diff options
861 files changed, 29351 insertions, 9710 deletions
diff --git a/.coveragerc b/.coveragerc deleted file mode 100644 index 9bc4ce49a..000000000 --- a/.coveragerc +++ /dev/null @@ -1,7 +0,0 @@ -[run] -branch = True -parallel = True -concurrency = multiprocessing,thread -include = Cython/* -source = Cython -omit = Test* diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 000000000..091606923 --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1,12 @@ +# These are supported funding model platforms + +github: scoder # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2] +patreon: # Replace with a single Patreon username +open_collective: # Replace with a single Open Collective username +ko_fi: # Replace with a single Ko-fi username +tidelift: pypi/Cython # Replace with a single Tidelift platform-name/package-name e.g., npm/babel +community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry +liberapay: # Replace with a single Liberapay username +issuehunt: # Replace with a single IssueHunt username +otechie: # Replace with a single Otechie username +custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2'] diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 000000000..04294a70b --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,34 @@ +--- +name: Bug report +about: Create a report to help us improve +title: "[BUG] " +labels: '' +assignees: '' + +--- + +<!-- +**PLEASE READ THIS FIRST:** +- Do not use the bug and feature tracker for support requests. Use the `cython-users` mailing list instead. +- Did you search for similar issues already? Please do, it helps to save us precious time that we otherwise could not invest into development. +- Did you try the latest master branch or pre-release? It might already have what you want to report. Also see the [Changelog](https://github.com/cython/cython/blob/master/CHANGES.rst) regarding recent changes. +--> + +**Describe the bug** +A clear and concise description of what the bug is. + +**To Reproduce** +Code to reproduce the behaviour: +```cython +``` + +**Expected behavior** +A clear and concise description of what you expected to happen. + +**Environment (please complete the following information):** + - OS: [e.g. Linux, Windows, macOS] + - Python version [e.g. 3.8.4] + - Cython version [e.g. 0.29.18] + +**Additional context** +Add any other context about the problem here. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 000000000..7b5be96a8 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,36 @@ +--- +name: Feature request +about: Suggest an idea for this project +title: "[ENH] " +labels: '' +assignees: '' + +--- + +<!-- +**Note:** +- Do not use the bug and feature tracker for support requests. Use the `cython-users` mailing list instead. +- Did you search for similar issues already? Please do, it helps to save us precious time that we otherwise could not invest into development. +- Did you try the latest master branch or pre-release? It might already have what you want to report. Also see the [Changelog](https://github.com/cython/cython/blob/master/CHANGES.rst) regarding recent changes. +--> + +**Is your feature request related to a problem? Please describe.** +A clear and concise description of what the problem is. Ex. In my code, I would like to do [...] +```cython +# add use case related code here +``` + +**Describe the solution you'd like** +A clear and concise description of what you want to happen, including code examples if applicable. +```cython +# add a proposed code/syntax example here +``` + +**Describe alternatives you've considered** +A clear and concise description of any alternative solutions or features you've considered. +```cython +# add alternative code/syntax proposals here +``` + +**Additional context** +Add any other context about the feature request here. diff --git a/.gitignore b/.gitignore index f59252e3f..9da27c0f1 100644 --- a/.gitignore +++ b/.gitignore @@ -6,27 +6,39 @@ __pycache__ *.egg *.egg-info +.*cache*/ +*venv*/ Cython/Compiler/*.c Cython/Plex/*.c Cython/Runtime/refnanny.c Cython/Tempita/*.c Cython/*.c +Cython/*.html +Cython/*/*.html Tools/*.elc +Demos/*.html +Demos/*/*.html /TEST_TMP/ /build/ +/cython_build/ /wheelhouse*/ !tests/build/ /dist/ .gitrev .coverage +*.patch +*.diff *.orig +*.prof *.rej +*.log *.dep *.swp *~ +callgrind.out.* .ipynb_checkpoints docs/build @@ -41,5 +53,8 @@ MANIFEST /.idea /*.iml +# Komodo EDIT/IDE project files +/*.komodoproject + # Visual Studio Code files .vscode diff --git a/.travis.yml b/.travis.yml index 4bb37d837..bb3aa5a94 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,6 +16,16 @@ cache: directories: - $HOME/.ccache +python: + - 3.9 + - 3.8 + - 2.7 + - 3.10-dev + - 3.7 + - 3.6 + - 3.5 + - 3.4 + env: global: - USE_CCACHE=1 @@ -24,65 +34,16 @@ env: - CCACHE_MAXSIZE=250M - PATH="/usr/lib/ccache:$HOME/miniconda/bin:$PATH" - BACKEND=c,cpp + matrix: + - BACKEND=c + - BACKEND=cpp matrix: include: - - python: 2.7 - env: BACKEND=c - - python: 2.7 - env: BACKEND=cpp - - python: 3.7 - dist: xenial # Required for Python 3.7 - sudo: required # travis-ci/travis-ci#9069 - env: BACKEND=c - - python: 3.7 - dist: xenial # Required for Python 3.7 - sudo: required # travis-ci/travis-ci#9069 - env: BACKEND=cpp - - python: 2.6 - env: BACKEND=c - - python: 2.6 - env: BACKEND=cpp -# Disabled: coverage analysis takes excessively long, several times longer than without. -# - python: 3.7 -# dist: xenial # Required for Python 3.7 -# sudo: required # travis-ci/travis-ci#9069 -# env: COVERAGE=1 - - python: 3.7 - dist: xenial # Required for Python 3.7 - sudo: required # travis-ci/travis-ci#9069 - env: TEST_CODE_STYLE=1 - - python: 3.4 - env: BACKEND=c - - python: 3.4 - env: BACKEND=cpp - - python: 3.5 - env: BACKEND=c - - python: 3.5 - env: BACKEND=cpp - - python: 3.6 - env: BACKEND=c - - python: 3.6 - env: BACKEND=cpp - - python: 3.8 - dist: xenial # Required for Python 3.7 - sudo: required # travis-ci/travis-ci#9069 - env: BACKEND=c - - python: 3.8 - dist: xenial # Required for Python 3.7 - sudo: required # travis-ci/travis-ci#9069 - env: BACKEND=cpp - - python: 3.9 - dist: xenial # Required for Python 3.7 - sudo: required # travis-ci/travis-ci#9069 - env: BACKEND=c - - python: 3.9 - dist: xenial # Required for Python 3.7 - sudo: required # travis-ci/travis-ci#9069 - env: BACKEND=cpp + # slowest first - os: osx - osx_image: xcode6.4 - env: PY=2 + osx_image: xcode10.3 + env: PY=2 MACOSX_DEPLOYMENT_TARGET=10.9 python: 2.7 language: c compiler: clang @@ -94,23 +55,53 @@ matrix: language: c compiler: clang cache: false - - python: pypy +# Disabled: coverage analysis takes excessively long, several times longer than without. +# - python: 3.7 +# env: COVERAGE=1 + - python: 3.7 + env: TEST_CODE_STYLE=1 + - python: 3.8 + env: LIMITED_API=--limited-api EXCLUDE=--no-file + - python: 3.7 + env: LIMITED_API=--limited-api EXCLUDE=--no-file + - python: 3.6 + env: LIMITED_API=--limited-api EXCLUDE=--no-file + - python: 3.8 + arch: arm64 env: BACKEND=c - - python: pypy3 + - python: 3.8 + arch: arm64 + env: BACKEND=cpp + - python: 3.8 + arch: ppc64le env: BACKEND=c + - python: 3.8 + arch: ppc64le + env: BACKEND=cpp +# s390x currently shows test failures. +# - python: 3.8 +# arch: s390x +# env: BACKEND=c +# - python: 3.8 +# arch: s390x +# env: BACKEND=cpp - env: STACKLESS=true BACKEND=c PY=2 python: 2.7 - env: STACKLESS=true BACKEND=c PY=3 python: 3.6 - allow_failures: - python: pypy + env: BACKEND=c - python: pypy3 - -branches: - only: - - master - - release - - 0.29.x + env: BACKEND=c +# a secondary pypy tests which is allowed to fail and which specifically +# tests known bugs + - python: pypy + env: BACKEND=c EXCLUDE="--listfile=tests/pypy_bugs.txt --listfile=tests/pypy2_bugs.txt bugs" + - python: pypy3 + env: BACKEND=c EXCLUDE="--listfile=tests/pypy_bugs.txt bugs" + allow_failures: + - env: BACKEND=c EXCLUDE="--listfile=tests/pypy_bugs.txt bugs" + - env: BACKEND=c EXCLUDE="--listfile=tests/pypy_bugs.txt --listfile=tests/pypy2_bugs.txt bugs" before_install: - | @@ -138,8 +129,8 @@ before_install: fi fi - - if [ -n "$CC" ]; then which $CC; $CC --version; fi - - if [ -n "$CXX" ]; then which $CXX; $CXX --version; fi + - if [ -n "$CC" ]; then which ${CC%% *}; $CC --version; fi + - if [ -n "$CXX" ]; then which ${CXX%% *}; $CXX --version; fi - if [ "$STACKLESS" == "true" ]; then conda config --add channels stackless; @@ -149,7 +140,7 @@ before_install: install: - python -c 'import sys; print("Python %s" % (sys.version,))' - if [ -z "${TRAVIS_PYTHON_VERSION##2.7}" ]; then [ "$TRAVIS_OS_NAME" == "osx" -a "$PY" == "3" ] || pip install -r test-requirements-27.txt ; fi - - if [ -n "${TRAVIS_PYTHON_VERSION##*-dev}" -a -n "${TRAVIS_PYTHON_VERSION##2.*}" ]; then pip install -r test-requirements.txt $( [ -z "${TRAVIS_PYTHON_VERSION##pypy*}" -o -z "${TRAVIS_PYTHON_VERSION##3.[47890]*}" ] || echo " -r test-requirements-cpython.txt" ) ; fi + - if [ -n "${TRAVIS_PYTHON_VERSION##*-dev}" -a -n "${TRAVIS_PYTHON_VERSION##2.7}" ]; then pip install -r test-requirements.txt $( [ -z "${TRAVIS_PYTHON_VERSION##pypy*}" -o -z "${TRAVIS_PYTHON_VERSION##3.[47891]*}" ] || echo " -r test-requirements-cpython.txt" ) ; fi # - CFLAGS="-O2 -ggdb -Wall -Wextra $(python -c 'import sys; print("-fno-strict-aliasing" if sys.version_info[0] == 2 else "")')" python setup.py build before_script: ccache -s || true @@ -161,9 +152,11 @@ script: else STYLE_ARGS=--no-code-style; if $PYTHON_DBG -V >&2; then CFLAGS="-O0 -ggdb" $PYTHON_DBG runtests.py -vv --no-code-style Debugger --backends=$BACKEND; fi; - if [ -z "${BACKEND##*cpp*}" -a -n "${TRAVIS_PYTHON_VERSION##*-dev}" ]; then pip install pythran; fi; - if [ "$BACKEND" != "cpp" -a -n "${TRAVIS_PYTHON_VERSION##2*}" -a -n "${TRAVIS_PYTHON_VERSION##*-dev}" -a -n "${TRAVIS_PYTHON_VERSION##*3.4}" ]; then pip install mypy; fi; + if [ -z "${BACKEND##*cpp*}" -a -n "${TRAVIS_PYTHON_VERSION##*-dev}" ]; then pip install pythran==0.9.7; fi; + if [ "$BACKEND" != "cpp" -a -n "${TRAVIS_PYTHON_VERSION##2*}" -a -n "${TRAVIS_PYTHON_VERSION##pypy*}" -a -n "${TRAVIS_PYTHON_VERSION##*-dev}" -a -n "${TRAVIS_PYTHON_VERSION##*3.4}" ]; then pip install mypy; fi; fi - - if [ "$COVERAGE" != "1" ]; then CFLAGS="-O2 -ggdb -Wall -Wextra $(python -c 'import sys; print("-fno-strict-aliasing" if sys.version_info[0] == 2 else "")')" python setup.py build_ext -i; fi - - CFLAGS="-O0 -ggdb -Wall -Wextra" python runtests.py -vv $STYLE_ARGS -x Debugger --backends=$BACKEND $(if [ "$COVERAGE" == "1" ]; then echo " --coverage"; fi) $(if [ -z "$TEST_CODE_STYLE" ]; then echo " -j7 "; fi) +# Need to clear the ccache? Try something like this: +# - if [ -n "${BACKEND##*cpp*}" -a -z "${TRAVIS_PYTHON_VERSION##*3.4}" ]; then ccache -C || true; fi + - if [ "$COVERAGE" != "1" ]; then CFLAGS="-O2 -ggdb -Wall -Wextra $(python -c 'import sys; print("-fno-strict-aliasing" if sys.version_info[0] == 2 else "")')" python setup.py build_ext -i $(python -c 'import sys; print("-j5" if sys.version_info >= (3,5) else "")'); fi + - CFLAGS="-O0 -ggdb -Wall -Wextra" python runtests.py -vv $STYLE_ARGS -x Debugger --backends=$BACKEND $LIMITED_API $EXCLUDE $(if [ "$COVERAGE" == "1" ]; then echo " --coverage"; fi) $(if [ -z "$TEST_CODE_STYLE" ]; then echo " -j7 "; fi) - ccache -s || true diff --git a/CHANGES.rst b/CHANGES.rst index 1437e94b1..41becd976 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -2,6 +2,582 @@ Cython Changelog ================ +3.0.0 alpha 7 (2020-0?-??) +========================== + +Features added +-------------- + +* ``__class_getitem__`` (`PEP-560`_) is supported for cdef classes. + Patch by Kmol Yuan. (Github issue :issue:`3764`) + +* ``__mro_entries__`` (`PEP-560`_) is supported for Python classes. + Patch by David Woods. (Github issue :issue:`3537`) + +* ``cython.array`` supports simple, non-strided views. + (Github issue :issue:`3775`) + +* The destructor is now called for fields in C++ structs. + Patch by David Woods. (Github issue :issue:`3226`) + +* ``asyncio.iscoroutinefunction()`` now recognises coroutine functions + also when compiled by Cython. + Patch by Pedro Marques da Luz. (Github issue :issue:`2273`) + +* ``float(…)`` is optimised for string arguments (str/bytes/bytearray). + +* Docstrings of ``cpdef`` enums are now copied to the enum class. + Patch by matham. (Github issue :issue:`3805`) + +* The type ``cython.Py_hash_t`` is available in Python mode. + +* The ``cpython.fileobject`` C-API declarations were added. + Patch by Zackery Spytz. (Github issue :issue:`3906`) + +* A new module ``cpython.time`` was added with some low-level alternatives to + Python's ``time`` module. + Patch by Brock Mendel. (Github issue :issue:`3767`) + +* The value ``PyBUF_MAX_NDIM`` was added to the ``cpython.buffer`` module. + Patch by John Kirkham. (Github issue :issue:`3811`) + +* "Declaration after use" is now an error for variables. + Patch by David Woods. (Github issue :issue:`3976`) + +Bugs fixed +---------- + +* Inline functions and other code in ``.pxd`` files could accidentally + inherit the compiler directives of the ``.pyx`` file that imported them. + Patch by David Woods. (Github issue :issue:`1071`) + +* Some issues were resolved that could lead to duplicated C names. + Patch by David Woods. (Github issue :issue:`3716`, :issue:`3741`, :issue:`3734`) + +* ``ndarray.shape`` failed to compile with Pythran and recent NumPy. + Patch by Serge Guelton. (Github issue :issue:`3762`) + +* Casting to ctuples is now allowed. + Patch by David Woods. (Github issue :issue:`3808`) + +* Nested C++ types were not usable through ctypedefs. + Patch by Vadim Pushtaev. (Github issue :issue:`4039`) + +* Cython compiled functions always provided a ``__self__`` attribute, regardless + of being used as a method or not. + Patch by David Woods. (Github issue :issue:`4036`) + +* A reference leak on import failures was resolved. + Patch by Max Bachmann. (Github issue :issue:`4056`) + +* A C compiler warning about unused code was resolved. + (Github issue :issue:`3763`) + +* A C compiler warning about enum value casting was resolved in GCC. + (Github issue :issue:`2749`) + +* Some C compiler warninge were resolved. + Patches by Max Bachmann. (Github issue :issue:`4053`, :issue:`4059`, :issue:`4054`) + +* A compile failure for C++ enums in Py3.4 / MSVC was resolved. + Patch by Ashwin Srinath. (Github issue :issue:`3782`) + +* An unsupported C-API call in PyPy was fixed. + Patch by Max Bachmann. (Github issue :issue:`4055`) + +* The Cython ``CodeWriter`` mishandled no-argument ``return`` statements. + Patch by Tao He. (Github issue :issue:`3795`) + +* ``complex`` wasn't supported in PEP-484 type annotations. + Patch by David Woods. (Github issue :issue:`3949`) + + +3.0.0 alpha 6 (2020-07-31) +========================== + +Features added +-------------- + +* Special methods for binary operators now follow Python semantics. + Rather than e.g. a single ``__add__`` method for cdef classes, where + "self" can be either the first or second argument, one can now define + both ``__add__`` and ``__radd__`` as for standard Python classes. + This behavior can be disabled with the ``c_api_binop_methods`` directive + to return to the previous semantics in Cython code (available from Cython + 0.29.20), or the reversed method (``__radd__``) can be implemented in + addition to an existing two-sided operator method (``__add__``) to get a + backwards compatible implementation. + (Github issue :issue:`2056`) + +* No/single argument functions now accept keyword arguments by default in order + to comply with Python semantics. The marginally faster calling conventions + ``METH_NOARGS`` and ``METH_O`` that reject keyword arguments are still available + with the directive ``@cython.always_allow_keywords(False)``. + (Github issue :issue:`3090`) + +* For-in-loop iteration over ``bytearray`` and memory views is optimised. + Patch by David Woods. (Github issue :issue:`2227`) + +* Type inference now works for memory views and slices. + Patch by David Woods. (Github issue :issue:`2227`) + +* The ``@returns()`` decorator propagates exceptions by default for suitable C + return types when no ``@exceptval()`` is defined. + (Github issues :issue:`3625`, :issue:`3664`) + +* A low-level inline function ``total_seconds(timedelta)`` was added to + ``cpython.datetime`` to bypass the Python method call. Note that this function + is not guaranteed to give exactly the same results for very large time intervals. + Patch by Brock Mendel. (Github issue :issue:`3616`) + +* Type inference now understands that ``a, *b = x`` assigns a list to ``b``. + +* Limited API support was improved. + Patches by Matthias Braun. (Github issues :issue:`3693`, :issue:`3707`) + +* The Cython ``CodeWriter`` can now handle more syntax constructs. + Patch by Tao He. (Github issue :issue:`3514`) + +Bugs fixed +---------- + +* The construct ``for x in cpp_function_call()`` failed to compile. + Patch by David Woods. (Github issue :issue:`3663`) + +* C++ references failed to compile when used as Python object indexes. + Patch by David Woods. (Github issue :issue:`3754`) + +* The C++ ``typeid()`` function was allowed in C mode. + Patch by Celelibi. (Github issue :issue:`3637`) + +* ``repr()`` was assumed to return ``str`` instead of ``unicode`` with ``language_level=3``. + (Github issue :issue:`3736`) + +* Includes all bug-fixes from the 0.29.21 release. + +Other changes +------------- + +* The ``numpy`` declarations were updated. + Patch by Brock Mendel. (Github issue #3630) + +* The names of Cython's internal types (functions, generator, coroutine, etc.) + are now qualified with the module name of the internal Cython module that is + used for sharing them across Cython implemented modules, for example + ``_cython_3_0a5.coroutine``. This was done to avoid making them look like + homeless builtins, to help with debugging, and in order to avoid a CPython + warning according to https://bugs.python.org/issue20204 + +3.0.0 alpha 5 (2020-05-19) +========================== + +Features added +-------------- + +* ``.pxd`` files can now be :ref:`versioned <versioning>` by adding an + extension like "``.cython-30.pxd``" to prevent older Cython versions (than + 3.0 in this case) from picking them up. (Github issue :issue:`3577`) + +* Several macros/functions declared in the NumPy API are now usable without + holding the GIL. + +* `libc.math` was extended to include all C99 function declarations. + Patch by Dean Scarff. (Github issue :issue:`3570`) + +Bugs fixed +---------- + +* Several issues with arithmetic overflow handling were resolved, including + undefined behaviour in C. + Patch by Sam Sneddon. (Github issue :issue:`3588`) + +* The improved GIL handling in ``nogil`` functions introduced in 3.0a3 + could fail to acquire the GIL in some cases on function exit. + (Github issue :issue:`3590` etc.) + +* A reference leak when processing keyword arguments in Py2 was resolved, + that appeared in 3.0a1. + (Github issue :issue:`3578`) + +* The outdated getbuffer/releasebuffer implementations in the NumPy + declarations were removed so that buffers declared as ``ndarray`` + now use the normal implementation in NumPy. + +* Includes all bug-fixes from the :ref:0.29.18 release. + + +3.0.0 alpha 4 (2020-05-05) +========================== + +Features added +-------------- + +* The ``print`` statement (not the ``print()`` function) is allowed in + ``nogil`` code without an explicit ``with gil`` section. + +* The ``assert`` statement is allowed in ``nogil`` sections. Here, the GIL is + only acquired if the ``AssertionError`` is really raised, which means that the + evaluation of the asserted condition only allows C expressions. + +* Cython generates C compiler branch hints for unlikely user defined if-clauses + in more cases, when they end up raising exceptions unconditionally. This now + includes exceptions being raised in ``nogil``/``with gil`` sections. + +* Some internal memoryview functions were tuned to reduce object overhead. + +Bugs fixed +---------- + +* Exception position reporting could run into race conditions on threaded code. + It now uses function-local variables again. + +* Error handling early in the module init code could lead to a crash. + +* Error handling in ``cython.array`` creation was improved to avoid calling + C-API functions with an error held. + +* Complex buffer item types of structs of arrays could fail to validate. + Patch by Leo and smutch. (Github issue :issue:`1407`) + +* When importing the old Cython ``build_ext`` integration with distutils, the + additional command line arguments leaked into the regular command. + Patch by Kamekameha. (Github issue :issue:`2209`) + +* The improved GIL handling in ``nogil`` functions introduced in 3.0a3 + could generate invalid C code. + (Github issue :issue:`3558`) + +* ``PyEval_InitThreads()`` is no longer used in Py3.7+ where it is a no-op. + +* Parallel builds of Cython itself (``setup.py build_ext -j N``) failed on Windows. + +Other changes +------------- + +* The C property feature has been rewritten and now requires C property methods + to be declared ``inline`` (:issue:`3571`). + + +3.0.0 alpha 3 (2020-04-27) +========================== + +Features added +-------------- + +* ``nogil`` functions now avoid acquiring the GIL on function exit if possible + even if they contain ``with gil`` blocks. + (Github issue :issue:`3554`) + +* Python private name mangling now falls back to unmangled names for non-Python + globals, since double-underscore names are not uncommon in C. Unmangled Python + names are also still found as a legacy fallback but produce a warning. + Patch by David Woods. (Github issue :issue:`3548`) + +Bugs fixed +---------- + +* Includes all bug-fixes from the :ref:0.29.17 release. + + +3.0.0 alpha 2 (2020-04-23) +========================== + +Features added +-------------- + +* ``std::move()`` is now used in C++ mode for internal temp variables to + make them work without copying values. + Patch by David Woods. (Github issues :issue:`3253`, :issue:`1612`) + +* ``__class_getitem__`` is supported for types on item access (`PEP-560`_). + Patch by msg555. (Github issue :issue:`2753`) + +* The simplified Py3.6 customisation of class creation is implemented (`PEP-487`_). + (Github issue #2781) + +* Conditional blocks in Python code that depend on ``cython.compiled`` are + eliminated at an earlier stage, which gives more freedom in writing + replacement Python code. + Patch by David Woods. (Github issue :issue:`3507`) + +* ``numpy.import_array()`` is automatically called if ``numpy`` has been cimported + and it has not been called in the module code. This is intended as a hidden + fail-safe so user code should continue to call ``numpy.import_array``. + Patch by David Woods. (Github issue :issue:`3524`) + +* The Cython AST code serialiser class ``CodeWriter`` in ``Cython.CodeWriter`` + supports more syntax nodes. + +* The fastcall/vectorcall protocols are used for several internal Python calls. + (Github issue :issue:`3540`) + +Bugs fixed +---------- + +* With ``language_level=3/3str``, Python classes without explicit base class + are now new-style (type) classes also in Py2. Previously, they were created + as old-style (non-type) classes. + (Github issue :issue:`3530`) + +* C++ ``typeid()`` failed for fused types. + Patch by David Woods. (Github issue :issue:`3203`) + +* ``__arg`` argument names in methods were not mangled with the class name. + Patch by David Woods. (Github issue :issue:`1382`) + +* Creating an empty unicode slice with large bounds could crash. + Patch by Sam Sneddon. (Github issue :issue:`3531`) + +* Decoding an empty bytes/char* slice with large bounds could crash. + Patch by Sam Sneddon. (Github issue :issue:`3534`) + +* Temporary buffer indexing variables were not released and could show up in + C compiler warnings, e.g. in generators. + Patch by David Woods. (Github issues :issue:`3430`, :issue:`3522`) + +* Several C compiler warnings were fixed. + + +3.0.0 alpha 1 (2020-04-12) +========================== + +Features added +-------------- + +* Cython functions now use the `PEP-590`_ vectorcall protocol in Py3.7+. + Patch by Jeroen Demeyer. (Github issue :issue:`2263`) + +* Unicode identifiers are supported in Cython code (`PEP-3131`_). + Patch by David Woods. (Github issue :issue:`2601`) + +* Unicode module names and imports are supported. + Patch by David Woods. (Github issue :issue:`3119`) + +* Annotations are no longer parsed, keeping them as strings following `PEP-563`_. + Patch by David Woods. (Github issue :issue:`3285`) + +* Preliminary support for the CPython's ``Py_LIMITED_API`` (stable ABI) is + available by setting the ``CYTHON_LIMITED_API`` C macro. Note that the + support is currently in an early stage and many features do not yet work. + You currently still have to define ``Py_LIMITED_API`` externally in order + to restrict the API usage. This will change when the feature stabilises. + Patches by Eddie Elizondo and David Woods. (Github issues :issue:`3223`, + :issue:`3311`, :issue:`3501`) + +* The dispatch to fused functions is now linear in the number of arguments, + which makes it much faster, often 2x or more, and several times faster for + larger fused types with many specialisations. + Patch by will-ca. (Github issue :issue:`1385`) + +* ``with gil/nogil`` statements can be conditional based on compile-time + constants, e.g. fused type checks. + Patch by Noam Hershtig. (Github issue :issue:`2579`) + +* ``const`` can be used together with fused types. + Patch by Thomas Vincent. (Github issue :issue:`1772`) + +* Reimports of already imported modules are substantially faster. + (Github issue :issue:`2854`) + +* Positional-only arguments are supported in Python functions (`PEP-570`_). + Patch by Josh Tobin. (Github issue :issue:`2915`) + +* The ``volatile`` C modifier is supported in Cython code. + Patch by Jeroen Demeyer. (Github issue #1667) + +* ``@cython.trashcan(True)`` can be used on an extension type to enable the + CPython :ref:`trashcan`. This allows deallocating deeply recursive objects + without overflowing the stack. Patch by Jeroen Demeyer. (Github issue :issue:`2842`) + +* Inlined properties can be defined for external extension types. + Patch by Matti Picus. (Github issue :issue:`2640`, redone later in :issue:`3571`) + +* The ``str()`` builtin now calls ``PyObject_Str()`` instead of going + through a Python call. + Patch by William Ayd. (Github issue :issue:`3279`) + +* String concatenation can now happen in place if possible, by extending the + existing string rather than always creating a new one. + Patch by David Woods. (Github issue :issue:`3453`) + +* Multiplication of Python numbers with small constant integers is faster. + (Github issue :issue:`2808`) + +* Some list copying is avoided internally when a new list needs to be created + but we already have a fresh one. + (Github issue :issue:`3494`) + +* Extension types that do not need their own ``tp_new`` implementation (because + they have no object attributes etc.) directly inherit the implementation of + their parent type if possible. + (Github issue :issue:`1555`) + +* The attributes ``gen.gi_frame`` and ``coro.cr_frame`` of Cython compiled + generators and coroutines now return an actual frame object for introspection. + (Github issue :issue:`2306`) + +* Several declarations in ``cpython.*``, ``libc.*`` and ``libcpp.*`` were added. + Patches by Jeroen Demeyer, Matthew Edwards, Chris Gyurgyik, Jerome Kieffer + and Zackery Spytz. + (Github issues :issue:`3468`, :issue:`3332`, :issue:`3202`, :issue:`3188`, + :issue:`3179`, :issue:`2891`, :issue:`2826`, :issue:`2713`) + +* Deprecated NumPy API usages were removed from ``numpy.pxd``. + Patch by Matti Picus. (Github issue :issue:`3365`) + +* ``cython.inline()`` now sets the ``NPY_NO_DEPRECATED_API=NPY_1_7_API_VERSION`` + C macro automatically when ``numpy`` is imported in the code, to avoid C compiler + warnings about deprecated NumPy C-API usage. + +* The builtin ``abs()`` function can now be used on C numbers in nogil code. + Patch by Elliott Sales de Andrade. (Github issue :issue:`2748`) + +* `PEP-479`_ (``generator_stop``) is now enabled by default with language level 3. + (Github issue :issue:`2580`) + +* The ``cython.view.array`` type supports inheritance. + Patch by David Woods. (Github issue :issue:`3413`) + +* Code annotation accepts a new debugging argument ``--annotate-fullc`` that + will include the complete syntax highlighted C file in the HTML output. + (Github issue :issue:`2855`) + +* ``--no-capture`` added to ``runtests.py`` to prevent stdout/stderr capturing + during srctree tests. + Patch by Matti Picus. (Github issue :issue:`2701`) + +* ``--no-docstrings`` option added to ``cythonize`` script. + Original patch by mo-han. (Github issue :issue:`2889`) + +* ``cygdb`` gives better error messages when it fails to initialise the + Python runtime support in gdb. + Patch by Volker Weissmann. (Github issue :issue:`3489`) + +* The Pythran ``shape`` attribute is supported. + Patch by Serge Guelton. (Github issue :issue:`3307`) + +Bugs fixed +---------- + +* The unicode methods ``.upper()``, ``.lower()`` and ``.title()`` were + incorrectly optimised for single character input values and only returned + the first character if multiple characters should have been returned. + They now use the original Python methods again. + +* Fused argument types were not correctly handled in type annotations and + ``cython.locals()``. + Patch by David Woods. (Github issues :issue:`3391`, :issue:`3142`) + +* Diverging from the usual behaviour, ``len(memoryview)``, ``len(char*)`` + and ``len(Py_UNICODE*)`` returned an unsigned ``size_t`` value. They now + return a signed ``Py_ssize_t``, like other usages of ``len()``. + +* Nested dict literals in function call kwargs could incorrectly raise an + error about duplicate keyword arguments, which are allowed when passing + them from dict literals. + (Github issue :issue:`2963`) + +* Item access (subscripting) with integer indices/keys always tried the + Sequence protocol before the Mapping protocol, which diverged from Python + semantics. It now passes through the Mapping protocol first when supported. + (Github issue :issue:`1807`) + +* Name lookups in class bodies no longer go through an attribute lookup. + Patch by Jeroen Demeyer. (Github issue :issue:`3100`) + +* Broadcast assignments to a multi-dimensional memory view slice could end + up in the wrong places when the underlying memory view is known to be + contiguous but the slice is not. + (Github issue :issue:`2941`) + +* Pickling unbound methods of Python classes failed. + Patch by Pierre Glaser. (Github issue :issue:`2972`) + +* The ``Py_hash_t`` type failed to accept arbitrary "index" values. + (Github issue :issue:`2752`) + +* The first function line number of functions with decorators pointed to the + signature line and not the first decorator line, as in Python. + Patch by Felix Kohlgrüber. (Github issue :issue:`2536`) + +* Constant integer expressions that used a negative exponent were evaluated + as integer 0 instead of the expected float value. + Patch by Kryštof Pilnáček. (Github issue :issue:`2133`) + +* The ``cython.declare()`` and ``cython.cast()`` functions could fail in pure mode. + Patch by Dmitry Shesterkin. (Github issue :issue:`3244`) + +* ``__doc__`` was not available inside of the class body during class creation. + (Github issue :issue:`1635`) + +* Setting ``language_level=2`` in a file did not work if ``language_level=3`` + was enabled globally before. + Patch by Jeroen Demeyer. (Github issue :issue:`2791`) + +* ``__init__.pyx`` files were not always considered as package indicators. + (Github issue :issue:`2665`) + +* Compiling package ``__init__`` files could fail under Windows due to an + undefined export symbol. (Github issue :issue:`2968`) + +* A C compiler cast warning was resolved. + Patch by Michael Buesch. (Github issue :issue:`2775`) + +* Binding staticmethods of Cython functions were not behaving like Python methods. + Patch by Jeroen Demeyer. (Github issue :issue:`3106`, :issue:`3102`) + +* Memoryviews failed to compile when the ``cache_builtins`` feature was disabled. + Patch by David Woods. (Github issue :issue:`3406`) + +Other changes +------------- + +* The default language level was changed to ``3str``, i.e. Python 3 semantics, + but with ``str`` literals (also in Python 2.7). This is a backwards incompatible + change from the previous default of Python 2 semantics. The previous behaviour + is available through the directive ``language_level=2``. + (Github issue :issue:`2565`) + +* Cython no longer generates ``__qualname__`` attributes for classes in Python + 2.x since they are problematic there and not correctly maintained for subclasses. + Patch by Jeroen Demeyer. (Github issue :issue:`2772`) + +* Source file fingerprinting now uses SHA-1 instead of MD5 since the latter + tends to be slower and less widely supported these days. + (Github issue :issue:`2790`) + +* The long deprecated include files ``python_*``, ``stdio``, ``stdlib`` and + ``stl`` in ``Cython/Includes/Deprecated/`` were removed. Use the ``libc.*`` + and ``cpython.*`` pxd modules instead. + Patch by Jeroen Demeyer. (Github issue :issue:`2904`) + +* The search order for include files was changed. Previously it was + ``include_directories``, ``Cython/Includes``, ``sys.path``. Now it is + ``include_directories``, ``sys.path``, ``Cython/Includes``. This was done to + allow third-party ``*.pxd`` files to override the ones in Cython. + Patch by Matti Picus. (Github issue :issue:`2905`) + +* The command line parser was rewritten and modernised using ``argparse``. + Patch by Egor Dranischnikow. (Github issue :issue:`2952`, :issue:`3001`) + +* Dotted filenames for qualified module names (``pkg.mod.pyx``) are deprecated. + Use the normal Python package directory layout instead. + (Github issue :issue:`2686`) + +* Binary Linux wheels now follow the manylinux2010 standard. + Patch by Alexey Stepanov. (Github issue :issue:`3355`) + +* Support for Python 2.6 was removed. + +.. _`PEP-560`: https://www.python.org/dev/peps/pep-0560 +.. _`PEP-570`: https://www.python.org/dev/peps/pep-0570 +.. _`PEP-487`: https://www.python.org/dev/peps/pep-0487 +.. _`PEP-590`: https://www.python.org/dev/peps/pep-0590 +.. _`PEP-3131`: https://www.python.org/dev/peps/pep-3131 +.. _`PEP-563`: https://www.python.org/dev/peps/pep-0563 +.. _`PEP-479`: https://www.python.org/dev/peps/pep-0479 + + 0.29.23 (2021-04-14) ==================== @@ -400,7 +976,6 @@ Bugs fixed * The declaration of ``PyGILState_STATE`` in ``cpython.pystate`` was unusable. Patch by Kirill Smelkov. (Github issue #2997) - Other changes ------------- @@ -1434,7 +2009,7 @@ Features added Patch by Syrtis Major (Github issue #1625). * ``abs()`` is optimised for C complex numbers. - Patch by da-woods (Github issue #1648). + Patch by David Woods (Github issue #1648). * The display of C lines in Cython tracebacks can now be enabled at runtime via ``import cython_runtime; cython_runtime.cline_in_traceback=True``. @@ -1445,7 +2020,7 @@ Features added * "cdef extern" include files are now also searched relative to the current file. Patch by Jeroen Demeyer (Github issue #1654). -* Optional optimization for re-aquiring the GIL, controlled by the +* Optional optimization for re-acquiring the GIL, controlled by the `fast_gil` directive. Bugs fixed @@ -1563,7 +2138,7 @@ Features added Patch by Claudio Freire. * Buffer variables are no longer excluded from ``locals()``. - Patch by da-woods. + Patch by David Woods. * Building f-strings is faster, especially when formatting C integers. @@ -1711,7 +2286,7 @@ Bugs fixed * Compile errors and warnings in integer type conversion code. This fixes ticket 877. Patches by Christian Neukirchen, Nikolaus Rath, Ian Henriksen. -* Reference leak when "*args" argument was reassigned in closures. +* Reference leak when ``*args`` argument was reassigned in closures. * Truth-testing Unicode strings could waste time and memory in Py3.3+. @@ -2167,7 +2742,7 @@ Features added * Generators have new properties ``__name__`` and ``__qualname__`` that provide the plain/qualified name of the generator function - (following CPython 3.5). See http://bugs.python.org/issue21205 + (following CPython 3.5). See https://bugs.python.org/issue21205 * The ``inline`` function modifier is available as a decorator ``@cython.inline`` in pure mode. @@ -2213,7 +2788,7 @@ Optimizations evaluation and generally improves the code flow in the generated C code. * The Python expression "2 ** N" is optimised into bit shifting. - See http://bugs.python.org/issue21420 + See https://bugs.python.org/issue21420 * Cascaded assignments (a = b = ...) try to minimise the number of type coercions. @@ -3002,7 +3577,7 @@ Features added * Extension type inheritance from builtin types, such as "cdef class MyUnicode(unicode)", now works without further external type redeclarations (which are also strongly discouraged now and continue to issue a warning). -* GDB support. http://docs.cython.org/src/userguide/debugging.html +* GDB support. https://docs.cython.org/src/userguide/debugging.html * A new build system with support for inline distutils directives, correct dependency tracking, and parallel compilation. https://github.com/cython/cython/wiki/enhancements-distutils_preprocessing diff --git a/Cython/Build/BuildExecutable.py b/Cython/Build/BuildExecutable.py index 2db9e5d74..57dd10ac6 100644 --- a/Cython/Build/BuildExecutable.py +++ b/Cython/Build/BuildExecutable.py @@ -28,7 +28,7 @@ if PYLIB_DYN == PYLIB: # no shared library PYLIB_DYN = '' else: - PYLIB_DYN = os.path.splitext(PYLIB_DYN[3:])[0] # 'lib(XYZ).so' -> XYZ + PYLIB_DYN = os.path.splitext(PYLIB_DYN[3:])[0] # 'lib(XYZ).so' -> XYZ CC = get_config_var('CC', os.environ.get('CC', '')) CFLAGS = get_config_var('CFLAGS') + ' ' + os.environ.get('CFLAGS', '') @@ -65,12 +65,8 @@ def runcmd(cmd, shell=True): else: _debug(' '.join(cmd)) - try: - import subprocess - except ImportError: # Python 2.3 ... - returncode = os.system(cmd) - else: - returncode = subprocess.call(cmd, shell=shell) + import subprocess + returncode = subprocess.call(cmd, shell=shell) if returncode: sys.exit(returncode) @@ -105,7 +101,7 @@ def build(input_file, compiler_args=(), force=False): if not force and os.path.abspath(exe_file) == os.path.abspath(input_file): raise ValueError("Input and output file names are the same, refusing to overwrite") if (not force and os.path.exists(exe_file) and os.path.exists(input_file) - and os.path.getmtime(input_file) <= os.path.getmtime(exe_file)): + and os.path.getmtime(input_file) <= os.path.getmtime(exe_file)): _debug("File is up to date, not regenerating %s", exe_file) return exe_file cycompile(input_file, compiler_args) diff --git a/Cython/Build/Cythonize.py b/Cython/Build/Cythonize.py index 9de84d5c8..a3774b385 100644 --- a/Cython/Build/Cythonize.py +++ b/Cython/Build/Cythonize.py @@ -38,38 +38,9 @@ class _FakePool(object): pass -def parse_directives(option, name, value, parser): - dest = option.dest - old_directives = dict(getattr(parser.values, dest, - Options.get_directive_defaults())) - directives = Options.parse_directive_list( - value, relaxed_bool=True, current_settings=old_directives) - setattr(parser.values, dest, directives) - - -def parse_options(option, name, value, parser): - dest = option.dest - options = dict(getattr(parser.values, dest, {})) - for opt in value.split(','): - if '=' in opt: - n, v = opt.split('=', 1) - v = v.lower() not in ('false', 'f', '0', 'no') - else: - n, v = opt, True - options[n] = v - setattr(parser.values, dest, options) - - -def parse_compile_time_env(option, name, value, parser): - dest = option.dest - old_env = dict(getattr(parser.values, dest, {})) - new_env = Options.parse_compile_time_env(value, current_settings=old_env) - setattr(parser.values, dest, new_env) - - def find_package_base(path): base_dir, package_path = os.path.split(path) - while os.path.isfile(os.path.join(base_dir, '__init__.py')): + while is_package_dir(base_dir): base_dir, parent = os.path.split(base_dir) package_path = '%s/%s' % (parent, package_path) return base_dir, package_path @@ -148,54 +119,80 @@ def run_distutils(args): shutil.rmtree(temp_dir) -def parse_args(args): - from optparse import OptionParser - parser = OptionParser(usage='%prog [options] [sources and packages]+') +def create_args_parser(): + from argparse import ArgumentParser + from ..Compiler.CmdLine import ParseDirectivesAction, ParseOptionsAction, ParseCompileTimeEnvAction - parser.add_option('-X', '--directive', metavar='NAME=VALUE,...', - dest='directives', default={}, type="str", - action='callback', callback=parse_directives, + parser = ArgumentParser() + + parser.add_argument('-X', '--directive', metavar='NAME=VALUE,...', + dest='directives', default={}, type=str, + action=ParseDirectivesAction, help='set a compiler directive') - parser.add_option('-E', '--compile-time-env', metavar='NAME=VALUE,...', - dest='compile_time_env', default={}, type="str", - action='callback', callback=parse_compile_time_env, + parser.add_argument('-E', '--compile-time-env', metavar='NAME=VALUE,...', + dest='compile_time_env', default={}, type=str, + action=ParseCompileTimeEnvAction, help='set a compile time environment variable') - parser.add_option('-s', '--option', metavar='NAME=VALUE', - dest='options', default={}, type="str", - action='callback', callback=parse_options, + parser.add_argument('-s', '--option', metavar='NAME=VALUE', + dest='options', default={}, type=str, + action=ParseOptionsAction, help='set a cythonize option') - parser.add_option('-2', dest='language_level', action='store_const', const=2, default=None, + parser.add_argument('-2', dest='language_level', action='store_const', const=2, default=None, help='use Python 2 syntax mode by default') - parser.add_option('-3', dest='language_level', action='store_const', const=3, + parser.add_argument('-3', dest='language_level', action='store_const', const=3, help='use Python 3 syntax mode by default') - parser.add_option('--3str', dest='language_level', action='store_const', const='3str', + parser.add_argument('--3str', dest='language_level', action='store_const', const='3str', help='use Python 3 syntax mode by default') - parser.add_option('-a', '--annotate', dest='annotate', action='store_true', - help='generate annotated HTML page for source files') - - parser.add_option('-x', '--exclude', metavar='PATTERN', dest='excludes', + parser.add_argument('-a', '--annotate', action='store_const', const='default', dest='annotate', + help='Produce a colorized HTML version of the source.') + parser.add_argument('--annotate-fullc', action='store_const', const='fullc', dest='annotate', + help='Produce a colorized HTML version of the source ' + 'which includes entire generated C/C++-code.') + parser.add_argument('-x', '--exclude', metavar='PATTERN', dest='excludes', action='append', default=[], help='exclude certain file patterns from the compilation') - parser.add_option('-b', '--build', dest='build', action='store_true', + parser.add_argument('-b', '--build', dest='build', action='store_true', default=None, help='build extension modules using distutils') - parser.add_option('-i', '--inplace', dest='build_inplace', action='store_true', + parser.add_argument('-i', '--inplace', dest='build_inplace', action='store_true', default=None, help='build extension modules in place using distutils (implies -b)') - parser.add_option('-j', '--parallel', dest='parallel', metavar='N', + parser.add_argument('-j', '--parallel', dest='parallel', metavar='N', type=int, default=parallel_compiles, help=('run builds in N parallel jobs (default: %d)' % parallel_compiles or 1)) - parser.add_option('-f', '--force', dest='force', action='store_true', + parser.add_argument('-f', '--force', dest='force', action='store_true', default=None, help='force recompilation') - parser.add_option('-q', '--quiet', dest='quiet', action='store_true', + parser.add_argument('-q', '--quiet', dest='quiet', action='store_true', default=None, help='be less verbose during compilation') - parser.add_option('--lenient', dest='lenient', action='store_true', + parser.add_argument('--lenient', dest='lenient', action='store_true', default=None, help='increase Python compatibility by ignoring some compile time errors') - parser.add_option('-k', '--keep-going', dest='keep_going', action='store_true', + parser.add_argument('-k', '--keep-going', dest='keep_going', action='store_true', default=None, help='compile as much as possible, ignore compilation failures') + parser.add_argument('--no-docstrings', dest='no_docstrings', action='store_true', default=None, + help='strip docstrings') + parser.add_argument('sources', nargs='*') + return parser + + +def parse_args_raw(parser, args): + options, unknown = parser.parse_known_args(args) + sources = options.sources + # if positional arguments were interspersed + # some of them are in unknown + for option in unknown: + if option.startswith('-'): + parser.error("unknown option "+option) + else: + sources.append(option) + del options.sources + return (options, sources) + + +def parse_args(args): + parser = create_args_parser() + options, args = parse_args_raw(parser, args) - options, args = parser.parse_args(args) if not args: parser.error("no source files provided") if options.build_inplace: @@ -205,11 +202,6 @@ def parse_args(args): if options.language_level: assert options.language_level in (2, 3, '3str') options.options['language_level'] = options.language_level - return options, args - - -def main(args=None): - options, paths = parse_args(args) if options.lenient: # increase Python compatibility by ignoring compile time errors @@ -217,7 +209,16 @@ def main(args=None): Options.error_on_uninitialized = False if options.annotate: - Options.annotate = True + Options.annotate = options.annotate + + if options.no_docstrings: + Options.docstrings = False + + return options, args + + +def main(args=None): + options, paths = parse_args(args) for path in paths: cython_compile(path, options) diff --git a/Cython/Build/Dependencies.py b/Cython/Build/Dependencies.py index c66afbf93..46a763c37 100644 --- a/Cython/Build/Dependencies.py +++ b/Cython/Build/Dependencies.py @@ -45,7 +45,9 @@ except: from .. import Utils from ..Utils import (cached_function, cached_method, path_exists, safe_makedirs, copy_file_to_dir_if_newer, is_package_dir, replace_suffix) -from ..Compiler.Main import Context, CompilationOptions, default_options +from ..Compiler import Errors +from ..Compiler.Main import Context +from ..Compiler.Options import CompilationOptions, default_options join_path = cached_function(os.path.join) copy_once_if_newer = cached_function(copy_file_to_dir_if_newer) @@ -118,7 +120,7 @@ def nonempty(it, error_msg="expected non-empty iterator"): def file_hash(filename): path = os.path.normpath(filename) prefix = ('%d:%s' % (len(path), path)).encode("UTF-8") - m = hashlib.md5(prefix) + m = hashlib.sha1(prefix) with open(path, 'rb') as f: data = f.read(65000) while data: @@ -322,7 +324,8 @@ def strip_string_literals(code, prefix='__Pyx_L'): in_quote = False hash_mark = single_q = double_q = -1 code_len = len(code) - quote_type = quote_len = None + quote_type = None + quote_len = -1 while True: if hash_mark < q: @@ -538,7 +541,7 @@ class DependencyTree(object): all.add(include_path) all.update(self.included_files(include_path)) elif not self.quiet: - print("Unable to locate '%s' referenced from '%s'" % (filename, include)) + print(u"Unable to locate '%s' referenced from '%s'" % (filename, include)) return all @cached_method @@ -610,10 +613,10 @@ class DependencyTree(object): @cached_method def immediate_dependencies(self, filename): - all = set([filename]) - all.update(self.cimported_files(filename)) - all.update(self.included_files(filename)) - return all + all_deps = {filename} + all_deps.update(self.cimported_files(filename)) + all_deps.update(self.included_files(filename)) + return all_deps def all_dependencies(self, filename): return self.transitive_merge(filename, self.immediate_dependencies, set.union) @@ -637,7 +640,7 @@ class DependencyTree(object): incorporate everything that has an influence on the generated code. """ try: - m = hashlib.md5(__version__.encode('UTF-8')) + m = hashlib.sha1(__version__.encode('UTF-8')) m.update(file_hash(filename).encode('UTF-8')) for x in sorted(self.all_dependencies(filename)): if os.path.splitext(x)[1] not in ('.c', '.cpp', '.h'): @@ -756,7 +759,7 @@ def create_extension_list(patterns, exclude=None, ctx=None, aliases=None, quiet= return [], {} elif isinstance(patterns, basestring) or not isinstance(patterns, Iterable): patterns = [patterns] - explicit_modules = set([m.name for m in patterns if isinstance(m, Extension)]) + explicit_modules = {m.name for m in patterns if isinstance(m, Extension)} seen = set() deps = create_dependency_tree(ctx, quiet=quiet) to_exclude = set() @@ -782,6 +785,8 @@ def create_extension_list(patterns, exclude=None, ctx=None, aliases=None, quiet= create_extension = ctx.options.create_extension or default_create_extension for pattern in patterns: + if not isinstance(pattern, (Extension_distutils, Extension_setuptools)): + pattern = encode_filename_in_py2(pattern) if isinstance(pattern, str): filepattern = pattern template = Extension(pattern, []) # Fake Extension without sources @@ -794,9 +799,9 @@ def create_extension_list(patterns, exclude=None, ctx=None, aliases=None, quiet= if cython_sources: filepattern = cython_sources[0] if len(cython_sources) > 1: - print("Warning: Multiple cython sources found for extension '%s': %s\n" - "See http://cython.readthedocs.io/en/latest/src/userguide/sharing_declarations.html " - "for sharing declarations among Cython files." % (pattern.name, cython_sources)) + print(u"Warning: Multiple cython sources found for extension '%s': %s\n" + u"See https://cython.readthedocs.io/en/latest/src/userguide/sharing_declarations.html " + u"for sharing declarations among Cython files." % (pattern.name, cython_sources)) else: # ignore non-cython modules module_list.append(pattern) @@ -870,7 +875,7 @@ def create_extension_list(patterns, exclude=None, ctx=None, aliases=None, quiet= m.sources.remove(target_file) except ValueError: # never seen this in the wild, but probably better to warn about this unexpected case - print("Warning: Cython source file not found in sources list, adding %s" % file) + print(u"Warning: Cython source file not found in sources list, adding %s" % file) m.sources.insert(0, file) seen.add(name) return module_list, module_metadata @@ -878,7 +883,7 @@ def create_extension_list(patterns, exclude=None, ctx=None, aliases=None, quiet= # This is the user-exposed entry point. def cythonize(module_list, exclude=None, nthreads=0, aliases=None, quiet=False, force=False, language=None, - exclude_failures=False, **options): + exclude_failures=False, show_all_warnings=False, **options): """ Compile a set of source modules into C/C++ files and return a list of distutils Extension objects for them. @@ -928,6 +933,9 @@ def cythonize(module_list, exclude=None, nthreads=0, aliases=None, quiet=False, really makes sense for compiling ``.py`` files which can also be used without compilation. + :param show_all_warnings: By default, not all Cython warnings are printed. + Set to true to show all warnings. + :param annotate: If ``True``, will produce a HTML file for each of the ``.pyx`` or ``.py`` files compiled. The HTML file gives an indication of how much Python interaction there is in @@ -940,6 +948,11 @@ def cythonize(module_list, exclude=None, nthreads=0, aliases=None, quiet=False, See examples in :ref:`determining_where_to_add_types` or :ref:`primes`. + + :param annotate-fullc: If ``True`` will produce a colorized HTML version of + the source which includes entire generated C/C++-code. + + :param compiler_directives: Allow to set compiler directives in the ``setup.py`` like this: ``compiler_directives={'embedsignature': True}``. See :ref:`compiler-directives`. @@ -960,7 +973,7 @@ def cythonize(module_list, exclude=None, nthreads=0, aliases=None, quiet=False, c_options = CompilationOptions(**options) cpp_options = CompilationOptions(**options); cpp_options.cplus = True - ctx = c_options.create_context() + ctx = Context.from_options(c_options) options = c_options module_list, module_metadata = create_extension_list( module_list, @@ -970,6 +983,9 @@ def cythonize(module_list, exclude=None, nthreads=0, aliases=None, quiet=False, exclude_failures=exclude_failures, language=language, aliases=aliases) + + fix_windows_unicode_modules(module_list) + deps = create_dependency_tree(ctx, quiet=quiet) build_dir = getattr(options, 'build_dir', None) @@ -1017,7 +1033,7 @@ def cythonize(module_list, exclude=None, nthreads=0, aliases=None, quiet=False, # setup for out of place build directory if enabled if build_dir: if os.path.isabs(c_file): - warnings.warn("build_dir has no effect for absolute source paths") + warnings.warn("build_dir has no effect for absolute source paths") c_file = os.path.join(build_dir, c_file) dir = os.path.dirname(c_file) safe_makedirs_once(dir) @@ -1038,9 +1054,12 @@ def cythonize(module_list, exclude=None, nthreads=0, aliases=None, quiet=False, if force or c_timestamp < dep_timestamp: if not quiet and not force: if source == dep: - print("Compiling %s because it changed." % source) + print(u"Compiling %s because it changed." % Utils.decode_filename(source)) else: - print("Compiling %s because it depends on %s." % (source, dep)) + print(u"Compiling %s because it depends on %s." % ( + Utils.decode_filename(source), + Utils.decode_filename(dep), + )) if not force and options.cache: fingerprint = deps.transitive_fingerprint(source, m, options) else: @@ -1048,7 +1067,7 @@ def cythonize(module_list, exclude=None, nthreads=0, aliases=None, quiet=False, to_compile.append(( priority, source, c_file, fingerprint, quiet, options, not exclude_failures, module_metadata.get(m.name), - full_module_name)) + full_module_name, show_all_warnings)) new_sources.append(c_file) modules_by_cfile[c_file].append(m) else: @@ -1072,32 +1091,26 @@ def cythonize(module_list, exclude=None, nthreads=0, aliases=None, quiet=False, if N <= 1: nthreads = 0 if nthreads: - # Requires multiprocessing (or Python >= 2.6) + import multiprocessing + pool = multiprocessing.Pool( + nthreads, initializer=_init_multiprocessing_helper) + # This is a bit more involved than it should be, because KeyboardInterrupts + # break the multiprocessing workers when using a normal pool.map(). + # See, for example: + # https://noswap.com/blog/python-multiprocessing-keyboardinterrupt try: - import multiprocessing - pool = multiprocessing.Pool( - nthreads, initializer=_init_multiprocessing_helper) - except (ImportError, OSError): - print("multiprocessing required for parallel cythonization") - nthreads = 0 - else: - # This is a bit more involved than it should be, because KeyboardInterrupts - # break the multiprocessing workers when using a normal pool.map(). - # See, for example: - # http://noswap.com/blog/python-multiprocessing-keyboardinterrupt - try: - result = pool.map_async(cythonize_one_helper, to_compile, chunksize=1) - pool.close() - while not result.ready(): - try: - result.get(99999) # seconds - except multiprocessing.TimeoutError: - pass - except KeyboardInterrupt: - pool.terminate() - raise - pool.join() - if not nthreads: + result = pool.map_async(cythonize_one_helper, to_compile, chunksize=1) + pool.close() + while not result.ready(): + try: + result.get(99999) # seconds + except multiprocessing.TimeoutError: + pass + except KeyboardInterrupt: + pool.terminate() + raise + pool.join() + else: for args in to_compile: cythonize_one(*args) @@ -1117,7 +1130,7 @@ def cythonize(module_list, exclude=None, nthreads=0, aliases=None, quiet=False, if failed_modules: for module in failed_modules: module_list.remove(module) - print("Failed compilations: %s" % ', '.join(sorted([ + print(u"Failed compilations: %s" % ', '.join(sorted([ module.name for module in failed_modules]))) if options.cache: @@ -1128,6 +1141,41 @@ def cythonize(module_list, exclude=None, nthreads=0, aliases=None, quiet=False, return module_list +def fix_windows_unicode_modules(module_list): + # Hack around a distutils 3.[5678] bug on Windows for unicode module names. + # https://bugs.python.org/issue39432 + if sys.platform != "win32": + return + if sys.version_info < (3, 5) or sys.version_info >= (3, 8, 2): + return + + def make_filtered_list(ignored_symbol, old_entries): + class FilteredExportSymbols(list): + # export_symbols for unicode filename cause link errors on Windows + # Cython doesn't need them (it already defines PyInit with the correct linkage) + # so use this class as a temporary fix to stop them from being generated + def __contains__(self, val): + # so distutils doesn't "helpfully" add PyInit_<name> + return val == ignored_symbol or list.__contains__(self, val) + + filtered_list = FilteredExportSymbols(old_entries) + if old_entries: + filtered_list.extend(name for name in old_entries if name != ignored_symbol) + return filtered_list + + for m in module_list: + # TODO: use m.name.isascii() in Py3.7+ + try: + m.name.encode("ascii") + continue + except UnicodeEncodeError: + pass + m.export_symbols = make_filtered_list( + "PyInit_" + m.name.rsplit(".", 1)[-1], + m.export_symbols, + ) + + if os.environ.get('XML_RESULTS'): compile_result_dir = os.environ['XML_RESULTS'] def record_results(func): @@ -1167,7 +1215,8 @@ else: # TODO: Share context? Issue: pyx processing leaks into pxd module @record_results def cythonize_one(pyx_file, c_file, fingerprint, quiet, options=None, - raise_on_failure=True, embedded_metadata=None, full_module_name=None, + raise_on_failure=True, embedded_metadata=None, + full_module_name=None, show_all_warnings=False, progress=""): from ..Compiler.Main import compile_single, default_options from ..Compiler.Errors import CompileError, PyrexError @@ -1183,7 +1232,7 @@ def cythonize_one(pyx_file, c_file, fingerprint, quiet, options=None, zip_fingerprint_file = fingerprint_file_base + '.zip' if os.path.exists(gz_fingerprint_file) or os.path.exists(zip_fingerprint_file): if not quiet: - print("%sFound compiled %s in cache" % (progress, pyx_file)) + print(u"%sFound compiled %s in cache" % (progress, pyx_file)) if os.path.exists(gz_fingerprint_file): os.utime(gz_fingerprint_file, None) with contextlib.closing(gzip_open(gz_fingerprint_file, 'rb')) as g: @@ -1197,12 +1246,16 @@ def cythonize_one(pyx_file, c_file, fingerprint, quiet, options=None, z.extract(artifact, os.path.join(dirname, artifact)) return if not quiet: - print("%sCythonizing %s" % (progress, pyx_file)) + print(u"%sCythonizing %s" % (progress, Utils.decode_filename(pyx_file))) if options is None: options = CompilationOptions(default_options) options.output_file = c_file options.embedded_metadata = embedded_metadata + old_warning_level = Errors.LEVEL + if show_all_warnings: + Errors.LEVEL = 0 + any_failures = 0 try: result = compile_single(pyx_file, options, full_module_name=full_module_name) @@ -1220,6 +1273,10 @@ def cythonize_one(pyx_file, c_file, fingerprint, quiet, options=None, import traceback traceback.print_exc() any_failures = 1 + finally: + if show_all_warnings: + Errors.LEVEL = old_warning_level + if any_failures: if raise_on_failure: raise CompileError(None, pyx_file) @@ -1237,7 +1294,7 @@ def cythonize_one(pyx_file, c_file, fingerprint, quiet, options=None, else: fingerprint_file = zip_fingerprint_file with contextlib.closing(zipfile.ZipFile( - fingerprint_file + '.tmp', 'w', zipfile_compression_mode)) as zip: + fingerprint_file + '.tmp', 'w', zipfile_compression_mode)) as zip: for artifact in artifacts: zip.write(artifact, os.path.basename(artifact)) os.rename(fingerprint_file + '.tmp', fingerprint_file) @@ -1261,9 +1318,10 @@ def _init_multiprocessing_helper(): def cleanup_cache(cache, target_size, ratio=.85): try: p = subprocess.Popen(['du', '-s', '-k', os.path.abspath(cache)], stdout=subprocess.PIPE) + stdout, _ = p.communicate() res = p.wait() if res == 0: - total_size = 1024 * int(p.stdout.read().strip().split()[0]) + total_size = 1024 * int(stdout.strip().split()[0]) if total_size < target_size: return except (OSError, ValueError): diff --git a/Cython/Build/Inline.py b/Cython/Build/Inline.py index a14862901..c56701b58 100644 --- a/Cython/Build/Inline.py +++ b/Cython/Build/Inline.py @@ -1,32 +1,31 @@ from __future__ import absolute_import -import sys, os, re, inspect -import imp - -try: - import hashlib -except ImportError: - import md5 as hashlib +import hashlib +import inspect +import os +import re +import sys from distutils.core import Distribution, Extension from distutils.command.build_ext import build_ext import Cython -from ..Compiler.Main import Context, CompilationOptions, default_options +from ..Compiler.Main import Context +from ..Compiler.Options import default_options -from ..Compiler.ParseTreeTransforms import (CythonTransform, - SkipDeclarations, AnalyseDeclarationsTransform, EnvTransform) +from ..Compiler.ParseTreeTransforms import CythonTransform, SkipDeclarations, EnvTransform from ..Compiler.TreeFragment import parse_from_strings from ..Compiler.StringEncoding import _unicode from .Dependencies import strip_string_literals, cythonize, cached_function -from ..Compiler import Pipeline, Nodes +from ..Compiler import Pipeline from ..Utils import get_cython_cache_dir import cython as cython_module -IS_PY3 = sys.version_info >= (3, 0) + +IS_PY3 = sys.version_info >= (3,) # A utility function to convert user-supplied ASCII strings to unicode. -if sys.version_info[0] < 3: +if not IS_PY3: def to_unicode(s): if isinstance(s, bytes): return s.decode('ascii') @@ -35,6 +34,7 @@ if sys.version_info[0] < 3: else: to_unicode = lambda x: x + if sys.version_info < (3, 5): import imp def load_dynamic(name, module_path): @@ -48,6 +48,7 @@ else: spec.loader.exec_module(module) return module + class UnboundSymbols(EnvTransform, SkipDeclarations): def __init__(self): CythonTransform.__init__(self, None) @@ -132,6 +133,7 @@ def _create_context(cython_include_dirs): _cython_inline_cache = {} _cython_inline_default_context = _create_context(('.',)) + def _populate_unbound(kwds, unbound_symbols, locals=None, globals=None): for symbol in unbound_symbols: if symbol not in kwds: @@ -148,10 +150,12 @@ def _populate_unbound(kwds, unbound_symbols, locals=None, globals=None): else: print("Couldn't find %r" % symbol) + def _inline_key(orig_code, arg_sigs, language_level): key = orig_code, arg_sigs, sys.version_info, sys.executable, language_level, Cython.__version__ return hashlib.sha1(_unicode(key).encode('utf-8')).hexdigest() + def cython_inline(code, get_type=unsafe_type, lib_dir=os.path.join(get_cython_cache_dir(), 'inline'), cython_include_dirs=None, cython_compiler_directives=None, @@ -161,12 +165,14 @@ def cython_inline(code, get_type=unsafe_type, get_type = lambda x: 'object' ctx = _create_context(tuple(cython_include_dirs)) if cython_include_dirs else _cython_inline_default_context - cython_compiler_directives = dict(cython_compiler_directives or {}) + cython_compiler_directives = dict(cython_compiler_directives) if cython_compiler_directives else {} if language_level is None and 'language_level' not in cython_compiler_directives: language_level = '3str' if language_level is not None: cython_compiler_directives['language_level'] = language_level + key_hash = None + # Fast path if this has been called in this session. _unbound_symbols = _cython_inline_cache.get(code) if _unbound_symbols is not None: @@ -202,7 +208,8 @@ def cython_inline(code, get_type=unsafe_type, del kwds[name] arg_names = sorted(kwds) arg_sigs = tuple([(get_type(kwds[arg], ctx), arg) for arg in arg_names]) - key_hash = _inline_key(orig_code, arg_sigs, language_level) + if key_hash is None: + key_hash = _inline_key(orig_code, arg_sigs, language_level) module_name = "_cython_inline_" + key_hash if module_name in sys.modules: @@ -221,6 +228,7 @@ def cython_inline(code, get_type=unsafe_type, os.makedirs(lib_dir) if force or not os.path.isfile(module_path): cflags = [] + define_macros = [] c_include_dirs = [] qualified = re.compile(r'([.\w]+)[.]') for type, _ in arg_sigs: @@ -231,6 +239,7 @@ def cython_inline(code, get_type=unsafe_type, if m.groups()[0] == 'numpy': import numpy c_include_dirs.append(numpy.get_include()) + define_macros.append(("NPY_NO_DEPRECATED_API", "NPY_1_7_API_VERSION")) # cflags.append('-Wno-unused') module_body, func_body = extract_func_code(code) params = ', '.join(['%s %s' % a for a in arg_sigs]) @@ -253,10 +262,12 @@ def __invoke(%(params)s): finally: fh.close() extension = Extension( - name = module_name, - sources = [pyx_file], - include_dirs = c_include_dirs, - extra_compile_args = cflags) + name=module_name, + sources=[pyx_file], + include_dirs=c_include_dirs or None, + extra_compile_args=cflags or None, + define_macros=define_macros or None, + ) if build_extension is None: build_extension = _get_build_extension() build_extension.extensions = cythonize( @@ -274,6 +285,7 @@ def __invoke(%(params)s): arg_list = [kwds[arg] for arg in arg_names] return module.__invoke(*arg_list) + # Cached suffix used by cython_inline above. None should get # overridden with actual value upon the first cython_inline invocation cython_inline.so_ext = None @@ -318,37 +330,6 @@ def extract_func_code(code): return '\n'.join(module), ' ' + '\n '.join(function) -try: - from inspect import getcallargs -except ImportError: - def getcallargs(func, *arg_values, **kwd_values): - all = {} - args, varargs, kwds, defaults = inspect.getargspec(func) - if varargs is not None: - all[varargs] = arg_values[len(args):] - for name, value in zip(args, arg_values): - all[name] = value - for name, value in list(kwd_values.items()): - if name in args: - if name in all: - raise TypeError("Duplicate argument %s" % name) - all[name] = kwd_values.pop(name) - if kwds is not None: - all[kwds] = kwd_values - elif kwd_values: - raise TypeError("Unexpected keyword arguments: %s" % list(kwd_values)) - if defaults is None: - defaults = () - first_default = len(args) - len(defaults) - for ix, name in enumerate(args): - if name not in all: - if ix >= first_default: - all[name] = defaults[ix - first_default] - else: - raise TypeError("Missing argument: %s" % name) - return all - - def get_body(source): ix = source.index(':') if source[:5] == 'lambda': @@ -366,7 +347,7 @@ class RuntimeCompiledFunction(object): self._body = get_body(inspect.getsource(f)) def __call__(self, *args, **kwds): - all = getcallargs(self._f, *args, **kwds) + all = inspect.getcallargs(self._f, *args, **kwds) if IS_PY3: return cython_inline(self._body, locals=self._f.__globals__, globals=self._f.__globals__, **all) else: diff --git a/Cython/Build/IpythonMagic.py b/Cython/Build/IpythonMagic.py index 7abb97ec7..7aa7bf666 100644 --- a/Cython/Build/IpythonMagic.py +++ b/Cython/Build/IpythonMagic.py @@ -60,15 +60,11 @@ IO_ENCODING = sys.getfilesystemencoding() IS_PY2 = sys.version_info[0] < 3 try: - reload -except NameError: # Python 3 - from imp import reload - -try: - import hashlib -except ImportError: - import md5 as hashlib + from importlib import reload +except ImportError: # Python 2 had a builtin function + pass +import hashlib from distutils.core import Distribution, Extension from distutils.command.build_ext import build_ext @@ -192,10 +188,15 @@ class CythonMagics(Magics): @magic_arguments.magic_arguments() @magic_arguments.argument( - '-a', '--annotate', action='store_true', default=False, + '-a', '--annotate', action='store_const', const='default', dest='annotate', help="Produce a colorized HTML version of the source." ) @magic_arguments.argument( + '--annotate-fullc', action='store_const', const='fullc', dest='annotate', + help="Produce a colorized HTML version of the source " + "which includes entire generated C/C++-code." + ) + @magic_arguments.argument( '-+', '--cplus', action='store_true', default=False, help="Output a C++ rather than C file." ) @@ -317,7 +318,7 @@ class CythonMagics(Magics): if args.name: module_name = str(args.name) # no-op in Py3 else: - module_name = "_cython_magic_" + hashlib.md5(str(key).encode('utf-8')).hexdigest() + module_name = "_cython_magic_" + hashlib.sha1(str(key).encode('utf-8')).hexdigest() html_file = os.path.join(lib_dir, module_name + '.html') module_path = os.path.join(lib_dir, module_name + self.so_ext) @@ -439,12 +440,11 @@ class CythonMagics(Magics): quiet=quiet, annotate=args.annotate, force=True, + language_level=min(3, sys.version_info[0]), ) if args.language_level is not None: assert args.language_level in (2, 3) opts['language_level'] = args.language_level - elif sys.version_info[0] >= 3: - opts['language_level'] = 3 return cythonize([extension], **opts) except CompileError: return None diff --git a/Cython/Build/Tests/TestCyCache.py b/Cython/Build/Tests/TestCyCache.py index a3224b417..7a44d89e9 100644 --- a/Cython/Build/Tests/TestCyCache.py +++ b/Cython/Build/Tests/TestCyCache.py @@ -33,25 +33,31 @@ class TestCyCache(CythonTest): a_pyx = os.path.join(self.src_dir, 'a.pyx') a_c = a_pyx[:-4] + '.c' - open(a_pyx, 'w').write(content1) + with open(a_pyx, 'w') as f: + f.write(content1) self.fresh_cythonize(a_pyx, cache=self.cache_dir) self.fresh_cythonize(a_pyx, cache=self.cache_dir) self.assertEqual(1, len(self.cache_files('a.c*'))) - a_contents1 = open(a_c).read() + with open(a_c) as f: + a_contents1 = f.read() os.unlink(a_c) - open(a_pyx, 'w').write(content2) + with open(a_pyx, 'w') as f: + f.write(content2) self.fresh_cythonize(a_pyx, cache=self.cache_dir) - a_contents2 = open(a_c).read() + with open(a_c) as f: + a_contents2 = f.read() os.unlink(a_c) self.assertNotEqual(a_contents1, a_contents2, 'C file not changed!') self.assertEqual(2, len(self.cache_files('a.c*'))) - open(a_pyx, 'w').write(content1) + with open(a_pyx, 'w') as f: + f.write(content1) self.fresh_cythonize(a_pyx, cache=self.cache_dir) self.assertEqual(2, len(self.cache_files('a.c*'))) - a_contents = open(a_c).read() + with open(a_c) as f: + a_contents = f.read() self.assertEqual( a_contents, a_contents1, msg='\n'.join(list(difflib.unified_diff( @@ -60,13 +66,15 @@ class TestCyCache(CythonTest): def test_cycache_uses_cache(self): a_pyx = os.path.join(self.src_dir, 'a.pyx') a_c = a_pyx[:-4] + '.c' - open(a_pyx, 'w').write('pass') + with open(a_pyx, 'w') as f: + f.write('pass') self.fresh_cythonize(a_pyx, cache=self.cache_dir) a_cache = os.path.join(self.cache_dir, os.listdir(self.cache_dir)[0]) gzip.GzipFile(a_cache, 'wb').write('fake stuff'.encode('ascii')) os.unlink(a_c) self.fresh_cythonize(a_pyx, cache=self.cache_dir) - a_contents = open(a_c).read() + with open(a_c) as f: + a_contents = f.read() self.assertEqual(a_contents, 'fake stuff', 'Unexpected contents: %s...' % a_contents[:100]) @@ -75,7 +83,8 @@ class TestCyCache(CythonTest): a_c = a_pyx[:-4] + '.c' a_h = a_pyx[:-4] + '.h' a_api_h = a_pyx[:-4] + '_api.h' - open(a_pyx, 'w').write('cdef public api int foo(int x): return x\n') + with open(a_pyx, 'w') as f: + f.write('cdef public api int foo(int x): return x\n') self.fresh_cythonize(a_pyx, cache=self.cache_dir) expected = [a_c, a_h, a_api_h] for output in expected: @@ -89,7 +98,8 @@ class TestCyCache(CythonTest): hash_pyx = os.path.join(self.src_dir, 'options.pyx') hash_c = hash_pyx[:-len('.pyx')] + '.c' - open(hash_pyx, 'w').write('pass') + with open(hash_pyx, 'w') as f: + f.write('pass') self.fresh_cythonize(hash_pyx, cache=self.cache_dir, cplus=False) self.assertEqual(1, len(self.cache_files('options.c*'))) diff --git a/Cython/Build/Tests/TestCythonizeArgsParser.py b/Cython/Build/Tests/TestCythonizeArgsParser.py new file mode 100644 index 000000000..c5a682dd6 --- /dev/null +++ b/Cython/Build/Tests/TestCythonizeArgsParser.py @@ -0,0 +1,482 @@ +from Cython.Build.Cythonize import ( + create_args_parser, parse_args_raw, parse_args, + parallel_compiles +) + +from Cython.Compiler import Options +from Cython.Compiler.Tests.Utils import backup_Options, restore_Options, check_global_options + +from unittest import TestCase + +import sys +try: + from StringIO import StringIO +except ImportError: + from io import StringIO # doesn't accept 'str' in Py2 + + +class TestCythonizeArgsParser(TestCase): + + def setUp(self): + TestCase.setUp(self) + self.parse_args = lambda x, parser=create_args_parser() : parse_args_raw(parser, x) + + + def are_default(self, options, skip): + # empty containers + empty_containers = ['directives', 'compile_time_env', 'options', 'excludes'] + are_none = ['language_level', 'annotate', 'build', 'build_inplace', 'force', 'quiet', 'lenient', 'keep_going', 'no_docstrings'] + for opt_name in empty_containers: + if len(getattr(options, opt_name))!=0 and (opt_name not in skip): + self.assertEqual(opt_name,"", msg="For option "+opt_name) + return False + for opt_name in are_none: + if (getattr(options, opt_name) is not None) and (opt_name not in skip): + self.assertEqual(opt_name,"", msg="For option "+opt_name) + return False + if options.parallel!=parallel_compiles and ('parallel' not in skip): + return False + return True + + # testing directives: + def test_directive_short(self): + options, args = self.parse_args(['-X', 'cdivision=True']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['directives'])) + self.assertEqual(options.directives['cdivision'], True) + + def test_directive_long(self): + options, args = self.parse_args(['--directive', 'cdivision=True']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['directives'])) + self.assertEqual(options.directives['cdivision'], True) + + def test_directive_multiple(self): + options, args = self.parse_args(['-X', 'cdivision=True', '-X', 'c_string_type=bytes']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['directives'])) + self.assertEqual(options.directives['cdivision'], True) + self.assertEqual(options.directives['c_string_type'], 'bytes') + + def test_directive_multiple_v2(self): + options, args = self.parse_args(['-X', 'cdivision=True,c_string_type=bytes']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['directives'])) + self.assertEqual(options.directives['cdivision'], True) + self.assertEqual(options.directives['c_string_type'], 'bytes') + + def test_directive_value_yes(self): + options, args = self.parse_args(['-X', 'cdivision=YeS']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['directives'])) + self.assertEqual(options.directives['cdivision'], True) + + def test_directive_value_no(self): + options, args = self.parse_args(['-X', 'cdivision=no']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['directives'])) + self.assertEqual(options.directives['cdivision'], False) + + def test_directive_value_invalid(self): + with self.assertRaises(ValueError) as context: + options, args = self.parse_args(['-X', 'cdivision=sadfasd']) + + def test_directive_key_invalid(self): + with self.assertRaises(ValueError) as context: + options, args = self.parse_args(['-X', 'abracadabra']) + + def test_directive_no_value(self): + with self.assertRaises(ValueError) as context: + options, args = self.parse_args(['-X', 'cdivision']) + + def test_directives_types(self): + directives = { + 'auto_pickle': True, + 'c_string_type': 'bytearray', + 'c_string_type': 'bytes', + 'c_string_type': 'str', + 'c_string_type': 'bytearray', + 'c_string_type': 'unicode', + 'c_string_encoding' : 'ascii', + 'language_level' : 2, + 'language_level' : 3, + 'language_level' : '3str', + 'set_initial_path' : 'my_initial_path', + } + for key, value in directives.items(): + cmd = '{key}={value}'.format(key=key, value=str(value)) + options, args = self.parse_args(['-X', cmd]) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['directives']), msg = "Error for option: "+cmd) + self.assertEqual(options.directives[key], value, msg = "Error for option: "+cmd) + + def test_directives_wrong(self): + directives = { + 'auto_pickle': 42, # for bool type + 'auto_pickle': 'NONONO', # for bool type + 'c_string_type': 'bites', + #'c_string_encoding' : 'a', + #'language_level' : 4, + } + for key, value in directives.items(): + cmd = '{key}={value}'.format(key=key, value=str(value)) + with self.assertRaises(ValueError, msg = "Error for option: "+cmd) as context: + options, args = self.parse_args(['-X', cmd]) + + def test_compile_time_env_short(self): + options, args = self.parse_args(['-E', 'MYSIZE=10']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['compile_time_env'])) + self.assertEqual(options.compile_time_env['MYSIZE'], 10) + + def test_compile_time_env_long(self): + options, args = self.parse_args(['--compile-time-env', 'MYSIZE=10']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['compile_time_env'])) + self.assertEqual(options.compile_time_env['MYSIZE'], 10) + + def test_compile_time_env_multiple(self): + options, args = self.parse_args(['-E', 'MYSIZE=10', '-E', 'ARRSIZE=11']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['compile_time_env'])) + self.assertEqual(options.compile_time_env['MYSIZE'], 10) + self.assertEqual(options.compile_time_env['ARRSIZE'], 11) + + def test_compile_time_env_multiple_v2(self): + options, args = self.parse_args(['-E', 'MYSIZE=10,ARRSIZE=11']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['compile_time_env'])) + self.assertEqual(options.compile_time_env['MYSIZE'], 10) + self.assertEqual(options.compile_time_env['ARRSIZE'], 11) + + #testing options + def test_option_short(self): + options, args = self.parse_args(['-s', 'docstrings=True']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['options'])) + self.assertEqual(options.options['docstrings'], True) + + def test_option_long(self): + options, args = self.parse_args(['--option', 'docstrings=True']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['options'])) + self.assertEqual(options.options['docstrings'], True) + + def test_option_multiple(self): + options, args = self.parse_args(['-s', 'docstrings=True', '-s', 'buffer_max_dims=8']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['options'])) + self.assertEqual(options.options['docstrings'], True) + self.assertEqual(options.options['buffer_max_dims'], True) # really? + + def test_option_multiple_v2(self): + options, args = self.parse_args(['-s', 'docstrings=True,buffer_max_dims=8']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['options'])) + self.assertEqual(options.options['docstrings'], True) + self.assertEqual(options.options['buffer_max_dims'], True) # really? + + def test_option_value_yes(self): + options, args = self.parse_args(['-s', 'docstrings=YeS']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['options'])) + self.assertEqual(options.options['docstrings'], True) + + def test_option_value_4242(self): + options, args = self.parse_args(['-s', 'docstrings=4242']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['options'])) + self.assertEqual(options.options['docstrings'], True) + + def test_option_value_0(self): + options, args = self.parse_args(['-s', 'docstrings=0']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['options'])) + self.assertEqual(options.options['docstrings'], False) + + def test_option_value_emptystr(self): + options, args = self.parse_args(['-s', 'docstrings=']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['options'])) + self.assertEqual(options.options['docstrings'], True) + + def test_option_value_a_str(self): + options, args = self.parse_args(['-s', 'docstrings=BB']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['options'])) + self.assertEqual(options.options['docstrings'], True) + + def test_option_value_no(self): + options, args = self.parse_args(['-s', 'docstrings=nO']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['options'])) + self.assertEqual(options.options['docstrings'], False) + + def test_option_no_value(self): + options, args = self.parse_args(['-s', 'docstrings']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['options'])) + self.assertEqual(options.options['docstrings'], True) + + def test_option_any_key(self): + options, args = self.parse_args(['-s', 'abracadabra']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['options'])) + self.assertEqual(options.options['abracadabra'], True) + + def test_language_level_2(self): + options, args = self.parse_args(['-2']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['language_level'])) + self.assertEqual(options.language_level, 2) + + def test_language_level_3(self): + options, args = self.parse_args(['-3']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['language_level'])) + self.assertEqual(options.language_level, 3) + + def test_language_level_3str(self): + options, args = self.parse_args(['--3str']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['language_level'])) + self.assertEqual(options.language_level, '3str') + + def test_annotate_short(self): + options, args = self.parse_args(['-a']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['annotate'])) + self.assertEqual(options.annotate, 'default') + + def test_annotate_long(self): + options, args = self.parse_args(['--annotate']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['annotate'])) + self.assertEqual(options.annotate, 'default') + + def test_annotate_fullc(self): + options, args = self.parse_args(['--annotate-fullc']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['annotate'])) + self.assertEqual(options.annotate, 'fullc') + + def test_annotate_and_positional(self): + options, args = self.parse_args(['-a', 'foo.pyx']) + self.assertEqual(args, ['foo.pyx']) + self.assertTrue(self.are_default(options, ['annotate'])) + self.assertEqual(options.annotate, 'default') + + def test_annotate_and_optional(self): + options, args = self.parse_args(['-a', '--3str']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['annotate', 'language_level'])) + self.assertEqual(options.annotate, 'default') + self.assertEqual(options.language_level, '3str') + + def test_exclude_short(self): + options, args = self.parse_args(['-x', '*.pyx']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['excludes'])) + self.assertTrue('*.pyx' in options.excludes) + + def test_exclude_long(self): + options, args = self.parse_args(['--exclude', '*.pyx']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['excludes'])) + self.assertTrue('*.pyx' in options.excludes) + + def test_exclude_multiple(self): + options, args = self.parse_args(['--exclude', '*.pyx', '--exclude', '*.py', ]) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['excludes'])) + self.assertEqual(options.excludes, ['*.pyx', '*.py']) + + def test_build_short(self): + options, args = self.parse_args(['-b']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['build'])) + self.assertEqual(options.build, True) + + def test_build_long(self): + options, args = self.parse_args(['--build']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['build'])) + self.assertEqual(options.build, True) + + def test_inplace_short(self): + options, args = self.parse_args(['-i']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['build_inplace'])) + self.assertEqual(options.build_inplace, True) + + def test_inplace_long(self): + options, args = self.parse_args(['--inplace']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['build_inplace'])) + self.assertEqual(options.build_inplace, True) + + def test_parallel_short(self): + options, args = self.parse_args(['-j', '42']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['parallel'])) + self.assertEqual(options.parallel, 42) + + def test_parallel_long(self): + options, args = self.parse_args(['--parallel', '42']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['parallel'])) + self.assertEqual(options.parallel, 42) + + def test_force_short(self): + options, args = self.parse_args(['-f']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['force'])) + self.assertEqual(options.force, True) + + def test_force_long(self): + options, args = self.parse_args(['--force']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['force'])) + self.assertEqual(options.force, True) + + def test_quite_short(self): + options, args = self.parse_args(['-q']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['quiet'])) + self.assertEqual(options.quiet, True) + + def test_quite_long(self): + options, args = self.parse_args(['--quiet']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['quiet'])) + self.assertEqual(options.quiet, True) + + def test_lenient_long(self): + options, args = self.parse_args(['--lenient']) + self.assertTrue(self.are_default(options, ['lenient'])) + self.assertFalse(args) + self.assertEqual(options.lenient, True) + + def test_keep_going_short(self): + options, args = self.parse_args(['-k']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['keep_going'])) + self.assertEqual(options.keep_going, True) + + def test_keep_going_long(self): + options, args = self.parse_args(['--keep-going']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['keep_going'])) + self.assertEqual(options.keep_going, True) + + def test_no_docstrings_long(self): + options, args = self.parse_args(['--no-docstrings']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['no_docstrings'])) + self.assertEqual(options.no_docstrings, True) + + def test_file_name(self): + options, args = self.parse_args(['file1.pyx', 'file2.pyx']) + self.assertEqual(len(args), 2) + self.assertEqual(args[0], 'file1.pyx') + self.assertEqual(args[1], 'file2.pyx') + self.assertTrue(self.are_default(options, [])) + + def test_option_first(self): + options, args = self.parse_args(['-i', 'file.pyx']) + self.assertEqual(args, ['file.pyx']) + self.assertEqual(options.build_inplace, True) + self.assertTrue(self.are_default(options, ['build_inplace'])) + + def test_file_inbetween(self): + options, args = self.parse_args(['-i', 'file.pyx', '-a']) + self.assertEqual(args, ['file.pyx']) + self.assertEqual(options.build_inplace, True) + self.assertEqual(options.annotate, 'default') + self.assertTrue(self.are_default(options, ['build_inplace', 'annotate'])) + + def test_option_trailing(self): + options, args = self.parse_args(['file.pyx', '-i']) + self.assertEqual(args, ['file.pyx']) + self.assertEqual(options.build_inplace, True) + self.assertTrue(self.are_default(options, ['build_inplace'])) + + def test_interspersed_positional(self): + options, sources = self.parse_args([ + 'file1.pyx', '-a', + 'file2.pyx' + ]) + self.assertEqual(sources, ['file1.pyx', 'file2.pyx']) + self.assertEqual(options.annotate, 'default') + self.assertTrue(self.are_default(options, ['annotate'])) + + def test_interspersed_positional2(self): + options, sources = self.parse_args([ + 'file1.pyx', '-a', + 'file2.pyx', '-a', 'file3.pyx' + ]) + self.assertEqual(sources, ['file1.pyx', 'file2.pyx', 'file3.pyx']) + self.assertEqual(options.annotate, 'default') + self.assertTrue(self.are_default(options, ['annotate'])) + + def test_interspersed_positional3(self): + options, sources = self.parse_args([ + '-f', 'f1', 'f2', '-a', + 'f3', 'f4', '-a', 'f5' + ]) + self.assertEqual(sources, ['f1', 'f2', 'f3', 'f4', 'f5']) + self.assertEqual(options.annotate, 'default') + self.assertEqual(options.force, True) + self.assertTrue(self.are_default(options, ['annotate', 'force'])) + + def test_wrong_option(self): + old_stderr = sys.stderr + stderr = sys.stderr = StringIO() + try: + self.assertRaises(SystemExit, self.parse_args, + ['--unknown-option'] + ) + finally: + sys.stderr = old_stderr + self.assertTrue(stderr.getvalue()) + + +class TestParseArgs(TestCase): + def setUp(self): + self._options_backup = backup_Options() + + def tearDown(self): + restore_Options(self._options_backup) + + def check_default_global_options(self, white_list=[]): + self.assertEqual(check_global_options(self._options_backup, white_list), "") + + def test_build_set_for_inplace(self): + options, args = parse_args(['foo.pyx', '-i']) + self.assertEqual(options.build, True) + self.check_default_global_options() + + def test_lenient(self): + options, sources = parse_args(['foo.pyx', '--lenient']) + self.assertEqual(sources, ['foo.pyx']) + self.assertEqual(Options.error_on_unknown_names, False) + self.assertEqual(Options.error_on_uninitialized, False) + self.check_default_global_options(['error_on_unknown_names', 'error_on_uninitialized']) + + def test_annotate(self): + options, sources = parse_args(['foo.pyx', '--annotate']) + self.assertEqual(sources, ['foo.pyx']) + self.assertEqual(Options.annotate, 'default') + self.check_default_global_options(['annotate']) + + def test_annotate_fullc(self): + options, sources = parse_args(['foo.pyx', '--annotate-fullc']) + self.assertEqual(sources, ['foo.pyx']) + self.assertEqual(Options.annotate, 'fullc') + self.check_default_global_options(['annotate']) + + def test_no_docstrings(self): + options, sources = parse_args(['foo.pyx', '--no-docstrings']) + self.assertEqual(sources, ['foo.pyx']) + self.assertEqual(Options.docstrings, False) + self.check_default_global_options(['docstrings']) diff --git a/Cython/Build/Tests/TestInline.py b/Cython/Build/Tests/TestInline.py index 5ef9fec4e..35d9a29cd 100644 --- a/Cython/Build/Tests/TestInline.py +++ b/Cython/Build/Tests/TestInline.py @@ -1,4 +1,6 @@ -import os, tempfile +import os +import tempfile +import unittest from Cython.Shadow import inline from Cython.Build.Inline import safe_type from Cython.TestUtils import CythonTest @@ -24,10 +26,10 @@ class TestInline(CythonTest): self.test_kwds['lib_dir'] = lib_dir def test_simple(self): - self.assertEquals(inline("return 1+2", **self.test_kwds), 3) + self.assertEqual(inline("return 1+2", **self.test_kwds), 3) def test_types(self): - self.assertEquals(inline(""" + self.assertEqual(inline(""" cimport cython return cython.typeof(a), cython.typeof(b) """, a=1.0, b=[], **self.test_kwds), ('double', 'list object')) @@ -35,13 +37,13 @@ class TestInline(CythonTest): def test_locals(self): a = 1 b = 2 - self.assertEquals(inline("return a+b", **self.test_kwds), 3) + self.assertEqual(inline("return a+b", **self.test_kwds), 3) def test_globals(self): - self.assertEquals(inline("return global_value + 1", **self.test_kwds), global_value + 1) + self.assertEqual(inline("return global_value + 1", **self.test_kwds), global_value + 1) def test_no_return(self): - self.assertEquals(inline(""" + self.assertEqual(inline(""" a = 1 cdef double b = 2 cdef c = [] @@ -49,7 +51,7 @@ class TestInline(CythonTest): def test_def_node(self): foo = inline("def foo(x): return x * x", **self.test_kwds)['foo'] - self.assertEquals(foo(7), 49) + self.assertEqual(foo(7), 49) def test_class_ref(self): class Type(object): @@ -64,7 +66,7 @@ class TestInline(CythonTest): c = cy.declare(cy.pointer(cy.float), &b) return b """, a=3, **self.test_kwds) - self.assertEquals(type(b), float) + self.assertEqual(type(b), float) def test_compiler_directives(self): self.assertEqual( @@ -85,12 +87,26 @@ class TestInline(CythonTest): inline(inline_divcode, language_level=3)['f'](5,2), 2.5 ) + self.assertEqual( + inline(inline_divcode, language_level=2)['f'](5,2), + 2 + ) - if has_numpy: - - def test_numpy(self): - import numpy - a = numpy.ndarray((10, 20)) - a[0,0] = 10 - self.assertEquals(safe_type(a), 'numpy.ndarray[numpy.float64_t, ndim=2]') - self.assertEquals(inline("return a[0,0]", a=a, **self.test_kwds), 10.0) + def test_repeated_use(self): + inline_mulcode = "def f(int a, int b): return a * b" + self.assertEqual(inline(inline_mulcode)['f'](5, 2), 10) + self.assertEqual(inline(inline_mulcode)['f'](5, 3), 15) + self.assertEqual(inline(inline_mulcode)['f'](6, 2), 12) + self.assertEqual(inline(inline_mulcode)['f'](5, 2), 10) + + f = inline(inline_mulcode)['f'] + self.assertEqual(f(5, 2), 10) + self.assertEqual(f(5, 3), 15) + + @unittest.skipIf(not has_numpy, "NumPy is not available") + def test_numpy(self): + import numpy + a = numpy.ndarray((10, 20)) + a[0,0] = 10 + self.assertEqual(safe_type(a), 'numpy.ndarray[numpy.float64_t, ndim=2]') + self.assertEqual(inline("return a[0,0]", a=a, **self.test_kwds), 10.0) diff --git a/Cython/Build/Tests/TestIpythonMagic.py b/Cython/Build/Tests/TestIpythonMagic.py index d9d8322a8..ed4db98cb 100644 --- a/Cython/Build/Tests/TestIpythonMagic.py +++ b/Cython/Build/Tests/TestIpythonMagic.py @@ -10,6 +10,7 @@ import sys from contextlib import contextmanager from Cython.Build import IpythonMagic from Cython.TestUtils import CythonTest +from Cython.Compiler.Annotate import AnnotationCCodeWriter try: import IPython.testing.globalipapp @@ -195,11 +196,37 @@ x = sin(0.0) ip.run_cell_magic('cython', '--verbose', code) ip.ex('g = f(10)') self.assertEqual(ip.user_ns['g'], 20.0) - self.assertEquals([verbose_log.INFO, verbose_log.DEBUG, verbose_log.INFO], + self.assertEqual([verbose_log.INFO, verbose_log.DEBUG, verbose_log.INFO], verbose_log.thresholds) with mock_distutils() as normal_log: ip.run_cell_magic('cython', '', code) ip.ex('g = f(10)') self.assertEqual(ip.user_ns['g'], 20.0) - self.assertEquals([normal_log.INFO], normal_log.thresholds) + self.assertEqual([normal_log.INFO], normal_log.thresholds) + + def test_cython_no_annotate(self): + ip = self._ip + html = ip.run_cell_magic('cython', '', code) + self.assertTrue(html is None) + + def test_cython_annotate(self): + ip = self._ip + html = ip.run_cell_magic('cython', '--annotate', code) + # somewhat brittle way to differentiate between annotated htmls + # with/without complete source code: + self.assertTrue(AnnotationCCodeWriter.COMPLETE_CODE_TITLE not in html.data) + + def test_cython_annotate_default(self): + ip = self._ip + html = ip.run_cell_magic('cython', '-a', code) + # somewhat brittle way to differentiate between annotated htmls + # with/without complete source code: + self.assertTrue(AnnotationCCodeWriter.COMPLETE_CODE_TITLE not in html.data) + + def test_cython_annotate_complete_c_code(self): + ip = self._ip + html = ip.run_cell_magic('cython', '--annotate-fullc', code) + # somewhat brittle way to differentiate between annotated htmls + # with/without complete source code: + self.assertTrue(AnnotationCCodeWriter.COMPLETE_CODE_TITLE in html.data) diff --git a/Cython/Build/Tests/TestStripLiterals.py b/Cython/Build/Tests/TestStripLiterals.py index a7572a508..cbe5c65a9 100644 --- a/Cython/Build/Tests/TestStripLiterals.py +++ b/Cython/Build/Tests/TestStripLiterals.py @@ -54,4 +54,3 @@ class TestStripLiterals(CythonTest): def test_extern(self): self.t("cdef extern from 'a.h': # comment", "cdef extern from '_L1_': #_L2_") - diff --git a/Cython/CodeWriter.py b/Cython/CodeWriter.py index 2e4646a65..1e5db318d 100644 --- a/Cython/CodeWriter.py +++ b/Cython/CodeWriter.py @@ -1,7 +1,6 @@ """ Serializes a Cython code tree to Cython code. This is primarily useful for debugging and testing purposes. - The output is in a strict format, no whitespace or comments from the input is preserved (and it could not be as it is not present in the code tree). """ @@ -10,6 +9,7 @@ from __future__ import absolute_import, print_function from .Compiler.Visitor import TreeVisitor from .Compiler.ExprNodes import * +from .Compiler.Nodes import CNameDeclaratorNode, CSimpleBaseTypeNode class LinesResult(object): @@ -28,7 +28,11 @@ class LinesResult(object): self.put(s) self.newline() + class DeclarationWriter(TreeVisitor): + """ + A Cython code writer that is limited to declarations nodes. + """ indent_string = u" " @@ -76,6 +80,14 @@ class DeclarationWriter(TreeVisitor): self.visit(item.default) self.put(u", ") self.visit(items[-1]) + if output_rhs and items[-1].default is not None: + self.put(u" = ") + self.visit(items[-1].default) + + def _visit_indented(self, node): + self.indent() + self.visit(node) + self.dedent() def visit_Node(self, node): raise AssertionError("Node not handled by serializer: %r" % node) @@ -92,9 +104,7 @@ class DeclarationWriter(TreeVisitor): else: file = u'"%s"' % node.include_file self.putline(u"cdef extern from %s:" % file) - self.indent() - self.visit(node.body) - self.dedent() + self._visit_indented(node.body) def visit_CPtrDeclaratorNode(self, node): self.put('*') @@ -111,13 +121,6 @@ class DeclarationWriter(TreeVisitor): self.visit(node.dimension) self.put(u']') - def visit_CArrayDeclaratorNode(self, node): - self.visit(node.base) - self.put(u'[') - if node.dimension is not None: - self.visit(node.dimension) - self.put(u']') - def visit_CFuncDeclaratorNode(self, node): # TODO: except, gil, etc. self.visit(node.base) @@ -136,13 +139,12 @@ class DeclarationWriter(TreeVisitor): self.put("short " * -node.longness) elif node.longness > 0: self.put("long " * node.longness) - self.put(node.name) + if node.name is not None: + self.put(node.name) def visit_CComplexBaseTypeNode(self, node): - self.put(u'(') self.visit(node.base_type) self.visit(node.declarator) - self.put(u')') def visit_CNestedBaseTypeNode(self, node): self.visit(node.base_type) @@ -162,7 +164,7 @@ class DeclarationWriter(TreeVisitor): self.comma_separated_list(node.declarators, output_rhs=True) self.endline() - def visit_container_node(self, node, decl, extras, attributes): + def _visit_container_node(self, node, decl, extras, attributes): # TODO: visibility self.startline(decl) if node.name: @@ -191,7 +193,7 @@ class DeclarationWriter(TreeVisitor): if node.packed: decl += u'packed ' decl += node.kind - self.visit_container_node(node, decl, None, node.attributes) + self._visit_container_node(node, decl, None, node.attributes) def visit_CppClassNode(self, node): extras = "" @@ -199,10 +201,10 @@ class DeclarationWriter(TreeVisitor): extras = u"[%s]" % ", ".join(node.templates) if node.base_classes: extras += "(%s)" % ", ".join(node.base_classes) - self.visit_container_node(node, u"cdef cppclass", extras, node.attributes) + self._visit_container_node(node, u"cdef cppclass", extras, node.attributes) def visit_CEnumDefNode(self, node): - self.visit_container_node(node, u"cdef enum", None, node.items) + self._visit_container_node(node, u"cdef enum", None, node.items) def visit_CEnumDefItemNode(self, node): self.startline(node.name) @@ -228,9 +230,7 @@ class DeclarationWriter(TreeVisitor): self.put(node.base_class_name) self.put(u")") self.endline(u":") - self.indent() - self.visit(node.body) - self.dedent() + self._visit_indented(node.body) def visit_CTypeDefNode(self, node): self.startline(u"ctypedef ") @@ -240,17 +240,49 @@ class DeclarationWriter(TreeVisitor): self.endline() def visit_FuncDefNode(self, node): + # TODO: support cdef + cpdef functions self.startline(u"def %s(" % node.name) self.comma_separated_list(node.args) self.endline(u"):") - self.indent() - self.visit(node.body) - self.dedent() + self._visit_indented(node.body) + + def visit_CFuncDefNode(self, node): + self.startline(u'cpdef ' if node.overridable else u'cdef ') + if node.modifiers: + self.put(' '.join(node.modifiers)) + self.put(' ') + if node.visibility != 'private': + self.put(node.visibility) + self.put(u' ') + if node.api: + self.put(u'api ') + + if node.base_type: + self.visit(node.base_type) + if node.base_type.name is not None: + self.put(u' ') + + # visit the CFuncDeclaratorNode, but put a `:` at the end of line + self.visit(node.declarator.base) + self.put(u'(') + self.comma_separated_list(node.declarator.args) + self.endline(u'):') + + self._visit_indented(node.body) def visit_CArgDeclNode(self, node): - if node.base_type.name is not None: + # For "CSimpleBaseTypeNode", the variable type may have been parsed as type. + # For other node types, the "name" is always None. + if not isinstance(node.base_type, CSimpleBaseTypeNode) or \ + node.base_type.name is not None: self.visit(node.base_type) - self.put(u" ") + + # If we printed something for "node.base_type", we may need to print an extra ' '. + # + # Special case: if "node.declarator" is a "CNameDeclaratorNode", + # its "name" might be an empty string, for example, for "cdef f(x)". + if node.declarator.declared_name(): + self.put(u" ") self.visit(node.declarator) if node.default is not None: self.put(u" = ") @@ -284,46 +316,20 @@ class DeclarationWriter(TreeVisitor): def visit_NameNode(self, node): self.put(node.name) - def visit_IntNode(self, node): - self.put(node.value) - - def visit_NoneNode(self, node): - self.put(u"None") - - def visit_NotNode(self, node): - self.put(u"(not ") - self.visit(node.operand) - self.put(u")") - def visit_DecoratorNode(self, node): self.startline("@") self.visit(node.decorator) self.endline() - def visit_BinopNode(self, node): - self.visit(node.operand1) - self.put(u" %s " % node.operator) - self.visit(node.operand2) - - def visit_AttributeNode(self, node): - self.visit(node.obj) - self.put(u".%s" % node.attribute) - - def visit_BoolNode(self, node): - self.put(str(node.value)) - - # FIXME: represent string nodes correctly - def visit_StringNode(self, node): - value = node.value - if value.encoding is not None: - value = value.encode(value.encoding) - self.put(repr(value)) - def visit_PassStatNode(self, node): self.startline(u"pass") self.endline() -class CodeWriter(DeclarationWriter): + +class StatementWriter(DeclarationWriter): + """ + A Cython code writer for most language statement features. + """ def visit_SingleAssignmentNode(self, node): self.startline() @@ -349,18 +355,17 @@ class CodeWriter(DeclarationWriter): def visit_ForInStatNode(self, node): self.startline(u"for ") - self.visit(node.target) + if node.target.is_sequence_constructor: + self.comma_separated_list(node.target.args) + else: + self.visit(node.target) self.put(u" in ") self.visit(node.iterator.sequence) self.endline(u":") - self.indent() - self.visit(node.body) - self.dedent() + self._visit_indented(node.body) if node.else_clause is not None: self.line(u"else:") - self.indent() - self.visit(node.else_clause) - self.dedent() + self._visit_indented(node.else_clause) def visit_IfStatNode(self, node): # The IfClauseNode is handled directly without a separate match @@ -368,50 +373,33 @@ class CodeWriter(DeclarationWriter): self.startline(u"if ") self.visit(node.if_clauses[0].condition) self.endline(":") - self.indent() - self.visit(node.if_clauses[0].body) - self.dedent() + self._visit_indented(node.if_clauses[0].body) for clause in node.if_clauses[1:]: self.startline("elif ") self.visit(clause.condition) self.endline(":") - self.indent() - self.visit(clause.body) - self.dedent() + self._visit_indented(clause.body) if node.else_clause is not None: self.line("else:") - self.indent() - self.visit(node.else_clause) - self.dedent() + self._visit_indented(node.else_clause) - def visit_SequenceNode(self, node): - self.comma_separated_list(node.args) # Might need to discover whether we need () around tuples...hmm... + def visit_WhileStatNode(self, node): + self.startline(u"while ") + self.visit(node.condition) + self.endline(u":") + self._visit_indented(node.body) + if node.else_clause is not None: + self.line("else:") + self._visit_indented(node.else_clause) - def visit_SimpleCallNode(self, node): - self.visit(node.function) - self.put(u"(") - self.comma_separated_list(node.args) - self.put(")") + def visit_ContinueStatNode(self, node): + self.line(u"continue") - def visit_GeneralCallNode(self, node): - self.visit(node.function) - self.put(u"(") - posarg = node.positional_args - if isinstance(posarg, AsTupleNode): - self.visit(posarg.arg) - else: - self.comma_separated_list(posarg.args) # TupleNode.args - if node.keyword_args: - if isinstance(node.keyword_args, DictNode): - for i, (name, value) in enumerate(node.keyword_args.key_value_pairs): - if i > 0: - self.put(', ') - self.visit(name) - self.put('=') - self.visit(value) - else: - raise Exception("Not implemented yet") - self.put(u")") + def visit_BreakStatNode(self, node): + self.line(u"break") + + def visit_SequenceNode(self, node): + self.comma_separated_list(node.args) # Might need to discover whether we need () around tuples...hmm... def visit_ExprStatNode(self, node): self.startline() @@ -433,25 +421,17 @@ class CodeWriter(DeclarationWriter): self.put(u" as ") self.visit(node.target) self.endline(u":") - self.indent() - self.visit(node.body) - self.dedent() + self._visit_indented(node.body) def visit_TryFinallyStatNode(self, node): self.line(u"try:") - self.indent() - self.visit(node.body) - self.dedent() + self._visit_indented(node.body) self.line(u"finally:") - self.indent() - self.visit(node.finally_clause) - self.dedent() + self._visit_indented(node.finally_clause) def visit_TryExceptStatNode(self, node): self.line(u"try:") - self.indent() - self.visit(node.body) - self.dedent() + self._visit_indented(node.body) for x in node.except_clauses: self.visit(x) if node.else_clause is not None: @@ -466,13 +446,13 @@ class CodeWriter(DeclarationWriter): self.put(u", ") self.visit(node.target) self.endline(":") - self.indent() - self.visit(node.body) - self.dedent() + self._visit_indented(node.body) def visit_ReturnStatNode(self, node): - self.startline("return ") - self.visit(node.value) + self.startline("return") + if node.value is not None: + self.put(u" ") + self.visit(node.value) self.endline() def visit_ReraiseStatNode(self, node): @@ -498,30 +478,10 @@ class CodeWriter(DeclarationWriter): self.put(self.tempnames[node.handle]) -class PxdWriter(DeclarationWriter): - def __call__(self, node): - print(u'\n'.join(self.write(node).lines)) - return node - - def visit_CFuncDefNode(self, node): - if 'inline' in node.modifiers: - return - if node.overridable: - self.startline(u'cpdef ') - else: - self.startline(u'cdef ') - if node.visibility != 'private': - self.put(node.visibility) - self.put(u' ') - if node.api: - self.put(u'api ') - self.visit(node.declarator) - - def visit_StatNode(self, node): - pass - - class ExpressionWriter(TreeVisitor): + """ + A Cython code writer that is intentionally limited to expressions. + """ def __init__(self, result=None): super(ExpressionWriter, self).__init__() @@ -551,12 +511,18 @@ class ExpressionWriter(TreeVisitor): def visit_Node(self, node): raise AssertionError("Node not handled by serializer: %r" % node) - def visit_NameNode(self, node): - self.put(node.name) + def visit_IntNode(self, node): + self.put(node.value) + + def visit_FloatNode(self, node): + self.put(node.value) def visit_NoneNode(self, node): self.put(u"None") + def visit_NameNode(self, node): + self.put(node.name) + def visit_EllipsisNode(self, node): self.put(u"...") @@ -814,3 +780,38 @@ class ExpressionWriter(TreeVisitor): # type(body) is Nodes.ExprStatNode body = body.expr.arg self.emit_comprehension(body, target, sequence, condition, u"()") + + +class PxdWriter(DeclarationWriter, ExpressionWriter): + """ + A Cython code writer for everything supported in pxd files. + (currently unused) + """ + + def __call__(self, node): + print(u'\n'.join(self.write(node).lines)) + return node + + def visit_CFuncDefNode(self, node): + if node.overridable: + self.startline(u'cpdef ') + else: + self.startline(u'cdef ') + if node.modifiers: + self.put(' '.join(node.modifiers)) + self.put(' ') + if node.visibility != 'private': + self.put(node.visibility) + self.put(u' ') + if node.api: + self.put(u'api ') + self.visit(node.declarator) + + def visit_StatNode(self, node): + pass + + +class CodeWriter(StatementWriter, ExpressionWriter): + """ + A complete Cython code writer. + """ diff --git a/Cython/Compiler/AnalysedTreeTransforms.py b/Cython/Compiler/AnalysedTreeTransforms.py index 07bf31f3e..d4941606e 100644 --- a/Cython/Compiler/AnalysedTreeTransforms.py +++ b/Cython/Compiler/AnalysedTreeTransforms.py @@ -10,9 +10,9 @@ from . import Symtab class AutoTestDictTransform(ScopeTrackingTransform): # Handles autotestdict directive - blacklist = ['__cinit__', '__dealloc__', '__richcmp__', - '__nonzero__', '__bool__', - '__len__', '__contains__'] + excludelist = ['__cinit__', '__dealloc__', '__richcmp__', + '__nonzero__', '__bool__', + '__len__', '__contains__'] def visit_ModuleNode(self, node): if node.is_pxd: @@ -81,7 +81,7 @@ class AutoTestDictTransform(ScopeTrackingTransform): name = node.entry.name else: name = node.name - if self.scope_type == 'cclass' and name in self.blacklist: + if self.scope_type == 'cclass' and name in self.excludelist: return node if self.scope_type == 'pyclass': class_name = self.scope_node.name diff --git a/Cython/Compiler/Annotate.py b/Cython/Compiler/Annotate.py index 5feac02d8..67e4662a8 100644 --- a/Cython/Compiler/Annotate.py +++ b/Cython/Compiler/Annotate.py @@ -23,8 +23,12 @@ from .. import Utils class AnnotationCCodeWriter(CCodeWriter): - def __init__(self, create_from=None, buffer=None, copy_formatting=True): + # also used as marker for detection of complete code emission in tests + COMPLETE_CODE_TITLE = "Complete cythonized code" + + def __init__(self, create_from=None, buffer=None, copy_formatting=True, show_entire_c_code=False, source_desc=None): CCodeWriter.__init__(self, create_from, buffer, copy_formatting=copy_formatting) + self.show_entire_c_code = show_entire_c_code if create_from is None: self.annotation_buffer = StringIO() self.last_annotated_pos = None @@ -198,17 +202,24 @@ class AnnotationCCodeWriter(CCodeWriter): for line in coverage_data.iterfind('lines/line') ) - def _htmlify_code(self, code): + def _htmlify_code(self, code, language): try: from pygments import highlight - from pygments.lexers import CythonLexer + from pygments.lexers import CythonLexer, CppLexer from pygments.formatters import HtmlFormatter except ImportError: # no Pygments, just escape the code return html_escape(code) + if language == "cython": + lexer = CythonLexer(stripnl=False, stripall=False) + elif language == "c/cpp": + lexer = CppLexer(stripnl=False, stripall=False) + else: + # unknown language, use fallback + return html_escape(code) html_code = highlight( - code, CythonLexer(stripnl=False, stripall=False), + code, lexer, HtmlFormatter(nowrap=True)) return html_code @@ -228,7 +239,7 @@ class AnnotationCCodeWriter(CCodeWriter): return u"<span class='%s'>%s</span>" % ( group_name, match.group(group_name)) - lines = self._htmlify_code(cython_code).splitlines() + lines = self._htmlify_code(cython_code, "cython").splitlines() lineno_width = len(str(len(lines))) if not covered_lines: covered_lines = None @@ -279,6 +290,19 @@ class AnnotationCCodeWriter(CCodeWriter): outlist.append(u"<pre class='cython code score-{score} {covered}'>{code}</pre>".format( score=score, covered=covered, code=c_code)) outlist.append(u"</div>") + + # now the whole c-code if needed: + if self.show_entire_c_code: + outlist.append(u'<p><div class="cython">') + onclick_title = u"<pre class='cython line'{onclick}>+ {title}</pre>\n" + outlist.append(onclick_title.format( + onclick=self._onclick_attr, + title=AnnotationCCodeWriter.COMPLETE_CODE_TITLE, + )) + complete_code_as_html = self._htmlify_code(self.buffer.getvalue(), "c/cpp") + outlist.append(u"<pre class='cython code'>{code}</pre>".format(code=complete_code_as_html)) + outlist.append(u"</div></p>") + return outlist diff --git a/Cython/Compiler/AutoDocTransforms.py b/Cython/Compiler/AutoDocTransforms.py index d3c0a1d0d..6c342f7ef 100644 --- a/Cython/Compiler/AutoDocTransforms.py +++ b/Cython/Compiler/AutoDocTransforms.py @@ -3,18 +3,48 @@ from __future__ import absolute_import, print_function from .Visitor import CythonTransform from .StringEncoding import EncodedString from . import Options -from . import PyrexTypes, ExprNodes +from . import PyrexTypes from ..CodeWriter import ExpressionWriter +from .Errors import warning class AnnotationWriter(ExpressionWriter): + """ + A Cython code writer for Python expressions in argument/variable annotations. + """ + def __init__(self, description=None): + """description is optional. If specified it is used in + warning messages for the nodes that don't convert to string properly. + If not specified then no messages are generated. + """ + ExpressionWriter.__init__(self) + self.description = description + self.incomplete = False def visit_Node(self, node): self.put(u"<???>") + self.incomplete = True + if self.description: + warning(node.pos, + "Failed to convert code to string representation in {0}".format( + self.description), level=1) def visit_LambdaNode(self, node): # XXX Should we do better? self.put("<lambda>") + self.incomplete = True + if self.description: + warning(node.pos, + "Failed to convert lambda to string representation in {0}".format( + self.description), level=1) + + def visit_UnicodeNode(self, node): + # Discard Unicode prefix in annotations. Any tool looking at them + # would probably expect Py3 string semantics. + self.emit_string(node, "") + + def visit_AnnotationNode(self, node): + self.put(node.string.unicode_value) class EmbedSignature(CythonTransform): @@ -25,6 +55,12 @@ class EmbedSignature(CythonTransform): self.class_node = None def _fmt_expr(self, node): + writer = ExpressionWriter() + result = writer.write(node) + # print(type(node).__name__, '-->', result) + return result + + def _fmt_annotation(self, node): writer = AnnotationWriter() result = writer.write(node) # print(type(node).__name__, '-->', result) @@ -37,7 +73,7 @@ class EmbedSignature(CythonTransform): doc = arg.type.declaration_code(arg.name, for_display=1) if arg.annotation: - annotation = self._fmt_expr(arg.annotation) + annotation = self._fmt_annotation(arg.annotation) doc = doc + (': %s' % annotation) if arg.default: default = self._fmt_expr(arg.default) @@ -50,12 +86,12 @@ class EmbedSignature(CythonTransform): def _fmt_star_arg(self, arg): arg_doc = arg.name if arg.annotation: - annotation = self._fmt_expr(arg.annotation) + annotation = self._fmt_annotation(arg.annotation) arg_doc = arg_doc + (': %s' % annotation) return arg_doc def _fmt_arglist(self, args, - npargs=0, pargs=None, + npoargs=0, npargs=0, pargs=None, nkargs=0, kargs=None, hide_self=False): arglist = [] @@ -65,9 +101,11 @@ class EmbedSignature(CythonTransform): arglist.append(arg_doc) if pargs: arg_doc = self._fmt_star_arg(pargs) - arglist.insert(npargs, '*%s' % arg_doc) + arglist.insert(npargs + npoargs, '*%s' % arg_doc) elif nkargs: - arglist.insert(npargs, '*') + arglist.insert(npargs + npoargs, '*') + if npoargs: + arglist.insert(npoargs, '/') if kargs: arg_doc = self._fmt_star_arg(kargs) arglist.append('**%s' % arg_doc) @@ -80,12 +118,12 @@ class EmbedSignature(CythonTransform): return ret.declaration_code("", for_display=1) def _fmt_signature(self, cls_name, func_name, args, - npargs=0, pargs=None, + npoargs=0, npargs=0, pargs=None, nkargs=0, kargs=None, return_expr=None, return_type=None, hide_self=False): arglist = self._fmt_arglist(args, - npargs, pargs, + npoargs, npargs, pargs, nkargs, kargs, hide_self=hide_self) arglist_doc = ', '.join(arglist) @@ -94,7 +132,7 @@ class EmbedSignature(CythonTransform): func_doc = '%s.%s' % (cls_name, func_doc) ret_doc = None if return_expr: - ret_doc = self._fmt_expr(return_expr) + ret_doc = self._fmt_annotation(return_expr) elif return_type: ret_doc = self._fmt_ret_type(return_type) if ret_doc: @@ -147,11 +185,12 @@ class EmbedSignature(CythonTransform): else: class_name, func_name = self.class_name, node.name + npoargs = getattr(node, 'num_posonly_args', 0) nkargs = getattr(node, 'num_kwonly_args', 0) - npargs = len(node.args) - nkargs + npargs = len(node.args) - nkargs - npoargs signature = self._fmt_signature( class_name, func_name, node.args, - npargs, node.star_arg, + npoargs, npargs, node.star_arg, nkargs, node.starstar_arg, return_expr=node.return_type_annotation, return_type=None, hide_self=hide_self) @@ -176,7 +215,7 @@ class EmbedSignature(CythonTransform): def visit_CFuncDefNode(self, node): if not self.current_directives['embedsignature']: return node - if not node.overridable: # not cpdef FOO(...): + if not node.overridable: # not cpdef FOO(...): return node signature = self._fmt_signature( @@ -192,8 +231,9 @@ class EmbedSignature(CythonTransform): old_doc = None new_doc = self._embed_signature(signature, old_doc) node.entry.doc = EncodedString(new_doc) - if hasattr(node, 'py_func') and node.py_func is not None: - node.py_func.entry.doc = EncodedString(new_doc) + py_func = getattr(node, 'py_func', None) + if py_func is not None: + py_func.entry.doc = EncodedString(new_doc) return node def visit_PropertyNode(self, node): diff --git a/Cython/Compiler/Buffer.py b/Cython/Compiler/Buffer.py index c62a24f56..ce8503626 100644 --- a/Cython/Compiler/Buffer.py +++ b/Cython/Compiler/Buffer.py @@ -85,7 +85,7 @@ class IntroduceBufferAuxiliaryVars(CythonTransform): aux_var = scope.declare_var(name=None, cname=cname, type=type, pos=node.pos) if entry.is_arg: - aux_var.used = True # otherwise, NameNode will mark whether it is used + aux_var.used = True # otherwise, NameNode will mark whether it is used return aux_var @@ -111,9 +111,9 @@ class IntroduceBufferAuxiliaryVars(CythonTransform): # # Analysis # -buffer_options = ("dtype", "ndim", "mode", "negative_indices", "cast") # ordered! +buffer_options = ("dtype", "ndim", "mode", "negative_indices", "cast") # ordered! buffer_defaults = {"ndim": 1, "mode": "full", "negative_indices": True, "cast": False} -buffer_positional_options_count = 1 # anything beyond this needs keyword argument +buffer_positional_options_count = 1 # anything beyond this needs keyword argument ERR_BUF_OPTION_UNKNOWN = '"%s" is not a buffer option' ERR_BUF_TOO_MANY = 'Too many buffer options' @@ -146,12 +146,12 @@ def analyse_buffer_options(globalpos, env, posargs, dictargs, defaults=None, nee options = {} for name, (value, pos) in dictargs.items(): - if not name in buffer_options: + if name not in buffer_options: raise CompileError(pos, ERR_BUF_OPTION_UNKNOWN % name) options[name] = value for name, (value, pos) in zip(buffer_options, posargs): - if not name in buffer_options: + if name not in buffer_options: raise CompileError(pos, ERR_BUF_OPTION_UNKNOWN % name) if name in options: raise CompileError(pos, ERR_BUF_DUP % name) @@ -159,7 +159,7 @@ def analyse_buffer_options(globalpos, env, posargs, dictargs, defaults=None, nee # Check that they are all there and copy defaults for name in buffer_options: - if not name in options: + if name not in options: try: options[name] = defaults[name] except KeyError: @@ -298,9 +298,10 @@ def put_unpack_buffer_aux_into_scope(buf_entry, code): ln = [] for i in range(buf_entry.type.ndim): for fldname in fldnames: - ln.append("%s.diminfo[%d].%s = %s.rcbuffer->pybuffer.%s[%d];" % \ - (pybuffernd_struct, i, fldname, - pybuffernd_struct, fldname, i)) + ln.append("%s.diminfo[%d].%s = %s.rcbuffer->pybuffer.%s[%d];" % ( + pybuffernd_struct, i, fldname, + pybuffernd_struct, fldname, i, + )) code.putln(' '.join(ln)) def put_init_vars(entry, code): @@ -373,7 +374,7 @@ def put_assign_to_buffer(lhs_cname, rhs_cname, buf_entry, code.putln("{") # Set up necessary stack for getbuffer code.putln("__Pyx_BufFmt_StackElem __pyx_stack[%d];" % buffer_type.dtype.struct_nesting_depth()) - getbuffer = get_getbuffer_call(code, "%s", buffer_aux, buffer_type) # fill in object below + getbuffer = get_getbuffer_call(code, "%s", buffer_aux, buffer_type) # fill in object below if is_initialized: # Release any existing buffer @@ -419,7 +420,7 @@ def put_assign_to_buffer(lhs_cname, rhs_cname, buf_entry, put_unpack_buffer_aux_into_scope(buf_entry, code) code.putln('}') - code.putln("}") # Release stack + code.putln("}") # Release stack def put_buffer_lookup_code(entry, index_signeds, index_cnames, directives, @@ -669,8 +670,8 @@ def get_type_information_cname(code, dtype, maxdepth=None): structinfo_name = "NULL" elif dtype.is_struct: struct_scope = dtype.scope - if dtype.is_const: - struct_scope = struct_scope.const_base_type_scope + if dtype.is_cv_qualified: + struct_scope = struct_scope.base_type_scope # Must pre-call all used types in order not to recurse during utility code writing. fields = struct_scope.var_entries assert len(fields) > 0 diff --git a/Cython/Compiler/Builtin.py b/Cython/Compiler/Builtin.py index 5fa717507..4e606b2d9 100644 --- a/Cython/Compiler/Builtin.py +++ b/Cython/Compiler/Builtin.py @@ -30,17 +30,19 @@ builtin_utility_code = { class _BuiltinOverride(object): def __init__(self, py_name, args, ret_type, cname, py_equiv="*", utility_code=None, sig=None, func_type=None, - is_strict_signature=False, builtin_return_type=None): + is_strict_signature=False, builtin_return_type=None, + nogil=None): self.py_name, self.cname, self.py_equiv = py_name, cname, py_equiv self.args, self.ret_type = args, ret_type self.func_type, self.sig = func_type, sig self.builtin_return_type = builtin_return_type self.is_strict_signature = is_strict_signature self.utility_code = utility_code + self.nogil = nogil def build_func_type(self, sig=None, self_arg=None): if sig is None: - sig = Signature(self.args, self.ret_type) + sig = Signature(self.args, self.ret_type, nogil=self.nogil) sig.exception_check = False # not needed for the current builtins func_type = sig.function_type(self_arg) if self.is_strict_signature: @@ -54,7 +56,7 @@ class BuiltinAttribute(object): def __init__(self, py_name, cname=None, field_type=None, field_type_name=None): self.py_name = py_name self.cname = cname or py_name - self.field_type_name = field_type_name # can't do the lookup before the type is declared! + self.field_type_name = field_type_name # can't do the lookup before the type is declared! self.field_type = field_type def declare_in_type(self, self_type): @@ -92,13 +94,13 @@ class BuiltinMethod(_BuiltinOverride): builtin_function_table = [ # name, args, return, C API func, py equiv = "*" BuiltinFunction('abs', "d", "d", "fabs", - is_strict_signature = True), + is_strict_signature=True, nogil=True), BuiltinFunction('abs', "f", "f", "fabsf", - is_strict_signature = True), + is_strict_signature=True, nogil=True), BuiltinFunction('abs', "i", "i", "abs", - is_strict_signature = True), + is_strict_signature=True, nogil=True), BuiltinFunction('abs', "l", "l", "labs", - is_strict_signature = True), + is_strict_signature=True, nogil=True), BuiltinFunction('abs', None, None, "__Pyx_abs_longlong", utility_code = UtilityCode.load("abs_longlong", "Builtins.c"), func_type = PyrexTypes.CFuncType( @@ -209,7 +211,7 @@ builtin_function_table = [ #('sum', "", "", ""), #('sorted', "", "", ""), #('type', "O", "O", "PyObject_Type"), - #('unichr', "", "", ""), + BuiltinFunction('unichr', "l", "O", "PyUnicode_FromOrdinal", builtin_return_type='unicode'), #('unicode', "", "", ""), #('vars', "", "", ""), #('zip', "", "", ""), @@ -344,15 +346,15 @@ builtin_types_table = [ ] -types_that_construct_their_instance = set([ +types_that_construct_their_instance = frozenset({ # some builtin types do not always return an instance of # themselves - these do: 'type', 'bool', 'long', 'float', 'complex', 'bytes', 'unicode', 'bytearray', - 'tuple', 'list', 'dict', 'set', 'frozenset' + 'tuple', 'list', 'dict', 'set', 'frozenset', # 'str', # only in Py3.x # 'file', # only in Py2.x -]) +}) builtin_structs_table = [ @@ -428,7 +430,7 @@ def init_builtins(): global list_type, tuple_type, dict_type, set_type, frozenset_type global bytes_type, str_type, unicode_type, basestring_type, slice_type - global float_type, bool_type, type_type, complex_type, bytearray_type + global float_type, long_type, bool_type, type_type, complex_type, bytearray_type type_type = builtin_scope.lookup('type').type list_type = builtin_scope.lookup('list').type tuple_type = builtin_scope.lookup('tuple').type @@ -442,6 +444,7 @@ def init_builtins(): basestring_type = builtin_scope.lookup('basestring').type bytearray_type = builtin_scope.lookup('bytearray').type float_type = builtin_scope.lookup('float').type + long_type = builtin_scope.lookup('long').type bool_type = builtin_scope.lookup('bool').type complex_type = builtin_scope.lookup('complex').type diff --git a/Cython/Compiler/CmdLine.py b/Cython/Compiler/CmdLine.py index e89e45ab4..ffff6a61c 100644 --- a/Cython/Compiler/CmdLine.py +++ b/Cython/Compiler/CmdLine.py @@ -5,220 +5,222 @@ from __future__ import absolute_import import os -import sys +from argparse import ArgumentParser, Action, SUPPRESS from . import Options -usage = """\ -Cython (http://cython.org) is a compiler for code written in the -Cython language. Cython is based on Pyrex by Greg Ewing. - -Usage: cython [options] sourcefile.{pyx,py} ... - -Options: - -V, --version Display version number of cython compiler - -l, --create-listing Write error messages to a listing file - -I, --include-dir <directory> Search for include files in named directory - (multiple include directories are allowed). - -o, --output-file <filename> Specify name of generated C file - -t, --timestamps Only compile newer source files - -f, --force Compile all source files (overrides implied -t) - -v, --verbose Be verbose, print file names on multiple compilation - -p, --embed-positions If specified, the positions in Cython files of each - function definition is embedded in its docstring. - --cleanup <level> Release interned objects on python exit, for memory debugging. - Level indicates aggressiveness, default 0 releases nothing. - -w, --working <directory> Sets the working directory for Cython (the directory modules - are searched from) - --gdb Output debug information for cygdb - --gdb-outdir <directory> Specify gdb debug information output directory. Implies --gdb. - - -D, --no-docstrings Strip docstrings from the compiled module. - -a, --annotate Produce a colorized HTML version of the source. - --annotate-coverage <cov.xml> Annotate and include coverage information from cov.xml. - --line-directives Produce #line directives pointing to the .pyx source - --cplus Output a C++ rather than C file. - --embed[=<method_name>] Generate a main() function that embeds the Python interpreter. - -2 Compile based on Python-2 syntax and code semantics. - -3 Compile based on Python-3 syntax and code semantics. - --3str Compile based on Python-3 syntax and code semantics without - assuming unicode by default for string literals under Python 2. - --lenient Change some compile time errors to runtime errors to - improve Python compatibility - --capi-reexport-cincludes Add cincluded headers to any auto-generated header files. - --fast-fail Abort the compilation on the first error - --warning-errors, -Werror Make all warnings into errors - --warning-extra, -Wextra Enable extra warnings - -X, --directive <name>=<value>[,<name=value,...] Overrides a compiler directive - -E, --compile-time-env name=value[,<name=value,...] Provides compile time env like DEF would do. -""" - - -# The following experimental options are supported only on MacOSX: -# -C, --compile Compile generated .c file to .o file -# --link Link .o file to produce extension module (implies -C) -# -+, --cplus Use C++ compiler for compiling and linking -# Additional .o files to link may be supplied when using -X.""" - -def bad_usage(): - sys.stderr.write(usage) - sys.exit(1) +class ParseDirectivesAction(Action): + def __call__(self, parser, namespace, values, option_string=None): + old_directives = dict(getattr(namespace, self.dest, + Options.get_directive_defaults())) + directives = Options.parse_directive_list( + values, relaxed_bool=True, current_settings=old_directives) + setattr(namespace, self.dest, directives) -def parse_command_line(args): - from .Main import CompilationOptions, default_options - - pending_arg = [] - - def pop_arg(): - if not args or pending_arg: - bad_usage() - if '=' in args[0] and args[0].startswith('--'): # allow "--long-option=xyz" - name, value = args.pop(0).split('=', 1) - pending_arg.append(value) - return name - return args.pop(0) - - def pop_value(default=None): - if pending_arg: - return pending_arg.pop() - elif default is not None: - return default - elif not args: - bad_usage() - return args.pop(0) - - def get_param(option): - tail = option[2:] - if tail: - return tail - else: - return pop_arg() - - options = CompilationOptions(default_options) - sources = [] - while args: - if args[0].startswith("-"): - option = pop_arg() - if option in ("-V", "--version"): - options.show_version = 1 - elif option in ("-l", "--create-listing"): - options.use_listing_file = 1 - elif option in ("-+", "--cplus"): - options.cplus = 1 - elif option == "--embed": - Options.embed = pop_value("main") - elif option.startswith("-I"): - options.include_path.append(get_param(option)) - elif option == "--include-dir": - options.include_path.append(pop_value()) - elif option in ("-w", "--working"): - options.working_path = pop_value() - elif option in ("-o", "--output-file"): - options.output_file = pop_value() - elif option in ("-t", "--timestamps"): - options.timestamps = 1 - elif option in ("-f", "--force"): - options.timestamps = 0 - elif option in ("-v", "--verbose"): - options.verbose += 1 - elif option in ("-p", "--embed-positions"): - Options.embed_pos_in_docstring = 1 - elif option in ("-z", "--pre-import"): - Options.pre_import = pop_value() - elif option == "--cleanup": - Options.generate_cleanup_code = int(pop_value()) - elif option in ("-D", "--no-docstrings"): - Options.docstrings = False - elif option in ("-a", "--annotate"): - Options.annotate = True - elif option == "--annotate-coverage": - Options.annotate = True - Options.annotate_coverage_xml = pop_value() - elif option == "--convert-range": - Options.convert_range = True - elif option == "--line-directives": - options.emit_linenums = True - elif option == "--no-c-in-traceback": - options.c_line_in_traceback = False - elif option == "--gdb": - options.gdb_debug = True - options.output_dir = os.curdir - elif option == "--gdb-outdir": - options.gdb_debug = True - options.output_dir = pop_value() - elif option == "--lenient": - Options.error_on_unknown_names = False - Options.error_on_uninitialized = False - elif option == '-2': - options.language_level = 2 - elif option == '-3': - options.language_level = 3 - elif option == '--3str': - options.language_level = '3str' - elif option == "--capi-reexport-cincludes": - options.capi_reexport_cincludes = True - elif option == "--fast-fail": - Options.fast_fail = True - elif option == "--cimport-from-pyx": - Options.cimport_from_pyx = True - elif option in ('-Werror', '--warning-errors'): - Options.warning_errors = True - elif option in ('-Wextra', '--warning-extra'): - options.compiler_directives.update(Options.extra_warnings) - elif option == "--old-style-globals": - Options.old_style_globals = True - elif option == "--directive" or option.startswith('-X'): - if option.startswith('-X') and option[2:].strip(): - x_args = option[2:] - else: - x_args = pop_value() - try: - options.compiler_directives = Options.parse_directive_list( - x_args, relaxed_bool=True, - current_settings=options.compiler_directives) - except ValueError as e: - sys.stderr.write("Error in compiler directive: %s\n" % e.args[0]) - sys.exit(1) - elif option == "--compile-time-env" or option.startswith('-E'): - if option.startswith('-E') and option[2:].strip(): - x_args = option[2:] - else: - x_args = pop_value() - try: - options.compile_time_env = Options.parse_compile_time_env( - x_args, current_settings=options.compile_time_env) - except ValueError as e: - sys.stderr.write("Error in compile-time-env: %s\n" % e.args[0]) - sys.exit(1) - elif option.startswith('--debug'): - option = option[2:].replace('-', '_') - from . import DebugFlags - if option in dir(DebugFlags): - setattr(DebugFlags, option, True) - else: - sys.stderr.write("Unknown debug flag: %s\n" % option) - bad_usage() - elif option in ('-h', '--help'): - sys.stdout.write(usage) - sys.exit(0) + +class ParseOptionsAction(Action): + def __call__(self, parser, namespace, values, option_string=None): + options = dict(getattr(namespace, self.dest, {})) + for opt in values.split(','): + if '=' in opt: + n, v = opt.split('=', 1) + v = v.lower() not in ('false', 'f', '0', 'no') + else: + n, v = opt, True + options[n] = v + setattr(namespace, self.dest, options) + + +class ParseCompileTimeEnvAction(Action): + def __call__(self, parser, namespace, values, option_string=None): + old_env = dict(getattr(namespace, self.dest, {})) + new_env = Options.parse_compile_time_env(values, current_settings=old_env) + setattr(namespace, self.dest, new_env) + + +class ActivateAllWarningsAction(Action): + def __call__(self, parser, namespace, values, option_string=None): + directives = getattr(namespace, 'compiler_directives', {}) + directives.update(Options.extra_warnings) + namespace.compiler_directives = directives + + +class SetLenientAction(Action): + def __call__(self, parser, namespace, values, option_string=None): + namespace.error_on_unknown_names = False + namespace.error_on_uninitialized = False + + +class SetGDBDebugAction(Action): + def __call__(self, parser, namespace, values, option_string=None): + namespace.gdb_debug = True + namespace.output_dir = os.curdir + + +class SetGDBDebugOutputAction(Action): + def __call__(self, parser, namespace, values, option_string=None): + namespace.gdb_debug = True + namespace.output_dir = values + + +class SetAnnotateCoverageAction(Action): + def __call__(self, parser, namespace, values, option_string=None): + namespace.annotate = True + namespace.annotate_coverage_xml = values + + +def create_cython_argparser(): + description = "Cython (https://cython.org/) is a compiler for code written in the "\ + "Cython language. Cython is based on Pyrex by Greg Ewing." + + parser = ArgumentParser(description=description, argument_default=SUPPRESS) + + parser.add_argument("-V", "--version", dest='show_version', action='store_const', const=1, + help='Display version number of cython compiler') + parser.add_argument("-l", "--create-listing", dest='use_listing_file', action='store_const', const=1, + help='Write error messages to a listing file') + parser.add_argument("-I", "--include-dir", dest='include_path', action='append', + help='Search for include files in named directory ' + '(multiple include directories are allowed).') + parser.add_argument("-o", "--output-file", dest='output_file', action='store', type=str, + help='Specify name of generated C file') + parser.add_argument("-t", "--timestamps", dest='timestamps', action='store_const', const=1, + help='Only compile newer source files') + parser.add_argument("-f", "--force", dest='timestamps', action='store_const', const=0, + help='Compile all source files (overrides implied -t)') + parser.add_argument("-v", "--verbose", dest='verbose', action='count', + help='Be verbose, print file names on multiple compilation') + parser.add_argument("-p", "--embed-positions", dest='embed_pos_in_docstring', action='store_const', const=1, + help='If specified, the positions in Cython files of each ' + 'function definition is embedded in its docstring.') + parser.add_argument("--cleanup", dest='generate_cleanup_code', action='store', type=int, + help='Release interned objects on python exit, for memory debugging. ' + 'Level indicates aggressiveness, default 0 releases nothing.') + parser.add_argument("-w", "--working", dest='working_path', action='store', type=str, + help='Sets the working directory for Cython (the directory modules are searched from)') + parser.add_argument("--gdb", action=SetGDBDebugAction, nargs=0, + help='Output debug information for cygdb') + parser.add_argument("--gdb-outdir", action=SetGDBDebugOutputAction, type=str, + help='Specify gdb debug information output directory. Implies --gdb.') + parser.add_argument("-D", "--no-docstrings", dest='docstrings', action='store_false', + help='Strip docstrings from the compiled module.') + parser.add_argument('-a', '--annotate', action='store_const', const='default', dest='annotate', + help='Produce a colorized HTML version of the source.') + parser.add_argument('--annotate-fullc', action='store_const', const='fullc', dest='annotate', + help='Produce a colorized HTML version of the source ' + 'which includes entire generated C/C++-code.') + parser.add_argument("--annotate-coverage", dest='annotate_coverage_xml', action=SetAnnotateCoverageAction, type=str, + help='Annotate and include coverage information from cov.xml.') + parser.add_argument("--line-directives", dest='emit_linenums', action='store_true', + help='Produce #line directives pointing to the .pyx source') + parser.add_argument("-+", "--cplus", dest='cplus', action='store_const', const=1, + help='Output a C++ rather than C file.') + parser.add_argument('--embed', action='store_const', const='main', + help='Generate a main() function that embeds the Python interpreter. ' + 'Pass --embed=<method_name> for a name other than main().') + parser.add_argument('-2', dest='language_level', action='store_const', const=2, + help='Compile based on Python-2 syntax and code semantics.') + parser.add_argument('-3', dest='language_level', action='store_const', const=3, + help='Compile based on Python-3 syntax and code semantics.') + parser.add_argument('--3str', dest='language_level', action='store_const', const='3str', + help='Compile based on Python-3 syntax and code semantics without ' + 'assuming unicode by default for string literals under Python 2.') + parser.add_argument("--lenient", action=SetLenientAction, nargs=0, + help='Change some compile time errors to runtime errors to ' + 'improve Python compatibility') + parser.add_argument("--capi-reexport-cincludes", dest='capi_reexport_cincludes', action='store_true', + help='Add cincluded headers to any auto-generated header files.') + parser.add_argument("--fast-fail", dest='fast_fail', action='store_true', + help='Abort the compilation on the first error') + parser.add_argument("-Werror", "--warning-errors", dest='warning_errors', action='store_true', + help='Make all warnings into errors') + parser.add_argument("-Wextra", "--warning-extra", action=ActivateAllWarningsAction, nargs=0, + help='Enable extra warnings') + + parser.add_argument('-X', '--directive', metavar='NAME=VALUE,...', + dest='compiler_directives', type=str, + action=ParseDirectivesAction, + help='Overrides a compiler directive') + parser.add_argument('-E', '--compile-time-env', metavar='NAME=VALUE,...', + dest='compile_time_env', type=str, + action=ParseCompileTimeEnvAction, + help='Provides compile time env like DEF would do.') + parser.add_argument('sources', nargs='*', default=[]) + + # TODO: add help + parser.add_argument("-z", "--pre-import", dest='pre_import', action='store', type=str, help=SUPPRESS) + parser.add_argument("--convert-range", dest='convert_range', action='store_true', help=SUPPRESS) + parser.add_argument("--no-c-in-traceback", dest='c_line_in_traceback', action='store_false', help=SUPPRESS) + parser.add_argument("--cimport-from-pyx", dest='cimport_from_pyx', action='store_true', help=SUPPRESS) + parser.add_argument("--old-style-globals", dest='old_style_globals', action='store_true', help=SUPPRESS) + + # debug stuff: + from . import DebugFlags + for name in vars(DebugFlags): + if name.startswith("debug"): + option_name = name.replace('_', '-') + parser.add_argument("--" + option_name, action='store_true', help=SUPPRESS) + + return parser + + +def parse_command_line_raw(parser, args): + # special handling for --embed and --embed=xxxx as they aren't correctly parsed + def filter_out_embed_options(args): + with_embed, without_embed = [], [] + for x in args: + if x == '--embed' or x.startswith('--embed='): + with_embed.append(x) else: - sys.stderr.write("Unknown compiler flag: %s\n" % option) - sys.exit(1) + without_embed.append(x) + return with_embed, without_embed + + with_embed, args_without_embed = filter_out_embed_options(args) + + arguments, unknown = parser.parse_known_args(args_without_embed) + + sources = arguments.sources + del arguments.sources + + # unknown can be either debug, embed or input files or really unknown + for option in unknown: + if option.startswith('-'): + parser.error("unknown option " + option) else: - sources.append(pop_arg()) + sources.append(option) - if pending_arg: - bad_usage() + # embed-stuff must be handled extra: + for x in with_embed: + if x == '--embed': + name = 'main' # default value + else: + name = x[len('--embed='):] + setattr(arguments, 'embed', name) + + return arguments, sources + + +def parse_command_line(args): + parser = create_cython_argparser() + arguments, sources = parse_command_line_raw(parser, args) + + options = Options.CompilationOptions(Options.default_options) + for name, value in vars(arguments).items(): + if name.startswith('debug'): + from . import DebugFlags + if name in dir(DebugFlags): + setattr(DebugFlags, name, value) + else: + parser.error("Unknown debug flag: %s\n" % name) + elif hasattr(Options, name): + setattr(Options, name, value) + else: + setattr(options, name, value) if options.use_listing_file and len(sources) > 1: - sys.stderr.write( - "cython: Only one source file allowed when using -o\n") - sys.exit(1) + parser.error("cython: Only one source file allowed when using -o\n") if len(sources) == 0 and not options.show_version: - bad_usage() + parser.error("cython: Need at least one source file\n") if Options.embed and len(sources) > 1: - sys.stderr.write( - "cython: Only one source file allowed when using -embed\n") - sys.exit(1) + parser.error("cython: Only one source file allowed when using -embed\n") return options, sources - diff --git a/Cython/Compiler/Code.pxd b/Cython/Compiler/Code.pxd index 01f7a71f5..3037a16ee 100644 --- a/Cython/Compiler/Code.pxd +++ b/Cython/Compiler/Code.pxd @@ -1,5 +1,4 @@ - -from __future__ import absolute_import +# cython: language_level=3 cimport cython from ..StringIOTree cimport StringIOTree diff --git a/Cython/Compiler/Code.py b/Cython/Compiler/Code.py index 26fb8a1cf..c7ee4d82d 100644 --- a/Cython/Compiler/Code.py +++ b/Cython/Compiler/Code.py @@ -1,4 +1,4 @@ -# cython: language_level = 2 +# cython: language_level=3str # cython: auto_pickle=False # # Code output module @@ -13,22 +13,17 @@ cython.declare(os=object, re=object, operator=object, textwrap=object, DebugFlags=object, basestring=object, defaultdict=object, closing=object, partial=object) +import hashlib +import operator import os import re import shutil -import sys -import operator import textwrap from string import Template from functools import partial from contextlib import closing from collections import defaultdict -try: - import hashlib -except ImportError: - import md5 as hashlib - from . import Naming from . import Options from . import DebugFlags @@ -43,8 +38,6 @@ try: except ImportError: from builtins import str as basestring -KEYWORDS_MUST_BE_BYTES = sys.version_info < (2, 7) - non_portable_builtins_map = { # builtins that have different names in different Python versions @@ -101,20 +94,18 @@ uncachable_builtins = [ '__build_class__', 'ascii', # might deserve an implementation in Cython #'exec', # implemented in Cython - ## - Py2.7+ - 'memoryview', ## - platform specific 'WindowsError', ## - others '_', # e.g. used by gettext ] -special_py_methods = set([ +special_py_methods = cython.declare(frozenset, frozenset(( '__cinit__', '__dealloc__', '__richcmp__', '__next__', '__await__', '__aiter__', '__anext__', '__getreadbuffer__', '__getwritebuffer__', '__getsegcount__', - '__getcharbuffer__', '__getbuffer__', '__releasebuffer__' -]) + '__getcharbuffer__', '__getbuffer__', '__releasebuffer__', +))) modifier_output_mapper = { 'inline': 'CYTHON_INLINE' @@ -203,6 +194,28 @@ def get_utility_dir(): Cython_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) return os.path.join(Cython_dir, "Utility") +read_utilities_hook = None +""" +Override the hook for reading a utilities file that contains code fragments used +by the codegen. + +The hook functions takes the path of the utilities file, and returns a list +of strings, one per line. + +The default behavior is to open a file relative to get_utility_dir(). +""" + +def read_utilities_from_utility_dir(path): + """ + Read all lines of the file at the provided path from a path relative + to get_utility_dir(). + """ + filename = os.path.join(get_utility_dir(), path) + with closing(Utils.open_source_file(filename, encoding='UTF-8')) as f: + return f.readlines() + +# by default, read utilities from the utility directory. +read_utilities_hook = read_utilities_from_utility_dir class UtilityCodeBase(object): """ @@ -224,6 +237,15 @@ class UtilityCodeBase(object): [definitions] + ##### MyUtility ##### + #@subsitute: tempita + + [requires tempita substitution + - context can't be specified here though so only + tempita utility that requires no external context + will benefit from this tag + - only necessary when @required from non-tempita code] + for prototypes and implementation respectively. For non-python or -cython files backslashes should be used instead. 5 to 30 comment characters may be used on either side. @@ -242,8 +264,7 @@ class UtilityCodeBase(object): return code = '\n'.join(lines) - if tags and 'substitute' in tags and tags['substitute'] == set(['naming']): - del tags['substitute'] + if tags and 'substitute' in tags and 'naming' in tags['substitute']: try: code = Template(code).substitute(vars(Naming)) except (KeyError, ValueError) as e: @@ -259,15 +280,11 @@ class UtilityCodeBase(object): utility[1] = code else: all_tags = utility[2] - if KEYWORDS_MUST_BE_BYTES: - type = type.encode('ASCII') all_tags[type] = code if tags: all_tags = utility[2] for name, values in tags.items(): - if KEYWORDS_MUST_BE_BYTES: - name = name.encode('ASCII') all_tags.setdefault(name, set()).update(values) @classmethod @@ -276,7 +293,6 @@ class UtilityCodeBase(object): if utilities: return utilities - filename = os.path.join(get_utility_dir(), path) _, ext = os.path.splitext(path) if ext in ('.pyx', '.py', '.pxd', '.pxi'): comment = '#' @@ -292,8 +308,7 @@ class UtilityCodeBase(object): {'C': comment}).match match_type = re.compile(r'(.+)[.](proto(?:[.]\S+)?|impl|init|cleanup)$').match - with closing(Utils.open_source_file(filename, encoding='UTF-8')) as f: - all_lines = f.readlines() + all_lines = read_utilities_hook(path) utilities = defaultdict(lambda: [None, None, {}]) lines = [] @@ -335,43 +350,22 @@ class UtilityCodeBase(object): return utilities @classmethod - def load(cls, util_code_name, from_file=None, **kwargs): + def load(cls, util_code_name, from_file, **kwargs): """ Load utility code from a file specified by from_file (relative to - Cython/Utility) and name util_code_name. If from_file is not given, - load it from the file util_code_name.*. There should be only one - file matched by this pattern. + Cython/Utility) and name util_code_name. """ + if '::' in util_code_name: from_file, util_code_name = util_code_name.rsplit('::', 1) - if not from_file: - utility_dir = get_utility_dir() - prefix = util_code_name + '.' - try: - listing = os.listdir(utility_dir) - except OSError: - # XXX the code below assumes as 'zipimport.zipimporter' instance - # XXX should be easy to generalize, but too lazy right now to write it - import zipfile - global __loader__ - loader = __loader__ - archive = loader.archive - with closing(zipfile.ZipFile(archive)) as fileobj: - listing = [os.path.basename(name) - for name in fileobj.namelist() - if os.path.join(archive, name).startswith(utility_dir)] - files = [filename for filename in listing - if filename.startswith(prefix)] - if not files: - raise ValueError("No match found for utility code " + util_code_name) - if len(files) > 1: - raise ValueError("More than one filename match found for utility code " + util_code_name) - from_file = files[0] - + assert from_file utilities = cls.load_utilities_from_file(from_file) proto, impl, tags = utilities[util_code_name] if tags: + if "substitute" in tags and "tempita" in tags["substitute"]: + if not issubclass(cls, TempitaUtilityCode): + return TempitaUtilityCode.load(util_code_name, from_file, **kwargs) orig_kwargs = kwargs.copy() for name, values in tags.items(): if name in kwargs: @@ -385,6 +379,12 @@ class UtilityCodeBase(object): # dependencies are rarely unique, so use load_cached() when we can values = [cls.load_cached(dep, from_file) for dep in sorted(values)] + elif name == 'substitute': + # don't want to pass "naming" or "tempita" to the constructor + # since these will have been handled + values = values - {'naming', 'tempita'} + if not values: + continue elif not values: values = None elif len(values) == 1: @@ -404,11 +404,11 @@ class UtilityCodeBase(object): return cls(**kwargs) @classmethod - def load_cached(cls, utility_code_name, from_file=None, __cache={}): + def load_cached(cls, utility_code_name, from_file, __cache={}): """ Calls .load(), but using a per-type cache based on utility name and file name. """ - key = (cls, from_file, utility_code_name) + key = (utility_code_name, from_file, cls) try: return __cache[key] except KeyError: @@ -417,7 +417,7 @@ class UtilityCodeBase(object): return code @classmethod - def load_as_string(cls, util_code_name, from_file=None, **kwargs): + def load_as_string(cls, util_code_name, from_file, **kwargs): """ Load a utility code as a string. Returns (proto, implementation) """ @@ -842,8 +842,8 @@ class FunctionState(object): A C string referring to the variable is returned. """ - if type.is_const and not type.is_reference: - type = type.const_base_type + if type.is_cv_qualified and not type.is_reference: + type = type.cv_base_type elif type.is_reference and not type.is_fake_reference: type = type.ref_base_type elif type.is_cfunction: @@ -912,7 +912,7 @@ class FunctionState(object): """ return [(name, type) for name, type, manage_ref in self.temps_in_use() - if manage_ref and type.is_pyobject] + if manage_ref and type.is_pyobject] def all_managed_temps(self): """Return a list of (cname, type) tuples of refcount-managed Python objects. @@ -1117,10 +1117,10 @@ class GlobalState(object): 'h_code', 'filename_table', 'utility_code_proto_before_types', - 'numeric_typedefs', # Let these detailed individual parts stay!, - 'complex_type_declarations', # as the proper solution is to make a full DAG... - 'type_declarations', # More coarse-grained blocks would simply hide - 'utility_code_proto', # the ugliness, not fix it + 'numeric_typedefs', # Let these detailed individual parts stay!, + 'complex_type_declarations', # as the proper solution is to make a full DAG... + 'type_declarations', # More coarse-grained blocks would simply hide + 'utility_code_proto', # the ugliness, not fix it 'module_declarations', 'typeinfo', 'before_global_var', @@ -1128,7 +1128,11 @@ class GlobalState(object): 'string_decls', 'decls', 'late_includes', - 'all_the_rest', + 'module_state', + 'module_state_clear', + 'module_state_traverse', + 'module_state_defines', # redefines names used in module_state/_clear/_traverse + 'module_code', # user code goes here 'pystring_table', 'cached_builtins', 'cached_constants', @@ -1141,6 +1145,14 @@ class GlobalState(object): 'end' ] + # h files can only have a much smaller list of sections + h_code_layout = [ + 'h_code', + 'utility_code_proto_before_types', + 'type_declarations', + 'utility_code_proto', + 'end' + ] def __init__(self, writer, module_node, code_config, common_utility_include_dir=None): self.filename_table = {} @@ -1152,8 +1164,8 @@ class GlobalState(object): self.code_config = code_config self.common_utility_include_dir = common_utility_include_dir self.parts = {} - self.module_node = module_node # because some utility code generation needs it - # (generating backwards-compatible Get/ReleaseBuffer + self.module_node = module_node # because some utility code generation needs it + # (generating backwards-compatible Get/ReleaseBuffer self.const_cnames_used = {} self.string_const_index = {} @@ -1169,8 +1181,10 @@ class GlobalState(object): def initialize_main_c_code(self): rootwriter = self.rootwriter - for part in self.code_layout: - self.parts[part] = rootwriter.insertion_point() + for i, part in enumerate(self.code_layout): + w = self.parts[part] = rootwriter.insertion_point() + if i > 0: + w.putln("/* #### Code section: %s ### */" % part) if not Options.cache_builtins: del self.parts['cached_builtins'] @@ -1184,7 +1198,7 @@ class GlobalState(object): w.putln("") w.putln("static CYTHON_SMALL_CODE int __Pyx_InitCachedConstants(void) {") w.put_declare_refcount_context() - w.put_setup_refcount_context("__Pyx_InitCachedConstants") + w.put_setup_refcount_context(StringEncoding.EncodedString("__Pyx_InitCachedConstants")) w = self.parts['init_globals'] w.enter_cfunc_scope() @@ -1209,6 +1223,11 @@ class GlobalState(object): code.putln("") code.putln("/* --- Runtime support code --- */") + def initialize_main_h_code(self): + rootwriter = self.rootwriter + for part in self.h_code_layout: + self.parts[part] = rootwriter.insertion_point() + def finalize_main_c_code(self): self.close_global_decls() @@ -1457,9 +1476,18 @@ class GlobalState(object): for c in self.py_constants] consts.sort() decls_writer = self.parts['decls'] + decls_writer.putln("#if !CYTHON_COMPILING_IN_LIMITED_API") for _, cname, c in consts: + self.parts['module_state'].putln("%s;" % c.type.declaration_code(cname)) + self.parts['module_state_defines'].putln( + "#define %s %s->%s" % (cname, Naming.modulestateglobal_cname, cname)) + self.parts['module_state_clear'].putln( + "Py_CLEAR(clear_module_state->%s);" % cname) + self.parts['module_state_traverse'].putln( + "Py_VISIT(traverse_module_state->%s);" % cname) decls_writer.putln( "static %s;" % c.type.declaration_code(cname)) + decls_writer.putln("#endif") def generate_cached_methods_decls(self): if not self.cached_cmethods: @@ -1471,11 +1499,14 @@ class GlobalState(object): for (type_cname, method_name), cname in sorted(self.cached_cmethods.items()): cnames.append(cname) method_name_cname = self.get_interned_identifier(StringEncoding.EncodedString(method_name)).cname - decl.putln('static __Pyx_CachedCFunction %s = {0, &%s, 0, 0, 0};' % ( - cname, method_name_cname)) + decl.putln('static __Pyx_CachedCFunction %s = {0, 0, 0, 0, 0};' % ( + cname)) # split type reference storage as it might not be static init.putln('%s.type = (PyObject*)&%s;' % ( cname, type_cname)) + # method name string isn't static in limited api + init.putln('%s.method_name = &%s;' % ( + cname, method_name_cname)) if Options.generate_cleanup_code: cleanup = self.parts['cleanup_globals'] @@ -1513,13 +1544,26 @@ class GlobalState(object): decls_writer.putln("static Py_UNICODE %s[] = { %s };" % (cname, utf16_array)) decls_writer.putln("#endif") + init_globals = self.parts['init_globals'] if py_strings: self.use_utility_code(UtilityCode.load_cached("InitStrings", "StringTools.c")) py_strings.sort() w = self.parts['pystring_table'] w.putln("") w.putln("static __Pyx_StringTabEntry %s[] = {" % Naming.stringtab_cname) - for c_cname, _, py_string in py_strings: + w.putln("#if CYTHON_COMPILING_IN_LIMITED_API") + w_limited_writer = w.insertion_point() + w.putln("#else") + w_not_limited_writer = w.insertion_point() + w.putln("#endif") + decls_writer.putln("#if !CYTHON_COMPILING_IN_LIMITED_API") + not_limited_api_decls_writer = decls_writer.insertion_point() + decls_writer.putln("#endif") + init_globals.putln("#if CYTHON_COMPILING_IN_LIMITED_API") + init_globals_limited_api = init_globals.insertion_point() + init_globals.putln("#endif") + for idx, py_string_args in enumerate(py_strings): + c_cname, _, py_string = py_string_args if not py_string.is_str or not py_string.encoding or \ py_string.encoding in ('ASCII', 'USASCII', 'US-ASCII', 'UTF8', 'UTF-8'): @@ -1527,19 +1571,28 @@ class GlobalState(object): else: encoding = '"%s"' % py_string.encoding.lower() - decls_writer.putln( + self.parts['module_state'].putln("PyObject *%s;" % py_string.cname) + self.parts['module_state_defines'].putln("#define %s %s->%s" % ( + py_string.cname, + Naming.modulestateglobal_cname, + py_string.cname)) + self.parts['module_state_clear'].putln("Py_CLEAR(clear_module_state->%s);" % + py_string.cname) + self.parts['module_state_traverse'].putln("Py_VISIT(traverse_module_state->%s);" % + py_string.cname) + not_limited_api_decls_writer.putln( "static PyObject *%s;" % py_string.cname) if py_string.py3str_cstring: - w.putln("#if PY_MAJOR_VERSION >= 3") - w.putln("{&%s, %s, sizeof(%s), %s, %d, %d, %d}," % ( + w_not_limited_writer.putln("#if PY_MAJOR_VERSION >= 3") + w_not_limited_writer.putln("{&%s, %s, sizeof(%s), %s, %d, %d, %d}," % ( py_string.cname, py_string.py3str_cstring.cname, py_string.py3str_cstring.cname, '0', 1, 0, py_string.intern )) - w.putln("#else") - w.putln("{&%s, %s, sizeof(%s), %s, %d, %d, %d}," % ( + w_not_limited_writer.putln("#else") + w_not_limited_writer.putln("{&%s, %s, sizeof(%s), %s, %d, %d, %d}," % ( py_string.cname, c_cname, c_cname, @@ -1549,24 +1602,46 @@ class GlobalState(object): py_string.intern )) if py_string.py3str_cstring: - w.putln("#endif") + w_not_limited_writer.putln("#endif") + w_limited_writer.putln("{0, %s, sizeof(%s), %s, %d, %d, %d}," % ( + c_cname if not py_string.py3str_cstring else py_string.py3str_cstring.cname, + c_cname if not py_string.py3str_cstring else py_string.py3str_cstring.cname, + encoding if not py_string.py3str_cstring else '0', + py_string.is_unicode, + py_string.is_str, + py_string.intern + )) + init_globals_limited_api.putln("if (__Pyx_InitString(%s[%d], &%s) < 0) %s;" % ( + Naming.stringtab_cname, + idx, + py_string.cname, + init_globals.error_goto(self.module_pos))) w.putln("{0, 0, 0, 0, 0, 0, 0}") w.putln("};") - init_globals = self.parts['init_globals'] + init_globals.putln("#if !CYTHON_COMPILING_IN_LIMITED_API") init_globals.putln( "if (__Pyx_InitStrings(%s) < 0) %s;" % ( Naming.stringtab_cname, init_globals.error_goto(self.module_pos))) + init_globals.putln("#endif") def generate_num_constants(self): consts = [(c.py_type, c.value[0] == '-', len(c.value), c.value, c.value_code, c) for c in self.num_const_index.values()] consts.sort() decls_writer = self.parts['decls'] + decls_writer.putln("#if !CYTHON_COMPILING_IN_LIMITED_API") init_globals = self.parts['init_globals'] for py_type, _, _, value, value_code, c in consts: cname = c.cname + self.parts['module_state'].putln("PyObject *%s;" % cname) + self.parts['module_state_defines'].putln("#define %s %s->%s" % ( + cname, Naming.modulestateglobal_cname, cname)) + self.parts['module_state_clear'].putln( + "Py_CLEAR(clear_module_state->%s);" % cname) + self.parts['module_state_traverse'].putln( + "Py_VISIT(traverse_module_state->%s);" % cname) decls_writer.putln("static PyObject *%s;" % cname) if py_type == 'float': function = 'PyFloat_FromDouble(%s)' @@ -1581,6 +1656,7 @@ class GlobalState(object): init_globals.putln('%s = %s; %s' % ( cname, function % value_code, init_globals.error_goto_if_null(cname, self.module_pos))) + decls_writer.putln("#endif") # The functions below are there in a transition phase only # and will be deprecated. They are called from Nodes.BlockNode. @@ -1755,10 +1831,13 @@ class CCodeWriter(object): return self.buffer.getvalue() def write(self, s): - # also put invalid markers (lineno 0), to indicate that those lines - # have no Cython source code correspondence - cython_lineno = self.last_marked_pos[1] if self.last_marked_pos else 0 - self.buffer.markers.extend([cython_lineno] * s.count('\n')) + # Cygdb needs to know which Cython source line corresponds to which C line. + # Therefore, we write this information into "self.buffer.markers" and then write it from there + # into cython_debug/cython_debug_info_* (see ModuleNode._serialize_lineno_map). + + filename_line = self.last_marked_pos[:2] if self.last_marked_pos else (None, 0) + self.buffer.markers.extend([filename_line] * s.count('\n')) + self.buffer.write(s) def insertion_point(self): @@ -1818,6 +1897,7 @@ class CCodeWriter(object): self.funcstate = FunctionState(self, scope=scope) def exit_cfunc_scope(self): + self.funcstate.validate_exit() self.funcstate = None # constant handling @@ -1908,7 +1988,7 @@ class CCodeWriter(object): include_dir = self.globalstate.common_utility_include_dir if include_dir and len(code) > 1024: include_file = "%s_%s.h" % ( - name, hashlib.md5(code.encode('utf8')).hexdigest()) + name, hashlib.sha1(code.encode('utf8')).hexdigest()) path = os.path.join(include_dir, include_file) if not os.path.exists(path): tmp_path = '%s.tmp%s' % (path, os.getpid()) @@ -2004,8 +2084,7 @@ class CCodeWriter(object): if type.is_pyobject: self.putln("%s = NULL;" % decl) elif type.is_memoryviewslice: - from . import MemoryView - self.putln("%s = %s;" % (decl, MemoryView.memslice_entry_init)) + self.putln("%s = %s;" % (decl, type.literal_code(type.default_value))) else: self.putln("%s%s;" % (static and "static " or "", decl)) @@ -2043,7 +2122,7 @@ class CCodeWriter(object): def entry_as_pyobject(self, entry): type = entry.type if (not entry.is_self_arg and not entry.type.is_complete() - or entry.type.is_extension_type): + or entry.type.is_extension_type): return "(PyObject *)" + entry.cname else: return entry.cname @@ -2052,123 +2131,89 @@ class CCodeWriter(object): from .PyrexTypes import py_object_type, typecast return typecast(py_object_type, type, cname) - def put_gotref(self, cname): - self.putln("__Pyx_GOTREF(%s);" % cname) + def put_gotref(self, cname, type): + type.generate_gotref(self, cname) - def put_giveref(self, cname): - self.putln("__Pyx_GIVEREF(%s);" % cname) + def put_giveref(self, cname, type): + type.generate_giveref(self, cname) - def put_xgiveref(self, cname): - self.putln("__Pyx_XGIVEREF(%s);" % cname) + def put_xgiveref(self, cname, type): + type.generate_xgiveref(self, cname) - def put_xgotref(self, cname): - self.putln("__Pyx_XGOTREF(%s);" % cname) + def put_xgotref(self, cname, type): + type.generate_xgotref(self, cname) def put_incref(self, cname, type, nanny=True): - if nanny: - self.putln("__Pyx_INCREF(%s);" % self.as_pyobject(cname, type)) - else: - self.putln("Py_INCREF(%s);" % self.as_pyobject(cname, type)) + # Note: original put_Memslice_Incref/Decref also added in some utility code + # this is unnecessary since the relevant utility code is loaded anyway if a memoryview is used + # and so has been removed. However, it's potentially a feature that might be useful here + type.generate_incref(self, cname, nanny=nanny) - def put_decref(self, cname, type, nanny=True): - self._put_decref(cname, type, nanny, null_check=False, clear=False) + def put_xincref(self, cname, type, nanny=True): + type.generate_xincref(self, cname, nanny=nanny) - def put_var_gotref(self, entry): - if entry.type.is_pyobject: - self.putln("__Pyx_GOTREF(%s);" % self.entry_as_pyobject(entry)) + def put_decref(self, cname, type, nanny=True, have_gil=True): + type.generate_decref(self, cname, nanny=nanny, have_gil=have_gil) - def put_var_giveref(self, entry): - if entry.type.is_pyobject: - self.putln("__Pyx_GIVEREF(%s);" % self.entry_as_pyobject(entry)) + def put_xdecref(self, cname, type, nanny=True, have_gil=True): + type.generate_xdecref(self, cname, nanny=nanny, have_gil=have_gil) - def put_var_xgotref(self, entry): - if entry.type.is_pyobject: - self.putln("__Pyx_XGOTREF(%s);" % self.entry_as_pyobject(entry)) + def put_decref_clear(self, cname, type, clear_before_decref=False, nanny=True, have_gil=True): + type.generate_decref_clear(self, cname, clear_before_decref=clear_before_decref, + nanny=nanny, have_gil=have_gil) - def put_var_xgiveref(self, entry): - if entry.type.is_pyobject: - self.putln("__Pyx_XGIVEREF(%s);" % self.entry_as_pyobject(entry)) + def put_xdecref_clear(self, cname, type, clear_before_decref=False, nanny=True, have_gil=True): + type.generate_xdecref_clear(self, cname, clear_before_decref=clear_before_decref, + nanny=nanny, have_gil=have_gil) - def put_var_incref(self, entry, nanny=True): - if entry.type.is_pyobject: - if nanny: - self.putln("__Pyx_INCREF(%s);" % self.entry_as_pyobject(entry)) - else: - self.putln("Py_INCREF(%s);" % self.entry_as_pyobject(entry)) + def put_decref_set(self, cname, type, rhs_cname): + type.generate_decref_set(self, cname, rhs_cname) - def put_var_xincref(self, entry): - if entry.type.is_pyobject: - self.putln("__Pyx_XINCREF(%s);" % self.entry_as_pyobject(entry)) + def put_xdecref_set(self, cname, type, rhs_cname): + type.generate_xdecref_set(self, cname, rhs_cname) - def put_decref_clear(self, cname, type, nanny=True, clear_before_decref=False): - self._put_decref(cname, type, nanny, null_check=False, - clear=True, clear_before_decref=clear_before_decref) + def put_incref_memoryviewslice(self, slice_cname, type, have_gil): + # TODO ideally this would just be merged into "put_incref" + type.generate_incref_memoryviewslice(self, slice_cname, have_gil=have_gil) - def put_xdecref(self, cname, type, nanny=True, have_gil=True): - self._put_decref(cname, type, nanny, null_check=True, - have_gil=have_gil, clear=False) + def put_var_incref_memoryviewslice(self, entry, have_gil): + self.put_incref_memoryviewslice(entry.cname, entry.type, have_gil=have_gil) - def put_xdecref_clear(self, cname, type, nanny=True, clear_before_decref=False): - self._put_decref(cname, type, nanny, null_check=True, - clear=True, clear_before_decref=clear_before_decref) + def put_var_gotref(self, entry): + self.put_gotref(entry.cname, entry.type) - def _put_decref(self, cname, type, nanny=True, null_check=False, - have_gil=True, clear=False, clear_before_decref=False): - if type.is_memoryviewslice: - self.put_xdecref_memoryviewslice(cname, have_gil=have_gil) - return + def put_var_giveref(self, entry): + self.put_giveref(entry.cname, entry.type) - prefix = '__Pyx' if nanny else 'Py' - X = 'X' if null_check else '' + def put_var_xgotref(self, entry): + self.put_xgotref(entry.cname, entry.type) - if clear: - if clear_before_decref: - if not nanny: - X = '' # CPython doesn't have a Py_XCLEAR() - self.putln("%s_%sCLEAR(%s);" % (prefix, X, cname)) - else: - self.putln("%s_%sDECREF(%s); %s = 0;" % ( - prefix, X, self.as_pyobject(cname, type), cname)) - else: - self.putln("%s_%sDECREF(%s);" % ( - prefix, X, self.as_pyobject(cname, type))) + def put_var_xgiveref(self, entry): + self.put_xgiveref(entry.cname, entry.type) - def put_decref_set(self, cname, rhs_cname): - self.putln("__Pyx_DECREF_SET(%s, %s);" % (cname, rhs_cname)) + def put_var_incref(self, entry, **kwds): + self.put_incref(entry.cname, entry.type, **kwds) - def put_xdecref_set(self, cname, rhs_cname): - self.putln("__Pyx_XDECREF_SET(%s, %s);" % (cname, rhs_cname)) + def put_var_xincref(self, entry, **kwds): + self.put_xincref(entry.cname, entry.type, **kwds) - def put_var_decref(self, entry): - if entry.type.is_pyobject: - self.putln("__Pyx_XDECREF(%s);" % self.entry_as_pyobject(entry)) + def put_var_decref(self, entry, **kwds): + self.put_decref(entry.cname, entry.type, **kwds) - def put_var_xdecref(self, entry, nanny=True): - if entry.type.is_pyobject: - if nanny: - self.putln("__Pyx_XDECREF(%s);" % self.entry_as_pyobject(entry)) - else: - self.putln("Py_XDECREF(%s);" % self.entry_as_pyobject(entry)) - - def put_var_decref_clear(self, entry): - self._put_var_decref_clear(entry, null_check=False) - - def put_var_xdecref_clear(self, entry): - self._put_var_decref_clear(entry, null_check=True) - - def _put_var_decref_clear(self, entry, null_check): - if entry.type.is_pyobject: - if entry.in_closure: - # reset before DECREF to make sure closure state is - # consistent during call to DECREF() - self.putln("__Pyx_%sCLEAR(%s);" % ( - null_check and 'X' or '', - entry.cname)) - else: - self.putln("__Pyx_%sDECREF(%s); %s = 0;" % ( - null_check and 'X' or '', - self.entry_as_pyobject(entry), - entry.cname)) + def put_var_xdecref(self, entry, **kwds): + self.put_xdecref(entry.cname, entry.type, **kwds) + + def put_var_decref_clear(self, entry, **kwds): + self.put_decref_clear(entry.cname, entry.type, clear_before_decref=entry.in_closure, **kwds) + + def put_var_decref_set(self, entry, rhs_cname, **kwds): + self.put_decref_set(entry.cname, entry.type, rhs_cname, **kwds) + + def put_var_xdecref_set(self, entry, rhs_cname, **kwds): + self.put_xdecref_set(entry.cname, entry.type, rhs_cname, **kwds) + + def put_var_xdecref_clear(self, entry, **kwds): + self.put_xdecref_clear(entry.cname, entry.type, clear_before_decref=entry.in_closure, **kwds) def put_var_decrefs(self, entries, used_only = 0): for entry in entries: @@ -2186,19 +2231,6 @@ class CCodeWriter(object): for entry in entries: self.put_var_xdecref_clear(entry) - def put_incref_memoryviewslice(self, slice_cname, have_gil=False): - from . import MemoryView - self.globalstate.use_utility_code(MemoryView.memviewslice_init_code) - self.putln("__PYX_INC_MEMVIEW(&%s, %d);" % (slice_cname, int(have_gil))) - - def put_xdecref_memoryviewslice(self, slice_cname, have_gil=False): - from . import MemoryView - self.globalstate.use_utility_code(MemoryView.memviewslice_init_code) - self.putln("__PYX_XDEC_MEMVIEW(&%s, %d);" % (slice_cname, int(have_gil))) - - def put_xgiveref_memoryviewslice(self, slice_cname): - self.put_xgiveref("%s.memview" % slice_cname) - def put_init_to_py_none(self, cname, type, nanny=True): from .PyrexTypes import py_object_type, typecast py_none = typecast(type, py_object_type, "Py_None") @@ -2234,14 +2266,13 @@ class CCodeWriter(object): method_flags += [TypeSlots.method_coexist] func_ptr = wrapper_code_writer.put_pymethoddef_wrapper(entry) if wrapper_code_writer else entry.func_cname # Add required casts, but try not to shadow real warnings. - cast = '__Pyx_PyCFunctionFast' if 'METH_FASTCALL' in method_flags else 'PyCFunction' - if 'METH_KEYWORDS' in method_flags: - cast += 'WithKeywords' + cast = entry.signature.method_function_type() if cast != 'PyCFunction': func_ptr = '(void*)(%s)%s' % (cast, func_ptr) + entry_name = entry.name.as_c_string_literal() self.putln( - '{"%s", (PyCFunction)%s, %s, %s}%s' % ( - entry.name, + '{%s, (PyCFunction)%s, %s, %s}%s' % ( + entry_name, func_ptr, "|".join(method_flags), entry.doc_cname if entry.doc else '0', @@ -2250,8 +2281,9 @@ class CCodeWriter(object): def put_pymethoddef_wrapper(self, entry): func_cname = entry.func_cname if entry.is_special: - method_flags = entry.signature.method_flags() - if method_flags and 'METH_NOARGS' in method_flags: + method_flags = entry.signature.method_flags() or [] + from .TypeSlots import method_noargs + if method_noargs in method_flags: # Special NOARGS methods really take no arguments besides 'self', but PyCFunction expects one. func_cname = Naming.method_wrapper_prefix + func_cname self.putln("static PyObject *%s(PyObject *self, CYTHON_UNUSED PyObject *arg) {return %s(self);}" % ( @@ -2260,6 +2292,12 @@ class CCodeWriter(object): # GIL methods + def use_fast_gil_utility_code(self): + if self.globalstate.directives['fast_gil']: + self.globalstate.use_utility_code(UtilityCode.load_cached("FastGil", "ModuleSetupCode.c")) + else: + self.globalstate.use_utility_code(UtilityCode.load_cached("NoFastGil", "ModuleSetupCode.c")) + def put_ensure_gil(self, declare_gilstate=True, variable=None): """ Acquire the GIL. The generated code is safe even when no PyThreadState @@ -2269,10 +2307,7 @@ class CCodeWriter(object): """ self.globalstate.use_utility_code( UtilityCode.load_cached("ForceInitThreads", "ModuleSetupCode.c")) - if self.globalstate.directives['fast_gil']: - self.globalstate.use_utility_code(UtilityCode.load_cached("FastGil", "ModuleSetupCode.c")) - else: - self.globalstate.use_utility_code(UtilityCode.load_cached("NoFastGil", "ModuleSetupCode.c")) + self.use_fast_gil_utility_code() self.putln("#ifdef WITH_THREAD") if not variable: variable = '__pyx_gilstate_save' @@ -2285,10 +2320,7 @@ class CCodeWriter(object): """ Releases the GIL, corresponds to `put_ensure_gil`. """ - if self.globalstate.directives['fast_gil']: - self.globalstate.use_utility_code(UtilityCode.load_cached("FastGil", "ModuleSetupCode.c")) - else: - self.globalstate.use_utility_code(UtilityCode.load_cached("NoFastGil", "ModuleSetupCode.c")) + self.use_fast_gil_utility_code() if not variable: variable = '__pyx_gilstate_save' self.putln("#ifdef WITH_THREAD") @@ -2300,10 +2332,7 @@ class CCodeWriter(object): Acquire the GIL. The thread's thread state must have been initialized by a previous `put_release_gil` """ - if self.globalstate.directives['fast_gil']: - self.globalstate.use_utility_code(UtilityCode.load_cached("FastGil", "ModuleSetupCode.c")) - else: - self.globalstate.use_utility_code(UtilityCode.load_cached("NoFastGil", "ModuleSetupCode.c")) + self.use_fast_gil_utility_code() self.putln("#ifdef WITH_THREAD") self.putln("__Pyx_FastGIL_Forget();") if variable: @@ -2313,10 +2342,7 @@ class CCodeWriter(object): def put_release_gil(self, variable=None): "Release the GIL, corresponds to `put_acquire_gil`." - if self.globalstate.directives['fast_gil']: - self.globalstate.use_utility_code(UtilityCode.load_cached("FastGil", "ModuleSetupCode.c")) - else: - self.globalstate.use_utility_code(UtilityCode.load_cached("NoFastGil", "ModuleSetupCode.c")) + self.use_fast_gil_utility_code() self.putln("#ifdef WITH_THREAD") self.putln("PyThreadState *_save;") self.putln("Py_UNBLOCK_THREADS") @@ -2386,7 +2412,8 @@ class CCodeWriter(object): return self.error_goto_if("!%s" % cname, pos) def error_goto_if_neg(self, cname, pos): - return self.error_goto_if("%s < 0" % cname, pos) + # Add extra parentheses to silence clang warnings about constant conditions. + return self.error_goto_if("(%s < 0)" % cname, pos) def error_goto_if_PyErr(self, pos): return self.error_goto_if("PyErr_Occurred()", pos) @@ -2398,13 +2425,14 @@ class CCodeWriter(object): self.putln('__Pyx_RefNannyDeclarations') def put_setup_refcount_context(self, name, acquire_gil=False): + name = name.as_c_string_literal() # handle unicode names if acquire_gil: self.globalstate.use_utility_code( UtilityCode.load_cached("ForceInitThreads", "ModuleSetupCode.c")) - self.putln('__Pyx_RefNannySetupContext("%s", %d);' % (name, acquire_gil and 1 or 0)) + self.putln('__Pyx_RefNannySetupContext(%s, %d);' % (name, acquire_gil and 1 or 0)) - def put_finish_refcount_context(self): - self.putln("__Pyx_RefNannyFinishContext();") + def put_finish_refcount_context(self, nogil=False): + self.putln("__Pyx_RefNannyFinishContextNogil()" if nogil else "__Pyx_RefNannyFinishContext();") def put_add_traceback(self, qualified_name, include_cline=True): """ @@ -2412,14 +2440,16 @@ class CCodeWriter(object): qualified_name should be the qualified name of the function. """ + qualified_name = qualified_name.as_c_string_literal() # handle unicode names format_tuple = ( qualified_name, Naming.clineno_cname if include_cline else 0, Naming.lineno_cname, Naming.filename_cname, ) + self.funcstate.uses_error_indicator = True - self.putln('__Pyx_AddTraceback("%s", %s, %s, %s);' % format_tuple) + self.putln('__Pyx_AddTraceback(%s, %s, %s, %s);' % format_tuple) def put_unraisable(self, qualified_name, nogil=False): """ diff --git a/Cython/Compiler/CythonScope.py b/Cython/Compiler/CythonScope.py index 1c25d1a6b..bb9e74aa6 100644 --- a/Cython/Compiler/CythonScope.py +++ b/Cython/Compiler/CythonScope.py @@ -125,7 +125,16 @@ class CythonScope(ModuleScope): view_utility_scope = MemoryView.view_utility_code.declare_in_scope( self.viewscope, cython_scope=self, - whitelist=MemoryView.view_utility_whitelist) + allowlist=MemoryView.view_utility_allowlist) + + # Marks the types as being cython_builtin_type so that they can be + # extended from without Cython attempting to import cython.view + ext_types = [ entry.type + for entry in view_utility_scope.entries.values() + if entry.type.is_extension_type ] + for ext_type in ext_types: + ext_type.is_cython_builtin_type = 1 + # self.entries["array"] = view_utility_scope.entries.pop("array") diff --git a/Cython/Compiler/Errors.py b/Cython/Compiler/Errors.py index 9761b52c3..ce011b616 100644 --- a/Cython/Compiler/Errors.py +++ b/Cython/Compiler/Errors.py @@ -24,6 +24,8 @@ class PyrexError(Exception): class PyrexWarning(Exception): pass +class CannotSpecialize(PyrexError): + pass def context(position): source = position[0] @@ -60,11 +62,9 @@ class CompileError(PyrexError): self.message_only = message self.formatted_message = format_error(message, position) self.reported = False - # Deprecated and withdrawn in 2.6: - # self.message = message Exception.__init__(self, self.formatted_message) # Python Exception subclass pickling is broken, - # see http://bugs.python.org/issue1692335 + # see https://bugs.python.org/issue1692335 self.args = (position, message) def __str__(self): @@ -74,8 +74,6 @@ class CompileWarning(PyrexWarning): def __init__(self, position = None, message = ""): self.position = position - # Deprecated and withdrawn in 2.6: - # self.message = message Exception.__init__(self, format_position(position) + message) class InternalError(Exception): @@ -114,7 +112,7 @@ class CompilerCrash(CompileError): message += u'%s: %s' % (cause.__class__.__name__, cause) CompileError.__init__(self, pos, message) # Python Exception subclass pickling is broken, - # see http://bugs.python.org/issue1692335 + # see https://bugs.python.org/issue1692335 self.args = (pos, context, message, cause, stacktrace) class NoElementTreeInstalledException(PyrexError): @@ -171,29 +169,34 @@ def report_error(err, use_stack=True): if Options.fast_fail: raise AbortError("fatal errors") - def error(position, message): #print("Errors.error:", repr(position), repr(message)) ### if position is None: raise InternalError(message) err = CompileError(position, message) - if DebugFlags.debug_exception_on_error: raise Exception(err) # debug + if DebugFlags.debug_exception_on_error: raise Exception(err) # debug report_error(err) return err -LEVEL = 1 # warn about all errors level 1 or higher +LEVEL = 1 # warn about all errors level 1 or higher + +def _write_file_encode(file, line): + try: + file.write(line) + except UnicodeEncodeError: + file.write(line.encode('ascii', 'replace')) def message(position, message, level=1): if level < LEVEL: return warn = CompileWarning(position, message) - line = "note: %s\n" % warn + line = u"note: %s\n" % warn if listing_file: - listing_file.write(line) + _write_file_encode(listing_file, line) if echo_file: - echo_file.write(line) + _write_file_encode(echo_file, line) return warn @@ -203,11 +206,11 @@ def warning(position, message, level=0): if Options.warning_errors and position: return error(position, message) warn = CompileWarning(position, message) - line = "warning: %s\n" % warn + line = u"warning: %s\n" % warn if listing_file: - listing_file.write(line) + _write_file_encode(listing_file, line) if echo_file: - echo_file.write(line) + _write_file_encode(echo_file, line) return warn @@ -216,11 +219,11 @@ def warn_once(position, message, level=0): if level < LEVEL or message in _warn_once_seen: return warn = CompileWarning(position, message) - line = "warning: %s\n" % warn + line = u"warning: %s\n" % warn if listing_file: - listing_file.write(line) + _write_file_encode(listing_file, line) if echo_file: - echo_file.write(line) + _write_file_encode(echo_file, line) _warn_once_seen[message] = True return warn diff --git a/Cython/Compiler/ExprNodes.py b/Cython/Compiler/ExprNodes.py index 4b424d50e..2714d3e4d 100644 --- a/Cython/Compiler/ExprNodes.py +++ b/Cython/Compiler/ExprNodes.py @@ -23,12 +23,13 @@ import os.path import operator from .Errors import ( - error, warning, InternalError, CompileError, report_error, local_errors) + error, warning, InternalError, CompileError, report_error, local_errors, + CannotSpecialize) from .Code import UtilityCode, TempitaUtilityCode from . import StringEncoding from . import Naming from . import Nodes -from .Nodes import Node, utility_code_for_imports, analyse_type_annotation +from .Nodes import Node, utility_code_for_imports, SingleAssignmentNode, PassStatNode from . import PyrexTypes from .PyrexTypes import py_object_type, c_long_type, typecast, error_type, \ unspecified_type @@ -182,7 +183,7 @@ def infer_sequence_item_type(env, seq_node, index_node=None, seq_type=None): else: return item.infer_type(env) # if we're lucky, all items have the same type - item_types = set([item.infer_type(env) for item in seq_node.args]) + item_types = {item.infer_type(env) for item in seq_node.args} if len(item_types) == 1: return item_types.pop() return None @@ -237,6 +238,7 @@ def get_exception_handler(exception_value): exception_value.entry.cname), False) + def maybe_check_py_error(code, check_py_exception, pos, nogil): if check_py_exception: if nogil: @@ -244,12 +246,13 @@ def maybe_check_py_error(code, check_py_exception, pos, nogil): else: code.putln(code.error_goto_if("PyErr_Occurred()", pos)) + def translate_cpp_exception(code, pos, inside, py_result, exception_value, nogil): raise_py_exception, check_py_exception = get_exception_handler(exception_value) code.putln("try {") code.putln("%s" % inside) if py_result: - code.putln(code.error_goto_if_null(py_result, pos)) + code.putln(code.error_goto_if_null(py_result, pos)) maybe_check_py_error(code, check_py_exception, pos, nogil) code.putln("} catch(...) {") if nogil: @@ -260,10 +263,10 @@ def translate_cpp_exception(code, pos, inside, py_result, exception_value, nogil code.putln(code.error_goto(pos)) code.putln("}") + # Used to handle the case where an lvalue expression and an overloaded assignment # both have an exception declaration. -def translate_double_cpp_exception(code, pos, lhs_type, lhs_code, rhs_code, - lhs_exc_val, assign_exc_val, nogil): +def translate_double_cpp_exception(code, pos, lhs_type, lhs_code, rhs_code, lhs_exc_val, assign_exc_val, nogil): handle_lhs_exc, lhc_check_py_exc = get_exception_handler(lhs_exc_val) handle_assignment_exc, assignment_check_py_exc = get_exception_handler(assign_exc_val) code.putln("try {") @@ -314,8 +317,8 @@ class ExprNode(Node): type = None annotation = None temp_code = None - old_temp = None # error checker for multiple frees etc. - use_managed_ref = True # can be set by optimisation transforms + old_temp = None # error checker for multiple frees etc. + use_managed_ref = True # can be set by optimisation transforms result_is_used = True is_numpy_attribute = False @@ -448,6 +451,7 @@ class ExprNode(Node): saved_subexpr_nodes = None is_temp = False + has_temp_moved = False # if True then attempting to do anything but free the temp is invalid is_target = False is_starred = False @@ -455,11 +459,13 @@ class ExprNode(Node): child_attrs = property(fget=operator.attrgetter('subexprs')) + def analyse_annotations(self, env): + pass + def not_implemented(self, method_name): - print_call_chain(method_name, "not implemented") ### + print_call_chain(method_name, "not implemented") raise InternalError( - "%s.%s not implemented" % - (self.__class__.__name__, method_name)) + "%s.%s not implemented" % (self.__class__.__name__, method_name)) def is_lvalue(self): return 0 @@ -498,6 +504,22 @@ class ExprNode(Node): else: return self.calculate_result_code() + def _make_move_result_rhs(self, result, optional=False): + if optional and not (self.is_temp and self.type.is_cpp_class and not self.type.is_reference): + return result + self.has_temp_moved = True + return "{}({})".format("__PYX_STD_MOVE_IF_SUPPORTED" if optional else "std::move", result) + + def move_result_rhs(self): + return self._make_move_result_rhs(self.result(), optional=True) + + def move_result_rhs_as(self, type): + result = self.result_as(type) + if not (type.is_reference or type.needs_refcounting): + requires_move = type.is_rvalue_reference and self.is_temp + result = self._make_move_result_rhs(result, optional=not requires_move) + return result + def pythran_result(self, type_=None): if is_pythran_supported_node_or_none(self): return to_pythran(self) @@ -614,7 +636,7 @@ class ExprNode(Node): def type_dependencies(self, env): # Returns the list of entries whose types must be determined # before the type of self can be inferred. - if hasattr(self, 'type') and self.type is not None: + if getattr(self, 'type', None) is not None: return () return sum([node.type_dependencies(env) for node in self.subexpr_nodes()], ()) @@ -623,12 +645,13 @@ class ExprNode(Node): # Differs from analyse_types as it avoids unnecessary # analysis of subexpressions, but can assume everything # in self.type_dependencies() has been resolved. - if hasattr(self, 'type') and self.type is not None: - return self.type - elif hasattr(self, 'entry') and self.entry is not None: - return self.entry.type - else: - self.not_implemented("infer_type") + type = getattr(self, 'type', None) + if type is not None: + return type + entry = getattr(self, 'entry', None) + if entry is not None: + return entry.type + self.not_implemented("infer_type") def nonlocally_immutable(self): # Returns whether this variable is a safe reference, i.e. @@ -655,6 +678,19 @@ class ExprNode(Node): # type, return that type, else None. return None + def analyse_as_specialized_type(self, env): + type = self.analyse_as_type(env) + if type and type.is_fused and env.fused_to_specific: + # while it would be nice to test "if entry.type in env.fused_to_specific" + # rather than try/catch this doesn't work reliably (mainly for nested fused types) + try: + return type.specialize(env.fused_to_specific) + except KeyError: + pass + if type and type.is_fused: + error(self.pos, "Type is not specific") + return type + def analyse_as_extension_type(self, env): # If this node can be interpreted as a reference to an # extension type or builtin type, return its type, else None. @@ -746,19 +782,20 @@ class ExprNode(Node): def make_owned_reference(self, code): """ - If result is a pyobject, make sure we own a reference to it. + Make sure we own a reference to result. If the result is in a temp, it is already a new reference. """ - if self.type.is_pyobject and not self.result_in_temp(): + if not self.result_in_temp(): code.put_incref(self.result(), self.ctype()) def make_owned_memoryviewslice(self, code): """ Make sure we own the reference to this memoryview slice. """ + # TODO ideally this would be shared with "make_owned_reference" if not self.result_in_temp(): - code.put_incref_memoryviewslice(self.result(), - have_gil=self.in_nogil_context) + code.put_incref_memoryviewslice(self.result(), self.type, + have_gil=not self.in_nogil_context) def generate_evaluation_code(self, code): # Generate code to evaluate this node and @@ -791,13 +828,11 @@ class ExprNode(Node): self.generate_subexpr_disposal_code(code) self.free_subexpr_temps(code) if self.result(): - if self.type.is_pyobject: - code.put_decref_clear(self.result(), self.ctype()) - elif self.type.is_memoryviewslice: - code.put_xdecref_memoryviewslice( - self.result(), have_gil=not self.in_nogil_context) - code.putln("%s.memview = NULL;" % self.result()) - code.putln("%s.data = NULL;" % self.result()) + code.put_decref_clear(self.result(), self.ctype(), + have_gil=not self.in_nogil_context) + if self.has_temp_moved: + code.globalstate.use_utility_code( + UtilityCode.load_cached("MoveIfSupported", "CppSupport.cpp")) else: # Already done if self.is_temp self.generate_subexpr_disposal_code(code) @@ -819,11 +854,15 @@ class ExprNode(Node): elif self.type.is_memoryviewslice: code.putln("%s.memview = NULL;" % self.result()) code.putln("%s.data = NULL;" % self.result()) + + if self.has_temp_moved: + code.globalstate.use_utility_code( + UtilityCode.load_cached("MoveIfSupported", "CppSupport.cpp")) else: self.generate_subexpr_disposal_code(code) def generate_assignment_code(self, rhs, code, overloaded_assignment=False, - exception_check=None, exception_value=None): + exception_check=None, exception_value=None): # Stub method for nodes which are not legal as # the LHS of an assignment. An error will have # been reported earlier. @@ -849,6 +888,32 @@ class ExprNode(Node): def generate_function_definitions(self, env, code): pass + # ----Generation of small bits of reference counting -- + + def generate_decref_set(self, code, rhs): + code.put_decref_set(self.result(), self.ctype(), rhs) + + def generate_xdecref_set(self, code, rhs): + code.put_xdecref_set(self.result(), self.ctype(), rhs) + + def generate_gotref(self, code, handle_null=False, + maybe_null_extra_check=True): + if not (handle_null and self.cf_is_null): + if (handle_null and self.cf_maybe_null + and maybe_null_extra_check): + self.generate_xgotref(code) + else: + code.put_gotref(self.result(), self.ctype()) + + def generate_xgotref(self, code): + code.put_xgotref(self.result(), self.ctype()) + + def generate_giveref(self, code): + code.put_giveref(self.result(), self.ctype()) + + def generate_xgiveref(self, code): + code.put_xgiveref(self.result(), self.ctype()) + # ---------------- Annotation --------------------- def annotate(self, code): @@ -883,8 +948,8 @@ class ExprNode(Node): if used_as_reference and not src_type.is_reference: dst_type = dst_type.ref_base_type - if src_type.is_const: - src_type = src_type.const_base_type + if src_type.is_cv_qualified: + src_type = src_type.cv_base_type if src_type.is_fused or dst_type.is_fused: # See if we are coercing a fused function to a pointer to a @@ -970,7 +1035,8 @@ class ExprNode(Node): and src_type != dst_type and dst_type.assignable_from(src_type)): src = CoerceToComplexNode(src, dst_type, env) - else: # neither src nor dst are py types + else: + # neither src nor dst are py types # Added the string comparison, since for c types that # is enough, but Cython gets confused when the types are # in different pxi files. @@ -1108,6 +1174,7 @@ class PyConstNode(AtomicExprNode): is_literal = 1 type = py_object_type + nogil_check = None def is_simple(self): return 1 @@ -1133,8 +1200,6 @@ class NoneNode(PyConstNode): constant_result = None - nogil_check = None - def compile_time_value(self, denv): return None @@ -1204,7 +1269,7 @@ class BoolNode(ConstNode): def calculate_result_code(self): if self.type.is_pyobject: - return self.value and 'Py_True' or 'Py_False' + return 'Py_True' if self.value else 'Py_False' else: return str(int(self.value)) @@ -1256,7 +1321,7 @@ class IntNode(ConstNode): unsigned = "" longness = "" - is_c_literal = None # unknown + is_c_literal = None # unknown def __init__(self, pos, **kwds): ExprNode.__init__(self, pos, **kwds) @@ -1434,11 +1499,7 @@ def _analyse_name_as_type(name, pos, env): return type global_entry = env.global_scope().lookup(name) - if global_entry and global_entry.type and ( - global_entry.type.is_extension_type - or global_entry.type.is_struct_or_union - or global_entry.type.is_builtin_type - or global_entry.type.is_cpp_class): + if global_entry and global_entry.is_type and global_entry.type: return global_entry.type from .TreeFragment import TreeFragment @@ -1539,7 +1600,7 @@ class BytesNode(ConstNode): self.result_code = result def get_constant_c_result_code(self): - return None # FIXME + return None # FIXME def calculate_result_code(self): return self.result_code @@ -1594,12 +1655,9 @@ class UnicodeNode(ConstNode): if dst_type.is_string and self.bytes_value is not None: # special case: '-3' enforced unicode literal used in a # C char* context - return BytesNode(self.pos, value=self.bytes_value - ).coerce_to(dst_type, env) + return BytesNode(self.pos, value=self.bytes_value).coerce_to(dst_type, env) if dst_type.is_pyunicode_ptr: - node = UnicodeNode(self.pos, value=self.value) - node.type = dst_type - return node + return UnicodeNode(self.pos, value=self.value, type=dst_type) error(self.pos, "Unicode literals do not support coercion to C types other " "than Py_UNICODE/Py_UCS4 (for characters) or Py_UNICODE* " @@ -1784,7 +1842,7 @@ class ImagNode(AtomicExprNode): self.result(), float(self.value), code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) class NewExprNode(AtomicExprNode): @@ -1837,7 +1895,7 @@ class NameNode(AtomicExprNode): is_name = True is_cython_module = False cython_attribute = None - lhs_of_first_assignment = False # TODO: remove me + lhs_of_first_assignment = False # TODO: remove me is_used_as_rvalue = 0 entry = None type_entry = None @@ -1919,30 +1977,47 @@ class NameNode(AtomicExprNode): def declare_from_annotation(self, env, as_target=False): """Implements PEP 526 annotation typing in a fairly relaxed way. - Annotations are ignored for global variables, Python class attributes and already declared variables. - String literals are allowed and ignored. - The ambiguous Python types 'int' and 'long' are ignored and the 'cython.int' form must be used instead. + Annotations are ignored for global variables. + All other annotations are stored on the entry in the symbol table. + String literals are allowed and not evaluated. + The ambiguous Python types 'int' and 'long' are not evaluated - the 'cython.int' form must be used instead. """ - if not env.directives['annotation_typing']: - return - if env.is_module_scope or env.is_py_class_scope: - # annotations never create global cdef names and Python classes don't support them anyway - return name = self.name - if self.entry or env.lookup_here(name) is not None: - # already declared => ignore annotation - return - annotation = self.annotation - if annotation.is_string_literal: - # name: "description" => not a type, but still a declared variable or attribute - atype = None - else: - _, atype = analyse_type_annotation(annotation, env) - if atype is None: - atype = unspecified_type if as_target and env.directives['infer_types'] != False else py_object_type - self.entry = env.declare_var(name, atype, self.pos, is_cdef=not as_target) - self.entry.annotation = annotation + entry = self.entry or env.lookup_here(name) + if not entry: + # annotations never create global cdef names + if env.is_module_scope: + return + if ( + # name: "description" => not a type, but still a declared variable or attribute + annotation.expr.is_string_literal + # don't do type analysis from annotations if not asked to, but still collect the annotation + or not env.directives['annotation_typing'] + ): + atype = None + else: + _, atype = annotation.analyse_type_annotation(env) + if atype is None: + atype = unspecified_type if as_target and env.directives['infer_types'] != False else py_object_type + if atype.is_fused and env.fused_to_specific: + try: + atype = atype.specialize(env.fused_to_specific) + except CannotSpecialize: + error(self.pos, + "'%s' cannot be specialized since its type is not a fused argument to this function" % + self.name) + atype = error_type + if as_target and env.is_c_class_scope and not (atype.is_pyobject or atype.is_error): + # TODO: this will need revising slightly if either cdef dataclasses or + # annotated cdef attributes are implemented + atype = py_object_type + warning(annotation.pos, "Annotation ignored since class-level attributes must be Python objects. " + "Were you trying to set up an instance attribute?", 2) + entry = self.entry = env.declare_var(name, atype, self.pos, is_cdef=not as_target) + # Even if the entry already exists, make sure we're supplying an annotation if we can. + if annotation and not entry.annotation: + entry.annotation = annotation def analyse_as_module(self, env): # Try to interpret this as a reference to a cimported module. @@ -2071,7 +2146,7 @@ class NameNode(AtomicExprNode): if self.is_used_as_rvalue: entry = self.entry if entry.is_builtin: - if not entry.is_const: # cached builtins are ok + if not entry.is_const: # cached builtins are ok self.gil_error() elif entry.is_pyglobal: self.gil_error() @@ -2096,7 +2171,7 @@ class NameNode(AtomicExprNode): entry = self.entry if entry.is_type and entry.type.is_extension_type: self.type_entry = entry - if entry.is_type and entry.type.is_enum: + if entry.is_type and (entry.type.is_enum or entry.type.is_cpp_enum): py_entry = Symtab.Entry(self.name, None, py_object_type) py_entry.is_pyglobal = True py_entry.scope = self.entry.scope @@ -2122,7 +2197,7 @@ class NameNode(AtomicExprNode): def may_be_none(self): if self.cf_state and self.type and (self.type.is_pyobject or self.type.is_memoryviewslice): - # gard against infinite recursion on self-dependencies + # guard against infinite recursion on self-dependencies if getattr(self, '_none_checking', False): # self-dependency - either this node receives a None # value from *another* node, or it can not reference @@ -2189,24 +2264,23 @@ class NameNode(AtomicExprNode): def calculate_result_code(self): entry = self.entry if not entry: - return "<error>" # There was an error earlier + return "<error>" # There was an error earlier return entry.cname def generate_result_code(self, code): - assert hasattr(self, 'entry') entry = self.entry if entry is None: - return # There was an error earlier + return # There was an error earlier if entry.utility_code: code.globalstate.use_utility_code(entry.utility_code) if entry.is_builtin and entry.is_const: - return # Lookup already cached + return # Lookup already cached elif entry.is_pyclass_attr: assert entry.type.is_pyobject, "Python global or builtin not a Python object" interned_cname = code.intern_identifier(self.entry.name) if entry.is_builtin: namespace = Naming.builtins_cname - else: # entry.is_pyglobal + else: # entry.is_pyglobal namespace = entry.scope.namespace_cname if not self.cf_is_null: code.putln( @@ -2225,7 +2299,7 @@ class NameNode(AtomicExprNode): if not self.cf_is_null: code.putln("}") code.putln(code.error_goto_if_null(self.result(), self.pos)) - code.put_gotref(self.py_result()) + self.generate_gotref(code) elif entry.is_builtin and not entry.scope.is_module_scope: # known builtin @@ -2238,7 +2312,7 @@ class NameNode(AtomicExprNode): self.result(), interned_cname, code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) elif entry.is_pyglobal or (entry.is_builtin and entry.scope.is_module_scope): # name in class body, global name or unknown builtin @@ -2262,7 +2336,7 @@ class NameNode(AtomicExprNode): entry.scope.namespace_cname, interned_cname, code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) elif entry.is_local or entry.in_closure or entry.from_closure or entry.type.is_memoryviewslice: # Raise UnboundLocalError for objects and memoryviewslices @@ -2276,11 +2350,11 @@ class NameNode(AtomicExprNode): code.put_error_if_unbound(self.pos, entry, self.in_nogil_context) def generate_assignment_code(self, rhs, code, overloaded_assignment=False, - exception_check=None, exception_value=None): + exception_check=None, exception_value=None): #print "NameNode.generate_assignment_code:", self.name ### entry = self.entry if entry is None: - return # There was an error earlier + return # There was an error earlier if (self.entry.type.is_ptr and isinstance(rhs, ListNode) and not self.lhs_of_first_assignment and not rhs.in_module_scope): @@ -2301,8 +2375,10 @@ class NameNode(AtomicExprNode): setter = 'PyDict_SetItem' namespace = Naming.moddict_cname elif entry.is_pyclass_attr: - code.globalstate.use_utility_code(UtilityCode.load_cached("SetNameInClass", "ObjectHandling.c")) - setter = '__Pyx_SetNameInClass' + # Special-case setting __new__ + n = "SetNewInClass" if self.name == "__new__" else "SetNameInClass" + code.globalstate.use_utility_code(UtilityCode.load_cached(n, "ObjectHandling.c")) + setter = '__Pyx_' + n else: assert False, repr(entry) code.put_error_if_neg( @@ -2344,31 +2420,24 @@ class NameNode(AtomicExprNode): rhs.make_owned_reference(code) is_external_ref = entry.is_cglobal or self.entry.in_closure or self.entry.from_closure if is_external_ref: - if not self.cf_is_null: - if self.cf_maybe_null: - code.put_xgotref(self.py_result()) - else: - code.put_gotref(self.py_result()) + self.generate_gotref(code, handle_null=True) assigned = True if entry.is_cglobal: - code.put_decref_set( - self.result(), rhs.result_as(self.ctype())) + self.generate_decref_set(code, rhs.result_as(self.ctype())) else: if not self.cf_is_null: if self.cf_maybe_null: - code.put_xdecref_set( - self.result(), rhs.result_as(self.ctype())) + self.generate_xdecref_set(code, rhs.result_as(self.ctype())) else: - code.put_decref_set( - self.result(), rhs.result_as(self.ctype())) + self.generate_decref_set(code, rhs.result_as(self.ctype())) else: assigned = False if is_external_ref: - code.put_giveref(rhs.py_result()) + rhs.generate_giveref(code) if not self.type.is_memoryviewslice: if not assigned: if overloaded_assignment: - result = rhs.result() + result = rhs.move_result_rhs() if exception_check == '+': translate_cpp_exception( code, self.pos, @@ -2378,7 +2447,7 @@ class NameNode(AtomicExprNode): else: code.putln('%s = %s;' % (self.result(), result)) else: - result = rhs.result_as(self.ctype()) + result = rhs.move_result_rhs_as(self.ctype()) if is_pythran_expr(self.type): code.putln('new (&%s) decltype(%s){%s};' % (self.result(), self.result(), result)) @@ -2431,7 +2500,7 @@ class NameNode(AtomicExprNode): def generate_deletion_code(self, code, ignore_nonexisting=False): if self.entry is None: - return # There was an error earlier + return # There was an error earlier elif self.entry.is_pyclass_attr: namespace = self.entry.scope.namespace_cname interned_cname = code.intern_identifier(self.entry.name) @@ -2467,26 +2536,20 @@ class NameNode(AtomicExprNode): if self.cf_maybe_null and not ignore_nonexisting: code.put_error_if_unbound(self.pos, self.entry) - if self.entry.type.is_pyobject: - if self.entry.in_closure: - # generator - if ignore_nonexisting and self.cf_maybe_null: - code.put_xgotref(self.result()) - else: - code.put_gotref(self.result()) - if ignore_nonexisting and self.cf_maybe_null: - code.put_xdecref(self.result(), self.ctype()) - else: - code.put_decref(self.result(), self.ctype()) - code.putln('%s = NULL;' % self.result()) + if self.entry.in_closure: + # generator + self.generate_gotref(code, handle_null=True, maybe_null_extra_check=ignore_nonexisting) + if ignore_nonexisting and self.cf_maybe_null: + code.put_xdecref_clear(self.result(), self.ctype(), + have_gil=not self.nogil) else: - code.put_xdecref_memoryviewslice(self.entry.cname, - have_gil=not self.nogil) + code.put_decref_clear(self.result(), self.ctype(), + have_gil=not self.nogil) else: error(self.pos, "Deletion of C names not supported") def annotate(self, code): - if hasattr(self, 'is_called') and self.is_called: + if getattr(self, 'is_called', False): pos = (self.pos[0], self.pos[1], self.pos[2] - len(self.name) - 1) if self.type.is_pyobject: style, text = 'py_call', 'python function (%s)' @@ -2520,7 +2583,7 @@ class BackquoteNode(ExprNode): self.result(), self.arg.py_result(), code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) class ImportNode(ExprNode): @@ -2539,44 +2602,61 @@ class ImportNode(ExprNode): # relative to the current module. # None: decide the level according to language level and # directives + # get_top_level_module int true: return top-level module, false: return imported module + # module_names TupleNode the separate names of the module and submodules, or None type = py_object_type + module_names = None + get_top_level_module = False + is_temp = True - subexprs = ['module_name', 'name_list'] + subexprs = ['module_name', 'name_list', 'module_names'] def analyse_types(self, env): if self.level is None: - if (env.directives['py2_import'] or - Future.absolute_import not in env.global_scope().context.future_directives): + # For modules in packages, and without 'absolute_import' enabled, try relative (Py2) import first. + if env.global_scope().parent_module and ( + env.directives['py2_import'] or + Future.absolute_import not in env.global_scope().context.future_directives): self.level = -1 else: self.level = 0 module_name = self.module_name.analyse_types(env) self.module_name = module_name.coerce_to_pyobject(env) + assert self.module_name.is_string_literal if self.name_list: name_list = self.name_list.analyse_types(env) self.name_list = name_list.coerce_to_pyobject(env) - self.is_temp = 1 + elif '.' in self.module_name.value: + self.module_names = TupleNode(self.module_name.pos, args=[ + IdentifierStringNode(self.module_name.pos, value=part, constant_result=part) + for part in map(StringEncoding.EncodedString, self.module_name.value.split('.')) + ]).analyse_types(env) return self gil_message = "Python import" def generate_result_code(self, code): - if self.name_list: - name_list_code = self.name_list.py_result() + assert self.module_name.is_string_literal + module_name = self.module_name.value + + if self.level == 0 and not self.name_list and not self.get_top_level_module: + if self.module_names: + assert self.module_names.is_literal # make sure we create the tuple only once + code.globalstate.use_utility_code(UtilityCode.load_cached("ImportDottedModule", "ImportExport.c")) + import_code = "__Pyx_ImportDottedModule(%s, %s)" % ( + self.module_name.py_result(), + self.module_names.py_result() if self.module_names else 'NULL', + ) else: - name_list_code = "0" + code.globalstate.use_utility_code(UtilityCode.load_cached("Import", "ImportExport.c")) + import_code = "__Pyx_Import(%s, %s, %d)" % ( + self.module_name.py_result(), + self.name_list.py_result() if self.name_list else '0', + self.level) - code.globalstate.use_utility_code(UtilityCode.load_cached("Import", "ImportExport.c")) - import_code = "__Pyx_Import(%s, %s, %d)" % ( - self.module_name.py_result(), - name_list_code, - self.level) - - if (self.level <= 0 and - self.module_name.is_string_literal and - self.module_name.value in utility_code_for_imports): - helper_func, code_name, code_file = utility_code_for_imports[self.module_name.value] + if self.level <= 0 and module_name in utility_code_for_imports: + helper_func, code_name, code_file = utility_code_for_imports[module_name] code.globalstate.use_utility_code(UtilityCode.load_cached(code_name, code_file)) import_code = '%s(%s)' % (helper_func, import_code) @@ -2584,7 +2664,7 @@ class ImportNode(ExprNode): self.result(), import_code, code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) class IteratorNode(ExprNode): @@ -2597,7 +2677,6 @@ class IteratorNode(ExprNode): type = py_object_type iter_func_ptr = None counter_cname = None - cpp_iterator_cname = None reversed = False # currently only used for list/tuple types (see Optimize.py) is_async = False @@ -2610,7 +2689,7 @@ class IteratorNode(ExprNode): # C array iteration will be transformed later on self.type = self.sequence.type elif self.sequence.type.is_cpp_class: - self.analyse_cpp_types(env) + return CppIteratorNode(self.pos, sequence=self.sequence).analyse_types(env) else: self.sequence = self.sequence.coerce_to_pyobject(env) if self.sequence.type in (list_type, tuple_type): @@ -2640,65 +2719,10 @@ class IteratorNode(ExprNode): return sequence_type return py_object_type - def analyse_cpp_types(self, env): - sequence_type = self.sequence.type - if sequence_type.is_ptr: - sequence_type = sequence_type.base_type - begin = sequence_type.scope.lookup("begin") - end = sequence_type.scope.lookup("end") - if (begin is None - or not begin.type.is_cfunction - or begin.type.args): - error(self.pos, "missing begin() on %s" % self.sequence.type) - self.type = error_type - return - if (end is None - or not end.type.is_cfunction - or end.type.args): - error(self.pos, "missing end() on %s" % self.sequence.type) - self.type = error_type - return - iter_type = begin.type.return_type - if iter_type.is_cpp_class: - if env.lookup_operator_for_types( - self.pos, - "!=", - [iter_type, end.type.return_type]) is None: - error(self.pos, "missing operator!= on result of begin() on %s" % self.sequence.type) - self.type = error_type - return - if env.lookup_operator_for_types(self.pos, '++', [iter_type]) is None: - error(self.pos, "missing operator++ on result of begin() on %s" % self.sequence.type) - self.type = error_type - return - if env.lookup_operator_for_types(self.pos, '*', [iter_type]) is None: - error(self.pos, "missing operator* on result of begin() on %s" % self.sequence.type) - self.type = error_type - return - self.type = iter_type - elif iter_type.is_ptr: - if not (iter_type == end.type.return_type): - error(self.pos, "incompatible types for begin() and end()") - self.type = iter_type - else: - error(self.pos, "result type of begin() on %s must be a C++ class or pointer" % self.sequence.type) - self.type = error_type - return - def generate_result_code(self, code): sequence_type = self.sequence.type if sequence_type.is_cpp_class: - if self.sequence.is_name: - # safe: C++ won't allow you to reassign to class references - begin_func = "%s.begin" % self.sequence.result() - else: - sequence_type = PyrexTypes.c_ptr_type(sequence_type) - self.cpp_iterator_cname = code.funcstate.allocate_temp(sequence_type, manage_ref=False) - code.putln("%s = &%s;" % (self.cpp_iterator_cname, self.sequence.result())) - begin_func = "%s->begin" % self.cpp_iterator_cname - # TODO: Limit scope. - code.putln("%s = %s();" % (self.result(), begin_func)) - return + assert False, "Should have been changed to CppIteratorNode" if sequence_type.is_array or sequence_type.is_ptr: raise InternalError("for in carray slice not transformed") @@ -2740,12 +2764,12 @@ class IteratorNode(ExprNode): self.result(), self.sequence.py_result(), code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) # PyObject_GetIter() fails if "tp_iternext" is not set, but the check below # makes it visible to the C compiler that the pointer really isn't NULL, so that # it can distinguish between the special cases and the generic case - code.putln("%s = Py_TYPE(%s)->tp_iternext; %s" % ( + code.putln("%s = __Pyx_PyObject_GetIterNextFunc(%s); %s" % ( self.iter_func_ptr, self.py_result(), code.error_goto_if_null(self.iter_func_ptr, self.pos))) if self.may_be_a_sequence: @@ -2787,28 +2811,14 @@ class IteratorNode(ExprNode): self.counter_cname, inc_dec, code.error_goto_if_null(result_name, self.pos))) - code.put_gotref(result_name) + code.put_gotref(result_name, py_object_type) code.putln("#endif") def generate_iter_next_result_code(self, result_name, code): sequence_type = self.sequence.type if self.reversed: code.putln("if (%s < 0) break;" % self.counter_cname) - if sequence_type.is_cpp_class: - if self.cpp_iterator_cname: - end_func = "%s->end" % self.cpp_iterator_cname - else: - end_func = "%s.end" % self.sequence.result() - # TODO: Cache end() call? - code.putln("if (!(%s != %s())) break;" % ( - self.result(), - end_func)) - code.putln("%s = *%s;" % ( - result_name, - self.result())) - code.putln("++%s;" % self.result()) - return - elif sequence_type is list_type: + if sequence_type is list_type: self.generate_next_sequence_item('List', result_name, code) return elif sequence_type is tuple_type: @@ -2838,7 +2848,7 @@ class IteratorNode(ExprNode): code.putln("}") code.putln("break;") code.putln("}") - code.put_gotref(result_name) + code.put_gotref(result_name, py_object_type) code.putln("}") def free_temps(self, code): @@ -2847,8 +2857,109 @@ class IteratorNode(ExprNode): if self.iter_func_ptr: code.funcstate.release_temp(self.iter_func_ptr) self.iter_func_ptr = None - if self.cpp_iterator_cname: - code.funcstate.release_temp(self.cpp_iterator_cname) + ExprNode.free_temps(self, code) + + +class CppIteratorNode(ExprNode): + # Iteration over a C++ container. + # Created at the analyse_types stage by IteratorNode + cpp_sequence_cname = None + cpp_attribute_op = "." + is_temp = True + + subexprs = ['sequence'] + + def analyse_types(self, env): + sequence_type = self.sequence.type + if sequence_type.is_ptr: + sequence_type = sequence_type.base_type + begin = sequence_type.scope.lookup("begin") + end = sequence_type.scope.lookup("end") + if (begin is None + or not begin.type.is_cfunction + or begin.type.args): + error(self.pos, "missing begin() on %s" % self.sequence.type) + self.type = error_type + return self + if (end is None + or not end.type.is_cfunction + or end.type.args): + error(self.pos, "missing end() on %s" % self.sequence.type) + self.type = error_type + return self + iter_type = begin.type.return_type + if iter_type.is_cpp_class: + if env.lookup_operator_for_types( + self.pos, + "!=", + [iter_type, end.type.return_type]) is None: + error(self.pos, "missing operator!= on result of begin() on %s" % self.sequence.type) + self.type = error_type + return self + if env.lookup_operator_for_types(self.pos, '++', [iter_type]) is None: + error(self.pos, "missing operator++ on result of begin() on %s" % self.sequence.type) + self.type = error_type + return self + if env.lookup_operator_for_types(self.pos, '*', [iter_type]) is None: + error(self.pos, "missing operator* on result of begin() on %s" % self.sequence.type) + self.type = error_type + return self + self.type = iter_type + elif iter_type.is_ptr: + if not (iter_type == end.type.return_type): + error(self.pos, "incompatible types for begin() and end()") + self.type = iter_type + else: + error(self.pos, "result type of begin() on %s must be a C++ class or pointer" % self.sequence.type) + self.type = error_type + return self + + def generate_result_code(self, code): + sequence_type = self.sequence.type + # essentially 3 options: + if self.sequence.is_name or self.sequence.is_attribute: + # 1) is a name and can be accessed directly; + # assigning to it may break the container, but that's the responsibility + # of the user + code.putln("%s = %s%sbegin();" % (self.result(), + self.sequence.result(), + self.cpp_attribute_op)) + else: + # (while it'd be nice to limit the scope of the loop temp, it's essentially + # impossible to do while supporting generators) + temp_type = sequence_type + if temp_type.is_reference: + # 2) Sequence is a reference (often obtained by dereferencing a pointer); + # make the temp a pointer so we are not sensitive to users reassigning + # the pointer than it came from + temp_type = PyrexTypes.CPtrType(sequence_type.ref_base_type) + if temp_type.is_ptr: + self.cpp_attribute_op = "->" + # 3) (otherwise) sequence comes from a function call or similar, so we must + # create a temp to store it in + self.cpp_sequence_cname = code.funcstate.allocate_temp(temp_type, manage_ref=False) + code.putln("%s = %s%s;" % (self.cpp_sequence_cname, + "&" if temp_type.is_ptr else "", + self.sequence.move_result_rhs())) + code.putln("%s = %s%sbegin();" % (self.result(), self.cpp_sequence_cname, + self.cpp_attribute_op)) + + def generate_iter_next_result_code(self, result_name, code): + # end call isn't cached to support containers that allow adding while iterating + # (much as this is usually a bad idea) + code.putln("if (!(%s != %s%send())) break;" % ( + self.result(), + self.cpp_sequence_cname or self.sequence.result(), + self.cpp_attribute_op)) + code.putln("%s = *%s;" % ( + result_name, + self.result())) + code.putln("++%s;" % self.result()) + + def free_temps(self, code): + if self.cpp_sequence_cname: + code.funcstate.release_temp(self.cpp_sequence_cname) + # skip over IteratorNode since we don't use any of the temps it does ExprNode.free_temps(self, code) @@ -2880,8 +2991,8 @@ class NextNode(AtomicExprNode): item_type = env.lookup_operator_for_types(self.pos, "*", [iterator_type]).type.return_type if item_type.is_reference: item_type = item_type.ref_base_type - if item_type.is_const: - item_type = item_type.const_base_type + if item_type.is_cv_qualified: + item_type = item_type.cv_base_type return item_type else: # Avoid duplication of complicated logic. @@ -2930,7 +3041,7 @@ class AsyncIteratorNode(ExprNode): self.result(), self.sequence.py_result(), code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.result()) + self.generate_gotref(code) class AsyncNextNode(AtomicExprNode): @@ -2960,7 +3071,7 @@ class AsyncNextNode(AtomicExprNode): self.result(), self.iterator.py_result(), code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.result()) + self.generate_gotref(code) class WithExitCallNode(ExprNode): @@ -3003,7 +3114,7 @@ class WithExitCallNode(ExprNode): self.args.free_temps(code) code.putln(code.error_goto_if_null(result_var, self.pos)) - code.put_gotref(result_var) + code.put_gotref(result_var, py_object_type) if self.await_expr: # FIXME: result_var temp currently leaks into the closure @@ -3135,6 +3246,7 @@ class JoinedStrNode(ExprNode): # type = unicode_type is_temp = True + gil_message = "String concatenation" subexprs = ['values'] @@ -3157,7 +3269,7 @@ class JoinedStrNode(ExprNode): list_var, num_items, code.error_goto_if_null(list_var, self.pos))) - code.put_gotref(list_var) + code.put_gotref(list_var, py_object_type) code.putln("%s = 0;" % ulength_var) code.putln("%s = 127;" % max_char_var) # at least ASCII character range @@ -3201,7 +3313,7 @@ class JoinedStrNode(ExprNode): max_char_var, max_char_value, max_char_var, max_char_value, max_char_var)) code.putln("%s += %s;" % (ulength_var, ulength)) - code.put_giveref(node.py_result()) + node.generate_giveref(code) code.putln('PyTuple_SET_ITEM(%s, %s, %s);' % (list_var, i, node.py_result())) node.generate_post_assignment_code(code) node.free_temps(code) @@ -3216,7 +3328,7 @@ class JoinedStrNode(ExprNode): ulength_var, max_char_var, code.error_goto_if_null(self.py_result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) code.put_decref_clear(list_var, py_object_type) code.funcstate.release_temp(list_var) @@ -3237,6 +3349,7 @@ class FormattedValueNode(ExprNode): type = unicode_type is_temp = True c_format_spec = None + gil_message = "String formatting" find_conversion_func = { 's': 'PyObject_Unicode', @@ -3274,7 +3387,7 @@ class FormattedValueNode(ExprNode): self.result(), convert_func_call, code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) return value_result = self.value.py_result() @@ -3313,7 +3426,7 @@ class FormattedValueNode(ExprNode): value_result, format_spec, code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) #------------------------------------------------------------------- @@ -3351,7 +3464,7 @@ class ParallelThreadsAvailableNode(AtomicExprNode): return self.temp_code -class ParallelThreadIdNode(AtomicExprNode): #, Nodes.ParallelNode): +class ParallelThreadIdNode(AtomicExprNode): #, Nodes.ParallelNode): """ Implements cython.parallel.threadid() """ @@ -3507,6 +3620,8 @@ class IndexNode(_IndexingBaseNode): bytearray_type, list_type, tuple_type): # slicing these returns the same type return base_type + elif base_type.is_memoryviewslice: + return base_type else: # TODO: Handle buffers (hopefully without too much redundancy). return py_object_type @@ -3549,6 +3664,23 @@ class IndexNode(_IndexingBaseNode): index += base_type.size if 0 <= index < base_type.size: return base_type.components[index] + elif base_type.is_memoryviewslice: + if base_type.ndim == 0: + pass # probably an error, but definitely don't know what to do - return pyobject for now + if base_type.ndim == 1: + return base_type.dtype + else: + return PyrexTypes.MemoryViewSliceType(base_type.dtype, base_type.axes[1:]) + + if self.index.is_sequence_constructor and base_type.is_memoryviewslice: + inferred_type = base_type + for a in self.index.args: + if not inferred_type.is_memoryviewslice: + break # something's gone wrong + inferred_type = IndexNode(self.pos, base=ExprNode(self.base.pos, type=inferred_type), + index=a).infer_type(env) + else: + return inferred_type if base_type.is_cpp_class: class FakeOperand: @@ -3626,6 +3758,8 @@ class IndexNode(_IndexingBaseNode): if not base_type.is_cfunction: self.index = self.index.analyse_types(env) self.original_index_type = self.index.type + if self.original_index_type.is_reference: + self.original_index_type = self.original_index_type.ref_base_type if base_type.is_unicode_char: # we infer Py_UNICODE/Py_UCS4 for unicode strings in some @@ -3711,6 +3845,8 @@ class IndexNode(_IndexingBaseNode): def analyse_as_c_array(self, env, is_slice): base_type = self.base.type self.type = base_type.base_type + if self.type.is_cpp_class: + self.type = PyrexTypes.CReferenceType(self.type) if is_slice: self.type = base_type elif self.index.type.is_pyobject: @@ -4060,7 +4196,7 @@ class IndexNode(_IndexingBaseNode): self.extra_index_params(code), code.error_goto_if(error_check % self.result(), self.pos))) if self.type.is_pyobject: - code.put_gotref(self.py_result()) + self.generate_gotref(code) def generate_setitem_code(self, value_code, code): if self.index.type.is_int: @@ -4096,7 +4232,7 @@ class IndexNode(_IndexingBaseNode): self.pos)) def generate_assignment_code(self, rhs, code, overloaded_assignment=False, - exception_check=None, exception_value=None): + exception_check=None, exception_value=None): self.generate_subexpr_evaluation_code(code) if self.type.is_pyobject: @@ -4105,8 +4241,7 @@ class IndexNode(_IndexingBaseNode): value_code = self._check_byte_value(code, rhs) self.generate_setitem_code(value_code, code) elif self.base.type.is_cpp_class and self.exception_check and self.exception_check == '+': - if overloaded_assignment and exception_check and \ - self.exception_value != exception_value: + if overloaded_assignment and exception_check and self.exception_value != exception_value: # Handle the case that both the index operator and the assignment # operator have a c++ exception handler and they are not the same. translate_double_cpp_exception(code, self.pos, self.type, @@ -4353,11 +4488,11 @@ class BufferIndexNode(_IndexingBaseNode): manage_ref=False) rhs_code = rhs.result() code.putln("%s = %s;" % (ptr, ptrexpr)) - code.put_gotref("*%s" % ptr) + code.put_gotref("*%s" % ptr, self.buffer_type.dtype) code.putln("__Pyx_INCREF(%s); __Pyx_DECREF(*%s);" % ( rhs_code, ptr)) code.putln("*%s %s= %s;" % (ptr, op, rhs_code)) - code.put_giveref("*%s" % ptr) + code.put_giveref("*%s" % ptr, self.buffer_type.dtype) code.funcstate.release_temp(ptr) else: # Simple case @@ -4620,7 +4755,7 @@ class MemoryViewSliceNode(MemoryViewIndexNode): assert not list(it) buffer_entry.generate_buffer_slice_code( - code, self.original_indices, self.result(), + code, self.original_indices, self.result(), self.type, have_gil=have_gil, have_slices=have_slices, directives=code.globalstate.directives) @@ -4723,8 +4858,17 @@ class MemoryCopyScalar(MemoryCopyNode): code.putln("%s __pyx_temp_slice = %s;" % (slice_decl, self.dst.result())) dst_temp = "__pyx_temp_slice" + force_strided = False + indices = self.dst.original_indices + for idx in indices: + if isinstance(idx, SliceNode) and not (idx.start.is_none and + idx.stop.is_none and + idx.step.is_none): + force_strided = True + slice_iter_obj = MemoryView.slice_iter(self.dst.type, dst_temp, - self.dst.type.ndim, code) + self.dst.type.ndim, code, + force_strided=force_strided) p = slice_iter_obj.start_loops() if dtype.is_pyobject: @@ -5057,15 +5201,14 @@ class SliceIndexNode(ExprNode): start_code, stop_code, code.error_goto_if_null(result, self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) def generate_assignment_code(self, rhs, code, overloaded_assignment=False, - exception_check=None, exception_value=None): + exception_check=None, exception_value=None): self.generate_subexpr_evaluation_code(code) if self.type.is_pyobject: code.globalstate.use_utility_code(self.set_slice_utility_code) - (has_c_start, has_c_stop, c_start, c_stop, - py_start, py_stop, py_slice) = self.get_slice_config() + has_c_start, has_c_stop, c_start, c_stop, py_start, py_stop, py_slice = self.get_slice_config() code.put_error_if_neg(self.pos, "__Pyx_PyObject_SetSlice(%s, %s, %s, %s, %s, %s, %s, %d, %d, %d)" % ( self.base.py_result(), @@ -5292,9 +5435,9 @@ class SliceNode(ExprNode): self.stop.py_result(), self.step.py_result(), code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) if self.is_literal: - code.put_giveref(self.py_result()) + self.generate_giveref(code) class SliceIntNode(SliceNode): # start:stop:step in subscript list @@ -5391,6 +5534,9 @@ class CallNode(ExprNode): return PyrexTypes.c_double_type elif function.entry.name in Builtin.types_that_construct_their_instance: return result_type + func_type = self.function.analyse_as_type(env) + if func_type and (func_type.is_struct_or_union or func_type.is_cpp_class): + return func_type return py_object_type def type_dependencies(self, env): @@ -5516,6 +5662,16 @@ class SimpleCallNode(CallNode): except Exception as e: self.compile_time_value_error(e) + @classmethod + def for_cproperty(cls, pos, obj, entry): + # Create a call node for C property access. + property_scope = entry.scope + getter_entry = property_scope.lookup_here(entry.name) + assert getter_entry, "Getter not found in scope %s: %s" % (property_scope, property_scope.entries) + function = NameNode(pos, name=entry.name, entry=getter_entry, type=getter_entry.type) + node = cls(pos, function=function, args=[obj]) + return node + def analyse_as_type(self, env): attr = self.function.as_cython_attribute() if attr == 'pointer': @@ -5581,6 +5737,7 @@ class SimpleCallNode(CallNode): self.analyse_c_function_call(env) if func_type.exception_check == '+': self.is_temp = True + return self def function_type(self): @@ -5632,8 +5789,18 @@ class SimpleCallNode(CallNode): else: alternatives = overloaded_entry.all_alternatives() - entry = PyrexTypes.best_match( - [arg.type for arg in args], alternatives, self.pos, env, args) + # For any argument/parameter pair A/P, if P is a forwarding reference, + # use lvalue-reference-to-A for deduction in place of A when the + # function call argument is an lvalue. See: + # https://en.cppreference.com/w/cpp/language/template_argument_deduction#Deduction_from_a_function_call + arg_types = [arg.type for arg in args] + if func_type.is_cfunction: + for i, formal_arg in enumerate(func_type.args): + if formal_arg.is_forwarding_reference(): + if self.args[i].is_lvalue(): + arg_types[i] = PyrexTypes.c_ref_type(arg_types[i]) + + entry = PyrexTypes.best_match(arg_types, alternatives, self.pos, env, args) if not entry: self.type = PyrexTypes.error_type @@ -5716,7 +5883,7 @@ class SimpleCallNode(CallNode): # but we must make sure it cannot be collected # before we return from the function, so we create # an owned temp reference to it - if i > 0: # first argument doesn't matter + if i > 0: # first argument doesn't matter some_args_in_temps = True arg = arg.coerce_to_temp(env) args[i] = arg @@ -5745,7 +5912,7 @@ class SimpleCallNode(CallNode): # case) for i in range(actual_nargs-1): if i == 0 and self.self is not None: - continue # self is ok + continue # self is ok arg = args[i] if arg.nonlocally_immutable(): # locals, C functions, unassignable types are safe. @@ -5761,7 +5928,7 @@ class SimpleCallNode(CallNode): else: #self.args[i] = arg.coerce_to_temp(env) # instead: issue a warning - if i > 0 or i == 1 and self.self is not None: # skip first arg + if i > 0 or i == 1 and self.self is not None: # skip first arg warning(arg.pos, "Argument evaluation order in C function call is undefined and may not be as expected", 0) break @@ -5793,8 +5960,8 @@ class SimpleCallNode(CallNode): # Called in 'nogil' context? self.nogil = env.nogil if (self.nogil and - func_type.exception_check and - func_type.exception_check != '+'): + func_type.exception_check and + func_type.exception_check != '+'): env.use_utility_code(pyerr_occurred_withgil_utility_code) # C++ exception handler if func_type.exception_check == '+': @@ -5817,8 +5984,8 @@ class SimpleCallNode(CallNode): expected_nargs = max_nargs - func_type.optional_arg_count actual_nargs = len(self.args) for formal_arg, actual_arg in args[:expected_nargs]: - arg_code = actual_arg.result_as(formal_arg.type) - arg_list_code.append(arg_code) + arg_code = actual_arg.move_result_rhs_as(formal_arg.type) + arg_list_code.append(arg_code) if func_type.is_overridable: arg_list_code.append(str(int(self.wrapper_call or self.function.entry.is_unbound_cmethod))) @@ -5831,7 +5998,7 @@ class SimpleCallNode(CallNode): arg_list_code.append(optional_args) for actual_arg in self.args[len(formal_args):]: - arg_list_code.append(actual_arg.result()) + arg_list_code.append(actual_arg.move_result_rhs()) result = "%s(%s)" % (self.function.result(), ', '.join(arg_list_code)) return result @@ -5892,7 +6059,7 @@ class SimpleCallNode(CallNode): arg.py_result(), code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) for subexpr in subexprs: if subexpr is not None: @@ -5911,7 +6078,7 @@ class SimpleCallNode(CallNode): self.function.py_result(), arg_code, code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) elif func_type.is_cfunction: if self.has_optional_args: actual_nargs = len(self.args) @@ -5966,7 +6133,7 @@ class SimpleCallNode(CallNode): goto_error = "" code.putln("%s%s; %s" % (lhs, rhs, goto_error)) if self.type.is_pyobject and self.result(): - code.put_gotref(self.py_result()) + self.generate_gotref(code) if self.has_optional_args: code.funcstate.release_temp(self.opt_arg_struct) @@ -6032,10 +6199,8 @@ class PyMethodCallNode(SimpleCallNode): self_arg = code.funcstate.allocate_temp(py_object_type, manage_ref=True) code.putln("%s = NULL;" % self_arg) - arg_offset_cname = None - if len(args) > 1: - arg_offset_cname = code.funcstate.allocate_temp(PyrexTypes.c_int_type, manage_ref=False) - code.putln("%s = 0;" % arg_offset_cname) + arg_offset_cname = code.funcstate.allocate_temp(PyrexTypes.c_int_type, manage_ref=False) + code.putln("%s = 0;" % arg_offset_cname) def attribute_is_likely_method(attr): obj = attr.obj @@ -6067,116 +6232,35 @@ class PyMethodCallNode(SimpleCallNode): code.put_incref(self_arg, py_object_type) code.put_incref("function", py_object_type) # free method object as early to possible to enable reuse from CPython's freelist - code.put_decref_set(function, "function") - if len(args) > 1: - code.putln("%s = 1;" % arg_offset_cname) + code.put_decref_set(function, py_object_type, "function") + code.putln("%s = 1;" % arg_offset_cname) code.putln("}") code.putln("}") - if not args: - # fastest special case: try to avoid tuple creation - code.globalstate.use_utility_code( - UtilityCode.load_cached("PyObjectCallNoArg", "ObjectHandling.c")) - code.globalstate.use_utility_code( - UtilityCode.load_cached("PyObjectCallOneArg", "ObjectHandling.c")) - code.putln( - "%s = (%s) ? __Pyx_PyObject_CallOneArg(%s, %s) : __Pyx_PyObject_CallNoArg(%s);" % ( - self.result(), self_arg, - function, self_arg, - function)) - code.put_xdecref_clear(self_arg, py_object_type) - code.funcstate.release_temp(self_arg) - code.putln(code.error_goto_if_null(self.result(), self.pos)) - code.put_gotref(self.py_result()) - elif len(args) == 1: - # fastest special case: try to avoid tuple creation - code.globalstate.use_utility_code( - UtilityCode.load_cached("PyObjectCall2Args", "ObjectHandling.c")) - code.globalstate.use_utility_code( - UtilityCode.load_cached("PyObjectCallOneArg", "ObjectHandling.c")) - arg = args[0] - code.putln( - "%s = (%s) ? __Pyx_PyObject_Call2Args(%s, %s, %s) : __Pyx_PyObject_CallOneArg(%s, %s);" % ( - self.result(), self_arg, - function, self_arg, arg.py_result(), - function, arg.py_result())) - code.put_xdecref_clear(self_arg, py_object_type) - code.funcstate.release_temp(self_arg) - arg.generate_disposal_code(code) - arg.free_temps(code) - code.putln(code.error_goto_if_null(self.result(), self.pos)) - code.put_gotref(self.py_result()) - else: - code.globalstate.use_utility_code( - UtilityCode.load_cached("PyFunctionFastCall", "ObjectHandling.c")) - code.globalstate.use_utility_code( - UtilityCode.load_cached("PyCFunctionFastCall", "ObjectHandling.c")) - for test_func, call_prefix in [('PyFunction_Check', 'Py'), ('__Pyx_PyFastCFunction_Check', 'PyC')]: - code.putln("#if CYTHON_FAST_%sCALL" % call_prefix.upper()) - code.putln("if (%s(%s)) {" % (test_func, function)) - code.putln("PyObject *%s[%d] = {%s, %s};" % ( - Naming.quick_temp_cname, - len(args)+1, - self_arg, - ', '.join(arg.py_result() for arg in args))) - code.putln("%s = __Pyx_%sFunction_FastCall(%s, %s+1-%s, %d+%s); %s" % ( - self.result(), - call_prefix, - function, - Naming.quick_temp_cname, - arg_offset_cname, - len(args), - arg_offset_cname, - code.error_goto_if_null(self.result(), self.pos))) - code.put_xdecref_clear(self_arg, py_object_type) - code.put_gotref(self.py_result()) - for arg in args: - arg.generate_disposal_code(code) - code.putln("} else") - code.putln("#endif") - - code.putln("{") - args_tuple = code.funcstate.allocate_temp(py_object_type, manage_ref=True) - code.putln("%s = PyTuple_New(%d+%s); %s" % ( - args_tuple, len(args), arg_offset_cname, - code.error_goto_if_null(args_tuple, self.pos))) - code.put_gotref(args_tuple) - - if len(args) > 1: - code.putln("if (%s) {" % self_arg) - code.putln("__Pyx_GIVEREF(%s); PyTuple_SET_ITEM(%s, 0, %s); %s = NULL;" % ( - self_arg, args_tuple, self_arg, self_arg)) # stealing owned ref in this case - code.funcstate.release_temp(self_arg) - if len(args) > 1: - code.putln("}") - - for i, arg in enumerate(args): - arg.make_owned_reference(code) - code.put_giveref(arg.py_result()) - code.putln("PyTuple_SET_ITEM(%s, %d+%s, %s);" % ( - args_tuple, i, arg_offset_cname, arg.py_result())) - if len(args) > 1: - code.funcstate.release_temp(arg_offset_cname) - - for arg in args: - arg.generate_post_assignment_code(code) - arg.free_temps(code) - - code.globalstate.use_utility_code( - UtilityCode.load_cached("PyObjectCall", "ObjectHandling.c")) - code.putln( - "%s = __Pyx_PyObject_Call(%s, %s, NULL); %s" % ( - self.result(), - function, args_tuple, - code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + # actually call the function + code.globalstate.use_utility_code( + UtilityCode.load_cached("PyObjectFastCall", "ObjectHandling.c")) - code.put_decref_clear(args_tuple, py_object_type) - code.funcstate.release_temp(args_tuple) + code.putln("{") + code.putln("PyObject *__pyx_callargs[%d] = {%s, %s};" % ( + len(args)+1, + self_arg, + ', '.join(arg.py_result() for arg in args))) + code.putln("%s = __Pyx_PyObject_FastCall(%s, __pyx_callargs+1-%s, %d+%s);" % ( + self.result(), + function, + arg_offset_cname, + len(args), + arg_offset_cname)) - if len(args) == 1: - code.putln("}") - code.putln("}") # !CYTHON_FAST_PYCALL + code.put_xdecref_clear(self_arg, py_object_type) + code.funcstate.release_temp(self_arg) + code.funcstate.release_temp(arg_offset_cname) + for arg in args: + arg.generate_disposal_code(code) + arg.free_temps(code) + code.putln(code.error_goto_if_null(self.result(), self.pos)) + self.generate_gotref(code) if reuse_function_temp: self.function.generate_disposal_code(code) @@ -6184,6 +6268,7 @@ class PyMethodCallNode(SimpleCallNode): else: code.put_decref_clear(function, py_object_type) code.funcstate.release_temp(function) + code.putln("}") class InlinedDefNodeCallNode(CallNode): @@ -6234,7 +6319,7 @@ class InlinedDefNodeCallNode(CallNode): # but we must make sure it cannot be collected # before we return from the function, so we create # an owned temp reference to it - if i > 0: # first argument doesn't matter + if i > 0: # first argument doesn't matter some_args_in_temps = True arg = arg.coerce_to_temp(env) self.args[i] = arg @@ -6281,7 +6366,7 @@ class InlinedDefNodeCallNode(CallNode): self.function.def_node.entry.pyfunc_cname, arg_code, code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) class PythonCapiFunctionNode(ExprNode): @@ -6351,7 +6436,7 @@ class CachedBuiltinMethodCallNode(CallNode): self.result(), call_code, code.error_goto_if_null(self.result(), self.pos) )) - code.put_gotref(self.result()) + self.generate_gotref(code) class GeneralCallNode(CallNode): @@ -6442,7 +6527,7 @@ class GeneralCallNode(CallNode): kwargs = self.keyword_args declared_args = function_type.args if entry.is_cmethod: - declared_args = declared_args[1:] # skip 'self' + declared_args = declared_args[1:] # skip 'self' if len(pos_args) > len(declared_args): error(self.pos, "function call got too many positional arguments, " @@ -6450,8 +6535,10 @@ class GeneralCallNode(CallNode): len(pos_args))) return None - matched_args = set([ arg.name for arg in declared_args[:len(pos_args)] - if arg.name ]) + matched_args = { + arg.name for arg in declared_args[:len(pos_args)] + if arg.name + } unmatched_args = declared_args[len(pos_args):] matched_kwargs_count = 0 args = list(pos_args) @@ -6569,7 +6656,7 @@ class GeneralCallNode(CallNode): self.positional_args.py_result(), kwargs, code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) class AsTupleNode(ExprNode): @@ -6611,7 +6698,7 @@ class AsTupleNode(ExprNode): self.result(), cfunc, self.arg.py_result(), code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) class MergedDictNode(ExprNode): @@ -6704,16 +6791,18 @@ class MergedDictNode(ExprNode): self.result(), item.py_result(), code.error_goto_if_null(self.result(), item.pos))) - code.put_gotref(self.result()) + self.generate_gotref(code) item.generate_disposal_code(code) if item.type is not dict_type: code.putln('} else {') - code.putln("%s = PyObject_CallFunctionObjArgs((PyObject*)&PyDict_Type, %s, NULL); %s" % ( + code.globalstate.use_utility_code(UtilityCode.load_cached( + "PyObjectCallOneArg", "ObjectHandling.c")) + code.putln("%s = __Pyx_PyObject_CallOneArg((PyObject*)&PyDict_Type, %s); %s" % ( self.result(), item.py_result(), code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) item.generate_disposal_code(code) code.putln('}') item.free_temps(code) @@ -6882,7 +6971,7 @@ class AttributeNode(ExprNode): if node is None: node = self.analyse_as_ordinary_attribute_node(env, target) assert node is not None - if node.entry: + if (node.is_attribute or node.is_name) and node.entry: node.entry.used = True if node.is_attribute: node.wrap_obj_in_nonecheck(env) @@ -6947,7 +7036,7 @@ class AttributeNode(ExprNode): ubcm_entry.is_unbound_cmethod = 1 ubcm_entry.scope = entry.scope return self.as_name_node(env, ubcm_entry, target=False) - elif type.is_enum: + elif type.is_enum or type.is_cpp_enum: if self.attribute in type.values: for entry in type.entry.enum_values: if entry.name == self.attribute: @@ -6964,7 +7053,7 @@ class AttributeNode(ExprNode): return module_scope.lookup_type(self.attribute) if not self.obj.is_string_literal: base_type = self.obj.analyse_as_type(env) - if base_type and hasattr(base_type, 'scope') and base_type.scope is not None: + if base_type and getattr(base_type, 'scope', None) is not None: return base_type.scope.lookup_type(self.attribute) return None @@ -7015,13 +7104,18 @@ class AttributeNode(ExprNode): self.result_ctype = py_object_type elif target and self.obj.type.is_builtin_type: error(self.pos, "Assignment to an immutable object field") + elif self.entry and self.entry.is_cproperty: + if not target: + return SimpleCallNode.for_cproperty(self.pos, self.obj, self.entry).analyse_types(env) + # TODO: implement writable C-properties? + error(self.pos, "Assignment to a read-only property") #elif self.type.is_memoryviewslice and not target: # self.is_temp = True return self def analyse_attribute(self, env, obj_type = None): # Look up attribute and set self.type and self.member. - immutable_obj = obj_type is not None # used during type inference + immutable_obj = obj_type is not None # used during type inference self.is_py_attr = 0 self.member = self.attribute if obj_type is None: @@ -7071,7 +7165,10 @@ class AttributeNode(ExprNode): # fused function go through assignment synthesis # (foo = pycfunction(foo_func_obj)) and need to go through # regular Python lookup as well - if (entry.is_variable and not entry.fused_cfunction) or entry.is_cmethod: + if entry.is_cproperty: + self.type = entry.type + return + elif (entry.is_variable and not entry.fused_cfunction) or entry.is_cmethod: self.type = entry.type self.member = entry.cname return @@ -7098,15 +7195,15 @@ class AttributeNode(ExprNode): if not obj_type.is_pyobject and not obj_type.is_error: # Expose python methods for immutable objects. if (obj_type.is_string or obj_type.is_cpp_string - or obj_type.is_buffer or obj_type.is_memoryviewslice - or obj_type.is_numeric - or (obj_type.is_ctuple and obj_type.can_coerce_to_pyobject(env)) - or (obj_type.is_struct and obj_type.can_coerce_to_pyobject(env))): + or obj_type.is_buffer or obj_type.is_memoryviewslice + or obj_type.is_numeric + or (obj_type.is_ctuple and obj_type.can_coerce_to_pyobject(env)) + or (obj_type.is_struct and obj_type.can_coerce_to_pyobject(env))): if not immutable_obj: self.obj = self.obj.coerce_to_pyobject(env) elif (obj_type.is_cfunction and (self.obj.is_name or self.obj.is_attribute) - and self.obj.entry.as_variable - and self.obj.entry.as_variable.type.is_pyobject): + and self.obj.entry.as_variable + and self.obj.entry.as_variable.type.is_pyobject): # might be an optimised builtin function => unpack it if not immutable_obj: self.obj = self.obj.coerce_to_pyobject(env) @@ -7220,7 +7317,7 @@ class AttributeNode(ExprNode): self.obj.py_result(), code.intern_identifier(self.attribute), code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) elif self.type.is_memoryviewslice: if self.is_memslice_transpose: # transpose the slice @@ -7231,10 +7328,11 @@ class AttributeNode(ExprNode): return code.putln("%s = %s;" % (self.result(), self.obj.result())) - code.put_incref_memoryviewslice(self.result(), have_gil=True) + code.put_incref_memoryviewslice(self.result(), self.type, + have_gil=True) - T = "__pyx_memslice_transpose(&%s) == 0" - code.putln(code.error_goto_if(T % self.result(), self.pos)) + T = "__pyx_memslice_transpose(&%s)" % self.result() + code.putln(code.error_goto_if_neg(T, self.pos)) elif self.initialized_check: code.putln( 'if (unlikely(!%s.memview)) {' @@ -7254,15 +7352,12 @@ class AttributeNode(ExprNode): def generate_disposal_code(self, code): if self.is_temp and self.type.is_memoryviewslice and self.is_memslice_transpose: # mirror condition for putting the memview incref here: - code.put_xdecref_memoryviewslice( - self.result(), have_gil=True) - code.putln("%s.memview = NULL;" % self.result()) - code.putln("%s.data = NULL;" % self.result()) + code.put_xdecref_clear(self.result(), self.type, have_gil=True) else: ExprNode.generate_disposal_code(self, code) def generate_assignment_code(self, rhs, code, overloaded_assignment=False, - exception_check=None, exception_value=None): + exception_check=None, exception_value=None): self.obj.generate_evaluation_code(code) if self.is_py_attr: code.globalstate.use_utility_code( @@ -7285,8 +7380,8 @@ class AttributeNode(ExprNode): select_code = self.result() if self.type.is_pyobject and self.use_managed_ref: rhs.make_owned_reference(code) - code.put_giveref(rhs.py_result()) - code.put_gotref(select_code) + rhs.generate_giveref(code) + code.put_gotref(select_code, self.type) code.put_decref(select_code, self.ctype()) elif self.type.is_memoryviewslice: from . import MemoryView @@ -7297,7 +7392,7 @@ class AttributeNode(ExprNode): code.putln( "%s = %s;" % ( select_code, - rhs.result_as(self.ctype()))) + rhs.move_result_rhs_as(self.ctype()))) #rhs.result())) rhs.generate_post_assignment_code(code) rhs.free_temps(code) @@ -7532,7 +7627,7 @@ class SequenceNode(ExprNode): len(self.args), ', '.join(arg.py_result() for arg in self.args), code.error_goto_if_null(target, self.pos))) - code.put_gotref(target) + code.put_gotref(target, py_object_type) elif self.type.is_ctuple: for i, arg in enumerate(self.args): code.putln("%s.f%s = %s;" % ( @@ -7549,7 +7644,7 @@ class SequenceNode(ExprNode): code.putln("%s = %s(%s%s); %s" % ( target, create_func, arg_count, size_factor, code.error_goto_if_null(target, self.pos))) - code.put_gotref(target) + code.put_gotref(target, py_object_type) if c_mult: # FIXME: can't use a temp variable here as the code may @@ -7573,7 +7668,7 @@ class SequenceNode(ExprNode): arg = self.args[i] if c_mult or not arg.result_in_temp(): code.put_incref(arg.result(), arg.ctype()) - code.put_giveref(arg.py_result()) + arg.generate_giveref(code) code.putln("%s(%s, %s, %s);" % ( set_item_func, target, @@ -7590,7 +7685,7 @@ class SequenceNode(ExprNode): Naming.quick_temp_cname, target, mult_factor.py_result(), code.error_goto_if_null(Naming.quick_temp_cname, self.pos) )) - code.put_gotref(Naming.quick_temp_cname) + code.put_gotref(Naming.quick_temp_cname, py_object_type) code.put_decref(target, py_object_type) code.putln('%s = %s;' % (target, Naming.quick_temp_cname)) code.putln('}') @@ -7612,7 +7707,7 @@ class SequenceNode(ExprNode): self.mult_factor.generate_disposal_code(code) def generate_assignment_code(self, rhs, code, overloaded_assignment=False, - exception_check=None, exception_value=None): + exception_check=None, exception_value=None): if self.starred_assignment: self.generate_starred_assignment_code(rhs, code) else: @@ -7708,7 +7803,7 @@ class SequenceNode(ExprNode): code.putln("%s = PySequence_ITEM(sequence, %d); %s" % ( item.result(), i, code.error_goto_if_null(item.result(), self.pos))) - code.put_gotref(item.result()) + code.put_gotref(item.result(), item.type) else: code.putln("{") code.putln("Py_ssize_t i;") @@ -7718,7 +7813,7 @@ class SequenceNode(ExprNode): code.putln("for (i=0; i < %s; i++) {" % len(self.unpacked_items)) code.putln("PyObject* item = PySequence_ITEM(sequence, i); %s" % ( code.error_goto_if_null('item', self.pos))) - code.put_gotref('item') + code.put_gotref('item', py_object_type) code.putln("*(temps[i]) = item;") code.putln("}") code.putln("}") @@ -7744,7 +7839,7 @@ class SequenceNode(ExprNode): def generate_generic_parallel_unpacking_code(self, code, rhs, unpacked_items, use_loop, terminate=True): code.globalstate.use_utility_code(raise_need_more_values_to_unpack) code.globalstate.use_utility_code(UtilityCode.load_cached("IterFinish", "ObjectHandling.c")) - code.putln("Py_ssize_t index = -1;") # must be at the start of a C block! + code.putln("Py_ssize_t index = -1;") # must be at the start of a C block! if use_loop: code.putln("PyObject** temps[%s] = {%s};" % ( @@ -7757,11 +7852,11 @@ class SequenceNode(ExprNode): iterator_temp, rhs.py_result(), code.error_goto_if_null(iterator_temp, self.pos))) - code.put_gotref(iterator_temp) + code.put_gotref(iterator_temp, py_object_type) rhs.generate_disposal_code(code) iternext_func = code.funcstate.allocate_temp(self._func_iternext_type, manage_ref=False) - code.putln("%s = Py_TYPE(%s)->tp_iternext;" % ( + code.putln("%s = __Pyx_PyObject_GetIterNextFunc(%s);" % ( iternext_func, iterator_temp)) unpacking_error_label = code.new_label('unpacking_failed') @@ -7770,7 +7865,7 @@ class SequenceNode(ExprNode): code.putln("for (index=0; index < %s; index++) {" % len(unpacked_items)) code.put("PyObject* item = %s; if (unlikely(!item)) " % unpack_code) code.put_goto(unpacking_error_label) - code.put_gotref("item") + code.put_gotref("item", py_object_type) code.putln("*(temps[index]) = item;") code.putln("}") else: @@ -7782,7 +7877,7 @@ class SequenceNode(ExprNode): unpack_code, item.result())) code.put_goto(unpacking_error_label) - code.put_gotref(item.py_result()) + item.generate_gotref(code) if terminate: code.globalstate.use_utility_code( @@ -7835,11 +7930,14 @@ class SequenceNode(ExprNode): starred_target.allocate(code) target_list = starred_target.result() - code.putln("%s = PySequence_List(%s); %s" % ( + code.putln("%s = %s(%s); %s" % ( target_list, + "__Pyx_PySequence_ListKeepNew" if ( + not iterator_temp and rhs.is_temp and rhs.type in (py_object_type, list_type)) + else "PySequence_List", iterator_temp or rhs.py_result(), code.error_goto_if_null(target_list, self.pos))) - code.put_gotref(target_list) + starred_target.generate_gotref(code) if iterator_temp: code.put_decref_clear(iterator_temp, py_object_type) @@ -7870,7 +7968,7 @@ class SequenceNode(ExprNode): code.putln("%s = PySequence_ITEM(%s, %s-%d); " % ( item.py_result(), target_list, length_temp, i+1)) code.putln('#endif') - code.put_gotref(item.py_result()) + item.generate_gotref(code) coerced_arg.generate_evaluation_code(code) code.putln('#if !CYTHON_COMPILING_IN_CPYTHON') @@ -7878,7 +7976,7 @@ class SequenceNode(ExprNode): code.putln('%s = PySequence_GetSlice(%s, 0, %s-%d); %s' % ( sublist_temp, target_list, length_temp, len(unpacked_fixed_items_right), code.error_goto_if_null(sublist_temp, self.pos))) - code.put_gotref(sublist_temp) + code.put_gotref(sublist_temp, py_object_type) code.funcstate.release_temp(length_temp) code.put_decref(target_list, py_object_type) code.putln('%s = %s; %s = NULL;' % (target_list, sublist_temp, sublist_temp)) @@ -8024,7 +8122,7 @@ class TupleNode(SequenceNode): # constant is not yet initialised const_code.mark_pos(self.pos) self.generate_sequence_packing_code(const_code, tuple_target, plain=not self.is_literal) - const_code.put_giveref(tuple_target) + const_code.put_giveref(tuple_target, py_object_type) if self.is_literal: self.result_code = tuple_target else: @@ -8032,7 +8130,7 @@ class TupleNode(SequenceNode): self.result(), tuple_target, self.mult_factor.py_result(), code.error_goto_if_null(self.result(), self.pos) )) - code.put_gotref(self.py_result()) + self.generate_gotref(code) else: self.type.entry.used = True self.generate_sequence_packing_code(code) @@ -8282,7 +8380,7 @@ class ScopedExprNode(ExprNode): for entry in py_entries: if entry.is_cglobal: code.put_var_gotref(entry) - code.put_decref_set(entry.cname, "Py_None") + code.put_var_decref_set(entry, "Py_None") else: code.put_var_xdecref_clear(entry) @@ -8299,7 +8397,7 @@ class ComprehensionNode(ScopedExprNode): return self.type def analyse_declarations(self, env): - self.append.target = self # this is used in the PyList_Append of the inner loop + self.append.target = self # this is used in the PyList_Append of the inner loop self.init_scope(env) def analyse_scoped_declarations(self, env): @@ -8334,7 +8432,7 @@ class ComprehensionNode(ScopedExprNode): self.result(), create_code, code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.result()) + self.generate_gotref(code) self.loop.generate_execution_code(code) def annotate(self, code): @@ -8459,7 +8557,7 @@ class InlinedGeneratorExpressionNode(ExprNode): code.putln("%s = __Pyx_Generator_Next(%s); %s" % ( self.result(), self.gen.result(), code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.result()) + self.generate_gotref(code) class MergedSequenceNode(ExprNode): @@ -8567,10 +8665,12 @@ class MergedSequenceNode(ExprNode): else: code.putln("%s = %s(%s); %s" % ( self.result(), - 'PySet_New' if is_set else 'PySequence_List', + 'PySet_New' if is_set + else "__Pyx_PySequence_ListKeepNew" if item.is_temp and item.type in (py_object_type, list_type) + else "PySequence_List", item.py_result(), code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) item.generate_disposal_code(code) item.free_temps(code) @@ -8620,7 +8720,7 @@ class MergedSequenceNode(ExprNode): self.result(), Naming.quick_temp_cname, code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.result()) + self.generate_gotref(code) code.putln("}") for helper in sorted(helpers): @@ -8653,7 +8753,7 @@ class SetNode(ExprNode): return False def calculate_constant_result(self): - self.constant_result = set([arg.constant_result for arg in self.args]) + self.constant_result = {arg.constant_result for arg in self.args} def compile_time_value(self, denv): values = [arg.compile_time_value(denv) for arg in self.args] @@ -8670,7 +8770,7 @@ class SetNode(ExprNode): "%s = PySet_New(0); %s" % ( self.result(), code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) for arg in self.args: code.put_error_if_neg( self.pos, @@ -8757,7 +8857,7 @@ class DictNode(ExprNode): error(item.key.pos, "Invalid struct field identifier") item.key = StringNode(item.key.pos, value="<error>") else: - key = str(item.key.value) # converts string literals to unicode in Py3 + key = str(item.key.value) # converts string literals to unicode in Py3 member = dst_type.scope.lookup_here(key) if not member: error(item.key.pos, "struct '%s' has no field '%s'" % (dst_type, key)) @@ -8792,7 +8892,7 @@ class DictNode(ExprNode): self.result(), len(self.key_value_pairs), code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) keys_seen = set() key_type = None @@ -8864,7 +8964,7 @@ class DictItemNode(ExprNode): # value ExprNode subexprs = ['key', 'value'] - nogil_check = None # Parent DictNode takes care of it + nogil_check = None # Parent DictNode takes care of it def calculate_constant_result(self): self.constant_result = ( @@ -8920,7 +9020,7 @@ class SortedDictKeysNode(ExprNode): code.putln('%s = PyDict_Keys(%s); %s' % ( self.result(), dict_result, code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) else: # originally used PyMapping_Keys() here, but that may return a tuple code.globalstate.use_utility_code(UtilityCode.load_cached( @@ -8929,11 +9029,11 @@ class SortedDictKeysNode(ExprNode): code.putln('%s = __Pyx_PyObject_CallMethod0(%s, %s); %s' % ( self.result(), dict_result, keys_cname, code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) code.putln("if (unlikely(!PyList_Check(%s))) {" % self.result()) - code.put_decref_set(self.result(), "PySequence_List(%s)" % self.result()) + self.generate_decref_set(code, "PySequence_List(%s)" % self.result()) code.putln(code.error_goto_if_null(self.result(), self.pos)) - code.put_gotref(self.py_result()) + self.generate_gotref(code) code.putln("}") code.put_error_if_neg( self.pos, 'PyList_Sort(%s)' % self.py_result()) @@ -8963,6 +9063,9 @@ class ClassNode(ExprNode, ModuleNameMixin): type = py_object_type is_temp = True + def analyse_annotations(self, env): + pass + def infer_type(self, env): # TODO: could return 'type' in some cases return py_object_type @@ -9001,7 +9104,7 @@ class ClassNode(ExprNode, ModuleNameMixin): qualname, py_mod_name, code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) class Py3ClassNode(ExprNode): @@ -9014,9 +9117,11 @@ class Py3ClassNode(ExprNode): # class_def_node PyClassDefNode PyClassDefNode defining this class # calculate_metaclass bool should call CalculateMetaclass() # allow_py2_metaclass bool should look for Py2 metaclass + # force_type bool always create a "new style" class, even with no bases subexprs = [] type = py_object_type + force_type = False is_temp = True def infer_type(self, env): @@ -9031,6 +9136,26 @@ class Py3ClassNode(ExprNode): gil_message = "Constructing Python class" + def analyse_annotations(self, env): + from .AutoDocTransforms import AnnotationWriter + position = self.class_def_node.pos + dict_items = [ + DictItemNode( + entry.pos, + key=IdentifierStringNode(entry.pos, value=entry.name), + value=entry.annotation.string + ) + for entry in env.entries.values() if entry.annotation + ] + # Annotations dict shouldn't exist for classes which don't declare any. + if dict_items: + annotations_dict = DictNode(position, key_value_pairs=dict_items) + lhs = NameNode(position, name=StringEncoding.EncodedString(u"__annotations__")) + lhs.entry = env.lookup_here(lhs.name) or env.declare_var(lhs.name, dict_type, position) + node = SingleAssignmentNode(position, lhs=lhs, rhs=annotations_dict) + node.analyse_declarations(env) + self.class_def_node.body.stats.insert(0, node) + def generate_result_code(self, code): code.globalstate.use_utility_code(UtilityCode.load_cached("Py3ClassCreate", "ObjectHandling.c")) cname = code.intern_identifier(self.name) @@ -9038,6 +9163,8 @@ class Py3ClassNode(ExprNode): mkw = class_def_node.mkw.py_result() if class_def_node.mkw else 'NULL' if class_def_node.metaclass: metaclass = class_def_node.metaclass.py_result() + elif self.force_type: + metaclass = "((PyObject*)&PyType_Type)" else: metaclass = "((PyObject*)&__Pyx_DefaultClassType)" code.putln( @@ -9051,7 +9178,7 @@ class Py3ClassNode(ExprNode): self.calculate_metaclass, self.allow_py2_metaclass, code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) class PyClassMetaclassNode(ExprNode): @@ -9087,7 +9214,7 @@ class PyClassMetaclassNode(ExprNode): "%s = %s; %s" % ( self.result(), call, code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) class PyClassNamespaceNode(ExprNode, ModuleNameMixin): @@ -9129,7 +9256,7 @@ class PyClassNamespaceNode(ExprNode, ModuleNameMixin): py_mod_name, doc_code, code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) class ClassCellInjectorNode(ExprNode): @@ -9148,7 +9275,7 @@ class ClassCellInjectorNode(ExprNode): '%s = PyList_New(0); %s' % ( self.result(), code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.result()) + self.generate_gotref(code) def generate_injection_code(self, code, classobj_cname): assert self.is_active @@ -9190,7 +9317,6 @@ class PyCFunctionNode(ExprNode, ModuleNameMixin): # from a PyMethodDef struct. # # pymethdef_cname string PyMethodDef structure - # self_object ExprNode or None # binding bool # def_node DefNode the Python function node # module_name EncodedString Name of defining module @@ -9199,7 +9325,6 @@ class PyCFunctionNode(ExprNode, ModuleNameMixin): subexprs = ['code_object', 'defaults_tuple', 'defaults_kwdict', 'annotations_dict'] - self_object = None code_object = None binding = False def_node = None @@ -9262,19 +9387,19 @@ class PyCFunctionNode(ExprNode, ModuleNameMixin): else: default_args.append(arg) if arg.annotation: - arg.annotation = self.analyse_annotation(env, arg.annotation) - annotations.append((arg.pos, arg.name, arg.annotation)) + arg.annotation = arg.annotation.analyse_types(env) + annotations.append((arg.pos, arg.name, arg.annotation.string)) for arg in (self.def_node.star_arg, self.def_node.starstar_arg): if arg and arg.annotation: - arg.annotation = self.analyse_annotation(env, arg.annotation) - annotations.append((arg.pos, arg.name, arg.annotation)) + arg.annotation = arg.annotation.analyse_types(env) + annotations.append((arg.pos, arg.name, arg.annotation.string)) annotation = self.def_node.return_type_annotation if annotation: - annotation = self.analyse_annotation(env, annotation) - self.def_node.return_type_annotation = annotation - annotations.append((annotation.pos, StringEncoding.EncodedString("return"), annotation)) + self.def_node.return_type_annotation = annotation.analyse_types(env) + annotations.append((annotation.pos, StringEncoding.EncodedString("return"), + annotation.string)) if nonliteral_objects or nonliteral_other: module_scope = env.global_scope() @@ -9282,7 +9407,10 @@ class PyCFunctionNode(ExprNode, ModuleNameMixin): scope = Symtab.StructOrUnionScope(cname) self.defaults = [] for arg in nonliteral_objects: - entry = scope.declare_var(arg.name, arg.type, None, + type_ = arg.type + if type_.is_buffer: + type_ = type_.base + entry = scope.declare_var(arg.name, type_, None, Naming.arg_prefix + arg.name, allow_pyobject=True) self.defaults.append((arg, entry)) @@ -9314,7 +9442,8 @@ class PyCFunctionNode(ExprNode, ModuleNameMixin): value=arg.default) for arg in default_kwargs]) self.defaults_kwdict = defaults_kwdict.analyse_types(env) - else: + elif not self.specialized_cpdefs: + # Fused dispatch functions do not support (dynamic) default arguments, only the specialisations do. if default_args: defaults_tuple = DefaultsTupleNode( self.pos, default_args, self.defaults_struct) @@ -9351,31 +9480,13 @@ class PyCFunctionNode(ExprNode, ModuleNameMixin): for pos, name, value in annotations]) self.annotations_dict = annotations_dict.analyse_types(env) - def analyse_annotation(self, env, annotation): - if annotation is None: - return None - atype = annotation.analyse_as_type(env) - if atype is not None: - # Keep parsed types as strings as they might not be Python representable. - annotation = UnicodeNode( - annotation.pos, - value=StringEncoding.EncodedString(atype.declaration_code('', for_display=True))) - annotation = annotation.analyse_types(env) - if not annotation.type.is_pyobject: - annotation = annotation.coerce_to_pyobject(env) - return annotation - def may_be_none(self): return False gil_message = "Constructing Python function" - def self_result_code(self): - if self.self_object is None: - self_result = "NULL" - else: - self_result = self.self_object.py_result() - return self_result + def closure_result_code(self): + return "NULL" def generate_result_code(self, code): if self.binding: @@ -9389,11 +9500,11 @@ class PyCFunctionNode(ExprNode, ModuleNameMixin): '%s = PyCFunction_NewEx(&%s, %s, %s); %s' % ( self.result(), self.pymethdef_cname, - self.self_result_code(), + self.closure_result_code(), py_mod_name, code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) def generate_cyfunction_code(self, code): if self.specialized_cpdefs: @@ -9424,6 +9535,9 @@ class PyCFunctionNode(ExprNode, ModuleNameMixin): if def_node.local_scope.parent_scope.is_c_class_scope and not def_node.entry.is_anonymous: flags.append('__Pyx_CYFUNCTION_CCLASS') + if def_node.is_coroutine: + flags.append('__Pyx_CYFUNCTION_COROUTINE') + if flags: flags = ' | '.join(flags) else: @@ -9436,13 +9550,13 @@ class PyCFunctionNode(ExprNode, ModuleNameMixin): self.pymethdef_cname, flags, self.get_py_qualified_name(code), - self.self_result_code(), + self.closure_result_code(), self.get_py_mod_name(code), Naming.moddict_cname, code_object_result, code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) if def_node.requires_classobj: assert code.pyclass_stack, "pyclass_stack is empty" @@ -9452,7 +9566,7 @@ class PyCFunctionNode(ExprNode, ModuleNameMixin): 'PyList_Append(%s, %s);' % ( class_node.class_cell.result(), self.result())) - code.put_giveref(self.py_result()) + self.generate_giveref(code) if self.defaults: code.putln( @@ -9468,27 +9582,29 @@ class PyCFunctionNode(ExprNode, ModuleNameMixin): if self.defaults_tuple: code.putln('__Pyx_CyFunction_SetDefaultsTuple(%s, %s);' % ( self.result(), self.defaults_tuple.py_result())) - if self.defaults_kwdict: - code.putln('__Pyx_CyFunction_SetDefaultsKwDict(%s, %s);' % ( - self.result(), self.defaults_kwdict.py_result())) - if def_node.defaults_getter and not self.specialized_cpdefs: - # Fused functions do not support dynamic defaults, only their specialisations can have them for now. - code.putln('__Pyx_CyFunction_SetDefaultsGetter(%s, %s);' % ( - self.result(), def_node.defaults_getter.entry.pyfunc_cname)) - if self.annotations_dict: - code.putln('__Pyx_CyFunction_SetAnnotationsDict(%s, %s);' % ( - self.result(), self.annotations_dict.py_result())) + if not self.specialized_cpdefs: + # disable introspection functions for fused dispatcher function since the user never sees it + # TODO: this is mostly disabled because the attributes end up pointing to ones belonging + # to the specializations - ideally this would be fixed instead + if self.defaults_kwdict: + code.putln('__Pyx_CyFunction_SetDefaultsKwDict(%s, %s);' % ( + self.result(), self.defaults_kwdict.py_result())) + if def_node.defaults_getter: + code.putln('__Pyx_CyFunction_SetDefaultsGetter(%s, %s);' % ( + self.result(), def_node.defaults_getter.entry.pyfunc_cname)) + if self.annotations_dict: + code.putln('__Pyx_CyFunction_SetAnnotationsDict(%s, %s);' % ( + self.result(), self.annotations_dict.py_result())) class InnerFunctionNode(PyCFunctionNode): # Special PyCFunctionNode that depends on a closure class - # binding = True - needs_self_code = True + needs_closure_code = True - def self_result_code(self): - if self.needs_self_code: + def closure_result_code(self): + if self.needs_closure_code: return "((PyObject*)%s)" % Naming.cur_scope_cname return "NULL" @@ -9546,9 +9662,10 @@ class CodeObjectNode(ExprNode): if self.def_node.starstar_arg: flags.append('CO_VARKEYWORDS') - code.putln("%s = (PyObject*)__Pyx_PyCode_New(%d, %d, %d, 0, %s, %s, %s, %s, %s, %s, %s, %s, %s, %d, %s); %s" % ( + code.putln("%s = (PyObject*)__Pyx_PyCode_New(%d, %d, %d, %d, 0, %s, %s, %s, %s, %s, %s, %s, %s, %s, %d, %s); %s" % ( self.result_code, len(func.args) - func.num_kwonly_args, # argcount + func.num_posonly_args, # posonlyargcount (Py3.8+ only) func.num_kwonly_args, # kwonlyargcount (Py3 only) len(self.varnames.args), # nlocals '|'.join(flags) or '0', # flags @@ -9670,8 +9787,8 @@ class LambdaNode(InnerFunctionNode): self.lambda_name = self.def_node.lambda_name = env.next_id('lambda') self.def_node.no_assignment_synthesis = True self.def_node.pymethdef_required = True - self.def_node.analyse_declarations(env) self.def_node.is_cyfunction = True + self.def_node.analyse_declarations(env) self.pymethdef_cname = self.def_node.entry.pymethdef_cname env.add_lambda_def(self.def_node) @@ -9710,9 +9827,9 @@ class GeneratorExpressionNode(LambdaNode): '%s = %s(%s); %s' % ( self.result(), self.def_node.entry.pyfunc_cname, - self.self_result_code(), + self.closure_result_code(), code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) class YieldExprNode(ExprNode): @@ -9771,11 +9888,10 @@ class YieldExprNode(ExprNode): for cname, type, manage_ref in code.funcstate.temps_in_use(): save_cname = code.funcstate.closure_temps.allocate_temp(type) saved.append((cname, save_cname, type)) - if type.is_pyobject: - code.put_xgiveref(cname) + code.put_xgiveref(cname, type) code.putln('%s->%s = %s;' % (Naming.cur_scope_cname, save_cname, cname)) - code.put_xgiveref(Naming.retval_cname) + code.put_xgiveref(Naming.retval_cname, py_object_type) profile = code.globalstate.directives['profile'] linetrace = code.globalstate.directives['linetrace'] if profile or linetrace: @@ -9806,7 +9922,7 @@ class YieldExprNode(ExprNode): code.putln('%s = %s->%s;' % (cname, Naming.cur_scope_cname, save_cname)) if type.is_pyobject: code.putln('%s->%s = 0;' % (Naming.cur_scope_cname, save_cname)) - code.put_xgotref(cname) + code.put_xgotref(cname, type) self.generate_sent_value_handling_code(code, Naming.sent_value_cname) if self.result_is_used: self.allocate_temp_result(code) @@ -9834,7 +9950,7 @@ class _YieldDelegationExprNode(YieldExprNode): self.arg.free_temps(code) elif decref_source: code.put_decref_clear(source_cname, py_object_type) - code.put_xgotref(Naming.retval_cname) + code.put_xgotref(Naming.retval_cname, py_object_type) code.putln("if (likely(%s)) {" % Naming.retval_cname) self.generate_yield_code(code) @@ -9850,7 +9966,7 @@ class _YieldDelegationExprNode(YieldExprNode): # YieldExprNode has allocated the result temp for us code.putln("%s = NULL;" % self.result()) code.put_error_if_neg(self.pos, "__Pyx_PyGen_FetchStopIterationValue(&%s)" % self.result()) - code.put_gotref(self.result()) + self.generate_gotref(code) def handle_iteration_exception(self, code): code.putln("PyObject* exc_type = __Pyx_PyErr_Occurred();") @@ -9942,7 +10058,7 @@ class GlobalsExprNode(AtomicExprNode): code.putln('%s = __Pyx_Globals(); %s' % ( self.result(), code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.result()) + self.generate_gotref(code) class LocalsDictItemNode(DictItemNode): @@ -10132,7 +10248,7 @@ class UnopNode(ExprNode): function, self.operand.py_result(), code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) def type_error(self): if not self.operand.type.is_error: @@ -10284,7 +10400,10 @@ class DereferenceNode(CUnopNode): def analyse_c_operation(self, env): if self.operand.type.is_ptr: - self.type = self.operand.type.base_type + if env.is_cpp: + self.type = PyrexTypes.CReferenceType(self.operand.type.base_type) + else: + self.type = self.operand.type.base_type else: self.type_error() @@ -10710,7 +10829,7 @@ class CythonArrayNode(ExprNode): type_info, code.error_goto_if_null(format_temp, self.pos), )) - code.put_gotref(format_temp) + code.put_gotref(format_temp, py_object_type) buildvalue_fmt = " __PYX_BUILD_PY_SSIZE_T " * len(shapes) code.putln('%s = Py_BuildValue((char*) "(" %s ")", %s); %s' % ( @@ -10719,15 +10838,14 @@ class CythonArrayNode(ExprNode): ", ".join(shapes), code.error_goto_if_null(shapes_temp, self.pos), )) - code.put_gotref(shapes_temp) + code.put_gotref(shapes_temp, py_object_type) - tup = (self.result(), shapes_temp, itemsize, format_temp, - self.mode, self.operand.result()) - code.putln('%s = __pyx_array_new(' - '%s, %s, PyBytes_AS_STRING(%s), ' - '(char *) "%s", (char *) %s);' % tup) - code.putln(code.error_goto_if_null(self.result(), self.pos)) - code.put_gotref(self.result()) + code.putln('%s = __pyx_array_new(%s, %s, PyBytes_AS_STRING(%s), (char *) "%s", (char *) %s); %s' % ( + self.result(), + shapes_temp, itemsize, format_temp, self.mode, self.operand.result(), + code.error_goto_if_null(self.result(), self.pos), + )) + self.generate_gotref(code) def dispose(temp): code.put_decref_clear(temp, py_object_type) @@ -10836,7 +10954,11 @@ class SizeofVarNode(SizeofNode): if operand_as_type: self.arg_type = operand_as_type if self.arg_type.is_fused: - self.arg_type = self.arg_type.specialize(env.fused_to_specific) + try: + self.arg_type = self.arg_type.specialize(env.fused_to_specific) + except CannotSpecialize: + error(self.operand.pos, + "Type cannot be specialized since it is not a fused argument to this function") self.__class__ = SizeofTypeNode self.check_type() else: @@ -10882,7 +11004,7 @@ class TypeidNode(ExprNode): self.error("The 'libcpp.typeinfo' module must be cimported to use the typeid() operator") return self self.type = type_info - as_type = self.operand.analyse_as_type(env) + as_type = self.operand.analyse_as_specialized_type(env) if as_type: self.arg_type = as_type self.is_type = True @@ -10930,11 +11052,11 @@ class TypeofNode(ExprNode): literal = None type = py_object_type - subexprs = ['literal'] # 'operand' will be ignored after type analysis! + subexprs = ['literal'] # 'operand' will be ignored after type analysis! def analyse_types(self, env): self.operand = self.operand.analyse_types(env) - value = StringEncoding.EncodedString(str(self.operand.type)) #self.operand.type.typeof_name()) + value = StringEncoding.EncodedString(str(self.operand.type)) #self.operand.type.typeof_name()) literal = StringNode(self.pos, value=value) literal = literal.analyse_types(env) self.literal = literal.coerce_to_pyobject(env) @@ -11178,7 +11300,7 @@ class BinopNode(ExprNode): self.operand2.py_result(), extra_args, code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) elif self.is_temp: # C++ overloaded operators with exception values are currently all # handled through temporaries. @@ -11302,8 +11424,8 @@ class NumBinopNode(BinopNode): def c_types_okay(self, type1, type2): #print "NumBinopNode.c_types_okay:", type1, type2 ### - return (type1.is_numeric or type1.is_enum) \ - and (type2.is_numeric or type2.is_enum) + return (type1.is_numeric or type1.is_enum) \ + and (type2.is_numeric or type2.is_enum) def generate_evaluation_code(self, code): if self.overflow_check: @@ -11414,7 +11536,7 @@ class AddNode(NumBinopNode): def py_operation_function(self, code): type1, type2 = self.operand1.type, self.operand2.type - + func = None if type1 is unicode_type or type2 is unicode_type: if type1 in (unicode_type, str_type) and type2 in (unicode_type, str_type): is_unicode_concat = True @@ -11426,10 +11548,22 @@ class AddNode(NumBinopNode): is_unicode_concat = False if is_unicode_concat: - if self.operand1.may_be_none() or self.operand2.may_be_none(): - return '__Pyx_PyUnicode_ConcatSafe' - else: - return '__Pyx_PyUnicode_Concat' + if self.inplace or self.operand1.is_temp: + code.globalstate.use_utility_code( + UtilityCode.load_cached("UnicodeConcatInPlace", "ObjectHandling.c")) + func = '__Pyx_PyUnicode_Concat' + elif type1 is str_type and type2 is str_type: + code.globalstate.use_utility_code( + UtilityCode.load_cached("StrConcatInPlace", "ObjectHandling.c")) + func = '__Pyx_PyStr_Concat' + + if func: + # any necessary utility code will be got by "NumberAdd" in generate_evaluation_code + if self.inplace or self.operand1.is_temp: + func += 'InPlace' # upper case to indicate unintuitive macro + if self.operand1.may_be_none() or self.operand2.may_be_none(): + func += 'Safe' + return func return super(AddNode, self).py_operation_function(code) @@ -11601,7 +11735,7 @@ class DivNode(NumBinopNode): minus1_check = '(!(((%s)-1) > 0)) && unlikely(%s == (%s)-1)' % ( type_of_op2, self.operand2.result(), type_of_op2) code.putln("else if (sizeof(%s) == sizeof(long) && %s " - " && unlikely(UNARY_NEG_WOULD_OVERFLOW(%s))) {" % ( + " && unlikely(__Pyx_UNARY_NEG_WOULD_OVERFLOW(%s))) {" % ( self.type.empty_declaration_code(), minus1_check, self.operand1.result())) @@ -11669,10 +11803,10 @@ _find_formatting_types = re.compile( br")").findall # These format conversion types can never trigger a Unicode string conversion in Py2. -_safe_bytes_formats = set([ +_safe_bytes_formats = frozenset({ # Excludes 's' and 'r', which can generate non-bytes strings. b'd', b'i', b'o', b'u', b'x', b'X', b'e', b'E', b'f', b'F', b'g', b'G', b'c', b'b', b'a', -]) +}) class ModNode(DivNode): @@ -11795,6 +11929,12 @@ class PowNode(NumBinopNode): error(self.pos, "got unexpected types for C power operator: %s, %s" % (self.operand1.type, self.operand2.type)) + def compute_c_result_type(self, type1, type2): + c_result_type = super(PowNode, self).compute_c_result_type(type1, type2) + if isinstance(self.operand2.constant_result, _py_int_types) and self.operand2.constant_result < 0: + c_result_type = PyrexTypes.widest_numeric_type(c_result_type, PyrexTypes.c_double_type) + return c_result_type + def calculate_result_code(self): # Work around MSVC overloading ambiguity. def typecast(operand): @@ -12065,6 +12205,9 @@ class BoolBinopResultNode(ExprNode): code.putln("}") self.arg.free_temps(code) + def analyse_types(self, env): + return self + class CondExprNode(ExprNode): # Short-circuiting conditional expression. @@ -12615,16 +12758,16 @@ class PrimaryCmpNode(ExprNode, CmpNode): elif self.find_special_bool_compare_function(env, self.operand1): if not self.operand1.type.is_pyobject: self.operand1 = self.operand1.coerce_to_pyobject(env) - common_type = None # if coercion needed, the method call above has already done it - self.is_pycmp = False # result is bint + common_type = None # if coercion needed, the method call above has already done it + self.is_pycmp = False # result is bint else: common_type = py_object_type self.is_pycmp = True elif self.find_special_bool_compare_function(env, self.operand1): if not self.operand1.type.is_pyobject: self.operand1 = self.operand1.coerce_to_pyobject(env) - common_type = None # if coercion needed, the method call above has already done it - self.is_pycmp = False # result is bint + common_type = None # if coercion needed, the method call above has already done it + self.is_pycmp = False # result is bint else: common_type = self.find_common_type(env, self.operator, self.operand1) self.is_pycmp = common_type.is_pyobject @@ -12812,7 +12955,7 @@ class CascadedCmpNode(Node, CmpNode): cascade = None coerced_operand2 = None - constant_result = constant_value_not_set # FIXME: where to calculate this? + constant_result = constant_value_not_set # FIXME: where to calculate this? def infer_type(self, env): # TODO: Actually implement this (after merging with -unstable). @@ -13011,9 +13154,10 @@ class PyTypeTestNode(CoercionNode): exact_builtin_type = True def __init__(self, arg, dst_type, env, notnone=False): - # The arg is know to be a Python object, and + # The arg is known to be a Python object, and # the dst_type is known to be an extension type. - assert dst_type.is_extension_type or dst_type.is_builtin_type, "PyTypeTest on non extension type" + assert dst_type.is_extension_type or dst_type.is_builtin_type, \ + "PyTypeTest for %s against non extension type %s" % (arg.type, dst_type) CoercionNode.__init__(self, arg) self.type = dst_type self.result_ctype = arg.ctype() @@ -13064,6 +13208,8 @@ class PyTypeTestNode(CoercionNode): type_test = self.type.type_test_code( self.arg.py_result(), self.notnone, exact=self.exact_builtin_type) + code.globalstate.use_utility_code(UtilityCode.load_cached( + "RaiseUnexpectedTypeError", "ObjectHandling.c")) else: type_test = self.type.type_test_code( self.arg.py_result(), self.notnone) @@ -13107,7 +13253,7 @@ class NoneCheckNode(CoercionNode): self.exception_message = exception_message self.exception_format_args = tuple(exception_format_args or ()) - nogil_check = None # this node only guards an operation that would fail already + nogil_check = None # this node only guards an operation that would fail already def analyse_types(self, env): return self @@ -13230,7 +13376,7 @@ class CoerceToPyTypeNode(CoercionNode): def coerce_to_boolean(self, env): arg_type = self.arg.type if (arg_type == PyrexTypes.c_bint_type or - (arg_type.is_pyobject and arg_type.name == 'bool')): + (arg_type.is_pyobject and arg_type.name == 'bool')): return self.arg.coerce_to_temp(env) else: return CoerceToBooleanNode(self, env) @@ -13254,7 +13400,7 @@ class CoerceToPyTypeNode(CoercionNode): self.target_type), code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) class CoerceIntToBytesNode(CoerceToPyTypeNode): @@ -13294,7 +13440,7 @@ class CoerceIntToBytesNode(CoerceToPyTypeNode): code.error_goto_if_null(self.result(), self.pos))) if temp is not None: code.funcstate.release_temp(temp) - code.put_gotref(self.py_result()) + self.generate_gotref(code) class CoerceFromPyTypeNode(CoercionNode): @@ -13332,7 +13478,7 @@ class CoerceFromPyTypeNode(CoercionNode): code.putln(self.type.from_py_call_code( self.arg.py_result(), self.result(), self.pos, code, from_py_function=from_py_function)) if self.type.is_pyobject: - code.put_gotref(self.py_result()) + self.generate_gotref(code) def nogil_check(self, env): error(self.pos, "Coercion from Python not allowed without the GIL") @@ -13389,6 +13535,9 @@ class CoerceToBooleanNode(CoercionNode): self.arg.py_result(), code.error_goto_if_neg(self.result(), self.pos))) + def analyse_types(self, env): + return self + class CoerceToComplexNode(CoercionNode): @@ -13414,6 +13563,9 @@ class CoerceToComplexNode(CoercionNode): def generate_result_code(self, code): pass + def analyse_types(self, env): + return self + class CoerceToTempNode(CoercionNode): # This node is used to force the result of another node # to be stored in a temporary. It is only used if the @@ -13433,6 +13585,9 @@ class CoerceToTempNode(CoercionNode): # The arg is always already analysed return self + def may_be_none(self): + return self.arg.may_be_none() + def coerce_to_boolean(self, env): self.arg = self.arg.coerce_to_boolean(env) if self.arg.is_simple(): @@ -13447,11 +13602,11 @@ class CoerceToTempNode(CoercionNode): code.putln("%s = %s;" % ( self.result(), self.arg.result_as(self.ctype()))) if self.use_managed_ref: - if self.type.is_pyobject: + if not self.type.is_memoryviewslice: code.put_incref(self.result(), self.ctype()) - elif self.type.is_memoryviewslice: - code.put_incref_memoryviewslice(self.result(), - not self.in_nogil_context) + else: + code.put_incref_memoryviewslice(self.result(), self.type, + have_gil=not self.in_nogil_context) class ProxyNode(CoercionNode): """ @@ -13478,11 +13633,13 @@ class ProxyNode(CoercionNode): return self.arg.infer_type(env) def _proxy_type(self): - if hasattr(self.arg, 'type'): - self.type = self.arg.type + type = getattr(self.arg, 'type', None) + if type: + self.type = type self.result_ctype = self.arg.result_ctype - if hasattr(self.arg, 'entry'): - self.entry = self.arg.entry + arg_entry = getattr(self.arg, 'entry', None) + if arg_entry: + self.entry = arg_entry def generate_result_code(self, code): self.arg.generate_result_code(code) @@ -13513,17 +13670,19 @@ class CloneNode(CoercionNode): # disposal code for it. The original owner of the argument # node is responsible for doing those things. - subexprs = [] # Arg is not considered a subexpr + subexprs = [] # Arg is not considered a subexpr nogil_check = None def __init__(self, arg): CoercionNode.__init__(self, arg) self.constant_result = arg.constant_result - if hasattr(arg, 'type'): - self.type = arg.type + type = getattr(arg, 'type', None) + if type: + self.type = type self.result_ctype = arg.result_ctype - if hasattr(arg, 'entry'): - self.entry = arg.entry + arg_entry = getattr(arg, 'entry', None) + if arg_entry: + self.entry = arg_entry def result(self): return self.arg.result() @@ -13541,8 +13700,9 @@ class CloneNode(CoercionNode): self.type = self.arg.type self.result_ctype = self.arg.result_ctype self.is_temp = 1 - if hasattr(self.arg, 'entry'): - self.entry = self.arg.entry + arg_entry = getattr(self.arg, 'entry', None) + if arg_entry: + self.entry = arg_entry return self def coerce_to(self, dest_type, env): @@ -13551,7 +13711,7 @@ class CloneNode(CoercionNode): return super(CloneNode, self).coerce_to(dest_type, env) def is_simple(self): - return True # result is always in a temp (or a name) + return True # result is always in a temp (or a name) def generate_evaluation_code(self, code): pass @@ -13617,9 +13777,97 @@ class DocstringRefNode(ExprNode): self.result(), self.body.result(), code.intern_identifier(StringEncoding.EncodedString("__doc__")), code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.result()) + self.generate_gotref(code) + +class AnnotationNode(ExprNode): + # Deals with the two possible uses of an annotation. + # 1. The post PEP-563 use where an annotation is stored + # as a string + # 2. The Cython use where the annotation can indicate an + # object type + # + # Doesn't handle the pre PEP-563 version where the + # annotation is evaluated into a Python Object. + + subexprs = [] + + # 'untyped' is set for fused specializations: + # Once a fused function has been created we don't want + # annotations to override an already set type. + untyped = False + + def __init__(self, pos, expr, string=None): + """string is expected to already be a StringNode or None""" + ExprNode.__init__(self, pos) + if string is None: + # import doesn't work at top of file? + from .AutoDocTransforms import AnnotationWriter + string = StringEncoding.EncodedString( + AnnotationWriter(description="annotation").write(expr)) + string = StringNode(pos, unicode_value=string, value=string.as_utf8_string()) + self.string = string + self.expr = expr + def analyse_types(self, env): + return self # nothing needs doing + def analyse_as_type(self, env): + # for compatibility when used as a return_type_node, have this interface too + return self.analyse_type_annotation(env)[1] + + def analyse_type_annotation(self, env, assigned_value=None): + if self.untyped: + # Already applied as a fused type, not re-evaluating it here. + return None, None + annotation = self.expr + base_type = None + is_ambiguous = False + explicit_pytype = explicit_ctype = False + if annotation.is_dict_literal: + warning(annotation.pos, + "Dicts should no longer be used as type annotations. Use 'cython.int' etc. directly.", level=1) + for name, value in annotation.key_value_pairs: + if not name.is_string_literal: + continue + if name.value in ('type', b'type'): + explicit_pytype = True + if not explicit_ctype: + annotation = value + elif name.value in ('ctype', b'ctype'): + explicit_ctype = True + annotation = value + if explicit_pytype and explicit_ctype: + warning(annotation.pos, "Duplicate type declarations found in signature annotation", level=1) + arg_type = annotation.analyse_as_type(env) + if annotation.is_name and not annotation.cython_attribute and annotation.name in ('int', 'long', 'float'): + # Map builtin numeric Python types to C types in safe cases. + if assigned_value is not None and arg_type is not None and not arg_type.is_pyobject: + assigned_type = assigned_value.infer_type(env) + if assigned_type and assigned_type.is_pyobject: + # C type seems unsafe, e.g. due to 'None' default value => ignore annotation type + is_ambiguous = True + arg_type = None + # ignore 'int' and require 'cython.int' to avoid unsafe integer declarations + if arg_type in (PyrexTypes.c_long_type, PyrexTypes.c_int_type, PyrexTypes.c_float_type): + arg_type = PyrexTypes.c_double_type if annotation.name == 'float' else py_object_type + elif arg_type is not None and annotation.is_string_literal: + warning(annotation.pos, + "Strings should no longer be used for type declarations. Use 'cython.int' etc. directly.", + level=1) + elif arg_type is not None and arg_type.is_complex: + # creating utility code needs to be special-cased for complex types + arg_type.create_declaration_utility_code(env) + if arg_type is not None: + if explicit_pytype and not explicit_ctype and not arg_type.is_pyobject: + warning(annotation.pos, + "Python type declaration in signature annotation does not refer to a Python type") + base_type = Nodes.CAnalysedBaseTypeNode( + annotation.pos, type=arg_type, is_arg=True) + elif is_ambiguous: + warning(annotation.pos, "Ambiguous types in annotation, ignoring") + else: + warning(annotation.pos, "Unknown type declaration in annotation, ignoring") + return base_type, arg_type #------------------------------------------------------------------------------------ # diff --git a/Cython/Compiler/FlowControl.pxd b/Cython/Compiler/FlowControl.pxd index c87370b81..c876ee3b1 100644 --- a/Cython/Compiler/FlowControl.pxd +++ b/Cython/Compiler/FlowControl.pxd @@ -1,4 +1,4 @@ -from __future__ import absolute_import +# cython: language_level=3 cimport cython @@ -105,6 +105,7 @@ cdef class ControlFlowAnalysis(CythonTransform): cdef list stack cdef object env cdef ControlFlow flow + cdef object object_expr cdef bint in_inplace_assignment cpdef mark_assignment(self, lhs, rhs=*) diff --git a/Cython/Compiler/FlowControl.py b/Cython/Compiler/FlowControl.py index df04471f9..5b51be0db 100644 --- a/Cython/Compiler/FlowControl.py +++ b/Cython/Compiler/FlowControl.py @@ -1,16 +1,17 @@ +# cython: language_level=3str +# cython: auto_pickle=True + from __future__ import absolute_import import cython cython.declare(PyrexTypes=object, ExprNodes=object, Nodes=object, Builtin=object, InternalError=object, error=object, warning=object, - py_object_type=object, unspecified_type=object, - object_expr=object, fake_rhs_expr=object, TypedExprNode=object) + fake_rhs_expr=object, TypedExprNode=object) from . import Builtin from . import ExprNodes from . import Nodes from . import Options -from .PyrexTypes import py_object_type, unspecified_type from . import PyrexTypes from .Visitor import TreeVisitor, CythonTransform @@ -28,9 +29,8 @@ class TypedExprNode(ExprNodes.ExprNode): def may_be_none(self): return self._may_be_none != False -object_expr = TypedExprNode(py_object_type, may_be_none=True) # Fake rhs to silence "unused variable" warning -fake_rhs_expr = TypedExprNode(unspecified_type) +fake_rhs_expr = TypedExprNode(PyrexTypes.unspecified_type) class ControlBlock(object): @@ -52,7 +52,7 @@ class ControlBlock(object): stats = [Assignment(a), NameReference(a), NameReference(c), Assignment(b)] gen = {Entry(a): Assignment(a), Entry(b): Assignment(b)} - bounded = set([Entry(a), Entry(c)]) + bounded = {Entry(a), Entry(c)} """ @@ -203,7 +203,7 @@ class ControlFlow(object): def normalize(self): """Delete unreachable and orphan blocks.""" - queue = set([self.entry_point]) + queue = {self.entry_point} visited = set() while queue: root = queue.pop() @@ -217,7 +217,7 @@ class ControlFlow(object): visited.remove(self.entry_point) for block in visited: if block.empty(): - for parent in block.parents: # Re-parent + for parent in block.parents: # Re-parent for child in block.children: parent.add_child(child) block.detach() @@ -373,9 +373,9 @@ class NameDeletion(NameAssignment): def infer_type(self): inferred_type = self.rhs.infer_type(self.entry.scope) - if (not inferred_type.is_pyobject and - inferred_type.can_coerce_to_pyobject(self.entry.scope)): - return py_object_type + if (not inferred_type.is_pyobject + and inferred_type.can_coerce_to_pyobject(self.entry.scope)): + return PyrexTypes.py_object_type self.inferred_type = inferred_type return inferred_type @@ -455,7 +455,7 @@ class GVContext(object): start = min(block.positions) stop = max(block.positions) srcdescr = start[0] - if not srcdescr in self.sources: + if srcdescr not in self.sources: self.sources[srcdescr] = list(srcdescr.get_lines()) lines = self.sources[srcdescr] return '\\n'.join([l.strip() for l in lines[start[1] - 1:stop[1]]]) @@ -621,7 +621,7 @@ def check_definitions(flow, compiler_directives): # Unused result for assmt in assignments: if (not assmt.refs and not assmt.entry.is_pyclass_attr - and not assmt.entry.in_closure): + and not assmt.entry.in_closure): if assmt.entry.cf_references and warn_unused_result: if assmt.is_arg: messages.warning(assmt.pos, "Unused argument value '%s'" % @@ -674,7 +674,8 @@ class AssignmentCollector(TreeVisitor): class ControlFlowAnalysis(CythonTransform): def visit_ModuleNode(self, node): - self.gv_ctx = GVContext() + dot_output = self.current_directives['control_flow.dot_output'] + self.gv_ctx = GVContext() if dot_output else None self.constant_folder = ConstantFolding() # Set of NameNode reductions @@ -685,18 +686,15 @@ class ControlFlowAnalysis(CythonTransform): self.env = node.scope self.stack = [] self.flow = ControlFlow() + self.object_expr = TypedExprNode(PyrexTypes.py_object_type, may_be_none=True) self.visitchildren(node) check_definitions(self.flow, self.current_directives) - dot_output = self.current_directives['control_flow.dot_output'] if dot_output: annotate_defs = self.current_directives['control_flow.dot_annotate_defs'] - fp = open(dot_output, 'wt') - try: + with open(dot_output, 'wt') as fp: self.gv_ctx.render(fp, 'module', annotate_defs=annotate_defs) - finally: - fp.close() return node def visit_FuncDefNode(self, node): @@ -744,7 +742,8 @@ class ControlFlowAnalysis(CythonTransform): check_definitions(self.flow, self.current_directives) self.flow.blocks.add(self.flow.entry_point) - self.gv_ctx.add(GV(node.local_scope.name, self.flow)) + if self.gv_ctx is not None: + self.gv_ctx.add(GV(node.local_scope.name, self.flow)) self.flow = self.stack.pop() self.env = self.env_stack.pop() @@ -769,19 +768,22 @@ class ControlFlowAnalysis(CythonTransform): self.flow.nextblock() if not rhs: - rhs = object_expr + rhs = self.object_expr if lhs.is_name: if lhs.entry is not None: entry = lhs.entry else: entry = self.env.lookup(lhs.name) - if entry is None: # TODO: This shouldn't happen... + if entry is None: # TODO: This shouldn't happen... return self.flow.mark_assignment(lhs, rhs, entry) elif lhs.is_sequence_constructor: for i, arg in enumerate(lhs.args): - if not rhs or arg.is_starred: - item_node = None + if arg.is_starred: + # "a, *b = x" assigns a list to "b" + item_node = TypedExprNode(Builtin.list_type, may_be_none=False, pos=arg.pos) + elif rhs is self.object_expr: + item_node = rhs else: item_node = rhs.inferable_item_node(i) self.mark_assignment(arg, item_node) @@ -806,7 +808,7 @@ class ControlFlowAnalysis(CythonTransform): return node def visit_AssignmentNode(self, node): - raise InternalError("Unhandled assignment node") + raise InternalError("Unhandled assignment node %s" % type(node)) def visit_SingleAssignmentNode(self, node): self._visit(node.rhs) @@ -916,6 +918,26 @@ class ControlFlowAnalysis(CythonTransform): self.flow.block = None return node + def visit_AssertStatNode(self, node): + """Essentially an if-condition that wraps a RaiseStatNode. + """ + self.mark_position(node) + next_block = self.flow.newblock() + parent = self.flow.block + # failure case + parent = self.flow.nextblock(parent) + self._visit(node.condition) + self.flow.nextblock() + self._visit(node.exception) + if self.flow.block: + self.flow.block.add_child(next_block) + parent.add_child(next_block) + if next_block.parents: + self.flow.block = next_block + else: + self.flow.block = None + return node + def visit_WhileStatNode(self, node): condition_block = self.flow.nextblock() next_block = self.flow.newblock() @@ -1013,7 +1035,7 @@ class ControlFlowAnalysis(CythonTransform): elif isinstance(node, Nodes.AsyncForStatNode): # not entirely correct, but good enough for now self.mark_assignment(node.target, node.item) - else: # Parallel + else: # Parallel self.mark_assignment(node.target) # Body block @@ -1308,10 +1330,12 @@ class ControlFlowAnalysis(CythonTransform): self.visitchildren(node, ('dict', 'metaclass', 'mkw', 'bases', 'class_result')) self.flow.mark_assignment(node.target, node.classobj, - self.env.lookup(node.name)) + self.env.lookup(node.target.name)) self.env_stack.append(self.env) self.env = node.scope self.flow.nextblock() + if node.doc_node: + self.flow.mark_assignment(node.doc_node, fake_rhs_expr, node.doc_node.entry) self.visitchildren(node, ('body',)) self.flow.nextblock() self.env = self.env_stack.pop() diff --git a/Cython/Compiler/FusedNode.py b/Cython/Compiler/FusedNode.py index 26d6ffd3d..918a05990 100644 --- a/Cython/Compiler/FusedNode.py +++ b/Cython/Compiler/FusedNode.py @@ -7,6 +7,7 @@ from . import (ExprNodes, PyrexTypes, MemoryView, from .ExprNodes import CloneNode, ProxyNode, TupleNode from .Nodes import FuncDefNode, CFuncDefNode, StatListNode, DefNode from ..Utils import OrderedSet +from .Errors import error, CannotSpecialize class FusedCFuncDefNode(StatListNode): @@ -141,7 +142,14 @@ class FusedCFuncDefNode(StatListNode): copied_node = copy.deepcopy(self.node) # Make the types in our CFuncType specific. - type = copied_node.type.specialize(fused_to_specific) + try: + type = copied_node.type.specialize(fused_to_specific) + except CannotSpecialize: + # unlike for the argument types, specializing the return type can fail + error(copied_node.pos, "Return type is a fused type that cannot " + "be determined from the function arguments") + self.py_func = None # this is just to let the compiler exit gracefully + return entry = copied_node.entry type.specialize_entry(entry, cname) @@ -220,6 +228,10 @@ class FusedCFuncDefNode(StatListNode): arg.type = arg.type.specialize(fused_to_specific) if arg.type.is_memoryviewslice: arg.type.validate_memslice_dtype(arg.pos) + if arg.annotation: + # TODO might be nice if annotations were specialized instead? + # (Or might be hard to do reliably) + arg.annotation.untyped = True def create_new_local_scope(self, node, env, f2s): """ @@ -402,7 +414,7 @@ class FusedCFuncDefNode(StatListNode): if itemsize == -1 or itemsize == {{sizeof_dtype}}: memslice = {{coerce_from_py_func}}(arg, 0) if memslice.memview: - __PYX_XDEC_MEMVIEW(&memslice, 1) + __PYX_XCLEAR_MEMVIEW(&memslice, 1) # print 'found a match for the buffer through format parsing' %s break @@ -481,7 +493,7 @@ class FusedCFuncDefNode(StatListNode): ctypedef struct {{memviewslice_cname}}: void *memview - void __PYX_XDEC_MEMVIEW({{memviewslice_cname}} *, int have_gil) + void __PYX_XCLEAR_MEMVIEW({{memviewslice_cname}} *, int have_gil) bint __pyx_memoryview_check(object) """) @@ -580,6 +592,26 @@ class FusedCFuncDefNode(StatListNode): {{endif}} """) + def _fused_signature_index(self, pyx_code): + """ + Generate Cython code for constructing a persistent nested dictionary index of + fused type specialization signatures. + """ + pyx_code.put_chunk( + u""" + if not _fused_sigindex: + for sig in <dict>signatures: + sigindex_node = _fused_sigindex + *sig_series, last_type = sig.strip('()').split('|') + for sig_type in sig_series: + if sig_type not in sigindex_node: + sigindex_node[sig_type] = sigindex_node = {} + else: + sigindex_node = sigindex_node[sig_type] + sigindex_node[last_type] = sig + """ + ) + def make_fused_cpdef(self, orig_py_func, env, is_def): """ This creates the function that is indexable from Python and does @@ -616,10 +648,14 @@ class FusedCFuncDefNode(StatListNode): pyx_code.put_chunk( u""" - def __pyx_fused_cpdef(signatures, args, kwargs, defaults): + def __pyx_fused_cpdef(signatures, args, kwargs, defaults, _fused_sigindex={}): # FIXME: use a typed signature - currently fails badly because # default arguments inherit the types we specify here! + cdef list search_list + + cdef dict sn, sigindex_node + dest_sig = [None] * {{n_fused}} if kwargs is not None and not kwargs: @@ -630,7 +666,7 @@ class FusedCFuncDefNode(StatListNode): # instance check body """) - pyx_code.indent() # indent following code to function body + pyx_code.indent() # indent following code to function body pyx_code.named_insertion_point("imports") pyx_code.named_insertion_point("func_defs") pyx_code.named_insertion_point("local_variable_declarations") @@ -687,23 +723,36 @@ class FusedCFuncDefNode(StatListNode): env.use_utility_code(Code.UtilityCode.load_cached("Import", "ImportExport.c")) env.use_utility_code(Code.UtilityCode.load_cached("ImportNumPyArray", "ImportExport.c")) + self._fused_signature_index(pyx_code) + pyx_code.put_chunk( u""" - candidates = [] - for sig in <dict>signatures: - match_found = False - src_sig = sig.strip('()').split('|') - for i in range(len(dest_sig)): - dst_type = dest_sig[i] - if dst_type is not None: - if src_sig[i] == dst_type: - match_found = True - else: - match_found = False - break + sigindex_matches = [] + sigindex_candidates = [_fused_sigindex] + + for dst_type in dest_sig: + found_matches = [] + found_candidates = [] + # Make two seperate lists: One for signature sub-trees + # with at least one definite match, and another for + # signature sub-trees with only ambiguous matches + # (where `dest_sig[i] is None`). + if dst_type is None: + for sn in sigindex_matches: + found_matches.extend(sn.values()) + for sn in sigindex_candidates: + found_candidates.extend(sn.values()) + else: + for search_list in (sigindex_matches, sigindex_candidates): + for sn in search_list: + if dst_type in sn: + found_matches.append(sn[dst_type]) + sigindex_matches = found_matches + sigindex_candidates = found_candidates + if not (found_matches or found_candidates): + break - if match_found: - candidates.append(sig) + candidates = sigindex_matches if not candidates: raise TypeError("No matching signature found") @@ -798,7 +847,8 @@ class FusedCFuncDefNode(StatListNode): for i, stat in enumerate(self.stats): stat = self.stats[i] = stat.analyse_expressions(env) - if isinstance(stat, FuncDefNode): + if isinstance(stat, FuncDefNode) and stat is not self.py_func: + # the dispatcher specifically doesn't want its defaults overriding for arg, default in zip(stat.args, defaults): if default is not None: arg.default = CloneNode(default).coerce_to(arg.type, env) @@ -829,6 +879,10 @@ class FusedCFuncDefNode(StatListNode): else: nodes = self.nodes + # For the moment, fused functions do not support METH_FASTCALL + for node in nodes: + node.entry.signature.use_fastcall = False + signatures = [StringEncoding.EncodedString(node.specialized_signature_string) for node in nodes] keys = [ExprNodes.StringNode(node.pos, value=sig) @@ -877,7 +931,7 @@ class FusedCFuncDefNode(StatListNode): "((__pyx_FusedFunctionObject *) %s)->__signatures__ = %s;" % (self.resulting_fused_function.result(), self.__signatures__.result())) - code.put_giveref(self.__signatures__.result()) + self.__signatures__.generate_giveref(code) self.__signatures__.generate_post_assignment_code(code) self.__signatures__.free_temps(code) diff --git a/Cython/Compiler/Future.py b/Cython/Compiler/Future.py index 848792e00..8de10c0cb 100644 --- a/Cython/Compiler/Future.py +++ b/Cython/Compiler/Future.py @@ -11,5 +11,6 @@ absolute_import = _get_feature("absolute_import") nested_scopes = _get_feature("nested_scopes") # dummy generators = _get_feature("generators") # dummy generator_stop = _get_feature("generator_stop") +annotations = _get_feature("annotations") del _get_feature diff --git a/Cython/Compiler/Interpreter.py b/Cython/Compiler/Interpreter.py index 9ec391f2a..244397264 100644 --- a/Cython/Compiler/Interpreter.py +++ b/Cython/Compiler/Interpreter.py @@ -47,8 +47,8 @@ def interpret_compiletime_options(optlist, optdict, type_env=None, type_args=()) raise CompileError(node.pos, "Type not allowed here.") else: if (sys.version_info[0] >=3 and - isinstance(node, StringNode) and - node.unicode_value is not None): + isinstance(node, StringNode) and + node.unicode_value is not None): return (node.unicode_value, node.pos) return (node.compile_time_value(empty_scope), node.pos) diff --git a/Cython/Compiler/Lexicon.py b/Cython/Compiler/Lexicon.py index 72c9ceaef..7c23b5de1 100644 --- a/Cython/Compiler/Lexicon.py +++ b/Cython/Compiler/Lexicon.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- # cython: language_level=3, py2_import=True # # Cython Scanner - Lexical Definitions @@ -16,28 +17,43 @@ IDENT = 'IDENT' def make_lexicon(): from ..Plex import \ Str, Any, AnyBut, AnyChar, Rep, Rep1, Opt, Bol, Eol, Eof, \ - TEXT, IGNORE, State, Lexicon - from .Scanning import Method + TEXT, IGNORE, Method, State, Lexicon, Range - letter = Any("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz_") + nonzero_digit = Any("123456789") digit = Any("0123456789") bindigit = Any("01") octdigit = Any("01234567") hexdigit = Any("0123456789ABCDEFabcdef") indentation = Bol + Rep(Any(" \t")) + # The list of valid unicode identifier characters are pretty slow to generate at runtime, + # and require Python3, so are just included directly here + # (via the generated code block at the bottom of the file) + unicode_start_character = (Any(unicode_start_ch_any) | Range(unicode_start_ch_range)) + unicode_continuation_character = ( + unicode_start_character | + Any(unicode_continuation_ch_any) | Range(unicode_continuation_ch_range)) + def underscore_digits(d): return Rep1(d) + Rep(Str("_") + Rep1(d)) + def prefixed_digits(prefix, digits): + return prefix + Opt(Str("_")) + underscore_digits(digits) + decimal = underscore_digits(digit) dot = Str(".") exponent = Any("Ee") + Opt(Any("+-")) + decimal decimal_fract = (decimal + dot + Opt(decimal)) | (dot + decimal) - name = letter + Rep(letter | digit) - intconst = decimal | (Str("0") + ((Any("Xx") + underscore_digits(hexdigit)) | - (Any("Oo") + underscore_digits(octdigit)) | - (Any("Bb") + underscore_digits(bindigit)) )) + #name = letter + Rep(letter | digit) + name = unicode_start_character + Rep(unicode_continuation_character) + intconst = (prefixed_digits(nonzero_digit, digit) | # decimal literals with underscores must not start with '0' + (Str("0") + (prefixed_digits(Any("Xx"), hexdigit) | + prefixed_digits(Any("Oo"), octdigit) | + prefixed_digits(Any("Bb"), bindigit) )) | + underscore_digits(Str('0')) # 0_0_0_0... is allowed as a decimal literal + | Rep1(digit) # FIXME: remove these Py2 style decimal/octal literals (PY_VERSION_HEX < 3) + ) intsuffix = (Opt(Any("Uu")) + Opt(Any("Ll")) + Opt(Any("Ll"))) | (Opt(Any("Ll")) + Opt(Any("Ll")) + Opt(Any("Uu"))) intliteral = intconst + intsuffix fltconst = (decimal_fract + Opt(exponent)) | (decimal + exponent) @@ -61,7 +77,7 @@ def make_lexicon(): punct = Any(":,;+-*/|&<>=.%`~^?!@") diphthong = Str("==", "<>", "!=", "<=", ">=", "<<", ">>", "**", "//", "+=", "-=", "*=", "/=", "%=", "|=", "^=", "&=", - "<<=", ">>=", "**=", "//=", "->", "@=") + "<<=", ">>=", "**=", "//=", "->", "@=", "&&", "||") spaces = Rep1(Any(" \t\f")) escaped_newline = Str("\\\n") lineterm = Eol + Opt(Str("\n")) @@ -69,7 +85,7 @@ def make_lexicon(): comment = Str("#") + Rep(AnyBut("\n")) return Lexicon([ - (name, IDENT), + (name, Method('normalize_ident')), (intliteral, Method('strip_underscores', symbol='INT')), (fltconst, Method('strip_underscores', symbol='FLOAT')), (imagconst, Method('strip_underscores', symbol='IMAG')), @@ -136,3 +152,50 @@ def make_lexicon(): #debug_file = scanner_dump_file ) + +# BEGIN GENERATED CODE +# generated with: +# cpython 3.10.0a0 (heads/master:2b0e654f91, May 29 2020, 16:17:52) + +unicode_start_ch_any = ( + u"_ªµºˬˮͿΆΌՙەۿܐޱߺࠚࠤࠨऽॐলঽৎৼਫ਼ઽૐૹଽୱஃஜௐఽಀಽೞഽൎලาຄລາຽໆༀဿၡႎჇჍቘዀៗៜᢪᪧᳺὙ" + u"ὛὝιⁱⁿℂℇℕℤΩℨⅎⴧⴭⵯꣻꧏꩺꪱꫀꫂיִמּﹱﹳﹷﹹﹻﹽ𐠈𐠼𐨀𐼧𑅄𑅇𑅶𑇚𑇜𑊈𑌽𑍐𑓇𑙄𑚸𑤉𑤿𑥁𑧡𑧣𑨀𑨺𑩐𑪝𑱀𑵆𑶘𑾰𖽐𖿣𝒢" + u"𝒻𝕆𞅎𞥋𞸤𞸧𞸹𞸻𞹂𞹇𞹉𞹋𞹔𞹗𞹙𞹛𞹝𞹟𞹤𞹾" +) +unicode_start_ch_range = ( + u"AZazÀÖØöøˁˆˑˠˤͰʹͶͷͻͽΈΊΎΡΣϵϷҁҊԯԱՖՠֈאתׯײؠيٮٯٱۓۥۦۮۯۺۼܒܯݍޥߊߪߴߵࠀࠕ" + u"ࡀࡘࡠࡪࢠࢴࢶࣇऄहक़ॡॱঀঅঌএঐওনপরশহড়ঢ়য়ৡৰৱਅਊਏਐਓਨਪਰਲਲ਼ਵਸ਼ਸਹਖ਼ੜੲੴઅઍએઑઓનપરલળવહ" + u"ૠૡଅଌଏଐଓନପରଲଳଵହଡ଼ଢ଼ୟୡஅஊஎஐஒகஙசஞடணதநபமஹఅఌఎఐఒనపహౘౚౠౡಅಌಎಐಒನಪಳವಹೠೡೱೲ" + u"ഄഌഎഐഒഺൔൖൟൡൺൿඅඖකනඳරවෆกะเๆກຂຆຊຌຣວະເໄໜໟཀཇཉཬྈྌကဪၐၕၚၝၥၦၮၰၵႁႠჅაჺჼቈ" + u"ቊቍቐቖቚቝበኈኊኍነኰኲኵኸኾዂዅወዖዘጐጒጕጘፚᎀᎏᎠᏵᏸᏽᐁᙬᙯᙿᚁᚚᚠᛪᛮᛸᜀᜌᜎᜑᜠᜱᝀᝑᝠᝬᝮᝰកឳᠠᡸᢀᢨ" + u"ᢰᣵᤀᤞᥐᥭᥰᥴᦀᦫᦰᧉᨀᨖᨠᩔᬅᬳᭅᭋᮃᮠᮮᮯᮺᯥᰀᰣᱍᱏᱚᱽᲀᲈᲐᲺᲽᲿᳩᳬᳮᳳᳵᳶᴀᶿḀἕἘἝἠὅὈὍὐὗὟώᾀᾴ" + u"ᾶᾼῂῄῆῌῐΐῖΊῠῬῲῴῶῼₐₜℊℓ℘ℝKℹℼℿⅅⅉⅠↈⰀⰮⰰⱞⱠⳤⳫⳮⳲⳳⴀⴥⴰⵧⶀⶖⶠⶦⶨⶮⶰⶶⶸⶾⷀⷆⷈⷎⷐⷖ" + u"ⷘⷞ々〇〡〩〱〵〸〼ぁゖゝゟァヺーヿㄅㄯㄱㆎㆠㆿㇰㇿ㐀䶿一鿼ꀀꒌꓐꓽꔀꘌꘐꘟꘪꘫꙀꙮꙿꚝꚠꛯꜗꜟꜢꞈꞋꞿꟂꟊꟵꠁꠃꠅꠇꠊ" + u"ꠌꠢꡀꡳꢂꢳꣲꣷꣽꣾꤊꤥꤰꥆꥠꥼꦄꦲꧠꧤꧦꧯꧺꧾꨀꨨꩀꩂꩄꩋꩠꩶꩾꪯꪵꪶꪹꪽꫛꫝꫠꫪꫲꫴꬁꬆꬉꬎꬑꬖꬠꬦꬨꬮꬰꭚꭜꭩꭰꯢ" + u"가힣ힰퟆퟋퟻ豈舘並龎ffstﬓﬗײַﬨשׁזּטּלּנּסּףּפּצּﮱﯓﱝﱤﴽﵐﶏﶒﷇﷰﷹﹿﻼAZazヲンᅠ하ᅦᅧᅬᅭᅲᅳᅵ𐀀𐀋𐀍𐀦𐀨𐀺" + u"𐀼𐀽𐀿𐁍𐁐𐁝𐂀𐃺𐅀𐅴𐊀𐊜𐊠𐋐𐌀𐌟𐌭𐍊𐍐𐍵𐎀𐎝𐎠𐏃𐏈𐏏𐏑𐏕𐐀𐒝𐒰𐓓𐓘𐓻𐔀𐔧𐔰𐕣𐘀𐜶𐝀𐝕𐝠𐝧𐠀𐠅𐠊𐠵𐠷𐠸𐠿𐡕𐡠𐡶𐢀𐢞𐣠𐣲𐣴𐣵" + u"𐤀𐤕𐤠𐤹𐦀𐦷𐦾𐦿𐨐𐨓𐨕𐨗𐨙𐨵𐩠𐩼𐪀𐪜𐫀𐫇𐫉𐫤𐬀𐬵𐭀𐭕𐭠𐭲𐮀𐮑𐰀𐱈𐲀𐲲𐳀𐳲𐴀𐴣𐺀𐺩𐺰𐺱𐼀𐼜𐼰𐽅𐾰𐿄𐿠𐿶𑀃𑀷𑂃𑂯𑃐𑃨𑄃𑄦𑅐𑅲" + u"𑆃𑆲𑇁𑇄𑈀𑈑𑈓𑈫𑊀𑊆𑊊𑊍𑊏𑊝𑊟𑊨𑊰𑋞𑌅𑌌𑌏𑌐𑌓𑌨𑌪𑌰𑌲𑌳𑌵𑌹𑍝𑍡𑐀𑐴𑑇𑑊𑑟𑑡𑒀𑒯𑓄𑓅𑖀𑖮𑗘𑗛𑘀𑘯𑚀𑚪𑜀𑜚𑠀𑠫𑢠𑣟𑣿𑤆𑤌𑤓" + u"𑤕𑤖𑤘𑤯𑦠𑦧𑦪𑧐𑨋𑨲𑩜𑪉𑫀𑫸𑰀𑰈𑰊𑰮𑱲𑲏𑴀𑴆𑴈𑴉𑴋𑴰𑵠𑵥𑵧𑵨𑵪𑶉𑻠𑻲𒀀𒎙𒐀𒑮𒒀𒕃𓀀𓐮𔐀𔙆𖠀𖨸𖩀𖩞𖫐𖫭𖬀𖬯𖭀𖭃𖭣𖭷𖭽𖮏𖹀𖹿" + u"𖼀𖽊𖾓𖾟𖿠𖿡𗀀𘟷𘠀𘳕𘴀𘴈𛀀𛄞𛅐𛅒𛅤𛅧𛅰𛋻𛰀𛱪𛱰𛱼𛲀𛲈𛲐𛲙𝐀𝑔𝑖𝒜𝒞𝒟𝒥𝒦𝒩𝒬𝒮𝒹𝒽𝓃𝓅𝔅𝔇𝔊𝔍𝔔𝔖𝔜𝔞𝔹𝔻𝔾𝕀𝕄𝕊𝕐𝕒𝚥" + u"𝚨𝛀𝛂𝛚𝛜𝛺𝛼𝜔𝜖𝜴𝜶𝝎𝝐𝝮𝝰𝞈𝞊𝞨𝞪𝟂𝟄𝟋𞄀𞄬𞄷𞄽𞋀𞋫𞠀𞣄𞤀𞥃𞸀𞸃𞸅𞸟𞸡𞸢𞸩𞸲𞸴𞸷𞹍𞹏𞹑𞹒𞹡𞹢𞹧𞹪𞹬𞹲𞹴𞹷𞹹𞹼𞺀𞺉𞺋𞺛" + u"𞺡𞺣𞺥𞺩𞺫𞺻𠀀𪛝𪜀𫜴𫝀𫠝𫠠𬺡𬺰𮯠丽𪘀" +) +unicode_continuation_ch_any = ( + u"··়ׇֿٰܑ߽ৗ਼৾ੑੵ઼଼ஂௗ಼ൗ්ූัັ༹༵༷࿆᳭ᢩ៝᳴⁔⵿⃡꙯ꠂ꠆ꠋ꠬ꧥꩃﬞꪰ꫁_𑅳𐨿𐇽𐋠𑈾𑍗𑑞𑥀𑧤𑩇𑴺𑵇𖽏𖿤𝩵" + u"𝪄" +) +unicode_continuation_ch_range = ( + u"09ֽׁׂًؚ֑ׅ̀ͯ҃҇ׄؐ٩۪ۭۖۜ۟ۤۧۨ۰۹ܰ݊ަް߀߉࡙࡛࣓ࣣ߫߳ࠖ࠙ࠛࠣࠥࠧࠩ࠭࣡ःऺ़ाॏ॑ॗॢॣ०९ঁঃ" + u"াৄেৈো্ৢৣ০৯ਁਃਾੂੇੈੋ੍੦ੱઁઃાૅેૉો્ૢૣ૦૯ૺ૿ଁଃାୄେୈୋ୍୕ୗୢୣ୦୯ாூெைொ்௦௯ఀఄాౄ" + u"ెైొ్ౕౖౢౣ౦౯ಁಃಾೄೆೈೊ್ೕೖೢೣ೦೯ഀഃ഻഼ാൄെൈൊ്ൢൣ൦൯ඁඃාුෘෟ෦෯ෲෳำฺ็๎๐๙ຳຼ່ໍ໐໙" + u"༘༙༠༩༾༿྄ཱ྆྇ྍྗྙྼါှ၀၉ၖၙၞၠၢၤၧၭၱၴႂႍႏႝ፝፟፩፱ᜒ᜔ᜲ᜴ᝒᝓᝲᝳ឴៓០៩᠋᠍᠐᠙ᤠᤫᤰ᤻᥆᥏᧐᧚" + u"ᨗᨛᩕᩞ᩠᩿᩼᪉᪐᪙᪽ᪿᫀ᪰ᬀᬄ᬴᭄᭐᭙᭫᭳ᮀᮂᮡᮭ᮰᮹᯦᯳ᰤ᰷᱀᱉᱐᱙᳔᳨᳐᳒᳷᷹᷿᳹᷀᷻‿⁀⃥゙゚〪〯⃐⃜⃰⳯⳱ⷠⷿ" + u"꘠꘩ꙴ꙽ꚞꚟ꛰꛱ꠣꠧꢀꢁꢴꣅ꣐꣙꣠꣱ꣿ꤉ꤦ꤭ꥇ꥓ꦀꦃ꦳꧀꧐꧙꧰꧹ꨩꨶꩌꩍ꩐꩙ꩻꩽꪴꪲꪷꪸꪾ꪿ꫫꫯꫵ꫶ꯣꯪ꯬꯭꯰꯹︀️︠︯" + u"︳︴﹍﹏09゙゚𐍶𐍺𐒠𐒩𐨁𐨃𐨅𐨆𐨌𐨺𐫦𐨏𐨸𐫥𐴤𐴧𐴰𐴹𐽆𐽐𐺫𐺬𑀀𑀂𑀸𑁆𑁦𑁯𑁿𑂂𑂰𑂺𑃰𑃹𑄀𑄂𑄧𑄴𑄶𑄿𑅅𑅆𑆀𑆂𑆳𑇀𑇉𑇌𑇎𑇙𑈬𑈷" + u"𑋟𑋪𑋰𑋹𑌀𑌃𑌻𑌼𑌾𑍄𑍇𑍈𑍋𑍍𑍢𑍣𑍦𑍬𑍰𑍴𑐵𑑆𑑐𑑙𑒰𑓃𑓐𑓙𑖯𑖵𑖸𑗀𑗜𑗝𑘰𑙀𑙐𑙙𑚫𑚷𑛀𑛉𑜝𑜫𑜰𑜹𑠬𑠺𑣠𑣩𑤰𑤵𑤷𑤸𑤻𑤾𑥂𑥃𑥐𑥙" + u"𑧑𑧗𑧚𑧠𑨁𑨊𑨳𑨹𑨻𑨾𑩑𑩛𑪊𑪙𑰯𑰶𑰸𑰿𑱐𑱙𑲒𑲧𑲩𑲶𑴱𑴶𑴼𑴽𑴿𑵅𑵐𑵙𑶊𑶎𑶐𑶑𑶓𑶗𑶠𑶩𑻳𑻶𖩠𖩩𖫰𖫴𖬰𖬶𖭐𖭙𖽑𖾇𖾏𖾒𖿰𖿱𛲝𛲞𝅩𝅥" + u"𝅲𝅻𝆂𝆋𝅭𝆅𝆪𝆭𝉂𝉄𝟎𝟿𝨀𝨶𝨻𝩬𝪛𝪟𝪡𝪯𞀀𞀆𞀈𞀘𞀛𞀡𞀣𞀤𞀦𞀪𞄰𞄶𞅀𞅉𞋬𞋹𞥊𞣐𞣖𞥄𞥐𞥙🯰🯹" +) + +# END GENERATED CODE diff --git a/Cython/Compiler/Main.py b/Cython/Compiler/Main.py index dc4add541..4ff5a6420 100644 --- a/Cython/Compiler/Main.py +++ b/Cython/Compiler/Main.py @@ -9,8 +9,8 @@ import re import sys import io -if sys.version_info[:2] < (2, 6) or (3, 0) <= sys.version_info[:2] < (3, 3): - sys.stderr.write("Sorry, Cython requires Python 2.6+ or 3.3+, found %d.%d\n" % tuple(sys.version_info[:2])) +if sys.version_info[:2] < (2, 7) or (3, 0) <= sys.version_info[:2] < (3, 3): + sys.stderr.write("Sorry, Cython requires Python 2.7 or 3.3+, found %d.%d\n" % tuple(sys.version_info[:2])) sys.exit(1) try: @@ -30,30 +30,28 @@ from .Errors import PyrexError, CompileError, error, warning from .Symtab import ModuleScope from .. import Utils from . import Options +from .Options import CompilationOptions, default_options +from .CmdLine import parse_command_line +from .Lexicon import (unicode_start_ch_any, unicode_continuation_ch_any, + unicode_start_ch_range, unicode_continuation_ch_range) -from . import Version # legacy import needed by old PyTables versions -version = Version.version # legacy attribute - use "Cython.__version__" instead -module_name_pattern = re.compile(r"[A-Za-z_][A-Za-z0-9_]*(\.[A-Za-z_][A-Za-z0-9_]*)*$") +def _make_range_re(chrs): + out = [] + for i in range(0, len(chrs), 2): + out.append(u"{0}-{1}".format(chrs[i], chrs[i+1])) + return u"".join(out) -verbose = 0 +# py2 version looked like r"[A-Za-z_][A-Za-z0-9_]*(\.[A-Za-z_][A-Za-z0-9_]*)*$" +module_name_pattern = u"[{0}{1}][{0}{2}{1}{3}]*".format( + unicode_start_ch_any, _make_range_re(unicode_start_ch_range), + unicode_continuation_ch_any, + _make_range_re(unicode_continuation_ch_range)) +module_name_pattern = re.compile(u"{0}(\\.{0})*$".format(module_name_pattern)) -standard_include_path = os.path.abspath(os.path.join(os.path.dirname(__file__), - os.path.pardir, 'Includes')) -class CompilationData(object): - # Bundles the information that is passed from transform to transform. - # (For now, this is only) - - # While Context contains every pxd ever loaded, path information etc., - # this only contains the data related to a single compilation pass - # - # pyx ModuleNode Main code tree of this compilation. - # pxds {string : ModuleNode} Trees for the pxds used in the pyx. - # codewriter CCodeWriter Where to output final code. - # options CompilationOptions - # result CompilationResult - pass +standard_include_path = os.path.abspath( + os.path.join(os.path.dirname(os.path.dirname(__file__)), 'Includes')) class Context(object): @@ -95,8 +93,13 @@ class Context(object): self.gdb_debug_outputwriter = None + @classmethod + def from_options(cls, options): + return cls(options.include_path, options.compiler_directives, + options.cplus, options.language_level, options=options) + def set_language_level(self, level): - from .Future import print_function, unicode_literals, absolute_import, division + from .Future import print_function, unicode_literals, absolute_import, division, generator_stop future_directives = set() if level == '3str': level = 3 @@ -105,7 +108,7 @@ class Context(object): if level >= 3: future_directives.add(unicode_literals) if level >= 3: - future_directives.update([print_function, absolute_import, division]) + future_directives.update([print_function, absolute_import, division, generator_stop]) self.language_level = level self.future_directives = future_directives if level >= 3: @@ -123,15 +126,6 @@ class Context(object): self._interned[key] = value return value - def intern_value(self, value, *key): - key = (type(value), value) + key - try: - return self._interned[key] - except KeyError: - pass - self._interned[key] = value - return value - # pipeline creation functions can now be found in Pipeline.py def process_pxd(self, source_desc, scope, module_name): @@ -179,7 +173,7 @@ class Context(object): if not module_name_pattern.match(qualified_name): raise CompileError(pos or (module_name, 0, 0), - "'%s' is not a valid module name" % module_name) + u"'%s' is not a valid module name" % module_name) if relative_to: if debug_find_module: @@ -216,7 +210,7 @@ class Context(object): # look for the non-existing pxd file next time. scope.pxd_file_loaded = True package_pathname = self.search_include_directories(qualified_name, ".py", pos) - if package_pathname and package_pathname.endswith('__init__.py'): + if package_pathname and package_pathname.endswith(Utils.PACKAGE_FILES): pass else: error(pos, "'%s.pxd' not found" % qualified_name.replace('.', os.sep)) @@ -248,25 +242,6 @@ class Context(object): # for a dotted filename, and its containing package root # directory is searched first for a non-dotted filename. pxd = self.search_include_directories(qualified_name, ".pxd", pos, sys_path=sys_path) - if pxd is None: # XXX Keep this until Includes/Deprecated is removed - if (qualified_name.startswith('python') or - qualified_name in ('stdlib', 'stdio', 'stl')): - standard_include_path = os.path.abspath(os.path.normpath( - os.path.join(os.path.dirname(__file__), os.path.pardir, 'Includes'))) - deprecated_include_path = os.path.join(standard_include_path, 'Deprecated') - self.include_directories.append(deprecated_include_path) - try: - pxd = self.search_include_directories(qualified_name, ".pxd", pos) - finally: - self.include_directories.pop() - if pxd: - name = qualified_name - if name.startswith('python'): - warning(pos, "'%s' is deprecated, use 'cpython'" % name, 1) - elif name in ('stdlib', 'stdio'): - warning(pos, "'%s' is deprecated, use 'libc.%s'" % (name, name), 1) - elif name in ('stl'): - warning(pos, "'%s' is deprecated, use 'libcpp.*.*'" % name, 1) if pxd is None and Options.cimport_from_pyx: return self.find_pyx_file(qualified_name, pos) return pxd @@ -315,7 +290,7 @@ class Context(object): if kind == "cimport": dep_path = self.find_pxd_file(name, pos) elif kind == "include": - dep_path = self.search_include_directories(name, pos) + dep_path = self.search_include_directories(name, "", pos) else: continue if dep_path and Utils.file_newer_than(dep_path, c_time): @@ -473,22 +448,29 @@ def create_default_resultobj(compilation_source, options): def run_pipeline(source, options, full_module_name=None, context=None): from . import Pipeline + # ensure that the inputs are unicode (for Python 2) + if sys.version_info[0] == 2: + source = Utils.decode_filename(source) + if full_module_name: + full_module_name = Utils.decode_filename(full_module_name) + source_ext = os.path.splitext(source)[1] - options.configure_language_defaults(source_ext[1:]) # py/pyx + options.configure_language_defaults(source_ext[1:]) # py/pyx if context is None: - context = options.create_context() + context = Context.from_options(options) # Set up source object cwd = os.getcwd() abs_path = os.path.abspath(source) full_module_name = full_module_name or context.extract_module_name(source, options) + full_module_name = EncodedString(full_module_name) Utils.raise_error_if_module_name_forbidden(full_module_name) if options.relative_path_in_code_position_comments: rel_path = full_module_name.replace('.', os.sep) + source_ext if not abs_path.endswith(rel_path): - rel_path = source # safety measure to prevent printing incorrect paths + rel_path = source # safety measure to prevent printing incorrect paths else: rel_path = abs_path source_desc = FileSourceDescriptor(abs_path, rel_path) @@ -512,6 +494,12 @@ def run_pipeline(source, options, full_module_name=None, context=None): pipeline = Pipeline.create_pyx_pipeline(context, options, result) context.setup_errors(options, result) + + if '.' in full_module_name and '.' in os.path.splitext(os.path.basename(abs_path))[0]: + warning((source_desc, 1, 0), + "Dotted filenames ('%s') are deprecated." + " Please use the normal Python package directory layout." % os.path.basename(abs_path), level=1) + err, enddata = Pipeline.run_pipeline(pipeline, source) context.teardown_errors(err, options, result) return result @@ -534,146 +522,6 @@ class CompilationSource(object): self.cwd = cwd -class CompilationOptions(object): - r""" - See default_options at the end of this module for a list of all possible - options and CmdLine.usage and CmdLine.parse_command_line() for their - meaning. - """ - def __init__(self, defaults=None, **kw): - self.include_path = [] - if defaults: - if isinstance(defaults, CompilationOptions): - defaults = defaults.__dict__ - else: - defaults = default_options - - options = dict(defaults) - options.update(kw) - - # let's assume 'default_options' contains a value for most known compiler options - # and validate against them - unknown_options = set(options) - set(default_options) - # ignore valid options that are not in the defaults - unknown_options.difference_update(['include_path']) - if unknown_options: - message = "got unknown compilation option%s, please remove: %s" % ( - 's' if len(unknown_options) > 1 else '', - ', '.join(unknown_options)) - raise ValueError(message) - - directive_defaults = Options.get_directive_defaults() - directives = dict(options['compiler_directives']) # copy mutable field - # check for invalid directives - unknown_directives = set(directives) - set(directive_defaults) - if unknown_directives: - message = "got unknown compiler directive%s: %s" % ( - 's' if len(unknown_directives) > 1 else '', - ', '.join(unknown_directives)) - raise ValueError(message) - options['compiler_directives'] = directives - if directives.get('np_pythran', False) and not options['cplus']: - import warnings - warnings.warn("C++ mode forced when in Pythran mode!") - options['cplus'] = True - if 'language_level' in directives and 'language_level' not in kw: - options['language_level'] = directives['language_level'] - elif not options.get('language_level'): - options['language_level'] = directive_defaults.get('language_level') - if 'formal_grammar' in directives and 'formal_grammar' not in kw: - options['formal_grammar'] = directives['formal_grammar'] - if options['cache'] is True: - options['cache'] = os.path.join(Utils.get_cython_cache_dir(), 'compiler') - - self.__dict__.update(options) - - def configure_language_defaults(self, source_extension): - if source_extension == 'py': - if self.compiler_directives.get('binding') is None: - self.compiler_directives['binding'] = True - - def create_context(self): - return Context(self.include_path, self.compiler_directives, - self.cplus, self.language_level, options=self) - - def get_fingerprint(self): - r""" - Return a string that contains all the options that are relevant for cache invalidation. - """ - # Collect only the data that can affect the generated file(s). - data = {} - - for key, value in self.__dict__.items(): - if key in ['show_version', 'errors_to_stderr', 'verbose', 'quiet']: - # verbosity flags have no influence on the compilation result - continue - elif key in ['output_file', 'output_dir']: - # ignore the exact name of the output file - continue - elif key in ['timestamps']: - # the cache cares about the content of files, not about the timestamps of sources - continue - elif key in ['cache']: - # hopefully caching has no influence on the compilation result - continue - elif key in ['compiler_directives']: - # directives passed on to the C compiler do not influence the generated C code - continue - elif key in ['include_path']: - # this path changes which headers are tracked as dependencies, - # it has no influence on the generated C code - continue - elif key in ['working_path']: - # this path changes where modules and pxd files are found; - # their content is part of the fingerprint anyway, their - # absolute path does not matter - continue - elif key in ['create_extension']: - # create_extension() has already mangled the options, e.g., - # embedded_metadata, when the fingerprint is computed so we - # ignore it here. - continue - elif key in ['build_dir']: - # the (temporary) directory where we collect dependencies - # has no influence on the C output - continue - elif key in ['use_listing_file', 'generate_pxi', 'annotate', 'annotate_coverage_xml']: - # all output files are contained in the cache so the types of - # files generated must be part of the fingerprint - data[key] = value - elif key in ['formal_grammar', 'evaluate_tree_assertions']: - # these bits can change whether compilation to C passes/fails - data[key] = value - elif key in ['embedded_metadata', 'emit_linenums', 'c_line_in_traceback', 'gdb_debug', 'relative_path_in_code_position_comments']: - # the generated code contains additional bits when these are set - data[key] = value - elif key in ['cplus', 'language_level', 'compile_time_env', 'np_pythran']: - # assorted bits that, e.g., influence the parser - data[key] = value - elif key == ['capi_reexport_cincludes']: - if self.capi_reexport_cincludes: - # our caching implementation does not yet include fingerprints of all the header files - raise NotImplementedError('capi_reexport_cincludes is not compatible with Cython caching') - elif key == ['common_utility_include_dir']: - if self.common_utility_include_dir: - raise NotImplementedError('common_utility_include_dir is not compatible with Cython caching yet') - else: - # any unexpected option should go into the fingerprint; it's better - # to recompile than to return incorrect results from the cache. - data[key] = value - - def to_fingerprint(item): - r""" - Recursively turn item into a string, turning dicts into lists with - deterministic ordering. - """ - if isinstance(item, dict): - item = sorted([(repr(key), to_fingerprint(value)) for key, value in item.items()]) - return repr(item) - - return to_fingerprint(data) - - class CompilationResult(object): """ Results from the Cython compiler: @@ -736,7 +584,7 @@ def compile_multiple(sources, options): if these are specified in the options. """ # run_pipeline creates the context - # context = options.create_context() + # context = Context.from_options(options) sources = [os.path.abspath(source) for source in sources] processed = set() results = CompilationResultSet() @@ -747,7 +595,7 @@ def compile_multiple(sources, options): for source in sources: if source not in processed: if context is None: - context = options.create_context() + context = Context.from_options(options) output_filename = get_output_filename(source, cwd, options) out_of_date = context.c_file_out_of_date(source, output_filename) if (not timestamps) or out_of_date: @@ -791,7 +639,6 @@ def search_include_directories(dirs, qualified_name, suffix, pos, include=False) The 'include' option will disable package dereferencing. """ - if pos: file_desc = pos[0] if not isinstance(file_desc, FileSourceDescriptor): @@ -801,35 +648,59 @@ def search_include_directories(dirs, qualified_name, suffix, pos, include=False) else: dirs = (Utils.find_root_package_dir(file_desc.filename),) + dirs + # search for dotted filename e.g. <dir>/foo.bar.pxd dotted_filename = qualified_name if suffix: dotted_filename += suffix - if not include: - names = qualified_name.split('.') - package_names = tuple(names[:-1]) - module_name = names[-1] - module_filename = module_name + suffix - package_filename = "__init__" + suffix - for dirname in dirs: path = os.path.join(dirname, dotted_filename) if os.path.exists(path): + if '.' in qualified_name and '.' in os.path.splitext(dotted_filename)[0]: + warning(pos, "Dotted filenames ('%s') are deprecated." + " Please use the normal Python package directory layout." % dotted_filename, level=1) return path - if not include: - package_dir = Utils.check_package_dir(dirname, package_names) + # search for filename in package structure e.g. <dir>/foo/bar.pxd or <dir>/foo/bar/__init__.pxd + if not include: + + names = qualified_name.split('.') + package_names = tuple(names[:-1]) + module_name = names[-1] + + # search for standard packages first - PEP420 + namespace_dirs = [] + for dirname in dirs: + package_dir, is_namespace = Utils.check_package_dir(dirname, package_names) if package_dir is not None: - path = os.path.join(package_dir, module_filename) - if os.path.exists(path): - return path - path = os.path.join(package_dir, module_name, - package_filename) - if os.path.exists(path): + if is_namespace: + namespace_dirs.append(package_dir) + continue + path = search_module_in_dir(package_dir, module_name, suffix) + if path: return path + + # search for namespaces second - PEP420 + for package_dir in namespace_dirs: + path = search_module_in_dir(package_dir, module_name, suffix) + if path: + return path + return None +@Utils.cached_function +def search_module_in_dir(package_dir, module_name, suffix): + # matches modules of the form: <dir>/foo/bar.pxd + path = Utils.find_versioned_file(package_dir, module_name, suffix) + + # matches modules of the form: <dir>/foo/bar/__init__.pxd + if not path and suffix: + path = Utils.find_versioned_file(os.path.join(package_dir, module_name), "__init__", suffix) + + return path + + # ------------------------------------------------------------------------ # # Main command-line entry point @@ -844,14 +715,14 @@ def main(command_line = 0): args = sys.argv[1:] any_failures = 0 if command_line: - from .CmdLine import parse_command_line options, sources = parse_command_line(args) else: options = CompilationOptions(default_options) sources = args if options.show_version: - sys.stderr.write("Cython version %s\n" % version) + from .. import __version__ + sys.stderr.write("Cython version %s\n" % __version__) if options.working_path!="": os.chdir(options.working_path) try: @@ -863,42 +734,3 @@ def main(command_line = 0): any_failures = 1 if any_failures: sys.exit(1) - - -# ------------------------------------------------------------------------ -# -# Set the default options depending on the platform -# -# ------------------------------------------------------------------------ - -default_options = dict( - show_version = 0, - use_listing_file = 0, - errors_to_stderr = 1, - cplus = 0, - output_file = None, - annotate = None, - annotate_coverage_xml = None, - generate_pxi = 0, - capi_reexport_cincludes = 0, - working_path = "", - timestamps = None, - verbose = 0, - quiet = 0, - compiler_directives = {}, - embedded_metadata = {}, - evaluate_tree_assertions = False, - emit_linenums = False, - relative_path_in_code_position_comments = True, - c_line_in_traceback = True, - language_level = None, # warn but default to 2 - formal_grammar = False, - gdb_debug = False, - compile_time_env = None, - common_utility_include_dir = None, - output_dir=None, - build_dir=None, - cache=None, - create_extension=None, - np_pythran=False -) diff --git a/Cython/Compiler/MemoryView.py b/Cython/Compiler/MemoryView.py index 0406d6c71..7f4c043c9 100644 --- a/Cython/Compiler/MemoryView.py +++ b/Cython/Compiler/MemoryView.py @@ -100,8 +100,15 @@ def put_acquire_memoryviewslice(lhs_cname, lhs_type, lhs_pos, rhs, code, def put_assign_to_memviewslice(lhs_cname, rhs, rhs_cname, memviewslicetype, code, have_gil=False, first_assignment=False): + if lhs_cname == rhs_cname: + # self assignment is tricky because memoryview xdecref clears the memoryview + # thus invalidating both sides of the assignment. Therefore make it actually do nothing + code.putln("/* memoryview self assignment no-op */") + return + if not first_assignment: - code.put_xdecref_memoryviewslice(lhs_cname, have_gil=have_gil) + code.put_xdecref(lhs_cname, memviewslicetype, + have_gil=have_gil) if not rhs.result_in_temp(): rhs.make_owned_memoryviewslice(code) @@ -167,7 +174,7 @@ def valid_memslice_dtype(dtype, i=0): valid_memslice_dtype(dtype.base_type, i + 1)) or dtype.is_numeric or dtype.is_pyobject or - dtype.is_fused or # accept this as it will be replaced by specializations later + dtype.is_fused or # accept this as it will be replaced by specializations later (dtype.is_typedef and valid_memslice_dtype(dtype.typedef_base_type)) ) @@ -248,7 +255,7 @@ class MemoryViewSliceBufferEntry(Buffer.BufferEntry): return bufp - def generate_buffer_slice_code(self, code, indices, dst, have_gil, + def generate_buffer_slice_code(self, code, indices, dst, dst_type, have_gil, have_slices, directives): """ Slice a memoryviewslice. @@ -265,7 +272,7 @@ class MemoryViewSliceBufferEntry(Buffer.BufferEntry): code.putln("%(dst)s.data = %(src)s.data;" % locals()) code.putln("%(dst)s.memview = %(src)s.memview;" % locals()) - code.put_incref_memoryviewslice(dst) + code.put_incref_memoryviewslice(dst, dst_type, have_gil=have_gil) all_dimensions_direct = all(access == 'direct' for access, packing in self.type.axes) suboffset_dim_temp = [] @@ -404,8 +411,8 @@ def get_is_contig_utility(contig_type, ndim): return utility -def slice_iter(slice_type, slice_result, ndim, code): - if slice_type.is_c_contig or slice_type.is_f_contig: +def slice_iter(slice_type, slice_result, ndim, code, force_strided=False): + if (slice_type.is_c_contig or slice_type.is_f_contig) and not force_strided: return ContigSliceIter(slice_type, slice_result, ndim, code) else: return StridedSliceIter(slice_type, slice_result, ndim, code) @@ -489,7 +496,7 @@ def copy_c_or_fortran_cname(memview): def get_copy_new_utility(pos, from_memview, to_memview): if (from_memview.dtype != to_memview.dtype and - not (from_memview.dtype.is_const and from_memview.dtype.const_base_type == to_memview.dtype)): + not (from_memview.dtype.is_cv_qualified and from_memview.dtype.cv_base_type == to_memview.dtype)): error(pos, "dtypes must be the same!") return if len(from_memview.axes) != len(to_memview.axes): @@ -654,13 +661,13 @@ def is_cf_contig(specs): is_c_contig = True elif (specs[-1] == ('direct','contig') and - all(axis == ('direct','follow') for axis in specs[:-1])): + all(axis == ('direct','follow') for axis in specs[:-1])): # c_contiguous: 'follow', 'follow', ..., 'follow', 'contig' is_c_contig = True elif (len(specs) > 1 and - specs[0] == ('direct','contig') and - all(axis == ('direct','follow') for axis in specs[1:])): + specs[0] == ('direct','contig') and + all(axis == ('direct','follow') for axis in specs[1:])): # f_contiguous: 'contig', 'follow', 'follow', ..., 'follow' is_f_contig = True @@ -809,7 +816,7 @@ context = { 'memview_struct_name': memview_objstruct_cname, 'max_dims': Options.buffer_max_dims, 'memviewslice_name': memviewslice_cname, - 'memslice_init': memslice_entry_init, + 'memslice_init': PyrexTypes.MemoryViewSliceType.default_value, } memviewslice_declare_code = load_memview_c_utility( "MemviewSliceStruct", @@ -835,7 +842,7 @@ overlapping_utility = load_memview_c_utility("OverlappingSlices", context) copy_contents_new_utility = load_memview_c_utility( "MemviewSliceCopyTemplate", context, - requires=[], # require cython_array_utility_code + requires=[], # require cython_array_utility_code ) view_utility_code = load_memview_cy_utility( @@ -850,7 +857,7 @@ view_utility_code = load_memview_cy_utility( copy_contents_new_utility, ModuleNode.capsule_utility_code], ) -view_utility_whitelist = ('array', 'memoryview', 'array_cwrapper', +view_utility_allowlist = ('array', 'memoryview', 'array_cwrapper', 'generic', 'strided', 'indirect', 'contiguous', 'indirect_contiguous') diff --git a/Cython/Compiler/ModuleNode.py b/Cython/Compiler/ModuleNode.py index fa896e619..77a92368c 100644 --- a/Cython/Compiler/ModuleNode.py +++ b/Cython/Compiler/ModuleNode.py @@ -14,6 +14,7 @@ import json import operator import os import re +import sys from .PyrexTypes import CPtrType from . import Future @@ -29,10 +30,22 @@ from . import Pythran from .Errors import error, warning from .PyrexTypes import py_object_type from ..Utils import open_new_file, replace_suffix, decode_filename, build_hex_version -from .Code import UtilityCode, IncludeCode -from .StringEncoding import EncodedString +from .Code import UtilityCode, IncludeCode, TempitaUtilityCode +from .StringEncoding import EncodedString, encoded_string_or_bytes_literal from .Pythran import has_np_pythran + +def replace_suffix_encoded(path, newsuf): + # calls replace suffix and returns a EncodedString or BytesLiteral with the encoding set + newpath = replace_suffix(path, newsuf) + return as_encoded_filename(newpath) + +def as_encoded_filename(path): + # wraps the path with either EncodedString or BytesLiteral (depending on its input type) + # and sets the encoding to the file system encoding + return encoded_string_or_bytes_literal(path, sys.getfilesystemencoding()) + + def check_c_declarations_pxd(module_node): module_node.scope.check_c_classes_pxd() return module_node @@ -50,6 +63,10 @@ def generate_c_code_config(env, options): else: emit_linenums = options.emit_linenums + if hasattr(options, "emit_code_comments"): + print('Warning: option emit_code_comments is deprecated. ' + 'Instead, use compiler directive emit_code_comments.') + return Code.CCodeConfig( emit_linenums=emit_linenums, emit_code_comments=env.directives['emit_code_comments'], @@ -80,6 +97,10 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): # CodeGenerator, and tell that CodeGenerator to generate code # from multiple sources. assert isinstance(self.body, Nodes.StatListNode) + if scope.directives != self.scope.directives: + # merged in nodes should keep their original compiler directives + # (for example inline cdef functions) + tree = Nodes.CompilerDirectivesNode(tree.pos, body=tree, directives=scope.directives) if isinstance(tree, Nodes.StatListNode): self.body.stats.extend(tree.stats) else: @@ -105,6 +126,13 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): self.scope.merge_in(scope) + def with_compiler_directives(self): + # When merging a utility code module into the user code we need to preserve + # the original compiler directives. This returns the body of the module node, + # wrapped in its set of directives. + body = Nodes.CompilerDirectivesNode(self.pos, directives=self.directives, body=self.body) + return body + def analyse_declarations(self, env): if has_np_pythran(env): Pythran.include_pythran_generic(env) @@ -131,8 +159,8 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): self.create_import_star_conversion_utility_code(env) for name, entry in sorted(env.entries.items()): if (entry.create_wrapper and entry.scope is env - and entry.is_type and entry.type.is_enum): - entry.type.create_type_wrapper(env) + and entry.is_type and (entry.type.is_enum or entry.type.is_cpp_enum)): + entry.type.create_type_wrapper(env) def process_implementation(self, options, result): env = self.scope @@ -161,65 +189,96 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): h_vars = h_entries(env.var_entries) h_funcs = h_entries(env.cfunc_entries) h_extension_types = h_entries(env.c_class_entries) - if h_types or h_vars or h_funcs or h_extension_types: - result.h_file = replace_suffix(result.c_file, ".h") - h_code = Code.CCodeWriter() + if h_types or h_vars or h_funcs or h_extension_types: + result.h_file = replace_suffix_encoded(result.c_file, ".h") + h_code_writer = Code.CCodeWriter() c_code_config = generate_c_code_config(env, options) - Code.GlobalState(h_code, self, c_code_config) + globalstate = Code.GlobalState(h_code_writer, self, c_code_config) + globalstate.initialize_main_h_code() # in-case utility code is used in the header + h_code_start = globalstate.parts['h_code'] + h_code_main = globalstate.parts['type_declarations'] + h_code_end = globalstate.parts['end'] if options.generate_pxi: - result.i_file = replace_suffix(result.c_file, ".pxi") + result.i_file = replace_suffix_encoded(result.c_file, ".pxi") i_code = Code.PyrexCodeWriter(result.i_file) else: i_code = None - h_code.put_generated_by() - h_guard = Naming.h_guard_prefix + self.api_name(env) - h_code.put_h_guard(h_guard) - h_code.putln("") - h_code.putln('#include "Python.h"') - self.generate_type_header_code(h_types, h_code) + h_code_start.put_generated_by() + h_guard = self.api_name(Naming.h_guard_prefix, env) + h_code_start.put_h_guard(h_guard) + h_code_start.putln("") + h_code_start.putln('#include "Python.h"') + self.generate_type_header_code(h_types, h_code_start) if options.capi_reexport_cincludes: - self.generate_includes(env, [], h_code) - h_code.putln("") - api_guard = Naming.api_guard_prefix + self.api_name(env) - h_code.putln("#ifndef %s" % api_guard) - h_code.putln("") - self.generate_extern_c_macro_definition(h_code) - h_code.putln("") - self.generate_dl_import_macro(h_code) + self.generate_includes(env, [], h_code_start) + h_code_start.putln("") + api_guard = self.api_name(Naming.api_guard_prefix, env) + h_code_start.putln("#ifndef %s" % api_guard) + h_code_start.putln("") + self.generate_extern_c_macro_definition(h_code_start) + h_code_start.putln("") + self.generate_dl_import_macro(h_code_start) if h_extension_types: - h_code.putln("") + h_code_main.putln("") for entry in h_extension_types: - self.generate_cclass_header_code(entry.type, h_code) + self.generate_cclass_header_code(entry.type, h_code_main) if i_code: self.generate_cclass_include_code(entry.type, i_code) if h_funcs: - h_code.putln("") + h_code_main.putln("") for entry in h_funcs: - self.generate_public_declaration(entry, h_code, i_code) + self.generate_public_declaration(entry, h_code_main, i_code) if h_vars: - h_code.putln("") + h_code_main.putln("") for entry in h_vars: - self.generate_public_declaration(entry, h_code, i_code) - h_code.putln("") - h_code.putln("#endif /* !%s */" % api_guard) - h_code.putln("") - h_code.putln("/* WARNING: the interface of the module init function changed in CPython 3.5. */") - h_code.putln("/* It now returns a PyModuleDef instance instead of a PyModule instance. */") - h_code.putln("") - h_code.putln("#if PY_MAJOR_VERSION < 3") - h_code.putln("PyMODINIT_FUNC init%s(void);" % env.module_name) - h_code.putln("#else") - h_code.putln("PyMODINIT_FUNC %s(void);" % self.mod_init_func_cname('PyInit', env)) - h_code.putln("#endif") - h_code.putln("") - h_code.putln("#endif /* !%s */" % h_guard) - - f = open_new_file(result.h_file) - try: - h_code.copyto(f) - finally: - f.close() + self.generate_public_declaration(entry, h_code_main, i_code) + h_code_main.putln("") + h_code_main.putln("#endif /* !%s */" % api_guard) + h_code_main.putln("") + h_code_main.putln("/* WARNING: the interface of the module init function changed in CPython 3.5. */") + h_code_main.putln("/* It now returns a PyModuleDef instance instead of a PyModule instance. */") + h_code_main.putln("") + h_code_main.putln("#if PY_MAJOR_VERSION < 3") + if env.module_name.isascii(): + py2_mod_name = env.module_name + else: + py2_mod_name = env.module_name.encode("ascii", errors="ignore").decode("utf-8") + h_code_main.putln('#error "Unicode module names are not supported in Python 2";') + h_code_main.putln("PyMODINIT_FUNC init%s(void);" % py2_mod_name) + h_code_main.putln("#else") + py3_mod_func_name = self.mod_init_func_cname('PyInit', env) + warning_string = EncodedString('Use PyImport_AppendInittab("%s", %s) instead of calling %s directly.' % ( + py2_mod_name, py3_mod_func_name, py3_mod_func_name)) + h_code_main.putln('/* WARNING: %s from Python 3.5 */' % warning_string.rstrip('.')) + h_code_main.putln("PyMODINIT_FUNC %s(void);" % py3_mod_func_name) + h_code_main.putln("") + h_code_main.putln("#if PY_VERSION_HEX >= 0x03050000 " + "&& (defined(__GNUC__) || defined(__clang__) || defined(_MSC_VER) " + "|| (defined(__cplusplus) && __cplusplus >= 201402L))") + h_code_main.putln("#if defined(__cplusplus) && __cplusplus >= 201402L") + h_code_main.putln("[[deprecated(%s)]] inline" % warning_string.as_c_string_literal()) + h_code_main.putln("#elif defined(__GNUC__) || defined(__clang__)") + h_code_main.putln('__attribute__ ((__deprecated__(%s), __unused__)) __inline__' % ( + warning_string.as_c_string_literal())) + h_code_main.putln("#elif defined(_MSC_VER)") + h_code_main.putln('__declspec(deprecated(%s)) __inline' % ( + warning_string.as_c_string_literal())) + h_code_main.putln('#endif') + h_code_main.putln("static PyObject* __PYX_WARN_IF_INIT_CALLED(PyObject* res) {") + h_code_main.putln("return res;") + h_code_main.putln("}") + # Function call is converted to warning macro; uncalled (pointer) is not + h_code_main.putln('#define %s() __PYX_WARN_IF_INIT_CALLED(%s())' % ( + py3_mod_func_name, py3_mod_func_name)) + h_code_main.putln('#endif') + h_code_main.putln('#endif') + + h_code_end.putln("") + h_code_end.putln("#endif /* !%s */" % h_guard) + + with open_new_file(result.h_file) as f: + h_code_writer.copyto(f) def generate_public_declaration(self, entry, h_code, i_code): h_code.putln("%s %s;" % ( @@ -229,8 +288,9 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): i_code.putln("cdef extern %s" % ( entry.type.declaration_code(entry.cname, pyrex=1))) - def api_name(self, env): - return env.qualified_name.replace(".", "__") + def api_name(self, prefix, env): + api_name = self.punycode_module_name(prefix, env.qualified_name) + return api_name.replace(".", "__") def generate_api_code(self, env, options, result): def api_entries(entries, pxd=0): @@ -240,12 +300,12 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): api_funcs = api_entries(env.cfunc_entries) api_extension_types = api_entries(env.c_class_entries) if api_vars or api_funcs or api_extension_types: - result.api_file = replace_suffix(result.c_file, "_api.h") + result.api_file = replace_suffix_encoded(result.c_file, "_api.h") h_code = Code.CCodeWriter() c_code_config = generate_c_code_config(env, options) Code.GlobalState(h_code, self, c_code_config) h_code.put_generated_by() - api_guard = Naming.api_guard_prefix + self.api_name(env) + api_guard = self.api_name(Naming.api_guard_prefix, env) h_code.put_h_guard(api_guard) # Work around https://bugs.python.org/issue4709 h_code.putln('#ifdef __MINGW64__') @@ -254,7 +314,9 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): h_code.putln('#include "Python.h"') if result.h_file: - h_code.putln('#include "%s"' % os.path.basename(result.h_file)) + h_filename = os.path.basename(result.h_file) + h_filename = as_encoded_filename(h_filename) + h_code.putln('#include %s' % h_filename.as_c_string_literal()) if api_extension_types: h_code.putln("") for entry in api_extension_types: @@ -274,7 +336,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): for entry in api_vars: type = CPtrType(entry.type) cname = env.mangle(Naming.varptr_prefix_api, entry.name) - h_code.putln("static %s = 0;" % type.declaration_code(cname)) + h_code.putln("static %s = 0;" % type.declaration_code(cname)) h_code.putln("#define %s (*%s)" % (entry.name, cname)) h_code.put(UtilityCode.load_as_string("PyIdentifierFromString", "ImportExport.c")[0]) if api_vars: @@ -285,22 +347,22 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): h_code.put(UtilityCode.load_as_string("TypeImport", "ImportExport.c")[0]) h_code.put(UtilityCode.load_as_string("TypeImport", "ImportExport.c")[1]) h_code.putln("") - h_code.putln("static int import_%s(void) {" % self.api_name(env)) + h_code.putln("static int %s(void) {" % self.api_name("import", env)) h_code.putln("PyObject *module = 0;") - h_code.putln('module = PyImport_ImportModule("%s");' % env.qualified_name) + h_code.putln('module = PyImport_ImportModule(%s);' % env.qualified_name.as_c_string_literal()) h_code.putln("if (!module) goto bad;") for entry in api_funcs: cname = env.mangle(Naming.func_prefix_api, entry.name) sig = entry.type.signature_string() h_code.putln( - 'if (__Pyx_ImportFunction(module, "%s", (void (**)(void))&%s, "%s") < 0) goto bad;' - % (entry.name, cname, sig)) + 'if (__Pyx_ImportFunction(module, %s, (void (**)(void))&%s, "%s") < 0) goto bad;' + % (entry.name.as_c_string_literal(), cname, sig)) for entry in api_vars: cname = env.mangle(Naming.varptr_prefix_api, entry.name) sig = entry.type.empty_declaration_code() h_code.putln( - 'if (__Pyx_ImportVoidPtr(module, "%s", (void **)&%s, "%s") < 0) goto bad;' - % (entry.name, cname, sig)) + 'if (__Pyx_ImportVoidPtr(module, %s, (void **)&%s, "%s") < 0) goto bad;' + % (entry.name.as_c_string_literal(), cname, sig)) with ModuleImportGenerator(h_code, imported_modules={env.qualified_name: 'module'}) as import_generator: for entry in api_extension_types: self.generate_type_import_call(entry.type, h_code, import_generator, error_code="goto bad;") @@ -342,7 +404,11 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): modules = self.referenced_modules if Options.annotate or options.annotate: - rootwriter = Annotate.AnnotationCCodeWriter() + show_entire_c_code = Options.annotate == "fullc" or options.annotate == "fullc" + rootwriter = Annotate.AnnotationCCodeWriter( + show_entire_c_code=show_entire_c_code, + source_desc=self.compilation_source.source_desc, + ) else: rootwriter = Code.CCodeWriter() @@ -364,24 +430,24 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): globalstate.use_utility_code(refnanny_utility_code) code = globalstate['before_global_var'] - code.putln('#define __Pyx_MODULE_NAME "%s"' % self.full_module_name) - module_is_main = "%s%s" % (Naming.module_is_main, self.full_module_name.replace('.', '__')) + code.putln('#define __Pyx_MODULE_NAME %s' % + self.full_module_name.as_c_string_literal()) + module_is_main = self.is_main_module_flag_cname() code.putln("extern int %s;" % module_is_main) code.putln("int %s = 0;" % module_is_main) code.putln("") - code.putln("/* Implementation of '%s' */" % env.qualified_name) + code.putln("/* Implementation of %s */" % env.qualified_name.as_c_string_literal()) code = globalstate['late_includes'] - code.putln("/* Late includes */") self.generate_includes(env, modules, code, early=False) - code = globalstate['all_the_rest'] + code = globalstate['module_code'] self.generate_cached_builtins_decls(env, code) - self.generate_lambda_definitions(env, code) + # generate normal variable and function definitions + self.generate_lambda_definitions(env, code) self.generate_variable_definitions(env, code) - self.body.generate_function_definitions(env, code) code.mark_pos(None) @@ -389,11 +455,15 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): self.generate_method_table(env, code) if env.has_import_star: self.generate_import_star(env, code) - self.generate_pymoduledef_struct(env, code) # initialise the macro to reduce the code size of one-time functionality code.putln(UtilityCode.load_as_string("SmallCodeConfig", "ModuleSetupCode.c")[0].strip()) + self.generate_module_state_start(env, globalstate['module_state']) + self.generate_module_state_defines(env, globalstate['module_state_defines']) + self.generate_module_state_clear(env, globalstate['module_state_clear']) + self.generate_module_state_traverse(env, globalstate['module_state_traverse']) + # init_globals is inserted before this self.generate_module_init_func(modules[:-1], env, globalstate['init_module']) self.generate_module_cleanup_func(env, globalstate['cleanup_module']) @@ -408,6 +478,8 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): globalstate.use_utility_code(utilcode) globalstate.finalize_main_c_code() + self.generate_module_state_end(env, modules, globalstate) + f = open_new_file(result.c_file) try: rootwriter.copyto(f) @@ -429,11 +501,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): except ImportError: import xml.etree.ElementTree as ET coverage_xml = ET.parse(coverage_xml_filename).getroot() - if hasattr(coverage_xml, 'iter'): - iterator = coverage_xml.iter() # Python 2.7 & 3.2+ - else: - iterator = coverage_xml.getiterator() - for el in iterator: + for el in coverage_xml.iter(): el.tail = None # save some memory else: coverage_xml = None @@ -469,16 +537,18 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): markers = ccodewriter.buffer.allmarkers() d = defaultdict(list) - for c_lineno, cython_lineno in enumerate(markers): - if cython_lineno > 0: - d[cython_lineno].append(c_lineno + 1) + for c_lineno, (src_desc, src_lineno) in enumerate(markers): + if src_lineno > 0 and src_desc.filename is not None: + d[src_desc, src_lineno].append(c_lineno + 1) tb.start('LineNumberMapping') - for cython_lineno, c_linenos in sorted(d.items()): + for (src_desc, src_lineno), c_linenos in sorted(d.items()): + assert src_desc.filename is not None tb.add_entry( 'LineNumber', c_linenos=' '.join(map(str, c_linenos)), - cython_lineno=str(cython_lineno), + src_path=src_desc.filename, + src_lineno=str(src_lineno), ) tb.end('LineNumberMapping') tb.serialize() @@ -619,8 +689,8 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): for module in modules: defined_here = module is env modulecode.putln("") - modulecode.putln("/* Module declarations from '%s' */" % module.qualified_name) - self.generate_c_class_declarations(module, modulecode, defined_here) + modulecode.putln("/* Module declarations from %s */" % module.qualified_name.as_c_string_literal()) + self.generate_c_class_declarations(module, modulecode, defined_here, globalstate) self.generate_cvariable_declarations(module, modulecode, defined_here) self.generate_cfunction_declarations(module, modulecode, defined_here) @@ -638,6 +708,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.putln("#ifndef PY_SSIZE_T_CLEAN") code.putln("#define PY_SSIZE_T_CLEAN") code.putln("#endif /* PY_SSIZE_T_CLEAN */") + self._put_setup_code(code, "InitLimitedAPI") for inc in sorted(env.c_includes.values(), key=IncludeCode.sortkey): if inc.location == inc.INITIAL: @@ -645,14 +716,16 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.putln("#ifndef Py_PYTHON_H") code.putln(" #error Python headers needed to compile C extensions, " "please install development version of Python.") - code.putln("#elif PY_VERSION_HEX < 0x02060000 || " + code.putln("#elif PY_VERSION_HEX < 0x02070000 || " "(0x03000000 <= PY_VERSION_HEX && PY_VERSION_HEX < 0x03030000)") - code.putln(" #error Cython requires Python 2.6+ or Python 3.3+.") + code.putln(" #error Cython requires Python 2.7+ or Python 3.3+.") code.putln("#else") code.globalstate["end"].putln("#endif /* Py_PYTHON_H */") from .. import __version__ code.putln('#define CYTHON_ABI "%s"' % __version__.replace('.', '_')) + code.putln('#define __PYX_ABI_MODULE_NAME "_cython_" CYTHON_ABI') + code.putln('#define __PYX_TYPE_MODULE_PREFIX __PYX_ABI_MODULE_NAME "."') code.putln('#define CYTHON_HEX_VERSION %s' % build_hex_version(__version__)) code.putln("#define CYTHON_FUTURE_DIVISION %d" % ( Future.division in env.context.future_directives)) @@ -683,8 +756,8 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): self.generate_extern_c_macro_definition(code) code.putln("") - code.putln("#define %s" % Naming.h_guard_prefix + self.api_name(env)) - code.putln("#define %s" % Naming.api_guard_prefix + self.api_name(env)) + code.putln("#define %s" % self.api_name(Naming.h_guard_prefix, env)) + code.putln("#define %s" % self.api_name(Naming.api_guard_prefix, env)) code.putln("/* Early includes */") self.generate_includes(env, cimported_modules, code, late=False) code.putln("") @@ -721,6 +794,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.putln('#define __Pyx_PyObject_FromString __Pyx_Py%s_FromString' % c_string_func_name) code.putln('#define __Pyx_PyObject_FromStringAndSize __Pyx_Py%s_FromStringAndSize' % c_string_func_name) code.put(UtilityCode.load_as_string("TypeConversions", "TypeConversion.c")[0]) + env.use_utility_code(UtilityCode.load_cached("FormatTypeName", "ObjectHandling.c")) # These utility functions are assumed to exist and used elsewhere. PyrexTypes.c_long_type.create_to_py_utility_code(env) @@ -730,6 +804,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.put(Nodes.branch_prediction_macros) code.putln('static CYTHON_INLINE void __Pyx_pretend_to_initialize(void* ptr) { (void)ptr; }') code.putln('') + code.putln('#if !CYTHON_COMPILING_IN_LIMITED_API') code.putln('static PyObject *%s = NULL;' % env.module_cname) code.putln('static PyObject *%s;' % env.module_dict_cname) code.putln('static PyObject *%s;' % Naming.builtins_cname) @@ -739,9 +814,11 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.putln('static PyObject *%s;' % Naming.empty_unicode) if Options.pre_import is not None: code.putln('static PyObject *%s;' % Naming.preimport_cname) + code.putln('#endif') + code.putln('static int %s;' % Naming.lineno_cname) code.putln('static int %s = 0;' % Naming.clineno_cname) - code.putln('static const char * %s= %s;' % (Naming.cfilenm_cname, Naming.file_c_macro)) + code.putln('static const char * %s = %s;' % (Naming.cfilenm_cname, Naming.file_c_macro)) code.putln('static const char *%s;' % Naming.filename_cname) env.use_utility_code(UtilityCode.load_cached("FastTypeChecks", "ModuleSetupCode.c")) @@ -764,7 +841,6 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.putln("#endif") def generate_includes(self, env, cimported_modules, code, early=True, late=True): - includes = [] for inc in sorted(env.c_includes.values(), key=IncludeCode.sortkey): if inc.location == inc.EARLY: if early: @@ -785,7 +861,8 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): if isabs(file_path): file_path = basename(file_path) # never include absolute paths escaped_filename = file_path.replace("\\", "\\\\").replace('"', r'\"') - code.putln('"%s",' % escaped_filename) + escaped_filename = as_encoded_filename(escaped_filename) + code.putln('%s,' % escaped_filename.as_c_string_literal()) else: # Some C compilers don't like an empty array code.putln("0") @@ -802,7 +879,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): if not entry.in_cinclude: #print "generate_type_header_code:", entry.name, repr(entry.type) ### type = entry.type - if type.is_typedef: # Must test this first! + if type.is_typedef: # Must test this first! pass elif type.is_struct_or_union or type.is_cpp_class: self.generate_struct_union_predeclaration(entry, code) @@ -815,9 +892,9 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): if not entry.in_cinclude: #print "generate_type_header_code:", entry.name, repr(entry.type) ### type = entry.type - if type.is_typedef: # Must test this first! + if type.is_typedef: # Must test this first! self.generate_typedef(entry, code) - elif type.is_enum: + elif type.is_enum or type.is_cpp_enum: self.generate_enum_definition(entry, code) elif type.is_struct_or_union: self.generate_struct_union_definition(entry, code) @@ -894,8 +971,6 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.putln("#endif") code.putln(header) var_entries = scope.var_entries - if not var_entries: - error(entry.pos, "Empty struct or union definition not allowed outside a 'cdef extern from' block") for attr in var_entries: code.putln( "%s;" % attr.type.declaration_code(attr.cname)) @@ -956,67 +1031,69 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): arg_decls = ["void"] arg_names = [] if is_implementing: - code.putln("%s(%s) {" % (type.cname, ", ".join(arg_decls))) - if py_attrs: - code.put_ensure_gil() - for attr in py_attrs: - code.put_init_var_to_py_none(attr, nanny=False); - if constructor: - code.putln("%s(%s);" % (constructor.cname, ", ".join(arg_names))) - if py_attrs: - code.put_release_ensured_gil() - code.putln("}") + code.putln("%s(%s) {" % (type.cname, ", ".join(arg_decls))) + if py_attrs: + code.put_ensure_gil() + for attr in py_attrs: + code.put_init_var_to_py_none(attr, nanny=False) + if constructor: + code.putln("%s(%s);" % (constructor.cname, ", ".join(arg_names))) + if py_attrs: + code.put_release_ensured_gil() + code.putln("}") else: - code.putln("%s(%s);" % (type.cname, ", ".join(arg_decls))) + code.putln("%s(%s);" % (type.cname, ", ".join(arg_decls))) if destructor or py_attrs or has_virtual_methods: if has_virtual_methods: code.put("virtual ") if is_implementing: - code.putln("~%s() {" % type.cname) - if py_attrs: - code.put_ensure_gil() - if destructor: - code.putln("%s();" % destructor.cname) - if py_attrs: - for attr in py_attrs: - code.put_var_xdecref(attr, nanny=False); - code.put_release_ensured_gil() - code.putln("}") + code.putln("~%s() {" % type.cname) + if py_attrs: + code.put_ensure_gil() + if destructor: + code.putln("%s();" % destructor.cname) + if py_attrs: + for attr in py_attrs: + code.put_var_xdecref(attr, nanny=False) + code.put_release_ensured_gil() + code.putln("}") else: - code.putln("~%s();" % type.cname) + code.putln("~%s();" % type.cname) if py_attrs: # Also need copy constructor and assignment operators. if is_implementing: - code.putln("%s(const %s& __Pyx_other) {" % (type.cname, type.cname)) - code.put_ensure_gil() - for attr in scope.var_entries: - if not attr.type.is_cfunction: - code.putln("%s = __Pyx_other.%s;" % (attr.cname, attr.cname)) - code.put_var_incref(attr, nanny=False) - code.put_release_ensured_gil() - code.putln("}") - code.putln("%s& operator=(const %s& __Pyx_other) {" % (type.cname, type.cname)) - code.putln("if (this != &__Pyx_other) {") - code.put_ensure_gil() - for attr in scope.var_entries: - if not attr.type.is_cfunction: - code.put_var_xdecref(attr, nanny=False); - code.putln("%s = __Pyx_other.%s;" % (attr.cname, attr.cname)) - code.put_var_incref(attr, nanny=False) - code.put_release_ensured_gil() - code.putln("}") - code.putln("return *this;") - code.putln("}") + code.putln("%s(const %s& __Pyx_other) {" % (type.cname, type.cname)) + code.put_ensure_gil() + for attr in scope.var_entries: + if not attr.type.is_cfunction: + code.putln("%s = __Pyx_other.%s;" % (attr.cname, attr.cname)) + code.put_var_incref(attr, nanny=False) + code.put_release_ensured_gil() + code.putln("}") + code.putln("%s& operator=(const %s& __Pyx_other) {" % (type.cname, type.cname)) + code.putln("if (this != &__Pyx_other) {") + code.put_ensure_gil() + for attr in scope.var_entries: + if not attr.type.is_cfunction: + code.put_var_xdecref(attr, nanny=False) + code.putln("%s = __Pyx_other.%s;" % (attr.cname, attr.cname)) + code.put_var_incref(attr, nanny=False) + code.put_release_ensured_gil() + code.putln("}") + code.putln("return *this;") + code.putln("}") else: - code.putln("%s(const %s& __Pyx_other);" % (type.cname, type.cname)) - code.putln("%s& operator=(const %s& __Pyx_other);" % (type.cname, type.cname)) + code.putln("%s(const %s& __Pyx_other);" % (type.cname, type.cname)) + code.putln("%s& operator=(const %s& __Pyx_other);" % (type.cname, type.cname)) code.putln("};") def generate_enum_definition(self, entry, code): code.mark_pos(entry.pos) type = entry.type name = entry.cname or entry.name or "" - header, footer = self.sue_header_footer(type, "enum", name) + + kind = "enum class" if entry.type.is_cpp_enum else "enum" + header, footer = self.sue_header_footer(type, kind, name) code.putln(header) enum_values = entry.enum_values if not enum_values: @@ -1030,18 +1107,20 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): for value_entry in enum_values: if value_entry.value_node is None: - value_code = value_entry.cname + value_code = value_entry.cname.split("::")[-1] else: value_code = ("%s = %s" % ( - value_entry.cname, + value_entry.cname.split("::")[-1], value_entry.value_node.result())) if value_entry is not last_entry: value_code += "," code.putln(value_code) code.putln(footer) - if entry.type.typedef_flag: - # Not pre-declared. - code.putln("typedef enum %s %s;" % (name, name)) + + if entry.type.is_enum: + if entry.type.typedef_flag: + # Not pre-declared. + code.putln("typedef enum %s %s;" % (name, name)) def generate_typeobj_predeclaration(self, entry, code): code.putln("") @@ -1120,7 +1199,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): # Generate object struct definition for an # extension type. if not type.scope: - return # Forward declared but never defined + return # Forward declared but never defined header, footer = \ self.sue_header_footer(type, "struct", type.objstruct_cname) code.putln(header) @@ -1155,11 +1234,40 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): # Only for exposing public typedef name. code.putln("typedef struct %s %s;" % (type.objstruct_cname, type.objtypedef_cname)) - def generate_c_class_declarations(self, env, code, definition): + def generate_c_class_declarations(self, env, code, definition, globalstate): + module_state = globalstate['module_state'] + module_state_defines = globalstate['module_state_defines'] + module_state_clear = globalstate['module_state_clear'] + module_state_traverse = globalstate['module_state_traverse'] + code.putln("#if !CYTHON_COMPILING_IN_LIMITED_API") for entry in env.c_class_entries: if definition or entry.defined_in_pxd: code.putln("static PyTypeObject *%s = 0;" % ( entry.type.typeptr_cname)) + module_state.putln("PyTypeObject *%s;" % entry.type.typeptr_cname) + module_state_defines.putln("#define %s %s->%s" % ( + entry.type.typeptr_cname, + Naming.modulestateglobal_cname, + entry.type.typeptr_cname)) + module_state_clear.putln( + "Py_CLEAR(clear_module_state->%s);" % + entry.type.typeptr_cname) + module_state_traverse.putln( + "Py_VISIT(traverse_module_state->%s);" % + entry.type.typeptr_cname) + if entry.type.typeobj_cname is not None: + module_state.putln("PyObject *%s;" % entry.type.typeobj_cname) + module_state_defines.putln("#define %s %s->%s" % ( + entry.type.typeobj_cname, + Naming.modulestateglobal_cname, + entry.type.typeobj_cname)) + module_state_clear.putln( + "Py_CLEAR(clear_module_state->%s);" % ( + entry.type.typeobj_cname)) + module_state_traverse.putln( + "Py_VISIT(traverse_module_state->%s);" % ( + entry.type.typeobj_cname)) + code.putln("#endif") def generate_cvariable_declarations(self, env, code, definition): if env.is_cython_builtin: @@ -1229,11 +1337,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): if entry.visibility != 'extern': type = entry.type scope = type.scope - if scope: # could be None if there was an error - if not scope.directives['c_api_binop_methods']: - error(self.pos, - "The 'c_api_binop_methods' directive is only supported for forward compatibility" - " and must be True.") + if scope: # could be None if there was an error self.generate_exttype_vtable(scope, code) self.generate_new_function(scope, code, entry) self.generate_dealloc_function(scope, code) @@ -1266,10 +1370,17 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): self.generate_dict_getter_function(scope, code) if scope.defines_any_special(TypeSlots.richcmp_special_methods): self.generate_richcmp_function(scope, code) + for slot in TypeSlots.PyNumberMethods: + if slot.is_binop and scope.defines_any_special(slot.user_methods): + self.generate_binop_function(scope, slot, code, entry.pos) self.generate_property_accessors(scope, code) self.generate_method_table(scope, code) self.generate_getset_table(scope, code) + code.putln("#if CYTHON_COMPILING_IN_LIMITED_API") + self.generate_typeobj_spec(entry, code) + code.putln("#else") self.generate_typeobj_definition(full_module_name, entry, code) + code.putln("#endif") def generate_exttype_vtable(self, scope, code): # Generate the definition of an extension type's vtable. @@ -1287,8 +1398,11 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): type.empty_declaration_code())) def generate_new_function(self, scope, code, cclass_entry): - tp_slot = TypeSlots.ConstructorSlot("tp_new", '__new__') + tp_slot = TypeSlots.ConstructorSlot("tp_new", "__cinit__") slot_func = scope.mangle_internal("tp_new") + if tp_slot.slot_code(scope) != slot_func: + return # never used + type = scope.parent_type base_type = type.base_type @@ -1298,12 +1412,14 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): if scope.is_internal: # internal classes (should) never need None inits, normal zeroing will do py_attrs = [] - cpp_class_attrs = [entry for entry in scope.var_entries - if entry.type.is_cpp_class] + cpp_constructable_attrs = [entry for entry in scope.var_entries + if entry.type.needs_cpp_construction] - new_func_entry = scope.lookup_here("__new__") - if base_type or (new_func_entry and new_func_entry.is_special - and not new_func_entry.trivial_signature): + cinit_func_entry = scope.lookup_here("__cinit__") + if cinit_func_entry and not cinit_func_entry.is_special: + cinit_func_entry = None + + if base_type or (cinit_func_entry and not cinit_func_entry.trivial_signature): unused_marker = '' else: unused_marker = 'CYTHON_UNUSED ' @@ -1331,23 +1447,33 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): need_self_cast = (type.vtabslot_cname or (py_buffers or memoryview_slices or py_attrs) or - cpp_class_attrs) + cpp_constructable_attrs) if need_self_cast: code.putln("%s;" % scope.parent_type.declaration_code("p")) if base_type: + code.putln("#if CYTHON_COMPILING_IN_LIMITED_API") + code.putln("newfunc new_func = (newfunc)PyType_GetSlot(%s, Py_tp_new);" % + base_type.typeptr_cname) + code.putln("PyObject *o = new_func(t, a, k);") + code.putln("#else") tp_new = TypeSlots.get_base_slot_function(scope, tp_slot) if tp_new is None: tp_new = "%s->tp_new" % base_type.typeptr_cname code.putln("PyObject *o = %s(t, a, k);" % tp_new) + code.putln("#endif") else: code.putln("PyObject *o;") + code.putln("#if CYTHON_COMPILING_IN_LIMITED_API") + code.putln("allocfunc alloc_func = (allocfunc)PyType_GetSlot(t, Py_tp_alloc);") + code.putln("o = alloc_func(t, 0);") + code.putln("#else") if freelist_size: code.globalstate.use_utility_code( UtilityCode.load_cached("IncludeStringH", "StringTools.c")) if is_final_type: type_safety_check = '' else: - type_safety_check = ' & ((t->tp_flags & (Py_TPFLAGS_IS_ABSTRACT | Py_TPFLAGS_HEAPTYPE)) == 0)' + type_safety_check = ' & (!__Pyx_PyType_HasFeature(t, (Py_TPFLAGS_IS_ABSTRACT | Py_TPFLAGS_HEAPTYPE)))' obj_struct = type.declaration_code("", deref=True) code.putln( "if (CYTHON_COMPILING_IN_CPYTHON && likely((%s > 0) & (t->tp_basicsize == sizeof(%s))%s)) {" % ( @@ -1360,7 +1486,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.putln("PyObject_GC_Track(o);") code.putln("} else {") if not is_final_type: - code.putln("if (likely((t->tp_flags & Py_TPFLAGS_IS_ABSTRACT) == 0)) {") + code.putln("if (likely(!__Pyx_PyType_HasFeature(t, Py_TPFLAGS_IS_ABSTRACT))) {") code.putln("o = (*t->tp_alloc)(t, 0);") if not is_final_type: code.putln("} else {") @@ -1369,6 +1495,8 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.putln("if (unlikely(!o)) return 0;") if freelist_size and not base_type: code.putln('}') + if not base_type: + code.putln("#endif") if need_self_cast: code.putln("p = %s;" % type.cast_code("o")) #if need_self_cast: @@ -1389,7 +1517,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): type.vtabslot_cname, struct_type_cast, type.vtabptr_cname)) - for entry in cpp_class_attrs: + for entry in cpp_constructable_attrs: code.putln("new((void*)&(p->%s)) %s();" % ( entry.cname, entry.type.empty_declaration_code())) @@ -1411,14 +1539,14 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): if cclass_entry.cname == '__pyx_memoryviewslice': code.putln("p->from_slice.memview = NULL;") - if new_func_entry and new_func_entry.is_special: - if new_func_entry.trivial_signature: + if cinit_func_entry: + if cinit_func_entry.trivial_signature: cinit_args = "o, %s, NULL" % Naming.empty_tuple else: cinit_args = "o, a, k" needs_error_cleanup = True code.putln("if (unlikely(%s(%s) < 0)) goto bad;" % ( - new_func_entry.func_cname, cinit_args)) + cinit_func_entry.func_cname, cinit_args)) code.putln( "return o;") @@ -1438,11 +1566,17 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): slot_func_cname = scope.mangle_internal("tp_dealloc") code.putln("") + cdealloc_func_entry = scope.lookup_here("__dealloc__") + if cdealloc_func_entry and not cdealloc_func_entry.is_special: + cdealloc_func_entry = None + if cdealloc_func_entry is None: + code.putln("#if !CYTHON_COMPILING_IN_LIMITED_API") code.putln( "static void %s(PyObject *o) {" % slot_func_cname) is_final_type = scope.parent_type.is_final_type needs_gc = scope.needs_gc() + needs_trashcan = scope.needs_trashcan() weakref_slot = scope.lookup_here("__weakref__") if not scope.is_closure_class_scope else None if weakref_slot not in scope.var_entries: @@ -1453,10 +1587,10 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): dict_slot = None _, (py_attrs, _, memoryview_slices) = scope.get_refcounted_entries() - cpp_class_attrs = [entry for entry in scope.var_entries - if entry.type.is_cpp_class] + cpp_destructable_attrs = [entry for entry in scope.var_entries + if entry.type.needs_cpp_construction] - if py_attrs or cpp_class_attrs or memoryview_slices or weakref_slot or dict_slot: + if py_attrs or cpp_destructable_attrs or memoryview_slices or weakref_slot or dict_slot: self.generate_self_cast(scope, code) if not is_final_type: @@ -1468,7 +1602,8 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): finalised_check = ( '(!PyType_IS_GC(Py_TYPE(o)) || !_PyGC_FINALIZED(o))') code.putln( - "if (unlikely(PyType_HasFeature(Py_TYPE(o), Py_TPFLAGS_HAVE_FINALIZE)" + "if (unlikely(" + "(PY_VERSION_HEX >= 0x03080000 || __Pyx_PyType_HasFeature(Py_TYPE(o), Py_TPFLAGS_HAVE_FINALIZE))" " && Py_TYPE(o)->tp_finalize) && %s) {" % finalised_check) # if instance was resurrected by finaliser, return code.putln("if (PyObject_CallFinalizerFromDealloc(o)) return;") @@ -1481,25 +1616,30 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): # running this destructor. code.putln("PyObject_GC_UnTrack(o);") - # call the user's __dealloc__ - self.generate_usr_dealloc_call(scope, code) + if needs_trashcan: + code.globalstate.use_utility_code( + UtilityCode.load_cached("PyTrashcan", "ExtensionTypes.c")) + code.putln("__Pyx_TRASHCAN_BEGIN(o, %s)" % slot_func_cname) if weakref_slot: + # We must clean the weakreferences before calling the user's __dealloc__ + # because if the __dealloc__ releases the GIL, a weakref can be + # dereferenced accessing the object in an inconsistent state or + # resurrecting it. code.putln("if (p->__weakref__) PyObject_ClearWeakRefs(o);") + # call the user's __dealloc__ + self.generate_usr_dealloc_call(scope, code) + if dict_slot: code.putln("if (p->__dict__) PyDict_Clear(p->__dict__);") - for entry in cpp_class_attrs: + for entry in cpp_destructable_attrs: code.putln("__Pyx_call_destructor(p->%s);" % entry.cname) - for entry in py_attrs: + for entry in (py_attrs + memoryview_slices): code.put_xdecref_clear("p->%s" % entry.cname, entry.type, nanny=False, - clear_before_decref=True) - - for entry in memoryview_slices: - code.put_xdecref_memoryviewslice("p->%s" % entry.cname, - have_gil=True) + clear_before_decref=True, have_gil=True) if base_type: if needs_gc: @@ -1539,7 +1679,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): type_safety_check = '' else: type_safety_check = ( - ' & ((Py_TYPE(o)->tp_flags & (Py_TPFLAGS_IS_ABSTRACT | Py_TPFLAGS_HEAPTYPE)) == 0)') + ' & (!__Pyx_PyType_HasFeature(Py_TYPE(o), (Py_TPFLAGS_IS_ABSTRACT | Py_TPFLAGS_HEAPTYPE)))') type = scope.parent_type code.putln( @@ -1554,12 +1694,18 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.putln("(*Py_TYPE(o)->tp_free)(o);") if freelist_size: code.putln("}") + + if needs_trashcan: + code.putln("__Pyx_TRASHCAN_END") + code.putln( "}") + if cdealloc_func_entry is None: + code.putln("#endif") def generate_usr_dealloc_call(self, scope, code): entry = scope.lookup_here("__dealloc__") - if not entry: + if not entry or not entry.is_special: return code.putln("{") @@ -1639,7 +1785,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): slot_func = scope.mangle_internal("tp_clear") base_type = scope.parent_type.base_type if tp_slot.slot_code(scope) != slot_func: - return # never used + return # never used have_entries, (py_attrs, py_buffers, memoryview_slices) = ( scope.get_refcounted_entries(include_gc_simple=False)) @@ -1697,7 +1843,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.putln("Py_CLEAR(p->%s.obj);" % entry.cname) if cclass_entry.cname == '__pyx_memoryviewslice': - code.putln("__PYX_XDEC_MEMVIEW(&p->from_slice, 1);") + code.putln("__PYX_XCLEAR_MEMVIEW(&p->from_slice, 1);") code.putln("return 0;") code.putln("}") @@ -1738,12 +1884,18 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): if set_entry: code.putln("return %s(o, i, v);" % set_entry.func_cname) else: + code.putln( + "__Pyx_TypeName o_type_name;") self.generate_guarded_basetype_call( base_type, "tp_as_mapping", "mp_ass_subscript", "o, i, v", code) code.putln( + "o_type_name = __Pyx_PyType_GetName(Py_TYPE(o));") + code.putln( "PyErr_Format(PyExc_NotImplementedError,") code.putln( - ' "Subscript assignment not supported by %.200s", Py_TYPE(o)->tp_name);') + ' "Subscript assignment not supported by " __Pyx_FMT_TYPENAME, o_type_name);') + code.putln( + "__Pyx_DECREF_TypeName(o_type_name);") code.putln( "return -1;") code.putln( @@ -1755,12 +1907,18 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): "return %s(o, i);" % ( del_entry.func_cname)) else: + code.putln( + "__Pyx_TypeName o_type_name;") self.generate_guarded_basetype_call( base_type, "tp_as_mapping", "mp_ass_subscript", "o, i, v", code) code.putln( + "o_type_name = __Pyx_PyType_GetName(Py_TYPE(o));") + code.putln( "PyErr_Format(PyExc_NotImplementedError,") code.putln( - ' "Subscript deletion not supported by %.200s", Py_TYPE(o)->tp_name);') + ' "Subscript deletion not supported by " __Pyx_FMT_TYPENAME, o_type_name);') + code.putln( + "__Pyx_DECREF_TypeName(o_type_name);") code.putln( "return -1;") code.putln( @@ -1805,12 +1963,18 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): "return %s(o, i, j, v);" % ( set_entry.func_cname)) else: + code.putln( + "__Pyx_TypeName o_type_name;") self.generate_guarded_basetype_call( base_type, "tp_as_sequence", "sq_ass_slice", "o, i, j, v", code) code.putln( + "o_type_name = __Pyx_PyType_GetName(Py_TYPE(o));") + code.putln( "PyErr_Format(PyExc_NotImplementedError,") code.putln( - ' "2-element slice assignment not supported by %.200s", Py_TYPE(o)->tp_name);') + ' "2-element slice assignment not supported by " __Pyx_FMT_TYPENAME, o_type_name);') + code.putln( + "__Pyx_DECREF_TypeName(o_type_name);") code.putln( "return -1;") code.putln( @@ -1822,12 +1986,18 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): "return %s(o, i, j);" % ( del_entry.func_cname)) else: + code.putln( + "__Pyx_TypeName o_type_name;") self.generate_guarded_basetype_call( base_type, "tp_as_sequence", "sq_ass_slice", "o, i, j, v", code) code.putln( + "o_type_name = __Pyx_PyType_GetName(Py_TYPE(o));") + code.putln( "PyErr_Format(PyExc_NotImplementedError,") code.putln( - ' "2-element slice deletion not supported by %.200s", Py_TYPE(o)->tp_name);') + ' "2-element slice deletion not supported by " __Pyx_FMT_TYPENAME, o_type_name);') + code.putln( + "__Pyx_DECREF_TypeName(o_type_name);") code.putln( "return -1;") code.putln( @@ -1905,6 +2075,70 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.putln("}") # switch code.putln("}") + def generate_binop_function(self, scope, slot, code, pos): + func_name = scope.mangle_internal(slot.slot_name) + if scope.directives['c_api_binop_methods']: + code.putln('#define %s %s' % (func_name, slot.left_slot.slot_code(scope))) + return + + code.putln() + preprocessor_guard = slot.preprocessor_guard_code() + if preprocessor_guard: + code.putln(preprocessor_guard) + + if slot.left_slot.signature == TypeSlots.binaryfunc: + slot_type = 'binaryfunc' + extra_arg = extra_arg_decl = '' + elif slot.left_slot.signature == TypeSlots.ternaryfunc: + slot_type = 'ternaryfunc' + extra_arg = ', extra_arg' + extra_arg_decl = ', PyObject* extra_arg' + else: + error(pos, "Unexpected type slot signature: %s" % slot) + return + + def get_slot_method_cname(method_name): + entry = scope.lookup(method_name) + return entry.func_cname if entry and entry.is_special else None + + def call_slot_method(method_name, reverse): + func_cname = get_slot_method_cname(method_name) + if func_cname: + return "%s(%s%s)" % ( + func_cname, + "right, left" if reverse else "left, right", + extra_arg) + else: + return '%s_maybe_call_slot(%s, left, right %s)' % ( + func_name, + 'Py_TYPE(right)->tp_base' if reverse else 'Py_TYPE(left)->tp_base', + extra_arg) + + if get_slot_method_cname(slot.left_slot.method_name) and not get_slot_method_cname(slot.right_slot.method_name): + warning(pos, "Extension type implements %s() but not %s(). " + "The behaviour has changed from previous Cython versions to match Python semantics. " + "You can implement both special methods in a backwards compatible way." % ( + slot.left_slot.method_name, + slot.right_slot.method_name, + )) + + code.putln( + TempitaUtilityCode.load_as_string( + "BinopSlot", "ExtensionTypes.c", + context={ + "func_name": func_name, + "slot_name": slot.slot_name, + "overloads_left": int(bool(get_slot_method_cname(slot.left_slot.method_name))), + "call_left": call_slot_method(slot.left_slot.method_name, reverse=False), + "call_right": call_slot_method(slot.right_slot.method_name, reverse=True), + "type_cname": scope.parent_type.typeptr_cname, + "slot_type": slot_type, + "extra_arg": extra_arg, + "extra_arg_decl": extra_arg_decl, + })[1]) + if preprocessor_guard: + code.putln("#endif") + def generate_getattro_function(self, scope, code): # First try to get the attribute using __getattribute__, if defined, or # PyObject_GenericGetAttr. @@ -2139,6 +2373,28 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.putln( "}") + def generate_typeobj_spec(self, entry, code): + ext_type = entry.type + scope = ext_type.scope + code.putln("static PyType_Slot %s_slots[] = {" % ext_type.typeobj_cname) + for slot in TypeSlots.slot_table: + slot.generate_spec(scope, code) + code.putln("{0, 0},") + code.putln("};") + + if ext_type.typedef_flag: + objstruct = ext_type.objstruct_cname + else: + objstruct = "struct %s" % ext_type.objstruct_cname + classname = scope.class_name.as_c_string_literal() + code.putln("static PyType_Spec %s_spec = {" % ext_type.typeobj_cname) + code.putln('"%s.%s",' % (self.full_module_name, classname.replace('"', ''))) + code.putln("sizeof(%s)," % objstruct) + code.putln("0,") + code.putln("%s," % TypeSlots.get_slot_by_name("tp_flags").slot_code(scope)) + code.putln("%s_slots," % ext_type.typeobj_cname) + code.putln("};") + def generate_typeobj_definition(self, modname, entry, code): type = entry.type scope = type.scope @@ -2153,9 +2409,11 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.putln(header % type.typeobj_cname) code.putln( "PyVarObject_HEAD_INIT(0, 0)") + classname = scope.class_name.as_c_string_literal() code.putln( - '"%s.%s", /*tp_name*/' % ( - self.full_module_name, scope.class_name)) + '"%s."%s, /*tp_name*/' % ( + self.full_module_name, + classname)) if type.typedef_flag: objstruct = type.objstruct_cname else: @@ -2218,12 +2476,12 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): if doc: if doc.is_unicode: doc = doc.as_utf8_string() - doc_code = doc.as_c_string_literal() + doc_code = "PyDoc_STR(%s)" % doc.as_c_string_literal() else: doc_code = "0" code.putln( - '{(char *)"%s", %s, %s, (char *)%s, 0},' % ( - entry.name, + '{(char *)%s, %s, %s, (char *)%s, 0},' % ( + entry.name.as_c_string_literal(), entry.getter_cname or "0", entry.setter_cname or "0", doc_code)) @@ -2299,7 +2557,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): if code.label_used(code.error_label): code.put_label(code.error_label) # This helps locate the offending name. - code.put_add_traceback(self.full_module_name) + code.put_add_traceback(EncodedString(self.full_module_name)) code.error_label = old_error_label code.putln("bad:") code.putln("return -1;") @@ -2308,20 +2566,197 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.putln(UtilityCode.load_as_string("ImportStar", "ImportExport.c")[1]) code.exit_cfunc_scope() # done with labels + def generate_module_state_start(self, env, code): + # TODO: Reactor LIMITED_API struct decl closer to the static decl + code.putln("#if CYTHON_COMPILING_IN_LIMITED_API") + code.putln('typedef struct {') + code.putln('PyObject *%s;' % env.module_dict_cname) + code.putln('PyObject *%s;' % Naming.builtins_cname) + code.putln('PyObject *%s;' % Naming.cython_runtime_cname) + code.putln('PyObject *%s;' % Naming.empty_tuple) + code.putln('PyObject *%s;' % Naming.empty_bytes) + code.putln('PyObject *%s;' % Naming.empty_unicode) + if Options.pre_import is not None: + code.putln('PyObject *%s;' % Naming.preimport_cname) + code.putln('#ifdef __Pyx_CyFunction_USED') + code.putln('PyTypeObject *%s;' % Naming.cyfunction_type_cname) + code.putln('#endif') + code.putln('#ifdef __Pyx_FusedFunction_USED') + code.putln('PyTypeObject *%s;' % Naming.fusedfunction_type_cname) + code.putln('#endif') + + def generate_module_state_end(self, env, modules, globalstate): + module_state = globalstate['module_state'] + module_state_defines = globalstate['module_state_defines'] + module_state_clear = globalstate['module_state_clear'] + module_state_traverse = globalstate['module_state_traverse'] + module_state.putln('} %s;' % Naming.modulestate_cname) + module_state.putln('') + module_state.putln('#ifdef __cplusplus') + module_state.putln('namespace {') + module_state.putln('extern struct PyModuleDef %s;' % Naming.pymoduledef_cname) + module_state.putln('} /* anonymous namespace */') + module_state.putln('#else') + module_state.putln('static struct PyModuleDef %s;' % Naming.pymoduledef_cname) + module_state.putln('#endif') + module_state.putln('') + module_state.putln('#define %s(o) ((%s *)__Pyx_PyModule_GetState(o))' % ( + Naming.modulestate_cname, + Naming.modulestate_cname)) + module_state.putln('') + module_state.putln('#define %s (%s(PyState_FindModule(&%s)))' % ( + Naming.modulestateglobal_cname, + Naming.modulestate_cname, + Naming.pymoduledef_cname)) + module_state.putln('') + module_state.putln('#define %s (PyState_FindModule(&%s))' % ( + env.module_cname, + Naming.pymoduledef_cname)) + module_state.putln("#endif") + module_state_defines.putln("#endif") + module_state_clear.putln("return 0;") + module_state_clear.putln("}") + module_state_clear.putln("#endif") + module_state_traverse.putln("return 0;") + module_state_traverse.putln("}") + module_state_traverse.putln("#endif") + + def generate_module_state_defines(self, env, code): + code.putln("#if CYTHON_COMPILING_IN_LIMITED_API") + code.putln('#define %s %s->%s' % ( + env.module_dict_cname, + Naming.modulestateglobal_cname, + env.module_dict_cname)) + code.putln('#define %s %s->%s' % ( + Naming.builtins_cname, + Naming.modulestateglobal_cname, + Naming.builtins_cname)) + code.putln('#define %s %s->%s' % ( + Naming.cython_runtime_cname, + Naming.modulestateglobal_cname, + Naming.cython_runtime_cname)) + code.putln('#define %s %s->%s' % ( + Naming.empty_tuple, + Naming.modulestateglobal_cname, + Naming.empty_tuple)) + code.putln('#define %s %s->%s' % ( + Naming.empty_bytes, + Naming.modulestateglobal_cname, + Naming.empty_bytes)) + code.putln('#define %s %s->%s' % ( + Naming.empty_unicode, + Naming.modulestateglobal_cname, + Naming.empty_unicode)) + if Options.pre_import is not None: + code.putln('#define %s %s->%s' % ( + Naming.preimport_cname, + Naming.modulestateglobal_cname, + Naming.preimport_cname)) + code.putln('#ifdef __Pyx_CyFunction_USED') + code.putln('#define %s %s->%s' % ( + Naming.cyfunction_type_cname, + Naming.modulestateglobal_cname, + Naming.cyfunction_type_cname)) + code.putln('#endif') + code.putln('#ifdef __Pyx_FusedFunction_USED') + code.putln('#define %s %s->%s' % + (Naming.fusedfunction_type_cname, + Naming.modulestateglobal_cname, + Naming.fusedfunction_type_cname)) + code.putln('#endif') + + def generate_module_state_clear(self, env, code): + code.putln("#if CYTHON_COMPILING_IN_LIMITED_API") + code.putln("static int %s_clear(PyObject *m) {" % Naming.module_cname) + code.putln("%s *clear_module_state = %s(m);" % ( + Naming.modulestate_cname, + Naming.modulestate_cname)) + code.putln("if (!clear_module_state) return 0;") + code.putln('Py_CLEAR(clear_module_state->%s);' % + env.module_dict_cname) + code.putln('Py_CLEAR(clear_module_state->%s);' % + Naming.builtins_cname) + code.putln('Py_CLEAR(clear_module_state->%s);' % + Naming.cython_runtime_cname) + code.putln('Py_CLEAR(clear_module_state->%s);' % + Naming.empty_tuple) + code.putln('Py_CLEAR(clear_module_state->%s);' % + Naming.empty_bytes) + code.putln('Py_CLEAR(clear_module_state->%s);' % + Naming.empty_unicode) + code.putln('#ifdef __Pyx_CyFunction_USED') + code.putln('Py_CLEAR(clear_module_state->%s);' % + Naming.cyfunction_type_cname) + code.putln('#endif') + code.putln('#ifdef __Pyx_FusedFunction_USED') + code.putln('Py_CLEAR(clear_module_state->%s);' % + Naming.fusedfunction_type_cname) + code.putln('#endif') + + def generate_module_state_traverse(self, env, code): + code.putln("#if CYTHON_COMPILING_IN_LIMITED_API") + code.putln("static int %s_traverse(PyObject *m, visitproc visit, void *arg) {" % Naming.module_cname) + code.putln("%s *traverse_module_state = %s(m);" % ( + Naming.modulestate_cname, + Naming.modulestate_cname)) + code.putln("if (!traverse_module_state) return 0;") + code.putln('Py_VISIT(traverse_module_state->%s);' % + env.module_dict_cname) + code.putln('Py_VISIT(traverse_module_state->%s);' % + Naming.builtins_cname) + code.putln('Py_VISIT(traverse_module_state->%s);' % + Naming.cython_runtime_cname) + code.putln('Py_VISIT(traverse_module_state->%s);' % + Naming.empty_tuple) + code.putln('Py_VISIT(traverse_module_state->%s);' % + Naming.empty_bytes) + code.putln('Py_VISIT(traverse_module_state->%s);' % + Naming.empty_unicode) + code.putln('#ifdef __Pyx_CyFunction_USED') + code.putln('Py_VISIT(traverse_module_state->%s);' % + Naming.cyfunction_type_cname) + code.putln('#endif') + code.putln('#ifdef __Pyx_FusedFunction_USED') + code.putln('Py_VISIT(traverse_module_state->%s);' % + Naming.fusedfunction_type_cname) + code.putln('#endif') + def generate_module_init_func(self, imported_modules, env, code): subfunction = self.mod_init_subfunction(self.pos, self.scope, code) + self.generate_pymoduledef_struct(env, code) + code.enter_cfunc_scope(self.scope) code.putln("") code.putln(UtilityCode.load_as_string("PyModInitFuncType", "ModuleSetupCode.c")[0]) - header2 = "__Pyx_PyMODINIT_FUNC init%s(void)" % env.module_name + if env.module_name.isascii(): + py2_mod_name = env.module_name + fail_compilation_in_py2 = False + else: + fail_compilation_in_py2 = True + # at this point py2_mod_name is largely a placeholder and the value doesn't matter + py2_mod_name = env.module_name.encode("ascii", errors="ignore").decode("utf8") + + header2 = "__Pyx_PyMODINIT_FUNC init%s(void)" % py2_mod_name header3 = "__Pyx_PyMODINIT_FUNC %s(void)" % self.mod_init_func_cname('PyInit', env) + header3 = EncodedString(header3) code.putln("#if PY_MAJOR_VERSION < 3") # Optimise for small code size as the module init function is only executed once. code.putln("%s CYTHON_SMALL_CODE; /*proto*/" % header2) + if fail_compilation_in_py2: + code.putln('#error "Unicode module names are not supported in Python 2";') + if self.scope.is_package: + code.putln("#if !defined(CYTHON_NO_PYINIT_EXPORT) && (defined(_WIN32) || defined(WIN32) || defined(MS_WINDOWS))") + code.putln("__Pyx_PyMODINIT_FUNC init__init__(void) { init%s(); }" % py2_mod_name) + code.putln("#endif") code.putln(header2) code.putln("#else") code.putln("%s CYTHON_SMALL_CODE; /*proto*/" % header3) + if self.scope.is_package: + code.putln("#if !defined(CYTHON_NO_PYINIT_EXPORT) && (defined(_WIN32) || defined(WIN32) || defined(MS_WINDOWS))") + code.putln("__Pyx_PyMODINIT_FUNC PyInit___init__(void) { return %s(); }" % ( + self.mod_init_func_cname('PyInit', env))) + code.putln("#endif") code.putln(header3) # CPython 3.5+ supports multi-phase module initialisation (gives access to __spec__, __file__, etc.) @@ -2336,7 +2771,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.putln("") # main module init code lives in Py_mod_exec function, not in PyInit function code.putln("static CYTHON_SMALL_CODE int %s(PyObject *%s)" % ( - self.mod_init_func_cname(Naming.pymodule_exec_func_cname, env), + self.module_init_func_cname(), Naming.pymodinit_module_arg)) code.putln("#endif") # PEP489 @@ -2350,6 +2785,8 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): profile = code.globalstate.directives['profile'] linetrace = code.globalstate.directives['linetrace'] if profile or linetrace: + if linetrace: + code.use_fast_gil_utility_code() code.globalstate.use_utility_code(UtilityCode.load_cached("Profile", "Profile.c")) code.put_declare_refcount_context() @@ -2364,7 +2801,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): )) code.putln('PyErr_SetString(PyExc_RuntimeError,' ' "Module \'%s\' has already been imported. Re-initialisation is not supported.");' % - env.module_name) + env.module_name.as_c_string_literal()[1:-1]) code.putln("return -1;") code.putln("}") code.putln("#elif PY_MAJOR_VERSION >= 3") @@ -2375,6 +2812,9 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): )) code.putln("#endif") + code.putln("/*--- Module creation code ---*/") + self.generate_module_creation_code(env, code) + if profile or linetrace: tempdecl_code.put_trace_declarations() code.put_trace_frame_init() @@ -2406,14 +2846,10 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.put_error_if_neg(self.pos, "_import_array()") code.putln("/*--- Threads initialization code ---*/") - code.putln("#if defined(__PYX_FORCE_INIT_THREADS) && __PYX_FORCE_INIT_THREADS") - code.putln("#ifdef WITH_THREAD /* Python build with threading support? */") + code.putln("#if defined(WITH_THREAD) && PY_VERSION_HEX < 0x030700F0 " + "&& defined(__PYX_FORCE_INIT_THREADS) && __PYX_FORCE_INIT_THREADS") code.putln("PyEval_InitThreads();") code.putln("#endif") - code.putln("#endif") - - code.putln("/*--- Module creation code ---*/") - self.generate_module_creation_code(env, code) code.putln("/*--- Initialize various global constants etc. ---*/") code.put_error_if_neg(self.pos, "__Pyx_InitGlobals()") @@ -2423,7 +2859,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.put_error_if_neg(self.pos, "__Pyx_init_sys_getdefaultencoding_params()") code.putln("#endif") - code.putln("if (%s%s) {" % (Naming.module_is_main, self.full_module_name.replace('.', '__'))) + code.putln("if (%s) {" % self.is_main_module_flag_cname()) code.put_error_if_neg(self.pos, 'PyObject_SetAttr(%s, %s, %s)' % ( env.module_cname, code.intern_identifier(EncodedString("__name__")), @@ -2500,7 +2936,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.put_xdecref(cname, type) code.putln('if (%s) {' % env.module_cname) code.putln('if (%s) {' % env.module_dict_cname) - code.put_add_traceback("init %s" % env.qualified_name) + code.put_add_traceback(EncodedString("init %s" % env.qualified_name)) code.globalstate.use_utility_code(Nodes.traceback_utility_code) # Module reference and module dict are in global variables which might still be needed # for cleanup, atexit code, etc., so leaking is better than crashing. @@ -2508,9 +2944,12 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): # user code in atexit or other global registries. ##code.put_decref_clear(env.module_dict_cname, py_object_type, nanny=False) code.putln('}') + code.putln("#if !CYTHON_COMPILING_IN_LIMITED_API") code.put_decref_clear(env.module_cname, py_object_type, nanny=False, clear_before_decref=True) + code.putln("#endif") code.putln('} else if (!PyErr_Occurred()) {') - code.putln('PyErr_SetString(PyExc_ImportError, "init %s");' % env.qualified_name) + code.putln('PyErr_SetString(PyExc_ImportError, "init %s");' % + env.qualified_name.as_c_string_literal()[1:-1]) code.putln('}') code.put_label(code.return_label) @@ -2560,7 +2999,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.putln("static int %s(void) {" % self.cfunc_name) code.put_declare_refcount_context() self.tempdecl_code = code.insertion_point() - code.put_setup_refcount_context(self.cfunc_name) + code.put_setup_refcount_context(EncodedString(self.cfunc_name)) # Leave a grepable marker that makes it easy to find the generator source. code.putln("/*--- %s ---*/" % self.description) return code @@ -2617,7 +3056,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): EncodedString(decode_filename( os.path.dirname(module_path)))).cname, code.error_goto_if_null(temp, self.pos))) - code.put_gotref(temp) + code.put_gotref(temp, py_object_type) code.putln( 'if (PyObject_SetAttrString(%s, "__path__", %s) < 0) %s;' % ( env.module_cname, temp, code.error_goto(self.pos))) @@ -2647,14 +3086,15 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): # CPython may not have put us into sys.modules yet, but relative imports and reimports require it fq_module_name = self.full_module_name if fq_module_name.endswith('.__init__'): - fq_module_name = fq_module_name[:-len('.__init__')] + fq_module_name = EncodedString(fq_module_name[:-len('.__init__')]) + fq_module_name_cstring = fq_module_name.as_c_string_literal() code.putln("#if PY_MAJOR_VERSION >= 3") code.putln("{") code.putln("PyObject *modules = PyImport_GetModuleDict(); %s" % code.error_goto_if_null("modules", self.pos)) - code.putln('if (!PyDict_GetItemString(modules, "%s")) {' % fq_module_name) - code.putln(code.error_goto_if_neg('PyDict_SetItemString(modules, "%s", %s)' % ( - fq_module_name, env.module_cname), self.pos)) + code.putln('if (!PyDict_GetItemString(modules, %s)) {' % fq_module_name_cstring) + code.putln(code.error_goto_if_neg('PyDict_SetItemString(modules, %s, %s)' % ( + fq_module_name_cstring, env.module_cname), self.pos)) code.putln("}") code.putln("}") code.putln("#endif") @@ -2727,11 +3167,12 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): if Options.pre_import is not None: code.put_decref_clear(Naming.preimport_cname, py_object_type, nanny=False, clear_before_decref=True) - for cname in [env.module_dict_cname, Naming.cython_runtime_cname, Naming.builtins_cname]: + for cname in [Naming.cython_runtime_cname, Naming.builtins_cname]: code.put_decref_clear(cname, py_object_type, nanny=False, clear_before_decref=True) + code.put_decref_clear(env.module_dict_cname, py_object_type, nanny=False, clear_before_decref=True) def generate_main_method(self, env, code): - module_is_main = "%s%s" % (Naming.module_is_main, self.full_module_name.replace('.', '__')) + module_is_main = self.is_main_module_flag_cname() if Options.embed == "main": wmain = "wmain" else: @@ -2744,8 +3185,25 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): main_method=Options.embed, wmain_method=wmain)) + def punycode_module_name(self, prefix, name): + # adapted from PEP483 + try: + name = '_' + name.encode('ascii').decode('ascii') + except UnicodeEncodeError: + name = 'U_' + name.encode('punycode').replace(b'-', b'_').decode('ascii') + return "%s%s" % (prefix, name) + def mod_init_func_cname(self, prefix, env): - return '%s_%s' % (prefix, env.module_name) + # from PEP483 + return self.punycode_module_name(prefix, env.module_name) + + # Returns the name of the C-function that corresponds to the module initialisation. + # (module initialisation == the cython code outside of functions) + # Note that this should never be the name of a wrapper and always the name of the + # function containing the actual code. Otherwise, cygdb will experience problems. + def module_init_func_cname(self): + env = self.scope + return self.mod_init_func_cname(Naming.pymodule_exec_func_cname, env) def generate_pymoduledef_struct(self, env, code): if env.doc: @@ -2760,7 +3218,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.putln("") code.putln("#if PY_MAJOR_VERSION >= 3") code.putln("#if CYTHON_PEP489_MULTI_PHASE_INIT") - exec_func_cname = self.mod_init_func_cname(Naming.pymodule_exec_func_cname, env) + exec_func_cname = self.module_init_func_cname() code.putln("static PyObject* %s(PyObject *spec, PyModuleDef *def); /*proto*/" % Naming.pymodule_create_func_cname) code.putln("static int %s(PyObject* module); /*proto*/" % exec_func_cname) @@ -2770,15 +3228,27 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.putln("{Py_mod_exec, (void*)%s}," % exec_func_cname) code.putln("{0, NULL}") code.putln("};") + if not env.module_name.isascii(): + code.putln("#else /* CYTHON_PEP489_MULTI_PHASE_INIT */") + code.putln('#error "Unicode module names are only supported with multi-phase init' + ' as per PEP489"') code.putln("#endif") code.putln("") - code.putln("static struct PyModuleDef %s = {" % Naming.pymoduledef_cname) + code.putln('#ifdef __cplusplus') + code.putln('namespace {') + code.putln("struct PyModuleDef %s =" % Naming.pymoduledef_cname) + code.putln('#else') + code.putln("static struct PyModuleDef %s =" % Naming.pymoduledef_cname) + code.putln('#endif') + code.putln('{') code.putln(" PyModuleDef_HEAD_INIT,") - code.putln(' "%s",' % env.module_name) + code.putln(' %s,' % env.module_name.as_c_string_literal()) code.putln(" %s, /* m_doc */" % doc) code.putln("#if CYTHON_PEP489_MULTI_PHASE_INIT") code.putln(" 0, /* m_size */") + code.putln("#elif CYTHON_COMPILING_IN_LIMITED_API") + code.putln(" sizeof(%s), /* m_size */" % Naming.modulestate_cname) code.putln("#else") code.putln(" -1, /* m_size */") code.putln("#endif") @@ -2788,10 +3258,19 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.putln("#else") code.putln(" NULL, /* m_reload */") code.putln("#endif") + code.putln("#if CYTHON_COMPILING_IN_LIMITED_API") + code.putln(" %s_traverse, /* m_traverse */" % Naming.module_cname) + code.putln(" %s_clear, /* m_clear */" % Naming.module_cname) + code.putln(" %s /* m_free */" % cleanup_func) + code.putln("#else") code.putln(" NULL, /* m_traverse */") code.putln(" NULL, /* m_clear */") code.putln(" %s /* m_free */" % cleanup_func) + code.putln("#endif") code.putln("};") + code.putln('#ifdef __cplusplus') + code.putln('} /* anonymous namespace */') + code.putln('#endif') code.putln("#endif") def generate_module_creation_code(self, env, code): @@ -2810,19 +3289,32 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.putln("#else") code.putln("#if PY_MAJOR_VERSION < 3") code.putln( - '%s = Py_InitModule4("%s", %s, %s, 0, PYTHON_API_VERSION); Py_XINCREF(%s);' % ( + '%s = Py_InitModule4(%s, %s, %s, 0, PYTHON_API_VERSION); Py_XINCREF(%s);' % ( env.module_cname, - env.module_name, + env.module_name.as_c_string_literal(), env.method_table_cname, doc, env.module_cname)) - code.putln("#else") + code.putln(code.error_goto_if_null(env.module_cname, self.pos)) + code.putln("#elif CYTHON_COMPILING_IN_LIMITED_API") + module_temp = code.funcstate.allocate_temp(py_object_type, manage_ref=True) + code.putln( + "%s = PyModule_Create(&%s); %s" % ( + module_temp, + Naming.pymoduledef_cname, + code.error_goto_if_null(module_temp, self.pos))) + code.put_gotref(module_temp, py_object_type) + code.putln(code.error_goto_if_neg("PyState_AddModule(%s, &%s)" % ( + module_temp, Naming.pymoduledef_cname), self.pos)) + code.put_decref_clear(module_temp, type=py_object_type) + code.funcstate.release_temp(module_temp) + code.putln('#else') code.putln( "%s = PyModule_Create(&%s);" % ( env.module_cname, Naming.pymoduledef_cname)) - code.putln("#endif") code.putln(code.error_goto_if_null(env.module_cname, self.pos)) + code.putln("#endif") code.putln("#endif") # CYTHON_PEP489_MULTI_PHASE_INIT code.putln( @@ -2913,8 +3405,8 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): # investigation shows that the resulting binary is smaller with repeated functions calls. for entry in entries: signature = entry.type.signature_string() - code.putln('if (__Pyx_ExportFunction("%s", (void (*)(void))%s, "%s") < 0) %s' % ( - entry.name, + code.putln('if (__Pyx_ExportFunction(%s, (void (*)(void))%s, "%s") < 0) %s' % ( + entry.name.as_c_string_literal(), entry.cname, signature, code.error_goto(self.pos))) @@ -2956,7 +3448,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): module.qualified_name, temp, code.error_goto(self.pos))) - code.put_gotref(temp) + code.put_gotref(temp, py_object_type) for entry in entries: if env is module: cname = entry.cname @@ -2986,12 +3478,12 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): module.qualified_name, temp, code.error_goto(self.pos))) - code.put_gotref(temp) + code.put_gotref(temp, py_object_type) for entry in entries: code.putln( - 'if (__Pyx_ImportFunction(%s, "%s", (void (**)(void))&%s, "%s") < 0) %s' % ( + 'if (__Pyx_ImportFunction(%s, %s, (void (**)(void))&%s, "%s") < 0) %s' % ( temp, - entry.name, + entry.name.as_c_string_literal(), entry.cname, entry.type.signature_string(), code.error_goto(self.pos))) @@ -3014,7 +3506,8 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): def generate_base_type_import_code(self, env, entry, code, import_generator): base_type = entry.type.base_type if (base_type and base_type.module_name != env.qualified_name and not - base_type.is_builtin_type and not entry.utility_code_definition): + (base_type.is_builtin_type or base_type.is_cython_builtin_type) + and not entry.utility_code_definition): self.generate_type_import_code(env, base_type, self.pos, code, import_generator) def generate_type_import_code(self, env, type, pos, code, import_generator): @@ -3031,7 +3524,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): if type.vtabptr_cname: code.globalstate.use_utility_code( UtilityCode.load_cached('GetVTable', 'ImportExport.c')) - code.putln("%s = (struct %s*)__Pyx_GetVtable(%s->tp_dict); %s" % ( + code.putln("%s = (struct %s*)__Pyx_GetVtable(%s); %s" % ( type.vtabptr_cname, type.vtabstruct_cname, type.typeptr_cname, @@ -3071,21 +3564,25 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): module, module_name)) + type_name = type.name.as_c_string_literal() + if condition and replacement: code.putln("") # start in new line code.putln("#if %s" % condition) code.putln('"%s",' % replacement) code.putln("#else") - code.putln('"%s",' % type.name) + code.putln('%s,' % type_name) code.putln("#endif") else: - code.put(' "%s", ' % type.name) + code.put(' %s, ' % type_name) if sizeof_objstruct != objstruct: if not condition: code.putln("") # start in new line code.putln("#if defined(PYPY_VERSION_NUM) && PYPY_VERSION_NUM < 0x050B0000") code.putln('sizeof(%s),' % objstruct) + code.putln("#elif CYTHON_COMPILING_IN_LIMITED_API") + code.putln('sizeof(%s),' % objstruct) code.putln("#else") code.putln('sizeof(%s),' % sizeof_objstruct) code.putln("#endif") @@ -3107,6 +3604,10 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): def generate_type_ready_code(self, entry, code): Nodes.CClassDefNode.generate_type_ready_code(entry, code) + def is_main_module_flag_cname(self): + full_module_name = self.full_module_name.replace('.', '__') + return self.punycode_module_name(Naming.module_is_main, full_module_name) + def generate_exttype_vtable_init_code(self, entry, code): # Generate code to initialise the C method table of an # extension type. @@ -3159,7 +3660,7 @@ class ModuleImportGenerator(object): self.temps.append(temp) code.putln('%s = PyImport_ImportModule(%s); if (unlikely(!%s)) %s' % ( temp, module_name_string, temp, error_code)) - code.put_gotref(temp) + code.put_gotref(temp, py_object_type) self.imported[module_name_string] = temp return temp @@ -3220,4 +3721,4 @@ packed_struct_utility_code = UtilityCode(proto=""" #endif """, impl="", proto_block='utility_code_proto_before_types') -capsule_utility_code = UtilityCode.load("Capsule") +capsule_utility_code = UtilityCode.load("Capsule", "Capsule.c") diff --git a/Cython/Compiler/Naming.py b/Cython/Compiler/Naming.py index 2c9b62078..b36239433 100644 --- a/Cython/Compiler/Naming.py +++ b/Cython/Compiler/Naming.py @@ -13,6 +13,8 @@ codewriter_temp_prefix = pyrex_prefix + "t_" temp_prefix = u"__cyt_" +pyunicode_identifier_prefix = pyrex_prefix + 'U' + builtin_prefix = pyrex_prefix + "builtin_" arg_prefix = pyrex_prefix + "arg_" funcdoc_prefix = pyrex_prefix + "doc_" @@ -45,12 +47,18 @@ pybufferstruct_prefix = pyrex_prefix + "pybuffer_" vtable_prefix = pyrex_prefix + "vtable_" vtabptr_prefix = pyrex_prefix + "vtabptr_" vtabstruct_prefix = pyrex_prefix + "vtabstruct_" +unicode_vtabentry_prefix = pyrex_prefix + "Uvtabentry_" +# vtab entries aren't normally mangled, +# but punycode names sometimes start with numbers leading to a C syntax error +unicode_structmember_prefix = pyrex_prefix + "Umember_" +# as above - +# not normally mangled but punycode names cause specific problems opt_arg_prefix = pyrex_prefix + "opt_args_" convert_func_prefix = pyrex_prefix + "convert_" closure_scope_prefix = pyrex_prefix + "scope_" closure_class_prefix = pyrex_prefix + "scope_struct_" lambda_func_prefix = pyrex_prefix + "lambda_" -module_is_main = pyrex_prefix + "module_is_main_" +module_is_main = pyrex_prefix + "module_is_main" defaults_struct_prefix = pyrex_prefix + "defaults" dynamic_args_cname = pyrex_prefix + "dynamic_args" @@ -67,6 +75,8 @@ interned_prefixes = { ctuple_type_prefix = pyrex_prefix + "ctuple_" args_cname = pyrex_prefix + "args" +nargs_cname = pyrex_prefix + "nargs" +kwvalues_cname = pyrex_prefix + "kwvalues" generator_cname = pyrex_prefix + "generator" sent_value_cname = pyrex_prefix + "sent_value" pykwdlist_cname = pyrex_prefix + "pyargnames" @@ -85,6 +95,8 @@ clineno_cname = pyrex_prefix + "clineno" cfilenm_cname = pyrex_prefix + "cfilenm" local_tstate_cname = pyrex_prefix + "tstate" module_cname = pyrex_prefix + "m" +modulestate_cname = pyrex_prefix + "mstate" +modulestateglobal_cname = pyrex_prefix + "mstate_global" moddoc_cname = pyrex_prefix + "mdoc" methtable_cname = pyrex_prefix + "methods" retval_cname = pyrex_prefix + "r" @@ -97,7 +109,7 @@ gilstate_cname = pyrex_prefix + "state" skip_dispatch_cname = pyrex_prefix + "skip_dispatch" empty_tuple = pyrex_prefix + "empty_tuple" empty_bytes = pyrex_prefix + "empty_bytes" -empty_unicode = pyrex_prefix + "empty_unicode" +empty_unicode = pyrex_prefix + "empty_unicode" print_function = pyrex_prefix + "print" print_function_kwargs = pyrex_prefix + "print_kwargs" cleanup_cname = pyrex_prefix + "module_cleanup" @@ -116,11 +128,13 @@ frame_cname = pyrex_prefix + "frame" frame_code_cname = pyrex_prefix + "frame_code" binding_cfunc = pyrex_prefix + "binding_PyCFunctionType" fused_func_prefix = pyrex_prefix + 'fuse_' -quick_temp_cname = pyrex_prefix + "temp" # temp variable for quick'n'dirty temping +quick_temp_cname = pyrex_prefix + "temp" # temp variable for quick'n'dirty temping tp_dict_version_temp = pyrex_prefix + "tp_dict_version" obj_dict_version_temp = pyrex_prefix + "obj_dict_version" type_dict_guard_temp = pyrex_prefix + "type_dict_guard" cython_runtime_cname = pyrex_prefix + "cython_runtime" +cyfunction_type_cname = pyrex_prefix + "CyFunctionType" +fusedfunction_type_cname = pyrex_prefix + "FusedFunctionType" global_code_object_cache_find = pyrex_prefix + 'find_code_object' global_code_object_cache_insert = pyrex_prefix + 'insert_code_object' @@ -152,8 +166,11 @@ exc_vars = (exc_type_name, exc_value_name, exc_tb_name) api_name = pyrex_prefix + "capi__" -h_guard_prefix = "__PYX_HAVE__" -api_guard_prefix = "__PYX_HAVE_API__" +# the h and api guards get changed to: +# __PYX_HAVE__FILENAME (for ascii filenames) +# __PYX_HAVE_U_PUNYCODEFILENAME (for non-ascii filenames) +h_guard_prefix = "__PYX_HAVE_" +api_guard_prefix = "__PYX_HAVE_API_" api_func_guard = "__PYX_HAVE_API_FUNC_" PYX_NAN = "__PYX_NAN()" diff --git a/Cython/Compiler/Nodes.py b/Cython/Compiler/Nodes.py index fe559031a..a095b2a72 100644 --- a/Cython/Compiler/Nodes.py +++ b/Cython/Compiler/Nodes.py @@ -5,6 +5,7 @@ from __future__ import absolute_import import cython + cython.declare(sys=object, os=object, copy=object, Builtin=object, error=object, warning=object, Naming=object, PyrexTypes=object, py_object_type=object, ModuleScope=object, LocalScope=object, ClosureScope=object, @@ -16,13 +17,14 @@ import sys, os, copy from itertools import chain from . import Builtin -from .Errors import error, warning, InternalError, CompileError +from .Errors import error, warning, InternalError, CompileError, CannotSpecialize from . import Naming from . import PyrexTypes from . import TypeSlots from .PyrexTypes import py_object_type, error_type -from .Symtab import (ModuleScope, LocalScope, ClosureScope, - StructOrUnionScope, PyClassScope, CppClassScope, TemplateScope) +from .Symtab import (ModuleScope, LocalScope, ClosureScope, PropertyScope, + StructOrUnionScope, PyClassScope, CppClassScope, TemplateScope, + CppScopedEnumScope, punycodify_name) from .Code import UtilityCode from .StringEncoding import EncodedString from . import Future @@ -38,6 +40,9 @@ else: _py_int_types = (int, long) +IMPLICIT_CLASSMETHODS = {"__init_subclass__", "__class_getitem__"} + + def relative_position(pos): return (pos[0].get_filenametable_entry(), pos[1]) @@ -68,53 +73,6 @@ def embed_position(pos, docstring): return doc -def analyse_type_annotation(annotation, env, assigned_value=None): - base_type = None - is_ambiguous = False - explicit_pytype = explicit_ctype = False - if annotation.is_dict_literal: - warning(annotation.pos, - "Dicts should no longer be used as type annotations. Use 'cython.int' etc. directly.") - for name, value in annotation.key_value_pairs: - if not name.is_string_literal: - continue - if name.value in ('type', b'type'): - explicit_pytype = True - if not explicit_ctype: - annotation = value - elif name.value in ('ctype', b'ctype'): - explicit_ctype = True - annotation = value - if explicit_pytype and explicit_ctype: - warning(annotation.pos, "Duplicate type declarations found in signature annotation") - arg_type = annotation.analyse_as_type(env) - if annotation.is_name and not annotation.cython_attribute and annotation.name in ('int', 'long', 'float'): - # Map builtin numeric Python types to C types in safe cases. - if assigned_value is not None and arg_type is not None and not arg_type.is_pyobject: - assigned_type = assigned_value.infer_type(env) - if assigned_type and assigned_type.is_pyobject: - # C type seems unsafe, e.g. due to 'None' default value => ignore annotation type - is_ambiguous = True - arg_type = None - # ignore 'int' and require 'cython.int' to avoid unsafe integer declarations - if arg_type in (PyrexTypes.c_long_type, PyrexTypes.c_int_type, PyrexTypes.c_float_type): - arg_type = PyrexTypes.c_double_type if annotation.name == 'float' else py_object_type - elif arg_type is not None and annotation.is_string_literal: - warning(annotation.pos, - "Strings should no longer be used for type declarations. Use 'cython.int' etc. directly.") - if arg_type is not None: - if explicit_pytype and not explicit_ctype and not arg_type.is_pyobject: - warning(annotation.pos, - "Python type declaration in signature annotation does not refer to a Python type") - base_type = CAnalysedBaseTypeNode( - annotation.pos, type=arg_type, is_arg=True) - elif is_ambiguous: - warning(annotation.pos, "Ambiguous types in annotation, ignoring") - else: - warning(annotation.pos, "Unknown type declaration in annotation, ignoring") - return base_type, arg_type - - def write_func_call(func, codewriter_class): def f(*args, **kwds): if len(args) > 1 and isinstance(args[1], codewriter_class): @@ -125,19 +83,16 @@ def write_func_call(func, codewriter_class): ' ' * code.call_level, node.__class__.__name__, func.__name__, - node.pos[1:]) - pristine = code.buffer.stream.tell() - code.putln(marker) + node.pos[1:], + ) + insertion_point = code.insertion_point() start = code.buffer.stream.tell() code.call_level += 4 res = func(*args, **kwds) code.call_level -= 4 - if start == code.buffer.stream.tell(): - # no code written => undo writing marker - code.buffer.stream.truncate(pristine) - else: - marker = marker.replace('->', '<-', 1) - code.putln(marker) + if start != code.buffer.stream.tell(): + code.putln(marker.replace('->', '<-', 1)) + insertion_point.putln(marker) return res else: return func(*args, **kwds) @@ -160,9 +115,11 @@ class VerboseCodeWriter(type): class CheckAnalysers(type): """Metaclass to check that type analysis functions return a node. """ - methods = set(['analyse_types', - 'analyse_expressions', - 'analyse_target_types']) + methods = frozenset({ + 'analyse_types', + 'analyse_expressions', + 'analyse_target_types', + }) def __new__(cls, name, bases, attrs): from types import FunctionType @@ -200,6 +157,7 @@ class Node(object): is_literal = 0 is_terminator = 0 is_wrapper = False # is a DefNode wrapper for a C function + is_cproperty = False temps = None # All descendants should set child_attrs to a list of the attributes @@ -254,7 +212,7 @@ class Node(object): # - # There are 3 phases of parse tree processing, applied in order to + # There are 3 main phases of parse tree processing, applied in order to # all the statements in a given scope-block: # # (0) analyse_declarations @@ -266,25 +224,25 @@ class Node(object): # Determine the result types of expressions and fill in the # 'type' attribute of each ExprNode. Insert coercion nodes into the # tree where needed to convert to and from Python objects. - # Allocate temporary locals for intermediate results. Fill - # in the 'result_code' attribute of each ExprNode with a C code - # fragment. + # Replace tree nodes with more appropriate implementations found by + # the type analysis. # # (2) generate_code # Emit C code for all declarations, statements and expressions. - # Recursively applies the 3 processing phases to the bodies of - # functions. + # + # These phases are triggered by tree transformations. + # See the full pipeline in Pipeline.py. # def analyse_declarations(self, env): pass def analyse_expressions(self, env): - raise InternalError("analyse_expressions not implemented for %s" % \ + raise InternalError("analyse_expressions not implemented for %s" % self.__class__.__name__) def generate_code(self, code): - raise InternalError("generate_code not implemented for %s" % \ + raise InternalError("generate_code not implemented for %s" % self.__class__.__name__) def annotate(self, code): @@ -469,7 +427,7 @@ class StatNode(Node): pass def generate_execution_code(self, code): - raise InternalError("generate_execution_code not implemented for %s" % \ + raise InternalError("generate_execution_code not implemented for %s" % self.__class__.__name__) @@ -499,8 +457,13 @@ class CDefExternNode(StatNode): env.add_include_file(self.include_file, self.verbatim_include, late) def analyse_expressions(self, env): + # Allow C properties, inline methods, etc. also in external types. + self.body = self.body.analyse_expressions(env) return self + def generate_function_definitions(self, env, code): + self.body.generate_function_definitions(env, code) + def generate_execution_code(self, code): pass @@ -525,6 +488,9 @@ class CDeclaratorNode(Node): calling_convention = "" + def declared_name(self): + return None + def analyse_templates(self): # Only C++ functions have templates. return None @@ -539,6 +505,9 @@ class CNameDeclaratorNode(CDeclaratorNode): default = None + def declared_name(self): + return self.name + def analyse(self, base_type, env, nonempty=0, visibility=None, in_pxd=False): if nonempty and self.name == '': # May have mistaken the name for the type. @@ -551,7 +520,12 @@ class CNameDeclaratorNode(CDeclaratorNode): base_type = py_object_type if base_type.is_fused and env.fused_to_specific: - base_type = base_type.specialize(env.fused_to_specific) + try: + base_type = base_type.specialize(env.fused_to_specific) + except CannotSpecialize: + error(self.pos, + "'%s' cannot be specialized since its type is not a fused argument to this function" % + self.name) self.type = base_type return self, base_type @@ -562,6 +536,9 @@ class CPtrDeclaratorNode(CDeclaratorNode): child_attrs = ["base"] + def declared_name(self): + return self.base.declared_name() + def analyse_templates(self): return self.base.analyse_templates() @@ -572,14 +549,17 @@ class CPtrDeclaratorNode(CDeclaratorNode): return self.base.analyse(ptr_type, env, nonempty=nonempty, visibility=visibility, in_pxd=in_pxd) -class CReferenceDeclaratorNode(CDeclaratorNode): - # base CDeclaratorNode - +class _CReferenceDeclaratorBaseNode(CDeclaratorNode): child_attrs = ["base"] + def declared_name(self): + return self.base.declared_name() + def analyse_templates(self): return self.base.analyse_templates() + +class CReferenceDeclaratorNode(_CReferenceDeclaratorBaseNode): def analyse(self, base_type, env, nonempty=0, visibility=None, in_pxd=False): if base_type.is_pyobject: error(self.pos, "Reference base type cannot be a Python object") @@ -587,6 +567,14 @@ class CReferenceDeclaratorNode(CDeclaratorNode): return self.base.analyse(ref_type, env, nonempty=nonempty, visibility=visibility, in_pxd=in_pxd) +class CppRvalueReferenceDeclaratorNode(_CReferenceDeclaratorBaseNode): + def analyse(self, base_type, env, nonempty=0, visibility=None, in_pxd=False): + if base_type.is_pyobject: + error(self.pos, "Rvalue-reference base type cannot be a Python object") + ref_type = PyrexTypes.cpp_rvalue_ref_type(base_type) + return self.base.analyse(ref_type, env, nonempty=nonempty, visibility=visibility, in_pxd=in_pxd) + + class CArrayDeclaratorNode(CDeclaratorNode): # base CDeclaratorNode # dimension ExprNode @@ -649,6 +637,9 @@ class CFuncDeclaratorNode(CDeclaratorNode): is_const_method = 0 templates = None + def declared_name(self): + return self.base.declared_name() + def analyse_templates(self): if isinstance(self.base, CArrayDeclaratorNode): from .ExprNodes import TupleNode, NameNode @@ -734,8 +725,8 @@ class CFuncDeclaratorNode(CDeclaratorNode): self.exception_value = ConstNode( self.pos, value=return_type.exception_value, type=return_type) if self.exception_value: - self.exception_value = self.exception_value.analyse_const_expression(env) if self.exception_check == '+': + self.exception_value = self.exception_value.analyse_const_expression(env) exc_val_type = self.exception_value.type if (not exc_val_type.is_error and not exc_val_type.is_pyobject @@ -748,16 +739,25 @@ class CFuncDeclaratorNode(CDeclaratorNode): "Exception value must be a Python exception or cdef function with no arguments or *.") exc_val = self.exception_value else: - self.exception_value = self.exception_value.coerce_to( + self.exception_value = self.exception_value.analyse_types(env).coerce_to( return_type, env).analyse_const_expression(env) exc_val = self.exception_value.get_constant_c_result_code() if exc_val is None: - raise InternalError( - "get_constant_c_result_code not implemented for %s" % - self.exception_value.__class__.__name__) + error(self.exception_value.pos, "Exception value must be constant") if not return_type.assignable_from(self.exception_value.type): error(self.exception_value.pos, "Exception value incompatible with function return type") + if (visibility != 'extern' + and (return_type.is_int or return_type.is_float) + and self.exception_value.has_constant_result()): + try: + type_default_value = float(return_type.default_value) + except ValueError: + pass + else: + if self.exception_value.constant_result == type_default_value: + warning(self.pos, "Ambiguous exception value, same as default return value: %r" % + self.exception_value.constant_result) exc_check = self.exception_check if return_type.is_cfunction: error(self.pos, "Function cannot return a function") @@ -789,6 +789,13 @@ class CFuncDeclaratorNode(CDeclaratorNode): error(self.pos, "cannot have both '%s' and '%s' " "calling conventions" % (current, callspec)) func_type.calling_convention = callspec + + if func_type.return_type.is_rvalue_reference: + warning(self.pos, "Rvalue-reference as function return type not supported", 1) + for arg in func_type.args: + if arg.type.is_rvalue_reference and not arg.is_forwarding_reference(): + warning(self.pos, "Rvalue-reference as function argument not supported", 1) + return self.base.analyse(func_type, env, visibility=visibility, in_pxd=in_pxd) def declare_optional_arg_struct(self, func_type, env, fused_cname=None): @@ -850,8 +857,12 @@ class CArgDeclNode(Node): # annotation ExprNode or None Py3 function arg annotation # is_self_arg boolean Is the "self" arg of an extension type method # is_type_arg boolean Is the "class" arg of an extension type classmethod - # is_kw_only boolean Is a keyword-only argument + # kw_only boolean Is a keyword-only argument # is_dynamic boolean Non-literal arg stored inside CyFunction + # pos_only boolean Is a positional-only argument + # + # name_cstring property that converts the name to a cstring taking care of unicode + # and quoting it child_attrs = ["base_type", "declarator", "default", "annotation"] outer_attrs = ["default", "annotation"] @@ -860,6 +871,7 @@ class CArgDeclNode(Node): is_type_arg = 0 is_generic = 1 kw_only = 0 + pos_only = 0 not_none = 0 or_none = 0 type = None @@ -868,6 +880,23 @@ class CArgDeclNode(Node): annotation = None is_dynamic = 0 + def declared_name(self): + return self.declarator.declared_name() + + @property + def name_cstring(self): + return self.name.as_c_string_literal() + + @property + def hdr_cname(self): + # done lazily - needs self.entry to be set to get the class-mangled + # name, which means it has to be generated relatively late + if self.needs_conversion: + return punycodify_name(Naming.arg_prefix + self.entry.name) + else: + return punycodify_name(Naming.var_prefix + self.entry.name) + + def analyse(self, env, nonempty=0, is_self_arg=False): if is_self_arg: self.base_type.is_self_arg = self.is_self_arg = True @@ -889,8 +918,9 @@ class CArgDeclNode(Node): could_be_name = False self.base_type.is_arg = True base_type = self.base_type.analyse(env, could_be_name=could_be_name) - if hasattr(self.base_type, 'arg_name') and self.base_type.arg_name: - self.declarator.name = self.base_type.arg_name + base_arg_name = getattr(self.base_type, 'arg_name', None) + if base_arg_name: + self.declarator.name = base_arg_name # The parser is unable to resolve the ambiguity of [] as part of the # type (e.g. in buffers) or empty declarator (as with arrays). @@ -906,7 +936,10 @@ class CArgDeclNode(Node): # inject type declaration from annotations # this is called without 'env' by AdjustDefByDirectives transform before declaration analysis - if self.annotation and env and env.directives['annotation_typing'] and self.base_type.name is None: + if (self.annotation and env and env.directives['annotation_typing'] + # CSimpleBaseTypeNode has a name attribute; CAnalysedBaseTypeNode + # (and maybe other options) doesn't + and getattr(self.base_type, "name", None) is None): arg_type = self.inject_type_from_annotations(env) if arg_type is not None: base_type = arg_type @@ -918,7 +951,7 @@ class CArgDeclNode(Node): annotation = self.annotation if not annotation: return None - base_type, arg_type = analyse_type_annotation(annotation, env, assigned_value=self.default) + base_type, arg_type = annotation.analyse_type_annotation(env, assigned_value=self.default) if base_type is not None: self.base_type = base_type return arg_type @@ -947,8 +980,7 @@ class CArgDeclNode(Node): default.make_owned_reference(code) result = default.result() if overloaded_assignment else default.result_as(self.type) code.putln("%s = %s;" % (target, result)) - if self.type.is_pyobject: - code.put_giveref(default.result()) + code.put_giveref(default.result(), self.type) default.generate_post_assignment_code(code) default.free_temps(code) @@ -1014,7 +1046,10 @@ class CSimpleBaseTypeNode(CBaseTypeNode): scope = env for item in self.module_path: entry = scope.lookup(item) - if entry is not None and entry.is_cpp_class: + if entry is not None and ( + entry.is_cpp_class or + entry.is_type and entry.type.is_cpp_class + ): scope = entry.type.scope else: scope = None @@ -1043,7 +1078,7 @@ class CSimpleBaseTypeNode(CBaseTypeNode): self.arg_name = EncodedString(self.name) else: if self.templates: - if not self.name in self.templates: + if self.name not in self.templates: error(self.pos, "'%s' is not a type identifier" % self.name) type = PyrexTypes.TemplatePlaceholderType(self.name) else: @@ -1199,7 +1234,12 @@ class TemplatedTypeNode(CBaseTypeNode): self.type = self.array_declarator.analyse(base_type, env)[1] if self.type.is_fused and env.fused_to_specific: - self.type = self.type.specialize(env.fused_to_specific) + try: + self.type = self.type.specialize(env.fused_to_specific) + except CannotSpecialize: + error(self.pos, + "'%s' cannot be specialized since its type is not a fused argument to this function" % + self.name) return self.type @@ -1273,8 +1313,10 @@ class FusedTypeNode(CBaseTypeNode): return PyrexTypes.FusedType(types, name=self.name) -class CConstTypeNode(CBaseTypeNode): +class CConstOrVolatileTypeNode(CBaseTypeNode): # base_type CBaseTypeNode + # is_const boolean + # is_volatile boolean child_attrs = ["base_type"] @@ -1282,8 +1324,8 @@ class CConstTypeNode(CBaseTypeNode): base = self.base_type.analyse(env, could_be_name) if base.is_pyobject: error(self.pos, - "Const base type cannot be a Python object") - return PyrexTypes.c_const_type(base) + "Const/volatile base type cannot be a Python object") + return PyrexTypes.c_const_or_volatile_type(base, self.is_const, self.is_volatile) class CVarDefNode(StatNode): @@ -1369,6 +1411,8 @@ class CVarDefNode(StatNode): return if type.is_reference and self.visibility != 'extern': error(declarator.pos, "C++ references cannot be declared; use a pointer instead") + if type.is_rvalue_reference and self.visibility != 'extern': + error(declarator.pos, "C++ rvalue-references cannot be declared") if type.is_cfunction: if 'staticmethod' in env.directives: type.is_static_method = True @@ -1529,36 +1573,62 @@ class CppClassNode(CStructOrUnionDefNode, BlockNode): class CEnumDefNode(StatNode): - # name string or None - # cname string or None - # items [CEnumDefItemNode] - # typedef_flag boolean - # visibility "public" or "private" or "extern" - # api boolean - # in_pxd boolean - # create_wrapper boolean - # entry Entry - - child_attrs = ["items"] + # name string or None + # cname string or None + # scoped boolean Is a C++ scoped enum + # underlying_type CSimpleBaseTypeNode The underlying value type (int or C++ type) + # items [CEnumDefItemNode] + # typedef_flag boolean + # visibility "public" or "private" or "extern" + # api boolean + # in_pxd boolean + # create_wrapper boolean + # entry Entry + # doc EncodedString or None Doc string + + child_attrs = ["items", "underlying_type"] + doc = None def declare(self, env): - self.entry = env.declare_enum( - self.name, self.pos, - cname=self.cname, typedef_flag=self.typedef_flag, - visibility=self.visibility, api=self.api, - create_wrapper=self.create_wrapper) + doc = None + if Options.docstrings: + doc = embed_position(self.pos, self.doc) + + self.entry = env.declare_enum( + self.name, self.pos, + cname=self.cname, + scoped=self.scoped, + typedef_flag=self.typedef_flag, + visibility=self.visibility, api=self.api, + create_wrapper=self.create_wrapper, doc=doc) def analyse_declarations(self, env): + scope = None + underlying_type = self.underlying_type.analyse(env) + + if not underlying_type.is_int: + error(self.underlying_type.pos, "underlying type is not an integral type") + + self.entry.type.underlying_type = underlying_type + + if self.scoped and self.items is not None: + scope = CppScopedEnumScope(self.name, env) + scope.type = self.entry.type + else: + scope = env + if self.items is not None: if self.in_pxd and not env.in_cinclude: self.entry.defined_in_pxd = 1 for item in self.items: - item.analyse_declarations(env, self.entry) + item.analyse_declarations(scope, self.entry) def analyse_expressions(self, env): return self def generate_execution_code(self, code): + if self.scoped: + return # nothing to do here for C++ enums if self.visibility == 'public' or self.api: code.mark_pos(self.pos) temp = code.funcstate.allocate_temp(PyrexTypes.py_object_type, manage_ref=True) @@ -1567,7 +1637,7 @@ class CEnumDefNode(StatNode): temp, item.cname, code.error_goto_if_null(temp, item.pos))) - code.put_gotref(temp) + code.put_gotref(temp, PyrexTypes.py_object_type) code.putln('if (PyDict_SetItemString(%s, "%s", %s) < 0) %s' % ( Naming.moddict_cname, item.name, @@ -1590,9 +1660,15 @@ class CEnumDefItemNode(StatNode): if not self.value.type.is_int: self.value = self.value.coerce_to(PyrexTypes.c_int_type, env) self.value = self.value.analyse_const_expression(env) + + if enum_entry.type.is_cpp_enum: + cname = "%s::%s" % (enum_entry.cname, self.name) + else: + cname = self.cname + entry = env.declare_const( self.name, enum_entry.type, - self.value, self.pos, cname=self.cname, + self.value, self.pos, cname=cname, visibility=enum_entry.visibility, api=enum_entry.api, create_wrapper=enum_entry.create_wrapper and enum_entry.name is None) enum_entry.enum_values.append(entry) @@ -1659,6 +1735,8 @@ class FuncDefNode(StatNode, BlockNode): needs_outer_scope = False pymethdef_required = False is_generator = False + is_coroutine = False + is_asyncgen = False is_generator_body = False is_async_def = False modifiers = [] @@ -1667,6 +1745,7 @@ class FuncDefNode(StatNode, BlockNode): starstar_arg = None is_cyfunction = False code_object = None + return_type_annotation = None def analyse_default_values(self, env): default_seen = 0 @@ -1684,18 +1763,12 @@ class FuncDefNode(StatNode, BlockNode): elif default_seen: error(arg.pos, "Non-default argument following default argument") - def analyse_annotation(self, env, annotation): - # Annotations can not only contain valid Python expressions but arbitrary type references. - if annotation is None: - return None - if not env.directives['annotation_typing'] or annotation.analyse_as_type(env) is None: - annotation = annotation.analyse_types(env) - return annotation - def analyse_annotations(self, env): for arg in self.args: if arg.annotation: - arg.annotation = self.analyse_annotation(env, arg.annotation) + arg.annotation = arg.annotation.analyse_types(env) + if self.return_type_annotation: + self.return_type_annotation = self.return_type_annotation.analyse_types(env) def align_argument_type(self, env, arg): # @cython.locals() @@ -1718,6 +1791,9 @@ class FuncDefNode(StatNode, BlockNode): error(type_node.pos, "Previous declaration here") else: arg.type = other_type + if arg.type.is_complex: + # utility code for complex types is special-cased and also important to ensure that it's run + arg.type.create_declaration_utility_code(env) return arg def need_gil_acquisition(self, lenv): @@ -1749,8 +1825,6 @@ class FuncDefNode(StatNode, BlockNode): def generate_function_definitions(self, env, code): from . import Buffer - if self.return_type.is_memoryviewslice: - from . import MemoryView lenv = self.local_scope if lenv.is_closure_scope and not lenv.is_passthrough: @@ -1778,6 +1852,8 @@ class FuncDefNode(StatNode, BlockNode): profile = code.globalstate.directives['profile'] linetrace = code.globalstate.directives['linetrace'] if profile or linetrace: + if linetrace: + code.use_fast_gil_utility_code() code.globalstate.use_utility_code( UtilityCode.load_cached("Profile", "Profile.c")) @@ -1823,14 +1899,15 @@ class FuncDefNode(StatNode, BlockNode): # Initialize the return variable __pyx_r init = "" - if not self.return_type.is_void: - if self.return_type.is_pyobject: + return_type = self.return_type + if not return_type.is_void: + if return_type.is_pyobject: init = " = NULL" - elif self.return_type.is_memoryviewslice: - init = ' = ' + MemoryView.memslice_entry_init + elif return_type.is_memoryviewslice: + init = ' = ' + return_type.literal_code(return_type.default_value) code.putln("%s%s;" % ( - self.return_type.declaration_code(Naming.retval_cname), + return_type.declaration_code(Naming.retval_cname), init)) tempvardecl_code = code.insertion_point() @@ -1862,11 +1939,12 @@ class FuncDefNode(StatNode, BlockNode): use_refnanny = not lenv.nogil or lenv.has_with_gil_block + gilstate_decl = None if acquire_gil or acquire_gil_for_var_decls_only: code.put_ensure_gil() code.funcstate.gil_owned = True - elif lenv.nogil and lenv.has_with_gil_block: - code.declare_gilstate() + else: + gilstate_decl = code.insertion_point() if profile or linetrace: if not self.is_generator: @@ -1908,7 +1986,7 @@ class FuncDefNode(StatNode, BlockNode): code.put_incref("Py_None", py_object_type) code.putln(code.error_goto(self.pos)) code.putln("} else {") - code.put_gotref(Naming.cur_scope_cname) + code.put_gotref(Naming.cur_scope_cname, lenv.scope_class.type) code.putln("}") # Note that it is unsafe to decref the scope at this point. if self.needs_outer_scope: @@ -1927,7 +2005,7 @@ class FuncDefNode(StatNode, BlockNode): elif self.needs_closure: # inner closures own a reference to their outer parent code.put_incref(outer_scope_cname, cenv.scope_class.type) - code.put_giveref(outer_scope_cname) + code.put_giveref(outer_scope_cname, cenv.scope_class.type) # ----- Trace function call if profile or linetrace: # this looks a bit late, but if we don't get here due to a @@ -1947,18 +2025,18 @@ class FuncDefNode(StatNode, BlockNode): # incref it to properly keep track of refcounts. is_cdef = isinstance(self, CFuncDefNode) for entry in lenv.arg_entries: - if entry.type.is_pyobject: - if (acquire_gil or len(entry.cf_assignments) > 1) and not entry.in_closure: + if not entry.type.is_memoryviewslice: + if (acquire_gil or entry.cf_is_reassigned) and not entry.in_closure: code.put_var_incref(entry) - # Note: defaults are always incref-ed. For def functions, we # we acquire arguments from object conversion, so we have # new references. If we are a cdef function, we need to # incref our arguments - elif is_cdef and entry.type.is_memoryviewslice and len(entry.cf_assignments) > 1: - code.put_incref_memoryviewslice(entry.cname, have_gil=code.funcstate.gil_owned) + elif is_cdef and entry.cf_is_reassigned: + code.put_var_incref_memoryviewslice(entry, + have_gil=code.funcstate.gil_owned) for entry in lenv.var_entries: - if entry.is_arg and len(entry.cf_assignments) > 1 and not entry.in_closure: + if entry.is_arg and entry.cf_is_reassigned and not entry.in_closure: if entry.xdecref_cleanup: code.put_var_xincref(entry) else: @@ -1989,27 +2067,45 @@ class FuncDefNode(StatNode, BlockNode): code.putln("") code.putln("/* function exit code */") + gil_owned = { + 'success': code.funcstate.gil_owned, + 'error': code.funcstate.gil_owned, + 'gil_state_declared': gilstate_decl is None, + } + def assure_gil(code_path, code=code): + if not gil_owned[code_path]: + if not gil_owned['gil_state_declared']: + gilstate_decl.declare_gilstate() + gil_owned['gil_state_declared'] = True + code.put_ensure_gil(declare_gilstate=False) + gil_owned[code_path] = True + # ----- Default return value + return_type = self.return_type if not self.body.is_terminator: - if self.return_type.is_pyobject: - #if self.return_type.is_extension_type: + if return_type.is_pyobject: + #if return_type.is_extension_type: # lhs = "(PyObject *)%s" % Naming.retval_cname #else: lhs = Naming.retval_cname - code.put_init_to_py_none(lhs, self.return_type) - else: - val = self.return_type.default_value + assure_gil('success') + code.put_init_to_py_none(lhs, return_type) + elif not return_type.is_memoryviewslice: + # memory view structs receive their default value on initialisation + val = return_type.default_value if val: code.putln("%s = %s;" % (Naming.retval_cname, val)) - elif not self.return_type.is_void: + elif not return_type.is_void: code.putln("__Pyx_pretend_to_initialize(&%s);" % Naming.retval_cname) + # ----- Error cleanup - if code.error_label in code.labels_used: + if code.label_used(code.error_label): if not self.body.is_terminator: code.put_goto(code.return_label) code.put_label(code.error_label) for cname, type in code.funcstate.all_managed_temps(): - code.put_xdecref(cname, type, have_gil=not lenv.nogil) + assure_gil('error') + code.put_xdecref(cname, type, have_gil=gil_owned['error']) # Clean up buffers -- this calls a Python function # so need to save and restore error state @@ -2019,6 +2115,7 @@ class FuncDefNode(StatNode, BlockNode): code.globalstate.use_utility_code(restore_exception_utility_code) code.putln("{ PyObject *__pyx_type, *__pyx_value, *__pyx_tb;") code.putln("__Pyx_PyThreadState_declare") + assure_gil('error') code.putln("__Pyx_PyThreadState_assign") code.putln("__Pyx_ErrFetch(&__pyx_type, &__pyx_value, &__pyx_tb);") for entry in used_buffer_entries: @@ -2026,7 +2123,8 @@ class FuncDefNode(StatNode, BlockNode): #code.putln("%s = 0;" % entry.cname) code.putln("__Pyx_ErrRestore(__pyx_type, __pyx_value, __pyx_tb);}") - if self.return_type.is_memoryviewslice: + if return_type.is_memoryviewslice: + from . import MemoryView MemoryView.put_init_entry(Naming.retval_cname, code) err_val = Naming.retval_cname else: @@ -2038,104 +2136,129 @@ class FuncDefNode(StatNode, BlockNode): # code.globalstate.use_utility_code(get_exception_tuple_utility_code) # code.put_trace_exception() - if lenv.nogil and not lenv.has_with_gil_block: - code.putln("{") - code.put_ensure_gil() - + assure_gil('error') code.put_add_traceback(self.entry.qualified_name) - - if lenv.nogil and not lenv.has_with_gil_block: - code.put_release_ensured_gil() - code.putln("}") else: warning(self.entry.pos, "Unraisable exception in function '%s'." % self.entry.qualified_name, 0) - code.put_unraisable(self.entry.qualified_name, lenv.nogil) - default_retval = self.return_type.default_value + assure_gil('error') + code.put_unraisable(self.entry.qualified_name) + default_retval = return_type.default_value if err_val is None and default_retval: err_val = default_retval if err_val is not None: if err_val != Naming.retval_cname: code.putln("%s = %s;" % (Naming.retval_cname, err_val)) - elif not self.return_type.is_void: + elif not return_type.is_void: code.putln("__Pyx_pretend_to_initialize(&%s);" % Naming.retval_cname) if is_getbuffer_slot: + assure_gil('error') self.getbuffer_error_cleanup(code) + def align_error_path_gil_to_success_path(code=code.insertion_point()): + # align error and success GIL state when both join + if gil_owned['success']: + assure_gil('error', code=code) + elif gil_owned['error']: + code.put_release_ensured_gil() + gil_owned['error'] = False + assert gil_owned['error'] == gil_owned['success'], "%s: error path %s != success path %s" % ( + self.pos, gil_owned['error'], gil_owned['success']) + # If we are using the non-error cleanup section we should # jump past it if we have an error. The if-test below determine # whether this section is used. - if buffers_present or is_getbuffer_slot or self.return_type.is_memoryviewslice: + if buffers_present or is_getbuffer_slot or return_type.is_memoryviewslice: + # In the buffer cases, we already called assure_gil('error') and own the GIL. + assert gil_owned['error'] or return_type.is_memoryviewslice code.put_goto(code.return_from_error_cleanup_label) + else: + # Adapt the GIL state to the success path right now. + align_error_path_gil_to_success_path() + else: + # No error path, no need to adapt the GIL state. + def align_error_path_gil_to_success_path(): pass # ----- Non-error return cleanup - code.put_label(code.return_label) - for entry in used_buffer_entries: - Buffer.put_release_buffer_code(code, entry) - if is_getbuffer_slot: - self.getbuffer_normal_cleanup(code) + if code.label_used(code.return_label) or not code.label_used(code.error_label): + code.put_label(code.return_label) - if self.return_type.is_memoryviewslice: - # See if our return value is uninitialized on non-error return - # from . import MemoryView - # MemoryView.err_if_nogil_initialized_check(self.pos, env) - cond = code.unlikely(self.return_type.error_condition(Naming.retval_cname)) - code.putln( - 'if (%s) {' % cond) - if env.nogil: - code.put_ensure_gil() - code.putln( - 'PyErr_SetString(PyExc_TypeError, "Memoryview return value is not initialized");') - if env.nogil: - code.put_release_ensured_gil() - code.putln( - '}') + for entry in used_buffer_entries: + assure_gil('success') + Buffer.put_release_buffer_code(code, entry) + if is_getbuffer_slot: + assure_gil('success') + self.getbuffer_normal_cleanup(code) + + if return_type.is_memoryviewslice: + # See if our return value is uninitialized on non-error return + # from . import MemoryView + # MemoryView.err_if_nogil_initialized_check(self.pos, env) + cond = code.unlikely(return_type.error_condition(Naming.retval_cname)) + code.putln( + 'if (%s) {' % cond) + if not gil_owned['success']: + code.put_ensure_gil() + code.putln( + 'PyErr_SetString(PyExc_TypeError, "Memoryview return value is not initialized");') + if not gil_owned['success']: + code.put_release_ensured_gil() + code.putln( + '}') # ----- Return cleanup for both error and no-error return - code.put_label(code.return_from_error_cleanup_label) + if code.label_used(code.return_from_error_cleanup_label): + align_error_path_gil_to_success_path() + code.put_label(code.return_from_error_cleanup_label) for entry in lenv.var_entries: if not entry.used or entry.in_closure: continue - if entry.type.is_memoryviewslice: - code.put_xdecref_memoryviewslice(entry.cname, have_gil=not lenv.nogil) - elif entry.type.is_pyobject: - if not entry.is_arg or len(entry.cf_assignments) > 1: - if entry.xdecref_cleanup: - code.put_var_xdecref(entry) - else: - code.put_var_decref(entry) + if entry.type.is_pyobject: + if entry.is_arg and not entry.cf_is_reassigned: + continue + if entry.type.needs_refcounting: + assure_gil('success') + # FIXME ideally use entry.xdecref_cleanup but this currently isn't reliable + code.put_var_xdecref(entry, have_gil=gil_owned['success']) # Decref any increfed args for entry in lenv.arg_entries: - if entry.type.is_pyobject: - if (acquire_gil or len(entry.cf_assignments) > 1) and not entry.in_closure: - code.put_var_decref(entry) - elif (entry.type.is_memoryviewslice and - (not is_cdef or len(entry.cf_assignments) > 1)): + if entry.type.is_memoryviewslice: # decref slices of def functions and acquired slices from cdef # functions, but not borrowed slices from cdef functions. - code.put_xdecref_memoryviewslice(entry.cname, - have_gil=not lenv.nogil) + if is_cdef and not entry.cf_is_reassigned: + continue + else: + if entry.in_closure: + continue + if not acquire_gil and not entry.cf_is_reassigned: + continue + if entry.type.needs_refcounting: + assure_gil('success') + + # FIXME use entry.xdecref_cleanup - del arg seems to be the problem + code.put_var_xdecref(entry, have_gil=gil_owned['success']) if self.needs_closure: + assure_gil('success') code.put_decref(Naming.cur_scope_cname, lenv.scope_class.type) # ----- Return # This code is duplicated in ModuleNode.generate_module_init_func if not lenv.nogil: - default_retval = self.return_type.default_value + default_retval = return_type.default_value err_val = self.error_value() if err_val is None and default_retval: err_val = default_retval # FIXME: why is err_val not used? - if self.return_type.is_pyobject: - code.put_xgiveref(self.return_type.as_pyobject(Naming.retval_cname)) + code.put_xgiveref(Naming.retval_cname, return_type) if self.entry.is_special and self.entry.name == "__hash__": # Returning -1 for __hash__ is supposed to signal an error # We do as Python instances and coerce -1 into -2. + assure_gil('success') # in special methods, the GIL is owned anyway code.putln("if (unlikely(%s == -1) && !PyErr_Occurred()) %s = -2;" % ( Naming.retval_cname, Naming.retval_cname)) @@ -2143,23 +2266,22 @@ class FuncDefNode(StatNode, BlockNode): code.funcstate.can_trace = False if not self.is_generator: # generators are traced when iterated, not at creation - if self.return_type.is_pyobject: + if return_type.is_pyobject: code.put_trace_return( - Naming.retval_cname, nogil=not code.funcstate.gil_owned) + Naming.retval_cname, nogil=not gil_owned['success']) else: code.put_trace_return( - "Py_None", nogil=not code.funcstate.gil_owned) + "Py_None", nogil=not gil_owned['success']) - if not lenv.nogil: - # GIL holding function - code.put_finish_refcount_context() + if use_refnanny: + code.put_finish_refcount_context(nogil=not gil_owned['success']) - if acquire_gil or (lenv.nogil and lenv.has_with_gil_block): + if acquire_gil or (lenv.nogil and gil_owned['success']): # release the GIL (note that with-gil blocks acquire it on exit in their EnsureGILNode) code.put_release_ensured_gil() code.funcstate.gil_owned = False - if not self.return_type.is_void: + if not return_type.is_void: code.putln("return %s;" % Naming.retval_cname) code.putln("}") @@ -2194,11 +2316,11 @@ class FuncDefNode(StatNode, BlockNode): typeptr_cname = arg.type.typeptr_cname arg_code = "((PyObject *)%s)" % arg.entry.cname code.putln( - 'if (unlikely(!__Pyx_ArgTypeTest(%s, %s, %d, "%s", %s))) %s' % ( + 'if (unlikely(!__Pyx_ArgTypeTest(%s, %s, %d, %s, %s))) %s' % ( arg_code, typeptr_cname, arg.accept_none, - arg.name, + arg.name_cstring, arg.type.is_builtin_type and arg.type.require_exact, code.error_goto(arg.pos))) else: @@ -2212,8 +2334,8 @@ class FuncDefNode(StatNode, BlockNode): cname = arg.entry.cname code.putln('if (unlikely(((PyObject *)%s) == Py_None)) {' % cname) - code.putln('''PyErr_Format(PyExc_TypeError, "Argument '%%.%ds' must not be None", "%s"); %s''' % ( - max(200, len(arg.name)), arg.name, + code.putln('''PyErr_Format(PyExc_TypeError, "Argument '%%.%ds' must not be None", %s); %s''' % ( + max(200, len(arg.name_cstring)), arg.name_cstring, code.error_goto(arg.pos))) code.putln('}') @@ -2249,7 +2371,7 @@ class FuncDefNode(StatNode, BlockNode): def getbuffer_check(self, code): py_buffer, _ = self._get_py_buffer_info() view = py_buffer.cname - code.putln("if (%s == NULL) {" % view) + code.putln("if (unlikely(%s == NULL)) {" % view) code.putln("PyErr_SetString(PyExc_BufferError, " "\"PyObject_GetBuffer: view==NULL argument is obsolete\");") code.putln("return -1;") @@ -2260,7 +2382,7 @@ class FuncDefNode(StatNode, BlockNode): view = py_buffer.cname if obj_type and obj_type.is_pyobject: code.put_init_to_py_none("%s->obj" % view, obj_type) - code.put_giveref("%s->obj" % view) # Do not refnanny object within structs + code.put_giveref("%s->obj" % view, obj_type) # Do not refnanny object within structs else: code.putln("%s->obj = NULL;" % view) @@ -2269,7 +2391,7 @@ class FuncDefNode(StatNode, BlockNode): view = py_buffer.cname if obj_type and obj_type.is_pyobject: code.putln("if (%s->obj != NULL) {" % view) - code.put_gotref("%s->obj" % view) + code.put_gotref("%s->obj" % view, obj_type) code.put_decref_clear("%s->obj" % view, obj_type) code.putln("}") else: @@ -2280,7 +2402,7 @@ class FuncDefNode(StatNode, BlockNode): view = py_buffer.cname if obj_type and obj_type.is_pyobject: code.putln("if (%s->obj == Py_None) {" % view) - code.put_gotref("%s->obj" % view) + code.put_gotref("%s->obj" % view, obj_type) code.put_decref_clear("%s->obj" % view, obj_type) code.putln("}") @@ -2322,7 +2444,8 @@ class CFuncDefNode(FuncDefNode): # is_static_method whether this is a static method # is_c_class_method whether this is a cclass method - child_attrs = ["base_type", "declarator", "body", "py_func_stat"] + child_attrs = ["base_type", "declarator", "body", "decorators", "py_func_stat"] + outer_attrs = ["decorators", "py_func_stat"] inline_in_pxd = False decorators = None @@ -2336,6 +2459,9 @@ class CFuncDefNode(FuncDefNode): def unqualified_name(self): return self.entry.name + def declared_name(self): + return self.declarator.declared_name() + @property def code_object(self): # share the CodeObject with the cpdef wrapper (if available) @@ -2356,20 +2482,20 @@ class CFuncDefNode(FuncDefNode): self.is_static_method = 'staticmethod' in env.directives and not env.lookup_here('staticmethod') # The 2 here is because we need both function and argument names. if isinstance(self.declarator, CFuncDeclaratorNode): - name_declarator, type = self.declarator.analyse( + name_declarator, typ = self.declarator.analyse( base_type, env, nonempty=2 * (self.body is not None), directive_locals=self.directive_locals, visibility=self.visibility) else: - name_declarator, type = self.declarator.analyse( + name_declarator, typ = self.declarator.analyse( base_type, env, nonempty=2 * (self.body is not None), visibility=self.visibility) - if not type.is_cfunction: + if not typ.is_cfunction: error(self.pos, "Suite attached to non-function declaration") # Remember the actual type according to the function header # written here, because the type in the symbol table entry # may be different if we're overriding a C method inherited # from the base type of an extension type. - self.type = type - type.is_overridable = self.overridable + self.type = typ + typ.is_overridable = self.overridable declarator = self.declarator while not hasattr(declarator, 'args'): declarator = declarator.base @@ -2382,11 +2508,11 @@ class CFuncDefNode(FuncDefNode): error(self.cfunc_declarator.pos, "Function with optional arguments may not be declared public or api") - if type.exception_check == '+' and self.visibility != 'extern': + if typ.exception_check == '+' and self.visibility != 'extern': warning(self.cfunc_declarator.pos, "Only extern functions can throw C++ exceptions.") - for formal_arg, type_arg in zip(self.args, type.args): + for formal_arg, type_arg in zip(self.args, typ.args): self.align_argument_type(env, type_arg) formal_arg.type = type_arg.type formal_arg.name = type_arg.name @@ -2407,20 +2533,21 @@ class CFuncDefNode(FuncDefNode): elif 'inline' in self.modifiers: warning(formal_arg.pos, "Buffer unpacking not optimized away.", 1) - self._validate_type_visibility(type.return_type, self.pos, env) + self._validate_type_visibility(typ.return_type, self.pos, env) name = name_declarator.name cname = name_declarator.cname - type.is_const_method = self.is_const_method - type.is_static_method = self.is_static_method + typ.is_const_method = self.is_const_method + typ.is_static_method = self.is_static_method + self.entry = env.declare_cfunction( - name, type, self.pos, + name, typ, self.pos, cname=cname, visibility=self.visibility, api=self.api, defining=self.body is not None, modifiers=self.modifiers, overridable=self.overridable) self.entry.inline_func_in_pxd = self.inline_in_pxd - self.return_type = type.return_type + self.return_type = typ.return_type if self.return_type.is_array and self.visibility != 'extern': error(self.pos, "Function cannot return an array") if self.return_type.is_cpp_class: @@ -2443,7 +2570,7 @@ class CFuncDefNode(FuncDefNode): py_func_body = self.call_self_node(is_module_scope=env.is_module_scope) if self.is_static_method: from .ExprNodes import NameNode - decorators = [DecoratorNode(self.pos, decorator=NameNode(self.pos, name='staticmethod'))] + decorators = [DecoratorNode(self.pos, decorator=NameNode(self.pos, name=EncodedString('staticmethod')))] decorators[0].decorator.analyse_types(env) else: decorators = [] @@ -2585,7 +2712,7 @@ class CFuncDefNode(FuncDefNode): header = self.return_type.declaration_code(entity, dll_linkage=dll_linkage) #print (storage_class, modifiers, header) - needs_proto = self.is_c_class_method + needs_proto = self.is_c_class_method or self.entry.is_cproperty if self.template_declaration: if needs_proto: code.globalstate.parts['module_declarations'].putln(self.template_declaration) @@ -2766,7 +2893,7 @@ class DefNode(FuncDefNode): self_in_stararg = 0 py_cfunc_node = None requires_classobj = False - defaults_struct = None # Dynamic kwrds structure name + defaults_struct = None # Dynamic kwrds structure name doc = None fused_py_func = False @@ -2779,14 +2906,17 @@ class DefNode(FuncDefNode): def __init__(self, pos, **kwds): FuncDefNode.__init__(self, pos, **kwds) - k = rk = r = 0 + p = k = rk = r = 0 for arg in self.args: + if arg.pos_only: + p += 1 if arg.kw_only: k += 1 if not arg.default: rk += 1 if not arg.default: r += 1 + self.num_posonly_args = p self.num_kwonly_args = k self.num_required_kw_args = rk self.num_required_args = r @@ -2884,8 +3014,18 @@ class DefNode(FuncDefNode): # staticmethod() was overridden - not much we can do here ... self.is_staticmethod = False - if self.name == '__new__' and env.is_py_class_scope: - self.is_staticmethod = 1 + if env.is_py_class_scope or env.is_c_class_scope: + if self.name == '__new__' and env.is_py_class_scope: + self.is_staticmethod = True + elif self.name == '__init_subclass__' and env.is_c_class_scope: + error(self.pos, "'__init_subclass__' is not supported by extension class") + elif self.name in IMPLICIT_CLASSMETHODS and not self.is_classmethod: + self.is_classmethod = True + # TODO: remove the need to generate a real decorator here, is_classmethod=True should suffice. + from .ExprNodes import NameNode + self.decorators = self.decorators or [] + self.decorators.insert(0, DecoratorNode( + self.pos, decorator=NameNode(self.pos, name=EncodedString('classmethod')))) self.analyse_argument_types(env) if self.name == '<lambda>': @@ -2898,7 +3038,7 @@ class DefNode(FuncDefNode): # if a signature annotation provides a more specific return object type, use it if self.return_type is py_object_type and self.return_type_annotation: if env.directives['annotation_typing'] and not self.entry.is_special: - _, return_type = analyse_type_annotation(self.return_type_annotation, env) + _, return_type = self.return_type_annotation.analyse_type_annotation(env) if return_type and return_type.is_pyobject: self.return_type = return_type @@ -2938,9 +3078,6 @@ class DefNode(FuncDefNode): arg.name = name_declarator.name arg.type = type - if type.is_fused: - self.has_fused_arguments = True - self.align_argument_type(env, arg) if name_declarator and name_declarator.cname: error(self.pos, "Python function argument cannot have C name specification") @@ -2966,11 +3103,14 @@ class DefNode(FuncDefNode): # probably just a plain 'object' arg.accept_none = True else: - arg.accept_none = True # won't be used, but must be there + arg.accept_none = True # won't be used, but must be there if arg.not_none: error(arg.pos, "Only Python type arguments can have 'not None'") if arg.or_none: error(arg.pos, "Only Python type arguments can have 'or None'") + + if arg.type.is_fused: + self.has_fused_arguments = True env.fused_to_specific = f2s if has_np_pythran(env): @@ -2983,8 +3123,10 @@ class DefNode(FuncDefNode): if self.decorators: error(self.pos, "special functions of cdef classes cannot have decorators") self.entry.trivial_signature = len(self.args) == 1 and not (self.star_arg or self.starstar_arg) - elif not env.directives['always_allow_keywords'] and not (self.star_arg or self.starstar_arg): - # Use the simpler calling signature for zero- and one-argument functions. + elif not (self.star_arg or self.starstar_arg) and ( + not env.directives['always_allow_keywords'] + or all([arg.pos_only for arg in self.args])): + # Use the simpler calling signature for zero- and one-argument pos-only functions. if self.entry.signature is TypeSlots.pyfunction_signature: if len(self.args) == 0: self.entry.signature = TypeSlots.pyfunction_noargs @@ -3005,7 +3147,7 @@ class DefNode(FuncDefNode): # this is the only case where a diverging number of # arguments is not an error - when we have no explicit # 'self' parameter as in method(*args) - sig = self.entry.signature = TypeSlots.pyfunction_signature # self is not 'really' used + sig = self.entry.signature = TypeSlots.pyfunction_signature # self is not 'really' used self.self_in_stararg = 1 nfixed = 0 @@ -3040,10 +3182,6 @@ class DefNode(FuncDefNode): arg.needs_type_test = 1 else: arg.needs_conversion = 1 - if arg.needs_conversion: - arg.hdr_cname = Naming.arg_prefix + arg.name - else: - arg.hdr_cname = Naming.var_prefix + arg.name if nfixed > len(self.args): self.bad_signature() @@ -3055,6 +3193,31 @@ class DefNode(FuncDefNode): if arg.is_generic and (arg.type.is_extension_type or arg.type.is_builtin_type): arg.needs_type_test = 1 + # Decide whether to use METH_FASTCALL + # 1. If we use METH_NOARGS or METH_O, keep that. We can only change + # METH_VARARGS to METH_FASTCALL + # 2. Special methods like __call__ always use the METH_VARGARGS + # calling convention + mf = sig.method_flags() + if mf and TypeSlots.method_varargs in mf and not self.entry.is_special: + # 3. If the function uses the full args tuple, it's more + # efficient to use METH_VARARGS. This happens when the function + # takes *args but no other positional arguments (apart from + # possibly self). We don't do the analogous check for keyword + # arguments since the kwargs dict is copied anyway. + if self.star_arg: + uses_args_tuple = True + for arg in self.args: + if (arg.is_generic and not arg.kw_only and + not arg.is_self_arg and not arg.is_type_arg): + # Other positional argument + uses_args_tuple = False + else: + uses_args_tuple = False + + if not uses_args_tuple: + sig = self.entry.signature = sig.with_fastcall() + def bad_signature(self): sig = self.entry.signature expected_str = "%d" % sig.num_fixed_args() @@ -3080,16 +3243,16 @@ class DefNode(FuncDefNode): entry = env.declare_pyfunction(name, self.pos, allow_redefine=not self.is_wrapper) self.entry = entry prefix = env.next_id(env.scope_prefix) - self.entry.pyfunc_cname = Naming.pyfunc_prefix + prefix + name + self.entry.pyfunc_cname = punycodify_name(Naming.pyfunc_prefix + prefix + name) if Options.docstrings: entry.doc = embed_position(self.pos, self.doc) - entry.doc_cname = Naming.funcdoc_prefix + prefix + name + entry.doc_cname = punycodify_name(Naming.funcdoc_prefix + prefix + name) if entry.is_special: if entry.name in TypeSlots.invisible or not entry.doc or ( entry.name in '__getattr__' and env.directives['fast_getattr']): entry.wrapperbase_cname = None else: - entry.wrapperbase_cname = Naming.wrapperbase_prefix + prefix + name + entry.wrapperbase_cname = punycodify_name(Naming.wrapperbase_prefix + prefix + name) else: entry.doc = None @@ -3132,8 +3295,6 @@ class DefNode(FuncDefNode): self.local_scope.directives = env.directives self.analyse_default_values(env) self.analyse_annotations(env) - if self.return_type_annotation: - self.return_type_annotation = self.analyse_annotation(env, self.return_type_annotation) if not self.needs_assignment_synthesis(env) and self.decorators: for decorator in self.decorators[::-1]: @@ -3257,10 +3418,11 @@ class DefNodeWrapper(FuncDefNode): # DefNode python wrapper code generator defnode = None - target = None # Target DefNode + target = None # Target DefNode def __init__(self, *args, **kwargs): FuncDefNode.__init__(self, *args, **kwargs) + self.num_posonly_args = self.target.num_posonly_args self.num_kwonly_args = self.target.num_kwonly_args self.num_required_kw_args = self.target.num_required_kw_args self.num_required_args = self.target.num_required_args @@ -3271,8 +3433,8 @@ class DefNodeWrapper(FuncDefNode): target_entry = self.target.entry name = self.name prefix = env.next_id(env.scope_prefix) - target_entry.func_cname = Naming.pywrap_prefix + prefix + name - target_entry.pymethdef_cname = Naming.pymethdef_prefix + prefix + name + target_entry.func_cname = punycodify_name(Naming.pywrap_prefix + prefix + name) + target_entry.pymethdef_cname = punycodify_name(Naming.pymethdef_prefix + prefix + name) self.signature = target_entry.signature @@ -3286,10 +3448,10 @@ class DefNodeWrapper(FuncDefNode): for arg in self.args: if not arg.type.is_pyobject: if not arg.type.create_from_py_utility_code(env): - pass # will fail later + pass # will fail later elif arg.hdr_type and not arg.hdr_type.is_pyobject: if not arg.hdr_type.create_to_py_utility_code(env): - pass # will fail later + pass # will fail later if self.starstar_arg and not self.starstar_arg.entry.cf_used: # we will set the kwargs argument to NULL instead of a new dict @@ -3316,7 +3478,13 @@ class DefNodeWrapper(FuncDefNode): if self.signature.has_dummy_arg: args.append(Naming.self_cname) for arg in self.args: - if arg.hdr_type and not (arg.type.is_memoryviewslice or + if arg.hdr_type and arg.type.is_cpp_class: + # it's safe to move converted C++ types because they aren't + # used again afterwards + code.globalstate.use_utility_code( + UtilityCode.load_cached("MoveIfSupported", "CppSupport.cpp")) + args.append("__PYX_STD_MOVE_IF_SUPPORTED(%s)" % arg.entry.cname) + elif arg.hdr_type and not (arg.type.is_memoryviewslice or arg.type.is_struct or arg.type.is_complex): args.append(arg.type.cast_code(arg.entry.cname)) @@ -3360,7 +3528,7 @@ class DefNodeWrapper(FuncDefNode): self.return_type.declaration_code(Naming.retval_cname), retval_init)) code.put_declare_refcount_context() - code.put_setup_refcount_context('%s (wrapper)' % self.name) + code.put_setup_refcount_context(EncodedString('%s (wrapper)' % self.name)) self.generate_argument_parsing_code(lenv, code) self.generate_argument_type_tests(code) @@ -3387,7 +3555,10 @@ class DefNodeWrapper(FuncDefNode): code.put_label(code.return_label) for entry in lenv.var_entries: if entry.is_arg and entry.type.is_pyobject: - code.put_var_decref(entry) + if entry.xdecref_cleanup: + code.put_var_xdecref(entry) + else: + code.put_var_decref(entry) code.put_finish_refcount_context() if not self.return_type.is_void: @@ -3420,9 +3591,16 @@ class DefNodeWrapper(FuncDefNode): if entry.scope.is_c_class_scope and entry.name == "__ipow__": arg_code_list.append("CYTHON_UNUSED PyObject *unused") if sig.has_generic_args: - arg_code_list.append( - "PyObject *%s, PyObject *%s" % ( - Naming.args_cname, Naming.kwds_cname)) + varargs_args = "PyObject *%s, PyObject *%s" % ( + Naming.args_cname, Naming.kwds_cname) + if sig.use_fastcall: + fastcall_args = "PyObject *const *%s, Py_ssize_t %s, PyObject *%s" % ( + Naming.args_cname, Naming.nargs_cname, Naming.kwds_cname) + arg_code_list.append( + "\n#if CYTHON_METH_FASTCALL\n%s\n#else\n%s\n#endif\n" % ( + fastcall_args, varargs_args)) + else: + arg_code_list.append(varargs_args) arg_code = ", ".join(arg_code_list) # Prevent warning: unused function '__pyx_pw_5numpy_7ndarray_1__getbuffer__' @@ -3456,7 +3634,7 @@ class DefNodeWrapper(FuncDefNode): docstr = docstr.as_utf8_string() if not (entry.is_special and entry.name in ('__getbuffer__', '__releasebuffer__')): - code.putln('static char %s[] = %s;' % ( + code.putln('PyDoc_STRVAR(%s, %s);' % ( entry.doc_cname, docstr.as_c_string_literal())) @@ -3483,6 +3661,23 @@ class DefNodeWrapper(FuncDefNode): if entry.is_arg: code.put_var_declaration(entry) + # Assign nargs variable as len(args), but avoid an "unused" warning in the few cases where we don't need it. + if self.signature_has_generic_args(): + nargs_code = "CYTHON_UNUSED const Py_ssize_t %s = PyTuple_GET_SIZE(%s);" % ( + Naming.nargs_cname, Naming.args_cname) + if self.signature.use_fastcall: + code.putln("#if !CYTHON_METH_FASTCALL") + code.putln(nargs_code) + code.putln("#endif") + else: + code.putln(nargs_code) + + # Array containing the values of keyword arguments when using METH_FASTCALL. + code.globalstate.use_utility_code( + UtilityCode.load_cached("fastcall", "FunctionArguments.c")) + code.putln('CYTHON_UNUSED PyObject *const *%s = __Pyx_KwValues_%s(%s, %s);' % ( + Naming.kwvalues_cname, self.signature.fastvar, Naming.args_cname, Naming.nargs_cname)) + def generate_argument_parsing_code(self, env, code): # Generate fast equivalent of PyArg_ParseTuple call for # generic arguments, if any, including args/kwargs @@ -3506,6 +3701,8 @@ class DefNodeWrapper(FuncDefNode): elif not self.signature_has_nongeneric_args(): # func(*args) or func(**kw) or func(*args, **kw) + # possibly with a "self" argument but no other non-star + # arguments self.generate_stararg_copy_code(code) else: @@ -3541,10 +3738,9 @@ class DefNodeWrapper(FuncDefNode): if not self.star_arg: code.globalstate.use_utility_code( UtilityCode.load_cached("RaiseArgTupleInvalid", "FunctionArguments.c")) - code.putln("if (unlikely(PyTuple_GET_SIZE(%s) > 0)) {" % - Naming.args_cname) - code.put('__Pyx_RaiseArgtupleInvalid("%s", 1, 0, 0, PyTuple_GET_SIZE(%s)); return %s;' % ( - self.name, Naming.args_cname, self.error_value())) + code.putln("if (unlikely(%s > 0)) {" % Naming.nargs_cname) + code.put('__Pyx_RaiseArgtupleInvalid(%s, 1, 0, 0, %s); return %s;' % ( + self.name.as_c_string_literal(), Naming.nargs_cname, self.error_value())) code.putln("}") if self.starstar_arg: @@ -3553,69 +3749,70 @@ class DefNodeWrapper(FuncDefNode): else: kwarg_check = "%s" % Naming.kwds_cname else: - kwarg_check = "unlikely(%s) && unlikely(PyDict_Size(%s) > 0)" % ( - Naming.kwds_cname, Naming.kwds_cname) + kwarg_check = "unlikely(%s) && __Pyx_NumKwargs_%s(%s)" % ( + Naming.kwds_cname, self.signature.fastvar, Naming.kwds_cname) code.globalstate.use_utility_code( UtilityCode.load_cached("KeywordStringCheck", "FunctionArguments.c")) code.putln( - "if (%s && unlikely(!__Pyx_CheckKeywordStrings(%s, \"%s\", %d))) return %s;" % ( - kwarg_check, Naming.kwds_cname, self.name, + "if (%s && unlikely(!__Pyx_CheckKeywordStrings(%s, %s, %d))) return %s;" % ( + kwarg_check, Naming.kwds_cname, self.name.as_c_string_literal(), bool(self.starstar_arg), self.error_value())) if self.starstar_arg and self.starstar_arg.entry.cf_used: - if all(ref.node.allow_null for ref in self.starstar_arg.entry.cf_references): - code.putln("if (%s) {" % kwarg_check) - code.putln("%s = PyDict_Copy(%s); if (unlikely(!%s)) return %s;" % ( - self.starstar_arg.entry.cname, - Naming.kwds_cname, - self.starstar_arg.entry.cname, - self.error_value())) - code.put_gotref(self.starstar_arg.entry.cname) - code.putln("} else {") + code.putln("if (%s) {" % kwarg_check) + code.putln("%s = __Pyx_KwargsAsDict_%s(%s, %s);" % ( + self.starstar_arg.entry.cname, + self.signature.fastvar, + Naming.kwds_cname, + Naming.kwvalues_cname)) + code.putln("if (unlikely(!%s)) return %s;" % ( + self.starstar_arg.entry.cname, self.error_value())) + code.put_gotref(self.starstar_arg.entry.cname, py_object_type) + code.putln("} else {") + allow_null = all(ref.node.allow_null for ref in self.starstar_arg.entry.cf_references) + if allow_null: code.putln("%s = NULL;" % (self.starstar_arg.entry.cname,)) - code.putln("}") - self.starstar_arg.entry.xdecref_cleanup = 1 else: - code.put("%s = (%s) ? PyDict_Copy(%s) : PyDict_New(); " % ( - self.starstar_arg.entry.cname, - Naming.kwds_cname, - Naming.kwds_cname)) + code.putln("%s = PyDict_New();" % (self.starstar_arg.entry.cname,)) code.putln("if (unlikely(!%s)) return %s;" % ( self.starstar_arg.entry.cname, self.error_value())) - self.starstar_arg.entry.xdecref_cleanup = 0 - code.put_gotref(self.starstar_arg.entry.cname) + code.put_var_gotref(self.starstar_arg.entry) + self.starstar_arg.entry.xdecref_cleanup = allow_null + code.putln("}") if self.self_in_stararg and not self.target.is_staticmethod: + assert not self.signature.use_fastcall # need to create a new tuple with 'self' inserted as first item - code.put("%s = PyTuple_New(PyTuple_GET_SIZE(%s)+1); if (unlikely(!%s)) " % ( + code.put("%s = PyTuple_New(%s + 1); if (unlikely(!%s)) " % ( self.star_arg.entry.cname, - Naming.args_cname, + Naming.nargs_cname, self.star_arg.entry.cname)) if self.starstar_arg and self.starstar_arg.entry.cf_used: code.putln("{") - code.put_xdecref_clear(self.starstar_arg.entry.cname, py_object_type) + code.put_var_xdecref_clear(self.starstar_arg.entry) code.putln("return %s;" % self.error_value()) code.putln("}") else: code.putln("return %s;" % self.error_value()) - code.put_gotref(self.star_arg.entry.cname) + code.put_var_gotref(self.star_arg.entry) code.put_incref(Naming.self_cname, py_object_type) - code.put_giveref(Naming.self_cname) + code.put_giveref(Naming.self_cname, py_object_type) code.putln("PyTuple_SET_ITEM(%s, 0, %s);" % ( self.star_arg.entry.cname, Naming.self_cname)) temp = code.funcstate.allocate_temp(PyrexTypes.c_py_ssize_t_type, manage_ref=False) - code.putln("for (%s=0; %s < PyTuple_GET_SIZE(%s); %s++) {" % ( - temp, temp, Naming.args_cname, temp)) + code.putln("for (%s=0; %s < %s; %s++) {" % ( + temp, temp, Naming.nargs_cname, temp)) code.putln("PyObject* item = PyTuple_GET_ITEM(%s, %s);" % ( Naming.args_cname, temp)) code.put_incref("item", py_object_type) - code.put_giveref("item") + code.put_giveref("item", py_object_type) code.putln("PyTuple_SET_ITEM(%s, %s+1, item);" % ( self.star_arg.entry.cname, temp)) code.putln("}") code.funcstate.release_temp(temp) self.star_arg.entry.xdecref_cleanup = 0 elif self.star_arg: + assert not self.signature.use_fastcall code.put_incref(Naming.args_cname, py_object_type) code.putln("%s = %s;" % ( self.star_arg.entry.cname, @@ -3623,11 +3820,17 @@ class DefNodeWrapper(FuncDefNode): self.star_arg.entry.xdecref_cleanup = 0 def generate_tuple_and_keyword_parsing_code(self, args, success_label, code): + code.globalstate.use_utility_code( + UtilityCode.load_cached("fastcall", "FunctionArguments.c")) + + self_name_csafe = self.name.as_c_string_literal() + argtuple_error_label = code.new_label("argtuple_error") positional_args = [] required_kw_only_args = [] optional_kw_only_args = [] + num_pos_only_args = 0 for arg in args: if arg.is_generic: if arg.default: @@ -3640,6 +3843,8 @@ class DefNodeWrapper(FuncDefNode): required_kw_only_args.append(arg) elif not arg.is_self_arg and not arg.is_type_arg: positional_args.append(arg) + if arg.pos_only: + num_pos_only_args += 1 # sort required kw-only args before optional ones to avoid special # cases in the unpacking code @@ -3658,10 +3863,18 @@ class DefNodeWrapper(FuncDefNode): code.putln('{') all_args = tuple(positional_args) + tuple(kw_only_args) - code.putln("static PyObject **%s[] = {%s,0};" % ( + non_posonly_args = [arg for arg in all_args if not arg.pos_only] + non_pos_args_id = ','.join( + ['&%s' % code.intern_identifier(arg.entry.name) for arg in non_posonly_args] + ['0']) + code.putln("#if CYTHON_COMPILING_IN_LIMITED_API") + code.putln("PyObject **%s[] = {%s};" % ( Naming.pykwdlist_cname, - ','.join(['&%s' % code.intern_identifier(arg.name) - for arg in all_args]))) + non_pos_args_id)) + code.putln("#else") + code.putln("static PyObject **%s[] = {%s};" % ( + Naming.pykwdlist_cname, + non_pos_args_id)) + code.putln("#endif") # Before being converted and assigned to the target variables, # borrowed references to all unpacked argument values are @@ -3673,14 +3886,43 @@ class DefNodeWrapper(FuncDefNode): # was passed for them. self.generate_argument_values_setup_code(all_args, code) + # If all args are positional-only, we can raise an error + # straight away if we receive a non-empty kw-dict. + # This requires a PyDict_Size call. This call is wasteful + # for functions which do accept kw-args, so we do not generate + # the PyDict_Size call unless all args are positional-only. + accept_kwd_args = non_posonly_args or self.starstar_arg + if accept_kwd_args: + kw_unpacking_condition = Naming.kwds_cname + else: + kw_unpacking_condition = "%s && __Pyx_NumKwargs_%s(%s) > 0" % ( + Naming.kwds_cname, self.signature.fastvar, Naming.kwds_cname) + + if self.num_required_kw_args > 0: + kw_unpacking_condition = "likely(%s)" % kw_unpacking_condition + # --- optimised code when we receive keyword arguments - code.putln("if (%s(%s)) {" % ( - (self.num_required_kw_args > 0) and "likely" or "unlikely", - Naming.kwds_cname)) - self.generate_keyword_unpacking_code( - min_positional_args, max_positional_args, - has_fixed_positional_count, has_kw_only_args, - all_args, argtuple_error_label, code) + code.putln("if (%s) {" % kw_unpacking_condition) + + if accept_kwd_args: + self.generate_keyword_unpacking_code( + min_positional_args, max_positional_args, + has_fixed_positional_count, has_kw_only_args, all_args, argtuple_error_label, code) + else: + # Here we do not accept kw-args but we are passed a non-empty kw-dict. + # We call ParseOptionalKeywords which will raise an appropriate error if + # the kw-args dict passed is non-empty (which it will be, since kw_unpacking_condition is true) + code.globalstate.use_utility_code( + UtilityCode.load_cached("ParseKeywords", "FunctionArguments.c")) + code.putln('if (likely(__Pyx_ParseOptionalKeywords(%s, %s, %s, %s, %s, %s, %s) < 0)) %s' % ( + Naming.kwds_cname, + Naming.kwvalues_cname, + Naming.pykwdlist_cname, + self.starstar_arg.entry.cname if self.starstar_arg else 0, + 'values', + 0, + self_name_csafe, + code.error_goto(self.pos))) # --- optimised code when we do not receive any keyword arguments if (self.num_required_kw_args and min_positional_args > 0) or min_positional_args == max_positional_args: @@ -3690,20 +3932,20 @@ class DefNodeWrapper(FuncDefNode): compare = '!=' else: compare = '<' - code.putln('} else if (PyTuple_GET_SIZE(%s) %s %d) {' % ( - Naming.args_cname, compare, min_positional_args)) + code.putln('} else if (unlikely(%s %s %d)) {' % ( + Naming.nargs_cname, compare, min_positional_args)) code.put_goto(argtuple_error_label) if self.num_required_kw_args: # pure error case: keywords required but not passed if max_positional_args > min_positional_args and not self.star_arg: - code.putln('} else if (PyTuple_GET_SIZE(%s) > %d) {' % ( - Naming.args_cname, max_positional_args)) + code.putln('} else if (unlikely(%s > %d)) {' % ( + Naming.nargs_cname, max_positional_args)) code.put_goto(argtuple_error_label) code.putln('} else {') for i, arg in enumerate(kw_only_args): if not arg.default: - pystring_cname = code.intern_identifier(arg.name) + pystring_cname = code.intern_identifier(arg.entry.name) # required keyword-only argument missing code.globalstate.use_utility_code( UtilityCode.load_cached("RaiseKeywordRequired", "FunctionArguments.c")) @@ -3720,11 +3962,12 @@ class DefNodeWrapper(FuncDefNode): # parse the exact number of positional arguments from # the args tuple for i, arg in enumerate(positional_args): - code.putln("values[%d] = PyTuple_GET_ITEM(%s, %d);" % (i, Naming.args_cname, i)) + code.putln("values[%d] = __Pyx_Arg_%s(%s, %d);" % ( + i, self.signature.fastvar, Naming.args_cname, i)) else: # parse the positional arguments from the variable length # args tuple and reject illegal argument tuple sizes - code.putln('switch (PyTuple_GET_SIZE(%s)) {' % Naming.args_cname) + code.putln('switch (%s) {' % Naming.nargs_cname) if self.star_arg: code.putln('default:') reversed_args = list(enumerate(positional_args))[::-1] @@ -3733,7 +3976,8 @@ class DefNodeWrapper(FuncDefNode): if i != reversed_args[0][0]: code.putln('CYTHON_FALLTHROUGH;') code.put('case %2d: ' % (i+1)) - code.putln("values[%d] = PyTuple_GET_ITEM(%s, %d);" % (i, Naming.args_cname, i)) + code.putln("values[%d] = __Pyx_Arg_%s(%s, %d);" % ( + i, self.signature.fastvar, Naming.args_cname, i)) if min_positional_args == 0: code.putln('CYTHON_FALLTHROUGH;') code.put('case 0: ') @@ -3748,7 +3992,7 @@ class DefNodeWrapper(FuncDefNode): code.put_goto(argtuple_error_label) code.putln('}') - code.putln('}') # end of the conditional unpacking blocks + code.putln('}') # end of the conditional unpacking blocks # Convert arg values to their final type and assign them. # Also inject non-Python default arguments, which do cannot @@ -3756,17 +4000,17 @@ class DefNodeWrapper(FuncDefNode): for i, arg in enumerate(all_args): self.generate_arg_assignment(arg, "values[%d]" % i, code) - code.putln('}') # end of the whole argument unpacking block + code.putln('}') # end of the whole argument unpacking block if code.label_used(argtuple_error_label): code.put_goto(success_label) code.put_label(argtuple_error_label) code.globalstate.use_utility_code( UtilityCode.load_cached("RaiseArgTupleInvalid", "FunctionArguments.c")) - code.put('__Pyx_RaiseArgtupleInvalid("%s", %d, %d, %d, PyTuple_GET_SIZE(%s)); ' % ( - self.name, has_fixed_positional_count, + code.put('__Pyx_RaiseArgtupleInvalid(%s, %d, %d, %d, %s); ' % ( + self_name_csafe, has_fixed_positional_count, min_positional_args, max_positional_args, - Naming.args_cname)) + Naming.nargs_cname)) code.putln(code.error_goto(self.pos)) def generate_arg_assignment(self, arg, item, code): @@ -3789,8 +4033,7 @@ class DefNodeWrapper(FuncDefNode): arg.entry.cname, arg.calculate_default_value_code(code))) if arg.type.is_memoryviewslice: - code.put_incref_memoryviewslice(arg.entry.cname, - have_gil=True) + code.put_var_incref_memoryviewslice(arg.entry, have_gil=True) code.putln('}') else: error(arg.pos, "Cannot convert Python object argument to type '%s'" % arg.type) @@ -3802,26 +4045,30 @@ class DefNodeWrapper(FuncDefNode): self.starstar_arg.entry.cname, self.starstar_arg.entry.cname, self.error_value())) - code.put_gotref(self.starstar_arg.entry.cname) + code.put_var_gotref(self.starstar_arg.entry) if self.star_arg: self.star_arg.entry.xdecref_cleanup = 0 - code.putln('if (PyTuple_GET_SIZE(%s) > %d) {' % ( - Naming.args_cname, - max_positional_args)) - code.putln('%s = PyTuple_GetSlice(%s, %d, PyTuple_GET_SIZE(%s));' % ( - self.star_arg.entry.cname, Naming.args_cname, - max_positional_args, Naming.args_cname)) - code.putln("if (unlikely(!%s)) {" % self.star_arg.entry.cname) - if self.starstar_arg: - code.put_decref_clear(self.starstar_arg.entry.cname, py_object_type) - code.put_finish_refcount_context() - code.putln('return %s;' % self.error_value()) - code.putln('}') - code.put_gotref(self.star_arg.entry.cname) - code.putln('} else {') - code.put("%s = %s; " % (self.star_arg.entry.cname, Naming.empty_tuple)) - code.put_incref(Naming.empty_tuple, py_object_type) - code.putln('}') + if max_positional_args == 0: + # If there are no positional arguments, use the args tuple + # directly + assert not self.signature.use_fastcall + code.put_incref(Naming.args_cname, py_object_type) + code.putln("%s = %s;" % (self.star_arg.entry.cname, Naming.args_cname)) + else: + # It is possible that this is a slice of "negative" length, + # as in args[5:3]. That's not a problem, the function below + # handles that efficiently and returns the empty tuple. + code.putln('%s = __Pyx_ArgsSlice_%s(%s, %d, %s);' % ( + self.star_arg.entry.cname, self.signature.fastvar, + Naming.args_cname, max_positional_args, Naming.nargs_cname)) + code.putln("if (unlikely(!%s)) {" % + self.star_arg.entry.type.nullcheck_string(self.star_arg.entry.cname)) + if self.starstar_arg: + code.put_var_decref_clear(self.starstar_arg.entry) + code.put_finish_refcount_context() + code.putln('return %s;' % self.error_value()) + code.putln('}') + code.put_var_gotref(self.star_arg.entry) def generate_argument_values_setup_code(self, args, code): max_args = len(args) @@ -3843,22 +4090,45 @@ class DefNodeWrapper(FuncDefNode): code.putln('values[%d] = %s;' % (i, arg.type.as_pyobject(default_value))) def generate_keyword_unpacking_code(self, min_positional_args, max_positional_args, - has_fixed_positional_count, has_kw_only_args, - all_args, argtuple_error_label, code): + has_fixed_positional_count, + has_kw_only_args, all_args, argtuple_error_label, code): + # First we count how many arguments must be passed as positional + num_required_posonly_args = num_pos_only_args = 0 + for i, arg in enumerate(all_args): + if arg.pos_only: + num_pos_only_args += 1 + if not arg.default: + num_required_posonly_args += 1 + code.putln('Py_ssize_t kw_args;') - code.putln('const Py_ssize_t pos_args = PyTuple_GET_SIZE(%s);' % Naming.args_cname) # copy the values from the args tuple and check that it's not too long - code.putln('switch (pos_args) {') + code.putln('switch (%s) {' % Naming.nargs_cname) if self.star_arg: code.putln('default:') - for i in range(max_positional_args-1, -1, -1): + + for i in range(max_positional_args-1, num_required_posonly_args-1, -1): + code.put('case %2d: ' % (i+1)) + code.putln("values[%d] = __Pyx_Arg_%s(%s, %d);" % ( + i, self.signature.fastvar, Naming.args_cname, i)) + code.putln('CYTHON_FALLTHROUGH;') + if num_required_posonly_args > 0: + code.put('case %2d: ' % num_required_posonly_args) + for i in range(num_required_posonly_args-1, -1, -1): + code.putln("values[%d] = __Pyx_Arg_%s(%s, %d);" % ( + i, self.signature.fastvar, Naming.args_cname, i)) + code.putln('break;') + for i in range(num_required_posonly_args-2, -1, -1): code.put('case %2d: ' % (i+1)) - code.putln("values[%d] = PyTuple_GET_ITEM(%s, %d);" % ( - i, Naming.args_cname, i)) code.putln('CYTHON_FALLTHROUGH;') - code.putln('case 0: break;') + + code.put('case 0: ') + if num_required_posonly_args == 0: + code.putln('break;') + else: + # catch-all for not enough pos-only args passed + code.put_goto(argtuple_error_label) if not self.star_arg: - code.put('default: ') # more arguments than allowed + code.put('default: ') # more arguments than allowed code.put_goto(argtuple_error_label) code.putln('}') @@ -3871,7 +4141,10 @@ class DefNodeWrapper(FuncDefNode): # If we received kwargs, fill up the positional/required # arguments with values from the kw dict - code.putln('kw_args = PyDict_Size(%s);' % Naming.kwds_cname) + self_name_csafe = self.name.as_c_string_literal() + + code.putln('kw_args = __Pyx_NumKwargs_%s(%s);' % ( + self.signature.fastvar, Naming.kwds_cname)) if self.num_required_args or max_positional_args > 0: last_required_arg = -1 for i, arg in enumerate(all_args): @@ -3879,30 +4152,32 @@ class DefNodeWrapper(FuncDefNode): last_required_arg = i if last_required_arg < max_positional_args: last_required_arg = max_positional_args-1 - if max_positional_args > 0: - code.putln('switch (pos_args) {') - for i, arg in enumerate(all_args[:last_required_arg+1]): - if max_positional_args > 0 and i <= max_positional_args: - if i != 0: + if max_positional_args > num_pos_only_args: + code.putln('switch (%s) {' % Naming.nargs_cname) + for i, arg in enumerate(all_args[num_pos_only_args:last_required_arg+1], num_pos_only_args): + if max_positional_args > num_pos_only_args and i <= max_positional_args: + if i != num_pos_only_args: code.putln('CYTHON_FALLTHROUGH;') if self.star_arg and i == max_positional_args: code.putln('default:') else: code.putln('case %2d:' % i) - pystring_cname = code.intern_identifier(arg.name) + pystring_cname = code.intern_identifier(arg.entry.name) if arg.default: if arg.kw_only: # optional kw-only args are handled separately below continue code.putln('if (kw_args > 0) {') # don't overwrite default argument - code.putln('PyObject* value = __Pyx_PyDict_GetItemStr(%s, %s);' % ( - Naming.kwds_cname, pystring_cname)) + code.putln('PyObject* value = __Pyx_GetKwValue_%s(%s, %s, %s);' % ( + self.signature.fastvar, Naming.kwds_cname, Naming.kwvalues_cname, pystring_cname)) code.putln('if (value) { values[%d] = value; kw_args--; }' % i) + code.putln('else if (unlikely(PyErr_Occurred())) %s' % code.error_goto(self.pos)) code.putln('}') else: - code.putln('if (likely((values[%d] = __Pyx_PyDict_GetItemStr(%s, %s)) != 0)) kw_args--;' % ( - i, Naming.kwds_cname, pystring_cname)) + code.putln('if (likely((values[%d] = __Pyx_GetKwValue_%s(%s, %s, %s)) != 0)) kw_args--;' % ( + i, self.signature.fastvar, Naming.kwds_cname, Naming.kwvalues_cname, pystring_cname)) + code.putln('else if (unlikely(PyErr_Occurred())) %s' % code.error_goto(self.pos)) if i < min_positional_args: if i == 0: # special case: we know arg 0 is missing @@ -3915,8 +4190,8 @@ class DefNodeWrapper(FuncDefNode): code.putln('else {') code.globalstate.use_utility_code( UtilityCode.load_cached("RaiseArgTupleInvalid", "FunctionArguments.c")) - code.put('__Pyx_RaiseArgtupleInvalid("%s", %d, %d, %d, %d); ' % ( - self.name, has_fixed_positional_count, + code.put('__Pyx_RaiseArgtupleInvalid(%s, %d, %d, %d, %d); ' % ( + self_name_csafe, has_fixed_positional_count, min_positional_args, max_positional_args, i)) code.putln(code.error_goto(self.pos)) code.putln('}') @@ -3924,11 +4199,11 @@ class DefNodeWrapper(FuncDefNode): code.putln('else {') code.globalstate.use_utility_code( UtilityCode.load_cached("RaiseKeywordRequired", "FunctionArguments.c")) - code.put('__Pyx_RaiseKeywordRequired("%s", %s); ' % ( - self.name, pystring_cname)) + code.put('__Pyx_RaiseKeywordRequired(%s, %s); ' % ( + self_name_csafe, pystring_cname)) code.putln(code.error_goto(self.pos)) code.putln('}') - if max_positional_args > 0: + if max_positional_args > num_pos_only_args: code.putln('}') if has_kw_only_args: @@ -3944,34 +4219,69 @@ class DefNodeWrapper(FuncDefNode): # arguments, this will always do the right thing for unpacking # keyword arguments, so that we can concentrate on optimising # common cases above. + # + # ParseOptionalKeywords() needs to know how many of the arguments + # that could be passed as keywords have in fact been passed as + # positional args. + if num_pos_only_args > 0: + # There are positional-only arguments which we don't want to count, + # since they cannot be keyword arguments. Subtract the number of + # pos-only arguments from the number of positional arguments we got. + # If we get a negative number then none of the keyword arguments were + # passed as positional args. + code.putln('const Py_ssize_t kwd_pos_args = (unlikely(%s < %d)) ? 0 : %s - %d;' % ( + Naming.nargs_cname, num_pos_only_args, + Naming.nargs_cname, num_pos_only_args, + )) + elif max_positional_args > 0: + code.putln('const Py_ssize_t kwd_pos_args = %s;' % Naming.nargs_cname) + if max_positional_args == 0: pos_arg_count = "0" elif self.star_arg: - code.putln("const Py_ssize_t used_pos_args = (pos_args < %d) ? pos_args : %d;" % ( - max_positional_args, max_positional_args)) + # If there is a *arg, the number of used positional args could be larger than + # the number of possible keyword arguments. But ParseOptionalKeywords() uses the + # number of positional args as an index into the keyword argument name array, + # if this is larger than the number of kwd args we get a segfault. So round + # this down to max_positional_args - num_pos_only_args (= num possible kwd args). + code.putln("const Py_ssize_t used_pos_args = (kwd_pos_args < %d) ? kwd_pos_args : %d;" % ( + max_positional_args - num_pos_only_args, max_positional_args - num_pos_only_args)) pos_arg_count = "used_pos_args" else: - pos_arg_count = "pos_args" + pos_arg_count = "kwd_pos_args" + if num_pos_only_args < len(all_args): + values_array = 'values + %d' % num_pos_only_args + else: + values_array = 'values' code.globalstate.use_utility_code( UtilityCode.load_cached("ParseKeywords", "FunctionArguments.c")) - code.putln('if (unlikely(__Pyx_ParseOptionalKeywords(%s, %s, %s, values, %s, "%s") < 0)) %s' % ( + code.putln('if (unlikely(__Pyx_ParseOptionalKeywords(%s, %s, %s, %s, %s, %s, %s) < 0)) %s' % ( Naming.kwds_cname, + Naming.kwvalues_cname, Naming.pykwdlist_cname, self.starstar_arg and self.starstar_arg.entry.cname or '0', + values_array, pos_arg_count, - self.name, + self_name_csafe, code.error_goto(self.pos))) code.putln('}') def generate_optional_kwonly_args_unpacking_code(self, all_args, code): optional_args = [] first_optional_arg = -1 + num_posonly_args = 0 for i, arg in enumerate(all_args): + if arg.pos_only: + num_posonly_args += 1 if not arg.kw_only or not arg.default: continue if not optional_args: first_optional_arg = i optional_args.append(arg.name) + if num_posonly_args > 0: + posonly_correction = '-%d' % num_posonly_args + else: + posonly_correction = '' if optional_args: if len(optional_args) > 1: # if we receive more than the named kwargs, we either have **kwargs @@ -3987,9 +4297,14 @@ class DefNodeWrapper(FuncDefNode): else: code.putln('if (kw_args == 1) {') code.putln('const Py_ssize_t index = %d;' % first_optional_arg) - code.putln('PyObject* value = __Pyx_PyDict_GetItemStr(%s, *%s[index]);' % ( - Naming.kwds_cname, Naming.pykwdlist_cname)) + code.putln('PyObject* value = __Pyx_GetKwValue_%s(%s, %s, *%s[index%s]);' % ( + self.signature.fastvar, + Naming.kwds_cname, + Naming.kwvalues_cname, + Naming.pykwdlist_cname, + posonly_correction)) code.putln('if (value) { values[index] = value; kw_args--; }') + code.putln('else if (unlikely(PyErr_Occurred())) %s' % code.error_goto(self.pos)) if len(optional_args) > 1: code.putln('}') code.putln('}') @@ -4070,9 +4385,7 @@ class GeneratorDefNode(DefNode): # is_generator = True - is_coroutine = False is_iterable_coroutine = False - is_asyncgen = False gen_type_name = 'Generator' needs_closure = True @@ -4107,7 +4420,7 @@ class GeneratorDefNode(DefNode): code.putln('%s = __Pyx_CyFunction_GetClassObj(%s);' % ( classobj_cname, Naming.self_cname)) code.put_incref(classobj_cname, py_object_type) - code.put_giveref(classobj_cname) + code.put_giveref(classobj_cname, py_object_type) code.put_finish_refcount_context() code.putln('return (PyObject *) gen;') code.putln('}') @@ -4227,7 +4540,7 @@ class GeneratorBodyDefNode(DefNode): code.putln("%s = %s; %s" % ( Naming.retval_cname, comp_init, code.error_goto_if_null(Naming.retval_cname, self.pos))) - code.put_gotref(Naming.retval_cname) + code.put_gotref(Naming.retval_cname, py_object_type) # ----- Function body self.generate_function_body(env, code) @@ -4273,7 +4586,7 @@ class GeneratorBodyDefNode(DefNode): # ----- Non-error return cleanup code.put_label(code.return_label) if self.is_inlined: - code.put_xgiveref(Naming.retval_cname) + code.put_xgiveref(Naming.retval_cname, py_object_type) else: code.put_xdecref_clear(Naming.retval_cname, py_object_type) # For Py3.7, clearing is already done below. @@ -4362,8 +4675,8 @@ class OverrideCheckNode(StatNode): if self.py_func.is_module_scope: code.putln("else {") else: - code.putln("else if (unlikely((Py_TYPE(%s)->tp_dictoffset != 0)" - " || (Py_TYPE(%s)->tp_flags & (Py_TPFLAGS_IS_ABSTRACT | Py_TPFLAGS_HEAPTYPE)))) {" % ( + code.putln("else if (unlikely((Py_TYPE(%s)->tp_dictoffset != 0) || " + "__Pyx_PyType_HasFeature(Py_TYPE(%s), (Py_TPFLAGS_IS_ABSTRACT | Py_TPFLAGS_HEAPTYPE)))) {" % ( self_arg, self_arg)) code.putln("#if CYTHON_USE_DICT_VERSIONS && CYTHON_USE_PYTYPE_LOOKUP && CYTHON_USE_TYPE_SLOTS") @@ -4389,7 +4702,7 @@ class OverrideCheckNode(StatNode): err = code.error_goto_if_null(func_node_temp, self.pos) code.putln("%s = __Pyx_PyObject_GetAttrStr(%s, %s); %s" % ( func_node_temp, self_arg, interned_attr_cname, err)) - code.put_gotref(func_node_temp) + code.put_gotref(func_node_temp, py_object_type) is_builtin_function_or_method = "PyCFunction_Check(%s)" % func_node_temp is_overridden = "(PyCFunction_GET_FUNCTION(%s) != (PyCFunction)(void*)%s)" % ( @@ -4436,25 +4749,31 @@ class PyClassDefNode(ClassDefNode): # A Python class definition. # # name EncodedString Name of the class - # doc string or None + # doc string or None The class docstring # body StatNode Attribute definition code # entry Symtab.Entry # scope PyClassScope # decorators [DecoratorNode] list of decorators or None + # bases ExprNode Expression that evaluates to a tuple of base classes # # The following subnodes are constructed internally: # + # doc_node NameNode '__doc__' name that is made available to the class body # dict DictNode Class dictionary or Py3 namespace # classobj ClassNode Class object # target NameNode Variable to assign class object to + # orig_bases None or ExprNode "bases" before transformation by PEP560 __mro_entries__, + # used to create the __orig_bases__ attribute - child_attrs = ["body", "dict", "metaclass", "mkw", "bases", "class_result", - "target", "class_cell", "decorators"] + child_attrs = ["doc_node", "body", "dict", "metaclass", "mkw", "bases", "class_result", + "target", "class_cell", "decorators", "orig_bases"] decorators = None class_result = None is_py3_style_class = False # Python3 style class (kwargs) metaclass = None mkw = None + doc_node = None + orig_bases = None def __init__(self, pos, name, bases, doc, body, decorators=None, keyword_args=None, force_py3_semantics=False): @@ -4468,6 +4787,7 @@ class PyClassDefNode(ClassDefNode): if self.doc and Options.docstrings: doc = embed_position(self.pos, self.doc) doc_node = ExprNodes.StringNode(pos, value=doc) + self.doc_node = ExprNodes.NameNode(name=EncodedString('__doc__'), type=py_object_type, pos=pos) else: doc_node = None @@ -4516,7 +4836,9 @@ class PyClassDefNode(ClassDefNode): self.classobj = ExprNodes.Py3ClassNode( pos, name=name, class_def_node=self, doc=doc_node, calculate_metaclass=needs_metaclass_calculation, - allow_py2_metaclass=allow_py2_metaclass) + allow_py2_metaclass=allow_py2_metaclass, + force_type=force_py3_semantics, + ) else: # no bases, no metaclass => old style class creation self.dict = ExprNodes.DictNode(pos, key_value_pairs=[]) @@ -4572,9 +4894,27 @@ class PyClassDefNode(ClassDefNode): cenv = self.create_scope(env) cenv.directives = env.directives cenv.class_obj_cname = self.target.entry.cname + if self.doc_node: + self.doc_node.analyse_target_declaration(cenv) self.body.analyse_declarations(cenv) + self.class_result.analyse_annotations(cenv) + + update_bases_functype = PyrexTypes.CFuncType( + PyrexTypes.py_object_type, [ + PyrexTypes.CFuncTypeArg("bases", PyrexTypes.py_object_type, None) + ]) def analyse_expressions(self, env): + if self.bases and not (self.bases.is_sequence_constructor and len(self.bases.args) == 0): + from .ExprNodes import PythonCapiCallNode, CloneNode + # handle the Python 3.7 __mro_entries__ transformation + orig_bases = self.bases.analyse_expressions(env) + self.bases = PythonCapiCallNode(orig_bases.pos, + function_name="__Pyx_PEP560_update_bases", + func_type=self.update_bases_functype, + utility_code=UtilityCode.load_cached('Py3UpdateBases', 'ObjectHandling.c'), + args=[CloneNode(orig_bases)]) + self.orig_bases = orig_bases if self.bases: self.bases = self.bases.analyse_expressions(env) if self.mkw: @@ -4597,6 +4937,8 @@ class PyClassDefNode(ClassDefNode): code.mark_pos(self.pos) code.pyclass_stack.append(self) cenv = self.scope + if self.orig_bases: + self.orig_bases.generate_evaluation_code(code) if self.bases: self.bases.generate_evaluation_code(code) if self.mkw: @@ -4604,6 +4946,17 @@ class PyClassDefNode(ClassDefNode): if self.metaclass: self.metaclass.generate_evaluation_code(code) self.dict.generate_evaluation_code(code) + if self.orig_bases: + # update __orig_bases__ if needed + code.putln("if (%s != %s) {" % (self.bases.result(), self.orig_bases.result())) + code.putln( + code.error_goto_if_neg('PyDict_SetItemString(%s, "__orig_bases__", %s)' % ( + self.dict.result(), self.orig_bases.result()), + self.pos + )) + code.putln("}") + self.orig_bases.generate_disposal_code(code) + self.orig_bases.free_temps(code) cenv.namespace_cname = cenv.class_obj_cname = self.dict.result() class_cell = self.class_cell @@ -4670,6 +5023,10 @@ class CClassDefNode(ClassDefNode): decorators = None shadow = False + @property + def punycode_class_name(self): + return punycodify_name(self.class_name) + def buffer_defaults(self, env): if not hasattr(self, '_buffer_defaults'): from . import Buffer @@ -4776,7 +5133,7 @@ class CClassDefNode(ClassDefNode): if self.visibility == 'extern': if (self.module_name == '__builtin__' and self.class_name in Builtin.builtin_types and - env.qualified_name[:8] != 'cpython.'): # allow overloaded names for cimporting from cpython + env.qualified_name[:8] != 'cpython.'): # allow overloaded names for cimporting from cpython warning(self.pos, "%s already a builtin Cython type" % self.class_name, 1) self.entry = home_scope.declare_c_class( @@ -4861,8 +5218,6 @@ class CClassDefNode(ClassDefNode): # This is needed to generate evaluation code for # default values of method arguments. code.mark_pos(self.pos) - if self.body: - self.body.generate_execution_code(code) if not self.entry.type.early_init: if self.type_init_args: self.type_init_args.generate_evaluation_code(code) @@ -4873,23 +5228,29 @@ class CClassDefNode(ClassDefNode): code.putln("%s = PyType_Type.tp_new(&PyType_Type, %s, NULL);" % ( trial_type, self.type_init_args.result())) code.putln(code.error_goto_if_null(trial_type, self.pos)) - code.put_gotref(trial_type) + code.put_gotref(trial_type, py_object_type) code.putln("if (((PyTypeObject*) %s)->tp_base != %s) {" % ( trial_type, first_base)) - code.putln("PyErr_Format(PyExc_TypeError, \"best base '%s' must be equal to first base '%s'\",") - code.putln(" ((PyTypeObject*) %s)->tp_base->tp_name, %s->tp_name);" % ( - trial_type, first_base)) + code.putln("__Pyx_TypeName base_name = __Pyx_PyType_GetName(((PyTypeObject*) %s)->tp_base);" % trial_type) + code.putln("__Pyx_TypeName type_name = __Pyx_PyType_GetName(%s);" % first_base) + code.putln("PyErr_Format(PyExc_TypeError, " + "\"best base '\" __Pyx_FMT_TYPENAME \"' must be equal to first base '\" __Pyx_FMT_TYPENAME \"'\",") + code.putln(" base_name, type_name);") + code.putln("__Pyx_DECREF_TypeName(base_name);") + code.putln("__Pyx_DECREF_TypeName(type_name);") code.putln(code.error_goto(self.pos)) code.putln("}") code.funcstate.release_temp(trial_type) code.put_incref(bases, PyrexTypes.py_object_type) - code.put_giveref(bases) + code.put_giveref(bases, py_object_type) code.putln("%s.tp_bases = %s;" % (self.entry.type.typeobj_cname, bases)) code.put_decref_clear(trial_type, PyrexTypes.py_object_type) self.type_init_args.generate_disposal_code(code) self.type_init_args.free_temps(code) self.generate_type_ready_code(self.entry, code, True) + if self.body: + self.body.generate_execution_code(code) # Also called from ModuleNode for early init types. @staticmethod @@ -4902,6 +5263,31 @@ class CClassDefNode(ClassDefNode): if not scope: # could be None if there was an error return if entry.visibility != 'extern': + code.putln("#if CYTHON_COMPILING_IN_LIMITED_API") + base_type = scope.parent_type.base_type + if base_type: + tuple_temp = code.funcstate.allocate_temp(py_object_type, manage_ref=True) + code.putln( + "%s = PyTuple_Pack(1, (PyObject *)%s); %s" % ( + tuple_temp, + base_type.typeptr_cname, + code.error_goto_if_null(tuple_temp, entry.pos))) + code.put_gotref(tuple_temp, py_object_type) + code.putln( + "%s = PyType_FromSpecWithBases(&%s_spec, %s); %s" % ( + typeobj_cname, + typeobj_cname, + tuple_temp, + code.error_goto_if_null(typeobj_cname, entry.pos))) + code.put_xdecref_clear(tuple_temp, type=py_object_type) + code.funcstate.release_temp(tuple_temp) + else: + code.putln( + "%s = PyType_FromSpec(&%s_spec); %s" % ( + typeobj_cname, + typeobj_cname, + code.error_goto_if_null(typeobj_cname, entry.pos))) + code.putln("#else") for slot in TypeSlots.slot_table: slot.generate_dynamic_init_code(scope, code) if heap_type_bases: @@ -4915,10 +5301,10 @@ class CClassDefNode(ClassDefNode): readyfunc, typeobj_cname, code.error_goto(entry.pos))) - # Don't inherit tp_print from builtin types, restoring the + # Don't inherit tp_print from builtin types in Python 2, restoring the # behavior of using tp_repr or tp_str instead. # ("tp_print" was renamed to "tp_vectorcall_offset" in Py3.8b1) - code.putln("#if PY_VERSION_HEX < 0x030800B1") + code.putln("#if PY_MAJOR_VERSION < 3") code.putln("%s.tp_print = 0;" % typeobj_cname) code.putln("#endif") @@ -4940,6 +5326,7 @@ class CClassDefNode(ClassDefNode): code.putln("%s.tp_getattro = %s;" % ( typeobj_cname, py_cfunc)) code.putln("}") + code.putln("#endif") # Fix special method docstrings. This is a bit of a hack, but # unless we let PyType_Ready create the slot wrappers we have @@ -4960,7 +5347,7 @@ class CClassDefNode(ClassDefNode): func.name, code.error_goto_if_null('wrapper', entry.pos))) code.putln( - "if (Py_TYPE(wrapper) == &PyWrapperDescr_Type) {") + "if (__Pyx_IS_TYPE(wrapper, &PyWrapperDescr_Type)) {") code.putln( "%s = *((PyWrapperDescrObject *)wrapper)->d_base;" % ( func.wrapperbase_cname)) @@ -4977,11 +5364,19 @@ class CClassDefNode(ClassDefNode): if type.vtable_cname: code.globalstate.use_utility_code( UtilityCode.load_cached('SetVTable', 'ImportExport.c')) + code.putln("#if CYTHON_COMPILING_IN_LIMITED_API") + code.putln( + "if (__Pyx_SetVtable(%s, %s) < 0) %s" % ( + typeobj_cname, + type.vtabptr_cname, + code.error_goto(entry.pos))) + code.putln("#else") code.putln( "if (__Pyx_SetVtable(%s.tp_dict, %s) < 0) %s" % ( typeobj_cname, type.vtabptr_cname, code.error_goto(entry.pos))) + code.putln("#endif") if heap_type_bases: code.globalstate.use_utility_code( UtilityCode.load_cached('MergeVTables', 'ImportExport.c')) @@ -4992,12 +5387,21 @@ class CClassDefNode(ClassDefNode): # scope.is_internal is set for types defined by # Cython (such as closures), the 'internal' # directive is set by users + code.putln("#if CYTHON_COMPILING_IN_LIMITED_API") + code.putln( + 'if (PyObject_SetAttr(%s, %s, %s) < 0) %s' % ( + Naming.module_cname, + code.intern_identifier(scope.class_name), + typeobj_cname, + code.error_goto(entry.pos))) + code.putln("#else") code.putln( 'if (PyObject_SetAttr(%s, %s, (PyObject *)&%s) < 0) %s' % ( Naming.module_cname, code.intern_identifier(scope.class_name), typeobj_cname, code.error_goto(entry.pos))) + code.putln("#endif") weakref_entry = scope.lookup_here("__weakref__") if not scope.is_closure_class_scope else None if weakref_entry: if weakref_entry.type is py_object_type: @@ -5019,15 +5423,23 @@ class CClassDefNode(ClassDefNode): # do so at runtime. code.globalstate.use_utility_code( UtilityCode.load_cached('SetupReduce', 'ExtensionTypes.c')) + code.putln("#if !CYTHON_COMPILING_IN_LIMITED_API") code.putln('if (__Pyx_setup_reduce((PyObject*)&%s) < 0) %s' % ( typeobj_cname, code.error_goto(entry.pos))) + code.putln("#endif") # Generate code to initialise the typeptr of an extension # type defined in this module to point to its type object. if type.typeobj_cname: + code.putln("#if CYTHON_COMPILING_IN_LIMITED_API") + code.putln( + "%s = (PyTypeObject *)%s;" % ( + type.typeptr_cname, type.typeobj_cname)) + code.putln("#else") code.putln( "%s = &%s;" % ( type.typeptr_cname, type.typeobj_cname)) + code.putln("#endif") def annotate(self, code): if self.type_init_args: @@ -5041,14 +5453,13 @@ class PropertyNode(StatNode): # # name string # doc EncodedString or None Doc string - # entry Symtab.Entry + # entry Symtab.Entry The Entry of the property attribute # body StatListNode child_attrs = ["body"] def analyse_declarations(self, env): self.entry = env.declare_property(self.name, self.doc, self.pos) - self.entry.scope.directives = env.directives self.body.analyse_declarations(self.entry.scope) def analyse_expressions(self, env): @@ -5065,6 +5476,44 @@ class PropertyNode(StatNode): self.body.annotate(code) +class CPropertyNode(StatNode): + """Definition of a C property, backed by a CFuncDefNode getter. + """ + # name string + # doc EncodedString or None Doc string of the property + # entry Symtab.Entry The Entry of the property attribute + # body StatListNode[CFuncDefNode] (for compatibility with PropertyNode) + + child_attrs = ["body"] + is_cproperty = True + + @property + def cfunc(self): + stats = self.body.stats + assert stats and isinstance(stats[0], CFuncDefNode), stats + return stats[0] + + def analyse_declarations(self, env): + scope = PropertyScope(self.name, class_scope=env) + self.body.analyse_declarations(scope) + entry = self.entry = env.declare_property( + self.name, self.doc, self.pos, ctype=self.cfunc.return_type, property_scope=scope) + entry.getter_cname = self.cfunc.entry.cname + + def analyse_expressions(self, env): + self.body = self.body.analyse_expressions(env) + return self + + def generate_function_definitions(self, env, code): + self.body.generate_function_definitions(env, code) + + def generate_execution_code(self, code): + pass + + def annotate(self, code): + self.body.annotate(code) + + class GlobalNode(StatNode): # Global variable declaration. # @@ -5814,7 +6263,7 @@ class ExecStatNode(StatNode): arg.free_temps(code) code.putln( code.error_goto_if_null(temp_result, self.pos)) - code.put_gotref(temp_result) + code.put_gotref(temp_result, py_object_type) code.put_decref_clear(temp_result, py_object_type) code.funcstate.release_temp(temp_result) @@ -6058,6 +6507,8 @@ class RaiseStatNode(StatNode): child_attrs = ["exc_type", "exc_value", "exc_tb", "cause"] is_terminator = True + builtin_exc_name = None + wrap_tuple_value = False def analyse_expressions(self, env): if self.exc_type: @@ -6065,6 +6516,12 @@ class RaiseStatNode(StatNode): self.exc_type = exc_type.coerce_to_pyobject(env) if self.exc_value: exc_value = self.exc_value.analyse_types(env) + if self.wrap_tuple_value: + if exc_value.type is Builtin.tuple_type or not exc_value.type.is_builtin_type: + # prevent tuple values from being interpreted as argument value tuples + from .ExprNodes import TupleNode + exc_value = TupleNode(exc_value.pos, args=[exc_value.coerce_to_pyobject(env)], slow=True) + exc_value = exc_value.analyse_types(env, skip_children=True) self.exc_value = exc_value.coerce_to_pyobject(env) if self.exc_tb: exc_tb = self.exc_tb.analyse_types(env) @@ -6073,7 +6530,6 @@ class RaiseStatNode(StatNode): cause = self.cause.analyse_types(env) self.cause = cause.coerce_to_pyobject(env) # special cases for builtin exceptions - self.builtin_exc_name = None if self.exc_type and not self.exc_value and not self.exc_tb: exc = self.exc_type from . import ExprNodes @@ -6083,7 +6539,7 @@ class RaiseStatNode(StatNode): if exc.is_name and exc.entry.is_builtin: self.builtin_exc_name = exc.name if self.builtin_exc_name == 'MemoryError': - self.exc_type = None # has a separate implementation + self.exc_type = None # has a separate implementation return self nogil_check = Node.gil_error @@ -6168,10 +6624,10 @@ class ReraiseStatNode(StatNode): vars = code.funcstate.exc_vars if vars: code.globalstate.use_utility_code(restore_exception_utility_code) - code.put_giveref(vars[0]) - code.put_giveref(vars[1]) + code.put_giveref(vars[0], py_object_type) + code.put_giveref(vars[1], py_object_type) # fresh exceptions may not have a traceback yet (-> finally!) - code.put_xgiveref(vars[2]) + code.put_xgiveref(vars[2], py_object_type) code.putln("__Pyx_ErrRestoreWithState(%s, %s, %s);" % tuple(vars)) for varname in vars: code.put("%s = 0; " % varname) @@ -6182,65 +6638,53 @@ class ReraiseStatNode(StatNode): UtilityCode.load_cached("ReRaiseException", "Exceptions.c")) code.putln("__Pyx_ReraiseException(); %s" % code.error_goto(self.pos)) + class AssertStatNode(StatNode): # assert statement # - # cond ExprNode - # value ExprNode or None + # condition ExprNode + # value ExprNode or None + # exception (Raise/GIL)StatNode created from 'value' in PostParse transform - child_attrs = ["cond", "value"] + child_attrs = ["condition", "value", "exception"] + exception = None + + def analyse_declarations(self, env): + assert self.value is None, "Message should have been replaced in PostParse()" + assert self.exception is not None, "Message should have been replaced in PostParse()" + self.exception.analyse_declarations(env) def analyse_expressions(self, env): - self.cond = self.cond.analyse_boolean_expression(env) - if self.value: - value = self.value.analyse_types(env) - if value.type is Builtin.tuple_type or not value.type.is_builtin_type: - # prevent tuple values from being interpreted as argument value tuples - from .ExprNodes import TupleNode - value = TupleNode(value.pos, args=[value], slow=True) - self.value = value.analyse_types(env, skip_children=True).coerce_to_pyobject(env) - else: - self.value = value.coerce_to_pyobject(env) + self.condition = self.condition.analyse_temp_boolean_expression(env) + self.exception = self.exception.analyse_expressions(env) return self - nogil_check = Node.gil_error - gil_message = "Raising exception" - def generate_execution_code(self, code): code.putln("#ifndef CYTHON_WITHOUT_ASSERTIONS") code.putln("if (unlikely(!Py_OptimizeFlag)) {") code.mark_pos(self.pos) - self.cond.generate_evaluation_code(code) - code.putln( - "if (unlikely(!%s)) {" % self.cond.result()) - if self.value: - self.value.generate_evaluation_code(code) - code.putln( - "PyErr_SetObject(PyExc_AssertionError, %s);" % self.value.py_result()) - self.value.generate_disposal_code(code) - self.value.free_temps(code) - else: - code.putln( - "PyErr_SetNone(PyExc_AssertionError);") + self.condition.generate_evaluation_code(code) code.putln( - code.error_goto(self.pos)) + "if (unlikely(!%s)) {" % self.condition.result()) + self.exception.generate_execution_code(code) code.putln( "}") - self.cond.generate_disposal_code(code) - self.cond.free_temps(code) + self.condition.generate_disposal_code(code) + self.condition.free_temps(code) code.putln( "}") + code.putln("#else") + # avoid unused labels etc. + code.putln("if ((1)); else %s" % code.error_goto(self.pos, used=False)) code.putln("#endif") def generate_function_definitions(self, env, code): - self.cond.generate_function_definitions(env, code) - if self.value is not None: - self.value.generate_function_definitions(env, code) + self.condition.generate_function_definitions(env, code) + self.exception.generate_function_definitions(env, code) def annotate(self, code): - self.cond.annotate(code) - if self.value: - self.value.annotate(code) + self.condition.annotate(code) + self.exception.annotate(code) class IfStatNode(StatNode): @@ -6267,13 +6711,9 @@ class IfStatNode(StatNode): code.mark_pos(self.pos) end_label = code.new_label() last = len(self.if_clauses) - if self.else_clause: - # If the 'else' clause is 'unlikely', then set the preceding 'if' clause to 'likely' to reflect that. - self._set_branch_hint(self.if_clauses[-1], self.else_clause, inverse=True) - else: + if not self.else_clause: last -= 1 # avoid redundant goto at end of last if-clause for i, if_clause in enumerate(self.if_clauses): - self._set_branch_hint(if_clause, if_clause.body) if_clause.generate_execution_code(code, end_label, is_last=i == last) if self.else_clause: code.mark_pos(self.else_clause.pos) @@ -6282,21 +6722,6 @@ class IfStatNode(StatNode): code.putln("}") code.put_label(end_label) - def _set_branch_hint(self, clause, statements_node, inverse=False): - if not statements_node.is_terminator: - return - if not isinstance(statements_node, StatListNode) or not statements_node.stats: - return - # Anything that unconditionally raises exceptions should be considered unlikely. - if isinstance(statements_node.stats[-1], (RaiseStatNode, ReraiseStatNode)): - if len(statements_node.stats) > 1: - # Allow simple statements before the 'raise', but no conditions, loops, etc. - non_branch_nodes = (ExprStatNode, AssignmentNode, DelStatNode, GlobalNode, NonlocalNode) - for node in statements_node.stats[:-1]: - if not isinstance(node, non_branch_nodes): - return - clause.branch_hint = 'likely' if inverse else 'unlikely' - def generate_function_definitions(self, env, code): for clause in self.if_clauses: clause.generate_function_definitions(env, code) @@ -6582,7 +7007,7 @@ class DictIterationNextNode(Node): # evaluate all coercions before the assignments for var, result, target in assignments: - code.put_gotref(var.result()) + var.generate_gotref(code) for var, result, target in assignments: result.generate_evaluation_code(code) for var, result, target in assignments: @@ -6644,7 +7069,7 @@ class SetIterationNextNode(Node): code.funcstate.release_temp(result_temp) # evaluate all coercions before the assignments - code.put_gotref(value_ref.result()) + value_ref.generate_gotref(code) self.coerced_value_var.generate_evaluation_code(code) self.value_target.generate_assignment_code(self.coerced_value_var, code) value_ref.release(code) @@ -6957,7 +7382,7 @@ class ForFromStatNode(LoopNode, StatNode): target_node.result(), interned_cname, code.error_goto_if_null(target_node.result(), self.target.pos))) - code.put_gotref(target_node.result()) + target_node.generate_gotref(code) else: target_node = self.target from_py_node = ExprNodes.CoerceFromPyTypeNode( @@ -7093,7 +7518,7 @@ class WithStatNode(StatNode): code.intern_identifier(EncodedString('__aexit__' if self.is_async else '__exit__')), code.error_goto_if_null(self.exit_var, self.pos), )) - code.put_gotref(self.exit_var) + code.put_gotref(self.exit_var, py_object_type) # need to free exit_var in the face of exceptions during setup old_error_label = code.new_error_label() @@ -7237,17 +7662,17 @@ class TryExceptStatNode(StatNode): save_exc.putln("__Pyx_ExceptionSave(%s);" % ( ', '.join(['&%s' % var for var in exc_save_vars]))) for var in exc_save_vars: - save_exc.put_xgotref(var) + save_exc.put_xgotref(var, py_object_type) def restore_saved_exception(): for name in exc_save_vars: - code.put_xgiveref(name) + code.put_xgiveref(name, py_object_type) code.putln("__Pyx_ExceptionReset(%s);" % ', '.join(exc_save_vars)) else: # try block cannot raise exceptions, but we had to allocate the temps above, # so just keep the C compiler from complaining about them being unused - mark_vars_used = ["(void)%s;" % var for var in exc_save_vars] + mark_vars_used = ["(void)%s;" % var for var in exc_save_vars] save_exc.putln("%s /* mark used */" % ' '.join(mark_vars_used)) def restore_saved_exception(): @@ -7390,17 +7815,40 @@ class ExceptClauseNode(Node): for _ in range(3)] code.globalstate.use_utility_code(UtilityCode.load_cached("PyErrFetchRestore", "Exceptions.c")) code.putln("__Pyx_ErrFetch(&%s, &%s, &%s);" % tuple(exc_vars)) - code.globalstate.use_utility_code(UtilityCode.load_cached("FastTypeChecks", "ModuleSetupCode.c")) - exc_test_func = "__Pyx_PyErr_GivenExceptionMatches(%s, %%s)" % exc_vars[0] + exc_type = exc_vars[0] else: - exc_vars = () - code.globalstate.use_utility_code(UtilityCode.load_cached("PyErrExceptionMatches", "Exceptions.c")) - exc_test_func = "__Pyx_PyErr_ExceptionMatches(%s)" + exc_vars = exc_type = None - exc_tests = [] for pattern in self.pattern: pattern.generate_evaluation_code(code) - exc_tests.append(exc_test_func % pattern.py_result()) + patterns = [pattern.py_result() for pattern in self.pattern] + + exc_tests = [] + if exc_type: + code.globalstate.use_utility_code( + UtilityCode.load_cached("FastTypeChecks", "ModuleSetupCode.c")) + if len(patterns) == 2: + exc_tests.append("__Pyx_PyErr_GivenExceptionMatches2(%s, %s, %s)" % ( + exc_type, patterns[0], patterns[1], + )) + else: + exc_tests.extend( + "__Pyx_PyErr_GivenExceptionMatches(%s, %s)" % (exc_type, pattern) + for pattern in patterns + ) + elif len(patterns) == 2: + code.globalstate.use_utility_code( + UtilityCode.load_cached("FastTypeChecks", "ModuleSetupCode.c")) + exc_tests.append("__Pyx_PyErr_ExceptionMatches2(%s, %s)" % ( + patterns[0], patterns[1], + )) + else: + code.globalstate.use_utility_code( + UtilityCode.load_cached("PyErrExceptionMatches", "Exceptions.c")) + exc_tests.extend( + "__Pyx_PyErr_ExceptionMatches(%s)" % pattern + for pattern in patterns + ) match_flag = code.funcstate.allocate_temp(PyrexTypes.c_int_type, manage_ref=False) code.putln("%s = %s;" % (match_flag, ' || '.join(exc_tests))) @@ -7408,7 +7856,7 @@ class ExceptClauseNode(Node): pattern.generate_disposal_code(code) pattern.free_temps(code) - if has_non_literals: + if exc_vars: code.putln("__Pyx_ErrRestore(%s, %s, %s);" % tuple(exc_vars)) code.putln(' '.join(["%s = 0;" % var for var in exc_vars])) for temp in exc_vars: @@ -7443,7 +7891,7 @@ class ExceptClauseNode(Node): code.putln("if (__Pyx_GetException(%s) < 0) %s" % ( exc_args, code.error_goto(self.pos))) for var in exc_vars: - code.put_gotref(var) + code.put_gotref(var, py_object_type) if self.target: self.exc_value.set_var(exc_vars[1]) self.exc_value.generate_evaluation_code(code) @@ -7723,7 +8171,7 @@ class TryFinallyStatNode(StatNode): " unlikely(__Pyx_GetException(&%s, &%s, &%s) < 0)) " "__Pyx_ErrFetch(&%s, &%s, &%s);" % (exc_vars[:3] * 2)) for var in exc_vars: - code.put_xgotref(var) + code.put_xgotref(var, py_object_type) if exc_lineno_cnames: code.putln("%s = %s; %s = %s; %s = %s;" % ( exc_lineno_cnames[0], Naming.lineno_cname, @@ -7744,11 +8192,11 @@ class TryFinallyStatNode(StatNode): # unused utility functions and/or temps code.putln("if (PY_MAJOR_VERSION >= 3) {") for var in exc_vars[3:]: - code.put_xgiveref(var) + code.put_xgiveref(var, py_object_type) code.putln("__Pyx_ExceptionReset(%s, %s, %s);" % exc_vars[3:]) code.putln("}") for var in exc_vars[:3]: - code.put_xgiveref(var) + code.put_xgiveref(var, py_object_type) code.putln("__Pyx_ErrRestore(%s, %s, %s);" % exc_vars[:3]) if self.is_try_finally_in_nogil: @@ -7770,7 +8218,7 @@ class TryFinallyStatNode(StatNode): # unused utility functions and/or temps code.putln("if (PY_MAJOR_VERSION >= 3) {") for var in exc_vars[3:]: - code.put_xgiveref(var) + code.put_xgiveref(var, py_object_type) code.putln("__Pyx_ExceptionReset(%s, %s, %s);" % exc_vars[3:]) code.putln("}") for var in exc_vars[:3]: @@ -7798,10 +8246,12 @@ class GILStatNode(NogilTryFinallyStatNode): # # state string 'gil' or 'nogil' + child_attrs = ["condition"] + NogilTryFinallyStatNode.child_attrs state_temp = None - def __init__(self, pos, state, body): + def __init__(self, pos, state, body, condition=None): self.state = state + self.condition = condition self.create_state_temp_if_needed(pos, state, body) TryFinallyStatNode.__init__( self, pos, @@ -7828,11 +8278,18 @@ class GILStatNode(NogilTryFinallyStatNode): if self.state == 'gil': env.has_with_gil_block = True + if self.condition is not None: + self.condition.analyse_declarations(env) + return super(GILStatNode, self).analyse_declarations(env) def analyse_expressions(self, env): env.use_utility_code( UtilityCode.load_cached("ForceInitThreads", "ModuleSetupCode.c")) + + if self.condition is not None: + self.condition = self.condition.analyse_expressions(env) + was_nogil = env.nogil env.nogil = self.state == 'nogil' node = TryFinallyStatNode.analyse_expressions(self, env) @@ -7919,6 +8376,33 @@ utility_code_for_imports = { 'inspect': ("__Pyx_patch_inspect", "PatchInspect", "Coroutine.c"), } +def cimport_numpy_check(node, code): + # shared code between CImportStatNode and FromCImportStatNode + # check to ensure that import_array is called + for mod in code.globalstate.module_node.scope.cimported_modules: + if mod.name != node.module_name: + continue + # there are sometimes several cimported modules with the same name + # so complete the loop if necessary + import_array = mod.lookup_here("import_array") + _import_array = mod.lookup_here("_import_array") + # at least one entry used + used = (import_array and import_array.used) or (_import_array and _import_array.used) + if ((import_array or _import_array) # at least one entry found + and not used): + # sanity check that this is actually numpy and not a user pxd called "numpy" + if _import_array and _import_array.type.is_cfunction: + # warning is mainly for the sake of testing + warning(node.pos, "'numpy.import_array()' has been added automatically " + "since 'numpy' was cimported but 'numpy.import_array' was not called.", 0) + from .Code import TempitaUtilityCode + code.globalstate.use_utility_code( + TempitaUtilityCode.load_cached("NumpyImportArray", "NumpyImportArray.c", + context = {'err_goto': code.error_goto(node.pos)}) + ) + return # no need to continue once the utility code is added + + class CImportStatNode(StatNode): # cimport statement @@ -7960,7 +8444,8 @@ class CImportStatNode(StatNode): return self def generate_execution_code(self, code): - pass + if self.module_name == "numpy": + cimport_numpy_check(self, code) class FromCImportStatNode(StatNode): @@ -8013,7 +8498,7 @@ class FromCImportStatNode(StatNode): local_name = as_name or name env.add_imported_entry(local_name, entry, pos) - if module_name.startswith('cpython') or module_name.startswith('cython'): # enough for now + if module_name.startswith('cpython') or module_name.startswith('cython'): # enough for now if module_name in utility_code_for_cimports: env.use_utility_code(utility_code_for_cimports[module_name]()) for _, name, _, _ in self.imported_names: @@ -8039,7 +8524,8 @@ class FromCImportStatNode(StatNode): return self def generate_execution_code(self, code): - pass + if self.module_name == "numpy": + cimport_numpy_check(self, code) class FromImportStatNode(StatNode): @@ -8121,7 +8607,7 @@ class FromImportStatNode(StatNode): self.module.py_result(), code.intern_identifier(name), code.error_goto_if_null(item_temp, self.pos))) - code.put_gotref(item_temp) + code.put_gotref(item_temp, py_object_type) if coerced_item is None: target.generate_assignment_code(self.item, code) else: @@ -8237,7 +8723,7 @@ class ParallelStatNode(StatNode, ParallelNode): seen.add(dictitem.key.value) if dictitem.key.value == 'num_threads': if not dictitem.value.is_none: - self.num_threads = dictitem.value + self.num_threads = dictitem.value elif self.is_prange and dictitem.key.value == 'chunksize': if not dictitem.value.is_none: self.chunksize = dictitem.value @@ -8498,11 +8984,7 @@ class ParallelStatNode(StatNode, ParallelNode): if self.is_parallel and not self.is_nested_prange: code.putln("/* Clean up any temporaries */") for temp, type in sorted(self.temps): - if type.is_memoryviewslice: - code.put_xdecref_memoryviewslice(temp, have_gil=False) - elif type.is_pyobject: - code.put_xdecref(temp, type) - code.putln("%s = NULL;" % temp) + code.put_xdecref_clear(temp, type, have_gil=False) def setup_parallel_control_flow_block(self, code): """ @@ -8535,7 +9017,7 @@ class ParallelStatNode(StatNode, ParallelNode): self.old_return_label = code.return_label code.return_label = code.new_label(name="return") - code.begin_block() # parallel control flow block + code.begin_block() # parallel control flow block self.begin_of_parallel_control_block_point = code.insertion_point() self.begin_of_parallel_control_block_point_after_decls = code.insertion_point() @@ -8665,7 +9147,7 @@ class ParallelStatNode(StatNode, ParallelNode): code.putln_openmp("#pragma omp critical(%s)" % section_name) ParallelStatNode.critical_section_counter += 1 - code.begin_block() # begin critical section + code.begin_block() # begin critical section c = self.begin_of_parallel_control_block_point @@ -8693,7 +9175,7 @@ class ParallelStatNode(StatNode, ParallelNode): self.parallel_private_temps.append((temp_cname, private_cname)) - code.end_block() # end critical section + code.end_block() # end critical section def fetch_parallel_exception(self, code): """ @@ -8733,7 +9215,7 @@ class ParallelStatNode(StatNode, ParallelNode): pos_info = chain(*zip(self.parallel_pos_info, self.pos_info)) code.funcstate.uses_error_indicator = True code.putln("%s = %s; %s = %s; %s = %s;" % tuple(pos_info)) - code.put_gotref(Naming.parallel_exc_type) + code.put_gotref(Naming.parallel_exc_type, py_object_type) code.putln( "}") @@ -8746,7 +9228,7 @@ class ParallelStatNode(StatNode, ParallelNode): code.begin_block() code.put_ensure_gil(declare_gilstate=True) - code.put_giveref(Naming.parallel_exc_type) + code.put_giveref(Naming.parallel_exc_type, py_object_type) code.putln("__Pyx_ErrRestoreWithState(%s, %s, %s);" % self.parallel_exc) pos_info = chain(*zip(self.pos_info, self.parallel_pos_info)) code.putln("%s = %s; %s = %s; %s = %s;" % tuple(pos_info)) @@ -8834,11 +9316,11 @@ class ParallelStatNode(StatNode, ParallelNode): self.restore_parallel_exception(code) code.put_goto(code.error_label) - code.putln("}") # end switch + code.putln("}") # end switch code.putln( - "}") # end if + "}") # end if - code.end_block() # end parallel control flow block + code.end_block() # end parallel control flow block self.redef_builtin_expect_apple_gcc_bug(code) # FIXME: improve with version number for OS X Lion @@ -8957,9 +9439,6 @@ class ParallelRangeNode(ParallelStatNode): else: self.start, self.stop, self.step = self.args - if hasattr(self.schedule, 'decode'): - self.schedule = self.schedule.decode('ascii') - if self.schedule not in (None, 'static', 'dynamic', 'guided', 'runtime'): error(self.pos, "Invalid schedule argument to prange: %s" % (self.schedule,)) @@ -9018,7 +9497,8 @@ class ParallelRangeNode(ParallelStatNode): # ensure lastprivate behaviour and propagation. If the target index is # not a NameNode, it won't have an entry, and an error was issued by # ParallelRangeTransform - if hasattr(self.target, 'entry'): + target_entry = getattr(self.target, 'entry', None) + if target_entry: self.assignments[self.target.entry] = self.target.pos, None node = super(ParallelRangeNode, self).analyse_expressions(env) @@ -9131,9 +9611,12 @@ class ParallelRangeNode(ParallelStatNode): # TODO: check if the step is 0 and if so, raise an exception in a # 'with gil' block. For now, just abort - code.putln("if ((%(step)s == 0)) abort();" % fmt_dict) + if self.step is not None and self.step.has_constant_result() and self.step.constant_result == 0: + error(node.pos, "Iteration with step 0 is invalid.") + elif not fmt_dict['step'].isdigit() or int(fmt_dict['step']) == 0: + code.putln("if (((%(step)s) == 0)) abort();" % fmt_dict) - self.setup_parallel_control_flow_block(code) # parallel control flow block + self.setup_parallel_control_flow_block(code) # parallel control flow block # Note: nsteps is private in an outer scope if present code.putln("%(nsteps)s = (%(stop)s - %(start)s + %(step)s - %(step)s/abs(%(step)s)) / %(step)s;" % fmt_dict) @@ -9145,9 +9628,9 @@ class ParallelRangeNode(ParallelStatNode): # erroneously believes that nsteps may be <= 0, leaving the private # target index uninitialized code.putln("if (%(nsteps)s > 0)" % fmt_dict) - code.begin_block() # if block + code.begin_block() # if block self.generate_loop(code, fmt_dict) - code.end_block() # end if block + code.end_block() # end if block self.restore_labels(code) @@ -9155,13 +9638,13 @@ class ParallelRangeNode(ParallelStatNode): if self.breaking_label_used: code.put("if (%s < 2)" % Naming.parallel_why) - code.begin_block() # else block + code.begin_block() # else block code.putln("/* else */") self.else_clause.generate_execution_code(code) - code.end_block() # end else block + code.end_block() # end else block # ------ cleanup ------ - self.end_parallel_control_flow_block(code) # end parallel control flow block + self.end_parallel_control_flow_block(code) # end parallel control flow block # And finally, release our privates and write back any closure # variables @@ -9192,7 +9675,7 @@ class ParallelRangeNode(ParallelStatNode): code.putln("") code.putln("#endif /* _OPENMP */") - code.begin_block() # pragma omp parallel begin block + code.begin_block() # pragma omp parallel begin block # Initialize the GIL if needed for this thread self.begin_parallel_block(code) diff --git a/Cython/Compiler/Optimize.py b/Cython/Compiler/Optimize.py index 69526a27c..25d654330 100644 --- a/Cython/Compiler/Optimize.py +++ b/Cython/Compiler/Optimize.py @@ -41,7 +41,7 @@ except ImportError: try: from __builtin__ import basestring except ImportError: - basestring = str # Python 3 + basestring = str # Python 3 def load_c_utility(name): @@ -192,7 +192,7 @@ class IterationTransform(Visitor.EnvTransform): def _optimise_for_loop(self, node, iterable, reversed=False): annotation_type = None if (iterable.is_name or iterable.is_attribute) and iterable.entry and iterable.entry.annotation: - annotation = iterable.entry.annotation + annotation = iterable.entry.annotation.expr if annotation.is_subscript: annotation = annotation.base # container base type # FIXME: generalise annotation evaluation => maybe provide a "qualified name" also for imported names? @@ -228,6 +228,12 @@ class IterationTransform(Visitor.EnvTransform): return self._transform_bytes_iteration(node, iterable, reversed=reversed) if iterable.type is Builtin.unicode_type: return self._transform_unicode_iteration(node, iterable, reversed=reversed) + # in principle _transform_indexable_iteration would work on most of the above, and + # also tuple and list. However, it probably isn't quite as optimized + if iterable.type is Builtin.bytearray_type: + return self._transform_indexable_iteration(node, iterable, is_mutable=True, reversed=reversed) + if isinstance(iterable, ExprNodes.CoerceToPyTypeNode) and iterable.arg.type.is_memoryviewslice: + return self._transform_indexable_iteration(node, iterable.arg, is_mutable=False, reversed=reversed) # the rest is based on function calls if not isinstance(iterable, ExprNodes.SimpleCallNode): @@ -333,6 +339,92 @@ class IterationTransform(Visitor.EnvTransform): PyrexTypes.CFuncTypeArg("s", Builtin.bytes_type, None) ]) + def _transform_indexable_iteration(self, node, slice_node, is_mutable, reversed=False): + """In principle can handle any iterable that Cython has a len() for and knows how to index""" + unpack_temp_node = UtilNodes.LetRefNode( + slice_node.as_none_safe_node("'NoneType' is not iterable"), + may_hold_none=False, is_temp=True + ) + + start_node = ExprNodes.IntNode( + node.pos, value='0', constant_result=0, type=PyrexTypes.c_py_ssize_t_type) + def make_length_call(): + # helper function since we need to create this node for a couple of places + builtin_len = ExprNodes.NameNode(node.pos, name="len", + entry=Builtin.builtin_scope.lookup("len")) + return ExprNodes.SimpleCallNode(node.pos, + function=builtin_len, + args=[unpack_temp_node] + ) + length_temp = UtilNodes.LetRefNode(make_length_call(), type=PyrexTypes.c_py_ssize_t_type, is_temp=True) + end_node = length_temp + + if reversed: + relation1, relation2 = '>', '>=' + start_node, end_node = end_node, start_node + else: + relation1, relation2 = '<=', '<' + + counter_ref = UtilNodes.LetRefNode(pos=node.pos, type=PyrexTypes.c_py_ssize_t_type) + + target_value = ExprNodes.IndexNode(slice_node.pos, base=unpack_temp_node, + index=counter_ref) + + target_assign = Nodes.SingleAssignmentNode( + pos = node.target.pos, + lhs = node.target, + rhs = target_value) + + # analyse with boundscheck and wraparound + # off (because we're confident we know the size) + env = self.current_env() + new_directives = Options.copy_inherited_directives(env.directives, boundscheck=False, wraparound=False) + target_assign = Nodes.CompilerDirectivesNode( + target_assign.pos, + directives=new_directives, + body=target_assign, + ) + + body = Nodes.StatListNode( + node.pos, + stats = [target_assign]) # exclude node.body for now to not reanalyse it + if is_mutable: + # We need to be slightly careful here that we are actually modifying the loop + # bounds and not a temp copy of it. Setting is_temp=True on length_temp seems + # to ensure this. + # If this starts to fail then we could insert an "if out_of_bounds: break" instead + loop_length_reassign = Nodes.SingleAssignmentNode(node.pos, + lhs = length_temp, + rhs = make_length_call()) + body.stats.append(loop_length_reassign) + + loop_node = Nodes.ForFromStatNode( + node.pos, + bound1=start_node, relation1=relation1, + target=counter_ref, + relation2=relation2, bound2=end_node, + step=None, body=body, + else_clause=node.else_clause, + from_range=True) + + ret = UtilNodes.LetNode( + unpack_temp_node, + UtilNodes.LetNode( + length_temp, + # TempResultFromStatNode provides the framework where the "counter_ref" + # temp is set up and can be assigned to. However, we don't need the + # result it returns so wrap it in an ExprStatNode. + Nodes.ExprStatNode(node.pos, + expr=UtilNodes.TempResultFromStatNode( + counter_ref, + loop_node + ) + ) + ) + ).analyse_expressions(env) + body.stats.insert(1, node.body) + return ret + def _transform_bytes_iteration(self, node, slice_node, reversed=False): target_type = node.target.type if not target_type.is_int and target_type is not Builtin.bytes_type: @@ -1144,7 +1236,7 @@ class SwitchTransform(Visitor.EnvTransform): # integers on iteration, whereas Py2 returns 1-char byte # strings characters = string_literal.value - characters = list(set([ characters[i:i+1] for i in range(len(characters)) ])) + characters = list({ characters[i:i+1] for i in range(len(characters)) }) characters.sort() return [ ExprNodes.CharNode(string_literal.pos, value=charval, constant_result=charval) @@ -1156,7 +1248,8 @@ class SwitchTransform(Visitor.EnvTransform): return self.NO_MATCH elif common_var is not None and not is_common_value(var, common_var): return self.NO_MATCH - elif not (var.type.is_int or var.type.is_enum) or sum([not (cond.type.is_int or cond.type.is_enum) for cond in conditions]): + elif not (var.type.is_int or var.type.is_enum) or any( + [not (cond.type.is_int or cond.type.is_enum) for cond in conditions]): return self.NO_MATCH return not_in, var, conditions @@ -1569,7 +1662,7 @@ class EarlyReplaceBuiltinCalls(Visitor.EnvTransform): utility_code = utility_code) def _error_wrong_arg_count(self, function_name, node, args, expected=None): - if not expected: # None or 0 + if not expected: # None or 0 arg_str = '' elif isinstance(expected, basestring) or expected > 1: arg_str = '...' @@ -1723,7 +1816,7 @@ class EarlyReplaceBuiltinCalls(Visitor.EnvTransform): arg = pos_args[0] if isinstance(arg, ExprNodes.ComprehensionNode) and arg.type is Builtin.list_type: - list_node = pos_args[0] + list_node = arg loop_node = list_node.loop elif isinstance(arg, ExprNodes.GeneratorExpressionNode): @@ -1753,7 +1846,11 @@ class EarlyReplaceBuiltinCalls(Visitor.EnvTransform): # Interestingly, PySequence_List works on a lot of non-sequence # things as well. list_node = loop_node = ExprNodes.PythonCapiCallNode( - node.pos, "PySequence_List", self.PySequence_List_func_type, + node.pos, + "__Pyx_PySequence_ListKeepNew" + if arg.is_temp and arg.type in (PyrexTypes.py_object_type, Builtin.list_type) + else "PySequence_List", + self.PySequence_List_func_type, args=pos_args, is_temp=True) result_node = UtilNodes.ResultRefNode( @@ -1799,7 +1896,7 @@ class EarlyReplaceBuiltinCalls(Visitor.EnvTransform): if not yield_expression.is_literal or not yield_expression.type.is_int: return node except AttributeError: - return node # in case we don't have a type yet + return node # in case we don't have a type yet # special case: old Py2 backwards compatible "sum([int_const for ...])" # can safely be unpacked into a genexpr @@ -2093,12 +2190,13 @@ class OptimizeBuiltinCalls(Visitor.NodeRefCleanupMixin, func_arg = arg.args[0] if func_arg.type is Builtin.float_type: return func_arg.as_none_safe_node("float() argument must be a string or a number, not 'NoneType'") - elif func_arg.type.is_pyobject: + elif func_arg.type.is_pyobject and arg.function.cname == "__Pyx_PyObject_AsDouble": return ExprNodes.PythonCapiCallNode( node.pos, '__Pyx_PyNumber_Float', self.PyNumber_Float_func_type, args=[func_arg], py_name='float', is_temp=node.is_temp, + utility_code = UtilityCode.load_cached("pynumber_float", "TypeConversion.c"), result_is_used=node.result_is_used, ).coerce_to(node.type, self.current_env()) return node @@ -2232,7 +2330,7 @@ class OptimizeBuiltinCalls(Visitor.NodeRefCleanupMixin, return node def _error_wrong_arg_count(self, function_name, node, args, expected=None): - if not expected: # None or 0 + if not expected: # None or 0 arg_str = '' elif isinstance(expected, basestring) or expected > 1: arg_str = '...' @@ -2312,6 +2410,38 @@ class OptimizeBuiltinCalls(Visitor.NodeRefCleanupMixin, return ExprNodes.CachedBuiltinMethodCallNode( node, function.obj, attr_name, arg_list) + PyObject_String_func_type = PyrexTypes.CFuncType( + PyrexTypes.py_object_type, [ # Change this to Builtin.str_type when removing Py2 support. + PyrexTypes.CFuncTypeArg("obj", PyrexTypes.py_object_type, None) + ]) + + def _handle_simple_function_str(self, node, function, pos_args): + """Optimize single argument calls to str(). + """ + if len(pos_args) != 1: + if len(pos_args) == 0: + return ExprNodes.StringNode(node.pos, value=EncodedString(), constant_result='') + return node + arg = pos_args[0] + + if arg.type is Builtin.str_type: + if not arg.may_be_none(): + return arg + + cname = "__Pyx_PyStr_Str" + utility_code = UtilityCode.load_cached('PyStr_Str', 'StringTools.c') + else: + cname = '__Pyx_PyObject_Str' + utility_code = UtilityCode.load_cached('PyObject_Str', 'StringTools.c') + + return ExprNodes.PythonCapiCallNode( + node.pos, cname, self.PyObject_String_func_type, + args=pos_args, + is_temp=node.is_temp, + utility_code=utility_code, + py_name="str" + ) + PyObject_Unicode_func_type = PyrexTypes.CFuncType( Builtin.unicode_type, [ PyrexTypes.CFuncTypeArg("obj", PyrexTypes.py_object_type, None) @@ -2383,8 +2513,14 @@ class OptimizeBuiltinCalls(Visitor.NodeRefCleanupMixin, return node arg = pos_args[0] return ExprNodes.PythonCapiCallNode( - node.pos, "PySequence_List", self.PySequence_List_func_type, - args=pos_args, is_temp=node.is_temp) + node.pos, + "__Pyx_PySequence_ListKeepNew" + if node.is_temp and arg.is_temp and arg.type in (PyrexTypes.py_object_type, Builtin.list_type) + else "PySequence_List", + self.PySequence_List_func_type, + args=pos_args, + is_temp=node.is_temp, + ) PyList_AsTuple_func_type = PyrexTypes.CFuncType( Builtin.tuple_type, [ @@ -2485,6 +2621,7 @@ class OptimizeBuiltinCalls(Visitor.NodeRefCleanupMixin, elif len(pos_args) != 1: self._error_wrong_arg_count('float', node, pos_args, '0 or 1') return node + func_arg = pos_args[0] if isinstance(func_arg, ExprNodes.CoerceToPyTypeNode): func_arg = func_arg.arg @@ -2493,12 +2630,37 @@ class OptimizeBuiltinCalls(Visitor.NodeRefCleanupMixin, elif node.type.assignable_from(func_arg.type) or func_arg.type.is_numeric: return ExprNodes.TypecastNode( node.pos, operand=func_arg, type=node.type) + + arg = None + if func_arg.type is Builtin.bytes_type: + cfunc_name = "__Pyx_PyBytes_AsDouble" + utility_code_name = 'pybytes_as_double' + elif func_arg.type is Builtin.bytearray_type: + cfunc_name = "__Pyx_PyByteArray_AsDouble" + utility_code_name = 'pybytes_as_double' + elif func_arg.type is Builtin.unicode_type: + cfunc_name = "__Pyx_PyUnicode_AsDouble" + utility_code_name = 'pyunicode_as_double' + elif func_arg.type is Builtin.str_type: + cfunc_name = "__Pyx_PyString_AsDouble" + utility_code_name = 'pystring_as_double' + elif func_arg.type is Builtin.long_type: + cfunc_name = "PyLong_AsDouble" + else: + arg = func_arg # no need for an additional None check + cfunc_name = "__Pyx_PyObject_AsDouble" + utility_code_name = 'pyobject_as_double' + + if arg is None: + arg = func_arg.as_none_safe_node( + "float() argument must be a string or a number, not 'NoneType'") + return ExprNodes.PythonCapiCallNode( - node.pos, "__Pyx_PyObject_AsDouble", + node.pos, cfunc_name, self.PyObject_AsDouble_func_type, - args = pos_args, + args = [arg], is_temp = node.is_temp, - utility_code = load_c_utility('pyobject_as_double'), + utility_code = load_c_utility(utility_code_name) if utility_code_name else None, py_name = "float") PyNumber_Int_func_type = PyrexTypes.CFuncType( @@ -2557,12 +2719,20 @@ class OptimizeBuiltinCalls(Visitor.NodeRefCleanupMixin, Pyx_strlen_func_type = PyrexTypes.CFuncType( PyrexTypes.c_size_t_type, [ PyrexTypes.CFuncTypeArg("bytes", PyrexTypes.c_const_char_ptr_type, None) - ]) + ], + nogil=True) + + Pyx_ssize_strlen_func_type = PyrexTypes.CFuncType( + PyrexTypes.c_py_ssize_t_type, [ + PyrexTypes.CFuncTypeArg("bytes", PyrexTypes.c_const_char_ptr_type, None) + ], + exception_value="-1") Pyx_Py_UNICODE_strlen_func_type = PyrexTypes.CFuncType( - PyrexTypes.c_size_t_type, [ + PyrexTypes.c_py_ssize_t_type, [ PyrexTypes.CFuncTypeArg("unicode", PyrexTypes.c_const_py_unicode_ptr_type, None) - ]) + ], + exception_value="-1") PyObject_Size_func_type = PyrexTypes.CFuncType( PyrexTypes.c_py_ssize_t_type, [ @@ -2581,7 +2751,7 @@ class OptimizeBuiltinCalls(Visitor.NodeRefCleanupMixin, Builtin.dict_type: "PyDict_Size", }.get - _ext_types_with_pysize = set(["cpython.array.array"]) + _ext_types_with_pysize = {"cpython.array.array"} def _handle_simple_function_len(self, node, function, pos_args): """Replace len(char*) by the equivalent call to strlen(), @@ -2596,18 +2766,19 @@ class OptimizeBuiltinCalls(Visitor.NodeRefCleanupMixin, arg = arg.arg if arg.type.is_string: new_node = ExprNodes.PythonCapiCallNode( - node.pos, "strlen", self.Pyx_strlen_func_type, + node.pos, "__Pyx_ssize_strlen", self.Pyx_ssize_strlen_func_type, args = [arg], is_temp = node.is_temp, - utility_code = UtilityCode.load_cached("IncludeStringH", "StringTools.c")) + utility_code = UtilityCode.load_cached("ssize_strlen", "StringTools.c")) elif arg.type.is_pyunicode_ptr: new_node = ExprNodes.PythonCapiCallNode( - node.pos, "__Pyx_Py_UNICODE_strlen", self.Pyx_Py_UNICODE_strlen_func_type, + node.pos, "__Pyx_Py_UNICODE_ssize_strlen", self.Pyx_Py_UNICODE_strlen_func_type, args = [arg], - is_temp = node.is_temp) + is_temp = node.is_temp, + utility_code = UtilityCode.load_cached("ssize_pyunicode_strlen", "StringTools.c")) elif arg.type.is_memoryviewslice: func_type = PyrexTypes.CFuncType( - PyrexTypes.c_size_t_type, [ + PyrexTypes.c_py_ssize_t_type, [ PyrexTypes.CFuncTypeArg("memoryviewslice", arg.type, None) ], nogil=True) new_node = ExprNodes.PythonCapiCallNode( @@ -2618,7 +2789,7 @@ class OptimizeBuiltinCalls(Visitor.NodeRefCleanupMixin, if cfunc_name is None: arg_type = arg.type if ((arg_type.is_extension_type or arg_type.is_builtin_type) - and arg_type.entry.qualified_name in self._ext_types_with_pysize): + and arg_type.entry.qualified_name in self._ext_types_with_pysize): cfunc_name = 'Py_SIZE' else: return node @@ -2777,11 +2948,9 @@ class OptimizeBuiltinCalls(Visitor.NodeRefCleanupMixin, return node type_arg = args[0] if not obj.is_name or not type_arg.is_name: - # play safe - return node + return node # not a simple case if obj.type != Builtin.type_type or type_arg.type != Builtin.type_type: - # not a known type, play safe - return node + return node # not a known type if not type_arg.type_entry or not obj.type_entry: if obj.name != type_arg.name: return node @@ -3178,6 +3347,9 @@ class OptimizeBuiltinCalls(Visitor.NodeRefCleanupMixin, def _handle_simple_method_object___sub__(self, node, function, args, is_unbound_method): return self._optimise_num_binop('Subtract', node, function, args, is_unbound_method) + def _handle_simple_method_object___mul__(self, node, function, args, is_unbound_method): + return self._optimise_num_binop('Multiply', node, function, args, is_unbound_method) + def _handle_simple_method_object___eq__(self, node, function, args, is_unbound_method): return self._optimise_num_binop('Eq', node, function, args, is_unbound_method) @@ -3385,6 +3557,8 @@ class OptimizeBuiltinCalls(Visitor.NodeRefCleanupMixin, PyrexTypes.CFuncTypeArg("uchar", PyrexTypes.c_py_ucs4_type, None), ]) + # DISABLED: Return value can only be one character, which is not correct. + ''' def _inject_unicode_character_conversion(self, node, function, args, is_unbound_method): if is_unbound_method or len(args) != 1: return node @@ -3403,9 +3577,10 @@ class OptimizeBuiltinCalls(Visitor.NodeRefCleanupMixin, func_call = func_call.coerce_to_pyobject(self.current_env) return func_call - _handle_simple_method_unicode_lower = _inject_unicode_character_conversion - _handle_simple_method_unicode_upper = _inject_unicode_character_conversion - _handle_simple_method_unicode_title = _inject_unicode_character_conversion + #_handle_simple_method_unicode_lower = _inject_unicode_character_conversion + #_handle_simple_method_unicode_upper = _inject_unicode_character_conversion + #_handle_simple_method_unicode_title = _inject_unicode_character_conversion + ''' PyUnicode_Splitlines_func_type = PyrexTypes.CFuncType( Builtin.list_type, [ @@ -3784,7 +3959,7 @@ class OptimizeBuiltinCalls(Visitor.NodeRefCleanupMixin, if not stop: # use strlen() to find the string length, just as CPython would if not string_node.is_name: - string_node = UtilNodes.LetRefNode(string_node) # used twice + string_node = UtilNodes.LetRefNode(string_node) # used twice temps.append(string_node) stop = ExprNodes.PythonCapiCallNode( string_node.pos, "strlen", self.Pyx_strlen_func_type, @@ -4435,25 +4610,25 @@ class ConstantFolding(Visitor.VisitorTransform, SkipDeclarations): args = [] items = [] - def add(arg): + def add(parent, arg): if arg.is_dict_literal: - if items: - items[0].key_value_pairs.extend(arg.key_value_pairs) + if items and items[-1].reject_duplicates == arg.reject_duplicates: + items[-1].key_value_pairs.extend(arg.key_value_pairs) else: items.append(arg) - elif isinstance(arg, ExprNodes.MergedDictNode): + elif isinstance(arg, ExprNodes.MergedDictNode) and parent.reject_duplicates == arg.reject_duplicates: for child_arg in arg.keyword_args: - add(child_arg) + add(arg, child_arg) else: if items: - args.append(items[0]) + args.extend(items) del items[:] args.append(arg) for arg in node.keyword_args: - add(arg) + add(node, arg) if items: - args.append(items[0]) + args.extend(items) if len(args) == 1: arg = args[0] @@ -4542,22 +4717,20 @@ class ConstantFolding(Visitor.VisitorTransform, SkipDeclarations): cascades = [[node.operand1]] final_false_result = [] - def split_cascades(cmp_node): + cmp_node = node + while cmp_node is not None: if cmp_node.has_constant_result(): if not cmp_node.constant_result: # False => short-circuit final_false_result.append(self._bool_node(cmp_node, False)) - return + break else: # True => discard and start new cascade cascades.append([cmp_node.operand2]) else: # not constant => append to current cascade cascades[-1].append(cmp_node) - if cmp_node.cascade: - split_cascades(cmp_node.cascade) - - split_cascades(node) + cmp_node = cmp_node.cascade cmp_nodes = [] for cascade in cascades: @@ -4703,6 +4876,30 @@ class ConstantFolding(Visitor.VisitorTransform, SkipDeclarations): return None return node + def visit_GILStatNode(self, node): + self.visitchildren(node) + if node.condition is None: + return node + + if node.condition.has_constant_result(): + # Condition is True - Modify node to be a normal + # GILStatNode with condition=None + if node.condition.constant_result: + node.condition = None + + # Condition is False - the body of the GILStatNode + # should run without changing the state of the gil + # return the body of the GILStatNode + else: + return node.body + + # If condition is not constant we keep the GILStatNode as it is. + # Either it will later become constant (e.g. a `numeric is int` + # expression in a fused type function) and then when ConstantFolding + # runs again it will be handled or a later transform (i.e. GilCheck) + # will raise an error + return node + # in the future, other nodes can have their own handler method here # that can replace them with a constant result node @@ -4719,6 +4916,7 @@ class FinalOptimizePhase(Visitor.EnvTransform, Visitor.NodeRefCleanupMixin): - isinstance -> typecheck for cdef types - eliminate checks for None and/or types that became redundant after tree changes - eliminate useless string formatting steps + - inject branch hints for unlikely if-cases that only raise exceptions - replace Python function calls that look like method calls by a faster PyMethodCallNode """ in_loop = False @@ -4817,6 +5015,48 @@ class FinalOptimizePhase(Visitor.EnvTransform, Visitor.NodeRefCleanupMixin): self.in_loop = old_val return node + def visit_IfStatNode(self, node): + """Assign 'unlikely' branch hints to if-clauses that only raise exceptions. + """ + self.visitchildren(node) + last_non_unlikely_clause = None + for i, if_clause in enumerate(node.if_clauses): + self._set_ifclause_branch_hint(if_clause, if_clause.body) + if not if_clause.branch_hint: + last_non_unlikely_clause = if_clause + if node.else_clause and last_non_unlikely_clause: + # If the 'else' clause is 'unlikely', then set the preceding 'if' clause to 'likely' to reflect that. + self._set_ifclause_branch_hint(last_non_unlikely_clause, node.else_clause, inverse=True) + return node + + def _set_ifclause_branch_hint(self, clause, statements_node, inverse=False): + """Inject a branch hint if the if-clause unconditionally leads to a 'raise' statement. + """ + if not statements_node.is_terminator: + return + # Allow simple statements, but no conditions, loops, etc. + non_branch_nodes = ( + Nodes.ExprStatNode, + Nodes.AssignmentNode, + Nodes.AssertStatNode, + Nodes.DelStatNode, + Nodes.GlobalNode, + Nodes.NonlocalNode, + ) + statements = [statements_node] + for next_node_pos, node in enumerate(statements, 1): + if isinstance(node, Nodes.GILStatNode): + statements.insert(next_node_pos, node.body) + continue + if isinstance(node, Nodes.StatListNode): + statements[next_node_pos:next_node_pos] = node.stats + continue + if not isinstance(node, non_branch_nodes): + if next_node_pos == len(statements) and isinstance(node, (Nodes.RaiseStatNode, Nodes.ReraiseStatNode)): + # Anything that unconditionally raises exceptions at the end should be considered unlikely. + clause.branch_hint = 'likely' if inverse else 'unlikely' + break + class ConsolidateOverflowCheck(Visitor.CythonTransform): """ diff --git a/Cython/Compiler/Options.py b/Cython/Compiler/Options.py index 48695dbfc..f20239321 100644 --- a/Cython/Compiler/Options.py +++ b/Cython/Compiler/Options.py @@ -4,6 +4,10 @@ from __future__ import absolute_import +import os + +from Cython import Utils + class ShouldBeFromDirective(object): @@ -25,15 +29,14 @@ class ShouldBeFromDirective(object): raise RuntimeError(repr(self)) def __repr__(self): - return ( - "Illegal access of '%s' from Options module rather than directive '%s'" - % (self.options_name, self.directive_name)) + return "Illegal access of '%s' from Options module rather than directive '%s'" % ( + self.options_name, self.directive_name) """ The members of this module are documented using autodata in Cython/docs/src/reference/compilation.rst. -See http://www.sphinx-doc.org/en/master/ext/autodoc.html#directive-autoattribute +See https://www.sphinx-doc.org/en/master/usage/extensions/autodoc.html#directive-autoattribute for how autodata works. Descriptions of those members should start with a #: Donc forget to keep the docs in sync by removing and adding @@ -48,11 +51,6 @@ docstrings = True #: Embed the source code position in the docstrings of functions and classes. embed_pos_in_docstring = False -#: Copy the original source code line by line into C code comments -#: in the generated code file to help with understanding the output. -#: This is also required for coverage analysis. -emit_code_comments = True - # undocumented pre_import = None @@ -168,8 +166,19 @@ def get_directive_defaults(): _directive_defaults[old_option.directive_name] = value return _directive_defaults +def copy_inherited_directives(outer_directives, **new_directives): + # A few directives are not copied downwards and this function removes them. + # For example, test_assert_path_exists and test_fail_if_path_exists should not be inherited + # otherwise they can produce very misleading test failures + new_directives_out = dict(outer_directives) + for name in ('test_assert_path_exists', 'test_fail_if_path_exists'): + new_directives_out.pop(name, None) + new_directives_out.update(new_directives) + return new_directives_out + # Declare compiler directives _directive_defaults = { + 'binding': True, # was False before 3.0 'boundscheck' : True, 'nonecheck' : False, 'initializedcheck' : True, @@ -178,10 +187,10 @@ _directive_defaults = { 'auto_pickle': None, 'cdivision': False, # was True before 0.12 'cdivision_warnings': False, - 'c_api_binop_methods': True, + 'c_api_binop_methods': False, # was True before 3.0 'overflowcheck': False, 'overflowcheck.fold': True, - 'always_allow_keywords': False, + 'always_allow_keywords': True, 'allow_none_for_extension_args': True, 'wraparound' : True, 'ccomplex' : False, # use C99/C++ for complex types and arith @@ -239,8 +248,6 @@ _directive_defaults = { 'test_fail_if_path_exists' : [], # experimental, subject to change - 'binding': None, - 'formal_grammar': False, } @@ -318,6 +325,7 @@ directive_types = { 'freelist': int, 'c_string_type': one_of('bytes', 'bytearray', 'str', 'unicode'), 'c_string_encoding': normalise_encoding_name, + 'trashcan': bool, } for key, val in _directive_defaults.items(): @@ -360,6 +368,7 @@ directive_scopes = { # defaults to available everywhere 'np_pythran': ('module',), 'fast_gil': ('module',), 'iterable_coroutine': ('module', 'function'), + 'trashcan' : ('cclass',), } @@ -548,3 +557,184 @@ def parse_compile_time_env(s, current_settings=None): name, value = [s.strip() for s in item.split('=', 1)] result[name] = parse_variable_value(value) return result + + +# ------------------------------------------------------------------------ +# CompilationOptions are constructed from user input and are the `option` +# object passed throughout the compilation pipeline. + +class CompilationOptions(object): + r""" + See default_options at the end of this module for a list of all possible + options and CmdLine.usage and CmdLine.parse_command_line() for their + meaning. + """ + def __init__(self, defaults=None, **kw): + self.include_path = [] + if defaults: + if isinstance(defaults, CompilationOptions): + defaults = defaults.__dict__ + else: + defaults = default_options + + options = dict(defaults) + options.update(kw) + + # let's assume 'default_options' contains a value for most known compiler options + # and validate against them + unknown_options = set(options) - set(default_options) + # ignore valid options that are not in the defaults + unknown_options.difference_update(['include_path']) + if unknown_options: + message = "got unknown compilation option%s, please remove: %s" % ( + 's' if len(unknown_options) > 1 else '', + ', '.join(unknown_options)) + raise ValueError(message) + + directive_defaults = get_directive_defaults() + directives = dict(options['compiler_directives']) # copy mutable field + # check for invalid directives + unknown_directives = set(directives) - set(directive_defaults) + if unknown_directives: + message = "got unknown compiler directive%s: %s" % ( + 's' if len(unknown_directives) > 1 else '', + ', '.join(unknown_directives)) + raise ValueError(message) + options['compiler_directives'] = directives + if directives.get('np_pythran', False) and not options['cplus']: + import warnings + warnings.warn("C++ mode forced when in Pythran mode!") + options['cplus'] = True + if 'language_level' in directives and 'language_level' not in kw: + options['language_level'] = directives['language_level'] + elif not options.get('language_level'): + options['language_level'] = directive_defaults.get('language_level') + if 'formal_grammar' in directives and 'formal_grammar' not in kw: + options['formal_grammar'] = directives['formal_grammar'] + if options['cache'] is True: + options['cache'] = os.path.join(Utils.get_cython_cache_dir(), 'compiler') + + self.__dict__.update(options) + + def configure_language_defaults(self, source_extension): + if source_extension == 'py': + if self.compiler_directives.get('binding') is None: + self.compiler_directives['binding'] = True + + def get_fingerprint(self): + r""" + Return a string that contains all the options that are relevant for cache invalidation. + """ + # Collect only the data that can affect the generated file(s). + data = {} + + for key, value in self.__dict__.items(): + if key in ['show_version', 'errors_to_stderr', 'verbose', 'quiet']: + # verbosity flags have no influence on the compilation result + continue + elif key in ['output_file', 'output_dir']: + # ignore the exact name of the output file + continue + elif key in ['timestamps']: + # the cache cares about the content of files, not about the timestamps of sources + continue + elif key in ['cache']: + # hopefully caching has no influence on the compilation result + continue + elif key in ['compiler_directives']: + # directives passed on to the C compiler do not influence the generated C code + continue + elif key in ['include_path']: + # this path changes which headers are tracked as dependencies, + # it has no influence on the generated C code + continue + elif key in ['working_path']: + # this path changes where modules and pxd files are found; + # their content is part of the fingerprint anyway, their + # absolute path does not matter + continue + elif key in ['create_extension']: + # create_extension() has already mangled the options, e.g., + # embedded_metadata, when the fingerprint is computed so we + # ignore it here. + continue + elif key in ['build_dir']: + # the (temporary) directory where we collect dependencies + # has no influence on the C output + continue + elif key in ['use_listing_file', 'generate_pxi', 'annotate', 'annotate_coverage_xml']: + # all output files are contained in the cache so the types of + # files generated must be part of the fingerprint + data[key] = value + elif key in ['formal_grammar', 'evaluate_tree_assertions']: + # these bits can change whether compilation to C passes/fails + data[key] = value + elif key in ['embedded_metadata', 'emit_linenums', + 'c_line_in_traceback', 'gdb_debug', + 'relative_path_in_code_position_comments']: + # the generated code contains additional bits when these are set + data[key] = value + elif key in ['cplus', 'language_level', 'compile_time_env', 'np_pythran']: + # assorted bits that, e.g., influence the parser + data[key] = value + elif key == ['capi_reexport_cincludes']: + if self.capi_reexport_cincludes: + # our caching implementation does not yet include fingerprints of all the header files + raise NotImplementedError('capi_reexport_cincludes is not compatible with Cython caching') + elif key == ['common_utility_include_dir']: + if self.common_utility_include_dir: + raise NotImplementedError('common_utility_include_dir is not compatible with Cython caching yet') + else: + # any unexpected option should go into the fingerprint; it's better + # to recompile than to return incorrect results from the cache. + data[key] = value + + def to_fingerprint(item): + r""" + Recursively turn item into a string, turning dicts into lists with + deterministic ordering. + """ + if isinstance(item, dict): + item = sorted([(repr(key), to_fingerprint(value)) for key, value in item.items()]) + return repr(item) + + return to_fingerprint(data) + + +# ------------------------------------------------------------------------ +# +# Set the default options depending on the platform +# +# ------------------------------------------------------------------------ + +default_options = dict( + show_version=0, + use_listing_file=0, + errors_to_stderr=1, + cplus=0, + output_file=None, + annotate=None, + annotate_coverage_xml=None, + generate_pxi=0, + capi_reexport_cincludes=0, + working_path="", + timestamps=None, + verbose=0, + quiet=0, + compiler_directives={}, + embedded_metadata={}, + evaluate_tree_assertions=False, + emit_linenums=False, + relative_path_in_code_position_comments=True, + c_line_in_traceback=True, + language_level=None, # warn but default to 2 + formal_grammar=False, + gdb_debug=False, + compile_time_env=None, + common_utility_include_dir=None, + output_dir=None, + build_dir=None, + cache=None, + create_extension=None, + np_pythran=False +) diff --git a/Cython/Compiler/ParseTreeTransforms.pxd b/Cython/Compiler/ParseTreeTransforms.pxd index 2c17901fa..4026429ac 100644 --- a/Cython/Compiler/ParseTreeTransforms.pxd +++ b/Cython/Compiler/ParseTreeTransforms.pxd @@ -1,5 +1,4 @@ - -from __future__ import absolute_import +# cython: language_level=3 cimport cython diff --git a/Cython/Compiler/ParseTreeTransforms.py b/Cython/Compiler/ParseTreeTransforms.py index e7a33341b..330208840 100644 --- a/Cython/Compiler/ParseTreeTransforms.py +++ b/Cython/Compiler/ParseTreeTransforms.py @@ -169,7 +169,6 @@ class PostParse(ScopeTrackingTransform): reorganization that can be refactored into this transform if a more pure Abstract Syntax Tree is wanted. """ - def __init__(self, context): super(PostParse, self).__init__(context) self.specialattribute_handlers = { @@ -244,7 +243,7 @@ class PostParse(ScopeTrackingTransform): if decl is not declbase: raise PostParseError(decl.pos, ERR_INVALID_SPECIALATTR_TYPE) handler(decl) - continue # Remove declaration + continue # Remove declaration raise PostParseError(decl.pos, ERR_CDEF_INCLASS) first_assignment = self.scope_type != 'module' stats.append(Nodes.SingleAssignmentNode(node.pos, @@ -347,6 +346,23 @@ class PostParse(ScopeTrackingTransform): self.visitchildren(node) return node + def visit_AssertStatNode(self, node): + """Extract the exception raising into a RaiseStatNode to simplify GIL handling. + """ + if node.exception is None: + node.exception = Nodes.RaiseStatNode( + node.pos, + exc_type=ExprNodes.NameNode(node.pos, name=EncodedString("AssertionError")), + exc_value=node.value, + exc_tb=None, + cause=None, + builtin_exc_name="AssertionError", + wrap_tuple_value=True, + ) + node.value = None + self.visitchildren(node) + return node + def eliminate_rhs_duplicates(expr_list_list, ref_node_sequence): """Replace rhs items by LetRefNodes if they appear more than once. @@ -417,7 +433,7 @@ def sort_common_subsequences(items): return b.is_sequence_constructor and contains(b.args, a) for pos, item in enumerate(items): - key = item[1] # the ResultRefNode which has already been injected into the sequences + key = item[1] # the ResultRefNode which has already been injected into the sequences new_pos = pos for i in range(pos-1, -1, -1): if lower_than(key, items[i][0]): @@ -447,7 +463,7 @@ def flatten_parallel_assignments(input, output): # recursively, so that nested structures get matched as well. rhs = input[-1] if (not (rhs.is_sequence_constructor or isinstance(rhs, ExprNodes.UnicodeNode)) - or not sum([lhs.is_sequence_constructor for lhs in input[:-1]])): + or not sum([lhs.is_sequence_constructor for lhs in input[:-1]])): output.append(input) return @@ -531,7 +547,7 @@ def map_starred_assignment(lhs_targets, starred_assignments, lhs_args, rhs_args) targets.append(expr) # the starred target itself, must be assigned a (potentially empty) list - target = lhs_args[starred].target # unpack starred node + target = lhs_args[starred].target # unpack starred node starred_rhs = rhs_args[starred:] if lhs_remaining: starred_rhs = starred_rhs[:-lhs_remaining] @@ -577,19 +593,19 @@ class PxdPostParse(CythonTransform, SkipDeclarations): err = self.ERR_INLINE_ONLY if (isinstance(node, Nodes.DefNode) and self.scope_type == 'cclass' - and node.name in ('__getbuffer__', '__releasebuffer__')): - err = None # allow these slots + and node.name in ('__getbuffer__', '__releasebuffer__')): + err = None # allow these slots if isinstance(node, Nodes.CFuncDefNode): if (u'inline' in node.modifiers and - self.scope_type in ('pxd', 'cclass')): + self.scope_type in ('pxd', 'cclass')): node.inline_in_pxd = True if node.visibility != 'private': err = self.ERR_NOGO_WITH_INLINE % node.visibility elif node.api: err = self.ERR_NOGO_WITH_INLINE % 'api' else: - err = None # allow inline function + err = None # allow inline function else: err = self.ERR_INLINE_ONLY @@ -628,6 +644,9 @@ class InterpretCompilerDirectives(CythonTransform): - Command-line arguments overriding these - @cython.directivename decorators - with cython.directivename: statements + - replaces "cython.compiled" with BoolNode(value=True) + allowing unreachable blocks to be removed at a fairly early stage + before cython typing rules are forced on applied This transform is responsible for interpreting these various sources and store the directive in two ways: @@ -666,17 +685,19 @@ class InterpretCompilerDirectives(CythonTransform): 'operator.comma' : ExprNodes.c_binop_constructor(','), } - special_methods = set(['declare', 'union', 'struct', 'typedef', - 'sizeof', 'cast', 'pointer', 'compiled', - 'NULL', 'fused_type', 'parallel']) + special_methods = { + 'declare', 'union', 'struct', 'typedef', + 'sizeof', 'cast', 'pointer', 'compiled', + 'NULL', 'fused_type', 'parallel', + } special_methods.update(unop_method_nodes) - valid_parallel_directives = set([ + valid_parallel_directives = { "parallel", "prange", "threadid", #"threadsavailable", - ]) + } def __init__(self, context, compilation_directive_defaults): super(InterpretCompilerDirectives, self).__init__(context) @@ -844,6 +865,16 @@ class InterpretCompilerDirectives(CythonTransform): directive = self.directive_names.get(node.name) if directive is not None: node.cython_attribute = directive + if node.as_cython_attribute() == "compiled": + return ExprNodes.BoolNode(node.pos, value=True) # replace early so unused branches can be dropped + # before they have a chance to cause compile-errors + return node + + def visit_AttributeNode(self, node): + self.visitchildren(node) + if node.as_cython_attribute() == "compiled": + return ExprNodes.BoolNode(node.pos, value=True) # replace early so unused branches can be dropped + # before they have a chance to cause compile-errors return node def visit_NewExprNode(self, node): @@ -963,8 +994,7 @@ class InterpretCompilerDirectives(CythonTransform): return self.visit_Node(node) old_directives = self.directives - new_directives = dict(old_directives) - new_directives.update(directives) + new_directives = Options.copy_inherited_directives(old_directives, **directives) if new_directives == old_directives: return self.visit_Node(node) @@ -1031,7 +1061,7 @@ class InterpretCompilerDirectives(CythonTransform): else: realdecs.append(dec) if realdecs and (scope_name == 'cclass' or - isinstance(node, (Nodes.CFuncDefNode, Nodes.CClassDefNode, Nodes.CVarDefNode))): + isinstance(node, (Nodes.CClassDefNode, Nodes.CVarDefNode))): raise PostParseError(realdecs[0].pos, "Cdef functions/classes cannot take arbitrary decorators.") node.decorators = realdecs[::-1] + both[::-1] # merge or override repeated directives @@ -1326,16 +1356,16 @@ class DecoratorTransform(ScopeTrackingTransform, SkipDeclarations): _properties = None _map_property_attribute = { - 'getter': '__get__', - 'setter': '__set__', - 'deleter': '__del__', + 'getter': EncodedString('__get__'), + 'setter': EncodedString('__set__'), + 'deleter': EncodedString('__del__'), }.get def visit_CClassDefNode(self, node): if self._properties is None: self._properties = [] self._properties.append({}) - super(DecoratorTransform, self).visit_CClassDefNode(node) + node = super(DecoratorTransform, self).visit_CClassDefNode(node) self._properties.pop() return node @@ -1345,6 +1375,25 @@ class DecoratorTransform(ScopeTrackingTransform, SkipDeclarations): warning(node.pos, "'property %s:' syntax is deprecated, use '@property'" % node.name, level) return node + def visit_CFuncDefNode(self, node): + node = self.visit_FuncDefNode(node) + if self.scope_type != 'cclass' or self.scope_node.visibility != "extern" or not node.decorators: + return node + + ret_node = node + decorator_node = self._find_property_decorator(node) + if decorator_node: + if decorator_node.decorator.is_name: + name = node.declared_name() + if name: + ret_node = self._add_property(node, name, decorator_node) + else: + error(decorator_node.pos, "C property decorator can only be @property") + + if node.decorators: + return self._reject_decorated_property(node, node.decorators[0]) + return ret_node + def visit_DefNode(self, node): scope_type = self.scope_type node = self.visit_FuncDefNode(node) @@ -1352,28 +1401,12 @@ class DecoratorTransform(ScopeTrackingTransform, SkipDeclarations): return node # transform @property decorators - properties = self._properties[-1] - for decorator_node in node.decorators[::-1]: + decorator_node = self._find_property_decorator(node) + if decorator_node is not None: decorator = decorator_node.decorator - if decorator.is_name and decorator.name == 'property': - if len(node.decorators) > 1: - return self._reject_decorated_property(node, decorator_node) - name = node.name - node.name = EncodedString('__get__') - node.decorators.remove(decorator_node) - stat_list = [node] - if name in properties: - prop = properties[name] - prop.pos = node.pos - prop.doc = node.doc - prop.body.stats = stat_list - return [] - prop = Nodes.PropertyNode(node.pos, name=name) - prop.doc = node.doc - prop.body = Nodes.StatListNode(node.pos, stats=stat_list) - properties[name] = prop - return [prop] - elif decorator.is_attribute and decorator.obj.name in properties: + if decorator.is_name: + return self._add_property(node, node.name, decorator_node) + else: handler_name = self._map_property_attribute(decorator.attribute) if handler_name: if decorator.obj.name != node.name: @@ -1384,7 +1417,7 @@ class DecoratorTransform(ScopeTrackingTransform, SkipDeclarations): elif len(node.decorators) > 1: return self._reject_decorated_property(node, decorator_node) else: - return self._add_to_property(properties, node, handler_name, decorator_node) + return self._add_to_property(node, handler_name, decorator_node) # we clear node.decorators, so we need to set the # is_staticmethod/is_classmethod attributes now @@ -1399,6 +1432,18 @@ class DecoratorTransform(ScopeTrackingTransform, SkipDeclarations): node.decorators = None return self.chain_decorators(node, decs, node.name) + def _find_property_decorator(self, node): + properties = self._properties[-1] + for decorator_node in node.decorators[::-1]: + decorator = decorator_node.decorator + if decorator.is_name and decorator.name == 'property': + # @property + return decorator_node + elif decorator.is_attribute and decorator.obj.name in properties: + # @prop.setter etc. + return decorator_node + return None + @staticmethod def _reject_decorated_property(node, decorator_node): # restrict transformation to outermost decorator as wrapped properties will probably not work @@ -1407,9 +1452,42 @@ class DecoratorTransform(ScopeTrackingTransform, SkipDeclarations): error(deco.pos, "Property methods with additional decorators are not supported") return node - @staticmethod - def _add_to_property(properties, node, name, decorator): + def _add_property(self, node, name, decorator_node): + if len(node.decorators) > 1: + return self._reject_decorated_property(node, decorator_node) + node.decorators.remove(decorator_node) + properties = self._properties[-1] + is_cproperty = isinstance(node, Nodes.CFuncDefNode) + body = Nodes.StatListNode(node.pos, stats=[node]) + if is_cproperty: + if name in properties: + error(node.pos, "C property redeclared") + if 'inline' not in node.modifiers: + error(node.pos, "C property method must be declared 'inline'") + prop = Nodes.CPropertyNode(node.pos, doc=node.doc, name=name, body=body) + elif name in properties: + prop = properties[name] + if prop.is_cproperty: + error(node.pos, "C property redeclared") + else: + node.name = EncodedString("__get__") + prop.pos = node.pos + prop.doc = node.doc + prop.body.stats = [node] + return None + else: + node.name = EncodedString("__get__") + prop = Nodes.PropertyNode( + node.pos, name=name, doc=node.doc, body=body) + properties[name] = prop + return prop + + def _add_to_property(self, node, name, decorator): + properties = self._properties[-1] prop = properties[node.name] + if prop.is_cproperty: + error(node.pos, "C property redeclared") + return None node.name = name node.decorators.remove(decorator) stats = prop.body.stats @@ -1419,7 +1497,7 @@ class DecoratorTransform(ScopeTrackingTransform, SkipDeclarations): break else: stats.append(node) - return [] + return None @staticmethod def chain_decorators(node, decorators, name): @@ -1497,6 +1575,10 @@ class CnameDirectivesTransform(CythonTransform, SkipDeclarations): class ForwardDeclareTypes(CythonTransform): + """ + Declare all global cdef names that we allow referencing in other places, + before declaring everything (else) in source code order. + """ def visit_CompilerDirectivesNode(self, node): env = self.module_scope @@ -1540,6 +1622,14 @@ class ForwardDeclareTypes(CythonTransform): entry.type.get_all_specialized_function_types() return node + def visit_FuncDefNode(self, node): + # no traversal needed + return node + + def visit_PyClassDefNode(self, node): + # no traversal needed + return node + class AnalyseDeclarationsTransform(EnvTransform): @@ -1631,8 +1721,8 @@ if VALUE is not None: if stats: node.body.stats += stats if (node.visibility != 'extern' - and not node.scope.lookup('__reduce__') - and not node.scope.lookup('__reduce_ex__')): + and not node.scope.lookup('__reduce__') + and not node.scope.lookup('__reduce_ex__')): self._inject_pickle_methods(node) return node @@ -1686,9 +1776,9 @@ if VALUE is not None: pickle_func = TreeFragment(u""" def __reduce_cython__(self): - raise TypeError("%(msg)s") + raise TypeError, "%(msg)s" def __setstate_cython__(self, __pyx_state): - raise TypeError("%(msg)s") + raise TypeError, "%(msg)s" """ % {'msg': msg}, level='c_class', pipeline=[NormalizeTree(None)]).substitute({}) pickle_func.analyse_declarations(node.scope) @@ -1701,8 +1791,8 @@ if VALUE is not None: e.type.create_to_py_utility_code(env) e.type.create_from_py_utility_code(env) all_members_names = sorted([e.name for e in all_members]) - checksum = '0x%s' % hashlib.md5(' '.join(all_members_names).encode('utf-8')).hexdigest()[:7] - unpickle_func_name = '__pyx_unpickle_%s' % node.class_name + checksum = '0x%s' % hashlib.sha1(' '.join(all_members_names).encode('utf-8')).hexdigest()[:7] + unpickle_func_name = '__pyx_unpickle_%s' % node.punycode_class_name # TODO(robertwb): Move the state into the third argument # so it can be pickled *after* self is memoized. @@ -1712,7 +1802,7 @@ if VALUE is not None: cdef object __pyx_result if __pyx_checksum != %(checksum)s: from pickle import PickleError as __pyx_PickleError - raise __pyx_PickleError("Incompatible checksums (%%s vs %(checksum)s = (%(members)s))" %% __pyx_checksum) + raise __pyx_PickleError, "Incompatible checksums (%%s vs %(checksum)s = (%(members)s))" %% __pyx_checksum __pyx_result = %(class_name)s.__new__(__pyx_type) if __pyx_state is not None: %(unpickle_func_name)s__set_state(<%(class_name)s> __pyx_result, __pyx_state) @@ -1780,8 +1870,8 @@ if VALUE is not None: for decorator in old_decorators: func = decorator.decorator if (not func.is_name or - func.name not in ('staticmethod', 'classmethod') or - env.lookup_here(func.name)): + func.name not in ('staticmethod', 'classmethod') or + env.lookup_here(func.name)): # not a static or classmethod decorators.append(decorator) @@ -1801,6 +1891,8 @@ if VALUE is not None: node.stats.insert(0, node.py_func) node.py_func = self.visit(node.py_func) node.update_fused_defnode_entry(env) + # For the moment, fused functions do not support METH_FASTCALL + node.py_func.entry.signature.use_fastcall = False pycfunc = ExprNodes.PyCFunctionNode.from_defnode(node.py_func, binding=True) pycfunc = ExprNodes.ProxyNode(pycfunc.coerce_to_temp(env)) node.resulting_fused_function = pycfunc @@ -1843,19 +1935,6 @@ if VALUE is not None: return node - def _handle_nogil_cleanup(self, lenv, node): - "Handle cleanup for 'with gil' blocks in nogil functions." - if lenv.nogil and lenv.has_with_gil_block: - # Acquire the GIL for cleanup in 'nogil' functions, by wrapping - # the entire function body in try/finally. - # The corresponding release will be taken care of by - # Nodes.FuncDefNode.generate_function_definitions() - node.body = Nodes.NogilTryFinallyStatNode( - node.body.pos, - body=node.body, - finally_clause=Nodes.EnsureGILNode(node.body.pos), - finally_except_clause=Nodes.EnsureGILNode(node.body.pos)) - def _handle_fused(self, node): if node.is_generator and node.has_fused_arguments: node.has_fused_arguments = False @@ -1887,6 +1966,8 @@ if VALUE is not None: for var, type_node in node.directive_locals.items(): if not lenv.lookup_here(var): # don't redeclare args type = type_node.analyse_as_type(lenv) + if type and type.is_fused and lenv.fused_to_specific: + type = type.specialize(lenv.fused_to_specific) if type: lenv.declare_var(var, type, type_node.pos) else: @@ -1896,7 +1977,6 @@ if VALUE is not None: node = self._create_fused_function(env, node) else: node.body.analyse_declarations(lenv) - self._handle_nogil_cleanup(lenv, node) self._super_visit_FuncDefNode(node) self.seen_vars_stack.pop() @@ -1990,7 +2070,7 @@ if VALUE is not None: # (so it can't happen later). # Note that we don't return the original node, as it is # never used after this phase. - if True: # private (default) + if True: # private (default) return None self_value = ExprNodes.AttributeNode( @@ -2082,8 +2162,8 @@ if VALUE is not None: if node.name in self.seen_vars_stack[-1]: entry = self.current_env().lookup(node.name) if (entry is None or entry.visibility != 'extern' - and not entry.scope.is_c_class_scope): - warning(node.pos, "cdef variable '%s' declared after it is used" % node.name, 2) + and not entry.scope.is_c_class_scope): + error(node.pos, "cdef variable '%s' declared after it is used" % node.name) self.visitchildren(node) return node @@ -2096,7 +2176,7 @@ if VALUE is not None: child_node = self.visit(node.node) if not child_node: return None - if type(child_node) is list: # Assignment synthesized + if type(child_node) is list: # Assignment synthesized node.child_node = child_node[0] return [node] + child_node[1:] node.node = child_node @@ -2198,7 +2278,7 @@ class CalculateQualifiedNamesTransform(EnvTransform): def visit_ClassDefNode(self, node): orig_qualified_name = self.qualified_name[:] entry = (getattr(node, 'entry', None) or # PyClass - self.current_env().lookup_here(node.name)) # CClass + self.current_env().lookup_here(node.target.name)) # CClass self._append_entry(entry) self._super_visit_ClassDefNode(node) self.qualified_name = orig_qualified_name @@ -2358,12 +2438,12 @@ class AdjustDefByDirectives(CythonTransform, SkipDeclarations): return_type_node = self.directives.get('returns') if return_type_node is None and self.directives['annotation_typing']: return_type_node = node.return_type_annotation - # for Python anntations, prefer safe exception handling by default + # for Python annotations, prefer safe exception handling by default if return_type_node is not None and except_val is None: except_val = (None, True) # except * elif except_val is None: - # backward compatible default: no exception check - except_val = (None, False) + # backward compatible default: no exception check, unless there's also a "@returns" declaration + except_val = (None, True if return_type_node else False) if 'ccall' in self.directives: node = node.as_cfunction( overridable=True, modifiers=modifiers, nogil=nogil, @@ -2456,7 +2536,7 @@ class AlignFunctionDefinitions(CythonTransform): return None node = node.as_cfunction(pxd_def) elif (self.scope.is_module_scope and self.directives['auto_cpdef'] - and not node.name in self.imported_names + and node.name not in self.imported_names and node.is_cdef_func_compatible()): # FIXME: cpdef-ing should be done in analyse_declarations() node = node.as_cfunction(scope=self.scope) @@ -2705,7 +2785,7 @@ class CreateClosureClasses(CythonTransform): if not node.py_cfunc_node: raise InternalError("DefNode does not have assignment node") inner_node = node.py_cfunc_node - inner_node.needs_self_code = False + inner_node.needs_closure_code = False node.needs_outer_scope = False if node.is_generator: @@ -2724,6 +2804,7 @@ class CreateClosureClasses(CythonTransform): as_name = '%s_%s' % ( target_module_scope.next_id(Naming.closure_class_prefix), node.entry.cname.replace('.','__')) + as_name = EncodedString(as_name) entry = target_module_scope.declare_c_class( name=as_name, pos=node.pos, defining=True, @@ -2803,20 +2884,20 @@ class InjectGilHandling(VisitorTransform, SkipDeclarations): Must run before the AnalyseDeclarationsTransform to make sure the GILStatNodes get set up, parallel sections know that the GIL is acquired inside of them, etc. """ - def __call__(self, root): - self.nogil = False - return super(InjectGilHandling, self).__call__(root) + nogil = False # special node handling - def visit_RaiseStatNode(self, node): - """Allow raising exceptions in nogil sections by wrapping them in a 'with gil' block.""" + def _inject_gil_in_nogil(self, node): + """Allow the (Python statement) node in nogil sections by wrapping it in a 'with gil' block.""" if self.nogil: node = Nodes.GILStatNode(node.pos, state='gil', body=node) return node + visit_RaiseStatNode = _inject_gil_in_nogil + visit_PrintStatNode = _inject_gil_in_nogil # sadly, not the function + # further candidates: - # def visit_AssertStatNode(self, node): # def visit_ReraiseStatNode(self, node): # nogil tracking @@ -2877,7 +2958,7 @@ class GilCheck(VisitorTransform): self.visitchildren(node, outer_attrs) self.nogil = gil_state - self.visitchildren(node, exclude=outer_attrs) + self.visitchildren(node, attrs=None, exclude=outer_attrs) self.nogil = was_nogil def visit_FuncDefNode(self, node): @@ -2899,6 +2980,12 @@ class GilCheck(VisitorTransform): return node def visit_GILStatNode(self, node): + if node.condition is not None: + error(node.condition.pos, + "Non-constant condition in a " + "`with %s(<condition>)` statement" % node.state) + return node + if self.nogil and node.nogil_check: node.nogil_check() @@ -2996,9 +3083,7 @@ class TransformBuiltinMethods(EnvTransform): def visit_cython_attribute(self, node): attribute = node.as_cython_attribute() if attribute: - if attribute == u'compiled': - node = ExprNodes.BoolNode(node.pos, value=True) - elif attribute == u'__version__': + if attribute == u'__version__': from .. import __version__ as version node = ExprNodes.StringNode(node.pos, value=EncodedString(version)) elif attribute == u'NULL': @@ -3043,9 +3128,9 @@ class TransformBuiltinMethods(EnvTransform): error(self.pos, "Builtin 'vars()' called with wrong number of args, expected 0-1, got %d" % len(node.args)) if len(node.args) > 0: - return node # nothing to do + return node # nothing to do return ExprNodes.LocalsExprNode(pos, self.current_scope_node(), lenv) - else: # dir() + else: # dir() if len(node.args) > 1: error(self.pos, "Builtin 'dir()' called with wrong number of args, expected 0-1, got %d" % len(node.args)) @@ -3080,8 +3165,8 @@ class TransformBuiltinMethods(EnvTransform): def _inject_eval(self, node, func_name): lenv = self.current_env() - entry = lenv.lookup_here(func_name) - if entry or len(node.args) != 1: + entry = lenv.lookup(func_name) + if len(node.args) != 1 or (entry and not entry.is_builtin): return node # Inject globals and locals node.args.append(ExprNodes.GlobalsExprNode(node.pos)) @@ -3098,8 +3183,7 @@ class TransformBuiltinMethods(EnvTransform): return node # Inject no-args super def_node = self.current_scope_node() - if (not isinstance(def_node, Nodes.DefNode) or not def_node.args or - len(self.env_stack) < 2): + if not isinstance(def_node, Nodes.DefNode) or not def_node.args or len(self.env_stack) < 2: return node class_node, class_scope = self.env_stack[-2] if class_scope.is_py_class_scope: @@ -3238,10 +3322,17 @@ class ReplaceFusedTypeChecks(VisitorTransform): self.visitchildren(node) return self.transform(node) + def visit_GILStatNode(self, node): + """ + Fold constant condition of GILStatNode. + """ + self.visitchildren(node) + return self.transform(node) + def visit_PrimaryCmpNode(self, node): with Errors.local_errors(ignore=True): - type1 = node.operand1.analyse_as_type(self.local_scope) - type2 = node.operand2.analyse_as_type(self.local_scope) + type1 = node.operand1.analyse_as_type(self.local_scope) + type2 = node.operand2.analyse_as_type(self.local_scope) if type1 and type2: false_node = ExprNodes.BoolNode(node.pos, value=False) @@ -3377,9 +3468,14 @@ class DebugTransform(CythonTransform): else: pf_cname = node.py_func.entry.func_cname + # For functions defined using def, cname will be pyfunc_cname=__pyx_pf_* + # For functions defined using cpdef or cdef, cname will be func_cname=__pyx_f_* + # In all cases, cname will be the name of the function containing the actual code + cname = node.entry.pyfunc_cname or node.entry.func_cname + attrs = dict( name=node.entry.name or getattr(node, 'name', '<unknown>'), - cname=node.entry.func_cname, + cname=cname, pf_cname=pf_cname, qualified_name=node.local_scope.qualified_name, lineno=str(node.pos[1])) @@ -3407,10 +3503,10 @@ class DebugTransform(CythonTransform): def visit_NameNode(self, node): if (self.register_stepinto and - node.type is not None and - node.type.is_cfunction and - getattr(node, 'is_called', False) and - node.entry.func_cname is not None): + node.type is not None and + node.type.is_cfunction and + getattr(node, 'is_called', False) and + node.entry.func_cname is not None): # don't check node.entry.in_cinclude, as 'cdef extern: ...' # declared functions are not 'in_cinclude'. # This means we will list called 'cdef' functions as @@ -3429,26 +3525,16 @@ class DebugTransform(CythonTransform): it's a "relevant frame" and it will know where to set the breakpoint for 'break modulename'. """ - name = node.full_module_name.rpartition('.')[-1] - - cname_py2 = 'init' + name - cname_py3 = 'PyInit_' + name - - py2_attrs = dict( - name=name, - cname=cname_py2, + self._serialize_modulenode_as_function(node, dict( + name=node.full_module_name.rpartition('.')[-1], + cname=node.module_init_func_cname(), pf_cname='', # Ignore the qualified_name, breakpoints should be set using # `cy break modulename:lineno` for module-level breakpoints. qualified_name='', lineno='1', is_initmodule_function="True", - ) - - py3_attrs = dict(py2_attrs, cname=cname_py3) - - self._serialize_modulenode_as_function(node, py2_attrs) - self._serialize_modulenode_as_function(node, py3_attrs) + )) def _serialize_modulenode_as_function(self, node, attrs): self.tb.start('Function', attrs=attrs) diff --git a/Cython/Compiler/Parsing.pxd b/Cython/Compiler/Parsing.pxd index 25453b39a..58c7b8e0b 100644 --- a/Cython/Compiler/Parsing.pxd +++ b/Cython/Compiler/Parsing.pxd @@ -1,3 +1,5 @@ +# cython: language_level=3 + # We declare all of these here to type the first argument. from __future__ import absolute_import @@ -24,7 +26,7 @@ cdef p_lambdef_nocond(PyrexScanner s) cdef p_test(PyrexScanner s) cdef p_test_nocond(PyrexScanner s) cdef p_or_test(PyrexScanner s) -cdef p_rassoc_binop_expr(PyrexScanner s, ops, p_sub_expr_func p_subexpr) +cdef p_rassoc_binop_expr(PyrexScanner s, unicode op, p_sub_expr_func p_subexpr) cdef p_and_test(PyrexScanner s) cdef p_not_test(PyrexScanner s) cdef p_comparison(PyrexScanner s) @@ -197,3 +199,4 @@ cdef dict p_compiler_directive_comments(PyrexScanner s) cdef p_template_definition(PyrexScanner s) cdef p_cpp_class_definition(PyrexScanner s, pos, ctx) cdef p_cpp_class_attribute(PyrexScanner s, ctx) +cdef p_annotation(PyrexScanner s) diff --git a/Cython/Compiler/Parsing.py b/Cython/Compiler/Parsing.py index 82bf82d3d..45641a7d3 100644 --- a/Cython/Compiler/Parsing.py +++ b/Cython/Compiler/Parsing.py @@ -14,7 +14,7 @@ cython.declare(Nodes=object, ExprNodes=object, EncodedString=object, Builtin=object, ModuleNode=object, Utils=object, _unicode=object, _bytes=object, re=object, sys=object, _parse_escape_sequences=object, _parse_escape_sequences_raw=object, partial=object, reduce=object, _IS_PY3=cython.bint, _IS_2BYTE_UNICODE=cython.bint, - _CDEF_MODIFIERS=tuple) + _CDEF_MODIFIERS=tuple, COMMON_BINOP_MISTAKES=dict) from io import StringIO import re @@ -65,7 +65,7 @@ class Ctx(object): def p_ident(s, message="Expected an identifier"): if s.sy == 'IDENT': - name = s.systring + name = s.context.intern_ustring(s.systring) s.next() return name else: @@ -74,7 +74,7 @@ def p_ident(s, message="Expected an identifier"): def p_ident_list(s): names = [] while s.sy == 'IDENT': - names.append(s.systring) + names.append(s.context.intern_ustring(s.systring)) s.next() if s.sy != ',': break @@ -103,7 +103,7 @@ def p_binop_expr(s, ops, p_sub_expr): if Future.division in s.context.future_directives: n1.truedivision = True else: - n1.truedivision = None # unknown + n1.truedivision = None # unknown return n1 #lambdef: 'lambda' [varargslist] ':' test @@ -159,24 +159,31 @@ def p_test_nocond(s): #or_test: and_test ('or' and_test)* +COMMON_BINOP_MISTAKES = {'||': 'or', '&&': 'and'} + def p_or_test(s): - return p_rassoc_binop_expr(s, ('or',), p_and_test) + return p_rassoc_binop_expr(s, u'or', p_and_test) -def p_rassoc_binop_expr(s, ops, p_subexpr): +def p_rassoc_binop_expr(s, op, p_subexpr): n1 = p_subexpr(s) - if s.sy in ops: + if s.sy == op: pos = s.position() op = s.sy s.next() - n2 = p_rassoc_binop_expr(s, ops, p_subexpr) + n2 = p_rassoc_binop_expr(s, op, p_subexpr) n1 = ExprNodes.binop_node(pos, op, n1, n2) + elif s.sy in COMMON_BINOP_MISTAKES and COMMON_BINOP_MISTAKES[s.sy] == op: + # Only report this for the current operator since we pass through here twice for 'and' and 'or'. + warning(s.position(), + "Found the C operator '%s', did you mean the Python operator '%s'?" % (s.sy, op), + level=1) return n1 #and_test: not_test ('and' not_test)* def p_and_test(s): #return p_binop_expr(s, ('and',), p_not_test) - return p_rassoc_binop_expr(s, ('and',), p_not_test) + return p_rassoc_binop_expr(s, u'and', p_not_test) #not_test: 'not' not_test | comparison @@ -250,10 +257,10 @@ def p_cmp_op(s): op = '!=' return op -comparison_ops = cython.declare(set, set([ +comparison_ops = cython.declare(frozenset, frozenset(( '<', '>', '==', '>=', '<=', '<>', '!=', 'in', 'is', 'not' -])) +))) #expr: xor_expr ('|' xor_expr)* @@ -316,10 +323,12 @@ def p_typecast(s): s.next() base_type = p_c_base_type(s) is_memslice = isinstance(base_type, Nodes.MemoryViewSliceTypeNode) - is_template = isinstance(base_type, Nodes.TemplatedTypeNode) - is_const = isinstance(base_type, Nodes.CConstTypeNode) - if (not is_memslice and not is_template and not is_const - and base_type.name is None): + is_other_unnamed_type = isinstance(base_type, ( + Nodes.TemplatedTypeNode, + Nodes.CConstOrVolatileTypeNode, + Nodes.CTupleBaseTypeNode, + )) + if not (is_memslice or is_other_unnamed_type) and base_type.name is None: s.error("Unknown type") declarator = p_c_declarator(s, empty = 1) if s.sy == '?': @@ -330,8 +339,7 @@ def p_typecast(s): s.expect(">") operand = p_factor(s) if is_memslice: - return ExprNodes.CythonArrayNode(pos, base_type_node=base_type, - operand=operand) + return ExprNodes.CythonArrayNode(pos, base_type_node=base_type, operand=operand) return ExprNodes.TypecastNode(pos, base_type = base_type, @@ -444,7 +452,7 @@ def p_trailer(s, node1): return p_call(s, node1) elif s.sy == '[': return p_index(s, node1) - else: # s.sy == '.' + else: # s.sy == '.' s.next() name = p_ident(s) return ExprNodes.AttributeNode(pos, @@ -821,7 +829,7 @@ def p_cat_string_literal(s): continue elif next_kind != kind: # concatenating f strings and normal strings is allowed and leads to an f string - if set([kind, next_kind]) in (set(['f', 'u']), set(['f', ''])): + if {kind, next_kind} in ({'f', 'u'}, {'f', ''}): kind = 'f' else: error(pos, "Cannot mix string literals of different types, expected %s'', got %s''" % ( @@ -1478,8 +1486,8 @@ def p_genexp(s, expr): expr.pos, expr = ExprNodes.YieldExprNode(expr.pos, arg=expr))) return ExprNodes.GeneratorExpressionNode(expr.pos, loop=loop) -expr_terminators = cython.declare(set, set([ - ')', ']', '}', ':', '=', 'NEWLINE'])) +expr_terminators = cython.declare(frozenset, frozenset(( + ')', ']', '}', ':', '=', 'NEWLINE'))) #------------------------------------------------------- @@ -1507,7 +1515,7 @@ def p_expression_or_assignment(s): expr = p_testlist_star_expr(s) if s.sy == ':' and (expr.is_name or expr.is_subscript or expr.is_attribute): s.next() - expr.annotation = p_test(s) + expr.annotation = p_annotation(s) if s.sy == '=' and expr.is_starred: # This is a common enough error to make when learning Cython to let # it fail as early as possible and give a very clear error message. @@ -1690,11 +1698,6 @@ def p_import_statement(s): as_name=as_name, is_absolute=is_absolute) else: - if as_name and "." in dotted_name: - name_list = ExprNodes.ListNode(pos, args=[ - ExprNodes.IdentifierStringNode(pos, value=s.context.intern_ustring("*"))]) - else: - name_list = None stat = Nodes.SingleAssignmentNode( pos, lhs=ExprNodes.NameNode(pos, name=as_name or target_name), @@ -1702,7 +1705,8 @@ def p_import_statement(s): pos, module_name=ExprNodes.IdentifierStringNode(pos, value=dotted_name), level=0 if is_absolute else None, - name_list=name_list)) + get_top_level_module='.' in dotted_name and as_name is None, + name_list=None)) stats.append(stat) return Nodes.StatListNode(pos, stats=stats) @@ -1788,7 +1792,8 @@ def p_from_import_statement(s, first_statement = 0): items = items) -imported_name_kinds = cython.declare(set, set(['class', 'struct', 'union'])) +imported_name_kinds = cython.declare(frozenset, frozenset(( + 'class', 'struct', 'union'))) def p_imported_name(s, is_cimport): pos = s.position() @@ -1832,10 +1837,11 @@ def p_assert_statement(s): value = p_test(s) else: value = None - return Nodes.AssertStatNode(pos, cond = cond, value = value) + return Nodes.AssertStatNode(pos, condition=cond, value=value) -statement_terminators = cython.declare(set, set([';', 'NEWLINE', 'EOF'])) +statement_terminators = cython.declare(frozenset, frozenset(( + ';', 'NEWLINE', 'EOF'))) def p_if_statement(s): # s.sy == 'if' @@ -1945,7 +1951,8 @@ def p_for_from_step(s): else: return None -inequality_relations = cython.declare(set, set(['<', '<=', '>', '>='])) +inequality_relations = cython.declare(frozenset, frozenset(( + '<', '<=', '>', '>='))) def p_target(s, terminator): pos = s.position() @@ -2035,7 +2042,7 @@ def p_except_clause(s): def p_include_statement(s, ctx): pos = s.position() - s.next() # 'include' + s.next() # 'include' unicode_include_file_name = p_string_literal(s, 'u')[2] s.expect_newline("Syntax error in include statement") if s.compile_time_eval: @@ -2070,12 +2077,20 @@ def p_with_items(s, is_async=False): s.error("with gil/nogil cannot be async") state = s.systring s.next() + + # support conditional gil/nogil + condition = None + if s.sy == '(': + s.next() + condition = p_test(s) + s.expect(')') + if s.sy == ',': s.next() body = p_with_items(s) else: body = p_suite(s) - return Nodes.GILStatNode(pos, state=state, body=body) + return Nodes.GILStatNode(pos, state=state, body=body, condition=condition) else: manager = p_test(s) target = None @@ -2193,7 +2208,7 @@ def p_compile_time_expr(s): def p_DEF_statement(s): pos = s.position() denv = s.compile_time_env - s.next() # 'DEF' + s.next() # 'DEF' name = p_ident(s) s.expect('=') expr = p_compile_time_expr(s) @@ -2211,7 +2226,7 @@ def p_IF_statement(s, ctx): denv = s.compile_time_env result = None while 1: - s.next() # 'IF' or 'ELIF' + s.next() # 'IF' or 'ELIF' expr = p_compile_time_expr(s) s.compile_time_eval = current_eval and bool(expr.compile_time_value(denv)) body = p_suite(s, ctx) @@ -2329,7 +2344,7 @@ def p_statement(s, ctx, first_statement = 0): return p_async_statement(s, ctx, decorators) elif decorators: s.error("Decorators can only be followed by functions or classes") - s.put_back('IDENT', ident_name) # re-insert original token + s.put_back(u'IDENT', ident_name) # re-insert original token return p_simple_statement_list(s, ctx, first_statement=first_statement) @@ -2397,7 +2412,7 @@ def p_positional_and_keyword_args(s, end_sy_set, templates = None): parsed_type = False if s.sy == 'IDENT' and s.peek()[0] == '=': ident = s.systring - s.next() # s.sy is '=' + s.next() # s.sy is '=' s.next() if looking_at_expr(s): arg = p_test(s) @@ -2451,8 +2466,8 @@ def p_calling_convention(s): return "" -calling_convention_words = cython.declare( - set, set(["__stdcall", "__cdecl", "__fastcall"])) +calling_convention_words = cython.declare(frozenset, frozenset(( + "__stdcall", "__cdecl", "__fastcall"))) def p_c_complex_base_type(s, templates = None): @@ -2492,16 +2507,31 @@ def p_c_simple_base_type(s, self_flag, nonempty, templates = None): complex = 0 module_path = [] pos = s.position() - if not s.sy == 'IDENT': - error(pos, "Expected an identifier, found '%s'" % s.sy) - if s.systring == 'const': + + # Handle const/volatile + is_const = is_volatile = 0 + while s.sy == 'IDENT': + if s.systring == 'const': + if is_const: error(pos, "Duplicate 'const'") + is_const = 1 + elif s.systring == 'volatile': + if is_volatile: error(pos, "Duplicate 'volatile'") + is_volatile = 1 + else: + break s.next() + if is_const or is_volatile: base_type = p_c_base_type(s, self_flag=self_flag, nonempty=nonempty, templates=templates) if isinstance(base_type, Nodes.MemoryViewSliceTypeNode): # reverse order to avoid having to write "(const int)[:]" - base_type.base_type_node = Nodes.CConstTypeNode(pos, base_type=base_type.base_type_node) + base_type.base_type_node = Nodes.CConstOrVolatileTypeNode(pos, + base_type=base_type.base_type_node, is_const=is_const, is_volatile=is_volatile) return base_type - return Nodes.CConstTypeNode(pos, base_type=base_type) + return Nodes.CConstOrVolatileTypeNode(pos, + base_type=base_type, is_const=is_const, is_volatile=is_volatile) + + if s.sy != 'IDENT': + error(pos, "Expected an identifier, found '%s'" % s.sy) if looking_at_base_type(s): #print "p_c_simple_base_type: looking_at_base_type at", s.position() is_basic = 1 @@ -2536,13 +2566,13 @@ def p_c_simple_base_type(s, self_flag, nonempty, templates = None): s.next() if (s.sy == '*' or s.sy == '**' or s.sy == '&' or (s.sy == 'IDENT' and s.systring in calling_convention_words)): - s.put_back('(', '(') + s.put_back(u'(', u'(') else: - s.put_back('(', '(') - s.put_back('IDENT', name) + s.put_back(u'(', u'(') + s.put_back(u'IDENT', name) name = None elif s.sy not in ('*', '**', '[', '&'): - s.put_back('IDENT', name) + s.put_back(u'IDENT', name) name = None type_node = Nodes.CSimpleBaseTypeNode(pos, @@ -2649,7 +2679,7 @@ def p_memoryviewslice_access(s, base_type_node): return result def looking_at_name(s): - return s.sy == 'IDENT' and not s.systring in calling_convention_words + return s.sy == 'IDENT' and s.systring not in calling_convention_words def looking_at_expr(s): if s.systring in base_type_start_words: @@ -2683,10 +2713,10 @@ def looking_at_expr(s): dotted_path.reverse() for p in dotted_path: - s.put_back('IDENT', p) - s.put_back('.', '.') + s.put_back(u'IDENT', p) + s.put_back(u'.', u'.') - s.put_back('IDENT', name) + s.put_back(u'IDENT', name) return not is_type and saved[0] else: return True @@ -2700,7 +2730,7 @@ def looking_at_dotted_name(s): name = s.systring s.next() result = s.sy == '.' - s.put_back('IDENT', name) + s.put_back(u'IDENT', name) return result else: return 0 @@ -2716,8 +2746,8 @@ def looking_at_call(s): s.start_line, s.start_col = position return result -basic_c_type_names = cython.declare( - set, set(["void", "char", "int", "float", "double", "bint"])) +basic_c_type_names = cython.declare(frozenset, frozenset(( + "void", "char", "int", "float", "double", "bint"))) special_basic_c_types = cython.declare(dict, { # name : (signed, longness) @@ -2731,8 +2761,8 @@ special_basic_c_types = cython.declare(dict, { "Py_tss_t" : (1, 0), }) -sign_and_longness_words = cython.declare( - set, set(["short", "long", "signed", "unsigned"])) +sign_and_longness_words = cython.declare(frozenset, frozenset(( + "short", "long", "signed", "unsigned"))) base_type_start_words = cython.declare( set, @@ -2740,8 +2770,8 @@ base_type_start_words = cython.declare( | sign_and_longness_words | set(special_basic_c_types)) -struct_enum_union = cython.declare( - set, set(["struct", "union", "enum", "packed"])) +struct_enum_union = cython.declare(frozenset, frozenset(( + "struct", "union", "enum", "packed"))) def p_sign_and_longness(s): signed = 1 @@ -2796,7 +2826,7 @@ def p_c_declarator(s, ctx = Ctx(), empty = 0, is_type = 0, cmethod_flag = 0, pos = s.position() if s.sy == '[': result = p_c_array_declarator(s, result) - else: # sy == '(' + else: # sy == '(' s.next() result = p_c_func_declarator(s, pos, ctx, result, cmethod_flag) cmethod_flag = 0 @@ -2804,7 +2834,7 @@ def p_c_declarator(s, ctx = Ctx(), empty = 0, is_type = 0, cmethod_flag = 0, def p_c_array_declarator(s, base): pos = s.position() - s.next() # '[' + s.next() # '[' if s.sy != ']': dim = p_testlist(s) else: @@ -2813,7 +2843,7 @@ def p_c_array_declarator(s, base): return Nodes.CArrayDeclaratorNode(pos, base = base, dimension = dim) def p_c_func_declarator(s, pos, ctx, base, cmethod_flag): - # Opening paren has already been skipped + # Opening paren has already been skipped args = p_c_arg_list(s, ctx, cmethod_flag = cmethod_flag, nonempty_declarators = 0) ellipsis = p_optional_ellipsis(s) @@ -2826,49 +2856,43 @@ def p_c_func_declarator(s, pos, ctx, base, cmethod_flag): exception_value = exc_val, exception_check = exc_check, nogil = nogil or ctx.nogil or with_gil, with_gil = with_gil) -supported_overloaded_operators = cython.declare(set, set([ +supported_overloaded_operators = cython.declare(frozenset, frozenset(( '+', '-', '*', '/', '%', '++', '--', '~', '|', '&', '^', '<<', '>>', ',', '==', '!=', '>=', '>', '<=', '<', '[]', '()', '!', '=', 'bool', -])) +))) def p_c_simple_declarator(s, ctx, empty, is_type, cmethod_flag, assignable, nonempty): pos = s.position() calling_convention = p_calling_convention(s) - if s.sy == '*': + if s.sy in ('*', '**'): + # scanner returns '**' as a single token + is_ptrptr = s.sy == '**' s.next() - if s.systring == 'const': - const_pos = s.position() + + const_pos = s.position() + is_const = s.systring == 'const' and s.sy == 'IDENT' + if is_const: s.next() - const_base = p_c_declarator(s, ctx, empty = empty, - is_type = is_type, - cmethod_flag = cmethod_flag, - assignable = assignable, - nonempty = nonempty) - base = Nodes.CConstDeclaratorNode(const_pos, base = const_base) - else: - base = p_c_declarator(s, ctx, empty = empty, is_type = is_type, - cmethod_flag = cmethod_flag, - assignable = assignable, nonempty = nonempty) - result = Nodes.CPtrDeclaratorNode(pos, - base = base) - elif s.sy == '**': # scanner returns this as a single token - s.next() - base = p_c_declarator(s, ctx, empty = empty, is_type = is_type, - cmethod_flag = cmethod_flag, - assignable = assignable, nonempty = nonempty) - result = Nodes.CPtrDeclaratorNode(pos, - base = Nodes.CPtrDeclaratorNode(pos, - base = base)) - elif s.sy == '&': - s.next() - base = p_c_declarator(s, ctx, empty = empty, is_type = is_type, - cmethod_flag = cmethod_flag, - assignable = assignable, nonempty = nonempty) - result = Nodes.CReferenceDeclaratorNode(pos, base = base) + + base = p_c_declarator(s, ctx, empty=empty, is_type=is_type, + cmethod_flag=cmethod_flag, + assignable=assignable, nonempty=nonempty) + if is_const: + base = Nodes.CConstDeclaratorNode(const_pos, base=base) + if is_ptrptr: + base = Nodes.CPtrDeclaratorNode(pos, base=base) + result = Nodes.CPtrDeclaratorNode(pos, base=base) + elif s.sy == '&' or (s.sy == '&&' and s.context.cpp): + node_class = Nodes.CppRvalueReferenceDeclaratorNode if s.sy == '&&' else Nodes.CReferenceDeclaratorNode + s.next() + base = p_c_declarator(s, ctx, empty=empty, is_type=is_type, + cmethod_flag=cmethod_flag, + assignable=assignable, nonempty=nonempty) + result = node_class(pos, base=base) else: rhs = None if s.sy == 'IDENT': @@ -2909,7 +2933,7 @@ def p_c_simple_declarator(s, ctx, empty, is_type, cmethod_flag, fatal=False) name += op elif op == 'IDENT': - op = s.systring; + op = s.systring if op not in supported_overloaded_operators: s.error("Overloading operator '%s' not yet supported." % op, fatal=False) @@ -2960,7 +2984,8 @@ def p_exception_value_clause(s): exc_val = p_test(s) return exc_val, exc_check -c_arg_list_terminators = cython.declare(set, set(['*', '**', '.', ')', ':'])) +c_arg_list_terminators = cython.declare(frozenset, frozenset(( + '*', '**', '.', ')', ':', '/'))) def p_c_arg_list(s, ctx = Ctx(), in_pyfunc = 0, cmethod_flag = 0, nonempty_declarators = 0, kw_only = 0, annotated = 1): @@ -3014,7 +3039,7 @@ def p_c_arg_decl(s, ctx, in_pyfunc, cmethod_flag = 0, nonempty = 0, not_none = kind == 'not' if annotated and s.sy == ':': s.next() - annotation = p_test(s) + annotation = p_annotation(s) if s.sy == '=': s.next() if 'pxd' in ctx.level: @@ -3116,6 +3141,12 @@ def p_cdef_extern_block(s, pos, ctx): def p_c_enum_definition(s, pos, ctx): # s.sy == ident 'enum' s.next() + + scoped = False + if s.context.cpp and (s.sy == 'class' or (s.sy == 'IDENT' and s.systring == 'struct')): + scoped = True + s.next() + if s.sy == 'IDENT': name = s.systring s.next() @@ -3123,24 +3154,51 @@ def p_c_enum_definition(s, pos, ctx): if cname is None and ctx.namespace is not None: cname = ctx.namespace + "::" + name else: - name = None - cname = None - items = None + name = cname = None + if scoped: + s.error("Unnamed scoped enum not allowed") + + if scoped and s.sy == '(': + s.next() + underlying_type = p_c_base_type(s) + s.expect(')') + else: + underlying_type = Nodes.CSimpleBaseTypeNode( + pos, + name="int", + module_path = [], + is_basic_c_type = True, + signed = 1, + complex = 0, + longness = 0 + ) + s.expect(':') items = [] + + doc = None if s.sy != 'NEWLINE': p_c_enum_line(s, ctx, items) else: - s.next() # 'NEWLINE' + s.next() # 'NEWLINE' s.expect_indent() + doc = p_doc_string(s) + while s.sy not in ('DEDENT', 'EOF'): p_c_enum_line(s, ctx, items) + s.expect_dedent() + + if not items and ctx.visibility != "extern": + error(pos, "Empty enum definition not allowed outside a 'cdef extern from' block") + return Nodes.CEnumDefNode( - pos, name = name, cname = cname, items = items, - typedef_flag = ctx.typedef_flag, visibility = ctx.visibility, - create_wrapper = ctx.overridable, - api = ctx.api, in_pxd = ctx.level == 'module_pxd') + pos, name=name, cname=cname, + scoped=scoped, items=items, + underlying_type=underlying_type, + typedef_flag=ctx.typedef_flag, visibility=ctx.visibility, + create_wrapper=ctx.overridable, + api=ctx.api, in_pxd=ctx.level == 'module_pxd', doc=doc) def p_c_enum_line(s, ctx, items): if s.sy != 'pass': @@ -3184,20 +3242,28 @@ def p_c_struct_or_union_definition(s, pos, ctx): attributes = None if s.sy == ':': s.next() - s.expect('NEWLINE') - s.expect_indent() attributes = [] - body_ctx = Ctx() - while s.sy != 'DEDENT': - if s.sy != 'pass': - attributes.append( - p_c_func_or_var_declaration(s, s.position(), body_ctx)) - else: - s.next() - s.expect_newline("Expected a newline") - s.expect_dedent() + if s.sy == 'pass': + s.next() + s.expect_newline("Expected a newline", ignore_semicolon=True) + else: + s.expect('NEWLINE') + s.expect_indent() + body_ctx = Ctx() + while s.sy != 'DEDENT': + if s.sy != 'pass': + attributes.append( + p_c_func_or_var_declaration(s, s.position(), body_ctx)) + else: + s.next() + s.expect_newline("Expected a newline") + s.expect_dedent() + + if not attributes and ctx.visibility != "extern": + error(pos, "Empty struct or union definition not allowed outside a 'cdef extern from' block") else: s.expect_newline("Syntax error in struct or union definition") + return Nodes.CStructOrUnionDefNode(pos, name = name, cname = cname, kind = kind, attributes = attributes, typedef_flag = ctx.typedef_flag, visibility = ctx.visibility, @@ -3224,7 +3290,7 @@ def p_fused_definition(s, pos, ctx): while s.sy != 'DEDENT': if s.sy != 'pass': #types.append(p_c_declarator(s)) - types.append(p_c_base_type(s)) #, nonempty=1)) + types.append(p_c_base_type(s)) #, nonempty=1)) else: s.next() @@ -3380,7 +3446,7 @@ def _reject_cdef_modifier_in_py(s, name): def p_def_statement(s, decorators=None, is_async_def=False): # s.sy == 'def' - pos = s.position() + pos = decorators[0].pos if decorators else s.position() # PEP 492 switches the async/await keywords on in "async def" functions if is_async_def: s.enter_async() @@ -3397,7 +3463,7 @@ def p_def_statement(s, decorators=None, is_async_def=False): return_type_annotation = None if s.sy == '->': s.next() - return_type_annotation = p_test(s) + return_type_annotation = p_annotation(s) _reject_cdef_modifier_in_py(s, s.systring) doc, body = p_suite_with_docstring(s, Ctx(level='function')) @@ -3415,6 +3481,20 @@ def p_varargslist(s, terminator=')', annotated=1): annotated = annotated) star_arg = None starstar_arg = None + if s.sy == '/': + if len(args) == 0: + s.error("Got zero positional-only arguments despite presence of " + "positional-only specifier '/'") + s.next() + # Mark all args to the left as pos only + for arg in args: + arg.pos_only = 1 + if s.sy == ',': + s.next() + args.extend(p_c_arg_list(s, in_pyfunc = 1, + nonempty_declarators = 1, annotated = annotated)) + elif s.sy != terminator: + s.error("Syntax error in Python function argument list") if s.sy == '*': s.next() if s.sy == 'IDENT': @@ -3438,7 +3518,7 @@ def p_py_arg_decl(s, annotated = 1): annotation = None if annotated and s.sy == ':': s.next() - annotation = p_test(s) + annotation = p_annotation(s) return Nodes.PyArgDeclNode(pos, name = name, annotation = annotation) @@ -3688,22 +3768,18 @@ def p_module(s, pxd, full_module_name, ctx=Ctx): s.parse_comments = False if s.context.language_level is None: - s.context.set_language_level(2) + s.context.set_language_level('3str') if pos[0].filename: import warnings warnings.warn( - "Cython directive 'language_level' not set, using 2 for now (Py2). " - "This will change in a later release! File: %s" % pos[0].filename, + "Cython directive 'language_level' not set, using '3str' for now (Py3). " + "This has changed from earlier releases! File: %s" % pos[0].filename, FutureWarning, stacklevel=1 if cython.compiled else 2, ) + level = 'module_pxd' if pxd else 'module' doc = p_doc_string(s) - if pxd: - level = 'module_pxd' - else: - level = 'module' - body = p_statement_list(s, ctx(level=level), first_statement = 1) if s.sy != 'EOF': s.error("Syntax error in statement [%s,%s]" % ( @@ -3725,7 +3801,6 @@ def p_template_definition(s): def p_cpp_class_definition(s, pos, ctx): # s.sy == 'cppclass' s.next() - module_path = [] class_name = p_ident(s) cname = p_opt_cname(s) if cname is None and ctx.namespace is not None: @@ -3759,6 +3834,10 @@ def p_cpp_class_definition(s, pos, ctx): s.next() s.expect('NEWLINE') s.expect_indent() + # Allow a cppclass to have docstrings. It will be discarded as comment. + # The goal of this is consistency: we can make docstrings inside cppclass methods, + # so why not on the cppclass itself ? + p_doc_string(s) attributes = [] body_ctx = Ctx(visibility = ctx.visibility, level='cpp_class', nogil=nogil or ctx.nogil) body_ctx.templates = template_names @@ -3842,3 +3921,14 @@ def print_parse_tree(f, node, level, key = None): f.write("%s]\n" % ind) return f.write("%s%s\n" % (ind, node)) + +def p_annotation(s): + """An annotation just has the "test" syntax, but also stores the string it came from + + Note that the string is *allowed* to be changed/processed (although isn't here) + so may not exactly match the string generated by Python, and if it doesn't + then it is not a bug. + """ + pos = s.position() + expr = p_test(s) + return ExprNodes.AnnotationNode(pos, expr=expr) diff --git a/Cython/Compiler/Pipeline.py b/Cython/Compiler/Pipeline.py index 5194c3e49..4e8e98c71 100644 --- a/Cython/Compiler/Pipeline.py +++ b/Cython/Compiler/Pipeline.py @@ -128,7 +128,8 @@ def inject_utility_code_stage_factory(context): module_node.scope.utility_code_list.append(dep) tree = utilcode.get_tree(cython_scope=context.cython_scope) if tree: - module_node.merge_in(tree.body, tree.scope, merge_scope=True) + module_node.merge_in(tree.with_compiler_directives(), + tree.scope, merge_scope=True) return module_node return inject_utility_code_stage @@ -238,7 +239,7 @@ def create_pyx_pipeline(context, options, result, py=False, exclude_classes=()): test_support.append(TreeAssertVisitor()) if options.gdb_debug: - from ..Debugger import DebugWriter # requires Py2.5+ + from ..Debugger import DebugWriter # requires Py2.5+ from .ParseTreeTransforms import DebugTransform context.gdb_debug_outputwriter = DebugWriter.CythonDebugWriter( options.output_dir) @@ -284,11 +285,25 @@ def create_pyx_as_pxd_pipeline(context, result): FlattenInListTransform, WithTransform ]) + from .Visitor import VisitorTransform + class SetInPxdTransform(VisitorTransform): + # A number of nodes have an "in_pxd" attribute which affects AnalyseDeclarationsTransform + # (for example controlling pickling generation). Set it, to make sure we don't mix them up with + # the importing main module. + # FIXME: This should be done closer to the parsing step. + def visit_StatNode(self, node): + if hasattr(node, "in_pxd"): + node.in_pxd = True + self.visitchildren(node) + return node + + visit_Node = VisitorTransform.recurse_to_children + for stage in pyx_pipeline: pipeline.append(stage) if isinstance(stage, AnalyseDeclarationsTransform): - # This is the last stage we need. - break + pipeline.insert(-1, SetInPxdTransform()) + break # This is the last stage we need. def fake_pxd(root): for entry in root.scope.entries.values(): if not entry.in_cinclude: diff --git a/Cython/Compiler/PyrexTypes.py b/Cython/Compiler/PyrexTypes.py index dcb51fe34..7e7154c00 100644 --- a/Cython/Compiler/PyrexTypes.py +++ b/Cython/Compiler/PyrexTypes.py @@ -12,13 +12,14 @@ try: reduce except NameError: from functools import reduce +from functools import partial from Cython.Utils import cached_function from .Code import UtilityCode, LazyUtilityCode, TempitaUtilityCode from . import StringEncoding from . import Naming -from .Errors import error, warning +from .Errors import error, warning, CannotSpecialize class BaseType(object): @@ -46,7 +47,9 @@ class BaseType(object): def cast_code(self, expr_code): return "((%s)%s)" % (self.empty_declaration_code(), expr_code) - def empty_declaration_code(self): + def empty_declaration_code(self, pyrex=False): + if pyrex: + return self.declaration_code('', pyrex=True) if self._empty_declaration is None: self._empty_declaration = self.declaration_code('') return self._empty_declaration @@ -115,7 +118,7 @@ class BaseType(object): Deduce any template params in this (argument) type given the actual argument type. - http://en.cppreference.com/w/cpp/language/function_template#Template_argument_deduction + https://en.cppreference.com/w/cpp/language/function_template#Template_argument_deduction """ return {} @@ -176,11 +179,15 @@ class PyrexType(BaseType): # is_ptr boolean Is a C pointer type # is_null_ptr boolean Is the type of NULL # is_reference boolean Is a C reference type - # is_const boolean Is a C const type. + # is_rvalue_reference boolean Is a C++ rvalue reference type + # is_const boolean Is a C const type + # is_volatile boolean Is a C volatile type + # is_cv_qualified boolean Is a C const or volatile type # is_cfunction boolean Is a C function type # is_struct_or_union boolean Is a C struct or union type # is_struct boolean Is a C struct type # is_enum boolean Is a C enum type + # is_cpp_enum boolean Is a C++ scoped enum type # is_typedef boolean Is a typedef type # is_string boolean Is a C char * type # is_pyunicode_ptr boolean Is a C PyUNICODE * type @@ -192,6 +199,9 @@ class PyrexType(BaseType): # is_pythran_expr boolean Is Pythran expr # is_numpy_buffer boolean Is Numpy array buffer # has_attributes boolean Has C dot-selectable attributes + # needs_cpp_construction boolean Needs C++ constructor and destructor when used in a cdef class + # needs_refcounting boolean Needs code to be generated similar to incref/gotref/decref. + # Largely used internally. # default_value string Initial value that can be assigned before first user assignment. # declaration_value string The value statically assigned on declaration (if any). # entry Entry The Entry for this type @@ -226,6 +236,7 @@ class PyrexType(BaseType): is_extension_type = 0 is_final_type = 0 is_builtin_type = 0 + is_cython_builtin_type = 0 is_numeric = 0 is_int = 0 is_float = 0 @@ -235,13 +246,17 @@ class PyrexType(BaseType): is_ptr = 0 is_null_ptr = 0 is_reference = 0 + is_rvalue_reference = 0 is_const = 0 + is_volatile = 0 + is_cv_qualified = 0 is_cfunction = 0 is_struct_or_union = 0 is_cpp_class = 0 is_cpp_string = 0 is_struct = 0 is_enum = 0 + is_cpp_enum = False is_typedef = 0 is_string = 0 is_pyunicode_ptr = 0 @@ -254,6 +269,8 @@ class PyrexType(BaseType): is_pythran_expr = 0 is_numpy_buffer = 0 has_attributes = 0 + needs_cpp_construction = 0 + needs_refcounting = 0 default_value = "" declaration_value = "" @@ -262,7 +279,8 @@ class PyrexType(BaseType): return self def specialize(self, values): - # TODO(danilo): Override wherever it makes sense. + # Returns the concrete type if this is a fused type, or otherwise the type itself. + # May raise Errors.CannotSpecialize on failure return self def literal_code(self, value): @@ -331,6 +349,31 @@ class PyrexType(BaseType): convert_call, code.error_goto_if(error_condition or self.error_condition(result_code), error_pos)) + def _generate_dummy_refcounting(self, code, *ignored_args, **ignored_kwds): + if self.needs_refcounting: + raise NotImplementedError("Ref-counting operation not yet implemented for type %s" % + self) + + def _generate_dummy_refcounting_assignment(self, code, cname, rhs_cname, *ignored_args, **ignored_kwds): + if self.needs_refcounting: + raise NotImplementedError("Ref-counting operation not yet implemented for type %s" % + self) + code.putln("%s = %s" % (cname, rhs_cname)) + + generate_incref = generate_xincref = generate_decref = generate_xdecref \ + = generate_decref_clear = generate_xdecref_clear \ + = generate_gotref = generate_xgotref = generate_giveref = generate_xgiveref \ + = _generate_dummy_refcounting + + generate_decref_set = generate_xdecref_set = _generate_dummy_refcounting_assignment + + def nullcheck_string(self, code, cname): + if self.needs_refcounting: + raise NotImplementedError("Ref-counting operation not yet implemented for type %s" % + self) + code.putln("1") + + def public_decl(base_code, dll_linkage): if dll_linkage: @@ -447,9 +490,9 @@ class CTypedefType(BaseType): "TO_PY_FUNCTION": self.to_py_function})) return True elif base_type.is_float: - pass # XXX implement! + pass # XXX implement! elif base_type.is_complex: - pass # XXX implement! + pass # XXX implement! pass elif base_type.is_cpp_string: cname = "__pyx_convert_PyObject_string_to_py_%s" % type_identifier(self) @@ -480,9 +523,9 @@ class CTypedefType(BaseType): "FROM_PY_FUNCTION": self.from_py_function})) return True elif base_type.is_float: - pass # XXX implement! + pass # XXX implement! elif base_type.is_complex: - pass # XXX implement! + pass # XXX implement! elif base_type.is_cpp_string: cname = '__pyx_convert_string_from_py_%s' % type_identifier(self) context = { @@ -561,8 +604,13 @@ class CTypedefType(BaseType): class MemoryViewSliceType(PyrexType): is_memoryviewslice = 1 + default_value = "{ 0, 0, { 0 }, { 0 }, { 0 } }" has_attributes = 1 + needs_refcounting = 1 # Ideally this would be true and reference counting for + # memoryview and pyobject code could be generated in the same way. + # However, memoryviews are sufficiently specialized that this doesn't + # seem practical. Implement a limited version of it for now scope = None # These are special cased in Defnode @@ -591,7 +639,7 @@ class MemoryViewSliceType(PyrexType): 'ptr' -- Pointer stored in this dimension. 'full' -- Check along this dimension, don't assume either. - the packing specifiers specify how the array elements are layed-out + the packing specifiers specify how the array elements are laid-out in memory. 'contig' -- The data is contiguous in memory along this dimension. @@ -633,6 +681,10 @@ class MemoryViewSliceType(PyrexType): else: return False + def __ne__(self, other): + # TODO drop when Python2 is dropped + return not (self == other) + def same_as_resolved_type(self, other_type): return ((other_type.is_memoryviewslice and #self.writable_needed == other_type.writable_needed and # FIXME: should be only uni-directional @@ -650,10 +702,10 @@ class MemoryViewSliceType(PyrexType): def declaration_code(self, entity_code, for_display = 0, dll_linkage = None, pyrex = 0): # XXX: we put these guards in for now... - assert not pyrex assert not dll_linkage from . import MemoryView - base_code = str(self) if for_display else MemoryView.memviewslice_cname + base_code = StringEncoding.EncodedString( + str(self) if pyrex or for_display else MemoryView.memviewslice_cname) return self.base_declaration_code( base_code, entity_code) @@ -713,8 +765,8 @@ class MemoryViewSliceType(PyrexType): to_axes_f = contig_dim + follow_dim * (ndim -1) dtype = self.dtype - if dtype.is_const: - dtype = dtype.const_base_type + if dtype.is_cv_qualified: + dtype = dtype.cv_base_type to_memview_c = MemoryViewSliceType(dtype, to_axes_c) to_memview_f = MemoryViewSliceType(dtype, to_axes_f) @@ -791,15 +843,18 @@ class MemoryViewSliceType(PyrexType): # return False src_dtype, dst_dtype = src.dtype, dst.dtype - if dst_dtype.is_const: - # Requesting read-only views is always ok => consider only the non-const base type. - dst_dtype = dst_dtype.const_base_type - if src_dtype.is_const: - # When assigning between read-only views, compare only the non-const base types. - src_dtype = src_dtype.const_base_type - elif copying and src_dtype.is_const: - # Copying by value => ignore const on source. - src_dtype = src_dtype.const_base_type + # We can add but not remove const/volatile modifiers + # (except if we are copying by value, then anything is fine) + if not copying: + if src_dtype.is_const and not dst_dtype.is_const: + return False + if src_dtype.is_volatile and not dst_dtype.is_volatile: + return False + # const/volatile checks are done, remove those qualifiers + if src_dtype.is_cv_qualified: + src_dtype = src_dtype.cv_base_type + if dst_dtype.is_cv_qualified: + dst_dtype = dst_dtype.cv_base_type if src_dtype != dst_dtype: return False @@ -1032,6 +1087,36 @@ class MemoryViewSliceType(PyrexType): def cast_code(self, expr_code): return expr_code + # When memoryviews are increfed currently seems heavily special-cased. + # Therefore, use our own function for now + def generate_incref(self, code, name, **kwds): + pass + + def generate_incref_memoryviewslice(self, code, slice_cname, have_gil): + # TODO ideally would be done separately + code.putln("__PYX_INC_MEMVIEW(&%s, %d);" % (slice_cname, int(have_gil))) + + # decref however did look to always apply for memoryview slices + # with "have_gil" set to True by default + def generate_xdecref(self, code, cname, nanny, have_gil): + code.putln("__PYX_XCLEAR_MEMVIEW(&%s, %d);" % (cname, int(have_gil))) + + def generate_decref(self, code, cname, nanny, have_gil): + # Fall back to xdecref since we don't care to have a separate decref version for this. + self.generate_xdecref(code, cname, nanny, have_gil) + + def generate_xdecref_clear(self, code, cname, clear_before_decref, **kwds): + self.generate_xdecref(code, cname, **kwds) + code.putln("%s.memview = NULL; %s.data = NULL;" % (cname, cname)) + + def generate_decref_clear(self, code, cname, **kwds): + # memoryviews don't currently distinguish between xdecref and decref + self.generate_xdecref_clear(code, cname, **kwds) + + # memoryviews don't participate in giveref/gotref + generate_gotref = generate_xgotref = generate_xgiveref = generate_giveref = lambda *args: None + + class BufferType(BaseType): # @@ -1129,6 +1214,8 @@ class PyObjectType(PyrexType): is_extern = False is_subclassed = False is_gc_simple = False + builtin_trashcan = False # builtin type using trashcan + needs_refcounting = True def __str__(self): return "Python object" @@ -1181,11 +1268,85 @@ class PyObjectType(PyrexType): def check_for_null_code(self, cname): return cname + def generate_incref(self, code, cname, nanny): + if nanny: + code.putln("__Pyx_INCREF(%s);" % self.as_pyobject(cname)) + else: + code.putln("Py_INCREF(%s);" % self.as_pyobject(cname)) + + def generate_xincref(self, code, cname, nanny): + if nanny: + code.putln("__Pyx_XINCREF(%s);" % self.as_pyobject(cname)) + else: + code.putln("Py_XINCREF(%s);" % self.as_pyobject(cname)) + + def generate_decref(self, code, cname, nanny, have_gil): + # have_gil is for the benefit of memoryviewslice - it's ignored here + assert have_gil + self._generate_decref(code, cname, nanny, null_check=False, clear=False) + + def generate_xdecref(self, code, cname, nanny, have_gil): + # in this (and other) PyObjectType functions, have_gil is being + # passed to provide a common interface with MemoryviewSlice. + # It's ignored here + self._generate_decref(code, cname, nanny, null_check=True, + clear=False) + + def generate_decref_clear(self, code, cname, clear_before_decref, nanny, have_gil): + self._generate_decref(code, cname, nanny, null_check=False, + clear=True, clear_before_decref=clear_before_decref) + + def generate_xdecref_clear(self, code, cname, clear_before_decref=False, nanny=True, have_gil=None): + self._generate_decref(code, cname, nanny, null_check=True, + clear=True, clear_before_decref=clear_before_decref) + + def generate_gotref(self, code, cname): + code.putln("__Pyx_GOTREF(%s);" % self.as_pyobject(cname)) + + def generate_xgotref(self, code, cname): + code.putln("__Pyx_XGOTREF(%s);" % self.as_pyobject(cname)) + + def generate_giveref(self, code, cname): + code.putln("__Pyx_GIVEREF(%s);" % self.as_pyobject(cname)) + + def generate_xgiveref(self, code, cname): + code.putln("__Pyx_XGIVEREF(%s);" % self.as_pyobject(cname)) + + def generate_decref_set(self, code, cname, rhs_cname): + code.putln("__Pyx_DECREF_SET(%s, %s);" % (cname, rhs_cname)) + + def generate_xdecref_set(self, code, cname, rhs_cname): + code.putln("__Pyx_XDECREF_SET(%s, %s);" % (cname, rhs_cname)) + + def _generate_decref(self, code, cname, nanny, null_check=False, + clear=False, clear_before_decref=False): + prefix = '__Pyx' if nanny else 'Py' + X = 'X' if null_check else '' + + if clear: + if clear_before_decref: + if not nanny: + X = '' # CPython doesn't have a Py_XCLEAR() + code.putln("%s_%sCLEAR(%s);" % (prefix, X, cname)) + else: + code.putln("%s_%sDECREF(%s); %s = 0;" % ( + prefix, X, self.as_pyobject(cname), cname)) + else: + code.putln("%s_%sDECREF(%s);" % ( + prefix, X, self.as_pyobject(cname))) + + def nullcheck_string(self, cname): + return cname + -builtin_types_that_cannot_create_refcycles = set([ - 'bool', 'int', 'long', 'float', 'complex', - 'bytearray', 'bytes', 'unicode', 'str', 'basestring' -]) +builtin_types_that_cannot_create_refcycles = frozenset({ + 'object', 'bool', 'int', 'long', 'float', 'complex', + 'bytearray', 'bytes', 'unicode', 'str', 'basestring', +}) + +builtin_types_with_trashcan = frozenset({ + 'dict', 'list', 'set', 'frozenset', 'tuple', 'type', +}) class BuiltinObjectType(PyObjectType): @@ -1211,6 +1372,7 @@ class BuiltinObjectType(PyObjectType): self.typeptr_cname = "(&%s)" % cname self.objstruct_cname = objstruct_cname self.is_gc_simple = name in builtin_types_that_cannot_create_refcycles + self.builtin_trashcan = name in builtin_types_with_trashcan if name == 'type': # Special case the type type, as many C API calls (and other # libraries) actually expect a PyTypeObject* for type arguments. @@ -1292,14 +1454,9 @@ class BuiltinObjectType(PyObjectType): check += '||((%s) == Py_None)' % arg if self.name == 'basestring': name = '(PY_MAJOR_VERSION < 3 ? "basestring" : "str")' - space_for_name = 16 else: name = '"%s"' % self.name - # avoid wasting too much space but limit number of different format strings - space_for_name = (len(self.name) // 16 + 1) * 16 - error = '(PyErr_Format(PyExc_TypeError, "Expected %%.%ds, got %%.200s", %s, Py_TYPE(%s)->tp_name), 0)' % ( - space_for_name, name, arg) - return check + '||' + error + return check + ' || __Pyx_RaiseUnexpectedTypeError(%s, %s)' % (name, arg) def declaration_code(self, entity_code, for_display = 0, dll_linkage = None, pyrex = 0): @@ -1318,7 +1475,7 @@ class BuiltinObjectType(PyObjectType): def cast_code(self, expr_code, to_object_struct = False): return "((%s*)%s)" % ( - to_object_struct and self.objstruct_cname or self.decl_type, # self.objstruct_cname may be None + to_object_struct and self.objstruct_cname or self.decl_type, # self.objstruct_cname may be None expr_code) def py_type_name(self): @@ -1543,8 +1700,14 @@ class PythranExpr(CType): self.scope = scope = Symtab.CClassScope('', None, visibility="extern") scope.parent_type = self scope.directives = {} - scope.declare_var("shape", CPtrType(c_long_type), None, cname="_shape", is_cdef=True) - scope.declare_var("ndim", c_long_type, None, cname="value", is_cdef=True) + + scope.declare_var("ndim", c_long_type, pos=None, cname="value", is_cdef=True) + scope.declare_cproperty( + "shape", c_ptr_type(c_long_type), "__Pyx_PythranShapeAccessor", + doc="Pythran array shape", + visibility="extern", + nogil=True, + ) return True @@ -1558,58 +1721,76 @@ class PythranExpr(CType): return hash(self.pythran_type) -class CConstType(BaseType): +class CConstOrVolatileType(BaseType): + "A C const or volatile type" - is_const = 1 + subtypes = ['cv_base_type'] - def __init__(self, const_base_type): - self.const_base_type = const_base_type - if const_base_type.has_attributes and const_base_type.scope is not None: - from . import Symtab - self.scope = Symtab.CConstScope(const_base_type.scope) + is_cv_qualified = 1 + + def __init__(self, base_type, is_const=0, is_volatile=0): + self.cv_base_type = base_type + self.is_const = is_const + self.is_volatile = is_volatile + if base_type.has_attributes and base_type.scope is not None: + from .Symtab import CConstOrVolatileScope + self.scope = CConstOrVolatileScope(base_type.scope, is_const, is_volatile) + + def cv_string(self): + cvstring = "" + if self.is_const: + cvstring = "const " + cvstring + if self.is_volatile: + cvstring = "volatile " + cvstring + return cvstring def __repr__(self): - return "<CConstType %s>" % repr(self.const_base_type) + return "<CConstOrVolatileType %s%r>" % (self.cv_string(), self.cv_base_type) def __str__(self): return self.declaration_code("", for_display=1) def declaration_code(self, entity_code, for_display = 0, dll_linkage = None, pyrex = 0): + cv = self.cv_string() if for_display or pyrex: - return "const " + self.const_base_type.declaration_code(entity_code, for_display, dll_linkage, pyrex) + return cv + self.cv_base_type.declaration_code(entity_code, for_display, dll_linkage, pyrex) else: - return self.const_base_type.declaration_code("const %s" % entity_code, for_display, dll_linkage, pyrex) + return self.cv_base_type.declaration_code(cv + entity_code, for_display, dll_linkage, pyrex) def specialize(self, values): - base_type = self.const_base_type.specialize(values) - if base_type == self.const_base_type: + base_type = self.cv_base_type.specialize(values) + if base_type == self.cv_base_type: return self - else: - return CConstType(base_type) + return CConstOrVolatileType(base_type, + self.is_const, self.is_volatile) def deduce_template_params(self, actual): - return self.const_base_type.deduce_template_params(actual) + return self.cv_base_type.deduce_template_params(actual) def can_coerce_to_pyobject(self, env): - return self.const_base_type.can_coerce_to_pyobject(env) + return self.cv_base_type.can_coerce_to_pyobject(env) def can_coerce_from_pyobject(self, env): - return self.const_base_type.can_coerce_from_pyobject(env) + return self.cv_base_type.can_coerce_from_pyobject(env) def create_to_py_utility_code(self, env): - if self.const_base_type.create_to_py_utility_code(env): - self.to_py_function = self.const_base_type.to_py_function + if self.cv_base_type.create_to_py_utility_code(env): + self.to_py_function = self.cv_base_type.to_py_function return True def same_as_resolved_type(self, other_type): - if other_type.is_const: - return self.const_base_type.same_as_resolved_type(other_type.const_base_type) - # Accept const LHS <- non-const RHS. - return self.const_base_type.same_as_resolved_type(other_type) + if other_type.is_cv_qualified: + return self.cv_base_type.same_as_resolved_type(other_type.cv_base_type) + # Accept cv LHS <- non-cv RHS. + return self.cv_base_type.same_as_resolved_type(other_type) def __getattr__(self, name): - return getattr(self.const_base_type, name) + return getattr(self.cv_base_type, name) + + +def CConstType(base_type): + return CConstOrVolatileType(base_type, is_const=1) class FusedType(CType): @@ -1652,7 +1833,10 @@ class FusedType(CType): return 'FusedType(name=%r)' % self.name def specialize(self, values): - return values[self] + if self in values: + return values[self] + else: + raise CannotSpecialize() def get_fused_types(self, result=None, seen=None): if result is None: @@ -1737,6 +1921,7 @@ class CNumericType(CType): base_code = type_name.replace('PY_LONG_LONG', 'long long') else: base_code = public_decl(type_name, dll_linkage) + base_code = StringEncoding.EncodedString(base_code) return self.base_declaration_code(base_code, entity_code) def attributes_known(self): @@ -2169,8 +2354,8 @@ class CComplexType(CNumericType): def assignable_from(self, src_type): # Temporary hack/feature disabling, see #441 if (not src_type.is_complex and src_type.is_numeric and src_type.is_typedef - and src_type.typedef_is_external): - return False + and src_type.typedef_is_external): + return False elif src_type.is_pyobject: return True else: @@ -2178,8 +2363,8 @@ class CComplexType(CNumericType): def assignable_from_resolved_type(self, src_type): return (src_type.is_complex and self.real_type.assignable_from_resolved_type(src_type.real_type) - or src_type.is_numeric and self.real_type.assignable_from_resolved_type(src_type) - or src_type is error_type) + or src_type.is_numeric and self.real_type.assignable_from_resolved_type(src_type) + or src_type is error_type) def attributes_known(self): if self.scope is None: @@ -2302,8 +2487,8 @@ class CPointerBaseType(CType): def __init__(self, base_type): self.base_type = base_type - if base_type.is_const: - base_type = base_type.const_base_type + if base_type.is_cv_qualified: + base_type = base_type.cv_base_type for char_type in (c_char_type, c_uchar_type, c_schar_type): if base_type.same_as(char_type): self.is_string = 1 @@ -2346,6 +2531,7 @@ class CPointerBaseType(CType): if self.is_string: assert isinstance(value, str) return '"%s"' % StringEncoding.escape_byte_string(value) + return str(value) class CArrayType(CPointerBaseType): @@ -2365,7 +2551,7 @@ class CArrayType(CPointerBaseType): return False def __hash__(self): - return hash(self.base_type) + 28 # arbitrarily chosen offset + return hash(self.base_type) + 28 # arbitrarily chosen offset def __repr__(self): return "<CArrayType %s %s>" % (self.size, repr(self.base_type)) @@ -2497,7 +2683,7 @@ class CPtrType(CPointerBaseType): default_value = "0" def __hash__(self): - return hash(self.base_type) + 27 # arbitrarily chosen offset + return hash(self.base_type) + 27 # arbitrarily chosen offset def __eq__(self, other): if isinstance(other, CType) and other.is_ptr: @@ -2527,8 +2713,8 @@ class CPtrType(CPointerBaseType): return 1 if other_type.is_null_ptr: return 1 - if self.base_type.is_const: - self = CPtrType(self.base_type.const_base_type) + if self.base_type.is_cv_qualified: + self = CPtrType(self.base_type.cv_base_type) if self.base_type.is_cfunction: if other_type.is_ptr: other_type = other_type.base_type.resolve() @@ -2570,26 +2756,17 @@ class CNullPtrType(CPtrType): is_null_ptr = 1 -class CReferenceType(BaseType): +class CReferenceBaseType(BaseType): - is_reference = 1 is_fake_reference = 0 + # Common base type for C reference and C++ rvalue reference types. + def __init__(self, base_type): self.ref_base_type = base_type def __repr__(self): - return "<CReferenceType %s>" % repr(self.ref_base_type) - - def __str__(self): - return "%s &" % self.ref_base_type - - def declaration_code(self, entity_code, - for_display = 0, dll_linkage = None, pyrex = 0): - #print "CReferenceType.declaration_code: pointer to", self.base_type ### - return self.ref_base_type.declaration_code( - "&%s" % entity_code, - for_display, dll_linkage, pyrex) + return "<%s %s>" % repr(self.__class__.__name__, self.ref_base_type) def specialize(self, values): base_type = self.ref_base_type.specialize(values) @@ -2605,13 +2782,25 @@ class CReferenceType(BaseType): return getattr(self.ref_base_type, name) +class CReferenceType(CReferenceBaseType): + + is_reference = 1 + + def __str__(self): + return "%s &" % self.ref_base_type + + def declaration_code(self, entity_code, + for_display = 0, dll_linkage = None, pyrex = 0): + #print "CReferenceType.declaration_code: pointer to", self.base_type ### + return self.ref_base_type.declaration_code( + "&%s" % entity_code, + for_display, dll_linkage, pyrex) + + class CFakeReferenceType(CReferenceType): is_fake_reference = 1 - def __repr__(self): - return "<CFakeReferenceType %s>" % repr(self.ref_base_type) - def __str__(self): return "%s [&]" % self.ref_base_type @@ -2621,6 +2810,20 @@ class CFakeReferenceType(CReferenceType): return "__Pyx_FakeReference<%s> %s" % (self.ref_base_type.empty_declaration_code(), entity_code) +class CppRvalueReferenceType(CReferenceBaseType): + + is_rvalue_reference = 1 + + def __str__(self): + return "%s &&" % self.ref_base_type + + def declaration_code(self, entity_code, + for_display = 0, dll_linkage = None, pyrex = 0): + return self.ref_base_type.declaration_code( + "&&%s" % entity_code, + for_display, dll_linkage, pyrex) + + class CFuncType(CType): # return_type CType # args [CFuncTypeArg] @@ -2792,8 +2995,8 @@ class CFuncType(CType): # is performed elsewhere). for i in range(as_cmethod, len(other_type.args)): if not self.args[i].type.same_as( - other_type.args[i].type): - return 0 + other_type.args[i].type): + return 0 if self.has_varargs != other_type.has_varargs: return 0 if not self.return_type.subtype_of_resolved_type(other_type.return_type): @@ -3049,8 +3252,16 @@ class CFuncType(CType): if not self.can_coerce_to_pyobject(env): return False from .UtilityCode import CythonUtilityCode - safe_typename = re.sub('[^a-zA-Z0-9]', '__', self.declaration_code("", pyrex=1)) - to_py_function = "__Pyx_CFunc_%s_to_py" % safe_typename + + # include argument names into the c function name to ensure cname is unique + # between functions with identical types but different argument names + from .Symtab import punycodify_name + def arg_name_part(arg): + return "%s%s" % (len(arg.name), punycodify_name(arg.name)) if arg.name else "0" + arg_names = [ arg_name_part(arg) for arg in self.args ] + arg_names = "_".join(arg_names) + safe_typename = type_identifier(self, pyrex=True) + to_py_function = "__Pyx_CFunc_%s_to_py_%s" % (safe_typename, arg_names) for arg in self.args: if not arg.type.is_pyobject and not arg.type.create_from_py_utility_code(env): @@ -3242,7 +3453,7 @@ class CFuncTypeArg(BaseType): self.annotation = annotation self.type = type self.pos = pos - self.needs_type_test = False # TODO: should these defaults be set in analyse_types()? + self.needs_type_test = False # TODO: should these defaults be set in analyse_types()? def __repr__(self): return "%s:%s" % (self.name, repr(self.type)) @@ -3253,6 +3464,12 @@ class CFuncTypeArg(BaseType): def specialize(self, values): return CFuncTypeArg(self.name, self.type.specialize(values), self.pos, self.cname) + def is_forwarding_reference(self): + if self.type.is_rvalue_reference: + if (isinstance(self.type.ref_base_type, TemplatePlaceholderType) + and not self.type.ref_base_type.is_cv_qualified): + return True + return False class ToPyStructUtilityCode(object): @@ -3320,7 +3537,7 @@ class CStructOrUnionType(CType): has_attributes = 1 exception_check = True - def __init__(self, name, kind, scope, typedef_flag, cname, packed=False): + def __init__(self, name, kind, scope, typedef_flag, cname, packed=False, in_cpp=False): self.name = name self.cname = cname self.kind = kind @@ -3335,6 +3552,7 @@ class CStructOrUnionType(CType): self._convert_to_py_code = None self._convert_from_py_code = None self.packed = packed + self.needs_cpp_construction = self.is_struct and in_cpp def can_coerce_to_pyobject(self, env): if self._convert_to_py_code is False: @@ -3412,6 +3630,7 @@ class CStructOrUnionType(CType): var_entries=self.scope.var_entries, funcname=self.from_py_function, ) + env.use_utility_code(UtilityCode.load_cached("RaiseUnexpectedTypeError", "ObjectHandling.c")) from .UtilityCode import CythonUtilityCode self._convert_from_py_code = CythonUtilityCode.load( "FromPyStructUtility" if self.is_struct else "FromPyUnionUtility", @@ -3504,6 +3723,7 @@ class CppClassType(CType): is_cpp_class = 1 has_attributes = 1 + needs_cpp_construction = 1 exception_check = True namespace = None @@ -3694,23 +3914,23 @@ class CppClassType(CType): specialized.namespace = self.namespace.specialize(values) specialized.scope = self.scope.specialize(values, specialized) if self.cname == 'std::vector': - # vector<bool> is special cased in the C++ standard, and its - # accessors do not necessarily return references to the underlying - # elements (which may be bit-packed). - # http://www.cplusplus.com/reference/vector/vector-bool/ - # Here we pretend that the various methods return bool values - # (as the actual returned values are coercable to such, and - # we don't support call expressions as lvalues). - T = values.get(self.templates[0], None) - if T and not T.is_fused and T.empty_declaration_code() == 'bool': - for bit_ref_returner in ('at', 'back', 'front'): - if bit_ref_returner in specialized.scope.entries: - specialized.scope.entries[bit_ref_returner].type.return_type = T + # vector<bool> is special cased in the C++ standard, and its + # accessors do not necessarily return references to the underlying + # elements (which may be bit-packed). + # http://www.cplusplus.com/reference/vector/vector-bool/ + # Here we pretend that the various methods return bool values + # (as the actual returned values are coercable to such, and + # we don't support call expressions as lvalues). + T = values.get(self.templates[0], None) + if T and not T.is_fused and T.empty_declaration_code() == 'bool': + for bit_ref_returner in ('at', 'back', 'front'): + if bit_ref_returner in specialized.scope.entries: + specialized.scope.entries[bit_ref_returner].type.return_type = T return specialized def deduce_template_params(self, actual): - if actual.is_const: - actual = actual.const_base_type + if actual.is_cv_qualified: + actual = actual.cv_base_type if actual.is_reference: actual = actual.ref_base_type if self == actual: @@ -3846,6 +4066,76 @@ class CppClassType(CType): if constructor is not None and best_match([], constructor.all_alternatives()) is None: error(pos, "C++ class must have a nullary constructor to be %s" % msg) +class CppScopedEnumType(CType): + # name string + # doc string or None + # cname string + + is_cpp_enum = True + + def __init__(self, name, cname, underlying_type, namespace=None, doc=None): + self.name = name + self.doc = doc + self.cname = cname + self.values = [] + self.underlying_type = underlying_type + self.namespace = namespace + + def __str__(self): + return self.name + + def declaration_code(self, entity_code, + for_display=0, dll_linkage=None, pyrex=0): + if pyrex or for_display: + type_name = self.name + else: + if self.namespace: + type_name = "%s::%s" % ( + self.namespace.empty_declaration_code(), + self.cname + ) + else: + type_name = "__PYX_ENUM_CLASS_DECL %s" % self.cname + type_name = public_decl(type_name, dll_linkage) + return self.base_declaration_code(type_name, entity_code) + + def create_from_py_utility_code(self, env): + if self.from_py_function: + return True + if self.underlying_type.create_from_py_utility_code(env): + self.from_py_function = '(%s)%s' % ( + self.cname, self.underlying_type.from_py_function + ) + return True + + def create_to_py_utility_code(self, env): + if self.to_py_function is not None: + return True + if self.underlying_type.create_to_py_utility_code(env): + # Using a C++11 lambda here, which is fine since + # scoped enums are a C++11 feature + self.to_py_function = '[](const %s& x){return %s((%s)x);}' % ( + self.cname, + self.underlying_type.to_py_function, + self.underlying_type.empty_declaration_code() + ) + return True + + def create_type_wrapper(self, env): + from .UtilityCode import CythonUtilityCode + rst = CythonUtilityCode.load( + "CppScopedEnumType", "CpdefEnums.pyx", + context={ + "name": self.name, + "cname": self.cname.split("::")[-1], + "items": tuple(self.values), + "underlying_type": self.underlying_type.empty_declaration_code(), + "enum_doc": self.doc, + }, + outer_module_scope=env.global_scope()) + + env.use_utility_code(rst) + class TemplatePlaceholderType(CType): @@ -3896,16 +4186,18 @@ def is_optional_template_param(type): class CEnumType(CIntLike, CType): # name string + # doc string or None # cname string or None # typedef_flag boolean # values [string], populated during declaration analysis is_enum = 1 signed = 1 - rank = -1 # Ranks below any integer type + rank = -1 # Ranks below any integer type - def __init__(self, name, cname, typedef_flag, namespace=None): + def __init__(self, name, cname, typedef_flag, namespace=None, doc=None): self.name = name + self.doc = doc self.cname = cname self.values = [] self.typedef_flag = typedef_flag @@ -3947,7 +4239,9 @@ class CEnumType(CIntLike, CType): env.use_utility_code(CythonUtilityCode.load( "EnumType", "CpdefEnums.pyx", context={"name": self.name, - "items": tuple(self.values)}, + "items": tuple(self.values), + "enum_doc": self.doc, + }, outer_module_scope=env.global_scope())) @@ -4085,14 +4379,14 @@ class ErrorType(PyrexType): rank_to_type_name = ( - "char", # 0 - "short", # 1 - "int", # 2 - "long", # 3 - "PY_LONG_LONG", # 4 - "float", # 5 - "double", # 6 - "long double", # 7 + "char", # 0 + "short", # 1 + "int", # 2 + "long", # 3 + "PY_LONG_LONG", # 4 + "float", # 5 + "double", # 6 + "long double", # 7 ) _rank_to_type_name = list(rank_to_type_name) @@ -4302,8 +4596,7 @@ def best_match(arg_types, functions, pos=None, env=None, args=None): # Check no. of args max_nargs = len(func_type.args) min_nargs = max_nargs - func_type.optional_arg_count - if actual_nargs < min_nargs or \ - (not func_type.has_varargs and actual_nargs > max_nargs): + if actual_nargs < min_nargs or (not func_type.has_varargs and actual_nargs > max_nargs): if max_nargs == min_nargs and not func_type.has_varargs: expectation = max_nargs elif actual_nargs < min_nargs: @@ -4455,10 +4748,10 @@ def widest_numeric_type(type1, type2): type1 = type1.ref_base_type if type2.is_reference: type2 = type2.ref_base_type - if type1.is_const: - type1 = type1.const_base_type - if type2.is_const: - type2 = type2.const_base_type + if type1.is_cv_qualified: + type1 = type1.cv_base_type + if type2.is_cv_qualified: + type2 = type2.cv_base_type if type1 == type2: widest_type = type1 elif type1.is_complex or type2.is_complex: @@ -4648,35 +4941,38 @@ def parse_basic_type(name): name = 'int' return simple_c_type(signed, longness, name) -def c_array_type(base_type, size): - # Construct a C array type. + +def _construct_type_from_base(cls, base_type, *args): if base_type is error_type: return error_type - else: - return CArrayType(base_type, size) + return cls(base_type, *args) + +def c_array_type(base_type, size): + # Construct a C array type. + return _construct_type_from_base(CArrayType, base_type, size) def c_ptr_type(base_type): # Construct a C pointer type. - if base_type is error_type: - return error_type - elif base_type.is_reference: - return CPtrType(base_type.ref_base_type) - else: - return CPtrType(base_type) + if base_type.is_reference: + base_type = base_type.ref_base_type + return _construct_type_from_base(CPtrType, base_type) def c_ref_type(base_type): # Construct a C reference type - if base_type is error_type: - return error_type - else: - return CReferenceType(base_type) + return _construct_type_from_base(CReferenceType, base_type) + +def cpp_rvalue_ref_type(base_type): + # Construct a C++ rvalue reference type + return _construct_type_from_base(CppRvalueReferenceType, base_type) def c_const_type(base_type): # Construct a C const type. - if base_type is error_type: - return error_type - else: - return CConstType(base_type) + return _construct_type_from_base(CConstType, base_type) + +def c_const_or_volatile_type(base_type, is_const, is_volatile): + # Construct a C const/volatile type. + return _construct_type_from_base(CConstOrVolatileType, base_type, is_const, is_volatile) + def same_type(type1, type2): return type1.same_as(type2) @@ -4702,28 +4998,42 @@ def typecast(to_type, from_type, expr_code): def type_list_identifier(types): return cap_length('__and_'.join(type_identifier(type) for type in types)) +_special_type_characters = { + '__': '__dunder', + 'const ': '__const_', + ' ': '__space_', + '*': '__ptr', + '&': '__ref', + '&&': '__fwref', + '[': '__lArr', + ']': '__rArr', + '<': '__lAng', + '>': '__rAng', + '(': '__lParen', + ')': '__rParen', + ',': '__comma_', + '...': '__EL', + '::': '__in_', + ':': '__D', +} + +_escape_special_type_characters = partial(re.compile( + # join substrings in reverse order to put longer matches first, e.g. "::" before ":" + " ?(%s) ?" % "|".join(re.escape(s) for s in sorted(_special_type_characters, reverse=True)) +).sub, lambda match: _special_type_characters[match.group(1)]) + +def type_identifier(type, pyrex=False): + decl = type.empty_declaration_code(pyrex=pyrex) + return type_identifier_from_declaration(decl) + _type_identifier_cache = {} -def type_identifier(type): - decl = type.empty_declaration_code() +def type_identifier_from_declaration(decl): safe = _type_identifier_cache.get(decl) if safe is None: safe = decl safe = re.sub(' +', ' ', safe) - safe = re.sub(' ([^a-zA-Z0-9_])', r'\1', safe) - safe = re.sub('([^a-zA-Z0-9_]) ', r'\1', safe) - safe = (safe.replace('__', '__dunder') - .replace('const ', '__const_') - .replace(' ', '__space_') - .replace('*', '__ptr') - .replace('&', '__ref') - .replace('[', '__lArr') - .replace(']', '__rArr') - .replace('<', '__lAng') - .replace('>', '__rAng') - .replace('(', '__lParen') - .replace(')', '__rParen') - .replace(',', '__comma_') - .replace('::', '__in_')) + safe = re.sub(' ?([^a-zA-Z0-9_]) ?', r'\1', safe) + safe = _escape_special_type_characters(safe) safe = cap_length(re.sub('[^a-zA-Z0-9_]', lambda x: '__%X' % ord(x.group(0)), safe)) _type_identifier_cache[decl] = safe return safe diff --git a/Cython/Compiler/Scanning.pxd b/Cython/Compiler/Scanning.pxd index 59593f88a..351d55560 100644 --- a/Cython/Compiler/Scanning.pxd +++ b/Cython/Compiler/Scanning.pxd @@ -1,4 +1,4 @@ -from __future__ import absolute_import +# cython: language_level=3 import cython @@ -9,11 +9,6 @@ cdef unicode any_string_prefix, IDENT cdef get_lexicon() cdef initial_compile_time_env() -cdef class Method: - cdef object name - cdef dict kwargs - cdef readonly object __name__ # for tracing the scanner - ## methods commented with '##' out are used by Parsing.py when compiled. @cython.final @@ -39,8 +34,8 @@ cdef class PyrexScanner(Scanner): cdef public indentation_char cdef public int bracket_nesting_level cdef readonly bint async_enabled - cdef public sy - cdef public systring + cdef public unicode sy + cdef public systring # EncodedString cdef long current_level(self) #cpdef commentline(self, text) diff --git a/Cython/Compiler/Scanning.py b/Cython/Compiler/Scanning.py index f61144033..feb428638 100644 --- a/Cython/Compiler/Scanning.py +++ b/Cython/Compiler/Scanning.py @@ -1,4 +1,4 @@ -# cython: infer_types=True, language_level=3, py2_import=True, auto_pickle=False +# cython: infer_types=True, language_level=3, auto_pickle=False # # Cython Scanner # @@ -12,6 +12,7 @@ cython.declare(make_lexicon=object, lexicon=object, import os import platform +from unicodedata import normalize from .. import Utils from ..Plex.Scanners import Scanner @@ -51,25 +52,6 @@ pyx_reserved_words = py_reserved_words + [ ] -class Method(object): - - def __init__(self, name, **kwargs): - self.name = name - self.kwargs = kwargs or None - self.__name__ = name # for Plex tracing - - def __call__(self, stream, text): - method = getattr(stream, self.name) - # self.kwargs is almost always unused => avoid call overhead - return method(text, **self.kwargs) if self.kwargs is not None else method(text) - - def __copy__(self): - return self # immutable, no need to copy - - def __deepcopy__(self, memo): - return self # immutable, no need to copy - - #------------------------------------------------------------------ class CompileTimeScope(object): @@ -154,7 +136,7 @@ class SourceDescriptor(object): _escaped_description = None _cmp_name = '' def __str__(self): - assert False # To catch all places where a descriptor is used directly as a filename + assert False # To catch all places where a descriptor is used directly as a filename def set_file_type_from_name(self, filename): name, ext = os.path.splitext(filename) @@ -360,6 +342,13 @@ class PyrexScanner(Scanner): self.sy = '' self.next() + def normalize_ident(self, text): + try: + text.encode('ascii') # really just name.isascii but supports Python 2 and 3 + except UnicodeEncodeError: + text = normalize('NFKC', text) + self.produce(IDENT, text) + def commentline(self, text): if self.parse_comments: self.produce('commentline', text) @@ -463,7 +452,7 @@ class PyrexScanner(Scanner): systring = self.context.intern_ustring(systring) self.sy = sy self.systring = systring - if False: # debug_scanner: + if False: # debug_scanner: _, line, col = self.position() if not self.systring or self.sy == self.systring: t = self.sy diff --git a/Cython/Compiler/StringEncoding.py b/Cython/Compiler/StringEncoding.py index c37e8aab7..192fc3de3 100644 --- a/Cython/Compiler/StringEncoding.py +++ b/Cython/Compiler/StringEncoding.py @@ -138,6 +138,24 @@ class EncodedString(_unicode): def as_utf8_string(self): return bytes_literal(self.utf8encode(), 'utf8') + def as_c_string_literal(self): + # first encodes the string then produces a c string literal + if self.encoding is None: + s = self.as_utf8_string() + else: + s = bytes_literal(self.byteencode(), self.encoding) + return s.as_c_string_literal() + + if not hasattr(_unicode, "isascii"): + def isascii(self): + # not defined for Python3.7+ since the class already has it + try: + self.encode("ascii") + except UnicodeEncodeError: + return False + else: + return True + def string_contains_surrogates(ustring): """ @@ -211,6 +229,11 @@ class BytesLiteral(_bytes): value = split_string_literal(escape_byte_string(self)) return '"%s"' % value + if not hasattr(_bytes, "isascii"): + def isascii(self): + # already defined for Python3.7+ + return True + def bytes_literal(s, encoding): assert isinstance(s, bytes) @@ -226,6 +249,12 @@ def encoded_string(s, encoding): s.encoding = encoding return s +def encoded_string_or_bytes_literal(s, encoding): + if isinstance(s, bytes): + return bytes_literal(s, encoding) + else: + return encoded_string(s, encoding) + char_from_escape_sequence = { r'\a' : u'\a', @@ -291,7 +320,7 @@ def escape_byte_string(s): """ s = _replace_specials(s) try: - return s.decode("ASCII") # trial decoding: plain ASCII => done + return s.decode("ASCII") # trial decoding: plain ASCII => done except UnicodeDecodeError: pass if IS_PYTHON3: @@ -324,7 +353,7 @@ def split_string_literal(s, limit=2000): while start < len(s): end = start + limit if len(s) > end-4 and '\\' in s[end-4:end]: - end -= 4 - s[end-4:end].find('\\') # just before the backslash + end -= 4 - s[end-4:end].find('\\') # just before the backslash while s[end-1] == '\\': end -= 1 if end == start: diff --git a/Cython/Compiler/Symtab.py b/Cython/Compiler/Symtab.py index dff22b7ae..9dfeb87bd 100644 --- a/Cython/Compiler/Symtab.py +++ b/Cython/Compiler/Symtab.py @@ -42,6 +42,28 @@ def c_safe_identifier(cname): cname = Naming.pyrex_prefix + cname return cname +def punycodify_name(cname, mangle_with=None): + # if passed the mangle_with should be a byte string + # modified from PEP489 + try: + cname.encode('ascii') + except UnicodeEncodeError: + cname = cname.encode('punycode').replace(b'-', b'_').decode('ascii') + if mangle_with: + # sometimes it necessary to mangle unicode names alone where + # they'll be inserted directly into C, because the punycode + # transformation can turn them into invalid identifiers + cname = "%s_%s" % (mangle_with, cname) + elif cname.startswith(Naming.pyrex_prefix): + # a punycode name could also be a valid ascii variable name so + # change the prefix to distinguish + cname = cname.replace(Naming.pyrex_prefix, + Naming.pyunicode_identifier_prefix, 1) + + return cname + + + class BufferAux(object): writable_needed = False @@ -87,6 +109,7 @@ class Entry(object): # doc_cname string or None C const holding the docstring # getter_cname string C func for getting property # setter_cname string C func for setting or deleting property + # is_cproperty boolean Is an inline property of an external type # is_self_arg boolean Is the "self" arg of an exttype method # is_arg boolean Is the arg of a method # is_local boolean Is a local variable @@ -134,6 +157,7 @@ class Entry(object): # cf_used boolean Entry is used # is_fused_specialized boolean Whether this entry of a cdef or def function # is a specialization + # is_cgetter boolean Is a c-level getter function # TODO: utility_code and utility_code_definition serves the same purpose... @@ -160,6 +184,7 @@ class Entry(object): is_cpp_class = 0 is_const = 0 is_property = 0 + is_cproperty = 0 doc_cname = None getter_cname = None setter_cname = None @@ -203,6 +228,7 @@ class Entry(object): error_on_uninitialized = False cf_used = True outer_entry = None + is_cgetter = False def __init__(self, name, cname, type, pos = None, init = None): self.name = name @@ -238,6 +264,10 @@ class Entry(object): else: return NotImplemented + @property + def cf_is_reassigned(self): + return len(self.cf_assignments) > 1 + class InnerEntry(Entry): """ @@ -347,7 +377,6 @@ class Scope(object): self.defined_c_classes = [] self.imported_c_classes = {} self.cname_to_entry = {} - self.string_to_entry = {} self.identifier_to_entry = {} self.num_to_entry = {} self.obj_to_entry = {} @@ -358,11 +387,11 @@ class Scope(object): def __deepcopy__(self, memo): return self - def merge_in(self, other, merge_unused=True, whitelist=None): + def merge_in(self, other, merge_unused=True, allowlist=None): # Use with care... entries = [] for name, entry in other.entries.items(): - if not whitelist or name in whitelist: + if not allowlist or name in allowlist: if entry.used or merge_unused: entries.append((name, entry)) @@ -390,7 +419,7 @@ class Scope(object): def mangle(self, prefix, name = None): if name: - return "%s%s%s" % (prefix, self.scope_prefix, name) + return punycodify_name("%s%s%s" % (prefix, self.scope_prefix, name)) else: return self.parent_scope.mangle(prefix, self.name) @@ -440,11 +469,12 @@ class Scope(object): # Create new entry, and add to dictionary if # name is not None. Reports a warning if already # declared. - if type.is_buffer and not isinstance(self, LocalScope): # and not is_type: + if type.is_buffer and not isinstance(self, LocalScope): # and not is_type: error(pos, 'Buffer types only allowed as function local variables') if not self.in_cinclude and cname and re.match("^_[_A-Z]+$", cname): - # See http://www.gnu.org/software/libc/manual/html_node/Reserved-Names.html#Reserved-Names + # See https://www.gnu.org/software/libc/manual/html_node/Reserved-Names.html#Reserved-Names warning(pos, "'%s' is a reserved name in C." % cname, -1) + entries = self.entries if name and name in entries and not shadow: old_entry = entries[name] @@ -487,8 +517,7 @@ class Scope(object): entries[name] = entry if type.is_memoryviewslice: - from . import MemoryView - entry.init = MemoryView.memslice_entry_init + entry.init = type.default_value entry.scope = self entry.visibility = visibility @@ -563,8 +592,10 @@ class Scope(object): cname = self.mangle(Naming.type_prefix, name) entry = self.lookup_here(name) if not entry: + in_cpp = self.is_cpp() type = PyrexTypes.CStructOrUnionType( - name, kind, scope, typedef_flag, cname, packed) + name, kind, scope, typedef_flag, cname, packed, + in_cpp = in_cpp) entry = self.declare_type(name, type, pos, cname, visibility = visibility, api = api, defining = scope is not None) @@ -650,12 +681,12 @@ class Scope(object): error(pos, "'%s' previously declared as '%s'" % ( entry.name, entry.visibility)) - def declare_enum(self, name, pos, cname, typedef_flag, - visibility = 'private', api = 0, create_wrapper = 0): + def declare_enum(self, name, pos, cname, scoped, typedef_flag, + visibility='private', api=0, create_wrapper=0, doc=None): if name: if not cname: if (self.in_cinclude or visibility == 'public' - or visibility == 'extern' or api): + or visibility == 'extern' or api): cname = name else: cname = self.mangle(Naming.type_prefix, name) @@ -663,13 +694,21 @@ class Scope(object): namespace = self.outer_scope.lookup(self.name).type else: namespace = None - type = PyrexTypes.CEnumType(name, cname, typedef_flag, namespace) + + if scoped: + type = PyrexTypes.CppScopedEnumType(name, cname, namespace, doc=doc) + else: + type = PyrexTypes.CEnumType(name, cname, typedef_flag, namespace, doc=doc) else: type = PyrexTypes.c_anon_enum_type entry = self.declare_type(name, type, pos, cname = cname, visibility = visibility, api = api) + if scoped: + entry.utility_code = Code.UtilityCode.load_cached("EnumClassDecl", "CppSupport.cpp") + self.use_entry_utility_code(entry) entry.create_wrapper = create_wrapper entry.enum_values = [] + self.sue_entries.append(entry) return entry @@ -698,6 +737,7 @@ class Scope(object): return entry def declare_builtin(self, name, pos): + name = self.mangle_class_private_name(name) return self.outer_scope.declare_builtin(name, pos) def _declare_pyfunction(self, name, pos, visibility='extern', entry=None): @@ -719,7 +759,7 @@ class Scope(object): entry.type = py_object_type elif entry.type is not py_object_type: return self._declare_pyfunction(name, pos, visibility=visibility, entry=entry) - else: # declare entry stub + else: # declare entry stub self.declare_var(name, py_object_type, pos, visibility=visibility) entry = self.declare_var(None, py_object_type, pos, cname=name, visibility='private') @@ -736,7 +776,7 @@ class Scope(object): qualified_name = self.qualify_name(lambda_name) entry = self.declare(None, func_cname, py_object_type, pos, 'private') - entry.name = lambda_name + entry.name = EncodedString(lambda_name) entry.qualified_name = qualified_name entry.pymethdef_cname = pymethdef_cname entry.func_cname = func_cname @@ -769,7 +809,8 @@ class Scope(object): entry.cname = cname entry.func_cname = cname if visibility != 'private' and visibility != entry.visibility: - warning(pos, "Function '%s' previously declared as '%s', now as '%s'" % (name, entry.visibility, visibility), 1) + warning(pos, "Function '%s' previously declared as '%s', now as '%s'" % ( + name, entry.visibility, visibility), 1) if overridable != entry.is_overridable: warning(pos, "Function '%s' previously declared as '%s'" % ( name, 'cpdef' if overridable else 'cdef'), 1) @@ -786,7 +827,7 @@ class Scope(object): # it's safe to allow signature overrides for alt_entry in entry.all_alternatives(): if not alt_entry.cname or cname == alt_entry.cname: - break # cname not unique! + break # cname not unique! else: can_override = True if can_override: @@ -822,6 +863,7 @@ class Scope(object): if overridable: # names of cpdef functions can be used as variables and can be assigned to var_entry = Entry(name, cname, py_object_type) # FIXME: cname? + var_entry.qualified_name = self.qualify_name(name) var_entry.is_variable = 1 var_entry.is_pyglobal = 1 var_entry.scope = entry.scope @@ -829,6 +871,23 @@ class Scope(object): type.entry = entry return entry + def declare_cgetter(self, name, return_type, pos=None, cname=None, + visibility="private", modifiers=(), defining=False, **cfunc_type_config): + assert all( + k in ('exception_value', 'exception_check', 'nogil', 'with_gil', 'is_const_method', 'is_static_method') + for k in cfunc_type_config + ) + cfunc_type = PyrexTypes.CFuncType( + return_type, + [PyrexTypes.CFuncTypeArg("self", self.parent_type, None)], + **cfunc_type_config) + entry = self.declare_cfunction( + name, cfunc_type, pos, cname=None, visibility=visibility, modifiers=modifiers, defining=defining) + entry.is_cgetter = True + if cname is not None: + entry.func_cname = cname + return entry + def add_cfunction(self, name, type, pos, cname, visibility, modifiers, inherited=False): # Add a C function entry without giving it a func_cname. entry = self.declare(name, cname, type, pos, visibility) @@ -874,12 +933,33 @@ class Scope(object): def lookup(self, name): # Look up name in this scope or an enclosing one. # Return None if not found. - return (self.lookup_here(name) - or (self.outer_scope and self.outer_scope.lookup(name)) - or None) + + mangled_name = self.mangle_class_private_name(name) + entry = (self.lookup_here(name) # lookup here also does mangling + or (self.outer_scope and self.outer_scope.lookup(mangled_name)) + or None) + if entry: + return entry + + # look up the original name in the outer scope + # Not strictly Python behaviour but see https://github.com/cython/cython/issues/3544 + entry = (self.outer_scope and self.outer_scope.lookup(name)) or None + if entry and entry.is_pyglobal: + self._emit_class_private_warning(entry.pos, name) + return entry def lookup_here(self, name): # Look up in this scope only, return None if not found. + + entry = self.entries.get(self.mangle_class_private_name(name), None) + if entry: + return entry + # Also check the unmangled name in the current scope + # (even if mangling should give us something else). + # This is to support things like global __foo which makes a declaration for __foo + return self.entries.get(name, None) + + def lookup_here_unmangled(self, name): return self.entries.get(name, None) def lookup_target(self, name): @@ -887,6 +967,10 @@ class Scope(object): # variable if not found. entry = self.lookup_here(name) if not entry: + entry = self.lookup_here_unmangled(name) + if entry and entry.is_pyglobal: + self._emit_class_private_warning(entry.pos, name) + if not entry: entry = self.declare_var(name, py_object_type, None) return entry @@ -903,8 +987,7 @@ class Scope(object): method = obj_type.scope.lookup("operator%s" % operator) if method is not None: arg_types = [arg.type for arg in operands[1:]] - res = PyrexTypes.best_match([arg.type for arg in operands[1:]], - method.all_alternatives()) + res = PyrexTypes.best_match(arg_types, method.all_alternatives()) if res is not None: return res function = self.lookup("operator%s" % operator) @@ -914,7 +997,7 @@ class Scope(object): # look-up nonmember methods listed within a class method_alternatives = [] - if len(operands)==2: # binary operators only + if len(operands) == 2: # binary operators only for n in range(2): if operands[n].type.is_cpp_class: obj_type = operands[n].type @@ -938,6 +1021,11 @@ class Scope(object): operands = [FakeOperand(pos, type=type) for type in types] return self.lookup_operator(operator, operands) + def _emit_class_private_warning(self, pos, name): + warning(pos, "Global name %s matched from within class scope " + "in contradiction to to Python 'class private name' rules. " + "This may change in a future release." % name, 1) + def use_utility_code(self, new_code): self.global_scope().use_utility_code(new_code) @@ -1026,14 +1114,14 @@ class BuiltinScope(Scope): # If python_equiv == "*", the Python equivalent has the same name # as the entry, otherwise it has the name specified by python_equiv. name = EncodedString(name) - entry = self.declare_cfunction(name, type, None, cname, visibility='extern', - utility_code=utility_code) + entry = self.declare_cfunction(name, type, None, cname, visibility='extern', utility_code=utility_code) if python_equiv: if python_equiv == "*": python_equiv = name else: python_equiv = EncodedString(python_equiv) var_entry = Entry(python_equiv, python_equiv, py_object_type) + var_entry.qualified_name = self.qualify_name(name) var_entry.is_variable = 1 var_entry.is_builtin = 1 var_entry.utility_code = utility_code @@ -1053,10 +1141,13 @@ class BuiltinScope(Scope): entry = self.declare_type(name, type, None, visibility='extern') entry.utility_code = utility_code - var_entry = Entry(name = entry.name, - type = self.lookup('type').type, # make sure "type" is the first type declared... - pos = entry.pos, - cname = entry.type.typeptr_cname) + var_entry = Entry( + name=entry.name, + type=self.lookup('type').type, # make sure "type" is the first type declared... + pos=entry.pos, + cname=entry.type.typeptr_cname, + ) + var_entry.qualified_name = self.qualify_name(name) var_entry.is_variable = 1 var_entry.is_cglobal = 1 var_entry.is_readonly = 1 @@ -1101,7 +1192,7 @@ class BuiltinScope(Scope): "True": ["Py_True", py_object_type], } -const_counter = 1 # As a temporary solution for compiling code in pxds +const_counter = 1 # As a temporary solution for compiling code in pxds class ModuleScope(Scope): # module_name string Python name of the module @@ -1113,7 +1204,6 @@ class ModuleScope(Scope): # utility_code_list [UtilityCode] Queuing utility codes for forwarding to Code.py # c_includes {key: IncludeCode} C headers or verbatim code to be generated # See process_include() for more documentation - # string_to_entry {string : Entry} Map string const to entry # identifier_to_entry {string : Entry} Map identifier string const to entry # context Context # parent_module Scope Parent in the import namespace @@ -1236,7 +1326,7 @@ class ModuleScope(Scope): entry = self.declare(None, None, py_object_type, pos, 'private') if Options.cache_builtins and name not in Code.uncachable_builtins: entry.is_builtin = 1 - entry.is_const = 1 # cached + entry.is_const = 1 # cached entry.name = name entry.cname = Naming.builtin_prefix + name self.cached_builtins.append(entry) @@ -1244,6 +1334,7 @@ class ModuleScope(Scope): else: entry.is_builtin = 1 entry.name = name + entry.qualified_name = self.builtin_scope().qualify_name(name) return entry def find_module(self, module_name, pos, relative_level=-1): @@ -1360,7 +1451,7 @@ class ModuleScope(Scope): entry = self.lookup_here(name) if entry: if entry.is_pyglobal and entry.as_module is scope: - return entry # Already declared as the same module + return entry # Already declared as the same module if not (entry.is_pyglobal and not entry.as_module): # SAGE -- I put this here so Pyrex # cimport's work across directories. @@ -1384,7 +1475,7 @@ class ModuleScope(Scope): # object type, and not declared with cdef, it will live # in the module dictionary, otherwise it will be a C # global variable. - if not visibility in ('private', 'public', 'extern'): + if visibility not in ('private', 'public', 'extern'): error(pos, "Module-level variable cannot be declared %s" % visibility) if not is_cdef: if type is unspecified_type: @@ -1501,7 +1592,7 @@ class ModuleScope(Scope): if entry and not shadow: type = entry.type if not (entry.is_type and type.is_extension_type): - entry = None # Will cause redeclaration and produce an error + entry = None # Will cause redeclaration and produce an error else: scope = type.scope if typedef_flag and (not scope or scope.defined): @@ -1707,6 +1798,7 @@ class ModuleScope(Scope): type = Builtin.type_type, pos = entry.pos, cname = entry.type.typeptr_cname) + var_entry.qualified_name = entry.qualified_name var_entry.is_variable = 1 var_entry.is_cglobal = 1 var_entry.is_readonly = 1 @@ -1735,10 +1827,11 @@ class LocalScope(Scope): Scope.__init__(self, name, outer_scope, parent_scope) def mangle(self, prefix, name): - return prefix + name + return punycodify_name(prefix + name) def declare_arg(self, name, type, pos): # Add an entry for an argument of a function. + name = self.mangle_class_private_name(name) cname = self.mangle(Naming.var_prefix, name) entry = self.declare(name, cname, type, pos, 'private') entry.is_variable = 1 @@ -1752,6 +1845,7 @@ class LocalScope(Scope): def declare_var(self, name, type, pos, cname = None, visibility = 'private', api = 0, in_pxd = 0, is_cdef = 0): + name = self.mangle_class_private_name(name) # Add an entry for a local variable. if visibility in ('public', 'readonly'): error(pos, "Local variable cannot be declared %s" % visibility) @@ -1788,6 +1882,7 @@ class LocalScope(Scope): def lookup(self, name): # Look up name in this scope or an enclosing one. # Return None if not found. + entry = Scope.lookup(self, name) if entry is not None: entry_scope = entry.scope @@ -1855,7 +1950,7 @@ class GeneratorExpressionScope(Scope): # if the outer scope defines a type for this variable, inherit it outer_entry = self.outer_scope.lookup(name) if outer_entry and outer_entry.is_variable: - type = outer_entry.type # may still be 'unspecified_type' ! + type = outer_entry.type # may still be 'unspecified_type' ! # the parent scope needs to generate code for the variable, but # this scope must hold its name exclusively cname = '%s%s' % (self.genexp_prefix, self.parent_scope.mangle(Naming.var_prefix, name or self.next_id())) @@ -1949,6 +2044,14 @@ class ClassScope(Scope): # declared in the class # doc string or None Doc string + def mangle_class_private_name(self, name): + # a few utilitycode names need to specifically be ignored + if name and name.lower().startswith("__pyx_"): + return name + if name and name.startswith('__') and not name.endswith('__'): + name = EncodedString('_%s%s' % (self.class_name.lstrip('_'), name)) + return name + def __init__(self, name, outer_scope): Scope.__init__(self, name, outer_scope, outer_scope) self.class_name = name @@ -1982,22 +2085,10 @@ class PyClassScope(ClassScope): is_py_class_scope = 1 - def mangle_class_private_name(self, name): - return self.mangle_special_name(name) - - def mangle_special_name(self, name): - if name and name.startswith('__') and not name.endswith('__'): - name = EncodedString('_%s%s' % (self.class_name.lstrip('_'), name)) - return name - - def lookup_here(self, name): - name = self.mangle_special_name(name) - return ClassScope.lookup_here(self, name) - def declare_var(self, name, type, pos, cname = None, visibility = 'private', api = 0, in_pxd = 0, is_cdef = 0): - name = self.mangle_special_name(name) + name = self.mangle_class_private_name(name) if type is unspecified_type: type = py_object_type # Add an entry for a class attribute. @@ -2038,7 +2129,7 @@ class PyClassScope(ClassScope): class CClassScope(ClassScope): # Namespace of an extension type. # - # parent_type CClassType + # parent_type PyExtensionType # #typeobj_cname string or None # #objstruct_cname string # method_table_cname string @@ -2057,7 +2148,7 @@ class CClassScope(ClassScope): has_pyobject_attrs = False has_memoryview_attrs = False - has_cpp_class_attrs = False + has_cpp_constructable_attrs = False has_cyclic_pyobject_attrs = False defined = False implemented = False @@ -2082,6 +2173,22 @@ class CClassScope(ClassScope): return not self.parent_type.is_gc_simple return False + def needs_trashcan(self): + # If the trashcan directive is explicitly set to False, + # unconditionally disable the trashcan. + directive = self.directives.get('trashcan') + if directive is False: + return False + # If the directive is set to True and the class has Python-valued + # C attributes, then it should use the trashcan in tp_dealloc. + if directive and self.has_cyclic_pyobject_attrs: + return True + # Use the trashcan if the base class uses it + base_type = self.parent_type.base_type + if base_type and base_type.scope is not None: + return base_type.scope.needs_trashcan() + return self.parent_type.builtin_trashcan + def needs_tp_clear(self): """ Do we need to generate an implementation for the tp_clear slot? Can @@ -2111,6 +2218,7 @@ class CClassScope(ClassScope): def declare_var(self, name, type, pos, cname = None, visibility = 'private', api = 0, in_pxd = 0, is_cdef = 0): + name = self.mangle_class_private_name(name) if is_cdef: # Add an entry for an attribute. if self.defined: @@ -2125,16 +2233,17 @@ class CClassScope(ClassScope): cname = name if visibility == 'private': cname = c_safe_identifier(cname) + cname = punycodify_name(cname, Naming.unicode_structmember_prefix) if type.is_cpp_class and visibility != 'extern': type.check_nullary_constructor(pos) - self.use_utility_code(Code.UtilityCode("#include <new>")) entry = self.declare(name, cname, type, pos, visibility) entry.is_variable = 1 self.var_entries.append(entry) if type.is_memoryviewslice: self.has_memoryview_attrs = True - elif type.is_cpp_class: - self.has_cpp_class_attrs = True + elif type.needs_cpp_construction: + self.use_utility_code(Code.UtilityCode("#include <new>")) + self.has_cpp_constructable_attrs = True elif type.is_pyobject and (self.is_closure_class_scope or name != '__weakref__'): self.has_pyobject_attrs = True if (not type.is_builtin_type @@ -2164,10 +2273,11 @@ class CClassScope(ClassScope): cname=cname, visibility=visibility, api=api, in_pxd=in_pxd, is_cdef=is_cdef) entry.is_member = 1 - entry.is_pyglobal = 1 # xxx: is_pyglobal changes behaviour in so many places that - # I keep it in for now. is_member should be enough - # later on + # xxx: is_pyglobal changes behaviour in so many places that I keep it in for now. + # is_member should be enough later on + entry.is_pyglobal = 1 self.namespace_cname = "(PyObject *)%s" % self.parent_type.typeptr_cname + return entry def declare_pyfunction(self, name, pos, allow_redefine=False): @@ -2215,6 +2325,7 @@ class CClassScope(ClassScope): def declare_cfunction(self, name, type, pos, cname=None, visibility='private', api=0, in_pxd=0, defining=0, modifiers=(), utility_code=None, overridable=False): + name = self.mangle_class_private_name(name) if get_special_method_signature(name) and not self.parent_type.is_builtin_type: error(pos, "Special methods must be declared with 'def', not 'cdef'") args = type.args @@ -2226,7 +2337,7 @@ class CClassScope(ClassScope): (args[0].type, name, self.parent_type)) entry = self.lookup_here(name) if cname is None: - cname = c_safe_identifier(name) + cname = punycodify_name(c_safe_identifier(name), Naming.unicode_vtabentry_prefix) if entry: if not entry.is_cfunction: warning(pos, "'%s' redeclared " % name, 0) @@ -2241,13 +2352,14 @@ class CClassScope(ClassScope): entry.type = entry.type.with_with_gil(type.with_gil) elif type.compatible_signature_with(entry.type, as_cmethod = 1) and type.nogil == entry.type.nogil: if (self.defined and not in_pxd - and not type.same_c_signature_as_resolved_type(entry.type, as_cmethod = 1, as_pxd_definition = 1)): + and not type.same_c_signature_as_resolved_type( + entry.type, as_cmethod=1, as_pxd_definition=1)): # TODO(robertwb): Make this an error. warning(pos, "Compatible but non-identical C method '%s' not redeclared " "in definition part of extension type '%s'. " "This may cause incorrect vtables to be generated." % ( - name, self.class_name), 2) + name, self.class_name), 2) warning(entry.pos, "Previous declaration is here", 2) entry = self.add_cfunction(name, type, pos, cname, visibility='ignore', modifiers=modifiers) else: @@ -2267,8 +2379,7 @@ class CClassScope(ClassScope): if u'inline' in modifiers: entry.is_inline_cmethod = True - if (self.parent_type.is_final_type or entry.is_inline_cmethod or - self.directives.get('final')): + if self.parent_type.is_final_type or entry.is_inline_cmethod or self.directives.get('final'): entry.is_final_cmethod = True entry.final_func_cname = entry.func_cname @@ -2277,8 +2388,8 @@ class CClassScope(ClassScope): def add_cfunction(self, name, type, pos, cname, visibility, modifiers, inherited=False): # Add a cfunction entry without giving it a func_cname. prev_entry = self.lookup_here(name) - entry = ClassScope.add_cfunction(self, name, type, pos, cname, - visibility, modifiers, inherited=inherited) + entry = ClassScope.add_cfunction( + self, name, type, pos, cname, visibility, modifiers, inherited=inherited) entry.is_cmethod = 1 entry.prev_entry = prev_entry return entry @@ -2287,9 +2398,10 @@ class CClassScope(ClassScope): # overridden methods of builtin types still have their Python # equivalent that must be accessible to support bound methods name = EncodedString(name) - entry = self.declare_cfunction(name, type, None, cname, visibility='extern', - utility_code=utility_code) + entry = self.declare_cfunction( + name, type, pos=None, cname=cname, visibility='extern', utility_code=utility_code) var_entry = Entry(name, name, py_object_type) + var_entry.qualified_name = name var_entry.is_variable = 1 var_entry.is_builtin = 1 var_entry.utility_code = utility_code @@ -2297,18 +2409,44 @@ class CClassScope(ClassScope): entry.as_variable = var_entry return entry - def declare_property(self, name, doc, pos): + def declare_property(self, name, doc, pos, ctype=None, property_scope=None): entry = self.lookup_here(name) if entry is None: - entry = self.declare(name, name, py_object_type, pos, 'private') - entry.is_property = 1 + entry = self.declare(name, name, py_object_type if ctype is None else ctype, pos, 'private') + entry.is_property = True + if ctype is not None: + entry.is_cproperty = True entry.doc = doc - entry.scope = PropertyScope(name, - outer_scope = self.global_scope(), parent_scope = self) - entry.scope.parent_type = self.parent_type + if property_scope is None: + entry.scope = PropertyScope(name, class_scope=self) + else: + entry.scope = property_scope self.property_entries.append(entry) return entry + def declare_cproperty(self, name, type, cfunc_name, doc=None, pos=None, visibility='extern', + nogil=False, with_gil=False, exception_value=None, exception_check=False, + utility_code=None): + """Internal convenience method to declare a C property function in one go. + """ + property_entry = self.declare_property(name, doc=doc, ctype=type, pos=pos) + cfunc_entry = property_entry.scope.declare_cfunction( + name=name, + type=PyrexTypes.CFuncType( + type, + [PyrexTypes.CFuncTypeArg("self", self.parent_type, pos=None)], + nogil=nogil, + with_gil=with_gil, + exception_value=exception_value, + exception_check=exception_check, + ), + cname=cfunc_name, + utility_code=utility_code, + visibility=visibility, + pos=pos, + ) + return property_entry, cfunc_entry + def declare_inherited_c_attributes(self, base_scope): # Declare entries for all the C attributes of an # inherited type, with cnames modified appropriately @@ -2337,9 +2475,9 @@ class CClassScope(ClassScope): is_builtin = var_entry and var_entry.is_builtin if not is_builtin: cname = adapt(cname) - entry = self.add_cfunction(base_entry.name, base_entry.type, - base_entry.pos, cname, - base_entry.visibility, base_entry.func_modifiers, inherited=True) + entry = self.add_cfunction( + base_entry.name, base_entry.type, base_entry.pos, cname, + base_entry.visibility, base_entry.func_modifiers, inherited=True) entry.is_inherited = 1 if base_entry.is_final_cmethod: entry.is_final_cmethod = True @@ -2403,7 +2541,7 @@ class CppClassScope(Scope): class_name = self.name.split('::')[-1] if name in (class_name, '__init__') and cname is None: cname = "%s__init__%s" % (Naming.func_prefix, class_name) - name = '<init>' + name = EncodedString('<init>') type.return_type = PyrexTypes.CVoidType() # This is called by the actual constructor, but need to support # arguments that cannot by called by value. @@ -2417,7 +2555,7 @@ class CppClassScope(Scope): type.args = [maybe_ref(arg) for arg in type.args] elif name == '__dealloc__' and cname is None: cname = "%s__dealloc__%s" % (Naming.func_prefix, class_name) - name = '<del>' + name = EncodedString('<del>') type.return_type = PyrexTypes.CVoidType() if name in ('<init>', '<del>') and type.nogil: for base in self.type.base_classes: @@ -2447,19 +2585,18 @@ class CppClassScope(Scope): # Declare entries for all the C++ attributes of an # inherited type, with cnames modified appropriately # to work with this type. - for base_entry in \ - base_scope.inherited_var_entries + base_scope.var_entries: - #constructor/destructor is not inherited - if base_entry.name in ("<init>", "<del>"): - continue - #print base_entry.name, self.entries - if base_entry.name in self.entries: - base_entry.name # FIXME: is there anything to do in this case? - entry = self.declare(base_entry.name, base_entry.cname, - base_entry.type, None, 'extern') - entry.is_variable = 1 - entry.is_inherited = 1 - self.inherited_var_entries.append(entry) + for base_entry in base_scope.inherited_var_entries + base_scope.var_entries: + #constructor/destructor is not inherited + if base_entry.name in ("<init>", "<del>"): + continue + #print base_entry.name, self.entries + if base_entry.name in self.entries: + base_entry.name # FIXME: is there anything to do in this case? + entry = self.declare(base_entry.name, base_entry.cname, + base_entry.type, None, 'extern') + entry.is_variable = 1 + entry.is_inherited = 1 + self.inherited_var_entries.append(entry) for base_entry in base_scope.cfunc_entries: entry = self.declare_cfunction(base_entry.name, base_entry.type, base_entry.pos, base_entry.cname, @@ -2501,6 +2638,22 @@ class CppClassScope(Scope): return scope +class CppScopedEnumScope(Scope): + # Namespace of a ScopedEnum + + def __init__(self, name, outer_scope): + Scope.__init__(self, name, outer_scope, None) + + def declare_var(self, name, type, pos, + cname=None, visibility='extern'): + # Add an entry for an attribute. + if not cname: + cname = name + entry = self.declare(name, cname, type, pos, visibility) + entry.is_variable = True + return entry + + class PropertyScope(Scope): # Scope holding the __get__, __set__ and __del__ methods for # a property of an extension type. @@ -2509,6 +2662,31 @@ class PropertyScope(Scope): is_property_scope = 1 + def __init__(self, name, class_scope): + # outer scope is None for some internal properties + outer_scope = class_scope.global_scope() if class_scope.outer_scope else None + Scope.__init__(self, name, outer_scope, parent_scope=class_scope) + self.parent_type = class_scope.parent_type + self.directives = class_scope.directives + + def declare_cfunction(self, name, type, pos, *args, **kwargs): + """Declare a C property function. + """ + if type.return_type.is_void: + error(pos, "C property method cannot return 'void'") + + if type.args and type.args[0].type is py_object_type: + # Set 'self' argument type to extension type. + type.args[0].type = self.parent_scope.parent_type + elif len(type.args) != 1: + error(pos, "C property method must have a single (self) argument") + elif not (type.args[0].type.is_pyobject or type.args[0].type is self.parent_scope.parent_type): + error(pos, "C property method must have a single (object) argument") + + entry = Scope.declare_cfunction(self, name, type, pos, *args, **kwargs) + entry.is_cproperty = True + return entry + def declare_pyfunction(self, name, pos, allow_redefine=False): # Add an entry for a method. signature = get_property_accessor_signature(name) @@ -2523,23 +2701,27 @@ class PropertyScope(Scope): return None -class CConstScope(Scope): +class CConstOrVolatileScope(Scope): - def __init__(self, const_base_type_scope): + def __init__(self, base_type_scope, is_const=0, is_volatile=0): Scope.__init__( self, - 'const_' + const_base_type_scope.name, - const_base_type_scope.outer_scope, - const_base_type_scope.parent_scope) - self.const_base_type_scope = const_base_type_scope + 'cv_' + base_type_scope.name, + base_type_scope.outer_scope, + base_type_scope.parent_scope) + self.base_type_scope = base_type_scope + self.is_const = is_const + self.is_volatile = is_volatile def lookup_here(self, name): - entry = self.const_base_type_scope.lookup_here(name) + entry = self.base_type_scope.lookup_here(name) if entry is not None: entry = copy.copy(entry) - entry.type = PyrexTypes.c_const_type(entry.type) + entry.type = PyrexTypes.c_const_or_volatile_type( + entry.type, self.is_const, self.is_volatile) return entry + class TemplateScope(Scope): def __init__(self, name, outer_scope): Scope.__init__(self, name, outer_scope, None) diff --git a/Cython/Compiler/Tests/TestBuffer.py b/Cython/Compiler/Tests/TestBuffer.py index fcb22b72f..2f653d0ff 100644 --- a/Cython/Compiler/Tests/TestBuffer.py +++ b/Cython/Compiler/Tests/TestBuffer.py @@ -21,7 +21,7 @@ class TestBufferParsing(CythonTest): def test_basic(self): t = self.parse(u"cdef object[float, 4, ndim=2, foo=foo] x") bufnode = t.stats[0].base_type - self.assert_(isinstance(bufnode, TemplatedTypeNode)) + self.assertTrue(isinstance(bufnode, TemplatedTypeNode)) self.assertEqual(2, len(bufnode.positional_args)) # print bufnode.dump() # should put more here... @@ -46,7 +46,7 @@ class TestBufferOptions(CythonTest): def nonfatal_error(self, error): # We're passing self as context to transform to trap this self.error = error - self.assert_(self.expect_error) + self.assertTrue(self.expect_error) def parse_opts(self, opts, expect_error=False): assert opts != "" @@ -55,14 +55,14 @@ class TestBufferOptions(CythonTest): root = self.fragment(s, pipeline=[NormalizeTree(self), PostParse(self)]).root if not expect_error: vardef = root.stats[0].body.stats[0] - assert isinstance(vardef, CVarDefNode) # use normal assert as this is to validate the test code + assert isinstance(vardef, CVarDefNode) # use normal assert as this is to validate the test code buftype = vardef.base_type - self.assert_(isinstance(buftype, TemplatedTypeNode)) - self.assert_(isinstance(buftype.base_type_node, CSimpleBaseTypeNode)) + self.assertTrue(isinstance(buftype, TemplatedTypeNode)) + self.assertTrue(isinstance(buftype.base_type_node, CSimpleBaseTypeNode)) self.assertEqual(u"object", buftype.base_type_node.name) return buftype else: - self.assert_(len(root.stats[0].body.stats) == 0) + self.assertTrue(len(root.stats[0].body.stats) == 0) def non_parse(self, expected_err, opts): self.parse_opts(opts, expect_error=True) @@ -71,14 +71,14 @@ class TestBufferOptions(CythonTest): def __test_basic(self): buf = self.parse_opts(u"unsigned short int, 3") - self.assert_(isinstance(buf.dtype_node, CSimpleBaseTypeNode)) - self.assert_(buf.dtype_node.signed == 0 and buf.dtype_node.longness == -1) + self.assertTrue(isinstance(buf.dtype_node, CSimpleBaseTypeNode)) + self.assertTrue(buf.dtype_node.signed == 0 and buf.dtype_node.longness == -1) self.assertEqual(3, buf.ndim) def __test_dict(self): buf = self.parse_opts(u"ndim=3, dtype=unsigned short int") - self.assert_(isinstance(buf.dtype_node, CSimpleBaseTypeNode)) - self.assert_(buf.dtype_node.signed == 0 and buf.dtype_node.longness == -1) + self.assertTrue(isinstance(buf.dtype_node, CSimpleBaseTypeNode)) + self.assertTrue(buf.dtype_node.signed == 0 and buf.dtype_node.longness == -1) self.assertEqual(3, buf.ndim) def __test_ndim(self): @@ -94,12 +94,12 @@ class TestBufferOptions(CythonTest): cdef object[ndim=ndim, dtype=int] y """, pipeline=[NormalizeTree(self), PostParse(self)]).root stats = t.stats[0].body.stats - self.assert_(stats[0].base_type.ndim == 3) - self.assert_(stats[1].base_type.ndim == 3) + self.assertTrue(stats[0].base_type.ndim == 3) + self.assertTrue(stats[1].base_type.ndim == 3) # add exotic and impossible combinations as they come along... + if __name__ == '__main__': import unittest unittest.main() - diff --git a/Cython/Compiler/Tests/TestCmdLine.py b/Cython/Compiler/Tests/TestCmdLine.py index abc7c0a89..5953112dc 100644 --- a/Cython/Compiler/Tests/TestCmdLine.py +++ b/Cython/Compiler/Tests/TestCmdLine.py @@ -1,4 +1,4 @@ - +import os import sys from unittest import TestCase try: @@ -9,19 +9,25 @@ except ImportError: from .. import Options from ..CmdLine import parse_command_line +from .Utils import backup_Options, restore_Options, check_global_options + class CmdLineParserTest(TestCase): def setUp(self): - backup = {} - for name, value in vars(Options).items(): - backup[name] = value - self._options_backup = backup + self._options_backup = backup_Options() def tearDown(self): + restore_Options(self._options_backup) + + def check_default_global_options(self, white_list=[]): + self.assertEqual(check_global_options(self._options_backup, white_list), "") + + def check_default_options(self, options, white_list=[]): + default_options = Options.CompilationOptions(Options.default_options) no_value = object() - for name, orig_value in self._options_backup.items(): - if getattr(Options, name, no_value) != orig_value: - setattr(Options, name, orig_value) + for name in default_options.__dict__.keys(): + if name not in white_list: + self.assertEqual(getattr(options, name, no_value), getattr(default_options, name), msg="error in option " + name) def test_short_options(self): options, sources = parse_command_line([ @@ -97,6 +103,397 @@ class CmdLineParserTest(TestCase): self.assertEqual(Options.annotate_coverage_xml, 'cov.xml') self.assertTrue(options.gdb_debug) self.assertEqual(options.output_dir, '/gdb/outdir') + self.assertEqual(options.compiler_directives['wraparound'], False) + + def test_embed_before_positional(self): + options, sources = parse_command_line([ + '--embed', + 'source.pyx', + ]) + self.assertEqual(sources, ['source.pyx']) + self.assertEqual(Options.embed, 'main') + + def test_two_embeds(self): + options, sources = parse_command_line([ + '--embed', '--embed=huhu', + 'source.pyx', + ]) + self.assertEqual(sources, ['source.pyx']) + self.assertEqual(Options.embed, 'huhu') + + def test_two_embeds2(self): + options, sources = parse_command_line([ + '--embed=huhu', '--embed', + 'source.pyx', + ]) + self.assertEqual(sources, ['source.pyx']) + self.assertEqual(Options.embed, 'main') + + def test_no_annotate(self): + options, sources = parse_command_line([ + '--embed=huhu', 'source.pyx' + ]) + self.assertFalse(Options.annotate) + + def test_annotate_short(self): + options, sources = parse_command_line([ + '-a', + 'source.pyx', + ]) + self.assertEqual(Options.annotate, 'default') + + def test_annotate_long(self): + options, sources = parse_command_line([ + '--annotate', + 'source.pyx', + ]) + self.assertEqual(Options.annotate, 'default') + + def test_annotate_fullc(self): + options, sources = parse_command_line([ + '--annotate-fullc', + 'source.pyx', + ]) + self.assertEqual(Options.annotate, 'fullc') + + def test_short_w(self): + options, sources = parse_command_line([ + '-w', 'my_working_path', + 'source.pyx' + ]) + self.assertEqual(options.working_path, 'my_working_path') + self.check_default_global_options() + self.check_default_options(options, ['working_path']) + + def test_short_o(self): + options, sources = parse_command_line([ + '-o', 'my_output', + 'source.pyx' + ]) + self.assertEqual(options.output_file, 'my_output') + self.check_default_global_options() + self.check_default_options(options, ['output_file']) + + def test_short_z(self): + options, sources = parse_command_line([ + '-z', 'my_preimport', + 'source.pyx' + ]) + self.assertEqual(Options.pre_import, 'my_preimport') + self.check_default_global_options(['pre_import']) + self.check_default_options(options) + + def test_convert_range(self): + options, sources = parse_command_line([ + '--convert-range', + 'source.pyx' + ]) + self.assertEqual(Options.convert_range, True) + self.check_default_global_options(['convert_range']) + self.check_default_options(options) + + def test_line_directives(self): + options, sources = parse_command_line([ + '--line-directives', + 'source.pyx' + ]) + self.assertEqual(options.emit_linenums, True) + self.check_default_global_options() + self.check_default_options(options, ['emit_linenums']) + + def test_no_c_in_traceback(self): + options, sources = parse_command_line([ + '--no-c-in-traceback', + 'source.pyx' + ]) + self.assertEqual(options.c_line_in_traceback, False) + self.check_default_global_options() + self.check_default_options(options, ['c_line_in_traceback']) + + def test_gdb(self): + options, sources = parse_command_line([ + '--gdb', + 'source.pyx' + ]) + self.assertEqual(options.gdb_debug, True) + self.assertEqual(options.output_dir, os.curdir) + self.check_default_global_options() + self.check_default_options(options, ['gdb_debug', 'output_dir']) + + def test_3str(self): + options, sources = parse_command_line([ + '--3str', + 'source.pyx' + ]) + self.assertEqual(options.language_level, '3str') + self.check_default_global_options() + self.check_default_options(options, ['language_level']) + + def test_capi_reexport_cincludes(self): + options, sources = parse_command_line([ + '--capi-reexport-cincludes', + 'source.pyx' + ]) + self.assertEqual(options.capi_reexport_cincludes, True) + self.check_default_global_options() + self.check_default_options(options, ['capi_reexport_cincludes']) + + def test_fast_fail(self): + options, sources = parse_command_line([ + '--fast-fail', + 'source.pyx' + ]) + self.assertEqual(Options.fast_fail, True) + self.check_default_global_options(['fast_fail']) + self.check_default_options(options) + + def test_cimport_from_pyx(self): + options, sources = parse_command_line([ + '--cimport-from-pyx', + 'source.pyx' + ]) + self.assertEqual(Options.cimport_from_pyx, True) + self.check_default_global_options(['cimport_from_pyx']) + self.check_default_options(options) + + def test_Werror(self): + options, sources = parse_command_line([ + '-Werror', + 'source.pyx' + ]) + self.assertEqual(Options.warning_errors, True) + self.check_default_global_options(['warning_errors']) + self.check_default_options(options) + + def test_warning_errors(self): + options, sources = parse_command_line([ + '--warning-errors', + 'source.pyx' + ]) + self.assertEqual(Options.warning_errors, True) + self.check_default_global_options(['warning_errors']) + self.check_default_options(options) + + def test_Wextra(self): + options, sources = parse_command_line([ + '-Wextra', + 'source.pyx' + ]) + self.assertEqual(options.compiler_directives, Options.extra_warnings) + self.check_default_global_options() + self.check_default_options(options, ['compiler_directives']) + + def test_warning_extra(self): + options, sources = parse_command_line([ + '--warning-extra', + 'source.pyx' + ]) + self.assertEqual(options.compiler_directives, Options.extra_warnings) + self.check_default_global_options() + self.check_default_options(options, ['compiler_directives']) + + def test_old_style_globals(self): + options, sources = parse_command_line([ + '--old-style-globals', + 'source.pyx' + ]) + self.assertEqual(Options.old_style_globals, True) + self.check_default_global_options(['old_style_globals']) + self.check_default_options(options) + + def test_directive_multiple(self): + options, source = parse_command_line([ + '-X', 'cdivision=True', + '-X', 'c_string_type=bytes', + 'source.pyx' + ]) + self.assertEqual(options.compiler_directives['cdivision'], True) + self.assertEqual(options.compiler_directives['c_string_type'], 'bytes') + self.check_default_global_options() + self.check_default_options(options, ['compiler_directives']) + + def test_directive_multiple_v2(self): + options, source = parse_command_line([ + '-X', 'cdivision=True,c_string_type=bytes', + 'source.pyx' + ]) + self.assertEqual(options.compiler_directives['cdivision'], True) + self.assertEqual(options.compiler_directives['c_string_type'], 'bytes') + self.check_default_global_options() + self.check_default_options(options, ['compiler_directives']) + + def test_directive_value_yes(self): + options, source = parse_command_line([ + '-X', 'cdivision=YeS', + 'source.pyx' + ]) + self.assertEqual(options.compiler_directives['cdivision'], True) + self.check_default_global_options() + self.check_default_options(options, ['compiler_directives']) + + def test_directive_value_no(self): + options, source = parse_command_line([ + '-X', 'cdivision=no', + 'source.pyx' + ]) + self.assertEqual(options.compiler_directives['cdivision'], False) + self.check_default_global_options() + self.check_default_options(options, ['compiler_directives']) + + def test_directive_value_invalid(self): + self.assertRaises(ValueError, parse_command_line, [ + '-X', 'cdivision=sadfasd', + 'source.pyx' + ]) + + def test_directive_key_invalid(self): + self.assertRaises(ValueError, parse_command_line, [ + '-X', 'abracadabra', + 'source.pyx' + ]) + + def test_directive_no_value(self): + self.assertRaises(ValueError, parse_command_line, [ + '-X', 'cdivision', + 'source.pyx' + ]) + + def test_compile_time_env_short(self): + options, source = parse_command_line([ + '-E', 'MYSIZE=10', + 'source.pyx' + ]) + self.assertEqual(options.compile_time_env['MYSIZE'], 10) + self.check_default_global_options() + self.check_default_options(options, ['compile_time_env']) + + def test_compile_time_env_long(self): + options, source = parse_command_line([ + '--compile-time-env', 'MYSIZE=10', + 'source.pyx' + ]) + self.assertEqual(options.compile_time_env['MYSIZE'], 10) + self.check_default_global_options() + self.check_default_options(options, ['compile_time_env']) + + def test_compile_time_env_multiple(self): + options, source = parse_command_line([ + '-E', 'MYSIZE=10', '-E', 'ARRSIZE=11', + 'source.pyx' + ]) + self.assertEqual(options.compile_time_env['MYSIZE'], 10) + self.assertEqual(options.compile_time_env['ARRSIZE'], 11) + self.check_default_global_options() + self.check_default_options(options, ['compile_time_env']) + + def test_compile_time_env_multiple_v2(self): + options, source = parse_command_line([ + '-E', 'MYSIZE=10,ARRSIZE=11', + 'source.pyx' + ]) + self.assertEqual(options.compile_time_env['MYSIZE'], 10) + self.assertEqual(options.compile_time_env['ARRSIZE'], 11) + self.check_default_global_options() + self.check_default_options(options, ['compile_time_env']) + + def test_option_first(self): + options, sources = parse_command_line(['-V', 'file.pyx']) + self.assertEqual(sources, ['file.pyx']) + + def test_file_inbetween(self): + options, sources = parse_command_line(['-V', 'file.pyx', '-a']) + self.assertEqual(sources, ['file.pyx']) + + def test_option_trailing(self): + options, sources = parse_command_line(['file.pyx', '-V']) + self.assertEqual(sources, ['file.pyx']) + + def test_multiple_files(self): + options, sources = parse_command_line([ + 'file1.pyx', '-V', + 'file2.pyx', '-a', + 'file3.pyx' + ]) + self.assertEqual(sources, ['file1.pyx', 'file2.pyx', 'file3.pyx']) + + def test_debug_flags(self): + options, sources = parse_command_line([ + '--debug-disposal-code', '--debug-coercion', + 'file3.pyx' + ]) + from Cython.Compiler import DebugFlags + for name in ['debug_disposal_code', 'debug_temp_alloc', 'debug_coercion']: + self.assertEqual(getattr(DebugFlags, name), name in ['debug_disposal_code', 'debug_coercion']) + setattr(DebugFlags, name, 0) # restore original value + + def test_gdb_overwrites_gdb_outdir(self): + options, sources = parse_command_line([ + '--gdb-outdir=my_dir', '--gdb', + 'file3.pyx' + ]) + self.assertEqual(options.gdb_debug, True) + self.assertEqual(options.output_dir, os.curdir) + self.check_default_global_options() + self.check_default_options(options, ['gdb_debug', 'output_dir']) + + def test_gdb_first(self): + options, sources = parse_command_line([ + '--gdb', '--gdb-outdir=my_dir', + 'file3.pyx' + ]) + self.assertEqual(options.gdb_debug, True) + self.assertEqual(options.output_dir, 'my_dir') + self.check_default_global_options() + self.check_default_options(options, ['gdb_debug', 'output_dir']) + + def test_coverage_overwrites_annotation(self): + options, sources = parse_command_line([ + '--annotate-fullc', '--annotate-coverage=my.xml', + 'file3.pyx' + ]) + self.assertEqual(Options.annotate, True) + self.assertEqual(Options.annotate_coverage_xml, 'my.xml') + self.check_default_global_options(['annotate', 'annotate_coverage_xml']) + self.check_default_options(options) + + def test_coverage_first(self): + options, sources = parse_command_line([ + '--annotate-coverage=my.xml', '--annotate-fullc', + 'file3.pyx' + ]) + self.assertEqual(Options.annotate, 'fullc') + self.assertEqual(Options.annotate_coverage_xml, 'my.xml') + self.check_default_global_options(['annotate', 'annotate_coverage_xml']) + self.check_default_options(options) + + def test_annotate_first_fullc_second(self): + options, sources = parse_command_line([ + '--annotate', '--annotate-fullc', + 'file3.pyx' + ]) + self.assertEqual(Options.annotate, 'fullc') + self.check_default_global_options(['annotate']) + self.check_default_options(options) + + def test_annotate_fullc_first(self): + options, sources = parse_command_line([ + '--annotate-fullc', '--annotate', + 'file3.pyx' + ]) + self.assertEqual(Options.annotate, 'default') + self.check_default_global_options(['annotate']) + self.check_default_options(options) + + def test_warning_extra_dont_overwrite(self): + options, sources = parse_command_line([ + '-X', 'cdivision=True', + '--warning-extra', + '-X', 'c_string_type=bytes', + 'source.pyx' + ]) + self.assertTrue(len(options.compiler_directives), len(Options.extra_warnings) + 1) + self.check_default_global_options() + self.check_default_options(options, ['compiler_directives']) def test_errors(self): def error(*args): @@ -116,3 +513,4 @@ class CmdLineParserTest(TestCase): error('--verbose=1') error('--verbose=1') error('--cleanup') + error('--debug-disposal-code-wrong-name', 'file3.pyx') diff --git a/Cython/Compiler/Tests/TestGrammar.py b/Cython/Compiler/Tests/TestGrammar.py index 3dddc960b..f80ec22d3 100644 --- a/Cython/Compiler/Tests/TestGrammar.py +++ b/Cython/Compiler/Tests/TestGrammar.py @@ -27,7 +27,15 @@ VALID_UNDERSCORE_LITERALS = [ '1e1_0', '.1_4', '.1_4e1', + '0b_0', + '0x_f', + '0o_5', + '1_00_00j', + '1_00_00.5j', + '1_00_00e5_1j', '.1_4j', + '(1_2.5+3_3j)', + '(.5_6j)', ] # Copied from CPython's test_grammar.py @@ -36,22 +44,29 @@ INVALID_UNDERSCORE_LITERALS = [ '0_', '42_', '1.4j_', + '0x_', '0b1_', '0xf_', '0o5_', + '0 if 1_Else 1', # Underscores in the base selector: '0_b0', '0_xf', '0_o5', - # Underscore right after the base selector: - '0b_0', - '0x_f', - '0o_5', # Old-style octal, still disallowed: - #'0_7', - #'09_99', - # Special case with exponent: - '0 if 1_Else 1', + # FIXME: still need to support PY_VERSION_HEX < 3 + '0_7', + '09_99', + # Multiple consecutive underscores: + '4_______2', + '0.1__4', + '0.1__4j', + '0b1001__0100', + '0xffff__ffff', + '0x___', + '0o5__77', + '1e1__0', + '1e1__0j', # Underscore right before a dot: '1_.4', '1_.4j', @@ -59,24 +74,24 @@ INVALID_UNDERSCORE_LITERALS = [ '1._4', '1._4j', '._5', + '._5j', # Underscore right after a sign: '1.0e+_1', - # Multiple consecutive underscores: - '4_______2', - '0.1__4', - '0b1001__0100', - '0xffff__ffff', - '0o5__77', - '1e1__0', + '1.0e+_1j', # Underscore right before j: '1.4_j', '1.4e5_j', # Underscore right before e: '1_e1', '1.4_e1', + '1.4_e1j', # Underscore right after e: '1e_1', '1.4e_1', + '1.4e_1j', + # Complex cases with parens: + '(1+1.5_j_)', + '(1+1.5_j)', # Whitespace in literals '1_ 2', '1 _2', @@ -117,11 +132,15 @@ class TestGrammar(CythonTest): # Add/MulNode() -> literal is first or second operand literal_node = literal_node.operand2 if i % 2 else literal_node.operand1 if 'j' in literal or 'J' in literal: - assert isinstance(literal_node, ExprNodes.ImagNode) + if '+' in literal: + # FIXME: tighten this test + assert isinstance(literal_node, ExprNodes.AddNode), (literal, literal_node) + else: + assert isinstance(literal_node, ExprNodes.ImagNode), (literal, literal_node) elif '.' in literal or 'e' in literal or 'E' in literal and not ('0x' in literal or '0X' in literal): - assert isinstance(literal_node, ExprNodes.FloatNode) + assert isinstance(literal_node, ExprNodes.FloatNode), (literal, literal_node) else: - assert isinstance(literal_node, ExprNodes.IntNode) + assert isinstance(literal_node, ExprNodes.IntNode), (literal, literal_node) if __name__ == "__main__": diff --git a/Cython/Compiler/Tests/TestMemView.py b/Cython/Compiler/Tests/TestMemView.py index 837bb8088..1d04a17fc 100644 --- a/Cython/Compiler/Tests/TestMemView.py +++ b/Cython/Compiler/Tests/TestMemView.py @@ -48,16 +48,16 @@ class TestMemviewParsing(CythonTest): def test_basic(self): t = self.parse(u"cdef int[:] x") memv_node = t.stats[0].base_type - self.assert_(isinstance(memv_node, MemoryViewSliceTypeNode)) + self.assertTrue(isinstance(memv_node, MemoryViewSliceTypeNode)) # we also test other similar declarations (buffers, anonymous C arrays) # since the parsing has to distinguish between them. - def disable_test_no_buf_arg(self): # TODO + def disable_test_no_buf_arg(self): # TODO self.not_parseable(u"Expected ']'", u"cdef extern foo(object[int, ndim=2])") - def disable_test_parse_sizeof(self): # TODO + def disable_test_parse_sizeof(self): # TODO self.parse(u"sizeof(int[NN])") self.parse(u"sizeof(int[])") self.parse(u"sizeof(int[][NN])") diff --git a/Cython/Compiler/Tests/TestParseTreeTransforms.py b/Cython/Compiler/Tests/TestParseTreeTransforms.py index 91e0e2f1d..e6889f8f2 100644 --- a/Cython/Compiler/Tests/TestParseTreeTransforms.py +++ b/Cython/Compiler/Tests/TestParseTreeTransforms.py @@ -3,7 +3,7 @@ import os from Cython.TestUtils import TransformTest from Cython.Compiler.ParseTreeTransforms import * from Cython.Compiler.Nodes import * -from Cython.Compiler import Main, Symtab +from Cython.Compiler import Main, Symtab, Options class TestNormalizeTree(TransformTest): @@ -87,9 +87,9 @@ class TestNormalizeTree(TransformTest): def test_pass_eliminated(self): t = self.run_pipeline([NormalizeTree(None)], u"pass") - self.assert_(len(t.stats) == 0) + self.assertTrue(len(t.stats) == 0) -class TestWithTransform(object): # (TransformTest): # Disabled! +class TestWithTransform(object): # (TransformTest): # Disabled! def test_simplified(self): t = self.run_pipeline([WithTransform(None)], u""" @@ -177,8 +177,8 @@ class TestInterpretCompilerDirectives(TransformTest): def setUp(self): super(TestInterpretCompilerDirectives, self).setUp() - compilation_options = Main.CompilationOptions(Main.default_options) - ctx = compilation_options.create_context() + compilation_options = Options.CompilationOptions(Options.default_options) + ctx = Main.Context.from_options(compilation_options) transform = InterpretCompilerDirectives(ctx, ctx.compiler_directives) transform.module_scope = Symtab.ModuleScope('__main__', None, ctx) diff --git a/Cython/Compiler/Tests/TestSignatureMatching.py b/Cython/Compiler/Tests/TestSignatureMatching.py index 166bb225b..57647c873 100644 --- a/Cython/Compiler/Tests/TestSignatureMatching.py +++ b/Cython/Compiler/Tests/TestSignatureMatching.py @@ -47,7 +47,7 @@ class SignatureMatcherTest(unittest.TestCase): self.assertMatches(function_types[1], [pt.c_long_type, pt.c_int_type], functions) def test_cpp_reference_cpp_class(self): - classes = [ cppclasstype("Test%d"%i, []) for i in range(2) ] + classes = [ cppclasstype("Test%d" % i, []) for i in range(2) ] function_types = [ cfunctype(pt.CReferenceType(classes[0])), cfunctype(pt.CReferenceType(classes[1])), @@ -58,7 +58,7 @@ class SignatureMatcherTest(unittest.TestCase): self.assertMatches(function_types[1], [classes[1]], functions) def test_cpp_reference_cpp_class_and_int(self): - classes = [ cppclasstype("Test%d"%i, []) for i in range(2) ] + classes = [ cppclasstype("Test%d" % i, []) for i in range(2) ] function_types = [ cfunctype(pt.CReferenceType(classes[0]), pt.c_int_type), cfunctype(pt.CReferenceType(classes[0]), pt.c_long_type), diff --git a/Cython/Compiler/Tests/TestTreeFragment.py b/Cython/Compiler/Tests/TestTreeFragment.py index 3f15b7457..9ee8da547 100644 --- a/Cython/Compiler/Tests/TestTreeFragment.py +++ b/Cython/Compiler/Tests/TestTreeFragment.py @@ -23,7 +23,7 @@ class TestTreeFragments(CythonTest): T = self.fragment(u"y + y").substitute({"y": NameNode(pos=None, name="x")}) self.assertEqual("x", T.stats[0].expr.operand1.name) self.assertEqual("x", T.stats[0].expr.operand2.name) - self.assert_(T.stats[0].expr.operand1 is not T.stats[0].expr.operand2) + self.assertTrue(T.stats[0].expr.operand1 is not T.stats[0].expr.operand2) def test_substitution(self): F = self.fragment(u"x = 4") @@ -35,7 +35,7 @@ class TestTreeFragments(CythonTest): F = self.fragment(u"PASS") pass_stat = PassStatNode(pos=None) T = F.substitute({"PASS" : pass_stat}) - self.assert_(isinstance(T.stats[0], PassStatNode), T) + self.assertTrue(isinstance(T.stats[0], PassStatNode), T) def test_pos_is_transferred(self): F = self.fragment(u""" @@ -55,9 +55,9 @@ class TestTreeFragments(CythonTest): """) T = F.substitute(temps=[u"TMP"]) s = T.body.stats - self.assert_(isinstance(s[0].expr, TempRefNode)) - self.assert_(isinstance(s[1].rhs, TempRefNode)) - self.assert_(s[0].expr.handle is s[1].rhs.handle) + self.assertTrue(isinstance(s[0].expr, TempRefNode)) + self.assertTrue(isinstance(s[1].rhs, TempRefNode)) + self.assertTrue(s[0].expr.handle is s[1].rhs.handle) if __name__ == "__main__": import unittest diff --git a/Cython/Compiler/Tests/TestTypes.py b/Cython/Compiler/Tests/TestTypes.py index f2f6f3773..4693dd806 100644 --- a/Cython/Compiler/Tests/TestTypes.py +++ b/Cython/Compiler/Tests/TestTypes.py @@ -17,3 +17,59 @@ class TestMethodDispatcherTransform(unittest.TestCase): cenum = PT.CEnumType("E", "cenum", typedef_flag=False) assert_widest(PT.c_int_type, cenum, PT.c_int_type) + + +class TestTypeIdentifiers(unittest.TestCase): + + TEST_DATA = [ + ("char*", "char__ptr"), + ("char *", "char__ptr"), + ("char **", "char__ptr__ptr"), + ("_typedef", "_typedef"), + ("__typedef", "__dundertypedef"), + ("___typedef", "__dunder_typedef"), + ("____typedef", "__dunder__dundertypedef"), + ("_____typedef", "__dunder__dunder_typedef"), + ("const __typedef", "__const___dundertypedef"), + ("int[42]", "int__lArr42__rArr"), + ("int[:]", "int__lArr__D__rArr"), + ("int[:,:]", "int__lArr__D__comma___D__rArr"), + ("int[:,:,:]", "int__lArr__D__comma___D__comma___D__rArr"), + ("int[:,:,...]", "int__lArr__D__comma___D__comma___EL__rArr"), + ("std::vector", "std__in_vector"), + ("std::vector&&", "std__in_vector__fwref"), + ("const std::vector", "__const_std__in_vector"), + ("const std::vector&", "__const_std__in_vector__ref"), + ("const_std", "const_std"), + ] + + def test_escape_special_type_characters(self): + test_func = PT._escape_special_type_characters # keep test usage visible for IDEs + function_name = "_escape_special_type_characters" + self._test_escape(function_name) + + def test_type_identifier_for_declaration(self): + test_func = PT.type_identifier_from_declaration # keep test usage visible for IDEs + function_name = test_func.__name__ + self._test_escape(function_name) + + # differences due to whitespace removal + test_data = [ + ("const &std::vector", "const__refstd__in_vector"), + ("const &std::vector<int>", "const__refstd__in_vector__lAngint__rAng"), + ("const &&std::vector", "const__fwrefstd__in_vector"), + ("const &&&std::vector", "const__fwref__refstd__in_vector"), + ("const &&std::vector", "const__fwrefstd__in_vector"), + ("void (*func)(int x, float y)", + "975d51__void__lParen__ptrfunc__rParen__lParenint__space_x__comma_float__space_y__rParen__etc"), + ("float ** (*func)(int x, int[:] y)", + "31883a__float__ptr__ptr__lParen__ptrfunc__rParen__lParenint__space_x__comma_int__lArr__D__rArry__rParen__etc"), + ] + self._test_escape(function_name, test_data) + + def _test_escape(self, func_name, test_data=TEST_DATA): + escape = getattr(PT, func_name) + for declaration, expected in test_data: + escaped_value = escape(declaration) + self.assertEqual(escaped_value, expected, "%s('%s') == '%s' != '%s'" % ( + func_name, declaration, escaped_value, expected)) diff --git a/Cython/Compiler/Tests/TestUtilityLoad.py b/Cython/Compiler/Tests/TestUtilityLoad.py index 3d1906ca0..44ae461ab 100644 --- a/Cython/Compiler/Tests/TestUtilityLoad.py +++ b/Cython/Compiler/Tests/TestUtilityLoad.py @@ -22,14 +22,11 @@ class TestUtilityLoader(unittest.TestCase): cls = Code.UtilityCode def test_load_as_string(self): - got = strip_2tup(self.cls.load_as_string(self.name)) - self.assertEqual(got, self.expected) - got = strip_2tup(self.cls.load_as_string(self.name, self.filename)) self.assertEqual(got, self.expected) def test_load(self): - utility = self.cls.load(self.name) + utility = self.cls.load(self.name, from_file=self.filename) got = strip_2tup((utility.proto, utility.impl)) self.assertEqual(got, self.expected) @@ -37,10 +34,6 @@ class TestUtilityLoader(unittest.TestCase): got = strip_2tup((required.proto, required.impl)) self.assertEqual(got, self.required) - utility = self.cls.load(self.name, from_file=self.filename) - got = strip_2tup((utility.proto, utility.impl)) - self.assertEqual(got, self.expected) - utility = self.cls.load_cached(self.name, from_file=self.filename) got = strip_2tup((utility.proto, utility.impl)) self.assertEqual(got, self.expected) @@ -59,11 +52,11 @@ class TestTempitaUtilityLoader(TestUtilityLoader): cls = Code.TempitaUtilityCode def test_load_as_string(self): - got = strip_2tup(self.cls.load_as_string(self.name, context=self.context)) + got = strip_2tup(self.cls.load_as_string(self.name, self.filename, context=self.context)) self.assertEqual(got, self.expected_tempita) def test_load(self): - utility = self.cls.load(self.name, context=self.context) + utility = self.cls.load(self.name, self.filename, context=self.context) got = strip_2tup((utility.proto, utility.impl)) self.assertEqual(got, self.expected_tempita) diff --git a/Cython/Compiler/Tests/Utils.py b/Cython/Compiler/Tests/Utils.py new file mode 100644 index 000000000..a158ecc50 --- /dev/null +++ b/Cython/Compiler/Tests/Utils.py @@ -0,0 +1,36 @@ +import copy + +from .. import Options + + +def backup_Options(): + backup = {} + for name, value in vars(Options).items(): + # we need a deep copy of _directive_defaults, because they can be changed + if name == '_directive_defaults': + value = copy.deepcopy(value) + backup[name] = value + return backup + + +def restore_Options(backup): + no_value = object() + for name, orig_value in backup.items(): + if getattr(Options, name, no_value) != orig_value: + setattr(Options, name, orig_value) + # strip Options from new keys that might have been added: + for name in vars(Options).keys(): + if name not in backup: + delattr(Options, name) + + +def check_global_options(expected_options, white_list=[]): + """ + returns error message of "" if check Ok + """ + no_value = object() + for name, orig_value in expected_options.items(): + if name not in white_list: + if getattr(Options, name, no_value) != orig_value: + return "error in option " + name + return "" diff --git a/Cython/Compiler/TreeFragment.py b/Cython/Compiler/TreeFragment.py index b85da8191..cef4469b5 100644 --- a/Cython/Compiler/TreeFragment.py +++ b/Cython/Compiler/TreeFragment.py @@ -29,8 +29,7 @@ class StringParseContext(Main.Context): include_directories = [] if compiler_directives is None: compiler_directives = {} - # TODO: see if "language_level=3" also works for our internal code here. - Main.Context.__init__(self, include_directories, compiler_directives, cpp=cpp, language_level=2) + Main.Context.__init__(self, include_directories, compiler_directives, cpp=cpp, language_level='3str') self.module_name = name def find_module(self, module_name, relative_to=None, pos=None, need_pxd=1, absolute_fallback=True): @@ -179,7 +178,7 @@ class TemplateTransform(VisitorTransform): if pos is None: pos = node.pos return ApplyPositionAndCopy(pos)(sub) else: - return self.visit_Node(node) # make copy as usual + return self.visit_Node(node) # make copy as usual def visit_NameNode(self, node): temphandle = self.tempmap.get(node.name) @@ -235,7 +234,7 @@ class TreeFragment(object): fmt_pxds[key] = fmt(value) mod = t = parse_from_strings(name, fmt_code, fmt_pxds, level=level, initial_pos=initial_pos) if level is None: - t = t.body # Make sure a StatListNode is at the top + t = t.body # Make sure a StatListNode is at the top if not isinstance(t, StatListNode): t = StatListNode(pos=mod.pos, stats=[t]) for transform in pipeline: diff --git a/Cython/Compiler/TypeInference.py b/Cython/Compiler/TypeInference.py index c7ffee7d2..1e46050d1 100644 --- a/Cython/Compiler/TypeInference.py +++ b/Cython/Compiler/TypeInference.py @@ -140,7 +140,6 @@ class MarkParallelAssignments(EnvTransform): '+', sequence.args[0], sequence.args[2])) - if not is_special: # A for-loop basically translates to subsequent calls to # __getitem__(), so using an IndexNode here allows us to @@ -178,7 +177,7 @@ class MarkParallelAssignments(EnvTransform): return node def visit_FromCImportStatNode(self, node): - pass # Can't be assigned to... + return node # Can't be assigned to... def visit_FromImportStatNode(self, node): for name, target in node.items: @@ -308,10 +307,10 @@ class MarkOverflowingArithmetic(CythonTransform): def visit_SimpleCallNode(self, node): if node.function.is_name and node.function.name == 'abs': - # Overflows for minimum value of fixed size ints. - return self.visit_dangerous_node(node) + # Overflows for minimum value of fixed size ints. + return self.visit_dangerous_node(node) else: - return self.visit_neutral_node(node) + return self.visit_neutral_node(node) visit_UnopNode = visit_neutral_node @@ -360,9 +359,11 @@ class SimpleAssignmentTypeInferer(object): applies to nested scopes in top-down order. """ def set_entry_type(self, entry, entry_type): - entry.type = entry_type for e in entry.all_entries(): e.type = entry_type + if e.type.is_memoryviewslice: + # memoryview slices crash if they don't get initialized + e.init = e.type.default_value def infer_types(self, scope): enabled = scope.directives['infer_types'] @@ -370,7 +371,7 @@ class SimpleAssignmentTypeInferer(object): if enabled == True: spanning_type = aggressive_spanning_type - elif enabled is None: # safe mode + elif enabled is None: # safe mode spanning_type = safe_spanning_type else: for entry in scope.entries.values(): @@ -533,8 +534,8 @@ def find_spanning_type(type1, type2): def simply_type(result_type, pos): if result_type.is_reference: result_type = result_type.ref_base_type - if result_type.is_const: - result_type = result_type.const_base_type + if result_type.is_cv_qualified: + result_type = result_type.cv_base_type if result_type.is_cpp_class: result_type.check_nullary_constructor(pos) if result_type.is_array: @@ -577,6 +578,8 @@ def safe_spanning_type(types, might_overflow, pos, scope): # used, won't arise in pure Python, and there shouldn't be side # effects, so I'm declaring this safe. return result_type + elif result_type.is_memoryviewslice: + return result_type # TODO: double complex should be OK as well, but we need # to make sure everything is supported. elif (result_type.is_int or result_type.is_enum) and not might_overflow: diff --git a/Cython/Compiler/TypeSlots.py b/Cython/Compiler/TypeSlots.py index 137ea4eba..1db912991 100644 --- a/Cython/Compiler/TypeSlots.py +++ b/Cython/Compiler/TypeSlots.py @@ -9,6 +9,8 @@ from . import Naming from . import PyrexTypes from .Errors import error +import copy + invisible = ['__cinit__', '__dealloc__', '__richcmp__', '__nonzero__', '__bool__'] @@ -23,6 +25,7 @@ class Signature(object): # fixed_arg_format string # ret_format string # error_value string + # use_fastcall boolean # # The formats are strings made up of the following # characters: @@ -86,20 +89,24 @@ class Signature(object): 'z': "-1", } - def __init__(self, arg_format, ret_format): - self.has_dummy_arg = 0 - self.has_generic_args = 0 + # Use METH_FASTCALL instead of METH_VARARGS + use_fastcall = False + + def __init__(self, arg_format, ret_format, nogil=False): + self.has_dummy_arg = False + self.has_generic_args = False if arg_format[:1] == '-': - self.has_dummy_arg = 1 + self.has_dummy_arg = True arg_format = arg_format[1:] if arg_format[-1:] == '*': - self.has_generic_args = 1 + self.has_generic_args = True arg_format = arg_format[:-1] self.fixed_arg_format = arg_format self.ret_format = ret_format self.error_value = self.error_value_map.get(ret_format, None) self.exception_check = ret_format != 'r' and self.error_value is not None self.is_staticmethod = False + self.nogil = nogil def __repr__(self): return '<Signature[%s(%s%s)]>' % ( @@ -149,7 +156,8 @@ class Signature(object): exc_value = self.exception_value() return PyrexTypes.CFuncType( ret_type, args, exception_value=exc_value, - exception_check=self.exception_check) + exception_check=self.exception_check, + nogil=self.nogil) def method_flags(self): if self.ret_format == "O": @@ -157,17 +165,50 @@ class Signature(object): if self.has_dummy_arg: full_args = "O" + full_args if full_args in ["O", "T"]: - if self.has_generic_args: - return [method_varargs, method_keywords] - else: + if not self.has_generic_args: return [method_noargs] + elif self.use_fastcall: + return [method_fastcall, method_keywords] + else: + return [method_varargs, method_keywords] elif full_args in ["OO", "TO"] and not self.has_generic_args: return [method_onearg] if self.is_staticmethod: - return [method_varargs, method_keywords] + if self.use_fastcall: + return [method_fastcall, method_keywords] + else: + return [method_varargs, method_keywords] + return None + + def method_function_type(self): + # Return the C function type + mflags = self.method_flags() + kw = "WithKeywords" if (method_keywords in mflags) else "" + for m in mflags: + if m == method_noargs or m == method_onearg: + return "PyCFunction" + if m == method_varargs: + return "PyCFunction" + kw + if m == method_fastcall: + return "__Pyx_PyCFunction_FastCall" + kw return None + def with_fastcall(self): + # Return a copy of this Signature with use_fastcall=True + sig = copy.copy(self) + sig.use_fastcall = True + return sig + + @property + def fastvar(self): + # Used to select variants of functions, one dealing with METH_VARARGS + # and one dealing with __Pyx_METH_FASTCALL + if self.use_fastcall: + return "FASTCALL" + else: + return "VARARGS" + class SlotDescriptor(object): # Abstract base class for type slot descriptors. @@ -180,13 +221,20 @@ class SlotDescriptor(object): # ifdef Full #ifdef string that slot is wrapped in. Using this causes py3, py2 and flags to be ignored.) def __init__(self, slot_name, dynamic=False, inherited=False, - py3=True, py2=True, ifdef=None): + py3=True, py2=True, ifdef=None, is_binop=False): self.slot_name = slot_name self.is_initialised_dynamically = dynamic self.is_inherited = inherited self.ifdef = ifdef self.py3 = py3 self.py2 = py2 + self.is_binop = is_binop + + def slot_code(self, scope): + raise NotImplemented() + + def spec_value(self, scope): + return self.slot_code(scope) def preprocessor_guard_code(self): ifdef = self.ifdef @@ -201,6 +249,19 @@ class SlotDescriptor(object): guard = ("#if PY_MAJOR_VERSION >= 3") return guard + def generate_spec(self, scope, code): + if self.is_initialised_dynamically: + return + value = self.spec_value(scope) + if value == "0": + return + preprocessor_guard = self.preprocessor_guard_code() + if preprocessor_guard: + code.putln(preprocessor_guard) + code.putln("{Py_%s, (void *)%s}," % (self.slot_name, value)) + if preprocessor_guard: + code.putln("#endif") + def generate(self, scope, code): preprocessor_guard = self.preprocessor_guard_code() if preprocessor_guard: @@ -248,14 +309,17 @@ class SlotDescriptor(object): def generate_dynamic_init_code(self, scope, code): if self.is_initialised_dynamically: - value = self.slot_code(scope) - if value != "0": - code.putln("%s.%s = %s;" % ( - scope.parent_type.typeobj_cname, - self.slot_name, - value - ) - ) + self.generate_set_slot_code( + self.slot_code(scope), scope, code) + + def generate_set_slot_code(self, value, scope, code): + if value == "0": + return + code.putln("%s.%s = %s;" % ( + scope.parent_type.typeobj_cname, + self.slot_name, + value, + )) class FixedSlot(SlotDescriptor): @@ -360,30 +424,59 @@ class GCClearReferencesSlot(GCDependentSlot): class ConstructorSlot(InternalMethodSlot): # Descriptor for tp_new and tp_dealloc. - def __init__(self, slot_name, method, **kargs): + def __init__(self, slot_name, method=None, **kargs): InternalMethodSlot.__init__(self, slot_name, **kargs) self.method = method - def slot_code(self, scope): - entry = scope.lookup_here(self.method) - if (self.slot_name != 'tp_new' - and scope.parent_type.base_type + def _needs_own(self, scope): + if (scope.parent_type.base_type and not scope.has_pyobject_attrs and not scope.has_memoryview_attrs - and not scope.has_cpp_class_attrs - and not (entry and entry.is_special)): + and not scope.has_cpp_constructable_attrs + and not (self.slot_name == 'tp_new' and scope.parent_type.vtabslot_cname)): + entry = scope.lookup_here(self.method) if self.method else None + if not (entry and entry.is_special): + return False + # Unless we can safely delegate to the parent, all types need a tp_new(). + return True + + def _parent_slot_function(self, scope): + parent_type_scope = scope.parent_type.base_type.scope + if scope.parent_scope is parent_type_scope.parent_scope: + entry = scope.parent_scope.lookup_here(scope.parent_type.base_type.name) + if entry.visibility != 'extern': + return self.slot_code(parent_type_scope) + return None + + def slot_code(self, scope): + if not self._needs_own(scope): # if the type does not have object attributes, it can # delegate GC methods to its parent - iff the parent # functions are defined in the same module - parent_type_scope = scope.parent_type.base_type.scope - if scope.parent_scope is parent_type_scope.parent_scope: - entry = scope.parent_scope.lookup_here(scope.parent_type.base_type.name) - if entry.visibility != 'extern': - return self.slot_code(parent_type_scope) - if entry and not entry.is_special: - return "0" + slot_code = self._parent_slot_function(scope) + return slot_code or '0' return InternalMethodSlot.slot_code(self, scope) + def spec_value(self, scope): + if self.slot_name == "tp_dealloc" and not scope.lookup_here("__dealloc__"): + return "0" + return self.slot_code(scope) + + def generate_dynamic_init_code(self, scope, code): + if self.slot_code(scope) != '0': + return + # If we don't have our own slot function and don't know the + # parent function statically, copy it dynamically. + base_type = scope.parent_type.base_type + if base_type.is_extension_type and base_type.typeobj_cname: + src = '%s.%s' % (base_type.typeobj_cname, self.slot_name) + elif base_type.typeptr_cname: + src = '%s->%s' % (base_type.typeptr_cname, self.slot_name) + else: + return + + self.generate_set_slot_code(src, scope, code) + class SyntheticSlot(InternalMethodSlot): # Type slot descriptor for a synthesized method which @@ -404,6 +497,22 @@ class SyntheticSlot(InternalMethodSlot): else: return self.default_value + def spec_value(self, scope): + if self.slot_name == "tp_getattro" and not scope.defines_any_special(self.user_methods): + return "PyObject_GenericGetAttr" + return self.slot_code(scope) + + +class BinopSlot(SyntheticSlot): + def __init__(self, signature, slot_name, left_method, **kargs): + assert left_method.startswith('__') + right_method = '__r' + left_method[2:] + SyntheticSlot.__init__( + self, slot_name, [left_method, right_method], "0", is_binop=True, **kargs) + # MethodSlot causes special method registration. + self.left_slot = MethodSlot(signature, "", left_method) + self.right_slot = MethodSlot(signature, "", right_method) + class RichcmpSlot(MethodSlot): def slot_code(self, scope): @@ -434,6 +543,10 @@ class TypeFlagsSlot(SlotDescriptor): value += "|Py_TPFLAGS_HAVE_GC" return value + def generate_spec(self, scope, code): + # Flags are stored in the PyType_Spec, not in a PyType_Slot. + return + class DocStringSlot(SlotDescriptor): # Descriptor for the docstring slot. @@ -444,7 +557,7 @@ class DocStringSlot(SlotDescriptor): return "0" if doc.is_unicode: doc = doc.as_utf8_string() - return doc.as_c_string_literal() + return "PyDoc_STR(%s)" % doc.as_c_string_literal() class SuiteSlot(SlotDescriptor): @@ -487,6 +600,13 @@ class SuiteSlot(SlotDescriptor): if self.ifdef: code.putln("#endif") + def generate_spec(self, scope, code): + if self.slot_name == "tp_as_buffer": + # Cannot currently support the buffer protocol in the limited C-API. + return + for slot in self.sub_slots: + slot.generate_spec(scope, code) + substructures = [] # List of all SuiteSlot instances class MethodTableSlot(SlotDescriptor): @@ -520,7 +640,7 @@ class BaseClassSlot(SlotDescriptor): # Slot descriptor for the base class slot. def __init__(self, name): - SlotDescriptor.__init__(self, name, dynamic = 1) + SlotDescriptor.__init__(self, name, dynamic=True) def generate_dynamic_init_code(self, scope, code): base_type = scope.parent_type.base_type @@ -668,7 +788,7 @@ ssizessizeargfunc = Signature("Tzz", "O") # typedef PyObject *(*ssizessizeargfu intobjargproc = Signature("TiO", 'r') # typedef int(*intobjargproc)(PyObject *, int, PyObject *); ssizeobjargproc = Signature("TzO", 'r') # typedef int(*ssizeobjargproc)(PyObject *, Py_ssize_t, PyObject *); intintobjargproc = Signature("TiiO", 'r') # typedef int(*intintobjargproc)(PyObject *, int, int, PyObject *); -ssizessizeobjargproc = Signature("TzzO", 'r') # typedef int(*ssizessizeobjargproc)(PyObject *, Py_ssize_t, Py_ssize_t, PyObject *); +ssizessizeobjargproc = Signature("TzzO", 'r') # typedef int(*ssizessizeobjargproc)(PyObject *, Py_ssize_t, Py_ssize_t, PyObject *); intintargproc = Signature("Tii", 'r') ssizessizeargproc = Signature("Tzz", 'r') @@ -728,23 +848,23 @@ property_accessor_signatures = { PyNumberMethods_Py3_GUARD = "PY_MAJOR_VERSION < 3 || (CYTHON_COMPILING_IN_PYPY && PY_VERSION_HEX < 0x03050000)" PyNumberMethods = ( - MethodSlot(binaryfunc, "nb_add", "__add__"), - MethodSlot(binaryfunc, "nb_subtract", "__sub__"), - MethodSlot(binaryfunc, "nb_multiply", "__mul__"), - MethodSlot(binaryfunc, "nb_divide", "__div__", ifdef = PyNumberMethods_Py3_GUARD), - MethodSlot(binaryfunc, "nb_remainder", "__mod__"), - MethodSlot(binaryfunc, "nb_divmod", "__divmod__"), - MethodSlot(ternaryfunc, "nb_power", "__pow__"), + BinopSlot(binaryfunc, "nb_add", "__add__"), + BinopSlot(binaryfunc, "nb_subtract", "__sub__"), + BinopSlot(binaryfunc, "nb_multiply", "__mul__"), + BinopSlot(binaryfunc, "nb_divide", "__div__", ifdef = PyNumberMethods_Py3_GUARD), + BinopSlot(binaryfunc, "nb_remainder", "__mod__"), + BinopSlot(binaryfunc, "nb_divmod", "__divmod__"), + BinopSlot(ternaryfunc, "nb_power", "__pow__"), MethodSlot(unaryfunc, "nb_negative", "__neg__"), MethodSlot(unaryfunc, "nb_positive", "__pos__"), MethodSlot(unaryfunc, "nb_absolute", "__abs__"), MethodSlot(inquiry, "nb_nonzero", "__nonzero__", py3 = ("nb_bool", "__bool__")), MethodSlot(unaryfunc, "nb_invert", "__invert__"), - MethodSlot(binaryfunc, "nb_lshift", "__lshift__"), - MethodSlot(binaryfunc, "nb_rshift", "__rshift__"), - MethodSlot(binaryfunc, "nb_and", "__and__"), - MethodSlot(binaryfunc, "nb_xor", "__xor__"), - MethodSlot(binaryfunc, "nb_or", "__or__"), + BinopSlot(binaryfunc, "nb_lshift", "__lshift__"), + BinopSlot(binaryfunc, "nb_rshift", "__rshift__"), + BinopSlot(binaryfunc, "nb_and", "__and__"), + BinopSlot(binaryfunc, "nb_xor", "__xor__"), + BinopSlot(binaryfunc, "nb_or", "__or__"), EmptySlot("nb_coerce", ifdef = PyNumberMethods_Py3_GUARD), MethodSlot(unaryfunc, "nb_int", "__int__", fallback="__long__"), MethodSlot(unaryfunc, "nb_long", "__long__", fallback="__int__", py3 = "<RESERVED>"), @@ -758,7 +878,7 @@ PyNumberMethods = ( MethodSlot(ibinaryfunc, "nb_inplace_multiply", "__imul__"), MethodSlot(ibinaryfunc, "nb_inplace_divide", "__idiv__", ifdef = PyNumberMethods_Py3_GUARD), MethodSlot(ibinaryfunc, "nb_inplace_remainder", "__imod__"), - MethodSlot(ibinaryfunc, "nb_inplace_power", "__ipow__"), # actually ternaryfunc!!! + MethodSlot(ibinaryfunc, "nb_inplace_power", "__ipow__"), # actually ternaryfunc!!! MethodSlot(ibinaryfunc, "nb_inplace_lshift", "__ilshift__"), MethodSlot(ibinaryfunc, "nb_inplace_rshift", "__irshift__"), MethodSlot(ibinaryfunc, "nb_inplace_and", "__iand__"), @@ -767,8 +887,8 @@ PyNumberMethods = ( # Added in release 2.2 # The following require the Py_TPFLAGS_HAVE_CLASS flag - MethodSlot(binaryfunc, "nb_floor_divide", "__floordiv__"), - MethodSlot(binaryfunc, "nb_true_divide", "__truediv__"), + BinopSlot(binaryfunc, "nb_floor_divide", "__floordiv__"), + BinopSlot(binaryfunc, "nb_true_divide", "__truediv__"), MethodSlot(ibinaryfunc, "nb_inplace_floor_divide", "__ifloordiv__"), MethodSlot(ibinaryfunc, "nb_inplace_true_divide", "__itruediv__"), @@ -776,21 +896,21 @@ PyNumberMethods = ( MethodSlot(unaryfunc, "nb_index", "__index__"), # Added in release 3.5 - MethodSlot(binaryfunc, "nb_matrix_multiply", "__matmul__", ifdef="PY_VERSION_HEX >= 0x03050000"), + BinopSlot(binaryfunc, "nb_matrix_multiply", "__matmul__", ifdef="PY_VERSION_HEX >= 0x03050000"), MethodSlot(ibinaryfunc, "nb_inplace_matrix_multiply", "__imatmul__", ifdef="PY_VERSION_HEX >= 0x03050000"), ) PySequenceMethods = ( MethodSlot(lenfunc, "sq_length", "__len__"), - EmptySlot("sq_concat"), # nb_add used instead - EmptySlot("sq_repeat"), # nb_multiply used instead + EmptySlot("sq_concat"), # nb_add used instead + EmptySlot("sq_repeat"), # nb_multiply used instead SyntheticSlot("sq_item", ["__getitem__"], "0"), #EmptySlot("sq_item"), # mp_subscript used instead MethodSlot(ssizessizeargfunc, "sq_slice", "__getslice__"), - EmptySlot("sq_ass_item"), # mp_ass_subscript used instead + EmptySlot("sq_ass_item"), # mp_ass_subscript used instead SyntheticSlot("sq_ass_slice", ["__setslice__", "__delslice__"], "0"), MethodSlot(cmpfunc, "sq_contains", "__contains__"), - EmptySlot("sq_inplace_concat"), # nb_inplace_add used instead - EmptySlot("sq_inplace_repeat"), # nb_inplace_multiply used instead + EmptySlot("sq_inplace_concat"), # nb_inplace_add used instead + EmptySlot("sq_inplace_repeat"), # nb_inplace_multiply used instead ) PyMappingMethods = ( @@ -844,8 +964,8 @@ slot_table = ( MethodSlot(callfunc, "tp_call", "__call__"), MethodSlot(reprfunc, "tp_str", "__str__"), - SyntheticSlot("tp_getattro", ["__getattr__","__getattribute__"], "0"), #"PyObject_GenericGetAttr"), - SyntheticSlot("tp_setattro", ["__setattr__", "__delattr__"], "0"), #"PyObject_GenericSetAttr"), + SyntheticSlot("tp_getattro", ["__getattr__","__getattribute__"], "0"), #"PyObject_GenericGetAttr"), + SyntheticSlot("tp_setattro", ["__setattr__", "__delattr__"], "0"), #"PyObject_GenericSetAttr"), SuiteSlot(PyBufferProcs, "PyBufferProcs", "tp_as_buffer"), @@ -866,7 +986,7 @@ slot_table = ( MemberTableSlot("tp_members"), GetSetSlot("tp_getset"), - BaseClassSlot("tp_base"), #EmptySlot("tp_base"), + BaseClassSlot("tp_base"), #EmptySlot("tp_base"), EmptySlot("tp_dict"), SyntheticSlot("tp_descr_get", ["__get__"], "0"), @@ -875,8 +995,8 @@ slot_table = ( DictOffsetSlot("tp_dictoffset"), MethodSlot(initproc, "tp_init", "__init__"), - EmptySlot("tp_alloc"), #FixedSlot("tp_alloc", "PyType_GenericAlloc"), - InternalMethodSlot("tp_new"), + EmptySlot("tp_alloc"), #FixedSlot("tp_alloc", "PyType_GenericAlloc"), + ConstructorSlot("tp_new", "__cinit__"), EmptySlot("tp_free"), EmptySlot("tp_is_gc"), @@ -890,6 +1010,8 @@ slot_table = ( EmptySlot("tp_finalize", ifdef="PY_VERSION_HEX >= 0x030400a1"), EmptySlot("tp_vectorcall", ifdef="PY_VERSION_HEX >= 0x030800b1"), EmptySlot("tp_print", ifdef="PY_VERSION_HEX >= 0x030800b4 && PY_VERSION_HEX < 0x03090000"), + # PyPy specific extension - only here to avoid C compiler warnings. + EmptySlot("tp_pypy_flags", ifdef="CYTHON_COMPILING_IN_PYPY && PYPY_VERSION_NUM+0 >= 0x06000000"), ) #------------------------------------------------------------------------------------------ @@ -920,5 +1042,6 @@ MethodSlot(descrdelfunc, "", "__delete__") method_noargs = "METH_NOARGS" method_onearg = "METH_O" method_varargs = "METH_VARARGS" +method_fastcall = "__Pyx_METH_FASTCALL" # Actually VARARGS on versions < 3.7 method_keywords = "METH_KEYWORDS" method_coexist = "METH_COEXIST" diff --git a/Cython/Compiler/UtilNodes.py b/Cython/Compiler/UtilNodes.py index c41748ace..0d5db9d33 100644 --- a/Cython/Compiler/UtilNodes.py +++ b/Cython/Compiler/UtilNodes.py @@ -45,7 +45,7 @@ class TempRefNode(AtomicExprNode): def calculate_result_code(self): result = self.handle.temp - if result is None: result = "<error>" # might be called and overwritten + if result is None: result = "<error>" # might be called and overwritten return result def generate_result_code(self, code): @@ -122,8 +122,7 @@ class ResultRefNode(AtomicExprNode): self.may_hold_none = may_hold_none if expression is not None: self.pos = expression.pos - if hasattr(expression, "type"): - self.type = expression.type + self.type = getattr(expression, "type", None) if pos is not None: self.pos = pos if type is not None: @@ -144,13 +143,14 @@ class ResultRefNode(AtomicExprNode): def update_expression(self, expression): self.expression = expression - if hasattr(expression, "type"): - self.type = expression.type + type = getattr(expression, "type", None) + if type: + self.type = type def analyse_types(self, env): if self.expression is not None: if not self.expression.type: - self.expression = self.expression.analyse_types(env) + self.expression = self.expression.analyse_types(env) self.type = self.expression.type return self @@ -175,7 +175,7 @@ class ResultRefNode(AtomicExprNode): return self.expression.may_be_none() if self.type is not None: return self.type.is_pyobject - return True # play safe + return True # play it safe def is_simple(self): return True @@ -354,6 +354,12 @@ class TempResultFromStatNode(ExprNodes.ExprNode): self.body = self.body.analyse_expressions(env) return self + def may_be_none(self): + return self.result_ref.may_be_none() + def generate_result_code(self, code): self.result_ref.result_code = self.result() self.body.generate_execution_code(code) + + def generate_function_definitions(self, env, code): + self.body.generate_function_definitions(env, code) diff --git a/Cython/Compiler/UtilityCode.py b/Cython/Compiler/UtilityCode.py index 98e9ab5bf..c58c8623c 100644 --- a/Cython/Compiler/UtilityCode.py +++ b/Cython/Compiler/UtilityCode.py @@ -131,7 +131,7 @@ class CythonUtilityCode(Code.UtilityCodeBase): p = [] for t in pipeline: p.append(t) - if isinstance(p, ParseTreeTransforms.AnalyseDeclarationsTransform): + if isinstance(t, ParseTreeTransforms.AnalyseDeclarationsTransform): break pipeline = p @@ -196,10 +196,10 @@ class CythonUtilityCode(Code.UtilityCodeBase): Load a utility code as a string. Returns (proto, implementation) """ util = cls.load(util_code_name, from_file, **kwargs) - return util.proto, util.impl # keep line numbers => no lstrip() + return util.proto, util.impl # keep line numbers => no lstrip() def declare_in_scope(self, dest_scope, used=False, cython_scope=None, - whitelist=None): + allowlist=None): """ Declare all entries from the utility code in dest_scope. Code will only be included for used entries. If module_name is given, declare the @@ -218,7 +218,7 @@ class CythonUtilityCode(Code.UtilityCodeBase): entry.used = used original_scope = tree.scope - dest_scope.merge_in(original_scope, merge_unused=True, whitelist=whitelist) + dest_scope.merge_in(original_scope, merge_unused=True, allowlist=allowlist) tree.scope = dest_scope for dep in self.requires: diff --git a/Cython/Compiler/Visitor.pxd b/Cython/Compiler/Visitor.pxd index d5d5692aa..6b8110383 100644 --- a/Cython/Compiler/Visitor.pxd +++ b/Cython/Compiler/Visitor.pxd @@ -1,4 +1,4 @@ -from __future__ import absolute_import +# cython: language_level=3 cimport cython diff --git a/Cython/Compiler/Visitor.py b/Cython/Compiler/Visitor.py index bd5655f5b..0cf5ee1eb 100644 --- a/Cython/Compiler/Visitor.py +++ b/Cython/Compiler/Visitor.py @@ -569,6 +569,17 @@ class MethodDispatcherTransform(EnvTransform): ### dispatch to specific handlers def _find_handler(self, match_name, has_kwargs): + try: + match_name.encode('ascii') + except UnicodeEncodeError: + # specifically when running the Cython compiler under Python 2 + # getattr can't take a unicode string. + # Classes with unicode names won't have specific handlers and thus it + # should be OK to return None. + # Doing the test here ensures that the same code gets run on + # Python 2 and 3 + return None + call_type = has_kwargs and 'general' or 'simple' handler = getattr(self, '_handle_%s_%s' % (call_type, match_name), None) if handler is None: @@ -821,6 +832,10 @@ class PrintTree(TreeVisitor): result += "(type=%s, name=\"%s\")" % (repr(node.type), node.name) elif isinstance(node, Nodes.DefNode): result += "(name=\"%s\")" % node.name + elif isinstance(node, ExprNodes.AttributeNode): + result += "(type=%s, attribute=\"%s\")" % (repr(node.type), node.attribute) + elif isinstance(node, (ExprNodes.ConstNode, ExprNodes.PyConstNode)): + result += "(type=%s, value=%r)" % (repr(node.type), node.value) elif isinstance(node, ExprNodes.ExprNode): t = node.type result += "(type=%s)" % repr(t) diff --git a/Cython/Coverage.py b/Cython/Coverage.py index 269896e13..0ef9815e7 100644 --- a/Cython/Coverage.py +++ b/Cython/Coverage.py @@ -57,10 +57,19 @@ class Plugin(CoveragePlugin): _c_files_map = None # map from parsed C files to their content _parsed_c_files = None + # map from traced files to lines that are excluded from coverage + _excluded_lines_map = None + # list of regex patterns for lines to exclude + _excluded_line_patterns = () def sys_info(self): return [('Cython version', __version__)] + def configure(self, config): + # Entry point for coverage "configurer". + # Read the regular expressions from the coverage config that match lines to be excluded from coverage. + self._excluded_line_patterns = config.get_option("report:exclude_lines") + def file_tracer(self, filename): """ Try to find a C source file for a file path found by the tracer. @@ -108,7 +117,13 @@ class Plugin(CoveragePlugin): rel_file_path, code = self._read_source_lines(c_file, filename) if code is None: return None # no source found - return CythonModuleReporter(c_file, filename, rel_file_path, code) + return CythonModuleReporter( + c_file, + filename, + rel_file_path, + code, + self._excluded_lines_map.get(rel_file_path, frozenset()) + ) def _find_source_files(self, filename): basename, ext = os.path.splitext(filename) @@ -218,10 +233,16 @@ class Plugin(CoveragePlugin): r'(?:struct|union|enum|class)' r'(\s+[^:]+|)\s*:' ).match + if self._excluded_line_patterns: + line_is_excluded = re.compile("|".join(["(?:%s)" % regex for regex in self._excluded_line_patterns])).search + else: + line_is_excluded = lambda line: False code_lines = defaultdict(dict) executable_lines = defaultdict(set) current_filename = None + if self._excluded_lines_map is None: + self._excluded_lines_map = defaultdict(set) with open(c_file) as lines: lines = iter(lines) @@ -242,6 +263,9 @@ class Plugin(CoveragePlugin): code_line = match.group(1).rstrip() if not_executable(code_line): break + if line_is_excluded(code_line): + self._excluded_lines_map[filename].add(lineno) + break code_lines[filename][lineno] = code_line break elif match_comment_end(comment_line): @@ -298,11 +322,12 @@ class CythonModuleReporter(FileReporter): """ Provide detailed trace information for one source file to coverage.py. """ - def __init__(self, c_file, source_file, rel_file_path, code): + def __init__(self, c_file, source_file, rel_file_path, code, excluded_lines): super(CythonModuleReporter, self).__init__(source_file) self.name = rel_file_path self.c_file = c_file self._code = code + self._excluded_lines = excluded_lines def lines(self): """ @@ -310,6 +335,12 @@ class CythonModuleReporter(FileReporter): """ return set(self._code) + def excluded_lines(self): + """ + Return set of line numbers that are excluded from coverage. + """ + return self._excluded_lines + def _iter_source_tokens(self): current_line = 1 for line_no, code_line in sorted(self._code.items()): @@ -345,4 +376,6 @@ class CythonModuleReporter(FileReporter): def coverage_init(reg, options): - reg.add_file_tracer(Plugin()) + plugin = Plugin() + reg.add_configurer(plugin) + reg.add_file_tracer(plugin) diff --git a/Cython/Debugger/Cygdb.py b/Cython/Debugger/Cygdb.py index 45f31ce6f..1b0a794e3 100644 --- a/Cython/Debugger/Cygdb.py +++ b/Cython/Debugger/Cygdb.py @@ -45,17 +45,23 @@ def make_command_file(path_to_debug_info, prefix_code='', no_import=False): set print pretty on python - # Activate virtualenv, if we were launched from one - import os - virtualenv = os.getenv('VIRTUAL_ENV') - if virtualenv: - path_to_activate_this_py = os.path.join(virtualenv, 'bin', 'activate_this.py') - print("gdb command file: Activating virtualenv: %s; path_to_activate_this_py: %s" % ( - virtualenv, path_to_activate_this_py)) - with open(path_to_activate_this_py) as f: - exec(f.read(), dict(__file__=path_to_activate_this_py)) - - from Cython.Debugger import libcython, libpython + try: + # Activate virtualenv, if we were launched from one + import os + virtualenv = os.getenv('VIRTUAL_ENV') + if virtualenv: + path_to_activate_this_py = os.path.join(virtualenv, 'bin', 'activate_this.py') + print("gdb command file: Activating virtualenv: %s; path_to_activate_this_py: %s" % ( + virtualenv, path_to_activate_this_py)) + with open(path_to_activate_this_py) as f: + exec(f.read(), dict(__file__=path_to_activate_this_py)) + from Cython.Debugger import libcython, libpython + except Exception as ex: + from traceback import print_exc + print("There was an error in Python code originating from the file ''' + str(__file__) + '''") + print("It used the Python interpreter " + str(sys.executable)) + print_exc() + exit(1) end ''')) @@ -79,8 +85,8 @@ def make_command_file(path_to_debug_info, prefix_code='', no_import=False): gdb.lookup_type('PyModuleObject') except RuntimeError: sys.stderr.write( - 'Python was not compiled with debug symbols (or it was ' - 'stripped). Some functionality may not work (properly).\\n') + "''' + interpreter + ''' was not compiled with debug symbols (or it was " + "stripped). Some functionality may not work (properly).\\n") end source .cygdbinit diff --git a/Cython/Debugger/DebugWriter.py b/Cython/Debugger/DebugWriter.py index 876a3a216..8b1fb75b0 100644 --- a/Cython/Debugger/DebugWriter.py +++ b/Cython/Debugger/DebugWriter.py @@ -5,8 +5,8 @@ import sys import errno try: - from lxml import etree - have_lxml = True + from lxml import etree + have_lxml = True except ImportError: have_lxml = False try: diff --git a/Cython/Debugger/Tests/TestLibCython.py b/Cython/Debugger/Tests/TestLibCython.py index 13560646f..fd7d00c05 100644 --- a/Cython/Debugger/Tests/TestLibCython.py +++ b/Cython/Debugger/Tests/TestLibCython.py @@ -56,13 +56,13 @@ def test_gdb(): stdout, _ = p.communicate() try: internal_python_version = list(map(int, stdout.decode('ascii', 'ignore').split())) - if internal_python_version < [2, 6]: + if internal_python_version < [2, 7]: have_gdb = False except ValueError: have_gdb = False if not have_gdb: - warnings.warn('Skipping gdb tests, need gdb >= 7.2 with Python >= 2.6') + warnings.warn('Skipping gdb tests, need gdb >= 7.2 with Python >= 2.7') return have_gdb diff --git a/Cython/Debugger/Tests/codefile b/Cython/Debugger/Tests/codefile index 6b4c6b6ad..ee587cbb1 100644 --- a/Cython/Debugger/Tests/codefile +++ b/Cython/Debugger/Tests/codefile @@ -37,14 +37,13 @@ def outer(): def inner(): b = 2 # access closed over variables - print a, b + print(a, b) return inner - outer()() spam() -print "bye!" +print("bye!") def use_ham(): ham() diff --git a/Cython/Debugger/Tests/test_libcython_in_gdb.py b/Cython/Debugger/Tests/test_libcython_in_gdb.py index bd7608d60..bb06c2905 100644 --- a/Cython/Debugger/Tests/test_libcython_in_gdb.py +++ b/Cython/Debugger/Tests/test_libcython_in_gdb.py @@ -134,7 +134,7 @@ class TestDebugInformationClasses(DebugTestCase): self.assertEqual(self.spam_func.arguments, ['a']) self.assertEqual(self.spam_func.step_into_functions, - set(['puts', 'some_c_function'])) + {'puts', 'some_c_function'}) expected_lineno = test_libcython.source_to_lineno['def spam(a=0):'] self.assertEqual(self.spam_func.lineno, expected_lineno) @@ -177,12 +177,13 @@ class TestBreak(DebugTestCase): assert step_result.rstrip().endswith(nextline) -class TestKilled(DebugTestCase): - - def test_abort(self): - gdb.execute("set args -c 'import os; os.abort()'") - output = gdb.execute('cy run', to_string=True) - assert 'abort' in output.lower() +# I removed this testcase, because it will never work, because +# gdb.execute(..., to_string=True) does not capture stdout and stderr of python. +# class TestKilled(DebugTestCase): +# def test_abort(self): +# gdb.execute("set args -c 'import os;print(123456789);os.abort()'") +# output = gdb.execute('cy run', to_string=True) +# assert 'abort' in output.lower() class DebugStepperTestCase(DebugTestCase): @@ -322,6 +323,61 @@ class TestPrint(DebugTestCase): self.break_and_run('c = 2') result = gdb.execute('cy print b', to_string=True) self.assertEqual('b = (int) 1\n', result) + result = gdb.execute('cy print python_var', to_string=True) + self.assertEqual('python_var = 13\n', result) + result = gdb.execute('cy print c_var', to_string=True) + self.assertEqual('c_var = (int) 12\n', result) + +correct_result_test_list_inside_func = '''\ + 14 int b, c + 15 + 16 b = c = d = 0 + 17 + 18 b = 1 +> 19 c = 2 + 20 int(10) + 21 puts("spam") + 22 os.path.join("foo", "bar") + 23 some_c_function() +''' +correct_result_test_list_outside_func = '''\ + 5 void some_c_function() + 6 + 7 import os + 8 + 9 cdef int c_var = 12 +> 10 python_var = 13 + 11 + 12 def spam(a=0): + 13 cdef: + 14 int b, c +''' + + +class TestList(DebugTestCase): + def workaround_for_coding_style_checker(self, correct_result_wrong_whitespace): + correct_result = "" + for line in correct_result_test_list_inside_func.split("\n"): + if len(line) < 10 and len(line) > 0: + line += " "*4 + correct_result += line + "\n" + correct_result = correct_result[:-1] + + def test_list_inside_func(self): + self.break_and_run('c = 2') + result = gdb.execute('cy list', to_string=True) + # We don't want to fail because of some trailing whitespace, + # so we remove trailing whitespaces with the following line + result = "\n".join([line.rstrip() for line in result.split("\n")]) + self.assertEqual(correct_result_test_list_inside_func, result) + + def test_list_outside_func(self): + self.break_and_run('python_var = 13') + result = gdb.execute('cy list', to_string=True) + # We don't want to fail because of some trailing whitespace, + # so we remove trailing whitespaces with the following line + result = "\n".join([line.rstrip() for line in result.split("\n")]) + self.assertEqual(correct_result_test_list_outside_func, result) class TestUpDown(DebugTestCase): @@ -362,6 +418,7 @@ class TestExec(DebugTestCase): # test normal behaviour self.assertEqual("[0]", self.eval_command('[a]')) + return #The test after this return freezes gdb, so I temporarily removed it. # test multiline code result = gdb.execute(textwrap.dedent('''\ cy exec diff --git a/Cython/Debugger/libcython.py b/Cython/Debugger/libcython.py index 23153789b..dd634cc35 100644 --- a/Cython/Debugger/libcython.py +++ b/Cython/Debugger/libcython.py @@ -11,7 +11,6 @@ except NameError: import sys import textwrap -import traceback import functools import itertools import collections @@ -69,19 +68,6 @@ _filesystemencoding = sys.getfilesystemencoding() or 'UTF-8' # decorators -def dont_suppress_errors(function): - "*sigh*, readline" - @functools.wraps(function) - def wrapper(*args, **kwargs): - try: - return function(*args, **kwargs) - except Exception: - traceback.print_exc() - raise - - return wrapper - - def default_selected_gdb_frame(err=True): def decorator(function): @functools.wraps(function) @@ -252,7 +238,9 @@ class CythonBase(object): filename = lineno = lexer = None if self.is_cython_function(frame): filename = self.get_cython_function(frame).module.filename - lineno = self.get_cython_lineno(frame) + filename_and_lineno = self.get_cython_lineno(frame) + assert filename == filename_and_lineno[0] + lineno = filename_and_lineno[1] if pygments: lexer = pygments.lexers.CythonLexer(stripall=False) elif self.is_python_function(frame): @@ -334,7 +322,7 @@ class CythonBase(object): func_name = cyfunc.name func_cname = cyfunc.cname - func_args = [] # [(arg, f(arg)) for arg in cyfunc.arguments] + func_args = [] # [(arg, f(arg)) for arg in cyfunc.arguments] else: source_desc, lineno = self.get_source_desc(frame) func_name = frame.name() @@ -392,7 +380,7 @@ class CythonBase(object): result = {} seen = set() - for k, v in pyobject_dict.items(): + for k, v in pyobject_dict.iteritems(): result[k.proxyval(seen)] = v return result @@ -410,7 +398,7 @@ class CythonBase(object): def is_initialized(self, cython_func, local_name): cyvar = cython_func.locals[local_name] - cur_lineno = self.get_cython_lineno() + cur_lineno = self.get_cython_lineno()[1] if '->' in cyvar.cname: # Closed over free variable @@ -695,6 +683,7 @@ class CyImport(CythonCommand): command_class = gdb.COMMAND_STATUS completer_class = gdb.COMPLETE_FILENAME + @libpython.dont_suppress_errors def invoke(self, args, from_tty): if isinstance(args, BYTES): args = args.decode(_filesystemencoding) @@ -742,11 +731,12 @@ class CyImport(CythonCommand): funcarg.tag for funcarg in function.find('Arguments')) for marker in module.find('LineNumberMapping'): - cython_lineno = int(marker.attrib['cython_lineno']) + src_lineno = int(marker.attrib['src_lineno']) + src_path = marker.attrib['src_path'] c_linenos = list(map(int, marker.attrib['c_linenos'].split())) - cython_module.lineno_cy2c[cython_lineno] = min(c_linenos) + cython_module.lineno_cy2c[src_path, src_lineno] = min(c_linenos) for c_lineno in c_linenos: - cython_module.lineno_c2cy[c_lineno] = cython_lineno + cython_module.lineno_c2cy[c_lineno] = (src_path, src_lineno) class CyBreak(CythonCommand): @@ -784,8 +774,8 @@ class CyBreak(CythonCommand): else: cython_module = self.get_cython_function().module - if lineno in cython_module.lineno_cy2c: - c_lineno = cython_module.lineno_cy2c[lineno] + if (cython_module.filename, lineno) in cython_module.lineno_cy2c: + c_lineno = cython_module.lineno_cy2c[cython_module.filename, lineno] breakpoint = '%s:%s' % (cython_module.c_filename, c_lineno) gdb.execute('break ' + breakpoint) else: @@ -841,6 +831,7 @@ class CyBreak(CythonCommand): if func.pf_cname: gdb.execute('break %s' % func.pf_cname) + @libpython.dont_suppress_errors def invoke(self, function_names, from_tty): if isinstance(function_names, BYTES): function_names = function_names.decode(_filesystemencoding) @@ -859,7 +850,7 @@ class CyBreak(CythonCommand): else: self._break_funcname(funcname) - @dont_suppress_errors + @libpython.dont_suppress_errors def complete(self, text, word): # Filter init-module functions (breakpoints can be set using # modulename:linenumber). @@ -904,7 +895,7 @@ class CythonInfo(CythonBase, libpython.PythonInfo): # stepping through Python code, but it would not step back into Cython- # related code. The C level should be dispatched to the 'step' command. if self.is_cython_function(frame): - return self.get_cython_lineno(frame) + return self.get_cython_lineno(frame)[1] return super(CythonInfo, self).lineno(frame) def get_source_line(self, frame): @@ -944,6 +935,7 @@ class CyStep(CythonExecutionControlCommand, libpython.PythonStepperMixin): name = 'cy -step' stepinto = True + @libpython.dont_suppress_errors def invoke(self, args, from_tty): if self.is_python_function(): self.python_step(self.stepinto) @@ -973,7 +965,7 @@ class CyRun(CythonExecutionControlCommand): name = 'cy run' - invoke = CythonExecutionControlCommand.run + invoke = libpython.dont_suppress_errors(CythonExecutionControlCommand.run) class CyCont(CythonExecutionControlCommand): @@ -983,7 +975,7 @@ class CyCont(CythonExecutionControlCommand): """ name = 'cy cont' - invoke = CythonExecutionControlCommand.cont + invoke = libpython.dont_suppress_errors(CythonExecutionControlCommand.cont) class CyFinish(CythonExecutionControlCommand): @@ -992,7 +984,7 @@ class CyFinish(CythonExecutionControlCommand): """ name = 'cy finish' - invoke = CythonExecutionControlCommand.finish + invoke = libpython.dont_suppress_errors(CythonExecutionControlCommand.finish) class CyUp(CythonCommand): @@ -1002,6 +994,7 @@ class CyUp(CythonCommand): name = 'cy up' _command = 'up' + @libpython.dont_suppress_errors def invoke(self, *args): try: gdb.execute(self._command, to_string=True) @@ -1036,6 +1029,7 @@ class CySelect(CythonCommand): name = 'cy select' + @libpython.dont_suppress_errors def invoke(self, stackno, from_tty): try: stackno = int(stackno) @@ -1062,6 +1056,7 @@ class CyBacktrace(CythonCommand): command_class = gdb.COMMAND_STACK completer_class = gdb.COMPLETE_NONE + @libpython.dont_suppress_errors @require_running_program def invoke(self, args, from_tty): # get the first frame @@ -1095,6 +1090,7 @@ class CyList(CythonCommand): command_class = gdb.COMMAND_FILES completer_class = gdb.COMPLETE_NONE + @libpython.dont_suppress_errors # @dispatch_on_frame(c_command='list') def invoke(self, _, from_tty): sd, lineno = self.get_source_desc() @@ -1111,8 +1107,28 @@ class CyPrint(CythonCommand): name = 'cy print' command_class = gdb.COMMAND_DATA - def invoke(self, name, from_tty, max_name_length=None): - if self.is_python_function(): + @libpython.dont_suppress_errors + def invoke(self, name, from_tty): + global_python_dict = self.get_cython_globals_dict() + module_globals = self.get_cython_function().module.globals + + if name in global_python_dict: + value = global_python_dict[name].get_truncated_repr(libpython.MAX_OUTPUT_LEN) + print('%s = %s' % (name, value)) + #This also would work, but beacause the output of cy exec is not captured in gdb.execute, TestPrint would fail + #self.cy.exec_.invoke("print('"+name+"','=', type(" + name + "), "+name+", flush=True )", from_tty) + elif name in module_globals: + cname = module_globals[name].cname + try: + value = gdb.parse_and_eval(cname) + except RuntimeError: + print("unable to get value of %s" % name) + else: + if not value.is_optimized_out: + self.print_gdb_value(name, value) + else: + print("%s is optimized out" % name) + elif self.is_python_function(): return gdb.execute('py-print ' + name) elif self.is_cython_function(): value = self.cy.cy_cvalue.invoke(name.lstrip('*')) @@ -1122,7 +1138,7 @@ class CyPrint(CythonCommand): else: break - self.print_gdb_value(name, value, max_name_length) + self.print_gdb_value(name, value) else: gdb.execute('print ' + name) @@ -1146,6 +1162,7 @@ class CyLocals(CythonCommand): command_class = gdb.COMMAND_STACK completer_class = gdb.COMPLETE_NONE + @libpython.dont_suppress_errors @dispatch_on_frame(c_command='info locals', python_command='py-locals') def invoke(self, args, from_tty): cython_function = self.get_cython_function() @@ -1173,6 +1190,7 @@ class CyGlobals(CyLocals): command_class = gdb.COMMAND_STACK completer_class = gdb.COMPLETE_NONE + @libpython.dont_suppress_errors @dispatch_on_frame(c_command='info variables', python_command='py-globals') def invoke(self, args, from_tty): global_python_dict = self.get_cython_globals_dict() @@ -1189,6 +1207,7 @@ class CyGlobals(CyLocals): seen = set() print('Python globals:') + for k, v in sorted(global_python_dict.items(), key=sortkey): v = v.get_truncated_repr(libpython.MAX_OUTPUT_LEN) seen.add(k) @@ -1219,7 +1238,9 @@ class EvaluateOrExecuteCodeMixin(object): cython_func = self.get_cython_function() for name, cyvar in cython_func.locals.items(): - if cyvar.type == PythonObject and self.is_initialized(cython_func, name): + if (cyvar.type == PythonObject + and self.is_initialized(cython_func, name)): + try: val = gdb.parse_and_eval(cyvar.cname) except RuntimeError: @@ -1247,8 +1268,8 @@ class EvaluateOrExecuteCodeMixin(object): def _find_first_cython_or_python_frame(self): frame = gdb.selected_frame() while frame: - if (self.is_cython_function(frame) or - self.is_python_function(frame)): + if (self.is_cython_function(frame) + or self.is_python_function(frame)): frame.select() return frame @@ -1295,10 +1316,11 @@ class CyExec(CythonCommand, libpython.PyExec, EvaluateOrExecuteCodeMixin): command_class = gdb.COMMAND_STACK completer_class = gdb.COMPLETE_NONE + @libpython.dont_suppress_errors def invoke(self, expr, from_tty): expr, input_type = self.readcode(expr) executor = libpython.PythonCodeExecutor() - executor.xdecref(self.evalcode(expr, executor.Py_single_input)) + executor.xdecref(self.evalcode(expr, executor.Py_file_input)) class CySet(CythonCommand): @@ -1317,6 +1339,7 @@ class CySet(CythonCommand): command_class = gdb.COMMAND_DATA completer_class = gdb.COMPLETE_NONE + @libpython.dont_suppress_errors @require_cython_frame def invoke(self, expr, from_tty): name_and_expr = expr.split('=', 1) @@ -1340,6 +1363,7 @@ class CyCName(gdb.Function, CythonBase): print $cy_cname("module.function") """ + @libpython.dont_suppress_errors @require_cython_frame @gdb_function_value_to_unicode def invoke(self, cyname, frame=None): @@ -1371,6 +1395,7 @@ class CyCValue(CyCName): Get the value of a Cython variable. """ + @libpython.dont_suppress_errors @require_cython_frame @gdb_function_value_to_unicode def invoke(self, cyname, frame=None): @@ -1391,9 +1416,10 @@ class CyLine(gdb.Function, CythonBase): Get the current Cython line. """ + @libpython.dont_suppress_errors @require_cython_frame def invoke(self): - return self.get_cython_lineno() + return self.get_cython_lineno()[1] class CyEval(gdb.Function, CythonBase, EvaluateOrExecuteCodeMixin): @@ -1401,6 +1427,7 @@ class CyEval(gdb.Function, CythonBase, EvaluateOrExecuteCodeMixin): Evaluate Python code in the nearest Python or Cython frame and return """ + @libpython.dont_suppress_errors @gdb_function_value_to_unicode def invoke(self, python_expression): input_type = libpython.PythonCodeExecutor.Py_eval_input diff --git a/Cython/Debugger/libpython.py b/Cython/Debugger/libpython.py index fea626dd7..30713e0d2 100644 --- a/Cython/Debugger/libpython.py +++ b/Cython/Debugger/libpython.py @@ -1,9 +1,10 @@ #!/usr/bin/python -# NOTE: this file is taken from the Python source distribution +# NOTE: Most of this file is taken from the Python source distribution # It can be found under Tools/gdb/libpython.py. It is shipped with Cython # because it's not installed as a python module, and because changes are only # merged into new python versions (v3.2+). +# We added some of our code below the "## added, not in CPython" comment. ''' From gdb 7 onwards, gdb's build can be configured --with-python, allowing gdb @@ -105,6 +106,8 @@ hexdigits = "0123456789abcdef" ENCODING = locale.getpreferredencoding() +FRAME_INFO_OPTIMIZED_OUT = '(frame information optimized out)' +UNABLE_READ_INFO_PYTHON_FRAME = 'Unable to read information on python frame' EVALFRAME = '_PyEval_EvalFrameDefault' class NullPyObjectPtr(RuntimeError): @@ -276,12 +279,13 @@ class PyObjectPtr(object): def safe_tp_name(self): try: - return self.type().field('tp_name').string() - except NullPyObjectPtr: - # NULL tp_name? - return 'unknown' - except RuntimeError: - # Can't even read the object at all? + ob_type = self.type() + tp_name = ob_type.field('tp_name') + return tp_name.string() + # NullPyObjectPtr: NULL tp_name? + # RuntimeError: Can't even read the object at all? + # UnicodeDecodeError: Failed to decode tp_name bytestring + except (NullPyObjectPtr, RuntimeError, UnicodeDecodeError): return 'unknown' def proxyval(self, visited): @@ -355,7 +359,9 @@ class PyObjectPtr(object): try: tp_name = t.field('tp_name').string() tp_flags = int(t.field('tp_flags')) - except RuntimeError: + # RuntimeError: NULL pointers + # UnicodeDecodeError: string() fails to decode the bytestring + except (RuntimeError, UnicodeDecodeError): # Handle any kind of error e.g. NULL ptrs by simply using the base # class return cls @@ -622,8 +628,11 @@ class PyCFunctionObjectPtr(PyObjectPtr): _typename = 'PyCFunctionObject' def proxyval(self, visited): - m_ml = self.field('m_ml') # m_ml is a (PyMethodDef*) - ml_name = m_ml['ml_name'].string() + m_ml = self.field('m_ml') # m_ml is a (PyMethodDef*) + try: + ml_name = m_ml['ml_name'].string() + except UnicodeDecodeError: + ml_name = '<ml_name:UnicodeDecodeError>' pyop_m_self = self.pyop_field('m_self') if pyop_m_self.is_null(): @@ -736,7 +745,7 @@ class PyDictObjectPtr(PyObjectPtr): else: offset = 8 * dk_size - ent_addr = keys['dk_indices']['as_1'].address + ent_addr = keys['dk_indices'].address ent_addr = ent_addr.cast(_type_unsigned_char_ptr()) + offset ent_ptr_t = gdb.lookup_type('PyDictKeyEntry').pointer() ent_addr = ent_addr.cast(ent_ptr_t) @@ -918,7 +927,7 @@ class PyFrameObjectPtr(PyObjectPtr): def filename(self): '''Get the path of the current Python source file, as a string''' if self.is_optimized_out(): - return '(frame information optimized out)' + return FRAME_INFO_OPTIMIZED_OUT return self.co_filename.proxyval(set()) def current_line_num(self): @@ -934,35 +943,50 @@ class PyFrameObjectPtr(PyObjectPtr): if long(f_trace) != 0: # we have a non-NULL f_trace: return self.f_lineno - else: - #try: + + try: return self.co.addr2line(self.f_lasti) - #except ValueError: - # return self.f_lineno + except Exception: + # bpo-34989: addr2line() is a complex function, it can fail in many + # ways. For example, it fails with a TypeError on "FakeRepr" if + # gdb fails to load debug symbols. Use a catch-all "except + # Exception" to make the whole function safe. The caller has to + # handle None anyway for optimized Python. + return None def current_line(self): '''Get the text of the current source line as a string, with a trailing newline character''' if self.is_optimized_out(): - return '(frame information optimized out)' + return FRAME_INFO_OPTIMIZED_OUT + + lineno = self.current_line_num() + if lineno is None: + return '(failed to get frame line number)' + filename = self.filename() try: - f = open(os_fsencode(filename), 'r') + with open(os_fsencode(filename), 'r') as fp: + lines = fp.readlines() except IOError: return None - with f: - all_lines = f.readlines() - # Convert from 1-based current_line_num to 0-based list offset: - return all_lines[self.current_line_num()-1] + + try: + # Convert from 1-based current_line_num to 0-based list offset + return lines[lineno - 1] + except IndexError: + return None def write_repr(self, out, visited): if self.is_optimized_out(): - out.write('(frame information optimized out)') + out.write(FRAME_INFO_OPTIMIZED_OUT) return - out.write('Frame 0x%x, for file %s, line %i, in %s (' + lineno = self.current_line_num() + lineno = str(lineno) if lineno is not None else "?" + out.write('Frame 0x%x, for file %s, line %s, in %s (' % (self.as_address(), self.co_filename.proxyval(visited), - self.current_line_num(), + lineno, self.co_name.proxyval(visited))) first = True for pyop_name, pyop_value in self.iter_locals(): @@ -978,12 +1002,14 @@ class PyFrameObjectPtr(PyObjectPtr): def print_traceback(self): if self.is_optimized_out(): - sys.stdout.write(' (frame information optimized out)\n') + sys.stdout.write(' %s\n' % FRAME_INFO_OPTIMIZED_OUT) return visited = set() - sys.stdout.write(' File "%s", line %i, in %s\n' + lineno = self.current_line_num() + lineno = str(lineno) if lineno is not None else "?" + sys.stdout.write(' File "%s", line %s, in %s\n' % (self.co_filename.proxyval(visited), - self.current_line_num(), + lineno, self.co_name.proxyval(visited))) class PySetObjectPtr(PyObjectPtr): @@ -1091,11 +1117,6 @@ class PyBytesObjectPtr(PyObjectPtr): out.write(byte) out.write(quote) - -class PyStringObjectPtr(PyBytesObjectPtr): - _typename = 'PyStringObject' - - class PyTupleObjectPtr(PyObjectPtr): _typename = 'PyTupleObject' @@ -1166,7 +1187,7 @@ class PyUnicodeObjectPtr(PyObjectPtr): def proxyval(self, visited): global _is_pep393 if _is_pep393 is None: - fields = gdb.lookup_type('PyUnicodeObject').target().fields() + fields = gdb.lookup_type('PyUnicodeObject').fields() _is_pep393 = 'data' in [f.name for f in fields] if _is_pep393: # Python 3.3 and newer @@ -1285,8 +1306,8 @@ class PyUnicodeObjectPtr(PyObjectPtr): # If sizeof(Py_UNICODE) is 2 here (in gdb), join # surrogate pairs before calling _unichr_is_printable. if (i < len(proxy) - and 0xD800 <= ord(ch) < 0xDC00 \ - and 0xDC00 <= ord(proxy[i]) <= 0xDFFF): + and 0xD800 <= ord(ch) < 0xDC00 + and 0xDC00 <= ord(proxy[i]) <= 0xDFFF): ch2 = proxy[i] ucs = ch + ch2 i += 1 @@ -1351,13 +1372,13 @@ class wrapperobject(PyObjectPtr): try: name = self.field('descr')['d_base']['name'].string() return repr(name) - except (NullPyObjectPtr, RuntimeError): + except (NullPyObjectPtr, RuntimeError, UnicodeDecodeError): return '<unknown name>' def safe_tp_name(self): try: return self.field('self')['ob_type']['tp_name'].string() - except (NullPyObjectPtr, RuntimeError): + except (NullPyObjectPtr, RuntimeError, UnicodeDecodeError): return '<unknown tp_name>' def safe_self_addresss(self): @@ -1380,7 +1401,7 @@ class wrapperobject(PyObjectPtr): def int_from_int(gdbval): - return int(str(gdbval)) + return int(gdbval) def stringify(val): @@ -1551,8 +1572,8 @@ class Frame(object): if not caller: return False - if caller in ('_PyCFunction_FastCallDict', - '_PyCFunction_FastCallKeywords'): + if (caller.startswith('cfunction_vectorcall_') or + caller == 'cfunction_call'): arg_name = 'func' # Within that frame: # "func" is the local containing the PyObject* of the @@ -1563,15 +1584,22 @@ class Frame(object): # Use the prettyprinter for the func: func = frame.read_var(arg_name) return str(func) + except ValueError: + return ('PyCFunction invocation (unable to read %s: ' + 'missing debuginfos?)' % arg_name) except RuntimeError: return 'PyCFunction invocation (unable to read %s)' % arg_name if caller == 'wrapper_call': + arg_name = 'wp' try: - func = frame.read_var('wp') + func = frame.read_var(arg_name) return str(func) + except ValueError: + return ('<wrapper_call invocation (unable to read %s: ' + 'missing debuginfos?)>' % arg_name) except RuntimeError: - return '<wrapper_call invocation>' + return '<wrapper_call invocation (unable to read %s)>' % arg_name # This frame isn't worth reporting: return False @@ -1581,7 +1609,7 @@ class Frame(object): # This assumes the _POSIX_THREADS version of Python/ceval_gil.h: name = self._gdbframe.name() if name: - return 'pthread_cond_timedwait' in name + return (name == 'take_gil') def is_gc_collect(self): '''Is this frame "collect" within the garbage-collector?''' @@ -1725,11 +1753,14 @@ class PyList(gdb.Command): pyop = frame.get_pyop() if not pyop or pyop.is_optimized_out(): - print('Unable to read information on python frame') + print(UNABLE_READ_INFO_PYTHON_FRAME) return filename = pyop.filename() lineno = pyop.current_line_num() + if lineno is None: + print('Unable to read python frame line number') + return if start is None: start = lineno - 5 @@ -1882,7 +1913,7 @@ class PyPrint(gdb.Command): pyop_frame = frame.get_pyop() if not pyop_frame: - print('Unable to read information on python frame') + print(UNABLE_READ_INFO_PYTHON_FRAME) return pyop_var, scope = pyop_frame.get_var_by_name(name) @@ -1899,9 +1930,9 @@ PyPrint() class PyLocals(gdb.Command): 'Look up the given python variable name, and print it' - def __init__(self, command="py-locals"): + def __init__(self): gdb.Command.__init__ (self, - command, + "py-locals", gdb.COMMAND_DATA, gdb.COMPLETE_NONE) @@ -1916,22 +1947,14 @@ class PyLocals(gdb.Command): pyop_frame = frame.get_pyop() if not pyop_frame: - print('Unable to read information on python frame') + print(UNABLE_READ_INFO_PYTHON_FRAME) return - namespace = self.get_namespace(pyop_frame) - namespace = [(name.proxyval(set()), val) for name, val in namespace] - - if namespace: - name, val = max(namespace, key=lambda item: len(item[0])) - max_name_length = len(name) - - for name, pyop_value in namespace: - value = pyop_value.get_truncated_repr(MAX_OUTPUT_LEN) - print('%-*s = %s' % (max_name_length, name, value)) - - def get_namespace(self, pyop_frame): - return pyop_frame.iter_locals() + for pyop_name, pyop_value in pyop_frame.iter_locals(): + print('%s = %s' % ( + pyop_name.proxyval(set()), + pyop_value.get_truncated_repr(MAX_OUTPUT_LEN), + )) PyLocals() @@ -1943,24 +1966,80 @@ PyLocals() import re import warnings import tempfile +import functools import textwrap import itertools +import traceback -class PyGlobals(PyLocals): + +def dont_suppress_errors(function): + "*sigh*, readline" + @functools.wraps(function) + def wrapper(*args, **kwargs): + try: + return function(*args, **kwargs) + except Exception: + traceback.print_exc() + raise + + return wrapper + +class PyGlobals(gdb.Command): 'List all the globals in the currently select Python frame' + def __init__(self): + gdb.Command.__init__ (self, + "py-globals", + gdb.COMMAND_DATA, + gdb.COMPLETE_NONE) + + @dont_suppress_errors + def invoke(self, args, from_tty): + name = str(args) + + frame = Frame.get_selected_python_frame() + if not frame: + print('Unable to locate python frame') + return + + pyop_frame = frame.get_pyop() + if not pyop_frame: + print(UNABLE_READ_INFO_PYTHON_FRAME) + return + + for pyop_name, pyop_value in pyop_frame.iter_locals(): + print('%s = %s' + % (pyop_name.proxyval(set()), + pyop_value.get_truncated_repr(MAX_OUTPUT_LEN))) def get_namespace(self, pyop_frame): return pyop_frame.iter_globals() -PyGlobals("py-globals") +PyGlobals() +# This function used to be a part of CPython's libpython.py (as a member function of frame). +# It isn't anymore, so I copied it. +def is_evalframeex(frame): + '''Is this a PyEval_EvalFrameEx frame?''' + if frame._gdbframe.name() == 'PyEval_EvalFrameEx': + ''' + I believe we also need to filter on the inline + struct frame_id.inline_depth, only regarding frames with + an inline depth of 0 as actually being this function + + So we reject those with type gdb.INLINE_FRAME + ''' + if frame._gdbframe.type() == gdb.NORMAL_FRAME: + # We have a PyEval_EvalFrameEx frame: + return True + + return False class PyNameEquals(gdb.Function): def _get_pycurframe_attr(self, attr): frame = Frame(gdb.selected_frame()) - if frame.is_evalframeex(): + if is_evalframeex(frame): pyframe = frame.get_pyop() if pyframe is None: warnings.warn("Use a Python debug build, Python breakpoints " @@ -1971,6 +2050,7 @@ class PyNameEquals(gdb.Function): return None + @dont_suppress_errors def invoke(self, funcname): attr = self._get_pycurframe_attr('co_name') return attr is not None and attr == funcname.string() @@ -1980,6 +2060,7 @@ PyNameEquals("pyname_equals") class PyModEquals(PyNameEquals): + @dont_suppress_errors def invoke(self, modname): attr = self._get_pycurframe_attr('co_filename') if attr is not None: @@ -2003,6 +2084,7 @@ class PyBreak(gdb.Command): py-break func """ + @dont_suppress_errors def invoke(self, funcname, from_tty): if '.' in funcname: modname, dot, funcname = funcname.rpartition('.') @@ -2457,6 +2539,7 @@ class PyStep(ExecutionControlCommandBase, PythonStepperMixin): stepinto = True + @dont_suppress_errors def invoke(self, args, from_tty): self.python_step(stepinto=self.stepinto) @@ -2470,18 +2553,18 @@ class PyNext(PyStep): class PyFinish(ExecutionControlCommandBase): "Execute until function returns to a caller." - invoke = ExecutionControlCommandBase.finish + invoke = dont_suppress_errors(ExecutionControlCommandBase.finish) class PyRun(ExecutionControlCommandBase): "Run the program." - invoke = ExecutionControlCommandBase.run + invoke = dont_suppress_errors(ExecutionControlCommandBase.run) class PyCont(ExecutionControlCommandBase): - invoke = ExecutionControlCommandBase.cont + invoke = dont_suppress_errors(ExecutionControlCommandBase.cont) def _pointervalue(gdbval): @@ -2574,7 +2657,7 @@ class PythonCodeExecutor(object): return pointer def free(self, pointer): - gdb.parse_and_eval("free((void *) %d)" % pointer) + gdb.parse_and_eval("(void) free((void *) %d)" % pointer) def incref(self, pointer): "Increment the reference count of a Python object in the inferior." @@ -2693,6 +2776,7 @@ class FixGdbCommand(gdb.Command): pass # warnings.resetwarnings() + @dont_suppress_errors def invoke(self, args, from_tty): self.fix_gdb() try: @@ -2726,7 +2810,10 @@ class PyExec(gdb.Command): lines = [] while True: try: - line = input('>') + if sys.version_info[0] == 2: + line = raw_input() + else: + line = input('>') except EOFError: break else: @@ -2737,6 +2824,7 @@ class PyExec(gdb.Command): return '\n'.join(lines), PythonCodeExecutor.Py_file_input + @dont_suppress_errors def invoke(self, expr, from_tty): expr, input_type = self.readcode(expr) executor = PythonCodeExecutor() diff --git a/Cython/Distutils/old_build_ext.py b/Cython/Distutils/old_build_ext.py index aa2a1cf22..7c2774aa2 100644 --- a/Cython/Distutils/old_build_ext.py +++ b/Cython/Distutils/old_build_ext.py @@ -26,17 +26,18 @@ except ImportError: def _check_stack(path): - try: - for frame in inspect.getouterframes(inspect.currentframe(), 0): - if path in frame[1].replace(os.sep, '/'): - return True - except Exception: - pass - return False + try: + for frame in inspect.getouterframes(inspect.currentframe(), 0): + if path in frame[1].replace(os.sep, '/'): + return True + except Exception: + pass + return False + if (not _check_stack('setuptools/extensions.py') - and not _check_stack('pyximport/pyxbuild.py') - and not _check_stack('Cython/Distutils/build_ext.py')): + and not _check_stack('pyximport/pyxbuild.py') + and not _check_stack('Cython/Distutils/build_ext.py')): warnings.warn( "Cython.Distutils.old_build_ext does not properly handle dependencies " "and is deprecated.") @@ -185,14 +186,11 @@ class old_build_ext(_build_ext.build_ext): _build_ext.build_ext.run(self) - def build_extensions(self): - # First, sanity-check the 'extensions' list - self.check_extensions_list(self.extensions) - + def check_extensions_list(self, extensions): + # Note: might get called multiple times. + _build_ext.build_ext.check_extensions_list(self, extensions) for ext in self.extensions: ext.sources = self.cython_sources(ext.sources, ext) - # Call original build_extensions - _build_ext.build_ext.build_extensions(self) def cython_sources(self, sources, extension): """ @@ -201,17 +199,6 @@ class old_build_ext(_build_ext.build_ext): found, and return a modified 'sources' list with Cython source files replaced by the generated C (or C++) files. """ - try: - from Cython.Compiler.Main \ - import CompilationOptions, \ - default_options as cython_default_options, \ - compile as cython_compile - from Cython.Compiler.Errors import PyrexError - except ImportError: - e = sys.exc_info()[1] - print("failed to import Cython: %s" % e) - raise DistutilsPlatformError("Cython does not appear to be installed") - new_sources = [] cython_sources = [] cython_targets = {} @@ -252,10 +239,10 @@ class old_build_ext(_build_ext.build_ext): # 2. Add in any (unique) paths from the extension # cython_include_dirs (if Cython.Distutils.extension is used). # 3. Add in any (unique) paths from the extension include_dirs - includes = self.cython_include_dirs + includes = list(self.cython_include_dirs) try: for i in extension.cython_include_dirs: - if not i in includes: + if i not in includes: includes.append(i) except AttributeError: pass @@ -264,19 +251,18 @@ class old_build_ext(_build_ext.build_ext): # result extension.include_dirs = list(extension.include_dirs) for i in extension.include_dirs: - if not i in includes: + if i not in includes: includes.append(i) # Set up Cython compiler directives: # 1. Start with the command line option. # 2. Add in any (unique) entries from the extension # cython_directives (if Cython.Distutils.extension is used). - directives = self.cython_directives + directives = dict(self.cython_directives) if hasattr(extension, "cython_directives"): directives.update(extension.cython_directives) - # Set the target_ext to '.c'. Cython will change this to '.cpp' if - # needed. + # Set the target file extension for C/C++ mode. if cplus: target_ext = '.cpp' else: @@ -314,6 +300,17 @@ class old_build_ext(_build_ext.build_ext): if not cython_sources: return new_sources + try: + from Cython.Compiler.Main \ + import CompilationOptions, \ + default_options as cython_default_options, \ + compile as cython_compile + from Cython.Compiler.Errors import PyrexError + except ImportError: + e = sys.exc_info()[1] + print("failed to import Cython: %s" % e) + raise DistutilsPlatformError("Cython does not appear to be installed") + module_name = extension.name for source in cython_sources: diff --git a/Cython/Includes/Deprecated/python.pxd b/Cython/Includes/Deprecated/python.pxd deleted file mode 100644 index 56236e925..000000000 --- a/Cython/Includes/Deprecated/python.pxd +++ /dev/null @@ -1,2 +0,0 @@ -# Present for backwards compatibility -from cpython cimport * diff --git a/Cython/Includes/Deprecated/python_bool.pxd b/Cython/Includes/Deprecated/python_bool.pxd deleted file mode 100644 index 9a6d253f4..000000000 --- a/Cython/Includes/Deprecated/python_bool.pxd +++ /dev/null @@ -1,2 +0,0 @@ -# Present for backwards compatibility -from cpython.bool cimport * diff --git a/Cython/Includes/Deprecated/python_buffer.pxd b/Cython/Includes/Deprecated/python_buffer.pxd deleted file mode 100644 index 2baeaae00..000000000 --- a/Cython/Includes/Deprecated/python_buffer.pxd +++ /dev/null @@ -1,2 +0,0 @@ -# Present for backwards compatibility -from cpython.buffer cimport * diff --git a/Cython/Includes/Deprecated/python_bytes.pxd b/Cython/Includes/Deprecated/python_bytes.pxd deleted file mode 100644 index 87af662de..000000000 --- a/Cython/Includes/Deprecated/python_bytes.pxd +++ /dev/null @@ -1,2 +0,0 @@ -# Present for backwards compatibility -from cpython.bytes cimport * diff --git a/Cython/Includes/Deprecated/python_cobject.pxd b/Cython/Includes/Deprecated/python_cobject.pxd deleted file mode 100644 index ed32c6b87..000000000 --- a/Cython/Includes/Deprecated/python_cobject.pxd +++ /dev/null @@ -1,2 +0,0 @@ -# Present for backwards compatibility -from cpython.cobject cimport * diff --git a/Cython/Includes/Deprecated/python_complex.pxd b/Cython/Includes/Deprecated/python_complex.pxd deleted file mode 100644 index 0a780b3b2..000000000 --- a/Cython/Includes/Deprecated/python_complex.pxd +++ /dev/null @@ -1,2 +0,0 @@ -# Present for backwards compatibility -from cpython.complex cimport * diff --git a/Cython/Includes/Deprecated/python_dict.pxd b/Cython/Includes/Deprecated/python_dict.pxd deleted file mode 100644 index 05b5f4796..000000000 --- a/Cython/Includes/Deprecated/python_dict.pxd +++ /dev/null @@ -1,2 +0,0 @@ -# Present for backwards compatibility -from cpython.dict cimport * diff --git a/Cython/Includes/Deprecated/python_exc.pxd b/Cython/Includes/Deprecated/python_exc.pxd deleted file mode 100644 index 6eb236bcc..000000000 --- a/Cython/Includes/Deprecated/python_exc.pxd +++ /dev/null @@ -1,2 +0,0 @@ -# Present for backwards compatibility -from cpython.exc cimport * diff --git a/Cython/Includes/Deprecated/python_float.pxd b/Cython/Includes/Deprecated/python_float.pxd deleted file mode 100644 index 7e133ef9b..000000000 --- a/Cython/Includes/Deprecated/python_float.pxd +++ /dev/null @@ -1,2 +0,0 @@ -# Present for backwards compatibility -from cpython.float cimport * diff --git a/Cython/Includes/Deprecated/python_function.pxd b/Cython/Includes/Deprecated/python_function.pxd deleted file mode 100644 index 1461c4e63..000000000 --- a/Cython/Includes/Deprecated/python_function.pxd +++ /dev/null @@ -1,2 +0,0 @@ -# Present for backwards compatibility -from cpython.function cimport * diff --git a/Cython/Includes/Deprecated/python_getargs.pxd b/Cython/Includes/Deprecated/python_getargs.pxd deleted file mode 100644 index 3852d6a6a..000000000 --- a/Cython/Includes/Deprecated/python_getargs.pxd +++ /dev/null @@ -1,2 +0,0 @@ -# Present for backwards compatibility -from cpython.getargs cimport * diff --git a/Cython/Includes/Deprecated/python_instance.pxd b/Cython/Includes/Deprecated/python_instance.pxd deleted file mode 100644 index 99cb5a909..000000000 --- a/Cython/Includes/Deprecated/python_instance.pxd +++ /dev/null @@ -1,2 +0,0 @@ -# Present for backwards compatibility -from cpython.instance cimport * diff --git a/Cython/Includes/Deprecated/python_int.pxd b/Cython/Includes/Deprecated/python_int.pxd deleted file mode 100644 index c1fd5178d..000000000 --- a/Cython/Includes/Deprecated/python_int.pxd +++ /dev/null @@ -1,2 +0,0 @@ -# Present for backwards compatibility -from cpython.int cimport * diff --git a/Cython/Includes/Deprecated/python_iterator.pxd b/Cython/Includes/Deprecated/python_iterator.pxd deleted file mode 100644 index e09aad279..000000000 --- a/Cython/Includes/Deprecated/python_iterator.pxd +++ /dev/null @@ -1,2 +0,0 @@ -# Present for backwards compatibility -from cpython.iterator cimport * diff --git a/Cython/Includes/Deprecated/python_list.pxd b/Cython/Includes/Deprecated/python_list.pxd deleted file mode 100644 index 64febcf96..000000000 --- a/Cython/Includes/Deprecated/python_list.pxd +++ /dev/null @@ -1,2 +0,0 @@ -# Present for backwards compatibility -from cpython.list cimport * diff --git a/Cython/Includes/Deprecated/python_long.pxd b/Cython/Includes/Deprecated/python_long.pxd deleted file mode 100644 index 1a24380c4..000000000 --- a/Cython/Includes/Deprecated/python_long.pxd +++ /dev/null @@ -1,2 +0,0 @@ -# Present for backwards compatibility -from cpython.long cimport * diff --git a/Cython/Includes/Deprecated/python_mapping.pxd b/Cython/Includes/Deprecated/python_mapping.pxd deleted file mode 100644 index cd01bee01..000000000 --- a/Cython/Includes/Deprecated/python_mapping.pxd +++ /dev/null @@ -1,2 +0,0 @@ -# Present for backwards compatibility -from cpython.mapping cimport * diff --git a/Cython/Includes/Deprecated/python_mem.pxd b/Cython/Includes/Deprecated/python_mem.pxd deleted file mode 100644 index d74429ea3..000000000 --- a/Cython/Includes/Deprecated/python_mem.pxd +++ /dev/null @@ -1,2 +0,0 @@ -# Present for backwards compatibility -from cpython.mem cimport * diff --git a/Cython/Includes/Deprecated/python_method.pxd b/Cython/Includes/Deprecated/python_method.pxd deleted file mode 100644 index e7da5154e..000000000 --- a/Cython/Includes/Deprecated/python_method.pxd +++ /dev/null @@ -1,2 +0,0 @@ -# Present for backwards compatibility -from cpython.method cimport * diff --git a/Cython/Includes/Deprecated/python_module.pxd b/Cython/Includes/Deprecated/python_module.pxd deleted file mode 100644 index 6310c0247..000000000 --- a/Cython/Includes/Deprecated/python_module.pxd +++ /dev/null @@ -1,2 +0,0 @@ -# Present for backwards compatibility -from cpython.module cimport * diff --git a/Cython/Includes/Deprecated/python_number.pxd b/Cython/Includes/Deprecated/python_number.pxd deleted file mode 100644 index ae67da1c3..000000000 --- a/Cython/Includes/Deprecated/python_number.pxd +++ /dev/null @@ -1,2 +0,0 @@ -# Present for backwards compatibility -from cpython.number cimport * diff --git a/Cython/Includes/Deprecated/python_object.pxd b/Cython/Includes/Deprecated/python_object.pxd deleted file mode 100644 index 3981bfa44..000000000 --- a/Cython/Includes/Deprecated/python_object.pxd +++ /dev/null @@ -1,2 +0,0 @@ -# Present for backwards compatibility -from cpython.object cimport * diff --git a/Cython/Includes/Deprecated/python_oldbuffer.pxd b/Cython/Includes/Deprecated/python_oldbuffer.pxd deleted file mode 100644 index e03e66a2e..000000000 --- a/Cython/Includes/Deprecated/python_oldbuffer.pxd +++ /dev/null @@ -1,2 +0,0 @@ -# Present for backwards compatibility -from cpython.oldbuffer cimport * diff --git a/Cython/Includes/Deprecated/python_pycapsule.pxd b/Cython/Includes/Deprecated/python_pycapsule.pxd deleted file mode 100644 index fe9cf8f8d..000000000 --- a/Cython/Includes/Deprecated/python_pycapsule.pxd +++ /dev/null @@ -1,2 +0,0 @@ -# Present for backwards compatibility -from cpython.pycapsule cimport * diff --git a/Cython/Includes/Deprecated/python_ref.pxd b/Cython/Includes/Deprecated/python_ref.pxd deleted file mode 100644 index 944741819..000000000 --- a/Cython/Includes/Deprecated/python_ref.pxd +++ /dev/null @@ -1,2 +0,0 @@ -# Present for backwards compatibility -from cpython.ref cimport * diff --git a/Cython/Includes/Deprecated/python_sequence.pxd b/Cython/Includes/Deprecated/python_sequence.pxd deleted file mode 100644 index fdef5b63e..000000000 --- a/Cython/Includes/Deprecated/python_sequence.pxd +++ /dev/null @@ -1,2 +0,0 @@ -# Present for backwards compatibility -from cpython.sequence cimport * diff --git a/Cython/Includes/Deprecated/python_set.pxd b/Cython/Includes/Deprecated/python_set.pxd deleted file mode 100644 index a2feb9371..000000000 --- a/Cython/Includes/Deprecated/python_set.pxd +++ /dev/null @@ -1,2 +0,0 @@ -# Present for backwards compatibility -from cpython.set cimport * diff --git a/Cython/Includes/Deprecated/python_string.pxd b/Cython/Includes/Deprecated/python_string.pxd deleted file mode 100644 index 24c818338..000000000 --- a/Cython/Includes/Deprecated/python_string.pxd +++ /dev/null @@ -1,2 +0,0 @@ -# Present for backwards compatibility -from cpython.string cimport * diff --git a/Cython/Includes/Deprecated/python_tuple.pxd b/Cython/Includes/Deprecated/python_tuple.pxd deleted file mode 100644 index 190713b02..000000000 --- a/Cython/Includes/Deprecated/python_tuple.pxd +++ /dev/null @@ -1,2 +0,0 @@ -# Present for backwards compatibility -from cpython.tuple cimport * diff --git a/Cython/Includes/Deprecated/python_type.pxd b/Cython/Includes/Deprecated/python_type.pxd deleted file mode 100644 index 3ac47d1b3..000000000 --- a/Cython/Includes/Deprecated/python_type.pxd +++ /dev/null @@ -1,2 +0,0 @@ -# Present for backwards compatibility -from cpython.type cimport * diff --git a/Cython/Includes/Deprecated/python_unicode.pxd b/Cython/Includes/Deprecated/python_unicode.pxd deleted file mode 100644 index 2b488b2dc..000000000 --- a/Cython/Includes/Deprecated/python_unicode.pxd +++ /dev/null @@ -1,2 +0,0 @@ -# Present for backwards compatibility -from cpython.unicode cimport * diff --git a/Cython/Includes/Deprecated/python_version.pxd b/Cython/Includes/Deprecated/python_version.pxd deleted file mode 100644 index c27ca4df9..000000000 --- a/Cython/Includes/Deprecated/python_version.pxd +++ /dev/null @@ -1,2 +0,0 @@ -# Present for backwards compatibility -from cpython.version cimport * diff --git a/Cython/Includes/Deprecated/python_weakref.pxd b/Cython/Includes/Deprecated/python_weakref.pxd deleted file mode 100644 index 1f84f1a17..000000000 --- a/Cython/Includes/Deprecated/python_weakref.pxd +++ /dev/null @@ -1,2 +0,0 @@ -# Present for backwards compatibility -from cpython.weakref cimport * diff --git a/Cython/Includes/Deprecated/stdio.pxd b/Cython/Includes/Deprecated/stdio.pxd deleted file mode 100644 index 41a4aebf1..000000000 --- a/Cython/Includes/Deprecated/stdio.pxd +++ /dev/null @@ -1,2 +0,0 @@ -# Present for backwards compatibility -from libc.stdio cimport * diff --git a/Cython/Includes/Deprecated/stdlib.pxd b/Cython/Includes/Deprecated/stdlib.pxd deleted file mode 100644 index 499511cde..000000000 --- a/Cython/Includes/Deprecated/stdlib.pxd +++ /dev/null @@ -1,2 +0,0 @@ -# Present for backwards compatibility -from libc.stdlib cimport * diff --git a/Cython/Includes/Deprecated/stl.pxd b/Cython/Includes/Deprecated/stl.pxd deleted file mode 100644 index 22248d265..000000000 --- a/Cython/Includes/Deprecated/stl.pxd +++ /dev/null @@ -1,91 +0,0 @@ -cdef extern from "<vector>" namespace std: - - cdef cppclass vector[TYPE]: - #constructors - __init__() - __init__(vector&) - __init__(int) - __init__(int, TYPE&) - __init__(iterator, iterator) - #operators - TYPE& __getitem__(int) - TYPE& __setitem__(int, TYPE&) - vector __new__(vector&) - bool __eq__(vector&, vector&) - bool __ne__(vector&, vector&) - bool __lt__(vector&, vector&) - bool __gt__(vector&, vector&) - bool __le__(vector&, vector&) - bool __ge__(vector&, vector&) - #others - void assign(int, TYPE) - #void assign(iterator, iterator) - TYPE& at(int) - TYPE& back() - iterator begin() - int capacity() - void clear() - bool empty() - iterator end() - iterator erase(iterator) - iterator erase(iterator, iterator) - TYPE& front() - iterator insert(iterator, TYPE&) - void insert(iterator, int, TYPE&) - void insert(iterator, iterator) - int max_size() - void pop_back() - void push_back(TYPE&) - iterator rbegin() - iterator rend() - void reserve(int) - void resize(int) - void resize(int, TYPE&) #void resize(size_type num, const TYPE& = TYPE()) - int size() - void swap(container&) - -cdef extern from "<deque>" namespace std: - - cdef cppclass deque[TYPE]: - #constructors - __init__() - __init__(deque&) - __init__(int) - __init__(int, TYPE&) - __init__(iterator, iterator) - #operators - TYPE& operator[]( size_type index ); - const TYPE& operator[]( size_type index ) const; - deque __new__(deque&); - bool __eq__(deque&, deque&); - bool __ne__(deque&, deque&); - bool __lt__(deque&, deque&); - bool __gt__(deque&, deque&); - bool __le__(deque&, deque&); - bool __ge__(deque&, deque&); - #others - void assign(int, TYPE&) - void assign(iterator, iterator) - TYPE& at(int) - TYPE& back() - iterator begin() - void clear() - bool empty() - iterator end() - iterator erase(iterator) - iterator erase(iterator, iterator) - TYPE& front() - iterator insert(iterator, TYPE&) - void insert(iterator, int, TYPE&) - void insert(iterator, iterator, iterator) - int max_size() - void pop_back() - void pop_front() - void push_back(TYPE&) - void push_front(TYPE&) - iterator rbegin() - iterator rend() - void resize(int) - void resize(int, TYPE&) - int size() - void swap(container&) diff --git a/Cython/Includes/cpython/array.pxd b/Cython/Includes/cpython/array.pxd index 19230a0a8..8431f7b66 100644 --- a/Cython/Includes/cpython/array.pxd +++ b/Cython/Includes/cpython/array.pxd @@ -46,8 +46,19 @@ : 2012-05-02 andreasvc : (see revision control) """ -from libc.string cimport strcat, strncat, \ - memset, memchr, memcmp, memcpy, memmove + +cdef extern from *: + """ + #if CYTHON_COMPILING_IN_PYPY + #ifdef _MSC_VER + #pragma message ("This module uses CPython specific internals of 'array.array', which are not available in PyPy.") + #else + #warning This module uses CPython specific internals of 'array.array', which are not available in PyPy. + #endif + #endif + """ + +from libc.string cimport memset, memcpy from cpython.object cimport Py_SIZE from cpython.ref cimport PyTypeObject, Py_TYPE diff --git a/Cython/Includes/cpython/bytes.pxd b/Cython/Includes/cpython/bytes.pxd index ea72c6aae..a7d1d09ac 100644 --- a/Cython/Includes/cpython/bytes.pxd +++ b/Cython/Includes/cpython/bytes.pxd @@ -68,6 +68,10 @@ cdef extern from "Python.h": # Return value: New reference. # Identical to PyBytes_FromFormat() except that it takes exactly two arguments. + bytes PyBytes_FromObject(object o) + # Return value: New reference. + # Return the bytes representation of object o that implements the buffer protocol. + Py_ssize_t PyBytes_Size(object string) except -1 # Return the length of the string in string object string. diff --git a/Cython/Includes/cpython/complex.pxd b/Cython/Includes/cpython/complex.pxd index f5ba33957..3fa145008 100644 --- a/Cython/Includes/cpython/complex.pxd +++ b/Cython/Includes/cpython/complex.pxd @@ -14,9 +14,14 @@ cdef extern from "Python.h": ctypedef class __builtin__.complex [object PyComplexObject]: cdef Py_complex cval - # not making these available to keep them read-only: - #cdef double imag "cval.imag" - #cdef double real "cval.real" + + @property + cdef inline double real(self): + return self.cval.real + + @property + cdef inline double imag(self): + return self.cval.imag # PyTypeObject PyComplex_Type # This instance of PyTypeObject represents the Python complex diff --git a/Cython/Includes/cpython/datetime.pxd b/Cython/Includes/cpython/datetime.pxd index cd0f90719..83fa20ab7 100644 --- a/Cython/Includes/cpython/datetime.pxd +++ b/Cython/Includes/cpython/datetime.pxd @@ -5,6 +5,17 @@ cdef extern from "Python.h": pass cdef extern from "datetime.h": + """ + #if PY_MAJOR_VERSION < 3 && !defined(PyDateTime_DELTA_GET_DAYS) + #define PyDateTime_DELTA_GET_DAYS(o) (((PyDateTime_Delta*)o)->days) + #endif + #if PY_MAJOR_VERSION < 3 && !defined(PyDateTime_DELTA_GET_SECONDS) + #define PyDateTime_DELTA_GET_SECONDS(o) (((PyDateTime_Delta*)o)->seconds) + #endif + #if PY_MAJOR_VERSION < 3 && !defined(PyDateTime_DELTA_GET_MICROSECONDS) + #define PyDateTime_DELTA_GET_MICROSECONDS(o) (((PyDateTime_Delta*)o)->microseconds) + #endif + """ ctypedef extern class datetime.date[object PyDateTime_Date]: pass @@ -13,7 +24,33 @@ cdef extern from "datetime.h": pass ctypedef extern class datetime.datetime[object PyDateTime_DateTime]: - pass + @property + cdef inline int year(self): + return PyDateTime_GET_YEAR(self) + + @property + cdef inline int month(self): + return PyDateTime_GET_MONTH(self) + + @property + cdef inline int day(self): + return PyDateTime_GET_DAY(self) + + @property + cdef inline int hour(self): + return PyDateTime_DATE_GET_HOUR(self) + + @property + cdef inline int minute(self): + return PyDateTime_DATE_GET_MINUTE(self) + + @property + cdef inline int second(self): + return PyDateTime_DATE_GET_SECOND(self) + + @property + cdef inline int microsecond(self): + return PyDateTime_DATE_GET_MICROSECOND(self) ctypedef extern class datetime.timedelta[object PyDateTime_Delta]: pass @@ -210,3 +247,14 @@ cdef inline int timedelta_seconds(object o): # Get microseconds of timedelta cdef inline int timedelta_microseconds(object o): return (<PyDateTime_Delta*>o).microseconds + +cdef inline double total_seconds(timedelta obj): + # Mirrors the "timedelta.total_seconds()" method. + # Note that this implementation is not guaranteed to give *exactly* the same + # result as the original method, due to potential differences in floating point rounding. + cdef: + double days, seconds, micros + days = <double>PyDateTime_DELTA_GET_DAYS(obj) + seconds = <double>PyDateTime_DELTA_GET_SECONDS(obj) + micros = <double>PyDateTime_DELTA_GET_MICROSECONDS(obj) + return days * 24 * 3600 + seconds + micros / 1_000_000 diff --git a/Cython/Includes/cpython/descr.pxd b/Cython/Includes/cpython/descr.pxd new file mode 100644 index 000000000..5075f0bbd --- /dev/null +++ b/Cython/Includes/cpython/descr.pxd @@ -0,0 +1,26 @@ +from .object cimport PyObject, PyTypeObject + +cdef extern from "Python.h": + ctypedef object (*wrapperfunc)(self, args, void* wrapped) + ctypedef object (*wrapperfunc_kwds)(self, args, void* wrapped, kwds) + + struct wrapperbase: + char* name + int offset + void* function + wrapperfunc wrapper + char* doc + int flags + PyObject* name_strobj + + int PyWrapperFlag_KEYWORDS + + ctypedef class __builtin__.wrapper_descriptor [object PyWrapperDescrObject]: + cdef type d_type + cdef d_name + cdef wrapperbase* d_base + cdef void* d_wrapped + + object PyDescr_NewWrapper(PyTypeObject* cls, wrapperbase* wrapper, void* wrapped) + + int PyDescr_IsData(descr) diff --git a/Cython/Includes/cpython/dict.pxd b/Cython/Includes/cpython/dict.pxd index 16dd5e145..979dd392a 100644 --- a/Cython/Includes/cpython/dict.pxd +++ b/Cython/Includes/cpython/dict.pxd @@ -1,6 +1,13 @@ from .object cimport PyObject +from .pyport cimport uint64_t cdef extern from "Python.h": + # On Python 2, PyDict_GetItemWithError is called _PyDict_GetItemWithError + """ + #if PY_MAJOR_VERSION <= 2 + #define PyDict_GetItemWithError _PyDict_GetItemWithError + #endif + """ ############################################################################ # 7.4.1 Dictionary Objects @@ -72,11 +79,25 @@ cdef extern from "Python.h": # NULL if the key key is not present, but without setting an # exception. + PyObject* PyDict_GetItemWithError(object p, object key) except? NULL + # Return value: Borrowed reference. + # Variant of PyDict_GetItem() that does not suppress exceptions. Return + # NULL with an exception set if an exception occurred. Return NULL + # without an exception set if the key wasn’t present. + PyObject* PyDict_GetItemString(object p, const char *key) # Return value: Borrowed reference. # This is the same as PyDict_GetItem(), but key is specified as a # char*, rather than a PyObject*. + PyObject* PyDict_SetDefault(object p, object key, object default) except NULL + # Return value: Borrowed reference. + # This is the same as the Python-level dict.setdefault(). If present, it + # returns the value corresponding to key from the dictionary p. If the key + # is not in the dict, it is inserted with value defaultobj and defaultobj + # is returned. This function evaluates the hash function of key only once, + # instead of evaluating it independently for the lookup and the insertion. + list PyDict_Items(object p) # Return value: New reference. # Return a PyListObject containing all the items from the diff --git a/Cython/Includes/cpython/fileobject.pxd b/Cython/Includes/cpython/fileobject.pxd new file mode 100644 index 000000000..e52cd33f5 --- /dev/null +++ b/Cython/Includes/cpython/fileobject.pxd @@ -0,0 +1,57 @@ +""" +From https://docs.python.org/3.9/c-api/file.html + +These APIs are a minimal emulation of the Python 2 C API for built-in file objects, +which used to rely on the buffered I/O (FILE*) support from the C standard library. +In Python 3, files and streams use the new io module, which defines several layers +over the low-level unbuffered I/O of the operating system. The functions described +below are convenience C wrappers over these new APIs, and meant mostly for internal +error reporting in the interpreter; + +third-party code is advised to access the io APIs instead. +""" + +cdef extern from "Python.h": + + ########################################################################### + # File Objects + ########################################################################### + + object PyFile_FromFd(int fd, const char *name, const char *mode, int buffering, + const char *encoding, const char *errors, const char *newline, int closefd) + # Return value: New reference. + # Create a Python file object from the file descriptor of an already + # opened file fd. The arguments name, encoding, errors and newline can be + # NULL to use the defaults; buffering can be -1 to use the default. name + # is ignored and kept for backward compatibility. Return NULL on failure. + # For a more comprehensive description of the arguments, please refer to + # the io.open() function documentation. + + # Warning: Since Python streams have their own buffering layer, mixing + # them with OS-level file descriptors can produce various issues (such as + # unexpected ordering of data). + + # Changed in version 3.2: Ignore name attribute. + + object PyFile_GetLine(object p, int n) + # Return value: New reference. + # Equivalent to p.readline([n]), this function reads one line from the + # object p. p may be a file object or any object with a readline() + # method. If n is 0, exactly one line is read, regardless of the length of + # the line. If n is greater than 0, no more than n bytes will be read from + # the file; a partial line can be returned. In both cases, an empty string + # is returned if the end of the file is reached immediately. If n is less + # than 0, however, one line is read regardless of length, but EOFError is + # raised if the end of the file is reached immediately. + + int PyFile_WriteObject(object obj, object p, int flags) except? -1 + # Write object obj to file object p. The only supported flag for flags + # is Py_PRINT_RAW; if given, the str() of the object is written instead of + # the repr(). Return 0 on success or -1 on failure; the appropriate + # exception will be set. + + int PyFile_WriteString(const char *s, object p) except? -1 + # Write string s to file object p. Return 0 on success or -1 on failure; + # the appropriate exception will be set. + + enum: Py_PRINT_RAW diff --git a/Cython/Includes/cpython/list.pxd b/Cython/Includes/cpython/list.pxd index c6a29535c..8ee9c891c 100644 --- a/Cython/Includes/cpython/list.pxd +++ b/Cython/Includes/cpython/list.pxd @@ -42,17 +42,19 @@ cdef extern from "Python.h": int PyList_SetItem(object list, Py_ssize_t index, object item) except -1 # Set the item at index index in list to item. Return 0 on success - # or -1 on failure. Note: This function ``steals'' a reference to - # item and discards a reference to an item already in the list at - # the affected position. + # or -1 on failure. + # + # WARNING: This function _steals_ a reference to item and discards a + # reference to an item already in the list at the affected position. void PyList_SET_ITEM(object list, Py_ssize_t i, object o) # Macro form of PyList_SetItem() without error checking. This is # normally only used to fill in new lists where there is no - # previous content. Note: This function ``steals'' a reference to - # item, and, unlike PyList_SetItem(), does not discard a reference - # to any item that it being replaced; any reference in list at - # position i will be *leaked*. + # previous content. + # + # WARNING: This function _steals_ a reference to item, and, unlike + # PyList_SetItem(), does not discard a reference to any item that + # it being replaced; any reference in list at position i will be *leaked*. int PyList_Insert(object list, Py_ssize_t index, object item) except -1 # Insert the item item into list list in front of index diff --git a/Cython/Includes/cpython/long.pxd b/Cython/Includes/cpython/long.pxd index eb8140d41..f65cd0073 100644 --- a/Cython/Includes/cpython/long.pxd +++ b/Cython/Includes/cpython/long.pxd @@ -89,7 +89,7 @@ cdef extern from "Python.h": # Return a C long representation of the contents of pylong. If # pylong is greater than LONG_MAX, an OverflowError is raised. - # long PyLong_AsLongAndOverflow(object pylong, int *overflow) except? -1 + long PyLong_AsLongAndOverflow(object pylong, int *overflow) except? -1 # Return a C long representation of the contents of pylong. If pylong is # greater than LONG_MAX or less than LONG_MIN, set *overflow to 1 or -1, # respectively, and return -1; otherwise, set *overflow to 0. If any other @@ -97,7 +97,7 @@ cdef extern from "Python.h": # be returned and *overflow will be 0. # New in version 2.7. - # PY_LONG_LONG PyLong_AsLongLongAndOverflow(object pylong, int *overflow) except? -1 + PY_LONG_LONG PyLong_AsLongLongAndOverflow(object pylong, int *overflow) except? -1 # Return a C long long representation of the contents of pylong. If pylong # is greater than PY_LLONG_MAX or less than PY_LLONG_MIN, set *overflow to # 1 or -1, respectively, and return -1; otherwise, set *overflow to 0. If diff --git a/Cython/Includes/cpython/marshal.pxd b/Cython/Includes/cpython/marshal.pxd new file mode 100644 index 000000000..6b6d6f480 --- /dev/null +++ b/Cython/Includes/cpython/marshal.pxd @@ -0,0 +1,66 @@ +from libc.stdio cimport FILE + +cdef extern from "Python.h": + + ########################################################################### + # Data marshalling support + ########################################################################### + + const int Py_MARSHAL_VERSION + + void PyMarshal_WriteLongToFile(long value, FILE *file, int version) + # Marshal a long integer, value, to file. This will only write the + # least-significant 32 bits of value, regardless of the size of the native + # long type. version indicates the file format. + + void PyMarshal_WriteObjectToFile(object value, FILE *file, int version) + # Marshal a Python object, value, to file. version indicates the file + # format. + + bytes PyMarshal_WriteObjectToString(object value, int version) + # Return value: New reference. + # Return a bytes object containing the marshalled representation of value. + # version indicates the file format. + + long PyMarshal_ReadLongFromFile(FILE *file) except? -1 + # Return a C long from the data stream in a FILE* opened for reading. Only + # a 32-bit value can be read in using this function, regardless of the + # native size of long. + + # On error, sets the appropriate exception (EOFError) and returns -1. + + int PyMarshal_ReadShortFromFile(FILE *file) except? -1 + # Return a C short from the data stream in a FILE* opened for reading. Only + # a 16-bit value can be read in using this function, regardless of the + # native size of short. + + # On error, sets the appropriate exception (EOFError) and returns -1. + + object PyMarshal_ReadObjectFromFile(FILE *file) + # Return value: New reference. + # Return a Python object from the data stream in a FILE* opened for + # reading. + + # On error, sets the appropriate exception (EOFError, ValueError or + # TypeError) and returns NULL. + + object PyMarshal_ReadLastObjectFromFile(FILE *file) + # Return value: New reference. + # Return a Python object from the data stream in a FILE* opened for + # reading. Unlike PyMarshal_ReadObjectFromFile(), this function assumes + # that no further objects will be read from the file, allowing it to + # aggressively load file data into memory so that the de-serialization can + # operate from data in memory, rather than reading a byte at a time from the + # file. Only use these variant if you are certain that you won’t be reading + # anything else from the file. + + # On error, sets the appropriate exception (EOFError, ValueError or + # TypeError) and returns NULL. + + object PyMarshal_ReadObjectFromString(const char *data, Py_ssize_t len) + # Return value: New reference. + # Return a Python object from the data stream in a byte buffer containing + # len bytes pointed to by data. + + # On error, sets the appropriate exception (EOFError, ValueError or + # TypeError) and returns NULL. diff --git a/Cython/Includes/cpython/module.pxd b/Cython/Includes/cpython/module.pxd index 8eb323b01..ea4c3817e 100644 --- a/Cython/Includes/cpython/module.pxd +++ b/Cython/Includes/cpython/module.pxd @@ -145,6 +145,12 @@ cdef extern from "Python.h": bint PyModule_CheckExact(object p) # Return true if p is a module object, but not a subtype of PyModule_Type. + object PyModule_NewObject(object name) + # Return a new module object with the __name__ attribute set to name. + # The module’s __name__, __doc__, __package__, and __loader__ + # attributes are filled in (all but __name__ are set to None); the caller + # is responsible for providing a __file__ attribute. + object PyModule_New(const char *name) # Return value: New reference. # Return a new module object with the __name__ attribute set to @@ -160,21 +166,35 @@ cdef extern from "Python.h": # use other PyModule_*() and PyObject_*() functions rather than # directly manipulate a module's __dict__. + object PyModule_GetNameObject(object module) + # Return module’s __name__ value. If the module does not provide one, or if + # it is not a string, SystemError is raised and NULL is returned. + char* PyModule_GetName(object module) except NULL - # Return module's __name__ value. If the module does not provide - # one, or if it is not a string, SystemError is raised and NULL is - # returned. + # Similar to PyModule_GetNameObject() but return the name encoded + # to 'utf-8'. + + void* PyModule_GetState(object module) + # Return the “state” of the module, that is, a pointer to the block of + # memory allocated at module creation time, or NULL. + # See PyModuleDef.m_size. + + object PyModule_GetFilenameObject(object module) + # Return the name of the file from which module was loaded using module’s + # __file__ attribute. If this is not defined, or if it is not a unicode + # string, raise SystemError and return NULL; otherwise return a reference + # to a Unicode object. char* PyModule_GetFilename(object module) except NULL - # Return the name of the file from which module was loaded using - # module's __file__ attribute. If this is not defined, or if it is - # not a string, raise SystemError and return NULL. + # Similar to PyModule_GetFilenameObject() but return the filename encoded + # to ‘utf-8’. int PyModule_AddObject(object module, const char *name, object value) except -1 # Add an object to module as name. This is a convenience function - # which can be used from the module's initialization - # function. This steals a reference to value. Return -1 on error, - # 0 on success. + # which can be used from the module's initialization function. + # Return -1 on error, 0 on success. + # + # WARNING: This _steals_ a reference to value. int PyModule_AddIntConstant(object module, const char *name, long value) except -1 # Add an integer constant to module as name. This convenience diff --git a/Cython/Includes/cpython/object.pxd b/Cython/Includes/cpython/object.pxd index 5a8116639..aa625523d 100644 --- a/Cython/Includes/cpython/object.pxd +++ b/Cython/Includes/cpython/object.pxd @@ -45,6 +45,8 @@ cdef extern from "Python.h": newfunc tp_new destructor tp_dealloc + destructor tp_del + destructor tp_finalize traverseproc tp_traverse inquiry tp_clear freefunc tp_free @@ -63,6 +65,8 @@ cdef extern from "Python.h": descrgetfunc tp_descr_get descrsetfunc tp_descr_set + unsigned int tp_version_tag + ctypedef struct PyObject: Py_ssize_t ob_refcnt PyTypeObject *ob_type @@ -128,6 +132,17 @@ cdef extern from "Python.h": # failure. This is the equivalent of the Python statement "del # o.attr_name". + object PyObject_GenericGetDict(object o, void *context) + # Return value: New reference. + # A generic implementation for the getter of a __dict__ descriptor. It + # creates the dictionary if necessary. + # New in version 3.3. + + int PyObject_GenericSetDict(object o, object value, void *context) except -1 + # A generic implementation for the setter of a __dict__ descriptor. This + # implementation does not allow the dictionary to be deleted. + # New in version 3.3. + int Py_LT, Py_LE, Py_EQ, Py_NE, Py_GT, Py_GE object PyObject_RichCompare(object o1, object o2, int opid) @@ -177,6 +192,14 @@ cdef extern from "Python.h": # equivalent of the Python expression "str(o)". Called by the # str() built-in function and by the print statement. + object PyObject_Bytes(object o) + # Return value: New reference. + # Compute a bytes representation of object o. Return NULL on + # failure and a bytes object on success. This is equivalent to + # the Python expression bytes(o), when o is not an integer. + # Unlike bytes(o), a TypeError is raised when o is an integer + # instead of a zero-initialized bytes object. + object PyObject_Unicode(object o) # Return value: New reference. # Compute a Unicode string representation of object o. Returns the @@ -320,6 +343,13 @@ cdef extern from "Python.h": # returned. On error, -1 is returned. This is the equivalent to # the Python expression "len(o)". + Py_ssize_t PyObject_LengthHint(object o, Py_ssize_t default) except -1 + # Return an estimated length for the object o. First try to return its + # actual length, then an estimate using __length_hint__(), and finally + # return the default value. On error, return -1. This is the equivalent to + # the Python expression "operator.length_hint(o, default)". + # New in version 3.4. + object PyObject_GetItem(object o, object key) # Return value: New reference. # Return element of o corresponding to the object key or NULL on @@ -397,3 +427,4 @@ cdef extern from "Python.h": long Py_TPFLAGS_DEFAULT_EXTERNAL long Py_TPFLAGS_DEFAULT_CORE long Py_TPFLAGS_DEFAULT + long Py_TPFLAGS_HAVE_FINALIZE diff --git a/Cython/Includes/cpython/pyport.pxd b/Cython/Includes/cpython/pyport.pxd new file mode 100644 index 000000000..fec59c9c8 --- /dev/null +++ b/Cython/Includes/cpython/pyport.pxd @@ -0,0 +1,8 @@ +cdef extern from "Python.h": + ctypedef int int32_t + ctypedef int int64_t + ctypedef unsigned int uint32_t + ctypedef unsigned int uint64_t + + const Py_ssize_t PY_SSIZE_T_MIN + const Py_ssize_t PY_SSIZE_T_MAX diff --git a/Cython/Includes/cpython/pystate.pxd b/Cython/Includes/cpython/pystate.pxd index 1af630793..ee8856b20 100644 --- a/Cython/Includes/cpython/pystate.pxd +++ b/Cython/Includes/cpython/pystate.pxd @@ -84,6 +84,9 @@ cdef extern from "Python.h": # PyGILState_Release on the same thread. void PyGILState_Release(PyGILState_STATE) + # Return 1 if the current thread holds the GIL and 0 otherwise. + int PyGILState_Check() + # Routines for advanced debuggers, requested by David Beazley. # Don't use unless you know what you are doing! PyInterpreterState * PyInterpreterState_Head() diff --git a/Cython/Includes/cpython/time.pxd b/Cython/Includes/cpython/time.pxd new file mode 100644 index 000000000..076abd931 --- /dev/null +++ b/Cython/Includes/cpython/time.pxd @@ -0,0 +1,51 @@ +""" +Cython implementation of (parts of) the standard library time module. +""" + +from libc.stdint cimport int64_t +from cpython.exc cimport PyErr_SetFromErrno + +cdef extern from "Python.h": + ctypedef int64_t _PyTime_t + _PyTime_t _PyTime_GetSystemClock() nogil + double _PyTime_AsSecondsDouble(_PyTime_t t) nogil + +from libc.time cimport ( + tm, + time_t, + localtime as libc_localtime, +) + + +cdef inline double time() nogil: + cdef: + _PyTime_t tic + + tic = _PyTime_GetSystemClock() + return _PyTime_AsSecondsDouble(tic) + + +cdef inline int _raise_from_errno() except -1 with gil: + PyErr_SetFromErrno(RuntimeError) + return <int> -1 # Let the C compiler know that this function always raises. + + +cdef inline tm localtime() nogil except *: + """ + Analogue to the stdlib time.localtime. The returned struct + has some entries that the stdlib version does not: tm_gmtoff, tm_zone + """ + cdef: + time_t tic = <time_t>time() + tm* result + + result = libc_localtime(&tic) + if result is NULL: + _raise_from_errno() + # Fix 0-based date values (and the 1900-based year). + # See tmtotuple() in https://github.com/python/cpython/blob/master/Modules/timemodule.c + result.tm_year += 1900 + result.tm_mon += 1 + result.tm_wday = (result.tm_wday + 6) % 7 + result.tm_yday += 1 + return result[0] diff --git a/Cython/Includes/cpython/tuple.pxd b/Cython/Includes/cpython/tuple.pxd index 09c46e0b4..b0d718047 100644 --- a/Cython/Includes/cpython/tuple.pxd +++ b/Cython/Includes/cpython/tuple.pxd @@ -47,13 +47,15 @@ cdef extern from "Python.h": int PyTuple_SetItem(object p, Py_ssize_t pos, object o) except -1 # Insert a reference to object o at position pos of the tuple - # pointed to by p. Return 0 on success. Note: This function - # ``steals'' a reference to o. + # pointed to by p. Return 0 on success. + # + # WARNING: This function _steals_ a reference to o. void PyTuple_SET_ITEM(object p, Py_ssize_t pos, object o) # Like PyTuple_SetItem(), but does no error checking, and should - # only be used to fill in brand new tuples. Note: This function - # ``steals'' a reference to o. + # only be used to fill in brand new tuples. + # + # WARNING: This function _steals_ a reference to o. int _PyTuple_Resize(PyObject **p, Py_ssize_t newsize) except -1 # Can be used to resize a tuple. newsize will be the new length of diff --git a/Cython/Includes/cpython/type.pxd b/Cython/Includes/cpython/type.pxd index a1d094e37..928a748cd 100644 --- a/Cython/Includes/cpython/type.pxd +++ b/Cython/Includes/cpython/type.pxd @@ -23,6 +23,11 @@ cdef extern from "Python.h": # of the standard type object. Return false in all other # cases. + void PyType_Modified(type type) + # Invalidate the internal lookup cache for the type and all of its + # subtypes. This function must be called after any manual modification + # of the attributes or base classes of the type. + bint PyType_HasFeature(object o, int feature) # Return true if the type object o sets the feature feature. Type # features are denoted by single bit flags. diff --git a/Cython/Includes/cpython/unicode.pxd b/Cython/Includes/cpython/unicode.pxd index 061be0905..63dde4363 100644 --- a/Cython/Includes/cpython/unicode.pxd +++ b/Cython/Includes/cpython/unicode.pxd @@ -103,6 +103,14 @@ cdef extern from *: # when u is NULL. unicode PyUnicode_FromUnicode(Py_UNICODE *u, Py_ssize_t size) + # Similar to PyUnicode_FromUnicode(), but u points to UTF-8 encoded + # bytes + unicode PyUnicode_FromStringAndSize(const char *u, Py_ssize_t size) + + # Similar to PyUnicode_FromUnicode(), but u points to null-terminated + # UTF-8 encoded bytes. The size is determined with strlen(). + unicode PyUnicode_FromString(const char *u) + # Create a Unicode Object from the given Unicode code point ordinal. # # The ordinal must be in range(0x10000) on narrow Python builds diff --git a/Cython/Includes/libc/complex.pxd b/Cython/Includes/libc/complex.pxd new file mode 100644 index 000000000..7cd740cb8 --- /dev/null +++ b/Cython/Includes/libc/complex.pxd @@ -0,0 +1,35 @@ +cdef extern from "<complex.h>" nogil: + # Trigonometric functions. + double complex cacos(double complex z) + double complex casin(double complex z) + double complex catan(double complex z) + double complex ccos(double complex z) + double complex csin(double complex z) + double complex ctan(double complex z) + + # Hyperbolic functions. + double complex cacosh(double complex z) + double complex casinh(double complex z) + double complex catanh(double complex z) + double complex ccosh(double complex z) + double complex csinh(double complex z) + double complex ctanh(double complex z) + + # Exponential and logarithmic functions. + double complex cexp(double complex z) + double complex clog(double complex z) + double complex clog10(double complex z) + + # Power functions. + double complex cpow(double complex x, double complex y) + double complex csqrt(double complex z) + + # Absolute value, conjugates, and projection. + double cabs(double complex z) + double carg(double complex z) + double complex conj(double complex z) + double complex cproj(double complex z) + + # Decomposing complex values. + double cimag(double complex z) + double creal(double complex z) diff --git a/Cython/Includes/libc/math.pxd b/Cython/Includes/libc/math.pxd index b002670b2..4a9858c2a 100644 --- a/Cython/Includes/libc/math.pxd +++ b/Cython/Includes/libc/math.pxd @@ -23,81 +23,178 @@ cdef extern from "<math.h>" nogil: const float HUGE_VALF const long double HUGE_VALL + # All C99 functions in alphabetical order double acos(double x) + float acosf(float) + double acosh(double x) + float acoshf(float) + long double acoshl(long double) + long double acosl(long double) double asin(double x) + float asinf(float) + double asinh(double x) + float asinhf(float) + long double asinhl(long double) + long double asinl(long double) double atan(double x) double atan2(double y, double x) - double cos(double x) - double sin(double x) - double tan(double x) - - double cosh(double x) - double sinh(double x) - double tanh(double x) - double acosh(double x) - double asinh(double x) + float atan2f(float, float) + long double atan2l(long double, long double) + float atanf(float) double atanh(double x) - - double hypot(double x, double y) - - double exp(double x) - double exp2(double x) - double expm1(double x) - double log(double x) - double logb(double x) - double log2(double x) - double log10(double x) - double log1p(double x) - int ilogb(double x) - - double lgamma(double x) - double tgamma(double x) - - double frexp(double x, int* exponent) - double ldexp(double x, int exponent) - - double modf(double x, double* iptr) - double fmod(double x, double y) - double remainder(double x, double y) - double remquo(double x, double y, int *quot) - double pow(double x, double y) - double sqrt(double x) + float atanhf(float) + long double atanhl(long double) + long double atanl(long double) double cbrt(double x) - - double fabs(double x) + float cbrtf(float) + long double cbrtl(long double) double ceil(double x) - double floor(double x) - double trunc(double x) - double rint(double x) - double round(double x) - double nearbyint(double x) - double nextafter(double, double) - double nexttoward(double, long double) - - long long llrint(double) - long lrint(double) - long long llround(double) - long lround(double) - + float ceilf(float) + long double ceill(long double) double copysign(double, double) float copysignf(float, float) long double copysignl(long double, long double) - + double cos(double x) + float cosf(float) + double cosh(double x) + float coshf(float) + long double coshl(long double) + long double cosl(long double) double erf(double) - float erff(float) - long double erfl(long double) double erfc(double) float erfcf(float) long double erfcl(long double) - + float erff(float) + long double erfl(long double) + double exp(double x) + double exp2(double x) + float exp2f(float) + long double exp2l(long double) + float expf(float) + long double expl(long double) + double expm1(double x) + float expm1f(float) + long double expm1l(long double) + double fabs(double x) + float fabsf(float) + long double fabsl(long double) double fdim(double x, double y) + float fdimf(float, float) + long double fdiml(long double, long double) + double floor(double x) + float floorf(float) + long double floorl(long double) double fma(double x, double y, double z) + float fmaf(float, float, float) + long double fmal(long double, long double, long double) double fmax(double x, double y) + float fmaxf(float, float) + long double fmaxl(long double, long double) double fmin(double x, double y) + float fminf(float, float) + long double fminl(long double, long double) + double fmod(double x, double y) + float fmodf(float, float) + long double fmodl(long double, long double) + double frexp(double x, int* exponent) + float frexpf(float, int* exponent) + long double frexpl(long double, int*) + double hypot(double x, double y) + float hypotf(float, float) + long double hypotl(long double, long double) + int ilogb(double x) + int ilogbf(float) + int ilogbl(long double) + double ldexp(double x, int exponent) + float ldexpf(float, int exponent) + long double ldexpl(long double, int exponent) + double lgamma(double x) + float lgammaf(float) + long double lgammal(long double) + long long llrint(double) + long long llrintf(float) + long long llrintl(long double) + long long llround(double) + long long llroundf(float) + long long llroundl(long double) + double log(double x) + double log10(double x) + float log10f(float) + long double log10l(long double) + double log1p(double x) + float log1pf(float) + long double log1pl(long double) + double log2(double x) + float log2f(float) + long double log2l(long double) + double logb(double x) + float logbf(float) + long double logbl(long double) + float logf(float) + long double logl(long double) + long lrint(double) + long lrintf(float) + long lrintl(long double) + long lround(double) + long lroundf(float) + long lroundl(long double) + double modf(double x, double* iptr) + float modff(float, float* iptr) + long double modfl(long double, long double* iptr) + double nan(const char*) + float nanf(const char*) + long double nanl(const char*) + double nearbyint(double x) + float nearbyintf(float) + long double nearbyintl(long double) + double nextafter(double, double) + float nextafterf(float, float) + long double nextafterl(long double, long double) + double nexttoward(double, long double) + float nexttowardf(float, long double) + long double nexttowardl(long double, long double) + double pow(double x, double y) + float powf(float, float) + long double powl(long double, long double) + double remainder(double x, double y) + float remainderf(float, float) + long double remainderl(long double, long double) + double remquo(double x, double y, int* quot) + float remquof(float, float, int* quot) + long double remquol(long double, long double, int* quot) + double rint(double x) + float rintf(float) + long double rintl(long double) + double round(double x) + float roundf(float) + long double roundl(long double) double scalbln(double x, long n) + float scalblnf(float, long) + long double scalblnl(long double, long) double scalbn(double x, int n) - - double nan(const char*) + float scalbnf(float, int) + long double scalbnl(long double, int) + double sin(double x) + float sinf(float) + double sinh(double x) + float sinhf(float) + long double sinhl(long double) + long double sinl(long double) + double sqrt(double x) + float sqrtf(float) + long double sqrtl(long double) + double tan(double x) + float tanf(float) + double tanh(double x) + float tanhf(float) + long double tanhl(long double) + long double tanl(long double) + double tgamma(double x) + float tgammaf(float) + long double tgammal(long double) + double trunc(double x) + float truncf(float) + long double truncl(long double) int isinf(long double) # -1 / 0 / 1 bint isfinite(long double) diff --git a/Cython/Includes/libc/time.pxd b/Cython/Includes/libc/time.pxd index 3aa15a2ee..318212eea 100644 --- a/Cython/Includes/libc/time.pxd +++ b/Cython/Includes/libc/time.pxd @@ -1,4 +1,4 @@ -# http://en.wikipedia.org/wiki/C_date_and_time_functions +# https://en.wikipedia.org/wiki/C_date_and_time_functions from libc.stddef cimport wchar_t @@ -20,8 +20,9 @@ cdef extern from "<time.h>" nogil: int tm_wday int tm_yday int tm_isdst - char *tm_zone - long tm_gmtoff + # GNU specific extensions + #char *tm_zone + #long tm_gmtoff int daylight # global state long timezone diff --git a/Cython/Includes/libcpp/algorithm.pxd b/Cython/Includes/libcpp/algorithm.pxd index ec7c3835b..c07bae139 100644 --- a/Cython/Includes/libcpp/algorithm.pxd +++ b/Cython/Includes/libcpp/algorithm.pxd @@ -1,43 +1,155 @@ from libcpp cimport bool +from libcpp.utility cimport pair +from libc.stddef import ptrdiff_t cdef extern from "<algorithm>" namespace "std" nogil: - # Sorting and searching - bool binary_search[Iter, T](Iter first, Iter last, const T& value) - bool binary_search[Iter, T, Compare](Iter first, Iter last, const T& value, - Compare comp) + # Non-modifying sequence operations + bool all_of[Iter, Pred](Iter first, Iter last, Pred pred) except + + bool any_of[Iter, Pred](Iter first, Iter last, Pred pred) except + + bool none_of[Iter, Pred](Iter first, Iter last, Pred pred) except + - Iter lower_bound[Iter, T](Iter first, Iter last, const T& value) - Iter lower_bound[Iter, T, Compare](Iter first, Iter last, const T& value, - Compare comp) + void for_each[Iter, UnaryFunction](Iter first, Iter last, UnaryFunction f) except + # actually returns f - Iter upper_bound[Iter, T](Iter first, Iter last, const T& value) - Iter upper_bound[Iter, T, Compare](Iter first, Iter last, const T& value, - Compare comp) + ptrdiff_t count[Iter, T](Iter first, Iter last, const T& value) except + + ptrdiff_t count_if[Iter, Pred](Iter first, Iter last, Pred pred) except + - void partial_sort[Iter](Iter first, Iter middle, Iter last) - void partial_sort[Iter, Compare](Iter first, Iter middle, Iter last, - Compare comp) + pair[Iter1, Iter2] mismatch[Iter1, Iter2]( + Iter1 first1, Iter1 last1, Iter2 first2) except + # other overloads are tricky - void sort[Iter](Iter first, Iter last) - void sort[Iter, Compare](Iter first, Iter last, Compare comp) + Iter find[Iter, T](Iter first, Iter last, const T& value) except + + Iter find_if[Iter, Pred](Iter first, Iter last, Pred pred) except + + Iter find_if_not[Iter, Pred](Iter first, Iter last, Pred pred) except + - # Removing duplicates - Iter unique[Iter](Iter first, Iter last) - Iter unique[Iter, BinaryPredicate](Iter first, Iter last, BinaryPredicate p) + Iter1 find_end[Iter1, Iter2](Iter1 first1, Iter1 last1, Iter2 first2, Iter2 last2) except + + Iter1 find_end[Iter1, Iter2, BinaryPred]( + Iter1 first1, Iter1 last1, Iter2 first2, Iter2 last2, BinaryPred pred) except + - # Binary heaps (priority queues) - void make_heap[Iter](Iter first, Iter last) - void make_heap[Iter, Compare](Iter first, Iter last, Compare comp) + Iter1 find_first_of[Iter1, Iter2](Iter1 first1, Iter1 last1, Iter2 first2, Iter2 last2) except + + Iter1 find_first_of[Iter1, Iter2, BinaryPred]( + Iter1 first1, Iter1 last1, Iter2 first2, Iter2 last2, BinaryPred pred) except + - void pop_heap[Iter](Iter first, Iter last) - void pop_heap[Iter, Compare](Iter first, Iter last, Compare comp) + Iter adjacent_find[Iter](Iter first, Iter last) except + + Iter adjacent_find[Iter, BinaryPred](Iter first, Iter last, BinaryPred pred) except + - void push_heap[Iter](Iter first, Iter last) - void push_heap[Iter, Compare](Iter first, Iter last, Compare comp) + Iter1 search[Iter1, Iter2](Iter1 first1, Iter1 last1, Iter2 first2, Iter2 last2) except + + Iter1 search[Iter1, Iter2, BinaryPred]( + Iter1 first1, Iter1 last1, Iter2 first2, Iter2 last2, BinaryPred pred) except + + Iter search_n[Iter, Size, T](Iter first1, Iter last1, Size count, const T& value) except + + Iter search_n[Iter, Size, T, BinaryPred]( + Iter first1, Iter last1, Size count, const T& value, BinaryPred pred) except + - void sort_heap[Iter](Iter first, Iter last) - void sort_heap[Iter, Compare](Iter first, Iter last, Compare comp) + # Modifying sequence operations + OutputIt copy[InputIt, OutputIt](InputIt first, InputIt last, OutputIt d_first) except + + OutputIt copy_if[InputIt, OutputIt, Pred](InputIt first, InputIt last, OutputIt d_first, Pred pred) except + + OutputIt copy_n[InputIt, Size, OutputIt](InputIt first, Size count, OutputIt result) except + + Iter2 copy_backward[Iter1, Iter2](Iter1 first, Iter1 last, Iter2 d_last) except + - # Copy - OutputIter copy[InputIter,OutputIter](InputIter,InputIter,OutputIter) + OutputIt move[InputIt, OutputIt](InputIt first, InputIt last, OutputIt d_first) except + + Iter2 move_backward[Iter1, Iter2](Iter1 first, Iter1 last, Iter2 d_last) except + + + void fill[Iter, T](Iter first, Iter last, const T& value) except + + Iter fill_n[Iter, Size, T](Iter first, Size count, const T& value) except + + + OutputIt transform[InputIt, OutputIt, UnaryOp]( + InputIt first1, InputIt last1, OutputIt d_first, UnaryOp unary_op) except + + OutputIt transform[InputIt1, InputIt2, OutputIt, BinaryOp]( + InputIt1 first1, InputIt1 last1, InputIt2 first2, OutputIt d_first, BinaryOp binary_op) except + + + void generate[Iter, Generator](Iter first, Iter last, Generator g) except + + void generate_n[Iter, Size, Generator](Iter first, Size count, Generator g) except + + + Iter remove[Iter, T](Iter first, Iter last, const T& value) except + + Iter remove_if[Iter, UnaryPred](Iter first, Iter last, UnaryPred pred) except + + OutputIt remove_copy[InputIt, OutputIt, T](InputIt first, InputIt last, OutputIt d_first, const T& value) except + + OutputIt remove_copy_if[InputIt, OutputIt, UnaryPred]( + InputIt first, InputIt last, OutputIt d_first, UnaryPred pred) except + + + void replace[Iter, T](Iter first, Iter last, const T& old_value, const T& new_value) except + + void replace_if[Iter, UnaryPred, T](Iter first, Iter last, UnaryPred pred, const T& new_value) except + + OutputIt replace_copy[InputIt, OutputIt, T]( + InputIt first, InputIt last, OutputIt d_first, const T& old_value, const T& new_value) except + + OutputIt replace_copy_if[InputIt, OutputIt, UnaryPred, T]( + InputIt first, InputIt last, OutputIt d_first, UnaryPred pred, const T& new_value) except + + + void swap[T](T& a, T& b) except + # array overload also works + Iter2 swap_ranges[Iter1, Iter2](Iter1 first1, Iter1 last1, Iter2 first2) except + + void iter_swap[Iter](Iter a, Iter b) except + + + void reverse[Iter](Iter first, Iter last) except + + OutputIt reverse_copy[InputIt, OutputIt](InputIt first, InputIt last, OutputIt d_first) except + + + Iter rotate[Iter](Iter first, Iter n_first, Iter last) except + + OutputIt rotate_copy[InputIt, OutputIt](InputIt first, InputIt n_first, InputIt last, OutputIt d_first) except + + + Iter unique[Iter](Iter first, Iter last) except + + Iter unique[Iter, BinaryPred](Iter first, Iter last, BinaryPred p) except + + OutputIt unique_copy[InputIt, OutputIt](InputIt first, InputIt last, OutputIt d_first) except + + OutputIt unique_copy[InputIt, OutputIt, BinaryPred]( + InputIt first, InputIt last, OutputIt d_first, BinaryPred pred) except + + + # Partitioning operations + bool is_partitioned[Iter, Pred](Iter first, Iter last, Pred p) except + + Iter partition[Iter, Pred](Iter first, Iter last, Pred p) except + + pair[OutputIt1, OutputIt2] partition_copy[InputIt, OutputIt1, OutputIt2, Pred]( + InputIt first, InputIt last, OutputIt1 d_first_true, OutputIt2 d_first_false, Pred p) except + + Iter stable_partition[Iter, Pred](Iter first, Iter last, Pred p) except + + Iter partition_point[Iter, Pred](Iter first, Iter last, Pred p) except + + + # Sorting operations + bool is_sorted[Iter](Iter first, Iter last) except + + bool is_sorted[Iter, Compare](Iter first, Iter last, Compare comp) except + + + Iter is_sorted_until[Iter](Iter first, Iter last) except + + Iter is_sorted_until[Iter, Compare](Iter first, Iter last, Compare comp) except + + + void sort[Iter](Iter first, Iter last) except + + void sort[Iter, Compare](Iter first, Iter last, Compare comp) except + + + void partial_sort[Iter](Iter first, Iter middle, Iter last) except + + void partial_sort[Iter, Compare](Iter first, Iter middle, Iter last, Compare comp) except + + + OutputIt partial_sort_copy[InputIt, OutputIt]( + InputIt first, InputIt last, OutputIt d_first, OutputIt d_last) except + + OutputIt partial_sort_copy[InputIt, OutputIt, Compare]( + InputIt first, InputIt last, OutputIt d_first, OutputIt d_last, Compare comp) except + + + void stable_sort[Iter](Iter first, Iter last) except + + void stable_sort[Iter, Compare](Iter first, Iter last, Compare comp) except + + + void nth_element[Iter](Iter first, Iter nth, Iter last) except + + void nth_element[Iter, Compare](Iter first, Iter nth, Iter last, Compare comp) except + + + # Binary search operations (on sorted ranges) + Iter lower_bound[Iter, T](Iter first, Iter last, const T& value) except + + Iter lower_bound[Iter, T, Compare](Iter first, Iter last, const T& value, Compare comp) except + + + Iter upper_bound[Iter, T](Iter first, Iter last, const T& value) except + + Iter upper_bound[Iter, T, Compare](Iter first, Iter last, const T& value, Compare comp) except + + + bool binary_search[Iter, T](Iter first, Iter last, const T& value) except + + bool binary_search[Iter, T, Compare](Iter first, Iter last, const T& value, Compare comp) except + + + # Other operations on sorted ranges + + # Set operations (on sorted ranges) + + # Heap operations + void make_heap[Iter](Iter first, Iter last) except + + void make_heap[Iter, Compare](Iter first, Iter last, Compare comp) except + + + void push_heap[Iter](Iter first, Iter last) except + + void push_heap[Iter, Compare](Iter first, Iter last, Compare comp) except + + + void pop_heap[Iter](Iter first, Iter last) except + + void pop_heap[Iter, Compare](Iter first, Iter last, Compare comp) except + + + void sort_heap[Iter](Iter first, Iter last) except + + void sort_heap[Iter, Compare](Iter first, Iter last, Compare comp) except + + + # Minimum/maximum operations + Iter min_element[Iter](Iter first, Iter last) except + + + # Comparison operations + + # Permutation operations diff --git a/Cython/Includes/libcpp/atomic.pxd b/Cython/Includes/libcpp/atomic.pxd new file mode 100644 index 000000000..3efd35aef --- /dev/null +++ b/Cython/Includes/libcpp/atomic.pxd @@ -0,0 +1,60 @@ + +cdef extern from "<atomic>" namespace "std" nogil: + + cdef enum memory_order: + memory_order_relaxed + memory_order_consume + memory_order_acquire + memory_order_release + memory_order_acq_rel + memory_order_seq_cst + + cdef cppclass atomic[T]: + atomic() + atomic(T) + + bint is_lock_free() + void store(T) + void store(T, memory_order) + T load() + T load(memory_order) + T exchange(T) + T exchange(T, memory_order) + + bint compare_exchange_weak(T&, T, memory_order, memory_order) + bint compare_exchange_weak(T&, T, memory_order) + bint compare_exchange_weak(T&, T) + bint compare_exchange_strong(T&, T, memory_order, memory_order) + bint compare_exchange_strong(T&, T, memory_order) + bint compare_exchange_strong(T&, T) + + T fetch_add(T, memory_order) + T fetch_add(T) + T fetch_sub(T, memory_order) + T fetch_sub(T) + T fetch_and(T, memory_order) + T fetch_and(T) + T fetch_or(T, memory_order) + T fetch_or(T) + T fetch_xor(T, memory_order) + T fetch_xor(T) + + T operator++() + T operator++(int) + T operator--() + T operator--(int) + + # modify-in-place operators not yet supported by Cython: + # T operator+=(T) + # T operator-=(T) + # T operator&=(T) + # T operator|=(T) + # T operator^=(T) + + bint operator==(atomic[T]&, atomic[T]&) + bint operator==(atomic[T]&, T&) + bint operator==(T&, atomic[T]&) + bint operator!=(atomic[T]&, atomic[T]&) + bint operator!=(atomic[T]&, T&) + bint operator!=(T&, atomic[T]&) + diff --git a/Cython/Includes/libcpp/functional.pxd b/Cython/Includes/libcpp/functional.pxd index 94cbd9e1d..4786d39eb 100644 --- a/Cython/Includes/libcpp/functional.pxd +++ b/Cython/Includes/libcpp/functional.pxd @@ -1,3 +1,5 @@ +from libcpp cimport bool + cdef extern from "<functional>" namespace "std" nogil: cdef cppclass function[T]: function() except + @@ -10,4 +12,10 @@ cdef extern from "<functional>" namespace "std" nogil: function operator=(void*) function operator=[U](U) - bint operator bool() + bool operator bool() + + # Comparisons + cdef cppclass greater[T=*]: + # https://github.com/cython/cython/issues/3193 + greater() except + + bool operator()(const T& lhs, const T& rhs) except + diff --git a/Cython/Includes/libcpp/iterator.pxd b/Cython/Includes/libcpp/iterator.pxd index e0f8bd8d6..0b50c586d 100644 --- a/Cython/Includes/libcpp/iterator.pxd +++ b/Cython/Includes/libcpp/iterator.pxd @@ -1,6 +1,8 @@ #Basic reference: http://www.cplusplus.com/reference/iterator/ #Most of these classes are in fact empty structs +from libc.stddef import ptrdiff_t + cdef extern from "<iterator>" namespace "std" nogil: cdef cppclass iterator[Category,T,Distance,Pointer,Reference]: pass @@ -29,4 +31,4 @@ cdef extern from "<iterator>" namespace "std" nogil: ##insert_iterator<Container> inserter (Container& x, typename Container::iterator it) insert_iterator[CONTAINER] inserter[CONTAINER,ITERATOR](CONTAINER &, ITERATOR) - + ptrdiff_t distance[It](It first, It last) diff --git a/Cython/Includes/libcpp/numeric.pxd b/Cython/Includes/libcpp/numeric.pxd new file mode 100644 index 000000000..d61b3a406 --- /dev/null +++ b/Cython/Includes/libcpp/numeric.pxd @@ -0,0 +1,23 @@ +cdef extern from "<numeric>" namespace "std" nogil: + T inner_product[InputIt1, InputIt2, T](InputIt1 first1, InputIt1 last1, InputIt2 first2, T init) + + T inner_product[InputIt1, InputIt2, T, BinaryOperation1, BinaryOperation2](InputIt1 first1, InputIt1 last1, + InputIt2 first2, T init, + BinaryOperation1 op1, + BinaryOperation2 op2) + + void iota[ForwardIt, T](ForwardIt first, ForwardIt last, T value) + + T accumulate[InputIt, T](InputIt first, InputIt last, T init) + + T accumulate[InputIt, T, BinaryOperation](InputIt first, InputIt last, T init, BinaryOperation op) + + void adjacent_difference[InputIt, OutputIt](InputIt in_first, InputIt in_last, OutputIt out_first) + + void adjacent_difference[InputIt, OutputIt, BinaryOperation](InputIt in_first, InputIt in_last, OutputIt out_first, + BinaryOperation op) + + void partial_sum[InputIt, OutputIt](InputIt in_first, OutputIt in_last, OutputIt out_first) + + void partial_sum[InputIt, OutputIt, BinaryOperation](InputIt in_first, InputIt in_last, OutputIt out_first, + BinaryOperation op) diff --git a/Cython/Includes/libcpp/string.pxd b/Cython/Includes/libcpp/string.pxd index 503e664a8..5d9e5b3ac 100644 --- a/Cython/Includes/libcpp/string.pxd +++ b/Cython/Includes/libcpp/string.pxd @@ -8,13 +8,6 @@ cdef extern from "<string>" namespace "std" nogil: size_t npos = -1 cdef cppclass string: - string() except + - string(const char *) except + - string(const char *, size_t) except + - string(const string&) except + - # as a string formed by a repetition of character c, n times. - string(size_t, char) except + - cppclass iterator: iterator() char& operator*() @@ -40,6 +33,15 @@ cdef extern from "<string>" namespace "std" nogil: cppclass const_reverse_iterator(reverse_iterator): pass + string() except + + string(const char *) except + + string(const char *, size_t) except + + string(const string&) except + + # as a string formed by a repetition of character c, n times. + string(size_t, char) except + + # from a pair of iterators + string(iterator first, iterator last) except + + iterator begin() const_iterator const_begin "begin"() iterator end() @@ -60,6 +62,10 @@ cdef extern from "<string>" namespace "std" nogil: void reserve(size_t) void clear() bint empty() + iterator erase(iterator position) + iterator erase(const_iterator position) + iterator erase(iterator first, iterator last) + iterator erase(const_iterator first, const_iterator last) char& at(size_t) char& operator[](size_t) diff --git a/Cython/Includes/numpy/__init__.pxd b/Cython/Includes/numpy/__init__.pxd index 15700c05e..3d9aff71b 100644 --- a/Cython/Includes/numpy/__init__.pxd +++ b/Cython/Includes/numpy/__init__.pxd @@ -1,30 +1,31 @@ # NumPy static imports for Cython # -# If any of the PyArray_* functions are called, import_array must be -# called first. -# -# This also defines backwards-compatibility buffer acquisition -# code for use in Python 2.x (or Python <= 2.5 when NumPy starts -# implementing PEP-3118 directly). +# NOTE: Do not make incompatible local changes to this file without contacting the NumPy project. +# This file is maintained by the NumPy project at +# https://github.com/numpy/numpy/tree/master/numpy # -# Because of laziness, the format string of the buffer is statically -# allocated. Increase the size if this is not enough, or submit a -# patch to do this properly. +# If any of the PyArray_* functions are called, import_array must be +# called first. This is done automatically by Cython 3.0+ if a call +# is not detected inside of the module. # # Author: Dag Sverre Seljebotn # -DEF _buffer_format_string_len = 255 - -cimport cpython.buffer as pybuf from cpython.ref cimport Py_INCREF -from cpython.mem cimport PyObject_Malloc, PyObject_Free -from cpython.object cimport PyObject, PyTypeObject -from cpython.type cimport type +from cpython.object cimport PyObject, PyTypeObject, PyObject_TypeCheck cimport libc.stdio as stdio + +cdef extern from *: + # Leave a marker that the NumPy declarations came from Cython and not from NumPy itself. + # See https://github.com/cython/cython/issues/3573 + """ + /* Using NumPy API declarations from "Cython/Includes/numpy/" */ + """ + + cdef extern from "Python.h": - ctypedef int Py_intptr_t + ctypedef Py_ssize_t Py_intptr_t cdef extern from "numpy/arrayobject.h": ctypedef Py_intptr_t npy_intp @@ -219,7 +220,7 @@ cdef extern from "numpy/arrayobject.h": cdef int type_num cdef int itemsize "elsize" cdef int alignment - cdef dict fields + cdef object fields cdef tuple names # Use PyDataType_HASSUBARRAY to test whether this field is # valid (the pointer can be NULL). Most users should access @@ -242,104 +243,56 @@ cdef extern from "numpy/arrayobject.h": ctypedef class numpy.ndarray [object PyArrayObject, check_size ignore]: cdef __cythonbufferdefaults__ = {"mode": "strided"} - cdef: - # Only taking a few of the most commonly used and stable fields. - # One should use PyArray_* macros instead to access the C fields. - char *data - int ndim "nd" - npy_intp *shape "dimensions" - npy_intp *strides - dtype descr # deprecated since NumPy 1.7 ! - PyObject* base - - # Note: This syntax (function definition in pxd files) is an - # experimental exception made for __getbuffer__ and __releasebuffer__ - # -- the details of this may change. - def __getbuffer__(ndarray self, Py_buffer* info, int flags): - # This implementation of getbuffer is geared towards Cython - # requirements, and does not yet fulfill the PEP. - # In particular strided access is always provided regardless - # of flags - - cdef int i, ndim - cdef int endian_detector = 1 - cdef bint little_endian = ((<char*>&endian_detector)[0] != 0) - - ndim = PyArray_NDIM(self) - - if ((flags & pybuf.PyBUF_C_CONTIGUOUS == pybuf.PyBUF_C_CONTIGUOUS) - and not PyArray_CHKFLAGS(self, NPY_ARRAY_C_CONTIGUOUS)): - raise ValueError(u"ndarray is not C contiguous") - - if ((flags & pybuf.PyBUF_F_CONTIGUOUS == pybuf.PyBUF_F_CONTIGUOUS) - and not PyArray_CHKFLAGS(self, NPY_ARRAY_F_CONTIGUOUS)): - raise ValueError(u"ndarray is not Fortran contiguous") - - info.buf = PyArray_DATA(self) - info.ndim = ndim - if sizeof(npy_intp) != sizeof(Py_ssize_t): - # Allocate new buffer for strides and shape info. - # This is allocated as one block, strides first. - info.strides = <Py_ssize_t*>PyObject_Malloc(sizeof(Py_ssize_t) * 2 * <size_t>ndim) - info.shape = info.strides + ndim - for i in range(ndim): - info.strides[i] = PyArray_STRIDES(self)[i] - info.shape[i] = PyArray_DIMS(self)[i] - else: - info.strides = <Py_ssize_t*>PyArray_STRIDES(self) - info.shape = <Py_ssize_t*>PyArray_DIMS(self) - info.suboffsets = NULL - info.itemsize = PyArray_ITEMSIZE(self) - info.readonly = not PyArray_ISWRITEABLE(self) - - cdef int t - cdef char* f = NULL - cdef dtype descr = <dtype>PyArray_DESCR(self) - cdef int offset - - info.obj = self - - if not PyDataType_HASFIELDS(descr): - t = descr.type_num - if ((descr.byteorder == c'>' and little_endian) or - (descr.byteorder == c'<' and not little_endian)): - raise ValueError(u"Non-native byte order not supported") - if t == NPY_BYTE: f = "b" - elif t == NPY_UBYTE: f = "B" - elif t == NPY_SHORT: f = "h" - elif t == NPY_USHORT: f = "H" - elif t == NPY_INT: f = "i" - elif t == NPY_UINT: f = "I" - elif t == NPY_LONG: f = "l" - elif t == NPY_ULONG: f = "L" - elif t == NPY_LONGLONG: f = "q" - elif t == NPY_ULONGLONG: f = "Q" - elif t == NPY_FLOAT: f = "f" - elif t == NPY_DOUBLE: f = "d" - elif t == NPY_LONGDOUBLE: f = "g" - elif t == NPY_CFLOAT: f = "Zf" - elif t == NPY_CDOUBLE: f = "Zd" - elif t == NPY_CLONGDOUBLE: f = "Zg" - elif t == NPY_OBJECT: f = "O" - else: - raise ValueError(u"unknown dtype code in numpy.pxd (%d)" % t) - info.format = f - return - else: - info.format = <char*>PyObject_Malloc(_buffer_format_string_len) - info.format[0] = c'^' # Native data types, manual alignment - offset = 0 - f = _util_dtypestring(descr, info.format + 1, - info.format + _buffer_format_string_len, - &offset) - f[0] = c'\0' # Terminate format string - - def __releasebuffer__(ndarray self, Py_buffer* info): - if PyArray_HASFIELDS(self): - PyObject_Free(info.format) - if sizeof(npy_intp) != sizeof(Py_ssize_t): - PyObject_Free(info.strides) - # info.shape was stored after info.strides in the same block + # NOTE: no field declarations since direct access is deprecated since NumPy 1.7 + # Instead, we use properties that map to the corresponding C-API functions. + + @property + cdef inline PyObject* base(self) nogil: + """Returns a borrowed reference to the object owning the data/memory. + """ + return PyArray_BASE(self) + + @property + cdef inline dtype descr(self): + """Returns an owned reference to the dtype of the array. + """ + return <dtype>PyArray_DESCR(self) + + @property + cdef inline int ndim(self) nogil: + """Returns the number of dimensions in the array. + """ + return PyArray_NDIM(self) + + @property + cdef inline npy_intp *shape(self) nogil: + """Returns a pointer to the dimensions/shape of the array. + The number of elements matches the number of dimensions of the array (ndim). + Can return NULL for 0-dimensional arrays. + """ + return PyArray_DIMS(self) + + @property + cdef inline npy_intp *strides(self) nogil: + """Returns a pointer to the strides of the array. + The number of elements matches the number of dimensions of the array (ndim). + """ + return PyArray_STRIDES(self) + + @property + cdef inline npy_intp size(self) nogil: + """Returns the total size (in number of elements) of the array. + """ + return PyArray_SIZE(self) + + @property + cdef inline char* data(self) nogil: + """The pointer to the data buffer as a char*. + This is provided for legacy reasons to avoid direct struct field access. + For new code that needs this access, you probably want to cast the result + of `PyArray_DATA()` instead, which returns a 'void*'. + """ + return PyArray_BYTES(self) ctypedef unsigned char npy_bool @@ -416,103 +369,110 @@ cdef extern from "numpy/arrayobject.h": int len int _import_array() except -1 + # A second definition so _import_array isn't marked as used when we use it here. + # Do not use - subject to change any time. + int __pyx_import_array "_import_array"() except -1 # # Macros from ndarrayobject.h # - bint PyArray_CHKFLAGS(ndarray m, int flags) - bint PyArray_IS_C_CONTIGUOUS(ndarray arr) - bint PyArray_IS_F_CONTIGUOUS(ndarray arr) - bint PyArray_ISCONTIGUOUS(ndarray m) - bint PyArray_ISWRITEABLE(ndarray m) - bint PyArray_ISALIGNED(ndarray m) - - int PyArray_NDIM(ndarray) - bint PyArray_ISONESEGMENT(ndarray) - bint PyArray_ISFORTRAN(ndarray) - int PyArray_FORTRANIF(ndarray) - - void* PyArray_DATA(ndarray) - char* PyArray_BYTES(ndarray) - npy_intp* PyArray_DIMS(ndarray) - npy_intp* PyArray_STRIDES(ndarray) - npy_intp PyArray_DIM(ndarray, size_t) - npy_intp PyArray_STRIDE(ndarray, size_t) - - PyObject *PyArray_BASE(ndarray) # returns borrowed reference! - PyArray_Descr *PyArray_DESCR(ndarray) # returns borrowed reference to dtype! - int PyArray_FLAGS(ndarray) - npy_intp PyArray_ITEMSIZE(ndarray) - int PyArray_TYPE(ndarray arr) + bint PyArray_CHKFLAGS(ndarray m, int flags) nogil + bint PyArray_IS_C_CONTIGUOUS(ndarray arr) nogil + bint PyArray_IS_F_CONTIGUOUS(ndarray arr) nogil + bint PyArray_ISCONTIGUOUS(ndarray m) nogil + bint PyArray_ISWRITEABLE(ndarray m) nogil + bint PyArray_ISALIGNED(ndarray m) nogil + + int PyArray_NDIM(ndarray) nogil + bint PyArray_ISONESEGMENT(ndarray) nogil + bint PyArray_ISFORTRAN(ndarray) nogil + int PyArray_FORTRANIF(ndarray) nogil + + void* PyArray_DATA(ndarray) nogil + char* PyArray_BYTES(ndarray) nogil + + npy_intp* PyArray_DIMS(ndarray) nogil + npy_intp* PyArray_STRIDES(ndarray) nogil + npy_intp PyArray_DIM(ndarray, size_t) nogil + npy_intp PyArray_STRIDE(ndarray, size_t) nogil + + PyObject *PyArray_BASE(ndarray) nogil # returns borrowed reference! + PyArray_Descr *PyArray_DESCR(ndarray) nogil # returns borrowed reference to dtype! + PyArray_Descr *PyArray_DTYPE(ndarray) nogil # returns borrowed reference to dtype! NP 1.7+ alias for descr. + int PyArray_FLAGS(ndarray) nogil + void PyArray_CLEARFLAGS(ndarray, int flags) nogil # Added in NumPy 1.7 + void PyArray_ENABLEFLAGS(ndarray, int flags) nogil # Added in NumPy 1.7 + npy_intp PyArray_ITEMSIZE(ndarray) nogil + int PyArray_TYPE(ndarray arr) nogil object PyArray_GETITEM(ndarray arr, void *itemptr) int PyArray_SETITEM(ndarray arr, void *itemptr, object obj) - bint PyTypeNum_ISBOOL(int) - bint PyTypeNum_ISUNSIGNED(int) - bint PyTypeNum_ISSIGNED(int) - bint PyTypeNum_ISINTEGER(int) - bint PyTypeNum_ISFLOAT(int) - bint PyTypeNum_ISNUMBER(int) - bint PyTypeNum_ISSTRING(int) - bint PyTypeNum_ISCOMPLEX(int) - bint PyTypeNum_ISPYTHON(int) - bint PyTypeNum_ISFLEXIBLE(int) - bint PyTypeNum_ISUSERDEF(int) - bint PyTypeNum_ISEXTENDED(int) - bint PyTypeNum_ISOBJECT(int) - - bint PyDataType_ISBOOL(dtype) - bint PyDataType_ISUNSIGNED(dtype) - bint PyDataType_ISSIGNED(dtype) - bint PyDataType_ISINTEGER(dtype) - bint PyDataType_ISFLOAT(dtype) - bint PyDataType_ISNUMBER(dtype) - bint PyDataType_ISSTRING(dtype) - bint PyDataType_ISCOMPLEX(dtype) - bint PyDataType_ISPYTHON(dtype) - bint PyDataType_ISFLEXIBLE(dtype) - bint PyDataType_ISUSERDEF(dtype) - bint PyDataType_ISEXTENDED(dtype) - bint PyDataType_ISOBJECT(dtype) - bint PyDataType_HASFIELDS(dtype) - bint PyDataType_HASSUBARRAY(dtype) - - bint PyArray_ISBOOL(ndarray) - bint PyArray_ISUNSIGNED(ndarray) - bint PyArray_ISSIGNED(ndarray) - bint PyArray_ISINTEGER(ndarray) - bint PyArray_ISFLOAT(ndarray) - bint PyArray_ISNUMBER(ndarray) - bint PyArray_ISSTRING(ndarray) - bint PyArray_ISCOMPLEX(ndarray) - bint PyArray_ISPYTHON(ndarray) - bint PyArray_ISFLEXIBLE(ndarray) - bint PyArray_ISUSERDEF(ndarray) - bint PyArray_ISEXTENDED(ndarray) - bint PyArray_ISOBJECT(ndarray) - bint PyArray_HASFIELDS(ndarray) - - bint PyArray_ISVARIABLE(ndarray) - - bint PyArray_SAFEALIGNEDCOPY(ndarray) - bint PyArray_ISNBO(char) # works on ndarray.byteorder - bint PyArray_IsNativeByteOrder(char) # works on ndarray.byteorder - bint PyArray_ISNOTSWAPPED(ndarray) - bint PyArray_ISBYTESWAPPED(ndarray) - - bint PyArray_FLAGSWAP(ndarray, int) - - bint PyArray_ISCARRAY(ndarray) - bint PyArray_ISCARRAY_RO(ndarray) - bint PyArray_ISFARRAY(ndarray) - bint PyArray_ISFARRAY_RO(ndarray) - bint PyArray_ISBEHAVED(ndarray) - bint PyArray_ISBEHAVED_RO(ndarray) - - - bint PyDataType_ISNOTSWAPPED(dtype) - bint PyDataType_ISBYTESWAPPED(dtype) + bint PyTypeNum_ISBOOL(int) nogil + bint PyTypeNum_ISUNSIGNED(int) nogil + bint PyTypeNum_ISSIGNED(int) nogil + bint PyTypeNum_ISINTEGER(int) nogil + bint PyTypeNum_ISFLOAT(int) nogil + bint PyTypeNum_ISNUMBER(int) nogil + bint PyTypeNum_ISSTRING(int) nogil + bint PyTypeNum_ISCOMPLEX(int) nogil + bint PyTypeNum_ISPYTHON(int) nogil + bint PyTypeNum_ISFLEXIBLE(int) nogil + bint PyTypeNum_ISUSERDEF(int) nogil + bint PyTypeNum_ISEXTENDED(int) nogil + bint PyTypeNum_ISOBJECT(int) nogil + + bint PyDataType_ISBOOL(dtype) nogil + bint PyDataType_ISUNSIGNED(dtype) nogil + bint PyDataType_ISSIGNED(dtype) nogil + bint PyDataType_ISINTEGER(dtype) nogil + bint PyDataType_ISFLOAT(dtype) nogil + bint PyDataType_ISNUMBER(dtype) nogil + bint PyDataType_ISSTRING(dtype) nogil + bint PyDataType_ISCOMPLEX(dtype) nogil + bint PyDataType_ISPYTHON(dtype) nogil + bint PyDataType_ISFLEXIBLE(dtype) nogil + bint PyDataType_ISUSERDEF(dtype) nogil + bint PyDataType_ISEXTENDED(dtype) nogil + bint PyDataType_ISOBJECT(dtype) nogil + bint PyDataType_HASFIELDS(dtype) nogil + bint PyDataType_HASSUBARRAY(dtype) nogil + + bint PyArray_ISBOOL(ndarray) nogil + bint PyArray_ISUNSIGNED(ndarray) nogil + bint PyArray_ISSIGNED(ndarray) nogil + bint PyArray_ISINTEGER(ndarray) nogil + bint PyArray_ISFLOAT(ndarray) nogil + bint PyArray_ISNUMBER(ndarray) nogil + bint PyArray_ISSTRING(ndarray) nogil + bint PyArray_ISCOMPLEX(ndarray) nogil + bint PyArray_ISPYTHON(ndarray) nogil + bint PyArray_ISFLEXIBLE(ndarray) nogil + bint PyArray_ISUSERDEF(ndarray) nogil + bint PyArray_ISEXTENDED(ndarray) nogil + bint PyArray_ISOBJECT(ndarray) nogil + bint PyArray_HASFIELDS(ndarray) nogil + + bint PyArray_ISVARIABLE(ndarray) nogil + + bint PyArray_SAFEALIGNEDCOPY(ndarray) nogil + bint PyArray_ISNBO(char) nogil # works on ndarray.byteorder + bint PyArray_IsNativeByteOrder(char) nogil # works on ndarray.byteorder + bint PyArray_ISNOTSWAPPED(ndarray) nogil + bint PyArray_ISBYTESWAPPED(ndarray) nogil + + bint PyArray_FLAGSWAP(ndarray, int) nogil + + bint PyArray_ISCARRAY(ndarray) nogil + bint PyArray_ISCARRAY_RO(ndarray) nogil + bint PyArray_ISFARRAY(ndarray) nogil + bint PyArray_ISFARRAY_RO(ndarray) nogil + bint PyArray_ISBEHAVED(ndarray) nogil + bint PyArray_ISBEHAVED_RO(ndarray) nogil + + + bint PyDataType_ISNOTSWAPPED(dtype) nogil + bint PyDataType_ISBYTESWAPPED(dtype) nogil bint PyArray_DescrCheck(object) @@ -532,10 +492,11 @@ cdef extern from "numpy/arrayobject.h": bint PyArray_IsPythonScalar(object) bint PyArray_IsAnyScalar(object) bint PyArray_CheckAnyScalar(object) + ndarray PyArray_GETCONTIGUOUS(ndarray) - bint PyArray_SAMESHAPE(ndarray, ndarray) - npy_intp PyArray_SIZE(ndarray) - npy_intp PyArray_NBYTES(ndarray) + bint PyArray_SAMESHAPE(ndarray, ndarray) nogil + npy_intp PyArray_SIZE(ndarray) nogil + npy_intp PyArray_NBYTES(ndarray) nogil object PyArray_FROM_O(object) object PyArray_FROM_OF(object m, int flags) @@ -548,16 +509,16 @@ cdef extern from "numpy/arrayobject.h": npy_intp PyArray_REFCOUNT(object) object PyArray_ContiguousFromAny(op, int, int min_depth, int max_depth) unsigned char PyArray_EquivArrTypes(ndarray a1, ndarray a2) - bint PyArray_EquivByteorders(int b1, int b2) + bint PyArray_EquivByteorders(int b1, int b2) nogil object PyArray_SimpleNew(int nd, npy_intp* dims, int typenum) object PyArray_SimpleNewFromData(int nd, npy_intp* dims, int typenum, void* data) #object PyArray_SimpleNewFromDescr(int nd, npy_intp* dims, dtype descr) object PyArray_ToScalar(void* data, ndarray arr) - void* PyArray_GETPTR1(ndarray m, npy_intp i) - void* PyArray_GETPTR2(ndarray m, npy_intp i, npy_intp j) - void* PyArray_GETPTR3(ndarray m, npy_intp i, npy_intp j, npy_intp k) - void* PyArray_GETPTR4(ndarray m, npy_intp i, npy_intp j, npy_intp k, npy_intp l) + void* PyArray_GETPTR1(ndarray m, npy_intp i) nogil + void* PyArray_GETPTR2(ndarray m, npy_intp i, npy_intp j) nogil + void* PyArray_GETPTR3(ndarray m, npy_intp i, npy_intp j, npy_intp k) nogil + void* PyArray_GETPTR4(ndarray m, npy_intp i, npy_intp j, npy_intp k, npy_intp l) nogil void PyArray_XDECREF_ERR(ndarray) # Cannot be supported due to out arg @@ -685,7 +646,7 @@ cdef extern from "numpy/arrayobject.h": object PyArray_Choose (ndarray, object, ndarray, NPY_CLIPMODE) int PyArray_Sort (ndarray, int, NPY_SORTKIND) object PyArray_ArgSort (ndarray, int, NPY_SORTKIND) - object PyArray_SearchSorted (ndarray, object, NPY_SEARCHSIDE, PyObject*) + object PyArray_SearchSorted (ndarray, object, NPY_SEARCHSIDE, PyObject *) object PyArray_ArgMax (ndarray, int, ndarray) object PyArray_ArgMin (ndarray, int, ndarray) object PyArray_Reshape (ndarray, object) @@ -838,72 +799,67 @@ cdef inline tuple PyDataType_SHAPE(dtype d): else: return () -cdef inline char* _util_dtypestring(dtype descr, char* f, char* end, int* offset) except NULL: - # Recursive utility function used in __getbuffer__ to get format - # string. The new location in the format string is returned. - - cdef dtype child - cdef int endian_detector = 1 - cdef bint little_endian = ((<char*>&endian_detector)[0] != 0) - cdef tuple fields - - for childname in descr.names: - fields = descr.fields[childname] - child, new_offset = fields - - if (end - f) - <int>(new_offset - offset[0]) < 15: - raise RuntimeError(u"Format string allocated too short, see comment in numpy.pxd") - - if ((child.byteorder == c'>' and little_endian) or - (child.byteorder == c'<' and not little_endian)): - raise ValueError(u"Non-native byte order not supported") - # One could encode it in the format string and have Cython - # complain instead, BUT: < and > in format strings also imply - # standardized sizes for datatypes, and we rely on native in - # order to avoid reencoding data types based on their size. - # - # A proper PEP 3118 exporter for other clients than Cython - # must deal properly with this! - - # Output padding bytes - while offset[0] < new_offset: - f[0] = 120 # "x"; pad byte - f += 1 - offset[0] += 1 - - offset[0] += child.itemsize - - if not PyDataType_HASFIELDS(child): - t = child.type_num - if end - f < 5: - raise RuntimeError(u"Format string allocated too short.") - - # Until ticket #99 is fixed, use integers to avoid warnings - if t == NPY_BYTE: f[0] = 98 #"b" - elif t == NPY_UBYTE: f[0] = 66 #"B" - elif t == NPY_SHORT: f[0] = 104 #"h" - elif t == NPY_USHORT: f[0] = 72 #"H" - elif t == NPY_INT: f[0] = 105 #"i" - elif t == NPY_UINT: f[0] = 73 #"I" - elif t == NPY_LONG: f[0] = 108 #"l" - elif t == NPY_ULONG: f[0] = 76 #"L" - elif t == NPY_LONGLONG: f[0] = 113 #"q" - elif t == NPY_ULONGLONG: f[0] = 81 #"Q" - elif t == NPY_FLOAT: f[0] = 102 #"f" - elif t == NPY_DOUBLE: f[0] = 100 #"d" - elif t == NPY_LONGDOUBLE: f[0] = 103 #"g" - elif t == NPY_CFLOAT: f[0] = 90; f[1] = 102; f += 1 # Zf - elif t == NPY_CDOUBLE: f[0] = 90; f[1] = 100; f += 1 # Zd - elif t == NPY_CLONGDOUBLE: f[0] = 90; f[1] = 103; f += 1 # Zg - elif t == NPY_OBJECT: f[0] = 79 #"O" - else: - raise ValueError(u"unknown dtype code in numpy.pxd (%d)" % t) - f += 1 - else: - # Cython ignores struct boundary information ("T{...}"), - # so don't output it - f = _util_dtypestring(child, f, end, offset) - return f + +cdef extern from "numpy/ndarrayobject.h": + PyTypeObject PyTimedeltaArrType_Type + PyTypeObject PyDatetimeArrType_Type + ctypedef int64_t npy_timedelta + ctypedef int64_t npy_datetime + +cdef extern from "numpy/ndarraytypes.h": + ctypedef struct PyArray_DatetimeMetaData: + NPY_DATETIMEUNIT base + int64_t num + +cdef extern from "numpy/arrayscalars.h": + + # abstract types + ctypedef class numpy.generic [object PyObject]: + pass + ctypedef class numpy.number [object PyObject]: + pass + ctypedef class numpy.integer [object PyObject]: + pass + ctypedef class numpy.signedinteger [object PyObject]: + pass + ctypedef class numpy.unsignedinteger [object PyObject]: + pass + ctypedef class numpy.inexact [object PyObject]: + pass + ctypedef class numpy.floating [object PyObject]: + pass + ctypedef class numpy.complexfloating [object PyObject]: + pass + ctypedef class numpy.flexible [object PyObject]: + pass + ctypedef class numpy.character [object PyObject]: + pass + + ctypedef struct PyDatetimeScalarObject: + # PyObject_HEAD + npy_datetime obval + PyArray_DatetimeMetaData obmeta + + ctypedef struct PyTimedeltaScalarObject: + # PyObject_HEAD + npy_timedelta obval + PyArray_DatetimeMetaData obmeta + + ctypedef enum NPY_DATETIMEUNIT: + NPY_FR_Y + NPY_FR_M + NPY_FR_W + NPY_FR_D + NPY_FR_B + NPY_FR_h + NPY_FR_m + NPY_FR_s + NPY_FR_ms + NPY_FR_us + NPY_FR_ns + NPY_FR_ps + NPY_FR_fs + NPY_FR_as # @@ -1032,7 +988,7 @@ cdef inline object get_array_base(ndarray arr): # Cython code. cdef inline int import_array() except -1: try: - _import_array() + __pyx_import_array() except Exception: raise ImportError("numpy.core.multiarray failed to import") @@ -1047,3 +1003,57 @@ cdef inline int import_ufunc() except -1: _import_umath() except Exception: raise ImportError("numpy.core.umath failed to import") + + +cdef inline bint is_timedelta64_object(object obj): + """ + Cython equivalent of `isinstance(obj, np.timedelta64)` + + Parameters + ---------- + obj : object + + Returns + ------- + bool + """ + return PyObject_TypeCheck(obj, &PyTimedeltaArrType_Type) + + +cdef inline bint is_datetime64_object(object obj): + """ + Cython equivalent of `isinstance(obj, np.datetime64)` + + Parameters + ---------- + obj : object + + Returns + ------- + bool + """ + return PyObject_TypeCheck(obj, &PyDatetimeArrType_Type) + + +cdef inline npy_datetime get_datetime64_value(object obj) nogil: + """ + returns the int64 value underlying scalar numpy datetime64 object + + Note that to interpret this as a datetime, the corresponding unit is + also needed. That can be found using `get_datetime64_unit`. + """ + return (<PyDatetimeScalarObject*>obj).obval + + +cdef inline npy_timedelta get_timedelta64_value(object obj) nogil: + """ + returns the int64 value underlying scalar numpy timedelta64 object + """ + return (<PyTimedeltaScalarObject*>obj).obval + + +cdef inline NPY_DATETIMEUNIT get_datetime64_unit(object obj) nogil: + """ + returns the unit part of the dtype for a numpy datetime64 object. + """ + return <NPY_DATETIMEUNIT>(<PyDatetimeScalarObject*>obj).obmeta.base diff --git a/Cython/Includes/posix/dlfcn.pxd b/Cython/Includes/posix/dlfcn.pxd index cff5bea15..bf61997f3 100644 --- a/Cython/Includes/posix/dlfcn.pxd +++ b/Cython/Includes/posix/dlfcn.pxd @@ -1,5 +1,5 @@ # POSIX dynamic linking/loading interface. -# http://pubs.opengroup.org/onlinepubs/9699919799/basedefs/dlfcn.h.html +# https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/dlfcn.h.html cdef extern from "<dlfcn.h>" nogil: void *dlopen(const char *, int) diff --git a/Cython/Includes/posix/mman.pxd b/Cython/Includes/posix/mman.pxd index c810f431b..9bcad66db 100644 --- a/Cython/Includes/posix/mman.pxd +++ b/Cython/Includes/posix/mman.pxd @@ -1,4 +1,4 @@ -# http://pubs.opengroup.org/onlinepubs/009695399/basedefs/sys/mman.h.html +# https://pubs.opengroup.org/onlinepubs/009695399/basedefs/sys/mman.h.html from posix.types cimport off_t, mode_t diff --git a/Cython/Includes/posix/resource.pxd b/Cython/Includes/posix/resource.pxd index 9f55c6ab4..ddcf4e1be 100644 --- a/Cython/Includes/posix/resource.pxd +++ b/Cython/Includes/posix/resource.pxd @@ -1,4 +1,4 @@ -# http://pubs.opengroup.org/onlinepubs/009695399/basedefs/sys/resource.h.html +# https://pubs.opengroup.org/onlinepubs/009695399/basedefs/sys/resource.h.html from posix.time cimport timeval from posix.types cimport id_t diff --git a/Cython/Includes/posix/stdio.pxd b/Cython/Includes/posix/stdio.pxd index 53913fdf4..38b815559 100644 --- a/Cython/Includes/posix/stdio.pxd +++ b/Cython/Includes/posix/stdio.pxd @@ -1,5 +1,5 @@ # POSIX additions to <stdio.h>. -# http://pubs.opengroup.org/onlinepubs/9699919799/basedefs/stdio.h.html +# https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/stdio.h.html from libc.stdio cimport FILE from libc.stddef cimport wchar_t diff --git a/Cython/Includes/posix/stdlib.pxd b/Cython/Includes/posix/stdlib.pxd index 513de938a..188e2e501 100644 --- a/Cython/Includes/posix/stdlib.pxd +++ b/Cython/Includes/posix/stdlib.pxd @@ -1,5 +1,5 @@ # POSIX additions to <stdlib.h> -# http://pubs.opengroup.org/onlinepubs/9699919799/basedefs/stdlib.h.html +# https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/stdlib.h.html cdef extern from "<stdlib.h>" nogil: void _Exit(int) diff --git a/Cython/Includes/posix/time.pxd b/Cython/Includes/posix/time.pxd index 6bc81bfea..a90cab577 100644 --- a/Cython/Includes/posix/time.pxd +++ b/Cython/Includes/posix/time.pxd @@ -1,4 +1,4 @@ -# http://pubs.opengroup.org/onlinepubs/009695399/basedefs/sys/time.h.html +# https://pubs.opengroup.org/onlinepubs/009695399/basedefs/sys/time.h.html from posix.types cimport suseconds_t, time_t, clockid_t, timer_t from posix.signal cimport sigevent diff --git a/Cython/Includes/posix/wait.pxd b/Cython/Includes/posix/wait.pxd index d18cff9cf..7be7c49b6 100644 --- a/Cython/Includes/posix/wait.pxd +++ b/Cython/Includes/posix/wait.pxd @@ -1,4 +1,4 @@ -# http://pubs.opengroup.org/onlinepubs/009695399/basedefs/sys/wait.h.html +# https://pubs.opengroup.org/onlinepubs/009695399/basedefs/sys/wait.h.html from posix.types cimport pid_t, id_t from posix.signal cimport siginfo_t diff --git a/Cython/Plex/Actions.pxd b/Cython/Plex/Actions.pxd index 34660a2d9..cd884ced8 100644 --- a/Cython/Plex/Actions.pxd +++ b/Cython/Plex/Actions.pxd @@ -1,25 +1,26 @@ +# cython: language_level=3 cdef class Action: - cdef perform(self, token_stream, text) - cpdef same_as(self, other) + cdef perform(self, token_stream, text) cdef class Return(Action): - cdef object value - cdef perform(self, token_stream, text) - cpdef same_as(self, other) + cdef object value + cdef perform(self, token_stream, text) cdef class Call(Action): - cdef object function - cdef perform(self, token_stream, text) - cpdef same_as(self, other) + cdef object function + cdef perform(self, token_stream, text) + +cdef class Method(Action): + cdef str name + cdef dict kwargs cdef class Begin(Action): - cdef object state_name - cdef perform(self, token_stream, text) - cpdef same_as(self, other) + cdef object state_name + cdef perform(self, token_stream, text) cdef class Ignore(Action): - cdef perform(self, token_stream, text) + cdef perform(self, token_stream, text) cdef class Text(Action): - cdef perform(self, token_stream, text) + cdef perform(self, token_stream, text) diff --git a/Cython/Plex/Actions.py b/Cython/Plex/Actions.py index c88176e71..725278ddf 100644 --- a/Cython/Plex/Actions.py +++ b/Cython/Plex/Actions.py @@ -1,18 +1,20 @@ +# cython: language_level=3str # cython: auto_pickle=False -#======================================================================= -# -# Python Lexical Analyser -# -# Actions for use in token specifications -# -#======================================================================= +""" +Python Lexical Analyser + +Actions for use in token specifications +""" class Action(object): def perform(self, token_stream, text): pass # abstract - def same_as(self, other): - return self is other + def __copy__(self): + return self # immutable, no need to copy + + def __deepcopy__(self, memo): + return self # immutable, no need to copy class Return(Action): @@ -27,11 +29,8 @@ class Return(Action): def perform(self, token_stream, text): return self.value - def same_as(self, other): - return isinstance(other, Return) and self.value == other.value - def __repr__(self): - return "Return(%s)" % repr(self.value) + return "Return(%r)" % self.value class Call(Action): @@ -48,8 +47,27 @@ class Call(Action): def __repr__(self): return "Call(%s)" % self.function.__name__ - def same_as(self, other): - return isinstance(other, Call) and self.function is other.function + +class Method(Action): + """ + Plex action that calls a specific method on the token stream, + passing the matched text and any provided constant keyword arguments. + """ + + def __init__(self, name, **kwargs): + self.name = name + self.kwargs = kwargs or None + + def perform(self, token_stream, text): + method = getattr(token_stream, self.name) + # self.kwargs is almost always unused => avoid call overhead + return method(text, **self.kwargs) if self.kwargs is not None else method(text) + + def __repr__(self): + kwargs = ( + ', '.join(sorted(['%s=%r' % item for item in self.kwargs.items()])) + if self.kwargs is not None else '') + return "Method(%s%s%s)" % (self.name, ', ' if kwargs else '', kwargs) class Begin(Action): @@ -68,9 +86,6 @@ class Begin(Action): def __repr__(self): return "Begin(%s)" % self.state_name - def same_as(self, other): - return isinstance(other, Begin) and self.state_name == other.state_name - class Ignore(Action): """ @@ -87,7 +102,6 @@ class Ignore(Action): IGNORE = Ignore() -#IGNORE.__doc__ = Ignore.__doc__ class Text(Action): @@ -105,6 +119,3 @@ class Text(Action): TEXT = Text() -#TEXT.__doc__ = Text.__doc__ - - diff --git a/Cython/Plex/DFA.pxd b/Cython/Plex/DFA.pxd new file mode 100644 index 000000000..226d07567 --- /dev/null +++ b/Cython/Plex/DFA.pxd @@ -0,0 +1,30 @@ +# cython: auto_pickle=False + +cimport cython + +from . cimport Machines +from .Transitions cimport TransitionMap + + +@cython.final +cdef class StateMap: + cdef Machines.FastMachine new_machine + cdef dict old_to_new_dict + cdef dict new_to_old_dict + + cdef old_to_new(self, dict old_state_set) + + @cython.locals(state=Machines.Node) + cdef highest_priority_action(self, dict state_set) + + cdef make_key(self, dict state_set) + + +@cython.locals(new_machine=Machines.FastMachine, transitions=TransitionMap) +cpdef nfa_to_dfa(Machines.Machine old_machine, debug=*) + +cdef set_epsilon_closure(dict state_set) +cdef dict epsilon_closure(Machines.Node state) + +@cython.locals(state_set_2=dict, state2=Machines.Node) +cdef add_to_epsilon_closure(dict state_set, Machines.Node state) diff --git a/Cython/Plex/DFA.py b/Cython/Plex/DFA.py index 76324621f..66dc4a379 100644 --- a/Cython/Plex/DFA.py +++ b/Cython/Plex/DFA.py @@ -1,11 +1,9 @@ -#======================================================================= -# -# Python Lexical Analyser -# -# Converting NFA to DFA -# -#======================================================================= +# cython: auto_cpdef=True +""" +Python Lexical Analyser +Converting NFA to DFA +""" from __future__ import absolute_import from . import Machines @@ -29,12 +27,14 @@ def nfa_to_dfa(old_machine, debug=None): # is reached. new_machine = Machines.FastMachine() state_map = StateMap(new_machine) + # Seed the process using the initial states of the old machine. # Make the corresponding new states into initial states of the new # machine with the same names. for (key, old_state) in old_machine.initial_states.items(): new_state = state_map.old_to_new(epsilon_closure(old_state)) new_machine.make_initial_state(key, new_state) + # Tricky bit here: we add things to the end of this list while we're # iterating over it. The iteration stops when closure is achieved. for new_state in new_machine.states: @@ -45,6 +45,7 @@ def nfa_to_dfa(old_machine, debug=None): transitions.add_set(event, set_epsilon_closure(old_target_states)) for event, old_states in transitions.items(): new_machine.add_transitions(new_state, event, state_map.old_to_new(old_states)) + if debug: debug.write("\n===== State Mapping =====\n") state_map.dump(debug) @@ -95,14 +96,11 @@ class StateMap(object): Helper class used by nfa_to_dfa() to map back and forth between sets of states from the old machine and states of the new machine. """ - new_machine = None # Machine - old_to_new_dict = None # {(old_state,...) : new_state} - new_to_old_dict = None # {id(new_state) : old_state_set} def __init__(self, new_machine): - self.new_machine = new_machine - self.old_to_new_dict = {} - self.new_to_old_dict = {} + self.new_machine = new_machine # Machine + self.old_to_new_dict = {} # {(old_state,...) : new_state} + self.new_to_old_dict = {} # {id(new_state) : old_state_set} def old_to_new(self, old_state_set): """ @@ -119,8 +117,6 @@ class StateMap(object): new_state = self.new_machine.new_state(action) self.old_to_new_dict[key] = new_state self.new_to_old_dict[id(new_state)] = old_state_set - #for old_state in old_state_set.keys(): - #new_state.merge_actions(old_state) return new_state def highest_priority_action(self, state_set): @@ -133,13 +129,6 @@ class StateMap(object): best_priority = priority return best_action - # def old_to_new_set(self, old_state_set): - # """ - # Return the new state corresponding to a set of old states as - # a singleton set. - # """ - # return {self.old_to_new(old_state_set):1} - def new_to_old(self, new_state): """Given a new state, return a set of corresponding old states.""" return self.new_to_old_dict[id(new_state)] @@ -149,9 +138,7 @@ class StateMap(object): Convert a set of states into a uniquified sorted tuple suitable for use as a dictionary key. """ - lst = list(state_set) - lst.sort() - return tuple(lst) + return tuple(sorted(state_set)) def dump(self, file): from .Transitions import state_set_str @@ -160,5 +147,3 @@ class StateMap(object): old_state_set = self.new_to_old_dict[id(new_state)] file.write(" State %s <-- %s\n" % ( new_state['number'], state_set_str(old_state_set))) - - diff --git a/Cython/Plex/Errors.py b/Cython/Plex/Errors.py index f460100d7..fa10374f8 100644 --- a/Cython/Plex/Errors.py +++ b/Cython/Plex/Errors.py @@ -1,10 +1,8 @@ -#======================================================================= -# -# Python Lexical Analyser -# -# Exception classes -# -#======================================================================= +""" +Python Lexical Analyser + +Exception classes +""" class PlexError(Exception): @@ -19,10 +17,6 @@ class PlexValueError(PlexError, ValueError): pass -class InvalidRegex(PlexError): - pass - - class InvalidToken(PlexError): def __init__(self, token_number, message): PlexError.__init__(self, "Token number %d: %s" % (token_number, message)) diff --git a/Cython/Plex/Lexicons.py b/Cython/Plex/Lexicons.py index 787f5854b..438e44bda 100644 --- a/Cython/Plex/Lexicons.py +++ b/Cython/Plex/Lexicons.py @@ -1,15 +1,10 @@ -#======================================================================= -# -# Python Lexical Analyser -# -# Lexical Analyser Specification -# -#======================================================================= +""" +Python Lexical Analyser +Lexical Analyser Specification +""" from __future__ import absolute_import -import types - from . import Actions from . import DFA from . import Errors @@ -114,17 +109,14 @@ class Lexicon(object): machine = None # Machine tables = None # StateTableMachine - def __init__(self, specifications, debug=None, debug_flags=7, timings=None): + def __init__(self, specifications, debug=None, debug_flags=7): if not isinstance(specifications, list): raise Errors.InvalidScanner("Scanner definition is not a list") - if timings: - from .Timing import time - total_time = 0.0 - time1 = time() nfa = Machines.Machine() default_initial_state = nfa.new_initial_state('') token_number = 1 + for spec in specifications: if isinstance(spec, State): user_initial_state = nfa.new_initial_state(spec.name) @@ -140,33 +132,22 @@ class Lexicon(object): raise Errors.InvalidToken( token_number, "Expected a token definition (tuple) or State instance") - if timings: - time2 = time() - total_time = total_time + (time2 - time1) - time3 = time() + if debug and (debug_flags & 1): debug.write("\n============= NFA ===========\n") nfa.dump(debug) + dfa = DFA.nfa_to_dfa(nfa, debug=(debug_flags & 3) == 3 and debug) - if timings: - time4 = time() - total_time = total_time + (time4 - time3) + if debug and (debug_flags & 2): debug.write("\n============= DFA ===========\n") dfa.dump(debug) - if timings: - timings.write("Constructing NFA : %5.2f\n" % (time2 - time1)) - timings.write("Converting to DFA: %5.2f\n" % (time4 - time3)) - timings.write("TOTAL : %5.2f\n" % total_time) + self.machine = dfa def add_token_to_machine(self, machine, initial_state, token_spec, token_number): try: (re, action_spec) = self.parse_token_definition(token_spec) - # Disabled this -- matching empty strings can be useful - #if re.nullable: - # raise Errors.InvalidToken( - # token_number, "Pattern can match 0 input symbols") if isinstance(action_spec, Actions.Action): action = action_spec else: @@ -188,6 +169,7 @@ class Lexicon(object): raise Errors.InvalidToken("Token definition is not a tuple") if len(token_spec) != 2: raise Errors.InvalidToken("Wrong number of items in token definition") + pattern, action = token_spec if not isinstance(pattern, Regexps.RE): raise Errors.InvalidToken("Pattern is not an RE instance") @@ -195,6 +177,3 @@ class Lexicon(object): def get_initial_state(self, name): return self.machine.get_initial_state(name) - - - diff --git a/Cython/Plex/Machines.pxd b/Cython/Plex/Machines.pxd new file mode 100644 index 000000000..8f785a07c --- /dev/null +++ b/Cython/Plex/Machines.pxd @@ -0,0 +1,33 @@ +cimport cython + +from .Actions cimport Action +from .Transitions cimport TransitionMap + +cdef long maxint + + +@cython.final +cdef class Machine: + cdef readonly list states + cdef readonly dict initial_states + cdef readonly Py_ssize_t next_state_number + + cpdef new_state(self) + cpdef new_initial_state(self, name) + + +@cython.final +cdef class Node: + cdef readonly TransitionMap transitions + cdef readonly Action action + cdef public dict epsilon_closure + cdef readonly Py_ssize_t number + cdef readonly long action_priority + + +@cython.final +cdef class FastMachine: + cdef readonly dict initial_states + cdef readonly dict new_state_template + cdef readonly list states + cdef readonly Py_ssize_t next_number diff --git a/Cython/Plex/Machines.py b/Cython/Plex/Machines.py index 398850976..baf3f1918 100644 --- a/Cython/Plex/Machines.py +++ b/Cython/Plex/Machines.py @@ -1,42 +1,33 @@ -#======================================================================= -# -# Python Lexical Analyser -# -# Classes for building NFAs and DFAs -# -#======================================================================= +# cython: auto_pickle=False +""" +Python Lexical Analyser +Classes for building NFAs and DFAs +""" from __future__ import absolute_import -import sys - +import cython from .Transitions import TransitionMap -try: - from sys import maxsize as maxint -except ImportError: - from sys import maxint +maxint = 2**31-1 # sentinel value -try: - unichr -except NameError: - unichr = chr +if not cython.compiled: + try: + unichr + except NameError: + unichr = chr LOWEST_PRIORITY = -maxint class Machine(object): """A collection of Nodes representing an NFA or DFA.""" - states = None # [Node] - next_state_number = 1 - initial_states = None # {(name, bol): Node} - def __init__(self): - self.states = [] - self.initial_states = {} + self.states = [] # [Node] + self.initial_states = {} # {(name, bol): Node} + self.next_state_number = 1 def __del__(self): - #print "Destroying", self ### for state in self.states: state.destroy() @@ -72,21 +63,17 @@ class Machine(object): class Node(object): """A state of an NFA or DFA.""" - transitions = None # TransitionMap - action = None # Action - action_priority = None # integer - number = 0 # for debug output - epsilon_closure = None # used by nfa_to_dfa() def __init__(self): # Preinitialise the list of empty transitions, because # the nfa-to-dfa algorithm needs it - #self.transitions = {'':[]} - self.transitions = TransitionMap() - self.action_priority = LOWEST_PRIORITY + self.transitions = TransitionMap() # TransitionMap + self.action_priority = LOWEST_PRIORITY # integer + self.action = None # Action + self.number = 0 # for debug output + self.epsilon_closure = None # used by nfa_to_dfa() def destroy(self): - #print "Destroying", self ### self.transitions = None self.action = None self.epsilon_closure = None @@ -133,23 +120,23 @@ class Node(object): def __lt__(self, other): return self.number < other.number + def __hash__(self): + # Prevent overflowing hash values due to arbitrarily large unsigned addresses. + return id(self) & maxint + class FastMachine(object): """ FastMachine is a deterministic machine represented in a way that allows fast scanning. """ - initial_states = None # {state_name:state} - states = None # [state] where state = {event:state, 'else':state, 'action':Action} - next_number = 1 # for debugging - - new_state_template = { - '': None, 'bol': None, 'eol': None, 'eof': None, 'else': None - } - def __init__(self): - self.initial_states = {} - self.states = [] + self.initial_states = {} # {state_name:state} + self.states = [] # [state] where state = {event:state, 'else':state, 'action':Action} + self.next_number = 1 # for debugging + self.new_state_template = { + '': None, 'bol': None, 'eol': None, 'eof': None, 'else': None + } def __del__(self): for state in self.states: @@ -167,6 +154,7 @@ class FastMachine(object): def make_initial_state(self, name, state): self.initial_states[name] = state + @cython.locals(code0=cython.long, code1=cython.long, maxint=cython.long, state=dict) def add_transitions(self, state, event, new_state, maxint=maxint): if type(event) is tuple: code0, code1 = event @@ -218,9 +206,7 @@ class FastMachine(object): if char_list: ranges = self.chars_to_ranges(char_list) ranges_to_state[ranges] = state - ranges_list = ranges_to_state.keys() - ranges_list.sort() - for ranges in ranges_list: + for ranges in sorted(ranges_to_state): key = self.ranges_to_string(ranges) state = ranges_to_state[ranges] file.write(" %s --> State %d\n" % (key, state['number'])) @@ -229,6 +215,7 @@ class FastMachine(object): if state: file.write(" %s --> State %d\n" % (key, state['number'])) + @cython.locals(char_list=list, i=cython.Py_ssize_t, n=cython.Py_ssize_t, c1=cython.long, c2=cython.long) def chars_to_ranges(self, char_list): char_list.sort() i = 0 diff --git a/Cython/Plex/Regexps.py b/Cython/Plex/Regexps.py index 41816c939..99d8c994a 100644 --- a/Cython/Plex/Regexps.py +++ b/Cython/Plex/Regexps.py @@ -1,21 +1,16 @@ -#======================================================================= -# -# Python Lexical Analyser -# -# Regular Expressions -# -#======================================================================= +""" +Python Lexical Analyser +Regular Expressions +""" from __future__ import absolute_import import types -try: - from sys import maxsize as maxint -except ImportError: - from sys import maxint from . import Errors +maxint = 2**31-1 # sentinel value + # # Constants # @@ -186,37 +181,6 @@ class RE(object): # These are the basic REs from which all others are built. # -## class Char(RE): -## """ -## Char(c) is an RE which matches the character |c|. -## """ - -## nullable = 0 - -## def __init__(self, char): -## self.char = char -## self.match_nl = char == '\n' - -## def build_machine(self, m, initial_state, final_state, match_bol, nocase): -## c = self.char -## if match_bol and c != BOL: -## s1 = self.build_opt(m, initial_state, BOL) -## else: -## s1 = initial_state -## if c == '\n' or c == EOF: -## s1 = self.build_opt(m, s1, EOL) -## if len(c) == 1: -## code = ord(self.char) -## s1.add_transition((code, code+1), final_state) -## if nocase and is_letter_code(code): -## code2 = other_case_code(code) -## s1.add_transition((code2, code2+1), final_state) -## else: -## s1.add_transition(c, final_state) - -## def calc_str(self): -## return "Char(%s)" % repr(self.char) - def Char(c): """ @@ -428,6 +392,7 @@ class SwitchCase(RE): name = "Case" return "%s(%s)" % (name, self.re) + # # Composite RE constructors # ------------------------- @@ -469,7 +434,6 @@ def Any(s): """ Any(s) is an RE which matches any character in the string |s|. """ - #result = apply(Alt, tuple(map(Char, s))) result = CodeRanges(chars_to_ranges(s)) result.str = "Any(%s)" % repr(s) return result @@ -549,6 +513,7 @@ def Case(re): """ return SwitchCase(re, nocase=0) + # # RE Constants # @@ -573,4 +538,3 @@ Eof.__doc__ = \ Eof is an RE which matches the end of the file. """ Eof.str = "Eof" - diff --git a/Cython/Plex/Scanners.pxd b/Cython/Plex/Scanners.pxd index 6e75f55e6..c6cb19b40 100644 --- a/Cython/Plex/Scanners.pxd +++ b/Cython/Plex/Scanners.pxd @@ -28,13 +28,11 @@ cdef class Scanner: cdef public level - @cython.final @cython.locals(input_state=long) - cdef next_char(self) + cdef inline next_char(self) @cython.locals(action=Action) cpdef tuple read(self) - @cython.final - cdef tuple scan_a_token(self) + cdef inline tuple scan_a_token(self) ##cdef tuple position(self) # used frequently by Parsing.py @cython.final @@ -44,7 +42,5 @@ cdef class Scanner: trace=bint, discard=Py_ssize_t, data=unicode, buffer=unicode) cdef run_machine_inlined(self) - @cython.final - cdef begin(self, state) - @cython.final - cdef produce(self, value, text = *) + cdef inline begin(self, state) + cdef inline produce(self, value, text = *) diff --git a/Cython/Plex/Scanners.py b/Cython/Plex/Scanners.py index 88f7e2da3..e850e0cc9 100644 --- a/Cython/Plex/Scanners.py +++ b/Cython/Plex/Scanners.py @@ -1,18 +1,15 @@ +# cython: language_level=3str # cython: auto_pickle=False -#======================================================================= -# -# Python Lexical Analyser -# -# -# Scanning an input stream -# -#======================================================================= +""" +Python Lexical Analyser +Scanning an input stream +""" from __future__ import absolute_import import cython -cython.declare(BOL=object, EOL=object, EOF=object, NOT_FOUND=object) +cython.declare(BOL=object, EOL=object, EOF=object, NOT_FOUND=object) # noqa:E402 from . import Errors from .Regexps import BOL, EOL, EOF @@ -173,26 +170,28 @@ class Scanner(object): buf_len = len(buffer) b_action, b_cur_pos, b_cur_line, b_cur_line_start, b_cur_char, b_input_state, b_next_pos = \ None, 0, 0, 0, u'', 0, 0 + trace = self.trace while 1: - if trace: #TRACE# - print("State %d, %d/%d:%s -->" % ( #TRACE# - state['number'], input_state, cur_pos, repr(cur_char))) #TRACE# + if trace: + print("State %d, %d/%d:%s -->" % ( + state['number'], input_state, cur_pos, repr(cur_char))) + # Begin inlined self.save_for_backup() - #action = state.action #@slow - action = state['action'] #@fast + action = state['action'] if action is not None: b_action, b_cur_pos, b_cur_line, b_cur_line_start, b_cur_char, b_input_state, b_next_pos = \ action, cur_pos, cur_line, cur_line_start, cur_char, input_state, next_pos # End inlined self.save_for_backup() + c = cur_char - #new_state = state.new_state(c) #@slow - new_state = state.get(c, NOT_FOUND) #@fast - if new_state is NOT_FOUND: #@fast - new_state = c and state.get('else') #@fast + new_state = state.get(c, NOT_FOUND) + if new_state is NOT_FOUND: + new_state = c and state.get('else') + if new_state: - if trace: #TRACE# - print("State %d" % new_state['number']) #TRACE# + if trace: + print("State %d" % new_state['number']) state = new_state # Begin inlined: self.next_char() if input_state == 1: @@ -240,8 +239,8 @@ class Scanner(object): cur_char = u'' # End inlined self.next_char() else: # not new_state - if trace: #TRACE# - print("blocked") #TRACE# + if trace: + print("blocked") # Begin inlined: action = self.back_up() if b_action is not None: (action, cur_pos, cur_line, cur_line_start, @@ -252,15 +251,16 @@ class Scanner(object): action = None break # while 1 # End inlined: action = self.back_up() + self.cur_pos = cur_pos self.cur_line = cur_line self.cur_line_start = cur_line_start self.cur_char = cur_char self.input_state = input_state self.next_pos = next_pos - if trace: #TRACE# - if action is not None: #TRACE# - print("Doing %s" % action) #TRACE# + if trace: + if action is not None: + print("Doing %s" % action) return action def next_char(self): @@ -306,7 +306,8 @@ class Scanner(object): return (self.name, self.start_line, self.start_col) def get_position(self): - """Python accessible wrapper around position(), only for error reporting. + """ + Python accessible wrapper around position(), only for error reporting. """ return self.position() @@ -336,3 +337,4 @@ class Scanner(object): Override this method if you want something to be done at end of file. """ + pass diff --git a/Cython/Plex/Timing.py b/Cython/Plex/Timing.py deleted file mode 100644 index 5c3692693..000000000 --- a/Cython/Plex/Timing.py +++ /dev/null @@ -1,23 +0,0 @@ -# -# Get time in platform-dependent way -# - -from __future__ import absolute_import - -import os -from sys import platform, exit, stderr - -if platform == 'mac': - import MacOS - def time(): - return MacOS.GetTicks() / 60.0 - timekind = "real" -elif hasattr(os, 'times'): - def time(): - t = os.times() - return t[0] + t[1] - timekind = "cpu" -else: - stderr.write( - "Don't know how to get time on platform %s\n" % repr(platform)) - exit(1) diff --git a/Cython/Plex/Traditional.py b/Cython/Plex/Traditional.py deleted file mode 100644 index ec7252dae..000000000 --- a/Cython/Plex/Traditional.py +++ /dev/null @@ -1,158 +0,0 @@ -#======================================================================= -# -# Python Lexical Analyser -# -# Traditional Regular Expression Syntax -# -#======================================================================= - -from __future__ import absolute_import - -from .Regexps import Alt, Seq, Rep, Rep1, Opt, Any, AnyBut, Bol, Eol, Char -from .Errors import PlexError - - -class RegexpSyntaxError(PlexError): - pass - - -def re(s): - """ - Convert traditional string representation of regular expression |s| - into Plex representation. - """ - return REParser(s).parse_re() - - -class REParser(object): - def __init__(self, s): - self.s = s - self.i = -1 - self.end = 0 - self.next() - - def parse_re(self): - re = self.parse_alt() - if not self.end: - self.error("Unexpected %s" % repr(self.c)) - return re - - def parse_alt(self): - """Parse a set of alternative regexps.""" - re = self.parse_seq() - if self.c == '|': - re_list = [re] - while self.c == '|': - self.next() - re_list.append(self.parse_seq()) - re = Alt(*re_list) - return re - - def parse_seq(self): - """Parse a sequence of regexps.""" - re_list = [] - while not self.end and not self.c in "|)": - re_list.append(self.parse_mod()) - return Seq(*re_list) - - def parse_mod(self): - """Parse a primitive regexp followed by *, +, ? modifiers.""" - re = self.parse_prim() - while not self.end and self.c in "*+?": - if self.c == '*': - re = Rep(re) - elif self.c == '+': - re = Rep1(re) - else: # self.c == '?' - re = Opt(re) - self.next() - return re - - def parse_prim(self): - """Parse a primitive regexp.""" - c = self.get() - if c == '.': - re = AnyBut("\n") - elif c == '^': - re = Bol - elif c == '$': - re = Eol - elif c == '(': - re = self.parse_alt() - self.expect(')') - elif c == '[': - re = self.parse_charset() - self.expect(']') - else: - if c == '\\': - c = self.get() - re = Char(c) - return re - - def parse_charset(self): - """Parse a charset. Does not include the surrounding [].""" - char_list = [] - invert = 0 - if self.c == '^': - invert = 1 - self.next() - if self.c == ']': - char_list.append(']') - self.next() - while not self.end and self.c != ']': - c1 = self.get() - if self.c == '-' and self.lookahead(1) != ']': - self.next() - c2 = self.get() - for a in range(ord(c1), ord(c2) + 1): - char_list.append(chr(a)) - else: - char_list.append(c1) - chars = ''.join(char_list) - if invert: - return AnyBut(chars) - else: - return Any(chars) - - def next(self): - """Advance to the next char.""" - s = self.s - i = self.i = self.i + 1 - if i < len(s): - self.c = s[i] - else: - self.c = '' - self.end = 1 - - def get(self): - if self.end: - self.error("Premature end of string") - c = self.c - self.next() - return c - - def lookahead(self, n): - """Look ahead n chars.""" - j = self.i + n - if j < len(self.s): - return self.s[j] - else: - return '' - - def expect(self, c): - """ - Expect to find character |c| at current position. - Raises an exception otherwise. - """ - if self.c == c: - self.next() - else: - self.error("Missing %s" % repr(c)) - - def error(self, mess): - """Raise exception to signal syntax error in regexp.""" - raise RegexpSyntaxError("Syntax error in regexp %s at position %d: %s" % ( - repr(self.s), self.i, mess)) - - - diff --git a/Cython/Plex/Transitions.pxd b/Cython/Plex/Transitions.pxd new file mode 100644 index 000000000..53dd4d58e --- /dev/null +++ b/Cython/Plex/Transitions.pxd @@ -0,0 +1,22 @@ +cimport cython + +cdef long maxint + +@cython.final +cdef class TransitionMap: + cdef list map + cdef dict special + + @cython.locals(i=cython.Py_ssize_t, j=cython.Py_ssize_t) + cpdef add(self, event, new_state) + + @cython.locals(i=cython.Py_ssize_t, j=cython.Py_ssize_t) + cpdef add_set(self, event, new_set) + + @cython.locals(i=cython.Py_ssize_t, n=cython.Py_ssize_t, else_set=cython.bint) + cpdef iteritems(self) + + @cython.locals(map=list, lo=cython.Py_ssize_t, mid=cython.Py_ssize_t, hi=cython.Py_ssize_t) + cdef split(self, long code) + + cdef get_special(self, event) diff --git a/Cython/Plex/Transitions.py b/Cython/Plex/Transitions.py index 383381794..f58dd538e 100644 --- a/Cython/Plex/Transitions.py +++ b/Cython/Plex/Transitions.py @@ -1,15 +1,11 @@ -# -# Plex - Transition Maps -# -# This version represents state sets directly as dicts for speed. -# +# cython: auto_pickle=False +""" +Plex - Transition Maps -from __future__ import absolute_import +This version represents state sets directly as dicts for speed. +""" -try: - from sys import maxsize as maxint -except ImportError: - from sys import maxint +maxint = 2**31-1 # sentinel value class TransitionMap(object): @@ -40,24 +36,19 @@ class TransitionMap(object): kept separately in a dictionary. """ - map = None # The list of codes and states - special = None # Mapping for special events - def __init__(self, map=None, special=None): if not map: map = [-maxint, {}, maxint] if not special: special = {} - self.map = map - self.special = special - #self.check() ### + self.map = map # The list of codes and states + self.special = special # Mapping for special events - def add(self, event, new_state, - TupleType=tuple): + def add(self, event, new_state): """ Add transition to |new_state| on |event|. """ - if type(event) is TupleType: + if type(event) is tuple: code0, code1 = event i = self.split(code0) j = self.split(code1) @@ -68,12 +59,11 @@ class TransitionMap(object): else: self.get_special(event)[new_state] = 1 - def add_set(self, event, new_set, - TupleType=tuple): + def add_set(self, event, new_set): """ Add transitions to the states in |new_set| on |event|. """ - if type(event) is TupleType: + if type(event) is tuple: code0, code1 = event i = self.split(code0) j = self.split(code1) @@ -84,15 +74,13 @@ class TransitionMap(object): else: self.get_special(event).update(new_set) - def get_epsilon(self, - none=None): + def get_epsilon(self): """ Return the mapping for epsilon, or None. """ - return self.special.get('', none) + return self.special.get('') - def iteritems(self, - len=len): + def iteritems(self): """ Return the mapping as an iterable of ((code1, code2), state_set) and (special_event, state_set) pairs. @@ -119,8 +107,7 @@ class TransitionMap(object): # ------------------- Private methods -------------------- - def split(self, code, - len=len, maxint=maxint): + def split(self, code): """ Search the list for the position of the split point for |code|, inserting a new split point if necessary. Returns index |i| such @@ -132,6 +119,7 @@ class TransitionMap(object): # Special case: code == map[-1] if code == maxint: return hi + # General case lo = 0 # loop invariant: map[lo] <= code < map[hi] and hi - lo >= 2 @@ -147,7 +135,6 @@ class TransitionMap(object): return lo else: map[hi:hi] = [code, map[hi - 1].copy()] - #self.check() ### return hi def get_special(self, event): @@ -243,9 +230,5 @@ class TransitionMap(object): # State set manipulation functions # -#def merge_state_sets(set1, set2): -# for state in set2.keys(): -# set1[state] = 1 - def state_set_str(set): return "[%s]" % ','.join(["S%d" % state.number for state in set]) diff --git a/Cython/Plex/__init__.py b/Cython/Plex/__init__.py index 81a066f78..83bb9239a 100644 --- a/Cython/Plex/__init__.py +++ b/Cython/Plex/__init__.py @@ -1,10 +1,6 @@ -#======================================================================= -# -# Python Lexical Analyser -# -#======================================================================= - """ +Python Lexical Analyser + The Plex module provides lexical analysers with similar capabilities to GNU Flex. The following classes and functions are exported; see the attached docstrings for more information. @@ -29,10 +25,10 @@ see the attached docstrings for more information. Actions for associating with patterns when creating a Lexicon. """ - +# flake8: noqa:F401 from __future__ import absolute_import -from .Actions import TEXT, IGNORE, Begin +from .Actions import TEXT, IGNORE, Begin, Method from .Lexicons import Lexicon, State from .Regexps import RE, Seq, Alt, Rep1, Empty, Str, Any, AnyBut, AnyChar, Range from .Regexps import Opt, Rep, Bol, Eol, Eof, Case, NoCase diff --git a/Cython/Runtime/refnanny.pyx b/Cython/Runtime/refnanny.pyx index d4b873fe9..5b68923f4 100644 --- a/Cython/Runtime/refnanny.pyx +++ b/Cython/Runtime/refnanny.pyx @@ -1,6 +1,6 @@ # cython: language_level=3, auto_pickle=False -from cpython.ref cimport PyObject, Py_INCREF, Py_DECREF, Py_XDECREF, Py_XINCREF +from cpython.ref cimport PyObject, Py_INCREF, Py_CLEAR, Py_XDECREF, Py_XINCREF from cpython.exc cimport PyErr_Fetch, PyErr_Restore from cpython.pystate cimport PyThreadState_Get @@ -10,6 +10,9 @@ loglevel = 0 reflog = [] cdef log(level, action, obj, lineno): + if reflog is None: + # can happen during finalisation + return if loglevel >= level: reflog.append((lineno, action, id(obj))) @@ -29,7 +32,7 @@ cdef class Context(object): self.refs = {} # id -> (count, [lineno]) self.errors = [] - cdef regref(self, obj, lineno, bint is_null): + cdef regref(self, obj, Py_ssize_t lineno, bint is_null): log(LOG_ALL, u'regref', u"<NULL>" if is_null else obj, lineno) if is_null: self.errors.append(f"NULL argument on line {lineno}") @@ -39,7 +42,7 @@ cdef class Context(object): self.refs[id_] = (count + 1, linenumbers) linenumbers.append(lineno) - cdef bint delref(self, obj, lineno, bint is_null) except -1: + cdef bint delref(self, obj, Py_ssize_t lineno, bint is_null) except -1: # returns whether it is ok to do the decref operation log(LOG_ALL, u'delref', u"<NULL>" if is_null else obj, lineno) if is_null: @@ -50,12 +53,11 @@ cdef class Context(object): if count == 0: self.errors.append(f"Too many decrefs on line {lineno}, reference acquired on lines {linenumbers!r}") return False - elif count == 1: + if count == 1: del self.refs[id_] - return True else: self.refs[id_] = (count - 1, linenumbers) - return True + return True cdef end(self): if self.refs: @@ -63,121 +65,117 @@ cdef class Context(object): for count, linenos in self.refs.itervalues(): msg += f"\n ({count}) acquired on lines: {u', '.join([f'{x}' for x in linenos])}" self.errors.append(msg) - if self.errors: - return u"\n".join([u'REFNANNY: '+error for error in self.errors]) - else: - return None + return u"\n".join([f'REFNANNY: {error}' for error in self.errors]) if self.errors else None + -cdef void report_unraisable(object e=None): +cdef void report_unraisable(filename, Py_ssize_t lineno, object e=None): try: if e is None: import sys e = sys.exc_info()[1] - print(f"refnanny raised an exception: {e}") - except: - pass # We absolutely cannot exit with an exception + print(f"refnanny raised an exception from {filename}:{lineno}: {e}") + finally: + return # We absolutely cannot exit with an exception + # All Python operations must happen after any existing # exception has been fetched, in case we are called from # exception-handling code. -cdef PyObject* SetupContext(char* funcname, int lineno, char* filename) except NULL: +cdef PyObject* SetupContext(char* funcname, Py_ssize_t lineno, char* filename) except NULL: if Context is None: # Context may be None during finalize phase. # In that case, we don't want to be doing anything fancy # like caching and resetting exceptions. return NULL cdef (PyObject*) type = NULL, value = NULL, tb = NULL, result = NULL - PyThreadState_Get() + PyThreadState_Get() # Check that we hold the GIL PyErr_Fetch(&type, &value, &tb) try: ctx = Context(funcname, lineno, filename) Py_INCREF(ctx) result = <PyObject*>ctx except Exception, e: - report_unraisable(e) + report_unraisable(filename, lineno, e) PyErr_Restore(type, value, tb) return result -cdef void GOTREF(PyObject* ctx, PyObject* p_obj, int lineno): +cdef void GOTREF(PyObject* ctx, PyObject* p_obj, Py_ssize_t lineno): if ctx == NULL: return cdef (PyObject*) type = NULL, value = NULL, tb = NULL PyErr_Fetch(&type, &value, &tb) try: - try: - if p_obj is NULL: - (<Context>ctx).regref(None, lineno, True) - else: - (<Context>ctx).regref(<object>p_obj, lineno, False) - except: - report_unraisable() + (<Context>ctx).regref( + <object>p_obj if p_obj is not NULL else None, + lineno, + is_null=p_obj is NULL, + ) except: - # __Pyx_GetException may itself raise errors - pass - PyErr_Restore(type, value, tb) + report_unraisable((<Context>ctx).filename, lineno=(<Context>ctx).start) + finally: + PyErr_Restore(type, value, tb) + return # swallow any exceptions -cdef int GIVEREF_and_report(PyObject* ctx, PyObject* p_obj, int lineno): +cdef bint GIVEREF_and_report(PyObject* ctx, PyObject* p_obj, Py_ssize_t lineno): if ctx == NULL: return 1 cdef (PyObject*) type = NULL, value = NULL, tb = NULL cdef bint decref_ok = False PyErr_Fetch(&type, &value, &tb) try: - try: - if p_obj is NULL: - decref_ok = (<Context>ctx).delref(None, lineno, True) - else: - decref_ok = (<Context>ctx).delref(<object>p_obj, lineno, False) - except: - report_unraisable() + decref_ok = (<Context>ctx).delref( + <object>p_obj if p_obj is not NULL else None, + lineno, + is_null=p_obj is NULL, + ) except: - # __Pyx_GetException may itself raise errors - pass - PyErr_Restore(type, value, tb) - return decref_ok + report_unraisable((<Context>ctx).filename, lineno=(<Context>ctx).start) + finally: + PyErr_Restore(type, value, tb) + return decref_ok # swallow any exceptions -cdef void GIVEREF(PyObject* ctx, PyObject* p_obj, int lineno): +cdef void GIVEREF(PyObject* ctx, PyObject* p_obj, Py_ssize_t lineno): GIVEREF_and_report(ctx, p_obj, lineno) -cdef void INCREF(PyObject* ctx, PyObject* obj, int lineno): +cdef void INCREF(PyObject* ctx, PyObject* obj, Py_ssize_t lineno): Py_XINCREF(obj) - PyThreadState_Get() + PyThreadState_Get() # Check that we hold the GIL GOTREF(ctx, obj, lineno) -cdef void DECREF(PyObject* ctx, PyObject* obj, int lineno): +cdef void DECREF(PyObject* ctx, PyObject* obj, Py_ssize_t lineno): if GIVEREF_and_report(ctx, obj, lineno): Py_XDECREF(obj) - PyThreadState_Get() + PyThreadState_Get() # Check that we hold the GIL cdef void FinishContext(PyObject** ctx): if ctx == NULL or ctx[0] == NULL: return cdef (PyObject*) type = NULL, value = NULL, tb = NULL cdef object errors = None cdef Context context - PyThreadState_Get() + PyThreadState_Get() # Check that we hold the GIL PyErr_Fetch(&type, &value, &tb) try: - try: - context = <Context>ctx[0] - errors = context.end() - if errors: - print(f"{context.filename.decode('latin1')}: {context.name.decode('latin1')}()") - print(errors) - context = None - except: - report_unraisable() + context = <Context>ctx[0] + errors = context.end() + if errors: + print(f"{context.filename.decode('latin1')}: {context.name.decode('latin1')}()") + print(errors) + context = None except: - # __Pyx_GetException may itself raise errors - pass - Py_XDECREF(ctx[0]) - ctx[0] = NULL - PyErr_Restore(type, value, tb) + report_unraisable( + context.filename if context is not None else None, + lineno=context.start if context is not None else 0, + ) + finally: + Py_CLEAR(ctx[0]) + PyErr_Restore(type, value, tb) + return # swallow any exceptions ctypedef struct RefNannyAPIStruct: - void (*INCREF)(PyObject*, PyObject*, int) - void (*DECREF)(PyObject*, PyObject*, int) - void (*GOTREF)(PyObject*, PyObject*, int) - void (*GIVEREF)(PyObject*, PyObject*, int) - PyObject* (*SetupContext)(char*, int, char*) except NULL + void (*INCREF)(PyObject*, PyObject*, Py_ssize_t) + void (*DECREF)(PyObject*, PyObject*, Py_ssize_t) + void (*GOTREF)(PyObject*, PyObject*, Py_ssize_t) + void (*GIVEREF)(PyObject*, PyObject*, Py_ssize_t) + PyObject* (*SetupContext)(char*, Py_ssize_t, char*) except NULL void (*FinishContext)(PyObject**) cdef RefNannyAPIStruct api diff --git a/Cython/Shadow.py b/Cython/Shadow.py index ddd2a311c..ae60bb0d2 100644 --- a/Cython/Shadow.py +++ b/Cython/Shadow.py @@ -1,7 +1,7 @@ # cython.* namespace for pure mode. from __future__ import absolute_import -__version__ = "0.29.23" +__version__ = "3.0a6" try: from __builtin__ import basestring @@ -71,7 +71,7 @@ def index_type(base_type, item): else: # int[8] etc. assert int(item) == item # array size must be a plain integer - array(base_type, item) + return array(base_type, item) # END shameless copy @@ -146,27 +146,33 @@ def compile(f): # Special functions def cdiv(a, b): - q = a / b - if q < 0: - q += 1 - return q + if a < 0: + a = -a + b = -b + if b < 0: + return (a + b + 1) // b + return a // b def cmod(a, b): r = a % b - if (a*b) < 0: + if (a * b) < 0 and r: r -= b return r # Emulated language constructs -def cast(type, *args, **kwargs): +def cast(t, *args, **kwargs): kwargs.pop('typecheck', None) assert not kwargs - if hasattr(type, '__call__'): - return type(*args) - else: - return args[0] + + if isinstance(t, typedef): + return t(*args) + elif isinstance(t, type): # Doesn't work with old-style classes of Python 2.x + if len(args) != 1 or not (args[0] is None or isinstance(args[0], t)): + return t(*args) + + return args[0] def sizeof(arg): return 1 @@ -178,14 +184,19 @@ def typeof(arg): def address(arg): return pointer(type(arg))([arg]) -def declare(type=None, value=_Unspecified, **kwds): - if type not in (None, object) and hasattr(type, '__call__'): - if value is not _Unspecified: - return type(value) - else: - return type() +def _is_value_type(t): + if isinstance(t, typedef): + return _is_value_type(t._basetype) + + return isinstance(t, type) and issubclass(t, (StructType, UnionType, ArrayType)) + +def declare(t=None, value=_Unspecified, **kwds): + if value is not _Unspecified: + return cast(t, value) + elif _is_value_type(t): + return t() else: - return value + return None class _nogil(object): """Support for 'with nogil' statement and @nogil decorator. @@ -258,8 +269,11 @@ class PointerType(CythonType): class ArrayType(PointerType): - def __init__(self): - self._items = [None] * self._n + def __init__(self, value=None): + if value is None: + self._items = [None] * self._n + else: + super(ArrayType, self).__init__(value) class StructType(CythonType): @@ -270,7 +284,7 @@ class StructType(CythonType): if len(data) > 0: raise ValueError('Cannot accept keyword arguments when casting.') if type(cast_from) is not type(self): - raise ValueError('Cannot cast from %s'%cast_from) + raise ValueError('Cannot cast from %s' % cast_from) for key, value in cast_from.__dict__.items(): setattr(self, key, value) else: @@ -296,7 +310,7 @@ class UnionType(CythonType): elif type(cast_from) is type(self): datadict = cast_from.__dict__ else: - raise ValueError('Cannot cast from %s'%cast_from) + raise ValueError('Cannot cast from %s' % cast_from) else: datadict = data if len(datadict) > 1: @@ -393,10 +407,34 @@ py_complex = typedef(complex, "double complex") # Predefined types -int_types = ['char', 'short', 'Py_UNICODE', 'int', 'Py_UCS4', 'long', 'longlong', 'Py_ssize_t', 'size_t'] -float_types = ['longdouble', 'double', 'float'] -complex_types = ['longdoublecomplex', 'doublecomplex', 'floatcomplex', 'complex'] -other_types = ['bint', 'void', 'Py_tss_t'] +int_types = [ + 'char', + 'short', + 'Py_UNICODE', + 'int', + 'Py_UCS4', + 'long', + 'longlong', + 'Py_hash_t', + 'Py_ssize_t', + 'size_t', +] +float_types = [ + 'longdouble', + 'double', + 'float', +] +complex_types = [ + 'longdoublecomplex', + 'doublecomplex', + 'floatcomplex', + 'complex', +] +other_types = [ + 'bint', + 'void', + 'Py_tss_t', +] to_repr = { 'longlong': 'long long', diff --git a/Cython/StringIOTree.py b/Cython/StringIOTree.py index d8239efed..a58fcc935 100644 --- a/Cython/StringIOTree.py +++ b/Cython/StringIOTree.py @@ -39,6 +39,7 @@ try: from cStringIO import StringIO except ImportError: from io import StringIO +import sys class StringIOTree(object): @@ -106,3 +107,44 @@ class StringIOTree(object): def allmarkers(self): children = self.prepended_children return [m for c in children for m in c.allmarkers()] + self.markers + + # Print the result of allmarkers in a nice human-readable form. Use it only for debugging. + # Prints e.g. + # /path/to/source.pyx: + # cython line 2 maps to 3299-3343 + # cython line 4 maps to 2236-2245 2306 3188-3201 + # /path/to/othersource.pyx: + # cython line 3 maps to 1234-1270 + # ... + # Note: In the example above, 3343 maps to line 2, 3344 does not. + def print_hr_allmarkers(self): + from collections import defaultdict + markers = self.allmarkers() + totmap = defaultdict(lambda: defaultdict(list)) + for c_lineno, (cython_desc, cython_lineno) in enumerate(markers): + if cython_lineno > 0 and cython_desc.filename is not None: + totmap[cython_desc.filename][cython_lineno].append(c_lineno + 1) + reprstr = "" + if totmap == 0: + reprstr += "allmarkers is empty\n" + try: + sorted(totmap.items()) + except: + print(totmap) + print(totmap.items()) + for cython_path, filemap in sorted(totmap.items()): + reprstr += cython_path + ":\n" + for cython_lineno, c_linenos in sorted(filemap.items()): + reprstr += "\tcython line " + str(cython_lineno) + " maps to " + i = 0 + while i < len(c_linenos): + reprstr += str(c_linenos[i]) + flag = False + while i+1 < len(c_linenos) and c_linenos[i+1] == c_linenos[i]+1: + i += 1 + flag = True + if flag: + reprstr += "-" + str(c_linenos[i]) + " " + i += 1 + reprstr += "\n" + sys.stdout.write(reprstr) diff --git a/Cython/Tempita/_tempita.py b/Cython/Tempita/_tempita.py index 22a7d233b..c72e76fe2 100644 --- a/Cython/Tempita/_tempita.py +++ b/Cython/Tempita/_tempita.py @@ -1,3 +1,5 @@ +# cython: language_level=3str + """ A small templating language @@ -144,9 +146,8 @@ class Template(object): def from_filename(cls, filename, namespace=None, encoding=None, default_inherit=None, get_template=get_file_template): - f = open(filename, 'rb') - c = f.read() - f.close() + with open(filename, 'rb') as f: + c = f.read() if encoding: c = c.decode(encoding) return cls(content=c, name=filename, namespace=namespace, @@ -335,7 +336,7 @@ class Template(object): if not isinstance(value, basestring_): value = coerce_text(value) if (is_unicode(value) - and self.default_encoding): + and self.default_encoding): value = value.encode(self.default_encoding) except Exception as e: e.args = (self._add_line_info(e.args[0], pos),) @@ -723,7 +724,7 @@ def trim_lex(tokens): else: next_chunk = tokens[i + 1] if (not isinstance(next_chunk, basestring_) - or not isinstance(prev, basestring_)): + or not isinstance(prev, basestring_)): continue prev_ok = not prev or trail_whitespace_re.search(prev) if i == 1 and not prev.strip(): @@ -735,7 +736,7 @@ def trim_lex(tokens): or (i == len(tokens) - 2 and not next_chunk.strip()))): if prev: if ((i == 1 and not prev.strip()) - or prev_ok == 'last'): + or prev_ok == 'last'): tokens[i - 1] = '' else: m = trail_whitespace_re.search(prev) @@ -887,7 +888,7 @@ def parse_cond(tokens, name, context): 'Missing {{endif}}', position=start, name=name) if (isinstance(tokens[0], tuple) - and tokens[0][0] == 'endif'): + and tokens[0][0] == 'endif'): return ('cond', start) + tuple(pieces), tokens[1:] next_chunk, tokens = parse_one_cond(tokens, name, context) pieces.append(next_chunk) @@ -949,7 +950,7 @@ def parse_for(tokens, name, context): 'No {{endfor}}', position=pos, name=name) if (isinstance(tokens[0], tuple) - and tokens[0][0] == 'endfor'): + and tokens[0][0] == 'endfor'): return ('for', pos, vars, expr, content), tokens[1:] next_chunk, tokens = parse_expr(tokens, name, context) content.append(next_chunk) @@ -1009,7 +1010,7 @@ def parse_def(tokens, name, context): 'Missing {{enddef}}', position=start, name=name) if (isinstance(tokens[0], tuple) - and tokens[0][0] == 'enddef'): + and tokens[0][0] == 'enddef'): return ('def', start, func_name, sig, content), tokens[1:] next_chunk, tokens = parse_expr(tokens, name, context) content.append(next_chunk) @@ -1072,7 +1073,7 @@ def parse_signature(sig_text, name, pos): raise TemplateError('Invalid signature: (%s)' % sig_text, position=pos, name=name) if (not nest_count and - (tok_type == tokenize.ENDMARKER or (tok_type == tokenize.OP and tok_string == ','))): + (tok_type == tokenize.ENDMARKER or (tok_type == tokenize.OP and tok_string == ','))): default_expr = isolate_expression(sig_text, start_pos, end_pos) defaults[var_name] = default_expr sig_args.append(var_name) @@ -1162,9 +1163,8 @@ def fill_command(args=None): template_content = sys.stdin.read() template_name = '<stdin>' else: - f = open(template_name, 'rb') - template_content = f.read() - f.close() + with open(template_name, 'rb') as f: + template_content = f.read() if options.use_html: TemplateClass = HTMLTemplate else: @@ -1172,9 +1172,8 @@ def fill_command(args=None): template = TemplateClass(template_content, name=template_name) result = template.substitute(vars) if options.output: - f = open(options.output, 'wb') - f.write(result) - f.close() + with open(options.output, 'wb') as f: + f.write(result) else: sys.stdout.write(result) diff --git a/Cython/TestUtils.py b/Cython/TestUtils.py index 9d6eb67fc..ba24490ce 100644 --- a/Cython/TestUtils.py +++ b/Cython/TestUtils.py @@ -2,7 +2,10 @@ from __future__ import absolute_import import os import unittest +import shlex +import sys import tempfile +from io import open from .Compiler import Errors from .CodeWriter import CodeWriter @@ -177,41 +180,48 @@ class TreeAssertVisitor(VisitorTransform): if TreePath.find_first(node, path) is not None: Errors.error( node.pos, - "Unexpected path '%s' found in result tree" % path) + "Unexpected path '%s' found in result tree" % path) self.visitchildren(node) return node visit_Node = VisitorTransform.recurse_to_children -def unpack_source_tree(tree_file, dir=None): - if dir is None: - dir = tempfile.mkdtemp() - header = [] - cur_file = None - f = open(tree_file) - try: - lines = f.readlines() - finally: - f.close() - del f - try: - for line in lines: - if line[:5] == '#####': - filename = line.strip().strip('#').strip().replace('/', os.path.sep) - path = os.path.join(dir, filename) - if not os.path.exists(os.path.dirname(path)): - os.makedirs(os.path.dirname(path)) - if cur_file is not None: - f, cur_file = cur_file, None - f.close() - cur_file = open(path, 'w') - elif cur_file is not None: - cur_file.write(line) - elif line.strip() and not line.lstrip().startswith('#'): - if line.strip() not in ('"""', "'''"): - header.append(line) - finally: - if cur_file is not None: - cur_file.close() - return dir, ''.join(header) +def unpack_source_tree(tree_file, workdir, cython_root): + programs = { + 'PYTHON': [sys.executable], + 'CYTHON': [sys.executable, os.path.join(cython_root, 'cython.py')], + 'CYTHONIZE': [sys.executable, os.path.join(cython_root, 'cythonize.py')] + } + + if workdir is None: + workdir = tempfile.mkdtemp() + header, cur_file = [], None + with open(tree_file, 'rb') as f: + try: + for line in f: + if line[:5] == b'#####': + filename = line.strip().strip(b'#').strip().decode('utf8').replace('/', os.path.sep) + path = os.path.join(workdir, filename) + if not os.path.exists(os.path.dirname(path)): + os.makedirs(os.path.dirname(path)) + if cur_file is not None: + to_close, cur_file = cur_file, None + to_close.close() + cur_file = open(path, 'wb') + elif cur_file is not None: + cur_file.write(line) + elif line.strip() and not line.lstrip().startswith(b'#'): + if line.strip() not in (b'"""', b"'''"): + command = shlex.split(line.decode('utf8')) + if not command: continue + # In Python 3: prog, *args = command + prog, args = command[0], command[1:] + try: + header.append(programs[prog]+args) + except KeyError: + header.append(command) + finally: + if cur_file is not None: + cur_file.close() + return workdir, header diff --git a/Cython/Tests/TestCodeWriter.py b/Cython/Tests/TestCodeWriter.py index 42e457da2..c3026cb1d 100644 --- a/Cython/Tests/TestCodeWriter.py +++ b/Cython/Tests/TestCodeWriter.py @@ -19,9 +19,9 @@ class TestCodeWriter(CythonTest): def test_print(self): self.t(u""" - print x, y - print x + y ** 2 - print x, y, z, + print(x + y ** 2) + print(x, y, z) + print(x + y, x + y * z, x * (y + z)) """) def test_if(self): @@ -47,6 +47,20 @@ class TestCodeWriter(CythonTest): pass """) + def test_cdef(self): + self.t(u""" + cdef f(x, y, z): + pass + cdef public void (x = 34, y = 54, z): + pass + cdef f(int *x, void *y, Value *z): + pass + cdef f(int **x, void **y, Value **z): + pass + cdef inline f(int &x, Value &z): + pass + """) + def test_longness_and_signedness(self): self.t(u"def f(unsigned long long long long long int y):\n pass") @@ -65,18 +79,50 @@ class TestCodeWriter(CythonTest): def test_for_loop(self): self.t(u""" for x, y, z in f(g(h(34) * 2) + 23): - print x, y, z + print(x, y, z) + else: + print(43) + """) + self.t(u""" + for abc in (1, 2, 3): + print(x, y, z) else: - print 43 + print(43) + """) + + def test_while_loop(self): + self.t(u""" + while True: + while True: + while True: + continue """) def test_inplace_assignment(self): self.t(u"x += 43") + def test_cascaded_assignment(self): + self.t(u"x = y = z = abc = 43") + def test_attribute(self): self.t(u"a.x") + def test_return_none(self): + self.t(u""" + def f(x, y, z): + return + cdef f(x, y, z): + return + def f(x, y, z): + return None + cdef f(x, y, z): + return None + def f(x, y, z): + return 1234 + cdef f(x, y, z): + return 1234 + """) + if __name__ == "__main__": import unittest unittest.main() - diff --git a/Cython/Tests/TestJediTyper.py b/Cython/Tests/TestJediTyper.py index 253adef17..ede99b3a8 100644 --- a/Cython/Tests/TestJediTyper.py +++ b/Cython/Tests/TestJediTyper.py @@ -11,7 +11,7 @@ from contextlib import contextmanager from tempfile import NamedTemporaryFile from Cython.Compiler.ParseTreeTransforms import NormalizeTree, InterpretCompilerDirectives -from Cython.Compiler import Main, Symtab, Visitor +from Cython.Compiler import Main, Symtab, Visitor, Options from Cython.TestUtils import TransformTest TOOLS_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', 'Tools')) @@ -210,8 +210,8 @@ class TestTypeInjection(TestJediTyper): """ def setUp(self): super(TestTypeInjection, self).setUp() - compilation_options = Main.CompilationOptions(Main.default_options) - ctx = compilation_options.create_context() + compilation_options = Options.CompilationOptions(Options.default_options) + ctx = Main.Context.from_options(compilation_options) transform = InterpretCompilerDirectives(ctx, ctx.compiler_directives) transform.module_scope = Symtab.ModuleScope('__main__', None, ctx) self.declarations_finder = DeclarationsFinder() diff --git a/Cython/Tests/xmlrunner.py b/Cython/Tests/xmlrunner.py index 665f3c241..a02d9d8c0 100644 --- a/Cython/Tests/xmlrunner.py +++ b/Cython/Tests/xmlrunner.py @@ -27,12 +27,12 @@ class TestSequenceFunctions(unittest.TestCase): def test_choice(self): element = random.choice(self.seq) - self.assert_(element in self.seq) + self.assertTrue(element in self.seq) def test_sample(self): self.assertRaises(ValueError, random.sample, self.seq, 20) for element in random.sample(self.seq, 5): - self.assert_(element in self.seq) + self.assertTrue(element in self.seq) if __name__ == '__main__': unittest.main(testRunner=xmlrunner.XMLTestRunner(output='test-reports')) @@ -109,8 +109,7 @@ class _XMLTestResult(_TextTestResult): self.elapsed_times = elapsed_times self.output_patched = False - def _prepare_callback(self, test_info, target_list, verbose_str, - short_str): + def _prepare_callback(self, test_info, target_list, verbose_str, short_str): """Append a _TestInfo to the given target list and sets a callback method to be called by stopTest method. """ @@ -125,7 +124,7 @@ class _XMLTestResult(_TextTestResult): self.start_time = self.stop_time = 0 if self.showAll: - self.stream.writeln('(%.3fs) %s' % \ + self.stream.writeln('(%.3fs) %s' % (test_info.get_elapsed_time(), verbose_str)) elif self.dots: self.stream.write(short_str) @@ -300,8 +299,7 @@ class _XMLTestResult(_TextTestResult): "Generates the XML reports to a given XMLTestRunner object." all_results = self._get_info_by_testcase() - if type(test_runner.output) == str and not \ - os.path.exists(test_runner.output): + if isinstance(test_runner.output, str) and not os.path.exists(test_runner.output): os.makedirs(test_runner.output) for suite, tests in all_results.items(): @@ -321,7 +319,7 @@ class _XMLTestResult(_TextTestResult): xml_content = doc.toprettyxml(indent='\t') if type(test_runner.output) is str: - report_file = open('%s%sTEST-%s.xml' % \ + report_file = open('%s%sTEST-%s.xml' % (test_runner.output, os.sep, suite), 'w') try: report_file.write(xml_content) @@ -348,7 +346,7 @@ class XMLTestRunner(TextTestRunner): """Create the TestResult object which will be used to store information about the executed tests. """ - return _XMLTestResult(self.stream, self.descriptions, \ + return _XMLTestResult(self.stream, self.descriptions, self.verbosity, self.elapsed_times) def run(self, test): diff --git a/Cython/Utility/AsyncGen.c b/Cython/Utility/AsyncGen.c index 4e952185b..ab2dbdb90 100644 --- a/Cython/Utility/AsyncGen.c +++ b/Cython/Utility/AsyncGen.c @@ -11,6 +11,7 @@ typedef struct { PyObject *ag_finalizer; int ag_hooks_inited; int ag_closed; + int ag_running_async; } __pyx_PyAsyncGenObject; static PyTypeObject *__pyx__PyAsyncGenWrappedValueType = 0; @@ -18,11 +19,11 @@ static PyTypeObject *__pyx__PyAsyncGenASendType = 0; static PyTypeObject *__pyx__PyAsyncGenAThrowType = 0; static PyTypeObject *__pyx_AsyncGenType = 0; -#define __Pyx_AsyncGen_CheckExact(obj) (Py_TYPE(obj) == __pyx_AsyncGenType) +#define __Pyx_AsyncGen_CheckExact(obj) __Pyx_IS_TYPE(obj, __pyx_AsyncGenType) #define __pyx_PyAsyncGenASend_CheckExact(o) \ - (Py_TYPE(o) == __pyx__PyAsyncGenASendType) + __Pyx_IS_TYPE(o, __pyx__PyAsyncGenASendType) #define __pyx_PyAsyncGenAThrow_CheckExact(o) \ - (Py_TYPE(o) == __pyx__PyAsyncGenAThrowType) + __Pyx_IS_TYPE(o, __pyx__PyAsyncGenAThrowType) static PyObject *__Pyx_async_gen_anext(PyObject *o); static CYTHON_INLINE PyObject *__Pyx_async_gen_asend_iternext(PyObject *o); @@ -42,6 +43,7 @@ static __pyx_CoroutineObject *__Pyx_AsyncGen_New( gen->ag_finalizer = NULL; gen->ag_closed = 0; gen->ag_hooks_inited = 0; + gen->ag_running_async = 0; return __Pyx__Coroutine_NewInit((__pyx_CoroutineObject*)gen, body, code, closure, name, qualname, module_name); } @@ -127,6 +129,8 @@ static PyObject *__Pyx_async_gen_athrow_new(__pyx_PyAsyncGenObject *, PyObject * static const char *__Pyx_NON_INIT_CORO_MSG = "can't send non-None value to a just-started coroutine"; static const char *__Pyx_ASYNC_GEN_IGNORED_EXIT_MSG = "async generator ignored GeneratorExit"; +static const char *__Pyx_ASYNC_GEN_CANNOT_REUSE_SEND_MSG = "cannot reuse already awaited __anext__()/asend()"; +static const char *__Pyx_ASYNC_GEN_CANNOT_REUSE_CLOSE_MSG = "cannot reuse already awaited aclose()/athrow()"; typedef enum { __PYX_AWAITABLE_STATE_INIT, /* new awaitable, has not yet been iterated */ @@ -178,7 +182,7 @@ static __pyx_PyAsyncGenASend *__Pyx_ag_asend_freelist[_PyAsyncGen_MAXFREELIST]; static int __Pyx_ag_asend_freelist_free = 0; #define __pyx__PyAsyncGenWrappedValue_CheckExact(o) \ - (Py_TYPE(o) == __pyx__PyAsyncGenWrappedValueType) + __Pyx_IS_TYPE(o, __pyx__PyAsyncGenWrappedValueType) static int @@ -253,7 +257,7 @@ static PyObject * __Pyx_async_gen_anext(PyObject *g) { __pyx_PyAsyncGenObject *o = (__pyx_PyAsyncGenObject*) g; - if (__Pyx_async_gen_init_hooks(o)) { + if (unlikely(__Pyx_async_gen_init_hooks(o))) { return NULL; } return __Pyx_async_gen_asend_new(o, NULL); @@ -268,7 +272,7 @@ __Pyx_async_gen_anext_method(PyObject *g, CYTHON_UNUSED PyObject *arg) { static PyObject * __Pyx_async_gen_asend(__pyx_PyAsyncGenObject *o, PyObject *arg) { - if (__Pyx_async_gen_init_hooks(o)) { + if (unlikely(__Pyx_async_gen_init_hooks(o))) { return NULL; } return __Pyx_async_gen_asend_new(o, arg); @@ -278,7 +282,7 @@ __Pyx_async_gen_asend(__pyx_PyAsyncGenObject *o, PyObject *arg) static PyObject * __Pyx_async_gen_aclose(__pyx_PyAsyncGenObject *o, CYTHON_UNUSED PyObject *arg) { - if (__Pyx_async_gen_init_hooks(o)) { + if (unlikely(__Pyx_async_gen_init_hooks(o))) { return NULL; } return __Pyx_async_gen_athrow_new(o, NULL); @@ -288,7 +292,7 @@ __Pyx_async_gen_aclose(__pyx_PyAsyncGenObject *o, CYTHON_UNUSED PyObject *arg) static PyObject * __Pyx_async_gen_athrow(__pyx_PyAsyncGenObject *o, PyObject *args) { - if (__Pyx_async_gen_init_hooks(o)) { + if (unlikely(__Pyx_async_gen_init_hooks(o))) { return NULL; } return __Pyx_async_gen_athrow_new(o, args); @@ -313,7 +317,7 @@ static PyGetSetDef __Pyx_async_gen_getsetlist[] = { static PyMemberDef __Pyx_async_gen_memberlist[] = { //REMOVED: {(char*) "ag_frame", T_OBJECT, offsetof(__pyx_PyAsyncGenObject, ag_frame), READONLY}, - {(char*) "ag_running", T_BOOL, offsetof(__pyx_CoroutineObject, is_running), READONLY, NULL}, + {(char*) "ag_running", T_BOOL, offsetof(__pyx_PyAsyncGenObject, ag_running_async), READONLY, NULL}, //REMOVED: {(char*) "ag_code", T_OBJECT, offsetof(__pyx_PyAsyncGenObject, ag_code), READONLY}, //ADDED: "ag_await" {(char*) "ag_await", T_OBJECT, offsetof(__pyx_CoroutineObject, yieldfrom), READONLY, @@ -360,7 +364,7 @@ static PyTypeObject __pyx_AsyncGenType_type = { sizeof(__pyx_PyAsyncGenObject), /* tp_basicsize */ 0, /* tp_itemsize */ (destructor)__Pyx_Coroutine_dealloc, /* tp_dealloc */ - 0, /* tp_print */ + 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ #if CYTHON_USE_ASYNC_SLOTS @@ -427,6 +431,9 @@ static PyTypeObject __pyx_AsyncGenType_type = { #if PY_VERSION_HEX >= 0x030800b4 && PY_VERSION_HEX < 0x03090000 0, /*tp_print*/ #endif +#if CYTHON_COMPILING_IN_PYPY && PYPY_VERSION_NUM+0 >= 0x06000000 + 0, /*tp_pypy_flags*/ +#endif }; @@ -445,7 +452,7 @@ __Pyx_PyAsyncGen_ClearFreeLists(void) while (__Pyx_ag_asend_freelist_free) { __pyx_PyAsyncGenASend *o; o = __Pyx_ag_asend_freelist[--__Pyx_ag_asend_freelist_free]; - assert(Py_TYPE(o) == __pyx__PyAsyncGenASendType); + assert(__Pyx_IS_TYPE(o, __pyx__PyAsyncGenASendType)); PyObject_GC_Del(o); } @@ -471,6 +478,7 @@ __Pyx_async_gen_unwrap_value(__pyx_PyAsyncGenObject *gen, PyObject *result) gen->ag_closed = 1; } + gen->ag_running_async = 0; return NULL; } @@ -478,6 +486,7 @@ __Pyx_async_gen_unwrap_value(__pyx_PyAsyncGenObject *gen, PyObject *result) /* async yield */ __Pyx_ReturnWithStopIteration(((__pyx__PyAsyncGenWrappedValue*)result)->agw_val); Py_DECREF(result); + gen->ag_running_async = 0; return NULL; } @@ -494,7 +503,7 @@ __Pyx_async_gen_asend_dealloc(__pyx_PyAsyncGenASend *o) PyObject_GC_UnTrack((PyObject *)o); Py_CLEAR(o->ags_gen); Py_CLEAR(o->ags_sendval); - if (__Pyx_ag_asend_freelist_free < _PyAsyncGen_MAXFREELIST) { + if (likely(__Pyx_ag_asend_freelist_free < _PyAsyncGen_MAXFREELIST)) { assert(__pyx_PyAsyncGenASend_CheckExact(o)); __Pyx_ag_asend_freelist[__Pyx_ag_asend_freelist_free++] = o; } else { @@ -518,17 +527,25 @@ __Pyx_async_gen_asend_send(PyObject *g, PyObject *arg) PyObject *result; if (unlikely(o->ags_state == __PYX_AWAITABLE_STATE_CLOSED)) { - PyErr_SetNone(PyExc_StopIteration); + PyErr_SetString(PyExc_RuntimeError, __Pyx_ASYNC_GEN_CANNOT_REUSE_SEND_MSG); return NULL; } if (o->ags_state == __PYX_AWAITABLE_STATE_INIT) { + if (unlikely(o->ags_gen->ag_running_async)) { + PyErr_SetString( + PyExc_RuntimeError, + "anext(): asynchronous generator is already running"); + return NULL; + } + if (arg == NULL || arg == Py_None) { arg = o->ags_sendval ? o->ags_sendval : Py_None; } o->ags_state = __PYX_AWAITABLE_STATE_ITER; } + o->ags_gen->ag_running_async = 1; result = __Pyx_Coroutine_Send((PyObject*)o->ags_gen, arg); result = __Pyx_async_gen_unwrap_value(o->ags_gen, result); @@ -553,7 +570,7 @@ __Pyx_async_gen_asend_throw(__pyx_PyAsyncGenASend *o, PyObject *args) PyObject *result; if (unlikely(o->ags_state == __PYX_AWAITABLE_STATE_CLOSED)) { - PyErr_SetNone(PyExc_StopIteration); + PyErr_SetString(PyExc_RuntimeError, __Pyx_ASYNC_GEN_CANNOT_REUSE_SEND_MSG); return NULL; } @@ -602,7 +619,7 @@ static PyTypeObject __pyx__PyAsyncGenASendType_type = { 0, /* tp_itemsize */ /* methods */ (destructor)__Pyx_async_gen_asend_dealloc, /* tp_dealloc */ - 0, /* tp_print */ + 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ #if CYTHON_USE_ASYNC_SLOTS @@ -662,6 +679,9 @@ static PyTypeObject __pyx__PyAsyncGenASendType_type = { #if PY_VERSION_HEX >= 0x030800b4 && PY_VERSION_HEX < 0x03090000 0, /*tp_print*/ #endif +#if CYTHON_COMPILING_IN_PYPY && PYPY_VERSION_NUM+0 >= 0x06000000 + 0, /*tp_pypy_flags*/ +#endif }; @@ -669,13 +689,13 @@ static PyObject * __Pyx_async_gen_asend_new(__pyx_PyAsyncGenObject *gen, PyObject *sendval) { __pyx_PyAsyncGenASend *o; - if (__Pyx_ag_asend_freelist_free) { + if (likely(__Pyx_ag_asend_freelist_free)) { __Pyx_ag_asend_freelist_free--; o = __Pyx_ag_asend_freelist[__Pyx_ag_asend_freelist_free]; _Py_NewReference((PyObject *)o); } else { o = PyObject_GC_New(__pyx_PyAsyncGenASend, __pyx__PyAsyncGenASendType); - if (o == NULL) { + if (unlikely(o == NULL)) { return NULL; } } @@ -701,7 +721,7 @@ __Pyx_async_gen_wrapped_val_dealloc(__pyx__PyAsyncGenWrappedValue *o) { PyObject_GC_UnTrack((PyObject *)o); Py_CLEAR(o->agw_val); - if (__Pyx_ag_value_freelist_free < _PyAsyncGen_MAXFREELIST) { + if (likely(__Pyx_ag_value_freelist_free < _PyAsyncGen_MAXFREELIST)) { assert(__pyx__PyAsyncGenWrappedValue_CheckExact(o)); __Pyx_ag_value_freelist[__Pyx_ag_value_freelist_free++] = o; } else { @@ -726,7 +746,7 @@ static PyTypeObject __pyx__PyAsyncGenWrappedValueType_type = { 0, /* tp_itemsize */ /* methods */ (destructor)__Pyx_async_gen_wrapped_val_dealloc, /* tp_dealloc */ - 0, /* tp_print */ + 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ 0, /* tp_as_async */ @@ -777,6 +797,9 @@ static PyTypeObject __pyx__PyAsyncGenWrappedValueType_type = { #if PY_VERSION_HEX >= 0x030800b4 && PY_VERSION_HEX < 0x03090000 0, /*tp_print*/ #endif +#if CYTHON_COMPILING_IN_PYPY && PYPY_VERSION_NUM+0 >= 0x06000000 + 0, /*tp_pypy_flags*/ +#endif }; @@ -787,7 +810,7 @@ __Pyx__PyAsyncGenValueWrapperNew(PyObject *val) __pyx__PyAsyncGenWrappedValue *o; assert(val); - if (__Pyx_ag_value_freelist_free) { + if (likely(__Pyx_ag_value_freelist_free)) { __Pyx_ag_value_freelist_free--; o = __Pyx_ag_value_freelist[__Pyx_ag_value_freelist_free]; assert(__pyx__PyAsyncGenWrappedValue_CheckExact(o)); @@ -832,34 +855,56 @@ static PyObject * __Pyx_async_gen_athrow_send(__pyx_PyAsyncGenAThrow *o, PyObject *arg) { __pyx_CoroutineObject *gen = (__pyx_CoroutineObject*)o->agt_gen; - PyObject *retval; + PyObject *retval, *exc_type; - if (o->agt_state == __PYX_AWAITABLE_STATE_CLOSED) { + if (unlikely(o->agt_state == __PYX_AWAITABLE_STATE_CLOSED)) { + PyErr_SetString(PyExc_RuntimeError, __Pyx_ASYNC_GEN_CANNOT_REUSE_CLOSE_MSG); + return NULL; + } + + if (unlikely(gen->resume_label == -1)) { + // already run past the end + o->agt_state = __PYX_AWAITABLE_STATE_CLOSED; PyErr_SetNone(PyExc_StopIteration); return NULL; } if (o->agt_state == __PYX_AWAITABLE_STATE_INIT) { - if (o->agt_gen->ag_closed) { - PyErr_SetNone(PyExc_StopIteration); + if (unlikely(o->agt_gen->ag_running_async)) { + o->agt_state = __PYX_AWAITABLE_STATE_CLOSED; + if (o->agt_args == NULL) { + PyErr_SetString( + PyExc_RuntimeError, + "aclose(): asynchronous generator is already running"); + } else { + PyErr_SetString( + PyExc_RuntimeError, + "athrow(): asynchronous generator is already running"); + } + return NULL; + } + + if (unlikely(o->agt_gen->ag_closed)) { + o->agt_state = __PYX_AWAITABLE_STATE_CLOSED; + PyErr_SetNone(__Pyx_PyExc_StopAsyncIteration); return NULL; } - if (arg != Py_None) { + if (unlikely(arg != Py_None)) { PyErr_SetString(PyExc_RuntimeError, __Pyx_NON_INIT_CORO_MSG); return NULL; } o->agt_state = __PYX_AWAITABLE_STATE_ITER; + o->agt_gen->ag_running_async = 1; if (o->agt_args == NULL) { /* aclose() mode */ o->agt_gen->ag_closed = 1; retval = __Pyx__Coroutine_Throw((PyObject*)gen, - /* Do not close generator when - PyExc_GeneratorExit is passed */ - PyExc_GeneratorExit, NULL, NULL, NULL, 0); + /* Do not close generator when PyExc_GeneratorExit is passed */ + PyExc_GeneratorExit, NULL, NULL, NULL, 0); if (retval && __pyx__PyAsyncGenWrappedValue_CheckExact(retval)) { Py_DECREF(retval); @@ -870,14 +915,13 @@ __Pyx_async_gen_athrow_send(__pyx_PyAsyncGenAThrow *o, PyObject *arg) PyObject *tb = NULL; PyObject *val = NULL; - if (!PyArg_UnpackTuple(o->agt_args, "athrow", 1, 3, - &typ, &val, &tb)) { + if (unlikely(!PyArg_UnpackTuple(o->agt_args, "athrow", 1, 3, &typ, &val, &tb))) { return NULL; } retval = __Pyx__Coroutine_Throw((PyObject*)gen, - /* Do not close generator when PyExc_GeneratorExit is passed */ - typ, val, tb, o->agt_args, 0); + /* Do not close generator when PyExc_GeneratorExit is passed */ + typ, val, tb, o->agt_args, 0); retval = __Pyx_async_gen_unwrap_value(o->agt_gen, retval); } if (retval == NULL) { @@ -894,7 +938,7 @@ __Pyx_async_gen_athrow_send(__pyx_PyAsyncGenAThrow *o, PyObject *arg) } else { /* aclose() mode */ if (retval) { - if (__pyx__PyAsyncGenWrappedValue_CheckExact(retval)) { + if (unlikely(__pyx__PyAsyncGenWrappedValue_CheckExact(retval))) { Py_DECREF(retval); goto yield_close; } @@ -908,26 +952,26 @@ __Pyx_async_gen_athrow_send(__pyx_PyAsyncGenAThrow *o, PyObject *arg) } yield_close: + o->agt_gen->ag_running_async = 0; + o->agt_state = __PYX_AWAITABLE_STATE_CLOSED; PyErr_SetString( PyExc_RuntimeError, __Pyx_ASYNC_GEN_IGNORED_EXIT_MSG); return NULL; check_error: - if (PyErr_ExceptionMatches(__Pyx_PyExc_StopAsyncIteration)) { - o->agt_state = __PYX_AWAITABLE_STATE_CLOSED; + o->agt_gen->ag_running_async = 0; + o->agt_state = __PYX_AWAITABLE_STATE_CLOSED; + exc_type = PyErr_Occurred(); + if (__Pyx_PyErr_GivenExceptionMatches2(exc_type, __Pyx_PyExc_StopAsyncIteration, PyExc_GeneratorExit)) { if (o->agt_args == NULL) { // when aclose() is called we don't want to propagate - // StopAsyncIteration; just raise StopIteration, signalling - // that 'aclose()' is done. + // StopAsyncIteration or GeneratorExit; just raise + // StopIteration, signalling that this 'aclose()' await + // is done. PyErr_Clear(); PyErr_SetNone(PyExc_StopIteration); } } - else if (PyErr_ExceptionMatches(PyExc_GeneratorExit)) { - o->agt_state = __PYX_AWAITABLE_STATE_CLOSED; - PyErr_Clear(); /* ignore these errors */ - PyErr_SetNone(PyExc_StopIteration); - } return NULL; } @@ -937,13 +981,8 @@ __Pyx_async_gen_athrow_throw(__pyx_PyAsyncGenAThrow *o, PyObject *args) { PyObject *retval; - if (o->agt_state == __PYX_AWAITABLE_STATE_INIT) { - PyErr_SetString(PyExc_RuntimeError, __Pyx_NON_INIT_CORO_MSG); - return NULL; - } - - if (o->agt_state == __PYX_AWAITABLE_STATE_CLOSED) { - PyErr_SetNone(PyExc_StopIteration); + if (unlikely(o->agt_state == __PYX_AWAITABLE_STATE_CLOSED)) { + PyErr_SetString(PyExc_RuntimeError, __Pyx_ASYNC_GEN_CANNOT_REUSE_CLOSE_MSG); return NULL; } @@ -951,12 +990,24 @@ __Pyx_async_gen_athrow_throw(__pyx_PyAsyncGenAThrow *o, PyObject *args) if (o->agt_args) { return __Pyx_async_gen_unwrap_value(o->agt_gen, retval); } else { - /* aclose() mode */ - if (retval && __pyx__PyAsyncGenWrappedValue_CheckExact(retval)) { + // aclose() mode + PyObject *exc_type; + if (unlikely(retval && __pyx__PyAsyncGenWrappedValue_CheckExact(retval))) { + o->agt_gen->ag_running_async = 0; + o->agt_state = __PYX_AWAITABLE_STATE_CLOSED; Py_DECREF(retval); PyErr_SetString(PyExc_RuntimeError, __Pyx_ASYNC_GEN_IGNORED_EXIT_MSG); return NULL; } + exc_type = PyErr_Occurred(); + if (__Pyx_PyErr_GivenExceptionMatches2(exc_type, __Pyx_PyExc_StopAsyncIteration, PyExc_GeneratorExit)) { + // when aclose() is called we don't want to propagate + // StopAsyncIteration or GeneratorExit; just raise + // StopIteration, signalling that this 'aclose()' await + // is done. + PyErr_Clear(); + PyErr_SetNone(PyExc_StopIteration); + } return retval; } } @@ -1002,7 +1053,7 @@ static PyTypeObject __pyx__PyAsyncGenAThrowType_type = { sizeof(__pyx_PyAsyncGenAThrow), /* tp_basicsize */ 0, /* tp_itemsize */ (destructor)__Pyx_async_gen_athrow_dealloc, /* tp_dealloc */ - 0, /* tp_print */ + 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ #if CYTHON_USE_ASYNC_SLOTS @@ -1062,6 +1113,9 @@ static PyTypeObject __pyx__PyAsyncGenAThrowType_type = { #if PY_VERSION_HEX >= 0x030800b4 && PY_VERSION_HEX < 0x03090000 0, /*tp_print*/ #endif +#if CYTHON_COMPILING_IN_PYPY && PYPY_VERSION_NUM+0 >= 0x06000000 + 0, /*tp_pypy_flags*/ +#endif }; @@ -1070,7 +1124,7 @@ __Pyx_async_gen_athrow_new(__pyx_PyAsyncGenObject *gen, PyObject *args) { __pyx_PyAsyncGenAThrow *o; o = PyObject_GC_New(__pyx_PyAsyncGenAThrow, __pyx__PyAsyncGenAThrowType); - if (o == NULL) { + if (unlikely(o == NULL)) { return NULL; } o->agt_gen = gen; diff --git a/Cython/Utility/Buffer.c b/Cython/Utility/Buffer.c index 3c7105fa3..8a6e055c2 100644 --- a/Cython/Utility/Buffer.c +++ b/Cython/Utility/Buffer.c @@ -111,6 +111,7 @@ typedef struct { #if PY_MAJOR_VERSION < 3 static int __Pyx_GetBuffer(PyObject *obj, Py_buffer *view, int flags) { + __Pyx_TypeName obj_type_name; if (PyObject_CheckBuffer(obj)) return PyObject_GetBuffer(obj, view, flags); {{for type_ptr, getbuffer, releasebuffer in types}} @@ -119,7 +120,11 @@ static int __Pyx_GetBuffer(PyObject *obj, Py_buffer *view, int flags) { {{endif}} {{endfor}} - PyErr_Format(PyExc_TypeError, "'%.200s' does not have the buffer interface", Py_TYPE(obj)->tp_name); + obj_type_name = __Pyx_PyType_GetName(Py_TYPE(obj)); + PyErr_Format(PyExc_TypeError, + "'" __Pyx_FMT_TYPENAME "' does not have the buffer interface", + obj_type_name); + __Pyx_DECREF_TypeName(obj_type_name); return -1; } @@ -224,8 +229,8 @@ fail:; // the format string; the access mode/flags is checked by the // exporter. See: // -// http://docs.python.org/3/library/struct.html -// http://legacy.python.org/dev/peps/pep-3118/#additions-to-the-struct-string-syntax +// https://docs.python.org/3/library/struct.html +// https://www.python.org/dev/peps/pep-3118/#additions-to-the-struct-string-syntax // // The alignment code is copied from _struct.c in Python. diff --git a/Cython/Utility/Builtins.c b/Cython/Utility/Builtins.c index 1945d10be..fc5a8441e 100644 --- a/Cython/Utility/Builtins.c +++ b/Cython/Utility/Builtins.c @@ -20,47 +20,7 @@ static PyObject* __Pyx_Globals(void); /*proto*/ // access requires a rewrite as a dedicated class. static PyObject* __Pyx_Globals(void) { - Py_ssize_t i; - PyObject *names; - PyObject *globals = $moddict_cname; - Py_INCREF(globals); - names = PyObject_Dir($module_cname); - if (!names) - goto bad; - for (i = PyList_GET_SIZE(names)-1; i >= 0; i--) { -#if CYTHON_COMPILING_IN_PYPY - PyObject* name = PySequence_ITEM(names, i); - if (!name) - goto bad; -#else - PyObject* name = PyList_GET_ITEM(names, i); -#endif - if (!PyDict_Contains(globals, name)) { - PyObject* value = __Pyx_GetAttr($module_cname, name); - if (!value) { -#if CYTHON_COMPILING_IN_PYPY - Py_DECREF(name); -#endif - goto bad; - } - if (PyDict_SetItem(globals, name, value) < 0) { -#if CYTHON_COMPILING_IN_PYPY - Py_DECREF(name); -#endif - Py_DECREF(value); - goto bad; - } - } -#if CYTHON_COMPILING_IN_PYPY - Py_DECREF(name); -#endif - } - Py_DECREF(names); - return globals; -bad: - Py_XDECREF(names); - Py_XDECREF(globals); - return NULL; + return __Pyx_NewRef($moddict_cname); } //////////////////// PyExecGlobals.proto //////////////////// @@ -68,17 +28,11 @@ bad: static PyObject* __Pyx_PyExecGlobals(PyObject*); //////////////////// PyExecGlobals //////////////////// -//@requires: Globals +//@substitute: naming //@requires: PyExec static PyObject* __Pyx_PyExecGlobals(PyObject* code) { - PyObject* result; - PyObject* globals = __Pyx_Globals(); - if (unlikely(!globals)) - return NULL; - result = __Pyx_PyExec2(code, globals); - Py_DECREF(globals); - return result; + return __Pyx_PyExec2(code, $moddict_cname); } //////////////////// PyExec.proto //////////////////// @@ -100,9 +54,13 @@ static PyObject* __Pyx_PyExec3(PyObject* o, PyObject* globals, PyObject* locals) if (!globals || globals == Py_None) { globals = $moddict_cname; - } else if (!PyDict_Check(globals)) { - PyErr_Format(PyExc_TypeError, "exec() arg 2 must be a dict, not %.200s", - Py_TYPE(globals)->tp_name); + } else if (unlikely(!PyDict_Check(globals))) { + __Pyx_TypeName globals_type_name = + __Pyx_PyType_GetName(Py_TYPE(globals)); + PyErr_Format(PyExc_TypeError, + "exec() arg 2 must be a dict, not " __Pyx_FMT_TYPENAME, + globals_type_name); + __Pyx_DECREF_TypeName(globals_type_name); goto bad; } if (!locals || locals == Py_None) { @@ -110,12 +68,12 @@ static PyObject* __Pyx_PyExec3(PyObject* o, PyObject* globals, PyObject* locals) } if (__Pyx_PyDict_GetItemStr(globals, PYIDENT("__builtins__")) == NULL) { - if (PyDict_SetItem(globals, PYIDENT("__builtins__"), PyEval_GetBuiltins()) < 0) + if (unlikely(PyDict_SetItem(globals, PYIDENT("__builtins__"), PyEval_GetBuiltins()) < 0)) goto bad; } if (PyCode_Check(o)) { - if (__Pyx_PyCode_HasFreeVars((PyCodeObject *)o)) { + if (unlikely(__Pyx_PyCode_HasFreeVars((PyCodeObject *)o))) { PyErr_SetString(PyExc_TypeError, "code object passed to exec() may not contain free variables"); goto bad; @@ -134,16 +92,18 @@ static PyObject* __Pyx_PyExec3(PyObject* o, PyObject* globals, PyObject* locals) if (PyUnicode_Check(o)) { cf.cf_flags = PyCF_SOURCE_IS_UTF8; s = PyUnicode_AsUTF8String(o); - if (!s) goto bad; + if (unlikely(!s)) goto bad; o = s; #if PY_MAJOR_VERSION >= 3 - } else if (!PyBytes_Check(o)) { + } else if (unlikely(!PyBytes_Check(o))) { #else - } else if (!PyString_Check(o)) { + } else if (unlikely(!PyString_Check(o))) { #endif + __Pyx_TypeName o_type_name = __Pyx_PyType_GetName(Py_TYPE(o)); PyErr_Format(PyExc_TypeError, - "exec: arg 1 must be string, bytes or code object, got %.200s", - Py_TYPE(o)->tp_name); + "exec: arg 1 must be string, bytes or code object, got " __Pyx_FMT_TYPENAME, + o_type_name); + __Pyx_DECREF_TypeName(o_type_name); goto bad; } #if PY_MAJOR_VERSION >= 3 @@ -170,7 +130,7 @@ bad: static CYTHON_INLINE PyObject *__Pyx_GetAttr3(PyObject *, PyObject *, PyObject *); /*proto*/ //////////////////// GetAttr3 //////////////////// -//@requires: ObjectHandling.c::GetAttr +//@requires: ObjectHandling.c::PyObjectGetAttrStr //@requires: Exceptions.c::PyThreadStateGet //@requires: Exceptions.c::PyErrFetchRestore //@requires: Exceptions.c::PyErrExceptionMatches @@ -186,7 +146,17 @@ static PyObject *__Pyx_GetAttr3Default(PyObject *d) { } static CYTHON_INLINE PyObject *__Pyx_GetAttr3(PyObject *o, PyObject *n, PyObject *d) { - PyObject *r = __Pyx_GetAttr(o, n); + PyObject *r; +#if CYTHON_USE_TYPE_SLOTS + if (likely(PyString_Check(n))) { + r = __Pyx_PyObject_GetAttrStrNoError(o, n); + if (unlikely(!r) && likely(!PyErr_Occurred())) { + r = __Pyx_NewRef(d); + } + return r; + } +#endif + r = PyObject_GetAttr(o, n); return (likely(r)) ? r : __Pyx_GetAttr3Default(d); } @@ -205,7 +175,7 @@ static CYTHON_INLINE int __Pyx_HasAttr(PyObject *o, PyObject *n) { return -1; } r = __Pyx_GetAttr(o, n); - if (unlikely(!r)) { + if (!r) { PyErr_Clear(); return 0; } else { @@ -219,11 +189,12 @@ static CYTHON_INLINE int __Pyx_HasAttr(PyObject *o, PyObject *n) { static PyObject* __Pyx_Intern(PyObject* s); /* proto */ //////////////////// Intern //////////////////// +//@requires: ObjectHandling.c::RaiseUnexpectedTypeError static PyObject* __Pyx_Intern(PyObject* s) { - if (!(likely(PyString_CheckExact(s)))) { - PyErr_Format(PyExc_TypeError, "Expected %.16s, got %.200s", "str", Py_TYPE(s)->tp_name); - return 0; + if (unlikely(!PyString_CheckExact(s))) { + __Pyx_RaiseUnexpectedTypeError("str", s); + return NULL; } Py_INCREF(s); #if PY_MAJOR_VERSION >= 3 @@ -332,8 +303,11 @@ static long __Pyx__PyObject_Ord(PyObject* c) { #endif } else { // FIXME: support character buffers - but CPython doesn't support them either + __Pyx_TypeName c_type_name = __Pyx_PyType_GetName(Py_TYPE(c)); PyErr_Format(PyExc_TypeError, - "ord() expected string of length 1, but %.200s found", c->ob_type->tp_name); + "ord() expected string of length 1, but " __Pyx_FMT_TYPENAME " found", + c_type_name); + __Pyx_DECREF_TypeName(c_type_name); return (long)(Py_UCS4)-1; } PyErr_Format(PyExc_TypeError, @@ -422,9 +396,6 @@ static CYTHON_INLINE PyObject* __Pyx_PyDict_IterItems(PyObject* d) { //////////////////// py_dict_viewkeys.proto //////////////////// -#if PY_VERSION_HEX < 0x02070000 -#error This module uses dict views, which require Python 2.7 or later -#endif static CYTHON_INLINE PyObject* __Pyx_PyDict_ViewKeys(PyObject* d); /*proto*/ //////////////////// py_dict_viewkeys //////////////////// @@ -438,9 +409,6 @@ static CYTHON_INLINE PyObject* __Pyx_PyDict_ViewKeys(PyObject* d) { //////////////////// py_dict_viewvalues.proto //////////////////// -#if PY_VERSION_HEX < 0x02070000 -#error This module uses dict views, which require Python 2.7 or later -#endif static CYTHON_INLINE PyObject* __Pyx_PyDict_ViewValues(PyObject* d); /*proto*/ //////////////////// py_dict_viewvalues //////////////////// @@ -454,9 +422,6 @@ static CYTHON_INLINE PyObject* __Pyx_PyDict_ViewValues(PyObject* d) { //////////////////// py_dict_viewitems.proto //////////////////// -#if PY_VERSION_HEX < 0x02070000 -#error This module uses dict views, which require Python 2.7 or later -#endif static CYTHON_INLINE PyObject* __Pyx_PyDict_ViewItems(PyObject* d); /*proto*/ //////////////////// py_dict_viewitems //////////////////// diff --git a/Cython/Utility/CConvert.pyx b/Cython/Utility/CConvert.pyx index 5969f6a58..4ae66162e 100644 --- a/Cython/Utility/CConvert.pyx +++ b/Cython/Utility/CConvert.pyx @@ -6,19 +6,20 @@ cdef extern from *: PyTypeObject *Py_TYPE(obj) bint PyMapping_Check(obj) object PyErr_Format(exc, const char *format, ...) + int __Pyx_RaiseUnexpectedTypeError(const char *expected, object obj) except 0 @cname("{{funcname}}") cdef {{struct_type}} {{funcname}}(obj) except *: cdef {{struct_type}} result if not PyMapping_Check(obj): - PyErr_Format(TypeError, b"Expected %.16s, got %.200s", b"a mapping", Py_TYPE(obj).tp_name) + __Pyx_RaiseUnexpectedTypeError(b"a mapping", obj) {{for member in var_entries:}} try: value = obj['{{member.name}}'] except KeyError: raise ValueError("No value specified for struct attribute '{{member.name}}'") - result.{{member.cname}} = value + result.{{member.name}} = value {{endfor}} return result @@ -31,13 +32,14 @@ cdef extern from *: PyTypeObject *Py_TYPE(obj) bint PyMapping_Check(obj) object PyErr_Format(exc, const char *format, ...) + int __Pyx_RaiseUnexpectedTypeError(const char *expected, object obj) except 0 @cname("{{funcname}}") cdef {{struct_type}} {{funcname}}(obj) except *: cdef {{struct_type}} result cdef Py_ssize_t length if not PyMapping_Check(obj): - PyErr_Format(TypeError, b"Expected %.16s, got %.200s", b"a mapping", Py_TYPE(obj).tp_name) + __Pyx_RaiseUnexpectedTypeError(b"a mapping", obj) last_found = None length = len(obj) diff --git a/Cython/Utility/Capsule.c b/Cython/Utility/Capsule.c index cc4fe0d88..14c744cdf 100644 --- a/Cython/Utility/Capsule.c +++ b/Cython/Utility/Capsule.c @@ -6,15 +6,7 @@ static CYTHON_INLINE PyObject *__pyx_capsule_create(void *p, const char *sig); //////////////// Capsule //////////////// static CYTHON_INLINE PyObject * -__pyx_capsule_create(void *p, CYTHON_UNUSED const char *sig) +__pyx_capsule_create(void *p, const char *sig) { - PyObject *cobj; - -#if PY_VERSION_HEX >= 0x02070000 - cobj = PyCapsule_New(p, sig, NULL); -#else - cobj = PyCObject_FromVoidPtr(p, NULL); -#endif - - return cobj; + return PyCapsule_New(p, sig, NULL); } diff --git a/Cython/Utility/CommonStructures.c b/Cython/Utility/CommonStructures.c index c7945feb4..4b572defb 100644 --- a/Cython/Utility/CommonStructures.c +++ b/Cython/Utility/CommonStructures.c @@ -1,51 +1,117 @@ /////////////// FetchCommonType.proto /////////////// static PyTypeObject* __Pyx_FetchCommonType(PyTypeObject* type); +#if CYTHON_COMPILING_IN_LIMITED_API +static PyTypeObject* __Pyx_FetchCommonTypeFromSpec(PyType_Spec *spec, PyObject *bases); +#endif /////////////// FetchCommonType /////////////// -static PyTypeObject* __Pyx_FetchCommonType(PyTypeObject* type) { - PyObject* fake_module; - PyTypeObject* cached_type = NULL; +static PyObject *__Pyx_FetchSharedCythonABIModule(void) { + PyObject *abi_module = PyImport_AddModule((char*) __PYX_ABI_MODULE_NAME); + if (!abi_module) return NULL; + Py_INCREF(abi_module); + return abi_module; +} + +static int __Pyx_VerifyCachedType(PyObject *cached_type, + const char *name, + Py_ssize_t basicsize, + Py_ssize_t expected_basicsize) { + if (!PyType_Check(cached_type)) { + PyErr_Format(PyExc_TypeError, + "Shared Cython type %.200s is not a type object", name); + return -1; + } + if (basicsize != expected_basicsize) { + PyErr_Format(PyExc_TypeError, + "Shared Cython type %.200s has the wrong size, try recompiling", + name); + return -1; + } + return 0; +} - fake_module = PyImport_AddModule((char*) "_cython_" CYTHON_ABI); - if (!fake_module) return NULL; - Py_INCREF(fake_module); +static PyTypeObject* __Pyx_FetchCommonType(PyTypeObject* type) { + PyObject* abi_module; + PyTypeObject *cached_type = NULL; - cached_type = (PyTypeObject*) PyObject_GetAttrString(fake_module, type->tp_name); + abi_module = __Pyx_FetchSharedCythonABIModule(); + if (!abi_module) return NULL; + cached_type = (PyTypeObject*) PyObject_GetAttrString(abi_module, type->tp_name); if (cached_type) { - if (!PyType_Check((PyObject*)cached_type)) { - PyErr_Format(PyExc_TypeError, - "Shared Cython type %.200s is not a type object", - type->tp_name); + if (__Pyx_VerifyCachedType( + (PyObject *)cached_type, + type->tp_name, + cached_type->tp_basicsize, + type->tp_basicsize) < 0) { goto bad; } - if (cached_type->tp_basicsize != type->tp_basicsize) { - PyErr_Format(PyExc_TypeError, - "Shared Cython type %.200s has the wrong size, try recompiling", - type->tp_name); + goto done; + } + + if (!PyErr_ExceptionMatches(PyExc_AttributeError)) goto bad; + PyErr_Clear(); + if (PyType_Ready(type) < 0) goto bad; + if (PyObject_SetAttrString(abi_module, type->tp_name, (PyObject *)type) < 0) + goto bad; + Py_INCREF(type); + cached_type = type; + +done: + Py_DECREF(abi_module); + // NOTE: always returns owned reference, or NULL on error + return cached_type; + +bad: + Py_XDECREF(cached_type); + cached_type = NULL; + goto done; +} + +#if CYTHON_COMPILING_IN_LIMITED_API +static PyTypeObject *__Pyx_FetchCommonTypeFromSpec(PyType_Spec *spec, PyObject *bases) { + PyObject *abi_module, *py_basicsize, *cached_type = NULL; + Py_ssize_t basicsize; + + abi_module = __Pyx_FetchSharedCythonABIModule(); + if (!abi_module) return NULL; + cached_type = PyObject_GetAttrString(abi_module, spec->name); + if (cached_type) { + py_basicsize = PyObject_GetAttrString(cached_type, "__basicsize__"); + if (!py_basicsize) goto bad; + basicsize = PyLong_AsSsize_t(py_basicsize); + Py_DECREF(py_basicsize); + py_basicsize = 0; + if (basicsize == (Py_ssize_t)-1 && PyErr_Occurred()) goto bad; + if (__Pyx_VerifyCachedType( + cached_type, + spec->name, + basicsize, + spec->basicsize) < 0) { goto bad; } - } else { - if (!PyErr_ExceptionMatches(PyExc_AttributeError)) goto bad; - PyErr_Clear(); - if (PyType_Ready(type) < 0) goto bad; - if (PyObject_SetAttrString(fake_module, type->tp_name, (PyObject*) type) < 0) - goto bad; - Py_INCREF(type); - cached_type = type; + goto done; } + if (!PyErr_ExceptionMatches(PyExc_AttributeError)) goto bad; + PyErr_Clear(); + cached_type = PyType_FromSpecWithBases(spec, bases); + if (unlikely(!cached_type)) goto bad; + if (PyObject_SetAttrString(abi_module, spec->name, cached_type) < 0) goto bad; + done: - Py_DECREF(fake_module); + Py_DECREF(abi_module); // NOTE: always returns owned reference, or NULL on error - return cached_type; + assert(cached_type == NULL || PyType_Check(cached_type)); + return (PyTypeObject *) cached_type; bad: Py_XDECREF(cached_type); cached_type = NULL; goto done; } +#endif /////////////// FetchCommonPointer.proto /////////////// @@ -56,31 +122,27 @@ static void* __Pyx_FetchCommonPointer(void* pointer, const char* name); static void* __Pyx_FetchCommonPointer(void* pointer, const char* name) { -#if PY_VERSION_HEX >= 0x02070000 - PyObject* fake_module = NULL; + PyObject* abi_module = NULL; PyObject* capsule = NULL; void* value = NULL; - fake_module = PyImport_AddModule((char*) "_cython_" CYTHON_ABI); - if (!fake_module) return NULL; - Py_INCREF(fake_module); + abi_module = PyImport_AddModule((char*) __PYX_ABI_MODULE_NAME); + if (!abi_module) return NULL; + Py_INCREF(abi_module); - capsule = PyObject_GetAttrString(fake_module, name); + capsule = PyObject_GetAttrString(abi_module, name); if (!capsule) { if (!PyErr_ExceptionMatches(PyExc_AttributeError)) goto bad; PyErr_Clear(); capsule = PyCapsule_New(pointer, name, NULL); if (!capsule) goto bad; - if (PyObject_SetAttrString(fake_module, name, capsule) < 0) + if (PyObject_SetAttrString(abi_module, name, capsule) < 0) goto bad; } value = PyCapsule_GetPointer(capsule, name); bad: Py_XDECREF(capsule); - Py_DECREF(fake_module); + Py_DECREF(abi_module); return value; -#else - return pointer; -#endif } diff --git a/Cython/Utility/Coroutine.c b/Cython/Utility/Coroutine.c index e5b4ce6ef..a2540d5d4 100644 --- a/Cython/Utility/Coroutine.c +++ b/Cython/Utility/Coroutine.c @@ -5,12 +5,15 @@ static CYTHON_INLINE PyObject* __Pyx_Generator_Yield_From(__pyx_CoroutineObject //////////////////// GeneratorYieldFrom //////////////////// //@requires: Generator -static void __PyxPyIter_CheckErrorAndDecref(PyObject *source) { +#if CYTHON_USE_TYPE_SLOTS +static void __Pyx_PyIter_CheckErrorAndDecref(PyObject *source) { + __Pyx_TypeName source_type_name = __Pyx_PyType_GetName(Py_TYPE(source)); PyErr_Format(PyExc_TypeError, - "iter() returned non-iterator of type '%.100s'", - Py_TYPE(source)->tp_name); + "iter() returned non-iterator of type '" __Pyx_FMT_TYPENAME "'", source_type_name); + __Pyx_DECREF_TypeName(source_type_name); Py_DECREF(source); } +#endif static CYTHON_INLINE PyObject* __Pyx_Generator_Yield_From(__pyx_CoroutineObject *gen, PyObject *source) { PyObject *source_gen, *retval; @@ -29,7 +32,7 @@ static CYTHON_INLINE PyObject* __Pyx_Generator_Yield_From(__pyx_CoroutineObject if (unlikely(!source_gen)) return NULL; if (unlikely(!PyIter_Check(source_gen))) { - __PyxPyIter_CheckErrorAndDecref(source_gen); + __Pyx_PyIter_CheckErrorAndDecref(source_gen); return NULL; } } else @@ -41,11 +44,7 @@ static CYTHON_INLINE PyObject* __Pyx_Generator_Yield_From(__pyx_CoroutineObject return NULL; } // source_gen is now the iterator, make the first next() call -#if CYTHON_USE_TYPE_SLOTS - retval = Py_TYPE(source_gen)->tp_iternext(source_gen); -#else - retval = PyIter_Next(source_gen); -#endif + retval = __Pyx_PyObject_GetIterNextFunc(source_gen)(source_gen); } if (likely(retval)) { gen->yieldfrom = source_gen; @@ -74,11 +73,7 @@ static PyObject* __Pyx__Coroutine_Yield_From_Generic(__pyx_CoroutineObject *gen, if (__Pyx_Coroutine_Check(source_gen)) { retval = __Pyx_Generator_Next(source_gen); } else { -#if CYTHON_USE_TYPE_SLOTS - retval = Py_TYPE(source_gen)->tp_iternext(source_gen); -#else - retval = PyIter_Next(source_gen); -#endif + retval = __Pyx_PyObject_GetIterNextFunc(source_gen)(source_gen); } if (retval) { gen->yieldfrom = source_gen; @@ -136,13 +131,13 @@ static CYTHON_INLINE PyObject *__Pyx_Coroutine_GetAwaitableIter(PyObject *o) { static void __Pyx_Coroutine_AwaitableIterError(PyObject *source) { #if PY_VERSION_HEX >= 0x030600B3 || defined(_PyErr_FormatFromCause) - _PyErr_FormatFromCause( - PyExc_TypeError, - "'async for' received an invalid object " - "from __anext__: %.100s", - Py_TYPE(source)->tp_name); + __Pyx_TypeName source_type_name = __Pyx_PyType_GetName(Py_TYPE(source)); + _PyErr_FormatFromCause(PyExc_TypeError, + "'async for' received an invalid object from __anext__: " __Pyx_FMT_TYPENAME, source_type_name); + __Pyx_DECREF_TypeName(source_type_name); #elif PY_MAJOR_VERSION >= 3 PyObject *exc, *val, *val2, *tb; + __Pyx_TypeName source_type_name = __Pyx_PyType_GetName(Py_TYPE(source)); assert(PyErr_Occurred()); PyErr_Fetch(&exc, &val, &tb); PyErr_NormalizeException(&exc, &val, &tb); @@ -152,11 +147,9 @@ static void __Pyx_Coroutine_AwaitableIterError(PyObject *source) { } Py_DECREF(exc); assert(!PyErr_Occurred()); - PyErr_Format( - PyExc_TypeError, - "'async for' received an invalid object " - "from __anext__: %.100s", - Py_TYPE(source)->tp_name); + PyErr_Format(PyExc_TypeError, + "'async for' received an invalid object from __anext__: " __Pyx_FMT_TYPENAME, source_type_name); + __Pyx_DECREF_TypeName(source_type_name); PyErr_Fetch(&exc, &val2, &tb); PyErr_NormalizeException(&exc, &val2, &tb); @@ -207,9 +200,10 @@ static PyObject *__Pyx__Coroutine_GetAwaitableIter(PyObject *obj) { goto bad; } if (unlikely(!PyIter_Check(res))) { + __Pyx_TypeName res_type_name = __Pyx_PyType_GetName(Py_TYPE(res)); PyErr_Format(PyExc_TypeError, - "__await__() returned non-iterator of type '%.100s'", - Py_TYPE(res)->tp_name); + "__await__() returned non-iterator of type '" __Pyx_FMT_TYPENAME "'", res_type_name); + __Pyx_DECREF_TypeName(res_type_name); Py_CLEAR(res); } else { int is_coroutine = 0; @@ -229,9 +223,12 @@ static PyObject *__Pyx__Coroutine_GetAwaitableIter(PyObject *obj) { } return res; slot_error: - PyErr_Format(PyExc_TypeError, - "object %.100s can't be used in 'await' expression", - Py_TYPE(obj)->tp_name); + { + __Pyx_TypeName obj_type_name = __Pyx_PyType_GetName(Py_TYPE(obj)); + PyErr_Format(PyExc_TypeError, + "object " __Pyx_FMT_TYPENAME " can't be used in 'await' expression", obj_type_name); + __Pyx_DECREF_TypeName(obj_type_name); + } bad: return NULL; } @@ -247,6 +244,7 @@ static CYTHON_INLINE PyObject *__Pyx_Coroutine_AsyncIterNext(PyObject *o); /*pro //@requires: ObjectHandling.c::PyObjectCallMethod0 static PyObject *__Pyx_Coroutine_GetAsyncIter_Generic(PyObject *obj) { + __Pyx_TypeName obj_type_name; #if PY_VERSION_HEX < 0x030500B1 { PyObject *iter = __Pyx_PyObject_CallMethod0(obj, PYIDENT("__aiter__")); @@ -261,8 +259,10 @@ static PyObject *__Pyx_Coroutine_GetAsyncIter_Generic(PyObject *obj) { if ((0)) (void) __Pyx_PyObject_CallMethod0(obj, PYIDENT("__aiter__")); #endif - PyErr_Format(PyExc_TypeError, "'async for' requires an object with __aiter__ method, got %.100s", - Py_TYPE(obj)->tp_name); + obj_type_name = __Pyx_PyType_GetName(Py_TYPE(obj)); + PyErr_Format(PyExc_TypeError, + "'async for' requires an object with __aiter__ method, got " __Pyx_FMT_TYPENAME, obj_type_name); + __Pyx_DECREF_TypeName(obj_type_name); return NULL; } @@ -295,8 +295,12 @@ static PyObject *__Pyx__Coroutine_AsyncIterNext(PyObject *obj) { // FIXME: for the sake of a nicely conforming exception message, assume any AttributeError meant '__anext__' if (PyErr_ExceptionMatches(PyExc_AttributeError)) #endif - PyErr_Format(PyExc_TypeError, "'async for' requires an object with __anext__ method, got %.100s", - Py_TYPE(obj)->tp_name); + { + __Pyx_TypeName obj_type_name = __Pyx_PyType_GetName(Py_TYPE(obj)); + PyErr_Format(PyExc_TypeError, + "'async for' requires an object with __anext__ method, got " __Pyx_FMT_TYPENAME, obj_type_name); + __Pyx_DECREF_TypeName(obj_type_name); + } return NULL; } @@ -362,7 +366,8 @@ static void __Pyx_Generator_Replace_StopIteration(CYTHON_UNUSED int in_async_gen //////////////////// CoroutineBase.proto //////////////////// //@substitute: naming -typedef PyObject *(*__pyx_coroutine_body_t)(PyObject *, PyThreadState *, PyObject *); +struct __pyx_CoroutineObject; +typedef PyObject *(*__pyx_coroutine_body_t)(struct __pyx_CoroutineObject *, PyThreadState *, PyObject *); #if CYTHON_USE_EXC_INFO_STACK // See https://bugs.python.org/issue25612 @@ -376,7 +381,7 @@ typedef struct { } __Pyx_ExcInfoStruct; #endif -typedef struct { +typedef struct __pyx_CoroutineObject { PyObject_HEAD __pyx_coroutine_body_t body; PyObject *closure; @@ -388,6 +393,7 @@ typedef struct { PyObject *gi_qualname; PyObject *gi_modulename; PyObject *gi_code; + PyObject *gi_frame; int resume_label; // using T_BOOL for property below requires char value char is_running; @@ -438,10 +444,10 @@ static CYTHON_INLINE void __Pyx_Coroutine_ResetFrameBackpointer(__Pyx_ExcInfoStr #define __Pyx_Coroutine_USED static PyTypeObject *__pyx_CoroutineType = 0; static PyTypeObject *__pyx_CoroutineAwaitType = 0; -#define __Pyx_Coroutine_CheckExact(obj) (Py_TYPE(obj) == __pyx_CoroutineType) +#define __Pyx_Coroutine_CheckExact(obj) __Pyx_IS_TYPE(obj, __pyx_CoroutineType) // __Pyx_Coroutine_Check(obj): see override for IterableCoroutine below #define __Pyx_Coroutine_Check(obj) __Pyx_Coroutine_CheckExact(obj) -#define __Pyx_CoroutineAwait_CheckExact(obj) (Py_TYPE(obj) == __pyx_CoroutineAwaitType) +#define __Pyx_CoroutineAwait_CheckExact(obj) __Pyx_IS_TYPE(obj, __pyx_CoroutineAwaitType) #define __Pyx_Coroutine_New(body, code, closure, name, qualname, module_name) \ __Pyx__Coroutine_New(__pyx_CoroutineType, body, code, closure, name, qualname, module_name) @@ -462,7 +468,7 @@ static PyObject *__Pyx_CoroutineAwait_Throw(__pyx_CoroutineAwaitObject *self, Py #define __Pyx_Generator_USED static PyTypeObject *__pyx_GeneratorType = 0; -#define __Pyx_Generator_CheckExact(obj) (Py_TYPE(obj) == __pyx_GeneratorType) +#define __Pyx_Generator_CheckExact(obj) __Pyx_IS_TYPE(obj, __pyx_GeneratorType) #define __Pyx_Generator_New(body, code, closure, name, qualname, module_name) \ __Pyx__Coroutine_New(__pyx_GeneratorType, body, code, closure, name, qualname, module_name) @@ -484,7 +490,10 @@ static int __pyx_Generator_init(void); /*proto*/ //@requires: Exceptions.c::RaiseException //@requires: Exceptions.c::SaveResetException //@requires: ObjectHandling.c::PyObjectCallMethod1 +//@requires: ObjectHandling.c::PyObjectCallNoArg +//@requires: ObjectHandling.c::PyObjectFastCall //@requires: ObjectHandling.c::PyObjectGetAttrStr +//@requires: ObjectHandling.c::PyObjectGetAttrStrNoError //@requires: CommonStructures.c::FetchCommonType #include <structmember.h> @@ -519,7 +528,7 @@ static int __Pyx_PyGen__FetchStopIterationValue(CYTHON_UNUSED PyThreadState *$lo value = Py_None; } #if PY_VERSION_HEX >= 0x030300A0 - else if (Py_TYPE(ev) == (PyTypeObject*)PyExc_StopIteration) { + else if (likely(__Pyx_IS_TYPE(ev, (PyTypeObject*)PyExc_StopIteration))) { value = ((PyStopIterationObject *)ev)->value; Py_INCREF(value); Py_DECREF(ev); @@ -738,7 +747,7 @@ PyObject *__Pyx_Coroutine_SendEx(__pyx_CoroutineObject *self, PyObject *value, i #endif self->is_running = 1; - retval = self->body((PyObject *) self, tstate, value); + retval = self->body(self, tstate, value); self->is_running = 0; #if CYTHON_USE_EXC_INFO_STACK @@ -866,7 +875,7 @@ static PyObject *__Pyx_Coroutine_Send(PyObject *self, PyObject *value) { #endif { if (value == Py_None) - ret = Py_TYPE(yf)->tp_iternext(yf); + ret = __Pyx_PyObject_GetIterNextFunc(yf)(yf); else ret = __Pyx_PyObject_CallMethod1(yf, PYIDENT("send"), value); } @@ -920,16 +929,15 @@ static int __Pyx_Coroutine_CloseIter(__pyx_CoroutineObject *gen, PyObject *yf) { { PyObject *meth; gen->is_running = 1; - meth = __Pyx_PyObject_GetAttrStr(yf, PYIDENT("close")); + meth = __Pyx_PyObject_GetAttrStrNoError(yf, PYIDENT("close")); if (unlikely(!meth)) { - if (!PyErr_ExceptionMatches(PyExc_AttributeError)) { + if (unlikely(PyErr_Occurred())) { PyErr_WriteUnraisable(yf); } - PyErr_Clear(); } else { - retval = PyObject_CallFunction(meth, NULL); + retval = __Pyx_PyObject_CallNoArg(meth); Py_DECREF(meth); - if (!retval) + if (unlikely(!retval)) err = -1; } gen->is_running = 0; @@ -965,7 +973,7 @@ static PyObject *__Pyx_Generator_Next(PyObject *self) { ret = __Pyx_Coroutine_Send(yf, Py_None); } else #endif - ret = Py_TYPE(yf)->tp_iternext(yf); + ret = __Pyx_PyObject_GetIterNextFunc(yf)(yf); gen->is_running = 0; //Py_DECREF(yf); if (likely(ret)) { @@ -1067,23 +1075,23 @@ static PyObject *__Pyx__Coroutine_Throw(PyObject *self, PyObject *typ, PyObject ret = __Pyx__Coroutine_Throw(((__pyx_CoroutineAwaitObject*)yf)->coroutine, typ, val, tb, args, close_on_genexit); #endif } else { - PyObject *meth = __Pyx_PyObject_GetAttrStr(yf, PYIDENT("throw")); + PyObject *meth = __Pyx_PyObject_GetAttrStrNoError(yf, PYIDENT("throw")); if (unlikely(!meth)) { Py_DECREF(yf); - if (!PyErr_ExceptionMatches(PyExc_AttributeError)) { + if (unlikely(PyErr_Occurred())) { gen->is_running = 0; return NULL; } - PyErr_Clear(); __Pyx_Coroutine_Undelegate(gen); gen->is_running = 0; goto throw_here; } if (likely(args)) { - ret = PyObject_CallObject(meth, args); + ret = __Pyx_PyObject_Call(meth, args, NULL); } else { // "tb" or even "val" might be NULL, but that also correctly terminates the argument list - ret = PyObject_CallFunctionObjArgs(meth, typ, val, tb, NULL); + PyObject *cargs[4] = {NULL, typ, val, tb}; + ret = __Pyx_PyObject_FastCall(meth, cargs+1, 3 | __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET); } Py_DECREF(meth); } @@ -1104,7 +1112,7 @@ static PyObject *__Pyx_Coroutine_Throw(PyObject *self, PyObject *args) { PyObject *val = NULL; PyObject *tb = NULL; - if (!PyArg_UnpackTuple(args, (char *)"throw", 1, 3, &typ, &val, &tb)) + if (unlikely(!PyArg_UnpackTuple(args, (char *)"throw", 1, 3, &typ, &val, &tb))) return NULL; return __Pyx__Coroutine_Throw(self, typ, val, tb, args, 1); @@ -1137,6 +1145,7 @@ static int __Pyx_Coroutine_clear(PyObject *self) { } #endif Py_CLEAR(gen->gi_code); + Py_CLEAR(gen->gi_frame); Py_CLEAR(gen->gi_name); Py_CLEAR(gen->gi_qualname); Py_CLEAR(gen->gi_modulename); @@ -1154,10 +1163,10 @@ static void __Pyx_Coroutine_dealloc(PyObject *self) { // Generator is paused or unstarted, so we need to close PyObject_GC_Track(self); #if PY_VERSION_HEX >= 0x030400a1 && CYTHON_USE_TP_FINALIZE - if (PyObject_CallFinalizerFromDealloc(self)) + if (unlikely(PyObject_CallFinalizerFromDealloc(self))) #else Py_TYPE(gen)->tp_del(self); - if (self->ob_refcnt > 0) + if (unlikely(Py_REFCNT(self) > 0)) #endif { // resurrected. :( @@ -1272,8 +1281,8 @@ static void __Pyx_Coroutine_del(PyObject *self) { #if !CYTHON_USE_TP_FINALIZE // Undo the temporary resurrection; can't use DECREF here, it would // cause a recursive call. - assert(self->ob_refcnt > 0); - if (--self->ob_refcnt == 0) { + assert(Py_REFCNT(self) > 0); + if (likely(--self->ob_refcnt == 0)) { // this is the normal path out return; } @@ -1281,12 +1290,12 @@ static void __Pyx_Coroutine_del(PyObject *self) { // close() resurrected it! Make it look like the original Py_DECREF // never happened. { - Py_ssize_t refcnt = self->ob_refcnt; + Py_ssize_t refcnt = Py_REFCNT(self); _Py_NewReference(self); __Pyx_SET_REFCNT(self, refcnt); } #if CYTHON_COMPILING_IN_CPYTHON - assert(PyType_IS_GC(self->ob_type) && + assert(PyType_IS_GC(Py_TYPE(self)) && _Py_AS_GC(self)->gc.gc_refs != _PyGC_REFS_UNTRACKED); // If Py_REF_DEBUG, _Py_NewReference bumped _Py_RefTotal, so @@ -1318,8 +1327,6 @@ __Pyx_Coroutine_get_name(__pyx_CoroutineObject *self, CYTHON_UNUSED void *contex static int __Pyx_Coroutine_set_name(__pyx_CoroutineObject *self, PyObject *value, CYTHON_UNUSED void *context) { - PyObject *tmp; - #if PY_MAJOR_VERSION >= 3 if (unlikely(value == NULL || !PyUnicode_Check(value))) #else @@ -1330,10 +1337,8 @@ __Pyx_Coroutine_set_name(__pyx_CoroutineObject *self, PyObject *value, CYTHON_UN "__name__ must be set to a string object"); return -1; } - tmp = self->gi_name; Py_INCREF(value); - self->gi_name = value; - Py_XDECREF(tmp); + __Pyx_Py_XDECREF_SET(self->gi_name, value); return 0; } @@ -1350,8 +1355,6 @@ __Pyx_Coroutine_get_qualname(__pyx_CoroutineObject *self, CYTHON_UNUSED void *co static int __Pyx_Coroutine_set_qualname(__pyx_CoroutineObject *self, PyObject *value, CYTHON_UNUSED void *context) { - PyObject *tmp; - #if PY_MAJOR_VERSION >= 3 if (unlikely(value == NULL || !PyUnicode_Check(value))) #else @@ -1362,13 +1365,35 @@ __Pyx_Coroutine_set_qualname(__pyx_CoroutineObject *self, PyObject *value, CYTHO "__qualname__ must be set to a string object"); return -1; } - tmp = self->gi_qualname; Py_INCREF(value); - self->gi_qualname = value; - Py_XDECREF(tmp); + __Pyx_Py_XDECREF_SET(self->gi_qualname, value); return 0; } +static PyObject * +__Pyx_Coroutine_get_frame(__pyx_CoroutineObject *self, CYTHON_UNUSED void *context) +{ + PyObject *frame = self->gi_frame; + if (!frame) { + if (unlikely(!self->gi_code)) { + // Avoid doing something stupid, e.g. during garbage collection. + Py_RETURN_NONE; + } + frame = (PyObject *) PyFrame_New( + PyThreadState_Get(), /*PyThreadState *tstate,*/ + (PyCodeObject*) self->gi_code, /*PyCodeObject *code,*/ + $moddict_cname, /*PyObject *globals,*/ + 0 /*PyObject *locals*/ + ); + if (unlikely(!frame)) + return NULL; + // keep the frame cached once it's created + self->gi_frame = frame; + } + Py_INCREF(frame); + return frame; +} + static __pyx_CoroutineObject *__Pyx__Coroutine_New( PyTypeObject* type, __pyx_coroutine_body_t body, PyObject *code, PyObject *closure, PyObject *name, PyObject *qualname, PyObject *module_name) { @@ -1403,6 +1428,7 @@ static __pyx_CoroutineObject *__Pyx__Coroutine_NewInit( gen->gi_modulename = module_name; Py_XINCREF(code); gen->gi_code = code; + gen->gi_frame = NULL; PyObject_GC_Track(gen); return gen; @@ -1470,7 +1496,7 @@ static PyMethodDef __pyx_CoroutineAwait_methods[] = { static PyTypeObject __pyx_CoroutineAwaitType_type = { PyVarObject_HEAD_INIT(0, 0) - "coroutine_wrapper", /*tp_name*/ + __PYX_TYPE_MODULE_PREFIX "coroutine_wrapper", /*tp_name*/ sizeof(__pyx_CoroutineAwaitObject), /*tp_basicsize*/ 0, /*tp_itemsize*/ (destructor) __Pyx_CoroutineAwait_dealloc,/*tp_dealloc*/ @@ -1529,6 +1555,9 @@ static PyTypeObject __pyx_CoroutineAwaitType_type = { #if PY_VERSION_HEX >= 0x030800b4 && PY_VERSION_HEX < 0x03090000 0, /*tp_print*/ #endif +#if CYTHON_COMPILING_IN_PYPY && PYPY_VERSION_NUM+0 >= 0x06000000 + 0, /*tp_pypy_flags*/ +#endif }; #if PY_VERSION_HEX < 0x030500B1 || defined(__Pyx_IterableCoroutine_USED) || CYTHON_USE_ASYNC_SLOTS @@ -1558,13 +1587,6 @@ static PyObject *__Pyx_Coroutine_await(PyObject *coroutine) { } #endif -static PyObject * -__Pyx_Coroutine_get_frame(CYTHON_UNUSED __pyx_CoroutineObject *self, CYTHON_UNUSED void *context) -{ - // Fake implementation that always returns None, but at least does not raise an AttributeError. - Py_RETURN_NONE; -} - #if CYTHON_COMPILING_IN_CPYTHON && PY_MAJOR_VERSION >= 3 && PY_VERSION_HEX < 0x030500B1 static PyObject *__Pyx_Coroutine_compare(PyObject *obj, PyObject *other, int op) { PyObject* result; @@ -1622,7 +1644,7 @@ static __Pyx_PyAsyncMethodsStruct __pyx_Coroutine_as_async = { static PyTypeObject __pyx_CoroutineType_type = { PyVarObject_HEAD_INIT(0, 0) - "coroutine", /*tp_name*/ + __PYX_TYPE_MODULE_PREFIX "coroutine", /*tp_name*/ sizeof(__pyx_CoroutineObject), /*tp_basicsize*/ 0, /*tp_itemsize*/ (destructor) __Pyx_Coroutine_dealloc,/*tp_dealloc*/ @@ -1693,6 +1715,9 @@ static PyTypeObject __pyx_CoroutineType_type = { #if PY_VERSION_HEX >= 0x030800b4 && PY_VERSION_HEX < 0x03090000 0, /*tp_print*/ #endif +#if CYTHON_COMPILING_IN_PYPY && PYPY_VERSION_NUM+0 >= 0x06000000 + 0, /*tp_pypy_flags*/ +#endif }; static int __pyx_Coroutine_init(void) { @@ -1721,7 +1746,7 @@ static int __pyx_Coroutine_init(void) { static PyTypeObject *__pyx_IterableCoroutineType = 0; #undef __Pyx_Coroutine_Check -#define __Pyx_Coroutine_Check(obj) (__Pyx_Coroutine_CheckExact(obj) || (Py_TYPE(obj) == __pyx_IterableCoroutineType)) +#define __Pyx_Coroutine_Check(obj) (__Pyx_Coroutine_CheckExact(obj) || __Pyx_IS_TYPE(obj, __pyx_IterableCoroutineType)) #define __Pyx_IterableCoroutine_New(body, code, closure, name, qualname, module_name) \ __Pyx__Coroutine_New(__pyx_IterableCoroutineType, body, code, closure, name, qualname, module_name) @@ -1735,7 +1760,7 @@ static int __pyx_IterableCoroutine_init(void);/*proto*/ static PyTypeObject __pyx_IterableCoroutineType_type = { PyVarObject_HEAD_INIT(0, 0) - "iterable_coroutine", /*tp_name*/ + __PYX_TYPE_MODULE_PREFIX "iterable_coroutine", /*tp_name*/ sizeof(__pyx_CoroutineObject), /*tp_basicsize*/ 0, /*tp_itemsize*/ (destructor) __Pyx_Coroutine_dealloc,/*tp_dealloc*/ @@ -1804,6 +1829,9 @@ static PyTypeObject __pyx_IterableCoroutineType_type = { #if PY_VERSION_HEX >= 0x030800b4 && PY_VERSION_HEX < 0x03090000 0, /*tp_print*/ #endif +#if CYTHON_COMPILING_IN_PYPY && PYPY_VERSION_NUM+0 >= 0x06000000 + 0, /*tp_pypy_flags*/ +#endif }; @@ -1844,12 +1872,14 @@ static PyGetSetDef __pyx_Generator_getsets[] = { (char*) PyDoc_STR("name of the generator"), 0}, {(char *) "__qualname__", (getter)__Pyx_Coroutine_get_qualname, (setter)__Pyx_Coroutine_set_qualname, (char*) PyDoc_STR("qualified name of the generator"), 0}, + {(char *) "gi_frame", (getter)__Pyx_Coroutine_get_frame, NULL, + (char*) PyDoc_STR("Frame of the coroutine"), 0}, {0, 0, 0, 0, 0} }; static PyTypeObject __pyx_GeneratorType_type = { PyVarObject_HEAD_INIT(0, 0) - "generator", /*tp_name*/ + __PYX_TYPE_MODULE_PREFIX "generator", /*tp_name*/ sizeof(__pyx_CoroutineObject), /*tp_basicsize*/ 0, /*tp_itemsize*/ (destructor) __Pyx_Coroutine_dealloc,/*tp_dealloc*/ @@ -1910,6 +1940,9 @@ static PyTypeObject __pyx_GeneratorType_type = { #if PY_VERSION_HEX >= 0x030800b4 && PY_VERSION_HEX < 0x03090000 0, /*tp_print*/ #endif +#if CYTHON_COMPILING_IN_PYPY && PYPY_VERSION_NUM+0 >= 0x06000000 + 0, /*tp_pypy_flags*/ +#endif }; static int __pyx_Generator_init(void) { @@ -2073,7 +2106,7 @@ static int __Pyx_patch_abc(void) { if (CYTHON_REGISTER_ABCS && !abc_patched) { PyObject *module; module = PyImport_ImportModule((PY_MAJOR_VERSION >= 3) ? "collections.abc" : "collections"); - if (!module) { + if (unlikely(!module)) { PyErr_WriteUnraisable(NULL); if (unlikely(PyErr_WarnEx(PyExc_RuntimeWarning, ((PY_MAJOR_VERSION >= 3) ? @@ -2306,6 +2339,9 @@ static PyTypeObject __Pyx__PyExc_StopAsyncIteration_type = { #if PY_VERSION_HEX >= 0x030400a1 0, /*tp_finalize*/ #endif +#if CYTHON_COMPILING_IN_PYPY && PYPY_VERSION_NUM+0 >= 0x06000000 + 0, /*tp_pypy_flags*/ +#endif }; #endif @@ -2331,7 +2367,7 @@ static int __pyx_StopAsyncIteration_init(void) { __Pyx_PyExc_StopAsyncIteration = (PyObject*) __Pyx_FetchCommonType(&__Pyx__PyExc_StopAsyncIteration_type); if (unlikely(!__Pyx_PyExc_StopAsyncIteration)) return -1; - if (builtins && unlikely(PyMapping_SetItemString(builtins, (char*) "StopAsyncIteration", __Pyx_PyExc_StopAsyncIteration) < 0)) + if (likely(builtins) && unlikely(PyMapping_SetItemString(builtins, (char*) "StopAsyncIteration", __Pyx_PyExc_StopAsyncIteration) < 0)) return -1; #endif return 0; diff --git a/Cython/Utility/CpdefEnums.pyx b/Cython/Utility/CpdefEnums.pyx index 148d776c2..aa19f45ed 100644 --- a/Cython/Utility/CpdefEnums.pyx +++ b/Cython/Utility/CpdefEnums.pyx @@ -6,10 +6,7 @@ cdef extern from *: int PY_VERSION_HEX cdef object __Pyx_OrderedDict -if PY_VERSION_HEX >= 0x02070000: - from collections import OrderedDict as __Pyx_OrderedDict -else: - __Pyx_OrderedDict = dict +from collections import OrderedDict as __Pyx_OrderedDict @cython.internal cdef class __Pyx_EnumMeta(type): @@ -23,8 +20,7 @@ cdef class __Pyx_EnumMeta(type): # @cython.internal cdef object __Pyx_EnumBase -class __Pyx_EnumBase(int): - __metaclass__ = __Pyx_EnumMeta +class __Pyx_EnumBase(int, metaclass=__Pyx_EnumMeta): def __new__(cls, value, name=None): for v in cls: if v == value: @@ -55,12 +51,38 @@ if PY_VERSION_HEX >= 0x03040000: ('{{item}}', {{item}}), {{endfor}} ])) + {{if enum_doc is not None}} + {{name}}.__doc__ = {{ repr(enum_doc) }} + {{endif}} + {{for item in items}} __Pyx_globals['{{item}}'] = {{name}}.{{item}} {{endfor}} else: class {{name}}(__Pyx_EnumBase): - pass + {{ repr(enum_doc) if enum_doc is not None else 'pass' }} {{for item in items}} __Pyx_globals['{{item}}'] = {{name}}({{item}}, '{{item}}') {{endfor}} + +#################### CppScopedEnumType #################### +#@requires: EnumBase +cdef dict __Pyx_globals = globals() + +if PY_VERSION_HEX >= 0x03040000: + # create new IntEnum() + __Pyx_globals["{{name}}"] = __Pyx_EnumBase('{{name}}', __Pyx_OrderedDict([ + {{for item in items}} + ('{{item}}', <{{underlying_type}}>({{name}}.{{item}})), + {{endfor}} + ])) + +else: + __Pyx_globals["{{name}}"] = type('{{name}}', (__Pyx_EnumBase,), {}) + {{for item in items}} + __Pyx_globals["{{name}}"](<{{underlying_type}}>({{name}}.{{item}}), '{{item}}') + {{endfor}} + +{{if enum_doc is not None}} +__Pyx_globals["{{name}}"].__doc__ = {{ repr(enum_doc) }} +{{endif}} diff --git a/Cython/Utility/CppSupport.cpp b/Cython/Utility/CppSupport.cpp index b8fcff064..37d3384b0 100644 --- a/Cython/Utility/CppSupport.cpp +++ b/Cython/Utility/CppSupport.cpp @@ -56,3 +56,27 @@ auto __Pyx_pythran_to_python(T &&value) -> decltype(to_python( using returnable_type = typename pythonic::returnable<typename std::remove_cv<typename std::remove_reference<T>::type>::type>::type; return to_python(returnable_type{std::forward<T>(value)}); } + +#define __Pyx_PythranShapeAccessor(x) (pythonic::builtins::getattr(pythonic::types::attr::SHAPE{}, x)) + +////////////// MoveIfSupported.proto ////////////////// + +#if __cplusplus >= 201103L || (defined(_MSC_VER) && _MSC_VER >= 1600) + // move should be defined for these versions of MSVC, but __cplusplus isn't set usefully + #include <utility> + #define __PYX_STD_MOVE_IF_SUPPORTED(x) std::move(x) +#else + #define __PYX_STD_MOVE_IF_SUPPORTED(x) x +#endif + +////////////// EnumClassDecl.proto ////////////////// + +#if defined (_MSC_VER) + #if PY_VERSION_HEX >= 0x03040000 && PY_VERSION_HEX < 0x03050000 + #define __PYX_ENUM_CLASS_DECL + #else + #define __PYX_ENUM_CLASS_DECL enum + #endif +#else + #define __PYX_ENUM_CLASS_DECL enum +#endif diff --git a/Cython/Utility/CythonFunction.c b/Cython/Utility/CythonFunction.c index 33d0a4750..69e325e12 100644 --- a/Cython/Utility/CythonFunction.c +++ b/Cython/Utility/CythonFunction.c @@ -1,11 +1,12 @@ //////////////////// CythonFunctionShared.proto //////////////////// -#define __Pyx_CyFunction_USED 1 +#define __Pyx_CyFunction_USED #define __Pyx_CYFUNCTION_STATICMETHOD 0x01 #define __Pyx_CYFUNCTION_CLASSMETHOD 0x02 #define __Pyx_CYFUNCTION_CCLASS 0x04 +#define __Pyx_CYFUNCTION_COROUTINE 0x08 #define __Pyx_CyFunction_GetClosure(f) \ (((__pyx_CyFunctionObject *) (f))->func_closure) @@ -20,6 +21,9 @@ typedef struct { PyCFunctionObject func; +#if CYTHON_BACKPORT_VECTORCALL + __pyx_vectorcallfunc func_vectorcall; +#endif #if PY_VERSION_HEX < 0x030500A0 PyObject *func_weakreflist; #endif @@ -44,15 +48,22 @@ typedef struct { PyObject *defaults_kwdict; /* Const kwonly defaults dict */ PyObject *(*defaults_getter)(PyObject *); PyObject *func_annotations; /* function annotations dict */ + + // Coroutine marker + PyObject *func_is_coroutine; } __pyx_CyFunctionObject; +#if !CYTHON_COMPILING_IN_LIMITED_API static PyTypeObject *__pyx_CyFunctionType = 0; +#endif -#define __Pyx_CyFunction_Check(obj) (__Pyx_TypeCheck(obj, __pyx_CyFunctionType)) +#define __Pyx_CyFunction_Check(obj) __Pyx_TypeCheck(obj, __pyx_CyFunctionType) +#define __Pyx_IsCyOrPyCFunction(obj) __Pyx_TypeCheck2(obj, __pyx_CyFunctionType, &PyCFunction_Type) +#define __Pyx_CyFunction_CheckExact(obj) __Pyx_IS_TYPE(obj, __pyx_CyFunctionType) static PyObject *__Pyx_CyFunction_Init(__pyx_CyFunctionObject* op, PyMethodDef *ml, int flags, PyObject* qualname, - PyObject *self, + PyObject *closure, PyObject *module, PyObject *globals, PyObject* code); @@ -69,11 +80,23 @@ static CYTHON_INLINE void __Pyx_CyFunction_SetAnnotationsDict(PyObject *m, static int __pyx_CyFunction_init(void); +#if CYTHON_METH_FASTCALL +static PyObject * __Pyx_CyFunction_Vectorcall_NOARGS(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames); +static PyObject * __Pyx_CyFunction_Vectorcall_O(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames); +static PyObject * __Pyx_CyFunction_Vectorcall_FASTCALL_KEYWORDS(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames); +#if CYTHON_BACKPORT_VECTORCALL +#define __Pyx_CyFunction_func_vectorcall(f) (((__pyx_CyFunctionObject*)f)->func_vectorcall) +#else +#define __Pyx_CyFunction_func_vectorcall(f) (((__pyx_CyFunctionObject*)f)->func.vectorcall) +#endif +#endif //////////////////// CythonFunctionShared //////////////////// //@substitute: naming //@requires: CommonStructures.c::FetchCommonType -////@requires: ObjectHandling.c::PyObjectGetAttrStr +//@requires: ObjectHandling.c::PyMethodNew +//@requires: ObjectHandling.c::PyVectorcallFastCallDict +//@requires: ObjectHandling.c::PyObjectGetAttrStr #include <structmember.h> @@ -101,14 +124,12 @@ __Pyx_CyFunction_get_doc(__pyx_CyFunctionObject *op, CYTHON_UNUSED void *closure static int __Pyx_CyFunction_set_doc(__pyx_CyFunctionObject *op, PyObject *value, CYTHON_UNUSED void *context) { - PyObject *tmp = op->func_doc; if (value == NULL) { // Mark as deleted value = Py_None; } Py_INCREF(value); - op->func_doc = value; - Py_XDECREF(tmp); + __Pyx_Py_XDECREF_SET(op->func_doc, value); return 0; } @@ -131,8 +152,6 @@ __Pyx_CyFunction_get_name(__pyx_CyFunctionObject *op, CYTHON_UNUSED void *contex static int __Pyx_CyFunction_set_name(__pyx_CyFunctionObject *op, PyObject *value, CYTHON_UNUSED void *context) { - PyObject *tmp; - #if PY_MAJOR_VERSION >= 3 if (unlikely(value == NULL || !PyUnicode_Check(value))) #else @@ -143,10 +162,8 @@ __Pyx_CyFunction_set_name(__pyx_CyFunctionObject *op, PyObject *value, CYTHON_UN "__name__ must be set to a string object"); return -1; } - tmp = op->func_name; Py_INCREF(value); - op->func_name = value; - Py_XDECREF(tmp); + __Pyx_Py_XDECREF_SET(op->func_name, value); return 0; } @@ -160,8 +177,6 @@ __Pyx_CyFunction_get_qualname(__pyx_CyFunctionObject *op, CYTHON_UNUSED void *co static int __Pyx_CyFunction_set_qualname(__pyx_CyFunctionObject *op, PyObject *value, CYTHON_UNUSED void *context) { - PyObject *tmp; - #if PY_MAJOR_VERSION >= 3 if (unlikely(value == NULL || !PyUnicode_Check(value))) #else @@ -172,26 +187,12 @@ __Pyx_CyFunction_set_qualname(__pyx_CyFunctionObject *op, PyObject *value, CYTHO "__qualname__ must be set to a string object"); return -1; } - tmp = op->func_qualname; Py_INCREF(value); - op->func_qualname = value; - Py_XDECREF(tmp); + __Pyx_Py_XDECREF_SET(op->func_qualname, value); return 0; } static PyObject * -__Pyx_CyFunction_get_self(__pyx_CyFunctionObject *m, CYTHON_UNUSED void *closure) -{ - PyObject *self; - - self = m->func_closure; - if (self == NULL) - self = Py_None; - Py_INCREF(self); - return self; -} - -static PyObject * __Pyx_CyFunction_get_dict(__pyx_CyFunctionObject *op, CYTHON_UNUSED void *context) { if (unlikely(op->func_dict == NULL)) { @@ -206,8 +207,6 @@ __Pyx_CyFunction_get_dict(__pyx_CyFunctionObject *op, CYTHON_UNUSED void *contex static int __Pyx_CyFunction_set_dict(__pyx_CyFunctionObject *op, PyObject *value, CYTHON_UNUSED void *context) { - PyObject *tmp; - if (unlikely(value == NULL)) { PyErr_SetString(PyExc_TypeError, "function's dictionary may not be deleted"); @@ -218,10 +217,8 @@ __Pyx_CyFunction_set_dict(__pyx_CyFunctionObject *op, PyObject *value, CYTHON_UN "setting function's dictionary to a non-dict"); return -1; } - tmp = op->func_dict; Py_INCREF(value); - op->func_dict = value; - Py_XDECREF(tmp); + __Pyx_Py_XDECREF_SET(op->func_dict, value); return 0; } @@ -274,19 +271,16 @@ __Pyx_CyFunction_init_defaults(__pyx_CyFunctionObject *op) { static int __Pyx_CyFunction_set_defaults(__pyx_CyFunctionObject *op, PyObject* value, CYTHON_UNUSED void *context) { - PyObject* tmp; if (!value) { // del => explicit None to prevent rebuilding value = Py_None; - } else if (value != Py_None && !PyTuple_Check(value)) { + } else if (unlikely(value != Py_None && !PyTuple_Check(value))) { PyErr_SetString(PyExc_TypeError, "__defaults__ must be set to a tuple object"); return -1; } Py_INCREF(value); - tmp = op->defaults_tuple; - op->defaults_tuple = value; - Py_XDECREF(tmp); + __Pyx_Py_XDECREF_SET(op->defaults_tuple, value); return 0; } @@ -295,7 +289,7 @@ __Pyx_CyFunction_get_defaults(__pyx_CyFunctionObject *op, CYTHON_UNUSED void *co PyObject* result = op->defaults_tuple; if (unlikely(!result)) { if (op->defaults_getter) { - if (__Pyx_CyFunction_init_defaults(op) < 0) return NULL; + if (unlikely(__Pyx_CyFunction_init_defaults(op) < 0)) return NULL; result = op->defaults_tuple; } else { result = Py_None; @@ -307,19 +301,16 @@ __Pyx_CyFunction_get_defaults(__pyx_CyFunctionObject *op, CYTHON_UNUSED void *co static int __Pyx_CyFunction_set_kwdefaults(__pyx_CyFunctionObject *op, PyObject* value, CYTHON_UNUSED void *context) { - PyObject* tmp; if (!value) { // del => explicit None to prevent rebuilding value = Py_None; - } else if (value != Py_None && !PyDict_Check(value)) { + } else if (unlikely(value != Py_None && !PyDict_Check(value))) { PyErr_SetString(PyExc_TypeError, "__kwdefaults__ must be set to a dict object"); return -1; } Py_INCREF(value); - tmp = op->defaults_kwdict; - op->defaults_kwdict = value; - Py_XDECREF(tmp); + __Pyx_Py_XDECREF_SET(op->defaults_kwdict, value); return 0; } @@ -328,7 +319,7 @@ __Pyx_CyFunction_get_kwdefaults(__pyx_CyFunctionObject *op, CYTHON_UNUSED void * PyObject* result = op->defaults_kwdict; if (unlikely(!result)) { if (op->defaults_getter) { - if (__Pyx_CyFunction_init_defaults(op) < 0) return NULL; + if (unlikely(__Pyx_CyFunction_init_defaults(op) < 0)) return NULL; result = op->defaults_kwdict; } else { result = Py_None; @@ -340,18 +331,15 @@ __Pyx_CyFunction_get_kwdefaults(__pyx_CyFunctionObject *op, CYTHON_UNUSED void * static int __Pyx_CyFunction_set_annotations(__pyx_CyFunctionObject *op, PyObject* value, CYTHON_UNUSED void *context) { - PyObject* tmp; if (!value || value == Py_None) { value = NULL; - } else if (!PyDict_Check(value)) { + } else if (unlikely(!PyDict_Check(value))) { PyErr_SetString(PyExc_TypeError, "__annotations__ must be set to a dict object"); return -1; } Py_XINCREF(value); - tmp = op->func_annotations; - op->func_annotations = value; - Py_XDECREF(tmp); + __Pyx_Py_XDECREF_SET(op->func_annotations, value); return 0; } @@ -367,6 +355,37 @@ __Pyx_CyFunction_get_annotations(__pyx_CyFunctionObject *op, CYTHON_UNUSED void return result; } +static PyObject * +__Pyx_CyFunction_get_is_coroutine(__pyx_CyFunctionObject *op, CYTHON_UNUSED void *context) { + int is_coroutine; + if (op->func_is_coroutine) { + return __Pyx_NewRef(op->func_is_coroutine); + } + + is_coroutine = op->flags & __Pyx_CYFUNCTION_COROUTINE; +#if PY_VERSION_HEX >= 0x03050000 + if (is_coroutine) { + PyObject *module, *fromlist, *marker = PYIDENT("_is_coroutine"); + fromlist = PyList_New(1); + if (unlikely(!fromlist)) return NULL; + PyList_SET_ITEM(fromlist, 0, marker); + module = PyImport_ImportModuleLevelObject(PYIDENT("asyncio.coroutines"), NULL, NULL, fromlist, 0); + Py_DECREF(fromlist); + if (unlikely(!module)) goto ignore; + op->func_is_coroutine = __Pyx_PyObject_GetAttrStr(module, marker); + Py_DECREF(module); + if (likely(op->func_is_coroutine)) { + return __Pyx_NewRef(op->func_is_coroutine); + } +ignore: + PyErr_Clear(); + } +#endif + + op->func_is_coroutine = __Pyx_PyBool_FromLong(is_coroutine); + return __Pyx_NewRef(op->func_is_coroutine); +} + //#if PY_VERSION_HEX >= 0x030400C1 //static PyObject * //__Pyx_CyFunction_get_signature(__pyx_CyFunctionObject *op, CYTHON_UNUSED void *context) { @@ -398,7 +417,6 @@ static PyGetSetDef __pyx_CyFunction_getsets[] = { {(char *) "func_name", (getter)__Pyx_CyFunction_get_name, (setter)__Pyx_CyFunction_set_name, 0, 0}, {(char *) "__name__", (getter)__Pyx_CyFunction_get_name, (setter)__Pyx_CyFunction_set_name, 0, 0}, {(char *) "__qualname__", (getter)__Pyx_CyFunction_get_qualname, (setter)__Pyx_CyFunction_set_qualname, 0, 0}, - {(char *) "__self__", (getter)__Pyx_CyFunction_get_self, 0, 0, 0}, {(char *) "func_dict", (getter)__Pyx_CyFunction_get_dict, (setter)__Pyx_CyFunction_set_dict, 0, 0}, {(char *) "__dict__", (getter)__Pyx_CyFunction_get_dict, (setter)__Pyx_CyFunction_set_dict, 0, 0}, {(char *) "func_globals", (getter)__Pyx_CyFunction_get_globals, 0, 0, 0}, @@ -411,6 +429,7 @@ static PyGetSetDef __pyx_CyFunction_getsets[] = { {(char *) "__defaults__", (getter)__Pyx_CyFunction_get_defaults, (setter)__Pyx_CyFunction_set_defaults, 0, 0}, {(char *) "__kwdefaults__", (getter)__Pyx_CyFunction_get_kwdefaults, (setter)__Pyx_CyFunction_set_kwdefaults, 0, 0}, {(char *) "__annotations__", (getter)__Pyx_CyFunction_get_annotations, (setter)__Pyx_CyFunction_set_annotations, 0, 0}, + {(char *) "_is_coroutine", (getter)__Pyx_CyFunction_get_is_coroutine, 0, 0, 0}, //#if PY_VERSION_HEX >= 0x030400C1 // {(char *) "__signature__", (getter)__Pyx_CyFunction_get_signature, 0, 0, 0}, //#endif @@ -419,6 +438,14 @@ static PyGetSetDef __pyx_CyFunction_getsets[] = { static PyMemberDef __pyx_CyFunction_members[] = { {(char *) "__module__", T_OBJECT, offsetof(PyCFunctionObject, m_module), PY_WRITE_RESTRICTED, 0}, +#if CYTHON_COMPILING_IN_LIMITED_API + {(char *) "__dictoffset__", T_PYSSIZET, offsetof(__pyx_CyFunctionObject, func_dict), READONLY, 0}, +#if PY_VERSION_HEX < 0x030500A0 + {(char *) "__weaklistoffset__", T_PYSSIZET, offsetof(__pyx_CyFunctionObject, func_weakreflist), READONLY, 0}, +#else + {(char *) "__weaklistoffset__", T_PYSSIZET, offsetof(PyCFunctionObject, m_weakreflist), READONLY, 0}, +#endif +#endif {0, 0, 0, 0, 0} }; @@ -426,7 +453,8 @@ static PyObject * __Pyx_CyFunction_reduce(__pyx_CyFunctionObject *m, CYTHON_UNUSED PyObject *args) { #if PY_MAJOR_VERSION >= 3 - return PyUnicode_FromString(m->func.m_ml->ml_name); + Py_INCREF(m->func_qualname); + return m->func_qualname; #else return PyString_FromString(m->func.m_ml->ml_name); #endif @@ -474,6 +502,29 @@ static PyObject *__Pyx_CyFunction_Init(__pyx_CyFunctionObject *op, PyMethodDef * op->defaults_kwdict = NULL; op->defaults_getter = NULL; op->func_annotations = NULL; + op->func_is_coroutine = NULL; +#if CYTHON_METH_FASTCALL + switch (ml->ml_flags & (METH_VARARGS | METH_FASTCALL | METH_NOARGS | METH_O | METH_KEYWORDS)) { + case METH_NOARGS: + __Pyx_CyFunction_func_vectorcall(op) = __Pyx_CyFunction_Vectorcall_NOARGS; + break; + case METH_O: + __Pyx_CyFunction_func_vectorcall(op) = __Pyx_CyFunction_Vectorcall_O; + break; + // case METH_FASTCALL is not used + case METH_FASTCALL | METH_KEYWORDS: + __Pyx_CyFunction_func_vectorcall(op) = __Pyx_CyFunction_Vectorcall_FASTCALL_KEYWORDS; + break; + // case METH_VARARGS is not used + case METH_VARARGS | METH_KEYWORDS: + __Pyx_CyFunction_func_vectorcall(op) = NULL; + break; + default: + PyErr_SetString(PyExc_SystemError, "Bad call flags for CyFunction"); + Py_DECREF(op); + return NULL; + } +#endif return (PyObject *) op; } @@ -492,6 +543,7 @@ __Pyx_CyFunction_clear(__pyx_CyFunctionObject *m) Py_CLEAR(m->defaults_tuple); Py_CLEAR(m->defaults_kwdict); Py_CLEAR(m->func_annotations); + Py_CLEAR(m->func_is_coroutine); if (m->defaults) { PyObject **pydefaults = __Pyx_CyFunction_Defaults(PyObject *, m); @@ -534,6 +586,7 @@ static int __Pyx_CyFunction_traverse(__pyx_CyFunctionObject *m, visitproc visit, Py_VISIT(m->func_classobj); Py_VISIT(m->defaults_tuple); Py_VISIT(m->defaults_kwdict); + Py_VISIT(m->func_is_coroutine); if (m->defaults) { PyObject **pydefaults = __Pyx_CyFunction_Defaults(PyObject *, m); @@ -546,28 +599,6 @@ static int __Pyx_CyFunction_traverse(__pyx_CyFunctionObject *m, visitproc visit, return 0; } -static PyObject *__Pyx_CyFunction_descr_get(PyObject *func, PyObject *obj, PyObject *type) -{ -#if PY_MAJOR_VERSION < 3 - __pyx_CyFunctionObject *m = (__pyx_CyFunctionObject *) func; - - if (m->flags & __Pyx_CYFUNCTION_STATICMETHOD) { - Py_INCREF(func); - return func; - } - - if (m->flags & __Pyx_CYFUNCTION_CLASSMETHOD) { - if (type == NULL) - type = (PyObject *)(Py_TYPE(obj)); - return __Pyx_PyMethod_New(func, type, (PyObject *)(Py_TYPE(type))); - } - - if (obj == Py_None) - obj = NULL; -#endif - return __Pyx_PyMethod_New(func, obj, type); -} - static PyObject* __Pyx_CyFunction_repr(__pyx_CyFunctionObject *op) { @@ -627,10 +658,7 @@ static PyObject * __Pyx_CyFunction_CallMethod(PyObject *func, PyObject *self, Py } break; default: - PyErr_SetString(PyExc_SystemError, "Bad call flags in " - "__Pyx_CyFunction_Call. METH_OLDARGS is no " - "longer supported!"); - + PyErr_SetString(PyExc_SystemError, "Bad call flags for CyFunction"); return NULL; } PyErr_Format(PyExc_TypeError, "%.200s() takes no keyword arguments", @@ -645,6 +673,22 @@ static CYTHON_INLINE PyObject *__Pyx_CyFunction_Call(PyObject *func, PyObject *a static PyObject *__Pyx_CyFunction_CallAsMethod(PyObject *func, PyObject *args, PyObject *kw) { PyObject *result; __pyx_CyFunctionObject *cyfunc = (__pyx_CyFunctionObject *) func; + +#if CYTHON_METH_FASTCALL + // Prefer vectorcall if available. This is not the typical case, as + // CPython would normally use vectorcall directly instead of tp_call. + __pyx_vectorcallfunc vc = __Pyx_CyFunction_func_vectorcall(cyfunc); + if (vc) { +#if CYTHON_ASSUME_SAFE_MACROS + return __Pyx_PyVectorcall_FastCallDict(func, vc, &PyTuple_GET_ITEM(args, 0), (size_t)PyTuple_GET_SIZE(args), kw); +#else + // avoid unused function warning + (void) &__Pyx_PyVectorcall_FastCallDict; + return PyVectorcall_Call(func, args, kw); +#endif + } +#endif + if ((cyfunc->flags & __Pyx_CYFUNCTION_CCLASS) && !(cyfunc->flags & __Pyx_CYFUNCTION_STATICMETHOD)) { Py_ssize_t argc; PyObject *new_args; @@ -670,19 +714,165 @@ static PyObject *__Pyx_CyFunction_CallAsMethod(PyObject *func, PyObject *args, P return result; } +#if CYTHON_METH_FASTCALL +// Check that kwnames is empty (if you want to allow keyword arguments, +// simply pass kwnames=NULL) and figure out what to do with "self". +// Return value: +// 1: self = args[0] +// 0: self = cyfunc->func.m_self +// -1: error +static CYTHON_INLINE int __Pyx_CyFunction_Vectorcall_CheckArgs(__pyx_CyFunctionObject *cyfunc, Py_ssize_t nargs, PyObject *kwnames) +{ + int ret = 0; + if ((cyfunc->flags & __Pyx_CYFUNCTION_CCLASS) && !(cyfunc->flags & __Pyx_CYFUNCTION_STATICMETHOD)) { + if (unlikely(nargs < 1)) { + PyErr_Format(PyExc_TypeError, "%.200s() needs an argument", + cyfunc->func.m_ml->ml_name); + return -1; + } + ret = 1; + } + if (unlikely(kwnames) && unlikely(PyTuple_GET_SIZE(kwnames))) { + PyErr_Format(PyExc_TypeError, + "%.200s() takes no keyword arguments", cyfunc->func.m_ml->ml_name); + return -1; + } + return ret; +} + +static PyObject * __Pyx_CyFunction_Vectorcall_NOARGS(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames) +{ + __pyx_CyFunctionObject *cyfunc = (__pyx_CyFunctionObject *)func; + PyMethodDef* def = cyfunc->func.m_ml; +#if CYTHON_BACKPORT_VECTORCALL + Py_ssize_t nargs = (Py_ssize_t)nargsf; +#else + Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); +#endif + PyObject *self; + switch (__Pyx_CyFunction_Vectorcall_CheckArgs(cyfunc, nargs, kwnames)) { + case 1: + self = args[0]; + args += 1; + nargs -= 1; + break; + case 0: + self = cyfunc->func.m_self; + break; + default: + return NULL; + } + + if (unlikely(nargs != 0)) { + PyErr_Format(PyExc_TypeError, + "%.200s() takes no arguments (%" CYTHON_FORMAT_SSIZE_T "d given)", + def->ml_name, nargs); + return NULL; + } + return def->ml_meth(self, NULL); +} + +static PyObject * __Pyx_CyFunction_Vectorcall_O(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames) +{ + __pyx_CyFunctionObject *cyfunc = (__pyx_CyFunctionObject *)func; + PyMethodDef* def = cyfunc->func.m_ml; +#if CYTHON_BACKPORT_VECTORCALL + Py_ssize_t nargs = (Py_ssize_t)nargsf; +#else + Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); +#endif + PyObject *self; + switch (__Pyx_CyFunction_Vectorcall_CheckArgs(cyfunc, nargs, kwnames)) { + case 1: + self = args[0]; + args += 1; + nargs -= 1; + break; + case 0: + self = cyfunc->func.m_self; + break; + default: + return NULL; + } + + if (unlikely(nargs != 1)) { + PyErr_Format(PyExc_TypeError, + "%.200s() takes exactly one argument (%" CYTHON_FORMAT_SSIZE_T "d given)", + def->ml_name, nargs); + return NULL; + } + return def->ml_meth(self, args[0]); +} + +static PyObject * __Pyx_CyFunction_Vectorcall_FASTCALL_KEYWORDS(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames) +{ + __pyx_CyFunctionObject *cyfunc = (__pyx_CyFunctionObject *)func; + PyMethodDef* def = cyfunc->func.m_ml; +#if CYTHON_BACKPORT_VECTORCALL + Py_ssize_t nargs = (Py_ssize_t)nargsf; +#else + Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); +#endif + PyObject *self; + switch (__Pyx_CyFunction_Vectorcall_CheckArgs(cyfunc, nargs, NULL)) { + case 1: + self = args[0]; + args += 1; + nargs -= 1; + break; + case 0: + self = cyfunc->func.m_self; + break; + default: + return NULL; + } + + return ((_PyCFunctionFastWithKeywords)(void(*)(void))def->ml_meth)(self, args, nargs, kwnames); +} +#endif + +#if CYTHON_COMPILING_IN_LIMITED_API +static PyType_Slot __pyx_CyFunctionType_slots[] = { + {Py_tp_dealloc, (void *)__Pyx_CyFunction_dealloc}, + {Py_tp_repr, (void *)__Pyx_CyFunction_repr}, + {Py_tp_call, (void *)__Pyx_CyFunction_CallAsMethod}, + {Py_tp_traverse, (void *)__Pyx_CyFunction_traverse}, + {Py_tp_clear, (void *)__Pyx_CyFunction_clear}, + {Py_tp_methods, (void *)__pyx_CyFunction_methods}, + {Py_tp_members, (void *)__pyx_CyFunction_members}, + {Py_tp_getset, (void *)__pyx_CyFunction_getsets}, + {Py_tp_descr_get, (void *)__Pyx_PyMethod_New}, + {0, 0}, +}; + +static PyType_Spec __pyx_CyFunctionType_spec = { + __PYX_TYPE_MODULE_PREFIX "cython_function_or_method", + sizeof(__pyx_CyFunctionObject), + 0, + // TODO: Support _Py_TPFLAGS_HAVE_VECTORCALL and _Py_TPFLAGS_HAVE_VECTORCALL + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /*tp_flags*/ + __pyx_CyFunctionType_slots +}; +#else static PyTypeObject __pyx_CyFunctionType_type = { PyVarObject_HEAD_INIT(0, 0) - "cython_function_or_method", /*tp_name*/ + __PYX_TYPE_MODULE_PREFIX "cython_function_or_method", /*tp_name*/ sizeof(__pyx_CyFunctionObject), /*tp_basicsize*/ 0, /*tp_itemsize*/ (destructor) __Pyx_CyFunction_dealloc, /*tp_dealloc*/ +#if !CYTHON_METH_FASTCALL 0, /*tp_print*/ +#elif CYTHON_BACKPORT_VECTORCALL + (printfunc)offsetof(__pyx_CyFunctionObject, func_vectorcall), /*tp_vectorcall_offset backported into tp_print*/ +#else + offsetof(__pyx_CyFunctionObject, func.vectorcall), /*tp_vectorcall_offset*/ +#endif 0, /*tp_getattr*/ 0, /*tp_setattr*/ #if PY_MAJOR_VERSION < 3 0, /*tp_compare*/ #else - 0, /*reserved*/ + 0, /*tp_as_async*/ #endif (reprfunc) __Pyx_CyFunction_repr, /*tp_repr*/ 0, /*tp_as_number*/ @@ -694,6 +884,12 @@ static PyTypeObject __pyx_CyFunctionType_type = { 0, /*tp_getattro*/ 0, /*tp_setattro*/ 0, /*tp_as_buffer*/ +#ifdef Py_TPFLAGS_METHOD_DESCRIPTOR + Py_TPFLAGS_METHOD_DESCRIPTOR | +#endif +#ifdef _Py_TPFLAGS_HAVE_VECTORCALL + _Py_TPFLAGS_HAVE_VECTORCALL | +#endif Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /*tp_flags*/ 0, /*tp_doc*/ (traverseproc) __Pyx_CyFunction_traverse, /*tp_traverse*/ @@ -711,7 +907,7 @@ static PyTypeObject __pyx_CyFunctionType_type = { __pyx_CyFunction_getsets, /*tp_getset*/ 0, /*tp_base*/ 0, /*tp_dict*/ - __Pyx_CyFunction_descr_get, /*tp_descr_get*/ + __Pyx_PyMethod_New, /*tp_descr_get*/ 0, /*tp_descr_set*/ offsetof(__pyx_CyFunctionObject, func_dict),/*tp_dictoffset*/ 0, /*tp_init*/ @@ -735,11 +931,19 @@ static PyTypeObject __pyx_CyFunctionType_type = { #if PY_VERSION_HEX >= 0x030800b4 && PY_VERSION_HEX < 0x03090000 0, /*tp_print*/ #endif +#if CYTHON_COMPILING_IN_PYPY && PYPY_VERSION_NUM+0 >= 0x06000000 + 0, /*tp_pypy_flags*/ +#endif }; +#endif /* CYTHON_COMPILING_IN_LIMITED_API */ static int __pyx_CyFunction_init(void) { +#if CYTHON_COMPILING_IN_LIMITED_API + __pyx_CyFunctionType = __Pyx_FetchCommonTypeFromSpec(&__pyx_CyFunctionType_spec, NULL); +#else __pyx_CyFunctionType = __Pyx_FetchCommonType(&__pyx_CyFunctionType_type); +#endif if (unlikely(__pyx_CyFunctionType == NULL)) { return -1; } @@ -834,7 +1038,6 @@ static int __Pyx_CyFunction_InitClassCell(PyObject *cyfunctions, PyObject *class typedef struct { __pyx_CyFunctionObject func; PyObject *__signatures__; - PyObject *type; PyObject *self; } __pyx_FusedFunctionObject; @@ -844,7 +1047,9 @@ static PyObject *__pyx_FusedFunction_New(PyMethodDef *ml, int flags, PyObject *code); static int __pyx_FusedFunction_clear(__pyx_FusedFunctionObject *self); +#if !CYTHON_COMPILING_IN_LIMITED_API static PyTypeObject *__pyx_FusedFunctionType = NULL; +#endif static int __pyx_FusedFunction_init(void); #define __Pyx_FusedFunction_USED @@ -866,7 +1071,6 @@ __pyx_FusedFunction_New(PyMethodDef *ml, int flags, if (likely(op)) { __pyx_FusedFunctionObject *fusedfunc = (__pyx_FusedFunctionObject *) op; fusedfunc->__signatures__ = NULL; - fusedfunc->type = NULL; fusedfunc->self = NULL; PyObject_GC_Track(op); } @@ -878,7 +1082,6 @@ __pyx_FusedFunction_dealloc(__pyx_FusedFunctionObject *self) { PyObject_GC_UnTrack(self); Py_CLEAR(self->self); - Py_CLEAR(self->type); Py_CLEAR(self->__signatures__); __Pyx__CyFunction_dealloc((__pyx_CyFunctionObject *) self); } @@ -889,7 +1092,6 @@ __pyx_FusedFunction_traverse(__pyx_FusedFunctionObject *self, void *arg) { Py_VISIT(self->self); - Py_VISIT(self->type); Py_VISIT(self->__signatures__); return __Pyx_CyFunction_traverse((__pyx_CyFunctionObject *) self, visit, arg); } @@ -898,7 +1100,6 @@ static int __pyx_FusedFunction_clear(__pyx_FusedFunctionObject *self) { Py_CLEAR(self->self); - Py_CLEAR(self->type); Py_CLEAR(self->__signatures__); return __Pyx_CyFunction_clear((__pyx_CyFunctionObject *) self); } @@ -928,7 +1129,7 @@ __pyx_FusedFunction_descr_get(PyObject *self, PyObject *obj, PyObject *type) ((PyCFunctionObject *) func)->m_module, ((__pyx_CyFunctionObject *) func)->func_globals, ((__pyx_CyFunctionObject *) func)->func_code); - if (!meth) + if (unlikely(!meth)) return NULL; // defaults needs copying fully rather than just copying the pointer @@ -938,9 +1139,10 @@ __pyx_FusedFunction_descr_get(PyObject *self, PyObject *obj, PyObject *type) PyObject **pydefaults; int i; - if (!__Pyx_CyFunction_InitDefaults((PyObject*)meth, - func->func.defaults_size, - func->func.defaults_pyobjects)) { + if (unlikely(!__Pyx_CyFunction_InitDefaults( + (PyObject*)meth, + func->func.defaults_size, + func->func.defaults_pyobjects))) { Py_XDECREF((PyObject*)meth); return NULL; } @@ -957,9 +1159,6 @@ __pyx_FusedFunction_descr_get(PyObject *self, PyObject *obj, PyObject *type) Py_XINCREF(func->__signatures__); meth->__signatures__ = func->__signatures__; - Py_XINCREF(type); - meth->type = type; - Py_XINCREF(func->func.defaults_tuple); meth->func.defaults_tuple = func->func.defaults_tuple; @@ -973,12 +1172,18 @@ __pyx_FusedFunction_descr_get(PyObject *self, PyObject *obj, PyObject *type) } static PyObject * -_obj_to_str(PyObject *obj) +_obj_to_string(PyObject *obj) { - if (PyType_Check(obj)) + if (PyUnicode_CheckExact(obj)) + return __Pyx_NewRef(obj); +#if PY_MAJOR_VERSION == 2 + else if (PyString_Check(obj)) + return PyUnicode_FromEncodedObject(obj, NULL, "strict"); +#endif + else if (PyType_Check(obj)) return PyObject_GetAttr(obj, PYIDENT("__name__")); else - return PyObject_Str(obj); + return PyObject_Unicode(obj); } static PyObject * @@ -988,65 +1193,56 @@ __pyx_FusedFunction_getitem(__pyx_FusedFunctionObject *self, PyObject *idx) PyObject *unbound_result_func; PyObject *result_func = NULL; - if (self->__signatures__ == NULL) { + if (unlikely(self->__signatures__ == NULL)) { PyErr_SetString(PyExc_TypeError, "Function is not fused"); return NULL; } if (PyTuple_Check(idx)) { - PyObject *list = PyList_New(0); Py_ssize_t n = PyTuple_GET_SIZE(idx); - PyObject *sep = NULL; + PyObject *list = PyList_New(n); int i; if (unlikely(!list)) return NULL; for (i = 0; i < n; i++) { - int ret; PyObject *string; #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS PyObject *item = PyTuple_GET_ITEM(idx, i); #else PyObject *item = PySequence_ITEM(idx, i); if (unlikely(!item)) goto __pyx_err; #endif - string = _obj_to_str(item); + string = _obj_to_string(item); #if !(CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS) Py_DECREF(item); #endif if (unlikely(!string)) goto __pyx_err; - ret = PyList_Append(list, string); - Py_DECREF(string); - if (unlikely(ret < 0)) goto __pyx_err; + PyList_SET_ITEM(list, i, string); } - sep = PyUnicode_FromString("|"); - if (likely(sep)) - signature = PyUnicode_Join(sep, list); -__pyx_err: -; + signature = PyUnicode_Join(PYUNICODE("|"), list); +__pyx_err:; Py_DECREF(list); - Py_XDECREF(sep); } else { - signature = _obj_to_str(idx); + signature = _obj_to_string(idx); } - if (!signature) + if (unlikely(!signature)) return NULL; unbound_result_func = PyObject_GetItem(self->__signatures__, signature); - if (unbound_result_func) { - if (self->self || self->type) { + if (likely(unbound_result_func)) { + if (self->self) { __pyx_FusedFunctionObject *unbound = (__pyx_FusedFunctionObject *) unbound_result_func; // TODO: move this to InitClassCell - Py_CLEAR(unbound->func.func_classobj); Py_XINCREF(self->func.func_classobj); - unbound->func.func_classobj = self->func.func_classobj; + __Pyx_Py_XDECREF_SET(unbound->func.func_classobj, self->func.func_classobj); result_func = __pyx_FusedFunction_descr_get(unbound_result_func, - self->self, self->type); + self->self, self->self); } else { result_func = unbound_result_func; Py_INCREF(result_func); @@ -1066,7 +1262,7 @@ __pyx_FusedFunction_callfunction(PyObject *func, PyObject *args, PyObject *kw) int static_specialized = (cyfunc->flags & __Pyx_CYFUNCTION_STATICMETHOD && !((__pyx_FusedFunctionObject *) func)->__signatures__); - if (cyfunc->flags & __Pyx_CYFUNCTION_CCLASS && !static_specialized) { + if ((cyfunc->flags & __Pyx_CYFUNCTION_CCLASS) && !static_specialized) { return __Pyx_CyFunction_CallAsMethod(func, args, kw); } else { return __Pyx_CyFunction_Call(func, args, kw); @@ -1087,23 +1283,21 @@ __pyx_FusedFunction_call(PyObject *func, PyObject *args, PyObject *kw) PyObject *new_args = NULL; __pyx_FusedFunctionObject *new_func = NULL; PyObject *result = NULL; - PyObject *self = NULL; int is_staticmethod = binding_func->func.flags & __Pyx_CYFUNCTION_STATICMETHOD; - int is_classmethod = binding_func->func.flags & __Pyx_CYFUNCTION_CLASSMETHOD; if (binding_func->self) { // Bound method call, put 'self' in the args tuple + PyObject *self; Py_ssize_t i; new_args = PyTuple_New(argc + 1); - if (!new_args) + if (unlikely(!new_args)) return NULL; self = binding_func->self; -#if !(CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS) - Py_INCREF(self); -#endif + Py_INCREF(self); PyTuple_SET_ITEM(new_args, 0, self); + self = NULL; for (i = 0; i < argc; i++) { #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS @@ -1116,35 +1310,7 @@ __pyx_FusedFunction_call(PyObject *func, PyObject *args, PyObject *kw) } args = new_args; - } else if (binding_func->type) { - // Unbound method call - if (argc < 1) { - PyErr_SetString(PyExc_TypeError, "Need at least one argument, 0 given."); - return NULL; - } -#if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS - self = PyTuple_GET_ITEM(args, 0); -#else - self = PySequence_ITEM(args, 0); if (unlikely(!self)) return NULL; -#endif - } - - if (self && !is_classmethod && !is_staticmethod) { - int is_instance = PyObject_IsInstance(self, binding_func->type); - if (unlikely(!is_instance)) { - PyErr_Format(PyExc_TypeError, - "First argument should be of type %.200s, got %.200s.", - ((PyTypeObject *) binding_func->type)->tp_name, - self->ob_type->tp_name); - goto bad; - } else if (unlikely(is_instance == -1)) { - goto bad; - } } -#if !(CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS) - Py_XDECREF(self); - self = NULL; -#endif if (binding_func->__signatures__) { PyObject *tup; @@ -1169,22 +1335,30 @@ __pyx_FusedFunction_call(PyObject *func, PyObject *args, PyObject *kw) goto bad; Py_XINCREF(binding_func->func.func_classobj); - Py_CLEAR(new_func->func.func_classobj); - new_func->func.func_classobj = binding_func->func.func_classobj; + __Pyx_Py_XDECREF_SET(new_func->func.func_classobj, binding_func->func.func_classobj); func = (PyObject *) new_func; } result = __pyx_FusedFunction_callfunction(func, args, kw); bad: -#if !(CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS) - Py_XDECREF(self); -#endif Py_XDECREF(new_args); Py_XDECREF((PyObject *) new_func); return result; } +static PyObject * +__Pyx_FusedFunction_get_self(__pyx_FusedFunctionObject *m, CYTHON_UNUSED void *closure) +{ + PyObject *self = m->self; + if (unlikely(!self)) { + PyErr_SetString(PyExc_AttributeError, "'function' object has no attribute '__self__'"); + } else { + Py_INCREF(self); + } + return self; +} + static PyMemberDef __pyx_FusedFunction_members[] = { {(char *) "__signatures__", T_OBJECT, @@ -1194,6 +1368,38 @@ static PyMemberDef __pyx_FusedFunction_members[] = { {0, 0, 0, 0, 0}, }; +static PyGetSetDef __pyx_FusedFunction_getsets[] = { + {(char *) "__self__", (getter)__Pyx_FusedFunction_get_self, 0, 0, 0}, + // __doc__ is None for the fused function type, but we need it to be + // a descriptor for the instance's __doc__, so rebuild the descriptor in our subclass + // (all other descriptors are inherited) + {(char *) "__doc__", (getter)__Pyx_CyFunction_get_doc, (setter)__Pyx_CyFunction_set_doc, 0, 0}, + {0, 0, 0, 0, 0} +}; + +#if CYTHON_COMPILING_IN_LIMITED_API +static PyType_Slot __pyx_FusedFunctionType_slots[] = { + {Py_tp_dealloc, (void *)__pyx_FusedFunction_dealloc}, + {Py_tp_call, (void *)__pyx_FusedFunction_call}, + {Py_tp_traverse, (void *)__pyx_FusedFunction_traverse}, + {Py_tp_clear, (void *)__pyx_FusedFunction_clear}, + {Py_tp_members, (void *)__pyx_FusedFunction_members}, + {Py_tp_getset, (void *)__pyx_FusedFunction_getsets}, + {Py_tp_descr_get, (void *)__pyx_FusedFunction_descr_get}, + {Py_mp_subscript, (void *)__pyx_FusedFunction_getitem}, + {0, 0}, +}; + +static PyType_Spec __pyx_FusedFunctionType_spec = { + __PYX_TYPE_MODULE_PREFIX "fused_cython_function", + sizeof(__pyx_FusedFunctionObject), + 0, + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_BASETYPE, /*tp_flags*/ + __pyx_FusedFunctionType_slots +}; + +#else /* !CYTHON_COMPILING_IN_LIMITED_API */ + static PyMappingMethods __pyx_FusedFunction_mapping_methods = { 0, (binaryfunc) __pyx_FusedFunction_getitem, @@ -1202,7 +1408,7 @@ static PyMappingMethods __pyx_FusedFunction_mapping_methods = { static PyTypeObject __pyx_FusedFunctionType_type = { PyVarObject_HEAD_INIT(0, 0) - "fused_cython_function", /*tp_name*/ + __PYX_TYPE_MODULE_PREFIX "fused_cython_function", /*tp_name*/ sizeof(__pyx_FusedFunctionObject), /*tp_basicsize*/ 0, /*tp_itemsize*/ (destructor) __pyx_FusedFunction_dealloc, /*tp_dealloc*/ @@ -1212,7 +1418,7 @@ static PyTypeObject __pyx_FusedFunctionType_type = { #if PY_MAJOR_VERSION < 3 0, /*tp_compare*/ #else - 0, /*reserved*/ + 0, /*tp_as_async*/ #endif 0, /*tp_repr*/ 0, /*tp_as_number*/ @@ -1234,9 +1440,7 @@ static PyTypeObject __pyx_FusedFunctionType_type = { 0, /*tp_iternext*/ 0, /*tp_methods*/ __pyx_FusedFunction_members, /*tp_members*/ - // __doc__ is None for the fused function type, but we need it to be - // a descriptor for the instance's __doc__, so rebuild descriptors in our subclass - __pyx_CyFunction_getsets, /*tp_getset*/ + __pyx_FusedFunction_getsets, /*tp_getset*/ // NOTE: tp_base may be changed later during module initialisation when importing CyFunction across modules. &__pyx_CyFunctionType_type, /*tp_base*/ 0, /*tp_dict*/ @@ -1264,13 +1468,26 @@ static PyTypeObject __pyx_FusedFunctionType_type = { #if PY_VERSION_HEX >= 0x030800b4 && PY_VERSION_HEX < 0x03090000 0, /*tp_print*/ #endif +#if CYTHON_COMPILING_IN_PYPY && PYPY_VERSION_NUM+0 >= 0x06000000 + 0, /*tp_pypy_flags*/ +#endif }; +#endif static int __pyx_FusedFunction_init(void) { +#if CYTHON_COMPILING_IN_LIMITED_API + PyObject *bases = PyTuple_Pack(1, __pyx_CyFunctionType); + if (unlikely(!bases)) { + return -1; + } + __pyx_FusedFunctionType = __Pyx_FetchCommonTypeFromSpec(&__pyx_FusedFunctionType_spec, bases); + Py_DECREF(bases); +#else // Set base from __Pyx_FetchCommonTypeFromSpec, in case it's different from the local static value. __pyx_FusedFunctionType_type.tp_base = __pyx_CyFunctionType; __pyx_FusedFunctionType = __Pyx_FetchCommonType(&__pyx_FusedFunctionType_type); - if (__pyx_FusedFunctionType == NULL) { +#endif + if (unlikely(__pyx_FusedFunctionType == NULL)) { return -1; } return 0; @@ -1297,9 +1514,9 @@ static PyObject* __Pyx_Method_ClassMethod(PyObject *method) { #if PY_MAJOR_VERSION == 2 // PyMethodDescr_Type is not exposed in the CPython C-API in Py2. static PyTypeObject *methoddescr_type = NULL; - if (methoddescr_type == NULL) { + if (unlikely(methoddescr_type == NULL)) { PyObject *meth = PyObject_GetAttrString((PyObject*)&PyList_Type, "append"); - if (!meth) return NULL; + if (unlikely(!meth)) return NULL; methoddescr_type = Py_TYPE(meth); Py_DECREF(meth); } diff --git a/Cython/Utility/Embed.c b/Cython/Utility/Embed.c index 8f7e8f46e..ec48bfa04 100644 --- a/Cython/Utility/Embed.c +++ b/Cython/Utility/Embed.c @@ -6,7 +6,7 @@ #if PY_MAJOR_VERSION < 3 int %(main_method)s(int argc, char** argv) { -#elif defined(WIN32) || defined(MS_WINDOWS) +#elif defined(_WIN32) || defined(WIN32) || defined(MS_WINDOWS) int %(wmain_method)s(int argc, wchar_t **argv) { #else static int __Pyx_main(int argc, wchar_t **argv) { @@ -24,32 +24,23 @@ static int __Pyx_main(int argc, wchar_t **argv) { #endif if (argc && argv) Py_SetProgramName(argv[0]); + + #if PY_MAJOR_VERSION < 3 + if (PyImport_AppendInittab("%(module_name)s", init%(module_name)s) < 0) return 1; + #else + if (PyImport_AppendInittab("%(module_name)s", PyInit_%(module_name)s) < 0) return 1; + #endif + Py_Initialize(); if (argc && argv) PySys_SetArgv(argc, argv); + { /* init module '%(module_name)s' as '__main__' */ PyObject* m = NULL; %(module_is_main)s = 1; - #if PY_MAJOR_VERSION < 3 - init%(module_name)s(); - #elif CYTHON_PEP489_MULTI_PHASE_INIT - m = PyInit_%(module_name)s(); - if (!PyModule_Check(m)) { - PyModuleDef *mdef = (PyModuleDef *) m; - PyObject *modname = PyUnicode_FromString("__main__"); - m = NULL; - if (modname) { - // FIXME: not currently calling PyModule_FromDefAndSpec() here because we do not have a module spec! - // FIXME: not currently setting __file__, __path__, __spec__, ... - m = PyModule_NewObject(modname); - Py_DECREF(modname); - if (m) PyModule_ExecDef(m, mdef); - } - } - #else - m = PyInit_%(module_name)s(); - #endif - if (PyErr_Occurred()) { + m = PyImport_ImportModule("%(module_name)s"); + + if (!m && PyErr_Occurred()) { PyErr_Print(); /* This exits with the right code if SystemExit. */ #if PY_MAJOR_VERSION < 3 if (Py_FlushLine()) PyErr_Clear(); @@ -68,9 +59,11 @@ static int __Pyx_main(int argc, wchar_t **argv) { } -#if PY_MAJOR_VERSION >= 3 && !defined(WIN32) && !defined(MS_WINDOWS) +#if PY_MAJOR_VERSION >= 3 && !defined(_WIN32) && !defined(WIN32) && !defined(MS_WINDOWS) #include <locale.h> +#if PY_VERSION_HEX < 0x03050000 + static wchar_t* __Pyx_char2wchar(char* arg) { @@ -175,6 +168,8 @@ oom: return NULL; } +#endif + int %(main_method)s(int argc, char **argv) { @@ -197,7 +192,12 @@ int res = 0; setlocale(LC_ALL, ""); for (i = 0; i < argc; i++) { - argv_copy2[i] = argv_copy[i] = __Pyx_char2wchar(argv[i]); + argv_copy2[i] = argv_copy[i] = +#if PY_VERSION_HEX < 0x03050000 + __Pyx_char2wchar(argv[i]); +#else + Py_DecodeLocale(argv[i], NULL); +#endif if (!argv_copy[i]) res = 1; /* failure, but continue to simplify cleanup */ } setlocale(LC_ALL, oldloc); diff --git a/Cython/Utility/Exceptions.c b/Cython/Utility/Exceptions.c index ebbf65e07..1509ed828 100644 --- a/Cython/Utility/Exceptions.c +++ b/Cython/Utility/Exceptions.c @@ -284,7 +284,7 @@ static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, PyObject PyErr_SetObject(type, value); if (tb) { -#if CYTHON_COMPILING_IN_PYPY +#if CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_LIMITED_API PyObject *tmp_type, *tmp_value, *tmp_tb; PyErr_Fetch(&tmp_type, &tmp_value, &tmp_tb); Py_INCREF(tb); @@ -641,7 +641,7 @@ static int __Pyx_CLineForTraceback(PyThreadState *tstate, int c_line);/*proto*/ #endif /////////////// CLineInTraceback /////////////// -//@requires: ObjectHandling.c::PyObjectGetAttrStr +//@requires: ObjectHandling.c::PyObjectGetAttrStrNoError //@requires: ObjectHandling.c::PyDictVersioning //@requires: PyErrFetchRestore //@substitute: naming @@ -670,7 +670,7 @@ static int __Pyx_CLineForTraceback(CYTHON_NCP_UNUSED PyThreadState *tstate, int } else #endif { - PyObject *use_cline_obj = __Pyx_PyObject_GetAttrStr(${cython_runtime_cname}, PYIDENT("cline_in_traceback")); + PyObject *use_cline_obj = __Pyx_PyObject_GetAttrStrNoError(${cython_runtime_cname}, PYIDENT("cline_in_traceback")); if (use_cline_obj) { use_cline = PyObject_Not(use_cline_obj) ? Py_False : Py_True; Py_DECREF(use_cline_obj); @@ -705,6 +705,17 @@ static void __Pyx_AddTraceback(const char *funcname, int c_line, #include "frameobject.h" #include "traceback.h" +#if CYTHON_COMPILING_IN_LIMITED_API +static void __Pyx_AddTraceback(const char *funcname, int c_line, + int py_line, const char *filename) { + if (c_line) { + // Avoid "unused" warning as long as we don't use this. + (void) $cfilenm_cname; + c_line = __Pyx_CLineForTraceback(__Pyx_PyThreadState_Current, c_line); + } + _PyTraceback_Add(funcname, filename, c_line ? -c_line : py_line); +} +#else static PyCodeObject* __Pyx_CreateCodeObjectForTraceback( const char *funcname, int c_line, int py_line, const char *filename) { @@ -735,6 +746,7 @@ static PyCodeObject* __Pyx_CreateCodeObjectForTraceback( if (!py_funcname) goto bad; py_code = __Pyx_PyCode_New( 0, /*int argcount,*/ + 0, /*int posonlyargcount,*/ 0, /*int kwonlyargcount,*/ 0, /*int nlocals,*/ 0, /*int stacksize,*/ @@ -790,3 +802,4 @@ bad: Py_XDECREF(py_code); Py_XDECREF(py_frame); } +#endif diff --git a/Cython/Utility/ExtensionTypes.c b/Cython/Utility/ExtensionTypes.c index 1b39c9e42..d8b8353cb 100644 --- a/Cython/Utility/ExtensionTypes.c +++ b/Cython/Utility/ExtensionTypes.c @@ -36,19 +36,26 @@ static int __Pyx_PyType_Ready(PyTypeObject *t) { } #endif b = (PyTypeObject*)b0; - if (!PyType_HasFeature(b, Py_TPFLAGS_HEAPTYPE)) + if (!__Pyx_PyType_HasFeature(b, Py_TPFLAGS_HEAPTYPE)) { - PyErr_Format(PyExc_TypeError, "base class '%.200s' is not a heap type", - b->tp_name); + __Pyx_TypeName b_name = __Pyx_PyType_GetName(b); + PyErr_Format(PyExc_TypeError, + "base class '" __Pyx_FMT_TYPENAME "' is not a heap type", b_name); + __Pyx_DECREF_TypeName(b_name); return -1; } if (t->tp_dictoffset == 0 && b->tp_dictoffset) { + __Pyx_TypeName t_name = __Pyx_PyType_GetName(t); + __Pyx_TypeName b_name = __Pyx_PyType_GetName(b); PyErr_Format(PyExc_TypeError, - "extension type '%.200s' has no __dict__ slot, but base type '%.200s' has: " + "extension type '" __Pyx_FMT_TYPENAME "' has no __dict__ slot, " + "but base type '" __Pyx_FMT_TYPENAME "' has: " "either add 'cdef dict __dict__' to the extension type " "or add '__slots__ = [...]' to the base type", - t->tp_name, b->tp_name); + t_name, b_name); + __Pyx_DECREF_TypeName(t_name); + __Pyx_DECREF_TypeName(b_name); return -1; } } @@ -119,6 +126,49 @@ static int __Pyx_PyType_Ready(PyTypeObject *t) { return r; } +/////////////// PyTrashcan.proto /////////////// + +// These macros are taken from https://github.com/python/cpython/pull/11841 +// Unlike the Py_TRASHCAN_SAFE_BEGIN/Py_TRASHCAN_SAFE_END macros, they +// allow dealing correctly with subclasses. + +// This requires CPython version >= 2.7.4 +// (or >= 3.2.4 but we don't support such old Python 3 versions anyway) +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x02070400 +#define __Pyx_TRASHCAN_BEGIN_CONDITION(op, cond) \ + do { \ + PyThreadState *_tstate = NULL; \ + // If "cond" is false, then _tstate remains NULL and the deallocator + // is run normally without involving the trashcan + if (cond) { \ + _tstate = PyThreadState_GET(); \ + if (_tstate->trash_delete_nesting >= PyTrash_UNWIND_LEVEL) { \ + // Store the object (to be deallocated later) and jump past + // Py_TRASHCAN_END, skipping the body of the deallocator + _PyTrash_thread_deposit_object((PyObject*)(op)); \ + break; \ + } \ + ++_tstate->trash_delete_nesting; \ + } + // The body of the deallocator is here. +#define __Pyx_TRASHCAN_END \ + if (_tstate) { \ + --_tstate->trash_delete_nesting; \ + if (_tstate->trash_delete_later && _tstate->trash_delete_nesting <= 0) \ + _PyTrash_thread_destroy_chain(); \ + } \ + } while (0); + +#define __Pyx_TRASHCAN_BEGIN(op, dealloc) __Pyx_TRASHCAN_BEGIN_CONDITION(op, \ + Py_TYPE(op)->tp_dealloc == (destructor)(dealloc)) + +#else +// The trashcan is a no-op on other Python implementations +// or old CPython versions +#define __Pyx_TRASHCAN_BEGIN(op, dealloc) +#define __Pyx_TRASHCAN_END +#endif + /////////////// CallNextTpDealloc.proto /////////////// static void __Pyx_call_next_tp_dealloc(PyObject* obj, destructor current_tp_dealloc); @@ -174,18 +224,21 @@ static void __Pyx_call_next_tp_clear(PyObject* obj, inquiry current_tp_clear) { /////////////// SetupReduce.proto /////////////// +#if !CYTHON_COMPILING_IN_LIMITED_API static int __Pyx_setup_reduce(PyObject* type_obj); +#endif /////////////// SetupReduce /////////////// //@requires: ObjectHandling.c::PyObjectGetAttrStrNoError //@requires: ObjectHandling.c::PyObjectGetAttrStr //@substitute: naming +#if !CYTHON_COMPILING_IN_LIMITED_API static int __Pyx_setup_reduce_is_named(PyObject* meth, PyObject* name) { int ret; PyObject *name_attr; - name_attr = __Pyx_PyObject_GetAttrStr(meth, PYIDENT("__name__")); + name_attr = __Pyx_PyObject_GetAttrStrNoError(meth, PYIDENT("__name__")); if (likely(name_attr)) { ret = PyObject_RichCompareBool(name_attr, name, Py_EQ); } else { @@ -244,7 +297,7 @@ static int __Pyx_setup_reduce(PyObject* type_obj) { goto __PYX_BAD; } - setstate = __Pyx_PyObject_GetAttrStr(type_obj, PYIDENT("__setstate__")); + setstate = __Pyx_PyObject_GetAttrStrNoError(type_obj, PYIDENT("__setstate__")); if (!setstate) PyErr_Clear(); if (!setstate || __Pyx_setup_reduce_is_named(setstate, PYIDENT("__setstate_cython__"))) { setstate_cython = __Pyx_PyObject_GetAttrStrNoError(type_obj, PYIDENT("__setstate_cython__")); @@ -263,8 +316,13 @@ static int __Pyx_setup_reduce(PyObject* type_obj) { goto __PYX_GOOD; __PYX_BAD: - if (!PyErr_Occurred()) - PyErr_Format(PyExc_RuntimeError, "Unable to initialize pickling for %s", ((PyTypeObject*)type_obj)->tp_name); + if (!PyErr_Occurred()) { + __Pyx_TypeName type_obj_name = + __Pyx_PyType_GetName((PyTypeObject*)type_obj); + PyErr_Format(PyExc_RuntimeError, + "Unable to initialize pickling for " __Pyx_FMT_TYPENAME, type_obj_name); + __Pyx_DECREF_TypeName(type_obj_name); + } ret = -1; __PYX_GOOD: #if !CYTHON_USE_PYTYPE_LOOKUP @@ -278,3 +336,58 @@ __PYX_GOOD: Py_XDECREF(setstate_cython); return ret; } +#endif + + +/////////////// BinopSlot /////////////// + +static CYTHON_INLINE PyObject *{{func_name}}_maybe_call_slot(PyTypeObject* type, PyObject *left, PyObject *right {{extra_arg_decl}}) { + {{slot_type}} slot; +#if CYTHON_USE_TYPE_SLOTS || PY_MAJOR_VERSION < 3 || CYTHON_COMPILING_IN_PYPY + slot = type->tp_as_number ? type->tp_as_number->{{slot_name}} : NULL; +#else + slot = ({{slot_type}}) PyType_GetSlot(type, Py_{{slot_name}}); +#endif + return slot ? slot(left, right {{extra_arg}}) : __Pyx_NewRef(Py_NotImplemented); +} + +static PyObject *{{func_name}}(PyObject *left, PyObject *right {{extra_arg_decl}}) { + int maybe_self_is_left, maybe_self_is_right = 0; + maybe_self_is_left = Py_TYPE(left) == Py_TYPE(right) +#if CYTHON_USE_TYPE_SLOTS + || (Py_TYPE(left)->tp_as_number && Py_TYPE(left)->tp_as_number->{{slot_name}} == &{{func_name}}) +#endif + || __Pyx_TypeCheck(left, {{type_cname}}); + // Optimize for the common case where the left operation is defined (and successful). + if (!({{overloads_left}})) { + maybe_self_is_right = Py_TYPE(left) == Py_TYPE(right) +#if CYTHON_USE_TYPE_SLOTS + || (Py_TYPE(right)->tp_as_number && Py_TYPE(right)->tp_as_number->{{slot_name}} == &{{func_name}}) +#endif + || __Pyx_TypeCheck(right, {{type_cname}}); + } + if (maybe_self_is_left) { + PyObject *res; + if (maybe_self_is_right && !({{overloads_left}})) { + res = {{call_right}}; + if (res != Py_NotImplemented) return res; + Py_DECREF(res); + // Don't bother calling it again. + maybe_self_is_right = 0; + } + res = {{call_left}}; + if (res != Py_NotImplemented) return res; + Py_DECREF(res); + } + if (({{overloads_left}})) { + maybe_self_is_right = Py_TYPE(left) == Py_TYPE(right) +#if CYTHON_USE_TYPE_SLOTS + || (Py_TYPE(right)->tp_as_number && Py_TYPE(right)->tp_as_number->{{slot_name}} == &{{func_name}}) +#endif + || PyType_IsSubtype(Py_TYPE(right), {{type_cname}}); + } + if (maybe_self_is_right) { + return {{call_right}}; + } + return __Pyx_NewRef(Py_NotImplemented); +} diff --git a/Cython/Utility/FunctionArguments.c b/Cython/Utility/FunctionArguments.c index 8333d9366..1882f826f 100644 --- a/Cython/Utility/FunctionArguments.c +++ b/Cython/Utility/FunctionArguments.c @@ -2,7 +2,7 @@ #define __Pyx_ArgTypeTest(obj, type, none_allowed, name, exact) \ - ((likely((Py_TYPE(obj) == type) | (none_allowed && (obj == Py_None)))) ? 1 : \ + ((likely(__Pyx_IS_TYPE(obj, type) | (none_allowed && (obj == Py_None)))) ? 1 : \ __Pyx__ArgTypeTest(obj, type, name, exact)) static int __Pyx__ArgTypeTest(PyObject *obj, PyTypeObject *type, const char *name, int exact); /*proto*/ @@ -11,6 +11,8 @@ static int __Pyx__ArgTypeTest(PyObject *obj, PyTypeObject *type, const char *nam static int __Pyx__ArgTypeTest(PyObject *obj, PyTypeObject *type, const char *name, int exact) { + __Pyx_TypeName type_name; + __Pyx_TypeName obj_type_name; if (unlikely(!type)) { PyErr_SetString(PyExc_SystemError, "Missing type object"); return 0; @@ -23,9 +25,13 @@ static int __Pyx__ArgTypeTest(PyObject *obj, PyTypeObject *type, const char *nam else { if (likely(__Pyx_TypeCheck(obj, type))) return 1; } + type_name = __Pyx_PyType_GetName(type); + obj_type_name = __Pyx_PyType_GetName(Py_TYPE(obj)); PyErr_Format(PyExc_TypeError, - "Argument '%.200s' has incorrect type (expected %.200s, got %.200s)", - name, type->tp_name, Py_TYPE(obj)->tp_name); + "Argument '%.200s' has incorrect type (expected " __Pyx_FMT_TYPENAME + ", got " __Pyx_FMT_TYPENAME ")", name, type_name, obj_type_name); + __Pyx_DECREF_TypeName(type_name); + __Pyx_DECREF_TypeName(obj_type_name); return 0; } @@ -111,22 +117,28 @@ static void __Pyx_RaiseMappingExpectedError(PyObject* arg); /*proto*/ //////////////////// RaiseMappingExpected //////////////////// static void __Pyx_RaiseMappingExpectedError(PyObject* arg) { - PyErr_Format(PyExc_TypeError, "'%.200s' object is not a mapping", Py_TYPE(arg)->tp_name); + __Pyx_TypeName arg_type_name = __Pyx_PyType_GetName(Py_TYPE(arg)); + PyErr_Format(PyExc_TypeError, + "'" __Pyx_FMT_TYPENAME "' object is not a mapping", arg_type_name); + __Pyx_DECREF_TypeName(arg_type_name); } //////////////////// KeywordStringCheck.proto //////////////////// -static int __Pyx_CheckKeywordStrings(PyObject *kwdict, const char* function_name, int kw_allowed); /*proto*/ +static int __Pyx_CheckKeywordStrings(PyObject *kw, const char* function_name, int kw_allowed); /*proto*/ //////////////////// KeywordStringCheck //////////////////// -// __Pyx_CheckKeywordStrings raises an error if non-string keywords -// were passed to a function, or if any keywords were passed to a -// function that does not accept them. +// __Pyx_CheckKeywordStrings raises an error if non-string keywords +// were passed to a function, or if any keywords were passed to a +// function that does not accept them. +// +// The "kw" argument is either a dict (for METH_VARARGS) or a tuple +// (for METH_FASTCALL). static int __Pyx_CheckKeywordStrings( - PyObject *kwdict, + PyObject *kw, const char* function_name, int kw_allowed) { @@ -134,18 +146,37 @@ static int __Pyx_CheckKeywordStrings( Py_ssize_t pos = 0; #if CYTHON_COMPILING_IN_PYPY /* PyPy appears to check keywords at call time, not at unpacking time => not much to do here */ - if (!kw_allowed && PyDict_Next(kwdict, &pos, &key, 0)) + if (!kw_allowed && PyDict_Next(kw, &pos, &key, 0)) goto invalid_keyword; return 1; #else - while (PyDict_Next(kwdict, &pos, &key, 0)) { + if (CYTHON_METH_FASTCALL && likely(PyTuple_Check(kw))) { + if (unlikely(PyTuple_GET_SIZE(kw) == 0)) + return 1; + if (!kw_allowed) { + key = PyTuple_GET_ITEM(kw, 0); + goto invalid_keyword; + } +#if PY_VERSION_HEX < 0x03090000 + // On CPython >= 3.9, the FASTCALL protocol guarantees that keyword + // names are strings (see https://bugs.python.org/issue37540) + for (pos = 0; pos < PyTuple_GET_SIZE(kw); pos++) { + key = PyTuple_GET_ITEM(kw, pos); + if (unlikely(!PyUnicode_Check(key))) + goto invalid_keyword_type; + } +#endif + return 1; + } + + while (PyDict_Next(kw, &pos, &key, 0)) { #if PY_MAJOR_VERSION < 3 if (unlikely(!PyString_Check(key))) #endif if (unlikely(!PyUnicode_Check(key))) goto invalid_keyword_type; } - if ((!kw_allowed) && unlikely(key)) + if (!kw_allowed && unlikely(key)) goto invalid_keyword; return 1; invalid_keyword_type: @@ -154,11 +185,12 @@ invalid_keyword_type: return 0; #endif invalid_keyword: - PyErr_Format(PyExc_TypeError, #if PY_MAJOR_VERSION < 3 + PyErr_Format(PyExc_TypeError, "%.200s() got an unexpected keyword argument '%.200s'", function_name, PyString_AsString(key)); #else + PyErr_Format(PyExc_TypeError, "%s() got an unexpected keyword argument '%U'", function_name, key); #endif @@ -168,17 +200,22 @@ invalid_keyword: //////////////////// ParseKeywords.proto //////////////////// -static int __Pyx_ParseOptionalKeywords(PyObject *kwds, PyObject **argnames[], \ - PyObject *kwds2, PyObject *values[], Py_ssize_t num_pos_args, \ +static int __Pyx_ParseOptionalKeywords(PyObject *kwds, PyObject *const *kwvalues, + PyObject **argnames[], + PyObject *kwds2, PyObject *values[], Py_ssize_t num_pos_args, const char* function_name); /*proto*/ //////////////////// ParseKeywords //////////////////// //@requires: RaiseDoubleKeywords // __Pyx_ParseOptionalKeywords copies the optional/unknown keyword -// arguments from the kwds dict into kwds2. If kwds2 is NULL, unknown +// arguments from kwds into the dict kwds2. If kwds2 is NULL, unknown // keywords will raise an invalid keyword error. // +// When not using METH_FASTCALL, kwds is a dict and kwvalues is NULL. +// Otherwise, kwds is a tuple with keyword names and kwvalues is a C +// array with the corresponding values. +// // Three kinds of errors are checked: 1) non-string keywords, 2) // unexpected keywords and 3) overlap with positional arguments. // @@ -190,6 +227,7 @@ static int __Pyx_ParseOptionalKeywords(PyObject *kwds, PyObject **argnames[], \ static int __Pyx_ParseOptionalKeywords( PyObject *kwds, + PyObject *const *kwvalues, PyObject **argnames[], PyObject *kwds2, PyObject *values[], @@ -200,8 +238,20 @@ static int __Pyx_ParseOptionalKeywords( Py_ssize_t pos = 0; PyObject*** name; PyObject*** first_kw_arg = argnames + num_pos_args; + int kwds_is_tuple = CYTHON_METH_FASTCALL && likely(PyTuple_Check(kwds)); + + while (1) { + if (kwds_is_tuple) { + if (pos >= PyTuple_GET_SIZE(kwds)) break; + key = PyTuple_GET_ITEM(kwds, pos); + value = kwvalues[pos]; + pos++; + } + else + { + if (!PyDict_Next(kwds, &pos, &key, &value)) break; + } - while (PyDict_Next(kwds, &pos, &key, &value)) { name = first_kw_arg; while (*name && (**name != key)) name++; if (*name) { @@ -237,12 +287,13 @@ static int __Pyx_ParseOptionalKeywords( #endif if (likely(PyUnicode_Check(key))) { while (*name) { - int cmp = (**name == key) ? 0 : + int cmp = ( #if !CYTHON_COMPILING_IN_PYPY && PY_MAJOR_VERSION >= 3 (__Pyx_PyUnicode_GET_LENGTH(**name) != __Pyx_PyUnicode_GET_LENGTH(key)) ? 1 : #endif // In Py2, we may need to convert the argument name from str to unicode for comparison. - PyUnicode_Compare(**name, key); + PyUnicode_Compare(**name, key) + ); if (cmp < 0 && unlikely(PyErr_Occurred())) goto bad; if (cmp == 0) { values[name-argnames] = value; @@ -284,11 +335,12 @@ invalid_keyword_type: "%.200s() keywords must be strings", function_name); goto bad; invalid_keyword: - PyErr_Format(PyExc_TypeError, #if PY_MAJOR_VERSION < 3 + PyErr_Format(PyExc_TypeError, "%.200s() got an unexpected keyword argument '%.200s'", function_name, PyString_AsString(key)); #else + PyErr_Format(PyExc_TypeError, "%s() got an unexpected keyword argument '%U'", function_name, key); #endif @@ -314,7 +366,7 @@ static int __Pyx_MergeKeywords(PyObject *kwdict, PyObject *source_mapping) { if (unlikely(!iter)) { // slow fallback: try converting to dict, then iterate PyObject *args; - if (!PyErr_ExceptionMatches(PyExc_AttributeError)) goto bad; + if (unlikely(!PyErr_ExceptionMatches(PyExc_AttributeError))) goto bad; PyErr_Clear(); args = PyTuple_Pack(1, source_mapping); if (likely(args)) { @@ -350,3 +402,74 @@ bad: Py_XDECREF(iter); return -1; } + + +/////////////// fastcall.proto /////////////// + +// We define various functions and macros with two variants: +//..._FASTCALL and ..._VARARGS + +// The first is used when METH_FASTCALL is enabled and the second is used +// otherwise. If the Python implementation does not support METH_FASTCALL +// (because it's an old version of CPython or it's not CPython at all), +// then the ..._FASTCALL macros simply alias ..._VARARGS + +#define __Pyx_Arg_VARARGS(args, i) PyTuple_GET_ITEM(args, i) +#define __Pyx_NumKwargs_VARARGS(kwds) PyDict_Size(kwds) +#define __Pyx_KwValues_VARARGS(args, nargs) NULL +#define __Pyx_GetKwValue_VARARGS(kw, kwvalues, s) __Pyx_PyDict_GetItemStrWithError(kw, s) +#define __Pyx_KwargsAsDict_VARARGS(kw, kwvalues) PyDict_Copy(kw) +#if CYTHON_METH_FASTCALL + #define __Pyx_Arg_FASTCALL(args, i) args[i] + #define __Pyx_NumKwargs_FASTCALL(kwds) PyTuple_GET_SIZE(kwds) + #define __Pyx_KwValues_FASTCALL(args, nargs) (&args[nargs]) + static CYTHON_INLINE PyObject * __Pyx_GetKwValue_FASTCALL(PyObject *kwnames, PyObject *const *kwvalues, PyObject *s); + #define __Pyx_KwargsAsDict_FASTCALL(kw, kwvalues) _PyStack_AsDict(kwvalues, kw) +#else + #define __Pyx_Arg_FASTCALL __Pyx_Arg_VARARGS + #define __Pyx_NumKwargs_FASTCALL __Pyx_NumKwargs_VARARGS + #define __Pyx_KwValues_FASTCALL __Pyx_KwValues_VARARGS + #define __Pyx_GetKwValue_FASTCALL __Pyx_GetKwValue_VARARGS + #define __Pyx_KwargsAsDict_FASTCALL __Pyx_KwargsAsDict_VARARGS +#endif + +#if CYTHON_COMPILING_IN_CPYTHON +#define __Pyx_ArgsSlice_VARARGS(args, start, stop) __Pyx_PyTuple_FromArray(&__Pyx_Arg_VARARGS(args, start), stop - start) +#define __Pyx_ArgsSlice_FASTCALL(args, start, stop) __Pyx_PyTuple_FromArray(&__Pyx_Arg_FASTCALL(args, start), stop - start) +#else +/* Not CPython, so certainly no METH_FASTCALL support */ +#define __Pyx_ArgsSlice_VARARGS(args, start, stop) PyTuple_GetSlice(args, start, stop) +#define __Pyx_ArgsSlice_FASTCALL(args, start, stop) PyTuple_GetSlice(args, start, stop) +#endif + + +/////////////// fastcall /////////////// +//@requires: ObjectHandling.c::TupleAndListFromArray +//@requires: StringTools.c::UnicodeEquals + +#if CYTHON_METH_FASTCALL +// kwnames: tuple with names of keyword arguments +// kwvalues: C array with values of keyword arguments +// s: str with the keyword name to look for +static CYTHON_INLINE PyObject * __Pyx_GetKwValue_FASTCALL(PyObject *kwnames, PyObject *const *kwvalues, PyObject *s) +{ + // Search the kwnames array for s and return the corresponding value. + // We do two loops: a first one to compare pointers (which will find a + // match if the name in kwnames is interned, given that s is interned + // by Cython). A second loop compares the actual strings. + Py_ssize_t i, n = PyTuple_GET_SIZE(kwnames); + for (i = 0; i < n; i++) + { + if (s == PyTuple_GET_ITEM(kwnames, i)) return kwvalues[i]; + } + for (i = 0; i < n; i++) + { + int eq = __Pyx_PyUnicode_Equals(s, PyTuple_GET_ITEM(kwnames, i), Py_EQ); + if (unlikely(eq != 0)) { + if (unlikely(eq < 0)) return NULL; // error + return kwvalues[i]; + } + } + return NULL; // not found (no exception set) +} +#endif diff --git a/Cython/Utility/ImportExport.c b/Cython/Utility/ImportExport.c index e3c0cafaf..655781ee1 100644 --- a/Cython/Utility/ImportExport.c +++ b/Cython/Utility/ImportExport.c @@ -9,6 +9,146 @@ #endif +/////////////// ImportDottedModule.proto /////////////// + +static PyObject *__Pyx_ImportDottedModule(PyObject *name, PyObject *parts_tuple); /*proto*/ + +/////////////// ImportDottedModule /////////////// +//@requires: Import + +#if PY_MAJOR_VERSION >= 3 +static PyObject *__Pyx__ImportDottedModule_Error(PyObject *name, PyObject *parts_tuple, Py_ssize_t count) { + PyObject *partial_name = NULL, *slice = NULL, *sep = NULL; + if (unlikely(PyErr_Occurred())) { + PyErr_Clear(); + } + if (likely(PyTuple_GET_SIZE(parts_tuple) == count)) { + partial_name = name; + } else { + slice = PySequence_GetSlice(parts_tuple, 0, count); + if (unlikely(!slice)) + goto bad; + sep = PyUnicode_FromStringAndSize(".", 1); + if (unlikely(!sep)) + goto bad; + partial_name = PyUnicode_Join(sep, slice); + } + + PyErr_Format( +#if PY_MAJOR_VERSION < 3 + PyExc_ImportError, + "No module named '%s'", PyString_AS_STRING(partial_name)); +#else +#if PY_VERSION_HEX >= 0x030600B1 + PyExc_ModuleNotFoundError, +#else + PyExc_ImportError, +#endif + "No module named '%U'", partial_name); +#endif + +bad: + Py_XDECREF(sep); + Py_XDECREF(slice); + Py_XDECREF(partial_name); + return NULL; +} +#endif + +#if PY_MAJOR_VERSION >= 3 +static PyObject *__Pyx__ImportDottedModule_Lookup(PyObject *name) { + PyObject *imported_module; +#if PY_VERSION_HEX < 0x030700A1 || (CYTHON_COMPILING_IN_PYPY && PYPY_VERSION_NUM < 0x07030400) + PyObject *modules = PyImport_GetModuleDict(); + if (unlikely(!modules)) + return NULL; + imported_module = __Pyx_PyDict_GetItemStr(modules, name); + Py_XINCREF(imported_module); +#else + imported_module = PyImport_GetModule(name); +#endif + return imported_module; +} +#endif + +static PyObject *__Pyx__ImportDottedModule(PyObject *name, CYTHON_UNUSED PyObject *parts_tuple) { +#if PY_MAJOR_VERSION < 3 + PyObject *module, *from_list, *star = PYIDENT("*"); + from_list = PyList_New(1); + if (unlikely(!from_list)) + return NULL; + Py_INCREF(star); + PyList_SET_ITEM(from_list, 0, star); + module = __Pyx_Import(name, from_list, 0); + Py_DECREF(from_list); + return module; +#else + Py_ssize_t i, nparts; + PyObject *imported_module; + PyObject *module = __Pyx_Import(name, NULL, 0); + if (!parts_tuple || unlikely(!module)) + return module; + + // Look up module in sys.modules, which is safer than the attribute lookups below. + imported_module = __Pyx__ImportDottedModule_Lookup(name); + if (likely(imported_module)) { + Py_DECREF(module); + return imported_module; + } + PyErr_Clear(); + + nparts = PyTuple_GET_SIZE(parts_tuple); + for (i=1; i < nparts && module; i++) { + PyObject *part, *submodule; +#if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + part = PyTuple_GET_ITEM(parts_tuple, i); +#else + part = PySequence_ITEM(parts_tuple, i); +#endif + submodule = __Pyx_PyObject_GetAttrStrNoError(module, part); +#if !(CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS) + Py_DECREF(part); +#endif + Py_DECREF(module); + module = submodule; + } + if (likely(module)) + return module; + return __Pyx__ImportDottedModule_Error(name, parts_tuple, i); +#endif +} + +static PyObject *__Pyx_ImportDottedModule(PyObject *name, PyObject *parts_tuple) { +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030400B1 + PyObject *module = __Pyx__ImportDottedModule_Lookup(name); + if (likely(module)) { + // CPython guards against thread-concurrent initialisation in importlib. + // In this case, we let PyImport_ImportModuleLevelObject() handle the locking. + PyObject *spec = __Pyx_PyObject_GetAttrStrNoError(module, PYIDENT("__spec__")); + if (likely(spec)) { + PyObject *unsafe = __Pyx_PyObject_GetAttrStrNoError(spec, PYIDENT("_initializing")); + if (likely(!unsafe || !__Pyx_PyObject_IsTrue(unsafe))) { + Py_DECREF(spec); + spec = NULL; + } + Py_XDECREF(unsafe); + } + if (likely(!spec)) { + // Not in initialisation phase => use modules as is. + PyErr_Clear(); + return module; + } + Py_DECREF(spec); + Py_DECREF(module); + } else if (PyErr_Occurred()) { + PyErr_Clear(); + } +#endif + + return __Pyx__ImportDottedModule(name, parts_tuple); +} + + /////////////// Import.proto /////////////// static PyObject *__Pyx_Import(PyObject *name, PyObject *from_list, int level); /*proto*/ @@ -18,30 +158,23 @@ static PyObject *__Pyx_Import(PyObject *name, PyObject *from_list, int level); / //@substitute: naming static PyObject *__Pyx_Import(PyObject *name, PyObject *from_list, int level) { - PyObject *empty_list = 0; PyObject *module = 0; - PyObject *global_dict = 0; PyObject *empty_dict = 0; - PyObject *list; + PyObject *empty_list = 0; #if PY_MAJOR_VERSION < 3 PyObject *py_import; py_import = __Pyx_PyObject_GetAttrStr($builtins_cname, PYIDENT("__import__")); - if (!py_import) + if (unlikely(!py_import)) goto bad; - #endif - if (from_list) - list = from_list; - else { + if (!from_list) { empty_list = PyList_New(0); - if (!empty_list) + if (unlikely(!empty_list)) goto bad; - list = empty_list; + from_list = empty_list; } - global_dict = PyModule_GetDict($module_cname); - if (!global_dict) - goto bad; + #endif empty_dict = PyDict_New(); - if (!empty_dict) + if (unlikely(!empty_dict)) goto bad; { #if PY_MAJOR_VERSION >= 3 @@ -49,10 +182,15 @@ static PyObject *__Pyx_Import(PyObject *name, PyObject *from_list, int level) { // Avoid C compiler warning if strchr() evaluates to false at compile time. if ((1) && (strchr(__Pyx_MODULE_NAME, '.'))) { /* try package relative import first */ + #if CYTHON_COMPILING_IN_LIMITED_API + module = PyImport_ImportModuleLevelObject( + name, empty_dict, empty_dict, from_list, 1); + #else module = PyImport_ImportModuleLevelObject( - name, global_dict, empty_dict, list, 1); - if (!module) { - if (!PyErr_ExceptionMatches(PyExc_ImportError)) + name, $moddict_cname, empty_dict, from_list, 1); + #endif + if (unlikely(!module)) { + if (unlikely(!PyErr_ExceptionMatches(PyExc_ImportError))) goto bad; PyErr_Clear(); } @@ -63,23 +201,28 @@ static PyObject *__Pyx_Import(PyObject *name, PyObject *from_list, int level) { if (!module) { #if PY_MAJOR_VERSION < 3 PyObject *py_level = PyInt_FromLong(level); - if (!py_level) + if (unlikely(!py_level)) goto bad; module = PyObject_CallFunctionObjArgs(py_import, - name, global_dict, empty_dict, list, py_level, (PyObject *)NULL); + name, $moddict_cname, empty_dict, from_list, py_level, (PyObject *)NULL); Py_DECREF(py_level); #else + #if CYTHON_COMPILING_IN_LIMITED_API + module = PyImport_ImportModuleLevelObject( + name, empty_dict, empty_dict, from_list, level); + #else module = PyImport_ImportModuleLevelObject( - name, global_dict, empty_dict, list, level); + name, $moddict_cname, empty_dict, from_list, level); + #endif #endif } } bad: + Py_XDECREF(empty_dict); + Py_XDECREF(empty_list); #if PY_MAJOR_VERSION < 3 Py_XDECREF(py_import); #endif - Py_XDECREF(empty_list); - Py_XDECREF(empty_dict); return module; } @@ -333,7 +476,7 @@ static PyTypeObject *__Pyx_ImportType(PyObject *module, const char *module_name, PyObject *result = 0; char warning[200]; Py_ssize_t basicsize; -#ifdef Py_LIMITED_API +#if CYTHON_COMPILING_IN_LIMITED_API PyObject *py_basicsize; #endif @@ -346,7 +489,7 @@ static PyTypeObject *__Pyx_ImportType(PyObject *module, const char *module_name, module_name, class_name); goto bad; } -#ifndef Py_LIMITED_API +#if !CYTHON_COMPILING_IN_LIMITED_API basicsize = ((PyTypeObject *)result)->tp_basicsize; #else py_basicsize = PyObject_GetAttrString(result, "__basicsize__"); @@ -414,7 +557,6 @@ static int __Pyx_ImportFunction(PyObject *module, const char *funcname, void (** PyModule_GetName(module), funcname); goto bad; } -#if PY_VERSION_HEX >= 0x02070000 if (!PyCapsule_IsValid(cobj, sig)) { PyErr_Format(PyExc_TypeError, "C function %.200s.%.200s has wrong signature (expected %.500s, got %.500s)", @@ -422,21 +564,6 @@ static int __Pyx_ImportFunction(PyObject *module, const char *funcname, void (** goto bad; } tmp.p = PyCapsule_GetPointer(cobj, sig); -#else - {const char *desc, *s1, *s2; - desc = (const char *)PyCObject_GetDesc(cobj); - if (!desc) - goto bad; - s1 = desc; s2 = sig; - while (*s1 != '\0' && *s1 == *s2) { s1++; s2++; } - if (*s1 != *s2) { - PyErr_Format(PyExc_TypeError, - "C function %.200s.%.200s has wrong signature (expected %.500s, got %.500s)", - PyModule_GetName(module), funcname, sig, desc); - goto bad; - } - tmp.p = PyCObject_AsVoidPtr(cobj);} -#endif *f = tmp.fp; if (!(*f)) goto bad; @@ -474,11 +601,7 @@ static int __Pyx_ExportFunction(const char *name, void (*f)(void), const char *s goto bad; } tmp.fp = f; -#if PY_VERSION_HEX >= 0x02070000 cobj = PyCapsule_New(tmp.p, sig, 0); -#else - cobj = PyCObject_FromVoidPtrAndDesc(tmp.p, (void *)sig, 0); -#endif if (!cobj) goto bad; if (PyDict_SetItemString(d, name, cobj) < 0) @@ -515,7 +638,6 @@ static int __Pyx_ImportVoidPtr(PyObject *module, const char *name, void **p, con PyModule_GetName(module), name); goto bad; } -#if PY_VERSION_HEX >= 0x02070000 if (!PyCapsule_IsValid(cobj, sig)) { PyErr_Format(PyExc_TypeError, "C variable %.200s.%.200s has wrong signature (expected %.500s, got %.500s)", @@ -523,21 +645,6 @@ static int __Pyx_ImportVoidPtr(PyObject *module, const char *name, void **p, con goto bad; } *p = PyCapsule_GetPointer(cobj, sig); -#else - {const char *desc, *s1, *s2; - desc = (const char *)PyCObject_GetDesc(cobj); - if (!desc) - goto bad; - s1 = desc; s2 = sig; - while (*s1 != '\0' && *s1 == *s2) { s1++; s2++; } - if (*s1 != *s2) { - PyErr_Format(PyExc_TypeError, - "C variable %.200s.%.200s has wrong signature (expected %.500s, got %.500s)", - PyModule_GetName(module), name, sig, desc); - goto bad; - } - *p = PyCObject_AsVoidPtr(cobj);} -#endif if (!(*p)) goto bad; Py_DECREF(d); @@ -569,11 +676,7 @@ static int __Pyx_ExportVoidPtr(PyObject *name, void *p, const char *sig) { if (__Pyx_PyObject_SetAttrStr($module_cname, PYIDENT("$api_name"), d) < 0) goto bad; } -#if PY_VERSION_HEX >= 0x02070000 cobj = PyCapsule_New(p, sig, 0); -#else - cobj = PyCObject_FromVoidPtrAndDesc(p, (void *)sig, 0); -#endif if (!cobj) goto bad; if (PyDict_SetItem(d, name, cobj) < 0) @@ -594,15 +697,19 @@ static int __Pyx_SetVtable(PyObject *dict, void *vtable); /*proto*/ /////////////// SetVTable /////////////// -static int __Pyx_SetVtable(PyObject *dict, void *vtable) { -#if PY_VERSION_HEX >= 0x02070000 - PyObject *ob = PyCapsule_New(vtable, 0, 0); +#if CYTHON_COMPILING_IN_LIMITED_API +static int __Pyx_SetVtable(PyObject *type, void *vtable) { #else - PyObject *ob = PyCObject_FromVoidPtr(vtable, 0); +static int __Pyx_SetVtable(PyObject *dict, void *vtable) { #endif + PyObject *ob = PyCapsule_New(vtable, 0, 0); if (!ob) goto bad; +#if CYTHON_COMPILING_IN_LIMITED_API + if (PyObject_SetAttr(type, PYIDENT("__pyx_vtable__"), ob) < 0) +#else if (PyDict_SetItem(dict, PYIDENT("__pyx_vtable__"), ob) < 0) +#endif goto bad; Py_DECREF(ob); return 0; @@ -614,20 +721,20 @@ bad: /////////////// GetVTable.proto /////////////// -static void* __Pyx_GetVtable(PyObject *dict); /*proto*/ +static void* __Pyx_GetVtable(PyTypeObject *type); /*proto*/ /////////////// GetVTable /////////////// -static void* __Pyx_GetVtable(PyObject *dict) { +static void* __Pyx_GetVtable(PyTypeObject *type) { void* ptr; - PyObject *ob = PyObject_GetItem(dict, PYIDENT("__pyx_vtable__")); +#if CYTHON_COMPILING_IN_LIMITED_API + PyObject *ob = PyObject_GetAttr((PyObject *)type, PYIDENT("__pyx_vtable__")); +#else + PyObject *ob = PyObject_GetItem(type->tp_dict, PYIDENT("__pyx_vtable__")); +#endif if (!ob) goto bad; -#if PY_VERSION_HEX >= 0x02070000 ptr = PyCapsule_GetPointer(ob, 0); -#else - ptr = PyCObject_AsVoidPtr(ob); -#endif if (!ptr && !PyErr_Occurred()) PyErr_SetString(PyExc_RuntimeError, "invalid vtable found for imported type"); Py_DECREF(ob); @@ -648,6 +755,8 @@ static int __Pyx_MergeVtables(PyTypeObject *type); /*proto*/ static int __Pyx_MergeVtables(PyTypeObject *type) { int i; void** base_vtables; + __Pyx_TypeName tp_base_name; + __Pyx_TypeName base_name; void* unknown = (void*)-1; PyObject* bases = type->tp_bases; int base_depth = 0; @@ -667,13 +776,13 @@ static int __Pyx_MergeVtables(PyTypeObject *type) { // instance struct is so extended. (It would be good to also do this // check when a multiple-base class is created in pure Python as well.) for (i = 1; i < PyTuple_GET_SIZE(bases); i++) { - void* base_vtable = __Pyx_GetVtable(((PyTypeObject*)PyTuple_GET_ITEM(bases, i))->tp_dict); + void* base_vtable = __Pyx_GetVtable(((PyTypeObject*)PyTuple_GET_ITEM(bases, i))); if (base_vtable != NULL) { int j; PyTypeObject* base = type->tp_base; for (j = 0; j < base_depth; j++) { if (base_vtables[j] == unknown) { - base_vtables[j] = __Pyx_GetVtable(base->tp_dict); + base_vtables[j] = __Pyx_GetVtable(base); base_vtables[j + 1] = unknown; } if (base_vtables[j] == base_vtable) { @@ -690,10 +799,12 @@ static int __Pyx_MergeVtables(PyTypeObject *type) { free(base_vtables); return 0; bad: - PyErr_Format( - PyExc_TypeError, - "multiple bases have vtable conflict: '%s' and '%s'", - type->tp_base->tp_name, ((PyTypeObject*)PyTuple_GET_ITEM(bases, i))->tp_name); + tp_base_name = __Pyx_PyType_GetName(type->tp_base); + base_name = __Pyx_PyType_GetName((PyTypeObject*)PyTuple_GET_ITEM(bases, i)); + PyErr_Format(PyExc_TypeError, + "multiple bases have vtable conflict: '" __Pyx_FMT_TYPENAME "' and '" __Pyx_FMT_TYPENAME "'", tp_base_name, base_name); + __Pyx_DECREF_TypeName(tp_base_name); + __Pyx_DECREF_TypeName(base_name); free(base_vtables); return -1; } diff --git a/Cython/Utility/MemoryView.pyx b/Cython/Utility/MemoryView.pyx index 6ca5fab9b..1496755f1 100644 --- a/Cython/Utility/MemoryView.pyx +++ b/Cython/Utility/MemoryView.pyx @@ -1,5 +1,8 @@ #################### View.MemoryView #################### +# cython: language_level=3str +# cython: binding=False + # This utility provides cython.array and cython.view.memoryview from __future__ import absolute_import @@ -8,8 +11,12 @@ cimport cython # from cpython cimport ... cdef extern from "Python.h": + ctypedef struct PyObject int PyIndex_Check(object) object PyLong_FromVoidPtr(void *) + PyObject *PyExc_IndexError + PyObject *PyExc_ValueError + PyObject *PyExc_MemoryError cdef extern from "pythread.h": ctypedef void *PyThread_type_lock @@ -49,7 +56,7 @@ cdef extern from *: Py_ssize_t suboffsets[{{max_dims}}] void __PYX_INC_MEMVIEW({{memviewslice_name}} *memslice, int have_gil) - void __PYX_XDEC_MEMVIEW({{memviewslice_name}} *memslice, int have_gil) + void __PYX_XCLEAR_MEMVIEW({{memviewslice_name}} *memslice, int have_gil) ctypedef struct __pyx_buffer "Py_buffer": PyObject *obj @@ -94,9 +101,6 @@ cdef extern from "<stdlib.h>": void free(void *) nogil void *memcpy(void *dest, void *src, size_t n) nogil - - - # ### cython.array class # @@ -123,17 +127,16 @@ cdef class array: mode="c", bint allocate_buffer=True): cdef int idx - cdef Py_ssize_t i, dim - cdef PyObject **p + cdef Py_ssize_t dim self.ndim = <int> len(shape) self.itemsize = itemsize if not self.ndim: - raise ValueError("Empty shape tuple for cython.array") + raise ValueError, "Empty shape tuple for cython.array" if itemsize <= 0: - raise ValueError("itemsize <= 0 for cython.array") + raise ValueError, "itemsize <= 0 for cython.array" if not isinstance(format, bytes): format = format.encode('ASCII') @@ -145,65 +148,58 @@ cdef class array: self._strides = self._shape + self.ndim if not self._shape: - raise MemoryError("unable to allocate shape and strides.") + raise MemoryError, "unable to allocate shape and strides." # cdef Py_ssize_t dim, stride for idx, dim in enumerate(shape): if dim <= 0: - raise ValueError("Invalid shape in axis %d: %d." % (idx, dim)) + raise ValueError, f"Invalid shape in axis {idx}: {dim}." self._shape[idx] = dim cdef char order - if mode == 'fortran': - order = b'F' - self.mode = u'fortran' - elif mode == 'c': + if mode == 'c': order = b'C' self.mode = u'c' + elif mode == 'fortran': + order = b'F' + self.mode = u'fortran' else: - raise ValueError("Invalid mode, expected 'c' or 'fortran', got %s" % mode) + raise ValueError, f"Invalid mode, expected 'c' or 'fortran', got {mode}" - self.len = fill_contig_strides_array(self._shape, self._strides, - itemsize, self.ndim, order) + self.len = fill_contig_strides_array(self._shape, self._strides, itemsize, self.ndim, order) self.free_data = allocate_buffer self.dtype_is_object = format == b'O' - if allocate_buffer: - # use malloc() for backwards compatibility - # in case external code wants to change the data pointer - self.data = <char *>malloc(self.len) - if not self.data: - raise MemoryError("unable to allocate array data.") - if self.dtype_is_object: - p = <PyObject **> self.data - for i in range(self.len / itemsize): - p[i] = Py_None - Py_INCREF(Py_None) + if allocate_buffer: + _allocate_buffer(self) @cname('getbuffer') def __getbuffer__(self, Py_buffer *info, int flags): cdef int bufmode = -1 - if self.mode == u"c": - bufmode = PyBUF_C_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS - elif self.mode == u"fortran": - bufmode = PyBUF_F_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS - if not (flags & bufmode): - raise ValueError("Can only create a buffer that is contiguous in memory.") + if flags & (PyBUF_C_CONTIGUOUS | PyBUF_F_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS): + if self.mode == u"c": + bufmode = PyBUF_C_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS + elif self.mode == u"fortran": + bufmode = PyBUF_F_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS + if not (flags & bufmode): + raise ValueError, "Can only create a buffer that is contiguous in memory." info.buf = self.data info.len = self.len - info.ndim = self.ndim - info.shape = self._shape - info.strides = self._strides - info.suboffsets = NULL - info.itemsize = self.itemsize - info.readonly = 0 - if flags & PyBUF_FORMAT: - info.format = self.format + if flags & PyBUF_STRIDES: + info.ndim = self.ndim + info.shape = self._shape + info.strides = self._strides else: - info.format = NULL + info.ndim = 1 + info.shape = &self.len if flags & PyBUF_ND else NULL + info.strides = NULL + info.suboffsets = NULL + info.itemsize = self.itemsize + info.readonly = 0 + info.format = self.format if flags & PyBUF_FORMAT else NULL info.obj = self __pyx_getbuffer = capsule(<void *> &__pyx_array_getbuffer, "getbuffer(obj, view, flags)") @@ -211,10 +207,9 @@ cdef class array: def __dealloc__(array self): if self.callback_free_data != NULL: self.callback_free_data(self.data) - elif self.free_data: + elif self.free_data and self.data is not NULL: if self.dtype_is_object: - refcount_objects_in_slice(self.data, self._shape, - self._strides, self.ndim, False) + refcount_objects_in_slice(self.data, self._shape, self._strides, self.ndim, inc=False) free(self.data) PyObject_Free(self._shape) @@ -240,16 +235,35 @@ cdef class array: self.memview[item] = value +@cname("__pyx_array_allocate_buffer") +cdef int _allocate_buffer(array self) except -1: + # use malloc() for backwards compatibility + # in case external code wants to change the data pointer + cdef Py_ssize_t i + cdef PyObject **p + + self.free_data = True + self.data = <char *>malloc(self.len) + if not self.data: + raise MemoryError, "unable to allocate array data." + + if self.dtype_is_object: + p = <PyObject **> self.data + for i in range(self.len // self.itemsize): + p[i] = Py_None + Py_INCREF(Py_None) + return 0 + + @cname("__pyx_array_new") -cdef array array_cwrapper(tuple shape, Py_ssize_t itemsize, char *format, - char *mode, char *buf): +cdef array array_cwrapper(tuple shape, Py_ssize_t itemsize, char *format, char *c_mode, char *buf): cdef array result + cdef str mode = "fortran" if c_mode[0] == b'f' else "c" # this often comes from a constant C string. - if buf == NULL: - result = array(shape, itemsize, format, mode.decode('ASCII')) + if buf is NULL: + result = array.__new__(array, shape, itemsize, format, mode) else: - result = array(shape, itemsize, format, mode.decode('ASCII'), - allocate_buffer=False) + result = array.__new__(array, shape, itemsize, format, mode, allocate_buffer=False) result.data = buf return result @@ -327,7 +341,7 @@ cdef PyThread_type_lock[THREAD_LOCKS_PREALLOCATED] __pyx_memoryview_thread_locks @cname('__pyx_memoryview') -cdef class memoryview(object): +cdef class memoryview: cdef object obj cdef object _size @@ -415,7 +429,7 @@ cdef class memoryview(object): def __setitem__(memoryview self, object index, object value): if self.view.readonly: - raise TypeError("Cannot assign to read-only memoryview") + raise TypeError, "Cannot assign to read-only memoryview" have_slices, index = _unellipsify(index, self.view.ndim) @@ -441,10 +455,10 @@ cdef class memoryview(object): cdef setitem_slice_assignment(self, dst, src): cdef {{memviewslice_name}} dst_slice cdef {{memviewslice_name}} src_slice + cdef {{memviewslice_name}} msrc = get_slice_from_memview(src, &src_slice)[0] + cdef {{memviewslice_name}} mdst = get_slice_from_memview(dst, &dst_slice)[0] - memoryview_copy_contents(get_slice_from_memview(src, &src_slice)[0], - get_slice_from_memview(dst, &dst_slice)[0], - src.ndim, dst.ndim, self.dtype_is_object) + memoryview_copy_contents(msrc, mdst, src.ndim, dst.ndim, self.dtype_is_object) cdef setitem_slice_assign_scalar(self, memoryview dst, value): cdef int array[128] @@ -492,7 +506,7 @@ cdef class memoryview(object): try: result = struct.unpack(self.view.format, bytesitem) except struct.error: - raise ValueError("Unable to convert item to object") + raise ValueError, "Unable to convert item to object" else: if len(self.view.format) == 1: return result[0] @@ -517,7 +531,7 @@ cdef class memoryview(object): @cname('getbuffer') def __getbuffer__(self, Py_buffer *info, int flags): if flags & PyBUF_WRITABLE and self.view.readonly: - raise ValueError("Cannot create writable memory view from read-only memoryview") + raise ValueError, "Cannot create writable memory view from read-only memoryview" if flags & PyBUF_ND: info.shape = self.view.shape @@ -557,6 +571,9 @@ cdef class memoryview(object): @property def base(self): + return self._get_base() + + cdef _get_base(self): return self.obj @property @@ -567,7 +584,7 @@ cdef class memoryview(object): def strides(self): if self.view.strides == NULL: # Note: we always ask for strides, so if this is not set it's a bug - raise ValueError("Buffer view does not expose strides") + raise ValueError, "Buffer view does not expose strides" return tuple([stride for stride in self.view.strides[:self.view.ndim]]) @@ -668,39 +685,34 @@ cdef tuple _unellipsify(object index, int ndim): Replace all ellipses with full slices and fill incomplete indices with full slices. """ - if not isinstance(index, tuple): - tup = (index,) - else: - tup = index + cdef Py_ssize_t idx + tup = <tuple>index if isinstance(index, tuple) else (index,) - result = [] + result = [slice(None)] * ndim have_slices = False seen_ellipsis = False - for idx, item in enumerate(tup): + idx = 0 + for item in tup: if item is Ellipsis: if not seen_ellipsis: - result.extend([slice(None)] * (ndim - len(tup) + 1)) + idx += ndim - len(tup) seen_ellipsis = True - else: - result.append(slice(None)) have_slices = True else: - if not isinstance(item, slice) and not PyIndex_Check(item): - raise TypeError("Cannot index with type '%s'" % type(item)) - - have_slices = have_slices or isinstance(item, slice) - result.append(item) - - nslices = ndim - len(result) - if nslices: - result.extend([slice(None)] * nslices) - + if isinstance(item, slice): + have_slices = True + elif not PyIndex_Check(item): + raise TypeError, f"Cannot index with type '{type(item)}'" + result[idx] = item + idx += 1 + + nslices = ndim - idx return have_slices or nslices, tuple(result) cdef assert_direct_dimensions(Py_ssize_t *suboffsets, int ndim): for suboffset in suboffsets[:ndim]: if suboffset >= 0: - raise ValueError("Indirect dimensions not supported") + raise ValueError, "Indirect dimensions not supported" # ### Slicing a memoryview @@ -740,15 +752,16 @@ cdef memoryview memview_slice(memoryview memview, object indices): # may not be as expected" cdef {{memviewslice_name}} *p_dst = &dst cdef int *p_suboffset_dim = &suboffset_dim - cdef Py_ssize_t start, stop, step + cdef Py_ssize_t start, stop, step, cindex cdef bint have_start, have_stop, have_step for dim, index in enumerate(indices): if PyIndex_Check(index): + cindex = index slice_memviewslice( p_dst, p_src.shape[dim], p_src.strides[dim], p_src.suboffsets[dim], dim, new_ndim, p_suboffset_dim, - index, 0, 0, # start, stop, step + cindex, 0, 0, # start, stop, step 0, 0, 0, # have_{start,stop,step} False) elif index is None: @@ -829,13 +842,16 @@ cdef int slice_memviewslice( if start < 0: start += shape if not 0 <= start < shape: - _err_dim(IndexError, "Index out of bounds (axis %d)", dim) + _err_dim(PyExc_IndexError, "Index out of bounds (axis %d)", dim) else: # index is a slice - negative_step = have_step != 0 and step < 0 - - if have_step and step == 0: - _err_dim(ValueError, "Step may not be zero (axis %d)", dim) + if have_step: + negative_step = step < 0 + if step == 0: + _err_dim(PyExc_ValueError, "Step may not be zero (axis %d)", dim) + else: + negative_step = False + step = 1 # check our bounds and set defaults if have_start: @@ -867,9 +883,6 @@ cdef int slice_memviewslice( else: stop = shape - if not have_step: - step = 1 - # len = ceil( (stop - start) / step ) with cython.cdivision(True): new_shape = (stop - start) // step @@ -885,7 +898,7 @@ cdef int slice_memviewslice( dst.shape[new_ndim] = new_shape dst.suboffsets[new_ndim] = suboffset - # Add the slicing or idexing offsets to the right suboffset or base data * + # Add the slicing or indexing offsets to the right suboffset or base data * if suboffset_dim[0] < 0: dst.data += start * stride else: @@ -896,7 +909,7 @@ cdef int slice_memviewslice( if new_ndim == 0: dst.data = (<char **> dst.data)[0] + suboffset else: - _err_dim(IndexError, "All dimensions preceding dimension %d " + _err_dim(PyExc_IndexError, "All dimensions preceding dimension %d " "must be indexed and not sliced", dim) else: suboffset_dim[0] = new_ndim @@ -914,7 +927,7 @@ cdef char *pybuffer_index(Py_buffer *view, char *bufp, Py_ssize_t index, cdef char *resultp if view.ndim == 0: - shape = view.len / itemsize + shape = view.len // itemsize stride = itemsize else: shape = view.shape[dim] @@ -925,10 +938,10 @@ cdef char *pybuffer_index(Py_buffer *view, char *bufp, Py_ssize_t index, if index < 0: index += view.shape[dim] if index < 0: - raise IndexError("Out of bounds on buffer access (axis %d)" % dim) + raise IndexError, f"Out of bounds on buffer access (axis {dim})" if index >= shape: - raise IndexError("Out of bounds on buffer access (axis %d)" % dim) + raise IndexError, f"Out of bounds on buffer access (axis {dim})" resultp = bufp + index * stride if suboffset >= 0: @@ -940,7 +953,7 @@ cdef char *pybuffer_index(Py_buffer *view, char *bufp, Py_ssize_t index, ### Transposing a memoryviewslice # @cname('__pyx_memslice_transpose') -cdef int transpose_memslice({{memviewslice_name}} *memslice) nogil except 0: +cdef int transpose_memslice({{memviewslice_name}} *memslice) nogil except -1: cdef int ndim = memslice.memview.view.ndim cdef Py_ssize_t *shape = memslice.shape @@ -948,15 +961,15 @@ cdef int transpose_memslice({{memviewslice_name}} *memslice) nogil except 0: # reverse strides and shape cdef int i, j - for i in range(ndim / 2): + for i in range(ndim // 2): j = ndim - 1 - i strides[i], strides[j] = strides[j], strides[i] shape[i], shape[j] = shape[j], shape[i] if memslice.suboffsets[i] >= 0 or memslice.suboffsets[j] >= 0: - _err(ValueError, "Cannot transpose memoryview with indirect dimensions") + _err(PyExc_ValueError, "Cannot transpose memoryview with indirect dimensions") - return 1 + return 0 # ### Creating new memoryview objects from slices and memoryviews @@ -974,7 +987,7 @@ cdef class _memoryviewslice(memoryview): cdef int (*to_dtype_func)(char *, object) except 0 def __dealloc__(self): - __PYX_XDEC_MEMVIEW(&self.from_slice, 1) + __PYX_XCLEAR_MEMVIEW(&self.from_slice, 1) cdef convert_item_to_object(self, char *itemp): if self.to_object_func != NULL: @@ -988,8 +1001,7 @@ cdef class _memoryviewslice(memoryview): else: memoryview.assign_item_from_object(self, itemp, value) - @property - def base(self): + cdef _get_base(self): return self.from_object __pyx_getbuffer = capsule(<void *> &__pyx_memoryview_getbuffer, "getbuffer(obj, view, flags)") @@ -1010,12 +1022,12 @@ cdef memoryview_fromslice({{memviewslice_name}} memviewslice, # assert 0 < ndim <= memviewslice.memview.view.ndim, ( # ndim, memviewslice.memview.view.ndim) - result = _memoryviewslice(None, 0, dtype_is_object) + result = _memoryviewslice.__new__(_memoryviewslice, None, 0, dtype_is_object) result.from_slice = memviewslice __PYX_INC_MEMVIEW(&memviewslice, 1) - result.from_object = (<memoryview> memviewslice.memview).base + result.from_object = (<memoryview> memviewslice.memview)._get_base() result.typeinfo = memviewslice.memview.typeinfo result.view = memviewslice.memview.view @@ -1107,10 +1119,7 @@ cdef memoryview_copy_from_slice(memoryview memview, {{memviewslice_name}} *memvi ### Copy the contents of a memoryview slices # cdef Py_ssize_t abs_py_ssize_t(Py_ssize_t arg) nogil: - if arg < 0: - return -arg - else: - return arg + return -arg if arg < 0 else arg @cname('__pyx_get_best_slice_order') cdef char get_best_order({{memviewslice_name}} *mslice, int ndim) nogil: @@ -1221,7 +1230,7 @@ cdef void *copy_data_to_temp({{memviewslice_name}} *src, result = malloc(size) if not result: - _err(MemoryError, NULL) + _err_no_memory() # tmpslice[0] = src tmpslice.data = <char *> result @@ -1230,8 +1239,7 @@ cdef void *copy_data_to_temp({{memviewslice_name}} *src, tmpslice.shape[i] = src.shape[i] tmpslice.suboffsets[i] = -1 - fill_contig_strides_array(&tmpslice.shape[0], &tmpslice.strides[0], itemsize, - ndim, order) + fill_contig_strides_array(&tmpslice.shape[0], &tmpslice.strides[0], itemsize, ndim, order) # We need to broadcast strides again for i in range(ndim): @@ -1250,19 +1258,20 @@ cdef void *copy_data_to_temp({{memviewslice_name}} *src, @cname('__pyx_memoryview_err_extents') cdef int _err_extents(int i, Py_ssize_t extent1, Py_ssize_t extent2) except -1 with gil: - raise ValueError("got differing extents in dimension %d (got %d and %d)" % - (i, extent1, extent2)) + raise ValueError, f"got differing extents in dimension {i} (got {extent1} and {extent2})" @cname('__pyx_memoryview_err_dim') -cdef int _err_dim(object error, char *msg, int dim) except -1 with gil: - raise error(msg.decode('ascii') % dim) +cdef int _err_dim(PyObject *error, str msg, int dim) except -1 with gil: + raise <object>error, msg % dim @cname('__pyx_memoryview_err') -cdef int _err(object error, char *msg) except -1 with gil: - if msg != NULL: - raise error(msg.decode('ascii')) - else: - raise error +cdef int _err(PyObject *error, str msg) except -1 with gil: + raise <object>error, msg + +@cname('__pyx_memoryview_err_no_memory') +cdef int _err_no_memory() except -1 with gil: + raise MemoryError + @cname('__pyx_memoryview_copy_contents') cdef int memoryview_copy_contents({{memviewslice_name}} src, @@ -1297,7 +1306,7 @@ cdef int memoryview_copy_contents({{memviewslice_name}} src, _err_extents(i, dst.shape[i], src.shape[i]) if src.suboffsets[i] >= 0: - _err_dim(ValueError, "Dimension %d is not direct", i) + _err_dim(PyExc_ValueError, "Dimension %d is not direct", i) if slices_overlap(&src, &dst, ndim, itemsize): # slices overlap, copy to temp, copy temp to dst @@ -1317,9 +1326,9 @@ cdef int memoryview_copy_contents({{memviewslice_name}} src, if direct_copy: # Contiguous slices with same order - refcount_copying(&dst, dtype_is_object, ndim, False) + refcount_copying(&dst, dtype_is_object, ndim, inc=False) memcpy(dst.data, src.data, slice_get_size(&src, ndim)) - refcount_copying(&dst, dtype_is_object, ndim, True) + refcount_copying(&dst, dtype_is_object, ndim, inc=True) free(tmpdata) return 0 @@ -1329,9 +1338,9 @@ cdef int memoryview_copy_contents({{memviewslice_name}} src, transpose_memslice(&src) transpose_memslice(&dst) - refcount_copying(&dst, dtype_is_object, ndim, False) + refcount_copying(&dst, dtype_is_object, ndim, inc=False) copy_strided_to_strided(&src, &dst, ndim, itemsize) - refcount_copying(&dst, dtype_is_object, ndim, True) + refcount_copying(&dst, dtype_is_object, ndim, inc=True) free(tmpdata) return 0 @@ -1359,13 +1368,10 @@ cdef void broadcast_leading({{memviewslice_name}} *mslice, # @cname('__pyx_memoryview_refcount_copying') -cdef void refcount_copying({{memviewslice_name}} *dst, bint dtype_is_object, - int ndim, bint inc) nogil: - # incref or decref the objects in the destination slice if the dtype is - # object +cdef void refcount_copying({{memviewslice_name}} *dst, bint dtype_is_object, int ndim, bint inc) nogil: + # incref or decref the objects in the destination slice if the dtype is object if dtype_is_object: - refcount_objects_in_slice_with_gil(dst.data, dst.shape, - dst.strides, ndim, inc) + refcount_objects_in_slice_with_gil(dst.data, dst.shape, dst.strides, ndim, inc) @cname('__pyx_memoryview_refcount_objects_in_slice_with_gil') cdef void refcount_objects_in_slice_with_gil(char *data, Py_ssize_t *shape, @@ -1377,6 +1383,7 @@ cdef void refcount_objects_in_slice_with_gil(char *data, Py_ssize_t *shape, cdef void refcount_objects_in_slice(char *data, Py_ssize_t *shape, Py_ssize_t *strides, int ndim, bint inc): cdef Py_ssize_t i + cdef Py_ssize_t stride = strides[0] for i in range(shape[0]): if ndim == 1: @@ -1385,10 +1392,9 @@ cdef void refcount_objects_in_slice(char *data, Py_ssize_t *shape, else: Py_DECREF((<PyObject **> data)[0]) else: - refcount_objects_in_slice(data, shape + 1, strides + 1, - ndim - 1, inc) + refcount_objects_in_slice(data, shape + 1, strides + 1, ndim - 1, inc) - data += strides[0] + data += stride # ### Scalar to slice assignment @@ -1397,10 +1403,9 @@ cdef void refcount_objects_in_slice(char *data, Py_ssize_t *shape, cdef void slice_assign_scalar({{memviewslice_name}} *dst, int ndim, size_t itemsize, void *item, bint dtype_is_object) nogil: - refcount_copying(dst, dtype_is_object, ndim, False) - _slice_assign_scalar(dst.data, dst.shape, dst.strides, ndim, - itemsize, item) - refcount_copying(dst, dtype_is_object, ndim, True) + refcount_copying(dst, dtype_is_object, ndim, inc=False) + _slice_assign_scalar(dst.data, dst.shape, dst.strides, ndim, itemsize, item) + refcount_copying(dst, dtype_is_object, ndim, inc=True) @cname('__pyx_memoryview__slice_assign_scalar') @@ -1417,8 +1422,7 @@ cdef void _slice_assign_scalar(char *data, Py_ssize_t *shape, data += stride else: for i in range(extent): - _slice_assign_scalar(data, shape + 1, strides + 1, - ndim - 1, itemsize, item) + _slice_assign_scalar(data, shape + 1, strides + 1, ndim - 1, itemsize, item) data += stride @@ -1464,6 +1468,7 @@ cdef bytes format_from_typeinfo(__Pyx_TypeInfo *type): cdef __Pyx_StructField *field cdef __pyx_typeinfo_string fmt cdef bytes part, result + cdef Py_ssize_t i if type.typegroup == 'S': assert type.fields != NULL @@ -1485,10 +1490,9 @@ cdef bytes format_from_typeinfo(__Pyx_TypeInfo *type): result = alignment.join(parts) + b'}' else: fmt = __Pyx_TypeInfoToFormat(type) + result = fmt.string if type.arraysize[0]: - extents = [unicode(type.arraysize[i]) for i in range(type.ndim)] - result = (u"(%s)" % u','.join(extents)).encode('ascii') + fmt.string - else: - result = fmt.string + extents = [f"{type.arraysize[i]}" for i in range(type.ndim)] + result = f"({u','.join(extents)})".encode('ascii') + result return result diff --git a/Cython/Utility/MemoryView_C.c b/Cython/Utility/MemoryView_C.c index 0a5d8ee2c..078f634ba 100644 --- a/Cython/Utility/MemoryView_C.c +++ b/Cython/Utility/MemoryView_C.c @@ -113,9 +113,9 @@ static CYTHON_INLINE int __pyx_sub_acquisition_count_locked( #define __pyx_get_slice_count_pointer(memview) (memview->acquisition_count_aligned_p) #define __pyx_get_slice_count(memview) (*__pyx_get_slice_count_pointer(memview)) #define __PYX_INC_MEMVIEW(slice, have_gil) __Pyx_INC_MEMVIEW(slice, have_gil, __LINE__) -#define __PYX_XDEC_MEMVIEW(slice, have_gil) __Pyx_XDEC_MEMVIEW(slice, have_gil, __LINE__) +#define __PYX_XCLEAR_MEMVIEW(slice, have_gil) __Pyx_XCLEAR_MEMVIEW(slice, have_gil, __LINE__) static CYTHON_INLINE void __Pyx_INC_MEMVIEW({{memviewslice_name}} *, int, int); -static CYTHON_INLINE void __Pyx_XDEC_MEMVIEW({{memviewslice_name}} *, int, int); +static CYTHON_INLINE void __Pyx_XCLEAR_MEMVIEW({{memviewslice_name}} *, int, int); /////////////// MemviewSliceIndex.proto /////////////// @@ -487,47 +487,49 @@ __pyx_sub_acquisition_count_locked(__pyx_atomic_int *acquisition_count, static CYTHON_INLINE void __Pyx_INC_MEMVIEW({{memviewslice_name}} *memslice, int have_gil, int lineno) { - int first_time; + __pyx_atomic_int_type old_acquisition_count; struct {{memview_struct_name}} *memview = memslice->memview; - if (unlikely(!memview || (PyObject *) memview == Py_None)) - return; /* allow uninitialized memoryview assignment */ - - if (unlikely(__pyx_get_slice_count(memview) < 0)) - __pyx_fatalerror("Acquisition count is %d (line %d)", - __pyx_get_slice_count(memview), lineno); - - first_time = __pyx_add_acquisition_count(memview) == 0; + if (unlikely(!memview || (PyObject *) memview == Py_None)) { + // Allow uninitialized memoryview assignment and do not ref-count None. + return; + } - if (unlikely(first_time)) { - if (have_gil) { - Py_INCREF((PyObject *) memview); + old_acquisition_count = __pyx_add_acquisition_count(memview); + if (unlikely(old_acquisition_count <= 0)) { + if (likely(old_acquisition_count == 0)) { + // First acquisition => keep the memoryview object alive. + if (have_gil) { + Py_INCREF((PyObject *) memview); + } else { + PyGILState_STATE _gilstate = PyGILState_Ensure(); + Py_INCREF((PyObject *) memview); + PyGILState_Release(_gilstate); + } } else { - PyGILState_STATE _gilstate = PyGILState_Ensure(); - Py_INCREF((PyObject *) memview); - PyGILState_Release(_gilstate); + __pyx_fatalerror("Acquisition count is %d (line %d)", + __pyx_get_slice_count(memview), lineno); } } } -static CYTHON_INLINE void __Pyx_XDEC_MEMVIEW({{memviewslice_name}} *memslice, +static CYTHON_INLINE void __Pyx_XCLEAR_MEMVIEW({{memviewslice_name}} *memslice, int have_gil, int lineno) { - int last_time; + __pyx_atomic_int_type old_acquisition_count; struct {{memview_struct_name}} *memview = memslice->memview; if (unlikely(!memview || (PyObject *) memview == Py_None)) { - // we do not ref-count None + // Do not ref-count None. memslice->memview = NULL; return; } - if (unlikely(__pyx_get_slice_count(memview) <= 0)) - __pyx_fatalerror("Acquisition count is %d (line %d)", - __pyx_get_slice_count(memview), lineno); - - last_time = __pyx_sub_acquisition_count(memview) == 1; + old_acquisition_count = __pyx_sub_acquisition_count(memview); memslice->data = NULL; - - if (unlikely(last_time)) { + if (likely(old_acquisition_count > 1)) { + // Still other slices out there => we do not own the reference. + memslice->memview = NULL; + } else if (likely(old_acquisition_count == 1)) { + // Last slice => discard owned Python reference to memoryview object. if (have_gil) { Py_CLEAR(memslice->memview); } else { @@ -536,7 +538,8 @@ static CYTHON_INLINE void __Pyx_XDEC_MEMVIEW({{memviewslice_name}} *memslice, PyGILState_Release(_gilstate); } } else { - memslice->memview = NULL; + __pyx_fatalerror("Acquisition count is %d (line %d)", + __pyx_get_slice_count(memview), lineno); } } @@ -774,7 +777,7 @@ static CYTHON_INLINE PyObject *{{get_function}}(const char *itemp) { {{if from_py_function}} static CYTHON_INLINE int {{set_function}}(const char *itemp, PyObject *obj) { {{dtype}} value = {{from_py_function}}(obj); - if ({{error_condition}}) + if (unlikely({{error_condition}})) return 0; *({{dtype}} *) itemp = value; return 1; @@ -925,7 +928,7 @@ __pyx_fill_slice_{{dtype_name}}({{type_decl}} *p, Py_ssize_t extent, Py_ssize_t ////////// FillStrided1DScalar ////////// /* Fill a slice with a scalar value. The dimension is direct and strided or contiguous */ -/* This can be used as a callback for the memoryview object to efficienty assign a scalar */ +/* This can be used as a callback for the memoryview object to efficiently assign a scalar */ /* Currently unused */ static void __pyx_fill_slice_{{dtype_name}}({{type_decl}} *p, Py_ssize_t extent, Py_ssize_t stride, diff --git a/Cython/Utility/ModuleSetupCode.c b/Cython/Utility/ModuleSetupCode.c index aa3f368c7..2354c00a8 100644 --- a/Cython/Utility/ModuleSetupCode.c +++ b/Cython/Utility/ModuleSetupCode.c @@ -1,3 +1,16 @@ +/////////////// InitLimitedAPI /////////////// + +#if defined(CYTHON_LIMITED_API) && 0 /* disabled: enabling Py_LIMITED_API needs more work */ + #ifndef Py_LIMITED_API + #if CYTHON_LIMITED_API+0 > 0x03030000 + #define Py_LIMITED_API CYTHON_LIMITED_API + #else + #define Py_LIMITED_API 0x03030000 + #endif + #endif +#endif + + /////////////// CModulePreamble /////////////// #include <stddef.h> /* For offsetof */ @@ -5,7 +18,7 @@ #define offsetof(type, member) ( (size_t) & ((type*)0) -> member ) #endif -#if !defined(WIN32) && !defined(MS_WINDOWS) +#if !defined(_WIN32) && !defined(WIN32) && !defined(MS_WINDOWS) #ifndef __stdcall #define __stdcall #endif @@ -29,9 +42,7 @@ #ifndef HAVE_LONG_LONG // CPython has required PY_LONG_LONG support for years, even if HAVE_LONG_LONG is not defined for us - #if PY_VERSION_HEX >= 0x02070000 - #define HAVE_LONG_LONG - #endif + #define HAVE_LONG_LONG #endif #ifndef PY_LONG_LONG @@ -46,6 +57,7 @@ #define CYTHON_COMPILING_IN_PYPY 1 #define CYTHON_COMPILING_IN_PYSTON 0 #define CYTHON_COMPILING_IN_CPYTHON 0 + #define CYTHON_COMPILING_IN_LIMITED_API 0 #undef CYTHON_USE_TYPE_SLOTS #define CYTHON_USE_TYPE_SLOTS 0 @@ -73,8 +85,15 @@ #define CYTHON_UNPACK_METHODS 0 #undef CYTHON_FAST_THREAD_STATE #define CYTHON_FAST_THREAD_STATE 0 + #undef CYTHON_FAST_GIL + #define CYTHON_FAST_GIL 0 + #undef CYTHON_METH_FASTCALL + #define CYTHON_METH_FASTCALL 0 #undef CYTHON_FAST_PYCALL #define CYTHON_FAST_PYCALL 0 + #ifndef CYTHON_PEP487_INIT_SUBCLASS + #define CYTHON_PEP487_INIT_SUBCLASS (PY_MAJOR_VERSION >= 3) + #endif #undef CYTHON_PEP489_MULTI_PHASE_INIT #define CYTHON_PEP489_MULTI_PHASE_INIT 0 #undef CYTHON_USE_TP_FINALIZE @@ -88,6 +107,7 @@ #define CYTHON_COMPILING_IN_PYPY 0 #define CYTHON_COMPILING_IN_PYSTON 1 #define CYTHON_COMPILING_IN_CPYTHON 0 + #define CYTHON_COMPILING_IN_LIMITED_API 0 #ifndef CYTHON_USE_TYPE_SLOTS #define CYTHON_USE_TYPE_SLOTS 1 @@ -116,8 +136,14 @@ #endif #undef CYTHON_FAST_THREAD_STATE #define CYTHON_FAST_THREAD_STATE 0 + #undef CYTHON_FAST_GIL + #define CYTHON_FAST_GIL 0 + #undef CYTHON_METH_FASTCALL + #define CYTHON_METH_FASTCALL 0 #undef CYTHON_FAST_PYCALL #define CYTHON_FAST_PYCALL 0 + #undef CYTHON_PEP487_INIT_SUBCLASS + #define CYTHON_PEP487_INIT_SUBCLASS 0 #undef CYTHON_PEP489_MULTI_PHASE_INIT #define CYTHON_PEP489_MULTI_PHASE_INIT 0 #undef CYTHON_USE_TP_FINALIZE @@ -127,19 +153,64 @@ #undef CYTHON_USE_EXC_INFO_STACK #define CYTHON_USE_EXC_INFO_STACK 0 +#elif defined(CYTHON_LIMITED_API) + #define CYTHON_COMPILING_IN_PYPY 0 + #define CYTHON_COMPILING_IN_PYSTON 0 + #define CYTHON_COMPILING_IN_CPYTHON 0 + #define CYTHON_COMPILING_IN_LIMITED_API 1 + + #undef CYTHON_USE_TYPE_SLOTS + #define CYTHON_USE_TYPE_SLOTS 0 + #undef CYTHON_USE_PYTYPE_LOOKUP + #define CYTHON_USE_PYTYPE_LOOKUP 0 + #undef CYTHON_USE_PYLIST_INTERNALS + #define CYTHON_USE_ASYNC_SLOTS 0 + #define CYTHON_USE_PYLIST_INTERNALS 0 + #undef CYTHON_USE_UNICODE_INTERNALS + #define CYTHON_USE_UNICODE_INTERNALS 0 + #ifndef CYTHON_USE_UNICODE_WRITER + #define CYTHON_USE_UNICODE_WRITER 1 + #endif + #undef CYTHON_USE_PYLONG_INTERNALS + #define CYTHON_USE_PYLONG_INTERNALS 0 + #ifndef CYTHON_AVOID_BORROWED_REFS + #define CYTHON_AVOID_BORROWED_REFS 0 + #endif + #undef CYTHON_ASSUME_SAFE_MACROS + #define CYTHON_ASSUME_SAFE_MACROS 0 + #undef CYTHON_UNPACK_METHODS + #define CYTHON_UNPACK_METHODS 0 + #undef CYTHON_FAST_THREAD_STATE + #define CYTHON_FAST_THREAD_STATE 0 + #undef CYTHON_FAST_GIL + #define CYTHON_FAST_GIL 0 + #undef CYTHON_METH_FASTCALL + #define CYTHON_METH_FASTCALL 0 + #undef CYTHON_FAST_PYCALL + #define CYTHON_FAST_PYCALL 0 + #ifndef CYTHON_PEP487_INIT_SUBCLASS + #define CYTHON_PEP487_INIT_SUBCLASS 1 + #endif + #undef CYTHON_PEP489_MULTI_PHASE_INIT + #define CYTHON_PEP489_MULTI_PHASE_INIT 0 + #ifndef CYTHON_USE_TP_FINALIZE + #define CYTHON_USE_TP_FINALIZE 1 + #endif + #undef CYTHON_USE_DICT_VERSIONS + #define CYTHON_USE_DICT_VERSIONS 0 + #undef CYTHON_USE_EXC_INFO_STACK + #define CYTHON_USE_EXC_INFO_STACK 0 + #else #define CYTHON_COMPILING_IN_PYPY 0 #define CYTHON_COMPILING_IN_PYSTON 0 #define CYTHON_COMPILING_IN_CPYTHON 1 + #define CYTHON_COMPILING_IN_LIMITED_API 0 #ifndef CYTHON_USE_TYPE_SLOTS #define CYTHON_USE_TYPE_SLOTS 1 #endif - #if PY_VERSION_HEX < 0x02070000 - // looks like calling _PyType_Lookup() isn't safe in Py<=2.6/3.1 - #undef CYTHON_USE_PYTYPE_LOOKUP - #define CYTHON_USE_PYTYPE_LOOKUP 0 - #elif !defined(CYTHON_USE_PYTYPE_LOOKUP) + #ifndef CYTHON_USE_PYTYPE_LOOKUP #define CYTHON_USE_PYTYPE_LOOKUP 1 #endif #if PY_MAJOR_VERSION < 3 @@ -148,10 +219,7 @@ #elif !defined(CYTHON_USE_ASYNC_SLOTS) #define CYTHON_USE_ASYNC_SLOTS 1 #endif - #if PY_VERSION_HEX < 0x02070000 - #undef CYTHON_USE_PYLONG_INTERNALS - #define CYTHON_USE_PYLONG_INTERNALS 0 - #elif !defined(CYTHON_USE_PYLONG_INTERNALS) + #ifndef CYTHON_USE_PYLONG_INTERNALS #define CYTHON_USE_PYLONG_INTERNALS 1 #endif #ifndef CYTHON_USE_PYLIST_INTERNALS @@ -178,9 +246,21 @@ #ifndef CYTHON_FAST_THREAD_STATE #define CYTHON_FAST_THREAD_STATE 1 #endif + #ifndef CYTHON_FAST_GIL + // Py3<3.5.2 does not support _PyThreadState_UncheckedGet(). + #define CYTHON_FAST_GIL (PY_MAJOR_VERSION < 3 || PY_VERSION_HEX >= 0x03060000) + #endif + #ifndef CYTHON_METH_FASTCALL + // CPython 3.6 introduced METH_FASTCALL but with slightly different + // semantics. It became stable starting from CPython 3.7. + #define CYTHON_METH_FASTCALL (PY_VERSION_HEX >= 0x030700A1) + #endif #ifndef CYTHON_FAST_PYCALL #define CYTHON_FAST_PYCALL 1 #endif + #ifndef CYTHON_PEP487_INIT_SUBCLASS + #define CYTHON_PEP487_INIT_SUBCLASS 1 + #endif #ifndef CYTHON_PEP489_MULTI_PHASE_INIT #define CYTHON_PEP489_MULTI_PHASE_INIT (PY_VERSION_HEX >= 0x03050000) #endif @@ -199,6 +279,13 @@ #define CYTHON_FAST_PYCCALL (CYTHON_FAST_PYCALL && PY_VERSION_HEX >= 0x030600B1) #endif +#if !defined(CYTHON_VECTORCALL) +#define CYTHON_VECTORCALL (CYTHON_FAST_PYCCALL && PY_VERSION_HEX >= 0x030800B1) +#endif + +/* Whether to use METH_FASTCALL with a fake backported implementation of vectorcall */ +#define CYTHON_BACKPORT_VECTORCALL (CYTHON_METH_FASTCALL && PY_VERSION_HEX < 0x030800B1) + #if CYTHON_USE_PYLONG_INTERNALS #include "longintrepr.h" /* These short defines can easily conflict with other code */ @@ -268,15 +355,31 @@ #ifdef _MSC_VER #ifndef _MSC_STDINT_H_ #if _MSC_VER < 1300 - typedef unsigned char uint8_t; - typedef unsigned int uint32_t; + typedef unsigned char uint8_t; + typedef unsigned short uint16_t; + typedef unsigned int uint32_t; #else - typedef unsigned __int8 uint8_t; - typedef unsigned __int32 uint32_t; + typedef unsigned __int8 uint8_t; + typedef unsigned __int16 uint16_t; + typedef unsigned __int32 uint32_t; + #endif + #endif + #if _MSC_VER < 1300 + #ifdef _WIN64 + typedef unsigned long long __pyx_uintptr_t; + #else + typedef unsigned int __pyx_uintptr_t; + #endif + #else + #ifdef _WIN64 + typedef unsigned __int64 __pyx_uintptr_t; + #else + typedef unsigned __int32 __pyx_uintptr_t; #endif #endif #else - #include <stdint.h> + #include <stdint.h> + typedef uintptr_t __pyx_uintptr_t; #endif @@ -340,7 +443,7 @@ #endif #endif -// Work around clang bug http://stackoverflow.com/questions/21847816/c-invoke-nested-template-class-destructor +// Work around clang bug https://stackoverflow.com/questions/21847816/c-invoke-nested-template-class-destructor template<typename T> void __Pyx_call_destructor(T& x) { x.~T(); @@ -376,19 +479,29 @@ class __Pyx_FakeReference { #if PY_MAJOR_VERSION < 3 #define __Pyx_BUILTIN_MODULE_NAME "__builtin__" - #define __Pyx_PyCode_New(a, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos) \ - PyCode_New(a+k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos) #define __Pyx_DefaultClassType PyClass_Type + #define __Pyx_PyCode_New(a, p, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos) \ + PyCode_New(a+k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos) #else #define __Pyx_BUILTIN_MODULE_NAME "builtins" -#if PY_VERSION_HEX >= 0x030800A4 && PY_VERSION_HEX < 0x030800B2 - #define __Pyx_PyCode_New(a, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos) \ - PyCode_New(a, 0, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos) + #define __Pyx_DefaultClassType PyType_Type +#if PY_VERSION_HEX >= 0x030800B2 + #define __Pyx_PyCode_New(a, p, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos) \ + PyCode_NewWithPosOnlyArgs(a, p, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos) +#elif PY_VERSION_HEX >= 0x030800A4 + // TODO: remove this special case once Py3.8 is released. + #define __Pyx_PyCode_New(a, p, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos) \ + PyCode_New(a, p, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos) #else - #define __Pyx_PyCode_New(a, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos) \ + #define __Pyx_PyCode_New(a, p, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos) \ PyCode_New(a, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos) #endif - #define __Pyx_DefaultClassType PyType_Type +#endif + +#if PY_VERSION_HEX >= 0x030900A4 || defined(Py_IS_TYPE) + #define __Pyx_IS_TYPE(ob, type) Py_IS_TYPE(ob, type) +#else + #define __Pyx_IS_TYPE(ob, type) (((const PyObject*)ob)->ob_type == (type)) #endif #ifndef Py_TPFLAGS_CHECKTYPES @@ -426,11 +539,29 @@ class __Pyx_FakeReference { #define __Pyx_PyCFunctionFast _PyCFunctionFast #define __Pyx_PyCFunctionFastWithKeywords _PyCFunctionFastWithKeywords #endif -#if CYTHON_FAST_PYCCALL -#define __Pyx_PyFastCFunction_Check(func) \ - ((PyCFunction_Check(func) && (METH_FASTCALL == (PyCFunction_GET_FLAGS(func) & ~(METH_CLASS | METH_STATIC | METH_COEXIST | METH_KEYWORDS | METH_STACKLESS))))) + +#if CYTHON_METH_FASTCALL + #define __Pyx_METH_FASTCALL METH_FASTCALL + #define __Pyx_PyCFunction_FastCall __Pyx_PyCFunctionFast + #define __Pyx_PyCFunction_FastCallWithKeywords __Pyx_PyCFunctionFastWithKeywords +#else + #define __Pyx_METH_FASTCALL METH_VARARGS + #define __Pyx_PyCFunction_FastCall PyCFunction + #define __Pyx_PyCFunction_FastCallWithKeywords PyCFunctionWithKeywords +#endif + +#if CYTHON_VECTORCALL + #define __pyx_vectorcallfunc vectorcallfunc + #define __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET PY_VECTORCALL_ARGUMENTS_OFFSET + #define __Pyx_PyVectorcall_NARGS(n) PyVectorcall_NARGS((size_t)(n)) +#elif CYTHON_BACKPORT_VECTORCALL + typedef PyObject *(*__pyx_vectorcallfunc)(PyObject *callable, PyObject *const *args, + size_t nargsf, PyObject *kwnames); + #define __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET ((size_t)1 << (8 * sizeof(size_t) - 1)) + #define __Pyx_PyVectorcall_NARGS(n) ((Py_ssize_t)(((size_t)(n)) & ~__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)) #else -#define __Pyx_PyFastCFunction_Check(func) 0 + #define __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET 0 + #define __Pyx_PyVectorcall_NARGS(n) ((Py_ssize_t)(n)) #endif #if CYTHON_COMPILING_IN_PYPY && !defined(PyObject_Malloc) @@ -449,12 +580,17 @@ class __Pyx_FakeReference { // special C-API functions only in Pyston #define __Pyx_PyCode_HasFreeVars(co) PyCode_HasFreeVars(co) #define __Pyx_PyFrame_SetLineNumber(frame, lineno) PyFrame_SetLineNumber(frame, lineno) +#elif CYTHON_COMPILING_IN_LIMITED_API + #define __Pyx_PyCode_HasFreeVars(co) (PyCode_GetNumFree(co) > 0) + #define __Pyx_PyFrame_SetLineNumber(frame, lineno) #else #define __Pyx_PyCode_HasFreeVars(co) (PyCode_GetNumFree(co) > 0) #define __Pyx_PyFrame_SetLineNumber(frame, lineno) (frame)->f_lineno = (lineno) #endif -#if !CYTHON_FAST_THREAD_STATE || PY_VERSION_HEX < 0x02070000 +#if CYTHON_COMPILING_IN_LIMITED_API + #define __Pyx_PyThreadState_Current PyThreadState_Get() +#elif !CYTHON_FAST_THREAD_STATE #define __Pyx_PyThreadState_Current PyThreadState_GET() #elif PY_VERSION_HEX >= 0x03060000 //#elif PY_VERSION_HEX >= 0x03050200 @@ -466,6 +602,18 @@ class __Pyx_FakeReference { #define __Pyx_PyThreadState_Current _PyThreadState_Current #endif +#if CYTHON_COMPILING_IN_LIMITED_API +static inline void *__Pyx_PyModule_GetState(PyObject *op) +{ + void *result; + + result = PyModule_GetState(op); + if (!result) + Py_FatalError("Couldn't find the module state"); + return result; +} +#endif + // TSS (Thread Specific Storage) API #if PY_VERSION_HEX < 0x030700A2 && !defined(PyThread_tss_create) && !defined(Py_tss_NEEDS_INIT) #include "pythread.h" @@ -514,21 +662,82 @@ static CYTHON_INLINE void * PyThread_tss_get(Py_tss_t *key) { #define __Pyx_PyNumber_InPlaceDivide(x,y) PyNumber_InPlaceDivide(x,y) #endif -#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030500A1 && CYTHON_USE_UNICODE_INTERNALS -#define __Pyx_PyDict_GetItemStr(dict, name) _PyDict_GetItem_KnownHash(dict, name, ((PyASCIIObject *) name)->hash) +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX > 0x030600B4 && CYTHON_USE_UNICODE_INTERNALS +// _PyDict_GetItem_KnownHash() exists since CPython 3.5, but it was +// dropping exceptions. Since 3.6, exceptions are kept. +#define __Pyx_PyDict_GetItemStrWithError(dict, name) _PyDict_GetItem_KnownHash(dict, name, ((PyASCIIObject *) name)->hash) +static CYTHON_INLINE PyObject * __Pyx_PyDict_GetItemStr(PyObject *dict, PyObject *name) { + PyObject *res = __Pyx_PyDict_GetItemStrWithError(dict, name); + if (res == NULL) PyErr_Clear(); + return res; +} +#elif PY_MAJOR_VERSION >= 3 && (!CYTHON_COMPILING_IN_PYPY || PYPY_VERSION_NUM >= 0x07020000) +#define __Pyx_PyDict_GetItemStrWithError PyDict_GetItemWithError +#define __Pyx_PyDict_GetItemStr PyDict_GetItem +#else +static CYTHON_INLINE PyObject * __Pyx_PyDict_GetItemStrWithError(PyObject *dict, PyObject *name) { + // This is tricky - we should return a borrowed reference but not swallow non-KeyError exceptions. 8-| + // But: this function is only used in Py2 and older PyPys, + // and currently only for argument parsing and other non-correctness-critical lookups + // and we know that 'name' is an interned 'str' with pre-calculated hash value (only comparisons can fail), + // thus, performance matters more than correctness here, especially in the "not found" case. +#if CYTHON_COMPILING_IN_PYPY + // So we ignore any exceptions in old PyPys ... + return PyDict_GetItem(dict, name); #else -#define __Pyx_PyDict_GetItemStr(dict, name) PyDict_GetItem(dict, name) + // and hack together a stripped-down and modified PyDict_GetItem() in CPython 2. + PyDictEntry *ep; + PyDictObject *mp = (PyDictObject*) dict; + long hash = ((PyStringObject *) name)->ob_shash; + assert(hash != -1); /* hash values of interned strings are always initialised */ + ep = (mp->ma_lookup)(mp, name, hash); + if (ep == NULL) { + // error occurred + return NULL; + } + // found or not found + return ep->me_value; +#endif +} +#define __Pyx_PyDict_GetItemStr PyDict_GetItem #endif -/* new Py3.3 unicode type (PEP 393) */ -#if PY_VERSION_HEX > 0x03030000 && defined(PyUnicode_KIND) +/* Type slots */ +#if CYTHON_COMPILING_IN_LIMITED_API + #define __Pyx_PyType_GetFlags(tp) (PyType_GetFlags((PyTypeObject *)tp)) +#else + #define __Pyx_PyType_GetFlags(tp) (((PyTypeObject *)tp)->tp_flags) +#endif + +#if CYTHON_USE_TYPE_SLOTS + #define __Pyx_PyType_HasFeature(type, feature) ((__Pyx_PyType_GetFlags(type) & (feature)) != 0) + #define __Pyx_PyObject_GetIterNextFunc(obj) (Py_TYPE(obj)->tp_iternext) +#else + #define __Pyx_PyType_HasFeature(type, feature) PyType_HasFeature(type, feature) + #define __Pyx_PyObject_GetIterNextFunc(obj) PyIter_Next +#endif + +#if CYTHON_COMPILING_IN_LIMITED_API + #define CYTHON_PEP393_ENABLED 1 + #define __Pyx_PyUnicode_READY(op) (0) + #define __Pyx_PyUnicode_GET_LENGTH(u) PyUnicode_GetLength(u) + #define __Pyx_PyUnicode_READ_CHAR(u, i) PyUnicode_ReadChar(u, i) + #define __Pyx_PyUnicode_MAX_CHAR_VALUE(u) ((void)u, 1114111) + #define __Pyx_PyUnicode_KIND(u) ((void)u, (0)) + // __Pyx_PyUnicode_DATA() and __Pyx_PyUnicode_READ() must go together, e.g. for iteration. + #define __Pyx_PyUnicode_DATA(u) ((void*)u) + #define __Pyx_PyUnicode_READ(k, d, i) ((void)k, PyUnicode_ReadChar((PyObject*)(d), i)) + //#define __Pyx_PyUnicode_WRITE(k, d, i, ch) /* not available */ + #define __Pyx_PyUnicode_IS_TRUE(u) (0 != PyUnicode_GetLength(u)) +#elif PY_VERSION_HEX > 0x03030000 && defined(PyUnicode_KIND) + /* new Py3.3 unicode type (PEP 393) */ #define CYTHON_PEP393_ENABLED 1 #define __Pyx_PyUnicode_READY(op) (likely(PyUnicode_IS_READY(op)) ? \ 0 : _PyUnicode_Ready((PyObject *)(op))) #define __Pyx_PyUnicode_GET_LENGTH(u) PyUnicode_GET_LENGTH(u) #define __Pyx_PyUnicode_READ_CHAR(u, i) PyUnicode_READ_CHAR(u, i) #define __Pyx_PyUnicode_MAX_CHAR_VALUE(u) PyUnicode_MAX_CHAR_VALUE(u) - #define __Pyx_PyUnicode_KIND(u) PyUnicode_KIND(u) + #define __Pyx_PyUnicode_KIND(u) ((int)PyUnicode_KIND(u)) #define __Pyx_PyUnicode_DATA(u) PyUnicode_DATA(u) #define __Pyx_PyUnicode_READ(k, d, i) PyUnicode_READ(k, d, i) #define __Pyx_PyUnicode_WRITE(k, d, i, ch) PyUnicode_WRITE(k, d, i, ch) @@ -546,9 +755,9 @@ static CYTHON_INLINE void * PyThread_tss_get(Py_tss_t *key) { #define __Pyx_PyUnicode_GET_LENGTH(u) PyUnicode_GET_SIZE(u) #define __Pyx_PyUnicode_READ_CHAR(u, i) ((Py_UCS4)(PyUnicode_AS_UNICODE(u)[i])) #define __Pyx_PyUnicode_MAX_CHAR_VALUE(u) ((sizeof(Py_UNICODE) == 2) ? 65535 : 1114111) - #define __Pyx_PyUnicode_KIND(u) (sizeof(Py_UNICODE)) + #define __Pyx_PyUnicode_KIND(u) ((int)sizeof(Py_UNICODE)) #define __Pyx_PyUnicode_DATA(u) ((void*)PyUnicode_AS_UNICODE(u)) - /* (void)(k) => avoid unused variable warning due to macro: */ + // (void)(k) => avoid unused variable warning due to macro: #define __Pyx_PyUnicode_READ(k, d, i) ((void)(k), (Py_UCS4)(((Py_UNICODE*)d)[i])) #define __Pyx_PyUnicode_WRITE(k, d, i, ch) (((void)(k)), ((Py_UNICODE*)d)[i] = ch) #define __Pyx_PyUnicode_IS_TRUE(u) (0 != PyUnicode_GET_SIZE(u)) @@ -609,10 +818,16 @@ static CYTHON_INLINE void * PyThread_tss_get(Py_tss_t *key) { #define __Pyx_PyBaseString_CheckExact(obj) (PyString_CheckExact(obj) || PyUnicode_CheckExact(obj)) #endif -#ifndef PySet_CheckExact - #define PySet_CheckExact(obj) (Py_TYPE(obj) == &PySet_Type) +#if CYTHON_COMPILING_IN_CPYTHON + #define __Pyx_PySequence_ListKeepNew(obj) \ + (likely(PyList_CheckExact(obj) && Py_REFCNT(obj) == 1) ? __Pyx_NewRef(obj) : PySequence_List(obj)) +#else + #define __Pyx_PySequence_ListKeepNew(obj) PySequence_List(obj) #endif +#ifndef PySet_CheckExact + #define PySet_CheckExact(obj) __Pyx_IS_TYPE(obj, &PySet_Type) +#endif #if PY_VERSION_HEX >= 0x030900A4 #define __Pyx_SET_REFCNT(obj, refcnt) Py_SET_REFCNT(obj, refcnt) @@ -660,16 +875,10 @@ static CYTHON_INLINE void * PyThread_tss_get(Py_tss_t *key) { #if PY_VERSION_HEX < 0x030200A4 typedef long Py_hash_t; #define __Pyx_PyInt_FromHash_t PyInt_FromLong - #define __Pyx_PyInt_AsHash_t PyInt_AsLong + #define __Pyx_PyInt_AsHash_t __Pyx_PyIndex_AsHash_t #else #define __Pyx_PyInt_FromHash_t PyInt_FromSsize_t - #define __Pyx_PyInt_AsHash_t PyInt_AsSsize_t -#endif - -#if PY_MAJOR_VERSION >= 3 - #define __Pyx_PyMethod_New(func, self, klass) ((self) ? ((void)(klass), PyMethod_New(func, self)) : __Pyx_NewRef(func)) -#else - #define __Pyx_PyMethod_New(func, self, klass) PyMethod_New(func, self, klass) + #define __Pyx_PyInt_AsHash_t __Pyx_PyIndex_AsSsize_t #endif // backport of PyAsyncMethods from Py3.5 to older Py3.x versions @@ -692,7 +901,6 @@ static CYTHON_INLINE void * PyThread_tss_get(Py_tss_t *key) { } __Pyx_PyAsyncMethodsStruct; #endif - /////////////// SmallCodeConfig.proto /////////////// #ifndef CYTHON_SMALL_CODE @@ -733,14 +941,18 @@ static CYTHON_INLINE void * PyThread_tss_get(Py_tss_t *key) { #if CYTHON_COMPILING_IN_CPYTHON #define __Pyx_TypeCheck(obj, type) __Pyx_IsSubtype(Py_TYPE(obj), (PyTypeObject *)type) +#define __Pyx_TypeCheck2(obj, type1, type2) __Pyx_IsAnySubtype2(Py_TYPE(obj), (PyTypeObject *)type1, (PyTypeObject *)type2) static CYTHON_INLINE int __Pyx_IsSubtype(PyTypeObject *a, PyTypeObject *b);/*proto*/ +static CYTHON_INLINE int __Pyx_IsAnySubtype2(PyTypeObject *cls, PyTypeObject *a, PyTypeObject *b);/*proto*/ static CYTHON_INLINE int __Pyx_PyErr_GivenExceptionMatches(PyObject *err, PyObject *type);/*proto*/ static CYTHON_INLINE int __Pyx_PyErr_GivenExceptionMatches2(PyObject *err, PyObject *type1, PyObject *type2);/*proto*/ #else #define __Pyx_TypeCheck(obj, type) PyObject_TypeCheck(obj, (PyTypeObject *)type) +#define __Pyx_TypeCheck2(obj, type1, type2) (PyObject_TypeCheck(obj, (PyTypeObject *)type1) || PyObject_TypeCheck(obj, (PyTypeObject *)type2)) #define __Pyx_PyErr_GivenExceptionMatches(err, type) PyErr_GivenExceptionMatches(err, type) #define __Pyx_PyErr_GivenExceptionMatches2(err, type1, type2) (PyErr_GivenExceptionMatches(err, type1) || PyErr_GivenExceptionMatches(err, type2)) #endif +#define __Pyx_PyErr_ExceptionMatches2(err1, err2) __Pyx_PyErr_GivenExceptionMatches2(__Pyx_PyErr_Occurred(), err1, err2) #define __Pyx_PyException_Check(obj) __Pyx_TypeCheck(obj, PyExc_Exception) @@ -775,6 +987,24 @@ static CYTHON_INLINE int __Pyx_IsSubtype(PyTypeObject *a, PyTypeObject *b) { return __Pyx_InBases(a, b); } +static CYTHON_INLINE int __Pyx_IsAnySubtype2(PyTypeObject *cls, PyTypeObject *a, PyTypeObject *b) { + PyObject *mro; + if (cls == a || cls == b) return 1; + mro = cls->tp_mro; + if (likely(mro)) { + Py_ssize_t i, n; + n = PyTuple_GET_SIZE(mro); + for (i = 0; i < n; i++) { + PyObject *base = PyTuple_GET_ITEM(mro, i); + if (base == (PyObject *)a || base == (PyObject *)b) + return 1; + } + return 0; + } + // should only get here for incompletely initialised types, i.e. never under normal usage patterns + return __Pyx_InBases(cls, a) || __Pyx_InBases(cls, b); +} + #if PY_MAJOR_VERSION == 2 static int __Pyx_inner_PyErr_GivenExceptionMatches2(PyObject *err, PyObject* exc_type1, PyObject* exc_type2) { @@ -805,11 +1035,11 @@ static int __Pyx_inner_PyErr_GivenExceptionMatches2(PyObject *err, PyObject* exc } #else static CYTHON_INLINE int __Pyx_inner_PyErr_GivenExceptionMatches2(PyObject *err, PyObject* exc_type1, PyObject *exc_type2) { - int res = exc_type1 ? __Pyx_IsSubtype((PyTypeObject*)err, (PyTypeObject*)exc_type1) : 0; - if (!res) { - res = __Pyx_IsSubtype((PyTypeObject*)err, (PyTypeObject*)exc_type2); + if (exc_type1) { + return __Pyx_IsAnySubtype2((PyTypeObject*)err, (PyTypeObject*)exc_type1, (PyTypeObject*)exc_type2); + } else { + return __Pyx_IsSubtype((PyTypeObject*)err, (PyTypeObject*)exc_type2); } - return res; } #endif @@ -870,7 +1100,7 @@ static CYTHON_INLINE int __Pyx_PyErr_GivenExceptionMatches2(PyObject *err, PyObj /////////////// MathInitCode /////////////// -#if defined(WIN32) || defined(MS_WINDOWS) +#if defined(_WIN32) || defined(WIN32) || defined(MS_WINDOWS) #define _USE_MATH_DEFINES #endif #include <math.h> @@ -909,7 +1139,7 @@ typedef struct {PyObject **p; const char *s; const Py_ssize_t n; const char* enc /////////////// InitThreads.init /////////////// -#ifdef WITH_THREAD +#if defined(WITH_THREAD) && PY_VERSION_HEX < 0x030700F0 PyEval_InitThreads(); #endif @@ -944,12 +1174,21 @@ static CYTHON_SMALL_CODE int __Pyx_check_single_interpreter(void) { return 0; } -static CYTHON_SMALL_CODE int __Pyx_copy_spec_to_module(PyObject *spec, PyObject *moddict, const char* from_name, const char* to_name, int allow_none) { +#if CYTHON_COMPILING_IN_LIMITED_API +static CYTHON_SMALL_CODE int __Pyx_copy_spec_to_module(PyObject *spec, PyObject *module, const char* from_name, const char* to_name, int allow_none) +#else +static CYTHON_SMALL_CODE int __Pyx_copy_spec_to_module(PyObject *spec, PyObject *moddict, const char* from_name, const char* to_name, int allow_none) +#endif +{ PyObject *value = PyObject_GetAttrString(spec, from_name); int result = 0; if (likely(value)) { if (allow_none || value != Py_None) { +#if CYTHON_COMPILING_IN_LIMITED_API + result = PyModule_AddObject(module, to_name, value); +#else result = PyDict_SetItemString(moddict, to_name, value); +#endif } Py_DECREF(value); } else if (PyErr_ExceptionMatches(PyExc_AttributeError)) { @@ -976,9 +1215,13 @@ static CYTHON_SMALL_CODE PyObject* ${pymodule_create_func_cname}(PyObject *spec, Py_DECREF(modname); if (unlikely(!module)) goto bad; +#if CYTHON_COMPILING_IN_LIMITED_API + moddict = module; +#else moddict = PyModule_GetDict(module); if (unlikely(!moddict)) goto bad; // moddict is a borrowed reference +#endif if (unlikely(__Pyx_copy_spec_to_module(spec, moddict, "loader", "__loader__", 1) < 0)) goto bad; if (unlikely(__Pyx_copy_spec_to_module(spec, moddict, "origin", "__file__", 1) < 0)) goto bad; @@ -995,6 +1238,7 @@ bad: /////////////// CodeObjectCache.proto /////////////// +#if !CYTHON_COMPILING_IN_LIMITED_API typedef struct { PyCodeObject* code_object; int code_line; @@ -1011,11 +1255,13 @@ static struct __Pyx_CodeObjectCache __pyx_code_cache = {0,0,NULL}; static int __pyx_bisect_code_objects(__Pyx_CodeObjectCacheEntry* entries, int count, int code_line); static PyCodeObject *__pyx_find_code_object(int code_line); static void __pyx_insert_code_object(int code_line, PyCodeObject* code_object); +#endif /////////////// CodeObjectCache /////////////// // Note that errors are simply ignored in the code below. // This is just a cache, if a lookup or insertion fails - so what? +#if !CYTHON_COMPILING_IN_LIMITED_API static int __pyx_bisect_code_objects(__Pyx_CodeObjectCacheEntry* entries, int count, int code_line) { int start = 0, mid = 0, end = count - 1; if (end >= 0 && code_line > entries[end].code_line) { @@ -1096,9 +1342,11 @@ static void __pyx_insert_code_object(int code_line, PyCodeObject* code_object) { __pyx_code_cache.count++; Py_INCREF(code_object); } +#endif /////////////// CodeObjectCache.cleanup /////////////// + #if !CYTHON_COMPILING_IN_LIMITED_API if (__pyx_code_cache.entries) { __Pyx_CodeObjectCacheEntry* entries = __pyx_code_cache.entries; int i, count = __pyx_code_cache.count; @@ -1110,6 +1358,7 @@ static void __pyx_insert_code_object(int code_line, PyCodeObject* code_object) { } PyMem_Free(entries); } + #endif /////////////// CheckBinaryVersion.proto /////////////// @@ -1156,11 +1405,11 @@ static CYTHON_INLINE int __Pyx_Is_Little_Endian(void) #if CYTHON_REFNANNY typedef struct { - void (*INCREF)(void*, PyObject*, int); - void (*DECREF)(void*, PyObject*, int); - void (*GOTREF)(void*, PyObject*, int); - void (*GIVEREF)(void*, PyObject*, int); - void* (*SetupContext)(const char*, int, const char*); + void (*INCREF)(void*, PyObject*, Py_ssize_t); + void (*DECREF)(void*, PyObject*, Py_ssize_t); + void (*GOTREF)(void*, PyObject*, Py_ssize_t); + void (*GIVEREF)(void*, PyObject*, Py_ssize_t); + void* (*SetupContext)(const char*, Py_ssize_t, const char*); void (*FinishContext)(void**); } __Pyx_RefNannyAPIStruct; static __Pyx_RefNannyAPIStruct *__Pyx_RefNanny = NULL; @@ -1170,28 +1419,35 @@ static CYTHON_INLINE int __Pyx_Is_Little_Endian(void) #define __Pyx_RefNannySetupContext(name, acquire_gil) \ if (acquire_gil) { \ PyGILState_STATE __pyx_gilstate_save = PyGILState_Ensure(); \ - __pyx_refnanny = __Pyx_RefNanny->SetupContext((name), __LINE__, __FILE__); \ + __pyx_refnanny = __Pyx_RefNanny->SetupContext((name), (__LINE__), (__FILE__)); \ PyGILState_Release(__pyx_gilstate_save); \ } else { \ - __pyx_refnanny = __Pyx_RefNanny->SetupContext((name), __LINE__, __FILE__); \ + __pyx_refnanny = __Pyx_RefNanny->SetupContext((name), (__LINE__), (__FILE__)); \ + } + #define __Pyx_RefNannyFinishContextNogil() { \ + PyGILState_STATE __pyx_gilstate_save = PyGILState_Ensure(); \ + __Pyx_RefNannyFinishContext(); \ + PyGILState_Release(__pyx_gilstate_save); \ } #else #define __Pyx_RefNannySetupContext(name, acquire_gil) \ - __pyx_refnanny = __Pyx_RefNanny->SetupContext((name), __LINE__, __FILE__) + __pyx_refnanny = __Pyx_RefNanny->SetupContext((name), (__LINE__), (__FILE__)) + #define __Pyx_RefNannyFinishContextNogil() __Pyx_RefNannyFinishContext() #endif #define __Pyx_RefNannyFinishContext() \ __Pyx_RefNanny->FinishContext(&__pyx_refnanny) - #define __Pyx_INCREF(r) __Pyx_RefNanny->INCREF(__pyx_refnanny, (PyObject *)(r), __LINE__) - #define __Pyx_DECREF(r) __Pyx_RefNanny->DECREF(__pyx_refnanny, (PyObject *)(r), __LINE__) - #define __Pyx_GOTREF(r) __Pyx_RefNanny->GOTREF(__pyx_refnanny, (PyObject *)(r), __LINE__) - #define __Pyx_GIVEREF(r) __Pyx_RefNanny->GIVEREF(__pyx_refnanny, (PyObject *)(r), __LINE__) - #define __Pyx_XINCREF(r) do { if((r) != NULL) {__Pyx_INCREF(r); }} while(0) - #define __Pyx_XDECREF(r) do { if((r) != NULL) {__Pyx_DECREF(r); }} while(0) - #define __Pyx_XGOTREF(r) do { if((r) != NULL) {__Pyx_GOTREF(r); }} while(0) - #define __Pyx_XGIVEREF(r) do { if((r) != NULL) {__Pyx_GIVEREF(r);}} while(0) + #define __Pyx_INCREF(r) __Pyx_RefNanny->INCREF(__pyx_refnanny, (PyObject *)(r), (__LINE__)) + #define __Pyx_DECREF(r) __Pyx_RefNanny->DECREF(__pyx_refnanny, (PyObject *)(r), (__LINE__)) + #define __Pyx_GOTREF(r) __Pyx_RefNanny->GOTREF(__pyx_refnanny, (PyObject *)(r), (__LINE__)) + #define __Pyx_GIVEREF(r) __Pyx_RefNanny->GIVEREF(__pyx_refnanny, (PyObject *)(r), (__LINE__)) + #define __Pyx_XINCREF(r) do { if((r) == NULL); else {__Pyx_INCREF(r); }} while(0) + #define __Pyx_XDECREF(r) do { if((r) == NULL); else {__Pyx_DECREF(r); }} while(0) + #define __Pyx_XGOTREF(r) do { if((r) == NULL); else {__Pyx_GOTREF(r); }} while(0) + #define __Pyx_XGIVEREF(r) do { if((r) == NULL); else {__Pyx_GIVEREF(r);}} while(0) #else #define __Pyx_RefNannyDeclarations #define __Pyx_RefNannySetupContext(name, acquire_gil) + #define __Pyx_RefNannyFinishContextNogil() #define __Pyx_RefNannyFinishContext() #define __Pyx_INCREF(r) Py_INCREF(r) #define __Pyx_DECREF(r) Py_DECREF(r) @@ -1203,6 +1459,10 @@ static CYTHON_INLINE int __Pyx_Is_Little_Endian(void) #define __Pyx_XGIVEREF(r) #endif /* CYTHON_REFNANNY */ +#define __Pyx_Py_XDECREF_SET(r, v) do { \ + PyObject *tmp = (PyObject *) r; \ + r = v; Py_XDECREF(tmp); \ + } while (0) #define __Pyx_XDECREF_SET(r, v) do { \ PyObject *tmp = (PyObject *) r; \ r = v; __Pyx_XDECREF(tmp); \ @@ -1350,6 +1610,8 @@ __Pyx_FastGilFuncInit(); /////////////// FastGil.proto /////////////// //@proto_block: utility_code_proto_before_types +#if CYTHON_FAST_GIL + struct __Pyx_FastGilVtab { PyGILState_STATE (*Fast_PyGILState_Ensure)(void); void (*Fast_PyGILState_Release)(PyGILState_STATE oldstate); @@ -1384,6 +1646,14 @@ static void __Pyx_FastGilFuncInit(void); #endif #endif +#else +#define __Pyx_PyGILState_Ensure PyGILState_Ensure +#define __Pyx_PyGILState_Release PyGILState_Release +#define __Pyx_FastGIL_Remember() +#define __Pyx_FastGIL_Forget() +#define __Pyx_FastGilFuncInit() +#endif + /////////////// FastGil /////////////// //@requires: CommonStructures.c::FetchCommonPointer // The implementations of PyGILState_Ensure/Release calls PyThread_get_key_value @@ -1393,15 +1663,13 @@ static void __Pyx_FastGilFuncInit(void); // To make optimal use of this thread local, we attempt to share it between // modules. -#define __Pyx_FastGIL_ABI_module "_cython_" CYTHON_ABI +#if CYTHON_FAST_GIL + +#define __Pyx_FastGIL_ABI_module __PYX_ABI_MODULE_NAME #define __Pyx_FastGIL_PyCapsuleName "FastGilFuncs" #define __Pyx_FastGIL_PyCapsule \ __Pyx_FastGIL_ABI_module "." __Pyx_FastGIL_PyCapsuleName -#if PY_VERSION_HEX < 0x02070000 - #undef CYTHON_THREAD_LOCAL -#endif - #ifdef CYTHON_THREAD_LOCAL #include "pythread.h" @@ -1495,11 +1763,7 @@ static void __Pyx_FastGilFuncInit0(void) { #endif static void __Pyx_FastGilFuncInit(void) { -#if PY_VERSION_HEX >= 0x02070000 struct __Pyx_FastGilVtab* shared = (struct __Pyx_FastGilVtab*)PyCapsule_Import(__Pyx_FastGIL_PyCapsule, 1); -#else - struct __Pyx_FastGilVtab* shared = NULL; -#endif if (shared) { __Pyx_FastGilFuncs = *shared; } else { @@ -1507,3 +1771,5 @@ static void __Pyx_FastGilFuncInit(void) { __Pyx_FastGilFuncInit0(); } } + +#endif diff --git a/Cython/Utility/NumpyImportArray.c b/Cython/Utility/NumpyImportArray.c new file mode 100644 index 000000000..aa8a18fea --- /dev/null +++ b/Cython/Utility/NumpyImportArray.c @@ -0,0 +1,21 @@ +///////////////////////// NumpyImportArray.init //////////////////// + +// comment below is deliberately kept in the generated C file to +// help users debug where this came from: +/* + * Cython has automatically inserted a call to _import_array since + * you didn't include one when you cimported numpy. To disable this + * add the line + * <void>numpy._import_array + */ +#ifdef NPY_NDARRAYOBJECT_H /* numpy headers have been included */ +// NO_IMPORT_ARRAY is Numpy's mechanism for indicating that import_array is handled elsewhere +#if !NO_IMPORT_ARRAY /* https://docs.scipy.org/doc/numpy-1.17.0/reference/c-api.array.html#c.NO_IMPORT_ARRAY */ +if (unlikely(_import_array() == -1)) { + PyErr_SetString(PyExc_ImportError, "numpy.core.multiarray failed to import " + "(auto-generated because you didn't call 'numpy.import_array()' after cimporting numpy; " + "use '<void>numpy._import_array' to disable if you are certain you don't need it)."); + {{ err_goto }}; +} +#endif +#endif diff --git a/Cython/Utility/ObjectHandling.c b/Cython/Utility/ObjectHandling.c index 2d42909c9..903c03833 100644 --- a/Cython/Utility/ObjectHandling.c +++ b/Cython/Utility/ObjectHandling.c @@ -133,7 +133,7 @@ static int __Pyx_unpack_tuple2_generic(PyObject* tuple, PyObject** pvalue1, PyOb if (unlikely(!iter)) goto bad; if (decref_tuple) { Py_DECREF(tuple); tuple = NULL; } - iternext = Py_TYPE(iter)->tp_iternext; + iternext = __Pyx_PyObject_GetIterNextFunc(iter); value1 = iternext(iter); if (unlikely(!value1)) { index = 0; goto unpacking_failed; } value2 = iternext(iter); if (unlikely(!value2)) { index = 1; goto unpacking_failed; } if (!has_known_size && unlikely(__Pyx_IternextUnpackEndCheck(iternext(iter), 2))) goto bad; @@ -185,8 +185,10 @@ static PyObject *__Pyx_PyIter_Next2Default(PyObject* defval) { } static void __Pyx_PyIter_Next_ErrorNoIterator(PyObject *iterator) { + __Pyx_TypeName iterator_type_name = __Pyx_PyType_GetName(Py_TYPE(iterator)); PyErr_Format(PyExc_TypeError, - "%.200s object is not an iterator", Py_TYPE(iterator)->tp_name); + __Pyx_FMT_TYPENAME " object is not an iterator", iterator_type_name); + __Pyx_DECREF_TypeName(iterator_type_name); } // originally copied from Py3's builtin_next() @@ -199,10 +201,8 @@ static CYTHON_INLINE PyObject *__Pyx_PyIter_Next2(PyObject* iterator, PyObject* next = iternext(iterator); if (likely(next)) return next; - #if PY_VERSION_HEX >= 0x02070000 if (unlikely(iternext == &_PyObject_NextNotImplemented)) return NULL; - #endif #else // Since the slot was set, assume that PyIter_Next() will likely succeed, and properly fail otherwise. // Note: PyIter_Next() crashes in CPython if "tp_iternext" is NULL. @@ -275,24 +275,21 @@ static CYTHON_INLINE int __Pyx_IterFinish(void) { /////////////// ObjectGetItem.proto /////////////// #if CYTHON_USE_TYPE_SLOTS -static CYTHON_INLINE PyObject *__Pyx_PyObject_GetItem(PyObject *obj, PyObject* key);/*proto*/ +static CYTHON_INLINE PyObject *__Pyx_PyObject_GetItem(PyObject *obj, PyObject *key);/*proto*/ #else #define __Pyx_PyObject_GetItem(obj, key) PyObject_GetItem(obj, key) #endif /////////////// ObjectGetItem /////////////// // //@requires: GetItemInt - added in IndexNode as it uses templating. +//@requires: PyObjectGetAttrStrNoError +//@requires: PyObjectCallOneArg #if CYTHON_USE_TYPE_SLOTS -static PyObject *__Pyx_PyObject_GetIndex(PyObject *obj, PyObject* index) { +static PyObject *__Pyx_PyObject_GetIndex(PyObject *obj, PyObject *index) { + // Get element from sequence object `obj` at index `index`. PyObject *runerr; Py_ssize_t key_value; - PySequenceMethods *m = Py_TYPE(obj)->tp_as_sequence; - if (unlikely(!(m && m->sq_item))) { - PyErr_Format(PyExc_TypeError, "'%.200s' object is not subscriptable", Py_TYPE(obj)->tp_name); - return NULL; - } - key_value = __Pyx_PyIndex_AsSsize_t(index); if (likely(key_value != -1 || !(runerr = PyErr_Occurred()))) { return __Pyx_GetItemInt_Fast(obj, key_value, 0, 1, 1); @@ -300,18 +297,46 @@ static PyObject *__Pyx_PyObject_GetIndex(PyObject *obj, PyObject* index) { // Error handling code -- only manage OverflowError differently. if (PyErr_GivenExceptionMatches(runerr, PyExc_OverflowError)) { + __Pyx_TypeName index_type_name = __Pyx_PyType_GetName(Py_TYPE(index)); PyErr_Clear(); - PyErr_Format(PyExc_IndexError, "cannot fit '%.200s' into an index-sized integer", Py_TYPE(index)->tp_name); + PyErr_Format(PyExc_IndexError, + "cannot fit '" __Pyx_FMT_TYPENAME "' into an index-sized integer", index_type_name); + __Pyx_DECREF_TypeName(index_type_name); + } + return NULL; +} + +static PyObject *__Pyx_PyObject_GetItem_Slow(PyObject *obj, PyObject *key) { + __Pyx_TypeName obj_type_name; + // Handles less common slow-path checks for GetItem + if (likely(PyType_Check(obj))) { + PyObject *meth = __Pyx_PyObject_GetAttrStrNoError(obj, PYIDENT("__class_getitem__")); + if (meth) { + PyObject *result = __Pyx_PyObject_CallOneArg(meth, key); + Py_DECREF(meth); + return result; + } } + + obj_type_name = __Pyx_PyType_GetName(Py_TYPE(obj)); + PyErr_Format(PyExc_TypeError, + "'" __Pyx_FMT_TYPENAME "' object is not subscriptable", obj_type_name); + __Pyx_DECREF_TypeName(obj_type_name); return NULL; } -static PyObject *__Pyx_PyObject_GetItem(PyObject *obj, PyObject* key) { - PyMappingMethods *m = Py_TYPE(obj)->tp_as_mapping; - if (likely(m && m->mp_subscript)) { - return m->mp_subscript(obj, key); +static PyObject *__Pyx_PyObject_GetItem(PyObject *obj, PyObject *key) { + PyTypeObject *tp = Py_TYPE(obj); + PyMappingMethods *mm = tp->tp_as_mapping; + PySequenceMethods *sm = tp->tp_as_sequence; + + if (likely(mm && mm->mp_subscript)) { + return mm->mp_subscript(obj, key); + } + if (likely(sm && sm->sq_item)) { + return __Pyx_PyObject_GetIndex(obj, key); } - return __Pyx_PyObject_GetIndex(obj, key); + return __Pyx_PyObject_GetItem_Slow(obj, key); } #endif @@ -358,6 +383,7 @@ static PyObject *__Pyx_PyDict_GetItem(PyObject *d, PyObject* key) { #endif /////////////// GetItemInt.proto /////////////// +//@substitute: tempita #define __Pyx_GetItemInt(o, i, type, is_signed, to_py_func, is_list, wraparound, boundscheck) \ (__Pyx_fits_Py_ssize_t(i, type, is_signed) ? \ @@ -380,10 +406,11 @@ static CYTHON_INLINE PyObject *__Pyx_GetItemInt_Fast(PyObject *o, Py_ssize_t i, int is_list, int wraparound, int boundscheck); /////////////// GetItemInt /////////////// +//@substitute: tempita static PyObject *__Pyx_GetItemInt_Generic(PyObject *o, PyObject* j) { PyObject *r; - if (!j) return NULL; + if (unlikely(!j)) return NULL; r = PyObject_GetItem(o, j); Py_DECREF(j); return r; @@ -431,10 +458,18 @@ static CYTHON_INLINE PyObject *__Pyx_GetItemInt_Fast(PyObject *o, Py_ssize_t i, } } else { // inlined PySequence_GetItem() + special cased length overflow - PySequenceMethods *m = Py_TYPE(o)->tp_as_sequence; - if (likely(m && m->sq_item)) { - if (wraparound && unlikely(i < 0) && likely(m->sq_length)) { - Py_ssize_t l = m->sq_length(o); + PyMappingMethods *mm = Py_TYPE(o)->tp_as_mapping; + PySequenceMethods *sm = Py_TYPE(o)->tp_as_sequence; + if (mm && mm->mp_subscript) { + PyObject *r, *key = PyInt_FromSsize_t(i); + if (unlikely(!key)) return NULL; + r = mm->mp_subscript(o, key); + Py_DECREF(key); + return r; + } + if (likely(sm && sm->sq_item)) { + if (wraparound && unlikely(i < 0) && likely(sm->sq_length)) { + Py_ssize_t l = sm->sq_length(o); if (likely(l >= 0)) { i += l; } else { @@ -444,7 +479,7 @@ static CYTHON_INLINE PyObject *__Pyx_GetItemInt_Fast(PyObject *o, Py_ssize_t i, PyErr_Clear(); } } - return m->sq_item(o, i); + return sm->sq_item(o, i); } } #else @@ -471,7 +506,7 @@ static CYTHON_INLINE int __Pyx_SetItemInt_Fast(PyObject *o, Py_ssize_t i, PyObje static int __Pyx_SetItemInt_Generic(PyObject *o, PyObject *j, PyObject *v) { int r; - if (!j) return -1; + if (unlikely(!j)) return -1; r = PyObject_SetItem(o, j, v); Py_DECREF(j); return r; @@ -491,10 +526,19 @@ static CYTHON_INLINE int __Pyx_SetItemInt_Fast(PyObject *o, Py_ssize_t i, PyObje } } else { // inlined PySequence_SetItem() + special cased length overflow - PySequenceMethods *m = Py_TYPE(o)->tp_as_sequence; - if (likely(m && m->sq_ass_item)) { - if (wraparound && unlikely(i < 0) && likely(m->sq_length)) { - Py_ssize_t l = m->sq_length(o); + PyMappingMethods *mm = Py_TYPE(o)->tp_as_mapping; + PySequenceMethods *sm = Py_TYPE(o)->tp_as_sequence; + if (mm && mm->mp_ass_subscript) { + int r; + PyObject *key = PyInt_FromSsize_t(i); + if (unlikely(!key)) return -1; + r = mm->mp_ass_subscript(o, key, v); + Py_DECREF(key); + return r; + } + if (likely(sm && sm->sq_ass_item)) { + if (wraparound && unlikely(i < 0) && likely(sm->sq_length)) { + Py_ssize_t l = sm->sq_length(o); if (likely(l >= 0)) { i += l; } else { @@ -504,7 +548,7 @@ static CYTHON_INLINE int __Pyx_SetItemInt_Fast(PyObject *o, Py_ssize_t i, PyObje PyErr_Clear(); } } - return m->sq_ass_item(o, i, v); + return sm->sq_ass_item(o, i, v); } } #else @@ -537,24 +581,29 @@ static CYTHON_INLINE int __Pyx_DelItemInt_Fast(PyObject *o, Py_ssize_t i, static int __Pyx_DelItem_Generic(PyObject *o, PyObject *j) { int r; - if (!j) return -1; + if (unlikely(!j)) return -1; r = PyObject_DelItem(o, j); Py_DECREF(j); return r; } static CYTHON_INLINE int __Pyx_DelItemInt_Fast(PyObject *o, Py_ssize_t i, - CYTHON_UNUSED int is_list, CYTHON_NCP_UNUSED int wraparound) { + int is_list, CYTHON_NCP_UNUSED int wraparound) { #if !CYTHON_USE_TYPE_SLOTS if (is_list || PySequence_Check(o)) { return PySequence_DelItem(o, i); } #else // inlined PySequence_DelItem() + special cased length overflow - PySequenceMethods *m = Py_TYPE(o)->tp_as_sequence; - if (likely(m && m->sq_ass_item)) { - if (wraparound && unlikely(i < 0) && likely(m->sq_length)) { - Py_ssize_t l = m->sq_length(o); + PyMappingMethods *mm = Py_TYPE(o)->tp_as_mapping; + PySequenceMethods *sm = Py_TYPE(o)->tp_as_sequence; + if ((!is_list) && mm && mm->mp_ass_subscript) { + PyObject *key = PyInt_FromSsize_t(i); + return likely(key) ? mm->mp_ass_subscript(o, key, (PyObject *)NULL) : -1; + } + if (likely(sm && sm->sq_ass_item)) { + if (wraparound && unlikely(i < 0) && likely(sm->sq_length)) { + Py_ssize_t l = sm->sq_length(o); if (likely(l >= 0)) { i += l; } else { @@ -564,7 +613,7 @@ static CYTHON_INLINE int __Pyx_DelItemInt_Fast(PyObject *o, Py_ssize_t i, PyErr_Clear(); } } - return m->sq_ass_item(o, i, (PyObject *)NULL); + return sm->sq_ass_item(o, i, (PyObject *)NULL); } #endif return __Pyx_DelItem_Generic(o, PyInt_FromSsize_t(i)); @@ -600,6 +649,7 @@ static CYTHON_INLINE int __Pyx_PyObject_SetSlice(PyObject* obj, PyObject* value, Py_ssize_t cstart, Py_ssize_t cstop, PyObject** _py_start, PyObject** _py_stop, PyObject** _py_slice, int has_cstart, int has_cstop, CYTHON_UNUSED int wraparound) { + __Pyx_TypeName obj_type_name; #if CYTHON_USE_TYPE_SLOTS PyMappingMethods* mp; #if PY_MAJOR_VERSION < 3 @@ -702,19 +752,70 @@ static CYTHON_INLINE int __Pyx_PyObject_SetSlice(PyObject* obj, PyObject* value, } return result; } + obj_type_name = __Pyx_PyType_GetName(Py_TYPE(obj)); PyErr_Format(PyExc_TypeError, {{if access == 'Get'}} - "'%.200s' object is unsliceable", Py_TYPE(obj)->tp_name); + "'" __Pyx_FMT_TYPENAME "' object is unsliceable", obj_type_name); {{else}} - "'%.200s' object does not support slice %.10s", - Py_TYPE(obj)->tp_name, value ? "assignment" : "deletion"); + "'" __Pyx_FMT_TYPENAME "' object does not support slice %.10s", + obj_type_name, value ? "assignment" : "deletion"); {{endif}} + __Pyx_DECREF_TypeName(obj_type_name); bad: return {{if access == 'Get'}}NULL{{else}}-1{{endif}}; } +/////////////// TupleAndListFromArray.proto /////////////// + +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyList_FromArray(PyObject *const *src, Py_ssize_t n); +static CYTHON_INLINE PyObject* __Pyx_PyTuple_FromArray(PyObject *const *src, Py_ssize_t n); +#endif + +/////////////// TupleAndListFromArray /////////////// +//@substitute: naming + +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE void __Pyx_copy_object_array(PyObject *const *CYTHON_RESTRICT src, PyObject** CYTHON_RESTRICT dest, Py_ssize_t length) { + PyObject *v; + Py_ssize_t i; + for (i = 0; i < length; i++) { + v = dest[i] = src[i]; + Py_INCREF(v); + } +} + +static CYTHON_INLINE PyObject * +__Pyx_PyTuple_FromArray(PyObject *const *src, Py_ssize_t n) +{ + PyObject *res; + if (n <= 0) { + Py_INCREF($empty_tuple); + return $empty_tuple; + } + res = PyTuple_New(n); + if (unlikely(res == NULL)) return NULL; + __Pyx_copy_object_array(src, ((PyTupleObject*)res)->ob_item, n); + return res; +} + +static CYTHON_INLINE PyObject * +__Pyx_PyList_FromArray(PyObject *const *src, Py_ssize_t n) +{ + PyObject *res; + if (n <= 0) { + return PyList_New(0); + } + res = PyList_New(n); + if (unlikely(res == NULL)) return NULL; + __Pyx_copy_object_array(src, ((PyListObject*)res)->ob_item, n); + return res; +} +#endif + + /////////////// SliceTupleAndList.proto /////////////// #if CYTHON_COMPILING_IN_CPYTHON @@ -726,6 +827,8 @@ static CYTHON_INLINE PyObject* __Pyx_PyTuple_GetSlice(PyObject* src, Py_ssize_t #endif /////////////// SliceTupleAndList /////////////// +//@requires: TupleAndListFromArray +//@substitute: tempita #if CYTHON_COMPILING_IN_CPYTHON static CYTHON_INLINE void __Pyx_crop_slice(Py_ssize_t* _start, Py_ssize_t* _stop, Py_ssize_t* _length) { @@ -746,32 +849,12 @@ static CYTHON_INLINE void __Pyx_crop_slice(Py_ssize_t* _start, Py_ssize_t* _stop *_stop = stop; } -static CYTHON_INLINE void __Pyx_copy_object_array(PyObject** CYTHON_RESTRICT src, PyObject** CYTHON_RESTRICT dest, Py_ssize_t length) { - PyObject *v; - Py_ssize_t i; - for (i = 0; i < length; i++) { - v = dest[i] = src[i]; - Py_INCREF(v); - } -} - {{for type in ['List', 'Tuple']}} static CYTHON_INLINE PyObject* __Pyx_Py{{type}}_GetSlice( PyObject* src, Py_ssize_t start, Py_ssize_t stop) { - PyObject* dest; Py_ssize_t length = Py{{type}}_GET_SIZE(src); __Pyx_crop_slice(&start, &stop, &length); - if (unlikely(length <= 0)) - return Py{{type}}_New(0); - - dest = Py{{type}}_New(length); - if (unlikely(!dest)) - return NULL; - __Pyx_copy_object_array( - ((Py{{type}}Object*)src)->ob_item + start, - ((Py{{type}}Object*)dest)->ob_item, - length); - return dest; + return __Pyx_Py{{type}}_FromArray(((Py{{type}}Object*)src)->ob_item + start, length); } {{endfor}} #endif @@ -913,10 +996,14 @@ static PyObject *__Pyx_CreateClass(PyObject *bases, PyObject *dict, PyObject *na PyObject *result; PyObject *metaclass; - if (PyDict_SetItem(dict, PYIDENT("__module__"), modname) < 0) + if (unlikely(PyDict_SetItem(dict, PYIDENT("__module__"), modname) < 0)) return NULL; - if (PyDict_SetItem(dict, PYIDENT("__qualname__"), qualname) < 0) +#if PY_VERSION_HEX >= 0x03030000 + if (unlikely(PyDict_SetItem(dict, PYIDENT("__qualname__"), qualname) < 0)) return NULL; +#else + CYTHON_MAYBE_UNUSED_VAR(qualname); +#endif /* Python2 __metaclass__ */ metaclass = __Pyx_PyDict_GetItemStr(dict, PYIDENT("__metaclass__")); @@ -937,6 +1024,94 @@ static PyObject *__Pyx_CreateClass(PyObject *bases, PyObject *dict, PyObject *na return result; } +/////////////// Py3UpdateBases.proto /////////////// + +static PyObject* __Pyx_PEP560_update_bases(PyObject *bases); /* proto */ + +/////////////// Py3UpdateBases ///////////////////// +//@requires: PyObjectCallOneArg +//@requires: PyObjectGetAttrStrNoError + +/* Shamelessly adapted from cpython/bltinmodule.c update_bases */ +static PyObject* +__Pyx_PEP560_update_bases(PyObject *bases) +{ + Py_ssize_t i, j, size_bases; + PyObject *base, *meth, *new_base, *result, *new_bases = NULL; + /*assert(PyTuple_Check(bases));*/ + + size_bases = PyTuple_GET_SIZE(bases); + for (i = 0; i < size_bases; i++) { + // original code in CPython: base = args[i]; + base = PyTuple_GET_ITEM(bases, i); + if (PyType_Check(base)) { + if (new_bases) { + // If we already have made a replacement, then we append every normal base, + // otherwise just skip it. + if (PyList_Append(new_bases, base) < 0) { + goto error; + } + } + continue; + } + // original code in CPython: + // if (_PyObject_LookupAttrId(base, &PyId___mro_entries__, &meth) < 0) { + meth = __Pyx_PyObject_GetAttrStrNoError(base, PYIDENT("__mro_entries__")); + if (!meth && PyErr_Occurred()) { + goto error; + } + if (!meth) { + if (new_bases) { + if (PyList_Append(new_bases, base) < 0) { + goto error; + } + } + continue; + } + new_base = __Pyx_PyObject_CallOneArg(meth, bases); + Py_DECREF(meth); + if (!new_base) { + goto error; + } + if (!PyTuple_Check(new_base)) { + PyErr_SetString(PyExc_TypeError, + "__mro_entries__ must return a tuple"); + Py_DECREF(new_base); + goto error; + } + if (!new_bases) { + // If this is a first successful replacement, create new_bases list and + // copy previously encountered bases. + if (!(new_bases = PyList_New(i))) { + goto error; + } + for (j = 0; j < i; j++) { + // original code in CPython: base = args[j]; + base = PyTuple_GET_ITEM(bases, j); + PyList_SET_ITEM(new_bases, j, base); + Py_INCREF(base); + } + } + j = PyList_GET_SIZE(new_bases); + if (PyList_SetSlice(new_bases, j, j, new_base) < 0) { + goto error; + } + Py_DECREF(new_base); + } + if (!new_bases) { + // unlike the CPython implementation, always return a new reference + Py_INCREF(bases); + return bases; + } + result = PyList_AsTuple(new_bases); + Py_DECREF(new_bases); + return result; + +error: + Py_XDECREF(new_bases); + return NULL; +} + /////////////// Py3ClassCreate.proto /////////////// static PyObject *__Pyx_Py3MetaclassPrepare(PyObject *metaclass, PyObject *bases, PyObject *name, PyObject *qualname, @@ -945,27 +1120,27 @@ static PyObject *__Pyx_Py3ClassCreate(PyObject *metaclass, PyObject *name, PyObj PyObject *mkw, int calculate_metaclass, int allow_py2_metaclass); /*proto*/ /////////////// Py3ClassCreate /////////////// -//@requires: PyObjectGetAttrStr +//@substitute: naming +//@requires: PyObjectGetAttrStrNoError //@requires: CalculateMetaclass +//@requires: PyObjectFastCall +//@requires: PyObjectCall2Args +//@requires: PyObjectLookupSpecial +// only in fallback code: +//@requires: GetBuiltinName static PyObject *__Pyx_Py3MetaclassPrepare(PyObject *metaclass, PyObject *bases, PyObject *name, PyObject *qualname, PyObject *mkw, PyObject *modname, PyObject *doc) { PyObject *ns; if (metaclass) { - PyObject *prep = __Pyx_PyObject_GetAttrStr(metaclass, PYIDENT("__prepare__")); + PyObject *prep = __Pyx_PyObject_GetAttrStrNoError(metaclass, PYIDENT("__prepare__")); if (prep) { - PyObject *pargs = PyTuple_Pack(2, name, bases); - if (unlikely(!pargs)) { - Py_DECREF(prep); - return NULL; - } - ns = PyObject_Call(prep, pargs, mkw); + PyObject *pargs[3] = {NULL, name, bases}; + ns = __Pyx_PyObject_FastCallDict(prep, pargs+1, 2 | __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET, mkw); Py_DECREF(prep); - Py_DECREF(pargs); } else { - if (unlikely(!PyErr_ExceptionMatches(PyExc_AttributeError))) + if (unlikely(PyErr_Occurred())) return NULL; - PyErr_Clear(); ns = PyDict_New(); } } else { @@ -977,7 +1152,11 @@ static PyObject *__Pyx_Py3MetaclassPrepare(PyObject *metaclass, PyObject *bases, /* Required here to emulate assignment order */ if (unlikely(PyObject_SetItem(ns, PYIDENT("__module__"), modname) < 0)) goto bad; +#if PY_VERSION_HEX >= 0x03030000 if (unlikely(PyObject_SetItem(ns, PYIDENT("__qualname__"), qualname) < 0)) goto bad; +#else + CYTHON_MAYBE_UNUSED_VAR(qualname); +#endif if (unlikely(doc && PyObject_SetItem(ns, PYIDENT("__doc__"), doc) < 0)) goto bad; return ns; bad: @@ -985,11 +1164,164 @@ bad: return NULL; } +#if PY_VERSION_HEX < 0x030600A4 && CYTHON_PEP487_INIT_SUBCLASS +// https://www.python.org/dev/peps/pep-0487/ +static int __Pyx_SetNamesPEP487(PyObject *type_obj) { + PyTypeObject *type = (PyTypeObject*) type_obj; + PyObject *names_to_set, *key, *value, *set_name, *tmp; + Py_ssize_t i = 0; + +#if CYTHON_USE_TYPE_SLOTS + names_to_set = PyDict_Copy(type->tp_dict); +#else + { + PyObject *d = PyObject_GetAttr(type_obj, PYIDENT("__dict__")); + names_to_set = NULL; + if (likely(d)) { + // d may not be a dict, e.g. PyDictProxy in PyPy2. + PyObject *names_to_set = PyDict_New(); + int ret = likely(names_to_set) ? PyDict_Update(names_to_set, d) : -1; + Py_DECREF(d); + if (unlikely(ret < 0)) + Py_CLEAR(names_to_set); + } + } +#endif + if (unlikely(names_to_set == NULL)) + goto bad; + + while (PyDict_Next(names_to_set, &i, &key, &value)) { + set_name = __Pyx_PyObject_LookupSpecialNoError(value, PYIDENT("__set_name__")); + if (unlikely(set_name != NULL)) { + tmp = __Pyx_PyObject_Call2Args(set_name, type_obj, key); + Py_DECREF(set_name); + if (unlikely(tmp == NULL)) { + __Pyx_TypeName value_type_name = + __Pyx_PyType_GetName(Py_TYPE(value)); + __Pyx_TypeName type_name = __Pyx_PyType_GetName(type); + PyErr_Format(PyExc_RuntimeError, +#if PY_MAJOR_VERSION >= 3 + "Error calling __set_name__ on '" __Pyx_FMT_TYPENAME "' instance %R " "in '" __Pyx_FMT_TYPENAME "'", + value_type_name, key, type_name); +#else + "Error calling __set_name__ on '" __Pyx_FMT_TYPENAME "' instance %.100s in '" __Pyx_FMT_TYPENAME "'", + value_type_name, + PyString_Check(key) ? PyString_AS_STRING(key) : "?", + type_name); +#endif + goto bad; + } else { + Py_DECREF(tmp); + } + } + else if (unlikely(PyErr_Occurred())) { + goto bad; + } + } + + Py_DECREF(names_to_set); + return 0; +bad: + Py_XDECREF(names_to_set); + return -1; +} + +static PyObject *__Pyx_InitSubclassPEP487(PyObject *type_obj, PyObject *mkw) { +#if CYTHON_USE_TYPE_SLOTS && CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS +// Stripped-down version of "super(type_obj, type_obj).__init_subclass__(**mkw)" in CPython 3.8. + PyTypeObject *type = (PyTypeObject*) type_obj; + PyObject *mro = type->tp_mro; + Py_ssize_t i, nbases; + if (unlikely(!mro)) goto done; + + // avoid "unused" warning + (void) &__Pyx_GetBuiltinName; + + Py_INCREF(mro); + nbases = PyTuple_GET_SIZE(mro); + + // Skip over the type itself and 'object'. + assert(PyTuple_GET_ITEM(mro, 0) == type_obj); + for (i = 1; i < nbases-1; i++) { + PyObject *base, *dict, *meth; + base = PyTuple_GET_ITEM(mro, i); + dict = ((PyTypeObject *)base)->tp_dict; + meth = __Pyx_PyDict_GetItemStrWithError(dict, PYIDENT("__init_subclass__")); + if (unlikely(meth)) { + descrgetfunc f = Py_TYPE(meth)->tp_descr_get; + PyObject *res; + Py_INCREF(meth); + if (likely(f)) { + res = f(meth, NULL, type_obj); + Py_DECREF(meth); + if (unlikely(!res)) goto bad; + meth = res; + } + res = __Pyx_PyObject_FastCallDict(meth, NULL, 0, mkw); + Py_DECREF(meth); + if (unlikely(!res)) goto bad; + Py_DECREF(res); + goto done; + } else if (unlikely(PyErr_Occurred())) { + goto bad; + } + } + +done: + Py_XDECREF(mro); + return type_obj; + +bad: + Py_XDECREF(mro); + Py_DECREF(type_obj); + return NULL; + +// CYTHON_USE_TYPE_SLOTS && CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS +#else +// Generic fallback: "super(type_obj, type_obj).__init_subclass__(**mkw)", as used in CPython 3.8. + PyObject *super_type, *super, *func, *res; + +#if CYTHON_COMPILING_IN_PYPY && !defined(PySuper_Type) + super_type = __Pyx_GetBuiltinName(PYIDENT("super")); +#else + super_type = (PyObject*) &PySuper_Type; + // avoid "unused" warning + (void) &__Pyx_GetBuiltinName; +#endif + super = likely(super_type) ? __Pyx_PyObject_Call2Args(super_type, type_obj, type_obj) : NULL; +#if CYTHON_COMPILING_IN_PYPY && !defined(PySuper_Type) + Py_XDECREF(super_type); +#endif + if (unlikely(!super)) { + Py_CLEAR(type_obj); + goto done; + } + func = __Pyx_PyObject_GetAttrStrNoError(super, PYIDENT("__init_subclass__")); + Py_DECREF(super); + if (likely(!func)) { + if (unlikely(PyErr_Occurred())) + Py_CLEAR(type_obj); + goto done; + } + res = __Pyx_PyObject_FastCallDict(func, NULL, 0, mkw); + Py_DECREF(func); + if (unlikely(!res)) + Py_CLEAR(type_obj); + Py_XDECREF(res); +done: + return type_obj; +#endif +} + +// PY_VERSION_HEX < 0x030600A4 && CYTHON_PEP487_INIT_SUBCLASS +#endif + static PyObject *__Pyx_Py3ClassCreate(PyObject *metaclass, PyObject *name, PyObject *bases, PyObject *dict, PyObject *mkw, int calculate_metaclass, int allow_py2_metaclass) { - PyObject *result, *margs; + PyObject *result; PyObject *owned_metaclass = NULL; + PyObject *margs[4] = {NULL, name, bases, dict}; if (allow_py2_metaclass) { /* honour Python2 __metaclass__ for backward compatibility */ owned_metaclass = PyObject_GetItem(dict, PYIDENT("__metaclass__")); @@ -1008,14 +1340,28 @@ static PyObject *__Pyx_Py3ClassCreate(PyObject *metaclass, PyObject *name, PyObj return NULL; owned_metaclass = metaclass; } - margs = PyTuple_Pack(3, name, bases, dict); - if (unlikely(!margs)) { - result = NULL; - } else { - result = PyObject_Call(metaclass, margs, mkw); - Py_DECREF(margs); - } + result = __Pyx_PyObject_FastCallDict(metaclass, margs+1, 3 | __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET, +#if PY_VERSION_HEX < 0x030600A4 + // Before PEP-487, type(a,b,c) did not accept any keyword arguments, so guard at least against that case. + (metaclass == (PyObject*)&PyType_Type) ? NULL : mkw +#else + mkw +#endif + ); Py_XDECREF(owned_metaclass); + +#if PY_VERSION_HEX < 0x030600A4 && CYTHON_PEP487_INIT_SUBCLASS + if (likely(result) && likely(PyType_Check(result))) { + if (unlikely(__Pyx_SetNamesPEP487(result) < 0)) { + Py_CLEAR(result); + } else { + result = __Pyx_InitSubclassPEP487(result, mkw); + } + } +#else + // avoid "unused" warning + (void) &__Pyx_GetBuiltinName; +#endif return result; } @@ -1026,14 +1372,21 @@ static CYTHON_INLINE int __Pyx_TypeTest(PyObject *obj, PyTypeObject *type); /*pr /////////////// ExtTypeTest /////////////// static CYTHON_INLINE int __Pyx_TypeTest(PyObject *obj, PyTypeObject *type) { + __Pyx_TypeName obj_type_name; + __Pyx_TypeName type_name; if (unlikely(!type)) { PyErr_SetString(PyExc_SystemError, "Missing type object"); return 0; } if (likely(__Pyx_TypeCheck(obj, type))) return 1; - PyErr_Format(PyExc_TypeError, "Cannot convert %.200s to %.200s", - Py_TYPE(obj)->tp_name, type->tp_name); + obj_type_name = __Pyx_PyType_GetName(Py_TYPE(obj)); + type_name = __Pyx_PyType_GetName(type); + PyErr_Format(PyExc_TypeError, + "Cannot convert " __Pyx_FMT_TYPENAME " to " __Pyx_FMT_TYPENAME, + obj_type_name, type_name); + __Pyx_DECREF_TypeName(obj_type_name); + __Pyx_DECREF_TypeName(type_name); return 0; } @@ -1101,12 +1454,12 @@ static CYTHON_INLINE PyObject* __Pyx_PyBoolOrNull_FromLong(long b) { static PyObject *__Pyx_GetBuiltinName(PyObject *name); /*proto*/ /////////////// GetBuiltinName /////////////// -//@requires: PyObjectGetAttrStr +//@requires: PyObjectGetAttrStrNoError //@substitute: naming static PyObject *__Pyx_GetBuiltinName(PyObject *name) { - PyObject* result = __Pyx_PyObject_GetAttrStr($builtins_cname, name); - if (unlikely(!result)) { + PyObject* result = __Pyx_PyObject_GetAttrStrNoError($builtins_cname, name); + if (unlikely(!result) && !PyErr_Occurred()) { PyErr_Format(PyExc_NameError, #if PY_MAJOR_VERSION >= 3 "name '%U' is not defined", name); @@ -1123,29 +1476,27 @@ static PyObject *__Pyx_GetBuiltinName(PyObject *name) { static PyObject *__Pyx__GetNameInClass(PyObject *nmspace, PyObject *name); /*proto*/ /////////////// GetNameInClass /////////////// -//@requires: PyObjectGetAttrStr //@requires: GetModuleGlobalName -//@requires: Exceptions.c::PyThreadStateGet -//@requires: Exceptions.c::PyErrFetchRestore -//@requires: Exceptions.c::PyErrExceptionMatches - -static PyObject *__Pyx_GetGlobalNameAfterAttributeLookup(PyObject *name) { - PyObject *result; - __Pyx_PyThreadState_declare - __Pyx_PyThreadState_assign - if (unlikely(!__Pyx_PyErr_ExceptionMatches(PyExc_AttributeError))) - return NULL; - __Pyx_PyErr_Clear(); - __Pyx_GetModuleGlobalNameUncached(result, name); - return result; -} static PyObject *__Pyx__GetNameInClass(PyObject *nmspace, PyObject *name) { PyObject *result; - result = __Pyx_PyObject_GetAttrStr(nmspace, name); - if (!result) { - result = __Pyx_GetGlobalNameAfterAttributeLookup(name); + PyObject *dict; + assert(PyType_Check(nmspace)); +#if CYTHON_USE_TYPE_SLOTS + dict = ((PyTypeObject*)nmspace)->tp_dict; + Py_XINCREF(dict); +#else + dict = PyObject_GetAttr(nmspace, PYIDENT("__dict__")); +#endif + if (likely(dict)) { + result = PyObject_GetItem(dict, name); + Py_DECREF(dict); + if (result) { + return result; + } } + PyErr_Clear(); + __Pyx_GetModuleGlobalNameUncached(result, name); return result; } @@ -1163,6 +1514,30 @@ static PyObject *__Pyx__GetNameInClass(PyObject *nmspace, PyObject *name) { #define __Pyx_SetNameInClass(ns, name, value) PyObject_SetItem(ns, name, value) #endif +/////////////// SetNewInClass.proto /////////////// + +static int __Pyx_SetNewInClass(PyObject *ns, PyObject *name, PyObject *value); + +/////////////// SetNewInClass /////////////// +//@requires: SetNameInClass + +// Special-case setting __new__: if it's a Cython function, wrap it in a +// staticmethod. This is similar to what Python does for a Python function +// called __new__. +static int __Pyx_SetNewInClass(PyObject *ns, PyObject *name, PyObject *value) { +#ifdef __Pyx_CyFunction_USED + int ret; + if (__Pyx_CyFunction_Check(value)) { + PyObject *staticnew = PyStaticMethod_New(value); + if (unlikely(!staticnew)) return -1; + ret = __Pyx_SetNameInClass(ns, name, staticnew); + Py_DECREF(staticnew); + return ret; + } +#endif + return __Pyx_SetNameInClass(ns, name, value); +} + /////////////// GetModuleGlobalName.proto /////////////// //@requires: PyDictVersioning @@ -1210,6 +1585,14 @@ static CYTHON_INLINE PyObject *__Pyx__GetModuleGlobalName(PyObject *name) } else if (unlikely(PyErr_Occurred())) { return NULL; } +#elif CYTHON_COMPILING_IN_LIMITED_API + if (unlikely(!$module_cname)) { + return NULL; + } + result = PyObject_GetAttr($module_cname, name); + if (likely(result)) { + return result; + } #else result = PyDict_GetItem($moddict_cname, name); __PYX_UPDATE_DICT_CACHE($moddict_cname, result, *dict_cached_value, *dict_version) @@ -1247,16 +1630,31 @@ static CYTHON_INLINE PyObject *__Pyx_GetAttr(PyObject *o, PyObject *n) { return PyObject_GetAttr(o, n); } + /////////////// PyObjectLookupSpecial.proto /////////////// + +#if CYTHON_USE_PYTYPE_LOOKUP && CYTHON_USE_TYPE_SLOTS +#define __Pyx_PyObject_LookupSpecialNoError(obj, attr_name) __Pyx__PyObject_LookupSpecial(obj, attr_name, 0) +#define __Pyx_PyObject_LookupSpecial(obj, attr_name) __Pyx__PyObject_LookupSpecial(obj, attr_name, 1) + +static CYTHON_INLINE PyObject* __Pyx__PyObject_LookupSpecial(PyObject* obj, PyObject* attr_name, int with_error); /*proto*/ + +#else +#define __Pyx_PyObject_LookupSpecialNoError(o,n) __Pyx_PyObject_GetAttrStrNoError(o,n) +#define __Pyx_PyObject_LookupSpecial(o,n) __Pyx_PyObject_GetAttrStr(o,n) +#endif + +/////////////// PyObjectLookupSpecial /////////////// //@requires: PyObjectGetAttrStr +//@requires: PyObjectGetAttrStrNoError #if CYTHON_USE_PYTYPE_LOOKUP && CYTHON_USE_TYPE_SLOTS -static CYTHON_INLINE PyObject* __Pyx_PyObject_LookupSpecial(PyObject* obj, PyObject* attr_name) { +static CYTHON_INLINE PyObject* __Pyx__PyObject_LookupSpecial(PyObject* obj, PyObject* attr_name, int with_error) { PyObject *res; PyTypeObject *tp = Py_TYPE(obj); #if PY_MAJOR_VERSION < 3 if (unlikely(PyInstance_Check(obj))) - return __Pyx_PyObject_GetAttrStr(obj, attr_name); + return with_error ? __Pyx_PyObject_GetAttrStr(obj, attr_name) : __Pyx_PyObject_GetAttrStrNoError(obj, attr_name); #endif // adapted from CPython's special_lookup() in ceval.c res = _PyType_Lookup(tp, attr_name); @@ -1267,13 +1665,11 @@ static CYTHON_INLINE PyObject* __Pyx_PyObject_LookupSpecial(PyObject* obj, PyObj } else { res = f(res, obj, (PyObject *)tp); } - } else { + } else if (with_error) { PyErr_SetObject(PyExc_AttributeError, attr_name); } return res; } -#else -#define __Pyx_PyObject_LookupSpecial(o,n) __Pyx_PyObject_GetAttrStr(o,n) #endif @@ -1292,19 +1688,21 @@ static CYTHON_INLINE PyObject* __Pyx_PyObject_GenericGetAttrNoDict(PyObject* obj #if CYTHON_USE_TYPE_SLOTS && CYTHON_USE_PYTYPE_LOOKUP && PY_VERSION_HEX < 0x03070000 static PyObject *__Pyx_RaiseGenericGetAttributeError(PyTypeObject *tp, PyObject *attr_name) { + __Pyx_TypeName type_name = __Pyx_PyType_GetName(tp); PyErr_Format(PyExc_AttributeError, #if PY_MAJOR_VERSION >= 3 - "'%.50s' object has no attribute '%U'", - tp->tp_name, attr_name); + "'" __Pyx_FMT_TYPENAME "' object has no attribute '%U'", + type_name, attr_name); #else - "'%.50s' object has no attribute '%.400s'", - tp->tp_name, PyString_AS_STRING(attr_name)); + "'" __Pyx_FMT_TYPENAME "' object has no attribute '%.400s'", + type_name, PyString_AS_STRING(attr_name)); #endif + __Pyx_DECREF_TypeName(type_name); return NULL; } static CYTHON_INLINE PyObject* __Pyx_PyObject_GenericGetAttrNoDict(PyObject* obj, PyObject* attr_name) { - // Copied and adapted from _PyObject_GenericGetAttrWithDict() in CPython 2.6/3.7. + // Copied and adapted from _PyObject_GenericGetAttrWithDict() in CPython 3.6/3.7. // To be used in the "tp_getattro" slot of extension types that have no instance dict and cannot be subclassed. PyObject *descr; PyTypeObject *tp = Py_TYPE(obj); @@ -1456,6 +1854,7 @@ static int __Pyx_PyObject_GetMethod(PyObject *obj, PyObject *name, PyObject **me static int __Pyx_PyObject_GetMethod(PyObject *obj, PyObject *name, PyObject **method) { PyObject *attr; #if CYTHON_UNPACK_METHODS && CYTHON_COMPILING_IN_CPYTHON && CYTHON_USE_PYTYPE_LOOKUP + __Pyx_TypeName type_name; // Copied from _PyObject_GetMethod() in CPython 3.7 PyTypeObject *tp = Py_TYPE(obj); PyObject *descr; @@ -1476,12 +1875,14 @@ static int __Pyx_PyObject_GetMethod(PyObject *obj, PyObject *name, PyObject **me descr = _PyType_Lookup(tp, name); if (likely(descr != NULL)) { Py_INCREF(descr); +#if defined(Py_TPFLAGS_METHOD_DESCRIPTOR) && Py_TPFLAGS_METHOD_DESCRIPTOR + if (__Pyx_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)) +#elif PY_MAJOR_VERSION >= 3 // Repeating the condition below accommodates for MSVC's inability to test macros inside of macro expansions. -#if PY_MAJOR_VERSION >= 3 #ifdef __Pyx_CyFunction_USED - if (likely(PyFunction_Check(descr) || (Py_TYPE(descr) == &PyMethodDescr_Type) || __Pyx_CyFunction_Check(descr))) + if (likely(PyFunction_Check(descr) || __Pyx_IS_TYPE(descr, &PyMethodDescr_Type) || __Pyx_CyFunction_Check(descr))) #else - if (likely(PyFunction_Check(descr) || (Py_TYPE(descr) == &PyMethodDescr_Type))) + if (likely(PyFunction_Check(descr) || __Pyx_IS_TYPE(descr, &PyMethodDescr_Type))) #endif #else // "PyMethodDescr_Type" is not part of the C-API in Py2. @@ -1527,19 +1928,21 @@ static int __Pyx_PyObject_GetMethod(PyObject *obj, PyObject *name, PyObject **me goto try_unpack; } - if (descr != NULL) { + if (likely(descr != NULL)) { *method = descr; return 0; } + type_name = __Pyx_PyType_GetName(tp); PyErr_Format(PyExc_AttributeError, #if PY_MAJOR_VERSION >= 3 - "'%.50s' object has no attribute '%U'", - tp->tp_name, name); + "'" __Pyx_FMT_TYPENAME "' object has no attribute '%U'", + type_name, name); #else - "'%.50s' object has no attribute '%.400s'", - tp->tp_name, PyString_AS_STRING(name)); + "'" __Pyx_FMT_TYPENAME "' object has no attribute '%.400s'", + type_name, PyString_AS_STRING(name)); #endif + __Pyx_DECREF_TypeName(type_name); return 0; // Generic fallback implementation using normal attribute lookup. @@ -1587,7 +1990,7 @@ static int __Pyx_TryUnpackUnboundCMethod(__Pyx_CachedCFunction* target) { target->method = method; #if CYTHON_COMPILING_IN_CPYTHON #if PY_MAJOR_VERSION >= 3 - // method dscriptor type isn't exported in Py2.x, cannot easily check the type there + // method descriptor type isn't exported in Py2.x, cannot easily check the type there if (likely(__Pyx_TypeCheck(method, &PyMethodDescr_Type))) #endif { @@ -1667,13 +2070,13 @@ static CYTHON_INLINE PyObject* __Pyx_CallUnboundCMethod1(__Pyx_CachedCFunction* // Not using #ifdefs for PY_VERSION_HEX to avoid C compiler warnings about unused functions. if (flag == METH_O) { return (*(cfunc->func))(self, arg); - } else if (PY_VERSION_HEX >= 0x030600B1 && flag == METH_FASTCALL) { - if (PY_VERSION_HEX >= 0x030700A0) { + } else if ((PY_VERSION_HEX >= 0x030600B1) && flag == METH_FASTCALL) { + if ((PY_VERSION_HEX >= 0x030700A0)) { return (*(__Pyx_PyCFunctionFast)(void*)(PyCFunction)cfunc->func)(self, &arg, 1); } else { return (*(__Pyx_PyCFunctionFastWithKeywords)(void*)(PyCFunction)cfunc->func)(self, &arg, 1, NULL); } - } else if (PY_VERSION_HEX >= 0x030700A0 && flag == (METH_FASTCALL | METH_KEYWORDS)) { + } else if ((PY_VERSION_HEX >= 0x030700A0) && flag == (METH_FASTCALL | METH_KEYWORDS)) { return (*(__Pyx_PyCFunctionFastWithKeywords)(void*)(PyCFunction)cfunc->func)(self, &arg, 1, NULL); } } @@ -1785,6 +2188,104 @@ bad: } +/////////////// PyObjectFastCall.proto /////////////// + +#define __Pyx_PyObject_FastCall(func, args, nargs) __Pyx_PyObject_FastCallDict(func, args, nargs, NULL) +static CYTHON_INLINE PyObject* __Pyx_PyObject_FastCallDict(PyObject *func, PyObject **args, size_t nargs, PyObject *kwargs); /*proto*/ + +/////////////// PyObjectFastCall /////////////// +//@requires: PyObjectCall +//@requires: PyFunctionFastCall +//@requires: PyObjectCallMethO +//@substitute: naming + +static PyObject* __Pyx_PyObject_FastCall_fallback(PyObject *func, PyObject **args, size_t nargs, PyObject *kwargs) { + PyObject *argstuple; + PyObject *result; + size_t i; + + argstuple = PyTuple_New(nargs); + if (unlikely(!argstuple)) return NULL; + for (i = 0; i < nargs; i++) { + Py_INCREF(args[i]); + PyTuple_SET_ITEM(argstuple, (Py_ssize_t)i, args[i]); + } + result = __Pyx_PyObject_Call(func, argstuple, kwargs); + Py_DECREF(argstuple); + return result; +} + +static CYTHON_INLINE PyObject* __Pyx_PyObject_FastCallDict(PyObject *func, PyObject **args, size_t _nargs, PyObject *kwargs) { + // Special fast paths for 0 and 1 arguments + // NOTE: in many cases, this is called with a constant value for nargs + // which is known at compile-time. So the branches below will typically + // be optimized away. + Py_ssize_t nargs = __Pyx_PyVectorcall_NARGS(_nargs); +#if CYTHON_COMPILING_IN_CPYTHON + if (nargs == 0 && kwargs == NULL) { +#ifdef __Pyx_CyFunction_USED + if (__Pyx_IsCyOrPyCFunction(func)) +#else + if (PyCFunction_Check(func)) +#endif + { + if (likely(PyCFunction_GET_FLAGS(func) & METH_NOARGS)) { + return __Pyx_PyObject_CallMethO(func, NULL); + } + } + } + else if (nargs == 1 && kwargs == NULL) { + if (PyCFunction_Check(func)) + { + if (likely(PyCFunction_GET_FLAGS(func) & METH_O)) { + return __Pyx_PyObject_CallMethO(func, args[0]); + } + } + } +#endif + + #if PY_VERSION_HEX < 0x030800B1 + #if CYTHON_FAST_PYCCALL + if (PyCFunction_Check(func)) { + if (kwargs) { + return _PyCFunction_FastCallDict(func, args, nargs, kwargs); + } else { + return _PyCFunction_FastCallKeywords(func, args, nargs, NULL); + } + } + #if PY_VERSION_HEX >= 0x030700A1 + if (!kwargs && __Pyx_IS_TYPE(func, &PyMethodDescr_Type)) { + return _PyMethodDescr_FastCallKeywords(func, args, nargs, NULL); + } + #endif + #endif + #if CYTHON_FAST_PYCALL + if (PyFunction_Check(func)) { + return __Pyx_PyFunction_FastCallDict(func, args, nargs, kwargs); + } + #endif + #endif + + #if CYTHON_VECTORCALL + vectorcallfunc f = _PyVectorcall_Function(func); + if (f) { + return f(func, args, (size_t)nargs, kwargs); + } + #elif defined(__Pyx_CyFunction_USED) && CYTHON_BACKPORT_VECTORCALL + // exclude fused functions for now + if (__Pyx_CyFunction_CheckExact(func)) { + __pyx_vectorcallfunc f = __Pyx_CyFunction_func_vectorcall(func); + if (f) return f(func, args, (size_t)nargs, kwargs); + } + #endif + + if (nargs == 0) { + return __Pyx_PyObject_Call(func, $empty_tuple, kwargs); + } + return __Pyx_PyObject_FastCall_fallback(func, args, (size_t)nargs, kwargs); +} + + /////////////// PyObjectCallMethod0.proto /////////////// static PyObject* __Pyx_PyObject_CallMethod0(PyObject* obj, PyObject* method_name); /*proto*/ @@ -1839,59 +2340,6 @@ static PyObject* __Pyx_PyObject_CallMethod1(PyObject* obj, PyObject* method_name } -/////////////// PyObjectCallMethod2.proto /////////////// - -static PyObject* __Pyx_PyObject_CallMethod2(PyObject* obj, PyObject* method_name, PyObject* arg1, PyObject* arg2); /*proto*/ - -/////////////// PyObjectCallMethod2 /////////////// -//@requires: PyObjectCall -//@requires: PyFunctionFastCall -//@requires: PyCFunctionFastCall -//@requires: PyObjectCall2Args - -static PyObject* __Pyx_PyObject_Call3Args(PyObject* function, PyObject* arg1, PyObject* arg2, PyObject* arg3) { - #if CYTHON_FAST_PYCALL - if (PyFunction_Check(function)) { - PyObject *args[3] = {arg1, arg2, arg3}; - return __Pyx_PyFunction_FastCall(function, args, 3); - } - #endif - #if CYTHON_FAST_PYCCALL - if (__Pyx_PyFastCFunction_Check(function)) { - PyObject *args[3] = {arg1, arg2, arg3}; - return __Pyx_PyFunction_FastCall(function, args, 3); - } - #endif - - args = PyTuple_New(3); - if (unlikely(!args)) goto done; - Py_INCREF(arg1); - PyTuple_SET_ITEM(args, 0, arg1); - Py_INCREF(arg2); - PyTuple_SET_ITEM(args, 1, arg2); - Py_INCREF(arg3); - PyTuple_SET_ITEM(args, 2, arg3); - - result = __Pyx_PyObject_Call(function, args, NULL); - Py_DECREF(args); - return result; -} - -static PyObject* __Pyx_PyObject_CallMethod2(PyObject* obj, PyObject* method_name, PyObject* arg1, PyObject* arg2) { - PyObject *args, *method = NULL, *result = NULL; - int is_method = __Pyx_PyObject_GetMethod(obj, method_name, &method); - if (likely(is_method)) { - result = __Pyx_PyObject_Call3Args(method, obj, arg1, arg2); - Py_DECREF(method); - return result; - } - if (unlikely(!method)) return NULL; - result = __Pyx_PyObject_Call2Args(method, arg1, arg2); - Py_DECREF(method); - return result; -} - - /////////////// tp_new.proto /////////////// #define __Pyx_tp_new(type_obj, args) __Pyx_tp_new_kwargs(type_obj, args, NULL) @@ -1913,7 +2361,7 @@ static CYTHON_INLINE PyObject* __Pyx_PyObject_Call(PyObject *func, PyObject *arg #if CYTHON_COMPILING_IN_CPYTHON static CYTHON_INLINE PyObject* __Pyx_PyObject_Call(PyObject *func, PyObject *arg, PyObject *kw) { PyObject *result; - ternaryfunc call = func->ob_type->tp_call; + ternaryfunc call = Py_TYPE(func)->tp_call; if (unlikely(!call)) return PyObject_Call(func, arg, kw); @@ -1963,14 +2411,12 @@ static CYTHON_INLINE PyObject* __Pyx_PyObject_CallMethO(PyObject *func, PyObject /////////////// PyFunctionFastCall.proto /////////////// #if CYTHON_FAST_PYCALL + +#if !CYTHON_VECTORCALL #define __Pyx_PyFunction_FastCall(func, args, nargs) \ __Pyx_PyFunction_FastCallDict((func), (args), (nargs), NULL) -// let's assume that the non-public C-API function might still change during the 3.6 beta phase -#if 1 || PY_VERSION_HEX < 0x030600B1 static PyObject *__Pyx_PyFunction_FastCallDict(PyObject *func, PyObject **args, Py_ssize_t nargs, PyObject *kwargs); -#else -#define __Pyx_PyFunction_FastCallDict(func, args, nargs, kwargs) _PyFunction_FastCallDict(func, args, nargs, kwargs) #endif // Backport from Python 3 @@ -1982,7 +2428,7 @@ static PyObject *__Pyx_PyFunction_FastCallDict(PyObject *func, PyObject **args, // ((char *)(foo) \ // + Py_BUILD_ASSERT_EXPR(offsetof(struct foo, string) == 0)) // -// Written by Rusty Russell, public domain, http://ccodearchive.net/ +// Written by Rusty Russell, public domain, https://ccodearchive.net/ #define __Pyx_BUILD_ASSERT_EXPR(cond) \ (sizeof(char [1 - 2*!(cond)]) - 1) @@ -2011,8 +2457,7 @@ static PyObject *__Pyx_PyFunction_FastCallDict(PyObject *func, PyObject **args, /////////////// PyFunctionFastCall /////////////// // copied from CPython 3.6 ceval.c -#if CYTHON_FAST_PYCALL - +#if CYTHON_FAST_PYCALL && !CYTHON_VECTORCALL static PyObject* __Pyx_PyFunction_FastCallNoKw(PyCodeObject *co, PyObject **args, Py_ssize_t na, PyObject *globals) { PyFrameObject *f; @@ -2048,7 +2493,6 @@ static PyObject* __Pyx_PyFunction_FastCallNoKw(PyCodeObject *co, PyObject **args } -#if 1 || PY_VERSION_HEX < 0x030600B1 static PyObject *__Pyx_PyFunction_FastCallDict(PyObject *func, PyObject **args, Py_ssize_t nargs, PyObject *kwargs) { PyCodeObject *co = (PyCodeObject *)PyFunction_GET_CODE(func); PyObject *globals = PyFunction_GET_GLOBALS(func); @@ -2069,7 +2513,7 @@ static PyObject *__Pyx_PyFunction_FastCallDict(PyObject *func, PyObject **args, assert(kwargs == NULL || PyDict_Check(kwargs)); nk = kwargs ? PyDict_Size(kwargs) : 0; - if (Py_EnterRecursiveCall((char*)" while calling a Python object")) { + if (unlikely(Py_EnterRecursiveCall((char*)" while calling a Python object"))) { return NULL; } @@ -2158,83 +2602,19 @@ done: Py_LeaveRecursiveCall(); return result; } -#endif /* CPython < 3.6 */ -#endif /* CYTHON_FAST_PYCALL */ - - -/////////////// PyCFunctionFastCall.proto /////////////// - -#if CYTHON_FAST_PYCCALL -static CYTHON_INLINE PyObject *__Pyx_PyCFunction_FastCall(PyObject *func, PyObject **args, Py_ssize_t nargs); -#else -#define __Pyx_PyCFunction_FastCall(func, args, nargs) (assert(0), NULL) -#endif - -/////////////// PyCFunctionFastCall /////////////// - -#if CYTHON_FAST_PYCCALL -static CYTHON_INLINE PyObject * __Pyx_PyCFunction_FastCall(PyObject *func_obj, PyObject **args, Py_ssize_t nargs) { - PyCFunctionObject *func = (PyCFunctionObject*)func_obj; - PyCFunction meth = PyCFunction_GET_FUNCTION(func); - PyObject *self = PyCFunction_GET_SELF(func); - int flags = PyCFunction_GET_FLAGS(func); - - assert(PyCFunction_Check(func)); - assert(METH_FASTCALL == (flags & ~(METH_CLASS | METH_STATIC | METH_COEXIST | METH_KEYWORDS | METH_STACKLESS))); - assert(nargs >= 0); - assert(nargs == 0 || args != NULL); - - /* _PyCFunction_FastCallDict() must not be called with an exception set, - because it may clear it (directly or indirectly) and so the - caller loses its exception */ - assert(!PyErr_Occurred()); - - if ((PY_VERSION_HEX < 0x030700A0) || unlikely(flags & METH_KEYWORDS)) { - return (*((__Pyx_PyCFunctionFastWithKeywords)(void*)meth)) (self, args, nargs, NULL); - } else { - return (*((__Pyx_PyCFunctionFast)(void*)meth)) (self, args, nargs); - } -} -#endif /* CYTHON_FAST_PYCCALL */ +#endif /* CYTHON_FAST_PYCALL && !CYTHON_VECTORCALL */ /////////////// PyObjectCall2Args.proto /////////////// -static CYTHON_UNUSED PyObject* __Pyx_PyObject_Call2Args(PyObject* function, PyObject* arg1, PyObject* arg2); /*proto*/ +static CYTHON_INLINE PyObject* __Pyx_PyObject_Call2Args(PyObject* function, PyObject* arg1, PyObject* arg2); /*proto*/ /////////////// PyObjectCall2Args /////////////// -//@requires: PyObjectCall -//@requires: PyFunctionFastCall -//@requires: PyCFunctionFastCall - -static CYTHON_UNUSED PyObject* __Pyx_PyObject_Call2Args(PyObject* function, PyObject* arg1, PyObject* arg2) { - PyObject *args, *result = NULL; - #if CYTHON_FAST_PYCALL - if (PyFunction_Check(function)) { - PyObject *args[2] = {arg1, arg2}; - return __Pyx_PyFunction_FastCall(function, args, 2); - } - #endif - #if CYTHON_FAST_PYCCALL - if (__Pyx_PyFastCFunction_Check(function)) { - PyObject *args[2] = {arg1, arg2}; - return __Pyx_PyCFunction_FastCall(function, args, 2); - } - #endif - - args = PyTuple_New(2); - if (unlikely(!args)) goto done; - Py_INCREF(arg1); - PyTuple_SET_ITEM(args, 0, arg1); - Py_INCREF(arg2); - PyTuple_SET_ITEM(args, 1, arg2); +//@requires: PyObjectFastCall - Py_INCREF(function); - result = __Pyx_PyObject_Call(function, args, NULL); - Py_DECREF(args); - Py_DECREF(function); -done: - return result; +static CYTHON_INLINE PyObject* __Pyx_PyObject_Call2Args(PyObject* function, PyObject* arg1, PyObject* arg2) { + PyObject *args[3] = {NULL, arg1, arg2}; + return __Pyx_PyObject_FastCall(function, args+1, 2 | __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET); } @@ -2243,88 +2623,97 @@ done: static CYTHON_INLINE PyObject* __Pyx_PyObject_CallOneArg(PyObject *func, PyObject *arg); /*proto*/ /////////////// PyObjectCallOneArg /////////////// -//@requires: PyObjectCallMethO -//@requires: PyObjectCall -//@requires: PyFunctionFastCall -//@requires: PyCFunctionFastCall - -#if CYTHON_COMPILING_IN_CPYTHON -static PyObject* __Pyx__PyObject_CallOneArg(PyObject *func, PyObject *arg) { - PyObject *result; - PyObject *args = PyTuple_New(1); - if (unlikely(!args)) return NULL; - Py_INCREF(arg); - PyTuple_SET_ITEM(args, 0, arg); - result = __Pyx_PyObject_Call(func, args, NULL); - Py_DECREF(args); - return result; -} +//@requires: PyObjectFastCall static CYTHON_INLINE PyObject* __Pyx_PyObject_CallOneArg(PyObject *func, PyObject *arg) { -#if CYTHON_FAST_PYCALL - if (PyFunction_Check(func)) { - return __Pyx_PyFunction_FastCall(func, &arg, 1); - } -#endif - if (likely(PyCFunction_Check(func))) { - if (likely(PyCFunction_GET_FLAGS(func) & METH_O)) { - // fast and simple case that we are optimising for - return __Pyx_PyObject_CallMethO(func, arg); -#if CYTHON_FAST_PYCCALL - } else if (__Pyx_PyFastCFunction_Check(func)) { - return __Pyx_PyCFunction_FastCall(func, &arg, 1); -#endif - } - } - return __Pyx__PyObject_CallOneArg(func, arg); + PyObject *args[2] = {NULL, arg}; + return __Pyx_PyObject_FastCall(func, args+1, 1 | __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET); } -#else -static CYTHON_INLINE PyObject* __Pyx_PyObject_CallOneArg(PyObject *func, PyObject *arg) { - PyObject *result; - PyObject *args = PyTuple_Pack(1, arg); - if (unlikely(!args)) return NULL; - result = __Pyx_PyObject_Call(func, args, NULL); - Py_DECREF(args); - return result; -} -#endif /////////////// PyObjectCallNoArg.proto /////////////// -//@requires: PyObjectCall -//@substitute: naming -#if CYTHON_COMPILING_IN_CPYTHON static CYTHON_INLINE PyObject* __Pyx_PyObject_CallNoArg(PyObject *func); /*proto*/ -#else -#define __Pyx_PyObject_CallNoArg(func) __Pyx_PyObject_Call(func, $empty_tuple, NULL) -#endif /////////////// PyObjectCallNoArg /////////////// -//@requires: PyObjectCallMethO -//@requires: PyObjectCall -//@requires: PyFunctionFastCall -//@substitute: naming +//@requires: PyObjectFastCall -#if CYTHON_COMPILING_IN_CPYTHON static CYTHON_INLINE PyObject* __Pyx_PyObject_CallNoArg(PyObject *func) { -#if CYTHON_FAST_PYCALL - if (PyFunction_Check(func)) { - return __Pyx_PyFunction_FastCall(func, NULL, 0); - } -#endif -#ifdef __Pyx_CyFunction_USED - if (likely(PyCFunction_Check(func) || __Pyx_CyFunction_Check(func))) -#else - if (likely(PyCFunction_Check(func))) + PyObject *arg = NULL; + return __Pyx_PyObject_FastCall(func, (&arg)+1, 0 | __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET); +} + + +/////////////// PyVectorcallFastCallDict.proto /////////////// + +#if CYTHON_METH_FASTCALL +static CYTHON_INLINE PyObject *__Pyx_PyVectorcall_FastCallDict(PyObject *func, __pyx_vectorcallfunc vc, PyObject *const *args, size_t nargs, PyObject *kw); #endif - { - if (likely(PyCFunction_GET_FLAGS(func) & METH_NOARGS)) { - // fast and simple case that we are optimising for - return __Pyx_PyObject_CallMethO(func, NULL); - } + +/////////////// PyVectorcallFastCallDict /////////////// + +#if CYTHON_METH_FASTCALL +// Slow path when kw is non-empty +static PyObject *__Pyx_PyVectorcall_FastCallDict_kw(PyObject *func, __pyx_vectorcallfunc vc, PyObject *const *args, size_t nargs, PyObject *kw) +{ + // Code based on _PyObject_FastCallDict() and _PyStack_UnpackDict() from CPython + PyObject *res = NULL; + PyObject *kwnames; + PyObject **newargs; + PyObject **kwvalues; + Py_ssize_t i, pos; + size_t j; + PyObject *key, *value; + unsigned long keys_are_strings; + Py_ssize_t nkw = PyDict_GET_SIZE(kw); + + // Copy positional arguments + newargs = (PyObject **)PyMem_Malloc((nargs + (size_t)nkw) * sizeof(args[0])); + if (unlikely(newargs == NULL)) { + PyErr_NoMemory(); + return NULL; + } + for (j = 0; j < nargs; j++) newargs[j] = args[j]; + + // Copy keyword arguments + kwnames = PyTuple_New(nkw); + if (unlikely(kwnames == NULL)) { + PyMem_Free(newargs); + return NULL; } - return __Pyx_PyObject_Call(func, $empty_tuple, NULL); + kwvalues = newargs + nargs; + pos = i = 0; + keys_are_strings = Py_TPFLAGS_UNICODE_SUBCLASS; + while (PyDict_Next(kw, &pos, &key, &value)) { + keys_are_strings &= Py_TYPE(key)->tp_flags; + Py_INCREF(key); + Py_INCREF(value); + PyTuple_SET_ITEM(kwnames, i, key); + kwvalues[i] = value; + i++; + } + if (unlikely(!keys_are_strings)) { + PyErr_SetString(PyExc_TypeError, "keywords must be strings"); + goto cleanup; + } + + // The actual call + res = vc(func, newargs, nargs, kwnames); + +cleanup: + Py_DECREF(kwnames); + for (i = 0; i < nkw; i++) + Py_DECREF(kwvalues[i]); + PyMem_Free(newargs); + return res; +} + +static CYTHON_INLINE PyObject *__Pyx_PyVectorcall_FastCallDict(PyObject *func, __pyx_vectorcallfunc vc, PyObject *const *args, size_t nargs, PyObject *kw) +{ + if (likely(kw == NULL) || PyDict_GET_SIZE(kw) == 0) { + return vc(func, args, nargs, NULL); + } + return __Pyx_PyVectorcall_FastCallDict_kw(func, vc, args, nargs, kw); } #endif @@ -2341,10 +2730,9 @@ static PyObject* __Pyx_PyNumber_InPlaceMatrixMultiply(PyObject* x, PyObject* y); #endif /////////////// MatrixMultiply /////////////// -//@requires: PyObjectGetAttrStr +//@requires: PyObjectGetAttrStrNoError //@requires: PyObjectCallOneArg -//@requires: PyFunctionFastCall -//@requires: PyCFunctionFastCall +//@requires: PyObjectCall2Args #if PY_VERSION_HEX < 0x03050000 static PyObject* __Pyx_PyObject_CallMatrixMethod(PyObject* method, PyObject* arg) { @@ -2354,34 +2742,9 @@ static PyObject* __Pyx_PyObject_CallMatrixMethod(PyObject* method, PyObject* arg if (likely(PyMethod_Check(method))) { PyObject *self = PyMethod_GET_SELF(method); if (likely(self)) { - PyObject *args; PyObject *function = PyMethod_GET_FUNCTION(method); - #if CYTHON_FAST_PYCALL - if (PyFunction_Check(function)) { - PyObject *args[2] = {self, arg}; - result = __Pyx_PyFunction_FastCall(function, args, 2); - goto done; - } - #endif - #if CYTHON_FAST_PYCCALL - if (__Pyx_PyFastCFunction_Check(function)) { - PyObject *args[2] = {self, arg}; - result = __Pyx_PyCFunction_FastCall(function, args, 2); - goto done; - } - #endif - args = PyTuple_New(2); - if (unlikely(!args)) goto done; - Py_INCREF(self); - PyTuple_SET_ITEM(args, 0, self); - Py_INCREF(arg); - PyTuple_SET_ITEM(args, 1, arg); - Py_INCREF(function); - Py_DECREF(method); method = NULL; - result = __Pyx_PyObject_Call(function, args, NULL); - Py_DECREF(args); - Py_DECREF(function); - return result; + result = __Pyx_PyObject_Call2Args(function, self, arg); + goto done; } } #endif @@ -2391,21 +2754,21 @@ done: return result; } -#define __Pyx_TryMatrixMethod(x, y, py_method_name) { \ - PyObject *func = __Pyx_PyObject_GetAttrStr(x, py_method_name); \ +#define __Pyx_TryMatrixMethod(x, y, py_method_name) { \ + PyObject *func = __Pyx_PyObject_GetAttrStrNoError(x, py_method_name); \ if (func) { \ PyObject *result = __Pyx_PyObject_CallMatrixMethod(func, y); \ if (result != Py_NotImplemented) \ return result; \ Py_DECREF(result); \ - } else { \ - if (!PyErr_ExceptionMatches(PyExc_AttributeError)) \ - return NULL; \ - PyErr_Clear(); \ + } else if (unlikely(PyErr_Occurred())) { \ + return NULL; \ } \ } static PyObject* __Pyx__PyNumber_MatrixMultiply(PyObject* x, PyObject* y, const char* op_name) { + __Pyx_TypeName x_type_name; + __Pyx_TypeName y_type_name; int right_is_subtype = PyObject_IsSubclass((PyObject*)Py_TYPE(y), (PyObject*)Py_TYPE(x)); if (unlikely(right_is_subtype == -1)) return NULL; @@ -2418,11 +2781,13 @@ static PyObject* __Pyx__PyNumber_MatrixMultiply(PyObject* x, PyObject* y, const if (!right_is_subtype) { __Pyx_TryMatrixMethod(y, x, PYIDENT("__rmatmul__")) } + x_type_name = __Pyx_PyType_GetName(Py_TYPE(x)); + y_type_name = __Pyx_PyType_GetName(Py_TYPE(y)); PyErr_Format(PyExc_TypeError, - "unsupported operand type(s) for %.2s: '%.100s' and '%.100s'", - op_name, - Py_TYPE(x)->tp_name, - Py_TYPE(y)->tp_name); + "unsupported operand type(s) for %.2s: '" __Pyx_FMT_TYPENAME "' and '" + __Pyx_FMT_TYPENAME "'", op_name, x_type_name, y_type_name); + __Pyx_DECREF_TypeName(x_type_name); + __Pyx_DECREF_TypeName(y_type_name); return NULL; } @@ -2493,3 +2858,184 @@ static CYTHON_INLINE int __Pyx_object_dict_version_matches(PyObject* obj, PY_UIN return obj_dict_version == __Pyx_get_object_dict_version(obj); } #endif + + +/////////////// PyMethodNew.proto /////////////// + +#if PY_MAJOR_VERSION >= 3 +// This should be an actual function (not a macro), such that we can put it +// directly in a tp_descr_get slot. +static PyObject *__Pyx_PyMethod_New(PyObject *func, PyObject *self, CYTHON_UNUSED PyObject *typ) { + if (!self) + return __Pyx_NewRef(func); + return PyMethod_New(func, self); +} +#else + #define __Pyx_PyMethod_New PyMethod_New +#endif + +/////////////// UnicodeConcatInPlace.proto //////////////// + +# if CYTHON_COMPILING_IN_CPYTHON && PY_MAJOR_VERSION >= 3 +// __Pyx_PyUnicode_ConcatInPlace may modify the first argument 'left' +// However, unlike `PyUnicode_Append` it will never NULL it. +// It behaves like a regular function - returns a new reference and NULL on error + #if CYTHON_REFNANNY + #define __Pyx_PyUnicode_ConcatInPlace(left, right) __Pyx_PyUnicode_ConcatInPlaceImpl(&left, right, __pyx_refnanny) + #else + #define __Pyx_PyUnicode_ConcatInPlace(left, right) __Pyx_PyUnicode_ConcatInPlaceImpl(&left, right) + #endif + // __Pyx_PyUnicode_ConcatInPlace is slightly odd because it has the potential to modify the input + // argument (but only in cases where no user should notice). Therefore, it needs to keep Cython's + // refnanny informed. + static CYTHON_INLINE PyObject *__Pyx_PyUnicode_ConcatInPlaceImpl(PyObject **p_left, PyObject *right + #if CYTHON_REFNANNY + , void* __pyx_refnanny + #endif + ); /* proto */ +#else +#define __Pyx_PyUnicode_ConcatInPlace __Pyx_PyUnicode_Concat +#endif +#define __Pyx_PyUnicode_ConcatInPlaceSafe(left, right) ((unlikely((left) == Py_None) || unlikely((right) == Py_None)) ? \ + PyNumber_InPlaceAdd(left, right) : __Pyx_PyUnicode_ConcatInPlace(left, right)) + +/////////////// UnicodeConcatInPlace //////////////// +//@substitute: naming + +# if CYTHON_COMPILING_IN_CPYTHON && PY_MAJOR_VERSION >= 3 +// copied directly from unicode_object.c "unicode_modifiable +// removing _PyUnicode_HASH since it's a macro we don't have +// - this is OK because trying PyUnicode_Resize on a non-modifyable +// object will still work, it just won't happen in place +static int +__Pyx_unicode_modifiable(PyObject *unicode) +{ + if (Py_REFCNT(unicode) != 1) + return 0; + if (!PyUnicode_CheckExact(unicode)) + return 0; + if (PyUnicode_CHECK_INTERNED(unicode)) + return 0; + return 1; +} + +static CYTHON_INLINE PyObject *__Pyx_PyUnicode_ConcatInPlaceImpl(PyObject **p_left, PyObject *right + #if CYTHON_REFNANNY + , void* __pyx_refnanny + #endif + ) { + // heavily based on PyUnicode_Append + PyObject *left = *p_left; + Py_ssize_t left_len, right_len, new_len; + + if (unlikely(PyUnicode_READY(left) == -1)) + return NULL; + if (unlikely(PyUnicode_READY(right) == -1)) + return NULL; + + // Shortcuts + left_len = PyUnicode_GET_LENGTH(left); + if (left_len == 0) { + Py_INCREF(right); + return right; + } + right_len = PyUnicode_GET_LENGTH(right); + if (right_len == 0) { + Py_INCREF(left); + return left; + } + if (unlikely(left_len > PY_SSIZE_T_MAX - right_len)) { + PyErr_SetString(PyExc_OverflowError, + "strings are too large to concat"); + return NULL; + } + new_len = left_len + right_len; + + if (__Pyx_unicode_modifiable(left) + && PyUnicode_CheckExact(right) + && PyUnicode_KIND(right) <= PyUnicode_KIND(left) + // Don't resize for ascii += latin1. Convert ascii to latin1 requires + // to change the structure size, but characters are stored just after + // the structure, and so it requires to move all characters which is + // not so different than duplicating the string. + && !(PyUnicode_IS_ASCII(left) && !PyUnicode_IS_ASCII(right))) { + + __Pyx_GIVEREF(*p_left); + if (unlikely(PyUnicode_Resize(p_left, new_len) != 0)) { + // on failure PyUnicode_Resize does not deallocate the the input + // so left will remain unchanged - simply undo the giveref + __Pyx_GOTREF(*p_left); + return NULL; + } + __Pyx_INCREF(*p_left); + + // copy 'right' into the newly allocated area of 'left' + _PyUnicode_FastCopyCharacters(*p_left, left_len, right, 0, right_len); + return *p_left; + } else { + return __Pyx_PyUnicode_Concat(left, right); + } + } +#endif + +////////////// StrConcatInPlace.proto /////////////////////// +//@requires: UnicodeConcatInPlace + +#if PY_MAJOR_VERSION >= 3 + // allow access to the more efficient versions where we know str_type is unicode + #define __Pyx_PyStr_Concat __Pyx_PyUnicode_Concat + #define __Pyx_PyStr_ConcatInPlace __Pyx_PyUnicode_ConcatInPlace +#else + #define __Pyx_PyStr_Concat PyNumber_Add + #define __Pyx_PyStr_ConcatInPlace PyNumber_InPlaceAdd +#endif +#define __Pyx_PyStr_ConcatSafe(a, b) ((unlikely((a) == Py_None) || unlikely((b) == Py_None)) ? \ + PyNumber_Add(a, b) : __Pyx_PyStr_Concat(a, b)) +#define __Pyx_PyStr_ConcatInPlaceSafe(a, b) ((unlikely((a) == Py_None) || unlikely((b) == Py_None)) ? \ + PyNumber_InPlaceAdd(a, b) : __Pyx_PyStr_ConcatInPlace(a, b)) + +/////////////// FormatTypeName.proto /////////////// + +#if CYTHON_COMPILING_IN_LIMITED_API +typedef PyObject *__Pyx_TypeName; +#define __Pyx_FMT_TYPENAME "%U" +static __Pyx_TypeName __Pyx_PyType_GetName(PyTypeObject* tp); /*proto*/ +#define __Pyx_DECREF_TypeName(obj) Py_XDECREF(obj) +#else +typedef const char *__Pyx_TypeName; +#define __Pyx_FMT_TYPENAME "%.200s" +#define __Pyx_PyType_GetName(tp) ((tp)->tp_name) +#define __Pyx_DECREF_TypeName(obj) +#endif + +/////////////// FormatTypeName /////////////// + +#if CYTHON_COMPILING_IN_LIMITED_API +static __Pyx_TypeName +__Pyx_PyType_GetName(PyTypeObject* tp) +{ + PyObject *name = __Pyx_PyObject_GetAttrStr((PyObject *)tp, + PYIDENT("__name__")); + if (unlikely(name == NULL) || unlikely(!PyUnicode_Check(name))) { + PyErr_Clear(); + Py_XSETREF(name, __Pyx_NewRef(PYIDENT("?"))); + } + return name; +} +#endif + +/////////////// RaiseUnexpectedTypeError.proto /////////////// + +static int __Pyx_RaiseUnexpectedTypeError(const char *expected, PyObject *obj); /*proto*/ + +/////////////// RaiseUnexpectedTypeError /////////////// + +static int +__Pyx_RaiseUnexpectedTypeError(const char *expected, PyObject *obj) +{ + __Pyx_TypeName obj_type_name = __Pyx_PyType_GetName(Py_TYPE(obj)); + PyErr_Format(PyExc_TypeError, "Expected %s, got " __Pyx_FMT_TYPENAME, + expected, obj_type_name); + __Pyx_DECREF_TypeName(obj_type_name); + return 0; +} diff --git a/Cython/Utility/Optimize.c b/Cython/Utility/Optimize.c index 35f3a67c9..f9637b670 100644 --- a/Cython/Utility/Optimize.c +++ b/Cython/Utility/Optimize.c @@ -94,7 +94,7 @@ static CYTHON_INLINE PyObject* __Pyx_PyList_Pop(PyObject* L); /*proto*/ //@requires: ObjectHandling.c::PyObjectCallMethod0 static CYTHON_INLINE PyObject* __Pyx__PyObject_Pop(PyObject* L) { - if (Py_TYPE(L) == &PySet_Type) { + if (__Pyx_IS_TYPE(L, &PySet_Type)) { return PySet_Pop(L); } return __Pyx_PyObject_CallMethod0(L, PYIDENT("pop")); @@ -190,7 +190,7 @@ static PyObject* __Pyx_PyDict_GetItemDefault(PyObject* d, PyObject* key, PyObjec static PyObject* __Pyx_PyDict_GetItemDefault(PyObject* d, PyObject* key, PyObject* default_value) { PyObject* value; -#if PY_MAJOR_VERSION >= 3 && !CYTHON_COMPILING_IN_PYPY +#if PY_MAJOR_VERSION >= 3 && (!CYTHON_COMPILING_IN_PYPY || PYPY_VERSION_NUM >= 0x07020000) value = PyDict_GetItemWithError(d, key); if (unlikely(!value)) { if (unlikely(PyErr_Occurred())) @@ -238,7 +238,7 @@ static CYTHON_INLINE PyObject *__Pyx_PyDict_SetDefault(PyObject *d, PyObject *ke #else if (is_safe_type == 1 || (is_safe_type == -1 && /* the following builtins presumably have repeatably safe and fast hash functions */ -#if PY_MAJOR_VERSION >= 3 && !CYTHON_COMPILING_IN_PYPY +#if PY_MAJOR_VERSION >= 3 && (!CYTHON_COMPILING_IN_PYPY || PYPY_VERSION_NUM >= 0x07020000) (PyUnicode_CheckExact(key) || PyString_CheckExact(key) || PyLong_CheckExact(key)))) { value = PyDict_GetItemWithError(d, key); if (unlikely(!value)) { @@ -591,51 +591,366 @@ static double __Pyx__PyObject_AsDouble(PyObject* obj); /* proto */ PyFloat_AsDouble(obj) : __Pyx__PyObject_AsDouble(obj)) #else #define __Pyx_PyObject_AsDouble(obj) \ -((likely(PyFloat_CheckExact(obj))) ? \ - PyFloat_AS_DOUBLE(obj) : __Pyx__PyObject_AsDouble(obj)) +((likely(PyFloat_CheckExact(obj))) ? PyFloat_AS_DOUBLE(obj) : \ + likely(PyLong_CheckExact(obj)) ? \ + PyLong_AsDouble(obj) : __Pyx__PyObject_AsDouble(obj)) #endif /////////////// pyobject_as_double /////////////// +//@requires: pybytes_as_double +//@requires: pyunicode_as_double +//@requires: ObjectHandling.c::PyObjectCallOneArg static double __Pyx__PyObject_AsDouble(PyObject* obj) { - PyObject* float_value; + if (PyUnicode_CheckExact(obj)) { + return __Pyx_PyUnicode_AsDouble(obj); + } else if (PyBytes_CheckExact(obj)) { + return __Pyx_PyBytes_AsDouble(obj); + } else if (PyByteArray_CheckExact(obj)) { + return __Pyx_PyByteArray_AsDouble(obj); + } else { + PyObject* float_value; #if !CYTHON_USE_TYPE_SLOTS - float_value = PyNumber_Float(obj); if ((0)) goto bad; + float_value = PyNumber_Float(obj); if ((0)) goto bad; + // avoid "unused" warnings + (void)__Pyx_PyObject_CallOneArg; #else - PyNumberMethods *nb = Py_TYPE(obj)->tp_as_number; - if (likely(nb) && likely(nb->nb_float)) { - float_value = nb->nb_float(obj); - if (likely(float_value) && unlikely(!PyFloat_Check(float_value))) { - PyErr_Format(PyExc_TypeError, - "__float__ returned non-float (type %.200s)", - Py_TYPE(float_value)->tp_name); - Py_DECREF(float_value); - goto bad; + PyNumberMethods *nb = Py_TYPE(obj)->tp_as_number; + if (likely(nb) && likely(nb->nb_float)) { + float_value = nb->nb_float(obj); + if (likely(float_value) && unlikely(!PyFloat_Check(float_value))) { + __Pyx_TypeName float_value_type_name = __Pyx_PyType_GetName(Py_TYPE(float_value)); + PyErr_Format(PyExc_TypeError, + "__float__ returned non-float (type " __Pyx_FMT_TYPENAME ")", + float_value_type_name); + __Pyx_DECREF_TypeName(float_value_type_name); + Py_DECREF(float_value); + goto bad; + } + } else { + float_value = __Pyx_PyObject_CallOneArg((PyObject*)&PyFloat_Type, obj); } - } else if (PyUnicode_CheckExact(obj) || PyBytes_CheckExact(obj)) { -#if PY_MAJOR_VERSION >= 3 - float_value = PyFloat_FromString(obj); -#else - float_value = PyFloat_FromString(obj, 0); #endif + if (likely(float_value)) { + double value = PyFloat_AS_DOUBLE(float_value); + Py_DECREF(float_value); + return value; + } + } +bad: + return (double)-1; +} + + +/////////////// pystring_as_double.proto /////////////// +//@requires: pyunicode_as_double +//@requires: pybytes_as_double + +static CYTHON_INLINE double __Pyx_PyString_AsDouble(PyObject *obj) { + #if PY_MAJOR_VERSION >= 3 + (void)__Pyx_PyBytes_AsDouble; + return __Pyx_PyUnicode_AsDouble(obj); + #else + (void)__Pyx_PyUnicode_AsDouble; + return __Pyx_PyBytes_AsDouble(obj); + #endif +} + + +/////////////// pyunicode_as_double.proto /////////////// + +static CYTHON_INLINE double __Pyx_PyUnicode_AsDouble(PyObject *obj);/*proto*/ + +/////////////// pyunicode_as_double.proto /////////////// +//@requires: pybytes_as_double + +#if PY_MAJOR_VERSION >= 3 && !CYTHON_COMPILING_IN_PYPY +static const char* __Pyx__PyUnicode_AsDouble_Copy(const void* data, const int kind, char* buffer, Py_ssize_t start, Py_ssize_t end) { + int last_was_punctuation; + Py_ssize_t i; + // number must not start with punctuation + last_was_punctuation = 1; + for (i=start; i <= end; i++) { + Py_UCS4 chr = PyUnicode_READ(kind, data, i); + int is_punctuation = (chr == '_') | (chr == '.'); + *buffer = (char)chr; + // reject sequences of '_' and '.' + buffer += (chr != '_'); + if (unlikely(chr > 127)) goto parse_failure; + if (unlikely(last_was_punctuation & is_punctuation)) goto parse_failure; + last_was_punctuation = is_punctuation; + } + if (unlikely(last_was_punctuation)) goto parse_failure; + *buffer = '\0'; + return buffer; + +parse_failure: + return NULL; +} + +static double __Pyx__PyUnicode_AsDouble_inf_nan(const void* data, int kind, Py_ssize_t start, Py_ssize_t length) { + int matches = 1; + Py_UCS4 chr; + Py_UCS4 sign = PyUnicode_READ(kind, data, start); + int is_signed = (sign == '-') | (sign == '+'); + start += is_signed; + length -= is_signed; + + switch (PyUnicode_READ(kind, data, start)) { + #ifdef Py_NAN + case 'n': + case 'N': + if (unlikely(length != 3)) goto parse_failure; + chr = PyUnicode_READ(kind, data, start+1); + matches &= (chr == 'a') | (chr == 'A'); + chr = PyUnicode_READ(kind, data, start+2); + matches &= (chr == 'n') | (chr == 'N'); + if (unlikely(!matches)) goto parse_failure; + return (sign == '-') ? -Py_NAN : Py_NAN; + #endif + case 'i': + case 'I': + if (unlikely(length < 3)) goto parse_failure; + chr = PyUnicode_READ(kind, data, start+1); + matches &= (chr == 'n') | (chr == 'N'); + chr = PyUnicode_READ(kind, data, start+2); + matches &= (chr == 'f') | (chr == 'F'); + if (likely(length == 3 && matches)) + return (sign == '-') ? -Py_HUGE_VAL : Py_HUGE_VAL; + if (unlikely(length != 8)) goto parse_failure; + chr = PyUnicode_READ(kind, data, start+3); + matches &= (chr == 'i') | (chr == 'I'); + chr = PyUnicode_READ(kind, data, start+4); + matches &= (chr == 'n') | (chr == 'N'); + chr = PyUnicode_READ(kind, data, start+5); + matches &= (chr == 'i') | (chr == 'I'); + chr = PyUnicode_READ(kind, data, start+6); + matches &= (chr == 't') | (chr == 'T'); + chr = PyUnicode_READ(kind, data, start+7); + matches &= (chr == 'y') | (chr == 'Y'); + if (unlikely(!matches)) goto parse_failure; + return (sign == '-') ? -Py_HUGE_VAL : Py_HUGE_VAL; + case '.': case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': + break; + default: + goto parse_failure; + } + return 0.0; +parse_failure: + return -1.0; +} + +static double __Pyx_PyUnicode_AsDouble_WithSpaces(PyObject *obj) { + double value; + const char *last; + char *end; + Py_ssize_t start, length = PyUnicode_GET_LENGTH(obj); + const int kind = PyUnicode_KIND(obj); + const void* data = PyUnicode_DATA(obj); + + // strip spaces at start and end + start = 0; + while (Py_UNICODE_ISSPACE(PyUnicode_READ(kind, data, start))) + start++; + while (start < length - 1 && Py_UNICODE_ISSPACE(PyUnicode_READ(kind, data, length - 1))) + length--; + length -= start; + if (unlikely(length <= 0)) goto fallback; + + // parse NaN / inf + value = __Pyx__PyUnicode_AsDouble_inf_nan(data, kind, start, length); + if (unlikely(value == -1.0)) goto fallback; + if (value != 0.0) return value; + + if (length < 40) { + char number[40]; + last = __Pyx__PyUnicode_AsDouble_Copy(data, kind, number, start, start + length); + if (unlikely(!last)) goto fallback; + value = PyOS_string_to_double(number, &end, NULL); } else { - PyObject* args = PyTuple_New(1); - if (unlikely(!args)) goto bad; - PyTuple_SET_ITEM(args, 0, obj); - float_value = PyObject_Call((PyObject*)&PyFloat_Type, args, 0); - PyTuple_SET_ITEM(args, 0, 0); - Py_DECREF(args); + char *number = (char*) PyMem_Malloc((length + 1) * sizeof(char)); + if (unlikely(!number)) goto fallback; + last = __Pyx__PyUnicode_AsDouble_Copy(data, kind, number, start, start + length); + if (unlikely(!last)) { + PyMem_Free(number); + goto fallback; + } + value = PyOS_string_to_double(number, &end, NULL); + PyMem_Free(number); + } + if (likely(end == last) || (value == (double)-1 && PyErr_Occurred())) { + return value; + } +fallback: + return __Pyx_SlowPyString_AsDouble(obj); +} +#endif + +static CYTHON_INLINE double __Pyx_PyUnicode_AsDouble(PyObject *obj) { + // Currently not optimised for Py2.7. +#if PY_MAJOR_VERSION >= 3 && !CYTHON_COMPILING_IN_PYPY + if (unlikely(PyUnicode_READY(obj) == -1)) + return (double)-1; + if (likely(PyUnicode_IS_ASCII(obj))) { + const char *s; + Py_ssize_t length; + s = PyUnicode_AsUTF8AndSize(obj, &length); + return __Pyx__PyBytes_AsDouble(obj, s, length); } + return __Pyx_PyUnicode_AsDouble_WithSpaces(obj); +#else + return __Pyx_SlowPyString_AsDouble(obj); +#endif +} + + +/////////////// pybytes_as_double.proto /////////////// + +static double __Pyx_SlowPyString_AsDouble(PyObject *obj);/*proto*/ +static double __Pyx__PyBytes_AsDouble(PyObject *obj, const char* start, Py_ssize_t length);/*proto*/ + +static CYTHON_INLINE double __Pyx_PyBytes_AsDouble(PyObject *obj) { + return __Pyx__PyBytes_AsDouble(obj, PyBytes_AS_STRING(obj), PyBytes_GET_SIZE(obj)); +} +static CYTHON_INLINE double __Pyx_PyByteArray_AsDouble(PyObject *obj) { + return __Pyx__PyBytes_AsDouble(obj, PyByteArray_AS_STRING(obj), PyByteArray_GET_SIZE(obj)); +} + + +/////////////// pybytes_as_double /////////////// + +static double __Pyx_SlowPyString_AsDouble(PyObject *obj) { + PyObject *float_value; +#if PY_MAJOR_VERSION >= 3 + float_value = PyFloat_FromString(obj); +#else + float_value = PyFloat_FromString(obj, 0); #endif if (likely(float_value)) { double value = PyFloat_AS_DOUBLE(float_value); Py_DECREF(float_value); return value; } -bad: return (double)-1; } +static const char* __Pyx__PyBytes_AsDouble_Copy(const char* start, char* buffer, Py_ssize_t length) { + // number must not start with punctuation + int last_was_punctuation = 1; + Py_ssize_t i; + for (i=0; i < length; i++) { + char chr = start[i]; + int is_punctuation = (chr == '_') | (chr == '.') | (chr == 'e') | (chr == 'E'); + *buffer = chr; + buffer += (chr != '_'); + // reject sequences of '_' and '.' + if (unlikely(last_was_punctuation & is_punctuation)) goto parse_failure; + last_was_punctuation = is_punctuation; + } + if (unlikely(last_was_punctuation)) goto parse_failure; + *buffer = '\0'; + return buffer; + +parse_failure: + return NULL; +} + +static double __Pyx__PyBytes_AsDouble_inf_nan(const char* start, Py_ssize_t length) { + int matches = 1; + char sign = start[0]; + int is_signed = (sign == '+') | (sign == '-'); + start += is_signed; + length -= is_signed; + + switch (start[0]) { + #ifdef Py_NAN + case 'n': + case 'N': + if (unlikely(length != 3)) goto parse_failure; + matches &= (start[1] == 'a' || start[1] == 'A'); + matches &= (start[2] == 'n' || start[2] == 'N'); + if (unlikely(!matches)) goto parse_failure; + return (sign == '-') ? -Py_NAN : Py_NAN; + #endif + case 'i': + case 'I': + if (unlikely(length < 3)) goto parse_failure; + matches &= (start[1] == 'n' || start[1] == 'N'); + matches &= (start[2] == 'f' || start[2] == 'F'); + if (likely(length == 3 && matches)) + return (sign == '-') ? -Py_HUGE_VAL : Py_HUGE_VAL; + if (unlikely(length != 8)) goto parse_failure; + matches &= (start[3] == 'i' || start[3] == 'I'); + matches &= (start[4] == 'n' || start[4] == 'N'); + matches &= (start[5] == 'i' || start[5] == 'I'); + matches &= (start[6] == 't' || start[6] == 'T'); + matches &= (start[7] == 'y' || start[7] == 'Y'); + if (unlikely(!matches)) goto parse_failure; + return (sign == '-') ? -Py_HUGE_VAL : Py_HUGE_VAL; + case '.': case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': + break; + default: + goto parse_failure; + } + return 0.0; +parse_failure: + return -1.0; +} + +static CYTHON_INLINE int __Pyx__PyBytes_AsDouble_IsSpace(char ch) { + // see Py_ISSPACE() in CPython + // https://github.com/python/cpython/blob/master/Python/pyctype.c + return (ch == 0x20) | !((ch < 0x9) | (ch > 0xd)); +} + +static CYTHON_UNUSED double __Pyx__PyBytes_AsDouble(PyObject *obj, const char* start, Py_ssize_t length) { + double value; + Py_ssize_t i, digits; + const char *last = start + length; + char *end; + + // strip spaces at start and end + while (__Pyx__PyBytes_AsDouble_IsSpace(*start)) + start++; + while (start < last - 1 && __Pyx__PyBytes_AsDouble_IsSpace(last[-1])) + last--; + length = last - start; + if (unlikely(length <= 0)) goto fallback; + + // parse NaN / inf + value = __Pyx__PyBytes_AsDouble_inf_nan(start, length); + if (unlikely(value == -1.0)) goto fallback; + if (value != 0.0) return value; + + // look for underscores + digits = 0; + for (i=0; i < length; digits += start[i++] != '_'); + + if (likely(digits == length)) { + value = PyOS_string_to_double(start, &end, NULL); + } else if (digits < 40) { + char number[40]; + last = __Pyx__PyBytes_AsDouble_Copy(start, number, length); + if (unlikely(!last)) goto fallback; + value = PyOS_string_to_double(number, &end, NULL); + } else { + char *number = (char*) PyMem_Malloc((digits + 1) * sizeof(char)); + if (unlikely(!number)) goto fallback; + last = __Pyx__PyBytes_AsDouble_Copy(start, number, length); + if (unlikely(!last)) { + PyMem_Free(number); + goto fallback; + } + value = PyOS_string_to_double(number, &end, NULL); + PyMem_Free(number); + } + if (likely(end == last) || (value == (double)-1 && PyErr_Occurred())) { + return value; + } +fallback: + return __Pyx_SlowPyString_AsDouble(obj); +} + /////////////// PyNumberPow2.proto /////////////// @@ -648,7 +963,7 @@ static PyObject* __Pyx__PyNumber_PowerOf2(PyObject *two, PyObject *exp, PyObject static PyObject* __Pyx__PyNumber_PowerOf2(PyObject *two, PyObject *exp, PyObject *none, int inplace) { // in CPython, 1<<N is substantially faster than 2**N -// see http://bugs.python.org/issue21420 +// see https://bugs.python.org/issue21420 #if !CYTHON_COMPILING_IN_PYPY Py_ssize_t shiftby; #if PY_MAJOR_VERSION < 3 @@ -776,7 +1091,11 @@ static CYTHON_INLINE {{c_ret_type}} __Pyx_PyInt_{{'' if ret_type.is_pyobject els if (PyFloat_CheckExact({{pyval}})) { const long {{'a' if order == 'CObj' else 'b'}} = intval; +#if CYTHON_COMPILING_IN_LIMITED_API + double {{ival}} = __pyx_PyFloat_AsDouble({{pyval}}); +#else double {{ival}} = PyFloat_AS_DOUBLE({{pyval}}); +#endif {{return_compare('(double)a', '(double)b', c_op)}} } @@ -807,29 +1126,26 @@ static {{c_ret_type}} __Pyx_PyInt_{{'' if ret_type.is_pyobject else 'Bool'}}{{op {{py: return_false = 'Py_RETURN_FALSE' if ret_type.is_pyobject else 'return 0'}} {{py: slot_name = {'TrueDivide': 'true_divide', 'FloorDivide': 'floor_divide'}.get(op, op.lower()) }} {{py: cfunc_name = '__Pyx_PyInt_%s%s%s' % ('' if ret_type.is_pyobject else 'Bool', op, order)}} -{{py: zerodiv_check = lambda operand, _cfunc_name=cfunc_name: '%s_ZeroDivisionError(%s)' % (_cfunc_name, operand)}} {{py: c_op = { - 'Add': '+', 'Subtract': '-', 'Remainder': '%', 'TrueDivide': '/', 'FloorDivide': '/', + 'Add': '+', 'Subtract': '-', 'Multiply': '*', 'Remainder': '%', 'TrueDivide': '/', 'FloorDivide': '/', 'Or': '|', 'Xor': '^', 'And': '&', 'Rshift': '>>', 'Lshift': '<<', 'Eq': '==', 'Ne': '!=', }[op] }} - -{{if op in ('TrueDivide', 'FloorDivide', 'Remainder')}} -#if PY_MAJOR_VERSION < 3 || CYTHON_USE_PYLONG_INTERNALS -#define {{zerodiv_check('operand')}} \ - if (unlikely(zerodivision_check && ((operand) == 0))) { \ - PyErr_SetString(PyExc_ZeroDivisionError, "integer division{{if op == 'Remainder'}} or modulo{{endif}} by zero"); \ - return NULL; \ - } -#endif -{{endif}} +{{py: +def zerodiv_check(operand, optype='integer', _is_mod=op == 'Remainder', _needs_check=(order == 'CObj' and c_op in '%/')): + return ((( + 'if (unlikely(zerodivision_check && ((%s) == 0))) {' + ' PyErr_SetString(PyExc_ZeroDivisionError, "%s division%s by zero");' + ' return NULL;' + '}') % (operand, optype, ' or modulo' if _is_mod else '') + ) if _needs_check else '') +}} static {{c_ret_type}} {{cfunc_name}}(PyObject *op1, PyObject *op2, CYTHON_UNUSED long intval, int inplace, int zerodivision_check) { // Prevent "unused" warnings. - (void)inplace; - (void)zerodivision_check; + (void)inplace; (void)zerodivision_check; {{if op in ('Eq', 'Ne')}} if (op1 == op2) { @@ -844,6 +1160,7 @@ static {{c_ret_type}} {{cfunc_name}}(PyObject *op1, PyObject *op2, CYTHON_UNUSED long x; {{endif}} long {{ival}} = PyInt_AS_LONG({{pyval}}); + {{zerodiv_check('b')}} {{if op in ('Eq', 'Ne')}} if (a {{c_op}} b) { @@ -859,21 +1176,18 @@ static {{c_ret_type}} {{cfunc_name}}(PyObject *op1, PyObject *op2, CYTHON_UNUSED return PyInt_FromLong(x); return PyLong_Type.tp_as_number->nb_{{slot_name}}(op1, op2); {{elif c_op == '%'}} - {{zerodiv_check('b')}} // see ExprNodes.py :: mod_int_utility_code x = a % b; x += ((x != 0) & ((x ^ b) < 0)) * b; return PyInt_FromLong(x); {{elif op == 'TrueDivide'}} - {{zerodiv_check('b')}} if (8 * sizeof(long) <= 53 || likely(labs({{ival}}) <= ((PY_LONG_LONG)1 << 53))) { return PyFloat_FromDouble((double)a / (double)b); } // let Python do the rounding return PyInt_Type.tp_as_number->nb_{{slot_name}}(op1, op2); {{elif op == 'FloorDivide'}} - // INT_MIN / -1 is the only case that overflows, b == 0 is an error case - {{zerodiv_check('b')}} + // INT_MIN / -1 is the only case that overflows if (unlikely(b == -1 && ((unsigned long)a) == 0-(unsigned long)a)) return PyInt_Type.tp_as_number->nb_{{slot_name}}(op1, op2); else { @@ -889,6 +1203,19 @@ static {{c_ret_type}} {{cfunc_name}}(PyObject *op1, PyObject *op2, CYTHON_UNUSED if (likely(b < (long) (sizeof(long)*8) && a == (a << b) >> b) || !a) { return PyInt_FromLong(a {{c_op}} b); } + {{elif c_op == '*'}} +#ifdef HAVE_LONG_LONG + if (sizeof(PY_LONG_LONG) > sizeof(long)) { + PY_LONG_LONG result = (PY_LONG_LONG)a {{c_op}} (PY_LONG_LONG)b; + return (result >= LONG_MIN && result <= LONG_MAX) ? + PyInt_FromLong((long)result) : PyLong_FromLongLong(result); + } +#endif +#if CYTHON_USE_TYPE_SLOTS + return PyInt_Type.tp_as_number->nb_{{slot_name}}(op1, op2); +#else + return PyNumber_{{op}}(op1, op2); +#endif {{else}} // other operations are safe, no overflow return PyInt_FromLong(a {{c_op}} b); @@ -908,6 +1235,38 @@ static {{c_ret_type}} {{cfunc_name}}(PyObject *op1, PyObject *op2, CYTHON_UNUSED {{endif}} const digit* digits = ((PyLongObject*){{pyval}})->ob_digit; const Py_ssize_t size = Py_SIZE({{pyval}}); + {{if c_op == '&'}} + // special case for &-ing arbitrarily large numbers with known single digit operands + if ((intval & PyLong_MASK) == intval) { + long result = 0; + if(likely(size)) { + result = intval & (likely(size>0) ? digits[0] : (PyLong_MASK - digits[0] + 1)); + } + return PyLong_FromLong(result); + } + {{endif}} + // special cases for 0: + - * % / // | ^ & >> << + if (unlikely(size == 0)) { + {{if order == 'CObj' and c_op in '%/'}} + // division by zero! + {{zerodiv_check('0')}} + {{elif order == 'CObj' and c_op in '+-|^>><<'}} + // x == x+0 == x-0 == x|0 == x^0 == x>>0 == x<<0 + return __Pyx_NewRef(op1); + {{elif order == 'CObj' and c_op in '*&'}} + // 0 == x*0 == x&0 + return __Pyx_NewRef(op2); + {{elif order == 'ObjC' and c_op in '+|^'}} + // x == 0+x == 0|x == 0^x + return __Pyx_NewRef(op2); + {{elif order == 'ObjC' and c_op == '-'}} + // -x == 0-x + return PyLong_FromLong(-intval); + {{elif order == 'ObjC' and (c_op in '*%&>><<' or op == 'FloorDivide')}} + // 0 == 0*x == 0%x == 0&x == 0>>x == 0<<x == 0//x + return __Pyx_NewRef(op1); + {{endif}} + } // handle most common case first to avoid indirect branch and optimise branch prediction if (likely(__Pyx_sst_abs(size) <= 1)) { {{ival}} = likely(size) ? digits[0] : 0; @@ -917,15 +1276,15 @@ static {{c_ret_type}} {{cfunc_name}}(PyObject *op1, PyObject *op2, CYTHON_UNUSED {{for _size in range(2, 5)}} {{for _case in (-_size, _size)}} case {{_case}}: - if (8 * sizeof(long) - 1 > {{_size}} * PyLong_SHIFT{{if op == 'TrueDivide'}} && {{_size-1}} * PyLong_SHIFT < 53{{endif}}) { + if (8 * sizeof(long) - 1 > {{_size}} * PyLong_SHIFT{{if c_op == '*'}}+30{{endif}}{{if op == 'TrueDivide'}} && {{_size-1}} * PyLong_SHIFT < 53{{endif}}) { {{ival}} = {{'-' if _case < 0 else ''}}(long) {{pylong_join(_size, 'digits')}}; break; {{if op not in ('Eq', 'Ne', 'TrueDivide')}} -#ifdef HAVE_LONG_LONG - } else if (8 * sizeof(PY_LONG_LONG) - 1 > {{_size}} * PyLong_SHIFT) { + #ifdef HAVE_LONG_LONG + } else if (8 * sizeof(PY_LONG_LONG) - 1 > {{_size}} * PyLong_SHIFT{{if c_op == '*'}}+30{{endif}}) { ll{{ival}} = {{'-' if _case < 0 else ''}}(PY_LONG_LONG) {{pylong_join(_size, 'digits', 'unsigned PY_LONG_LONG')}}; goto long_long; -#endif + #endif {{endif}} } // if size doesn't fit into a long or PY_LONG_LONG anymore, fall through to default @@ -954,20 +1313,25 @@ static {{c_ret_type}} {{cfunc_name}}(PyObject *op1, PyObject *op2, CYTHON_UNUSED {{return_false}}; } {{else}} - {{if c_op == '%'}} - {{zerodiv_check('b')}} + {{if c_op == '*'}} + (void)a; (void)b; + #ifdef HAVE_LONG_LONG + ll{{ival}} = {{ival}}; + goto long_long; + #else + return PyLong_Type.tp_as_number->nb_{{slot_name}}(op1, op2); + #endif + {{elif c_op == '%'}} // see ExprNodes.py :: mod_int_utility_code x = a % b; x += ((x != 0) & ((x ^ b) < 0)) * b; {{elif op == 'TrueDivide'}} - {{zerodiv_check('b')}} if ((8 * sizeof(long) <= 53 || likely(labs({{ival}}) <= ((PY_LONG_LONG)1 << 53))) || __Pyx_sst_abs(size) <= 52 / PyLong_SHIFT) { return PyFloat_FromDouble((double)a / (double)b); } return PyLong_Type.tp_as_number->nb_{{slot_name}}(op1, op2); {{elif op == 'FloorDivide'}} - {{zerodiv_check('b')}} { long q, r; // see ExprNodes.py :: div_int_utility_code @@ -1020,10 +1384,14 @@ static {{c_ret_type}} {{cfunc_name}}(PyObject *op1, PyObject *op2, CYTHON_UNUSED } #endif - {{if c_op in '+-' or op in ('TrueDivide', 'Eq', 'Ne')}} + {{if c_op in '+-*' or op in ('TrueDivide', 'Eq', 'Ne')}} if (PyFloat_CheckExact({{pyval}})) { const long {{'a' if order == 'CObj' else 'b'}} = intval; +#if CYTHON_COMPILING_IN_LIMITED_API + double {{ival}} = __pyx_PyFloat_AsDouble({{pyval}}); +#else double {{ival}} = PyFloat_AS_DOUBLE({{pyval}}); +#endif {{if op in ('Eq', 'Ne')}} if ((double)a {{c_op}} (double)b) { {{return_true}}; @@ -1032,12 +1400,7 @@ static {{c_ret_type}} {{cfunc_name}}(PyObject *op1, PyObject *op2, CYTHON_UNUSED } {{else}} double result; - {{if op == 'TrueDivide'}} - if (unlikely(zerodivision_check && b == 0)) { - PyErr_SetString(PyExc_ZeroDivisionError, "float division by zero"); - return NULL; - } - {{endif}} + {{zerodiv_check('b', 'float')}} // copied from floatobject.c in Py3.5: PyFPE_START_PROTECT("{{op.lower() if not op.endswith('Divide') else 'divide'}}", return NULL) result = ((double)a) {{c_op}} (double)b; @@ -1078,27 +1441,27 @@ static {{c_ret_type}} __Pyx_PyFloat_{{'' if ret_type.is_pyobject else 'Bool'}}{{ {{py: return_false = 'Py_RETURN_FALSE' if ret_type.is_pyobject else 'return 0'}} {{py: pyval, fval = ('op2', 'b') if order == 'CObj' else ('op1', 'a') }} {{py: cfunc_name = '__Pyx_PyFloat_%s%s%s' % ('' if ret_type.is_pyobject else 'Bool', op, order) }} -{{py: zerodiv_check = lambda operand, _cfunc_name=cfunc_name: '%s_ZeroDivisionError(%s)' % (_cfunc_name, operand)}} {{py: c_op = { 'Add': '+', 'Subtract': '-', 'TrueDivide': '/', 'Divide': '/', 'Remainder': '%', 'Eq': '==', 'Ne': '!=', }[op] }} - -{{if order == 'CObj' and c_op in '%/'}} -#define {{zerodiv_check('operand')}} if (unlikely(zerodivision_check && ((operand) == 0))) { \ - PyErr_SetString(PyExc_ZeroDivisionError, "float division{{if op == 'Remainder'}} or modulo{{endif}} by zero"); \ - return NULL; \ -} -{{endif}} +{{py: +def zerodiv_check(operand, _is_mod=op == 'Remainder', _needs_check=(order == 'CObj' and c_op in '%/')): + return ((( + 'if (unlikely(zerodivision_check && ((%s) == 0.0))) {' + ' PyErr_SetString(PyExc_ZeroDivisionError, "float division%s by zero");' + ' return NULL;' + '}') % (operand, ' or modulo' if _is_mod else '') + ) if _needs_check else '') +}} static {{c_ret_type}} {{cfunc_name}}(PyObject *op1, PyObject *op2, double floatval, int inplace, int zerodivision_check) { const double {{'a' if order == 'CObj' else 'b'}} = floatval; double {{fval}}{{if op not in ('Eq', 'Ne')}}, result{{endif}}; // Prevent "unused" warnings. - (void)inplace; - (void)zerodivision_check; + (void)inplace; (void)zerodivision_check; {{if op in ('Eq', 'Ne')}} if (op1 == op2) { @@ -1107,14 +1470,18 @@ static {{c_ret_type}} {{cfunc_name}}(PyObject *op1, PyObject *op2, double floatv {{endif}} if (likely(PyFloat_CheckExact({{pyval}}))) { +#if CYTHON_COMPILING_IN_LIMITED_API + {{fval}} = __pyx_PyFloat_AsDouble({{pyval}}); +#else {{fval}} = PyFloat_AS_DOUBLE({{pyval}}); - {{if order == 'CObj' and c_op in '%/'}}{{zerodiv_check(fval)}}{{endif}} +#endif + {{zerodiv_check(fval)}} } else #if PY_MAJOR_VERSION < 3 if (likely(PyInt_CheckExact({{pyval}}))) { {{fval}} = (double) PyInt_AS_LONG({{pyval}}); - {{if order == 'CObj' and c_op in '%/'}}{{zerodiv_check(fval)}}{{endif}} + {{zerodiv_check(fval)}} } else #endif @@ -1123,7 +1490,7 @@ static {{c_ret_type}} {{cfunc_name}}(PyObject *op1, PyObject *op2, double floatv const digit* digits = ((PyLongObject*){{pyval}})->ob_digit; const Py_ssize_t size = Py_SIZE({{pyval}}); switch (size) { - case 0: {{if order == 'CObj' and c_op in '%/'}}{{zerodiv_check('0')}}{{else}}{{fval}} = 0.0;{{endif}} break; + case 0: {{fval}} = 0.0; {{zerodiv_check(fval)}} break; case -1: {{fval}} = -(double) digits[0]; break; case 1: {{fval}} = (double) digits[0]; break; {{for _size in (2, 3, 4)}} @@ -1155,7 +1522,11 @@ static {{c_ret_type}} {{cfunc_name}}(PyObject *op1, PyObject *op2, double floatv {{else}} {{fval}} = PyLong_AsDouble({{pyval}}); if (unlikely({{fval}} == -1.0 && PyErr_Occurred())) return NULL; - {{if order == 'CObj' and c_op in '%/'}}{{zerodiv_check(fval)}}{{endif}} + {{if zerodiv_check(fval)}} + #if !CYTHON_USE_PYLONG_INTERNALS + {{zerodiv_check(fval)}} + #endif + {{endif}} {{endif}} } } else { @@ -1177,7 +1548,6 @@ static {{c_ret_type}} {{cfunc_name}}(PyObject *op1, PyObject *op2, double floatv } {{else}} // copied from floatobject.c in Py3.5: - {{if order == 'CObj' and c_op in '%/'}}{{zerodiv_check('b')}}{{endif}} PyFPE_START_PROTECT("{{op.lower() if not op.endswith('Divide') else 'divide'}}", return NULL) {{if c_op == '%'}} result = fmod(a, b); diff --git a/Cython/Utility/Overflow.c b/Cython/Utility/Overflow.c index 6dff81cc1..2e58ee54a 100644 --- a/Cython/Utility/Overflow.c +++ b/Cython/Utility/Overflow.c @@ -46,6 +46,24 @@ static int __Pyx_check_twos_complement(void) { #define __Pyx_div_no_overflow(a, b, overflow) ((a) / (b)) #define __Pyx_div_const_no_overflow(a, b, overflow) ((a) / (b)) +#if defined(__has_builtin) +# if __has_builtin(__builtin_add_overflow) && !defined(__ibmxl__) +# define __PYX_HAVE_BUILTIN_OVERFLOW +# endif +#elif defined(__GNUC__) && (__GNUC__ >= 5) && (!defined(__INTEL_COMPILER) || (__INTEL_COMPILER >= 1800)) +# define __PYX_HAVE_BUILTIN_OVERFLOW +#endif + +#if defined(__GNUC__) +# define __Pyx_is_constant(x) (__builtin_constant_p(x)) +#elif defined(__has_builtin) +# if __has_builtin(__builtin_constant_p) +# define __Pyx_is_constant(x) (__builtin_constant_p(x)) +# endif +#else +# define __Pyx_is_constant(x) (0) +#endif + /////////////// Common.init /////////////// //@substitute: naming @@ -56,6 +74,8 @@ if (unlikely(__Pyx_check_twos_complement())) { /////////////// BaseCaseUnsigned.proto /////////////// +{{if UINT == "long long"}}#ifdef HAVE_LONG_LONG{{endif}} + static CYTHON_INLINE {{UINT}} __Pyx_add_{{NAME}}_checking_overflow({{UINT}} a, {{UINT}} b, int *overflow); static CYTHON_INLINE {{UINT}} __Pyx_sub_{{NAME}}_checking_overflow({{UINT}} a, {{UINT}} b, int *overflow); static CYTHON_INLINE {{UINT}} __Pyx_mul_{{NAME}}_checking_overflow({{UINT}} a, {{UINT}} b, int *overflow); @@ -64,11 +84,41 @@ static CYTHON_INLINE {{UINT}} __Pyx_div_{{NAME}}_checking_overflow({{UINT}} a, { // Use these when b is known at compile time. #define __Pyx_add_const_{{NAME}}_checking_overflow __Pyx_add_{{NAME}}_checking_overflow #define __Pyx_sub_const_{{NAME}}_checking_overflow __Pyx_sub_{{NAME}}_checking_overflow +#if defined(__PYX_HAVE_BUILTIN_OVERFLOW) +#define __Pyx_mul_const_{{NAME}}_checking_overflow __Pyx_mul_{{NAME}}_checking_overflow +#else static CYTHON_INLINE {{UINT}} __Pyx_mul_const_{{NAME}}_checking_overflow({{UINT}} a, {{UINT}} constant, int *overflow); +#endif #define __Pyx_div_const_{{NAME}}_checking_overflow __Pyx_div_{{NAME}}_checking_overflow +{{if UINT == "long long"}}#endif{{endif}} + /////////////// BaseCaseUnsigned /////////////// +{{if UINT == "long long"}}#ifdef HAVE_LONG_LONG{{endif}} + +#if defined(__PYX_HAVE_BUILTIN_OVERFLOW) + +static CYTHON_INLINE {{UINT}} __Pyx_add_{{NAME}}_checking_overflow({{UINT}} a, {{UINT}} b, int *overflow) { + {{UINT}} result; + *overflow |= __builtin_add_overflow(a, b, &result); + return result; +} + +static CYTHON_INLINE {{UINT}} __Pyx_sub_{{NAME}}_checking_overflow({{UINT}} a, {{UINT}} b, int *overflow) { + {{UINT}} result; + *overflow |= __builtin_sub_overflow(a, b, &result); + return result; +} + +static CYTHON_INLINE {{UINT}} __Pyx_mul_{{NAME}}_checking_overflow({{UINT}} a, {{UINT}} b, int *overflow) { + {{UINT}} result; + *overflow |= __builtin_mul_overflow(a, b, &result); + return result; +} + +#else + static CYTHON_INLINE {{UINT}} __Pyx_add_{{NAME}}_checking_overflow({{UINT}} a, {{UINT}} b, int *overflow) { {{UINT}} r = a + b; *overflow |= r < a; @@ -82,7 +132,12 @@ static CYTHON_INLINE {{UINT}} __Pyx_sub_{{NAME}}_checking_overflow({{UINT}} a, { } static CYTHON_INLINE {{UINT}} __Pyx_mul_{{NAME}}_checking_overflow({{UINT}} a, {{UINT}} b, int *overflow) { - if ((sizeof({{UINT}}) < sizeof(unsigned long))) { + // if we have a constant, use the constant version + if (__Pyx_is_constant(b)) { + return __Pyx_mul_const_{{NAME}}_checking_overflow(a, b, overflow); + } else if (__Pyx_is_constant(a)) { + return __Pyx_mul_const_{{NAME}}_checking_overflow(b, a, overflow); + } else if ((sizeof({{UINT}}) < sizeof(unsigned long))) { unsigned long big_r = ((unsigned long) a) * ((unsigned long) b); {{UINT}} r = ({{UINT}}) big_r; *overflow |= big_r != r; @@ -95,21 +150,27 @@ static CYTHON_INLINE {{UINT}} __Pyx_mul_{{NAME}}_checking_overflow({{UINT}} a, { return r; #endif } else { - {{UINT}} prod = a * b; - double dprod = ((double) a) * ((double) b); - // Overflow results in an error of at least 2^sizeof(UINT), - // whereas rounding represents an error on the order of 2^(sizeof(UINT)-53). - *overflow |= fabs(dprod - prod) > (__PYX_MAX({{UINT}}) / 2); - return prod; + return __Pyx_mul_const_{{NAME}}_checking_overflow(a, b, overflow); } } static CYTHON_INLINE {{UINT}} __Pyx_mul_const_{{NAME}}_checking_overflow({{UINT}} a, {{UINT}} b, int *overflow) { - if (b > 1) { - *overflow |= a > __PYX_MAX({{UINT}}) / b; + // note that deliberately the overflow check is written such that it divides by b; this + // function is used when b is a constant thus the compiler should be able to eliminate the + // (very slow on most CPUs!) division operation + {{UINT}} prod; + if (__Pyx_is_constant(a) && !__Pyx_is_constant(b)) { + // if a is a compile-time constant and b isn't, swap them + {{UINT}} temp = b; + b = a; + a = temp; } - return a * b; + prod = a * b; + if (b != 0) + *overflow |= a > (__PYX_MAX({{UINT}}) / b); + return prod; } +#endif // __PYX_HAVE_BUILTIN_OVERFLOW static CYTHON_INLINE {{UINT}} __Pyx_div_{{NAME}}_checking_overflow({{UINT}} a, {{UINT}} b, int *overflow) { @@ -120,9 +181,13 @@ static CYTHON_INLINE {{UINT}} __Pyx_div_{{NAME}}_checking_overflow({{UINT}} a, { return a / b; } +{{if UINT == "long long"}}#endif{{endif}} + /////////////// BaseCaseSigned.proto /////////////// +{{if INT == "long long"}}#ifdef HAVE_LONG_LONG{{endif}} + static CYTHON_INLINE {{INT}} __Pyx_add_{{NAME}}_checking_overflow({{INT}} a, {{INT}} b, int *overflow); static CYTHON_INLINE {{INT}} __Pyx_sub_{{NAME}}_checking_overflow({{INT}} a, {{INT}} b, int *overflow); static CYTHON_INLINE {{INT}} __Pyx_mul_{{NAME}}_checking_overflow({{INT}} a, {{INT}} b, int *overflow); @@ -130,13 +195,43 @@ static CYTHON_INLINE {{INT}} __Pyx_div_{{NAME}}_checking_overflow({{INT}} a, {{I // Use when b is known at compile time. -static CYTHON_INLINE {{INT}} __Pyx_add_const_{{NAME}}_checking_overflow({{INT}} a, {{INT}} b, int *overflow); -static CYTHON_INLINE {{INT}} __Pyx_sub_const_{{NAME}}_checking_overflow({{INT}} a, {{INT}} b, int *overflow); +#define __Pyx_add_const_{{NAME}}_checking_overflow __Pyx_add_{{NAME}}_checking_overflow +#define __Pyx_sub_const_{{NAME}}_checking_overflow __Pyx_sub_{{NAME}}_checking_overflow +#if defined(__PYX_HAVE_BUILTIN_OVERFLOW) +#define __Pyx_mul_const_{{NAME}}_checking_overflow __Pyx_mul_{{NAME}}_checking_overflow +#else static CYTHON_INLINE {{INT}} __Pyx_mul_const_{{NAME}}_checking_overflow({{INT}} a, {{INT}} constant, int *overflow); +#endif #define __Pyx_div_const_{{NAME}}_checking_overflow __Pyx_div_{{NAME}}_checking_overflow +{{if INT == "long long"}}#endif{{endif}} + /////////////// BaseCaseSigned /////////////// +{{if INT == "long long"}}#ifdef HAVE_LONG_LONG{{endif}} + +#if defined(__PYX_HAVE_BUILTIN_OVERFLOW) + +static CYTHON_INLINE {{INT}} __Pyx_add_{{NAME}}_checking_overflow({{INT}} a, {{INT}} b, int *overflow) { + {{INT}} result; + *overflow |= __builtin_add_overflow(a, b, &result); + return result; +} + +static CYTHON_INLINE {{INT}} __Pyx_sub_{{NAME}}_checking_overflow({{INT}} a, {{INT}} b, int *overflow) { + {{INT}} result; + *overflow |= __builtin_sub_overflow(a, b, &result); + return result; +} + +static CYTHON_INLINE {{INT}} __Pyx_mul_{{NAME}}_checking_overflow({{INT}} a, {{INT}} b, int *overflow) { + {{INT}} result; + *overflow |= __builtin_mul_overflow(a, b, &result); + return result; +} + +#else + static CYTHON_INLINE {{INT}} __Pyx_add_{{NAME}}_checking_overflow({{INT}} a, {{INT}} b, int *overflow) { if ((sizeof({{INT}}) < sizeof(long))) { long big_r = ((long) a) + ((long) b); @@ -151,40 +246,33 @@ static CYTHON_INLINE {{INT}} __Pyx_add_{{NAME}}_checking_overflow({{INT}} a, {{I return r; #endif } else { - // Signed overflow undefined, but unsigned overflow is well defined. - {{INT}} r = ({{INT}}) ((unsigned {{INT}}) a + (unsigned {{INT}}) b); + // Signed overflow undefined, but unsigned overflow is well defined. Casting is + // implementation-defined, but we assume two's complement (see __Pyx_check_twos_complement + // above), and arithmetic in two's-complement is the same as unsigned arithmetic. + unsigned {{INT}} r = (unsigned {{INT}}) a + (unsigned {{INT}}) b; // Overflow happened if the operands have the same sign, but the result // has opposite sign. - // sign(a) == sign(b) != sign(r) - {{INT}} sign_a = __PYX_SIGN_BIT({{INT}}) & a; - {{INT}} sign_b = __PYX_SIGN_BIT({{INT}}) & b; - {{INT}} sign_r = __PYX_SIGN_BIT({{INT}}) & r; - *overflow |= (sign_a == sign_b) & (sign_a != sign_r); - return r; - } -} - -static CYTHON_INLINE {{INT}} __Pyx_add_const_{{NAME}}_checking_overflow({{INT}} a, {{INT}} b, int *overflow) { - if (b > 0) { - *overflow |= a > __PYX_MAX({{INT}}) - b; - } else if (b < 0) { - *overflow |= a < __PYX_MIN({{INT}}) - b; + *overflow |= (((unsigned {{INT}})a ^ r) & ((unsigned {{INT}})b ^ r)) >> (8 * sizeof({{INT}}) - 1); + return ({{INT}}) r; } - return a + b; } static CYTHON_INLINE {{INT}} __Pyx_sub_{{NAME}}_checking_overflow({{INT}} a, {{INT}} b, int *overflow) { - *overflow |= b == __PYX_MIN({{INT}}); - return __Pyx_add_{{NAME}}_checking_overflow(a, -b, overflow); -} - -static CYTHON_INLINE {{INT}} __Pyx_sub_const_{{NAME}}_checking_overflow({{INT}} a, {{INT}} b, int *overflow) { - *overflow |= b == __PYX_MIN({{INT}}); - return __Pyx_add_const_{{NAME}}_checking_overflow(a, -b, overflow); + // Compilers don't handle widening as well in the subtraction case, so don't bother + unsigned {{INT}} r = (unsigned {{INT}}) a - (unsigned {{INT}}) b; + // Overflow happened if the operands differing signs, and the result + // has opposite sign to a. + *overflow |= (((unsigned {{INT}})a ^ (unsigned {{INT}})b) & ((unsigned {{INT}})a ^ r)) >> (8 * sizeof({{INT}}) - 1); + return ({{INT}}) r; } static CYTHON_INLINE {{INT}} __Pyx_mul_{{NAME}}_checking_overflow({{INT}} a, {{INT}} b, int *overflow) { - if ((sizeof({{INT}}) < sizeof(long))) { + // if we have a constant, use the constant version + if (__Pyx_is_constant(b)) { + return __Pyx_mul_const_{{NAME}}_checking_overflow(a, b, overflow); + } else if (__Pyx_is_constant(a)) { + return __Pyx_mul_const_{{NAME}}_checking_overflow(b, a, overflow); + } else if ((sizeof({{INT}}) < sizeof(long))) { long big_r = ((long) a) * ((long) b); {{INT}} r = ({{INT}}) big_r; *overflow |= big_r != r; @@ -197,16 +285,20 @@ static CYTHON_INLINE {{INT}} __Pyx_mul_{{NAME}}_checking_overflow({{INT}} a, {{I return ({{INT}}) r; #endif } else { - {{INT}} prod = a * b; - double dprod = ((double) a) * ((double) b); - // Overflow results in an error of at least 2^sizeof(INT), - // whereas rounding represents an error on the order of 2^(sizeof(INT)-53). - *overflow |= fabs(dprod - prod) > (__PYX_MAX({{INT}}) / 2); - return prod; + return __Pyx_mul_const_{{NAME}}_checking_overflow(a, b, overflow); } } static CYTHON_INLINE {{INT}} __Pyx_mul_const_{{NAME}}_checking_overflow({{INT}} a, {{INT}} b, int *overflow) { + // note that deliberately all these comparisons are written such that they divide by b; this + // function is used when b is a constant thus the compiler should be able to eliminate the + // (very slow on most CPUs!) division operations + if (__Pyx_is_constant(a) && !__Pyx_is_constant(b)) { + // if a is a compile-time constant and b isn't, swap them + {{INT}} temp = b; + b = a; + a = temp; + } if (b > 1) { *overflow |= a > __PYX_MAX({{INT}}) / b; *overflow |= a < __PYX_MIN({{INT}}) / b; @@ -216,18 +308,21 @@ static CYTHON_INLINE {{INT}} __Pyx_mul_const_{{NAME}}_checking_overflow({{INT}} *overflow |= a > __PYX_MIN({{INT}}) / b; *overflow |= a < __PYX_MAX({{INT}}) / b; } - return a * b; + return ({{INT}}) (((unsigned {{INT}})a) * ((unsigned {{INT}}) b)); } +#endif // defined(__PYX_HAVE_BUILTIN_OVERFLOW) static CYTHON_INLINE {{INT}} __Pyx_div_{{NAME}}_checking_overflow({{INT}} a, {{INT}} b, int *overflow) { if (b == 0) { *overflow |= 1; return 0; } - *overflow |= (a == __PYX_MIN({{INT}})) & (b == -1); - return a / b; + *overflow |= a == __PYX_MIN({{INT}}) && b == -1; + return ({{INT}}) ((unsigned {{INT}}) a / (unsigned {{INT}}) b); } +{{if INT == "long long"}}#endif{{endif}} + /////////////// SizeCheck.init /////////////// //@substitute: naming @@ -265,24 +360,24 @@ static CYTHON_INLINE {{TYPE}} __Pyx_{{BINOP}}_{{NAME}}_checking_overflow({{TYPE} return __Pyx_{{BINOP}}_no_overflow(a, b, overflow); } else if (__PYX_IS_UNSIGNED({{TYPE}})) { if ((sizeof({{TYPE}}) == sizeof(unsigned int))) { - return __Pyx_{{BINOP}}_unsigned_int_checking_overflow(a, b, overflow); + return ({{TYPE}}) __Pyx_{{BINOP}}_unsigned_int_checking_overflow(a, b, overflow); } else if ((sizeof({{TYPE}}) == sizeof(unsigned long))) { - return __Pyx_{{BINOP}}_unsigned_long_checking_overflow(a, b, overflow); + return ({{TYPE}}) __Pyx_{{BINOP}}_unsigned_long_checking_overflow(a, b, overflow); #ifdef HAVE_LONG_LONG } else if ((sizeof({{TYPE}}) == sizeof(unsigned PY_LONG_LONG))) { - return __Pyx_{{BINOP}}_unsigned_long_long_checking_overflow(a, b, overflow); + return ({{TYPE}}) __Pyx_{{BINOP}}_unsigned_long_long_checking_overflow(a, b, overflow); #endif } else { abort(); return 0; /* handled elsewhere */ } } else { if ((sizeof({{TYPE}}) == sizeof(int))) { - return __Pyx_{{BINOP}}_int_checking_overflow(a, b, overflow); + return ({{TYPE}}) __Pyx_{{BINOP}}_int_checking_overflow(a, b, overflow); } else if ((sizeof({{TYPE}}) == sizeof(long))) { - return __Pyx_{{BINOP}}_long_checking_overflow(a, b, overflow); + return ({{TYPE}}) __Pyx_{{BINOP}}_long_checking_overflow(a, b, overflow); #ifdef HAVE_LONG_LONG } else if ((sizeof({{TYPE}}) == sizeof(PY_LONG_LONG))) { - return __Pyx_{{BINOP}}_long_long_checking_overflow(a, b, overflow); + return ({{TYPE}}) __Pyx_{{BINOP}}_long_long_checking_overflow(a, b, overflow); #endif } else { abort(); return 0; /* handled elsewhere */ @@ -293,19 +388,24 @@ static CYTHON_INLINE {{TYPE}} __Pyx_{{BINOP}}_{{NAME}}_checking_overflow({{TYPE} /////////////// LeftShift.proto /////////////// static CYTHON_INLINE {{TYPE}} __Pyx_lshift_{{NAME}}_checking_overflow({{TYPE}} a, {{TYPE}} b, int *overflow) { - *overflow |= + int overflow_check = #if {{SIGNED}} - (b < 0) | + (a < 0) || (b < 0) || #endif - (b > ({{TYPE}}) (8 * sizeof({{TYPE}}))) | (a > (__PYX_MAX({{TYPE}}) >> b)); - return a << b; + // the following must be a _logical_ OR as the RHS is undefined if the LHS is true + (b >= ({{TYPE}}) (8 * sizeof({{TYPE}}))) || (a > (__PYX_MAX({{TYPE}}) >> b)); + if (overflow_check) { + *overflow |= 1; + return 0; + } else { + return a << b; + } } #define __Pyx_lshift_const_{{NAME}}_checking_overflow __Pyx_lshift_{{NAME}}_checking_overflow /////////////// UnaryNegOverflows.proto /////////////// -//FIXME: shouldn't the macro name be prefixed by "__Pyx_" ? Too late now, I guess... // from intobject.c -#define UNARY_NEG_WOULD_OVERFLOW(x) \ +#define __Pyx_UNARY_NEG_WOULD_OVERFLOW(x) \ (((x) < 0) & ((unsigned long)(x) == 0-(unsigned long)(x))) diff --git a/Cython/Utility/Profile.c b/Cython/Utility/Profile.c index 5f2de7e54..15ceb41cc 100644 --- a/Cython/Utility/Profile.c +++ b/Cython/Utility/Profile.c @@ -6,7 +6,7 @@ // but maybe some other profilers don't. #ifndef CYTHON_PROFILE -#if CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_PYSTON +#if CYTHON_COMPILING_IN_LIMITED_API || CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_PYSTON #define CYTHON_PROFILE 0 #else #define CYTHON_PROFILE 1 @@ -197,12 +197,12 @@ if (CYTHON_TRACE_NOGIL) { \ int ret = 0; \ PyThreadState *tstate; \ - PyGILState_STATE state = PyGILState_Ensure(); \ + PyGILState_STATE state = __Pyx_PyGILState_Ensure(); \ tstate = __Pyx_PyThreadState_Current; \ if (unlikely(tstate->use_tracing && tstate->c_tracefunc && $frame_cname->f_trace)) { \ ret = __Pyx_call_line_trace_func(tstate, $frame_cname, lineno); \ } \ - PyGILState_Release(state); \ + __Pyx_PyGILState_Release(state); \ if (unlikely(ret)) goto_error; \ } \ } else { \ diff --git a/Cython/Utility/StringTools.c b/Cython/Utility/StringTools.c index 7a3426215..feb6c99df 100644 --- a/Cython/Utility/StringTools.c +++ b/Cython/Utility/StringTools.c @@ -7,15 +7,78 @@ #include <string> + +//////////////////// ssize_strlen.proto //////////////////// + +static CYTHON_INLINE Py_ssize_t __Pyx_ssize_strlen(const char *s);/*proto*/ + +//////////////////// ssize_strlen //////////////////// +//@requires: IncludeStringH + +static CYTHON_INLINE Py_ssize_t __Pyx_ssize_strlen(const char *s) { + size_t len = strlen(s); + if (unlikely(len > PY_SSIZE_T_MAX)) { + PyErr_SetString(PyExc_OverflowError, "byte string is too long"); + return -1; + } + return (Py_ssize_t) len; +} + + +//////////////////// ssize_pyunicode_strlen.proto //////////////////// + +static CYTHON_INLINE Py_ssize_t __Pyx_Py_UNICODE_ssize_strlen(const Py_UNICODE *u);/*proto*/ + +//////////////////// ssize_pyunicode_strlen //////////////////// + +static CYTHON_INLINE Py_ssize_t __Pyx_Py_UNICODE_ssize_strlen(const Py_UNICODE *u) { + size_t len = __Pyx_Py_UNICODE_strlen(u); + if (unlikely(len > PY_SSIZE_T_MAX)) { + PyErr_SetString(PyExc_OverflowError, "Py_UNICODE string is too long"); + return -1; + } + return (Py_ssize_t) len; +} + + //////////////////// InitStrings.proto //////////////////// +#if CYTHON_COMPILING_IN_LIMITED_API +static int __Pyx_InitString(__Pyx_StringTabEntry t, PyObject **str); /*proto*/ +#else static int __Pyx_InitStrings(__Pyx_StringTabEntry *t); /*proto*/ +#endif //////////////////// InitStrings //////////////////// +#if PY_MAJOR_VERSION >= 3 +static int __Pyx_InitString(__Pyx_StringTabEntry t, PyObject **str) { + if (t.is_unicode | t.is_str) { + if (t.intern) { + *str = PyUnicode_InternFromString(t.s); + } else if (t.encoding) { + *str = PyUnicode_Decode(t.s, t.n - 1, t.encoding, NULL); + } else { + *str = PyUnicode_FromStringAndSize(t.s, t.n - 1); + } + } else { + *str = PyBytes_FromStringAndSize(t.s, t.n - 1); + } + if (!*str) + return -1; + // initialise cached hash value + if (PyObject_Hash(*str) == -1) + return -1; + return 0; +} +#endif + +#if !CYTHON_COMPILING_IN_LIMITED_API static int __Pyx_InitStrings(__Pyx_StringTabEntry *t) { while (t->p) { - #if PY_MAJOR_VERSION < 3 + #if PY_MAJOR_VERSION >= 3 /* Python 3+ has unicode identifiers */ + __Pyx_InitString(*t, t->p); + #else if (t->is_unicode) { *t->p = PyUnicode_DecodeUTF8(t->s, t->n - 1, NULL); } else if (t->intern) { @@ -23,28 +86,17 @@ static int __Pyx_InitStrings(__Pyx_StringTabEntry *t) { } else { *t->p = PyString_FromStringAndSize(t->s, t->n - 1); } - #else /* Python 3+ has unicode identifiers */ - if (t->is_unicode | t->is_str) { - if (t->intern) { - *t->p = PyUnicode_InternFromString(t->s); - } else if (t->encoding) { - *t->p = PyUnicode_Decode(t->s, t->n - 1, t->encoding, NULL); - } else { - *t->p = PyUnicode_FromStringAndSize(t->s, t->n - 1); - } - } else { - *t->p = PyBytes_FromStringAndSize(t->s, t->n - 1); - } - #endif if (!*t->p) return -1; // initialise cached hash value if (PyObject_Hash(*t->p) == -1) return -1; + #endif ++t; } return 0; } +#endif //////////////////// BytesContains.proto //////////////////// @@ -166,7 +218,7 @@ static CYTHON_INLINE int __Pyx_PyUnicode_Equals(PyObject* s1, PyObject* s2, int //@requires: BytesEquals static CYTHON_INLINE int __Pyx_PyUnicode_Equals(PyObject* s1, PyObject* s2, int equals) { -#if CYTHON_COMPILING_IN_PYPY +#if CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_LIMITED_API return PyObject_RichCompareBool(s1, s2, equals); #else #if PY_MAJOR_VERSION < 3 @@ -277,7 +329,7 @@ static CYTHON_INLINE int __Pyx_PyBytes_Equals(PyObject* s1, PyObject* s2, int eq //@requires: IncludeStringH static CYTHON_INLINE int __Pyx_PyBytes_Equals(PyObject* s1, PyObject* s2, int equals) { -#if CYTHON_COMPILING_IN_PYPY +#if CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_LIMITED_API return PyObject_RichCompareBool(s1, s2, equals); #else if (s1 == s2) { @@ -574,6 +626,8 @@ static CYTHON_INLINE PyObject* __Pyx_PyUnicode_Substring( stop = length; if (stop <= start) return __Pyx_NewRef($empty_unicode); + if (start == 0 && stop == length) + return __Pyx_NewRef(text); #if CYTHON_PEP393_ENABLED return PyUnicode_FromKindAndData(PyUnicode_KIND(text), PyUnicode_1BYTE_DATA(text) + start*PyUnicode_KIND(text), stop-start); @@ -821,7 +875,7 @@ static PyObject* __Pyx_PyUnicode_Join(PyObject* value_tuple, Py_ssize_t value_co CYTHON_UNUSED Py_UCS4 max_char) { #if CYTHON_USE_UNICODE_INTERNALS && CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS PyObject *result_uval; - int result_ukind; + int result_ukind, kind_shift; Py_ssize_t i, char_pos; void *result_udata; #if CYTHON_PEP393_ENABLED @@ -829,14 +883,17 @@ static PyObject* __Pyx_PyUnicode_Join(PyObject* value_tuple, Py_ssize_t value_co result_uval = PyUnicode_New(result_ulength, max_char); if (unlikely(!result_uval)) return NULL; result_ukind = (max_char <= 255) ? PyUnicode_1BYTE_KIND : (max_char <= 65535) ? PyUnicode_2BYTE_KIND : PyUnicode_4BYTE_KIND; + kind_shift = (result_ukind == PyUnicode_4BYTE_KIND) ? 2 : result_ukind - 1; result_udata = PyUnicode_DATA(result_uval); #else // Py 2.x/3.2 (pre PEP-393) result_uval = PyUnicode_FromUnicode(NULL, result_ulength); if (unlikely(!result_uval)) return NULL; result_ukind = sizeof(Py_UNICODE); + kind_shift = (result_ukind == 4) ? 2 : result_ukind - 1; result_udata = PyUnicode_AS_UNICODE(result_uval); #endif + assert(kind_shift == 2 || kind_shift == 1 || kind_shift == 0); char_pos = 0; for (i=0; i < value_count; i++) { @@ -849,12 +906,12 @@ static PyObject* __Pyx_PyUnicode_Join(PyObject* value_tuple, Py_ssize_t value_co ulength = __Pyx_PyUnicode_GET_LENGTH(uval); if (unlikely(!ulength)) continue; - if (unlikely(char_pos + ulength < 0)) + if (unlikely((PY_SSIZE_T_MAX >> kind_shift) - ulength < char_pos)) goto overflow; ukind = __Pyx_PyUnicode_KIND(uval); udata = __Pyx_PyUnicode_DATA(uval); if (!CYTHON_PEP393_ENABLED || ukind == result_ukind) { - memcpy((char *)result_udata + char_pos * result_ukind, udata, (size_t) (ulength * result_ukind)); + memcpy((char *)result_udata + (char_pos << kind_shift), udata, (size_t) (ulength << kind_shift)); } else { #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030300F0 || defined(_PyUnicode_FastCopyCharacters) _PyUnicode_FastCopyCharacters(result_uval, char_pos, uval, 0, ulength); @@ -1113,11 +1170,12 @@ static PyObject* __Pyx_PyObject_Format(PyObject* obj, PyObject* format_spec) { likely(PyString_CheckExact(s)) ? PyUnicode_FromEncodedObject(s, NULL, "strict") : \ PyObject_Format(s, f)) #elif CYTHON_USE_TYPE_SLOTS - // Py3 nicely returns unicode strings from str() which makes this quite efficient for builtin types + // Py3 nicely returns unicode strings from str() and repr(), which makes this quite efficient for builtin types. + // In Py3.8+, tp_str() delegates to tp_repr(), so we call tp_repr() directly here. #define __Pyx_PyObject_FormatSimple(s, f) ( \ likely(PyUnicode_CheckExact(s)) ? (Py_INCREF(s), s) : \ - likely(PyLong_CheckExact(s)) ? PyLong_Type.tp_str(s) : \ - likely(PyFloat_CheckExact(s)) ? PyFloat_Type.tp_str(s) : \ + likely(PyLong_CheckExact(s)) ? PyLong_Type.tp_repr(s) : \ + likely(PyFloat_CheckExact(s)) ? PyFloat_Type.tp_repr(s) : \ PyObject_Format(s, f)) #else #define __Pyx_PyObject_FormatSimple(s, f) ( \ @@ -1176,3 +1234,22 @@ static CYTHON_INLINE PyObject* __Pyx_PyUnicode_Unicode(PyObject *obj) { #define __Pyx_PyObject_Unicode(obj) \ (likely(PyUnicode_CheckExact(obj)) ? __Pyx_NewRef(obj) : PyObject_Unicode(obj)) #endif + + +//////////////////// PyStr_Str.proto //////////////////// + +static CYTHON_INLINE PyObject* __Pyx_PyStr_Str(PyObject *obj);/*proto*/ + +//////////////////// PyStr_Str //////////////////// + +static CYTHON_INLINE PyObject* __Pyx_PyStr_Str(PyObject *obj) { + if (unlikely(obj == Py_None)) + obj = PYIDENT("None"); + return __Pyx_NewRef(obj); +} + + +//////////////////// PyObject_Str.proto //////////////////// + +#define __Pyx_PyObject_Str(obj) \ + (likely(PyString_CheckExact(obj)) ? __Pyx_NewRef(obj) : PyObject_Str(obj)) diff --git a/Cython/Utility/TestCythonScope.pyx b/Cython/Utility/TestCythonScope.pyx index f585be298..7da7665c3 100644 --- a/Cython/Utility/TestCythonScope.pyx +++ b/Cython/Utility/TestCythonScope.pyx @@ -1,6 +1,11 @@ ########## TestClass ########## # These utilities are for testing purposes +# The "cythonscope" test calls METH_O functions with their (self, arg) signature. +# cython: always_allow_keywords=False + +from __future__ import print_function + cdef extern from *: cdef object __pyx_test_dep(object) @@ -12,32 +17,32 @@ cdef class TestClass(object): self.value = value def __str__(self): - return 'TestClass(%d)' % self.value + return f'TestClass({self.value})' cdef cdef_method(self, int value): - print 'Hello from cdef_method', value + print('Hello from cdef_method', value) cpdef cpdef_method(self, int value): - print 'Hello from cpdef_method', value + print('Hello from cpdef_method', value) def def_method(self, int value): - print 'Hello from def_method', value + print('Hello from def_method', value) @cname('cdef_cname') cdef cdef_cname_method(self, int value): - print "Hello from cdef_cname_method", value + print("Hello from cdef_cname_method", value) @cname('cpdef_cname') cpdef cpdef_cname_method(self, int value): - print "Hello from cpdef_cname_method", value + print("Hello from cpdef_cname_method", value) @cname('def_cname') def def_cname_method(self, int value): - print "Hello from def_cname_method", value + print("Hello from def_cname_method", value) @cname('__pyx_test_call_other_cy_util') cdef test_call(obj): - print 'test_call' + print('test_call') __pyx_test_dep(obj) @cname('__pyx_TestClass_New') @@ -46,19 +51,20 @@ cdef _testclass_new(int value): ########### TestDep ########## +from __future__ import print_function + @cname('__pyx_test_dep') cdef test_dep(obj): - print 'test_dep', obj + print('test_dep', obj) ########## TestScope ########## @cname('__pyx_testscope') cdef object _testscope(int value): - return "hello from cython scope, value=%d" % value + return f"hello from cython scope, value={value}" ########## View.TestScope ########## @cname('__pyx_view_testscope') cdef object _testscope(int value): - return "hello from cython.view scope, value=%d" % value - + return f"hello from cython.view scope, value={value}" diff --git a/Cython/Utility/TypeConversion.c b/Cython/Utility/TypeConversion.c index 42e474443..99e668545 100644 --- a/Cython/Utility/TypeConversion.c +++ b/Cython/Utility/TypeConversion.c @@ -68,9 +68,9 @@ static CYTHON_INLINE PyObject* __Pyx_PyUnicode_FromString(const char*); #define __Pyx_PyBytes_AsString(s) ((const char*) PyBytes_AS_STRING(s)) #define __Pyx_PyBytes_AsSString(s) ((const signed char*) PyBytes_AS_STRING(s)) #define __Pyx_PyBytes_AsUString(s) ((const unsigned char*) PyBytes_AS_STRING(s)) -#define __Pyx_PyObject_AsWritableString(s) ((char*) __Pyx_PyObject_AsString(s)) -#define __Pyx_PyObject_AsWritableSString(s) ((signed char*) __Pyx_PyObject_AsString(s)) -#define __Pyx_PyObject_AsWritableUString(s) ((unsigned char*) __Pyx_PyObject_AsString(s)) +#define __Pyx_PyObject_AsWritableString(s) ((char*)(__pyx_uintptr_t) __Pyx_PyObject_AsString(s)) +#define __Pyx_PyObject_AsWritableSString(s) ((signed char*)(__pyx_uintptr_t) __Pyx_PyObject_AsString(s)) +#define __Pyx_PyObject_AsWritableUString(s) ((unsigned char*)(__pyx_uintptr_t) __Pyx_PyObject_AsString(s)) #define __Pyx_PyObject_AsSString(s) ((const signed char*) __Pyx_PyObject_AsString(s)) #define __Pyx_PyObject_AsUString(s) ((const unsigned char*) __Pyx_PyObject_AsString(s)) #define __Pyx_PyObject_FromCString(s) __Pyx_PyObject_FromString((const char*)s) @@ -80,11 +80,21 @@ static CYTHON_INLINE PyObject* __Pyx_PyUnicode_FromString(const char*); #define __Pyx_PyUnicode_FromCString(s) __Pyx_PyUnicode_FromString((const char*)s) // There used to be a Py_UNICODE_strlen() in CPython 3.x, but it is deprecated since Py3.3. -static CYTHON_INLINE size_t __Pyx_Py_UNICODE_strlen(const Py_UNICODE *u) { +#if CYTHON_COMPILING_IN_LIMITED_API +static CYTHON_INLINE size_t __Pyx_Py_UNICODE_strlen(const wchar_t *u) +{ + const wchar_t *u_end = u; + while (*u_end++) ; + return (size_t)(u_end - u - 1); +} +#else +static CYTHON_INLINE size_t __Pyx_Py_UNICODE_strlen(const Py_UNICODE *u) +{ const Py_UNICODE *u_end = u; while (*u_end++) ; return (size_t)(u_end - u - 1); } +#endif #define __Pyx_PyUnicode_FromUnicode(u) PyUnicode_FromUnicode(u, __Pyx_Py_UNICODE_strlen(u)) #define __Pyx_PyUnicode_FromUnicodeAndLength PyUnicode_FromUnicode @@ -102,6 +112,7 @@ static CYTHON_INLINE PyObject* __Pyx_PyNumber_IntOrLong(PyObject* x); static CYTHON_INLINE Py_ssize_t __Pyx_PyIndex_AsSsize_t(PyObject*); static CYTHON_INLINE PyObject * __Pyx_PyInt_FromSize_t(size_t); +static CYTHON_INLINE Py_hash_t __Pyx_PyIndex_AsHash_t(PyObject*); #if CYTHON_ASSUME_SAFE_MACROS #define __pyx_PyFloat_AsDouble(x) (PyFloat_CheckExact(x) ? PyFloat_AS_DOUBLE(x) : PyFloat_AsDouble(x)) @@ -115,7 +126,8 @@ static CYTHON_INLINE PyObject * __Pyx_PyInt_FromSize_t(size_t); #else #define __Pyx_PyNumber_Int(x) (PyInt_CheckExact(x) ? __Pyx_NewRef(x) : PyNumber_Int(x)) #endif -#define __Pyx_PyNumber_Float(x) (PyFloat_CheckExact(x) ? __Pyx_NewRef(x) : PyNumber_Float(x)) +// __Pyx_PyNumber_Float is now in it's own section since it has dependencies (needed to make +// string conversion work the same in all circumstances) #if PY_MAJOR_VERSION < 3 && __PYX_DEFAULT_STRING_ENCODING_IS_ASCII static int __Pyx_sys_getdefaultencoding_not_ascii; @@ -270,7 +282,7 @@ static CYTHON_INLINE const char* __Pyx_PyObject_AsStringAndSize(PyObject* o, Py_ } else #endif /* __PYX_DEFAULT_STRING_ENCODING_IS_ASCII || __PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT */ -#if (!CYTHON_COMPILING_IN_PYPY) || (defined(PyByteArray_AS_STRING) && defined(PyByteArray_GET_SIZE)) +#if (!CYTHON_COMPILING_IN_PYPY && !CYTHON_COMPILING_IN_LIMITED_API) || (defined(PyByteArray_AS_STRING) && defined(PyByteArray_GET_SIZE)) if (PyByteArray_Check(o)) { *length = PyByteArray_GET_SIZE(o); return PyByteArray_AS_STRING(o); @@ -303,23 +315,27 @@ static CYTHON_INLINE int __Pyx_PyObject_IsTrueAndDecref(PyObject* x) { } static PyObject* __Pyx_PyNumber_IntOrLongWrongResultType(PyObject* result, const char* type_name) { + __Pyx_TypeName result_type_name = __Pyx_PyType_GetName(Py_TYPE(result)); #if PY_MAJOR_VERSION >= 3 if (PyLong_Check(result)) { // CPython issue #17576: warn if 'result' not of exact type int. if (PyErr_WarnFormat(PyExc_DeprecationWarning, 1, - "__int__ returned non-int (type %.200s). " - "The ability to return an instance of a strict subclass of int " - "is deprecated, and may be removed in a future version of Python.", - Py_TYPE(result)->tp_name)) { + "__int__ returned non-int (type " __Pyx_FMT_TYPENAME "). " + "The ability to return an instance of a strict subclass of int is deprecated, " + "and may be removed in a future version of Python.", + result_type_name)) { + __Pyx_DECREF_TypeName(result_type_name); Py_DECREF(result); return NULL; } + __Pyx_DECREF_TypeName(result_type_name); return result; } #endif PyErr_Format(PyExc_TypeError, - "__%.4s__ returned non-%.4s (type %.200s)", - type_name, type_name, Py_TYPE(result)->tp_name); + "__%.4s__ returned non-%.4s (type " __Pyx_FMT_TYPENAME ")", + type_name, type_name, result_type_name); + __Pyx_DECREF_TypeName(result_type_name); Py_DECREF(result); return NULL; } @@ -420,6 +436,25 @@ static CYTHON_INLINE Py_ssize_t __Pyx_PyIndex_AsSsize_t(PyObject* b) { } +static CYTHON_INLINE Py_hash_t __Pyx_PyIndex_AsHash_t(PyObject* o) { + if (sizeof(Py_hash_t) == sizeof(Py_ssize_t)) { + return (Py_hash_t) __Pyx_PyIndex_AsSsize_t(o); +#if PY_MAJOR_VERSION < 3 + } else if (likely(PyInt_CheckExact(o))) { + return PyInt_AS_LONG(o); +#endif + } else { + Py_ssize_t ival; + PyObject *x; + x = PyNumber_Index(o); + if (!x) return -1; + ival = PyInt_AsLong(x); + Py_DECREF(x); + return ival; + } +} + + static CYTHON_INLINE PyObject * __Pyx_PyBool_FromLong(long b) { return b ? __Pyx_NewRef(Py_True) : __Pyx_NewRef(Py_False); } @@ -429,6 +464,27 @@ static CYTHON_INLINE PyObject * __Pyx_PyInt_FromSize_t(size_t ival) { return PyInt_FromSize_t(ival); } +/////////////// pynumber_float.proto /////////////// + +static CYTHON_INLINE PyObject* __Pyx__PyNumber_Float(PyObject* obj); /* proto */ +#define __Pyx_PyNumber_Float(x) (PyFloat_CheckExact(x) ? __Pyx_NewRef(x) : __Pyx__PyNumber_Float(x)) + +/////////////// pynumber_float /////////////// +//@requires: Optimize.c::pybytes_as_double +//@requires: Optimize.c::pyunicode_as_double + +static CYTHON_INLINE PyObject* __Pyx__PyNumber_Float(PyObject* obj) { + // obj is PyFloat is handled in the calling macro + if (PyUnicode_CheckExact(obj)) { + return PyFloat_FromDouble(__Pyx_PyUnicode_AsDouble(obj)); + } else if (PyBytes_CheckExact(obj)) { + return PyFloat_FromDouble(__Pyx_PyBytes_AsDouble(obj)); + } else if (PyByteArray_CheckExact(obj)) { + return PyFloat_FromDouble(__Pyx_PyByteArray_AsDouble(obj)); + } else { + return PyNumber_Float(obj); + } +} /////////////// GCCDiagnostics.proto /////////////// @@ -471,7 +527,10 @@ static {{struct_type_decl}} {{funcname}}(PyObject * o) { {{struct_type_decl}} result; if (!PyTuple_Check(o) || PyTuple_GET_SIZE(o) != {{size}}) { - PyErr_Format(PyExc_TypeError, "Expected %.16s of size %d, got %.200s", "a tuple", {{size}}, Py_TYPE(o)->tp_name); + __Pyx_TypeName o_type_name = __Pyx_PyType_GetName(Py_TYPE(o)); + PyErr_Format(PyExc_TypeError, + "Expected a tuple of size %d, got " __Pyx_FMT_TYPENAME, {{size}}, o_type_name); + __Pyx_DECREF_TypeName(o_type_name); goto bad; } @@ -706,18 +765,6 @@ static CYTHON_INLINE PyObject* {{TO_PY_FUNCTION}}({{TYPE}} value, Py_ssize_t wid //@requires: CIntToDigits //@requires: GCCDiagnostics -#ifdef _MSC_VER - #ifndef _MSC_STDINT_H_ - #if _MSC_VER < 1300 - typedef unsigned short uint16_t; - #else - typedef unsigned __int16 uint16_t; - #endif - #endif -#else - #include <stdint.h> -#endif - // NOTE: inlining because most arguments are constant, which collapses lots of code below static CYTHON_INLINE PyObject* {{TO_PY_FUNCTION}}({{TYPE}} value, Py_ssize_t width, char padding_char, char format_char) { @@ -775,10 +822,10 @@ static CYTHON_INLINE PyObject* {{TO_PY_FUNCTION}}({{TYPE}} value, Py_ssize_t wid } } while (unlikely(remaining != 0)); - if (last_one_off) { - assert(*dpos == '0'); - dpos++; - } + // Correct dpos by 1 if we read an excess digit. + assert(!last_one_off || *dpos == '0'); + dpos += last_one_off; + length = end - dpos; ulength = length; prepend_sign = 0; @@ -877,7 +924,7 @@ static CYTHON_INLINE {{TYPE}} {{FROM_PY_FUNCTION}}(PyObject *x) { const int is_unsigned = neg_one > const_zero; #if PY_MAJOR_VERSION < 3 if (likely(PyInt_Check(x))) { - if (sizeof({{TYPE}}) < sizeof(long)) { + if ((sizeof({{TYPE}}) < sizeof(long))) { __PYX_VERIFY_RETURN_INT({{TYPE}}, long, PyInt_AS_LONG(x)) } else { long val = PyInt_AS_LONG(x); @@ -897,10 +944,10 @@ static CYTHON_INLINE {{TYPE}} {{FROM_PY_FUNCTION}}(PyObject *x) { case 1: __PYX_VERIFY_RETURN_INT({{TYPE}}, digit, digits[0]) {{for _size in (2, 3, 4)}} case {{_size}}: - if (8 * sizeof({{TYPE}}) > {{_size-1}} * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > {{_size}} * PyLong_SHIFT) { + if ((8 * sizeof({{TYPE}}) > {{_size-1}} * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > {{_size}} * PyLong_SHIFT)) { __PYX_VERIFY_RETURN_INT({{TYPE}}, unsigned long, {{pylong_join(_size, 'digits')}}) - } else if (8 * sizeof({{TYPE}}) >= {{_size}} * PyLong_SHIFT) { + } else if ((8 * sizeof({{TYPE}}) >= {{_size}} * PyLong_SHIFT)) { return ({{TYPE}}) {{pylong_join(_size, 'digits', TYPE)}}; } } @@ -922,10 +969,10 @@ static CYTHON_INLINE {{TYPE}} {{FROM_PY_FUNCTION}}(PyObject *x) { goto raise_neg_overflow; } #endif - if (sizeof({{TYPE}}) <= sizeof(unsigned long)) { + if ((sizeof({{TYPE}}) <= sizeof(unsigned long))) { __PYX_VERIFY_RETURN_INT_EXC({{TYPE}}, unsigned long, PyLong_AsUnsignedLong(x)) #ifdef HAVE_LONG_LONG - } else if (sizeof({{TYPE}}) <= sizeof(unsigned PY_LONG_LONG)) { + } else if ((sizeof({{TYPE}}) <= sizeof(unsigned PY_LONG_LONG))) { __PYX_VERIFY_RETURN_INT_EXC({{TYPE}}, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) #endif } @@ -940,10 +987,10 @@ static CYTHON_INLINE {{TYPE}} {{FROM_PY_FUNCTION}}(PyObject *x) { {{for _size in (2, 3, 4)}} {{for _case in (-_size, _size)}} case {{_case}}: - if (8 * sizeof({{TYPE}}){{' - 1' if _case < 0 else ''}} > {{_size-1}} * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > {{_size}} * PyLong_SHIFT) { + if ((8 * sizeof({{TYPE}}){{' - 1' if _case < 0 else ''}} > {{_size-1}} * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > {{_size}} * PyLong_SHIFT)) { __PYX_VERIFY_RETURN_INT({{TYPE}}, {{'long' if _case < 0 else 'unsigned long'}}, {{'-(long) ' if _case < 0 else ''}}{{pylong_join(_size, 'digits')}}) - } else if (8 * sizeof({{TYPE}}) - 1 > {{_size}} * PyLong_SHIFT) { + } else if ((8 * sizeof({{TYPE}}) - 1 > {{_size}} * PyLong_SHIFT)) { return ({{TYPE}}) ({{'((%s)-1)*' % TYPE if _case < 0 else ''}}{{pylong_join(_size, 'digits', TYPE)}}); } } @@ -952,18 +999,18 @@ static CYTHON_INLINE {{TYPE}} {{FROM_PY_FUNCTION}}(PyObject *x) { {{endfor}} } #endif - if (sizeof({{TYPE}}) <= sizeof(long)) { + if ((sizeof({{TYPE}}) <= sizeof(long))) { __PYX_VERIFY_RETURN_INT_EXC({{TYPE}}, long, PyLong_AsLong(x)) #ifdef HAVE_LONG_LONG - } else if (sizeof({{TYPE}}) <= sizeof(PY_LONG_LONG)) { + } else if ((sizeof({{TYPE}}) <= sizeof(PY_LONG_LONG))) { __PYX_VERIFY_RETURN_INT_EXC({{TYPE}}, PY_LONG_LONG, PyLong_AsLongLong(x)) #endif } } { -#if CYTHON_COMPILING_IN_PYPY && !defined(_PyLong_AsByteArray) +#if (CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_LIMITED_API) && !defined(_PyLong_AsByteArray) PyErr_SetString(PyExc_RuntimeError, - "_PyLong_AsByteArray() not available in PyPy, cannot convert large numbers"); + "_PyLong_AsByteArray() not available, cannot convert large numbers"); #else {{TYPE}} val; PyObject *v = __Pyx_PyNumber_IntOrLong(x); diff --git a/Cython/Utils.py b/Cython/Utils.py index d59d67d78..c8f716467 100644 --- a/Cython/Utils.py +++ b/Cython/Utils.py @@ -1,7 +1,7 @@ -# -# Cython -- Things that don't belong -# anywhere else in particular -# +""" +Cython -- Things that don't belong + anywhere else in particular +""" from __future__ import absolute_import @@ -20,31 +20,43 @@ import sys import re import io import codecs +import glob import shutil import tempfile from contextlib import contextmanager +from . import __version__ as cython_version + +PACKAGE_FILES = ("__init__.py", "__init__.pyc", "__init__.pyx", "__init__.pxd") + modification_time = os.path.getmtime _function_caches = [] + + def clear_function_caches(): for cache in _function_caches: cache.clear() + def cached_function(f): cache = {} _function_caches.append(cache) uncomputed = object() + def wrapper(*args): res = cache.get(args, uncomputed) if res is uncomputed: res = cache[args] = f(*args) return res + wrapper.uncached = f return wrapper + def cached_method(f): cache_name = '__%s_cache' % f.__name__ + def wrapper(self, *args): cache = getattr(self, cache_name, None) if cache is None: @@ -54,8 +66,10 @@ def cached_method(f): return cache[args] res = cache[args] = f(self, *args) return res + return wrapper + def replace_suffix(path, newsuf): base, _ = os.path.splitext(path) return base + newsuf @@ -92,6 +106,7 @@ def castrate_file(path, st): if st: os.utime(path, (st.st_atime, st.st_mtime-1)) + def file_newer_than(path, time): ftime = modification_time(path) return ftime > time @@ -134,24 +149,31 @@ def find_root_package_dir(file_path): else: return dir + @cached_function -def check_package_dir(dir, package_names): +def check_package_dir(dir_path, package_names): + namespace = True for dirname in package_names: - dir = os.path.join(dir, dirname) - if not is_package_dir(dir): - return None - return dir + dir_path = os.path.join(dir_path, dirname) + has_init = contains_init(dir_path) + if has_init: + namespace = False + return dir_path, namespace + @cached_function -def is_package_dir(dir_path): - for filename in ("__init__.py", - "__init__.pyc", - "__init__.pyx", - "__init__.pxd"): +def contains_init(dir_path): + for filename in PACKAGE_FILES: path = os.path.join(dir_path, filename) if path_exists(path): return 1 + +def is_package_dir(dir_path): + if contains_init(dir_path): + return 1 + + @cached_function def path_exists(path): # try on the filesystem first @@ -176,6 +198,40 @@ def path_exists(path): pass return False + +_parse_file_version = re.compile(r".*[.]cython-([0-9]+)[.][^./\\]+$").findall + + +@cached_function +def find_versioned_file(directory, filename, suffix, + _current_version=int(re.sub(r"^([0-9]+)[.]([0-9]+).*", r"\1\2", cython_version))): + """ + Search a directory for versioned pxd files, e.g. "lib.cython-30.pxd" for a Cython 3.0+ version. + + @param directory: the directory to search + @param filename: the filename without suffix + @param suffix: the filename extension including the dot, e.g. ".pxd" + @return: the file path if found, or None + """ + assert not suffix or suffix[:1] == '.' + path_prefix = os.path.join(directory, filename) + + matching_files = glob.glob(path_prefix + ".cython-*" + suffix) + path = path_prefix + suffix + if not os.path.exists(path): + path = None + best_match = (-1, path) # last resort, if we do not have versioned .pxd files + + for path in matching_files: + versions = _parse_file_version(path) + if versions: + int_version = int(versions[0]) + # Let's assume no duplicates. + if best_match[0] < int_version <= _current_version: + best_match = (int_version, path) + return best_match[1] + + # file name encodings def decode_filename(filename): @@ -189,12 +245,13 @@ def decode_filename(filename): pass return filename + # support for source file encoding detection _match_file_encoding = re.compile(br"(\w*coding)[:=]\s*([-\w.]+)").search -def detect_opened_file_encoding(f): +def detect_opened_file_encoding(f, default='UTF-8'): # PEPs 263 and 3120 # Most of the time the first two lines fall in the first couple of hundred chars, # and this bulk read/split is much faster. @@ -206,6 +263,7 @@ def detect_opened_file_encoding(f): lines = start.split(b"\n") if not data: break + m = _match_file_encoding(lines[0]) if m and m.group(1) != b'c_string_encoding': return m.group(2).decode('iso8859-1') @@ -213,7 +271,7 @@ def detect_opened_file_encoding(f): m = _match_file_encoding(lines[1]) if m: return m.group(2).decode('iso8859-1') - return "UTF-8" + return default def skip_bom(f): @@ -372,33 +430,41 @@ def print_bytes(s, header_text=None, end=b'\n', file=sys.stdout, flush=True): if flush: out.flush() + class LazyStr: def __init__(self, callback): self.callback = callback + def __str__(self): return self.callback() + def __repr__(self): return self.callback() + def __add__(self, right): return self.callback() + right + def __radd__(self, left): return left + self.callback() class OrderedSet(object): - def __init__(self, elements=()): - self._list = [] - self._set = set() - self.update(elements) - def __iter__(self): - return iter(self._list) - def update(self, elements): - for e in elements: - self.add(e) - def add(self, e): - if e not in self._set: - self._list.append(e) - self._set.add(e) + def __init__(self, elements=()): + self._list = [] + self._set = set() + self.update(elements) + + def __iter__(self): + return iter(self._list) + + def update(self, elements): + for e in elements: + self.add(e) + + def add(self, e): + if e not in self._set: + self._list.append(e) + self._set.add(e) # Class decorator that adds a metaclass and recreates the class with it. @@ -420,7 +486,7 @@ def add_metaclass(metaclass): def raise_error_if_module_name_forbidden(full_module_name): - #it is bad idea to call the pyx-file cython.pyx, so fail early + # it is bad idea to call the pyx-file cython.pyx, so fail early if full_module_name == 'cython' or full_module_name.startswith('cython.'): raise ValueError('cython is a special module, cannot be used as a module name') diff --git a/Demos/benchmarks/bpnn3.py b/Demos/benchmarks/bpnn3.py index 362bc2703..8498bd898 100644 --- a/Demos/benchmarks/bpnn3.py +++ b/Demos/benchmarks/bpnn3.py @@ -1,7 +1,7 @@ #!/usr/bin/python # Back-Propagation Neural Networks # -# Written in Python. See http://www.python.org/ +# Written in Python. See https://www.python.org/ # # Neil Schemenauer <nascheme@enme.ucalgary.ca> @@ -29,7 +29,7 @@ class NN(object): # print 'class NN' def __init__(self, ni, nh, no): # number of input, hidden, and output nodes - self.ni = ni + 1 # +1 for bias node + self.ni = ni + 1 # +1 for bias node self.nh = nh self.no = no @@ -67,7 +67,7 @@ class NN(object): for j in range(self.nh): sum = 0.0 for i in range(self.ni): - sum = sum + self.ai[i] * self.wi[i][j] + sum = sum + self.ai[i] * self.wi[i][j] self.ah[j] = 1.0/(1.0+math.exp(-sum)) # output activations diff --git a/Demos/benchmarks/chaos.py b/Demos/benchmarks/chaos.py index 36bd1bcd3..d770ff41c 100644 --- a/Demos/benchmarks/chaos.py +++ b/Demos/benchmarks/chaos.py @@ -130,7 +130,7 @@ class Spline(object): I = ii break else: - I = dom[1] - 1 + I = dom[1] - 1 return I def __len__(self): diff --git a/Demos/benchmarks/meteor_contest.py b/Demos/benchmarks/meteor_contest.py index 7eb6ca299..7610d5c08 100644 --- a/Demos/benchmarks/meteor_contest.py +++ b/Demos/benchmarks/meteor_contest.py @@ -64,7 +64,7 @@ def get_senh(board, cti): def get_puzzle(w=w, h=h): - board = [E*x + S*y + (y%2) for y in range(h) for x in range(w)] + board = [E*x + S*y + (y % 2) for y in range(h) for x in range(w)] cti = dict((board[i], i) for i in range(len(board))) idos = [[E, E, E, SE], # incremental direction offsets @@ -152,4 +152,3 @@ if __name__ == "__main__": options, args = parser.parse_args() util.run_benchmark(options, options.num_runs, main) - diff --git a/Demos/benchmarks/nqueens.py b/Demos/benchmarks/nqueens.py index 87e63df11..918f9e589 100644 --- a/Demos/benchmarks/nqueens.py +++ b/Demos/benchmarks/nqueens.py @@ -43,7 +43,7 @@ def permutations(iterable): else: return -# From http://code.activestate.com/recipes/576647/ +# From https://code.activestate.com/recipes/576647/ @cython.locals(queen_count=int, i=int, vec=list) def n_queens(queen_count): """N-Queens solver. diff --git a/Demos/benchmarks/richards.py b/Demos/benchmarks/richards.py index 913ec5daf..76a92c3a0 100644 --- a/Demos/benchmarks/richards.py +++ b/Demos/benchmarks/richards.py @@ -333,7 +333,7 @@ class WorkTask(Task): pkt.ident = dest pkt.datum = 0 - for i in BUFSIZE_RANGE: # range(BUFSIZE) + for i in BUFSIZE_RANGE: # range(BUFSIZE) w.count += 1 if w.count > 26: w.count = 1 @@ -382,9 +382,9 @@ class Richards(object): wkq = Packet(wkq , I_DEVB, K_DEV) HandlerTask(I_HANDLERB, 3000, wkq, TaskState().waitingWithPacket(), HandlerTaskRec()) - wkq = None; - DeviceTask(I_DEVA, 4000, wkq, TaskState().waiting(), DeviceTaskRec()); - DeviceTask(I_DEVB, 5000, wkq, TaskState().waiting(), DeviceTaskRec()); + wkq = None + DeviceTask(I_DEVA, 4000, wkq, TaskState().waiting(), DeviceTaskRec()) + DeviceTask(I_DEVB, 5000, wkq, TaskState().waiting(), DeviceTaskRec()) schedule() diff --git a/Demos/callback/run_cheese.py b/Demos/callback/run_cheese.py index e04910e7e..f0feb3c30 100644 --- a/Demos/callback/run_cheese.py +++ b/Demos/callback/run_cheese.py @@ -4,5 +4,3 @@ def report_cheese(name): print("Found cheese: " + name) cheese.find(report_cheese) - - diff --git a/Demos/freeze/README.rst b/Demos/freeze/README.rst index 31c4b4c12..20383ef28 100644 --- a/Demos/freeze/README.rst +++ b/Demos/freeze/README.rst @@ -106,6 +106,6 @@ Cython 0.11.2 (or newer, assuming the API does not change) SEE ALSO ======== -* `Python <http://www.python.org>`_ +* `Python <https://www.python.org/>`_ * `Cython <http://www.cython.org>`_ -* `freeze.py <http://wiki.python.org/moin/Freeze>`_ +* `freeze.py <https://wiki.python.org/moin/Freeze>`_ diff --git a/Demos/pyprimes.py b/Demos/pyprimes.py index 7c5242244..5725a8e29 100644 --- a/Demos/pyprimes.py +++ b/Demos/pyprimes.py @@ -5,9 +5,9 @@ def primes(kmax): while k < kmax: i = 0 while i < k and n % p[i] != 0: - i = i + 1 + i += 1 if i == k: p.append(n) - k = k + 1 - n = n + 1 + k += 1 + n += 1 return p diff --git a/Doc/s5/cython-ep2008.txt b/Doc/s5/cython-ep2008.txt index a29ca35d6..aeb56ae66 100644 --- a/Doc/s5/cython-ep2008.txt +++ b/Doc/s5/cython-ep2008.txt @@ -51,7 +51,7 @@ Cython is * an Open-Source project - * http://cython.org + * https://cython.org/ * a Python compiler (almost) @@ -115,7 +115,7 @@ Major Cython Developers * many, *many* others - see - * http://cython.org/ + * https://cython.org/ * the mailing list archives of Cython and Pyrex @@ -398,4 +398,4 @@ Cython \... use it, and join the project! - http://cython.org/ + https://cython.org/ diff --git a/Doc/s5/ui/default/cython-logo64.png b/Doc/s5/ui/default/cython-logo64.png Binary files differindex 7ff4f6e92..c24c9e6d5 100644 --- a/Doc/s5/ui/default/cython-logo64.png +++ b/Doc/s5/ui/default/cython-logo64.png diff --git a/Doc/s5/ui/default/iepngfix.htc b/Doc/s5/ui/default/iepngfix.htc index 4d90c87a9..5fd31ff9e 100644 --- a/Doc/s5/ui/default/iepngfix.htc +++ b/Doc/s5/ui/default/iepngfix.htc @@ -3,7 +3,7 @@ <script>
-// IE5.5+ PNG Alpha Fix v1.0 by Angus Turnbull http://www.twinhelix.com
+// IE5.5+ PNG Alpha Fix v1.0 by Angus Turnbull https://www.twinhelix.com/
// Free usage permitted as long as this notice remains intact.
// This must be a path to a blank image. That's all the configuration you need here.
diff --git a/Doc/s5/ui/default/slides.js b/Doc/s5/ui/default/slides.js index 452203586..6f7d08a25 100644 --- a/Doc/s5/ui/default/slides.js +++ b/Doc/s5/ui/default/slides.js @@ -1,6 +1,6 @@ // S5 v1.1 slides.js -- released into the Public Domain // -// Please see http://www.meyerweb.com/eric/tools/s5/credits.html for information +// Please see https://meyerweb.com/eric/tools/s5/credits.html for information // about all the wonderful and talented contributors to this code! var undef; diff --git a/LICENSE.txt b/LICENSE.txt index d9a10c0d8..4f6f63985 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -1,6 +1,6 @@ Apache License Version 2.0, January 2004 - http://www.apache.org/licenses/ + https://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION @@ -3,14 +3,18 @@ PYTHON?=python TESTOPTS?= REPO = git://github.com/cython/cython.git VERSION?=$(shell sed -ne 's|^__version__\s*=\s*"\([^"]*\)".*|\1|p' Cython/Shadow.py) +PARALLEL?=$(shell ${PYTHON} -c 'import sys; print("-j5" if sys.version_info >= (3,5) else "")' || true) -MANYLINUX_IMAGE_X86_64=quay.io/pypa/manylinux1_x86_64 -MANYLINUX_IMAGE_686=quay.io/pypa/manylinux1_i686 +MANYLINUX_IMAGE_X86_64=quay.io/pypa/manylinux2010_x86_64 +MANYLINUX_IMAGE_686=quay.io/pypa/manylinux2010_i686 all: local local: - ${PYTHON} setup.py build_ext --inplace + ${PYTHON} setup.py build_ext --inplace $(PARALLEL) + +plocal: + ${PYTHON} setup.py build_ext --inplace --cython-profile $(PARALLEL) sdist: dist/$(PACKAGENAME)-$(VERSION).tar.gz @@ -44,6 +48,7 @@ clean: @rm -f *.pyd */*.pyd */*/*.pyd @rm -f *~ */*~ */*/*~ @rm -f core */core + @rm -f Cython/*.c @rm -f Cython/Compiler/*.c @rm -f Cython/Plex/*.c @rm -f Cython/Tempita/*.c @@ -56,6 +61,9 @@ testclean: test: testclean ${PYTHON} runtests.py -vv ${TESTOPTS} +checks: + ${PYTHON} runtests.py -vv --no-unit --no-doctest --no-file --no-pyregr --no-examples + s5: $(MAKE) -C Doc/s5 slides diff --git a/README.rst b/README.rst index fe4b5958a..6b9c9f773 100644 --- a/README.rst +++ b/README.rst @@ -1,4 +1,4 @@ -Welcome to Cython! +Welcome to Cython! ================== Cython is a language that makes writing C extensions for @@ -14,20 +14,24 @@ code from Cython code. This makes Cython the ideal language for wrapping external C libraries, and for fast C modules that speed up the execution of Python code. -* Official website: http://cython.org/ -* Documentation: http://docs.cython.org/en/latest/ +* Official website: https://cython.org/ +* Documentation: https://docs.cython.org/ * Github repository: https://github.com/cython/cython * Wiki: https://github.com/cython/cython/wiki +You can **support the Cython project** via +`Github Sponsors <https://github.com/users/scoder/sponsorship>`_ or +`Tidelift <https://tidelift.com/subscription/pkg/pypi-cython>`_. + Installation: ------------- -If you already have a C compiler, just do:: +If you already have a C compiler, just run following command:: pip install Cython -otherwise, see `the installation page <http://docs.cython.org/en/latest/src/quickstart/install.html>`_. +otherwise, see `the installation page <https://docs.cython.org/en/latest/src/quickstart/install.html>`_. License: @@ -45,6 +49,10 @@ Contributing: Want to contribute to the Cython project? Here is some `help to get you started <https://github.com/cython/cython/blob/master/docs/CONTRIBUTING.rst>`_. +We are currently building the next great Cython edition: +`Cython 3.0 <https://github.com/cython/cython/milestone/58>`_. +You can help us make the life of Python 3.x users easier. + Get the full source history: ---------------------------- @@ -63,21 +71,21 @@ The following is from Pyrex: This is a development version of Pyrex, a language for writing Python extension modules. -For more info, see: +For more info, take a look at: * Doc/About.html for a description of the language * INSTALL.txt for installation instructions * USAGE.txt for usage instructions * Demos for usage examples -Comments, suggestions, bug reports, etc. are +Comments, suggestions, bug reports, etc. are most welcome! Copyright stuff: Pyrex is free of restrictions. You may use, redistribute, modify and distribute modified versions. -The latest version of Pyrex can be found `here <http://www.cosc.canterbury.ac.nz/~greg/python/Pyrex/>`_. +The latest version of Pyrex can be found `here <https://www.cosc.canterbury.ac.nz/~greg/python/Pyrex/>`_. | Greg Ewing, Computer Science Dept | University of Canterbury diff --git a/Tools/BUILD.bazel b/Tools/BUILD.bazel index e69de29bb..8b1378917 100644 --- a/Tools/BUILD.bazel +++ b/Tools/BUILD.bazel @@ -0,0 +1 @@ + diff --git a/Tools/cython-mode.el b/Tools/cython-mode.el index e4be99f5b..058dd7b20 100644 --- a/Tools/cython-mode.el +++ b/Tools/cython-mode.el @@ -103,7 +103,7 @@ (defgroup cython nil "Major mode for editing and compiling Cython files" :group 'languages :prefix "cython-" - :link '(url-link :tag "Homepage" "http://cython.org")) + :link '(url-link :tag "Homepage" "https://cython.org/")) ;;;###autoload (defcustom cython-default-compile-format "cython -a %s" diff --git a/Tools/rules.bzl b/Tools/rules.bzl index cd3eed58f..c59af6a99 100644 --- a/Tools/rules.bzl +++ b/Tools/rules.bzl @@ -11,8 +11,8 @@ load("@cython//Tools:rules.bzl", "pyx_library") pyx_library(name = 'mylib', srcs = ['a.pyx', 'a.pxd', 'b.py', 'pkg/__init__.py', 'pkg/c.pyx'], - py_deps = ['//py_library/dep'], - data = ['//other/data'], + # python library deps passed to py_library + deps = ['//py_library/dep'] ) The __init__.py file must be in your srcs list so that Cython can resolve diff --git a/appveyor.yml b/appveyor.yml index 5804d2c24..a6ad04ea7 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -5,17 +5,23 @@ environment: global: # SDK v7.0 MSVC Express 2008's SetEnv.cmd script will fail if the # /E:ON and /V:ON options are not enabled in the batch script interpreter - # See: http://stackoverflow.com/a/13751649/163740 + # See: https://stackoverflow.com/questions/11267463/compiling-python-modules-on-windows-x64/13751649#13751649 WITH_ENV: "cmd /E:ON /V:ON /C .\\appveyor\\run_with_env.cmd" + BACKEND: c + PARALLEL: "-j4" matrix: - PYTHON: "C:\\Python27" PYTHON_VERSION: "2.7" PYTHON_ARCH: "32" + PYTHONIOENCODING: "utf-8" + PARALLEL: "" - PYTHON: "C:\\Python27-x64" PYTHON_VERSION: "2.7" PYTHON_ARCH: "64" + PYTHONIOENCODING: "utf-8" + PARALLEL: "" - PYTHON: "C:\\Python39" PYTHON_VERSION: "3.9" @@ -32,10 +38,12 @@ environment: - PYTHON: "C:\\Python38-x64" PYTHON_VERSION: "3.8" PYTHON_ARCH: "64" + BACKEND: c,cpp - PYTHON: "C:\\Python37" PYTHON_VERSION: "3.7" PYTHON_ARCH: "32" + BACKEND: c,cpp - PYTHON: "C:\\Python37-x64" PYTHON_VERSION: "3.7" @@ -60,10 +68,19 @@ environment: - PYTHON: "C:\\Python34" PYTHON_VERSION: "3.4" PYTHON_ARCH: "32" + PARALLEL: "" - PYTHON: "C:\\Python34-x64" PYTHON_VERSION: "3.4" PYTHON_ARCH: "64" + PARALLEL: "" + + - PYTHON: "C:\\Python27-x64" + PYTHON_VERSION: "2.7" + PYTHON_ARCH: "64" + BACKEND: cpp + PYTHONIOENCODING: "utf-8" + PARALLEL: "" clone_depth: 5 @@ -84,14 +101,15 @@ install: build: off build_script: - - "%WITH_ENV% %PYTHON%\\python.exe setup.py build_ext --inplace" + - "%WITH_ENV% %PYTHON%\\python.exe setup.py build_ext --inplace %PARALLEL%" - "%WITH_ENV% %PYTHON%\\python.exe setup.py bdist_wheel" test: off test_script: - "%PYTHON%\\Scripts\\pip.exe install -r test-requirements.txt" - - "set CFLAGS=/Od" - - "%WITH_ENV% %PYTHON%\\python.exe runtests.py -vv --no-cpp --no-code-style -j5" + - "%PYTHON%\\Scripts\\pip.exe install win_unicode_console" + - "set CFLAGS=/Od /W3" + - "%WITH_ENV% %PYTHON%\\python.exe runtests.py -vv --backend=%BACKEND% --no-code-style -j5" artifacts: - path: dist\* diff --git a/appveyor/install.ps1 b/appveyor/install.ps1 index ab027fcfe..6a8fc6ad9 100644 --- a/appveyor/install.ps1 +++ b/appveyor/install.ps1 @@ -1,6 +1,6 @@ # Sample script to install Python and pip under Windows # Authors: Olivier Grisel and Kyle Kastner -# License: CC0 1.0 Universal: http://creativecommons.org/publicdomain/zero/1.0/ +# License: CC0 1.0 Universal: https://creativecommons.org/publicdomain/zero/1.0/ $PYTHON_BASE_URL = "https://www.python.org/ftp/python/" $GET_PIP_URL = "https://bootstrap.pypa.io/get-pip.py" diff --git a/bin/cython-generate-lexicon.py b/bin/cython-generate-lexicon.py new file mode 100755 index 000000000..e28441585 --- /dev/null +++ b/bin/cython-generate-lexicon.py @@ -0,0 +1,132 @@ +#!/usr/bin/env python3 + +# +# Updates Cython's Lexicon.py with the unicode characters that are accepted as +# identifiers. Should be run with the most recent version of Python possible +# to ensure that Lexicon is as complete as possible. +# +# Python3 only (it relies on str.isidentifier which is a Python 3 addition) +# +# Run with either +# --overwrite to update the existing Lexicon.py file +# --here to create a copy of Lexicon.py in the current directory + +import functools +import re +import os +import sys + +# Make sure we import the right Cython +cythonpath, _ = os.path.split(os.path.realpath(__file__)) # bin directory +cythonpath, _ = os.path.split(cythonpath) +if os.path.exists(os.path.join(cythonpath, "Cython")): + sys.path.insert(0, cythonpath) + print("Found (and using) local cython directory") +# else we aren't in a development directory + +from Cython.Compiler import Lexicon + + +def main(): + arg = '--overwrite' + if len(sys.argv) == 2: + arg = sys.argv[1] + if len(sys.argv) > 2 or arg not in ['--overwrite','--here']: + print("""Call the script with either: + --overwrite to update the existing Lexicon.py file (default) + --here to create an version of Lexicon.py in the current directory +""") + return + + generated_code = ( + f"# generated with:\n" + f"# {sys.implementation.name} {sys.version.splitlines()[0].strip()}\n" + "\n" + f"{generate_character_sets()}\n" + ) + + print("Reading file", Lexicon.__file__) + with open(Lexicon.__file__, 'r') as f: + parts = re.split(r"(# (?:BEGIN|END) GENERATED CODE\n?)", f.read()) + + if len(parts) not in (4,5) or ' GENERATED CODE' not in parts[1] or ' GENERATED CODE' not in parts[3]: + print("Warning: generated code section not found - code not inserted") + return + + parts[2] = generated_code + output = "".join(parts) + + if arg == "--here": + outfile = "Lexicon.py" + else: + assert arg == "--overwrite" + outfile = Lexicon.__file__ + + print("Writing to file", outfile) + with open(outfile, 'w') as f: + f.write(output) + + +# The easiest way to generate an appropriate character set is just to use the str.isidentifier method +# An alternative approach for getting character sets is at https://stackoverflow.com/a/49332214/4657412 +@functools.lru_cache() +def get_start_characters_as_number(): + return [ i for i in range(sys.maxunicode) if str.isidentifier(chr(i)) ] + + +def get_continue_characters_as_number(): + return [ i for i in range(sys.maxunicode) if str.isidentifier('a'+chr(i)) ] + + +def get_continue_not_start_as_number(): + start = get_start_characters_as_number() + cont = get_continue_characters_as_number() + assert set(start) <= set(cont), \ + "We assume that all identifier start characters are also continuation characters." + return sorted(set(cont).difference(start)) + + +def to_ranges(char_num_list): + # Convert the large lists of character digits to + # list of characters + # a list pairs of characters representing closed ranges + char_num_list = sorted(char_num_list) + first_good_val = char_num_list[0] + + single_chars = [] + ranges = [] + for n in range(1, len(char_num_list)): + if char_num_list[n]-1 != char_num_list[n-1]: + # discontinuous + if first_good_val == char_num_list[n-1]: + single_chars.append(chr(char_num_list[n-1])) + else: + ranges.append(chr(first_good_val) + chr(char_num_list[n-1])) + first_good_val = char_num_list[n] + + return ''.join(single_chars), ''.join(ranges) + + +def make_split_strings(chars, splitby=60, indent=" "): + lines = [f'u"{chars[i:i+splitby]}"' for i in range(0, len(chars), splitby)] + return indent + f"\n{indent}".join(lines) + + +def generate_character_sets(): + declarations = [] + for char_type, char_generator in [ + ("unicode_start_ch", get_start_characters_as_number), + ("unicode_continuation_ch", get_continue_not_start_as_number), + ]: + for set_type, chars in zip(("any", "range"), to_ranges(char_generator())): + declarations.append( + f"{char_type}_{set_type} = (\n" + f"{make_split_strings(chars)}\n" + f")\n" + ) + + return "".join(declarations) + + +if __name__ == "__main__": + main() diff --git a/doc-requirements.txt b/doc-requirements.txt new file mode 100644 index 000000000..2494b1d37 --- /dev/null +++ b/doc-requirements.txt @@ -0,0 +1,2 @@ +sphinx==3.5.3 +sphinx-issues==1.2.0 diff --git a/docs/CONTRIBUTING.rst b/docs/CONTRIBUTING.rst index 57cd10499..1cfc52fe6 100644 --- a/docs/CONTRIBUTING.rst +++ b/docs/CONTRIBUTING.rst @@ -5,13 +5,20 @@ If you are looking for a good way to contribute to the Cython project, please * have a look at the `Cython Hacker Guide <https://github.com/cython/cython/wiki/HackerGuide>`_, especially the section on `getting started <https://github.com/cython/cython/wiki/HackerGuide#getting-started>`_. -* look through the `issues that need help <https://github.com/cython/cython/issues?q=is%3Aissue+is%3Aopen+view+label%3A%22help+wanted%22>`_. -* look through the `issues that are a good entry point for beginners <https://github.com/cython/cython/issues?q=is%3Aissue+is%3Aopen+view+label%3A%22good+first+issue%22>`_. +* look through the `issues that need help <https://github.com/cython/cython/labels/help%20wanted>`_. +* look through the `issues that are a good entry point for beginners <https://github.com/cython/cython/labels/good%20first%20issue>`_. * ask on the `core developers mailing list <https://mail.python.org/mailman/listinfo/cython-devel>`_ for guidance. +Note that some (but not all) "good first issue"s also require an understanding of C +and a bit of the CPython C-API – usually those that also have the ``Code Generation`` +label. We generally consider a ticket a "good first issue" if it has a limited scope +that new contributors will have to learn about, e.g. only needs changes to the parser, +the type analysis or the code generation, but does not require changes all across the +compiler pipeline. + If you have code that you want to contribute, please make sure that it -* includes tests in the `tests/` directory (see the `Hacker Guide on Testing <https://github.com/cython/cython/wiki/HackerGuide#the-test-suite>`_) +* includes tests in the ``tests/`` directory (see the `Hacker Guide on Testing <https://github.com/cython/cython/wiki/HackerGuide#the-test-suite>`_) * comes in form of a pull request We use `travis <https://travis-ci.org/cython/cython>`_ and `appveyor <https://ci.appveyor.com/project/cython/cython>`_ for cross-platform testing, including pull requests. diff --git a/docs/README b/docs/README index 7b3f61b5a..db5a80c91 100644 --- a/docs/README +++ b/docs/README @@ -11,7 +11,7 @@ On windows systems, you only need Sphinx. Open PowerShell and type:: You can then see the documentation by opening in a browser ``cython/docs/build/html/index.html``. The current Cython documentation files are hosted at -https://cython.readthedocs.io/en/latest/ +https://cython.readthedocs.io/ Notes @@ -17,7 +17,7 @@ Eventually, it seems all of the old users manual could be whittled down into independent tutorial topics. Much discussion of what we'd like to see is at -http://www.mail-archive.com/cython-dev@codespeak.net/msg06945.html +https://www.mail-archive.com/cython-dev@codespeak.net/msg06945.html There is currently a huge amount of redundancy, but no one section has it all. diff --git a/docs/_templates/layout.html b/docs/_templates/layout.html index a071c96c8..814971239 100644 --- a/docs/_templates/layout.html +++ b/docs/_templates/layout.html @@ -1,7 +1,18 @@ {% extends "!layout.html" %} +{% block relbar1 %} +{{ super() }} +{% if pagename != "src/donating" %} +<div style="width: 100%; height: 3em; vertical-align: middle; text-align: center; font-weight: bold; background-color: lightgray; border: solid red 1px"> + <p style="font-size: 110%">🤝 Like the tool? Help making it better! <a href="{{ pathto("src/donating") }}">Your donation helps!</a> 🤝</p> +</div> +{% endif %} +{% endblock %} + + {% block footer %} {{ super() }} +{# ## Removed to avoid tracking our users. <script type="text/javascript"> var gaJsHost = (("https:" == document.location.protocol) ? "https://ssl." : "http://www."); document.write(unescape("%3Cscript src='" + gaJsHost + "google-analytics.com/ga.js' type='text/javascript'%3E%3C/script%3E")); @@ -11,4 +22,5 @@ try { var pageTracker = _gat._getTracker("UA-6139100-3"); pageTracker._trackPageview(); } catch(err) {}</script> +#} {% endblock %} diff --git a/docs/conf.py b/docs/conf.py index 10662e28c..5ffc995f9 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -41,10 +41,11 @@ highlight_language = 'cython' extensions = [ 'ipython_console_highlighting', 'cython_highlighting', - 'sphinx.ext.pngmath', + 'sphinx.ext.imgmath', 'sphinx.ext.todo', 'sphinx.ext.intersphinx', - 'sphinx.ext.autodoc' + 'sphinx.ext.autodoc', + 'sphinx_issues', # if this is missing, pip install sphinx-issues ] try: import rst2pdf @@ -132,6 +133,12 @@ intersphinx_mapping = {'python': ('https://docs.python.org/3/', None)} # If true, keep warnings as "system message" paragraphs in the built documents. #keep_warnings = False +# The output image format. The default is 'png'. It should be either 'png' or 'svg'. +imgmath_image_format = "svg" + +# For sphinx-issues + +issues_github_path = "cython/cython" # -- Options for HTML output --------------------------------------------------- diff --git a/docs/examples/Cython Magics.ipynb b/docs/examples/Cython Magics.ipynb index 0a8c8f56f..9bbf934c8 100644 --- a/docs/examples/Cython Magics.ipynb +++ b/docs/examples/Cython Magics.ipynb @@ -356,7 +356,7 @@ "cell_type": "markdown",
"metadata": {},
"source": [
- "You can similarly use the `-I/--include` flag to add include directories to the search path, and `-c/--compile-args` to add extra flags that are passed to Cython via the `extra_compile_args` of the distutils `Extension` class. Please see [the Cython docs on C library usage](http://docs.cython.org/src/tutorial/clibraries.html) for more details on the use of these flags."
+ "You can similarly use the `-I/--include` flag to add include directories to the search path, and `-c/--compile-args` to add extra flags that are passed to Cython via the `extra_compile_args` of the distutils `Extension` class. Please see [the Cython docs on C library usage](https://docs.cython.org/src/tutorial/clibraries.html) for more details on the use of these flags."
]
}
],
diff --git a/docs/examples/README.rst b/docs/examples/README.rst index 02c21a5fe..f998f4f33 100644 --- a/docs/examples/README.rst +++ b/docs/examples/README.rst @@ -1,3 +1,5 @@ -This example directory is organized like the ``Cython/docs/src/`` directory,
-with one directory per ``.rst`` file. All files in this directory are tested
-in the :file:`runtests.py` with the mode `compile`.
+:orphan: + +This example directory is organized like the ``Cython/docs/src/`` directory, +with one directory per ``.rst`` file. All files in this directory are tested +in the :file:`runtests.py` with the mode `compile`. diff --git a/docs/examples/not_in_docs/great_circle/p1.py b/docs/examples/not_in_docs/great_circle/p1.py index c0694a235..e60b9723f 100644 --- a/docs/examples/not_in_docs/great_circle/p1.py +++ b/docs/examples/not_in_docs/great_circle/p1.py @@ -1,7 +1,7 @@ import math def great_circle(lon1, lat1, lon2, lat2): - radius = 3956 # miles + radius = 3956 # miles x = math.pi/180.0 a = (90.0 - lat1)*x diff --git a/docs/examples/quickstart/build/hello.pyx b/docs/examples/quickstart/build/hello.pyx index 47fc8d1cf..da1b827ac 100644 --- a/docs/examples/quickstart/build/hello.pyx +++ b/docs/examples/quickstart/build/hello.pyx @@ -1,2 +1,2 @@ -def say_hello_to(name):
- print("Hello %s!" % name)
+def say_hello_to(name): + print("Hello %s!" % name) diff --git a/docs/examples/quickstart/build/setup.py b/docs/examples/quickstart/build/setup.py index 4fb8c8154..fe959a106 100644 --- a/docs/examples/quickstart/build/setup.py +++ b/docs/examples/quickstart/build/setup.py @@ -1,5 +1,8 @@ -from distutils.core import setup
-from Cython.Build import cythonize
-
-setup(name='Hello world app',
- ext_modules=cythonize("hello.pyx"))
+from setuptools import setup +from Cython.Build import cythonize + +setup( + name='Hello world app', + ext_modules=cythonize("hello.pyx"), + zip_safe=False, +) diff --git a/docs/examples/quickstart/cythonize/cdef_keyword.pyx b/docs/examples/quickstart/cythonize/cdef_keyword.pyx index 16503ee89..ba1e46272 100644 --- a/docs/examples/quickstart/cythonize/cdef_keyword.pyx +++ b/docs/examples/quickstart/cythonize/cdef_keyword.pyx @@ -1,2 +1,2 @@ -cdef double f(double x) except? -2:
- return x ** 2 - x
+cdef double f(double x) except? -2: + return x ** 2 - x diff --git a/docs/examples/quickstart/cythonize/integrate.py b/docs/examples/quickstart/cythonize/integrate.py index 8d420b923..80d6d13a7 100644 --- a/docs/examples/quickstart/cythonize/integrate.py +++ b/docs/examples/quickstart/cythonize/integrate.py @@ -1,10 +1,10 @@ -def f(x):
- return x ** 2 - x
-
-
-def integrate_f(a, b, N):
- s = 0
- dx = (b - a) / N
- for i in range(N):
- s += f(a + i * dx)
- return s * dx
+def f(x): + return x ** 2 - x + + +def integrate_f(a, b, N): + s = 0 + dx = (b - a) / N + for i in range(N): + s += f(a + i * dx) + return s * dx diff --git a/docs/examples/quickstart/cythonize/integrate_cy.pyx b/docs/examples/quickstart/cythonize/integrate_cy.pyx index 0bb0cd548..f676969d8 100644 --- a/docs/examples/quickstart/cythonize/integrate_cy.pyx +++ b/docs/examples/quickstart/cythonize/integrate_cy.pyx @@ -1,12 +1,12 @@ -def f(double x):
- return x ** 2 - x
-
-
-def integrate_f(double a, double b, int N):
- cdef int i
- cdef double s, dx
- s = 0
- dx = (b - a) / N
- for i in range(N):
- s += f(a + i * dx)
- return s * dx
+def f(double x): + return x ** 2 - x + + +def integrate_f(double a, double b, int N): + cdef int i + cdef double s, dx + s = 0 + dx = (b - a) / N + for i in range(N): + s += f(a + i * dx) + return s * dx diff --git a/docs/examples/tutorial/array/clone.pyx b/docs/examples/tutorial/array/clone.pyx index e2bac0e4a..2eb803499 100644 --- a/docs/examples/tutorial/array/clone.pyx +++ b/docs/examples/tutorial/array/clone.pyx @@ -1,8 +1,8 @@ -from cpython cimport array
-import array
-
-cdef array.array int_array_template = array.array('i', [])
-cdef array.array newarray
-
-# create an array with 3 elements with same type as template
-newarray = array.clone(int_array_template, 3, zero=False)
+from cpython cimport array +import array + +cdef array.array int_array_template = array.array('i', []) +cdef array.array newarray + +# create an array with 3 elements with same type as template +newarray = array.clone(int_array_template, 3, zero=False) diff --git a/docs/examples/tutorial/array/overhead.pyx b/docs/examples/tutorial/array/overhead.pyx index e385bff3f..88bc97500 100644 --- a/docs/examples/tutorial/array/overhead.pyx +++ b/docs/examples/tutorial/array/overhead.pyx @@ -1,15 +1,15 @@ -from cpython cimport array
-import array
-
-cdef array.array a = array.array('i', [1, 2, 3])
-cdef int[:] ca = a
-
-cdef int overhead(object a):
- cdef int[:] ca = a
- return ca[0]
-
-cdef int no_overhead(int[:] ca):
- return ca[0]
-
-print(overhead(a)) # new memory view will be constructed, overhead
-print(no_overhead(ca)) # ca is already a memory view, so no overhead
+from cpython cimport array +import array + +cdef array.array a = array.array('i', [1, 2, 3]) +cdef int[:] ca = a + +cdef int overhead(object a): + cdef int[:] ca = a + return ca[0] + +cdef int no_overhead(int[:] ca): + return ca[0] + +print(overhead(a)) # new memory view will be constructed, overhead +print(no_overhead(ca)) # ca is already a memory view, so no overhead diff --git a/docs/examples/tutorial/array/resize.pyx b/docs/examples/tutorial/array/resize.pyx index a11fbde7b..7b92958b4 100644 --- a/docs/examples/tutorial/array/resize.pyx +++ b/docs/examples/tutorial/array/resize.pyx @@ -1,10 +1,10 @@ -from cpython cimport array
-import array
-
-cdef array.array a = array.array('i', [1, 2, 3])
-cdef array.array b = array.array('i', [4, 5, 6])
-
-# extend a with b, resize as needed
-array.extend(a, b)
-# resize a, leaving just original three elements
-array.resize(a, len(a) - len(b))
+from cpython cimport array +import array + +cdef array.array a = array.array('i', [1, 2, 3]) +cdef array.array b = array.array('i', [4, 5, 6]) + +# extend a with b, resize as needed +array.extend(a, b) +# resize a, leaving just original three elements +array.resize(a, len(a) - len(b)) diff --git a/docs/examples/tutorial/array/safe_usage.pyx b/docs/examples/tutorial/array/safe_usage.pyx index 61d6b39eb..15107ae92 100644 --- a/docs/examples/tutorial/array/safe_usage.pyx +++ b/docs/examples/tutorial/array/safe_usage.pyx @@ -1,6 +1,6 @@ -from cpython cimport array
-import array
-cdef array.array a = array.array('i', [1, 2, 3])
-cdef int[:] ca = a
-
-print(ca[0])
+from cpython cimport array +import array +cdef array.array a = array.array('i', [1, 2, 3]) +cdef int[:] ca = a + +print(ca[0]) diff --git a/docs/examples/tutorial/array/unsafe_usage.pyx b/docs/examples/tutorial/array/unsafe_usage.pyx index 2aefeb102..d1f498c68 100644 --- a/docs/examples/tutorial/array/unsafe_usage.pyx +++ b/docs/examples/tutorial/array/unsafe_usage.pyx @@ -1,11 +1,11 @@ -from cpython cimport array
-import array
-
-cdef array.array a = array.array('i', [1, 2, 3])
-
-# access underlying pointer:
-print(a.data.as_ints[0])
-
-from libc.string cimport memset
-
-memset(a.data.as_voidptr, 0, len(a) * sizeof(int))
+from cpython cimport array +import array + +cdef array.array a = array.array('i', [1, 2, 3]) + +# access underlying pointer: +print(a.data.as_ints[0]) + +from libc.string cimport memset + +memset(a.data.as_voidptr, 0, len(a) * sizeof(int)) diff --git a/docs/examples/tutorial/cdef_classes/integrate.pyx b/docs/examples/tutorial/cdef_classes/integrate.pyx index a3bbcbfec..a3e96f398 100644 --- a/docs/examples/tutorial/cdef_classes/integrate.pyx +++ b/docs/examples/tutorial/cdef_classes/integrate.pyx @@ -1,14 +1,14 @@ -from sin_of_square cimport Function, SinOfSquareFunction
-
-def integrate(Function f, double a, double b, int N):
- cdef int i
- cdef double s, dx
- if f is None:
- raise ValueError("f cannot be None")
- s = 0
- dx = (b - a) / N
- for i in range(N):
- s += f.evaluate(a + i * dx)
- return s * dx
-
-print(integrate(SinOfSquareFunction(), 0, 1, 10000))
+from sin_of_square cimport Function, SinOfSquareFunction + +def integrate(Function f, double a, double b, int N): + cdef int i + cdef double s, dx + if f is None: + raise ValueError("f cannot be None") + s = 0 + dx = (b - a) / N + for i in range(N): + s += f.evaluate(a + i * dx) + return s * dx + +print(integrate(SinOfSquareFunction(), 0, 1, 10000)) diff --git a/docs/examples/tutorial/cdef_classes/math_function.py b/docs/examples/tutorial/cdef_classes/math_function.py index 21281cc9b..1a6ee896c 100644 --- a/docs/examples/tutorial/cdef_classes/math_function.py +++ b/docs/examples/tutorial/cdef_classes/math_function.py @@ -1,7 +1,7 @@ -class MathFunction(object):
- def __init__(self, name, operator):
- self.name = name
- self.operator = operator
-
- def __call__(self, *operands):
- return self.operator(*operands)
+class MathFunction(object): + def __init__(self, name, operator): + self.name = name + self.operator = operator + + def __call__(self, *operands): + return self.operator(*operands) diff --git a/docs/examples/tutorial/cdef_classes/math_function_2.pyx b/docs/examples/tutorial/cdef_classes/math_function_2.pyx index 1793ef689..7d1446b7c 100644 --- a/docs/examples/tutorial/cdef_classes/math_function_2.pyx +++ b/docs/examples/tutorial/cdef_classes/math_function_2.pyx @@ -1,3 +1,3 @@ -cdef class Function:
- cpdef double evaluate(self, double x) except *:
- return 0
+cdef class Function: + cpdef double evaluate(self, double x) except *: + return 0 diff --git a/docs/examples/tutorial/cdef_classes/nonecheck.pyx b/docs/examples/tutorial/cdef_classes/nonecheck.pyx index b9e12c8d5..51c61006d 100644 --- a/docs/examples/tutorial/cdef_classes/nonecheck.pyx +++ b/docs/examples/tutorial/cdef_classes/nonecheck.pyx @@ -1,19 +1,19 @@ -# cython: nonecheck=True
-# ^^^ Turns on nonecheck globally
-
-import cython
-
-cdef class MyClass:
- pass
-
-# Turn off nonecheck locally for the function
-@cython.nonecheck(False)
-def func():
- cdef MyClass obj = None
- try:
- # Turn nonecheck on again for a block
- with cython.nonecheck(True):
- print(obj.myfunc()) # Raises exception
- except AttributeError:
- pass
- print(obj.myfunc()) # Hope for a crash!
+# cython: nonecheck=True +# ^^^ Turns on nonecheck globally + +import cython + +cdef class MyClass: + pass + +# Turn off nonecheck locally for the function +@cython.nonecheck(False) +def func(): + cdef MyClass obj = None + try: + # Turn nonecheck on again for a block + with cython.nonecheck(True): + print(obj.myfunc()) # Raises exception + except AttributeError: + pass + print(obj.myfunc()) # Hope for a crash! diff --git a/docs/examples/tutorial/cdef_classes/sin_of_square.pyx b/docs/examples/tutorial/cdef_classes/sin_of_square.pyx index 7aab96056..d39ff374d 100644 --- a/docs/examples/tutorial/cdef_classes/sin_of_square.pyx +++ b/docs/examples/tutorial/cdef_classes/sin_of_square.pyx @@ -1,9 +1,9 @@ -from libc.math cimport sin
-
-cdef class Function:
- cpdef double evaluate(self, double x) except *:
- return 0
-
-cdef class SinOfSquareFunction(Function):
- cpdef double evaluate(self, double x) except *:
- return sin(x ** 2)
+from libc.math cimport sin + +cdef class Function: + cpdef double evaluate(self, double x) except *: + return 0 + +cdef class SinOfSquareFunction(Function): + cpdef double evaluate(self, double x) except *: + return sin(x ** 2) diff --git a/docs/examples/tutorial/cdef_classes/wave_function.pyx b/docs/examples/tutorial/cdef_classes/wave_function.pyx index aa35d954e..82ba1c7bc 100644 --- a/docs/examples/tutorial/cdef_classes/wave_function.pyx +++ b/docs/examples/tutorial/cdef_classes/wave_function.pyx @@ -1,21 +1,21 @@ -from sin_of_square cimport Function
-
-cdef class WaveFunction(Function):
-
- # Not available in Python-space:
- cdef double offset
-
- # Available in Python-space:
- cdef public double freq
-
- # Available in Python-space, but only for reading:
- cdef readonly double scale
-
- # Available in Python-space:
- @property
- def period(self):
- return 1.0 / self.freq
-
- @period.setter
- def period(self, value):
- self.freq = 1.0 / value
+from sin_of_square cimport Function + +cdef class WaveFunction(Function): + + # Not available in Python-space: + cdef double offset + + # Available in Python-space: + cdef public double freq + + # Available in Python-space, but only for reading: + cdef readonly double scale + + # Available in Python-space: + @property + def period(self): + return 1.0 / self.freq + + @period.setter + def period(self, value): + self.freq = 1.0 / value diff --git a/docs/examples/tutorial/clibraries/queue.pyx b/docs/examples/tutorial/clibraries/queue.pyx index 5363ee4f5..20dd8c4e6 100644 --- a/docs/examples/tutorial/clibraries/queue.pyx +++ b/docs/examples/tutorial/clibraries/queue.pyx @@ -1,9 +1,9 @@ -# queue.pyx
-
-cimport cqueue
-
-cdef class Queue:
- cdef cqueue.Queue* _c_queue
-
- def __cinit__(self):
- self._c_queue = cqueue.queue_new()
+# queue.pyx + +cimport cqueue + +cdef class Queue: + cdef cqueue.Queue* _c_queue + + def __cinit__(self): + self._c_queue = cqueue.queue_new() diff --git a/docs/examples/tutorial/clibraries/queue2.pyx b/docs/examples/tutorial/clibraries/queue2.pyx index 9278fbf4b..3685a4a02 100644 --- a/docs/examples/tutorial/clibraries/queue2.pyx +++ b/docs/examples/tutorial/clibraries/queue2.pyx @@ -1,11 +1,11 @@ -# queue.pyx
-
-cimport cqueue
-
-cdef class Queue:
- cdef cqueue.Queue* _c_queue
-
- def __cinit__(self):
- self._c_queue = cqueue.queue_new()
- if self._c_queue is NULL:
- raise MemoryError()
+# queue.pyx + +cimport cqueue + +cdef class Queue: + cdef cqueue.Queue* _c_queue + + def __cinit__(self): + self._c_queue = cqueue.queue_new() + if self._c_queue is NULL: + raise MemoryError() diff --git a/docs/examples/tutorial/clibraries/test_queue.py b/docs/examples/tutorial/clibraries/test_queue.py index 5390a82c1..41b267395 100644 --- a/docs/examples/tutorial/clibraries/test_queue.py +++ b/docs/examples/tutorial/clibraries/test_queue.py @@ -1,36 +1,36 @@ -from __future__ import print_function
-
-import time
-
-import queue
-
-Q = queue.Queue()
-
-Q.append(10)
-Q.append(20)
-print(Q.peek())
-print(Q.pop())
-print(Q.pop())
-try:
- print(Q.pop())
-except IndexError as e:
- print("Error message:", e) # Prints "Queue is empty"
-
-i = 10000
-
-values = range(i)
-
-start_time = time.time()
-
-Q.extend(values)
-
-end_time = time.time() - start_time
-
-print("Adding {} items took {:1.3f} msecs.".format(i, 1000 * end_time))
-
-for i in range(41):
- Q.pop()
-
-Q.pop()
-print("The answer is:")
-print(Q.pop())
+from __future__ import print_function + +import time + +import queue + +Q = queue.Queue() + +Q.append(10) +Q.append(20) +print(Q.peek()) +print(Q.pop()) +print(Q.pop()) +try: + print(Q.pop()) +except IndexError as e: + print("Error message:", e) # Prints "Queue is empty" + +i = 10000 + +values = range(i) + +start_time = time.time() + +Q.extend(values) + +end_time = time.time() - start_time + +print("Adding {} items took {:1.3f} msecs.".format(i, 1000 * end_time)) + +for i in range(41): + Q.pop() + +Q.pop() +print("The answer is:") +print(Q.pop()) diff --git a/docs/examples/tutorial/cython_tutorial/primes_cpp.pyx b/docs/examples/tutorial/cython_tutorial/primes_cpp.pyx index 57bfe9cc2..7b905a27b 100644 --- a/docs/examples/tutorial/cython_tutorial/primes_cpp.pyx +++ b/docs/examples/tutorial/cython_tutorial/primes_cpp.pyx @@ -1,21 +1,21 @@ -# distutils: language=c++
-
-from libcpp.vector cimport vector
-
-def primes(unsigned int nb_primes):
- cdef int n, i
- cdef vector[int] p
- p.reserve(nb_primes) # allocate memory for 'nb_primes' elements.
-
- n = 2
- while p.size() < nb_primes: # size() for vectors is similar to len()
- for i in p:
- if n % i == 0:
- break
- else:
- p.push_back(n) # push_back is similar to append()
- n += 1
-
- # Vectors are automatically converted to Python
- # lists when converted to Python objects.
- return p
+# distutils: language=c++ + +from libcpp.vector cimport vector + +def primes(unsigned int nb_primes): + cdef int n, i + cdef vector[int] p + p.reserve(nb_primes) # allocate memory for 'nb_primes' elements. + + n = 2 + while p.size() < nb_primes: # size() for vectors is similar to len() + for i in p: + if n % i == 0: + break + else: + p.push_back(n) # push_back is similar to append() + n += 1 + + # Vectors are automatically converted to Python + # lists when converted to Python objects. + return p diff --git a/docs/examples/tutorial/cython_tutorial/setup.py b/docs/examples/tutorial/cython_tutorial/setup.py index 302a08e5f..45941a33c 100644 --- a/docs/examples/tutorial/cython_tutorial/setup.py +++ b/docs/examples/tutorial/cython_tutorial/setup.py @@ -1,4 +1,4 @@ -from distutils.core import setup +from setuptools import setup from Cython.Build import cythonize setup( diff --git a/docs/examples/tutorial/embedding/embedded.pyx b/docs/examples/tutorial/embedding/embedded.pyx new file mode 100644 index 000000000..26704d45f --- /dev/null +++ b/docs/examples/tutorial/embedding/embedded.pyx @@ -0,0 +1,11 @@ +# embedded.pyx + +# The following two lines are for test purposed only, please ignore them. +# distutils: sources = embedded_main.c +# tag: py3only + +TEXT_TO_SAY = 'Hello from Python!' + +cdef public int say_hello_from_python() except -1: + print(TEXT_TO_SAY) + return 0 diff --git a/docs/examples/tutorial/embedding/embedded_main.c b/docs/examples/tutorial/embedding/embedded_main.c new file mode 100644 index 000000000..e14901a5e --- /dev/null +++ b/docs/examples/tutorial/embedding/embedded_main.c @@ -0,0 +1,69 @@ +/* embedded_main.c */ + +/* This include file is automatically generated by Cython for 'public' functions. */ +#include "embedded.h" + +#ifdef __cplusplus +extern "C" { +#endif + +int +main(int argc, char *argv[]) +{ + PyObject *pmodule; + wchar_t *program; + + program = Py_DecodeLocale(argv[0], NULL); + if (program == NULL) { + fprintf(stderr, "Fatal error: cannot decode argv[0], got %d arguments\n", argc); + exit(1); + } + + /* Add a built-in module, before Py_Initialize */ + if (PyImport_AppendInittab("embedded", PyInit_embedded) == -1) { + fprintf(stderr, "Error: could not extend in-built modules table\n"); + exit(1); + } + + /* Pass argv[0] to the Python interpreter */ + Py_SetProgramName(program); + + /* Initialize the Python interpreter. Required. + If this step fails, it will be a fatal error. */ + Py_Initialize(); + + /* Optionally import the module; alternatively, + import can be deferred until the embedded script + imports it. */ + pmodule = PyImport_ImportModule("embedded"); + if (!pmodule) { + PyErr_Print(); + fprintf(stderr, "Error: could not import module 'embedded'\n"); + goto exit_with_error; + } + + /* Now call into your module code. */ + if (say_hello_from_python() < 0) { + PyErr_Print(); + fprintf(stderr, "Error in Python code, exception was printed.\n"); + goto exit_with_error; + } + + /* ... */ + + /* Clean up after using CPython. */ + PyMem_RawFree(program); + Py_Finalize(); + + return 0; + + /* Clean up in the error cases above. */ +exit_with_error: + PyMem_RawFree(program); + Py_Finalize(); + return 1; +} + +#ifdef __cplusplus +} +#endif diff --git a/docs/examples/tutorial/external/atoi.pyx b/docs/examples/tutorial/external/atoi.pyx index 48643bbf2..96a67d73c 100644 --- a/docs/examples/tutorial/external/atoi.pyx +++ b/docs/examples/tutorial/external/atoi.pyx @@ -1,5 +1,5 @@ -from libc.stdlib cimport atoi
-
-cdef parse_charptr_to_py_int(char* s):
- assert s is not NULL, "byte string value is NULL"
- return atoi(s) # note: atoi() has no error detection!
+from libc.stdlib cimport atoi + +cdef parse_charptr_to_py_int(char* s): + assert s is not NULL, "byte string value is NULL" + return atoi(s) # note: atoi() has no error detection! diff --git a/docs/examples/tutorial/external/cpdef_sin.pyx b/docs/examples/tutorial/external/cpdef_sin.pyx index 47e09f433..eee2326a4 100644 --- a/docs/examples/tutorial/external/cpdef_sin.pyx +++ b/docs/examples/tutorial/external/cpdef_sin.pyx @@ -1,7 +1,7 @@ -"""
->>> sin(0)
-0.0
-"""
-
-cdef extern from "math.h":
- cpdef double sin(double x)
+""" +>>> sin(0) +0.0 +""" + +cdef extern from "math.h": + cpdef double sin(double x) diff --git a/docs/examples/tutorial/external/keyword_args.pyx b/docs/examples/tutorial/external/keyword_args.pyx index 327e4e08b..7c2a786cc 100644 --- a/docs/examples/tutorial/external/keyword_args.pyx +++ b/docs/examples/tutorial/external/keyword_args.pyx @@ -1,2 +1,2 @@ -cdef extern from "string.h":
- char* strstr(const char *haystack, const char *needle)
+cdef extern from "string.h": + char* strstr(const char *haystack, const char *needle) diff --git a/docs/examples/tutorial/external/keyword_args_call.pyx b/docs/examples/tutorial/external/keyword_args_call.pyx index 4be5f755d..de2b6f2b2 100644 --- a/docs/examples/tutorial/external/keyword_args_call.pyx +++ b/docs/examples/tutorial/external/keyword_args_call.pyx @@ -1,7 +1,7 @@ -cdef extern from "string.h":
- char* strstr(const char *haystack, const char *needle)
-
-cdef char* data = "hfvcakdfagbcffvschvxcdfgccbcfhvgcsnfxjh"
-
-cdef char* pos = strstr(needle='akd', haystack=data)
-print(pos is not NULL)
+cdef extern from "string.h": + char* strstr(const char *haystack, const char *needle) + +cdef char* data = "hfvcakdfagbcffvschvxcdfgccbcfhvgcsnfxjh" + +cdef char* pos = strstr(needle='akd', haystack=data) +print(pos is not NULL) diff --git a/docs/examples/tutorial/external/libc_sin.pyx b/docs/examples/tutorial/external/libc_sin.pyx index 25a4430e3..fb247c03d 100644 --- a/docs/examples/tutorial/external/libc_sin.pyx +++ b/docs/examples/tutorial/external/libc_sin.pyx @@ -1,4 +1,4 @@ -from libc.math cimport sin
-
-cdef double f(double x):
- return sin(x * x)
+from libc.math cimport sin + +cdef double f(double x): + return sin(x * x) diff --git a/docs/examples/tutorial/external/py_version_hex.pyx b/docs/examples/tutorial/external/py_version_hex.pyx index d732b00e7..e33f207c1 100644 --- a/docs/examples/tutorial/external/py_version_hex.pyx +++ b/docs/examples/tutorial/external/py_version_hex.pyx @@ -1,4 +1,4 @@ -from cpython.version cimport PY_VERSION_HEX
-
-# Python version >= 3.2 final ?
-print(PY_VERSION_HEX >= 0x030200F0)
+from cpython.version cimport PY_VERSION_HEX + +# Python version >= 3.2 final ? +print(PY_VERSION_HEX >= 0x030200F0) diff --git a/docs/examples/tutorial/external/setup.py b/docs/examples/tutorial/external/setup.py index 653214c84..289bc9534 100644 --- a/docs/examples/tutorial/external/setup.py +++ b/docs/examples/tutorial/external/setup.py @@ -1,13 +1,12 @@ -from distutils.core import setup
-from distutils.extension import Extension
-from Cython.Build import cythonize
-
-ext_modules = [
- Extension("demo",
- sources=["demo.pyx"],
- libraries=["m"] # Unix-like specific
- )
-]
-
-setup(name="Demos",
- ext_modules=cythonize(ext_modules))
+from setuptools import Extension, setup +from Cython.Build import cythonize + +ext_modules = [ + Extension("demo", + sources=["demo.pyx"], + libraries=["m"] # Unix-like specific + ) +] + +setup(name="Demos", + ext_modules=cythonize(ext_modules)) diff --git a/docs/examples/tutorial/memory_allocation/malloc.pyx b/docs/examples/tutorial/memory_allocation/malloc.pyx index e01a378e3..c185187d4 100644 --- a/docs/examples/tutorial/memory_allocation/malloc.pyx +++ b/docs/examples/tutorial/memory_allocation/malloc.pyx @@ -1,23 +1,23 @@ -import random
-from libc.stdlib cimport malloc, free
-
-def random_noise(int number=1):
- cdef int i
- # allocate number * sizeof(double) bytes of memory
- cdef double *my_array = <double *> malloc(number * sizeof(double))
- if not my_array:
- raise MemoryError()
-
- try:
- ran = random.normalvariate
- for i in range(number):
- my_array[i] = ran(0, 1)
-
- # ... let's just assume we do some more heavy C calculations here to make up
- # for the work that it takes to pack the C double values into Python float
- # objects below, right after throwing away the existing objects above.
-
- return [x for x in my_array[:number]]
- finally:
- # return the previously allocated memory to the system
- free(my_array)
+import random +from libc.stdlib cimport malloc, free + +def random_noise(int number=1): + cdef int i + # allocate number * sizeof(double) bytes of memory + cdef double *my_array = <double *> malloc(number * sizeof(double)) + if not my_array: + raise MemoryError() + + try: + ran = random.normalvariate + for i in range(number): + my_array[i] = ran(0, 1) + + # ... let's just assume we do some more heavy C calculations here to make up + # for the work that it takes to pack the C double values into Python float + # objects below, right after throwing away the existing objects above. + + return [x for x in my_array[:number]] + finally: + # return the previously allocated memory to the system + free(my_array) diff --git a/docs/examples/tutorial/memory_allocation/some_memory.pyx b/docs/examples/tutorial/memory_allocation/some_memory.pyx index 2e639ac4d..fb272a88d 100644 --- a/docs/examples/tutorial/memory_allocation/some_memory.pyx +++ b/docs/examples/tutorial/memory_allocation/some_memory.pyx @@ -1,25 +1,25 @@ -from cpython.mem cimport PyMem_Malloc, PyMem_Realloc, PyMem_Free
-
-cdef class SomeMemory:
-
- cdef double* data
-
- def __cinit__(self, size_t number):
- # allocate some memory (uninitialised, may contain arbitrary data)
- self.data = <double*> PyMem_Malloc(number * sizeof(double))
- if not self.data:
- raise MemoryError()
-
- def resize(self, size_t new_number):
- # Allocates new_number * sizeof(double) bytes,
- # preserving the current content and making a best-effort to
- # re-use the original data location.
- mem = <double*> PyMem_Realloc(self.data, new_number * sizeof(double))
- if not mem:
- raise MemoryError()
- # Only overwrite the pointer if the memory was really reallocated.
- # On error (mem is NULL), the originally memory has not been freed.
- self.data = mem
-
- def __dealloc__(self):
- PyMem_Free(self.data) # no-op if self.data is NULL
+from cpython.mem cimport PyMem_Malloc, PyMem_Realloc, PyMem_Free + +cdef class SomeMemory: + + cdef double* data + + def __cinit__(self, size_t number): + # allocate some memory (uninitialised, may contain arbitrary data) + self.data = <double*> PyMem_Malloc(number * sizeof(double)) + if not self.data: + raise MemoryError() + + def resize(self, size_t new_number): + # Allocates new_number * sizeof(double) bytes, + # preserving the current content and making a best-effort to + # re-use the original data location. + mem = <double*> PyMem_Realloc(self.data, new_number * sizeof(double)) + if not mem: + raise MemoryError() + # Only overwrite the pointer if the memory was really reallocated. + # On error (mem is NULL), the originally memory has not been freed. + self.data = mem + + def __dealloc__(self): + PyMem_Free(self.data) # no-op if self.data is NULL diff --git a/docs/examples/tutorial/numpy/convolve2.pyx b/docs/examples/tutorial/numpy/convolve2.pyx index be7512fe1..c122aeb3c 100644 --- a/docs/examples/tutorial/numpy/convolve2.pyx +++ b/docs/examples/tutorial/numpy/convolve2.pyx @@ -1,4 +1,4 @@ -# tag: numpy_old +# tag: numpy # You can ignore the previous line. # It's for internal testing of the cython documentation. diff --git a/docs/examples/tutorial/numpy/convolve_py.py b/docs/examples/tutorial/numpy/convolve_py.py index c3cbc5f86..39b276a04 100644 --- a/docs/examples/tutorial/numpy/convolve_py.py +++ b/docs/examples/tutorial/numpy/convolve_py.py @@ -1,43 +1,43 @@ -import numpy as np
-
-
-def naive_convolve(f, g):
- # f is an image and is indexed by (v, w)
- # g is a filter kernel and is indexed by (s, t),
- # it needs odd dimensions
- # h is the output image and is indexed by (x, y),
- # it is not cropped
- if g.shape[0] % 2 != 1 or g.shape[1] % 2 != 1:
- raise ValueError("Only odd dimensions on filter supported")
- # smid and tmid are number of pixels between the center pixel
- # and the edge, ie for a 5x5 filter they will be 2.
- #
- # The output size is calculated by adding smid, tmid to each
- # side of the dimensions of the input image.
- vmax = f.shape[0]
- wmax = f.shape[1]
- smax = g.shape[0]
- tmax = g.shape[1]
- smid = smax // 2
- tmid = tmax // 2
- xmax = vmax + 2 * smid
- ymax = wmax + 2 * tmid
- # Allocate result image.
- h = np.zeros([xmax, ymax], dtype=f.dtype)
- # Do convolution
- for x in range(xmax):
- for y in range(ymax):
- # Calculate pixel value for h at (x,y). Sum one component
- # for each pixel (s, t) of the filter g.
- s_from = max(smid - x, -smid)
- s_to = min((xmax - x) - smid, smid + 1)
- t_from = max(tmid - y, -tmid)
- t_to = min((ymax - y) - tmid, tmid + 1)
- value = 0
- for s in range(s_from, s_to):
- for t in range(t_from, t_to):
- v = x - smid + s
- w = y - tmid + t
- value += g[smid - s, tmid - t] * f[v, w]
- h[x, y] = value
- return h
+import numpy as np + + +def naive_convolve(f, g): + # f is an image and is indexed by (v, w) + # g is a filter kernel and is indexed by (s, t), + # it needs odd dimensions + # h is the output image and is indexed by (x, y), + # it is not cropped + if g.shape[0] % 2 != 1 or g.shape[1] % 2 != 1: + raise ValueError("Only odd dimensions on filter supported") + # smid and tmid are number of pixels between the center pixel + # and the edge, ie for a 5x5 filter they will be 2. + # + # The output size is calculated by adding smid, tmid to each + # side of the dimensions of the input image. + vmax = f.shape[0] + wmax = f.shape[1] + smax = g.shape[0] + tmax = g.shape[1] + smid = smax // 2 + tmid = tmax // 2 + xmax = vmax + 2 * smid + ymax = wmax + 2 * tmid + # Allocate result image. + h = np.zeros([xmax, ymax], dtype=f.dtype) + # Do convolution + for x in range(xmax): + for y in range(ymax): + # Calculate pixel value for h at (x,y). Sum one component + # for each pixel (s, t) of the filter g. + s_from = max(smid - x, -smid) + s_to = min((xmax - x) - smid, smid + 1) + t_from = max(tmid - y, -tmid) + t_to = min((ymax - y) - tmid, tmid + 1) + value = 0 + for s in range(s_from, s_to): + for t in range(t_from, t_to): + v = x - smid + s + w = y - tmid + t + value += g[smid - s, tmid - t] * f[v, w] + h[x, y] = value + return h diff --git a/docs/examples/tutorial/profiling_tutorial/calc_pi.py b/docs/examples/tutorial/profiling_tutorial/calc_pi.py index bc265e560..08c9ec2d4 100644 --- a/docs/examples/tutorial/profiling_tutorial/calc_pi.py +++ b/docs/examples/tutorial/profiling_tutorial/calc_pi.py @@ -1,10 +1,10 @@ -# calc_pi.py
-
-def recip_square(i):
- return 1. / i ** 2
-
-def approx_pi(n=10000000):
- val = 0.
- for k in range(1, n + 1):
- val += recip_square(k)
- return (6 * val) ** .5
+# calc_pi.py + +def recip_square(i): + return 1. / i ** 2 + +def approx_pi(n=10000000): + val = 0. + for k in range(1, n + 1): + val += recip_square(k) + return (6 * val) ** .5 diff --git a/docs/examples/tutorial/profiling_tutorial/calc_pi_2.pyx b/docs/examples/tutorial/profiling_tutorial/calc_pi_2.pyx index dab8d238d..c4c6a1bb9 100644 --- a/docs/examples/tutorial/profiling_tutorial/calc_pi_2.pyx +++ b/docs/examples/tutorial/profiling_tutorial/calc_pi_2.pyx @@ -1,13 +1,13 @@ -# cython: profile=True
-
-# calc_pi.pyx
-
-def recip_square(int i):
- return 1. / i ** 2
-
-def approx_pi(int n=10000000):
- cdef double val = 0.
- cdef int k
- for k in range(1, n + 1):
- val += recip_square(k)
- return (6 * val) ** .5
+# cython: profile=True + +# calc_pi.pyx + +def recip_square(int i): + return 1. / i ** 2 + +def approx_pi(int n=10000000): + cdef double val = 0. + cdef int k + for k in range(1, n + 1): + val += recip_square(k) + return (6 * val) ** .5 diff --git a/docs/examples/tutorial/profiling_tutorial/calc_pi_3.pyx b/docs/examples/tutorial/profiling_tutorial/calc_pi_3.pyx index 0f0bdb18a..f6b4f6546 100644 --- a/docs/examples/tutorial/profiling_tutorial/calc_pi_3.pyx +++ b/docs/examples/tutorial/profiling_tutorial/calc_pi_3.pyx @@ -1,13 +1,13 @@ -# cython: profile=True
-
-# calc_pi.pyx
-
-cdef inline double recip_square(int i):
- return 1. / (i * i)
-
-def approx_pi(int n=10000000):
- cdef double val = 0.
- cdef int k
- for k in range(1, n + 1):
- val += recip_square(k)
- return (6 * val) ** .5
+# cython: profile=True + +# calc_pi.pyx + +cdef inline double recip_square(int i): + return 1. / (i * i) + +def approx_pi(int n=10000000): + cdef double val = 0. + cdef int k + for k in range(1, n + 1): + val += recip_square(k) + return (6 * val) ** .5 diff --git a/docs/examples/tutorial/profiling_tutorial/calc_pi_4.pyx b/docs/examples/tutorial/profiling_tutorial/calc_pi_4.pyx index ab3f9ea9f..054018408 100644 --- a/docs/examples/tutorial/profiling_tutorial/calc_pi_4.pyx +++ b/docs/examples/tutorial/profiling_tutorial/calc_pi_4.pyx @@ -1,16 +1,16 @@ -# cython: profile=True
-
-# calc_pi.pyx
-
-cimport cython
-
-@cython.profile(False)
-cdef inline double recip_square(int i):
- return 1. / (i * i)
-
-def approx_pi(int n=10000000):
- cdef double val = 0.
- cdef int k
- for k in range(1, n + 1):
- val += recip_square(k)
- return (6 * val) ** .5
+# cython: profile=True + +# calc_pi.pyx + +cimport cython + +@cython.profile(False) +cdef inline double recip_square(int i): + return 1. / (i * i) + +def approx_pi(int n=10000000): + cdef double val = 0. + cdef int k + for k in range(1, n + 1): + val += recip_square(k) + return (6 * val) ** .5 diff --git a/docs/examples/tutorial/profiling_tutorial/often_called.pyx b/docs/examples/tutorial/profiling_tutorial/often_called.pyx index 77c689c9c..3699dce4f 100644 --- a/docs/examples/tutorial/profiling_tutorial/often_called.pyx +++ b/docs/examples/tutorial/profiling_tutorial/often_called.pyx @@ -1,5 +1,5 @@ -cimport cython
-
-@cython.profile(False)
-def my_often_called_function():
- pass
+cimport cython + +@cython.profile(False) +def my_often_called_function(): + pass diff --git a/docs/examples/tutorial/profiling_tutorial/profile.py b/docs/examples/tutorial/profiling_tutorial/profile.py index 1c12bf971..60e90c8ba 100644 --- a/docs/examples/tutorial/profiling_tutorial/profile.py +++ b/docs/examples/tutorial/profiling_tutorial/profile.py @@ -1,10 +1,10 @@ -# profile.py
-
-import pstats, cProfile
-
-import calc_pi
-
-cProfile.runctx("calc_pi.approx_pi()", globals(), locals(), "Profile.prof")
-
-s = pstats.Stats("Profile.prof")
-s.strip_dirs().sort_stats("time").print_stats()
+# profile.py + +import pstats, cProfile + +import calc_pi + +cProfile.runctx("calc_pi.approx_pi()", globals(), locals(), "Profile.prof") + +s = pstats.Stats("Profile.prof") +s.strip_dirs().sort_stats("time").print_stats() diff --git a/docs/examples/tutorial/profiling_tutorial/profile_2.py b/docs/examples/tutorial/profiling_tutorial/profile_2.py index 38fb50104..b0e91b07f 100644 --- a/docs/examples/tutorial/profiling_tutorial/profile_2.py +++ b/docs/examples/tutorial/profiling_tutorial/profile_2.py @@ -1,13 +1,13 @@ -# profile.py
-
-import pstats, cProfile
-
-import pyximport
-pyximport.install()
-
-import calc_pi
-
-cProfile.runctx("calc_pi.approx_pi()", globals(), locals(), "Profile.prof")
-
-s = pstats.Stats("Profile.prof")
-s.strip_dirs().sort_stats("time").print_stats()
+# profile.py + +import pstats, cProfile + +import pyximport +pyximport.install() + +import calc_pi + +cProfile.runctx("calc_pi.approx_pi()", globals(), locals(), "Profile.prof") + +s = pstats.Stats("Profile.prof") +s.strip_dirs().sort_stats("time").print_stats() diff --git a/docs/examples/tutorial/pure/A.py b/docs/examples/tutorial/pure/A.py index 1e0cea950..98d5530c8 100644 --- a/docs/examples/tutorial/pure/A.py +++ b/docs/examples/tutorial/pure/A.py @@ -1,14 +1,14 @@ -def myfunction(x, y=2):
- a = x - y
- return a + x * y
-
-def _helper(a):
- return a + 1
-
-class A:
- def __init__(self, b=0):
- self.a = 3
- self.b = b
-
- def foo(self, x):
- print(x + _helper(1.0))
+def myfunction(x, y=2): + a = x - y + return a + x * y + +def _helper(a): + return a + 1 + +class A: + def __init__(self, b=0): + self.a = 3 + self.b = b + + def foo(self, x): + print(x + _helper(1.0)) diff --git a/docs/examples/tutorial/pure/A_equivalent.pyx b/docs/examples/tutorial/pure/A_equivalent.pyx index 1b256a010..ab9e0081c 100644 --- a/docs/examples/tutorial/pure/A_equivalent.pyx +++ b/docs/examples/tutorial/pure/A_equivalent.pyx @@ -1,15 +1,15 @@ -cpdef int myfunction(int x, int y=2):
- a = x - y
- return a + x * y
-
-cdef double _helper(double a):
- return a + 1
-
-cdef class A:
- cdef public int a, b
- def __init__(self, b=0):
- self.a = 3
- self.b = b
-
- cpdef foo(self, double x):
- print(x + _helper(1.0))
+cpdef int myfunction(int x, int y=2): + a = x - y + return a + x * y + +cdef double _helper(double a): + return a + 1 + +cdef class A: + cdef public int a, b + def __init__(self, b=0): + self.a = 3 + self.b = b + + cpdef foo(self, double x): + print(x + _helper(1.0)) diff --git a/docs/examples/tutorial/pure/annotations.py b/docs/examples/tutorial/pure/annotations.py index 2b8487c0b..09682c352 100644 --- a/docs/examples/tutorial/pure/annotations.py +++ b/docs/examples/tutorial/pure/annotations.py @@ -1,5 +1,5 @@ -import cython
-
-def func(foo: dict, bar: cython.int) -> tuple:
- foo["hello world"] = 3 + bar
- return foo, 5
+import cython + +def func(foo: dict, bar: cython.int) -> tuple: + foo["hello world"] = 3 + bar + return foo, 5 diff --git a/docs/examples/tutorial/pure/c_arrays.py b/docs/examples/tutorial/pure/c_arrays.py index 33067da68..f221b7ae8 100644 --- a/docs/examples/tutorial/pure/c_arrays.py +++ b/docs/examples/tutorial/pure/c_arrays.py @@ -1,15 +1,15 @@ -import cython
-
-
-@cython.locals(counts=cython.int[10], digit=cython.int)
-def count_digits(digits):
- """
- >>> digits = '01112222333334445667788899'
- >>> count_digits(map(int, digits))
- [1, 3, 4, 5, 3, 1, 2, 2, 3, 2]
- """
- counts = [0] * 10
- for digit in digits:
- assert 0 <= digit <= 9
- counts[digit] += 1
- return counts
+import cython + + +@cython.locals(counts=cython.int[10], digit=cython.int) +def count_digits(digits): + """ + >>> digits = '01112222333334445667788899' + >>> count_digits(map(int, digits)) + [1, 3, 4, 5, 3, 1, 2, 2, 3, 2] + """ + counts = [0] * 10 + for digit in digits: + assert 0 <= digit <= 9 + counts[digit] += 1 + return counts diff --git a/docs/examples/tutorial/pure/cclass.py b/docs/examples/tutorial/pure/cclass.py index 61c65183d..7f9cb1a04 100644 --- a/docs/examples/tutorial/pure/cclass.py +++ b/docs/examples/tutorial/pure/cclass.py @@ -1,16 +1,16 @@ -import cython
-
-
-@cython.cclass
-class A:
- cython.declare(a=cython.int, b=cython.int)
- c = cython.declare(cython.int, visibility='public')
- d = cython.declare(cython.int) # private by default.
- e = cython.declare(cython.int, visibility='readonly')
-
- def __init__(self, a, b, c, d=5, e=3):
- self.a = a
- self.b = b
- self.c = c
- self.d = d
- self.e = e
+import cython + + +@cython.cclass +class A: + cython.declare(a=cython.int, b=cython.int) + c = cython.declare(cython.int, visibility='public') + d = cython.declare(cython.int) # private by default. + e = cython.declare(cython.int, visibility='readonly') + + def __init__(self, a, b, c, d=5, e=3): + self.a = a + self.b = b + self.c = c + self.d = d + self.e = e diff --git a/docs/examples/tutorial/pure/compiled_switch.py b/docs/examples/tutorial/pure/compiled_switch.py index 47d62a3e6..a35cac2c6 100644 --- a/docs/examples/tutorial/pure/compiled_switch.py +++ b/docs/examples/tutorial/pure/compiled_switch.py @@ -1,6 +1,6 @@ -import cython
-
-if cython.compiled:
- print("Yep, I'm compiled.")
-else:
- print("Just a lowly interpreted script.")
+import cython + +if cython.compiled: + print("Yep, I'm compiled.") +else: + print("Just a lowly interpreted script.") diff --git a/docs/examples/tutorial/pure/cython_declare.py b/docs/examples/tutorial/pure/cython_declare.py index cf6d58bba..50a4ab8aa 100644 --- a/docs/examples/tutorial/pure/cython_declare.py +++ b/docs/examples/tutorial/pure/cython_declare.py @@ -1,4 +1,4 @@ -import cython
-
-x = cython.declare(cython.int) # cdef int x
-y = cython.declare(cython.double, 0.57721) # cdef double y = 0.57721
+import cython + +x = cython.declare(cython.int) # cdef int x +y = cython.declare(cython.double, 0.57721) # cdef double y = 0.57721 diff --git a/docs/examples/tutorial/pure/cython_declare2.py b/docs/examples/tutorial/pure/cython_declare2.py index 35fae7d7b..ee491d62b 100644 --- a/docs/examples/tutorial/pure/cython_declare2.py +++ b/docs/examples/tutorial/pure/cython_declare2.py @@ -1,3 +1,3 @@ -import cython
-
-cython.declare(x=cython.int, y=cython.double) # cdef int x; cdef double y
+import cython + +cython.declare(x=cython.int, y=cython.double) # cdef int x; cdef double y diff --git a/docs/examples/tutorial/pure/dostuff.py b/docs/examples/tutorial/pure/dostuff.py index 748c31b10..7a88533c5 100644 --- a/docs/examples/tutorial/pure/dostuff.py +++ b/docs/examples/tutorial/pure/dostuff.py @@ -1,5 +1,5 @@ -def dostuff(n):
- t = 0
- for i in range(n):
- t += i
- return t
+def dostuff(n): + t = 0 + for i in range(n): + t += i + return t diff --git a/docs/examples/tutorial/pure/exceptval.py b/docs/examples/tutorial/pure/exceptval.py index 8bf564040..3d991f7c1 100644 --- a/docs/examples/tutorial/pure/exceptval.py +++ b/docs/examples/tutorial/pure/exceptval.py @@ -1,7 +1,7 @@ -import cython
-
-@cython.exceptval(-1)
-def func(x: cython.int) -> cython.int:
- if x < 0:
- raise ValueError("need integer >= 0")
- return x + 1
+import cython + +@cython.exceptval(-1) +def func(x: cython.int) -> cython.int: + if x < 0: + raise ValueError("need integer >= 0") + return x + 1 diff --git a/docs/examples/tutorial/pure/locals.py b/docs/examples/tutorial/pure/locals.py index 8eda7114a..b273a9ebe 100644 --- a/docs/examples/tutorial/pure/locals.py +++ b/docs/examples/tutorial/pure/locals.py @@ -1,6 +1,6 @@ -import cython
-
-@cython.locals(a=cython.long, b=cython.long, n=cython.longlong)
-def foo(a, b, x, y):
- n = a * b
- # ...
+import cython + +@cython.locals(a=cython.long, b=cython.long, n=cython.longlong) +def foo(a, b, x, y): + n = a * b + # ... diff --git a/docs/examples/tutorial/pure/mymodule.py b/docs/examples/tutorial/pure/mymodule.py index 62d4c76ac..83f5cdc28 100644 --- a/docs/examples/tutorial/pure/mymodule.py +++ b/docs/examples/tutorial/pure/mymodule.py @@ -1,10 +1,10 @@ -# mymodule.py
-
-import cython
-
-# override with Python import if not in compiled code
-if not cython.compiled:
- from math import sin
-
-# calls sin() from math.h when compiled with Cython and math.sin() in Python
-print(sin(0))
+# mymodule.py + +import cython + +# override with Python import if not in compiled code +if not cython.compiled: + from math import sin + +# calls sin() from math.h when compiled with Cython and math.sin() in Python +print(sin(0)) diff --git a/docs/examples/tutorial/pure/pep_526.py b/docs/examples/tutorial/pure/pep_526.py index 163d97859..ecb3bfcdf 100644 --- a/docs/examples/tutorial/pure/pep_526.py +++ b/docs/examples/tutorial/pure/pep_526.py @@ -1,22 +1,22 @@ -import cython
-
-def func():
- # Cython types are evaluated as for cdef declarations
- x: cython.int # cdef int x
- y: cython.double = 0.57721 # cdef double y = 0.57721
- z: cython.float = 0.57721 # cdef float z = 0.57721
-
- # Python types shadow Cython types for compatibility reasons
- a: float = 0.54321 # cdef double a = 0.54321
- b: int = 5 # cdef object b = 5
- c: long = 6 # cdef object c = 6
- pass
-
-@cython.cclass
-class A:
- a: cython.int
- b: cython.int
-
- def __init__(self, b=0):
- self.a = 3
- self.b = b
+import cython + +def func(): + # Cython types are evaluated as for cdef declarations + x: cython.int # cdef int x + y: cython.double = 0.57721 # cdef double y = 0.57721 + z: cython.float = 0.57721 # cdef float z = 0.57721 + + # Python types shadow Cython types for compatibility reasons + a: float = 0.54321 # cdef double a = 0.54321 + b: int = 5 # cdef object b = 5 + c: long = 6 # cdef object c = 6 + pass + +@cython.cclass +class A: + a: cython.int + b: cython.int + + def __init__(self, b=0): + self.a = 3 + self.b = b diff --git a/docs/examples/tutorial/string/api_func.pyx b/docs/examples/tutorial/string/api_func.pyx index ec6b27751..c9e05f9e3 100644 --- a/docs/examples/tutorial/string/api_func.pyx +++ b/docs/examples/tutorial/string/api_func.pyx @@ -1,5 +1,5 @@ -from to_unicode cimport _text
-
-def api_func(s):
- text_input = _text(s)
- # ...
+from to_unicode cimport _text + +def api_func(s): + text_input = _text(s) + # ... diff --git a/docs/examples/tutorial/string/arg_memview.pyx b/docs/examples/tutorial/string/arg_memview.pyx index 63a18f943..e2b6d75be 100644 --- a/docs/examples/tutorial/string/arg_memview.pyx +++ b/docs/examples/tutorial/string/arg_memview.pyx @@ -1,5 +1,5 @@ -def process_byte_data(unsigned char[:] data):
- length = data.shape[0]
- first_byte = data[0]
- slice_view = data[1:-1]
- # ...
+def process_byte_data(unsigned char[:] data): + length = data.shape[0] + first_byte = data[0] + slice_view = data[1:-1] + # ... diff --git a/docs/examples/tutorial/string/auto_conversion_1.pyx b/docs/examples/tutorial/string/auto_conversion_1.pyx index 929c03d68..ee2b116e4 100644 --- a/docs/examples/tutorial/string/auto_conversion_1.pyx +++ b/docs/examples/tutorial/string/auto_conversion_1.pyx @@ -1,9 +1,9 @@ -# cython: c_string_type=unicode, c_string_encoding=utf8
-
-cdef char* c_string = 'abcdefg'
-
-# implicit decoding:
-cdef object py_unicode_object = c_string
-
-# explicit conversion to Python bytes:
-py_bytes_object = <bytes>c_string
+# cython: c_string_type=unicode, c_string_encoding=utf8 + +cdef char* c_string = 'abcdefg' + +# implicit decoding: +cdef object py_unicode_object = c_string + +# explicit conversion to Python bytes: +py_bytes_object = <bytes>c_string diff --git a/docs/examples/tutorial/string/auto_conversion_2.pyx b/docs/examples/tutorial/string/auto_conversion_2.pyx index 84436661d..9f7a5ad04 100644 --- a/docs/examples/tutorial/string/auto_conversion_2.pyx +++ b/docs/examples/tutorial/string/auto_conversion_2.pyx @@ -1,12 +1,12 @@ -# cython: c_string_type=str, c_string_encoding=ascii
-
-cdef char* c_string = 'abcdefg'
-
-# implicit decoding in Py3, bytes conversion in Py2:
-cdef object py_str_object = c_string
-
-# explicit conversion to Python bytes:
-py_bytes_object = <bytes>c_string
-
-# explicit conversion to Python unicode:
-py_bytes_object = <unicode>c_string
+# cython: c_string_type=str, c_string_encoding=ascii + +cdef char* c_string = 'abcdefg' + +# implicit decoding in Py3, bytes conversion in Py2: +cdef object py_str_object = c_string + +# explicit conversion to Python bytes: +py_bytes_object = <bytes>c_string + +# explicit conversion to Python unicode: +py_bytes_object = <unicode>c_string diff --git a/docs/examples/tutorial/string/auto_conversion_3.pyx b/docs/examples/tutorial/string/auto_conversion_3.pyx index 509d7e324..b9a1b7517 100644 --- a/docs/examples/tutorial/string/auto_conversion_3.pyx +++ b/docs/examples/tutorial/string/auto_conversion_3.pyx @@ -1,6 +1,6 @@ -# cython: c_string_type=unicode, c_string_encoding=ascii
-
-def func():
- ustring = u'abc'
- cdef char* s = ustring
- return s[0] # returns u'a'
+# cython: c_string_type=unicode, c_string_encoding=ascii + +def func(): + ustring = u'abc' + cdef char* s = ustring + return s[0] # returns u'a' diff --git a/docs/examples/tutorial/string/c_func.pyx b/docs/examples/tutorial/string/c_func.pyx index a456b815b..3c94587fe 100644 --- a/docs/examples/tutorial/string/c_func.pyx +++ b/docs/examples/tutorial/string/c_func.pyx @@ -1,22 +1,22 @@ -from libc.stdlib cimport malloc
-from libc.string cimport strcpy, strlen
-
-cdef char* hello_world = 'hello world'
-cdef Py_ssize_t n = strlen(hello_world)
-
-
-cdef char* c_call_returning_a_c_string():
- cdef char* c_string = <char *> malloc((n + 1) * sizeof(char))
- if not c_string:
- raise MemoryError()
- strcpy(c_string, hello_world)
- return c_string
-
-
-cdef void get_a_c_string(char** c_string_ptr, Py_ssize_t *length):
- c_string_ptr[0] = <char *> malloc((n + 1) * sizeof(char))
- if not c_string_ptr[0]:
- raise MemoryError()
-
- strcpy(c_string_ptr[0], hello_world)
- length[0] = n
+from libc.stdlib cimport malloc +from libc.string cimport strcpy, strlen + +cdef char* hello_world = 'hello world' +cdef Py_ssize_t n = strlen(hello_world) + + +cdef char* c_call_returning_a_c_string(): + cdef char* c_string = <char *> malloc((n + 1) * sizeof(char)) + if not c_string: + raise MemoryError() + strcpy(c_string, hello_world) + return c_string + + +cdef void get_a_c_string(char** c_string_ptr, Py_ssize_t *length): + c_string_ptr[0] = <char *> malloc((n + 1) * sizeof(char)) + if not c_string_ptr[0]: + raise MemoryError() + + strcpy(c_string_ptr[0], hello_world) + length[0] = n diff --git a/docs/examples/tutorial/string/const.pyx b/docs/examples/tutorial/string/const.pyx index e066c77ac..0b89b9e41 100644 --- a/docs/examples/tutorial/string/const.pyx +++ b/docs/examples/tutorial/string/const.pyx @@ -1,4 +1,4 @@ -cdef extern from "someheader.h":
- ctypedef const char specialChar
- int process_string(const char* s)
- const unsigned char* look_up_cached_string(const unsigned char* key)
+cdef extern from "someheader.h": + ctypedef const char specialChar + int process_string(const char* s) + const unsigned char* look_up_cached_string(const unsigned char* key) diff --git a/docs/examples/tutorial/string/cpp_string.pyx b/docs/examples/tutorial/string/cpp_string.pyx index b6c9e44f9..313e72a67 100644 --- a/docs/examples/tutorial/string/cpp_string.pyx +++ b/docs/examples/tutorial/string/cpp_string.pyx @@ -1,12 +1,12 @@ -# distutils: language = c++
-
-from libcpp.string cimport string
-
-def get_bytes():
- py_bytes_object = b'hello world'
- cdef string s = py_bytes_object
-
- s.append('abc')
- py_bytes_object = s
- return py_bytes_object
-
+# distutils: language = c++ + +from libcpp.string cimport string + +def get_bytes(): + py_bytes_object = b'hello world' + cdef string s = py_bytes_object + + s.append('abc') + py_bytes_object = s + return py_bytes_object + diff --git a/docs/examples/tutorial/string/decode.pyx b/docs/examples/tutorial/string/decode.pyx index c3e2faf68..79fc41835 100644 --- a/docs/examples/tutorial/string/decode.pyx +++ b/docs/examples/tutorial/string/decode.pyx @@ -1,9 +1,9 @@ -from c_func cimport get_a_c_string
-
-cdef char* c_string = NULL
-cdef Py_ssize_t length = 0
-
-# get pointer and length from a C function
-get_a_c_string(&c_string, &length)
-
-ustring = c_string[:length].decode('UTF-8')
+from c_func cimport get_a_c_string + +cdef char* c_string = NULL +cdef Py_ssize_t length = 0 + +# get pointer and length from a C function +get_a_c_string(&c_string, &length) + +ustring = c_string[:length].decode('UTF-8') diff --git a/docs/examples/tutorial/string/decode_cpp_string.pyx b/docs/examples/tutorial/string/decode_cpp_string.pyx index 8f1d01af8..52861c209 100644 --- a/docs/examples/tutorial/string/decode_cpp_string.pyx +++ b/docs/examples/tutorial/string/decode_cpp_string.pyx @@ -1,10 +1,10 @@ -# distutils: language = c++
-
-from libcpp.string cimport string
-
-def get_ustrings():
- cdef string s = string(b'abcdefg')
-
- ustring1 = s.decode('UTF-8')
- ustring2 = s[2:-2].decode('UTF-8')
- return ustring1, ustring2
+# distutils: language = c++ + +from libcpp.string cimport string + +def get_ustrings(): + cdef string s = string(b'abcdefg') + + ustring1 = s.decode('UTF-8') + ustring2 = s[2:-2].decode('UTF-8') + return ustring1, ustring2 diff --git a/docs/examples/tutorial/string/for_bytes.pyx b/docs/examples/tutorial/string/for_bytes.pyx index 69e9202ae..d4d3e1f81 100644 --- a/docs/examples/tutorial/string/for_bytes.pyx +++ b/docs/examples/tutorial/string/for_bytes.pyx @@ -1,6 +1,6 @@ -cdef bytes bytes_string = b"hello to A bytes' world"
-
-cdef char c
-for c in bytes_string:
- if c == 'A':
- print("Found the letter A")
+cdef bytes bytes_string = b"hello to A bytes' world" + +cdef char c +for c in bytes_string: + if c == 'A': + print("Found the letter A") diff --git a/docs/examples/tutorial/string/for_char.pyx b/docs/examples/tutorial/string/for_char.pyx index adc16bcd8..81abae97c 100644 --- a/docs/examples/tutorial/string/for_char.pyx +++ b/docs/examples/tutorial/string/for_char.pyx @@ -1,6 +1,6 @@ -cdef char* c_string = "Hello to A C-string's world"
-
-cdef char c
-for c in c_string[:11]:
- if c == 'A':
- print("Found the letter A")
+cdef char* c_string = "Hello to A C-string's world" + +cdef char c +for c in c_string[:11]: + if c == 'A': + print("Found the letter A") diff --git a/docs/examples/tutorial/string/for_unicode.pyx b/docs/examples/tutorial/string/for_unicode.pyx index aabd562e4..0f8ab0c7d 100644 --- a/docs/examples/tutorial/string/for_unicode.pyx +++ b/docs/examples/tutorial/string/for_unicode.pyx @@ -1,6 +1,6 @@ -cdef unicode ustring = u'Hello world'
-
-# NOTE: no typing required for 'uchar' !
-for uchar in ustring:
- if uchar == u'A':
- print("Found the letter A")
+cdef unicode ustring = u'Hello world' + +# NOTE: no typing required for 'uchar' ! +for uchar in ustring: + if uchar == u'A': + print("Found the letter A") diff --git a/docs/examples/tutorial/string/if_char_in.pyx b/docs/examples/tutorial/string/if_char_in.pyx index 73521b2de..e33e18d59 100644 --- a/docs/examples/tutorial/string/if_char_in.pyx +++ b/docs/examples/tutorial/string/if_char_in.pyx @@ -1,5 +1,5 @@ -cpdef void is_in(Py_UCS4 uchar_val):
- if uchar_val in u'abcABCxY':
- print("The character is in the string.")
- else:
- print("The character is not in the string")
+cpdef void is_in(Py_UCS4 uchar_val): + if uchar_val in u'abcABCxY': + print("The character is in the string.") + else: + print("The character is not in the string") diff --git a/docs/examples/tutorial/string/naive_decode.pyx b/docs/examples/tutorial/string/naive_decode.pyx index 186d2affa..b3c44d9af 100644 --- a/docs/examples/tutorial/string/naive_decode.pyx +++ b/docs/examples/tutorial/string/naive_decode.pyx @@ -1,4 +1,4 @@ -from c_func cimport c_call_returning_a_c_string
-
-cdef char* some_c_string = c_call_returning_a_c_string()
-ustring = some_c_string.decode('UTF-8')
+from c_func cimport c_call_returning_a_c_string + +cdef char* some_c_string = c_call_returning_a_c_string() +ustring = some_c_string.decode('UTF-8') diff --git a/docs/examples/tutorial/string/return_memview.pyx b/docs/examples/tutorial/string/return_memview.pyx index f6233436a..be9b597a4 100644 --- a/docs/examples/tutorial/string/return_memview.pyx +++ b/docs/examples/tutorial/string/return_memview.pyx @@ -1,9 +1,9 @@ -def process_byte_data(unsigned char[:] data):
- # ... process the data, here, dummy processing.
- cdef bint return_all = (data[0] == 108)
-
- if return_all:
- return bytes(data)
- else:
- # example for returning a slice
- return bytes(data[5:7])
+def process_byte_data(unsigned char[:] data): + # ... process the data, here, dummy processing. + cdef bint return_all = (data[0] == 108) + + if return_all: + return bytes(data) + else: + # example for returning a slice + return bytes(data[5:7]) diff --git a/docs/examples/tutorial/string/slicing_c_string.pyx b/docs/examples/tutorial/string/slicing_c_string.pyx index 2e937430e..f8d272e32 100644 --- a/docs/examples/tutorial/string/slicing_c_string.pyx +++ b/docs/examples/tutorial/string/slicing_c_string.pyx @@ -1,15 +1,15 @@ -from libc.stdlib cimport free
-from c_func cimport get_a_c_string
-
-
-def main():
- cdef char* c_string = NULL
- cdef Py_ssize_t length = 0
-
- # get pointer and length from a C function
- get_a_c_string(&c_string, &length)
-
- try:
- py_bytes_string = c_string[:length] # Performs a copy of the data
- finally:
- free(c_string)
+from libc.stdlib cimport free +from c_func cimport get_a_c_string + + +def main(): + cdef char* c_string = NULL + cdef Py_ssize_t length = 0 + + # get pointer and length from a C function + get_a_c_string(&c_string, &length) + + try: + py_bytes_string = c_string[:length] # Performs a copy of the data + finally: + free(c_string) diff --git a/docs/examples/tutorial/string/to_char.pyx b/docs/examples/tutorial/string/to_char.pyx index 5e4a16161..7912ea485 100644 --- a/docs/examples/tutorial/string/to_char.pyx +++ b/docs/examples/tutorial/string/to_char.pyx @@ -1,8 +1,8 @@ -# define a global name for whatever char type is used in the module
-ctypedef unsigned char char_type
-
-cdef char_type[:] _chars(s):
- if isinstance(s, unicode):
- # encode to the specific encoding used inside of the module
- s = (<unicode>s).encode('utf8')
- return s
+# define a global name for whatever char type is used in the module +ctypedef unsigned char char_type + +cdef char_type[:] _chars(s): + if isinstance(s, unicode): + # encode to the specific encoding used inside of the module + s = (<unicode>s).encode('utf8') + return s diff --git a/docs/examples/tutorial/string/to_unicode.pyx b/docs/examples/tutorial/string/to_unicode.pyx index 8ab8f2662..188a8776e 100644 --- a/docs/examples/tutorial/string/to_unicode.pyx +++ b/docs/examples/tutorial/string/to_unicode.pyx @@ -1,22 +1,22 @@ -# to_unicode.pyx
-
-from cpython.version cimport PY_MAJOR_VERSION
-
-cdef unicode _text(s):
- if type(s) is unicode:
- # Fast path for most common case(s).
- return <unicode>s
-
- elif PY_MAJOR_VERSION < 3 and isinstance(s, bytes):
- # Only accept byte strings as text input in Python 2.x, not in Py3.
- return (<bytes>s).decode('ascii')
-
- elif isinstance(s, unicode):
- # We know from the fast path above that 's' can only be a subtype here.
- # An evil cast to <unicode> might still work in some(!) cases,
- # depending on what the further processing does. To be safe,
- # we can always create a copy instead.
- return unicode(s)
-
- else:
- raise TypeError("Could not convert to unicode.")
+# to_unicode.pyx + +from cpython.version cimport PY_MAJOR_VERSION + +cdef unicode _text(s): + if type(s) is unicode: + # Fast path for most common case(s). + return <unicode>s + + elif PY_MAJOR_VERSION < 3 and isinstance(s, bytes): + # Only accept byte strings as text input in Python 2.x, not in Py3. + return (<bytes>s).decode('ascii') + + elif isinstance(s, unicode): + # We know from the fast path above that 's' can only be a subtype here. + # An evil cast to <unicode> might still work in some(!) cases, + # depending on what the further processing does. To be safe, + # we can always create a copy instead. + return unicode(s) + + else: + raise TypeError("Could not convert to unicode.") diff --git a/docs/examples/tutorial/string/try_finally.pyx b/docs/examples/tutorial/string/try_finally.pyx index aea786f5b..4cf943e56 100644 --- a/docs/examples/tutorial/string/try_finally.pyx +++ b/docs/examples/tutorial/string/try_finally.pyx @@ -1,9 +1,9 @@ -from libc.stdlib cimport free
-from c_func cimport c_call_returning_a_c_string
-
-cdef bytes py_string
-cdef char* c_string = c_call_returning_a_c_string()
-try:
- py_string = c_string
-finally:
- free(c_string)
+from libc.stdlib cimport free +from c_func cimport c_call_returning_a_c_string + +cdef bytes py_string +cdef char* c_string = c_call_returning_a_c_string() +try: + py_string = c_string +finally: + free(c_string) diff --git a/docs/examples/tutorial/string/utf_eight.pyx b/docs/examples/tutorial/string/utf_eight.pyx index 7113947f4..654c51ae8 100644 --- a/docs/examples/tutorial/string/utf_eight.pyx +++ b/docs/examples/tutorial/string/utf_eight.pyx @@ -1,15 +1,15 @@ -from libc.stdlib cimport free
-
-cdef unicode tounicode(char* s):
- return s.decode('UTF-8', 'strict')
-
-cdef unicode tounicode_with_length(
- char* s, size_t length):
- return s[:length].decode('UTF-8', 'strict')
-
-cdef unicode tounicode_with_length_and_free(
- char* s, size_t length):
- try:
- return s[:length].decode('UTF-8', 'strict')
- finally:
+from libc.stdlib cimport free + +cdef unicode tounicode(char* s): + return s.decode('UTF-8', 'strict') + +cdef unicode tounicode_with_length( + char* s, size_t length): + return s[:length].decode('UTF-8', 'strict') + +cdef unicode tounicode_with_length_and_free( + char* s, size_t length): + try: + return s[:length].decode('UTF-8', 'strict') + finally: free(s)
\ No newline at end of file diff --git a/docs/examples/userguide/buffer/matrix.pyx b/docs/examples/userguide/buffer/matrix.pyx index abdb2d3c7..ca597c2f2 100644 --- a/docs/examples/userguide/buffer/matrix.pyx +++ b/docs/examples/userguide/buffer/matrix.pyx @@ -1,16 +1,16 @@ -# distutils: language = c++
-
-# matrix.pyx
-
-from libcpp.vector cimport vector
-
-cdef class Matrix:
- cdef unsigned ncols
- cdef vector[float] v
-
- def __cinit__(self, unsigned ncols):
- self.ncols = ncols
-
- def add_row(self):
- """Adds a row, initially zero-filled."""
- self.v.resize(self.v.size() + self.ncols)
+# distutils: language = c++ + +# matrix.pyx + +from libcpp.vector cimport vector + +cdef class Matrix: + cdef unsigned ncols + cdef vector[float] v + + def __cinit__(self, unsigned ncols): + self.ncols = ncols + + def add_row(self): + """Adds a row, initially zero-filled.""" + self.v.resize(self.v.size() + self.ncols) diff --git a/docs/examples/userguide/buffer/matrix_with_buffer.pyx b/docs/examples/userguide/buffer/matrix_with_buffer.pyx index 985991cbe..c355f0fe8 100644 --- a/docs/examples/userguide/buffer/matrix_with_buffer.pyx +++ b/docs/examples/userguide/buffer/matrix_with_buffer.pyx @@ -1,45 +1,45 @@ -# distutils: language = c++
-
-from cpython cimport Py_buffer
-from libcpp.vector cimport vector
-
-cdef class Matrix:
- cdef Py_ssize_t ncols
- cdef Py_ssize_t shape[2]
- cdef Py_ssize_t strides[2]
- cdef vector[float] v
-
- def __cinit__(self, Py_ssize_t ncols):
- self.ncols = ncols
-
- def add_row(self):
- """Adds a row, initially zero-filled."""
- self.v.resize(self.v.size() + self.ncols)
-
- def __getbuffer__(self, Py_buffer *buffer, int flags):
- cdef Py_ssize_t itemsize = sizeof(self.v[0])
-
- self.shape[0] = self.v.size() / self.ncols
- self.shape[1] = self.ncols
-
- # Stride 1 is the distance, in bytes, between two items in a row;
- # this is the distance between two adjacent items in the vector.
- # Stride 0 is the distance between the first elements of adjacent rows.
- self.strides[1] = <Py_ssize_t>( <char *>&(self.v[1])
- - <char *>&(self.v[0]))
- self.strides[0] = self.ncols * self.strides[1]
-
- buffer.buf = <char *>&(self.v[0])
- buffer.format = 'f' # float
- buffer.internal = NULL # see References
- buffer.itemsize = itemsize
- buffer.len = self.v.size() * itemsize # product(shape) * itemsize
- buffer.ndim = 2
- buffer.obj = self
- buffer.readonly = 0
- buffer.shape = self.shape
- buffer.strides = self.strides
- buffer.suboffsets = NULL # for pointer arrays only
-
- def __releasebuffer__(self, Py_buffer *buffer):
- pass
+# distutils: language = c++ + +from cpython cimport Py_buffer +from libcpp.vector cimport vector + +cdef class Matrix: + cdef Py_ssize_t ncols + cdef Py_ssize_t shape[2] + cdef Py_ssize_t strides[2] + cdef vector[float] v + + def __cinit__(self, Py_ssize_t ncols): + self.ncols = ncols + + def add_row(self): + """Adds a row, initially zero-filled.""" + self.v.resize(self.v.size() + self.ncols) + + def __getbuffer__(self, Py_buffer *buffer, int flags): + cdef Py_ssize_t itemsize = sizeof(self.v[0]) + + self.shape[0] = self.v.size() / self.ncols + self.shape[1] = self.ncols + + # Stride 1 is the distance, in bytes, between two items in a row; + # this is the distance between two adjacent items in the vector. + # Stride 0 is the distance between the first elements of adjacent rows. + self.strides[1] = <Py_ssize_t>( <char *>&(self.v[1]) + - <char *>&(self.v[0])) + self.strides[0] = self.ncols * self.strides[1] + + buffer.buf = <char *>&(self.v[0]) + buffer.format = 'f' # float + buffer.internal = NULL # see References + buffer.itemsize = itemsize + buffer.len = self.v.size() * itemsize # product(shape) * itemsize + buffer.ndim = 2 + buffer.obj = self + buffer.readonly = 0 + buffer.shape = self.shape + buffer.strides = self.strides + buffer.suboffsets = NULL # for pointer arrays only + + def __releasebuffer__(self, Py_buffer *buffer): + pass diff --git a/docs/examples/userguide/buffer/view_count.pyx b/docs/examples/userguide/buffer/view_count.pyx index ee8d5085d..8027f3ee9 100644 --- a/docs/examples/userguide/buffer/view_count.pyx +++ b/docs/examples/userguide/buffer/view_count.pyx @@ -1,29 +1,29 @@ -# distutils: language = c++
-
-from cpython cimport Py_buffer
-from libcpp.vector cimport vector
-
-cdef class Matrix:
-
- cdef int view_count
-
- cdef Py_ssize_t ncols
- cdef vector[float] v
- # ...
-
- def __cinit__(self, Py_ssize_t ncols):
- self.ncols = ncols
- self.view_count = 0
-
- def add_row(self):
- if self.view_count > 0:
- raise ValueError("can't add row while being viewed")
- self.v.resize(self.v.size() + self.ncols)
-
- def __getbuffer__(self, Py_buffer *buffer, int flags):
- # ... as before
-
- self.view_count += 1
-
- def __releasebuffer__(self, Py_buffer *buffer):
+# distutils: language = c++ + +from cpython cimport Py_buffer +from libcpp.vector cimport vector + +cdef class Matrix: + + cdef int view_count + + cdef Py_ssize_t ncols + cdef vector[float] v + # ... + + def __cinit__(self, Py_ssize_t ncols): + self.ncols = ncols + self.view_count = 0 + + def add_row(self): + if self.view_count > 0: + raise ValueError("can't add row while being viewed") + self.v.resize(self.v.size() + self.ncols) + + def __getbuffer__(self, Py_buffer *buffer, int flags): + # ... as before + + self.view_count += 1 + + def __releasebuffer__(self, Py_buffer *buffer): self.view_count -= 1
\ No newline at end of file diff --git a/docs/examples/userguide/early_binding_for_speed/rectangle.pyx b/docs/examples/userguide/early_binding_for_speed/rectangle.pyx index ffc5593b0..de70b0263 100644 --- a/docs/examples/userguide/early_binding_for_speed/rectangle.pyx +++ b/docs/examples/userguide/early_binding_for_speed/rectangle.pyx @@ -1,19 +1,19 @@ -cdef class Rectangle:
- cdef int x0, y0
- cdef int x1, y1
-
- def __init__(self, int x0, int y0, int x1, int y1):
- self.x0 = x0
- self.y0 = y0
- self.x1 = x1
- self.y1 = y1
-
- def area(self):
- area = (self.x1 - self.x0) * (self.y1 - self.y0)
- if area < 0:
- area = -area
- return area
-
-def rectArea(x0, y0, x1, y1):
- rect = Rectangle(x0, y0, x1, y1)
- return rect.area()
+cdef class Rectangle: + cdef int x0, y0 + cdef int x1, y1 + + def __init__(self, int x0, int y0, int x1, int y1): + self.x0 = x0 + self.y0 = y0 + self.x1 = x1 + self.y1 = y1 + + def area(self): + area = (self.x1 - self.x0) * (self.y1 - self.y0) + if area < 0: + area = -area + return area + +def rectArea(x0, y0, x1, y1): + rect = Rectangle(x0, y0, x1, y1) + return rect.area() diff --git a/docs/examples/userguide/early_binding_for_speed/rectangle_cdef.pyx b/docs/examples/userguide/early_binding_for_speed/rectangle_cdef.pyx index 5502b4568..25c9ce88a 100644 --- a/docs/examples/userguide/early_binding_for_speed/rectangle_cdef.pyx +++ b/docs/examples/userguide/early_binding_for_speed/rectangle_cdef.pyx @@ -1,22 +1,22 @@ -cdef class Rectangle:
- cdef int x0, y0
- cdef int x1, y1
-
- def __init__(self, int x0, int y0, int x1, int y1):
- self.x0 = x0
- self.y0 = y0
- self.x1 = x1
- self.y1 = y1
-
- cdef int _area(self):
- area = (self.x1 - self.x0) * (self.y1 - self.y0)
- if area < 0:
- area = -area
- return area
-
- def area(self):
- return self._area()
-
-def rectArea(x0, y0, x1, y1):
- rect = Rectangle(x0, y0, x1, y1)
- return rect.area()
+cdef class Rectangle: + cdef int x0, y0 + cdef int x1, y1 + + def __init__(self, int x0, int y0, int x1, int y1): + self.x0 = x0 + self.y0 = y0 + self.x1 = x1 + self.y1 = y1 + + cdef int _area(self): + area = (self.x1 - self.x0) * (self.y1 - self.y0) + if area < 0: + area = -area + return area + + def area(self): + return self._area() + +def rectArea(x0, y0, x1, y1): + cdef Rectangle rect = Rectangle(x0, y0, x1, y1) + return rect._area() diff --git a/docs/examples/userguide/early_binding_for_speed/rectangle_cpdef.pyx b/docs/examples/userguide/early_binding_for_speed/rectangle_cpdef.pyx index 01df3759f..f8b7d86a8 100644 --- a/docs/examples/userguide/early_binding_for_speed/rectangle_cpdef.pyx +++ b/docs/examples/userguide/early_binding_for_speed/rectangle_cpdef.pyx @@ -1,19 +1,19 @@ -cdef class Rectangle:
- cdef int x0, y0
- cdef int x1, y1
-
- def __init__(self, int x0, int y0, int x1, int y1):
- self.x0 = x0
- self.y0 = y0
- self.x1 = x1
- self.y1 = y1
-
- cpdef int area(self):
- area = (self.x1 - self.x0) * (self.y1 - self.y0)
- if area < 0:
- area = -area
- return area
-
-def rectArea(x0, y0, x1, y1):
- rect = Rectangle(x0, y0, x1, y1)
- return rect.area()
+cdef class Rectangle: + cdef int x0, y0 + cdef int x1, y1 + + def __init__(self, int x0, int y0, int x1, int y1): + self.x0 = x0 + self.y0 = y0 + self.x1 = x1 + self.y1 = y1 + + cpdef int area(self): + area = (self.x1 - self.x0) * (self.y1 - self.y0) + if area < 0: + area = -area + return area + +def rectArea(x0, y0, x1, y1): + cdef Rectangle rect = Rectangle(x0, y0, x1, y1) + return rect.area() diff --git a/docs/examples/userguide/extension_types/c_property.pyx b/docs/examples/userguide/extension_types/c_property.pyx new file mode 100644 index 000000000..b759ceec1 --- /dev/null +++ b/docs/examples/userguide/extension_types/c_property.pyx @@ -0,0 +1,20 @@ +cdef extern from "complexobject.h": + + struct Py_complex: + double real + double imag + + ctypedef class __builtin__.complex [object PyComplexObject]: + cdef Py_complex cval + + @property + cdef inline double real(self): + return self.cval.real + + @property + cdef inline double imag(self): + return self.cval.imag + + +def cprint(complex c): + print(f"{c.real :.4f}{c.imag :+.4f}j") # uses C calls to the above property methods. diff --git a/docs/examples/userguide/extension_types/dict_animal.pyx b/docs/examples/userguide/extension_types/dict_animal.pyx index ab66900fe..1aa0ccc11 100644 --- a/docs/examples/userguide/extension_types/dict_animal.pyx +++ b/docs/examples/userguide/extension_types/dict_animal.pyx @@ -1,11 +1,11 @@ -cdef class Animal:
-
- cdef int number_of_legs
- cdef dict __dict__
-
- def __cinit__(self, int number_of_legs):
- self.number_of_legs = number_of_legs
-
-
-dog = Animal(4)
-dog.has_tail = True
+cdef class Animal: + + cdef int number_of_legs + cdef dict __dict__ + + def __cinit__(self, int number_of_legs): + self.number_of_legs = number_of_legs + + +dog = Animal(4) +dog.has_tail = True diff --git a/docs/examples/userguide/extension_types/extendable_animal.pyx b/docs/examples/userguide/extension_types/extendable_animal.pyx index fe53218b5..701a93148 100644 --- a/docs/examples/userguide/extension_types/extendable_animal.pyx +++ b/docs/examples/userguide/extension_types/extendable_animal.pyx @@ -1,14 +1,14 @@ -cdef class Animal:
-
- cdef int number_of_legs
-
- def __cinit__(self, int number_of_legs):
- self.number_of_legs = number_of_legs
-
-
-class ExtendableAnimal(Animal): # Note that we use class, not cdef class
- pass
-
-
-dog = ExtendableAnimal(4)
+cdef class Animal: + + cdef int number_of_legs + + def __cinit__(self, int number_of_legs): + self.number_of_legs = number_of_legs + + +class ExtendableAnimal(Animal): # Note that we use class, not cdef class + pass + + +dog = ExtendableAnimal(4) dog.has_tail = True
\ No newline at end of file diff --git a/docs/examples/userguide/extension_types/my_module.pyx b/docs/examples/userguide/extension_types/my_module.pyx index 6ceb057ca..fb0701c12 100644 --- a/docs/examples/userguide/extension_types/my_module.pyx +++ b/docs/examples/userguide/extension_types/my_module.pyx @@ -1,11 +1,11 @@ -from __future__ import print_function
-
-cdef class Shrubbery:
-
- def __init__(self, w, h):
- self.width = w
- self.height = h
-
- def describe(self):
- print("This shrubbery is", self.width,
- "by", self.height, "cubits.")
+from __future__ import print_function + +cdef class Shrubbery: + + def __init__(self, w, h): + self.width = w + self.height = h + + def describe(self): + print("This shrubbery is", self.width, + "by", self.height, "cubits.") diff --git a/docs/examples/userguide/extension_types/python_access.pyx b/docs/examples/userguide/extension_types/python_access.pyx index 13e19091e..6d5225ec0 100644 --- a/docs/examples/userguide/extension_types/python_access.pyx +++ b/docs/examples/userguide/extension_types/python_access.pyx @@ -1,3 +1,3 @@ -cdef class Shrubbery:
- cdef public int width, height
- cdef readonly float depth
+cdef class Shrubbery: + cdef public int width, height + cdef readonly float depth diff --git a/docs/examples/userguide/extension_types/shrubbery.pyx b/docs/examples/userguide/extension_types/shrubbery.pyx index 9d2a5481a..780735c9d 100644 --- a/docs/examples/userguide/extension_types/shrubbery.pyx +++ b/docs/examples/userguide/extension_types/shrubbery.pyx @@ -1,12 +1,12 @@ -from __future__ import print_function
-
-cdef class Shrubbery:
- cdef int width, height
-
- def __init__(self, w, h):
- self.width = w
- self.height = h
-
- def describe(self):
- print("This shrubbery is", self.width,
- "by", self.height, "cubits.")
+from __future__ import print_function + +cdef class Shrubbery: + cdef int width, height + + def __init__(self, w, h): + self.width = w + self.height = h + + def describe(self): + print("This shrubbery is", self.width, + "by", self.height, "cubits.") diff --git a/docs/examples/userguide/extension_types/shrubbery_2.pyx b/docs/examples/userguide/extension_types/shrubbery_2.pyx index e0d8c45b5..d05d28243 100644 --- a/docs/examples/userguide/extension_types/shrubbery_2.pyx +++ b/docs/examples/userguide/extension_types/shrubbery_2.pyx @@ -1,8 +1,8 @@ -from my_module cimport Shrubbery
-
-cdef Shrubbery another_shrubbery(Shrubbery sh1):
- cdef Shrubbery sh2
- sh2 = Shrubbery()
- sh2.width = sh1.width
- sh2.height = sh1.height
- return sh2
+from my_module cimport Shrubbery + +cdef Shrubbery another_shrubbery(Shrubbery sh1): + cdef Shrubbery sh2 + sh2 = Shrubbery() + sh2.width = sh1.width + sh2.height = sh1.height + return sh2 diff --git a/docs/examples/userguide/extension_types/widen_shrubbery.pyx b/docs/examples/userguide/extension_types/widen_shrubbery.pyx index aff3bd116..a312fbfd9 100644 --- a/docs/examples/userguide/extension_types/widen_shrubbery.pyx +++ b/docs/examples/userguide/extension_types/widen_shrubbery.pyx @@ -1,4 +1,4 @@ -from my_module cimport Shrubbery
-
-cdef widen_shrubbery(Shrubbery sh, extra_width):
- sh.width = sh.width + extra_width
+from my_module cimport Shrubbery + +cdef widen_shrubbery(Shrubbery sh, extra_width): + sh.width = sh.width + extra_width diff --git a/docs/examples/userguide/external_C_code/delorean.pyx b/docs/examples/userguide/external_C_code/delorean.pyx index 52c713616..d74e5de2b 100644 --- a/docs/examples/userguide/external_C_code/delorean.pyx +++ b/docs/examples/userguide/external_C_code/delorean.pyx @@ -1,9 +1,9 @@ -# delorean.pyx
-
-cdef public struct Vehicle:
- int speed
- float power
-
-cdef api void activate(Vehicle *v):
- if v.speed >= 88 and v.power >= 1.21:
+# delorean.pyx + +cdef public struct Vehicle: + int speed + float power + +cdef api void activate(Vehicle *v): + if v.speed >= 88 and v.power >= 1.21: print("Time travel achieved")
\ No newline at end of file diff --git a/docs/examples/userguide/external_C_code/platform_adaptation.pyx b/docs/examples/userguide/external_C_code/platform_adaptation.pyx new file mode 100644 index 000000000..fa193f83d --- /dev/null +++ b/docs/examples/userguide/external_C_code/platform_adaptation.pyx @@ -0,0 +1,15 @@ +cdef extern from *: + """ + #if defined(_WIN32) || defined(MS_WINDOWS) || defined(_MSC_VER) + #define WIN32_LEAN_AND_MEAN + #include <windows.h> + #define myapp_sleep(m) Sleep(m) + #else + #include <unistd.h> + #define myapp_sleep(m) ((void) usleep((m) * 1000)) + #endif + """ + # using "myapp_" prefix in the C code to prevent C naming conflicts + void msleep "myapp_sleep"(int milliseconds) nogil + +msleep(milliseconds=1) diff --git a/docs/examples/userguide/external_C_code/c_code_docstring.pyx b/docs/examples/userguide/external_C_code/verbatim_c_code.pyx index 430e89c48..fb1937166 100644 --- a/docs/examples/userguide/external_C_code/c_code_docstring.pyx +++ b/docs/examples/userguide/external_C_code/verbatim_c_code.pyx @@ -1,9 +1,9 @@ -cdef extern from *:
- """
- /* This is C code which will be put
- * in the .c file output by Cython */
- static long square(long x) {return x * x;}
- #define assign(x, y) ((x) = (y))
- """
- long square(long x)
- void assign(long& x, long y)
+cdef extern from *: + """ + /* This is C code which will be put + * in the .c file output by Cython */ + static long square(long x) {return x * x;} + #define assign(x, y) ((x) = (y)) + """ + long square(long x) + void assign(long& x, long y) diff --git a/docs/examples/userguide/fusedtypes/char_or_float.pyx b/docs/examples/userguide/fusedtypes/char_or_float.pyx index 6db746831..5a525431f 100644 --- a/docs/examples/userguide/fusedtypes/char_or_float.pyx +++ b/docs/examples/userguide/fusedtypes/char_or_float.pyx @@ -1,17 +1,17 @@ -from __future__ import print_function
-
-ctypedef fused char_or_float:
- char
- float
-
-
-cpdef char_or_float plus_one(char_or_float var):
- return var + 1
-
-
-def show_me():
- cdef:
- char a = 127
- float b = 127
- print('char', plus_one(a))
- print('float', plus_one(b))
+from __future__ import print_function + +ctypedef fused char_or_float: + char + float + + +cpdef char_or_float plus_one(char_or_float var): + return var + 1 + + +def show_me(): + cdef: + char a = 127 + float b = 127 + print('char', plus_one(a)) + print('float', plus_one(b)) diff --git a/docs/examples/userguide/language_basics/casting_python.pyx b/docs/examples/userguide/language_basics/casting_python.pyx index fe84acde2..4cc819ae3 100644 --- a/docs/examples/userguide/language_basics/casting_python.pyx +++ b/docs/examples/userguide/language_basics/casting_python.pyx @@ -1,19 +1,19 @@ -from cpython.ref cimport PyObject
-
-cdef extern from *:
- ctypedef Py_ssize_t Py_intptr_t
-
-python_string = "foo"
-
-cdef void* ptr = <void*>python_string
-cdef Py_intptr_t adress_in_c = <Py_intptr_t>ptr
-address_from_void = adress_in_c # address_from_void is a python int
-
-cdef PyObject* ptr2 = <PyObject*>python_string
-cdef Py_intptr_t address_in_c2 = <Py_intptr_t>ptr2
-address_from_PyObject = address_in_c2 # address_from_PyObject is a python int
-
-assert address_from_void == address_from_PyObject == id(python_string)
-
-print(<object>ptr) # Prints "foo"
-print(<object>ptr2) # prints "foo"
+from cpython.ref cimport PyObject + +cdef extern from *: + ctypedef Py_ssize_t Py_intptr_t + +python_string = "foo" + +cdef void* ptr = <void*>python_string +cdef Py_intptr_t adress_in_c = <Py_intptr_t>ptr +address_from_void = adress_in_c # address_from_void is a python int + +cdef PyObject* ptr2 = <PyObject*>python_string +cdef Py_intptr_t address_in_c2 = <Py_intptr_t>ptr2 +address_from_PyObject = address_in_c2 # address_from_PyObject is a python int + +assert address_from_void == address_from_PyObject == id(python_string) + +print(<object>ptr) # Prints "foo" +print(<object>ptr2) # prints "foo" diff --git a/docs/examples/userguide/language_basics/cdef_block.pyx b/docs/examples/userguide/language_basics/cdef_block.pyx index 4132aeee1..3b569963e 100644 --- a/docs/examples/userguide/language_basics/cdef_block.pyx +++ b/docs/examples/userguide/language_basics/cdef_block.pyx @@ -1,12 +1,12 @@ -from __future__ import print_function
-
-cdef:
- struct Spam:
- int tons
-
- int i
- float a
- Spam *p
-
- void f(Spam *s):
- print(s.tons, "Tons of spam")
+from __future__ import print_function + +cdef: + struct Spam: + int tons + + int i + float a + Spam *p + + void f(Spam *s): + print(s.tons, "Tons of spam") diff --git a/docs/examples/userguide/language_basics/compile_time.pyx b/docs/examples/userguide/language_basics/compile_time.pyx index fcaf75f29..28465f62a 100644 --- a/docs/examples/userguide/language_basics/compile_time.pyx +++ b/docs/examples/userguide/language_basics/compile_time.pyx @@ -1,9 +1,9 @@ -from __future__ import print_function
-
-DEF FavouriteFood = u"spam"
-DEF ArraySize = 42
-DEF OtherArraySize = 2 * ArraySize + 17
-
-cdef int a1[ArraySize]
-cdef int a2[OtherArraySize]
+from __future__ import print_function + +DEF FavouriteFood = u"spam" +DEF ArraySize = 42 +DEF OtherArraySize = 2 * ArraySize + 17 + +cdef int a1[ArraySize] +cdef int a2[OtherArraySize] print("I like", FavouriteFood)
\ No newline at end of file diff --git a/docs/examples/userguide/language_basics/kwargs_1.pyx b/docs/examples/userguide/language_basics/kwargs_1.pyx index e5e18c008..1117c967c 100644 --- a/docs/examples/userguide/language_basics/kwargs_1.pyx +++ b/docs/examples/userguide/language_basics/kwargs_1.pyx @@ -1,6 +1,6 @@ -def f(a, b, *args, c, d = 42, e, **kwds):
- ...
-
-
-# We cannot call f with less verbosity than this.
-foo = f(4, "bar", c=68, e=1.0)
+def f(a, b, *args, c, d = 42, e, **kwds): + ... + + +# We cannot call f with less verbosity than this. +foo = f(4, "bar", c=68, e=1.0) diff --git a/docs/examples/userguide/language_basics/kwargs_2.pyx b/docs/examples/userguide/language_basics/kwargs_2.pyx index a2c639ea6..902df694c 100644 --- a/docs/examples/userguide/language_basics/kwargs_2.pyx +++ b/docs/examples/userguide/language_basics/kwargs_2.pyx @@ -1,5 +1,5 @@ -def g(a, b, *, c, d):
- ...
-
-# We cannot call g with less verbosity than this.
-foo = g(4.0, "something", c=68, d="other")
+def g(a, b, *, c, d): + ... + +# We cannot call g with less verbosity than this. +foo = g(4.0, "something", c=68, d="other") diff --git a/docs/examples/userguide/language_basics/open_file.pyx b/docs/examples/userguide/language_basics/open_file.pyx index 19eac104e..ad45fc8c4 100644 --- a/docs/examples/userguide/language_basics/open_file.pyx +++ b/docs/examples/userguide/language_basics/open_file.pyx @@ -1,18 +1,18 @@ -from libc.stdio cimport FILE, fopen
-from libc.stdlib cimport malloc, free
-from cpython.exc cimport PyErr_SetFromErrnoWithFilenameObject
-
-def open_file():
- cdef FILE* p
- p = fopen("spam.txt", "r")
- if p is NULL:
- PyErr_SetFromErrnoWithFilenameObject(OSError, "spam.txt")
- ...
-
-
-def allocating_memory(number=10):
- cdef double *my_array = <double *> malloc(number * sizeof(double))
- if not my_array: # same as 'is NULL' above
- raise MemoryError()
- ...
- free(my_array)
+from libc.stdio cimport FILE, fopen +from libc.stdlib cimport malloc, free +from cpython.exc cimport PyErr_SetFromErrnoWithFilenameObject + +def open_file(): + cdef FILE* p + p = fopen("spam.txt", "r") + if p is NULL: + PyErr_SetFromErrnoWithFilenameObject(OSError, "spam.txt") + ... + + +def allocating_memory(number=10): + cdef double *my_array = <double *> malloc(number * sizeof(double)) + if not my_array: # same as 'is NULL' above + raise MemoryError() + ... + free(my_array) diff --git a/docs/examples/userguide/language_basics/optional_subclassing.pyx b/docs/examples/userguide/language_basics/optional_subclassing.pyx index f655cadf3..88371ab41 100644 --- a/docs/examples/userguide/language_basics/optional_subclassing.pyx +++ b/docs/examples/userguide/language_basics/optional_subclassing.pyx @@ -1,13 +1,13 @@ -from __future__ import print_function
-
-cdef class A:
- cdef foo(self):
- print("A")
-
-cdef class B(A):
- cdef foo(self, x=None):
- print("B", x)
-
-cdef class C(B):
- cpdef foo(self, x=True, int k=3):
- print("C", x, k)
+from __future__ import print_function + +cdef class A: + cdef foo(self): + print("A") + +cdef class B(A): + cdef foo(self, x=None): + print("B", x) + +cdef class C(B): + cpdef foo(self, x=True, int k=3): + print("C", x, k) diff --git a/docs/examples/userguide/language_basics/override.pyx b/docs/examples/userguide/language_basics/override.pyx index 873a7ec6e..13f0b2a64 100644 --- a/docs/examples/userguide/language_basics/override.pyx +++ b/docs/examples/userguide/language_basics/override.pyx @@ -1,13 +1,13 @@ -from __future__ import print_function
-
-cdef class A:
- cdef foo(self):
- print("A")
-
-cdef class B(A):
- cpdef foo(self):
- print("B")
-
-class C(B): # NOTE: not cdef class
- def foo(self):
- print("C")
+from __future__ import print_function + +cdef class A: + cdef foo(self): + print("A") + +cdef class B(A): + cpdef foo(self): + print("B") + +class C(B): # NOTE: not cdef class + def foo(self): + print("C") diff --git a/docs/examples/userguide/language_basics/parameter_refcount.pyx b/docs/examples/userguide/language_basics/parameter_refcount.pyx new file mode 100644 index 000000000..5ffd28c44 --- /dev/null +++ b/docs/examples/userguide/language_basics/parameter_refcount.pyx @@ -0,0 +1,20 @@ +from __future__ import print_function + +from cpython.ref cimport PyObject + +import sys + +python_dict = {"abc": 123} +python_dict_refcount = sys.getrefcount(python_dict) + +cdef owned_reference(object obj): + refcount = sys.getrefcount(python_dict) + print('Inside owned_reference: {refcount}'.format(refcount=refcount)) + +cdef borrowed_reference(PyObject * obj): + refcount = obj.ob_refcnt + print('Inside borrowed_reference: {refcount}'.format(refcount=refcount)) + +print('Initial refcount: {refcount}'.format(refcount=python_dict_refcount)) +owned_reference(python_dict) +borrowed_reference(<PyObject *>python_dict) diff --git a/docs/examples/userguide/language_basics/struct_union_enum.pyx b/docs/examples/userguide/language_basics/struct_union_enum.pyx index ccbc28d42..af9b06d9a 100644 --- a/docs/examples/userguide/language_basics/struct_union_enum.pyx +++ b/docs/examples/userguide/language_basics/struct_union_enum.pyx @@ -1,16 +1,16 @@ -cdef struct Grail:
- int age
- float volume
-
-cdef union Food:
- char *spam
- float *eggs
-
-cdef enum CheeseType:
- cheddar, edam,
- camembert
-
-cdef enum CheeseState:
- hard = 1
- soft = 2
- runny = 3
+cdef struct Grail: + int age + float volume + +cdef union Food: + char *spam + float *eggs + +cdef enum CheeseType: + cheddar, edam, + camembert + +cdef enum CheeseState: + hard = 1 + soft = 2 + runny = 3 diff --git a/docs/examples/userguide/memoryviews/add_one.pyx b/docs/examples/userguide/memoryviews/add_one.pyx index cbe65b069..7de7a1274 100644 --- a/docs/examples/userguide/memoryviews/add_one.pyx +++ b/docs/examples/userguide/memoryviews/add_one.pyx @@ -1,12 +1,12 @@ -import numpy as np
-
-def add_one(int[:,:] buf):
- for x in range(buf.shape[0]):
- for y in range(buf.shape[1]):
- buf[x, y] += 1
-
-# exporting_object must be a Python object
-# implementing the buffer interface, e.g. a numpy array.
-exporting_object = np.zeros((10, 20), dtype=np.intc)
-
-add_one(exporting_object)
+import numpy as np + +def add_one(int[:,:] buf): + for x in range(buf.shape[0]): + for y in range(buf.shape[1]): + buf[x, y] += 1 + +# exporting_object must be a Python object +# implementing the buffer interface, e.g. a numpy array. +exporting_object = np.zeros((10, 20), dtype=np.intc) + +add_one(exporting_object) diff --git a/docs/examples/userguide/memoryviews/copy.pyx b/docs/examples/userguide/memoryviews/copy.pyx index 9f000a3b4..9eb1307bf 100644 --- a/docs/examples/userguide/memoryviews/copy.pyx +++ b/docs/examples/userguide/memoryviews/copy.pyx @@ -1,12 +1,12 @@ -import numpy as np
-
-cdef int[:, :, :] to_view, from_view
-to_view = np.empty((20, 15, 30), dtype=np.intc)
-from_view = np.ones((20, 15, 30), dtype=np.intc)
-
-# copy the elements in from_view to to_view
-to_view[...] = from_view
-# or
-to_view[:] = from_view
-# or
-to_view[:, :, :] = from_view
+import numpy as np + +cdef int[:, :, :] to_view, from_view +to_view = np.empty((20, 15, 30), dtype=np.intc) +from_view = np.ones((20, 15, 30), dtype=np.intc) + +# copy the elements in from_view to to_view +to_view[...] = from_view +# or +to_view[:] = from_view +# or +to_view[:, :, :] = from_view diff --git a/docs/examples/userguide/memoryviews/memory_layout.pyx b/docs/examples/userguide/memoryviews/memory_layout.pyx index 5c2818dc0..8f9d8a23c 100644 --- a/docs/examples/userguide/memoryviews/memory_layout.pyx +++ b/docs/examples/userguide/memoryviews/memory_layout.pyx @@ -1,11 +1,11 @@ -from cython cimport view
-
-# direct access in both dimensions, strided in the first dimension, contiguous in the last
-cdef int[:, ::view.contiguous] a
-
-# contiguous list of pointers to contiguous lists of ints
-cdef int[::view.indirect_contiguous, ::1] b
-
-# direct or indirect in the first dimension, direct in the second dimension
-# strided in both dimensions
-cdef int[::view.generic, :] c
+from cython cimport view + +# direct access in both dimensions, strided in the first dimension, contiguous in the last +cdef int[:, ::view.contiguous] a + +# contiguous list of pointers to contiguous lists of ints +cdef int[::view.indirect_contiguous, ::1] b + +# direct or indirect in the first dimension, direct in the second dimension +# strided in both dimensions +cdef int[::view.generic, :] c diff --git a/docs/examples/userguide/memoryviews/memory_layout_2.pyx b/docs/examples/userguide/memoryviews/memory_layout_2.pyx index 1cb039dd4..71d2cceb2 100644 --- a/docs/examples/userguide/memoryviews/memory_layout_2.pyx +++ b/docs/examples/userguide/memoryviews/memory_layout_2.pyx @@ -1,6 +1,6 @@ -from cython cimport view
-
-# VALID
-cdef int[::view.indirect, ::1, :] a
-cdef int[::view.indirect, :, ::1] b
-cdef int[::view.indirect_contiguous, ::1, :] c
+from cython cimport view + +# VALID +cdef int[::view.indirect, ::1, :] a +cdef int[::view.indirect, :, ::1] b +cdef int[::view.indirect_contiguous, ::1, :] c diff --git a/docs/examples/userguide/memoryviews/memview_to_c.pyx b/docs/examples/userguide/memoryviews/memview_to_c.pyx index c5abc19ac..ad6190cc7 100644 --- a/docs/examples/userguide/memoryviews/memview_to_c.pyx +++ b/docs/examples/userguide/memoryviews/memview_to_c.pyx @@ -1,28 +1,28 @@ -cdef extern from "C_func_file.c":
- # C is include here so that it doesn't need to be compiled externally
- pass
-
-cdef extern from "C_func_file.h":
- void multiply_by_10_in_C(double *, unsigned int)
-
-import numpy as np
-
-def multiply_by_10(arr): # 'arr' is a one-dimensional numpy array
-
- if not arr.flags['C_CONTIGUOUS']:
- arr = np.ascontiguousarray(arr) # Makes a contiguous copy of the numpy array.
-
- cdef double[::1] arr_memview = arr
-
- multiply_by_10_in_C(&arr_memview[0], arr_memview.shape[0])
-
- return arr
-
-
-a = np.ones(5, dtype=np.double)
-print(multiply_by_10(a))
-
-b = np.ones(10, dtype=np.double)
-b = b[::2] # b is not contiguous.
-
-print(multiply_by_10(b)) # but our function still works as expected.
+cdef extern from "C_func_file.c": + # C is include here so that it doesn't need to be compiled externally + pass + +cdef extern from "C_func_file.h": + void multiply_by_10_in_C(double *, unsigned int) + +import numpy as np + +def multiply_by_10(arr): # 'arr' is a one-dimensional numpy array + + if not arr.flags['C_CONTIGUOUS']: + arr = np.ascontiguousarray(arr) # Makes a contiguous copy of the numpy array. + + cdef double[::1] arr_memview = arr + + multiply_by_10_in_C(&arr_memview[0], arr_memview.shape[0]) + + return arr + + +a = np.ones(5, dtype=np.double) +print(multiply_by_10(a)) + +b = np.ones(10, dtype=np.double) +b = b[::2] # b is not contiguous. + +print(multiply_by_10(b)) # but our function still works as expected. diff --git a/docs/examples/userguide/memoryviews/not_none.pyx b/docs/examples/userguide/memoryviews/not_none.pyx index ae3b6c936..f6c0fed8a 100644 --- a/docs/examples/userguide/memoryviews/not_none.pyx +++ b/docs/examples/userguide/memoryviews/not_none.pyx @@ -1,11 +1,11 @@ -import numpy as np
-
-def process_buffer(int[:,:] input_view not None,
- int[:,:] output_view=None):
-
- if output_view is None:
- # Creating a default view, e.g.
- output_view = np.empty_like(input_view)
-
- # process 'input_view' into 'output_view'
- return output_view
+import numpy as np + +def process_buffer(int[:,:] input_view not None, + int[:,:] output_view=None): + + if output_view is None: + # Creating a default view, e.g. + output_view = np.empty_like(input_view) + + # process 'input_view' into 'output_view' + return output_view diff --git a/docs/examples/userguide/memoryviews/np_flag_const.pyx b/docs/examples/userguide/memoryviews/np_flag_const.pyx index 03f0ea4a3..54eb3d338 100644 --- a/docs/examples/userguide/memoryviews/np_flag_const.pyx +++ b/docs/examples/userguide/memoryviews/np_flag_const.pyx @@ -1,7 +1,7 @@ -import numpy as np
-
-cdef const double[:] myslice # const item type => read-only view
-
-a = np.linspace(0, 10, num=50)
-a.setflags(write=False)
-myslice = a
+import numpy as np + +cdef const double[:] myslice # const item type => read-only view + +a = np.linspace(0, 10, num=50) +a.setflags(write=False) +myslice = a diff --git a/docs/examples/userguide/memoryviews/slicing.pyx b/docs/examples/userguide/memoryviews/slicing.pyx index a6134aae2..d7bd896e6 100644 --- a/docs/examples/userguide/memoryviews/slicing.pyx +++ b/docs/examples/userguide/memoryviews/slicing.pyx @@ -1,10 +1,10 @@ -import numpy as np
-
-exporting_object = np.arange(0, 15 * 10 * 20, dtype=np.intc).reshape((15, 10, 20))
-
-cdef int[:, :, :] my_view = exporting_object
-
-# These are all equivalent
-my_view[10]
-my_view[10, :, :]
-my_view[10, ...]
+import numpy as np + +exporting_object = np.arange(0, 15 * 10 * 20, dtype=np.intc).reshape((15, 10, 20)) + +cdef int[:, :, :] my_view = exporting_object + +# These are all equivalent +my_view[10] +my_view[10, :, :] +my_view[10, ...] diff --git a/docs/examples/userguide/memoryviews/transpose.pyx b/docs/examples/userguide/memoryviews/transpose.pyx index 7611529c2..8a53f7140 100644 --- a/docs/examples/userguide/memoryviews/transpose.pyx +++ b/docs/examples/userguide/memoryviews/transpose.pyx @@ -1,6 +1,6 @@ -import numpy as np
-
-array = np.arange(20, dtype=np.intc).reshape((2, 10))
-
-cdef int[:, ::1] c_contig = array
-cdef int[::1, :] f_contig = c_contig.T
+import numpy as np + +array = np.arange(20, dtype=np.intc).reshape((2, 10)) + +cdef int[:, ::1] c_contig = array +cdef int[::1, :] f_contig = c_contig.T diff --git a/docs/examples/userguide/memoryviews/view_string.pyx b/docs/examples/userguide/memoryviews/view_string.pyx index 7aace3ea5..9fdeae053 100644 --- a/docs/examples/userguide/memoryviews/view_string.pyx +++ b/docs/examples/userguide/memoryviews/view_string.pyx @@ -1,9 +1,9 @@ -cdef bint is_y_in(const unsigned char[:] string_view):
- cdef int i
- for i in range(string_view.shape[0]):
- if string_view[i] == b'y':
- return True
- return False
-
-print(is_y_in(b'hello world')) # False
-print(is_y_in(b'hello Cython')) # True
+cdef bint is_y_in(const unsigned char[:] string_view): + cdef int i + for i in range(string_view.shape[0]): + if string_view[i] == b'y': + return True + return False + +print(is_y_in(b'hello world')) # False +print(is_y_in(b'hello Cython')) # True diff --git a/docs/examples/userguide/numpy_tutorial/compute_fused_types.pyx b/docs/examples/userguide/numpy_tutorial/compute_fused_types.pyx index 2fc87907d..af5ef9071 100644 --- a/docs/examples/userguide/numpy_tutorial/compute_fused_types.pyx +++ b/docs/examples/userguide/numpy_tutorial/compute_fused_types.pyx @@ -1,44 +1,44 @@ -# cython: infer_types=True
-import numpy as np
-cimport cython
-
-ctypedef fused my_type:
- int
- double
- long long
-
-
-cdef my_type clip(my_type a, my_type min_value, my_type max_value):
- return min(max(a, min_value), max_value)
-
-
-@cython.boundscheck(False)
-@cython.wraparound(False)
-def compute(my_type[:, ::1] array_1, my_type[:, ::1] array_2, my_type a, my_type b, my_type c):
-
- x_max = array_1.shape[0]
- y_max = array_1.shape[1]
-
- assert tuple(array_1.shape) == tuple(array_2.shape)
-
- if my_type is int:
- dtype = np.intc
- elif my_type is double:
- dtype = np.double
- elif my_type is cython.longlong:
- dtype = np.longlong
-
- result = np.zeros((x_max, y_max), dtype=dtype)
- cdef my_type[:, ::1] result_view = result
-
- cdef my_type tmp
- cdef Py_ssize_t x, y
-
- for x in range(x_max):
- for y in range(y_max):
-
- tmp = clip(array_1[x, y], 2, 10)
- tmp = tmp * a + array_2[x, y] * b
- result_view[x, y] = tmp + c
-
- return result
+# cython: infer_types=True +import numpy as np +cimport cython + +ctypedef fused my_type: + int + double + long long + + +cdef my_type clip(my_type a, my_type min_value, my_type max_value): + return min(max(a, min_value), max_value) + + +@cython.boundscheck(False) +@cython.wraparound(False) +def compute(my_type[:, ::1] array_1, my_type[:, ::1] array_2, my_type a, my_type b, my_type c): + + x_max = array_1.shape[0] + y_max = array_1.shape[1] + + assert tuple(array_1.shape) == tuple(array_2.shape) + + if my_type is int: + dtype = np.intc + elif my_type is double: + dtype = np.double + elif my_type is cython.longlong: + dtype = np.longlong + + result = np.zeros((x_max, y_max), dtype=dtype) + cdef my_type[:, ::1] result_view = result + + cdef my_type tmp + cdef Py_ssize_t x, y + + for x in range(x_max): + for y in range(y_max): + + tmp = clip(array_1[x, y], 2, 10) + tmp = tmp * a + array_2[x, y] * b + result_view[x, y] = tmp + c + + return result diff --git a/docs/examples/userguide/numpy_tutorial/compute_infer_types.pyx b/docs/examples/userguide/numpy_tutorial/compute_infer_types.pyx index 98a683de7..3882c289d 100644 --- a/docs/examples/userguide/numpy_tutorial/compute_infer_types.pyx +++ b/docs/examples/userguide/numpy_tutorial/compute_infer_types.pyx @@ -1,34 +1,34 @@ -# cython: infer_types=True
-import numpy as np
-cimport cython
-
-DTYPE = np.intc
-
-
-cdef int clip(int a, int min_value, int max_value):
- return min(max(a, min_value), max_value)
-
-
-@cython.boundscheck(False)
-@cython.wraparound(False)
-def compute(int[:, ::1] array_1, int[:, ::1] array_2, int a, int b, int c):
-
- x_max = array_1.shape[0]
- y_max = array_1.shape[1]
-
- assert tuple(array_1.shape) == tuple(array_2.shape)
-
- result = np.zeros((x_max, y_max), dtype=DTYPE)
- cdef int[:, ::1] result_view = result
-
- cdef int tmp
- cdef Py_ssize_t x, y
-
- for x in range(x_max):
- for y in range(y_max):
-
- tmp = clip(array_1[x, y], 2, 10)
- tmp = tmp * a + array_2[x, y] * b
- result_view[x, y] = tmp + c
-
- return result
+# cython: infer_types=True +import numpy as np +cimport cython + +DTYPE = np.intc + + +cdef int clip(int a, int min_value, int max_value): + return min(max(a, min_value), max_value) + + +@cython.boundscheck(False) +@cython.wraparound(False) +def compute(int[:, ::1] array_1, int[:, ::1] array_2, int a, int b, int c): + + x_max = array_1.shape[0] + y_max = array_1.shape[1] + + assert tuple(array_1.shape) == tuple(array_2.shape) + + result = np.zeros((x_max, y_max), dtype=DTYPE) + cdef int[:, ::1] result_view = result + + cdef int tmp + cdef Py_ssize_t x, y + + for x in range(x_max): + for y in range(y_max): + + tmp = clip(array_1[x, y], 2, 10) + tmp = tmp * a + array_2[x, y] * b + result_view[x, y] = tmp + c + + return result diff --git a/docs/examples/userguide/numpy_tutorial/compute_memview.pyx b/docs/examples/userguide/numpy_tutorial/compute_memview.pyx index d264e773a..166cd6df3 100644 --- a/docs/examples/userguide/numpy_tutorial/compute_memview.pyx +++ b/docs/examples/userguide/numpy_tutorial/compute_memview.pyx @@ -1,34 +1,34 @@ -import numpy as np
-
-DTYPE = np.intc
-
-
-cdef int clip(int a, int min_value, int max_value):
- return min(max(a, min_value), max_value)
-
-
-def compute(int[:, :] array_1, int[:, :] array_2, int a, int b, int c):
-
- cdef Py_ssize_t x_max = array_1.shape[0]
- cdef Py_ssize_t y_max = array_1.shape[1]
-
- # array_1.shape is now a C array, no it's not possible
- # to compare it simply by using == without a for-loop.
- # To be able to compare it to array_2.shape easily,
- # we convert them both to Python tuples.
- assert tuple(array_1.shape) == tuple(array_2.shape)
-
- result = np.zeros((x_max, y_max), dtype=DTYPE)
- cdef int[:, :] result_view = result
-
- cdef int tmp
- cdef Py_ssize_t x, y
-
- for x in range(x_max):
- for y in range(y_max):
-
- tmp = clip(array_1[x, y], 2, 10)
- tmp = tmp * a + array_2[x, y] * b
- result_view[x, y] = tmp + c
-
- return result
+import numpy as np + +DTYPE = np.intc + + +cdef int clip(int a, int min_value, int max_value): + return min(max(a, min_value), max_value) + + +def compute(int[:, :] array_1, int[:, :] array_2, int a, int b, int c): + + cdef Py_ssize_t x_max = array_1.shape[0] + cdef Py_ssize_t y_max = array_1.shape[1] + + # array_1.shape is now a C array, no it's not possible + # to compare it simply by using == without a for-loop. + # To be able to compare it to array_2.shape easily, + # we convert them both to Python tuples. + assert tuple(array_1.shape) == tuple(array_2.shape) + + result = np.zeros((x_max, y_max), dtype=DTYPE) + cdef int[:, :] result_view = result + + cdef int tmp + cdef Py_ssize_t x, y + + for x in range(x_max): + for y in range(y_max): + + tmp = clip(array_1[x, y], 2, 10) + tmp = tmp * a + array_2[x, y] * b + result_view[x, y] = tmp + c + + return result diff --git a/docs/examples/userguide/numpy_tutorial/compute_prange.pyx b/docs/examples/userguide/numpy_tutorial/compute_prange.pyx index c00d2261b..562c73070 100644 --- a/docs/examples/userguide/numpy_tutorial/compute_prange.pyx +++ b/docs/examples/userguide/numpy_tutorial/compute_prange.pyx @@ -1,53 +1,53 @@ -# tag: openmp
-# You can ignore the previous line.
-# It's for internal testing of the cython documentation.
-
-# distutils: extra_compile_args=-fopenmp
-# distutils: extra_link_args=-fopenmp
-
-import numpy as np
-cimport cython
-from cython.parallel import prange
-
-ctypedef fused my_type:
- int
- double
- long long
-
-
-# We declare our plain c function nogil
-cdef my_type clip(my_type a, my_type min_value, my_type max_value) nogil:
- return min(max(a, min_value), max_value)
-
-
-@cython.boundscheck(False)
-@cython.wraparound(False)
-def compute(my_type[:, ::1] array_1, my_type[:, ::1] array_2, my_type a, my_type b, my_type c):
-
- cdef Py_ssize_t x_max = array_1.shape[0]
- cdef Py_ssize_t y_max = array_1.shape[1]
-
- assert tuple(array_1.shape) == tuple(array_2.shape)
-
- if my_type is int:
- dtype = np.intc
- elif my_type is double:
- dtype = np.double
- elif my_type is cython.longlong:
- dtype = np.longlong
-
- result = np.zeros((x_max, y_max), dtype=dtype)
- cdef my_type[:, ::1] result_view = result
-
- cdef my_type tmp
- cdef Py_ssize_t x, y
-
- # We use prange here.
- for x in prange(x_max, nogil=True):
- for y in range(y_max):
-
- tmp = clip(array_1[x, y], 2, 10)
- tmp = tmp * a + array_2[x, y] * b
- result_view[x, y] = tmp + c
-
- return result
+# tag: openmp +# You can ignore the previous line. +# It's for internal testing of the cython documentation. + +# distutils: extra_compile_args=-fopenmp +# distutils: extra_link_args=-fopenmp + +import numpy as np +cimport cython +from cython.parallel import prange + +ctypedef fused my_type: + int + double + long long + + +# We declare our plain c function nogil +cdef my_type clip(my_type a, my_type min_value, my_type max_value) nogil: + return min(max(a, min_value), max_value) + + +@cython.boundscheck(False) +@cython.wraparound(False) +def compute(my_type[:, ::1] array_1, my_type[:, ::1] array_2, my_type a, my_type b, my_type c): + + cdef Py_ssize_t x_max = array_1.shape[0] + cdef Py_ssize_t y_max = array_1.shape[1] + + assert tuple(array_1.shape) == tuple(array_2.shape) + + if my_type is int: + dtype = np.intc + elif my_type is double: + dtype = np.double + elif my_type is cython.longlong: + dtype = np.longlong + + result = np.zeros((x_max, y_max), dtype=dtype) + cdef my_type[:, ::1] result_view = result + + cdef my_type tmp + cdef Py_ssize_t x, y + + # We use prange here. + for x in prange(x_max, nogil=True): + for y in range(y_max): + + tmp = clip(array_1[x, y], 2, 10) + tmp = tmp * a + array_2[x, y] * b + result_view[x, y] = tmp + c + + return result diff --git a/docs/examples/userguide/numpy_tutorial/compute_py.py b/docs/examples/userguide/numpy_tutorial/compute_py.py index 00bcf244c..4a5f90b4d 100644 --- a/docs/examples/userguide/numpy_tutorial/compute_py.py +++ b/docs/examples/userguide/numpy_tutorial/compute_py.py @@ -1,28 +1,28 @@ -import numpy as np
-
-
-def clip(a, min_value, max_value):
- return min(max(a, min_value), max_value)
-
-
-def compute(array_1, array_2, a, b, c):
- """
- This function must implement the formula
- np.clip(array_1, 2, 10) * a + array_2 * b + c
-
- array_1 and array_2 are 2D.
- """
- x_max = array_1.shape[0]
- y_max = array_1.shape[1]
-
- assert array_1.shape == array_2.shape
-
- result = np.zeros((x_max, y_max), dtype=array_1.dtype)
-
- for x in range(x_max):
- for y in range(y_max):
- tmp = clip(array_1[x, y], 2, 10)
- tmp = tmp * a + array_2[x, y] * b
- result[x, y] = tmp + c
-
- return result
+import numpy as np + + +def clip(a, min_value, max_value): + return min(max(a, min_value), max_value) + + +def compute(array_1, array_2, a, b, c): + """ + This function must implement the formula + np.clip(array_1, 2, 10) * a + array_2 * b + c + + array_1 and array_2 are 2D. + """ + x_max = array_1.shape[0] + y_max = array_1.shape[1] + + assert array_1.shape == array_2.shape + + result = np.zeros((x_max, y_max), dtype=array_1.dtype) + + for x in range(x_max): + for y in range(y_max): + tmp = clip(array_1[x, y], 2, 10) + tmp = tmp * a + array_2[x, y] * b + result[x, y] = tmp + c + + return result diff --git a/docs/examples/userguide/numpy_tutorial/compute_typed.pyx b/docs/examples/userguide/numpy_tutorial/compute_typed.pyx index 8218aa709..cccc1aa3b 100644 --- a/docs/examples/userguide/numpy_tutorial/compute_typed.pyx +++ b/docs/examples/userguide/numpy_tutorial/compute_typed.pyx @@ -1,50 +1,50 @@ -import numpy as np
-
-# We now need to fix a datatype for our arrays. I've used the variable
-# DTYPE for this, which is assigned to the usual NumPy runtime
-# type info object.
-DTYPE = np.intc
-
-# cdef means here that this function is a plain C function (so faster).
-# To get all the benefits, we type the arguments and the return value.
-cdef int clip(int a, int min_value, int max_value):
- return min(max(a, min_value), max_value)
-
-
-def compute(array_1, array_2, int a, int b, int c):
-
- # The "cdef" keyword is also used within functions to type variables. It
- # can only be used at the top indentation level (there are non-trivial
- # problems with allowing them in other places, though we'd love to see
- # good and thought out proposals for it).
- cdef Py_ssize_t x_max = array_1.shape[0]
- cdef Py_ssize_t y_max = array_1.shape[1]
-
- assert array_1.shape == array_2.shape
- assert array_1.dtype == DTYPE
- assert array_2.dtype == DTYPE
-
- result = np.zeros((x_max, y_max), dtype=DTYPE)
-
- # It is very important to type ALL your variables. You do not get any
- # warnings if not, only much slower code (they are implicitly typed as
- # Python objects).
- # For the "tmp" variable, we want to use the same data type as is
- # stored in the array, so we use int because it correspond to np.intc.
- # NB! An important side-effect of this is that if "tmp" overflows its
- # datatype size, it will simply wrap around like in C, rather than raise
- # an error like in Python.
-
- cdef int tmp
-
- # Py_ssize_t is the proper C type for Python array indices.
- cdef Py_ssize_t x, y
-
- for x in range(x_max):
- for y in range(y_max):
-
- tmp = clip(array_1[x, y], 2, 10)
- tmp = tmp * a + array_2[x, y] * b
- result[x, y] = tmp + c
-
- return result
+import numpy as np + +# We now need to fix a datatype for our arrays. I've used the variable +# DTYPE for this, which is assigned to the usual NumPy runtime +# type info object. +DTYPE = np.intc + +# cdef means here that this function is a plain C function (so faster). +# To get all the benefits, we type the arguments and the return value. +cdef int clip(int a, int min_value, int max_value): + return min(max(a, min_value), max_value) + + +def compute(array_1, array_2, int a, int b, int c): + + # The "cdef" keyword is also used within functions to type variables. It + # can only be used at the top indentation level (there are non-trivial + # problems with allowing them in other places, though we'd love to see + # good and thought out proposals for it). + cdef Py_ssize_t x_max = array_1.shape[0] + cdef Py_ssize_t y_max = array_1.shape[1] + + assert array_1.shape == array_2.shape + assert array_1.dtype == DTYPE + assert array_2.dtype == DTYPE + + result = np.zeros((x_max, y_max), dtype=DTYPE) + + # It is very important to type ALL your variables. You do not get any + # warnings if not, only much slower code (they are implicitly typed as + # Python objects). + # For the "tmp" variable, we want to use the same data type as is + # stored in the array, so we use int because it correspond to np.intc. + # NB! An important side-effect of this is that if "tmp" overflows its + # datatype size, it will simply wrap around like in C, rather than raise + # an error like in Python. + + cdef int tmp + + # Py_ssize_t is the proper C type for Python array indices. + cdef Py_ssize_t x, y + + for x in range(x_max): + for y in range(y_max): + + tmp = clip(array_1[x, y], 2, 10) + tmp = tmp * a + array_2[x, y] * b + result[x, y] = tmp + c + + return result diff --git a/docs/examples/userguide/parallelism/breaking_loop.pyx b/docs/examples/userguide/parallelism/breaking_loop.pyx index d11b179d9..5a30da5c3 100644 --- a/docs/examples/userguide/parallelism/breaking_loop.pyx +++ b/docs/examples/userguide/parallelism/breaking_loop.pyx @@ -1,13 +1,13 @@ -from cython.parallel import prange
-
-cdef int func(Py_ssize_t n):
- cdef Py_ssize_t i
-
- for i in prange(n, nogil=True):
- if i == 8:
- with gil:
- raise Exception()
- elif i == 4:
- break
- elif i == 2:
- return i
+from cython.parallel import prange + +cdef int func(Py_ssize_t n): + cdef Py_ssize_t i + + for i in prange(n, nogil=True): + if i == 8: + with gil: + raise Exception() + elif i == 4: + break + elif i == 2: + return i diff --git a/docs/examples/userguide/parallelism/cimport_openmp.pyx b/docs/examples/userguide/parallelism/cimport_openmp.pyx index 235ee10bc..797936fe7 100644 --- a/docs/examples/userguide/parallelism/cimport_openmp.pyx +++ b/docs/examples/userguide/parallelism/cimport_openmp.pyx @@ -1,13 +1,13 @@ -# tag: openmp
-# You can ignore the previous line.
-# It's for internal testing of the Cython documentation.
-
-from cython.parallel cimport parallel
-cimport openmp
-
-cdef int num_threads
-
-openmp.omp_set_dynamic(1)
-with nogil, parallel():
- num_threads = openmp.omp_get_num_threads()
- # ...
+# tag: openmp +# You can ignore the previous line. +# It's for internal testing of the Cython documentation. + +from cython.parallel cimport parallel +cimport openmp + +cdef int num_threads + +openmp.omp_set_dynamic(1) +with nogil, parallel(): + num_threads = openmp.omp_get_num_threads() + # ... diff --git a/docs/examples/userguide/parallelism/setup.py b/docs/examples/userguide/parallelism/setup.py index 0fb6026f7..fe6d0a64c 100644 --- a/docs/examples/userguide/parallelism/setup.py +++ b/docs/examples/userguide/parallelism/setup.py @@ -1,17 +1,16 @@ -from distutils.core import setup
-from distutils.extension import Extension
-from Cython.Build import cythonize
-
-ext_modules = [
- Extension(
- "hello",
- ["hello.pyx"],
- extra_compile_args=['-fopenmp'],
- extra_link_args=['-fopenmp'],
- )
-]
-
-setup(
- name='hello-parallel-world',
- ext_modules=cythonize(ext_modules),
-)
+from setuptools import Extension, setup +from Cython.Build import cythonize + +ext_modules = [ + Extension( + "hello", + ["hello.pyx"], + extra_compile_args=['-fopenmp'], + extra_link_args=['-fopenmp'], + ) +] + +setup( + name='hello-parallel-world', + ext_modules=cythonize(ext_modules), +) diff --git a/docs/examples/userguide/parallelism/simple_sum.pyx b/docs/examples/userguide/parallelism/simple_sum.pyx index 83b862ea6..929a988e5 100644 --- a/docs/examples/userguide/parallelism/simple_sum.pyx +++ b/docs/examples/userguide/parallelism/simple_sum.pyx @@ -1,10 +1,10 @@ -from cython.parallel import prange
-
-cdef int i
-cdef int n = 30
-cdef int sum = 0
-
-for i in prange(n, nogil=True):
- sum += i
-
-print(sum)
+from cython.parallel import prange + +cdef int i +cdef int n = 30 +cdef int sum = 0 + +for i in prange(n, nogil=True): + sum += i + +print(sum) diff --git a/docs/examples/userguide/sharing_declarations/landscaping.pyx b/docs/examples/userguide/sharing_declarations/landscaping.pyx index c54e74fd0..afc999e53 100644 --- a/docs/examples/userguide/sharing_declarations/landscaping.pyx +++ b/docs/examples/userguide/sharing_declarations/landscaping.pyx @@ -1,7 +1,7 @@ -cimport shrubbing
-import shrubbing
-
-def main():
- cdef shrubbing.Shrubbery sh
- sh = shrubbing.standard_shrubbery()
- print("Shrubbery size is", sh.width, 'x', sh.length)
+cimport shrubbing +import shrubbing + +def main(): + cdef shrubbing.Shrubbery sh + sh = shrubbing.standard_shrubbery() + print("Shrubbery size is", sh.width, 'x', sh.length) diff --git a/docs/examples/userguide/sharing_declarations/lunch.pyx b/docs/examples/userguide/sharing_declarations/lunch.pyx index 7bb2d4756..8b0911510 100644 --- a/docs/examples/userguide/sharing_declarations/lunch.pyx +++ b/docs/examples/userguide/sharing_declarations/lunch.pyx @@ -1,4 +1,4 @@ -cimport c_lunch
-
-def eject_tomato(float speed):
- c_lunch.eject_tomato(speed)
+cimport c_lunch + +def eject_tomato(float speed): + c_lunch.eject_tomato(speed) diff --git a/docs/examples/userguide/sharing_declarations/restaurant.pyx b/docs/examples/userguide/sharing_declarations/restaurant.pyx index 0c1dbf5c0..3257c681b 100644 --- a/docs/examples/userguide/sharing_declarations/restaurant.pyx +++ b/docs/examples/userguide/sharing_declarations/restaurant.pyx @@ -1,12 +1,12 @@ -from __future__ import print_function
-cimport dishes
-from dishes cimport spamdish
-
-cdef void prepare(spamdish *d):
- d.oz_of_spam = 42
- d.filler = dishes.sausage
-
-def serve():
- cdef spamdish d
- prepare(&d)
- print(f'{d.oz_of_spam} oz spam, filler no. {d.filler}')
+from __future__ import print_function +cimport dishes +from dishes cimport spamdish + +cdef void prepare(spamdish *d): + d.oz_of_spam = 42 + d.filler = dishes.sausage + +def serve(): + cdef spamdish d + prepare(&d) + print(f'{d.oz_of_spam} oz spam, filler no. {d.filler}') diff --git a/docs/examples/userguide/sharing_declarations/setup.py b/docs/examples/userguide/sharing_declarations/setup.py index 64804f97d..505b53e9d 100644 --- a/docs/examples/userguide/sharing_declarations/setup.py +++ b/docs/examples/userguide/sharing_declarations/setup.py @@ -1,4 +1,4 @@ -from distutils.core import setup
-from Cython.Build import cythonize
-
-setup(ext_modules=cythonize(["landscaping.pyx", "shrubbing.pyx"]))
+from setuptools import setup +from Cython.Build import cythonize + +setup(ext_modules=cythonize(["landscaping.pyx", "shrubbing.pyx"])) diff --git a/docs/examples/userguide/sharing_declarations/shrubbing.pyx b/docs/examples/userguide/sharing_declarations/shrubbing.pyx index a8b70dae2..bb97e7e77 100644 --- a/docs/examples/userguide/sharing_declarations/shrubbing.pyx +++ b/docs/examples/userguide/sharing_declarations/shrubbing.pyx @@ -1,7 +1,7 @@ -cdef class Shrubbery:
- def __cinit__(self, int w, int l):
- self.width = w
- self.length = l
-
-def standard_shrubbery():
- return Shrubbery(3, 7)
+cdef class Shrubbery: + def __cinit__(self, int w, int l): + self.width = w + self.length = l + +def standard_shrubbery(): + return Shrubbery(3, 7) diff --git a/docs/examples/userguide/sharing_declarations/spammery.pyx b/docs/examples/userguide/sharing_declarations/spammery.pyx index f65cf63d7..16cbda06e 100644 --- a/docs/examples/userguide/sharing_declarations/spammery.pyx +++ b/docs/examples/userguide/sharing_declarations/spammery.pyx @@ -1,11 +1,11 @@ -from __future__ import print_function
-
-from volume cimport cube
-
-def menu(description, size):
- print(description, ":", cube(size),
- "cubic metres of spam")
-
-menu("Entree", 1)
-menu("Main course", 3)
-menu("Dessert", 2)
+from __future__ import print_function + +from volume cimport cube + +def menu(description, size): + print(description, ":", cube(size), + "cubic metres of spam") + +menu("Entree", 1) +menu("Main course", 3) +menu("Dessert", 2) diff --git a/docs/examples/userguide/sharing_declarations/volume.pyx b/docs/examples/userguide/sharing_declarations/volume.pyx index 0fbab6fb7..0476b6068 100644 --- a/docs/examples/userguide/sharing_declarations/volume.pyx +++ b/docs/examples/userguide/sharing_declarations/volume.pyx @@ -1,2 +1,2 @@ -cdef float cube(float x):
- return x * x * x
+cdef float cube(float x): + return x * x * x diff --git a/docs/examples/userguide/wrapping_CPlusPlus/Rectangle.pxd b/docs/examples/userguide/wrapping_CPlusPlus/Rectangle.pxd index 68f949122..a26e69b51 100644 --- a/docs/examples/userguide/wrapping_CPlusPlus/Rectangle.pxd +++ b/docs/examples/userguide/wrapping_CPlusPlus/Rectangle.pxd @@ -1,7 +1,7 @@ cdef extern from "Rectangle.cpp":
pass
-# Decalre the class with cdef
+# Declare the class with cdef
cdef extern from "Rectangle.h" namespace "shapes":
cdef cppclass Rectangle:
Rectangle() except +
diff --git a/docs/examples/userguide/wrapping_CPlusPlus/cython_usage.pyx b/docs/examples/userguide/wrapping_CPlusPlus/cython_usage.pyx index 24192bf96..d074fa5ab 100644 --- a/docs/examples/userguide/wrapping_CPlusPlus/cython_usage.pyx +++ b/docs/examples/userguide/wrapping_CPlusPlus/cython_usage.pyx @@ -1,12 +1,12 @@ -# distutils: language = c++
-
-from Rectangle cimport Rectangle
-
-def main():
- rec_ptr = new Rectangle(1, 2, 3, 4) # Instantiate a Rectangle object on the heap
- try:
- rec_area = rec_ptr.getArea()
- finally:
- del rec_ptr # delete heap allocated object
-
- cdef Rectangle rec_stack # Instantiate a Rectangle object on the stack
+# distutils: language = c++ + +from Rectangle cimport Rectangle + +def main(): + rec_ptr = new Rectangle(1, 2, 3, 4) # Instantiate a Rectangle object on the heap + try: + rec_area = rec_ptr.getArea() + finally: + del rec_ptr # delete heap allocated object + + cdef Rectangle rec_stack # Instantiate a Rectangle object on the stack diff --git a/docs/examples/userguide/wrapping_CPlusPlus/function_templates.pyx b/docs/examples/userguide/wrapping_CPlusPlus/function_templates.pyx index 13c75426e..35d064fdd 100644 --- a/docs/examples/userguide/wrapping_CPlusPlus/function_templates.pyx +++ b/docs/examples/userguide/wrapping_CPlusPlus/function_templates.pyx @@ -1,7 +1,7 @@ -# distutils: language = c++
-
-cdef extern from "<algorithm>" namespace "std":
- T max[T](T a, T b)
-
-print(max[long](3, 4))
-print(max(1.5, 2.5)) # simple template argument deduction
+# distutils: language = c++ + +cdef extern from "<algorithm>" namespace "std": + T max[T](T a, T b) + +print(max[long](3, 4)) +print(max(1.5, 2.5)) # simple template argument deduction diff --git a/docs/examples/userguide/wrapping_CPlusPlus/iterate.pyx b/docs/examples/userguide/wrapping_CPlusPlus/iterate.pyx index ea0007e6a..cdce8910f 100644 --- a/docs/examples/userguide/wrapping_CPlusPlus/iterate.pyx +++ b/docs/examples/userguide/wrapping_CPlusPlus/iterate.pyx @@ -1,12 +1,12 @@ -# distutils: language = c++
-
-from libcpp.vector cimport vector
-
-def main():
- cdef vector[int] v = [4, 6, 5, 10, 3]
-
- cdef int value
- for value in v:
- print(value)
-
- return [x*x for x in v if x % 2 == 0]
+# distutils: language = c++ + +from libcpp.vector cimport vector + +def main(): + cdef vector[int] v = [4, 6, 5, 10, 3] + + cdef int value + for value in v: + print(value) + + return [x*x for x in v if x % 2 == 0] diff --git a/docs/examples/userguide/wrapping_CPlusPlus/nested_class.pyx b/docs/examples/userguide/wrapping_CPlusPlus/nested_class.pyx index e53f39b98..c5c764468 100644 --- a/docs/examples/userguide/wrapping_CPlusPlus/nested_class.pyx +++ b/docs/examples/userguide/wrapping_CPlusPlus/nested_class.pyx @@ -1,17 +1,17 @@ -# distutils: language = c++
-
-cdef extern from "<vector>" namespace "std":
- cdef cppclass vector[T]:
- cppclass iterator:
- T operator*()
- iterator operator++()
- bint operator==(iterator)
- bint operator!=(iterator)
- vector()
- void push_back(T&)
- T& operator[](int)
- T& at(int)
- iterator begin()
- iterator end()
-
-cdef vector[int].iterator iter #iter is declared as being of type vector<int>::iterator
+# distutils: language = c++ + +cdef extern from "<vector>" namespace "std": + cdef cppclass vector[T]: + cppclass iterator: + T operator*() + iterator operator++() + bint operator==(iterator) + bint operator!=(iterator) + vector() + void push_back(T&) + T& operator[](int) + T& at(int) + iterator begin() + iterator end() + +cdef vector[int].iterator iter #iter is declared as being of type vector<int>::iterator diff --git a/docs/examples/userguide/wrapping_CPlusPlus/python_to_cpp.pyx b/docs/examples/userguide/wrapping_CPlusPlus/python_to_cpp.pyx index 30bdb7bcb..b4be72c16 100644 --- a/docs/examples/userguide/wrapping_CPlusPlus/python_to_cpp.pyx +++ b/docs/examples/userguide/wrapping_CPlusPlus/python_to_cpp.pyx @@ -1,19 +1,19 @@ -# distutils: language = c++
-
-from libcpp.string cimport string
-from libcpp.vector cimport vector
-
-py_bytes_object = b'The knights who say ni'
-py_unicode_object = u'Those who hear them seldom live to tell the tale.'
-
-cdef string s = py_bytes_object
-print(s) # b'The knights who say ni'
-
-cdef string cpp_string = <string> py_unicode_object.encode('utf-8')
-print(cpp_string) # b'Those who hear them seldom live to tell the tale.'
-
-cdef vector[int] vect = range(1, 10, 2)
-print(vect) # [1, 3, 5, 7, 9]
-
-cdef vector[string] cpp_strings = b'It is a good shrubbery'.split()
-print(cpp_strings[1]) # b'is'
+# distutils: language = c++ + +from libcpp.string cimport string +from libcpp.vector cimport vector + +py_bytes_object = b'The knights who say ni' +py_unicode_object = u'Those who hear them seldom live to tell the tale.' + +cdef string s = py_bytes_object +print(s) # b'The knights who say ni' + +cdef string cpp_string = <string> py_unicode_object.encode('utf-8') +print(cpp_string) # b'Those who hear them seldom live to tell the tale.' + +cdef vector[int] vect = range(1, 10, 2) +print(vect) # [1, 3, 5, 7, 9] + +cdef vector[string] cpp_strings = b'It is a good shrubbery'.split() +print(cpp_strings[1]) # b'is' diff --git a/docs/examples/userguide/wrapping_CPlusPlus/rect_ptr.pyx b/docs/examples/userguide/wrapping_CPlusPlus/rect_ptr.pyx index 508e55dc6..0c48689e7 100644 --- a/docs/examples/userguide/wrapping_CPlusPlus/rect_ptr.pyx +++ b/docs/examples/userguide/wrapping_CPlusPlus/rect_ptr.pyx @@ -1,12 +1,12 @@ -# distutils: language = c++
-
-from Rectangle cimport Rectangle
-
-cdef class PyRectangle:
- cdef Rectangle*c_rect # hold a pointer to the C++ instance which we're wrapping
-
- def __cinit__(self, int x0, int y0, int x1, int y1):
- self.c_rect = new Rectangle(x0, y0, x1, y1)
-
- def __dealloc__(self):
- del self.c_rect
+# distutils: language = c++ + +from Rectangle cimport Rectangle + +cdef class PyRectangle: + cdef Rectangle*c_rect # hold a pointer to the C++ instance which we're wrapping + + def __cinit__(self, int x0, int y0, int x1, int y1): + self.c_rect = new Rectangle(x0, y0, x1, y1) + + def __dealloc__(self): + del self.c_rect diff --git a/docs/examples/userguide/wrapping_CPlusPlus/setup.py b/docs/examples/userguide/wrapping_CPlusPlus/setup.py index 0c89865d6..09009d28d 100644 --- a/docs/examples/userguide/wrapping_CPlusPlus/setup.py +++ b/docs/examples/userguide/wrapping_CPlusPlus/setup.py @@ -1,4 +1,4 @@ -from distutils.core import setup +from setuptools import setup from Cython.Build import cythonize diff --git a/docs/examples/userguide/wrapping_CPlusPlus/templates.pyx b/docs/examples/userguide/wrapping_CPlusPlus/templates.pyx index 8e7383ca2..4ff232b82 100644 --- a/docs/examples/userguide/wrapping_CPlusPlus/templates.pyx +++ b/docs/examples/userguide/wrapping_CPlusPlus/templates.pyx @@ -1,30 +1,30 @@ -# distutils: language = c++
-
-# import dereference and increment operators
-from cython.operator cimport dereference as deref, preincrement as inc
-
-cdef extern from "<vector>" namespace "std":
- cdef cppclass vector[T]:
- cppclass iterator:
- T operator*()
- iterator operator++()
- bint operator==(iterator)
- bint operator!=(iterator)
- vector()
- void push_back(T&)
- T& operator[](int)
- T& at(int)
- iterator begin()
- iterator end()
-
-cdef vector[int] *v = new vector[int]()
-cdef int i
-for i in range(10):
- v.push_back(i)
-
-cdef vector[int].iterator it = v.begin()
-while it != v.end():
- print(deref(it))
- inc(it)
-
-del v
+# distutils: language = c++ + +# import dereference and increment operators +from cython.operator cimport dereference as deref, preincrement as inc + +cdef extern from "<vector>" namespace "std": + cdef cppclass vector[T]: + cppclass iterator: + T operator*() + iterator operator++() + bint operator==(iterator) + bint operator!=(iterator) + vector() + void push_back(T&) + T& operator[](int) + T& at(int) + iterator begin() + iterator end() + +cdef vector[int] *v = new vector[int]() +cdef int i +for i in range(10): + v.push_back(i) + +cdef vector[int].iterator it = v.begin() +while it != v.end(): + print(deref(it)) + inc(it) + +del v diff --git a/docs/examples/userguide/wrapping_CPlusPlus/vector_demo.pyx b/docs/examples/userguide/wrapping_CPlusPlus/vector_demo.pyx index d7fdfc969..f1697e1ec 100644 --- a/docs/examples/userguide/wrapping_CPlusPlus/vector_demo.pyx +++ b/docs/examples/userguide/wrapping_CPlusPlus/vector_demo.pyx @@ -1,15 +1,15 @@ -# distutils: language = c++
-
-from libcpp.vector cimport vector
-
-cdef vector[int] vect
-cdef int i, x
-
-for i in range(10):
- vect.push_back(i)
-
-for i in range(10):
- print(vect[i])
-
-for x in vect:
- print(x)
+# distutils: language = c++ + +from libcpp.vector cimport vector + +cdef vector[int] vect +cdef int i, x + +for i in range(10): + vect.push_back(i) + +for i in range(10): + print(vect[i]) + +for x in vect: + print(x) diff --git a/docs/examples/userguide/wrapping_CPlusPlus/wrapper_vector.pyx b/docs/examples/userguide/wrapping_CPlusPlus/wrapper_vector.pyx index 592e83ad9..4cdf12fc2 100644 --- a/docs/examples/userguide/wrapping_CPlusPlus/wrapper_vector.pyx +++ b/docs/examples/userguide/wrapping_CPlusPlus/wrapper_vector.pyx @@ -1,17 +1,17 @@ -# distutils: language = c++
-
-from libcpp.vector cimport vector
-
-
-cdef class VectorStack:
- cdef vector[int] v
-
- def push(self, x):
- self.v.push_back(x)
-
- def pop(self):
- if self.v.empty():
- raise IndexError()
- x = self.v.back()
- self.v.pop_back()
- return x
+# distutils: language = c++ + +from libcpp.vector cimport vector + + +cdef class VectorStack: + cdef vector[int] v + + def push(self, x): + self.v.push_back(x) + + def pop(self): + if self.v.empty(): + raise IndexError() + x = self.v.back() + self.v.pop_back() + return x diff --git a/docs/index.rst b/docs/index.rst index 3e14d8dad..0103d8d9f 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -2,7 +2,7 @@ Welcome to Cython's Documentation ================================= -Also see the `Cython project homepage <http://cython.org/>`_. +Also see the `Cython project homepage <https://cython.org/>`_. .. toctree:: :maxdepth: 2 @@ -10,4 +10,6 @@ Also see the `Cython project homepage <http://cython.org/>`_. src/quickstart/index src/tutorial/index src/userguide/index + src/reference/index + Contributing <CONTRIBUTING> src/changes diff --git a/docs/make.bat b/docs/make.bat index fb25acc6a..3e719777d 100644 --- a/docs/make.bat +++ b/docs/make.bat @@ -56,7 +56,7 @@ if errorlevel 9009 ( echo.may add the Sphinx directory to PATH.
echo.
echo.If you don't have Sphinx installed, grab it from
- echo.http://sphinx-doc.org/
+ echo.https://sphinx-doc.org/
exit /b 1
)
diff --git a/docs/sphinxext/cython_highlighting.py b/docs/sphinxext/cython_highlighting.py index 06e11b891..37c1d6058 100644 --- a/docs/sphinxext/cython_highlighting.py +++ b/docs/sphinxext/cython_highlighting.py @@ -14,7 +14,7 @@ line_re = re.compile('.*?\n') class CythonLexer(RegexLexer): """ - For `Cython <http://cython.org>`_ source code. + For `Cython <https://cython.org/>`_ source code. """ name = 'Cython' @@ -128,7 +128,7 @@ class CythonLexer(RegexLexer): (r'(\s+)(as)(\s+)', bygroups(Text, Keyword, Text)), (r'[a-zA-Z_][a-zA-Z0-9_.]*', Name.Namespace), (r'(\s*)(,)(\s*)', bygroups(Text, Operator, Text)), - (r'', Text, '#pop') # all else: go back + (r'', Text, '#pop') # all else: go back ], 'fromimport': [ (r'(\s+)(c?import)\b', bygroups(Text, Keyword), '#pop'), @@ -155,12 +155,12 @@ class CythonLexer(RegexLexer): ], 'dqs': [ (r'"', String, '#pop'), - (r'\\\\|\\"|\\\n', String.Escape), # included here again for raw strings + (r'\\\\|\\"|\\\n', String.Escape), # included here again for raw strings include('strings') ], 'sqs': [ (r"'", String, '#pop'), - (r"\\\\|\\'|\\\n", String.Escape), # included here again for raw strings + (r"\\\\|\\'|\\\n", String.Escape), # included here again for raw strings include('strings') ], 'tdqs': [ diff --git a/docs/src/donating.rst b/docs/src/donating.rst new file mode 100644 index 000000000..3d12a8691 --- /dev/null +++ b/docs/src/donating.rst @@ -0,0 +1,44 @@ +:orphan: + +🌷️ Thank you for your interest in supporting Cython! 🌷️ +========================================================= + +Managing, maintaining and advancing a project as large as Cython takes +**a lot of time and dedication**. + +**Your support can make a difference** +for a great tool that helps you every day! + +Please consider signing a subscription for continuous project support via + +* `GitHub Sponsors <https://github.com/users/scoder/sponsorship>`_ +* `Tidelift <https://tidelift.com/subscription/pkg/pypi-cython>`_ +* `PayPal <https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=HLS9JEYD4ETB6&source=url>`_ + +or donating via + +* `PayPal <https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=HLS9JEYD4ETB6&source=url>`_ + +Note that PayPal takes 5 - 15% fees for small non-EUR payments, +which is money that *you pay without helping us*. +Consider signing up for a GitHub Sponsors subscription instead, +which is currently free of additional charges. + + +Legal Notice for Donations +-------------------------- + +Any donation that you make to the Cython project is voluntary and +is not a fee for any services, goods, or advantages. By making +a donation to the Cython project, you acknowledge that we have the +right to use the money you donate in any lawful way and for any +lawful purpose we see fit and we are not obligated to disclose +the way and purpose to any party unless required by applicable +law. Although Cython is free software, to the best of our knowledge +the Cython project does not have any tax exempt status. The Cython +project is neither a registered non-profit corporation nor a +registered charity in any country. Your donation may or may not +be tax-deductible; please consult your tax advisor in this matter. +We will not publish or disclose your name and/or e-mail address +without your consent, unless required by applicable law. Your +donation is non-refundable. diff --git a/docs/src/quickstart/build.rst b/docs/src/quickstart/build.rst index 628ed2604..0da55750f 100644 --- a/docs/src/quickstart/build.rst +++ b/docs/src/quickstart/build.rst @@ -8,18 +8,18 @@ Cython code must, unlike Python, be compiled. This happens in two stages: - The ``.c`` file is compiled by a C compiler to a ``.so`` file (or ``.pyd`` on Windows) which can be ``import``-ed directly into a Python session. - Distutils or setuptools take care of this part. + `setuptools <https://setuptools.readthedocs.io/>`_ takes care of this part. Although Cython can call them for you in certain cases. - -To understand fully the Cython + distutils/setuptools build process, + +To understand fully the Cython + setuptools build process, one may want to read more about `distributing Python modules <https://docs.python.org/3/distributing/index.html>`_. There are several ways to build Cython code: - - Write a distutils/setuptools ``setup.py``. This is the normal and recommended way. + - Write a setuptools ``setup.py``. This is the normal and recommended way. - Use :ref:`Pyximport<pyximport>`, importing Cython ``.pyx`` files as if they - were ``.py`` files (using distutils to compile and build in the background). + were ``.py`` files (using setuptools to compile and build in the background). This method is easier than writing a ``setup.py``, but is not very flexible. So you'll need to write a ``setup.py`` if, for example, you need certain compilations options. - Run the ``cython`` command-line utility manually to produce the ``.c`` file @@ -30,12 +30,12 @@ There are several ways to build Cython code: both of which allow Cython code inline. This is the easiest way to get started writing Cython code and running it. -Currently, using distutils or setuptools is the most common way Cython files are built and distributed. +Currently, using setuptools is the most common way Cython files are built and distributed. The other methods are described in more detail in the :ref:`compilation` section of the reference manual. -Building a Cython module using distutils ----------------------------------------- +Building a Cython module using setuptools +----------------------------------------- Imagine a simple "hello world" script in a file ``hello.pyx``: @@ -49,11 +49,10 @@ To build, run ``python setup.py build_ext --inplace``. Then simply start a Python session and do ``from hello import say_hello_to`` and use the imported function as you see fit. -One caveat if you use setuptools instead of distutils, the default -action when running ``python setup.py install`` is to create a zipped -``egg`` file which will not work with ``cimport`` for ``pxd`` files -when you try to use them from a dependent package. -To prevent this, include ``zip_safe=False`` in the arguments to ``setup()``. +One caveat: the default action when running ``python setup.py install`` is to +create a zipped ``egg`` file which will not work with ``cimport`` for ``pxd`` +files when you try to use them from a dependent package. To prevent this, +include ``zip_safe=False`` in the arguments to ``setup()``. .. _jupyter-notebook: @@ -104,5 +103,6 @@ Using the Sage notebook functions defined in a Cython cell imported into the running session. -.. [Jupyter] http://jupyter.org/ -.. [Sage] W. Stein et al., Sage Mathematics Software, http://www.sagemath.org/ +.. [Jupyter] https://jupyter.org/ +.. + [Sage] W. Stein et al., Sage Mathematics Software, https://www.sagemath.org/ diff --git a/docs/src/quickstart/htmlreport.png b/docs/src/quickstart/htmlreport.png Binary files differindex cc30cec9f..0fc5e7bbe 100644 --- a/docs/src/quickstart/htmlreport.png +++ b/docs/src/quickstart/htmlreport.png diff --git a/docs/src/quickstart/install.rst b/docs/src/quickstart/install.rst index a71adffb5..ec30076cf 100644 --- a/docs/src/quickstart/install.rst +++ b/docs/src/quickstart/install.rst @@ -7,9 +7,7 @@ Many scientific Python distributions, such as Anaconda [Anaconda]_, Enthought Canopy [Canopy]_, and Sage [Sage]_, bundle Cython and no setup is needed. Note however that if your distribution ships a version of Cython which is too old you can still -use the instructions below to update Cython. Everything in this -tutorial should work with Cython 0.11.2 and newer, unless a footnote -says otherwise. +use the instructions below to update Cython. Unlike most Python software, Cython requires a C compiler to be present on the system. The details of getting a C compiler varies @@ -41,7 +39,7 @@ The simplest way of installing Cython is by using ``pip``:: The newest Cython release can always be downloaded from -http://cython.org. Unpack the tarball or zip file, enter the +https://cython.org/. Unpack the tarball or zip file, enter the directory, and then run:: python setup.py install @@ -59,4 +57,4 @@ with .. [Anaconda] https://docs.anaconda.com/anaconda/ .. [Canopy] https://www.enthought.com/product/canopy/ -.. [Sage] W. Stein et al., Sage Mathematics Software, http://www.sagemath.org/ +.. [Sage] W. Stein et al., Sage Mathematics Software, https://www.sagemath.org/ diff --git a/docs/src/quickstart/jupyter.png b/docs/src/quickstart/jupyter.png Binary files differindex 84b3543ad..34b38df6d 100644 --- a/docs/src/quickstart/jupyter.png +++ b/docs/src/quickstart/jupyter.png diff --git a/docs/src/quickstart/overview.rst b/docs/src/quickstart/overview.rst index 1585f89fe..1a378e837 100644 --- a/docs/src/quickstart/overview.rst +++ b/docs/src/quickstart/overview.rst @@ -1,7 +1,7 @@ Cython - an overview ==================== -[Cython] is a programming language that makes writing C extensions +[Cython]_ is a programming language that makes writing C extensions for the Python language as easy as Python itself. It aims to become a superset of the [Python]_ language which gives it high-level, object-oriented, functional, and dynamic programming. Its main feature @@ -44,13 +44,13 @@ thus merges the two worlds into a very broadly applicable programming language. .. [Cython] G. Ewing, R. W. Bradshaw, S. Behnel, D. S. Seljebotn et al., - The Cython compiler, http://cython.org. + The Cython compiler, https://cython.org/. .. [IronPython] Jim Hugunin et al., https://archive.codeplex.com/?p=IronPython. .. [Jython] J. Huginin, B. Warsaw, F. Bock, et al., - Jython: Python for the Java platform, http://www.jython.org. + Jython: Python for the Java platform, https://www.jython.org. .. [PyPy] The PyPy Group, PyPy: a Python implementation written in Python, - http://pypy.org. + https://pypy.org/. .. [Pyrex] G. Ewing, Pyrex: C-Extensions for Python, - http://www.cosc.canterbury.ac.nz/greg.ewing/python/Pyrex/ + https://www.cosc.canterbury.ac.nz/greg.ewing/python/Pyrex/ .. [Python] G. van Rossum et al., The Python programming language, https://www.python.org/. diff --git a/docs/src/reference/directives.rst b/docs/src/reference/directives.rst index f527536a8..46d7fa9c4 100644 --- a/docs/src/reference/directives.rst +++ b/docs/src/reference/directives.rst @@ -1,3 +1,5 @@ +:orphan: + Compiler Directives =================== diff --git a/docs/src/reference/extension_types.rst b/docs/src/reference/extension_types.rst index ea26f38a0..3c5cc828d 100644 --- a/docs/src/reference/extension_types.rst +++ b/docs/src/reference/extension_types.rst @@ -1,3 +1,5 @@ +:orphan: + .. highlight:: cython *************** diff --git a/docs/src/reference/language_basics.rst b/docs/src/reference/language_basics.rst index bd8b1e38c..8d7cd0b06 100644 --- a/docs/src/reference/language_basics.rst +++ b/docs/src/reference/language_basics.rst @@ -1,3 +1,5 @@ +:orphan: + .. highlight:: cython diff --git a/docs/src/tutorial/appendix.rst b/docs/src/tutorial/appendix.rst index b0ab0426e..82f225bbf 100644 --- a/docs/src/tutorial/appendix.rst +++ b/docs/src/tutorial/appendix.rst @@ -2,7 +2,7 @@ Appendix: Installing MinGW on Windows ===================================== 1. Download the MinGW installer from - http://www.mingw.org/wiki/HOWTO_Install_the_MinGW_GCC_Compiler_Suite. + https://www.mingw.org/wiki/HOWTO_Install_the_MinGW_GCC_Compiler_Suite. (As of this writing, the download link is a bit difficult to find; it's under "About" in the menu on the left-hand side). You want the file @@ -28,4 +28,65 @@ procedure. Any contributions towards making the Windows install process smoother is welcomed; it is an unfortunate fact that none of the regular Cython developers have convenient access to Windows. +Python 3.8+ +----------- + +Since Python 3.8, the search paths of DLL dependencies has been reset. +(`changelog <https://docs.python.org/3/whatsnew/3.8.html#bpo-36085-whatsnew>`_) + +Only the system paths, the directory containing the DLL or PYD file +are searched for load-time dependencies. +Instead, a new function `os.add_dll_directory() <https://docs.python.org/3.8/library/os.html#os.add_dll_directory>`_ +was added to supply additional search paths. But such a runtime update is not applicable in all situations. + +Unlike MSVC, MinGW has its owned standard libraries such as ``libstdc++-6.dll``, +which are not placed in the system path (such as ``C:\Windows\System32``). +For a C++ example, you can check the dependencies by MSVC tool ``dumpbin``:: + + > dumpbin /dependents my_gnu_extension.cp38-win_amd64.pyd + ... + Dump of file my_gnu_extension.cp38-win_amd64.pyd + + File Type: DLL + + Image has the following dependencies: + + python38.dll + KERNEL32.dll + msvcrt.dll + libgcc_s_seh-1.dll + libstdc++-6.dll + ... + +These standard libraries can be embedded via static linking, by adding the following options to the linker:: + + -static-libgcc -static-libstdc++ -Wl,-Bstatic,--whole-archive -lwinpthread -Wl,--no-whole-archive + +In ``setup.py``, a cross platform config can be added through +extending ``build_ext`` class:: + + from setuptools import setup + from setuptools.command.build_ext import build_ext + + link_args = ['-static-libgcc', + '-static-libstdc++', + '-Wl,-Bstatic,--whole-archive', + '-lwinpthread', + '-Wl,--no-whole-archive'] + + ... # Add extensions + + class Build(build_ext): + def build_extensions(self): + if self.compiler.compiler_type == 'mingw32': + for e in self.extensions: + e.extra_link_args = link_args + super(Build, self).build_extensions() + + setup( + ... + cmdclass={'build_ext': Build}, + ... + ) + .. [WinInst] https://github.com/cython/cython/wiki/CythonExtensionsOnWindows diff --git a/docs/src/tutorial/caveats.rst b/docs/src/tutorial/caveats.rst index 65443ae26..192313162 100644 --- a/docs/src/tutorial/caveats.rst +++ b/docs/src/tutorial/caveats.rst @@ -5,7 +5,6 @@ Since Cython mixes C and Python semantics, some things may be a bit surprising or unintuitive. Work always goes on to make Cython more natural for Python users, so this list may change in the future. - - ``10**-2 == 0``, instead of ``0.01`` like in Python. - Given two typed ``int`` variables ``a`` and ``b``, ``a % b`` has the same sign as the second argument (following Python semantics) rather than having the same sign as the first (as in C). The C behavior can be diff --git a/docs/src/tutorial/clibraries.rst b/docs/src/tutorial/clibraries.rst index dd91d9ef2..08c491d36 100644 --- a/docs/src/tutorial/clibraries.rst +++ b/docs/src/tutorial/clibraries.rst @@ -24,7 +24,7 @@ decide to use its double ended queue implementation. To make the handling easier, however, you decide to wrap it in a Python extension type that can encapsulate all memory management. -.. [CAlg] Simon Howard, C Algorithms library, http://c-algorithms.sourceforge.net/ +.. [CAlg] Simon Howard, C Algorithms library, https://fragglet.github.io/c-algorithms/ Defining external declarations @@ -180,11 +180,10 @@ Compiling and linking ===================== At this point, we have a working Cython module that we can test. To -compile it, we need to configure a ``setup.py`` script for distutils. +compile it, we need to configure a ``setup.py`` script for setuptools. Here is the most basic script for compiling a Cython module:: - from distutils.core import setup - from distutils.extension import Extension + from setuptools import Extension, setup from Cython.Build import cythonize setup( @@ -193,7 +192,7 @@ Here is the most basic script for compiling a Cython module:: To build against the external C library, we need to make sure Cython finds the necessary libraries. -There are two ways to archive this. First we can tell distutils where to find +There are two ways to archive this. First we can tell setuptools where to find the c-source to compile the :file:`queue.c` implementation automatically. Alternatively, we can build and install C-Alg as system library and dynamically link it. The latter is useful if other applications also use C-Alg. @@ -221,7 +220,7 @@ To build the c-code automatically we need to include compiler directives in `que cqueue.queue_free(self._c_queue) The ``sources`` compiler directive gives the path of the C -files that distutils is going to compile and +files that setuptools is going to compile and link (statically) into the resulting extension module. In general all relevant header files should be found in ``include_dirs``. Now we can build the project using:: diff --git a/docs/src/tutorial/cython_tutorial.rst b/docs/src/tutorial/cython_tutorial.rst index f80a8b016..c4cb64101 100644 --- a/docs/src/tutorial/cython_tutorial.rst +++ b/docs/src/tutorial/cython_tutorial.rst @@ -18,7 +18,7 @@ serve for now.) The Cython compiler will convert it into C code which makes equivalent calls to the Python/C API. But Cython is much more than that, because parameters and variables can be -declared to have C data types. Code which manipulates Python values and C +declared to have C data types. Code which manipulates :term:`Python values<Python object>` and C values can be freely intermixed, with conversions occurring automatically wherever possible. Reference count maintenance and error checking of Python operations is also automatic, and the full power of Python's exception @@ -40,7 +40,7 @@ Save this code in a file named :file:`helloworld.pyx`. Now we need to create the :file:`setup.py`, which is like a python Makefile (for more information see :ref:`compilation`). Your :file:`setup.py` should look like:: - from distutils.core import setup + from setuptools import setup from Cython.Build import cythonize setup( @@ -148,7 +148,7 @@ The result is stored in the C array ``p`` during processing, and will be copied into a Python list at the end (line 22). .. NOTE:: You cannot create very large arrays in this manner, because - they are allocated on the C function call stack, which is a + they are allocated on the C function call :term:`stack<Stack allocation>`, which is a rather precious and scarce resource. To request larger arrays, or even arrays with a length only known at runtime, @@ -273,7 +273,7 @@ compile it with Cython (without changing the code). We will also change the name file to ``example_py_cy.py`` to differentiate it from the others. Now the ``setup.py`` looks like this:: - from distutils.core import setup + from setuptools import setup from Cython.Build import cythonize setup( diff --git a/docs/src/tutorial/embedding.rst b/docs/src/tutorial/embedding.rst new file mode 100644 index 000000000..3f6325428 --- /dev/null +++ b/docs/src/tutorial/embedding.rst @@ -0,0 +1,77 @@ +.. highlight:: cython + +.. _embedding: + +********************************************** +Embedding Cython modules in C/C++ applications +********************************************** + +**This is a stub documentation page. PRs very welcome.** + +Quick links: + +* `CPython docs <https://docs.python.org/3/extending/embedding.html>`_ + +* `Cython Wiki <https://github.com/cython/cython/wiki/EmbeddingCython>`_ + +* See the ``--embed`` option to the ``cython`` and ``cythonize`` frontends + for generating a C main function and the + `cython_freeze <https://github.com/cython/cython/blob/master/bin/cython_freeze>`_ + script for merging multiple extension modules into one library. + +* `Embedding demo program <https://github.com/cython/cython/tree/master/Demos/embed>`_ + +* See the documentation of the `module init function + <https://docs.python.org/3/extending/extending.html#the-module-s-method-table-and-initialization-function>`_ + in CPython and `PEP 489 <https://www.python.org/dev/peps/pep-0489/>`_ regarding the module + initialisation mechanism in CPython 3.5 and later. + + +Initialising your main module +============================= + +Most importantly, DO NOT call the module init function instead of importing +the module. This is not the right way to initialise an extension module. +(It was always wrong but used to work before, but since Python 3.5, it is +wrong *and* no longer works.) + +For details, see the documentation of the +`module init function <https://docs.python.org/3/extending/extending.html#the-module-s-method-table-and-initialization-function>`_ +in CPython and `PEP 489 <https://www.python.org/dev/peps/pep-0489/>`_ regarding the module +initialisation mechanism in CPython 3.5 and later. + +The `PyImport_AppendInittab() <https://docs.python.org/3/c-api/import.html#c.PyImport_AppendInittab>`_ +function in CPython allows registering statically (or dynamically) linked extension +modules for later imports. An example is given in the documentation of the module +init function that is linked above. + + +Embedding example code +====================== + +The following is a simple example that shows the main steps for embedding a +Cython module (``embedded.pyx``) in Python 3.x. + +First, here is a Cython module that exports a C function to be called by external +code. Note that the ``say_hello_from_python()`` function is declared as ``public`` +to export it as a linker symbol that can be used by other C files, which in this +case is ``embedded_main.c``. + +.. literalinclude:: ../../examples/tutorial/embedding/embedded.pyx + +The C ``main()`` function of your program could look like this: + +.. literalinclude:: ../../examples/tutorial/embedding/embedded_main.c + :linenos: + :language: c + +(Adapted from the `CPython documentation +<https://docs.python.org/3/extending/extending.html#the-module-s-method-table-and-initialization-function>`_.) + +Instead of writing such a ``main()`` function yourself, you can also let +Cython generate one into your module's C file with the ``cython --embed`` +option. Or use the +`cython_freeze <https://github.com/cython/cython/blob/master/bin/cython_freeze>`_ +script to embed multiple modules. See the +`embedding demo program <https://github.com/cython/cython/tree/master/Demos/embed>`_ +for a complete example setup. diff --git a/docs/src/tutorial/external.rst b/docs/src/tutorial/external.rst index b55b96505..83ec75375 100644 --- a/docs/src/tutorial/external.rst +++ b/docs/src/tutorial/external.rst @@ -30,6 +30,8 @@ your code is being compiled with, you can do this: .. literalinclude:: ../../examples/tutorial/external/py_version_hex.pyx +.. _libc.math: + Cython also provides declarations for the C math library: .. literalinclude:: ../../examples/tutorial/external/libc_sin.pyx @@ -41,7 +43,7 @@ Dynamic linking The libc math library is special in that it is not linked by default on some Unix-like systems, such as Linux. In addition to cimporting the declarations, you must configure your build system to link against the -shared library ``m``. For distutils, it is enough to add it to the +shared library ``m``. For setuptools, it is enough to add it to the ``libraries`` parameter of the ``Extension()`` setup: .. literalinclude:: ../../examples/tutorial/external/setup.py diff --git a/docs/src/tutorial/htmlreport.png b/docs/src/tutorial/htmlreport.png Binary files differindex 4fc98f3e0..08c3d3472 100644 --- a/docs/src/tutorial/htmlreport.png +++ b/docs/src/tutorial/htmlreport.png diff --git a/docs/src/tutorial/index.rst b/docs/src/tutorial/index.rst index 14bc5d9ee..67ff4ff1c 100644 --- a/docs/src/tutorial/index.rst +++ b/docs/src/tutorial/index.rst @@ -13,10 +13,10 @@ Tutorials profiling_tutorial strings memory_allocation + embedding pure numpy array readings related_work appendix - diff --git a/docs/src/tutorial/memory_allocation.rst b/docs/src/tutorial/memory_allocation.rst index f53c1119a..a6dc0ae1b 100644 --- a/docs/src/tutorial/memory_allocation.rst +++ b/docs/src/tutorial/memory_allocation.rst @@ -20,9 +20,9 @@ amount of overhead, which can then makes a case for doing manual memory management in C. Simple C values and structs (such as a local variable ``cdef double x``) are -usually allocated on the stack and passed by value, but for larger and more +usually :term:`allocated on the stack<Stack allocation>` and passed by value, but for larger and more complicated objects (e.g. a dynamically-sized list of doubles), the memory must -be manually requested and released. C provides the functions :c:func:`malloc`, +be :term:`manually requested and released<Heap allocation>`. C provides the functions :c:func:`malloc`, :c:func:`realloc`, and :c:func:`free` for this purpose, which can be imported in cython from ``clibc.stdlib``. Their signatures are: diff --git a/docs/src/tutorial/profiling_tutorial.rst b/docs/src/tutorial/profiling_tutorial.rst index a7cfab0a8..903522de2 100644 --- a/docs/src/tutorial/profiling_tutorial.rst +++ b/docs/src/tutorial/profiling_tutorial.rst @@ -75,8 +75,8 @@ Enabling coverage analysis -------------------------- Since Cython 0.23, line tracing (see above) also enables support for coverage -reporting with the `coverage.py <http://coverage.readthedocs.io/>`_ tool. -To make the coverage analysis understand Cython modules, you also need to enable +reporting with the `coverage.py <https://coverage.readthedocs.io/>`_ tool. To +make the coverage analysis understand Cython modules, you also need to enable Cython's coverage plugin in your ``.coveragerc`` file as follows: .. code-block:: ini @@ -274,5 +274,3 @@ faster than calling pow(x,0.5). Even so, the result we achieved here is quite satisfactory: we came up with a solution that is much faster then our original Python version while retaining functionality and readability. - - diff --git a/docs/src/tutorial/pure.rst b/docs/src/tutorial/pure.rst index 775e7719c..a44ff8967 100644 --- a/docs/src/tutorial/pure.rst +++ b/docs/src/tutorial/pure.rst @@ -13,7 +13,7 @@ To go beyond that, Cython provides language constructs to add static typing and cythonic functionalities to a Python module to make it run much faster when compiled, while still allowing it to be interpreted. This is accomplished via an augmenting ``.pxd`` file, via Python -type annotations (following +type :ref:`pep484_type_annotations` (following `PEP 484 <https://www.python.org/dev/peps/pep-0484/>`_ and `PEP 526 <https://www.python.org/dev/peps/pep-0526/>`_), and/or via special functions and decorators available after importing the magic @@ -82,7 +82,7 @@ in the :file:`.pxd`, that is, to be accessible from Python, In the example above, the type of the local variable `a` in `myfunction()` -is not fixed and will thus be a Python object. To statically type it, one +is not fixed and will thus be a :term:`Python object`. To statically type it, one can use Cython's ``@cython.locals`` decorator (see :ref:`magic_attributes`, and :ref:`magic_attributes_pxd`). @@ -153,26 +153,10 @@ Static typing @exceptval(-1, check=False) # cdef int func() except -1: @exceptval(check=True) # cdef int func() except *: @exceptval(-1, check=True) # cdef int func() except? -1: + @exceptval(check=False) # no exception checking/propagation -* Python annotations can be used to declare argument types, as shown in the - following example. To avoid conflicts with other kinds of annotation - usages, this can be disabled with the directive ``annotation_typing=False``. - - .. literalinclude:: ../../examples/tutorial/pure/annotations.py - - This can be combined with the ``@cython.exceptval()`` decorator for non-Python - return types: - - .. literalinclude:: ../../examples/tutorial/pure/exceptval.py - - Since version 0.27, Cython also supports the variable annotations defined - in `PEP 526 <https://www.python.org/dev/peps/pep-0526/>`_. This allows to - declare types of variables in a Python 3.6 compatible way as follows: - - .. literalinclude:: ../../examples/tutorial/pure/pep_526.py - - There is currently no way to express the visibility of object attributes. - + If exception propagation is disabled, any Python exceptions that are raised + inside of the function will be printed and ignored. C types ^^^^^^^ @@ -289,6 +273,52 @@ can be augmented with the following :file:`.pxd` file :file:`dostuff.pxd`: The :func:`cython.declare()` function can be used to specify types for global variables in the augmenting :file:`.pxd` file. +.. _pep484_type_annotations: + +PEP-484 type annotations +------------------------ + +Python `type hints <https://www.python.org/dev/peps/pep-0484>`_ +can be used to declare argument types, as shown in the +following example. To avoid conflicts with other kinds of annotation +usages, this can be disabled with the directive ``annotation_typing=False``. + + .. literalinclude:: ../../examples/tutorial/pure/annotations.py + +Note the use of ``cython.int`` rather than ``int`` - Cython does not translate +an ``int`` annotation to a C integer by default since the behaviour can be +quite different with respect to overflow and division. + +Annotations can be combined with the ``@cython.exceptval()`` decorator for non-Python +return types: + + .. literalinclude:: ../../examples/tutorial/pure/exceptval.py + +Note that the default exception handling behaviour when returning C numeric types +is to check for ``-1``, and if that was returned, check Python's error indicator +for an exception. This means, if no ``@exceptval`` decorator is provided, and the +return type is a numeric type, then the default with type annotations is +``@exceptval(-1, check=True)``, in order to make sure that exceptions are correctly +and efficiently reported to the caller. Exception propagation can be disabled +explicitly with ``@exceptval(check=False)``, in which case any Python exceptions +raised inside of the function will be printed and ignored. + +Since version 0.27, Cython also supports the variable annotations defined +in `PEP 526 <https://www.python.org/dev/peps/pep-0526/>`_. This allows to +declare types of variables in a Python 3.6 compatible way as follows: + +.. literalinclude:: ../../examples/tutorial/pure/pep_526.py + +There is currently no way to express the visibility of object attributes. + +Cython does not support the full range of annotations described by PEP-484. +For example it does not currently understand features from the ``typing`` module +such as ``Optional[]`` or typed containers such as ``List[str]``. This is partly +because some of these type hints are not relevant for the compilation to +efficient C code. In other cases, however, where the generated C code could +benefit from these type hints but does not currently, help is welcome to +improve the type analysis in Cython. + Tips and Tricks --------------- diff --git a/docs/src/tutorial/pxd_files.rst b/docs/src/tutorial/pxd_files.rst index 5fc5b0a89..75fc7b1af 100644 --- a/docs/src/tutorial/pxd_files.rst +++ b/docs/src/tutorial/pxd_files.rst @@ -11,27 +11,25 @@ using the ``cimport`` keyword. ``pxd`` files have many use-cases: - 1. They can be used for sharing external C declarations. - 2. They can contain functions which are well suited for inlining by - the C compiler. Such functions should be marked ``inline``, example: - :: +1. They can be used for sharing external C declarations. +2. They can contain functions which are well suited for inlining by + the C compiler. Such functions should be marked ``inline``, example:: cdef inline int int_min(int a, int b): return b if b < a else a - 3. When accompanying an equally named ``pyx`` file, they +3. When accompanying an equally named ``pyx`` file, they provide a Cython interface to the Cython module so that other Cython modules can communicate with it using a more efficient protocol than the Python one. In our integration example, we might break it up into ``pxd`` files like this: - 1. Add a ``cmath.pxd`` function which defines the C functions available from +1. Add a ``cmath.pxd`` function which defines the C functions available from the C ``math.h`` header file, like ``sin``. Then one would simply do ``from cmath cimport sin`` in ``integrate.pyx``. - 2. Add a ``integrate.pxd`` so that other modules written in Cython - can define fast custom functions to integrate. - :: +2. Add a ``integrate.pxd`` so that other modules written in Cython + can define fast custom functions to integrate:: cdef class Function: cpdef evaluate(self, double x) @@ -41,3 +39,35 @@ In our integration example, we might break it up into ``pxd`` files like this: Note that if you have a cdef class with attributes, the attributes must be declared in the class declaration ``pxd`` file (if you use one), not the ``pyx`` file. The compiler will tell you about this. + + +__init__.pxd +^^^^^^^^^^^^ + +Cython also supports ``__init__.pxd`` files for declarations in package's +namespaces, similar to ``__init__.py`` files in Python. + +Continuing the integration example, we could package the module as follows: + +1. Place the module files in a directory tree as one usually would for + Python:: + + CyIntegration/ + ├── __init__.pyx + ├── __init__.pxd + ├── integrate.pyx + └── integrate.pxd + +2. In ``__init__.pxd``, use ``cimport`` for any declarations that one + would want to be available from the package's main namespace:: + + from CyIntegration cimport integrate + + Other modules would then be able to use ``cimport`` on the package in + order to recursively gain faster, Cython access to the entire package + and the data declared in its modules:: + + cimport CyIntegration + + cpdef do_integration(CyIntegration.integrate.Function f): + return CyIntegration.integrate.integrate(f, 0., 2., 1) diff --git a/docs/src/tutorial/python_division.png b/docs/src/tutorial/python_division.png Binary files differindex 617be942c..0fe83f053 100644 --- a/docs/src/tutorial/python_division.png +++ b/docs/src/tutorial/python_division.png diff --git a/docs/src/tutorial/readings.rst b/docs/src/tutorial/readings.rst index a3f09d39e..80ed26e66 100644 --- a/docs/src/tutorial/readings.rst +++ b/docs/src/tutorial/readings.rst @@ -1,7 +1,7 @@ Further reading =============== -The main documentation is located at http://docs.cython.org/. Some +The main documentation is located at https://docs.cython.org/. Some recent features might not have documentation written yet, in such cases some notes can usually be found in the form of a Cython Enhancement Proposal (CEP) on https://github.com/cython/cython/wiki/enhancements. @@ -16,7 +16,7 @@ features for managing it. Finally, don't hesitate to ask questions (or post reports on successes!) on the Cython users mailing list [UserList]_. The Cython developer mailing list, [DevList]_, is also open to everybody, but -focusses on core development issues. Feel free to use it to report a +focuses on core development issues. Feel free to use it to report a clear bug, to ask for guidance if you have time to spare to develop Cython, or if you have suggestions for future development. diff --git a/docs/src/tutorial/related_work.rst b/docs/src/tutorial/related_work.rst index 01cc5b327..af55ae88b 100644 --- a/docs/src/tutorial/related_work.rst +++ b/docs/src/tutorial/related_work.rst @@ -41,8 +41,9 @@ Python modules. .. [ctypes] https://docs.python.org/library/ctypes.html. .. there's also the original ctypes home page: http://python.net/crew/theller/ctypes/ -.. [Pyrex] G. Ewing, Pyrex: C-Extensions for Python, - http://www.cosc.canterbury.ac.nz/greg.ewing/python/Pyrex/ +.. + [Pyrex] G. Ewing, Pyrex: C-Extensions for Python, + https://www.cosc.canterbury.ac.nz/greg.ewing/python/Pyrex/ .. [ShedSkin] M. Dufour, J. Coughlan, ShedSkin, https://github.com/shedskin/shedskin .. [SWIG] David M. Beazley et al., diff --git a/docs/src/tutorial/strings.rst b/docs/src/tutorial/strings.rst index a4ce2b9d0..43c9ee6a8 100644 --- a/docs/src/tutorial/strings.rst +++ b/docs/src/tutorial/strings.rst @@ -441,7 +441,7 @@ characters and is compatible with plain ASCII encoded text that it encodes efficiently. This makes it a very good choice for source code files which usually consist mostly of ASCII characters. -.. _`UTF-8`: http://en.wikipedia.org/wiki/UTF-8 +.. _`UTF-8`: https://en.wikipedia.org/wiki/UTF-8 As an example, putting the following line into a UTF-8 encoded source file will print ``5``, as UTF-8 encodes the letter ``'ö'`` in the two @@ -554,7 +554,7 @@ above character. For more information on this topic, it is worth reading the `Wikipedia article about the UTF-16 encoding`_. -.. _`Wikipedia article about the UTF-16 encoding`: http://en.wikipedia.org/wiki/UTF-16/UCS-2 +.. _`Wikipedia article about the UTF-16 encoding`: https://en.wikipedia.org/wiki/UTF-16/UCS-2 The same properties apply to Cython code that gets compiled for a narrow CPython runtime environment. In most cases, e.g. when diff --git a/docs/src/userguide/compute_typed_html.jpg b/docs/src/userguide/compute_typed_html.jpg Binary files differindex f8607bdd8..a1e006573 100644 --- a/docs/src/userguide/compute_typed_html.jpg +++ b/docs/src/userguide/compute_typed_html.jpg diff --git a/docs/src/userguide/debugging.rst b/docs/src/userguide/debugging.rst index 06af18bbf..a33ff8dd8 100644 --- a/docs/src/userguide/debugging.rst +++ b/docs/src/userguide/debugging.rst @@ -21,19 +21,20 @@ source, and then running:: make sudo make install +Installing the Cython debugger can be quite tricky. `This installation script and example code <https://gitlab.com/volkerweissmann/cygdb_installation>`_ might be useful. + The debugger will need debug information that the Cython compiler can export. This can be achieved from within the setup script by passing ``gdb_debug=True`` to ``cythonize()``:: - from distutils.core import setup - from distutils.extension import Extension + from setuptools import Extension, setup extensions = [Extension('source', ['source.pyx'])] setup(..., ext_modules=cythonize(extensions, gdb_debug=True)) For development it's often helpful to pass the ``--inplace`` flag to -the ``setup.py`` script, which makes distutils build your project +the ``setup.py`` script, which makes setuptools build your project "in place", i.e., not in a separate `build` directory. When invoking Cython from the command line directly you can have it write diff --git a/docs/src/userguide/extension_types.rst b/docs/src/userguide/extension_types.rst index 63b407871..e7f95a7ab 100644 --- a/docs/src/userguide/extension_types.rst +++ b/docs/src/userguide/extension_types.rst @@ -11,7 +11,7 @@ Introduction As well as creating normal user-defined classes with the Python class statement, Cython also lets you create new built-in Python types, known as -extension types. You define an extension type using the :keyword:`cdef` class +:term:`extension types<Extension type>`. You define an extension type using the :keyword:`cdef` class statement. Here's an example: .. literalinclude:: ../../examples/userguide/extension_types/shrubbery.pyx @@ -327,7 +327,8 @@ when it is deleted.:: Subclassing ============= -An extension type may inherit from a built-in type or another extension type:: +If an extension type inherits from other types, the first base class must be +a built-in type or another extension type:: cdef class Parrot: ... @@ -342,7 +343,9 @@ extern extension type. If the base type is defined in another Cython module, it must either be declared as an extern extension type or imported using the :keyword:`cimport` statement. -An extension type can only have one base class (no multiple inheritance). +Multiple inheritance is supported, however the second and subsequent base +classes must be an ordinary Python class (not an extension type or a built-in +type). Cython extension types can also be subclassed in Python. A Python class can inherit from multiple extension types provided that the usual Python rules for @@ -504,7 +507,7 @@ It is quite common to want to instantiate an extension class from an existing (pointer to a) data structure, often as returned by external C/C++ functions. As extension classes can only accept Python objects as arguments in their -contructors, this necessitates the use of factory functions. For example, :: +constructors, this necessitates the use of factory functions. For example, :: from libc.stdlib cimport malloc, free @@ -611,10 +614,97 @@ object called :attr:`__weakref__`. For example,:: cdef object __weakref__ -Controlling cyclic garbage collection in CPython -================================================ +Controlling deallocation and garbage collection in CPython +========================================================== -By default each extension type will support the cyclic garbage collector of +.. NOTE:: + + This section only applies to the usual CPython implementation + of Python. Other implementations like PyPy work differently. + +.. _dealloc_intro: + +Introduction +------------ + +First of all, it is good to understand that there are two ways to +trigger deallocation of Python objects in CPython: +CPython uses reference counting for all objects and any object with a +reference count of zero is immediately deallocated. This is the most +common way of deallocating an object. For example, consider :: + + >>> x = "foo" + >>> x = "bar" + +After executing the second line, the string ``"foo"`` is no longer referenced, +so it is deallocated. This is done using the ``tp_dealloc`` slot, which can be +customized in Cython by implementing ``__dealloc__``. + +The second mechanism is the cyclic garbage collector. +This is meant to resolve cyclic reference cycles such as :: + + >>> class Object: + ... pass + >>> def make_cycle(): + ... x = Object() + ... y = [x] + ... x.attr = y + +When calling ``make_cycle``, a reference cycle is created since ``x`` +references ``y`` and vice versa. Even though neither ``x`` or ``y`` +are accessible after ``make_cycle`` returns, both have a reference count +of 1, so they are not immediately deallocated. At regular times, the garbage +collector runs, which will notice the reference cycle +(using the ``tp_traverse`` slot) and break it. +Breaking a reference cycle means taking an object in the cycle +and removing all references from it to other Python objects (we call this +*clearing* an object). Clearing is almost the same as deallocating, except +that the actual object is not yet freed. For ``x`` in the example above, +the attributes of ``x`` would be removed from ``x``. + +Note that it suffices to clear just one object in the reference cycle, +since there is no longer a cycle after clearing one object. Once the cycle +is broken, the usual refcount-based deallocation will actually remove the +objects from memory. Clearing is implemented in the ``tp_clear`` slot. +As we just explained, it is sufficient that one object in the cycle +implements ``tp_clear``. + +.. _trashcan: + +Enabling the deallocation trashcan +---------------------------------- + +In CPython, it is possible to create deeply recursive objects. For example:: + + >>> L = None + >>> for i in range(2**20): + ... L = [L] + +Now imagine that we delete the final ``L``. Then ``L`` deallocates +``L[0]``, which deallocates ``L[0][0]`` and so on until we reach a +recursion depth of ``2**20``. This deallocation is done in C and such +a deep recursion will likely overflow the C call stack, crashing Python. + +CPython invented a mechanism for this called the *trashcan*. It limits the +recursion depth of deallocations by delaying some deallocations. + +By default, Cython extension types do not use the trashcan but it can be +enabled by setting the ``trashcan`` directive to ``True``. For example:: + + cimport cython + @cython.trashcan(True) + cdef class Object: + cdef dict __dict__ + +Trashcan usage is inherited by subclasses +(unless explicitly disabled by ``@cython.trashcan(False)``). +Some builtin types like ``list`` use the trashcan, so subclasses of it +use the trashcan by default. + +Disabling cycle breaking (``tp_clear``) +--------------------------------------- + +By default, each extension type will support the cyclic garbage collector of CPython. If any Python objects can be referenced, Cython will automatically generate the ``tp_traverse`` and ``tp_clear`` slots. This is usually what you want. @@ -622,13 +712,13 @@ want. There is at least one reason why this might not be what you want: If you need to cleanup some external resources in the ``__dealloc__`` special function and your object happened to be in a reference cycle, the garbage collector may -have triggered a call to ``tp_clear`` to drop references. This is the way that -reference cycles are broken so that the garbage can actually be reclaimed. +have triggered a call to ``tp_clear`` to clear the object +(see :ref:`dealloc_intro`). -In that case any object references have vanished by the time when -``__dealloc__`` is called. Now your cleanup code lost access to the objects it -has to clean up. In that case you can disable the cycle breaker ``tp_clear`` -by using the ``no_gc_clear`` decorator :: +In that case, any object references have vanished when ``__dealloc__`` +is called. Now your cleanup code lost access to the objects it has to clean up. +To fix this, you can disable clearing instances of a specific class by using +the ``no_gc_clear`` directive:: @cython.no_gc_clear cdef class DBCursor: @@ -641,17 +731,21 @@ by using the ``no_gc_clear`` decorator :: This example tries to close a cursor via a database connection when the Python object is destroyed. The ``DBConnection`` object is kept alive by the reference from ``DBCursor``. But if a cursor happens to be in a reference cycle, the -garbage collector may effectively "steal" the database connection reference, +garbage collector may delete the database connection reference, which makes it impossible to clean up the cursor. -Using the ``no_gc_clear`` decorator this can not happen anymore because the -references of a cursor object will not be cleared anymore. +If you use ``no_gc_clear``, it is important that any given reference cycle +contains at least one object *without* ``no_gc_clear``. Otherwise, the cycle +cannot be broken, which is a memory leak. + +Disabling cyclic garbage collection +----------------------------------- In rare cases, extension types can be guaranteed not to participate in cycles, but the compiler won't be able to prove this. This would be the case if the class can never reference itself, even indirectly. In that case, you can manually disable cycle collection by using the -``no_gc`` decorator, but beware that doing so when in fact the extension type +``no_gc`` directive, but beware that doing so when in fact the extension type can participate in cycles could cause memory leaks :: @cython.no_gc @@ -805,8 +899,7 @@ write ``dtype.itemsize`` in Cython code which will be compiled into direct access of the C struct field, without going through a C-API equivalent of ``dtype.__getattr__('itemsize')``. -For example we may have an extension -module ``foo_extension``:: +For example, we may have an extension module ``foo_extension``:: cdef class Foo: cdef public int field0, field1, field2; @@ -850,6 +943,7 @@ use the C-API equivalent of:: instead of the desired C equivalent of ``return f->f0 + f->f1 + f->f2``. We can alias the fields by using:: + cdef extern from "foo_nominal.h": ctypedef class foo_extension.Foo [object FooStructNominal]: @@ -867,6 +961,19 @@ code. No changes to Python need be made to achieve significant speedups, even though the field names in Python and C are different. Of course, one should make sure the fields are equivalent. +C inline properties +------------------- + +Similar to Python property attributes, Cython provides a way to declare C-level +properties on external extension types. This is often used to shadow Python +attributes through faster C level data access, but can also be used to add certain +functionality to existing types when using them from Cython. The declarations +must use `cdef inline`. + +For example, the above ``complex`` type could also be declared like this: + +.. literalinclude:: ../../examples/userguide/extension_types/c_property.pyx + Implicit importing ------------------ diff --git a/docs/src/userguide/external_C_code.rst b/docs/src/userguide/external_C_code.rst index e6605223d..4b8ddd8a0 100644 --- a/docs/src/userguide/external_C_code.rst +++ b/docs/src/userguide/external_C_code.rst @@ -223,6 +223,38 @@ See also use of :ref:`external_extension_types`. Note that in all the cases below, you refer to the type in Cython code simply as :c:type:`Foo`, not ``struct Foo``. +Pointers +-------- +When interacting with a C-api there may be functions that require pointers as arguments. +Pointers are variables that contain a memory address to another variable. + +For example:: + + cdef extern from "<my_lib.h>": + cdef void increase_by_one(int *my_var) + +This function takes a pointer to an integer as argument. Knowing the address of the +integer allows the function to modify the value in place, so that the caller can see +the changes afterwards. In order to get the address from an existing variable, +use the ``&`` operator:: + + cdef int some_int = 42 + cdef int *some_int_pointer = &some_int + increase_by_one(some_int_pointer) + # Or without creating the extra variable + increase_by_one(&some_int) + print(some_int) # prints 44 (== 42+1+1) + +If you want to manipulate the variable the pointer points to, you can access it by +referencing its first element like you would in python ``my_pointer[0]``. For example:: + + cdef void increase_by_one(int *my_var): + my_var[0] += 1 + +For a deeper introduction to pointers, you can read `this tutorial at tutorialspoint +<https://www.tutorialspoint.com/cprogramming/c_pointers.htm>`_. For differences between +Cython and C syntax for manipulating pointers, see :ref:`statements_and_expressions`. + Accessing Python/C API routines --------------------------------- @@ -335,7 +367,7 @@ Including verbatim C code For advanced use cases, Cython allows you to directly write C code as "docstring" of a ``cdef extern from`` block: -.. literalinclude:: ../../examples/userguide/external_C_code/c_code_docstring.pyx +.. literalinclude:: ../../examples/userguide/external_C_code/verbatim_c_code.pyx The above is essentially equivalent to having the C code in a file ``header.h`` and writing :: @@ -344,6 +376,11 @@ The above is essentially equivalent to having the C code in a file long square(long x) void assign(long& x, long y) +This feature is commonly used for platform specific adaptations at +compile time, for example: + +.. literalinclude:: ../../examples/userguide/external_C_code/platform_adaptation.pyx + It is also possible to combine a header file and verbatim C code:: cdef extern from "badheader.h": @@ -524,7 +561,7 @@ Acquiring and Releasing the GIL --------------------------------- Cython provides facilities for acquiring and releasing the -`Global Interpreter Lock (GIL) <http://docs.python.org/dev/glossary.html#term-global-interpreter-lock>`_. +`Global Interpreter Lock (GIL) <https://docs.python.org/dev/glossary.html#term-global-interpreter-lock>`_. This may be useful when calling from multi-threaded code into (external C) code that may block, or when wanting to use Python from a (native) C thread callback. Releasing the GIL should @@ -549,14 +586,18 @@ You can release the GIL around a section of code using the with nogil: <code to be executed with the GIL released> -Code in the body of the with-statement must not raise exceptions or -manipulate Python objects in any way, and must not call anything that -manipulates Python objects without first re-acquiring the GIL. Cython -validates these operations at compile time, but cannot look into -external C functions, for example. They must be correctly declared -as requiring or not requiring the GIL (see below) in order to make +Code in the body of the with-statement must not manipulate Python objects +in any way, and must not call anything that manipulates Python objects without +first re-acquiring the GIL. Cython validates these operations at compile time, +but cannot look into external C functions, for example. They must be correctly +declared as requiring or not requiring the GIL (see below) in order to make Cython's checks effective. +Since Cython 3.0, some simple Python statements can be used inside of ``nogil`` +sections: ``raise``, ``assert`` and ``print`` (the Py2 statement, not the function). +Since they tend to be lone Python statements, Cython will automatically acquire +and release the GIL around them for convenience. + .. _gil: Acquiring the GIL @@ -580,6 +621,26 @@ The GIL may also be acquired through the ``with gil`` statement:: with gil: <execute this block with the GIL acquired> +.. _gil_conditional: + +Conditional Acquiring / Releasing the GIL +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Sometimes it is helpful to use a condition to decide whether to run a +certain piece of code with or without the GIL. This code would run anyway, +the difference is whether the GIL will be held or released. +The condition must be constant (at compile time). + +This could be useful for profiling, debugging, performance testing, and +for fused types (see :ref:`fused_gil_conditional`).:: + + DEF FREE_GIL = True + + with nogil(FREE_GIL): + <code to be executed with the GIL released> + + with gil(False): + <GIL is still released> + Declaring a function as callable without the GIL -------------------------------------------------- diff --git a/docs/src/userguide/fusedtypes.rst b/docs/src/userguide/fusedtypes.rst index b50bb0efd..b108b944e 100644 --- a/docs/src/userguide/fusedtypes.rst +++ b/docs/src/userguide/fusedtypes.rst @@ -249,6 +249,34 @@ to figure out whether a specialization is part of another set of types if bunch_of_types in string_t: print("s is a string!") +.. _fused_gil_conditional: + +Conditional GIL Acquiring / Releasing +===================================== + +Acquiring and releasing the GIL can be controlled by a condition +which is known at compile time (see :ref:`gil_conditional`). + +This is most useful when combined with fused types. +A fused type function may have to handle both cython native types +(e.g. cython.int or cython.double) and python types (e.g. object or bytes). +Conditional Acquiring / Releasing the GIL provides a method for running +the same piece of code either with the GIL released (for cython native types) +and with the GIL held (for python types).:: + + cimport cython + + ctypedef fused double_or_object: + cython.double + object + + def increment(double_or_object x): + with nogil(double_or_object is cython.double): + # Same code handles both cython.double (GIL is released) + # and python object (GIL is not released). + x = x + 1 + return x + __signatures__ ============== diff --git a/docs/src/userguide/glossary.rst b/docs/src/userguide/glossary.rst new file mode 100644 index 000000000..1ccbb4408 --- /dev/null +++ b/docs/src/userguide/glossary.rst @@ -0,0 +1,46 @@ +.. glossary:: + +Glossary +======== + + Extension type + "Extension type" can refer to either a Cython class defined with ``cdef class`` or more generally to any Python type that is ultimately implemented as a native C struct (including the built-in types like `int` or `dict`). + + Dynamic allocation or Heap allocation + A C variable allocated with ``malloc`` (in C) or ``new`` (in C++) is + `allocated dynamically/heap allocated <https://en.wikipedia.org/wiki/C_dynamic_memory_allocation>`_. + Its lifetime is until the user deletes it explicitly (with ``free`` in C or ``del`` in C++). + This can happen in a different function than the allocation. + + pointer + A **pointer** is a variable that stores the address of another variable + (i.e. direct address of the memory location). They allow for + dynamic memory allocation and deallocation. They can be used to build + dynamic data structures. `Read more <https://en.wikipedia.org/wiki/Pointer_(computer_programming)#C_pointers>`__. + + Python object + When using Python, the contents of every variable is a Python object + (including Cython extension types). Key features of Python objects are that + they are passed *by reference* and that their lifetime is *managed* automatically + so that they are destroyed when no more references exist to them. + In Cython, they are distinct from C types, which are passed *by value* and whose + lifetime is managed depending on whether they are allocated on the stack or heap. + To explicitly declare a Python object variable in Cython use ``cdef object abc``. + Internally in C, they are referred to as ``PyObject*``. + + Stack allocation + A C variable declared within a function as ``cdef SomeType a`` + is said to be allocated on the stack. + It exists for the duration of the function only. + + Typed memoryview + A useful Cython type for getting quick access to blocks of memory. + A memoryview alone does not actually own any memory. + However, it can be initialized with a Python object that supports the + `buffer protocol`_ (typically "array" types, for example a Numpy array). + The memoryview keeps a reference to that Python object alive + and provides quick access to the memory without needing to go + through the Python API of the object and its ``__getitem__``/``__setitem__`` methods. + For more information, see :ref:`memoryviews`. + +.. _buffer protocol: https://docs.python.org/3/c-api/buffer.html diff --git a/docs/src/userguide/index.rst b/docs/src/userguide/index.rst index cfbee2fbd..9dc3a75a3 100644 --- a/docs/src/userguide/index.rst +++ b/docs/src/userguide/index.rst @@ -16,6 +16,7 @@ Contents: wrapping_CPlusPlus fusedtypes pypy + migrating_to_cy30 limitations pyrex_differences memoryviews @@ -34,3 +35,4 @@ Indices and tables .. toctree:: + glossary diff --git a/docs/src/userguide/language_basics.rst b/docs/src/userguide/language_basics.rst index 0fdd87783..402cd14ba 100644 --- a/docs/src/userguide/language_basics.rst +++ b/docs/src/userguide/language_basics.rst @@ -139,10 +139,11 @@ For example:: cdef list foo = [] -This requires an *exact* match of the class, it does not allow -subclasses. This allows Cython to optimize code by accessing -internals of the builtin class. -For this kind of typing, Cython uses internally a C variable of type ``PyObject*``. +This requires an *exact* match of the class, it does not allow subclasses. +This allows Cython to optimize code by accessing internals of the builtin class, +which is the main reason for declaring builtin types in the first place. + +For declared builtin types, Cython uses internally a C variable of type ``PyObject*``. The Python types int, long, and float are not available for static typing and instead interpreted as C ``int``, ``long``, and ``float`` respectively, as statically typing variables with these Python @@ -189,7 +190,7 @@ Python functions vs. C functions There are two kinds of function definition in Cython: Python functions are defined using the def statement, as in Python. They take -Python objects as parameters and return Python objects. +:term:`Python objects<Python object>` as parameters and return Python objects. C functions are defined using the new :keyword:`cdef` statement. They take either Python objects or C values as parameters, and can return either Python @@ -288,6 +289,17 @@ In the interests of clarity, it is probably a good idea to always be explicit about object parameters in C functions. +To create a borrowed reference, specify the parameter type as ``PyObject*``. +Cython won't perform automatic ``Py_INCREF``, or ``Py_DECREF``, e.g.: + +.. literalinclude:: ../../examples/userguide/language_basics/parameter_refcount.pyx + +will display:: + + Initial refcount: 2 + Inside owned_reference: 3 + Inside borrowed_reference: 2 + .. _optional_arguments: Optional Arguments @@ -594,6 +606,10 @@ Here is an example: The precedence of ``<...>`` is such that ``<type>a.b.c`` is interpreted as ``<type>(a.b.c)``. +Casting to ``<object>`` creates an owned reference. Cython will automatically +perform a ``Py_INCREF`` and ``Py_DECREF`` operation. Casting to +``<PyObject *>`` creates a borrowed reference, leaving the refcount unchanged. + .. _checked_type_casts: Checked Type Casts diff --git a/docs/src/userguide/limitations.rst b/docs/src/userguide/limitations.rst index 6128c308a..670f01d03 100644 --- a/docs/src/userguide/limitations.rst +++ b/docs/src/userguide/limitations.rst @@ -8,9 +8,12 @@ Limitations This page used to list bugs in Cython that made the semantics of compiled code differ from that in Python. Most of the missing -features have been fixed in Cython 0.15. Note that a -future version 1.0 of Cython is planned to provide full Python -language compatibility. +features have been fixed in Cython 0.15. A future version of +Cython is planned to provide full Python language compatibility. +For now, the issue tracker can provide an overview of deviations +that we are aware of and would like to see fixed. + +https://github.com/cython/cython/labels/Python%20Semantics Below is a list of differences that we will probably not be addressing. Most of these things that fall more into the implementation details rather diff --git a/docs/src/userguide/memoryviews.rst b/docs/src/userguide/memoryviews.rst index 328831e86..ce83a3005 100644 --- a/docs/src/userguide/memoryviews.rst +++ b/docs/src/userguide/memoryviews.rst @@ -20,6 +20,9 @@ A memoryview can be used in any context (function parameters, module-level, cdef class attribute, etc) and can be obtained from nearly any object that exposes writable buffer through the `PEP 3118`_ buffer interface. +.. _`PEP 3118`: https://www.python.org/dev/peps/pep-3118/ + + .. _view_quickstart: Quickstart @@ -56,16 +59,11 @@ A complete 3D view:: cdef int[:,:,:] view3D = exporting_object -A 2D view that restricts the first dimension of a buffer to 100 rows -starting at the second (index 1) and then skips every second (odd) row:: - - cdef int[1:102:2,:] partial_view = exporting_object - -This also works conveniently as function arguments: +They also work conveniently as function arguments: .. code-block:: cython - def process_3d_buffer(int[1:102:2,:] view not None): + def process_3d_buffer(int[:,:,:] view not None): ... The ``not None`` declaration for the argument automatically rejects @@ -234,6 +232,8 @@ NumPy arrays support this interface, as do :ref:`view_cython_arrays`. The data array to themselves be pointers; Cython memoryviews do not yet support this. +.. _`new style buffers`: https://docs.python.org/3/c-api/buffer.html + .. _view_memory_layout: Memory layout @@ -660,6 +660,6 @@ object handling. For the details of how to compile and call functions in C files, see :ref:`using_c_libraries`. -.. _GIL: http://docs.python.org/dev/glossary.html#term-global-interpreter-lock +.. _GIL: https://docs.python.org/dev/glossary.html#term-global-interpreter-lock .. _NumPy: https://docs.scipy.org/doc/numpy/reference/arrays.ndarray.html#memory-layout .. _example: https://docs.scipy.org/doc/numpy/reference/arrays.indexing.html diff --git a/docs/src/userguide/migrating_to_cy30.rst b/docs/src/userguide/migrating_to_cy30.rst new file mode 100644 index 000000000..1dd29a9a2 --- /dev/null +++ b/docs/src/userguide/migrating_to_cy30.rst @@ -0,0 +1,159 @@ +.. highlight:: cython + +.. _cython30: + +********************************* +Migrating from Cython 0.29 to 3.0 +********************************* + +Cython 3.0 is a major revision of the compiler and the language +that comes with some backwards incompatible changes. +This document lists the important ones and explains how to deal with +them in existing code. + + +Python 3 syntax/semantics +========================= + +Cython 3.0 now uses Python 3 syntax and semantics by default, which previously +required setting the ``language_level`` `directive <compiler-directives>` to +either ``3`` or ``3str``. +The new default setting is now ``language_level=3str``, which means Python 3 +semantics, but unprefixed strings are ``str`` objects, i.e. unicode text strings +under Python 3 and byte strings under Python 2.7. + +You can revert your code to the previous (Python 2.x) semantics by setting +``language_level=2``. + +Further semantic changes due to the language level include: + +* ``/``-division uses the true (float) division operator, unless ``cdivision`` is enabled. +* ``print`` is a function, not a statement. +* Python classes that are defined without bases (``class C: ...``) are "new-style" + classes also in Py2.x (if you never heard about "old-style classes", you're probably + happy without them). +* Annotations (type hints) are now stored as strings. + (`PEP 563 <https://github.com/cython/cython/issues/2863>`_) +* ``StopIteration`` handling in generators has been changed according to + `PEP 479 <https://www.python.org/dev/peps/pep-0479/>`_. + + +Python semantics +================ + +Some Python compatibility bugs were fixed, e.g. + +* Subscripting (``x[1]``) now tries the mapping protocol before the sequence protocol. + (https://github.com/cython/cython/issues/1807) +* Exponentiation of integer literals now follows Python semantics and not C semantics. + (https://github.com/cython/cython/issues/2133) + + +Binding functions +================= + +The :ref:`binding directive <compiler-directives>` is now enabled by default. +This makes Cython compiled Python (``def``) functions mostly compatible +with normal (non-compiled) Python functions, regarding signature introspection, +annotations, etc. + +It also makes them bind as methods in Python classes on attribute assignments, +thus the name. +If this is not intended, i.e. if a function is really meant to be a function +and never a method, you can disable the binding (and all other Python function +features) by setting ``binding=False`` or selectively adding a decorator +``@cython.binding(False)``. +In pure Python mode, the decorator was not available in Cython 0.29.16 yet, +but compiled code does not suffer from this. + +We recommend, however, to keep the new function features and instead deal +with the binding issue using the standard Python ``staticmethod()`` builtin. + +:: + + def func(self, b): ... + + class MyClass(object): + binding_method = func + + no_method = staticmethod(func) + + +Namespace packages +================== + +Cython now has support for loading pxd files also from namespace packages +according to `PEP-420 <https://www.python.org/dev/peps/pep-0420/>`_. +This might have an impact on the import path. + + +NumPy C-API +=========== + +Cython used to generate code that depended on the deprecated pre-NumPy-1.7 C-API. +This is no longer the case with Cython 3.0. + +You can now define the macro ``NPY_NO_DEPRECATED_API=NPY_1_7_API_VERSION`` +to get rid of the long-standing build warnings that the compiled C module +uses a deprecated API. Either per file:: + + # distutils: define_macros=NPY_NO_DEPRECATED_API=NPY_1_7_API_VERSION + +or by setting it in your Extensions in ``setup.py``:: + + Extension(... + define_macros=[("NPY_NO_DEPRECATED_API", "NPY_1_7_API_VERSION")] + ) + +One side-effect of the different C-API usage is that your code may now +require a call to the `NumPy C-API initialisation function +<https://docs.scipy.org/doc/numpy-1.17.0/reference/c-api.array.html#importing-the-api>`_ +where it previously got away without doing so. + +In order to reduce the user impact here, Cython 3.0 will now call it +automatically when it sees ``numpy`` being cimported, but the function +not being used. +In the (hopefully rare) cases where this gets in the way, the internal +C-API initialisation can be disabled by faking the use of the function +without actually calling it, e.g. + +:: + + # Explicitly disable the automatic initialisation of NumPy's C-API. + <void>import_array + +Class-private name mangling +=========================== + +Cython has been updated to follow the `Python rules for class-private names +<https://docs.python.org/3/tutorial/classes.html#private-variables>`_ +more closely. Essentially any name that starts with and doesn't end with +``__`` within a class is mangled with the class name. Most user code +should be unaffected -- unlike in Python unmangled global names will +still be matched to ensure it is possible to access C names +beginning with ``__``:: + + cdef extern void __foo() + + class C: # or "cdef class" + def call_foo(self): + return __foo() # still calls the global name + +What will no-longer work is overriding methods starting with ``__`` in +a ``cdef class``:: + + cdef class Base: + cdef __bar(self): + return 1 + + def call_bar(self): + return self.__bar() + + cdef class Derived(Base): + cdef __bar(self): + return 2 + +Here ``Base.__bar`` is mangled to ``_Base__bar`` and ``Derived.__bar`` +to ``_Derived__bar``. Therefore ``call_bar`` will always call +``_Base__bar``. This matches established Python behaviour and applies +for ``def``, ``cdef`` and ``cpdef`` methods and attributes. diff --git a/docs/src/userguide/numpy_pythran.rst b/docs/src/userguide/numpy_pythran.rst index 185f7c654..cb075d729 100644 --- a/docs/src/userguide/numpy_pythran.rst +++ b/docs/src/userguide/numpy_pythran.rst @@ -16,22 +16,22 @@ This can lead to really interesting speedup in some cases, going from 2 up to Please note that this feature is experimental. -Usage example with distutils ----------------------------- +Usage example with setuptools +----------------------------- You first need to install Pythran. See its `documentation -<http://pythran.readthedocs.io/en/latest/>`_ for more information. +<https://pythran.readthedocs.io/>`_ for more information. Then, simply add a ``cython: np_pythran=True`` directive at the top of the Python files that needs to be compiled using Pythran numpy support. -Here is an example of a simple ``setup.py`` file using distutils: +Here is an example of a simple ``setup.py`` file using setuptools: .. code:: - from distutils.core import setup + from setuptools import setup from Cython.Build import cythonize - + setup( name = "My hello app", ext_modules = cythonize('hello_pythran.pyx') diff --git a/docs/src/userguide/numpy_tutorial.rst b/docs/src/userguide/numpy_tutorial.rst index 3d1cd5a74..f34b2a0da 100644 --- a/docs/src/userguide/numpy_tutorial.rst +++ b/docs/src/userguide/numpy_tutorial.rst @@ -61,11 +61,11 @@ Using Cython consists of these steps: However there are several options to automate these steps: -1. The `SAGE <http://sagemath.org>`_ mathematics software system provides +1. The `SAGE <https://sagemath.org>`_ mathematics software system provides excellent support for using Cython and NumPy from an interactive command line or through a notebook interface (like Maple/Mathematica). See `this documentation - <http://doc.sagemath.org/html/en/developer/coding_in_cython.html>`_. + <https://doc.sagemath.org/html/en/developer/coding_in_cython.html>`_. 2. Cython can be used as an extension within a Jupyter notebook, making it easy to compile and use Cython code with just a ``%%cython`` at the top of a cell. For more information see @@ -73,7 +73,7 @@ However there are several options to automate these steps: 3. A version of pyximport is shipped with Cython, so that you can import pyx-files dynamically into Python and have them compiled automatically (See :ref:`pyximport`). -4. Cython supports distutils so that you can very easily create build scripts +4. Cython supports setuptools so that you can very easily create build scripts which automate the process, this is the preferred method for Cython implemented libraries and packages. See :ref:`Basic setup.py <basic_setup.py>`. @@ -487,8 +487,8 @@ than NumPy! Where to go from here? ====================== -* If you want to learn how to make use of `BLAS <http://www.netlib.org/blas/>`_ - or `LAPACK <http://www.netlib.org/lapack/>`_ with Cython, you can watch +* If you want to learn how to make use of `BLAS <https://www.netlib.org/blas/>`_ + or `LAPACK <https://www.netlib.org/lapack/>`_ with Cython, you can watch `the presentation of Ian Henriksen at SciPy 2015 <https://www.youtube.com/watch?v=R4yB-8tB0J0&t=693s&ab_channel=Enthought>`_. * If you want to learn how to use Pythran as backend in Cython, you diff --git a/docs/src/userguide/parallelism.rst b/docs/src/userguide/parallelism.rst index ad97885e1..e9d473e66 100644 --- a/docs/src/userguide/parallelism.rst +++ b/docs/src/userguide/parallelism.rst @@ -118,7 +118,7 @@ Example with a reduction: .. literalinclude:: ../../examples/userguide/parallelism/simple_sum.pyx -Example with a typed memoryview (e.g. a NumPy array):: +Example with a :term:`typed memoryview<Typed memoryview>` (e.g. a NumPy array):: from cython.parallel import prange diff --git a/docs/src/userguide/pypy.rst b/docs/src/userguide/pypy.rst index cf1fbb24d..893277bda 100644 --- a/docs/src/userguide/pypy.rst +++ b/docs/src/userguide/pypy.rst @@ -2,7 +2,7 @@ Porting Cython code to PyPy =========================== Cython has basic support for cpyext, the layer in -`PyPy <http://pypy.org/>`_ that emulates CPython's C-API. This is +`PyPy <https://pypy.org/>`_ that emulates CPython's C-API. This is achieved by making the generated C code adapt at C compile time, so the generated code will compile in both CPython and PyPy unchanged. diff --git a/docs/src/userguide/pyrex_differences.rst b/docs/src/userguide/pyrex_differences.rst index 8a958ec9a..aa2eb75da 100644 --- a/docs/src/userguide/pyrex_differences.rst +++ b/docs/src/userguide/pyrex_differences.rst @@ -310,7 +310,7 @@ Automatic ``typecheck`` Rather than introducing a new keyword ``typecheck`` as explained in the `Pyrex docs -<http://www.cosc.canterbury.ac.nz/greg.ewing/python/Pyrex/version/Doc/Manual/special_methods.html>`_, +<https://www.cosc.canterbury.ac.nz/greg.ewing/python/Pyrex/version/Doc/Manual/special_methods.html>`_, Cython emits a (non-spoofable and faster) typecheck whenever :func:`isinstance` is used with an extension type as the second parameter. diff --git a/docs/src/userguide/sharing_declarations.rst b/docs/src/userguide/sharing_declarations.rst index 57f41e38d..70e29e2b2 100644 --- a/docs/src/userguide/sharing_declarations.rst +++ b/docs/src/userguide/sharing_declarations.rst @@ -226,3 +226,24 @@ Some things to note about this example: ``egg`` file which will not work with ``cimport`` for ``pxd`` files when you try to use them from a dependent package. To prevent this, include ``zip_safe=False`` in the arguments to ``setup()``. + +.. _versioning: + +Versioning +========== + +``.pxd`` files can be labelled with a minimum Cython version as part of +their file name, similar to the version tagging of ``.so`` files in PEP 3149. +For example a file called :file:`Shrubbing.cython-30.pxd` will only be +found by ``cimport Shrubbing`` on Cython 3.0 and higher. Cython will use the +file tagged with the highest compatible version number. + +Note that versioned files that are distributed across different directories +will not be found. Only the first directory in the Python module search +path in which a matching ``.pxd`` file is found will be considered. + +The purpose of this feature is to allow third-party packages to release +Cython interfaces to their packages that take advantage of the latest Cython +features while not breaking compatibility for users with older versions of Cython. +Users intending to use ``.pxd`` files solely within their own project +need not produce these tagged files. diff --git a/docs/src/userguide/source_files_and_compilation.rst b/docs/src/userguide/source_files_and_compilation.rst index c4f01806d..bc3b08677 100644 --- a/docs/src/userguide/source_files_and_compilation.rst +++ b/docs/src/userguide/source_files_and_compilation.rst @@ -53,7 +53,7 @@ compiled with the C compiler using whatever options are appropriate on your platform for generating an extension module. For these options look at the official Python documentation. -The other, and probably better, way is to use the :mod:`distutils` extension +The other, and probably better, way is to use the :mod:`setuptools` extension provided with Cython. The benefit of this method is that it will give the platform specific compilation options, acting like a stripped down autotools. @@ -93,7 +93,7 @@ to libraries you want to link with.) After compilation, a ``yourmod.so`` (:file:`yourmod.pyd` for Windows) file is written into the target directory and your module, ``yourmod``, is available for you to import as with any other -Python module. Note that if you are not relying on ``cythonize`` or distutils, +Python module. Note that if you are not relying on ``cythonize`` or setuptools, you will not automatically benefit from the platform specific file extension that CPython generates for disambiguation, such as ``yourmod.cpython-35m-x86_64-linux-gnu.so`` on a regular 64bit Linux installation @@ -103,23 +103,31 @@ of CPython 3.5. Basic setup.py =============== -The distutils extension provided with Cython allows you to pass ``.pyx`` files +The setuptools extension provided with Cython allows you to pass ``.pyx`` files directly to the ``Extension`` constructor in your setup file. If you have a single Cython file that you want to turn into a compiled extension, say with filename :file:`example.pyx` the associated :file:`setup.py` would be:: - from distutils.core import setup + from setuptools import setup from Cython.Build import cythonize setup( ext_modules = cythonize("example.pyx") ) -To understand the :file:`setup.py` more fully look at the official -:mod:`distutils` documentation. To compile the extension for use in the -current directory use: +If your build depends directly on Cython in this way, +then you may also want to inform pip that :mod:`Cython` is required for +:file:`setup.py` to execute, following `PEP 518 +<https://www.python.org/dev/peps/pep-0518/>`, creating a :file:`pyproject.toml` +file containing, at least:: + + [build-system] + requires = ["setuptools", "wheel", "Cython"] + +To understand the :file:`setup.py` more fully look at the official `setuptools +documentation`_. To compile the extension for use in the current directory use: .. sourcecode:: text @@ -131,7 +139,7 @@ Configuring the C-Build If you have include files in non-standard places you can pass an ``include_path`` parameter to ``cythonize``:: - from distutils.core import setup + from setuptools import setup from Cython.Build import cythonize setup( @@ -150,20 +158,38 @@ the necessary include files, e.g. for NumPy:: you have to add the path to NumPy include files. You need to add this path only if you use ``cimport numpy``. -Despite this, you will still get warnings like the -following from the compiler, because Cython is using a deprecated Numpy API:: +Despite this, you may still get warnings like the following from the compiler, +because Cython is not disabling the usage of the old deprecated Numpy API:: .../include/numpy/npy_1_7_deprecated_api.h:15:2: warning: #warning "Using deprecated NumPy API, disable it by " "#defining NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION" [-Wcpp] -For the time being, it is just a warning that you can ignore. +In Cython 3.0, you can get rid of this warning by defining the C macro +``NPY_NO_DEPRECATED_API`` as ``NPY_1_7_API_VERSION`` +in your build, e.g.:: + + # distutils: define_macros=NPY_NO_DEPRECATED_API=NPY_1_7_API_VERSION + +or (see below):: + + Extension( + ..., + define_macros=[("NPY_NO_DEPRECATED_API", "NPY_1_7_API_VERSION")], + ) + +With older Cython releases, setting this macro will fail the C compilation, +because Cython generates code that uses this deprecated C-API. However, the +warning has no negative effects even in recent NumPy versions including 1.18.x. +You can ignore it until you (or your library's users) switch to a newer NumPy +version that removes this long deprecated API, in which case you also need to +use Cython 3.0 or later. Thus, the earlier you switch to Cython 3.0, the +better for your users. If you need to specify compiler options, libraries to link with or other linker options you will need to create ``Extension`` instances manually (note that glob syntax can still be used to specify multiple extensions in one line):: - from distutils.core import setup - from distutils.extension import Extension + from setuptools import Extension, setup from Cython.Build import cythonize extensions = [ @@ -182,11 +208,10 @@ in one line):: ext_modules=cythonize(extensions), ) -Note that when using setuptools, you should import it before Cython as -setuptools may replace the ``Extension`` class in distutils. Otherwise, +Note that when using setuptools, you should import it before Cython, otherwise, both might disagree about the class to use here. -Note also that if you use setuptools instead of distutils, the default +Note also that if you use setuptools instead of :mod:`distutils`, the default action when running ``python setup.py install`` is to create a zipped ``egg`` file which will not work with ``cimport`` for ``pxd`` files when you try to use them from a dependent package. @@ -204,7 +229,7 @@ merges the list of libraries, so this works as expected (similarly with other options, like ``include_dirs`` above). If you have some C files that have been wrapped with Cython and you want to -compile them into your extension, you can define the distutils ``sources`` +compile them into your extension, you can define the setuptools ``sources`` parameter:: # distutils: sources = helper.c, another_helper.c @@ -213,9 +238,8 @@ Note that these sources are added to the list of sources of the current extension module. Spelling this out in the :file:`setup.py` file looks as follows:: - from distutils.core import setup + from setuptools import Extension, setup from Cython.Build import cythonize - from distutils.extension import Extension sourcefiles = ['example.pyx', 'helper.c', 'another_helper.c'] @@ -226,14 +250,14 @@ as follows:: ) The :class:`Extension` class takes many options, and a fuller explanation can -be found in the `distutils documentation`_. Some useful options to know about +be found in the `setuptools documentation`_. Some useful options to know about are ``include_dirs``, ``libraries``, and ``library_dirs`` which specify where to find the ``.h`` and library files when linking to external libraries. -.. _distutils documentation: https://docs.python.org/extending/building.html +.. _setuptools documentation: https://setuptools.readthedocs.io/ Sometimes this is not enough and you need finer customization of the -distutils :class:`Extension`. +setuptools :class:`Extension`. To do this, you can provide a custom function ``create_extension`` to create the final :class:`Extension` object after Cython has processed the sources, dependencies and ``# distutils`` directives but before the @@ -329,11 +353,10 @@ doesn't want to use it just to install your module. Also, the installed version may not be the same one you used, and may not compile your sources correctly. This simply means that the :file:`setup.py` file that you ship with will just -be a normal distutils file on the generated `.c` files, for the basic example +be a normal setuptools file on the generated `.c` files, for the basic example we would have instead:: - from distutils.core import setup - from distutils.extension import Extension + from setuptools import Extension, setup setup( ext_modules = [Extension("example", ["example.c"])] @@ -342,8 +365,7 @@ we would have instead:: This is easy to combine with :func:`cythonize` by changing the file extension of the extension module sources:: - from distutils.core import setup - from distutils.extension import Extension + from setuptools import Extension, setup USE_CYTHON = ... # command line option, try-import, ... @@ -385,14 +407,17 @@ Another option is to make Cython a setup dependency of your system and use Cython's build_ext module which runs ``cythonize`` as part of the build process:: setup( - setup_requires=[ - 'cython>=0.x', - ], extensions = [Extension("*", ["*.pyx"])], - cmdclass={'build_ext': Cython.Build.build_ext}, + cmdclass={'build_ext': Cython.Build.new_build_ext}, ... ) +This depends on pip knowing that :mod:`Cython` is a setup dependency, by having +a :file:`pyproject.toml` file:: + + [build-system] + requires = ["setuptools", "wheel", "Cython"] + If you want to expose the C-level interface of your library for other libraries to cimport from, use package_data to install the ``.pxd`` files, e.g.:: @@ -522,7 +547,7 @@ glob must be on a separate line. Pyximport will check the file date for each of those files before deciding whether to rebuild the module. In order to keep track of the fact that the dependency has been handled, Pyximport updates the modification time of your ".pyx" source file. Future versions may do -something more sophisticated like informing distutils of the dependencies +something more sophisticated like informing setuptools of the dependencies directly. @@ -538,7 +563,7 @@ compiled. Usually the defaults are fine. You might run into problems if you wanted to write your program in half-C, half-Cython and build them into a single library. -Pyximport does not hide the Distutils/GCC warnings and errors generated +Pyximport does not hide the setuptools/GCC warnings and errors generated by the import process. Arguably this will give you better feedback if something went wrong and why. And if nothing went wrong it will give you the warm fuzzy feeling that pyximport really did rebuild your module as it @@ -582,7 +607,7 @@ The Sage notebook allows transparently editing and compiling Cython code simply by typing ``%cython`` at the top of a cell and evaluate it. Variables and functions defined in a Cython cell are imported into the running session. Please check `Sage documentation -<http://www.sagemath.org/doc/>`_ for details. +<https://www.sagemath.org/doc/>`_ for details. You can tailor the behavior of the Cython compiler by specifying the directives below. @@ -623,6 +648,8 @@ You can see them also by typing ```%%cython?`` in IPython or a Jupyter notebook. -a, --annotate Produce a colorized HTML version of the source. +--annotate-fullc Produce a colorized HTML version of the source which includes entire generated C/C++-code. + -+, --cplus Output a C++ rather than C file. -f, --force Force the compilation of a new module, even if the source has been previously compiled. @@ -648,6 +675,7 @@ You can see them also by typing ```%%cython?`` in IPython or a Jupyter notebook. --pgo Enable profile guided optimisation in the C compiler. Compiles the cell twice and executes it in between to generate a runtime profile. --verbose Print debug information like generated .c/.cpp file location and exact gcc/g++ command invoked. + ============================================ ======================================================================================================================================= @@ -659,7 +687,7 @@ Compiler options Compiler options can be set in the :file:`setup.py`, before calling :func:`cythonize`, like this:: - from distutils.core import setup + from setuptools import setup from Cython.Build import cythonize from Cython.Compiler import Options @@ -675,7 +703,6 @@ Here are the options that are available: .. autodata:: Cython.Compiler.Options.docstrings .. autodata:: Cython.Compiler.Options.embed_pos_in_docstring -.. autodata:: Cython.Compiler.Options.emit_code_comments .. pre_import .. autodata:: Cython.Compiler.Options.generate_cleanup_code .. autodata:: Cython.Compiler.Options.clear_to_none @@ -711,7 +738,11 @@ Cython code. Here is the list of currently supported directives: class attribute (hence the name) and will emulate the attributes of Python functions, including introspections like argument names and annotations. - Default is False. + + Default is True. + + .. versionchanged:: 3.0.0 + Default changed from False to True ``boundscheck`` (True / False) If set to False, Cython is free to assume that indexing operations @@ -788,12 +819,19 @@ Cython code. Here is the list of currently supported directives: False. ``always_allow_keywords`` (True / False) - Avoid the ``METH_NOARGS`` and ``METH_O`` when constructing - functions/methods which take zero or one arguments. Has no effect - on special methods and functions with more than one argument. The - ``METH_NOARGS`` and ``METH_O`` signatures provide faster + When disabled, uses the ``METH_NOARGS`` and ``METH_O`` signatures when + constructing functions/methods which take zero or one arguments. Has no + effect on special methods and functions with more than one argument. The + ``METH_NOARGS`` and ``METH_O`` signatures provide slightly faster calling conventions but disallow the use of keywords. +``c_api_binop_methods`` (True / False) + When enabled, makes the special binary operator methods (``__add__``, etc.) + behave according to the low-level C-API slot semantics, i.e. only a single + method implements both the normal and reversed operator. This used to be + the default in Cython 0.x and was now replaced by Python semantics, i.e. the + default in Cython 3.x and later is ``False``. + ``profile`` (True / False) Write hooks for Python profilers into the compiled C code. Default is False. @@ -803,7 +841,7 @@ Cython code. Here is the list of currently supported directives: into the compiled C code. This also enables profiling. Default is False. Note that the generated module will not actually use line tracing, unless you additionally pass the C macro definition - ``CYTHON_TRACE=1`` to the C compiler (e.g. using the distutils option + ``CYTHON_TRACE=1`` to the C compiler (e.g. using the setuptools option ``define_macros``). Define ``CYTHON_TRACE_NOGIL=1`` to also include ``nogil`` functions and sections. @@ -856,7 +894,17 @@ Cython code. Here is the list of currently supported directives: asyncio before Python 3.5. This directive can be applied in modules or selectively as decorator on an async-def coroutine to make the affected coroutine(s) iterable and thus directly interoperable with yield-from. - + +``annotation_typing`` (True / False) + Uses function argument annotations to determine the type of variables. Default + is True, but can be disabled. Since Python does not enforce types given in + annotations, setting to False gives greater compatibility with Python code. + Must be set globally. + +``emit_code_comments`` (True / False) + Copy the original source code line by line into C code comments in the generated + code file to help with understanding the output. + This is also required for coverage analysis. .. _configurable_optimisations: @@ -964,7 +1012,7 @@ In :file:`setup.py` Compiler directives can also be set in the :file:`setup.py` file by passing a keyword argument to ``cythonize``:: - from distutils.core import setup + from setuptools import setup from Cython.Build import cythonize setup( diff --git a/docs/src/userguide/special_methods.rst b/docs/src/userguide/special_methods.rst index 8bb4d9be4..9b59fe738 100644 --- a/docs/src/userguide/special_methods.rst +++ b/docs/src/userguide/special_methods.rst @@ -43,9 +43,11 @@ There are two methods concerned with initialising the object. The :meth:`__cinit__` method is where you should perform basic C-level initialisation of the object, including allocation of any C data structures that your object will own. You need to be careful what you do in the -:meth:`__cinit__` method, because the object may not yet be fully valid Python +:meth:`__cinit__` method, because the object may not yet be a fully valid Python object when it is called. Therefore, you should be careful invoking any Python -operations which might touch the object; in particular, its methods. +operations which might touch the object; in particular, its methods and anything +that could be overridden by subtypes (and thus depend on their subtype state being +initialised already). By the time your :meth:`__cinit__` method is called, memory has been allocated for the object and any C attributes it has have been initialised to 0 or null. (Any @@ -53,12 +55,13 @@ Python attributes have also been initialised to None, but you probably shouldn't rely on that.) Your :meth:`__cinit__` method is guaranteed to be called exactly once. -If your extension type has a base type, the :meth:`__cinit__` method of the base type -is automatically called before your :meth:`__cinit__` method is called; you cannot -explicitly call the inherited :meth:`__cinit__` method. If you need to pass a modified -argument list to the base type, you will have to do the relevant part of the -initialisation in the :meth:`__init__` method instead (where the normal rules for -calling inherited methods apply). +If your extension type has a base type, any existing :meth:`__cinit__` methods in +the base type hierarchy are automatically called before your :meth:`__cinit__` +method. You cannot explicitly call the inherited :meth:`__cinit__` methods, and the +base types are free to choose whether they implement :meth:`__cinit__` at all. +If you need to pass a modified argument list to the base type, you will have to do +the relevant part of the initialisation in the :meth:`__init__` method instead, where +the normal rules for calling inherited methods apply. Any initialisation which cannot safely be done in the :meth:`__cinit__` method should be done in the :meth:`__init__` method. By the time :meth:`__init__` is called, the object is @@ -126,20 +129,36 @@ executed unless they are explicitly called by the subclass. Arithmetic methods ------------------- -Arithmetic operator methods, such as :meth:`__add__`, behave differently from their -Python counterparts. There are no separate "reversed" versions of these -methods (:meth:`__radd__`, etc.) Instead, if the first operand cannot perform the -operation, the same method of the second operand is called, with the operands -in the same order. - -This means that you can't rely on the first parameter of these methods being -"self" or being the right type, and you should test the types of both operands -before deciding what to do. If you can't handle the combination of types you've -been given, you should return `NotImplemented`. - -This also applies to the in-place arithmetic method :meth:`__ipow__`. It doesn't apply -to any of the other in-place methods (:meth:`__iadd__`, etc.) which always -take `self` as the first argument. +Arithmetic operator methods, such as :meth:`__add__`, used to behave differently +from their Python counterparts in Cython 0.x, following the low-level semantics +of the C-API slot functions. Since Cython 3.0, they are called in the same way +as in Python, including the separate "reversed" versions of these methods +(:meth:`__radd__`, etc.). + +Previously, if the first operand could not perform the operation, the same method +of the second operand was called, with the operands in the same order. +This means that you could not rely on the first parameter of these methods being +"self" or being the right type, and you needed to test the types of both operands +before deciding what to do. + +If backwards compatibility is needed, the normal operator method (``__add__``, etc.) +can still be implemented to support both variants, applying a type check to the +arguments. The reversed method (``__radd__``, etc.) can always be implemented +with ``self`` as first argument and will be ignored by older Cython versions, whereas +Cython 3.x and later will only call the normal method with the expected argument order, +and otherwise call the reversed method instead. + +Alternatively, the old Cython 0.x (or native C-API) behaviour is still available with +the directive ``c_api_binop_methods=True``. + +If you can't handle the combination of types you've been given, you should return +`NotImplemented`. This will let Python's operator implementation first try to apply +the reversed operator to the second operand, and failing that as well, report an +appropriate error to the user. + +This change in behaviour also applies to the in-place arithmetic method :meth:`__ipow__`. +It does not apply to any of the other in-place methods (:meth:`__iadd__`, etc.) +which always take `self` as the first argument. .. _righ_comparisons: @@ -212,13 +231,13 @@ https://docs.python.org/3/reference/datamodel.html#special-method-names +-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ | __dealloc__ |self | | Basic deallocation (no direct Python equivalent) | +-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ -| __cmp__ |x, y | int | 3-way comparison | +| __cmp__ |x, y | int | 3-way comparison (Python 2 only) | +-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ | __str__ |self | object | str(self) | +-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ | __repr__ |self | object | repr(self) | +-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ -| __hash__ |self | int | Hash function | +| __hash__ |self | Py_hash_t | Hash function (returns 32/64 bit integer) | +-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ | __call__ |self, ... | object | self(...) | +-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ @@ -372,7 +391,7 @@ https://docs.python.org/3/reference/datamodel.html#emulating-container-types +-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ | Name | Parameters | Return type | Description | +=======================+=======================================+=============+=====================================================+ -| __len__ | self int | | len(self) | +| __len__ | self | Py_ssize_t | len(self) | +-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ | __getitem__ | self, x | object | self[x] | +-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ diff --git a/docs/src/userguide/wrapping_CPlusPlus.rst b/docs/src/userguide/wrapping_CPlusPlus.rst index e12bf38be..1d8a25835 100644 --- a/docs/src/userguide/wrapping_CPlusPlus.rst +++ b/docs/src/userguide/wrapping_CPlusPlus.rst @@ -11,8 +11,8 @@ Overview Cython has native support for most of the C++ language. Specifically: -* C++ objects can be dynamically allocated with ``new`` and ``del`` keywords. -* C++ objects can be stack-allocated. +* C++ objects can be :term:`dynamically allocated<Dynamic allocation>` with ``new`` and ``del`` keywords. +* C++ objects can be :term:`stack-allocated<Stack allocation>`. * C++ classes can be declared with the new keyword ``cppclass``. * Templated classes and functions are supported. * Overloaded functions are supported. @@ -97,7 +97,7 @@ We use the lines:: pass to include the C++ code from :file:`Rectangle.cpp`. It is also possible to specify to -distutils that :file:`Rectangle.cpp` is a source. To do that, you can add this directive at the +setuptools that :file:`Rectangle.cpp` is a source. To do that, you can add this directive at the top of the ``.pyx`` (not ``.pxd``) file:: # distutils: sources = Rectangle.cpp @@ -331,19 +331,27 @@ arguments) or by an explicit cast, e.g.: The following coercions are available: -+------------------+----------------+-----------------+ -| Python type => | *C++ type* | => Python type | -+==================+================+=================+ -| bytes | std::string | bytes | -+------------------+----------------+-----------------+ -| iterable | std::vector | list | -+------------------+----------------+-----------------+ -| iterable | std::list | list | -+------------------+----------------+-----------------+ -| iterable | std::set | set | -+------------------+----------------+-----------------+ -| iterable (len 2) | std::pair | tuple (len 2) | -+------------------+----------------+-----------------+ ++------------------+------------------------+-----------------+ +| Python type => | *C++ type* | => Python type | ++==================+========================+=================+ +| bytes | std::string | bytes | ++------------------+------------------------+-----------------+ +| iterable | std::vector | list | ++------------------+------------------------+-----------------+ +| iterable | std::list | list | ++------------------+------------------------+-----------------+ +| iterable | std::set | set | ++------------------+------------------------+-----------------+ +| iterable | std::unordered_set | set | ++------------------+------------------------+-----------------+ +| mapping | std::map | dict | ++------------------+------------------------+-----------------+ +| mapping | std::unordered_map | dict | ++------------------+------------------------+-----------------+ +| iterable (len 2) | std::pair | tuple (len 2) | ++------------------+------------------------+-----------------+ +| complex | std::complex | complex | ++------------------+------------------------+-----------------+ All conversions create a new container and copy the data into it. The items in the containers are converted to a corresponding type @@ -482,6 +490,33 @@ Note, however, that it is unnecessary to declare the arguments of extern functions as references (const or otherwise) as it has no impact on the caller's syntax. +Scoped Enumerations +------------------- + +Cython supports scoped enumerations (:keyword:`enum class`) in C++ mode:: + + cdef enum class Cheese: + cheddar = 1 + camembert = 2 + +As with "plain" enums, you may access the enumerators as attributes of the type. +Unlike plain enums however, the enumerators are not visible to the +enclosing scope:: + + cdef Cheese c1 = Cheese.cheddar # OK + cdef Cheese c2 = cheddar # ERROR! + +Optionally, you may specify the underlying type of a scoped enumeration. +This is especially important when declaring an external scoped enumeration +with an underlying type:: + + cdef extern from "Foo.h": + cdef enum class Spam(unsigned int): + x = 10 + y = 20 + ... + +Declaring an enum class as ``cpdef`` will create a :pep:`435`-style Python wrapper. ``auto`` Keyword ---------------- @@ -527,7 +562,7 @@ Specify C++ language in setup.py Instead of specifying the language and the sources in the source files, it is possible to declare them in the :file:`setup.py` file:: - from distutils.core import setup + from setuptools import setup from Cython.Build import cythonize setup(ext_modules = cythonize( @@ -553,7 +588,7 @@ recognize the ``language`` option and it needs to be specified as an option to an :class:`Extension` that describes your extension and that is then handled by ``cythonize()`` as follows:: - from distutils.core import setup, Extension + from setuptools import Extension, setup from Cython.Build import cythonize setup(ext_modules = cythonize(Extension( @@ -568,7 +603,7 @@ often preferable (and overrides any global option). Starting with version 0.17, Cython also allows passing external source files into the ``cythonize()`` command this way. Here is a simplified setup.py file:: - from distutils.core import setup + from setuptools import setup from Cython.Build import cythonize setup( @@ -586,8 +621,8 @@ any source code, to compile it in C++ mode and link it statically against the .. note:: When using distutils directives, the paths are relative to the working - directory of the distutils run (which is usually the - project root where the :file:`setup.py` resides). + directory of the setuptools run (which is usually the project root where + the :file:`setup.py` resides). To compile manually (e.g. using ``make``), the ``cython`` command-line utility can be used to generate a C++ ``.cpp`` file, and then compile it @@ -10,7 +10,7 @@ # Profiled execution. profile=no -# Add files or directories to the blacklist. They should be base names, not +# Add files or directories to the ignorelist. They should be base names, not # paths. ignore=.git,.gitmarker diff --git a/pyximport/pyxbuild.py b/pyximport/pyxbuild.py index de4a2241f..842e54502 100644 --- a/pyximport/pyxbuild.py +++ b/pyximport/pyxbuild.py @@ -103,7 +103,7 @@ def pyx_to_dll(filename, ext=None, force_rebuild=0, build_in_temp=False, pyxbuil so_path = obj_build_ext.get_outputs()[0] if obj_build_ext.inplace: # Python distutils get_outputs()[ returns a wrong so_path - # when --inplace ; see http://bugs.python.org/issue5977 + # when --inplace ; see https://bugs.python.org/issue5977 # workaround: so_path = os.path.join(os.path.dirname(filename), os.path.basename(so_path)) @@ -119,9 +119,9 @@ def pyx_to_dll(filename, ext=None, force_rebuild=0, build_in_temp=False, pyxbuil while count < 100: count += 1 r_path = os.path.join(obj_build_ext.build_lib, - basename + '.reload%s'%count) + basename + '.reload%s' % count) try: - import shutil # late import / reload_support is: debugging + import shutil # late import / reload_support is: debugging try: # Try to unlink first --- if the .so file # is mmapped by another process, @@ -140,7 +140,7 @@ def pyx_to_dll(filename, ext=None, force_rebuild=0, build_in_temp=False, pyxbuil break else: # used up all 100 slots - raise ImportError("reload count for %s reached maximum"%org_path) + raise ImportError("reload count for %s reached maximum" % org_path) _reloads[org_path]=(timestamp, so_path, count) return so_path except KeyboardInterrupt: @@ -157,4 +157,3 @@ def pyx_to_dll(filename, ext=None, force_rebuild=0, build_in_temp=False, pyxbuil if __name__=="__main__": pyx_to_dll("dummy.pyx") from . import test - diff --git a/pyximport/pyximport.py b/pyximport/pyximport.py index 5628f301b..7ff2ad219 100644 --- a/pyximport/pyximport.py +++ b/pyximport/pyximport.py @@ -191,7 +191,7 @@ def build_module(name, pyxfilename, pyxbuild_dir=None, inplace=False, language_l reload_support=pyxargs.reload_support) assert os.path.exists(so_path), "Cannot find: %s" % so_path - junkpath = os.path.join(os.path.dirname(so_path), name+"_*") #very dangerous with --inplace ? yes, indeed, trying to eat my files ;) + junkpath = os.path.join(os.path.dirname(so_path), name+"_*") #very dangerous with --inplace ? yes, indeed, trying to eat my files ;) junkstuff = glob.glob(junkpath) for path in junkstuff: if path != so_path: @@ -269,7 +269,7 @@ class PyxImporter(object): pyxbuild_dir=self.pyxbuild_dir, inplace=self.inplace, language_level=self.language_level) - if ty != imp.C_EXTENSION: # only when an extension, check if we have a .pyx next! + if ty != imp.C_EXTENSION: # only when an extension, check if we have a .pyx next! return None # find .pyx fast, when .so/.pyd exist --inplace @@ -350,7 +350,7 @@ class PyImporter(PyxImporter): language_level=language_level) self.uncompilable_modules = {} self.blocked_modules = ['Cython', 'pyxbuild', 'pyximport.pyxbuild', - 'distutils.extension', 'distutils.sysconfig'] + 'distutils'] def find_module(self, fullname, package_path=None): if fullname in sys.modules: diff --git a/pyximport/test/test_pyximport.py b/pyximport/test/test_pyximport.py index b3a4a9058..0b6b7432b 100644 --- a/pyximport/test/test_pyximport.py +++ b/pyximport/test/test_pyximport.py @@ -66,9 +66,9 @@ def make_ext(name, filename): assert len(pyximport._test_files)==1, pyximport._test_files reload(dummy) - time.sleep(1) # sleep a second to get safer mtimes + time.sleep(1) # sleep a second to get safer mtimes open(os.path.join(tempdir, "abc.txt"), "w").write(" ") - print("Here goes the reolad") + print("Here goes the reload") reload(dummy) assert len(pyximport._test_files) == 1, pyximport._test_files diff --git a/pyximport/test/test_reload.py b/pyximport/test/test_reload.py index ba53746f9..8c4c7962b 100644 --- a/pyximport/test/test_reload.py +++ b/pyximport/test/test_reload.py @@ -22,8 +22,8 @@ def test(): import hello assert hello.x == 1 - time.sleep(1) # sleep to make sure that new "hello.pyx" has later - # timestamp than object file. + time.sleep(1) # sleep to make sure that new "hello.pyx" has later + # timestamp than object file. open(hello_file, "w").write("x = 2; print x; after = 'after'\n") reload(hello) diff --git a/runtests.py b/runtests.py index a29bfb806..91f190951 100755 --- a/runtests.py +++ b/runtests.py @@ -3,6 +3,7 @@ from __future__ import print_function import atexit +import base64 import os import sys import re @@ -30,6 +31,7 @@ try: except (ImportError, AttributeError): IS_CPYTHON = True IS_PYPY = False +IS_PY2 = sys.version_info[0] < 3 from io import open as io_open try: @@ -64,11 +66,9 @@ except NameError: basestring = str WITH_CYTHON = True -CY3_DIR = None from distutils.command.build_ext import build_ext as _build_ext from distutils import sysconfig -from distutils import ccompiler _to_clean = [] @atexit.register @@ -96,7 +96,7 @@ def get_distutils_distro(_cache=[]): distutils_distro = Distribution() if sys.platform == 'win32': - # TODO: Figure out why this hackery (see http://thread.gmane.org/gmane.comp.python.cython.devel/8280/). + # TODO: Figure out why this hackery (see https://thread.gmane.org/gmane.comp.python.cython.devel/8280/). config_files = distutils_distro.find_config_files() try: config_files.remove('setup.cfg') @@ -116,7 +116,6 @@ def get_distutils_distro(_cache=[]): EXT_DEP_MODULES = { 'tag:numpy': 'numpy', - 'tag:numpy_old': 'numpy', 'tag:pythran': 'pythran', 'tag:setuptools': 'setuptools.sandbox', 'tag:asyncio': 'asyncio', @@ -236,17 +235,13 @@ def update_linetrace_extension(ext): return ext -def update_old_numpy_extension(ext): - update_numpy_extension(ext, set_api17_macro=False) - - def update_numpy_extension(ext, set_api17_macro=True): import numpy from numpy.distutils.misc_util import get_info ext.include_dirs.append(numpy.get_include()) - if set_api17_macro: + if set_api17_macro and getattr(numpy, '__version__', '') not in ('1.19.0', '1.19.1'): ext.define_macros.append(('NPY_NO_DEPRECATED_API', 'NPY_1_7_API_VERSION')) # We need the npymath library for numpy.math. @@ -255,6 +250,22 @@ def update_numpy_extension(ext, set_api17_macro=True): getattr(ext, attr).extend(value) +def update_gdb_extension(ext, _has_gdb=[None]): + # We should probably also check for Python support. + if not include_debugger: + _has_gdb[0] = False + if _has_gdb[0] is None: + try: + subprocess.check_call(["gdb", "--version"]) + except (IOError, subprocess.CalledProcessError): + _has_gdb[0] = False + else: + _has_gdb[0] = True + if not _has_gdb[0]: + return EXCLUDE_EXT + return ext + + def update_openmp_extension(ext): ext.openmp = True language = ext.language @@ -310,7 +321,8 @@ def get_cc_version(language): else: cc = sysconfig.get_config_var('CC') if not cc: - cc = ccompiler.get_default_compiler() + from distutils import ccompiler + cc = ccompiler.get_default_compiler() if not cc: return '' @@ -380,12 +392,13 @@ EXCLUDE_EXT = object() EXT_EXTRAS = { 'tag:numpy' : update_numpy_extension, - 'tag:numpy_old' : update_old_numpy_extension, 'tag:openmp': update_openmp_extension, + 'tag:gdb': update_gdb_extension, 'tag:cpp11': update_cpp11_extension, 'tag:trace' : update_linetrace_extension, 'tag:bytesformat': exclude_extension_in_pyver((3, 3), (3, 4)), # no %-bytes formatting 'tag:no-macos': exclude_extension_on_platform('darwin'), + 'tag:py3only': exclude_extension_in_pyver((2, 7)), } @@ -393,28 +406,27 @@ EXT_EXTRAS = { VER_DEP_MODULES = { # tests are excluded if 'CurrentPythonVersion OP VersionTuple', i.e. # (2,4) : (operator.lt, ...) excludes ... when PyVer < 2.4.x - (2,7) : (operator.lt, lambda x: x in ['run.withstat_py27', # multi context with statement - 'run.yield_inside_lambda', - 'run.test_dictviews', - 'run.pyclass_special_methods', - 'run.set_literals', - ]), + # The next line should start (3,); but this is a dictionary, so # we can only have one (3,) key. Since 2.7 is supposed to be the # last 2.x release, things would have to change drastically for this # to be unsafe... (2,999): (operator.lt, lambda x: x in ['run.special_methods_T561_py3', 'run.test_raisefrom', + 'run.different_package_names', + 'run.unicode_imports', # encoding problems on appveyor in Py2 'run.reimport_failure', # reimports don't do anything in Py2 ]), (3,): (operator.ge, lambda x: x in ['run.non_future_division', 'compile.extsetslice', 'compile.extdelslice', - 'run.special_methods_T561_py2' + 'run.special_methods_T561_py2', ]), (3,3) : (operator.lt, lambda x: x in ['build.package_compilation', + 'build.cythonize_pep420_namespace', 'run.yield_from_py33', 'pyximport.pyximport_namespace', + 'run.qualname', ]), (3,4): (operator.lt, lambda x: x in ['run.py34_signature', 'run.test_unicode', # taken from Py3.7, difficult to backport @@ -426,6 +438,7 @@ VER_DEP_MODULES = { 'run.mod__spec__', 'run.pep526_variable_annotations', # typing module 'run.test_exceptions', # copied from Py3.7+ + 'run.time_pxd', # _PyTime_GetSystemClock doesn't exist in 3.4 ]), } @@ -483,7 +496,7 @@ def parse_tags(filepath): return tags -list_unchanging_dir = memoize(lambda x: os.listdir(x)) +list_unchanging_dir = memoize(lambda x: os.listdir(x)) # needs lambda to set function attribute @memoize @@ -529,9 +542,14 @@ class build_ext(_build_ext): class ErrorWriter(object): match_error = re.compile(r'(warning:)?(?:.*:)?\s*([-0-9]+)\s*:\s*([-0-9]+)\s*:\s*(.*)').match - def __init__(self): + def __init__(self, encoding=None): self.output = [] - self.write = self.output.append + self.encoding = encoding + + def write(self, value): + if self.encoding: + value = value.encode('ISO-8859-1').decode(self.encoding) + self.output.append(value) def _collect(self): s = ''.join(self.output) @@ -629,11 +647,13 @@ class TestBuilder(object): self.default_mode = default_mode self.stats = stats self.add_embedded_test = add_embedded_test + self.capture = options.capture def build_suite(self): suite = unittest.TestSuite() filenames = os.listdir(self.rootdir) filenames.sort() + # TODO: parallelise I/O with a thread pool for the different directories once we drop Py2 support for filename in filenames: path = os.path.join(self.rootdir, filename) if os.path.isdir(path) and filename != TEST_SUPPORT_DIR: @@ -648,7 +668,7 @@ class TestBuilder(object): and (sys.version_info < (3, 8) or sys.platform != 'darwin')): # Non-Windows makefile. if [1 for selector in self.selectors if selector("embedded")] \ - and not [1 for selector in self.exclude_selectors if selector("embedded")]: + and not [1 for selector in self.exclude_selectors if selector("embedded")]: suite.addTest(unittest.makeSuite(EmbedTest)) return suite @@ -688,7 +708,9 @@ class TestBuilder(object): if ext == '.srctree': if 'cpp' not in tags['tag'] or 'cpp' in self.languages: - suite.addTest(EndToEndTest(filepath, workdir, self.cleanup_workdir, stats=self.stats)) + suite.addTest(EndToEndTest(filepath, workdir, + self.cleanup_workdir, stats=self.stats, + capture=self.capture)) continue # Choose the test suite. @@ -743,6 +765,8 @@ class TestBuilder(object): if not languages: return [] + language_levels = [2, 3] if 'all_language_levels' in tags['tag'] else [None] + pythran_dir = self.pythran_dir if 'pythran' in tags['tag'] and not pythran_dir and 'cpp' in languages: import pythran.config @@ -753,21 +777,25 @@ class TestBuilder(object): pythran_dir = pythran_ext['include_dirs'][0] preparse_list = tags.get('preparse', ['id']) - tests = [ self.build_test(test_class, path, workdir, module, tags, language, + tests = [ self.build_test(test_class, path, workdir, module, tags, language, language_level, expect_errors, expect_warnings, warning_errors, preparse, pythran_dir if language == "cpp" else None) for language in languages - for preparse in preparse_list ] + for preparse in preparse_list + for language_level in language_levels + ] return tests - def build_test(self, test_class, path, workdir, module, tags, language, + def build_test(self, test_class, path, workdir, module, tags, language, language_level, expect_errors, expect_warnings, warning_errors, preparse, pythran_dir): language_workdir = os.path.join(workdir, language) if not os.path.exists(language_workdir): os.makedirs(language_workdir) workdir = os.path.join(language_workdir, module) if preparse != 'id': - workdir += '_%s' % str(preparse) + workdir += '_%s' % (preparse,) + if language_level: + workdir += '_cy%d' % (language_level,) return test_class(path, workdir, module, tags, language=language, preparse=preparse, @@ -780,7 +808,7 @@ class TestBuilder(object): cython_only=self.cython_only, doctest_selector=self.doctest_selector, fork=self.fork, - language_level=self.language_level, + language_level=language_level or self.language_level, warning_errors=warning_errors, test_determinism=self.test_determinism, common_utility_dir=self.common_utility_dir, @@ -848,13 +876,27 @@ class CythonCompileTestCase(unittest.TestCase): unittest.TestCase.__init__(self) def shortDescription(self): - return "compiling (%s%s) %s" % (self.language, "/pythran" if self.pythran_dir is not None else "", self.name) + return "compiling (%s%s%s) %s" % ( + self.language, + "/cy2" if self.language_level == 2 else "/cy3" if self.language_level == 3 else "", + "/pythran" if self.pythran_dir is not None else "", + self.description_name() + ) + + def description_name(self): + return self.name def setUp(self): from Cython.Compiler import Options self._saved_options = [ (name, getattr(Options, name)) - for name in ('warning_errors', 'clear_to_none', 'error_on_unknown_names', 'error_on_uninitialized') + for name in ( + 'warning_errors', + 'clear_to_none', + 'error_on_unknown_names', + 'error_on_uninitialized', + # 'cache_builtins', # not currently supported due to incorrect global caching + ) ] self._saved_default_directives = list(Options.get_directive_defaults().items()) Options.warning_errors = self.warning_errors @@ -891,13 +933,17 @@ class CythonCompileTestCase(unittest.TestCase): shutil.rmtree(self.workdir, ignore_errors=True) else: for rmfile in os.listdir(self.workdir): + ext = os.path.splitext(rmfile)[1] if not cleanup_c_files: - if (rmfile[-2:] in (".c", ".h") or - rmfile[-4:] == ".cpp" or - rmfile.endswith(".html") and rmfile.startswith(self.module)): + # Keep C, C++ files, header files, preprocessed sources + # and assembly sources (typically the .i and .s files + # are intentionally generated when -save-temps is given) + if ext in (".c", ".cpp", ".h", ".i", ".ii", ".s"): + continue + if ext == ".html" and rmfile.startswith(self.module): continue - is_shared_obj = rmfile.endswith(".so") or rmfile.endswith(".dll") + is_shared_obj = ext in (".so", ".dll") if not cleanup_lib_files and is_shared_obj: continue @@ -973,6 +1019,13 @@ class CythonCompileTestCase(unittest.TestCase): def split_source_and_output(self, test_directory, module, workdir): source_file = self.find_module_source_file(os.path.join(test_directory, module) + '.pyx') + + from Cython.Utils import detect_opened_file_encoding + with io_open(source_file, 'rb') as f: + # encoding is passed to ErrorWriter but not used on the source + # since it is sometimes deliberately wrong + encoding = detect_opened_file_encoding(f, default=None) + with io_open(source_file, 'r', encoding='ISO-8859-1') as source_and_output: error_writer = warnings_writer = None out = io_open(os.path.join(workdir, module + os.path.splitext(source_file)[1]), @@ -981,10 +1034,10 @@ class CythonCompileTestCase(unittest.TestCase): for line in source_and_output: if line.startswith("_ERRORS"): out.close() - out = error_writer = ErrorWriter() + out = error_writer = ErrorWriter(encoding=encoding) elif line.startswith("_WARNINGS"): out.close() - out = warnings_writer = ErrorWriter() + out = warnings_writer = ErrorWriter(encoding=encoding) else: out.write(line) finally: @@ -1017,9 +1070,9 @@ class CythonCompileTestCase(unittest.TestCase): try: CompilationOptions except NameError: - from Cython.Compiler.Main import CompilationOptions + from Cython.Compiler.Options import CompilationOptions from Cython.Compiler.Main import compile as cython_compile - from Cython.Compiler.Main import default_options + from Cython.Compiler.Options import default_options common_utility_include_dir = self.common_utility_dir options = CompilationOptions( @@ -1106,6 +1159,8 @@ class CythonCompileTestCase(unittest.TestCase): extension = newext or extension if self.language == 'cpp': extension.language = 'c++' + if IS_PY2: + workdir = str(workdir) # work around type check in distutils that disallows unicode strings build_extension.extensions = [extension] build_extension.build_temp = workdir build_extension.build_lib = workdir @@ -1202,11 +1257,14 @@ class CythonCompileTestCase(unittest.TestCase): finally: if show_output: stdout = get_stdout and get_stdout().strip() + stderr = get_stderr and filter_stderr(get_stderr()).strip() + if so_path and not stderr: + # normal success case => ignore non-error compiler output + stdout = None if stdout: print_bytes( stdout, header_text="\n=== C/C++ compiler output: =========\n", end=None, file=sys.__stderr__) - stderr = get_stderr and filter_stderr(get_stderr()).strip() if stderr: print_bytes( stderr, header_text="\n=== C/C++ compiler error output: ===\n", @@ -1240,11 +1298,8 @@ class CythonRunTestCase(CythonCompileTestCase): from Cython.Compiler import Options Options.clear_to_none = False - def shortDescription(self): - if self.cython_only: - return CythonCompileTestCase.shortDescription(self) - else: - return "compiling (%s%s) and running %s" % (self.language, "/pythran" if self.pythran_dir is not None else "", self.name) + def description_name(self): + return self.name if self.cython_only else "and running %s" % self.name def run(self, result=None): if result is None: @@ -1255,8 +1310,7 @@ class CythonRunTestCase(CythonCompileTestCase): try: self.success = False ext_so_path = self.runCompileTest() - # Py2.6 lacks "_TextTestResult.skipped" - failures, errors, skipped = len(result.failures), len(result.errors), len(getattr(result, 'skipped', [])) + failures, errors, skipped = len(result.failures), len(result.errors), len(result.skipped) if not self.cython_only and ext_so_path is not None: self.run_tests(result, ext_so_path) if failures == len(result.failures) and errors == len(result.errors): @@ -1441,10 +1495,6 @@ class PartialTestResult(_TextTestResult): _TextTestResult.__init__( self, self._StringIO(), True, base_result.dots + base_result.showAll*2) - try: - self.skipped - except AttributeError: - self.skipped = [] # Py2.6 def strip_error_results(self, results): for test_case, error in results: @@ -1469,10 +1519,7 @@ class PartialTestResult(_TextTestResult): if output: result.stream.write(output) result.errors.extend(errors) - try: - result.skipped.extend(skipped) - except AttributeError: - pass # Py2.6 + result.skipped.extend(skipped) result.failures.extend(failures) result.testsRun += tests_run @@ -1485,7 +1532,7 @@ class PartialTestResult(_TextTestResult): class CythonUnitTestCase(CythonRunTestCase): def shortDescription(self): - return "compiling (%s) tests in %s" % (self.language, self.name) + return "compiling (%s) tests in %s" % (self.language, self.description_name()) def run_tests(self, result, ext_so_path): with self.stats.time(self.name, self.language, 'import'): @@ -1578,10 +1625,12 @@ class TestCodeFormat(unittest.TestCase): def runTest(self): import pycodestyle - config_file = os.path.join(self.cython_dir, "tox.ini") + config_file = os.path.join(self.cython_dir, "setup.cfg") if not os.path.exists(config_file): - config_file=os.path.join(os.path.dirname(__file__), "tox.ini") - paths = glob.glob(os.path.join(self.cython_dir, "**/*.py"), recursive=True) + config_file = os.path.join(os.path.dirname(__file__), "setup.cfg") + paths = [] + for codedir in ['Cython', 'Demos', 'docs', 'pyximport', 'tests']: + paths += glob.glob(os.path.join(self.cython_dir, codedir + "/**/*.py"), recursive=True) style = pycodestyle.StyleGuide(config_file=config_file) print("") # Fix the first line of the report. result = style.check_files(paths) @@ -1638,13 +1687,13 @@ def collect_doctests(path, module_prefix, suite, selectors, exclude_selectors): return dirname not in ("Mac", "Distutils", "Plex", "Tempita") def file_matches(filename): filename, ext = os.path.splitext(filename) - blacklist = ['libcython', 'libpython', 'test_libcython_in_gdb', - 'TestLibCython'] + excludelist = ['libcython', 'libpython', 'test_libcython_in_gdb', + 'TestLibCython'] return (ext == '.py' and not '~' in filename and not '#' in filename and not filename.startswith('.') and not - filename in blacklist) + filename in excludelist) import doctest for dirpath, dirnames, filenames in os.walk(path): for dir in list(dirnames): @@ -1681,12 +1730,13 @@ class EndToEndTest(unittest.TestCase): """ cython_root = os.path.dirname(os.path.abspath(__file__)) - def __init__(self, treefile, workdir, cleanup_workdir=True, stats=None): + def __init__(self, treefile, workdir, cleanup_workdir=True, stats=None, capture=True): self.name = os.path.splitext(os.path.basename(treefile))[0] self.treefile = treefile self.workdir = os.path.join(workdir, self.name) self.cleanup_workdir = cleanup_workdir self.stats = stats + self.capture = capture cython_syspath = [self.cython_root] for path in sys.path: if path.startswith(self.cython_root) and path not in cython_syspath: @@ -1702,11 +1752,9 @@ class EndToEndTest(unittest.TestCase): def setUp(self): from Cython.TestUtils import unpack_source_tree - _, self.commands = unpack_source_tree(self.treefile, self.workdir) + _, self.commands = unpack_source_tree(self.treefile, self.workdir, self.cython_root) self.old_dir = os.getcwd() os.chdir(self.workdir) - if self.workdir not in sys.path: - sys.path.insert(0, self.workdir) def tearDown(self): if self.cleanup_workdir: @@ -1727,31 +1775,31 @@ class EndToEndTest(unittest.TestCase): def runTest(self): self.success = False - commands = (self.commands - .replace("CYTHON", "PYTHON %s" % os.path.join(self.cython_root, 'cython.py')) - .replace("PYTHON", sys.executable)) old_path = os.environ.get('PYTHONPATH') env = dict(os.environ) new_path = self.cython_syspath if old_path: - new_path = new_path + os.pathsep + old_path + new_path = new_path + os.pathsep + self.workdir + os.pathsep + old_path env['PYTHONPATH'] = new_path + if not env.get("PYTHONIOENCODING"): + env["PYTHONIOENCODING"] = sys.stdout.encoding or sys.getdefaultencoding() cmd = [] out = [] err = [] - for command_no, command in enumerate(filter(None, commands.splitlines()), 1): + for command_no, command in enumerate(self.commands, 1): with self.stats.time('%s(%d)' % (self.name, command_no), 'c', - 'etoe-build' if ' setup.py ' in command else 'etoe-run'): - p = subprocess.Popen(command, - stderr=subprocess.PIPE, - stdout=subprocess.PIPE, - shell=True, - env=env) - _out, _err = p.communicate() + 'etoe-build' if 'setup.py' in command else 'etoe-run'): + if self.capture: + p = subprocess.Popen(command, stderr=subprocess.PIPE, stdout=subprocess.PIPE, env=env) + _out, _err = p.communicate() + res = p.returncode + else: + p = subprocess.call(command, env=env) + _out, _err = b'', b'' + res = p cmd.append(command) out.append(_out) err.append(_err) - res = p.returncode if res == 0 and b'REFNANNY: ' in _out: res = -1 if res != 0: @@ -1793,13 +1841,10 @@ class EmbedTest(unittest.TestCase): if not os.path.isdir(libdir) or libname not in os.listdir(libdir): # report the error for the original directory libdir = sysconfig.get_config_var('LIBDIR') - cython = 'cython.py' - if sys.version_info[0] >=3 and CY3_DIR: - cython = os.path.join(CY3_DIR, cython) - cython = os.path.abspath(os.path.join('..', '..', cython)) + cython = os.path.abspath(os.path.join('..', '..', 'cython.py')) try: - subprocess.check_call([ + subprocess.check_output([ "make", "PYTHON='%s'" % sys.executable, "CYTHON='%s'" % cython, @@ -1812,17 +1857,43 @@ class EmbedTest(unittest.TestCase): self.assertTrue(True) # :) +def load_listfile(filename): + # just re-use the FileListExclude implementation + fle = FileListExcluder(filename) + return list(fle.excludes) class MissingDependencyExcluder(object): def __init__(self, deps): # deps: { matcher func : module name } self.exclude_matchers = [] - for matcher, mod in deps.items(): + for matcher, module_name in deps.items(): try: - __import__(mod) + module = __import__(module_name) except ImportError: self.exclude_matchers.append(string_selector(matcher)) + print("Test dependency not found: '%s'" % module_name) + else: + version = self.find_dep_version(module_name, module) + print("Test dependency found: '%s' version %s" % (module_name, version)) self.tests_missing_deps = [] + + def find_dep_version(self, name, module): + try: + version = module.__version__ + except AttributeError: + stdlib_dir = os.path.dirname(shutil.__file__) + os.sep + module_path = getattr(module, '__file__', stdlib_dir) # no __file__? => builtin stdlib module + if module_path.startswith(stdlib_dir): + # stdlib module + version = sys.version.partition(' ')[0] + elif '.' in name: + # incrementally look for a parent package with version + name = name.rpartition('.')[0] + return self.find_dep_version(name, __import__(name)) + else: + version = '?.?' + return version + def __call__(self, testname, tags=None): for matcher in self.exclude_matchers: if matcher(testname, tags): @@ -1860,8 +1931,7 @@ class FileListExcluder(object): self.excludes[line.split()[0]] = True def __call__(self, testname, tags=None): - exclude = (testname in self.excludes - or testname.split('.')[-1] in self.excludes) + exclude = any(string_selector(ex)(testname) for ex in self.excludes) if exclude and self.verbose: print("Excluding %s because it's listed in %s" % (testname, self._list_file)) @@ -1903,14 +1973,18 @@ class ShardExcludeSelector(object): # This is an exclude selector so it can override the (include) selectors. # It may not provide uniform distribution (in time or count), but is a # determanistic partition of the tests which is important. + + # Random seed to improve the hash distribution. + _seed = base64.b64decode(b'2ged1EtsGz/GkisJr22UcLeP6n9XIaA5Vby2wM49Wvg=') + def __init__(self, shard_num, shard_count): self.shard_num = shard_num self.shard_count = shard_count - def __call__(self, testname, tags=None, _hash=zlib.crc32, _is_py2=sys.version_info[0] < 3): + def __call__(self, testname, tags=None, _hash=zlib.crc32, _is_py2=IS_PY2): # Cannot use simple hash() here as shard processes might use different hash seeds. # CRC32 is fast and simple, but might return negative values in Py2. - hashval = _hash(testname) & 0x7fffffff if _is_py2 else _hash(testname.encode()) + hashval = _hash(self._seed + testname) & 0x7fffffff if _is_py2 else _hash(self._seed + testname.encode()) return hashval % self.shard_count != self.shard_num @@ -1982,6 +2056,10 @@ def flush_and_terminate(status): def main(): global DISTDIR, WITH_CYTHON + + # Set an environment variable to the top directory + os.environ['CYTHON_PROJECT_DIR'] = os.path.abspath(os.path.dirname(__file__)) + DISTDIR = os.path.join(os.getcwd(), os.path.dirname(sys.argv[0])) from Cython.Compiler import DebugFlags @@ -2050,6 +2128,9 @@ def main(): parser.add_option("-x", "--exclude", dest="exclude", action="append", metavar="PATTERN", help="exclude tests matching the PATTERN") + parser.add_option("--listfile", dest="listfile", + action="append", + help="specify a file containing a list of tests to run") parser.add_option("-j", "--shard_count", dest="shard_count", metavar="N", type=int, default=1, help="shard this run into several parallel runs") @@ -2115,6 +2196,10 @@ def main(): help="test whether Cython's output is deterministic") parser.add_option("--pythran-dir", dest="pythran_dir", default=None, help="specify Pythran include directory. This will run the C++ tests using Pythran backend for Numpy") + parser.add_option("--no-capture", dest="capture", default=True, action="store_false", + help="do not capture stdout, stderr in srctree tests. Makes pdb.set_trace interactive") + parser.add_option("--limited-api", dest="limited_api", default=False, action="store_true", + help="Compiles Cython using CPython's LIMITED_API") options, cmd_args = parser.parse_args(args) @@ -2141,7 +2226,18 @@ def main(): if options.xml_output_dir: shutil.rmtree(options.xml_output_dir, ignore_errors=True) + if options.listfile: + for listfile in options.listfile: + cmd_args.extend(load_listfile(listfile)) + + if options.capture and not options.for_debugging: + keep_alive_interval = 10 + else: + keep_alive_interval = None if options.shard_count > 1 and options.shard_num == -1: + if "PYTHONIOENCODING" not in os.environ: + # Make sure subprocesses can print() Unicode text. + os.environ["PYTHONIOENCODING"] = sys.stdout.encoding or sys.getdefaultencoding() import multiprocessing pool = multiprocessing.Pool(options.shard_count) tasks = [(options, cmd_args, shard_num) for shard_num in range(options.shard_count)] @@ -2150,7 +2246,7 @@ def main(): # NOTE: create process pool before time stamper thread to avoid forking issues. total_time = time.time() stats = Stats() - with time_stamper_thread(): + with time_stamper_thread(interval=keep_alive_interval): for shard_num, shard_stats, return_code, failure_output in pool.imap_unordered(runtests_callback, tasks): if return_code != 0: error_shards.append(shard_num) @@ -2170,7 +2266,7 @@ def main(): else: return_code = 0 else: - with time_stamper_thread(): + with time_stamper_thread(interval=keep_alive_interval): _, stats, return_code, _ = runtests(options, cmd_args, coverage) if coverage: @@ -2200,20 +2296,30 @@ def time_stamper_thread(interval=10): Print regular time stamps into the build logs to find slow tests. @param interval: time interval in seconds """ + if not interval or interval < 0: + # Do nothing + yield + return + try: _xrange = xrange except NameError: _xrange = range import threading - from datetime import datetime + import datetime from time import sleep interval = _xrange(interval * 4) - now = datetime.now - write = sys.__stderr__.write + now = datetime.datetime.now stop = False + # We capture stderr in some places. + # => make sure we write to the real (original) stderr of the test runner. + stderr = os.dup(2) + def write(s): + os.write(stderr, s if type(s) is bytes else s.encode('ascii')) + def time_stamper(): while True: for _ in interval: @@ -2223,18 +2329,19 @@ def time_stamper_thread(interval=10): write('\n#### %s\n' % now()) thread = threading.Thread(target=time_stamper, name='time_stamper') - thread.setDaemon(True) # Py2.6 ... + thread.setDaemon(True) # Py2 ... thread.start() try: yield finally: stop = True thread.join() + os.close(stderr) def configure_cython(options): global CompilationOptions, pyrex_default_options, cython_compile - from Cython.Compiler.Main import \ + from Cython.Compiler.Options import \ CompilationOptions, \ default_options as pyrex_default_options from Cython.Compiler.Options import _directive_defaults as directive_defaults @@ -2268,6 +2375,24 @@ def runtests_callback(args): def runtests(options, cmd_args, coverage=None): + # faulthandler should be able to provide a limited traceback + # in the event of a segmentation fault. Hopefully better than Travis + # just keeping running until timeout. Only available on Python 3.3+ + try: + import faulthandler + except ImportError: + pass # OK - not essential + else: + faulthandler.enable() + + if sys.platform == "win32" and sys.version_info < (3, 6): + # enable Unicode console output, if possible + try: + import win_unicode_console + except ImportError: + pass + else: + win_unicode_console.enable() WITH_CYTHON = options.with_cython ROOTDIR = os.path.abspath(options.root_dir) @@ -2306,7 +2431,7 @@ def runtests(options, cmd_args, coverage=None): options.cleanup_sharedlibs = False options.fork = False if WITH_CYTHON and include_debugger: - from Cython.Compiler.Main import default_options as compiler_default_options + from Cython.Compiler.Options import default_options as compiler_default_options compiler_default_options['gdb_debug'] = True compiler_default_options['output_dir'] = os.getcwd() @@ -2323,6 +2448,10 @@ def runtests(options, cmd_args, coverage=None): sys.path.insert(0, os.path.split(libpath)[0]) CDEFS.append(('CYTHON_REFNANNY', '1')) + if options.limited_api: + CFLAGS.append("-DCYTHON_LIMITED_API=1") + CFLAGS.append('-Wno-unused-function') + if xml_output_dir and options.fork: # doesn't currently work together sys.stderr.write("Disabling forked testing to support XML test output\n") @@ -2381,6 +2510,10 @@ def runtests(options, cmd_args, coverage=None): bug_files = [ ('bugs.txt', True), ('pypy_bugs.txt', IS_PYPY), + ('pypy2_bugs.txt', IS_PYPY and IS_PY2), + ('pypy_crash_bugs.txt', IS_PYPY), + ('pypy_implementation_detail_bugs.txt', IS_PYPY), + ('limited_api_bugs.txt', options.limited_api), ('windows_bugs.txt', sys.platform == 'win32'), ('cygwin_bugs.txt', sys.platform == 'cygwin') ] @@ -2486,10 +2619,7 @@ def runtests(options, cmd_args, coverage=None): else: text_runner_options = {} if options.failfast: - if sys.version_info < (2, 7): - sys.stderr.write("--failfast not supported with Python < 2.7\n") - else: - text_runner_options['failfast'] = True + text_runner_options['failfast'] = True test_runner = unittest.TextTestRunner(verbosity=options.verbosity, **text_runner_options) if options.pyximport_py: diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 000000000..25db8b4bd --- /dev/null +++ b/setup.cfg @@ -0,0 +1,43 @@ +[flake8] +max-complexity = 10 + +[pycodestyle] +exclude = .git,build,__pycache__,venv*,TEST*,tests/run/test*.py,Cython/Debugger/libpython.py +max-line-length = 150 +format = pylint +# See https://pycodestyle.pycqa.org/en/latest/intro.html#configuration +select = + E711, E713, E714, E501, W291, E502, E703, + # indentation + E101, E111, E112, E113, E117 + E121, E125, E129, + # E114, E115, E116, E122, + # whitespace + E223, E224, E228, E261, E273, E274, E275, + # E201, E202, E203, E211, E265 + # E303, E306, + W1, W2, W3 +#ignore = W, E +ignore = + W504, + # W504 line break after binary operator + S001, + # S001 found module formatter + E226, + # E226 missing whitespace around operator[run] + +[coverage:run] +branch = True +parallel = True +concurrency = multiprocessing,thread +include = Cython/* +source = Cython +omit = Test* + +[bdist_wheel] +universal = 1 + +[metadata] +license_files = + LICENSE.txt + COPYING.txt @@ -14,7 +14,7 @@ is_cpython = platform.python_implementation() == 'CPython' # this specifies which versions of python we support, pip >= 9 knows to skip # versions of packages which are not compatible with the running python -PYTHON_REQUIRES = '>=2.6, !=3.0.*, !=3.1.*, !=3.2.*' +PYTHON_REQUIRES = '>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*' if sys.platform == "darwin": # Don't create resource files on OS X tar. @@ -42,8 +42,7 @@ pxd_include_dirs = [ directory for directory, dirs, files in os.walk(os.path.join('Cython', 'Includes')) if '__init__.pyx' in files or '__init__.pxd' in files - or directory == os.path.join('Cython', 'Includes') - or directory == os.path.join('Cython', 'Includes', 'Deprecated')] + or directory == os.path.join('Cython', 'Includes')] pxd_include_patterns = [ p+'/*.pxd' for p in pxd_include_dirs ] + [ @@ -85,6 +84,9 @@ def compile_cython_modules(profile=False, compile_more=False, cython_with_refnan compiled_modules = [ "Cython.Plex.Scanners", "Cython.Plex.Actions", + "Cython.Plex.Machines", + "Cython.Plex.Transitions", + "Cython.Plex.DFA", "Cython.Compiler.Scanning", "Cython.Compiler.Visitor", "Cython.Compiler.FlowControl", @@ -157,7 +159,12 @@ def compile_cython_modules(profile=False, compile_more=False, cython_with_refnan from Cython.Distutils.build_ext import new_build_ext from Cython.Compiler.Options import get_directive_defaults - get_directive_defaults()['language_level'] = 2 + get_directive_defaults().update( + language_level=2, + binding=False, + always_allow_keywords=False, + autotestdict=False, + ) if profile: get_directive_defaults()['profile'] = True sys.stderr.write("Enabled profiling for the Cython binary modules\n") @@ -189,15 +196,10 @@ try: except ValueError: compile_cython_itself = True -if compile_cython_itself and (is_cpython or cython_compile_more): - compile_cython_modules(cython_profile, cython_compile_more, cython_with_refnanny) - setup_args.update(setuptools_extra_args) -from Cython import __version__ as version - -def dev_status(): +def dev_status(version): if 'b' in version or 'c' in version: # 1b1, 1beta1, 2rc1, ... return 'Development Status :: 4 - Beta' @@ -225,65 +227,74 @@ packages = [ 'pyximport', ] -setup( - name='Cython', - version=version, - url='http://cython.org/', - author='Robert Bradshaw, Stefan Behnel, Dag Seljebotn, Greg Ewing, et al.', - author_email='cython-devel@python.org', - description="The Cython compiler for writing C extensions for the Python language.", - long_description=textwrap.dedent("""\ - The Cython language makes writing C extensions for the Python language as - easy as Python itself. Cython is a source code translator based on Pyrex_, - but supports more cutting edge functionality and optimizations. - - The Cython language is a superset of the Python language (almost all Python - code is also valid Cython code), but Cython additionally supports optional - static typing to natively call C functions, operate with C++ classes and - declare fast C types on variables and class attributes. This allows the - compiler to generate very efficient C code from Cython code. - - This makes Cython the ideal language for writing glue code for external - C/C++ libraries, and for fast C modules that speed up the execution of - Python code. - - Note that for one-time builds, e.g. for CI/testing, on platforms that are not - covered by one of the wheel packages provided on PyPI *and* the pure Python wheel - that we provide is not used, it is substantially faster than a full source build - to install an uncompiled (slower) version of Cython with:: - - pip install Cython --install-option="--no-cython-compile" - - .. _Pyrex: http://www.cosc.canterbury.ac.nz/greg.ewing/python/Pyrex/ - """), - license='Apache', - classifiers=[ - dev_status(), - "Intended Audience :: Developers", - "License :: OSI Approved :: Apache Software License", - "Operating System :: OS Independent", - "Programming Language :: Python", - "Programming Language :: Python :: 2", - "Programming Language :: Python :: 2.6", - "Programming Language :: Python :: 2.7", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.4", - "Programming Language :: Python :: 3.5", - "Programming Language :: Python :: 3.6", - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: Implementation :: CPython", - "Programming Language :: Python :: Implementation :: PyPy", - "Programming Language :: C", - "Programming Language :: Cython", - "Topic :: Software Development :: Code Generators", - "Topic :: Software Development :: Compilers", - "Topic :: Software Development :: Libraries :: Python Modules" - ], - - scripts=scripts, - packages=packages, - py_modules=["cython"], - **setup_args -) + +def run_build(): + if compile_cython_itself and (is_cpython or cython_compile_more): + compile_cython_modules(cython_profile, cython_compile_more, cython_with_refnanny) + + from Cython import __version__ as version + setup( + name='Cython', + version=version, + url='https://cython.org/', + author='Robert Bradshaw, Stefan Behnel, Dag Seljebotn, Greg Ewing, et al.', + author_email='cython-devel@python.org', + description="The Cython compiler for writing C extensions for the Python language.", + long_description=textwrap.dedent("""\ + The Cython language makes writing C extensions for the Python language as + easy as Python itself. Cython is a source code translator based on Pyrex_, + but supports more cutting edge functionality and optimizations. + + The Cython language is a superset of the Python language (almost all Python + code is also valid Cython code), but Cython additionally supports optional + static typing to natively call C functions, operate with C++ classes and + declare fast C types on variables and class attributes. This allows the + compiler to generate very efficient C code from Cython code. + + This makes Cython the ideal language for writing glue code for external + C/C++ libraries, and for fast C modules that speed up the execution of + Python code. + + Note that for one-time builds, e.g. for CI/testing, on platforms that are not + covered by one of the wheel packages provided on PyPI *and* the pure Python wheel + that we provide is not used, it is substantially faster than a full source build + to install an uncompiled (slower) version of Cython with:: + + pip install Cython --install-option="--no-cython-compile" + + .. _Pyrex: https://www.cosc.canterbury.ac.nz/greg.ewing/python/Pyrex/ + """), + license='Apache', + classifiers=[ + dev_status(version), + "Intended Audience :: Developers", + "License :: OSI Approved :: Apache Software License", + "Operating System :: OS Independent", + "Programming Language :: Python", + "Programming Language :: Python :: 2", + "Programming Language :: Python :: 2.7", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.4", + "Programming Language :: Python :: 3.5", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python :: Implementation :: PyPy", + "Programming Language :: C", + "Programming Language :: Cython", + "Topic :: Software Development :: Code Generators", + "Topic :: Software Development :: Compilers", + "Topic :: Software Development :: Libraries :: Python Modules" + ], + + scripts=scripts, + packages=packages, + py_modules=["cython"], + **setup_args + ) + + +if __name__ == '__main__': + run_build() diff --git a/test-requirements.txt b/test-requirements.txt index 2cb479eee..8697eff4b 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,3 +1,3 @@ -numpy != 1.19.0 +numpy < 1.19.0 coverage pycodestyle diff --git a/tests/broken/cdefemptysue.pyx b/tests/broken/cdefemptysue.pyx deleted file mode 100644 index d2dda1b25..000000000 --- a/tests/broken/cdefemptysue.pyx +++ /dev/null @@ -1,14 +0,0 @@ -cdef extern from "cdefemptysue.h": - - cdef struct spam: - pass - - ctypedef union eggs: - pass - - cdef enum ham: - pass - -cdef extern spam s -cdef extern eggs e -cdef extern ham h diff --git a/tests/broken/cdefexternblock.pyx b/tests/broken/cdefexternblock.pyx deleted file mode 100644 index 89fca223f..000000000 --- a/tests/broken/cdefexternblock.pyx +++ /dev/null @@ -1,19 +0,0 @@ -cdef extern from "cheese.h": - - ctypedef int camembert - - struct roquefort: - int x - - char *swiss - - void cheddar() - - class external.runny [object runny_obj]: - cdef int a - def __init__(self): - pass - -cdef runny r -r = x -r.a = 42 diff --git a/tests/buffers/bufaccess.pyx b/tests/buffers/bufaccess.pyx index 8761e6eb9..6b0b4ac30 100644 --- a/tests/buffers/bufaccess.pyx +++ b/tests/buffers/bufaccess.pyx @@ -13,8 +13,6 @@ from cpython.object cimport PyObject from cpython.ref cimport Py_INCREF, Py_DECREF cimport cython -__test__ = {} - import sys #import re exclude = []#re.compile('object').search] @@ -27,8 +25,7 @@ if getattr(sys, 'pypy_version_info', None) is not None: def testcase(func): for e in exclude: if e(func.__name__): - return func - __test__[func.__name__] = func.__doc__ + func.__doc__ = "" # disable the test return func @@ -959,6 +956,8 @@ def addref(*args): def decref(*args): for item in args: Py_DECREF(item) +@cython.binding(False) +@cython.always_allow_keywords(False) def get_refcount(x): return (<PyObject*>x).ob_refcnt @@ -991,15 +990,16 @@ def assign_to_object(object[object] buf, int idx, obj): See comments on printbuf_object above. >>> a, b = [1, 2, 3], [4, 5, 6] - >>> get_refcount(a), get_refcount(b) - (2, 2) + >>> rca1, rcb1 = get_refcount(a), get_refcount(b) + >>> rca1 == rcb1 + True >>> addref(a) >>> A = ObjectMockBuffer(None, [1, a]) # 1, ...,otherwise it thinks nested lists... - >>> get_refcount(a), get_refcount(b) - (3, 2) + >>> get_refcount(a) == rca1+1, get_refcount(b) == rcb1 + (True, True) >>> assign_to_object(A, 1, b) - >>> get_refcount(a), get_refcount(b) - (2, 3) + >>> get_refcount(a) == rca1, get_refcount(b) == rcb1+1 + (True, True) >>> decref(b) """ buf[idx] = obj @@ -1010,15 +1010,14 @@ def assign_temporary_to_object(object[object] buf): See comments on printbuf_object above. >>> a, b = [1, 2, 3], {4:23} - >>> get_refcount(a) - 2 + >>> rc1 = get_refcount(a) >>> addref(a) >>> A = ObjectMockBuffer(None, [b, a]) - >>> get_refcount(a) - 3 + >>> get_refcount(a) == rc1+1 + True >>> assign_temporary_to_object(A) - >>> get_refcount(a) - 2 + >>> get_refcount(a) == rc1 + True >>> printbuf_object(A, (2,)) {4: 23} 2 diff --git a/tests/buffers/buffmt.pyx b/tests/buffers/buffmt.pyx index eba10020d..0a9757270 100644 --- a/tests/buffers/buffmt.pyx +++ b/tests/buffers/buffmt.pyx @@ -3,10 +3,6 @@ import struct # Tests buffer format string parsing. -__test__ = {} -def testcase(func): - __test__[func.__name__] = func.__doc__ - return func from libc cimport stdlib @@ -56,7 +52,6 @@ cdef class MockBuffer: info.format = self.format info.itemsize = self.itemsize -@testcase def _int(fmt): """ >>> _int("i") @@ -78,14 +73,12 @@ def _int(fmt): """ cdef object[int] buf = MockBuffer(fmt, sizeof(int)) -@testcase def _ulong(fmt): """ >>> _ulong("L") """ cdef object[unsigned long] buf = MockBuffer(fmt, sizeof(unsigned long)) -@testcase def wrongsize(): """ >>> wrongsize() @@ -96,7 +89,6 @@ def wrongsize(): """ cdef object[float] buf = MockBuffer("f", 1) -@testcase def _obj(fmt): """ >>> _obj("O") @@ -151,7 +143,6 @@ cdef struct UnpackedStruct4: char d int e, f, g -@testcase def char3int(fmt): """ >>> char3int("ciii") @@ -166,7 +157,7 @@ def char3int(fmt): >>> char3int("c3xiii") >>> char3int("cxxxiii") - Standard alignment (assming int size is 4) + Standard alignment (assuming int size is 4) >>> char3int("=c3xiii") >>> char3int("=ciii") Traceback (most recent call last): @@ -185,7 +176,6 @@ def char3int(fmt): cdef object[Char3Int, ndim=1] buf = obj -@testcase def long_string(fmt): """ >>> long_string("90198s") @@ -194,7 +184,6 @@ def long_string(fmt): cdef object[LongString, ndim=1] buf = obj -@testcase def unpacked_struct(fmt): """ Native formats: @@ -218,7 +207,6 @@ def unpacked_struct(fmt): cdef struct ComplexTest: ComplexFloat a, b, c -@testcase def complex_test(fmt): """ >>> complex_test("ZfZfZf") @@ -236,7 +224,6 @@ def complex_test(fmt): cdef object[ComplexTest] buf1 = obj -@testcase def alignment_string(fmt, exc=None): """ >>> alignment_string("@i") @@ -258,7 +245,6 @@ def alignment_string(fmt, exc=None): print "fail" -@testcase def int_and_long_are_same(): """ >>> int_and_long_are_same() @@ -273,7 +259,6 @@ cdef struct MixedComplex: double real float imag -@testcase def mixed_complex_struct(): """ Triggering a specific execution path for this case. @@ -311,7 +296,6 @@ cdef packed struct PartiallyPackedStruct2: char b int c -@testcase def packed_struct(fmt): """ Assuming int is four bytes: @@ -334,7 +318,6 @@ def packed_struct(fmt): """ cdef object[PackedStruct] buf = MockBuffer(fmt, sizeof(PackedStruct)) -@testcase def partially_packed_struct(fmt): """ Assuming int is four bytes: @@ -362,7 +345,6 @@ def partially_packed_struct(fmt): cdef object[PartiallyPackedStruct] buf = MockBuffer( fmt, sizeof(PartiallyPackedStruct)) -@testcase def partially_packed_struct_2(fmt): """ Assuming int is four bytes: @@ -380,7 +362,7 @@ def partially_packed_struct_2(fmt): Traceback (most recent call last): ... ValueError: Buffer dtype mismatch; next field is at offset 8 but 5 expected - + >>> partially_packed_struct_2("ccici") Traceback (most recent call last): ... @@ -398,7 +380,6 @@ cdef packed struct PackedStructWithCharArrays: char[3] d -@testcase def packed_struct_with_strings(fmt): """ >>> packed_struct_with_strings("T{f:a:i:b:5s:c:3s:d:}") @@ -430,7 +411,6 @@ ctypedef struct PackedStructWithNDArrays: float d -@testcase def packed_struct_with_arrays(fmt): """ >>> packed_struct_with_arrays("T{(16)d:a:(16)d:b:d:c:}") @@ -440,7 +420,6 @@ def packed_struct_with_arrays(fmt): fmt, sizeof(PackedStructWithArrays)) -@testcase def unpacked_struct_with_arrays(fmt): """ >>> if struct.calcsize('P') == 8: # 64 bit @@ -453,7 +432,6 @@ def unpacked_struct_with_arrays(fmt): fmt, sizeof(UnpackedStructWithArrays)) -@testcase def packed_struct_with_ndarrays(fmt): """ >>> packed_struct_with_ndarrays("T{d:a:(2,2)d:b:f:c:f:d:}") diff --git a/tests/buffers/mockbuffers.pxi b/tests/buffers/mockbuffers.pxi index 09395cc51..8703b789b 100644 --- a/tests/buffers/mockbuffers.pxi +++ b/tests/buffers/mockbuffers.pxi @@ -1,5 +1,4 @@ from libc cimport stdlib -from libc cimport stdio cimport cpython.buffer import sys @@ -34,7 +33,7 @@ cdef class MockBuffer: cdef Py_ssize_t x, s, cumprod, itemsize self.label = label self.release_ok = True - self.log = "" + self.log = u"" self.offset = offset self.itemsize = itemsize = self.get_itemsize() self.writable = writable @@ -138,7 +137,7 @@ cdef class MockBuffer: self.received_flags.append(name) if flags & cpython.buffer.PyBUF_WRITABLE and not self.writable: - raise BufferError("Writable buffer requested from read-only mock: %s" % ' | '.join(self.received_flags)) + raise BufferError(f"Writable buffer requested from read-only mock: {' | '.join(self.received_flags)}") buffer.buf = <void*>(<char*>self.buffer + (<int>self.offset * self.itemsize)) buffer.obj = self @@ -152,29 +151,29 @@ cdef class MockBuffer: buffer.itemsize = self.itemsize buffer.internal = NULL if self.label: - msg = "acquired %s" % self.label - print msg - self.log += msg + "\n" + msg = f"acquired {self.label}" + print(msg) + self.log += msg + u"\n" def __releasebuffer__(MockBuffer self, Py_buffer* buffer): if buffer.suboffsets != self.suboffsets: self.release_ok = False if self.label: - msg = "released %s" % self.label - print msg - self.log += msg + "\n" + msg = f"released {self.label}" + print(msg) + self.log += msg + u"\n" def printlog(self): - print self.log[:-1] + print(self.log[:-1]) def resetlog(self): - self.log = "" + self.log = u"" cdef int write(self, char* buf, object value) except -1: raise Exception() cdef get_itemsize(self): - print "ERROR, not subclassed", self.__class__ + print(f"ERROR, not subclassed: {self.__class__}") cdef get_default_format(self): - print "ERROR, not subclassed", self.__class__ + print(f"ERROR, not subclassed {self.__class__}") cdef class CharMockBuffer(MockBuffer): cdef int write(self, char* buf, object value) except -1: @@ -246,10 +245,10 @@ cdef class ErrorBuffer: self.label = label def __getbuffer__(ErrorBuffer self, Py_buffer* buffer, int flags): - raise Exception("acquiring %s" % self.label) + raise Exception(f"acquiring {self.label}") def __releasebuffer__(ErrorBuffer self, Py_buffer* buffer): - raise Exception("releasing %s" % self.label) + raise Exception(f"releasing {self.label}") # # Structs @@ -336,13 +335,12 @@ cdef class LongComplexMockBuffer(MockBuffer): def print_offsets(*args, size, newline=True): - sys.stdout.write(' '.join([str(item // size) for item in args])) - if newline: - sys.stdout.write('\n') + sys.stdout.write(' '.join([str(item // size) for item in args]) + ('\n' if newline else '')) def print_int_offsets(*args, newline=True): print_offsets(*args, size=sizeof(int), newline=newline) + shape_5_3_4_list = [[list(range(k * 12 + j * 4, k * 12 + j * 4 + 4)) for j in range(3)] for k in range(5)] diff --git a/tests/buffers/userbuffer.pyx b/tests/buffers/userbuffer.pyx index b9c871970..df774a07b 100644 --- a/tests/buffers/userbuffer.pyx +++ b/tests/buffers/userbuffer.pyx @@ -1,21 +1,11 @@ -import sys -__doc__ = u"" - -if sys.version_info[:2] == (2, 6): - __doc__ += u""" ->>> memoryview = _memoryview -""" - -__doc__ += u""" +__doc__ = u""" >>> b1 = UserBuffer1() >>> m1 = memoryview(b1) >>> m1.tolist() [0, 1, 2, 3, 4] >>> del m1, b1 -""" -__doc__ += u""" >>> b2 = UserBuffer2() >>> m2 = memoryview(b2) UserBuffer2: getbuffer diff --git a/tests/build/cythonize_options.srctree b/tests/build/cythonize_options.srctree index 56e38be61..fcef9645b 100644 --- a/tests/build/cythonize_options.srctree +++ b/tests/build/cythonize_options.srctree @@ -1,3 +1,5 @@ +# mode: run + PYTHON setup.py build_ext --inplace PYTHON -c "import a" @@ -5,11 +7,35 @@ PYTHON -c "import a" from Cython.Build.Dependencies import cythonize +import sys from distutils.core import setup +try: + # io.StringIO cannot handle 'str' in Py2 + from StringIO import StringIO +except ImportError: + from io import StringIO + +old_stderr = sys.stderr +captured = sys.stderr = StringIO() +try: + setup( + ext_modules = cythonize( + "*.pyx", include_path=['subdir'], + compiler_directives={'cdivision': True}, + show_all_warnings=True, + ), + ) + output = sys.stderr.getvalue() +finally: + sys.stderr = old_stderr + sys.stderr.write(captured.getvalue()) + +assert "Unraisable exception in function" in output, output + -setup( - ext_modules = cythonize("*.pyx", include_path=['subdir'], compiler_directives={'cdivision': True}), -) +######## subdir/x.pxd ######## + +######## subdir/y.pxi ######## ######## a.pyx ######## @@ -22,8 +48,6 @@ def mod_int_c(int a, int b): assert mod_int_c(-1, 10) < 0 - -######## subdir/x.pxd ######## - -######## subdir/y.pxi ######## - +# unraisable exceptions should produce a warning +cdef int no_exc_propagate(): + raise TypeError() diff --git a/tests/build/cythonize_pep420_namespace.srctree b/tests/build/cythonize_pep420_namespace.srctree new file mode 100644 index 000000000..99649376a --- /dev/null +++ b/tests/build/cythonize_pep420_namespace.srctree @@ -0,0 +1,59 @@ +PYTHON setup.py build_ext --inplace +PYTHON -c "import runner" + +######## setup.py ######## + +from Cython.Build.Dependencies import cythonize + +from distutils.core import setup, Extension + +setup( + ext_modules=cythonize([ + Extension("nsp.m1.a", ["nsp/m1/a.pyx"]), + Extension("nsp.m2.b", ["nsp/m2/b.pyx"]), + Extension("nsp.m3.c.d", ["nsp/m3/c/d.pyx"]), + ]), +) + +######## nsp/m1/__init__.py ######## + +######## nsp/m1/a.pyx ######## + +cdef class A: + pass + +######## nsp/m1/a.pxd ######## + +cdef class A: + pass + +######## nsp/m2/__init__.py ######## + +######## nsp/m2/b.pyx ######## + +from nsp.m1.a cimport A +from nsp.m3.c.d cimport D + +cdef class B(A): + pass + +######## nsp/m3/__init__.py ######## + +######## nsp/m3/c/d.pyx ######## + +cdef class D: + pass + +######## nsp/m3/c/d.pxd ######## + +cdef class D: + pass + +######## runner.py ######## + +from nsp.m1.a import A +from nsp.m2.b import B +from nsp.m3.c.d import D + +a = A() +b = B() diff --git a/tests/build/cythonize_with_annotate.srctree b/tests/build/cythonize_with_annotate.srctree new file mode 100644 index 000000000..f529d8620 --- /dev/null +++ b/tests/build/cythonize_with_annotate.srctree @@ -0,0 +1,45 @@ +PYTHON setup.py build_ext --inplace +PYTHON -c "import not_annotated; not_annotated.check()" +PYTHON -c "import default_annotated; default_annotated.check()" +PYTHON -c "import fullc_annotated; fullc_annotated.check()" +######## setup.py ######## + +from Cython.Build.Dependencies import cythonize + +from distutils.core import setup + +setup( + ext_modules = cythonize(["not_annotated.pyx"], language_level=3) + + cythonize(["default_annotated.pyx"], annotate=True, language_level=3) + + cythonize(["fullc_annotated.pyx"], annotate='fullc', language_level=3) +) +######## not_annotated.pyx ######## +# check that html-file doesn't exist: +def check(): + import os.path as os_path + assert not os_path.isfile(__name__+'.html') + + + +######## default_annotated.pyx ######## +# load html-site and check that the marker isn't there: +def check(): + from codecs import open + with open(__name__+'.html', 'r', 'utf8') as html_file: + html = html_file.read() + + from Cython.Compiler.Annotate import AnnotationCCodeWriter + assert (AnnotationCCodeWriter.COMPLETE_CODE_TITLE not in html) # per default no complete c code + + + +######## fullc_annotated.pyx ######## +# load html-site and check that the marker is there: +def check(): + from codecs import open + with open(__name__+'.html', 'r', 'utf8') as html_file: + html = html_file.read() + + from Cython.Compiler.Annotate import AnnotationCCodeWriter + assert (AnnotationCCodeWriter.COMPLETE_CODE_TITLE in html) + diff --git a/tests/build/cythonize_with_annotate_via_Options.srctree b/tests/build/cythonize_with_annotate_via_Options.srctree new file mode 100644 index 000000000..f08875506 --- /dev/null +++ b/tests/build/cythonize_with_annotate_via_Options.srctree @@ -0,0 +1,27 @@ +PYTHON setup.py build_ext --inplace +PYTHON -c "import fullc_annotated; fullc_annotated.check()" + +######## setup.py ######## + +from Cython.Build.Dependencies import cythonize +from Cython.Compiler import Options + +Options.annotate = 'fullc' + +from distutils.core import setup + +setup( + ext_modules = cythonize(["fullc_annotated.pyx"], language_level=3) +) + +######## fullc_annotated.pyx ######## +# load html-site and check that the marker is there: + +def check(): + from codecs import open + with open(__name__+'.html', 'r', 'utf8') as html_file: + html = html_file.read() + + from Cython.Compiler.Annotate import AnnotationCCodeWriter + assert (AnnotationCCodeWriter.COMPLETE_CODE_TITLE in html) + diff --git a/tests/build/cythonize_with_annotate_via_cli.srctree b/tests/build/cythonize_with_annotate_via_cli.srctree new file mode 100644 index 000000000..5ca615cd4 --- /dev/null +++ b/tests/build/cythonize_with_annotate_via_cli.srctree @@ -0,0 +1,36 @@ +CYTHONIZE -i -3 not_annotated.pyx +PYTHON -c "import not_annotated; not_annotated.check()" +CYTHONIZE -i -3 --annotate default_annotated.pyx +PYTHON -c "import default_annotated; default_annotated.check()" +CYTHONIZE -i -3 --annotate-fullc fullc_annotated.pyx +PYTHON -c "import fullc_annotated; fullc_annotated.check()" + +######## not_annotated.pyx ######## +# check that html-file doesn't exist: +def check(): + import os.path as os_path + assert not os_path.isfile(__name__+'.html') + + + +######## default_annotated.pyx ######## +# load html-site and check that the marker isn't there: +def check(): + from codecs import open + with open(__name__+'.html', 'r', 'utf8') as html_file: + html = html_file.read() + + from Cython.Compiler.Annotate import AnnotationCCodeWriter + assert (AnnotationCCodeWriter.COMPLETE_CODE_TITLE not in html) # per default no complete c code + + + +######## fullc_annotated.pyx ######## +# load html-site and check that the marker is there: +def check(): + from codecs import open + with open(__name__+'.html', 'r', 'utf8') as html_file: + html = html_file.read() + + from Cython.Compiler.Annotate import AnnotationCCodeWriter + assert (AnnotationCCodeWriter.COMPLETE_CODE_TITLE in html) diff --git a/tests/build/dotted.filename.modules.pxd b/tests/build/dotted.filename.modules.pxd new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/tests/build/dotted.filename.modules.pxd diff --git a/tests/build/dotted.filename.modules.pyx b/tests/build/dotted.filename.modules.pyx new file mode 100644 index 000000000..681cbf6c8 --- /dev/null +++ b/tests/build/dotted.filename.modules.pyx @@ -0,0 +1,7 @@ +# mode: compile +# tag: warnings + +_WARNINGS=""" +1:0: Dotted filenames ('dotted.filename.modules.pxd') are deprecated. Please use the normal Python package directory layout. +1:0: Dotted filenames ('dotted.filename.modules.pyx') are deprecated. Please use the normal Python package directory layout. +""" diff --git a/tests/build/inline_distutils.srctree b/tests/build/inline_distutils.srctree index 7f735796a..d0b21373d 100644 --- a/tests/build/inline_distutils.srctree +++ b/tests/build/inline_distutils.srctree @@ -38,4 +38,4 @@ namespace A { from my_lib cimport x -print x +print(x) diff --git a/tests/compile/branch_hints.pyx b/tests/compile/branch_hints.pyx new file mode 100644 index 000000000..575ee6cba --- /dev/null +++ b/tests/compile/branch_hints.pyx @@ -0,0 +1,91 @@ +# mode: compile +# tag: if, unlikely + +cimport cython + + +@cython.test_assert_path_exists( + "//IfClauseNode", + "//IfClauseNode[not(@branch_hint)]", +) +def if_simple(x): + if x: + x = 2 + + +@cython.test_assert_path_exists( + "//IfClauseNode", + "//IfClauseNode[not(@branch_hint)]", +) +def if_return(x): + if x: + return 1 + raise TypeError() + + +@cython.test_assert_path_exists( + "//IfClauseNode", + "//IfClauseNode[@branch_hint = 'unlikely']", +) +def if_raise_else(x): + if x: + raise TypeError() + else: + return 1 + + +@cython.test_assert_path_exists( + "//IfClauseNode", + "//IfClauseNode[@branch_hint = 'likely']", +) +def if_else_raise(x): + if x: + return 1 + else: + raise TypeError() + + +@cython.test_assert_path_exists( + "//IfClauseNode", + "//IfClauseNode[@branch_hint = 'unlikely']", +) +def if_raise_else_raise(x): + if x: + raise ValueError() + else: + raise TypeError() + + +@cython.test_assert_path_exists( + "//IfClauseNode", + "//IfClauseNode[@branch_hint = 'unlikely']", +) +@cython.test_fail_if_path_exists( + "//IfClauseNode[@branch_hint = 'likely']", + "//IfClauseNode[not(@branch_hint)]", +) +def if_elif_raise_else_raise(x): + if x: + raise ValueError() + elif not x: + raise AttributeError() + else: + raise TypeError() + + +@cython.test_assert_path_exists( + "//IfClauseNode", + "//IfClauseNode[@branch_hint = 'unlikely']", + "//IfClauseNode[@branch_hint = 'unlikely']//GILStatNode", +) +@cython.test_fail_if_path_exists( + "//IfClauseNode[@branch_hint = 'likely']", + "//IfClauseNode[not(@branch_hint)]", +) +cpdef int nogil_if_raise(int x) nogil except -1: + if x: + raise TypeError() + elif not x: + raise ValueError() + else: + x = 2 diff --git a/tests/compile/buildenv.pyx b/tests/compile/buildenv.pyx index ec2445c59..def37de59 100644 --- a/tests/compile/buildenv.pyx +++ b/tests/compile/buildenv.pyx @@ -34,6 +34,7 @@ cdef extern from *: # Cython config cdef int CYTHON_COMPILING_IN_CPYTHON + cdef int CYTHON_COMPILING_IN_LIMITED_API cdef int CYTHON_COMPILING_IN_PYPY cdef int CYTHON_COMPILING_IN_PYSTON cdef int CYTHON_USE_PYLONG_INTERNALS @@ -42,6 +43,7 @@ cdef extern from *: cdef int CYTHON_USE_UNICODE_WRITER cdef int CYTHON_AVOID_BORROWED_REFS cdef int CYTHON_ASSUME_SAFE_MACROS + cdef int CYTHON_USE_TYPE_SLOTS cdef int CYTHON_UNPACK_METHODS cdef int CYTHON_FAST_THREAD_STATE cdef int CYTHON_FAST_PYCALL @@ -76,6 +78,7 @@ Python {sys.version_info} PY_VERSION_HEX 0x{PY_VERSION_HEX:X} CYTHON_COMPILING_IN_CPYTHON {CYTHON_COMPILING_IN_CPYTHON} +CYTHON_COMPILING_IN_LIMITED_API {CYTHON_COMPILING_IN_LIMITED_API} CYTHON_COMPILING_IN_PYPY {CYTHON_COMPILING_IN_PYPY} CYTHON_COMPILING_IN_PYSTON {CYTHON_COMPILING_IN_PYSTON} @@ -85,6 +88,7 @@ CYTHON_USE_UNICODE_INTERNALS {CYTHON_USE_UNICODE_INTERNALS} CYTHON_USE_UNICODE_WRITER {CYTHON_USE_UNICODE_WRITER} CYTHON_AVOID_BORROWED_REFS {CYTHON_AVOID_BORROWED_REFS} CYTHON_ASSUME_SAFE_MACROS {CYTHON_ASSUME_SAFE_MACROS} +CYTHON_USE_TYPE_SLOTS {CYTHON_USE_TYPE_SLOTS} CYTHON_UNPACK_METHODS {CYTHON_UNPACK_METHODS} CYTHON_FAST_THREAD_STATE {CYTHON_FAST_THREAD_STATE} CYTHON_FAST_PYCALL {CYTHON_FAST_PYCALL} @@ -107,6 +111,15 @@ SIZEOF_VOID_P {SIZEOF_VOID_P} ({sizeof(void*)}) SIZEOF_UINTPTR_T {SIZEOF_UINTPTR_T} ({sizeof(unsigned int *)}) SIZEOF_OFF_T {SIZEOF_OFF_T} +Paths: +sys.executable = {sys.executable} +sys.exec_prefix = {sys.exec_prefix} +sys.base_exec_prefix = {getattr(sys, 'base_exec_prefix', "")} +sys.prefix = {sys.prefix} +sys.path = {sys.path} +PYTHONPATH (env) = {get_env('PYTHONPATH', '')} +PYTHONHOME (env) = {get_env('PYTHONHOME', '')} + Distutils: INCDIR = {sysconfig.get_python_inc()} LIBS = {config_var('LIBS')} diff --git a/tests/compile/builtinbuffer.py b/tests/compile/builtinbuffer.py index c18ec1bf7..833fa9954 100644 --- a/tests/compile/builtinbuffer.py +++ b/tests/compile/builtinbuffer.py @@ -4,4 +4,3 @@ import cython @cython.cclass class BuiltinRef: cython.declare(pybuf = 'Py_buffer') - diff --git a/tests/compile/cascmp.pyx b/tests/compile/cascmp.pyx deleted file mode 100644 index c36997fbb..000000000 --- a/tests/compile/cascmp.pyx +++ /dev/null @@ -1,17 +0,0 @@ -# mode: compile - -cdef void foo(): - cdef int bool, int1=0, int2=0, int3=0, int4=0 - cdef object obj1, obj2, obj3, obj4 - obj1 = 1 - obj2 = 2 - obj3 = 3 - obj4 = 4 - bool = int1 < int2 < int3 - bool = obj1 < obj2 < obj3 - bool = int1 < int2 < obj3 - bool = obj1 < 2 < 3 - bool = obj1 < 2 < 3 < 4 - bool = int1 < (int2 == int3) < int4 - -foo() diff --git a/tests/compile/cast_ctypedef_array_T518.pyx b/tests/compile/cast_ctypedef_array_T518.pyx index a62f4cf4c..6a52374c3 100644 --- a/tests/compile/cast_ctypedef_array_T518.pyx +++ b/tests/compile/cast_ctypedef_array_T518.pyx @@ -1,4 +1,4 @@ -# ticket: 518 +# ticket: t518 # mode: compile cdef extern from "cast_ctypedef_array_T518_helper.h": diff --git a/tests/compile/cdefemptysue.pyx b/tests/compile/cdefemptysue.pyx new file mode 100644 index 000000000..1baf67d36 --- /dev/null +++ b/tests/compile/cdefemptysue.pyx @@ -0,0 +1,43 @@ +# mode: compile +# tag: struct, union, enum, cdefextern + +cdef extern from *: + """ + struct spam { int a; }; + struct flat_spam { int a; }; + typedef struct { int a; } flat_spam_type; + + typedef union { int a; long b; } eggs; + typedef union { int a; long b; } flat_eggs; + + enum ham { TOAST }; + enum flat_ham { FLAT_TOAST }; + """ + + cdef struct spam: + pass + + cdef struct flat_spam: pass + + ctypedef struct flat_spam_type: pass + + ctypedef union eggs: + pass + + ctypedef union flat_eggs: pass + + cdef enum ham: + pass + + cdef enum flat_ham: pass + + +cdef extern spam s +cdef extern flat_spam fs +cdef extern flat_spam_type fst + +cdef extern eggs e +cdef extern flat_eggs fe + +cdef extern ham h +cdef extern flat_ham fh diff --git a/tests/compile/cdefexternblock.pyx b/tests/compile/cdefexternblock.pyx new file mode 100644 index 000000000..865a59345 --- /dev/null +++ b/tests/compile/cdefexternblock.pyx @@ -0,0 +1,23 @@ +# mode: compile +# tag: struct, union, enum, cdefextern + +cdef extern from "cheese.h": + + ctypedef int camembert + + struct roquefort: + int x + + char *swiss + + void cheddar() + + # FIXME: find a real declaration here. + #class external.runny [object runny_obj]: + # cdef int a + # def __init__(self): + # pass + + +#cdef runny r = runny() +#r.a = 42 diff --git a/tests/compile/cimport_package_module_T4.pyx b/tests/compile/cimport_package_module_T4.pyx index 28403f122..717f0bf8f 100644 --- a/tests/compile/cimport_package_module_T4.pyx +++ b/tests/compile/cimport_package_module_T4.pyx @@ -1,4 +1,4 @@ -# ticket: 4 +# ticket: t4 # mode: compile from a cimport b diff --git a/tests/compile/cimportfrom_T248.pyx b/tests/compile/cimportfrom_T248.pyx index 13796baf1..1be252f3c 100644 --- a/tests/compile/cimportfrom_T248.pyx +++ b/tests/compile/cimportfrom_T248.pyx @@ -1,4 +1,4 @@ -# ticket: 248 +# ticket: t248 # mode: compile from ewing8 cimport (Foo, diff --git a/tests/compile/complex_annotations.pyx b/tests/compile/complex_annotations.pyx new file mode 100644 index 000000000..5d95611ac --- /dev/null +++ b/tests/compile/complex_annotations.pyx @@ -0,0 +1,7 @@ +# mode: compile + +# Complex numbers defined in annotations weren't having their utility code imported directly +# leading to compile-errors that the type wasn't defined. The test is intentionally minimal since +# anything more thorough ends up creating the utility code +cdef f(x: complex): + pass diff --git a/tests/compile/complex_decorators.pyx b/tests/compile/complex_decorators.pyx new file mode 100644 index 000000000..e8d65244d --- /dev/null +++ b/tests/compile/complex_decorators.pyx @@ -0,0 +1,10 @@ +# mode: compile + +cimport cython + +# Complex numbers defined in "cython.locals" weren't having their utility code imported directly +# leading to compile-errors that the type wasn't defined. The test is intentionally minimal since +# anything more thorough ends up creating the utility code +@cython.locals(x=complex) +cdef f(x): + pass diff --git a/tests/compile/const_decl.pyx b/tests/compile/const_decl.pyx index 5424c8f90..c47f952df 100644 --- a/tests/compile/const_decl.pyx +++ b/tests/compile/const_decl.pyx @@ -1,12 +1,17 @@ # mode: compile -cdef const_args(const int a, const int *b, const (int*) c, int *const d): +cdef const_args(const int a, const int *b, const (int*) c, int *const d, int **const e, int *const *f): print a print b[0] - b = NULL # OK, the pointer itself is not const - c[0] = 4 # OK, the value is not const - d[0] = 7 # OK, the value is not const + b = NULL # OK, the pointer itself is not const + c[0] = 4 # OK, the value is not const + d[0] = 7 # OK, the value is not const + e[0][0] = 1 # OK, the value is not const + e[0] = NULL # OK, the pointed pointer is not const + f[0][0] = 1 # OK, the value is not const + f = NULL # OK, the pointer is not const def call_const_args(x): cdef int k = x - const_args(x, &k, &k, &k) + cdef int* arr = [x] + const_args(x, &k, &k, &k, &arr, &arr) diff --git a/tests/compile/cpp_enums.h b/tests/compile/cpp_enums.h deleted file mode 100644 index 4ea444ee4..000000000 --- a/tests/compile/cpp_enums.h +++ /dev/null @@ -1,11 +0,0 @@ -enum Enum1 { - Item1, - Item2 -}; - -namespace Namespace1 { - enum Enum2 { - Item3, - Item4 - }; -} diff --git a/tests/compile/cpp_enums.pyx b/tests/compile/cpp_enums.pyx deleted file mode 100644 index b738dee05..000000000 --- a/tests/compile/cpp_enums.pyx +++ /dev/null @@ -1,27 +0,0 @@ -# tag: cpp -# mode: compile - -cdef extern from "cpp_enums.h": - cdef enum Enum1: - Item1 - Item2 - -a = Item1 -b = Item2 - -cdef Enum1 x, y -x = Item1 -y = Item2 - -cdef extern from "cpp_enums.h" namespace "Namespace1": - cdef enum Enum2: - Item3 - Item4 - -c = Item3 -d = Item4 - -cdef Enum2 z, w -z = Item3 -w = Item4 - diff --git a/tests/compile/cpp_rvalue_reference_binding.pyx b/tests/compile/cpp_rvalue_reference_binding.pyx new file mode 100644 index 000000000..492a950ac --- /dev/null +++ b/tests/compile/cpp_rvalue_reference_binding.pyx @@ -0,0 +1,22 @@ +# tag: cpp, cpp11 +# mode: compile + +cdef extern from *: + """ + template <typename T> + void accept(T&& x) { (void) x; } + """ + cdef void accept[T](T&& x) + +cdef int make_int_py() except *: + # might raise Python exception (thus needs a temp) + return 1 + +cdef int make_int_cpp() except +: + # might raise C++ exception (thus needs a temp) + return 1 + +def test_func_arg(): + # won't compile if move() isn't called on the temp: + accept(make_int_py()) + accept(make_int_cpp()) diff --git a/tests/compile/cpp_temp_assignment.pyx b/tests/compile/cpp_temp_assignment.pyx new file mode 100644 index 000000000..40a2cc671 --- /dev/null +++ b/tests/compile/cpp_temp_assignment.pyx @@ -0,0 +1,65 @@ +# tag: cpp +# mode: compile + +cdef extern from *: + """ + #if __cplusplus >= 201103L + class NoAssign { + public: + NoAssign() {} + NoAssign(NoAssign&) = delete; + NoAssign(NoAssign&&) {} + NoAssign& operator=(NoAssign&) = delete; + NoAssign& operator=(NoAssign&&) { return *this; } + void func() {} + }; + #else + // the test becomes meaningless + // (but just declare something to ensure it passes) + class NoAssign { + public: + void func() {} + }; + #endif + + NoAssign get_NoAssign_Py() { + return NoAssign(); + } + NoAssign get_NoAssign_Cpp() { + return NoAssign(); + } + """ + cdef cppclass NoAssign: + void func() + + # might raise Python exception (thus needs a temp) + NoAssign get_NoAssign_Py() except * + # might raise C++ exception (thus needs a temp) + NoAssign get_NoAssign_Cpp() except + + +cdef internal_cpp_func(NoAssign arg): + pass + +def test_call_to_function(): + # will fail to compile if move constructors aren't used + internal_cpp_func(get_NoAssign_Py()) + internal_cpp_func(get_NoAssign_Cpp()) + +def test_assignment_to_name(): + # will fail if move constructors aren't used + cdef NoAssign value + value = get_NoAssign_Py() + value = get_NoAssign_Cpp() + +def test_assignment_to_scope(): + cdef NoAssign value + value = get_NoAssign_Py() + value = get_NoAssign_Cpp() + def inner(): + value.func() + +cdef class AssignToClassAttr: + cdef NoAssign attr + def __init__(self): + self.attr = get_NoAssign_Py() + self.attr = get_NoAssign_Cpp() diff --git a/tests/compile/cpp_templates.pyx b/tests/compile/cpp_templates.pyx index cab981e38..7952a1610 100644 --- a/tests/compile/cpp_templates.pyx +++ b/tests/compile/cpp_templates.pyx @@ -1,6 +1,6 @@ # tag: cpp # mode: compile -# ticket: 767 +# ticket: t767 cdef extern from "templates.h": cdef cppclass TemplateTest1[T]: diff --git a/tests/compile/cppenum.pyx b/tests/compile/cppenum.pyx new file mode 100644 index 000000000..8431ac83b --- /dev/null +++ b/tests/compile/cppenum.pyx @@ -0,0 +1,31 @@ +# mode: compile +# tag: cpp,cpp11 + + +cpdef enum class Spam: + a, b + c + d + e + f = 42 + + +cpdef enum class Cheese(unsigned int): + x = 1 + y = 2 + + +cdef enum struct parrot_state: + alive = 1 + dead = 0 + + +cdef void eggs(): + cdef Spam s1 + s1 = Spam.a + s2 = Spam.b + + cdef Cheese c1 + c1 = Cheese.x + +eggs() diff --git a/tests/compile/crunchytype.h b/tests/compile/crunchytype.h deleted file mode 100644 index 6ea0e37c0..000000000 --- a/tests/compile/crunchytype.h +++ /dev/null @@ -1,5 +0,0 @@ - -struct CrunchyType { - int number; - PyObject* string; -}; diff --git a/tests/compile/crunchytype.pxd b/tests/compile/crunchytype.pxd index c03e38dad..c1f2b555b 100644 --- a/tests/compile/crunchytype.pxd +++ b/tests/compile/crunchytype.pxd @@ -1,4 +1,10 @@ -cdef extern from "crunchytype.h": +cdef extern from *: + """ + struct CrunchyType { + int number; + PyObject* string; + }; + """ cdef class crunchytype.Crunchy [ object CrunchyType ]: cdef int number cdef object string diff --git a/tests/compile/ctypedef_public_class_T355.pyx b/tests/compile/ctypedef_public_class_T355.pyx index 505f9d67f..ceb2d65e4 100644 --- a/tests/compile/ctypedef_public_class_T355.pyx +++ b/tests/compile/ctypedef_public_class_T355.pyx @@ -1,4 +1,4 @@ -# ticket: 355 +# ticket: t355 # mode: compile ctypedef public class Time [type MyTime_Type, object MyTimeObject]: diff --git a/tests/compile/cython_compiled_folding.pxd b/tests/compile/cython_compiled_folding.pxd new file mode 100644 index 000000000..4c85d4311 --- /dev/null +++ b/tests/compile/cython_compiled_folding.pxd @@ -0,0 +1 @@ +from libc.math cimport sin, cos, sqrt, tan, log diff --git a/tests/compile/cython_compiled_folding.py b/tests/compile/cython_compiled_folding.py new file mode 100644 index 000000000..047b61689 --- /dev/null +++ b/tests/compile/cython_compiled_folding.py @@ -0,0 +1,39 @@ +# mode: compile + +# libc sin, cos and sqrt cimported in the pxd file + +import cython +from cython import compiled + +if not cython.compiled: + from math import sin + +if cython.compiled: + pass +else: + from math import cos + +if "aa" == "bb": + pass +elif cython.compiled: + pass +elif True: + from math import sqrt + +if "aa" == "bb": + pass +elif compiled: + pass +else: + from math import tan + +# log10 isn't defined in the pxd file +from math import log10 + +@cython.test_fail_if_path_exists("//FromImportStatNode//ImportNode") +@cython.test_assert_path_exists("//AddNode") +def import_log(x, y): + if compiled: + return x+y + else: + from math import log diff --git a/tests/compile/ellipsis_T488.pyx b/tests/compile/ellipsis_T488.pyx index d90d21634..804b18497 100644 --- a/tests/compile/ellipsis_T488.pyx +++ b/tests/compile/ellipsis_T488.pyx @@ -1,4 +1,4 @@ -# ticket: 488 +# ticket: t488 # mode: compile #from ... import foo diff --git a/tests/compile/find_pxd.srctree b/tests/compile/find_pxd.srctree index 75d2765c9..23fc5c5c7 100644 --- a/tests/compile/find_pxd.srctree +++ b/tests/compile/find_pxd.srctree @@ -41,7 +41,7 @@ ctypedef int my_type ######## path/numpy/__init__.pxd ######## -# gh-2905: This should be found before Cython/Inlude/numpy/__init__.pxd +# gh-2905: This should be found before Cython/Includes/numpy/__init__.pxd ctypedef int my_type diff --git a/tests/compile/fused_redeclare_T3111.pyx b/tests/compile/fused_redeclare_T3111.pyx index baf932bd4..80e211fad 100644 --- a/tests/compile/fused_redeclare_T3111.pyx +++ b/tests/compile/fused_redeclare_T3111.pyx @@ -22,10 +22,14 @@ def foo(dtype_t[:] a, dtype_t_out[:, :] b): # "__pyxutil:16:4: '___pyx_npy_uint8' redeclared". The remaining warnings are # unrelated to this test. _WARNINGS = """ -20:10: 'cpdef_method' redeclared -31:10: 'cpdef_cname_method' redeclared -446:72: Argument evaluation order in C function call is undefined and may not be as expected -446:72: Argument evaluation order in C function call is undefined and may not be as expected -749:34: Argument evaluation order in C function call is undefined and may not be as expected -749:34: Argument evaluation order in C function call is undefined and may not be as expected +# cpdef redeclaration bug, from TestCythonScope.pyx +25:10: 'cpdef_method' redeclared +36:10: 'cpdef_cname_method' redeclared +# from MemoryView.pyx +987:29: Ambiguous exception value, same as default return value: 0 +987:29: Ambiguous exception value, same as default return value: 0 +1014:46: Ambiguous exception value, same as default return value: 0 +1014:46: Ambiguous exception value, same as default return value: 0 +1104:29: Ambiguous exception value, same as default return value: 0 +1104:29: Ambiguous exception value, same as default return value: 0 """ diff --git a/tests/compile/fused_wraparound.pyx b/tests/compile/fused_wraparound.pyx new file mode 100644 index 000000000..e9facf9c1 --- /dev/null +++ b/tests/compile/fused_wraparound.pyx @@ -0,0 +1,22 @@ +# mode: compile +# tag: fused, werror + +""" +Very short test for https://github.com/cython/cython/issues/3492 +(Needs its own file since werror fails on the main fused-test files) +wraparound and boundscheck directives shouldn't break the fused +dispatcher function +""" + +cimport cython + +ctypedef fused fused_t: + str + int + long + complex + +@cython.wraparound(False) +@cython.boundscheck(False) +def func(fused_t a, cython.floating b): + return a, b diff --git a/tests/compile/extern_packed_struct.pyx b/tests/compile/packed_structs.pyx index 6df4d20d0..6e722b46d 100644 --- a/tests/compile/extern_packed_struct.pyx +++ b/tests/compile/packed_structs.pyx @@ -3,3 +3,8 @@ cdef extern from *: cdef packed struct MyStruct: char a + +cdef public packed struct PublicStruct: + int a + unsigned char b + int c diff --git a/tests/compile/tree_assertions.pyx b/tests/compile/tree_assertions.pyx new file mode 100644 index 000000000..e311bfd25 --- /dev/null +++ b/tests/compile/tree_assertions.pyx @@ -0,0 +1,20 @@ +# mode: compile + +# This is a sort of meta test - to test the functionality of "test_assert_path_exists" + +cimport cython + +@cython.test_assert_path_exists("//ReturnStatNode") +def not_in_inner_compiler_directives(): + # used to fail because ReturnStatNode wasn't in *this* CompilerDirectivesNode + with cython.boundscheck(False): + pass + return 1 # should pass + +@cython.test_assert_path_exists("//ReturnStatNode") +def in_inner_compiler_directives(): + # used to fail because ReturnStatNode wasn't in *this* CompilerDirectivesNode + with cython.boundscheck(False): + return 1 + +# it's hard to come up with a corresponding test for fail_if_path_exists.. diff --git a/tests/compile/volatile.pyx b/tests/compile/volatile.pyx new file mode 100644 index 000000000..d69d8b355 --- /dev/null +++ b/tests/compile/volatile.pyx @@ -0,0 +1,17 @@ +# mode: compile + +cdef volatile int x = 1 + +cdef const volatile char* greeting1 = "hello world" +cdef volatile const char* greeting2 = "goodbye" + + +cdef extern from "stdlib.h": + volatile void* malloc(size_t) + +cdef volatile long* test(volatile size_t s): + cdef volatile long* arr = <long*><volatile long*>malloc(s) + return arr + + +test(64) diff --git a/tests/compile/weakref_T276.pyx b/tests/compile/weakref_T276.pyx index d56f67303..7fbc52819 100644 --- a/tests/compile/weakref_T276.pyx +++ b/tests/compile/weakref_T276.pyx @@ -1,4 +1,4 @@ -# ticket: 276 +# ticket: t276 # mode: compile __doc__ = u""" diff --git a/tests/errors/bufaccess_noassignT444.pyx b/tests/errors/bufaccess_noassignT444.pyx index 44d2ebd0b..c618106fb 100644 --- a/tests/errors/bufaccess_noassignT444.pyx +++ b/tests/errors/bufaccess_noassignT444.pyx @@ -1,4 +1,4 @@ -# ticket: 444 +# ticket: t444 # mode: error def test(): diff --git a/tests/errors/buffertypedef_T117.pyx b/tests/errors/buffertypedef_T117.pyx index d88a3895b..cefa79939 100644 --- a/tests/errors/buffertypedef_T117.pyx +++ b/tests/errors/buffertypedef_T117.pyx @@ -1,4 +1,4 @@ -# ticket: 117 +# ticket: t117 # mode: error ctypedef object[float] mybuffer diff --git a/tests/errors/callingnonexisting_T307.pyx b/tests/errors/callingnonexisting_T307.pyx index e9409fcab..ac767e581 100644 --- a/tests/errors/callingnonexisting_T307.pyx +++ b/tests/errors/callingnonexisting_T307.pyx @@ -1,4 +1,4 @@ -# ticket: 307 +# ticket: t307 # mode: error nonexisting(3, with_kw_arg=4) diff --git a/tests/errors/cdef_class_properties_decorated.pyx b/tests/errors/cdef_class_properties_decorated.pyx index 7485d8891..f796b7128 100644 --- a/tests/errors/cdef_class_properties_decorated.pyx +++ b/tests/errors/cdef_class_properties_decorated.pyx @@ -1,5 +1,5 @@ # mode: error -# ticket: 264 +# ticket: t264 # tag: property, decorator diff --git a/tests/errors/cdef_members_T517.pyx b/tests/errors/cdef_members_T517.pyx index 9bd3b111c..34bfd79fd 100644 --- a/tests/errors/cdef_members_T517.pyx +++ b/tests/errors/cdef_members_T517.pyx @@ -1,4 +1,4 @@ -# ticket: 517 +# ticket: t517 # mode: error ctypedef void* VoidP diff --git a/tests/errors/cdefkwargs.pyx b/tests/errors/cdefkwargs.pyx index a4477e2da..2fedeb04a 100644 --- a/tests/errors/cdefkwargs.pyx +++ b/tests/errors/cdefkwargs.pyx @@ -6,10 +6,6 @@ __doc__ = u""" >>> call4() """ -import sys, re -if sys.version_info >= (2,6): - __doc__ = re.sub(u"Error: (.*)exactly(.*)", u"Error: \\1at most\\2", __doc__) - # the calls: def call2(): diff --git a/tests/errors/cfunc_directive_in_pyclass.pyx b/tests/errors/cfunc_directive_in_pyclass.pyx index bb12dc271..d6b373a40 100644 --- a/tests/errors/cfunc_directive_in_pyclass.pyx +++ b/tests/errors/cfunc_directive_in_pyclass.pyx @@ -7,5 +7,5 @@ class Pyclass(object): pass _ERRORS = """ - 6:4: cfunc directive is not allowed here + 5:4: cfunc directive is not allowed here """ diff --git a/tests/errors/cimport_attributes.pyx b/tests/errors/cimport_attributes.pyx index 53e303a42..7037b9aa3 100644 --- a/tests/errors/cimport_attributes.pyx +++ b/tests/errors/cimport_attributes.pyx @@ -1,4 +1,5 @@ # mode: error +# tag: cpp cimport libcpp @@ -24,8 +25,8 @@ print my_map_with_shadow.python_attribute # OK (if such a module existed at ru _ERRORS = u""" -5:12: cimported module has no attribute 'no_such_attribute' -8:16: cimported module has no attribute 'no_such_attribute' -11:12: cimported module has no attribute 'no_such_attribute' -14:15: cimported module has no attribute 'no_such_attribute' +6:12: cimported module has no attribute 'no_such_attribute' +9:16: cimported module has no attribute 'no_such_attribute' +12:12: cimported module has no attribute 'no_such_attribute' +15:15: cimported module has no attribute 'no_such_attribute' """ diff --git a/tests/errors/compile_time_unraisable_T370.pyx b/tests/errors/compile_time_unraisable_T370.pyx index 963f280fb..d22f0c6c0 100644 --- a/tests/errors/compile_time_unraisable_T370.pyx +++ b/tests/errors/compile_time_unraisable_T370.pyx @@ -1,4 +1,4 @@ -# ticket: 370 +# ticket: t370 # mode: error cdef int raiseit(): diff --git a/tests/errors/const_decl_errors.pyx b/tests/errors/const_decl_errors.pyx index 459480adb..29c9896ea 100644 --- a/tests/errors/const_decl_errors.pyx +++ b/tests/errors/const_decl_errors.pyx @@ -10,23 +10,34 @@ cdef const int x = 10 cdef struct S: int member -cdef func(const int a, const int* b, const (int*) c, const S s, int *const d, +cdef func(const int a, const int* b, const (int*) c, const S s, int *const d, int **const e, int *const *f, const S *const t): a = 10 c = NULL b[0] = 100 s.member = 1000 d = NULL + e[0][0] = 1 # ok + e[0] = NULL # ok + e = NULL # nok + f[0][0] = 1 # ok + f[0] = NULL # nok + f = NULL # ok t = &s +cdef volatile object v + _ERRORS = """ -3:5: Const base type cannot be a Python object +3:5: Const/volatile base type cannot be a Python object 8:5: Assignment to const 'x' 15:4: Assignment to const 'a' 16:4: Assignment to const 'c' 17:5: Assignment to const dereference 18:5: Assignment to const attribute 'member' 19:4: Assignment to const 'd' -20:4: Assignment to const 't' +22:4: Assignment to const 'e' +24:5: Assignment to const dereference +26:4: Assignment to const 't' +28:5: Const/volatile base type cannot be a Python object """ diff --git a/tests/errors/cpp_enum_redeclare.pyx b/tests/errors/cpp_enum_redeclare.pyx new file mode 100644 index 000000000..3d2ae7763 --- /dev/null +++ b/tests/errors/cpp_enum_redeclare.pyx @@ -0,0 +1,13 @@ +# mode: error +# tag: cpp + +cdef enum class Spam: + a + +cdef enum class Spam: + b + +_ERRORS=""" +7:5: 'Spam' redeclared +4:5: Previous declaration is here +""" diff --git a/tests/errors/cpp_object_template.pyx b/tests/errors/cpp_object_template.pyx index a90bdedff..93d6c28da 100644 --- a/tests/errors/cpp_object_template.pyx +++ b/tests/errors/cpp_object_template.pyx @@ -1,4 +1,5 @@ # mode: error +# tag: cpp from libcpp.vector cimport vector @@ -12,6 +13,6 @@ def main(): va.push_back(A()) _ERRORS = u""" -9:16: Python object type 'Python object' cannot be used as a template argument -11:16: Python object type 'A' cannot be used as a template argument +10:16: Python object type 'Python object' cannot be used as a template argument +12:16: Python object type 'A' cannot be used as a template argument """ diff --git a/tests/errors/cpp_rvalue_reference_support.pyx b/tests/errors/cpp_rvalue_reference_support.pyx new file mode 100644 index 000000000..e1e7015c9 --- /dev/null +++ b/tests/errors/cpp_rvalue_reference_support.pyx @@ -0,0 +1,32 @@ +# mode: error +# tag: werror, cpp, cpp11 + +# These tests check for unsupported use of rvalue-references (&&) +# and should be removed or cleaned up when support is added. + +cdef int&& x + +cdef void foo(int&& x): + pass + +cdef int&& bar(): + pass + +cdef extern from *: + """ + void baz(int x, int&& y) {} + + template <typename T> + void qux(const T&& x) {} + """ + cdef void baz(int x, int&& y) + cdef void qux[T](const T&& x) + + +_ERRORS=""" +7:8: C++ rvalue-references cannot be declared +9:13: Rvalue-reference as function argument not supported +12:14: Rvalue-reference as function return type not supported +22:17: Rvalue-reference as function argument not supported +23:20: Rvalue-reference as function argument not supported +""" diff --git a/tests/errors/declareafteruse_T158.pyx b/tests/errors/declareafteruse_T158.pyx index 34e5f097e..a6ff6da13 100644 --- a/tests/errors/declareafteruse_T158.pyx +++ b/tests/errors/declareafteruse_T158.pyx @@ -1,4 +1,4 @@ -# ticket: 158 +# ticket: t158 # mode: error def mult_decl_test(): @@ -52,26 +52,19 @@ cdef int *baz print var[0][0] cdef unsigned long long[100][100] var -# in 0.11.1 these are warnings -FUTURE_ERRORS = u""" -6:13: cdef variable 's' declared after it is used -6:16: cdef variable 'vv' declared after it is used -11:14: cdef variable 'i' declared after it is used -17:14: cdef variable 'i' declared after it is used -23:14: cdef variable 'i' declared after it is used -26:9: cdef variable 's' declared after it is used -32:17: cdef variable 't' declared after it is used -36:13: cdef variable 'r' declared after it is used -42:17: cdef variable 't' declared after it is used -49:10: cdef variable 'baz' declared after it is used -52:24: cdef variable 'var' declared after it is used -""" - -syntax error - _ERRORS = u""" -42:17: cdef variable 't' declared after it is used -49:10: cdef variable 'baz' declared after it is used -52:24: cdef variable 'var' declared after it is used -70:7: Syntax error in simple statement list +5:17: local variable 'vv' referenced before assignment +6:17: local variable 's' referenced before assignment +7:13: cdef variable 's' declared after it is used +7:16: cdef variable 'vv' declared after it is used +12:14: cdef variable 'i' declared after it is used +18:14: cdef variable 'i' declared after it is used +24:14: cdef variable 'i' declared after it is used +27:9: cdef variable 's' declared after it is used +33:17: cdef variable 't' declared after it is used +43:17: cdef variable 't' declared after it is used +50:10: cdef variable 'baz' declared after it is used +53:34: cdef variable 'var' declared after it is used """ +# FIXME not detected +#37:13: cdef variable 'r' declared after it is used diff --git a/tests/errors/duplicate_const.pyx b/tests/errors/duplicate_const.pyx new file mode 100644 index 000000000..1367dd13b --- /dev/null +++ b/tests/errors/duplicate_const.pyx @@ -0,0 +1,13 @@ +# mode: error + +cdef extern from *: + cdef const const int a + cdef const volatile int b + cdef volatile const int c + cdef volatile volatile int d + + +_ERRORS = """ +4:9: Duplicate 'const' +7:9: Duplicate 'volatile' +""" diff --git a/tests/errors/e2_packedstruct_T290.pyx b/tests/errors/e2_packedstruct_T290.pyx index 1a0f40f18..084b36d71 100644 --- a/tests/errors/e2_packedstruct_T290.pyx +++ b/tests/errors/e2_packedstruct_T290.pyx @@ -1,4 +1,4 @@ -# ticket: 290 +# ticket: t290 # mode: error cdef packed foo: diff --git a/tests/errors/e_assert.pyx b/tests/errors/e_assert.pyx new file mode 100644 index 000000000..616d86ff1 --- /dev/null +++ b/tests/errors/e_assert.pyx @@ -0,0 +1,25 @@ +# mode: error +# tag: assert + +def nontrivial_assert_in_nogil(int a, obj): + with nogil: + # NOK + assert obj + assert a*obj + assert obj, "abc" + + # OK + assert a + assert a*a + assert a, "abc" + assert a, u"abc" + assert a, f"123{a}xyz" + + +_ERRORS = """ +7:15: Truth-testing Python object not allowed without gil +8:15: Converting to Python object not allowed without gil +8:16: Operation not allowed without gil +8:16: Truth-testing Python object not allowed without gil +9:15: Truth-testing Python object not allowed without gil +""" diff --git a/tests/errors/e_autotestdict.pyx b/tests/errors/e_autotestdict.pyx index a5d56f3d2..097b09fb2 100644 --- a/tests/errors/e_autotestdict.pyx +++ b/tests/errors/e_autotestdict.pyx @@ -7,5 +7,5 @@ def foo(): pass _ERRORS = u""" -6:0: The autotestdict compiler directive is not allowed in function scope +5:0: The autotestdict compiler directive is not allowed in function scope """ diff --git a/tests/errors/e_binop_and.pyx b/tests/errors/e_binop_and.pyx new file mode 100644 index 000000000..195677bdf --- /dev/null +++ b/tests/errors/e_binop_and.pyx @@ -0,0 +1,14 @@ +# mode: error +# tag: and, binop, warnings + +def test_and(a, b): + return a && b + + +_WARNINGS = """ +5:13: Found the C operator '&&', did you mean the Python operator 'and'? +""" + +_ERRORS = """ +5:13: Syntax error in simple statement list +""" diff --git a/tests/errors/e_binop_or.pyx b/tests/errors/e_binop_or.pyx new file mode 100644 index 000000000..1e4109244 --- /dev/null +++ b/tests/errors/e_binop_or.pyx @@ -0,0 +1,14 @@ +# mode: error +# tag: or, binop, warnings + +def test_or(a, b): + return a || b + + +_WARNINGS = """ +5:13: Found the C operator '||', did you mean the Python operator 'or'? +""" + +_ERRORS = """ +5:13: Syntax error in simple statement list +""" diff --git a/tests/errors/e_cdef_keywords_T241.pyx b/tests/errors/e_cdef_keywords_T241.pyx index 87524ebdd..28a2783fe 100644 --- a/tests/errors/e_cdef_keywords_T241.pyx +++ b/tests/errors/e_cdef_keywords_T241.pyx @@ -1,4 +1,4 @@ -# ticket: 241 +# ticket: t241 # mode: error cdef some_function(x, y): diff --git a/tests/errors/e_cdefemptysue.pyx b/tests/errors/e_cdefemptysue.pyx index f71370f4a..7f218b1fe 100644 --- a/tests/errors/e_cdefemptysue.pyx +++ b/tests/errors/e_cdefemptysue.pyx @@ -8,8 +8,21 @@ ctypedef union eggs: cdef enum ham: pass + + +cdef struct flat_spam: pass + +ctypedef union flat_eggs: pass + +cdef enum flat_ham: pass + + _ERRORS = u""" 3:5: Empty struct or union definition not allowed outside a 'cdef extern from' block 6:0: Empty struct or union definition not allowed outside a 'cdef extern from' block 9:5: Empty enum definition not allowed outside a 'cdef extern from' block + +13:5: Empty struct or union definition not allowed outside a 'cdef extern from' block +15:0: Empty struct or union definition not allowed outside a 'cdef extern from' block +17:5: Empty enum definition not allowed outside a 'cdef extern from' block """ diff --git a/tests/errors/e_cenum_with_type.pyx b/tests/errors/e_cenum_with_type.pyx new file mode 100644 index 000000000..43c0e08fc --- /dev/null +++ b/tests/errors/e_cenum_with_type.pyx @@ -0,0 +1,8 @@ +# mode: error + +cdef enum Spam(int): + a, b + +_ERRORS = u""" +3:14: Expected ':', found '(' +""" diff --git a/tests/errors/e_cpp_references.pyx b/tests/errors/e_cpp_references.pyx new file mode 100644 index 000000000..39ace0a13 --- /dev/null +++ b/tests/errors/e_cpp_references.pyx @@ -0,0 +1,10 @@ +# mode: error +# tag: cpp, cpp11 + +cdef foo(object& x): pass +cdef bar(object&& x): pass + +_ERRORS=""" +4:15: Reference base type cannot be a Python object +5:15: Rvalue-reference base type cannot be a Python object +""" diff --git a/tests/errors/e_decorators.pyx b/tests/errors/e_decorators.pyx index 5abc1fc29..33ef2355d 100644 --- a/tests/errors/e_decorators.pyx +++ b/tests/errors/e_decorators.pyx @@ -1,13 +1,12 @@ # mode: error -_ERRORS = u""" -4:4 Expected a newline after decorator -""" - - class A: pass @A().a def f(): pass + +_ERRORS = u""" +6:4: Expected a newline after decorator +""" diff --git a/tests/errors/e_int_literals_py2.py b/tests/errors/e_int_literals_py2.py index 9eef36673..750214e5e 100644 --- a/tests/errors/e_int_literals_py2.py +++ b/tests/errors/e_int_literals_py2.py @@ -3,7 +3,7 @@ def int_literals(): a = 1L # ok - b = 10000000000000L # ok + b = 10000000000000L # ok c = 1UL d = 10000000000000UL e = 10000000000000LL diff --git a/tests/errors/e_tuple_args_T692.py b/tests/errors/e_tuple_args_T692.py index 715433d01..3ca30519b 100644 --- a/tests/errors/e_tuple_args_T692.py +++ b/tests/errors/e_tuple_args_T692.py @@ -1,4 +1,4 @@ -# ticket: 692 +# ticket: t692 # mode: error def func((a, b)): @@ -9,4 +9,3 @@ _ERRORS = u""" 5:11: undeclared name not builtin: a 5:15: undeclared name not builtin: b """ - diff --git a/tests/errors/fused_types.pyx b/tests/errors/fused_types.pyx index 438e46584..378ac5506 100644 --- a/tests/errors/fused_types.pyx +++ b/tests/errors/fused_types.pyx @@ -1,4 +1,5 @@ # mode: error +# ticket: 1772 cimport cython from cython import fused_type @@ -48,6 +49,18 @@ def outer(cython.floating f): def inner(): cdef cython.floating g + +# Mixing const and non-const type makes fused type ambiguous +cdef fused mix_const_t: + int + const int + +cdef cdef_func_with_mix_const_type(mix_const_t val): + print(val) + +cdef_func_with_mix_const_type(1) + + # This is all valid dtype5 = fused_type(int, long, float) dtype6 = cython.fused_type(int, long) @@ -63,18 +76,40 @@ ctypedef fused fused2: func(x, y) +cdef floating return_type_unfindable1(cython.integral x): + return 1.0 + +cpdef floating return_type_unfindable2(cython.integral x): + return 1.0 + +cdef void contents_unfindable1(cython.integral x): + z: floating = 1 # note: cdef variables also fail with an error but not by the time this test aborts + sz = sizeof(floating) + _ERRORS = u""" -10:15: fused_type does not take keyword arguments -15:33: Type specified multiple times -26:0: Invalid use of fused types, type cannot be specialized -26:4: Not enough types specified to specialize the function, int2_t is still fused +11:15: fused_type does not take keyword arguments +16:33: Type specified multiple times 27:0: Invalid use of fused types, type cannot be specialized 27:4: Not enough types specified to specialize the function, int2_t is still fused -28:16: Call with wrong number of arguments (expected 2, got 1) -29:16: Call with wrong number of arguments (expected 2, got 3) -36:6: Invalid base type for memoryview slice: int * -39:0: Fused lambdas not allowed -42:5: Fused types not allowed here -45:9: Fused types not allowed here +28:0: Invalid use of fused types, type cannot be specialized +28:4: Not enough types specified to specialize the function, int2_t is still fused +29:16: Call with wrong number of arguments (expected 2, got 1) +30:16: Call with wrong number of arguments (expected 2, got 3) +37:6: Invalid base type for memoryview slice: int * +40:0: Fused lambdas not allowed +43:5: Fused types not allowed here +43:21: cdef variable 'x' declared after it is used +46:9: Fused types not allowed here +61:0: Invalid use of fused types, type cannot be specialized +61:29: ambiguous overloaded method +# Possibly duplicates the errors more often than we want +79:5: Return type is a fused type that cannot be determined from the function arguments +82:6: Return type is a fused type that cannot be determined from the function arguments +86:4: 'z' cannot be specialized since its type is not a fused argument to this function +86:4: 'z' cannot be specialized since its type is not a fused argument to this function +86:4: 'z' cannot be specialized since its type is not a fused argument to this function +87:24: Type cannot be specialized since it is not a fused argument to this function +87:24: Type cannot be specialized since it is not a fused argument to this function +87:24: Type cannot be specialized since it is not a fused argument to this function """ diff --git a/tests/errors/missing_baseclass_in_predecl_T262.pyx b/tests/errors/missing_baseclass_in_predecl_T262.pyx index ece07b155..907f072f5 100644 --- a/tests/errors/missing_baseclass_in_predecl_T262.pyx +++ b/tests/errors/missing_baseclass_in_predecl_T262.pyx @@ -1,4 +1,4 @@ -# ticket: 262 +# ticket: t262 # mode: error cdef class Album diff --git a/tests/errors/missing_self_in_cpdef_method_T156.pyx b/tests/errors/missing_self_in_cpdef_method_T156.pyx index 21241a221..e8b0c5369 100644 --- a/tests/errors/missing_self_in_cpdef_method_T156.pyx +++ b/tests/errors/missing_self_in_cpdef_method_T156.pyx @@ -1,4 +1,4 @@ -# ticket: 156 +# ticket: t156 # mode: error cdef class B: diff --git a/tests/errors/missing_self_in_cpdef_method_T165.pyx b/tests/errors/missing_self_in_cpdef_method_T165.pyx index 6a95922d5..89763cd2a 100644 --- a/tests/errors/missing_self_in_cpdef_method_T165.pyx +++ b/tests/errors/missing_self_in_cpdef_method_T165.pyx @@ -1,4 +1,4 @@ -# ticket: 165 +# ticket: t165 # mode: error cdef class A: diff --git a/tests/errors/nogil.pyx b/tests/errors/nogil.pyx index 1d22f9d9b..aa3011d00 100644 --- a/tests/errors/nogil.pyx +++ b/tests/errors/nogil.pyx @@ -8,14 +8,14 @@ cdef void g(int x) nogil: cdef object z z = None -cdef void h(int x) nogil: +cdef void h(int x) nogil: # allowed p() cdef object p() nogil: pass -cdef void r() nogil: - q() +cdef void r() nogil: # allowed + q() # allowed cdef object m(): cdef object x, y = 0, obj @@ -23,11 +23,11 @@ cdef object m(): global fred q() with nogil: - r() + r() # allowed to call plain C functions q() - i = 42 + i = 42 # allowed with type inference obj = None - 17L + 17L # allowed <object>7j help xxx = `"Hello"` @@ -45,14 +45,14 @@ cdef object m(): {x, y} obj and x t(obj) -# f(42) # Cython handles this internally + f(42) x + obj -obj x = y = obj x, y = y, x obj[i] = x obj.fred = x - print obj + print obj # allowed! del fred return obj raise obj # allowed! @@ -90,8 +90,13 @@ def bare_pyvar_name(object x): with nogil: x -# For m(), the important thing is that there are errors on all lines in the range 23-69 -# except these: 29, 34, 44, 56, 58, 60, 62-64 +cdef int fstrings(int x, object obj) nogil except -1: + f"" # allowed + f"a" # allowed + f"a"f"b" # allowed + f"{x}" + f"{obj}" + _ERRORS = u""" 4:5: Function with Python return type cannot be declared nogil @@ -105,14 +110,10 @@ _ERRORS = u""" 31:16: Constructing complex number not allowed without gil 33:8: Assignment of Python object not allowed without gil 33:14: Backquote expression not allowed without gil -33:15: Operation not allowed without gil 34:15: Assignment of Python object not allowed without gil -34:15: Operation not allowed without gil 34:15: Python import not allowed without gil -35:8: Operation not allowed without gil 35:13: Python import not allowed without gil 35:25: Constructing Python list not allowed without gil -35:25: Operation not allowed without gil 36:17: Iterating over Python object not allowed without gil 38:11: Discarding owned Python object not allowed without gil 38:11: Indexing Python object not allowed without gil @@ -137,6 +138,7 @@ _ERRORS = u""" 46:12: Discarding owned Python object not allowed without gil 46:12: Truth-testing Python object not allowed without gil 47:10: Python type test not allowed without gil +48:9: Discarding owned Python object not allowed without gil 49:10: Discarding owned Python object not allowed without gil 49:10: Operation not allowed without gil 50:8: Discarding owned Python object not allowed without gil @@ -151,10 +153,10 @@ _ERRORS = u""" 53:11: Indexing Python object not allowed without gil 54:11: Accessing Python attribute not allowed without gil 54:11: Assignment of Python object not allowed without gil -55:8: Constructing Python tuple not allowed without gil -55:8: Python print statement not allowed without gil + 56:8: Deleting Python object not allowed without gil 57:8: Returning Python object not allowed without gil + 59:11: Truth-testing Python object not allowed without gil 61:14: Truth-testing Python object not allowed without gil 63:8: For-loop using object bounds or target not allowed without gil @@ -162,4 +164,9 @@ _ERRORS = u""" 63:24: Coercion from Python not allowed without the GIL 65:8: Try-except statement not allowed without gil 86:8: For-loop using object bounds or target not allowed without gil + +97:4: Discarding owned Python object not allowed without gil +97:6: String formatting not allowed without gil +98:4: Discarding owned Python object not allowed without gil +98:6: String formatting not allowed without gil """ diff --git a/tests/errors/nogil_conditional.pyx b/tests/errors/nogil_conditional.pyx new file mode 100644 index 000000000..e800eb199 --- /dev/null +++ b/tests/errors/nogil_conditional.pyx @@ -0,0 +1,81 @@ +# cython: remove_unreachable=False +# mode: error + +cdef int f_nogil(int x) nogil: + cdef int y + y = x + 10 + return y + + +def f_gil(x): + y = 0 + y = x + 100 + return y + + +def illegal_gil_usage(): + cdef int res = 0 + with nogil(True): + res = f_gil(res) + + with nogil(True): + res = f_gil(res) + + with gil(False): + res = f_gil(res) + + with nogil(False): + res = f_nogil(res) + + +def foo(a): + return a < 10 + + +def non_constant_condition(int x) -> int: + cdef int res = x + with nogil(x < 10): + res = f_nogil(res) + + with gil(foo(x)): + res = f_gil(res) + + +ctypedef fused number_or_object: + float + object + + +def fused_type(number_or_object x): + with nogil(number_or_object is object): + res = x + 1 + + # This should be fine + with nogil(number_or_object is float): + res = x + 1 + + return res + + +_ERRORS = u""" +19:14: Accessing Python global or builtin not allowed without gil +19:19: Calling gil-requiring function not allowed without gil +19:19: Coercion from Python not allowed without the GIL +19:19: Constructing Python tuple not allowed without gil +19:20: Converting to Python object not allowed without gil +21:13: Trying to release the GIL while it was previously released. +22:18: Accessing Python global or builtin not allowed without gil +22:23: Calling gil-requiring function not allowed without gil +22:23: Coercion from Python not allowed without the GIL +22:23: Constructing Python tuple not allowed without gil +22:24: Converting to Python object not allowed without gil +25:18: Accessing Python global or builtin not allowed without gil +25:23: Calling gil-requiring function not allowed without gil +25:23: Coercion from Python not allowed without the GIL +25:23: Constructing Python tuple not allowed without gil +25:24: Converting to Python object not allowed without gil +37:17: Non-constant condition in a `with nogil(<condition>)` statement +40:16: Non-constant condition in a `with gil(<condition>)` statement +51:8: Assignment of Python object not allowed without gil +51:16: Calling gil-requiring function not allowed without gil +""" diff --git a/tests/errors/nonconst_excval.pyx b/tests/errors/nonconst_excval.pyx new file mode 100644 index 000000000..ad80f2e27 --- /dev/null +++ b/tests/errors/nonconst_excval.pyx @@ -0,0 +1,12 @@ +# mode: error + +import math + +cdef double cfunc(double x) except math.nan: + return x + + +_ERRORS = """ +5:39: Exception value must be constant +5:39: Not allowed in a constant expression +""" diff --git a/tests/errors/notcimportedT418.pyx b/tests/errors/notcimportedT418.pyx index 980d66555..c2821fb83 100644 --- a/tests/errors/notcimportedT418.pyx +++ b/tests/errors/notcimportedT418.pyx @@ -1,4 +1,4 @@ -# ticket: 418 +# ticket: t418 # mode: error import somemod.child diff --git a/tests/errors/pep487_exttype.pyx b/tests/errors/pep487_exttype.pyx new file mode 100644 index 000000000..e9e4f44a2 --- /dev/null +++ b/tests/errors/pep487_exttype.pyx @@ -0,0 +1,13 @@ +# mode: error + +cdef class Imp: + def __init_subclass__(cls, a=None, **kwargs): + super().__init_subclass__(**kwargs) + print(a) + +cdef class ExImp1(Imp): pass +class ExImp2(Imp, a=60): pass + +_ERRORS = u""" +4:4: '__init_subclass__' is not supported by extension class +""" diff --git a/tests/errors/pep492_badsyntax_async2.pyx b/tests/errors/pep492_badsyntax_async2.pyx deleted file mode 100644 index 4ea30dba1..000000000 --- a/tests/errors/pep492_badsyntax_async2.pyx +++ /dev/null @@ -1,11 +0,0 @@ -# mode: error -# tag: pep492, async - -async def foo(): - def foo(a:await list()): - pass - -_ERRORS = """ -5:14: 'await' not supported here -5:14: 'await' not supported here -""" diff --git a/tests/errors/posonly.pyx b/tests/errors/posonly.pyx new file mode 100644 index 000000000..2724f3037 --- /dev/null +++ b/tests/errors/posonly.pyx @@ -0,0 +1,102 @@ +# mode: error +# tag: posonly + +def f(a, b = 5, /, c): + pass + +def f(a = 5, b, /, c): + pass + +def f(a = 5, b, /): + pass + +def f(a, /, a): + pass + +def f(a, /, *, a): + pass + +#def f(a, b/2, c): #D +# pass + +#def f(*args, /): #D +# pass + +#def f(*args, a, /): +# pass + +#def f(**kwargs, /): +# pass + +#def f(/, a = 1): # D +# pass + +#def f(/, a): +# pass + +#def f(/): +# pass + +#def f(*, a, /): +# pass + +#def f(*, /, a): +# pass + +#def f(a, /, c, /): +# pass + +#def f(a, /, c, /, d): +# pass + +#def f(a, /, c, /, d, *, e): +# pass + +#def f(a, *, c, /, d, e): +# pass + +def test_invalid_syntax_lambda(self): + lambda a, b = 5, /, c: None + lambda a = 5, b, /, c: None + lambda a = 5, b, /: None + lambda a, /, a: None + lambda a, /, *, a: None +# lambda *args, /: None +# lambda *args, a, /: None +# lambda **kwargs, /: None +# lambda /, a = 1: None +# lambda /, a: None +# lambda /: None +# lambda *, a, /: None +# lambda *, /, a: None + +async def f(a, b = 5, /, c): + pass + +#def test_multiple_seps(a,/,b,/): +# pass + +_ERRORS = u""" +4:19: Non-default argument following default argument +7:13: Non-default argument following default argument +7:19: Non-default argument following default argument +10:13: Non-default argument following default argument +13:6: Previous declaration is here +13:12: 'a' redeclared +16:6: Previous declaration is here +16:15: 'a' redeclared +59:24: Non-default argument following default argument +60:18: Non-default argument following default argument +60:24: Non-default argument following default argument +61:18: Non-default argument following default argument +62:11: Previous declaration is here +62:17: 'a' redeclared +63:11: Previous declaration is here +63:20: 'a' redeclared +73:25: Non-default argument following default argument +""" + + + + + diff --git a/tests/errors/posonly2.pyx b/tests/errors/posonly2.pyx new file mode 100644 index 000000000..dab079047 --- /dev/null +++ b/tests/errors/posonly2.pyx @@ -0,0 +1,9 @@ +# mode: error +# tag: posonly + +def f(a, b/2, c): + pass + +_ERRORS = u""" +4:11: Syntax error in Python function argument list +""" diff --git a/tests/errors/posonly3.pyx b/tests/errors/posonly3.pyx new file mode 100644 index 000000000..4b4f0db75 --- /dev/null +++ b/tests/errors/posonly3.pyx @@ -0,0 +1,13 @@ +# mode: error +# tag: posonly + +def f(*args, /): + pass + +def f(*args, a, /): + pass + + +_ERRORS = u""" +4:13: Expected ')', found '/' +""" diff --git a/tests/errors/posonly4.pyx b/tests/errors/posonly4.pyx new file mode 100644 index 000000000..d1a1f61f9 --- /dev/null +++ b/tests/errors/posonly4.pyx @@ -0,0 +1,9 @@ +# mode: error +# tag: posonly + +def f(/, a = 1): + pass + +_ERRORS = u""" +4:6: Got zero positional-only arguments despite presence of positional-only specifier '/' +""" diff --git a/tests/errors/posonly5.pyx b/tests/errors/posonly5.pyx new file mode 100644 index 000000000..9e359226a --- /dev/null +++ b/tests/errors/posonly5.pyx @@ -0,0 +1,11 @@ +# mode: error +# tag: posonly + +def test_multiple_seps(a,/,b,/): + pass + +_ERRORS = u""" +4:29: Expected ')', found '/' +""" + + diff --git a/tests/errors/pure_errors.py b/tests/errors/pure_errors.py index 4a9dca53b..570dc006f 100644 --- a/tests/errors/pure_errors.py +++ b/tests/errors/pure_errors.py @@ -53,5 +53,5 @@ def pyfunc(x): # invalid _ERRORS = """ 44:22: Calling gil-requiring function not allowed without gil 45:24: Calling gil-requiring function not allowed without gil -49:0: Python functions cannot be declared 'nogil' +48:0: Python functions cannot be declared 'nogil' """ diff --git a/tests/errors/pxd_cdef_class_declaration_T286.pyx b/tests/errors/pxd_cdef_class_declaration_T286.pyx index 0c968f701..b1a8bc844 100644 --- a/tests/errors/pxd_cdef_class_declaration_T286.pyx +++ b/tests/errors/pxd_cdef_class_declaration_T286.pyx @@ -1,4 +1,4 @@ -# ticket: 286 +# ticket: t286 # mode: error cdef class A: diff --git a/tests/errors/pyobjcastdisallow_T313.pyx b/tests/errors/pyobjcastdisallow_T313.pyx index 5048719d8..30973380f 100644 --- a/tests/errors/pyobjcastdisallow_T313.pyx +++ b/tests/errors/pyobjcastdisallow_T313.pyx @@ -1,4 +1,4 @@ -# ticket: 313 +# ticket: t313 # mode: error a = 3 diff --git a/tests/errors/return_outside_function_T135.pyx b/tests/errors/return_outside_function_T135.pyx index d7432ca50..4788c2a72 100644 --- a/tests/errors/return_outside_function_T135.pyx +++ b/tests/errors/return_outside_function_T135.pyx @@ -1,5 +1,5 @@ # cython: remove_unreachable=False -# ticket: 135 +# ticket: t135 # mode: error def _runtime_True(): diff --git a/tests/errors/tree_assert.pyx b/tests/errors/tree_assert.pyx index 81ae9a1d3..b76990cab 100644 --- a/tests/errors/tree_assert.pyx +++ b/tests/errors/tree_assert.pyx @@ -11,8 +11,8 @@ def test(): _ERRORS = u""" -9:0: Expected path '//ComprehensionNode' not found in result tree -9:0: Expected path '//ComprehensionNode//FuncDefNode' not found in result tree -9:0: Unexpected path '//NameNode' found in result tree -9:0: Unexpected path '//SimpleCallNode' found in result tree +5:0: Expected path '//ComprehensionNode' not found in result tree +5:0: Expected path '//ComprehensionNode//FuncDefNode' not found in result tree +5:0: Unexpected path '//NameNode' found in result tree +5:0: Unexpected path '//SimpleCallNode' found in result tree """ diff --git a/tests/errors/typoT304.pyx b/tests/errors/typoT304.pyx index 7c736c898..2de5573d4 100644 --- a/tests/errors/typoT304.pyx +++ b/tests/errors/typoT304.pyx @@ -1,4 +1,4 @@ -# ticket: 304 +# ticket: t304 # mode: error def f(): diff --git a/tests/errors/unicode_identifiers_e1.pyx b/tests/errors/unicode_identifiers_e1.pyx new file mode 100644 index 000000000..5087a3d43 --- /dev/null +++ b/tests/errors/unicode_identifiers_e1.pyx @@ -0,0 +1,8 @@ +# -*- coding: utf-8 -*- +# mode: error + +★1 = 5 # invalid start symbol + +_ERRORS = u""" +4:0: Unrecognized character +""" diff --git a/tests/errors/unicode_identifiers_e2.pyx b/tests/errors/unicode_identifiers_e2.pyx new file mode 100644 index 000000000..e9e53aa5b --- /dev/null +++ b/tests/errors/unicode_identifiers_e2.pyx @@ -0,0 +1,9 @@ +# -*- coding: utf-8 -*- +# mode: error + +class MyClass₡: # invalid continue symbol + pass + +_ERRORS = u""" +4:13: Unrecognized character +""" diff --git a/tests/errors/unicode_identifiers_e3.pyx b/tests/errors/unicode_identifiers_e3.pyx new file mode 100644 index 000000000..e5cba9caa --- /dev/null +++ b/tests/errors/unicode_identifiers_e3.pyx @@ -0,0 +1,11 @@ +# -*- coding: utf-8 -*- +# mode: error + +def f(): + a = 1 + ́b = 2 # looks like an indentation error but is actually a combining accent as the first letter of column 4 + c = 3 + +_ERRORS = u""" +6:4: Unrecognized character +""" diff --git a/tests/errors/unicode_identifiers_e4.pyx b/tests/errors/unicode_identifiers_e4.pyx new file mode 100644 index 000000000..035eff2c9 --- /dev/null +++ b/tests/errors/unicode_identifiers_e4.pyx @@ -0,0 +1,13 @@ +# -*- coding: utf-8 -*- +# mode: error + +cdef class C: + # these two symbols "\u1e69" and "\u1e9b\u0323" normalize to the same thing + # so the two attributes can't coexist + cdef int ṩomething + cdef double ẛ̣omething + +_ERRORS = u""" +7:13: Previous declaration is here +8:16: 'ṩomething' redeclared +""" diff --git a/tests/errors/uninitialized_lhs.pyx b/tests/errors/uninitialized_lhs.pyx index 5d5ece854..8550761a4 100644 --- a/tests/errors/uninitialized_lhs.pyx +++ b/tests/errors/uninitialized_lhs.pyx @@ -1,7 +1,7 @@ # cython: warn.maybe_uninitialized=True # mode: error # tag: werror -# ticket: 739 +# ticket: t739 def index_lhs(a): cdef object idx diff --git a/tests/errors/w_numpy_arr_as_cppvec_ref.pyx b/tests/errors/w_numpy_arr_as_cppvec_ref.pyx index e7fbaeece..b8dd1f536 100644 --- a/tests/errors/w_numpy_arr_as_cppvec_ref.pyx +++ b/tests/errors/w_numpy_arr_as_cppvec_ref.pyx @@ -5,6 +5,8 @@ import numpy as np cimport numpy as np from libcpp.vector cimport vector +np.import_array() + cdef extern from *: void cpp_function_vector1(vector[int]) void cpp_function_vector2(vector[int] &) @@ -24,8 +26,8 @@ def main(): _ERRORS = """ -17:25: Cannot pass Python object as C++ data structure reference (vector[int] &), will pass by copy. -18:25: Cannot pass Python object as C++ data structure reference (vector[int] &), will pass by copy. -19:28: Cannot pass Python object as C++ data structure reference (vector[int] &), will pass by copy. -19:33: Cannot pass Python object as C++ data structure reference (vector[int] &), will pass by copy. +19:25: Cannot pass Python object as C++ data structure reference (vector[int] &), will pass by copy. +20:25: Cannot pass Python object as C++ data structure reference (vector[int] &), will pass by copy. +21:28: Cannot pass Python object as C++ data structure reference (vector[int] &), will pass by copy. +21:33: Cannot pass Python object as C++ data structure reference (vector[int] &), will pass by copy. """ diff --git a/tests/limited_api_bugs.txt b/tests/limited_api_bugs.txt new file mode 100644 index 000000000..60baf6eb4 --- /dev/null +++ b/tests/limited_api_bugs.txt @@ -0,0 +1,8 @@ +# This file contains tests corresponding to unresolved bugs using CPython's +# Limited API which will be skipped in the normal testing run. + +convolve2 +dict_animal +not_none +queue3 +test_queue diff --git a/tests/memoryview/cfunc_convert_with_memoryview.pyx b/tests/memoryview/cfunc_convert_with_memoryview.pyx new file mode 100644 index 000000000..55bd02205 --- /dev/null +++ b/tests/memoryview/cfunc_convert_with_memoryview.pyx @@ -0,0 +1,51 @@ +# mode: run +# tag: autowrap +# cython: always_allow_keywords=True + +cdef void memoryview_func_a(double [:] x): + x[0] = 1 + +cdef void memoryview_func_b(double [::1] x): + x[0] = 2 + +cdef void memoryview_func_c(int [:] x): + x[0] = 1 + +cdef void memoryview_func_d(int [:] x): + x[0] = 2 + +cdef void memoryview_func_e(int [:,::1] x): + x[0,0] = 4 + +cdef void memoryview_func_f(int [::1,:] x): + x[0,0] = 4 + + +def test_memview_wrapping(): + """ + We're mainly concerned that the code compiles without the names clashing + >>> test_memview_wrapping() + 1.0 + 2.0 + 1 + 2 + """ + cdef a = memoryview_func_a + cdef b = memoryview_func_b + cdef c = memoryview_func_c + cdef d = memoryview_func_d + cdef e = memoryview_func_e + cdef f = memoryview_func_f + cdef double[1] double_arr = [0] + cdef int[1] int_arr = [0] + + a(<double[:1]> double_arr) + print(double_arr[0]) + b(<double[:1:1]> double_arr) + print(double_arr[0]) + c(<int[:1]> int_arr) + print(int_arr[0]) + d(<int[:1:1]> int_arr) + print(int_arr[0]) + # don't call e and f because it's harder without needing extra dependencies + # it's mostly a compile test for them diff --git a/tests/memoryview/cythonarray.pyx b/tests/memoryview/cythonarray.pyx index 1f87a4e18..d0408ff23 100644 --- a/tests/memoryview/cythonarray.pyx +++ b/tests/memoryview/cythonarray.pyx @@ -209,3 +209,80 @@ def test_cyarray_from_carray(): mslice = a print mslice[0, 0], mslice[1, 0], mslice[2, 5] + +class InheritFrom(v.array): + """ + Test is just to confirm it works, not to do anything meaningful with it + (Be aware that itemsize isn't necessarily right) + >>> inst = InheritFrom(shape=(3, 3, 3), itemsize=4, format="i") + """ + pass + + +def test_char_array_in_python_api(*shape): + """ + >>> import sys + >>> if sys.version_info[0] < 3: + ... def bytes(b): return memoryview(b).tobytes() # don't call str() + + >>> arr1d = test_char_array_in_python_api(10) + >>> print(bytes(arr1d).decode('ascii')) + xxxxxxxxxx + >>> len(bytes(arr1d)) + 10 + >>> arr2d = test_char_array_in_python_api(10, 2) + >>> print(bytes(arr2d).decode('ascii')) + xxxxxxxxxxxxxxxxxxxx + >>> len(bytes(arr2d)) + 20 + + # memoryview + >>> len(bytes(memoryview(arr1d))) + 10 + >>> bytes(memoryview(arr1d)) == bytes(arr1d) + True + >>> len(bytes(memoryview(arr2d))) + 20 + >>> bytes(memoryview(arr2d)) == bytes(arr2d) + True + + # BytesIO + >>> from io import BytesIO + >>> BytesIO(arr1d).read() == bytes(arr1d) + True + >>> BytesIO(arr2d).read() == bytes(arr2d) + True + + >>> b = BytesIO() + >>> print(b.write(arr1d)) + 10 + >>> b.getvalue() == bytes(arr1d) or b.getvalue() + True + + >>> b = BytesIO() + >>> print(b.write(arr2d)) + 20 + >>> b.getvalue() == bytes(arr2d) or b.getvalue() + True + + # BufferedWriter (uses PyBUF_SIMPLE, see https://github.com/cython/cython/issues/3775) + >>> from io import BufferedWriter + >>> b = BytesIO() + >>> bw = BufferedWriter(b) + >>> print(bw.write(arr1d)) + 10 + >>> bw.flush() + >>> b.getvalue() == bytes(arr1d) + True + + >>> b = BytesIO() + >>> bw = BufferedWriter(b) + >>> print(bw.write(arr2d)) + 20 + >>> bw.flush() + >>> b.getvalue() == bytes(arr2d) + True + """ + arr = array(shape=shape, itemsize=sizeof(char), format='c', mode='c') + arr[:] = b'x' + return arr diff --git a/tests/memoryview/memoryview.pyx b/tests/memoryview/memoryview.pyx index 15c71d481..273a1fa68 100644 --- a/tests/memoryview/memoryview.pyx +++ b/tests/memoryview/memoryview.pyx @@ -1,5 +1,7 @@ # mode: run +# Test declarations, behaviour and coercions of the memoryview type itself. + u''' >>> f() >>> g() @@ -155,6 +157,7 @@ def assignmvs(): cdef int[:] mv3 mv1 = array((10,), itemsize=sizeof(int), format='i') mv2 = mv1 + mv1 = mv1 mv1 = mv2 mv3 = mv2 @@ -247,7 +250,7 @@ def basic_struct(MyStruct[:] mslice): >>> basic_struct(MyStructMockBuffer(None, [(1, 2, 3, 4, 5)], format="ccqii")) [('a', 1), ('b', 2), ('c', 3), ('d', 4), ('e', 5)] """ - buf = mslice + cdef object buf = mslice print sorted([(k, int(v)) for k, v in buf[0].items()]) def nested_struct(NestedStruct[:] mslice): @@ -259,7 +262,7 @@ def nested_struct(NestedStruct[:] mslice): >>> nested_struct(NestedStructMockBuffer(None, [(1, 2, 3, 4, 5)], format="T{ii}T{2i}i")) 1 2 3 4 5 """ - buf = mslice + cdef object buf = mslice d = buf[0] print d['x']['a'], d['x']['b'], d['y']['a'], d['y']['b'], d['z'] @@ -275,7 +278,7 @@ def packed_struct(PackedStruct[:] mslice): 1 2 """ - buf = mslice + cdef object buf = mslice print buf[0]['a'], buf[0]['b'] def nested_packed_struct(NestedPackedStruct[:] mslice): @@ -289,7 +292,7 @@ def nested_packed_struct(NestedPackedStruct[:] mslice): >>> nested_packed_struct(NestedPackedStructMockBuffer(None, [(1, 2, 3, 4, 5)], format="^c@i^ci@i")) 1 2 3 4 5 """ - buf = mslice + cdef object buf = mslice d = buf[0] print d['a'], d['b'], d['sub']['a'], d['sub']['b'], d['c'] @@ -299,7 +302,7 @@ def complex_dtype(long double complex[:] mslice): >>> complex_dtype(LongComplexMockBuffer(None, [(0, -1)])) -1j """ - buf = mslice + cdef object buf = mslice print buf[0] def complex_inplace(long double complex[:] mslice): @@ -307,7 +310,7 @@ def complex_inplace(long double complex[:] mslice): >>> complex_inplace(LongComplexMockBuffer(None, [(0, -1)])) (1+1j) """ - buf = mslice + cdef object buf = mslice buf[0] = buf[0] + 1 + 2j print buf[0] @@ -318,7 +321,7 @@ def complex_struct_dtype(LongComplex[:] mslice): >>> complex_struct_dtype(LongComplexMockBuffer(None, [(0, -1)])) 0.0 -1.0 """ - buf = mslice + cdef object buf = mslice print buf[0]['real'], buf[0]['imag'] # @@ -356,7 +359,7 @@ def get_int_2d(int[:, :] mslice, int i, int j): ... IndexError: Out of bounds on buffer access (axis 1) """ - buf = mslice + cdef object buf = mslice return buf[i, j] def set_int_2d(int[:, :] mslice, int i, int j, int value): @@ -409,11 +412,48 @@ def set_int_2d(int[:, :] mslice, int i, int j, int value): IndexError: Out of bounds on buffer access (axis 1) """ - buf = mslice + cdef object buf = mslice buf[i, j] = value # +# auto type inference +# (note that for most numeric types "might_overflow" stops the type inference from working well) +# +def type_infer(double[:, :] arg): + """ + >>> type_infer(DoubleMockBuffer(None, range(6), (2,3))) + double + double[:] + double[:] + double[:, :] + """ + a = arg[0,0] + print(cython.typeof(a)) + b = arg[0] + print(cython.typeof(b)) + c = arg[0,:] + print(cython.typeof(c)) + d = arg[:,:] + print(cython.typeof(d)) + +# +# Loop optimization +# +@cython.test_fail_if_path_exists("//CoerceToPyTypeNode") +def memview_iter(double[:, :] arg): + """ + memview_iter(DoubleMockBuffer("C", range(6), (2,3))) + True + """ + cdef double total = 0 + for mview1d in arg: + for val in mview1d: + total += val + if total == 15: + return True + +# # Test all kinds of indexing and flags # @@ -426,7 +466,7 @@ def writable(unsigned short int[:, :, :] mslice): >>> [str(x) for x in R.received_flags] # Py2/3 ['FORMAT', 'ND', 'STRIDES', 'WRITABLE'] """ - buf = mslice + cdef object buf = mslice buf[2, 2, 1] = 23 def strided(int[:] mslice): @@ -441,7 +481,7 @@ def strided(int[:] mslice): >>> A.release_ok True """ - buf = mslice + cdef object buf = mslice return buf[2] def c_contig(int[::1] mslice): @@ -450,7 +490,7 @@ def c_contig(int[::1] mslice): >>> c_contig(A) 2 """ - buf = mslice + cdef object buf = mslice return buf[2] def c_contig_2d(int[:, ::1] mslice): @@ -461,7 +501,7 @@ def c_contig_2d(int[:, ::1] mslice): >>> c_contig_2d(A) 7 """ - buf = mslice + cdef object buf = mslice return buf[1, 3] def f_contig(int[::1, :] mslice): @@ -470,7 +510,7 @@ def f_contig(int[::1, :] mslice): >>> f_contig(A) 2 """ - buf = mslice + cdef object buf = mslice return buf[0, 1] def f_contig_2d(int[::1, :] mslice): @@ -481,7 +521,7 @@ def f_contig_2d(int[::1, :] mslice): >>> f_contig_2d(A) 7 """ - buf = mslice + cdef object buf = mslice return buf[3, 1] def generic(int[::view.generic, ::view.generic] mslice1, @@ -552,7 +592,7 @@ def printbuf_td_cy_int(td_cy_int[:] mslice, shape): ... ValueError: Buffer dtype mismatch, expected 'td_cy_int' but got 'short' """ - buf = mslice + cdef object buf = mslice cdef int i for i in range(shape[0]): print buf[i], @@ -567,7 +607,7 @@ def printbuf_td_h_short(td_h_short[:] mslice, shape): ... ValueError: Buffer dtype mismatch, expected 'td_h_short' but got 'int' """ - buf = mslice + cdef object buf = mslice cdef int i for i in range(shape[0]): print buf[i], @@ -582,7 +622,7 @@ def printbuf_td_h_cy_short(td_h_cy_short[:] mslice, shape): ... ValueError: Buffer dtype mismatch, expected 'td_h_cy_short' but got 'int' """ - buf = mslice + cdef object buf = mslice cdef int i for i in range(shape[0]): print buf[i], @@ -597,7 +637,7 @@ def printbuf_td_h_ushort(td_h_ushort[:] mslice, shape): ... ValueError: Buffer dtype mismatch, expected 'td_h_ushort' but got 'short' """ - buf = mslice + cdef object buf = mslice cdef int i for i in range(shape[0]): print buf[i], @@ -612,7 +652,7 @@ def printbuf_td_h_double(td_h_double[:] mslice, shape): ... ValueError: Buffer dtype mismatch, expected 'td_h_double' but got 'float' """ - buf = mslice + cdef object buf = mslice cdef int i for i in range(shape[0]): print buf[i], @@ -626,6 +666,8 @@ def addref(*args): def decref(*args): for item in args: Py_DECREF(item) +@cython.binding(False) +@cython.always_allow_keywords(False) def get_refcount(x): return (<PyObject*>x).ob_refcnt @@ -647,7 +689,7 @@ def printbuf_object(object[:] mslice, shape): {4: 23} 2 [34, 3] 2 """ - buf = mslice + cdef object buf = mslice cdef int i for i in range(shape[0]): print repr(buf[i]), (<PyObject*>buf[i]).ob_refcnt @@ -668,7 +710,7 @@ def assign_to_object(object[:] mslice, int idx, obj): (2, 3) >>> decref(b) """ - buf = mslice + cdef object buf = mslice buf[idx] = obj def assign_temporary_to_object(object[:] mslice): @@ -695,7 +737,7 @@ def assign_temporary_to_object(object[:] mslice): >>> assign_to_object(A, 1, a) >>> decref(a) """ - buf = mslice + cdef object buf = mslice buf[1] = {3-2: 2+(2*4)-2} @@ -743,7 +785,7 @@ def test_generic_slicing(arg, indirect=False): """ cdef int[::view.generic, ::view.generic, :] _a = arg - a = _a + cdef object a = _a b = a[2:8:2, -4:1:-1, 1:3] print b.shape @@ -826,7 +868,7 @@ def test_direct_slicing(arg): released A """ cdef int[:, :, :] _a = arg - a = _a + cdef object a = _a b = a[2:8:2, -4:1:-1, 1:3] print b.shape @@ -854,7 +896,7 @@ def test_slicing_and_indexing(arg): released A """ cdef int[:, :, :] _a = arg - a = _a + cdef object a = _a b = a[-5:, 1, 1::2] c = b[4:1:-1, ::-1] d = c[2, 1:2] @@ -1099,7 +1141,7 @@ def optimised_index_of_slice(int[:,:,:] arr, int x, int y, int z): def test_assign_from_byteslike(byteslike): - # Once http://python3statement.org is accepted, should be just + # Once https://python3statement.org/ is accepted, should be just # >>> test_assign_from_byteslike(bytes(b'hello')) # b'hello' # ... diff --git a/tests/memoryview/memoryview_cache_builtins.srctree b/tests/memoryview/memoryview_cache_builtins.srctree new file mode 100644 index 000000000..91b1c4753 --- /dev/null +++ b/tests/memoryview/memoryview_cache_builtins.srctree @@ -0,0 +1,21 @@ +PYTHON setup.py build_ext --inplace + +################ setup.py ##################### + +from distutils.core import setup +from Cython.Build import cythonize +from Cython.Compiler import Options + +Options.cache_builtins = False + +setup( + ext_modules = cythonize("mview.pyx") +) + +############### mview.pyx ################ + +# https://github.com/cython/cython/issues/3406 +# Failure was at Cython compilation stage + +def f(double [::1] x): + pass diff --git a/tests/memoryview/memoryview_no_binding_T3613.pyx b/tests/memoryview/memoryview_no_binding_T3613.pyx new file mode 100644 index 000000000..1c736359e --- /dev/null +++ b/tests/memoryview/memoryview_no_binding_T3613.pyx @@ -0,0 +1,21 @@ +# mode: compile +# tag: memoryview + +# cython: binding=False + +# See GH 3613 - when memoryviews were compiled with binding off they ended up in an +# inconsistent state where different directives were applied at different stages +# of the pipeline + +import cython + +def f(double[:] a): + pass + +@cython.binding(False) +def g(double[:] a): + pass + +@cython.binding(True) +def h(double[:] a): + pass diff --git a/tests/memoryview/memoryview_pep489_typing.pyx b/tests/memoryview/memoryview_pep484_typing.pyx index 0a62ff624..66445a1b7 100644 --- a/tests/memoryview/memoryview_pep489_typing.pyx +++ b/tests/memoryview/memoryview_pep484_typing.pyx @@ -1,5 +1,5 @@ # mode: run -# tag: pep489, memoryview +# tag: pep484, memoryview cimport cython diff --git a/tests/memoryview/memoryviewattrs.pyx b/tests/memoryview/memoryviewattrs.pyx index 66bc9da56..7322424c3 100644 --- a/tests/memoryview/memoryviewattrs.pyx +++ b/tests/memoryview/memoryviewattrs.pyx @@ -1,13 +1,6 @@ # mode: run # tag: numpy -__test__ = {} - -def testcase(func): - __test__[func.__name__] = func.__doc__ - return func - - cimport cython from cython.view cimport array @@ -15,7 +8,6 @@ import numpy as np cimport numpy as np -@testcase def test_shape_stride_suboffset(): u''' >>> test_shape_stride_suboffset() @@ -49,7 +41,6 @@ def test_shape_stride_suboffset(): print c_contig.suboffsets[0], c_contig.suboffsets[1], c_contig.suboffsets[2] -@testcase def test_copy_to(): u''' >>> test_copy_to() @@ -72,7 +63,6 @@ def test_copy_to(): print ' '.join(str(to_data[i]) for i in range(2*2*2)) -@testcase def test_overlapping_copy(): """ >>> test_overlapping_copy() @@ -88,7 +78,6 @@ def test_overlapping_copy(): assert slice[i] == 10 - 1 - i -@testcase def test_copy_return_type(): """ >>> test_copy_return_type() @@ -103,7 +92,6 @@ def test_copy_return_type(): print(f_contig[2, 2]) -@testcase def test_partly_overlapping(): """ >>> test_partly_overlapping() @@ -119,7 +107,6 @@ def test_partly_overlapping(): for i in range(5): assert slice2[i] == i + 4 -@testcase @cython.nonecheck(True) def test_nonecheck1(): u''' @@ -131,7 +118,6 @@ def test_nonecheck1(): cdef int[:,:,:] uninitialized print uninitialized.is_c_contig() -@testcase @cython.nonecheck(True) def test_nonecheck2(): u''' @@ -143,7 +129,6 @@ def test_nonecheck2(): cdef int[:,:,:] uninitialized print uninitialized.is_f_contig() -@testcase @cython.nonecheck(True) def test_nonecheck3(): u''' @@ -155,7 +140,6 @@ def test_nonecheck3(): cdef int[:,:,:] uninitialized uninitialized.copy() -@testcase @cython.nonecheck(True) def test_nonecheck4(): u''' @@ -167,7 +151,6 @@ def test_nonecheck4(): cdef int[:,:,:] uninitialized uninitialized.copy_fortran() -@testcase @cython.nonecheck(True) def test_nonecheck5(): u''' @@ -179,7 +162,6 @@ def test_nonecheck5(): cdef int[:,:,:] uninitialized uninitialized._data -@testcase def test_copy_mismatch(): u''' >>> test_copy_mismatch() @@ -193,7 +175,6 @@ def test_copy_mismatch(): mv1[...] = mv2 -@testcase def test_is_contiguous(): u""" >>> test_is_contiguous() @@ -222,7 +203,6 @@ def test_is_contiguous(): print 'strided', strided[::2].is_c_contig() -@testcase def call(): u''' >>> call() @@ -265,7 +245,6 @@ def call(): assert len(mv3) == 3 -@testcase def two_dee(): u''' >>> two_dee() @@ -313,7 +292,6 @@ def two_dee(): print (<long*>mv3._data)[0] , (<long*>mv3._data)[1] , (<long*>mv3._data)[2] , (<long*>mv3._data)[3] -@testcase def fort_two_dee(): u''' >>> fort_two_dee() diff --git a/tests/memoryview/memslice.pyx b/tests/memoryview/memslice.pyx index 827012c1e..d68ee43fe 100644 --- a/tests/memoryview/memslice.pyx +++ b/tests/memoryview/memslice.pyx @@ -1,5 +1,7 @@ # mode: run +# Test the behaviour of memoryview slicing and indexing, contiguity, etc. + # Note: see also bufaccess.pyx from __future__ import unicode_literals @@ -1058,6 +1060,8 @@ def addref(*args): def decref(*args): for item in args: Py_DECREF(item) +@cython.binding(False) +@cython.always_allow_keywords(False) def get_refcount(x): return (<PyObject*>x).ob_refcnt @@ -1577,7 +1581,7 @@ def test_index_slicing_away_direct_indirect(): All dimensions preceding dimension 1 must be indexed and not sliced """ cdef int[:, ::view.indirect, :] a = TestIndexSlicingDirectIndirectDims() - a_obj = a + cdef object a_obj = a print a[1][2][3] print a[1, 2, 3] @@ -1907,11 +1911,7 @@ def test_struct_attributes_format(): """ cdef TestAttrs[10] array cdef TestAttrs[:] struct_memview = array - - if sys.version_info[:2] >= (2, 7): - print builtins.memoryview(struct_memview).format - else: - print "T{i:int_attrib:c:char_attrib:}" + print builtins.memoryview(struct_memview).format # Test padding at the end of structs in the buffer support @@ -2199,7 +2199,7 @@ def test_object_dtype_copying(): 7 8 9 - 2 5 + 5 1 5 """ cdef int i @@ -2220,10 +2220,12 @@ def test_object_dtype_copying(): print m2[i] obj = m2[5] - print get_refcount(obj), obj + refcount1 = get_refcount(obj) + print obj del m2 - print get_refcount(obj), obj + refcount2 = get_refcount(obj) + print refcount1 - refcount2, obj assert unique_refcount == get_refcount(unique), (unique_refcount, get_refcount(unique)) @@ -2310,7 +2312,9 @@ def test_contig_scalar_to_slice_assignment(): """ >>> test_contig_scalar_to_slice_assignment() 14 14 14 14 + 30 14 30 14 20 20 20 20 + 30 30 20 20 """ cdef int[5][10] a cdef int[:, ::1] m = a @@ -2318,9 +2322,15 @@ def test_contig_scalar_to_slice_assignment(): m[...] = 14 print m[0, 0], m[-1, -1], m[3, 2], m[4, 9] + m[:, :1] = 30 + print m[0, 0], m[0, 1], m[1, 0], m[1, 1] + m[:, :] = 20 print m[0, 0], m[-1, -1], m[3, 2], m[4, 9] + m[:1, :] = 30 + print m[0, 0], m[0, 1], m[1, 0], m[1, 1] + @testcase def test_dtype_object_scalar_assignment(): """ diff --git a/tests/memoryview/numpy_memoryview.pyx b/tests/memoryview/numpy_memoryview.pyx index 9b18be615..6860fafeb 100644 --- a/tests/memoryview/numpy_memoryview.pyx +++ b/tests/memoryview/numpy_memoryview.pyx @@ -17,6 +17,10 @@ include "../buffers/mockbuffers.pxi" ctypedef np.int32_t dtype_t +IS_PYPY = hasattr(sys, 'pypy_version_info') +NUMPY_VERSION = tuple(int(v) for v in np.__version__.split('.')[:2]) +print(NUMPY_VERSION) + def get_array(): # We need to type our array to get a __pyx_get_buffer() that typechecks # for np.ndarray and calls __getbuffer__ in numpy.pxd @@ -32,22 +36,13 @@ def ae(*args): if x != args[0]: raise AssertionError(args) -__test__ = {} - -def testcase(f): - __test__[f.__name__] = f.__doc__ - return f - -def testcase_numpy_1_5(f): - major, minor, *rest = np.__version__.split('.') - if (int(major), int(minor)) >= (1, 5): - __test__[f.__name__] = f.__doc__ +def testcase_no_pypy(f, _is_pypy=hasattr(sys, "pypy_version_info")): + if _is_pypy: + f.__doc__ = "" # disable the tests return f - def gc_collect_if_required(): - major, minor, *rest = np.__version__.split('.') - if (int(major), int(minor)) >= (1, 14): + if NUMPY_VERSION >= (1, 14) or IS_PYPY: import gc gc.collect() @@ -56,7 +51,6 @@ def gc_collect_if_required(): ### Test slicing memoryview slices # -@testcase def test_partial_slicing(array): """ >>> test_partial_slicing(a) @@ -72,7 +66,6 @@ def test_partial_slicing(array): ae(b.strides[0], c.strides[0], obj.strides[0]) ae(b.strides[1], c.strides[1], obj.strides[1]) -@testcase def test_ellipsis(array): """ >>> test_ellipsis(a) @@ -114,7 +107,6 @@ def test_ellipsis(array): # ### Test slicing memoryview objects # -@testcase def test_partial_slicing_memoryview(array): """ >>> test_partial_slicing_memoryview(a) @@ -131,7 +123,6 @@ def test_partial_slicing_memoryview(array): ae(b.strides[0], c.strides[0], obj.strides[0]) ae(b.strides[1], c.strides[1], obj.strides[1]) -@testcase def test_ellipsis_memoryview(array): """ >>> test_ellipsis_memoryview(a) @@ -172,7 +163,6 @@ def test_ellipsis_memoryview(array): ae(e.strides[0], e_obj.strides[0]) -@testcase def test_transpose(): """ >>> test_transpose() @@ -186,7 +176,7 @@ def test_transpose(): numpy_obj = np.arange(4 * 3, dtype=np.int32).reshape(4, 3) a = numpy_obj - a_obj = a + cdef object a_obj = a cdef dtype_t[:, :] b = a.T print a.T.shape[0], a.T.shape[1] @@ -203,7 +193,6 @@ def test_transpose(): print a[3, 2], a.T[2, 3], a_obj[3, 2], a_obj.T[2, 3], numpy_obj[3, 2], numpy_obj.T[2, 3] -@testcase def test_transpose_type(a): """ >>> a = np.zeros((5, 10), dtype=np.float64) @@ -216,12 +205,8 @@ def test_transpose_type(a): print m_transpose[6, 4] -@testcase_numpy_1_5 def test_numpy_like_attributes(cyarray): """ - For some reason this fails in numpy 1.4, with shape () and strides (40, 8) - instead of 20, 4 on my machine. Investigate this. - >>> cyarray = create_array(shape=(8, 5), mode="c") >>> test_numpy_like_attributes(cyarray) >>> test_numpy_like_attributes(cyarray.memview) @@ -237,14 +222,13 @@ def test_numpy_like_attributes(cyarray): cdef int[:, :] mslice = numarray assert (<object> mslice).base is numarray -@testcase_numpy_1_5 def test_copy_and_contig_attributes(a): """ >>> a = np.arange(20, dtype=np.int32).reshape(5, 4) >>> test_copy_and_contig_attributes(a) """ cdef np.int32_t[:, :] mslice = a - m = mslice + cdef object m = mslice # object copy # Test object copy attributes assert np.all(a == np.array(m.copy())) @@ -274,7 +258,7 @@ def build_numarray(array array): def index(array array): print build_numarray(array)[3, 2] -@testcase_numpy_1_5 +@testcase_no_pypy def test_coerce_to_numpy(): """ Test coercion to NumPy arrays, especially with automatically @@ -355,6 +339,7 @@ def test_coerce_to_numpy(): 'e': 800, } + smallstructs[idx] = { 'a': 600, 'b': 700 } nestedstructs[idx] = { @@ -412,7 +397,7 @@ def test_coerce_to_numpy(): index(<td_h_ushort[:4, :5]> <td_h_ushort *> h_ushorts) -@testcase_numpy_1_5 +@testcase_no_pypy def test_memslice_getbuffer(): """ >>> test_memslice_getbuffer(); gc_collect_if_required() @@ -451,7 +436,6 @@ cdef packed struct StructArray: int a[4] signed char b[5] -@testcase_numpy_1_5 def test_memslice_structarray(data, dtype): """ >>> def b(s): return s.encode('ascii') @@ -507,7 +491,6 @@ def test_memslice_structarray(data, dtype): print myslice[i].a[j] print myslice[i].b.decode('ASCII') -@testcase_numpy_1_5 def test_structarray_errors(StructArray[:] a): """ >>> dtype = np.dtype([('a', '4i'), ('b', '5b')]) @@ -554,7 +537,6 @@ def stringstructtest(StringStruct[:] view): def stringtest(String[:] view): pass -@testcase_numpy_1_5 def test_string_invalid_dims(): """ >>> def b(s): return s.encode('ascii') @@ -575,7 +557,6 @@ ctypedef struct AttributesStruct: float attrib2 StringStruct attrib3 -@testcase_numpy_1_5 def test_struct_attributes(): """ >>> test_struct_attributes() @@ -631,7 +612,6 @@ cdef class SuboffsetsNoStridesBuffer(Buffer): getbuffer(self, info) info.suboffsets = self._shape -@testcase def test_null_strides(Buffer buffer_obj): """ >>> test_null_strides(Buffer()) @@ -651,7 +631,6 @@ def test_null_strides(Buffer buffer_obj): assert m2[i, j] == buffer_obj.m[i, j], (i, j, m2[i, j], buffer_obj.m[i, j]) assert m3[i, j] == buffer_obj.m[i, j] -@testcase def test_null_strides_error(buffer_obj): """ >>> test_null_strides_error(Buffer()) @@ -725,7 +704,6 @@ ctypedef struct SameTypeAfterArraysStructSimple: double b[16] double c -@testcase def same_type_after_arrays_simple(): """ >>> same_type_after_arrays_simple() @@ -747,7 +725,6 @@ ctypedef struct SameTypeAfterArraysStructComposite: double h[4] int i -@testcase def same_type_after_arrays_composite(): """ >>> same_type_after_arrays_composite() if sys.version_info[:2] >= (3, 5) else None diff --git a/tests/memoryview/numpy_memoryview_readonly.pyx b/tests/memoryview/numpy_memoryview_readonly.pyx index 20b6c7393..f1b289968 100644 --- a/tests/memoryview/numpy_memoryview_readonly.pyx +++ b/tests/memoryview/numpy_memoryview_readonly.pyx @@ -1,10 +1,14 @@ # mode: run # tag: readonly, const, numpy +# ticket: 1772 import numpy as np +cimport cython -def new_array(): - return np.arange(10).astype('float') +def new_array(dtype='float', writeable=True): + array = np.arange(10, dtype=dtype) + array.setflags(write=writeable) + return array ARRAY = new_array() @@ -124,3 +128,45 @@ def test_copy(): rw[1] = 2 rw2[2] = 2 return rw[0], rw[1], rw[2], rw2[0], rw2[1], rw2[2] + + +cdef getmax_floating(const cython.floating[:] x): + """Function with fused type, should work with both ro and rw memoryviews""" + cdef cython.floating max_val = - float('inf') + for val in x: + if val > max_val: + max_val = val + return max_val + + +def test_mmview_const_fused_cdef(): + """Test cdef function with const fused type memory view as argument. + + >>> test_mmview_const_fused_cdef() + """ + cdef float[:] data_rw = new_array(dtype='float32') + assert getmax_floating(data_rw) == 9 + + cdef const float[:] data_ro = new_array(dtype='float32', writeable=False) + assert getmax_floating(data_ro) == 9 + + +def test_mmview_const_fused_def(const cython.floating[:] x): + """Test def function with const fused type memory view as argument. + + With read-write numpy array: + + >>> test_mmview_const_fused_def(new_array('float32', writeable=True)) + 0.0 + >>> test_mmview_const_fused_def(new_array('float64', writeable=True)) + 0.0 + + With read-only numpy array: + + >>> test_mmview_const_fused_def(new_array('float32', writeable=False)) + 0.0 + >>> test_mmview_const_fused_def(new_array('float64', writeable=False)) + 0.0 + """ + cdef cython.floating result = x[0] + return result diff --git a/tests/memoryview/relaxed_strides.pyx b/tests/memoryview/relaxed_strides.pyx index 1743f23d0..6f90f4db5 100644 --- a/tests/memoryview/relaxed_strides.pyx +++ b/tests/memoryview/relaxed_strides.pyx @@ -10,12 +10,12 @@ Thanks to Nathaniel Smith and Sebastian Berg. See also: Mailing list threads: - http://thread.gmane.org/gmane.comp.python.cython.devel/14762 - http://thread.gmane.org/gmane.comp.python.cython.devel/14634 + https://thread.gmane.org/gmane.comp.python.cython.devel/14762 + https://thread.gmane.org/gmane.comp.python.cython.devel/14634 Detailed discussion of the difference between numpy/cython's current definition of "contiguity", and the correct definition: - http://thread.gmane.org/gmane.comp.python.cython.devel/14634/focus=14640 + https://thread.gmane.org/gmane.comp.python.cython.devel/14634/focus=14640 The PR implementing NPY_RELAXED_STRIDES_CHECKING: https://github.com/numpy/numpy/pull/3162 @@ -37,13 +37,6 @@ NUMPY_HAS_RELAXED_STRIDES = ( np.ones((10, 1), order="C").flags.f_contiguous) -def not_py26(f): - import sys - if sys.version_info < (2, 7): - return lambda a: None - return f - - def test_one_sized(array): """ >>> contig = np.ascontiguousarray(np.arange(10, dtype=np.double)[::100]) @@ -81,7 +74,6 @@ def test_zero_sized_multidim_ccontig(array): cdef double[:, :, ::1] a = array return a -@not_py26 def test_zero_sized_multidim_fcontig(array): """ >>> contig = np.ascontiguousarray(np.zeros((4,4,4))[::2, 2:2, ::2]) diff --git a/tests/memoryview/view_return_errors.pyx b/tests/memoryview/view_return_errors.pyx index 8e9443108..6cedef969 100644 --- a/tests/memoryview/view_return_errors.pyx +++ b/tests/memoryview/view_return_errors.pyx @@ -13,7 +13,18 @@ cdef double[:] foo(int i): raise TypeError('dummy') -def propagate(i): +cdef double[:] foo_nogil(int i) nogil: + if i == 1: + raise AttributeError('dummy') + if i == 2: + raise RuntimeError('dummy') + if i == 3: + raise ValueError('dummy') + if i == 4: + raise TypeError('dummy') + + +def propagate(i, bint nogil=False): """ >>> propagate(0) TypeError('Memoryview return value is not initialized') @@ -25,9 +36,20 @@ def propagate(i): ValueError('dummy') >>> propagate(4) TypeError('dummy') + + >>> propagate(0, nogil=True) + TypeError('Memoryview return value is not initialized') + >>> propagate(1, nogil=True) + AttributeError('dummy') + >>> propagate(2, nogil=True) + RuntimeError('dummy') + >>> propagate(3, nogil=True) + ValueError('dummy') + >>> propagate(4, nogil=True) + TypeError('dummy') """ try: - foo(i) + foo_nogil(i) if nogil else foo(i) except Exception as e: print '%s(%r)' % (e.__class__.__name__, e.args[0]) else: diff --git a/tests/pypy2_bugs.txt b/tests/pypy2_bugs.txt new file mode 100644 index 000000000..18fbcd204 --- /dev/null +++ b/tests/pypy2_bugs.txt @@ -0,0 +1,29 @@ +# Specific bugs that only apply to pypy2 + +build.cythonize_script +build.cythonize_script_package +run.initial_file_path +run.reduce_pickle +run.final_in_pxd +run.cdef_multiple_inheritance +run.cdef_multiple_inheritance_nodict +run.extstarargs +run.cpython_capi +run.isnot + +# pypy 2 seems to be preferring .py files to .so files +# https://foss.heptapod.net/pypy/pypy/issues/3185 +run.language_level +run.pure_pxd + +# Silly error with doctest matching slightly different string outputs rather than +# an actual bug but one I can't easily resolve +run.with_gil + + +# looks like a "when does the GC run?" issue - slightly surprised it's OK on pypy3 +memoryview.numpy_memoryview + +# type features that are disabled in PyPy2: +#run.test_genericclass +run.test_subclassinit diff --git a/tests/pypy_bugs.txt b/tests/pypy_bugs.txt index 77b814cf1..2ed1eb970 100644 --- a/tests/pypy_bugs.txt +++ b/tests/pypy_bugs.txt @@ -4,17 +4,60 @@ broken_exception bufaccess -memoryview -memslice +memoryview.memoryview$ sequential_parallel yield_from_pep380 memoryview_inplace_division +run.unicodemethods +run.unicode_imports +run.test_genericclass +run.tp_new +run.test_fstring +run.test_exceptions +run.test_dictviews +run.str_subclass_kwargs +run.special_method_docstrings +run.slice_ptr +compile.min_async +run.cython_includes +run.pyarray +run.test_unicode +run.__getattribute__ +run.__getattribute_subclasses__ +run.__debug__ +run.array_cimport +run.builtin_abs +run.builtincomplex +run.cdef_multiple_inheritance_errors +run.cdivision_CEP_516 +run.cyfunction +run.final_cdef_class +run.index +run.pyclass_special_methods +run.reimport_from_package +run.reimport_from_subinterpreter run.reimport_failure +pkg.cimportfrom +embedded +TestCyCache +run.ext_auto_richcmp +run.coverage_cmd +run.coverage_cmd_src_layout +run.coverage_installed_pkg +run.coverage_api +run.coverage_nogil + +# very little coroutine-related seems to work +run.test_asyncgen +run.test_coroutines_pep492 +run.async_iter_pep492 +run.embedsignatures +run.py35_asyncio_async_def +run.asyncio_generators # gc issue? -memoryview_in_subclasses external_ref_reassignment run.exttype_dealloc @@ -22,20 +65,13 @@ run.exttype_dealloc run.special_methods_T561 run.special_methods_T561_py2 -# tests for things that don't exist in cpyext -compile.pylong -run.datetime_pxd -run.datetime_cimport -run.datetime_members -run.extern_builtins_T258 -run.line_trace -run.line_profile_test -run.pstats_profile_test -run.longintrepr - -# refcounting-specific tests -double_dealloc_T796 -run.exceptionrefcount -run.capiimpl -run.refcount_in_meth - +# looks to be fixed in PyPy 7.3.0 +# TODO - remove when Travis updates +run.py_unicode_strings +run.unicodeliterals +run.unicode_identifiers +run.unicode_identifiers_import +errors.unicode_identifiers_e4 +run.tracebacks +run.fstring +run.unicode_identifiers_normalization diff --git a/tests/pypy_crash_bugs.txt b/tests/pypy_crash_bugs.txt new file mode 100644 index 000000000..779603146 --- /dev/null +++ b/tests/pypy_crash_bugs.txt @@ -0,0 +1,13 @@ +# Bugs that causes hard crashes that we certainly don't +# want to run because it will break the testsuite + +# segfault +run.fastcall +memslice + +# """Fatal RPython error: NotImplementedError +# Aborted (core dumped)""" +run.py35_pep492_interop + +# gc issue? +memoryview_in_subclasses diff --git a/tests/pypy_implementation_detail_bugs.txt b/tests/pypy_implementation_detail_bugs.txt new file mode 100644 index 000000000..ecf0b71dc --- /dev/null +++ b/tests/pypy_implementation_detail_bugs.txt @@ -0,0 +1,45 @@ +# PyPy "bugs" that are probably implementation differences from +# CPython rather than actual bugs. Therefore they aren't targets +# to be fixed (but there *may* be other details in the testfile +# that should be tested on PyPy?) + +run.starargs + +# refcounting-specific tests +double_dealloc_T796 +run.exceptionrefcount +run.capiimpl +run.refcount_in_meth +# Ideally just disable the reference-counting tests on PyPy? +run.fused_types +run.generator_frame_cycle +run.generators_in_refcycles +run.generators_py +run.parallel + +# "sys.getsizeof(object, default) will always return default on PyPy, and +# raise a TypeError if default is not provided." +buildenv + +# tests for things that don't exist in cpyext +compile.pylong +run.datetime_pxd +run.datetime_cimport +run.datetime_members +run.extern_builtins_T258 +run.line_trace +run.line_profile_test +run.pstats_profile_test +run.longintrepr + +# tests probably rely on immediate GC (although maybe the tests could be tweaked so +# only these bits don't run in PyPy?) +buffers.buffer +buffers.userbuffer +memoryview.cythonarray +memoryview.memoryview_pep484_typing +run.cpp_classes +run.cpp_classes_def + +# missing pypy feature? +matrix_multiplier diff --git a/tests/pyximport/pyximport_pyimport.srctree b/tests/pyximport/pyximport_pyimport.srctree index 69bc47988..753cef83f 100644 --- a/tests/pyximport/pyximport_pyimport.srctree +++ b/tests/pyximport/pyximport_pyimport.srctree @@ -8,7 +8,6 @@ import pyximport # blacklist for speed import pyximport.pyxbuild, Cython.Compiler.Pipeline -import distutils.core, distutils.ccompiler, distutils.command.build pyximport.install(pyximport=False, pyimport=True, build_dir=os.path.join(os.path.dirname(__file__), "TEST_TMP")) diff --git a/tests/run/addop.pyx b/tests/run/addop.pyx index 0e345d0e4..d6c3a3f65 100644 --- a/tests/run/addop.pyx +++ b/tests/run/addop.pyx @@ -166,3 +166,20 @@ def add_large_x(x): ... except TypeError: pass """ return 2**30 + x + + +def add0(x): + """ + >>> add0(0) + (0, 0) + >>> add0(1) + (1, 1) + >>> add0(-1) + (-1, -1) + >>> a, b = add0(2**32) + >>> bigint(a) + 4294967296 + >>> bigint(b) + 4294967296 + """ + return x + 0, 0 + x diff --git a/tests/run/always_allow_keywords_T295.pyx b/tests/run/always_allow_keywords_T295.pyx index 694839fcf..8e6e07739 100644 --- a/tests/run/always_allow_keywords_T295.pyx +++ b/tests/run/always_allow_keywords_T295.pyx @@ -1,7 +1,12 @@ -# ticket: 295 +# mode: run +# ticket: t295 cimport cython +import sys +IS_PY2 = sys.version_info[0] == 2 + + def assert_typeerror_no_keywords(func, *args, **kwds): # Python 3.9 produces an slightly different error message # to previous versions, so doctest isn't matching the @@ -14,13 +19,18 @@ def assert_typeerror_no_keywords(func, *args, **kwds): assert False, "call did not raise TypeError" +def func0(): + """ + >>> func0() + >>> func0(**{}) + """ + def func1(arg): """ >>> func1(None) >>> func1(*[None]) - >>> assert_typeerror_no_keywords(func1, arg=None) + >>> func1(arg=None) """ - pass @cython.always_allow_keywords(False) def func2(arg): @@ -29,7 +39,6 @@ def func2(arg): >>> func2(*[None]) >>> assert_typeerror_no_keywords(func2, arg=None) """ - pass @cython.always_allow_keywords(True) def func3(arg): @@ -40,26 +49,132 @@ def func3(arg): """ pass + cdef class A: """ - >>> A().meth1(None) - >>> A().meth1(*[None]) - >>> assert_typeerror_no_keywords(A().meth1, arg=None) - >>> A().meth2(None) - >>> A().meth2(*[None]) - >>> assert_typeerror_no_keywords(A().meth2, arg=None) - >>> A().meth3(None) - >>> A().meth3(*[None]) - >>> A().meth3(arg=None) + >>> class PyA(object): + ... def meth0(self): pass + ... def meth1(self, arg): pass + + >>> PyA().meth0() + >>> PyA.meth0(PyA()) + >>> if not IS_PY2: PyA.meth0(self=PyA()) + >>> try: PyA().meth0(self=PyA()) + ... except TypeError as exc: assert 'multiple' in str(exc), "Unexpected message: %s" % exc + ... else: assert False, "No TypeError when passing 'self' argument twice" + + >>> PyA().meth1(1) + >>> PyA.meth1(PyA(), 1) + >>> PyA.meth1(PyA(), arg=1) + >>> if not IS_PY2: PyA.meth1(self=PyA(), arg=1) """ + @cython.always_allow_keywords(False) + def meth0_nokw(self): + """ + >>> A().meth0_nokw() + >>> A().meth0_nokw(**{}) + >>> try: pass #A.meth0_nokw(self=A()) + ... except TypeError as exc: assert 'needs an argument' in str(exc), "Unexpected message: %s" % exc + ... else: pass #assert False, "No TypeError for missing 'self' positional argument" + """ + + @cython.always_allow_keywords(True) + def meth0_kw(self): + """ + >>> A().meth0_kw() + >>> A().meth0_kw(**{}) + >>> A.meth0_kw(A()) + >>> #A.meth0_kw(self=A()) + >>> try: pass #A().meth0_kw(self=A()) + ... except TypeError as exc: assert 'multiple' in str(exc), "Unexpected message: %s" % exc + ... else: pass #assert False, "No TypeError when passing 'self' argument twice" + """ + + @cython.always_allow_keywords(True) + def meth1_kw(self, arg): + """ + >>> A().meth1_kw(None) + >>> A().meth1_kw(*[None]) + >>> A().meth1_kw(arg=None) + >>> A.meth1_kw(A(), arg=None) + >>> #A.meth1_kw(self=A(), arg=None) + """ + + @cython.always_allow_keywords(False) + def meth1_nokw(self, arg): + """ + >>> A().meth1_nokw(None) + >>> A().meth1_nokw(*[None]) + >>> assert_typeerror_no_keywords(A().meth1_nokw, arg=None) + >>> assert_typeerror_no_keywords(A.meth1_nokw, A(), arg=None) + >>> try: pass # A.meth1_nokw(self=A(), arg=None) + ... except TypeError as exc: assert 'needs an argument' in str(exc), "Unexpected message: %s" % exc + ... else: pass # assert False, "No TypeError for missing 'self' positional argument" + """ + + @cython.always_allow_keywords(False) + def meth2(self, arg): + """ + >>> A().meth2(None) + >>> A().meth2(*[None]) + >>> assert_typeerror_no_keywords(A().meth2, arg=None) + """ + + @cython.always_allow_keywords(True) + def meth3(self, arg): + """ + >>> A().meth3(None) + >>> A().meth3(*[None]) + >>> A().meth3(arg=None) + """ + + +class B(object): + @cython.always_allow_keywords(False) + def meth0_nokw(self): + """ + >>> B().meth0_nokw() + >>> B().meth0_nokw(**{}) + >>> if not IS_PY2: assert_typeerror_no_keywords(B.meth0_nokw, self=B()) + """ + + @cython.always_allow_keywords(True) + def meth0_kw(self): + """ + >>> B().meth0_kw() + >>> B().meth0_kw(**{}) + >>> B.meth0_kw(B()) + >>> if not IS_PY2: B.meth0_kw(self=B()) + >>> try: B().meth0_kw(self=B()) + ... except TypeError as exc: assert 'multiple' in str(exc), "Unexpected message: %s" % exc + ... else: assert False, "No TypeError when passing 'self' argument twice" + """ + + @cython.always_allow_keywords(True) def meth1(self, arg): - pass + """ + >>> B().meth1(None) + >>> B().meth1(*[None]) + >>> B().meth1(arg=None) + >>> B.meth1(B(), arg=None) + >>> if not IS_PY2: B.meth1(self=B(), arg=None) + """ @cython.always_allow_keywords(False) def meth2(self, arg): - pass + """ + >>> B().meth2(None) + >>> B().meth2(*[None]) + >>> B.meth2(B(), None) + >>> if not IS_PY2: B.meth2(self=B(), arg=None) + >>> B().meth2(arg=None) # assert_typeerror_no_keywords(B().meth2, arg=None) -> not a cdef class! + """ @cython.always_allow_keywords(True) def meth3(self, arg): - pass + """ + >>> B().meth3(None) + >>> B().meth3(*[None]) + >>> B().meth3(arg=None) + """ diff --git a/tests/run/annotate_html.pyx b/tests/run/annotate_html.pyx index 765c2e13f..3db7bf190 100644 --- a/tests/run/annotate_html.pyx +++ b/tests/run/annotate_html.pyx @@ -11,6 +11,8 @@ >>> import re >>> assert re.search('<pre .*def.* .*mixed_test.*</pre>', html) +>>> from Cython.Compiler.Annotate import AnnotationCCodeWriter +>>> assert (AnnotationCCodeWriter.COMPLETE_CODE_TITLE not in html) # per default no complete c code """ diff --git a/tests/run/annotation_typing.pyx b/tests/run/annotation_typing.pyx index 2582c3d03..3604efe30 100644 --- a/tests/run/annotation_typing.pyx +++ b/tests/run/annotation_typing.pyx @@ -196,6 +196,37 @@ def call_exception_default(raise_exc=False): return exception_default(raise_exc) +@cython.test_assert_path_exists( + "//CFuncDefNode", + "//CFuncDefNode//DefNode", + "//CFuncDefNode[@return_type]", + "//CFuncDefNode[@return_type.is_int = True]", +) +@cython.ccall +def exception_default_uint(raise_exc : cython.bint = False) -> cython.uint: + """ + >>> print(exception_default_uint(raise_exc=False)) + 10 + >>> exception_default_uint(raise_exc=True) + Traceback (most recent call last): + ValueError: huhu! + """ + if raise_exc: + raise ValueError("huhu!") + return 10 + + +def call_exception_default_uint(raise_exc=False): + """ + >>> print(call_exception_default_uint(raise_exc=False)) + 10 + >>> call_exception_default_uint(raise_exc=True) + Traceback (most recent call last): + ValueError: huhu! + """ + return exception_default_uint(raise_exc) + + class EarlyClass(object): """ >>> a = EarlyClass(1) @@ -231,6 +262,10 @@ def py_float_default(price : float=None, ndigits=4): return price, ndigits +cdef class ClassAttribute: + cls_attr : float = 1. + + _WARNINGS = """ 8:32: Strings should no longer be used for type declarations. Use 'cython.int' etc. directly. 8:47: Dicts should no longer be used as type annotations. Use 'cython.int' etc. directly. @@ -238,11 +273,13 @@ _WARNINGS = """ 8:77: Dicts should no longer be used as type annotations. Use 'cython.int' etc. directly. 8:85: Python type declaration in signature annotation does not refer to a Python type 8:85: Strings should no longer be used for type declarations. Use 'cython.int' etc. directly. -211:44: Unknown type declaration in annotation, ignoring -218:29: Ambiguous types in annotation, ignoring +242:44: Unknown type declaration in annotation, ignoring +249:29: Ambiguous types in annotation, ignoring +266:15: Annotation ignored since class-level attributes must be Python objects. Were you trying to set up an instance attribute? # BUG: 46:6: 'pytypes_cpdef' redeclared -121:0: 'struct_io' redeclared -156:0: 'struct_convert' redeclared -175:0: 'exception_default' redeclared +120:0: 'struct_io' redeclared +149:0: 'struct_convert' redeclared +168:0: 'exception_default' redeclared +199:0: 'exception_default_uint' redeclared """ diff --git a/tests/run/args_unpacking_in_closure_T658.pyx b/tests/run/args_unpacking_in_closure_T658.pyx index 98b57743b..b52d689c9 100644 --- a/tests/run/args_unpacking_in_closure_T658.pyx +++ b/tests/run/args_unpacking_in_closure_T658.pyx @@ -1,6 +1,6 @@ # mode: run # tag: closures -# ticket: 658 +# ticket: t658 def outer(int x, *args, **kwargs): """ diff --git a/tests/run/argument_unpacking_closure_T736.py b/tests/run/argument_unpacking_closure_T736.py index 6c83d7026..80c824417 100644 --- a/tests/run/argument_unpacking_closure_T736.py +++ b/tests/run/argument_unpacking_closure_T736.py @@ -1,5 +1,5 @@ # mode: run -# ticket: 736 +# ticket: t736 # tag: default arguments, closure def default_args_for_closure(a=1, b=2): diff --git a/tests/run/arithmetic_analyse_types.pyx b/tests/run/arithmetic_analyse_types.pyx index 7d5e41666..c8c7f7ac8 100644 --- a/tests/run/arithmetic_analyse_types.pyx +++ b/tests/run/arithmetic_analyse_types.pyx @@ -1,4 +1,4 @@ -# ticket: 676 +# ticket: t676 # tag: cpp from cython cimport typeof diff --git a/tests/run/array_cimport.srctree b/tests/run/array_cimport.srctree index 4fefade5f..5951f2604 100644 --- a/tests/run/array_cimport.srctree +++ b/tests/run/array_cimport.srctree @@ -32,6 +32,6 @@ from tt cimport Foo cdef array a = array('i', [1,2,3]) cdef Foo x -print a.data.as_ints[0] +print(a.data.as_ints[0]) x = Foo(a) -print x.obj.data.as_ints[0] +print(x.obj.data.as_ints[0]) diff --git a/tests/run/assert.pyx b/tests/run/assert.pyx index 1e6cc17be..2e3dedf8e 100644 --- a/tests/run/assert.pyx +++ b/tests/run/assert.pyx @@ -2,6 +2,10 @@ cimport cython +@cython.test_assert_path_exists( + '//AssertStatNode', + '//AssertStatNode//RaiseStatNode', +) def f(a, b, int i): """ >>> f(1, 2, 1) @@ -22,7 +26,9 @@ def f(a, b, int i): @cython.test_assert_path_exists( '//AssertStatNode', - '//AssertStatNode//TupleNode') + '//AssertStatNode//RaiseStatNode', + '//AssertStatNode//RaiseStatNode//TupleNode', +) def g(a, b): """ >>> g(1, "works") @@ -38,7 +44,9 @@ def g(a, b): @cython.test_assert_path_exists( '//AssertStatNode', - '//AssertStatNode//TupleNode') + '//AssertStatNode//RaiseStatNode', + '//AssertStatNode//RaiseStatNode//TupleNode', +) def g(a, b): """ >>> g(1, "works") @@ -54,8 +62,9 @@ def g(a, b): @cython.test_assert_path_exists( '//AssertStatNode', - '//AssertStatNode//TupleNode', - '//AssertStatNode//TupleNode//TupleNode') + '//AssertStatNode//RaiseStatNode', + '//AssertStatNode//RaiseStatNode//TupleNode', + '//AssertStatNode//RaiseStatNode//TupleNode//TupleNode',) def assert_with_tuple_arg(a): """ >>> assert_with_tuple_arg(True) @@ -67,9 +76,12 @@ def assert_with_tuple_arg(a): @cython.test_assert_path_exists( - '//AssertStatNode') + '//AssertStatNode', + '//AssertStatNode//RaiseStatNode', +) @cython.test_fail_if_path_exists( - '//AssertStatNode//TupleNode') + '//AssertStatNode//TupleNode', +) def assert_with_str_arg(a): """ >>> assert_with_str_arg(True) diff --git a/tests/run/bad_c_struct_T252.pyx b/tests/run/bad_c_struct_T252.pyx index 247a55202..57b1155c2 100644 --- a/tests/run/bad_c_struct_T252.pyx +++ b/tests/run/bad_c_struct_T252.pyx @@ -1,4 +1,4 @@ -# ticket: 252 +# ticket: t252 cdef cf(default=None): return default diff --git a/tests/run/binop_reverse_methods_GH2056.pyx b/tests/run/binop_reverse_methods_GH2056.pyx new file mode 100644 index 000000000..8a89cfa38 --- /dev/null +++ b/tests/run/binop_reverse_methods_GH2056.pyx @@ -0,0 +1,85 @@ +cimport cython + +@cython.c_api_binop_methods(False) +@cython.cclass +class Base(object): + """ + >>> Base() + 2 + 'Base.__add__(Base(), 2)' + >>> 2 + Base() + 'Base.__radd__(Base(), 2)' + + >>> Base() ** 2 + 'Base.__pow__(Base(), 2, None)' + >>> 2 ** Base() + 'Base.__rpow__(Base(), 2, None)' + >>> pow(Base(), 2, 100) + 'Base.__pow__(Base(), 2, 100)' + """ + def __add__(self, other): + return "Base.__add__(%s, %s)" % (self, other) + + def __radd__(self, other): + return "Base.__radd__(%s, %s)" % (self, other) + + def __pow__(self, other, mod): + return "Base.__pow__(%s, %s, %s)" % (self, other, mod) + + def __rpow__(self, other, mod): + return "Base.__rpow__(%s, %s, %s)" % (self, other, mod) + + def __repr__(self): + return "%s()" % (self.__class__.__name__) + +@cython.c_api_binop_methods(False) +@cython.cclass +class OverloadLeft(Base): + """ + >>> OverloadLeft() + 2 + 'OverloadLeft.__add__(OverloadLeft(), 2)' + >>> 2 + OverloadLeft() + 'Base.__radd__(OverloadLeft(), 2)' + + >>> OverloadLeft() + Base() + 'OverloadLeft.__add__(OverloadLeft(), Base())' + >>> Base() + OverloadLeft() + 'Base.__add__(Base(), OverloadLeft())' + """ + def __add__(self, other): + return "OverloadLeft.__add__(%s, %s)" % (self, other) + + +@cython.c_api_binop_methods(False) +@cython.cclass +class OverloadRight(Base): + """ + >>> OverloadRight() + 2 + 'Base.__add__(OverloadRight(), 2)' + >>> 2 + OverloadRight() + 'OverloadRight.__radd__(OverloadRight(), 2)' + + >>> OverloadRight() + Base() + 'Base.__add__(OverloadRight(), Base())' + >>> Base() + OverloadRight() + 'OverloadRight.__radd__(OverloadRight(), Base())' + """ + def __radd__(self, other): + return "OverloadRight.__radd__(%s, %s)" % (self, other) + +@cython.c_api_binop_methods(True) +@cython.cclass +class OverloadCApi(Base): + """ + >>> OverloadCApi() + 2 + 'OverloadCApi.__add__(OverloadCApi(), 2)' + >>> 2 + OverloadCApi() + 'OverloadCApi.__add__(2, OverloadCApi())' + + >>> OverloadCApi() + Base() + 'OverloadCApi.__add__(OverloadCApi(), Base())' + >>> Base() + OverloadCApi() + 'OverloadCApi.__add__(Base(), OverloadCApi())' + """ + def __add__(self, other): + return "OverloadCApi.__add__(%s, %s)" % (self, other) + diff --git a/tests/run/bint_binop_T145.pyx b/tests/run/bint_binop_T145.pyx index c6df52303..f0f0ef560 100644 --- a/tests/run/bint_binop_T145.pyx +++ b/tests/run/bint_binop_T145.pyx @@ -1,4 +1,4 @@ -# ticket: 145 +# ticket: t145 cimport cython diff --git a/tests/run/bint_property_T354.pyx b/tests/run/bint_property_T354.pyx index 5a461ac76..ef4c623ee 100644 --- a/tests/run/bint_property_T354.pyx +++ b/tests/run/bint_property_T354.pyx @@ -1,4 +1,4 @@ -# ticket: 354 +# ticket: t354 cdef class Test: """ diff --git a/tests/run/bound_builtin_methods_T589.pyx b/tests/run/bound_builtin_methods_T589.pyx index 18a517cbd..1d2fb9f66 100644 --- a/tests/run/bound_builtin_methods_T589.pyx +++ b/tests/run/bound_builtin_methods_T589.pyx @@ -1,4 +1,4 @@ -# ticket: 589 +# ticket: t589 cimport cython diff --git a/tests/run/builtin_abs.pyx b/tests/run/builtin_abs.pyx index ba6351d9b..59f3a93c4 100644 --- a/tests/run/builtin_abs.pyx +++ b/tests/run/builtin_abs.pyx @@ -1,5 +1,5 @@ # mode: run -# ticket: 698 +# ticket: t698 # distutils: extra_compile_args=-fwrapv cdef extern from *: @@ -60,6 +60,27 @@ def int_abs(int a): """ return abs(a) +@cython.overflowcheck(True) +@cython.test_assert_path_exists("//ReturnStatNode//NameNode[@entry.name = 'abs']", + "//ReturnStatNode//NameNode[@entry.cname = 'abs']") +cdef int c_int_abs(int a) nogil except *: + return abs(a) + +def test_c_int_abs(int a): + """ + >>> test_c_int_abs(-5) == 5 + True + >>> test_c_int_abs(-5.1) == 5 + True + >>> test_c_int_abs(-max_int-1) #doctest: +ELLIPSIS + Traceback (most recent call last): + ... + OverflowError: ... + >>> test_c_int_abs(max_int) == abs(max_int) or (max_int, test_c_int_abs(max_int), abs(max_int)) + True + """ + return c_int_abs(a) + @cython.test_assert_path_exists("//ReturnStatNode//NameNode[@entry.name = 'abs']") @cython.test_fail_if_path_exists("//ReturnStatNode//NameNode[@entry.cname = 'abs']", "//ReturnStatNode//NameNode[@entry.cname = 'labs']") @@ -70,6 +91,19 @@ def uint_abs(unsigned int a): """ return abs(a) +@cython.test_assert_path_exists("//ReturnStatNode//NameNode[@entry.name = 'abs']") +@cython.test_fail_if_path_exists("//ReturnStatNode//NameNode[@entry.cname = 'abs']", + "//ReturnStatNode//NameNode[@entry.cname = 'labs']") +cdef unsigned int c_uint_abs(unsigned int a) nogil: + return abs(a) + +def test_c_uint_abs(unsigned int a): + """ + >>> test_c_uint_abs(max_int) == abs(max_int) or (max_int, test_c_uint_abs(max_int), abs(max_int)) + True + """ + return c_uint_abs(a) + @cython.overflowcheck(True) @cython.test_assert_path_exists("//ReturnStatNode//NameNode[@entry.name = 'abs']", "//ReturnStatNode//NameNode[@entry.cname = 'labs']") @@ -88,6 +122,27 @@ def long_abs(long a): """ return abs(a) +@cython.overflowcheck(True) +@cython.test_assert_path_exists("//ReturnStatNode//NameNode[@entry.name = 'abs']", + "//ReturnStatNode//NameNode[@entry.cname = 'labs']") +cdef long c_long_abs(long a) nogil except *: + return abs(a) + +def test_c_long_abs(long a): + """ + >>> test_c_long_abs(-5) == 5 + True + >>> test_c_long_abs(-5.1) == 5 + True + >>> test_c_long_abs(-max_long-1) #doctest: +ELLIPSIS + Traceback (most recent call last): + ... + OverflowError: ... + >>> test_c_long_abs(max_long) == abs(max_long) or (max_long, test_c_long_abs(max_long), abs(max_long)) + True + """ + return c_long_abs(a) + @cython.test_assert_path_exists("//ReturnStatNode//NameNode[@entry.name = 'abs']") @cython.test_fail_if_path_exists("//ReturnStatNode//NameNode[@entry.cname = 'abs']", "//ReturnStatNode//NameNode[@entry.cname = 'labs']") @@ -100,6 +155,21 @@ def ulong_abs(unsigned long a): """ return abs(a) +@cython.test_assert_path_exists("//ReturnStatNode//NameNode[@entry.name = 'abs']") +@cython.test_fail_if_path_exists("//ReturnStatNode//NameNode[@entry.cname = 'abs']", + "//ReturnStatNode//NameNode[@entry.cname = 'labs']") +cdef unsigned long c_ulong_abs(unsigned long a) nogil: + return abs(a) + +def test_c_ulong_abs(unsigned long a): + """ + >>> test_c_ulong_abs(max_long) == abs(max_long) or (max_int, test_c_ulong_abs(max_long), abs(max_long)) + True + >>> test_c_ulong_abs(max_long + 5) == abs(max_long + 5) or (max_long + 5, test_c_ulong_abs(max_long + 5), abs(max_long + 5)) + True + """ + return c_ulong_abs(a) + @cython.overflowcheck(True) @cython.test_assert_path_exists("//ReturnStatNode//NameNode[@entry.name = 'abs']", "//ReturnStatNode//NameNode[@entry.cname = '__Pyx_abs_longlong']") @@ -116,6 +186,25 @@ def long_long_abs(long long a): """ return abs(a) +@cython.overflowcheck(True) +@cython.test_assert_path_exists("//ReturnStatNode//NameNode[@entry.name = 'abs']", + "//ReturnStatNode//NameNode[@entry.cname = '__Pyx_abs_longlong']") +cdef long long c_long_long_abs(long long a) nogil except *: + return abs(a) + +def test_c_long_long_abs(long long a): + """ + >>> test_c_long_long_abs(-(2**33)) == 2**33 + True + >>> test_c_long_long_abs(-max_long_long-1) #doctest: +ELLIPSIS + Traceback (most recent call last): + ... + OverflowError: ... + >>> test_c_long_long_abs(max_long_long) == abs(max_long_long) or (max_long_long, test_c_long_long_abs(max_long_long), abs(max_long_long)) + True + """ + return c_long_long_abs(a) + @cython.test_assert_path_exists("//ReturnStatNode//NameNode[@entry.name = 'abs']", "//ReturnStatNode//NameNode[@entry.cname = 'fabs']") def double_abs(double a): @@ -128,6 +217,20 @@ def double_abs(double a): return abs(a) @cython.test_assert_path_exists("//ReturnStatNode//NameNode[@entry.name = 'abs']", + "//ReturnStatNode//NameNode[@entry.cname = 'fabs']") +cdef double c_double_abs(double a) nogil: + return abs(a) + +def test_c_double_abs(double a): + """ + >>> test_c_double_abs(-5) + 5.0 + >>> test_c_double_abs(-5.5) + 5.5 + """ + return c_double_abs(a) + +@cython.test_assert_path_exists("//ReturnStatNode//NameNode[@entry.name = 'abs']", "//ReturnStatNode//NameNode[@entry.cname = 'fabsf']") def float_abs(float a): """ @@ -139,6 +242,20 @@ def float_abs(float a): return abs(a) @cython.test_assert_path_exists("//ReturnStatNode//NameNode[@entry.name = 'abs']", + "//ReturnStatNode//NameNode[@entry.cname = 'fabsf']") +cdef float c_float_abs(float a) nogil: + return abs(a) + +def test_c_float_abs(float a): + """ + >>> test_c_float_abs(-5) + 5.0 + >>> test_c_float_abs(-5.5) + 5.5 + """ + return c_float_abs(a) + +@cython.test_assert_path_exists("//ReturnStatNode//NameNode[@entry.name = 'abs']", "//ReturnStatNode//NameNode[@entry.cname = '__Pyx_c_abs_double']") def complex_abs(complex a): """ @@ -148,3 +265,17 @@ def complex_abs(complex a): 5.5 """ return abs(a) + +@cython.test_assert_path_exists("//ReturnStatNode//NameNode[@entry.name = 'abs']", + "//ReturnStatNode//NameNode[@entry.cname = '__Pyx_c_abs_double']") +cdef double c_complex_abs(complex a) nogil: + return abs(a) + +def test_c_complex_abs(complex a): + """ + >>> test_c_complex_abs(-5j) + 5.0 + >>> test_c_complex_abs(-5.5j) + 5.5 + """ + return c_complex_abs(a) diff --git a/tests/run/builtin_float.py b/tests/run/builtin_float.py index f2eff2be8..dc31223bb 100644 --- a/tests/run/builtin_float.py +++ b/tests/run/builtin_float.py @@ -1,6 +1,20 @@ +# mode: run +# tag: pure3.0 +import cython import sys +def fix_underscores(s): + if sys.version_info < (3, 6) or getattr(sys, 'pypy_version_info', (9, 9)) < (3, 7, 4): + # Py2 float() does not support PEP-515 underscore literals + if isinstance(s, bytes): + if not cython.compiled and b'_' in s: + return s.replace(b'_', b'') + elif '_' in s: + return s.replace('_', '') + return s + + def empty_float(): """ >>> float() @@ -11,24 +25,212 @@ def empty_float(): x = float() return x + def float_conjugate(): """ >>> float_call_conjugate() 1.5 """ - if sys.version_info >= (2,6): - x = 1.5 .conjugate() - else: - x = 1.5 + x = 1.5 .conjugate() return x + def float_call_conjugate(): """ >>> float_call_conjugate() 1.5 """ - if sys.version_info >= (2,6): - x = float(1.5).conjugate() - else: - x = 1.5 + x = float(1.5).conjugate() return x + + +@cython.test_assert_path_exists( + "//CoerceToPyTypeNode", + "//CoerceToPyTypeNode//PythonCapiCallNode", +) +def from_bytes(s: bytes): + """ + >>> from_bytes(b"123") + 123.0 + >>> from_bytes(b"123.25") + 123.25 + >>> from_bytes(fix_underscores(b"98_5_6.2_1")) + 9856.21 + >>> from_bytes(fix_underscores(b"12_4_131_123123_1893798127398123_19238712_128937198237.8222113_519879812387")) + 1.2413112312318938e+47 + >>> from_bytes(b"123E100") + 1.23e+102 + >>> from_bytes(b"12__._3") # doctest: +ELLIPSIS + Traceback (most recent call last): + ValueError: ...12__._3... + >>> from_bytes(b"_12.3") # doctest: +ELLIPSIS + Traceback (most recent call last): + ValueError: ..._12.3... + >>> from_bytes(b"12.3_") # doctest: +ELLIPSIS + Traceback (most recent call last): + ValueError: ...12.3_... + >>> from_bytes(b"na_n") # doctest: +ELLIPSIS + Traceback (most recent call last): + ValueError: ...na_n... + >>> from_bytes(None) # doctest: +ELLIPSIS + Traceback (most recent call last): + TypeError... + """ + return float(s) + + +@cython.test_assert_path_exists( + "//CoerceToPyTypeNode", + "//CoerceToPyTypeNode//PythonCapiCallNode", +) +def from_bytes_literals(): + """ + >>> from_bytes_literals() + (123.0, 123.23, 123.76, 1e+100) + """ + return float(b"123"), float(b"123.23"), float(fix_underscores(b"12_3.7_6")), float(b"1e100") + + +@cython.test_assert_path_exists( + "//CoerceToPyTypeNode", + "//CoerceToPyTypeNode//PythonCapiCallNode", +) +def from_bytearray(s: bytearray): + """ + >>> from_bytearray(bytearray(b"123")) + 123.0 + >>> from_bytearray(bytearray(b"123.25")) + 123.25 + >>> from_bytearray(bytearray(fix_underscores(b"98_5_6.2_1"))) + 9856.21 + >>> from_bytearray(bytearray(fix_underscores(b"12_4_131_123123_1893798127398123_19238712_128937198237.8222113_519879812387"))) + 1.2413112312318938e+47 + >>> from_bytearray(bytearray(b"123E100")) + 1.23e+102 + >>> from_bytearray(bytearray(b"12__._3")) # doctest: +ELLIPSIS + Traceback (most recent call last): + ValueError: ...12__._3... + >>> from_bytearray(bytearray(b"_12.3")) # doctest: +ELLIPSIS + Traceback (most recent call last): + ValueError: ..._12.3... + >>> from_bytearray(bytearray(b"12.3_")) # doctest: +ELLIPSIS + Traceback (most recent call last): + ValueError: ...12.3_... + >>> from_bytearray(bytearray(b"in_f")) # doctest: +ELLIPSIS + Traceback (most recent call last): + ValueError: ...in_f... + >>> from_bytearray(None) # doctest: +ELLIPSIS + Traceback (most recent call last): + TypeError... + """ + return float(s) + + +@cython.test_assert_path_exists( + "//CoerceToPyTypeNode", + "//CoerceToPyTypeNode//PythonCapiCallNode", +) +def from_str(s: 'str'): + """ + >>> from_str("123") + 123.0 + >>> from_str("123.25") + 123.25 + >>> from_str(fix_underscores("3_21.2_5")) + 321.25 + >>> from_str(fix_underscores("12_4_131_123123_1893798127398123_19238712_128937198237.8222113_519879812387")) + 1.2413112312318938e+47 + >>> from_str("123E100") + 1.23e+102 + >>> from_str("12__._3") # doctest: +ELLIPSIS + Traceback (most recent call last): + ValueError: ...12__._3... + >>> from_str("_12.3") # doctest: +ELLIPSIS + Traceback (most recent call last): + ValueError: ..._12.3... + >>> from_str("12.3_") # doctest: +ELLIPSIS + Traceback (most recent call last): + ValueError: ...12.3_... + >>> from_str("n_an") # doctest: +ELLIPSIS + Traceback (most recent call last): + ValueError: ...n_an... + >>> from_str(None) # doctest: +ELLIPSIS + Traceback (most recent call last): + TypeError... + """ + return float(s) + + +@cython.test_assert_path_exists( + "//CoerceToPyTypeNode", + "//CoerceToPyTypeNode//PythonCapiCallNode", +) +def from_str_literals(): + """ + >>> from_str_literals() + (123.0, 123.23, 124.23, 1e+100) + """ + return float("123"), float("123.23"), float(fix_underscores("1_2_4.2_3")), float("1e100") + + +@cython.test_assert_path_exists( + "//CoerceToPyTypeNode", + "//CoerceToPyTypeNode//PythonCapiCallNode", +) +def from_unicode(s: 'unicode'): + """ + >>> from_unicode(u"123") + 123.0 + >>> from_unicode(u"123.25") + 123.25 + >>> from_unicode(fix_underscores(u"12_4.8_5")) + 124.85 + >>> from_unicode(fix_underscores(u"12_4_131_123123_1893798127398123_19238712_128937198237.8222113_519879812387")) + 1.2413112312318938e+47 + >>> from_unicode(u"123E100") + 1.23e+102 + >>> from_unicode(u"123.23\\N{PUNCTUATION SPACE}") + 123.23 + >>> from_unicode(u"\\N{PUNCTUATION SPACE} 123.23 \\N{PUNCTUATION SPACE}") + 123.23 + >>> from_unicode(fix_underscores(u"\\N{PUNCTUATION SPACE} 12_3.2_3 \\N{PUNCTUATION SPACE}")) + 123.23 + >>> from_unicode(u"\\N{PUNCTUATION SPACE} " * 25 + u"123.54 " + u"\\N{PUNCTUATION SPACE} " * 22) # >= 40 chars + 123.54 + >>> from_unicode(fix_underscores(u"\\N{PUNCTUATION SPACE} " * 25 + u"1_23.5_4 " + u"\\N{PUNCTUATION SPACE} " * 22)) + 123.54 + >>> from_unicode(u"\\N{PUNCTUATION SPACE} " + u"123.54 " * 2 + u"\\N{PUNCTUATION SPACE}") # doctest: +ELLIPSIS + Traceback (most recent call last): + ValueError: ...123.54 123.54... + >>> from_unicode(u"\\N{PUNCTUATION SPACE} " * 25 + u"123.54 " * 2 + u"\\N{PUNCTUATION SPACE} " * 22) # doctest: +ELLIPSIS + Traceback (most recent call last): + ValueError: ...123.54 123.54... + >>> from_unicode(u"_12__._3") # doctest: +ELLIPSIS + Traceback (most recent call last): + ValueError: ..._12__._3... + >>> from_unicode(u"_12.3") # doctest: +ELLIPSIS + Traceback (most recent call last): + ValueError: ..._12.3... + >>> from_unicode(u"12.3_") # doctest: +ELLIPSIS + Traceback (most recent call last): + ValueError: ...12.3_... + >>> from_unicode(u"i_nf") # doctest: +ELLIPSIS + Traceback (most recent call last): + ValueError: ...i_nf... + >>> from_unicode(None) # doctest: +ELLIPSIS + Traceback (most recent call last): + TypeError... + """ + return float(s) + + +@cython.test_assert_path_exists( + "//CoerceToPyTypeNode", + "//CoerceToPyTypeNode//PythonCapiCallNode", +) +def from_unicode_literals(): + """ + >>> from_unicode_literals() + (123.0, 123.23, 123.45, 1e+100, 123.23) + """ + return float(u"123"), float(u"123.23"), float(fix_underscores(u"12_3.4_5")), float(u"1e100"), float(u"123.23\N{PUNCTUATION SPACE}") diff --git a/tests/run/builtin_methods_return_values.pyx b/tests/run/builtin_methods_return_values.pyx index 50f1427ca..09a25273c 100644 --- a/tests/run/builtin_methods_return_values.pyx +++ b/tests/run/builtin_methods_return_values.pyx @@ -1,6 +1,6 @@ # mode: run # tag: list, set, builtins -# ticket: 688 +# ticket: t688 _set = set diff --git a/tests/run/builtin_subtype_methods_T653.pyx b/tests/run/builtin_subtype_methods_T653.pyx index bcfda81f0..522cffff8 100644 --- a/tests/run/builtin_subtype_methods_T653.pyx +++ b/tests/run/builtin_subtype_methods_T653.pyx @@ -1,6 +1,6 @@ #cython: language_level=2 # mode: run -# ticket: 653 +# ticket: t653 cimport cython diff --git a/tests/run/builtin_subtype_methods_cy3.pyx b/tests/run/builtin_subtype_methods_cy3.pyx index 1cff75fb7..1f54ac598 100644 --- a/tests/run/builtin_subtype_methods_cy3.pyx +++ b/tests/run/builtin_subtype_methods_cy3.pyx @@ -1,6 +1,6 @@ # cython: language_level=3 # mode: run -# ticket: 653 +# ticket: t653 class DictPySubtype(dict): diff --git a/tests/run/builtin_type_inheritance_T608.pyx b/tests/run/builtin_type_inheritance_T608.pyx index 67e97ec1e..1214b6841 100644 --- a/tests/run/builtin_type_inheritance_T608.pyx +++ b/tests/run/builtin_type_inheritance_T608.pyx @@ -1,4 +1,4 @@ -# ticket: 608 +# ticket: t608 cdef class MyInt(int): """ diff --git a/tests/run/builtin_types_none_T166.pyx b/tests/run/builtin_types_none_T166.pyx index 33cabffa8..276f52724 100644 --- a/tests/run/builtin_types_none_T166.pyx +++ b/tests/run/builtin_types_none_T166.pyx @@ -1,4 +1,4 @@ -# ticket: 166 +# ticket: t166 __doc__ = u""" >>> l = None diff --git a/tests/run/bytearray_iter.py b/tests/run/bytearray_iter.py new file mode 100644 index 000000000..1865f057b --- /dev/null +++ b/tests/run/bytearray_iter.py @@ -0,0 +1,90 @@ +# mode: run +# tag: pure3, pure2 + +import cython + +@cython.test_assert_path_exists("//ForFromStatNode") +@cython.test_fail_if_path_exists("//ForInStatNode") +@cython.locals(x=bytearray) +def basic_bytearray_iter(x): + """ + >>> basic_bytearray_iter(bytearray(b"hello")) + h + e + l + l + o + """ + for a in x: + print(chr(a)) + +@cython.test_assert_path_exists("//ForFromStatNode") +@cython.test_fail_if_path_exists("//ForInStatNode") +@cython.locals(x=bytearray) +def reversed_bytearray_iter(x): + """ + >>> reversed_bytearray_iter(bytearray(b"hello")) + o + l + l + e + h + """ + for a in reversed(x): + print(chr(a)) + +@cython.test_assert_path_exists("//ForFromStatNode") +@cython.test_fail_if_path_exists("//ForInStatNode") +@cython.locals(x=bytearray) +def modifying_bytearray_iter1(x): + """ + >>> modifying_bytearray_iter1(bytearray(b"abcdef")) + a + b + c + 3 + """ + count = 0 + for a in x: + print(chr(a)) + del x[-1] + count += 1 + print(count) + +@cython.test_assert_path_exists("//ForFromStatNode") +@cython.test_fail_if_path_exists("//ForInStatNode") +@cython.locals(x=bytearray) +def modifying_bytearray_iter2(x): + """ + >>> modifying_bytearray_iter2(bytearray(b"abcdef")) + a + c + e + 3 + """ + count = 0 + for a in x: + print(chr(a)) + del x[0] + count += 1 + print(count) + +@cython.test_assert_path_exists("//ForFromStatNode") +@cython.test_fail_if_path_exists("//ForInStatNode") +@cython.locals(x=bytearray) +def modifying_reversed_bytearray_iter(x): + """ + NOTE - I'm not 100% sure how well-defined this behaviour is in Python. + However, for the moment Python and Cython seem to do the same thing. + Testing that it doesn't crash is probably more important than the exact output! + >>> modifying_reversed_bytearray_iter(bytearray(b"abcdef")) + f + f + f + f + f + f + """ + for a in reversed(x): + print(chr(a)) + del x[0] diff --git a/tests/run/c_int_types_T255.pyx b/tests/run/c_int_types_T255.pyx index eec5d5e10..58b7839d9 100644 --- a/tests/run/c_int_types_T255.pyx +++ b/tests/run/c_int_types_T255.pyx @@ -1,4 +1,4 @@ -# ticket: 255 +# ticket: t255 __doc__ = u"" @@ -685,14 +685,13 @@ class MyBadInt2(MyInt2): def test_convert_pyint(x): u""" - >>> test_convert_pyint(None) + >>> test_convert_pyint(None) # doctest: +ELLIPSIS Traceback (most recent call last): - ... - TypeError: an integer is required - >>> test_convert_pyint("123") + TypeError:... int... + >>> test_convert_pyint("123") # doctest: +ELLIPSIS Traceback (most recent call last): ... - TypeError: an integer is required + TypeError:... int... >>> test_convert_pyint(MyBadInt(0)) #doctest: +ELLIPSIS Traceback (most recent call last): ... @@ -733,14 +732,14 @@ class MyBadLong(MyLong): def test_convert_pylong(x): u""" - >>> test_convert_pylong(None) + >>> test_convert_pylong(None) # doctest: +ELLIPSIS Traceback (most recent call last): ... - TypeError: an integer is required - >>> test_convert_pylong("123") + TypeError:... int... + >>> test_convert_pylong("123") # doctest: +ELLIPSIS Traceback (most recent call last): ... - TypeError: an integer is required + TypeError:... int... >>> test_convert_pylong(MyBadLong(0)) #doctest: +ELLIPSIS Traceback (most recent call last): ... diff --git a/tests/run/c_type_methods_T236.pyx b/tests/run/c_type_methods_T236.pyx index 5ac22feae..b1ef5d023 100644 --- a/tests/run/c_type_methods_T236.pyx +++ b/tests/run/c_type_methods_T236.pyx @@ -1,10 +1,8 @@ -# ticket: 236 - -__doc__ = '' +# ticket: t236 import sys -if sys.version_info >= (2,6): - __doc__ += ''' + +__doc__ = ''' >>> float_is_integer(1.0) True >>> float_is_integer(1.1) @@ -19,7 +17,6 @@ True ''' def float_is_integer(float f): - # requires Python 2.6+ return f.is_integer() def int_bit_length(int i): diff --git a/tests/run/callargs.pyx b/tests/run/callargs.pyx index 05f3f639b..c87f57962 100644 --- a/tests/run/callargs.pyx +++ b/tests/run/callargs.pyx @@ -168,15 +168,15 @@ def test_int_kwargs(f): """ >>> test_int_kwargs(e) # doctest: +ELLIPSIS Traceback (most recent call last): - TypeError: ...keywords must be strings + TypeError: ...keywords must be strings... >>> test_int_kwargs(f) # doctest: +ELLIPSIS Traceback (most recent call last): - TypeError: ...keywords must be strings + TypeError: ...keywords must be strings... >>> test_int_kwargs(g) # doctest: +ELLIPSIS Traceback (most recent call last): - TypeError: ...keywords must be strings + TypeError: ...keywords must be strings... >>> test_int_kwargs(h) # doctest: +ELLIPSIS Traceback (most recent call last): - TypeError: ...keywords must be strings + TypeError: ...keywords must be strings... """ f(a=1,b=2,c=3, **{10:20,30:40}) diff --git a/tests/run/cascaded_list_unpacking_T467.pyx b/tests/run/cascaded_list_unpacking_T467.pyx index 3008bddaf..fc0963d0d 100644 --- a/tests/run/cascaded_list_unpacking_T467.pyx +++ b/tests/run/cascaded_list_unpacking_T467.pyx @@ -1,4 +1,4 @@ -# ticket: 467 +# ticket: t467 def simple_parallel_assignment_from_call(): """ diff --git a/tests/run/cascaded_typed_assignments_T466.pyx b/tests/run/cascaded_typed_assignments_T466.pyx index eef9d5b7c..c2081bf91 100644 --- a/tests/run/cascaded_typed_assignments_T466.pyx +++ b/tests/run/cascaded_typed_assignments_T466.pyx @@ -1,5 +1,5 @@ # mode: run -# ticket: 466 +# ticket: t466 # extension to T409 cimport cython diff --git a/tests/run/cascadedassignment.pyx b/tests/run/cascadedassignment.pyx index 5bde089f7..ad606fe14 100644 --- a/tests/run/cascadedassignment.pyx +++ b/tests/run/cascadedassignment.pyx @@ -56,3 +56,15 @@ def test_cascaded_assignment_evaluate_expr(): """ a = b = c = float(expr()) return a, b, c + + +def test_overwrite(): + """ + >>> test_overwrite() + {0: {1: {2: {}}}} + """ + x = a = {} + for i in range(3): + a[i] = a = {} + assert a == {} + return x diff --git a/tests/run/cascmp.pyx b/tests/run/cascmp.pyx new file mode 100644 index 000000000..becae3fc5 --- /dev/null +++ b/tests/run/cascmp.pyx @@ -0,0 +1,38 @@ +# mode: run +# tag: cascade, compare + +def ints_and_objects(): + """ + >>> ints_and_objects() + (0, 1, 0, 1, 1, 0) + """ + cdef int int1=0, int2=0, int3=0, int4=0 + cdef int r1, r2, r3, r4, r5, r6 + cdef object obj1, obj2, obj3, obj4 + obj1 = 1 + obj2 = 2 + obj3 = 3 + obj4 = 4 + r1 = int1 < int2 < int3 + r2 = obj1 < obj2 < obj3 + r3 = int1 < int2 < obj3 + r4 = obj1 < 2 < 3 + r5 = obj1 < 2 < 3 < 4 + r6 = int1 < (int2 == int3) < int4 + return r1, r2, r3, r4, r5, r6 + + +def const_cascade(x): + """ + >>> const_cascade(2) + (True, False, True, False, False, True, False) + """ + return ( + 0 <= 1, + 1 <= 0, + 1 <= 1 <= 2, + 1 <= 0 < 1, + 1 <= 1 <= 0, + 1 <= 1 <= x <= 2 <= 3 > x <= 2 <= 2, + 1 <= 1 <= x <= 1 <= 1 <= x <= 2, + ) diff --git a/tests/run/cclass_assign_attr_GH3100.pyx b/tests/run/cclass_assign_attr_GH3100.pyx new file mode 100644 index 000000000..21785b81a --- /dev/null +++ b/tests/run/cclass_assign_attr_GH3100.pyx @@ -0,0 +1,19 @@ +cdef class Foo: + """ + >>> D = Foo.__dict__ + >>> D["meth"] is D["meth2"] + True + >>> D["classmeth"] is D["classmeth2"] + True + >>> D["staticmeth"] is D["staticmeth2"] + True + """ + def meth(self): pass + @classmethod + def classmeth(cls): pass + @staticmethod + def staticmeth(): pass + + meth2 = meth + classmeth2 = classmeth + staticmeth2 = staticmeth diff --git a/tests/run/cdef_bool_T227.pyx b/tests/run/cdef_bool_T227.pyx index 889963951..be403e695 100644 --- a/tests/run/cdef_bool_T227.pyx +++ b/tests/run/cdef_bool_T227.pyx @@ -1,4 +1,4 @@ -# ticket: 227 +# ticket: t227 from cpython.bool cimport bool diff --git a/tests/run/cdef_class_field.pyx b/tests/run/cdef_class_field.pyx index 9fb745ecb..5012bab89 100644 --- a/tests/run/cdef_class_field.pyx +++ b/tests/run/cdef_class_field.pyx @@ -1,6 +1,6 @@ # mode: run # tag: exttype -# ticket: 677 +# ticket: t677 """ >>> str(Foo(4)) diff --git a/tests/run/cdef_class_property_decorator_T264.pyx b/tests/run/cdef_class_property_decorator_T264.pyx index 421f762bc..b53bd7ec1 100644 --- a/tests/run/cdef_class_property_decorator_T264.pyx +++ b/tests/run/cdef_class_property_decorator_T264.pyx @@ -1,5 +1,5 @@ # mode: run -# ticket: 264 +# ticket: t264 # tag: property, decorator my_property = property diff --git a/tests/run/cdef_decorator_directives_T183.pyx b/tests/run/cdef_decorator_directives_T183.pyx index 84b05a8df..dc362b255 100644 --- a/tests/run/cdef_decorator_directives_T183.pyx +++ b/tests/run/cdef_decorator_directives_T183.pyx @@ -1,4 +1,4 @@ -# ticket: 183 +# ticket: t183 cimport cython diff --git a/tests/run/cdef_locals_decorator_T477.pyx b/tests/run/cdef_locals_decorator_T477.pyx index 2b40d05ad..5af71e748 100644 --- a/tests/run/cdef_locals_decorator_T477.pyx +++ b/tests/run/cdef_locals_decorator_T477.pyx @@ -1,4 +1,4 @@ -# ticket: 477 +# ticket: t477 import cython @cython.locals(x=double) diff --git a/tests/run/cdef_members_T517.pyx b/tests/run/cdef_members_T517.pyx index d69d72ca4..389879f67 100644 --- a/tests/run/cdef_members_T517.pyx +++ b/tests/run/cdef_members_T517.pyx @@ -1,4 +1,4 @@ -# ticket: 517 +# ticket: t517 #cython: embedsignature=True __doc__ = u""" diff --git a/tests/run/cdef_methods_T462.pyx b/tests/run/cdef_methods_T462.pyx index dde15adbc..80bfa4b6f 100644 --- a/tests/run/cdef_methods_T462.pyx +++ b/tests/run/cdef_methods_T462.pyx @@ -1,4 +1,4 @@ -# ticket: 462 +# ticket: t462 cimport cython diff --git a/tests/run/cdef_multiple_inheritance.pyx b/tests/run/cdef_multiple_inheritance.pyx index 4213d35d3..0f8eaef24 100644 --- a/tests/run/cdef_multiple_inheritance.pyx +++ b/tests/run/cdef_multiple_inheritance.pyx @@ -1,3 +1,5 @@ +cimport cython + cdef class CBase(object): cdef int a cdef c_method(self): @@ -9,7 +11,8 @@ class PyBase(object): def py_method(self): return "PyBase" -cdef class Both(CBase, PyBase): +@cython.binding(True) +cdef class BothBound(CBase, PyBase): cdef dict __dict__ """ >>> b = Both() @@ -32,7 +35,7 @@ cdef class Both(CBase, PyBase): def call_c_method(self): return self.c_method() -cdef class BothSub(Both): +cdef class BothSub(BothBound): """ >>> b = BothSub() >>> b.py_method() @@ -43,3 +46,27 @@ cdef class BothSub(Both): 'Both' """ pass + +@cython.binding(False) +cdef class BothUnbound(CBase, PyBase): + cdef dict __dict__ + """ + >>> b = Both() + >>> b.py_method() + 'PyBase' + >>> b.cp_method() + 'Both' + >>> b.call_c_method() + 'Both' + + >>> isinstance(b, CBase) + True + >>> isinstance(b, PyBase) + True + """ + cdef c_method(self): + return "Both" + cpdef cp_method(self): + return "Both" + def call_c_method(self): + return self.c_method() diff --git a/tests/run/cdef_multiple_inheritance_errors.srctree b/tests/run/cdef_multiple_inheritance_errors.srctree index e6b3426ba..dd4aaa200 100644 --- a/tests/run/cdef_multiple_inheritance_errors.srctree +++ b/tests/run/cdef_multiple_inheritance_errors.srctree @@ -48,6 +48,7 @@ cdef class X(Base, Py): pass ######## oldstyle.pyx ######## +# cython: language_level=2 cdef class Base: cdef dict __dict__ diff --git a/tests/run/cdef_setitem_T284.pyx b/tests/run/cdef_setitem_T284.pyx index 2c885d5be..389b8c409 100644 --- a/tests/run/cdef_setitem_T284.pyx +++ b/tests/run/cdef_setitem_T284.pyx @@ -1,4 +1,4 @@ -# ticket: 284 +# ticket: t284 def no_cdef(): """ diff --git a/tests/run/cdivision_CEP_516.pyx b/tests/run/cdivision_CEP_516.pyx index c8b24a0e1..fbd2def3a 100644 --- a/tests/run/cdivision_CEP_516.pyx +++ b/tests/run/cdivision_CEP_516.pyx @@ -27,6 +27,9 @@ True >>> [test_cdiv_cmod(a, b) for a, b in v] [(1, 7), (-1, -7), (1, -7), (-1, 7)] +>>> [test_cdiv_cmod(a, b) for a, b in [(4, -4), (4, -2), (4, -1)]] +[(-1, 0), (-2, 0), (-4, 0)] + >>> all([mod_int_py(a,b) == a % b for a in range(-10, 10) for b in range(-10, 10) if b != 0]) True >>> all([div_int_py(a,b) == a // b for a in range(-10, 10) for b in range(-10, 10) if b != 0]) diff --git a/tests/run/cfunc_call_tuple_args_T408.pyx b/tests/run/cfunc_call_tuple_args_T408.pyx index 165329737..e32eb036c 100644 --- a/tests/run/cfunc_call_tuple_args_T408.pyx +++ b/tests/run/cfunc_call_tuple_args_T408.pyx @@ -1,4 +1,4 @@ -# ticket: 408 +# ticket: t408 __doc__ = """ >>> call_with_tuple(1, 1.2, 'test', [1,2,3]) diff --git a/tests/run/cfunc_convert.pyx b/tests/run/cfunc_convert.pyx index 3391cd226..6db0765d4 100644 --- a/tests/run/cfunc_convert.pyx +++ b/tests/run/cfunc_convert.pyx @@ -1,4 +1,5 @@ # mode: run +# tag: autowrap # cython: always_allow_keywords=True cimport cython @@ -82,7 +83,7 @@ def test_global(): cdef long long rad(long long x): cdef long long rad = 1 - for p in range(2, <long long>sqrt(x) + 1): + for p in range(2, <long long>sqrt(<double>x) + 1): # MSVC++ fails without the input cast if x % p == 0: rad *= p while x % p == 0: @@ -229,3 +230,39 @@ def test_cdef_class_params(a, b): TypeError: Argument 'b' has incorrect type (expected cfunc_convert.B, got cfunc_convert.A) """ return (<object>test_cdef_class_params_cfunc)(a, b) + +# There were a few cases where duplicate utility code definitions (i.e. with the same name) +# could be generated, causing C compile errors. This file tests them. + +cdef cfunc_dup_f1(x, r): + return "f1" + +cdef cfunc_dup_f2(x1, r): + return "f2" + +def make_map(): + """ + https://github.com/cython/cython/issues/3716 + This is testing the generation of wrappers for f1 and f2 + >>> for k, f in make_map().items(): + ... print(k == f(0, 0)) # in both cases the functions should just return their name + True + True + + # Test passing of keyword arguments + >>> print(make_map()['f1'](x=1, r=2)) + f1 + >>> make_map()['f1'](x1=1, r=2) # doctest: +ELLIPSIS + Traceback (most recent call last): + TypeError: ... + >>> print(make_map()['f2'](x1=1, r=2)) + f2 + >>> make_map()['f2'](x=1, r=2) # doctest: +ELLIPSIS + Traceback (most recent call last): + TypeError: ... + """ + cdef map = { + "f1": cfunc_dup_f1, + "f2": cfunc_dup_f2, + } + return map diff --git a/tests/run/char_constants_T99.pyx b/tests/run/char_constants_T99.pyx index aa2550b52..241b461a1 100644 --- a/tests/run/char_constants_T99.pyx +++ b/tests/run/char_constants_T99.pyx @@ -1,4 +1,4 @@ -# ticket: 99 +# ticket: t99 cdef char c = 'c' cdef char* s = 'abcdef' diff --git a/tests/run/charcomparisonT412.pyx b/tests/run/charcomparisonT412.pyx index b9316a36c..97afb4769 100644 --- a/tests/run/charcomparisonT412.pyx +++ b/tests/run/charcomparisonT412.pyx @@ -1,4 +1,4 @@ -# ticket: 412 +# ticket: t412 def f(): """ diff --git a/tests/run/charptr_comparison_T582.pyx b/tests/run/charptr_comparison_T582.pyx index 3701921cf..1e5e43cd2 100644 --- a/tests/run/charptr_comparison_T582.pyx +++ b/tests/run/charptr_comparison_T582.pyx @@ -1,4 +1,4 @@ -# ticket: 582 +# ticket: t582 cimport cython diff --git a/tests/run/cimport_cython_T505.pyx b/tests/run/cimport_cython_T505.pyx index 20d2daf3a..69229856f 100644 --- a/tests/run/cimport_cython_T505.pyx +++ b/tests/run/cimport_cython_T505.pyx @@ -1,4 +1,4 @@ -# ticket: 505 +# ticket: t505 cimport cython diff --git a/tests/run/cimport_from_pyx.srctree b/tests/run/cimport_from_pyx.srctree index 602188748..d25e21b10 100644 --- a/tests/run/cimport_from_pyx.srctree +++ b/tests/run/cimport_from_pyx.srctree @@ -15,14 +15,22 @@ setup( ######## a.pyx ######## -from b cimport Bclass, Bfunc, Bstruct, Benum, Benum_value, Btypedef, Py_EQ, Py_NE +from b cimport (Bclass, Bfunc, Bstruct, Benum, Benum_value, Btypedef, Py_EQ, Py_NE, + DecoratedClass, cfuncOutside) cdef Bclass b = Bclass(5) assert Bfunc(&b.value) == b.value +assert b.anotherValue == 6, b.anotherValue assert b.asStruct().value == b.value cdef Btypedef b_type = &b.value cdef Benum b_enum = Benum_value cdef int tmp = Py_EQ +cdef DecoratedClass dc = DecoratedClass() +assert dc.cfuncInClass().value == 5 +assert dc.cpdefInClass() == 1.0 + +assert cfuncOutside().value == 2 + #from c cimport ClassC #cdef ClassC c = ClassC() #print c.value @@ -31,6 +39,8 @@ cdef int tmp = Py_EQ from cpython.object cimport Py_EQ, Py_NE +cimport cython + cdef enum Benum: Benum_value @@ -41,14 +51,34 @@ ctypedef long *Btypedef cdef class Bclass: cdef long value + anotherValue: cython.double def __init__(self, value): self.value = value + self.anotherValue = value + 1 cdef Bstruct asStruct(self): return Bstruct(value=self.value) + cdef double getOtherValue(self): + return self.anotherValue cdef long Bfunc(Btypedef x): return x[0] +@cython.cclass +class DecoratedClass: + @cython.cfunc + @cython.returns(Bstruct) + def cfuncInClass(self): + return Bstruct(value=5) + @cython.ccall + @cython.returns(cython.double) + def cpdefInClass(self): + return 1.0 + +@cython.cfunc +@cython.returns(Bstruct) +def cfuncOutside(): + return Bstruct(value=2) + ######## c.pxd ######## cdef class ClassC: diff --git a/tests/run/cimport_from_sys_path.srctree b/tests/run/cimport_from_sys_path.srctree index e6f619d78..e1541d57c 100644 --- a/tests/run/cimport_from_sys_path.srctree +++ b/tests/run/cimport_from_sys_path.srctree @@ -32,7 +32,7 @@ static int foo(int a) ######## a.pyx ######## from b.other cimport foo -print foo(10) +print(foo(10)) cimport b.other -print b.other.foo(10) +print(b.other.foo(10)) diff --git a/tests/run/class_attribute_init_values_T18.pyx b/tests/run/class_attribute_init_values_T18.pyx index 5e12f665f..7887853df 100644 --- a/tests/run/class_attribute_init_values_T18.pyx +++ b/tests/run/class_attribute_init_values_T18.pyx @@ -1,4 +1,4 @@ -# ticket: 18 +# ticket: t18 __doc__ = u""" >>> f = PyFoo() diff --git a/tests/run/class_func_in_control_structures_T87.pyx b/tests/run/class_func_in_control_structures_T87.pyx index 5c23ceff9..14ee440a0 100644 --- a/tests/run/class_func_in_control_structures_T87.pyx +++ b/tests/run/class_func_in_control_structures_T87.pyx @@ -1,4 +1,4 @@ -# ticket: 87 +# ticket: t87 __doc__ = u""" >>> d = Defined() diff --git a/tests/run/class_scope_del_T684.py b/tests/run/class_scope_del_T684.py index 43368f333..f52b31864 100644 --- a/tests/run/class_scope_del_T684.py +++ b/tests/run/class_scope_del_T684.py @@ -1,6 +1,6 @@ # mode:run # tag: class, scope, del -# ticket: 684 +# ticket: t684 class DelInClass(object): """ diff --git a/tests/run/classdecorators_T336.pyx b/tests/run/classdecorators_T336.pyx index c29c1cbaa..b56e08c07 100644 --- a/tests/run/classdecorators_T336.pyx +++ b/tests/run/classdecorators_T336.pyx @@ -1,4 +1,4 @@ -# ticket: 336 +# ticket: t336 __doc__ = u""" >>> print('\\n'.join(calls)) diff --git a/tests/run/clear_to_null.pyx b/tests/run/clear_to_null.pyx index c8a355570..c54faa583 100644 --- a/tests/run/clear_to_null.pyx +++ b/tests/run/clear_to_null.pyx @@ -2,7 +2,7 @@ Check that Cython generates a tp_clear function that actually clears object references to NULL instead of None. -Discussed here: http://article.gmane.org/gmane.comp.python.cython.devel/14833 +Discussed here: https://article.gmane.org/gmane.comp.python.cython.devel/14833 """ from cpython.ref cimport PyObject, Py_TYPE diff --git a/tests/run/closure_class_T596.pyx b/tests/run/closure_class_T596.pyx index 6a92d9f82..7808a8d9e 100644 --- a/tests/run/closure_class_T596.pyx +++ b/tests/run/closure_class_T596.pyx @@ -1,6 +1,6 @@ # mode: run # tag: closures -# ticket: 596 +# ticket: t596 def simple(a, b): """ diff --git a/tests/run/closure_decorators_T478.pyx b/tests/run/closure_decorators_T478.pyx index e1c5f4918..1bccdcc19 100644 --- a/tests/run/closure_decorators_T478.pyx +++ b/tests/run/closure_decorators_T478.pyx @@ -1,6 +1,6 @@ # mode: run # tag: closures -# ticket: 478 +# ticket: t478 __doc__ = """ >>> Num(13).is_prime() diff --git a/tests/run/closure_inside_cdef_T554.pyx b/tests/run/closure_inside_cdef_T554.pyx index 3a112868d..3406259b5 100644 --- a/tests/run/closure_inside_cdef_T554.pyx +++ b/tests/run/closure_inside_cdef_T554.pyx @@ -1,6 +1,6 @@ # mode: run # tag: closures -# ticket: 554 +# ticket: t554 def call_f(x): """ diff --git a/tests/run/closure_name_mangling_T537.pyx b/tests/run/closure_name_mangling_T537.pyx index 0324109ec..148f9f1ae 100644 --- a/tests/run/closure_name_mangling_T537.pyx +++ b/tests/run/closure_name_mangling_T537.pyx @@ -1,6 +1,6 @@ # mode: run # tag: closures -# ticket: 537 +# ticket: t537 __doc__ = u""" >>> f1 = nested1() diff --git a/tests/run/closure_names.pyx b/tests/run/closure_names.pyx index b6d253983..0ceb320d5 100644 --- a/tests/run/closure_names.pyx +++ b/tests/run/closure_names.pyx @@ -1,6 +1,6 @@ # mode: run # tag: closures -# ticket: gh-1797 +# ticket: 1797 def func(): diff --git a/tests/run/closures_T82.pyx b/tests/run/closures_T82.pyx index 85ce9043e..c8a228a19 100644 --- a/tests/run/closures_T82.pyx +++ b/tests/run/closures_T82.pyx @@ -1,6 +1,6 @@ # mode: run # tag: closures -# ticket: 82 +# ticket: t82 # preparse: id # preparse: def_to_cdef diff --git a/tests/run/cmethod_inline_T474.pyx b/tests/run/cmethod_inline_T474.pyx index ddc33a62b..d09db3bf4 100644 --- a/tests/run/cmethod_inline_T474.pyx +++ b/tests/run/cmethod_inline_T474.pyx @@ -1,5 +1,5 @@ # mode: run -# ticket: 474 +# ticket: t474 cimport cython diff --git a/tests/run/common_utility_types.srctree b/tests/run/common_utility_types.srctree index 1f3d2aea3..35daad505 100644 --- a/tests/run/common_utility_types.srctree +++ b/tests/run/common_utility_types.srctree @@ -31,7 +31,7 @@ print("importing...") import a, b print(type(a.funcA)) -assert type(a.funcA).__name__ == 'cython_function_or_method' +assert type(a.funcA).__name__.endswith('cython_function_or_method') assert type(a.funcA) is type(b.funcB) assert a.funcA.func_globals is a.__dict__ diff --git a/tests/run/complex_cast_T445.pyx b/tests/run/complex_cast_T445.pyx index 62709c9c1..b4069ec9d 100644 --- a/tests/run/complex_cast_T445.pyx +++ b/tests/run/complex_cast_T445.pyx @@ -1,4 +1,4 @@ -# ticket: 445 +# ticket: t445 def complex_double_cast(double x, double complex z): """ diff --git a/tests/run/complex_coercion_sideeffects_T693.pyx b/tests/run/complex_coercion_sideeffects_T693.pyx index 92b1f94a9..18a9575dc 100644 --- a/tests/run/complex_coercion_sideeffects_T693.pyx +++ b/tests/run/complex_coercion_sideeffects_T693.pyx @@ -1,5 +1,5 @@ # mode: run -# ticket: 693 +# ticket: t693 cdef double complex func(double complex x): print "hello" diff --git a/tests/run/complex_int_T446.pyx b/tests/run/complex_int_T446.pyx index a6f6aac2d..16e991c7c 100644 --- a/tests/run/complex_int_T446.pyx +++ b/tests/run/complex_int_T446.pyx @@ -1,9 +1,14 @@ -# ticket: 446 +# ticket: t446 import cython -cdef extern from "complex_int_T446_fix.h": - pass +cdef extern from *: + """ + #if defined _MSC_VER && defined __cplusplus + #define CYTHON_CCOMPLEX 0 + #endif + """ + def test_arith(int complex a, int complex b): """ diff --git a/tests/run/complex_int_T446_fix.h b/tests/run/complex_int_T446_fix.h deleted file mode 100644 index c6d11d15e..000000000 --- a/tests/run/complex_int_T446_fix.h +++ /dev/null @@ -1,3 +0,0 @@ -#if defined _MSC_VER && defined __cplusplus -#define CYTHON_CCOMPLEX 0 -#endif diff --git a/tests/run/complex_numbers_T305.pyx b/tests/run/complex_numbers_T305.pyx index 310b7233e..acbc0a5fa 100644 --- a/tests/run/complex_numbers_T305.pyx +++ b/tests/run/complex_numbers_T305.pyx @@ -1,4 +1,4 @@ -# ticket: 305 +# ticket: t305 from cpython.object cimport Py_EQ, Py_NE diff --git a/tests/run/complex_numbers_T305_long_double.pyx b/tests/run/complex_numbers_T305_long_double.pyx index 891d44271..9bc1e73fb 100644 --- a/tests/run/complex_numbers_T305_long_double.pyx +++ b/tests/run/complex_numbers_T305_long_double.pyx @@ -1,4 +1,4 @@ -# ticket: 305 +# ticket: t305 cimport cython diff --git a/tests/run/complex_numbers_c89_T398.pyx b/tests/run/complex_numbers_c89_T398.pyx index 7d680a167..546d37c31 100644 --- a/tests/run/complex_numbers_c89_T398.pyx +++ b/tests/run/complex_numbers_c89_T398.pyx @@ -1,4 +1,4 @@ -# ticket: 398 +# ticket: t398 cdef extern from "complex_numbers_c89_T398.h": pass include "complex_numbers_T305.pyx" diff --git a/tests/run/complex_numbers_c89_T398_long_double.pyx b/tests/run/complex_numbers_c89_T398_long_double.pyx index 91450e22b..9ac1607e4 100644 --- a/tests/run/complex_numbers_c89_T398_long_double.pyx +++ b/tests/run/complex_numbers_c89_T398_long_double.pyx @@ -1,4 +1,4 @@ -# ticket: 398 +# ticket: t398 cdef extern from "complex_numbers_c89_T398.h": pass include "complex_numbers_T305_long_double.pyx" diff --git a/tests/run/complex_numbers_c99_T398.pyx b/tests/run/complex_numbers_c99_T398.pyx index 762c24b02..332029cab 100644 --- a/tests/run/complex_numbers_c99_T398.pyx +++ b/tests/run/complex_numbers_c99_T398.pyx @@ -1,4 +1,4 @@ -# ticket: 398 +# ticket: t398 cdef extern from "complex_numbers_c99_T398.h": pass include "complex_numbers_T305.pyx" diff --git a/tests/run/complex_numbers_cmath_T2891.pyx b/tests/run/complex_numbers_cmath_T2891.pyx new file mode 100644 index 000000000..a5b1a65ea --- /dev/null +++ b/tests/run/complex_numbers_cmath_T2891.pyx @@ -0,0 +1,15 @@ +# ticket: 2891 +# tag: c, no-cpp + +cdef extern from "complex_numbers_c99_T398.h": pass + +from libc.complex cimport cimag, creal, cabs, carg + + +def test_decomposing(double complex z): + """ + >>> test_decomposing(3+4j) + (3.0, 4.0, 5.0, 0.9272952180016122) + """ + + return (creal(z), cimag(z), cabs(z), carg(z)) diff --git a/tests/run/complex_numbers_cxx_T398.pyx b/tests/run/complex_numbers_cxx_T398.pyx index 9b9b69a73..f5e535ab7 100644 --- a/tests/run/complex_numbers_cxx_T398.pyx +++ b/tests/run/complex_numbers_cxx_T398.pyx @@ -1,4 +1,4 @@ -# ticket: 398 +# ticket: t398 cdef extern from "complex_numbers_cxx_T398.h": pass include "complex_numbers_T305.pyx" diff --git a/tests/run/constant_folding.py b/tests/run/constant_folding.py index 2df8da308..4dd70a625 100644 --- a/tests/run/constant_folding.py +++ b/tests/run/constant_folding.py @@ -136,6 +136,17 @@ def binop_mul_pow(): return (mul_int, mul_large_int, pow_int, pow_large_int) +def binop_pow_negative(): + """ + >>> print_big_ints(binop_pow_negative()) + (4.018775720164609e-06, 8.020807320287816e-38, 0.1) + """ + pow_int = 12 ** -5 + pow_large_int = 1234 ** -12 + pow_expression_int = 10 ** (1-2) + return (pow_int, pow_large_int, pow_expression_int) + + @cython.test_fail_if_path_exists( "//SliceIndexNode", ) diff --git a/tests/run/constants.pyx b/tests/run/constants.pyx index 37772eaef..5f70e3ac2 100644 --- a/tests/run/constants.pyx +++ b/tests/run/constants.pyx @@ -156,8 +156,17 @@ def multiplied_lists_nonconst_left(x): """ return x * [1,2,3] + +@cython.test_fail_if_path_exists("//MulNode") +def multiplied_nonconst_list_const_int(x): + """ + >>> multiplied_nonconst_list_const_int(2) + [1, 2, 3, 1, 2, 3] + """ + return [1,x,3] * 2 + + @cython.test_fail_if_path_exists("//MulNode//ListNode") -@cython.test_assert_path_exists("//MulNode") def multiplied_lists_nonconst_expression(x): """ >>> multiplied_lists_nonconst_expression(5) == [1,2,3] * (5 * 2) diff --git a/tests/run/contains_T455.pyx b/tests/run/contains_T455.pyx index 5caed0b86..8bb087acf 100644 --- a/tests/run/contains_T455.pyx +++ b/tests/run/contains_T455.pyx @@ -1,4 +1,4 @@ -# ticket: 455 +# ticket: t455 def in_sequence(x, seq): """ diff --git a/tests/run/coroutines.py b/tests/run/coroutines.py new file mode 100644 index 000000000..3b236a925 --- /dev/null +++ b/tests/run/coroutines.py @@ -0,0 +1,60 @@ +# cython: language_level=3 +# mode: run +# tag: pep492, pure3.5, gh1462, async, await + + +async def test_coroutine_frame(awaitable): + """ + >>> class Awaitable(object): + ... def __await__(self): + ... return iter([2]) + + >>> coro = test_coroutine_frame(Awaitable()) + >>> import types + >>> isinstance(coro.cr_frame, types.FrameType) or coro.cr_frame + True + >>> coro.cr_frame is coro.cr_frame # assert that it's cached + True + >>> coro.cr_frame.f_code is not None + True + >>> code_obj = coro.cr_frame.f_code + >>> code_obj.co_argcount + 1 + >>> code_obj.co_varnames + ('awaitable', 'b') + + >>> next(coro.__await__()) # avoid "not awaited" warning + 2 + """ + b = await awaitable + return b + + +# gh1462: Using decorators on coroutines. + +def pass_through(func): + return func + + +@pass_through +async def test_pass_through(): + """ + >>> t = test_pass_through() + >>> try: t.send(None) + ... except StopIteration as ex: + ... print(ex.args[0] if ex.args else None) + ... else: print("NOT STOPPED!") + None + """ + + +@pass_through(pass_through) +async def test_pass_through_with_args(): + """ + >>> t = test_pass_through_with_args() + >>> try: t.send(None) + ... except StopIteration as ex: + ... print(ex.args[0] if ex.args else None) + ... else: print("NOT STOPPED!") + None + """ diff --git a/tests/run/coverage_cmd.srctree b/tests/run/coverage_cmd.srctree index 9504cc6f8..70408409c 100644 --- a/tests/run/coverage_cmd.srctree +++ b/tests/run/coverage_cmd.srctree @@ -40,6 +40,13 @@ def func2(a): return a * 2 # 11 +def func3(a): + x = 1 # 15 + a *= 2 # # pragma: no cover + a += x # + return a * 42 # 18 # pragma: no cover + + ######## pkg/coverage_test_pyx.pyx ######## # cython: linetrace=True # distutils: define_macros=CYTHON_TRACE=1 @@ -54,6 +61,13 @@ def func2(int a): return a * 2 # 11 +def func3(int a): + cdef int x = 1 # 15 + a *= 2 # # pragma: no cover + a += x # + return a * 42 # 18 # pragma: no cover + + ######## coverage_test_include_pyx.pyx ######## # cython: linetrace=True # distutils: define_macros=CYTHON_TRACE=1 @@ -152,8 +166,9 @@ def run_report(): missing.append(int(start)) files[os.path.basename(name)] = (statements, missed, covered, missing) - assert 7 not in files['coverage_test_pyx.pyx'][-1], files['coverage_test_pyx.pyx'] - assert 12 not in files['coverage_test_pyx.pyx'][-1], files['coverage_test_pyx.pyx'] + report = files['coverage_test_pyx.pyx'] + assert 7 not in report[-1], report + assert 12 not in report[-1], report def run_xml_report(): @@ -170,35 +185,74 @@ def run_xml_report(): for line in module.findall('lines/line') ) - assert files['pkg/coverage_test_pyx.pyx'][5] > 0, files['pkg/coverage_test_pyx.pyx'] - assert files['pkg/coverage_test_pyx.pyx'][6] > 0, files['pkg/coverage_test_pyx.pyx'] - assert files['pkg/coverage_test_pyx.pyx'][7] > 0, files['pkg/coverage_test_pyx.pyx'] + report = files['pkg/coverage_test_pyx.pyx'] + assert report[5] > 0, report + assert report[6] > 0, report + assert report[7] > 0, report + + +def run_json_report(): + import coverage + if coverage.version_info < (5, 0): + # JSON output comes in coverage 5.0 + return + + stdout = run_coverage_command('json', '-o', '-') + + import json + files = json.loads(stdout.decode("ascii"))['files'] + + for filename in [ + 'pkg/coverage_test_py.py', + 'pkg/coverage_test_pyx.pyx', + ]: + report = files[filename.replace('/', os.sep)] + summary = report['summary'] + assert summary['missing_lines'] == 2, summary + assert summary['excluded_lines'] == 2, summary + assert report['missing_lines'] == [15, 17], report + assert report['excluded_lines'] == [16, 18], report + + assert not frozenset( + report['missing_lines'] + report['excluded_lines'] + ).intersection(report['executed_lines']) def run_html_report(): + from collections import defaultdict + stdout = run_coverage_command('html', '-d', 'html') _parse_lines = re.compile( r'<p[^>]* id=["\'][^0-9"\']*(?P<id>[0-9]+)[^0-9"\']*["\'][^>]*' - r' class=["\'][^"\']*(?P<run>mis|run)[^"\']*["\']').findall + r' class=["\'][^"\']*(?P<run>mis|run|exc)[^"\']*["\']').findall files = {} for file_path in iglob('html/*.html'): with open(file_path) as f: page = f.read() - executed = set() - missing = set() - for line, has_run in _parse_lines(page): - (executed if has_run == 'run' else missing).add(int(line)) - files[file_path] = (executed, missing) + report = defaultdict(set) + for line, state in _parse_lines(page): + report[state].add(int(line)) + files[file_path] = report - executed, missing = [data for path, data in files.items() if 'coverage_test_pyx' in path][0] - assert executed - assert 5 in executed, executed - assert 6 in executed, executed - assert 7 in executed, executed + for filename, report in files.items(): + if "coverage_test_pyx" not in filename: + continue + executed = report["run"] + missing = report["mis"] + excluded = report["exc"] + assert executed + assert 5 in executed, executed + assert 6 in executed, executed + assert 7 in executed, executed + assert 15 in missing, missing + assert 16 in excluded, excluded + assert 17 in missing, missing + assert 18 in excluded, excluded if __name__ == '__main__': run_report() run_xml_report() + run_json_report() run_html_report() diff --git a/tests/run/coverage_nogil.srctree b/tests/run/coverage_nogil.srctree index 053008cc2..d51993ea5 100644 --- a/tests/run/coverage_nogil.srctree +++ b/tests/run/coverage_nogil.srctree @@ -1,5 +1,5 @@ # mode: run -# tag: coverage,trace,nogil +# tag: coverage,trace,nogil,fastgil """ PYTHON setup.py build_ext -i @@ -21,22 +21,34 @@ setup(ext_modules = cythonize([ plugins = Cython.Coverage -######## coverage_test_nogil.pyx ######## -# cython: linetrace=True +######## coverage_test_nogil_fastgil.pyx ######## +# cython: linetrace=True,fast_gil=True # distutils: define_macros=CYTHON_TRACE=1 CYTHON_TRACE_NOGIL=1 +include "_coverage_test_nogil.pxi" + +######## coverage_test_nogil_nofastgil.pyx ######## +# cython: linetrace=True,fast_gil=False +# distutils: define_macros=CYTHON_TRACE=1 CYTHON_TRACE_NOGIL=1 +include "_coverage_test_nogil.pxi" + + +######## _coverage_test_nogil.pxi ######## +# 1 +# 2 +# 3 cdef int func1(int a, int b) nogil: # 4 cdef int x # 5 with gil: # 6 x = 1 # 7 cdef int c = func2(a) + b # 8 return x + c # 9 - - +# 10 +# 11 cdef int func2(int a) with gil: # 12 return a * 2 # 13 - - +# 14 +# 15 def call(int a, int b): # 16 a, b = b, a # 17 with nogil: # 18 @@ -56,11 +68,12 @@ except ImportError: from coverage import coverage -def run_coverage(): +def run_coverage(module_name): + print("Testing module %s" % module_name) cov = coverage() cov.start() - import coverage_test_nogil as module + module = __import__(module_name) module_name = module.__name__ module_path = module_name + '.pyx' assert not any(module.__file__.endswith(ext) @@ -69,7 +82,6 @@ def run_coverage(): assert module.call(1, 2) == (1 * 2) + 2 + 1 cov.stop() - out = StringIO() cov.report(file=out) #cov.report([module], file=out) @@ -77,15 +89,17 @@ def run_coverage(): assert any(module_path in line for line in lines), \ "'%s' not found in coverage report:\n\n%s" % (module_path, out.getvalue()) - mod_file, exec_lines, excl_lines, missing_lines, _ = cov.analysis2(os.path.abspath(module_path)) - assert module_path in mod_file + module_pxi = "_coverage_test_nogil.pxi" + mod_file, exec_lines, excl_lines, missing_lines, _ = cov.analysis2(os.path.abspath(module_pxi)) + assert module_pxi in mod_file executed = set(exec_lines) - set(missing_lines) - # check that everything that runs with the gil owned was executed - assert all(line in executed for line in [12, 13, 16, 17, 18, 20]), '%s / %s' % (exec_lines, missing_lines) + # check that everything that runs with the gil owned was executed (missing due to pxi: 4, 12, 16) + assert all(line in executed for line in [13, 17, 18, 20]), '%s / %s' % (exec_lines, missing_lines) # check that everything that runs in nogil sections was executed - assert all(line in executed for line in [4, 6, 7, 8, 9]), '%s / %s' % (exec_lines, missing_lines) + assert all(line in executed for line in [6, 7, 8, 9]), '%s / %s' % (exec_lines, missing_lines) if __name__ == '__main__': - run_coverage() + for module_name in ["coverage_test_nogil_fastgil", "coverage_test_nogil_nofastgil"]: + run_coverage(module_name) diff --git a/tests/run/cpdef_enums.pxd b/tests/run/cpdef_enums.pxd index 14f64501f..9140e2d5e 100644 --- a/tests/run/cpdef_enums.pxd +++ b/tests/run/cpdef_enums.pxd @@ -11,5 +11,19 @@ cpdef enum PxdEnum: RANK_1 = 37 RANK_2 = 389 +cpdef enum cpdefPxdDocEnum: + """Home is where... + """ + RANK_6 = 159 + +cpdef enum cpdefPxdDocLineEnum: + """Home is where...""" + RANK_7 = 889 + cdef enum PxdSecretEnum: - RANK_3 = 5077 + RANK_8 = 5077 + +cdef enum cdefPxdDocEnum: + """the heart is. + """ + RANK_9 = 2458 diff --git a/tests/run/cpdef_enums.pyx b/tests/run/cpdef_enums.pyx index c8d9d8205..d93e19582 100644 --- a/tests/run/cpdef_enums.pyx +++ b/tests/run/cpdef_enums.pyx @@ -11,6 +11,8 @@ True True >>> FIVE == 5 or FIVE True +>>> ELEVEN == 11 or ELEVEN +True >>> SEVEN # doctest: +ELLIPSIS Traceback (most recent call last): NameError: ...name 'SEVEN' is not defined @@ -29,6 +31,10 @@ True True >>> RANK_2 == 389 or RANK_2 True +>>> RANK_6 == 159 or RANK_6 +True +>>> RANK_7 == 889 or RANK_7 +True >>> RANK_3 # doctest: +ELLIPSIS Traceback (most recent call last): NameError: ...name 'RANK_3' is not defined @@ -48,7 +54,6 @@ Traceback (most recent call last): NameError: ...name 'IntEnum' is not defined """ - cdef extern from *: cpdef enum: # ExternPyx ONE "1" @@ -63,21 +68,30 @@ cpdef enum PyxEnum: THREE = 3 FIVE = 5 +cpdef enum cpdefPyxDocEnum: + """Home is where... + """ + ELEVEN = 11 + +cpdef enum cpdefPyxDocLineEnum: + """Home is where...""" + FOURTEEN = 14 + cdef enum SecretPyxEnum: SEVEN = 7 +cdef enum cdefPyxDocEnum: + """the heart is. + """ + FIVE_AND_SEVEN = 5077 + + def test_as_variable_from_cython(): """ >>> test_as_variable_from_cython() """ - import sys - if sys.version_info >= (2, 7): - assert list(PyxEnum) == [TWO, THREE, FIVE], list(PyxEnum) - assert list(PxdEnum) == [RANK_0, RANK_1, RANK_2], list(PxdEnum) - else: - # No OrderedDict. - assert set(PyxEnum) == {TWO, THREE, FIVE}, list(PyxEnum) - assert set(PxdEnum) == {RANK_0, RANK_1, RANK_2}, list(PxdEnum) + assert list(PyxEnum) == [TWO, THREE, FIVE], list(PyxEnum) + assert list(PxdEnum) == [RANK_0, RANK_1, RANK_2], list(PxdEnum) cdef int verify_pure_c() nogil: cdef int x = TWO @@ -95,3 +109,21 @@ def verify_resolution_GH1533(): """ THREE = 100 return int(PyxEnum.THREE) + + +def check_docs(): + """ + >>> PxdEnum.__doc__ not in ("Home is where...\\n ", "Home is where...") + True + >>> PyxEnum.__doc__ not in ("Home is where...\\n ", "Home is where...") + True + >>> cpdefPyxDocEnum.__doc__ == "Home is where...\\n " + True + >>> cpdefPxdDocEnum.__doc__ == "Home is where...\\n " + True + >>> cpdefPyxDocLineEnum.__doc__ + 'Home is where...' + >>> cpdefPxdDocLineEnum.__doc__ + 'Home is where...' + """ + pass diff --git a/tests/run/cpdef_method_override.pyx b/tests/run/cpdef_method_override.pyx index 97aafe562..aadef67a6 100644 --- a/tests/run/cpdef_method_override.pyx +++ b/tests/run/cpdef_method_override.pyx @@ -1,6 +1,6 @@ # mode: run # tag: cpdef -# ticket: gh-1771 +# ticket: 1771 def _call_method(cls): obj = cls() diff --git a/tests/run/cpdef_nogil.pyx b/tests/run/cpdef_nogil.pyx new file mode 100644 index 000000000..b44e6f551 --- /dev/null +++ b/tests/run/cpdef_nogil.pyx @@ -0,0 +1,19 @@ +# cython: binding=True +# mode: run +# tag: cyfunction + +cpdef int simple() nogil: + """ + >>> simple() + 1 + """ + return 1 + + +cpdef int call_nogil(): + """ + >>> call_nogil() + 1 + """ + with nogil: + return simple() diff --git a/tests/run/cpdef_scoped_enums.pyx b/tests/run/cpdef_scoped_enums.pyx new file mode 100644 index 000000000..8787b7751 --- /dev/null +++ b/tests/run/cpdef_scoped_enums.pyx @@ -0,0 +1,42 @@ +# mode: run +# tag: cpp, cpp11 + +cdef extern from *: + """ + enum class Enum1 { + Item1 = 1, + Item2 = 2 + }; + + enum class Enum2 { + Item4 = 4, + Item5 = 5 + }; + """ + cpdef enum class Enum1: + Item1 + Item2 + + cpdef enum class Enum2: + """Apricots and other fruits. + """ + Item4 + Item5 + + +def test_enum_to_list(): + """ + >>> test_enum_to_list() + """ + assert list(Enum1) == [1, 2] + assert list(Enum2) == [4, 5] + + +def test_enum_doc(): + """ + >>> Enum2.__doc__ == "Apricots and other fruits.\\n " + True + >>> Enum1.__doc__ != "Apricots and other fruits.\\n " + True + """ + pass diff --git a/tests/run/cpdef_scoped_enums_import.srctree b/tests/run/cpdef_scoped_enums_import.srctree new file mode 100644 index 000000000..2d79fb05a --- /dev/null +++ b/tests/run/cpdef_scoped_enums_import.srctree @@ -0,0 +1,71 @@ +# mode: run +# tag: cpp, cpp11 + +""" +PYTHON setup.py build_ext --inplace +PYTHON -c "import runner" +""" + +######## setup.py ######## + +from Cython.Build.Dependencies import cythonize +from distutils.core import setup +setup(ext_modules=cythonize("*.pyx", language='c++')) + +setup( + ext_modules = cythonize([ + "cheese.pyx", + "import_scoped_enum_test.pyx", + "dotted_import_scoped_enum_test.pyx" + ]) +) + +######## cheese.pxd ######## +# distutils: language = c++ +# distutils: extra_compile_args = -std=c++11 + + +cdef extern from * namespace "Namespace": + """ + namespace Namespace { + enum class Cheese { + cheddar = 1, + camembert = 2 + }; + } + """ + cpdef enum class Cheese: + cheddar + camembert + +######## cheese.pyx ######## +# distutils: language = c++ +# distutils: extra_compile_args = -std=c++11 + +pass + +######## import_scoped_enum_test.pyx ######## +# distutils: language = c++ +# distutils: extra_compile_args = -std=c++11 + +from cheese import Cheese +from cheese cimport Cheese + +cdef Cheese c = Cheese.cheddar +assert list(Cheese) == [1, 2] + +######## dotted_import_scoped_enum_test.pyx ######## +# distutils: language = c++ +# distutils: extra_compile_args = -std=c++11 + + +cimport cheese + +cdef cheese.Cheese c = cheese.Cheese.cheddar +assert [cheese.Cheese.cheddar, cheese.Cheese.camembert] == [1, 2] +cdef cheese.Cheese d = int(1) + +######## runner.py ######## + +import import_scoped_enum_test +import dotted_import_scoped_enum_test diff --git a/tests/run/cpdef_temps_T411.pyx b/tests/run/cpdef_temps_T411.pyx index 061d9ef13..66f08331d 100644 --- a/tests/run/cpdef_temps_T411.pyx +++ b/tests/run/cpdef_temps_T411.pyx @@ -1,4 +1,4 @@ -# ticket: 411 +# ticket: t411 cdef class A: """ diff --git a/tests/run/cpp_class_attrib.srctree b/tests/run/cpp_class_attrib.srctree new file mode 100644 index 000000000..3ae850839 --- /dev/null +++ b/tests/run/cpp_class_attrib.srctree @@ -0,0 +1,26 @@ +# tag: cpp + +PYTHON setup.py build_ext --inplace +PYTHON -c "import runner" + +######## setup.py ######## + +from Cython.Build.Dependencies import cythonize +from distutils.core import setup +import os + +example_dir = os.path.abspath(os.path.join(os.environ['CYTHON_PROJECT_DIR'], + 'docs/examples/userguide/wrapping_CPlusPlus')) + +ext_modules= cythonize(os.path.join(example_dir, "rect_with_attributes.pyx"), + include_path=[example_dir]) +setup(ext_modules=ext_modules) + +######## runner.py ######## + +import rect_with_attributes + +x0, y0, x1, y1 = 1, 2, 3, 4 +rect_obj = rect_with_attributes.PyRectangle(x0, y0, x1, y1) + +assert rect_obj.x0 == x0 diff --git a/tests/run/cpp_classes.pyx b/tests/run/cpp_classes.pyx index c7654c065..bdaf4b7de 100644 --- a/tests/run/cpp_classes.pyx +++ b/tests/run/cpp_classes.pyx @@ -130,6 +130,10 @@ def test_value_call(int w): del sqr +cdef struct StructWithEmpty: + Empty empty + + def get_destructor_count(): return destructor_count @@ -146,6 +150,15 @@ def test_stack_allocation(int w, int h): print rect.method(<int>5) return destructor_count +def test_stack_allocation_in_struct(): + """ + >>> d = test_stack_allocation_in_struct() + >>> get_destructor_count() - d + 1 + """ + cdef StructWithEmpty swe + sizeof(swe.empty) # use it for something + return destructor_count cdef class EmptyHolder: cdef Empty empty @@ -153,6 +166,9 @@ cdef class EmptyHolder: cdef class AnotherEmptyHolder(EmptyHolder): cdef Empty another_empty +cdef class EmptyViaStructHolder: + cdef StructWithEmpty swe + def test_class_member(): """ >>> test_class_member() @@ -183,6 +199,18 @@ def test_derived_class_member(): assert destructor_count - start_destructor_count == 2, \ destructor_count - start_destructor_count +def test_class_in_struct_member(): + """ + >>> test_class_in_struct_member() + """ + start_constructor_count = constructor_count + start_destructor_count = destructor_count + e = EmptyViaStructHolder() + #assert constructor_count - start_constructor_count == 1, \ + # constructor_count - start_constructor_count + del e + assert destructor_count - start_destructor_count == 1, \ + destructor_count - start_destructor_count cdef class TemplateClassMember: cdef vector[int] x diff --git a/tests/run/cpp_enums.pyx b/tests/run/cpp_enums.pyx new file mode 100644 index 000000000..2c91d5187 --- /dev/null +++ b/tests/run/cpp_enums.pyx @@ -0,0 +1,58 @@ +# tag: cpp +# mode: run + +cdef extern from *: + """ + enum Enum1 { + Item1, + Item2 + }; + + """ + cdef enum Enum1: + Item1 + Item2 + +a = Item1 +b = Item2 + +cdef Enum1 x, y +x = Item1 +y = Item2 + + +def compare_enums(): + """ + >>> compare_enums() + (True, True, True, True) + """ + return x == a, a == Item1, b == y, y == Item2 + + +cdef extern from * namespace "Namespace1": + """ + namespace Namespace1 { + enum Enum2 { + Item3, + Item4 + }; + } + """ + cdef enum Enum2: + Item3 + Item4 + +c = Item3 +d = Item4 + +cdef Enum2 z, w +z = Item3 +w = Item4 + + +def compare_namespace_enums(): + """ + >>> compare_namespace_enums() + (True, True, True, True) + """ + return z == c, c == Item3, d == w, d == Item4 diff --git a/tests/run/cpp_forwarding_ref.pyx b/tests/run/cpp_forwarding_ref.pyx new file mode 100644 index 000000000..48995442e --- /dev/null +++ b/tests/run/cpp_forwarding_ref.pyx @@ -0,0 +1,37 @@ +# mode: run +# tag: cpp, cpp11 + +from libcpp.utility cimport move + + +cdef extern from *: + """ + #include <utility> + + const char* f(int& x) { + (void) x; + return "lvalue-ref"; + } + + const char* f(int&& x) { + (void) x; + return "rvalue-ref"; + } + + template <typename T> + const char* foo(T&& x) + { + return f(std::forward<T>(x)); + } + """ + const char* foo[T](T&& x) + + +def test_forwarding_ref(): + """ + >>> test_forwarding_ref() + """ + cdef int x = 1 + assert foo(x) == b"lvalue-ref" + assert foo(<int>(1)) == b"rvalue-ref" + assert foo(move(x)) == b"rvalue-ref" diff --git a/tests/run/cpp_iterators.pyx b/tests/run/cpp_iterators.pyx index e5c54c9db..36782710f 100644 --- a/tests/run/cpp_iterators.pyx +++ b/tests/run/cpp_iterators.pyx @@ -25,7 +25,7 @@ def test_vector(py_v): def test_deque_iterator_subtraction(py_v): """ - >>> test_deque_iterator_subtraction([1, 2, 3]) + >>> print(test_deque_iterator_subtraction([1, 2, 3])) 3 """ cdef deque[int] dint @@ -38,7 +38,7 @@ def test_deque_iterator_subtraction(py_v): def test_vector_iterator_subtraction(py_v): """ - >>> test_vector_iterator_subtraction([1, 2, 3]) + >>> print(test_vector_iterator_subtraction([1, 2, 3])) 3 """ cdef vector[int] vint = py_v @@ -140,3 +140,40 @@ def test_iteration_in_generator_reassigned(): if vint is not orig_vint: del vint del orig_vint + +cdef extern from *: + """ + std::vector<int> make_vec1() { + std::vector<int> vint; + vint.push_back(1); + vint.push_back(2); + return vint; + } + """ + cdef vector[int] make_vec1() except + + +cdef vector[int] make_vec2() except *: + return make_vec1() + +cdef vector[int] make_vec3(): + try: + return make_vec1() + except: + pass + +def test_iteration_from_function_call(): + """ + >>> test_iteration_from_function_call() + 1 + 2 + 1 + 2 + 1 + 2 + """ + for i in make_vec1(): + print(i) + for i in make_vec2(): + print(i) + for i in make_vec3(): + print(i) diff --git a/tests/run/cpp_nested_classes.pyx b/tests/run/cpp_nested_classes.pyx index 6008b2379..98ea514f1 100644 --- a/tests/run/cpp_nested_classes.pyx +++ b/tests/run/cpp_nested_classes.pyx @@ -26,6 +26,10 @@ cdef extern from "cpp_nested_classes_support.h": pass +ctypedef A AliasA1 +ctypedef AliasA1 AliasA2 + + def test_nested_classes(): """ >>> test_nested_classes() @@ -47,6 +51,20 @@ def test_nested_typedef(py_x): cdef A.my_int x = py_x assert A.negate(x) == -py_x +def test_typedef_for_nested(py_x): + """ + >>> test_typedef_for_nested(5) + """ + cdef AliasA1.my_int x = py_x + assert A.negate(x) == -py_x + +def test_typedef_for_nested_deep(py_x): + """ + >>> test_typedef_for_nested_deep(5) + """ + cdef AliasA2.my_int x = py_x + assert A.negate(x) == -py_x + def test_typed_nested_typedef(x): """ >>> test_typed_nested_typedef(4) diff --git a/tests/run/cpp_operators_helper.h b/tests/run/cpp_operators_helper.h index ec77d7f62..6c5a545fe 100644 --- a/tests/run/cpp_operators_helper.h +++ b/tests/run/cpp_operators_helper.h @@ -84,10 +84,12 @@ NONMEMBER_BIN_OP2(COMMA) #define REF_BIN_OP(op) int& operator op (int x) { x++; return value; } class RefTestOps { - int value = 0; + int value; public: + RefTestOps() { value = 0; } + REF_UN_OP(-); REF_UN_OP(+); REF_UN_OP(*); diff --git a/tests/run/cpp_scoped_enums.pyx b/tests/run/cpp_scoped_enums.pyx new file mode 100644 index 000000000..7a2bc912f --- /dev/null +++ b/tests/run/cpp_scoped_enums.pyx @@ -0,0 +1,136 @@ +# mode: run +# tag: cpp, cpp11 + +from libcpp.limits cimport numeric_limits + +cdef extern from *: + """ + enum class Enum1 { + Item1, + Item2 + }; + """ + cdef enum class Enum1: + Item1 + Item2 + + +cdef extern from * namespace "Namespace1": + """ + namespace Namespace1 { + enum class Enum2 { + Item1, + Item2 + }; + } + """ + cdef enum class Enum2: + Item1 + Item2 + + +cdef enum class Enum3(int): + a = 1 + b = 2 + + +cdef extern from *: + """ + enum class sorted + { + a = 1, + b = 0 + }; + """ + cdef enum class Enum4 "sorted": + a + b + + +cdef extern from *: + """ + #include <limits> + + enum class LongIntEnum : long int { + val = std::numeric_limits<long int>::max(), + }; + """ + enum class LongIntEnum(long int): + val + + +def test_compare_enums(): + """ + >>> test_compare_enums() + (True, True, False, False) + """ + cdef Enum1 x, y + x = Enum1.Item1 + y = Enum1.Item2 + + return ( + x == Enum1.Item1, + y == Enum1.Item2, + x == Enum1.Item2, + y == Enum1.Item1 + ) + + +def test_compare_namespace_enums(): + """ + >>> test_compare_enums() + (True, True, False, False) + """ + cdef Enum2 z, w + + z = Enum2.Item1 + w = Enum2.Item2 + + return ( + z == Enum2.Item1, + w == Enum2.Item2, + z == Enum2.Item2, + w == Enum2.Item1 + ) + + +def test_coerce_to_from_py_value(object i): + """ + >>> test_coerce_to_from_py_value(1) + (True, False) + + >>> test_coerce_to_from_py_value(2) + (False, True) + + >>> test_coerce_to_from_py_value(3) + (False, False) + + >>> test_coerce_to_from_py_value(11111111111111111111111111111111111111111111) + Traceback (most recent call last): + OverflowError: Python int too large to convert to C long + """ + cdef Enum3 x = i + y = Enum3.b + + return ( + x == Enum3.a, + y == int(i) + ) + + +def test_reserved_cname(): + """ + >>> test_reserved_cname() + True + """ + cdef Enum4 x = Enum4.a + return Enum4.a == int(1) + + +def test_large_enum(): + """ + >>> test_large_enum() + True + """ + long_max = int(numeric_limits[long].max()) + return LongIntEnum.val == long_max diff --git a/tests/run/cpp_stl_algo_modifying_sequence_ops.pyx b/tests/run/cpp_stl_algo_modifying_sequence_ops.pyx new file mode 100644 index 000000000..156a28a11 --- /dev/null +++ b/tests/run/cpp_stl_algo_modifying_sequence_ops.pyx @@ -0,0 +1,434 @@ +# mode: run +# tag: cpp, werror, cpp11 + +from __future__ import print_function + +from cython.operator cimport dereference as deref +from cython.operator cimport preincrement, postincrement +from libcpp cimport bool +from libcpp.algorithm cimport copy, copy_if, copy_n, copy_backward, move, move_backward, fill, fill_n, transform +from libcpp.algorithm cimport generate, generate_n, remove, remove_if, remove_copy, remove_copy_if, replace, replace_if +from libcpp.algorithm cimport replace_copy, replace_copy_if, swap, swap_ranges, iter_swap, reverse, reverse_copy +from libcpp.algorithm cimport rotate, rotate_copy, unique, unique_copy +from libcpp.algorithm cimport sort, upper_bound, min_element +from libcpp.iterator cimport back_inserter +from libcpp.string cimport string +from libcpp.vector cimport vector + + +def copy_int(vector[int] values): + """ + Test copy. + + >>> copy_int(range(5)) + [0, 1, 2, 3, 4] + """ + cdef vector[int] out + copy(values.begin(), values.end(), back_inserter(out)) + return out + + +cdef bool is_odd(int i): + return i % 2 + + +def copy_int_if_odd(vector[int] values): + """ + Test copy_if. + + >>> copy_int_if_odd(range(5)) + [1, 3] + """ + cdef vector[int] out + copy_if(values.begin(), values.end(), back_inserter(out), is_odd) + return out + + +def copy_int_n(vector[int] values, int count): + """ + Test copy_n. + + >>> copy_int_n(range(5), 2) + [0, 1] + """ + cdef vector[int] out + copy_n(values.begin(), count, back_inserter(out)) + return out + + +def copy_int_backward(vector[int] values): + """ + Test copy_backward. + + >>> copy_int_backward(range(5)) + [0, 0, 0, 0, 1, 2, 3, 4] + """ + out = vector[int](values.size() + 3) + copy_backward(values.begin(), values.end(), out.end()) + return out + + +def move_int(vector[int] values): + """ + Test move. + + >>> move_int(range(5)) + [0, 1, 2, 3, 4] + """ + cdef vector[int] out + move(values.begin(), values.end(), back_inserter(out)) + return out + + +def move_int_backward(vector[int] values): + """ + Test move_backward. + + >>> move_int_backward(range(5)) + [0, 0, 0, 0, 1, 2, 3, 4] + """ + out = vector[int](values.size() + 3) + move_backward(values.begin(), values.end(), out.end()) + return out + + +def fill_int(vector[int] array, int value): + """ + Test fill. + + >>> fill_int(range(5), -1) + [-1, -1, -1, -1, -1] + """ + fill(array.begin(), array.end(), value) + return array + + +def fill_int_n(vector[int] array, int count, int value): + """ + Test fill_n. + + >>> fill_int_n(range(5), 3, -1) + [-1, -1, -1, 3, 4] + """ + fill_n(array.begin(), count, value) + return array + + +cdef int to_ord(unsigned char c): + return c + + +def string_to_ord(string s): + """ + Test transform (unary version). + + >> string_to_ord(b"HELLO") + [72, 69, 76, 76, 79] + """ + cdef vector[int] ordinals + transform(s.begin(), s.end(), back_inserter(ordinals), to_ord) + return ordinals + + +cdef int add_ints(int lhs, int rhs): + return lhs + rhs + + +def add_int_vectors(vector[int] lhs, vector[int] rhs): + """ + Test transform (binary version). + + >>> add_int_vectors([1, 2, 3], [4, 5, 6]) + [5, 7, 9] + """ + transform(lhs.begin(), lhs.end(), rhs.begin(), lhs.begin(), add_ints) + return lhs + + +cdef int i = 0 +cdef int generator(): + return postincrement(i) + + +def generate_ints(int count): + """ + Test generate. + + >> generate_ints(5) + [0, 1, 2, 3, 4] + """ + out = vector[int](count) + generate(out.begin(), out.end(), generator) + return out + + +cdef int j = 0 +cdef int generator2(): + return postincrement(j) + + +def generate_n_ints(int count): + """ + Test generate_n. + + >> generate_n_ints(5) + [0, 1, 2, 3, 4, 0, 0, 0] + """ + out = vector[int](count + 3) + generate_n(out.begin(), count, generator2) + return out + + +def remove_spaces(string s): + """ + Test remove. + + >>> print(remove_spaces(b"Text with some spaces").decode("ascii")) + Textwithsomespaces + """ + s.erase(remove(s.begin(), s.end(), ord(" ")), s.end()) + return s + + +cdef bool is_whitespace(unsigned char c) except -1: + # std::isspace from <cctype> + return chr(c) in " \f\n\r\t\v" + + +def remove_whitespace(string s): + r""" + Test remove_if. + + >>> print(remove_whitespace(b"Text\n with\tsome \t whitespaces\n\n").decode("ascii")) + Textwithsomewhitespaces + """ + s.erase(remove_if(s.begin(), s.end(), &is_whitespace), s.end()) + return s + + +def remove_spaces2(string s): + """ + Test remove_copy. + + >>> print(remove_spaces2(b"Text with some spaces").decode("ascii")) + Textwithsomespaces + """ + cdef string out + remove_copy(s.begin(), s.end(), back_inserter(out), ord(" ")) + return out + + +def remove_whitespace2(string s): + r""" + Test remove_copy_if. + + >>> print(remove_whitespace2(b"Text\n with\tsome \t whitespaces\n\n").decode("ascii")) + Textwithsomewhitespaces + """ + cdef string out + remove_copy_if(s.begin(), s.end(), back_inserter(out), &is_whitespace) + return out + + +def replace_ints(vector[int] values, int old, int new): + """ + Test replace. + + >>> replace_ints([5, 7, 4, 2, 8, 6, 1, 9, 0, 3], 8, 88) + [5, 7, 4, 2, 88, 6, 1, 9, 0, 3] + """ + replace(values.begin(), values.end(), old, new) + return values + + +cdef bool less_than_five(int i): + return i < 5 + + +def replace_ints_less_than_five(vector[int] values, int new): + """ + Test replace_if (using cppreference example that doesn't translate well). + + >>> replace_ints_less_than_five([5, 7, 4, 2, 88, 6, 1, 9, 0, 3], 55) + [5, 7, 55, 55, 88, 6, 55, 9, 55, 55] + """ + replace_if(values.begin(), values.end(), less_than_five, new) + return values + + +def replace_ints2(vector[int] values, int old, int new): + """ + Test replace_copy. + + >>> replace_ints2([5, 7, 4, 2, 8, 6, 1, 9, 0, 3], 8, 88) + [5, 7, 4, 2, 88, 6, 1, 9, 0, 3] + """ + cdef vector[int] out + replace_copy(values.begin(), values.end(), back_inserter(out), old, new) + return out + + +def replace_ints_less_than_five2(vector[int] values, int new): + """ + Test replace_copy_if (using cppreference example that doesn't translate well). + + >>> replace_ints_less_than_five2([5, 7, 4, 2, 88, 6, 1, 9, 0, 3], 55) + [5, 7, 55, 55, 88, 6, 55, 9, 55, 55] + """ + cdef vector[int] out + replace_copy_if(values.begin(), values.end(), back_inserter(out), less_than_five, new) + return out + + +def test_swap_ints(): + """ + >>> test_swap_ints() + 5 3 + 3 5 + """ + cdef int a = 5, b = 3 + print(a, b) + swap(a, b) + print(a, b) + + +def test_swap_vectors(): + """ + >>> test_swap_vectors() + [1, 2, 3] [4, 5, 6] + [4, 5, 6] [1, 2, 3] + """ + cdef vector[int] a = [1, 2, 3], b = [4, 5, 6] + print(a, b) + swap(a, b) + print(a, b) + + +def test_swap_ranges(): + """ + >>> test_swap_ranges() + [1, 2, 3] [4, 5, 6] + [4, 5, 6] [1, 2, 3] + """ + cdef vector[int] a = [1, 2, 3], b = [4, 5, 6] + print(a, b) + swap_ranges(a.begin(), a.end(), b.begin()) + print(a, b) + + +def selection_sort(vector[int] values): + """ + Test iter_swap using cppreference example. + + >>> selection_sort([-7, 6, 2, 4, -1, 6, -9, -1, 2, -5, 10, -9, -5, -3, -5, -3, 6, 6, 1, 8]) + [-9, -9, -7, -5, -5, -5, -3, -3, -1, -1, 1, 2, 2, 4, 6, 6, 6, 6, 8, 10] + """ + i = values.begin() + end = values.end() + while i < end: + iter_swap(i, min_element(i, end)) + preincrement(i) + return values + + +def reverse_ints(vector[int] values): + """ + Test reverse. + + >>> reverse_ints([1, 2, 3]) + [3, 2, 1] + """ + reverse(values.begin(), values.end()) + return values + + +def reverse_ints2(vector[int] values): + """ + Test reverse_copy. + + >>> reverse_ints2([1, 2, 3]) + [3, 2, 1] + """ + cdef vector[int] out + reverse_copy(values.begin(), values.end(), back_inserter(out)) + return out + + +def insertion_sort(vector[int] values): + """ + Test rotate using cppreference example. + + >>> insertion_sort([2, 4, 2, 0, 5, 10, 7, 3, 7, 1]) + [0, 1, 2, 2, 3, 4, 5, 7, 7, 10] + """ + i = values.begin() + while i < values.end(): + rotate(upper_bound(values.begin(), i, deref(i)), i, i + 1) + preincrement(i) + return values + + +def rotate_ints_about_middle(vector[int] values): + """ + Test rotate_copy. + + >>> rotate_ints_about_middle([1, 2, 3, 4, 5]) + [3, 4, 5, 1, 2] + """ + cdef vector[int] out + cdef vector[int].iterator pivot = values.begin() + values.size()/2 + rotate_copy(values.begin(), pivot, values.end(), back_inserter(out)) + return out + + +def unique_ints(vector[int] values): + """ + Test unique. + + >>> unique_ints([1, 2, 3, 1, 2, 3, 3, 4, 5, 4, 5, 6, 7]) + [1, 2, 3, 4, 5, 6, 7] + """ + sort(values.begin(), values.end()) + values.erase(unique(values.begin(), values.end()), values.end()) + return values + + +cdef bool both_space(unsigned char lhs, unsigned char rhs): + return lhs == rhs == ord(' ') + + +def collapse_spaces(string text): + """ + Test unique (predicate version) using cppreference example for unique_copy. + + >>> print(collapse_spaces(b"The string with many spaces!").decode("ascii")) + The string with many spaces! + """ + last = unique(text.begin(), text.end(), &both_space) + text.erase(last, text.end()) + return text + + +def unique_ints2(vector[int] values): + """ + Test unique_copy. + + >>> unique_ints2([1, 2, 3, 1, 2, 3, 3, 4, 5, 4, 5, 6, 7]) + [1, 2, 3, 4, 5, 6, 7] + """ + cdef vector[int] out + sort(values.begin(), values.end()) + unique_copy(values.begin(), values.end(), back_inserter(out)) + return out + + +def collapse_spaces2(string text): + """ + Test unique_copy (predicate version) using cppreference example. + + >>> print(collapse_spaces2(b"The string with many spaces!").decode("ascii")) + The string with many spaces! + """ + cdef string out + unique_copy(text.begin(), text.end(), back_inserter(out), &both_space) + return out diff --git a/tests/run/cpp_stl_algo_nonmodifying_sequence_ops.pyx b/tests/run/cpp_stl_algo_nonmodifying_sequence_ops.pyx new file mode 100644 index 000000000..2b001b5f2 --- /dev/null +++ b/tests/run/cpp_stl_algo_nonmodifying_sequence_ops.pyx @@ -0,0 +1,307 @@ +# mode: run +# tag: cpp, werror, cpp11 + +from cython.operator cimport dereference as deref + +from libcpp cimport bool +from libcpp.algorithm cimport all_of, any_of, none_of, for_each, count, count_if, mismatch, find, find_if, find_if_not +from libcpp.algorithm cimport find_end, find_first_of, adjacent_find, search, search_n +from libcpp.iterator cimport distance +from libcpp.string cimport string +from libcpp.vector cimport vector + + +cdef bool is_odd(int i): + return i % 2 + + +def all_odd(vector[int] values): + """ + Test all_of with is_odd predicate. + + >>> all_odd([3, 5, 7, 11, 13, 17, 19, 23]) + True + >>> all_odd([3, 4]) + False + """ + return all_of(values.begin(), values.end(), is_odd) + + +def any_odd(vector[int] values): + """ + Test any_of with is_odd predicate. + + >>> any_odd([1, 2, 3, 4]) + True + >>> any_odd([2, 4, 6, 8]) + False + """ + return any_of(values.begin(), values.end(), is_odd) + + +def none_odd(vector[int] values): + """ + Test none_of with is_odd predicate. + + >>> none_odd([2, 4, 6, 8]) + True + >>> none_odd([1, 2, 3, 4]) + False + """ + return none_of(values.begin(), values.end(), is_odd) + + +def count_ones(vector[int] values): + """ + Test count. + + >>> count_ones([1, 2, 1, 2]) + 2 + """ + return count(values.begin(), values.end(), 1) + + +cdef void add_one(int &i): + # https://github.com/cython/cython/issues/1863 + (&i)[0] += 1 + + +def increment_ints(vector[int] values): + """ + Test for_each. + + >>> increment_ints([3, 4, 2, 8, 15, 267]) + [4, 5, 3, 9, 16, 268] + """ + for_each(values.begin(), values.end(), &add_one) + return values + + +def count_odd(vector[int] values): + """ + Test count_if with is_odd predicate. + + >>> count_odd([1, 2, 3, 4]) + 2 + >>> count_odd([2, 4, 6, 8]) + 0 + """ + return count_if(values.begin(), values.end(), is_odd) + + +def mirror_ends(string data): + """ + Test mismatch using cppreference example. + + This program determines the longest substring that is simultaneously found at the very beginning of the given string + and at the very end of it, in reverse order (possibly overlapping). + + >>> print(mirror_ends(b'abXYZba').decode('ascii')) + ab + >>> print(mirror_ends(b'abca').decode('ascii')) + a + >>> print(mirror_ends(b'aba').decode('ascii')) + aba + """ + return string(data.begin(), mismatch(data.begin(), data.end(), data.rbegin()).first) + + +def mismatch_ints(vector[int] values1, vector[int] values2): + """ + Test mismatch(first1, last1, first2). + + >>> mismatch_ints([1, 2, 3], [1, 2, 3]) + >>> mismatch_ints([1, 2], [1, 2, 3]) + >>> mismatch_ints([1, 3], [1, 2, 3]) + (3, 2) + """ + result = mismatch(values1.begin(), values1.end(), values2.begin()) + if result.first == values1.end(): + return + return deref(result.first), deref(result.second) + + +def is_int_in(vector[int] values, int target): + """ + Test find. + + >>> is_int_in(range(5), 3) + True + >>> is_int_in(range(5), 10) + False + """ + return find(values.begin(), values.end(), target) != values.end() + + +def find_odd(vector[int] values): + """ + Test find_if using is_odd predicate. + + >>> find_odd([2, 3, 4]) + 3 + >>> find_odd([2, 4, 6]) + """ + result = find_if(values.begin(), values.end(), is_odd) + if result != values.end(): + return deref(result) + else: + return None + + +def find_even(vector[int] values): + """ + Test find_if_not using is_odd predicate. + + >>> find_even([3, 4, 5]) + 4 + >>> find_even([1, 3, 5]) + """ + result = find_if_not(values.begin(), values.end(), is_odd) + if result != values.end(): + return deref(result) + else: + return None + + +def find_last_int_sequence(vector[int] values, vector[int] target): + """ + Test find_end. + + >>> find_last_int_sequence([1, 2, 3, 1, 2, 3], [2, 3]) + 4 + >>> find_last_int_sequence([1, 2, 3], [4, 5]) + """ + result = find_end(values.begin(), values.end(), target.begin(), target.end()) + if result != values.end(): + return distance(values.begin(), result) + else: + return None + + +cdef bool is_equal(int lhs, int rhs): + return lhs == rhs + + +def find_last_int_sequence2(vector[int] values, vector[int] target): + """ + Test find_end (using is_equal predicate). + + >>> find_last_int_sequence2([1, 2, 3, 1, 2, 3], [2, 3]) + 4 + >>> find_last_int_sequence2([1, 2, 3], [4, 5]) + """ + result = find_end(values.begin(), values.end(), target.begin(), target.end(), &is_equal) + if result != values.end(): + return distance(values.begin(), result) + else: + return None + + +def find_first_int_in_set(values, target): + """ + Test find_first_of. + + >>> find_first_int_in_set([1, 2, 3, 4, 5], [3, 5]) + 2 + >>> find_first_int_in_set([1, 2, 3], [4, 5]) + """ + cdef vector[int] v = values + cdef vector[int] t = target + result = find_first_of(v.begin(), v.end(), t.begin(), t.end()) + if result != v.end(): + return distance(v.begin(), result) + else: + return None + + +def find_first_int_in_set2(vector[int] values, vector[int] target): + """ + Test find_first_of with is_equal predicate. + + >>> find_first_int_in_set2([1, 2, 3, 4, 5], [3, 5]) + 2 + >>> find_first_int_in_set2([1, 2, 3], [4, 5]) + """ + result = find_first_of(values.begin(), values.end(), target.begin(), target.end(), is_equal) + if result != values.end(): + return distance(values.begin(), result) + else: + return None + + +def find_adjacent_int(vector[int] values): + """ + Test adjacent_find. + + >>> find_adjacent_int([0, 1, 2, 3, 40, 40, 41, 41, 5]) + 4 + >>> find_adjacent_int(range(5)) + """ + result = adjacent_find(values.begin(), values.end()) + if result != values.end(): + return distance(values.begin(), result) + else: + return None + + +def find_adjacent_int2(vector[int] values): + """ + Test find_adjacent with is_equal predicate. + + >>> find_adjacent_int2([0, 1, 2, 3, 40, 40, 41, 41, 5]) + 4 + >>> find_adjacent_int2(range(5)) + """ + result = adjacent_find(values.begin(), values.end(), is_equal) + if result != values.end(): + return distance(values.begin(), result) + else: + return None + + +def in_quote(string quote, string word): + """ + Test search using cppreference example. + + >>> in_quote(b"why waste time learning, when ignorance is instantaneous?", b"learning") + True + >>> in_quote(b"why waste time learning, when ignorance is instantaneous?", b"lemming") + False + """ + return search(quote.begin(), quote.end(), word.begin(), word.end()) != quote.end() + + +def in_quote2(string quote, string word): + """ + Test search using cppreference example (with is_equal predicate). + + >>> in_quote2(b"why waste time learning, when ignorance is instantaneous?", b"learning") + True + >>> in_quote2(b"why waste time learning, when ignorance is instantaneous?", b"lemming") + False + """ + return search(quote.begin(), quote.end(), word.begin(), word.end(), &is_equal) != quote.end() + + +def consecutive_values(string c, int count, char v): + """ + Test search_n using cppreference example (without std::begin and std::end). + + >>> consecutive_values(b"1001010100010101001010101", 4, ord("0")) + False + >>> consecutive_values(b"1001010100010101001010101", 3, ord("0")) + True + """ + return search_n(c.begin(), c.end(), count, v) != c.end() + + +def consecutive_values2(string c, int count, char v): + """ + Test search_n using cppreference example (with is_equal predicate). + + >>> consecutive_values2(b"1001010100010101001010101", 4, ord("0")) + False + >>> consecutive_values2(b"1001010100010101001010101", 3, ord("0")) + True + """ + return search_n(c.begin(), c.end(), count, v, &is_equal) != c.end() diff --git a/tests/run/cpp_stl_algo_partitioning_ops.pyx b/tests/run/cpp_stl_algo_partitioning_ops.pyx new file mode 100644 index 000000000..3bdc3d2af --- /dev/null +++ b/tests/run/cpp_stl_algo_partitioning_ops.pyx @@ -0,0 +1,90 @@ +# mode: run +# tag: cpp, werror, cpp11 + +from __future__ import print_function + +from libcpp cimport bool +from libcpp.algorithm cimport is_partitioned, partition, partition_copy, stable_partition, partition_point +from libcpp.algorithm cimport for_each, copy, reverse +from libcpp.iterator cimport back_inserter +from libcpp.vector cimport vector + + +cdef bool is_even(int i): + return i % 2 == 0 + + +def test_is_partitioned(): + """ + >>> test_is_partitioned() + False + True + False + """ + cdef vector[int] values = range(10) + print(is_partitioned(values.begin(), values.end(), is_even)) + + partition(values.begin(), values.end(), &is_even) + print(is_partitioned(values.begin(), values.end(), is_even)) + + reverse(values.begin(), values.end()) + print(is_partitioned(values.begin(), values.end(), is_even)) + + +cdef int print_int(int v) except -1: + print(v, end=" ") + + +def print_partition(vector[int] values): + """ + Test partition. + + >> print_partition(range(10)) + 0 8 2 6 4 * 5 3 7 1 9 + """ + it = partition(values.begin(), values.end(), &is_even) + for_each(values.begin(), it, &print_int) + print("*", end=" ") + for_each(it, values.end(), &print_int) + print() + + +def partition_ints_even(vector[int] values): + """ + Test partition_copy. + + >>> partition_ints_even(range(10)) + ([0, 2, 4, 6, 8], [1, 3, 5, 7, 9]) + """ + cdef vector[int] even_values, odd_values + partition_copy(values.begin(), values.end(), back_inserter(even_values), back_inserter(odd_values), &is_even) + return even_values, odd_values + + +cdef bool is_positive(int v): + return v > 0 + + +def partition_ints_positive(vector[int] values): + """ + Test stable_partition. + + >>> partition_ints_positive([0, 0, 3, 0, 2, 4, 5, 0, 7]) + [3, 2, 4, 5, 7, 0, 0, 0, 0] + """ + stable_partition(values.begin(), values.end(), &is_positive) + return values + + +def partition_point_ints_even(vector[int] values): + """ + Test partition_point. + + >>> partition_point_ints_even([0, 8, 2, 6, 4, 5, 3, 7, 1, 9]) + ([0, 8, 2, 6, 4], [5, 3, 7, 1, 9]) + """ + it = partition_point(values.begin(), values.end(), is_even) + cdef vector[int] even_values, odd_values + copy(values.begin(), it, back_inserter(even_values)) + copy(it, values.end(), back_inserter(odd_values)) + return even_values, odd_values diff --git a/tests/run/cpp_stl_algo_sorting_ops.pyx b/tests/run/cpp_stl_algo_sorting_ops.pyx new file mode 100644 index 000000000..299089c4c --- /dev/null +++ b/tests/run/cpp_stl_algo_sorting_ops.pyx @@ -0,0 +1,187 @@ +# mode: run +# tag: cpp, werror, cpp11 + +from __future__ import print_function + +from libcpp cimport bool +from libcpp.algorithm cimport is_sorted, is_sorted_until, sort, partial_sort, partial_sort_copy, stable_sort +from libcpp.algorithm cimport nth_element +from libcpp.functional cimport greater +from libcpp.iterator cimport distance +from libcpp.string cimport string +from libcpp.vector cimport vector + + +def is_sorted_ints(vector[int] values): + """ + Test is_sorted. + + >>> is_sorted_ints([3, 1, 4, 1, 5]) + False + >>> is_sorted_ints([1, 1, 3, 4, 5]) + True + """ + return is_sorted(values.begin(), values.end()) + + +def initial_sorted_elements(vector[int] values): + """ + Test is_sorted_until. + + >>> initial_sorted_elements([4, 1, 9, 5, 1, 3]) + 1 + >>> initial_sorted_elements([4, 5, 9, 3, 1, 1]) + 3 + >>> initial_sorted_elements([9, 3, 1, 4, 5, 1]) + 1 + >>> initial_sorted_elements([1, 3, 5, 4, 1, 9]) + 3 + >>> initial_sorted_elements([5, 9, 1, 1, 3, 4]) + 2 + >>> initial_sorted_elements([4, 9, 1, 5, 1, 3]) + 2 + >>> initial_sorted_elements([1, 1, 4, 9, 5, 3]) + 4 + """ + sorted_end = is_sorted_until(values.begin(), values.end()) + return distance(values.begin(), sorted_end) + + +def sort_ints(vector[int] values): + """Test sort using the default operator<. + + >>> sort_ints([5, 7, 4, 2, 8, 6, 1, 9, 0, 3]) + [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + """ + sort(values.begin(), values.end()) + return values + + +def sort_ints_reverse(vector[int] values): + """Test sort using a standard library comparison function object. + + >>> sort_ints_reverse([5, 7, 4, 2, 8, 6, 1, 9, 0, 3]) + [9, 8, 7, 6, 5, 4, 3, 2, 1, 0] + """ + sort(values.begin(), values.end(), greater[int]()) + return values + + +def partial_sort_ints(vector[int] values, int k): + """ + Test partial_sort using the default operator<. + + >>> partial_sort_ints([4, 2, 3, 1, 5], 2)[:2] + [1, 2] + """ + partial_sort(values.begin(), values.begin() + k, values.end()) + return values + + +def partial_sort_ints_reverse(vector[int] values, int k): + """ + Test partial_sort using a standard library comparison function object. + + >>> partial_sort_ints_reverse([4, 2, 3, 1, 5], 2)[:2] + [5, 4] + """ + partial_sort(values.begin(), values.begin() + k, values.end(), greater[int]()) + return values + + +def partial_sort_ints2(vector[int] values, int k): + """ + Test partial_sort_copy using the default operator<. + + >>> partial_sort_ints2([4, 2, 3, 1, 5], 2) + [1, 2] + """ + output = vector[int](2) + partial_sort_copy(values.begin(), values.end(), output.begin(), output.end()) + return output + + +def partial_sort_ints_reverse2(vector[int] values, int k): + """ + Test partial_sort_copy using a standard library comparison function object. + + >>> partial_sort_ints_reverse2([4, 2, 3, 1, 5], 2) + [5, 4] + """ + output = vector[int](2) + partial_sort_copy(values.begin(), values.end(), output.begin(), output.end(), greater[int]()) + return output + + +cdef extern from *: + """ + struct Employee + { + Employee() = default; + Employee(int age, std::string name): age(age), name(name) {} + int age; + std::string name; // Does not participate in comparisons + }; + + bool operator<(const Employee& lhs, const Employee& rhs) + { + return lhs.age < rhs.age; + } + """ + cppclass Employee: + Employee() + Employee(int, string) + int age + string name + +cdef bool Employee_greater(const Employee& lhs, const Employee& rhs): + return lhs.age > rhs.age + +def test_stable_sort(): + """ + Test stable_sort using cppreference example. + + >>> test_stable_sort() + 32, Arthur + 108, Zaphod + 108, Ford + 108, Zaphod + 108, Ford + 32, Arthur + """ + cdef vector[Employee] employees + employees.push_back(Employee(108, <string>b"Zaphod")) + employees.push_back(Employee(32, <string>b"Arthur")) + employees.push_back(Employee(108, <string>b"Ford")) + + stable_sort(employees.begin(), employees.end()) + + for e in employees: + print("%s, %s" % (e.age, <str>(e.name).decode("ascii"))) + + stable_sort(employees.begin(), employees.end(), &Employee_greater) + + for e in employees: + print("%s, %s" % (e.age, <str>(e.name).decode("ascii"))) + + +def second_smallest(vector[int] values): + """ + Test nth_element using the default operator<. + + >>> second_smallest([5, 6, 4, 3, 2, 6, 7, 9, 3]) + 3 + """ + nth_element(values.begin(), values.begin() + 1, values.end()) + return values[1] + + +def second_largest(vector[int] values): + """ + Test nth_element using a standard library comparison function object. + + >>> second_largest([5, 6, 4, 3, 2, 6, 7, 9, 3]) + 7 + """ + nth_element(values.begin(), values.begin() + 1, values.end(), greater[int]()) + return values[1] diff --git a/tests/run/cpp_stl_atomic.pyx b/tests/run/cpp_stl_atomic.pyx new file mode 100644 index 000000000..e504c70eb --- /dev/null +++ b/tests/run/cpp_stl_atomic.pyx @@ -0,0 +1,85 @@ +# mode: run +# tag: cpp, cpp11, werror + +from cython.operator cimport preincrement as incr, dereference as deref +from libc.stdint cimport * + +from libcpp.atomic cimport atomic + +def int_test(int x): + """ + >>> int_test(55) + 3 + >>> int_test(42) + 3 + >>> int_test(100000) + 3 + """ + atom = new atomic[int](x) + try: + atom.store(0) + incr(deref(atom)) + incr(deref(atom)) + incr(deref(atom)) + return atom.load() + finally: + del atom + +ctypedef atomic[int32_t] atomint32_t + +def typedef_test(int x): + """ + >>> typedef_test(55) + 3 + >>> typedef_test(42) + 3 + >>> typedef_test(100000) + 3 + """ + atom = new atomint32_t(x) + try: + atom.store(0) + incr(deref(atom)) + incr(deref(atom)) + incr(deref(atom)) + return atom.load() + finally: + del atom + +def stack_allocation_test(int x): + """ + >>> stack_allocation_test(55) + 3 + >>> stack_allocation_test(42) + 3 + >>> stack_allocation_test(100000) + 3 + """ + cdef atomint32_t atom + atom.store(x) + try: + atom.store(0) + incr(atom) + incr(atom) + incr(atom) + return atom.load() + finally: + pass + +def nogil_int_test(int x): + """ + >>> nogil_int_test(55) + 55 + >>> nogil_int_test(42) + 42 + >>> nogil_int_test(100000) + 100000 + """ + with nogil: + atom = new atomic[int](0) + try: + with nogil: + atom.store(x) + return atom.load() + finally: + del atom diff --git a/tests/run/cpp_stl_conversion.pyx b/tests/run/cpp_stl_conversion.pyx index 578915a0b..ccebc700e 100644 --- a/tests/run/cpp_stl_conversion.pyx +++ b/tests/run/cpp_stl_conversion.pyx @@ -15,7 +15,7 @@ py_set = set py_xrange = xrange py_unicode = unicode -cdef string add_strings(string a, string b): +cdef string add_strings(string a, string b) except *: return a + b def normalize(bytes b): @@ -143,10 +143,12 @@ def test_typedef_vector(o): Traceback (most recent call last): ... OverflowError: ... + + "TypeError: an integer is required" on CPython >>> test_typedef_vector([1, 2, None]) #doctest: +ELLIPSIS Traceback (most recent call last): ... - TypeError: an integer is required + TypeError: ...int... """ cdef vector[my_int] v = o return v diff --git a/tests/run/cpp_stl_numeric_ops.pyx b/tests/run/cpp_stl_numeric_ops.pyx new file mode 100644 index 000000000..52f3c58ec --- /dev/null +++ b/tests/run/cpp_stl_numeric_ops.pyx @@ -0,0 +1,147 @@ +# mode: run +# tag: cpp, werror, cpp11 + +from libcpp.numeric cimport inner_product, iota, accumulate, adjacent_difference, partial_sum +from libcpp.vector cimport vector +from libcpp cimport bool + +# Subtracts two integers. +cdef int subtract_integers(int lhs, int rhs): + return lhs - rhs + +# Adds two integers. +cdef int add_integers(int lhs, int rhs): + return lhs + rhs + +# Multiplies two integers. +cdef int multiply_integers(int lhs, int rhs): + return lhs * rhs + +# Determines equality for two integers. +# If lhs == rhs, returns true. Returns false otherwise. +cdef bool is_equal(int lhs, int rhs): + return lhs == rhs + +def test_inner_product(vector[int] v1, vector[int] v2, int init): + """ + Test inner_product with integer values. + >>> test_inner_product([1, 2, 3], [1, 2, 3], 1) + 15 + """ + return inner_product(v1.begin(), v1.end(), v2.begin(), init) + + +def test_inner_product_with_zero(vector[int] v1, vector[int] v2, int init): + """ + Test inner_product with a zero value in the container. + >>> test_inner_product_with_zero([1, 2, 0], [1, 1, 1], 0) + 3 + """ + return inner_product(v1.begin(), v1.end(), v2.begin(), init) + +def test_inner_product_with_bin_op(vector[int] v1, vector[int] v2, int init): + """ + Test inner_product with two binary operations. In this case, + Looks at number of pairwise matches between v1 and v2. + [5, 1, 2, 3, 4] + [5, 4, 2, 3, 1] + There are 3 matches (5, 2, 3). So, 1 + 1 + 1 = 3. + + >>> test_inner_product_with_bin_op([5, 1, 2, 3, 4], [5, 4, 2, 3, 1], 0) + 3 + """ + return inner_product(v1.begin(), v1.end(), v2.begin(), init, add_integers, is_equal) + +def test_iota(vector[int] v, int value): + """ + Test iota with beginning value of 0. + >>> test_iota(range(6), 0) + [0, 1, 2, 3, 4, 5] + """ + iota(v.begin(), v.end(), value) + return v + +def test_iota_negative_init(vector[int] v, int value): + """ + Test iota with a negative beginning value. + >>> test_iota_negative_init(range(7), -4) + [-4, -3, -2, -1, 0, 1, 2] + """ + iota(v.begin(), v.end(), value) + return v + +def test_accumulate(vector[int] v, int init): + """ + Test accumulate. + 0 + 1 = 1 + 1 + 2 = 3 + 3 + 3 = 6 + >>> test_accumulate([1, 2, 3], 0) + 6 + """ + return accumulate(v.begin(), v.end(), init) + +def test_accumulate_with_subtraction(vector[int] v, int init): + """ + Test accumulate with subtraction. Note that accumulate is a fold-left operation. + 0 - 1 = -1 + -1 - 2 = -3 + -3 - 3 = -6 + >>> test_accumulate_with_subtraction([1, 2, 3], 0) + -6 + """ + return accumulate(v.begin(), v.end(), init, subtract_integers) + +def test_adjacent_difference(vector[int] v): + """ + Test adjacent_difference with integer values. + 2 - 0, -> 2 + 4 - 2, -> 2 + 6 - 4, -> 2 + 8 - 6, -> 2 + 10 - 8, -> 2 + 12 - 10 -> 2 + >>> test_adjacent_difference([2, 4, 6, 8, 10, 12]) + [2, 2, 2, 2, 2, 2] + """ + adjacent_difference(v.begin(), v.end(), v.begin()) + return v + +def test_adjacent_difference_with_bin_op(vector[int] v): + """ + Test adjacent_difference with a binary operation. + 0 + 1 -> 1 + 1 + 2 -> 3 + 2 + 4 -> 6 + 4 + 5 -> 9 + 5 + 6 -> 11 + >>> test_adjacent_difference_with_bin_op([1, 2, 4, 5, 6]) + [1, 3, 6, 9, 11] + """ + adjacent_difference(v.begin(), v.end(), v.begin(), add_integers) + return v + +def test_partial_sum(vector[int] v): + """ + Test partial_sum with integer values. + 2 + 0 -> 2 + 2 + 2 -> 4 + 4 + 2 -> 6 + 6 + 2 -> 8 + 8 + 2 -> 10 + 10 + 2 -> 12 + >>> test_partial_sum([2, 2, 2, 2, 2, 2]) + [2, 4, 6, 8, 10, 12] + """ + partial_sum(v.begin(), v.end(), v.begin()) + return v + +def test_partial_sum_with_bin_op(vector[int] v): + """ + Test partial_sum with a binary operation. + Using multiply_integers, partial_sum will calculate the first 5 powers of 2. + >>> test_partial_sum_with_bin_op([2, 2, 2, 2, 2]) + [2, 4, 8, 16, 32] + """ + partial_sum(v.begin(), v.end(), v.begin(), multiply_integers) + return v diff --git a/tests/run/cpp_type_inference.pyx b/tests/run/cpp_type_inference.pyx index 95631ebfb..f75836a42 100644 --- a/tests/run/cpp_type_inference.pyx +++ b/tests/run/cpp_type_inference.pyx @@ -12,6 +12,11 @@ cdef extern from "shapes.h" namespace "shapes": cdef cppclass Square(Shape): Square(int) + cdef cppclass Empty(Shape): + Empty() + + cdef Empty make_Empty "shapes::Empty"() + from cython cimport typeof from cython.operator cimport dereference as d @@ -48,3 +53,35 @@ def test_derived_types(int size, bint round): ptr = new Square(size) print typeof(ptr) del ptr + +def test_stack_allocated(bint b=True): + """ + >>> test_stack_allocated() + """ + e1 = Empty() + e2 = Empty() + e = e1 if b else e2 + assert typeof(e1) == "Empty", typeof(e1) + assert typeof(e2) == "Empty", typeof(e2) + assert typeof(e) == "Empty", typeof(e) + +cdef extern from *: + """ + template <typename T> + struct MyTemplate {}; + """ + cdef cppclass MyTemplate[T]: + MyTemplate() + +def test_template_types(): + """ + >>> test_template_types() + """ + t_stack = MyTemplate[int]() + assert typeof(t_stack) == "MyTemplate[int]", typeof(t_stack) + + t_ptr = new MyTemplate[double]() + try: + assert typeof(t_ptr) == "MyTemplate[double] *", typeof(t_ptr) + finally: + del t_ptr diff --git a/tests/run/crashT245.pyx b/tests/run/crashT245.pyx index 862deca37..0ef9cb447 100644 --- a/tests/run/crashT245.pyx +++ b/tests/run/crashT245.pyx @@ -1,4 +1,4 @@ -# ticket: 245 +# ticket: t245 cimport crashT245_pxd diff --git a/tests/run/ctuple.pyx b/tests/run/ctuple.pyx index 235265bb1..4053df42b 100644 --- a/tests/run/ctuple.pyx +++ b/tests/run/ctuple.pyx @@ -142,6 +142,17 @@ cpdef (int, double) cpdef_ctuple_return_type(int x, double y): return x, y +def cast_to_ctuple(*o): + """ + >>> cast_to_ctuple(1, 2.) + (1, 2.0) + """ + cdef int x + cdef double y + x, y = <(int, double)>o + return x, y + + @cython.infer_types(True) def test_type_inference(): """ diff --git a/tests/run/ctypedef_int_types_T333.pyx b/tests/run/ctypedef_int_types_T333.pyx index 3f1a99f69..b0a47d484 100644 --- a/tests/run/ctypedef_int_types_T333.pyx +++ b/tests/run/ctypedef_int_types_T333.pyx @@ -1,4 +1,4 @@ -# ticket: 333 +# ticket: t333 #cython: autotestdict=True # ------------------------------------------------------------------- diff --git a/tests/run/cyfunction.pyx b/tests/run/cyfunction.pyx index 3811b2694..619d11f55 100644 --- a/tests/run/cyfunction.pyx +++ b/tests/run/cyfunction.pyx @@ -271,13 +271,13 @@ def test_annotations(a: "test", b: "other" = 2, c: 123 = 4) -> "ret": >>> isinstance(test_annotations.__annotations__, dict) True >>> sorted(test_annotations.__annotations__.items()) - [('a', 'test'), ('b', 'other'), ('c', 123), ('return', 'ret')] + [('a', "'test'"), ('b', "'other'"), ('c', '123'), ('return', "'ret'")] >>> def func_b(): return 42 >>> def func_c(): return 99 >>> inner = test_annotations(1, func_b, func_c) >>> sorted(inner.__annotations__.items()) - [('return', 99), ('x', 'banana'), ('y', 42)] + [('return', 'c()'), ('x', "'banana'"), ('y', 'b()')] >>> inner.__annotations__ = {234: 567} >>> inner.__annotations__ @@ -293,14 +293,14 @@ def test_annotations(a: "test", b: "other" = 2, c: 123 = 4) -> "ret": >>> inner = test_annotations(1, func_b, func_c) >>> sorted(inner.__annotations__.items()) - [('return', 99), ('x', 'banana'), ('y', 42)] + [('return', 'c()'), ('x', "'banana'"), ('y', 'b()')] >>> inner.__annotations__['abc'] = 66 >>> sorted(inner.__annotations__.items()) - [('abc', 66), ('return', 99), ('x', 'banana'), ('y', 42)] + [('abc', 66), ('return', 'c()'), ('x', "'banana'"), ('y', 'b()')] >>> inner = test_annotations(1, func_b, func_c) >>> sorted(inner.__annotations__.items()) - [('return', 99), ('x', 'banana'), ('y', 42)] + [('return', 'c()'), ('x', "'banana'"), ('y', 'b()')] """ def inner(x: "banana", y: b()) -> c(): return x,y @@ -376,6 +376,18 @@ class TestUnboundMethod: def meth(self): pass +class TestStaticmethod(object): + """ + >>> x = TestStaticmethod() + >>> x.staticmeth(42) + 42 + >>> x.staticmeth.__get__(42)() + 42 + """ + @staticmethod + def staticmeth(arg): return arg + + cdef class TestOptimisedBuiltinMethod: """ >>> obj = TestOptimisedBuiltinMethod() @@ -391,3 +403,22 @@ cdef class TestOptimisedBuiltinMethod: def call(self, arg, obj=None): (obj or self).append(arg+1) # optimistically optimised => uses fast fallback method call + + +def do_nothing(f): + """Dummy decorator for `test_firstlineno_decorated_function`""" + return f + + +@do_nothing +@do_nothing +def test_firstlineno_decorated_function(): + """ + check that `test_firstlineno_decorated_function` starts 5 lines below `do_nothing` + + >>> test_firstlineno_decorated_function() + 5 + """ + l1 = do_nothing.__code__.co_firstlineno + l2 = test_firstlineno_decorated_function.__code__.co_firstlineno + return l2 - l1 diff --git a/tests/run/cyfunction_METH_O_GH1728.pyx b/tests/run/cyfunction_METH_O_GH1728.pyx index 621fc565f..5b3b0779b 100644 --- a/tests/run/cyfunction_METH_O_GH1728.pyx +++ b/tests/run/cyfunction_METH_O_GH1728.pyx @@ -8,9 +8,9 @@ cdef class TestMethodOneArg: def call_meth(x): """ - >>> call_meth(TestMethodOneArg()) + >>> call_meth(TestMethodOneArg()) # doctest: +ELLIPSIS Traceback (most recent call last): ... - TypeError: meth() takes exactly one argument (0 given) + TypeError: meth() takes exactly ... argument (0 given) """ return x.meth() diff --git a/tests/run/cython3.pyx b/tests/run/cython3.pyx index 81e4ea6ca..9dae55301 100644 --- a/tests/run/cython3.pyx +++ b/tests/run/cython3.pyx @@ -1,6 +1,6 @@ -# cython: language_level=3, binding=True +# cython: language_level=3, binding=True, annotation_typing=False # mode: run -# tag: generators, python3, exceptions +# tag: generators, python3, exceptions, gh2230, gh2811 print(end='') # test that language_level 3 applies immediately at the module start, for the first token. @@ -16,6 +16,7 @@ x = u'abc' >>> except_as_deletes True + >>> no_match_does_not_touch_target True """ @@ -25,11 +26,24 @@ IS_PY2 = sys.version_info[0] < 3 if not IS_PY2: __doc__ = __doc__.replace(" u'", " '") + def locals_function(a, b=2): x = 'abc' return locals() +### "new style" classes + +class T: + """ + >>> t = T() + >>> isinstance(t, T) + True + >>> isinstance(T, type) # not a Py2 old style class! + True + """ + + ### true division def truediv(x): @@ -340,7 +354,7 @@ def unicode_literals(): def non_ascii_unprefixed_str(): - u""" + """ >>> s = non_ascii_unprefixed_str() >>> isinstance(s, bytes) False @@ -353,7 +367,7 @@ def non_ascii_unprefixed_str(): def non_ascii_raw_str(): - u""" + """ >>> s = non_ascii_raw_str() >>> isinstance(s, bytes) False @@ -366,7 +380,7 @@ def non_ascii_raw_str(): def non_ascii_raw_prefixed_unicode(): - u""" + """ >>> s = non_ascii_raw_prefixed_unicode() >>> isinstance(s, bytes) False @@ -604,39 +618,49 @@ def annotation_syntax(a: "test new test", b : "other" = 2, *args: "ARGS", **kwar >>> len(annotation_syntax.__annotations__) 5 >>> print(annotation_syntax.__annotations__['a']) - test new test + 'test new test' >>> print(annotation_syntax.__annotations__['b']) - other + 'other' >>> print(annotation_syntax.__annotations__['args']) - ARGS + 'ARGS' >>> print(annotation_syntax.__annotations__['kwargs']) - KWARGS + 'KWARGS' >>> print(annotation_syntax.__annotations__['return']) - ret + 'ret' """ result : int = a + b return result +def builtin_as_annotation(text: str): + # See https://github.com/cython/cython/issues/2811 + """ + >>> builtin_as_annotation("abc") + a + b + c + """ + for c in text: + print(c) + + async def async_def_annotations(x: 'int') -> 'float': """ >>> ret, arg = sorted(async_def_annotations.__annotations__.items()) >>> print(ret[0]); print(ret[1]) return - float + 'float' >>> print(arg[0]); print(arg[1]) x - int + 'int' """ return float(x) -''' def repr_returns_str(x) -> str: """ >>> repr_returns_str(123) '123' """ return repr(x) -''' diff --git a/tests/run/cython3_no_unicode_literals.pyx b/tests/run/cython3_no_unicode_literals.pyx index c9690b305..99e8b3f1a 100644 --- a/tests/run/cython3_no_unicode_literals.pyx +++ b/tests/run/cython3_no_unicode_literals.pyx @@ -140,6 +140,12 @@ def str_type_is_str(): cdef str s = 'abc' return str, s +def strip_wrapped_string(s): + # PEP 563 translates an annotation of "test new test" to '"test new test"' + # but choice of string delimiters is a bit arbitrary + # this function handles that + assert s[0] == s[-1] # delimiters on either end are the same + return s[1:-1] # strip them def annotation_syntax(a: "test new test", b : "other" = 2, *args: "ARGS", **kwargs: "KWARGS") -> "ret": """ @@ -150,15 +156,15 @@ def annotation_syntax(a: "test new test", b : "other" = 2, *args: "ARGS", **kwar >>> len(annotation_syntax.__annotations__) 5 - >>> annotation_syntax.__annotations__['a'] + >>> strip_wrapped_string(annotation_syntax.__annotations__['a']) 'test new test' - >>> annotation_syntax.__annotations__['b'] + >>> strip_wrapped_string(annotation_syntax.__annotations__['b']) 'other' - >>> annotation_syntax.__annotations__['args'] + >>> strip_wrapped_string(annotation_syntax.__annotations__['args']) 'ARGS' - >>> annotation_syntax.__annotations__['kwargs'] + >>> strip_wrapped_string(annotation_syntax.__annotations__['kwargs']) 'KWARGS' - >>> annotation_syntax.__annotations__['return'] + >>> strip_wrapped_string(annotation_syntax.__annotations__['return']) 'ret' """ result : int = a + b diff --git a/tests/run/cython_includes.pyx b/tests/run/cython_includes.pyx index 5aab63f99..1fcc575ec 100644 --- a/tests/run/cython_includes.pyx +++ b/tests/run/cython_includes.pyx @@ -19,6 +19,7 @@ cimport cpython.conversion cimport cpython.datetime cimport cpython.dict cimport cpython.exc +cimport cpython.fileobject cimport cpython.float cimport cpython.function cimport cpython.genobject @@ -31,6 +32,7 @@ cimport cpython.list cimport cpython.long cimport cpython.longintrepr cimport cpython.mapping +cimport cpython.marshal cimport cpython.mem cimport cpython.memoryview cimport cpython.method diff --git a/tests/run/datetime_pxd.pyx b/tests/run/datetime_pxd.pyx index 64c4980db..8d1931399 100644 --- a/tests/run/datetime_pxd.pyx +++ b/tests/run/datetime_pxd.pyx @@ -1,10 +1,8 @@ # coding: utf-8 -#cimport cpython.datetime as cy_datetime -#from datetime import time, date, datetime, timedelta, tzinfo +cimport cython - -from cpython.datetime cimport import_datetime +from cpython.datetime cimport import_datetime, timedelta from cpython.datetime cimport time_new, date_new, datetime_new, timedelta_new from cpython.datetime cimport time_tzinfo, datetime_tzinfo from cpython.datetime cimport time_hour, time_minute, time_second, time_microsecond @@ -12,6 +10,15 @@ from cpython.datetime cimport date_day, date_month, date_year from cpython.datetime cimport datetime_day, datetime_month, datetime_year from cpython.datetime cimport datetime_hour, datetime_minute, datetime_second, \ datetime_microsecond +from cpython.datetime cimport datetime, total_seconds + +# These were added in Py3, make sure that their backport works. +from cpython.datetime cimport ( + timedelta as timedelta_ext_type, + PyDateTime_DELTA_GET_DAYS, + PyDateTime_DELTA_GET_SECONDS, + PyDateTime_DELTA_GET_MICROSECONDS, +) import datetime as py_datetime @@ -37,7 +44,23 @@ class FixedOffset(py_datetime.tzinfo): def dst(self, dt): return ZERO - + + +def do_timedelta_macros(timedelta_ext_type delta): + """ + >>> delta = py_datetime.timedelta(days=13, hours=7, seconds=31, microseconds=993322) + >>> (delta.days, delta.seconds, delta.microseconds) + (13, 25231, 993322) + >>> do_timedelta_macros(delta) + (13, 25231, 993322) + """ + return ( + PyDateTime_DELTA_GET_DAYS(delta), + PyDateTime_DELTA_GET_SECONDS(delta), + PyDateTime_DELTA_GET_MICROSECONDS(delta), + ) + + def do_date(int year, int month, int day): """ >>> do_date(2012, 12, 31) @@ -46,7 +69,7 @@ def do_date(int year, int month, int day): v = date_new(year, month, day) return type(v) is py_datetime.date, v.year == year, v.month == month, v.day == day -def do_datetime(int year, int month, int day, +def do_datetime(int year, int month, int day, int hour, int minute, int second, int microsecond): """ >>> do_datetime(2012, 12, 31, 12, 23, 0, 0) @@ -69,7 +92,7 @@ def do_time(int hour, int minute, int second, int microsecond): def do_time_tzinfo(int hour, int minute, int second, int microsecond, object tz): """ - >>> tz = FixedOffset(60*3, 'Moscow') + >>> tz = FixedOffset(60*3, 'Moscow') >>> do_time_tzinfo(12, 23, 0, 0, tz) (True, True, True, True, True, True) """ @@ -79,10 +102,10 @@ def do_time_tzinfo(int hour, int minute, int second, int microsecond, object tz) v.microsecond == microsecond, v.tzinfo is tz -def do_datetime_tzinfo(int year, int month, int day, +def do_datetime_tzinfo(int year, int month, int day, int hour, int minute, int second, int microsecond, object tz): """ - >>> tz = FixedOffset(60*3, 'Moscow') + >>> tz = FixedOffset(60*3, 'Moscow') >>> do_datetime_tzinfo(2012, 12, 31, 12, 23, 0, 0, tz) (True, True, True, True, True, True, True, True, True) """ @@ -90,35 +113,35 @@ def do_datetime_tzinfo(int year, int month, int day, return type(v) is py_datetime.datetime, v.year == year, v.month == month, v.day == day, \ v.hour == hour, v.minute == minute, v.second == second, \ v.microsecond == microsecond, v.tzinfo is tz - + def do_time_tzinfo2(int hour, int minute, int second, int microsecond, object tz): """ - >>> tz = FixedOffset(60*3, 'Moscow') + >>> tz = FixedOffset(60*3, 'Moscow') >>> do_time_tzinfo2(12, 23, 0, 0, tz) (True, True, True, True, True, True, True, True) """ v = time_new(hour, minute, second, microsecond, None) v1 = time_new( - time_hour(v), - time_minute(v), - time_second(v), - time_microsecond(v), + time_hour(v), + time_minute(v), + time_second(v), + time_microsecond(v), tz) r1 = (v1.tzinfo == tz) r2 = (tz == time_tzinfo(v1)) v2 = time_new( - time_hour(v1), - time_minute(v1), - time_second(v1), - time_microsecond(v1), + time_hour(v1), + time_minute(v1), + time_second(v1), + time_microsecond(v1), None) r3 = (v2.tzinfo == None) r4 = (None == time_tzinfo(v2)) v3 = time_new( - time_hour(v2), - time_minute(v2), - time_second(v2), - time_microsecond(v2), + time_hour(v2), + time_minute(v2), + time_second(v2), + time_microsecond(v2), tz) r5 = (v3.tzinfo == tz) r6 = (tz == time_tzinfo(v3)) @@ -130,44 +153,88 @@ def do_time_tzinfo2(int hour, int minute, int second, int microsecond, object tz def do_datetime_tzinfo2(int year, int month, int day, int hour, int minute, int second, int microsecond, object tz): """ - >>> tz = FixedOffset(60*3, 'Moscow') + >>> tz = FixedOffset(60*3, 'Moscow') >>> do_datetime_tzinfo2(2012, 12, 31, 12, 23, 0, 0, tz) (True, True, True, True, True, True, True, True) """ v = datetime_new(year, month, day, hour, minute, second, microsecond, None) v1 = datetime_new( - datetime_year(v), - datetime_month(v), - datetime_day(v), - datetime_hour(v), - datetime_minute(v), - datetime_second(v), - datetime_microsecond(v), + datetime_year(v), + datetime_month(v), + datetime_day(v), + datetime_hour(v), + datetime_minute(v), + datetime_second(v), + datetime_microsecond(v), tz) r1 = (v1.tzinfo == tz) r2 = (tz == datetime_tzinfo(v1)) v2 = datetime_new( - datetime_year(v1), - datetime_month(v1), - datetime_day(v1), - datetime_hour(v1), - datetime_minute(v1), - datetime_second(v1), - datetime_microsecond(v1), + datetime_year(v1), + datetime_month(v1), + datetime_day(v1), + datetime_hour(v1), + datetime_minute(v1), + datetime_second(v1), + datetime_microsecond(v1), None) r3 = (v2.tzinfo == None) r4 = (None == datetime_tzinfo(v2)) v3 = datetime_new( - datetime_year(v2), - datetime_month(v2), - datetime_day(v2), - datetime_hour(v2), - datetime_minute(v2), - datetime_second(v2), - datetime_microsecond(v2), + datetime_year(v2), + datetime_month(v2), + datetime_day(v2), + datetime_hour(v2), + datetime_minute(v2), + datetime_second(v2), + datetime_microsecond(v2), tz) r5 = (v3.tzinfo == tz) r6 = (tz == datetime_tzinfo(v3)) r7 = (v2 == v) r8 = (v3 == v1) return r1, r2, r3, r4, r5, r6, r7, r8 + + +def test_timedelta_total_seconds(): + """ + >>> cytotal, pytotal = test_timedelta_total_seconds() + >>> assert cytotal == pytotal, (cytotal, pytotal) + >>> cytotal == pytotal + True + """ + cdef: + datetime now = py_datetime.datetime.now() + timedelta td = now - py_datetime.datetime(1970, 1, 1) + + pytd = now - py_datetime.datetime(1970, 1, 1) + + return total_seconds(td), pytd.total_seconds() + + +@cython.test_fail_if_path_exists( + "//CoerceFromPyTypeNode", + "//AttributeNode", +) +def test_datetime_attrs_inlined(datetime dt): + # GH#3737 + """ + >>> from datetime import datetime + >>> py_dt = datetime(2020, 8, 18, 4, 9) + >>> dt = test_datetime_attrs_inlined(py_dt) + >>> dt[:5] + (2020, 8, 18, 4, 9) + >>> dt[5] == py_dt.second or (dt[5], py_dt.second) + True + >>> dt[6] == py_dt.microsecond or (dt[6], py_dt.microsecond) + True + """ + return ( + dt.year, + dt.month, + dt.day, + dt.hour, + dt.minute, + dt.second, + dt.microsecond, + ) diff --git a/tests/run/decorators_T593.pyx b/tests/run/decorators_T593.pyx index 9ac3508b8..824e5fb2a 100644 --- a/tests/run/decorators_T593.pyx +++ b/tests/run/decorators_T593.pyx @@ -1,5 +1,5 @@ # mode: run -# ticket: 593 +# ticket: t593 # tag: property, decorator """ @@ -110,8 +110,8 @@ class Base(type): class Bar(metaclass=Base): """ - >>> Bar._order - ['__module__', '__qualname__', '__doc__', 'bar'] + >>> [n for n in Bar._order if n not in {"__qualname__", "__annotations__"}] + ['__module__', '__doc__', 'bar'] """ @property def bar(self): diff --git a/tests/run/decorators_py_T593.py b/tests/run/decorators_py_T593.py index 98cffa79f..339e575de 100644 --- a/tests/run/decorators_py_T593.py +++ b/tests/run/decorators_py_T593.py @@ -1,5 +1,5 @@ # mode: run -# ticket: 593 +# ticket: t593 # tag: property, decorator """ diff --git a/tests/run/default_args_T674.py b/tests/run/default_args_T674.py index 1ca9381bc..7acf84048 100644 --- a/tests/run/default_args_T674.py +++ b/tests/run/default_args_T674.py @@ -1,5 +1,5 @@ # mode: run -# ticket: 674 +# ticket: t674 def test_inner(a): """ diff --git a/tests/run/dict_getitem.pyx b/tests/run/dict_getitem.pyx index 7a17663ba..f8e0210b8 100644 --- a/tests/run/dict_getitem.pyx +++ b/tests/run/dict_getitem.pyx @@ -146,3 +146,17 @@ def getitem_not_none(dict d not None, key): KeyError: (1, 2) """ return d[key] + + +def getitem_int_key(d, int key): + """ + >>> d = {-1: 10} + >>> getitem_int_key(d, -1) # dict + 10 + >>> class D(dict): pass + >>> d = D({-1: 10}) + >>> getitem_int_key(d, -1) # D + 10 + """ + # Based on GH-1807: must check Mapping protocol first, even for integer "index" keys. + return d[key] diff --git a/tests/run/different_package_names.srctree b/tests/run/different_package_names.srctree new file mode 100644 index 000000000..f70699012 --- /dev/null +++ b/tests/run/different_package_names.srctree @@ -0,0 +1,43 @@ +# mode: run +# tag: import,cimport,packages + +PYTHON setup.py build_ext --inplace +PYTHON -c "import pkg_py" +PYTHON -c "import pkg_py.pkg_pyx" +PYTHON -c "import pkg_py.pkg_pyx.module as module; module.run_test()" + +######## setup.py ######## + +from distutils.core import setup +from Cython.Build import cythonize + +setup( + ext_modules=cythonize('**/*.pyx', language_level=3), +) + + +######## pkg_py/__init__.py ######## + +TYPE = 'py' + +######## pkg_py/pkg_pyx/__init__.pyx ######## + +TYPE = 'pyx' + +######## pkg_py/pkg_pyx/pkg_pxd/__init__.pxd ######## + +# Not what Python would consider a package, but Cython can use it for cimports. +from libc.math cimport fabs + +######## pkg_py/pkg_pyx/module.pyx ######## + +from pkg_py.pkg_pyx.pkg_pxd cimport fabs + +def run_test(): + import pkg_py + assert pkg_py.TYPE == 'py' + + import pkg_py.pkg_pyx + assert pkg_py.pkg_pyx.TYPE == 'pyx' + + assert fabs(-2.0) == 2.0 diff --git a/tests/run/division_T384.pyx b/tests/run/division_T384.pyx index 301dc3a61..5da5475fd 100644 --- a/tests/run/division_T384.pyx +++ b/tests/run/division_T384.pyx @@ -1,4 +1,4 @@ -# ticket: 384 +# ticket: t384 """ >>> test(3) diff --git a/tests/run/duplicate_keyword_in_call.py b/tests/run/duplicate_keyword_in_call.py index e3d041d76..aba5772c3 100644 --- a/tests/run/duplicate_keyword_in_call.py +++ b/tests/run/duplicate_keyword_in_call.py @@ -1,6 +1,6 @@ # mode: run # tag: kwargs, call -# ticket: 717 +# ticket: t717 def f(**kwargs): return sorted(kwargs.items()) diff --git a/tests/run/duplicate_utilitycode_from_pyx.srctree b/tests/run/duplicate_utilitycode_from_pyx.srctree new file mode 100644 index 000000000..c539af7a6 --- /dev/null +++ b/tests/run/duplicate_utilitycode_from_pyx.srctree @@ -0,0 +1,29 @@ + + +PYTHON setup.py build_ext --inplace +PYTHON -c "import modb; modb.ClassB()" + +#################### moda.pyx #################### + +cdef class ClassA: + cdef int[2] a + +#################### modb.pyx ##################### + +from moda cimport ClassA + +cdef class ClassB(ClassA): + cdef int[2] b + +###################### setup.py ################### + +from setuptools import setup +from Cython.Build import cythonize +import Cython.Compiler.Options + +Cython.Compiler.Options.cimport_from_pyx = True + +setup( + ext_modules = cythonize(["moda.pyx", "modb.pyx"], + compiler_directives={'language_level': 3}) +) diff --git a/tests/run/dynamic_args.pyx b/tests/run/dynamic_args.pyx index 900671028..1205b06b0 100644 --- a/tests/run/dynamic_args.pyx +++ b/tests/run/dynamic_args.pyx @@ -1,5 +1,5 @@ # mode: run -# ticket: 674 +# ticket: t674 cdef class Foo: cdef str name diff --git a/tests/run/ellipsis_T488.pyx b/tests/run/ellipsis_T488.pyx index e7892dbc9..ab04b317c 100644 --- a/tests/run/ellipsis_T488.pyx +++ b/tests/run/ellipsis_T488.pyx @@ -1,4 +1,4 @@ -# ticket: 488 +# ticket: t488 """ >>> test() diff --git a/tests/run/embedsignatures.pyx b/tests/run/embedsignatures.pyx index 147f7afdd..233359127 100644 --- a/tests/run/embedsignatures.pyx +++ b/tests/run/embedsignatures.pyx @@ -1,10 +1,16 @@ #cython: embedsignature=True, annotation_typing=False +# signatures here are a little fragile - when they are +# generated during the build process gives slightly +# different (but equivalent) forms - therefore tests +# may need changing occasionally to reflect behaviour +# and this isn't necessarily a bug + import sys if sys.version_info >= (3, 4): def funcdoc(f): - if not f.__text_signature__: + if not getattr(f, "__text_signature__", None): return f.__doc__ doc = '%s%s' % (f.__name__, f.__text_signature__) if f.__doc__: @@ -83,6 +89,9 @@ __doc__ = ur""" >>> print (Ext.n.__doc__) Ext.n(self, a: int, b: float = 1.0, *args: tuple, **kwargs: dict) -> (None, True) + >>> print (Ext.o.__doc__) + Ext.o(self, a, b=1, /, c=5, *args, **kwargs) + >>> print (Ext.get_int.__doc__) Ext.get_int(self) -> int @@ -265,6 +274,9 @@ cdef class Ext: def n(self, a: int, b: float = 1.0, *args: tuple, **kwargs: dict) -> (None, True): pass + def o(self, a, b=1, /, c=5, *args, **kwargs): + pass + cpdef int get_int(self): return 0 @@ -441,16 +453,16 @@ Foo.m01(self, a: ...) -> Ellipsis Foo.m02(self, a: True, b: False) -> bool >>> print(Foo.m03.__doc__) -Foo.m03(self, a: 42, b: 42, c: -42) -> int +Foo.m03(self, a: 42, b: +42, c: -42) -> int >>> print(Foo.m04.__doc__) -Foo.m04(self, a: 3.14, b: 3.14, c: -3.14) -> float +Foo.m04(self, a: 3.14, b: +3.14, c: -3.14) -> float >>> print(Foo.m05.__doc__) Foo.m05(self, a: 1 + 2j, b: +2j, c: -2j) -> complex >>> print(Foo.m06.__doc__) -Foo.m06(self, a: 'abc', b: b'abc', c: u'abc') -> (str, bytes, unicode) +Foo.m06(self, a: 'abc', b: b'abc', c: 'abc') -> (str, bytes, unicode) >>> print(Foo.m07.__doc__) Foo.m07(self, a: [1, 2, 3], b: []) -> list @@ -459,7 +471,7 @@ Foo.m07(self, a: [1, 2, 3], b: []) -> list Foo.m08(self, a: (1, 2, 3), b: ()) -> tuple >>> print(Foo.m09.__doc__) -Foo.m09(self, a: {1, 2, 3}, b: set()) -> set +Foo.m09(self, a: {1, 2, 3}, b: {i for i in ()}) -> set >>> print(Foo.m10.__doc__) Foo.m10(self, a: {1: 1, 2: 2, 3: 3}, b: {}) -> dict diff --git a/tests/run/empty_for_loop_T208.pyx b/tests/run/empty_for_loop_T208.pyx index 88a134f25..70b3be512 100644 --- a/tests/run/empty_for_loop_T208.pyx +++ b/tests/run/empty_for_loop_T208.pyx @@ -1,4 +1,4 @@ -# ticket: 208 +# ticket: t208 def go_py_empty(): """ diff --git a/tests/run/enumerate_T316.pyx b/tests/run/enumerate_T316.pyx index f148e1a22..89755b76b 100644 --- a/tests/run/enumerate_T316.pyx +++ b/tests/run/enumerate_T316.pyx @@ -1,4 +1,4 @@ -# ticket: 316 +# ticket: t316 cimport cython diff --git a/tests/run/exceptionrefcount.pyx b/tests/run/exceptionrefcount.pyx index d4ce39fd9..1d1a6742f 100644 --- a/tests/run/exceptionrefcount.pyx +++ b/tests/run/exceptionrefcount.pyx @@ -27,8 +27,11 @@ __doc__ = u""" >>> run_test(50, test_finally) """ +cimport cython from cpython.ref cimport PyObject +@cython.binding(False) +@cython.always_allow_keywords(False) def get_refcount(obj): return (<PyObject*>obj).ob_refcnt diff --git a/tests/run/ext_attr_getter.srctree b/tests/run/ext_attr_getter.srctree index 6fdf7503c..76f059ed7 100644 --- a/tests/run/ext_attr_getter.srctree +++ b/tests/run/ext_attr_getter.srctree @@ -1,17 +1,60 @@ +# mode: run +# tag: cgetter, property + +""" PYTHON setup.py build_ext --inplace -PYTHON -c "import runner" +PYTHON run_failure_tests.py +PYTHON runner.py +""" ######## setup.py ######## from Cython.Build.Dependencies import cythonize from distutils.core import setup -# force the build order -setup(ext_modules= cythonize("foo_extension.pyx")) +# Enforce the right build order +setup(ext_modules = cythonize("foo_extension.pyx", language_level=3)) +setup(ext_modules = cythonize("getter[0-9].pyx", language_level=3)) + + +######## run_failure_tests.py ######## + +import glob +import sys + +from Cython.Build.Dependencies import cythonize +from Cython.Compiler.Errors import CompileError + +# Run the failure tests +failed_tests = [] +passed_tests = [] + +def run_test(name): + title = name + with open(name, 'r') as f: + for line in f: + if 'TEST' in line: + title = line.partition('TEST:')[2].strip() + break + sys.stderr.write("\n### TESTING: %s\n" % title) + + try: + cythonize(name, language_level=3) + except CompileError as e: + sys.stderr.write("\nOK: got expected exception\n") + passed_tests.append(name) + else: + sys.stderr.write("\nFAIL: compilation did not detect the error\n") + failed_tests.append(name) -setup(ext_modules = cythonize("getter*.pyx")) +for name in sorted(glob.glob("getter_fail*.pyx")): + run_test(name) -######## foo_nominal.h ######## +assert not failed_tests, "Failed tests: %s" % failed_tests +assert passed_tests # check that tests were found at all + + +######## foo.h ######## #include <Python.h> @@ -24,30 +67,77 @@ typedef struct { int f0; int f1; int f2; + int v[10]; } FooStructNominal; +typedef struct { + PyObject_HEAD +} FooStructOpaque; + + +#define PyFoo_GET0M(a) (((FooStructNominal*)a)->f0) +#define PyFoo_GET1M(a) (((FooStructNominal*)a)->f1) +#define PyFoo_GET2M(a) (((FooStructNominal*)a)->f2) + +int PyFoo_Get0F(FooStructOpaque *f) +{ + return PyFoo_GET0M(f); +} + +int PyFoo_Get1F(FooStructOpaque *f) +{ + return PyFoo_GET1M(f); +} + +int PyFoo_Get2F(FooStructOpaque *f) +{ + return PyFoo_GET2M(f); +} + +int *PyFoo_GetV(FooStructOpaque *f) +{ + return ((FooStructNominal*)f)->v; +} + #ifdef __cplusplus } #endif + ######## foo_extension.pyx ######## cdef class Foo: - cdef public int field0, field1, field2; - - def __init__(self, f0, f1, f2): - self.field0 = f0 - self.field1 = f1 - self.field2 = f2 + cdef public int _field0, _field1, _field2; + cdef public int _vector[10]; -cdef get_field0(Foo f): - return f.field0 + @property + def field0(self): + return self._field0 -cdef get_field1(Foo f): - return f.field1 + @property + def field1(self): + return self._field1 -cdef get_field2(Foo f): - return f.field2 + @property + def field2(self): + return self._field2 + + def __init__(self, f0, f1, f2, vec=None): + if vec is None: + vec = () + if not isinstance(vec, tuple): + raise ValueError("v must be None or a tuple") + self._field0 = f0 + self._field1 = f1 + self._field2 = f2 + i = 0 + for v in vec: + self._vector[i] = v + if i > 9: + break + i += 1 + for j in range(i,10): + self._vector[j] = 0 # A pure-python class that disallows direct access to fields class OpaqueFoo(Foo): @@ -69,7 +159,7 @@ class OpaqueFoo(Foo): # Access base Foo fields from C via aliased field names -cdef extern from "foo_nominal.h": +cdef extern from "foo.h": ctypedef class foo_extension.Foo [object FooStructNominal]: cdef: @@ -78,13 +168,164 @@ cdef extern from "foo_nominal.h": int field2 "f2" def sum(Foo f): - # the f.__getattr__('field0') is replaced in c by f->f0 + # Note - not a cdef function but compiling the f.__getattr__('field0') + # notices the alias and replaces the __getattr__ in c by f->f0 anyway return f.field0 + f.field1 + f.field2 +def check_pyobj(Foo f): + # compare the c code to the check_pyobj in getter2.pyx + return bool(f.field1) + + +######## getter.pxd ######## + +# Access base Foo fields from C via getter functions + + +cdef extern from "foo.h": + ctypedef class foo_extension.Foo [object FooStructOpaque, check_size ignore]: + @property + cdef inline int fieldM0(self): + return PyFoo_GET0M(self) + + @property + cdef inline int fieldF1(self) except -123: + return PyFoo_Get1F(self) + + @property + cdef inline int fieldM2(self): + return PyFoo_GET2M(self) + + @property + cdef inline int *vector(self): + return PyFoo_GetV(self) + + @property + cdef inline int meaning_of_life(self) except -99: + cdef int ret = 21 + ret *= 2 + return ret + + int PyFoo_GET0M(Foo); # this is actually a macro ! + int PyFoo_Get1F(Foo); + int PyFoo_GET2M(Foo); # this is actually a macro ! + int *PyFoo_GetV(Foo); + + +######## getter1.pyx ######## + +cimport getter + +def sum(getter.Foo f): + # Note - not a cdef function but compiling the f.__getattr__('field0') + # notices the getter and replaces the __getattr__ in c by PyFoo_GET anyway + return f.fieldM0 + f.fieldF1 + f.fieldM2 + +def check_10(getter.Foo f): + return f.fieldF1 != 10 + +def vec0(getter.Foo f): + return f.vector[0] + +def check_binop(getter.Foo f): + return f.fieldF1 / 10 + + +######## getter2.pyx ######## + +cimport getter + +def check_pyobj(getter.Foo f): + return bool(f.fieldF1) + +def check_unary(getter.Foo f): + return -f.fieldF1 + +def check_meaning_of_life(getter.Foo f): + return f.meaning_of_life + + +######## getter_fail_classmethod.pyx ######## + +# TEST: Make sure not all decorators are accepted. + +cdef extern from "foo.h": + ctypedef class foo_extension.Foo [object FooStructOpaque]: + @property + @classmethod + cdef inline int field0(cls): + print('in classmethod of Foo') + + +######## getter_fail_dot_getter.pyx ######## + +# TEST: Make sure not all decorators are accepted. + +cdef extern from "foo.h": + ctypedef class foo_extension.Foo [object FooStructOpaque]: + @property + cdef inline int field0(self): + pass + + @field0.getter + cdef inline void field1(self): + pass + + +######## getter_fail_no_inline.pyx ######## + +# TEST: Properties must be declared "inline". + +cdef extern from "foo.h": + ctypedef class foo_extension.Foo [object FooStructOpaque]: + @property + cdef int field0(self): + pass + + +######## getter_fail_void.pyx ######## + +# TEST: Properties must have a non-void return type. + +cdef extern from "foo.h": + ctypedef class foo_extension.Foo [object FooStructOpaque]: + @property + cdef void field0(self): + pass + + +######## getter_fail_no_args.pyx ######## + +# TEST: Properties must have the right signature. + +cdef extern from "foo.h": + ctypedef class foo_extension.Foo [object FooStructOpaque]: + @property + cdef int field0(): + pass + + +######## getter_fail_too_many_args.pyx ######## + +# TEST: Properties must have the right signature. + +cdef extern from "foo.h": + ctypedef class foo_extension.Foo [object FooStructOpaque]: + @property + cdef int field0(x, y): + pass + + ######## runner.py ######## -import foo_extension, getter0 +import warnings +import foo_extension, getter0, getter1, getter2 +def sum(f): + # pure python field access, but code is identical to cython cdef sum + return f.field0 + f.field1 + f.field2 + +# Baseline test: if this fails something else is wrong foo = foo_extension.Foo(23, 123, 1023) assert foo.field0 == 23 @@ -92,18 +333,36 @@ assert foo.field1 == 123 assert foo.field2 == 1023 ret = getter0.sum(foo) -assert ret == foo.field0 + foo.field1 + foo.field2 +assert ret == sum(foo) + +# Aliasing test. Check 'cdef int field0 "f0" works as advertised: +# - C can access the fields through the aliases +# - Python cannot access the fields at all opaque_foo = foo_extension.OpaqueFoo(23, 123, 1023) -# C can access the fields through the aliases opaque_ret = getter0.sum(opaque_foo) assert opaque_ret == ret + +val = getter2.check_pyobj(opaque_foo) +assert val is True +val = getter2.check_unary(opaque_foo) +assert val == -123 +val = getter2.check_meaning_of_life(opaque_foo) +assert val == 42 + try: - # Python cannot access the fields f0 = opaque_ret.field0 assert False except AttributeError as e: pass +# Getter test. Check C-level getter works as advertised: +# - C accesses the fields through getter calls (maybe macros) +# - Python accesses the fields through attribute lookup + +opaque_foo = foo_extension.OpaqueFoo(23, 123, 1023, (1, 2, 3)) + +opaque_ret = getter1.sum(opaque_foo) +assert opaque_ret == ret diff --git a/tests/run/ext_instance_type_T232.pyx b/tests/run/ext_instance_type_T232.pyx index 9538a9b96..c089b1603 100644 --- a/tests/run/ext_instance_type_T232.pyx +++ b/tests/run/ext_instance_type_T232.pyx @@ -1,4 +1,4 @@ -# ticket: 232 +# ticket: t232 cdef class MyExt: cdef object attr diff --git a/tests/run/extended_unpacking_T235.pyx b/tests/run/extended_unpacking_T235.pyx index 2388e2587..8d38120a1 100644 --- a/tests/run/extended_unpacking_T235.pyx +++ b/tests/run/extended_unpacking_T235.pyx @@ -1,4 +1,4 @@ -# ticket: 235 +# ticket: t235 __doc__ = u""" >>> class FakeSeq(object): diff --git a/tests/run/extended_unpacking_T409.pyx b/tests/run/extended_unpacking_T409.pyx index 1198a15f3..7c3b71d51 100644 --- a/tests/run/extended_unpacking_T409.pyx +++ b/tests/run/extended_unpacking_T409.pyx @@ -1,4 +1,4 @@ -# ticket: 409 +# ticket: t409 def simple(): """ diff --git a/tests/run/extern_builtins_T258.pyx b/tests/run/extern_builtins_T258.pyx index d732931dd..1c6ea9843 100644 --- a/tests/run/extern_builtins_T258.pyx +++ b/tests/run/extern_builtins_T258.pyx @@ -1,4 +1,4 @@ -# ticket: 258 +# ticket: t258 cdef extern from "Python.h": diff --git a/tests/run/extstarargs.pyx b/tests/run/extstarargs.pyx index 041603865..b1dcd4957 100644 --- a/tests/run/extstarargs.pyx +++ b/tests/run/extstarargs.pyx @@ -1,124 +1,169 @@ -__doc__ = u""" - >>> s = Silly(1,2,3, 'test') - >>> (spam,grail,swallow,creosote,onlyt,onlyk,tk) = ( - ... s.spam,s.grail,s.swallow,s.creosote,s.onlyt,s.onlyk,s.tk) - - >>> spam(1,2,3) - (1, 2, 3) - >>> spam(1,2) - Traceback (most recent call last): - TypeError: spam() takes exactly 3 positional arguments (2 given) - >>> spam(1,2,3,4) - Traceback (most recent call last): - TypeError: spam() takes exactly 3 positional arguments (4 given) - >>> spam(1,2,3, a=1) #doctest: +ELLIPSIS - Traceback (most recent call last): - TypeError: spam() got an unexpected keyword argument 'a' - - >>> grail(1,2,3) - (1, 2, 3, ()) - >>> grail(1,2,3,4) - (1, 2, 3, (4,)) - >>> grail(1,2,3,4,5,6,7,8,9) - (1, 2, 3, (4, 5, 6, 7, 8, 9)) - >>> grail(1,2) - Traceback (most recent call last): - TypeError: grail() takes at least 3 positional arguments (2 given) - >>> grail(1,2,3, a=1) #doctest: +ELLIPSIS - Traceback (most recent call last): - TypeError: grail() got an unexpected keyword argument 'a' - - >>> swallow(1,2,3) - (1, 2, 3, ()) - >>> swallow(1,2,3,4) - Traceback (most recent call last): - TypeError: swallow() takes exactly 3 positional arguments (4 given) - >>> swallow(1,2,3, a=1, b=2) - (1, 2, 3, (('a', 1), ('b', 2))) - >>> swallow(1,2,3, x=1) - Traceback (most recent call last): - TypeError: swallow() got multiple values for keyword argument 'x' - - >>> creosote(1,2,3) - (1, 2, 3, (), ()) - >>> creosote(1,2,3,4) - (1, 2, 3, (4,), ()) - >>> creosote(1,2,3, a=1) - (1, 2, 3, (), (('a', 1),)) - >>> creosote(1,2,3,4, a=1, b=2) - (1, 2, 3, (4,), (('a', 1), ('b', 2))) - >>> creosote(1,2,3,4, x=1) - Traceback (most recent call last): - TypeError: creosote() got multiple values for keyword argument 'x' - - >>> onlyt(1) - (1,) - >>> onlyt(1,2) - (1, 2) - >>> onlyt(a=1) - Traceback (most recent call last): - TypeError: onlyt() got an unexpected keyword argument 'a' - >>> onlyt(1, a=2) - Traceback (most recent call last): - TypeError: onlyt() got an unexpected keyword argument 'a' - - >>> onlyk(a=1) - (('a', 1),) - >>> onlyk(a=1, b=2) - (('a', 1), ('b', 2)) - >>> onlyk(1) - Traceback (most recent call last): - TypeError: onlyk() takes exactly 0 positional arguments (1 given) - >>> onlyk(1, 2) - Traceback (most recent call last): - TypeError: onlyk() takes exactly 0 positional arguments (2 given) - >>> onlyk(1, a=1, b=2) - Traceback (most recent call last): - TypeError: onlyk() takes exactly 0 positional arguments (1 given) - - >>> tk(a=1) - (('a', 1),) - >>> tk(a=1, b=2) - (('a', 1), ('b', 2)) - >>> tk(1) - (1,) - >>> tk(1, 2) - (1, 2) - >>> tk(1, a=1, b=2) - (1, ('a', 1), ('b', 2)) -""" - -import sys, re -if sys.version_info >= (2,6): - __doc__ = re.sub(u"(ELLIPSIS[^>]*Error: )[^\n]*\n", u"\\1...\n", __doc__) +cimport cython cdef sorteditems(d): - l = list(d.items()) - l.sort() - return tuple(l) + return tuple(sorted(d.items())) + cdef class Silly: def __init__(self, *a): - pass + """ + >>> s = Silly(1,2,3, 'test') + """ def spam(self, x, y, z): + """ + >>> s = Silly() + >>> s.spam(1,2,3) + (1, 2, 3) + >>> s.spam(1,2) + Traceback (most recent call last): + TypeError: spam() takes exactly 3 positional arguments (2 given) + >>> s.spam(1,2,3,4) + Traceback (most recent call last): + TypeError: spam() takes exactly 3 positional arguments (4 given) + >>> s.spam(1,2,3, a=1) + Traceback (most recent call last): + TypeError: spam() got an unexpected keyword argument 'a' + """ return (x, y, z) def grail(self, x, y, z, *a): + """ + >>> s = Silly() + >>> s.grail(1,2,3) + (1, 2, 3, ()) + >>> s.grail(1,2,3,4) + (1, 2, 3, (4,)) + >>> s.grail(1,2,3,4,5,6,7,8,9) + (1, 2, 3, (4, 5, 6, 7, 8, 9)) + >>> s.grail(1,2) + Traceback (most recent call last): + TypeError: grail() takes at least 3 positional arguments (2 given) + >>> s.grail(1,2,3, a=1) + Traceback (most recent call last): + TypeError: grail() got an unexpected keyword argument 'a' + """ return (x, y, z, a) def swallow(self, x, y, z, **k): + """ + >>> s = Silly() + >>> s.swallow(1,2,3) + (1, 2, 3, ()) + >>> s.swallow(1,2,3,4) + Traceback (most recent call last): + TypeError: swallow() takes exactly 3 positional arguments (4 given) + >>> s.swallow(1,2,3, a=1, b=2) + (1, 2, 3, (('a', 1), ('b', 2))) + >>> s.swallow(1,2,3, x=1) + Traceback (most recent call last): + TypeError: swallow() got multiple values for keyword argument 'x' + """ return (x, y, z, sorteditems(k)) def creosote(self, x, y, z, *a, **k): + """ + >>> s = Silly() + >>> s.creosote(1,2,3) + (1, 2, 3, (), ()) + >>> s.creosote(1,2,3,4) + (1, 2, 3, (4,), ()) + >>> s.creosote(1,2,3, a=1) + (1, 2, 3, (), (('a', 1),)) + >>> s.creosote(1,2,3,4, a=1, b=2) + (1, 2, 3, (4,), (('a', 1), ('b', 2))) + >>> s.creosote(1,2,3,4, x=1) + Traceback (most recent call last): + TypeError: creosote() got multiple values for keyword argument 'x' + """ return (x, y, z, a, sorteditems(k)) def onlyt(self, *a): + """ + >>> s = Silly() + >>> s.onlyt(1) + (1,) + >>> s.onlyt(1,2) + (1, 2) + >>> s.onlyt(a=1) + Traceback (most recent call last): + TypeError: onlyt() got an unexpected keyword argument 'a' + >>> s.onlyt(1, a=2) + Traceback (most recent call last): + TypeError: onlyt() got an unexpected keyword argument 'a' + """ + return a + + @cython.binding(False) # passthrough of exact same tuple can't work with binding + def onlyt_nobinding(self, *a): + """ + >>> s = Silly() + >>> s.onlyt_nobinding(1) + (1,) + >>> s.onlyt_nobinding(1,2) + (1, 2) + >>> s.onlyt_nobinding(a=1) + Traceback (most recent call last): + TypeError: onlyt_nobinding() got an unexpected keyword argument 'a' + >>> s.onlyt_nobinding(1, a=2) + Traceback (most recent call last): + TypeError: onlyt_nobinding() got an unexpected keyword argument 'a' + >>> test_no_copy_args(s.onlyt_nobinding) + True + """ return a def onlyk(self, **k): + """ + >>> s = Silly() + >>> s.onlyk(a=1) + (('a', 1),) + >>> s.onlyk(a=1, b=2) + (('a', 1), ('b', 2)) + >>> s.onlyk(1) + Traceback (most recent call last): + TypeError: onlyk() takes exactly 0 positional arguments (1 given) + >>> s.onlyk(1, 2) + Traceback (most recent call last): + TypeError: onlyk() takes exactly 0 positional arguments (2 given) + >>> s.onlyk(1, a=1, b=2) + Traceback (most recent call last): + TypeError: onlyk() takes exactly 0 positional arguments (1 given) + """ return sorteditems(k) def tk(self, *a, **k): + """ + >>> s = Silly() + >>> s.tk(a=1) + (('a', 1),) + >>> s.tk(a=1, b=2) + (('a', 1), ('b', 2)) + >>> s.tk(1) + (1,) + >>> s.tk(1, 2) + (1, 2) + >>> s.tk(1, a=1, b=2) + (1, ('a', 1), ('b', 2)) + """ return a + sorteditems(k) + + @cython.binding(False) # passthrough of exact same tuple can't work with binding + def t_kwonly(self, *a, k): + """ + >>> s = Silly() + >>> test_no_copy_args(s.t_kwonly, k=None) + True + """ + return a + + +def test_no_copy_args(func, **kw): + """ + func is a function such that func(*args, **kw) returns args. + We test that no copy is made of the args tuple. + This tests both the caller side and the callee side. + """ + args = (1, 2, 3) + return func(*args, **kw) is args diff --git a/tests/run/exttype.pyx b/tests/run/exttype.pyx index 61d0c169a..c14c1e4ed 100644 --- a/tests/run/exttype.pyx +++ b/tests/run/exttype.pyx @@ -1,6 +1,52 @@ +# mode: run +# tag: exttype, tpnew + +from __future__ import print_function + +from cpython.object cimport PyTypeObject + cdef gobble(a, b): - print a, b + print(a, b) + + +def tp_new_ptr(exttype): + assert isinstance(exttype, type) + tp = <PyTypeObject*> exttype + return <unsigned long long><void*>tp.tp_new + + +cdef class Empty: + """ + >>> n = Empty() + >>> isinstance(n, Empty) + True + >>> tp_new_ptr(Empty) != 0 + True + """ + + +cdef class EmptySubclass(Empty): + """ + >>> n = EmptySubclass() + >>> isinstance(n, EmptySubclass) + True + >>> tp_new_ptr(EmptySubclass) != 0 + True + >>> tp_new_ptr(EmptySubclass) == tp_new_ptr(Empty) + True + """ + + +cdef class CInit: + """ + >>> c = CInit() + >>> isinstance(c, CInit) + True + """ + def __cinit__(self): + assert self is not None + cdef class Spam: """ @@ -21,6 +67,7 @@ cdef class Spam: def eat(self): gobble(self.eggs, self.ham) + def f(Spam spam): """ >>> s = Spam(12) diff --git a/tests/run/fastcall.pyx b/tests/run/fastcall.pyx index ec9f5ed5b..a770fa9e3 100644 --- a/tests/run/fastcall.pyx +++ b/tests/run/fastcall.pyx @@ -1,6 +1,8 @@ # mode: run # tag: METH_FASTCALL +cimport cython + import sys import struct from collections import deque @@ -63,3 +65,64 @@ cdef class SelfCast: """ def index_of_self(self, list orbit not None): return orbit.index(self) + + +cdef extern from *: + int PyCFunction_GET_FLAGS(op) + + +def has_fastcall(meth): + """ + Given a builtin_function_or_method or cyfunction ``meth``, + return whether it uses ``METH_FASTCALL``. + """ + # Hardcode METH_FASTCALL constant equal to 0x80 for simplicity + return bool(PyCFunction_GET_FLAGS(meth) & 0x80) + + +def assert_fastcall(meth): + """ + Assert that ``meth`` uses ``METH_FASTCALL`` if the Python + implementation supports it. + """ + # getattr uses METH_FASTCALL on CPython >= 3.7 + if has_fastcall(getattr) and not has_fastcall(meth): + raise AssertionError(f"{meth} does not use METH_FASTCALL") + + +@cython.binding(False) +def fastcall_function(**kw): + """ + >>> assert_fastcall(fastcall_function) + """ + return kw + +@cython.binding(True) +def fastcall_cyfunction(**kw): + """ + >>> assert_fastcall(fastcall_cyfunction) + """ + return kw + +cdef class Dummy: + @cython.binding(False) + def fastcall_method(self, x, *args, **kw): + """ + >>> assert_fastcall(Dummy().fastcall_method) + """ + return tuple(args) + tuple(kw) + +cdef class CyDummy: + @cython.binding(True) + def fastcall_method(self, x, *args, **kw): + """ + >>> assert_fastcall(CyDummy.fastcall_method) + """ + return tuple(args) + tuple(kw) + +class PyDummy: + def fastcall_method(self, x, *args, **kw): + """ + >>> assert_fastcall(PyDummy.fastcall_method) + """ + return tuple(args) + tuple(kw) diff --git a/tests/run/file_encoding_T740.py b/tests/run/file_encoding_T740.py index f61973353..a71b04be9 100644 --- a/tests/run/file_encoding_T740.py +++ b/tests/run/file_encoding_T740.py @@ -1,6 +1,6 @@ # encoding: koi8-r # mode: run -# ticket: 740 +# ticket: t740 """ >>> wtf 'wtf' diff --git a/tests/run/final_method_T586.pyx b/tests/run/final_method_T586.pyx index 8fa1ba25c..f50bb8aa8 100644 --- a/tests/run/final_method_T586.pyx +++ b/tests/run/final_method_T586.pyx @@ -1,5 +1,5 @@ # mode: run -# ticket: 568 +# ticket: t568 cimport cython diff --git a/tests/run/float_floor_division_T260.pyx b/tests/run/float_floor_division_T260.pyx index 66f6f83f7..58d13a157 100644 --- a/tests/run/float_floor_division_T260.pyx +++ b/tests/run/float_floor_division_T260.pyx @@ -1,4 +1,4 @@ -# ticket: 260 +# ticket: t260 def floor_div_float(double a, double b): """ diff --git a/tests/run/float_len_T480.pyx b/tests/run/float_len_T480.pyx index efb456c54..4f623030a 100644 --- a/tests/run/float_len_T480.pyx +++ b/tests/run/float_len_T480.pyx @@ -1,4 +1,4 @@ -# ticket: 480 +# ticket: t480 def f(x): return x diff --git a/tests/run/for_from_float_T254.pyx b/tests/run/for_from_float_T254.pyx index 0ea153a9c..bf07f42b6 100644 --- a/tests/run/for_from_float_T254.pyx +++ b/tests/run/for_from_float_T254.pyx @@ -1,4 +1,4 @@ -# ticket: 254 +# ticket: t254 def double_target(a, b): """ diff --git a/tests/run/for_from_pyvar_loop_T601.pyx b/tests/run/for_from_pyvar_loop_T601.pyx index 2d5890d9b..949fcef06 100644 --- a/tests/run/for_from_pyvar_loop_T601.pyx +++ b/tests/run/for_from_pyvar_loop_T601.pyx @@ -1,4 +1,4 @@ -# ticket: 601 +# ticket: t601 cdef unsigned long size2(): return 3 diff --git a/tests/run/for_in_break_continue_T533.pyx b/tests/run/for_in_break_continue_T533.pyx index 0baa9fa49..32f646aaf 100644 --- a/tests/run/for_in_break_continue_T533.pyx +++ b/tests/run/for_in_break_continue_T533.pyx @@ -1,4 +1,4 @@ -# ticket: 533 +# ticket: t533 def for_in(): """ diff --git a/tests/run/for_in_range_T372.pyx b/tests/run/for_in_range_T372.pyx index bff686295..11ef4b592 100644 --- a/tests/run/for_in_range_T372.pyx +++ b/tests/run/for_in_range_T372.pyx @@ -1,5 +1,5 @@ # mode: run -# ticket: 372 +# ticket: t372 cimport cython diff --git a/tests/run/fstring.pyx b/tests/run/fstring.pyx index 45bfaf5e3..dccc8dff3 100644 --- a/tests/run/fstring.pyx +++ b/tests/run/fstring.pyx @@ -18,6 +18,65 @@ min_long = LONG_MIN @cython.test_fail_if_path_exists( + "//JoinedStrNode", +) +@cython.test_assert_path_exists( + "//AddNode", +) +def concat_strings(a, b): + """ + >>> concat_strings("", "") + x + <BLANKLINE> + x + x + x + xx + >>> concat_strings("a", "") + ax + a + x + ax + ax + axx + >>> concat_strings("", "b") + x + b + xb + xb + xb + xxb + >>> concat_strings("a", "b") + ax + ab + xb + axb + axb + axxb + >>> concat_strings("".join(["a", "b"]), "") # fresh temp string left + abx + ab + x + abx + abx + abxx + >>> concat_strings("", "".join(["a", "b"])) # fresh temp string right + x + ab + xab + xab + xab + xxab + """ + print(f"{a}x") + print(f"{a}{b}") + print(f"x{b}") + print(f"{a+'x'}{b}") # fresh temp string left + print(f"{a}{'x'+b}") # fresh temp string right + print(f"{a+'x'}{'x'+b}") # fresh temp strings right and left + + +@cython.test_fail_if_path_exists( "//FormattedValueNode", "//JoinedStrNode", "//AddNode", @@ -519,6 +578,27 @@ def percent_s_unicode(u, int i): return u"%s-%d" % (u, i) +@cython.test_assert_path_exists( + "//FormattedValueNode", +) +def sideeffect(l): + """ + >>> class Listish(list): + ... def __format__(self, format_spec): + ... self.append("format called") + ... return repr(self) + ... def append(self, item): + ... list.append(self, item) + ... return self + + >>> l = Listish() + >>> sideeffect(l) + [123, 'format called'] + """ + f"{l.append(123)}" # unused f-string ! + return list(l) + + ######################################## # await inside f-string diff --git a/tests/run/funcexc_iter_T228.pyx b/tests/run/funcexc_iter_T228.pyx index 0c5bde250..4b81166f6 100644 --- a/tests/run/funcexc_iter_T228.pyx +++ b/tests/run/funcexc_iter_T228.pyx @@ -1,4 +1,4 @@ -# ticket: 228 +# ticket: t228 __doc__ = u""" >>> def py_iterator(): diff --git a/tests/run/function_as_method_T494.pyx b/tests/run/function_as_method_T494.pyx index 34728bc8c..92deafc6d 100644 --- a/tests/run/function_as_method_T494.pyx +++ b/tests/run/function_as_method_T494.pyx @@ -1,4 +1,4 @@ -# ticket: 494 +# ticket: t494 # cython: binding=True __doc__ = """ @@ -12,3 +12,39 @@ class A: def foo(self): return self is not None + +# assignment of functions used in a "static method" type way behaves differently +# in Python2 and 3 +import sys +if sys.version_info[0] == 2: + __doc__ = """>>> B.plus1(1) #doctest: +IGNORE_EXCEPTION_DETAIL +Traceback (most recent call last): + ... +TypeError: unbound +""" +else: + __doc__ = """>>> B.plus1(1) +2 +""" + +# with binding==False assignment of functions always worked - doesn't match Python +# behaviour but ensures Cython behaviour stays consistent +__doc__ += """ +>>> B.plus1_nobind(1) +2 +""" + +cimport cython + +def f_plus(a): + return a + 1 + +@cython.binding(False) +def f_plus_nobind(a): + return a+1 + +cdef class B: + plus1 = f_plus + plus1_nobind = f_plus_nobind + + diff --git a/tests/run/function_as_method_py_T494.py b/tests/run/function_as_method_py_T494.py index 49d5f27ce..5317d8848 100644 --- a/tests/run/function_as_method_py_T494.py +++ b/tests/run/function_as_method_py_T494.py @@ -1,4 +1,4 @@ -# ticket: 494 +# ticket: t494 __doc__ = """ >>> A.foo = foo @@ -11,3 +11,35 @@ class A: def foo(self): return self is not None + + +# assignment of functions used in a "static method" type way behaves differently +# in Python2 and 3 +import sys +if sys.version_info[0] == 2: + __doc__ = u""" +>>> B.plus1(1) #doctest: +IGNORE_EXCEPTION_DETAIL +Traceback (most recent call last): + ... +TypeError: unbound +>>> C.plus1(1) #doctest: +IGNORE_EXCEPTION_DETAIL +Traceback (most recent call last): + ... +TypeError: unbound +""" +else: + __doc__ = u""" +>>> B.plus1(1) +2 +>>> C.plus1(1) +2 +""" + +def f_plus(a): + return a + 1 + +class B: + plus1 = f_plus + +class C(object): + plus1 = f_plus diff --git a/tests/run/function_binding_T494.pyx b/tests/run/function_binding_T494.pyx index b41221e2c..86c64c9c0 100644 --- a/tests/run/function_binding_T494.pyx +++ b/tests/run/function_binding_T494.pyx @@ -1,4 +1,4 @@ -# ticket: 494 +# ticket: t494 cimport cython diff --git a/tests/run/function_self.py b/tests/run/function_self.py new file mode 100644 index 000000000..e4796c46b --- /dev/null +++ b/tests/run/function_self.py @@ -0,0 +1,91 @@ +# mode: run +# tag: pure2.7 + +# cython: binding=True + +import cython +import sys + +def regular(x): + """ + >>> hasattr(regular, "__self__") + False + >>> nested = regular(10) + >>> hasattr(nested, "__self__") + False + """ + def nested(y): + return x+y + return nested + +@cython.locals(x=cython.floating) +def fused(x): + """ + >>> nested = fused(10.) + >>> hasattr(nested, "__self__") + False + + #>>> hasattr(fused, "__self__") # FIXME this fails for fused functions + #False + >>> fused.__self__ # but this is OK + Traceback (most recent call last): + ... + AttributeError: 'function' object has no attribute '__self__' + """ + def nested_in_fused(y): + return x+y + return nested_in_fused + +# FIXME - doesn't currently work at all +#def get_nested_fused(x): +# @cython.locals(x=cython.floating) +# def nested_fused(y): +# return x+y +# return nested_fused + +class C: + """ + >>> c = C() + >>> c.regular.__self__ is c + True + >>> c.fused.__self__ is c + True + """ + def regular(self): + pass + + @cython.locals(x=cython.floating) + def fused(self, x): + return x + +__doc__ = "" +if sys.version_info[0] > 2 or cython.compiled: + __doc__ += """ + >>> hasattr(C.regular, "__self__") # __self__==None on pure-python 2 + False + >>> C.fused.__self__ # returns None on pure-python 2 + Traceback (most recent call last): + ... + AttributeError: 'function' object has no attribute '__self__' + """ + +if cython.compiled: + __doc__ = """ + >>> fused['double'].__self__ + Traceback (most recent call last): + ... + AttributeError: 'function' object has no attribute '__self__' + + >>> C.fused['double'].__self__ + Traceback (most recent call last): + ... + AttributeError: 'function' object has no attribute '__self__' + + >>> c = C() + >>> c.fused['double'].__self__ is c + True + + # The PR that changed __self__ also changed how __doc__ is set up slightly + >>> fused['double'].__doc__ == fused.__doc__ and isinstance(fused.__doc__, str) + True + """ diff --git a/tests/run/fused_bound_functions.py b/tests/run/fused_bound_functions.py new file mode 100644 index 000000000..650bc51ff --- /dev/null +++ b/tests/run/fused_bound_functions.py @@ -0,0 +1,151 @@ +# mode: run +# tag: pure3.0 +# cython: binding=True + +""" +Test that fused functions can be used in the same way as CyFunctions with respect to +assigning them to class attributes. Previously they enforced extra type/argument checks +beyond those which CyFunctions did. +""" + +import cython + +MyFusedClass = cython.fused_type( + float, + 'Cdef', + object) + +def fused_func(x: MyFusedClass): + return (type(x).__name__, cython.typeof(x)) + +IntOrFloat = cython.fused_type(int, float) + +def fused_func_0(x: IntOrFloat = 0): + """ + Fused functions can legitimately take 0 arguments + >>> fused_func_0() + ('int', 'int') + + # subscripted in module __doc__ conditionally + """ + return (type(x).__name__, cython.typeof(x)) + +def regular_func(x): + return (type(x).__name__, cython.typeof(x)) + +def regular_func_0(): + return + +@cython.cclass +class Cdef: + __doc__ = """ + >>> c = Cdef() + + # functions are callable with an instance of c + >>> c.fused_func() + ('Cdef', 'Cdef') + >>> c.regular_func() + ('Cdef', '{typeofCdef}') + >>> c.fused_in_class(1.5) + ('float', 'float') + + # Fused functions are callable without an instance + # (This applies to everything in Py3 - see __doc__ below) + >>> Cdef.fused_func(1.5) + ('float', 'float') + >>> Cdef.fused_in_class(c, 1.5) + ('float', 'float') + >>> Cdef.fused_func_0() + ('int', 'int') + + # Functions not expecting an argument don't work with an instance + >>> c.regular_func_0() # doctest: +ELLIPSIS + Traceback (most recent call last): + TypeError: regular_func_0() takes ... arguments ...1... given... + """.format(typeofCdef = 'Python object' if cython.compiled else 'Cdef') + + if cython.compiled: + __doc__ += """ + + # fused_func_0 does not accept a "Cdef" instance + >>> c.fused_func_0() + Traceback (most recent call last): + TypeError: No matching signature found + + # subscripting requires fused methods (so not pure Python) + >>> Cdef.fused_func_0['float']() + ('float', 'float') + >>> c.fused_func_0['float']() # doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + TypeError: (Exception looks quite different in Python2 and 3 so no way to match both) + """ + fused_func = fused_func + fused_func_0 = fused_func_0 + regular_func = regular_func + regular_func_0 = regular_func_0 + + def fused_in_class(self, x: MyFusedClass): + return (type(x).__name__, cython.typeof(x)) + + def regular_in_class(self): + return type(self).__name__ + +class Regular(object): + __doc__ = """ + >>> c = Regular() + + # Functions are callable with an instance of C + >>> c.fused_func() + ('Regular', '{typeofRegular}') + >>> c.regular_func() + ('Regular', '{typeofRegular}') + + # Fused functions are callable without an instance + # (This applies to everything in Py3 - see __doc__ below) + >>> Regular.fused_func(1.5) + ('float', 'float') + >>> Regular.fused_func_0() + ('int', 'int') + + # Functions not expecting an argument don't work with an instance + >>> c.regular_func_0() # doctest: +ELLIPSIS + Traceback (most recent call last): + TypeError: regular_func_0() takes ... arguments ...1... given... + """.format(typeofRegular = "Python object" if cython.compiled else 'Regular') + if cython.compiled: + __doc__ += """ + # fused_func_0 does not accept a "Regular" instance + >>> c.fused_func_0() + Traceback (most recent call last): + TypeError: No matching signature found + + # subscripting requires fused methods (so not pure Python) + >>> c.fused_func_0['float']() # doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + TypeError: (Exception looks quite different in Python2 and 3 so no way to match both) + >>> Regular.fused_func_0['float']() + ('float', 'float') + """ + + fused_func = fused_func + fused_func_0 = fused_func_0 + regular_func = regular_func + regular_func_0 = regular_func_0 + +import sys +if sys.version_info[0] > 2: + # extra Py3 only tests - shows that functions added to a class can be called + # with an type as the first argument + __doc__ = """ + >>> Cdef.regular_func(1.5) + ('float', '{typeoffloat}') + >>> Regular.regular_func(1.5) + ('float', '{typeoffloat}') + >>> Cdef.regular_func_0() + >>> Regular.regular_func_0() + """.format(typeoffloat='Python object' if cython.compiled else 'float') +if cython.compiled: + __doc__ += """ + >>> fused_func_0['float']() + ('float', 'float') + """ diff --git a/tests/run/fused_cpdef.pyx b/tests/run/fused_cpdef.pyx index 0b63c8b98..4a614e0f4 100644 --- a/tests/run/fused_cpdef.pyx +++ b/tests/run/fused_cpdef.pyx @@ -1,13 +1,17 @@ +# cython: language_level=3 +# mode: run + cimport cython +import sys, io cy = __import__("cython") cpdef func1(self, cython.integral x): - print "%s," % (self,), + print(f"{self},", end=' ') if cython.integral is int: - print 'x is int', x, cython.typeof(x) + print('x is int', x, cython.typeof(x)) else: - print 'x is long', x, cython.typeof(x) + print('x is long', x, cython.typeof(x)) class A(object): @@ -16,6 +20,18 @@ class A(object): def __str__(self): return "A" +cdef class B: + cpdef int meth(self, cython.integral x): + print(f"{self},", end=' ') + if cython.integral is int: + print('x is int', x, cython.typeof(x)) + else: + print('x is long', x, cython.typeof(x)) + return 0 + + def __str__(self): + return "B" + pyfunc = func1 def test_fused_cpdef(): @@ -32,23 +48,71 @@ def test_fused_cpdef(): A, x is long 2 long A, x is long 2 long A, x is long 2 long + <BLANKLINE> + B, x is long 2 long """ func1[int](None, 2) func1[long](None, 2) func1(None, 2) - print + print() pyfunc[cy.int](None, 2) pyfunc(None, 2) - print + print() A.meth[cy.int](A(), 2) A.meth(A(), 2) A().meth[cy.long](2) A().meth(2) + print() + + B().meth(2) + + +midimport_run = io.StringIO() +if sys.version_info.major < 3: + # Monkey-patch midimport_run.write to accept non-unicode strings under Python 2. + midimport_run.write = lambda c: io.StringIO.write(midimport_run, unicode(c)) + +realstdout = sys.stdout +sys.stdout = midimport_run + +try: + # Run `test_fused_cpdef()` during import and save the result for + # `test_midimport_run()`. + test_fused_cpdef() +except Exception as e: + midimport_run.write(f"{e!r}\n") +finally: + sys.stdout = realstdout + +def test_midimport_run(): + # At one point, dynamically calling fused cpdef functions during import + # would fail because the type signature-matching indices weren't + # yet initialized. + # (See Compiler.FusedNode.FusedCFuncDefNode._fused_signature_index, + # GH-3366.) + """ + >>> test_midimport_run() + None, x is int 2 int + None, x is long 2 long + None, x is long 2 long + <BLANKLINE> + None, x is int 2 int + None, x is long 2 long + <BLANKLINE> + A, x is int 2 int + A, x is long 2 long + A, x is long 2 long + A, x is long 2 long + <BLANKLINE> + B, x is long 2 long + """ + print(midimport_run.getvalue(), end='') + def assert_raise(func, *args): try: @@ -70,23 +134,31 @@ def test_badcall(): assert_raise(A.meth) assert_raise(A().meth[cy.int]) assert_raise(A.meth[cy.int]) + assert_raise(B().meth, 1, 2, 3) + +def test_nomatch(): + """ + >>> func1(None, ()) + Traceback (most recent call last): + TypeError: No matching signature found + """ ctypedef long double long_double cpdef multiarg(cython.integral x, cython.floating y): if cython.integral is int: - print "x is an int,", + print("x is an int,", end=' ') else: - print "x is a long,", + print("x is a long,", end=' ') if cython.floating is long_double: - print "y is a long double:", + print("y is a long double:", end=' ') elif float is cython.floating: - print "y is a float:", + print("y is a float:", end=' ') else: - print "y is a double:", + print("y is a double:", end=' ') - print x, y + print(x, y) def test_multiarg(): """ @@ -104,3 +176,13 @@ def test_multiarg(): multiarg[int, float](1, 2.0) multiarg[cy.int, cy.float](1, 2.0) multiarg(4, 5.0) + +def test_ambiguousmatch(): + """ + >>> multiarg(5, ()) + Traceback (most recent call last): + TypeError: Function call with ambiguous argument types + >>> multiarg((), 2.0) + Traceback (most recent call last): + TypeError: Function call with ambiguous argument types + """ diff --git a/tests/run/fused_cpp.pyx b/tests/run/fused_cpp.pyx index c5a0d7d34..9f3bb5104 100644 --- a/tests/run/fused_cpp.pyx +++ b/tests/run/fused_cpp.pyx @@ -2,6 +2,8 @@ cimport cython from libcpp.vector cimport vector +from libcpp.typeinfo cimport type_info +from cython.operator cimport typeid def test_cpp_specialization(cython.floating element): """ @@ -14,3 +16,28 @@ def test_cpp_specialization(cython.floating element): cdef vector[cython.floating] *v = new vector[cython.floating]() v.push_back(element) print cython.typeof(v), cython.typeof(element), v.at(0) + +cdef fused C: + int + object + +cdef const type_info* tidint = &typeid(int) +def typeid_call(C x): + """ + For GH issue 3203 + >>> typeid_call(1) + True + """ + cdef const type_info* a = &typeid(C) + return a[0] == tidint[0] + +cimport cython + +def typeid_call2(cython.integral x): + """ + For GH issue 3203 + >>> typeid_call2[int](1) + True + """ + cdef const type_info* a = &typeid(cython.integral) + return a[0] == tidint[0] diff --git a/tests/run/fused_def.pyx b/tests/run/fused_def.pyx index f0e1be4c8..0f5ec1bbd 100644 --- a/tests/run/fused_def.pyx +++ b/tests/run/fused_def.pyx @@ -54,9 +54,13 @@ f = 5.6 i = 9 -def opt_func(fused_t obj, cython.floating myf = 1.2, cython.integral myi = 7): +def opt_func(fused_t obj, cython.floating myf = 1.2, cython.integral myi = 7, + another_opt = 2, yet_another_opt=3): """ - Test runtime dispatch, indexing of various kinds and optional arguments + Test runtime dispatch, indexing of various kinds and optional arguments. + Use 5 arguments because at one point the optional argument from the + 5th argument was overwriting that of the __pyx_fused dispatcher. + https://github.com/cython/cython/issues/3511 >>> opt_func("spam", f, i) str object double long @@ -121,9 +125,9 @@ def opt_func(fused_t obj, cython.floating myf = 1.2, cython.integral myi = 7): >>> opt_func() Traceback (most recent call last): TypeError: Expected at least 1 argument, got 0 - >>> opt_func("abc", f, i, 5) # doctest: +ELLIPSIS + >>> opt_func("abc", f, i, 5, 5, 5) # doctest: +ELLIPSIS Traceback (most recent call last): - TypeError: ...at most 3... + TypeError: ...at most 5... >>> opt_func[ExtClassA, cy.float, cy.long](object(), f) Traceback (most recent call last): TypeError: Argument 'obj' has incorrect type (expected fused_def.ExtClassA, got object) @@ -139,7 +143,7 @@ def run_cyfunction_check(): fused_cython_function 1 """ - print(type(opt_func).__name__) + print(type(opt_func).__name__.rsplit('.', 1)[-1]) print(__Pyx_CyFunction_Check(opt_func)) # should be True def test_opt_func(): @@ -154,19 +158,19 @@ def test_opt_func(): def test_opt_func_introspection(): """ >>> opt_func.__defaults__ - (1.2, 7) + (1.2, 7, 2, 3) >>> opt_func.__kwdefaults__ >>> opt_func.__annotations__ {} >>> opt_func[str, float, int].__defaults__ - (1.2, 7) + (1.2, 7, 2, 3) >>> opt_func[str, float, int].__kwdefaults__ >>> opt_func[str, float, int].__annotations__ {} >>> opt_func[str, cy.double, cy.long].__defaults__ - (1.2, 7) + (1.2, 7, 2, 3) >>> opt_func[str, cy.double, cy.long].__kwdefaults__ >>> opt_func[str, cy.double, cy.long].__annotations__ {} diff --git a/tests/run/fused_types.pyx b/tests/run/fused_types.pyx index 4e96bab3a..3a0009bdd 100644 --- a/tests/run/fused_types.pyx +++ b/tests/run/fused_types.pyx @@ -1,4 +1,5 @@ # mode: run +# ticket: t1772 cimport cython from cython.view cimport array @@ -20,6 +21,7 @@ ctypedef double *p_double ctypedef int *p_int fused_type3 = cython.fused_type(int, double) fused_composite = cython.fused_type(fused_type2, fused_type3) +just_float = cython.fused_type(float) def test_pure(): """ @@ -363,6 +365,22 @@ def test_fused_memslice_dtype_repeated_2(cython.floating[:] array1, cython.float """ print cython.typeof(array1), cython.typeof(array2), cython.typeof(array3) +def test_fused_const_memslice_dtype_repeated(const cython.floating[:] array1, cython.floating[:] array2): + """Test fused types memory view with one being const + + >>> sorted(test_fused_const_memslice_dtype_repeated.__signatures__) + ['double', 'float'] + + >>> test_fused_const_memslice_dtype_repeated(get_array(8, 'd'), get_array(8, 'd')) + const double[:] double[:] + >>> test_fused_const_memslice_dtype_repeated(get_array(4, 'f'), get_array(4, 'f')) + const float[:] float[:] + >>> test_fused_const_memslice_dtype_repeated(get_array(8, 'd'), get_array(4, 'f')) + Traceback (most recent call last): + ValueError: Buffer dtype mismatch, expected 'double' but got 'float' + """ + print cython.typeof(array1), cython.typeof(array2) + def test_cython_numeric(cython.numeric arg): """ Test to see whether complex numbers have their utility code declared @@ -388,6 +406,18 @@ def test_index_fused_args(cython.floating f, ints_t i): """ _test_index_fused_args[cython.floating, ints_t](f, i) +cdef _test_index_const_fused_args(const cython.floating f, const ints_t i): + print(cython.typeof(f), cython.typeof(i)) + +def test_index_const_fused_args(const cython.floating f, const ints_t i): + """Test indexing function implementation with const fused type args + + >>> import cython + >>> test_index_const_fused_args[cython.double, cython.int](2.0, 3) + ('const double', 'const int') + """ + _test_index_const_fused_args[cython.floating, ints_t](f, i) + def test_composite(fused_composite x): """ @@ -404,6 +434,60 @@ def test_composite(fused_composite x): return 2 * x +cdef cdef_func_const_fused_arg(const cython.floating val, + const fused_type1 * ptr_to_const, + const (cython.floating *) const_ptr): + print(val, cython.typeof(val)) + print(ptr_to_const[0], cython.typeof(ptr_to_const[0])) + print(const_ptr[0], cython.typeof(const_ptr[0])) + + ptr_to_const = NULL # pointer is not const, value is const + const_ptr[0] = 0.0 # pointer is const, value is not const + +def test_cdef_func_with_const_fused_arg(): + """Test cdef function with const fused type argument + + >>> test_cdef_func_with_const_fused_arg() + (0.0, 'const float') + (1, 'const int') + (2.0, 'float') + """ + cdef float arg0 = 0.0 + cdef int arg1 = 1 + cdef float arg2 = 2.0 + cdef_func_const_fused_arg(arg0, &arg1, &arg2) + + +cdef in_check_1(just_float x): + return just_float in floating + +cdef in_check_2(just_float x, floating y): + # the "floating" on the right-hand side of the in statement should not be specialized + # - the test should still work. + return just_float in floating + +cdef in_check_3(floating x): + # the floating on the left-hand side of the in statement should be specialized + # but the one of the right-hand side should not (so that the test can still work). + return floating in floating + +def test_fused_in_check(): + """ + It should be possible to use fused types on in "x in ...fused_type" statements + even if that type is specialized in the function. + + >>> test_fused_in_check() + True + True + True + True + """ + print(in_check_1(1.0)) + print(in_check_2(1.0, 2.0)) + print(in_check_2[float, double](1.0, 2.0)) + print(in_check_3[float](1.0)) + + ### see GH3642 - presence of cdef inside "unrelated" caused a type to be incorrectly inferred cdef unrelated(cython.floating x): cdef cython.floating t = 1 diff --git a/tests/run/generator_expressions_and_locals.pyx b/tests/run/generator_expressions_and_locals.pyx index 7a87164ff..a239f9b29 100644 --- a/tests/run/generator_expressions_and_locals.pyx +++ b/tests/run/generator_expressions_and_locals.pyx @@ -1,6 +1,6 @@ # mode: run # tag: genexpr, locals -# ticket: 715 +# ticket: t715 def genexpr_not_in_locals(): """ diff --git a/tests/run/generators.pyx b/tests/run/generators.pyx index b8a795428..75551a71e 100644 --- a/tests/run/generators.pyx +++ b/tests/run/generators.pyx @@ -504,6 +504,26 @@ def test_generator_abc(): yield 1 +def test_generator_frame(a=1): + """ + >>> gen = test_generator_frame() + >>> import types + >>> isinstance(gen.gi_frame, types.FrameType) or gen.gi_frame + True + >>> gen.gi_frame is gen.gi_frame # assert that it's cached + True + >>> gen.gi_frame.f_code is not None + True + >>> code_obj = gen.gi_frame.f_code + >>> code_obj.co_argcount + 1 + >>> code_obj.co_varnames + ('a', 'b') + """ + b = a + 1 + yield b + + # GH Issue 3265 - **kwds could cause a crash in some cases due to not # handling NULL pointers (in testing it shows as a REFNANNY error). # This was on creation of the generator and diff --git a/tests/run/generators_GH1731.pyx b/tests/run/generators_GH1731.pyx index 99cae90ca..77b0877f2 100644 --- a/tests/run/generators_GH1731.pyx +++ b/tests/run/generators_GH1731.pyx @@ -1,5 +1,5 @@ # mode: run -# ticket: gh1731 +# ticket: 1731 def cygen(): diff --git a/tests/run/genexpr_T491.pyx b/tests/run/genexpr_T491.pyx index 7fa6c1754..7640b3518 100644 --- a/tests/run/genexpr_T491.pyx +++ b/tests/run/genexpr_T491.pyx @@ -1,4 +1,4 @@ -# ticket: 491 +# ticket: t491 def test_genexpr(): """ diff --git a/tests/run/genexpr_T715.pyx b/tests/run/genexpr_T715.pyx index 2c6f5d6c8..028a93128 100644 --- a/tests/run/genexpr_T715.pyx +++ b/tests/run/genexpr_T715.pyx @@ -1,5 +1,5 @@ # mode: run -# ticket: 715 +# ticket: t715 # tag: genexpr, comprehension def t715(*items): diff --git a/tests/run/genexpr_iterable_lookup_T600.pyx b/tests/run/genexpr_iterable_lookup_T600.pyx index 220098a1f..945652717 100644 --- a/tests/run/genexpr_iterable_lookup_T600.pyx +++ b/tests/run/genexpr_iterable_lookup_T600.pyx @@ -1,5 +1,5 @@ # mode: run -# ticket: 600 +# ticket: t600 # tag: genexpr # cython: language_level=3 diff --git a/tests/run/hash_T326.pyx b/tests/run/hash_T326.pyx index 11184837f..bc9566879 100644 --- a/tests/run/hash_T326.pyx +++ b/tests/run/hash_T326.pyx @@ -1,5 +1,5 @@ # mode: run -# ticket: 326 +# ticket: t326 # tag: hash diff --git a/tests/run/if_and_or.pyx b/tests/run/if_and_or.pyx new file mode 100644 index 000000000..6f2534078 --- /dev/null +++ b/tests/run/if_and_or.pyx @@ -0,0 +1,119 @@ +# mode: run +# tag: if, and, or + +def if_x(x): + """ + >>> if_x(0) + 2 + >>> if_x(1) + 1 + """ + if x: + return 1 + else: + return 2 + +def if_not(x): + """ + >>> if_not(0) + 1 + >>> if_not(1) + 2 + """ + if not x: + return 1 + else: + return 2 + + +def if_and(a, b): + """ + >>> if_and(3, 0) + 2 + >>> if_and(0, 3) + 2 + >>> if_and(0, 0) + 2 + >>> if_and(3, 3) + 1 + """ + if a and b: + return 1 + else: + return 2 + + +def if_not_and(a, b): + """ + >>> if_not_and(3, 0) + 1 + >>> if_not_and(0, 3) + 1 + >>> if_not_and(0, 0) + 1 + >>> if_not_and(3, 3) + 2 + """ + if not (a and b): + return 1 + else: + return 2 + + +def if_or(a, b): + """ + >>> if_or(3, 0) + 1 + >>> if_or(0, 3) + 1 + >>> if_or(0, 0) + 2 + >>> if_or(3, 3) + 1 + """ + if a or b: + return 1 + else: + return 2 + + +def if_not_or(a, b): + """ + >>> if_not_or(3, 0) + 2 + >>> if_not_or(0, 3) + 2 + >>> if_not_or(0, 0) + 1 + >>> if_not_or(3, 3) + 2 + """ + if not (a or b): + return 1 + else: + return 2 + + +def if_and_or(a, b, c, d): + """ + >>> if_and_or(3, 0, 0, 3) + 1 + >>> if_and_or(0, 3, 0, 3) + 1 + >>> if_and_or(0, 3, 3, 0) + 1 + >>> if_and_or(0, 3, 3, 0) + 1 + >>> if_and_or(0, 0, 0, 0) + 2 + >>> if_and_or(0, 3, 0, 0) + 2 + >>> if_and_or(0, 0, 3, 0) + 2 + >>> if_and_or(0, 0, 0, 3) + 2 + """ + if (a or b) and (c or d): + return 1 + else: + return 2 diff --git a/tests/run/ifelseexpr_T267.pyx b/tests/run/ifelseexpr_T267.pyx index 24dabc442..973c1979a 100644 --- a/tests/run/ifelseexpr_T267.pyx +++ b/tests/run/ifelseexpr_T267.pyx @@ -1,6 +1,6 @@ # mode: run # tag: condexpr -# ticket: 267 +# ticket: t267 cimport cython diff --git a/tests/run/import_error_T734.py b/tests/run/import_error_T734.py index 4138fba1c..efcd79944 100644 --- a/tests/run/import_error_T734.py +++ b/tests/run/import_error_T734.py @@ -1,5 +1,5 @@ # mode: run -# ticket: 734 +# ticket: t734 def test_import_error(): """ diff --git a/tests/run/importas.pyx b/tests/run/importas.pyx index c57b057a2..6b3c98304 100644 --- a/tests/run/importas.pyx +++ b/tests/run/importas.pyx @@ -1,4 +1,14 @@ +# mode: run +# tag: all_language_levels + __doc__ = u""" +>>> try: sys +... except NameError: pass +... else: print("sys was defined!") +>>> try: distutils +... except NameError: pass +... else: print("distutils was defined!") + >>> import sys as sous >>> import distutils.core as corey >>> from copy import deepcopy as copey diff --git a/tests/run/importfrom.pyx b/tests/run/importfrom.pyx index 52bc67611..a3e5de78d 100644 --- a/tests/run/importfrom.pyx +++ b/tests/run/importfrom.pyx @@ -68,6 +68,10 @@ def typed_imports(): try: from sys import version_info as maxunicode except TypeError, e: + if getattr(sys, "pypy_version_info", None): + # translate message + if e.args[0].startswith("int() argument must be"): + e = "an integer is required" print(e) try: diff --git a/tests/run/in_list_with_side_effects_T544.pyx b/tests/run/in_list_with_side_effects_T544.pyx index 3bb3954c3..4a75f206a 100644 --- a/tests/run/in_list_with_side_effects_T544.pyx +++ b/tests/run/in_list_with_side_effects_T544.pyx @@ -1,4 +1,4 @@ -# ticket: 544 +# ticket: t544 def count(i=[0]): i[0] += 1 diff --git a/tests/run/initial_file_path.srctree b/tests/run/initial_file_path.srctree index a55df688b..e90cf6866 100644 --- a/tests/run/initial_file_path.srctree +++ b/tests/run/initial_file_path.srctree @@ -29,8 +29,8 @@ except ImportError as e: traceback.print_exc() def test(): - print "FILE: ", initial_file - print "PATH: ", initial_path + print("FILE: ", initial_file) + print("PATH: ", initial_path) assert initial_path[0].endswith('my_test_package'), initial_path assert initial_file.endswith('__init__.py'), initial_file assert import_error is None, import_error @@ -51,8 +51,8 @@ except ImportError as e: traceback.print_exc() def test(): - print "FILE: ", initial_file - print "PATH: ", initial_path + print("FILE: ", initial_file) + print("PATH: ", initial_path) assert initial_path[0].endswith('another'), initial_path assert initial_file.endswith('__init__.py'), initial_file assert import_error is None, import_error diff --git a/tests/run/inlinepxd.pyx b/tests/run/inlinepxd.pyx index 3d724f7d3..65c596c36 100644 --- a/tests/run/inlinepxd.pyx +++ b/tests/run/inlinepxd.pyx @@ -1,3 +1,8 @@ +# mode: run +# tag: inline, pxd + +# cython: wraparound = False + __doc__ = u""" >>> f() 3 @@ -28,3 +33,12 @@ def i(): def j(): return my_add3(2, 4) + +def test_wraparound(): + """ + >>> test_wraparound() + 1.0 + """ + # the wraparound directive from this scope should not affect the inline pxd + a = [ 0.0, 1.0 ] + return inlinepxd_support.index(a) diff --git a/tests/run/inlinepxd_support.pxd b/tests/run/inlinepxd_support.pxd index c2941e2b3..863f4eaf1 100644 --- a/tests/run/inlinepxd_support.pxd +++ b/tests/run/inlinepxd_support.pxd @@ -1,3 +1,9 @@ cdef inline int my_add(int a, int b=1, int c=0): return a + b + c + +cdef inline index(list L): + # This function should *not* be affected by directives set in the outer scope, such as "wraparound". + # See https://github.com/cython/cython/issues/1071 + return L[-1] + diff --git a/tests/run/int128.pyx b/tests/run/int128.pyx index cb18ccbd8..8e31ee141 100644 --- a/tests/run/int128.pyx +++ b/tests/run/int128.pyx @@ -117,3 +117,73 @@ def signed_conversion(x): """ cdef int128_t n = x return n + + +def get_int_distribution(shuffle=True): + """ + >>> L = get_int_distribution() + >>> bigint(L[0]) + 682 + >>> bigint(L[ len(L) // 2 ]) + 5617771410183435 + >>> bigint(L[-1]) + 52818775009509558395695966805 + >>> len(L) + 66510 + """ + # Large integers that cover 1-4 (30 bits) or 1-7 (15 bits) PyLong digits. + # Uses only integer calculations to avoid rounding issues. + pow2 = [2**exp for exp in range(98)] + ints = [ + n // 3 + for i in range(11, len(pow2) - 1) + # Take a low but growing number of integers from each power-of-2 range. + for n in range(pow2[i], pow2[i+1], pow2[i - 8] - 1) + ] + return ints * 3 # longer list, but keeps median in the middle + + +def intsum(L): + """ + >>> L = get_int_distribution() + >>> bigint(intsum(L)) + 61084913298497804284622382871263 + >>> bigint(sum(L)) + 61084913298497804284622382871263 + + >>> from random import shuffle + >>> shuffle(L) + >>> bigint(intsum(L)) + 61084913298497804284622382871263 + """ + cdef uint128_t i, x = 0 + for i in L: + x += i + return x + + +def intxor(L): + """ + >>> L = get_int_distribution() + >>> bigint(intxor(L)) + 31773794341658093722410838161 + >>> bigint(intxor(L * 2)) + 0 + >>> import operator + >>> from functools import reduce + >>> bigint(reduce(operator.xor, L)) + 31773794341658093722410838161 + >>> bigint(reduce(operator.xor, L * 2)) + 0 + + >>> from random import shuffle + >>> shuffle(L) + >>> bigint(intxor(L)) + 31773794341658093722410838161 + >>> bigint(intxor(L * 2)) + 0 + """ + cdef uint128_t i, x = 0 + for i in L: + x ^= i + return x diff --git a/tests/run/int_float_builtins_as_casts_T400.pyx b/tests/run/int_float_builtins_as_casts_T400.pyx index 6d02eff36..99d1796ed 100644 --- a/tests/run/int_float_builtins_as_casts_T400.pyx +++ b/tests/run/int_float_builtins_as_casts_T400.pyx @@ -1,4 +1,4 @@ -# ticket: 400 +# ticket: t400 cimport cython diff --git a/tests/run/int_float_builtins_as_casts_T400_long_double.pyx b/tests/run/int_float_builtins_as_casts_T400_long_double.pyx index d7e61f43b..776434ee8 100644 --- a/tests/run/int_float_builtins_as_casts_T400_long_double.pyx +++ b/tests/run/int_float_builtins_as_casts_T400_long_double.pyx @@ -1,4 +1,4 @@ -# ticket: 400 +# ticket: t400 cimport cython diff --git a/tests/run/intern_T431.pyx b/tests/run/intern_T431.pyx index 5851d9741..6b7ef0516 100644 --- a/tests/run/intern_T431.pyx +++ b/tests/run/intern_T431.pyx @@ -1,4 +1,4 @@ -# ticket: 431 +# ticket: t431 __doc__ = u""" >>> s == s_interned diff --git a/tests/run/ipow_crash_T562.pyx b/tests/run/ipow_crash_T562.pyx index 6fea958b5..7b074ea5d 100644 --- a/tests/run/ipow_crash_T562.pyx +++ b/tests/run/ipow_crash_T562.pyx @@ -1,4 +1,4 @@ -# ticket: 562 +# ticket: t562 class IPOW: """ diff --git a/tests/run/isnot.pyx b/tests/run/isnot.pyx index 9baabbc7e..f7efd017d 100644 --- a/tests/run/isnot.pyx +++ b/tests/run/isnot.pyx @@ -3,12 +3,17 @@ cimport cython +# Use a single global object for identity checks. +# PyPy can optimise away integer objects, for example, and may fail the 'is' test. +obj = object() + + @cython.test_fail_if_path_exists('//NotNode') def is_not(a, b): """ >>> is_not(1, 2) True - >>> x = 1 + >>> x = obj >>> is_not(x, x) False """ @@ -20,7 +25,7 @@ def not_is_not(a, b): """ >>> not_is_not(1, 2) False - >>> x = 1 + >>> x = obj >>> not_is_not(x, x) True """ @@ -32,7 +37,7 @@ def not_is(a, b): """ >>> not_is(1, 2) True - >>> x = 1 + >>> x = obj >>> not_is(x, x) False """ diff --git a/tests/run/iterdict.pyx b/tests/run/iterdict.pyx index 0e8eaba1c..e2d697e0f 100644 --- a/tests/run/iterdict.pyx +++ b/tests/run/iterdict.pyx @@ -555,3 +555,19 @@ def for_in_iteritems_of_expression(*args, **kwargs): for k, v in dict(*args, **kwargs).iteritems(): result.append((k, v)) return result + + +cdef class NotADict: + """ + >>> NotADict().listvalues() # doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + ... + TypeError: descriptor 'values' for 'mappingproxy' objects doesn't apply to a 'iterdict.NotADict' object + """ + cdef long v + def __cinit__(self): + self.v = 1 + itervalues = type(object.__dict__).values + + def listvalues(self): + return [v for v in self.itervalues()] diff --git a/tests/run/knuth_man_or_boy_test.pyx b/tests/run/knuth_man_or_boy_test.pyx index 068cec524..d2b5c8825 100644 --- a/tests/run/knuth_man_or_boy_test.pyx +++ b/tests/run/knuth_man_or_boy_test.pyx @@ -46,9 +46,13 @@ def compute(val): def a(in_k, x1, x2, x3, x4, x5): """ >>> import sys - >>> sys.setrecursionlimit(1350) + >>> old_limit = sys.getrecursionlimit() + >>> sys.setrecursionlimit(1350 if not getattr(sys, 'pypy_version_info', None) else 2700) + >>> a(10, 1, -1, -1, 1, 0) -67 + + >>> sys.setrecursionlimit(old_limit) """ k = [in_k] def b(): diff --git a/tests/run/kwargproblems.pyx b/tests/run/kwargproblems.pyx index 88e3ac53a..7984fbb08 100644 --- a/tests/run/kwargproblems.pyx +++ b/tests/run/kwargproblems.pyx @@ -4,7 +4,7 @@ def test(**kw): >>> d = {1 : 2} >>> test(**d) # doctest: +ELLIPSIS Traceback (most recent call last): - TypeError: ...keywords must be strings + TypeError: ...keywords must be strings... >>> d {1: 2} >>> d = {} diff --git a/tests/run/kwargs_passthrough.pyx b/tests/run/kwargs_passthrough.pyx index 576306efd..c09b6cba4 100644 --- a/tests/run/kwargs_passthrough.pyx +++ b/tests/run/kwargs_passthrough.pyx @@ -138,6 +138,18 @@ def wrap_modify_mix(f): >>> wrapped(a=2, test=3) CALLED (2, 1) + + >>> def py_modify(**kwargs): + ... print(sorted(kwargs.items())) + ... kwargs['new'] = len(kwargs) + ... return kwargs + + >>> wrapped_modify = wrap_modify_mix(py_modify) + >>> sorted(wrapped_modify(a=1).items()) + CALLED + [('a', 1)] + [('a', 1), ('test', 1)] + [('a', 1), ('new', 2), ('test', 1)] """ def wrapper(*args, **kwargs): print("CALLED") diff --git a/tests/run/lambda_T195.pyx b/tests/run/lambda_T195.pyx index fdbde9982..bfae08fea 100644 --- a/tests/run/lambda_T195.pyx +++ b/tests/run/lambda_T195.pyx @@ -1,6 +1,6 @@ # mode: run # tag: lambda -# ticket: 195 +# ticket: t195 __doc__ = u""" #>>> py_identity = lambda x:x diff --git a/tests/run/lambda_T723.pyx b/tests/run/lambda_T723.pyx index e746a3f58..39ae66851 100644 --- a/tests/run/lambda_T723.pyx +++ b/tests/run/lambda_T723.pyx @@ -1,5 +1,5 @@ # mode: run -# ticket: 723 +# ticket: t723 # tag: lambda def t723(a): diff --git a/tests/run/lambda_class_T605.pyx b/tests/run/lambda_class_T605.pyx index 82e1ff8a7..2efe77e7c 100644 --- a/tests/run/lambda_class_T605.pyx +++ b/tests/run/lambda_class_T605.pyx @@ -1,6 +1,6 @@ # mode: run # tag: lambda -# ticket: 605 +# ticket: t605 cdef int cdef_CONST = 123 CONST = 456 diff --git a/tests/run/lambda_module_T603.pyx b/tests/run/lambda_module_T603.pyx index 245f86937..c92fce5df 100644 --- a/tests/run/lambda_module_T603.pyx +++ b/tests/run/lambda_module_T603.pyx @@ -1,6 +1,6 @@ # mode: run # tag: lambda -# ticket: 603 +# ticket: t603 # Module scope lambda functions diff --git a/tests/run/large_consts_T237.pyx b/tests/run/large_consts_T237.pyx index f11a24541..7c448401f 100644 --- a/tests/run/large_consts_T237.pyx +++ b/tests/run/large_consts_T237.pyx @@ -1,4 +1,4 @@ -# ticket: 237 +# ticket: t237 #def add_large_c(): # cdef unsigned long long val = 2**30 + 2**30 # return val diff --git a/tests/run/letnode_T766.pyx b/tests/run/letnode_T766.pyx index 337808ee4..0f2d53e0b 100644 --- a/tests/run/letnode_T766.pyx +++ b/tests/run/letnode_T766.pyx @@ -1,5 +1,5 @@ # mode: run -# ticket: 766 +# ticket: t766 # tag: letnode def test_letnode_range(int n): diff --git a/tests/run/libc_math.pyx b/tests/run/libc_math.pyx index c3d768e82..9bb93678c 100644 --- a/tests/run/libc_math.pyx +++ b/tests/run/libc_math.pyx @@ -2,8 +2,8 @@ from libc.math cimport (M_E, M_LOG2E, M_LOG10E, M_LN2, M_LN10, M_PI, M_PI_2, M_PI_4, M_1_PI, M_2_PI, M_2_SQRTPI, M_SQRT2, M_SQRT1_2) -from libc.math cimport (acos, asin, atan, atan2, cos, sin, tan, cosh, sinh, - tanh, acosh, asinh, atanh, exp, log, log10, pow, sqrt) +from libc.math cimport (acos, asin, atan, atan2, cos, modf, sin, sinf, sinl, + tan, cosh, sinh, tanh, acosh, asinh, atanh, exp, log, log10, pow, sqrt) cimport libc.math as libc_math @@ -34,3 +34,21 @@ def test_sin(x): [True, True, True, True, True, True, True, True, True, True] """ return sin(x) + + +def test_sin_kwarg(x): + """ + >>> test_sin_kwarg(0) + 0.0 + """ + return sin(x=x) + + +def test_modf(x): + """ + >>> test_modf(2.5) + (0.5, 2.0) + """ + cdef double i + cdef double f = modf(x, &i) + return (f, i) diff --git a/tests/run/libcpp_algo.pyx b/tests/run/libcpp_algo.pyx index 40285cbd1..758da7705 100644 --- a/tests/run/libcpp_algo.pyx +++ b/tests/run/libcpp_algo.pyx @@ -1,12 +1,13 @@ +# mode: run # tag: cpp from libcpp cimport bool -from libcpp.algorithm cimport make_heap, sort_heap, sort, partial_sort +from libcpp.algorithm cimport make_heap, sort_heap from libcpp.vector cimport vector # XXX should use std::greater, but I don't know how to wrap that. -cdef inline bool greater(int x, int y): +cdef inline bool greater(const int &x, const int &y): return x > y @@ -27,33 +28,3 @@ def heapsort(l, bool reverse=False): sort_heap(v.begin(), v.end()) return v - - -def partialsort(l, int k, reverse=False): - """ - >>> partialsort([4, 2, 3, 1, 5], k=2)[:2] - [1, 2] - >>> partialsort([4, 2, 3, 1, 5], k=2, reverse=True)[:2] - [5, 4] - """ - cdef vector[int] v = l - if reverse: - partial_sort(v.begin(), v.begin() + k, v.end(), &greater) - else: - partial_sort(v.begin(), v.begin() + k, v.end()) - return v - - -def stdsort(l, reverse=False): - """ - >>> stdsort([3, 2, 1, 4, 5]) - [1, 2, 3, 4, 5] - >>> stdsort([3, 2, 1, 4, 5], reverse=True) - [5, 4, 3, 2, 1] - """ - cdef vector[int] v = l - if reverse: - sort(v.begin(), v.end(), &greater) - else: - sort(v.begin(), v.end()) - return v diff --git a/tests/run/libcpp_all.pyx b/tests/run/libcpp_all.pyx index 2930807d9..a45de63f6 100644 --- a/tests/run/libcpp_all.pyx +++ b/tests/run/libcpp_all.pyx @@ -4,6 +4,7 @@ import cython cimport libcpp +# cimport libcpp.atomic cimport libcpp.deque cimport libcpp.list cimport libcpp.map @@ -15,6 +16,7 @@ cimport libcpp.vector cimport libcpp.complex cimport libcpp.limits +# from libcpp.atomic cimport * from libcpp.deque cimport * from libcpp.list cimport * from libcpp.map cimport * @@ -26,6 +28,7 @@ from libcpp.vector cimport * from libcpp.complex cimport * from libcpp.limits cimport * +# cdef libcpp.atomic.atomc[int] a1 = atomic[int]() cdef libcpp.deque.deque[int] d1 = deque[int]() cdef libcpp.list.list[int] l1 = list[int]() cdef libcpp.map.map[int,int] m1 = map[int,int]() diff --git a/tests/run/list_comp_in_closure_T598.pyx b/tests/run/list_comp_in_closure_T598.pyx index 3f418add1..45b572ac0 100644 --- a/tests/run/list_comp_in_closure_T598.pyx +++ b/tests/run/list_comp_in_closure_T598.pyx @@ -1,6 +1,6 @@ # mode: run # tag: closures -# ticket: 598 +# ticket: t598 # cython: language_level=3 def list_comp_in_closure(): diff --git a/tests/run/list_pop.pyx b/tests/run/list_pop.pyx index b1379f199..3e7c5bdf5 100644 --- a/tests/run/list_pop.pyx +++ b/tests/run/list_pop.pyx @@ -206,7 +206,7 @@ def crazy_pop(L): """ >>> crazy_pop(list(range(10))) # doctest: +ELLIPSIS Traceback (most recent call last): - TypeError: pop... at most ... argument... + TypeError: pop... argument... >>> crazy_pop(A()) (1, 2, 3) """ diff --git a/tests/run/locals_T732.pyx b/tests/run/locals_T732.pyx index d134948af..6ce8c7447 100644 --- a/tests/run/locals_T732.pyx +++ b/tests/run/locals_T732.pyx @@ -1,5 +1,5 @@ # mode: run -# ticket: 731 +# ticket: t731 # tag: locals, vars, dir cimport cython @@ -23,8 +23,8 @@ def test_class_locals_and_dir(): >>> klass = test_class_locals_and_dir() >>> 'visible' in klass.locs and 'not_visible' not in klass.locs True - >>> klass.names - ['__module__', '__qualname__', 'visible'] + >>> [n for n in klass.names if n not in {"__qualname__", "__annotations__"}] + ['__module__', 'visible'] """ not_visible = 1234 class Foo: diff --git a/tests/run/locals_expressions_T430.pyx b/tests/run/locals_expressions_T430.pyx index a0f8a0d62..a0e9dff3e 100644 --- a/tests/run/locals_expressions_T430.pyx +++ b/tests/run/locals_expressions_T430.pyx @@ -1,4 +1,4 @@ -# ticket: 430 +# ticket: t430 __doc__ = u""" >>> sorted( get_locals(1,2,3, k=5) .items()) diff --git a/tests/run/locals_rebind_T429.pyx b/tests/run/locals_rebind_T429.pyx index 9b3f06a5a..e84a13947 100644 --- a/tests/run/locals_rebind_T429.pyx +++ b/tests/run/locals_rebind_T429.pyx @@ -1,4 +1,4 @@ -# ticket: 429 +# ticket: t429 __doc__ = u""" >>> sorted( get_locals(1,2,3, k=5) .items()) diff --git a/tests/run/lvalue_refs.pyx b/tests/run/lvalue_refs.pyx index d42f2407e..6bd0d88bc 100644 --- a/tests/run/lvalue_refs.pyx +++ b/tests/run/lvalue_refs.pyx @@ -28,3 +28,15 @@ def test_lvalue_ref_assignment(): assert bar[0] == &baz[0][0] assert bar[0][0] == bongle + +# not *strictly* lvalue refs but this file seems the closest applicable place for it. +# GH 3754 - std::vector operator[] returns a reference, and this causes problems if +# the reference is passed into Cython __Pyx_GetItemInt +def test_ref_used_for_indexing(): + """ + >>> test_ref_used_for_indexing() + 'looked up correctly' + """ + cdef vector[int] idx = [1,2,3] + d = {1: "looked up correctly", 2:"oops"} + return d[idx[0]] diff --git a/tests/run/metaclass.pyx b/tests/run/metaclass.pyx index d5464fb17..9e4db4c8b 100644 --- a/tests/run/metaclass.pyx +++ b/tests/run/metaclass.pyx @@ -69,8 +69,8 @@ class Py3ClassMCOnly(object, metaclass=Py3MetaclassPlusAttr): 321 >>> obj.metaclass_was_here True - >>> obj._order - ['__module__', '__qualname__', '__doc__', 'bar', 'metaclass_was_here'] + >>> [n for n in obj._order if n not in {"__qualname__", "__annotations__"}] + ['__module__', '__doc__', 'bar', 'metaclass_was_here'] """ bar = 321 @@ -81,8 +81,8 @@ class Py3InheritedMetaclass(Py3ClassMCOnly): 345 >>> obj.metaclass_was_here True - >>> obj._order - ['__module__', '__qualname__', '__doc__', 'bar', 'metaclass_was_here'] + >>> [n for n in obj._order if n not in {"__qualname__", "__annotations__"}] + ['__module__', '__doc__', 'bar', 'metaclass_was_here'] """ bar = 345 @@ -109,8 +109,8 @@ class Py3Foo(object, metaclass=Py3Base, foo=123): 123 >>> obj.bar 321 - >>> obj._order - ['__module__', '__qualname__', '__doc__', 'bar', 'foo'] + >>> [n for n in obj._order if n not in {"__qualname__", "__annotations__"}] + ['__module__', '__doc__', 'bar', 'foo'] """ bar = 321 @@ -122,8 +122,8 @@ class Py3FooInherited(Py3Foo, foo=567): 567 >>> obj.bar 321 - >>> obj._order - ['__module__', '__qualname__', '__doc__', 'bar', 'foo'] + >>> [n for n in obj._order if n not in {"__qualname__", "__annotations__"}] + ['__module__', '__doc__', 'bar', 'foo'] """ bar = 321 diff --git a/tests/run/method_module_name_T422.pyx b/tests/run/method_module_name_T422.pyx index fbefa2144..731cd1b30 100644 --- a/tests/run/method_module_name_T422.pyx +++ b/tests/run/method_module_name_T422.pyx @@ -1,4 +1,4 @@ -# ticket: 422 +# ticket: t422 """ >>> Foo.incr.__module__ is not None diff --git a/tests/run/methodmangling_T5.py b/tests/run/methodmangling_T5.py index 1cfa85310..9e2b7c63a 100644 --- a/tests/run/methodmangling_T5.py +++ b/tests/run/methodmangling_T5.py @@ -1,5 +1,10 @@ # mode: run -# ticket: 5 +# ticket: t5 + +# A small number of extra tests checking: +# 1) this works correctly with pure-Python-mode decorators - methodmangling_pure.py. +# 2) this works correctly with cdef classes - methodmangling_cdef.pyx +# 3) with "error_on_unknown_names" - methodmangling_unknown_names.py class CyTest(object): """ @@ -15,8 +20,23 @@ class CyTest(object): >>> '__x' in dir(cy) False + >>> cy._CyTest__y + 2 + + >>> '_CyTest___more_than_two' in dir(cy) + True + >>> '___more_than_two' in dir(cy) + False + >>> '___more_than_two_special___' in dir(cy) + True """ __x = 1 + ___more_than_two = 3 + ___more_than_two_special___ = 4 + + def __init__(self): + self.__y = 2 + def __private(self): return 8 def get(self): @@ -88,8 +108,285 @@ class _UnderscoreTest(object): 1 >>> ut.get() 1 + >>> ut._UnderscoreTest__UnderscoreNested().ret1() + 1 + >>> ut._UnderscoreTest__UnderscoreNested.__name__ + '__UnderscoreNested' + >>> ut._UnderscoreTest__prop + 1 """ __x = 1 def get(self): return self.__x + + class __UnderscoreNested(object): + def ret1(self): + return 1 + + @property + def __prop(self): + return self.__x + +class C: + error = """Traceback (most recent call last): +... +TypeError: +""" + __doc__ = """ +>>> instance = C() + +Instance methods have their arguments mangled +>>> instance.method1(__arg=1) # doctest: +IGNORE_EXCEPTION_DETAIL +{error} +>>> instance.method1(_C__arg=1) +1 +>>> instance.method2(__arg=1) # doctest: +IGNORE_EXCEPTION_DETAIL +{error} +>>> instance.method2(_C__arg=1) +1 + +Works when optional argument isn't passed +>>> instance.method2() +None + +Where args are in the function's **kwargs dict, names aren't mangled +>>> instance.method3(__arg=1) # doctest: +1 +>>> instance.method3(_C__arg=1) # doctest: +IGNORE_EXCEPTION_DETAIL +Traceback (most recent call last): +... +KeyError: + +Lambda functions behave in the same way: +>>> instance.method_lambda(__arg=1) # doctest: +IGNORE_EXCEPTION_DETAIL +{error} +>>> instance.method_lambda(_C__arg=1) +1 + +Class methods - have their arguments mangled +>>> instance.class_meth(__arg=1) # doctest: +IGNORE_EXCEPTION_DETAIL +{error} +>>> instance.class_meth(_C__arg=1) +1 +>>> C.class_meth(__arg=1) # doctest: +IGNORE_EXCEPTION_DETAIL +{error} +>>> C.class_meth(_C__arg=1) +1 + +Static methods - have their arguments mangled +>>> instance.static_meth(__arg=1) # doctest: +IGNORE_EXCEPTION_DETAIL +{error} +>>> instance.static_meth(_C__arg=1) +1 +>>> C.static_meth(__arg=1) # doctest: +IGNORE_EXCEPTION_DETAIL +{error} +>>> C.static_meth(_C__arg=1) +1 + +Functions assigned to the class don't have their arguments mangled +>>> instance.class_assigned_function(__arg=1) +1 +>>> instance.class_assigned_function(_C__arg=1) # doctest: +IGNORE_EXCEPTION_DETAIL +{error} + +Functions assigned to an instance don't have their arguments mangled +>>> instance.instance_assigned_function = free_function2 +>>> instance.instance_assigned_function(__arg=1) +1 +>>> instance.instance_assigned_function(_C__arg=1) # doctest: +IGNORE_EXCEPTION_DETAIL +{error} + +Locals are reported as mangled +>>> list(sorted(k for k in instance.get_locals(1).keys())) +['_C__arg', 'self'] +""".format(error=error) + + def method1(self, __arg): + print(__arg) + + def method2(self, __arg=None): + # __arg is optional + print(__arg) + + def method3(self, **kwargs): + print(kwargs['__arg']) + + method_lambda = lambda self, __arg: __arg + + def get_locals(self, __arg): + return locals() + + @classmethod + def class_meth(cls, __arg): + print(__arg) + + @staticmethod + def static_meth(__arg, dummy_arg=None): + # dummy_arg is to mask https://github.com/cython/cython/issues/3090 + print(__arg) + +def free_function1(x, __arg): + print(__arg) + +def free_function2(__arg, dummy_arg=None): + # dummy_arg is to mask https://github.com/cython/cython/issues/3090 + print(__arg) + +C.class_assigned_function = free_function1 + +__global_arg = True + +_D__arg1 = None +_D__global_arg = False # define these because otherwise Cython gives a compile-time error + # while Python gives a runtime error (which is difficult to test) +def can_find_global_arg(): + """ + >>> can_find_global_arg() + True + """ + return __global_arg + +def cant_find_global_arg(): + """ + Gets _D_global_arg instead + >>> cant_find_global_arg() + False + """ + class D: + def f(self): + return __global_arg + return D().f() + +class CMultiplyNested: + def f1(self, __arg, name=None, return_closure=False): + """ + >>> inst = CMultiplyNested() + >>> for name in [None, '__arg', '_CMultiplyNested__arg', '_D__arg']: + ... try: + ... print(inst.f1(1,name)) + ... except TypeError: + ... print("TypeError") # not concerned about exact details + ... # now test behaviour is the same in closures + ... closure = inst.f1(1, return_closure=True) + ... try: + ... if name is None: + ... print(closure(2)) + ... else: + ... print(closure(**{ name: 2})) + ... except TypeError: + ... print("TypeError") + 2 + 2 + TypeError + TypeError + TypeError + TypeError + 2 + 2 + """ + class D: + def g(self, __arg): + return __arg + if return_closure: + return D().g + if name is not None: + return D().g(**{ name: 2 }) + else: + return D().g(2) + + def f2(self, __arg1): + """ + This finds the global name '_D__arg1' + It's tested in this way because without the global + Python gives a runtime error and Cython a compile error + >>> print(CMultiplyNested().f2(1)) + None + """ + class D: + def g(self): + return __arg1 + return D().g() + + def f3(self, arg, name): + """ + >>> inst = CMultiplyNested() + >>> inst.f3(1, None) + 2 + >>> inst.f3(1, '__arg') # doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + ... + TypeError: + >>> inst.f3(1, '_CMultiplyNested__arg') + 2 + """ + def g(__arg, dummy=1): + return __arg + if name is not None: + return g(**{ name: 2}) + else: + return g(2) + + def f4(self, __arg): + """ + >>> CMultiplyNested().f4(1) + 1 + """ + def g(): + return __arg + return g() + + def f5(self, __arg): + """ + Default values are found in the outer scope correcly + >>> CMultiplyNested().f5(1) + 1 + """ + def g(x=__arg): + return x + return g() + + def f6(self, __arg1): + """ + This will find the global name _D__arg1 + >>> print(CMultiplyNested().f6(1)) + None + """ + class D: + def g(self, x=__arg1): + return x + return D().g() + + def f7(self, __arg): + """ + Lookup works in generator expressions + >>> list(CMultiplyNested().f7(1)) + [1] + """ + return (__arg for x in range(1)) + +class __NameWithDunder: + """ + >>> __NameWithDunder.__name__ + '__NameWithDunder' + """ + pass + +class Inherits(__NameWithDunder): + """ + Compile check that it can find the base class + >>> x = Inherits() + """ + pass + +def regular_function(__x, dummy=None): + # as before, dummy stops Cython creating a 1 arg, non-keyword call + return __x + +class CallsRegularFunction: + def call(self): + """ + >>> CallsRegularFunction().call() + 1 + """ + return regular_function(__x=1) # __x shouldn't be mangled as an argument elsewhere diff --git a/tests/run/methodmangling_cdef.pxd b/tests/run/methodmangling_cdef.pxd new file mode 100644 index 000000000..58a9130a4 --- /dev/null +++ b/tests/run/methodmangling_cdef.pxd @@ -0,0 +1,3 @@ +cdef class InPxd: + cdef public int __y + cdef int __private_cdef(self) diff --git a/tests/run/methodmangling_cdef.pyx b/tests/run/methodmangling_cdef.pyx new file mode 100644 index 000000000..83f02fa6a --- /dev/null +++ b/tests/run/methodmangling_cdef.pyx @@ -0,0 +1,95 @@ +# mode: run + +def call_cdt_private_cdef(CDefTest o): + return o._CDefTest__private_cdef() + +cdef __c_func(): + return "cdef function" + +cdef __c_var = "Shouldn't see this" + +cdef class CDefTest: + """ + >>> cd = CDefTest() + >>> '_CDefTest__private' in dir(cd) + True + >>> cd._CDefTest__private() + 8 + >>> call_cdt_private_cdef(cd) + 8 + >>> '__private' in dir(cd) + False + >>> '_CDefTest__x' in dir(cd) + True + + >>> '__x' in dir(cd) + False + >>> cd._CDefTest__y + 2 + """ + __x = 1 + cdef public int __y + + def __init__(self): + self.__y = 2 + + def __private(self): return 8 + + cdef __private_cdef(self): return 8 + + def get(self): + """ + >>> CDefTest().get() + (1, 1, 8) + """ + return self._CDefTest__x, self.__x, self.__private() + + def get_inner(self): + """ + >>> CDefTest().get_inner() + (1, 1, 8) + """ + def get(o): + return o._CDefTest__x, o.__x, o.__private() + return get(self) + + def get_c_func(self): + """ + Should still be able to access C function with __names + >>> CDefTest().get_c_func() + 'cdef function' + """ + return __c_func() + + def get_c_func2(self): + """ + Should find mangled name before C __name + >>> CDefTest().get_c_func2() + 'lambda' + """ + _CDefTest__c_func = lambda: "lambda" + return __c_func() + + def get_c_var(self): + """ + >>> CDefTest().get_c_var() + 'c var' + """ + global __c_var + __c_var = "c var" + return __c_var + +def call_inpdx_private_cdef(InPxd o): + return o._InPxd__private_cdef() + +cdef class InPxd: + """ + >>> InPxd()._InPxd__y + 2 + >>> call_inpdx_private_cdef(InPxd()) + 8 + """ + def __init__(self): + self.__y = 2 + + cdef int __private_cdef(self): return 8 diff --git a/tests/run/methodmangling_pure.py b/tests/run/methodmangling_pure.py new file mode 100644 index 000000000..10b811e2f --- /dev/null +++ b/tests/run/methodmangling_pure.py @@ -0,0 +1,76 @@ +# mode: run +# cython: language_level=3 + +# This file tests that methodmangling is applied correctly to +# pure Python decorated classes. + +import cython + +if cython.compiled: + # don't run in Python mode since a significant number of the tests + # are only for Cython features + + def declare(**kwargs): + return kwargs['__x'] + + class RegularClass: + @cython.locals(__x=cython.int) + def f1(self, __x, dummy=None): + """ + Is the locals decorator correctly applied + >>> c = RegularClass() + >>> c.f1(1) + 1 + >>> c.f1("a") + Traceback (most recent call last): + ... + TypeError: an integer is required + >>> c.f1(_RegularClass__x = 1) + 1 + """ + return __x + + def f2(self, x): + """ + Is the locals decorator correctly applied + >>> c = RegularClass() + >>> c.f2(1) + 1 + >>> c.f2("a") + Traceback (most recent call last): + ... + TypeError: an integer is required + """ + __x = cython.declare(cython.int, x) + + return __x + + def f3(self, x): + """ + Is the locals decorator correctly applied + >>> c = RegularClass() + >>> c.f3(1) + 1 + >>> c.f3("a") + Traceback (most recent call last): + ... + TypeError: an integer is required + """ + cython.declare(__x=cython.int) + __x = x + + return __x + + def f4(self, x): + """ + We shouldn't be tripped up by a function called + "declare" that is nothing to do with cython + >>> RegularClass().f4(1) + 1 + """ + return declare(__x=x) +else: + __doc__ = """ + >>> True + True + """ # stops Python2 from failing diff --git a/tests/run/methodmangling_unknown_names.py b/tests/run/methodmangling_unknown_names.py new file mode 100644 index 000000000..dae1f5c22 --- /dev/null +++ b/tests/run/methodmangling_unknown_names.py @@ -0,0 +1,25 @@ +# mode: run +# tag: allow_unknown_names, pure2.0, pure3.0 + +class Test(object): + def run(self): + """ + >>> Test().run() + NameError1 + NameError2 + found mangled + """ + try: + print(__something) + except NameError: + print("NameError1") # correct - shouldn't exist + globals()['__something'] = 'found unmangled' + try: + print(__something) + except NameError: + print("NameError2") # correct - shouldn't exist + globals()['_Test__something'] = 'found mangled' + try: + print(__something) # should print this + except NameError: + print("NameError3") diff --git a/tests/run/modop.pyx b/tests/run/modop.pyx index c7b4c7d57..82e43d095 100644 --- a/tests/run/modop.pyx +++ b/tests/run/modop.pyx @@ -9,7 +9,7 @@ def modobj(obj2, obj3): '5' >>> modobj(1, 0) # doctest: +ELLIPSIS Traceback (most recent call last): - ZeroDivisionError: integer division... + ZeroDivisionError: integer... modulo by zero """ obj1 = obj2 % obj3 return obj1 @@ -19,11 +19,22 @@ def mod_10_obj(int2): """ >>> mod_10_obj(0) # doctest: +ELLIPSIS Traceback (most recent call last): - ZeroDivisionError: integer division... + ZeroDivisionError: ... modulo by zero + >>> 10 % 1 + 0 + >>> mod_10_obj(1) + 0 >>> mod_10_obj(3) 1 + >>> 10 % -1 + 0 + >>> mod_10_obj(-1) + 0 + >>> mod_10_obj(-10) + 0 """ - return 10 % int2 + int1 = 10 % int2 + return int1 def mod_obj_10(int2): @@ -168,6 +179,53 @@ def mod_obj_17(int2): return int1 +def mod_int_17(int int2): + """ + >>> 0 % 17 + 0 + >>> mod_int_17(0) + 0 + >>> 1 % 17 + 1 + >>> mod_int_17(1) + 1 + >>> (-1) % 17 + 16 + >>> mod_int_17(-1) + 16 + >>> 9 % 17 + 9 + >>> mod_int_17(16) + 16 + >>> 17 % 17 + 0 + >>> mod_int_17(17) + 0 + >>> (-17) % 17 + 0 + >>> mod_int_17(-17) + 0 + >>> (-18) % 17 + 16 + >>> mod_int_17(-18) + 16 + >>> 10002 % 17 + 6 + >>> mod_int_17(10002) + 6 + >>> int((2**25) % 17) + 2 + >>> int(mod_int_17(2**25)) + 2 + >>> int((-2**25) % 17) + 15 + >>> int(mod_int_17(-2**25)) + 15 + """ + int1 = int2 % 17 + return int1 + + def mod_obj_m2(int2): """ >>> 0 % -2 diff --git a/tests/run/mulop.pyx b/tests/run/mulop.pyx new file mode 100644 index 000000000..0fba7dba9 --- /dev/null +++ b/tests/run/mulop.pyx @@ -0,0 +1,166 @@ +# mode: run +# tag: multiply + +import sys +IS_PY2 = sys.version_info[0] < 3 + + +def print_long(x): + if IS_PY2: + x = str(x).rstrip('L') + print(x) + + +def mul_10_obj(x): + """ + >>> mul_10_obj(0) + 0 + >>> mul_10_obj(10) + 100 + >>> mul_10_obj(-10) + -100 + >>> 10 * (2**14) + 163840 + >>> mul_10_obj(2**14) + 163840 + >>> mul_10_obj(-2**14) + -163840 + >>> print_long(10 * (2**29)) + 5368709120 + >>> print_long(mul_10_obj(2**29)) + 5368709120 + >>> print_long(mul_10_obj(-2**29)) + -5368709120 + >>> print_long(10 * (2**30)) + 10737418240 + >>> print_long(mul_10_obj(2**30)) + 10737418240 + >>> print_long(mul_10_obj(-2**30)) + -10737418240 + >>> print_long(10 * (2**63)) + 92233720368547758080 + >>> print_long(mul_10_obj(2**63)) + 92233720368547758080 + >>> print_long(mul_10_obj(-2**63)) + -92233720368547758080 + >>> print_long(10 * (2**128)) + 3402823669209384634633746074317682114560 + >>> print_long(mul_10_obj(2**128)) + 3402823669209384634633746074317682114560 + >>> print_long(mul_10_obj(-2**128)) + -3402823669209384634633746074317682114560 + """ + result = 10 * x + return result + + +def mul_obj_10(x): + """ + >>> mul_obj_10(0) + 0 + >>> mul_obj_10(10) + 100 + >>> mul_obj_10(-10) + -100 + >>> 10 * (2**14) + 163840 + >>> mul_obj_10(2**14) + 163840 + >>> mul_obj_10(-2**14) + -163840 + >>> print_long(10 * (2**29)) + 5368709120 + >>> print_long(mul_obj_10(2**29)) + 5368709120 + >>> print_long(mul_obj_10(-2**29)) + -5368709120 + >>> print_long(10 * (2**30)) + 10737418240 + >>> print_long(mul_obj_10(2**30)) + 10737418240 + >>> print_long(mul_obj_10(-2**30)) + -10737418240 + >>> print_long(10 * (2**63)) + 92233720368547758080 + >>> print_long(mul_obj_10(2**63)) + 92233720368547758080 + >>> print_long(mul_obj_10(-2**63)) + -92233720368547758080 + >>> print_long(10 * (2**128)) + 3402823669209384634633746074317682114560 + >>> print_long(mul_obj_10(2**128)) + 3402823669209384634633746074317682114560 + >>> print_long(mul_obj_10(-2**128)) + -3402823669209384634633746074317682114560 + """ + result = x * 10 + return result + + +def mul_bigint_obj(x): + """ + >>> mul_bigint_obj(0) + 0 + >>> print_long(mul_bigint_obj(1)) + 536870912 + >>> print_long(mul_bigint_obj(2)) + 1073741824 + >>> print_long(mul_bigint_obj(2**29)) + 288230376151711744 + >>> print_long(mul_bigint_obj(-2**29)) + -288230376151711744 + >>> print_long(mul_bigint_obj(2**30)) + 576460752303423488 + >>> print_long(mul_bigint_obj(-2**30)) + -576460752303423488 + >>> print_long(mul_bigint_obj(2**59)) + 309485009821345068724781056 + >>> print_long(mul_bigint_obj(-2**59)) + -309485009821345068724781056 + """ + result = (2**29) * x + return result + + +def mul_obj_float(x): + """ + >>> mul_obj_float(-0.0) + -0.0 + >>> mul_obj_float(0) + 0.0 + >>> mul_obj_float(1.0) + 2.0 + >>> mul_obj_float(-2.0) + -4.0 + >>> mul_obj_float(-0.5) + -1.0 + """ + result = x * 2.0 + return result + + +def mul_float_obj(x): + """ + >>> mul_float_obj(0) + 0.0 + >>> mul_float_obj(2) + 4.0 + >>> mul_float_obj(-2) + -4.0 + >>> 2.0 * (2**30-1) + 2147483646.0 + >>> mul_float_obj(2**30-1) + 2147483646.0 + >>> mul_float_obj(-(2**30-1)) + -2147483646.0 + >>> mul_float_obj(-0.0) + -0.0 + >>> mul_float_obj(1.0) + 2.0 + >>> mul_float_obj(-2.0) + -4.0 + >>> mul_float_obj(-0.5) + -1.0 + """ + result = 2.0 * x + return result diff --git a/tests/run/no_gc_clear.pyx b/tests/run/no_gc_clear.pyx index 643ec4c6e..9afca8af8 100644 --- a/tests/run/no_gc_clear.pyx +++ b/tests/run/no_gc_clear.pyx @@ -3,7 +3,7 @@ Check that the @cython.no_gc_clear decorator disables generation of the tp_clear slot so that __dealloc__ will still see the original reference contents. -Discussed here: http://article.gmane.org/gmane.comp.python.cython.devel/14986 +Discussed here: https://article.gmane.org/gmane.comp.python.cython.devel/14986 """ cimport cython diff --git a/tests/run/nogil.pyx b/tests/run/nogil.pyx index c6f6d8b53..6649ee071 100644 --- a/tests/run/nogil.pyx +++ b/tests/run/nogil.pyx @@ -28,10 +28,10 @@ cdef int g(int x) nogil: y = x + 42 return y -cdef int with_gil_func() except 0 with gil: +cdef int with_gil_func() except -1 with gil: raise Exception("error!") -cdef int nogil_func() nogil except 0: +cdef int nogil_func() nogil except -1: with_gil_func() def test_nogil_exception_propagation(): diff --git a/tests/run/nogil_conditional.pyx b/tests/run/nogil_conditional.pyx new file mode 100644 index 000000000..eba22d5b2 --- /dev/null +++ b/tests/run/nogil_conditional.pyx @@ -0,0 +1,271 @@ +# mode: run + +try: + from StringIO import StringIO +except ImportError: + from io import StringIO + + +def test(int x): + """ + >>> test(0) + 110 + """ + with nogil(True): + x = f_nogil(x) + with gil(True): + x = f_gil(x) + return x + + +cdef int f_nogil(int x) nogil: + cdef int y + y = x + 10 + return y + + +def f_gil(x): + y = 0 + y = x + 100 + return y + + +cdef int with_gil_func() except? -1 with gil: + raise Exception("error!") + + +cdef int nogil_func() nogil except? -1: + with_gil_func() + + +def test_nogil_exception_propagation(): + """ + >>> test_nogil_exception_propagation() + Traceback (most recent call last): + ... + Exception: error! + """ + with nogil: + with gil: + with nogil(True): + nogil_func() + + +cdef int write_unraisable() nogil: + with gil: + raise ValueError() + + +def test_unraisable(): + """ + >>> print(test_unraisable()) # doctest: +ELLIPSIS + ValueError + Exception...ignored... + """ + import sys + old_stderr = sys.stderr + stderr = sys.stderr = StringIO() + try: + write_unraisable() + finally: + sys.stderr = old_stderr + return stderr.getvalue().strip() + + +def test_nested(): + """ + >>> test_nested() + 240 + """ + cdef int res = 0 + + with nogil(True): + res = f_nogil(res) + with gil(1 < 2): + res = f_gil(res) + with nogil: + res = f_nogil(res) + + with gil: + res = f_gil(res) + with nogil(True): + res = f_nogil(res) + with nogil: + res = f_nogil(res) + + return res + + +DEF FREE_GIL = True +DEF FREE_GIL_FALSE = False + + +def test_nested_condition_false(): + """ + >>> test_nested_condition_false() + 220 + """ + cdef int res = 0 + + with gil(FREE_GIL_FALSE): + res = f_gil(res) + with nogil(False): + res = f_gil(res) + + with nogil(FREE_GIL): + res = f_nogil(res) + with gil(False): + res = f_nogil(res) + + return res + +def test_try_finally(): + """ + >>> test_try_finally() + 113 + """ + cdef int res = 0 + + try: + with nogil(True): + try: + res = f_nogil(res) + with gil(1 < 2): + try: + res = f_gil(res) + finally: + res += 1 + finally: + res = res + 1 + finally: + res += 1 + + return res + + +ctypedef fused number_or_object: + int + float + object + + +def test_fused(number_or_object x) -> number_or_object: + """ + >>> test_fused[int](1) + 2 + >>> test_fused[float](1.0) + 2.0 + >>> test_fused[object](1) + 2 + >>> test_fused[object](1.0) + 2.0 + """ + cdef number_or_object res = x + + with nogil(number_or_object is not object): + res = res + 1 + + return res + + +ctypedef fused int_or_object: + int + object + + +def test_fused_object(int_or_object x): + """ + >>> test_fused_object[object]("spam") + 456 + >>> test_fused_object[int](1000) + 1000 + """ + cdef int res = 0 + + if int_or_object is object: + with nogil(False): + res += len(x) + + try: + with nogil(int_or_object is object): + try: + with gil(int_or_object is object): + res = f_gil(res) + with gil: + res = f_gil(res) + with gil(False): + res = f_nogil(res) + + with gil(int_or_object is not object): + res = f_nogil(res) + with nogil(False): + res = f_nogil(res) + + res = f_nogil(res) + finally: + res = res + 1 + + with nogil(int_or_object is not object): + res = f_gil(res) + + with gil(int_or_object is not object): + res = f_gil(res) + + with nogil(int_or_object is object): + res = f_nogil(res) + + finally: + res += 1 + else: + res = x + + return res + + +def test_fused_int(int_or_object x): + """ + >>> test_fused_int[object]("spam") + 4 + >>> test_fused_int[int](1000) + 1452 + """ + cdef int res = 0 + + if int_or_object is int: + res += x + + try: + with nogil(int_or_object is int): + try: + with gil(int_or_object is int): + res = f_gil(res) + with gil: + res = f_gil(res) + with gil(False): + res = f_nogil(res) + + with gil(int_or_object is not int): + res = f_nogil(res) + with nogil(False): + res = f_nogil(res) + + res = f_nogil(res) + finally: + res = res + 1 + + with nogil(int_or_object is not int): + res = f_gil(res) + + with gil(int_or_object is not int): + res = f_gil(res) + + with nogil(int_or_object is int): + res = f_nogil(res) + + finally: + res += 1 + else: + with nogil(False): + res = len(x) + + return res diff --git a/tests/run/non_dict_kwargs_T470.pyx b/tests/run/non_dict_kwargs_T470.pyx index b60d56210..be6f8c67b 100644 --- a/tests/run/non_dict_kwargs_T470.pyx +++ b/tests/run/non_dict_kwargs_T470.pyx @@ -1,5 +1,5 @@ # mode: run -# ticket: 470 +# ticket: t470 def func(**kwargs): diff --git a/tests/run/numpy_ValueError_T172.pyx b/tests/run/numpy_ValueError_T172.pyx index 66558eefc..233048ce7 100644 --- a/tests/run/numpy_ValueError_T172.pyx +++ b/tests/run/numpy_ValueError_T172.pyx @@ -1,4 +1,4 @@ -# ticket: 172 +# ticket: t172 # tag: numpy __doc__ = u""" diff --git a/tests/run/numpy_attributes.pyx b/tests/run/numpy_attributes.pyx new file mode 100644 index 000000000..c77346a49 --- /dev/null +++ b/tests/run/numpy_attributes.pyx @@ -0,0 +1,87 @@ +# mode: run +# tag: numpy + +cimport cython + +import numpy as np +cimport numpy as cnp + +cnp.import_array() + + +@cython.test_assert_path_exists( + "//ReturnStatNode", + "//ReturnStatNode//IndexNode", + "//ReturnStatNode//IndexNode//SimpleCallNode", +) +@cython.test_fail_if_path_exists( + "//ReturnStatNode//AttributeNode", +) +def access_shape(): + """ + >>> print(access_shape()) + 10 + """ + cdef cnp.ndarray[double, ndim=2, mode='c'] array_in = \ + 1e10 * np.ones((10, 10)) + + return array_in.shape[0] + + +@cython.test_assert_path_exists( + "//ReturnStatNode", + "//ReturnStatNode//SimpleCallNode", +) +@cython.test_fail_if_path_exists( + "//ReturnStatNode//AttributeNode", +) +def access_size(): + """ + >>> print(access_size()) + 100 + """ + cdef cnp.ndarray[double, ndim=2, mode='c'] array_in = \ + 1e10 * np.ones((10, 10)) + + return array_in.size + + +@cython.test_assert_path_exists( + "//ReturnStatNode", + "//ReturnStatNode//IndexNode", + "//ReturnStatNode//IndexNode//SimpleCallNode", +) +@cython.test_fail_if_path_exists( + "//ReturnStatNode//AttributeNode", +) +def access_strides(): + """ + >>> x, y = access_strides() + >>> print(x) + 80 + >>> print(y) + 8 + """ + cdef cnp.ndarray[double, ndim=2, mode='c'] array_in = \ + 1e10 * np.ones((10, 10), dtype=np.float64) + + return (array_in.strides[0], array_in.strides[1]) + + +@cython.test_assert_path_exists( + "//ReturnStatNode", + "//ReturnStatNode//PrimaryCmpNode", + "//ReturnStatNode//PrimaryCmpNode//SimpleCallNode", +) +@cython.test_fail_if_path_exists( + "//ReturnStatNode//AttributeNode", +) +def access_data(): + """ + >>> access_data() + True + """ + cdef cnp.ndarray[double, ndim=2, mode='c'] array_in = \ + 1e10 * np.ones((10, 10), dtype=np.float64) + + return array_in.data is not NULL diff --git a/tests/run/numpy_bufacc_T155.pyx b/tests/run/numpy_bufacc_T155.pyx index 927eddb29..2c637ebc4 100644 --- a/tests/run/numpy_bufacc_T155.pyx +++ b/tests/run/numpy_bufacc_T155.pyx @@ -1,5 +1,5 @@ -# ticket: 155 -# tag: numpy_old +# ticket: t155 +# tag: numpy """ >>> myfunc() @@ -17,4 +17,3 @@ def myfunc(): A[i, :] /= 2 return A[0,0] -include "numpy_common.pxi" diff --git a/tests/run/numpy_cimport.pyx b/tests/run/numpy_cimport.pyx index 7c23d7725..8aaea8597 100644 --- a/tests/run/numpy_cimport.pyx +++ b/tests/run/numpy_cimport.pyx @@ -6,4 +6,3 @@ True """ cimport numpy as np -include "numpy_common.pxi" diff --git a/tests/run/numpy_cimport_1.pyx b/tests/run/numpy_cimport_1.pyx new file mode 100644 index 000000000..05754f591 --- /dev/null +++ b/tests/run/numpy_cimport_1.pyx @@ -0,0 +1,25 @@ +# mode: run +# tag: warnings, numpy + +cimport numpy as np +# np.import_array not called - should generate warning + +cdef extern from *: + """ + static void** _check_array_api(void) { + return PyArray_API; /* should be non NULL */ + } + """ + void** _check_array_api() + +def check_array_api(): + """ + >>> check_array_api() + True + """ + return _check_array_api() != NULL + + +_WARNINGS = """ +4:8: 'numpy.import_array()' has been added automatically since 'numpy' was cimported but 'numpy.import_array' was not called. +""" diff --git a/tests/run/numpy_cimport_2.pyx b/tests/run/numpy_cimport_2.pyx new file mode 100644 index 000000000..cdf8783c9 --- /dev/null +++ b/tests/run/numpy_cimport_2.pyx @@ -0,0 +1,25 @@ +# mode: run +# tag: warnings, numpy + +cimport numpy as np +np.import_array() +# np.import_array is called - no warning necessary + +cdef extern from *: + """ + static void** _check_array_api(void) { + return PyArray_API; /* should be non NULL */ + } + """ + void** _check_array_api() + +def check_array_api(): + """ + >>> check_array_api() + True + """ + return _check_array_api() != NULL + + +_WARNINGS = """ +""" diff --git a/tests/run/numpy_cimport_3.pyx b/tests/run/numpy_cimport_3.pyx new file mode 100644 index 000000000..b5b24661d --- /dev/null +++ b/tests/run/numpy_cimport_3.pyx @@ -0,0 +1,8 @@ +# mode: compile +# tag: warnings, numpy + +import numpy as np +# Numpy is only imported - no warning necessary + +_WARNINGS = """ +""" diff --git a/tests/run/numpy_cimport_4.pyx b/tests/run/numpy_cimport_4.pyx new file mode 100644 index 000000000..44e112054 --- /dev/null +++ b/tests/run/numpy_cimport_4.pyx @@ -0,0 +1,24 @@ +# mode: run +# tag: warnings, numpy + +cimport numpy +<void>numpy.import_array # dummy call should stop Cython auto-generating call to import_array + +cdef extern from *: + """ + static void** _check_array_api(void) { + return PyArray_API; /* should be non NULL if initialized */ + } + """ + void** _check_array_api() + +def check_array_api(): + """ + >>> check_array_api() + True + """ + return _check_array_api() == NULL # not initialized + + +_WARNINGS = """ +""" diff --git a/tests/run/numpy_cimport_5.pyx b/tests/run/numpy_cimport_5.pyx new file mode 100644 index 000000000..2768f2dae --- /dev/null +++ b/tests/run/numpy_cimport_5.pyx @@ -0,0 +1,25 @@ +# mode: run +# tag: warnings, numpy + +from numpy cimport ndarray +# np.import_array not called - should generate warning + +cdef extern from *: + """ + static void** _check_array_api(void) { + return PyArray_API; /* should be non NULL */ + } + """ + void** _check_array_api() + +def check_array_api(): + """ + >>> check_array_api() + True + """ + return _check_array_api() != NULL + + +_WARNINGS = """ +4:0: 'numpy.import_array()' has been added automatically since 'numpy' was cimported but 'numpy.import_array' was not called. +""" diff --git a/tests/run/numpy_cimport_6.pyx b/tests/run/numpy_cimport_6.pyx new file mode 100644 index 000000000..5cda5246d --- /dev/null +++ b/tests/run/numpy_cimport_6.pyx @@ -0,0 +1,25 @@ +# mode: run +# tag: warnings, numpy + +from numpy cimport ndarray, import_array +import_array() +# np.import_array is called - no warning necessary + +cdef extern from *: + """ + static void** _check_array_api(void) { + return PyArray_API; /* should be non NULL */ + } + """ + void** _check_array_api() + +def check_array_api(): + """ + >>> check_array_api() + True + """ + return _check_array_api() != NULL + + +_WARNINGS = """ +""" diff --git a/tests/run/numpy_common.pxi b/tests/run/numpy_common.pxi deleted file mode 100644 index 615bf701a..000000000 --- a/tests/run/numpy_common.pxi +++ /dev/null @@ -1,10 +0,0 @@ -# hack to avoid C compiler warnings about unused functions in the NumPy header files - -from numpy cimport import_array # , import_umath - -cdef extern from *: - bint FALSE "0" - -if FALSE: - import_array() -# import_umath() diff --git a/tests/run/numpy_parallel.pyx b/tests/run/numpy_parallel.pyx index 7dcf2b27a..96a60be14 100644 --- a/tests/run/numpy_parallel.pyx +++ b/tests/run/numpy_parallel.pyx @@ -1,10 +1,9 @@ -# tag: numpy_old +# tag: numpy # tag: openmp cimport cython from cython.parallel import prange cimport numpy as np -include "numpy_common.pxi" @cython.boundscheck(False) @@ -22,7 +21,7 @@ def test_parallel_numpy_arrays(): 3 4 """ - cdef Py_ssize_t i + cdef Py_ssize_t i, length cdef np.ndarray[np.int_t] x try: @@ -33,10 +32,10 @@ def test_parallel_numpy_arrays(): return x = numpy.zeros(10, dtype=numpy.int) + length = x.shape[0] - for i in prange(x.shape[0], nogil=True): + for i in prange(length, nogil=True): x[i] = i - 5 for i in x: - print i - + print(i) diff --git a/tests/run/numpy_pythran.pyx b/tests/run/numpy_pythran.pyx index 363d9ea85..8bc5fa5f2 100644 --- a/tests/run/numpy_pythran.pyx +++ b/tests/run/numpy_pythran.pyx @@ -55,3 +55,13 @@ def calculate_tax(cnp.ndarray[double, ndim=1] d): np.sum(seg3 * prog_seg3 + 939.57) + np.sum(seg4 * prog_seg4) ) / np.sum(d) + +def access_shape(): + """ + >>> access_shape() + 10 + """ + cdef cnp.ndarray[double, ndim=2, mode='c'] array_in = \ + 1e10 * np.ones((10, 10)) + + return array_in.shape[0] diff --git a/tests/run/numpy_subarray.pyx b/tests/run/numpy_subarray.pyx index c5d82d4b1..d032a3a01 100644 --- a/tests/run/numpy_subarray.pyx +++ b/tests/run/numpy_subarray.pyx @@ -1,4 +1,4 @@ -# tag: numpy_old +# tag: numpy cimport numpy as np cimport cython diff --git a/tests/run/numpy_test.pyx b/tests/run/numpy_test.pyx index 662dd4ad0..d2718a463 100644 --- a/tests/run/numpy_test.pyx +++ b/tests/run/numpy_test.pyx @@ -1,30 +1,19 @@ -# tag: numpy_old -# cannot be named "numpy" in order to not clash with the numpy module! +# tag: numpy cimport numpy as np cimport cython import re -import sys - -# initialise NumPy C-API -np.import_array() def little_endian(): cdef int endian_detector = 1 return (<char*>&endian_detector)[0] != 0 -__test__ = {} def testcase(f): - __test__[f.__name__] = f.__doc__ - return f - -def testcase_have_buffer_interface(f): - major, minor, *rest = np.__version__.split('.') - if (int(major), int(minor)) >= (1, 5) and sys.version_info[:2] >= (2, 6): - __test__[f.__name__] = f.__doc__ + # testcase decorator now does nothing (following changes to doctest) + # but is a useful indicator of what functions are designed as tests return f if little_endian(): @@ -272,8 +261,6 @@ try: except: __doc__ = u"" -__test__[__name__] = __doc__ - def assert_dtype_sizes(): assert sizeof(np.int8_t) == 1 @@ -684,7 +671,6 @@ def get_Foo_array(): data[5].b = 9.0 return np.asarray(<Foo[:]>data).copy() -@testcase_have_buffer_interface def test_fused_ndarray(fused_ndarray a): """ >>> import cython @@ -733,9 +719,6 @@ cpdef test_fused_cpdef_ndarray(fused_ndarray a): else: print b[5] -testcase_have_buffer_interface(test_fused_cpdef_ndarray) - -@testcase_have_buffer_interface def test_fused_cpdef_ndarray_cdef_call(): """ >>> test_fused_cpdef_ndarray_cdef_call() diff --git a/tests/run/overflow_check.pxi b/tests/run/overflow_check.pxi index 9bfd068a2..0b9844c89 100644 --- a/tests/run/overflow_check.pxi +++ b/tests/run/overflow_check.pxi @@ -1,14 +1,15 @@ cimport cython cdef object two = 2 -cdef int size_in_bits = sizeof(INT) * 8 +cdef int size_in_bits_ = sizeof(INT) * 8 cdef bint is_signed_ = not ((<INT>-1) > 0) -cdef INT max_value_ = <INT>(two ** (size_in_bits - is_signed_) - 1) +cdef INT max_value_ = <INT>(two ** (size_in_bits_ - is_signed_) - 1) cdef INT min_value_ = ~max_value_ cdef INT half_ = max_value_ // <INT>2 # Python visible. +size_in_bits = size_in_bits_ is_signed = is_signed_ max_value = max_value_ min_value = min_value_ @@ -230,6 +231,17 @@ def test_lshift(INT a, int b): """ >>> test_lshift(1, 10) 1024 + >>> test_lshift(1, size_in_bits - 2) == 1 << (size_in_bits - 2) + True + >>> test_lshift(0, size_in_bits - 1) + 0 + >>> test_lshift(1, size_in_bits - 1) == 1 << (size_in_bits - 1) if not is_signed else True + True + >>> if is_signed: expect_overflow(test_lshift, 1, size_in_bits - 1) + >>> expect_overflow(test_lshift, 0, size_in_bits) + >>> expect_overflow(test_lshift, 1, size_in_bits) + >>> expect_overflow(test_lshift, 0, size_in_bits + 1) + >>> expect_overflow(test_lshift, 1, size_in_bits + 1) >>> expect_overflow(test_lshift, 1, 100) >>> expect_overflow(test_lshift, max_value, 1) >>> test_lshift(max_value, 0) == max_value diff --git a/tests/run/packedstruct_T290.pyx b/tests/run/packedstruct_T290.pyx index 203008ca7..d9381567e 100644 --- a/tests/run/packedstruct_T290.pyx +++ b/tests/run/packedstruct_T290.pyx @@ -1,4 +1,4 @@ -# ticket: 290 +# ticket: t290 """ >>> f() diff --git a/tests/run/parallel_swap_assign_T425.pyx b/tests/run/parallel_swap_assign_T425.pyx index fb7d1ac29..b6e7e233d 100644 --- a/tests/run/parallel_swap_assign_T425.pyx +++ b/tests/run/parallel_swap_assign_T425.pyx @@ -1,4 +1,4 @@ -# ticket: 425 +# ticket: t425 cimport cython diff --git a/tests/run/pep448_extended_unpacking.pyx b/tests/run/pep448_extended_unpacking.pyx index 61414f423..cb6501f07 100644 --- a/tests/run/pep448_extended_unpacking.pyx +++ b/tests/run/pep448_extended_unpacking.pyx @@ -464,6 +464,10 @@ def unpack_dict_simple(it): return {**it} +@cython.test_assert_path_exists('//MergedDictNode') +@cython.test_fail_if_path_exists( + '//MergedDictNode//MergedDictNode', +) def unpack_dict_from_iterable(it): """ >>> d = unpack_dict_from_iterable(dict(a=1, b=2, c=3)) @@ -536,3 +540,28 @@ def unpack_dict_keep_originals(a, b, c): True """ return {**a, **b, 2: 4, **c} + + +@cython.test_assert_path_exists( + '//MergedDictNode', + '//MergedDictNode//MergedDictNode', + '//MergedDictNode//MergedDictNode//DictNode', +) +def unpack_in_call(f): + """ + >>> def f(a=1, test=2, **kwargs): + ... return a, test, sorted(kwargs.items()) + >>> wrapped = unpack_in_call(f) + >>> wrapped(1) + (1, 1, [('more', 2)]) + >>> wrapped(test='overwritten') + (1, 1, [('more', 2)]) + >>> wrapped(b=3) + (1, 1, [('b', 3), ('more', 2)]) + >>> wrapped(more=4) + Traceback (most recent call last): + TypeError: function() got multiple values for keyword argument 'more' + """ + def wrapper(*args, **kwargs): + return f(*args, more=2, **{**kwargs, 'test': 1}) + return wrapper diff --git a/tests/run/pep448_test_extcall.pyx b/tests/run/pep448_test_extcall.pyx index 03121556c..208ba4f7a 100644 --- a/tests/run/pep448_test_extcall.pyx +++ b/tests/run/pep448_test_extcall.pyx @@ -326,7 +326,7 @@ def errors_non_string_kwarg(): """ >>> errors_non_string_kwarg() # doctest: +ELLIPSIS Traceback (most recent call last): - TypeError: ...keywords must be strings + TypeError: ...keywords must be strings... """ f(**{1:2}) @@ -463,10 +463,10 @@ def call_builtin_empty_dict(): def call_builtin_nonempty_dict(): """ - >>> call_builtin_nonempty_dict() + >>> call_builtin_nonempty_dict() # doctest: +ELLIPSIS Traceback (most recent call last): ... - TypeError: id() takes no keyword arguments + TypeError: id() ... keyword argument... """ return id(1, **{'foo': 1}) @@ -474,7 +474,7 @@ def call_builtin_nonempty_dict(): ''' Cython: currently just passes empty kwargs into f() while CPython keeps the content # A corner case of keyword dictionary items being deleted during -# the function call setup. See <http://bugs.python.org/issue2016>. +# the function call setup. See <https://bugs.python.org/issue2016>. def call_kwargs_modified_while_building(): """ diff --git a/tests/run/pep526_variable_annotations.py b/tests/run/pep526_variable_annotations.py index 9a43f35f6..d91314aeb 100644 --- a/tests/run/pep526_variable_annotations.py +++ b/tests/run/pep526_variable_annotations.py @@ -156,6 +156,8 @@ _WARNINGS = """ 37:19: Unknown type declaration in annotation, ignoring 38:12: Unknown type declaration in annotation, ignoring 39:18: Unknown type declaration in annotation, ignoring +57:19: Unknown type declaration in annotation, ignoring +73:11: Annotation ignored since class-level attributes must be Python objects. Were you trying to set up an instance attribute? 73:19: Unknown type declaration in annotation, ignoring # FIXME: these are sort-of evaluated now, so the warning is misleading 126:21: Unknown type declaration in annotation, ignoring diff --git a/tests/run/pep563_annotations.py b/tests/run/pep563_annotations.py new file mode 100644 index 000000000..26f3c8dbf --- /dev/null +++ b/tests/run/pep563_annotations.py @@ -0,0 +1,26 @@ +# mode: run +# tag: pep563, pure3.7 + +from __future__ import annotations + +def f(a: 1+2==3, b: list, c: this_cant_evaluate, d: "Hello from inside a string") -> "Return me!": + """ + The absolute exact strings aren't reproducible according to the PEP, + so be careful to avoid being too specific + >>> stypes = (type(""), type(u"")) # Python 2 is a bit awkward here + >>> eval(f.__annotations__['a']) + True + >>> isinstance(f.__annotations__['a'], stypes) + True + >>> print(f.__annotations__['b']) + list + >>> print(f.__annotations__['c']) + this_cant_evaluate + >>> isinstance(eval(f.__annotations__['d']), stypes) + True + >>> print(f.__annotations__['return'][1:-1]) # First and last could be either " or ' + Return me! + >>> f.__annotations__['return'][0] == f.__annotations__['return'][-1] + True + """ + pass diff --git a/tests/run/posonly.py b/tests/run/posonly.py new file mode 100644 index 000000000..6ea0c3127 --- /dev/null +++ b/tests/run/posonly.py @@ -0,0 +1,568 @@ +# cython: always_allow_keywords=True +# mode: run +# tag: posonly, pure3.8 + +import cython +import sys +import pickle + +def test_optional_posonly_args1(a, b=10, /, c=100): + """ + >>> test_optional_posonly_args1(1, 2, 3) + 6 + >>> test_optional_posonly_args1(1, 2, c=3) + 6 + >>> test_optional_posonly_args1(1, b=2, c=3) # doctest: +ELLIPSIS + Traceback (most recent call last): + TypeError: test_optional_posonly_args1() got ... keyword argument... 'b' + >>> test_optional_posonly_args1(1, 2) + 103 + >>> test_optional_posonly_args1(1, b=2) # doctest: +ELLIPSIS + Traceback (most recent call last): + TypeError: test_optional_posonly_args1() got ... keyword argument... 'b' + """ + return a + b + c + +def test_optional_posonly_args2(a=1, b=10, /, c=100): + """ + >>> test_optional_posonly_args2(1, 2, 3) + 6 + >>> test_optional_posonly_args2(1, 2, c=3) + 6 + >>> test_optional_posonly_args2(1, b=2, c=3) # doctest: +ELLIPSIS + Traceback (most recent call last): + TypeError: test_optional_posonly_args2() got ... keyword argument... 'b' + >>> test_optional_posonly_args2(1, 2) + 103 + >>> test_optional_posonly_args2(1, b=2) # doctest: +ELLIPSIS + Traceback (most recent call last): + TypeError: test_optional_posonly_args2() got ... keyword argument... 'b' + >>> test_optional_posonly_args2(1, c=2) + 13 + """ + return a + b + c + +# TODO: this causes a line that is too long for old versions of Clang +#def many_args(a1,a2,a3,a4,a5,a6,a7,a8,a9,a10,a11,a12,a13,a14,a15,a16,a17,a18,a19,a20,a21, +# a22,a23,a24,a25,a26,a27,a28,a29,a30,a31,a32,a33,a34,a35,a36,a37,a38,a39,a40, +# a41,a42,a43,a44,a45,a46,a47,a48,a49,a50,a51,a52,a53,a54,a55,a56,a57,a58,a59, +# a60,a61,a62,a63,a64,a65,a66,a67,a68,a69,a70,a71,a72,a73,a74,a75,a76,a77,a78, +# a79,a80,a81,a82,a83,a84,a85,a86,a87,a88,a89,a90,a91,a92,a93,a94,a95,a96,a97, +# a98,a99,a100,a101,a102,a103,a104,a105,a106,a107,a108,a109,a110,a111,a112, +# a113,a114,a115,a116,a117,a118,a119,a120,a121,a122,a123,a124,a125,a126,a127, +# a128,a129,a130,a131,a132,a133,a134,a135,a136,a137,a138,a139,a140,a141,a142, +# a143,a144,a145,a146,a147,a148,a149,a150,a151,a152,a153,a154,a155,a156,a157, +# a158,a159,a160,a161,a162,a163,a164,a165,a166,a167,a168,a169,a170,a171,a172, +# a173,a174,a175,a176,a177,a178,a179,a180,a181,a182,a183,a184,a185,a186,a187, +# a188,a189,a190,a191,a192,a193,a194,a195,a196,a197,a198,a199,a200,a201,a202, +# a203,a204,a205,a206,a207,a208,a209,a210,a211,a212,a213,a214,a215,a216,a217, +# a218,a219,a220,a221,a222,a223,a224,a225,a226,a227,a228,a229,a230,a231,a232, +# a233,a234,a235,a236,a237,a238,a239,a240,a241,a242,a243,a244,a245,a246,a247, +# a248,a249,a250,a251,a252,a253,a254,a255,a256,a257,a258,a259,a260,a261,a262, +# a263,a264,a265,a266,a267,a268,a269,a270,a271,a272,a273,a274,a275,a276,a277, +# a278,a279,a280,a281,a282,a283,a284,a285,a286,a287,a288,a289,a290,a291,a292, +# a293,a294,a295,a296,a297,a298,a299,/,b,c=42,*,d): +# """ +# >>> many_args(*range(299),b=1,c=2,d=3) +# (298, 1, 2, 3) +# >>> many_args(*range(299),b=1,d=3) +# (298, 1, 42, 3) +# >>> many_args(*range(300),d=3) +# (298, 299, 42, 3) +# """ +# return (a299, b, c, d) + +#TODO: update this test for Python 3.8 final +@cython.binding(True) +def func_introspection1(a, b, c, /, d, e=1, *, f, g=2): + """ + >>> if sys.version_info[0] < 3: + ... assert func_introspection2.__code__.co_argcount == 7, func_introspection2.__code__.co_argcount + ... else: + ... assert func_introspection2.__code__.co_argcount == 5, func_introspection2.__code__.co_argcount + >>> func_introspection1.__defaults__ + (1,) + """ + +@cython.binding(True) +def func_introspection2(a, b, c=1, /, d=2, e=3, *, f, g=4): + """ + >>> if sys.version_info[0] < 3: + ... assert func_introspection2.__code__.co_argcount == 7, func_introspection2.__code__.co_argcount + ... else: + ... assert func_introspection2.__code__.co_argcount == 5, func_introspection2.__code__.co_argcount + >>> func_introspection2.__defaults__ + (1, 2, 3) + """ + +def test_pos_only_call_via_unpacking(a, b, /): + """ + >>> test_pos_only_call_via_unpacking(*[1,2]) + 3 + """ + return a + b + +def test_use_positional_as_keyword1(a, /): + """ + >>> test_use_positional_as_keyword1(1) + >>> test_use_positional_as_keyword1(a=1) # doctest: +ELLIPSIS + Traceback (most recent call last): + TypeError: test_use_positional_as_keyword1() ... keyword arguments... + """ + +def test_use_positional_as_keyword2(a, /, b): + """ + >>> test_use_positional_as_keyword2(1, 2) + >>> test_use_positional_as_keyword2(1, b=2) + >>> test_use_positional_as_keyword2(a=1, b=2) # doctest: +ELLIPSIS + Traceback (most recent call last): + TypeError: test_use_positional_as_keyword2() ... positional...arguments... + """ + +def test_use_positional_as_keyword3(a, b, /): + """ + >>> test_use_positional_as_keyword3(1, 2) + >>> test_use_positional_as_keyword3(a=1, b=2) # doctest:+ELLIPSIS + Traceback (most recent call last): + TypeError: test_use_positional_as_keyword3() got ... keyword argument... + """ + +def test_positional_only_and_arg_invalid_calls(a, b, /, c): + """ + >>> test_positional_only_and_arg_invalid_calls(1, 2, 3) + >>> test_positional_only_and_arg_invalid_calls(1, 2, c=3) + >>> test_positional_only_and_arg_invalid_calls(1, 2) # doctest: +ELLIPSIS + Traceback (most recent call last): + TypeError: test_positional_only_and_arg_invalid_calls() ... positional argument... + >>> test_positional_only_and_arg_invalid_calls(1) # doctest: +ELLIPSIS + Traceback (most recent call last): + TypeError: test_positional_only_and_arg_invalid_calls() ... positional arguments... + >>> test_positional_only_and_arg_invalid_calls(1,2,3,4) # doctest: +ELLIPSIS + Traceback (most recent call last): + TypeError: test_positional_only_and_arg_invalid_calls() takes ... positional arguments ...4 ...given... + """ + +def test_positional_only_and_optional_arg_invalid_calls(a, b, /, c=3): + """ + >>> test_positional_only_and_optional_arg_invalid_calls(1, 2) + >>> test_positional_only_and_optional_arg_invalid_calls(1) # doctest: +ELLIPSIS + Traceback (most recent call last): + TypeError: test_positional_only_and_optional_arg_invalid_calls() ... positional argument... + >>> test_positional_only_and_optional_arg_invalid_calls() # doctest: +ELLIPSIS + Traceback (most recent call last): + TypeError: test_positional_only_and_optional_arg_invalid_calls() ... positional arguments... + >>> test_positional_only_and_optional_arg_invalid_calls(1, 2, 3, 4) # doctest: +ELLIPSIS + Traceback (most recent call last): + TypeError: test_positional_only_and_optional_arg_invalid_calls() takes ... positional arguments ...4 ...given... + """ + +def test_positional_only_and_kwonlyargs_invalid_calls(a, b, /, c, *, d, e): + """ + >>> test_positional_only_and_kwonlyargs_invalid_calls(1, 2, 3, d=1, e=2) + >>> test_positional_only_and_kwonlyargs_invalid_calls(1, 2, 3, e=2) # doctest: +ELLIPSIS + Traceback (most recent call last): + TypeError: test_positional_only_and_kwonlyargs_invalid_calls() ... keyword-only argument...d... + >>> test_positional_only_and_kwonlyargs_invalid_calls(1, 2, 3) # doctest: +ELLIPSIS + Traceback (most recent call last): + TypeError: test_positional_only_and_kwonlyargs_invalid_calls() ... keyword-only argument...d... + >>> test_positional_only_and_kwonlyargs_invalid_calls(1, 2) # doctest: +ELLIPSIS + Traceback (most recent call last): + TypeError: test_positional_only_and_kwonlyargs_invalid_calls() ... positional argument... + >>> test_positional_only_and_kwonlyargs_invalid_calls(1) # doctest: +ELLIPSIS + Traceback (most recent call last): + TypeError: test_positional_only_and_kwonlyargs_invalid_calls() ... positional arguments... + >>> test_positional_only_and_kwonlyargs_invalid_calls() # doctest: +ELLIPSIS + Traceback (most recent call last): + TypeError: test_positional_only_and_kwonlyargs_invalid_calls() ... positional arguments... + >>> test_positional_only_and_kwonlyargs_invalid_calls(1, 2, 3, 4, 5, 6, d=7, e=8) # doctest: +ELLIPSIS + Traceback (most recent call last): + TypeError: test_positional_only_and_kwonlyargs_invalid_calls() takes ... positional arguments ... + >>> test_positional_only_and_kwonlyargs_invalid_calls(1, 2, 3, d=1, e=4, f=56) # doctest: +ELLIPSIS + Traceback (most recent call last): + TypeError: test_positional_only_and_kwonlyargs_invalid_calls() got an unexpected keyword argument 'f' + """ + +def test_positional_only_invalid_calls(a, b, /): + """ + >>> test_positional_only_invalid_calls(1, 2) + >>> test_positional_only_invalid_calls(1) # doctest: +ELLIPSIS + Traceback (most recent call last): + TypeError: test_positional_only_invalid_calls() ... positional argument... + >>> test_positional_only_invalid_calls() # doctest: +ELLIPSIS + Traceback (most recent call last): + TypeError: test_positional_only_invalid_calls() ... positional arguments... + >>> test_positional_only_invalid_calls(1, 2, 3) # doctest: +ELLIPSIS + Traceback (most recent call last): + TypeError: test_positional_only_invalid_calls() takes ... positional arguments ...3 ...given... + """ + +def test_positional_only_with_optional_invalid_calls(a, b=2, /): + """ + >>> test_positional_only_with_optional_invalid_calls(1) + >>> test_positional_only_with_optional_invalid_calls() # doctest: +ELLIPSIS + Traceback (most recent call last): + TypeError: test_positional_only_with_optional_invalid_calls() ... positional argument... + >>> test_positional_only_with_optional_invalid_calls(1, 2, 3) # doctest: +ELLIPSIS + Traceback (most recent call last): + TypeError: test_positional_only_with_optional_invalid_calls() takes ... positional arguments ...3 ...given... + """ + +def test_no_standard_args_usage(a, b, /, *, c): + """ + >>> test_no_standard_args_usage(1, 2, c=3) + >>> test_no_standard_args_usage(1, b=2, c=3) # doctest: +ELLIPSIS + Traceback (most recent call last): + TypeError: test_no_standard_args_usage() ... positional... arguments... + """ + +#def test_change_default_pos_only(): +# TODO: probably remove this, since __defaults__ is not writable in Cython? +# """ +# >>> test_change_default_pos_only() +# True +# True +# """ +# def f(a, b=2, /, c=3): +# return a + b + c +# +# print((2,3) == f.__defaults__) +# f.__defaults__ = (1, 2, 3) +# print(f(1, 2, 3) == 6) + +def test_lambdas(): + """ + >>> test_lambdas() + 3 + 3 + 3 + 3 + 3 + """ + x = lambda a, /, b: a + b + print(x(1,2)) + print(x(1,b=2)) + + x = lambda a, /, b=2: a + b + print(x(1)) + + x = lambda a, b, /: a + b + print(x(1, 2)) + + x = lambda a, b, /, : a + b + print(x(1, 2)) + +class TestPosonlyMethods(object): + """ + >>> TestPosonlyMethods().f(1,2) + (1, 2) + >>> TestPosonlyMethods.f(TestPosonlyMethods(), 1, 2) + (1, 2) + >>> try: + ... TestPosonlyMethods.f(1,2) + ... except TypeError: + ... print("Got type error") + Got type error + >>> TestPosonlyMethods().f(1, b=2) # doctest: +ELLIPSIS + Traceback (most recent call last): + TypeError: ...f() got ... keyword argument... 'b' + """ + def f(self, a, b, /): + return a, b + +class TestMangling(object): + """ + >>> TestMangling().f() + 42 + >>> TestMangling().f2() + 42 + + #>>> TestMangling().f3() + #(42, 43) + #>>> TestMangling().f4() + #(42, 43, 44) + + >>> TestMangling().f2(1) + 1 + + #>>> TestMangling().f3(1, _TestMangling__b=2) + #(1, 2) + #>>> TestMangling().f4(1, _TestMangling__b=2, _TestMangling__c=3) + #(1, 2, 3) + """ + def f(self, *, __a=42): + return __a + + def f2(self, __a=42, /): + return __a + +# FIXME: https://github.com/cython/cython/issues/1382 +# def f3(self, __a=42, /, __b=43): +# return (__a, __b) + +# def f4(self, __a=42, /, __b=43, *, __c=44): +# return (__a, __b, __c) + +def test_module_function(a, b, /): + """ + >>> test_module_function(1, 2) + >>> test_module_function() # doctest: +ELLIPSIS + Traceback (most recent call last): + TypeError: test_module_function() ... positional arguments... + """ + +def test_closures1(x,y): + """ + >>> test_closures1(1,2)(3,4) + 10 + >>> test_closures1(1,2)(3) # doctest: +ELLIPSIS + Traceback (most recent call last): + TypeError: ...g() ... positional argument... + >>> test_closures1(1,2)(3,4,5) # doctest: +ELLIPSIS + Traceback (most recent call last): + TypeError: ...g() ... positional argument... + """ + def g(x2,/,y2): + return x + y + x2 + y2 + return g + +def test_closures2(x,/,y): + """ + >>> test_closures2(1,2)(3,4) + 10 + """ + def g(x2,y2): + return x + y + x2 + y2 + return g + + +def test_closures3(x,/,y): + """ + >>> test_closures3(1,2)(3,4) + 10 + >>> test_closures3(1,2)(3) # doctest: +ELLIPSIS + Traceback (most recent call last): + TypeError: ...g() ... positional argument... + >>> test_closures3(1,2)(3,4,5) # doctest: +ELLIPSIS + Traceback (most recent call last): + TypeError: ...g() ... positional argument... + """ + def g(x2,/,y2): + return x + y + x2 + y2 + return g + + +def test_same_keyword_as_positional_with_kwargs(something, /, **kwargs): + """ + >>> test_same_keyword_as_positional_with_kwargs(42, something=42) + (42, {'something': 42}) + >>> test_same_keyword_as_positional_with_kwargs(something=42) # doctest: +ELLIPSIS + Traceback (most recent call last): + TypeError: test_same_keyword_as_positional_with_kwargs() ... positional argument... + >>> test_same_keyword_as_positional_with_kwargs(42) + (42, {}) + """ + return (something, kwargs) + +def test_serialization1(a, b, /): + """ + >>> pickled_posonly = pickle.dumps(test_serialization1) + >>> unpickled_posonly = pickle.loads(pickled_posonly) + >>> unpickled_posonly(1, 2) + (1, 2) + >>> unpickled_posonly(a=1, b=2) # doctest: +ELLIPSIS + Traceback (most recent call last): + TypeError: test_serialization1() got ... keyword argument... + """ + return (a, b) + +def test_serialization2(a, /, b): + """ + >>> pickled_optional = pickle.dumps(test_serialization2) + >>> unpickled_optional = pickle.loads(pickled_optional) + >>> unpickled_optional(1, 2) + (1, 2) + >>> unpickled_optional(a=1, b=2) # doctest: +ELLIPSIS + Traceback (most recent call last): + TypeError: test_serialization2() ... positional... arguments... + """ + return (a, b) + +def test_serialization3(a=1, /, b=2): + """ + >>> pickled_defaults = pickle.dumps(test_serialization3) + >>> unpickled_defaults = pickle.loads(pickled_defaults) + >>> unpickled_defaults(1, 2) + (1, 2) + >>> unpickled_defaults(a=1, b=2) # doctest: +ELLIPSIS + Traceback (most recent call last): + TypeError: test_serialization3() got ... keyword argument... 'a' + """ + return (a, b) + + +async def test_async(a=1, /, b=2): + """ + >>> test_async(a=1, b=2) # doctest: +ELLIPSIS + Traceback (most recent call last): + TypeError: test_async() got ... keyword argument... 'a' + """ + return a, b + + +def test_async_call(*args, **kwargs): + """ + >>> test_async_call(1, 2) + >>> test_async_call(1, b=2) + >>> test_async_call(1) + >>> test_async_call() + """ + if sys.version_info < (3, 6): + return + try: + coro = test_async(*args, **kwargs) + coro.send(None) + except StopIteration as e: + result = e.value + assert result == (1, 2), result + + +def test_generator(a=1, /, b=2): + """ + >>> test_generator(a=1, b=2) # doctest: +ELLIPSIS + Traceback (most recent call last): + TypeError: test_generator() got ... keyword argument... 'a' + >>> gen = test_generator(1, 2) + >>> next(gen) + (1, 2) + >>> gen = test_generator(1, b=2) + >>> next(gen) + (1, 2) + >>> gen = test_generator(1) + >>> next(gen) + (1, 2) + >>> gen = test_generator() + >>> next(gen) + (1, 2) + """ + yield a, b + +def f_call_1_0_0(a,/): + """ + >>> f_call_1_0_0(1) + (1,) + """ + return (a,) + +def f_call_1_1_0(a,/,b): + """ + >>> f_call_1_1_0(1,2) + (1, 2) + """ + return (a,b) + +def f_call_1_1_1(a,/,b,*,c): + """ + >>> f_call_1_1_1(1,2,c=3) + (1, 2, 3) + """ + return (a,b,c) + +def f_call_1_1_1_star(a,/,b,*args,c): + """ + >>> f_call_1_1_1_star(1,2,c=3) + (1, 2, (), 3) + >>> f_call_1_1_1_star(1,2,3,4,5,6,7,8,c=9) + (1, 2, (3, 4, 5, 6, 7, 8), 9) + """ + return (a,b,args,c) + +def f_call_1_1_1_kwds(a,/,b,*,c,**kwds): + """ + >>> f_call_1_1_1_kwds(1,2,c=3) + (1, 2, 3, {}) + >>> f_call_1_1_1_kwds(1,2,c=3,d=4,e=5) == (1, 2, 3, {'d': 4, 'e': 5}) + True + """ + return (a,b,c,kwds) + +def f_call_1_1_1_star_kwds(a,/,b,*args,c,**kwds): + """ + >>> f_call_1_1_1_star_kwds(1,2,c=3,d=4,e=5) == (1, 2, (), 3, {'d': 4, 'e': 5}) + True + >>> f_call_1_1_1_star_kwds(1,2,3,4,c=5,d=6,e=7) == (1, 2, (3, 4), 5, {'d': 6, 'e': 7}) + True + """ + return (a,b,args,c,kwds) + +def f_call_one_optional_kwd(a,/,*,b=2): + """ + >>> f_call_one_optional_kwd(1) + (1, 2) + >>> f_call_one_optional_kwd(1, b=3) + (1, 3) + """ + return (a,b) + +def f_call_posonly_stararg(a,/,*args): + """ + >>> f_call_posonly_stararg(1) + (1, ()) + >>> f_call_posonly_stararg(1, 2, 3, 4) + (1, (2, 3, 4)) + """ + return (a,args) + +def f_call_posonly_kwarg(a,/,**kw): + """ + >>> f_call_posonly_kwarg(1) + (1, {}) + >>> all_args = f_call_posonly_kwarg(1, b=2, c=3, d=4) + >>> all_args == (1, {'b': 2, 'c': 3, 'd': 4}) or all_args + True + """ + return (a,kw) + +def f_call_posonly_stararg_kwarg(a,/,*args,**kw): + """ + >>> f_call_posonly_stararg_kwarg(1) + (1, (), {}) + >>> f_call_posonly_stararg_kwarg(1, 2) + (1, (2,), {}) + >>> all_args = f_call_posonly_stararg_kwarg(1, b=3, c=4) + >>> all_args == (1, (), {'b': 3, 'c': 4}) or all_args + True + >>> all_args = f_call_posonly_stararg_kwarg(1, 2, b=3, c=4) + >>> all_args == (1, (2,), {'b': 3, 'c': 4}) or all_args + True + """ + return (a,args,kw) + +def test_empty_kwargs(a, b, /): + """ + >>> test_empty_kwargs(1, 2) + (1, 2) + >>> test_empty_kwargs(1, 2, **{}) + (1, 2) + >>> test_empty_kwargs(1, 2, **{'c': 3}) + Traceback (most recent call last): + TypeError: test_empty_kwargs() got an unexpected keyword argument 'c' + """ + return (a,b) + + +@cython.cclass +class TestExtensionClass: + """ + >>> t = TestExtensionClass() + >>> t.f(1,2) + (1, 2, 3) + >>> t.f(1,2,4) + (1, 2, 4) + >>> t.f(1, 2, c=4) + (1, 2, 4) + >>> t.f(1, 2, 5, c=6) # doctest: +ELLIPSIS + Traceback (most recent call last): + TypeError: ...f() got multiple values for ...argument 'c' + """ + def f(self, a, b, /, c=3): + return (a,b,c) diff --git a/tests/run/powop.pyx b/tests/run/powop.pyx index 414774562..3f9fdd077 100644 --- a/tests/run/powop.pyx +++ b/tests/run/powop.pyx @@ -123,9 +123,9 @@ def optimised_pow2(n): 0.5 >>> optimised_pow2(0.5) == 2 ** 0.5 True - >>> optimised_pow2('test') + >>> optimised_pow2('test') # doctest: +ELLIPSIS Traceback (most recent call last): - TypeError: unsupported operand type(s) for ** or pow(): 'int' and 'str' + TypeError: ...operand... **... """ if isinstance(n, (int, long)) and 0 <= n < 1000: assert isinstance(2.0 ** n, float), 'float %s' % n @@ -153,9 +153,9 @@ def optimised_pow2_inplace(n): 0.5 >>> optimised_pow2_inplace(0.5) == 2 ** 0.5 True - >>> optimised_pow2_inplace('test') #doctest: +ELLIPSIS + >>> optimised_pow2_inplace('test') # doctest: +ELLIPSIS Traceback (most recent call last): - TypeError: unsupported operand type(s) for ...: 'int' and 'str' + TypeError: ...operand... **... """ x = 2 x **= n diff --git a/tests/run/property_decorator_T593.py b/tests/run/property_decorator_T593.py index 45ddb57a8..d34db8951 100644 --- a/tests/run/property_decorator_T593.py +++ b/tests/run/property_decorator_T593.py @@ -1,5 +1,5 @@ # mode: run -# ticket: 593 +# ticket: t593 # tag: property, decorator my_property = property diff --git a/tests/run/pstats_profile_test.pyx b/tests/run/pstats_profile_test.pyx index da17f08c5..74a135ad5 100644 --- a/tests/run/pstats_profile_test.pyx +++ b/tests/run/pstats_profile_test.pyx @@ -12,9 +12,7 @@ __doc__ = u""" >>> short_stats['f_cdef'] 100 >>> short_stats['f_cpdef'] - 200 - >>> short_stats['f_cpdef (wrapper)'] - 100 + 300 >>> short_stats['f_inline'] 100 >>> short_stats['f_inline_prof'] @@ -50,9 +48,7 @@ __doc__ = u""" >>> short_stats['m_cdef'] 100 >>> short_stats['m_cpdef'] - 200 - >>> short_stats['m_cpdef (wrapper)'] - 100 + 300 >>> try: ... os.unlink(statsfile) @@ -60,10 +56,10 @@ __doc__ = u""" ... pass >>> sorted(callees(s, 'test_profile')) #doctest: +NORMALIZE_WHITESPACE - ['f_cdef', 'f_cpdef', 'f_cpdef (wrapper)', 'f_def', + ['f_cdef', 'f_cpdef', 'f_def', 'f_inline', 'f_inline_prof', 'f_raise', - 'm_cdef', 'm_cpdef', 'm_cpdef (wrapper)', 'm_def', + 'm_cdef', 'm_cpdef', 'm_def', 'withgil_prof'] >>> profile.runctx("test_generators()", locals(), globals(), statsfile) diff --git a/tests/run/pstats_profile_test_pycfunc.pyx b/tests/run/pstats_profile_test_pycfunc.pyx new file mode 100644 index 000000000..717811082 --- /dev/null +++ b/tests/run/pstats_profile_test_pycfunc.pyx @@ -0,0 +1,228 @@ +# tag: pstats +# cython: profile = True +# cython: binding = False + +__doc__ = u""" + >>> import os, tempfile, cProfile as profile, pstats + >>> statsfile = tempfile.mkstemp()[1] + >>> profile.runctx("test_profile(100)", locals(), globals(), statsfile) + >>> s = pstats.Stats(statsfile) + >>> short_stats = dict([(k[2], v[1]) for k,v in s.stats.items()]) + >>> short_stats['f_def'] + 100 + >>> short_stats['f_cdef'] + 100 + >>> short_stats['f_cpdef'] + 200 + >>> short_stats['f_cpdef (wrapper)'] + 100 + >>> short_stats['f_inline'] + 100 + >>> short_stats['f_inline_prof'] + 100 + >>> short_stats['f_noprof'] + Traceback (most recent call last): + ... + KeyError: 'f_noprof' + >>> short_stats['f_raise'] + 100 + + >>> short_stats['withgil_prof'] + 100 + >>> short_stats['withgil_noprof'] + Traceback (most recent call last): + ... + KeyError: 'withgil_noprof' + + >>> short_stats['nogil_prof'] + Traceback (most recent call last): + ... + KeyError: 'nogil_prof' + >>> short_stats['nogil_noprof'] + Traceback (most recent call last): + ... + KeyError: 'nogil_noprof' + + >>> short_stats['f_raise'] + 100 + + >>> short_stats['m_def'] + 200 + >>> short_stats['m_cdef'] + 100 + >>> short_stats['m_cpdef'] + 200 + >>> short_stats['m_cpdef (wrapper)'] + 100 + + >>> try: + ... os.unlink(statsfile) + ... except: + ... pass + + >>> sorted(callees(s, 'test_profile')) #doctest: +NORMALIZE_WHITESPACE + ['f_cdef', 'f_cpdef', 'f_cpdef (wrapper)', 'f_def', + 'f_inline', 'f_inline_prof', + 'f_raise', + 'm_cdef', 'm_cpdef', 'm_cpdef (wrapper)', 'm_def', + 'withgil_prof'] + + >>> profile.runctx("test_generators()", locals(), globals(), statsfile) + >>> s = pstats.Stats(statsfile) + >>> short_stats = dict([(k[2], v[1]) for k,v in s.stats.items()]) + >>> short_stats['generator'] + 3 + + >>> short_stats['generator_exception'] + 2 + + >>> short_stats['genexpr'] + 11 + + >>> sorted(callees(s, 'test_generators')) + ['call_generator', 'call_generator_exception', 'generator_expr'] + + >>> list(callees(s, 'call_generator')) + ['generator'] + + >>> list(callees(s, 'generator')) + [] + + >>> list(callees(s, 'generator_exception')) + [] + + >>> list(callees(s, 'generator_expr')) + ['genexpr'] + + >>> list(callees(s, 'genexpr')) + [] + + >>> def python_generator(): + ... yield 1 + ... yield 2 + >>> def call_python_generator(): + ... list(python_generator()) + + >>> profile.runctx("call_python_generator()", locals(), globals(), statsfile) + >>> python_stats = pstats.Stats(statsfile) + >>> python_stats_dict = dict([(k[2], v[1]) for k,v in python_stats.stats.items()]) + + >>> profile.runctx("call_generator()", locals(), globals(), statsfile) + >>> cython_stats = pstats.Stats(statsfile) + >>> cython_stats_dict = dict([(k[2], v[1]) for k,v in cython_stats.stats.items()]) + + >>> python_stats_dict['python_generator'] == cython_stats_dict['generator'] + True + + >>> try: + ... os.unlink(statsfile) + ... except: + ... pass +""" + +cimport cython + +def callees(pstats, target_caller): + pstats.calc_callees() + for (_, _, caller), callees in pstats.all_callees.items(): + if caller == target_caller: + for (file, line, callee) in callees.keys(): + if 'pyx' in file: + yield callee + +def test_profile(long N): + cdef long i, n = 0 + cdef A a = A() + for i from 0 <= i < N: + n += f_def(i) + n += f_cdef(i) + n += f_cpdef(i) + n += (<object>f_cpdef)(i) + n += f_inline(i) + n += f_inline_prof(i) + n += f_noprof(i) + n += nogil_noprof(i) + n += nogil_prof(i) + n += withgil_noprof(i) + n += withgil_prof(i) + n += a.m_def(i) + n += (<object>a).m_def(i) + n += a.m_cpdef(i) + n += (<object>a).m_cpdef(i) + n += a.m_cdef(i) + try: + n += f_raise(i+2) + except RuntimeError: + pass + return n + +def f_def(long a): + return a + +cdef long f_cdef(long a): + return a + +cpdef long f_cpdef(long a): + return a + +cdef inline long f_inline(long a): + return a + +@cython.profile(True) +cdef inline long f_inline_prof(long a): + return a + +@cython.profile(False) +cdef int f_noprof(long a): + return a + +cdef long f_raise(long) except -2: + raise RuntimeError + +@cython.profile(False) +cdef int withgil_noprof(long a) with gil: + return (a) +@cython.profile(True) +cdef int withgil_prof(long a) with gil: + return (a) + +@cython.profile(False) +cdef int nogil_noprof(long a) nogil: + return a +@cython.profile(True) +cdef int nogil_prof(long a) nogil: + return a + +cdef class A(object): + def m_def(self, long a): + return a + cpdef m_cpdef(self, long a): + return a + cdef m_cdef(self, long a): + return a + +def test_generators(): + call_generator() + call_generator_exception() + generator_expr() + +def call_generator(): + list(generator()) + +def generator(): + yield 1 + yield 2 + +def call_generator_exception(): + try: + list(generator_exception()) + except ValueError: + pass + +def generator_exception(): + yield 1 + raise ValueError(2) + +def generator_expr(): + e = (x for x in range(10)) + return sum(e) diff --git a/tests/run/ptr_warning_T714.pyx b/tests/run/ptr_warning_T714.pyx index 3ec8b5b12..3bd330525 100644 --- a/tests/run/ptr_warning_T714.pyx +++ b/tests/run/ptr_warning_T714.pyx @@ -1,6 +1,6 @@ # mode: run # tag: werror -# ticket: 714 +# ticket: t714 def test_ptr(): """ diff --git a/tests/run/public_fused_types.srctree b/tests/run/public_fused_types.srctree index 54c7aa148..2474e0eaa 100644 --- a/tests/run/public_fused_types.srctree +++ b/tests/run/public_fused_types.srctree @@ -196,8 +196,8 @@ import a as a_mod def ae(result, expected): "assert equals" if result != expected: - print 'result :', result - print 'expected:', expected + print('result :', result) + print('expected:', expected) assert result == expected @@ -227,7 +227,7 @@ ae(myobj.cpdef_method[cy.int, cy.float](10, 10.0), (10, 10.0)) d = {'obj': obj, 'myobj': myobj, 'ae': ae} -exec s in d +exec(s, d) # Test def methods # ae(obj.def_method(12, 14.9), 26) diff --git a/tests/run/pure.pyx b/tests/run/pure.pyx index 57a575a33..23e41c77b 100644 --- a/tests/run/pure.pyx +++ b/tests/run/pure.pyx @@ -25,10 +25,12 @@ def test_declare(n): (100, 100) >>> test_declare(100.5) (100, 100) - >>> test_declare(None) + + # CPython: "TypeError: an integer is required" + >>> test_declare(None) # doctest: +ELLIPSIS Traceback (most recent call last): ... - TypeError: an integer is required + TypeError: ...int... """ x = cython.declare(cython.int) y = cython.declare(cython.int, n) diff --git a/tests/run/pure_cdef_class_property_decorator_T264.pxd b/tests/run/pure_cdef_class_property_decorator_T264.pxd index 4c1d879cf..e5e616a00 100644 --- a/tests/run/pure_cdef_class_property_decorator_T264.pxd +++ b/tests/run/pure_cdef_class_property_decorator_T264.pxd @@ -1,5 +1,5 @@ # mode: run -# ticket: 264 +# ticket: t264 # tag: property, decorator cdef class Prop: diff --git a/tests/run/pure_cdef_class_property_decorator_T264.py b/tests/run/pure_cdef_class_property_decorator_T264.py index 70f299b4f..8ca2fc7b2 100644 --- a/tests/run/pure_cdef_class_property_decorator_T264.py +++ b/tests/run/pure_cdef_class_property_decorator_T264.py @@ -1,5 +1,5 @@ # mode: run -# ticket: 264 +# ticket: t264 # tag: property, decorator class Prop(object): diff --git a/tests/run/pure_fused.pxd b/tests/run/pure_fused.pxd new file mode 100644 index 000000000..c00105087 --- /dev/null +++ b/tests/run/pure_fused.pxd @@ -0,0 +1,9 @@ +cimport cython + +ctypedef fused NotInPy: + int + float + +cdef class TestCls: + @cython.locals(loc = NotInPy) + cpdef cpfunc(self, NotInPy arg) diff --git a/tests/run/pure_fused.py b/tests/run/pure_fused.py new file mode 100644 index 000000000..35a9d27d7 --- /dev/null +++ b/tests/run/pure_fused.py @@ -0,0 +1,64 @@ +# mode: run +# tag: fused, pure3.6 + +#cython: annotation_typing=True + +import cython + +InPy = cython.fused_type(cython.int, cython.float) + +class TestCls: + # although annotations as strings isn't recommended and generates a warning + # it does allow the test to run on more (pure) Python versions + def func1(self, arg: 'NotInPy'): + """ + >>> TestCls().func1(1.0) + 'float' + >>> TestCls().func1(2) + 'int' + """ + loc: 'NotInPy' = arg + return cython.typeof(arg) + + if cython.compiled: + @cython.locals(arg=NotInPy, loc=NotInPy) # NameError for 'NotInPy' in pure Python + def func2(self, arg): + """ + >>> TestCls().func2(1.0) + 'float' + >>> TestCls().func2(2) + 'int' + """ + loc = arg + return cython.typeof(arg) + + def cpfunc(self, arg): + """ + >>> TestCls().cpfunc(1.0) + 'float' + >>> TestCls().cpfunc(2) + 'int' + """ + loc = arg + return cython.typeof(arg) + + def func1_inpy(self, arg: InPy): + """ + >>> TestCls().func1_inpy(1.0) + 'float' + >>> TestCls().func1_inpy(2) + 'int' + """ + loc: InPy = arg + return cython.typeof(arg) + + @cython.locals(arg = InPy, loc = InPy) + def func2_inpy(self, arg): + """ + >>> TestCls().func2_inpy(1.0) + 'float' + >>> TestCls().func2_inpy(2) + 'int' + """ + loc = arg + return cython.typeof(arg) diff --git a/tests/run/pure_py.py b/tests/run/pure_py.py index 89139155c..aa13ead8a 100644 --- a/tests/run/pure_py.py +++ b/tests/run/pure_py.py @@ -36,10 +36,6 @@ def test_declare(n): (100, 100) >>> test_declare(100.5) (100, 100) - >>> test_declare(None) #doctest: +ELLIPSIS - Traceback (most recent call last): - ... - TypeError: ... """ x = cython.declare(cython.int) y = cython.declare(cython.int, n) @@ -221,6 +217,8 @@ def test_declare_c_types(n): @cython.ccall @cython.returns(cython.double) def c_call(x): + if x == -2.0: + raise RuntimeError("huhu!") return x @@ -239,6 +237,10 @@ def call_ccall(x): 1.0 >>> (is_compiled and 1) or result 1 + + >>> call_ccall(-2) + Traceback (most recent call last): + RuntimeError: huhu! """ ret = c_call(x) return ret, cython.typeof(ret) @@ -248,6 +250,8 @@ def call_ccall(x): @cython.inline @cython.returns(cython.double) def cdef_inline(x): + if x == -2.0: + raise RuntimeError("huhu!") return x + 1 @@ -262,6 +266,10 @@ def call_cdef_inline(x): 'int' >>> result == 2.0 or result True + + >>> call_cdef_inline(-2) + Traceback (most recent call last): + RuntimeError: huhu! """ ret = cdef_inline(x) return ret, cython.typeof(ret) @@ -396,6 +404,23 @@ def ccall_except_check_always(x): return x+1 +@cython.test_fail_if_path_exists("//CFuncDeclaratorNode//IntNode[@value = '-1']") +@cython.test_assert_path_exists("//CFuncDeclaratorNode") +@cython.ccall +@cython.returns(cython.long) +@cython.exceptval(check=False) +def ccall_except_no_check(x): + """ + >>> ccall_except_no_check(41) + 42 + >>> try: _ = ccall_except_no_check(0) # no exception propagated! + ... except ValueError: assert not is_compiled + """ + if x == 0: + raise ValueError + return x+1 + + @cython.final @cython.cclass class CClass(object): @@ -422,3 +447,132 @@ class TestUnboundMethod: True """ def meth(self): pass + +@cython.cclass +class Foo: + a = cython.declare(cython.double) + b = cython.declare(cython.double) + c = cython.declare(cython.double) + + @cython.locals(a=cython.double, b=cython.double, c=cython.double) + def __init__(self, a, b, c): + self.a = a + self.b = b + self.c = c + +@cython.cclass +class EmptyClass(object): + def __init__(self, *args): + pass + +def same_type_cast(): + """ + >>> same_type_cast() + True + """ + + f = EmptyClass() + return f is cython.cast(EmptyClass, f) + +def multi_args_init_cast(): + """ + >>> multi_args_init_cast() + True + """ + f = Foo(10, 20, 30) + return cython.cast(Foo, f) is f + +def multi_args_init_declare(): + """ + >>> multi_args_init_declare() is None + True + """ + f = cython.declare(Foo) + + if cython.compiled: + f = None + + return f + +EmptyClassSyn = cython.typedef(EmptyClass) + +def empty_declare(): + """ + >>> empty_declare() + [] + """ + + r0 = cython.declare(EmptyClass) + r1 = cython.declare(EmptyClassSyn) + r2 = cython.declare(MyStruct) + r3 = cython.declare(MyUnion) + r4 = cython.declare(MyStruct2) + r5 = cython.declare(cython.int[2]) + + if cython.compiled: + r0 = None + r1 = None + + res = [ + r0 is None, + r1 is None, + r2 is not None, + r3 is not None, + r4 is not None, + r5 is not None + ] + + r2.is_integral = True + assert( r2.is_integral == True ) + + r3.x = 12.3 + assert( r3.x == 12.3 ) + + #It generates a correct C code, but raises an exception when interpreted + if cython.compiled: + r4[0].is_integral = True + assert( r4[0].is_integral == True ) + + r5[0] = 42 + assert ( r5[0] == 42 ) + + return [i for i, x in enumerate(res) if not x] + +def same_declare(): + """ + >>> same_declare() + True + """ + + f = EmptyClass() + f2 = cython.declare(EmptyClass, f) + return f2 is f + +def none_cast(): + """ + >>> none_cast() is None + True + """ + + f = None + return cython.cast(EmptyClass, f) + +def none_declare(): + """ + >>> none_declare() is None + True + """ + + f = None + f2 = cython.declare(Foo, f) + return f2 + +def array_init_with_list(): + """ + >>> array_init_with_list() + [10, 42] + """ + x = cython.declare(cython.int[20], list(range(20))) + x[12] = 42 + + return [x[10], x[12]] diff --git a/tests/run/pxd_argument_names.srctree b/tests/run/pxd_argument_names.srctree index 885643609..94262ef92 100644 --- a/tests/run/pxd_argument_names.srctree +++ b/tests/run/pxd_argument_names.srctree @@ -1,5 +1,5 @@ # mode: run -# ticket: gh1888 +# ticket: 1888 PYTHON setup.py build_ext --inplace PYTHON -c "import a; a.test()" diff --git a/tests/run/py35_asyncio_async_def.srctree b/tests/run/py35_asyncio_async_def.srctree index 9da5560b3..bbfe8bbe6 100644 --- a/tests/run/py35_asyncio_async_def.srctree +++ b/tests/run/py35_asyncio_async_def.srctree @@ -1,5 +1,5 @@ # mode: run -# tag: asyncio, gh1685 +# tag: asyncio, gh1685, gh2273 PYTHON setup.py build_ext -i PYTHON main.py @@ -19,6 +19,7 @@ setup( import asyncio import cy_test +import py_test from contextlib import closing async def main(): @@ -31,6 +32,12 @@ with closing(asyncio.get_event_loop()) as loop: print("Running Cython coroutine ...") loop.run_until_complete(cy_test.say()) +assert asyncio.iscoroutinefunction(cy_test.cy_async_def_example) == True +assert asyncio.iscoroutinefunction(cy_test.cy_async_def_example) == True +assert asyncio.iscoroutinefunction(py_test.py_async_def_example) == True +assert asyncio.iscoroutinefunction(py_test.py_async_def_example) == True +assert asyncio.iscoroutinefunction(cy_test.cy_def_example) == False +assert asyncio.iscoroutinefunction(py_test.py_def_example) == False ######## cy_test.pyx ######## @@ -51,8 +58,19 @@ async def cb(): await asyncio.sleep(0.5) print("done!") +async def cy_async_def_example(): + return 1 + +def cy_def_example(): + return 1 ######## py_test.py ######## async def py_async(): print("- and this one is from Python") + +async def py_async_def_example(): + return 1 + +def py_def_example(): + return 1 diff --git a/tests/run/py_hash_t.pyx b/tests/run/py_hash_t.pyx index f18a8c3ac..3b20584d0 100644 --- a/tests/run/py_hash_t.pyx +++ b/tests/run/py_hash_t.pyx @@ -2,12 +2,30 @@ cimport cython +class IntLike(object): + def __init__(self, value): + self.value = value + def __index__(self): + return self.value + + def assign_py_hash_t(x): """ >>> assign_py_hash_t(12) 12 >>> assign_py_hash_t(-12) -12 + + >>> assign_py_hash_t(IntLike(-3)) + -3 + >>> assign_py_hash_t(IntLike(1 << 100)) # doctest: +ELLIPSIS + Traceback (most recent call last): + ... + OverflowError: ... + >>> assign_py_hash_t(IntLike(1.5)) # doctest: +ELLIPSIS + Traceback (most recent call last): + ... + TypeError: __index__ ... (type ...float...) """ cdef Py_hash_t h = x return h diff --git a/tests/run/py_ucs4_type.pyx b/tests/run/py_ucs4_type.pyx index 7193319c6..afd45fca3 100644 --- a/tests/run/py_ucs4_type.pyx +++ b/tests/run/py_ucs4_type.pyx @@ -132,15 +132,24 @@ def unicode_type_methods(Py_UCS4 uchar): uchar.isupper(), ] -@cython.test_assert_path_exists('//PythonCapiCallNode') -@cython.test_fail_if_path_exists('//SimpleCallNode') +#@cython.test_assert_path_exists('//PythonCapiCallNode') +#@cython.test_fail_if_path_exists('//SimpleCallNode') def unicode_methods(Py_UCS4 uchar): """ - >>> unicode_methods(ord('A')) == ['a', 'A', 'A'] + >>> unicode_methods(ord('A')) == ['a', 'A', 'A'] or unicode_methods(ord('A')) + True + >>> unicode_methods(ord('a')) == ['a', 'A', 'A'] or unicode_methods(ord('a')) True - >>> unicode_methods(ord('a')) == ['a', 'A', 'A'] + >>> unicode_methods(0x1E9E) == [u'\\xdf', u'\\u1e9e', u'\\u1e9e'] or unicode_methods(0x1E9E) + True + >>> unicode_methods(0x0130) in ( + ... [u'i\\u0307', u'\\u0130', u'\\u0130'], # Py3 + ... [u'i', u'\\u0130', u'\\u0130'], # Py2 + ... ) or unicode_methods(0x0130) True """ + # \u1E9E == 'LATIN CAPITAL LETTER SHARP S' + # \u0130 == 'LATIN CAPITAL LETTER I WITH DOT ABOVE' return [ # character conversion uchar.lower(), @@ -149,11 +158,11 @@ def unicode_methods(Py_UCS4 uchar): ] -@cython.test_assert_path_exists('//PythonCapiCallNode') -@cython.test_fail_if_path_exists( - '//SimpleCallNode', - '//CoerceFromPyTypeNode', -) +#@cython.test_assert_path_exists('//PythonCapiCallNode') +#@cython.test_fail_if_path_exists( +# '//SimpleCallNode', +# '//CoerceFromPyTypeNode', +#) def unicode_method_return_type(Py_UCS4 uchar): """ >>> unicode_method_return_type(ord('A')) @@ -366,5 +375,5 @@ def uchar_lookup_in_dict(obj, Py_UCS4 uchar): _WARNINGS = """ -364:16: Item lookup of unicode character codes now always converts to a Unicode string. Use an explicit C integer cast to get back the previous integer lookup behaviour. +373:16: Item lookup of unicode character codes now always converts to a Unicode string. Use an explicit C integer cast to get back the previous integer lookup behaviour. """ diff --git a/tests/run/py_unicode_type.pyx b/tests/run/py_unicode_type.pyx index d8d172bc9..0d33be927 100644 --- a/tests/run/py_unicode_type.pyx +++ b/tests/run/py_unicode_type.pyx @@ -123,8 +123,8 @@ def unicode_type_methods(Py_UNICODE uchar): uchar.isupper(), ] -@cython.test_assert_path_exists('//PythonCapiCallNode') -@cython.test_fail_if_path_exists('//SimpleCallNode') +#@cython.test_assert_path_exists('//PythonCapiCallNode') +#@cython.test_fail_if_path_exists('//SimpleCallNode') def unicode_methods(Py_UNICODE uchar): """ >>> unicode_methods(ord('A')) == ['a', 'A', 'A'] diff --git a/tests/run/pyclass_annotations_pep526.py b/tests/run/pyclass_annotations_pep526.py new file mode 100644 index 000000000..c972b9455 --- /dev/null +++ b/tests/run/pyclass_annotations_pep526.py @@ -0,0 +1,47 @@ +# cython: language_level=3 +# mode: run +# tag: pure3.7, pep526, pep484 + +from __future__ import annotations + +try: + from typing import ClassVar +except ImportError: # Py<=3.5 + ClassVar = {int: int} + + +class PyAnnotatedClass: + """ + >>> PyAnnotatedClass.__annotations__["CLASS_VAR"] + 'ClassVar[int]' + >>> PyAnnotatedClass.__annotations__["obj"] + 'str' + >>> PyAnnotatedClass.__annotations__["literal"] + "'int'" + >>> PyAnnotatedClass.__annotations__["recurse"] + "'PyAnnotatedClass'" + >>> PyAnnotatedClass.__annotations__["default"] + 'bool' + >>> PyAnnotatedClass.CLASS_VAR + 1 + >>> PyAnnotatedClass.default + False + >>> PyAnnotatedClass.obj + Traceback (most recent call last): + ... + AttributeError: type object 'PyAnnotatedClass' has no attribute 'obj' + """ + CLASS_VAR: ClassVar[int] = 1 + obj: str + literal: "int" + recurse: "PyAnnotatedClass" + default: bool = False + + +class PyVanillaClass: + """ + >>> PyVanillaClass.__annotations__ + Traceback (most recent call last): + ... + AttributeError: type object 'PyVanillaClass' has no attribute '__annotations__' + """ diff --git a/tests/run/pyclass_scope_T671.py b/tests/run/pyclass_scope_T671.py index 911268e22..00a10de11 100644 --- a/tests/run/pyclass_scope_T671.py +++ b/tests/run/pyclass_scope_T671.py @@ -1,5 +1,5 @@ # mode: run -# ticket: 671 +# ticket: t671 A = 1234 diff --git a/tests/run/pyfunction_redefine_T489.pyx b/tests/run/pyfunction_redefine_T489.pyx index cf393407f..5931ff1b6 100644 --- a/tests/run/pyfunction_redefine_T489.pyx +++ b/tests/run/pyfunction_redefine_T489.pyx @@ -1,4 +1,4 @@ -# ticket: 489 +# ticket: t489 """ >>> xxx diff --git a/tests/run/pyintop.pyx b/tests/run/pyintop.pyx index c101542d0..a0bb1e9d9 100644 --- a/tests/run/pyintop.pyx +++ b/tests/run/pyintop.pyx @@ -25,6 +25,8 @@ def or_obj(obj2, obj3): @cython.test_fail_if_path_exists('//IntBinopNode') def or_int(obj2): """ + >>> or_int(0) + 16 >>> or_int(1) 17 >>> or_int(16) @@ -47,6 +49,8 @@ def xor_obj(obj2, obj3): @cython.test_fail_if_path_exists('//IntBinopNode') def xor_int(obj2): """ + >>> xor_int(0) + 16 >>> xor_int(2) 18 >>> xor_int(16) @@ -69,10 +73,14 @@ def and_obj(obj2, obj3): @cython.test_fail_if_path_exists('//IntBinopNode') def and_int(obj2): """ + >>> and_int(0) + 0 >>> and_int(1) 0 >>> and_int(18) 16 + >>> and_int(-1) + 16 """ obj1 = obj2 & 0x10 return obj1 @@ -98,9 +106,30 @@ def rshift_obj(obj2, obj3): return obj1 +@cython.test_assert_path_exists('//IntBinopNode') +def rshift_int_obj(obj3): + """ + >>> rshift_int_obj(3) + 0 + >>> rshift_int_obj(2) + 0 + >>> rshift_int_obj(1) + 1 + >>> rshift_int_obj(0) + 2 + >>> rshift_int_obj(-1) + Traceback (most recent call last): + ValueError: negative shift count + """ + obj1 = 2 >> obj3 + return obj1 + + @cython.test_fail_if_path_exists('//IntBinopNode') def rshift_int(obj2): """ + >>> rshift_int(0) + 0 >>> rshift_int(2) 0 @@ -226,6 +255,8 @@ def mixed_obj(obj2, obj3): ) def mixed_int(obj2): """ + >>> mixed_int(0) + 16 >>> mixed_int(2) 18 >>> mixed_int(16) diff --git a/tests/run/pyobjcast_T313.pyx b/tests/run/pyobjcast_T313.pyx index fb4ef80a0..0e5fc7e8e 100644 --- a/tests/run/pyobjcast_T313.pyx +++ b/tests/run/pyobjcast_T313.pyx @@ -1,4 +1,4 @@ -# ticket: 313 +# ticket: t313 # Ensure casting still works to void* """ diff --git a/tests/run/r_docstrings.pyx b/tests/run/r_docstrings.pyx index b01a8a4ed..4ee3f8735 100644 --- a/tests/run/r_docstrings.pyx +++ b/tests/run/r_docstrings.pyx @@ -14,10 +14,21 @@ doctest = u"""# Python 3 gets all of these right ... >>> C.__doc__ '\\n This is a class docstring.\\n ' + >>> C.docstring_copy_C + '\\n This is a class docstring.\\n ' + >>> CS.docstring_copy_C + '\\n This is a class docstring.\\n ' + >>> CS.__doc__ '\\n This is a subclass docstring.\\n ' + >>> CS.docstring_copy_CS + '\\n This is a subclass docstring.\\n ' + >>> CSS.docstring_copy_CS + '\\n This is a subclass docstring.\\n ' >>> print(CSS.__doc__) None + >>> CSS.docstring_copy_CSS + 'A module docstring' >>> T.__doc__ '\\n This is an extension type docstring.\\n ' @@ -34,22 +45,38 @@ Compare with standard Python: >>> Pyf.__doc__ '\\n This is a function docstring.\\n ' - >>> class PyC: + >>> class PyC(object): ... ''' ... This is a class docstring. ... ''' - >>> class PyCS(C): + ... docstring_copy_C = __doc__ + >>> class PyCS(PyC): ... ''' ... This is a subclass docstring. ... ''' - >>> class PyCSS(CS): - ... pass + ... docstring_copy_CS = __doc__ + >>> class PyCSS(PyCS): + ... docstring_copy_CSS = __doc__ >>> PyC.__doc__ '\\n This is a class docstring.\\n ' + >>> PyC.docstring_copy_C + '\\n This is a class docstring.\\n ' + >>> PyCS.docstring_copy_C + '\\n This is a class docstring.\\n ' + >>> PyCSS.docstring_copy_C + '\\n This is a class docstring.\\n ' + >>> PyCS.__doc__ '\\n This is a subclass docstring.\\n ' + >>> PyCS.docstring_copy_CS + '\\n This is a subclass docstring.\\n ' + >>> PyCSS.docstring_copy_CS + '\\n This is a subclass docstring.\\n ' + >>> PyCSS.__doc__ + >>> PyCSS.docstring_copy_CSS + 'A module docstring' """ __test__ = {"test_docstrings" : doctest} @@ -59,18 +86,24 @@ def f(): This is a function docstring. """ -class C: + +class C(object): """ This is a class docstring. """ + docstring_copy_C = __doc__ + class CS(C): """ This is a subclass docstring. """ + docstring_copy_CS = __doc__ + class CSS(CS): - pass + docstring_copy_CSS = __doc__ + cdef class T: """ diff --git a/tests/run/raise_memory_error_T650.pyx b/tests/run/raise_memory_error_T650.pyx index 76b1ef4ab..8bced08e0 100644 --- a/tests/run/raise_memory_error_T650.pyx +++ b/tests/run/raise_memory_error_T650.pyx @@ -1,4 +1,4 @@ -# ticket: 650 +# ticket: t650 cimport cython diff --git a/tests/run/range_optimisation_T203.pyx b/tests/run/range_optimisation_T203.pyx index 880d8952d..bf8c0c978 100644 --- a/tests/run/range_optimisation_T203.pyx +++ b/tests/run/range_optimisation_T203.pyx @@ -1,4 +1,4 @@ -# ticket: 203 +# ticket: t203 cdef int get_bound(int m): print u"get_bound(%s)"%m diff --git a/tests/run/reduce_pickle.pyx b/tests/run/reduce_pickle.pyx index 1b4c6a73f..537f720a1 100644 --- a/tests/run/reduce_pickle.pyx +++ b/tests/run/reduce_pickle.pyx @@ -287,3 +287,20 @@ cdef class Wrapper(object): return "Wrapper(...)" else: return "Wrapper(%r)" % self.ref + + +# Non-regression test for pickling bound and unbound methods of non-extension +# classes +if sys.version_info[:2] >= (3, 5): + # builtin methods not picklable for python <= 3.4 + class MyClass(object): + """ + >>> import pickle + >>> pickle.loads(pickle.dumps(MyClass.my_method)) is MyClass.my_method + True + >>> bound_method = pickle.loads(pickle.dumps(MyClass().my_method)) + >>> bound_method(1) + 1 + """ + def my_method(self, x): + return x diff --git a/tests/run/refcount_in_meth.pyx b/tests/run/refcount_in_meth.pyx index ed21b437b..af6212779 100644 --- a/tests/run/refcount_in_meth.pyx +++ b/tests/run/refcount_in_meth.pyx @@ -12,8 +12,10 @@ True True """ +cimport cython from cpython.ref cimport PyObject +@cython.always_allow_keywords(False) def get_refcount(obj): return (<PyObject*>obj).ob_refcnt diff --git a/tests/run/reimport_from_package.srctree b/tests/run/reimport_from_package.srctree index 027d26de5..91e4b9d0a 100644 --- a/tests/run/reimport_from_package.srctree +++ b/tests/run/reimport_from_package.srctree @@ -17,8 +17,8 @@ setup( import sys import a -assert a in sys.modules.values(), list(sys.modules) -assert sys.modules['a'] is a, list(sys.modules) +assert a in sys.modules.values(), sorted(sys.modules) +assert sys.modules['a'] is a, sorted(sys.modules) from atest.package import module @@ -33,8 +33,8 @@ assert 'atest.package.module' in sys.modules import a import atest.package.module as module -assert module in sys.modules.values(), list(sys.modules) -assert sys.modules['atest.package.module'] is module, list(sys.modules) +assert module in sys.modules.values(), sorted(sys.modules) +assert sys.modules['atest.package.module'] is module, sorted(sys.modules) if sys.version_info >= (3, 5): from . import pymodule diff --git a/tests/run/relative_cimport.srctree b/tests/run/relative_cimport.srctree index 0184fe464..2989f9ac5 100644 --- a/tests/run/relative_cimport.srctree +++ b/tests/run/relative_cimport.srctree @@ -3,6 +3,7 @@ PYTHON setup.py build_ext --inplace PYTHON -c "from pkg.b import test; assert test() == (1, 2)" +PYTHON -c "from pkg.b_py2 import test; assert test() == (1, 2)" PYTHON -c "from pkg.sub.c import test; assert test() == (1, 2)" ######## setup.py ######## @@ -42,7 +43,23 @@ cdef class test_pxd: from . cimport a from .a cimport test_pxd -cimport a as implicitly_relative_a + +assert a.test_pxd is test_pxd + +def test(): + cdef test_pxd obj = test_pxd() + obj.x = 1 + obj.y = 2 + return (obj.x, obj.y) + + +######## pkg/b_py2.pyx ######## + +# cython: language_level=2 + +from . cimport a +from .a cimport test_pxd +cimport a as implicitly_relative_a # <-- Py2 "feature" assert a.test_pxd is test_pxd assert implicitly_relative_a.test_pxd is test_pxd diff --git a/tests/run/relativeimport_T542.srctree b/tests/run/relativeimport_T542.srctree index 232c5f06f..27020174f 100644 --- a/tests/run/relativeimport_T542.srctree +++ b/tests/run/relativeimport_T542.srctree @@ -26,7 +26,8 @@ try: except ImportError: pass else: - assert False, "absolute import succeeded" + import sys + assert False, "absolute import succeeded: %s" % sorted(sys.modules) import relimport.a import relimport.bmod diff --git a/tests/run/reraise.py b/tests/run/reraise.py index de7aad7bd..43becf59d 100644 --- a/tests/run/reraise.py +++ b/tests/run/reraise.py @@ -31,6 +31,6 @@ def test_reraise_error(): ... else: print("FAILED") """ import sys - if hasattr(sys, 'exc_clear'): # Py2 + if hasattr(sys, 'exc_clear'): # Py2 sys.exc_clear() raise diff --git a/tests/run/self_in_ext_type_closure.pyx b/tests/run/self_in_ext_type_closure.pyx index efb116659..ae45cce76 100644 --- a/tests/run/self_in_ext_type_closure.pyx +++ b/tests/run/self_in_ext_type_closure.pyx @@ -1,5 +1,5 @@ # mode: run -# ticket: 742 +# ticket: t742 import cython diff --git a/tests/run/sequential_parallel.pyx b/tests/run/sequential_parallel.pyx index b383b9cb5..3d8e1efff 100644 --- a/tests/run/sequential_parallel.pyx +++ b/tests/run/sequential_parallel.pyx @@ -656,7 +656,7 @@ def outer_parallel_section(): sum += inner_parallel_section() return sum -cdef int nogil_cdef_except_clause() nogil except 0: +cdef int nogil_cdef_except_clause() nogil except -1: return 1 cdef void nogil_cdef_except_star() nogil except *: diff --git a/tests/run/set_item.pyx b/tests/run/set_item.pyx new file mode 100644 index 000000000..4086351ca --- /dev/null +++ b/tests/run/set_item.pyx @@ -0,0 +1,75 @@ +# mode: run +# tag: list, dict, setitem, delitem + +def set_item(obj, key, value): + """ + >>> set_item([1, 2, 3], 1, -1) + [1, -1, 3] + >>> set_item([1, 2, 3], -1, -1) + [1, 2, -1] + >>> set_item({}, 'abc', 5) + {'abc': 5} + >>> set_item({}, -1, 5) + {-1: 5} + >>> class D(dict): pass + >>> set_item(D({}), 'abc', 5) + {'abc': 5} + >>> set_item(D({}), -1, 5) + {-1: 5} + """ + obj[key] = value + return obj + + +def set_item_int(obj, int key, value): + """ + >>> set_item_int([1, 2, 3], 1, -1) + [1, -1, 3] + >>> set_item_int([1, 2, 3], -1, -1) + [1, 2, -1] + >>> set_item_int({}, 1, 5) + {1: 5} + >>> set_item_int({}, -1, 5) + {-1: 5} + >>> class D(dict): pass + >>> set_item_int(D({}), 1, 5) + {1: 5} + >>> set_item_int(D({}), -1, 5) + {-1: 5} + """ + obj[key] = value + return obj + + +def del_item(obj, key): + """ + >>> del_item([1, 2, 3], 1) + [1, 3] + >>> del_item([1, 2, 3], -3) + [2, 3] + >>> class D(dict): pass + >>> del_item({'abc': 1, 'def': 2}, 'abc') + {'def': 2} + >>> del_item(D({'abc': 1, 'def': 2}), 'abc') + {'def': 2} + >>> del_item(D({-1: 1, -2: 2}), -1) + {-2: 2} + """ + del obj[key] + return obj + + +def del_item_int(obj, int key): + """ + >>> del_item_int([1, 2, 3], 1) + [1, 3] + >>> del_item_int([1, 2, 3], -3) + [2, 3] + >>> class D(dict): pass + >>> del_item_int(D({-1: 1, 1: 2}), 1) + {-1: 1} + >>> del_item_int(D({-1: 1, -2: 2}), -1) + {-2: 2} + """ + del obj[key] + return obj diff --git a/tests/run/set_new.py b/tests/run/set_new.py new file mode 100644 index 000000000..d1c2c4acb --- /dev/null +++ b/tests/run/set_new.py @@ -0,0 +1,21 @@ +""" +>>> X = make_class_with_new(cynew) +>>> X.__new__ is cynew +True +>>> X().__new__ is cynew +True +>>> def pynew(cls): return object.__new__(cls) +>>> X = make_class_with_new(pynew) +>>> X.__new__ is pynew +True +>>> X().__new__ is pynew +True +""" + +def make_class_with_new(n): + class X(object): + __new__ = n + return X + +def cynew(cls): + return object.__new__(cls) diff --git a/tests/run/short_circuit_T404.pyx b/tests/run/short_circuit_T404.pyx index 09affaf7a..dbb24820e 100644 --- a/tests/run/short_circuit_T404.pyx +++ b/tests/run/short_circuit_T404.pyx @@ -1,4 +1,4 @@ -# ticket: 404 +# ticket: t404 cdef long foo(long x): print "foo(%s)" % x diff --git a/tests/run/special_methods_T561.pyx b/tests/run/special_methods_T561.pyx index 350d452f4..d585f35d0 100644 --- a/tests/run/special_methods_T561.pyx +++ b/tests/run/special_methods_T561.pyx @@ -1,6 +1,6 @@ # mode: run -# ticket: 561 -# ticket: 3 +# ticket: tt561 +# ticket: tt3 # The patch in #561 changes code generation for most special methods # to remove the Cython-generated wrapper and let PyType_Ready() diff --git a/tests/run/special_methods_T561_py2.pyx b/tests/run/special_methods_T561_py2.pyx index 285f820f6..9781376b3 100644 --- a/tests/run/special_methods_T561_py2.pyx +++ b/tests/run/special_methods_T561_py2.pyx @@ -1,4 +1,4 @@ -# ticket: 561 +# ticket: t561 # tag: py2 # This file tests the behavior of special methods under Python 2 # after #561. (Only methods whose behavior differs between Python 2 and 3 diff --git a/tests/run/special_methods_T561_py3.pyx b/tests/run/special_methods_T561_py3.pyx index 932002254..0a5f53547 100644 --- a/tests/run/special_methods_T561_py3.pyx +++ b/tests/run/special_methods_T561_py3.pyx @@ -1,4 +1,4 @@ -# ticket: 561 +# ticket: t561 # tag: py3 # This file tests the behavior of special methods under Python 3 # after #561. (Only methods whose behavior differs between Python 2 and 3 diff --git a/tests/run/ssize_t_T399.pyx b/tests/run/ssize_t_T399.pyx index 4a8f65d68..265a3c924 100644 --- a/tests/run/ssize_t_T399.pyx +++ b/tests/run/ssize_t_T399.pyx @@ -1,4 +1,4 @@ -# ticket: 399 +# ticket: t399 __doc__ = u""" >>> test(-2) diff --git a/tests/run/starargs.pyx b/tests/run/starargs.pyx index 5e17faa47..7da9c6442 100644 --- a/tests/run/starargs.pyx +++ b/tests/run/starargs.pyx @@ -1,7 +1,6 @@ cdef sorteditems(d): - l = list(d.items()) - l.sort() - return tuple(l) + return tuple(sorted(d.items())) + def spam(x, y, z): """ @@ -79,6 +78,8 @@ def onlyt(*a): >>> onlyt(1, a=2) Traceback (most recent call last): TypeError: onlyt() got an unexpected keyword argument 'a' + >>> test_no_copy_args(onlyt) + True """ return a @@ -114,3 +115,20 @@ def tk(*a, **k): (1, ('a', 1), ('b', 2)) """ return a + sorteditems(k) + +def t_kwonly(*a, k): + """ + >>> test_no_copy_args(t_kwonly, k=None) + True + """ + return a + + +def test_no_copy_args(func, **kw): + """ + func is a function such that func(*args, **kw) returns args. + We test that no copy is made of the args tuple. + This tests both the caller side and the callee side. + """ + args = (1, 2, 3) + return func(*args, **kw) is args diff --git a/tests/run/starred_target_T664.pyx b/tests/run/starred_target_T664.pyx index 0d291c648..5a88d5ab2 100644 --- a/tests/run/starred_target_T664.pyx +++ b/tests/run/starred_target_T664.pyx @@ -1,4 +1,4 @@ -# ticket: 664 +# ticket: t664 def assign(): """ diff --git a/tests/run/staticmethod.pyx b/tests/run/staticmethod.pyx index ceaf616fb..5ede6a642 100644 --- a/tests/run/staticmethod.pyx +++ b/tests/run/staticmethod.pyx @@ -1,35 +1,17 @@ -__doc__ = u""" ->>> class1.plus1(1) -2 ->>> class2.plus1(1) -2 ->>> class3.plus1(1) -2 ->>> class4.plus1(1) -2 ->>> class4().plus1(1) -2 ->>> class4.bplus1(1) -2 ->>> class4().bplus1(1) -2 -""" - cimport cython -def f_plus(a): - return a + 1 class class1: - plus1 = f_plus - -class class2(object): - plus1 = f_plus - -cdef class class3: - plus1 = f_plus - -class class4: + u""" + >>> class1.plus1(1) + 2 + >>> class1().plus1(1) + 2 + >>> class1.bplus1(1) + 2 + >>> class1().bplus1(1) + 2 + """ @staticmethod def plus1(a): return a + 1 @@ -49,14 +31,14 @@ def nested_class(): >>> obj.plus1(1) 2 """ - class class5(object): + class class2(object): def __new__(cls): # implicit staticmethod return object.__new__(cls) @staticmethod def plus1(a): return a + 1 - return class5 + return class2 cdef class BaseClass(object): diff --git a/tests/run/str_char_coercion_T412.pyx b/tests/run/str_char_coercion_T412.pyx index 6a470e2b7..c1c0ba709 100644 --- a/tests/run/str_char_coercion_T412.pyx +++ b/tests/run/str_char_coercion_T412.pyx @@ -1,4 +1,4 @@ -# ticket: 412 +# ticket: t412 cdef int i = 'x' cdef char c = 'x' diff --git a/tests/run/str_subclass_kwargs.pyx b/tests/run/str_subclass_kwargs.pyx new file mode 100644 index 000000000..94c6712f4 --- /dev/null +++ b/tests/run/str_subclass_kwargs.pyx @@ -0,0 +1,21 @@ +def test_str_subclass_kwargs(k=None): + """ + Test passing keywords with names that are not of type ``str`` + but a subclass: + + >>> class StrSubclass(str): + ... pass + >>> class StrNoCompare(str): + ... def __eq__(self, other): + ... raise RuntimeError("do not compare me") + ... def __hash__(self): + ... return hash(str(self)) + >>> kwargs = {StrSubclass('k'): 'value'} + >>> test_str_subclass_kwargs(**kwargs) + 'value' + >>> kwargs = {StrNoCompare('k'): 'value'} + >>> test_str_subclass_kwargs(**kwargs) + Traceback (most recent call last): + RuntimeError: do not compare me + """ + return k diff --git a/tests/run/strfunction.pyx b/tests/run/strfunction.pyx index dc6adaf79..d4a1c95d4 100644 --- a/tests/run/strfunction.pyx +++ b/tests/run/strfunction.pyx @@ -5,6 +5,8 @@ __doc__ = u""" 'test' """ +cimport cython + s = str z = str('test') @@ -39,3 +41,33 @@ def sub(string): #def csub(string): # return csubs(string) + + +@cython.test_fail_if_path_exists("//SimpleCallNode") +@cython.test_assert_path_exists("//PythonCapiCallNode") +def typed(str s): + """ + >>> print(typed(None)) + None + >>> type(typed(None)) is type(typed(None)) + True + >>> print(typed('abc')) + abc + >>> type(typed('abc')) is type(typed('abc')) + True + """ + return str(s) + + +@cython.test_fail_if_path_exists( + "//SimpleCallNode", + "//PythonCapiCallNode", +) +def typed_not_none(str s not None): + """ + >>> print(typed('abc')) + abc + >>> type(typed('abc')) is type(typed('abc')) + True + """ + return str(s) diff --git a/tests/run/struct_conversion.pyx b/tests/run/struct_conversion.pyx index 26bd62686..8ef652a5d 100644 --- a/tests/run/struct_conversion.pyx +++ b/tests/run/struct_conversion.pyx @@ -28,10 +28,9 @@ def test_constructor_kwds(x, y, color): """ >>> sorted(test_constructor_kwds(1.25, 2.5, 128).items()) [('color', 128), ('x', 1.25), ('y', 2.5)] - >>> test_constructor_kwds(1.25, 2.5, None) + >>> test_constructor_kwds(1.25, 2.5, None) # doctest: +ELLIPSIS Traceback (most recent call last): - ... - TypeError: an integer is required + TypeError:... int... """ cdef Point p = Point(x=x, y=y, color=color) return p @@ -41,10 +40,9 @@ def return_constructor_kwds(double x, y, color): """ >>> sorted(return_constructor_kwds(1.25, 2.5, 128).items()) [('color', 128), ('x', 1.25), ('y', 2.5)] - >>> return_constructor_kwds(1.25, 2.5, None) + >>> return_constructor_kwds(1.25, 2.5, None) # doctest: +ELLIPSIS Traceback (most recent call last): - ... - TypeError: an integer is required + TypeError:... int... """ return Point(x=x, y=y, color=color) @@ -169,3 +167,19 @@ def test_nested_obj_to_struct(NestedStruct nested): nested.mystruct.s.decode('UTF-8'), nested.d) +cdef struct OverriddenCname: + int x "not_x" + +def test_obj_to_struct_cnames(OverriddenCname s): + """ + >>> test_obj_to_struct_cnames({ 'x': 1 }) + 1 + """ + print(s.x) + +def test_struct_to_obj_cnames(): + """ + >>> test_struct_to_obj_cnames() + {'x': 2} + """ + return OverriddenCname(2) diff --git a/tests/run/subop.pyx b/tests/run/subop.pyx index 027ec07bb..fbb92331f 100644 --- a/tests/run/subop.pyx +++ b/tests/run/subop.pyx @@ -191,3 +191,27 @@ def sub_large_x(x): ... except TypeError: pass """ return 2**30 - x + + +def sub0(x): + """ + >>> sub0(0) + (0, 0) + >>> sub0(1) + (1, -1) + >>> sub0(-1) + (-1, 1) + >>> sub0(99) + (99, -99) + >>> a, b = sub0(2**32) + >>> bigint(a) + 4294967296 + >>> bigint(b) + -4294967296 + >>> a, b = sub0(-2**32) + >>> bigint(a) + -4294967296 + >>> bigint(b) + 4294967296 + """ + return x - 0, 0 - x diff --git a/tests/run/temp_alloc_T409.pyx b/tests/run/temp_alloc_T409.pyx index 383e1ef6f..425b7064b 100644 --- a/tests/run/temp_alloc_T409.pyx +++ b/tests/run/temp_alloc_T409.pyx @@ -1,4 +1,4 @@ -# ticket: 409 +# ticket: t409 # Extracted from sage/plot/plot3d/index_face_set.pyx:502 # Turns out to be a bug in implementation of PEP 3132 (Extended Iterable Unpacking) diff --git a/tests/run/temp_sideeffects_T654.pyx b/tests/run/temp_sideeffects_T654.pyx index 8a00f7e5b..fe8f03fee 100644 --- a/tests/run/temp_sideeffects_T654.pyx +++ b/tests/run/temp_sideeffects_T654.pyx @@ -1,4 +1,4 @@ -# ticket: 654 +# ticket: t654 # function call arguments diff --git a/tests/run/test_asyncgen.py b/tests/run/test_asyncgen.py index ec1599434..410d4fe0b 100644 --- a/tests/run/test_asyncgen.py +++ b/tests/run/test_asyncgen.py @@ -247,16 +247,6 @@ class AsyncGenTest(unittest.TestCase): else: self.assertTrue(False) - if sys.version_info < (2, 7): - def assertIn(self, x, container): - self.assertTrue(x in container) - - def assertIs(self, x, y): - self.assertTrue(x is y) - - assertRaises = assertRaisesRegex - - def compare_generators(self, sync_gen, async_gen): def sync_iterate(g): res = [] @@ -273,19 +263,26 @@ class AsyncGenTest(unittest.TestCase): def async_iterate(g): res = [] while True: + an = g.__anext__() try: - next(g.__anext__()) + while True: + try: + next(an) + except StopIteration as ex: + if ex.args: + res.append(ex.args[0]) + break + else: + res.append('EMPTY StopIteration') + break + except StopAsyncIteration: + raise + except Exception as ex: + res.append(str(type(ex))) + break except StopAsyncIteration: res.append('STOP') break - except StopIteration as ex: - if ex.args: - res.append(ex.args[0]) - else: - res.append('EMPTY StopIteration') - break - except Exception as ex: - res.append(str(type(ex))) return res sync_gen_result = sync_iterate(sync_gen) @@ -313,19 +310,22 @@ class AsyncGenTest(unittest.TestCase): g = gen() ai = g.__aiter__() - self.assertEqual(next(ai.__anext__()), ('result',)) + + an = ai.__anext__() + self.assertEqual(next(an), ('result',)) try: - next(ai.__anext__()) + next(an) except StopIteration as ex: self.assertEqual(ex.args[0], 123) else: self.fail('StopIteration was not raised') - self.assertEqual(next(ai.__anext__()), ('result',)) + an = ai.__anext__() + self.assertEqual(next(an), ('result',)) try: - next(ai.__anext__()) + next(an) except StopAsyncIteration as ex: self.assertFalse(ex.args) else: @@ -349,10 +349,12 @@ class AsyncGenTest(unittest.TestCase): g = gen() ai = g.__aiter__() - self.assertEqual(next(ai.__anext__()), ('result',)) + + an = ai.__anext__() + self.assertEqual(next(an), ('result',)) try: - next(ai.__anext__()) + next(an) except StopIteration as ex: self.assertEqual(ex.args[0], 123) else: @@ -459,6 +461,37 @@ class AsyncGenTest(unittest.TestCase): "non-None value .* async generator"): gen().__anext__().send(100) + def test_async_gen_exception_11(self): + def sync_gen(): + yield 10 + yield 20 + + def sync_gen_wrapper(): + yield 1 + sg = sync_gen() + sg.send(None) + try: + sg.throw(GeneratorExit()) + except GeneratorExit: + yield 2 + yield 3 + + async def async_gen(): + yield 10 + yield 20 + + async def async_gen_wrapper(): + yield 1 + asg = async_gen() + await asg.asend(None) + try: + await asg.athrow(GeneratorExit()) + except GeneratorExit: + yield 2 + yield 3 + + self.compare_generators(sync_gen_wrapper(), async_gen_wrapper()) + def test_async_gen_api_01(self): async def gen(): yield 123 @@ -752,17 +785,13 @@ class AsyncGenAsyncioTest(unittest.TestCase): gen = foo() it = gen.__aiter__() self.assertEqual(await it.__anext__(), 1) - t = self.loop.create_task(it.__anext__()) - await asyncio.sleep(0.01) await gen.aclose() - return t - t = self.loop.run_until_complete(run()) + self.loop.run_until_complete(run()) self.assertEqual(DONE, 1) # Silence ResourceWarnings fut.cancel() - t.cancel() self.loop.run_until_complete(asyncio.sleep(0.01)) @needs_py36_asyncio @@ -785,7 +814,7 @@ class AsyncGenAsyncioTest(unittest.TestCase): await g.__anext__() del g - await asyncio.sleep(0.1) + await asyncio.sleep(0.2) self.loop.run_until_complete(run()) self.assertEqual(DONE, 1) @@ -860,6 +889,33 @@ class AsyncGenAsyncioTest(unittest.TestCase): self.loop.run_until_complete(run()) self.assertEqual(DONE, 10) + def test_async_gen_asyncio_aclose_12(self): + DONE = 0 + + async def target(): + await asyncio.sleep(0.01) + 1 / ZERO + + async def foo(): + nonlocal DONE + task = self.loop.create_task(target()) + try: + yield 1 + finally: + try: + await task + except ZeroDivisionError: + DONE = 1 + + async def run(): + gen = foo() + it = gen.__aiter__() + await it.__anext__() + await gen.aclose() + + self.loop.run_until_complete(run()) + self.assertEqual(DONE, 1) + def test_async_gen_asyncio_asend_01(self): DONE = 0 @@ -1162,47 +1218,156 @@ class AsyncGenAsyncioTest(unittest.TestCase): self.loop.run_until_complete(asyncio.sleep(0.1)) - self.loop.run_until_complete(self.loop.shutdown_asyncgens()) - self.assertEqual(finalized, 2) # Silence warnings t1.cancel() t2.cancel() - self.loop.run_until_complete(asyncio.sleep(0.1)) - @needs_py36_asyncio - def test_async_gen_asyncio_shutdown_02(self): - logged = 0 + with self.assertRaises(asyncio.CancelledError): + self.loop.run_until_complete(t1) + with self.assertRaises(asyncio.CancelledError): + self.loop.run_until_complete(t2) - def logger(loop, context): - nonlocal logged - self.assertIn('asyncgen', context) - expected = 'an error occurred during closing of asynchronous' - if expected in context['message']: - logged += 1 + self.loop.run_until_complete(self.loop.shutdown_asyncgens()) - async def waiter(timeout): - try: - await asyncio.sleep(timeout) - yield 1 - finally: - 1 / ZERO + self.assertEqual(finalized, 2) - async def wait(): - async for _ in waiter(1): + """ + def test_async_gen_expression_01(self): + async def arange(n): + for i in range(n): + await asyncio.sleep(0.01) + yield i + + def make_arange(n): + # This syntax is legal starting with Python 3.7 + return (i * 2 async for i in arange(n)) + + async def run(): + return [i async for i in make_arange(10)] + + res = self.loop.run_until_complete(run()) + self.assertEqual(res, [i * 2 for i in range(10)]) + + def test_async_gen_expression_02(self): + async def wrap(n): + await asyncio.sleep(0.01) + return n + + def make_arange(n): + # This syntax is legal starting with Python 3.7 + return (i * 2 for i in range(n) if await wrap(i)) + + async def run(): + return [i async for i in make_arange(10)] + + res = self.loop.run_until_complete(run()) + self.assertEqual(res, [i * 2 for i in range(1, 10)]) + """ + + def test_asyncgen_nonstarted_hooks_are_cancellable(self): + # See https://bugs.python.org/issue38013 + messages = [] + + def exception_handler(loop, context): + messages.append(context) + + async def async_iterate(): + yield 1 + yield 2 + + async def main(): + # loop = asyncio.get_running_loop() + loop = self.loop + loop.set_exception_handler(exception_handler) + + async for i in async_iterate(): + break + + # asyncio.run(main()) + self.loop.run_until_complete(main()) + + self.assertEqual([], messages) + + def test_async_gen_await_same_anext_coro_twice(self): + async def async_iterate(): + yield 1 + yield 2 + + async def run(): + it = async_iterate() + nxt = it.__anext__() + await nxt + with self.assertRaisesRegex( + RuntimeError, + r"cannot reuse already awaited __anext__\(\)/asend\(\)" + ): + await nxt + + await it.aclose() # prevent unfinished iterator warning + + self.loop.run_until_complete(run()) + + def test_async_gen_await_same_aclose_coro_twice(self): + async def async_iterate(): + yield 1 + yield 2 + + async def run(): + it = async_iterate() + nxt = it.aclose() + await nxt + with self.assertRaisesRegex( + RuntimeError, + r"cannot reuse already awaited aclose\(\)/athrow\(\)" + ): + await nxt + + self.loop.run_until_complete(run()) + + def test_async_gen_aclose_twice_with_different_coros(self): + # Regression test for https://bugs.python.org/issue39606 + async def async_iterate(): + yield 1 + yield 2 + + async def run(): + it = async_iterate() + await it.aclose() + await it.aclose() + + self.loop.run_until_complete(run()) + + def test_async_gen_aclose_after_exhaustion(self): + # Regression test for https://bugs.python.org/issue39606 + async def async_iterate(): + yield 1 + yield 2 + + async def run(): + it = async_iterate() + async for _ in it: pass + await it.aclose() - t = self.loop.create_task(wait()) - self.loop.run_until_complete(asyncio.sleep(0.1)) + self.loop.run_until_complete(run()) - self.loop.set_exception_handler(logger) - self.loop.run_until_complete(self.loop.shutdown_asyncgens()) + """ + def test_async_gen_aclose_compatible_with_get_stack(self): + async def async_generator(): + yield object() - self.assertEqual(logged, 1) + async def run(): + ag = async_generator() + self.loop.create_task(ag.aclose()) + tasks = asyncio.all_tasks() + for task in tasks: + # No AttributeError raised + task.get_stack() + + self.loop.run_until_complete(run()) + """ - # Silence warnings - t.cancel() - self.loop.run_until_complete(asyncio.sleep(0.1)) if __name__ == "__main__": unittest.main() diff --git a/tests/run/test_coroutines_pep492.pyx b/tests/run/test_coroutines_pep492.pyx index a010b701f..2841d97af 100644 --- a/tests/run/test_coroutines_pep492.pyx +++ b/tests/run/test_coroutines_pep492.pyx @@ -70,6 +70,12 @@ except ImportError: return (<PyObject*>obj).ob_refcnt +def no_pypy(f): + import platform + if platform.python_implementation() == 'PyPy': + return unittest.skip("excluded in PyPy") + + # compiled exec() def exec(code_string, l, g): from Cython.Shadow import inline @@ -109,7 +115,7 @@ class AsyncYield(object): def run_async(coro): #assert coro.__class__ is types.GeneratorType - assert coro.__class__.__name__ in ('coroutine', '_GeneratorWrapper'), coro.__class__.__name__ + assert coro.__class__.__name__.rsplit('.', 1)[-1] in ('coroutine', '_GeneratorWrapper'), coro.__class__.__name__ buffer = [] result = None @@ -123,7 +129,7 @@ def run_async(coro): def run_async__await__(coro): - assert coro.__class__.__name__ in ('coroutine', '_GeneratorWrapper'), coro.__class__.__name__ + assert coro.__class__.__name__.rsplit('.', 1)[-1] in ('coroutine', '_GeneratorWrapper'), coro.__class__.__name__ aw = coro.__await__() buffer = [] result = None @@ -149,17 +155,6 @@ def silence_coro_gc(): gc.collect() -def min_py27(method): - return None if sys.version_info < (2, 7) else method - - -def ignore_py26(manager): - @contextlib.contextmanager - def dummy(): - yield - return dummy() if sys.version_info < (2, 7) else manager - - @contextlib.contextmanager def captured_stderr(): try: @@ -213,9 +208,10 @@ class AsyncBadSyntaxTest(unittest.TestCase): pass """, - """async def foo(a:await something()): - pass - """, + #"""async def foo(a:await something()): + # pass + #""", # No longer an error with pep-563 (although still nonsense) + # Some other similar tests have also been commented out """async def foo(): def bar(): @@ -413,9 +409,9 @@ class AsyncBadSyntaxTest(unittest.TestCase): pass """, - """async def foo(a:await b): - pass - """, + #"""async def foo(a:await b): + # pass + #""", """def baz(): async def foo(a=await b): @@ -628,9 +624,9 @@ class AsyncBadSyntaxTest(unittest.TestCase): pass """, - """async def foo(a:await b): - pass - """, + #"""async def foo(a:await b): + # pass + #""", """def baz(): async def foo(a=await b): @@ -905,7 +901,7 @@ class CoroutineTest(unittest.TestCase): raise StopIteration with silence_coro_gc(): - self.assertRegex(repr(foo()), '^<coroutine object.* at 0x.*>$') + self.assertRegex(repr(foo()), '^<[^\s]*coroutine object.* at 0x.*>$') def test_func_4(self): async def foo(): @@ -1091,7 +1087,7 @@ class CoroutineTest(unittest.TestCase): c.close() def test_func_15(self): - # See http://bugs.python.org/issue25887 for details + # See https://bugs.python.org/issue25887 for details async def spammer(): return 'spam' @@ -1108,7 +1104,7 @@ class CoroutineTest(unittest.TestCase): reader(spammer_coro).send(None) def test_func_16(self): - # See http://bugs.python.org/issue25887 for details + # See https://bugs.python.org/issue25887 for details @types_coroutine def nop(): @@ -1139,7 +1135,7 @@ class CoroutineTest(unittest.TestCase): reader.throw(Exception('wat')) def test_func_17(self): - # See http://bugs.python.org/issue25887 for details + # See https://bugs.python.org/issue25887 for details async def coroutine(): return 'spam' @@ -1162,7 +1158,7 @@ class CoroutineTest(unittest.TestCase): coro.close() def test_func_18(self): - # See http://bugs.python.org/issue25887 for details + # See https://bugs.python.org/issue25887 for details async def coroutine(): return 'spam' @@ -1831,7 +1827,7 @@ class CoroutineTest(unittest.TestCase): buffer = [] async def test1(): - with ignore_py26(self.assertWarnsRegex(DeprecationWarning, "legacy")): + with self.assertWarnsRegex(DeprecationWarning, "legacy"): async for i1, i2 in AsyncIter(): buffer.append(i1 + i2) @@ -1845,7 +1841,7 @@ class CoroutineTest(unittest.TestCase): buffer = [] async def test2(): nonlocal buffer - with ignore_py26(self.assertWarnsRegex(DeprecationWarning, "legacy")): + with self.assertWarnsRegex(DeprecationWarning, "legacy"): async for i in AsyncIter(): buffer.append(i[0]) if i[0] == 20: @@ -1864,7 +1860,7 @@ class CoroutineTest(unittest.TestCase): buffer = [] async def test3(): nonlocal buffer - with ignore_py26(self.assertWarnsRegex(DeprecationWarning, "legacy")): + with self.assertWarnsRegex(DeprecationWarning, "legacy"): async for i in AsyncIter(): if i[0] > 20: continue @@ -2081,7 +2077,6 @@ class CoroutineTest(unittest.TestCase): self.assertEqual(CNT, 0) # old-style pre-Py3.5.2 protocol - no longer supported - @min_py27 def __test_for_9(self): # Test that DeprecationWarning can safely be converted into # an exception (__aiter__ should not have a chance to raise @@ -2099,7 +2094,6 @@ class CoroutineTest(unittest.TestCase): run_async(foo()) # old-style pre-Py3.5.2 protocol - no longer supported - @min_py27 def __test_for_10(self): # Test that DeprecationWarning can safely be converted into # an exception. @@ -2418,6 +2412,7 @@ class CoroutineTest(unittest.TestCase): finally: aw.close() + @no_pypy def test_fatal_coro_warning(self): # Issue 27811 async def func(): pass diff --git a/tests/run/test_fstring.pyx b/tests/run/test_fstring.pyx index 309696c28..a670e2ccb 100644 --- a/tests/run/test_fstring.pyx +++ b/tests/run/test_fstring.pyx @@ -3,14 +3,13 @@ # tag: allow_unknown_names, f_strings, pep498 import ast +import os import types import decimal import unittest -import contextlib import sys IS_PY2 = sys.version_info[0] < 3 -IS_PY26 = sys.version_info[:2] < (2, 7) from Cython.Build.Inline import cython_inline from Cython.TestUtils import CythonTest @@ -56,32 +55,14 @@ class TestCase(CythonTest): def assertEqual(self, first, second, msg=None): # strip u'' string prefixes in Py2 if first != second and isinstance(first, unicode): - stripped_first = first.replace("u'", "'").replace('u"', '"') + stripped_first = first.replace(u"u'", u"'").replace(u'u"', u'"') if stripped_first == second: first = stripped_first - elif stripped_first.decode('unicode_escape') == second: + elif u'\\' in stripped_first and stripped_first.decode('unicode_escape') == second: first = stripped_first.decode('unicode_escape') super(TestCase, self).assertEqual(first, second, msg) - if IS_PY26: - @contextlib.contextmanager - def assertRaises(self, exc): - try: - yield - except exc: - pass - else: - assert False, "exception '%s' not raised" % exc - - def assertIn(self, value, collection): - self.assertTrue(value in collection) - def test__format__lookup(self): - if IS_PY26: - return - elif IS_PY2: - raise unittest.SkipTest("Py3-only") - # Make sure __format__ is looked up on the type, not the instance. class X: def __format__(self, spec): @@ -109,7 +90,7 @@ class TestCase(CythonTest): self.assertEqual(type(y).__format__(y, ''), 'class') def __test_ast(self): - # Inspired by http://bugs.python.org/issue24975 + # Inspired by https://bugs.python.org/issue24975 class X: def __init__(self): self.called = False @@ -132,14 +113,263 @@ f'{a * x()}'""" # Make sure x was called. self.assertTrue(x.called) + def __test_ast_line_numbers(self): + expr = """ +a = 10 +f'{a * x()}'""" + t = ast.parse(expr) + self.assertEqual(type(t), ast.Module) + self.assertEqual(len(t.body), 2) + # check `a = 10` + self.assertEqual(type(t.body[0]), ast.Assign) + self.assertEqual(t.body[0].lineno, 2) + # check `f'...'` + self.assertEqual(type(t.body[1]), ast.Expr) + self.assertEqual(type(t.body[1].value), ast.JoinedStr) + self.assertEqual(len(t.body[1].value.values), 1) + self.assertEqual(type(t.body[1].value.values[0]), ast.FormattedValue) + self.assertEqual(t.body[1].lineno, 3) + self.assertEqual(t.body[1].value.lineno, 3) + self.assertEqual(t.body[1].value.values[0].lineno, 3) + # check the binop location + binop = t.body[1].value.values[0].value + self.assertEqual(type(binop), ast.BinOp) + self.assertEqual(type(binop.left), ast.Name) + self.assertEqual(type(binop.op), ast.Mult) + self.assertEqual(type(binop.right), ast.Call) + self.assertEqual(binop.lineno, 3) + self.assertEqual(binop.left.lineno, 3) + self.assertEqual(binop.right.lineno, 3) + self.assertEqual(binop.col_offset, 3) + self.assertEqual(binop.left.col_offset, 3) + self.assertEqual(binop.right.col_offset, 7) + + def __test_ast_line_numbers_multiple_formattedvalues(self): + expr = """ +f'no formatted values' +f'eggs {a * x()} spam {b + y()}'""" + t = ast.parse(expr) + self.assertEqual(type(t), ast.Module) + self.assertEqual(len(t.body), 2) + # check `f'no formatted value'` + self.assertEqual(type(t.body[0]), ast.Expr) + self.assertEqual(type(t.body[0].value), ast.JoinedStr) + self.assertEqual(t.body[0].lineno, 2) + # check `f'...'` + self.assertEqual(type(t.body[1]), ast.Expr) + self.assertEqual(type(t.body[1].value), ast.JoinedStr) + self.assertEqual(len(t.body[1].value.values), 4) + self.assertEqual(type(t.body[1].value.values[0]), ast.Constant) + self.assertEqual(type(t.body[1].value.values[0].value), str) + self.assertEqual(type(t.body[1].value.values[1]), ast.FormattedValue) + self.assertEqual(type(t.body[1].value.values[2]), ast.Constant) + self.assertEqual(type(t.body[1].value.values[2].value), str) + self.assertEqual(type(t.body[1].value.values[3]), ast.FormattedValue) + self.assertEqual(t.body[1].lineno, 3) + self.assertEqual(t.body[1].value.lineno, 3) + self.assertEqual(t.body[1].value.values[0].lineno, 3) + self.assertEqual(t.body[1].value.values[1].lineno, 3) + self.assertEqual(t.body[1].value.values[2].lineno, 3) + self.assertEqual(t.body[1].value.values[3].lineno, 3) + # check the first binop location + binop1 = t.body[1].value.values[1].value + self.assertEqual(type(binop1), ast.BinOp) + self.assertEqual(type(binop1.left), ast.Name) + self.assertEqual(type(binop1.op), ast.Mult) + self.assertEqual(type(binop1.right), ast.Call) + self.assertEqual(binop1.lineno, 3) + self.assertEqual(binop1.left.lineno, 3) + self.assertEqual(binop1.right.lineno, 3) + self.assertEqual(binop1.col_offset, 8) + self.assertEqual(binop1.left.col_offset, 8) + self.assertEqual(binop1.right.col_offset, 12) + # check the second binop location + binop2 = t.body[1].value.values[3].value + self.assertEqual(type(binop2), ast.BinOp) + self.assertEqual(type(binop2.left), ast.Name) + self.assertEqual(type(binop2.op), ast.Add) + self.assertEqual(type(binop2.right), ast.Call) + self.assertEqual(binop2.lineno, 3) + self.assertEqual(binop2.left.lineno, 3) + self.assertEqual(binop2.right.lineno, 3) + self.assertEqual(binop2.col_offset, 23) + self.assertEqual(binop2.left.col_offset, 23) + self.assertEqual(binop2.right.col_offset, 27) + + def __test_ast_line_numbers_nested(self): + expr = """ +a = 10 +f'{a * f"-{x()}-"}'""" + t = ast.parse(expr) + self.assertEqual(type(t), ast.Module) + self.assertEqual(len(t.body), 2) + # check `a = 10` + self.assertEqual(type(t.body[0]), ast.Assign) + self.assertEqual(t.body[0].lineno, 2) + # check `f'...'` + self.assertEqual(type(t.body[1]), ast.Expr) + self.assertEqual(type(t.body[1].value), ast.JoinedStr) + self.assertEqual(len(t.body[1].value.values), 1) + self.assertEqual(type(t.body[1].value.values[0]), ast.FormattedValue) + self.assertEqual(t.body[1].lineno, 3) + self.assertEqual(t.body[1].value.lineno, 3) + self.assertEqual(t.body[1].value.values[0].lineno, 3) + # check the binop location + binop = t.body[1].value.values[0].value + self.assertEqual(type(binop), ast.BinOp) + self.assertEqual(type(binop.left), ast.Name) + self.assertEqual(type(binop.op), ast.Mult) + self.assertEqual(type(binop.right), ast.JoinedStr) + self.assertEqual(binop.lineno, 3) + self.assertEqual(binop.left.lineno, 3) + self.assertEqual(binop.right.lineno, 3) + self.assertEqual(binop.col_offset, 3) + self.assertEqual(binop.left.col_offset, 3) + self.assertEqual(binop.right.col_offset, 7) + # check the nested call location + self.assertEqual(len(binop.right.values), 3) + self.assertEqual(type(binop.right.values[0]), ast.Constant) + self.assertEqual(type(binop.right.values[0].value), str) + self.assertEqual(type(binop.right.values[1]), ast.FormattedValue) + self.assertEqual(type(binop.right.values[2]), ast.Constant) + self.assertEqual(type(binop.right.values[2].value), str) + self.assertEqual(binop.right.values[0].lineno, 3) + self.assertEqual(binop.right.values[1].lineno, 3) + self.assertEqual(binop.right.values[2].lineno, 3) + call = binop.right.values[1].value + self.assertEqual(type(call), ast.Call) + self.assertEqual(call.lineno, 3) + self.assertEqual(call.col_offset, 11) + + def __test_ast_line_numbers_duplicate_expression(self): + """Duplicate expression + + NOTE: this is currently broken, always sets location of the first + expression. + """ + expr = """ +a = 10 +f'{a * x()} {a * x()} {a * x()}' +""" + t = ast.parse(expr) + self.assertEqual(type(t), ast.Module) + self.assertEqual(len(t.body), 2) + # check `a = 10` + self.assertEqual(type(t.body[0]), ast.Assign) + self.assertEqual(t.body[0].lineno, 2) + # check `f'...'` + self.assertEqual(type(t.body[1]), ast.Expr) + self.assertEqual(type(t.body[1].value), ast.JoinedStr) + self.assertEqual(len(t.body[1].value.values), 5) + self.assertEqual(type(t.body[1].value.values[0]), ast.FormattedValue) + self.assertEqual(type(t.body[1].value.values[1]), ast.Constant) + self.assertEqual(type(t.body[1].value.values[1].value), str) + self.assertEqual(type(t.body[1].value.values[2]), ast.FormattedValue) + self.assertEqual(type(t.body[1].value.values[3]), ast.Constant) + self.assertEqual(type(t.body[1].value.values[3].value), str) + self.assertEqual(type(t.body[1].value.values[4]), ast.FormattedValue) + self.assertEqual(t.body[1].lineno, 3) + self.assertEqual(t.body[1].value.lineno, 3) + self.assertEqual(t.body[1].value.values[0].lineno, 3) + self.assertEqual(t.body[1].value.values[1].lineno, 3) + self.assertEqual(t.body[1].value.values[2].lineno, 3) + self.assertEqual(t.body[1].value.values[3].lineno, 3) + self.assertEqual(t.body[1].value.values[4].lineno, 3) + # check the first binop location + binop = t.body[1].value.values[0].value + self.assertEqual(type(binop), ast.BinOp) + self.assertEqual(type(binop.left), ast.Name) + self.assertEqual(type(binop.op), ast.Mult) + self.assertEqual(type(binop.right), ast.Call) + self.assertEqual(binop.lineno, 3) + self.assertEqual(binop.left.lineno, 3) + self.assertEqual(binop.right.lineno, 3) + self.assertEqual(binop.col_offset, 3) + self.assertEqual(binop.left.col_offset, 3) + self.assertEqual(binop.right.col_offset, 7) + # check the second binop location + binop = t.body[1].value.values[2].value + self.assertEqual(type(binop), ast.BinOp) + self.assertEqual(type(binop.left), ast.Name) + self.assertEqual(type(binop.op), ast.Mult) + self.assertEqual(type(binop.right), ast.Call) + self.assertEqual(binop.lineno, 3) + self.assertEqual(binop.left.lineno, 3) + self.assertEqual(binop.right.lineno, 3) + self.assertEqual(binop.col_offset, 3) # FIXME: this is wrong + self.assertEqual(binop.left.col_offset, 3) # FIXME: this is wrong + self.assertEqual(binop.right.col_offset, 7) # FIXME: this is wrong + # check the third binop location + binop = t.body[1].value.values[4].value + self.assertEqual(type(binop), ast.BinOp) + self.assertEqual(type(binop.left), ast.Name) + self.assertEqual(type(binop.op), ast.Mult) + self.assertEqual(type(binop.right), ast.Call) + self.assertEqual(binop.lineno, 3) + self.assertEqual(binop.left.lineno, 3) + self.assertEqual(binop.right.lineno, 3) + self.assertEqual(binop.col_offset, 3) # FIXME: this is wrong + self.assertEqual(binop.left.col_offset, 3) # FIXME: this is wrong + self.assertEqual(binop.right.col_offset, 7) # FIXME: this is wrong + + def __test_ast_line_numbers_multiline_fstring(self): + # See bpo-30465 for details. + expr = """ +a = 10 +f''' + {a + * + x()} +non-important content +''' +""" + t = ast.parse(expr) + self.assertEqual(type(t), ast.Module) + self.assertEqual(len(t.body), 2) + # check `a = 10` + self.assertEqual(type(t.body[0]), ast.Assign) + self.assertEqual(t.body[0].lineno, 2) + # check `f'...'` + self.assertEqual(type(t.body[1]), ast.Expr) + self.assertEqual(type(t.body[1].value), ast.JoinedStr) + self.assertEqual(len(t.body[1].value.values), 3) + self.assertEqual(type(t.body[1].value.values[0]), ast.Constant) + self.assertEqual(type(t.body[1].value.values[0].value), str) + self.assertEqual(type(t.body[1].value.values[1]), ast.FormattedValue) + self.assertEqual(type(t.body[1].value.values[2]), ast.Constant) + self.assertEqual(type(t.body[1].value.values[2].value), str) + self.assertEqual(t.body[1].lineno, 3) + self.assertEqual(t.body[1].value.lineno, 3) + self.assertEqual(t.body[1].value.values[0].lineno, 3) + self.assertEqual(t.body[1].value.values[1].lineno, 3) + self.assertEqual(t.body[1].value.values[2].lineno, 3) + self.assertEqual(t.body[1].col_offset, 0) + self.assertEqual(t.body[1].value.col_offset, 0) + self.assertEqual(t.body[1].value.values[0].col_offset, 0) + self.assertEqual(t.body[1].value.values[1].col_offset, 0) + self.assertEqual(t.body[1].value.values[2].col_offset, 0) + # NOTE: the following lineno information and col_offset is correct for + # expressions within FormattedValues. + binop = t.body[1].value.values[1].value + self.assertEqual(type(binop), ast.BinOp) + self.assertEqual(type(binop.left), ast.Name) + self.assertEqual(type(binop.op), ast.Mult) + self.assertEqual(type(binop.right), ast.Call) + self.assertEqual(binop.lineno, 4) + self.assertEqual(binop.left.lineno, 4) + self.assertEqual(binop.right.lineno, 6) + self.assertEqual(binop.col_offset, 4) + self.assertEqual(binop.left.col_offset, 4) + self.assertEqual(binop.right.col_offset, 7) + def test_docstring(self): def f(): f'''Not a docstring''' - self.assertTrue(f.__doc__ is None) + self.assertIsNone(f.__doc__) def g(): '''Not a docstring''' \ f'' - self.assertTrue(g.__doc__ is None) + self.assertIsNone(g.__doc__) def __test_literal_eval(self): with self.assertRaisesRegex(ValueError, 'malformed node or string'): @@ -175,9 +405,27 @@ f'{a * x()}'""" ]) def test_mismatched_parens(self): - self.assertAllRaise(SyntaxError, 'f-string: mismatched', + self.assertAllRaise(SyntaxError, r"f-string: closing parenthesis '\}' " + r"does not match opening parenthesis '\('", ["f'{((}'", ]) + self.assertAllRaise(SyntaxError, r"f-string: closing parenthesis '\)' " + r"does not match opening parenthesis '\['", + ["f'{a[4)}'", + ]) + self.assertAllRaise(SyntaxError, r"f-string: closing parenthesis '\]' " + r"does not match opening parenthesis '\('", + ["f'{a(4]}'", + ]) + self.assertAllRaise(SyntaxError, r"f-string: closing parenthesis '\}' " + r"does not match opening parenthesis '\['", + ["f'{a[4}'", + ]) + self.assertAllRaise(SyntaxError, r"f-string: closing parenthesis '\}' " + r"does not match opening parenthesis '\('", + ["f'{a(4}'", + ]) + self.assertRaises(SyntaxError, eval, "f'{" + "("*500 + "}'") def test_double_braces(self): self.assertEqual(f'{{', '{') @@ -255,7 +503,9 @@ f'{a * x()}'""" ["f'{1#}'", # error because the expression becomes "(1#)" "f'{3(#)}'", "f'{#}'", - "f'{)#}'", # When wrapped in parens, this becomes + ]) + self.assertAllRaise(SyntaxError, r"f-string: unmatched '\)'", + ["f'{)#}'", # When wrapped in parens, this becomes # '()#)'. Make sure that doesn't compile. ]) @@ -271,29 +521,35 @@ f'{a * x()}'""" width = 1 # Test around 256. - for i in range(250, 260): - self.assertEqual(cy_eval(build_fstr(i), x=x, width=width), (x+' ')*i) + # for i in range(250, 260): + # self.assertEqual(eval(build_fstr(i)), (x+' ')*i) + self.assertEqual( + cy_eval('[' + ', '.join(build_fstr(i) for i in range(250, 260)) + ']', x=x, width=width), + [(x+' ')*i for i in range(250, 260)], + ) # Test concatenating 2 largs fstrings. + # self.assertEqual(eval(build_fstr(255)*256), (x+' ')*(255*256)) self.assertEqual(cy_eval(build_fstr(255)*3, x=x, width=width), (x+' ')*(255*3)) # CPython uses 255*256 s = build_fstr(253, '{x:{width}} ') + # self.assertEqual(eval(s), (x+' ')*254) self.assertEqual(cy_eval(s, x=x, width=width), (x+' ')*254) # Test lots of expressions and constants, concatenated. s = "f'{1}' 'x' 'y'" * 1024 + # self.assertEqual(eval(s), '1xy' * 1024) self.assertEqual(cy_eval(s, x=x, width=width), '1xy' * 1024) def test_format_specifier_expressions(self): width = 10 precision = 4 value = decimal.Decimal('12.34567') - if not IS_PY26: - self.assertEqual(f'result: {value:{width}.{precision}}', 'result: 12.35') - self.assertEqual(f'result: {value:{width!r}.{precision}}', 'result: 12.35') - self.assertEqual(f'result: {value:{width:0}.{precision:1}}', 'result: 12.35') - self.assertEqual(f'result: {value:{1}{0:0}.{precision:1}}', 'result: 12.35') - self.assertEqual(f'result: {value:{ 1}{ 0:0}.{ precision:1}}', 'result: 12.35') + self.assertEqual(f'result: {value:{width}.{precision}}', 'result: 12.35') + self.assertEqual(f'result: {value:{width!r}.{precision}}', 'result: 12.35') + self.assertEqual(f'result: {value:{width:0}.{precision:1}}', 'result: 12.35') + self.assertEqual(f'result: {value:{1}{0:0}.{precision:1}}', 'result: 12.35') + self.assertEqual(f'result: {value:{ 1}{ 0:0}.{ precision:1}}', 'result: 12.35') self.assertEqual(f'{10:#{1}0x}', ' 0xa') self.assertEqual(f'{10:{"#"}1{0}{"x"}}', ' 0xa') self.assertEqual(f'{-10:-{"#"}1{0}x}', ' -0xa') @@ -306,14 +562,13 @@ f'{a * x()}'""" # This looks like a nested format spec. ]) - self.assertAllRaise(SyntaxError, "invalid syntax", + self.assertAllRaise(SyntaxError, "f-string: invalid syntax", [# Invalid syntax inside a nested spec. "f'{4:{/5}}'", ]) # CYTHON: The nesting restriction seems rather arbitrary. Ignoring it for now and instead test that it works. - if not IS_PY26: - self.assertEqual(f'result: {value:{width:{0}}.{precision:1}}', 'result: 12.35') + self.assertEqual(f'result: {value:{width:{0}}.{precision:1}}', 'result: 12.35') #self.assertAllRaise(SyntaxError, "f-string: expressions nested too deeply", # [# Can't nest format specifiers. # "f'result: {value:{width:{0}}.{precision:1}}'", @@ -371,7 +626,7 @@ f'{a * x()}'""" ]) # Different error message is raised for other whitespace characters. - self.assertAllRaise(SyntaxError, 'invalid character in identifier', + self.assertAllRaise(SyntaxError, r"invalid non-printable character U\+00A0", ["f'''{\xa0}'''", #"\xa0", ]) @@ -383,12 +638,12 @@ f'{a * x()}'""" # are added around it. But we shouldn't go from an invalid # expression to a valid one. The added parens are just # supposed to allow whitespace (including newlines). - self.assertAllRaise(SyntaxError, 'invalid syntax', + self.assertAllRaise(SyntaxError, 'f-string: invalid syntax', ["f'{,}'", "f'{,}'", # this is (,), which is an error ]) - self.assertAllRaise(SyntaxError, "f-string: expecting '}'", + self.assertAllRaise(SyntaxError, r"f-string: unmatched '\)'", ["f'{3)+(4}'", ]) @@ -502,7 +757,7 @@ f'{a * x()}'""" # lambda doesn't work without parens, because the colon # makes the parser think it's a format_spec - self.assertAllRaise(SyntaxError, 'unexpected EOF while parsing', + self.assertAllRaise(SyntaxError, 'f-string: invalid syntax', ["f'{lambda x:x}'", ]) @@ -511,9 +766,11 @@ f'{a * x()}'""" # a function into a generator def fn(y): f'y:{yield y*2}' + f'{yield}' g = fn(4) self.assertEqual(next(g), 8) + self.assertEqual(next(g), None) def test_yield_send(self): def fn(x): @@ -630,8 +887,7 @@ f'{a * x()}'""" self.assertEqual(f'{f"{y}"*3}', '555') def test_invalid_string_prefixes(self): - self.assertAllRaise(SyntaxError, 'unexpected EOF while parsing', - ["fu''", + single_quote_cases = ["fu''", "uf''", "Fu''", "fU''", @@ -652,8 +908,10 @@ f'{a * x()}'""" "bf''", "bF''", "Bf''", - "BF''", - ]) + "BF''",] + double_quote_cases = [case.replace("'", '"') for case in single_quote_cases] + self.assertAllRaise(SyntaxError, 'unexpected EOF while parsing', + single_quote_cases + double_quote_cases) def test_leading_trailing_spaces(self): self.assertEqual(f'{ 3}', '3') @@ -676,12 +934,17 @@ f'{a * x()}'""" self.assertEqual(f'{3!=4!s}', 'True') self.assertEqual(f'{3!=4!s:.3}', 'Tru') + def test_equal_equal(self): + # Because an expression ending in = has special meaning, + # there's a special test for ==. Make sure it works. + + self.assertEqual(f'{0==1}', 'False') + def test_conversions(self): self.assertEqual(f'{3.14:10.10}', ' 3.14') - if not IS_PY26: - self.assertEqual(f'{3.14!s:10.10}', '3.14 ') - self.assertEqual(f'{3.14!r:10.10}', '3.14 ') - self.assertEqual(f'{3.14!a:10.10}', '3.14 ') + self.assertEqual(f'{3.14!s:10.10}', '3.14 ') + self.assertEqual(f'{3.14!r:10.10}', '3.14 ') + self.assertEqual(f'{3.14!a:10.10}', '3.14 ') self.assertEqual(f'{"a"}', 'a') self.assertEqual(f'{"a"!r}', "'a'") @@ -816,12 +1079,6 @@ f'{a * x()}'""" self.assertEqual('{d[a]}'.format(d=d), 'string') self.assertEqual('{d[0]}'.format(d=d), 'integer') - def test_invalid_expressions(self): - self.assertAllRaise(SyntaxError, 'invalid syntax', - [r"f'{a[4)}'", - r"f'{a(4]}'", - ]) - def test_errors(self): # see issue 26287 exc = ValueError if sys.version_info < (3, 4) else TypeError @@ -834,6 +1091,16 @@ f'{a * x()}'""" r"f'{1000:j}'", ]) + def __test_filename_in_syntaxerror(self): + # see issue 38964 + with temp_cwd() as cwd: + file_path = os.path.join(cwd, 't.py') + with open(file_path, 'w') as f: + f.write('f"{a b}"') # This generates a SyntaxError + _, _, stderr = assert_python_failure(file_path, + PYTHONIOENCODING='ascii') + self.assertIn(file_path.encode('ascii', 'backslashreplace'), stderr) + def test_loop(self): for i in range(1000): self.assertEqual(f'i:{i}', 'i:' + str(i)) @@ -855,5 +1122,129 @@ f'{a * x()}'""" self.assertEqual(cy_eval('f"\\\n"'), '') self.assertEqual(cy_eval('f"\\\r"'), '') + """ + def __test_debug_conversion(self): + x = 'A string' + self.assertEqual(f'{x=}', 'x=' + repr(x)) + self.assertEqual(f'{x =}', 'x =' + repr(x)) + self.assertEqual(f'{x=!s}', 'x=' + str(x)) + self.assertEqual(f'{x=!r}', 'x=' + repr(x)) + self.assertEqual(f'{x=!a}', 'x=' + ascii(x)) + + x = 2.71828 + self.assertEqual(f'{x=:.2f}', 'x=' + format(x, '.2f')) + self.assertEqual(f'{x=:}', 'x=' + format(x, '')) + self.assertEqual(f'{x=!r:^20}', 'x=' + format(repr(x), '^20')) + self.assertEqual(f'{x=!s:^20}', 'x=' + format(str(x), '^20')) + self.assertEqual(f'{x=!a:^20}', 'x=' + format(ascii(x), '^20')) + + x = 9 + self.assertEqual(f'{3*x+15=}', '3*x+15=42') + + # There is code in ast.c that deals with non-ascii expression values. So, + # use a unicode identifier to trigger that. + tenπ = 31.4 + self.assertEqual(f'{tenπ=:.2f}', 'tenπ=31.40') + + # Also test with Unicode in non-identifiers. + self.assertEqual(f'{"Σ"=}', '"Σ"=\'Σ\'') + + # Make sure nested fstrings still work. + self.assertEqual(f'{f"{3.1415=:.1f}":*^20}', '*****3.1415=3.1*****') + + # Make sure text before and after an expression with = works + # correctly. + pi = 'π' + self.assertEqual(f'alpha α {pi=} ω omega', "alpha α pi='π' ω omega") + + # Check multi-line expressions. + self.assertEqual(f'''{ +3 +=}''', '\n3\n=3') + + # Since = is handled specially, make sure all existing uses of + # it still work. + + self.assertEqual(f'{0==1}', 'False') + self.assertEqual(f'{0!=1}', 'True') + self.assertEqual(f'{0<=1}', 'True') + self.assertEqual(f'{0>=1}', 'False') + self.assertEqual(f'{(x:="5")}', '5') + self.assertEqual(x, '5') + self.assertEqual(f'{(x:=5)}', '5') + self.assertEqual(x, 5) + self.assertEqual(f'{"="}', '=') + + x = 20 + # This isn't an assignment expression, it's 'x', with a format + # spec of '=10'. See test_walrus: you need to use parens. + self.assertEqual(f'{x:=10}', ' 20') + + # Test named function parameters, to make sure '=' parsing works + # there. + def f(a): + nonlocal x + oldx = x + x = a + return oldx + x = 0 + self.assertEqual(f'{f(a="3=")}', '0') + self.assertEqual(x, '3=') + self.assertEqual(f'{f(a=4)}', '3=') + self.assertEqual(x, 4) + + # Make sure __format__ is being called. + class C: + def __format__(self, s): + return f'FORMAT-{s}' + def __repr__(self): + return 'REPR' + + self.assertEqual(f'{C()=}', 'C()=REPR') + self.assertEqual(f'{C()=!r}', 'C()=REPR') + self.assertEqual(f'{C()=:}', 'C()=FORMAT-') + self.assertEqual(f'{C()=: }', 'C()=FORMAT- ') + self.assertEqual(f'{C()=:x}', 'C()=FORMAT-x') + self.assertEqual(f'{C()=!r:*^20}', 'C()=********REPR********') + + self.assertRaises(SyntaxError, eval, "f'{C=]'") + + # Make sure leading and following text works. + x = 'foo' + self.assertEqual(f'X{x=}Y', 'Xx='+repr(x)+'Y') + + # Make sure whitespace around the = works. + self.assertEqual(f'X{x =}Y', 'Xx ='+repr(x)+'Y') + self.assertEqual(f'X{x= }Y', 'Xx= '+repr(x)+'Y') + self.assertEqual(f'X{x = }Y', 'Xx = '+repr(x)+'Y') + + # These next lines contains tabs. Backslash escapes don't + # work in f-strings. + # patchcheck doesn't like these tabs. So the only way to test + # this will be to dynamically created and exec the f-strings. But + # that's such a hassle I'll save it for another day. For now, convert + # the tabs to spaces just to shut up patchcheck. + #self.assertEqual(f'X{x =}Y', 'Xx\t='+repr(x)+'Y') + #self.assertEqual(f'X{x = }Y', 'Xx\t=\t'+repr(x)+'Y') + """ + + def test_walrus(self): + x = 20 + # This isn't an assignment expression, it's 'x', with a format + # spec of '=10'. + self.assertEqual(f'{x:=10}', ' 20') + + """ + # This is an assignment expression, which requires parens. + self.assertEqual(f'{(x:=10)}', '10') + self.assertEqual(x, 10) + """ + + def test_invalid_syntax_error_message(self): + # with self.assertRaisesRegex(SyntaxError, "f-string: invalid syntax"): + # compile("f'{a $ b}'", "?", "exec") + self.assertAllRaise(CompileError, "f-string: invalid syntax", ["f'{a $ b}'"]) + + if __name__ == '__main__': unittest.main() diff --git a/tests/run/test_genericclass.py b/tests/run/test_genericclass.py new file mode 100644 index 000000000..e52fc2dc7 --- /dev/null +++ b/tests/run/test_genericclass.py @@ -0,0 +1,289 @@ +# mode: run +# tag: pure3.7 +# cython: language_level=3 + +# COPIED FROM CPython 3.7 + +import contextlib +import unittest +import sys + +class TestMROEntry(unittest.TestCase): + def test_mro_entry_signature(self): + tested = [] + class B: ... + class C: + def __mro_entries__(self, *args, **kwargs): + tested.extend([args, kwargs]) + return (C,) + c = C() + self.assertEqual(tested, []) + class D(B, c): ... + self.assertEqual(tested[0], ((B, c),)) + self.assertEqual(tested[1], {}) + + def test_mro_entry(self): + tested = [] + class A: ... + class B: ... + class C: + def __mro_entries__(self, bases): + tested.append(bases) + return (self.__class__,) + c = C() + self.assertEqual(tested, []) + class D(A, c, B): ... + self.assertEqual(tested[-1], (A, c, B)) + self.assertEqual(D.__bases__, (A, C, B)) + self.assertEqual(D.__orig_bases__, (A, c, B)) + self.assertEqual(D.__mro__, (D, A, C, B, object)) + d = D() + class E(d): ... + self.assertEqual(tested[-1], (d,)) + self.assertEqual(E.__bases__, (D,)) + + def test_mro_entry_none(self): + tested = [] + class A: ... + class B: ... + class C: + def __mro_entries__(self, bases): + tested.append(bases) + return () + c = C() + self.assertEqual(tested, []) + class D(A, c, B): ... + self.assertEqual(tested[-1], (A, c, B)) + self.assertEqual(D.__bases__, (A, B)) + self.assertEqual(D.__orig_bases__, (A, c, B)) + self.assertEqual(D.__mro__, (D, A, B, object)) + class E(c): ... + self.assertEqual(tested[-1], (c,)) + if sys.version_info[0] > 2: + # not all of it works on Python 2 + self.assertEqual(E.__bases__, (object,)) + self.assertEqual(E.__orig_bases__, (c,)) + if sys.version_info[0] > 2: + # not all of it works on Python 2 + self.assertEqual(E.__mro__, (E, object)) + + def test_mro_entry_with_builtins(self): + tested = [] + class A: ... + class C: + def __mro_entries__(self, bases): + tested.append(bases) + return (dict,) + c = C() + self.assertEqual(tested, []) + class D(A, c): ... + self.assertEqual(tested[-1], (A, c)) + self.assertEqual(D.__bases__, (A, dict)) + self.assertEqual(D.__orig_bases__, (A, c)) + self.assertEqual(D.__mro__, (D, A, dict, object)) + + def test_mro_entry_with_builtins_2(self): + tested = [] + class C: + def __mro_entries__(self, bases): + tested.append(bases) + return (C,) + c = C() + self.assertEqual(tested, []) + class D(c, dict): ... + self.assertEqual(tested[-1], (c, dict)) + self.assertEqual(D.__bases__, (C, dict)) + self.assertEqual(D.__orig_bases__, (c, dict)) + self.assertEqual(D.__mro__, (D, C, dict, object)) + + def test_mro_entry_errors(self): + class C_too_many: + def __mro_entries__(self, bases, something, other): + return () + c = C_too_many() + with self.assertRaises(TypeError): + class D(c): ... + class C_too_few: + def __mro_entries__(self): + return () + d = C_too_few() + with self.assertRaises(TypeError): + class D(d): ... + + def test_mro_entry_errors_2(self): + class C_not_callable: + __mro_entries__ = "Surprise!" + c = C_not_callable() + with self.assertRaises(TypeError): + class D(c): ... + class C_not_tuple: + def __mro_entries__(self): + return object + c = C_not_tuple() + with self.assertRaises(TypeError): + class D(c): ... + + def test_mro_entry_metaclass(self): + meta_args = [] + class Meta(type): + def __new__(mcls, name, bases, ns): + meta_args.extend([mcls, name, bases, ns]) + return super().__new__(mcls, name, bases, ns) + class A: ... + class C: + def __mro_entries__(self, bases): + return (A,) + c = C() + class D(c, metaclass=Meta): + x = 1 + self.assertEqual(meta_args[0], Meta) + self.assertEqual(meta_args[1], 'D') + self.assertEqual(meta_args[2], (A,)) + self.assertEqual(meta_args[3]['x'], 1) + self.assertEqual(D.__bases__, (A,)) + self.assertEqual(D.__orig_bases__, (c,)) + self.assertEqual(D.__mro__, (D, A, object)) + self.assertEqual(D.__class__, Meta) + + @unittest.skipIf(sys.version_info < (3, 7), "'type' checks for __mro_entries__ not implemented") + def test_mro_entry_type_call(self): + # Substitution should _not_ happen in direct type call + class C: + def __mro_entries__(self, bases): + return () + c = C() + with self.assertRaisesRegex(TypeError, + "MRO entry resolution; " + "use types.new_class()"): + type('Bad', (c,), {}) + + +class TestClassGetitem(unittest.TestCase): + # BEGIN - Additional tests from cython + def test_no_class_getitem(self): + class C: ... + with self.assertRaises(TypeError): + C[int] + + # END - Additional tests from cython + + def test_class_getitem(self): + getitem_args = [] + class C: + def __class_getitem__(*args, **kwargs): + getitem_args.extend([args, kwargs]) + return None + C[int, str] + self.assertEqual(getitem_args[0], (C, (int, str))) + self.assertEqual(getitem_args[1], {}) + + def test_class_getitem_format(self): + class C: + def __class_getitem__(cls, item): + return f'C[{item.__name__}]' + self.assertEqual(C[int], 'C[int]') + self.assertEqual(C[C], 'C[C]') + + def test_class_getitem_inheritance(self): + class C: + def __class_getitem__(cls, item): + return f'{cls.__name__}[{item.__name__}]' + class D(C): ... + self.assertEqual(D[int], 'D[int]') + self.assertEqual(D[D], 'D[D]') + + def test_class_getitem_inheritance_2(self): + class C: + def __class_getitem__(cls, item): + return 'Should not see this' + class D(C): + def __class_getitem__(cls, item): + return f'{cls.__name__}[{item.__name__}]' + self.assertEqual(D[int], 'D[int]') + self.assertEqual(D[D], 'D[D]') + + def test_class_getitem_classmethod(self): + class C: + @classmethod + def __class_getitem__(cls, item): + return f'{cls.__name__}[{item.__name__}]' + class D(C): ... + self.assertEqual(D[int], 'D[int]') + self.assertEqual(D[D], 'D[D]') + + @unittest.skipIf(sys.version_info < (3, 6), "__init_subclass__() requires Py3.6+ (PEP 487)") + def test_class_getitem_patched(self): + class C: + def __init_subclass__(cls): + def __class_getitem__(cls, item): + return f'{cls.__name__}[{item.__name__}]' + cls.__class_getitem__ = classmethod(__class_getitem__) + class D(C): ... + self.assertEqual(D[int], 'D[int]') + self.assertEqual(D[D], 'D[D]') + + def test_class_getitem_with_builtins(self): + class A(dict): + called_with = None + + def __class_getitem__(cls, item): + cls.called_with = item + class B(A): + pass + self.assertIs(B.called_with, None) + B[int] + self.assertIs(B.called_with, int) + + def test_class_getitem_errors(self): + class C_too_few: + def __class_getitem__(cls): + return None + with self.assertRaises(TypeError): + C_too_few[int] + class C_too_many: + def __class_getitem__(cls, one, two): + return None + with self.assertRaises(TypeError): + C_too_many[int] + + def test_class_getitem_errors_2(self): + class C: + def __class_getitem__(cls, item): + return None + with self.assertRaises(TypeError): + C()[int] + class E: ... + e = E() + e.__class_getitem__ = lambda cls, item: 'This will not work' + with self.assertRaises(TypeError): + e[int] + class C_not_callable: + __class_getitem__ = "Surprise!" + with self.assertRaises(TypeError): + C_not_callable[int] + + def test_class_getitem_metaclass(self): + class Meta(type): + def __class_getitem__(cls, item): + return f'{cls.__name__}[{item.__name__}]' + self.assertEqual(Meta[int], 'Meta[int]') + + def test_class_getitem_with_metaclass(self): + class Meta(type): pass + class C(metaclass=Meta): + def __class_getitem__(cls, item): + return f'{cls.__name__}[{item.__name__}]' + self.assertEqual(C[int], 'C[int]') + + def test_class_getitem_metaclass_first(self): + class Meta(type): + def __getitem__(cls, item): + return 'from metaclass' + class C(metaclass=Meta): + def __class_getitem__(cls, item): + return 'from __class_getitem__' + self.assertEqual(C[int], 'from metaclass') + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/run/test_genericclass_exttype.pyx b/tests/run/test_genericclass_exttype.pyx new file mode 100644 index 000000000..b8cdd0c47 --- /dev/null +++ b/tests/run/test_genericclass_exttype.pyx @@ -0,0 +1,96 @@ +# mode: run +# cython: language_level=3 + +import unittest +import sys + + +cdef class UnSupport: pass + +cdef class Unpack: + para_list = [] + def __class_getitem__(*args, **kwargs): + Unpack.para_list.extend([args, kwargs]) + +cdef class Format: + def __class_getitem__(cls, item): + return f'{cls.__name__}[{item.__name__}]' + +cdef class ExFormat(Format): pass + +cdef class Override: + def __class_getitem__(cls, item): + return 'Should not see this' + +cdef class Covered(Override): + def __class_getitem__(cls, item): + return f'{cls.__name__}[{item.__name__}]' + +cdef class Decorated: + @classmethod + def __class_getitem__(cls, item): + return f'{cls.__name__}[{item.__name__}]' + +cdef class ExDecorated(Decorated): pass + +cdef class Invalid1: + def __class_getitem__(cls): pass + +cdef class Invalid2: + def __class_getitem__(cls, item1, item2): pass + +cdef class Invalid3: + cdef dict __dict__ + def __init__(self): + self.__class_getitem__ = lambda cls, items: 'This will not work' + +cdef class Invalid4: + __class_getitem__ = "Surprise!" + + +class TestClassGetitem(unittest.TestCase): + # BEGIN - Additional tests from cython + def test_no_class_getitem(self): + with self.assertRaises(TypeError): + UnSupport[int] + + # END - Additional tests from cython + + def test_class_getitem(self): + Unpack[int, str] + self.assertEqual(Unpack.para_list[0], (Unpack, (int, str))) + self.assertEqual(Unpack.para_list[1], {}) + + def test_class_getitem_format(self): + self.assertEqual(Format[int], 'Format[int]') + self.assertEqual(Format[Format], 'Format[Format]') + + def test_class_getitem_inheritance(self): + self.assertEqual(ExFormat[int], 'ExFormat[int]') + self.assertEqual(ExFormat[ExFormat], 'ExFormat[ExFormat]') + + def test_class_getitem_inheritance_2(self): + self.assertEqual(Covered[int], 'Covered[int]') + self.assertEqual(Covered[Covered], 'Covered[Covered]') + + def test_class_getitem_classmethod(self): + self.assertEqual(ExDecorated[int], 'ExDecorated[int]') + self.assertEqual(ExDecorated[ExDecorated], 'ExDecorated[ExDecorated]') + + def test_class_getitem_errors(self): + with self.assertRaises(TypeError): + Invalid1[int] + with self.assertRaises(TypeError): + Invalid2[int] + + def test_class_getitem_errors_2(self): + with self.assertRaises(TypeError): + Format()[int] + with self.assertRaises(TypeError): + Invalid3()[int] + with self.assertRaises(TypeError): + Invalid4[int] + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/run/test_grammar.py b/tests/run/test_grammar.py index 3bdadbefa..c41b75f55 100644 --- a/tests/run/test_grammar.py +++ b/tests/run/test_grammar.py @@ -1,9 +1,26 @@ -### COPIED FROM CPython 3.5 - ADDED PART FOLLOWS ### +### COPIED FROM CPython 3.9 - ADDED PART FOLLOWS ### # cython: language_level=3 +import cython import contextlib from tempfile import NamedTemporaryFile -from Cython.Compiler.Main import compile as cython_compile +from Cython.Compiler.Main import compile as cython_compile, CompileError +from Cython.Build.Inline import cython_inline + + +@contextlib.contextmanager +def hidden_stderr(): + try: + from StringIO import StringIO + except ImportError: + from io import StringIO + + old_stderr = sys.stderr + try: + sys.stderr = StringIO() + yield + finally: + sys.stderr = old_stderr def _compile(code): @@ -11,36 +28,44 @@ def _compile(code): f.write(code.encode('utf8')) f.flush() - try: - from StringIO import StringIO - except ImportError: - from io import StringIO - - old_stderr = sys.stderr - try: - sys.stderr = StringIO() + with hidden_stderr(): result = cython_compile(f.name, language_level=3) - finally: - sys.stderr = old_stderr return result -def check_syntax_error(test, code): +def check_syntax_error(test, code, msg=None): result = _compile(code) assert not result.c_file -def compile(code, name, what): - assert what == 'exec' - result = _compile(code) - if not result.c_file: - raise SyntaxError('unexpected EOF') # see usage of compile() below +def check_syntax_warning(test, code, *args): + # ignore for now + return _compile(code) -def exec(code): - result = _compile(code) - if not result.c_file: - raise SyntaxError('unexpected EOF') # see usage of compile() below +if cython.compiled: + def compile(code, name, what): + assert what == 'exec' + result = _compile(code) + if not result.c_file: + raise SyntaxError('unexpected EOF') # see usage of compile() below + + def exec(code): + result = _compile(code) + if not result.c_file: + raise SyntaxError('unexpected EOF') # see usage of compile() below + + def eval(code): + try: + with hidden_stderr(): + return cython_inline(code) + except CompileError as exc: + raise SyntaxError(str(exc)) + + +def use_old_parser(): + # FIXME: currently disabling new PEG parser tests. + return True import unittest @@ -72,16 +97,109 @@ skip = unittest.skip # Python test set -- part 1, grammar. # This just tests whether the parser accepts them all. -#from test.support import check_syntax_error +#from test.support import check_syntax_error, check_syntax_warning, use_old_parser import inspect import unittest import sys +import warnings # testing import * from sys import * +# different import patterns to check that __annotations__ does not interfere +# with import machinery +#import test.ann_module as ann_module +#import typing +#from collections import ChainMap +#from test import ann_module2 +#import test + +# These are shared with test_tokenize and other test modules. +# +# Note: since several test cases filter out floats by looking for "e" and ".", +# don't add hexadecimal literals that contain "e" or "E". +VALID_UNDERSCORE_LITERALS = [ + '0_0_0', + '4_2', + '1_0000_0000', + '0b1001_0100', + '0xffff_ffff', + '0o5_7_7', + '1_00_00.5', + '1_00_00.5e5', + '1_00_00e5_1', + '1e1_0', + '.1_4', + '.1_4e1', + '0b_0', + '0x_f', + '0o_5', + '1_00_00j', + '1_00_00.5j', + '1_00_00e5_1j', + '.1_4j', + '(1_2.5+3_3j)', + '(.5_6j)', +] +INVALID_UNDERSCORE_LITERALS = [ + # Trailing underscores: + '0_', + '42_', + '1.4j_', + '0x_', + '0b1_', + '0xf_', + '0o5_', + '0 if 1_Else 1', + # Underscores in the base selector: + '0_b0', + '0_xf', + '0_o5', + # Old-style octal, still disallowed: + '0_7', + '09_99', + # Multiple consecutive underscores: + '4_______2', + '0.1__4', + '0.1__4j', + '0b1001__0100', + '0xffff__ffff', + '0x___', + '0o5__77', + '1e1__0', + '1e1__0j', + # Underscore right before a dot: + '1_.4', + '1_.4j', + # Underscore right after a dot: + '1._4', + '1._4j', + '._5', + '._5j', + # Underscore right after a sign: + '1.0e+_1', + '1.0e+_1j', + # Underscore right before j: + '1.4_j', + '1.4e5_j', + # Underscore right before e: + '1_e1', + '1.4_e1', + '1.4_e1j', + # Underscore right after e: + '1e_1', + '1.4e_1', + '1.4e_1j', + # Complex cases with parens: + '(1+1.5_j_)', + '(1+1.5_j)', +] + class TokenTests(unittest.TestCase): + #from test.support import check_syntax_error + check_syntax_error = check_syntax_error + def test_backslash(self): # Backslash means line continuation: x = 1 \ @@ -158,6 +276,40 @@ class TokenTests(unittest.TestCase): self.assertEqual(1 if 0else 0, 0) self.assertRaises(SyntaxError, eval, "0 if 1Else 0") + @skip("Done more efficiently in TestGrammar") + def test_underscore_literals(self): + for lit in VALID_UNDERSCORE_LITERALS: + self.assertEqual(eval(lit), eval(lit.replace('_', ''))) + for lit in INVALID_UNDERSCORE_LITERALS: + self.assertRaises(SyntaxError, eval, lit) + # Sanity check: no literal begins with an underscore + self.assertRaises(NameError, eval, "_0") + + def test_bad_numerical_literals(self): + check = self.check_syntax_error + check("0b12", "invalid digit '2' in binary literal") + check("0b1_2", "invalid digit '2' in binary literal") + check("0b2", "invalid digit '2' in binary literal") + check("0b1_", "invalid binary literal") + check("0b", "invalid binary literal") + check("0o18", "invalid digit '8' in octal literal") + check("0o1_8", "invalid digit '8' in octal literal") + check("0o8", "invalid digit '8' in octal literal") + check("0o1_", "invalid octal literal") + check("0o", "invalid octal literal") + check("0x1_", "invalid hexadecimal literal") + check("0x", "invalid hexadecimal literal") + check("1_", "invalid decimal literal") + # FIXME: not currently a syntax error + """ + check("012", + "leading zeros in decimal integer literals are not permitted; " + "use an 0o prefix for octal integers") + """ + check("1.2_", "invalid decimal literal") + check("1e2_", "invalid decimal literal") + check("1e+", "invalid decimal literal") + def test_string_literals(self): x = ''; y = ""; self.assertTrue(len(x) == 0 and x == y) x = '\''; y = "'"; self.assertTrue(len(x) == 1 and x == y and ord(x) == 39) @@ -201,7 +353,8 @@ the \'lazy\' dog.\n\ def test_ellipsis(self): x = ... self.assertTrue(x is Ellipsis) - self.assertRaises(SyntaxError, eval, ".. .") + # FIXME: why is this not rejected ??? + #self.assertRaises(SyntaxError, eval, ".. .") def test_eof_error(self): samples = ("def foo(", "\ndef foo(", "def foo(\n") @@ -210,7 +363,7 @@ the \'lazy\' dog.\n\ compile(s, "<test>", "exec") self.assertIn("unexpected EOF", str(cm.exception)) -var_annot_global: int # a global annotated is necessary for test_var_annot +var_annot_global: int # a global annotated is necessary for test_var_annot # custom namespace for testing __annotations__ @@ -225,6 +378,18 @@ class CNS: class GrammarTests(unittest.TestCase): + #from test.support import check_syntax_error, check_syntax_warning + check_syntax_error, check_syntax_warning = check_syntax_error, check_syntax_warning + + if not hasattr(unittest.TestCase, 'subTest'): + @contextlib.contextmanager + def subTest(self, source=None, case=None, **kwargs): + try: + yield + except Exception: + print(source or case) + raise + # single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE # XXX can't test in a script -- this rule is only used when interactive @@ -250,7 +415,7 @@ class GrammarTests(unittest.TestCase): my_lst[one()-1]: int = 5 self.assertEqual(my_lst, [5]) - @skip("Bug: global vs. local declarations do not currently raise an error") + @skip("Cython Bug: global vs. local declarations do not currently raise an error") def test_var_annot_syntax_errors(self): # parser pass check_syntax_error(self, "def f: int") @@ -271,7 +436,6 @@ class GrammarTests(unittest.TestCase): " global x\n" " x: int\n") - @skip("Class annotations not implemented") def test_var_annot_basic_semantics(self): # execution order with self.assertRaises(ZeroDivisionError): @@ -289,21 +453,21 @@ class GrammarTests(unittest.TestCase): def f_OK(): x: 1/0 f_OK() - - ### The following are compile time errors in Cython. - - #def fbad(): - # x: int - # print(x) - #with self.assertRaises(UnboundLocalError): - # fbad() - #def f2bad(): - # (no_such_global): int - # print(no_such_global) - #try: - # f2bad() - #except Exception as e: - # self.assertIs(type(e), NameError) + # Compile-time errors in Cython: + """ + def fbad(): + x: int + print(x) + with self.assertRaises(UnboundLocalError): + fbad() + def f2bad(): + (no_such_global): int + print(no_such_global) + try: + f2bad() + except Exception as e: + self.assertIs(type(e), NameError) + """ # class semantics class C: @@ -312,7 +476,8 @@ class GrammarTests(unittest.TestCase): z = 2 def __init__(self, x): self.x: int = x - self.assertEqual(C.__annotations__, {'_C__foo': int, 's': str}) + + self.assertEqual(C.__annotations__, {'_C__foo': 'int', 's': 'str'}) with self.assertRaises(NameError): class CBad: no_such_name_defined.attr: int = 0 @@ -321,7 +486,7 @@ class GrammarTests(unittest.TestCase): x: int x.y: list = [] - @skip("Class annotations not implemented") + @skip("Not currently supported: https://github.com/cython/cython/issues/3839") def test_var_annot_metaclass_semantics(self): class CMeta(type): @classmethod @@ -403,9 +568,23 @@ class GrammarTests(unittest.TestCase): exec('X: str', {}, CNS2()) self.assertEqual(nonloc_ns['__annotations__']['x'], str) + @skip("Depends on 3-args compiled exec()") + def test_var_annot_rhs(self): + ns = {} + exec('x: tuple = 1, 2', ns) + self.assertEqual(ns['x'], (1, 2)) + stmt = ('def f():\n' + ' x: int = yield') + exec(stmt, ns) + self.assertEqual(list(ns['f']()), [None]) + + ns = {"a": 1, 'b': (2, 3, 4), "c":5, "Tuple": typing.Tuple} + exec('x: Tuple[int, ...] = a,*b,c', ns) + self.assertEqual(ns['x'], (1, 2, 3, 4, 5)) + def test_funcdef(self): ### [decorators] 'def' NAME parameters ['->' test] ':' suite - ### decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE + ### decorator: '@' namedexpr_test NEWLINE ### decorators: decorator+ ### parameters: '(' [typedargslist] ')' ### typedargslist: ((tfpdef ['=' test] ',')* @@ -544,9 +723,10 @@ class GrammarTests(unittest.TestCase): pos2key2dict(1,2,k2=100,tokwarg1=100,tokwarg2=200) pos2key2dict(1,2,tokwarg1=100,tokwarg2=200, k2=100) - self.assertRaises(SyntaxError, eval, "def f(*): pass") - self.assertRaises(SyntaxError, eval, "def f(*,): pass") - self.assertRaises(SyntaxError, eval, "def f(*, **kwds): pass") + # FIXME: currently does not raise an error + #self.assertRaises(SyntaxError, eval, "def f(*): pass") + #self.assertRaises(SyntaxError, eval, "def f(*,): pass") + #self.assertRaises(SyntaxError, eval, "def f(*, **kwds): pass") # keyword arguments after *arglist def f(*args, **kwargs): @@ -566,39 +746,75 @@ class GrammarTests(unittest.TestCase): # argument annotation tests def f(x) -> list: pass - self.assertEqual(f.__annotations__, {'return': list}) + self.assertEqual(f.__annotations__, {'return': 'list'}) def f(x: int): pass - self.assertEqual(f.__annotations__, {'x': int}) + self.assertEqual(f.__annotations__, {'x': 'int'}) + def f(x: int, /): pass + self.assertEqual(f.__annotations__, {'x': 'int'}) + def f(x: int = 34, /): pass + self.assertEqual(f.__annotations__, {'x': 'int'}) def f(*x: str): pass - self.assertEqual(f.__annotations__, {'x': str}) + self.assertEqual(f.__annotations__, {'x': 'str'}) def f(**x: float): pass - self.assertEqual(f.__annotations__, {'x': float}) + self.assertEqual(f.__annotations__, {'x': 'float'}) def f(x, y: 1+2): pass - self.assertEqual(f.__annotations__, {'y': 3}) + self.assertEqual(f.__annotations__, {'y': '1 + 2'}) + def f(x, y: 1+2, /): pass + self.assertEqual(f.__annotations__, {'y': '1 + 2'}) def f(a, b: 1, c: 2, d): pass - self.assertEqual(f.__annotations__, {'b': 1, 'c': 2}) + self.assertEqual(f.__annotations__, {'b': '1', 'c': '2'}) + def f(a, b: 1, /, c: 2, d): pass + self.assertEqual(f.__annotations__, {'b': '1', 'c': '2'}) def f(a, b: 1, c: 2, d, e: 3 = 4, f=5, *g: 6): pass self.assertEqual(f.__annotations__, - {'b': 1, 'c': 2, 'e': 3, 'g': 6}) + {'b': '1', 'c': '2', 'e': '3', 'g': '6'}) def f(a, b: 1, c: 2, d, e: 3 = 4, f=5, *g: 6, h: 7, i=8, j: 9 = 10, **k: 11) -> 12: pass self.assertEqual(f.__annotations__, - {'b': 1, 'c': 2, 'e': 3, 'g': 6, 'h': 7, 'j': 9, - 'k': 11, 'return': 12}) + {'b': '1', 'c': '2', 'e': '3', 'g': '6', 'h': '7', 'j': '9', + 'k': '11', 'return': '12'}) + # FIXME: compile failure on positional-only argument declaration + """ + def f(a, b: 1, c: 2, d, e: 3 = 4, f: int = 5, /, *g: 6, h: 7, i=8, j: 9 = 10, + **k: 11) -> 12: pass + self.assertEqual(f.__annotations__, + {'b': 1, 'c': 2, 'e': 3, 'f': int, 'g': 6, 'h': 7, 'j': 9, + 'k': 11, 'return': 12}) + """ # Check for issue #20625 -- annotations mangling class Spam: def f(self, *, __kw: 1): pass class Ham(Spam): pass - self.assertEqual(Spam.f.__annotations__, {'_Spam__kw': 1}) - self.assertEqual(Ham.f.__annotations__, {'_Spam__kw': 1}) + # FIXME: not currently mangled + """ + self.assertEqual(Spam.f.__annotations__, {'_Spam__kw': '1'}) + self.assertEqual(Ham.f.__annotations__, {'_Spam__kw': '1'}) + """ # Check for SF Bug #1697248 - mixing decorators and a return annotation def null(x): return x @null def f(x) -> list: pass - self.assertEqual(f.__annotations__, {'return': list}) - - # test MAKE_CLOSURE with a variety of oparg's + self.assertEqual(f.__annotations__, {'return': 'list'}) + + # Test expressions as decorators (PEP 614): + # FIXME: implement PEP 614 + """ + @False or null + def f(x): pass + @d := null + def f(x): pass + @lambda f: null(f) + def f(x): pass + @[..., null, ...][1] + def f(x): pass + @null(null)(null) + def f(x): pass + @[null][0].__call__.__call__ + def f(x): pass + """ + + # test closures with a variety of opargs closure = 1 def f(): return closure def f(x=1): return closure @@ -626,7 +842,7 @@ class GrammarTests(unittest.TestCase): ### lambdef: 'lambda' [varargslist] ':' test l1 = lambda : 0 self.assertEqual(l1(), 0) - l2 = lambda : a[d] # XXX just testing the expression + l2 = lambda : a[d] # XXX just testing the expression l3 = lambda : [2 < x for x in [-1, 3, 0]] self.assertEqual(l3(), [0, 1, 0]) l4 = lambda x = lambda y = lambda z=1 : z : y() : x() @@ -703,11 +919,13 @@ class GrammarTests(unittest.TestCase): for case in cases: source = case.format(keyword) with self.subTest(source=source): - with self.assertRaisesRegex(SyntaxError, custom_msg): + #with self.assertRaisesRegex(SyntaxError, custom_msg): + with self.assertRaises(SyntaxError): exec(source) source = source.replace("foo", "(foo.)") with self.subTest(source=source): - with self.assertRaisesRegex(SyntaxError, "invalid syntax"): + #with self.assertRaisesRegex(SyntaxError, "invalid syntax"): + with self.assertRaises(SyntaxError): exec(source) def test_del_stmt(self): @@ -719,6 +937,24 @@ class GrammarTests(unittest.TestCase): del abc del x, y, (z, xyz) + x, y, z = "xyz" + del x + del y, + del (z) + del () + + a, b, c, d, e, f, g = "abcdefg" + del a, (b, c), (d, (e, f)) + + a, b, c, d, e, f, g = "abcdefg" + del a, [b, c], (d, [e, f]) + + abcd = list("abcd") + del abcd[1:2] + + # FIXME: currently fails to compile + #compile("del a, (b[0].c, (d.e, f.g[1:2])), [h.i.j], ()", "<testcase>", "exec") + def test_pass_stmt(self): # 'pass' pass @@ -848,6 +1084,59 @@ class GrammarTests(unittest.TestCase): break self.assertEqual(count, 0) + def test_continue_in_finally(self): + count = 0 + while count < 2: + count += 1 + try: + pass + finally: + continue + break + self.assertEqual(count, 2) + + count = 0 + while count < 2: + count += 1 + try: + break + finally: + continue + self.assertEqual(count, 2) + + count = 0 + while count < 2: + count += 1 + try: + 1/0 + finally: + continue + break + self.assertEqual(count, 2) + + for count in [0, 1]: + try: + pass + finally: + continue + break + self.assertEqual(count, 1) + + for count in [0, 1]: + try: + break + finally: + continue + self.assertEqual(count, 1) + + for count in [0, 1]: + try: + 1/0 + finally: + continue + break + self.assertEqual(count, 1) + def test_return_in_finally(self): def g1(): try: @@ -870,6 +1159,62 @@ class GrammarTests(unittest.TestCase): return 4 self.assertEqual(g3(), 4) + @skip("FIXME: currently crashes because the iterable is cleaned up on 'return', not on loop exit") + def test_break_in_finally_after_return(self): + # See issue #37830 + def g1(x): + for count in [0, 1]: + count2 = 0 + while count2 < 20: + count2 += 10 + try: + return count + count2 + finally: + if x: + break + return 'end', count, count2 + self.assertEqual(g1(False), 10) + self.assertEqual(g1(True), ('end', 1, 10)) + + def g2(x): + for count in [0, 1]: + for count2 in [10, 20]: + try: + return count + count2 + finally: + if x: + break + return 'end', count, count2 + self.assertEqual(g2(False), 10) + self.assertEqual(g2(True), ('end', 1, 10)) + + @skip("FIXME: currently crashes because the iterable is cleaned up on 'return', not on loop exit") + def test_continue_in_finally_after_return(self): + # See issue #37830 + def g1(x): + count = 0 + while count < 100: + count += 1 + try: + return count + finally: + if x: + continue + return 'end', count + self.assertEqual(g1(False), 1) + self.assertEqual(g1(True), ('end', 100)) + + def g2(x): + for count in [0, 1]: + try: + return count + finally: + if x: + continue + return 'end', count + self.assertEqual(g2(False), 0) + self.assertEqual(g2(True), ('end', 1)) + def test_yield(self): # Allowed as standalone statement def g(): yield 1 @@ -895,7 +1240,7 @@ class GrammarTests(unittest.TestCase): def g(): f((yield from ()), 1) # Do not require parenthesis for tuple unpacking def g(): rest = 4, 5, 6; yield 1, 2, 3, *rest - self.assertEquals(list(g()), [(1, 2, 3, 4, 5, 6)]) + self.assertEqual(list(g()), [(1, 2, 3, 4, 5, 6)]) check_syntax_error(self, "def g(): f(yield 1)") check_syntax_error(self, "def g(): f(yield 1, 1)") check_syntax_error(self, "def g(): f(yield from ())") @@ -907,23 +1252,15 @@ class GrammarTests(unittest.TestCase): check_syntax_error(self, "class foo:yield 1") check_syntax_error(self, "class foo:yield from ()") # Check annotation refleak on SyntaxError - check_syntax_error(self, "def g(a:(yield)): pass") + #check_syntax_error(self, "def g(a:(yield)): pass") # no longer a syntax error with PEP-563 - @skip("DeprecationWarning not implemented") + @skip("Not currently a syntax error") def test_yield_in_comprehensions(self): # Check yield in comprehensions def g(): [x for x in [(yield 1)]] def g(): [x for x in [(yield from ())]] - def check(code, warntext): - with self.assertWarnsRegex(DeprecationWarning, warntext): - compile(code, '<test string>', 'exec') - import warnings - with warnings.catch_warnings(): - warnings.filterwarnings('error', category=DeprecationWarning) - with self.assertRaisesRegex(SyntaxError, warntext): - compile(code, '<test string>', 'exec') - + check = self.check_syntax_error check("def g(): [(yield x) for x in ()]", "'yield' inside list comprehension") check("def g(): [x for x in () if not (yield x)]", @@ -1014,6 +1351,15 @@ class GrammarTests(unittest.TestCase): else: self.fail("AssertionError not raised by 'assert False'") + self.check_syntax_warning('assert(x, "msg")', + 'assertion is always true') + # FIXME: currently fails to compile + """ + with warnings.catch_warnings(): + warnings.simplefilter('error', SyntaxWarning) + compile('assert x, "msg"', '<testcase>', 'exec') + """ + ### compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | funcdef | classdef # Tested below @@ -1076,7 +1422,7 @@ class GrammarTests(unittest.TestCase): def test_try(self): ### try_stmt: 'try' ':' suite (except_clause ':' suite)+ ['else' ':' suite] ### | 'try' ':' suite 'finally' ':' suite - ### except_clause: 'except' [expr ['as' expr]] + ### except_clause: 'except' [expr ['as' NAME]] try: 1/0 except ZeroDivisionError: @@ -1094,6 +1440,9 @@ class GrammarTests(unittest.TestCase): except (EOFError, TypeError, ZeroDivisionError) as msg: pass try: pass finally: pass + with self.assertRaises(SyntaxError): + compile("try:\n pass\nexcept Exception as a.b:\n pass", "?", "exec") + compile("try:\n pass\nexcept Exception as a[b]:\n pass", "?", "exec") def test_suite(self): # simple_stmt | NEWLINE INDENT NEWLINE* (stmt NEWLINE*)+ DEDENT @@ -1132,11 +1481,116 @@ class GrammarTests(unittest.TestCase): if 1 > 1: pass if 1 <= 1: pass if 1 >= 1: pass - if 1 is 1: pass - if 1 is not 1: pass + if x is x: pass + if x is not x: pass if 1 in (): pass if 1 not in (): pass - if 1 < 1 > 1 == 1 >= 1 <= 1 != 1 in 1 not in 1 is 1 is not 1: pass + if 1 < 1 > 1 == 1 >= 1 <= 1 != 1 in 1 not in x is x is not x: pass + + def test_comparison_is_literal(self): + def check(test, msg='"is" with a literal'): + self.check_syntax_warning(test, msg) + + check('x is 1') + check('x is "thing"') + check('1 is x') + check('x is y is 1') + check('x is not 1', '"is not" with a literal') + + # FIXME: this fails to compile + """ + with warnings.catch_warnings(): + warnings.simplefilter('error', SyntaxWarning) + compile('x is None', '<testcase>', 'exec') + compile('x is False', '<testcase>', 'exec') + compile('x is True', '<testcase>', 'exec') + compile('x is ...', '<testcase>', 'exec') + """ + + def test_warn_missed_comma(self): + # FIXME: would be nice if this could actually raise a compile time warning as well + def check(test): + self.check_syntax_warning(test, msg) + + msg=r'is not callable; perhaps you missed a comma\?' + check('[(1, 2) (3, 4)]') + check('[(x, y) (3, 4)]') + check('[[1, 2] (3, 4)]') + check('[{1, 2} (3, 4)]') + check('[{1: 2} (3, 4)]') + check('[[i for i in range(5)] (3, 4)]') + check('[{i for i in range(5)} (3, 4)]') + check('[(i for i in range(5)) (3, 4)]') + check('[{i: i for i in range(5)} (3, 4)]') + check('[f"{x}" (3, 4)]') + check('[f"x={x}" (3, 4)]') + check('["abc" (3, 4)]') + check('[b"abc" (3, 4)]') + check('[123 (3, 4)]') + check('[12.3 (3, 4)]') + check('[12.3j (3, 4)]') + check('[None (3, 4)]') + check('[True (3, 4)]') + check('[... (3, 4)]') + + msg=r'is not subscriptable; perhaps you missed a comma\?' + check('[{1, 2} [i, j]]') + check('[{i for i in range(5)} [i, j]]') + check('[(i for i in range(5)) [i, j]]') + check('[(lambda x, y: x) [i, j]]') + check('[123 [i, j]]') + check('[12.3 [i, j]]') + check('[12.3j [i, j]]') + check('[None [i, j]]') + check('[True [i, j]]') + check('[... [i, j]]') + + msg=r'indices must be integers or slices, not tuple; perhaps you missed a comma\?' + check('[(1, 2) [i, j]]') + check('[(x, y) [i, j]]') + check('[[1, 2] [i, j]]') + check('[[i for i in range(5)] [i, j]]') + check('[f"{x}" [i, j]]') + check('[f"x={x}" [i, j]]') + check('["abc" [i, j]]') + check('[b"abc" [i, j]]') + + msg=r'indices must be integers or slices, not tuple;' + check('[[1, 2] [3, 4]]') + msg=r'indices must be integers or slices, not list;' + check('[[1, 2] [[3, 4]]]') + check('[[1, 2] [[i for i in range(5)]]]') + msg=r'indices must be integers or slices, not set;' + check('[[1, 2] [{3, 4}]]') + check('[[1, 2] [{i for i in range(5)}]]') + msg=r'indices must be integers or slices, not dict;' + check('[[1, 2] [{3: 4}]]') + check('[[1, 2] [{i: i for i in range(5)}]]') + msg=r'indices must be integers or slices, not generator;' + check('[[1, 2] [(i for i in range(5))]]') + msg=r'indices must be integers or slices, not function;' + check('[[1, 2] [(lambda x, y: x)]]') + msg=r'indices must be integers or slices, not str;' + check('[[1, 2] [f"{x}"]]') + check('[[1, 2] [f"x={x}"]]') + check('[[1, 2] ["abc"]]') + msg=r'indices must be integers or slices, not' + check('[[1, 2] [b"abc"]]') + check('[[1, 2] [12.3]]') + check('[[1, 2] [12.3j]]') + check('[[1, 2] [None]]') + check('[[1, 2] [...]]') + + """ + with warnings.catch_warnings(): + warnings.simplefilter('error', SyntaxWarning) + compile('[(lambda x, y: x) (3, 4)]', '<testcase>', 'exec') + compile('[[1, 2] [i]]', '<testcase>', 'exec') + compile('[[1, 2] [0]]', '<testcase>', 'exec') + compile('[[1, 2] [True]]', '<testcase>', 'exec') + compile('[[1, 2] [1:2]]', '<testcase>', 'exec') + compile('[{(1, 2): 3} [i, j]]', '<testcase>', 'exec') + """ def test_binary_mask_ops(self): x = 1 & 1 @@ -1244,13 +1698,30 @@ class GrammarTests(unittest.TestCase): def meth2(self, arg): pass def meth3(self, a1, a2): pass - # decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE + # decorator: '@' namedexpr_test NEWLINE # decorators: decorator+ # decorated: decorators (classdef | funcdef) def class_decorator(x): return x @class_decorator class G: pass + # Test expressions as decorators (PEP 614): + # FIXME: implement PEP 614 + """ + @False or class_decorator + class H: pass + @d := class_decorator + class I: pass + @lambda c: class_decorator(c) + class J: pass + @[..., class_decorator, ...][1] + class K: pass + @class_decorator(class_decorator)(class_decorator) + class L: pass + @[class_decorator][0].__call__.__call__ + class M: pass + """ + def test_dictcomps(self): # dictorsetmaker: ( (test ':' test (comp_for | # (',' test ':' test)* [','])) | @@ -1347,7 +1818,7 @@ class GrammarTests(unittest.TestCase): self.assertEqual(sum(b), sum([x for x in range(10)])) self.assertEqual(sum(x**2 for x in range(10)), sum([x**2 for x in range(10)])) - self.assertEqual(sum(x*x for x in range(10) if x%2), sum([x*x for x in range(10) if x%2])) + self.assertEqual(sum(x*x for x in range(10) if x % 2), sum([x*x for x in range(10) if x % 2])) self.assertEqual(sum(x for x in (y for y in range(10))), sum([x for x in range(10)])) self.assertEqual(sum(x for x in (y for y in (z for z in range(10)))), sum([x for x in range(10)])) self.assertEqual(sum(x for x in [y for y in (z for z in range(10))]), sum([x for x in range(10)])) @@ -1358,6 +1829,8 @@ class GrammarTests(unittest.TestCase): def test_comprehension_specials(self): # test for outmost iterable precomputation + # FIXME: https://github.com/cython/cython/issues/1159 + """ x = 10; g = (i for i in range(x)); x = 5 self.assertEqual(len(list(g)), 10) @@ -1365,6 +1838,7 @@ class GrammarTests(unittest.TestCase): x = 10; t = False; g = ((i,j) for i in range(x) if t for j in range(x)) x = 5; t = True; self.assertEqual([(i,j) for i in range(10) for j in range(5)], list(g)) + """ # Grammar allows multiple adjacent 'if's in listcomps and genexps, # even though it's silly. Make sure it works (ifelse broke this.) @@ -1395,11 +1869,75 @@ class GrammarTests(unittest.TestCase): with manager() as x, manager(): pass + if not use_old_parser(): + test_cases = [ + """if 1: + with ( + manager() + ): + pass + """, + """if 1: + with ( + manager() as x + ): + pass + """, + """if 1: + with ( + manager() as (x, y), + manager() as z, + ): + pass + """, + """if 1: + with ( + manager(), + manager() + ): + pass + """, + """if 1: + with ( + manager() as x, + manager() as y + ): + pass + """, + """if 1: + with ( + manager() as x, + manager() + ): + pass + """, + """if 1: + with ( + manager() as x, + manager() as y, + manager() as z, + ): + pass + """, + """if 1: + with ( + manager() as x, + manager() as y, + manager(), + ): + pass + """, + ] + for case in test_cases: + with self.subTest(case=case): + compile(case, "<string>", "exec") + + def test_if_else_expr(self): # Test ifelse expressions in various cases def _checkeval(msg, ret): "helper to check that evaluation of expressions is done correctly" - print(x) + print(msg) return ret # the next line is not allowed anymore @@ -1426,9 +1964,11 @@ class GrammarTests(unittest.TestCase): self.assertEqual(16 // (4 // 2), 8) self.assertEqual((16 // 4) // 2, 2) self.assertEqual(16 // 4 // 2, 2) - self.assertTrue(False is (2 is 3)) - self.assertFalse((False is 2) is 3) - self.assertFalse(False is 2 is 3) + x = 2 + y = 3 + self.assertTrue(False is (x is y)) + self.assertFalse((False is x) is y) + self.assertFalse(False is x is y) def test_matrix_mul(self): # This is not intended to be a comprehensive test, rather just to be few @@ -1516,54 +2056,5 @@ class GrammarTests(unittest.TestCase): foo().send(None) -### END OF COPY ### - -GrammarTests.assertRaisesRegex = lambda self, exc, msg: self.assertRaises(exc) - -if sys.version_info < (2, 7): - def assertRaises(self, exc_type, func=None, *args, **kwargs): - if func is not None: - return unittest.TestCase.assertRaises(self, exc_type, func, *args, **kwargs) - @contextlib.contextmanager - def assertRaisesCM(): - class Result(object): - exception = exc_type("unexpected EOF") # see usage above - try: - yield Result() - except exc_type: - self.assertTrue(True) - else: - self.assertTrue(False) - return assertRaisesCM() - GrammarTests.assertRaises = assertRaises - TokenTests.assertRaises = assertRaises - - -if not hasattr(unittest.TestCase, 'subTest'): - @contextlib.contextmanager - def subTest(self, source, **kwargs): - try: - yield - except Exception: - print(source) - raise - GrammarTests.subTest = subTest - - -if not hasattr(unittest.TestCase, 'assertIn'): - def assertIn(self, member, container, msg=None): - self.assertTrue(member in container, msg) - TokenTests.assertIn = assertIn - - -# FIXME: disabling some tests for real Cython bugs here -del GrammarTests.test_comprehension_specials # iterable pre-calculation in generator expression -del GrammarTests.test_funcdef # annotation mangling - -# this test is difficult to enable in Py2.6 -if sys.version_info < (2,7): - del GrammarTests.test_former_statements_refer_to_builtins - - if __name__ == '__main__': unittest.main() diff --git a/tests/run/test_subclassinit.py b/tests/run/test_subclassinit.py new file mode 100644 index 000000000..7661c6062 --- /dev/null +++ b/tests/run/test_subclassinit.py @@ -0,0 +1,316 @@ +# mode: run +# tag: pure3.6 +# cython: language_level=3str + +import sys +HAS_NATIVE_SUPPORT = sys.version_info >= (3, 6) +IS_PY2 = sys.version_info[0] == 2 + +import re +import types +import unittest + +ZERO = 0 + +skip_if_not_native = unittest.skipIf(not HAS_NATIVE_SUPPORT, "currently requires Python 3.6+") + + +class Test(unittest.TestCase): + if not hasattr(unittest.TestCase, 'assertRegex'): + def assertRegex(self, value, regex): + self.assertTrue(re.search(regex, str(value)), + "'%s' did not match '%s'" % (value, regex)) + + if not hasattr(unittest.TestCase, 'assertCountEqual'): + def assertCountEqual(self, first, second): + self.assertEqual(set(first), set(second)) + self.assertEqual(len(first), len(second)) + + def test_init_subclass(self): + class A: + initialized = False + + def __init_subclass__(cls): + if HAS_NATIVE_SUPPORT: + super().__init_subclass__() + cls.initialized = True + + class B(A): + pass + + self.assertFalse(A.initialized) + self.assertTrue(B.initialized) + + def test_init_subclass_dict(self): + class A(dict): + initialized = False + + def __init_subclass__(cls): + if HAS_NATIVE_SUPPORT: + super().__init_subclass__() + cls.initialized = True + + class B(A): + pass + + self.assertFalse(A.initialized) + self.assertTrue(B.initialized) + + def test_init_subclass_kwargs(self): + class A: + def __init_subclass__(cls, **kwargs): + cls.kwargs = kwargs + + class B(A, x=3): + pass + + self.assertEqual(B.kwargs, dict(x=3)) + + def test_init_subclass_error(self): + class A: + def __init_subclass__(cls): + raise RuntimeError + + with self.assertRaises(RuntimeError): + class B(A): + pass + + def test_init_subclass_wrong(self): + class A: + def __init_subclass__(cls, whatever): + pass + + with self.assertRaises(TypeError): + class B(A): + pass + + def test_init_subclass_skipped(self): + class BaseWithInit: + def __init_subclass__(cls, **kwargs): + if HAS_NATIVE_SUPPORT: + super().__init_subclass__(**kwargs) + cls.initialized = cls + + class BaseWithoutInit(BaseWithInit): + pass + + class A(BaseWithoutInit): + pass + + self.assertIs(A.initialized, A) + self.assertIs(BaseWithoutInit.initialized, BaseWithoutInit) + + def test_init_subclass_diamond(self): + class Base: + def __init_subclass__(cls, **kwargs): + if HAS_NATIVE_SUPPORT: + super().__init_subclass__(**kwargs) + cls.calls = [] + + class Left(Base): + pass + + class Middle: + def __init_subclass__(cls, middle, **kwargs): + super().__init_subclass__(**kwargs) + cls.calls += [middle] + + class Right(Base): + def __init_subclass__(cls, right="right", **kwargs): + super().__init_subclass__(**kwargs) + cls.calls += [right] + + class A(Left, Middle, Right, middle="middle"): + pass + + self.assertEqual(A.calls, ["right", "middle"]) + self.assertEqual(Left.calls, []) + self.assertEqual(Right.calls, []) + + def test_set_name(self): + class Descriptor: + def __set_name__(self, owner, name): + self.owner = owner + self.name = name + + class A: + d = Descriptor() + + self.assertEqual(A.d.name, "d") + self.assertIs(A.d.owner, A) + + @skip_if_not_native + def test_set_name_metaclass(self): + class Meta(type): + def __new__(cls, name, bases, ns): + ret = super().__new__(cls, name, bases, ns) + self.assertEqual(ret.d.name, "d") + self.assertIs(ret.d.owner, ret) + return 0 + + class Descriptor: + def __set_name__(self, owner, name): + self.owner = owner + self.name = name + + class A(metaclass=Meta): + d = Descriptor() + self.assertEqual(A, 0) + + def test_set_name_error(self): + class Descriptor: + def __set_name__(self, owner, name): + 1 / ZERO + + with self.assertRaises(RuntimeError) as cm: + class NotGoingToWork: + attr = Descriptor() + + exc = cm.exception + self.assertRegex(str(exc), r'\bNotGoingToWork\b') + self.assertRegex(str(exc), r'\battr\b') + self.assertRegex(str(exc), r'\bDescriptor\b') + if HAS_NATIVE_SUPPORT: + self.assertIsInstance(exc.__cause__, ZeroDivisionError) + + def test_set_name_wrong(self): + class Descriptor: + def __set_name__(self): + pass + + with self.assertRaises(RuntimeError) as cm: + class NotGoingToWork: + attr = Descriptor() + + exc = cm.exception + self.assertRegex(str(exc), r'\bNotGoingToWork\b') + self.assertRegex(str(exc), r'\battr\b') + self.assertRegex(str(exc), r'\bDescriptor\b') + if HAS_NATIVE_SUPPORT: + self.assertIsInstance(exc.__cause__, TypeError) + + def test_set_name_lookup(self): + resolved = [] + class NonDescriptor: + def __getattr__(self, name): + resolved.append(name) + + class A: + d = NonDescriptor() + + self.assertNotIn('__set_name__', resolved, + '__set_name__ is looked up in instance dict') + + @skip_if_not_native + def test_set_name_init_subclass(self): + class Descriptor: + def __set_name__(self, owner, name): + self.owner = owner + self.name = name + + class Meta(type): + def __new__(cls, name, bases, ns): + self = super().__new__(cls, name, bases, ns) + self.meta_owner = self.owner + self.meta_name = self.name + return self + + class A: + def __init_subclass__(cls): + cls.owner = cls.d.owner + cls.name = cls.d.name + + class B(A, metaclass=Meta): + d = Descriptor() + + self.assertIs(B.owner, B) + self.assertEqual(B.name, 'd') + self.assertIs(B.meta_owner, B) + self.assertEqual(B.name, 'd') + + def test_set_name_modifying_dict(self): + notified = [] + class Descriptor: + def __set_name__(self, owner, name): + setattr(owner, name + 'x', None) + notified.append(name) + + class A: + a = Descriptor() + b = Descriptor() + c = Descriptor() + d = Descriptor() + e = Descriptor() + + self.assertCountEqual(notified, ['a', 'b', 'c', 'd', 'e']) + + def test_errors(self): + class MyMeta(type): + pass + + with self.assertRaises(TypeError): + class MyClass(metaclass=MyMeta, otherarg=1): + pass + + if not IS_PY2: + with self.assertRaises(TypeError): + types.new_class("MyClass", (object,), + dict(metaclass=MyMeta, otherarg=1)) + types.prepare_class("MyClass", (object,), + dict(metaclass=MyMeta, otherarg=1)) + + class MyMeta(type): + def __init__(self, name, bases, namespace, otherarg): + super().__init__(name, bases, namespace) + + with self.assertRaises(TypeError): + class MyClass(metaclass=MyMeta, otherarg=1): + pass + + class MyMeta(type): + def __new__(cls, name, bases, namespace, otherarg): + return super().__new__(cls, name, bases, namespace) + + def __init__(self, name, bases, namespace, otherarg): + super().__init__(name, bases, namespace) + self.otherarg = otherarg + + class MyClass(metaclass=MyMeta, otherarg=1): + pass + + self.assertEqual(MyClass.otherarg, 1) + + @skip_if_not_native + def test_errors_changed_pep487(self): + # These tests failed before Python 3.6, PEP 487 + class MyMeta(type): + def __new__(cls, name, bases, namespace): + return super().__new__(cls, name=name, bases=bases, + dict=namespace) + + with self.assertRaises(TypeError): + class MyClass(metaclass=MyMeta): + pass + + class MyMeta(type): + def __new__(cls, name, bases, namespace, otherarg): + self = super().__new__(cls, name, bases, namespace) + self.otherarg = otherarg + return self + + class MyClass(metaclass=MyMeta, otherarg=1): + pass + + self.assertEqual(MyClass.otherarg, 1) + + def test_type(self): + t = type('NewClass', (object,), {}) + self.assertIsInstance(t, type) + self.assertEqual(t.__name__, 'NewClass') + + with self.assertRaises(TypeError): + type(name='NewClass', bases=(object,), dict={}) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/run/test_unicode.pyx b/tests/run/test_unicode.pyx index 4616acf2c..2386b5124 100644 --- a/tests/run/test_unicode.pyx +++ b/tests/run/test_unicode.pyx @@ -846,7 +846,6 @@ class UnicodeTest(CommonTest, self.assertEqual('finnish'.capitalize(), 'FInnish') else: self.assertEqual('finnish'.capitalize(), 'Finnish') - self.assertEqual('A\u0345\u03a3'.capitalize(), 'A\u0345\u03c2') def test_title(self): diff --git a/tests/run/time_pxd.pyx b/tests/run/time_pxd.pyx new file mode 100644 index 000000000..91e6e8d24 --- /dev/null +++ b/tests/run/time_pxd.pyx @@ -0,0 +1,59 @@ +# mode: run +# tag: py3only,pytime + +import time + +from cpython cimport time as ctime + + +def test_time(): + """ + >>> tic1, tic2, tic3 = test_time() + >>> tic1 <= tic3 # sanity check + True + >>> tic1 <= tic2 + True + >>> tic2 <= tic3 + True + """ + # check that ctime.time() matches time.time() to within call-time tolerance + tic1 = time.time() + tic2 = ctime.time() + tic3 = time.time() + + return tic1, tic2, tic3 + + +def test_localtime(): + """ + >>> ltp, ltc = test_localtime() + >>> ltp.tm_year == ltc['tm_year'] or (ltp.tm_year, ltc['tm_year']) + True + >>> ltp.tm_mon == ltc['tm_mon'] or (ltp.tm_mon, ltc['tm_mon']) + True + >>> ltp.tm_mday == ltc['tm_mday'] or (ltp.tm_mday, ltc['tm_mday']) + True + >>> ltp.tm_hour == ltc['tm_hour'] or (ltp.tm_hour, ltc['tm_hour']) + True + >>> ltp.tm_min == ltc['tm_min'] or (ltp.tm_min, ltc['tm_min']) + True + >>> ltp.tm_sec == ltc['tm_sec'] or (ltp.tm_sec, ltc['tm_sec']) + True + >>> ltp.tm_wday == ltc['tm_wday'] or (ltp.tm_wday, ltc['tm_wday']) + True + >>> ltp.tm_yday == ltc['tm_yday'] or (ltp.tm_yday, ltc['tm_yday']) + True + >>> ltp.tm_isdst == ltc['tm_isdst'] or (ltp.tm_isdst, ltc['tm_isdst']) + True + """ + ltp = time.localtime() + ltc = ctime.localtime() + + if ltp.tm_sec != ltc.tm_sec: + # If the time.localtime call is just before the end of a second and the + # ctime.localtime call is just after the beginning of the next second, + # re-call. This should not occur twice in a row. + ltp = time.localtime() + ltc = ctime.localtime() + + return ltp, ltc diff --git a/tests/run/tp_new.pyx b/tests/run/tp_new.pyx index c25a84289..45b52259f 100644 --- a/tests/run/tp_new.pyx +++ b/tests/run/tp_new.pyx @@ -1,4 +1,6 @@ -# ticket: 808 +# mode: run +# tag: exttype, tpnew +# ticket: t808 cimport cython diff --git a/tests/run/tp_new_T454.pyx b/tests/run/tp_new_T454.pyx index 893ef9b1b..d3401442c 100644 --- a/tests/run/tp_new_T454.pyx +++ b/tests/run/tp_new_T454.pyx @@ -1,4 +1,4 @@ -# ticket: 454 +# ticket: t454 cimport cython diff --git a/tests/run/trashcan.pyx b/tests/run/trashcan.pyx new file mode 100644 index 000000000..29331fdf6 --- /dev/null +++ b/tests/run/trashcan.pyx @@ -0,0 +1,148 @@ +# mode: run + +cimport cython + + +# Count number of times an object was deallocated twice. This should remain 0. +cdef int double_deallocations = 0 +def assert_no_double_deallocations(): + global double_deallocations + err = double_deallocations + double_deallocations = 0 + assert not err + + +# Compute x = f(f(f(...(None)...))) nested n times and throw away the result. +# The real test happens when exiting this function: then a big recursive +# deallocation of x happens. We are testing two things in the tests below: +# that Python does not crash and that no double deallocation happens. +# See also https://github.com/python/cpython/pull/11841 +def recursion_test(f, int n=2**20): + x = None + cdef int i + for i in range(n): + x = f(x) + + +@cython.trashcan(True) +cdef class Recurse: + """ + >>> recursion_test(Recurse) + >>> assert_no_double_deallocations() + """ + cdef public attr + cdef int deallocated + + def __cinit__(self, x): + self.attr = x + + def __dealloc__(self): + # Check that we're not being deallocated twice + global double_deallocations + double_deallocations += self.deallocated + self.deallocated = 1 + + +cdef class RecurseSub(Recurse): + """ + >>> recursion_test(RecurseSub) + >>> assert_no_double_deallocations() + """ + cdef int subdeallocated + + def __dealloc__(self): + # Check that we're not being deallocated twice + global double_deallocations + double_deallocations += self.subdeallocated + self.subdeallocated = 1 + + +@cython.freelist(4) +@cython.trashcan(True) +cdef class RecurseFreelist: + """ + >>> recursion_test(RecurseFreelist) + >>> recursion_test(RecurseFreelist, 1000) + >>> assert_no_double_deallocations() + """ + cdef public attr + cdef int deallocated + + def __cinit__(self, x): + self.attr = x + + def __dealloc__(self): + # Check that we're not being deallocated twice + global double_deallocations + double_deallocations += self.deallocated + self.deallocated = 1 + + +# Subclass of list => uses trashcan by default +# As long as https://github.com/python/cpython/pull/11841 is not fixed, +# this does lead to double deallocations, so we skip that check. +cdef class RecurseList(list): + """ + >>> RecurseList(42) + [42] + >>> recursion_test(RecurseList) + """ + def __init__(self, x): + super().__init__((x,)) + + +# Some tests where the trashcan is NOT used. When the trashcan is not used +# in a big recursive deallocation, the __dealloc__s of the base classes are +# only run after the __dealloc__s of the subclasses. +# We use this to detect trashcan usage. +cdef int base_deallocated = 0 +cdef int trashcan_used = 0 +def assert_no_trashcan_used(): + global base_deallocated, trashcan_used + err = trashcan_used + trashcan_used = base_deallocated = 0 + assert not err + + +cdef class Base: + def __dealloc__(self): + global base_deallocated + base_deallocated = 1 + + +# Trashcan disabled by default +cdef class Sub1(Base): + """ + >>> recursion_test(Sub1, 100) + >>> assert_no_trashcan_used() + """ + cdef public attr + + def __cinit__(self, x): + self.attr = x + + def __dealloc__(self): + global base_deallocated, trashcan_used + trashcan_used += base_deallocated + + +@cython.trashcan(True) +cdef class Middle(Base): + cdef public foo + + +# Trashcan disabled explicitly +@cython.trashcan(False) +cdef class Sub2(Middle): + """ + >>> recursion_test(Sub2, 1000) + >>> assert_no_trashcan_used() + """ + cdef public attr + + def __cinit__(self, x): + self.attr = x + + def __dealloc__(self): + global base_deallocated, trashcan_used + trashcan_used += base_deallocated diff --git a/tests/run/tupleunpack_T298.pyx b/tests/run/tupleunpack_T298.pyx index 7763b1fc7..7e3150243 100644 --- a/tests/run/tupleunpack_T298.pyx +++ b/tests/run/tupleunpack_T298.pyx @@ -1,4 +1,4 @@ -# ticket: 298 +# ticket: t298 """ >>> func() diff --git a/tests/run/tupleunpack_T712.pyx b/tests/run/tupleunpack_T712.pyx index 24d750849..73b3c1446 100644 --- a/tests/run/tupleunpack_T712.pyx +++ b/tests/run/tupleunpack_T712.pyx @@ -1,5 +1,5 @@ # mode: run -# ticket: 712 +# ticket: t712 def single_from_string(): """ diff --git a/tests/run/type_inference.pyx b/tests/run/type_inference.pyx index d2bef6431..3504c8525 100644 --- a/tests/run/type_inference.pyx +++ b/tests/run/type_inference.pyx @@ -276,6 +276,9 @@ def cascade(): assert typeof(e) == "double" def cascaded_assignment(): + """ + >>> cascaded_assignment() + """ a = b = c = d = 1.0 assert typeof(a) == "double" assert typeof(b) == "double" @@ -284,6 +287,36 @@ def cascaded_assignment(): e = a + b + c + d assert typeof(e) == "double" + +def unpacking(x): + """ + >>> unpacking(0) + """ + a, b, c, (d, e) = x, 1, 2.0, [3, [5, 6]] + assert typeof(a) == "Python object", typeof(a) + assert typeof(b) == "long", typeof(b) + assert typeof(c) == "double", typeof(c) + assert typeof(d) == "long", typeof(d) + assert typeof(e) == "list object", typeof(e) + + +def star_unpacking(*x): + """ + >>> star_unpacking(1, 2) + """ + a, b = x + c, *d = x + *e, f = x + *g, g = x # re-assignment + assert typeof(a) == "Python object", typeof(a) + assert typeof(b) == "Python object", typeof(b) + assert typeof(c) == "Python object", typeof(c) + assert typeof(d) == "list object", typeof(d) + assert typeof(e) == "list object", typeof(e) + assert typeof(f) == "Python object", typeof(f) + assert typeof(g) == "Python object", typeof(f) + + def increment(): """ >>> increment() diff --git a/tests/run/type_inference_T768.pyx b/tests/run/type_inference_T768.pyx index cb7b9eccb..8f1f2e61e 100644 --- a/tests/run/type_inference_T768.pyx +++ b/tests/run/type_inference_T768.pyx @@ -1,5 +1,5 @@ # mode: run -# ticket: 768 +# ticket: t768 from cython cimport typeof def type_inference_del_int(): diff --git a/tests/run/type_inference_T768_cpp.pyx b/tests/run/type_inference_T768_cpp.pyx index 56fad4dab..295351914 100644 --- a/tests/run/type_inference_T768_cpp.pyx +++ b/tests/run/type_inference_T768_cpp.pyx @@ -1,6 +1,6 @@ # mode: run # tag: cpp -# ticket: 768 +# ticket: t768 from cython cimport typeof cdef extern from "shapes.h" namespace "shapes": diff --git a/tests/run/type_slots_int_long_T287.pyx b/tests/run/type_slots_int_long_T287.pyx index 2426225b5..7b66125b9 100644 --- a/tests/run/type_slots_int_long_T287.pyx +++ b/tests/run/type_slots_int_long_T287.pyx @@ -1,4 +1,4 @@ -# ticket: 287 +# ticket: t287 __doc__ = u""" >>> print( "%d" % Int() ) diff --git a/tests/run/typeddefaultargT373.pyx b/tests/run/typeddefaultargT373.pyx index 98ac85c3a..d817c97c7 100644 --- a/tests/run/typeddefaultargT373.pyx +++ b/tests/run/typeddefaultargT373.pyx @@ -1,4 +1,4 @@ -# ticket: 373 +# ticket: t373 import math diff --git a/tests/run/typedfieldbug_T303.pyx b/tests/run/typedfieldbug_T303.pyx index 4298a1cce..e1790c574 100644 --- a/tests/run/typedfieldbug_T303.pyx +++ b/tests/run/typedfieldbug_T303.pyx @@ -1,5 +1,5 @@ # mode: run -# ticket: 303 +# ticket: t303 __doc__ = """ >>> try: readonly() diff --git a/tests/run/typetest_T417.pyx b/tests/run/typetest_T417.pyx index 7d7b24902..622b35416 100644 --- a/tests/run/typetest_T417.pyx +++ b/tests/run/typetest_T417.pyx @@ -1,4 +1,4 @@ -# ticket: 417 +# ticket: t417 #cython: autotestdict=True cdef class Foo: diff --git a/tests/run/unicode_formatting.pyx b/tests/run/unicode_formatting.pyx index fa51f8b94..cddcdf136 100644 --- a/tests/run/unicode_formatting.pyx +++ b/tests/run/unicode_formatting.pyx @@ -38,21 +38,21 @@ def mix_format(a, int b, list c): class PySubtype(unicode): def __rmod__(self, other): - return f'PyRMOD({other})' + return f'PyRMOD({self}, {other})' cdef class ExtSubtype(unicode): - def __mod__(one, other): - return f'ExtMOD({one}, {other})' + def __rmod__(self, other): + return f'ExtRMOD({self}, {other})' def subtypes(): """ >>> py, ext = subtypes() >>> print(py) - PyRMOD(-%s-) + PyRMOD(PySub, -%s-) >>> print(ext) - ExtMOD(-%s-, ExtSub) + ExtRMOD(ExtSub, -%s-) """ return [ '-%s-' % PySubtype("PySub"), diff --git a/tests/run/unicode_identifiers.pxd b/tests/run/unicode_identifiers.pxd new file mode 100644 index 000000000..06469e6c7 --- /dev/null +++ b/tests/run/unicode_identifiers.pxd @@ -0,0 +1,10 @@ +# -*- coding: utf-8 -*- +# cython: language_level=3 + +cdef Fα1() +cdef class Γναμε2: + cdef public int α + cdef boring_cdef(self) + cdef εxciting_cdef(self) + cpdef boring_cpdef(self) + cpdef εxciting_cpdef(self) diff --git a/tests/run/unicode_identifiers.pyx b/tests/run/unicode_identifiers.pyx new file mode 100644 index 000000000..767e3283e --- /dev/null +++ b/tests/run/unicode_identifiers.pyx @@ -0,0 +1,243 @@ +# -*- coding: utf-8 -*- +# mode: run +# tag: pep3131, traceback + +# cython: language_level=3 + +# Code with unicode identifiers can be compiled with Cython running either Python 2 or 3. +# However Python access to unicode identifiers is only possible in Python 3. In Python 2 +# it's only really safe to use the unicode identifiers for purely Cython interfaces +# (although this isn't enforced...). Therefore the majority of the doctests are +# Python3 only and only a limited set are run in Python2. +# This is controlled by putting the Python3 only tests in the module __doc__ attribute +# Most of the individual function and class docstrings are only present as a compile test + +cimport cython + +import sys + + +if sys.version_info[0] > 2: + __doc__ = u""" + >>> f()() + 2 + >>> f().__name__ + 'nεsted' + + The test is mainly to see if the traceback is generated correctly + >>> print_traceback_name() + unicode_identifiers.Fα1 + + Just check that a cpdef function is callable + >>> Fα3() + 1 + + >>> Γναμε2.ναμε3 + 1 + >>> x = Γναμε2() + >>> print(x.α) + 100 + >>> x.α = 200 + >>> print(x.α) + 200 + + >>> B().Ƒ() + >>> C().Ƒ() + + Test generation of locals() + >>> sorted(Γναμε2().boring_function(1,2).keys()) + ['self', 'somevalue', 'x', 'ναμε5', 'ναμε6'] + + >>> Γναμε2().boring_cpdef() - Γναμε2().εxciting_cpdef() + 0 + >>> function_taking_fancy_argument(Γναμε2()).ναμε3 + 1 + >>> metho_function_taking_fancy_argument(Γναμε2()).ναμε3 + 1 + >>> NormalClassΓΓ().ναμε + 10 + >>> NormalClassΓΓ().εxciting_function(None).__qualname__ + 'NormalClassΓΓ.εxciting_function.<locals>.nestεd' + + Do kwargs work? + >>> unicode_kwarg(αrγ=5) + 5 + >>> unicode_kwarg_from_cy() + 1 + + Normalization of attributes + (The cdef class version is testable in Python 2 too) + >>> NormalizeAttrPy().get() + 5 + """ +else: + __doc__ = "" + +global_ναμε1 = None +cdef double global_ναμε2 = 1.2 + +def f(): + """docstring""" + ναμε2 = 2 + def nεsted(): + return ναμε2 + return nεsted + +# Ƒ is notably awkward because its punycode starts with "2" causing +# C compile errors. Therefore try a few different variations... +cdef class A: + cdef int ναμε + def __init__(self): + self.ναμε = 1 + cdef Ƒ(self): + return self.ναμε == 1 + def regular_function(self): + """ + Can use unicode cdef functions and (private) attributes internally + >>> A().regular_function() + True + """ + return self.Ƒ() +cdef class B: + cpdef Ƒ(self): + pass +cdef class C: + def Ƒ(self): + pass +cdef class D: + cdef int Ƒ + +def regular_function(): + """ + Unicode names can be used internally on python2 + >>> regular_function() + 10 + """ + cdef int variableƑ = 5 + ναμε2 = 2 + return variableƑ*ναμε2 + +cdef Fα1(): + """docstring""" + ναμε2 = 2 + raise RuntimeError() # forces generation of a traceback + +def print_traceback_name(): + try: + Fα1() + except RuntimeError as e: + import traceback + # get the name of one level up in the traceback + print(traceback.extract_tb(e.__traceback__,2)[1][2]) + + +def Fα2(): + """docstring""" + def nested_normal(): + """docstring""" + pass + def nεstεd_uni(): + """docstring""" + pass + return nested_normal, nεstεd_uni + +cpdef Fα3(): + """docstring""" + return 1 + +cdef class Γναμε2: + """ + docstring + """ + ναμε3 = 1 + + def __init__(self): + self.α = 100 + def boring_function(self,x,ναμε5): + """docstring""" + ναμε6 = ναμε5 + somevalue = global_ναμε1 == self.ναμε3 + return locals() + def εxciting_function(self,y): + """docstring""" + def nestεd(): + pass + return nestεd + + cdef boring_cdef(self): + """docstring""" + pass + cdef εxciting_cdef(self): + """docstring""" + pass + + cpdef boring_cpdef(self): + """docstring""" + return 2 + cpdef εxciting_cpdef(self): + """docstring""" + return 2 + +cdef class Derived(Γναμε2): + pass + +cdef Γναμε2 global_ναμε3 = Γναμε2() + + +@cython.always_allow_keywords(False) # METH_O signature +def metho_function_taking_fancy_argument(Γναμε2 αrγ): + return αrγ + +@cython.always_allow_keywords(True) +def function_taking_fancy_argument(Γναμε2 αrγ): + return αrγ + + +class NormalClassΓΓ(Γναμε2): + """ + docstring + """ + def __init__(self): + self.ναμε = 10 + + def boring_function(self,x,ναμε5): + """docstring""" + ναμε6 = ναμε5 + somevalue = global_ναμε1 == self.ναμε3 + return locals() + def εxciting_function(self,y): + """docstring""" + def nestεd(): + pass + return nestεd + +def unicode_kwarg(*, αrγ): + return αrγ + +def unicode_kwarg_from_cy(): + return unicode_kwarg(αrγ=1) + +class NormalizeAttrPy: + """Python normalizes identifier names before they are used; + therefore fi and fi should access the same attribute""" + def __init__(self): + self.fi = 5 # note unicode ligature symbol + def get(self): + return self.fi + +cdef class NormalizeAttrCdef: + """Python normalizes identifier names before they are used; + therefore fi and fi should access the same attribute + >>> NormalizeAttrCdef().get() + 5 + """ + cdef int fi # note unicode ligature symbol + def __init__(self): + self.fi = 5 + def get(self): + return self.fi + +if sys.version_info[0]<=2: + # These symbols are causing problems for doctest + del NormalClassΓΓ + del globals()[u'Γναμε2'.encode('utf-8')] diff --git a/tests/run/unicode_identifiers_import.pyx b/tests/run/unicode_identifiers_import.pyx new file mode 100644 index 000000000..17e091697 --- /dev/null +++ b/tests/run/unicode_identifiers_import.pyx @@ -0,0 +1,31 @@ +# -*- coding: utf-8 -*- +# cython: language_level = 3 +# mode: compile +# tag: pep3131 + +# compile only test since there's no way to get +# it to import another test module at runtime + +# this test looks at [c]importing unicode stuff +from unicode_identifiers cimport Fα1, Γναμε2 +cimport unicode_identifiers +from unicode_identifiers cimport Γναμε2 as Γναμε3 + +from unicode_identifiers import NormalClassΓΓ +from unicode_identifiers import NormalClassΓΓ as NörmalCläss + + +cdef class C(unicode_identifiers.Γναμε2): + pass + +cdef class D(Γναμε2): + pass + +cdef class E(Γναμε3): + pass + +def f(): + Fα1() + unicode_identifiers.Fα1() + + diff --git a/tests/run/unicode_identifiers_normalization.srctree b/tests/run/unicode_identifiers_normalization.srctree new file mode 100644 index 000000000..764384455 --- /dev/null +++ b/tests/run/unicode_identifiers_normalization.srctree @@ -0,0 +1,83 @@ +# -*- coding: utf-8 -*- +# mode: run +# tag: pure3.0, pep3131 + +PYTHON build_tests.py +# show behaviour in Python mode +PYTHON -m doctest test0.py +PYTHON -m doctest test1.py +PYTHON -m doctest test2.py + +PYTHON setup.py build_ext --inplace +# test in Cython mode +PYTHON -c "import doctest; import test0 as m; exit(doctest.testmod(m)[0])" +PYTHON -c "import doctest; import test1 as m; exit(doctest.testmod(m)[0])" +PYTHON -c "import doctest; import test2 as m; exit(doctest.testmod(m)[0])" + +########## setup.py ######### + +from Cython.Build.Dependencies import cythonize +from distutils.core import setup + +setup( + ext_modules = cythonize("test*.py"), +) + +######### build_tests.py ######## +# -*- coding: utf-8 -*- +from __future__ import unicode_literals +import sys +import unicodedata + +# a few pairs of unicode strings that should be equivalent after normalization +string_pairs = [("fi", "fi"), # ligature and two letters + ("a\u0301", '\u00e1'), # a with acute accent with combining character or as 1 character + ("α\u0334\u0362", "α\u0362\u0334") # alpha with a pair of combining characters + # in a different order. No single character to normalize to + ] + +# Show that the pairs genuinely aren't equal before normalization +for sp in string_pairs: + assert sp[0] != sp[1] + assert unicodedata.normalize('NFKC', sp[0]) == unicodedata.normalize('NFKC', sp[1]) + +# some code that accesses the identifiers through the two different names +# contains doctests +example_code = [ +""" +class C: + ''' + >>> C().get() + True + ''' + def __init__(self): + self.{0} = True + def get(self): + return self.{1} +""", """ +def pass_through({0}): + ''' + >>> pass_through(True) + True + ''' + return {1} +""", """ +import cython +{0} = True +def test(): + ''' + >>> test() + True + ''' + return {1} +"""] + +from io import open + +for idx, (code, strings) in enumerate(zip(example_code, string_pairs)): + with open("test{0}.py".format(idx), "w", encoding="utf8") as f: + code = code.format(*strings) + f.write("# -*- coding: utf-8 -*-\n") + # The code isn't Py2 compatible. Only write actual code in Py3+. + if sys.version_info[0] > 2: + f.write(code) diff --git a/tests/run/unicode_imports.srctree b/tests/run/unicode_imports.srctree new file mode 100644 index 000000000..262f4949a --- /dev/null +++ b/tests/run/unicode_imports.srctree @@ -0,0 +1,105 @@ +# -*- coding: utf-8 -*- +# tag: py3, pep489 + +PYTHON setup.py build_ext --inplace +PYTHON -m mydoctest + +########### mydoctest.py ####### + +import sys +if sys.version_info < (3, 5): + # The module is only Cythonized and not build for these versions + # so don't run the tests + exit() + +import doctest +import from_py +val = doctest.testmod(from_py)[0] +import from_cy +val += doctest.testmod(from_cy)[0] + +exit(val) + +########### setup.py ######## + +# -*- coding: utf-8 -*- + +from __future__ import unicode_literals + +import sys +from Cython.Build import cythonize + +files = ["mymoð.pyx", "from_cy.pyx"] + + +# For Python 2 and Python <= 3.4 just run pyx->c; +# don't compile the C file +modules = cythonize(files) + +if sys.version_info >= (3, 5): + from distutils.core import setup + + setup( + ext_modules = modules + ) + +############ mymoð.pyx ######### + +def f(): + return True + +cdef public api void cdef_func(): + pass + +############ pxd_moð.pxd ########## + +cdef struct S: + int x + +cdef public api void cdef_func() # just to test generation of headers + +############ from_py.py ######### + +# -*- coding: utf-8 -*- + +import mymoð +from mymoð import f + +__doc__ = """ +>>> mymoð.f() +True +>>> f() +True +""" + +######### from_cy.pyx ########## + +# -*- coding: utf-8 -*- + +import mymoð + +from mymoð import f + +cimport pxd_moð +from pxd_moð cimport S + + +def test_imported(): + """ + >>> test_imported() + True + """ + return mymoð.f() and f() # True and True + + +def test_cimported(): + """ + >>> test_cimported() + 3 + """ + cdef pxd_moð.S v1 + v1.x = 1 + cdef S v2 + v2.x = 2 + return v1.x + v2.x + diff --git a/tests/run/unicodeliterals.pyx b/tests/run/unicodeliterals.pyx index bcc34d213..1449f5534 100644 --- a/tests/run/unicodeliterals.pyx +++ b/tests/run/unicodeliterals.pyx @@ -102,11 +102,7 @@ __doc__ = br""" True >>> ustring_in_constant_tuple == ('a', u'abc', u'\\N{SNOWMAN}', u'x' * 3, u'\\N{SNOWMAN}' * 4 + u'O') or ustring_in_constant_tuple # unescaped by Python True -""" -if sys.version_info >= (2,6,5): - # this doesn't work well in older Python versions - __doc__ += u"""\ >>> expected = u'\U00101234' # unescaped by Cython >>> if wide_literal == expected: print(True) ... else: print(repr(wide_literal), repr(expected), sys.maxunicode) diff --git a/tests/run/unsigned_char_ptr_bytes_conversion_T359.pyx b/tests/run/unsigned_char_ptr_bytes_conversion_T359.pyx index fb611511b..09df291f5 100644 --- a/tests/run/unsigned_char_ptr_bytes_conversion_T359.pyx +++ b/tests/run/unsigned_char_ptr_bytes_conversion_T359.pyx @@ -1,4 +1,4 @@ -# ticket: 359 +# ticket: t359 cdef unsigned char* some_c_unstring = 'test toast taste' diff --git a/tests/run/unsignedbehaviour_T184.pyx b/tests/run/unsignedbehaviour_T184.pyx index 9cbf65fc1..4dcb86398 100644 --- a/tests/run/unsignedbehaviour_T184.pyx +++ b/tests/run/unsignedbehaviour_T184.pyx @@ -1,4 +1,4 @@ -# ticket: 184 +# ticket: t184 """ >>> c_call() diff --git a/tests/run/versioned_pxds.srctree b/tests/run/versioned_pxds.srctree new file mode 100644 index 000000000..fb46cb26f --- /dev/null +++ b/tests/run/versioned_pxds.srctree @@ -0,0 +1,79 @@ +# mode: run +# tag: pxd + +""" +PYTHON setup.py build_ext --inplace +PYTHON -c "import runner" +""" + +######## setup.py ######## + +from Cython.Build.Dependencies import cythonize + +from distutils.core import setup, Extension + +setup( + ext_modules=cythonize([ + Extension("pkg.m1.a", ["pkg/m1/a.pyx"]), + Extension("pkg.m2.b", ["pkg/m2/b.pyx"]) + ]), +) + +######## pkg/__init__.py ######## + +######## pkg/m1/__init__.py ######## + + +######## pkg/m1/a.pyx ######## + +cdef class A: + def __init__(self): + self.x = 5 + +######## pkg/m1/a.pxd ######## + +to be ignored if there is a more specific file + +######## pkg/m1/a.cython-2.pxd ######## + +very outdated, not to be picked up + +######## pkg/m1/a.cython-20.pxd ######## + +outdated, not to be picked up + +######## pkg/m1/a.cython-29.pxd ######## + +# closest version should get found! + +cdef class A: + cdef public float x + +######## pkg/m1/a.cython-300000.pxd ######## + +Invalid distant future syntax right here! + +######## pkg/m1/a.cython-100000.pxd ######## + +Invalid future syntax right here! + + +######## pkg/m2/__init__.py ######## + +######## pkg/m2/b.pyx ######## + +from pkg.m1.a cimport A + +cdef class B(A): + pass + +######## runner.py ######## + +from pkg.m1.a import A +from pkg.m2.b import B + +a = A() +b = B() + +assert a.x == 5 +assert isinstance(a.x, float), type(a.x) diff --git a/tests/run/with_gil.pyx b/tests/run/with_gil.pyx index 46e9e6c2a..6fee3f192 100644 --- a/tests/run/with_gil.pyx +++ b/tests/run/with_gil.pyx @@ -1,3 +1,6 @@ +# mode: run +# tag: nogil, withgil + """ Test the 'with gil:' statement. """ @@ -460,6 +463,35 @@ def test_nogil_try_finally_error_label(): print e.args[0] +def void_with_python_objects(): + """ + >>> void_with_python_objects() + """ + with nogil: + _void_with_python_objects() + + +cdef void _void_with_python_objects() nogil: + c = 123 + with gil: + obj1 = [123] + obj2 = [456] + + +def void_with_py_arg_reassigned(x): + """ + >>> void_with_py_arg_reassigned(123) + """ + with nogil: + _void_with_py_arg_reassigned(x) + + +cdef void _void_with_py_arg_reassigned(x) nogil: + c = 123 + with gil: + x = [456] + + cdef void test_timing_callback() with gil: pass diff --git a/tests/run/with_gil_automatic.pyx b/tests/run/with_gil_automatic.pyx new file mode 100644 index 000000000..425dbbce7 --- /dev/null +++ b/tests/run/with_gil_automatic.pyx @@ -0,0 +1,138 @@ +# mode: run +# tag: nogil +# cython: language_level=2 + +cimport cython + + +#### print + +@cython.test_assert_path_exists( + "//GILStatNode", + "//GILStatNode//GILStatNode", + "//GILStatNode//GILStatNode//PrintStatNode", +) +def test_print_in_nogil_section(x): + """ + >>> test_print_in_nogil_section(123) + --123-- + """ + with nogil: + print f"--{x}--" + + +@cython.test_assert_path_exists( + "//GILStatNode", + "//GILStatNode//PrintStatNode", +) +@cython.test_fail_if_path_exists( + "//GILStatNode//GILStatNode", +) +cpdef int test_print_in_nogil_func(x) nogil except -1: + """ + >>> _ = test_print_in_nogil_func(123) + --123-- + """ + print f"--{x}--" + + +#### raise + +@cython.test_assert_path_exists( + "//GILStatNode", + "//GILStatNode//GILStatNode", + "//GILStatNode//GILStatNode//RaiseStatNode", +) +def test_raise_in_nogil_section(x): + """ + >>> try: test_raise_in_nogil_section(123) + ... except ValueError as exc: print(exc) + ... else: print("NOT RAISED !") + --123-- + """ + with nogil: + raise ValueError(f"--{x}--") + + +@cython.test_assert_path_exists( + "//GILStatNode", + "//GILStatNode//RaiseStatNode", +) +@cython.test_fail_if_path_exists( + "//GILStatNode//GILStatNode", +) +cpdef int test_raise_in_nogil_func(x) nogil except -1: + """ + >>> test_raise_in_nogil_func(123) + Traceback (most recent call last): + ValueError: --123-- + """ + raise ValueError(f"--{x}--") + + +#### assert + +@cython.test_assert_path_exists( + "//GILStatNode", + "//GILStatNode//AssertStatNode", + "//GILStatNode//AssertStatNode//GILStatNode", + "//GILStatNode//AssertStatNode//GILStatNode//RaiseStatNode", +) +def assert_in_nogil_section(int x): + """ + >>> assert_in_nogil_section(123) + >>> assert_in_nogil_section(0) + Traceback (most recent call last): + AssertionError + """ + with nogil: + assert x + + +@cython.test_assert_path_exists( + "//GILStatNode", + "//GILStatNode//AssertStatNode", + "//GILStatNode//AssertStatNode//GILStatNode", + "//GILStatNode//AssertStatNode//GILStatNode//RaiseStatNode", +) +def assert_in_nogil_section_ustring(int x): + """ + >>> assert_in_nogil_section_string(123) + >>> assert_in_nogil_section_string(0) + Traceback (most recent call last): + AssertionError: failed! + """ + with nogil: + assert x, u"failed!" + + +@cython.test_assert_path_exists( + "//GILStatNode", + "//GILStatNode//AssertStatNode", + "//GILStatNode//AssertStatNode//GILStatNode", + "//GILStatNode//AssertStatNode//GILStatNode//RaiseStatNode", +) +def assert_in_nogil_section_string(int x): + """ + >>> assert_in_nogil_section_string(123) + >>> assert_in_nogil_section_string(0) + Traceback (most recent call last): + AssertionError: failed! + """ + with nogil: + assert x, "failed!" + + +@cython.test_assert_path_exists( + "//AssertStatNode", + "//AssertStatNode//GILStatNode", + "//AssertStatNode//GILStatNode//RaiseStatNode", +) +cpdef int assert_in_nogil_func(int x) nogil except -1: + """ + >>> _ = assert_in_nogil_func(123) + >>> assert_in_nogil_func(0) + Traceback (most recent call last): + AssertionError: failed! + """ + assert x, "failed!" diff --git a/tests/run/with_statement_module_level_T536.pyx b/tests/run/with_statement_module_level_T536.pyx index 506c362f9..a18925f46 100644 --- a/tests/run/with_statement_module_level_T536.pyx +++ b/tests/run/with_statement_module_level_T536.pyx @@ -1,4 +1,4 @@ -# ticket: 536 +# ticket: t536 __doc__ = """ >>> inner_result diff --git a/tests/run/yield_from_pep380.pyx b/tests/run/yield_from_pep380.pyx index a1b7500d5..d73144c9e 100644 --- a/tests/run/yield_from_pep380.pyx +++ b/tests/run/yield_from_pep380.pyx @@ -4,7 +4,7 @@ Test suite for PEP 380 implementation adapted from original tests written by Greg Ewing -see <http://www.cosc.canterbury.ac.nz/greg.ewing/python/yield-from/YieldFrom-Python3.1.2-rev5.zip> +see <https://www.cosc.canterbury.ac.nz/greg.ewing/python/yield-from/YieldFrom-Python3.1.2-rev5.zip> """ import sys diff --git a/tests/windows_bugs.txt b/tests/windows_bugs.txt index ea5b23902..efca82325 100644 --- a/tests/windows_bugs.txt +++ b/tests/windows_bugs.txt @@ -24,3 +24,9 @@ queue queue2 queue3 lunch + +# "C linkage function cannot return C++ class" (uses public C++ cdef function) +cpp_template_subclasses + +# MSVC lacks "complex.h" +complex_numbers_cmath_T2891 @@ -10,9 +10,3 @@ envlist = py26, py27, py32, py33, py34, pypy setenv = CFLAGS=-O0 -ggdb commands = {envpython} runtests.py -vv - -[pycodestyle] -ignore = W, E -select = E711, E714, E501, W291 -max-line-length = 150 -format = pylint |