diff options
985 files changed, 39491 insertions, 11912 deletions
diff --git a/.coveragerc b/.coveragerc deleted file mode 100644 index 9bc4ce49a..000000000 --- a/.coveragerc +++ /dev/null @@ -1,7 +0,0 @@ -[run] -branch = True -parallel = True -concurrency = multiprocessing,thread -include = Cython/* -source = Cython -omit = Test* diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 000000000..091606923 --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1,12 @@ +# These are supported funding model platforms + +github: scoder # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2] +patreon: # Replace with a single Patreon username +open_collective: # Replace with a single Open Collective username +ko_fi: # Replace with a single Ko-fi username +tidelift: pypi/Cython # Replace with a single Tidelift platform-name/package-name e.g., npm/babel +community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry +liberapay: # Replace with a single Liberapay username +issuehunt: # Replace with a single IssueHunt username +otechie: # Replace with a single Otechie username +custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2'] diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 000000000..04294a70b --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,34 @@ +--- +name: Bug report +about: Create a report to help us improve +title: "[BUG] " +labels: '' +assignees: '' + +--- + +<!-- +**PLEASE READ THIS FIRST:** +- Do not use the bug and feature tracker for support requests. Use the `cython-users` mailing list instead. +- Did you search for similar issues already? Please do, it helps to save us precious time that we otherwise could not invest into development. +- Did you try the latest master branch or pre-release? It might already have what you want to report. Also see the [Changelog](https://github.com/cython/cython/blob/master/CHANGES.rst) regarding recent changes. +--> + +**Describe the bug** +A clear and concise description of what the bug is. + +**To Reproduce** +Code to reproduce the behaviour: +```cython +``` + +**Expected behavior** +A clear and concise description of what you expected to happen. + +**Environment (please complete the following information):** + - OS: [e.g. Linux, Windows, macOS] + - Python version [e.g. 3.8.4] + - Cython version [e.g. 0.29.18] + +**Additional context** +Add any other context about the problem here. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 000000000..7b5be96a8 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,36 @@ +--- +name: Feature request +about: Suggest an idea for this project +title: "[ENH] " +labels: '' +assignees: '' + +--- + +<!-- +**Note:** +- Do not use the bug and feature tracker for support requests. Use the `cython-users` mailing list instead. +- Did you search for similar issues already? Please do, it helps to save us precious time that we otherwise could not invest into development. +- Did you try the latest master branch or pre-release? It might already have what you want to report. Also see the [Changelog](https://github.com/cython/cython/blob/master/CHANGES.rst) regarding recent changes. +--> + +**Is your feature request related to a problem? Please describe.** +A clear and concise description of what the problem is. Ex. In my code, I would like to do [...] +```cython +# add use case related code here +``` + +**Describe the solution you'd like** +A clear and concise description of what you want to happen, including code examples if applicable. +```cython +# add a proposed code/syntax example here +``` + +**Describe alternatives you've considered** +A clear and concise description of any alternative solutions or features you've considered. +```cython +# add alternative code/syntax proposals here +``` + +**Additional context** +Add any other context about the feature request here. diff --git a/.github/code-of-conduct.md b/.github/code-of-conduct.md new file mode 100644 index 000000000..c3590a96f --- /dev/null +++ b/.github/code-of-conduct.md @@ -0,0 +1,78 @@ +# Cython Code of Conduct + +## Introduction + +This Code of Conduct applies to all spaces managed by the Cython project, including all public and private mailing lists, issue trackers, wikis, and any other communication channel used by our community. The Cython project may also organise or participate in in-person or virtual events. This Code of Conduct applies to events organized by the Cython project, and we expect other events related to our community to have a code of conduct similar in spirit to this one. + +This Code of Conduct should be honored by everyone who participates in the Cython community formally or informally, or claims any affiliation with the project, in any project-related activities and especially when representing the project, in any role. + +This code is not exhaustive or complete. It serves to distill our common understanding of a collaborative, shared environment and goals. Please try to follow this code in spirit as much as in letter, to create a friendly and productive environment that enriches the surrounding community. + +## Specific Guidelines + +We strive to: + +1. Be open. We invite anyone to participate in our community. We prefer to use public methods of communication for project-related messages, unless discussing something sensitive. This applies to messages for help or project-related support, too; not only is a public support request much more likely to result in an answer to a question, it also ensures that any inadvertent mistakes in answering are more easily detected and corrected. +2. Be empathetic, welcoming, friendly, and patient. We work together to resolve conflict, and assume good intentions. We may all experience some frustration from time to time, but we do not allow frustration to turn into a personal attack. A community where people feel uncomfortable or threatened is not a productive one. +3. Be collaborative. Our work will be used by other people, and in turn we will depend on the work of others. When we make something for the benefit of the project, we are willing to explain to others how it works, so that they can build on the work to make it even better. Any decision we make will affect users and colleagues, and we take those consequences seriously when making decisions. +4. Be inquisitive. Nobody knows everything! Asking questions early avoids many problems later, so we encourage questions, although we may direct them to the appropriate forum. We will try hard to be responsive and helpful. +5. Be careful in the words that we choose. We are careful and respectful in our communication, and we take responsibility for our own speech. Be kind to others. Do not insult or put down other participants. We will not accept harassment or other exclusionary behaviour, such as: + * Violent threats or language directed against another person. + * Sexist, racist, or otherwise discriminatory jokes and language. + * Posting sexually explicit or violent material. + * Posting (or threatening to post) other people’s personally identifying information (“doxing”). + * Sharing private content, such as emails sent privately or non-publicly, or unlogged forums such as IRC channel history, without the sender’s consent. + * Personal insults, especially those using racist or sexist terms. + * Unwelcome sexual attention. + * Excessive profanity. Please avoid swearwords; people differ greatly in their sensitivity to swearing. + * Repeated harassment of others. In general, if someone asks you to stop, then stop. + * Advocating for, or encouraging, any of the above behaviour. + +## Diversity Statement + +The Cython project welcomes and encourages participation by everyone. We are committed to being a community that everyone enjoys being part of. Although we may not always be able to accommodate each individual’s preferences, we try our best to treat everyone kindly. + +No matter how you identify yourself or how others perceive you: we welcome you. Though no list can hope to be comprehensive, we explicitly honour diversity in: age, culture, ethnicity, genotype, gender identity or expression, language, national origin, neurotype, phenotype, political beliefs, profession, race, religion, sexual orientation, socioeconomic status, subculture and technical ability, to the extent that these do not conflict with this code of conduct. + +Though we welcome people fluent in all languages, Cython development is conducted in English. + +Standards for behaviour in the Cython community are detailed in the Code of Conduct above. Participants in our community should uphold these standards in all their interactions and help others to do so as well (see next section). + +## Reporting Guidelines + +We know that it is painfully common for Internet communication to start at or devolve into obvious and flagrant abuse. We also recognize that sometimes people may have a bad day, or be unaware of some of the guidelines in this Code of Conduct. Please keep this in mind when deciding on how to respond to a breach of this Code. + +For clearly intentional breaches, report those to the Code of Conduct Committee (see below). For possibly unintentional breaches, you may reply to the person and point out this code of conduct (either in public or in private, whatever is most appropriate). If you would prefer not to do that, please feel free to report to the Code of Conduct Committee directly, or ask the Committee for advice, in confidence. + +You can report issues to the Cython Code of Conduct Committee at cython-conduct@googlegroups.com. + +Currently, the Committee consists of: + +* Stefan Behnel +* Robert Bradshaw +* Ralf Gommers + +If your report involves any members of the Committee, or if they feel they have a conflict of interest in handling it, then they will step aside and not involve themselves from considering your report. Alternatively, if for any reason you feel uncomfortable making a report to the Committee, then you can also contact senior NumFOCUS staff at [conduct@numfocus.org](https://numfocus.org/code-of-conduct#persons-responsible). + +## Incident reporting resolution & Code of Conduct enforcement + +_This section summarizes the most important points, more details can be found in_ [Cython Code of Conduct - How to follow up on a report](report-handling-manual.md). + +We will investigate and respond to all complaints. The Cython Code of Conduct Committee will protect the identity of the reporter, and treat the content of complaints as confidential (unless the reporter agrees otherwise). + +In case of severe and obvious breaches, e.g. personal threat or violent, sexist or racist language, we will immediately disconnect the originator from Cython communication channels; please see the manual for details. + +In cases not involving clear severe and obvious breaches of this Code of Conduct the process for acting on any received Code of Conduct violation report will be: + +1. acknowledge report is received, +2. reasonable discussion/feedback, +3. mediation (if feedback didn’t help, and only if both reporter and reportee agree to this), +4. enforcement via transparent decision (see [Resolutions](report-handling-manual.md#resolutions)) by the Code of Conduct Committee. + +The Committee will respond to any report as soon as possible, and at most within 72 hours. + +## Endnotes + +We are thankful to the groups behind the following documents, from which we drew content and inspiration: + +- [The SciPy Code of Conduct](https://docs.scipy.org/doc/scipy/reference/dev/conduct/code_of_conduct.html) diff --git a/.github/report-handling-manual.md b/.github/report-handling-manual.md new file mode 100644 index 000000000..5c33c14b4 --- /dev/null +++ b/.github/report-handling-manual.md @@ -0,0 +1,92 @@ +# Cython Code of Conduct - How to follow up on a report + +This is the manual followed by Cython’s Code of Conduct Committee. It’s used when we respond to an issue to make sure we’re consistent and fair. + +Enforcing the [Code of Conduct](code-of-conduct.md) impacts our community today and for the future. It’s an action that we do not take lightly. When reviewing enforcement measures, the Code of Conduct Committee will keep the following values and guidelines in mind: + +* Act in a personal manner rather than impersonal. The Committee can engage the parties to understand the situation while respecting the privacy and any necessary confidentiality of reporters. However, sometimes it is necessary to communicate with one or more individuals directly: the Committee’s goal is to improve the health of our community rather than only produce a formal decision. +* Emphasize empathy for individuals rather than judging behavior, avoiding binary labels of “good” and “bad/evil”. Overt, clear-cut aggression and harassment exist, and we will address them firmly. But many scenarios that can prove challenging to resolve are those where normal disagreements devolve into unhelpful or harmful behavior from multiple parties. Understanding the full context and finding a path that re-engages all is hard, but ultimately the most productive for our community. +* We understand that email is a difficult medium and can be isolating. Receiving criticism over email, without personal contact, can be particularly painful. This makes it especially important to keep an atmosphere of open-minded respect for the views of others. It also means that we must be transparent in our actions, and that we will do everything in our power to make sure that all our members are treated fairly and with sympathy. +* Discrimination can be subtle and it can be unconscious. It can show itself as unfairness and hostility in otherwise ordinary interactions. We know that this does occur, and we will take care to look out for it. We would very much like to hear from you if you feel you have been treated unfairly, and we will use these procedures to make sure that your complaint is heard and addressed. +* Help increase engagement in good discussion practice: try to identify where discussion may have broken down, and provide actionable information, pointers, and resources that can lead to positive change on these points. +* Be mindful of the needs of new members: provide them with explicit support and consideration, with the aim of increasing participation from underrepresented groups in particular. +* Individuals come from different cultural backgrounds and native languages. Try to identify any honest misunderstandings caused by a non-native speaker and help them understand the issue and what they can change to avoid causing offence. Complex discussion in a foreign language can be very intimidating, and we want to grow our diversity also across nationalities and cultures. + + +## Mediation + +Voluntary informal mediation is a tool at our disposal. In contexts such as when two or more parties have all escalated to the point of inappropriate behavior (something sadly common in human conflict), it may be useful to facilitate a mediation process. This is only an example: the Committee can consider mediation in any case, mindful that the process is meant to be strictly voluntary and no party can be pressured to participate. If the Committee suggests mediation, it should: + +* Find a candidate who can serve as a mediator. +* Obtain the agreement of the reporter(s). The reporter(s) have complete freedom to decline the mediation idea or to propose an alternate mediator. +* Obtain the agreement of the reported person(s). +* Settle on the mediator: while parties can propose a different mediator than the suggested candidate, only if a common agreement is reached on all terms can the process move forward. +* Establish a timeline for mediation to complete, ideally within two weeks. + +The mediator will engage with all the parties and seek a resolution that is satisfactory to all. Upon completion, the mediator will provide a report (vetted by all parties to the process) to the Committee, with recommendations on further steps. The Committee will then evaluate these results (whether a satisfactory resolution was achieved or not) and decide on any additional action deemed necessary. + + +## How the Committee will respond to reports + +When the Committee (or a Committee member) receives a report, they will first determine whether the report is about a clear and severe breach (as defined below). If so, immediate action needs to be taken in addition to the regular report handling process. + + +## Clear and severe breach actions + +We know that it is painfully common for internet communication to start at or devolve into obvious and flagrant abuse. We will deal quickly with clear and severe breaches like personal threats, violent, sexist or racist language. + +When a member of the Code of Conduct Committee becomes aware of a clear and severe breach, they will do the following: + +* Immediately disconnect the originator from all Cython communication channels. +* Reply to the reporter that their report has been received and that the originator has been disconnected. +* In every case, the moderator should make a reasonable effort to contact the originator, and tell them specifically how their language or actions qualify as a “clear and severe breach”. The moderator should also say that, if the originator believes this is unfair or they want to be reconnected to Cython, they have the right to ask for a review, as below, by the Code of Conduct Committee. The moderator should copy this explanation to the Code of Conduct Committee. +* The Code of Conduct Committee will formally review and sign off on all cases where this mechanism has been applied to make sure it is not being used to control ordinary heated disagreement. + + +## Report handling + +When a report is sent to the Committee they will immediately reply to the reporter to confirm receipt. This reply must be sent within 72 hours, and the group should strive to respond much quicker than that. + +If a report doesn’t contain enough information, the Committee will obtain all relevant data before acting. It may contact any individuals involved to get a more complete account of events. + +The Committee will then review the incident and determine, to the best of their ability: + +* What happened. +* Whether this event constitutes a Code of Conduct violation. +* Who are the responsible party(ies). +* Whether this is an ongoing situation, and there is a threat to anyone’s physical safety. + +This information will be collected in writing, and whenever possible the group’s deliberations will be recorded and retained (i.e. chat transcripts, email discussions, recorded conference calls, summaries of voice conversations, etc). + +It is important to retain an archive of all activities of this Committee to ensure consistency in behavior and provide institutional memory for the project. To assist in this, the default channel of discussion for this Committee will be a private mailing list accessible to current and future members of the Committee. If the Committee finds the need to use off-list communications (e.g. phone calls for early/rapid response), it should in all cases summarize these back to the list so there’s a good record of the process. + +The Code of Conduct Committee should aim to have a resolution agreed upon within two weeks. In the event that a resolution can’t be determined in that time, the Committee will respond to the reporter(s) with an update and projected timeline for resolution. + + +## Resolutions + +The Committee must agree on a resolution by consensus. If the group cannot reach consensus and deadlocks for over a week, the group will turn the matter over to the NumFOCUS Code of Conduct Enforcement Team for resolution. + +Possible responses may include: + +* Taking no further action: + - if we determine no violations have occurred; + - if the matter has been resolved publicly while the Committee was considering responses. +* Coordinating voluntary mediation: if all involved parties agree, the Committee may facilitate a mediation process as detailed above. +* Remind publicly, and point out that some behavior/actions/language have been judged inappropriate and why in the current context, or can but hurtful to some people, requesting the community to self-adjust. +* A private reprimand from the Committee to the individual(s) involved. In this case, the group chair will deliver that reprimand to the individual(s) over email, cc’ing the group. +* A public reprimand. In this case, the Committee chair will deliver that reprimand in the same venue that the violation occurred, within the limits of practicality. E.g., the original mailing list for an email violation, but for a chat room discussion where the person/context may be gone, they can be reached by other means. The group may choose to publish this message elsewhere for documentation purposes. +* A request for a public or private apology, assuming the reporter agrees to this idea: they may at their discretion refuse further contact with the violator. The chair will deliver this request. The Committee may, if it chooses, attach “strings” to this request: for example, the group may ask a violator to apologize in order to retain one’s membership on a mailing list. +* A “mutually agreed upon hiatus” where the Committee asks the individual to temporarily refrain from community participation. If the individual chooses not to take a temporary break voluntarily, the Committee may issue a “mandatory cooling off period”. +* A permanent or temporary ban from some or all Cython spaces (mailing list, GitHub, etc.). The group will maintain records of all such bans so that they may be reviewed in the future or otherwise maintained. + +Once a resolution is agreed upon, but before it is enacted, the Committee will contact the original reporter and any other affected parties and explain the proposed resolution. The Committee will ask if this resolution is acceptable, and must note feedback for the record. + +Finally, the Committee will make a report to the Cython project leadership (as well as the Cython core team in the event of an ongoing resolution, such as a ban). + +The Committee will never publicly discuss the issue; all public statements will be made by the chair of the Code of Conduct Committee. + + +## Conflicts of Interest + +In the event of any conflict of interest, a Committee member must immediately notify the other members, and recuse themselves if necessary. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 000000000..c885c4419 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,309 @@ +name: CI + +on: [push, pull_request] + +jobs: + ci: + strategy: + # Allows for matrix sub-jobs to fail without canceling the rest + fail-fast: false + + # MATRIX: + # ======= + # Required parameters: + # os the os to run on + # python-version the python version to use + # backend the backend to use + # env any additional env variables. Set to '{}' for none + # Optional parameters: + # allowed_failure whether the job is allowed to fail + # extra_hash extra hash str to differentiate from other caches with similar name (must always start with '-') + matrix: + # Tests [amd64] + # + # FIXME: 'cpp' tests seems to fail due to compilation errors (numpy_pythran_unit) + # in all python versions and test failures (builtin_float) in 3.5< + os: [ubuntu-18.04] + backend: [c, cpp] + python-version: [2.7, 3.4, 3.5, 3.6, 3.7, 3.8, 3.9, 3.10-dev] + env: [{}] + + include: + # Temporary - Allow failure on all Python -dev jobs until they are considered stable + #- python-version: 3.10-dev + # allowed_failure: true + + # Ubuntu sub-jobs: + # ================ + # GCC 11 + - os: ubuntu-18.04 + python-version: 3.9 + backend: c + env: { GCC_VERSION: 11 } + extra_hash: "-gcc11" + - os: ubuntu-18.04 + python-version: 3.9 + backend: cpp + env: { GCC_VERSION: 11 } + extra_hash: "-gcc11" + # compile all modules + - os: ubuntu-18.04 + python-version: 2.7 + backend: c + env: { CYTHON_COMPILE_ALL: 1 } + extra_hash: "-all" + - os: ubuntu-18.04 + python-version: 2.7 + backend: cpp + env: { CYTHON_COMPILE_ALL: 1 } + extra_hash: "-all" + - os: ubuntu-18.04 + python-version: 3.9 + backend: c + env: { CYTHON_COMPILE_ALL: 1 } + extra_hash: "-all" + - os: ubuntu-18.04 + python-version: 3.9 + backend: cpp + env: { CYTHON_COMPILE_ALL: 1 } + extra_hash: "-all" + # Linting + - os: ubuntu-18.04 + python-version: 3.7 + backend: "c,cpp" + env: { TEST_CODE_STYLE: 1, NO_CYTHON_COMPILE: 1 } + extra_hash: "-codestyle" + # Limited API + - os: ubuntu-18.04 + python-version: 3.6 + backend: "c,cpp" + env: { LIMITED_API: "--limited-api", EXCLUDE: "--no-file" } + extra_hash: "-limited_api" + - os: ubuntu-18.04 + python-version: 3.7 + backend: "c,cpp" + env: { LIMITED_API: "--limited-api", EXCLUDE: "--no-file" } + extra_hash: "-limited_api" + - os: ubuntu-18.04 + python-version: 3.8 + backend: "c,cpp" + env: { LIMITED_API: "--limited-api", EXCLUDE: "--no-file" } + extra_hash: "-limited_api" + # Type specs + - os: ubuntu-18.04 + python-version: 3.9 + backend: c + env: { EXTRA_CFLAGS: "-DCYTHON_USE_TYPE_SPECS=1" } + extra_hash: "-typespecs" + - os: ubuntu-18.04 + python-version: 3.8 + backend: c + env: { EXTRA_CFLAGS: "-DCYTHON_USE_TYPE_SPECS=1" } + extra_hash: "-typespecs" + - os: ubuntu-18.04 + python-version: 3.7 + backend: c + env: { EXTRA_CFLAGS: "-DCYTHON_USE_TYPE_SPECS=1" } + extra_hash: "-typespecs" + - os: ubuntu-18.04 + python-version: 3.6 + backend: c + env: { EXTRA_CFLAGS: "-DCYTHON_USE_TYPE_SPECS=1" } + extra_hash: "-typespecs" + - os: ubuntu-18.04 + python-version: 3.5 + backend: c + env: { EXTRA_CFLAGS: "-DCYTHON_USE_TYPE_SPECS=1" } + extra_hash: "-typespecs" + allowed_failure: true + # Stackless + - os: ubuntu-18.04 + python-version: 2.7 + backend: c + env: { STACKLESS: true, PY: 2 } + extra_hash: "-stackless" + - os: ubuntu-18.04 + python-version: 3.6 + backend: c + env: { STACKLESS: true, PY: 3 } + extra_hash: "-stackless" + # Pypy + - os: ubuntu-18.04 + python-version: pypy-2.7 + backend: c + env: { NO_CYTHON_COMPILE: 1 } + - os: ubuntu-18.04 + python-version: pypy-3.7 + backend: c + env: { NO_CYTHON_COMPILE: 1 } + # Pypy [allowed-failures] - These specifically test known bugs + - os: ubuntu-18.04 + python-version: pypy-2.7 + backend: c + env: + { + NO_CYTHON_COMPILE: 1, + EXCLUDE: "--listfile=tests/pypy_bugs.txt --listfile=tests/pypy2_bugs.txt bugs", + } + allowed_failure: true + extra_hash: "-allowed_failures" + - os: ubuntu-18.04 + python-version: pypy-3.7 + backend: c + env: { NO_CYTHON_COMPILE: 1, EXCLUDE: "--listfile=tests/pypy_bugs.txt bugs" } + allowed_failure: true + extra_hash: "-allowed_failures" + # Coverage - Disabled due to taking too long to run + # - os: ubuntu-18.04 + # python-version: 3.7 + # backend: "c,cpp" + # env: { COVERAGE: 1 } + # extra_hash: '-coverage' + + # MacOS sub-jobs + # ============== + # (C-only builds are used to create wheels) + - os: macos-10.15 + python-version: 2.7 + backend: c + env: { MACOSX_DEPLOYMENT_TARGET: 10.14 } + - os: macos-10.15 + python-version: 2.7 + backend: cpp + env: { MACOSX_DEPLOYMENT_TARGET: 10.14 } + - os: macos-10.15 + python-version: 3.5 + backend: c + env: { MACOSX_DEPLOYMENT_TARGET: 10.14 } + - os: macos-10.15 + python-version: 3.6 + backend: c + env: { MACOSX_DEPLOYMENT_TARGET: 10.14 } + - os: macos-10.15 + python-version: 3.7 + backend: c + env: { MACOSX_DEPLOYMENT_TARGET: 10.14 } + - os: macos-10.15 + python-version: 3.8 + backend: c + env: { MACOSX_DEPLOYMENT_TARGET: 10.14 } + - os: macos-10.15 + python-version: 3.9 + backend: c + env: { MACOSX_DEPLOYMENT_TARGET: 10.14 } + - os: macos-10.15 + python-version: 3.9 + backend: cpp + env: { MACOSX_DEPLOYMENT_TARGET: 10.14 } + + # This defaults to 360 minutes (6h) which is way too long and if a test gets stuck, it can block other pipelines. + # From testing, the runs tend to take ~20 minutes, so a limit of 30 minutes should be enough. This can always be + # changed in the future if needed. + timeout-minutes: 30 + runs-on: ${{ matrix.os }} + + env: + BACKEND: ${{ matrix.backend }} + OS_NAME: ${{ matrix.os }} + PYTHON_VERSION: ${{ matrix.python-version }} + GCC_VERSION: 8 + USE_CCACHE: 1 + CCACHE_SLOPPINESS: "pch_defines,time_macros" + CCACHE_COMPRESS: 1 + CCACHE_MAXSIZE: "200M" + + steps: + - name: Checkout repo + uses: actions/checkout@v2 + with: + fetch-depth: 1 + + - name: Setup python + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Cache [ccache] + uses: pat-s/always-upload-cache@v2.1.3 + if: startsWith(runner.os, 'Linux') + with: + path: ~/.ccache + key: ${{ runner.os }}-ccache${{ matrix.extra_hash }}-${{ matrix.python-version }}-${{ matrix.backend == 'c' || matrix.backend == 'c,cpp' }}-${{ contains(matrix.backend, 'cpp') }}-${{ hashFiles('**/test-requirements*.txt', '**/ci.yml', '**/ci-run.sh') }} + + - name: Run CI + continue-on-error: ${{ matrix.allowed_failure || false }} + env: ${{ matrix.env }} + run: bash ./Tools/ci-run.sh + + - name: Upload HTML docs + uses: actions/upload-artifact@v2 + with: + name: htmldocs + path: docs/build/html + if-no-files-found: ignore + + - name: Upload wheels + uses: actions/upload-artifact@v2 + with: + name: wheels-${{ runner.os }} + path: dist/*.whl + if-no-files-found: ignore + + + pycoverage: + runs-on: ubuntu-18.04 + + env: + BACKEND: c,cpp + OS_NAME: ubuntu-18.04 + PYTHON_VERSION: 3.9 + + steps: + - name: Checkout repo + uses: actions/checkout@v2 + with: + fetch-depth: 1 + + - name: Setup python + uses: actions/setup-python@v2 + with: + python-version: 3.9 + + - name: Run Coverage + env: { COVERAGE: 1, NO_CYTHON_COMPILE: 1 } + run: bash ./Tools/ci-run.sh + + - name: Upload Coverage Report + uses: actions/upload-artifact@v2 + with: + name: pycoverage_html + path: coverage-report-html + +# cycoverage: +# runs-on: ubuntu-18.04 +# +# env: +# BACKEND: c,cpp +# OS_NAME: ubuntu-18.04 +# PYTHON_VERSION: 3.9 +# +# steps: +# - name: Checkout repo +# uses: actions/checkout@v2 +# with: +# fetch-depth: 1 +# +# - name: Setup python +# uses: actions/setup-python@v2 +# with: +# python-version: 3.9 +# +# - name: Run Coverage +# env: { COVERAGE: 1 } +# run: bash ./Tools/ci-run.sh +# +# - name: Upload Coverage Report +# uses: actions/upload-artifact@v2 +# with: +# name: cycoverage_html +# path: coverage-report-html diff --git a/.github/workflows/wheel-manylinux.yml b/.github/workflows/wheel-manylinux.yml new file mode 100644 index 000000000..8a0b673af --- /dev/null +++ b/.github/workflows/wheel-manylinux.yml @@ -0,0 +1,123 @@ +name: Linux wheel build + +on: + release: + types: [created] + +jobs: + python: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - name: Set up Python + uses: actions/setup-python@v1 + with: + python-version: 3.8 + + - name: Install build dependencies + run: pip install -U setuptools pip wheel + + - name: Make sdist and Python wheel + run: make sdist pywheel + + - name: Upload sdist + uses: actions/upload-artifact@v2 + with: + name: sdist + path: dist/*.tar.gz + if-no-files-found: ignore + + - name: Upload Python wheel + uses: actions/upload-artifact@v2 + with: + name: wheel-Python + path: dist/*-none-any.whl + if-no-files-found: ignore + + manylinux1-i686: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - name: Set up Python + uses: actions/setup-python@v1 + with: + python-version: 3.8 + + - name: Build Linux wheels + uses: RalfG/python-wheels-manylinux-build@v0.3.4-manylinux1_i686 + with: + python-versions: 'cp27-cp27m cp27-cp27mu cp35-cp35m cp36-cp36m cp37-cp37m cp38-cp38 cp39-cp39 cp39-cp39' + + - name: Upload wheels + uses: actions/upload-artifact@v2 + with: + name: wheels-Linux + path: dist/*-manylinux*.whl + if-no-files-found: ignore + + manylinux1-x86_64: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - name: Set up Python + uses: actions/setup-python@v1 + with: + python-version: 3.8 + + - name: Build Linux wheels + uses: RalfG/python-wheels-manylinux-build@v0.3.4-manylinux1_x86_64 + with: + python-versions: 'cp27-cp27m cp27-cp27mu cp35-cp35m cp36-cp36m cp37-cp37m cp38-cp38 cp39-cp39' + + - name: Upload wheels + uses: actions/upload-artifact@v2 + with: + name: wheels-Linux + path: wheels/*-manylinux*.whl + if-no-files-found: ignore + + manylinux2014-i686: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - name: Set up Python + uses: actions/setup-python@v1 + with: + python-version: 3.8 + + - name: Build Linux wheels + uses: RalfG/python-wheels-manylinux-build@v0.3.4-manylinux2014_i686 + with: + python-versions: 'cp36-cp36m cp37-cp37m cp38-cp38 cp39-cp39' + + - name: Upload wheels + uses: actions/upload-artifact@v2 + with: + name: wheels-Linux + path: wheels/*-manylinux2014*.whl + if-no-files-found: ignore + + manylinux2014-x86_64: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - name: Set up Python + uses: actions/setup-python@v1 + with: + python-version: 3.8 + + - name: Build Linux wheels + uses: RalfG/python-wheels-manylinux-build@v0.3.4-manylinux2014_x86_64 + with: + python-versions: 'cp36-cp36m cp37-cp37m cp38-cp38 cp39-cp39' + - name: Upload wheels + uses: actions/upload-artifact@v2 + with: + name: wheels-Linux + path: wheels/*-manylinux2014*.whl + if-no-files-found: ignore diff --git a/.gitignore b/.gitignore index f59252e3f..deb4c6fce 100644 --- a/.gitignore +++ b/.gitignore @@ -6,30 +6,44 @@ __pycache__ *.egg *.egg-info +.*cache*/ +*venv*/ Cython/Compiler/*.c Cython/Plex/*.c Cython/Runtime/refnanny.c Cython/Tempita/*.c Cython/*.c +Cython/*.html +Cython/*/*.html Tools/*.elc +Demos/*.html +Demos/*/*.html /TEST_TMP/ /build/ +/cython_build/ +cython_debug/ /wheelhouse*/ !tests/build/ /dist/ .gitrev .coverage +*.patch +*.diff *.orig +*.prof *.rej +*.log *.dep *.swp *~ +callgrind.out.* .ipynb_checkpoints docs/build +docs/_build tags TAGS @@ -41,5 +55,8 @@ MANIFEST /.idea /*.iml +# Komodo EDIT/IDE project files +/*.komodoproject + # Visual Studio Code files .vscode diff --git a/.travis.yml b/.travis.yml index 8054d7799..551a38cb7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,148 +22,27 @@ env: - CCACHE_SLOPPINESS=pch_defines,time_macros - CCACHE_COMPRESS=1 - CCACHE_MAXSIZE=250M - - PATH="/usr/lib/ccache:$HOME/miniconda/bin:$PATH" - - BACKEND=c,cpp + - PATH="/usr/lib/ccache:$PATH" + - PYTHON_VERSION=3.8 + - OS_NAME=ubuntu + +python: 3.8 matrix: include: - - python: 2.7 - env: BACKEND=c - - python: 2.7 - env: BACKEND=cpp - - python: 3.7 - dist: xenial # Required for Python 3.7 - sudo: required # travis-ci/travis-ci#9069 - env: BACKEND=c - - python: 3.7 - dist: xenial # Required for Python 3.7 - sudo: required # travis-ci/travis-ci#9069 - env: BACKEND=cpp - - python: 2.6 - env: BACKEND=c - - python: 2.6 - env: BACKEND=cpp -# Disabled: coverage analysis takes excessively long, several times longer than without. -# - python: 3.7 -# dist: xenial # Required for Python 3.7 -# sudo: required # travis-ci/travis-ci#9069 -# env: COVERAGE=1 - - python: 3.7 - dist: xenial # Required for Python 3.7 - sudo: required # travis-ci/travis-ci#9069 - env: TEST_CODE_STYLE=1 - - python: 3.4 - env: BACKEND=c - - python: 3.4 - env: BACKEND=cpp - - python: 3.5 - env: BACKEND=c - - python: 3.5 - env: BACKEND=cpp - - python: 3.6 + - arch: arm64 env: BACKEND=c - - python: 3.6 + - arch: arm64 env: BACKEND=cpp - - python: 3.8 - dist: xenial # Required for Python 3.7 - sudo: required # travis-ci/travis-ci#9069 + - arch: ppc64le env: BACKEND=c - - python: 3.8 - dist: xenial # Required for Python 3.7 - sudo: required # travis-ci/travis-ci#9069 + - arch: ppc64le env: BACKEND=cpp - - python: 3.9 - dist: xenial # Required for Python 3.7 - sudo: required # travis-ci/travis-ci#9069 - env: BACKEND=c - - python: 3.9 - dist: xenial # Required for Python 3.7 - sudo: required # travis-ci/travis-ci#9069 - env: BACKEND=cpp - - os: osx - osx_image: xcode6.4 - env: PY=2 - python: 2.7 - language: c - compiler: clang - cache: false - - os: osx - osx_image: xcode10.3 - env: PY=3 MACOSX_DEPLOYMENT_TARGET=10.9 - python: 3.9 - language: c - compiler: clang - cache: false - - python: pypy - env: BACKEND=c - - python: pypy3 - env: BACKEND=c - - env: STACKLESS=true BACKEND=c PY=2 - python: 2.7 - - env: STACKLESS=true BACKEND=c PY=3 - python: 3.6 - allow_failures: - - python: pypy - - python: pypy3 - -branches: - only: - - master - - release - - 0.29.x - -before_install: - - | - if [ "$TRAVIS_OS_NAME" == "linux" ]; then - # adding apt repos in travis is really fragile => retry a couple of times. - for i in {1..10}; do travis_retry sudo apt-add-repository --yes 'ppa:ubuntu-toolchain-r/test' && break; sleep 2; done - for i in {1..10}; do travis_retry sudo apt-get update && travis_retry sudo apt-get install --yes gcc-8 $(if [ -z "${BACKEND##*cpp*}" ]; then echo -n "g++-8"; fi ) && break; sleep 2; done - sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-8 60 $(if [ -z "${BACKEND##*cpp*}" ]; then echo " --slave /usr/bin/g++ g++ /usr/bin/g++-8"; fi) - sudo update-alternatives --set gcc /usr/bin/gcc-8 - export CC=gcc - if [ -z "${BACKEND##*cpp*}" ]; then sudo update-alternatives --set g++ /usr/bin/g++-8; export CXX=g++; fi - fi - - - | - if [ "$TRAVIS_OS_NAME" == "osx" -o "$STACKLESS" == "true" ]; then - echo "Installing Miniconda" - if [ "$TRAVIS_OS_NAME" == "osx" ]; then CONDA_PLATFORM=MacOSX; else CONDA_PLATFORM=Linux; fi - travis_retry wget -O miniconda.sh https://repo.continuum.io/miniconda/Miniconda$PY-latest-${CONDA_PLATFORM}-x86_64.sh || exit 1 - bash miniconda.sh -b -p $HOME/miniconda && rm miniconda.sh || exit 1 - conda --version || exit 1 - #conda install --quiet --yes nomkl --file=test-requirements.txt --file=test-requirements-cpython.txt - if [ "$TRAVIS_OS_NAME" == "osx" ]; then - which clang && clang --version && export CC="clang -Wno-deprecated-declarations" || true - which clang++ && clang++ --version && export CXX="clang++ -stdlib=libc++ -Wno-deprecated-declarations" || true - fi - fi - - - if [ -n "$CC" ]; then which $CC; $CC --version; fi - - if [ -n "$CXX" ]; then which $CXX; $CXX --version; fi - - - if [ "$STACKLESS" == "true" ]; then - conda config --add channels stackless; - travis_retry conda install --quiet --yes stackless; - fi - -install: - - python -c 'import sys; print("Python %s" % (sys.version,))' - - if [ -z "${TRAVIS_PYTHON_VERSION##2.7}" ]; then [ "$TRAVIS_OS_NAME" == "osx" -a "$PY" == "3" ] || pip install -r test-requirements-27.txt ; fi - - if [ -n "${TRAVIS_PYTHON_VERSION##*-dev}" -a -n "${TRAVIS_PYTHON_VERSION##2.*}" ]; then pip install -r test-requirements.txt $( [ -z "${TRAVIS_PYTHON_VERSION##pypy*}" -o -z "${TRAVIS_PYTHON_VERSION##3.[47890]*}" ] || echo " -r test-requirements-cpython.txt" ) ; fi -# - CFLAGS="-O2 -ggdb -Wall -Wextra $(python -c 'import sys; print("-fno-strict-aliasing" if sys.version_info[0] == 2 else "")')" python setup.py build - -before_script: ccache -s || true + # Disabled due to test errors + # - arch: s390x + # env: BACKEND=c + # - arch: s390x + # env: BACKEND=cpp script: - - PYTHON_DBG="python$( python -c 'import sys; print("%d.%d" % sys.version_info[:2])' )-dbg" - - if [ "$TEST_CODE_STYLE" = "1" ]; then - STYLE_ARGS="--no-unit --no-doctest --no-file --no-pyregr --no-examples"; - else - STYLE_ARGS=--no-code-style; - if $PYTHON_DBG -V >&2; then CFLAGS="-O0 -ggdb" $PYTHON_DBG runtests.py -vv --no-code-style Debugger --backends=$BACKEND; fi; - #if [ -z "${BACKEND##*cpp*}" -a -n "${TRAVIS_PYTHON_VERSION##*-dev}" ]; then pip install pythran; fi; - if [ "$BACKEND" != "cpp" -a -n "${TRAVIS_PYTHON_VERSION##2*}" -a -n "${TRAVIS_PYTHON_VERSION##*-dev}" -a -n "${TRAVIS_PYTHON_VERSION##*3.4}" ]; then pip install mypy; fi; - fi - - if [ "$COVERAGE" != "1" ]; then CFLAGS="-O2 -ggdb -Wall -Wextra $(python -c 'import sys; print("-fno-strict-aliasing" if sys.version_info[0] == 2 else "")')" python setup.py build_ext -i; fi - - CFLAGS="-O0 -ggdb -Wall -Wextra" python runtests.py -vv $STYLE_ARGS -x Debugger --backends=$BACKEND $(if [ "$COVERAGE" == "1" ]; then echo " --coverage"; fi) $(if [ -z "$TEST_CODE_STYLE" ]; then echo " -j7 "; fi) - - ccache -s || true + - bash ./Tools/ci-run.sh diff --git a/CHANGES.rst b/CHANGES.rst index ef4fab222..e87b198fb 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -2,6 +2,769 @@ Cython Changelog ================ +3.0.0 alpha 9 (2021-07-21) +========================== + +Features added +-------------- + +* Declarations for ``libcpp.algorithms``, ``libcpp.set`` and ``libcpp.unordered_set`` + were extended. + Patch by David Woods. (Github issues :issue:`4271`, :issue:`4273`) + +* ``cygdb`` has a new option ``--skip-interpreter`` that allows using a different + Python runtime than the one used to generate the debugging information. + Patch by Alessandro Molina. (Github issue :issue:`4186`) + +Bugs fixed +---------- + +* Several issues with the new ``cpp_locals`` directive were resolved and + its test coverage improved. + Patch by David Woods. (Github issues :issue:`4266`, :issue:`4265`) + +* Generated utility code for C++ conversions no longer depends on several user + definable directives that may make it behave incorrectly. + Patch by David Woods. (Github issue :issue:`4206`) + +* A reference counting bug in the new ``@cython.total_ordering`` decorator was fixed. + +* Includes all bug-fixes from the :ref:`0.29.24` release. + +Other changes +------------- + +* Parts of the documentation were (and are being) rewritten to show the + Cython language syntax next to the equivalent Python syntax. + Patches by 0dminnimda and Matus Valo. (Github issue :issue:`4187`) + + +3.0.0 alpha 8 (2021-07-02) +========================== + +Features added +-------------- + +* A ``@cython.total_ordering`` decorator has been added to automatically + implement all comparison operators, similar to ``functools.total_ordering``. + Patch by Spencer Brown. (Github issue :issue:`2090`) + +* A new directive ``cpp_locals`` was added that allows local C++ variables to + be lazily initialised (without default constructor), thus making them behave + more like Python variables. + Patch by David Woods. (Github issue :issue:`4160`) + +* C++17 execution policies are supported in ``libcpp.algorithm``. + Patch by Ashwin Srinath. (Github issue :issue:`3790`) + +* New C feature flags: ``CYTHON_USE_MODULE_STATE``, ``CYTHON_USE_TYPE_SPECS`` + Both are currently considered experimental. + (Github issue :issue:`3611`) + +* ``[...] * N`` is optimised for C integer multipliers ``N``. + (Github issue :issue:`3922`) + +Bugs fixed +---------- + +* The dispatch code for binary operators to special methods could run into infinite recursion. + Patch by David Woods. (Github issue :issue:`4172`) + +* Code optimisations were not applied to methods of Cython implemented C++ classes. + Patch by David Woods. (Github issue :issue:`4212`) + +* The special ``cython`` module was not always detected in PEP-484 type annotations. + Patch by David Woods. (Github issue :issue:`4243`) + +* Conversion from Python dicts to ``std::map`` was broken. + Patch by David Woods and Mikkel Skofelt. (Github issues :issue:`4231`, :issue:`4228`) + +* The exception handling annotation ``except +*`` was broken. + Patch by David Woods. (Github issues :issue:`3065`, :issue:`3066`) + +* Attribute annotations in Python classes are now ignored, because they are + just Python objects in a dict (as opposed to the fields of extension types). + Patch by David Woods. (Github issues :issue:`4196`, :issue:`4198`) + +* An unnecessary slow-down at import time was removed from ``Cython.Distutils``. + Original patch by Anthony Sottile. (Github issue :issue:`4224`) + +* Python modules were not automatically recompiled when only their ``.pxd`` file changed. + Patch by Golden Rockefeller. (Github issue :issue:`1428`) + +* The signature of ``PyFloat_FromString()`` in ``cpython.float`` was changed + to match the signature in Py3. It still has an automatic fallback for Py2. + (Github issue :issue:`3909`) + +* A compile error on MSVC was resolved. + Patch by David Woods. (Github issue :issue:`4202`) + +* A C compiler warning in PyPy3 regarding ``PyEval_EvalCode()`` was resolved. + +* Directives starting with ``optimization.*`` in pure Python mode were incorrectly named. + It should have been ``optimize.*``. + Patch by David Woods. (Github issue :issue:`4258`) + +Other changes +------------- + +* Variables can no longer be declared with ``cpdef``. + Patch by David Woods. (Github issue :issue:`887`) + +* Support for the now unsupported Pyston V1 was removed in favour of Pyston V2. + Patch by Marius Wachtler. (Github issue :issue:`4211`) + +* The ``Cython.Build.BuildExecutable`` tool no longer executes the program automatically. + Use ``cythonrun`` for that. + + +3.0.0 alpha 7 (2021-05-24) +========================== + +Features added +-------------- + +* A ``cimport`` is now supported in pure Python code by prefixing the + imported module name with ``cython.cimports.``, e.g. + ``from cython.cimports.libc.math import sin``. + (GIthub issue :issue:`4190`) + +* ``__class_getitem__`` (`PEP-560`_) is supported for cdef classes. + Patch by Kmol Yuan. (Github issue :issue:`3764`) + +* ``__mro_entries__`` (`PEP-560`_) is supported for Python classes. + Patch by David Woods. (Github issue :issue:`3537`) + +* ``cython.array`` supports simple, non-strided views. + (Github issue :issue:`3775`) + +* Self-documenting f-strings (``=``) were implemented. + Patch by davfsa. (Github issue :issue:`3796`) + +* The destructor is now called for fields in C++ structs. + Patch by David Woods. (Github issue :issue:`3226`) + +* ``std::move()`` is now also called for temps during ``yield``. + Patch by Yu Feng. (Github issue :issue:`4154`) + +* ``asyncio.iscoroutinefunction()`` now recognises coroutine functions + also when compiled by Cython. + Patch by Pedro Marques da Luz. (Github issue :issue:`2273`) + +* C compiler warnings and errors are now shown in Jupyter notebooks. + Patch by Egor Dranischnikow. (Github issue :issue:`3751`) + +* ``float(…)`` is optimised for string arguments (str/bytes/bytearray). + +* Converting C++ containers to Python lists uses less memory allocations. + Patch by Max Bachmann. (Github issue :issue:`4081`) + +* Docstrings of ``cpdef`` enums are now copied to the enum class. + Patch by matham. (Github issue :issue:`3805`) + +* The type ``cython.Py_hash_t`` is available in Python mode. + +* C-API declarations for ``cpython.fileobject`` were added. + Patch by Zackery Spytz. (Github issue :issue:`3906`) + +* C-API declarations for context variables in Python 3.7 were added. + Original patch by Zolisa Bleki. (Github issue :issue:`2281`) + +* More C-API declarations for ``cpython.datetime`` were added. + Patch by Bluenix2. (Github issue :issue:`4128`) + +* A new module ``cpython.time`` was added with some low-level alternatives to + Python's ``time`` module. + Patch by Brock Mendel. (Github issue :issue:`3767`) + +* The value ``PyBUF_MAX_NDIM`` was added to the ``cpython.buffer`` module. + Patch by John Kirkham. (Github issue :issue:`3811`) + +* "Declaration after use" is now an error for variables. + Patch by David Woods. (Github issue :issue:`3976`) + +* More declarations for C++ string methods were added. + +* Cython now detects when existing output files were not previously generated + by itself and refuses to overwrite them. It is a common mistake to name + the module file of a wrapper after the library (source file) that it wraps, + which can lead to surprising errors when the file gets overwritten. + +Bugs fixed +---------- + +* Annotations were not exposed on annotated (data-)classes. + Patch by matsjoyce. (Github issue :issue:`4151`) + +* Inline functions and other code in ``.pxd`` files could accidentally + inherit the compiler directives of the ``.pyx`` file that imported them. + Patch by David Woods. (Github issue :issue:`1071`) + +* Some issues were resolved that could lead to duplicated C names. + Patch by David Woods. (Github issue :issue:`3716`, :issue:`3741`, :issue:`3734`) + +* Modules with unicode names failed to build on Windows. + Patch by David Woods. (Github issue :issue:`4125`) + +* ``ndarray.shape`` failed to compile with Pythran and recent NumPy. + Patch by Serge Guelton. (Github issue :issue:`3762`) + +* Casting to ctuples is now allowed. + Patch by David Woods. (Github issue :issue:`3808`) + +* Structs could not be instantiated with positional arguments in + pure Python mode. + +* Literal list assignments to pointer variables declared in PEP-526 + notation failed to compile. + +* Nested C++ types were not usable through ctypedefs. + Patch by Vadim Pushtaev. (Github issue :issue:`4039`) + +* Overloaded C++ static methods were lost. + Patch by Ashwin Srinath. (Github :issue:`1851`) + +* Cython compiled functions always provided a ``__self__`` attribute, + regardless of being used as a method or not. + Patch by David Woods. (Github issue :issue:`4036`) + +* Calls to ``.__class__()`` of a known extension type failed. + Patch by David Woods. (Github issue :issue:`3954`) + +* Generator expressions in pxd-overridden ``cdef`` functions could + fail to compile. + Patch by Matus Valo. (Github issue :issue:`3477`) + +* A reference leak on import failures was resolved. + Patch by Max Bachmann. (Github issue :issue:`4056`) + +* A C compiler warning about unused code was resolved. + (Github issue :issue:`3763`) + +* A C compiler warning about enum value casting was resolved in GCC. + (Github issue :issue:`2749`) + +* Some C compiler warninge were resolved. + Patches by Max Bachmann. (Github issue :issue:`4053`, :issue:`4059`, :issue:`4054`, :issue:`4148`, :issue:`4162`) + +* A compile failure for C++ enums in Py3.4 / MSVC was resolved. + Patch by Ashwin Srinath. (Github issue :issue:`3782`) + +* Some C++ STL methods did not propagate exceptions. + Patch by Max Bachmann. (Github issue :issue:`4079`) + +* An unsupported C-API call in PyPy was fixed. + Patch by Max Bachmann. (Github issue :issue:`4055`) + +* The Cython ``CodeWriter`` mishandled no-argument ``return`` statements. + Patch by Tao He. (Github issue :issue:`3795`) + +* ``complex`` wasn't supported in PEP-484 type annotations. + Patch by David Woods. (Github issue :issue:`3949`) + +* Default arguments of methods were not exposed for introspection. + Patch by Vladimir Matveev. (Github issue :issue:`4061`) + +* Extension types inheriting from Python classes could not safely + be exposed in ``.pxd`` files. + (Github issue :issue:`4106`) + +* The profiling/tracing code was adapted to work with Python 3.10b1. + +* The internal CPython macro ``Py_ISSPACE()`` is no longer used. + Original patch by Andrew Jones. (Github issue :issue:`4111`) + +* Includes all bug-fixes from the :ref:`0.29.23` release. + + +3.0.0 alpha 6 (2020-07-31) +========================== + +Features added +-------------- + +* Special methods for binary operators now follow Python semantics. + Rather than e.g. a single ``__add__`` method for cdef classes, where + "self" can be either the first or second argument, one can now define + both ``__add__`` and ``__radd__`` as for standard Python classes. + This behavior can be disabled with the ``c_api_binop_methods`` directive + to return to the previous semantics in Cython code (available from Cython + 0.29.20), or the reversed method (``__radd__``) can be implemented in + addition to an existing two-sided operator method (``__add__``) to get a + backwards compatible implementation. + (Github issue :issue:`2056`) + +* No/single argument functions now accept keyword arguments by default in order + to comply with Python semantics. The marginally faster calling conventions + ``METH_NOARGS`` and ``METH_O`` that reject keyword arguments are still available + with the directive ``@cython.always_allow_keywords(False)``. + (Github issue :issue:`3090`) + +* For-in-loop iteration over ``bytearray`` and memory views is optimised. + Patch by David Woods. (Github issue :issue:`2227`) + +* Type inference now works for memory views and slices. + Patch by David Woods. (Github issue :issue:`2227`) + +* The ``@returns()`` decorator propagates exceptions by default for suitable C + return types when no ``@exceptval()`` is defined. + (Github issues :issue:`3625`, :issue:`3664`) + +* A low-level inline function ``total_seconds(timedelta)`` was added to + ``cpython.datetime`` to bypass the Python method call. Note that this function + is not guaranteed to give exactly the same results for very large time intervals. + Patch by Brock Mendel. (Github issue :issue:`3616`) + +* Type inference now understands that ``a, *b = x`` assigns a list to ``b``. + +* Limited API support was improved. + Patches by Matthias Braun. (Github issues :issue:`3693`, :issue:`3707`) + +* The Cython ``CodeWriter`` can now handle more syntax constructs. + Patch by Tao He. (Github issue :issue:`3514`) + +Bugs fixed +---------- + +* The construct ``for x in cpp_function_call()`` failed to compile. + Patch by David Woods. (Github issue :issue:`3663`) + +* C++ references failed to compile when used as Python object indexes. + Patch by David Woods. (Github issue :issue:`3754`) + +* The C++ ``typeid()`` function was allowed in C mode. + Patch by Celelibi. (Github issue :issue:`3637`) + +* ``repr()`` was assumed to return ``str`` instead of ``unicode`` with ``language_level=3``. + (Github issue :issue:`3736`) + +* Includes all bug-fixes from the :ref:`0.29.21` release. + +Other changes +------------- + +* The ``numpy`` declarations were updated. + Patch by Brock Mendel. (Github issue :issue:`3630`) + +* The names of Cython's internal types (functions, generator, coroutine, etc.) + are now qualified with the module name of the internal Cython module that is + used for sharing them across Cython implemented modules, for example + ``_cython_3_0a5.coroutine``. This was done to avoid making them look like + homeless builtins, to help with debugging, and in order to avoid a CPython + warning according to https://bugs.python.org/issue20204 + +3.0.0 alpha 5 (2020-05-19) +========================== + +Features added +-------------- + +* ``.pxd`` files can now be :ref:`versioned <versioning>` by adding an + extension like "``.cython-30.pxd``" to prevent older Cython versions (than + 3.0 in this case) from picking them up. (Github issue :issue:`3577`) + +* Several macros/functions declared in the NumPy API are now usable without + holding the GIL. + +* `libc.math` was extended to include all C99 function declarations. + Patch by Dean Scarff. (Github issue :issue:`3570`) + +Bugs fixed +---------- + +* Several issues with arithmetic overflow handling were resolved, including + undefined behaviour in C. + Patch by Sam Sneddon. (Github issue :issue:`3588`) + +* The improved GIL handling in ``nogil`` functions introduced in 3.0a3 + could fail to acquire the GIL in some cases on function exit. + (Github issue :issue:`3590` etc.) + +* A reference leak when processing keyword arguments in Py2 was resolved, + that appeared in 3.0a1. + (Github issue :issue:`3578`) + +* The outdated getbuffer/releasebuffer implementations in the NumPy + declarations were removed so that buffers declared as ``ndarray`` + now use the normal implementation in NumPy. + +* Includes all bug-fixes from the :ref:`0.29.18` release. + + +3.0.0 alpha 4 (2020-05-05) +========================== + +Features added +-------------- + +* The ``print`` statement (not the ``print()`` function) is allowed in + ``nogil`` code without an explicit ``with gil`` section. + +* The ``assert`` statement is allowed in ``nogil`` sections. Here, the GIL is + only acquired if the ``AssertionError`` is really raised, which means that the + evaluation of the asserted condition only allows C expressions. + +* Cython generates C compiler branch hints for unlikely user defined if-clauses + in more cases, when they end up raising exceptions unconditionally. This now + includes exceptions being raised in ``nogil``/``with gil`` sections. + +* Some internal memoryview functions were tuned to reduce object overhead. + +Bugs fixed +---------- + +* Exception position reporting could run into race conditions on threaded code. + It now uses function-local variables again. + +* Error handling early in the module init code could lead to a crash. + +* Error handling in ``cython.array`` creation was improved to avoid calling + C-API functions with an error held. + +* Complex buffer item types of structs of arrays could fail to validate. + Patch by Leo and smutch. (Github issue :issue:`1407`) + +* When importing the old Cython ``build_ext`` integration with distutils, the + additional command line arguments leaked into the regular command. + Patch by Kamekameha. (Github issue :issue:`2209`) + +* The improved GIL handling in ``nogil`` functions introduced in 3.0a3 + could generate invalid C code. + (Github issue :issue:`3558`) + +* ``PyEval_InitThreads()`` is no longer used in Py3.7+ where it is a no-op. + +* Parallel builds of Cython itself (``setup.py build_ext -j N``) failed on Windows. + +Other changes +------------- + +* The C property feature has been rewritten and now requires C property methods + to be declared ``inline`` (:issue:`3571`). + + +3.0.0 alpha 3 (2020-04-27) +========================== + +Features added +-------------- + +* ``nogil`` functions now avoid acquiring the GIL on function exit if possible + even if they contain ``with gil`` blocks. + (Github issue :issue:`3554`) + +* Python private name mangling now falls back to unmangled names for non-Python + globals, since double-underscore names are not uncommon in C. Unmangled Python + names are also still found as a legacy fallback but produce a warning. + Patch by David Woods. (Github issue :issue:`3548`) + +Bugs fixed +---------- + +* Includes all bug-fixes from the :ref:`0.29.17` release. + + +3.0.0 alpha 2 (2020-04-23) +========================== + +Features added +-------------- + +* ``std::move()`` is now used in C++ mode for internal temp variables to + make them work without copying values. + Patch by David Woods. (Github issues :issue:`3253`, :issue:`1612`) + +* ``__class_getitem__`` is supported for types on item access (`PEP-560`_). + Patch by msg555. (Github issue :issue:`2753`) + +* The simplified Py3.6 customisation of class creation is implemented (`PEP-487`_). + (Github issue :issue:`2781`) + +* Conditional blocks in Python code that depend on ``cython.compiled`` are + eliminated at an earlier stage, which gives more freedom in writing + replacement Python code. + Patch by David Woods. (Github issue :issue:`3507`) + +* ``numpy.import_array()`` is automatically called if ``numpy`` has been cimported + and it has not been called in the module code. This is intended as a hidden + fail-safe so user code should continue to call ``numpy.import_array``. + Patch by David Woods. (Github issue :issue:`3524`) + +* The Cython AST code serialiser class ``CodeWriter`` in ``Cython.CodeWriter`` + supports more syntax nodes. + +* The fastcall/vectorcall protocols are used for several internal Python calls. + (Github issue :issue:`3540`) + +Bugs fixed +---------- + +* With ``language_level=3/3str``, Python classes without explicit base class + are now new-style (type) classes also in Py2. Previously, they were created + as old-style (non-type) classes. + (Github issue :issue:`3530`) + +* C++ ``typeid()`` failed for fused types. + Patch by David Woods. (Github issue :issue:`3203`) + +* ``__arg`` argument names in methods were not mangled with the class name. + Patch by David Woods. (Github issue :issue:`1382`) + +* Creating an empty unicode slice with large bounds could crash. + Patch by Sam Sneddon. (Github issue :issue:`3531`) + +* Decoding an empty bytes/char* slice with large bounds could crash. + Patch by Sam Sneddon. (Github issue :issue:`3534`) + +* Temporary buffer indexing variables were not released and could show up in + C compiler warnings, e.g. in generators. + Patch by David Woods. (Github issues :issue:`3430`, :issue:`3522`) + +* Several C compiler warnings were fixed. + + +3.0.0 alpha 1 (2020-04-12) +========================== + +Features added +-------------- + +* Cython functions now use the `PEP-590`_ vectorcall protocol in Py3.7+. + Patch by Jeroen Demeyer. (Github issue :issue:`2263`) + +* Unicode identifiers are supported in Cython code (`PEP-3131`_). + Patch by David Woods. (Github issue :issue:`2601`) + +* Unicode module names and imports are supported. + Patch by David Woods. (Github issue :issue:`3119`) + +* Annotations are no longer parsed, keeping them as strings following `PEP-563`_. + Patch by David Woods. (Github issue :issue:`3285`) + +* Preliminary support for the CPython's ``Py_LIMITED_API`` (stable ABI) is + available by setting the ``CYTHON_LIMITED_API`` C macro. Note that the + support is currently in an early stage and many features do not yet work. + You currently still have to define ``Py_LIMITED_API`` externally in order + to restrict the API usage. This will change when the feature stabilises. + Patches by Eddie Elizondo and David Woods. (Github issues :issue:`3223`, + :issue:`3311`, :issue:`3501`) + +* The dispatch to fused functions is now linear in the number of arguments, + which makes it much faster, often 2x or more, and several times faster for + larger fused types with many specialisations. + Patch by will-ca. (Github issue :issue:`1385`) + +* ``with gil/nogil`` statements can be conditional based on compile-time + constants, e.g. fused type checks. + Patch by Noam Hershtig. (Github issue :issue:`2579`) + +* ``const`` can be used together with fused types. + Patch by Thomas Vincent. (Github issue :issue:`1772`) + +* Reimports of already imported modules are substantially faster. + (Github issue :issue:`2854`) + +* Positional-only arguments are supported in Python functions (`PEP-570`_). + Patch by Josh Tobin. (Github issue :issue:`2915`) + +* The ``volatile`` C modifier is supported in Cython code. + Patch by Jeroen Demeyer. (Github issue :issue:`1667`) + +* ``@cython.trashcan(True)`` can be used on an extension type to enable the + CPython :ref:`trashcan`. This allows deallocating deeply recursive objects + without overflowing the stack. Patch by Jeroen Demeyer. (Github issue :issue:`2842`) + +* Inlined properties can be defined for external extension types. + Patch by Matti Picus. (Github issue :issue:`2640`, redone later in :issue:`3571`) + +* The ``str()`` builtin now calls ``PyObject_Str()`` instead of going + through a Python call. + Patch by William Ayd. (Github issue :issue:`3279`) + +* String concatenation can now happen in place if possible, by extending the + existing string rather than always creating a new one. + Patch by David Woods. (Github issue :issue:`3453`) + +* Multiplication of Python numbers with small constant integers is faster. + (Github issue :issue:`2808`) + +* Some list copying is avoided internally when a new list needs to be created + but we already have a fresh one. + (Github issue :issue:`3494`) + +* Extension types that do not need their own ``tp_new`` implementation (because + they have no object attributes etc.) directly inherit the implementation of + their parent type if possible. + (Github issue :issue:`1555`) + +* The attributes ``gen.gi_frame`` and ``coro.cr_frame`` of Cython compiled + generators and coroutines now return an actual frame object for introspection. + (Github issue :issue:`2306`) + +* Several declarations in ``cpython.*``, ``libc.*`` and ``libcpp.*`` were added. + Patches by Jeroen Demeyer, Matthew Edwards, Chris Gyurgyik, Jerome Kieffer + and Zackery Spytz. + (Github issues :issue:`3468`, :issue:`3332`, :issue:`3202`, :issue:`3188`, + :issue:`3179`, :issue:`2891`, :issue:`2826`, :issue:`2713`) + +* Deprecated NumPy API usages were removed from ``numpy.pxd``. + Patch by Matti Picus. (Github issue :issue:`3365`) + +* ``cython.inline()`` now sets the ``NPY_NO_DEPRECATED_API=NPY_1_7_API_VERSION`` + C macro automatically when ``numpy`` is imported in the code, to avoid C compiler + warnings about deprecated NumPy C-API usage. + +* The builtin ``abs()`` function can now be used on C numbers in nogil code. + Patch by Elliott Sales de Andrade. (Github issue :issue:`2748`) + +* `PEP-479`_ (``generator_stop``) is now enabled by default with language level 3. + (Github issue :issue:`2580`) + +* The ``cython.view.array`` type supports inheritance. + Patch by David Woods. (Github issue :issue:`3413`) + +* Code annotation accepts a new debugging argument ``--annotate-fullc`` that + will include the complete syntax highlighted C file in the HTML output. + (Github issue :issue:`2855`) + +* ``--no-capture`` added to ``runtests.py`` to prevent stdout/stderr capturing + during srctree tests. + Patch by Matti Picus. (Github issue :issue:`2701`) + +* ``--no-docstrings`` option added to ``cythonize`` script. + Original patch by mo-han. (Github issue :issue:`2889`) + +* ``cygdb`` gives better error messages when it fails to initialise the + Python runtime support in gdb. + Patch by Volker Weissmann. (Github issue :issue:`3489`) + +* The Pythran ``shape`` attribute is supported. + Patch by Serge Guelton. (Github issue :issue:`3307`) + +Bugs fixed +---------- + +* The unicode methods ``.upper()``, ``.lower()`` and ``.title()`` were + incorrectly optimised for single character input values and only returned + the first character if multiple characters should have been returned. + They now use the original Python methods again. + +* Fused argument types were not correctly handled in type annotations and + ``cython.locals()``. + Patch by David Woods. (Github issues :issue:`3391`, :issue:`3142`) + +* Diverging from the usual behaviour, ``len(memoryview)``, ``len(char*)`` + and ``len(Py_UNICODE*)`` returned an unsigned ``size_t`` value. They now + return a signed ``Py_ssize_t``, like other usages of ``len()``. + +* Nested dict literals in function call kwargs could incorrectly raise an + error about duplicate keyword arguments, which are allowed when passing + them from dict literals. + (Github issue :issue:`2963`) + +* Item access (subscripting) with integer indices/keys always tried the + Sequence protocol before the Mapping protocol, which diverged from Python + semantics. It now passes through the Mapping protocol first when supported. + (Github issue :issue:`1807`) + +* Name lookups in class bodies no longer go through an attribute lookup. + Patch by Jeroen Demeyer. (Github issue :issue:`3100`) + +* Broadcast assignments to a multi-dimensional memory view slice could end + up in the wrong places when the underlying memory view is known to be + contiguous but the slice is not. + (Github issue :issue:`2941`) + +* Pickling unbound methods of Python classes failed. + Patch by Pierre Glaser. (Github issue :issue:`2972`) + +* The ``Py_hash_t`` type failed to accept arbitrary "index" values. + (Github issue :issue:`2752`) + +* The first function line number of functions with decorators pointed to the + signature line and not the first decorator line, as in Python. + Patch by Felix Kohlgrüber. (Github issue :issue:`2536`) + +* Constant integer expressions that used a negative exponent were evaluated + as integer 0 instead of the expected float value. + Patch by Kryštof Pilnáček. (Github issue :issue:`2133`) + +* The ``cython.declare()`` and ``cython.cast()`` functions could fail in pure mode. + Patch by Dmitry Shesterkin. (Github issue :issue:`3244`) + +* ``__doc__`` was not available inside of the class body during class creation. + (Github issue :issue:`1635`) + +* Setting ``language_level=2`` in a file did not work if ``language_level=3`` + was enabled globally before. + Patch by Jeroen Demeyer. (Github issue :issue:`2791`) + +* ``__init__.pyx`` files were not always considered as package indicators. + (Github issue :issue:`2665`) + +* Compiling package ``__init__`` files could fail under Windows due to an + undefined export symbol. (Github issue :issue:`2968`) + +* A C compiler cast warning was resolved. + Patch by Michael Buesch. (Github issue :issue:`2775`) + +* Binding staticmethods of Cython functions were not behaving like Python methods. + Patch by Jeroen Demeyer. (Github issue :issue:`3106`, :issue:`3102`) + +* Memoryviews failed to compile when the ``cache_builtins`` feature was disabled. + Patch by David Woods. (Github issue :issue:`3406`) + +Other changes +------------- + +* The default language level was changed to ``3str``, i.e. Python 3 semantics, + but with ``str`` literals (also in Python 2.7). This is a backwards incompatible + change from the previous default of Python 2 semantics. The previous behaviour + is available through the directive ``language_level=2``. + (Github issue :issue:`2565`) + +* Cython no longer generates ``__qualname__`` attributes for classes in Python + 2.x since they are problematic there and not correctly maintained for subclasses. + Patch by Jeroen Demeyer. (Github issue :issue:`2772`) + +* Source file fingerprinting now uses SHA-1 instead of MD5 since the latter + tends to be slower and less widely supported these days. + (Github issue :issue:`2790`) + +* The long deprecated include files ``python_*``, ``stdio``, ``stdlib`` and + ``stl`` in ``Cython/Includes/Deprecated/`` were removed. Use the ``libc.*`` + and ``cpython.*`` pxd modules instead. + Patch by Jeroen Demeyer. (Github issue :issue:`2904`) + +* The search order for include files was changed. Previously it was + ``include_directories``, ``Cython/Includes``, ``sys.path``. Now it is + ``include_directories``, ``sys.path``, ``Cython/Includes``. This was done to + allow third-party ``*.pxd`` files to override the ones in Cython. + Patch by Matti Picus. (Github issue :issue:`2905`) + +* The command line parser was rewritten and modernised using ``argparse``. + Patch by Egor Dranischnikow. (Github issue :issue:`2952`, :issue:`3001`) + +* Dotted filenames for qualified module names (``pkg.mod.pyx``) are deprecated. + Use the normal Python package directory layout instead. + (Github issue :issue:`2686`) + +* Binary Linux wheels now follow the manylinux2010 standard. + Patch by Alexey Stepanov. (Github issue :issue:`3355`) + +* Support for Python 2.6 was removed. + +.. _`PEP-560`: https://www.python.org/dev/peps/pep-0560 +.. _`PEP-570`: https://www.python.org/dev/peps/pep-0570 +.. _`PEP-487`: https://www.python.org/dev/peps/pep-0487 +.. _`PEP-590`: https://www.python.org/dev/peps/pep-0590 +.. _`PEP-3131`: https://www.python.org/dev/peps/pep-3131 +.. _`PEP-563`: https://www.python.org/dev/peps/pep-0563 +.. _`PEP-479`: https://www.python.org/dev/peps/pep-0479 + + +.. _0.29.24: + 0.29.24 (2021-07-14) ==================== @@ -10,32 +773,34 @@ Bugs fixed * Inline functions in pxd files that used memory views could lead to invalid C code if the module that imported from them does not use memory views. - Patch by David Woods. (Github issue #1415) + Patch by David Woods. (Github issue :issue:`1415`) * Several declarations in ``libcpp.string`` were added and corrected. - Patch by Janek Bevendorff. (Github issue #4268) + Patch by Janek Bevendorff. (Github issue :issue:`4268`) * Pickling unbound Cython compiled methods failed. - Patch by Pierre Glaser. (Github issue #2972) + Patch by Pierre Glaser. (Github issue :issue:`2972`) * The tracing code was adapted to work with CPython 3.10. * The optimised ``in`` operator failed on unicode strings in Py3.9 and later that were constructed from an external ``wchar_t`` source. Also, related C compiler warnings about deprecated C-API usage were resolved. - (Github issue #3925) + (Github issue :issue:`3925`) * Some compiler crashes were resolved. - Patch by David Woods. (Github issues #4214, #2811) + Patch by David Woods. (Github issues :issue:`4214`, :issue:`2811`) * An incorrect warning about 'unused' generator expressions was removed. - (GIthub issue #1699) + (GIthub issue :issue:`1699`) * The attributes ``gen.gi_frame`` and ``coro.cr_frame`` of Cython compiled generators and coroutines now return an actual frame object for introspection, instead of ``None``. - (Github issue #2306) + (Github issue :issue:`2306`) + +.. _0.29.23: 0.29.23 (2021-04-14) ==================== @@ -44,19 +809,21 @@ Bugs fixed ---------- * Some problems with Python 3.10 were resolved. - Patches by Victor Stinner and David Woods. (Github issues #4046, #4100) + Patches by Victor Stinner and David Woods. (Github issues :issue:`4046`, :issue:`4100`) * An incorrect "optimisation" was removed that allowed changes to a keyword dict to leak into keyword arguments passed into a function. - Patch by Peng Weikang. (Github issue #3227) + Patch by Peng Weikang. (Github issue :issue:`3227`) * Multiplied str constants could end up as bytes constants with language_level=2. - Patch by Alphadelta14 and David Woods. (Github issue #3951) + Patch by Alphadelta14 and David Woods. (Github issue :issue:`3951`) * ``PY_SSIZE_T_CLEAN`` does not get defined any more if it is already defined. - Patch by Andrew Jones. (Github issue #4104) + Patch by Andrew Jones. (Github issue :issue:`4104`) +.. _0.29.22: + 0.29.22 (2021-02-20) ==================== @@ -65,41 +832,41 @@ Features added * Some declarations were added to the provided pxd includes. Patches by Zackery Spytz and John Kirkham. - (Github issues #3811, #3882, #3899, #3901) + (Github issues :issue:`3811`, :issue:`3882`, :issue:`3899`, :issue:`3901`) Bugs fixed ---------- * A crash when calling certain functions in Py3.9 and later was resolved. - (Github issue #3917) + (Github issue :issue:`3917`) * ``const`` memory views of structs failed to compile. - (Github issue #2251) + (Github issue :issue:`2251`) * ``const`` template declarations could not be nested. - Patch by Ashwin Srinath. (Github issue #1355) + Patch by Ashwin Srinath. (Github issue :issue:`1355`) * The declarations in the ``cpython.pycapsule`` module were missing their ``const`` modifiers and generated incorrect C code. - Patch by Warren Weckesser. (Github issue #3964) + Patch by Warren Weckesser. (Github issue :issue:`3964`) * Casts to memory views failed for fused dtypes. - Patch by David Woods. (Github issue #3881) + Patch by David Woods. (Github issue :issue:`3881`) * ``repr()`` was assumed to return ``str`` instead of ``unicode`` with ``language_level=3``. - (Github issue #3736) + (Github issue :issue:`3736`) * Calling ``cpdef`` functions from cimported modules crashed the compiler. - Patch by David Woods. (Github issue #4000) + Patch by David Woods. (Github issue :issue:`4000`) * Cython no longer validates the ABI size of the NumPy classes it compiled against. See the discussion in https://github.com/numpy/numpy/pull/432 * A C compiler warning about enum value casting was resolved in GCC. - (Github issue #2749) + (Github issue :issue:`2749`) * Coverage reporting in the annotated HTML file failed in Py3.9. - Patch by Nick Pope. (Github issue #3865) + Patch by Nick Pope. (Github issue :issue:`3865`) * The embedding code now reports Python errors as exit status. @@ -111,8 +878,10 @@ Other changes * Variables defined as ``cpdef`` now generate a warning since this is currently useless and thus does not do what users would expect. - Patch by David Woods. (Github issue #3959) + Patch by David Woods. (Github issue :issue:`3959`) + +.. _0.29.21: 0.29.21 (2020-07-09) ==================== @@ -121,39 +890,42 @@ Bugs fixed ---------- * Fix a regression in 0.29.20 where ``__div__`` failed to be found in extension types. - (Github issue #3688) + (Github issue :issue:`3688`) * Fix a regression in 0.29.20 where a call inside of a finally clause could fail to compile. - Patch by David Woods. (Github issue #3712) + Patch by David Woods. (Github issue :issue:`3712`) * Zero-sized buffers could fail to validate as C/Fortran-contiguous. - Patch by Clemens Hofreither. (Github issue #2093) + Patch by Clemens Hofreither. (Github issue :issue:`2093`) * ``exec()`` did not allow recent Python syntax features in Py3.8+ due to https://bugs.python.org/issue35975. - (Github issue #3695) + (Github issue :issue:`3695`) * Binding staticmethods of Cython functions were not behaving like Python methods in Py3. - Patch by Jeroen Demeyer and Michał Górny. (Github issue #3106) + Patch by Jeroen Demeyer and Michał Górny. (Github issue :issue:`3106`) * Pythran calls to NumPy methods no longer generate useless method lookup code. * The ``PyUnicode_GET_LENGTH()`` macro was missing from the ``cpython.*`` declarations. - Patch by Thomas Caswell. (Github issue #3692) + Patch by Thomas Caswell. (Github issue :issue:`3692`) * The deprecated ``PyUnicode_*()`` C-API functions are no longer used, except for Unicode strings that contain lone surrogates. Unicode strings that contain non-BMP characters or surrogate pairs now generate different C code on 16-bit Python 2.x Unicode deployments (such as MS-Windows). Generating the C code on Python 3.x is recommended in this case. - Original patches by Inada Naoki and Victor Stinner. (Github issues #3677, #3721, #3697) + Original patches by Inada Naoki and Victor Stinner. + (Github issues :issue:`3677`, :issue:`3721`, :issue:`3697`) * Some template parameters were missing from the C++ ``std::unordered_map`` declaration. - Patch by will. (Github issue #3685) + Patch by will. (Github issue :issue:`3685`) * Several internal code generation issues regarding temporary variables were resolved. - (Github issue #3708) + (Github issue :issue:`3708`) +.. _0.29.20: + 0.29.20 (2020-06-10) ==================== @@ -162,36 +934,38 @@ Bugs fixed * Nested try-except statements with multiple ``return`` statements could crash due to incorrect deletion of the ``except as`` target variable. - (Github issue #3666) + (Github issue :issue:`3666`) * The ``@classmethod`` decorator no longer rejects unknown input from other decorators. - Patch by David Woods. (Github issue #3660) + Patch by David Woods. (Github issue :issue:`3660`) * Fused types could leak into unrelated usages. - Patch by David Woods. (Github issue #3642) + Patch by David Woods. (Github issue :issue:`3642`) * Now uses ``Py_SET_SIZE()`` and ``Py_SET_REFCNT()`` in Py3.9+ to avoid low-level write access to these object fields. - Patch by Victor Stinner. (Github issue #3639) + Patch by Victor Stinner. (Github issue :issue:`3639`) * The built-in ``abs()`` function could lead to undefined behaviour when used on the negative-most value of a signed C integer type. - Patch by Serge Guelton. (Github issue #1911) + Patch by Serge Guelton. (Github issue :issue:`1911`) * Usages of ``sizeof()`` and ``typeid()`` on uninitialised variables no longer produce a warning. - Patch by Celelibi. (Github issue #3575) + Patch by Celelibi. (Github issue :issue:`3575`) * The C++ ``typeid()`` function was allowed in C mode. - Patch by Celelibi. (Github issue #3637) + Patch by Celelibi. (Github issue :issue:`3637`) * The error position reported for errors found in f-strings was misleading. - (Github issue #3674) + (Github issue :issue:`3674`) * The new ``c_api_binop_methods`` directive was added for forward compatibility, but can only be set to True (the current default value). It can be disabled in Cython 3.0. +.. _0.29.19: + 0.29.19 (2020-05-20) ==================== @@ -199,11 +973,13 @@ Bugs fixed ---------- * A typo in Windows specific code in 0.29.18 was fixed that broke "libc.math". - (Github issue #3622) + (Github issue :issue:`3622`) * A platform specific test failure in 0.29.18 was fixed. - Patch by smutch. (Github issue #3620) + Patch by smutch. (Github issue :issue:`3620`) + +.. _0.29.18: 0.29.18 (2020-05-18) ==================== @@ -221,45 +997,47 @@ Bugs fixed * A memory corruption was fixed when garbage collection was triggered during calls to ``PyType_Ready()`` of extension type subclasses. - (Github issue #3603) + (Github issue :issue:`3603`) * Memory view slicing generated unused error handling code which could negatively impact the C compiler optimisations for parallel OpenMP code etc. Also, it is now helped by static branch hints. - (Github issue #2987) + (Github issue :issue:`2987`) * Cython's built-in OpenMP functions were not translated inside of call arguments. - Original patch by Celelibi and David Woods. (Github issue #3594) + Original patch by Celelibi and David Woods. (Github issue :issue:`3594`) * Complex buffer item types of structs of arrays could fail to validate. - Patch by Leo and smutch. (Github issue #1407) + Patch by Leo and smutch. (Github issue :issue:`1407`) * Decorators were not allowed on nested `async def` functions. - (Github issue #1462) + (Github issue :issue:`1462`) * C-tuples could use invalid C struct casting. - Patch by MegaIng. (Github issue #3038) + Patch by MegaIng. (Github issue :issue:`3038`) * Optimised ``%d`` string formatting into f-strings failed on float values. - (Github issue #3092) + (Github issue :issue:`3092`) * Optimised aligned string formatting (``%05s``, ``%-5s``) failed. - (Github issue #3476) + (Github issue :issue:`3476`) * When importing the old Cython ``build_ext`` integration with distutils, the additional command line arguments leaked into the regular command. - Patch by Kamekameha. (Github issue #2209) + Patch by Kamekameha. (Github issue :issue:`2209`) * When using the ``CYTHON_NO_PYINIT_EXPORT`` option in C++, the module init function was not declared as ``extern "C"``. - (Github issue #3414) + (Github issue :issue:`3414`) * Three missing timedelta access macros were added in ``cpython.datetime``. * The signature of the NumPy C-API function ``PyArray_SearchSorted()`` was fixed. - Patch by Brock Mendel. (Github issue #3606) + Patch by Brock Mendel. (Github issue :issue:`3606`) +.. _0.29.17: + 0.29.17 (2020-04-26) ==================== @@ -267,42 +1045,44 @@ Features added -------------- * ``std::move()`` is now available from ``libcpp.utility``. - Patch by Omer Ozarslan. (Github issue #2169) + Patch by Omer Ozarslan. (Github issue :issue:`2169`) * The ``@cython.binding`` decorator is available in Python code. - (Github issue #3505) + (Github issue :issue:`3505`) Bugs fixed ---------- * Creating an empty unicode slice with large bounds could crash. - Patch by Sam Sneddon. (Github issue #3531) + Patch by Sam Sneddon. (Github issue :issue:`3531`) * Decoding an empty bytes/char* slice with large bounds could crash. - Patch by Sam Sneddon. (Github issue #3534) + Patch by Sam Sneddon. (Github issue :issue:`3534`) * Re-importing a Cython extension no longer raises the error "``__reduce_cython__ not found``". - (Github issue #3545) + (Github issue :issue:`3545`) * Unused C-tuples could generate incorrect code in 0.29.16. - Patch by Kirk Meyer. (Github issue #3543) + Patch by Kirk Meyer. (Github issue :issue:`3543`) * Creating a fused function attached it to the garbage collector before it was fully initialised, thus risking crashes in rare failure cases. - Original patch by achernomorov. (Github issue #3215) + Original patch by achernomorov. (Github issue :issue:`3215`) * Temporary buffer indexing variables were not released and could show up in C compiler warnings, e.g. in generators. - Patch by David Woods. (Github issues #3430, #3522) + Patch by David Woods. (Github issues :issue:`3430`, :issue:`3522`) * The compilation cache in ``cython.inline("…")`` failed to take the language level into account. - Patch by will-ca. (Github issue #3419) + Patch by will-ca. (Github issue :issue:`3419`) * The deprecated ``PyUnicode_GET_SIZE()`` function is no longer used in Py3. +.. _0.29.16: + 0.29.16 (2020-03-24) ==================== @@ -310,50 +1090,53 @@ Bugs fixed ---------- * Temporary internal variables in nested prange loops could leak into other - threads. Patch by Frank Schlimbach. (Github issue #3348) + threads. Patch by Frank Schlimbach. (Github issue :issue:`3348`) * Default arguments on fused functions could crash. - Patch by David Woods. (Github issue #3370) + Patch by David Woods. (Github issue :issue:`3370`) * C-tuples declared in ``.pxd`` files could generate incomplete C code. - Patch by Kirk Meyer. (Github issue #1427) + Patch by Kirk Meyer. (Github issue :issue:`1427`) * Fused functions were not always detected and optimised as Cython implemented functions. - Patch by David Woods. (Github issue #3384) + Patch by David Woods. (Github issue :issue:`3384`) * Valid Python object concatenation of (iterable) strings to non-strings could fail with an exception. - Patch by David Woods. (Github issue #3433) + Patch by David Woods. (Github issue :issue:`3433`) * Using C functions as temporary values lead to invalid C code. - Original patch by David Woods. (Github issue #3418) + Original patch by David Woods. (Github issue :issue:`3418`) * Fix an unhandled C++ exception in comparisons. - Patch by David Woods. (Github issue #3361) + Patch by David Woods. (Github issue :issue:`3361`) * Fix deprecated import of "imp" module. - Patch by Matti Picus. (Github issue #3350) + Patch by Matti Picus. (Github issue :issue:`3350`) * Fix compatibility with Pythran 0.9.6 and later. - Patch by Serge Guelton. (Github issue #3308) + Patch by Serge Guelton. (Github issue :issue:`3308`) * The ``_Py_PyAtExit()`` function in ``cpython.pylifecycle`` was misdeclared. - Patch by Zackery Spytz. (Github issue #3382) + Patch by Zackery Spytz. (Github issue :issue:`3382`) * Several missing declarations in ``cpython.*`` were added. - Patches by Zackery Spytz. (Github issue #3452, #3421, #3411, #3402) + Patches by Zackery Spytz. (Github issue :issue:`3452`, :issue:`3421`, :issue:`3411`, :issue:`3402`) * A declaration for ``libc.math.fpclassify()`` was added. - Patch by Zackery Spytz. (Github issue #2514) + Patch by Zackery Spytz. (Github issue :issue:`2514`) * Avoid "undeclared" warning about automatically generated pickle methods. - Patch by David Woods. (Github issue #3353) + Patch by David Woods. (Github issue :issue:`3353`) * Avoid C compiler warning about unreachable code in ``prange()``. * Some C compiler warnings in PyPy were resolved. - Patch by Matti Picus. (Github issue #3437) + Patch by Matti Picus. (Github issue :issue:`3437`) + + +.. _0.29.15: 0.29.15 (2020-02-06) @@ -363,27 +1146,29 @@ Bugs fixed ---------- * Crash when returning a temporary Python object from an async-def function. - (Github issue #3337) + (Github issue :issue:`3337`) * Crash when using ``**kwargs`` in generators. - Patch by David Woods. (Github issue #3265) + Patch by David Woods. (Github issue :issue:`3265`) * Double reference free in ``__class__`` cell handling for ``super()`` calls. - (Github issue #3246) + (Github issue :issue:`3246`) * Compile error when using ``*args`` as Python class bases. - (Github issue #3338) + (Github issue :issue:`3338`) * Import failure in IPython 7.11. - (Github issue #3297) + (Github issue :issue:`3297`) * Fixed C name collision in the auto-pickle code. - Patch by ThePrez. (Github issue #3238) + Patch by ThePrez. (Github issue :issue:`3238`) * Deprecated import failed in Python 3.9. - (Github issue #3266) + (Github issue :issue:`3266`) +.. _0.29.14: + 0.29.14 (2019-11-01) ==================== @@ -391,36 +1176,36 @@ Bugs fixed ---------- * The generated code failed to initialise the ``tp_print`` slot in CPython 3.8. - Patches by Pablo Galindo and Orivej Desh. (Github issues #3171, #3201) + Patches by Pablo Galindo and Orivej Desh. (Github issues :issue:`3171`, :issue:`3201`) * ``?`` for ``bool`` was missing from the supported NumPy dtypes. - Patch by Max Klein. (Github issue #2675) + Patch by Max Klein. (Github issue :issue:`2675`) * ``await`` was not allowed inside of f-strings. - Patch by Dmitro Getz. (Github issue #2877) + Patch by Dmitro Getz. (Github issue :issue:`2877`) * Coverage analysis failed for projects where the code resides in separate source sub-directories. - Patch by Antonio Valentino. (Github issue #1985) + Patch by Antonio Valentino. (Github issue :issue:`1985`) * An incorrect compiler warning was fixed in automatic C++ string conversions. - Patch by Gerion Entrup. (Github issue #3108) + Patch by Gerion Entrup. (Github issue :issue:`3108`) * Error reports in the Jupyter notebook showed unhelpful stack traces. - Patch by Matthew Edwards (Github issue #3196). + Patch by Matthew Edwards (Github issue :issue:`3196`). * ``Python.h`` is now also included explicitly from ``public`` header files. - (Github issue #3133). + (Github issue :issue:`3133`). * Distutils builds with ``--parallel`` did not work when using Cython's deprecated ``build_ext`` command. - Patch by Alphadelta14 (Github issue #3187). + Patch by Alphadelta14 (Github issue :issue:`3187`). Other changes ------------- * The ``PyMemoryView_*()`` C-API is available in ``cpython.memoryview``. - Patch by Nathan Manville. (Github issue #2541) + Patch by Nathan Manville. (Github issue :issue:`2541`) 0.29.13 (2019-07-26) @@ -430,17 +1215,16 @@ Bugs fixed ---------- * A reference leak for ``None`` was fixed when converting a memoryview - to a Python object. (Github issue #3023) + to a Python object. (Github issue :issue:`3023`) * The declaration of ``PyGILState_STATE`` in ``cpython.pystate`` was unusable. - Patch by Kirill Smelkov. (Github issue #2997) - + Patch by Kirill Smelkov. (Github issue :issue:`2997`) Other changes ------------- * The declarations in ``posix.mman`` were extended. - Patches by Kirill Smelkov. (Github issues #2893, #2894, #3012) + Patches by Kirill Smelkov. (Github issues :issue:`2893`, :issue:`2894`, :issue:`3012`) 0.29.12 (2019-07-07) @@ -450,16 +1234,16 @@ Bugs fixed ---------- * Fix compile error in CPython 3.8b2 regarding the ``PyCode_New()`` signature. - (Github issue #3031) + (Github issue :issue:`3031`) * Fix a C compiler warning about a missing ``int`` downcast. - (Github issue #3028) + (Github issue :issue:`3028`) * Fix reported error positions of undefined builtins and constants. - Patch by Orivej Desh. (Github issue #3030) + Patch by Orivej Desh. (Github issue :issue:`3030`) * A 32 bit issue in the Pythran support was resolved. - Patch by Serge Guelton. (Github issue #3032) + Patch by Serge Guelton. (Github issue :issue:`3032`) 0.29.11 (2019-06-30) @@ -469,26 +1253,26 @@ Bugs fixed ---------- * Fix compile error in CPython 3.8b2 regarding the ``PyCode_New()`` signature. - Patch by Nick Coghlan. (Github issue #3009) + Patch by Nick Coghlan. (Github issue :issue:`3009`) * Invalid C code generated for lambda functions in cdef methods. - Patch by Josh Tobin. (Github issue #2967) + Patch by Josh Tobin. (Github issue :issue:`2967`) * Support slice handling in newer Pythran versions. - Patch by Serge Guelton. (Github issue #2989) + Patch by Serge Guelton. (Github issue :issue:`2989`) * A reference leak in power-of-2 calculation was fixed. - Patch by Sebastian Berg. (Github issue #3022) + Patch by Sebastian Berg. (Github issue :issue:`3022`) * The search order for include files was changed. Previously it was ``include_directories``, ``Cython/Includes``, ``sys.path``. Now it is ``include_directories``, ``sys.path``, ``Cython/Includes``. This was done to allow third-party ``*.pxd`` files to override the ones in Cython. - Original patch by Matti Picus. (Github issue #2905) + Original patch by Matti Picus. (Github issue :issue:`2905`) * Setting ``language_level=2`` in a file did not work if ``language_level=3`` was enabled globally before. - Patch by Jeroen Demeyer. (Github issue #2791) + Patch by Jeroen Demeyer. (Github issue :issue:`2791`) 0.29.10 (2019-06-02) @@ -498,7 +1282,7 @@ Bugs fixed ---------- * Fix compile errors in CPython 3.8b1 due to the new "tp_vectorcall" slots. - (Github issue #2976) + (Github issue :issue:`2976`) 0.29.9 (2019-05-29) @@ -510,7 +1294,7 @@ Bugs fixed * Fix a crash regression in 0.29.8 when creating code objects fails. * Remove an incorrect cast when using true-division in C++ operations. - (Github issue #1950) + (Github issue :issue:`1950`) 0.29.8 (2019-05-28) @@ -520,20 +1304,20 @@ Bugs fixed ---------- * C compile errors with CPython 3.8 were resolved. - Patch by Marcel Plch. (Github issue #2938) + Patch by Marcel Plch. (Github issue :issue:`2938`) * Python tuple constants that compare equal but have different item types could incorrectly be merged into a single constant. - (Github issue #2919) + (Github issue :issue:`2919`) * Non-ASCII characters in unprefixed strings could crash the compiler when used with language level ``3str``. * Starred expressions in %-formatting tuples could fail to compile for - unicode strings. (Github issue #2939) + unicode strings. (Github issue :issue:`2939`) * Passing Python class references through ``cython.inline()`` was broken. - (Github issue #2936) + (Github issue :issue:`2936`) 0.29.7 (2019-04-14) @@ -545,18 +1329,18 @@ Bugs fixed * Crash when the shared Cython config module gets unloaded and another Cython module reports an exceptions. Cython now makes sure it keeps an owned reference to the module. - (Github issue #2885) + (Github issue :issue:`2885`) * Resolved a C89 compilation problem when enabling the fast-gil sharing feature. * Coverage reporting did not include the signature line of ``cdef`` functions. - (Github issue #1461) + (Github issue :issue:`1461`) * Casting a GIL-requiring function into a nogil function now issues a warning. - (Github issue #2879) + (Github issue :issue:`2879`) * Generators and coroutines were missing their return type annotation. - (Github issue #2884) + (Github issue :issue:`2884`) 0.29.6 (2019-02-27) @@ -566,17 +1350,17 @@ Bugs fixed ---------- * Fix a crash when accessing the ``__kwdefaults__`` special attribute of - fused functions. (Github issue #1470) + fused functions. (Github issue :issue:`1470`) * Fix the parsing of buffer format strings that contain numeric sizes, which - could lead to incorrect input rejections. (Github issue #2845) + could lead to incorrect input rejections. (Github issue :issue:`2845`) * Avoid a C #pragma in old gcc versions that was only added in GCC 4.6. - Patch by Michael Anselmi. (Github issue #2838) + Patch by Michael Anselmi. (Github issue :issue:`2838`) * Auto-encoding of Unicode strings to UTF-8 C/C++ strings failed in Python 3, even though the default encoding there is UTF-8. - (Github issue #2819) + (Github issue :issue:`2819`) 0.29.5 (2019-02-09) @@ -586,16 +1370,16 @@ Bugs fixed ---------- * Crash when defining a Python subclass of an extension type and repeatedly calling - a cpdef method on it. (Github issue #2823) + a cpdef method on it. (Github issue :issue:`2823`) * Compiler crash when ``prange()`` loops appear inside of with-statements. - (Github issue #2780) + (Github issue :issue:`2780`) * Some C compiler warnings were resolved. - Patches by Christoph Gohlke. (Github issues #2815, #2816, #2817, #2822) + Patches by Christoph Gohlke. (Github issues :issue:`2815`, :issue:`2816`, :issue:`2817`, :issue:`2822`) * Python conversion of C++ enums failed in 0.29. - Patch by Orivej Desh. (Github issue #2767) + Patch by Orivej Desh. (Github issue :issue:`2767`) 0.29.4 (2019-02-01) @@ -605,7 +1389,7 @@ Bugs fixed ---------- * Division of numeric constants by a runtime value of 0 could fail to raise a - ``ZeroDivisionError``. (Github issue #2820) + ``ZeroDivisionError``. (Github issue :issue:`2820`) 0.29.3 (2019-01-19) @@ -615,19 +1399,19 @@ Bugs fixed ---------- * Some C code for memoryviews was generated in a non-deterministic order. - Patch by Martijn van Steenbergen. (Github issue #2779) + Patch by Martijn van Steenbergen. (Github issue :issue:`2779`) * C89 compatibility was accidentally lost since 0.28. - Patches by gastineau and true-pasky. (Github issues #2778, #2801) + Patches by gastineau and true-pasky. (Github issues :issue:`2778`, :issue:`2801`) * A C compiler cast warning was resolved. - Patch by Michael Buesch. (Github issue #2774) + Patch by Michael Buesch. (Github issue :issue:`2774`) * An compilation failure with complex numbers under MSVC++ was resolved. - (Github issue #2797) + (Github issue :issue:`2797`) * Coverage reporting could fail when modules were moved around after the build. - Patch by Wenjun Si. (Github issue #2776) + Patch by Wenjun Si. (Github issue :issue:`2776`) 0.29.2 (2018-12-14) @@ -637,16 +1421,16 @@ Bugs fixed ---------- * The code generated for deduplicated constants leaked some references. - (Github issue #2750) + (Github issue :issue:`2750`) * The declaration of ``sigismember()`` in ``libc.signal`` was corrected. - (Github issue #2756) + (Github issue :issue:`2756`) * Crashes in compiler and test runner were fixed. - (Github issue #2736, #2755) + (Github issue :issue:`2736`, :issue:`2755`) * A C compiler warning about an invalid safety check was resolved. - (Github issue #2731) + (Github issue :issue:`2731`) 0.29.1 (2018-11-24) @@ -657,47 +1441,47 @@ Bugs fixed * Extensions compiled with MinGW-64 under Windows could misinterpret integer objects larger than 15 bit and return incorrect results. - (Github issue #2670) + (Github issue :issue:`2670`) * Cython no longer requires the source to be writable when copying its data into a memory view slice. - Patch by Andrey Paramonov. (Github issue #2644) + Patch by Andrey Paramonov. (Github issue :issue:`2644`) * Line tracing of ``try``-statements generated invalid C code. - (Github issue #2274) + (Github issue :issue:`2274`) * When using the ``warn.undeclared`` directive, Cython's own code generated warnings that are now fixed. - Patch by Nicolas Pauss. (Github issue #2685) + Patch by Nicolas Pauss. (Github issue :issue:`2685`) * Cython's memoryviews no longer require strides for setting the shape field but only the ``PyBUF_ND`` flag to be set. - Patch by John Kirkham. (Github issue #2716) + Patch by John Kirkham. (Github issue :issue:`2716`) * Some C compiler warnings about unused memoryview code were fixed. - Patch by Ho Cheuk Ting. (Github issue #2588) + Patch by Ho Cheuk Ting. (Github issue :issue:`2588`) * A C compiler warning about implicit signed/unsigned conversion was fixed. - (Github issue #2729) + (Github issue :issue:`2729`) * Assignments to C++ references returned by ``operator[]`` could fail to compile. - (Github issue #2671) + (Github issue :issue:`2671`) * The power operator and the support for NumPy math functions were fixed in Pythran expressions. - Patch by Serge Guelton. (Github issues #2702, #2709) + Patch by Serge Guelton. (Github issues :issue:`2702`, :issue:`2709`) * Signatures with memory view arguments now show the expected type when embedded in docstrings. - Patch by Matthew Chan and Benjamin Weigel. (Github issue #2634) + Patch by Matthew Chan and Benjamin Weigel. (Github issue :issue:`2634`) * Some ``from ... cimport ...`` constructs were not correctly considered when searching modified dependencies in ``cythonize()`` to decide whether to recompile a module. - Patch by Kryštof Pilnáček. (Github issue #2638) + Patch by Kryštof Pilnáček. (Github issue :issue:`2638`) * A struct field type in the ``cpython.array`` declarations was corrected. - Patch by John Kirkham. (Github issue #2712) + Patch by John Kirkham. (Github issue :issue:`2712`) 0.29 (2018-10-14) @@ -714,20 +1498,20 @@ Features added types to integrate with static analysers in typed Python code. They are available in the ``Cython/Shadow.pyi`` module and describe the types in the special ``cython`` module that can be used for typing in Python code. - Original patch by Julian Gethmann. (Github issue #1965) + Original patch by Julian Gethmann. (Github issue :issue:`1965`) * Memoryviews are supported in PEP-484/526 style type declarations. - (Github issue #2529) + (Github issue :issue:`2529`) * ``@cython.nogil`` is supported as a C-function decorator in Python code. - (Github issue #2557) + (Github issue :issue:`2557`) * Raising exceptions from nogil code will automatically acquire the GIL, instead of requiring an explicit ``with gil`` block. * C++ functions can now be declared as potentially raising both C++ and Python exceptions, so that Cython can handle both correctly. - (Github issue #2615) + (Github issue :issue:`2615`) * ``cython.inline()`` supports a direct ``language_level`` keyword argument that was previously only available via a directive. @@ -741,14 +1525,14 @@ Features added * In CPython 3.6 and later, looking up globals in the module dict is almost as fast as looking up C globals. - (Github issue #2313) + (Github issue :issue:`2313`) * For a Python subclass of an extension type, repeated method calls to non-overridden cpdef methods can avoid the attribute lookup in Py3.6+, which makes them 4x faster. - (Github issue #2313) + (Github issue :issue:`2313`) * (In-)equality comparisons of objects to integer literals are faster. - (Github issue #2188) + (Github issue :issue:`2188`) * Some internal and 1-argument method calls are faster. @@ -756,26 +1540,26 @@ Features added execute less import requests during module initialisation. * Constant tuples and slices are deduplicated and only created once per module. - (Github issue #2292) + (Github issue :issue:`2292`) * The coverage plugin considers more C file extensions such as ``.cc`` and ``.cxx``. - (Github issue #2266) + (Github issue :issue:`2266`) * The ``cythonize`` command accepts compile time variable values (as set by ``DEF``) through the new ``-E`` option. - Patch by Jerome Kieffer. (Github issue #2315) + Patch by Jerome Kieffer. (Github issue :issue:`2315`) * ``pyximport`` can import from namespace packages. - Patch by Prakhar Goel. (Github issue #2294) + Patch by Prakhar Goel. (Github issue :issue:`2294`) * Some missing numpy and CPython C-API declarations were added. - Patch by John Kirkham. (Github issues #2523, #2520, #2537) + Patch by John Kirkham. (Github issues :issue:`2523`, :issue:`2520`, :issue:`2537`) * Declarations for the ``pylifecycle`` C-API functions were added in a new .pxd file ``cpython.pylifecycle``. * The Pythran support was updated to work with the latest Pythran 0.8.7. - Original patch by Adrien Guinet. (Github issue #2600) + Original patch by Adrien Guinet. (Github issue :issue:`2600`) * ``%a`` is included in the string formatting types that are optimised into f-strings. In this case, it is also automatically mapped to ``%r`` in Python 2.x. @@ -788,7 +1572,7 @@ Features added * An additional ``check_size`` clause was added to the ``ctypedef class`` name specification to allow suppressing warnings when importing modules with backwards-compatible ``PyTypeObject`` size changes. - Patch by Matti Picus. (Github issue #2627) + Patch by Matti Picus. (Github issue :issue:`2627`) Bugs fixed ---------- @@ -796,13 +1580,13 @@ Bugs fixed * The exception handling in generators and coroutines under CPython 3.7 was adapted to the newly introduced exception stack. Users of Cython 0.28 who want to support Python 3.7 are encouraged to upgrade to 0.29 to avoid potentially incorrect error - reporting and tracebacks. (Github issue #1958) + reporting and tracebacks. (Github issue :issue:`1958`) * Crash when importing a module under Stackless Python that was built for CPython. - Patch by Anselm Kruis. (Github issue #2534) + Patch by Anselm Kruis. (Github issue :issue:`2534`) * 2-value slicing of typed sequences failed if the start or stop index was None. - Patch by Christian Gibson. (Github issue #2508) + Patch by Christian Gibson. (Github issue :issue:`2508`) * Multiplied string literals lost their factor when they are part of another constant expression (e.g. 'x' * 10 + 'y' => 'xy'). @@ -812,38 +1596,38 @@ Bugs fixed (Python issue 28598) * The directive ``language_level=3`` did not apply to the first token in the - source file. (Github issue #2230) + source file. (Github issue :issue:`2230`) * Overriding cpdef methods did not work in Python subclasses with slots. Note that this can have a performance impact on calls from Cython code. - (Github issue #1771) + (Github issue :issue:`1771`) * Fix declarations of builtin or C types using strings in pure python mode. - (Github issue #2046) + (Github issue :issue:`2046`) * Generator expressions and lambdas failed to compile in ``@cfunc`` functions. - (Github issue #459) + (Github issue :issue:`459`) * Global names with ``const`` types were not excluded from star-import assignments which could lead to invalid C code. - (Github issue #2621) + (Github issue :issue:`2621`) * Several internal function signatures were fixed that lead to warnings in gcc-8. - (Github issue #2363) + (Github issue :issue:`2363`) * The numpy helper functions ``set_array_base()`` and ``get_array_base()`` were adapted to the current numpy C-API recommendations. - Patch by Matti Picus. (Github issue #2528) + Patch by Matti Picus. (Github issue :issue:`2528`) * Some NumPy related code was updated to avoid deprecated API usage. - Original patch by jbrockmendel. (Github issue #2559) + Original patch by jbrockmendel. (Github issue :issue:`2559`) * Several C++ STL declarations were extended and corrected. - Patch by Valentin Valls. (Github issue #2207) + Patch by Valentin Valls. (Github issue :issue:`2207`) * C lines of the module init function were unconditionally not reported in exception stack traces. - Patch by Jeroen Demeyer. (Github issue #2492) + Patch by Jeroen Demeyer. (Github issue :issue:`2492`) * When PEP-489 support is enabled, reloading the module overwrote any static module state. It now raises an exception instead, given that reloading is @@ -851,11 +1635,11 @@ Bugs fixed * Object-returning, C++ exception throwing functions were not checking that the return value was non-null. - Original patch by Matt Wozniski (Github Issue #2603) + Original patch by Matt Wozniski (Github issue :issue:`2603`) * The source file encoding detection could get confused if the ``c_string_encoding`` directive appeared within the first two lines. - (Github issue #2632) + (Github issue :issue:`2632`) * Cython generated modules no longer emit a warning during import when the size of the NumPy array type is larger than what was found at compile time. @@ -877,7 +1661,7 @@ Other changes * The documentation was restructured, cleaned up and examples are now tested. The NumPy tutorial was also rewritten to simplify the running example. - Contributed by Gabriel de Marmiesse. (Github issue #2245) + Contributed by Gabriel de Marmiesse. (Github issue :issue:`2245`) * Cython compiles less of its own modules at build time to reduce the installed package size to about half of its previous size. This makes the compiler @@ -892,7 +1676,7 @@ Bugs fixed * Extensions compiled with MinGW-64 under Windows could misinterpret integer objects larger than 15 bit and return incorrect results. - (Github issue #2670) + (Github issue :issue:`2670`) * Multiplied string literals lost their factor when they are part of another constant expression (e.g. 'x' * 10 + 'y' => 'xy'). @@ -906,7 +1690,7 @@ Bugs fixed * The discouraged usage of GCC's attribute ``optimize("Os")`` was replaced by the similar attribute ``cold`` to reduce the code impact of the module init functions. - (Github issue #2494) + (Github issue :issue:`2494`) * A reference leak in Py2.x was fixed when comparing str to unicode for equality. @@ -919,12 +1703,12 @@ Bugs fixed * Reallowing ``tp_clear()`` in a subtype of an ``@no_gc_clear`` extension type generated an invalid C function call to the (non-existent) base type implementation. - (Github issue #2309) + (Github issue :issue:`2309`) * Exception catching based on a non-literal (runtime) tuple could fail to match the - exception. (Github issue #2425) + exception. (Github issue :issue:`2425`) -* Compile fix for CPython 3.7.0a2. (Github issue #2477) +* Compile fix for CPython 3.7.0a2. (Github issue :issue:`2477`) 0.28.3 (2018-05-27) @@ -936,13 +1720,13 @@ Bugs fixed * Set iteration was broken in non-CPython since 0.28. * ``UnicodeEncodeError`` in Py2 when ``%s`` formatting is optimised for - unicode strings. (Github issue #2276) + unicode strings. (Github issue :issue:`2276`) * Work around a crash bug in g++ 4.4.x by disabling the size reduction setting - of the module init function in this version. (Github issue #2235) + of the module init function in this version. (Github issue :issue:`2235`) * Crash when exceptions occur early during module initialisation. - (Github issue #2199) + (Github issue :issue:`2199`) 0.28.2 (2018-04-13) @@ -954,10 +1738,10 @@ Features added * ``abs()`` is faster for Python long objects. * The C++11 methods ``front()`` and ``end()`` were added to the declaration of - ``libcpp.string``. Patch by Alex Huszagh. (Github issue #2123) + ``libcpp.string``. Patch by Alex Huszagh. (Github issue :issue:`2123`) * The C++11 methods ``reserve()`` and ``bucket_count()`` are declared for - ``libcpp.unordered_map``. Patch by Valentin Valls. (Github issue #2168) + ``libcpp.unordered_map``. Patch by Valentin Valls. (Github issue :issue:`2168`) Bugs fixed ---------- @@ -965,24 +1749,24 @@ Bugs fixed * The copy of a read-only memoryview was considered read-only as well, whereas a common reason to copy a read-only view is to make it writable. The result of the copying is now a writable buffer by default. - (Github issue #2134) + (Github issue :issue:`2134`) * The ``switch`` statement generation failed to apply recursively to the body of converted if-statements. * ``NULL`` was sometimes rejected as exception return value when the returned type is a fused pointer type. - Patch by Callie LeFave. (Github issue #2177) + Patch by Callie LeFave. (Github issue :issue:`2177`) * Fixed compatibility with PyPy 5.11. - Patch by Matti Picus. (Github issue #2165) + Patch by Matti Picus. (Github issue :issue:`2165`) Other changes ------------- * The NumPy tutorial was rewritten to use memoryviews instead of the older buffer declaration syntax. - Contributed by Gabriel de Marmiesse. (Github issue #2162) + Contributed by Gabriel de Marmiesse. (Github issue :issue:`2162`) 0.28.1 (2018-03-18) @@ -996,18 +1780,18 @@ Bugs fixed * Assignment between some C++ templated types were incorrectly rejected when the templates mix ``const`` with ``ctypedef``. - (Github issue #2148) + (Github issue :issue:`2148`) * Undeclared C++ no-args constructors in subclasses could make the compilation fail if the base class constructor was declared without ``nogil``. - (Github issue #2157) + (Github issue :issue:`2157`) * Bytes %-formatting inferred ``basestring`` (bytes or unicode) as result type in some cases where ``bytes`` would have been safe to infer. - (Github issue #2153) + (Github issue :issue:`2153`) * ``None`` was accidentally disallowed as typed return value of ``dict.pop()``. - (Github issue #2152) + (Github issue :issue:`2152`) 0.28 (2018-03-13) @@ -1021,19 +1805,19 @@ Features added the other bases must *not* be cdef classes.) * Type inference is now supported for Pythran compiled NumPy expressions. - Patch by Nils Braun. (Github issue #1954) + Patch by Nils Braun. (Github issue :issue:`1954`) * The ``const`` modifier can be applied to memoryview declarations to allow - read-only buffers as input. (Github issues #1605, #1869) + read-only buffers as input. (Github issues :issue:`1605`, :issue:`1869`) * C code in the docstring of a ``cdef extern`` block is copied verbatimly into the generated file. - Patch by Jeroen Demeyer. (Github issue #1915) + Patch by Jeroen Demeyer. (Github issue :issue:`1915`) * When compiling with gcc, the module init function is now tuned for small code size instead of whatever compile flags were provided externally. Cython now also disables some code intensive optimisations in that function - to further reduce the code size. (Github issue #2102) + to further reduce the code size. (Github issue :issue:`2102`) * Decorating an async coroutine with ``@cython.iterable_coroutine`` changes its type at compile time to make it iterable. While this is not strictly in line @@ -1041,23 +1825,23 @@ Features added use ``yield from`` instead of ``await``. * The IPython magic has preliminary support for JupyterLab. - (Github issue #1775) + (Github issue :issue:`1775`) * The new TSS C-API in CPython 3.7 is supported and has been backported. - Patch by Naotoshi Seo. (Github issue #1932) + Patch by Naotoshi Seo. (Github issue :issue:`1932`) * Cython knows the new ``Py_tss_t`` type defined in PEP-539 and automatically initialises variables declared with that type to ``Py_tss_NEEDS_INIT``, a value which cannot be used outside of static assignments. * The set methods ``.remove()`` and ``.discard()`` are optimised. - Patch by Antoine Pitrou. (Github issue #2042) + Patch by Antoine Pitrou. (Github issue :issue:`2042`) * ``dict.pop()`` is optimised. - Original patch by Antoine Pitrou. (Github issue #2047) + Original patch by Antoine Pitrou. (Github issue :issue:`2047`) * Iteration over sets and frozensets is optimised. - (Github issue #2048) + (Github issue :issue:`2048`) * Safe integer loops (< range(2^30)) are automatically optimised into C loops. @@ -1066,7 +1850,7 @@ Features added * Calls to builtin methods that are not specifically optimised into C-API calls now use a cache that avoids repeated lookups of the underlying C function. - (Github issue #2054) + (Github issue :issue:`2054`) * Single argument function calls can avoid the argument tuple creation in some cases. @@ -1094,19 +1878,19 @@ Features added * Python attribute lookups on extension types without instance dict are faster. * Some missing signals were added to ``libc/signal.pxd``. - Patch by Jeroen Demeyer. (Github issue #1914) + Patch by Jeroen Demeyer. (Github issue :issue:`1914`) * The warning about repeated extern declarations is now visible by default. - (Github issue #1874) + (Github issue :issue:`1874`) * The exception handling of the function types used by CPython's type slot functions was corrected to match the de-facto standard behaviour, so that code that uses them directly benefits from automatic and correct exception - propagation. Patch by Jeroen Demeyer. (Github issue #1980) + propagation. Patch by Jeroen Demeyer. (Github issue :issue:`1980`) * Defining the macro ``CYTHON_NO_PYINIT_EXPORT`` will prevent the module init function from being exported as symbol, e.g. when linking modules statically - in an embedding setup. Patch by AraHaan. (Github issue #1944) + in an embedding setup. Patch by AraHaan. (Github issue :issue:`1944`) Bugs fixed ---------- @@ -1114,11 +1898,11 @@ Bugs fixed * If a module name is explicitly provided for an ``Extension()`` that is compiled via ``cythonize()``, it was previously ignored and replaced by the source file name. It can now be used to override the target module name, e.g. for compiling - prefixed accelerator modules from Python files. (Github issue #2038) + prefixed accelerator modules from Python files. (Github issue :issue:`2038`) * The arguments of the ``num_threads`` parameter of parallel sections were not sufficiently validated and could lead to invalid C code. - (Github issue #1957) + (Github issue :issue:`1957`) * Catching exceptions with a non-trivial exception pattern could call into CPython with a live exception set. This triggered incorrect behaviour @@ -1134,32 +1918,32 @@ Bugs fixed under Python 2. * Some async helper functions were not defined in the generated C code when - compiling simple async code. (Github issue #2075) + compiling simple async code. (Github issue :issue:`2075`) * Line tracing did not include generators and coroutines. - (Github issue #1949) + (Github issue :issue:`1949`) * C++ declarations for ``unordered_map`` were corrected. - Patch by Michael Schatzow. (Github issue #1484) + Patch by Michael Schatzow. (Github issue :issue:`1484`) * Iterator declarations in C++ ``deque`` and ``vector`` were corrected. - Patch by Alex Huszagh. (Github issue #1870) + Patch by Alex Huszagh. (Github issue :issue:`1870`) * The const modifiers in the C++ ``string`` declarations were corrected, together with the coercion behaviour of string literals into C++ strings. - (Github issue #2132) + (Github issue :issue:`2132`) * Some declaration types in ``libc.limits`` were corrected. - Patch by Jeroen Demeyer. (Github issue #2016) + Patch by Jeroen Demeyer. (Github issue :issue:`2016`) * ``@cython.final`` was not accepted on Python classes with an ``@cython.cclass`` - decorator. (Github issue #2040) + decorator. (Github issue :issue:`2040`) * Cython no longer creates useless and incorrect ``PyInstanceMethod`` wrappers for - methods in Python 3. Patch by Jeroen Demeyer. (Github issue #2105) + methods in Python 3. Patch by Jeroen Demeyer. (Github issue :issue:`2105`) * The builtin ``bytearray`` type could not be used as base type of cdef classes. - (Github issue #2106) + (Github issue :issue:`2106`) Other changes ------------- @@ -1172,18 +1956,18 @@ Bugs fixed ---------- * String forward references to extension types like ``@cython.locals(x="ExtType")`` - failed to find the named type. (Github issue #1962) + failed to find the named type. (Github issue :issue:`1962`) * NumPy slicing generated incorrect results when compiled with Pythran. - Original patch by Serge Guelton (Github issue #1946). + Original patch by Serge Guelton (Github issue :issue:`1946`). * Fix "undefined reference" linker error for generators on Windows in Py3.3-3.5. - (Github issue #1968) + (Github issue :issue:`1968`) * Adapt to recent C-API change of ``PyThreadState`` in CPython 3.7. * Fix signature of ``PyWeakref_GetObject()`` API declaration. - Patch by Jeroen Demeyer (Github issue #1975). + Patch by Jeroen Demeyer (Github issue :issue:`1975`). 0.27.2 (2017-10-22) @@ -1193,27 +1977,27 @@ Bugs fixed ---------- * Comprehensions could incorrectly be optimised away when they appeared in boolean - test contexts. (Github issue #1920) + test contexts. (Github issue :issue:`1920`) * The special methods ``__eq__``, ``__lt__`` etc. in extension types did not type - their first argument as the type of the class but ``object``. (Github issue #1935) + their first argument as the type of the class but ``object``. (Github issue :issue:`1935`) * Crash on first lookup of "cline_in_traceback" option during exception handling. - (Github issue #1907) + (Github issue :issue:`1907`) * Some nested module level comprehensions failed to compile. - (Github issue #1906) + (Github issue :issue:`1906`) * Compiler crash on some complex type declarations in pure mode. - (Github issue #1908) + (Github issue :issue:`1908`) * ``std::unordered_map.erase()`` was declared with an incorrect ``void`` return - type in ``libcpp.unordered_map``. (Github issue #1484) + type in ``libcpp.unordered_map``. (Github issue :issue:`1484`) * Invalid use of C++ ``fallthrough`` attribute before C++11 and similar issue in clang. - (Github issue #1930) + (Github issue :issue:`1930`) -* Compiler crash on misnamed properties. (Github issue #1905) +* Compiler crash on misnamed properties. (Github issue :issue:`1905`) 0.27.1 (2017-10-01) @@ -1223,34 +2007,34 @@ Features added -------------- * The Jupyter magic has a new debug option ``--verbose`` that shows details about - the distutils invocation. Patch by Boris Filippov (Github issue #1881). + the distutils invocation. Patch by Boris Filippov (Github issue :issue:`1881`). Bugs fixed ---------- * Py3 list comprehensions in class bodies resulted in invalid C code. - (Github issue #1889) + (Github issue :issue:`1889`) * Modules built for later CPython 3.5.x versions failed to import in 3.5.0/3.5.1. - (Github issue #1880) + (Github issue :issue:`1880`) * Deallocating fused types functions and methods kept their GC tracking enabled, which could potentially lead to recursive deallocation attempts. * Crash when compiling in C++ mode with old setuptools versions. - (Github issue #1879) + (Github issue :issue:`1879`) * C++ object arguments for the constructor of Cython implemented C++ are now passed by reference and not by value to allow for non-copyable arguments, such as ``unique_ptr``. * API-exported C++ classes with Python object members failed to compile. - (Github issue #1866) + (Github issue :issue:`1866`) * Some issues with the new relaxed exception value handling were resolved. * Python classes as annotation types could prevent compilation. - (Github issue #1887) + (Github issue :issue:`1887`) * Cython annotation types in Python files could lead to import failures with a "cython undefined" error. Recognised types are now turned into strings. @@ -1278,7 +2062,7 @@ Features added resolves several differences with regard to normal Python modules. This makes the global names ``__file__`` and ``__path__`` correctly available to module level code and improves the support for module-level relative imports. - (Github issues #1715, #1753, #1035) + (Github issues :issue:`1715`, :issue:`1753`, :issue:`1035`) * Asynchronous generators (`PEP 525 <https://www.python.org/dev/peps/pep-0525/>`_) and asynchronous comprehensions (`PEP 530 <https://www.python.org/dev/peps/pep-0530/>`_) @@ -1291,18 +2075,18 @@ Features added ``cython.int``) are evaluated as C type declarations and everything else as Python types. This can be disabled with the directive ``annotation_typing=False``. Note that most complex PEP-484 style annotations are currently ignored. This will - change in future releases. (Github issue #1850) + change in future releases. (Github issue :issue:`1850`) * Extension types (also in pure Python mode) can implement the normal special methods ``__eq__``, ``__lt__`` etc. for comparisons instead of the low-level ``__richcmp__`` - method. (Github issue #690) + method. (Github issue :issue:`690`) * New decorator ``@cython.exceptval(x=None, check=False)`` that makes the signature declarations ``except x``, ``except? x`` and ``except *`` available to pure Python - code. Original patch by Antonio Cuni. (Github issue #1653) + code. Original patch by Antonio Cuni. (Github issue :issue:`1653`) * Signature annotations are now included in the signature docstring generated by - the ``embedsignature`` directive. Patch by Lisandro Dalcin (Github issue #1781). + the ``embedsignature`` directive. Patch by Lisandro Dalcin (Github issue :issue:`1781`). * The gdb support for Python code (``libpython.py``) was updated to the latest version in CPython 3.7 (git rev 5fe59f8). @@ -1322,15 +2106,15 @@ Features added profile for C compiler optimisation. Currently only tested with gcc. * ``len(memoryview)`` can be used in nogil sections to get the size of the - first dimension of a memory view (``shape[0]``). (Github issue #1733) + first dimension of a memory view (``shape[0]``). (Github issue :issue:`1733`) * C++ classes can now contain (properly refcounted) Python objects. * NumPy dtype subarrays are now accessible through the C-API. - Patch by Gerald Dalley (Github issue #245). + Patch by Gerald Dalley (Github issue :issue:`245`). * Resolves several issues with PyPy and uses faster async slots in PyPy3. - Patch by Ronan Lamy (Github issues #1871, #1878). + Patch by Ronan Lamy (Github issues :issue:`1871`, :issue:`1878`). Bugs fixed ---------- @@ -1341,11 +2125,11 @@ Bugs fixed changes to the ordering of fused methods in the call table, which may break existing compiled modules that call fused cdef methods across module boundaries, if these methods were implemented in a different order than they were declared - in the corresponding .pxd file. (Github issue #1873) + in the corresponding .pxd file. (Github issue :issue:`1873`) * The exception state handling in generators and coroutines could lead to exceptions in the caller being lost if an exception was raised and handled - inside of the coroutine when yielding. (Github issue #1731) + inside of the coroutine when yielding. (Github issue :issue:`1731`) * Loops over ``range(enum)`` were not converted into C for-loops. Note that it is still recommended to use an explicit cast to a C integer type in this case. @@ -1354,27 +2138,27 @@ Bugs fixed name and not at the beginning of the name. * Compile time ``DEF`` assignments were evaluated even when they occur inside of - falsy ``IF`` blocks. (Github issue #1796) + falsy ``IF`` blocks. (Github issue :issue:`1796`) * Disabling the line tracing from a trace function could fail. - Original patch by Dmitry Trofimov. (Github issue #1769) + Original patch by Dmitry Trofimov. (Github issue :issue:`1769`) * Several issues with the Pythran integration were resolved. * abs(signed int) now returns a signed rather than unsigned int. - (Github issue #1837) + (Github issue :issue:`1837`) * Reading ``frame.f_locals`` of a Cython function (e.g. from a debugger or profiler - could modify the module globals. (Github issue #1836) + could modify the module globals. (Github issue :issue:`1836`) * Buffer type mismatches in the NumPy buffer support could leak a reference to the buffer owner. * Using the "is_f_contig" and "is_c_contig" memoryview methods together could leave - one of them undeclared. (Github issue #1872) + one of them undeclared. (Github issue :issue:`1872`) * Compilation failed if the for-in-range loop target was not a variable but a more - complex expression, e.g. an item assignment. (Github issue #1831) + complex expression, e.g. an item assignment. (Github issue :issue:`1831`) * Compile time evaluations of (partially) constant f-strings could show incorrect results. @@ -1392,7 +2176,7 @@ Other changes typing. Only Cython types (e.g. ``cython.int``) and Python builtin types are currently considered as type declarations. Everything else is ignored, but this will change in a future Cython release. - (Github issue #1672) + (Github issue :issue:`1672`) * The directive ``annotation_typing`` is now ``True`` by default, which enables parsing type declarations from annotations. @@ -1414,31 +2198,31 @@ Bugs fixed * Extension types with a ``.pxd`` override for their ``__releasebuffer__`` slot (e.g. as provided by Cython for the Python ``array.array`` type) could leak a reference to the buffer owner on release, thus not freeing the memory. - (Github issue #1638) + (Github issue :issue:`1638`) * Auto-decoding failed in 0.26 for strings inside of C++ containers. - (Github issue #1790) + (Github issue :issue:`1790`) * Compile error when inheriting from C++ container types. - (Github issue #1788) + (Github issue :issue:`1788`) * Invalid C code in generators (declaration after code). - (Github issue #1801) + (Github issue :issue:`1801`) * Arithmetic operations on ``const`` integer variables could generate invalid code. - (Github issue #1798) + (Github issue :issue:`1798`) * Local variables with names of special Python methods failed to compile inside of - closures. (Github issue #1797) + closures. (Github issue :issue:`1797`) * Problem with indirect Emacs buffers in cython-mode. - Patch by Martin Albrecht (Github issue #1743). + Patch by Martin Albrecht (Github issue :issue:`1743`). * Extension types named ``result`` or ``PickleError`` generated invalid unpickling code. - Patch by Jason Madden (Github issue #1786). + Patch by Jason Madden (Github issue :issue:`1786`). * Bazel integration failed to compile ``.py`` files. - Patch by Guro Bokum (Github issue #1784). + Patch by Guro Bokum (Github issue :issue:`1784`). * Some include directories and dependencies were referenced with their absolute paths in the generated files despite lying within the project directory. @@ -1453,23 +2237,23 @@ Features added -------------- * Pythran can be used as a backend for evaluating NumPy array expressions. - Patch by Adrien Guinet (Github issue #1607). + Patch by Adrien Guinet (Github issue :issue:`1607`). * cdef classes now support pickling by default when possible. This can be disabled with the ``auto_pickle`` directive. * Speed up comparisons of strings if their hash value is available. - Patch by Claudio Freire (Github issue #1571). + Patch by Claudio Freire (Github issue :issue:`1571`). * Support pyximport from zip files. - Patch by Sergei Lebedev (Github issue #1485). + Patch by Sergei Lebedev (Github issue :issue:`1485`). * IPython magic now respects the ``__all__`` variable and ignores names with leading-underscore (like ``import *`` does). - Patch by Syrtis Major (Github issue #1625). + Patch by Syrtis Major (Github issue :issue:`1625`). * ``abs()`` is optimised for C complex numbers. - Patch by da-woods (Github issue #1648). + Patch by David Woods (Github issue :issue:`1648`). * The display of C lines in Cython tracebacks can now be enabled at runtime via ``import cython_runtime; cython_runtime.cline_in_traceback=True``. @@ -1478,9 +2262,9 @@ Features added * The overhead of calling fused types generic functions was reduced. * "cdef extern" include files are now also searched relative to the current file. - Patch by Jeroen Demeyer (Github issue #1654). + Patch by Jeroen Demeyer (Github issue :issue:`1654`). -* Optional optimization for re-aquiring the GIL, controlled by the +* Optional optimization for re-acquiring the GIL, controlled by the `fast_gil` directive. Bugs fixed @@ -1490,24 +2274,24 @@ Bugs fixed (explicitly or implicitly) as ``Py_UCS4`` or ``Py_UNICODE`` used the integer value instead of the Unicode string value. Code that relied on the previous behaviour now triggers a warning that can be disabled by - applying an explicit cast. (Github issue #1602) + applying an explicit cast. (Github issue :issue:`1602`) * f-string processing was adapted to changes in PEP 498 and CPython 3.6. * Invalid C code when decoding from UTF-16(LE/BE) byte strings. - (Github issue #1696) + (Github issue :issue:`1696`) * Unicode escapes in 'ur' raw-unicode strings were not resolved in Py2 code. - Original patch by Aaron Gallagher (Github issue #1594). + Original patch by Aaron Gallagher (Github issue :issue:`1594`). * File paths of code objects are now relative. - Original patch by Jelmer Vernooij (Github issue #1565). + Original patch by Jelmer Vernooij (Github issue :issue:`1565`). * Decorators of cdef class methods could be executed twice. - Patch by Jeroen Demeyer (Github issue #1724). + Patch by Jeroen Demeyer (Github issue :issue:`1724`). * Dict iteration using the Py2 ``iter*`` methods failed in PyPy3. - Patch by Armin Rigo (Github issue #1631). + Patch by Armin Rigo (Github issue :issue:`1631`). * Several warnings in the generated code are now suppressed. @@ -1517,15 +2301,15 @@ Other changes * The ``unraisable_tracebacks`` option now defaults to ``True``. * Coercion of C++ containers to Python is no longer automatic on attribute - access (Github issue #1521). + access (Github issue :issue:`1521`). * Access to Python attributes of cimported modules without the corresponding import is now a compile-time (rather than runtime) error. * Do not use special dll linkage for "cdef public" functions. - Patch by Jeroen Demeyer (Github issue #1687). + Patch by Jeroen Demeyer (Github issue :issue:`1687`). -* cdef/cpdef methods must match their declarations. See Github Issue #1732. +* cdef/cpdef methods must match their declarations. See Github issue :issue:`1732`. This is now a warning and will be an error in future releases. @@ -1537,13 +2321,13 @@ Bugs fixed * Fixes several issues with C++ template deduction. -* Fixes a issue with bound method type inference (Github issue #551). +* Fixes a issue with bound method type inference (Github issue :issue:`551`). -* Fixes a bug with cascaded tuple assignment (Github issue #1523). +* Fixes a bug with cascaded tuple assignment (Github issue :issue:`1523`). * Fixed or silenced many Clang warnings. -* Fixes bug with powers of pure real complex numbers (Github issue #1538). +* Fixes bug with powers of pure real complex numbers (Github issue :issue:`1538`). 0.25.1 (2016-10-26) @@ -1552,10 +2336,10 @@ Bugs fixed Bugs fixed ---------- -* Fixes a bug with ``isinstance(o, Exception)`` (Github issue #1496). +* Fixes a bug with ``isinstance(o, Exception)`` (Github issue :issue:`1496`). * Fixes bug with ``cython.view.array`` missing utility code in some cases - (Github issue #1502). + (Github issue :issue:`1502`). Other changes ------------- @@ -1598,7 +2382,7 @@ Features added Patch by Claudio Freire. * Buffer variables are no longer excluded from ``locals()``. - Patch by da-woods. + Patch by David Woods. * Building f-strings is faster, especially when formatting C integers. @@ -1746,7 +2530,7 @@ Bugs fixed * Compile errors and warnings in integer type conversion code. This fixes ticket 877. Patches by Christian Neukirchen, Nikolaus Rath, Ian Henriksen. -* Reference leak when "*args" argument was reassigned in closures. +* Reference leak when ``*args`` argument was reassigned in closures. * Truth-testing Unicode strings could waste time and memory in Py3.3+. @@ -2202,7 +2986,7 @@ Features added * Generators have new properties ``__name__`` and ``__qualname__`` that provide the plain/qualified name of the generator function - (following CPython 3.5). See http://bugs.python.org/issue21205 + (following CPython 3.5). See https://bugs.python.org/issue21205 * The ``inline`` function modifier is available as a decorator ``@cython.inline`` in pure mode. @@ -2248,7 +3032,7 @@ Optimizations evaluation and generally improves the code flow in the generated C code. * The Python expression "2 ** N" is optimised into bit shifting. - See http://bugs.python.org/issue21420 + See https://bugs.python.org/issue21420 * Cascaded assignments (a = b = ...) try to minimise the number of type coercions. @@ -3037,7 +3821,7 @@ Features added * Extension type inheritance from builtin types, such as "cdef class MyUnicode(unicode)", now works without further external type redeclarations (which are also strongly discouraged now and continue to issue a warning). -* GDB support. http://docs.cython.org/src/userguide/debugging.html +* GDB support. https://docs.cython.org/src/userguide/debugging.html * A new build system with support for inline distutils directives, correct dependency tracking, and parallel compilation. https://github.com/cython/cython/wiki/enhancements-distutils_preprocessing @@ -3217,7 +4001,7 @@ Features added cdef np.ndarray[np.complex128_t, ndim=2] arr = \ np.zeros(10, np.complex128) -* Cython can now generate a main()-method for embedding of the Python interpreter into an executable (see #289) [Robert Bradshaw] +* Cython can now generate a main()-method for embedding of the Python interpreter into an executable (see :issue:`289`) [Robert Bradshaw] * @wraparound directive (another way to disable arr[idx] for negative idx) [Dag Sverre Seljebotn] diff --git a/Cython/Build/BuildExecutable.py b/Cython/Build/BuildExecutable.py index 2db9e5d74..334fbf069 100644 --- a/Cython/Build/BuildExecutable.py +++ b/Cython/Build/BuildExecutable.py @@ -1,10 +1,10 @@ """ -Compile a Python script into an executable that embeds CPython and run it. +Compile a Python script into an executable that embeds CPython. Requires CPython to be built as a shared library ('libpythonX.Y'). Basic usage: - python cythonrun somefile.py [ARGS] + python -m Cython.Build.BuildExecutable [ARGS] somefile.py """ from __future__ import absolute_import @@ -28,7 +28,7 @@ if PYLIB_DYN == PYLIB: # no shared library PYLIB_DYN = '' else: - PYLIB_DYN = os.path.splitext(PYLIB_DYN[3:])[0] # 'lib(XYZ).so' -> XYZ + PYLIB_DYN = os.path.splitext(PYLIB_DYN[3:])[0] # 'lib(XYZ).so' -> XYZ CC = get_config_var('CC', os.environ.get('CC', '')) CFLAGS = get_config_var('CFLAGS') + ' ' + os.environ.get('CFLAGS', '') @@ -38,12 +38,14 @@ LIBS = get_config_var('LIBS') SYSLIBS = get_config_var('SYSLIBS') EXE_EXT = sysconfig.get_config_var('EXE') + def _debug(msg, *args): if DEBUG: if args: msg = msg % args sys.stderr.write(msg + '\n') + def dump_config(): _debug('INCDIR: %s', INCDIR) _debug('LIBDIR1: %s', LIBDIR1) @@ -58,6 +60,26 @@ def dump_config(): _debug('SYSLIBS: %s', SYSLIBS) _debug('EXE_EXT: %s', EXE_EXT) + +def _parse_args(args): + cy_args = [] + last_arg = None + for i, arg in enumerate(args): + if arg.startswith('-'): + cy_args.append(arg) + elif last_arg in ('-X', '--directive'): + cy_args.append(arg) + else: + input_file = arg + args = args[i+1:] + break + last_arg = arg + else: + raise ValueError('no input file provided') + + return input_file, cy_args, args + + def runcmd(cmd, shell=True): if shell: cmd = ' '.join(cmd) @@ -65,24 +87,23 @@ def runcmd(cmd, shell=True): else: _debug(' '.join(cmd)) - try: - import subprocess - except ImportError: # Python 2.3 ... - returncode = os.system(cmd) - else: - returncode = subprocess.call(cmd, shell=shell) + import subprocess + returncode = subprocess.call(cmd, shell=shell) if returncode: sys.exit(returncode) + def clink(basename): runcmd([LINKCC, '-o', basename + EXE_EXT, basename+'.o', '-L'+LIBDIR1, '-L'+LIBDIR2] + [PYLIB_DYN and ('-l'+PYLIB_DYN) or os.path.join(LIBDIR1, PYLIB)] + LIBS.split() + SYSLIBS.split() + LINKFORSHARED.split()) + def ccompile(basename): runcmd([CC, '-c', '-o', basename+'.o', basename+'.c', '-I' + INCDIR] + CFLAGS.split()) + def cycompile(input_file, options=()): from ..Compiler import Version, CmdLine, Main options, sources = CmdLine.parse_command_line(list(options or ()) + ['--embed', input_file]) @@ -91,9 +112,11 @@ def cycompile(input_file, options=()): if result.num_errors > 0: sys.exit(1) + def exec_file(program_name, args=()): runcmd([os.path.abspath(program_name)] + list(args), shell=False) + def build(input_file, compiler_args=(), force=False): """ Build an executable program from a Cython module. @@ -105,7 +128,7 @@ def build(input_file, compiler_args=(), force=False): if not force and os.path.abspath(exe_file) == os.path.abspath(input_file): raise ValueError("Input and output file names are the same, refusing to overwrite") if (not force and os.path.exists(exe_file) and os.path.exists(input_file) - and os.path.getmtime(input_file) <= os.path.getmtime(exe_file)): + and os.path.getmtime(input_file) <= os.path.getmtime(exe_file)): _debug("File is up to date, not regenerating %s", exe_file) return exe_file cycompile(input_file, compiler_args) @@ -113,30 +136,22 @@ def build(input_file, compiler_args=(), force=False): clink(basename) return exe_file + def build_and_run(args): """ - Build an executable program from a Cython module and runs it. + Build an executable program from a Cython module and run it. - Arguments after the module name will be passed verbatimely to the - program. + Arguments after the module name will be passed verbatimly to the program. """ - cy_args = [] - last_arg = None - for i, arg in enumerate(args): - if arg.startswith('-'): - cy_args.append(arg) - elif last_arg in ('-X', '--directive'): - cy_args.append(arg) - else: - input_file = arg - args = args[i+1:] - break - last_arg = arg - else: - raise ValueError('no input file provided') + program_name, args = _build(args) + exec_file(program_name, args) + +def _build(args): + input_file, cy_args, args = _parse_args(args) program_name = build(input_file, cy_args) - exec_file(program_name, args) + return program_name, args + if __name__ == '__main__': - build_and_run(sys.argv[1:]) + _build(sys.argv[1:]) diff --git a/Cython/Build/Cythonize.py b/Cython/Build/Cythonize.py index 9de84d5c8..a3774b385 100644 --- a/Cython/Build/Cythonize.py +++ b/Cython/Build/Cythonize.py @@ -38,38 +38,9 @@ class _FakePool(object): pass -def parse_directives(option, name, value, parser): - dest = option.dest - old_directives = dict(getattr(parser.values, dest, - Options.get_directive_defaults())) - directives = Options.parse_directive_list( - value, relaxed_bool=True, current_settings=old_directives) - setattr(parser.values, dest, directives) - - -def parse_options(option, name, value, parser): - dest = option.dest - options = dict(getattr(parser.values, dest, {})) - for opt in value.split(','): - if '=' in opt: - n, v = opt.split('=', 1) - v = v.lower() not in ('false', 'f', '0', 'no') - else: - n, v = opt, True - options[n] = v - setattr(parser.values, dest, options) - - -def parse_compile_time_env(option, name, value, parser): - dest = option.dest - old_env = dict(getattr(parser.values, dest, {})) - new_env = Options.parse_compile_time_env(value, current_settings=old_env) - setattr(parser.values, dest, new_env) - - def find_package_base(path): base_dir, package_path = os.path.split(path) - while os.path.isfile(os.path.join(base_dir, '__init__.py')): + while is_package_dir(base_dir): base_dir, parent = os.path.split(base_dir) package_path = '%s/%s' % (parent, package_path) return base_dir, package_path @@ -148,54 +119,80 @@ def run_distutils(args): shutil.rmtree(temp_dir) -def parse_args(args): - from optparse import OptionParser - parser = OptionParser(usage='%prog [options] [sources and packages]+') +def create_args_parser(): + from argparse import ArgumentParser + from ..Compiler.CmdLine import ParseDirectivesAction, ParseOptionsAction, ParseCompileTimeEnvAction - parser.add_option('-X', '--directive', metavar='NAME=VALUE,...', - dest='directives', default={}, type="str", - action='callback', callback=parse_directives, + parser = ArgumentParser() + + parser.add_argument('-X', '--directive', metavar='NAME=VALUE,...', + dest='directives', default={}, type=str, + action=ParseDirectivesAction, help='set a compiler directive') - parser.add_option('-E', '--compile-time-env', metavar='NAME=VALUE,...', - dest='compile_time_env', default={}, type="str", - action='callback', callback=parse_compile_time_env, + parser.add_argument('-E', '--compile-time-env', metavar='NAME=VALUE,...', + dest='compile_time_env', default={}, type=str, + action=ParseCompileTimeEnvAction, help='set a compile time environment variable') - parser.add_option('-s', '--option', metavar='NAME=VALUE', - dest='options', default={}, type="str", - action='callback', callback=parse_options, + parser.add_argument('-s', '--option', metavar='NAME=VALUE', + dest='options', default={}, type=str, + action=ParseOptionsAction, help='set a cythonize option') - parser.add_option('-2', dest='language_level', action='store_const', const=2, default=None, + parser.add_argument('-2', dest='language_level', action='store_const', const=2, default=None, help='use Python 2 syntax mode by default') - parser.add_option('-3', dest='language_level', action='store_const', const=3, + parser.add_argument('-3', dest='language_level', action='store_const', const=3, help='use Python 3 syntax mode by default') - parser.add_option('--3str', dest='language_level', action='store_const', const='3str', + parser.add_argument('--3str', dest='language_level', action='store_const', const='3str', help='use Python 3 syntax mode by default') - parser.add_option('-a', '--annotate', dest='annotate', action='store_true', - help='generate annotated HTML page for source files') - - parser.add_option('-x', '--exclude', metavar='PATTERN', dest='excludes', + parser.add_argument('-a', '--annotate', action='store_const', const='default', dest='annotate', + help='Produce a colorized HTML version of the source.') + parser.add_argument('--annotate-fullc', action='store_const', const='fullc', dest='annotate', + help='Produce a colorized HTML version of the source ' + 'which includes entire generated C/C++-code.') + parser.add_argument('-x', '--exclude', metavar='PATTERN', dest='excludes', action='append', default=[], help='exclude certain file patterns from the compilation') - parser.add_option('-b', '--build', dest='build', action='store_true', + parser.add_argument('-b', '--build', dest='build', action='store_true', default=None, help='build extension modules using distutils') - parser.add_option('-i', '--inplace', dest='build_inplace', action='store_true', + parser.add_argument('-i', '--inplace', dest='build_inplace', action='store_true', default=None, help='build extension modules in place using distutils (implies -b)') - parser.add_option('-j', '--parallel', dest='parallel', metavar='N', + parser.add_argument('-j', '--parallel', dest='parallel', metavar='N', type=int, default=parallel_compiles, help=('run builds in N parallel jobs (default: %d)' % parallel_compiles or 1)) - parser.add_option('-f', '--force', dest='force', action='store_true', + parser.add_argument('-f', '--force', dest='force', action='store_true', default=None, help='force recompilation') - parser.add_option('-q', '--quiet', dest='quiet', action='store_true', + parser.add_argument('-q', '--quiet', dest='quiet', action='store_true', default=None, help='be less verbose during compilation') - parser.add_option('--lenient', dest='lenient', action='store_true', + parser.add_argument('--lenient', dest='lenient', action='store_true', default=None, help='increase Python compatibility by ignoring some compile time errors') - parser.add_option('-k', '--keep-going', dest='keep_going', action='store_true', + parser.add_argument('-k', '--keep-going', dest='keep_going', action='store_true', default=None, help='compile as much as possible, ignore compilation failures') + parser.add_argument('--no-docstrings', dest='no_docstrings', action='store_true', default=None, + help='strip docstrings') + parser.add_argument('sources', nargs='*') + return parser + + +def parse_args_raw(parser, args): + options, unknown = parser.parse_known_args(args) + sources = options.sources + # if positional arguments were interspersed + # some of them are in unknown + for option in unknown: + if option.startswith('-'): + parser.error("unknown option "+option) + else: + sources.append(option) + del options.sources + return (options, sources) + + +def parse_args(args): + parser = create_args_parser() + options, args = parse_args_raw(parser, args) - options, args = parser.parse_args(args) if not args: parser.error("no source files provided") if options.build_inplace: @@ -205,11 +202,6 @@ def parse_args(args): if options.language_level: assert options.language_level in (2, 3, '3str') options.options['language_level'] = options.language_level - return options, args - - -def main(args=None): - options, paths = parse_args(args) if options.lenient: # increase Python compatibility by ignoring compile time errors @@ -217,7 +209,16 @@ def main(args=None): Options.error_on_uninitialized = False if options.annotate: - Options.annotate = True + Options.annotate = options.annotate + + if options.no_docstrings: + Options.docstrings = False + + return options, args + + +def main(args=None): + options, paths = parse_args(args) for path in paths: cython_compile(path, options) diff --git a/Cython/Build/Dependencies.py b/Cython/Build/Dependencies.py index c66afbf93..ae491c7c6 100644 --- a/Cython/Build/Dependencies.py +++ b/Cython/Build/Dependencies.py @@ -45,7 +45,9 @@ except: from .. import Utils from ..Utils import (cached_function, cached_method, path_exists, safe_makedirs, copy_file_to_dir_if_newer, is_package_dir, replace_suffix) -from ..Compiler.Main import Context, CompilationOptions, default_options +from ..Compiler import Errors +from ..Compiler.Main import Context +from ..Compiler.Options import CompilationOptions, default_options join_path = cached_function(os.path.join) copy_once_if_newer = cached_function(copy_file_to_dir_if_newer) @@ -118,7 +120,7 @@ def nonempty(it, error_msg="expected non-empty iterator"): def file_hash(filename): path = os.path.normpath(filename) prefix = ('%d:%s' % (len(path), path)).encode("UTF-8") - m = hashlib.md5(prefix) + m = hashlib.sha1(prefix) with open(path, 'rb') as f: data = f.read(65000) while data: @@ -322,7 +324,8 @@ def strip_string_literals(code, prefix='__Pyx_L'): in_quote = False hash_mark = single_q = double_q = -1 code_len = len(code) - quote_type = quote_len = None + quote_type = None + quote_len = -1 while True: if hash_mark < q: @@ -531,14 +534,14 @@ class DependencyTree(object): for include in self.parse_dependencies(filename)[1]: include_path = join_path(os.path.dirname(filename), include) if not path_exists(include_path): - include_path = self.context.find_include_file(include, None) + include_path = self.context.find_include_file(include, source_file_path=filename) if include_path: if '.' + os.path.sep in include_path: include_path = os.path.normpath(include_path) all.add(include_path) all.update(self.included_files(include_path)) elif not self.quiet: - print("Unable to locate '%s' referenced from '%s'" % (filename, include)) + print(u"Unable to locate '%s' referenced from '%s'" % (filename, include)) return all @cached_method @@ -585,17 +588,18 @@ class DependencyTree(object): return None # FIXME: error? module_path.pop(0) relative = '.'.join(package_path + module_path) - pxd = self.context.find_pxd_file(relative, None) + pxd = self.context.find_pxd_file(relative, source_file_path=filename) if pxd: return pxd if is_relative: return None # FIXME: error? - return self.context.find_pxd_file(module, None) + return self.context.find_pxd_file(module, source_file_path=filename) @cached_method def cimported_files(self, filename): - if filename[-4:] == '.pyx' and path_exists(filename[:-4] + '.pxd'): - pxd_list = [filename[:-4] + '.pxd'] + filename_root, filename_ext = os.path.splitext(filename) + if filename_ext in ('.pyx', '.py') and path_exists(filename_root + '.pxd'): + pxd_list = [filename_root + '.pxd'] else: pxd_list = [] # Cimports generates all possible combinations package.module @@ -610,10 +614,10 @@ class DependencyTree(object): @cached_method def immediate_dependencies(self, filename): - all = set([filename]) - all.update(self.cimported_files(filename)) - all.update(self.included_files(filename)) - return all + all_deps = {filename} + all_deps.update(self.cimported_files(filename)) + all_deps.update(self.included_files(filename)) + return all_deps def all_dependencies(self, filename): return self.transitive_merge(filename, self.immediate_dependencies, set.union) @@ -637,7 +641,7 @@ class DependencyTree(object): incorporate everything that has an influence on the generated code. """ try: - m = hashlib.md5(__version__.encode('UTF-8')) + m = hashlib.sha1(__version__.encode('UTF-8')) m.update(file_hash(filename).encode('UTF-8')) for x in sorted(self.all_dependencies(filename)): if os.path.splitext(x)[1] not in ('.c', '.cpp', '.h'): @@ -756,7 +760,7 @@ def create_extension_list(patterns, exclude=None, ctx=None, aliases=None, quiet= return [], {} elif isinstance(patterns, basestring) or not isinstance(patterns, Iterable): patterns = [patterns] - explicit_modules = set([m.name for m in patterns if isinstance(m, Extension)]) + explicit_modules = {m.name for m in patterns if isinstance(m, Extension)} seen = set() deps = create_dependency_tree(ctx, quiet=quiet) to_exclude = set() @@ -782,6 +786,8 @@ def create_extension_list(patterns, exclude=None, ctx=None, aliases=None, quiet= create_extension = ctx.options.create_extension or default_create_extension for pattern in patterns: + if not isinstance(pattern, (Extension_distutils, Extension_setuptools)): + pattern = encode_filename_in_py2(pattern) if isinstance(pattern, str): filepattern = pattern template = Extension(pattern, []) # Fake Extension without sources @@ -794,9 +800,9 @@ def create_extension_list(patterns, exclude=None, ctx=None, aliases=None, quiet= if cython_sources: filepattern = cython_sources[0] if len(cython_sources) > 1: - print("Warning: Multiple cython sources found for extension '%s': %s\n" - "See http://cython.readthedocs.io/en/latest/src/userguide/sharing_declarations.html " - "for sharing declarations among Cython files." % (pattern.name, cython_sources)) + print(u"Warning: Multiple cython sources found for extension '%s': %s\n" + u"See https://cython.readthedocs.io/en/latest/src/userguide/sharing_declarations.html " + u"for sharing declarations among Cython files." % (pattern.name, cython_sources)) else: # ignore non-cython modules module_list.append(pattern) @@ -870,7 +876,7 @@ def create_extension_list(patterns, exclude=None, ctx=None, aliases=None, quiet= m.sources.remove(target_file) except ValueError: # never seen this in the wild, but probably better to warn about this unexpected case - print("Warning: Cython source file not found in sources list, adding %s" % file) + print(u"Warning: Cython source file not found in sources list, adding %s" % file) m.sources.insert(0, file) seen.add(name) return module_list, module_metadata @@ -878,7 +884,7 @@ def create_extension_list(patterns, exclude=None, ctx=None, aliases=None, quiet= # This is the user-exposed entry point. def cythonize(module_list, exclude=None, nthreads=0, aliases=None, quiet=False, force=False, language=None, - exclude_failures=False, **options): + exclude_failures=False, show_all_warnings=False, **options): """ Compile a set of source modules into C/C++ files and return a list of distutils Extension objects for them. @@ -928,6 +934,9 @@ def cythonize(module_list, exclude=None, nthreads=0, aliases=None, quiet=False, really makes sense for compiling ``.py`` files which can also be used without compilation. + :param show_all_warnings: By default, not all Cython warnings are printed. + Set to true to show all warnings. + :param annotate: If ``True``, will produce a HTML file for each of the ``.pyx`` or ``.py`` files compiled. The HTML file gives an indication of how much Python interaction there is in @@ -940,6 +949,11 @@ def cythonize(module_list, exclude=None, nthreads=0, aliases=None, quiet=False, See examples in :ref:`determining_where_to_add_types` or :ref:`primes`. + + :param annotate-fullc: If ``True`` will produce a colorized HTML version of + the source which includes entire generated C/C++-code. + + :param compiler_directives: Allow to set compiler directives in the ``setup.py`` like this: ``compiler_directives={'embedsignature': True}``. See :ref:`compiler-directives`. @@ -960,7 +974,7 @@ def cythonize(module_list, exclude=None, nthreads=0, aliases=None, quiet=False, c_options = CompilationOptions(**options) cpp_options = CompilationOptions(**options); cpp_options.cplus = True - ctx = c_options.create_context() + ctx = Context.from_options(c_options) options = c_options module_list, module_metadata = create_extension_list( module_list, @@ -970,6 +984,9 @@ def cythonize(module_list, exclude=None, nthreads=0, aliases=None, quiet=False, exclude_failures=exclude_failures, language=language, aliases=aliases) + + fix_windows_unicode_modules(module_list) + deps = create_dependency_tree(ctx, quiet=quiet) build_dir = getattr(options, 'build_dir', None) @@ -1017,7 +1034,7 @@ def cythonize(module_list, exclude=None, nthreads=0, aliases=None, quiet=False, # setup for out of place build directory if enabled if build_dir: if os.path.isabs(c_file): - warnings.warn("build_dir has no effect for absolute source paths") + warnings.warn("build_dir has no effect for absolute source paths") c_file = os.path.join(build_dir, c_file) dir = os.path.dirname(c_file) safe_makedirs_once(dir) @@ -1038,9 +1055,12 @@ def cythonize(module_list, exclude=None, nthreads=0, aliases=None, quiet=False, if force or c_timestamp < dep_timestamp: if not quiet and not force: if source == dep: - print("Compiling %s because it changed." % source) + print(u"Compiling %s because it changed." % Utils.decode_filename(source)) else: - print("Compiling %s because it depends on %s." % (source, dep)) + print(u"Compiling %s because it depends on %s." % ( + Utils.decode_filename(source), + Utils.decode_filename(dep), + )) if not force and options.cache: fingerprint = deps.transitive_fingerprint(source, m, options) else: @@ -1048,7 +1068,7 @@ def cythonize(module_list, exclude=None, nthreads=0, aliases=None, quiet=False, to_compile.append(( priority, source, c_file, fingerprint, quiet, options, not exclude_failures, module_metadata.get(m.name), - full_module_name)) + full_module_name, show_all_warnings)) new_sources.append(c_file) modules_by_cfile[c_file].append(m) else: @@ -1072,32 +1092,26 @@ def cythonize(module_list, exclude=None, nthreads=0, aliases=None, quiet=False, if N <= 1: nthreads = 0 if nthreads: - # Requires multiprocessing (or Python >= 2.6) + import multiprocessing + pool = multiprocessing.Pool( + nthreads, initializer=_init_multiprocessing_helper) + # This is a bit more involved than it should be, because KeyboardInterrupts + # break the multiprocessing workers when using a normal pool.map(). + # See, for example: + # https://noswap.com/blog/python-multiprocessing-keyboardinterrupt try: - import multiprocessing - pool = multiprocessing.Pool( - nthreads, initializer=_init_multiprocessing_helper) - except (ImportError, OSError): - print("multiprocessing required for parallel cythonization") - nthreads = 0 - else: - # This is a bit more involved than it should be, because KeyboardInterrupts - # break the multiprocessing workers when using a normal pool.map(). - # See, for example: - # http://noswap.com/blog/python-multiprocessing-keyboardinterrupt - try: - result = pool.map_async(cythonize_one_helper, to_compile, chunksize=1) - pool.close() - while not result.ready(): - try: - result.get(99999) # seconds - except multiprocessing.TimeoutError: - pass - except KeyboardInterrupt: - pool.terminate() - raise - pool.join() - if not nthreads: + result = pool.map_async(cythonize_one_helper, to_compile, chunksize=1) + pool.close() + while not result.ready(): + try: + result.get(99999) # seconds + except multiprocessing.TimeoutError: + pass + except KeyboardInterrupt: + pool.terminate() + raise + pool.join() + else: for args in to_compile: cythonize_one(*args) @@ -1117,7 +1131,7 @@ def cythonize(module_list, exclude=None, nthreads=0, aliases=None, quiet=False, if failed_modules: for module in failed_modules: module_list.remove(module) - print("Failed compilations: %s" % ', '.join(sorted([ + print(u"Failed compilations: %s" % ', '.join(sorted([ module.name for module in failed_modules]))) if options.cache: @@ -1128,6 +1142,41 @@ def cythonize(module_list, exclude=None, nthreads=0, aliases=None, quiet=False, return module_list +def fix_windows_unicode_modules(module_list): + # Hack around a distutils 3.[5678] bug on Windows for unicode module names. + # https://bugs.python.org/issue39432 + if sys.platform != "win32": + return + if sys.version_info < (3, 5) or sys.version_info >= (3, 8, 2): + return + + def make_filtered_list(ignored_symbol, old_entries): + class FilteredExportSymbols(list): + # export_symbols for unicode filename cause link errors on Windows + # Cython doesn't need them (it already defines PyInit with the correct linkage) + # so use this class as a temporary fix to stop them from being generated + def __contains__(self, val): + # so distutils doesn't "helpfully" add PyInit_<name> + return val == ignored_symbol or list.__contains__(self, val) + + filtered_list = FilteredExportSymbols(old_entries) + if old_entries: + filtered_list.extend(name for name in old_entries if name != ignored_symbol) + return filtered_list + + for m in module_list: + # TODO: use m.name.isascii() in Py3.7+ + try: + m.name.encode("ascii") + continue + except UnicodeEncodeError: + pass + m.export_symbols = make_filtered_list( + "PyInit_" + m.name.rsplit(".", 1)[-1], + m.export_symbols, + ) + + if os.environ.get('XML_RESULTS'): compile_result_dir = os.environ['XML_RESULTS'] def record_results(func): @@ -1167,7 +1216,8 @@ else: # TODO: Share context? Issue: pyx processing leaks into pxd module @record_results def cythonize_one(pyx_file, c_file, fingerprint, quiet, options=None, - raise_on_failure=True, embedded_metadata=None, full_module_name=None, + raise_on_failure=True, embedded_metadata=None, + full_module_name=None, show_all_warnings=False, progress=""): from ..Compiler.Main import compile_single, default_options from ..Compiler.Errors import CompileError, PyrexError @@ -1183,7 +1233,7 @@ def cythonize_one(pyx_file, c_file, fingerprint, quiet, options=None, zip_fingerprint_file = fingerprint_file_base + '.zip' if os.path.exists(gz_fingerprint_file) or os.path.exists(zip_fingerprint_file): if not quiet: - print("%sFound compiled %s in cache" % (progress, pyx_file)) + print(u"%sFound compiled %s in cache" % (progress, pyx_file)) if os.path.exists(gz_fingerprint_file): os.utime(gz_fingerprint_file, None) with contextlib.closing(gzip_open(gz_fingerprint_file, 'rb')) as g: @@ -1197,12 +1247,16 @@ def cythonize_one(pyx_file, c_file, fingerprint, quiet, options=None, z.extract(artifact, os.path.join(dirname, artifact)) return if not quiet: - print("%sCythonizing %s" % (progress, pyx_file)) + print(u"%sCythonizing %s" % (progress, Utils.decode_filename(pyx_file))) if options is None: options = CompilationOptions(default_options) options.output_file = c_file options.embedded_metadata = embedded_metadata + old_warning_level = Errors.LEVEL + if show_all_warnings: + Errors.LEVEL = 0 + any_failures = 0 try: result = compile_single(pyx_file, options, full_module_name=full_module_name) @@ -1220,6 +1274,10 @@ def cythonize_one(pyx_file, c_file, fingerprint, quiet, options=None, import traceback traceback.print_exc() any_failures = 1 + finally: + if show_all_warnings: + Errors.LEVEL = old_warning_level + if any_failures: if raise_on_failure: raise CompileError(None, pyx_file) @@ -1237,7 +1295,7 @@ def cythonize_one(pyx_file, c_file, fingerprint, quiet, options=None, else: fingerprint_file = zip_fingerprint_file with contextlib.closing(zipfile.ZipFile( - fingerprint_file + '.tmp', 'w', zipfile_compression_mode)) as zip: + fingerprint_file + '.tmp', 'w', zipfile_compression_mode)) as zip: for artifact in artifacts: zip.write(artifact, os.path.basename(artifact)) os.rename(fingerprint_file + '.tmp', fingerprint_file) @@ -1261,9 +1319,10 @@ def _init_multiprocessing_helper(): def cleanup_cache(cache, target_size, ratio=.85): try: p = subprocess.Popen(['du', '-s', '-k', os.path.abspath(cache)], stdout=subprocess.PIPE) + stdout, _ = p.communicate() res = p.wait() if res == 0: - total_size = 1024 * int(p.stdout.read().strip().split()[0]) + total_size = 1024 * int(stdout.strip().split()[0]) if total_size < target_size: return except (OSError, ValueError): diff --git a/Cython/Build/Inline.py b/Cython/Build/Inline.py index a14862901..15d26dbf8 100644 --- a/Cython/Build/Inline.py +++ b/Cython/Build/Inline.py @@ -1,32 +1,32 @@ from __future__ import absolute_import -import sys, os, re, inspect -import imp - -try: - import hashlib -except ImportError: - import md5 as hashlib +import hashlib +import inspect +import os +import re +import sys from distutils.core import Distribution, Extension from distutils.command.build_ext import build_ext import Cython -from ..Compiler.Main import Context, CompilationOptions, default_options +from ..Compiler.Main import Context +from ..Compiler.Options import default_options -from ..Compiler.ParseTreeTransforms import (CythonTransform, - SkipDeclarations, AnalyseDeclarationsTransform, EnvTransform) +from ..Compiler.Visitor import CythonTransform, EnvTransform +from ..Compiler.ParseTreeTransforms import SkipDeclarations from ..Compiler.TreeFragment import parse_from_strings from ..Compiler.StringEncoding import _unicode from .Dependencies import strip_string_literals, cythonize, cached_function -from ..Compiler import Pipeline, Nodes +from ..Compiler import Pipeline from ..Utils import get_cython_cache_dir import cython as cython_module -IS_PY3 = sys.version_info >= (3, 0) + +IS_PY3 = sys.version_info >= (3,) # A utility function to convert user-supplied ASCII strings to unicode. -if sys.version_info[0] < 3: +if not IS_PY3: def to_unicode(s): if isinstance(s, bytes): return s.decode('ascii') @@ -35,6 +35,7 @@ if sys.version_info[0] < 3: else: to_unicode = lambda x: x + if sys.version_info < (3, 5): import imp def load_dynamic(name, module_path): @@ -48,6 +49,7 @@ else: spec.loader.exec_module(module) return module + class UnboundSymbols(EnvTransform, SkipDeclarations): def __init__(self): CythonTransform.__init__(self, None) @@ -132,6 +134,7 @@ def _create_context(cython_include_dirs): _cython_inline_cache = {} _cython_inline_default_context = _create_context(('.',)) + def _populate_unbound(kwds, unbound_symbols, locals=None, globals=None): for symbol in unbound_symbols: if symbol not in kwds: @@ -148,10 +151,12 @@ def _populate_unbound(kwds, unbound_symbols, locals=None, globals=None): else: print("Couldn't find %r" % symbol) + def _inline_key(orig_code, arg_sigs, language_level): key = orig_code, arg_sigs, sys.version_info, sys.executable, language_level, Cython.__version__ return hashlib.sha1(_unicode(key).encode('utf-8')).hexdigest() + def cython_inline(code, get_type=unsafe_type, lib_dir=os.path.join(get_cython_cache_dir(), 'inline'), cython_include_dirs=None, cython_compiler_directives=None, @@ -161,12 +166,14 @@ def cython_inline(code, get_type=unsafe_type, get_type = lambda x: 'object' ctx = _create_context(tuple(cython_include_dirs)) if cython_include_dirs else _cython_inline_default_context - cython_compiler_directives = dict(cython_compiler_directives or {}) + cython_compiler_directives = dict(cython_compiler_directives) if cython_compiler_directives else {} if language_level is None and 'language_level' not in cython_compiler_directives: language_level = '3str' if language_level is not None: cython_compiler_directives['language_level'] = language_level + key_hash = None + # Fast path if this has been called in this session. _unbound_symbols = _cython_inline_cache.get(code) if _unbound_symbols is not None: @@ -202,7 +209,8 @@ def cython_inline(code, get_type=unsafe_type, del kwds[name] arg_names = sorted(kwds) arg_sigs = tuple([(get_type(kwds[arg], ctx), arg) for arg in arg_names]) - key_hash = _inline_key(orig_code, arg_sigs, language_level) + if key_hash is None: + key_hash = _inline_key(orig_code, arg_sigs, language_level) module_name = "_cython_inline_" + key_hash if module_name in sys.modules: @@ -221,6 +229,7 @@ def cython_inline(code, get_type=unsafe_type, os.makedirs(lib_dir) if force or not os.path.isfile(module_path): cflags = [] + define_macros = [] c_include_dirs = [] qualified = re.compile(r'([.\w]+)[.]') for type, _ in arg_sigs: @@ -231,6 +240,7 @@ def cython_inline(code, get_type=unsafe_type, if m.groups()[0] == 'numpy': import numpy c_include_dirs.append(numpy.get_include()) + define_macros.append(("NPY_NO_DEPRECATED_API", "NPY_1_7_API_VERSION")) # cflags.append('-Wno-unused') module_body, func_body = extract_func_code(code) params = ', '.join(['%s %s' % a for a in arg_sigs]) @@ -253,10 +263,12 @@ def __invoke(%(params)s): finally: fh.close() extension = Extension( - name = module_name, - sources = [pyx_file], - include_dirs = c_include_dirs, - extra_compile_args = cflags) + name=module_name, + sources=[pyx_file], + include_dirs=c_include_dirs or None, + extra_compile_args=cflags or None, + define_macros=define_macros or None, + ) if build_extension is None: build_extension = _get_build_extension() build_extension.extensions = cythonize( @@ -274,6 +286,7 @@ def __invoke(%(params)s): arg_list = [kwds[arg] for arg in arg_names] return module.__invoke(*arg_list) + # Cached suffix used by cython_inline above. None should get # overridden with actual value upon the first cython_inline invocation cython_inline.so_ext = None @@ -318,37 +331,6 @@ def extract_func_code(code): return '\n'.join(module), ' ' + '\n '.join(function) -try: - from inspect import getcallargs -except ImportError: - def getcallargs(func, *arg_values, **kwd_values): - all = {} - args, varargs, kwds, defaults = inspect.getargspec(func) - if varargs is not None: - all[varargs] = arg_values[len(args):] - for name, value in zip(args, arg_values): - all[name] = value - for name, value in list(kwd_values.items()): - if name in args: - if name in all: - raise TypeError("Duplicate argument %s" % name) - all[name] = kwd_values.pop(name) - if kwds is not None: - all[kwds] = kwd_values - elif kwd_values: - raise TypeError("Unexpected keyword arguments: %s" % list(kwd_values)) - if defaults is None: - defaults = () - first_default = len(args) - len(defaults) - for ix, name in enumerate(args): - if name not in all: - if ix >= first_default: - all[name] = defaults[ix - first_default] - else: - raise TypeError("Missing argument: %s" % name) - return all - - def get_body(source): ix = source.index(':') if source[:5] == 'lambda': @@ -366,7 +348,7 @@ class RuntimeCompiledFunction(object): self._body = get_body(inspect.getsource(f)) def __call__(self, *args, **kwds): - all = getcallargs(self._f, *args, **kwds) + all = inspect.getcallargs(self._f, *args, **kwds) if IS_PY3: return cython_inline(self._body, locals=self._f.__globals__, globals=self._f.__globals__, **all) else: diff --git a/Cython/Build/IpythonMagic.py b/Cython/Build/IpythonMagic.py index 7abb97ec7..f074be168 100644 --- a/Cython/Build/IpythonMagic.py +++ b/Cython/Build/IpythonMagic.py @@ -60,15 +60,11 @@ IO_ENCODING = sys.getfilesystemencoding() IS_PY2 = sys.version_info[0] < 3 try: - reload -except NameError: # Python 3 - from imp import reload - -try: - import hashlib -except ImportError: - import md5 as hashlib + from importlib import reload +except ImportError: # Python 2 had a builtin function + pass +import hashlib from distutils.core import Distribution, Extension from distutils.command.build_ext import build_ext @@ -86,6 +82,7 @@ from ..Shadow import __version__ as cython_version from ..Compiler.Errors import CompileError from .Inline import cython_inline from .Dependencies import cythonize +from ..Utils import captured_fd, print_captured PGO_CONFIG = { @@ -192,10 +189,15 @@ class CythonMagics(Magics): @magic_arguments.magic_arguments() @magic_arguments.argument( - '-a', '--annotate', action='store_true', default=False, + '-a', '--annotate', action='store_const', const='default', dest='annotate', help="Produce a colorized HTML version of the source." ) @magic_arguments.argument( + '--annotate-fullc', action='store_const', const='fullc', dest='annotate', + help="Produce a colorized HTML version of the source " + "which includes entire generated C/C++-code." + ) + @magic_arguments.argument( '-+', '--cplus', action='store_true', default=False, help="Output a C++ rather than C file." ) @@ -317,7 +319,7 @@ class CythonMagics(Magics): if args.name: module_name = str(args.name) # no-op in Py3 else: - module_name = "_cython_magic_" + hashlib.md5(str(key).encode('utf-8')).hexdigest() + module_name = "_cython_magic_" + hashlib.sha1(str(key).encode('utf-8')).hexdigest() html_file = os.path.join(lib_dir, module_name + '.html') module_path = os.path.join(lib_dir, module_name + self.so_ext) @@ -341,13 +343,25 @@ class CythonMagics(Magics): if args.pgo: self._profile_pgo_wrapper(extension, lib_dir) + def print_compiler_output(stdout, stderr, where): + # On windows, errors are printed to stdout, we redirect both to sys.stderr. + print_captured(stdout, where, u"Content of stdout:\n") + print_captured(stderr, where, u"Content of stderr:\n") + + get_stderr = get_stdout = None try: - self._build_extension(extension, lib_dir, pgo_step_name='use' if args.pgo else None, - quiet=args.quiet) - except distutils.errors.CompileError: - # Build failed and printed error message + with captured_fd(1) as get_stdout: + with captured_fd(2) as get_stderr: + self._build_extension( + extension, lib_dir, pgo_step_name='use' if args.pgo else None, quiet=args.quiet) + except (distutils.errors.CompileError, distutils.errors.LinkError): + # Build failed, print error message from compiler/linker + print_compiler_output(get_stdout(), get_stderr(), sys.stderr) return None + # Build seems ok, but we might still want to show any warnings that occurred + print_compiler_output(get_stdout(), get_stderr(), sys.stdout) + module = imp.load_dynamic(module_name, module_path) self._import_all(module) @@ -439,12 +453,11 @@ class CythonMagics(Magics): quiet=quiet, annotate=args.annotate, force=True, + language_level=min(3, sys.version_info[0]), ) if args.language_level is not None: assert args.language_level in (2, 3) opts['language_level'] = args.language_level - elif sys.version_info[0] >= 3: - opts['language_level'] = 3 return cythonize([extension], **opts) except CompileError: return None diff --git a/Cython/Build/Tests/TestCyCache.py b/Cython/Build/Tests/TestCyCache.py index a3224b417..7a44d89e9 100644 --- a/Cython/Build/Tests/TestCyCache.py +++ b/Cython/Build/Tests/TestCyCache.py @@ -33,25 +33,31 @@ class TestCyCache(CythonTest): a_pyx = os.path.join(self.src_dir, 'a.pyx') a_c = a_pyx[:-4] + '.c' - open(a_pyx, 'w').write(content1) + with open(a_pyx, 'w') as f: + f.write(content1) self.fresh_cythonize(a_pyx, cache=self.cache_dir) self.fresh_cythonize(a_pyx, cache=self.cache_dir) self.assertEqual(1, len(self.cache_files('a.c*'))) - a_contents1 = open(a_c).read() + with open(a_c) as f: + a_contents1 = f.read() os.unlink(a_c) - open(a_pyx, 'w').write(content2) + with open(a_pyx, 'w') as f: + f.write(content2) self.fresh_cythonize(a_pyx, cache=self.cache_dir) - a_contents2 = open(a_c).read() + with open(a_c) as f: + a_contents2 = f.read() os.unlink(a_c) self.assertNotEqual(a_contents1, a_contents2, 'C file not changed!') self.assertEqual(2, len(self.cache_files('a.c*'))) - open(a_pyx, 'w').write(content1) + with open(a_pyx, 'w') as f: + f.write(content1) self.fresh_cythonize(a_pyx, cache=self.cache_dir) self.assertEqual(2, len(self.cache_files('a.c*'))) - a_contents = open(a_c).read() + with open(a_c) as f: + a_contents = f.read() self.assertEqual( a_contents, a_contents1, msg='\n'.join(list(difflib.unified_diff( @@ -60,13 +66,15 @@ class TestCyCache(CythonTest): def test_cycache_uses_cache(self): a_pyx = os.path.join(self.src_dir, 'a.pyx') a_c = a_pyx[:-4] + '.c' - open(a_pyx, 'w').write('pass') + with open(a_pyx, 'w') as f: + f.write('pass') self.fresh_cythonize(a_pyx, cache=self.cache_dir) a_cache = os.path.join(self.cache_dir, os.listdir(self.cache_dir)[0]) gzip.GzipFile(a_cache, 'wb').write('fake stuff'.encode('ascii')) os.unlink(a_c) self.fresh_cythonize(a_pyx, cache=self.cache_dir) - a_contents = open(a_c).read() + with open(a_c) as f: + a_contents = f.read() self.assertEqual(a_contents, 'fake stuff', 'Unexpected contents: %s...' % a_contents[:100]) @@ -75,7 +83,8 @@ class TestCyCache(CythonTest): a_c = a_pyx[:-4] + '.c' a_h = a_pyx[:-4] + '.h' a_api_h = a_pyx[:-4] + '_api.h' - open(a_pyx, 'w').write('cdef public api int foo(int x): return x\n') + with open(a_pyx, 'w') as f: + f.write('cdef public api int foo(int x): return x\n') self.fresh_cythonize(a_pyx, cache=self.cache_dir) expected = [a_c, a_h, a_api_h] for output in expected: @@ -89,7 +98,8 @@ class TestCyCache(CythonTest): hash_pyx = os.path.join(self.src_dir, 'options.pyx') hash_c = hash_pyx[:-len('.pyx')] + '.c' - open(hash_pyx, 'w').write('pass') + with open(hash_pyx, 'w') as f: + f.write('pass') self.fresh_cythonize(hash_pyx, cache=self.cache_dir, cplus=False) self.assertEqual(1, len(self.cache_files('options.c*'))) diff --git a/Cython/Build/Tests/TestCythonizeArgsParser.py b/Cython/Build/Tests/TestCythonizeArgsParser.py new file mode 100644 index 000000000..c5a682dd6 --- /dev/null +++ b/Cython/Build/Tests/TestCythonizeArgsParser.py @@ -0,0 +1,482 @@ +from Cython.Build.Cythonize import ( + create_args_parser, parse_args_raw, parse_args, + parallel_compiles +) + +from Cython.Compiler import Options +from Cython.Compiler.Tests.Utils import backup_Options, restore_Options, check_global_options + +from unittest import TestCase + +import sys +try: + from StringIO import StringIO +except ImportError: + from io import StringIO # doesn't accept 'str' in Py2 + + +class TestCythonizeArgsParser(TestCase): + + def setUp(self): + TestCase.setUp(self) + self.parse_args = lambda x, parser=create_args_parser() : parse_args_raw(parser, x) + + + def are_default(self, options, skip): + # empty containers + empty_containers = ['directives', 'compile_time_env', 'options', 'excludes'] + are_none = ['language_level', 'annotate', 'build', 'build_inplace', 'force', 'quiet', 'lenient', 'keep_going', 'no_docstrings'] + for opt_name in empty_containers: + if len(getattr(options, opt_name))!=0 and (opt_name not in skip): + self.assertEqual(opt_name,"", msg="For option "+opt_name) + return False + for opt_name in are_none: + if (getattr(options, opt_name) is not None) and (opt_name not in skip): + self.assertEqual(opt_name,"", msg="For option "+opt_name) + return False + if options.parallel!=parallel_compiles and ('parallel' not in skip): + return False + return True + + # testing directives: + def test_directive_short(self): + options, args = self.parse_args(['-X', 'cdivision=True']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['directives'])) + self.assertEqual(options.directives['cdivision'], True) + + def test_directive_long(self): + options, args = self.parse_args(['--directive', 'cdivision=True']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['directives'])) + self.assertEqual(options.directives['cdivision'], True) + + def test_directive_multiple(self): + options, args = self.parse_args(['-X', 'cdivision=True', '-X', 'c_string_type=bytes']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['directives'])) + self.assertEqual(options.directives['cdivision'], True) + self.assertEqual(options.directives['c_string_type'], 'bytes') + + def test_directive_multiple_v2(self): + options, args = self.parse_args(['-X', 'cdivision=True,c_string_type=bytes']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['directives'])) + self.assertEqual(options.directives['cdivision'], True) + self.assertEqual(options.directives['c_string_type'], 'bytes') + + def test_directive_value_yes(self): + options, args = self.parse_args(['-X', 'cdivision=YeS']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['directives'])) + self.assertEqual(options.directives['cdivision'], True) + + def test_directive_value_no(self): + options, args = self.parse_args(['-X', 'cdivision=no']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['directives'])) + self.assertEqual(options.directives['cdivision'], False) + + def test_directive_value_invalid(self): + with self.assertRaises(ValueError) as context: + options, args = self.parse_args(['-X', 'cdivision=sadfasd']) + + def test_directive_key_invalid(self): + with self.assertRaises(ValueError) as context: + options, args = self.parse_args(['-X', 'abracadabra']) + + def test_directive_no_value(self): + with self.assertRaises(ValueError) as context: + options, args = self.parse_args(['-X', 'cdivision']) + + def test_directives_types(self): + directives = { + 'auto_pickle': True, + 'c_string_type': 'bytearray', + 'c_string_type': 'bytes', + 'c_string_type': 'str', + 'c_string_type': 'bytearray', + 'c_string_type': 'unicode', + 'c_string_encoding' : 'ascii', + 'language_level' : 2, + 'language_level' : 3, + 'language_level' : '3str', + 'set_initial_path' : 'my_initial_path', + } + for key, value in directives.items(): + cmd = '{key}={value}'.format(key=key, value=str(value)) + options, args = self.parse_args(['-X', cmd]) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['directives']), msg = "Error for option: "+cmd) + self.assertEqual(options.directives[key], value, msg = "Error for option: "+cmd) + + def test_directives_wrong(self): + directives = { + 'auto_pickle': 42, # for bool type + 'auto_pickle': 'NONONO', # for bool type + 'c_string_type': 'bites', + #'c_string_encoding' : 'a', + #'language_level' : 4, + } + for key, value in directives.items(): + cmd = '{key}={value}'.format(key=key, value=str(value)) + with self.assertRaises(ValueError, msg = "Error for option: "+cmd) as context: + options, args = self.parse_args(['-X', cmd]) + + def test_compile_time_env_short(self): + options, args = self.parse_args(['-E', 'MYSIZE=10']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['compile_time_env'])) + self.assertEqual(options.compile_time_env['MYSIZE'], 10) + + def test_compile_time_env_long(self): + options, args = self.parse_args(['--compile-time-env', 'MYSIZE=10']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['compile_time_env'])) + self.assertEqual(options.compile_time_env['MYSIZE'], 10) + + def test_compile_time_env_multiple(self): + options, args = self.parse_args(['-E', 'MYSIZE=10', '-E', 'ARRSIZE=11']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['compile_time_env'])) + self.assertEqual(options.compile_time_env['MYSIZE'], 10) + self.assertEqual(options.compile_time_env['ARRSIZE'], 11) + + def test_compile_time_env_multiple_v2(self): + options, args = self.parse_args(['-E', 'MYSIZE=10,ARRSIZE=11']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['compile_time_env'])) + self.assertEqual(options.compile_time_env['MYSIZE'], 10) + self.assertEqual(options.compile_time_env['ARRSIZE'], 11) + + #testing options + def test_option_short(self): + options, args = self.parse_args(['-s', 'docstrings=True']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['options'])) + self.assertEqual(options.options['docstrings'], True) + + def test_option_long(self): + options, args = self.parse_args(['--option', 'docstrings=True']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['options'])) + self.assertEqual(options.options['docstrings'], True) + + def test_option_multiple(self): + options, args = self.parse_args(['-s', 'docstrings=True', '-s', 'buffer_max_dims=8']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['options'])) + self.assertEqual(options.options['docstrings'], True) + self.assertEqual(options.options['buffer_max_dims'], True) # really? + + def test_option_multiple_v2(self): + options, args = self.parse_args(['-s', 'docstrings=True,buffer_max_dims=8']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['options'])) + self.assertEqual(options.options['docstrings'], True) + self.assertEqual(options.options['buffer_max_dims'], True) # really? + + def test_option_value_yes(self): + options, args = self.parse_args(['-s', 'docstrings=YeS']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['options'])) + self.assertEqual(options.options['docstrings'], True) + + def test_option_value_4242(self): + options, args = self.parse_args(['-s', 'docstrings=4242']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['options'])) + self.assertEqual(options.options['docstrings'], True) + + def test_option_value_0(self): + options, args = self.parse_args(['-s', 'docstrings=0']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['options'])) + self.assertEqual(options.options['docstrings'], False) + + def test_option_value_emptystr(self): + options, args = self.parse_args(['-s', 'docstrings=']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['options'])) + self.assertEqual(options.options['docstrings'], True) + + def test_option_value_a_str(self): + options, args = self.parse_args(['-s', 'docstrings=BB']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['options'])) + self.assertEqual(options.options['docstrings'], True) + + def test_option_value_no(self): + options, args = self.parse_args(['-s', 'docstrings=nO']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['options'])) + self.assertEqual(options.options['docstrings'], False) + + def test_option_no_value(self): + options, args = self.parse_args(['-s', 'docstrings']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['options'])) + self.assertEqual(options.options['docstrings'], True) + + def test_option_any_key(self): + options, args = self.parse_args(['-s', 'abracadabra']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['options'])) + self.assertEqual(options.options['abracadabra'], True) + + def test_language_level_2(self): + options, args = self.parse_args(['-2']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['language_level'])) + self.assertEqual(options.language_level, 2) + + def test_language_level_3(self): + options, args = self.parse_args(['-3']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['language_level'])) + self.assertEqual(options.language_level, 3) + + def test_language_level_3str(self): + options, args = self.parse_args(['--3str']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['language_level'])) + self.assertEqual(options.language_level, '3str') + + def test_annotate_short(self): + options, args = self.parse_args(['-a']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['annotate'])) + self.assertEqual(options.annotate, 'default') + + def test_annotate_long(self): + options, args = self.parse_args(['--annotate']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['annotate'])) + self.assertEqual(options.annotate, 'default') + + def test_annotate_fullc(self): + options, args = self.parse_args(['--annotate-fullc']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['annotate'])) + self.assertEqual(options.annotate, 'fullc') + + def test_annotate_and_positional(self): + options, args = self.parse_args(['-a', 'foo.pyx']) + self.assertEqual(args, ['foo.pyx']) + self.assertTrue(self.are_default(options, ['annotate'])) + self.assertEqual(options.annotate, 'default') + + def test_annotate_and_optional(self): + options, args = self.parse_args(['-a', '--3str']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['annotate', 'language_level'])) + self.assertEqual(options.annotate, 'default') + self.assertEqual(options.language_level, '3str') + + def test_exclude_short(self): + options, args = self.parse_args(['-x', '*.pyx']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['excludes'])) + self.assertTrue('*.pyx' in options.excludes) + + def test_exclude_long(self): + options, args = self.parse_args(['--exclude', '*.pyx']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['excludes'])) + self.assertTrue('*.pyx' in options.excludes) + + def test_exclude_multiple(self): + options, args = self.parse_args(['--exclude', '*.pyx', '--exclude', '*.py', ]) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['excludes'])) + self.assertEqual(options.excludes, ['*.pyx', '*.py']) + + def test_build_short(self): + options, args = self.parse_args(['-b']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['build'])) + self.assertEqual(options.build, True) + + def test_build_long(self): + options, args = self.parse_args(['--build']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['build'])) + self.assertEqual(options.build, True) + + def test_inplace_short(self): + options, args = self.parse_args(['-i']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['build_inplace'])) + self.assertEqual(options.build_inplace, True) + + def test_inplace_long(self): + options, args = self.parse_args(['--inplace']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['build_inplace'])) + self.assertEqual(options.build_inplace, True) + + def test_parallel_short(self): + options, args = self.parse_args(['-j', '42']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['parallel'])) + self.assertEqual(options.parallel, 42) + + def test_parallel_long(self): + options, args = self.parse_args(['--parallel', '42']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['parallel'])) + self.assertEqual(options.parallel, 42) + + def test_force_short(self): + options, args = self.parse_args(['-f']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['force'])) + self.assertEqual(options.force, True) + + def test_force_long(self): + options, args = self.parse_args(['--force']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['force'])) + self.assertEqual(options.force, True) + + def test_quite_short(self): + options, args = self.parse_args(['-q']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['quiet'])) + self.assertEqual(options.quiet, True) + + def test_quite_long(self): + options, args = self.parse_args(['--quiet']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['quiet'])) + self.assertEqual(options.quiet, True) + + def test_lenient_long(self): + options, args = self.parse_args(['--lenient']) + self.assertTrue(self.are_default(options, ['lenient'])) + self.assertFalse(args) + self.assertEqual(options.lenient, True) + + def test_keep_going_short(self): + options, args = self.parse_args(['-k']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['keep_going'])) + self.assertEqual(options.keep_going, True) + + def test_keep_going_long(self): + options, args = self.parse_args(['--keep-going']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['keep_going'])) + self.assertEqual(options.keep_going, True) + + def test_no_docstrings_long(self): + options, args = self.parse_args(['--no-docstrings']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['no_docstrings'])) + self.assertEqual(options.no_docstrings, True) + + def test_file_name(self): + options, args = self.parse_args(['file1.pyx', 'file2.pyx']) + self.assertEqual(len(args), 2) + self.assertEqual(args[0], 'file1.pyx') + self.assertEqual(args[1], 'file2.pyx') + self.assertTrue(self.are_default(options, [])) + + def test_option_first(self): + options, args = self.parse_args(['-i', 'file.pyx']) + self.assertEqual(args, ['file.pyx']) + self.assertEqual(options.build_inplace, True) + self.assertTrue(self.are_default(options, ['build_inplace'])) + + def test_file_inbetween(self): + options, args = self.parse_args(['-i', 'file.pyx', '-a']) + self.assertEqual(args, ['file.pyx']) + self.assertEqual(options.build_inplace, True) + self.assertEqual(options.annotate, 'default') + self.assertTrue(self.are_default(options, ['build_inplace', 'annotate'])) + + def test_option_trailing(self): + options, args = self.parse_args(['file.pyx', '-i']) + self.assertEqual(args, ['file.pyx']) + self.assertEqual(options.build_inplace, True) + self.assertTrue(self.are_default(options, ['build_inplace'])) + + def test_interspersed_positional(self): + options, sources = self.parse_args([ + 'file1.pyx', '-a', + 'file2.pyx' + ]) + self.assertEqual(sources, ['file1.pyx', 'file2.pyx']) + self.assertEqual(options.annotate, 'default') + self.assertTrue(self.are_default(options, ['annotate'])) + + def test_interspersed_positional2(self): + options, sources = self.parse_args([ + 'file1.pyx', '-a', + 'file2.pyx', '-a', 'file3.pyx' + ]) + self.assertEqual(sources, ['file1.pyx', 'file2.pyx', 'file3.pyx']) + self.assertEqual(options.annotate, 'default') + self.assertTrue(self.are_default(options, ['annotate'])) + + def test_interspersed_positional3(self): + options, sources = self.parse_args([ + '-f', 'f1', 'f2', '-a', + 'f3', 'f4', '-a', 'f5' + ]) + self.assertEqual(sources, ['f1', 'f2', 'f3', 'f4', 'f5']) + self.assertEqual(options.annotate, 'default') + self.assertEqual(options.force, True) + self.assertTrue(self.are_default(options, ['annotate', 'force'])) + + def test_wrong_option(self): + old_stderr = sys.stderr + stderr = sys.stderr = StringIO() + try: + self.assertRaises(SystemExit, self.parse_args, + ['--unknown-option'] + ) + finally: + sys.stderr = old_stderr + self.assertTrue(stderr.getvalue()) + + +class TestParseArgs(TestCase): + def setUp(self): + self._options_backup = backup_Options() + + def tearDown(self): + restore_Options(self._options_backup) + + def check_default_global_options(self, white_list=[]): + self.assertEqual(check_global_options(self._options_backup, white_list), "") + + def test_build_set_for_inplace(self): + options, args = parse_args(['foo.pyx', '-i']) + self.assertEqual(options.build, True) + self.check_default_global_options() + + def test_lenient(self): + options, sources = parse_args(['foo.pyx', '--lenient']) + self.assertEqual(sources, ['foo.pyx']) + self.assertEqual(Options.error_on_unknown_names, False) + self.assertEqual(Options.error_on_uninitialized, False) + self.check_default_global_options(['error_on_unknown_names', 'error_on_uninitialized']) + + def test_annotate(self): + options, sources = parse_args(['foo.pyx', '--annotate']) + self.assertEqual(sources, ['foo.pyx']) + self.assertEqual(Options.annotate, 'default') + self.check_default_global_options(['annotate']) + + def test_annotate_fullc(self): + options, sources = parse_args(['foo.pyx', '--annotate-fullc']) + self.assertEqual(sources, ['foo.pyx']) + self.assertEqual(Options.annotate, 'fullc') + self.check_default_global_options(['annotate']) + + def test_no_docstrings(self): + options, sources = parse_args(['foo.pyx', '--no-docstrings']) + self.assertEqual(sources, ['foo.pyx']) + self.assertEqual(Options.docstrings, False) + self.check_default_global_options(['docstrings']) diff --git a/Cython/Build/Tests/TestInline.py b/Cython/Build/Tests/TestInline.py index 5ef9fec4e..35d9a29cd 100644 --- a/Cython/Build/Tests/TestInline.py +++ b/Cython/Build/Tests/TestInline.py @@ -1,4 +1,6 @@ -import os, tempfile +import os +import tempfile +import unittest from Cython.Shadow import inline from Cython.Build.Inline import safe_type from Cython.TestUtils import CythonTest @@ -24,10 +26,10 @@ class TestInline(CythonTest): self.test_kwds['lib_dir'] = lib_dir def test_simple(self): - self.assertEquals(inline("return 1+2", **self.test_kwds), 3) + self.assertEqual(inline("return 1+2", **self.test_kwds), 3) def test_types(self): - self.assertEquals(inline(""" + self.assertEqual(inline(""" cimport cython return cython.typeof(a), cython.typeof(b) """, a=1.0, b=[], **self.test_kwds), ('double', 'list object')) @@ -35,13 +37,13 @@ class TestInline(CythonTest): def test_locals(self): a = 1 b = 2 - self.assertEquals(inline("return a+b", **self.test_kwds), 3) + self.assertEqual(inline("return a+b", **self.test_kwds), 3) def test_globals(self): - self.assertEquals(inline("return global_value + 1", **self.test_kwds), global_value + 1) + self.assertEqual(inline("return global_value + 1", **self.test_kwds), global_value + 1) def test_no_return(self): - self.assertEquals(inline(""" + self.assertEqual(inline(""" a = 1 cdef double b = 2 cdef c = [] @@ -49,7 +51,7 @@ class TestInline(CythonTest): def test_def_node(self): foo = inline("def foo(x): return x * x", **self.test_kwds)['foo'] - self.assertEquals(foo(7), 49) + self.assertEqual(foo(7), 49) def test_class_ref(self): class Type(object): @@ -64,7 +66,7 @@ class TestInline(CythonTest): c = cy.declare(cy.pointer(cy.float), &b) return b """, a=3, **self.test_kwds) - self.assertEquals(type(b), float) + self.assertEqual(type(b), float) def test_compiler_directives(self): self.assertEqual( @@ -85,12 +87,26 @@ class TestInline(CythonTest): inline(inline_divcode, language_level=3)['f'](5,2), 2.5 ) + self.assertEqual( + inline(inline_divcode, language_level=2)['f'](5,2), + 2 + ) - if has_numpy: - - def test_numpy(self): - import numpy - a = numpy.ndarray((10, 20)) - a[0,0] = 10 - self.assertEquals(safe_type(a), 'numpy.ndarray[numpy.float64_t, ndim=2]') - self.assertEquals(inline("return a[0,0]", a=a, **self.test_kwds), 10.0) + def test_repeated_use(self): + inline_mulcode = "def f(int a, int b): return a * b" + self.assertEqual(inline(inline_mulcode)['f'](5, 2), 10) + self.assertEqual(inline(inline_mulcode)['f'](5, 3), 15) + self.assertEqual(inline(inline_mulcode)['f'](6, 2), 12) + self.assertEqual(inline(inline_mulcode)['f'](5, 2), 10) + + f = inline(inline_mulcode)['f'] + self.assertEqual(f(5, 2), 10) + self.assertEqual(f(5, 3), 15) + + @unittest.skipIf(not has_numpy, "NumPy is not available") + def test_numpy(self): + import numpy + a = numpy.ndarray((10, 20)) + a[0,0] = 10 + self.assertEqual(safe_type(a), 'numpy.ndarray[numpy.float64_t, ndim=2]') + self.assertEqual(inline("return a[0,0]", a=a, **self.test_kwds), 10.0) diff --git a/Cython/Build/Tests/TestIpythonMagic.py b/Cython/Build/Tests/TestIpythonMagic.py index d9d8322a8..febb480ac 100644 --- a/Cython/Build/Tests/TestIpythonMagic.py +++ b/Cython/Build/Tests/TestIpythonMagic.py @@ -6,10 +6,12 @@ from __future__ import absolute_import import os +import io import sys from contextlib import contextmanager from Cython.Build import IpythonMagic from Cython.TestUtils import CythonTest +from Cython.Compiler.Annotate import AnnotationCCodeWriter try: import IPython.testing.globalipapp @@ -28,6 +30,26 @@ try: except ImportError: pass + +@contextmanager +def capture_output(): + backup = sys.stdout, sys.stderr + try: + replacement = [ + io.TextIOWrapper(io.BytesIO(), encoding=sys.stdout.encoding), + io.TextIOWrapper(io.BytesIO(), encoding=sys.stderr.encoding), + ] + sys.stdout, sys.stderr = replacement + output = [] + yield output + finally: + sys.stdout, sys.stderr = backup + for wrapper in replacement: + wrapper.seek(0) # rewind + output.append(wrapper.read()) + wrapper.close() + + code = u"""\ def f(x): return 2*x @@ -47,6 +69,27 @@ def main(): main() """ +compile_error_code = u'''\ +cdef extern from *: + """ + xxx a=1; + """ + int a; +def doit(): + return a +''' + +compile_warning_code = u'''\ +cdef extern from *: + """ + #pragma message ( "CWarning" ) + int a = 42; + """ + int a; +def doit(): + return a +''' + if sys.platform == 'win32': # not using IPython's decorators here because they depend on "nose" @@ -142,6 +185,39 @@ class TestIPythonMagic(CythonTest): self.assertEqual(ip.user_ns['g'], 2 // 10) self.assertEqual(ip.user_ns['h'], 2 // 10) + def test_cython_compile_error_shown(self): + ip = self._ip + with capture_output() as out: + ip.run_cell_magic('cython', '-3', compile_error_code) + captured_out, captured_err = out + + # it could be that c-level output is captured by distutil-extension + # (and not by us) and is printed to stdout: + captured_all = captured_out + "\n" + captured_err + self.assertTrue("error" in captured_all, msg="error in " + captured_all) + + def test_cython_link_error_shown(self): + ip = self._ip + with capture_output() as out: + ip.run_cell_magic('cython', '-3 -l=xxxxxxxx', code) + captured_out, captured_err = out + + # it could be that c-level output is captured by distutil-extension + # (and not by us) and is printed to stdout: + captured_all = captured_out + "\n!" + captured_err + self.assertTrue("error" in captured_all, msg="error in " + captured_all) + + def test_cython_warning_shown(self): + ip = self._ip + with capture_output() as out: + # force rebuild, otherwise no warning as after the first success + # no build step is performed + ip.run_cell_magic('cython', '-3 -f', compile_warning_code) + captured_out, captured_err = out + + # check that warning was printed to stdout even if build hasn't failed + self.assertTrue("CWarning" in captured_out) + @skip_win32('Skip on Windows') def test_cython3_pgo(self): # The Cython cell defines the functions f() and call(). @@ -195,11 +271,37 @@ x = sin(0.0) ip.run_cell_magic('cython', '--verbose', code) ip.ex('g = f(10)') self.assertEqual(ip.user_ns['g'], 20.0) - self.assertEquals([verbose_log.INFO, verbose_log.DEBUG, verbose_log.INFO], + self.assertEqual([verbose_log.INFO, verbose_log.DEBUG, verbose_log.INFO], verbose_log.thresholds) with mock_distutils() as normal_log: ip.run_cell_magic('cython', '', code) ip.ex('g = f(10)') self.assertEqual(ip.user_ns['g'], 20.0) - self.assertEquals([normal_log.INFO], normal_log.thresholds) + self.assertEqual([normal_log.INFO], normal_log.thresholds) + + def test_cython_no_annotate(self): + ip = self._ip + html = ip.run_cell_magic('cython', '', code) + self.assertTrue(html is None) + + def test_cython_annotate(self): + ip = self._ip + html = ip.run_cell_magic('cython', '--annotate', code) + # somewhat brittle way to differentiate between annotated htmls + # with/without complete source code: + self.assertTrue(AnnotationCCodeWriter.COMPLETE_CODE_TITLE not in html.data) + + def test_cython_annotate_default(self): + ip = self._ip + html = ip.run_cell_magic('cython', '-a', code) + # somewhat brittle way to differentiate between annotated htmls + # with/without complete source code: + self.assertTrue(AnnotationCCodeWriter.COMPLETE_CODE_TITLE not in html.data) + + def test_cython_annotate_complete_c_code(self): + ip = self._ip + html = ip.run_cell_magic('cython', '--annotate-fullc', code) + # somewhat brittle way to differentiate between annotated htmls + # with/without complete source code: + self.assertTrue(AnnotationCCodeWriter.COMPLETE_CODE_TITLE in html.data) diff --git a/Cython/Build/Tests/TestRecythonize.py b/Cython/Build/Tests/TestRecythonize.py new file mode 100644 index 000000000..eb87018cb --- /dev/null +++ b/Cython/Build/Tests/TestRecythonize.py @@ -0,0 +1,212 @@ +import shutil +import os +import tempfile +import time + +import Cython.Build.Dependencies +import Cython.Utils +from Cython.TestUtils import CythonTest + + +def fresh_cythonize(*args, **kwargs): + Cython.Utils.clear_function_caches() + Cython.Build.Dependencies._dep_tree = None # discard method caches + Cython.Build.Dependencies.cythonize(*args, **kwargs) + +class TestRecythonize(CythonTest): + + def setUp(self): + CythonTest.setUp(self) + self.temp_dir = ( + tempfile.mkdtemp( + prefix='recythonize-test', + dir='TEST_TMP' if os.path.isdir('TEST_TMP') else None + ) + ) + + def tearDown(self): + CythonTest.tearDown(self) + shutil.rmtree(self.temp_dir) + + def test_recythonize_pyx_on_pxd_change(self): + + src_dir = tempfile.mkdtemp(prefix='src', dir=self.temp_dir) + + a_pxd = os.path.join(src_dir, 'a.pxd') + a_pyx = os.path.join(src_dir, 'a.pyx') + a_c = os.path.join(src_dir, 'a.c') + dep_tree = Cython.Build.Dependencies.create_dependency_tree() + + with open(a_pxd, 'w') as f: + f.write('cdef int value\n') + + with open(a_pyx, 'w') as f: + f.write('value = 1\n') + + + # The dependencies for "a.pyx" are "a.pxd" and "a.pyx". + self.assertEqual({a_pxd, a_pyx}, dep_tree.all_dependencies(a_pyx)) + + # Cythonize to create a.c + fresh_cythonize(a_pyx) + + # Sleep to address coarse time-stamp precision. + time.sleep(1) + + with open(a_c) as f: + a_c_contents1 = f.read() + + with open(a_pxd, 'w') as f: + f.write('cdef double value\n') + + fresh_cythonize(a_pyx) + + with open(a_c) as f: + a_c_contents2 = f.read() + + self.assertTrue("__pyx_v_1a_value = 1;" in a_c_contents1) + self.assertFalse("__pyx_v_1a_value = 1;" in a_c_contents2) + self.assertTrue("__pyx_v_1a_value = 1.0;" in a_c_contents2) + self.assertFalse("__pyx_v_1a_value = 1.0;" in a_c_contents1) + + + def test_recythonize_py_on_pxd_change(self): + + src_dir = tempfile.mkdtemp(prefix='src', dir=self.temp_dir) + + a_pxd = os.path.join(src_dir, 'a.pxd') + a_py = os.path.join(src_dir, 'a.py') + a_c = os.path.join(src_dir, 'a.c') + dep_tree = Cython.Build.Dependencies.create_dependency_tree() + + with open(a_pxd, 'w') as f: + f.write('cdef int value\n') + + with open(a_py, 'w') as f: + f.write('value = 1\n') + + + # The dependencies for "a.py" are "a.pxd" and "a.py". + self.assertEqual({a_pxd, a_py}, dep_tree.all_dependencies(a_py)) + + # Cythonize to create a.c + fresh_cythonize(a_py) + + # Sleep to address coarse time-stamp precision. + time.sleep(1) + + with open(a_c) as f: + a_c_contents1 = f.read() + + with open(a_pxd, 'w') as f: + f.write('cdef double value\n') + + fresh_cythonize(a_py) + + with open(a_c) as f: + a_c_contents2 = f.read() + + + self.assertTrue("__pyx_v_1a_value = 1;" in a_c_contents1) + self.assertFalse("__pyx_v_1a_value = 1;" in a_c_contents2) + self.assertTrue("__pyx_v_1a_value = 1.0;" in a_c_contents2) + self.assertFalse("__pyx_v_1a_value = 1.0;" in a_c_contents1) + + def test_recythonize_pyx_on_dep_pxd_change(self): + src_dir = tempfile.mkdtemp(prefix='src', dir=self.temp_dir) + + a_pxd = os.path.join(src_dir, 'a.pxd') + a_pyx = os.path.join(src_dir, 'a.pyx') + b_pyx = os.path.join(src_dir, 'b.pyx') + b_c = os.path.join(src_dir, 'b.c') + dep_tree = Cython.Build.Dependencies.create_dependency_tree() + + with open(a_pxd, 'w') as f: + f.write('cdef int value\n') + + with open(a_pyx, 'w') as f: + f.write('value = 1\n') + + with open(b_pyx, 'w') as f: + f.write('cimport a\n' + 'a.value = 2\n') + + + # The dependencies for "b.pyx" are "a.pxd" and "b.pyx". + self.assertEqual({a_pxd, b_pyx}, dep_tree.all_dependencies(b_pyx)) + + + # Cythonize to create b.c + fresh_cythonize([a_pyx, b_pyx]) + + # Sleep to address coarse time-stamp precision. + time.sleep(1) + + with open(b_c) as f: + b_c_contents1 = f.read() + + with open(a_pxd, 'w') as f: + f.write('cdef double value\n') + + fresh_cythonize([a_pyx, b_pyx]) + + with open(b_c) as f: + b_c_contents2 = f.read() + + + + self.assertTrue("__pyx_v_1a_value = 2;" in b_c_contents1) + self.assertFalse("__pyx_v_1a_value = 2;" in b_c_contents2) + self.assertTrue("__pyx_v_1a_value = 2.0;" in b_c_contents2) + self.assertFalse("__pyx_v_1a_value = 2.0;" in b_c_contents1) + + + + def test_recythonize_py_on_dep_pxd_change(self): + + src_dir = tempfile.mkdtemp(prefix='src', dir=self.temp_dir) + + a_pxd = os.path.join(src_dir, 'a.pxd') + a_pyx = os.path.join(src_dir, 'a.pyx') + b_pxd = os.path.join(src_dir, 'b.pxd') + b_py = os.path.join(src_dir, 'b.py') + b_c = os.path.join(src_dir, 'b.c') + dep_tree = Cython.Build.Dependencies.create_dependency_tree() + + with open(a_pxd, 'w') as f: + f.write('cdef int value\n') + + with open(a_pyx, 'w') as f: + f.write('value = 1\n') + + with open(b_pxd, 'w') as f: + f.write('cimport a\n') + + with open(b_py, 'w') as f: + f.write('a.value = 2\n') + + + # The dependencies for b.py are "a.pxd", "b.pxd" and "b.py". + self.assertEqual({a_pxd, b_pxd, b_py}, dep_tree.all_dependencies(b_py)) + + + # Cythonize to create b.c + fresh_cythonize([a_pyx, b_py]) + + # Sleep to address coarse time-stamp precision. + time.sleep(1) + + with open(b_c) as f: + b_c_contents1 = f.read() + + with open(a_pxd, 'w') as f: + f.write('cdef double value\n') + + fresh_cythonize([a_pyx, b_py]) + + with open(b_c) as f: + b_c_contents2 = f.read() + + self.assertTrue("__pyx_v_1a_value = 2;" in b_c_contents1) + self.assertFalse("__pyx_v_1a_value = 2;" in b_c_contents2) + self.assertTrue("__pyx_v_1a_value = 2.0;" in b_c_contents2) + self.assertFalse("__pyx_v_1a_value = 2.0;" in b_c_contents1) diff --git a/Cython/Build/Tests/TestStripLiterals.py b/Cython/Build/Tests/TestStripLiterals.py index a7572a508..cbe5c65a9 100644 --- a/Cython/Build/Tests/TestStripLiterals.py +++ b/Cython/Build/Tests/TestStripLiterals.py @@ -54,4 +54,3 @@ class TestStripLiterals(CythonTest): def test_extern(self): self.t("cdef extern from 'a.h': # comment", "cdef extern from '_L1_': #_L2_") - diff --git a/Cython/CodeWriter.py b/Cython/CodeWriter.py index 2e4646a65..1e5db318d 100644 --- a/Cython/CodeWriter.py +++ b/Cython/CodeWriter.py @@ -1,7 +1,6 @@ """ Serializes a Cython code tree to Cython code. This is primarily useful for debugging and testing purposes. - The output is in a strict format, no whitespace or comments from the input is preserved (and it could not be as it is not present in the code tree). """ @@ -10,6 +9,7 @@ from __future__ import absolute_import, print_function from .Compiler.Visitor import TreeVisitor from .Compiler.ExprNodes import * +from .Compiler.Nodes import CNameDeclaratorNode, CSimpleBaseTypeNode class LinesResult(object): @@ -28,7 +28,11 @@ class LinesResult(object): self.put(s) self.newline() + class DeclarationWriter(TreeVisitor): + """ + A Cython code writer that is limited to declarations nodes. + """ indent_string = u" " @@ -76,6 +80,14 @@ class DeclarationWriter(TreeVisitor): self.visit(item.default) self.put(u", ") self.visit(items[-1]) + if output_rhs and items[-1].default is not None: + self.put(u" = ") + self.visit(items[-1].default) + + def _visit_indented(self, node): + self.indent() + self.visit(node) + self.dedent() def visit_Node(self, node): raise AssertionError("Node not handled by serializer: %r" % node) @@ -92,9 +104,7 @@ class DeclarationWriter(TreeVisitor): else: file = u'"%s"' % node.include_file self.putline(u"cdef extern from %s:" % file) - self.indent() - self.visit(node.body) - self.dedent() + self._visit_indented(node.body) def visit_CPtrDeclaratorNode(self, node): self.put('*') @@ -111,13 +121,6 @@ class DeclarationWriter(TreeVisitor): self.visit(node.dimension) self.put(u']') - def visit_CArrayDeclaratorNode(self, node): - self.visit(node.base) - self.put(u'[') - if node.dimension is not None: - self.visit(node.dimension) - self.put(u']') - def visit_CFuncDeclaratorNode(self, node): # TODO: except, gil, etc. self.visit(node.base) @@ -136,13 +139,12 @@ class DeclarationWriter(TreeVisitor): self.put("short " * -node.longness) elif node.longness > 0: self.put("long " * node.longness) - self.put(node.name) + if node.name is not None: + self.put(node.name) def visit_CComplexBaseTypeNode(self, node): - self.put(u'(') self.visit(node.base_type) self.visit(node.declarator) - self.put(u')') def visit_CNestedBaseTypeNode(self, node): self.visit(node.base_type) @@ -162,7 +164,7 @@ class DeclarationWriter(TreeVisitor): self.comma_separated_list(node.declarators, output_rhs=True) self.endline() - def visit_container_node(self, node, decl, extras, attributes): + def _visit_container_node(self, node, decl, extras, attributes): # TODO: visibility self.startline(decl) if node.name: @@ -191,7 +193,7 @@ class DeclarationWriter(TreeVisitor): if node.packed: decl += u'packed ' decl += node.kind - self.visit_container_node(node, decl, None, node.attributes) + self._visit_container_node(node, decl, None, node.attributes) def visit_CppClassNode(self, node): extras = "" @@ -199,10 +201,10 @@ class DeclarationWriter(TreeVisitor): extras = u"[%s]" % ", ".join(node.templates) if node.base_classes: extras += "(%s)" % ", ".join(node.base_classes) - self.visit_container_node(node, u"cdef cppclass", extras, node.attributes) + self._visit_container_node(node, u"cdef cppclass", extras, node.attributes) def visit_CEnumDefNode(self, node): - self.visit_container_node(node, u"cdef enum", None, node.items) + self._visit_container_node(node, u"cdef enum", None, node.items) def visit_CEnumDefItemNode(self, node): self.startline(node.name) @@ -228,9 +230,7 @@ class DeclarationWriter(TreeVisitor): self.put(node.base_class_name) self.put(u")") self.endline(u":") - self.indent() - self.visit(node.body) - self.dedent() + self._visit_indented(node.body) def visit_CTypeDefNode(self, node): self.startline(u"ctypedef ") @@ -240,17 +240,49 @@ class DeclarationWriter(TreeVisitor): self.endline() def visit_FuncDefNode(self, node): + # TODO: support cdef + cpdef functions self.startline(u"def %s(" % node.name) self.comma_separated_list(node.args) self.endline(u"):") - self.indent() - self.visit(node.body) - self.dedent() + self._visit_indented(node.body) + + def visit_CFuncDefNode(self, node): + self.startline(u'cpdef ' if node.overridable else u'cdef ') + if node.modifiers: + self.put(' '.join(node.modifiers)) + self.put(' ') + if node.visibility != 'private': + self.put(node.visibility) + self.put(u' ') + if node.api: + self.put(u'api ') + + if node.base_type: + self.visit(node.base_type) + if node.base_type.name is not None: + self.put(u' ') + + # visit the CFuncDeclaratorNode, but put a `:` at the end of line + self.visit(node.declarator.base) + self.put(u'(') + self.comma_separated_list(node.declarator.args) + self.endline(u'):') + + self._visit_indented(node.body) def visit_CArgDeclNode(self, node): - if node.base_type.name is not None: + # For "CSimpleBaseTypeNode", the variable type may have been parsed as type. + # For other node types, the "name" is always None. + if not isinstance(node.base_type, CSimpleBaseTypeNode) or \ + node.base_type.name is not None: self.visit(node.base_type) - self.put(u" ") + + # If we printed something for "node.base_type", we may need to print an extra ' '. + # + # Special case: if "node.declarator" is a "CNameDeclaratorNode", + # its "name" might be an empty string, for example, for "cdef f(x)". + if node.declarator.declared_name(): + self.put(u" ") self.visit(node.declarator) if node.default is not None: self.put(u" = ") @@ -284,46 +316,20 @@ class DeclarationWriter(TreeVisitor): def visit_NameNode(self, node): self.put(node.name) - def visit_IntNode(self, node): - self.put(node.value) - - def visit_NoneNode(self, node): - self.put(u"None") - - def visit_NotNode(self, node): - self.put(u"(not ") - self.visit(node.operand) - self.put(u")") - def visit_DecoratorNode(self, node): self.startline("@") self.visit(node.decorator) self.endline() - def visit_BinopNode(self, node): - self.visit(node.operand1) - self.put(u" %s " % node.operator) - self.visit(node.operand2) - - def visit_AttributeNode(self, node): - self.visit(node.obj) - self.put(u".%s" % node.attribute) - - def visit_BoolNode(self, node): - self.put(str(node.value)) - - # FIXME: represent string nodes correctly - def visit_StringNode(self, node): - value = node.value - if value.encoding is not None: - value = value.encode(value.encoding) - self.put(repr(value)) - def visit_PassStatNode(self, node): self.startline(u"pass") self.endline() -class CodeWriter(DeclarationWriter): + +class StatementWriter(DeclarationWriter): + """ + A Cython code writer for most language statement features. + """ def visit_SingleAssignmentNode(self, node): self.startline() @@ -349,18 +355,17 @@ class CodeWriter(DeclarationWriter): def visit_ForInStatNode(self, node): self.startline(u"for ") - self.visit(node.target) + if node.target.is_sequence_constructor: + self.comma_separated_list(node.target.args) + else: + self.visit(node.target) self.put(u" in ") self.visit(node.iterator.sequence) self.endline(u":") - self.indent() - self.visit(node.body) - self.dedent() + self._visit_indented(node.body) if node.else_clause is not None: self.line(u"else:") - self.indent() - self.visit(node.else_clause) - self.dedent() + self._visit_indented(node.else_clause) def visit_IfStatNode(self, node): # The IfClauseNode is handled directly without a separate match @@ -368,50 +373,33 @@ class CodeWriter(DeclarationWriter): self.startline(u"if ") self.visit(node.if_clauses[0].condition) self.endline(":") - self.indent() - self.visit(node.if_clauses[0].body) - self.dedent() + self._visit_indented(node.if_clauses[0].body) for clause in node.if_clauses[1:]: self.startline("elif ") self.visit(clause.condition) self.endline(":") - self.indent() - self.visit(clause.body) - self.dedent() + self._visit_indented(clause.body) if node.else_clause is not None: self.line("else:") - self.indent() - self.visit(node.else_clause) - self.dedent() + self._visit_indented(node.else_clause) - def visit_SequenceNode(self, node): - self.comma_separated_list(node.args) # Might need to discover whether we need () around tuples...hmm... + def visit_WhileStatNode(self, node): + self.startline(u"while ") + self.visit(node.condition) + self.endline(u":") + self._visit_indented(node.body) + if node.else_clause is not None: + self.line("else:") + self._visit_indented(node.else_clause) - def visit_SimpleCallNode(self, node): - self.visit(node.function) - self.put(u"(") - self.comma_separated_list(node.args) - self.put(")") + def visit_ContinueStatNode(self, node): + self.line(u"continue") - def visit_GeneralCallNode(self, node): - self.visit(node.function) - self.put(u"(") - posarg = node.positional_args - if isinstance(posarg, AsTupleNode): - self.visit(posarg.arg) - else: - self.comma_separated_list(posarg.args) # TupleNode.args - if node.keyword_args: - if isinstance(node.keyword_args, DictNode): - for i, (name, value) in enumerate(node.keyword_args.key_value_pairs): - if i > 0: - self.put(', ') - self.visit(name) - self.put('=') - self.visit(value) - else: - raise Exception("Not implemented yet") - self.put(u")") + def visit_BreakStatNode(self, node): + self.line(u"break") + + def visit_SequenceNode(self, node): + self.comma_separated_list(node.args) # Might need to discover whether we need () around tuples...hmm... def visit_ExprStatNode(self, node): self.startline() @@ -433,25 +421,17 @@ class CodeWriter(DeclarationWriter): self.put(u" as ") self.visit(node.target) self.endline(u":") - self.indent() - self.visit(node.body) - self.dedent() + self._visit_indented(node.body) def visit_TryFinallyStatNode(self, node): self.line(u"try:") - self.indent() - self.visit(node.body) - self.dedent() + self._visit_indented(node.body) self.line(u"finally:") - self.indent() - self.visit(node.finally_clause) - self.dedent() + self._visit_indented(node.finally_clause) def visit_TryExceptStatNode(self, node): self.line(u"try:") - self.indent() - self.visit(node.body) - self.dedent() + self._visit_indented(node.body) for x in node.except_clauses: self.visit(x) if node.else_clause is not None: @@ -466,13 +446,13 @@ class CodeWriter(DeclarationWriter): self.put(u", ") self.visit(node.target) self.endline(":") - self.indent() - self.visit(node.body) - self.dedent() + self._visit_indented(node.body) def visit_ReturnStatNode(self, node): - self.startline("return ") - self.visit(node.value) + self.startline("return") + if node.value is not None: + self.put(u" ") + self.visit(node.value) self.endline() def visit_ReraiseStatNode(self, node): @@ -498,30 +478,10 @@ class CodeWriter(DeclarationWriter): self.put(self.tempnames[node.handle]) -class PxdWriter(DeclarationWriter): - def __call__(self, node): - print(u'\n'.join(self.write(node).lines)) - return node - - def visit_CFuncDefNode(self, node): - if 'inline' in node.modifiers: - return - if node.overridable: - self.startline(u'cpdef ') - else: - self.startline(u'cdef ') - if node.visibility != 'private': - self.put(node.visibility) - self.put(u' ') - if node.api: - self.put(u'api ') - self.visit(node.declarator) - - def visit_StatNode(self, node): - pass - - class ExpressionWriter(TreeVisitor): + """ + A Cython code writer that is intentionally limited to expressions. + """ def __init__(self, result=None): super(ExpressionWriter, self).__init__() @@ -551,12 +511,18 @@ class ExpressionWriter(TreeVisitor): def visit_Node(self, node): raise AssertionError("Node not handled by serializer: %r" % node) - def visit_NameNode(self, node): - self.put(node.name) + def visit_IntNode(self, node): + self.put(node.value) + + def visit_FloatNode(self, node): + self.put(node.value) def visit_NoneNode(self, node): self.put(u"None") + def visit_NameNode(self, node): + self.put(node.name) + def visit_EllipsisNode(self, node): self.put(u"...") @@ -814,3 +780,38 @@ class ExpressionWriter(TreeVisitor): # type(body) is Nodes.ExprStatNode body = body.expr.arg self.emit_comprehension(body, target, sequence, condition, u"()") + + +class PxdWriter(DeclarationWriter, ExpressionWriter): + """ + A Cython code writer for everything supported in pxd files. + (currently unused) + """ + + def __call__(self, node): + print(u'\n'.join(self.write(node).lines)) + return node + + def visit_CFuncDefNode(self, node): + if node.overridable: + self.startline(u'cpdef ') + else: + self.startline(u'cdef ') + if node.modifiers: + self.put(' '.join(node.modifiers)) + self.put(' ') + if node.visibility != 'private': + self.put(node.visibility) + self.put(u' ') + if node.api: + self.put(u'api ') + self.visit(node.declarator) + + def visit_StatNode(self, node): + pass + + +class CodeWriter(StatementWriter, ExpressionWriter): + """ + A complete Cython code writer. + """ diff --git a/Cython/Compiler/AnalysedTreeTransforms.py b/Cython/Compiler/AnalysedTreeTransforms.py index 07bf31f3e..d4941606e 100644 --- a/Cython/Compiler/AnalysedTreeTransforms.py +++ b/Cython/Compiler/AnalysedTreeTransforms.py @@ -10,9 +10,9 @@ from . import Symtab class AutoTestDictTransform(ScopeTrackingTransform): # Handles autotestdict directive - blacklist = ['__cinit__', '__dealloc__', '__richcmp__', - '__nonzero__', '__bool__', - '__len__', '__contains__'] + excludelist = ['__cinit__', '__dealloc__', '__richcmp__', + '__nonzero__', '__bool__', + '__len__', '__contains__'] def visit_ModuleNode(self, node): if node.is_pxd: @@ -81,7 +81,7 @@ class AutoTestDictTransform(ScopeTrackingTransform): name = node.entry.name else: name = node.name - if self.scope_type == 'cclass' and name in self.blacklist: + if self.scope_type == 'cclass' and name in self.excludelist: return node if self.scope_type == 'pyclass': class_name = self.scope_node.name diff --git a/Cython/Compiler/Annotate.py b/Cython/Compiler/Annotate.py index 5feac02d8..48e73f853 100644 --- a/Cython/Compiler/Annotate.py +++ b/Cython/Compiler/Annotate.py @@ -23,8 +23,12 @@ from .. import Utils class AnnotationCCodeWriter(CCodeWriter): - def __init__(self, create_from=None, buffer=None, copy_formatting=True): + # also used as marker for detection of complete code emission in tests + COMPLETE_CODE_TITLE = "Complete cythonized code" + + def __init__(self, create_from=None, buffer=None, copy_formatting=True, show_entire_c_code=False, source_desc=None): CCodeWriter.__init__(self, create_from, buffer, copy_formatting=copy_formatting) + self.show_entire_c_code = show_entire_c_code if create_from is None: self.annotation_buffer = StringIO() self.last_annotated_pos = None @@ -83,7 +87,7 @@ class AnnotationCCodeWriter(CCodeWriter): body.cython { font-family: courier; font-size: 12; } .cython.tag { } - .cython.line { margin: 0em } + .cython.line { color: #000000; margin: 0em } .cython.code { font-size: 9; color: #444444; display: none; margin: 0px 0px 0px 8px; border-left: 8px none; } .cython.line .run { background-color: #B0FFB0; } @@ -198,17 +202,24 @@ class AnnotationCCodeWriter(CCodeWriter): for line in coverage_data.iterfind('lines/line') ) - def _htmlify_code(self, code): + def _htmlify_code(self, code, language): try: from pygments import highlight - from pygments.lexers import CythonLexer + from pygments.lexers import CythonLexer, CppLexer from pygments.formatters import HtmlFormatter except ImportError: # no Pygments, just escape the code return html_escape(code) + if language == "cython": + lexer = CythonLexer(stripnl=False, stripall=False) + elif language == "c/cpp": + lexer = CppLexer(stripnl=False, stripall=False) + else: + # unknown language, use fallback + return html_escape(code) html_code = highlight( - code, CythonLexer(stripnl=False, stripall=False), + code, lexer, HtmlFormatter(nowrap=True)) return html_code @@ -228,7 +239,7 @@ class AnnotationCCodeWriter(CCodeWriter): return u"<span class='%s'>%s</span>" % ( group_name, match.group(group_name)) - lines = self._htmlify_code(cython_code).splitlines() + lines = self._htmlify_code(cython_code, "cython").splitlines() lineno_width = len(str(len(lines))) if not covered_lines: covered_lines = None @@ -279,6 +290,19 @@ class AnnotationCCodeWriter(CCodeWriter): outlist.append(u"<pre class='cython code score-{score} {covered}'>{code}</pre>".format( score=score, covered=covered, code=c_code)) outlist.append(u"</div>") + + # now the whole c-code if needed: + if self.show_entire_c_code: + outlist.append(u'<p><div class="cython">') + onclick_title = u"<pre class='cython line'{onclick}>+ {title}</pre>\n" + outlist.append(onclick_title.format( + onclick=self._onclick_attr, + title=AnnotationCCodeWriter.COMPLETE_CODE_TITLE, + )) + complete_code_as_html = self._htmlify_code(self.buffer.getvalue(), "c/cpp") + outlist.append(u"<pre class='cython code'>{code}</pre>".format(code=complete_code_as_html)) + outlist.append(u"</div></p>") + return outlist diff --git a/Cython/Compiler/AutoDocTransforms.py b/Cython/Compiler/AutoDocTransforms.py index d3c0a1d0d..6c342f7ef 100644 --- a/Cython/Compiler/AutoDocTransforms.py +++ b/Cython/Compiler/AutoDocTransforms.py @@ -3,18 +3,48 @@ from __future__ import absolute_import, print_function from .Visitor import CythonTransform from .StringEncoding import EncodedString from . import Options -from . import PyrexTypes, ExprNodes +from . import PyrexTypes from ..CodeWriter import ExpressionWriter +from .Errors import warning class AnnotationWriter(ExpressionWriter): + """ + A Cython code writer for Python expressions in argument/variable annotations. + """ + def __init__(self, description=None): + """description is optional. If specified it is used in + warning messages for the nodes that don't convert to string properly. + If not specified then no messages are generated. + """ + ExpressionWriter.__init__(self) + self.description = description + self.incomplete = False def visit_Node(self, node): self.put(u"<???>") + self.incomplete = True + if self.description: + warning(node.pos, + "Failed to convert code to string representation in {0}".format( + self.description), level=1) def visit_LambdaNode(self, node): # XXX Should we do better? self.put("<lambda>") + self.incomplete = True + if self.description: + warning(node.pos, + "Failed to convert lambda to string representation in {0}".format( + self.description), level=1) + + def visit_UnicodeNode(self, node): + # Discard Unicode prefix in annotations. Any tool looking at them + # would probably expect Py3 string semantics. + self.emit_string(node, "") + + def visit_AnnotationNode(self, node): + self.put(node.string.unicode_value) class EmbedSignature(CythonTransform): @@ -25,6 +55,12 @@ class EmbedSignature(CythonTransform): self.class_node = None def _fmt_expr(self, node): + writer = ExpressionWriter() + result = writer.write(node) + # print(type(node).__name__, '-->', result) + return result + + def _fmt_annotation(self, node): writer = AnnotationWriter() result = writer.write(node) # print(type(node).__name__, '-->', result) @@ -37,7 +73,7 @@ class EmbedSignature(CythonTransform): doc = arg.type.declaration_code(arg.name, for_display=1) if arg.annotation: - annotation = self._fmt_expr(arg.annotation) + annotation = self._fmt_annotation(arg.annotation) doc = doc + (': %s' % annotation) if arg.default: default = self._fmt_expr(arg.default) @@ -50,12 +86,12 @@ class EmbedSignature(CythonTransform): def _fmt_star_arg(self, arg): arg_doc = arg.name if arg.annotation: - annotation = self._fmt_expr(arg.annotation) + annotation = self._fmt_annotation(arg.annotation) arg_doc = arg_doc + (': %s' % annotation) return arg_doc def _fmt_arglist(self, args, - npargs=0, pargs=None, + npoargs=0, npargs=0, pargs=None, nkargs=0, kargs=None, hide_self=False): arglist = [] @@ -65,9 +101,11 @@ class EmbedSignature(CythonTransform): arglist.append(arg_doc) if pargs: arg_doc = self._fmt_star_arg(pargs) - arglist.insert(npargs, '*%s' % arg_doc) + arglist.insert(npargs + npoargs, '*%s' % arg_doc) elif nkargs: - arglist.insert(npargs, '*') + arglist.insert(npargs + npoargs, '*') + if npoargs: + arglist.insert(npoargs, '/') if kargs: arg_doc = self._fmt_star_arg(kargs) arglist.append('**%s' % arg_doc) @@ -80,12 +118,12 @@ class EmbedSignature(CythonTransform): return ret.declaration_code("", for_display=1) def _fmt_signature(self, cls_name, func_name, args, - npargs=0, pargs=None, + npoargs=0, npargs=0, pargs=None, nkargs=0, kargs=None, return_expr=None, return_type=None, hide_self=False): arglist = self._fmt_arglist(args, - npargs, pargs, + npoargs, npargs, pargs, nkargs, kargs, hide_self=hide_self) arglist_doc = ', '.join(arglist) @@ -94,7 +132,7 @@ class EmbedSignature(CythonTransform): func_doc = '%s.%s' % (cls_name, func_doc) ret_doc = None if return_expr: - ret_doc = self._fmt_expr(return_expr) + ret_doc = self._fmt_annotation(return_expr) elif return_type: ret_doc = self._fmt_ret_type(return_type) if ret_doc: @@ -147,11 +185,12 @@ class EmbedSignature(CythonTransform): else: class_name, func_name = self.class_name, node.name + npoargs = getattr(node, 'num_posonly_args', 0) nkargs = getattr(node, 'num_kwonly_args', 0) - npargs = len(node.args) - nkargs + npargs = len(node.args) - nkargs - npoargs signature = self._fmt_signature( class_name, func_name, node.args, - npargs, node.star_arg, + npoargs, npargs, node.star_arg, nkargs, node.starstar_arg, return_expr=node.return_type_annotation, return_type=None, hide_self=hide_self) @@ -176,7 +215,7 @@ class EmbedSignature(CythonTransform): def visit_CFuncDefNode(self, node): if not self.current_directives['embedsignature']: return node - if not node.overridable: # not cpdef FOO(...): + if not node.overridable: # not cpdef FOO(...): return node signature = self._fmt_signature( @@ -192,8 +231,9 @@ class EmbedSignature(CythonTransform): old_doc = None new_doc = self._embed_signature(signature, old_doc) node.entry.doc = EncodedString(new_doc) - if hasattr(node, 'py_func') and node.py_func is not None: - node.py_func.entry.doc = EncodedString(new_doc) + py_func = getattr(node, 'py_func', None) + if py_func is not None: + py_func.entry.doc = EncodedString(new_doc) return node def visit_PropertyNode(self, node): diff --git a/Cython/Compiler/Buffer.py b/Cython/Compiler/Buffer.py index c62a24f56..e33a95e29 100644 --- a/Cython/Compiler/Buffer.py +++ b/Cython/Compiler/Buffer.py @@ -85,7 +85,7 @@ class IntroduceBufferAuxiliaryVars(CythonTransform): aux_var = scope.declare_var(name=None, cname=cname, type=type, pos=node.pos) if entry.is_arg: - aux_var.used = True # otherwise, NameNode will mark whether it is used + aux_var.used = True # otherwise, NameNode will mark whether it is used return aux_var @@ -111,9 +111,9 @@ class IntroduceBufferAuxiliaryVars(CythonTransform): # # Analysis # -buffer_options = ("dtype", "ndim", "mode", "negative_indices", "cast") # ordered! +buffer_options = ("dtype", "ndim", "mode", "negative_indices", "cast") # ordered! buffer_defaults = {"ndim": 1, "mode": "full", "negative_indices": True, "cast": False} -buffer_positional_options_count = 1 # anything beyond this needs keyword argument +buffer_positional_options_count = 1 # anything beyond this needs keyword argument ERR_BUF_OPTION_UNKNOWN = '"%s" is not a buffer option' ERR_BUF_TOO_MANY = 'Too many buffer options' @@ -146,12 +146,12 @@ def analyse_buffer_options(globalpos, env, posargs, dictargs, defaults=None, nee options = {} for name, (value, pos) in dictargs.items(): - if not name in buffer_options: + if name not in buffer_options: raise CompileError(pos, ERR_BUF_OPTION_UNKNOWN % name) options[name] = value for name, (value, pos) in zip(buffer_options, posargs): - if not name in buffer_options: + if name not in buffer_options: raise CompileError(pos, ERR_BUF_OPTION_UNKNOWN % name) if name in options: raise CompileError(pos, ERR_BUF_DUP % name) @@ -159,7 +159,7 @@ def analyse_buffer_options(globalpos, env, posargs, dictargs, defaults=None, nee # Check that they are all there and copy defaults for name in buffer_options: - if not name in options: + if name not in options: try: options[name] = defaults[name] except KeyError: @@ -298,9 +298,10 @@ def put_unpack_buffer_aux_into_scope(buf_entry, code): ln = [] for i in range(buf_entry.type.ndim): for fldname in fldnames: - ln.append("%s.diminfo[%d].%s = %s.rcbuffer->pybuffer.%s[%d];" % \ - (pybuffernd_struct, i, fldname, - pybuffernd_struct, fldname, i)) + ln.append("%s.diminfo[%d].%s = %s.rcbuffer->pybuffer.%s[%d];" % ( + pybuffernd_struct, i, fldname, + pybuffernd_struct, fldname, i, + )) code.putln(' '.join(ln)) def put_init_vars(entry, code): @@ -373,7 +374,7 @@ def put_assign_to_buffer(lhs_cname, rhs_cname, buf_entry, code.putln("{") # Set up necessary stack for getbuffer code.putln("__Pyx_BufFmt_StackElem __pyx_stack[%d];" % buffer_type.dtype.struct_nesting_depth()) - getbuffer = get_getbuffer_call(code, "%s", buffer_aux, buffer_type) # fill in object below + getbuffer = get_getbuffer_call(code, "%s", buffer_aux, buffer_type) # fill in object below if is_initialized: # Release any existing buffer @@ -419,7 +420,7 @@ def put_assign_to_buffer(lhs_cname, rhs_cname, buf_entry, put_unpack_buffer_aux_into_scope(buf_entry, code) code.putln('}') - code.putln("}") # Release stack + code.putln("}") # Release stack def put_buffer_lookup_code(entry, index_signeds, index_cnames, directives, @@ -669,17 +670,25 @@ def get_type_information_cname(code, dtype, maxdepth=None): structinfo_name = "NULL" elif dtype.is_struct: struct_scope = dtype.scope - if dtype.is_const: - struct_scope = struct_scope.const_base_type_scope + if dtype.is_cv_qualified: + struct_scope = struct_scope.base_type_scope # Must pre-call all used types in order not to recurse during utility code writing. fields = struct_scope.var_entries assert len(fields) > 0 types = [get_type_information_cname(code, f.type, maxdepth - 1) for f in fields] typecode.putln("static __Pyx_StructField %s[] = {" % structinfo_name, safe=True) + + if dtype.is_cv_qualified: + # roughly speaking, remove "const" from struct_type + struct_type = dtype.cv_base_type.empty_declaration_code() + else: + struct_type = dtype.empty_declaration_code() + for f, typeinfo in zip(fields, types): typecode.putln(' {&%s, "%s", offsetof(%s, %s)},' % - (typeinfo, f.name, dtype.empty_declaration_code(), f.cname), safe=True) + (typeinfo, f.name, struct_type, f.cname), safe=True) + typecode.putln(' {NULL, NULL, 0}', safe=True) typecode.putln("};", safe=True) else: diff --git a/Cython/Compiler/Builtin.py b/Cython/Compiler/Builtin.py index 5fa717507..4e606b2d9 100644 --- a/Cython/Compiler/Builtin.py +++ b/Cython/Compiler/Builtin.py @@ -30,17 +30,19 @@ builtin_utility_code = { class _BuiltinOverride(object): def __init__(self, py_name, args, ret_type, cname, py_equiv="*", utility_code=None, sig=None, func_type=None, - is_strict_signature=False, builtin_return_type=None): + is_strict_signature=False, builtin_return_type=None, + nogil=None): self.py_name, self.cname, self.py_equiv = py_name, cname, py_equiv self.args, self.ret_type = args, ret_type self.func_type, self.sig = func_type, sig self.builtin_return_type = builtin_return_type self.is_strict_signature = is_strict_signature self.utility_code = utility_code + self.nogil = nogil def build_func_type(self, sig=None, self_arg=None): if sig is None: - sig = Signature(self.args, self.ret_type) + sig = Signature(self.args, self.ret_type, nogil=self.nogil) sig.exception_check = False # not needed for the current builtins func_type = sig.function_type(self_arg) if self.is_strict_signature: @@ -54,7 +56,7 @@ class BuiltinAttribute(object): def __init__(self, py_name, cname=None, field_type=None, field_type_name=None): self.py_name = py_name self.cname = cname or py_name - self.field_type_name = field_type_name # can't do the lookup before the type is declared! + self.field_type_name = field_type_name # can't do the lookup before the type is declared! self.field_type = field_type def declare_in_type(self, self_type): @@ -92,13 +94,13 @@ class BuiltinMethod(_BuiltinOverride): builtin_function_table = [ # name, args, return, C API func, py equiv = "*" BuiltinFunction('abs', "d", "d", "fabs", - is_strict_signature = True), + is_strict_signature=True, nogil=True), BuiltinFunction('abs', "f", "f", "fabsf", - is_strict_signature = True), + is_strict_signature=True, nogil=True), BuiltinFunction('abs', "i", "i", "abs", - is_strict_signature = True), + is_strict_signature=True, nogil=True), BuiltinFunction('abs', "l", "l", "labs", - is_strict_signature = True), + is_strict_signature=True, nogil=True), BuiltinFunction('abs', None, None, "__Pyx_abs_longlong", utility_code = UtilityCode.load("abs_longlong", "Builtins.c"), func_type = PyrexTypes.CFuncType( @@ -209,7 +211,7 @@ builtin_function_table = [ #('sum', "", "", ""), #('sorted', "", "", ""), #('type', "O", "O", "PyObject_Type"), - #('unichr', "", "", ""), + BuiltinFunction('unichr', "l", "O", "PyUnicode_FromOrdinal", builtin_return_type='unicode'), #('unicode', "", "", ""), #('vars', "", "", ""), #('zip', "", "", ""), @@ -344,15 +346,15 @@ builtin_types_table = [ ] -types_that_construct_their_instance = set([ +types_that_construct_their_instance = frozenset({ # some builtin types do not always return an instance of # themselves - these do: 'type', 'bool', 'long', 'float', 'complex', 'bytes', 'unicode', 'bytearray', - 'tuple', 'list', 'dict', 'set', 'frozenset' + 'tuple', 'list', 'dict', 'set', 'frozenset', # 'str', # only in Py3.x # 'file', # only in Py2.x -]) +}) builtin_structs_table = [ @@ -428,7 +430,7 @@ def init_builtins(): global list_type, tuple_type, dict_type, set_type, frozenset_type global bytes_type, str_type, unicode_type, basestring_type, slice_type - global float_type, bool_type, type_type, complex_type, bytearray_type + global float_type, long_type, bool_type, type_type, complex_type, bytearray_type type_type = builtin_scope.lookup('type').type list_type = builtin_scope.lookup('list').type tuple_type = builtin_scope.lookup('tuple').type @@ -442,6 +444,7 @@ def init_builtins(): basestring_type = builtin_scope.lookup('basestring').type bytearray_type = builtin_scope.lookup('bytearray').type float_type = builtin_scope.lookup('float').type + long_type = builtin_scope.lookup('long').type bool_type = builtin_scope.lookup('bool').type complex_type = builtin_scope.lookup('complex').type diff --git a/Cython/Compiler/CmdLine.py b/Cython/Compiler/CmdLine.py index e89e45ab4..ffff6a61c 100644 --- a/Cython/Compiler/CmdLine.py +++ b/Cython/Compiler/CmdLine.py @@ -5,220 +5,222 @@ from __future__ import absolute_import import os -import sys +from argparse import ArgumentParser, Action, SUPPRESS from . import Options -usage = """\ -Cython (http://cython.org) is a compiler for code written in the -Cython language. Cython is based on Pyrex by Greg Ewing. - -Usage: cython [options] sourcefile.{pyx,py} ... - -Options: - -V, --version Display version number of cython compiler - -l, --create-listing Write error messages to a listing file - -I, --include-dir <directory> Search for include files in named directory - (multiple include directories are allowed). - -o, --output-file <filename> Specify name of generated C file - -t, --timestamps Only compile newer source files - -f, --force Compile all source files (overrides implied -t) - -v, --verbose Be verbose, print file names on multiple compilation - -p, --embed-positions If specified, the positions in Cython files of each - function definition is embedded in its docstring. - --cleanup <level> Release interned objects on python exit, for memory debugging. - Level indicates aggressiveness, default 0 releases nothing. - -w, --working <directory> Sets the working directory for Cython (the directory modules - are searched from) - --gdb Output debug information for cygdb - --gdb-outdir <directory> Specify gdb debug information output directory. Implies --gdb. - - -D, --no-docstrings Strip docstrings from the compiled module. - -a, --annotate Produce a colorized HTML version of the source. - --annotate-coverage <cov.xml> Annotate and include coverage information from cov.xml. - --line-directives Produce #line directives pointing to the .pyx source - --cplus Output a C++ rather than C file. - --embed[=<method_name>] Generate a main() function that embeds the Python interpreter. - -2 Compile based on Python-2 syntax and code semantics. - -3 Compile based on Python-3 syntax and code semantics. - --3str Compile based on Python-3 syntax and code semantics without - assuming unicode by default for string literals under Python 2. - --lenient Change some compile time errors to runtime errors to - improve Python compatibility - --capi-reexport-cincludes Add cincluded headers to any auto-generated header files. - --fast-fail Abort the compilation on the first error - --warning-errors, -Werror Make all warnings into errors - --warning-extra, -Wextra Enable extra warnings - -X, --directive <name>=<value>[,<name=value,...] Overrides a compiler directive - -E, --compile-time-env name=value[,<name=value,...] Provides compile time env like DEF would do. -""" - - -# The following experimental options are supported only on MacOSX: -# -C, --compile Compile generated .c file to .o file -# --link Link .o file to produce extension module (implies -C) -# -+, --cplus Use C++ compiler for compiling and linking -# Additional .o files to link may be supplied when using -X.""" - -def bad_usage(): - sys.stderr.write(usage) - sys.exit(1) +class ParseDirectivesAction(Action): + def __call__(self, parser, namespace, values, option_string=None): + old_directives = dict(getattr(namespace, self.dest, + Options.get_directive_defaults())) + directives = Options.parse_directive_list( + values, relaxed_bool=True, current_settings=old_directives) + setattr(namespace, self.dest, directives) -def parse_command_line(args): - from .Main import CompilationOptions, default_options - - pending_arg = [] - - def pop_arg(): - if not args or pending_arg: - bad_usage() - if '=' in args[0] and args[0].startswith('--'): # allow "--long-option=xyz" - name, value = args.pop(0).split('=', 1) - pending_arg.append(value) - return name - return args.pop(0) - - def pop_value(default=None): - if pending_arg: - return pending_arg.pop() - elif default is not None: - return default - elif not args: - bad_usage() - return args.pop(0) - - def get_param(option): - tail = option[2:] - if tail: - return tail - else: - return pop_arg() - - options = CompilationOptions(default_options) - sources = [] - while args: - if args[0].startswith("-"): - option = pop_arg() - if option in ("-V", "--version"): - options.show_version = 1 - elif option in ("-l", "--create-listing"): - options.use_listing_file = 1 - elif option in ("-+", "--cplus"): - options.cplus = 1 - elif option == "--embed": - Options.embed = pop_value("main") - elif option.startswith("-I"): - options.include_path.append(get_param(option)) - elif option == "--include-dir": - options.include_path.append(pop_value()) - elif option in ("-w", "--working"): - options.working_path = pop_value() - elif option in ("-o", "--output-file"): - options.output_file = pop_value() - elif option in ("-t", "--timestamps"): - options.timestamps = 1 - elif option in ("-f", "--force"): - options.timestamps = 0 - elif option in ("-v", "--verbose"): - options.verbose += 1 - elif option in ("-p", "--embed-positions"): - Options.embed_pos_in_docstring = 1 - elif option in ("-z", "--pre-import"): - Options.pre_import = pop_value() - elif option == "--cleanup": - Options.generate_cleanup_code = int(pop_value()) - elif option in ("-D", "--no-docstrings"): - Options.docstrings = False - elif option in ("-a", "--annotate"): - Options.annotate = True - elif option == "--annotate-coverage": - Options.annotate = True - Options.annotate_coverage_xml = pop_value() - elif option == "--convert-range": - Options.convert_range = True - elif option == "--line-directives": - options.emit_linenums = True - elif option == "--no-c-in-traceback": - options.c_line_in_traceback = False - elif option == "--gdb": - options.gdb_debug = True - options.output_dir = os.curdir - elif option == "--gdb-outdir": - options.gdb_debug = True - options.output_dir = pop_value() - elif option == "--lenient": - Options.error_on_unknown_names = False - Options.error_on_uninitialized = False - elif option == '-2': - options.language_level = 2 - elif option == '-3': - options.language_level = 3 - elif option == '--3str': - options.language_level = '3str' - elif option == "--capi-reexport-cincludes": - options.capi_reexport_cincludes = True - elif option == "--fast-fail": - Options.fast_fail = True - elif option == "--cimport-from-pyx": - Options.cimport_from_pyx = True - elif option in ('-Werror', '--warning-errors'): - Options.warning_errors = True - elif option in ('-Wextra', '--warning-extra'): - options.compiler_directives.update(Options.extra_warnings) - elif option == "--old-style-globals": - Options.old_style_globals = True - elif option == "--directive" or option.startswith('-X'): - if option.startswith('-X') and option[2:].strip(): - x_args = option[2:] - else: - x_args = pop_value() - try: - options.compiler_directives = Options.parse_directive_list( - x_args, relaxed_bool=True, - current_settings=options.compiler_directives) - except ValueError as e: - sys.stderr.write("Error in compiler directive: %s\n" % e.args[0]) - sys.exit(1) - elif option == "--compile-time-env" or option.startswith('-E'): - if option.startswith('-E') and option[2:].strip(): - x_args = option[2:] - else: - x_args = pop_value() - try: - options.compile_time_env = Options.parse_compile_time_env( - x_args, current_settings=options.compile_time_env) - except ValueError as e: - sys.stderr.write("Error in compile-time-env: %s\n" % e.args[0]) - sys.exit(1) - elif option.startswith('--debug'): - option = option[2:].replace('-', '_') - from . import DebugFlags - if option in dir(DebugFlags): - setattr(DebugFlags, option, True) - else: - sys.stderr.write("Unknown debug flag: %s\n" % option) - bad_usage() - elif option in ('-h', '--help'): - sys.stdout.write(usage) - sys.exit(0) + +class ParseOptionsAction(Action): + def __call__(self, parser, namespace, values, option_string=None): + options = dict(getattr(namespace, self.dest, {})) + for opt in values.split(','): + if '=' in opt: + n, v = opt.split('=', 1) + v = v.lower() not in ('false', 'f', '0', 'no') + else: + n, v = opt, True + options[n] = v + setattr(namespace, self.dest, options) + + +class ParseCompileTimeEnvAction(Action): + def __call__(self, parser, namespace, values, option_string=None): + old_env = dict(getattr(namespace, self.dest, {})) + new_env = Options.parse_compile_time_env(values, current_settings=old_env) + setattr(namespace, self.dest, new_env) + + +class ActivateAllWarningsAction(Action): + def __call__(self, parser, namespace, values, option_string=None): + directives = getattr(namespace, 'compiler_directives', {}) + directives.update(Options.extra_warnings) + namespace.compiler_directives = directives + + +class SetLenientAction(Action): + def __call__(self, parser, namespace, values, option_string=None): + namespace.error_on_unknown_names = False + namespace.error_on_uninitialized = False + + +class SetGDBDebugAction(Action): + def __call__(self, parser, namespace, values, option_string=None): + namespace.gdb_debug = True + namespace.output_dir = os.curdir + + +class SetGDBDebugOutputAction(Action): + def __call__(self, parser, namespace, values, option_string=None): + namespace.gdb_debug = True + namespace.output_dir = values + + +class SetAnnotateCoverageAction(Action): + def __call__(self, parser, namespace, values, option_string=None): + namespace.annotate = True + namespace.annotate_coverage_xml = values + + +def create_cython_argparser(): + description = "Cython (https://cython.org/) is a compiler for code written in the "\ + "Cython language. Cython is based on Pyrex by Greg Ewing." + + parser = ArgumentParser(description=description, argument_default=SUPPRESS) + + parser.add_argument("-V", "--version", dest='show_version', action='store_const', const=1, + help='Display version number of cython compiler') + parser.add_argument("-l", "--create-listing", dest='use_listing_file', action='store_const', const=1, + help='Write error messages to a listing file') + parser.add_argument("-I", "--include-dir", dest='include_path', action='append', + help='Search for include files in named directory ' + '(multiple include directories are allowed).') + parser.add_argument("-o", "--output-file", dest='output_file', action='store', type=str, + help='Specify name of generated C file') + parser.add_argument("-t", "--timestamps", dest='timestamps', action='store_const', const=1, + help='Only compile newer source files') + parser.add_argument("-f", "--force", dest='timestamps', action='store_const', const=0, + help='Compile all source files (overrides implied -t)') + parser.add_argument("-v", "--verbose", dest='verbose', action='count', + help='Be verbose, print file names on multiple compilation') + parser.add_argument("-p", "--embed-positions", dest='embed_pos_in_docstring', action='store_const', const=1, + help='If specified, the positions in Cython files of each ' + 'function definition is embedded in its docstring.') + parser.add_argument("--cleanup", dest='generate_cleanup_code', action='store', type=int, + help='Release interned objects on python exit, for memory debugging. ' + 'Level indicates aggressiveness, default 0 releases nothing.') + parser.add_argument("-w", "--working", dest='working_path', action='store', type=str, + help='Sets the working directory for Cython (the directory modules are searched from)') + parser.add_argument("--gdb", action=SetGDBDebugAction, nargs=0, + help='Output debug information for cygdb') + parser.add_argument("--gdb-outdir", action=SetGDBDebugOutputAction, type=str, + help='Specify gdb debug information output directory. Implies --gdb.') + parser.add_argument("-D", "--no-docstrings", dest='docstrings', action='store_false', + help='Strip docstrings from the compiled module.') + parser.add_argument('-a', '--annotate', action='store_const', const='default', dest='annotate', + help='Produce a colorized HTML version of the source.') + parser.add_argument('--annotate-fullc', action='store_const', const='fullc', dest='annotate', + help='Produce a colorized HTML version of the source ' + 'which includes entire generated C/C++-code.') + parser.add_argument("--annotate-coverage", dest='annotate_coverage_xml', action=SetAnnotateCoverageAction, type=str, + help='Annotate and include coverage information from cov.xml.') + parser.add_argument("--line-directives", dest='emit_linenums', action='store_true', + help='Produce #line directives pointing to the .pyx source') + parser.add_argument("-+", "--cplus", dest='cplus', action='store_const', const=1, + help='Output a C++ rather than C file.') + parser.add_argument('--embed', action='store_const', const='main', + help='Generate a main() function that embeds the Python interpreter. ' + 'Pass --embed=<method_name> for a name other than main().') + parser.add_argument('-2', dest='language_level', action='store_const', const=2, + help='Compile based on Python-2 syntax and code semantics.') + parser.add_argument('-3', dest='language_level', action='store_const', const=3, + help='Compile based on Python-3 syntax and code semantics.') + parser.add_argument('--3str', dest='language_level', action='store_const', const='3str', + help='Compile based on Python-3 syntax and code semantics without ' + 'assuming unicode by default for string literals under Python 2.') + parser.add_argument("--lenient", action=SetLenientAction, nargs=0, + help='Change some compile time errors to runtime errors to ' + 'improve Python compatibility') + parser.add_argument("--capi-reexport-cincludes", dest='capi_reexport_cincludes', action='store_true', + help='Add cincluded headers to any auto-generated header files.') + parser.add_argument("--fast-fail", dest='fast_fail', action='store_true', + help='Abort the compilation on the first error') + parser.add_argument("-Werror", "--warning-errors", dest='warning_errors', action='store_true', + help='Make all warnings into errors') + parser.add_argument("-Wextra", "--warning-extra", action=ActivateAllWarningsAction, nargs=0, + help='Enable extra warnings') + + parser.add_argument('-X', '--directive', metavar='NAME=VALUE,...', + dest='compiler_directives', type=str, + action=ParseDirectivesAction, + help='Overrides a compiler directive') + parser.add_argument('-E', '--compile-time-env', metavar='NAME=VALUE,...', + dest='compile_time_env', type=str, + action=ParseCompileTimeEnvAction, + help='Provides compile time env like DEF would do.') + parser.add_argument('sources', nargs='*', default=[]) + + # TODO: add help + parser.add_argument("-z", "--pre-import", dest='pre_import', action='store', type=str, help=SUPPRESS) + parser.add_argument("--convert-range", dest='convert_range', action='store_true', help=SUPPRESS) + parser.add_argument("--no-c-in-traceback", dest='c_line_in_traceback', action='store_false', help=SUPPRESS) + parser.add_argument("--cimport-from-pyx", dest='cimport_from_pyx', action='store_true', help=SUPPRESS) + parser.add_argument("--old-style-globals", dest='old_style_globals', action='store_true', help=SUPPRESS) + + # debug stuff: + from . import DebugFlags + for name in vars(DebugFlags): + if name.startswith("debug"): + option_name = name.replace('_', '-') + parser.add_argument("--" + option_name, action='store_true', help=SUPPRESS) + + return parser + + +def parse_command_line_raw(parser, args): + # special handling for --embed and --embed=xxxx as they aren't correctly parsed + def filter_out_embed_options(args): + with_embed, without_embed = [], [] + for x in args: + if x == '--embed' or x.startswith('--embed='): + with_embed.append(x) else: - sys.stderr.write("Unknown compiler flag: %s\n" % option) - sys.exit(1) + without_embed.append(x) + return with_embed, without_embed + + with_embed, args_without_embed = filter_out_embed_options(args) + + arguments, unknown = parser.parse_known_args(args_without_embed) + + sources = arguments.sources + del arguments.sources + + # unknown can be either debug, embed or input files or really unknown + for option in unknown: + if option.startswith('-'): + parser.error("unknown option " + option) else: - sources.append(pop_arg()) + sources.append(option) - if pending_arg: - bad_usage() + # embed-stuff must be handled extra: + for x in with_embed: + if x == '--embed': + name = 'main' # default value + else: + name = x[len('--embed='):] + setattr(arguments, 'embed', name) + + return arguments, sources + + +def parse_command_line(args): + parser = create_cython_argparser() + arguments, sources = parse_command_line_raw(parser, args) + + options = Options.CompilationOptions(Options.default_options) + for name, value in vars(arguments).items(): + if name.startswith('debug'): + from . import DebugFlags + if name in dir(DebugFlags): + setattr(DebugFlags, name, value) + else: + parser.error("Unknown debug flag: %s\n" % name) + elif hasattr(Options, name): + setattr(Options, name, value) + else: + setattr(options, name, value) if options.use_listing_file and len(sources) > 1: - sys.stderr.write( - "cython: Only one source file allowed when using -o\n") - sys.exit(1) + parser.error("cython: Only one source file allowed when using -o\n") if len(sources) == 0 and not options.show_version: - bad_usage() + parser.error("cython: Need at least one source file\n") if Options.embed and len(sources) > 1: - sys.stderr.write( - "cython: Only one source file allowed when using -embed\n") - sys.exit(1) + parser.error("cython: Only one source file allowed when using -embed\n") return options, sources - diff --git a/Cython/Compiler/Code.pxd b/Cython/Compiler/Code.pxd index 01f7a71f5..e17e0fb1d 100644 --- a/Cython/Compiler/Code.pxd +++ b/Cython/Compiler/Code.pxd @@ -1,5 +1,4 @@ - -from __future__ import absolute_import +# cython: language_level=3 cimport cython from ..StringIOTree cimport StringIOTree @@ -48,6 +47,7 @@ cdef class FunctionState: cdef public list temps_allocated cdef public dict temps_free cdef public dict temps_used_type + cdef public set zombie_temps cdef public size_t temp_counter cdef public list collect_temps_stack diff --git a/Cython/Compiler/Code.py b/Cython/Compiler/Code.py index 26fb8a1cf..a2affbe89 100644 --- a/Cython/Compiler/Code.py +++ b/Cython/Compiler/Code.py @@ -1,4 +1,4 @@ -# cython: language_level = 2 +# cython: language_level=3str # cython: auto_pickle=False # # Code output module @@ -13,22 +13,17 @@ cython.declare(os=object, re=object, operator=object, textwrap=object, DebugFlags=object, basestring=object, defaultdict=object, closing=object, partial=object) +import hashlib +import operator import os import re import shutil -import sys -import operator import textwrap from string import Template from functools import partial from contextlib import closing from collections import defaultdict -try: - import hashlib -except ImportError: - import md5 as hashlib - from . import Naming from . import Options from . import DebugFlags @@ -43,8 +38,6 @@ try: except ImportError: from builtins import str as basestring -KEYWORDS_MUST_BE_BYTES = sys.version_info < (2, 7) - non_portable_builtins_map = { # builtins that have different names in different Python versions @@ -101,20 +94,18 @@ uncachable_builtins = [ '__build_class__', 'ascii', # might deserve an implementation in Cython #'exec', # implemented in Cython - ## - Py2.7+ - 'memoryview', ## - platform specific 'WindowsError', ## - others '_', # e.g. used by gettext ] -special_py_methods = set([ +special_py_methods = cython.declare(frozenset, frozenset(( '__cinit__', '__dealloc__', '__richcmp__', '__next__', '__await__', '__aiter__', '__anext__', '__getreadbuffer__', '__getwritebuffer__', '__getsegcount__', - '__getcharbuffer__', '__getbuffer__', '__releasebuffer__' -]) + '__getcharbuffer__', '__getbuffer__', '__releasebuffer__', +))) modifier_output_mapper = { 'inline': 'CYTHON_INLINE' @@ -203,6 +194,28 @@ def get_utility_dir(): Cython_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) return os.path.join(Cython_dir, "Utility") +read_utilities_hook = None +""" +Override the hook for reading a utilities file that contains code fragments used +by the codegen. + +The hook functions takes the path of the utilities file, and returns a list +of strings, one per line. + +The default behavior is to open a file relative to get_utility_dir(). +""" + +def read_utilities_from_utility_dir(path): + """ + Read all lines of the file at the provided path from a path relative + to get_utility_dir(). + """ + filename = os.path.join(get_utility_dir(), path) + with closing(Utils.open_source_file(filename, encoding='UTF-8')) as f: + return f.readlines() + +# by default, read utilities from the utility directory. +read_utilities_hook = read_utilities_from_utility_dir class UtilityCodeBase(object): """ @@ -224,6 +237,15 @@ class UtilityCodeBase(object): [definitions] + ##### MyUtility ##### + #@subsitute: tempita + + [requires tempita substitution + - context can't be specified here though so only + tempita utility that requires no external context + will benefit from this tag + - only necessary when @required from non-tempita code] + for prototypes and implementation respectively. For non-python or -cython files backslashes should be used instead. 5 to 30 comment characters may be used on either side. @@ -242,8 +264,7 @@ class UtilityCodeBase(object): return code = '\n'.join(lines) - if tags and 'substitute' in tags and tags['substitute'] == set(['naming']): - del tags['substitute'] + if tags and 'substitute' in tags and 'naming' in tags['substitute']: try: code = Template(code).substitute(vars(Naming)) except (KeyError, ValueError) as e: @@ -259,15 +280,11 @@ class UtilityCodeBase(object): utility[1] = code else: all_tags = utility[2] - if KEYWORDS_MUST_BE_BYTES: - type = type.encode('ASCII') all_tags[type] = code if tags: all_tags = utility[2] for name, values in tags.items(): - if KEYWORDS_MUST_BE_BYTES: - name = name.encode('ASCII') all_tags.setdefault(name, set()).update(values) @classmethod @@ -276,7 +293,6 @@ class UtilityCodeBase(object): if utilities: return utilities - filename = os.path.join(get_utility_dir(), path) _, ext = os.path.splitext(path) if ext in ('.pyx', '.py', '.pxd', '.pxi'): comment = '#' @@ -292,8 +308,7 @@ class UtilityCodeBase(object): {'C': comment}).match match_type = re.compile(r'(.+)[.](proto(?:[.]\S+)?|impl|init|cleanup)$').match - with closing(Utils.open_source_file(filename, encoding='UTF-8')) as f: - all_lines = f.readlines() + all_lines = read_utilities_hook(path) utilities = defaultdict(lambda: [None, None, {}]) lines = [] @@ -335,43 +350,22 @@ class UtilityCodeBase(object): return utilities @classmethod - def load(cls, util_code_name, from_file=None, **kwargs): + def load(cls, util_code_name, from_file, **kwargs): """ Load utility code from a file specified by from_file (relative to - Cython/Utility) and name util_code_name. If from_file is not given, - load it from the file util_code_name.*. There should be only one - file matched by this pattern. + Cython/Utility) and name util_code_name. """ + if '::' in util_code_name: from_file, util_code_name = util_code_name.rsplit('::', 1) - if not from_file: - utility_dir = get_utility_dir() - prefix = util_code_name + '.' - try: - listing = os.listdir(utility_dir) - except OSError: - # XXX the code below assumes as 'zipimport.zipimporter' instance - # XXX should be easy to generalize, but too lazy right now to write it - import zipfile - global __loader__ - loader = __loader__ - archive = loader.archive - with closing(zipfile.ZipFile(archive)) as fileobj: - listing = [os.path.basename(name) - for name in fileobj.namelist() - if os.path.join(archive, name).startswith(utility_dir)] - files = [filename for filename in listing - if filename.startswith(prefix)] - if not files: - raise ValueError("No match found for utility code " + util_code_name) - if len(files) > 1: - raise ValueError("More than one filename match found for utility code " + util_code_name) - from_file = files[0] - + assert from_file utilities = cls.load_utilities_from_file(from_file) proto, impl, tags = utilities[util_code_name] if tags: + if "substitute" in tags and "tempita" in tags["substitute"]: + if not issubclass(cls, TempitaUtilityCode): + return TempitaUtilityCode.load(util_code_name, from_file, **kwargs) orig_kwargs = kwargs.copy() for name, values in tags.items(): if name in kwargs: @@ -385,6 +379,12 @@ class UtilityCodeBase(object): # dependencies are rarely unique, so use load_cached() when we can values = [cls.load_cached(dep, from_file) for dep in sorted(values)] + elif name == 'substitute': + # don't want to pass "naming" or "tempita" to the constructor + # since these will have been handled + values = values - {'naming', 'tempita'} + if not values: + continue elif not values: values = None elif len(values) == 1: @@ -404,11 +404,11 @@ class UtilityCodeBase(object): return cls(**kwargs) @classmethod - def load_cached(cls, utility_code_name, from_file=None, __cache={}): + def load_cached(cls, utility_code_name, from_file, __cache={}): """ Calls .load(), but using a per-type cache based on utility name and file name. """ - key = (cls, from_file, utility_code_name) + key = (utility_code_name, from_file, cls) try: return __cache[key] except KeyError: @@ -417,7 +417,7 @@ class UtilityCodeBase(object): return code @classmethod - def load_as_string(cls, util_code_name, from_file=None, **kwargs): + def load_as_string(cls, util_code_name, from_file, **kwargs): """ Load a utility code as a string. Returns (proto, implementation) """ @@ -842,13 +842,15 @@ class FunctionState(object): A C string referring to the variable is returned. """ - if type.is_const and not type.is_reference: - type = type.const_base_type + if type.is_cv_qualified and not type.is_reference: + type = type.cv_base_type elif type.is_reference and not type.is_fake_reference: type = type.ref_base_type elif type.is_cfunction: from . import PyrexTypes type = PyrexTypes.c_ptr_type(type) # A function itself isn't an l-value + elif type.is_cpp_class and not type.is_fake_reference and self.scope.directives['cpp_locals']: + self.scope.use_utility_code(UtilityCode.load_cached("OptionalLocals", "CppSupport.cpp")) if not type.is_pyobject and not type.is_memoryviewslice: # Make manage_ref canonical, so that manage_ref will always mean # a decref is needed. @@ -912,7 +914,7 @@ class FunctionState(object): """ return [(name, type) for name, type, manage_ref in self.temps_in_use() - if manage_ref and type.is_pyobject] + if manage_ref and type.is_pyobject] def all_managed_temps(self): """Return a list of (cname, type) tuples of refcount-managed Python objects. @@ -1117,10 +1119,10 @@ class GlobalState(object): 'h_code', 'filename_table', 'utility_code_proto_before_types', - 'numeric_typedefs', # Let these detailed individual parts stay!, - 'complex_type_declarations', # as the proper solution is to make a full DAG... - 'type_declarations', # More coarse-grained blocks would simply hide - 'utility_code_proto', # the ugliness, not fix it + 'numeric_typedefs', # Let these detailed individual parts stay!, + 'complex_type_declarations', # as the proper solution is to make a full DAG... + 'type_declarations', # More coarse-grained blocks would simply hide + 'utility_code_proto', # the ugliness, not fix it 'module_declarations', 'typeinfo', 'before_global_var', @@ -1128,7 +1130,11 @@ class GlobalState(object): 'string_decls', 'decls', 'late_includes', - 'all_the_rest', + 'module_state', + 'module_state_clear', + 'module_state_traverse', + 'module_state_defines', # redefines names used in module_state/_clear/_traverse + 'module_code', # user code goes here 'pystring_table', 'cached_builtins', 'cached_constants', @@ -1137,10 +1143,20 @@ class GlobalState(object): 'cleanup_globals', 'cleanup_module', 'main_method', + 'utility_code_pragmas', # silence some irrelevant warnings in utility code 'utility_code_def', + 'utility_code_pragmas_end', # clean-up the utility_code_pragmas 'end' ] + # h files can only have a much smaller list of sections + h_code_layout = [ + 'h_code', + 'utility_code_proto_before_types', + 'type_declarations', + 'utility_code_proto', + 'end' + ] def __init__(self, writer, module_node, code_config, common_utility_include_dir=None): self.filename_table = {} @@ -1152,8 +1168,8 @@ class GlobalState(object): self.code_config = code_config self.common_utility_include_dir = common_utility_include_dir self.parts = {} - self.module_node = module_node # because some utility code generation needs it - # (generating backwards-compatible Get/ReleaseBuffer + self.module_node = module_node # because some utility code generation needs it + # (generating backwards-compatible Get/ReleaseBuffer self.const_cnames_used = {} self.string_const_index = {} @@ -1169,8 +1185,10 @@ class GlobalState(object): def initialize_main_c_code(self): rootwriter = self.rootwriter - for part in self.code_layout: - self.parts[part] = rootwriter.insertion_point() + for i, part in enumerate(self.code_layout): + w = self.parts[part] = rootwriter.insertion_point() + if i > 0: + w.putln("/* #### Code section: %s ### */" % part) if not Options.cache_builtins: del self.parts['cached_builtins'] @@ -1184,7 +1202,7 @@ class GlobalState(object): w.putln("") w.putln("static CYTHON_SMALL_CODE int __Pyx_InitCachedConstants(void) {") w.put_declare_refcount_context() - w.put_setup_refcount_context("__Pyx_InitCachedConstants") + w.put_setup_refcount_context(StringEncoding.EncodedString("__Pyx_InitCachedConstants")) w = self.parts['init_globals'] w.enter_cfunc_scope() @@ -1209,6 +1227,11 @@ class GlobalState(object): code.putln("") code.putln("/* --- Runtime support code --- */") + def initialize_main_h_code(self): + rootwriter = self.rootwriter + for part in self.h_code_layout: + self.parts[part] = rootwriter.insertion_point() + def finalize_main_c_code(self): self.close_global_decls() @@ -1220,6 +1243,18 @@ class GlobalState(object): code.put(util.format_code(util.impl)) code.putln("") + # + # utility code pragmas + # + code = self.parts['utility_code_pragmas'] + util = UtilityCode.load_cached("UtilityCodePragmas", "ModuleSetupCode.c") + code.putln(util.format_code(util.impl)) + code.putln("") + code = self.parts['utility_code_pragmas_end'] + util = UtilityCode.load_cached("UtilityCodePragmasEnd", "ModuleSetupCode.c") + code.putln(util.format_code(util.impl)) + code.putln("") + def __getitem__(self, key): return self.parts[key] @@ -1457,9 +1492,18 @@ class GlobalState(object): for c in self.py_constants] consts.sort() decls_writer = self.parts['decls'] + decls_writer.putln("#if !CYTHON_USE_MODULE_STATE") for _, cname, c in consts: + self.parts['module_state'].putln("%s;" % c.type.declaration_code(cname)) + self.parts['module_state_defines'].putln( + "#define %s %s->%s" % (cname, Naming.modulestateglobal_cname, cname)) + self.parts['module_state_clear'].putln( + "Py_CLEAR(clear_module_state->%s);" % cname) + self.parts['module_state_traverse'].putln( + "Py_VISIT(traverse_module_state->%s);" % cname) decls_writer.putln( "static %s;" % c.type.declaration_code(cname)) + decls_writer.putln("#endif") def generate_cached_methods_decls(self): if not self.cached_cmethods: @@ -1471,11 +1515,14 @@ class GlobalState(object): for (type_cname, method_name), cname in sorted(self.cached_cmethods.items()): cnames.append(cname) method_name_cname = self.get_interned_identifier(StringEncoding.EncodedString(method_name)).cname - decl.putln('static __Pyx_CachedCFunction %s = {0, &%s, 0, 0, 0};' % ( - cname, method_name_cname)) + decl.putln('static __Pyx_CachedCFunction %s = {0, 0, 0, 0, 0};' % ( + cname)) # split type reference storage as it might not be static init.putln('%s.type = (PyObject*)&%s;' % ( cname, type_cname)) + # method name string isn't static in limited api + init.putln('%s.method_name = &%s;' % ( + cname, method_name_cname)) if Options.generate_cleanup_code: cleanup = self.parts['cleanup_globals'] @@ -1513,13 +1560,26 @@ class GlobalState(object): decls_writer.putln("static Py_UNICODE %s[] = { %s };" % (cname, utf16_array)) decls_writer.putln("#endif") + init_globals = self.parts['init_globals'] if py_strings: self.use_utility_code(UtilityCode.load_cached("InitStrings", "StringTools.c")) py_strings.sort() w = self.parts['pystring_table'] w.putln("") w.putln("static __Pyx_StringTabEntry %s[] = {" % Naming.stringtab_cname) - for c_cname, _, py_string in py_strings: + w.putln("#if CYTHON_USE_MODULE_STATE") + w_in_module_state = w.insertion_point() + w.putln("#else") + w_not_in_module_state = w.insertion_point() + w.putln("#endif") + decls_writer.putln("#if !CYTHON_USE_MODULE_STATE") + not_limited_api_decls_writer = decls_writer.insertion_point() + decls_writer.putln("#endif") + init_globals.putln("#if CYTHON_USE_MODULE_STATE") + init_globals_in_module_state = init_globals.insertion_point() + init_globals.putln("#endif") + for idx, py_string_args in enumerate(py_strings): + c_cname, _, py_string = py_string_args if not py_string.is_str or not py_string.encoding or \ py_string.encoding in ('ASCII', 'USASCII', 'US-ASCII', 'UTF8', 'UTF-8'): @@ -1527,19 +1587,28 @@ class GlobalState(object): else: encoding = '"%s"' % py_string.encoding.lower() - decls_writer.putln( + self.parts['module_state'].putln("PyObject *%s;" % py_string.cname) + self.parts['module_state_defines'].putln("#define %s %s->%s" % ( + py_string.cname, + Naming.modulestateglobal_cname, + py_string.cname)) + self.parts['module_state_clear'].putln("Py_CLEAR(clear_module_state->%s);" % + py_string.cname) + self.parts['module_state_traverse'].putln("Py_VISIT(traverse_module_state->%s);" % + py_string.cname) + not_limited_api_decls_writer.putln( "static PyObject *%s;" % py_string.cname) if py_string.py3str_cstring: - w.putln("#if PY_MAJOR_VERSION >= 3") - w.putln("{&%s, %s, sizeof(%s), %s, %d, %d, %d}," % ( + w_not_in_module_state.putln("#if PY_MAJOR_VERSION >= 3") + w_not_in_module_state.putln("{&%s, %s, sizeof(%s), %s, %d, %d, %d}," % ( py_string.cname, py_string.py3str_cstring.cname, py_string.py3str_cstring.cname, '0', 1, 0, py_string.intern )) - w.putln("#else") - w.putln("{&%s, %s, sizeof(%s), %s, %d, %d, %d}," % ( + w_not_in_module_state.putln("#else") + w_not_in_module_state.putln("{&%s, %s, sizeof(%s), %s, %d, %d, %d}," % ( py_string.cname, c_cname, c_cname, @@ -1549,24 +1618,46 @@ class GlobalState(object): py_string.intern )) if py_string.py3str_cstring: - w.putln("#endif") + w_not_in_module_state.putln("#endif") + w_in_module_state.putln("{0, %s, sizeof(%s), %s, %d, %d, %d}," % ( + c_cname if not py_string.py3str_cstring else py_string.py3str_cstring.cname, + c_cname if not py_string.py3str_cstring else py_string.py3str_cstring.cname, + encoding if not py_string.py3str_cstring else '0', + py_string.is_unicode, + py_string.is_str, + py_string.intern + )) + init_globals_in_module_state.putln("if (__Pyx_InitString(%s[%d], &%s) < 0) %s;" % ( + Naming.stringtab_cname, + idx, + py_string.cname, + init_globals.error_goto(self.module_pos))) w.putln("{0, 0, 0, 0, 0, 0, 0}") w.putln("};") - init_globals = self.parts['init_globals'] + init_globals.putln("#if !CYTHON_USE_MODULE_STATE") init_globals.putln( "if (__Pyx_InitStrings(%s) < 0) %s;" % ( Naming.stringtab_cname, init_globals.error_goto(self.module_pos))) + init_globals.putln("#endif") def generate_num_constants(self): consts = [(c.py_type, c.value[0] == '-', len(c.value), c.value, c.value_code, c) for c in self.num_const_index.values()] consts.sort() decls_writer = self.parts['decls'] + decls_writer.putln("#if !CYTHON_USE_MODULE_STATE") init_globals = self.parts['init_globals'] for py_type, _, _, value, value_code, c in consts: cname = c.cname + self.parts['module_state'].putln("PyObject *%s;" % cname) + self.parts['module_state_defines'].putln("#define %s %s->%s" % ( + cname, Naming.modulestateglobal_cname, cname)) + self.parts['module_state_clear'].putln( + "Py_CLEAR(clear_module_state->%s);" % cname) + self.parts['module_state_traverse'].putln( + "Py_VISIT(traverse_module_state->%s);" % cname) decls_writer.putln("static PyObject *%s;" % cname) if py_type == 'float': function = 'PyFloat_FromDouble(%s)' @@ -1581,6 +1672,7 @@ class GlobalState(object): init_globals.putln('%s = %s; %s' % ( cname, function % value_code, init_globals.error_goto_if_null(cname, self.module_pos))) + decls_writer.putln("#endif") # The functions below are there in a transition phase only # and will be deprecated. They are called from Nodes.BlockNode. @@ -1755,10 +1847,13 @@ class CCodeWriter(object): return self.buffer.getvalue() def write(self, s): - # also put invalid markers (lineno 0), to indicate that those lines - # have no Cython source code correspondence - cython_lineno = self.last_marked_pos[1] if self.last_marked_pos else 0 - self.buffer.markers.extend([cython_lineno] * s.count('\n')) + # Cygdb needs to know which Cython source line corresponds to which C line. + # Therefore, we write this information into "self.buffer.markers" and then write it from there + # into cython_debug/cython_debug_info_* (see ModuleNode._serialize_lineno_map). + + filename_line = self.last_marked_pos[:2] if self.last_marked_pos else (None, 0) + self.buffer.markers.extend([filename_line] * s.count('\n')) + self.buffer.write(s) def insertion_point(self): @@ -1818,6 +1913,7 @@ class CCodeWriter(object): self.funcstate = FunctionState(self, scope=scope) def exit_cfunc_scope(self): + self.funcstate.validate_exit() self.funcstate = None # constant handling @@ -1908,7 +2004,7 @@ class CCodeWriter(object): include_dir = self.globalstate.common_utility_include_dir if include_dir and len(code) > 1024: include_file = "%s_%s.h" % ( - name, hashlib.md5(code.encode('utf8')).hexdigest()) + name, hashlib.sha1(code.encode('utf8')).hexdigest()) path = os.path.join(include_dir, include_file) if not os.path.exists(path): tmp_path = '%s.tmp%s' % (path, os.getpid()) @@ -1990,22 +2086,29 @@ class CCodeWriter(object): self.put("%s " % storage_class) if not entry.cf_used: self.put('CYTHON_UNUSED ') - self.put(entry.type.declaration_code( - entry.cname, dll_linkage=dll_linkage)) + if entry.is_cpp_optional: + self.put(entry.type.cpp_optional_declaration_code( + entry.cname, dll_linkage=dll_linkage)) + else: + self.put(entry.type.declaration_code( + entry.cname, dll_linkage=dll_linkage)) if entry.init is not None: self.put_safe(" = %s" % entry.type.literal_code(entry.init)) elif entry.type.is_pyobject: self.put(" = NULL") self.putln(";") + self.funcstate.scope.use_entry_utility_code(entry) def put_temp_declarations(self, func_context): for name, type, manage_ref, static in func_context.temps_allocated: - decl = type.declaration_code(name) + if type.is_cpp_class and not type.is_fake_reference and func_context.scope.directives['cpp_locals']: + decl = type.cpp_optional_declaration_code(name) + else: + decl = type.declaration_code(name) if type.is_pyobject: self.putln("%s = NULL;" % decl) elif type.is_memoryviewslice: - from . import MemoryView - self.putln("%s = %s;" % (decl, MemoryView.memslice_entry_init)) + self.putln("%s = %s;" % (decl, type.literal_code(type.default_value))) else: self.putln("%s%s;" % (static and "static " or "", decl)) @@ -2043,7 +2146,7 @@ class CCodeWriter(object): def entry_as_pyobject(self, entry): type = entry.type if (not entry.is_self_arg and not entry.type.is_complete() - or entry.type.is_extension_type): + or entry.type.is_extension_type): return "(PyObject *)" + entry.cname else: return entry.cname @@ -2052,123 +2155,89 @@ class CCodeWriter(object): from .PyrexTypes import py_object_type, typecast return typecast(py_object_type, type, cname) - def put_gotref(self, cname): - self.putln("__Pyx_GOTREF(%s);" % cname) + def put_gotref(self, cname, type): + type.generate_gotref(self, cname) - def put_giveref(self, cname): - self.putln("__Pyx_GIVEREF(%s);" % cname) + def put_giveref(self, cname, type): + type.generate_giveref(self, cname) - def put_xgiveref(self, cname): - self.putln("__Pyx_XGIVEREF(%s);" % cname) + def put_xgiveref(self, cname, type): + type.generate_xgiveref(self, cname) - def put_xgotref(self, cname): - self.putln("__Pyx_XGOTREF(%s);" % cname) + def put_xgotref(self, cname, type): + type.generate_xgotref(self, cname) def put_incref(self, cname, type, nanny=True): - if nanny: - self.putln("__Pyx_INCREF(%s);" % self.as_pyobject(cname, type)) - else: - self.putln("Py_INCREF(%s);" % self.as_pyobject(cname, type)) + # Note: original put_Memslice_Incref/Decref also added in some utility code + # this is unnecessary since the relevant utility code is loaded anyway if a memoryview is used + # and so has been removed. However, it's potentially a feature that might be useful here + type.generate_incref(self, cname, nanny=nanny) - def put_decref(self, cname, type, nanny=True): - self._put_decref(cname, type, nanny, null_check=False, clear=False) + def put_xincref(self, cname, type, nanny=True): + type.generate_xincref(self, cname, nanny=nanny) - def put_var_gotref(self, entry): - if entry.type.is_pyobject: - self.putln("__Pyx_GOTREF(%s);" % self.entry_as_pyobject(entry)) + def put_decref(self, cname, type, nanny=True, have_gil=True): + type.generate_decref(self, cname, nanny=nanny, have_gil=have_gil) - def put_var_giveref(self, entry): - if entry.type.is_pyobject: - self.putln("__Pyx_GIVEREF(%s);" % self.entry_as_pyobject(entry)) + def put_xdecref(self, cname, type, nanny=True, have_gil=True): + type.generate_xdecref(self, cname, nanny=nanny, have_gil=have_gil) - def put_var_xgotref(self, entry): - if entry.type.is_pyobject: - self.putln("__Pyx_XGOTREF(%s);" % self.entry_as_pyobject(entry)) + def put_decref_clear(self, cname, type, clear_before_decref=False, nanny=True, have_gil=True): + type.generate_decref_clear(self, cname, clear_before_decref=clear_before_decref, + nanny=nanny, have_gil=have_gil) - def put_var_xgiveref(self, entry): - if entry.type.is_pyobject: - self.putln("__Pyx_XGIVEREF(%s);" % self.entry_as_pyobject(entry)) + def put_xdecref_clear(self, cname, type, clear_before_decref=False, nanny=True, have_gil=True): + type.generate_xdecref_clear(self, cname, clear_before_decref=clear_before_decref, + nanny=nanny, have_gil=have_gil) - def put_var_incref(self, entry, nanny=True): - if entry.type.is_pyobject: - if nanny: - self.putln("__Pyx_INCREF(%s);" % self.entry_as_pyobject(entry)) - else: - self.putln("Py_INCREF(%s);" % self.entry_as_pyobject(entry)) + def put_decref_set(self, cname, type, rhs_cname): + type.generate_decref_set(self, cname, rhs_cname) - def put_var_xincref(self, entry): - if entry.type.is_pyobject: - self.putln("__Pyx_XINCREF(%s);" % self.entry_as_pyobject(entry)) + def put_xdecref_set(self, cname, type, rhs_cname): + type.generate_xdecref_set(self, cname, rhs_cname) - def put_decref_clear(self, cname, type, nanny=True, clear_before_decref=False): - self._put_decref(cname, type, nanny, null_check=False, - clear=True, clear_before_decref=clear_before_decref) + def put_incref_memoryviewslice(self, slice_cname, type, have_gil): + # TODO ideally this would just be merged into "put_incref" + type.generate_incref_memoryviewslice(self, slice_cname, have_gil=have_gil) - def put_xdecref(self, cname, type, nanny=True, have_gil=True): - self._put_decref(cname, type, nanny, null_check=True, - have_gil=have_gil, clear=False) + def put_var_incref_memoryviewslice(self, entry, have_gil): + self.put_incref_memoryviewslice(entry.cname, entry.type, have_gil=have_gil) - def put_xdecref_clear(self, cname, type, nanny=True, clear_before_decref=False): - self._put_decref(cname, type, nanny, null_check=True, - clear=True, clear_before_decref=clear_before_decref) + def put_var_gotref(self, entry): + self.put_gotref(entry.cname, entry.type) - def _put_decref(self, cname, type, nanny=True, null_check=False, - have_gil=True, clear=False, clear_before_decref=False): - if type.is_memoryviewslice: - self.put_xdecref_memoryviewslice(cname, have_gil=have_gil) - return + def put_var_giveref(self, entry): + self.put_giveref(entry.cname, entry.type) - prefix = '__Pyx' if nanny else 'Py' - X = 'X' if null_check else '' + def put_var_xgotref(self, entry): + self.put_xgotref(entry.cname, entry.type) - if clear: - if clear_before_decref: - if not nanny: - X = '' # CPython doesn't have a Py_XCLEAR() - self.putln("%s_%sCLEAR(%s);" % (prefix, X, cname)) - else: - self.putln("%s_%sDECREF(%s); %s = 0;" % ( - prefix, X, self.as_pyobject(cname, type), cname)) - else: - self.putln("%s_%sDECREF(%s);" % ( - prefix, X, self.as_pyobject(cname, type))) + def put_var_xgiveref(self, entry): + self.put_xgiveref(entry.cname, entry.type) - def put_decref_set(self, cname, rhs_cname): - self.putln("__Pyx_DECREF_SET(%s, %s);" % (cname, rhs_cname)) + def put_var_incref(self, entry, **kwds): + self.put_incref(entry.cname, entry.type, **kwds) - def put_xdecref_set(self, cname, rhs_cname): - self.putln("__Pyx_XDECREF_SET(%s, %s);" % (cname, rhs_cname)) + def put_var_xincref(self, entry, **kwds): + self.put_xincref(entry.cname, entry.type, **kwds) - def put_var_decref(self, entry): - if entry.type.is_pyobject: - self.putln("__Pyx_XDECREF(%s);" % self.entry_as_pyobject(entry)) + def put_var_decref(self, entry, **kwds): + self.put_decref(entry.cname, entry.type, **kwds) - def put_var_xdecref(self, entry, nanny=True): - if entry.type.is_pyobject: - if nanny: - self.putln("__Pyx_XDECREF(%s);" % self.entry_as_pyobject(entry)) - else: - self.putln("Py_XDECREF(%s);" % self.entry_as_pyobject(entry)) - - def put_var_decref_clear(self, entry): - self._put_var_decref_clear(entry, null_check=False) - - def put_var_xdecref_clear(self, entry): - self._put_var_decref_clear(entry, null_check=True) - - def _put_var_decref_clear(self, entry, null_check): - if entry.type.is_pyobject: - if entry.in_closure: - # reset before DECREF to make sure closure state is - # consistent during call to DECREF() - self.putln("__Pyx_%sCLEAR(%s);" % ( - null_check and 'X' or '', - entry.cname)) - else: - self.putln("__Pyx_%sDECREF(%s); %s = 0;" % ( - null_check and 'X' or '', - self.entry_as_pyobject(entry), - entry.cname)) + def put_var_xdecref(self, entry, **kwds): + self.put_xdecref(entry.cname, entry.type, **kwds) + + def put_var_decref_clear(self, entry, **kwds): + self.put_decref_clear(entry.cname, entry.type, clear_before_decref=entry.in_closure, **kwds) + + def put_var_decref_set(self, entry, rhs_cname, **kwds): + self.put_decref_set(entry.cname, entry.type, rhs_cname, **kwds) + + def put_var_xdecref_set(self, entry, rhs_cname, **kwds): + self.put_xdecref_set(entry.cname, entry.type, rhs_cname, **kwds) + + def put_var_xdecref_clear(self, entry, **kwds): + self.put_xdecref_clear(entry.cname, entry.type, clear_before_decref=entry.in_closure, **kwds) def put_var_decrefs(self, entries, used_only = 0): for entry in entries: @@ -2186,19 +2255,6 @@ class CCodeWriter(object): for entry in entries: self.put_var_xdecref_clear(entry) - def put_incref_memoryviewslice(self, slice_cname, have_gil=False): - from . import MemoryView - self.globalstate.use_utility_code(MemoryView.memviewslice_init_code) - self.putln("__PYX_INC_MEMVIEW(&%s, %d);" % (slice_cname, int(have_gil))) - - def put_xdecref_memoryviewslice(self, slice_cname, have_gil=False): - from . import MemoryView - self.globalstate.use_utility_code(MemoryView.memviewslice_init_code) - self.putln("__PYX_XDEC_MEMVIEW(&%s, %d);" % (slice_cname, int(have_gil))) - - def put_xgiveref_memoryviewslice(self, slice_cname): - self.put_xgiveref("%s.memview" % slice_cname) - def put_init_to_py_none(self, cname, type, nanny=True): from .PyrexTypes import py_object_type, typecast py_none = typecast(type, py_object_type, "Py_None") @@ -2234,14 +2290,13 @@ class CCodeWriter(object): method_flags += [TypeSlots.method_coexist] func_ptr = wrapper_code_writer.put_pymethoddef_wrapper(entry) if wrapper_code_writer else entry.func_cname # Add required casts, but try not to shadow real warnings. - cast = '__Pyx_PyCFunctionFast' if 'METH_FASTCALL' in method_flags else 'PyCFunction' - if 'METH_KEYWORDS' in method_flags: - cast += 'WithKeywords' + cast = entry.signature.method_function_type() if cast != 'PyCFunction': func_ptr = '(void*)(%s)%s' % (cast, func_ptr) + entry_name = entry.name.as_c_string_literal() self.putln( - '{"%s", (PyCFunction)%s, %s, %s}%s' % ( - entry.name, + '{%s, (PyCFunction)%s, %s, %s}%s' % ( + entry_name, func_ptr, "|".join(method_flags), entry.doc_cname if entry.doc else '0', @@ -2250,8 +2305,9 @@ class CCodeWriter(object): def put_pymethoddef_wrapper(self, entry): func_cname = entry.func_cname if entry.is_special: - method_flags = entry.signature.method_flags() - if method_flags and 'METH_NOARGS' in method_flags: + method_flags = entry.signature.method_flags() or [] + from .TypeSlots import method_noargs + if method_noargs in method_flags: # Special NOARGS methods really take no arguments besides 'self', but PyCFunction expects one. func_cname = Naming.method_wrapper_prefix + func_cname self.putln("static PyObject *%s(PyObject *self, CYTHON_UNUSED PyObject *arg) {return %s(self);}" % ( @@ -2260,6 +2316,12 @@ class CCodeWriter(object): # GIL methods + def use_fast_gil_utility_code(self): + if self.globalstate.directives['fast_gil']: + self.globalstate.use_utility_code(UtilityCode.load_cached("FastGil", "ModuleSetupCode.c")) + else: + self.globalstate.use_utility_code(UtilityCode.load_cached("NoFastGil", "ModuleSetupCode.c")) + def put_ensure_gil(self, declare_gilstate=True, variable=None): """ Acquire the GIL. The generated code is safe even when no PyThreadState @@ -2269,10 +2331,7 @@ class CCodeWriter(object): """ self.globalstate.use_utility_code( UtilityCode.load_cached("ForceInitThreads", "ModuleSetupCode.c")) - if self.globalstate.directives['fast_gil']: - self.globalstate.use_utility_code(UtilityCode.load_cached("FastGil", "ModuleSetupCode.c")) - else: - self.globalstate.use_utility_code(UtilityCode.load_cached("NoFastGil", "ModuleSetupCode.c")) + self.use_fast_gil_utility_code() self.putln("#ifdef WITH_THREAD") if not variable: variable = '__pyx_gilstate_save' @@ -2285,10 +2344,7 @@ class CCodeWriter(object): """ Releases the GIL, corresponds to `put_ensure_gil`. """ - if self.globalstate.directives['fast_gil']: - self.globalstate.use_utility_code(UtilityCode.load_cached("FastGil", "ModuleSetupCode.c")) - else: - self.globalstate.use_utility_code(UtilityCode.load_cached("NoFastGil", "ModuleSetupCode.c")) + self.use_fast_gil_utility_code() if not variable: variable = '__pyx_gilstate_save' self.putln("#ifdef WITH_THREAD") @@ -2300,10 +2356,7 @@ class CCodeWriter(object): Acquire the GIL. The thread's thread state must have been initialized by a previous `put_release_gil` """ - if self.globalstate.directives['fast_gil']: - self.globalstate.use_utility_code(UtilityCode.load_cached("FastGil", "ModuleSetupCode.c")) - else: - self.globalstate.use_utility_code(UtilityCode.load_cached("NoFastGil", "ModuleSetupCode.c")) + self.use_fast_gil_utility_code() self.putln("#ifdef WITH_THREAD") self.putln("__Pyx_FastGIL_Forget();") if variable: @@ -2313,10 +2366,7 @@ class CCodeWriter(object): def put_release_gil(self, variable=None): "Release the GIL, corresponds to `put_acquire_gil`." - if self.globalstate.directives['fast_gil']: - self.globalstate.use_utility_code(UtilityCode.load_cached("FastGil", "ModuleSetupCode.c")) - else: - self.globalstate.use_utility_code(UtilityCode.load_cached("NoFastGil", "ModuleSetupCode.c")) + self.use_fast_gil_utility_code() self.putln("#ifdef WITH_THREAD") self.putln("PyThreadState *_save;") self.putln("Py_UNBLOCK_THREADS") @@ -2337,23 +2387,34 @@ class CCodeWriter(object): # return self.putln("if (unlikely(%s < 0)) %s" % (value, self.error_goto(pos))) return self.putln("if (%s < 0) %s" % (value, self.error_goto(pos))) - def put_error_if_unbound(self, pos, entry, in_nogil_context=False): - from . import ExprNodes + def put_error_if_unbound(self, pos, entry, in_nogil_context=False, unbound_check_code=None): if entry.from_closure: func = '__Pyx_RaiseClosureNameError' self.globalstate.use_utility_code( - ExprNodes.raise_closure_name_error_utility_code) + UtilityCode.load_cached("RaiseClosureNameError", "ObjectHandling.c")) elif entry.type.is_memoryviewslice and in_nogil_context: func = '__Pyx_RaiseUnboundMemoryviewSliceNogil' self.globalstate.use_utility_code( - ExprNodes.raise_unbound_memoryview_utility_code_nogil) + UtilityCode.load_cached("RaiseUnboundMemoryviewSliceNogil", "ObjectHandling.c")) + elif entry.type.is_cpp_class and entry.is_cglobal: + func = '__Pyx_RaiseCppGlobalNameError' + self.globalstate.use_utility_code( + UtilityCode.load_cached("RaiseCppGlobalNameError", "ObjectHandling.c")) + elif entry.type.is_cpp_class and entry.is_variable and not entry.is_member and entry.scope.is_c_class_scope: + # there doesn't seem to be a good way to detecting an instance-attribute of a C class + # (is_member is only set for class attributes) + func = '__Pyx_RaiseCppAttributeError' + self.globalstate.use_utility_code( + UtilityCode.load_cached("RaiseCppAttributeError", "ObjectHandling.c")) else: func = '__Pyx_RaiseUnboundLocalError' self.globalstate.use_utility_code( - ExprNodes.raise_unbound_local_error_utility_code) + UtilityCode.load_cached("RaiseUnboundLocalError", "ObjectHandling.c")) + if not unbound_check_code: + unbound_check_code = entry.type.check_for_null_code(entry.cname) self.putln('if (unlikely(!%s)) { %s("%s"); %s }' % ( - entry.type.check_for_null_code(entry.cname), + unbound_check_code, func, entry.name, self.error_goto(pos))) @@ -2386,7 +2447,8 @@ class CCodeWriter(object): return self.error_goto_if("!%s" % cname, pos) def error_goto_if_neg(self, cname, pos): - return self.error_goto_if("%s < 0" % cname, pos) + # Add extra parentheses to silence clang warnings about constant conditions. + return self.error_goto_if("(%s < 0)" % cname, pos) def error_goto_if_PyErr(self, pos): return self.error_goto_if("PyErr_Occurred()", pos) @@ -2398,13 +2460,14 @@ class CCodeWriter(object): self.putln('__Pyx_RefNannyDeclarations') def put_setup_refcount_context(self, name, acquire_gil=False): + name = name.as_c_string_literal() # handle unicode names if acquire_gil: self.globalstate.use_utility_code( UtilityCode.load_cached("ForceInitThreads", "ModuleSetupCode.c")) - self.putln('__Pyx_RefNannySetupContext("%s", %d);' % (name, acquire_gil and 1 or 0)) + self.putln('__Pyx_RefNannySetupContext(%s, %d);' % (name, acquire_gil and 1 or 0)) - def put_finish_refcount_context(self): - self.putln("__Pyx_RefNannyFinishContext();") + def put_finish_refcount_context(self, nogil=False): + self.putln("__Pyx_RefNannyFinishContextNogil()" if nogil else "__Pyx_RefNannyFinishContext();") def put_add_traceback(self, qualified_name, include_cline=True): """ @@ -2412,14 +2475,16 @@ class CCodeWriter(object): qualified_name should be the qualified name of the function. """ + qualified_name = qualified_name.as_c_string_literal() # handle unicode names format_tuple = ( qualified_name, Naming.clineno_cname if include_cline else 0, Naming.lineno_cname, Naming.filename_cname, ) + self.funcstate.uses_error_indicator = True - self.putln('__Pyx_AddTraceback("%s", %s, %s, %s);' % format_tuple) + self.putln('__Pyx_AddTraceback(%s, %s, %s, %s);' % format_tuple) def put_unraisable(self, qualified_name, nogil=False): """ diff --git a/Cython/Compiler/CythonScope.py b/Cython/Compiler/CythonScope.py index 1c25d1a6b..bb9e74aa6 100644 --- a/Cython/Compiler/CythonScope.py +++ b/Cython/Compiler/CythonScope.py @@ -125,7 +125,16 @@ class CythonScope(ModuleScope): view_utility_scope = MemoryView.view_utility_code.declare_in_scope( self.viewscope, cython_scope=self, - whitelist=MemoryView.view_utility_whitelist) + allowlist=MemoryView.view_utility_allowlist) + + # Marks the types as being cython_builtin_type so that they can be + # extended from without Cython attempting to import cython.view + ext_types = [ entry.type + for entry in view_utility_scope.entries.values() + if entry.type.is_extension_type ] + for ext_type in ext_types: + ext_type.is_cython_builtin_type = 1 + # self.entries["array"] = view_utility_scope.entries.pop("array") diff --git a/Cython/Compiler/Errors.py b/Cython/Compiler/Errors.py index 9761b52c3..ce011b616 100644 --- a/Cython/Compiler/Errors.py +++ b/Cython/Compiler/Errors.py @@ -24,6 +24,8 @@ class PyrexError(Exception): class PyrexWarning(Exception): pass +class CannotSpecialize(PyrexError): + pass def context(position): source = position[0] @@ -60,11 +62,9 @@ class CompileError(PyrexError): self.message_only = message self.formatted_message = format_error(message, position) self.reported = False - # Deprecated and withdrawn in 2.6: - # self.message = message Exception.__init__(self, self.formatted_message) # Python Exception subclass pickling is broken, - # see http://bugs.python.org/issue1692335 + # see https://bugs.python.org/issue1692335 self.args = (position, message) def __str__(self): @@ -74,8 +74,6 @@ class CompileWarning(PyrexWarning): def __init__(self, position = None, message = ""): self.position = position - # Deprecated and withdrawn in 2.6: - # self.message = message Exception.__init__(self, format_position(position) + message) class InternalError(Exception): @@ -114,7 +112,7 @@ class CompilerCrash(CompileError): message += u'%s: %s' % (cause.__class__.__name__, cause) CompileError.__init__(self, pos, message) # Python Exception subclass pickling is broken, - # see http://bugs.python.org/issue1692335 + # see https://bugs.python.org/issue1692335 self.args = (pos, context, message, cause, stacktrace) class NoElementTreeInstalledException(PyrexError): @@ -171,29 +169,34 @@ def report_error(err, use_stack=True): if Options.fast_fail: raise AbortError("fatal errors") - def error(position, message): #print("Errors.error:", repr(position), repr(message)) ### if position is None: raise InternalError(message) err = CompileError(position, message) - if DebugFlags.debug_exception_on_error: raise Exception(err) # debug + if DebugFlags.debug_exception_on_error: raise Exception(err) # debug report_error(err) return err -LEVEL = 1 # warn about all errors level 1 or higher +LEVEL = 1 # warn about all errors level 1 or higher + +def _write_file_encode(file, line): + try: + file.write(line) + except UnicodeEncodeError: + file.write(line.encode('ascii', 'replace')) def message(position, message, level=1): if level < LEVEL: return warn = CompileWarning(position, message) - line = "note: %s\n" % warn + line = u"note: %s\n" % warn if listing_file: - listing_file.write(line) + _write_file_encode(listing_file, line) if echo_file: - echo_file.write(line) + _write_file_encode(echo_file, line) return warn @@ -203,11 +206,11 @@ def warning(position, message, level=0): if Options.warning_errors and position: return error(position, message) warn = CompileWarning(position, message) - line = "warning: %s\n" % warn + line = u"warning: %s\n" % warn if listing_file: - listing_file.write(line) + _write_file_encode(listing_file, line) if echo_file: - echo_file.write(line) + _write_file_encode(echo_file, line) return warn @@ -216,11 +219,11 @@ def warn_once(position, message, level=0): if level < LEVEL or message in _warn_once_seen: return warn = CompileWarning(position, message) - line = "warning: %s\n" % warn + line = u"warning: %s\n" % warn if listing_file: - listing_file.write(line) + _write_file_encode(listing_file, line) if echo_file: - echo_file.write(line) + _write_file_encode(echo_file, line) _warn_once_seen[message] = True return warn diff --git a/Cython/Compiler/ExprNodes.py b/Cython/Compiler/ExprNodes.py index 719b7f2bf..a2ccc465c 100644 --- a/Cython/Compiler/ExprNodes.py +++ b/Cython/Compiler/ExprNodes.py @@ -23,18 +23,22 @@ import os.path import operator from .Errors import ( - error, warning, InternalError, CompileError, report_error, local_errors) + error, warning, InternalError, CompileError, report_error, local_errors, + CannotSpecialize) from .Code import UtilityCode, TempitaUtilityCode from . import StringEncoding from . import Naming from . import Nodes -from .Nodes import Node, utility_code_for_imports, analyse_type_annotation +from .Nodes import Node, utility_code_for_imports, SingleAssignmentNode, PassStatNode from . import PyrexTypes from .PyrexTypes import py_object_type, c_long_type, typecast, error_type, \ unspecified_type from . import TypeSlots -from .Builtin import list_type, tuple_type, set_type, dict_type, type_type, \ - unicode_type, str_type, bytes_type, bytearray_type, basestring_type, slice_type +from .Builtin import ( + list_type, tuple_type, set_type, dict_type, type_type, + unicode_type, str_type, bytes_type, bytearray_type, basestring_type, + slice_type, long_type, +) from . import Builtin from . import Symtab from .. import Utils @@ -182,7 +186,7 @@ def infer_sequence_item_type(env, seq_node, index_node=None, seq_type=None): else: return item.infer_type(env) # if we're lucky, all items have the same type - item_types = set([item.infer_type(env) for item in seq_node.args]) + item_types = {item.infer_type(env) for item in seq_node.args} if len(item_types) == 1: return item_types.pop() return None @@ -205,6 +209,11 @@ def make_dedup_key(outer_type, item_nodes): # For constants, look at the Python value type if we don't know the concrete Cython type. else (node.type, node.constant_result, type(node.constant_result) if node.type is py_object_type else None) if node.has_constant_result() + # IdentifierStringNode doesn't usually have a "constant_result" set because: + # 1. it doesn't usually have unicode_value + # 2. it's often created later in the compilation process after ConstantFolding + # but should be cacheable + else (node.type, node.value, node.unicode_value, "IdentifierStringNode") if isinstance(node, IdentifierStringNode) else None # something we cannot handle => short-circuit below for node in item_nodes ] @@ -237,19 +246,23 @@ def get_exception_handler(exception_value): exception_value.entry.cname), False) + def maybe_check_py_error(code, check_py_exception, pos, nogil): if check_py_exception: if nogil: + code.globalstate.use_utility_code( + UtilityCode.load_cached("ErrOccurredWithGIL", "Exceptions.c")) code.putln(code.error_goto_if("__Pyx_ErrOccurredWithGIL()", pos)) else: code.putln(code.error_goto_if("PyErr_Occurred()", pos)) + def translate_cpp_exception(code, pos, inside, py_result, exception_value, nogil): raise_py_exception, check_py_exception = get_exception_handler(exception_value) code.putln("try {") code.putln("%s" % inside) if py_result: - code.putln(code.error_goto_if_null(py_result, pos)) + code.putln(code.error_goto_if_null(py_result, pos)) maybe_check_py_error(code, check_py_exception, pos, nogil) code.putln("} catch(...) {") if nogil: @@ -260,10 +273,24 @@ def translate_cpp_exception(code, pos, inside, py_result, exception_value, nogil code.putln(code.error_goto(pos)) code.putln("}") +def needs_cpp_exception_conversion(node): + assert node.exception_check == "+" + if node.exception_value is None: + return True + # exception_value can be a NameNode + # (in which case it's used as a handler function and no conversion is needed) + if node.exception_value.is_name: + return False + # or a CharNode with a value of "*" + if isinstance(node.exception_value, CharNode) and node.exception_value.value == "*": + return True + # Most other const-nodes are disallowed after "+" by the parser + return False + + # Used to handle the case where an lvalue expression and an overloaded assignment # both have an exception declaration. -def translate_double_cpp_exception(code, pos, lhs_type, lhs_code, rhs_code, - lhs_exc_val, assign_exc_val, nogil): +def translate_double_cpp_exception(code, pos, lhs_type, lhs_code, rhs_code, lhs_exc_val, assign_exc_val, nogil): handle_lhs_exc, lhc_check_py_exc = get_exception_handler(lhs_exc_val) handle_assignment_exc, assignment_check_py_exc = get_exception_handler(assign_exc_val) code.putln("try {") @@ -314,8 +341,8 @@ class ExprNode(Node): type = None annotation = None temp_code = None - old_temp = None # error checker for multiple frees etc. - use_managed_ref = True # can be set by optimisation transforms + old_temp = None # error checker for multiple frees etc. + use_managed_ref = True # can be set by optimisation transforms result_is_used = True is_numpy_attribute = False @@ -448,6 +475,7 @@ class ExprNode(Node): saved_subexpr_nodes = None is_temp = False + has_temp_moved = False # if True then attempting to do anything but free the temp is invalid is_target = False is_starred = False @@ -455,11 +483,13 @@ class ExprNode(Node): child_attrs = property(fget=operator.attrgetter('subexprs')) + def analyse_annotations(self, env): + pass + def not_implemented(self, method_name): - print_call_chain(method_name, "not implemented") ### + print_call_chain(method_name, "not implemented") raise InternalError( - "%s.%s not implemented" % - (self.__class__.__name__, method_name)) + "%s.%s not implemented" % (self.__class__.__name__, method_name)) def is_lvalue(self): return 0 @@ -498,6 +528,22 @@ class ExprNode(Node): else: return self.calculate_result_code() + def _make_move_result_rhs(self, result, optional=False): + if optional and not (self.is_temp and self.type.is_cpp_class and not self.type.is_reference): + return result + self.has_temp_moved = True + return "{}({})".format("__PYX_STD_MOVE_IF_SUPPORTED" if optional else "std::move", result) + + def move_result_rhs(self): + return self._make_move_result_rhs(self.result(), optional=True) + + def move_result_rhs_as(self, type): + result = self.result_as(type) + if not (type.is_reference or type.needs_refcounting): + requires_move = type.is_rvalue_reference and self.is_temp + result = self._make_move_result_rhs(result, optional=not requires_move) + return result + def pythran_result(self, type_=None): if is_pythran_supported_node_or_none(self): return to_pythran(self) @@ -614,7 +660,7 @@ class ExprNode(Node): def type_dependencies(self, env): # Returns the list of entries whose types must be determined # before the type of self can be inferred. - if hasattr(self, 'type') and self.type is not None: + if getattr(self, 'type', None) is not None: return () return sum([node.type_dependencies(env) for node in self.subexpr_nodes()], ()) @@ -623,12 +669,13 @@ class ExprNode(Node): # Differs from analyse_types as it avoids unnecessary # analysis of subexpressions, but can assume everything # in self.type_dependencies() has been resolved. - if hasattr(self, 'type') and self.type is not None: - return self.type - elif hasattr(self, 'entry') and self.entry is not None: - return self.entry.type - else: - self.not_implemented("infer_type") + type = getattr(self, 'type', None) + if type is not None: + return type + entry = getattr(self, 'entry', None) + if entry is not None: + return entry.type + self.not_implemented("infer_type") def nonlocally_immutable(self): # Returns whether this variable is a safe reference, i.e. @@ -655,6 +702,19 @@ class ExprNode(Node): # type, return that type, else None. return None + def analyse_as_specialized_type(self, env): + type = self.analyse_as_type(env) + if type and type.is_fused and env.fused_to_specific: + # while it would be nice to test "if entry.type in env.fused_to_specific" + # rather than try/catch this doesn't work reliably (mainly for nested fused types) + try: + return type.specialize(env.fused_to_specific) + except KeyError: + pass + if type and type.is_fused: + error(self.pos, "Type is not specific") + return type + def analyse_as_extension_type(self, env): # If this node can be interpreted as a reference to an # extension type or builtin type, return its type, else None. @@ -746,19 +806,20 @@ class ExprNode(Node): def make_owned_reference(self, code): """ - If result is a pyobject, make sure we own a reference to it. + Make sure we own a reference to result. If the result is in a temp, it is already a new reference. """ - if self.type.is_pyobject and not self.result_in_temp(): + if not self.result_in_temp(): code.put_incref(self.result(), self.ctype()) def make_owned_memoryviewslice(self, code): """ Make sure we own the reference to this memoryview slice. """ + # TODO ideally this would be shared with "make_owned_reference" if not self.result_in_temp(): - code.put_incref_memoryviewslice(self.result(), - have_gil=self.in_nogil_context) + code.put_incref_memoryviewslice(self.result(), self.type, + have_gil=not self.in_nogil_context) def generate_evaluation_code(self, code): # Generate code to evaluate this node and @@ -785,19 +846,17 @@ class ExprNode(Node): self.not_implemented("generate_result_code") def generate_disposal_code(self, code): + if self.has_temp_moved: + code.globalstate.use_utility_code( + UtilityCode.load_cached("MoveIfSupported", "CppSupport.cpp")) if self.is_temp: if self.type.is_string or self.type.is_pyunicode_ptr: # postponed from self.generate_evaluation_code() self.generate_subexpr_disposal_code(code) self.free_subexpr_temps(code) if self.result(): - if self.type.is_pyobject: - code.put_decref_clear(self.result(), self.ctype()) - elif self.type.is_memoryviewslice: - code.put_xdecref_memoryviewslice( - self.result(), have_gil=not self.in_nogil_context) - code.putln("%s.memview = NULL;" % self.result()) - code.putln("%s.data = NULL;" % self.result()) + code.put_decref_clear(self.result(), self.ctype(), + have_gil=not self.in_nogil_context) else: # Already done if self.is_temp self.generate_subexpr_disposal_code(code) @@ -819,11 +878,15 @@ class ExprNode(Node): elif self.type.is_memoryviewslice: code.putln("%s.memview = NULL;" % self.result()) code.putln("%s.data = NULL;" % self.result()) + + if self.has_temp_moved: + code.globalstate.use_utility_code( + UtilityCode.load_cached("MoveIfSupported", "CppSupport.cpp")) else: self.generate_subexpr_disposal_code(code) def generate_assignment_code(self, rhs, code, overloaded_assignment=False, - exception_check=None, exception_value=None): + exception_check=None, exception_value=None): # Stub method for nodes which are not legal as # the LHS of an assignment. An error will have # been reported earlier. @@ -849,6 +912,32 @@ class ExprNode(Node): def generate_function_definitions(self, env, code): pass + # ----Generation of small bits of reference counting -- + + def generate_decref_set(self, code, rhs): + code.put_decref_set(self.result(), self.ctype(), rhs) + + def generate_xdecref_set(self, code, rhs): + code.put_xdecref_set(self.result(), self.ctype(), rhs) + + def generate_gotref(self, code, handle_null=False, + maybe_null_extra_check=True): + if not (handle_null and self.cf_is_null): + if (handle_null and self.cf_maybe_null + and maybe_null_extra_check): + self.generate_xgotref(code) + else: + code.put_gotref(self.result(), self.ctype()) + + def generate_xgotref(self, code): + code.put_xgotref(self.result(), self.ctype()) + + def generate_giveref(self, code): + code.put_giveref(self.result(), self.ctype()) + + def generate_xgiveref(self, code): + code.put_xgiveref(self.result(), self.ctype()) + # ---------------- Annotation --------------------- def annotate(self, code): @@ -883,8 +972,8 @@ class ExprNode(Node): if used_as_reference and not src_type.is_reference: dst_type = dst_type.ref_base_type - if src_type.is_const: - src_type = src_type.const_base_type + if src_type.is_cv_qualified: + src_type = src_type.cv_base_type if src_type.is_fused or dst_type.is_fused: # See if we are coercing a fused function to a pointer to a @@ -970,7 +1059,8 @@ class ExprNode(Node): and src_type != dst_type and dst_type.assignable_from(src_type)): src = CoerceToComplexNode(src, dst_type, env) - else: # neither src nor dst are py types + else: + # neither src nor dst are py types # Added the string comparison, since for c types that # is enough, but Cython gets confused when the types are # in different pxi files. @@ -1108,6 +1198,7 @@ class PyConstNode(AtomicExprNode): is_literal = 1 type = py_object_type + nogil_check = None def is_simple(self): return 1 @@ -1133,8 +1224,6 @@ class NoneNode(PyConstNode): constant_result = None - nogil_check = None - def compile_time_value(self, denv): return None @@ -1204,7 +1293,7 @@ class BoolNode(ConstNode): def calculate_result_code(self): if self.type.is_pyobject: - return self.value and 'Py_True' or 'Py_False' + return 'Py_True' if self.value else 'Py_False' else: return str(int(self.value)) @@ -1256,7 +1345,7 @@ class IntNode(ConstNode): unsigned = "" longness = "" - is_c_literal = None # unknown + is_c_literal = None # unknown def __init__(self, pos, **kwds): ExprNode.__init__(self, pos, **kwds) @@ -1434,11 +1523,7 @@ def _analyse_name_as_type(name, pos, env): return type global_entry = env.global_scope().lookup(name) - if global_entry and global_entry.type and ( - global_entry.type.is_extension_type - or global_entry.type.is_struct_or_union - or global_entry.type.is_builtin_type - or global_entry.type.is_cpp_class): + if global_entry and global_entry.is_type and global_entry.type: return global_entry.type from .TreeFragment import TreeFragment @@ -1539,7 +1624,7 @@ class BytesNode(ConstNode): self.result_code = result def get_constant_c_result_code(self): - return None # FIXME + return None # FIXME def calculate_result_code(self): return self.result_code @@ -1594,12 +1679,9 @@ class UnicodeNode(ConstNode): if dst_type.is_string and self.bytes_value is not None: # special case: '-3' enforced unicode literal used in a # C char* context - return BytesNode(self.pos, value=self.bytes_value - ).coerce_to(dst_type, env) + return BytesNode(self.pos, value=self.bytes_value).coerce_to(dst_type, env) if dst_type.is_pyunicode_ptr: - node = UnicodeNode(self.pos, value=self.value) - node.type = dst_type - return node + return UnicodeNode(self.pos, value=self.value, type=dst_type) error(self.pos, "Unicode literals do not support coercion to C types other " "than Py_UNICODE/Py_UCS4 (for characters) or Py_UNICODE* " @@ -1784,7 +1866,7 @@ class ImagNode(AtomicExprNode): self.result(), float(self.value), code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) class NewExprNode(AtomicExprNode): @@ -1837,7 +1919,7 @@ class NameNode(AtomicExprNode): is_name = True is_cython_module = False cython_attribute = None - lhs_of_first_assignment = False # TODO: remove me + lhs_of_first_assignment = False # TODO: remove me is_used_as_rvalue = 0 entry = None type_entry = None @@ -1919,30 +2001,50 @@ class NameNode(AtomicExprNode): def declare_from_annotation(self, env, as_target=False): """Implements PEP 526 annotation typing in a fairly relaxed way. - Annotations are ignored for global variables, Python class attributes and already declared variables. - String literals are allowed and ignored. - The ambiguous Python types 'int' and 'long' are ignored and the 'cython.int' form must be used instead. + Annotations are ignored for global variables. + All other annotations are stored on the entry in the symbol table. + String literals are allowed and not evaluated. + The ambiguous Python types 'int' and 'long' are not evaluated - the 'cython.int' form must be used instead. """ - if not env.directives['annotation_typing']: - return - if env.is_module_scope or env.is_py_class_scope: - # annotations never create global cdef names and Python classes don't support them anyway - return name = self.name - if self.entry or env.lookup_here(name) is not None: - # already declared => ignore annotation - return - annotation = self.annotation - if annotation.is_string_literal: - # name: "description" => not a type, but still a declared variable or attribute - atype = None - else: - _, atype = analyse_type_annotation(annotation, env) - if atype is None: - atype = unspecified_type if as_target and env.directives['infer_types'] != False else py_object_type - self.entry = env.declare_var(name, atype, self.pos, is_cdef=not as_target) - self.entry.annotation = annotation + entry = self.entry or env.lookup_here(name) + if not entry: + # annotations never create global cdef names + if env.is_module_scope: + return + if ( + # name: "description" => not a type, but still a declared variable or attribute + annotation.expr.is_string_literal + # don't do type analysis from annotations if not asked to, but still collect the annotation + or not env.directives['annotation_typing'] + ): + atype = None + elif env.is_py_class_scope: + # For Python class scopes every attribute is a Python object + atype = py_object_type + else: + _, atype = annotation.analyse_type_annotation(env) + if atype is None: + atype = unspecified_type if as_target and env.directives['infer_types'] != False else py_object_type + if atype.is_fused and env.fused_to_specific: + try: + atype = atype.specialize(env.fused_to_specific) + except CannotSpecialize: + error(self.pos, + "'%s' cannot be specialized since its type is not a fused argument to this function" % + self.name) + atype = error_type + if as_target and env.is_c_class_scope and not (atype.is_pyobject or atype.is_error): + # TODO: this will need revising slightly if either cdef dataclasses or + # annotated cdef attributes are implemented + atype = py_object_type + warning(annotation.pos, "Annotation ignored since class-level attributes must be Python objects. " + "Were you trying to set up an instance attribute?", 2) + entry = self.entry = env.declare_var(name, atype, self.pos, is_cdef=not as_target) + # Even if the entry already exists, make sure we're supplying an annotation if we can. + if annotation and not entry.annotation: + entry.annotation = annotation def analyse_as_module(self, env): # Try to interpret this as a reference to a cimported module. @@ -1981,6 +2083,7 @@ class NameNode(AtomicExprNode): return None def analyse_target_declaration(self, env): + self.is_target = True if not self.entry: self.entry = env.lookup_here(self.name) if not self.entry and self.annotation is not None: @@ -2071,7 +2174,7 @@ class NameNode(AtomicExprNode): if self.is_used_as_rvalue: entry = self.entry if entry.is_builtin: - if not entry.is_const: # cached builtins are ok + if not entry.is_const: # cached builtins are ok self.gil_error() elif entry.is_pyglobal: self.gil_error() @@ -2096,7 +2199,7 @@ class NameNode(AtomicExprNode): entry = self.entry if entry.is_type and entry.type.is_extension_type: self.type_entry = entry - if entry.is_type and entry.type.is_enum: + if entry.is_type and (entry.type.is_enum or entry.type.is_cpp_enum): py_entry = Symtab.Entry(self.name, None, py_object_type) py_entry.is_pyglobal = True py_entry.scope = self.entry.scope @@ -2122,7 +2225,7 @@ class NameNode(AtomicExprNode): def may_be_none(self): if self.cf_state and self.type and (self.type.is_pyobject or self.type.is_memoryviewslice): - # gard against infinite recursion on self-dependencies + # guard against infinite recursion on self-dependencies if getattr(self, '_none_checking', False): # self-dependency - either this node receives a None # value from *another* node, or it can not reference @@ -2189,24 +2292,25 @@ class NameNode(AtomicExprNode): def calculate_result_code(self): entry = self.entry if not entry: - return "<error>" # There was an error earlier + return "<error>" # There was an error earlier + if self.entry.is_cpp_optional and not self.is_target: + return "(*%s)" % entry.cname return entry.cname def generate_result_code(self, code): - assert hasattr(self, 'entry') entry = self.entry if entry is None: - return # There was an error earlier + return # There was an error earlier if entry.utility_code: code.globalstate.use_utility_code(entry.utility_code) if entry.is_builtin and entry.is_const: - return # Lookup already cached + return # Lookup already cached elif entry.is_pyclass_attr: assert entry.type.is_pyobject, "Python global or builtin not a Python object" interned_cname = code.intern_identifier(self.entry.name) if entry.is_builtin: namespace = Naming.builtins_cname - else: # entry.is_pyglobal + else: # entry.is_pyglobal namespace = entry.scope.namespace_cname if not self.cf_is_null: code.putln( @@ -2225,7 +2329,7 @@ class NameNode(AtomicExprNode): if not self.cf_is_null: code.putln("}") code.putln(code.error_goto_if_null(self.result(), self.pos)) - code.put_gotref(self.py_result()) + self.generate_gotref(code) elif entry.is_builtin and not entry.scope.is_module_scope: # known builtin @@ -2238,7 +2342,7 @@ class NameNode(AtomicExprNode): self.result(), interned_cname, code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) elif entry.is_pyglobal or (entry.is_builtin and entry.scope.is_module_scope): # name in class body, global name or unknown builtin @@ -2262,25 +2366,34 @@ class NameNode(AtomicExprNode): entry.scope.namespace_cname, interned_cname, code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) elif entry.is_local or entry.in_closure or entry.from_closure or entry.type.is_memoryviewslice: # Raise UnboundLocalError for objects and memoryviewslices raise_unbound = ( (self.cf_maybe_null or self.cf_is_null) and not self.allow_null) - null_code = entry.type.check_for_null_code(entry.cname) memslice_check = entry.type.is_memoryviewslice and self.initialized_check + optional_cpp_check = entry.is_cpp_optional and self.initialized_check - if null_code and raise_unbound and (entry.type.is_pyobject or memslice_check): - code.put_error_if_unbound(self.pos, entry, self.in_nogil_context) + if optional_cpp_check: + unbound_check_code = entry.type.cpp_optional_check_for_null_code(entry.cname) + else: + unbound_check_code = entry.type.check_for_null_code(entry.cname) + + if unbound_check_code and raise_unbound and (entry.type.is_pyobject or memslice_check or optional_cpp_check): + code.put_error_if_unbound(self.pos, entry, self.in_nogil_context, unbound_check_code=unbound_check_code) + + elif entry.is_cglobal and entry.is_cpp_optional and self.initialized_check: + unbound_check_code = entry.type.cpp_optional_check_for_null_code(entry.cname) + code.put_error_if_unbound(self.pos, entry, unbound_check_code=unbound_check_code) def generate_assignment_code(self, rhs, code, overloaded_assignment=False, - exception_check=None, exception_value=None): + exception_check=None, exception_value=None): #print "NameNode.generate_assignment_code:", self.name ### entry = self.entry if entry is None: - return # There was an error earlier + return # There was an error earlier if (self.entry.type.is_ptr and isinstance(rhs, ListNode) and not self.lhs_of_first_assignment and not rhs.in_module_scope): @@ -2301,8 +2414,10 @@ class NameNode(AtomicExprNode): setter = 'PyDict_SetItem' namespace = Naming.moddict_cname elif entry.is_pyclass_attr: - code.globalstate.use_utility_code(UtilityCode.load_cached("SetNameInClass", "ObjectHandling.c")) - setter = '__Pyx_SetNameInClass' + # Special-case setting __new__ + n = "SetNewInClass" if self.name == "__new__" else "SetNameInClass" + code.globalstate.use_utility_code(UtilityCode.load_cached(n, "ObjectHandling.c")) + setter = '__Pyx_' + n else: assert False, repr(entry) code.put_error_if_neg( @@ -2344,31 +2459,24 @@ class NameNode(AtomicExprNode): rhs.make_owned_reference(code) is_external_ref = entry.is_cglobal or self.entry.in_closure or self.entry.from_closure if is_external_ref: - if not self.cf_is_null: - if self.cf_maybe_null: - code.put_xgotref(self.py_result()) - else: - code.put_gotref(self.py_result()) + self.generate_gotref(code, handle_null=True) assigned = True if entry.is_cglobal: - code.put_decref_set( - self.result(), rhs.result_as(self.ctype())) + self.generate_decref_set(code, rhs.result_as(self.ctype())) else: if not self.cf_is_null: if self.cf_maybe_null: - code.put_xdecref_set( - self.result(), rhs.result_as(self.ctype())) + self.generate_xdecref_set(code, rhs.result_as(self.ctype())) else: - code.put_decref_set( - self.result(), rhs.result_as(self.ctype())) + self.generate_decref_set(code, rhs.result_as(self.ctype())) else: assigned = False if is_external_ref: - code.put_giveref(rhs.py_result()) + rhs.generate_giveref(code) if not self.type.is_memoryviewslice: if not assigned: if overloaded_assignment: - result = rhs.result() + result = rhs.move_result_rhs() if exception_check == '+': translate_cpp_exception( code, self.pos, @@ -2378,7 +2486,7 @@ class NameNode(AtomicExprNode): else: code.putln('%s = %s;' % (self.result(), result)) else: - result = rhs.result_as(self.ctype()) + result = rhs.move_result_rhs_as(self.ctype()) if is_pythran_expr(self.type): code.putln('new (&%s) decltype(%s){%s};' % (self.result(), self.result(), result)) @@ -2431,7 +2539,7 @@ class NameNode(AtomicExprNode): def generate_deletion_code(self, code, ignore_nonexisting=False): if self.entry is None: - return # There was an error earlier + return # There was an error earlier elif self.entry.is_pyclass_attr: namespace = self.entry.scope.namespace_cname interned_cname = code.intern_identifier(self.entry.name) @@ -2467,26 +2575,20 @@ class NameNode(AtomicExprNode): if self.cf_maybe_null and not ignore_nonexisting: code.put_error_if_unbound(self.pos, self.entry) - if self.entry.type.is_pyobject: - if self.entry.in_closure: - # generator - if ignore_nonexisting and self.cf_maybe_null: - code.put_xgotref(self.result()) - else: - code.put_gotref(self.result()) - if ignore_nonexisting and self.cf_maybe_null: - code.put_xdecref(self.result(), self.ctype()) - else: - code.put_decref(self.result(), self.ctype()) - code.putln('%s = NULL;' % self.result()) + if self.entry.in_closure: + # generator + self.generate_gotref(code, handle_null=True, maybe_null_extra_check=ignore_nonexisting) + if ignore_nonexisting and self.cf_maybe_null: + code.put_xdecref_clear(self.result(), self.ctype(), + have_gil=not self.nogil) else: - code.put_xdecref_memoryviewslice(self.entry.cname, - have_gil=not self.nogil) + code.put_decref_clear(self.result(), self.ctype(), + have_gil=not self.nogil) else: error(self.pos, "Deletion of C names not supported") def annotate(self, code): - if hasattr(self, 'is_called') and self.is_called: + if getattr(self, 'is_called', False): pos = (self.pos[0], self.pos[1], self.pos[2] - len(self.name) - 1) if self.type.is_pyobject: style, text = 'py_call', 'python function (%s)' @@ -2520,7 +2622,7 @@ class BackquoteNode(ExprNode): self.result(), self.arg.py_result(), code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) class ImportNode(ExprNode): @@ -2539,44 +2641,61 @@ class ImportNode(ExprNode): # relative to the current module. # None: decide the level according to language level and # directives + # get_top_level_module int true: return top-level module, false: return imported module + # module_names TupleNode the separate names of the module and submodules, or None type = py_object_type + module_names = None + get_top_level_module = False + is_temp = True - subexprs = ['module_name', 'name_list'] + subexprs = ['module_name', 'name_list', 'module_names'] def analyse_types(self, env): if self.level is None: - if (env.directives['py2_import'] or - Future.absolute_import not in env.global_scope().context.future_directives): + # For modules in packages, and without 'absolute_import' enabled, try relative (Py2) import first. + if env.global_scope().parent_module and ( + env.directives['py2_import'] or + Future.absolute_import not in env.global_scope().context.future_directives): self.level = -1 else: self.level = 0 module_name = self.module_name.analyse_types(env) self.module_name = module_name.coerce_to_pyobject(env) + assert self.module_name.is_string_literal if self.name_list: name_list = self.name_list.analyse_types(env) self.name_list = name_list.coerce_to_pyobject(env) - self.is_temp = 1 + elif '.' in self.module_name.value: + self.module_names = TupleNode(self.module_name.pos, args=[ + IdentifierStringNode(self.module_name.pos, value=part, constant_result=part) + for part in map(StringEncoding.EncodedString, self.module_name.value.split('.')) + ]).analyse_types(env) return self gil_message = "Python import" def generate_result_code(self, code): - if self.name_list: - name_list_code = self.name_list.py_result() + assert self.module_name.is_string_literal + module_name = self.module_name.value + + if self.level == 0 and not self.name_list and not self.get_top_level_module: + if self.module_names: + assert self.module_names.is_literal # make sure we create the tuple only once + code.globalstate.use_utility_code(UtilityCode.load_cached("ImportDottedModule", "ImportExport.c")) + import_code = "__Pyx_ImportDottedModule(%s, %s)" % ( + self.module_name.py_result(), + self.module_names.py_result() if self.module_names else 'NULL', + ) else: - name_list_code = "0" - - code.globalstate.use_utility_code(UtilityCode.load_cached("Import", "ImportExport.c")) - import_code = "__Pyx_Import(%s, %s, %d)" % ( - self.module_name.py_result(), - name_list_code, - self.level) + code.globalstate.use_utility_code(UtilityCode.load_cached("Import", "ImportExport.c")) + import_code = "__Pyx_Import(%s, %s, %d)" % ( + self.module_name.py_result(), + self.name_list.py_result() if self.name_list else '0', + self.level) - if (self.level <= 0 and - self.module_name.is_string_literal and - self.module_name.value in utility_code_for_imports): - helper_func, code_name, code_file = utility_code_for_imports[self.module_name.value] + if self.level <= 0 and module_name in utility_code_for_imports: + helper_func, code_name, code_file = utility_code_for_imports[module_name] code.globalstate.use_utility_code(UtilityCode.load_cached(code_name, code_file)) import_code = '%s(%s)' % (helper_func, import_code) @@ -2584,7 +2703,7 @@ class ImportNode(ExprNode): self.result(), import_code, code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) class IteratorNode(ExprNode): @@ -2597,7 +2716,6 @@ class IteratorNode(ExprNode): type = py_object_type iter_func_ptr = None counter_cname = None - cpp_iterator_cname = None reversed = False # currently only used for list/tuple types (see Optimize.py) is_async = False @@ -2610,7 +2728,7 @@ class IteratorNode(ExprNode): # C array iteration will be transformed later on self.type = self.sequence.type elif self.sequence.type.is_cpp_class: - self.analyse_cpp_types(env) + return CppIteratorNode(self.pos, sequence=self.sequence).analyse_types(env) else: self.sequence = self.sequence.coerce_to_pyobject(env) if self.sequence.type in (list_type, tuple_type): @@ -2640,65 +2758,10 @@ class IteratorNode(ExprNode): return sequence_type return py_object_type - def analyse_cpp_types(self, env): - sequence_type = self.sequence.type - if sequence_type.is_ptr: - sequence_type = sequence_type.base_type - begin = sequence_type.scope.lookup("begin") - end = sequence_type.scope.lookup("end") - if (begin is None - or not begin.type.is_cfunction - or begin.type.args): - error(self.pos, "missing begin() on %s" % self.sequence.type) - self.type = error_type - return - if (end is None - or not end.type.is_cfunction - or end.type.args): - error(self.pos, "missing end() on %s" % self.sequence.type) - self.type = error_type - return - iter_type = begin.type.return_type - if iter_type.is_cpp_class: - if env.lookup_operator_for_types( - self.pos, - "!=", - [iter_type, end.type.return_type]) is None: - error(self.pos, "missing operator!= on result of begin() on %s" % self.sequence.type) - self.type = error_type - return - if env.lookup_operator_for_types(self.pos, '++', [iter_type]) is None: - error(self.pos, "missing operator++ on result of begin() on %s" % self.sequence.type) - self.type = error_type - return - if env.lookup_operator_for_types(self.pos, '*', [iter_type]) is None: - error(self.pos, "missing operator* on result of begin() on %s" % self.sequence.type) - self.type = error_type - return - self.type = iter_type - elif iter_type.is_ptr: - if not (iter_type == end.type.return_type): - error(self.pos, "incompatible types for begin() and end()") - self.type = iter_type - else: - error(self.pos, "result type of begin() on %s must be a C++ class or pointer" % self.sequence.type) - self.type = error_type - return - def generate_result_code(self, code): sequence_type = self.sequence.type if sequence_type.is_cpp_class: - if self.sequence.is_name: - # safe: C++ won't allow you to reassign to class references - begin_func = "%s.begin" % self.sequence.result() - else: - sequence_type = PyrexTypes.c_ptr_type(sequence_type) - self.cpp_iterator_cname = code.funcstate.allocate_temp(sequence_type, manage_ref=False) - code.putln("%s = &%s;" % (self.cpp_iterator_cname, self.sequence.result())) - begin_func = "%s->begin" % self.cpp_iterator_cname - # TODO: Limit scope. - code.putln("%s = %s();" % (self.result(), begin_func)) - return + assert False, "Should have been changed to CppIteratorNode" if sequence_type.is_array or sequence_type.is_ptr: raise InternalError("for in carray slice not transformed") @@ -2740,12 +2803,12 @@ class IteratorNode(ExprNode): self.result(), self.sequence.py_result(), code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) # PyObject_GetIter() fails if "tp_iternext" is not set, but the check below # makes it visible to the C compiler that the pointer really isn't NULL, so that # it can distinguish between the special cases and the generic case - code.putln("%s = Py_TYPE(%s)->tp_iternext; %s" % ( + code.putln("%s = __Pyx_PyObject_GetIterNextFunc(%s); %s" % ( self.iter_func_ptr, self.py_result(), code.error_goto_if_null(self.iter_func_ptr, self.pos))) if self.may_be_a_sequence: @@ -2787,28 +2850,14 @@ class IteratorNode(ExprNode): self.counter_cname, inc_dec, code.error_goto_if_null(result_name, self.pos))) - code.put_gotref(result_name) + code.put_gotref(result_name, py_object_type) code.putln("#endif") def generate_iter_next_result_code(self, result_name, code): sequence_type = self.sequence.type if self.reversed: code.putln("if (%s < 0) break;" % self.counter_cname) - if sequence_type.is_cpp_class: - if self.cpp_iterator_cname: - end_func = "%s->end" % self.cpp_iterator_cname - else: - end_func = "%s.end" % self.sequence.result() - # TODO: Cache end() call? - code.putln("if (!(%s != %s())) break;" % ( - self.result(), - end_func)) - code.putln("%s = *%s;" % ( - result_name, - self.result())) - code.putln("++%s;" % self.result()) - return - elif sequence_type is list_type: + if sequence_type is list_type: self.generate_next_sequence_item('List', result_name, code) return elif sequence_type is tuple_type: @@ -2838,7 +2887,7 @@ class IteratorNode(ExprNode): code.putln("}") code.putln("break;") code.putln("}") - code.put_gotref(result_name) + code.put_gotref(result_name, py_object_type) code.putln("}") def free_temps(self, code): @@ -2847,8 +2896,114 @@ class IteratorNode(ExprNode): if self.iter_func_ptr: code.funcstate.release_temp(self.iter_func_ptr) self.iter_func_ptr = None - if self.cpp_iterator_cname: - code.funcstate.release_temp(self.cpp_iterator_cname) + ExprNode.free_temps(self, code) + + +class CppIteratorNode(ExprNode): + # Iteration over a C++ container. + # Created at the analyse_types stage by IteratorNode + cpp_sequence_cname = None + cpp_attribute_op = "." + extra_dereference = "" + is_temp = True + + subexprs = ['sequence'] + + def analyse_types(self, env): + sequence_type = self.sequence.type + if sequence_type.is_ptr: + sequence_type = sequence_type.base_type + begin = sequence_type.scope.lookup("begin") + end = sequence_type.scope.lookup("end") + if (begin is None + or not begin.type.is_cfunction + or begin.type.args): + error(self.pos, "missing begin() on %s" % self.sequence.type) + self.type = error_type + return self + if (end is None + or not end.type.is_cfunction + or end.type.args): + error(self.pos, "missing end() on %s" % self.sequence.type) + self.type = error_type + return self + iter_type = begin.type.return_type + if iter_type.is_cpp_class: + if env.directives['cpp_locals']: + self.extra_dereference = "*" + if env.lookup_operator_for_types( + self.pos, + "!=", + [iter_type, end.type.return_type]) is None: + error(self.pos, "missing operator!= on result of begin() on %s" % self.sequence.type) + self.type = error_type + return self + if env.lookup_operator_for_types(self.pos, '++', [iter_type]) is None: + error(self.pos, "missing operator++ on result of begin() on %s" % self.sequence.type) + self.type = error_type + return self + if env.lookup_operator_for_types(self.pos, '*', [iter_type]) is None: + error(self.pos, "missing operator* on result of begin() on %s" % self.sequence.type) + self.type = error_type + return self + self.type = iter_type + elif iter_type.is_ptr: + if not (iter_type == end.type.return_type): + error(self.pos, "incompatible types for begin() and end()") + self.type = iter_type + else: + error(self.pos, "result type of begin() on %s must be a C++ class or pointer" % self.sequence.type) + self.type = error_type + return self + + def generate_result_code(self, code): + sequence_type = self.sequence.type + # essentially 3 options: + if self.sequence.is_name or self.sequence.is_attribute: + # 1) is a name and can be accessed directly; + # assigning to it may break the container, but that's the responsibility + # of the user + code.putln("%s = %s%sbegin();" % (self.result(), + self.sequence.result(), + self.cpp_attribute_op)) + else: + # (while it'd be nice to limit the scope of the loop temp, it's essentially + # impossible to do while supporting generators) + temp_type = sequence_type + if temp_type.is_reference: + # 2) Sequence is a reference (often obtained by dereferencing a pointer); + # make the temp a pointer so we are not sensitive to users reassigning + # the pointer than it came from + temp_type = PyrexTypes.CPtrType(sequence_type.ref_base_type) + if temp_type.is_ptr or code.globalstate.directives['cpp_locals']: + self.cpp_attribute_op = "->" + # 3) (otherwise) sequence comes from a function call or similar, so we must + # create a temp to store it in + self.cpp_sequence_cname = code.funcstate.allocate_temp(temp_type, manage_ref=False) + code.putln("%s = %s%s;" % (self.cpp_sequence_cname, + "&" if temp_type.is_ptr else "", + self.sequence.move_result_rhs())) + code.putln("%s = %s%sbegin();" % (self.result(), self.cpp_sequence_cname, + self.cpp_attribute_op)) + + def generate_iter_next_result_code(self, result_name, code): + # end call isn't cached to support containers that allow adding while iterating + # (much as this is usually a bad idea) + code.putln("if (!(%s%s != %s%send())) break;" % ( + self.extra_dereference, + self.result(), + self.cpp_sequence_cname or self.sequence.result(), + self.cpp_attribute_op)) + code.putln("%s = *%s%s;" % ( + result_name, + self.extra_dereference, + self.result())) + code.putln("++%s%s;" % (self.extra_dereference, self.result())) + + def free_temps(self, code): + if self.cpp_sequence_cname: + code.funcstate.release_temp(self.cpp_sequence_cname) + # skip over IteratorNode since we don't use any of the temps it does ExprNode.free_temps(self, code) @@ -2880,8 +3035,8 @@ class NextNode(AtomicExprNode): item_type = env.lookup_operator_for_types(self.pos, "*", [iterator_type]).type.return_type if item_type.is_reference: item_type = item_type.ref_base_type - if item_type.is_const: - item_type = item_type.const_base_type + if item_type.is_cv_qualified: + item_type = item_type.cv_base_type return item_type else: # Avoid duplication of complicated logic. @@ -2930,7 +3085,7 @@ class AsyncIteratorNode(ExprNode): self.result(), self.sequence.py_result(), code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.result()) + self.generate_gotref(code) class AsyncNextNode(AtomicExprNode): @@ -2960,7 +3115,7 @@ class AsyncNextNode(AtomicExprNode): self.result(), self.iterator.py_result(), code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.result()) + self.generate_gotref(code) class WithExitCallNode(ExprNode): @@ -3003,7 +3158,7 @@ class WithExitCallNode(ExprNode): self.args.free_temps(code) code.putln(code.error_goto_if_null(result_var, self.pos)) - code.put_gotref(result_var) + code.put_gotref(result_var, py_object_type) if self.await_expr: # FIXME: result_var temp currently leaks into the closure @@ -3068,7 +3223,7 @@ class TempNode(ExprNode): return self def analyse_target_declaration(self, env): - pass + self.is_target = True def generate_result_code(self, code): pass @@ -3135,6 +3290,7 @@ class JoinedStrNode(ExprNode): # type = unicode_type is_temp = True + gil_message = "String concatenation" subexprs = ['values'] @@ -3157,7 +3313,7 @@ class JoinedStrNode(ExprNode): list_var, num_items, code.error_goto_if_null(list_var, self.pos))) - code.put_gotref(list_var) + code.put_gotref(list_var, py_object_type) code.putln("%s = 0;" % ulength_var) code.putln("%s = 127;" % max_char_var) # at least ASCII character range @@ -3201,7 +3357,7 @@ class JoinedStrNode(ExprNode): max_char_var, max_char_value, max_char_var, max_char_value, max_char_var)) code.putln("%s += %s;" % (ulength_var, ulength)) - code.put_giveref(node.py_result()) + node.generate_giveref(code) code.putln('PyTuple_SET_ITEM(%s, %s, %s);' % (list_var, i, node.py_result())) node.generate_post_assignment_code(code) node.free_temps(code) @@ -3216,7 +3372,7 @@ class JoinedStrNode(ExprNode): ulength_var, max_char_var, code.error_goto_if_null(self.py_result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) code.put_decref_clear(list_var, py_object_type) code.funcstate.release_temp(list_var) @@ -3228,7 +3384,7 @@ class FormattedValueNode(ExprNode): # {}-delimited portions of an f-string # # value ExprNode The expression itself - # conversion_char str or None Type conversion (!s, !r, !a, or none, or 'd' for integer conversion) + # conversion_char str or None Type conversion (!s, !r, !a, none, or 'd' for integer conversion) # format_spec JoinedStrNode or None Format string passed to __format__ # c_format_spec str or None If not None, formatting can be done at the C level @@ -3237,6 +3393,7 @@ class FormattedValueNode(ExprNode): type = unicode_type is_temp = True c_format_spec = None + gil_message = "String formatting" find_conversion_func = { 's': 'PyObject_Unicode', @@ -3274,7 +3431,7 @@ class FormattedValueNode(ExprNode): self.result(), convert_func_call, code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) return value_result = self.value.py_result() @@ -3313,7 +3470,7 @@ class FormattedValueNode(ExprNode): value_result, format_spec, code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) #------------------------------------------------------------------- @@ -3351,7 +3508,7 @@ class ParallelThreadsAvailableNode(AtomicExprNode): return self.temp_code -class ParallelThreadIdNode(AtomicExprNode): #, Nodes.ParallelNode): +class ParallelThreadIdNode(AtomicExprNode): #, Nodes.ParallelNode): """ Implements cython.parallel.threadid() """ @@ -3507,6 +3664,8 @@ class IndexNode(_IndexingBaseNode): bytearray_type, list_type, tuple_type): # slicing these returns the same type return base_type + elif base_type.is_memoryviewslice: + return base_type else: # TODO: Handle buffers (hopefully without too much redundancy). return py_object_type @@ -3549,6 +3708,23 @@ class IndexNode(_IndexingBaseNode): index += base_type.size if 0 <= index < base_type.size: return base_type.components[index] + elif base_type.is_memoryviewslice: + if base_type.ndim == 0: + pass # probably an error, but definitely don't know what to do - return pyobject for now + if base_type.ndim == 1: + return base_type.dtype + else: + return PyrexTypes.MemoryViewSliceType(base_type.dtype, base_type.axes[1:]) + + if self.index.is_sequence_constructor and base_type.is_memoryviewslice: + inferred_type = base_type + for a in self.index.args: + if not inferred_type.is_memoryviewslice: + break # something's gone wrong + inferred_type = IndexNode(self.pos, base=ExprNode(self.base.pos, type=inferred_type), + index=a).infer_type(env) + else: + return inferred_type if base_type.is_cpp_class: class FakeOperand: @@ -3626,6 +3802,8 @@ class IndexNode(_IndexingBaseNode): if not base_type.is_cfunction: self.index = self.index.analyse_types(env) self.original_index_type = self.index.type + if self.original_index_type.is_reference: + self.original_index_type = self.original_index_type.ref_base_type if base_type.is_unicode_char: # we infer Py_UNICODE/Py_UCS4 for unicode strings in some @@ -3711,6 +3889,8 @@ class IndexNode(_IndexingBaseNode): def analyse_as_c_array(self, env, is_slice): base_type = self.base.type self.type = base_type.base_type + if self.type.is_cpp_class: + self.type = PyrexTypes.CReferenceType(self.type) if is_slice: self.type = base_type elif self.index.type.is_pyobject: @@ -3735,7 +3915,7 @@ class IndexNode(_IndexingBaseNode): if self.exception_check: if not setting: self.is_temp = True - if self.exception_value is None: + if needs_cpp_exception_conversion(self): env.use_utility_code(UtilityCode.load_cached("CppExceptionConversion", "CppSupport.cpp")) self.index = self.index.coerce_to(func_type.args[0].type, env) self.type = func_type.return_type @@ -4060,7 +4240,7 @@ class IndexNode(_IndexingBaseNode): self.extra_index_params(code), code.error_goto_if(error_check % self.result(), self.pos))) if self.type.is_pyobject: - code.put_gotref(self.py_result()) + self.generate_gotref(code) def generate_setitem_code(self, value_code, code): if self.index.type.is_int: @@ -4096,7 +4276,7 @@ class IndexNode(_IndexingBaseNode): self.pos)) def generate_assignment_code(self, rhs, code, overloaded_assignment=False, - exception_check=None, exception_value=None): + exception_check=None, exception_value=None): self.generate_subexpr_evaluation_code(code) if self.type.is_pyobject: @@ -4105,8 +4285,7 @@ class IndexNode(_IndexingBaseNode): value_code = self._check_byte_value(code, rhs) self.generate_setitem_code(value_code, code) elif self.base.type.is_cpp_class and self.exception_check and self.exception_check == '+': - if overloaded_assignment and exception_check and \ - self.exception_value != exception_value: + if overloaded_assignment and exception_check and self.exception_value != exception_value: # Handle the case that both the index operator and the assignment # operator have a c++ exception handler and they are not the same. translate_double_cpp_exception(code, self.pos, self.type, @@ -4353,11 +4532,11 @@ class BufferIndexNode(_IndexingBaseNode): manage_ref=False) rhs_code = rhs.result() code.putln("%s = %s;" % (ptr, ptrexpr)) - code.put_gotref("*%s" % ptr) + code.put_gotref("*%s" % ptr, self.buffer_type.dtype) code.putln("__Pyx_INCREF(%s); __Pyx_DECREF(*%s);" % ( rhs_code, ptr)) code.putln("*%s %s= %s;" % (ptr, op, rhs_code)) - code.put_giveref("*%s" % ptr) + code.put_giveref("*%s" % ptr, self.buffer_type.dtype) code.funcstate.release_temp(ptr) else: # Simple case @@ -4620,7 +4799,7 @@ class MemoryViewSliceNode(MemoryViewIndexNode): assert not list(it) buffer_entry.generate_buffer_slice_code( - code, self.original_indices, self.result(), + code, self.original_indices, self.result(), self.type, have_gil=have_gil, have_slices=have_slices, directives=code.globalstate.directives) @@ -4723,8 +4902,17 @@ class MemoryCopyScalar(MemoryCopyNode): code.putln("%s __pyx_temp_slice = %s;" % (slice_decl, self.dst.result())) dst_temp = "__pyx_temp_slice" + force_strided = False + indices = self.dst.original_indices + for idx in indices: + if isinstance(idx, SliceNode) and not (idx.start.is_none and + idx.stop.is_none and + idx.step.is_none): + force_strided = True + slice_iter_obj = MemoryView.slice_iter(self.dst.type, dst_temp, - self.dst.type.ndim, code) + self.dst.type.ndim, code, + force_strided=force_strided) p = slice_iter_obj.start_loops() if dtype.is_pyobject: @@ -5057,15 +5245,14 @@ class SliceIndexNode(ExprNode): start_code, stop_code, code.error_goto_if_null(result, self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) def generate_assignment_code(self, rhs, code, overloaded_assignment=False, - exception_check=None, exception_value=None): + exception_check=None, exception_value=None): self.generate_subexpr_evaluation_code(code) if self.type.is_pyobject: code.globalstate.use_utility_code(self.set_slice_utility_code) - (has_c_start, has_c_stop, c_start, c_stop, - py_start, py_stop, py_slice) = self.get_slice_config() + has_c_start, has_c_stop, c_start, c_stop, py_start, py_stop, py_slice = self.get_slice_config() code.put_error_if_neg(self.pos, "__Pyx_PyObject_SetSlice(%s, %s, %s, %s, %s, %s, %s, %d, %d, %d)" % ( self.base.py_result(), @@ -5292,9 +5479,9 @@ class SliceNode(ExprNode): self.stop.py_result(), self.step.py_result(), code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) if self.is_literal: - code.put_giveref(self.py_result()) + self.generate_giveref(code) class SliceIntNode(SliceNode): # start:stop:step in subscript list @@ -5391,6 +5578,9 @@ class CallNode(ExprNode): return PyrexTypes.c_double_type elif function.entry.name in Builtin.types_that_construct_their_instance: return result_type + func_type = self.function.analyse_as_type(env) + if func_type and (func_type.is_struct_or_union or func_type.is_cpp_class): + return func_type return py_object_type def type_dependencies(self, env): @@ -5516,6 +5706,16 @@ class SimpleCallNode(CallNode): except Exception as e: self.compile_time_value_error(e) + @classmethod + def for_cproperty(cls, pos, obj, entry): + # Create a call node for C property access. + property_scope = entry.scope + getter_entry = property_scope.lookup_here(entry.name) + assert getter_entry, "Getter not found in scope %s: %s" % (property_scope, property_scope.entries) + function = NameNode(pos, name=entry.name, entry=getter_entry, type=getter_entry.type) + node = cls(pos, function=function, args=[obj]) + return node + def analyse_as_type(self, env): attr = self.function.as_cython_attribute() if attr == 'pointer': @@ -5581,6 +5781,7 @@ class SimpleCallNode(CallNode): self.analyse_c_function_call(env) if func_type.exception_check == '+': self.is_temp = True + return self def function_type(self): @@ -5632,8 +5833,8 @@ class SimpleCallNode(CallNode): else: alternatives = overloaded_entry.all_alternatives() - entry = PyrexTypes.best_match( - [arg.type for arg in args], alternatives, self.pos, env, args) + entry = PyrexTypes.best_match([arg.type for arg in args], + alternatives, self.pos, env, args) if not entry: self.type = PyrexTypes.error_type @@ -5716,7 +5917,7 @@ class SimpleCallNode(CallNode): # but we must make sure it cannot be collected # before we return from the function, so we create # an owned temp reference to it - if i > 0: # first argument doesn't matter + if i > 0: # first argument doesn't matter some_args_in_temps = True arg = arg.coerce_to_temp(env) args[i] = arg @@ -5745,7 +5946,7 @@ class SimpleCallNode(CallNode): # case) for i in range(actual_nargs-1): if i == 0 and self.self is not None: - continue # self is ok + continue # self is ok arg = args[i] if arg.nonlocally_immutable(): # locals, C functions, unassignable types are safe. @@ -5761,7 +5962,7 @@ class SimpleCallNode(CallNode): else: #self.args[i] = arg.coerce_to_temp(env) # instead: issue a warning - if i > 0 or i == 1 and self.self is not None: # skip first arg + if i > 0 or i == 1 and self.self is not None: # skip first arg warning(arg.pos, "Argument evaluation order in C function call is undefined and may not be as expected", 0) break @@ -5792,13 +5993,9 @@ class SimpleCallNode(CallNode): # Called in 'nogil' context? self.nogil = env.nogil - if (self.nogil and - func_type.exception_check and - func_type.exception_check != '+'): - env.use_utility_code(pyerr_occurred_withgil_utility_code) # C++ exception handler if func_type.exception_check == '+': - if func_type.exception_value is None: + if needs_cpp_exception_conversion(func_type): env.use_utility_code(UtilityCode.load_cached("CppExceptionConversion", "CppSupport.cpp")) self.overflowcheck = env.directives['overflowcheck'] @@ -5817,8 +6014,8 @@ class SimpleCallNode(CallNode): expected_nargs = max_nargs - func_type.optional_arg_count actual_nargs = len(self.args) for formal_arg, actual_arg in args[:expected_nargs]: - arg_code = actual_arg.result_as(formal_arg.type) - arg_list_code.append(arg_code) + arg_code = actual_arg.move_result_rhs_as(formal_arg.type) + arg_list_code.append(arg_code) if func_type.is_overridable: arg_list_code.append(str(int(self.wrapper_call or self.function.entry.is_unbound_cmethod))) @@ -5831,7 +6028,7 @@ class SimpleCallNode(CallNode): arg_list_code.append(optional_args) for actual_arg in self.args[len(formal_args):]: - arg_list_code.append(actual_arg.result()) + arg_list_code.append(actual_arg.move_result_rhs()) result = "%s(%s)" % (self.function.result(), ', '.join(arg_list_code)) return result @@ -5892,7 +6089,7 @@ class SimpleCallNode(CallNode): arg.py_result(), code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) for subexpr in subexprs: if subexpr is not None: @@ -5911,7 +6108,7 @@ class SimpleCallNode(CallNode): self.function.py_result(), arg_code, code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) elif func_type.is_cfunction: if self.has_optional_args: actual_nargs = len(self.args) @@ -5941,6 +6138,8 @@ class SimpleCallNode(CallNode): exc_checks.append("%s == %s" % (self.result(), func_type.return_type.cast_code(exc_val))) if exc_check: if self.nogil: + code.globalstate.use_utility_code( + UtilityCode.load_cached("ErrOccurredWithGIL", "Exceptions.c")) exc_checks.append("__Pyx_ErrOccurredWithGIL()") else: exc_checks.append("PyErr_Occurred()") @@ -5966,7 +6165,7 @@ class SimpleCallNode(CallNode): goto_error = "" code.putln("%s%s; %s" % (lhs, rhs, goto_error)) if self.type.is_pyobject and self.result(): - code.put_gotref(self.py_result()) + self.generate_gotref(code) if self.has_optional_args: code.funcstate.release_temp(self.opt_arg_struct) @@ -6032,10 +6231,8 @@ class PyMethodCallNode(SimpleCallNode): self_arg = code.funcstate.allocate_temp(py_object_type, manage_ref=True) code.putln("%s = NULL;" % self_arg) - arg_offset_cname = None - if len(args) > 1: - arg_offset_cname = code.funcstate.allocate_temp(PyrexTypes.c_int_type, manage_ref=False) - code.putln("%s = 0;" % arg_offset_cname) + arg_offset_cname = code.funcstate.allocate_temp(PyrexTypes.c_int_type, manage_ref=False) + code.putln("%s = 0;" % arg_offset_cname) def attribute_is_likely_method(attr): obj = attr.obj @@ -6067,116 +6264,35 @@ class PyMethodCallNode(SimpleCallNode): code.put_incref(self_arg, py_object_type) code.put_incref("function", py_object_type) # free method object as early to possible to enable reuse from CPython's freelist - code.put_decref_set(function, "function") - if len(args) > 1: - code.putln("%s = 1;" % arg_offset_cname) + code.put_decref_set(function, py_object_type, "function") + code.putln("%s = 1;" % arg_offset_cname) code.putln("}") code.putln("}") - if not args: - # fastest special case: try to avoid tuple creation - code.globalstate.use_utility_code( - UtilityCode.load_cached("PyObjectCallNoArg", "ObjectHandling.c")) - code.globalstate.use_utility_code( - UtilityCode.load_cached("PyObjectCallOneArg", "ObjectHandling.c")) - code.putln( - "%s = (%s) ? __Pyx_PyObject_CallOneArg(%s, %s) : __Pyx_PyObject_CallNoArg(%s);" % ( - self.result(), self_arg, - function, self_arg, - function)) - code.put_xdecref_clear(self_arg, py_object_type) - code.funcstate.release_temp(self_arg) - code.putln(code.error_goto_if_null(self.result(), self.pos)) - code.put_gotref(self.py_result()) - elif len(args) == 1: - # fastest special case: try to avoid tuple creation - code.globalstate.use_utility_code( - UtilityCode.load_cached("PyObjectCall2Args", "ObjectHandling.c")) - code.globalstate.use_utility_code( - UtilityCode.load_cached("PyObjectCallOneArg", "ObjectHandling.c")) - arg = args[0] - code.putln( - "%s = (%s) ? __Pyx_PyObject_Call2Args(%s, %s, %s) : __Pyx_PyObject_CallOneArg(%s, %s);" % ( - self.result(), self_arg, - function, self_arg, arg.py_result(), - function, arg.py_result())) - code.put_xdecref_clear(self_arg, py_object_type) - code.funcstate.release_temp(self_arg) - arg.generate_disposal_code(code) - arg.free_temps(code) - code.putln(code.error_goto_if_null(self.result(), self.pos)) - code.put_gotref(self.py_result()) - else: - code.globalstate.use_utility_code( - UtilityCode.load_cached("PyFunctionFastCall", "ObjectHandling.c")) - code.globalstate.use_utility_code( - UtilityCode.load_cached("PyCFunctionFastCall", "ObjectHandling.c")) - for test_func, call_prefix in [('PyFunction_Check', 'Py'), ('__Pyx_PyFastCFunction_Check', 'PyC')]: - code.putln("#if CYTHON_FAST_%sCALL" % call_prefix.upper()) - code.putln("if (%s(%s)) {" % (test_func, function)) - code.putln("PyObject *%s[%d] = {%s, %s};" % ( - Naming.quick_temp_cname, - len(args)+1, - self_arg, - ', '.join(arg.py_result() for arg in args))) - code.putln("%s = __Pyx_%sFunction_FastCall(%s, %s+1-%s, %d+%s); %s" % ( - self.result(), - call_prefix, - function, - Naming.quick_temp_cname, - arg_offset_cname, - len(args), - arg_offset_cname, - code.error_goto_if_null(self.result(), self.pos))) - code.put_xdecref_clear(self_arg, py_object_type) - code.put_gotref(self.py_result()) - for arg in args: - arg.generate_disposal_code(code) - code.putln("} else") - code.putln("#endif") - - code.putln("{") - args_tuple = code.funcstate.allocate_temp(py_object_type, manage_ref=True) - code.putln("%s = PyTuple_New(%d+%s); %s" % ( - args_tuple, len(args), arg_offset_cname, - code.error_goto_if_null(args_tuple, self.pos))) - code.put_gotref(args_tuple) - - if len(args) > 1: - code.putln("if (%s) {" % self_arg) - code.putln("__Pyx_GIVEREF(%s); PyTuple_SET_ITEM(%s, 0, %s); %s = NULL;" % ( - self_arg, args_tuple, self_arg, self_arg)) # stealing owned ref in this case - code.funcstate.release_temp(self_arg) - if len(args) > 1: - code.putln("}") - - for i, arg in enumerate(args): - arg.make_owned_reference(code) - code.put_giveref(arg.py_result()) - code.putln("PyTuple_SET_ITEM(%s, %d+%s, %s);" % ( - args_tuple, i, arg_offset_cname, arg.py_result())) - if len(args) > 1: - code.funcstate.release_temp(arg_offset_cname) - - for arg in args: - arg.generate_post_assignment_code(code) - arg.free_temps(code) - - code.globalstate.use_utility_code( - UtilityCode.load_cached("PyObjectCall", "ObjectHandling.c")) - code.putln( - "%s = __Pyx_PyObject_Call(%s, %s, NULL); %s" % ( - self.result(), - function, args_tuple, - code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + # actually call the function + code.globalstate.use_utility_code( + UtilityCode.load_cached("PyObjectFastCall", "ObjectHandling.c")) - code.put_decref_clear(args_tuple, py_object_type) - code.funcstate.release_temp(args_tuple) + code.putln("{") + code.putln("PyObject *__pyx_callargs[%d] = {%s, %s};" % ( + len(args)+1, + self_arg, + ', '.join(arg.py_result() for arg in args))) + code.putln("%s = __Pyx_PyObject_FastCall(%s, __pyx_callargs+1-%s, %d+%s);" % ( + self.result(), + function, + arg_offset_cname, + len(args), + arg_offset_cname)) - if len(args) == 1: - code.putln("}") - code.putln("}") # !CYTHON_FAST_PYCALL + code.put_xdecref_clear(self_arg, py_object_type) + code.funcstate.release_temp(self_arg) + code.funcstate.release_temp(arg_offset_cname) + for arg in args: + arg.generate_disposal_code(code) + arg.free_temps(code) + code.putln(code.error_goto_if_null(self.result(), self.pos)) + self.generate_gotref(code) if reuse_function_temp: self.function.generate_disposal_code(code) @@ -6184,6 +6300,7 @@ class PyMethodCallNode(SimpleCallNode): else: code.put_decref_clear(function, py_object_type) code.funcstate.release_temp(function) + code.putln("}") class InlinedDefNodeCallNode(CallNode): @@ -6234,7 +6351,7 @@ class InlinedDefNodeCallNode(CallNode): # but we must make sure it cannot be collected # before we return from the function, so we create # an owned temp reference to it - if i > 0: # first argument doesn't matter + if i > 0: # first argument doesn't matter some_args_in_temps = True arg = arg.coerce_to_temp(env) self.args[i] = arg @@ -6281,7 +6398,7 @@ class InlinedDefNodeCallNode(CallNode): self.function.def_node.entry.pyfunc_cname, arg_code, code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) class PythonCapiFunctionNode(ExprNode): @@ -6351,7 +6468,7 @@ class CachedBuiltinMethodCallNode(CallNode): self.result(), call_code, code.error_goto_if_null(self.result(), self.pos) )) - code.put_gotref(self.result()) + self.generate_gotref(code) class GeneralCallNode(CallNode): @@ -6442,7 +6559,7 @@ class GeneralCallNode(CallNode): kwargs = self.keyword_args declared_args = function_type.args if entry.is_cmethod: - declared_args = declared_args[1:] # skip 'self' + declared_args = declared_args[1:] # skip 'self' if len(pos_args) > len(declared_args): error(self.pos, "function call got too many positional arguments, " @@ -6450,8 +6567,10 @@ class GeneralCallNode(CallNode): len(pos_args))) return None - matched_args = set([ arg.name for arg in declared_args[:len(pos_args)] - if arg.name ]) + matched_args = { + arg.name for arg in declared_args[:len(pos_args)] + if arg.name + } unmatched_args = declared_args[len(pos_args):] matched_kwargs_count = 0 args = list(pos_args) @@ -6569,7 +6688,7 @@ class GeneralCallNode(CallNode): self.positional_args.py_result(), kwargs, code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) class AsTupleNode(ExprNode): @@ -6611,7 +6730,7 @@ class AsTupleNode(ExprNode): self.result(), cfunc, self.arg.py_result(), code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) class MergedDictNode(ExprNode): @@ -6704,16 +6823,18 @@ class MergedDictNode(ExprNode): self.result(), item.py_result(), code.error_goto_if_null(self.result(), item.pos))) - code.put_gotref(self.result()) + self.generate_gotref(code) item.generate_disposal_code(code) if item.type is not dict_type: code.putln('} else {') - code.putln("%s = PyObject_CallFunctionObjArgs((PyObject*)&PyDict_Type, %s, NULL); %s" % ( + code.globalstate.use_utility_code(UtilityCode.load_cached( + "PyObjectCallOneArg", "ObjectHandling.c")) + code.putln("%s = __Pyx_PyObject_CallOneArg((PyObject*)&PyDict_Type, %s); %s" % ( self.result(), item.py_result(), code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) item.generate_disposal_code(code) code.putln('}') item.free_temps(code) @@ -6785,7 +6906,6 @@ class AttributeNode(ExprNode): is_attribute = 1 subexprs = ['obj'] - type = PyrexTypes.error_type entry = None is_called = 0 needs_none_check = True @@ -6864,7 +6984,7 @@ class AttributeNode(ExprNode): return self.type def analyse_target_declaration(self, env): - pass + self.is_target = True def analyse_target_types(self, env): node = self.analyse_types(env, target = 1) @@ -6875,6 +6995,8 @@ class AttributeNode(ExprNode): return node def analyse_types(self, env, target = 0): + if not self.type: + self.type = PyrexTypes.error_type # default value if it isn't analysed successfully self.initialized_check = env.directives['initializedcheck'] node = self.analyse_as_cimported_attribute_node(env, target) if node is None and not target: @@ -6882,7 +7004,7 @@ class AttributeNode(ExprNode): if node is None: node = self.analyse_as_ordinary_attribute_node(env, target) assert node is not None - if node.entry: + if (node.is_attribute or node.is_name) and node.entry: node.entry.used = True if node.is_attribute: node.wrap_obj_in_nonecheck(env) @@ -6901,6 +7023,7 @@ class AttributeNode(ExprNode): or entry.is_type or entry.is_const): return self.as_name_node(env, entry, target) if self.is_cimported_module_without_shadow(env): + # TODO: search for submodule error(self.pos, "cimported module has no attribute '%s'" % self.attribute) return self return None @@ -6923,31 +7046,13 @@ class AttributeNode(ExprNode): return None ubcm_entry = entry else: - # Create a temporary entry describing the C method - # as an ordinary function. - if entry.func_cname and not hasattr(entry.type, 'op_arg_struct'): - cname = entry.func_cname - if entry.type.is_static_method or ( - env.parent_scope and env.parent_scope.is_cpp_class_scope): - ctype = entry.type - elif type.is_cpp_class: - error(self.pos, "%s not a static member of %s" % (entry.name, type)) - ctype = PyrexTypes.error_type - else: - # Fix self type. - ctype = copy.copy(entry.type) - ctype.args = ctype.args[:] - ctype.args[0] = PyrexTypes.CFuncTypeArg('self', type, 'self', None) - else: - cname = "%s->%s" % (type.vtabptr_cname, entry.cname) - ctype = entry.type - ubcm_entry = Symtab.Entry(entry.name, cname, ctype) - ubcm_entry.is_cfunction = 1 - ubcm_entry.func_cname = entry.func_cname - ubcm_entry.is_unbound_cmethod = 1 - ubcm_entry.scope = entry.scope + ubcm_entry = self._create_unbound_cmethod_entry(type, entry, env) + ubcm_entry.overloaded_alternatives = [ + self._create_unbound_cmethod_entry(type, overloaded_alternative, env) + for overloaded_alternative in entry.overloaded_alternatives + ] return self.as_name_node(env, ubcm_entry, target=False) - elif type.is_enum: + elif type.is_enum or type.is_cpp_enum: if self.attribute in type.values: for entry in type.entry.enum_values: if entry.name == self.attribute: @@ -6958,13 +7063,39 @@ class AttributeNode(ExprNode): error(self.pos, "%s not a known value of %s" % (self.attribute, type)) return None + def _create_unbound_cmethod_entry(self, type, entry, env): + # Create a temporary entry describing the unbound C method in `entry` + # as an ordinary function. + if entry.func_cname and entry.type.op_arg_struct is None: + cname = entry.func_cname + if entry.type.is_static_method or ( + env.parent_scope and env.parent_scope.is_cpp_class_scope): + ctype = entry.type + elif type.is_cpp_class: + error(self.pos, "%s not a static member of %s" % (entry.name, type)) + ctype = PyrexTypes.error_type + else: + # Fix self type. + ctype = copy.copy(entry.type) + ctype.args = ctype.args[:] + ctype.args[0] = PyrexTypes.CFuncTypeArg('self', type, 'self', None) + else: + cname = "%s->%s" % (type.vtabptr_cname, entry.cname) + ctype = entry.type + ubcm_entry = Symtab.Entry(entry.name, cname, ctype) + ubcm_entry.is_cfunction = 1 + ubcm_entry.func_cname = entry.func_cname + ubcm_entry.is_unbound_cmethod = 1 + ubcm_entry.scope = entry.scope + return ubcm_entry + def analyse_as_type(self, env): module_scope = self.obj.analyse_as_module(env) if module_scope: return module_scope.lookup_type(self.attribute) if not self.obj.is_string_literal: base_type = self.obj.analyse_as_type(env) - if base_type and hasattr(base_type, 'scope') and base_type.scope is not None: + if base_type and getattr(base_type, 'scope', None) is not None: return base_type.scope.lookup_type(self.attribute) return None @@ -7015,13 +7146,18 @@ class AttributeNode(ExprNode): self.result_ctype = py_object_type elif target and self.obj.type.is_builtin_type: error(self.pos, "Assignment to an immutable object field") + elif self.entry and self.entry.is_cproperty: + if not target: + return SimpleCallNode.for_cproperty(self.pos, self.obj, self.entry).analyse_types(env) + # TODO: implement writable C-properties? + error(self.pos, "Assignment to a read-only property") #elif self.type.is_memoryviewslice and not target: # self.is_temp = True return self def analyse_attribute(self, env, obj_type = None): # Look up attribute and set self.type and self.member. - immutable_obj = obj_type is not None # used during type inference + immutable_obj = obj_type is not None # used during type inference self.is_py_attr = 0 self.member = self.attribute if obj_type is None: @@ -7071,7 +7207,10 @@ class AttributeNode(ExprNode): # fused function go through assignment synthesis # (foo = pycfunction(foo_func_obj)) and need to go through # regular Python lookup as well - if (entry.is_variable and not entry.fused_cfunction) or entry.is_cmethod: + if entry.is_cproperty: + self.type = entry.type + return + elif (entry.is_variable and not entry.fused_cfunction) or entry.is_cmethod: self.type = entry.type self.member = entry.cname return @@ -7098,15 +7237,15 @@ class AttributeNode(ExprNode): if not obj_type.is_pyobject and not obj_type.is_error: # Expose python methods for immutable objects. if (obj_type.is_string or obj_type.is_cpp_string - or obj_type.is_buffer or obj_type.is_memoryviewslice - or obj_type.is_numeric - or (obj_type.is_ctuple and obj_type.can_coerce_to_pyobject(env)) - or (obj_type.is_struct and obj_type.can_coerce_to_pyobject(env))): + or obj_type.is_buffer or obj_type.is_memoryviewslice + or obj_type.is_numeric + or (obj_type.is_ctuple and obj_type.can_coerce_to_pyobject(env)) + or (obj_type.is_struct and obj_type.can_coerce_to_pyobject(env))): if not immutable_obj: self.obj = self.obj.coerce_to_pyobject(env) elif (obj_type.is_cfunction and (self.obj.is_name or self.obj.is_attribute) - and self.obj.entry.as_variable - and self.obj.entry.as_variable.type.is_pyobject): + and self.obj.entry.as_variable + and self.obj.entry.as_variable.type.is_pyobject): # might be an optimised builtin function => unpack it if not immutable_obj: self.obj = self.obj.coerce_to_pyobject(env) @@ -7167,9 +7306,14 @@ class AttributeNode(ExprNode): return NameNode.is_ephemeral(self) def calculate_result_code(self): - #print "AttributeNode.calculate_result_code:", self.member ### - #print "...obj node =", self.obj, "code", self.obj.result() ### - #print "...obj type", self.obj.type, "ctype", self.obj.ctype() ### + result = self.calculate_access_code() + if self.entry and self.entry.is_cpp_optional and not self.is_target: + result = "(*%s)" % result + return result + + def calculate_access_code(self): + # Does the job of calculate_result_code but doesn't dereference cpp_optionals + # Therefore allowing access to the holder variable obj = self.obj obj_code = obj.result_as(obj.type) #print "...obj_code =", obj_code ### @@ -7220,7 +7364,7 @@ class AttributeNode(ExprNode): self.obj.py_result(), code.intern_identifier(self.attribute), code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) elif self.type.is_memoryviewslice: if self.is_memslice_transpose: # transpose the slice @@ -7231,10 +7375,11 @@ class AttributeNode(ExprNode): return code.putln("%s = %s;" % (self.result(), self.obj.result())) - code.put_incref_memoryviewslice(self.result(), have_gil=True) + code.put_incref_memoryviewslice(self.result(), self.type, + have_gil=True) - T = "__pyx_memslice_transpose(&%s) == 0" - code.putln(code.error_goto_if(T % self.result(), self.pos)) + T = "__pyx_memslice_transpose(&%s)" % self.result() + code.putln(code.error_goto_if_neg(T, self.pos)) elif self.initialized_check: code.putln( 'if (unlikely(!%s.memview)) {' @@ -7242,6 +7387,14 @@ class AttributeNode(ExprNode): '"Memoryview is not initialized");' '%s' '}' % (self.result(), code.error_goto(self.pos))) + elif self.entry.is_cpp_optional and self.initialized_check: + if self.is_target: + undereferenced_result = self.result() + else: + assert not self.is_temp # calculate_access_code() only makes sense for non-temps + undereferenced_result = self.calculate_access_code() + unbound_check_code = self.type.cpp_optional_check_for_null_code(undereferenced_result) + code.put_error_if_unbound(self.pos, self.entry, unbound_check_code=unbound_check_code) else: # result_code contains what is needed, but we may need to insert # a check and raise an exception @@ -7254,15 +7407,12 @@ class AttributeNode(ExprNode): def generate_disposal_code(self, code): if self.is_temp and self.type.is_memoryviewslice and self.is_memslice_transpose: # mirror condition for putting the memview incref here: - code.put_xdecref_memoryviewslice( - self.result(), have_gil=True) - code.putln("%s.memview = NULL;" % self.result()) - code.putln("%s.data = NULL;" % self.result()) + code.put_xdecref_clear(self.result(), self.type, have_gil=True) else: ExprNode.generate_disposal_code(self, code) def generate_assignment_code(self, rhs, code, overloaded_assignment=False, - exception_check=None, exception_value=None): + exception_check=None, exception_value=None): self.obj.generate_evaluation_code(code) if self.is_py_attr: code.globalstate.use_utility_code( @@ -7285,8 +7435,8 @@ class AttributeNode(ExprNode): select_code = self.result() if self.type.is_pyobject and self.use_managed_ref: rhs.make_owned_reference(code) - code.put_giveref(rhs.py_result()) - code.put_gotref(select_code) + rhs.generate_giveref(code) + code.put_gotref(select_code, self.type) code.put_decref(select_code, self.ctype()) elif self.type.is_memoryviewslice: from . import MemoryView @@ -7297,7 +7447,7 @@ class AttributeNode(ExprNode): code.putln( "%s = %s;" % ( select_code, - rhs.result_as(self.ctype()))) + rhs.move_result_rhs_as(self.ctype()))) #rhs.result())) rhs.generate_post_assignment_code(code) rhs.free_temps(code) @@ -7428,9 +7578,10 @@ class SequenceNode(ExprNode): arg = arg.analyse_types(env) self.args[i] = arg.coerce_to_pyobject(env) if self.mult_factor: - self.mult_factor = self.mult_factor.analyse_types(env) - if not self.mult_factor.type.is_int: - self.mult_factor = self.mult_factor.coerce_to_pyobject(env) + mult_factor = self.mult_factor.analyse_types(env) + if not mult_factor.type.is_int: + mult_factor = mult_factor.coerce_to_pyobject(env) + self.mult_factor = mult_factor.coerce_to_simple(env) self.is_temp = 1 # not setting self.type here, subtypes do this return self @@ -7532,7 +7683,7 @@ class SequenceNode(ExprNode): len(self.args), ', '.join(arg.py_result() for arg in self.args), code.error_goto_if_null(target, self.pos))) - code.put_gotref(target) + code.put_gotref(target, py_object_type) elif self.type.is_ctuple: for i, arg in enumerate(self.args): code.putln("%s.f%s = %s;" % ( @@ -7549,7 +7700,7 @@ class SequenceNode(ExprNode): code.putln("%s = %s(%s%s); %s" % ( target, create_func, arg_count, size_factor, code.error_goto_if_null(target, self.pos))) - code.put_gotref(target) + code.put_gotref(target, py_object_type) if c_mult: # FIXME: can't use a temp variable here as the code may @@ -7573,7 +7724,7 @@ class SequenceNode(ExprNode): arg = self.args[i] if c_mult or not arg.result_in_temp(): code.put_incref(arg.result(), arg.ctype()) - code.put_giveref(arg.py_result()) + arg.generate_giveref(code) code.putln("%s(%s, %s, %s);" % ( set_item_func, target, @@ -7590,7 +7741,7 @@ class SequenceNode(ExprNode): Naming.quick_temp_cname, target, mult_factor.py_result(), code.error_goto_if_null(Naming.quick_temp_cname, self.pos) )) - code.put_gotref(Naming.quick_temp_cname) + code.put_gotref(Naming.quick_temp_cname, py_object_type) code.put_decref(target, py_object_type) code.putln('%s = %s;' % (target, Naming.quick_temp_cname)) code.putln('}') @@ -7612,7 +7763,7 @@ class SequenceNode(ExprNode): self.mult_factor.generate_disposal_code(code) def generate_assignment_code(self, rhs, code, overloaded_assignment=False, - exception_check=None, exception_value=None): + exception_check=None, exception_value=None): if self.starred_assignment: self.generate_starred_assignment_code(rhs, code) else: @@ -7676,10 +7827,12 @@ class SequenceNode(ExprNode): # list/tuple => check size code.putln("Py_ssize_t size = __Pyx_PySequence_SIZE(sequence);") code.putln("if (unlikely(size != %d)) {" % len(self.args)) - code.globalstate.use_utility_code(raise_too_many_values_to_unpack) + code.globalstate.use_utility_code( + UtilityCode.load_cached("RaiseTooManyValuesToUnpack", "ObjectHandling.c")) code.putln("if (size > %d) __Pyx_RaiseTooManyValuesError(%d);" % ( len(self.args), len(self.args))) - code.globalstate.use_utility_code(raise_need_more_values_to_unpack) + code.globalstate.use_utility_code( + UtilityCode.load_cached("RaiseNeedMoreValuesToUnpack", "ObjectHandling.c")) code.putln("else if (size >= 0) __Pyx_RaiseNeedMoreValuesError(size);") # < 0 => exception code.putln(code.error_goto(self.pos)) @@ -7708,7 +7861,7 @@ class SequenceNode(ExprNode): code.putln("%s = PySequence_ITEM(sequence, %d); %s" % ( item.result(), i, code.error_goto_if_null(item.result(), self.pos))) - code.put_gotref(item.result()) + code.put_gotref(item.result(), item.type) else: code.putln("{") code.putln("Py_ssize_t i;") @@ -7718,7 +7871,7 @@ class SequenceNode(ExprNode): code.putln("for (i=0; i < %s; i++) {" % len(self.unpacked_items)) code.putln("PyObject* item = PySequence_ITEM(sequence, i); %s" % ( code.error_goto_if_null('item', self.pos))) - code.put_gotref('item') + code.put_gotref('item', py_object_type) code.putln("*(temps[i]) = item;") code.putln("}") code.putln("}") @@ -7742,9 +7895,11 @@ class SequenceNode(ExprNode): code.putln("}") def generate_generic_parallel_unpacking_code(self, code, rhs, unpacked_items, use_loop, terminate=True): - code.globalstate.use_utility_code(raise_need_more_values_to_unpack) - code.globalstate.use_utility_code(UtilityCode.load_cached("IterFinish", "ObjectHandling.c")) - code.putln("Py_ssize_t index = -1;") # must be at the start of a C block! + code.globalstate.use_utility_code( + UtilityCode.load_cached("RaiseNeedMoreValuesToUnpack", "ObjectHandling.c")) + code.globalstate.use_utility_code( + UtilityCode.load_cached("IterFinish", "ObjectHandling.c")) + code.putln("Py_ssize_t index = -1;") # must be at the start of a C block! if use_loop: code.putln("PyObject** temps[%s] = {%s};" % ( @@ -7757,11 +7912,11 @@ class SequenceNode(ExprNode): iterator_temp, rhs.py_result(), code.error_goto_if_null(iterator_temp, self.pos))) - code.put_gotref(iterator_temp) + code.put_gotref(iterator_temp, py_object_type) rhs.generate_disposal_code(code) iternext_func = code.funcstate.allocate_temp(self._func_iternext_type, manage_ref=False) - code.putln("%s = Py_TYPE(%s)->tp_iternext;" % ( + code.putln("%s = __Pyx_PyObject_GetIterNextFunc(%s);" % ( iternext_func, iterator_temp)) unpacking_error_label = code.new_label('unpacking_failed') @@ -7770,7 +7925,7 @@ class SequenceNode(ExprNode): code.putln("for (index=0; index < %s; index++) {" % len(unpacked_items)) code.put("PyObject* item = %s; if (unlikely(!item)) " % unpack_code) code.put_goto(unpacking_error_label) - code.put_gotref("item") + code.put_gotref("item", py_object_type) code.putln("*(temps[index]) = item;") code.putln("}") else: @@ -7782,7 +7937,7 @@ class SequenceNode(ExprNode): unpack_code, item.result())) code.put_goto(unpacking_error_label) - code.put_gotref(item.py_result()) + item.generate_gotref(code) if terminate: code.globalstate.use_utility_code( @@ -7835,11 +7990,14 @@ class SequenceNode(ExprNode): starred_target.allocate(code) target_list = starred_target.result() - code.putln("%s = PySequence_List(%s); %s" % ( + code.putln("%s = %s(%s); %s" % ( target_list, + "__Pyx_PySequence_ListKeepNew" if ( + not iterator_temp and rhs.is_temp and rhs.type in (py_object_type, list_type)) + else "PySequence_List", iterator_temp or rhs.py_result(), code.error_goto_if_null(target_list, self.pos))) - code.put_gotref(target_list) + starred_target.generate_gotref(code) if iterator_temp: code.put_decref_clear(iterator_temp, py_object_type) @@ -7848,7 +8006,8 @@ class SequenceNode(ExprNode): rhs.generate_disposal_code(code) if unpacked_fixed_items_right: - code.globalstate.use_utility_code(raise_need_more_values_to_unpack) + code.globalstate.use_utility_code( + UtilityCode.load_cached("RaiseNeedMoreValuesToUnpack", "ObjectHandling.c")) length_temp = code.funcstate.allocate_temp(PyrexTypes.c_py_ssize_t_type, manage_ref=False) code.putln('%s = PyList_GET_SIZE(%s);' % (length_temp, target_list)) code.putln("if (unlikely(%s < %d)) {" % (length_temp, len(unpacked_fixed_items_right))) @@ -7870,7 +8029,7 @@ class SequenceNode(ExprNode): code.putln("%s = PySequence_ITEM(%s, %s-%d); " % ( item.py_result(), target_list, length_temp, i+1)) code.putln('#endif') - code.put_gotref(item.py_result()) + item.generate_gotref(code) coerced_arg.generate_evaluation_code(code) code.putln('#if !CYTHON_COMPILING_IN_CPYTHON') @@ -7878,7 +8037,7 @@ class SequenceNode(ExprNode): code.putln('%s = PySequence_GetSlice(%s, 0, %s-%d); %s' % ( sublist_temp, target_list, length_temp, len(unpacked_fixed_items_right), code.error_goto_if_null(sublist_temp, self.pos))) - code.put_gotref(sublist_temp) + code.put_gotref(sublist_temp, py_object_type) code.funcstate.release_temp(length_temp) code.put_decref(target_list, py_object_type) code.putln('%s = %s; %s = NULL;' % (target_list, sublist_temp, sublist_temp)) @@ -8024,7 +8183,7 @@ class TupleNode(SequenceNode): # constant is not yet initialised const_code.mark_pos(self.pos) self.generate_sequence_packing_code(const_code, tuple_target, plain=not self.is_literal) - const_code.put_giveref(tuple_target) + const_code.put_giveref(tuple_target, py_object_type) if self.is_literal: self.result_code = tuple_target else: @@ -8032,7 +8191,7 @@ class TupleNode(SequenceNode): self.result(), tuple_target, self.mult_factor.py_result(), code.error_goto_if_null(self.result(), self.pos) )) - code.put_gotref(self.py_result()) + self.generate_gotref(code) else: self.type.entry.used = True self.generate_sequence_packing_code(code) @@ -8215,7 +8374,7 @@ class ScopedExprNode(ExprNode): if expr_scope is not None: self.expr_scope = expr_scope elif self.has_local_scope: - self.expr_scope = Symtab.GeneratorExpressionScope(outer_scope) + self.expr_scope = Symtab.ComprehensionScope(outer_scope) else: self.expr_scope = None @@ -8282,7 +8441,7 @@ class ScopedExprNode(ExprNode): for entry in py_entries: if entry.is_cglobal: code.put_var_gotref(entry) - code.put_decref_set(entry.cname, "Py_None") + code.put_var_decref_set(entry, "Py_None") else: code.put_var_xdecref_clear(entry) @@ -8299,7 +8458,7 @@ class ComprehensionNode(ScopedExprNode): return self.type def analyse_declarations(self, env): - self.append.target = self # this is used in the PyList_Append of the inner loop + self.append.target = self # this is used in the PyList_Append of the inner loop self.init_scope(env) def analyse_scoped_declarations(self, env): @@ -8334,7 +8493,7 @@ class ComprehensionNode(ScopedExprNode): self.result(), create_code, code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.result()) + self.generate_gotref(code) self.loop.generate_execution_code(code) def annotate(self, code): @@ -8459,7 +8618,7 @@ class InlinedGeneratorExpressionNode(ExprNode): code.putln("%s = __Pyx_Generator_Next(%s); %s" % ( self.result(), self.gen.result(), code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.result()) + self.generate_gotref(code) class MergedSequenceNode(ExprNode): @@ -8567,10 +8726,12 @@ class MergedSequenceNode(ExprNode): else: code.putln("%s = %s(%s); %s" % ( self.result(), - 'PySet_New' if is_set else 'PySequence_List', + 'PySet_New' if is_set + else "__Pyx_PySequence_ListKeepNew" if item.is_temp and item.type in (py_object_type, list_type) + else "PySequence_List", item.py_result(), code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) item.generate_disposal_code(code) item.free_temps(code) @@ -8620,7 +8781,7 @@ class MergedSequenceNode(ExprNode): self.result(), Naming.quick_temp_cname, code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.result()) + self.generate_gotref(code) code.putln("}") for helper in sorted(helpers): @@ -8653,7 +8814,7 @@ class SetNode(ExprNode): return False def calculate_constant_result(self): - self.constant_result = set([arg.constant_result for arg in self.args]) + self.constant_result = {arg.constant_result for arg in self.args} def compile_time_value(self, denv): values = [arg.compile_time_value(denv) for arg in self.args] @@ -8670,7 +8831,7 @@ class SetNode(ExprNode): "%s = PySet_New(0); %s" % ( self.result(), code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) for arg in self.args: code.put_error_if_neg( self.pos, @@ -8757,7 +8918,7 @@ class DictNode(ExprNode): error(item.key.pos, "Invalid struct field identifier") item.key = StringNode(item.key.pos, value="<error>") else: - key = str(item.key.value) # converts string literals to unicode in Py3 + key = str(item.key.value) # converts string literals to unicode in Py3 member = dst_type.scope.lookup_here(key) if not member: error(item.key.pos, "struct '%s' has no field '%s'" % (dst_type, key)) @@ -8767,8 +8928,7 @@ class DictNode(ExprNode): value = value.arg item.value = value.coerce_to(member.type, env) else: - self.type = error_type - error(self.pos, "Cannot interpret dict as type '%s'" % dst_type) + return super(DictNode, self).coerce_to(dst_type, env) return self def release_errors(self): @@ -8792,7 +8952,7 @@ class DictNode(ExprNode): self.result(), len(self.key_value_pairs), code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) keys_seen = set() key_type = None @@ -8864,7 +9024,7 @@ class DictItemNode(ExprNode): # value ExprNode subexprs = ['key', 'value'] - nogil_check = None # Parent DictNode takes care of it + nogil_check = None # Parent DictNode takes care of it def calculate_constant_result(self): self.constant_result = ( @@ -8920,7 +9080,7 @@ class SortedDictKeysNode(ExprNode): code.putln('%s = PyDict_Keys(%s); %s' % ( self.result(), dict_result, code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) else: # originally used PyMapping_Keys() here, but that may return a tuple code.globalstate.use_utility_code(UtilityCode.load_cached( @@ -8929,11 +9089,11 @@ class SortedDictKeysNode(ExprNode): code.putln('%s = __Pyx_PyObject_CallMethod0(%s, %s); %s' % ( self.result(), dict_result, keys_cname, code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) code.putln("if (unlikely(!PyList_Check(%s))) {" % self.result()) - code.put_decref_set(self.result(), "PySequence_List(%s)" % self.result()) + self.generate_decref_set(code, "PySequence_List(%s)" % self.result()) code.putln(code.error_goto_if_null(self.result(), self.pos)) - code.put_gotref(self.py_result()) + self.generate_gotref(code) code.putln("}") code.put_error_if_neg( self.pos, 'PyList_Sort(%s)' % self.py_result()) @@ -8963,6 +9123,9 @@ class ClassNode(ExprNode, ModuleNameMixin): type = py_object_type is_temp = True + def analyse_annotations(self, env): + pass + def infer_type(self, env): # TODO: could return 'type' in some cases return py_object_type @@ -9001,7 +9164,7 @@ class ClassNode(ExprNode, ModuleNameMixin): qualname, py_mod_name, code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) class Py3ClassNode(ExprNode): @@ -9014,9 +9177,11 @@ class Py3ClassNode(ExprNode): # class_def_node PyClassDefNode PyClassDefNode defining this class # calculate_metaclass bool should call CalculateMetaclass() # allow_py2_metaclass bool should look for Py2 metaclass + # force_type bool always create a "new style" class, even with no bases subexprs = [] type = py_object_type + force_type = False is_temp = True def infer_type(self, env): @@ -9031,6 +9196,26 @@ class Py3ClassNode(ExprNode): gil_message = "Constructing Python class" + def analyse_annotations(self, env): + from .AutoDocTransforms import AnnotationWriter + position = self.class_def_node.pos + dict_items = [ + DictItemNode( + entry.pos, + key=IdentifierStringNode(entry.pos, value=entry.name), + value=entry.annotation.string + ) + for entry in env.entries.values() if entry.annotation + ] + # Annotations dict shouldn't exist for classes which don't declare any. + if dict_items: + annotations_dict = DictNode(position, key_value_pairs=dict_items) + lhs = NameNode(position, name=StringEncoding.EncodedString(u"__annotations__")) + lhs.entry = env.lookup_here(lhs.name) or env.declare_var(lhs.name, dict_type, position) + node = SingleAssignmentNode(position, lhs=lhs, rhs=annotations_dict) + node.analyse_declarations(env) + self.class_def_node.body.stats.insert(0, node) + def generate_result_code(self, code): code.globalstate.use_utility_code(UtilityCode.load_cached("Py3ClassCreate", "ObjectHandling.c")) cname = code.intern_identifier(self.name) @@ -9038,6 +9223,8 @@ class Py3ClassNode(ExprNode): mkw = class_def_node.mkw.py_result() if class_def_node.mkw else 'NULL' if class_def_node.metaclass: metaclass = class_def_node.metaclass.py_result() + elif self.force_type: + metaclass = "((PyObject*)&PyType_Type)" else: metaclass = "((PyObject*)&__Pyx_DefaultClassType)" code.putln( @@ -9051,7 +9238,7 @@ class Py3ClassNode(ExprNode): self.calculate_metaclass, self.allow_py2_metaclass, code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) class PyClassMetaclassNode(ExprNode): @@ -9087,7 +9274,7 @@ class PyClassMetaclassNode(ExprNode): "%s = %s; %s" % ( self.result(), call, code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) class PyClassNamespaceNode(ExprNode, ModuleNameMixin): @@ -9129,7 +9316,7 @@ class PyClassNamespaceNode(ExprNode, ModuleNameMixin): py_mod_name, doc_code, code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) class ClassCellInjectorNode(ExprNode): @@ -9148,7 +9335,7 @@ class ClassCellInjectorNode(ExprNode): '%s = PyList_New(0); %s' % ( self.result(), code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.result()) + self.generate_gotref(code) def generate_injection_code(self, code, classobj_cname): assert self.is_active @@ -9190,7 +9377,6 @@ class PyCFunctionNode(ExprNode, ModuleNameMixin): # from a PyMethodDef struct. # # pymethdef_cname string PyMethodDef structure - # self_object ExprNode or None # binding bool # def_node DefNode the Python function node # module_name EncodedString Name of defining module @@ -9199,7 +9385,6 @@ class PyCFunctionNode(ExprNode, ModuleNameMixin): subexprs = ['code_object', 'defaults_tuple', 'defaults_kwdict', 'annotations_dict'] - self_object = None code_object = None binding = False def_node = None @@ -9248,33 +9433,35 @@ class PyCFunctionNode(ExprNode, ModuleNameMixin): must_use_constants = env.is_c_class_scope or (self.def_node.is_wrapper and env.is_module_scope) for arg in self.def_node.args: - if arg.default and not must_use_constants: - if not arg.default.is_literal: - arg.is_dynamic = True - if arg.type.is_pyobject: - nonliteral_objects.append(arg) + if arg.default: + if not must_use_constants: + if arg.default.is_literal: + arg.default = DefaultLiteralArgNode(arg.pos, arg.default) else: - nonliteral_other.append(arg) - else: - arg.default = DefaultLiteralArgNode(arg.pos, arg.default) - if arg.kw_only: - default_kwargs.append(arg) - else: - default_args.append(arg) + arg.is_dynamic = True + if arg.type.is_pyobject: + nonliteral_objects.append(arg) + else: + nonliteral_other.append(arg) + if arg.default.type and arg.default.type.can_coerce_to_pyobject(env): + if arg.kw_only: + default_kwargs.append(arg) + else: + default_args.append(arg) if arg.annotation: - arg.annotation = self.analyse_annotation(env, arg.annotation) - annotations.append((arg.pos, arg.name, arg.annotation)) + arg.annotation = arg.annotation.analyse_types(env) + annotations.append((arg.pos, arg.name, arg.annotation.string)) for arg in (self.def_node.star_arg, self.def_node.starstar_arg): if arg and arg.annotation: - arg.annotation = self.analyse_annotation(env, arg.annotation) - annotations.append((arg.pos, arg.name, arg.annotation)) + arg.annotation = arg.annotation.analyse_types(env) + annotations.append((arg.pos, arg.name, arg.annotation.string)) annotation = self.def_node.return_type_annotation if annotation: - annotation = self.analyse_annotation(env, annotation) - self.def_node.return_type_annotation = annotation - annotations.append((annotation.pos, StringEncoding.EncodedString("return"), annotation)) + self.def_node.return_type_annotation = annotation.analyse_types(env) + annotations.append((annotation.pos, StringEncoding.EncodedString("return"), + annotation.string)) if nonliteral_objects or nonliteral_other: module_scope = env.global_scope() @@ -9282,7 +9469,10 @@ class PyCFunctionNode(ExprNode, ModuleNameMixin): scope = Symtab.StructOrUnionScope(cname) self.defaults = [] for arg in nonliteral_objects: - entry = scope.declare_var(arg.name, arg.type, None, + type_ = arg.type + if type_.is_buffer: + type_ = type_.base + entry = scope.declare_var(arg.name, type_, None, Naming.arg_prefix + arg.name, allow_pyobject=True) self.defaults.append((arg, entry)) @@ -9314,7 +9504,8 @@ class PyCFunctionNode(ExprNode, ModuleNameMixin): value=arg.default) for arg in default_kwargs]) self.defaults_kwdict = defaults_kwdict.analyse_types(env) - else: + elif not self.specialized_cpdefs: + # Fused dispatch functions do not support (dynamic) default arguments, only the specialisations do. if default_args: defaults_tuple = DefaultsTupleNode( self.pos, default_args, self.defaults_struct) @@ -9351,31 +9542,13 @@ class PyCFunctionNode(ExprNode, ModuleNameMixin): for pos, name, value in annotations]) self.annotations_dict = annotations_dict.analyse_types(env) - def analyse_annotation(self, env, annotation): - if annotation is None: - return None - atype = annotation.analyse_as_type(env) - if atype is not None: - # Keep parsed types as strings as they might not be Python representable. - annotation = UnicodeNode( - annotation.pos, - value=StringEncoding.EncodedString(atype.declaration_code('', for_display=True))) - annotation = annotation.analyse_types(env) - if not annotation.type.is_pyobject: - annotation = annotation.coerce_to_pyobject(env) - return annotation - def may_be_none(self): return False gil_message = "Constructing Python function" - def self_result_code(self): - if self.self_object is None: - self_result = "NULL" - else: - self_result = self.self_object.py_result() - return self_result + def closure_result_code(self): + return "NULL" def generate_result_code(self, code): if self.binding: @@ -9389,11 +9562,11 @@ class PyCFunctionNode(ExprNode, ModuleNameMixin): '%s = PyCFunction_NewEx(&%s, %s, %s); %s' % ( self.result(), self.pymethdef_cname, - self.self_result_code(), + self.closure_result_code(), py_mod_name, code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) def generate_cyfunction_code(self, code): if self.specialized_cpdefs: @@ -9424,6 +9597,9 @@ class PyCFunctionNode(ExprNode, ModuleNameMixin): if def_node.local_scope.parent_scope.is_c_class_scope and not def_node.entry.is_anonymous: flags.append('__Pyx_CYFUNCTION_CCLASS') + if def_node.is_coroutine: + flags.append('__Pyx_CYFUNCTION_COROUTINE') + if flags: flags = ' | '.join(flags) else: @@ -9436,13 +9612,13 @@ class PyCFunctionNode(ExprNode, ModuleNameMixin): self.pymethdef_cname, flags, self.get_py_qualified_name(code), - self.self_result_code(), + self.closure_result_code(), self.get_py_mod_name(code), Naming.moddict_cname, code_object_result, code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) if def_node.requires_classobj: assert code.pyclass_stack, "pyclass_stack is empty" @@ -9452,7 +9628,7 @@ class PyCFunctionNode(ExprNode, ModuleNameMixin): 'PyList_Append(%s, %s);' % ( class_node.class_cell.result(), self.result())) - code.put_giveref(self.py_result()) + self.generate_giveref(code) if self.defaults: code.putln( @@ -9468,27 +9644,29 @@ class PyCFunctionNode(ExprNode, ModuleNameMixin): if self.defaults_tuple: code.putln('__Pyx_CyFunction_SetDefaultsTuple(%s, %s);' % ( self.result(), self.defaults_tuple.py_result())) - if self.defaults_kwdict: - code.putln('__Pyx_CyFunction_SetDefaultsKwDict(%s, %s);' % ( - self.result(), self.defaults_kwdict.py_result())) - if def_node.defaults_getter and not self.specialized_cpdefs: - # Fused functions do not support dynamic defaults, only their specialisations can have them for now. - code.putln('__Pyx_CyFunction_SetDefaultsGetter(%s, %s);' % ( - self.result(), def_node.defaults_getter.entry.pyfunc_cname)) - if self.annotations_dict: - code.putln('__Pyx_CyFunction_SetAnnotationsDict(%s, %s);' % ( - self.result(), self.annotations_dict.py_result())) + if not self.specialized_cpdefs: + # disable introspection functions for fused dispatcher function since the user never sees it + # TODO: this is mostly disabled because the attributes end up pointing to ones belonging + # to the specializations - ideally this would be fixed instead + if self.defaults_kwdict: + code.putln('__Pyx_CyFunction_SetDefaultsKwDict(%s, %s);' % ( + self.result(), self.defaults_kwdict.py_result())) + if def_node.defaults_getter: + code.putln('__Pyx_CyFunction_SetDefaultsGetter(%s, %s);' % ( + self.result(), def_node.defaults_getter.entry.pyfunc_cname)) + if self.annotations_dict: + code.putln('__Pyx_CyFunction_SetAnnotationsDict(%s, %s);' % ( + self.result(), self.annotations_dict.py_result())) class InnerFunctionNode(PyCFunctionNode): # Special PyCFunctionNode that depends on a closure class - # binding = True - needs_self_code = True + needs_closure_code = True - def self_result_code(self): - if self.needs_self_code: + def closure_result_code(self): + if self.needs_closure_code: return "((PyObject*)%s)" % Naming.cur_scope_cname return "NULL" @@ -9546,9 +9724,10 @@ class CodeObjectNode(ExprNode): if self.def_node.starstar_arg: flags.append('CO_VARKEYWORDS') - code.putln("%s = (PyObject*)__Pyx_PyCode_New(%d, %d, %d, 0, %s, %s, %s, %s, %s, %s, %s, %s, %s, %d, %s); %s" % ( + code.putln("%s = (PyObject*)__Pyx_PyCode_New(%d, %d, %d, %d, 0, %s, %s, %s, %s, %s, %s, %s, %s, %s, %d, %s); %s" % ( self.result_code, len(func.args) - func.num_kwonly_args, # argcount + func.num_posonly_args, # posonlyargcount (Py3.8+ only) func.num_kwonly_args, # kwonlyargcount (Py3 only) len(self.varnames.args), # nlocals '|'.join(flags) or '0', # flags @@ -9670,8 +9849,8 @@ class LambdaNode(InnerFunctionNode): self.lambda_name = self.def_node.lambda_name = env.next_id('lambda') self.def_node.no_assignment_synthesis = True self.def_node.pymethdef_required = True - self.def_node.analyse_declarations(env) self.def_node.is_cyfunction = True + self.def_node.analyse_declarations(env) self.pymethdef_cname = self.def_node.entry.pymethdef_cname env.add_lambda_def(self.def_node) @@ -9710,9 +9889,9 @@ class GeneratorExpressionNode(LambdaNode): '%s = %s(%s); %s' % ( self.result(), self.def_node.entry.pyfunc_cname, - self.self_result_code(), + self.closure_result_code(), code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) class YieldExprNode(ExprNode): @@ -9771,11 +9950,15 @@ class YieldExprNode(ExprNode): for cname, type, manage_ref in code.funcstate.temps_in_use(): save_cname = code.funcstate.closure_temps.allocate_temp(type) saved.append((cname, save_cname, type)) - if type.is_pyobject: - code.put_xgiveref(cname) + if type.is_cpp_class: + code.globalstate.use_utility_code( + UtilityCode.load_cached("MoveIfSupported", "CppSupport.cpp")) + cname = "__PYX_STD_MOVE_IF_SUPPORTED(%s)" % cname + else: + code.put_xgiveref(cname, type) code.putln('%s->%s = %s;' % (Naming.cur_scope_cname, save_cname, cname)) - code.put_xgiveref(Naming.retval_cname) + code.put_xgiveref(Naming.retval_cname, py_object_type) profile = code.globalstate.directives['profile'] linetrace = code.globalstate.directives['linetrace'] if profile or linetrace: @@ -9803,10 +9986,13 @@ class YieldExprNode(ExprNode): code.put_label(label_name) for cname, save_cname, type in saved: - code.putln('%s = %s->%s;' % (cname, Naming.cur_scope_cname, save_cname)) + save_cname = "%s->%s" % (Naming.cur_scope_cname, save_cname) + if type.is_cpp_class: + save_cname = "__PYX_STD_MOVE_IF_SUPPORTED(%s)" % save_cname + code.putln('%s = %s;' % (cname, save_cname)) if type.is_pyobject: - code.putln('%s->%s = 0;' % (Naming.cur_scope_cname, save_cname)) - code.put_xgotref(cname) + code.putln('%s = 0;' % save_cname) + code.put_xgotref(cname, type) self.generate_sent_value_handling_code(code, Naming.sent_value_cname) if self.result_is_used: self.allocate_temp_result(code) @@ -9834,7 +10020,7 @@ class _YieldDelegationExprNode(YieldExprNode): self.arg.free_temps(code) elif decref_source: code.put_decref_clear(source_cname, py_object_type) - code.put_xgotref(Naming.retval_cname) + code.put_xgotref(Naming.retval_cname, py_object_type) code.putln("if (likely(%s)) {" % Naming.retval_cname) self.generate_yield_code(code) @@ -9850,7 +10036,7 @@ class _YieldDelegationExprNode(YieldExprNode): # YieldExprNode has allocated the result temp for us code.putln("%s = NULL;" % self.result()) code.put_error_if_neg(self.pos, "__Pyx_PyGen_FetchStopIterationValue(&%s)" % self.result()) - code.put_gotref(self.result()) + self.generate_gotref(code) def handle_iteration_exception(self, code): code.putln("PyObject* exc_type = __Pyx_PyErr_Occurred();") @@ -9942,7 +10128,7 @@ class GlobalsExprNode(AtomicExprNode): code.putln('%s = __Pyx_Globals(); %s' % ( self.result(), code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.result()) + self.generate_gotref(code) class LocalsDictItemNode(DictItemNode): @@ -10132,7 +10318,7 @@ class UnopNode(ExprNode): function, self.operand.py_result(), code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) def type_error(self): if not self.operand.type.is_error: @@ -10150,7 +10336,7 @@ class UnopNode(ExprNode): self.exception_value = entry.type.exception_value if self.exception_check == '+': self.is_temp = True - if self.exception_value is None: + if needs_cpp_exception_conversion(self): env.use_utility_code(UtilityCode.load_cached("CppExceptionConversion", "CppSupport.cpp")) else: self.exception_check = '' @@ -10284,7 +10470,10 @@ class DereferenceNode(CUnopNode): def analyse_c_operation(self, env): if self.operand.type.is_ptr: - self.type = self.operand.type.base_type + if env.is_cpp: + self.type = PyrexTypes.CReferenceType(self.operand.type.base_type) + else: + self.type = self.operand.type.base_type else: self.type_error() @@ -10710,7 +10899,7 @@ class CythonArrayNode(ExprNode): type_info, code.error_goto_if_null(format_temp, self.pos), )) - code.put_gotref(format_temp) + code.put_gotref(format_temp, py_object_type) buildvalue_fmt = " __PYX_BUILD_PY_SSIZE_T " * len(shapes) code.putln('%s = Py_BuildValue((char*) "(" %s ")", %s); %s' % ( @@ -10719,15 +10908,14 @@ class CythonArrayNode(ExprNode): ", ".join(shapes), code.error_goto_if_null(shapes_temp, self.pos), )) - code.put_gotref(shapes_temp) + code.put_gotref(shapes_temp, py_object_type) - tup = (self.result(), shapes_temp, itemsize, format_temp, - self.mode, self.operand.result()) - code.putln('%s = __pyx_array_new(' - '%s, %s, PyBytes_AS_STRING(%s), ' - '(char *) "%s", (char *) %s);' % tup) - code.putln(code.error_goto_if_null(self.result(), self.pos)) - code.put_gotref(self.result()) + code.putln('%s = __pyx_array_new(%s, %s, PyBytes_AS_STRING(%s), (char *) "%s", (char *) %s); %s' % ( + self.result(), + shapes_temp, itemsize, format_temp, self.mode, self.operand.result(), + code.error_goto_if_null(self.result(), self.pos), + )) + self.generate_gotref(code) def dispose(temp): code.put_decref_clear(temp, py_object_type) @@ -10836,7 +11024,11 @@ class SizeofVarNode(SizeofNode): if operand_as_type: self.arg_type = operand_as_type if self.arg_type.is_fused: - self.arg_type = self.arg_type.specialize(env.fused_to_specific) + try: + self.arg_type = self.arg_type.specialize(env.fused_to_specific) + except CannotSpecialize: + error(self.operand.pos, + "Type cannot be specialized since it is not a fused argument to this function") self.__class__ = SizeofTypeNode self.check_type() else: @@ -10857,8 +11049,6 @@ class TypeidNode(ExprNode): # arg_type ExprNode # is_variable boolean - type = PyrexTypes.error_type - subexprs = ['operand'] arg_type = None @@ -10876,19 +11066,25 @@ class TypeidNode(ExprNode): cpp_message = 'typeid operator' def analyse_types(self, env): + if not self.type: + self.type = PyrexTypes.error_type # default value if it isn't analysed successfully self.cpp_check(env) type_info = self.get_type_info_type(env) if not type_info: self.error("The 'libcpp.typeinfo' module must be cimported to use the typeid() operator") return self + if self.operand is None: + return self # already analysed, no need to repeat self.type = type_info - as_type = self.operand.analyse_as_type(env) + as_type = self.operand.analyse_as_specialized_type(env) if as_type: self.arg_type = as_type self.is_type = True + self.operand = None # nothing further uses self.operand - will only cause problems if its used in code generation else: self.arg_type = self.operand.analyse_types(env) self.is_type = False + self.operand = None # nothing further uses self.operand - will only cause problems if its used in code generation if self.arg_type.type.is_pyobject: self.error("Cannot use typeid on a Python object") return self @@ -10930,11 +11126,11 @@ class TypeofNode(ExprNode): literal = None type = py_object_type - subexprs = ['literal'] # 'operand' will be ignored after type analysis! + subexprs = ['literal'] # 'operand' will be ignored after type analysis! def analyse_types(self, env): self.operand = self.operand.analyse_types(env) - value = StringEncoding.EncodedString(str(self.operand.type)) #self.operand.type.typeof_name()) + value = StringEncoding.EncodedString(str(self.operand.type)) #self.operand.type.typeof_name()) literal = StringNode(self.pos, value=value) literal = literal.analyse_types(env) self.literal = literal.coerce_to_pyobject(env) @@ -11093,7 +11289,7 @@ class BinopNode(ExprNode): # Used by NumBinopNodes to break up expressions involving multiple # operators so that exceptions can be handled properly. self.is_temp = 1 - if self.exception_value is None: + if needs_cpp_exception_conversion(self): env.use_utility_code(UtilityCode.load_cached("CppExceptionConversion", "CppSupport.cpp")) if func_type.is_ptr: func_type = func_type.base_type @@ -11178,7 +11374,7 @@ class BinopNode(ExprNode): self.operand2.py_result(), extra_args, code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) elif self.is_temp: # C++ overloaded operators with exception values are currently all # handled through temporaries. @@ -11302,8 +11498,8 @@ class NumBinopNode(BinopNode): def c_types_okay(self, type1, type2): #print "NumBinopNode.c_types_okay:", type1, type2 ### - return (type1.is_numeric or type1.is_enum) \ - and (type2.is_numeric or type2.is_enum) + return (type1.is_numeric or type1.is_enum) \ + and (type2.is_numeric or type2.is_enum) def generate_evaluation_code(self, code): if self.overflow_check: @@ -11414,7 +11610,7 @@ class AddNode(NumBinopNode): def py_operation_function(self, code): type1, type2 = self.operand1.type, self.operand2.type - + func = None if type1 is unicode_type or type2 is unicode_type: if type1 in (unicode_type, str_type) and type2 in (unicode_type, str_type): is_unicode_concat = True @@ -11426,10 +11622,22 @@ class AddNode(NumBinopNode): is_unicode_concat = False if is_unicode_concat: - if self.operand1.may_be_none() or self.operand2.may_be_none(): - return '__Pyx_PyUnicode_ConcatSafe' - else: - return '__Pyx_PyUnicode_Concat' + if self.inplace or self.operand1.is_temp: + code.globalstate.use_utility_code( + UtilityCode.load_cached("UnicodeConcatInPlace", "ObjectHandling.c")) + func = '__Pyx_PyUnicode_Concat' + elif type1 is str_type and type2 is str_type: + code.globalstate.use_utility_code( + UtilityCode.load_cached("StrConcatInPlace", "ObjectHandling.c")) + func = '__Pyx_PyStr_Concat' + + if func: + # any necessary utility code will be got by "NumberAdd" in generate_evaluation_code + if self.inplace or self.operand1.is_temp: + func += 'InPlace' # upper case to indicate unintuitive macro + if self.operand1.may_be_none() or self.operand2.may_be_none(): + func += 'Safe' + return func return super(AddNode, self).py_operation_function(code) @@ -11450,6 +11658,24 @@ class SubNode(NumBinopNode): class MulNode(NumBinopNode): # '*' operator. + def analyse_types(self, env): + # TODO: we could also optimise the case of "[...] * 2 * n", i.e. with an existing 'mult_factor' + if self.operand1.is_sequence_constructor and self.operand1.mult_factor is None: + operand2 = self.operand2.analyse_types(env) + if operand2.type.is_int or operand2.type is long_type: + return self.analyse_sequence_mul(env, self.operand1, operand2) + elif self.operand2.is_sequence_constructor and self.operand2.mult_factor is None: + operand1 = self.operand1.analyse_types(env) + if operand1.type.is_int or operand1.type is long_type: + return self.analyse_sequence_mul(env, self.operand2, operand1) + + return NumBinopNode.analyse_types(self, env) + + def analyse_sequence_mul(self, env, seq, mult): + assert seq.mult_factor is None + seq.mult_factor = mult + return seq.analyse_types(env) + def is_py_operation_types(self, type1, type2): if ((type1.is_string and type2.is_int) or (type2.is_string and type1.is_int)): @@ -11601,7 +11827,7 @@ class DivNode(NumBinopNode): minus1_check = '(!(((%s)-1) > 0)) && unlikely(%s == (%s)-1)' % ( type_of_op2, self.operand2.result(), type_of_op2) code.putln("else if (sizeof(%s) == sizeof(long) && %s " - " && unlikely(UNARY_NEG_WOULD_OVERFLOW(%s))) {" % ( + " && unlikely(__Pyx_UNARY_NEG_WOULD_OVERFLOW(%s))) {" % ( self.type.empty_declaration_code(), minus1_check, self.operand1.result())) @@ -11669,10 +11895,10 @@ _find_formatting_types = re.compile( br")").findall # These format conversion types can never trigger a Unicode string conversion in Py2. -_safe_bytes_formats = set([ +_safe_bytes_formats = frozenset({ # Excludes 's' and 'r', which can generate non-bytes strings. b'd', b'i', b'o', b'u', b'x', b'X', b'e', b'E', b'f', b'F', b'g', b'G', b'c', b'b', b'a', -]) +}) class ModNode(DivNode): @@ -11795,6 +12021,12 @@ class PowNode(NumBinopNode): error(self.pos, "got unexpected types for C power operator: %s, %s" % (self.operand1.type, self.operand2.type)) + def compute_c_result_type(self, type1, type2): + c_result_type = super(PowNode, self).compute_c_result_type(type1, type2) + if isinstance(self.operand2.constant_result, _py_int_types) and self.operand2.constant_result < 0: + c_result_type = PyrexTypes.widest_numeric_type(c_result_type, PyrexTypes.c_double_type) + return c_result_type + def calculate_result_code(self): # Work around MSVC overloading ambiguity. def typecast(operand): @@ -12065,6 +12297,9 @@ class BoolBinopResultNode(ExprNode): code.putln("}") self.arg.free_temps(code) + def analyse_types(self, env): + return self + class CondExprNode(ExprNode): # Short-circuiting conditional expression. @@ -12615,16 +12850,16 @@ class PrimaryCmpNode(ExprNode, CmpNode): elif self.find_special_bool_compare_function(env, self.operand1): if not self.operand1.type.is_pyobject: self.operand1 = self.operand1.coerce_to_pyobject(env) - common_type = None # if coercion needed, the method call above has already done it - self.is_pycmp = False # result is bint + common_type = None # if coercion needed, the method call above has already done it + self.is_pycmp = False # result is bint else: common_type = py_object_type self.is_pycmp = True elif self.find_special_bool_compare_function(env, self.operand1): if not self.operand1.type.is_pyobject: self.operand1 = self.operand1.coerce_to_pyobject(env) - common_type = None # if coercion needed, the method call above has already done it - self.is_pycmp = False # result is bint + common_type = None # if coercion needed, the method call above has already done it + self.is_pycmp = False # result is bint else: common_type = self.find_common_type(env, self.operator, self.operand1) self.is_pycmp = common_type.is_pyobject @@ -12671,7 +12906,7 @@ class PrimaryCmpNode(ExprNode, CmpNode): self.exception_value = func_type.exception_value if self.exception_check == '+': self.is_temp = True - if self.exception_value is None: + if needs_cpp_exception_conversion(self): env.use_utility_code(UtilityCode.load_cached("CppExceptionConversion", "CppSupport.cpp")) if len(func_type.args) == 1: self.operand2 = self.operand2.coerce_to(func_type.args[0].type, env) @@ -12812,7 +13047,7 @@ class CascadedCmpNode(Node, CmpNode): cascade = None coerced_operand2 = None - constant_result = constant_value_not_set # FIXME: where to calculate this? + constant_result = constant_value_not_set # FIXME: where to calculate this? def infer_type(self, env): # TODO: Actually implement this (after merging with -unstable). @@ -13010,9 +13245,10 @@ class PyTypeTestNode(CoercionNode): exact_builtin_type = True def __init__(self, arg, dst_type, env, notnone=False): - # The arg is know to be a Python object, and + # The arg is known to be a Python object, and # the dst_type is known to be an extension type. - assert dst_type.is_extension_type or dst_type.is_builtin_type, "PyTypeTest on non extension type" + assert dst_type.is_extension_type or dst_type.is_builtin_type, \ + "PyTypeTest for %s against non extension type %s" % (arg.type, dst_type) CoercionNode.__init__(self, arg) self.type = dst_type self.result_ctype = arg.ctype() @@ -13063,6 +13299,8 @@ class PyTypeTestNode(CoercionNode): type_test = self.type.type_test_code( self.arg.py_result(), self.notnone, exact=self.exact_builtin_type) + code.globalstate.use_utility_code(UtilityCode.load_cached( + "RaiseUnexpectedTypeError", "ObjectHandling.c")) else: type_test = self.type.type_test_code( self.arg.py_result(), self.notnone) @@ -13106,7 +13344,7 @@ class NoneCheckNode(CoercionNode): self.exception_message = exception_message self.exception_format_args = tuple(exception_format_args or ()) - nogil_check = None # this node only guards an operation that would fail already + nogil_check = None # this node only guards an operation that would fail already def analyse_types(self, env): return self @@ -13229,7 +13467,7 @@ class CoerceToPyTypeNode(CoercionNode): def coerce_to_boolean(self, env): arg_type = self.arg.type if (arg_type == PyrexTypes.c_bint_type or - (arg_type.is_pyobject and arg_type.name == 'bool')): + (arg_type.is_pyobject and arg_type.name == 'bool')): return self.arg.coerce_to_temp(env) else: return CoerceToBooleanNode(self, env) @@ -13253,7 +13491,7 @@ class CoerceToPyTypeNode(CoercionNode): self.target_type), code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) class CoerceIntToBytesNode(CoerceToPyTypeNode): @@ -13293,7 +13531,7 @@ class CoerceIntToBytesNode(CoerceToPyTypeNode): code.error_goto_if_null(self.result(), self.pos))) if temp is not None: code.funcstate.release_temp(temp) - code.put_gotref(self.py_result()) + self.generate_gotref(code) class CoerceFromPyTypeNode(CoercionNode): @@ -13331,7 +13569,7 @@ class CoerceFromPyTypeNode(CoercionNode): code.putln(self.type.from_py_call_code( self.arg.py_result(), self.result(), self.pos, code, from_py_function=from_py_function)) if self.type.is_pyobject: - code.put_gotref(self.py_result()) + self.generate_gotref(code) def nogil_check(self, env): error(self.pos, "Coercion from Python not allowed without the GIL") @@ -13388,6 +13626,9 @@ class CoerceToBooleanNode(CoercionNode): self.arg.py_result(), code.error_goto_if_neg(self.result(), self.pos))) + def analyse_types(self, env): + return self + class CoerceToComplexNode(CoercionNode): @@ -13413,6 +13654,9 @@ class CoerceToComplexNode(CoercionNode): def generate_result_code(self, code): pass + def analyse_types(self, env): + return self + class CoerceToTempNode(CoercionNode): # This node is used to force the result of another node # to be stored in a temporary. It is only used if the @@ -13432,6 +13676,9 @@ class CoerceToTempNode(CoercionNode): # The arg is always already analysed return self + def may_be_none(self): + return self.arg.may_be_none() + def coerce_to_boolean(self, env): self.arg = self.arg.coerce_to_boolean(env) if self.arg.is_simple(): @@ -13446,11 +13693,12 @@ class CoerceToTempNode(CoercionNode): code.putln("%s = %s;" % ( self.result(), self.arg.result_as(self.ctype()))) if self.use_managed_ref: - if self.type.is_pyobject: + if not self.type.is_memoryviewslice: code.put_incref(self.result(), self.ctype()) - elif self.type.is_memoryviewslice: - code.put_incref_memoryviewslice(self.result(), - not self.in_nogil_context) + else: + code.put_incref_memoryviewslice(self.result(), self.type, + have_gil=not self.in_nogil_context) + class ProxyNode(CoercionNode): """ @@ -13477,11 +13725,13 @@ class ProxyNode(CoercionNode): return self.arg.infer_type(env) def _proxy_type(self): - if hasattr(self.arg, 'type'): - self.type = self.arg.type + type = getattr(self.arg, 'type', None) + if type: + self.type = type self.result_ctype = self.arg.result_ctype - if hasattr(self.arg, 'entry'): - self.entry = self.arg.entry + arg_entry = getattr(self.arg, 'entry', None) + if arg_entry: + self.entry = arg_entry def generate_result_code(self, code): self.arg.generate_result_code(code) @@ -13512,17 +13762,19 @@ class CloneNode(CoercionNode): # disposal code for it. The original owner of the argument # node is responsible for doing those things. - subexprs = [] # Arg is not considered a subexpr + subexprs = [] # Arg is not considered a subexpr nogil_check = None def __init__(self, arg): CoercionNode.__init__(self, arg) self.constant_result = arg.constant_result - if hasattr(arg, 'type'): - self.type = arg.type + type = getattr(arg, 'type', None) + if type: + self.type = type self.result_ctype = arg.result_ctype - if hasattr(arg, 'entry'): - self.entry = arg.entry + arg_entry = getattr(arg, 'entry', None) + if arg_entry: + self.entry = arg_entry def result(self): return self.arg.result() @@ -13540,8 +13792,9 @@ class CloneNode(CoercionNode): self.type = self.arg.type self.result_ctype = self.arg.result_ctype self.is_temp = 1 - if hasattr(self.arg, 'entry'): - self.entry = self.arg.entry + arg_entry = getattr(self.arg, 'entry', None) + if arg_entry: + self.entry = arg_entry return self def coerce_to(self, dest_type, env): @@ -13550,7 +13803,7 @@ class CloneNode(CoercionNode): return super(CloneNode, self).coerce_to(dest_type, env) def is_simple(self): - return True # result is always in a temp (or a name) + return True # result is always in a temp (or a name) def generate_evaluation_code(self, code): pass @@ -13565,6 +13818,28 @@ class CloneNode(CoercionNode): pass +class CppOptionalTempCoercion(CoercionNode): + """ + Used only in CoerceCppTemps - handles cases the temp is actually a OptionalCppClassType (and thus needs dereferencing when on the rhs) + """ + is_temp = False + + @property + def type(self): + return self.arg.type + + def calculate_result_code(self): + return "(*%s)" % self.arg.result() + + def generate_result_code(self, code): + pass + + def _make_move_result_rhs(self, result, optional=False): + # this wouldn't normally get moved (because it isn't a temp), but force it to be because it + # is a thin wrapper around a temp + return super(CppOptionalTempCoercion, self)._make_move_result_rhs(result, optional=False) + + class CMethodSelfCloneNode(CloneNode): # Special CloneNode for the self argument of builtin C methods # that accepts subtypes of the builtin type. This is safe only @@ -13616,77 +13891,94 @@ class DocstringRefNode(ExprNode): self.result(), self.body.result(), code.intern_identifier(StringEncoding.EncodedString("__doc__")), code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.result()) - - + self.generate_gotref(code) + +class AnnotationNode(ExprNode): + # Deals with the two possible uses of an annotation. + # 1. The post PEP-563 use where an annotation is stored + # as a string + # 2. The Cython use where the annotation can indicate an + # object type + # + # Doesn't handle the pre PEP-563 version where the + # annotation is evaluated into a Python Object. -#------------------------------------------------------------------------------------ -# -# Runtime support code -# -#------------------------------------------------------------------------------------ - -pyerr_occurred_withgil_utility_code= UtilityCode( -proto = """ -static CYTHON_INLINE int __Pyx_ErrOccurredWithGIL(void); /* proto */ -""", -impl = """ -static CYTHON_INLINE int __Pyx_ErrOccurredWithGIL(void) { - int err; - #ifdef WITH_THREAD - PyGILState_STATE _save = PyGILState_Ensure(); - #endif - err = !!PyErr_Occurred(); - #ifdef WITH_THREAD - PyGILState_Release(_save); - #endif - return err; -} -""" -) + subexprs = [] -#------------------------------------------------------------------------------------ + # 'untyped' is set for fused specializations: + # Once a fused function has been created we don't want + # annotations to override an already set type. + untyped = False -raise_unbound_local_error_utility_code = UtilityCode( -proto = """ -static CYTHON_INLINE void __Pyx_RaiseUnboundLocalError(const char *varname); -""", -impl = """ -static CYTHON_INLINE void __Pyx_RaiseUnboundLocalError(const char *varname) { - PyErr_Format(PyExc_UnboundLocalError, "local variable '%s' referenced before assignment", varname); -} -""") - -raise_closure_name_error_utility_code = UtilityCode( -proto = """ -static CYTHON_INLINE void __Pyx_RaiseClosureNameError(const char *varname); -""", -impl = """ -static CYTHON_INLINE void __Pyx_RaiseClosureNameError(const char *varname) { - PyErr_Format(PyExc_NameError, "free variable '%s' referenced before assignment in enclosing scope", varname); -} -""") - -# Don't inline the function, it should really never be called in production -raise_unbound_memoryview_utility_code_nogil = UtilityCode( -proto = """ -static void __Pyx_RaiseUnboundMemoryviewSliceNogil(const char *varname); -""", -impl = """ -static void __Pyx_RaiseUnboundMemoryviewSliceNogil(const char *varname) { - #ifdef WITH_THREAD - PyGILState_STATE gilstate = PyGILState_Ensure(); - #endif - __Pyx_RaiseUnboundLocalError(varname); - #ifdef WITH_THREAD - PyGILState_Release(gilstate); - #endif -} -""", -requires = [raise_unbound_local_error_utility_code]) + def __init__(self, pos, expr, string=None): + """string is expected to already be a StringNode or None""" + ExprNode.__init__(self, pos) + if string is None: + # import doesn't work at top of file? + from .AutoDocTransforms import AnnotationWriter + string = StringEncoding.EncodedString( + AnnotationWriter(description="annotation").write(expr)) + string = StringNode(pos, unicode_value=string, value=string.as_utf8_string()) + self.string = string + self.expr = expr -#------------------------------------------------------------------------------------ + def analyse_types(self, env): + return self # nothing needs doing -raise_too_many_values_to_unpack = UtilityCode.load_cached("RaiseTooManyValuesToUnpack", "ObjectHandling.c") -raise_need_more_values_to_unpack = UtilityCode.load_cached("RaiseNeedMoreValuesToUnpack", "ObjectHandling.c") -tuple_unpacking_error_code = UtilityCode.load_cached("UnpackTupleError", "ObjectHandling.c") + def analyse_as_type(self, env): + # for compatibility when used as a return_type_node, have this interface too + return self.analyse_type_annotation(env)[1] + + def analyse_type_annotation(self, env, assigned_value=None): + if self.untyped: + # Already applied as a fused type, not re-evaluating it here. + return None, None + annotation = self.expr + base_type = None + is_ambiguous = False + explicit_pytype = explicit_ctype = False + if annotation.is_dict_literal: + warning(annotation.pos, + "Dicts should no longer be used as type annotations. Use 'cython.int' etc. directly.", level=1) + for name, value in annotation.key_value_pairs: + if not name.is_string_literal: + continue + if name.value in ('type', b'type'): + explicit_pytype = True + if not explicit_ctype: + annotation = value + elif name.value in ('ctype', b'ctype'): + explicit_ctype = True + annotation = value + if explicit_pytype and explicit_ctype: + warning(annotation.pos, "Duplicate type declarations found in signature annotation", level=1) + arg_type = annotation.analyse_as_type(env) + if annotation.is_name and not annotation.cython_attribute and annotation.name in ('int', 'long', 'float'): + # Map builtin numeric Python types to C types in safe cases. + if assigned_value is not None and arg_type is not None and not arg_type.is_pyobject: + assigned_type = assigned_value.infer_type(env) + if assigned_type and assigned_type.is_pyobject: + # C type seems unsafe, e.g. due to 'None' default value => ignore annotation type + is_ambiguous = True + arg_type = None + # ignore 'int' and require 'cython.int' to avoid unsafe integer declarations + if arg_type in (PyrexTypes.c_long_type, PyrexTypes.c_int_type, PyrexTypes.c_float_type): + arg_type = PyrexTypes.c_double_type if annotation.name == 'float' else py_object_type + elif arg_type is not None and annotation.is_string_literal: + warning(annotation.pos, + "Strings should no longer be used for type declarations. Use 'cython.int' etc. directly.", + level=1) + elif arg_type is not None and arg_type.is_complex: + # creating utility code needs to be special-cased for complex types + arg_type.create_declaration_utility_code(env) + if arg_type is not None: + if explicit_pytype and not explicit_ctype and not arg_type.is_pyobject: + warning(annotation.pos, + "Python type declaration in signature annotation does not refer to a Python type") + base_type = Nodes.CAnalysedBaseTypeNode( + annotation.pos, type=arg_type, is_arg=True) + elif is_ambiguous: + warning(annotation.pos, "Ambiguous types in annotation, ignoring") + else: + warning(annotation.pos, "Unknown type declaration in annotation, ignoring") + return base_type, arg_type diff --git a/Cython/Compiler/FlowControl.pxd b/Cython/Compiler/FlowControl.pxd index c87370b81..c876ee3b1 100644 --- a/Cython/Compiler/FlowControl.pxd +++ b/Cython/Compiler/FlowControl.pxd @@ -1,4 +1,4 @@ -from __future__ import absolute_import +# cython: language_level=3 cimport cython @@ -105,6 +105,7 @@ cdef class ControlFlowAnalysis(CythonTransform): cdef list stack cdef object env cdef ControlFlow flow + cdef object object_expr cdef bint in_inplace_assignment cpdef mark_assignment(self, lhs, rhs=*) diff --git a/Cython/Compiler/FlowControl.py b/Cython/Compiler/FlowControl.py index df04471f9..e25947011 100644 --- a/Cython/Compiler/FlowControl.py +++ b/Cython/Compiler/FlowControl.py @@ -1,16 +1,17 @@ +# cython: language_level=3str +# cython: auto_pickle=True + from __future__ import absolute_import import cython cython.declare(PyrexTypes=object, ExprNodes=object, Nodes=object, Builtin=object, InternalError=object, error=object, warning=object, - py_object_type=object, unspecified_type=object, - object_expr=object, fake_rhs_expr=object, TypedExprNode=object) + fake_rhs_expr=object, TypedExprNode=object) from . import Builtin from . import ExprNodes from . import Nodes from . import Options -from .PyrexTypes import py_object_type, unspecified_type from . import PyrexTypes from .Visitor import TreeVisitor, CythonTransform @@ -28,9 +29,8 @@ class TypedExprNode(ExprNodes.ExprNode): def may_be_none(self): return self._may_be_none != False -object_expr = TypedExprNode(py_object_type, may_be_none=True) # Fake rhs to silence "unused variable" warning -fake_rhs_expr = TypedExprNode(unspecified_type) +fake_rhs_expr = TypedExprNode(PyrexTypes.unspecified_type) class ControlBlock(object): @@ -52,7 +52,7 @@ class ControlBlock(object): stats = [Assignment(a), NameReference(a), NameReference(c), Assignment(b)] gen = {Entry(a): Assignment(a), Entry(b): Assignment(b)} - bounded = set([Entry(a), Entry(c)]) + bounded = {Entry(a), Entry(c)} """ @@ -160,7 +160,7 @@ class ControlFlow(object): (entry.type.is_struct_or_union or entry.type.is_complex or entry.type.is_array or - entry.type.is_cpp_class)): + (entry.type.is_cpp_class and not entry.is_cpp_optional))): # stack allocated structured variable => never uninitialised return True return False @@ -203,7 +203,7 @@ class ControlFlow(object): def normalize(self): """Delete unreachable and orphan blocks.""" - queue = set([self.entry_point]) + queue = {self.entry_point} visited = set() while queue: root = queue.pop() @@ -217,7 +217,7 @@ class ControlFlow(object): visited.remove(self.entry_point) for block in visited: if block.empty(): - for parent in block.parents: # Re-parent + for parent in block.parents: # Re-parent for child in block.children: parent.add_child(child) block.detach() @@ -373,9 +373,9 @@ class NameDeletion(NameAssignment): def infer_type(self): inferred_type = self.rhs.infer_type(self.entry.scope) - if (not inferred_type.is_pyobject and - inferred_type.can_coerce_to_pyobject(self.entry.scope)): - return py_object_type + if (not inferred_type.is_pyobject + and inferred_type.can_coerce_to_pyobject(self.entry.scope)): + return PyrexTypes.py_object_type self.inferred_type = inferred_type return inferred_type @@ -455,7 +455,7 @@ class GVContext(object): start = min(block.positions) stop = max(block.positions) srcdescr = start[0] - if not srcdescr in self.sources: + if srcdescr not in self.sources: self.sources[srcdescr] = list(srcdescr.get_lines()) lines = self.sources[srcdescr] return '\\n'.join([l.strip() for l in lines[start[1] - 1:stop[1]]]) @@ -621,7 +621,7 @@ def check_definitions(flow, compiler_directives): # Unused result for assmt in assignments: if (not assmt.refs and not assmt.entry.is_pyclass_attr - and not assmt.entry.in_closure): + and not assmt.entry.in_closure): if assmt.entry.cf_references and warn_unused_result: if assmt.is_arg: messages.warning(assmt.pos, "Unused argument value '%s'" % @@ -674,7 +674,8 @@ class AssignmentCollector(TreeVisitor): class ControlFlowAnalysis(CythonTransform): def visit_ModuleNode(self, node): - self.gv_ctx = GVContext() + dot_output = self.current_directives['control_flow.dot_output'] + self.gv_ctx = GVContext() if dot_output else None self.constant_folder = ConstantFolding() # Set of NameNode reductions @@ -685,18 +686,15 @@ class ControlFlowAnalysis(CythonTransform): self.env = node.scope self.stack = [] self.flow = ControlFlow() + self.object_expr = TypedExprNode(PyrexTypes.py_object_type, may_be_none=True) self.visitchildren(node) check_definitions(self.flow, self.current_directives) - dot_output = self.current_directives['control_flow.dot_output'] if dot_output: annotate_defs = self.current_directives['control_flow.dot_annotate_defs'] - fp = open(dot_output, 'wt') - try: + with open(dot_output, 'wt') as fp: self.gv_ctx.render(fp, 'module', annotate_defs=annotate_defs) - finally: - fp.close() return node def visit_FuncDefNode(self, node): @@ -744,7 +742,8 @@ class ControlFlowAnalysis(CythonTransform): check_definitions(self.flow, self.current_directives) self.flow.blocks.add(self.flow.entry_point) - self.gv_ctx.add(GV(node.local_scope.name, self.flow)) + if self.gv_ctx is not None: + self.gv_ctx.add(GV(node.local_scope.name, self.flow)) self.flow = self.stack.pop() self.env = self.env_stack.pop() @@ -769,19 +768,22 @@ class ControlFlowAnalysis(CythonTransform): self.flow.nextblock() if not rhs: - rhs = object_expr + rhs = self.object_expr if lhs.is_name: if lhs.entry is not None: entry = lhs.entry else: entry = self.env.lookup(lhs.name) - if entry is None: # TODO: This shouldn't happen... + if entry is None: # TODO: This shouldn't happen... return self.flow.mark_assignment(lhs, rhs, entry) elif lhs.is_sequence_constructor: for i, arg in enumerate(lhs.args): - if not rhs or arg.is_starred: - item_node = None + if arg.is_starred: + # "a, *b = x" assigns a list to "b" + item_node = TypedExprNode(Builtin.list_type, may_be_none=False, pos=arg.pos) + elif rhs is self.object_expr: + item_node = rhs else: item_node = rhs.inferable_item_node(i) self.mark_assignment(arg, item_node) @@ -806,7 +808,7 @@ class ControlFlowAnalysis(CythonTransform): return node def visit_AssignmentNode(self, node): - raise InternalError("Unhandled assignment node") + raise InternalError("Unhandled assignment node %s" % type(node)) def visit_SingleAssignmentNode(self, node): self._visit(node.rhs) @@ -916,6 +918,26 @@ class ControlFlowAnalysis(CythonTransform): self.flow.block = None return node + def visit_AssertStatNode(self, node): + """Essentially an if-condition that wraps a RaiseStatNode. + """ + self.mark_position(node) + next_block = self.flow.newblock() + parent = self.flow.block + # failure case + parent = self.flow.nextblock(parent) + self._visit(node.condition) + self.flow.nextblock() + self._visit(node.exception) + if self.flow.block: + self.flow.block.add_child(next_block) + parent.add_child(next_block) + if next_block.parents: + self.flow.block = next_block + else: + self.flow.block = None + return node + def visit_WhileStatNode(self, node): condition_block = self.flow.nextblock() next_block = self.flow.newblock() @@ -1013,7 +1035,7 @@ class ControlFlowAnalysis(CythonTransform): elif isinstance(node, Nodes.AsyncForStatNode): # not entirely correct, but good enough for now self.mark_assignment(node.target, node.item) - else: # Parallel + else: # Parallel self.mark_assignment(node.target) # Body block @@ -1308,10 +1330,12 @@ class ControlFlowAnalysis(CythonTransform): self.visitchildren(node, ('dict', 'metaclass', 'mkw', 'bases', 'class_result')) self.flow.mark_assignment(node.target, node.classobj, - self.env.lookup(node.name)) + self.env.lookup(node.target.name)) self.env_stack.append(self.env) self.env = node.scope self.flow.nextblock() + if node.doc_node: + self.flow.mark_assignment(node.doc_node, fake_rhs_expr, node.doc_node.entry) self.visitchildren(node, ('body',)) self.flow.nextblock() self.env = self.env_stack.pop() diff --git a/Cython/Compiler/FusedNode.py b/Cython/Compiler/FusedNode.py index 26d6ffd3d..918a05990 100644 --- a/Cython/Compiler/FusedNode.py +++ b/Cython/Compiler/FusedNode.py @@ -7,6 +7,7 @@ from . import (ExprNodes, PyrexTypes, MemoryView, from .ExprNodes import CloneNode, ProxyNode, TupleNode from .Nodes import FuncDefNode, CFuncDefNode, StatListNode, DefNode from ..Utils import OrderedSet +from .Errors import error, CannotSpecialize class FusedCFuncDefNode(StatListNode): @@ -141,7 +142,14 @@ class FusedCFuncDefNode(StatListNode): copied_node = copy.deepcopy(self.node) # Make the types in our CFuncType specific. - type = copied_node.type.specialize(fused_to_specific) + try: + type = copied_node.type.specialize(fused_to_specific) + except CannotSpecialize: + # unlike for the argument types, specializing the return type can fail + error(copied_node.pos, "Return type is a fused type that cannot " + "be determined from the function arguments") + self.py_func = None # this is just to let the compiler exit gracefully + return entry = copied_node.entry type.specialize_entry(entry, cname) @@ -220,6 +228,10 @@ class FusedCFuncDefNode(StatListNode): arg.type = arg.type.specialize(fused_to_specific) if arg.type.is_memoryviewslice: arg.type.validate_memslice_dtype(arg.pos) + if arg.annotation: + # TODO might be nice if annotations were specialized instead? + # (Or might be hard to do reliably) + arg.annotation.untyped = True def create_new_local_scope(self, node, env, f2s): """ @@ -402,7 +414,7 @@ class FusedCFuncDefNode(StatListNode): if itemsize == -1 or itemsize == {{sizeof_dtype}}: memslice = {{coerce_from_py_func}}(arg, 0) if memslice.memview: - __PYX_XDEC_MEMVIEW(&memslice, 1) + __PYX_XCLEAR_MEMVIEW(&memslice, 1) # print 'found a match for the buffer through format parsing' %s break @@ -481,7 +493,7 @@ class FusedCFuncDefNode(StatListNode): ctypedef struct {{memviewslice_cname}}: void *memview - void __PYX_XDEC_MEMVIEW({{memviewslice_cname}} *, int have_gil) + void __PYX_XCLEAR_MEMVIEW({{memviewslice_cname}} *, int have_gil) bint __pyx_memoryview_check(object) """) @@ -580,6 +592,26 @@ class FusedCFuncDefNode(StatListNode): {{endif}} """) + def _fused_signature_index(self, pyx_code): + """ + Generate Cython code for constructing a persistent nested dictionary index of + fused type specialization signatures. + """ + pyx_code.put_chunk( + u""" + if not _fused_sigindex: + for sig in <dict>signatures: + sigindex_node = _fused_sigindex + *sig_series, last_type = sig.strip('()').split('|') + for sig_type in sig_series: + if sig_type not in sigindex_node: + sigindex_node[sig_type] = sigindex_node = {} + else: + sigindex_node = sigindex_node[sig_type] + sigindex_node[last_type] = sig + """ + ) + def make_fused_cpdef(self, orig_py_func, env, is_def): """ This creates the function that is indexable from Python and does @@ -616,10 +648,14 @@ class FusedCFuncDefNode(StatListNode): pyx_code.put_chunk( u""" - def __pyx_fused_cpdef(signatures, args, kwargs, defaults): + def __pyx_fused_cpdef(signatures, args, kwargs, defaults, _fused_sigindex={}): # FIXME: use a typed signature - currently fails badly because # default arguments inherit the types we specify here! + cdef list search_list + + cdef dict sn, sigindex_node + dest_sig = [None] * {{n_fused}} if kwargs is not None and not kwargs: @@ -630,7 +666,7 @@ class FusedCFuncDefNode(StatListNode): # instance check body """) - pyx_code.indent() # indent following code to function body + pyx_code.indent() # indent following code to function body pyx_code.named_insertion_point("imports") pyx_code.named_insertion_point("func_defs") pyx_code.named_insertion_point("local_variable_declarations") @@ -687,23 +723,36 @@ class FusedCFuncDefNode(StatListNode): env.use_utility_code(Code.UtilityCode.load_cached("Import", "ImportExport.c")) env.use_utility_code(Code.UtilityCode.load_cached("ImportNumPyArray", "ImportExport.c")) + self._fused_signature_index(pyx_code) + pyx_code.put_chunk( u""" - candidates = [] - for sig in <dict>signatures: - match_found = False - src_sig = sig.strip('()').split('|') - for i in range(len(dest_sig)): - dst_type = dest_sig[i] - if dst_type is not None: - if src_sig[i] == dst_type: - match_found = True - else: - match_found = False - break + sigindex_matches = [] + sigindex_candidates = [_fused_sigindex] + + for dst_type in dest_sig: + found_matches = [] + found_candidates = [] + # Make two seperate lists: One for signature sub-trees + # with at least one definite match, and another for + # signature sub-trees with only ambiguous matches + # (where `dest_sig[i] is None`). + if dst_type is None: + for sn in sigindex_matches: + found_matches.extend(sn.values()) + for sn in sigindex_candidates: + found_candidates.extend(sn.values()) + else: + for search_list in (sigindex_matches, sigindex_candidates): + for sn in search_list: + if dst_type in sn: + found_matches.append(sn[dst_type]) + sigindex_matches = found_matches + sigindex_candidates = found_candidates + if not (found_matches or found_candidates): + break - if match_found: - candidates.append(sig) + candidates = sigindex_matches if not candidates: raise TypeError("No matching signature found") @@ -798,7 +847,8 @@ class FusedCFuncDefNode(StatListNode): for i, stat in enumerate(self.stats): stat = self.stats[i] = stat.analyse_expressions(env) - if isinstance(stat, FuncDefNode): + if isinstance(stat, FuncDefNode) and stat is not self.py_func: + # the dispatcher specifically doesn't want its defaults overriding for arg, default in zip(stat.args, defaults): if default is not None: arg.default = CloneNode(default).coerce_to(arg.type, env) @@ -829,6 +879,10 @@ class FusedCFuncDefNode(StatListNode): else: nodes = self.nodes + # For the moment, fused functions do not support METH_FASTCALL + for node in nodes: + node.entry.signature.use_fastcall = False + signatures = [StringEncoding.EncodedString(node.specialized_signature_string) for node in nodes] keys = [ExprNodes.StringNode(node.pos, value=sig) @@ -877,7 +931,7 @@ class FusedCFuncDefNode(StatListNode): "((__pyx_FusedFunctionObject *) %s)->__signatures__ = %s;" % (self.resulting_fused_function.result(), self.__signatures__.result())) - code.put_giveref(self.__signatures__.result()) + self.__signatures__.generate_giveref(code) self.__signatures__.generate_post_assignment_code(code) self.__signatures__.free_temps(code) diff --git a/Cython/Compiler/Future.py b/Cython/Compiler/Future.py index 848792e00..8de10c0cb 100644 --- a/Cython/Compiler/Future.py +++ b/Cython/Compiler/Future.py @@ -11,5 +11,6 @@ absolute_import = _get_feature("absolute_import") nested_scopes = _get_feature("nested_scopes") # dummy generators = _get_feature("generators") # dummy generator_stop = _get_feature("generator_stop") +annotations = _get_feature("annotations") del _get_feature diff --git a/Cython/Compiler/Interpreter.py b/Cython/Compiler/Interpreter.py index 9ec391f2a..244397264 100644 --- a/Cython/Compiler/Interpreter.py +++ b/Cython/Compiler/Interpreter.py @@ -47,8 +47,8 @@ def interpret_compiletime_options(optlist, optdict, type_env=None, type_args=()) raise CompileError(node.pos, "Type not allowed here.") else: if (sys.version_info[0] >=3 and - isinstance(node, StringNode) and - node.unicode_value is not None): + isinstance(node, StringNode) and + node.unicode_value is not None): return (node.unicode_value, node.pos) return (node.compile_time_value(empty_scope), node.pos) diff --git a/Cython/Compiler/Lexicon.py b/Cython/Compiler/Lexicon.py index 72c9ceaef..7c23b5de1 100644 --- a/Cython/Compiler/Lexicon.py +++ b/Cython/Compiler/Lexicon.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- # cython: language_level=3, py2_import=True # # Cython Scanner - Lexical Definitions @@ -16,28 +17,43 @@ IDENT = 'IDENT' def make_lexicon(): from ..Plex import \ Str, Any, AnyBut, AnyChar, Rep, Rep1, Opt, Bol, Eol, Eof, \ - TEXT, IGNORE, State, Lexicon - from .Scanning import Method + TEXT, IGNORE, Method, State, Lexicon, Range - letter = Any("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz_") + nonzero_digit = Any("123456789") digit = Any("0123456789") bindigit = Any("01") octdigit = Any("01234567") hexdigit = Any("0123456789ABCDEFabcdef") indentation = Bol + Rep(Any(" \t")) + # The list of valid unicode identifier characters are pretty slow to generate at runtime, + # and require Python3, so are just included directly here + # (via the generated code block at the bottom of the file) + unicode_start_character = (Any(unicode_start_ch_any) | Range(unicode_start_ch_range)) + unicode_continuation_character = ( + unicode_start_character | + Any(unicode_continuation_ch_any) | Range(unicode_continuation_ch_range)) + def underscore_digits(d): return Rep1(d) + Rep(Str("_") + Rep1(d)) + def prefixed_digits(prefix, digits): + return prefix + Opt(Str("_")) + underscore_digits(digits) + decimal = underscore_digits(digit) dot = Str(".") exponent = Any("Ee") + Opt(Any("+-")) + decimal decimal_fract = (decimal + dot + Opt(decimal)) | (dot + decimal) - name = letter + Rep(letter | digit) - intconst = decimal | (Str("0") + ((Any("Xx") + underscore_digits(hexdigit)) | - (Any("Oo") + underscore_digits(octdigit)) | - (Any("Bb") + underscore_digits(bindigit)) )) + #name = letter + Rep(letter | digit) + name = unicode_start_character + Rep(unicode_continuation_character) + intconst = (prefixed_digits(nonzero_digit, digit) | # decimal literals with underscores must not start with '0' + (Str("0") + (prefixed_digits(Any("Xx"), hexdigit) | + prefixed_digits(Any("Oo"), octdigit) | + prefixed_digits(Any("Bb"), bindigit) )) | + underscore_digits(Str('0')) # 0_0_0_0... is allowed as a decimal literal + | Rep1(digit) # FIXME: remove these Py2 style decimal/octal literals (PY_VERSION_HEX < 3) + ) intsuffix = (Opt(Any("Uu")) + Opt(Any("Ll")) + Opt(Any("Ll"))) | (Opt(Any("Ll")) + Opt(Any("Ll")) + Opt(Any("Uu"))) intliteral = intconst + intsuffix fltconst = (decimal_fract + Opt(exponent)) | (decimal + exponent) @@ -61,7 +77,7 @@ def make_lexicon(): punct = Any(":,;+-*/|&<>=.%`~^?!@") diphthong = Str("==", "<>", "!=", "<=", ">=", "<<", ">>", "**", "//", "+=", "-=", "*=", "/=", "%=", "|=", "^=", "&=", - "<<=", ">>=", "**=", "//=", "->", "@=") + "<<=", ">>=", "**=", "//=", "->", "@=", "&&", "||") spaces = Rep1(Any(" \t\f")) escaped_newline = Str("\\\n") lineterm = Eol + Opt(Str("\n")) @@ -69,7 +85,7 @@ def make_lexicon(): comment = Str("#") + Rep(AnyBut("\n")) return Lexicon([ - (name, IDENT), + (name, Method('normalize_ident')), (intliteral, Method('strip_underscores', symbol='INT')), (fltconst, Method('strip_underscores', symbol='FLOAT')), (imagconst, Method('strip_underscores', symbol='IMAG')), @@ -136,3 +152,50 @@ def make_lexicon(): #debug_file = scanner_dump_file ) + +# BEGIN GENERATED CODE +# generated with: +# cpython 3.10.0a0 (heads/master:2b0e654f91, May 29 2020, 16:17:52) + +unicode_start_ch_any = ( + u"_ªµºˬˮͿΆΌՙەۿܐޱߺࠚࠤࠨऽॐলঽৎৼਫ਼ઽૐૹଽୱஃஜௐఽಀಽೞഽൎලาຄລາຽໆༀဿၡႎჇჍቘዀៗៜᢪᪧᳺὙ" + u"ὛὝιⁱⁿℂℇℕℤΩℨⅎⴧⴭⵯꣻꧏꩺꪱꫀꫂיִמּﹱﹳﹷﹹﹻﹽ𐠈𐠼𐨀𐼧𑅄𑅇𑅶𑇚𑇜𑊈𑌽𑍐𑓇𑙄𑚸𑤉𑤿𑥁𑧡𑧣𑨀𑨺𑩐𑪝𑱀𑵆𑶘𑾰𖽐𖿣𝒢" + u"𝒻𝕆𞅎𞥋𞸤𞸧𞸹𞸻𞹂𞹇𞹉𞹋𞹔𞹗𞹙𞹛𞹝𞹟𞹤𞹾" +) +unicode_start_ch_range = ( + u"AZazÀÖØöøˁˆˑˠˤͰʹͶͷͻͽΈΊΎΡΣϵϷҁҊԯԱՖՠֈאתׯײؠيٮٯٱۓۥۦۮۯۺۼܒܯݍޥߊߪߴߵࠀࠕ" + u"ࡀࡘࡠࡪࢠࢴࢶࣇऄहक़ॡॱঀঅঌএঐওনপরশহড়ঢ়য়ৡৰৱਅਊਏਐਓਨਪਰਲਲ਼ਵਸ਼ਸਹਖ਼ੜੲੴઅઍએઑઓનપરલળવહ" + u"ૠૡଅଌଏଐଓନପରଲଳଵହଡ଼ଢ଼ୟୡஅஊஎஐஒகஙசஞடணதநபமஹఅఌఎఐఒనపహౘౚౠౡಅಌಎಐಒನಪಳವಹೠೡೱೲ" + u"ഄഌഎഐഒഺൔൖൟൡൺൿඅඖකනඳරවෆกะเๆກຂຆຊຌຣວະເໄໜໟཀཇཉཬྈྌကဪၐၕၚၝၥၦၮၰၵႁႠჅაჺჼቈ" + u"ቊቍቐቖቚቝበኈኊኍነኰኲኵኸኾዂዅወዖዘጐጒጕጘፚᎀᎏᎠᏵᏸᏽᐁᙬᙯᙿᚁᚚᚠᛪᛮᛸᜀᜌᜎᜑᜠᜱᝀᝑᝠᝬᝮᝰកឳᠠᡸᢀᢨ" + u"ᢰᣵᤀᤞᥐᥭᥰᥴᦀᦫᦰᧉᨀᨖᨠᩔᬅᬳᭅᭋᮃᮠᮮᮯᮺᯥᰀᰣᱍᱏᱚᱽᲀᲈᲐᲺᲽᲿᳩᳬᳮᳳᳵᳶᴀᶿḀἕἘἝἠὅὈὍὐὗὟώᾀᾴ" + u"ᾶᾼῂῄῆῌῐΐῖΊῠῬῲῴῶῼₐₜℊℓ℘ℝKℹℼℿⅅⅉⅠↈⰀⰮⰰⱞⱠⳤⳫⳮⳲⳳⴀⴥⴰⵧⶀⶖⶠⶦⶨⶮⶰⶶⶸⶾⷀⷆⷈⷎⷐⷖ" + u"ⷘⷞ々〇〡〩〱〵〸〼ぁゖゝゟァヺーヿㄅㄯㄱㆎㆠㆿㇰㇿ㐀䶿一鿼ꀀꒌꓐꓽꔀꘌꘐꘟꘪꘫꙀꙮꙿꚝꚠꛯꜗꜟꜢꞈꞋꞿꟂꟊꟵꠁꠃꠅꠇꠊ" + u"ꠌꠢꡀꡳꢂꢳꣲꣷꣽꣾꤊꤥꤰꥆꥠꥼꦄꦲꧠꧤꧦꧯꧺꧾꨀꨨꩀꩂꩄꩋꩠꩶꩾꪯꪵꪶꪹꪽꫛꫝꫠꫪꫲꫴꬁꬆꬉꬎꬑꬖꬠꬦꬨꬮꬰꭚꭜꭩꭰꯢ" + u"가힣ힰퟆퟋퟻ豈舘並龎ffstﬓﬗײַﬨשׁזּטּלּנּסּףּפּצּﮱﯓﱝﱤﴽﵐﶏﶒﷇﷰﷹﹿﻼＡＺａｚｦﾝﾠﾾￂￇￊￏￒￗￚￜ𐀀𐀋𐀍𐀦𐀨𐀺" + u"𐀼𐀽𐀿𐁍𐁐𐁝𐂀𐃺𐅀𐅴𐊀𐊜𐊠𐋐𐌀𐌟𐌭𐍊𐍐𐍵𐎀𐎝𐎠𐏃𐏈𐏏𐏑𐏕𐐀𐒝𐒰𐓓𐓘𐓻𐔀𐔧𐔰𐕣𐘀𐜶𐝀𐝕𐝠𐝧𐠀𐠅𐠊𐠵𐠷𐠸𐠿𐡕𐡠𐡶𐢀𐢞𐣠𐣲𐣴𐣵" + u"𐤀𐤕𐤠𐤹𐦀𐦷𐦾𐦿𐨐𐨓𐨕𐨗𐨙𐨵𐩠𐩼𐪀𐪜𐫀𐫇𐫉𐫤𐬀𐬵𐭀𐭕𐭠𐭲𐮀𐮑𐰀𐱈𐲀𐲲𐳀𐳲𐴀𐴣𐺀𐺩𐺰𐺱𐼀𐼜𐼰𐽅𐾰𐿄𐿠𐿶𑀃𑀷𑂃𑂯𑃐𑃨𑄃𑄦𑅐𑅲" + u"𑆃𑆲𑇁𑇄𑈀𑈑𑈓𑈫𑊀𑊆𑊊𑊍𑊏𑊝𑊟𑊨𑊰𑋞𑌅𑌌𑌏𑌐𑌓𑌨𑌪𑌰𑌲𑌳𑌵𑌹𑍝𑍡𑐀𑐴𑑇𑑊𑑟𑑡𑒀𑒯𑓄𑓅𑖀𑖮𑗘𑗛𑘀𑘯𑚀𑚪𑜀𑜚𑠀𑠫𑢠𑣟𑣿𑤆𑤌𑤓" + u"𑤕𑤖𑤘𑤯𑦠𑦧𑦪𑧐𑨋𑨲𑩜𑪉𑫀𑫸𑰀𑰈𑰊𑰮𑱲𑲏𑴀𑴆𑴈𑴉𑴋𑴰𑵠𑵥𑵧𑵨𑵪𑶉𑻠𑻲𒀀𒎙𒐀𒑮𒒀𒕃𓀀𓐮𔐀𔙆𖠀𖨸𖩀𖩞𖫐𖫭𖬀𖬯𖭀𖭃𖭣𖭷𖭽𖮏𖹀𖹿" + u"𖼀𖽊𖾓𖾟𖿠𖿡𗀀𘟷𘠀𘳕𘴀𘴈𛀀𛄞𛅐𛅒𛅤𛅧𛅰𛋻𛰀𛱪𛱰𛱼𛲀𛲈𛲐𛲙𝐀𝑔𝑖𝒜𝒞𝒟𝒥𝒦𝒩𝒬𝒮𝒹𝒽𝓃𝓅𝔅𝔇𝔊𝔍𝔔𝔖𝔜𝔞𝔹𝔻𝔾𝕀𝕄𝕊𝕐𝕒𝚥" + u"𝚨𝛀𝛂𝛚𝛜𝛺𝛼𝜔𝜖𝜴𝜶𝝎𝝐𝝮𝝰𝞈𝞊𝞨𝞪𝟂𝟄𝟋𞄀𞄬𞄷𞄽𞋀𞋫𞠀𞣄𞤀𞥃𞸀𞸃𞸅𞸟𞸡𞸢𞸩𞸲𞸴𞸷𞹍𞹏𞹑𞹒𞹡𞹢𞹧𞹪𞹬𞹲𞹴𞹷𞹹𞹼𞺀𞺉𞺋𞺛" + u"𞺡𞺣𞺥𞺩𞺫𞺻𠀀𪛝𪜀𫜴𫝀𫠝𫠠𬺡𬺰𮯠丽𪘀" +) +unicode_continuation_ch_any = ( + u"··়ׇֿٰܑ߽ৗ਼৾ੑੵ઼଼ஂௗ಼ൗ්ූัັ༹༵༷࿆᳭ᢩ៝᳴⁔⵿⃡꙯ꠂ꠆ꠋ꠬ꧥꩃﬞꪰ꫁＿𑅳𐨿𐇽𐋠𑈾𑍗𑑞𑥀𑧤𑩇𑴺𑵇𖽏𖿤𝩵" + u"𝪄" +) +unicode_continuation_ch_range = ( + u"09ֽׁׂًؚ֑ׅ̀ͯ҃҇ׄؐ٩۪ۭۖۜ۟ۤۧۨ۰۹ܰ݊ަް߀߉࡙࡛࣓ࣣ߫߳ࠖ࠙ࠛࠣࠥࠧࠩ࠭࣡ःऺ़ाॏ॑ॗॢॣ०९ঁঃ" + u"াৄেৈো্ৢৣ০৯ਁਃਾੂੇੈੋ੍੦ੱઁઃાૅેૉો્ૢૣ૦૯ૺ૿ଁଃାୄେୈୋ୍୕ୗୢୣ୦୯ாூெைொ்௦௯ఀఄాౄ" + u"ెైొ్ౕౖౢౣ౦౯ಁಃಾೄೆೈೊ್ೕೖೢೣ೦೯ഀഃ഻഼ാൄെൈൊ്ൢൣ൦൯ඁඃාුෘෟ෦෯ෲෳำฺ็๎๐๙ຳຼ່ໍ໐໙" + u"༘༙༠༩༾༿྄ཱ྆྇ྍྗྙྼါှ၀၉ၖၙၞၠၢၤၧၭၱၴႂႍႏႝ፝፟፩፱ᜒ᜔ᜲ᜴ᝒᝓᝲᝳ឴៓០៩᠋᠍᠐᠙ᤠᤫᤰ᤻᥆᥏᧐᧚" + u"ᨗᨛᩕᩞ᩠᩿᩼᪉᪐᪙᪽ᪿᫀ᪰ᬀᬄ᬴᭄᭐᭙᭫᭳ᮀᮂᮡᮭ᮰᮹᯦᯳ᰤ᰷᱀᱉᱐᱙᳔᳨᳐᳒᳷᷹᷿᳹᷀᷻‿⁀⃥゙゚〪〯⃐⃜⃰⳯⳱ⷠⷿ" + u"꘠꘩ꙴ꙽ꚞꚟ꛰꛱ꠣꠧꢀꢁꢴꣅ꣐꣙꣠꣱ꣿ꤉ꤦ꤭ꥇ꥓ꦀꦃ꦳꧀꧐꧙꧰꧹ꨩꨶꩌꩍ꩐꩙ꩻꩽꪴꪲꪷꪸꪾ꪿ꫫꫯꫵ꫶ꯣꯪ꯬꯭꯰꯹︀️︠︯" + u"︳︴﹍﹏０９ﾞﾟ𐍶𐍺𐒠𐒩𐨁𐨃𐨅𐨆𐨌𐨺𐫦𐨏𐨸𐫥𐴤𐴧𐴰𐴹𐽆𐽐𐺫𐺬𑀀𑀂𑀸𑁆𑁦𑁯𑁿𑂂𑂰𑂺𑃰𑃹𑄀𑄂𑄧𑄴𑄶𑄿𑅅𑅆𑆀𑆂𑆳𑇀𑇉𑇌𑇎𑇙𑈬𑈷" + u"𑋟𑋪𑋰𑋹𑌀𑌃𑌻𑌼𑌾𑍄𑍇𑍈𑍋𑍍𑍢𑍣𑍦𑍬𑍰𑍴𑐵𑑆𑑐𑑙𑒰𑓃𑓐𑓙𑖯𑖵𑖸𑗀𑗜𑗝𑘰𑙀𑙐𑙙𑚫𑚷𑛀𑛉𑜝𑜫𑜰𑜹𑠬𑠺𑣠𑣩𑤰𑤵𑤷𑤸𑤻𑤾𑥂𑥃𑥐𑥙" + u"𑧑𑧗𑧚𑧠𑨁𑨊𑨳𑨹𑨻𑨾𑩑𑩛𑪊𑪙𑰯𑰶𑰸𑰿𑱐𑱙𑲒𑲧𑲩𑲶𑴱𑴶𑴼𑴽𑴿𑵅𑵐𑵙𑶊𑶎𑶐𑶑𑶓𑶗𑶠𑶩𑻳𑻶𖩠𖩩𖫰𖫴𖬰𖬶𖭐𖭙𖽑𖾇𖾏𖾒𖿰𖿱𛲝𛲞𝅩𝅥" + u"𝅲𝅻𝆂𝆋𝅭𝆅𝆪𝆭𝉂𝉄𝟎𝟿𝨀𝨶𝨻𝩬𝪛𝪟𝪡𝪯𞀀𞀆𞀈𞀘𞀛𞀡𞀣𞀤𞀦𞀪𞄰𞄶𞅀𞅉𞋬𞋹𞥊𞣐𞣖𞥄𞥐𞥙🯰🯹" +) + +# END GENERATED CODE diff --git a/Cython/Compiler/Main.py b/Cython/Compiler/Main.py index dc4add541..31c0e1747 100644 --- a/Cython/Compiler/Main.py +++ b/Cython/Compiler/Main.py @@ -9,8 +9,8 @@ import re import sys import io -if sys.version_info[:2] < (2, 6) or (3, 0) <= sys.version_info[:2] < (3, 3): - sys.stderr.write("Sorry, Cython requires Python 2.6+ or 3.3+, found %d.%d\n" % tuple(sys.version_info[:2])) +if sys.version_info[:2] < (2, 7) or (3, 0) <= sys.version_info[:2] < (3, 3): + sys.stderr.write("Sorry, Cython requires Python 2.7 or 3.3+, found %d.%d\n" % tuple(sys.version_info[:2])) sys.exit(1) try: @@ -30,30 +30,28 @@ from .Errors import PyrexError, CompileError, error, warning from .Symtab import ModuleScope from .. import Utils from . import Options +from .Options import CompilationOptions, default_options +from .CmdLine import parse_command_line +from .Lexicon import (unicode_start_ch_any, unicode_continuation_ch_any, + unicode_start_ch_range, unicode_continuation_ch_range) -from . import Version # legacy import needed by old PyTables versions -version = Version.version # legacy attribute - use "Cython.__version__" instead -module_name_pattern = re.compile(r"[A-Za-z_][A-Za-z0-9_]*(\.[A-Za-z_][A-Za-z0-9_]*)*$") +def _make_range_re(chrs): + out = [] + for i in range(0, len(chrs), 2): + out.append(u"{0}-{1}".format(chrs[i], chrs[i+1])) + return u"".join(out) -verbose = 0 +# py2 version looked like r"[A-Za-z_][A-Za-z0-9_]*(\.[A-Za-z_][A-Za-z0-9_]*)*$" +module_name_pattern = u"[{0}{1}][{0}{2}{1}{3}]*".format( + unicode_start_ch_any, _make_range_re(unicode_start_ch_range), + unicode_continuation_ch_any, + _make_range_re(unicode_continuation_ch_range)) +module_name_pattern = re.compile(u"{0}(\\.{0})*$".format(module_name_pattern)) -standard_include_path = os.path.abspath(os.path.join(os.path.dirname(__file__), - os.path.pardir, 'Includes')) -class CompilationData(object): - # Bundles the information that is passed from transform to transform. - # (For now, this is only) - - # While Context contains every pxd ever loaded, path information etc., - # this only contains the data related to a single compilation pass - # - # pyx ModuleNode Main code tree of this compilation. - # pxds {string : ModuleNode} Trees for the pxds used in the pyx. - # codewriter CCodeWriter Where to output final code. - # options CompilationOptions - # result CompilationResult - pass +standard_include_path = os.path.abspath( + os.path.join(os.path.dirname(os.path.dirname(__file__)), 'Includes')) class Context(object): @@ -95,8 +93,13 @@ class Context(object): self.gdb_debug_outputwriter = None + @classmethod + def from_options(cls, options): + return cls(options.include_path, options.compiler_directives, + options.cplus, options.language_level, options=options) + def set_language_level(self, level): - from .Future import print_function, unicode_literals, absolute_import, division + from .Future import print_function, unicode_literals, absolute_import, division, generator_stop future_directives = set() if level == '3str': level = 3 @@ -105,7 +108,7 @@ class Context(object): if level >= 3: future_directives.add(unicode_literals) if level >= 3: - future_directives.update([print_function, absolute_import, division]) + future_directives.update([print_function, absolute_import, division, generator_stop]) self.language_level = level self.future_directives = future_directives if level >= 3: @@ -123,15 +126,6 @@ class Context(object): self._interned[key] = value return value - def intern_value(self, value, *key): - key = (type(value), value) + key - try: - return self._interned[key] - except KeyError: - pass - self._interned[key] = value - return value - # pipeline creation functions can now be found in Pipeline.py def process_pxd(self, source_desc, scope, module_name): @@ -179,7 +173,7 @@ class Context(object): if not module_name_pattern.match(qualified_name): raise CompileError(pos or (module_name, 0, 0), - "'%s' is not a valid module name" % module_name) + u"'%s' is not a valid module name" % module_name) if relative_to: if debug_find_module: @@ -215,8 +209,9 @@ class Context(object): # Set pxd_file_loaded such that we don't need to # look for the non-existing pxd file next time. scope.pxd_file_loaded = True - package_pathname = self.search_include_directories(qualified_name, ".py", pos) - if package_pathname and package_pathname.endswith('__init__.py'): + package_pathname = self.search_include_directories( + qualified_name, suffix=".py", source_pos=pos) + if package_pathname and package_pathname.endswith(Utils.PACKAGE_FILES): pass else: error(pos, "'%s.pxd' not found" % qualified_name.replace('.', os.sep)) @@ -238,7 +233,7 @@ class Context(object): pass return scope - def find_pxd_file(self, qualified_name, pos, sys_path=True): + def find_pxd_file(self, qualified_name, pos=None, sys_path=True, source_file_path=None): # Search include path (and sys.path if sys_path is True) for # the .pxd file corresponding to the given fully-qualified # module name. @@ -247,53 +242,36 @@ class Context(object): # the directory containing the source file is searched first # for a dotted filename, and its containing package root # directory is searched first for a non-dotted filename. - pxd = self.search_include_directories(qualified_name, ".pxd", pos, sys_path=sys_path) - if pxd is None: # XXX Keep this until Includes/Deprecated is removed - if (qualified_name.startswith('python') or - qualified_name in ('stdlib', 'stdio', 'stl')): - standard_include_path = os.path.abspath(os.path.normpath( - os.path.join(os.path.dirname(__file__), os.path.pardir, 'Includes'))) - deprecated_include_path = os.path.join(standard_include_path, 'Deprecated') - self.include_directories.append(deprecated_include_path) - try: - pxd = self.search_include_directories(qualified_name, ".pxd", pos) - finally: - self.include_directories.pop() - if pxd: - name = qualified_name - if name.startswith('python'): - warning(pos, "'%s' is deprecated, use 'cpython'" % name, 1) - elif name in ('stdlib', 'stdio'): - warning(pos, "'%s' is deprecated, use 'libc.%s'" % (name, name), 1) - elif name in ('stl'): - warning(pos, "'%s' is deprecated, use 'libcpp.*.*'" % name, 1) + pxd = self.search_include_directories( + qualified_name, suffix=".pxd", source_pos=pos, sys_path=sys_path, source_file_path=source_file_path) if pxd is None and Options.cimport_from_pyx: return self.find_pyx_file(qualified_name, pos) return pxd - def find_pyx_file(self, qualified_name, pos): + def find_pyx_file(self, qualified_name, pos=None, source_file_path=None): # Search include path for the .pyx file corresponding to the # given fully-qualified module name, as for find_pxd_file(). - return self.search_include_directories(qualified_name, ".pyx", pos) + return self.search_include_directories( + qualified_name, suffix=".pyx", source_pos=pos, source_file_path=source_file_path) - def find_include_file(self, filename, pos): + def find_include_file(self, filename, pos=None, source_file_path=None): # Search list of include directories for filename. # Reports an error and returns None if not found. - path = self.search_include_directories(filename, "", pos, - include=True) + path = self.search_include_directories( + filename, source_pos=pos, include=True, source_file_path=source_file_path) if not path: error(pos, "'%s' not found" % filename) return path - def search_include_directories(self, qualified_name, suffix, pos, - include=False, sys_path=False): + def search_include_directories(self, qualified_name, + suffix=None, source_pos=None, include=False, sys_path=False, source_file_path=None): include_dirs = self.include_directories if sys_path: include_dirs = include_dirs + sys.path # include_dirs must be hashable for caching in @cached_function include_dirs = tuple(include_dirs + [standard_include_path]) - return search_include_directories(include_dirs, qualified_name, - suffix, pos, include) + return search_include_directories( + include_dirs, qualified_name, suffix or "", source_pos, include, source_file_path) def find_root_package_dir(self, file_path): return Utils.find_root_package_dir(file_path) @@ -307,15 +285,14 @@ class Context(object): c_time = Utils.modification_time(output_path) if Utils.file_newer_than(source_path, c_time): return 1 - pos = [source_path] pxd_path = Utils.replace_suffix(source_path, ".pxd") if os.path.exists(pxd_path) and Utils.file_newer_than(pxd_path, c_time): return 1 for kind, name in self.read_dependency_file(source_path): if kind == "cimport": - dep_path = self.find_pxd_file(name, pos) + dep_path = self.find_pxd_file(name, source_file_path=source_path) elif kind == "include": - dep_path = self.search_include_directories(name, pos) + dep_path = self.search_include_directories(name, source_file_path=source_path) else: continue if dep_path and Utils.file_newer_than(dep_path, c_time): @@ -473,22 +450,29 @@ def create_default_resultobj(compilation_source, options): def run_pipeline(source, options, full_module_name=None, context=None): from . import Pipeline + # ensure that the inputs are unicode (for Python 2) + if sys.version_info[0] == 2: + source = Utils.decode_filename(source) + if full_module_name: + full_module_name = Utils.decode_filename(full_module_name) + source_ext = os.path.splitext(source)[1] - options.configure_language_defaults(source_ext[1:]) # py/pyx + options.configure_language_defaults(source_ext[1:]) # py/pyx if context is None: - context = options.create_context() + context = Context.from_options(options) # Set up source object cwd = os.getcwd() abs_path = os.path.abspath(source) full_module_name = full_module_name or context.extract_module_name(source, options) + full_module_name = EncodedString(full_module_name) Utils.raise_error_if_module_name_forbidden(full_module_name) if options.relative_path_in_code_position_comments: rel_path = full_module_name.replace('.', os.sep) + source_ext if not abs_path.endswith(rel_path): - rel_path = source # safety measure to prevent printing incorrect paths + rel_path = source # safety measure to prevent printing incorrect paths else: rel_path = abs_path source_desc = FileSourceDescriptor(abs_path, rel_path) @@ -512,6 +496,12 @@ def run_pipeline(source, options, full_module_name=None, context=None): pipeline = Pipeline.create_pyx_pipeline(context, options, result) context.setup_errors(options, result) + + if '.' in full_module_name and '.' in os.path.splitext(os.path.basename(abs_path))[0]: + warning((source_desc, 1, 0), + "Dotted filenames ('%s') are deprecated." + " Please use the normal Python package directory layout." % os.path.basename(abs_path), level=1) + err, enddata = Pipeline.run_pipeline(pipeline, source) context.teardown_errors(err, options, result) return result @@ -534,146 +524,6 @@ class CompilationSource(object): self.cwd = cwd -class CompilationOptions(object): - r""" - See default_options at the end of this module for a list of all possible - options and CmdLine.usage and CmdLine.parse_command_line() for their - meaning. - """ - def __init__(self, defaults=None, **kw): - self.include_path = [] - if defaults: - if isinstance(defaults, CompilationOptions): - defaults = defaults.__dict__ - else: - defaults = default_options - - options = dict(defaults) - options.update(kw) - - # let's assume 'default_options' contains a value for most known compiler options - # and validate against them - unknown_options = set(options) - set(default_options) - # ignore valid options that are not in the defaults - unknown_options.difference_update(['include_path']) - if unknown_options: - message = "got unknown compilation option%s, please remove: %s" % ( - 's' if len(unknown_options) > 1 else '', - ', '.join(unknown_options)) - raise ValueError(message) - - directive_defaults = Options.get_directive_defaults() - directives = dict(options['compiler_directives']) # copy mutable field - # check for invalid directives - unknown_directives = set(directives) - set(directive_defaults) - if unknown_directives: - message = "got unknown compiler directive%s: %s" % ( - 's' if len(unknown_directives) > 1 else '', - ', '.join(unknown_directives)) - raise ValueError(message) - options['compiler_directives'] = directives - if directives.get('np_pythran', False) and not options['cplus']: - import warnings - warnings.warn("C++ mode forced when in Pythran mode!") - options['cplus'] = True - if 'language_level' in directives and 'language_level' not in kw: - options['language_level'] = directives['language_level'] - elif not options.get('language_level'): - options['language_level'] = directive_defaults.get('language_level') - if 'formal_grammar' in directives and 'formal_grammar' not in kw: - options['formal_grammar'] = directives['formal_grammar'] - if options['cache'] is True: - options['cache'] = os.path.join(Utils.get_cython_cache_dir(), 'compiler') - - self.__dict__.update(options) - - def configure_language_defaults(self, source_extension): - if source_extension == 'py': - if self.compiler_directives.get('binding') is None: - self.compiler_directives['binding'] = True - - def create_context(self): - return Context(self.include_path, self.compiler_directives, - self.cplus, self.language_level, options=self) - - def get_fingerprint(self): - r""" - Return a string that contains all the options that are relevant for cache invalidation. - """ - # Collect only the data that can affect the generated file(s). - data = {} - - for key, value in self.__dict__.items(): - if key in ['show_version', 'errors_to_stderr', 'verbose', 'quiet']: - # verbosity flags have no influence on the compilation result - continue - elif key in ['output_file', 'output_dir']: - # ignore the exact name of the output file - continue - elif key in ['timestamps']: - # the cache cares about the content of files, not about the timestamps of sources - continue - elif key in ['cache']: - # hopefully caching has no influence on the compilation result - continue - elif key in ['compiler_directives']: - # directives passed on to the C compiler do not influence the generated C code - continue - elif key in ['include_path']: - # this path changes which headers are tracked as dependencies, - # it has no influence on the generated C code - continue - elif key in ['working_path']: - # this path changes where modules and pxd files are found; - # their content is part of the fingerprint anyway, their - # absolute path does not matter - continue - elif key in ['create_extension']: - # create_extension() has already mangled the options, e.g., - # embedded_metadata, when the fingerprint is computed so we - # ignore it here. - continue - elif key in ['build_dir']: - # the (temporary) directory where we collect dependencies - # has no influence on the C output - continue - elif key in ['use_listing_file', 'generate_pxi', 'annotate', 'annotate_coverage_xml']: - # all output files are contained in the cache so the types of - # files generated must be part of the fingerprint - data[key] = value - elif key in ['formal_grammar', 'evaluate_tree_assertions']: - # these bits can change whether compilation to C passes/fails - data[key] = value - elif key in ['embedded_metadata', 'emit_linenums', 'c_line_in_traceback', 'gdb_debug', 'relative_path_in_code_position_comments']: - # the generated code contains additional bits when these are set - data[key] = value - elif key in ['cplus', 'language_level', 'compile_time_env', 'np_pythran']: - # assorted bits that, e.g., influence the parser - data[key] = value - elif key == ['capi_reexport_cincludes']: - if self.capi_reexport_cincludes: - # our caching implementation does not yet include fingerprints of all the header files - raise NotImplementedError('capi_reexport_cincludes is not compatible with Cython caching') - elif key == ['common_utility_include_dir']: - if self.common_utility_include_dir: - raise NotImplementedError('common_utility_include_dir is not compatible with Cython caching yet') - else: - # any unexpected option should go into the fingerprint; it's better - # to recompile than to return incorrect results from the cache. - data[key] = value - - def to_fingerprint(item): - r""" - Recursively turn item into a string, turning dicts into lists with - deterministic ordering. - """ - if isinstance(item, dict): - item = sorted([(repr(key), to_fingerprint(value)) for key, value in item.items()]) - return repr(item) - - return to_fingerprint(data) - - class CompilationResult(object): """ Results from the Cython compiler: @@ -736,7 +586,7 @@ def compile_multiple(sources, options): if these are specified in the options. """ # run_pipeline creates the context - # context = options.create_context() + # context = Context.from_options(options) sources = [os.path.abspath(source) for source in sources] processed = set() results = CompilationResultSet() @@ -747,7 +597,7 @@ def compile_multiple(sources, options): for source in sources: if source not in processed: if context is None: - context = options.create_context() + context = Context.from_options(options) output_filename = get_output_filename(source, cwd, options) out_of_date = context.c_file_out_of_date(source, output_filename) if (not timestamps) or out_of_date: @@ -781,55 +631,79 @@ def compile(source, options = None, full_module_name = None, **kwds): @Utils.cached_function -def search_include_directories(dirs, qualified_name, suffix, pos, include=False): +def search_include_directories(dirs, qualified_name, suffix="", pos=None, include=False, source_file_path=None): """ Search the list of include directories for the given file name. - If a source file position is given, first searches the directory - containing that file. Returns None if not found, but does not - report an error. + If a source file path or position is given, first searches the directory + containing that file. Returns None if not found, but does not report an error. The 'include' option will disable package dereferencing. """ - - if pos: + if pos and not source_file_path: file_desc = pos[0] if not isinstance(file_desc, FileSourceDescriptor): raise RuntimeError("Only file sources for code supported") + source_file_path = file_desc.filename + if source_file_path: if include: - dirs = (os.path.dirname(file_desc.filename),) + dirs + dirs = (os.path.dirname(source_file_path),) + dirs else: - dirs = (Utils.find_root_package_dir(file_desc.filename),) + dirs + dirs = (Utils.find_root_package_dir(source_file_path),) + dirs + # search for dotted filename e.g. <dir>/foo.bar.pxd dotted_filename = qualified_name if suffix: dotted_filename += suffix - if not include: - names = qualified_name.split('.') - package_names = tuple(names[:-1]) - module_name = names[-1] - module_filename = module_name + suffix - package_filename = "__init__" + suffix - for dirname in dirs: path = os.path.join(dirname, dotted_filename) if os.path.exists(path): + if '.' in qualified_name and '.' in os.path.splitext(dotted_filename)[0]: + warning(pos, "Dotted filenames ('%s') are deprecated." + " Please use the normal Python package directory layout." % dotted_filename, level=1) return path - if not include: - package_dir = Utils.check_package_dir(dirname, package_names) + # search for filename in package structure e.g. <dir>/foo/bar.pxd or <dir>/foo/bar/__init__.pxd + if not include: + + names = qualified_name.split('.') + package_names = tuple(names[:-1]) + module_name = names[-1] + + # search for standard packages first - PEP420 + namespace_dirs = [] + for dirname in dirs: + package_dir, is_namespace = Utils.check_package_dir(dirname, package_names) if package_dir is not None: - path = os.path.join(package_dir, module_filename) - if os.path.exists(path): - return path - path = os.path.join(package_dir, module_name, - package_filename) - if os.path.exists(path): + if is_namespace: + namespace_dirs.append(package_dir) + continue + path = search_module_in_dir(package_dir, module_name, suffix) + if path: return path + + # search for namespaces second - PEP420 + for package_dir in namespace_dirs: + path = search_module_in_dir(package_dir, module_name, suffix) + if path: + return path + return None +@Utils.cached_function +def search_module_in_dir(package_dir, module_name, suffix): + # matches modules of the form: <dir>/foo/bar.pxd + path = Utils.find_versioned_file(package_dir, module_name, suffix) + + # matches modules of the form: <dir>/foo/bar/__init__.pxd + if not path and suffix: + path = Utils.find_versioned_file(os.path.join(package_dir, module_name), "__init__", suffix) + + return path + + # ------------------------------------------------------------------------ # # Main command-line entry point @@ -844,14 +718,14 @@ def main(command_line = 0): args = sys.argv[1:] any_failures = 0 if command_line: - from .CmdLine import parse_command_line options, sources = parse_command_line(args) else: options = CompilationOptions(default_options) sources = args if options.show_version: - sys.stderr.write("Cython version %s\n" % version) + from .. import __version__ + sys.stderr.write("Cython version %s\n" % __version__) if options.working_path!="": os.chdir(options.working_path) try: @@ -863,42 +737,3 @@ def main(command_line = 0): any_failures = 1 if any_failures: sys.exit(1) - - -# ------------------------------------------------------------------------ -# -# Set the default options depending on the platform -# -# ------------------------------------------------------------------------ - -default_options = dict( - show_version = 0, - use_listing_file = 0, - errors_to_stderr = 1, - cplus = 0, - output_file = None, - annotate = None, - annotate_coverage_xml = None, - generate_pxi = 0, - capi_reexport_cincludes = 0, - working_path = "", - timestamps = None, - verbose = 0, - quiet = 0, - compiler_directives = {}, - embedded_metadata = {}, - evaluate_tree_assertions = False, - emit_linenums = False, - relative_path_in_code_position_comments = True, - c_line_in_traceback = True, - language_level = None, # warn but default to 2 - formal_grammar = False, - gdb_debug = False, - compile_time_env = None, - common_utility_include_dir = None, - output_dir=None, - build_dir=None, - cache=None, - create_extension=None, - np_pythran=False -) diff --git a/Cython/Compiler/MemoryView.py b/Cython/Compiler/MemoryView.py index 0406d6c71..d39c6b36f 100644 --- a/Cython/Compiler/MemoryView.py +++ b/Cython/Compiler/MemoryView.py @@ -22,10 +22,6 @@ ERR_UNINITIALIZED = ("Cannot check if memoryview %s is initialized without the " "GIL, consider using initializedcheck(False)") -def concat_flags(*flags): - return "(%s)" % "|".join(flags) - - format_flag = "PyBUF_FORMAT" memview_c_contiguous = "(PyBUF_C_CONTIGUOUS | PyBUF_FORMAT)" @@ -100,8 +96,15 @@ def put_acquire_memoryviewslice(lhs_cname, lhs_type, lhs_pos, rhs, code, def put_assign_to_memviewslice(lhs_cname, rhs, rhs_cname, memviewslicetype, code, have_gil=False, first_assignment=False): + if lhs_cname == rhs_cname: + # self assignment is tricky because memoryview xdecref clears the memoryview + # thus invalidating both sides of the assignment. Therefore make it actually do nothing + code.putln("/* memoryview self assignment no-op */") + return + if not first_assignment: - code.put_xdecref_memoryviewslice(lhs_cname, have_gil=have_gil) + code.put_xdecref(lhs_cname, memviewslicetype, + have_gil=have_gil) if not rhs.result_in_temp(): rhs.make_owned_memoryviewslice(code) @@ -167,7 +170,7 @@ def valid_memslice_dtype(dtype, i=0): valid_memslice_dtype(dtype.base_type, i + 1)) or dtype.is_numeric or dtype.is_pyobject or - dtype.is_fused or # accept this as it will be replaced by specializations later + dtype.is_fused or # accept this as it will be replaced by specializations later (dtype.is_typedef and valid_memslice_dtype(dtype.typedef_base_type)) ) @@ -248,7 +251,7 @@ class MemoryViewSliceBufferEntry(Buffer.BufferEntry): return bufp - def generate_buffer_slice_code(self, code, indices, dst, have_gil, + def generate_buffer_slice_code(self, code, indices, dst, dst_type, have_gil, have_slices, directives): """ Slice a memoryviewslice. @@ -265,7 +268,7 @@ class MemoryViewSliceBufferEntry(Buffer.BufferEntry): code.putln("%(dst)s.data = %(src)s.data;" % locals()) code.putln("%(dst)s.memview = %(src)s.memview;" % locals()) - code.put_incref_memoryviewslice(dst) + code.put_incref_memoryviewslice(dst, dst_type, have_gil=have_gil) all_dimensions_direct = all(access == 'direct' for access, packing in self.type.axes) suboffset_dim_temp = [] @@ -404,8 +407,8 @@ def get_is_contig_utility(contig_type, ndim): return utility -def slice_iter(slice_type, slice_result, ndim, code): - if slice_type.is_c_contig or slice_type.is_f_contig: +def slice_iter(slice_type, slice_result, ndim, code, force_strided=False): + if (slice_type.is_c_contig or slice_type.is_f_contig) and not force_strided: return ContigSliceIter(slice_type, slice_result, ndim, code) else: return StridedSliceIter(slice_type, slice_result, ndim, code) @@ -489,7 +492,7 @@ def copy_c_or_fortran_cname(memview): def get_copy_new_utility(pos, from_memview, to_memview): if (from_memview.dtype != to_memview.dtype and - not (from_memview.dtype.is_const and from_memview.dtype.const_base_type == to_memview.dtype)): + not (from_memview.dtype.is_cv_qualified and from_memview.dtype.cv_base_type == to_memview.dtype)): error(pos, "dtypes must be the same!") return if len(from_memview.axes) != len(to_memview.axes): @@ -507,7 +510,8 @@ def get_copy_new_utility(pos, from_memview, to_memview): if to_memview.is_c_contig: mode = 'c' contig_flag = memview_c_contiguous - elif to_memview.is_f_contig: + else: + assert to_memview.is_f_contig mode = 'fortran' contig_flag = memview_f_contiguous @@ -654,13 +658,13 @@ def is_cf_contig(specs): is_c_contig = True elif (specs[-1] == ('direct','contig') and - all(axis == ('direct','follow') for axis in specs[:-1])): + all(axis == ('direct','follow') for axis in specs[:-1])): # c_contiguous: 'follow', 'follow', ..., 'follow', 'contig' is_c_contig = True elif (len(specs) > 1 and - specs[0] == ('direct','contig') and - all(axis == ('direct','follow') for axis in specs[1:])): + specs[0] == ('direct','contig') and + all(axis == ('direct','follow') for axis in specs[1:])): # f_contiguous: 'contig', 'follow', 'follow', ..., 'follow' is_f_contig = True @@ -809,7 +813,7 @@ context = { 'memview_struct_name': memview_objstruct_cname, 'max_dims': Options.buffer_max_dims, 'memviewslice_name': memviewslice_cname, - 'memslice_init': memslice_entry_init, + 'memslice_init': PyrexTypes.MemoryViewSliceType.default_value, } memviewslice_declare_code = load_memview_c_utility( "MemviewSliceStruct", @@ -835,7 +839,7 @@ overlapping_utility = load_memview_c_utility("OverlappingSlices", context) copy_contents_new_utility = load_memview_c_utility( "MemviewSliceCopyTemplate", context, - requires=[], # require cython_array_utility_code + requires=[], # require cython_array_utility_code ) view_utility_code = load_memview_cy_utility( @@ -850,7 +854,7 @@ view_utility_code = load_memview_cy_utility( copy_contents_new_utility, ModuleNode.capsule_utility_code], ) -view_utility_whitelist = ('array', 'memoryview', 'array_cwrapper', +view_utility_allowlist = ('array', 'memoryview', 'array_cwrapper', 'generic', 'strided', 'indirect', 'contiguous', 'indirect_contiguous') diff --git a/Cython/Compiler/ModuleNode.py b/Cython/Compiler/ModuleNode.py index f89af8ca5..d0687624f 100644 --- a/Cython/Compiler/ModuleNode.py +++ b/Cython/Compiler/ModuleNode.py @@ -14,6 +14,7 @@ import json import operator import os import re +import sys from .PyrexTypes import CPtrType from . import Future @@ -26,13 +27,25 @@ from . import TypeSlots from . import PyrexTypes from . import Pythran -from .Errors import error, warning +from .Errors import error, warning, CompileError from .PyrexTypes import py_object_type -from ..Utils import open_new_file, replace_suffix, decode_filename, build_hex_version -from .Code import UtilityCode, IncludeCode -from .StringEncoding import EncodedString +from ..Utils import open_new_file, replace_suffix, decode_filename, build_hex_version, is_cython_generated_file +from .Code import UtilityCode, IncludeCode, TempitaUtilityCode +from .StringEncoding import EncodedString, encoded_string_or_bytes_literal from .Pythran import has_np_pythran + +def replace_suffix_encoded(path, newsuf): + # calls replace suffix and returns a EncodedString or BytesLiteral with the encoding set + newpath = replace_suffix(path, newsuf) + return as_encoded_filename(newpath) + +def as_encoded_filename(path): + # wraps the path with either EncodedString or BytesLiteral (depending on its input type) + # and sets the encoding to the file system encoding + return encoded_string_or_bytes_literal(path, sys.getfilesystemencoding()) + + def check_c_declarations_pxd(module_node): module_node.scope.check_c_classes_pxd() return module_node @@ -50,11 +63,50 @@ def generate_c_code_config(env, options): else: emit_linenums = options.emit_linenums + if hasattr(options, "emit_code_comments"): + print('Warning: option emit_code_comments is deprecated. ' + 'Instead, use compiler directive emit_code_comments.') + return Code.CCodeConfig( emit_linenums=emit_linenums, emit_code_comments=env.directives['emit_code_comments'], c_line_in_traceback=options.c_line_in_traceback) +# The code required to generate one comparison from another. +# The keys are (from, to). +# The comparison operator always goes first, with equality possibly second. +# The first value specifies if the comparison is inverted. The second is the +# logic op to use, and the third is if the equality is inverted or not. +TOTAL_ORDERING = { + # a > b from (not a < b) and (a != b) + ('__lt__', '__gt__'): (True, '&&', True), + # a <= b from (a < b) or (a == b) + ('__lt__', '__le__'): (False, '||', False), + # a >= b from (not a < b). + ('__lt__', '__ge__'): (True, '', None), + + # a >= b from (not a <= b) or (a == b) + ('__le__', '__ge__'): (True, '||', False), + # a < b, from (a <= b) and (a != b) + ('__le__', '__lt__'): (False, '&&', True), + # a > b from (not a <= b) + ('__le__', '__gt__'): (True, '', None), + + # a < b from (not a > b) and (a != b) + ('__gt__', '__lt__'): (True, '&&', True), + # a >= b from (a > b) or (a == b) + ('__gt__', '__ge__'): (False, '||', False), + # a <= b from (not a > b) + ('__gt__', '__le__'): (True, '', None), + + # Return a <= b from (not a >= b) or (a == b) + ('__ge__', '__le__'): (True, '||', False), + # a > b from (a >= b) and (a != b) + ('__ge__', '__gt__'): (False, '&&', True), + # a < b from (not a >= b) + ('__ge__', '__lt__'): (True, '', None), +} + class ModuleNode(Nodes.Node, Nodes.BlockNode): # doc string or None @@ -80,6 +132,10 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): # CodeGenerator, and tell that CodeGenerator to generate code # from multiple sources. assert isinstance(self.body, Nodes.StatListNode) + if scope.directives != self.scope.directives: + # merged in nodes should keep their original compiler directives + # (for example inline cdef functions) + tree = Nodes.CompilerDirectivesNode(tree.pos, body=tree, directives=scope.directives) if isinstance(tree, Nodes.StatListNode): self.body.stats.extend(tree.stats) else: @@ -105,6 +161,13 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): self.scope.merge_in(scope) + def with_compiler_directives(self): + # When merging a utility code module into the user code we need to preserve + # the original compiler directives. This returns the body of the module node, + # wrapped in its set of directives. + body = Nodes.CompilerDirectivesNode(self.pos, directives=self.directives, body=self.body) + return body + def analyse_declarations(self, env): if has_np_pythran(env): Pythran.include_pythran_generic(env) @@ -131,8 +194,8 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): self.create_import_star_conversion_utility_code(env) for name, entry in sorted(env.entries.items()): if (entry.create_wrapper and entry.scope is env - and entry.is_type and entry.type.is_enum): - entry.type.create_type_wrapper(env) + and entry.is_type and (entry.type.is_enum or entry.type.is_cpp_enum)): + entry.type.create_type_wrapper(env) def process_implementation(self, options, result): env = self.scope @@ -151,6 +214,14 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): return 1 return 0 + def assure_safe_target(self, path, allow_failed=False): + # Check for a common gotcha for new users: naming your .pyx file after the .c file you want to wrap + if not is_cython_generated_file(path, allow_failed=allow_failed, if_not_found=True): + # Raising a fatal CompileError instead of calling error() to prevent castrating an existing file. + raise CompileError( + self.pos, 'The output file already exists and does not look like it was generated by Cython: "%s"' % + os.path.basename(path)) + def generate_h_code(self, env, options, result): def h_entries(entries, api=0, pxd=0): return [entry for entry in entries @@ -161,65 +232,99 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): h_vars = h_entries(env.var_entries) h_funcs = h_entries(env.cfunc_entries) h_extension_types = h_entries(env.c_class_entries) - if h_types or h_vars or h_funcs or h_extension_types: - result.h_file = replace_suffix(result.c_file, ".h") - h_code = Code.CCodeWriter() + + if h_types or h_vars or h_funcs or h_extension_types: + result.h_file = replace_suffix_encoded(result.c_file, ".h") + self.assure_safe_target(result.h_file) + + h_code_writer = Code.CCodeWriter() c_code_config = generate_c_code_config(env, options) - Code.GlobalState(h_code, self, c_code_config) + globalstate = Code.GlobalState(h_code_writer, self, c_code_config) + globalstate.initialize_main_h_code() # in-case utility code is used in the header + h_code_start = globalstate.parts['h_code'] + h_code_main = globalstate.parts['type_declarations'] + h_code_end = globalstate.parts['end'] if options.generate_pxi: - result.i_file = replace_suffix(result.c_file, ".pxi") + result.i_file = replace_suffix_encoded(result.c_file, ".pxi") i_code = Code.PyrexCodeWriter(result.i_file) else: i_code = None - h_code.put_generated_by() - h_guard = Naming.h_guard_prefix + self.api_name(env) - h_code.put_h_guard(h_guard) - h_code.putln("") - h_code.putln('#include "Python.h"') - self.generate_type_header_code(h_types, h_code) + h_code_start.put_generated_by() + h_guard = self.api_name(Naming.h_guard_prefix, env) + h_code_start.put_h_guard(h_guard) + h_code_start.putln("") + h_code_start.putln('#include "Python.h"') + self.generate_type_header_code(h_types, h_code_start) if options.capi_reexport_cincludes: - self.generate_includes(env, [], h_code) - h_code.putln("") - api_guard = Naming.api_guard_prefix + self.api_name(env) - h_code.putln("#ifndef %s" % api_guard) - h_code.putln("") - self.generate_extern_c_macro_definition(h_code) - h_code.putln("") - self.generate_dl_import_macro(h_code) + self.generate_includes(env, [], h_code_start) + h_code_start.putln("") + api_guard = self.api_name(Naming.api_guard_prefix, env) + h_code_start.putln("#ifndef %s" % api_guard) + h_code_start.putln("") + self.generate_extern_c_macro_definition(h_code_start) + h_code_start.putln("") + self.generate_dl_import_macro(h_code_start) if h_extension_types: - h_code.putln("") + h_code_main.putln("") for entry in h_extension_types: - self.generate_cclass_header_code(entry.type, h_code) + self.generate_cclass_header_code(entry.type, h_code_main) if i_code: self.generate_cclass_include_code(entry.type, i_code) if h_funcs: - h_code.putln("") + h_code_main.putln("") for entry in h_funcs: - self.generate_public_declaration(entry, h_code, i_code) + self.generate_public_declaration(entry, h_code_main, i_code) if h_vars: - h_code.putln("") + h_code_main.putln("") for entry in h_vars: - self.generate_public_declaration(entry, h_code, i_code) - h_code.putln("") - h_code.putln("#endif /* !%s */" % api_guard) - h_code.putln("") - h_code.putln("/* WARNING: the interface of the module init function changed in CPython 3.5. */") - h_code.putln("/* It now returns a PyModuleDef instance instead of a PyModule instance. */") - h_code.putln("") - h_code.putln("#if PY_MAJOR_VERSION < 3") - h_code.putln("PyMODINIT_FUNC init%s(void);" % env.module_name) - h_code.putln("#else") - h_code.putln("PyMODINIT_FUNC %s(void);" % self.mod_init_func_cname('PyInit', env)) - h_code.putln("#endif") - h_code.putln("") - h_code.putln("#endif /* !%s */" % h_guard) - - f = open_new_file(result.h_file) - try: - h_code.copyto(f) - finally: - f.close() + self.generate_public_declaration(entry, h_code_main, i_code) + h_code_main.putln("") + h_code_main.putln("#endif /* !%s */" % api_guard) + h_code_main.putln("") + h_code_main.putln("/* WARNING: the interface of the module init function changed in CPython 3.5. */") + h_code_main.putln("/* It now returns a PyModuleDef instance instead of a PyModule instance. */") + h_code_main.putln("") + h_code_main.putln("#if PY_MAJOR_VERSION < 3") + if env.module_name.isascii(): + py2_mod_name = env.module_name + else: + py2_mod_name = env.module_name.encode("ascii", errors="ignore").decode("utf-8") + h_code_main.putln('#error "Unicode module names are not supported in Python 2";') + h_code_main.putln("PyMODINIT_FUNC init%s(void);" % py2_mod_name) + h_code_main.putln("#else") + py3_mod_func_name = self.mod_init_func_cname('PyInit', env) + warning_string = EncodedString('Use PyImport_AppendInittab("%s", %s) instead of calling %s directly.' % ( + py2_mod_name, py3_mod_func_name, py3_mod_func_name)) + h_code_main.putln('/* WARNING: %s from Python 3.5 */' % warning_string.rstrip('.')) + h_code_main.putln("PyMODINIT_FUNC %s(void);" % py3_mod_func_name) + h_code_main.putln("") + h_code_main.putln("#if PY_VERSION_HEX >= 0x03050000 " + "&& (defined(__GNUC__) || defined(__clang__) || defined(_MSC_VER) " + "|| (defined(__cplusplus) && __cplusplus >= 201402L))") + h_code_main.putln("#if defined(__cplusplus) && __cplusplus >= 201402L") + h_code_main.putln("[[deprecated(%s)]] inline" % warning_string.as_c_string_literal()) + h_code_main.putln("#elif defined(__GNUC__) || defined(__clang__)") + h_code_main.putln('__attribute__ ((__deprecated__(%s), __unused__)) __inline__' % ( + warning_string.as_c_string_literal())) + h_code_main.putln("#elif defined(_MSC_VER)") + h_code_main.putln('__declspec(deprecated(%s)) __inline' % ( + warning_string.as_c_string_literal())) + h_code_main.putln('#endif') + h_code_main.putln("static PyObject* __PYX_WARN_IF_%s_INIT_CALLED(PyObject* res) {" % py3_mod_func_name) + h_code_main.putln("return res;") + h_code_main.putln("}") + # Function call is converted to warning macro; uncalled (pointer) is not + h_code_main.putln('#define %s() __PYX_WARN_IF_%s_INIT_CALLED(%s())' % ( + py3_mod_func_name, py3_mod_func_name, py3_mod_func_name)) + h_code_main.putln('#endif') + h_code_main.putln('#endif') + + h_code_end.putln("") + h_code_end.putln("#endif /* !%s */" % h_guard) + + with open_new_file(result.h_file) as f: + h_code_writer.copyto(f) def generate_public_declaration(self, entry, h_code, i_code): h_code.putln("%s %s;" % ( @@ -229,8 +334,9 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): i_code.putln("cdef extern %s" % ( entry.type.declaration_code(entry.cname, pyrex=1))) - def api_name(self, env): - return env.qualified_name.replace(".", "__") + def api_name(self, prefix, env): + api_name = self.punycode_module_name(prefix, env.qualified_name) + return api_name.replace(".", "__") def generate_api_code(self, env, options, result): def api_entries(entries, pxd=0): @@ -239,13 +345,16 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): api_vars = api_entries(env.var_entries) api_funcs = api_entries(env.cfunc_entries) api_extension_types = api_entries(env.c_class_entries) + if api_vars or api_funcs or api_extension_types: - result.api_file = replace_suffix(result.c_file, "_api.h") + result.api_file = replace_suffix_encoded(result.c_file, "_api.h") + self.assure_safe_target(result.api_file) + h_code = Code.CCodeWriter() c_code_config = generate_c_code_config(env, options) Code.GlobalState(h_code, self, c_code_config) h_code.put_generated_by() - api_guard = Naming.api_guard_prefix + self.api_name(env) + api_guard = self.api_name(Naming.api_guard_prefix, env) h_code.put_h_guard(api_guard) # Work around https://bugs.python.org/issue4709 h_code.putln('#ifdef __MINGW64__') @@ -254,7 +363,9 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): h_code.putln('#include "Python.h"') if result.h_file: - h_code.putln('#include "%s"' % os.path.basename(result.h_file)) + h_filename = os.path.basename(result.h_file) + h_filename = as_encoded_filename(h_filename) + h_code.putln('#include %s' % h_filename.as_c_string_literal()) if api_extension_types: h_code.putln("") for entry in api_extension_types: @@ -274,7 +385,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): for entry in api_vars: type = CPtrType(entry.type) cname = env.mangle(Naming.varptr_prefix_api, entry.name) - h_code.putln("static %s = 0;" % type.declaration_code(cname)) + h_code.putln("static %s = 0;" % type.declaration_code(cname)) h_code.putln("#define %s (*%s)" % (entry.name, cname)) h_code.put(UtilityCode.load_as_string("PyIdentifierFromString", "ImportExport.c")[0]) if api_vars: @@ -285,22 +396,22 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): h_code.put(UtilityCode.load_as_string("TypeImport", "ImportExport.c")[0]) h_code.put(UtilityCode.load_as_string("TypeImport", "ImportExport.c")[1]) h_code.putln("") - h_code.putln("static int import_%s(void) {" % self.api_name(env)) + h_code.putln("static int %s(void) {" % self.api_name("import", env)) h_code.putln("PyObject *module = 0;") - h_code.putln('module = PyImport_ImportModule("%s");' % env.qualified_name) + h_code.putln('module = PyImport_ImportModule(%s);' % env.qualified_name.as_c_string_literal()) h_code.putln("if (!module) goto bad;") for entry in api_funcs: cname = env.mangle(Naming.func_prefix_api, entry.name) sig = entry.type.signature_string() h_code.putln( - 'if (__Pyx_ImportFunction(module, "%s", (void (**)(void))&%s, "%s") < 0) goto bad;' - % (entry.name, cname, sig)) + 'if (__Pyx_ImportFunction(module, %s, (void (**)(void))&%s, "%s") < 0) goto bad;' + % (entry.name.as_c_string_literal(), cname, sig)) for entry in api_vars: cname = env.mangle(Naming.varptr_prefix_api, entry.name) sig = entry.type.empty_declaration_code() h_code.putln( - 'if (__Pyx_ImportVoidPtr(module, "%s", (void **)&%s, "%s") < 0) goto bad;' - % (entry.name, cname, sig)) + 'if (__Pyx_ImportVoidPtr(module, %s, (void **)&%s, "%s") < 0) goto bad;' + % (entry.name.as_c_string_literal(), cname, sig)) with ModuleImportGenerator(h_code, imported_modules={env.qualified_name: 'module'}) as import_generator: for entry in api_extension_types: self.generate_type_import_call(entry.type, h_code, import_generator, error_code="goto bad;") @@ -339,10 +450,15 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): i_code.dedent() def generate_c_code(self, env, options, result): + self.assure_safe_target(result.c_file, allow_failed=True) modules = self.referenced_modules if Options.annotate or options.annotate: - rootwriter = Annotate.AnnotationCCodeWriter() + show_entire_c_code = Options.annotate == "fullc" or options.annotate == "fullc" + rootwriter = Annotate.AnnotationCCodeWriter( + show_entire_c_code=show_entire_c_code, + source_desc=self.compilation_source.source_desc, + ) else: rootwriter = Code.CCodeWriter() @@ -364,24 +480,24 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): globalstate.use_utility_code(refnanny_utility_code) code = globalstate['before_global_var'] - code.putln('#define __Pyx_MODULE_NAME "%s"' % self.full_module_name) - module_is_main = "%s%s" % (Naming.module_is_main, self.full_module_name.replace('.', '__')) + code.putln('#define __Pyx_MODULE_NAME %s' % + self.full_module_name.as_c_string_literal()) + module_is_main = self.is_main_module_flag_cname() code.putln("extern int %s;" % module_is_main) code.putln("int %s = 0;" % module_is_main) code.putln("") - code.putln("/* Implementation of '%s' */" % env.qualified_name) + code.putln("/* Implementation of %s */" % env.qualified_name.as_c_string_literal()) code = globalstate['late_includes'] - code.putln("/* Late includes */") self.generate_includes(env, modules, code, early=False) - code = globalstate['all_the_rest'] + code = globalstate['module_code'] self.generate_cached_builtins_decls(env, code) - self.generate_lambda_definitions(env, code) + # generate normal variable and function definitions + self.generate_lambda_definitions(env, code) self.generate_variable_definitions(env, code) - self.body.generate_function_definitions(env, code) code.mark_pos(None) @@ -389,11 +505,15 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): self.generate_method_table(env, code) if env.has_import_star: self.generate_import_star(env, code) - self.generate_pymoduledef_struct(env, code) # initialise the macro to reduce the code size of one-time functionality code.putln(UtilityCode.load_as_string("SmallCodeConfig", "ModuleSetupCode.c")[0].strip()) + self.generate_module_state_start(env, globalstate['module_state']) + self.generate_module_state_defines(env, globalstate['module_state_defines']) + self.generate_module_state_clear(env, globalstate['module_state_clear']) + self.generate_module_state_traverse(env, globalstate['module_state_traverse']) + # init_globals is inserted before this self.generate_module_init_func(modules[:-1], env, globalstate['init_module']) self.generate_module_cleanup_func(env, globalstate['cleanup_module']) @@ -408,6 +528,8 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): globalstate.use_utility_code(utilcode) globalstate.finalize_main_c_code() + self.generate_module_state_end(env, modules, globalstate) + f = open_new_file(result.c_file) try: rootwriter.copyto(f) @@ -429,11 +551,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): except ImportError: import xml.etree.ElementTree as ET coverage_xml = ET.parse(coverage_xml_filename).getroot() - if hasattr(coverage_xml, 'iter'): - iterator = coverage_xml.iter() # Python 2.7 & 3.2+ - else: - iterator = coverage_xml.getiterator() - for el in iterator: + for el in coverage_xml.iter(): el.tail = None # save some memory else: coverage_xml = None @@ -452,7 +570,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): if not target_file_dir.startswith(target_dir): # any other directories may not be writable => avoid trying continue - source_file = search_include_file(included_file, "", self.pos, include=True) + source_file = search_include_file(included_file, source_pos=self.pos, include=True) if not source_file: continue if target_file_dir != target_dir and not os.path.exists(target_file_dir): @@ -469,16 +587,18 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): markers = ccodewriter.buffer.allmarkers() d = defaultdict(list) - for c_lineno, cython_lineno in enumerate(markers): - if cython_lineno > 0: - d[cython_lineno].append(c_lineno + 1) + for c_lineno, (src_desc, src_lineno) in enumerate(markers): + if src_lineno > 0 and src_desc.filename is not None: + d[src_desc, src_lineno].append(c_lineno + 1) tb.start('LineNumberMapping') - for cython_lineno, c_linenos in sorted(d.items()): + for (src_desc, src_lineno), c_linenos in sorted(d.items()): + assert src_desc.filename is not None tb.add_entry( 'LineNumber', c_linenos=' '.join(map(str, c_linenos)), - cython_lineno=str(cython_lineno), + src_path=src_desc.filename, + src_lineno=str(src_lineno), ) tb.end('LineNumberMapping') tb.serialize() @@ -619,12 +739,13 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): for module in modules: defined_here = module is env modulecode.putln("") - modulecode.putln("/* Module declarations from '%s' */" % module.qualified_name) - self.generate_c_class_declarations(module, modulecode, defined_here) + modulecode.putln("/* Module declarations from %s */" % module.qualified_name.as_c_string_literal()) + self.generate_c_class_declarations(module, modulecode, defined_here, globalstate) self.generate_cvariable_declarations(module, modulecode, defined_here) self.generate_cfunction_declarations(module, modulecode, defined_here) - def _put_setup_code(self, code, name): + @staticmethod + def _put_setup_code(code, name): code.put(UtilityCode.load_as_string(name, "ModuleSetupCode.c")[1]) def generate_module_preamble(self, env, options, cimported_modules, metadata, code): @@ -638,6 +759,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.putln("#ifndef PY_SSIZE_T_CLEAN") code.putln("#define PY_SSIZE_T_CLEAN") code.putln("#endif /* PY_SSIZE_T_CLEAN */") + self._put_setup_code(code, "InitLimitedAPI") for inc in sorted(env.c_includes.values(), key=IncludeCode.sortkey): if inc.location == inc.INITIAL: @@ -645,14 +767,16 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.putln("#ifndef Py_PYTHON_H") code.putln(" #error Python headers needed to compile C extensions, " "please install development version of Python.") - code.putln("#elif PY_VERSION_HEX < 0x02060000 || " + code.putln("#elif PY_VERSION_HEX < 0x02070000 || " "(0x03000000 <= PY_VERSION_HEX && PY_VERSION_HEX < 0x03030000)") - code.putln(" #error Cython requires Python 2.6+ or Python 3.3+.") + code.putln(" #error Cython requires Python 2.7+ or Python 3.3+.") code.putln("#else") code.globalstate["end"].putln("#endif /* Py_PYTHON_H */") from .. import __version__ code.putln('#define CYTHON_ABI "%s"' % __version__.replace('.', '_')) + code.putln('#define __PYX_ABI_MODULE_NAME "_cython_" CYTHON_ABI') + code.putln('#define __PYX_TYPE_MODULE_PREFIX __PYX_ABI_MODULE_NAME "."') code.putln('#define CYTHON_HEX_VERSION %s' % build_hex_version(__version__)) code.putln("#define CYTHON_FUTURE_DIVISION %d" % ( Future.division in env.context.future_directives)) @@ -683,8 +807,8 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): self.generate_extern_c_macro_definition(code) code.putln("") - code.putln("#define %s" % Naming.h_guard_prefix + self.api_name(env)) - code.putln("#define %s" % Naming.api_guard_prefix + self.api_name(env)) + code.putln("#define %s" % self.api_name(Naming.h_guard_prefix, env)) + code.putln("#define %s" % self.api_name(Naming.api_guard_prefix, env)) code.putln("/* Early includes */") self.generate_includes(env, cimported_modules, code, late=False) code.putln("") @@ -721,6 +845,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.putln('#define __Pyx_PyObject_FromString __Pyx_Py%s_FromString' % c_string_func_name) code.putln('#define __Pyx_PyObject_FromStringAndSize __Pyx_Py%s_FromStringAndSize' % c_string_func_name) code.put(UtilityCode.load_as_string("TypeConversions", "TypeConversion.c")[0]) + env.use_utility_code(UtilityCode.load_cached("FormatTypeName", "ObjectHandling.c")) # These utility functions are assumed to exist and used elsewhere. PyrexTypes.c_long_type.create_to_py_utility_code(env) @@ -730,6 +855,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.put(Nodes.branch_prediction_macros) code.putln('static CYTHON_INLINE void __Pyx_pretend_to_initialize(void* ptr) { (void)ptr; }') code.putln('') + code.putln('#if !CYTHON_USE_MODULE_STATE') code.putln('static PyObject *%s = NULL;' % env.module_cname) code.putln('static PyObject *%s;' % env.module_dict_cname) code.putln('static PyObject *%s;' % Naming.builtins_cname) @@ -739,9 +865,11 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.putln('static PyObject *%s;' % Naming.empty_unicode) if Options.pre_import is not None: code.putln('static PyObject *%s;' % Naming.preimport_cname) + code.putln('#endif') + code.putln('static int %s;' % Naming.lineno_cname) code.putln('static int %s = 0;' % Naming.clineno_cname) - code.putln('static const char * %s= %s;' % (Naming.cfilenm_cname, Naming.file_c_macro)) + code.putln('static const char * %s = %s;' % (Naming.cfilenm_cname, Naming.file_c_macro)) code.putln('static const char *%s;' % Naming.filename_cname) env.use_utility_code(UtilityCode.load_cached("FastTypeChecks", "ModuleSetupCode.c")) @@ -764,7 +892,6 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.putln("#endif") def generate_includes(self, env, cimported_modules, code, early=True, late=True): - includes = [] for inc in sorted(env.c_includes.values(), key=IncludeCode.sortkey): if inc.location == inc.EARLY: if early: @@ -785,7 +912,8 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): if isabs(file_path): file_path = basename(file_path) # never include absolute paths escaped_filename = file_path.replace("\\", "\\\\").replace('"', r'\"') - code.putln('"%s",' % escaped_filename) + escaped_filename = as_encoded_filename(escaped_filename) + code.putln('%s,' % escaped_filename.as_c_string_literal()) else: # Some C compilers don't like an empty array code.putln("0") @@ -802,7 +930,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): if not entry.in_cinclude: #print "generate_type_header_code:", entry.name, repr(entry.type) ### type = entry.type - if type.is_typedef: # Must test this first! + if type.is_typedef: # Must test this first! pass elif type.is_struct_or_union or type.is_cpp_class: self.generate_struct_union_predeclaration(entry, code) @@ -815,9 +943,9 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): if not entry.in_cinclude: #print "generate_type_header_code:", entry.name, repr(entry.type) ### type = entry.type - if type.is_typedef: # Must test this first! + if type.is_typedef: # Must test this first! self.generate_typedef(entry, code) - elif type.is_enum: + elif type.is_enum or type.is_cpp_enum: self.generate_enum_definition(entry, code) elif type.is_struct_or_union: self.generate_struct_union_definition(entry, code) @@ -894,8 +1022,6 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.putln("#endif") code.putln(header) var_entries = scope.var_entries - if not var_entries: - error(entry.pos, "Empty struct or union definition not allowed outside a 'cdef extern from' block") for attr in var_entries: code.putln( "%s;" % attr.type.declaration_code(attr.cname)) @@ -956,67 +1082,69 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): arg_decls = ["void"] arg_names = [] if is_implementing: - code.putln("%s(%s) {" % (type.cname, ", ".join(arg_decls))) - if py_attrs: - code.put_ensure_gil() - for attr in py_attrs: - code.put_init_var_to_py_none(attr, nanny=False); - if constructor: - code.putln("%s(%s);" % (constructor.cname, ", ".join(arg_names))) - if py_attrs: - code.put_release_ensured_gil() - code.putln("}") + code.putln("%s(%s) {" % (type.cname, ", ".join(arg_decls))) + if py_attrs: + code.put_ensure_gil() + for attr in py_attrs: + code.put_init_var_to_py_none(attr, nanny=False) + if constructor: + code.putln("%s(%s);" % (constructor.cname, ", ".join(arg_names))) + if py_attrs: + code.put_release_ensured_gil() + code.putln("}") else: - code.putln("%s(%s);" % (type.cname, ", ".join(arg_decls))) + code.putln("%s(%s);" % (type.cname, ", ".join(arg_decls))) if destructor or py_attrs or has_virtual_methods: if has_virtual_methods: code.put("virtual ") if is_implementing: - code.putln("~%s() {" % type.cname) - if py_attrs: - code.put_ensure_gil() - if destructor: - code.putln("%s();" % destructor.cname) - if py_attrs: - for attr in py_attrs: - code.put_var_xdecref(attr, nanny=False); - code.put_release_ensured_gil() - code.putln("}") + code.putln("~%s() {" % type.cname) + if py_attrs: + code.put_ensure_gil() + if destructor: + code.putln("%s();" % destructor.cname) + if py_attrs: + for attr in py_attrs: + code.put_var_xdecref(attr, nanny=False) + code.put_release_ensured_gil() + code.putln("}") else: - code.putln("~%s();" % type.cname) + code.putln("~%s();" % type.cname) if py_attrs: # Also need copy constructor and assignment operators. if is_implementing: - code.putln("%s(const %s& __Pyx_other) {" % (type.cname, type.cname)) - code.put_ensure_gil() - for attr in scope.var_entries: - if not attr.type.is_cfunction: - code.putln("%s = __Pyx_other.%s;" % (attr.cname, attr.cname)) - code.put_var_incref(attr, nanny=False) - code.put_release_ensured_gil() - code.putln("}") - code.putln("%s& operator=(const %s& __Pyx_other) {" % (type.cname, type.cname)) - code.putln("if (this != &__Pyx_other) {") - code.put_ensure_gil() - for attr in scope.var_entries: - if not attr.type.is_cfunction: - code.put_var_xdecref(attr, nanny=False); - code.putln("%s = __Pyx_other.%s;" % (attr.cname, attr.cname)) - code.put_var_incref(attr, nanny=False) - code.put_release_ensured_gil() - code.putln("}") - code.putln("return *this;") - code.putln("}") + code.putln("%s(const %s& __Pyx_other) {" % (type.cname, type.cname)) + code.put_ensure_gil() + for attr in scope.var_entries: + if not attr.type.is_cfunction: + code.putln("%s = __Pyx_other.%s;" % (attr.cname, attr.cname)) + code.put_var_incref(attr, nanny=False) + code.put_release_ensured_gil() + code.putln("}") + code.putln("%s& operator=(const %s& __Pyx_other) {" % (type.cname, type.cname)) + code.putln("if (this != &__Pyx_other) {") + code.put_ensure_gil() + for attr in scope.var_entries: + if not attr.type.is_cfunction: + code.put_var_xdecref(attr, nanny=False) + code.putln("%s = __Pyx_other.%s;" % (attr.cname, attr.cname)) + code.put_var_incref(attr, nanny=False) + code.put_release_ensured_gil() + code.putln("}") + code.putln("return *this;") + code.putln("}") else: - code.putln("%s(const %s& __Pyx_other);" % (type.cname, type.cname)) - code.putln("%s& operator=(const %s& __Pyx_other);" % (type.cname, type.cname)) + code.putln("%s(const %s& __Pyx_other);" % (type.cname, type.cname)) + code.putln("%s& operator=(const %s& __Pyx_other);" % (type.cname, type.cname)) code.putln("};") def generate_enum_definition(self, entry, code): code.mark_pos(entry.pos) type = entry.type name = entry.cname or entry.name or "" - header, footer = self.sue_header_footer(type, "enum", name) + + kind = "enum class" if entry.type.is_cpp_enum else "enum" + header, footer = self.sue_header_footer(type, kind, name) code.putln(header) enum_values = entry.enum_values if not enum_values: @@ -1030,18 +1158,20 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): for value_entry in enum_values: if value_entry.value_node is None: - value_code = value_entry.cname + value_code = value_entry.cname.split("::")[-1] else: value_code = ("%s = %s" % ( - value_entry.cname, + value_entry.cname.split("::")[-1], value_entry.value_node.result())) if value_entry is not last_entry: value_code += "," code.putln(value_code) code.putln(footer) - if entry.type.typedef_flag: - # Not pre-declared. - code.putln("typedef enum %s %s;" % (name, name)) + + if entry.type.is_enum: + if entry.type.typedef_flag: + # Not pre-declared. + code.putln("typedef enum %s %s;" % (name, name)) def generate_typeobj_predeclaration(self, entry, code): code.putln("") @@ -1120,7 +1250,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): # Generate object struct definition for an # extension type. if not type.scope: - return # Forward declared but never defined + return # Forward declared but never defined header, footer = \ self.sue_header_footer(type, "struct", type.objstruct_cname) code.putln(header) @@ -1148,18 +1278,51 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): attr_type = py_object_type else: attr_type = attr.type - code.putln( - "%s;" % attr_type.declaration_code(attr.cname)) + if attr.is_cpp_optional: + decl = attr_type.cpp_optional_declaration_code(attr.cname) + else: + decl = attr_type.declaration_code(attr.cname) + type.scope.use_entry_utility_code(attr) + code.putln("%s;" % decl) code.putln(footer) if type.objtypedef_cname is not None: # Only for exposing public typedef name. code.putln("typedef struct %s %s;" % (type.objstruct_cname, type.objtypedef_cname)) - def generate_c_class_declarations(self, env, code, definition): + def generate_c_class_declarations(self, env, code, definition, globalstate): + module_state = globalstate['module_state'] + module_state_defines = globalstate['module_state_defines'] + module_state_clear = globalstate['module_state_clear'] + module_state_traverse = globalstate['module_state_traverse'] + code.putln("#if !CYTHON_USE_MODULE_STATE") for entry in env.c_class_entries: if definition or entry.defined_in_pxd: code.putln("static PyTypeObject *%s = 0;" % ( entry.type.typeptr_cname)) + module_state.putln("PyTypeObject *%s;" % entry.type.typeptr_cname) + module_state_defines.putln("#define %s %s->%s" % ( + entry.type.typeptr_cname, + Naming.modulestateglobal_cname, + entry.type.typeptr_cname)) + module_state_clear.putln( + "Py_CLEAR(clear_module_state->%s);" % + entry.type.typeptr_cname) + module_state_traverse.putln( + "Py_VISIT(traverse_module_state->%s);" % + entry.type.typeptr_cname) + if entry.type.typeobj_cname is not None: + module_state.putln("PyObject *%s;" % entry.type.typeobj_cname) + module_state_defines.putln("#define %s %s->%s" % ( + entry.type.typeobj_cname, + Naming.modulestateglobal_cname, + entry.type.typeobj_cname)) + module_state_clear.putln( + "Py_CLEAR(clear_module_state->%s);" % ( + entry.type.typeobj_cname)) + module_state_traverse.putln( + "Py_VISIT(traverse_module_state->%s);" % ( + entry.type.typeobj_cname)) + code.putln("#endif") def generate_cvariable_declarations(self, env, code, definition): if env.is_cython_builtin: @@ -1199,13 +1362,18 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): if storage_class: code.put("%s " % storage_class) - code.put(type.declaration_code( - cname, dll_linkage=dll_linkage)) + if entry.is_cpp_optional: + code.put(type.cpp_optional_declaration_code( + cname, dll_linkage=dll_linkage)) + else: + code.put(type.declaration_code( + cname, dll_linkage=dll_linkage)) if init is not None: code.put_safe(" = %s" % init) code.putln(";") if entry.cname != cname: code.putln("#define %s (*%s)" % (entry.cname, cname)) + env.use_entry_utility_code(entry) def generate_cfunction_declarations(self, env, code, definition): for entry in env.cfunc_entries: @@ -1229,14 +1397,11 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): if entry.visibility != 'extern': type = entry.type scope = type.scope - if scope: # could be None if there was an error - if not scope.directives['c_api_binop_methods']: - error(self.pos, - "The 'c_api_binop_methods' directive is only supported for forward compatibility" - " and must be True.") + if scope: # could be None if there was an error self.generate_exttype_vtable(scope, code) self.generate_new_function(scope, code, entry) self.generate_dealloc_function(scope, code) + if scope.needs_gc(): self.generate_traverse_function(scope, code, entry) if scope.needs_tp_clear(): @@ -1264,12 +1429,26 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): self.generate_descr_set_function(scope, code) if not scope.is_closure_class_scope and scope.defines_any(["__dict__"]): self.generate_dict_getter_function(scope, code) + if scope.defines_any_special(TypeSlots.richcmp_special_methods): self.generate_richcmp_function(scope, code) + elif scope.directives.get('total_ordering'): + # Warn if this is used when it can't have any effect. + warning(scope.parent_type.pos, + "total_ordering directive used, but no comparison and equality methods defined") + + for slot in TypeSlots.PyNumberMethods: + if slot.is_binop and scope.defines_any_special(slot.user_methods): + self.generate_binop_function(scope, slot, code, entry.pos) + self.generate_property_accessors(scope, code) self.generate_method_table(scope, code) self.generate_getset_table(scope, code) + code.putln("#if CYTHON_USE_TYPE_SPECS") + self.generate_typeobj_spec(entry, code) + code.putln("#else") self.generate_typeobj_definition(full_module_name, entry, code) + code.putln("#endif") def generate_exttype_vtable(self, scope, code): # Generate the definition of an extension type's vtable. @@ -1287,8 +1466,11 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): type.empty_declaration_code())) def generate_new_function(self, scope, code, cclass_entry): - tp_slot = TypeSlots.ConstructorSlot("tp_new", '__new__') + tp_slot = TypeSlots.ConstructorSlot("tp_new", "__cinit__") slot_func = scope.mangle_internal("tp_new") + if tp_slot.slot_code(scope) != slot_func: + return # never used + type = scope.parent_type base_type = type.base_type @@ -1298,12 +1480,13 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): if scope.is_internal: # internal classes (should) never need None inits, normal zeroing will do py_attrs = [] - cpp_class_attrs = [entry for entry in scope.var_entries - if entry.type.is_cpp_class] + cpp_constructable_attrs = [entry for entry in scope.var_entries if entry.type.needs_cpp_construction] - new_func_entry = scope.lookup_here("__new__") - if base_type or (new_func_entry and new_func_entry.is_special - and not new_func_entry.trivial_signature): + cinit_func_entry = scope.lookup_here("__cinit__") + if cinit_func_entry and not cinit_func_entry.is_special: + cinit_func_entry = None + + if base_type or (cinit_func_entry and not cinit_func_entry.trivial_signature): unused_marker = '' else: unused_marker = 'CYTHON_UNUSED ' @@ -1331,23 +1514,33 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): need_self_cast = (type.vtabslot_cname or (py_buffers or memoryview_slices or py_attrs) or - cpp_class_attrs) + cpp_constructable_attrs) if need_self_cast: code.putln("%s;" % scope.parent_type.declaration_code("p")) if base_type: + code.putln("#if CYTHON_COMPILING_IN_LIMITED_API") + code.putln("newfunc new_func = (newfunc)PyType_GetSlot(%s, Py_tp_new);" % + base_type.typeptr_cname) + code.putln("PyObject *o = new_func(t, a, k);") + code.putln("#else") tp_new = TypeSlots.get_base_slot_function(scope, tp_slot) if tp_new is None: tp_new = "%s->tp_new" % base_type.typeptr_cname code.putln("PyObject *o = %s(t, a, k);" % tp_new) + code.putln("#endif") else: code.putln("PyObject *o;") + code.putln("#if CYTHON_COMPILING_IN_LIMITED_API") + code.putln("allocfunc alloc_func = (allocfunc)PyType_GetSlot(t, Py_tp_alloc);") + code.putln("o = alloc_func(t, 0);") + code.putln("#else") if freelist_size: code.globalstate.use_utility_code( UtilityCode.load_cached("IncludeStringH", "StringTools.c")) if is_final_type: type_safety_check = '' else: - type_safety_check = ' & ((t->tp_flags & (Py_TPFLAGS_IS_ABSTRACT | Py_TPFLAGS_HEAPTYPE)) == 0)' + type_safety_check = ' & (!__Pyx_PyType_HasFeature(t, (Py_TPFLAGS_IS_ABSTRACT | Py_TPFLAGS_HEAPTYPE)))' obj_struct = type.declaration_code("", deref=True) code.putln( "if (CYTHON_COMPILING_IN_CPYTHON && likely((%s > 0) & (t->tp_basicsize == sizeof(%s))%s)) {" % ( @@ -1360,7 +1553,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.putln("PyObject_GC_Track(o);") code.putln("} else {") if not is_final_type: - code.putln("if (likely((t->tp_flags & Py_TPFLAGS_IS_ABSTRACT) == 0)) {") + code.putln("if (likely(!__Pyx_PyType_HasFeature(t, Py_TPFLAGS_IS_ABSTRACT))) {") code.putln("o = (*t->tp_alloc)(t, 0);") if not is_final_type: code.putln("} else {") @@ -1369,6 +1562,8 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.putln("if (unlikely(!o)) return 0;") if freelist_size and not base_type: code.putln('}') + if not base_type: + code.putln("#endif") if need_self_cast: code.putln("p = %s;" % type.cast_code("o")) #if need_self_cast: @@ -1389,9 +1584,13 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): type.vtabslot_cname, struct_type_cast, type.vtabptr_cname)) - for entry in cpp_class_attrs: + for entry in cpp_constructable_attrs: + if entry.is_cpp_optional: + decl_code = entry.type.cpp_optional_declaration_code("") + else: + decl_code = entry.type.empty_declaration_code() code.putln("new((void*)&(p->%s)) %s();" % ( - entry.cname, entry.type.empty_declaration_code())) + entry.cname, decl_code)) for entry in py_attrs: if entry.name == "__dict__": @@ -1411,14 +1610,14 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): if cclass_entry.cname == '__pyx_memoryviewslice': code.putln("p->from_slice.memview = NULL;") - if new_func_entry and new_func_entry.is_special: - if new_func_entry.trivial_signature: + if cinit_func_entry: + if cinit_func_entry.trivial_signature: cinit_args = "o, %s, NULL" % Naming.empty_tuple else: cinit_args = "o, a, k" needs_error_cleanup = True code.putln("if (unlikely(%s(%s) < 0)) goto bad;" % ( - new_func_entry.func_cname, cinit_args)) + cinit_func_entry.func_cname, cinit_args)) code.putln( "return o;") @@ -1443,6 +1642,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): is_final_type = scope.parent_type.is_final_type needs_gc = scope.needs_gc() + needs_trashcan = scope.needs_trashcan() weakref_slot = scope.lookup_here("__weakref__") if not scope.is_closure_class_scope else None if weakref_slot not in scope.var_entries: @@ -1453,10 +1653,10 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): dict_slot = None _, (py_attrs, _, memoryview_slices) = scope.get_refcounted_entries() - cpp_class_attrs = [entry for entry in scope.var_entries - if entry.type.is_cpp_class] + cpp_destructable_attrs = [entry for entry in scope.var_entries + if entry.type.needs_cpp_construction] - if py_attrs or cpp_class_attrs or memoryview_slices or weakref_slot or dict_slot: + if py_attrs or cpp_destructable_attrs or memoryview_slices or weakref_slot or dict_slot: self.generate_self_cast(scope, code) if not is_final_type: @@ -1468,7 +1668,8 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): finalised_check = ( '(!PyType_IS_GC(Py_TYPE(o)) || !_PyGC_FINALIZED(o))') code.putln( - "if (unlikely(PyType_HasFeature(Py_TYPE(o), Py_TPFLAGS_HAVE_FINALIZE)" + "if (unlikely(" + "(PY_VERSION_HEX >= 0x03080000 || __Pyx_PyType_HasFeature(Py_TYPE(o), Py_TPFLAGS_HAVE_FINALIZE))" " && Py_TYPE(o)->tp_finalize) && %s) {" % finalised_check) # if instance was resurrected by finaliser, return code.putln("if (PyObject_CallFinalizerFromDealloc(o)) return;") @@ -1481,25 +1682,30 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): # running this destructor. code.putln("PyObject_GC_UnTrack(o);") - # call the user's __dealloc__ - self.generate_usr_dealloc_call(scope, code) + if needs_trashcan: + code.globalstate.use_utility_code( + UtilityCode.load_cached("PyTrashcan", "ExtensionTypes.c")) + code.putln("__Pyx_TRASHCAN_BEGIN(o, %s)" % slot_func_cname) if weakref_slot: + # We must clean the weakreferences before calling the user's __dealloc__ + # because if the __dealloc__ releases the GIL, a weakref can be + # dereferenced accessing the object in an inconsistent state or + # resurrecting it. code.putln("if (p->__weakref__) PyObject_ClearWeakRefs(o);") + # call the user's __dealloc__ + self.generate_usr_dealloc_call(scope, code) + if dict_slot: code.putln("if (p->__dict__) PyDict_Clear(p->__dict__);") - for entry in cpp_class_attrs: + for entry in cpp_destructable_attrs: code.putln("__Pyx_call_destructor(p->%s);" % entry.cname) - for entry in py_attrs: + for entry in (py_attrs + memoryview_slices): code.put_xdecref_clear("p->%s" % entry.cname, entry.type, nanny=False, - clear_before_decref=True) - - for entry in memoryview_slices: - code.put_xdecref_memoryviewslice("p->%s" % entry.cname, - have_gil=True) + clear_before_decref=True, have_gil=True) if base_type: if needs_gc: @@ -1539,7 +1745,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): type_safety_check = '' else: type_safety_check = ( - ' & ((Py_TYPE(o)->tp_flags & (Py_TPFLAGS_IS_ABSTRACT | Py_TPFLAGS_HEAPTYPE)) == 0)') + ' & (!__Pyx_PyType_HasFeature(Py_TYPE(o), (Py_TPFLAGS_IS_ABSTRACT | Py_TPFLAGS_HEAPTYPE)))') type = scope.parent_type code.putln( @@ -1554,12 +1760,16 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.putln("(*Py_TYPE(o)->tp_free)(o);") if freelist_size: code.putln("}") + + if needs_trashcan: + code.putln("__Pyx_TRASHCAN_END") + code.putln( "}") def generate_usr_dealloc_call(self, scope, code): entry = scope.lookup_here("__dealloc__") - if not entry: + if not entry or not entry.is_special: return code.putln("{") @@ -1639,7 +1849,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): slot_func = scope.mangle_internal("tp_clear") base_type = scope.parent_type.base_type if tp_slot.slot_code(scope) != slot_func: - return # never used + return # never used have_entries, (py_attrs, py_buffers, memoryview_slices) = ( scope.get_refcounted_entries(include_gc_simple=False)) @@ -1697,7 +1907,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.putln("Py_CLEAR(p->%s.obj);" % entry.cname) if cclass_entry.cname == '__pyx_memoryviewslice': - code.putln("__PYX_XDEC_MEMVIEW(&p->from_slice, 1);") + code.putln("__PYX_XCLEAR_MEMVIEW(&p->from_slice, 1);") code.putln("return 0;") code.putln("}") @@ -1738,12 +1948,18 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): if set_entry: code.putln("return %s(o, i, v);" % set_entry.func_cname) else: + code.putln( + "__Pyx_TypeName o_type_name;") self.generate_guarded_basetype_call( base_type, "tp_as_mapping", "mp_ass_subscript", "o, i, v", code) code.putln( + "o_type_name = __Pyx_PyType_GetName(Py_TYPE(o));") + code.putln( "PyErr_Format(PyExc_NotImplementedError,") code.putln( - ' "Subscript assignment not supported by %.200s", Py_TYPE(o)->tp_name);') + ' "Subscript assignment not supported by " __Pyx_FMT_TYPENAME, o_type_name);') + code.putln( + "__Pyx_DECREF_TypeName(o_type_name);") code.putln( "return -1;") code.putln( @@ -1755,12 +1971,18 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): "return %s(o, i);" % ( del_entry.func_cname)) else: + code.putln( + "__Pyx_TypeName o_type_name;") self.generate_guarded_basetype_call( base_type, "tp_as_mapping", "mp_ass_subscript", "o, i, v", code) code.putln( + "o_type_name = __Pyx_PyType_GetName(Py_TYPE(o));") + code.putln( "PyErr_Format(PyExc_NotImplementedError,") code.putln( - ' "Subscript deletion not supported by %.200s", Py_TYPE(o)->tp_name);') + ' "Subscript deletion not supported by " __Pyx_FMT_TYPENAME, o_type_name);') + code.putln( + "__Pyx_DECREF_TypeName(o_type_name);") code.putln( "return -1;") code.putln( @@ -1805,12 +2027,18 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): "return %s(o, i, j, v);" % ( set_entry.func_cname)) else: + code.putln( + "__Pyx_TypeName o_type_name;") self.generate_guarded_basetype_call( base_type, "tp_as_sequence", "sq_ass_slice", "o, i, j, v", code) code.putln( + "o_type_name = __Pyx_PyType_GetName(Py_TYPE(o));") + code.putln( "PyErr_Format(PyExc_NotImplementedError,") code.putln( - ' "2-element slice assignment not supported by %.200s", Py_TYPE(o)->tp_name);') + ' "2-element slice assignment not supported by " __Pyx_FMT_TYPENAME, o_type_name);') + code.putln( + "__Pyx_DECREF_TypeName(o_type_name);") code.putln( "return -1;") code.putln( @@ -1822,12 +2050,18 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): "return %s(o, i, j);" % ( del_entry.func_cname)) else: + code.putln( + "__Pyx_TypeName o_type_name;") self.generate_guarded_basetype_call( base_type, "tp_as_sequence", "sq_ass_slice", "o, i, j, v", code) code.putln( + "o_type_name = __Pyx_PyType_GetName(Py_TYPE(o));") + code.putln( "PyErr_Format(PyExc_NotImplementedError,") code.putln( - ' "2-element slice deletion not supported by %.200s", Py_TYPE(o)->tp_name);') + ' "2-element slice deletion not supported by " __Pyx_FMT_TYPENAME, o_type_name);') + code.putln( + "__Pyx_DECREF_TypeName(o_type_name);") code.putln( "return -1;") code.putln( @@ -1857,37 +2091,112 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): # need to call up into base classes as we may not know all implemented comparison methods extern_parent = cls if cls.typeptr_cname else scope.parent_type.base_type - eq_entry = None - has_ne = False + total_ordering = scope.directives.get('total_ordering', False) + + comp_entry = {} + for cmp_method in TypeSlots.richcmp_special_methods: for class_scope in class_scopes: entry = class_scope.lookup_here(cmp_method) if entry is not None: + comp_entry[cmp_method] = entry break + + if total_ordering: + # Check this is valid - we must have at least 1 operation defined. + comp_names = [from_name for from_name, to_name in TOTAL_ORDERING if from_name in comp_entry] + if not comp_names: + if '__eq__' not in comp_entry and '__ne__' not in comp_entry: + warning(scope.parent_type.pos, + "total_ordering directive used, but no comparison and equality methods defined") + else: + warning(scope.parent_type.pos, + "total_ordering directive used, but no comparison methods defined") + total_ordering = False else: - continue + if '__eq__' not in comp_entry and '__ne__' not in comp_entry: + warning(scope.parent_type.pos, "total_ordering directive used, but no equality method defined") + total_ordering = False + + # Same priority as functools, prefers + # __lt__ to __le__ to __gt__ to __ge__ + ordering_source = max(comp_names) + for cmp_method in TypeSlots.richcmp_special_methods: cmp_type = cmp_method.strip('_').upper() # e.g. "__eq__" -> EQ + entry = comp_entry.get(cmp_method) + if entry is None and (not total_ordering or cmp_type in ('NE', 'EQ')): + # No definition, fall back to superclasses. + # eq/ne methods shouldn't use the total_ordering code. + continue + code.putln("case Py_%s: {" % cmp_type) - if cmp_method == '__eq__': - eq_entry = entry - # Python itself does not do this optimisation, it seems... - #code.putln("if (o1 == o2) return __Pyx_NewRef(Py_True);") - elif cmp_method == '__ne__': - has_ne = True - # Python itself does not do this optimisation, it seems... - #code.putln("if (o1 == o2) return __Pyx_NewRef(Py_False);") - code.putln("return %s(o1, o2);" % entry.func_cname) - code.putln("}") + if entry is None: + assert total_ordering + # We need to generate this from the other methods. + invert_comp, comp_op, invert_equals = TOTAL_ORDERING[ordering_source, cmp_method] + + # First we always do the comparison. + code.putln("PyObject *ret;") + code.putln("ret = %s(o1, o2);" % comp_entry[ordering_source].func_cname) + code.putln("if (likely(ret && ret != Py_NotImplemented)) {") + code.putln("int order_res = __Pyx_PyObject_IsTrue(ret);") + code.putln("Py_DECREF(ret);") + code.putln("if (unlikely(order_res < 0)) return NULL;") + # We may need to check equality too. For some combos it's never required. + if invert_equals is not None: + # Implement the and/or check with an if. + if comp_op == '&&': + code.putln("if (%s order_res) {" % ('!!' if invert_comp else '!')) + code.putln("ret = __Pyx_NewRef(Py_False);") + code.putln("} else {") + elif comp_op == '||': + code.putln("if (%s order_res) {" % ('!' if invert_comp else '')) + code.putln("ret = __Pyx_NewRef(Py_True);") + code.putln("} else {") + else: + raise AssertionError('Unknown op %s' % (comp_op, )) + if '__eq__' in comp_entry: + eq_func = '__eq__' + else: + # Fall back to NE, which is defined here. + eq_func = '__ne__' + invert_equals = not invert_equals + + code.putln("ret = %s(o1, o2);" % comp_entry[eq_func].func_cname) + code.putln("if (likely(ret && ret != Py_NotImplemented)) {") + code.putln("int eq_res = __Pyx_PyObject_IsTrue(ret);") + code.putln("Py_DECREF(ret);") + code.putln("if (unlikely(eq_res < 0)) return NULL;") + if invert_equals: + code.putln("ret = eq_res ? Py_False : Py_True;") + else: + code.putln("ret = eq_res ? Py_True : Py_False;") + code.putln("Py_INCREF(ret);") + code.putln("}") # equals success + code.putln("}") # Needs to try equals + else: + # Convert direct to a boolean. + if invert_comp: + code.putln("ret = order_res ? Py_False : Py_True;") + else: + code.putln("ret = order_res ? Py_True : Py_False;") + code.putln("Py_INCREF(ret);") + code.putln("}") # comp_op + code.putln("return ret;") + else: + code.putln("return %s(o1, o2);" % entry.func_cname) + code.putln("}") # Case - if eq_entry and not has_ne and not extern_parent: + if '__eq__' in comp_entry and '__ne__' not in comp_entry and not extern_parent: code.putln("case Py_NE: {") code.putln("PyObject *ret;") # Python itself does not do this optimisation, it seems... #code.putln("if (o1 == o2) return __Pyx_NewRef(Py_False);") - code.putln("ret = %s(o1, o2);" % eq_entry.func_cname) + code.putln("ret = %s(o1, o2);" % comp_entry['__eq__'].func_cname) code.putln("if (likely(ret && ret != Py_NotImplemented)) {") - code.putln("int b = __Pyx_PyObject_IsTrue(ret); Py_DECREF(ret);") + code.putln("int b = __Pyx_PyObject_IsTrue(ret);") + code.putln("Py_DECREF(ret);") code.putln("if (unlikely(b < 0)) return NULL;") code.putln("ret = (b) ? Py_False : Py_True;") code.putln("Py_INCREF(ret);") @@ -1905,6 +2214,73 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.putln("}") # switch code.putln("}") + def generate_binop_function(self, scope, slot, code, pos): + func_name = scope.mangle_internal(slot.slot_name) + if scope.directives['c_api_binop_methods']: + code.putln('#define %s %s' % (func_name, slot.left_slot.slot_code(scope))) + return + + code.putln() + preprocessor_guard = slot.preprocessor_guard_code() + if preprocessor_guard: + code.putln(preprocessor_guard) + + if slot.left_slot.signature == TypeSlots.binaryfunc: + slot_type = 'binaryfunc' + extra_arg = extra_arg_decl = '' + elif slot.left_slot.signature == TypeSlots.ternaryfunc: + slot_type = 'ternaryfunc' + extra_arg = ', extra_arg' + extra_arg_decl = ', PyObject* extra_arg' + else: + error(pos, "Unexpected type slot signature: %s" % slot) + return + + def get_slot_method_cname(method_name): + entry = scope.lookup(method_name) + return entry.func_cname if entry and entry.is_special else None + + def call_slot_method(method_name, reverse): + func_cname = get_slot_method_cname(method_name) + if func_cname: + return "%s(%s%s)" % ( + func_cname, + "right, left" if reverse else "left, right", + extra_arg) + else: + return '%s_maybe_call_slot(%s->tp_base, left, right %s)' % ( + func_name, + scope.parent_type.typeptr_cname, + extra_arg) + + if get_slot_method_cname(slot.left_slot.method_name) and not get_slot_method_cname(slot.right_slot.method_name): + warning(pos, "Extension type implements %s() but not %s(). " + "The behaviour has changed from previous Cython versions to match Python semantics. " + "You can implement both special methods in a backwards compatible way." % ( + slot.left_slot.method_name, + slot.right_slot.method_name, + )) + + overloads_left = int(bool(get_slot_method_cname(slot.left_slot.method_name))) + overloads_right = int(bool(get_slot_method_cname(slot.right_slot.method_name))) + code.putln( + TempitaUtilityCode.load_as_string( + "BinopSlot", "ExtensionTypes.c", + context={ + "func_name": func_name, + "slot_name": slot.slot_name, + "overloads_left": overloads_left, + "overloads_right": overloads_right, + "call_left": call_slot_method(slot.left_slot.method_name, reverse=False), + "call_right": call_slot_method(slot.right_slot.method_name, reverse=True), + "type_cname": scope.parent_type.typeptr_cname, + "slot_type": slot_type, + "extra_arg": extra_arg, + "extra_arg_decl": extra_arg_decl, + })[1]) + if preprocessor_guard: + code.putln("#endif") + def generate_getattro_function(self, scope, code): # First try to get the attribute using __getattribute__, if defined, or # PyObject_GenericGetAttr. @@ -2139,6 +2515,38 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.putln( "}") + def generate_typeobj_spec(self, entry, code): + ext_type = entry.type + scope = ext_type.scope + + members_slot = TypeSlots.get_slot_by_name("tp_members") + members_slot.generate_substructure_spec(scope, code) + + buffer_slot = TypeSlots.get_slot_by_name("tp_as_buffer") + if not buffer_slot.is_empty(scope): + code.putln("#if !CYTHON_COMPILING_IN_LIMITED_API") + buffer_slot.generate_substructure(scope, code) + code.putln("#endif") + + code.putln("static PyType_Slot %s_slots[] = {" % ext_type.typeobj_cname) + for slot in TypeSlots.slot_table: + slot.generate_spec(scope, code) + code.putln("{0, 0},") + code.putln("};") + + if ext_type.typedef_flag: + objstruct = ext_type.objstruct_cname + else: + objstruct = "struct %s" % ext_type.objstruct_cname + classname = scope.class_name.as_c_string_literal() + code.putln("static PyType_Spec %s_spec = {" % ext_type.typeobj_cname) + code.putln('"%s.%s",' % (self.full_module_name, classname.replace('"', ''))) + code.putln("sizeof(%s)," % objstruct) + code.putln("0,") + code.putln("%s," % TypeSlots.get_slot_by_name("tp_flags").slot_code(scope)) + code.putln("%s_slots," % ext_type.typeobj_cname) + code.putln("};") + def generate_typeobj_definition(self, modname, entry, code): type = entry.type scope = type.scope @@ -2153,9 +2561,11 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.putln(header % type.typeobj_cname) code.putln( "PyVarObject_HEAD_INIT(0, 0)") + classname = scope.class_name.as_c_string_literal() code.putln( - '"%s.%s", /*tp_name*/' % ( - self.full_module_name, scope.class_name)) + '"%s."%s, /*tp_name*/' % ( + self.full_module_name, + classname)) if type.typedef_flag: objstruct = type.objstruct_cname else: @@ -2218,12 +2628,12 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): if doc: if doc.is_unicode: doc = doc.as_utf8_string() - doc_code = doc.as_c_string_literal() + doc_code = "PyDoc_STR(%s)" % doc.as_c_string_literal() else: doc_code = "0" code.putln( - '{(char *)"%s", %s, %s, (char *)%s, 0},' % ( - entry.name, + '{(char *)%s, %s, %s, (char *)%s, 0},' % ( + entry.name.as_c_string_literal(), entry.getter_cname or "0", entry.setter_cname or "0", doc_code)) @@ -2299,7 +2709,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): if code.label_used(code.error_label): code.put_label(code.error_label) # This helps locate the offending name. - code.put_add_traceback(self.full_module_name) + code.put_add_traceback(EncodedString(self.full_module_name)) code.error_label = old_error_label code.putln("bad:") code.putln("return -1;") @@ -2308,20 +2718,205 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.putln(UtilityCode.load_as_string("ImportStar", "ImportExport.c")[1]) code.exit_cfunc_scope() # done with labels + def generate_module_state_start(self, env, code): + # TODO: Refactor to move module state struct decl closer to the static decl + code.putln("#if CYTHON_USE_MODULE_STATE") + code.putln('typedef struct {') + code.putln('PyObject *%s;' % env.module_dict_cname) + code.putln('PyObject *%s;' % Naming.builtins_cname) + code.putln('PyObject *%s;' % Naming.cython_runtime_cname) + code.putln('PyObject *%s;' % Naming.empty_tuple) + code.putln('PyObject *%s;' % Naming.empty_bytes) + code.putln('PyObject *%s;' % Naming.empty_unicode) + if Options.pre_import is not None: + code.putln('PyObject *%s;' % Naming.preimport_cname) + code.putln('#ifdef __Pyx_CyFunction_USED') + code.putln('PyTypeObject *%s;' % Naming.cyfunction_type_cname) + code.putln('#endif') + code.putln('#ifdef __Pyx_FusedFunction_USED') + code.putln('PyTypeObject *%s;' % Naming.fusedfunction_type_cname) + code.putln('#endif') + + def generate_module_state_end(self, env, modules, globalstate): + module_state = globalstate['module_state'] + module_state_defines = globalstate['module_state_defines'] + module_state_clear = globalstate['module_state_clear'] + module_state_traverse = globalstate['module_state_traverse'] + module_state.putln('} %s;' % Naming.modulestate_cname) + module_state.putln('') + module_state.putln('#ifdef __cplusplus') + module_state.putln('namespace {') + module_state.putln('extern struct PyModuleDef %s;' % Naming.pymoduledef_cname) + module_state.putln('} /* anonymous namespace */') + module_state.putln('#else') + module_state.putln('static struct PyModuleDef %s;' % Naming.pymoduledef_cname) + module_state.putln('#endif') + module_state.putln('') + module_state.putln('#define %s(o) ((%s *)__Pyx_PyModule_GetState(o))' % ( + Naming.modulestate_cname, + Naming.modulestate_cname)) + module_state.putln('') + module_state.putln('#define %s (%s(PyState_FindModule(&%s)))' % ( + Naming.modulestateglobal_cname, + Naming.modulestate_cname, + Naming.pymoduledef_cname)) + module_state.putln('') + module_state.putln('#define %s (PyState_FindModule(&%s))' % ( + env.module_cname, + Naming.pymoduledef_cname)) + module_state.putln("#endif") + module_state_defines.putln("#endif") + module_state_clear.putln("return 0;") + module_state_clear.putln("}") + module_state_clear.putln("#endif") + module_state_traverse.putln("return 0;") + module_state_traverse.putln("}") + module_state_traverse.putln("#endif") + + def generate_module_state_defines(self, env, code): + code.putln("#if CYTHON_USE_MODULE_STATE") + code.putln('#define %s %s->%s' % ( + env.module_dict_cname, + Naming.modulestateglobal_cname, + env.module_dict_cname)) + code.putln('#define %s %s->%s' % ( + Naming.builtins_cname, + Naming.modulestateglobal_cname, + Naming.builtins_cname)) + code.putln('#define %s %s->%s' % ( + Naming.cython_runtime_cname, + Naming.modulestateglobal_cname, + Naming.cython_runtime_cname)) + code.putln('#define %s %s->%s' % ( + Naming.empty_tuple, + Naming.modulestateglobal_cname, + Naming.empty_tuple)) + code.putln('#define %s %s->%s' % ( + Naming.empty_bytes, + Naming.modulestateglobal_cname, + Naming.empty_bytes)) + code.putln('#define %s %s->%s' % ( + Naming.empty_unicode, + Naming.modulestateglobal_cname, + Naming.empty_unicode)) + if Options.pre_import is not None: + code.putln('#define %s %s->%s' % ( + Naming.preimport_cname, + Naming.modulestateglobal_cname, + Naming.preimport_cname)) + code.putln('#ifdef __Pyx_CyFunction_USED') + code.putln('#define %s %s->%s' % ( + Naming.cyfunction_type_cname, + Naming.modulestateglobal_cname, + Naming.cyfunction_type_cname)) + code.putln('#endif') + code.putln('#ifdef __Pyx_FusedFunction_USED') + code.putln('#define %s %s->%s' % + (Naming.fusedfunction_type_cname, + Naming.modulestateglobal_cname, + Naming.fusedfunction_type_cname)) + code.putln('#endif') + + def generate_module_state_clear(self, env, code): + code.putln("#if CYTHON_USE_MODULE_STATE") + code.putln("static int %s_clear(PyObject *m) {" % Naming.module_cname) + code.putln("%s *clear_module_state = %s(m);" % ( + Naming.modulestate_cname, + Naming.modulestate_cname)) + code.putln("if (!clear_module_state) return 0;") + code.putln('Py_CLEAR(clear_module_state->%s);' % + env.module_dict_cname) + code.putln('Py_CLEAR(clear_module_state->%s);' % + Naming.builtins_cname) + code.putln('Py_CLEAR(clear_module_state->%s);' % + Naming.cython_runtime_cname) + code.putln('Py_CLEAR(clear_module_state->%s);' % + Naming.empty_tuple) + code.putln('Py_CLEAR(clear_module_state->%s);' % + Naming.empty_bytes) + code.putln('Py_CLEAR(clear_module_state->%s);' % + Naming.empty_unicode) + code.putln('#ifdef __Pyx_CyFunction_USED') + code.putln('Py_CLEAR(clear_module_state->%s);' % + Naming.cyfunction_type_cname) + code.putln('#endif') + code.putln('#ifdef __Pyx_FusedFunction_USED') + code.putln('Py_CLEAR(clear_module_state->%s);' % + Naming.fusedfunction_type_cname) + code.putln('#endif') + + def generate_module_state_traverse(self, env, code): + code.putln("#if CYTHON_USE_MODULE_STATE") + code.putln("static int %s_traverse(PyObject *m, visitproc visit, void *arg) {" % Naming.module_cname) + code.putln("%s *traverse_module_state = %s(m);" % ( + Naming.modulestate_cname, + Naming.modulestate_cname)) + code.putln("if (!traverse_module_state) return 0;") + code.putln('Py_VISIT(traverse_module_state->%s);' % + env.module_dict_cname) + code.putln('Py_VISIT(traverse_module_state->%s);' % + Naming.builtins_cname) + code.putln('Py_VISIT(traverse_module_state->%s);' % + Naming.cython_runtime_cname) + code.putln('Py_VISIT(traverse_module_state->%s);' % + Naming.empty_tuple) + code.putln('Py_VISIT(traverse_module_state->%s);' % + Naming.empty_bytes) + code.putln('Py_VISIT(traverse_module_state->%s);' % + Naming.empty_unicode) + code.putln('#ifdef __Pyx_CyFunction_USED') + code.putln('Py_VISIT(traverse_module_state->%s);' % + Naming.cyfunction_type_cname) + code.putln('#endif') + code.putln('#ifdef __Pyx_FusedFunction_USED') + code.putln('Py_VISIT(traverse_module_state->%s);' % + Naming.fusedfunction_type_cname) + code.putln('#endif') + def generate_module_init_func(self, imported_modules, env, code): subfunction = self.mod_init_subfunction(self.pos, self.scope, code) + self.generate_pymoduledef_struct(env, code) + code.enter_cfunc_scope(self.scope) code.putln("") code.putln(UtilityCode.load_as_string("PyModInitFuncType", "ModuleSetupCode.c")[0]) - header2 = "__Pyx_PyMODINIT_FUNC init%s(void)" % env.module_name + if env.module_name.isascii(): + py2_mod_name = env.module_name + fail_compilation_in_py2 = False + else: + fail_compilation_in_py2 = True + # at this point py2_mod_name is largely a placeholder and the value doesn't matter + py2_mod_name = env.module_name.encode("ascii", errors="ignore").decode("utf8") + + header2 = "__Pyx_PyMODINIT_FUNC init%s(void)" % py2_mod_name header3 = "__Pyx_PyMODINIT_FUNC %s(void)" % self.mod_init_func_cname('PyInit', env) + header3 = EncodedString(header3) code.putln("#if PY_MAJOR_VERSION < 3") # Optimise for small code size as the module init function is only executed once. code.putln("%s CYTHON_SMALL_CODE; /*proto*/" % header2) + if fail_compilation_in_py2: + code.putln('#error "Unicode module names are not supported in Python 2";') + if self.scope.is_package: + code.putln("#if !defined(CYTHON_NO_PYINIT_EXPORT) && (defined(_WIN32) || defined(WIN32) || defined(MS_WINDOWS))") + code.putln("__Pyx_PyMODINIT_FUNC init__init__(void) { init%s(); }" % py2_mod_name) + code.putln("#endif") code.putln(header2) code.putln("#else") code.putln("%s CYTHON_SMALL_CODE; /*proto*/" % header3) + if self.scope.is_package: + code.putln("#if !defined(CYTHON_NO_PYINIT_EXPORT) && (defined(_WIN32) || defined(WIN32) || defined(MS_WINDOWS))") + code.putln("__Pyx_PyMODINIT_FUNC PyInit___init__(void) { return %s(); }" % ( + self.mod_init_func_cname('PyInit', env))) + code.putln("#endif") + # Hack for a distutils bug - https://bugs.python.org/issue39432 + # distutils attempts to make visible a slightly wrong PyInitU module name. Just create a dummy + # function to keep it quiet + wrong_punycode_module_name = self.wrong_punycode_module_name(env.module_name) + if wrong_punycode_module_name: + code.putln("#if !defined(CYTHON_NO_PYINIT_EXPORT) && (defined(_WIN32) || defined(WIN32) || defined(MS_WINDOWS))") + code.putln("void %s(void) {} /* workaround for https://bugs.python.org/issue39432 */" % wrong_punycode_module_name) + code.putln("#endif") code.putln(header3) # CPython 3.5+ supports multi-phase module initialisation (gives access to __spec__, __file__, etc.) @@ -2336,7 +2931,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.putln("") # main module init code lives in Py_mod_exec function, not in PyInit function code.putln("static CYTHON_SMALL_CODE int %s(PyObject *%s)" % ( - self.mod_init_func_cname(Naming.pymodule_exec_func_cname, env), + self.module_init_func_cname(), Naming.pymodinit_module_arg)) code.putln("#endif") # PEP489 @@ -2350,6 +2945,8 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): profile = code.globalstate.directives['profile'] linetrace = code.globalstate.directives['linetrace'] if profile or linetrace: + if linetrace: + code.use_fast_gil_utility_code() code.globalstate.use_utility_code(UtilityCode.load_cached("Profile", "Profile.c")) code.put_declare_refcount_context() @@ -2364,7 +2961,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): )) code.putln('PyErr_SetString(PyExc_RuntimeError,' ' "Module \'%s\' has already been imported. Re-initialisation is not supported.");' % - env.module_name) + env.module_name.as_c_string_literal()[1:-1]) code.putln("return -1;") code.putln("}") code.putln("#elif PY_MAJOR_VERSION >= 3") @@ -2375,6 +2972,9 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): )) code.putln("#endif") + code.putln("/*--- Module creation code ---*/") + self.generate_module_creation_code(env, code) + if profile or linetrace: tempdecl_code.put_trace_declarations() code.put_trace_frame_init() @@ -2398,7 +2998,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): for ext_type in ('CyFunction', 'FusedFunction', 'Coroutine', 'Generator', 'AsyncGen', 'StopAsyncIteration'): code.putln("#ifdef __Pyx_%s_USED" % ext_type) - code.put_error_if_neg(self.pos, "__pyx_%s_init()" % ext_type) + code.put_error_if_neg(self.pos, "__pyx_%s_init(%s)" % (ext_type, env.module_cname)) code.putln("#endif") code.putln("/*--- Library function declarations ---*/") @@ -2411,9 +3011,6 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.putln("PyEval_InitThreads();") code.putln("#endif") - code.putln("/*--- Module creation code ---*/") - self.generate_module_creation_code(env, code) - code.putln("/*--- Initialize various global constants etc. ---*/") code.put_error_if_neg(self.pos, "__Pyx_InitGlobals()") @@ -2422,7 +3019,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.put_error_if_neg(self.pos, "__Pyx_init_sys_getdefaultencoding_params()") code.putln("#endif") - code.putln("if (%s%s) {" % (Naming.module_is_main, self.full_module_name.replace('.', '__'))) + code.putln("if (%s) {" % self.is_main_module_flag_cname()) code.put_error_if_neg(self.pos, 'PyObject_SetAttr(%s, %s, %s)' % ( env.module_cname, code.intern_identifier(EncodedString("__name__")), @@ -2499,7 +3096,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.put_xdecref(cname, type) code.putln('if (%s) {' % env.module_cname) code.putln('if (%s) {' % env.module_dict_cname) - code.put_add_traceback("init %s" % env.qualified_name) + code.put_add_traceback(EncodedString("init %s" % env.qualified_name)) code.globalstate.use_utility_code(Nodes.traceback_utility_code) # Module reference and module dict are in global variables which might still be needed # for cleanup, atexit code, etc., so leaking is better than crashing. @@ -2507,9 +3104,12 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): # user code in atexit or other global registries. ##code.put_decref_clear(env.module_dict_cname, py_object_type, nanny=False) code.putln('}') + code.putln("#if !CYTHON_COMPILING_IN_LIMITED_API") code.put_decref_clear(env.module_cname, py_object_type, nanny=False, clear_before_decref=True) + code.putln("#endif") code.putln('} else if (!PyErr_Occurred()) {') - code.putln('PyErr_SetString(PyExc_ImportError, "init %s");' % env.qualified_name) + code.putln('PyErr_SetString(PyExc_ImportError, "init %s");' % + env.qualified_name.as_c_string_literal()[1:-1]) code.putln('}') code.put_label(code.return_label) @@ -2559,7 +3159,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.putln("static int %s(void) {" % self.cfunc_name) code.put_declare_refcount_context() self.tempdecl_code = code.insertion_point() - code.put_setup_refcount_context(self.cfunc_name) + code.put_setup_refcount_context(EncodedString(self.cfunc_name)) # Leave a grepable marker that makes it easy to find the generator source. code.putln("/*--- %s ---*/" % self.description) return code @@ -2616,7 +3216,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): EncodedString(decode_filename( os.path.dirname(module_path)))).cname, code.error_goto_if_null(temp, self.pos))) - code.put_gotref(temp) + code.put_gotref(temp, py_object_type) code.putln( 'if (PyObject_SetAttrString(%s, "__path__", %s) < 0) %s;' % ( env.module_cname, temp, code.error_goto(self.pos))) @@ -2646,14 +3246,15 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): # CPython may not have put us into sys.modules yet, but relative imports and reimports require it fq_module_name = self.full_module_name if fq_module_name.endswith('.__init__'): - fq_module_name = fq_module_name[:-len('.__init__')] + fq_module_name = EncodedString(fq_module_name[:-len('.__init__')]) + fq_module_name_cstring = fq_module_name.as_c_string_literal() code.putln("#if PY_MAJOR_VERSION >= 3") code.putln("{") code.putln("PyObject *modules = PyImport_GetModuleDict(); %s" % code.error_goto_if_null("modules", self.pos)) - code.putln('if (!PyDict_GetItemString(modules, "%s")) {' % fq_module_name) - code.putln(code.error_goto_if_neg('PyDict_SetItemString(modules, "%s", %s)' % ( - fq_module_name, env.module_cname), self.pos)) + code.putln('if (!PyDict_GetItemString(modules, %s)) {' % fq_module_name_cstring) + code.putln(code.error_goto_if_neg('PyDict_SetItemString(modules, %s, %s)' % ( + fq_module_name_cstring, env.module_cname), self.pos)) code.putln("}") code.putln("}") code.putln("#endif") @@ -2726,11 +3327,12 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): if Options.pre_import is not None: code.put_decref_clear(Naming.preimport_cname, py_object_type, nanny=False, clear_before_decref=True) - for cname in [env.module_dict_cname, Naming.cython_runtime_cname, Naming.builtins_cname]: + for cname in [Naming.cython_runtime_cname, Naming.builtins_cname]: code.put_decref_clear(cname, py_object_type, nanny=False, clear_before_decref=True) + code.put_decref_clear(env.module_dict_cname, py_object_type, nanny=False, clear_before_decref=True) def generate_main_method(self, env, code): - module_is_main = "%s%s" % (Naming.module_is_main, self.full_module_name.replace('.', '__')) + module_is_main = self.is_main_module_flag_cname() if Options.embed == "main": wmain = "wmain" else: @@ -2743,8 +3345,33 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): main_method=Options.embed, wmain_method=wmain)) + def punycode_module_name(self, prefix, name): + # adapted from PEP483 + try: + name = '_' + name.encode('ascii').decode('ascii') + except UnicodeEncodeError: + name = 'U_' + name.encode('punycode').replace(b'-', b'_').decode('ascii') + return "%s%s" % (prefix, name) + + def wrong_punycode_module_name(self, name): + # to work around a distutils bug by also generating an incorrect symbol... + try: + name.encode("ascii") + return None # workaround is not needed + except UnicodeEncodeError: + return "PyInitU" + (u"_"+name).encode('punycode').replace(b'-', b'_').decode('ascii') + def mod_init_func_cname(self, prefix, env): - return '%s_%s' % (prefix, env.module_name) + # from PEP483 + return self.punycode_module_name(prefix, env.module_name) + + # Returns the name of the C-function that corresponds to the module initialisation. + # (module initialisation == the cython code outside of functions) + # Note that this should never be the name of a wrapper and always the name of the + # function containing the actual code. Otherwise, cygdb will experience problems. + def module_init_func_cname(self): + env = self.scope + return self.mod_init_func_cname(Naming.pymodule_exec_func_cname, env) def generate_pymoduledef_struct(self, env, code): if env.doc: @@ -2759,7 +3386,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.putln("") code.putln("#if PY_MAJOR_VERSION >= 3") code.putln("#if CYTHON_PEP489_MULTI_PHASE_INIT") - exec_func_cname = self.mod_init_func_cname(Naming.pymodule_exec_func_cname, env) + exec_func_cname = self.module_init_func_cname() code.putln("static PyObject* %s(PyObject *spec, PyModuleDef *def); /*proto*/" % Naming.pymodule_create_func_cname) code.putln("static int %s(PyObject* module); /*proto*/" % exec_func_cname) @@ -2769,15 +3396,27 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.putln("{Py_mod_exec, (void*)%s}," % exec_func_cname) code.putln("{0, NULL}") code.putln("};") + if not env.module_name.isascii(): + code.putln("#else /* CYTHON_PEP489_MULTI_PHASE_INIT */") + code.putln('#error "Unicode module names are only supported with multi-phase init' + ' as per PEP489"') code.putln("#endif") code.putln("") - code.putln("static struct PyModuleDef %s = {" % Naming.pymoduledef_cname) + code.putln('#ifdef __cplusplus') + code.putln('namespace {') + code.putln("struct PyModuleDef %s =" % Naming.pymoduledef_cname) + code.putln('#else') + code.putln("static struct PyModuleDef %s =" % Naming.pymoduledef_cname) + code.putln('#endif') + code.putln('{') code.putln(" PyModuleDef_HEAD_INIT,") - code.putln(' "%s",' % env.module_name) + code.putln(' %s,' % env.module_name.as_c_string_literal()) code.putln(" %s, /* m_doc */" % doc) code.putln("#if CYTHON_PEP489_MULTI_PHASE_INIT") code.putln(" 0, /* m_size */") + code.putln("#elif CYTHON_USE_MODULE_STATE") # FIXME: should allow combination with PEP-489 + code.putln(" sizeof(%s), /* m_size */" % Naming.modulestate_cname) code.putln("#else") code.putln(" -1, /* m_size */") code.putln("#endif") @@ -2787,10 +3426,19 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.putln("#else") code.putln(" NULL, /* m_reload */") code.putln("#endif") + code.putln("#if CYTHON_USE_MODULE_STATE") + code.putln(" %s_traverse, /* m_traverse */" % Naming.module_cname) + code.putln(" %s_clear, /* m_clear */" % Naming.module_cname) + code.putln(" %s /* m_free */" % cleanup_func) + code.putln("#else") code.putln(" NULL, /* m_traverse */") code.putln(" NULL, /* m_clear */") code.putln(" %s /* m_free */" % cleanup_func) + code.putln("#endif") code.putln("};") + code.putln('#ifdef __cplusplus') + code.putln('} /* anonymous namespace */') + code.putln('#endif') code.putln("#endif") def generate_module_creation_code(self, env, code): @@ -2809,19 +3457,32 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.putln("#else") code.putln("#if PY_MAJOR_VERSION < 3") code.putln( - '%s = Py_InitModule4("%s", %s, %s, 0, PYTHON_API_VERSION); Py_XINCREF(%s);' % ( + '%s = Py_InitModule4(%s, %s, %s, 0, PYTHON_API_VERSION); Py_XINCREF(%s);' % ( env.module_cname, - env.module_name, + env.module_name.as_c_string_literal(), env.method_table_cname, doc, env.module_cname)) - code.putln("#else") + code.putln(code.error_goto_if_null(env.module_cname, self.pos)) + code.putln("#elif CYTHON_COMPILING_IN_LIMITED_API") + module_temp = code.funcstate.allocate_temp(py_object_type, manage_ref=True) + code.putln( + "%s = PyModule_Create(&%s); %s" % ( + module_temp, + Naming.pymoduledef_cname, + code.error_goto_if_null(module_temp, self.pos))) + code.put_gotref(module_temp, py_object_type) + code.putln(code.error_goto_if_neg("PyState_AddModule(%s, &%s)" % ( + module_temp, Naming.pymoduledef_cname), self.pos)) + code.put_decref_clear(module_temp, type=py_object_type) + code.funcstate.release_temp(module_temp) + code.putln('#else') code.putln( "%s = PyModule_Create(&%s);" % ( env.module_cname, Naming.pymoduledef_cname)) - code.putln("#endif") code.putln(code.error_goto_if_null(env.module_cname, self.pos)) + code.putln("#endif") code.putln("#endif") # CYTHON_PEP489_MULTI_PHASE_INIT code.putln( @@ -2912,8 +3573,8 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): # investigation shows that the resulting binary is smaller with repeated functions calls. for entry in entries: signature = entry.type.signature_string() - code.putln('if (__Pyx_ExportFunction("%s", (void (*)(void))%s, "%s") < 0) %s' % ( - entry.name, + code.putln('if (__Pyx_ExportFunction(%s, (void (*)(void))%s, "%s") < 0) %s' % ( + entry.name.as_c_string_literal(), entry.cname, signature, code.error_goto(self.pos))) @@ -2955,7 +3616,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): module.qualified_name, temp, code.error_goto(self.pos))) - code.put_gotref(temp) + code.put_gotref(temp, py_object_type) for entry in entries: if env is module: cname = entry.cname @@ -2985,12 +3646,12 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): module.qualified_name, temp, code.error_goto(self.pos))) - code.put_gotref(temp) + code.put_gotref(temp, py_object_type) for entry in entries: code.putln( - 'if (__Pyx_ImportFunction(%s, "%s", (void (**)(void))&%s, "%s") < 0) %s' % ( + 'if (__Pyx_ImportFunction(%s, %s, (void (**)(void))&%s, "%s") < 0) %s' % ( temp, - entry.name, + entry.name.as_c_string_literal(), entry.cname, entry.type.signature_string(), code.error_goto(self.pos))) @@ -3013,7 +3674,8 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): def generate_base_type_import_code(self, env, entry, code, import_generator): base_type = entry.type.base_type if (base_type and base_type.module_name != env.qualified_name and not - base_type.is_builtin_type and not entry.utility_code_definition): + (base_type.is_builtin_type or base_type.is_cython_builtin_type) + and not entry.utility_code_definition): self.generate_type_import_code(env, base_type, self.pos, code, import_generator) def generate_type_import_code(self, env, type, pos, code, import_generator): @@ -3030,7 +3692,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): if type.vtabptr_cname: code.globalstate.use_utility_code( UtilityCode.load_cached('GetVTable', 'ImportExport.c')) - code.putln("%s = (struct %s*)__Pyx_GetVtable(%s->tp_dict); %s" % ( + code.putln("%s = (struct %s*)__Pyx_GetVtable(%s); %s" % ( type.vtabptr_cname, type.vtabstruct_cname, type.typeptr_cname, @@ -3070,21 +3732,25 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): module, module_name)) + type_name = type.name.as_c_string_literal() + if condition and replacement: code.putln("") # start in new line code.putln("#if %s" % condition) code.putln('"%s",' % replacement) code.putln("#else") - code.putln('"%s",' % type.name) + code.putln('%s,' % type_name) code.putln("#endif") else: - code.put(' "%s", ' % type.name) + code.put(' %s, ' % type_name) if sizeof_objstruct != objstruct: if not condition: code.putln("") # start in new line code.putln("#if defined(PYPY_VERSION_NUM) && PYPY_VERSION_NUM < 0x050B0000") code.putln('sizeof(%s),' % objstruct) + code.putln("#elif CYTHON_COMPILING_IN_LIMITED_API") + code.putln('sizeof(%s),' % objstruct) code.putln("#else") code.putln('sizeof(%s),' % sizeof_objstruct) code.putln("#endif") @@ -3106,6 +3772,10 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): def generate_type_ready_code(self, entry, code): Nodes.CClassDefNode.generate_type_ready_code(entry, code) + def is_main_module_flag_cname(self): + full_module_name = self.full_module_name.replace('.', '__') + return self.punycode_module_name(Naming.module_is_main, full_module_name) + def generate_exttype_vtable_init_code(self, entry, code): # Generate code to initialise the C method table of an # extension type. @@ -3158,7 +3828,7 @@ class ModuleImportGenerator(object): self.temps.append(temp) code.putln('%s = PyImport_ImportModule(%s); if (unlikely(!%s)) %s' % ( temp, module_name_string, temp, error_code)) - code.put_gotref(temp) + code.put_gotref(temp, py_object_type) self.imported[module_name_string] = temp return temp @@ -3219,4 +3889,4 @@ packed_struct_utility_code = UtilityCode(proto=""" #endif """, impl="", proto_block='utility_code_proto_before_types') -capsule_utility_code = UtilityCode.load("Capsule") +capsule_utility_code = UtilityCode.load("Capsule", "Capsule.c") diff --git a/Cython/Compiler/Naming.py b/Cython/Compiler/Naming.py index 2c9b62078..005bd0530 100644 --- a/Cython/Compiler/Naming.py +++ b/Cython/Compiler/Naming.py @@ -13,6 +13,8 @@ codewriter_temp_prefix = pyrex_prefix + "t_" temp_prefix = u"__cyt_" +pyunicode_identifier_prefix = pyrex_prefix + 'U' + builtin_prefix = pyrex_prefix + "builtin_" arg_prefix = pyrex_prefix + "arg_" funcdoc_prefix = pyrex_prefix + "doc_" @@ -45,12 +47,18 @@ pybufferstruct_prefix = pyrex_prefix + "pybuffer_" vtable_prefix = pyrex_prefix + "vtable_" vtabptr_prefix = pyrex_prefix + "vtabptr_" vtabstruct_prefix = pyrex_prefix + "vtabstruct_" +unicode_vtabentry_prefix = pyrex_prefix + "Uvtabentry_" +# vtab entries aren't normally mangled, +# but punycode names sometimes start with numbers leading to a C syntax error +unicode_structmember_prefix = pyrex_prefix + "Umember_" +# as above - +# not normally mangled but punycode names cause specific problems opt_arg_prefix = pyrex_prefix + "opt_args_" convert_func_prefix = pyrex_prefix + "convert_" closure_scope_prefix = pyrex_prefix + "scope_" closure_class_prefix = pyrex_prefix + "scope_struct_" lambda_func_prefix = pyrex_prefix + "lambda_" -module_is_main = pyrex_prefix + "module_is_main_" +module_is_main = pyrex_prefix + "module_is_main" defaults_struct_prefix = pyrex_prefix + "defaults" dynamic_args_cname = pyrex_prefix + "dynamic_args" @@ -67,6 +75,8 @@ interned_prefixes = { ctuple_type_prefix = pyrex_prefix + "ctuple_" args_cname = pyrex_prefix + "args" +nargs_cname = pyrex_prefix + "nargs" +kwvalues_cname = pyrex_prefix + "kwvalues" generator_cname = pyrex_prefix + "generator" sent_value_cname = pyrex_prefix + "sent_value" pykwdlist_cname = pyrex_prefix + "pyargnames" @@ -85,6 +95,8 @@ clineno_cname = pyrex_prefix + "clineno" cfilenm_cname = pyrex_prefix + "cfilenm" local_tstate_cname = pyrex_prefix + "tstate" module_cname = pyrex_prefix + "m" +modulestate_cname = pyrex_prefix + "mstate" +modulestateglobal_cname = pyrex_prefix + "mstate_global" moddoc_cname = pyrex_prefix + "mdoc" methtable_cname = pyrex_prefix + "methods" retval_cname = pyrex_prefix + "r" @@ -97,7 +109,7 @@ gilstate_cname = pyrex_prefix + "state" skip_dispatch_cname = pyrex_prefix + "skip_dispatch" empty_tuple = pyrex_prefix + "empty_tuple" empty_bytes = pyrex_prefix + "empty_bytes" -empty_unicode = pyrex_prefix + "empty_unicode" +empty_unicode = pyrex_prefix + "empty_unicode" print_function = pyrex_prefix + "print" print_function_kwargs = pyrex_prefix + "print_kwargs" cleanup_cname = pyrex_prefix + "module_cleanup" @@ -116,11 +128,13 @@ frame_cname = pyrex_prefix + "frame" frame_code_cname = pyrex_prefix + "frame_code" binding_cfunc = pyrex_prefix + "binding_PyCFunctionType" fused_func_prefix = pyrex_prefix + 'fuse_' -quick_temp_cname = pyrex_prefix + "temp" # temp variable for quick'n'dirty temping +quick_temp_cname = pyrex_prefix + "temp" # temp variable for quick'n'dirty temping tp_dict_version_temp = pyrex_prefix + "tp_dict_version" obj_dict_version_temp = pyrex_prefix + "obj_dict_version" -type_dict_guard_temp = pyrex_prefix + "type_dict_guard" +type_dict_guard_temp = pyrex_prefix + "typedict_guard" cython_runtime_cname = pyrex_prefix + "cython_runtime" +cyfunction_type_cname = pyrex_prefix + "CyFunctionType" +fusedfunction_type_cname = pyrex_prefix + "FusedFunctionType" global_code_object_cache_find = pyrex_prefix + 'find_code_object' global_code_object_cache_insert = pyrex_prefix + 'insert_code_object' @@ -152,8 +166,11 @@ exc_vars = (exc_type_name, exc_value_name, exc_tb_name) api_name = pyrex_prefix + "capi__" -h_guard_prefix = "__PYX_HAVE__" -api_guard_prefix = "__PYX_HAVE_API__" +# the h and api guards get changed to: +# __PYX_HAVE__FILENAME (for ascii filenames) +# __PYX_HAVE_U_PUNYCODEFILENAME (for non-ascii filenames) +h_guard_prefix = "__PYX_HAVE_" +api_guard_prefix = "__PYX_HAVE_API_" api_func_guard = "__PYX_HAVE_API_FUNC_" PYX_NAN = "__PYX_NAN()" diff --git a/Cython/Compiler/Nodes.py b/Cython/Compiler/Nodes.py index c6d0b02f0..4b0da59cf 100644 --- a/Cython/Compiler/Nodes.py +++ b/Cython/Compiler/Nodes.py @@ -5,6 +5,7 @@ from __future__ import absolute_import import cython + cython.declare(sys=object, os=object, copy=object, Builtin=object, error=object, warning=object, Naming=object, PyrexTypes=object, py_object_type=object, ModuleScope=object, LocalScope=object, ClosureScope=object, @@ -16,13 +17,14 @@ import sys, os, copy from itertools import chain from . import Builtin -from .Errors import error, warning, InternalError, CompileError +from .Errors import error, warning, InternalError, CompileError, CannotSpecialize from . import Naming from . import PyrexTypes from . import TypeSlots from .PyrexTypes import py_object_type, error_type -from .Symtab import (ModuleScope, LocalScope, ClosureScope, - StructOrUnionScope, PyClassScope, CppClassScope, TemplateScope) +from .Symtab import (ModuleScope, LocalScope, ClosureScope, PropertyScope, + StructOrUnionScope, PyClassScope, CppClassScope, TemplateScope, + CppScopedEnumScope, punycodify_name) from .Code import UtilityCode from .StringEncoding import EncodedString from . import Future @@ -38,6 +40,9 @@ else: _py_int_types = (int, long) +IMPLICIT_CLASSMETHODS = {"__init_subclass__", "__class_getitem__"} + + def relative_position(pos): return (pos[0].get_filenametable_entry(), pos[1]) @@ -68,53 +73,6 @@ def embed_position(pos, docstring): return doc -def analyse_type_annotation(annotation, env, assigned_value=None): - base_type = None - is_ambiguous = False - explicit_pytype = explicit_ctype = False - if annotation.is_dict_literal: - warning(annotation.pos, - "Dicts should no longer be used as type annotations. Use 'cython.int' etc. directly.") - for name, value in annotation.key_value_pairs: - if not name.is_string_literal: - continue - if name.value in ('type', b'type'): - explicit_pytype = True - if not explicit_ctype: - annotation = value - elif name.value in ('ctype', b'ctype'): - explicit_ctype = True - annotation = value - if explicit_pytype and explicit_ctype: - warning(annotation.pos, "Duplicate type declarations found in signature annotation") - arg_type = annotation.analyse_as_type(env) - if annotation.is_name and not annotation.cython_attribute and annotation.name in ('int', 'long', 'float'): - # Map builtin numeric Python types to C types in safe cases. - if assigned_value is not None and arg_type is not None and not arg_type.is_pyobject: - assigned_type = assigned_value.infer_type(env) - if assigned_type and assigned_type.is_pyobject: - # C type seems unsafe, e.g. due to 'None' default value => ignore annotation type - is_ambiguous = True - arg_type = None - # ignore 'int' and require 'cython.int' to avoid unsafe integer declarations - if arg_type in (PyrexTypes.c_long_type, PyrexTypes.c_int_type, PyrexTypes.c_float_type): - arg_type = PyrexTypes.c_double_type if annotation.name == 'float' else py_object_type - elif arg_type is not None and annotation.is_string_literal: - warning(annotation.pos, - "Strings should no longer be used for type declarations. Use 'cython.int' etc. directly.") - if arg_type is not None: - if explicit_pytype and not explicit_ctype and not arg_type.is_pyobject: - warning(annotation.pos, - "Python type declaration in signature annotation does not refer to a Python type") - base_type = CAnalysedBaseTypeNode( - annotation.pos, type=arg_type, is_arg=True) - elif is_ambiguous: - warning(annotation.pos, "Ambiguous types in annotation, ignoring") - else: - warning(annotation.pos, "Unknown type declaration in annotation, ignoring") - return base_type, arg_type - - def write_func_call(func, codewriter_class): def f(*args, **kwds): if len(args) > 1 and isinstance(args[1], codewriter_class): @@ -125,19 +83,16 @@ def write_func_call(func, codewriter_class): ' ' * code.call_level, node.__class__.__name__, func.__name__, - node.pos[1:]) - pristine = code.buffer.stream.tell() - code.putln(marker) + node.pos[1:], + ) + insertion_point = code.insertion_point() start = code.buffer.stream.tell() code.call_level += 4 res = func(*args, **kwds) code.call_level -= 4 - if start == code.buffer.stream.tell(): - # no code written => undo writing marker - code.buffer.stream.truncate(pristine) - else: - marker = marker.replace('->', '<-', 1) - code.putln(marker) + if start != code.buffer.stream.tell(): + code.putln(marker.replace('->', '<-', 1)) + insertion_point.putln(marker) return res else: return func(*args, **kwds) @@ -160,9 +115,11 @@ class VerboseCodeWriter(type): class CheckAnalysers(type): """Metaclass to check that type analysis functions return a node. """ - methods = set(['analyse_types', - 'analyse_expressions', - 'analyse_target_types']) + methods = frozenset({ + 'analyse_types', + 'analyse_expressions', + 'analyse_target_types', + }) def __new__(cls, name, bases, attrs): from types import FunctionType @@ -200,6 +157,7 @@ class Node(object): is_literal = 0 is_terminator = 0 is_wrapper = False # is a DefNode wrapper for a C function + is_cproperty = False temps = None # All descendants should set child_attrs to a list of the attributes @@ -254,7 +212,7 @@ class Node(object): # - # There are 3 phases of parse tree processing, applied in order to + # There are 3 main phases of parse tree processing, applied in order to # all the statements in a given scope-block: # # (0) analyse_declarations @@ -266,25 +224,25 @@ class Node(object): # Determine the result types of expressions and fill in the # 'type' attribute of each ExprNode. Insert coercion nodes into the # tree where needed to convert to and from Python objects. - # Allocate temporary locals for intermediate results. Fill - # in the 'result_code' attribute of each ExprNode with a C code - # fragment. + # Replace tree nodes with more appropriate implementations found by + # the type analysis. # # (2) generate_code # Emit C code for all declarations, statements and expressions. - # Recursively applies the 3 processing phases to the bodies of - # functions. + # + # These phases are triggered by tree transformations. + # See the full pipeline in Pipeline.py. # def analyse_declarations(self, env): pass def analyse_expressions(self, env): - raise InternalError("analyse_expressions not implemented for %s" % \ + raise InternalError("analyse_expressions not implemented for %s" % self.__class__.__name__) def generate_code(self, code): - raise InternalError("generate_code not implemented for %s" % \ + raise InternalError("generate_code not implemented for %s" % self.__class__.__name__) def annotate(self, code): @@ -360,6 +318,7 @@ class Node(object): return u'"%s":%d:%d\n%s\n' % ( source_desc.get_escaped_description(), line, col, u''.join(lines)) + class CompilerDirectivesNode(Node): """ Sets compiler directives for the children nodes @@ -402,6 +361,7 @@ class CompilerDirectivesNode(Node): self.body.annotate(code) code.globalstate.directives = old + class BlockNode(object): # Mixin class for nodes representing a declaration block. @@ -415,6 +375,7 @@ class BlockNode(object): for node in env.lambda_defs: node.generate_function_definitions(env, code) + class StatListNode(Node): # stats a list of StatNode @@ -469,7 +430,7 @@ class StatNode(Node): pass def generate_execution_code(self, code): - raise InternalError("generate_execution_code not implemented for %s" % \ + raise InternalError("generate_execution_code not implemented for %s" % self.__class__.__name__) @@ -499,8 +460,13 @@ class CDefExternNode(StatNode): env.add_include_file(self.include_file, self.verbatim_include, late) def analyse_expressions(self, env): + # Allow C properties, inline methods, etc. also in external types. + self.body = self.body.analyse_expressions(env) return self + def generate_function_definitions(self, env, code): + self.body.generate_function_definitions(env, code) + def generate_execution_code(self, code): pass @@ -525,6 +491,9 @@ class CDeclaratorNode(Node): calling_convention = "" + def declared_name(self): + return None + def analyse_templates(self): # Only C++ functions have templates. return None @@ -539,6 +508,9 @@ class CNameDeclaratorNode(CDeclaratorNode): default = None + def declared_name(self): + return self.name + def analyse(self, base_type, env, nonempty=0, visibility=None, in_pxd=False): if nonempty and self.name == '': # May have mistaken the name for the type. @@ -551,7 +523,12 @@ class CNameDeclaratorNode(CDeclaratorNode): base_type = py_object_type if base_type.is_fused and env.fused_to_specific: - base_type = base_type.specialize(env.fused_to_specific) + try: + base_type = base_type.specialize(env.fused_to_specific) + except CannotSpecialize: + error(self.pos, + "'%s' cannot be specialized since its type is not a fused argument to this function" % + self.name) self.type = base_type return self, base_type @@ -562,6 +539,9 @@ class CPtrDeclaratorNode(CDeclaratorNode): child_attrs = ["base"] + def declared_name(self): + return self.base.declared_name() + def analyse_templates(self): return self.base.analyse_templates() @@ -572,14 +552,17 @@ class CPtrDeclaratorNode(CDeclaratorNode): return self.base.analyse(ptr_type, env, nonempty=nonempty, visibility=visibility, in_pxd=in_pxd) -class CReferenceDeclaratorNode(CDeclaratorNode): - # base CDeclaratorNode - +class _CReferenceDeclaratorBaseNode(CDeclaratorNode): child_attrs = ["base"] + def declared_name(self): + return self.base.declared_name() + def analyse_templates(self): return self.base.analyse_templates() + +class CReferenceDeclaratorNode(_CReferenceDeclaratorBaseNode): def analyse(self, base_type, env, nonempty=0, visibility=None, in_pxd=False): if base_type.is_pyobject: error(self.pos, "Reference base type cannot be a Python object") @@ -587,6 +570,14 @@ class CReferenceDeclaratorNode(CDeclaratorNode): return self.base.analyse(ref_type, env, nonempty=nonempty, visibility=visibility, in_pxd=in_pxd) +class CppRvalueReferenceDeclaratorNode(_CReferenceDeclaratorBaseNode): + def analyse(self, base_type, env, nonempty=0, visibility=None, in_pxd=False): + if base_type.is_pyobject: + error(self.pos, "Rvalue-reference base type cannot be a Python object") + ref_type = PyrexTypes.cpp_rvalue_ref_type(base_type) + return self.base.analyse(ref_type, env, nonempty=nonempty, visibility=visibility, in_pxd=in_pxd) + + class CArrayDeclaratorNode(CDeclaratorNode): # base CDeclaratorNode # dimension ExprNode @@ -636,8 +627,8 @@ class CFuncDeclaratorNode(CDeclaratorNode): # args [CArgDeclNode] # templates [TemplatePlaceholderType] # has_varargs boolean - # exception_value ConstNode - # exception_check boolean True if PyErr_Occurred check needed + # exception_value ConstNode or NameNode NameNode when the name of a c++ exception conversion function + # exception_check boolean or "+" True if PyErr_Occurred check needed, "+" for a c++ check # nogil boolean Can be called without gil # with_gil boolean Acquire gil around function body # is_const_method boolean Whether this is a const method @@ -649,6 +640,9 @@ class CFuncDeclaratorNode(CDeclaratorNode): is_const_method = 0 templates = None + def declared_name(self): + return self.base.declared_name() + def analyse_templates(self): if isinstance(self.base, CArrayDeclaratorNode): from .ExprNodes import TupleNode, NameNode @@ -734,8 +728,8 @@ class CFuncDeclaratorNode(CDeclaratorNode): self.exception_value = ConstNode( self.pos, value=return_type.exception_value, type=return_type) if self.exception_value: - self.exception_value = self.exception_value.analyse_const_expression(env) if self.exception_check == '+': + self.exception_value = self.exception_value.analyse_const_expression(env) exc_val_type = self.exception_value.type if (not exc_val_type.is_error and not exc_val_type.is_pyobject @@ -748,16 +742,25 @@ class CFuncDeclaratorNode(CDeclaratorNode): "Exception value must be a Python exception or cdef function with no arguments or *.") exc_val = self.exception_value else: - self.exception_value = self.exception_value.coerce_to( + self.exception_value = self.exception_value.analyse_types(env).coerce_to( return_type, env).analyse_const_expression(env) exc_val = self.exception_value.get_constant_c_result_code() if exc_val is None: - raise InternalError( - "get_constant_c_result_code not implemented for %s" % - self.exception_value.__class__.__name__) + error(self.exception_value.pos, "Exception value must be constant") if not return_type.assignable_from(self.exception_value.type): error(self.exception_value.pos, "Exception value incompatible with function return type") + if (visibility != 'extern' + and (return_type.is_int or return_type.is_float) + and self.exception_value.has_constant_result()): + try: + type_default_value = float(return_type.default_value) + except ValueError: + pass + else: + if self.exception_value.constant_result == type_default_value: + warning(self.pos, "Ambiguous exception value, same as default return value: %r" % + self.exception_value.constant_result) exc_check = self.exception_check if return_type.is_cfunction: error(self.pos, "Function cannot return a function") @@ -789,6 +792,13 @@ class CFuncDeclaratorNode(CDeclaratorNode): error(self.pos, "cannot have both '%s' and '%s' " "calling conventions" % (current, callspec)) func_type.calling_convention = callspec + + if func_type.return_type.is_rvalue_reference: + warning(self.pos, "Rvalue-reference as function return type not supported", 1) + for arg in func_type.args: + if arg.type.is_rvalue_reference and not arg.is_forwarding_reference(): + warning(self.pos, "Rvalue-reference as function argument not supported", 1) + return self.base.analyse(func_type, env, visibility=visibility, in_pxd=in_pxd) def declare_optional_arg_struct(self, func_type, env, fused_cname=None): @@ -850,8 +860,12 @@ class CArgDeclNode(Node): # annotation ExprNode or None Py3 function arg annotation # is_self_arg boolean Is the "self" arg of an extension type method # is_type_arg boolean Is the "class" arg of an extension type classmethod - # is_kw_only boolean Is a keyword-only argument + # kw_only boolean Is a keyword-only argument # is_dynamic boolean Non-literal arg stored inside CyFunction + # pos_only boolean Is a positional-only argument + # + # name_cstring property that converts the name to a cstring taking care of unicode + # and quoting it child_attrs = ["base_type", "declarator", "default", "annotation"] outer_attrs = ["default", "annotation"] @@ -860,6 +874,7 @@ class CArgDeclNode(Node): is_type_arg = 0 is_generic = 1 kw_only = 0 + pos_only = 0 not_none = 0 or_none = 0 type = None @@ -868,57 +883,78 @@ class CArgDeclNode(Node): annotation = None is_dynamic = 0 + def declared_name(self): + return self.declarator.declared_name() + + @property + def name_cstring(self): + return self.name.as_c_string_literal() + + @property + def hdr_cname(self): + # done lazily - needs self.entry to be set to get the class-mangled + # name, which means it has to be generated relatively late + if self.needs_conversion: + return punycodify_name(Naming.arg_prefix + self.entry.name) + else: + return punycodify_name(Naming.var_prefix + self.entry.name) + + def analyse(self, env, nonempty=0, is_self_arg=False): if is_self_arg: - self.base_type.is_self_arg = self.is_self_arg = True - if self.type is None: - # The parser may misinterpret names as types. We fix that here. - if isinstance(self.declarator, CNameDeclaratorNode) and self.declarator.name == '': - if nonempty: - if self.base_type.is_basic_c_type: - # char, short, long called "int" - type = self.base_type.analyse(env, could_be_name=True) - arg_name = type.empty_declaration_code() - else: - arg_name = self.base_type.name - self.declarator.name = EncodedString(arg_name) - self.base_type.name = None - self.base_type.is_basic_c_type = False - could_be_name = True - else: - could_be_name = False - self.base_type.is_arg = True - base_type = self.base_type.analyse(env, could_be_name=could_be_name) - if hasattr(self.base_type, 'arg_name') and self.base_type.arg_name: - self.declarator.name = self.base_type.arg_name - - # The parser is unable to resolve the ambiguity of [] as part of the - # type (e.g. in buffers) or empty declarator (as with arrays). - # This is only arises for empty multi-dimensional arrays. - if (base_type.is_array - and isinstance(self.base_type, TemplatedTypeNode) - and isinstance(self.declarator, CArrayDeclaratorNode)): - declarator = self.declarator - while isinstance(declarator.base, CArrayDeclaratorNode): - declarator = declarator.base - declarator.base = self.base_type.array_declarator - base_type = base_type.base_type - - # inject type declaration from annotations - # this is called without 'env' by AdjustDefByDirectives transform before declaration analysis - if self.annotation and env and env.directives['annotation_typing'] and self.base_type.name is None: - arg_type = self.inject_type_from_annotations(env) - if arg_type is not None: - base_type = arg_type - return self.declarator.analyse(base_type, env, nonempty=nonempty) - else: + self.base_type.is_self_arg = self.is_self_arg = is_self_arg + if self.type is not None: return self.name_declarator, self.type + # The parser may misinterpret names as types. We fix that here. + if isinstance(self.declarator, CNameDeclaratorNode) and self.declarator.name == '': + if nonempty: + if self.base_type.is_basic_c_type: + # char, short, long called "int" + type = self.base_type.analyse(env, could_be_name=True) + arg_name = type.empty_declaration_code() + else: + arg_name = self.base_type.name + self.declarator.name = EncodedString(arg_name) + self.base_type.name = None + self.base_type.is_basic_c_type = False + could_be_name = True + else: + could_be_name = False + self.base_type.is_arg = True + base_type = self.base_type.analyse(env, could_be_name=could_be_name) + base_arg_name = getattr(self.base_type, 'arg_name', None) + if base_arg_name: + self.declarator.name = base_arg_name + + # The parser is unable to resolve the ambiguity of [] as part of the + # type (e.g. in buffers) or empty declarator (as with arrays). + # This is only arises for empty multi-dimensional arrays. + if (base_type.is_array + and isinstance(self.base_type, TemplatedTypeNode) + and isinstance(self.declarator, CArrayDeclaratorNode)): + declarator = self.declarator + while isinstance(declarator.base, CArrayDeclaratorNode): + declarator = declarator.base + declarator.base = self.base_type.array_declarator + base_type = base_type.base_type + + # inject type declaration from annotations + # this is called without 'env' by AdjustDefByDirectives transform before declaration analysis + if (self.annotation and env and env.directives['annotation_typing'] + # CSimpleBaseTypeNode has a name attribute; CAnalysedBaseTypeNode + # (and maybe other options) doesn't + and getattr(self.base_type, "name", None) is None): + arg_type = self.inject_type_from_annotations(env) + if arg_type is not None: + base_type = arg_type + return self.declarator.analyse(base_type, env, nonempty=nonempty) + def inject_type_from_annotations(self, env): annotation = self.annotation if not annotation: return None - base_type, arg_type = analyse_type_annotation(annotation, env, assigned_value=self.default) + base_type, arg_type = annotation.analyse_type_annotation(env, assigned_value=self.default) if base_type is not None: self.base_type = base_type return arg_type @@ -947,8 +983,7 @@ class CArgDeclNode(Node): default.make_owned_reference(code) result = default.result() if overloaded_assignment else default.result_as(self.type) code.putln("%s = %s;" % (target, result)) - if self.type.is_pyobject: - code.put_giveref(default.result()) + code.put_giveref(default.result(), self.type) default.generate_post_assignment_code(code) default.free_temps(code) @@ -989,6 +1024,7 @@ class CSimpleBaseTypeNode(CBaseTypeNode): module_path = [] is_basic_c_type = False complex = False + is_self_arg = False def analyse(self, env, could_be_name=False): # Return type descriptor. @@ -1014,7 +1050,10 @@ class CSimpleBaseTypeNode(CBaseTypeNode): scope = env for item in self.module_path: entry = scope.lookup(item) - if entry is not None and entry.is_cpp_class: + if entry is not None and ( + entry.is_cpp_class or + entry.is_type and entry.type.is_cpp_class + ): scope = entry.type.scope else: scope = None @@ -1043,7 +1082,7 @@ class CSimpleBaseTypeNode(CBaseTypeNode): self.arg_name = EncodedString(self.name) else: if self.templates: - if not self.name in self.templates: + if self.name not in self.templates: error(self.pos, "'%s' is not a type identifier" % self.name) type = PyrexTypes.TemplatePlaceholderType(self.name) else: @@ -1199,7 +1238,12 @@ class TemplatedTypeNode(CBaseTypeNode): self.type = self.array_declarator.analyse(base_type, env)[1] if self.type.is_fused and env.fused_to_specific: - self.type = self.type.specialize(env.fused_to_specific) + try: + self.type = self.type.specialize(env.fused_to_specific) + except CannotSpecialize: + error(self.pos, + "'%s' cannot be specialized since its type is not a fused argument to this function" % + self.name) return self.type @@ -1273,8 +1317,10 @@ class FusedTypeNode(CBaseTypeNode): return PyrexTypes.FusedType(types, name=self.name) -class CConstTypeNode(CBaseTypeNode): +class CConstOrVolatileTypeNode(CBaseTypeNode): # base_type CBaseTypeNode + # is_const boolean + # is_volatile boolean child_attrs = ["base_type"] @@ -1282,8 +1328,8 @@ class CConstTypeNode(CBaseTypeNode): base = self.base_type.analyse(env, could_be_name) if base.is_pyobject: error(self.pos, - "Const base type cannot be a Python object") - return PyrexTypes.c_const_type(base) + "Const/volatile base type cannot be a Python object") + return PyrexTypes.c_const_or_volatile_type(base, self.is_const, self.is_volatile) class CVarDefNode(StatNode): @@ -1369,6 +1415,8 @@ class CVarDefNode(StatNode): return if type.is_reference and self.visibility != 'extern': error(declarator.pos, "C++ references cannot be declared; use a pointer instead") + if type.is_rvalue_reference and self.visibility != 'extern': + error(declarator.pos, "C++ rvalue-references cannot be declared") if type.is_cfunction: if 'staticmethod' in env.directives: type.is_static_method = True @@ -1383,8 +1431,7 @@ class CVarDefNode(StatNode): self.entry.create_wrapper = True else: if self.overridable: - warning(self.pos, "cpdef variables will not be supported in Cython 3; " - "currently they are no different from cdef variables", 2) + error(self.pos, "Variables cannot be declared with 'cpdef'. Use 'cdef' instead.") if self.directive_locals: error(self.pos, "Decorators can only be followed by functions") self.entry = dest_scope.declare_var( @@ -1529,36 +1576,62 @@ class CppClassNode(CStructOrUnionDefNode, BlockNode): class CEnumDefNode(StatNode): - # name string or None - # cname string or None - # items [CEnumDefItemNode] - # typedef_flag boolean - # visibility "public" or "private" or "extern" - # api boolean - # in_pxd boolean - # create_wrapper boolean - # entry Entry - - child_attrs = ["items"] + # name string or None + # cname string or None + # scoped boolean Is a C++ scoped enum + # underlying_type CSimpleBaseTypeNode The underlying value type (int or C++ type) + # items [CEnumDefItemNode] + # typedef_flag boolean + # visibility "public" or "private" or "extern" + # api boolean + # in_pxd boolean + # create_wrapper boolean + # entry Entry + # doc EncodedString or None Doc string + + child_attrs = ["items", "underlying_type"] + doc = None def declare(self, env): - self.entry = env.declare_enum( - self.name, self.pos, - cname=self.cname, typedef_flag=self.typedef_flag, - visibility=self.visibility, api=self.api, - create_wrapper=self.create_wrapper) + doc = None + if Options.docstrings: + doc = embed_position(self.pos, self.doc) + + self.entry = env.declare_enum( + self.name, self.pos, + cname=self.cname, + scoped=self.scoped, + typedef_flag=self.typedef_flag, + visibility=self.visibility, api=self.api, + create_wrapper=self.create_wrapper, doc=doc) def analyse_declarations(self, env): + scope = None + underlying_type = self.underlying_type.analyse(env) + + if not underlying_type.is_int: + error(self.underlying_type.pos, "underlying type is not an integral type") + + self.entry.type.underlying_type = underlying_type + + if self.scoped and self.items is not None: + scope = CppScopedEnumScope(self.name, env) + scope.type = self.entry.type + else: + scope = env + if self.items is not None: if self.in_pxd and not env.in_cinclude: self.entry.defined_in_pxd = 1 for item in self.items: - item.analyse_declarations(env, self.entry) + item.analyse_declarations(scope, self.entry) def analyse_expressions(self, env): return self def generate_execution_code(self, code): + if self.scoped: + return # nothing to do here for C++ enums if self.visibility == 'public' or self.api: code.mark_pos(self.pos) temp = code.funcstate.allocate_temp(PyrexTypes.py_object_type, manage_ref=True) @@ -1567,7 +1640,7 @@ class CEnumDefNode(StatNode): temp, item.cname, code.error_goto_if_null(temp, item.pos))) - code.put_gotref(temp) + code.put_gotref(temp, PyrexTypes.py_object_type) code.putln('if (PyDict_SetItemString(%s, "%s", %s) < 0) %s' % ( Naming.moddict_cname, item.name, @@ -1590,9 +1663,15 @@ class CEnumDefItemNode(StatNode): if not self.value.type.is_int: self.value = self.value.coerce_to(PyrexTypes.c_int_type, env) self.value = self.value.analyse_const_expression(env) + + if enum_entry.type.is_cpp_enum: + cname = "%s::%s" % (enum_entry.cname, self.name) + else: + cname = self.cname + entry = env.declare_const( self.name, enum_entry.type, - self.value, self.pos, cname=self.cname, + self.value, self.pos, cname=cname, visibility=enum_entry.visibility, api=enum_entry.api, create_wrapper=enum_entry.create_wrapper and enum_entry.name is None) enum_entry.enum_values.append(entry) @@ -1659,6 +1738,8 @@ class FuncDefNode(StatNode, BlockNode): needs_outer_scope = False pymethdef_required = False is_generator = False + is_coroutine = False + is_asyncgen = False is_generator_body = False is_async_def = False modifiers = [] @@ -1667,6 +1748,7 @@ class FuncDefNode(StatNode, BlockNode): starstar_arg = None is_cyfunction = False code_object = None + return_type_annotation = None def analyse_default_values(self, env): default_seen = 0 @@ -1684,18 +1766,12 @@ class FuncDefNode(StatNode, BlockNode): elif default_seen: error(arg.pos, "Non-default argument following default argument") - def analyse_annotation(self, env, annotation): - # Annotations can not only contain valid Python expressions but arbitrary type references. - if annotation is None: - return None - if not env.directives['annotation_typing'] or annotation.analyse_as_type(env) is None: - annotation = annotation.analyse_types(env) - return annotation - def analyse_annotations(self, env): for arg in self.args: if arg.annotation: - arg.annotation = self.analyse_annotation(env, arg.annotation) + arg.annotation = arg.annotation.analyse_types(env) + if self.return_type_annotation: + self.return_type_annotation = self.return_type_annotation.analyse_types(env) def align_argument_type(self, env, arg): # @cython.locals() @@ -1718,6 +1794,9 @@ class FuncDefNode(StatNode, BlockNode): error(type_node.pos, "Previous declaration here") else: arg.type = other_type + if arg.type.is_complex: + # utility code for complex types is special-cased and also important to ensure that it's run + arg.type.create_declaration_utility_code(env) return arg def need_gil_acquisition(self, lenv): @@ -1749,8 +1828,6 @@ class FuncDefNode(StatNode, BlockNode): def generate_function_definitions(self, env, code): from . import Buffer - if self.return_type.is_memoryviewslice: - from . import MemoryView lenv = self.local_scope if lenv.is_closure_scope and not lenv.is_passthrough: @@ -1778,6 +1855,8 @@ class FuncDefNode(StatNode, BlockNode): profile = code.globalstate.directives['profile'] linetrace = code.globalstate.directives['linetrace'] if profile or linetrace: + if linetrace: + code.use_fast_gil_utility_code() code.globalstate.use_utility_code( UtilityCode.load_cached("Profile", "Profile.c")) @@ -1823,14 +1902,15 @@ class FuncDefNode(StatNode, BlockNode): # Initialize the return variable __pyx_r init = "" - if not self.return_type.is_void: - if self.return_type.is_pyobject: + return_type = self.return_type + if not return_type.is_void: + if return_type.is_pyobject: init = " = NULL" - elif self.return_type.is_memoryviewslice: - init = ' = ' + MemoryView.memslice_entry_init + elif return_type.is_memoryviewslice: + init = ' = ' + return_type.literal_code(return_type.default_value) code.putln("%s%s;" % ( - self.return_type.declaration_code(Naming.retval_cname), + return_type.declaration_code(Naming.retval_cname), init)) tempvardecl_code = code.insertion_point() @@ -1862,11 +1942,12 @@ class FuncDefNode(StatNode, BlockNode): use_refnanny = not lenv.nogil or lenv.has_with_gil_block + gilstate_decl = None if acquire_gil or acquire_gil_for_var_decls_only: code.put_ensure_gil() code.funcstate.gil_owned = True - elif lenv.nogil and lenv.has_with_gil_block: - code.declare_gilstate() + else: + gilstate_decl = code.insertion_point() if profile or linetrace: if not self.is_generator: @@ -1908,7 +1989,7 @@ class FuncDefNode(StatNode, BlockNode): code.put_incref("Py_None", py_object_type) code.putln(code.error_goto(self.pos)) code.putln("} else {") - code.put_gotref(Naming.cur_scope_cname) + code.put_gotref(Naming.cur_scope_cname, lenv.scope_class.type) code.putln("}") # Note that it is unsafe to decref the scope at this point. if self.needs_outer_scope: @@ -1927,7 +2008,7 @@ class FuncDefNode(StatNode, BlockNode): elif self.needs_closure: # inner closures own a reference to their outer parent code.put_incref(outer_scope_cname, cenv.scope_class.type) - code.put_giveref(outer_scope_cname) + code.put_giveref(outer_scope_cname, cenv.scope_class.type) # ----- Trace function call if profile or linetrace: # this looks a bit late, but if we don't get here due to a @@ -1947,18 +2028,18 @@ class FuncDefNode(StatNode, BlockNode): # incref it to properly keep track of refcounts. is_cdef = isinstance(self, CFuncDefNode) for entry in lenv.arg_entries: - if entry.type.is_pyobject: - if (acquire_gil or len(entry.cf_assignments) > 1) and not entry.in_closure: + if not entry.type.is_memoryviewslice: + if (acquire_gil or entry.cf_is_reassigned) and not entry.in_closure: code.put_var_incref(entry) - # Note: defaults are always incref-ed. For def functions, we # we acquire arguments from object conversion, so we have # new references. If we are a cdef function, we need to # incref our arguments - elif is_cdef and entry.type.is_memoryviewslice and len(entry.cf_assignments) > 1: - code.put_incref_memoryviewslice(entry.cname, have_gil=code.funcstate.gil_owned) + elif is_cdef and entry.cf_is_reassigned: + code.put_var_incref_memoryviewslice(entry, + have_gil=code.funcstate.gil_owned) for entry in lenv.var_entries: - if entry.is_arg and len(entry.cf_assignments) > 1 and not entry.in_closure: + if entry.is_arg and entry.cf_is_reassigned and not entry.in_closure: if entry.xdecref_cleanup: code.put_var_xincref(entry) else: @@ -1989,27 +2070,45 @@ class FuncDefNode(StatNode, BlockNode): code.putln("") code.putln("/* function exit code */") + gil_owned = { + 'success': code.funcstate.gil_owned, + 'error': code.funcstate.gil_owned, + 'gil_state_declared': gilstate_decl is None, + } + def assure_gil(code_path, code=code): + if not gil_owned[code_path]: + if not gil_owned['gil_state_declared']: + gilstate_decl.declare_gilstate() + gil_owned['gil_state_declared'] = True + code.put_ensure_gil(declare_gilstate=False) + gil_owned[code_path] = True + # ----- Default return value + return_type = self.return_type if not self.body.is_terminator: - if self.return_type.is_pyobject: - #if self.return_type.is_extension_type: + if return_type.is_pyobject: + #if return_type.is_extension_type: # lhs = "(PyObject *)%s" % Naming.retval_cname #else: lhs = Naming.retval_cname - code.put_init_to_py_none(lhs, self.return_type) - else: - val = self.return_type.default_value + assure_gil('success') + code.put_init_to_py_none(lhs, return_type) + elif not return_type.is_memoryviewslice: + # memory view structs receive their default value on initialisation + val = return_type.default_value if val: code.putln("%s = %s;" % (Naming.retval_cname, val)) - elif not self.return_type.is_void: + elif not return_type.is_void: code.putln("__Pyx_pretend_to_initialize(&%s);" % Naming.retval_cname) + # ----- Error cleanup - if code.error_label in code.labels_used: + if code.label_used(code.error_label): if not self.body.is_terminator: code.put_goto(code.return_label) code.put_label(code.error_label) for cname, type in code.funcstate.all_managed_temps(): - code.put_xdecref(cname, type, have_gil=not lenv.nogil) + assure_gil('error') + code.put_xdecref(cname, type, have_gil=gil_owned['error']) # Clean up buffers -- this calls a Python function # so need to save and restore error state @@ -2019,6 +2118,7 @@ class FuncDefNode(StatNode, BlockNode): code.globalstate.use_utility_code(restore_exception_utility_code) code.putln("{ PyObject *__pyx_type, *__pyx_value, *__pyx_tb;") code.putln("__Pyx_PyThreadState_declare") + assure_gil('error') code.putln("__Pyx_PyThreadState_assign") code.putln("__Pyx_ErrFetch(&__pyx_type, &__pyx_value, &__pyx_tb);") for entry in used_buffer_entries: @@ -2026,7 +2126,8 @@ class FuncDefNode(StatNode, BlockNode): #code.putln("%s = 0;" % entry.cname) code.putln("__Pyx_ErrRestore(__pyx_type, __pyx_value, __pyx_tb);}") - if self.return_type.is_memoryviewslice: + if return_type.is_memoryviewslice: + from . import MemoryView MemoryView.put_init_entry(Naming.retval_cname, code) err_val = Naming.retval_cname else: @@ -2038,104 +2139,129 @@ class FuncDefNode(StatNode, BlockNode): # code.globalstate.use_utility_code(get_exception_tuple_utility_code) # code.put_trace_exception() - if lenv.nogil and not lenv.has_with_gil_block: - code.putln("{") - code.put_ensure_gil() - + assure_gil('error') code.put_add_traceback(self.entry.qualified_name) - - if lenv.nogil and not lenv.has_with_gil_block: - code.put_release_ensured_gil() - code.putln("}") else: warning(self.entry.pos, "Unraisable exception in function '%s'." % self.entry.qualified_name, 0) - code.put_unraisable(self.entry.qualified_name, lenv.nogil) - default_retval = self.return_type.default_value + assure_gil('error') + code.put_unraisable(self.entry.qualified_name) + default_retval = return_type.default_value if err_val is None and default_retval: err_val = default_retval if err_val is not None: if err_val != Naming.retval_cname: code.putln("%s = %s;" % (Naming.retval_cname, err_val)) - elif not self.return_type.is_void: + elif not return_type.is_void: code.putln("__Pyx_pretend_to_initialize(&%s);" % Naming.retval_cname) if is_getbuffer_slot: + assure_gil('error') self.getbuffer_error_cleanup(code) + def align_error_path_gil_to_success_path(code=code.insertion_point()): + # align error and success GIL state when both join + if gil_owned['success']: + assure_gil('error', code=code) + elif gil_owned['error']: + code.put_release_ensured_gil() + gil_owned['error'] = False + assert gil_owned['error'] == gil_owned['success'], "%s: error path %s != success path %s" % ( + self.pos, gil_owned['error'], gil_owned['success']) + # If we are using the non-error cleanup section we should # jump past it if we have an error. The if-test below determine # whether this section is used. - if buffers_present or is_getbuffer_slot or self.return_type.is_memoryviewslice: + if buffers_present or is_getbuffer_slot or return_type.is_memoryviewslice: + # In the buffer cases, we already called assure_gil('error') and own the GIL. + assert gil_owned['error'] or return_type.is_memoryviewslice code.put_goto(code.return_from_error_cleanup_label) + else: + # Adapt the GIL state to the success path right now. + align_error_path_gil_to_success_path() + else: + # No error path, no need to adapt the GIL state. + def align_error_path_gil_to_success_path(): pass # ----- Non-error return cleanup - code.put_label(code.return_label) - for entry in used_buffer_entries: - Buffer.put_release_buffer_code(code, entry) - if is_getbuffer_slot: - self.getbuffer_normal_cleanup(code) + if code.label_used(code.return_label) or not code.label_used(code.error_label): + code.put_label(code.return_label) - if self.return_type.is_memoryviewslice: - # See if our return value is uninitialized on non-error return - # from . import MemoryView - # MemoryView.err_if_nogil_initialized_check(self.pos, env) - cond = code.unlikely(self.return_type.error_condition(Naming.retval_cname)) - code.putln( - 'if (%s) {' % cond) - if env.nogil: - code.put_ensure_gil() - code.putln( - 'PyErr_SetString(PyExc_TypeError, "Memoryview return value is not initialized");') - if env.nogil: - code.put_release_ensured_gil() - code.putln( - '}') + for entry in used_buffer_entries: + assure_gil('success') + Buffer.put_release_buffer_code(code, entry) + if is_getbuffer_slot: + assure_gil('success') + self.getbuffer_normal_cleanup(code) + + if return_type.is_memoryviewslice: + # See if our return value is uninitialized on non-error return + # from . import MemoryView + # MemoryView.err_if_nogil_initialized_check(self.pos, env) + cond = code.unlikely(return_type.error_condition(Naming.retval_cname)) + code.putln( + 'if (%s) {' % cond) + if not gil_owned['success']: + code.put_ensure_gil() + code.putln( + 'PyErr_SetString(PyExc_TypeError, "Memoryview return value is not initialized");') + if not gil_owned['success']: + code.put_release_ensured_gil() + code.putln( + '}') # ----- Return cleanup for both error and no-error return - code.put_label(code.return_from_error_cleanup_label) + if code.label_used(code.return_from_error_cleanup_label): + align_error_path_gil_to_success_path() + code.put_label(code.return_from_error_cleanup_label) for entry in lenv.var_entries: if not entry.used or entry.in_closure: continue - if entry.type.is_memoryviewslice: - code.put_xdecref_memoryviewslice(entry.cname, have_gil=not lenv.nogil) - elif entry.type.is_pyobject: - if not entry.is_arg or len(entry.cf_assignments) > 1: - if entry.xdecref_cleanup: - code.put_var_xdecref(entry) - else: - code.put_var_decref(entry) + if entry.type.is_pyobject: + if entry.is_arg and not entry.cf_is_reassigned: + continue + if entry.type.needs_refcounting: + assure_gil('success') + # FIXME ideally use entry.xdecref_cleanup but this currently isn't reliable + code.put_var_xdecref(entry, have_gil=gil_owned['success']) # Decref any increfed args for entry in lenv.arg_entries: - if entry.type.is_pyobject: - if (acquire_gil or len(entry.cf_assignments) > 1) and not entry.in_closure: - code.put_var_decref(entry) - elif (entry.type.is_memoryviewslice and - (not is_cdef or len(entry.cf_assignments) > 1)): + if entry.type.is_memoryviewslice: # decref slices of def functions and acquired slices from cdef # functions, but not borrowed slices from cdef functions. - code.put_xdecref_memoryviewslice(entry.cname, - have_gil=not lenv.nogil) + if is_cdef and not entry.cf_is_reassigned: + continue + else: + if entry.in_closure: + continue + if not acquire_gil and not entry.cf_is_reassigned: + continue + if entry.type.needs_refcounting: + assure_gil('success') + + # FIXME use entry.xdecref_cleanup - del arg seems to be the problem + code.put_var_xdecref(entry, have_gil=gil_owned['success']) if self.needs_closure: + assure_gil('success') code.put_decref(Naming.cur_scope_cname, lenv.scope_class.type) # ----- Return # This code is duplicated in ModuleNode.generate_module_init_func if not lenv.nogil: - default_retval = self.return_type.default_value + default_retval = return_type.default_value err_val = self.error_value() if err_val is None and default_retval: err_val = default_retval # FIXME: why is err_val not used? - if self.return_type.is_pyobject: - code.put_xgiveref(self.return_type.as_pyobject(Naming.retval_cname)) + code.put_xgiveref(Naming.retval_cname, return_type) if self.entry.is_special and self.entry.name == "__hash__": # Returning -1 for __hash__ is supposed to signal an error # We do as Python instances and coerce -1 into -2. + assure_gil('success') # in special methods, the GIL is owned anyway code.putln("if (unlikely(%s == -1) && !PyErr_Occurred()) %s = -2;" % ( Naming.retval_cname, Naming.retval_cname)) @@ -2143,23 +2269,22 @@ class FuncDefNode(StatNode, BlockNode): code.funcstate.can_trace = False if not self.is_generator: # generators are traced when iterated, not at creation - if self.return_type.is_pyobject: + if return_type.is_pyobject: code.put_trace_return( - Naming.retval_cname, nogil=not code.funcstate.gil_owned) + Naming.retval_cname, nogil=not gil_owned['success']) else: code.put_trace_return( - "Py_None", nogil=not code.funcstate.gil_owned) + "Py_None", nogil=not gil_owned['success']) - if not lenv.nogil: - # GIL holding function - code.put_finish_refcount_context() + if use_refnanny: + code.put_finish_refcount_context(nogil=not gil_owned['success']) - if acquire_gil or (lenv.nogil and lenv.has_with_gil_block): + if acquire_gil or (lenv.nogil and gil_owned['success']): # release the GIL (note that with-gil blocks acquire it on exit in their EnsureGILNode) code.put_release_ensured_gil() code.funcstate.gil_owned = False - if not self.return_type.is_void: + if not return_type.is_void: code.putln("return %s;" % Naming.retval_cname) code.putln("}") @@ -2194,11 +2319,11 @@ class FuncDefNode(StatNode, BlockNode): typeptr_cname = arg.type.typeptr_cname arg_code = "((PyObject *)%s)" % arg.entry.cname code.putln( - 'if (unlikely(!__Pyx_ArgTypeTest(%s, %s, %d, "%s", %s))) %s' % ( + 'if (unlikely(!__Pyx_ArgTypeTest(%s, %s, %d, %s, %s))) %s' % ( arg_code, typeptr_cname, arg.accept_none, - arg.name, + arg.name_cstring, arg.type.is_builtin_type and arg.type.require_exact, code.error_goto(arg.pos))) else: @@ -2212,8 +2337,8 @@ class FuncDefNode(StatNode, BlockNode): cname = arg.entry.cname code.putln('if (unlikely(((PyObject *)%s) == Py_None)) {' % cname) - code.putln('''PyErr_Format(PyExc_TypeError, "Argument '%%.%ds' must not be None", "%s"); %s''' % ( - max(200, len(arg.name)), arg.name, + code.putln('''PyErr_Format(PyExc_TypeError, "Argument '%%.%ds' must not be None", %s); %s''' % ( + max(200, len(arg.name_cstring)), arg.name_cstring, code.error_goto(arg.pos))) code.putln('}') @@ -2249,7 +2374,7 @@ class FuncDefNode(StatNode, BlockNode): def getbuffer_check(self, code): py_buffer, _ = self._get_py_buffer_info() view = py_buffer.cname - code.putln("if (%s == NULL) {" % view) + code.putln("if (unlikely(%s == NULL)) {" % view) code.putln("PyErr_SetString(PyExc_BufferError, " "\"PyObject_GetBuffer: view==NULL argument is obsolete\");") code.putln("return -1;") @@ -2260,7 +2385,7 @@ class FuncDefNode(StatNode, BlockNode): view = py_buffer.cname if obj_type and obj_type.is_pyobject: code.put_init_to_py_none("%s->obj" % view, obj_type) - code.put_giveref("%s->obj" % view) # Do not refnanny object within structs + code.put_giveref("%s->obj" % view, obj_type) # Do not refnanny object within structs else: code.putln("%s->obj = NULL;" % view) @@ -2269,7 +2394,7 @@ class FuncDefNode(StatNode, BlockNode): view = py_buffer.cname if obj_type and obj_type.is_pyobject: code.putln("if (%s->obj != NULL) {" % view) - code.put_gotref("%s->obj" % view) + code.put_gotref("%s->obj" % view, obj_type) code.put_decref_clear("%s->obj" % view, obj_type) code.putln("}") else: @@ -2280,7 +2405,7 @@ class FuncDefNode(StatNode, BlockNode): view = py_buffer.cname if obj_type and obj_type.is_pyobject: code.putln("if (%s->obj == Py_None) {" % view) - code.put_gotref("%s->obj" % view) + code.put_gotref("%s->obj" % view, obj_type) code.put_decref_clear("%s->obj" % view, obj_type) code.putln("}") @@ -2322,7 +2447,8 @@ class CFuncDefNode(FuncDefNode): # is_static_method whether this is a static method # is_c_class_method whether this is a cclass method - child_attrs = ["base_type", "declarator", "body", "py_func_stat"] + child_attrs = ["base_type", "declarator", "body", "decorators", "py_func_stat"] + outer_attrs = ["decorators", "py_func_stat"] inline_in_pxd = False decorators = None @@ -2336,6 +2462,9 @@ class CFuncDefNode(FuncDefNode): def unqualified_name(self): return self.entry.name + def declared_name(self): + return self.declarator.declared_name() + @property def code_object(self): # share the CodeObject with the cpdef wrapper (if available) @@ -2356,20 +2485,20 @@ class CFuncDefNode(FuncDefNode): self.is_static_method = 'staticmethod' in env.directives and not env.lookup_here('staticmethod') # The 2 here is because we need both function and argument names. if isinstance(self.declarator, CFuncDeclaratorNode): - name_declarator, type = self.declarator.analyse( + name_declarator, typ = self.declarator.analyse( base_type, env, nonempty=2 * (self.body is not None), directive_locals=self.directive_locals, visibility=self.visibility) else: - name_declarator, type = self.declarator.analyse( + name_declarator, typ = self.declarator.analyse( base_type, env, nonempty=2 * (self.body is not None), visibility=self.visibility) - if not type.is_cfunction: + if not typ.is_cfunction: error(self.pos, "Suite attached to non-function declaration") # Remember the actual type according to the function header # written here, because the type in the symbol table entry # may be different if we're overriding a C method inherited # from the base type of an extension type. - self.type = type - type.is_overridable = self.overridable + self.type = typ + typ.is_overridable = self.overridable declarator = self.declarator while not hasattr(declarator, 'args'): declarator = declarator.base @@ -2382,11 +2511,18 @@ class CFuncDefNode(FuncDefNode): error(self.cfunc_declarator.pos, "Function with optional arguments may not be declared public or api") - if type.exception_check == '+' and self.visibility != 'extern': - warning(self.cfunc_declarator.pos, + if typ.exception_check == '+' and self.visibility != 'extern': + if typ.exception_value and typ.exception_value.is_name: + # it really is impossible to reason about what the user wants to happens + # if they've specified a C++ exception translation function. Therefore, + # raise an error. + error(self.cfunc_declarator.pos, "Only extern functions can throw C++ exceptions.") + else: + warning(self.cfunc_declarator.pos, + "Only extern functions can throw C++ exceptions.", 2) - for formal_arg, type_arg in zip(self.args, type.args): + for formal_arg, type_arg in zip(self.args, typ.args): self.align_argument_type(env, type_arg) formal_arg.type = type_arg.type formal_arg.name = type_arg.name @@ -2407,20 +2543,21 @@ class CFuncDefNode(FuncDefNode): elif 'inline' in self.modifiers: warning(formal_arg.pos, "Buffer unpacking not optimized away.", 1) - self._validate_type_visibility(type.return_type, self.pos, env) + self._validate_type_visibility(typ.return_type, self.pos, env) name = name_declarator.name cname = name_declarator.cname - type.is_const_method = self.is_const_method - type.is_static_method = self.is_static_method + typ.is_const_method = self.is_const_method + typ.is_static_method = self.is_static_method + self.entry = env.declare_cfunction( - name, type, self.pos, + name, typ, self.pos, cname=cname, visibility=self.visibility, api=self.api, defining=self.body is not None, modifiers=self.modifiers, overridable=self.overridable) self.entry.inline_func_in_pxd = self.inline_in_pxd - self.return_type = type.return_type + self.return_type = typ.return_type if self.return_type.is_array and self.visibility != 'extern': error(self.pos, "Function cannot return an array") if self.return_type.is_cpp_class: @@ -2435,38 +2572,45 @@ class CFuncDefNode(FuncDefNode): self.create_local_scope(env) def declare_cpdef_wrapper(self, env): - if self.overridable: - if self.is_static_method: - # TODO(robertwb): Finish this up, perhaps via more function refactoring. - error(self.pos, "static cpdef methods not yet supported") - name = self.entry.name - py_func_body = self.call_self_node(is_module_scope=env.is_module_scope) - if self.is_static_method: - from .ExprNodes import NameNode - decorators = [DecoratorNode(self.pos, decorator=NameNode(self.pos, name='staticmethod'))] - decorators[0].decorator.analyse_types(env) + if not self.overridable: + return + if self.is_static_method: + # TODO(robertwb): Finish this up, perhaps via more function refactoring. + error(self.pos, "static cpdef methods not yet supported") + + name = self.entry.name + py_func_body = self.call_self_node(is_module_scope=env.is_module_scope) + if self.is_static_method: + from .ExprNodes import NameNode + decorators = [DecoratorNode(self.pos, decorator=NameNode(self.pos, name=EncodedString('staticmethod')))] + decorators[0].decorator.analyse_types(env) + else: + decorators = [] + self.py_func = DefNode(pos=self.pos, + name=self.entry.name, + args=self.args, + star_arg=None, + starstar_arg=None, + doc=self.doc, + body=py_func_body, + decorators=decorators, + is_wrapper=1) + self.py_func.is_module_scope = env.is_module_scope + self.py_func.analyse_declarations(env) + self.py_func.entry.is_overridable = True + self.py_func_stat = StatListNode(self.pos, stats=[self.py_func]) + self.py_func.type = PyrexTypes.py_object_type + self.entry.as_variable = self.py_func.entry + self.entry.used = self.entry.as_variable.used = True + # Reset scope entry the above cfunction + env.entries[name] = self.entry + if (not self.entry.is_final_cmethod and + (not env.is_module_scope or Options.lookup_module_cpdef)): + if self.override: + # This is a hack: we shouldn't create the wrapper twice, but we do for fused functions. + assert self.entry.is_fused_specialized # should not happen for non-fused cpdef functions + self.override.py_func = self.py_func else: - decorators = [] - self.py_func = DefNode(pos=self.pos, - name=self.entry.name, - args=self.args, - star_arg=None, - starstar_arg=None, - doc=self.doc, - body=py_func_body, - decorators=decorators, - is_wrapper=1) - self.py_func.is_module_scope = env.is_module_scope - self.py_func.analyse_declarations(env) - self.py_func.entry.is_overridable = True - self.py_func_stat = StatListNode(self.pos, stats=[self.py_func]) - self.py_func.type = PyrexTypes.py_object_type - self.entry.as_variable = self.py_func.entry - self.entry.used = self.entry.as_variable.used = True - # Reset scope entry the above cfunction - env.entries[name] = self.entry - if (not self.entry.is_final_cmethod and - (not env.is_module_scope or Options.lookup_module_cpdef)): self.override = OverrideCheckNode(self.pos, py_func=self.py_func) self.body = StatListNode(self.pos, stats=[self.override, self.body]) @@ -2585,7 +2729,7 @@ class CFuncDefNode(FuncDefNode): header = self.return_type.declaration_code(entity, dll_linkage=dll_linkage) #print (storage_class, modifiers, header) - needs_proto = self.is_c_class_method + needs_proto = self.is_c_class_method or self.entry.is_cproperty if self.template_declaration: if needs_proto: code.globalstate.parts['module_declarations'].putln(self.template_declaration) @@ -2667,7 +2811,6 @@ class CFuncDefNode(FuncDefNode): if self.return_type.is_pyobject: return "0" else: - #return None return self.entry.type.exception_value def caller_will_check_exceptions(self): @@ -2766,7 +2909,7 @@ class DefNode(FuncDefNode): self_in_stararg = 0 py_cfunc_node = None requires_classobj = False - defaults_struct = None # Dynamic kwrds structure name + defaults_struct = None # Dynamic kwrds structure name doc = None fused_py_func = False @@ -2779,14 +2922,17 @@ class DefNode(FuncDefNode): def __init__(self, pos, **kwds): FuncDefNode.__init__(self, pos, **kwds) - k = rk = r = 0 + p = k = rk = r = 0 for arg in self.args: + if arg.pos_only: + p += 1 if arg.kw_only: k += 1 if not arg.default: rk += 1 if not arg.default: r += 1 + self.num_posonly_args = p self.num_kwonly_args = k self.num_required_kw_args = rk self.num_required_args = r @@ -2884,8 +3030,18 @@ class DefNode(FuncDefNode): # staticmethod() was overridden - not much we can do here ... self.is_staticmethod = False - if self.name == '__new__' and env.is_py_class_scope: - self.is_staticmethod = 1 + if env.is_py_class_scope or env.is_c_class_scope: + if self.name == '__new__' and env.is_py_class_scope: + self.is_staticmethod = True + elif self.name == '__init_subclass__' and env.is_c_class_scope: + error(self.pos, "'__init_subclass__' is not supported by extension class") + elif self.name in IMPLICIT_CLASSMETHODS and not self.is_classmethod: + self.is_classmethod = True + # TODO: remove the need to generate a real decorator here, is_classmethod=True should suffice. + from .ExprNodes import NameNode + self.decorators = self.decorators or [] + self.decorators.insert(0, DecoratorNode( + self.pos, decorator=NameNode(self.pos, name=EncodedString('classmethod')))) self.analyse_argument_types(env) if self.name == '<lambda>': @@ -2898,7 +3054,7 @@ class DefNode(FuncDefNode): # if a signature annotation provides a more specific return object type, use it if self.return_type is py_object_type and self.return_type_annotation: if env.directives['annotation_typing'] and not self.entry.is_special: - _, return_type = analyse_type_annotation(self.return_type_annotation, env) + _, return_type = self.return_type_annotation.analyse_type_annotation(env) if return_type and return_type.is_pyobject: self.return_type = return_type @@ -2938,9 +3094,6 @@ class DefNode(FuncDefNode): arg.name = name_declarator.name arg.type = type - if type.is_fused: - self.has_fused_arguments = True - self.align_argument_type(env, arg) if name_declarator and name_declarator.cname: error(self.pos, "Python function argument cannot have C name specification") @@ -2966,11 +3119,14 @@ class DefNode(FuncDefNode): # probably just a plain 'object' arg.accept_none = True else: - arg.accept_none = True # won't be used, but must be there + arg.accept_none = True # won't be used, but must be there if arg.not_none: error(arg.pos, "Only Python type arguments can have 'not None'") if arg.or_none: error(arg.pos, "Only Python type arguments can have 'or None'") + + if arg.type.is_fused: + self.has_fused_arguments = True env.fused_to_specific = f2s if has_np_pythran(env): @@ -2983,8 +3139,10 @@ class DefNode(FuncDefNode): if self.decorators: error(self.pos, "special functions of cdef classes cannot have decorators") self.entry.trivial_signature = len(self.args) == 1 and not (self.star_arg or self.starstar_arg) - elif not env.directives['always_allow_keywords'] and not (self.star_arg or self.starstar_arg): - # Use the simpler calling signature for zero- and one-argument functions. + elif not (self.star_arg or self.starstar_arg) and ( + not env.directives['always_allow_keywords'] + or all([arg.pos_only for arg in self.args])): + # Use the simpler calling signature for zero- and one-argument pos-only functions. if self.entry.signature is TypeSlots.pyfunction_signature: if len(self.args) == 0: self.entry.signature = TypeSlots.pyfunction_noargs @@ -3005,7 +3163,7 @@ class DefNode(FuncDefNode): # this is the only case where a diverging number of # arguments is not an error - when we have no explicit # 'self' parameter as in method(*args) - sig = self.entry.signature = TypeSlots.pyfunction_signature # self is not 'really' used + sig = self.entry.signature = TypeSlots.pyfunction_signature # self is not 'really' used self.self_in_stararg = 1 nfixed = 0 @@ -3040,10 +3198,6 @@ class DefNode(FuncDefNode): arg.needs_type_test = 1 else: arg.needs_conversion = 1 - if arg.needs_conversion: - arg.hdr_cname = Naming.arg_prefix + arg.name - else: - arg.hdr_cname = Naming.var_prefix + arg.name if nfixed > len(self.args): self.bad_signature() @@ -3055,6 +3209,31 @@ class DefNode(FuncDefNode): if arg.is_generic and (arg.type.is_extension_type or arg.type.is_builtin_type): arg.needs_type_test = 1 + # Decide whether to use METH_FASTCALL + # 1. If we use METH_NOARGS or METH_O, keep that. We can only change + # METH_VARARGS to METH_FASTCALL + # 2. Special methods like __call__ always use the METH_VARGARGS + # calling convention + mf = sig.method_flags() + if mf and TypeSlots.method_varargs in mf and not self.entry.is_special: + # 3. If the function uses the full args tuple, it's more + # efficient to use METH_VARARGS. This happens when the function + # takes *args but no other positional arguments (apart from + # possibly self). We don't do the analogous check for keyword + # arguments since the kwargs dict is copied anyway. + if self.star_arg: + uses_args_tuple = True + for arg in self.args: + if (arg.is_generic and not arg.kw_only and + not arg.is_self_arg and not arg.is_type_arg): + # Other positional argument + uses_args_tuple = False + else: + uses_args_tuple = False + + if not uses_args_tuple: + sig = self.entry.signature = sig.with_fastcall() + def bad_signature(self): sig = self.entry.signature expected_str = "%d" % sig.num_fixed_args() @@ -3080,16 +3259,16 @@ class DefNode(FuncDefNode): entry = env.declare_pyfunction(name, self.pos, allow_redefine=not self.is_wrapper) self.entry = entry prefix = env.next_id(env.scope_prefix) - self.entry.pyfunc_cname = Naming.pyfunc_prefix + prefix + name + self.entry.pyfunc_cname = punycodify_name(Naming.pyfunc_prefix + prefix + name) if Options.docstrings: entry.doc = embed_position(self.pos, self.doc) - entry.doc_cname = Naming.funcdoc_prefix + prefix + name + entry.doc_cname = punycodify_name(Naming.funcdoc_prefix + prefix + name) if entry.is_special: if entry.name in TypeSlots.invisible or not entry.doc or ( entry.name in '__getattr__' and env.directives['fast_getattr']): entry.wrapperbase_cname = None else: - entry.wrapperbase_cname = Naming.wrapperbase_prefix + prefix + name + entry.wrapperbase_cname = punycodify_name(Naming.wrapperbase_prefix + prefix + name) else: entry.doc = None @@ -3132,8 +3311,6 @@ class DefNode(FuncDefNode): self.local_scope.directives = env.directives self.analyse_default_values(env) self.analyse_annotations(env) - if self.return_type_annotation: - self.return_type_annotation = self.analyse_annotation(env, self.return_type_annotation) if not self.needs_assignment_synthesis(env) and self.decorators: for decorator in self.decorators[::-1]: @@ -3257,10 +3434,11 @@ class DefNodeWrapper(FuncDefNode): # DefNode python wrapper code generator defnode = None - target = None # Target DefNode + target = None # Target DefNode def __init__(self, *args, **kwargs): FuncDefNode.__init__(self, *args, **kwargs) + self.num_posonly_args = self.target.num_posonly_args self.num_kwonly_args = self.target.num_kwonly_args self.num_required_kw_args = self.target.num_required_kw_args self.num_required_args = self.target.num_required_args @@ -3271,8 +3449,8 @@ class DefNodeWrapper(FuncDefNode): target_entry = self.target.entry name = self.name prefix = env.next_id(env.scope_prefix) - target_entry.func_cname = Naming.pywrap_prefix + prefix + name - target_entry.pymethdef_cname = Naming.pymethdef_prefix + prefix + name + target_entry.func_cname = punycodify_name(Naming.pywrap_prefix + prefix + name) + target_entry.pymethdef_cname = punycodify_name(Naming.pymethdef_prefix + prefix + name) self.signature = target_entry.signature @@ -3286,10 +3464,10 @@ class DefNodeWrapper(FuncDefNode): for arg in self.args: if not arg.type.is_pyobject: if not arg.type.create_from_py_utility_code(env): - pass # will fail later + pass # will fail later elif arg.hdr_type and not arg.hdr_type.is_pyobject: if not arg.hdr_type.create_to_py_utility_code(env): - pass # will fail later + pass # will fail later if self.starstar_arg and not self.starstar_arg.entry.cf_used: # we will set the kwargs argument to NULL instead of a new dict @@ -3316,7 +3494,13 @@ class DefNodeWrapper(FuncDefNode): if self.signature.has_dummy_arg: args.append(Naming.self_cname) for arg in self.args: - if arg.hdr_type and not (arg.type.is_memoryviewslice or + if arg.type.is_cpp_class: + # it's safe to move converted C++ types because they aren't + # used again afterwards + code.globalstate.use_utility_code( + UtilityCode.load_cached("MoveIfSupported", "CppSupport.cpp")) + args.append("__PYX_STD_MOVE_IF_SUPPORTED(%s)" % arg.entry.cname) + elif arg.hdr_type and not (arg.type.is_memoryviewslice or arg.type.is_struct or arg.type.is_complex): args.append(arg.type.cast_code(arg.entry.cname)) @@ -3360,7 +3544,7 @@ class DefNodeWrapper(FuncDefNode): self.return_type.declaration_code(Naming.retval_cname), retval_init)) code.put_declare_refcount_context() - code.put_setup_refcount_context('%s (wrapper)' % self.name) + code.put_setup_refcount_context(EncodedString('%s (wrapper)' % self.name)) self.generate_argument_parsing_code(lenv, code) self.generate_argument_type_tests(code) @@ -3387,7 +3571,10 @@ class DefNodeWrapper(FuncDefNode): code.put_label(code.return_label) for entry in lenv.var_entries: if entry.is_arg and entry.type.is_pyobject: - code.put_var_decref(entry) + if entry.xdecref_cleanup: + code.put_var_xdecref(entry) + else: + code.put_var_decref(entry) code.put_finish_refcount_context() if not self.return_type.is_void: @@ -3420,9 +3607,16 @@ class DefNodeWrapper(FuncDefNode): if entry.scope.is_c_class_scope and entry.name == "__ipow__": arg_code_list.append("CYTHON_UNUSED PyObject *unused") if sig.has_generic_args: - arg_code_list.append( - "PyObject *%s, PyObject *%s" % ( - Naming.args_cname, Naming.kwds_cname)) + varargs_args = "PyObject *%s, PyObject *%s" % ( + Naming.args_cname, Naming.kwds_cname) + if sig.use_fastcall: + fastcall_args = "PyObject *const *%s, Py_ssize_t %s, PyObject *%s" % ( + Naming.args_cname, Naming.nargs_cname, Naming.kwds_cname) + arg_code_list.append( + "\n#if CYTHON_METH_FASTCALL\n%s\n#else\n%s\n#endif\n" % ( + fastcall_args, varargs_args)) + else: + arg_code_list.append(varargs_args) arg_code = ", ".join(arg_code_list) # Prevent warning: unused function '__pyx_pw_5numpy_7ndarray_1__getbuffer__' @@ -3456,7 +3650,7 @@ class DefNodeWrapper(FuncDefNode): docstr = docstr.as_utf8_string() if not (entry.is_special and entry.name in ('__getbuffer__', '__releasebuffer__')): - code.putln('static char %s[] = %s;' % ( + code.putln('PyDoc_STRVAR(%s, %s);' % ( entry.doc_cname, docstr.as_c_string_literal())) @@ -3483,6 +3677,23 @@ class DefNodeWrapper(FuncDefNode): if entry.is_arg: code.put_var_declaration(entry) + # Assign nargs variable as len(args), but avoid an "unused" warning in the few cases where we don't need it. + if self.signature_has_generic_args(): + nargs_code = "CYTHON_UNUSED const Py_ssize_t %s = PyTuple_GET_SIZE(%s);" % ( + Naming.nargs_cname, Naming.args_cname) + if self.signature.use_fastcall: + code.putln("#if !CYTHON_METH_FASTCALL") + code.putln(nargs_code) + code.putln("#endif") + else: + code.putln(nargs_code) + + # Array containing the values of keyword arguments when using METH_FASTCALL. + code.globalstate.use_utility_code( + UtilityCode.load_cached("fastcall", "FunctionArguments.c")) + code.putln('CYTHON_UNUSED PyObject *const *%s = __Pyx_KwValues_%s(%s, %s);' % ( + Naming.kwvalues_cname, self.signature.fastvar, Naming.args_cname, Naming.nargs_cname)) + def generate_argument_parsing_code(self, env, code): # Generate fast equivalent of PyArg_ParseTuple call for # generic arguments, if any, including args/kwargs @@ -3506,6 +3717,8 @@ class DefNodeWrapper(FuncDefNode): elif not self.signature_has_nongeneric_args(): # func(*args) or func(**kw) or func(*args, **kw) + # possibly with a "self" argument but no other non-star + # arguments self.generate_stararg_copy_code(code) else: @@ -3541,10 +3754,9 @@ class DefNodeWrapper(FuncDefNode): if not self.star_arg: code.globalstate.use_utility_code( UtilityCode.load_cached("RaiseArgTupleInvalid", "FunctionArguments.c")) - code.putln("if (unlikely(PyTuple_GET_SIZE(%s) > 0)) {" % - Naming.args_cname) - code.put('__Pyx_RaiseArgtupleInvalid("%s", 1, 0, 0, PyTuple_GET_SIZE(%s)); return %s;' % ( - self.name, Naming.args_cname, self.error_value())) + code.putln("if (unlikely(%s > 0)) {" % Naming.nargs_cname) + code.put('__Pyx_RaiseArgtupleInvalid(%s, 1, 0, 0, %s); return %s;' % ( + self.name.as_c_string_literal(), Naming.nargs_cname, self.error_value())) code.putln("}") if self.starstar_arg: @@ -3553,69 +3765,70 @@ class DefNodeWrapper(FuncDefNode): else: kwarg_check = "%s" % Naming.kwds_cname else: - kwarg_check = "unlikely(%s) && unlikely(PyDict_Size(%s) > 0)" % ( - Naming.kwds_cname, Naming.kwds_cname) + kwarg_check = "unlikely(%s) && __Pyx_NumKwargs_%s(%s)" % ( + Naming.kwds_cname, self.signature.fastvar, Naming.kwds_cname) code.globalstate.use_utility_code( UtilityCode.load_cached("KeywordStringCheck", "FunctionArguments.c")) code.putln( - "if (%s && unlikely(!__Pyx_CheckKeywordStrings(%s, \"%s\", %d))) return %s;" % ( - kwarg_check, Naming.kwds_cname, self.name, + "if (%s && unlikely(!__Pyx_CheckKeywordStrings(%s, %s, %d))) return %s;" % ( + kwarg_check, Naming.kwds_cname, self.name.as_c_string_literal(), bool(self.starstar_arg), self.error_value())) if self.starstar_arg and self.starstar_arg.entry.cf_used: - if all(ref.node.allow_null for ref in self.starstar_arg.entry.cf_references): - code.putln("if (%s) {" % kwarg_check) - code.putln("%s = PyDict_Copy(%s); if (unlikely(!%s)) return %s;" % ( - self.starstar_arg.entry.cname, - Naming.kwds_cname, - self.starstar_arg.entry.cname, - self.error_value())) - code.put_gotref(self.starstar_arg.entry.cname) - code.putln("} else {") + code.putln("if (%s) {" % kwarg_check) + code.putln("%s = __Pyx_KwargsAsDict_%s(%s, %s);" % ( + self.starstar_arg.entry.cname, + self.signature.fastvar, + Naming.kwds_cname, + Naming.kwvalues_cname)) + code.putln("if (unlikely(!%s)) return %s;" % ( + self.starstar_arg.entry.cname, self.error_value())) + code.put_gotref(self.starstar_arg.entry.cname, py_object_type) + code.putln("} else {") + allow_null = all(ref.node.allow_null for ref in self.starstar_arg.entry.cf_references) + if allow_null: code.putln("%s = NULL;" % (self.starstar_arg.entry.cname,)) - code.putln("}") - self.starstar_arg.entry.xdecref_cleanup = 1 else: - code.put("%s = (%s) ? PyDict_Copy(%s) : PyDict_New(); " % ( - self.starstar_arg.entry.cname, - Naming.kwds_cname, - Naming.kwds_cname)) + code.putln("%s = PyDict_New();" % (self.starstar_arg.entry.cname,)) code.putln("if (unlikely(!%s)) return %s;" % ( self.starstar_arg.entry.cname, self.error_value())) - self.starstar_arg.entry.xdecref_cleanup = 0 - code.put_gotref(self.starstar_arg.entry.cname) + code.put_var_gotref(self.starstar_arg.entry) + self.starstar_arg.entry.xdecref_cleanup = allow_null + code.putln("}") if self.self_in_stararg and not self.target.is_staticmethod: + assert not self.signature.use_fastcall # need to create a new tuple with 'self' inserted as first item - code.put("%s = PyTuple_New(PyTuple_GET_SIZE(%s)+1); if (unlikely(!%s)) " % ( + code.put("%s = PyTuple_New(%s + 1); if (unlikely(!%s)) " % ( self.star_arg.entry.cname, - Naming.args_cname, + Naming.nargs_cname, self.star_arg.entry.cname)) if self.starstar_arg and self.starstar_arg.entry.cf_used: code.putln("{") - code.put_xdecref_clear(self.starstar_arg.entry.cname, py_object_type) + code.put_var_xdecref_clear(self.starstar_arg.entry) code.putln("return %s;" % self.error_value()) code.putln("}") else: code.putln("return %s;" % self.error_value()) - code.put_gotref(self.star_arg.entry.cname) + code.put_var_gotref(self.star_arg.entry) code.put_incref(Naming.self_cname, py_object_type) - code.put_giveref(Naming.self_cname) + code.put_giveref(Naming.self_cname, py_object_type) code.putln("PyTuple_SET_ITEM(%s, 0, %s);" % ( self.star_arg.entry.cname, Naming.self_cname)) temp = code.funcstate.allocate_temp(PyrexTypes.c_py_ssize_t_type, manage_ref=False) - code.putln("for (%s=0; %s < PyTuple_GET_SIZE(%s); %s++) {" % ( - temp, temp, Naming.args_cname, temp)) + code.putln("for (%s=0; %s < %s; %s++) {" % ( + temp, temp, Naming.nargs_cname, temp)) code.putln("PyObject* item = PyTuple_GET_ITEM(%s, %s);" % ( Naming.args_cname, temp)) code.put_incref("item", py_object_type) - code.put_giveref("item") + code.put_giveref("item", py_object_type) code.putln("PyTuple_SET_ITEM(%s, %s+1, item);" % ( self.star_arg.entry.cname, temp)) code.putln("}") code.funcstate.release_temp(temp) self.star_arg.entry.xdecref_cleanup = 0 elif self.star_arg: + assert not self.signature.use_fastcall code.put_incref(Naming.args_cname, py_object_type) code.putln("%s = %s;" % ( self.star_arg.entry.cname, @@ -3623,11 +3836,17 @@ class DefNodeWrapper(FuncDefNode): self.star_arg.entry.xdecref_cleanup = 0 def generate_tuple_and_keyword_parsing_code(self, args, success_label, code): + code.globalstate.use_utility_code( + UtilityCode.load_cached("fastcall", "FunctionArguments.c")) + + self_name_csafe = self.name.as_c_string_literal() + argtuple_error_label = code.new_label("argtuple_error") positional_args = [] required_kw_only_args = [] optional_kw_only_args = [] + num_pos_only_args = 0 for arg in args: if arg.is_generic: if arg.default: @@ -3640,6 +3859,8 @@ class DefNodeWrapper(FuncDefNode): required_kw_only_args.append(arg) elif not arg.is_self_arg and not arg.is_type_arg: positional_args.append(arg) + if arg.pos_only: + num_pos_only_args += 1 # sort required kw-only args before optional ones to avoid special # cases in the unpacking code @@ -3658,10 +3879,18 @@ class DefNodeWrapper(FuncDefNode): code.putln('{') all_args = tuple(positional_args) + tuple(kw_only_args) - code.putln("static PyObject **%s[] = {%s,0};" % ( + non_posonly_args = [arg for arg in all_args if not arg.pos_only] + non_pos_args_id = ','.join( + ['&%s' % code.intern_identifier(arg.entry.name) for arg in non_posonly_args] + ['0']) + code.putln("#if CYTHON_USE_MODULE_STATE") + code.putln("PyObject **%s[] = {%s};" % ( + Naming.pykwdlist_cname, + non_pos_args_id)) + code.putln("#else") + code.putln("static PyObject **%s[] = {%s};" % ( Naming.pykwdlist_cname, - ','.join(['&%s' % code.intern_identifier(arg.name) - for arg in all_args]))) + non_pos_args_id)) + code.putln("#endif") # Before being converted and assigned to the target variables, # borrowed references to all unpacked argument values are @@ -3673,14 +3902,43 @@ class DefNodeWrapper(FuncDefNode): # was passed for them. self.generate_argument_values_setup_code(all_args, code) + # If all args are positional-only, we can raise an error + # straight away if we receive a non-empty kw-dict. + # This requires a PyDict_Size call. This call is wasteful + # for functions which do accept kw-args, so we do not generate + # the PyDict_Size call unless all args are positional-only. + accept_kwd_args = non_posonly_args or self.starstar_arg + if accept_kwd_args: + kw_unpacking_condition = Naming.kwds_cname + else: + kw_unpacking_condition = "%s && __Pyx_NumKwargs_%s(%s) > 0" % ( + Naming.kwds_cname, self.signature.fastvar, Naming.kwds_cname) + + if self.num_required_kw_args > 0: + kw_unpacking_condition = "likely(%s)" % kw_unpacking_condition + # --- optimised code when we receive keyword arguments - code.putln("if (%s(%s)) {" % ( - (self.num_required_kw_args > 0) and "likely" or "unlikely", - Naming.kwds_cname)) - self.generate_keyword_unpacking_code( - min_positional_args, max_positional_args, - has_fixed_positional_count, has_kw_only_args, - all_args, argtuple_error_label, code) + code.putln("if (%s) {" % kw_unpacking_condition) + + if accept_kwd_args: + self.generate_keyword_unpacking_code( + min_positional_args, max_positional_args, + has_fixed_positional_count, has_kw_only_args, all_args, argtuple_error_label, code) + else: + # Here we do not accept kw-args but we are passed a non-empty kw-dict. + # We call ParseOptionalKeywords which will raise an appropriate error if + # the kw-args dict passed is non-empty (which it will be, since kw_unpacking_condition is true) + code.globalstate.use_utility_code( + UtilityCode.load_cached("ParseKeywords", "FunctionArguments.c")) + code.putln('if (likely(__Pyx_ParseOptionalKeywords(%s, %s, %s, %s, %s, %s, %s) < 0)) %s' % ( + Naming.kwds_cname, + Naming.kwvalues_cname, + Naming.pykwdlist_cname, + self.starstar_arg.entry.cname if self.starstar_arg else 0, + 'values', + 0, + self_name_csafe, + code.error_goto(self.pos))) # --- optimised code when we do not receive any keyword arguments if (self.num_required_kw_args and min_positional_args > 0) or min_positional_args == max_positional_args: @@ -3690,20 +3948,20 @@ class DefNodeWrapper(FuncDefNode): compare = '!=' else: compare = '<' - code.putln('} else if (PyTuple_GET_SIZE(%s) %s %d) {' % ( - Naming.args_cname, compare, min_positional_args)) + code.putln('} else if (unlikely(%s %s %d)) {' % ( + Naming.nargs_cname, compare, min_positional_args)) code.put_goto(argtuple_error_label) if self.num_required_kw_args: # pure error case: keywords required but not passed if max_positional_args > min_positional_args and not self.star_arg: - code.putln('} else if (PyTuple_GET_SIZE(%s) > %d) {' % ( - Naming.args_cname, max_positional_args)) + code.putln('} else if (unlikely(%s > %d)) {' % ( + Naming.nargs_cname, max_positional_args)) code.put_goto(argtuple_error_label) code.putln('} else {') for i, arg in enumerate(kw_only_args): if not arg.default: - pystring_cname = code.intern_identifier(arg.name) + pystring_cname = code.intern_identifier(arg.entry.name) # required keyword-only argument missing code.globalstate.use_utility_code( UtilityCode.load_cached("RaiseKeywordRequired", "FunctionArguments.c")) @@ -3720,11 +3978,12 @@ class DefNodeWrapper(FuncDefNode): # parse the exact number of positional arguments from # the args tuple for i, arg in enumerate(positional_args): - code.putln("values[%d] = PyTuple_GET_ITEM(%s, %d);" % (i, Naming.args_cname, i)) + code.putln("values[%d] = __Pyx_Arg_%s(%s, %d);" % ( + i, self.signature.fastvar, Naming.args_cname, i)) else: # parse the positional arguments from the variable length # args tuple and reject illegal argument tuple sizes - code.putln('switch (PyTuple_GET_SIZE(%s)) {' % Naming.args_cname) + code.putln('switch (%s) {' % Naming.nargs_cname) if self.star_arg: code.putln('default:') reversed_args = list(enumerate(positional_args))[::-1] @@ -3733,7 +3992,8 @@ class DefNodeWrapper(FuncDefNode): if i != reversed_args[0][0]: code.putln('CYTHON_FALLTHROUGH;') code.put('case %2d: ' % (i+1)) - code.putln("values[%d] = PyTuple_GET_ITEM(%s, %d);" % (i, Naming.args_cname, i)) + code.putln("values[%d] = __Pyx_Arg_%s(%s, %d);" % ( + i, self.signature.fastvar, Naming.args_cname, i)) if min_positional_args == 0: code.putln('CYTHON_FALLTHROUGH;') code.put('case 0: ') @@ -3748,7 +4008,7 @@ class DefNodeWrapper(FuncDefNode): code.put_goto(argtuple_error_label) code.putln('}') - code.putln('}') # end of the conditional unpacking blocks + code.putln('}') # end of the conditional unpacking blocks # Convert arg values to their final type and assign them. # Also inject non-Python default arguments, which do cannot @@ -3756,17 +4016,17 @@ class DefNodeWrapper(FuncDefNode): for i, arg in enumerate(all_args): self.generate_arg_assignment(arg, "values[%d]" % i, code) - code.putln('}') # end of the whole argument unpacking block + code.putln('}') # end of the whole argument unpacking block if code.label_used(argtuple_error_label): code.put_goto(success_label) code.put_label(argtuple_error_label) code.globalstate.use_utility_code( UtilityCode.load_cached("RaiseArgTupleInvalid", "FunctionArguments.c")) - code.put('__Pyx_RaiseArgtupleInvalid("%s", %d, %d, %d, PyTuple_GET_SIZE(%s)); ' % ( - self.name, has_fixed_positional_count, + code.put('__Pyx_RaiseArgtupleInvalid(%s, %d, %d, %d, %s); ' % ( + self_name_csafe, has_fixed_positional_count, min_positional_args, max_positional_args, - Naming.args_cname)) + Naming.nargs_cname)) code.putln(code.error_goto(self.pos)) def generate_arg_assignment(self, arg, item, code): @@ -3789,8 +4049,7 @@ class DefNodeWrapper(FuncDefNode): arg.entry.cname, arg.calculate_default_value_code(code))) if arg.type.is_memoryviewslice: - code.put_incref_memoryviewslice(arg.entry.cname, - have_gil=True) + code.put_var_incref_memoryviewslice(arg.entry, have_gil=True) code.putln('}') else: error(arg.pos, "Cannot convert Python object argument to type '%s'" % arg.type) @@ -3802,26 +4061,30 @@ class DefNodeWrapper(FuncDefNode): self.starstar_arg.entry.cname, self.starstar_arg.entry.cname, self.error_value())) - code.put_gotref(self.starstar_arg.entry.cname) + code.put_var_gotref(self.starstar_arg.entry) if self.star_arg: self.star_arg.entry.xdecref_cleanup = 0 - code.putln('if (PyTuple_GET_SIZE(%s) > %d) {' % ( - Naming.args_cname, - max_positional_args)) - code.putln('%s = PyTuple_GetSlice(%s, %d, PyTuple_GET_SIZE(%s));' % ( - self.star_arg.entry.cname, Naming.args_cname, - max_positional_args, Naming.args_cname)) - code.putln("if (unlikely(!%s)) {" % self.star_arg.entry.cname) - if self.starstar_arg: - code.put_decref_clear(self.starstar_arg.entry.cname, py_object_type) - code.put_finish_refcount_context() - code.putln('return %s;' % self.error_value()) - code.putln('}') - code.put_gotref(self.star_arg.entry.cname) - code.putln('} else {') - code.put("%s = %s; " % (self.star_arg.entry.cname, Naming.empty_tuple)) - code.put_incref(Naming.empty_tuple, py_object_type) - code.putln('}') + if max_positional_args == 0: + # If there are no positional arguments, use the args tuple + # directly + assert not self.signature.use_fastcall + code.put_incref(Naming.args_cname, py_object_type) + code.putln("%s = %s;" % (self.star_arg.entry.cname, Naming.args_cname)) + else: + # It is possible that this is a slice of "negative" length, + # as in args[5:3]. That's not a problem, the function below + # handles that efficiently and returns the empty tuple. + code.putln('%s = __Pyx_ArgsSlice_%s(%s, %d, %s);' % ( + self.star_arg.entry.cname, self.signature.fastvar, + Naming.args_cname, max_positional_args, Naming.nargs_cname)) + code.putln("if (unlikely(!%s)) {" % + self.star_arg.entry.type.nullcheck_string(self.star_arg.entry.cname)) + if self.starstar_arg: + code.put_var_decref_clear(self.starstar_arg.entry) + code.put_finish_refcount_context() + code.putln('return %s;' % self.error_value()) + code.putln('}') + code.put_var_gotref(self.star_arg.entry) def generate_argument_values_setup_code(self, args, code): max_args = len(args) @@ -3843,22 +4106,45 @@ class DefNodeWrapper(FuncDefNode): code.putln('values[%d] = %s;' % (i, arg.type.as_pyobject(default_value))) def generate_keyword_unpacking_code(self, min_positional_args, max_positional_args, - has_fixed_positional_count, has_kw_only_args, - all_args, argtuple_error_label, code): + has_fixed_positional_count, + has_kw_only_args, all_args, argtuple_error_label, code): + # First we count how many arguments must be passed as positional + num_required_posonly_args = num_pos_only_args = 0 + for i, arg in enumerate(all_args): + if arg.pos_only: + num_pos_only_args += 1 + if not arg.default: + num_required_posonly_args += 1 + code.putln('Py_ssize_t kw_args;') - code.putln('const Py_ssize_t pos_args = PyTuple_GET_SIZE(%s);' % Naming.args_cname) # copy the values from the args tuple and check that it's not too long - code.putln('switch (pos_args) {') + code.putln('switch (%s) {' % Naming.nargs_cname) if self.star_arg: code.putln('default:') - for i in range(max_positional_args-1, -1, -1): + + for i in range(max_positional_args-1, num_required_posonly_args-1, -1): code.put('case %2d: ' % (i+1)) - code.putln("values[%d] = PyTuple_GET_ITEM(%s, %d);" % ( - i, Naming.args_cname, i)) + code.putln("values[%d] = __Pyx_Arg_%s(%s, %d);" % ( + i, self.signature.fastvar, Naming.args_cname, i)) code.putln('CYTHON_FALLTHROUGH;') - code.putln('case 0: break;') + if num_required_posonly_args > 0: + code.put('case %2d: ' % num_required_posonly_args) + for i in range(num_required_posonly_args-1, -1, -1): + code.putln("values[%d] = __Pyx_Arg_%s(%s, %d);" % ( + i, self.signature.fastvar, Naming.args_cname, i)) + code.putln('break;') + for i in range(num_required_posonly_args-2, -1, -1): + code.put('case %2d: ' % (i+1)) + code.putln('CYTHON_FALLTHROUGH;') + + code.put('case 0: ') + if num_required_posonly_args == 0: + code.putln('break;') + else: + # catch-all for not enough pos-only args passed + code.put_goto(argtuple_error_label) if not self.star_arg: - code.put('default: ') # more arguments than allowed + code.put('default: ') # more arguments than allowed code.put_goto(argtuple_error_label) code.putln('}') @@ -3871,7 +4157,10 @@ class DefNodeWrapper(FuncDefNode): # If we received kwargs, fill up the positional/required # arguments with values from the kw dict - code.putln('kw_args = PyDict_Size(%s);' % Naming.kwds_cname) + self_name_csafe = self.name.as_c_string_literal() + + code.putln('kw_args = __Pyx_NumKwargs_%s(%s);' % ( + self.signature.fastvar, Naming.kwds_cname)) if self.num_required_args or max_positional_args > 0: last_required_arg = -1 for i, arg in enumerate(all_args): @@ -3879,30 +4168,32 @@ class DefNodeWrapper(FuncDefNode): last_required_arg = i if last_required_arg < max_positional_args: last_required_arg = max_positional_args-1 - if max_positional_args > 0: - code.putln('switch (pos_args) {') - for i, arg in enumerate(all_args[:last_required_arg+1]): - if max_positional_args > 0 and i <= max_positional_args: - if i != 0: + if max_positional_args > num_pos_only_args: + code.putln('switch (%s) {' % Naming.nargs_cname) + for i, arg in enumerate(all_args[num_pos_only_args:last_required_arg+1], num_pos_only_args): + if max_positional_args > num_pos_only_args and i <= max_positional_args: + if i != num_pos_only_args: code.putln('CYTHON_FALLTHROUGH;') if self.star_arg and i == max_positional_args: code.putln('default:') else: code.putln('case %2d:' % i) - pystring_cname = code.intern_identifier(arg.name) + pystring_cname = code.intern_identifier(arg.entry.name) if arg.default: if arg.kw_only: # optional kw-only args are handled separately below continue code.putln('if (kw_args > 0) {') # don't overwrite default argument - code.putln('PyObject* value = __Pyx_PyDict_GetItemStr(%s, %s);' % ( - Naming.kwds_cname, pystring_cname)) + code.putln('PyObject* value = __Pyx_GetKwValue_%s(%s, %s, %s);' % ( + self.signature.fastvar, Naming.kwds_cname, Naming.kwvalues_cname, pystring_cname)) code.putln('if (value) { values[%d] = value; kw_args--; }' % i) + code.putln('else if (unlikely(PyErr_Occurred())) %s' % code.error_goto(self.pos)) code.putln('}') else: - code.putln('if (likely((values[%d] = __Pyx_PyDict_GetItemStr(%s, %s)) != 0)) kw_args--;' % ( - i, Naming.kwds_cname, pystring_cname)) + code.putln('if (likely((values[%d] = __Pyx_GetKwValue_%s(%s, %s, %s)) != 0)) kw_args--;' % ( + i, self.signature.fastvar, Naming.kwds_cname, Naming.kwvalues_cname, pystring_cname)) + code.putln('else if (unlikely(PyErr_Occurred())) %s' % code.error_goto(self.pos)) if i < min_positional_args: if i == 0: # special case: we know arg 0 is missing @@ -3915,8 +4206,8 @@ class DefNodeWrapper(FuncDefNode): code.putln('else {') code.globalstate.use_utility_code( UtilityCode.load_cached("RaiseArgTupleInvalid", "FunctionArguments.c")) - code.put('__Pyx_RaiseArgtupleInvalid("%s", %d, %d, %d, %d); ' % ( - self.name, has_fixed_positional_count, + code.put('__Pyx_RaiseArgtupleInvalid(%s, %d, %d, %d, %d); ' % ( + self_name_csafe, has_fixed_positional_count, min_positional_args, max_positional_args, i)) code.putln(code.error_goto(self.pos)) code.putln('}') @@ -3924,11 +4215,11 @@ class DefNodeWrapper(FuncDefNode): code.putln('else {') code.globalstate.use_utility_code( UtilityCode.load_cached("RaiseKeywordRequired", "FunctionArguments.c")) - code.put('__Pyx_RaiseKeywordRequired("%s", %s); ' % ( - self.name, pystring_cname)) + code.put('__Pyx_RaiseKeywordRequired(%s, %s); ' % ( + self_name_csafe, pystring_cname)) code.putln(code.error_goto(self.pos)) code.putln('}') - if max_positional_args > 0: + if max_positional_args > num_pos_only_args: code.putln('}') if has_kw_only_args: @@ -3944,34 +4235,69 @@ class DefNodeWrapper(FuncDefNode): # arguments, this will always do the right thing for unpacking # keyword arguments, so that we can concentrate on optimising # common cases above. + # + # ParseOptionalKeywords() needs to know how many of the arguments + # that could be passed as keywords have in fact been passed as + # positional args. + if num_pos_only_args > 0: + # There are positional-only arguments which we don't want to count, + # since they cannot be keyword arguments. Subtract the number of + # pos-only arguments from the number of positional arguments we got. + # If we get a negative number then none of the keyword arguments were + # passed as positional args. + code.putln('const Py_ssize_t kwd_pos_args = (unlikely(%s < %d)) ? 0 : %s - %d;' % ( + Naming.nargs_cname, num_pos_only_args, + Naming.nargs_cname, num_pos_only_args, + )) + elif max_positional_args > 0: + code.putln('const Py_ssize_t kwd_pos_args = %s;' % Naming.nargs_cname) + if max_positional_args == 0: pos_arg_count = "0" elif self.star_arg: - code.putln("const Py_ssize_t used_pos_args = (pos_args < %d) ? pos_args : %d;" % ( - max_positional_args, max_positional_args)) + # If there is a *arg, the number of used positional args could be larger than + # the number of possible keyword arguments. But ParseOptionalKeywords() uses the + # number of positional args as an index into the keyword argument name array, + # if this is larger than the number of kwd args we get a segfault. So round + # this down to max_positional_args - num_pos_only_args (= num possible kwd args). + code.putln("const Py_ssize_t used_pos_args = (kwd_pos_args < %d) ? kwd_pos_args : %d;" % ( + max_positional_args - num_pos_only_args, max_positional_args - num_pos_only_args)) pos_arg_count = "used_pos_args" else: - pos_arg_count = "pos_args" + pos_arg_count = "kwd_pos_args" + if num_pos_only_args < len(all_args): + values_array = 'values + %d' % num_pos_only_args + else: + values_array = 'values' code.globalstate.use_utility_code( UtilityCode.load_cached("ParseKeywords", "FunctionArguments.c")) - code.putln('if (unlikely(__Pyx_ParseOptionalKeywords(%s, %s, %s, values, %s, "%s") < 0)) %s' % ( + code.putln('if (unlikely(__Pyx_ParseOptionalKeywords(%s, %s, %s, %s, %s, %s, %s) < 0)) %s' % ( Naming.kwds_cname, + Naming.kwvalues_cname, Naming.pykwdlist_cname, self.starstar_arg and self.starstar_arg.entry.cname or '0', + values_array, pos_arg_count, - self.name, + self_name_csafe, code.error_goto(self.pos))) code.putln('}') def generate_optional_kwonly_args_unpacking_code(self, all_args, code): optional_args = [] first_optional_arg = -1 + num_posonly_args = 0 for i, arg in enumerate(all_args): + if arg.pos_only: + num_posonly_args += 1 if not arg.kw_only or not arg.default: continue if not optional_args: first_optional_arg = i optional_args.append(arg.name) + if num_posonly_args > 0: + posonly_correction = '-%d' % num_posonly_args + else: + posonly_correction = '' if optional_args: if len(optional_args) > 1: # if we receive more than the named kwargs, we either have **kwargs @@ -3987,9 +4313,14 @@ class DefNodeWrapper(FuncDefNode): else: code.putln('if (kw_args == 1) {') code.putln('const Py_ssize_t index = %d;' % first_optional_arg) - code.putln('PyObject* value = __Pyx_PyDict_GetItemStr(%s, *%s[index]);' % ( - Naming.kwds_cname, Naming.pykwdlist_cname)) + code.putln('PyObject* value = __Pyx_GetKwValue_%s(%s, %s, *%s[index%s]);' % ( + self.signature.fastvar, + Naming.kwds_cname, + Naming.kwvalues_cname, + Naming.pykwdlist_cname, + posonly_correction)) code.putln('if (value) { values[index] = value; kw_args--; }') + code.putln('else if (unlikely(PyErr_Occurred())) %s' % code.error_goto(self.pos)) if len(optional_args) > 1: code.putln('}') code.putln('}') @@ -4070,9 +4401,7 @@ class GeneratorDefNode(DefNode): # is_generator = True - is_coroutine = False is_iterable_coroutine = False - is_asyncgen = False gen_type_name = 'Generator' needs_closure = True @@ -4107,7 +4436,7 @@ class GeneratorDefNode(DefNode): code.putln('%s = __Pyx_CyFunction_GetClassObj(%s);' % ( classobj_cname, Naming.self_cname)) code.put_incref(classobj_cname, py_object_type) - code.put_giveref(classobj_cname) + code.put_giveref(classobj_cname, py_object_type) code.put_finish_refcount_context() code.putln('return (PyObject *) gen;') code.putln('}') @@ -4231,7 +4560,7 @@ class GeneratorBodyDefNode(DefNode): code.putln("%s = %s; %s" % ( Naming.retval_cname, comp_init, code.error_goto_if_null(Naming.retval_cname, self.pos))) - code.put_gotref(Naming.retval_cname) + code.put_gotref(Naming.retval_cname, py_object_type) # ----- Function body self.generate_function_body(env, code) @@ -4277,7 +4606,7 @@ class GeneratorBodyDefNode(DefNode): # ----- Non-error return cleanup code.put_label(code.return_label) if self.is_inlined: - code.put_xgiveref(Naming.retval_cname) + code.put_xgiveref(Naming.retval_cname, py_object_type) else: code.put_xdecref_clear(Naming.retval_cname, py_object_type) # For Py3.7, clearing is already done below. @@ -4354,7 +4683,10 @@ class OverrideCheckNode(StatNode): return self def generate_execution_code(self, code): - interned_attr_cname = code.intern_identifier(self.py_func.entry.name) + # For fused functions, look up the dispatch function, not the specialisation. + method_entry = self.py_func.fused_py_func.entry if self.py_func.fused_py_func else self.py_func.entry + interned_attr_cname = code.intern_identifier(method_entry.name) + # Check to see if we are an extension type if self.py_func.is_module_scope: self_arg = "((PyObject *)%s)" % Naming.module_cname @@ -4366,8 +4698,8 @@ class OverrideCheckNode(StatNode): if self.py_func.is_module_scope: code.putln("else {") else: - code.putln("else if (unlikely((Py_TYPE(%s)->tp_dictoffset != 0)" - " || (Py_TYPE(%s)->tp_flags & (Py_TPFLAGS_IS_ABSTRACT | Py_TPFLAGS_HEAPTYPE)))) {" % ( + code.putln("else if (unlikely((Py_TYPE(%s)->tp_dictoffset != 0) || " + "__Pyx_PyType_HasFeature(Py_TYPE(%s), (Py_TPFLAGS_IS_ABSTRACT | Py_TPFLAGS_HEAPTYPE)))) {" % ( self_arg, self_arg)) code.putln("#if CYTHON_USE_DICT_VERSIONS && CYTHON_USE_PYTYPE_LOOKUP && CYTHON_USE_TYPE_SLOTS") @@ -4393,12 +4725,16 @@ class OverrideCheckNode(StatNode): err = code.error_goto_if_null(func_node_temp, self.pos) code.putln("%s = __Pyx_PyObject_GetAttrStr(%s, %s); %s" % ( func_node_temp, self_arg, interned_attr_cname, err)) - code.put_gotref(func_node_temp) + code.put_gotref(func_node_temp, py_object_type) - is_builtin_function_or_method = "PyCFunction_Check(%s)" % func_node_temp is_overridden = "(PyCFunction_GET_FUNCTION(%s) != (PyCFunction)(void*)%s)" % ( - func_node_temp, self.py_func.entry.func_cname) - code.putln("if (!%s || %s) {" % (is_builtin_function_or_method, is_overridden)) + func_node_temp, method_entry.func_cname) + code.putln("#ifdef __Pyx_CyFunction_USED") + code.putln("if (!__Pyx_IsCyOrPyCFunction(%s)" % func_node_temp) + code.putln("#else") + code.putln("if (!PyCFunction_Check(%s)" % func_node_temp) + code.putln("#endif") + code.putln(" || %s) {" % is_overridden) self.body.generate_execution_code(code) code.putln("}") @@ -4440,25 +4776,31 @@ class PyClassDefNode(ClassDefNode): # A Python class definition. # # name EncodedString Name of the class - # doc string or None + # doc string or None The class docstring # body StatNode Attribute definition code # entry Symtab.Entry # scope PyClassScope # decorators [DecoratorNode] list of decorators or None + # bases ExprNode Expression that evaluates to a tuple of base classes # # The following subnodes are constructed internally: # + # doc_node NameNode '__doc__' name that is made available to the class body # dict DictNode Class dictionary or Py3 namespace # classobj ClassNode Class object # target NameNode Variable to assign class object to + # orig_bases None or ExprNode "bases" before transformation by PEP560 __mro_entries__, + # used to create the __orig_bases__ attribute - child_attrs = ["body", "dict", "metaclass", "mkw", "bases", "class_result", - "target", "class_cell", "decorators"] + child_attrs = ["doc_node", "body", "dict", "metaclass", "mkw", "bases", "class_result", + "target", "class_cell", "decorators", "orig_bases"] decorators = None class_result = None is_py3_style_class = False # Python3 style class (kwargs) metaclass = None mkw = None + doc_node = None + orig_bases = None def __init__(self, pos, name, bases, doc, body, decorators=None, keyword_args=None, force_py3_semantics=False): @@ -4472,6 +4814,7 @@ class PyClassDefNode(ClassDefNode): if self.doc and Options.docstrings: doc = embed_position(self.pos, self.doc) doc_node = ExprNodes.StringNode(pos, value=doc) + self.doc_node = ExprNodes.NameNode(name=EncodedString('__doc__'), type=py_object_type, pos=pos) else: doc_node = None @@ -4520,7 +4863,9 @@ class PyClassDefNode(ClassDefNode): self.classobj = ExprNodes.Py3ClassNode( pos, name=name, class_def_node=self, doc=doc_node, calculate_metaclass=needs_metaclass_calculation, - allow_py2_metaclass=allow_py2_metaclass) + allow_py2_metaclass=allow_py2_metaclass, + force_type=force_py3_semantics, + ) else: # no bases, no metaclass => old style class creation self.dict = ExprNodes.DictNode(pos, key_value_pairs=[]) @@ -4557,7 +4902,7 @@ class PyClassDefNode(ClassDefNode): return cenv def analyse_declarations(self, env): - class_result = self.classobj + unwrapped_class_result = class_result = self.classobj if self.decorators: from .ExprNodes import SimpleCallNode for decorator in self.decorators[::-1]: @@ -4576,9 +4921,27 @@ class PyClassDefNode(ClassDefNode): cenv = self.create_scope(env) cenv.directives = env.directives cenv.class_obj_cname = self.target.entry.cname + if self.doc_node: + self.doc_node.analyse_target_declaration(cenv) self.body.analyse_declarations(cenv) + unwrapped_class_result.analyse_annotations(cenv) + + update_bases_functype = PyrexTypes.CFuncType( + PyrexTypes.py_object_type, [ + PyrexTypes.CFuncTypeArg("bases", PyrexTypes.py_object_type, None) + ]) def analyse_expressions(self, env): + if self.bases and not (self.bases.is_sequence_constructor and len(self.bases.args) == 0): + from .ExprNodes import PythonCapiCallNode, CloneNode + # handle the Python 3.7 __mro_entries__ transformation + orig_bases = self.bases.analyse_expressions(env) + self.bases = PythonCapiCallNode(orig_bases.pos, + function_name="__Pyx_PEP560_update_bases", + func_type=self.update_bases_functype, + utility_code=UtilityCode.load_cached('Py3UpdateBases', 'ObjectHandling.c'), + args=[CloneNode(orig_bases)]) + self.orig_bases = orig_bases if self.bases: self.bases = self.bases.analyse_expressions(env) if self.mkw: @@ -4589,7 +4952,7 @@ class PyClassDefNode(ClassDefNode): self.class_result = self.class_result.analyse_expressions(env) cenv = self.scope self.body = self.body.analyse_expressions(cenv) - self.target.analyse_target_expression(env, self.classobj) + self.target = self.target.analyse_target_expression(env, self.classobj) self.class_cell = self.class_cell.analyse_expressions(cenv) return self @@ -4601,6 +4964,8 @@ class PyClassDefNode(ClassDefNode): code.mark_pos(self.pos) code.pyclass_stack.append(self) cenv = self.scope + if self.orig_bases: + self.orig_bases.generate_evaluation_code(code) if self.bases: self.bases.generate_evaluation_code(code) if self.mkw: @@ -4608,6 +4973,17 @@ class PyClassDefNode(ClassDefNode): if self.metaclass: self.metaclass.generate_evaluation_code(code) self.dict.generate_evaluation_code(code) + if self.orig_bases: + # update __orig_bases__ if needed + code.putln("if (%s != %s) {" % (self.bases.result(), self.orig_bases.result())) + code.putln( + code.error_goto_if_neg('PyDict_SetItemString(%s, "__orig_bases__", %s)' % ( + self.dict.result(), self.orig_bases.result()), + self.pos + )) + code.putln("}") + self.orig_bases.generate_disposal_code(code) + self.orig_bases.free_temps(code) cenv.namespace_cname = cenv.class_obj_cname = self.dict.result() class_cell = self.class_cell @@ -4674,6 +5050,10 @@ class CClassDefNode(ClassDefNode): decorators = None shadow = False + @property + def punycode_class_name(self): + return punycodify_name(self.class_name) + def buffer_defaults(self, env): if not hasattr(self, '_buffer_defaults'): from . import Buffer @@ -4780,7 +5160,7 @@ class CClassDefNode(ClassDefNode): if self.visibility == 'extern': if (self.module_name == '__builtin__' and self.class_name in Builtin.builtin_types and - env.qualified_name[:8] != 'cpython.'): # allow overloaded names for cimporting from cpython + env.qualified_name[:8] != 'cpython.'): # allow overloaded names for cimporting from cpython warning(self.pos, "%s already a builtin Cython type" % self.class_name, 1) self.entry = home_scope.declare_c_class( @@ -4865,71 +5245,184 @@ class CClassDefNode(ClassDefNode): # This is needed to generate evaluation code for # default values of method arguments. code.mark_pos(self.pos) - if self.body: - self.body.generate_execution_code(code) if not self.entry.type.early_init: + bases = None if self.type_init_args: + # Extract bases tuple and validate 'best base' by actually calling 'type()'. + bases = code.funcstate.allocate_temp(PyrexTypes.py_object_type, manage_ref=True) + self.type_init_args.generate_evaluation_code(code) - bases = "PyTuple_GET_ITEM(%s, 1)" % self.type_init_args.result() + code.putln("%s = PyTuple_GET_ITEM(%s, 1);" % (bases, self.type_init_args.result())) + code.put_incref(bases, PyrexTypes.py_object_type) + first_base = "((PyTypeObject*)PyTuple_GET_ITEM(%s, 0))" % bases # Let Python do the base types compatibility checking. - trial_type = code.funcstate.allocate_temp(PyrexTypes.py_object_type, True) + trial_type = code.funcstate.allocate_temp(PyrexTypes.py_object_type, manage_ref=True) code.putln("%s = PyType_Type.tp_new(&PyType_Type, %s, NULL);" % ( trial_type, self.type_init_args.result())) code.putln(code.error_goto_if_null(trial_type, self.pos)) - code.put_gotref(trial_type) + code.put_gotref(trial_type, py_object_type) code.putln("if (((PyTypeObject*) %s)->tp_base != %s) {" % ( trial_type, first_base)) - code.putln("PyErr_Format(PyExc_TypeError, \"best base '%s' must be equal to first base '%s'\",") - code.putln(" ((PyTypeObject*) %s)->tp_base->tp_name, %s->tp_name);" % ( - trial_type, first_base)) + code.putln("__Pyx_TypeName base_name = __Pyx_PyType_GetName(((PyTypeObject*) %s)->tp_base);" % trial_type) + code.putln("__Pyx_TypeName type_name = __Pyx_PyType_GetName(%s);" % first_base) + code.putln("PyErr_Format(PyExc_TypeError, " + "\"best base '\" __Pyx_FMT_TYPENAME \"' must be equal to first base '\" __Pyx_FMT_TYPENAME \"'\",") + code.putln(" base_name, type_name);") + code.putln("__Pyx_DECREF_TypeName(base_name);") + code.putln("__Pyx_DECREF_TypeName(type_name);") code.putln(code.error_goto(self.pos)) code.putln("}") - code.funcstate.release_temp(trial_type) - code.put_incref(bases, PyrexTypes.py_object_type) - code.put_giveref(bases) - code.putln("%s.tp_bases = %s;" % (self.entry.type.typeobj_cname, bases)) + code.put_decref_clear(trial_type, PyrexTypes.py_object_type) + code.funcstate.release_temp(trial_type) + self.type_init_args.generate_disposal_code(code) self.type_init_args.free_temps(code) - self.generate_type_ready_code(self.entry, code, True) + self.generate_type_ready_code(self.entry, code, bases_tuple_cname=bases, check_heap_type_bases=True) + if bases is not None: + code.put_decref_clear(bases, PyrexTypes.py_object_type) + code.funcstate.release_temp(bases) + + if self.body: + self.body.generate_execution_code(code) # Also called from ModuleNode for early init types. @staticmethod - def generate_type_ready_code(entry, code, heap_type_bases=False): + def generate_type_ready_code(entry, code, bases_tuple_cname=None, check_heap_type_bases=False): # Generate a call to PyType_Ready for an extension # type defined in this module. type = entry.type - typeobj_cname = type.typeobj_cname + typeptr_cname = type.typeptr_cname scope = type.scope if not scope: # could be None if there was an error return - if entry.visibility != 'extern': + if entry.visibility == 'extern': + # Generate code to initialise the typeptr of an external extension + # type defined in this module to point to its type object. + if type.typeobj_cname: + # FIXME: this should not normally be set :-? + assert not type.typeobj_cname + code.putln("%s = &%s;" % ( + type.typeptr_cname, + type.typeobj_cname, + )) + return + # TODO: remove 'else:' and dedent + else: + assert typeptr_cname + assert type.typeobj_cname + typespec_cname = "%s_spec" % type.typeobj_cname + code.putln("#if CYTHON_USE_TYPE_SPECS") + tuple_temp = None + if not bases_tuple_cname and scope.parent_type.base_type: + tuple_temp = code.funcstate.allocate_temp(py_object_type, manage_ref=True) + code.putln("%s = PyTuple_Pack(1, (PyObject *)%s); %s" % ( + tuple_temp, + scope.parent_type.base_type.typeptr_cname, + code.error_goto_if_null(tuple_temp, entry.pos), + )) + code.put_gotref(tuple_temp, py_object_type) + + if bases_tuple_cname or tuple_temp: + if check_heap_type_bases: + code.globalstate.use_utility_code( + UtilityCode.load_cached('ValidateBasesTuple', 'ExtensionTypes.c')) + code.put_error_if_neg(entry.pos, "__Pyx_validate_bases_tuple(%s.name, %s, %s)" % ( + typespec_cname, + TypeSlots.get_slot_by_name("tp_dictoffset").slot_code(scope), + bases_tuple_cname or tuple_temp, + )) + + code.putln("%s = (PyTypeObject *) __Pyx_PyType_FromModuleAndSpec(%s, &%s, %s);" % ( + typeptr_cname, + Naming.module_cname, + typespec_cname, + bases_tuple_cname or tuple_temp, + )) + if tuple_temp: + code.put_xdecref_clear(tuple_temp, type=py_object_type) + code.funcstate.release_temp(tuple_temp) + code.putln(code.error_goto_if_null(typeptr_cname, entry.pos)) + else: + code.putln( + "%s = (PyTypeObject *) __Pyx_PyType_FromModuleAndSpec(%s, &%s, NULL); %s" % ( + typeptr_cname, + Naming.module_cname, + typespec_cname, + code.error_goto_if_null(typeptr_cname, entry.pos), + )) + + # The buffer interface is not currently supported by PyType_FromSpec(). + buffer_slot = TypeSlots.get_slot_by_name("tp_as_buffer") + if not buffer_slot.is_empty(scope): + code.putln("#if !CYTHON_COMPILING_IN_LIMITED_API") + code.putln("%s->%s = %s;" % ( + typeptr_cname, + buffer_slot.slot_name, + buffer_slot.slot_code(scope), + )) + # Still need to inherit buffer methods since PyType_Ready() didn't do it for us. + for buffer_method_name in ("__getbuffer__", "__releasebuffer__"): + buffer_slot = TypeSlots.get_slot_by_method_name(buffer_method_name) + if buffer_slot.slot_code(scope) == "0" and not TypeSlots.get_base_slot_function(scope, buffer_slot): + code.putln("if (!%s->tp_as_buffer->%s &&" + " %s->tp_base->tp_as_buffer &&" + " %s->tp_base->tp_as_buffer->%s) {" % ( + typeptr_cname, buffer_slot.slot_name, + typeptr_cname, + typeptr_cname, buffer_slot.slot_name, + )) + code.putln("%s->tp_as_buffer->%s = %s->tp_base->tp_as_buffer->%s;" % ( + typeptr_cname, buffer_slot.slot_name, + typeptr_cname, buffer_slot.slot_name, + )) + code.putln("}") + code.putln("#else") + code.putln("#warning The buffer protocol is not supported in the Limited C-API.") + code.putln("#endif") + + code.globalstate.use_utility_code( + UtilityCode.load_cached("FixUpExtensionType", "ExtensionTypes.c")) + code.put_error_if_neg(entry.pos, "__Pyx_fix_up_extension_type_from_spec(&%s, %s)" % ( + typespec_cname, typeptr_cname)) + + code.putln("#else") + if bases_tuple_cname: + code.put_incref(bases_tuple_cname, py_object_type) + code.put_giveref(bases_tuple_cname, py_object_type) + code.putln("%s.tp_bases = %s;" % (type.typeobj_cname, bases_tuple_cname)) + code.putln("%s = &%s;" % ( + typeptr_cname, + type.typeobj_cname, + )) + code.putln("#endif") # if CYTHON_USE_TYPE_SPECS + + code.putln("#if !CYTHON_COMPILING_IN_LIMITED_API") + # FIXME: these still need to get initialised even with the limited-API for slot in TypeSlots.slot_table: slot.generate_dynamic_init_code(scope, code) - if heap_type_bases: - code.globalstate.use_utility_code( - UtilityCode.load_cached('PyType_Ready', 'ExtensionTypes.c')) - readyfunc = "__Pyx_PyType_Ready" - else: - readyfunc = "PyType_Ready" - code.putln( - "if (%s(&%s) < 0) %s" % ( - readyfunc, - typeobj_cname, - code.error_goto(entry.pos))) - # Don't inherit tp_print from builtin types, restoring the + code.putln("#endif") + + code.putln("#if !CYTHON_USE_TYPE_SPECS") + code.globalstate.use_utility_code( + UtilityCode.load_cached('PyType_Ready', 'ExtensionTypes.c')) + code.put_error_if_neg(entry.pos, "__Pyx_PyType_Ready(%s)" % typeptr_cname) + code.putln("#endif") + + # Don't inherit tp_print from builtin types in Python 2, restoring the # behavior of using tp_repr or tp_str instead. # ("tp_print" was renamed to "tp_vectorcall_offset" in Py3.8b1) - code.putln("#if PY_VERSION_HEX < 0x030800B1") - code.putln("%s.tp_print = 0;" % typeobj_cname) + code.putln("#if PY_MAJOR_VERSION < 3") + code.putln("%s->tp_print = 0;" % typeptr_cname) code.putln("#endif") # Use specialised attribute lookup for types with generic lookup but no instance dict. getattr_slot_func = TypeSlots.get_slot_code_by_name(scope, 'tp_getattro') dictoffset_slot_func = TypeSlots.get_slot_code_by_name(scope, 'tp_dictoffset') if getattr_slot_func == '0' and dictoffset_slot_func == '0': + code.putln("#if !CYTHON_COMPILING_IN_LIMITED_API") # FIXME if type.is_final_type: py_cfunc = "__Pyx_PyObject_GenericGetAttrNoDict" # grepable utility_func = "PyObject_GenericGetAttrNoDict" @@ -4939,11 +5432,12 @@ class CClassDefNode(ClassDefNode): code.globalstate.use_utility_code(UtilityCode.load_cached(utility_func, "ObjectHandling.c")) code.putln("if ((CYTHON_USE_TYPE_SLOTS && CYTHON_USE_PYTYPE_LOOKUP) &&" - " likely(!%s.tp_dictoffset && %s.tp_getattro == PyObject_GenericGetAttr)) {" % ( - typeobj_cname, typeobj_cname)) - code.putln("%s.tp_getattro = %s;" % ( - typeobj_cname, py_cfunc)) + " likely(!%s->tp_dictoffset && %s->tp_getattro == PyObject_GenericGetAttr)) {" % ( + typeptr_cname, typeptr_cname)) + code.putln("%s->tp_getattro = %s;" % ( + typeptr_cname, py_cfunc)) code.putln("}") + code.putln("#endif") # if !CYTHON_COMPILING_IN_LIMITED_API # Fix special method docstrings. This is a bit of a hack, but # unless we let PyType_Ready create the slot wrappers we have @@ -4959,12 +5453,12 @@ class CClassDefNode(ClassDefNode): code.putln('#if CYTHON_COMPILING_IN_CPYTHON') code.putln("{") code.putln( - 'PyObject *wrapper = PyObject_GetAttrString((PyObject *)&%s, "%s"); %s' % ( - typeobj_cname, + 'PyObject *wrapper = PyObject_GetAttrString((PyObject *)%s, "%s"); %s' % ( + typeptr_cname, func.name, code.error_goto_if_null('wrapper', entry.pos))) code.putln( - "if (Py_TYPE(wrapper) == &PyWrapperDescr_Type) {") + "if (__Pyx_IS_TYPE(wrapper, &PyWrapperDescr_Type)) {") code.putln( "%s = *((PyWrapperDescrObject *)wrapper)->d_base;" % ( func.wrapperbase_cname)) @@ -4978,34 +5472,34 @@ class CClassDefNode(ClassDefNode): code.putln('#endif') if preprocessor_guard: code.putln('#endif') + if type.vtable_cname: code.globalstate.use_utility_code( UtilityCode.load_cached('SetVTable', 'ImportExport.c')) - code.putln( - "if (__Pyx_SetVtable(%s.tp_dict, %s) < 0) %s" % ( - typeobj_cname, - type.vtabptr_cname, - code.error_goto(entry.pos))) - if heap_type_bases: - code.globalstate.use_utility_code( - UtilityCode.load_cached('MergeVTables', 'ImportExport.c')) - code.putln("if (__Pyx_MergeVtables(&%s) < 0) %s" % ( - typeobj_cname, - code.error_goto(entry.pos))) + code.put_error_if_neg(entry.pos, "__Pyx_SetVtable(%s, %s)" % ( + typeptr_cname, + type.vtabptr_cname, + )) + # TODO: find a way to make this work with the Limited API! + code.putln("#if !CYTHON_COMPILING_IN_LIMITED_API") + code.globalstate.use_utility_code( + UtilityCode.load_cached('MergeVTables', 'ImportExport.c')) + code.put_error_if_neg(entry.pos, "__Pyx_MergeVtables(%s)" % typeptr_cname) + code.putln("#endif") if not type.scope.is_internal and not type.scope.directives.get('internal'): # scope.is_internal is set for types defined by # Cython (such as closures), the 'internal' # directive is set by users - code.putln( - 'if (PyObject_SetAttr(%s, %s, (PyObject *)&%s) < 0) %s' % ( - Naming.module_cname, - code.intern_identifier(scope.class_name), - typeobj_cname, - code.error_goto(entry.pos))) + code.put_error_if_neg(entry.pos, "PyObject_SetAttr(%s, %s, (PyObject *) %s)" % ( + Naming.module_cname, + code.intern_identifier(scope.class_name), + typeptr_cname, + )) + weakref_entry = scope.lookup_here("__weakref__") if not scope.is_closure_class_scope else None if weakref_entry: if weakref_entry.type is py_object_type: - tp_weaklistoffset = "%s.tp_weaklistoffset" % typeobj_cname + tp_weaklistoffset = "%s->tp_weaklistoffset" % typeptr_cname if type.typedef_flag: objstruct = type.objstruct_cname else: @@ -5017,21 +5511,16 @@ class CClassDefNode(ClassDefNode): weakref_entry.cname)) else: error(weakref_entry.pos, "__weakref__ slot must be of type 'object'") + if scope.lookup_here("__reduce_cython__") if not scope.is_closure_class_scope else None: # Unfortunately, we cannot reliably detect whether a # superclass defined __reduce__ at compile time, so we must # do so at runtime. code.globalstate.use_utility_code( UtilityCode.load_cached('SetupReduce', 'ExtensionTypes.c')) - code.putln('if (__Pyx_setup_reduce((PyObject*)&%s) < 0) %s' % ( - typeobj_cname, - code.error_goto(entry.pos))) - # Generate code to initialise the typeptr of an extension - # type defined in this module to point to its type object. - if type.typeobj_cname: - code.putln( - "%s = &%s;" % ( - type.typeptr_cname, type.typeobj_cname)) + code.putln("#if !CYTHON_COMPILING_IN_LIMITED_API") # FIXME + code.put_error_if_neg(entry.pos, "__Pyx_setup_reduce((PyObject *) %s)" % typeptr_cname) + code.putln("#endif") def annotate(self, code): if self.type_init_args: @@ -5045,14 +5534,13 @@ class PropertyNode(StatNode): # # name string # doc EncodedString or None Doc string - # entry Symtab.Entry + # entry Symtab.Entry The Entry of the property attribute # body StatListNode child_attrs = ["body"] def analyse_declarations(self, env): self.entry = env.declare_property(self.name, self.doc, self.pos) - self.entry.scope.directives = env.directives self.body.analyse_declarations(self.entry.scope) def analyse_expressions(self, env): @@ -5069,6 +5557,44 @@ class PropertyNode(StatNode): self.body.annotate(code) +class CPropertyNode(StatNode): + """Definition of a C property, backed by a CFuncDefNode getter. + """ + # name string + # doc EncodedString or None Doc string of the property + # entry Symtab.Entry The Entry of the property attribute + # body StatListNode[CFuncDefNode] (for compatibility with PropertyNode) + + child_attrs = ["body"] + is_cproperty = True + + @property + def cfunc(self): + stats = self.body.stats + assert stats and isinstance(stats[0], CFuncDefNode), stats + return stats[0] + + def analyse_declarations(self, env): + scope = PropertyScope(self.name, class_scope=env) + self.body.analyse_declarations(scope) + entry = self.entry = env.declare_property( + self.name, self.doc, self.pos, ctype=self.cfunc.return_type, property_scope=scope) + entry.getter_cname = self.cfunc.entry.cname + + def analyse_expressions(self, env): + self.body = self.body.analyse_expressions(env) + return self + + def generate_function_definitions(self, env, code): + self.body.generate_function_definitions(env, code) + + def generate_execution_code(self, code): + pass + + def annotate(self, code): + self.body.annotate(code) + + class GlobalNode(StatNode): # Global variable declaration. # @@ -5317,8 +5843,8 @@ class SingleAssignmentNode(AssignmentNode): elif self.lhs.type.is_array: if not isinstance(self.lhs, ExprNodes.SliceIndexNode): # cannot assign to C array, only to its full slice - self.lhs = ExprNodes.SliceIndexNode(self.lhs.pos, base=self.lhs, start=None, stop=None) - self.lhs = self.lhs.analyse_target_types(env) + lhs = ExprNodes.SliceIndexNode(self.lhs.pos, base=self.lhs, start=None, stop=None) + self.lhs = lhs.analyse_target_types(env) if self.lhs.type.is_cpp_class: op = env.lookup_operator_for_types(self.pos, '=', [self.lhs.type, self.rhs.type]) @@ -5818,7 +6344,7 @@ class ExecStatNode(StatNode): arg.free_temps(code) code.putln( code.error_goto_if_null(temp_result, self.pos)) - code.put_gotref(temp_result) + code.put_gotref(temp_result, py_object_type) code.put_decref_clear(temp_result, py_object_type) code.funcstate.release_temp(temp_result) @@ -6062,6 +6588,8 @@ class RaiseStatNode(StatNode): child_attrs = ["exc_type", "exc_value", "exc_tb", "cause"] is_terminator = True + builtin_exc_name = None + wrap_tuple_value = False def analyse_expressions(self, env): if self.exc_type: @@ -6069,6 +6597,12 @@ class RaiseStatNode(StatNode): self.exc_type = exc_type.coerce_to_pyobject(env) if self.exc_value: exc_value = self.exc_value.analyse_types(env) + if self.wrap_tuple_value: + if exc_value.type is Builtin.tuple_type or not exc_value.type.is_builtin_type: + # prevent tuple values from being interpreted as argument value tuples + from .ExprNodes import TupleNode + exc_value = TupleNode(exc_value.pos, args=[exc_value.coerce_to_pyobject(env)], slow=True) + exc_value = exc_value.analyse_types(env, skip_children=True) self.exc_value = exc_value.coerce_to_pyobject(env) if self.exc_tb: exc_tb = self.exc_tb.analyse_types(env) @@ -6077,7 +6611,6 @@ class RaiseStatNode(StatNode): cause = self.cause.analyse_types(env) self.cause = cause.coerce_to_pyobject(env) # special cases for builtin exceptions - self.builtin_exc_name = None if self.exc_type and not self.exc_value and not self.exc_tb: exc = self.exc_type from . import ExprNodes @@ -6087,7 +6620,7 @@ class RaiseStatNode(StatNode): if exc.is_name and exc.entry.is_builtin: self.builtin_exc_name = exc.name if self.builtin_exc_name == 'MemoryError': - self.exc_type = None # has a separate implementation + self.exc_type = None # has a separate implementation return self nogil_check = Node.gil_error @@ -6172,10 +6705,10 @@ class ReraiseStatNode(StatNode): vars = code.funcstate.exc_vars if vars: code.globalstate.use_utility_code(restore_exception_utility_code) - code.put_giveref(vars[0]) - code.put_giveref(vars[1]) + code.put_giveref(vars[0], py_object_type) + code.put_giveref(vars[1], py_object_type) # fresh exceptions may not have a traceback yet (-> finally!) - code.put_xgiveref(vars[2]) + code.put_xgiveref(vars[2], py_object_type) code.putln("__Pyx_ErrRestoreWithState(%s, %s, %s);" % tuple(vars)) for varname in vars: code.put("%s = 0; " % varname) @@ -6186,65 +6719,53 @@ class ReraiseStatNode(StatNode): UtilityCode.load_cached("ReRaiseException", "Exceptions.c")) code.putln("__Pyx_ReraiseException(); %s" % code.error_goto(self.pos)) + class AssertStatNode(StatNode): # assert statement # - # cond ExprNode - # value ExprNode or None + # condition ExprNode + # value ExprNode or None + # exception (Raise/GIL)StatNode created from 'value' in PostParse transform - child_attrs = ["cond", "value"] + child_attrs = ["condition", "value", "exception"] + exception = None + + def analyse_declarations(self, env): + assert self.value is None, "Message should have been replaced in PostParse()" + assert self.exception is not None, "Message should have been replaced in PostParse()" + self.exception.analyse_declarations(env) def analyse_expressions(self, env): - self.cond = self.cond.analyse_boolean_expression(env) - if self.value: - value = self.value.analyse_types(env) - if value.type is Builtin.tuple_type or not value.type.is_builtin_type: - # prevent tuple values from being interpreted as argument value tuples - from .ExprNodes import TupleNode - value = TupleNode(value.pos, args=[value], slow=True) - self.value = value.analyse_types(env, skip_children=True).coerce_to_pyobject(env) - else: - self.value = value.coerce_to_pyobject(env) + self.condition = self.condition.analyse_temp_boolean_expression(env) + self.exception = self.exception.analyse_expressions(env) return self - nogil_check = Node.gil_error - gil_message = "Raising exception" - def generate_execution_code(self, code): code.putln("#ifndef CYTHON_WITHOUT_ASSERTIONS") code.putln("if (unlikely(!Py_OptimizeFlag)) {") code.mark_pos(self.pos) - self.cond.generate_evaluation_code(code) - code.putln( - "if (unlikely(!%s)) {" % self.cond.result()) - if self.value: - self.value.generate_evaluation_code(code) - code.putln( - "PyErr_SetObject(PyExc_AssertionError, %s);" % self.value.py_result()) - self.value.generate_disposal_code(code) - self.value.free_temps(code) - else: - code.putln( - "PyErr_SetNone(PyExc_AssertionError);") + self.condition.generate_evaluation_code(code) code.putln( - code.error_goto(self.pos)) + "if (unlikely(!%s)) {" % self.condition.result()) + self.exception.generate_execution_code(code) code.putln( "}") - self.cond.generate_disposal_code(code) - self.cond.free_temps(code) + self.condition.generate_disposal_code(code) + self.condition.free_temps(code) code.putln( "}") + code.putln("#else") + # avoid unused labels etc. + code.putln("if ((1)); else %s" % code.error_goto(self.pos, used=False)) code.putln("#endif") def generate_function_definitions(self, env, code): - self.cond.generate_function_definitions(env, code) - if self.value is not None: - self.value.generate_function_definitions(env, code) + self.condition.generate_function_definitions(env, code) + self.exception.generate_function_definitions(env, code) def annotate(self, code): - self.cond.annotate(code) - if self.value: - self.value.annotate(code) + self.condition.annotate(code) + self.exception.annotate(code) class IfStatNode(StatNode): @@ -6271,13 +6792,9 @@ class IfStatNode(StatNode): code.mark_pos(self.pos) end_label = code.new_label() last = len(self.if_clauses) - if self.else_clause: - # If the 'else' clause is 'unlikely', then set the preceding 'if' clause to 'likely' to reflect that. - self._set_branch_hint(self.if_clauses[-1], self.else_clause, inverse=True) - else: + if not self.else_clause: last -= 1 # avoid redundant goto at end of last if-clause for i, if_clause in enumerate(self.if_clauses): - self._set_branch_hint(if_clause, if_clause.body) if_clause.generate_execution_code(code, end_label, is_last=i == last) if self.else_clause: code.mark_pos(self.else_clause.pos) @@ -6286,21 +6803,6 @@ class IfStatNode(StatNode): code.putln("}") code.put_label(end_label) - def _set_branch_hint(self, clause, statements_node, inverse=False): - if not statements_node.is_terminator: - return - if not isinstance(statements_node, StatListNode) or not statements_node.stats: - return - # Anything that unconditionally raises exceptions should be considered unlikely. - if isinstance(statements_node.stats[-1], (RaiseStatNode, ReraiseStatNode)): - if len(statements_node.stats) > 1: - # Allow simple statements before the 'raise', but no conditions, loops, etc. - non_branch_nodes = (ExprStatNode, AssignmentNode, DelStatNode, GlobalNode, NonlocalNode) - for node in statements_node.stats[:-1]: - if not isinstance(node, non_branch_nodes): - return - clause.branch_hint = 'likely' if inverse else 'unlikely' - def generate_function_definitions(self, env, code): for clause in self.if_clauses: clause.generate_function_definitions(env, code) @@ -6586,7 +7088,7 @@ class DictIterationNextNode(Node): # evaluate all coercions before the assignments for var, result, target in assignments: - code.put_gotref(var.result()) + var.generate_gotref(code) for var, result, target in assignments: result.generate_evaluation_code(code) for var, result, target in assignments: @@ -6648,7 +7150,7 @@ class SetIterationNextNode(Node): code.funcstate.release_temp(result_temp) # evaluate all coercions before the assignments - code.put_gotref(value_ref.result()) + value_ref.generate_gotref(code) self.coerced_value_var.generate_evaluation_code(code) self.value_target.generate_assignment_code(self.coerced_value_var, code) value_ref.release(code) @@ -6961,7 +7463,7 @@ class ForFromStatNode(LoopNode, StatNode): target_node.result(), interned_cname, code.error_goto_if_null(target_node.result(), self.target.pos))) - code.put_gotref(target_node.result()) + target_node.generate_gotref(code) else: target_node = self.target from_py_node = ExprNodes.CoerceFromPyTypeNode( @@ -7097,7 +7599,7 @@ class WithStatNode(StatNode): code.intern_identifier(EncodedString('__aexit__' if self.is_async else '__exit__')), code.error_goto_if_null(self.exit_var, self.pos), )) - code.put_gotref(self.exit_var) + code.put_gotref(self.exit_var, py_object_type) # need to free exit_var in the face of exceptions during setup old_error_label = code.new_error_label() @@ -7241,17 +7743,17 @@ class TryExceptStatNode(StatNode): save_exc.putln("__Pyx_ExceptionSave(%s);" % ( ', '.join(['&%s' % var for var in exc_save_vars]))) for var in exc_save_vars: - save_exc.put_xgotref(var) + save_exc.put_xgotref(var, py_object_type) def restore_saved_exception(): for name in exc_save_vars: - code.put_xgiveref(name) + code.put_xgiveref(name, py_object_type) code.putln("__Pyx_ExceptionReset(%s);" % ', '.join(exc_save_vars)) else: # try block cannot raise exceptions, but we had to allocate the temps above, # so just keep the C compiler from complaining about them being unused - mark_vars_used = ["(void)%s;" % var for var in exc_save_vars] + mark_vars_used = ["(void)%s;" % var for var in exc_save_vars] save_exc.putln("%s /* mark used */" % ' '.join(mark_vars_used)) def restore_saved_exception(): @@ -7394,17 +7896,40 @@ class ExceptClauseNode(Node): for _ in range(3)] code.globalstate.use_utility_code(UtilityCode.load_cached("PyErrFetchRestore", "Exceptions.c")) code.putln("__Pyx_ErrFetch(&%s, &%s, &%s);" % tuple(exc_vars)) - code.globalstate.use_utility_code(UtilityCode.load_cached("FastTypeChecks", "ModuleSetupCode.c")) - exc_test_func = "__Pyx_PyErr_GivenExceptionMatches(%s, %%s)" % exc_vars[0] + exc_type = exc_vars[0] else: - exc_vars = () - code.globalstate.use_utility_code(UtilityCode.load_cached("PyErrExceptionMatches", "Exceptions.c")) - exc_test_func = "__Pyx_PyErr_ExceptionMatches(%s)" + exc_vars = exc_type = None - exc_tests = [] for pattern in self.pattern: pattern.generate_evaluation_code(code) - exc_tests.append(exc_test_func % pattern.py_result()) + patterns = [pattern.py_result() for pattern in self.pattern] + + exc_tests = [] + if exc_type: + code.globalstate.use_utility_code( + UtilityCode.load_cached("FastTypeChecks", "ModuleSetupCode.c")) + if len(patterns) == 2: + exc_tests.append("__Pyx_PyErr_GivenExceptionMatches2(%s, %s, %s)" % ( + exc_type, patterns[0], patterns[1], + )) + else: + exc_tests.extend( + "__Pyx_PyErr_GivenExceptionMatches(%s, %s)" % (exc_type, pattern) + for pattern in patterns + ) + elif len(patterns) == 2: + code.globalstate.use_utility_code( + UtilityCode.load_cached("FastTypeChecks", "ModuleSetupCode.c")) + exc_tests.append("__Pyx_PyErr_ExceptionMatches2(%s, %s)" % ( + patterns[0], patterns[1], + )) + else: + code.globalstate.use_utility_code( + UtilityCode.load_cached("PyErrExceptionMatches", "Exceptions.c")) + exc_tests.extend( + "__Pyx_PyErr_ExceptionMatches(%s)" % pattern + for pattern in patterns + ) match_flag = code.funcstate.allocate_temp(PyrexTypes.c_int_type, manage_ref=False) code.putln("%s = %s;" % (match_flag, ' || '.join(exc_tests))) @@ -7412,7 +7937,7 @@ class ExceptClauseNode(Node): pattern.generate_disposal_code(code) pattern.free_temps(code) - if has_non_literals: + if exc_vars: code.putln("__Pyx_ErrRestore(%s, %s, %s);" % tuple(exc_vars)) code.putln(' '.join(["%s = 0;" % var for var in exc_vars])) for temp in exc_vars: @@ -7447,7 +7972,7 @@ class ExceptClauseNode(Node): code.putln("if (__Pyx_GetException(%s) < 0) %s" % ( exc_args, code.error_goto(self.pos))) for var in exc_vars: - code.put_gotref(var) + code.put_gotref(var, py_object_type) if self.target: self.exc_value.set_var(exc_vars[1]) self.exc_value.generate_evaluation_code(code) @@ -7727,7 +8252,7 @@ class TryFinallyStatNode(StatNode): " unlikely(__Pyx_GetException(&%s, &%s, &%s) < 0)) " "__Pyx_ErrFetch(&%s, &%s, &%s);" % (exc_vars[:3] * 2)) for var in exc_vars: - code.put_xgotref(var) + code.put_xgotref(var, py_object_type) if exc_lineno_cnames: code.putln("%s = %s; %s = %s; %s = %s;" % ( exc_lineno_cnames[0], Naming.lineno_cname, @@ -7748,11 +8273,11 @@ class TryFinallyStatNode(StatNode): # unused utility functions and/or temps code.putln("if (PY_MAJOR_VERSION >= 3) {") for var in exc_vars[3:]: - code.put_xgiveref(var) + code.put_xgiveref(var, py_object_type) code.putln("__Pyx_ExceptionReset(%s, %s, %s);" % exc_vars[3:]) code.putln("}") for var in exc_vars[:3]: - code.put_xgiveref(var) + code.put_xgiveref(var, py_object_type) code.putln("__Pyx_ErrRestore(%s, %s, %s);" % exc_vars[:3]) if self.is_try_finally_in_nogil: @@ -7774,7 +8299,7 @@ class TryFinallyStatNode(StatNode): # unused utility functions and/or temps code.putln("if (PY_MAJOR_VERSION >= 3) {") for var in exc_vars[3:]: - code.put_xgiveref(var) + code.put_xgiveref(var, py_object_type) code.putln("__Pyx_ExceptionReset(%s, %s, %s);" % exc_vars[3:]) code.putln("}") for var in exc_vars[:3]: @@ -7802,10 +8327,12 @@ class GILStatNode(NogilTryFinallyStatNode): # # state string 'gil' or 'nogil' + child_attrs = ["condition"] + NogilTryFinallyStatNode.child_attrs state_temp = None - def __init__(self, pos, state, body): + def __init__(self, pos, state, body, condition=None): self.state = state + self.condition = condition self.create_state_temp_if_needed(pos, state, body) TryFinallyStatNode.__init__( self, pos, @@ -7832,11 +8359,18 @@ class GILStatNode(NogilTryFinallyStatNode): if self.state == 'gil': env.has_with_gil_block = True + if self.condition is not None: + self.condition.analyse_declarations(env) + return super(GILStatNode, self).analyse_declarations(env) def analyse_expressions(self, env): env.use_utility_code( UtilityCode.load_cached("ForceInitThreads", "ModuleSetupCode.c")) + + if self.condition is not None: + self.condition = self.condition.analyse_expressions(env) + was_nogil = env.nogil env.nogil = self.state == 'nogil' node = TryFinallyStatNode.analyse_expressions(self, env) @@ -7923,6 +8457,33 @@ utility_code_for_imports = { 'inspect': ("__Pyx_patch_inspect", "PatchInspect", "Coroutine.c"), } +def cimport_numpy_check(node, code): + # shared code between CImportStatNode and FromCImportStatNode + # check to ensure that import_array is called + for mod in code.globalstate.module_node.scope.cimported_modules: + if mod.name != node.module_name: + continue + # there are sometimes several cimported modules with the same name + # so complete the loop if necessary + import_array = mod.lookup_here("import_array") + _import_array = mod.lookup_here("_import_array") + # at least one entry used + used = (import_array and import_array.used) or (_import_array and _import_array.used) + if ((import_array or _import_array) # at least one entry found + and not used): + # sanity check that this is actually numpy and not a user pxd called "numpy" + if _import_array and _import_array.type.is_cfunction: + # warning is mainly for the sake of testing + warning(node.pos, "'numpy.import_array()' has been added automatically " + "since 'numpy' was cimported but 'numpy.import_array' was not called.", 0) + from .Code import TempitaUtilityCode + code.globalstate.use_utility_code( + TempitaUtilityCode.load_cached("NumpyImportArray", "NumpyImportArray.c", + context = {'err_goto': code.error_goto(node.pos)}) + ) + return # no need to continue once the utility code is added + + class CImportStatNode(StatNode): # cimport statement @@ -7964,7 +8525,8 @@ class CImportStatNode(StatNode): return self def generate_execution_code(self, code): - pass + if self.module_name == "numpy": + cimport_numpy_check(self, code) class FromCImportStatNode(StatNode): @@ -8017,7 +8579,7 @@ class FromCImportStatNode(StatNode): local_name = as_name or name env.add_imported_entry(local_name, entry, pos) - if module_name.startswith('cpython') or module_name.startswith('cython'): # enough for now + if module_name.startswith('cpython') or module_name.startswith('cython'): # enough for now if module_name in utility_code_for_cimports: env.use_utility_code(utility_code_for_cimports[module_name]()) for _, name, _, _ in self.imported_names: @@ -8043,7 +8605,8 @@ class FromCImportStatNode(StatNode): return self def generate_execution_code(self, code): - pass + if self.module_name == "numpy": + cimport_numpy_check(self, code) class FromImportStatNode(StatNode): @@ -8125,7 +8688,7 @@ class FromImportStatNode(StatNode): self.module.py_result(), code.intern_identifier(name), code.error_goto_if_null(item_temp, self.pos))) - code.put_gotref(item_temp) + code.put_gotref(item_temp, py_object_type) if coerced_item is None: target.generate_assignment_code(self.item, code) else: @@ -8241,7 +8804,7 @@ class ParallelStatNode(StatNode, ParallelNode): seen.add(dictitem.key.value) if dictitem.key.value == 'num_threads': if not dictitem.value.is_none: - self.num_threads = dictitem.value + self.num_threads = dictitem.value elif self.is_prange and dictitem.key.value == 'chunksize': if not dictitem.value.is_none: self.chunksize = dictitem.value @@ -8502,11 +9065,7 @@ class ParallelStatNode(StatNode, ParallelNode): if self.is_parallel and not self.is_nested_prange: code.putln("/* Clean up any temporaries */") for temp, type in sorted(self.temps): - if type.is_memoryviewslice: - code.put_xdecref_memoryviewslice(temp, have_gil=False) - elif type.is_pyobject: - code.put_xdecref(temp, type) - code.putln("%s = NULL;" % temp) + code.put_xdecref_clear(temp, type, have_gil=False) def setup_parallel_control_flow_block(self, code): """ @@ -8539,7 +9098,7 @@ class ParallelStatNode(StatNode, ParallelNode): self.old_return_label = code.return_label code.return_label = code.new_label(name="return") - code.begin_block() # parallel control flow block + code.begin_block() # parallel control flow block self.begin_of_parallel_control_block_point = code.insertion_point() self.begin_of_parallel_control_block_point_after_decls = code.insertion_point() @@ -8669,7 +9228,7 @@ class ParallelStatNode(StatNode, ParallelNode): code.putln_openmp("#pragma omp critical(%s)" % section_name) ParallelStatNode.critical_section_counter += 1 - code.begin_block() # begin critical section + code.begin_block() # begin critical section c = self.begin_of_parallel_control_block_point @@ -8697,7 +9256,7 @@ class ParallelStatNode(StatNode, ParallelNode): self.parallel_private_temps.append((temp_cname, private_cname)) - code.end_block() # end critical section + code.end_block() # end critical section def fetch_parallel_exception(self, code): """ @@ -8737,7 +9296,7 @@ class ParallelStatNode(StatNode, ParallelNode): pos_info = chain(*zip(self.parallel_pos_info, self.pos_info)) code.funcstate.uses_error_indicator = True code.putln("%s = %s; %s = %s; %s = %s;" % tuple(pos_info)) - code.put_gotref(Naming.parallel_exc_type) + code.put_gotref(Naming.parallel_exc_type, py_object_type) code.putln( "}") @@ -8750,7 +9309,7 @@ class ParallelStatNode(StatNode, ParallelNode): code.begin_block() code.put_ensure_gil(declare_gilstate=True) - code.put_giveref(Naming.parallel_exc_type) + code.put_giveref(Naming.parallel_exc_type, py_object_type) code.putln("__Pyx_ErrRestoreWithState(%s, %s, %s);" % self.parallel_exc) pos_info = chain(*zip(self.pos_info, self.parallel_pos_info)) code.putln("%s = %s; %s = %s; %s = %s;" % tuple(pos_info)) @@ -8838,11 +9397,11 @@ class ParallelStatNode(StatNode, ParallelNode): self.restore_parallel_exception(code) code.put_goto(code.error_label) - code.putln("}") # end switch + code.putln("}") # end switch code.putln( - "}") # end if + "}") # end if - code.end_block() # end parallel control flow block + code.end_block() # end parallel control flow block self.redef_builtin_expect_apple_gcc_bug(code) # FIXME: improve with version number for OS X Lion @@ -8961,9 +9520,6 @@ class ParallelRangeNode(ParallelStatNode): else: self.start, self.stop, self.step = self.args - if hasattr(self.schedule, 'decode'): - self.schedule = self.schedule.decode('ascii') - if self.schedule not in (None, 'static', 'dynamic', 'guided', 'runtime'): error(self.pos, "Invalid schedule argument to prange: %s" % (self.schedule,)) @@ -9022,7 +9578,8 @@ class ParallelRangeNode(ParallelStatNode): # ensure lastprivate behaviour and propagation. If the target index is # not a NameNode, it won't have an entry, and an error was issued by # ParallelRangeTransform - if hasattr(self.target, 'entry'): + target_entry = getattr(self.target, 'entry', None) + if target_entry: self.assignments[self.target.entry] = self.target.pos, None node = super(ParallelRangeNode, self).analyse_expressions(env) @@ -9135,9 +9692,12 @@ class ParallelRangeNode(ParallelStatNode): # TODO: check if the step is 0 and if so, raise an exception in a # 'with gil' block. For now, just abort - code.putln("if ((%(step)s == 0)) abort();" % fmt_dict) + if self.step is not None and self.step.has_constant_result() and self.step.constant_result == 0: + error(node.pos, "Iteration with step 0 is invalid.") + elif not fmt_dict['step'].isdigit() or int(fmt_dict['step']) == 0: + code.putln("if (((%(step)s) == 0)) abort();" % fmt_dict) - self.setup_parallel_control_flow_block(code) # parallel control flow block + self.setup_parallel_control_flow_block(code) # parallel control flow block # Note: nsteps is private in an outer scope if present code.putln("%(nsteps)s = (%(stop)s - %(start)s + %(step)s - %(step)s/abs(%(step)s)) / %(step)s;" % fmt_dict) @@ -9149,9 +9709,9 @@ class ParallelRangeNode(ParallelStatNode): # erroneously believes that nsteps may be <= 0, leaving the private # target index uninitialized code.putln("if (%(nsteps)s > 0)" % fmt_dict) - code.begin_block() # if block + code.begin_block() # if block self.generate_loop(code, fmt_dict) - code.end_block() # end if block + code.end_block() # end if block self.restore_labels(code) @@ -9159,13 +9719,13 @@ class ParallelRangeNode(ParallelStatNode): if self.breaking_label_used: code.put("if (%s < 2)" % Naming.parallel_why) - code.begin_block() # else block + code.begin_block() # else block code.putln("/* else */") self.else_clause.generate_execution_code(code) - code.end_block() # end else block + code.end_block() # end else block # ------ cleanup ------ - self.end_parallel_control_flow_block(code) # end parallel control flow block + self.end_parallel_control_flow_block(code) # end parallel control flow block # And finally, release our privates and write back any closure # variables @@ -9196,7 +9756,7 @@ class ParallelRangeNode(ParallelStatNode): code.putln("") code.putln("#endif /* _OPENMP */") - code.begin_block() # pragma omp parallel begin block + code.begin_block() # pragma omp parallel begin block # Initialize the GIL if needed for this thread self.begin_parallel_block(code) diff --git a/Cython/Compiler/Optimize.py b/Cython/Compiler/Optimize.py index 69526a27c..1ea68feff 100644 --- a/Cython/Compiler/Optimize.py +++ b/Cython/Compiler/Optimize.py @@ -41,7 +41,7 @@ except ImportError: try: from __builtin__ import basestring except ImportError: - basestring = str # Python 3 + basestring = str # Python 3 def load_c_utility(name): @@ -192,7 +192,7 @@ class IterationTransform(Visitor.EnvTransform): def _optimise_for_loop(self, node, iterable, reversed=False): annotation_type = None if (iterable.is_name or iterable.is_attribute) and iterable.entry and iterable.entry.annotation: - annotation = iterable.entry.annotation + annotation = iterable.entry.annotation.expr if annotation.is_subscript: annotation = annotation.base # container base type # FIXME: generalise annotation evaluation => maybe provide a "qualified name" also for imported names? @@ -228,6 +228,12 @@ class IterationTransform(Visitor.EnvTransform): return self._transform_bytes_iteration(node, iterable, reversed=reversed) if iterable.type is Builtin.unicode_type: return self._transform_unicode_iteration(node, iterable, reversed=reversed) + # in principle _transform_indexable_iteration would work on most of the above, and + # also tuple and list. However, it probably isn't quite as optimized + if iterable.type is Builtin.bytearray_type: + return self._transform_indexable_iteration(node, iterable, is_mutable=True, reversed=reversed) + if isinstance(iterable, ExprNodes.CoerceToPyTypeNode) and iterable.arg.type.is_memoryviewslice: + return self._transform_indexable_iteration(node, iterable.arg, is_mutable=False, reversed=reversed) # the rest is based on function calls if not isinstance(iterable, ExprNodes.SimpleCallNode): @@ -333,6 +339,92 @@ class IterationTransform(Visitor.EnvTransform): PyrexTypes.CFuncTypeArg("s", Builtin.bytes_type, None) ]) + def _transform_indexable_iteration(self, node, slice_node, is_mutable, reversed=False): + """In principle can handle any iterable that Cython has a len() for and knows how to index""" + unpack_temp_node = UtilNodes.LetRefNode( + slice_node.as_none_safe_node("'NoneType' is not iterable"), + may_hold_none=False, is_temp=True + ) + + start_node = ExprNodes.IntNode( + node.pos, value='0', constant_result=0, type=PyrexTypes.c_py_ssize_t_type) + def make_length_call(): + # helper function since we need to create this node for a couple of places + builtin_len = ExprNodes.NameNode(node.pos, name="len", + entry=Builtin.builtin_scope.lookup("len")) + return ExprNodes.SimpleCallNode(node.pos, + function=builtin_len, + args=[unpack_temp_node] + ) + length_temp = UtilNodes.LetRefNode(make_length_call(), type=PyrexTypes.c_py_ssize_t_type, is_temp=True) + end_node = length_temp + + if reversed: + relation1, relation2 = '>', '>=' + start_node, end_node = end_node, start_node + else: + relation1, relation2 = '<=', '<' + + counter_ref = UtilNodes.LetRefNode(pos=node.pos, type=PyrexTypes.c_py_ssize_t_type) + + target_value = ExprNodes.IndexNode(slice_node.pos, base=unpack_temp_node, + index=counter_ref) + + target_assign = Nodes.SingleAssignmentNode( + pos = node.target.pos, + lhs = node.target, + rhs = target_value) + + # analyse with boundscheck and wraparound + # off (because we're confident we know the size) + env = self.current_env() + new_directives = Options.copy_inherited_directives(env.directives, boundscheck=False, wraparound=False) + target_assign = Nodes.CompilerDirectivesNode( + target_assign.pos, + directives=new_directives, + body=target_assign, + ) + + body = Nodes.StatListNode( + node.pos, + stats = [target_assign]) # exclude node.body for now to not reanalyse it + if is_mutable: + # We need to be slightly careful here that we are actually modifying the loop + # bounds and not a temp copy of it. Setting is_temp=True on length_temp seems + # to ensure this. + # If this starts to fail then we could insert an "if out_of_bounds: break" instead + loop_length_reassign = Nodes.SingleAssignmentNode(node.pos, + lhs = length_temp, + rhs = make_length_call()) + body.stats.append(loop_length_reassign) + + loop_node = Nodes.ForFromStatNode( + node.pos, + bound1=start_node, relation1=relation1, + target=counter_ref, + relation2=relation2, bound2=end_node, + step=None, body=body, + else_clause=node.else_clause, + from_range=True) + + ret = UtilNodes.LetNode( + unpack_temp_node, + UtilNodes.LetNode( + length_temp, + # TempResultFromStatNode provides the framework where the "counter_ref" + # temp is set up and can be assigned to. However, we don't need the + # result it returns so wrap it in an ExprStatNode. + Nodes.ExprStatNode(node.pos, + expr=UtilNodes.TempResultFromStatNode( + counter_ref, + loop_node + ) + ) + ) + ).analyse_expressions(env) + body.stats.insert(1, node.body) + return ret + def _transform_bytes_iteration(self, node, slice_node, reversed=False): target_type = node.target.type if not target_type.is_int and target_type is not Builtin.bytes_type: @@ -1144,7 +1236,7 @@ class SwitchTransform(Visitor.EnvTransform): # integers on iteration, whereas Py2 returns 1-char byte # strings characters = string_literal.value - characters = list(set([ characters[i:i+1] for i in range(len(characters)) ])) + characters = list({ characters[i:i+1] for i in range(len(characters)) }) characters.sort() return [ ExprNodes.CharNode(string_literal.pos, value=charval, constant_result=charval) @@ -1156,7 +1248,8 @@ class SwitchTransform(Visitor.EnvTransform): return self.NO_MATCH elif common_var is not None and not is_common_value(var, common_var): return self.NO_MATCH - elif not (var.type.is_int or var.type.is_enum) or sum([not (cond.type.is_int or cond.type.is_enum) for cond in conditions]): + elif not (var.type.is_int or var.type.is_enum) or any( + [not (cond.type.is_int or cond.type.is_enum) for cond in conditions]): return self.NO_MATCH return not_in, var, conditions @@ -1569,7 +1662,7 @@ class EarlyReplaceBuiltinCalls(Visitor.EnvTransform): utility_code = utility_code) def _error_wrong_arg_count(self, function_name, node, args, expected=None): - if not expected: # None or 0 + if not expected: # None or 0 arg_str = '' elif isinstance(expected, basestring) or expected > 1: arg_str = '...' @@ -1723,7 +1816,7 @@ class EarlyReplaceBuiltinCalls(Visitor.EnvTransform): arg = pos_args[0] if isinstance(arg, ExprNodes.ComprehensionNode) and arg.type is Builtin.list_type: - list_node = pos_args[0] + list_node = arg loop_node = list_node.loop elif isinstance(arg, ExprNodes.GeneratorExpressionNode): @@ -1753,7 +1846,11 @@ class EarlyReplaceBuiltinCalls(Visitor.EnvTransform): # Interestingly, PySequence_List works on a lot of non-sequence # things as well. list_node = loop_node = ExprNodes.PythonCapiCallNode( - node.pos, "PySequence_List", self.PySequence_List_func_type, + node.pos, + "__Pyx_PySequence_ListKeepNew" + if arg.is_temp and arg.type in (PyrexTypes.py_object_type, Builtin.list_type) + else "PySequence_List", + self.PySequence_List_func_type, args=pos_args, is_temp=True) result_node = UtilNodes.ResultRefNode( @@ -1799,7 +1896,7 @@ class EarlyReplaceBuiltinCalls(Visitor.EnvTransform): if not yield_expression.is_literal or not yield_expression.type.is_int: return node except AttributeError: - return node # in case we don't have a type yet + return node # in case we don't have a type yet # special case: old Py2 backwards compatible "sum([int_const for ...])" # can safely be unpacked into a genexpr @@ -2093,12 +2190,13 @@ class OptimizeBuiltinCalls(Visitor.NodeRefCleanupMixin, func_arg = arg.args[0] if func_arg.type is Builtin.float_type: return func_arg.as_none_safe_node("float() argument must be a string or a number, not 'NoneType'") - elif func_arg.type.is_pyobject: + elif func_arg.type.is_pyobject and arg.function.cname == "__Pyx_PyObject_AsDouble": return ExprNodes.PythonCapiCallNode( node.pos, '__Pyx_PyNumber_Float', self.PyNumber_Float_func_type, args=[func_arg], py_name='float', is_temp=node.is_temp, + utility_code = UtilityCode.load_cached("pynumber_float", "TypeConversion.c"), result_is_used=node.result_is_used, ).coerce_to(node.type, self.current_env()) return node @@ -2232,7 +2330,7 @@ class OptimizeBuiltinCalls(Visitor.NodeRefCleanupMixin, return node def _error_wrong_arg_count(self, function_name, node, args, expected=None): - if not expected: # None or 0 + if not expected: # None or 0 arg_str = '' elif isinstance(expected, basestring) or expected > 1: arg_str = '...' @@ -2312,6 +2410,38 @@ class OptimizeBuiltinCalls(Visitor.NodeRefCleanupMixin, return ExprNodes.CachedBuiltinMethodCallNode( node, function.obj, attr_name, arg_list) + PyObject_String_func_type = PyrexTypes.CFuncType( + PyrexTypes.py_object_type, [ # Change this to Builtin.str_type when removing Py2 support. + PyrexTypes.CFuncTypeArg("obj", PyrexTypes.py_object_type, None) + ]) + + def _handle_simple_function_str(self, node, function, pos_args): + """Optimize single argument calls to str(). + """ + if len(pos_args) != 1: + if len(pos_args) == 0: + return ExprNodes.StringNode(node.pos, value=EncodedString(), constant_result='') + return node + arg = pos_args[0] + + if arg.type is Builtin.str_type: + if not arg.may_be_none(): + return arg + + cname = "__Pyx_PyStr_Str" + utility_code = UtilityCode.load_cached('PyStr_Str', 'StringTools.c') + else: + cname = '__Pyx_PyObject_Str' + utility_code = UtilityCode.load_cached('PyObject_Str', 'StringTools.c') + + return ExprNodes.PythonCapiCallNode( + node.pos, cname, self.PyObject_String_func_type, + args=pos_args, + is_temp=node.is_temp, + utility_code=utility_code, + py_name="str" + ) + PyObject_Unicode_func_type = PyrexTypes.CFuncType( Builtin.unicode_type, [ PyrexTypes.CFuncTypeArg("obj", PyrexTypes.py_object_type, None) @@ -2383,8 +2513,14 @@ class OptimizeBuiltinCalls(Visitor.NodeRefCleanupMixin, return node arg = pos_args[0] return ExprNodes.PythonCapiCallNode( - node.pos, "PySequence_List", self.PySequence_List_func_type, - args=pos_args, is_temp=node.is_temp) + node.pos, + "__Pyx_PySequence_ListKeepNew" + if node.is_temp and arg.is_temp and arg.type in (PyrexTypes.py_object_type, Builtin.list_type) + else "PySequence_List", + self.PySequence_List_func_type, + args=pos_args, + is_temp=node.is_temp, + ) PyList_AsTuple_func_type = PyrexTypes.CFuncType( Builtin.tuple_type, [ @@ -2485,6 +2621,7 @@ class OptimizeBuiltinCalls(Visitor.NodeRefCleanupMixin, elif len(pos_args) != 1: self._error_wrong_arg_count('float', node, pos_args, '0 or 1') return node + func_arg = pos_args[0] if isinstance(func_arg, ExprNodes.CoerceToPyTypeNode): func_arg = func_arg.arg @@ -2493,12 +2630,37 @@ class OptimizeBuiltinCalls(Visitor.NodeRefCleanupMixin, elif node.type.assignable_from(func_arg.type) or func_arg.type.is_numeric: return ExprNodes.TypecastNode( node.pos, operand=func_arg, type=node.type) + + arg = None + if func_arg.type is Builtin.bytes_type: + cfunc_name = "__Pyx_PyBytes_AsDouble" + utility_code_name = 'pybytes_as_double' + elif func_arg.type is Builtin.bytearray_type: + cfunc_name = "__Pyx_PyByteArray_AsDouble" + utility_code_name = 'pybytes_as_double' + elif func_arg.type is Builtin.unicode_type: + cfunc_name = "__Pyx_PyUnicode_AsDouble" + utility_code_name = 'pyunicode_as_double' + elif func_arg.type is Builtin.str_type: + cfunc_name = "__Pyx_PyString_AsDouble" + utility_code_name = 'pystring_as_double' + elif func_arg.type is Builtin.long_type: + cfunc_name = "PyLong_AsDouble" + else: + arg = func_arg # no need for an additional None check + cfunc_name = "__Pyx_PyObject_AsDouble" + utility_code_name = 'pyobject_as_double' + + if arg is None: + arg = func_arg.as_none_safe_node( + "float() argument must be a string or a number, not 'NoneType'") + return ExprNodes.PythonCapiCallNode( - node.pos, "__Pyx_PyObject_AsDouble", + node.pos, cfunc_name, self.PyObject_AsDouble_func_type, - args = pos_args, + args = [arg], is_temp = node.is_temp, - utility_code = load_c_utility('pyobject_as_double'), + utility_code = load_c_utility(utility_code_name) if utility_code_name else None, py_name = "float") PyNumber_Int_func_type = PyrexTypes.CFuncType( @@ -2557,12 +2719,20 @@ class OptimizeBuiltinCalls(Visitor.NodeRefCleanupMixin, Pyx_strlen_func_type = PyrexTypes.CFuncType( PyrexTypes.c_size_t_type, [ PyrexTypes.CFuncTypeArg("bytes", PyrexTypes.c_const_char_ptr_type, None) - ]) + ], + nogil=True) + + Pyx_ssize_strlen_func_type = PyrexTypes.CFuncType( + PyrexTypes.c_py_ssize_t_type, [ + PyrexTypes.CFuncTypeArg("bytes", PyrexTypes.c_const_char_ptr_type, None) + ], + exception_value="-1") Pyx_Py_UNICODE_strlen_func_type = PyrexTypes.CFuncType( - PyrexTypes.c_size_t_type, [ + PyrexTypes.c_py_ssize_t_type, [ PyrexTypes.CFuncTypeArg("unicode", PyrexTypes.c_const_py_unicode_ptr_type, None) - ]) + ], + exception_value="-1") PyObject_Size_func_type = PyrexTypes.CFuncType( PyrexTypes.c_py_ssize_t_type, [ @@ -2581,7 +2751,7 @@ class OptimizeBuiltinCalls(Visitor.NodeRefCleanupMixin, Builtin.dict_type: "PyDict_Size", }.get - _ext_types_with_pysize = set(["cpython.array.array"]) + _ext_types_with_pysize = {"cpython.array.array"} def _handle_simple_function_len(self, node, function, pos_args): """Replace len(char*) by the equivalent call to strlen(), @@ -2596,18 +2766,19 @@ class OptimizeBuiltinCalls(Visitor.NodeRefCleanupMixin, arg = arg.arg if arg.type.is_string: new_node = ExprNodes.PythonCapiCallNode( - node.pos, "strlen", self.Pyx_strlen_func_type, + node.pos, "__Pyx_ssize_strlen", self.Pyx_ssize_strlen_func_type, args = [arg], is_temp = node.is_temp, - utility_code = UtilityCode.load_cached("IncludeStringH", "StringTools.c")) + utility_code = UtilityCode.load_cached("ssize_strlen", "StringTools.c")) elif arg.type.is_pyunicode_ptr: new_node = ExprNodes.PythonCapiCallNode( - node.pos, "__Pyx_Py_UNICODE_strlen", self.Pyx_Py_UNICODE_strlen_func_type, + node.pos, "__Pyx_Py_UNICODE_ssize_strlen", self.Pyx_Py_UNICODE_strlen_func_type, args = [arg], - is_temp = node.is_temp) + is_temp = node.is_temp, + utility_code = UtilityCode.load_cached("ssize_pyunicode_strlen", "StringTools.c")) elif arg.type.is_memoryviewslice: func_type = PyrexTypes.CFuncType( - PyrexTypes.c_size_t_type, [ + PyrexTypes.c_py_ssize_t_type, [ PyrexTypes.CFuncTypeArg("memoryviewslice", arg.type, None) ], nogil=True) new_node = ExprNodes.PythonCapiCallNode( @@ -2618,7 +2789,7 @@ class OptimizeBuiltinCalls(Visitor.NodeRefCleanupMixin, if cfunc_name is None: arg_type = arg.type if ((arg_type.is_extension_type or arg_type.is_builtin_type) - and arg_type.entry.qualified_name in self._ext_types_with_pysize): + and arg_type.entry.qualified_name in self._ext_types_with_pysize): cfunc_name = 'Py_SIZE' else: return node @@ -2777,11 +2948,9 @@ class OptimizeBuiltinCalls(Visitor.NodeRefCleanupMixin, return node type_arg = args[0] if not obj.is_name or not type_arg.is_name: - # play safe - return node + return node # not a simple case if obj.type != Builtin.type_type or type_arg.type != Builtin.type_type: - # not a known type, play safe - return node + return node # not a known type if not type_arg.type_entry or not obj.type_entry: if obj.name != type_arg.name: return node @@ -2843,6 +3012,13 @@ class OptimizeBuiltinCalls(Visitor.NodeRefCleanupMixin, is_temp=node.is_temp ) + def _handle_any_slot__class__(self, node, function, args, + is_unbound_method, kwargs=None): + # The purpose of this function is to handle calls to instance.__class__() so that + # it doesn't get handled by the __Pyx_CallUnboundCMethod0 mechanism. + # TODO: optimizations of the instance.__class__() call might be possible in future. + return node + ### methods of builtin types PyObject_Append_func_type = PyrexTypes.CFuncType( @@ -3178,6 +3354,9 @@ class OptimizeBuiltinCalls(Visitor.NodeRefCleanupMixin, def _handle_simple_method_object___sub__(self, node, function, args, is_unbound_method): return self._optimise_num_binop('Subtract', node, function, args, is_unbound_method) + def _handle_simple_method_object___mul__(self, node, function, args, is_unbound_method): + return self._optimise_num_binop('Multiply', node, function, args, is_unbound_method) + def _handle_simple_method_object___eq__(self, node, function, args, is_unbound_method): return self._optimise_num_binop('Eq', node, function, args, is_unbound_method) @@ -3385,6 +3564,8 @@ class OptimizeBuiltinCalls(Visitor.NodeRefCleanupMixin, PyrexTypes.CFuncTypeArg("uchar", PyrexTypes.c_py_ucs4_type, None), ]) + # DISABLED: Return value can only be one character, which is not correct. + ''' def _inject_unicode_character_conversion(self, node, function, args, is_unbound_method): if is_unbound_method or len(args) != 1: return node @@ -3403,9 +3584,10 @@ class OptimizeBuiltinCalls(Visitor.NodeRefCleanupMixin, func_call = func_call.coerce_to_pyobject(self.current_env) return func_call - _handle_simple_method_unicode_lower = _inject_unicode_character_conversion - _handle_simple_method_unicode_upper = _inject_unicode_character_conversion - _handle_simple_method_unicode_title = _inject_unicode_character_conversion + #_handle_simple_method_unicode_lower = _inject_unicode_character_conversion + #_handle_simple_method_unicode_upper = _inject_unicode_character_conversion + #_handle_simple_method_unicode_title = _inject_unicode_character_conversion + ''' PyUnicode_Splitlines_func_type = PyrexTypes.CFuncType( Builtin.list_type, [ @@ -3784,7 +3966,7 @@ class OptimizeBuiltinCalls(Visitor.NodeRefCleanupMixin, if not stop: # use strlen() to find the string length, just as CPython would if not string_node.is_name: - string_node = UtilNodes.LetRefNode(string_node) # used twice + string_node = UtilNodes.LetRefNode(string_node) # used twice temps.append(string_node) stop = ExprNodes.PythonCapiCallNode( string_node.pos, "strlen", self.Pyx_strlen_func_type, @@ -4435,25 +4617,25 @@ class ConstantFolding(Visitor.VisitorTransform, SkipDeclarations): args = [] items = [] - def add(arg): + def add(parent, arg): if arg.is_dict_literal: - if items: - items[0].key_value_pairs.extend(arg.key_value_pairs) + if items and items[-1].reject_duplicates == arg.reject_duplicates: + items[-1].key_value_pairs.extend(arg.key_value_pairs) else: items.append(arg) - elif isinstance(arg, ExprNodes.MergedDictNode): + elif isinstance(arg, ExprNodes.MergedDictNode) and parent.reject_duplicates == arg.reject_duplicates: for child_arg in arg.keyword_args: - add(child_arg) + add(arg, child_arg) else: if items: - args.append(items[0]) + args.extend(items) del items[:] args.append(arg) for arg in node.keyword_args: - add(arg) + add(node, arg) if items: - args.append(items[0]) + args.extend(items) if len(args) == 1: arg = args[0] @@ -4542,22 +4724,20 @@ class ConstantFolding(Visitor.VisitorTransform, SkipDeclarations): cascades = [[node.operand1]] final_false_result = [] - def split_cascades(cmp_node): + cmp_node = node + while cmp_node is not None: if cmp_node.has_constant_result(): if not cmp_node.constant_result: # False => short-circuit final_false_result.append(self._bool_node(cmp_node, False)) - return + break else: # True => discard and start new cascade cascades.append([cmp_node.operand2]) else: # not constant => append to current cascade cascades[-1].append(cmp_node) - if cmp_node.cascade: - split_cascades(cmp_node.cascade) - - split_cascades(node) + cmp_node = cmp_node.cascade cmp_nodes = [] for cascade in cascades: @@ -4703,6 +4883,30 @@ class ConstantFolding(Visitor.VisitorTransform, SkipDeclarations): return None return node + def visit_GILStatNode(self, node): + self.visitchildren(node) + if node.condition is None: + return node + + if node.condition.has_constant_result(): + # Condition is True - Modify node to be a normal + # GILStatNode with condition=None + if node.condition.constant_result: + node.condition = None + + # Condition is False - the body of the GILStatNode + # should run without changing the state of the gil + # return the body of the GILStatNode + else: + return node.body + + # If condition is not constant we keep the GILStatNode as it is. + # Either it will later become constant (e.g. a `numeric is int` + # expression in a fused type function) and then when ConstantFolding + # runs again it will be handled or a later transform (i.e. GilCheck) + # will raise an error + return node + # in the future, other nodes can have their own handler method here # that can replace them with a constant result node @@ -4719,6 +4923,7 @@ class FinalOptimizePhase(Visitor.EnvTransform, Visitor.NodeRefCleanupMixin): - isinstance -> typecheck for cdef types - eliminate checks for None and/or types that became redundant after tree changes - eliminate useless string formatting steps + - inject branch hints for unlikely if-cases that only raise exceptions - replace Python function calls that look like method calls by a faster PyMethodCallNode """ in_loop = False @@ -4817,6 +5022,48 @@ class FinalOptimizePhase(Visitor.EnvTransform, Visitor.NodeRefCleanupMixin): self.in_loop = old_val return node + def visit_IfStatNode(self, node): + """Assign 'unlikely' branch hints to if-clauses that only raise exceptions. + """ + self.visitchildren(node) + last_non_unlikely_clause = None + for i, if_clause in enumerate(node.if_clauses): + self._set_ifclause_branch_hint(if_clause, if_clause.body) + if not if_clause.branch_hint: + last_non_unlikely_clause = if_clause + if node.else_clause and last_non_unlikely_clause: + # If the 'else' clause is 'unlikely', then set the preceding 'if' clause to 'likely' to reflect that. + self._set_ifclause_branch_hint(last_non_unlikely_clause, node.else_clause, inverse=True) + return node + + def _set_ifclause_branch_hint(self, clause, statements_node, inverse=False): + """Inject a branch hint if the if-clause unconditionally leads to a 'raise' statement. + """ + if not statements_node.is_terminator: + return + # Allow simple statements, but no conditions, loops, etc. + non_branch_nodes = ( + Nodes.ExprStatNode, + Nodes.AssignmentNode, + Nodes.AssertStatNode, + Nodes.DelStatNode, + Nodes.GlobalNode, + Nodes.NonlocalNode, + ) + statements = [statements_node] + for next_node_pos, node in enumerate(statements, 1): + if isinstance(node, Nodes.GILStatNode): + statements.insert(next_node_pos, node.body) + continue + if isinstance(node, Nodes.StatListNode): + statements[next_node_pos:next_node_pos] = node.stats + continue + if not isinstance(node, non_branch_nodes): + if next_node_pos == len(statements) and isinstance(node, (Nodes.RaiseStatNode, Nodes.ReraiseStatNode)): + # Anything that unconditionally raises exceptions at the end should be considered unlikely. + clause.branch_hint = 'likely' if inverse else 'unlikely' + break + class ConsolidateOverflowCheck(Visitor.CythonTransform): """ diff --git a/Cython/Compiler/Options.py b/Cython/Compiler/Options.py index 48695dbfc..16c21393b 100644 --- a/Cython/Compiler/Options.py +++ b/Cython/Compiler/Options.py @@ -4,6 +4,10 @@ from __future__ import absolute_import +import os + +from Cython import Utils + class ShouldBeFromDirective(object): @@ -25,15 +29,14 @@ class ShouldBeFromDirective(object): raise RuntimeError(repr(self)) def __repr__(self): - return ( - "Illegal access of '%s' from Options module rather than directive '%s'" - % (self.options_name, self.directive_name)) + return "Illegal access of '%s' from Options module rather than directive '%s'" % ( + self.options_name, self.directive_name) """ The members of this module are documented using autodata in Cython/docs/src/reference/compilation.rst. -See http://www.sphinx-doc.org/en/master/ext/autodoc.html#directive-autoattribute +See https://www.sphinx-doc.org/en/master/usage/extensions/autodoc.html#directive-autoattribute for how autodata works. Descriptions of those members should start with a #: Donc forget to keep the docs in sync by removing and adding @@ -48,11 +51,6 @@ docstrings = True #: Embed the source code position in the docstrings of functions and classes. embed_pos_in_docstring = False -#: Copy the original source code line by line into C code comments -#: in the generated code file to help with understanding the output. -#: This is also required for coverage analysis. -emit_code_comments = True - # undocumented pre_import = None @@ -168,8 +166,19 @@ def get_directive_defaults(): _directive_defaults[old_option.directive_name] = value return _directive_defaults +def copy_inherited_directives(outer_directives, **new_directives): + # A few directives are not copied downwards and this function removes them. + # For example, test_assert_path_exists and test_fail_if_path_exists should not be inherited + # otherwise they can produce very misleading test failures + new_directives_out = dict(outer_directives) + for name in ('test_assert_path_exists', 'test_fail_if_path_exists'): + new_directives_out.pop(name, None) + new_directives_out.update(new_directives) + return new_directives_out + # Declare compiler directives _directive_defaults = { + 'binding': True, # was False before 3.0 'boundscheck' : True, 'nonecheck' : False, 'initializedcheck' : True, @@ -178,10 +187,10 @@ _directive_defaults = { 'auto_pickle': None, 'cdivision': False, # was True before 0.12 'cdivision_warnings': False, - 'c_api_binop_methods': True, + 'c_api_binop_methods': False, # was True before 3.0 'overflowcheck': False, 'overflowcheck.fold': True, - 'always_allow_keywords': False, + 'always_allow_keywords': True, 'allow_none_for_extension_args': True, 'wraparound' : True, 'ccomplex' : False, # use C99/C++ for complex types and arith @@ -208,6 +217,7 @@ _directive_defaults = { 'old_style_globals': False, 'np_pythran': False, 'fast_gil': False, + 'cpp_locals': False, # uses std::optional for C++ locals, so that they work more like Python locals # set __file__ and/or __path__ to known source/target path at import time (instead of not having them available) 'set_initial_path' : None, # SOURCEFILE or "/full/path/to/module" @@ -239,8 +249,6 @@ _directive_defaults = { 'test_fail_if_path_exists' : [], # experimental, subject to change - 'binding': None, - 'formal_grammar': False, } @@ -318,6 +326,8 @@ directive_types = { 'freelist': int, 'c_string_type': one_of('bytes', 'bytearray', 'str', 'unicode'), 'c_string_encoding': normalise_encoding_name, + 'trashcan': bool, + 'total_ordering': bool, } for key, val in _directive_defaults.items(): @@ -360,6 +370,9 @@ directive_scopes = { # defaults to available everywhere 'np_pythran': ('module',), 'fast_gil': ('module',), 'iterable_coroutine': ('module', 'function'), + 'trashcan' : ('cclass',), + 'total_ordering': ('cclass', ), + 'cpp_locals': ('module', 'function', 'cclass'), # I don't think they make sense in a with_statement } @@ -548,3 +561,184 @@ def parse_compile_time_env(s, current_settings=None): name, value = [s.strip() for s in item.split('=', 1)] result[name] = parse_variable_value(value) return result + + +# ------------------------------------------------------------------------ +# CompilationOptions are constructed from user input and are the `option` +# object passed throughout the compilation pipeline. + +class CompilationOptions(object): + r""" + See default_options at the end of this module for a list of all possible + options and CmdLine.usage and CmdLine.parse_command_line() for their + meaning. + """ + def __init__(self, defaults=None, **kw): + self.include_path = [] + if defaults: + if isinstance(defaults, CompilationOptions): + defaults = defaults.__dict__ + else: + defaults = default_options + + options = dict(defaults) + options.update(kw) + + # let's assume 'default_options' contains a value for most known compiler options + # and validate against them + unknown_options = set(options) - set(default_options) + # ignore valid options that are not in the defaults + unknown_options.difference_update(['include_path']) + if unknown_options: + message = "got unknown compilation option%s, please remove: %s" % ( + 's' if len(unknown_options) > 1 else '', + ', '.join(unknown_options)) + raise ValueError(message) + + directive_defaults = get_directive_defaults() + directives = dict(options['compiler_directives']) # copy mutable field + # check for invalid directives + unknown_directives = set(directives) - set(directive_defaults) + if unknown_directives: + message = "got unknown compiler directive%s: %s" % ( + 's' if len(unknown_directives) > 1 else '', + ', '.join(unknown_directives)) + raise ValueError(message) + options['compiler_directives'] = directives + if directives.get('np_pythran', False) and not options['cplus']: + import warnings + warnings.warn("C++ mode forced when in Pythran mode!") + options['cplus'] = True + if 'language_level' in directives and 'language_level' not in kw: + options['language_level'] = directives['language_level'] + elif not options.get('language_level'): + options['language_level'] = directive_defaults.get('language_level') + if 'formal_grammar' in directives and 'formal_grammar' not in kw: + options['formal_grammar'] = directives['formal_grammar'] + if options['cache'] is True: + options['cache'] = os.path.join(Utils.get_cython_cache_dir(), 'compiler') + + self.__dict__.update(options) + + def configure_language_defaults(self, source_extension): + if source_extension == 'py': + if self.compiler_directives.get('binding') is None: + self.compiler_directives['binding'] = True + + def get_fingerprint(self): + r""" + Return a string that contains all the options that are relevant for cache invalidation. + """ + # Collect only the data that can affect the generated file(s). + data = {} + + for key, value in self.__dict__.items(): + if key in ['show_version', 'errors_to_stderr', 'verbose', 'quiet']: + # verbosity flags have no influence on the compilation result + continue + elif key in ['output_file', 'output_dir']: + # ignore the exact name of the output file + continue + elif key in ['timestamps']: + # the cache cares about the content of files, not about the timestamps of sources + continue + elif key in ['cache']: + # hopefully caching has no influence on the compilation result + continue + elif key in ['compiler_directives']: + # directives passed on to the C compiler do not influence the generated C code + continue + elif key in ['include_path']: + # this path changes which headers are tracked as dependencies, + # it has no influence on the generated C code + continue + elif key in ['working_path']: + # this path changes where modules and pxd files are found; + # their content is part of the fingerprint anyway, their + # absolute path does not matter + continue + elif key in ['create_extension']: + # create_extension() has already mangled the options, e.g., + # embedded_metadata, when the fingerprint is computed so we + # ignore it here. + continue + elif key in ['build_dir']: + # the (temporary) directory where we collect dependencies + # has no influence on the C output + continue + elif key in ['use_listing_file', 'generate_pxi', 'annotate', 'annotate_coverage_xml']: + # all output files are contained in the cache so the types of + # files generated must be part of the fingerprint + data[key] = value + elif key in ['formal_grammar', 'evaluate_tree_assertions']: + # these bits can change whether compilation to C passes/fails + data[key] = value + elif key in ['embedded_metadata', 'emit_linenums', + 'c_line_in_traceback', 'gdb_debug', + 'relative_path_in_code_position_comments']: + # the generated code contains additional bits when these are set + data[key] = value + elif key in ['cplus', 'language_level', 'compile_time_env', 'np_pythran']: + # assorted bits that, e.g., influence the parser + data[key] = value + elif key == ['capi_reexport_cincludes']: + if self.capi_reexport_cincludes: + # our caching implementation does not yet include fingerprints of all the header files + raise NotImplementedError('capi_reexport_cincludes is not compatible with Cython caching') + elif key == ['common_utility_include_dir']: + if self.common_utility_include_dir: + raise NotImplementedError('common_utility_include_dir is not compatible with Cython caching yet') + else: + # any unexpected option should go into the fingerprint; it's better + # to recompile than to return incorrect results from the cache. + data[key] = value + + def to_fingerprint(item): + r""" + Recursively turn item into a string, turning dicts into lists with + deterministic ordering. + """ + if isinstance(item, dict): + item = sorted([(repr(key), to_fingerprint(value)) for key, value in item.items()]) + return repr(item) + + return to_fingerprint(data) + + +# ------------------------------------------------------------------------ +# +# Set the default options depending on the platform +# +# ------------------------------------------------------------------------ + +default_options = dict( + show_version=0, + use_listing_file=0, + errors_to_stderr=1, + cplus=0, + output_file=None, + annotate=None, + annotate_coverage_xml=None, + generate_pxi=0, + capi_reexport_cincludes=0, + working_path="", + timestamps=None, + verbose=0, + quiet=0, + compiler_directives={}, + embedded_metadata={}, + evaluate_tree_assertions=False, + emit_linenums=False, + relative_path_in_code_position_comments=True, + c_line_in_traceback=True, + language_level=None, # warn but default to 2 + formal_grammar=False, + gdb_debug=False, + compile_time_env=None, + common_utility_include_dir=None, + output_dir=None, + build_dir=None, + cache=None, + create_extension=None, + np_pythran=False +) diff --git a/Cython/Compiler/ParseTreeTransforms.pxd b/Cython/Compiler/ParseTreeTransforms.pxd index 2c17901fa..c6a7b2ddb 100644 --- a/Cython/Compiler/ParseTreeTransforms.pxd +++ b/Cython/Compiler/ParseTreeTransforms.pxd @@ -1,5 +1,4 @@ - -from __future__ import absolute_import +# cython: language_level=3str cimport cython diff --git a/Cython/Compiler/ParseTreeTransforms.py b/Cython/Compiler/ParseTreeTransforms.py index e7a33341b..0eb91163a 100644 --- a/Cython/Compiler/ParseTreeTransforms.py +++ b/Cython/Compiler/ParseTreeTransforms.py @@ -1,3 +1,5 @@ +# cython: language_level=3str + from __future__ import absolute_import import cython @@ -52,6 +54,12 @@ class SkipDeclarations(object): def visit_CStructOrUnionDefNode(self, node): return node + def visit_CppClassNode(self, node): + if node.visibility != "extern": + # Need to traverse methods. + self.visitchildren(node) + return node + class NormalizeTree(CythonTransform): """ @@ -79,6 +87,13 @@ class NormalizeTree(CythonTransform): self.is_in_statlist = False self.is_in_expr = False + def visit_ModuleNode(self, node): + self.visitchildren(node) + if not isinstance(node.body, Nodes.StatListNode): + # This can happen when the body only consists of a single (unused) declaration and no statements. + node.body = Nodes.StatListNode(pos=node.pos, stats=[node.body]) + return node + def visit_ExprNode(self, node): stacktmp = self.is_in_expr self.is_in_expr = True @@ -169,7 +184,6 @@ class PostParse(ScopeTrackingTransform): reorganization that can be refactored into this transform if a more pure Abstract Syntax Tree is wanted. """ - def __init__(self, context): super(PostParse, self).__init__(context) self.specialattribute_handlers = { @@ -244,7 +258,7 @@ class PostParse(ScopeTrackingTransform): if decl is not declbase: raise PostParseError(decl.pos, ERR_INVALID_SPECIALATTR_TYPE) handler(decl) - continue # Remove declaration + continue # Remove declaration raise PostParseError(decl.pos, ERR_CDEF_INCLASS) first_assignment = self.scope_type != 'module' stats.append(Nodes.SingleAssignmentNode(node.pos, @@ -347,6 +361,23 @@ class PostParse(ScopeTrackingTransform): self.visitchildren(node) return node + def visit_AssertStatNode(self, node): + """Extract the exception raising into a RaiseStatNode to simplify GIL handling. + """ + if node.exception is None: + node.exception = Nodes.RaiseStatNode( + node.pos, + exc_type=ExprNodes.NameNode(node.pos, name=EncodedString("AssertionError")), + exc_value=node.value, + exc_tb=None, + cause=None, + builtin_exc_name="AssertionError", + wrap_tuple_value=True, + ) + node.value = None + self.visitchildren(node) + return node + def eliminate_rhs_duplicates(expr_list_list, ref_node_sequence): """Replace rhs items by LetRefNodes if they appear more than once. @@ -417,7 +448,7 @@ def sort_common_subsequences(items): return b.is_sequence_constructor and contains(b.args, a) for pos, item in enumerate(items): - key = item[1] # the ResultRefNode which has already been injected into the sequences + key = item[1] # the ResultRefNode which has already been injected into the sequences new_pos = pos for i in range(pos-1, -1, -1): if lower_than(key, items[i][0]): @@ -447,7 +478,7 @@ def flatten_parallel_assignments(input, output): # recursively, so that nested structures get matched as well. rhs = input[-1] if (not (rhs.is_sequence_constructor or isinstance(rhs, ExprNodes.UnicodeNode)) - or not sum([lhs.is_sequence_constructor for lhs in input[:-1]])): + or not sum([lhs.is_sequence_constructor for lhs in input[:-1]])): output.append(input) return @@ -531,7 +562,7 @@ def map_starred_assignment(lhs_targets, starred_assignments, lhs_args, rhs_args) targets.append(expr) # the starred target itself, must be assigned a (potentially empty) list - target = lhs_args[starred].target # unpack starred node + target = lhs_args[starred].target # unpack starred node starred_rhs = rhs_args[starred:] if lhs_remaining: starred_rhs = starred_rhs[:-lhs_remaining] @@ -577,19 +608,19 @@ class PxdPostParse(CythonTransform, SkipDeclarations): err = self.ERR_INLINE_ONLY if (isinstance(node, Nodes.DefNode) and self.scope_type == 'cclass' - and node.name in ('__getbuffer__', '__releasebuffer__')): - err = None # allow these slots + and node.name in ('__getbuffer__', '__releasebuffer__')): + err = None # allow these slots if isinstance(node, Nodes.CFuncDefNode): if (u'inline' in node.modifiers and - self.scope_type in ('pxd', 'cclass')): + self.scope_type in ('pxd', 'cclass')): node.inline_in_pxd = True if node.visibility != 'private': err = self.ERR_NOGO_WITH_INLINE % node.visibility elif node.api: err = self.ERR_NOGO_WITH_INLINE % 'api' else: - err = None # allow inline function + err = None # allow inline function else: err = self.ERR_INLINE_ONLY @@ -628,6 +659,9 @@ class InterpretCompilerDirectives(CythonTransform): - Command-line arguments overriding these - @cython.directivename decorators - with cython.directivename: statements + - replaces "cython.compiled" with BoolNode(value=True) + allowing unreachable blocks to be removed at a fairly early stage + before cython typing rules are forced on applied This transform is responsible for interpreting these various sources and store the directive in two ways: @@ -666,17 +700,19 @@ class InterpretCompilerDirectives(CythonTransform): 'operator.comma' : ExprNodes.c_binop_constructor(','), } - special_methods = set(['declare', 'union', 'struct', 'typedef', - 'sizeof', 'cast', 'pointer', 'compiled', - 'NULL', 'fused_type', 'parallel']) + special_methods = { + 'declare', 'union', 'struct', 'typedef', + 'sizeof', 'cast', 'pointer', 'compiled', + 'NULL', 'fused_type', 'parallel', + } special_methods.update(unop_method_nodes) - valid_parallel_directives = set([ + valid_parallel_directives = { "parallel", "prange", "threadid", #"threadsavailable", - ]) + } def __init__(self, context, compilation_directive_defaults): super(InterpretCompilerDirectives, self).__init__(context) @@ -747,22 +783,33 @@ class InterpretCompilerDirectives(CythonTransform): return result def visit_CImportStatNode(self, node): - if node.module_name == u"cython": + module_name = node.module_name + if module_name == u"cython.cimports": + error(node.pos, "Cannot cimport the 'cython.cimports' package directly, only submodules.") + if module_name.startswith(u"cython.cimports."): + if node.as_name and node.as_name != u'cython': + node.module_name = module_name[len(u"cython.cimports."):] + return node + error(node.pos, + "Python cimports must use 'from cython.cimports... import ...'" + " or 'import ... as ...', not just 'import ...'") + + if module_name == u"cython": self.cython_module_names.add(node.as_name or u"cython") - elif node.module_name.startswith(u"cython."): - if node.module_name.startswith(u"cython.parallel."): + elif module_name.startswith(u"cython."): + if module_name.startswith(u"cython.parallel."): error(node.pos, node.module_name + " is not a module") - if node.module_name == u"cython.parallel": + if module_name == u"cython.parallel": if node.as_name and node.as_name != u"cython": - self.parallel_directives[node.as_name] = node.module_name + self.parallel_directives[node.as_name] = module_name else: self.cython_module_names.add(u"cython") self.parallel_directives[ - u"cython.parallel"] = node.module_name + u"cython.parallel"] = module_name self.module_scope.use_utility_code( UtilityCode.load_cached("InitThreads", "ModuleSetupCode.c")) elif node.as_name: - self.directive_names[node.as_name] = node.module_name[7:] + self.directive_names[node.as_name] = module_name[7:] else: self.cython_module_names.add(u"cython") # if this cimport was a compiler directive, we don't @@ -771,9 +818,14 @@ class InterpretCompilerDirectives(CythonTransform): return node def visit_FromCImportStatNode(self, node): - if not node.relative_level and ( - node.module_name == u"cython" or node.module_name.startswith(u"cython.")): - submodule = (node.module_name + u".")[7:] + module_name = node.module_name + if module_name == u"cython.cimports" or module_name.startswith(u"cython.cimports."): + # only supported for convenience + return self._create_cimport_from_import( + node.pos, module_name, node.relative_level, node.imported_names) + elif not node.relative_level and ( + module_name == u"cython" or module_name.startswith(u"cython.")): + submodule = (module_name + u".")[7:] newimp = [] for pos, name, as_name, kind in node.imported_names: @@ -799,9 +851,17 @@ class InterpretCompilerDirectives(CythonTransform): return node def visit_FromImportStatNode(self, node): - if (node.module.module_name.value == u"cython") or \ - node.module.module_name.value.startswith(u"cython."): - submodule = (node.module.module_name.value + u".")[7:] + import_node = node.module + module_name = import_node.module_name.value + if module_name == u"cython.cimports" or module_name.startswith(u"cython.cimports."): + imported_names = [] + for name, name_node in node.items: + imported_names.append( + (name_node.pos, name, None if name == name_node.name else name_node.name, None)) + return self._create_cimport_from_import( + node.pos, module_name, import_node.level, imported_names) + elif module_name == u"cython" or module_name.startswith(u"cython."): + submodule = (module_name + u".")[7:] newimp = [] for name, name_node in node.items: full_name = submodule + name @@ -817,20 +877,35 @@ class InterpretCompilerDirectives(CythonTransform): node.items = newimp return node + def _create_cimport_from_import(self, node_pos, module_name, level, imported_names): + if module_name == u"cython.cimports" or module_name.startswith(u"cython.cimports."): + module_name = EncodedString(module_name[len(u"cython.cimports."):]) # may be empty + + if module_name: + # from cython.cimports.a.b import x, y, z => from a.b cimport x, y, z + return Nodes.FromCImportStatNode( + node_pos, module_name=module_name, + relative_level=level, + imported_names=imported_names) + else: + # from cython.cimports import x, y, z => cimport x; cimport y; cimport z + return [ + Nodes.CImportStatNode( + pos, + module_name=dotted_name, + as_name=as_name, + is_absolute=level == 0) + for pos, dotted_name, as_name, _ in imported_names + ] + def visit_SingleAssignmentNode(self, node): if isinstance(node.rhs, ExprNodes.ImportNode): module_name = node.rhs.module_name.value - is_parallel = (module_name + u".").startswith(u"cython.parallel.") - - if module_name != u"cython" and not is_parallel: + is_special_module = (module_name + u".").startswith((u"cython.parallel.", u"cython.cimports.")) + if module_name != u"cython" and not is_special_module: return node - module_name = node.rhs.module_name.value - as_name = node.lhs.name - - node = Nodes.CImportStatNode(node.pos, - module_name = module_name, - as_name = as_name) + node = Nodes.CImportStatNode(node.pos, module_name=module_name, as_name=node.lhs.name) node = self.visit_CImportStatNode(node) else: self.visitchildren(node) @@ -844,6 +919,23 @@ class InterpretCompilerDirectives(CythonTransform): directive = self.directive_names.get(node.name) if directive is not None: node.cython_attribute = directive + if node.as_cython_attribute() == "compiled": + return ExprNodes.BoolNode(node.pos, value=True) # replace early so unused branches can be dropped + # before they have a chance to cause compile-errors + return node + + def visit_AttributeNode(self, node): + self.visitchildren(node) + if node.as_cython_attribute() == "compiled": + return ExprNodes.BoolNode(node.pos, value=True) # replace early so unused branches can be dropped + # before they have a chance to cause compile-errors + return node + + def visit_AnnotationNode(self, node): + # for most transforms annotations are left unvisited (because they're unevaluated) + # however, it is important to pick up compiler directives from them + if node.expr: + self.visitchildren(node.expr) return node def visit_NewExprNode(self, node): @@ -963,8 +1055,7 @@ class InterpretCompilerDirectives(CythonTransform): return self.visit_Node(node) old_directives = self.directives - new_directives = dict(old_directives) - new_directives.update(directives) + new_directives = Options.copy_inherited_directives(old_directives, **directives) if new_directives == old_directives: return self.visit_Node(node) @@ -1031,7 +1122,8 @@ class InterpretCompilerDirectives(CythonTransform): else: realdecs.append(dec) if realdecs and (scope_name == 'cclass' or - isinstance(node, (Nodes.CFuncDefNode, Nodes.CClassDefNode, Nodes.CVarDefNode))): + isinstance(node, (Nodes.CClassDefNode, Nodes.CVarDefNode))): + # Note - arbitrary C function decorators are caught later in DecoratorTransform raise PostParseError(realdecs[0].pos, "Cdef functions/classes cannot take arbitrary decorators.") node.decorators = realdecs[::-1] + both[::-1] # merge or override repeated directives @@ -1326,16 +1418,16 @@ class DecoratorTransform(ScopeTrackingTransform, SkipDeclarations): _properties = None _map_property_attribute = { - 'getter': '__get__', - 'setter': '__set__', - 'deleter': '__del__', + 'getter': EncodedString('__get__'), + 'setter': EncodedString('__set__'), + 'deleter': EncodedString('__del__'), }.get def visit_CClassDefNode(self, node): if self._properties is None: self._properties = [] self._properties.append({}) - super(DecoratorTransform, self).visit_CClassDefNode(node) + node = super(DecoratorTransform, self).visit_CClassDefNode(node) self._properties.pop() return node @@ -1345,6 +1437,32 @@ class DecoratorTransform(ScopeTrackingTransform, SkipDeclarations): warning(node.pos, "'property %s:' syntax is deprecated, use '@property'" % node.name, level) return node + def visit_CFuncDefNode(self, node): + node = self.visit_FuncDefNode(node) + if not node.decorators: + return node + elif self.scope_type != 'cclass' or self.scope_node.visibility != "extern": + # at the moment cdef functions are very restricted in what decorators they can take + # so it's simple to test for the small number of allowed decorators.... + if not (len(node.decorators) == 1 and node.decorators[0].decorator.is_name and + node.decorators[0].decorator.name == "staticmethod"): + error(node.decorators[0].pos, "Cdef functions cannot take arbitrary decorators.") + return node + + ret_node = node + decorator_node = self._find_property_decorator(node) + if decorator_node: + if decorator_node.decorator.is_name: + name = node.declared_name() + if name: + ret_node = self._add_property(node, name, decorator_node) + else: + error(decorator_node.pos, "C property decorator can only be @property") + + if node.decorators: + return self._reject_decorated_property(node, node.decorators[0]) + return ret_node + def visit_DefNode(self, node): scope_type = self.scope_type node = self.visit_FuncDefNode(node) @@ -1352,28 +1470,12 @@ class DecoratorTransform(ScopeTrackingTransform, SkipDeclarations): return node # transform @property decorators - properties = self._properties[-1] - for decorator_node in node.decorators[::-1]: + decorator_node = self._find_property_decorator(node) + if decorator_node is not None: decorator = decorator_node.decorator - if decorator.is_name and decorator.name == 'property': - if len(node.decorators) > 1: - return self._reject_decorated_property(node, decorator_node) - name = node.name - node.name = EncodedString('__get__') - node.decorators.remove(decorator_node) - stat_list = [node] - if name in properties: - prop = properties[name] - prop.pos = node.pos - prop.doc = node.doc - prop.body.stats = stat_list - return [] - prop = Nodes.PropertyNode(node.pos, name=name) - prop.doc = node.doc - prop.body = Nodes.StatListNode(node.pos, stats=stat_list) - properties[name] = prop - return [prop] - elif decorator.is_attribute and decorator.obj.name in properties: + if decorator.is_name: + return self._add_property(node, node.name, decorator_node) + else: handler_name = self._map_property_attribute(decorator.attribute) if handler_name: if decorator.obj.name != node.name: @@ -1384,7 +1486,7 @@ class DecoratorTransform(ScopeTrackingTransform, SkipDeclarations): elif len(node.decorators) > 1: return self._reject_decorated_property(node, decorator_node) else: - return self._add_to_property(properties, node, handler_name, decorator_node) + return self._add_to_property(node, handler_name, decorator_node) # we clear node.decorators, so we need to set the # is_staticmethod/is_classmethod attributes now @@ -1399,6 +1501,18 @@ class DecoratorTransform(ScopeTrackingTransform, SkipDeclarations): node.decorators = None return self.chain_decorators(node, decs, node.name) + def _find_property_decorator(self, node): + properties = self._properties[-1] + for decorator_node in node.decorators[::-1]: + decorator = decorator_node.decorator + if decorator.is_name and decorator.name == 'property': + # @property + return decorator_node + elif decorator.is_attribute and decorator.obj.name in properties: + # @prop.setter etc. + return decorator_node + return None + @staticmethod def _reject_decorated_property(node, decorator_node): # restrict transformation to outermost decorator as wrapped properties will probably not work @@ -1407,9 +1521,42 @@ class DecoratorTransform(ScopeTrackingTransform, SkipDeclarations): error(deco.pos, "Property methods with additional decorators are not supported") return node - @staticmethod - def _add_to_property(properties, node, name, decorator): + def _add_property(self, node, name, decorator_node): + if len(node.decorators) > 1: + return self._reject_decorated_property(node, decorator_node) + node.decorators.remove(decorator_node) + properties = self._properties[-1] + is_cproperty = isinstance(node, Nodes.CFuncDefNode) + body = Nodes.StatListNode(node.pos, stats=[node]) + if is_cproperty: + if name in properties: + error(node.pos, "C property redeclared") + if 'inline' not in node.modifiers: + error(node.pos, "C property method must be declared 'inline'") + prop = Nodes.CPropertyNode(node.pos, doc=node.doc, name=name, body=body) + elif name in properties: + prop = properties[name] + if prop.is_cproperty: + error(node.pos, "C property redeclared") + else: + node.name = EncodedString("__get__") + prop.pos = node.pos + prop.doc = node.doc + prop.body.stats = [node] + return None + else: + node.name = EncodedString("__get__") + prop = Nodes.PropertyNode( + node.pos, name=name, doc=node.doc, body=body) + properties[name] = prop + return prop + + def _add_to_property(self, node, name, decorator): + properties = self._properties[-1] prop = properties[node.name] + if prop.is_cproperty: + error(node.pos, "C property redeclared") + return None node.name = name node.decorators.remove(decorator) stats = prop.body.stats @@ -1419,7 +1566,7 @@ class DecoratorTransform(ScopeTrackingTransform, SkipDeclarations): break else: stats.append(node) - return [] + return None @staticmethod def chain_decorators(node, decorators, name): @@ -1497,6 +1644,10 @@ class CnameDirectivesTransform(CythonTransform, SkipDeclarations): class ForwardDeclareTypes(CythonTransform): + """ + Declare all global cdef names that we allow referencing in other places, + before declaring everything (else) in source code order. + """ def visit_CompilerDirectivesNode(self, node): env = self.module_scope @@ -1540,6 +1691,14 @@ class ForwardDeclareTypes(CythonTransform): entry.type.get_all_specialized_function_types() return node + def visit_FuncDefNode(self, node): + # no traversal needed + return node + + def visit_PyClassDefNode(self, node): + # no traversal needed + return node + class AnalyseDeclarationsTransform(EnvTransform): @@ -1631,8 +1790,8 @@ if VALUE is not None: if stats: node.body.stats += stats if (node.visibility != 'extern' - and not node.scope.lookup('__reduce__') - and not node.scope.lookup('__reduce_ex__')): + and not node.scope.lookup('__reduce__') + and not node.scope.lookup('__reduce_ex__')): self._inject_pickle_methods(node) return node @@ -1686,9 +1845,9 @@ if VALUE is not None: pickle_func = TreeFragment(u""" def __reduce_cython__(self): - raise TypeError("%(msg)s") + raise TypeError, "%(msg)s" def __setstate_cython__(self, __pyx_state): - raise TypeError("%(msg)s") + raise TypeError, "%(msg)s" """ % {'msg': msg}, level='c_class', pipeline=[NormalizeTree(None)]).substitute({}) pickle_func.analyse_declarations(node.scope) @@ -1701,8 +1860,8 @@ if VALUE is not None: e.type.create_to_py_utility_code(env) e.type.create_from_py_utility_code(env) all_members_names = sorted([e.name for e in all_members]) - checksum = '0x%s' % hashlib.md5(' '.join(all_members_names).encode('utf-8')).hexdigest()[:7] - unpickle_func_name = '__pyx_unpickle_%s' % node.class_name + checksum = '0x%s' % hashlib.sha1(' '.join(all_members_names).encode('utf-8')).hexdigest()[:7] + unpickle_func_name = '__pyx_unpickle_%s' % node.punycode_class_name # TODO(robertwb): Move the state into the third argument # so it can be pickled *after* self is memoized. @@ -1712,7 +1871,7 @@ if VALUE is not None: cdef object __pyx_result if __pyx_checksum != %(checksum)s: from pickle import PickleError as __pyx_PickleError - raise __pyx_PickleError("Incompatible checksums (%%s vs %(checksum)s = (%(members)s))" %% __pyx_checksum) + raise __pyx_PickleError, "Incompatible checksums (%%s vs %(checksum)s = (%(members)s))" %% __pyx_checksum __pyx_result = %(class_name)s.__new__(__pyx_type) if __pyx_state is not None: %(unpickle_func_name)s__set_state(<%(class_name)s> __pyx_result, __pyx_state) @@ -1780,8 +1939,8 @@ if VALUE is not None: for decorator in old_decorators: func = decorator.decorator if (not func.is_name or - func.name not in ('staticmethod', 'classmethod') or - env.lookup_here(func.name)): + func.name not in ('staticmethod', 'classmethod') or + env.lookup_here(func.name)): # not a static or classmethod decorators.append(decorator) @@ -1801,6 +1960,8 @@ if VALUE is not None: node.stats.insert(0, node.py_func) node.py_func = self.visit(node.py_func) node.update_fused_defnode_entry(env) + # For the moment, fused functions do not support METH_FASTCALL + node.py_func.entry.signature.use_fastcall = False pycfunc = ExprNodes.PyCFunctionNode.from_defnode(node.py_func, binding=True) pycfunc = ExprNodes.ProxyNode(pycfunc.coerce_to_temp(env)) node.resulting_fused_function = pycfunc @@ -1843,19 +2004,6 @@ if VALUE is not None: return node - def _handle_nogil_cleanup(self, lenv, node): - "Handle cleanup for 'with gil' blocks in nogil functions." - if lenv.nogil and lenv.has_with_gil_block: - # Acquire the GIL for cleanup in 'nogil' functions, by wrapping - # the entire function body in try/finally. - # The corresponding release will be taken care of by - # Nodes.FuncDefNode.generate_function_definitions() - node.body = Nodes.NogilTryFinallyStatNode( - node.body.pos, - body=node.body, - finally_clause=Nodes.EnsureGILNode(node.body.pos), - finally_except_clause=Nodes.EnsureGILNode(node.body.pos)) - def _handle_fused(self, node): if node.is_generator and node.has_fused_arguments: node.has_fused_arguments = False @@ -1887,6 +2035,8 @@ if VALUE is not None: for var, type_node in node.directive_locals.items(): if not lenv.lookup_here(var): # don't redeclare args type = type_node.analyse_as_type(lenv) + if type and type.is_fused and lenv.fused_to_specific: + type = type.specialize(lenv.fused_to_specific) if type: lenv.declare_var(var, type, type_node.pos) else: @@ -1896,7 +2046,6 @@ if VALUE is not None: node = self._create_fused_function(env, node) else: node.body.analyse_declarations(lenv) - self._handle_nogil_cleanup(lenv, node) self._super_visit_FuncDefNode(node) self.seen_vars_stack.pop() @@ -1990,7 +2139,7 @@ if VALUE is not None: # (so it can't happen later). # Note that we don't return the original node, as it is # never used after this phase. - if True: # private (default) + if True: # private (default) return None self_value = ExprNodes.AttributeNode( @@ -2082,8 +2231,8 @@ if VALUE is not None: if node.name in self.seen_vars_stack[-1]: entry = self.current_env().lookup(node.name) if (entry is None or entry.visibility != 'extern' - and not entry.scope.is_c_class_scope): - warning(node.pos, "cdef variable '%s' declared after it is used" % node.name, 2) + and not entry.scope.is_c_class_scope): + error(node.pos, "cdef variable '%s' declared after it is used" % node.name) self.visitchildren(node) return node @@ -2096,7 +2245,7 @@ if VALUE is not None: child_node = self.visit(node.node) if not child_node: return None - if type(child_node) is list: # Assignment synthesized + if type(child_node) is list: # Assignment synthesized node.child_node = child_node[0] return [node] + child_node[1:] node.node = child_node @@ -2198,7 +2347,7 @@ class CalculateQualifiedNamesTransform(EnvTransform): def visit_ClassDefNode(self, node): orig_qualified_name = self.qualified_name[:] entry = (getattr(node, 'entry', None) or # PyClass - self.current_env().lookup_here(node.name)) # CClass + self.current_env().lookup_here(node.target.name)) # CClass self._append_entry(entry) self._super_visit_ClassDefNode(node) self.qualified_name = orig_qualified_name @@ -2307,8 +2456,8 @@ class ExpandInplaceOperators(EnvTransform): operand2 = rhs, inplace=True) # Manually analyse types for new node. - lhs.analyse_target_types(env) - dup.analyse_types(env) + lhs = lhs.analyse_target_types(env) + dup.analyse_types(env) # FIXME: no need to reanalyse the copy, right? binop.analyse_operation(env) node = Nodes.SingleAssignmentNode( node.pos, @@ -2358,12 +2507,12 @@ class AdjustDefByDirectives(CythonTransform, SkipDeclarations): return_type_node = self.directives.get('returns') if return_type_node is None and self.directives['annotation_typing']: return_type_node = node.return_type_annotation - # for Python anntations, prefer safe exception handling by default + # for Python annotations, prefer safe exception handling by default if return_type_node is not None and except_val is None: except_val = (None, True) # except * elif except_val is None: - # backward compatible default: no exception check - except_val = (None, False) + # backward compatible default: no exception check, unless there's also a "@returns" declaration + except_val = (None, True if return_type_node else False) if 'ccall' in self.directives: node = node.as_cfunction( overridable=True, modifiers=modifiers, nogil=nogil, @@ -2416,8 +2565,6 @@ class AlignFunctionDefinitions(CythonTransform): def visit_ModuleNode(self, node): self.scope = node.scope - self.directives = node.directives - self.imported_names = set() # hack, see visit_FromImportStatNode() self.visitchildren(node) return node @@ -2455,15 +2602,45 @@ class AlignFunctionDefinitions(CythonTransform): error(pxd_def.pos, "previous declaration here") return None node = node.as_cfunction(pxd_def) - elif (self.scope.is_module_scope and self.directives['auto_cpdef'] - and not node.name in self.imported_names - and node.is_cdef_func_compatible()): - # FIXME: cpdef-ing should be done in analyse_declarations() - node = node.as_cfunction(scope=self.scope) # Enable this when nested cdef functions are allowed. # self.visitchildren(node) return node + def visit_ExprNode(self, node): + # ignore lambdas and everything else that appears in expressions + return node + + +class AutoCpdefFunctionDefinitions(CythonTransform): + + def visit_ModuleNode(self, node): + self.directives = node.directives + self.imported_names = set() # hack, see visit_FromImportStatNode() + self.scope = node.scope + self.visitchildren(node) + return node + + def visit_DefNode(self, node): + if (self.scope.is_module_scope and self.directives['auto_cpdef'] + and node.name not in self.imported_names + and node.is_cdef_func_compatible()): + # FIXME: cpdef-ing should be done in analyse_declarations() + node = node.as_cfunction(scope=self.scope) + return node + + def visit_CClassDefNode(self, node, pxd_def=None): + if pxd_def is None: + pxd_def = self.scope.lookup(node.class_name) + if pxd_def: + if not pxd_def.defined_in_pxd: + return node + outer_scope = self.scope + self.scope = pxd_def.type.scope + self.visitchildren(node) + if pxd_def: + self.scope = outer_scope + return node + def visit_FromImportStatNode(self, node): # hack to prevent conditional import fallback functions from # being cdpef-ed (global Python variables currently conflict @@ -2705,7 +2882,7 @@ class CreateClosureClasses(CythonTransform): if not node.py_cfunc_node: raise InternalError("DefNode does not have assignment node") inner_node = node.py_cfunc_node - inner_node.needs_self_code = False + inner_node.needs_closure_code = False node.needs_outer_scope = False if node.is_generator: @@ -2724,6 +2901,7 @@ class CreateClosureClasses(CythonTransform): as_name = '%s_%s' % ( target_module_scope.next_id(Naming.closure_class_prefix), node.entry.cname.replace('.','__')) + as_name = EncodedString(as_name) entry = target_module_scope.declare_c_class( name=as_name, pos=node.pos, defining=True, @@ -2803,20 +2981,20 @@ class InjectGilHandling(VisitorTransform, SkipDeclarations): Must run before the AnalyseDeclarationsTransform to make sure the GILStatNodes get set up, parallel sections know that the GIL is acquired inside of them, etc. """ - def __call__(self, root): - self.nogil = False - return super(InjectGilHandling, self).__call__(root) + nogil = False # special node handling - def visit_RaiseStatNode(self, node): - """Allow raising exceptions in nogil sections by wrapping them in a 'with gil' block.""" + def _inject_gil_in_nogil(self, node): + """Allow the (Python statement) node in nogil sections by wrapping it in a 'with gil' block.""" if self.nogil: node = Nodes.GILStatNode(node.pos, state='gil', body=node) return node + visit_RaiseStatNode = _inject_gil_in_nogil + visit_PrintStatNode = _inject_gil_in_nogil # sadly, not the function + # further candidates: - # def visit_AssertStatNode(self, node): # def visit_ReraiseStatNode(self, node): # nogil tracking @@ -2877,7 +3055,7 @@ class GilCheck(VisitorTransform): self.visitchildren(node, outer_attrs) self.nogil = gil_state - self.visitchildren(node, exclude=outer_attrs) + self.visitchildren(node, attrs=None, exclude=outer_attrs) self.nogil = was_nogil def visit_FuncDefNode(self, node): @@ -2899,6 +3077,12 @@ class GilCheck(VisitorTransform): return node def visit_GILStatNode(self, node): + if node.condition is not None: + error(node.condition.pos, + "Non-constant condition in a " + "`with %s(<condition>)` statement" % node.state) + return node + if self.nogil and node.nogil_check: node.nogil_check() @@ -2974,6 +3158,32 @@ class GilCheck(VisitorTransform): return node +class CoerceCppTemps(EnvTransform, SkipDeclarations): + """ + For temporary expression that are implemented using std::optional it's necessary the temps are + assigned using `__pyx_t_x = value;` but accessed using `something = (*__pyx_t_x)`. This transform + inserts a coercion node to take care of this, and runs absolutely last (once nothing else can be + inserted into the tree) + + TODO: a possible alternative would be to split ExprNode.result() into ExprNode.rhs_rhs() and ExprNode.lhs_rhs()??? + """ + def visit_ModuleNode(self, node): + if self.current_env().cpp: + # skipping this makes it essentially free for C files + self.visitchildren(node) + return node + + def visit_ExprNode(self, node): + self.visitchildren(node) + if (self.current_env().directives['cpp_locals'] and + node.is_temp and node.type.is_cpp_class and + # Fake references are not replaced with "std::optional()". + not node.type.is_fake_reference): + node = ExprNodes.CppOptionalTempCoercion(node) + + return node + + class TransformBuiltinMethods(EnvTransform): """ Replace Cython's own cython.* builtins by the corresponding tree nodes. @@ -2996,9 +3206,7 @@ class TransformBuiltinMethods(EnvTransform): def visit_cython_attribute(self, node): attribute = node.as_cython_attribute() if attribute: - if attribute == u'compiled': - node = ExprNodes.BoolNode(node.pos, value=True) - elif attribute == u'__version__': + if attribute == u'__version__': from .. import __version__ as version node = ExprNodes.StringNode(node.pos, value=EncodedString(version)) elif attribute == u'NULL': @@ -3043,9 +3251,9 @@ class TransformBuiltinMethods(EnvTransform): error(self.pos, "Builtin 'vars()' called with wrong number of args, expected 0-1, got %d" % len(node.args)) if len(node.args) > 0: - return node # nothing to do + return node # nothing to do return ExprNodes.LocalsExprNode(pos, self.current_scope_node(), lenv) - else: # dir() + else: # dir() if len(node.args) > 1: error(self.pos, "Builtin 'dir()' called with wrong number of args, expected 0-1, got %d" % len(node.args)) @@ -3080,8 +3288,8 @@ class TransformBuiltinMethods(EnvTransform): def _inject_eval(self, node, func_name): lenv = self.current_env() - entry = lenv.lookup_here(func_name) - if entry or len(node.args) != 1: + entry = lenv.lookup(func_name) + if len(node.args) != 1 or (entry and not entry.is_builtin): return node # Inject globals and locals node.args.append(ExprNodes.GlobalsExprNode(node.pos)) @@ -3098,8 +3306,7 @@ class TransformBuiltinMethods(EnvTransform): return node # Inject no-args super def_node = self.current_scope_node() - if (not isinstance(def_node, Nodes.DefNode) or not def_node.args or - len(self.env_stack) < 2): + if not isinstance(def_node, Nodes.DefNode) or not def_node.args or len(self.env_stack) < 2: return node class_node, class_scope = self.env_stack[-2] if class_scope.is_py_class_scope: @@ -3238,10 +3445,17 @@ class ReplaceFusedTypeChecks(VisitorTransform): self.visitchildren(node) return self.transform(node) + def visit_GILStatNode(self, node): + """ + Fold constant condition of GILStatNode. + """ + self.visitchildren(node) + return self.transform(node) + def visit_PrimaryCmpNode(self, node): with Errors.local_errors(ignore=True): - type1 = node.operand1.analyse_as_type(self.local_scope) - type2 = node.operand2.analyse_as_type(self.local_scope) + type1 = node.operand1.analyse_as_type(self.local_scope) + type2 = node.operand2.analyse_as_type(self.local_scope) if type1 and type2: false_node = ExprNodes.BoolNode(node.pos, value=False) @@ -3377,9 +3591,14 @@ class DebugTransform(CythonTransform): else: pf_cname = node.py_func.entry.func_cname + # For functions defined using def, cname will be pyfunc_cname=__pyx_pf_* + # For functions defined using cpdef or cdef, cname will be func_cname=__pyx_f_* + # In all cases, cname will be the name of the function containing the actual code + cname = node.entry.pyfunc_cname or node.entry.func_cname + attrs = dict( name=node.entry.name or getattr(node, 'name', '<unknown>'), - cname=node.entry.func_cname, + cname=cname, pf_cname=pf_cname, qualified_name=node.local_scope.qualified_name, lineno=str(node.pos[1])) @@ -3407,10 +3626,10 @@ class DebugTransform(CythonTransform): def visit_NameNode(self, node): if (self.register_stepinto and - node.type is not None and - node.type.is_cfunction and - getattr(node, 'is_called', False) and - node.entry.func_cname is not None): + node.type is not None and + node.type.is_cfunction and + getattr(node, 'is_called', False) and + node.entry.func_cname is not None): # don't check node.entry.in_cinclude, as 'cdef extern: ...' # declared functions are not 'in_cinclude'. # This means we will list called 'cdef' functions as @@ -3429,26 +3648,16 @@ class DebugTransform(CythonTransform): it's a "relevant frame" and it will know where to set the breakpoint for 'break modulename'. """ - name = node.full_module_name.rpartition('.')[-1] - - cname_py2 = 'init' + name - cname_py3 = 'PyInit_' + name - - py2_attrs = dict( - name=name, - cname=cname_py2, + self._serialize_modulenode_as_function(node, dict( + name=node.full_module_name.rpartition('.')[-1], + cname=node.module_init_func_cname(), pf_cname='', # Ignore the qualified_name, breakpoints should be set using # `cy break modulename:lineno` for module-level breakpoints. qualified_name='', lineno='1', is_initmodule_function="True", - ) - - py3_attrs = dict(py2_attrs, cname=cname_py3) - - self._serialize_modulenode_as_function(node, py2_attrs) - self._serialize_modulenode_as_function(node, py3_attrs) + )) def _serialize_modulenode_as_function(self, node, attrs): self.tb.start('Function', attrs=attrs) diff --git a/Cython/Compiler/Parsing.pxd b/Cython/Compiler/Parsing.pxd index 25453b39a..d6c96a3d3 100644 --- a/Cython/Compiler/Parsing.pxd +++ b/Cython/Compiler/Parsing.pxd @@ -1,3 +1,5 @@ +# cython: language_level=3 + # We declare all of these here to type the first argument. from __future__ import absolute_import @@ -24,7 +26,7 @@ cdef p_lambdef_nocond(PyrexScanner s) cdef p_test(PyrexScanner s) cdef p_test_nocond(PyrexScanner s) cdef p_or_test(PyrexScanner s) -cdef p_rassoc_binop_expr(PyrexScanner s, ops, p_sub_expr_func p_subexpr) +cdef p_rassoc_binop_expr(PyrexScanner s, unicode op, p_sub_expr_func p_subexpr) cdef p_and_test(PyrexScanner s) cdef p_not_test(PyrexScanner s) cdef p_comparison(PyrexScanner s) @@ -139,10 +141,10 @@ cdef tuple p_suite_with_docstring(PyrexScanner s, ctx, bint with_doc_only=*) cdef tuple _extract_docstring(node) cdef p_positional_and_keyword_args(PyrexScanner s, end_sy_set, templates = *) -cpdef p_c_base_type(PyrexScanner s, bint self_flag = *, bint nonempty = *, templates = *) +cpdef p_c_base_type(PyrexScanner s, bint nonempty = *, templates = *) cdef p_calling_convention(PyrexScanner s) cdef p_c_complex_base_type(PyrexScanner s, templates = *) -cdef p_c_simple_base_type(PyrexScanner s, bint self_flag, bint nonempty, templates = *) +cdef p_c_simple_base_type(PyrexScanner s, bint nonempty, templates = *) cdef p_buffer_or_template(PyrexScanner s, base_type_node, templates) cdef p_bracketed_base_type(PyrexScanner s, base_type_node, nonempty, empty) cdef is_memoryviewslice_access(PyrexScanner s) @@ -197,3 +199,4 @@ cdef dict p_compiler_directive_comments(PyrexScanner s) cdef p_template_definition(PyrexScanner s) cdef p_cpp_class_definition(PyrexScanner s, pos, ctx) cdef p_cpp_class_attribute(PyrexScanner s, ctx) +cdef p_annotation(PyrexScanner s) diff --git a/Cython/Compiler/Parsing.py b/Cython/Compiler/Parsing.py index 82bf82d3d..b2dbc0c22 100644 --- a/Cython/Compiler/Parsing.py +++ b/Cython/Compiler/Parsing.py @@ -14,7 +14,7 @@ cython.declare(Nodes=object, ExprNodes=object, EncodedString=object, Builtin=object, ModuleNode=object, Utils=object, _unicode=object, _bytes=object, re=object, sys=object, _parse_escape_sequences=object, _parse_escape_sequences_raw=object, partial=object, reduce=object, _IS_PY3=cython.bint, _IS_2BYTE_UNICODE=cython.bint, - _CDEF_MODIFIERS=tuple) + _CDEF_MODIFIERS=tuple, COMMON_BINOP_MISTAKES=dict) from io import StringIO import re @@ -65,7 +65,7 @@ class Ctx(object): def p_ident(s, message="Expected an identifier"): if s.sy == 'IDENT': - name = s.systring + name = s.context.intern_ustring(s.systring) s.next() return name else: @@ -74,7 +74,7 @@ def p_ident(s, message="Expected an identifier"): def p_ident_list(s): names = [] while s.sy == 'IDENT': - names.append(s.systring) + names.append(s.context.intern_ustring(s.systring)) s.next() if s.sy != ',': break @@ -103,7 +103,7 @@ def p_binop_expr(s, ops, p_sub_expr): if Future.division in s.context.future_directives: n1.truedivision = True else: - n1.truedivision = None # unknown + n1.truedivision = None # unknown return n1 #lambdef: 'lambda' [varargslist] ':' test @@ -159,24 +159,31 @@ def p_test_nocond(s): #or_test: and_test ('or' and_test)* +COMMON_BINOP_MISTAKES = {'||': 'or', '&&': 'and'} + def p_or_test(s): - return p_rassoc_binop_expr(s, ('or',), p_and_test) + return p_rassoc_binop_expr(s, u'or', p_and_test) -def p_rassoc_binop_expr(s, ops, p_subexpr): +def p_rassoc_binop_expr(s, op, p_subexpr): n1 = p_subexpr(s) - if s.sy in ops: + if s.sy == op: pos = s.position() op = s.sy s.next() - n2 = p_rassoc_binop_expr(s, ops, p_subexpr) + n2 = p_rassoc_binop_expr(s, op, p_subexpr) n1 = ExprNodes.binop_node(pos, op, n1, n2) + elif s.sy in COMMON_BINOP_MISTAKES and COMMON_BINOP_MISTAKES[s.sy] == op: + # Only report this for the current operator since we pass through here twice for 'and' and 'or'. + warning(s.position(), + "Found the C operator '%s', did you mean the Python operator '%s'?" % (s.sy, op), + level=1) return n1 #and_test: not_test ('and' not_test)* def p_and_test(s): #return p_binop_expr(s, ('and',), p_not_test) - return p_rassoc_binop_expr(s, ('and',), p_not_test) + return p_rassoc_binop_expr(s, u'and', p_not_test) #not_test: 'not' not_test | comparison @@ -250,10 +257,10 @@ def p_cmp_op(s): op = '!=' return op -comparison_ops = cython.declare(set, set([ +comparison_ops = cython.declare(frozenset, frozenset(( '<', '>', '==', '>=', '<=', '<>', '!=', 'in', 'is', 'not' -])) +))) #expr: xor_expr ('|' xor_expr)* @@ -316,10 +323,12 @@ def p_typecast(s): s.next() base_type = p_c_base_type(s) is_memslice = isinstance(base_type, Nodes.MemoryViewSliceTypeNode) - is_template = isinstance(base_type, Nodes.TemplatedTypeNode) - is_const = isinstance(base_type, Nodes.CConstTypeNode) - if (not is_memslice and not is_template and not is_const - and base_type.name is None): + is_other_unnamed_type = isinstance(base_type, ( + Nodes.TemplatedTypeNode, + Nodes.CConstOrVolatileTypeNode, + Nodes.CTupleBaseTypeNode, + )) + if not (is_memslice or is_other_unnamed_type) and base_type.name is None: s.error("Unknown type") declarator = p_c_declarator(s, empty = 1) if s.sy == '?': @@ -330,8 +339,7 @@ def p_typecast(s): s.expect(">") operand = p_factor(s) if is_memslice: - return ExprNodes.CythonArrayNode(pos, base_type_node=base_type, - operand=operand) + return ExprNodes.CythonArrayNode(pos, base_type_node=base_type, operand=operand) return ExprNodes.TypecastNode(pos, base_type = base_type, @@ -444,7 +452,7 @@ def p_trailer(s, node1): return p_call(s, node1) elif s.sy == '[': return p_index(s, node1) - else: # s.sy == '.' + else: # s.sy == '.' s.next() name = p_ident(s) return ExprNodes.AttributeNode(pos, @@ -821,7 +829,7 @@ def p_cat_string_literal(s): continue elif next_kind != kind: # concatenating f strings and normal strings is allowed and leads to an f string - if set([kind, next_kind]) in (set(['f', 'u']), set(['f', ''])): + if {kind, next_kind} in ({'f', 'u'}, {'f', ''}): kind = 'f' else: error(pos, "Cannot mix string literals of different types, expected %s'', got %s''" % ( @@ -1073,8 +1081,8 @@ def p_f_string(s, unicode_value, pos, is_raw): if builder.chars: values.append(ExprNodes.UnicodeNode(pos, value=builder.getstring())) builder = StringEncoding.UnicodeLiteralBuilder() - next_start, expr_node = p_f_string_expr(s, unicode_value, pos, next_start, is_raw) - values.append(expr_node) + next_start, expr_nodes = p_f_string_expr(s, unicode_value, pos, next_start, is_raw) + values.extend(expr_nodes) elif c == '}': if part == '}}': builder.append('}') @@ -1090,12 +1098,16 @@ def p_f_string(s, unicode_value, pos, is_raw): def p_f_string_expr(s, unicode_value, pos, starting_index, is_raw): - # Parses a {}-delimited expression inside an f-string. Returns a FormattedValueNode - # and the index in the string that follows the expression. + # Parses a {}-delimited expression inside an f-string. Returns a list of nodes + # [UnicodeNode?, FormattedValueNode] and the index in the string that follows + # the expression. + # + # ? = Optional i = starting_index size = len(unicode_value) conversion_char = terminal_char = format_spec = None format_spec_str = None + expr_text = None NO_CHAR = 2**30 nested_depth = 0 @@ -1135,12 +1147,15 @@ def p_f_string_expr(s, unicode_value, pos, starting_index, is_raw): elif c == '#': error(_f_string_error_pos(pos, unicode_value, i), "format string cannot include #") - elif nested_depth == 0 and c in '!:}': - # allow != as a special case - if c == '!' and i + 1 < size and unicode_value[i + 1] == '=': - i += 1 - continue - + elif nested_depth == 0 and c in '><=!:}': + # allow special cases with '!' and '=' + if i + 1 < size and c in '!=><': + if unicode_value[i + 1] == '=': + i += 2 # we checked 2, so we can skip 2: '!=', '==', '>=', '<=' + continue + elif c in '><': # allow single '<' and '>' + i += 1 + continue terminal_char = c break i += 1 @@ -1153,6 +1168,16 @@ def p_f_string_expr(s, unicode_value, pos, starting_index, is_raw): error(_f_string_error_pos(pos, unicode_value, starting_index), "empty expression not allowed in f-string") + if terminal_char == '=': + i += 1 + while i < size and unicode_value[i].isspace(): + i += 1 + + if i < size: + terminal_char = unicode_value[i] + expr_text = unicode_value[starting_index:i] + # otherwise: error will be reported below + if terminal_char == '!': i += 1 if i + 2 > size: @@ -1190,6 +1215,9 @@ def p_f_string_expr(s, unicode_value, pos, starting_index, is_raw): format_spec_str = unicode_value[start_format_spec:i] + if expr_text and conversion_char is None and format_spec_str is None: + conversion_char = 'r' + if terminal_char != '}': error(_f_string_error_pos(pos, unicode_value, i), "missing '}' in format string expression" + ( @@ -1208,8 +1236,12 @@ def p_f_string_expr(s, unicode_value, pos, starting_index, is_raw): if format_spec_str: format_spec = ExprNodes.JoinedStrNode(pos, values=p_f_string(s, format_spec_str, pos, is_raw)) - return i + 1, ExprNodes.FormattedValueNode( - pos, value=expr, conversion_char=conversion_char, format_spec=format_spec) + nodes = [] + if expr_text: + nodes.append(ExprNodes.UnicodeNode(pos, value=StringEncoding.EncodedString(expr_text))) + nodes.append(ExprNodes.FormattedValueNode(pos, value=expr, conversion_char=conversion_char, format_spec=format_spec)) + + return i + 1, nodes # since PEP 448: @@ -1478,8 +1510,8 @@ def p_genexp(s, expr): expr.pos, expr = ExprNodes.YieldExprNode(expr.pos, arg=expr))) return ExprNodes.GeneratorExpressionNode(expr.pos, loop=loop) -expr_terminators = cython.declare(set, set([ - ')', ']', '}', ':', '=', 'NEWLINE'])) +expr_terminators = cython.declare(frozenset, frozenset(( + ')', ']', '}', ':', '=', 'NEWLINE'))) #------------------------------------------------------- @@ -1505,15 +1537,19 @@ def p_nonlocal_statement(s): def p_expression_or_assignment(s): expr = p_testlist_star_expr(s) + has_annotation = False if s.sy == ':' and (expr.is_name or expr.is_subscript or expr.is_attribute): + has_annotation = True s.next() - expr.annotation = p_test(s) + expr.annotation = p_annotation(s) + if s.sy == '=' and expr.is_starred: # This is a common enough error to make when learning Cython to let # it fail as early as possible and give a very clear error message. s.error("a starred assignment target must be in a list or tuple" " - maybe you meant to use an index assignment: var[0] = ...", pos=expr.pos) + expr_list = [expr] while s.sy == '=': s.next() @@ -1545,7 +1581,7 @@ def p_expression_or_assignment(s): rhs = expr_list[-1] if len(expr_list) == 2: - return Nodes.SingleAssignmentNode(rhs.pos, lhs=expr_list[0], rhs=rhs) + return Nodes.SingleAssignmentNode(rhs.pos, lhs=expr_list[0], rhs=rhs, first=has_annotation) else: return Nodes.CascadedAssignmentNode(rhs.pos, lhs_list=expr_list[:-1], rhs=rhs) @@ -1690,11 +1726,6 @@ def p_import_statement(s): as_name=as_name, is_absolute=is_absolute) else: - if as_name and "." in dotted_name: - name_list = ExprNodes.ListNode(pos, args=[ - ExprNodes.IdentifierStringNode(pos, value=s.context.intern_ustring("*"))]) - else: - name_list = None stat = Nodes.SingleAssignmentNode( pos, lhs=ExprNodes.NameNode(pos, name=as_name or target_name), @@ -1702,7 +1733,8 @@ def p_import_statement(s): pos, module_name=ExprNodes.IdentifierStringNode(pos, value=dotted_name), level=0 if is_absolute else None, - name_list=name_list)) + get_top_level_module='.' in dotted_name and as_name is None, + name_list=None)) stats.append(stat) return Nodes.StatListNode(pos, stats=stats) @@ -1788,7 +1820,8 @@ def p_from_import_statement(s, first_statement = 0): items = items) -imported_name_kinds = cython.declare(set, set(['class', 'struct', 'union'])) +imported_name_kinds = cython.declare(frozenset, frozenset(( + 'class', 'struct', 'union'))) def p_imported_name(s, is_cimport): pos = s.position() @@ -1832,10 +1865,11 @@ def p_assert_statement(s): value = p_test(s) else: value = None - return Nodes.AssertStatNode(pos, cond = cond, value = value) + return Nodes.AssertStatNode(pos, condition=cond, value=value) -statement_terminators = cython.declare(set, set([';', 'NEWLINE', 'EOF'])) +statement_terminators = cython.declare(frozenset, frozenset(( + ';', 'NEWLINE', 'EOF'))) def p_if_statement(s): # s.sy == 'if' @@ -1945,7 +1979,8 @@ def p_for_from_step(s): else: return None -inequality_relations = cython.declare(set, set(['<', '<=', '>', '>='])) +inequality_relations = cython.declare(frozenset, frozenset(( + '<', '<=', '>', '>='))) def p_target(s, terminator): pos = s.position() @@ -2035,7 +2070,7 @@ def p_except_clause(s): def p_include_statement(s, ctx): pos = s.position() - s.next() # 'include' + s.next() # 'include' unicode_include_file_name = p_string_literal(s, 'u')[2] s.expect_newline("Syntax error in include statement") if s.compile_time_eval: @@ -2070,12 +2105,20 @@ def p_with_items(s, is_async=False): s.error("with gil/nogil cannot be async") state = s.systring s.next() + + # support conditional gil/nogil + condition = None + if s.sy == '(': + s.next() + condition = p_test(s) + s.expect(')') + if s.sy == ',': s.next() body = p_with_items(s) else: body = p_suite(s) - return Nodes.GILStatNode(pos, state=state, body=body) + return Nodes.GILStatNode(pos, state=state, body=body, condition=condition) else: manager = p_test(s) target = None @@ -2193,7 +2236,7 @@ def p_compile_time_expr(s): def p_DEF_statement(s): pos = s.position() denv = s.compile_time_env - s.next() # 'DEF' + s.next() # 'DEF' name = p_ident(s) s.expect('=') expr = p_compile_time_expr(s) @@ -2211,7 +2254,7 @@ def p_IF_statement(s, ctx): denv = s.compile_time_env result = None while 1: - s.next() # 'IF' or 'ELIF' + s.next() # 'IF' or 'ELIF' expr = p_compile_time_expr(s) s.compile_time_eval = current_eval and bool(expr.compile_time_value(denv)) body = p_suite(s, ctx) @@ -2329,7 +2372,7 @@ def p_statement(s, ctx, first_statement = 0): return p_async_statement(s, ctx, decorators) elif decorators: s.error("Decorators can only be followed by functions or classes") - s.put_back('IDENT', ident_name) # re-insert original token + s.put_back(u'IDENT', ident_name) # re-insert original token return p_simple_statement_list(s, ctx, first_statement=first_statement) @@ -2397,7 +2440,7 @@ def p_positional_and_keyword_args(s, end_sy_set, templates = None): parsed_type = False if s.sy == 'IDENT' and s.peek()[0] == '=': ident = s.systring - s.next() # s.sy is '=' + s.next() # s.sy is '=' s.next() if looking_at_expr(s): arg = p_test(s) @@ -2434,13 +2477,11 @@ def p_positional_and_keyword_args(s, end_sy_set, templates = None): s.next() return positional_args, keyword_args -def p_c_base_type(s, self_flag = 0, nonempty = 0, templates = None): - # If self_flag is true, this is the base type for the - # self argument of a C method of an extension type. +def p_c_base_type(s, nonempty=False, templates=None): if s.sy == '(': return p_c_complex_base_type(s, templates = templates) else: - return p_c_simple_base_type(s, self_flag, nonempty = nonempty, templates = templates) + return p_c_simple_base_type(s, nonempty=nonempty, templates=templates) def p_calling_convention(s): if s.sy == 'IDENT' and s.systring in calling_convention_words: @@ -2451,8 +2492,8 @@ def p_calling_convention(s): return "" -calling_convention_words = cython.declare( - set, set(["__stdcall", "__cdecl", "__fastcall"])) +calling_convention_words = cython.declare(frozenset, frozenset(( + "__stdcall", "__cdecl", "__fastcall"))) def p_c_complex_base_type(s, templates = None): @@ -2484,24 +2525,38 @@ def p_c_complex_base_type(s, templates = None): return type_node -def p_c_simple_base_type(s, self_flag, nonempty, templates = None): - #print "p_c_simple_base_type: self_flag =", self_flag, nonempty +def p_c_simple_base_type(s, nonempty, templates=None): is_basic = 0 signed = 1 longness = 0 complex = 0 module_path = [] pos = s.position() - if not s.sy == 'IDENT': - error(pos, "Expected an identifier, found '%s'" % s.sy) - if s.systring == 'const': + + # Handle const/volatile + is_const = is_volatile = 0 + while s.sy == 'IDENT': + if s.systring == 'const': + if is_const: error(pos, "Duplicate 'const'") + is_const = 1 + elif s.systring == 'volatile': + if is_volatile: error(pos, "Duplicate 'volatile'") + is_volatile = 1 + else: + break s.next() - base_type = p_c_base_type(s, self_flag=self_flag, nonempty=nonempty, templates=templates) + if is_const or is_volatile: + base_type = p_c_base_type(s, nonempty=nonempty, templates=templates) if isinstance(base_type, Nodes.MemoryViewSliceTypeNode): # reverse order to avoid having to write "(const int)[:]" - base_type.base_type_node = Nodes.CConstTypeNode(pos, base_type=base_type.base_type_node) + base_type.base_type_node = Nodes.CConstOrVolatileTypeNode(pos, + base_type=base_type.base_type_node, is_const=is_const, is_volatile=is_volatile) return base_type - return Nodes.CConstTypeNode(pos, base_type=base_type) + return Nodes.CConstOrVolatileTypeNode(pos, + base_type=base_type, is_const=is_const, is_volatile=is_volatile) + + if s.sy != 'IDENT': + error(pos, "Expected an identifier, found '%s'" % s.sy) if looking_at_base_type(s): #print "p_c_simple_base_type: looking_at_base_type at", s.position() is_basic = 1 @@ -2536,20 +2591,20 @@ def p_c_simple_base_type(s, self_flag, nonempty, templates = None): s.next() if (s.sy == '*' or s.sy == '**' or s.sy == '&' or (s.sy == 'IDENT' and s.systring in calling_convention_words)): - s.put_back('(', '(') + s.put_back(u'(', u'(') else: - s.put_back('(', '(') - s.put_back('IDENT', name) + s.put_back(u'(', u'(') + s.put_back(u'IDENT', name) name = None elif s.sy not in ('*', '**', '[', '&'): - s.put_back('IDENT', name) + s.put_back(u'IDENT', name) name = None type_node = Nodes.CSimpleBaseTypeNode(pos, name = name, module_path = module_path, is_basic_c_type = is_basic, signed = signed, complex = complex, longness = longness, - is_self_arg = self_flag, templates = templates) + templates = templates) # declarations here. if s.sy == '[': @@ -2649,7 +2704,7 @@ def p_memoryviewslice_access(s, base_type_node): return result def looking_at_name(s): - return s.sy == 'IDENT' and not s.systring in calling_convention_words + return s.sy == 'IDENT' and s.systring not in calling_convention_words def looking_at_expr(s): if s.systring in base_type_start_words: @@ -2683,10 +2738,10 @@ def looking_at_expr(s): dotted_path.reverse() for p in dotted_path: - s.put_back('IDENT', p) - s.put_back('.', '.') + s.put_back(u'IDENT', p) + s.put_back(u'.', u'.') - s.put_back('IDENT', name) + s.put_back(u'IDENT', name) return not is_type and saved[0] else: return True @@ -2700,7 +2755,7 @@ def looking_at_dotted_name(s): name = s.systring s.next() result = s.sy == '.' - s.put_back('IDENT', name) + s.put_back(u'IDENT', name) return result else: return 0 @@ -2716,8 +2771,8 @@ def looking_at_call(s): s.start_line, s.start_col = position return result -basic_c_type_names = cython.declare( - set, set(["void", "char", "int", "float", "double", "bint"])) +basic_c_type_names = cython.declare(frozenset, frozenset(( + "void", "char", "int", "float", "double", "bint"))) special_basic_c_types = cython.declare(dict, { # name : (signed, longness) @@ -2731,17 +2786,17 @@ special_basic_c_types = cython.declare(dict, { "Py_tss_t" : (1, 0), }) -sign_and_longness_words = cython.declare( - set, set(["short", "long", "signed", "unsigned"])) +sign_and_longness_words = cython.declare(frozenset, frozenset(( + "short", "long", "signed", "unsigned"))) base_type_start_words = cython.declare( - set, + frozenset, basic_c_type_names | sign_and_longness_words - | set(special_basic_c_types)) + | frozenset(special_basic_c_types)) -struct_enum_union = cython.declare( - set, set(["struct", "union", "enum", "packed"])) +struct_enum_union = cython.declare(frozenset, frozenset(( + "struct", "union", "enum", "packed"))) def p_sign_and_longness(s): signed = 1 @@ -2796,7 +2851,7 @@ def p_c_declarator(s, ctx = Ctx(), empty = 0, is_type = 0, cmethod_flag = 0, pos = s.position() if s.sy == '[': result = p_c_array_declarator(s, result) - else: # sy == '(' + else: # sy == '(' s.next() result = p_c_func_declarator(s, pos, ctx, result, cmethod_flag) cmethod_flag = 0 @@ -2804,7 +2859,7 @@ def p_c_declarator(s, ctx = Ctx(), empty = 0, is_type = 0, cmethod_flag = 0, def p_c_array_declarator(s, base): pos = s.position() - s.next() # '[' + s.next() # '[' if s.sy != ']': dim = p_testlist(s) else: @@ -2813,7 +2868,7 @@ def p_c_array_declarator(s, base): return Nodes.CArrayDeclaratorNode(pos, base = base, dimension = dim) def p_c_func_declarator(s, pos, ctx, base, cmethod_flag): - # Opening paren has already been skipped + # Opening paren has already been skipped args = p_c_arg_list(s, ctx, cmethod_flag = cmethod_flag, nonempty_declarators = 0) ellipsis = p_optional_ellipsis(s) @@ -2826,49 +2881,43 @@ def p_c_func_declarator(s, pos, ctx, base, cmethod_flag): exception_value = exc_val, exception_check = exc_check, nogil = nogil or ctx.nogil or with_gil, with_gil = with_gil) -supported_overloaded_operators = cython.declare(set, set([ +supported_overloaded_operators = cython.declare(frozenset, frozenset(( '+', '-', '*', '/', '%', '++', '--', '~', '|', '&', '^', '<<', '>>', ',', '==', '!=', '>=', '>', '<=', '<', '[]', '()', '!', '=', 'bool', -])) +))) def p_c_simple_declarator(s, ctx, empty, is_type, cmethod_flag, assignable, nonempty): pos = s.position() calling_convention = p_calling_convention(s) - if s.sy == '*': + if s.sy in ('*', '**'): + # scanner returns '**' as a single token + is_ptrptr = s.sy == '**' s.next() - if s.systring == 'const': - const_pos = s.position() + + const_pos = s.position() + is_const = s.systring == 'const' and s.sy == 'IDENT' + if is_const: s.next() - const_base = p_c_declarator(s, ctx, empty = empty, - is_type = is_type, - cmethod_flag = cmethod_flag, - assignable = assignable, - nonempty = nonempty) - base = Nodes.CConstDeclaratorNode(const_pos, base = const_base) - else: - base = p_c_declarator(s, ctx, empty = empty, is_type = is_type, - cmethod_flag = cmethod_flag, - assignable = assignable, nonempty = nonempty) - result = Nodes.CPtrDeclaratorNode(pos, - base = base) - elif s.sy == '**': # scanner returns this as a single token - s.next() - base = p_c_declarator(s, ctx, empty = empty, is_type = is_type, - cmethod_flag = cmethod_flag, - assignable = assignable, nonempty = nonempty) - result = Nodes.CPtrDeclaratorNode(pos, - base = Nodes.CPtrDeclaratorNode(pos, - base = base)) - elif s.sy == '&': - s.next() - base = p_c_declarator(s, ctx, empty = empty, is_type = is_type, - cmethod_flag = cmethod_flag, - assignable = assignable, nonempty = nonempty) - result = Nodes.CReferenceDeclaratorNode(pos, base = base) + + base = p_c_declarator(s, ctx, empty=empty, is_type=is_type, + cmethod_flag=cmethod_flag, + assignable=assignable, nonempty=nonempty) + if is_const: + base = Nodes.CConstDeclaratorNode(const_pos, base=base) + if is_ptrptr: + base = Nodes.CPtrDeclaratorNode(pos, base=base) + result = Nodes.CPtrDeclaratorNode(pos, base=base) + elif s.sy == '&' or (s.sy == '&&' and s.context.cpp): + node_class = Nodes.CppRvalueReferenceDeclaratorNode if s.sy == '&&' else Nodes.CReferenceDeclaratorNode + s.next() + base = p_c_declarator(s, ctx, empty=empty, is_type=is_type, + cmethod_flag=cmethod_flag, + assignable=assignable, nonempty=nonempty) + result = node_class(pos, base=base) else: rhs = None if s.sy == 'IDENT': @@ -2909,7 +2958,7 @@ def p_c_simple_declarator(s, ctx, empty, is_type, cmethod_flag, fatal=False) name += op elif op == 'IDENT': - op = s.systring; + op = s.systring if op not in supported_overloaded_operators: s.error("Overloading operator '%s' not yet supported." % op, fatal=False) @@ -2960,7 +3009,8 @@ def p_exception_value_clause(s): exc_val = p_test(s) return exc_val, exc_check -c_arg_list_terminators = cython.declare(set, set(['*', '**', '.', ')', ':'])) +c_arg_list_terminators = cython.declare(frozenset, frozenset(( + '*', '**', '.', ')', ':', '/'))) def p_c_arg_list(s, ctx = Ctx(), in_pyfunc = 0, cmethod_flag = 0, nonempty_declarators = 0, kw_only = 0, annotated = 1): @@ -2999,7 +3049,7 @@ def p_c_arg_decl(s, ctx, in_pyfunc, cmethod_flag = 0, nonempty = 0, complex = 0, longness = 0, is_self_arg = cmethod_flag, templates = None) else: - base_type = p_c_base_type(s, cmethod_flag, nonempty = nonempty) + base_type = p_c_base_type(s, nonempty=nonempty) declarator = p_c_declarator(s, ctx, nonempty = nonempty) if s.sy in ('not', 'or') and not s.in_python_file: kind = s.sy @@ -3014,7 +3064,7 @@ def p_c_arg_decl(s, ctx, in_pyfunc, cmethod_flag = 0, nonempty = 0, not_none = kind == 'not' if annotated and s.sy == ':': s.next() - annotation = p_test(s) + annotation = p_annotation(s) if s.sy == '=': s.next() if 'pxd' in ctx.level: @@ -3116,6 +3166,12 @@ def p_cdef_extern_block(s, pos, ctx): def p_c_enum_definition(s, pos, ctx): # s.sy == ident 'enum' s.next() + + scoped = False + if s.context.cpp and (s.sy == 'class' or (s.sy == 'IDENT' and s.systring == 'struct')): + scoped = True + s.next() + if s.sy == 'IDENT': name = s.systring s.next() @@ -3123,24 +3179,51 @@ def p_c_enum_definition(s, pos, ctx): if cname is None and ctx.namespace is not None: cname = ctx.namespace + "::" + name else: - name = None - cname = None - items = None + name = cname = None + if scoped: + s.error("Unnamed scoped enum not allowed") + + if scoped and s.sy == '(': + s.next() + underlying_type = p_c_base_type(s) + s.expect(')') + else: + underlying_type = Nodes.CSimpleBaseTypeNode( + pos, + name="int", + module_path = [], + is_basic_c_type = True, + signed = 1, + complex = 0, + longness = 0 + ) + s.expect(':') items = [] + + doc = None if s.sy != 'NEWLINE': p_c_enum_line(s, ctx, items) else: - s.next() # 'NEWLINE' + s.next() # 'NEWLINE' s.expect_indent() + doc = p_doc_string(s) + while s.sy not in ('DEDENT', 'EOF'): p_c_enum_line(s, ctx, items) + s.expect_dedent() + + if not items and ctx.visibility != "extern": + error(pos, "Empty enum definition not allowed outside a 'cdef extern from' block") + return Nodes.CEnumDefNode( - pos, name = name, cname = cname, items = items, - typedef_flag = ctx.typedef_flag, visibility = ctx.visibility, - create_wrapper = ctx.overridable, - api = ctx.api, in_pxd = ctx.level == 'module_pxd') + pos, name=name, cname=cname, + scoped=scoped, items=items, + underlying_type=underlying_type, + typedef_flag=ctx.typedef_flag, visibility=ctx.visibility, + create_wrapper=ctx.overridable, + api=ctx.api, in_pxd=ctx.level == 'module_pxd', doc=doc) def p_c_enum_line(s, ctx, items): if s.sy != 'pass': @@ -3184,20 +3267,28 @@ def p_c_struct_or_union_definition(s, pos, ctx): attributes = None if s.sy == ':': s.next() - s.expect('NEWLINE') - s.expect_indent() attributes = [] - body_ctx = Ctx() - while s.sy != 'DEDENT': - if s.sy != 'pass': - attributes.append( - p_c_func_or_var_declaration(s, s.position(), body_ctx)) - else: - s.next() - s.expect_newline("Expected a newline") - s.expect_dedent() + if s.sy == 'pass': + s.next() + s.expect_newline("Expected a newline", ignore_semicolon=True) + else: + s.expect('NEWLINE') + s.expect_indent() + body_ctx = Ctx() + while s.sy != 'DEDENT': + if s.sy != 'pass': + attributes.append( + p_c_func_or_var_declaration(s, s.position(), body_ctx)) + else: + s.next() + s.expect_newline("Expected a newline") + s.expect_dedent() + + if not attributes and ctx.visibility != "extern": + error(pos, "Empty struct or union definition not allowed outside a 'cdef extern from' block") else: s.expect_newline("Syntax error in struct or union definition") + return Nodes.CStructOrUnionDefNode(pos, name = name, cname = cname, kind = kind, attributes = attributes, typedef_flag = ctx.typedef_flag, visibility = ctx.visibility, @@ -3224,7 +3315,7 @@ def p_fused_definition(s, pos, ctx): while s.sy != 'DEDENT': if s.sy != 'pass': #types.append(p_c_declarator(s)) - types.append(p_c_base_type(s)) #, nonempty=1)) + types.append(p_c_base_type(s)) #, nonempty=1)) else: s.next() @@ -3380,7 +3471,7 @@ def _reject_cdef_modifier_in_py(s, name): def p_def_statement(s, decorators=None, is_async_def=False): # s.sy == 'def' - pos = s.position() + pos = decorators[0].pos if decorators else s.position() # PEP 492 switches the async/await keywords on in "async def" functions if is_async_def: s.enter_async() @@ -3397,7 +3488,7 @@ def p_def_statement(s, decorators=None, is_async_def=False): return_type_annotation = None if s.sy == '->': s.next() - return_type_annotation = p_test(s) + return_type_annotation = p_annotation(s) _reject_cdef_modifier_in_py(s, s.systring) doc, body = p_suite_with_docstring(s, Ctx(level='function')) @@ -3415,6 +3506,20 @@ def p_varargslist(s, terminator=')', annotated=1): annotated = annotated) star_arg = None starstar_arg = None + if s.sy == '/': + if len(args) == 0: + s.error("Got zero positional-only arguments despite presence of " + "positional-only specifier '/'") + s.next() + # Mark all args to the left as pos only + for arg in args: + arg.pos_only = 1 + if s.sy == ',': + s.next() + args.extend(p_c_arg_list(s, in_pyfunc = 1, + nonempty_declarators = 1, annotated = annotated)) + elif s.sy != terminator: + s.error("Syntax error in Python function argument list") if s.sy == '*': s.next() if s.sy == 'IDENT': @@ -3438,7 +3543,7 @@ def p_py_arg_decl(s, annotated = 1): annotation = None if annotated and s.sy == ':': s.next() - annotation = p_test(s) + annotation = p_annotation(s) return Nodes.PyArgDeclNode(pos, name = name, annotation = annotation) @@ -3688,22 +3793,18 @@ def p_module(s, pxd, full_module_name, ctx=Ctx): s.parse_comments = False if s.context.language_level is None: - s.context.set_language_level(2) + s.context.set_language_level('3str') if pos[0].filename: import warnings warnings.warn( - "Cython directive 'language_level' not set, using 2 for now (Py2). " - "This will change in a later release! File: %s" % pos[0].filename, + "Cython directive 'language_level' not set, using '3str' for now (Py3). " + "This has changed from earlier releases! File: %s" % pos[0].filename, FutureWarning, stacklevel=1 if cython.compiled else 2, ) + level = 'module_pxd' if pxd else 'module' doc = p_doc_string(s) - if pxd: - level = 'module_pxd' - else: - level = 'module' - body = p_statement_list(s, ctx(level=level), first_statement = 1) if s.sy != 'EOF': s.error("Syntax error in statement [%s,%s]" % ( @@ -3725,7 +3826,6 @@ def p_template_definition(s): def p_cpp_class_definition(s, pos, ctx): # s.sy == 'cppclass' s.next() - module_path = [] class_name = p_ident(s) cname = p_opt_cname(s) if cname is None and ctx.namespace is not None: @@ -3759,6 +3859,10 @@ def p_cpp_class_definition(s, pos, ctx): s.next() s.expect('NEWLINE') s.expect_indent() + # Allow a cppclass to have docstrings. It will be discarded as comment. + # The goal of this is consistency: we can make docstrings inside cppclass methods, + # so why not on the cppclass itself ? + p_doc_string(s) attributes = [] body_ctx = Ctx(visibility = ctx.visibility, level='cpp_class', nogil=nogil or ctx.nogil) body_ctx.templates = template_names @@ -3842,3 +3946,14 @@ def print_parse_tree(f, node, level, key = None): f.write("%s]\n" % ind) return f.write("%s%s\n" % (ind, node)) + +def p_annotation(s): + """An annotation just has the "test" syntax, but also stores the string it came from + + Note that the string is *allowed* to be changed/processed (although isn't here) + so may not exactly match the string generated by Python, and if it doesn't + then it is not a bug. + """ + pos = s.position() + expr = p_test(s) + return ExprNodes.AnnotationNode(pos, expr=expr) diff --git a/Cython/Compiler/Pipeline.py b/Cython/Compiler/Pipeline.py index 5194c3e49..864e6bc2a 100644 --- a/Cython/Compiler/Pipeline.py +++ b/Cython/Compiler/Pipeline.py @@ -128,7 +128,8 @@ def inject_utility_code_stage_factory(context): module_node.scope.utility_code_list.append(dep) tree = utilcode.get_tree(cython_scope=context.cython_scope) if tree: - module_node.merge_in(tree.body, tree.scope, merge_scope=True) + module_node.merge_in(tree.with_compiler_directives(), + tree.scope, merge_scope=True) return module_node return inject_utility_code_stage @@ -148,8 +149,8 @@ def create_pipeline(context, mode, exclude_classes=()): from .ParseTreeTransforms import ExpandInplaceOperators, ParallelRangeTransform from .ParseTreeTransforms import CalculateQualifiedNamesTransform from .TypeInference import MarkParallelAssignments, MarkOverflowingArithmetic - from .ParseTreeTransforms import AdjustDefByDirectives, AlignFunctionDefinitions - from .ParseTreeTransforms import RemoveUnreachableCode, GilCheck + from .ParseTreeTransforms import AdjustDefByDirectives, AlignFunctionDefinitions, AutoCpdefFunctionDefinitions + from .ParseTreeTransforms import RemoveUnreachableCode, GilCheck, CoerceCppTemps from .FlowControl import ControlFlowAnalysis from .AnalysedTreeTransforms import AutoTestDictTransform from .AutoDocTransforms import EmbedSignature @@ -185,10 +186,11 @@ def create_pipeline(context, mode, exclude_classes=()): TrackNumpyAttributes(), InterpretCompilerDirectives(context, context.compiler_directives), ParallelRangeTransform(context), - AdjustDefByDirectives(context), WithTransform(context), - MarkClosureVisitor(context), + AdjustDefByDirectives(context), _align_function_definitions, + MarkClosureVisitor(context), + AutoCpdefFunctionDefinitions(context), RemoveUnreachableCode(context), ConstantFolding(), FlattenInListTransform(), @@ -219,6 +221,7 @@ def create_pipeline(context, mode, exclude_classes=()): ConsolidateOverflowCheck(context), DropRefcountingTransform(), FinalOptimizePhase(context), + CoerceCppTemps(context), GilCheck(), ] filtered_stages = [] @@ -238,7 +241,7 @@ def create_pyx_pipeline(context, options, result, py=False, exclude_classes=()): test_support.append(TreeAssertVisitor()) if options.gdb_debug: - from ..Debugger import DebugWriter # requires Py2.5+ + from ..Debugger import DebugWriter # requires Py2.5+ from .ParseTreeTransforms import DebugTransform context.gdb_debug_outputwriter = DebugWriter.CythonDebugWriter( options.output_dir) @@ -284,11 +287,25 @@ def create_pyx_as_pxd_pipeline(context, result): FlattenInListTransform, WithTransform ]) + from .Visitor import VisitorTransform + class SetInPxdTransform(VisitorTransform): + # A number of nodes have an "in_pxd" attribute which affects AnalyseDeclarationsTransform + # (for example controlling pickling generation). Set it, to make sure we don't mix them up with + # the importing main module. + # FIXME: This should be done closer to the parsing step. + def visit_StatNode(self, node): + if hasattr(node, "in_pxd"): + node.in_pxd = True + self.visitchildren(node) + return node + + visit_Node = VisitorTransform.recurse_to_children + for stage in pyx_pipeline: pipeline.append(stage) if isinstance(stage, AnalyseDeclarationsTransform): - # This is the last stage we need. - break + pipeline.insert(-1, SetInPxdTransform()) + break # This is the last stage we need. def fake_pxd(root): for entry in root.scope.entries.values(): if not entry.in_cinclude: diff --git a/Cython/Compiler/PyrexTypes.py b/Cython/Compiler/PyrexTypes.py index dcb51fe34..4345bfdb2 100644 --- a/Cython/Compiler/PyrexTypes.py +++ b/Cython/Compiler/PyrexTypes.py @@ -12,13 +12,14 @@ try: reduce except NameError: from functools import reduce +from functools import partial from Cython.Utils import cached_function from .Code import UtilityCode, LazyUtilityCode, TempitaUtilityCode from . import StringEncoding from . import Naming -from .Errors import error, warning +from .Errors import error, warning, CannotSpecialize class BaseType(object): @@ -46,7 +47,9 @@ class BaseType(object): def cast_code(self, expr_code): return "((%s)%s)" % (self.empty_declaration_code(), expr_code) - def empty_declaration_code(self): + def empty_declaration_code(self, pyrex=False): + if pyrex: + return self.declaration_code('', pyrex=True) if self._empty_declaration is None: self._empty_declaration = self.declaration_code('') return self._empty_declaration @@ -115,7 +118,7 @@ class BaseType(object): Deduce any template params in this (argument) type given the actual argument type. - http://en.cppreference.com/w/cpp/language/function_template#Template_argument_deduction + https://en.cppreference.com/w/cpp/language/function_template#Template_argument_deduction """ return {} @@ -176,11 +179,17 @@ class PyrexType(BaseType): # is_ptr boolean Is a C pointer type # is_null_ptr boolean Is the type of NULL # is_reference boolean Is a C reference type - # is_const boolean Is a C const type. + # is_rvalue_reference boolean Is a C++ rvalue reference type + # is_const boolean Is a C const type + # is_volatile boolean Is a C volatile type + # is_cv_qualified boolean Is a C const or volatile type # is_cfunction boolean Is a C function type # is_struct_or_union boolean Is a C struct or union type # is_struct boolean Is a C struct type + # is_cpp_class boolean Is a C++ class + # is_optional_cpp_class boolean Is a C++ class with variable lifetime handled with std::optional # is_enum boolean Is a C enum type + # is_cpp_enum boolean Is a C++ scoped enum type # is_typedef boolean Is a typedef type # is_string boolean Is a C char * type # is_pyunicode_ptr boolean Is a C PyUNICODE * type @@ -192,6 +201,9 @@ class PyrexType(BaseType): # is_pythran_expr boolean Is Pythran expr # is_numpy_buffer boolean Is Numpy array buffer # has_attributes boolean Has C dot-selectable attributes + # needs_cpp_construction boolean Needs C++ constructor and destructor when used in a cdef class + # needs_refcounting boolean Needs code to be generated similar to incref/gotref/decref. + # Largely used internally. # default_value string Initial value that can be assigned before first user assignment. # declaration_value string The value statically assigned on declaration (if any). # entry Entry The Entry for this type @@ -226,6 +238,7 @@ class PyrexType(BaseType): is_extension_type = 0 is_final_type = 0 is_builtin_type = 0 + is_cython_builtin_type = 0 is_numeric = 0 is_int = 0 is_float = 0 @@ -235,13 +248,19 @@ class PyrexType(BaseType): is_ptr = 0 is_null_ptr = 0 is_reference = 0 + is_fake_reference = 0 + is_rvalue_reference = 0 is_const = 0 + is_volatile = 0 + is_cv_qualified = 0 is_cfunction = 0 is_struct_or_union = 0 is_cpp_class = 0 + is_optional_cpp_class = 0 is_cpp_string = 0 is_struct = 0 is_enum = 0 + is_cpp_enum = False is_typedef = 0 is_string = 0 is_pyunicode_ptr = 0 @@ -254,6 +273,8 @@ class PyrexType(BaseType): is_pythran_expr = 0 is_numpy_buffer = 0 has_attributes = 0 + needs_cpp_construction = 0 + needs_refcounting = 0 default_value = "" declaration_value = "" @@ -262,7 +283,8 @@ class PyrexType(BaseType): return self def specialize(self, values): - # TODO(danilo): Override wherever it makes sense. + # Returns the concrete type if this is a fused type, or otherwise the type itself. + # May raise Errors.CannotSpecialize on failure return self def literal_code(self, value): @@ -331,6 +353,35 @@ class PyrexType(BaseType): convert_call, code.error_goto_if(error_condition or self.error_condition(result_code), error_pos)) + def _generate_dummy_refcounting(self, code, *ignored_args, **ignored_kwds): + if self.needs_refcounting: + raise NotImplementedError("Ref-counting operation not yet implemented for type %s" % + self) + + def _generate_dummy_refcounting_assignment(self, code, cname, rhs_cname, *ignored_args, **ignored_kwds): + if self.needs_refcounting: + raise NotImplementedError("Ref-counting operation not yet implemented for type %s" % + self) + code.putln("%s = %s" % (cname, rhs_cname)) + + generate_incref = generate_xincref = generate_decref = generate_xdecref \ + = generate_decref_clear = generate_xdecref_clear \ + = generate_gotref = generate_xgotref = generate_giveref = generate_xgiveref \ + = _generate_dummy_refcounting + + generate_decref_set = generate_xdecref_set = _generate_dummy_refcounting_assignment + + def nullcheck_string(self, code, cname): + if self.needs_refcounting: + raise NotImplementedError("Ref-counting operation not yet implemented for type %s" % + self) + code.putln("1") + + def cpp_optional_declaration_code(self, entity_code, dll_linkage=None): + # declares an std::optional c++ variable + raise NotImplementedError( + "cpp_optional_declaration_code only implemented for c++ classes and not type %s" % self) + def public_decl(base_code, dll_linkage): if dll_linkage: @@ -338,20 +389,15 @@ def public_decl(base_code, dll_linkage): else: return base_code -def create_typedef_type(name, base_type, cname, is_external=0, namespace=None): - is_fused = base_type.is_fused - if base_type.is_complex or is_fused: - if is_external: - if is_fused: - msg = "Fused" - else: - msg = "Complex" - - raise ValueError("%s external typedefs not supported" % msg) +def create_typedef_type(name, base_type, cname, is_external=0, namespace=None): + if is_external: + if base_type.is_complex or base_type.is_fused: + raise ValueError("%s external typedefs not supported" % ( + "Fused" if base_type.is_fused else "Complex")) + if base_type.is_complex or base_type.is_fused: return base_type - else: - return CTypedefType(name, base_type, cname, is_external, namespace) + return CTypedefType(name, base_type, cname, is_external, namespace) class CTypedefType(BaseType): @@ -447,9 +493,9 @@ class CTypedefType(BaseType): "TO_PY_FUNCTION": self.to_py_function})) return True elif base_type.is_float: - pass # XXX implement! + pass # XXX implement! elif base_type.is_complex: - pass # XXX implement! + pass # XXX implement! pass elif base_type.is_cpp_string: cname = "__pyx_convert_PyObject_string_to_py_%s" % type_identifier(self) @@ -480,9 +526,9 @@ class CTypedefType(BaseType): "FROM_PY_FUNCTION": self.from_py_function})) return True elif base_type.is_float: - pass # XXX implement! + pass # XXX implement! elif base_type.is_complex: - pass # XXX implement! + pass # XXX implement! elif base_type.is_cpp_string: cname = '__pyx_convert_string_from_py_%s' % type_identifier(self) context = { @@ -561,8 +607,13 @@ class CTypedefType(BaseType): class MemoryViewSliceType(PyrexType): is_memoryviewslice = 1 + default_value = "{ 0, 0, { 0 }, { 0 }, { 0 } }" has_attributes = 1 + needs_refcounting = 1 # Ideally this would be true and reference counting for + # memoryview and pyobject code could be generated in the same way. + # However, memoryviews are sufficiently specialized that this doesn't + # seem practical. Implement a limited version of it for now scope = None # These are special cased in Defnode @@ -591,7 +642,7 @@ class MemoryViewSliceType(PyrexType): 'ptr' -- Pointer stored in this dimension. 'full' -- Check along this dimension, don't assume either. - the packing specifiers specify how the array elements are layed-out + the packing specifiers specify how the array elements are laid-out in memory. 'contig' -- The data is contiguous in memory along this dimension. @@ -633,6 +684,10 @@ class MemoryViewSliceType(PyrexType): else: return False + def __ne__(self, other): + # TODO drop when Python2 is dropped + return not (self == other) + def same_as_resolved_type(self, other_type): return ((other_type.is_memoryviewslice and #self.writable_needed == other_type.writable_needed and # FIXME: should be only uni-directional @@ -650,10 +705,10 @@ class MemoryViewSliceType(PyrexType): def declaration_code(self, entity_code, for_display = 0, dll_linkage = None, pyrex = 0): # XXX: we put these guards in for now... - assert not pyrex assert not dll_linkage from . import MemoryView - base_code = str(self) if for_display else MemoryView.memviewslice_cname + base_code = StringEncoding.EncodedString( + str(self) if pyrex or for_display else MemoryView.memviewslice_cname) return self.base_declaration_code( base_code, entity_code) @@ -713,8 +768,8 @@ class MemoryViewSliceType(PyrexType): to_axes_f = contig_dim + follow_dim * (ndim -1) dtype = self.dtype - if dtype.is_const: - dtype = dtype.const_base_type + if dtype.is_cv_qualified: + dtype = dtype.cv_base_type to_memview_c = MemoryViewSliceType(dtype, to_axes_c) to_memview_f = MemoryViewSliceType(dtype, to_axes_f) @@ -791,15 +846,18 @@ class MemoryViewSliceType(PyrexType): # return False src_dtype, dst_dtype = src.dtype, dst.dtype - if dst_dtype.is_const: - # Requesting read-only views is always ok => consider only the non-const base type. - dst_dtype = dst_dtype.const_base_type - if src_dtype.is_const: - # When assigning between read-only views, compare only the non-const base types. - src_dtype = src_dtype.const_base_type - elif copying and src_dtype.is_const: - # Copying by value => ignore const on source. - src_dtype = src_dtype.const_base_type + # We can add but not remove const/volatile modifiers + # (except if we are copying by value, then anything is fine) + if not copying: + if src_dtype.is_const and not dst_dtype.is_const: + return False + if src_dtype.is_volatile and not dst_dtype.is_volatile: + return False + # const/volatile checks are done, remove those qualifiers + if src_dtype.is_cv_qualified: + src_dtype = src_dtype.cv_base_type + if dst_dtype.is_cv_qualified: + dst_dtype = dst_dtype.cv_base_type if src_dtype != dst_dtype: return False @@ -1032,6 +1090,36 @@ class MemoryViewSliceType(PyrexType): def cast_code(self, expr_code): return expr_code + # When memoryviews are increfed currently seems heavily special-cased. + # Therefore, use our own function for now + def generate_incref(self, code, name, **kwds): + pass + + def generate_incref_memoryviewslice(self, code, slice_cname, have_gil): + # TODO ideally would be done separately + code.putln("__PYX_INC_MEMVIEW(&%s, %d);" % (slice_cname, int(have_gil))) + + # decref however did look to always apply for memoryview slices + # with "have_gil" set to True by default + def generate_xdecref(self, code, cname, nanny, have_gil): + code.putln("__PYX_XCLEAR_MEMVIEW(&%s, %d);" % (cname, int(have_gil))) + + def generate_decref(self, code, cname, nanny, have_gil): + # Fall back to xdecref since we don't care to have a separate decref version for this. + self.generate_xdecref(code, cname, nanny, have_gil) + + def generate_xdecref_clear(self, code, cname, clear_before_decref, **kwds): + self.generate_xdecref(code, cname, **kwds) + code.putln("%s.memview = NULL; %s.data = NULL;" % (cname, cname)) + + def generate_decref_clear(self, code, cname, **kwds): + # memoryviews don't currently distinguish between xdecref and decref + self.generate_xdecref_clear(code, cname, **kwds) + + # memoryviews don't participate in giveref/gotref + generate_gotref = generate_xgotref = generate_xgiveref = generate_giveref = lambda *args: None + + class BufferType(BaseType): # @@ -1129,6 +1217,8 @@ class PyObjectType(PyrexType): is_extern = False is_subclassed = False is_gc_simple = False + builtin_trashcan = False # builtin type using trashcan + needs_refcounting = True def __str__(self): return "Python object" @@ -1181,11 +1271,85 @@ class PyObjectType(PyrexType): def check_for_null_code(self, cname): return cname + def generate_incref(self, code, cname, nanny): + if nanny: + code.putln("__Pyx_INCREF(%s);" % self.as_pyobject(cname)) + else: + code.putln("Py_INCREF(%s);" % self.as_pyobject(cname)) + + def generate_xincref(self, code, cname, nanny): + if nanny: + code.putln("__Pyx_XINCREF(%s);" % self.as_pyobject(cname)) + else: + code.putln("Py_XINCREF(%s);" % self.as_pyobject(cname)) + + def generate_decref(self, code, cname, nanny, have_gil): + # have_gil is for the benefit of memoryviewslice - it's ignored here + assert have_gil + self._generate_decref(code, cname, nanny, null_check=False, clear=False) -builtin_types_that_cannot_create_refcycles = set([ - 'bool', 'int', 'long', 'float', 'complex', - 'bytearray', 'bytes', 'unicode', 'str', 'basestring' -]) + def generate_xdecref(self, code, cname, nanny, have_gil): + # in this (and other) PyObjectType functions, have_gil is being + # passed to provide a common interface with MemoryviewSlice. + # It's ignored here + self._generate_decref(code, cname, nanny, null_check=True, + clear=False) + + def generate_decref_clear(self, code, cname, clear_before_decref, nanny, have_gil): + self._generate_decref(code, cname, nanny, null_check=False, + clear=True, clear_before_decref=clear_before_decref) + + def generate_xdecref_clear(self, code, cname, clear_before_decref=False, nanny=True, have_gil=None): + self._generate_decref(code, cname, nanny, null_check=True, + clear=True, clear_before_decref=clear_before_decref) + + def generate_gotref(self, code, cname): + code.putln("__Pyx_GOTREF(%s);" % self.as_pyobject(cname)) + + def generate_xgotref(self, code, cname): + code.putln("__Pyx_XGOTREF(%s);" % self.as_pyobject(cname)) + + def generate_giveref(self, code, cname): + code.putln("__Pyx_GIVEREF(%s);" % self.as_pyobject(cname)) + + def generate_xgiveref(self, code, cname): + code.putln("__Pyx_XGIVEREF(%s);" % self.as_pyobject(cname)) + + def generate_decref_set(self, code, cname, rhs_cname): + code.putln("__Pyx_DECREF_SET(%s, %s);" % (cname, rhs_cname)) + + def generate_xdecref_set(self, code, cname, rhs_cname): + code.putln("__Pyx_XDECREF_SET(%s, %s);" % (cname, rhs_cname)) + + def _generate_decref(self, code, cname, nanny, null_check=False, + clear=False, clear_before_decref=False): + prefix = '__Pyx' if nanny else 'Py' + X = 'X' if null_check else '' + + if clear: + if clear_before_decref: + if not nanny: + X = '' # CPython doesn't have a Py_XCLEAR() + code.putln("%s_%sCLEAR(%s);" % (prefix, X, cname)) + else: + code.putln("%s_%sDECREF(%s); %s = 0;" % ( + prefix, X, self.as_pyobject(cname), cname)) + else: + code.putln("%s_%sDECREF(%s);" % ( + prefix, X, self.as_pyobject(cname))) + + def nullcheck_string(self, cname): + return cname + + +builtin_types_that_cannot_create_refcycles = frozenset({ + 'object', 'bool', 'int', 'long', 'float', 'complex', + 'bytearray', 'bytes', 'unicode', 'str', 'basestring', +}) + +builtin_types_with_trashcan = frozenset({ + 'dict', 'list', 'set', 'frozenset', 'tuple', 'type', +}) class BuiltinObjectType(PyObjectType): @@ -1211,6 +1375,7 @@ class BuiltinObjectType(PyObjectType): self.typeptr_cname = "(&%s)" % cname self.objstruct_cname = objstruct_cname self.is_gc_simple = name in builtin_types_that_cannot_create_refcycles + self.builtin_trashcan = name in builtin_types_with_trashcan if name == 'type': # Special case the type type, as many C API calls (and other # libraries) actually expect a PyTypeObject* for type arguments. @@ -1292,14 +1457,9 @@ class BuiltinObjectType(PyObjectType): check += '||((%s) == Py_None)' % arg if self.name == 'basestring': name = '(PY_MAJOR_VERSION < 3 ? "basestring" : "str")' - space_for_name = 16 else: name = '"%s"' % self.name - # avoid wasting too much space but limit number of different format strings - space_for_name = (len(self.name) // 16 + 1) * 16 - error = '(PyErr_Format(PyExc_TypeError, "Expected %%.%ds, got %%.200s", %s, Py_TYPE(%s)->tp_name), 0)' % ( - space_for_name, name, arg) - return check + '||' + error + return check + ' || __Pyx_RaiseUnexpectedTypeError(%s, %s)' % (name, arg) def declaration_code(self, entity_code, for_display = 0, dll_linkage = None, pyrex = 0): @@ -1318,7 +1478,7 @@ class BuiltinObjectType(PyObjectType): def cast_code(self, expr_code, to_object_struct = False): return "((%s*)%s)" % ( - to_object_struct and self.objstruct_cname or self.decl_type, # self.objstruct_cname may be None + to_object_struct and self.objstruct_cname or self.decl_type, # self.objstruct_cname may be None expr_code) def py_type_name(self): @@ -1543,8 +1703,14 @@ class PythranExpr(CType): self.scope = scope = Symtab.CClassScope('', None, visibility="extern") scope.parent_type = self scope.directives = {} - scope.declare_var("shape", CPtrType(c_long_type), None, cname="_shape", is_cdef=True) - scope.declare_var("ndim", c_long_type, None, cname="value", is_cdef=True) + + scope.declare_var("ndim", c_long_type, pos=None, cname="value", is_cdef=True) + scope.declare_cproperty( + "shape", c_ptr_type(c_long_type), "__Pyx_PythranShapeAccessor", + doc="Pythran array shape", + visibility="extern", + nogil=True, + ) return True @@ -1558,58 +1724,76 @@ class PythranExpr(CType): return hash(self.pythran_type) -class CConstType(BaseType): +class CConstOrVolatileType(BaseType): + "A C const or volatile type" - is_const = 1 + subtypes = ['cv_base_type'] - def __init__(self, const_base_type): - self.const_base_type = const_base_type - if const_base_type.has_attributes and const_base_type.scope is not None: - from . import Symtab - self.scope = Symtab.CConstScope(const_base_type.scope) + is_cv_qualified = 1 + + def __init__(self, base_type, is_const=0, is_volatile=0): + self.cv_base_type = base_type + self.is_const = is_const + self.is_volatile = is_volatile + if base_type.has_attributes and base_type.scope is not None: + from .Symtab import CConstOrVolatileScope + self.scope = CConstOrVolatileScope(base_type.scope, is_const, is_volatile) + + def cv_string(self): + cvstring = "" + if self.is_const: + cvstring = "const " + cvstring + if self.is_volatile: + cvstring = "volatile " + cvstring + return cvstring def __repr__(self): - return "<CConstType %s>" % repr(self.const_base_type) + return "<CConstOrVolatileType %s%r>" % (self.cv_string(), self.cv_base_type) def __str__(self): return self.declaration_code("", for_display=1) def declaration_code(self, entity_code, for_display = 0, dll_linkage = None, pyrex = 0): + cv = self.cv_string() if for_display or pyrex: - return "const " + self.const_base_type.declaration_code(entity_code, for_display, dll_linkage, pyrex) + return cv + self.cv_base_type.declaration_code(entity_code, for_display, dll_linkage, pyrex) else: - return self.const_base_type.declaration_code("const %s" % entity_code, for_display, dll_linkage, pyrex) + return self.cv_base_type.declaration_code(cv + entity_code, for_display, dll_linkage, pyrex) def specialize(self, values): - base_type = self.const_base_type.specialize(values) - if base_type == self.const_base_type: + base_type = self.cv_base_type.specialize(values) + if base_type == self.cv_base_type: return self - else: - return CConstType(base_type) + return CConstOrVolatileType(base_type, + self.is_const, self.is_volatile) def deduce_template_params(self, actual): - return self.const_base_type.deduce_template_params(actual) + return self.cv_base_type.deduce_template_params(actual) def can_coerce_to_pyobject(self, env): - return self.const_base_type.can_coerce_to_pyobject(env) + return self.cv_base_type.can_coerce_to_pyobject(env) def can_coerce_from_pyobject(self, env): - return self.const_base_type.can_coerce_from_pyobject(env) + return self.cv_base_type.can_coerce_from_pyobject(env) def create_to_py_utility_code(self, env): - if self.const_base_type.create_to_py_utility_code(env): - self.to_py_function = self.const_base_type.to_py_function + if self.cv_base_type.create_to_py_utility_code(env): + self.to_py_function = self.cv_base_type.to_py_function return True def same_as_resolved_type(self, other_type): - if other_type.is_const: - return self.const_base_type.same_as_resolved_type(other_type.const_base_type) - # Accept const LHS <- non-const RHS. - return self.const_base_type.same_as_resolved_type(other_type) + if other_type.is_cv_qualified: + return self.cv_base_type.same_as_resolved_type(other_type.cv_base_type) + # Accept cv LHS <- non-cv RHS. + return self.cv_base_type.same_as_resolved_type(other_type) def __getattr__(self, name): - return getattr(self.const_base_type, name) + return getattr(self.cv_base_type, name) + + +def CConstType(base_type): + return CConstOrVolatileType(base_type, is_const=1) class FusedType(CType): @@ -1652,7 +1836,10 @@ class FusedType(CType): return 'FusedType(name=%r)' % self.name def specialize(self, values): - return values[self] + if self in values: + return values[self] + else: + raise CannotSpecialize() def get_fused_types(self, result=None, seen=None): if result is None: @@ -1737,6 +1924,7 @@ class CNumericType(CType): base_code = type_name.replace('PY_LONG_LONG', 'long long') else: base_code = public_decl(type_name, dll_linkage) + base_code = StringEncoding.EncodedString(base_code) return self.base_declaration_code(base_code, entity_code) def attributes_known(self): @@ -2169,8 +2357,8 @@ class CComplexType(CNumericType): def assignable_from(self, src_type): # Temporary hack/feature disabling, see #441 if (not src_type.is_complex and src_type.is_numeric and src_type.is_typedef - and src_type.typedef_is_external): - return False + and src_type.typedef_is_external): + return False elif src_type.is_pyobject: return True else: @@ -2178,8 +2366,8 @@ class CComplexType(CNumericType): def assignable_from_resolved_type(self, src_type): return (src_type.is_complex and self.real_type.assignable_from_resolved_type(src_type.real_type) - or src_type.is_numeric and self.real_type.assignable_from_resolved_type(src_type) - or src_type is error_type) + or src_type.is_numeric and self.real_type.assignable_from_resolved_type(src_type) + or src_type is error_type) def attributes_known(self): if self.scope is None: @@ -2302,8 +2490,8 @@ class CPointerBaseType(CType): def __init__(self, base_type): self.base_type = base_type - if base_type.is_const: - base_type = base_type.const_base_type + if base_type.is_cv_qualified: + base_type = base_type.cv_base_type for char_type in (c_char_type, c_uchar_type, c_schar_type): if base_type.same_as(char_type): self.is_string = 1 @@ -2346,6 +2534,7 @@ class CPointerBaseType(CType): if self.is_string: assert isinstance(value, str) return '"%s"' % StringEncoding.escape_byte_string(value) + return str(value) class CArrayType(CPointerBaseType): @@ -2365,7 +2554,7 @@ class CArrayType(CPointerBaseType): return False def __hash__(self): - return hash(self.base_type) + 28 # arbitrarily chosen offset + return hash(self.base_type) + 28 # arbitrarily chosen offset def __repr__(self): return "<CArrayType %s %s>" % (self.size, repr(self.base_type)) @@ -2497,7 +2686,7 @@ class CPtrType(CPointerBaseType): default_value = "0" def __hash__(self): - return hash(self.base_type) + 27 # arbitrarily chosen offset + return hash(self.base_type) + 27 # arbitrarily chosen offset def __eq__(self, other): if isinstance(other, CType) and other.is_ptr: @@ -2527,8 +2716,8 @@ class CPtrType(CPointerBaseType): return 1 if other_type.is_null_ptr: return 1 - if self.base_type.is_const: - self = CPtrType(self.base_type.const_base_type) + if self.base_type.is_cv_qualified: + self = CPtrType(self.base_type.cv_base_type) if self.base_type.is_cfunction: if other_type.is_ptr: other_type = other_type.base_type.resolve() @@ -2570,26 +2759,17 @@ class CNullPtrType(CPtrType): is_null_ptr = 1 -class CReferenceType(BaseType): +class CReferenceBaseType(BaseType): - is_reference = 1 is_fake_reference = 0 + # Common base type for C reference and C++ rvalue reference types. + def __init__(self, base_type): self.ref_base_type = base_type def __repr__(self): - return "<CReferenceType %s>" % repr(self.ref_base_type) - - def __str__(self): - return "%s &" % self.ref_base_type - - def declaration_code(self, entity_code, - for_display = 0, dll_linkage = None, pyrex = 0): - #print "CReferenceType.declaration_code: pointer to", self.base_type ### - return self.ref_base_type.declaration_code( - "&%s" % entity_code, - for_display, dll_linkage, pyrex) + return "<%r %s>" % (self.__class__.__name__, self.ref_base_type) def specialize(self, values): base_type = self.ref_base_type.specialize(values) @@ -2605,13 +2785,25 @@ class CReferenceType(BaseType): return getattr(self.ref_base_type, name) +class CReferenceType(CReferenceBaseType): + + is_reference = 1 + + def __str__(self): + return "%s &" % self.ref_base_type + + def declaration_code(self, entity_code, + for_display = 0, dll_linkage = None, pyrex = 0): + #print "CReferenceType.declaration_code: pointer to", self.base_type ### + return self.ref_base_type.declaration_code( + "&%s" % entity_code, + for_display, dll_linkage, pyrex) + + class CFakeReferenceType(CReferenceType): is_fake_reference = 1 - def __repr__(self): - return "<CFakeReferenceType %s>" % repr(self.ref_base_type) - def __str__(self): return "%s [&]" % self.ref_base_type @@ -2621,6 +2813,20 @@ class CFakeReferenceType(CReferenceType): return "__Pyx_FakeReference<%s> %s" % (self.ref_base_type.empty_declaration_code(), entity_code) +class CppRvalueReferenceType(CReferenceBaseType): + + is_rvalue_reference = 1 + + def __str__(self): + return "%s &&" % self.ref_base_type + + def declaration_code(self, entity_code, + for_display = 0, dll_linkage = None, pyrex = 0): + return self.ref_base_type.declaration_code( + "&&%s" % entity_code, + for_display, dll_linkage, pyrex) + + class CFuncType(CType): # return_type CType # args [CFuncTypeArg] @@ -2638,12 +2844,14 @@ class CFuncType(CType): # (used for optimisation overrides) # is_const_method boolean # is_static_method boolean + # op_arg_struct CPtrType Pointer to optional argument struct is_cfunction = 1 original_sig = None cached_specialized_types = None from_fused = False is_const_method = False + op_arg_struct = None subtypes = ['return_type', 'args'] @@ -2792,8 +3000,8 @@ class CFuncType(CType): # is performed elsewhere). for i in range(as_cmethod, len(other_type.args)): if not self.args[i].type.same_as( - other_type.args[i].type): - return 0 + other_type.args[i].type): + return 0 if self.has_varargs != other_type.has_varargs: return 0 if not self.return_type.subtype_of_resolved_type(other_type.return_type): @@ -3049,8 +3257,16 @@ class CFuncType(CType): if not self.can_coerce_to_pyobject(env): return False from .UtilityCode import CythonUtilityCode - safe_typename = re.sub('[^a-zA-Z0-9]', '__', self.declaration_code("", pyrex=1)) - to_py_function = "__Pyx_CFunc_%s_to_py" % safe_typename + + # include argument names into the c function name to ensure cname is unique + # between functions with identical types but different argument names + from .Symtab import punycodify_name + def arg_name_part(arg): + return "%s%s" % (len(arg.name), punycodify_name(arg.name)) if arg.name else "0" + arg_names = [ arg_name_part(arg) for arg in self.args ] + arg_names = "_".join(arg_names) + safe_typename = type_identifier(self, pyrex=True) + to_py_function = "__Pyx_CFunc_%s_to_py_%s" % (safe_typename, arg_names) for arg in self.args: if not arg.type.is_pyobject and not arg.type.create_from_py_utility_code(env): @@ -3242,7 +3458,7 @@ class CFuncTypeArg(BaseType): self.annotation = annotation self.type = type self.pos = pos - self.needs_type_test = False # TODO: should these defaults be set in analyse_types()? + self.needs_type_test = False # TODO: should these defaults be set in analyse_types()? def __repr__(self): return "%s:%s" % (self.name, repr(self.type)) @@ -3253,6 +3469,12 @@ class CFuncTypeArg(BaseType): def specialize(self, values): return CFuncTypeArg(self.name, self.type.specialize(values), self.pos, self.cname) + def is_forwarding_reference(self): + if self.type.is_rvalue_reference: + if (isinstance(self.type.ref_base_type, TemplatePlaceholderType) + and not self.type.ref_base_type.is_cv_qualified): + return True + return False class ToPyStructUtilityCode(object): @@ -3320,7 +3542,7 @@ class CStructOrUnionType(CType): has_attributes = 1 exception_check = True - def __init__(self, name, kind, scope, typedef_flag, cname, packed=False): + def __init__(self, name, kind, scope, typedef_flag, cname, packed=False, in_cpp=False): self.name = name self.cname = cname self.kind = kind @@ -3335,6 +3557,7 @@ class CStructOrUnionType(CType): self._convert_to_py_code = None self._convert_from_py_code = None self.packed = packed + self.needs_cpp_construction = self.is_struct and in_cpp def can_coerce_to_pyobject(self, env): if self._convert_to_py_code is False: @@ -3412,6 +3635,7 @@ class CStructOrUnionType(CType): var_entries=self.scope.var_entries, funcname=self.from_py_function, ) + env.use_utility_code(UtilityCode.load_cached("RaiseUnexpectedTypeError", "ObjectHandling.c")) from .UtilityCode import CythonUtilityCode self._convert_from_py_code = CythonUtilityCode.load( "FromPyStructUtility" if self.is_struct else "FromPyUnionUtility", @@ -3504,6 +3728,7 @@ class CppClassType(CType): is_cpp_class = 1 has_attributes = 1 + needs_cpp_construction = 1 exception_check = True namespace = None @@ -3577,10 +3802,12 @@ class CppClassType(CType): 'maybe_unordered': self.maybe_unordered(), 'type': self.cname, }) + # Override directives that should not be inherited from user code. from .UtilityCode import CythonUtilityCode + directives = CythonUtilityCode.filter_inherited_directives(env.directives) env.use_utility_code(CythonUtilityCode.load( cls.replace('unordered_', '') + ".from_py", "CppConvert.pyx", - context=context, compiler_directives=env.directives)) + context=context, compiler_directives=directives)) self.from_py_function = cname return True @@ -3623,9 +3850,11 @@ class CppClassType(CType): 'type': self.cname, }) from .UtilityCode import CythonUtilityCode + # Override directives that should not be inherited from user code. + directives = CythonUtilityCode.filter_inherited_directives(env.directives) env.use_utility_code(CythonUtilityCode.load( cls.replace('unordered_', '') + ".to_py", "CppConvert.pyx", - context=context, compiler_directives=env.directives)) + context=context, compiler_directives=directives)) self.to_py_function = cname return True @@ -3694,23 +3923,23 @@ class CppClassType(CType): specialized.namespace = self.namespace.specialize(values) specialized.scope = self.scope.specialize(values, specialized) if self.cname == 'std::vector': - # vector<bool> is special cased in the C++ standard, and its - # accessors do not necessarily return references to the underlying - # elements (which may be bit-packed). - # http://www.cplusplus.com/reference/vector/vector-bool/ - # Here we pretend that the various methods return bool values - # (as the actual returned values are coercable to such, and - # we don't support call expressions as lvalues). - T = values.get(self.templates[0], None) - if T and not T.is_fused and T.empty_declaration_code() == 'bool': - for bit_ref_returner in ('at', 'back', 'front'): - if bit_ref_returner in specialized.scope.entries: - specialized.scope.entries[bit_ref_returner].type.return_type = T + # vector<bool> is special cased in the C++ standard, and its + # accessors do not necessarily return references to the underlying + # elements (which may be bit-packed). + # http://www.cplusplus.com/reference/vector/vector-bool/ + # Here we pretend that the various methods return bool values + # (as the actual returned values are coercable to such, and + # we don't support call expressions as lvalues). + T = values.get(self.templates[0], None) + if T and not T.is_fused and T.empty_declaration_code() == 'bool': + for bit_ref_returner in ('at', 'back', 'front'): + if bit_ref_returner in specialized.scope.entries: + specialized.scope.entries[bit_ref_returner].type.return_type = T return specialized def deduce_template_params(self, actual): - if actual.is_const: - actual = actual.const_base_type + if actual.is_cv_qualified: + actual = actual.cv_base_type if actual.is_reference: actual = actual.ref_base_type if self == actual: @@ -3764,6 +3993,12 @@ class CppClassType(CType): base_code = public_decl(base_code, dll_linkage) return self.base_declaration_code(base_code, entity_code) + def cpp_optional_declaration_code(self, entity_code, dll_linkage=None, template_params=None): + return "__Pyx_Optional_Type<%s> %s" % ( + self.declaration_code("", False, dll_linkage, False, + template_params), + entity_code) + def is_subclass(self, other_type): if self.same_as_resolved_type(other_type): return 1 @@ -3846,6 +4081,81 @@ class CppClassType(CType): if constructor is not None and best_match([], constructor.all_alternatives()) is None: error(pos, "C++ class must have a nullary constructor to be %s" % msg) + def cpp_optional_check_for_null_code(self, cname): + # only applies to c++ classes that are being declared as std::optional + return "(%s.has_value())" % cname + + +class CppScopedEnumType(CType): + # name string + # doc string or None + # cname string + + is_cpp_enum = True + + def __init__(self, name, cname, underlying_type, namespace=None, doc=None): + self.name = name + self.doc = doc + self.cname = cname + self.values = [] + self.underlying_type = underlying_type + self.namespace = namespace + + def __str__(self): + return self.name + + def declaration_code(self, entity_code, + for_display=0, dll_linkage=None, pyrex=0): + if pyrex or for_display: + type_name = self.name + else: + if self.namespace: + type_name = "%s::%s" % ( + self.namespace.empty_declaration_code(), + self.cname + ) + else: + type_name = "__PYX_ENUM_CLASS_DECL %s" % self.cname + type_name = public_decl(type_name, dll_linkage) + return self.base_declaration_code(type_name, entity_code) + + def create_from_py_utility_code(self, env): + if self.from_py_function: + return True + if self.underlying_type.create_from_py_utility_code(env): + self.from_py_function = '(%s)%s' % ( + self.cname, self.underlying_type.from_py_function + ) + return True + + def create_to_py_utility_code(self, env): + if self.to_py_function is not None: + return True + if self.underlying_type.create_to_py_utility_code(env): + # Using a C++11 lambda here, which is fine since + # scoped enums are a C++11 feature + self.to_py_function = '[](const %s& x){return %s((%s)x);}' % ( + self.cname, + self.underlying_type.to_py_function, + self.underlying_type.empty_declaration_code() + ) + return True + + def create_type_wrapper(self, env): + from .UtilityCode import CythonUtilityCode + rst = CythonUtilityCode.load( + "CppScopedEnumType", "CpdefEnums.pyx", + context={ + "name": self.name, + "cname": self.cname.split("::")[-1], + "items": tuple(self.values), + "underlying_type": self.underlying_type.empty_declaration_code(), + "enum_doc": self.doc, + }, + outer_module_scope=env.global_scope()) + + env.use_utility_code(rst) + class TemplatePlaceholderType(CType): @@ -3896,16 +4206,18 @@ def is_optional_template_param(type): class CEnumType(CIntLike, CType): # name string + # doc string or None # cname string or None # typedef_flag boolean # values [string], populated during declaration analysis is_enum = 1 signed = 1 - rank = -1 # Ranks below any integer type + rank = -1 # Ranks below any integer type - def __init__(self, name, cname, typedef_flag, namespace=None): + def __init__(self, name, cname, typedef_flag, namespace=None, doc=None): self.name = name + self.doc = doc self.cname = cname self.values = [] self.typedef_flag = typedef_flag @@ -3947,7 +4259,9 @@ class CEnumType(CIntLike, CType): env.use_utility_code(CythonUtilityCode.load( "EnumType", "CpdefEnums.pyx", context={"name": self.name, - "items": tuple(self.values)}, + "items": tuple(self.values), + "enum_doc": self.doc, + }, outer_module_scope=env.global_scope())) @@ -4085,14 +4399,14 @@ class ErrorType(PyrexType): rank_to_type_name = ( - "char", # 0 - "short", # 1 - "int", # 2 - "long", # 3 - "PY_LONG_LONG", # 4 - "float", # 5 - "double", # 6 - "long double", # 7 + "char", # 0 + "short", # 1 + "int", # 2 + "long", # 3 + "PY_LONG_LONG", # 4 + "float", # 5 + "double", # 6 + "long double", # 7 ) _rank_to_type_name = list(rank_to_type_name) @@ -4302,8 +4616,7 @@ def best_match(arg_types, functions, pos=None, env=None, args=None): # Check no. of args max_nargs = len(func_type.args) min_nargs = max_nargs - func_type.optional_arg_count - if actual_nargs < min_nargs or \ - (not func_type.has_varargs and actual_nargs > max_nargs): + if actual_nargs < min_nargs or (not func_type.has_varargs and actual_nargs > max_nargs): if max_nargs == min_nargs and not func_type.has_varargs: expectation = max_nargs elif actual_nargs < min_nargs: @@ -4315,12 +4628,23 @@ def best_match(arg_types, functions, pos=None, env=None, args=None): errors.append((func, error_mesg)) continue if func_type.templates: + # For any argument/parameter pair A/P, if P is a forwarding reference, + # use lvalue-reference-to-A for deduction in place of A when the + # function call argument is an lvalue. See: + # https://en.cppreference.com/w/cpp/language/template_argument_deduction#Deduction_from_a_function_call + arg_types_for_deduction = list(arg_types) + if func.type.is_cfunction and args: + for i, formal_arg in enumerate(func.type.args): + if formal_arg.is_forwarding_reference(): + if args[i].is_lvalue(): + arg_types_for_deduction[i] = c_ref_type(arg_types[i]) deductions = reduce( merge_template_deductions, - [pattern.type.deduce_template_params(actual) for (pattern, actual) in zip(func_type.args, arg_types)], + [pattern.type.deduce_template_params(actual) for (pattern, actual) in zip(func_type.args, arg_types_for_deduction)], {}) if deductions is None: - errors.append((func, "Unable to deduce type parameters for %s given (%s)" % (func_type, ', '.join(map(str, arg_types))))) + errors.append((func, "Unable to deduce type parameters for %s given (%s)" % ( + func_type, ', '.join(map(str, arg_types_for_deduction))))) elif len(deductions) < len(func_type.templates): errors.append((func, "Unable to deduce type parameter %s" % ( ", ".join([param.name for param in set(func_type.templates) - set(deductions.keys())])))) @@ -4455,10 +4779,10 @@ def widest_numeric_type(type1, type2): type1 = type1.ref_base_type if type2.is_reference: type2 = type2.ref_base_type - if type1.is_const: - type1 = type1.const_base_type - if type2.is_const: - type2 = type2.const_base_type + if type1.is_cv_qualified: + type1 = type1.cv_base_type + if type2.is_cv_qualified: + type2 = type2.cv_base_type if type1 == type2: widest_type = type1 elif type1.is_complex or type2.is_complex: @@ -4648,35 +4972,38 @@ def parse_basic_type(name): name = 'int' return simple_c_type(signed, longness, name) -def c_array_type(base_type, size): - # Construct a C array type. + +def _construct_type_from_base(cls, base_type, *args): if base_type is error_type: return error_type - else: - return CArrayType(base_type, size) + return cls(base_type, *args) + +def c_array_type(base_type, size): + # Construct a C array type. + return _construct_type_from_base(CArrayType, base_type, size) def c_ptr_type(base_type): # Construct a C pointer type. - if base_type is error_type: - return error_type - elif base_type.is_reference: - return CPtrType(base_type.ref_base_type) - else: - return CPtrType(base_type) + if base_type.is_reference: + base_type = base_type.ref_base_type + return _construct_type_from_base(CPtrType, base_type) def c_ref_type(base_type): # Construct a C reference type - if base_type is error_type: - return error_type - else: - return CReferenceType(base_type) + return _construct_type_from_base(CReferenceType, base_type) + +def cpp_rvalue_ref_type(base_type): + # Construct a C++ rvalue reference type + return _construct_type_from_base(CppRvalueReferenceType, base_type) def c_const_type(base_type): # Construct a C const type. - if base_type is error_type: - return error_type - else: - return CConstType(base_type) + return _construct_type_from_base(CConstType, base_type) + +def c_const_or_volatile_type(base_type, is_const, is_volatile): + # Construct a C const/volatile type. + return _construct_type_from_base(CConstOrVolatileType, base_type, is_const, is_volatile) + def same_type(type1, type2): return type1.same_as(type2) @@ -4702,28 +5029,42 @@ def typecast(to_type, from_type, expr_code): def type_list_identifier(types): return cap_length('__and_'.join(type_identifier(type) for type in types)) +_special_type_characters = { + '__': '__dunder', + 'const ': '__const_', + ' ': '__space_', + '*': '__ptr', + '&': '__ref', + '&&': '__fwref', + '[': '__lArr', + ']': '__rArr', + '<': '__lAng', + '>': '__rAng', + '(': '__lParen', + ')': '__rParen', + ',': '__comma_', + '...': '__EL', + '::': '__in_', + ':': '__D', +} + +_escape_special_type_characters = partial(re.compile( + # join substrings in reverse order to put longer matches first, e.g. "::" before ":" + " ?(%s) ?" % "|".join(re.escape(s) for s in sorted(_special_type_characters, reverse=True)) +).sub, lambda match: _special_type_characters[match.group(1)]) + +def type_identifier(type, pyrex=False): + decl = type.empty_declaration_code(pyrex=pyrex) + return type_identifier_from_declaration(decl) + _type_identifier_cache = {} -def type_identifier(type): - decl = type.empty_declaration_code() +def type_identifier_from_declaration(decl): safe = _type_identifier_cache.get(decl) if safe is None: safe = decl safe = re.sub(' +', ' ', safe) - safe = re.sub(' ([^a-zA-Z0-9_])', r'\1', safe) - safe = re.sub('([^a-zA-Z0-9_]) ', r'\1', safe) - safe = (safe.replace('__', '__dunder') - .replace('const ', '__const_') - .replace(' ', '__space_') - .replace('*', '__ptr') - .replace('&', '__ref') - .replace('[', '__lArr') - .replace(']', '__rArr') - .replace('<', '__lAng') - .replace('>', '__rAng') - .replace('(', '__lParen') - .replace(')', '__rParen') - .replace(',', '__comma_') - .replace('::', '__in_')) + safe = re.sub(' ?([^a-zA-Z0-9_]) ?', r'\1', safe) + safe = _escape_special_type_characters(safe) safe = cap_length(re.sub('[^a-zA-Z0-9_]', lambda x: '__%X' % ord(x.group(0)), safe)) _type_identifier_cache[decl] = safe return safe diff --git a/Cython/Compiler/Scanning.pxd b/Cython/Compiler/Scanning.pxd index 59593f88a..351d55560 100644 --- a/Cython/Compiler/Scanning.pxd +++ b/Cython/Compiler/Scanning.pxd @@ -1,4 +1,4 @@ -from __future__ import absolute_import +# cython: language_level=3 import cython @@ -9,11 +9,6 @@ cdef unicode any_string_prefix, IDENT cdef get_lexicon() cdef initial_compile_time_env() -cdef class Method: - cdef object name - cdef dict kwargs - cdef readonly object __name__ # for tracing the scanner - ## methods commented with '##' out are used by Parsing.py when compiled. @cython.final @@ -39,8 +34,8 @@ cdef class PyrexScanner(Scanner): cdef public indentation_char cdef public int bracket_nesting_level cdef readonly bint async_enabled - cdef public sy - cdef public systring + cdef public unicode sy + cdef public systring # EncodedString cdef long current_level(self) #cpdef commentline(self, text) diff --git a/Cython/Compiler/Scanning.py b/Cython/Compiler/Scanning.py index f61144033..feb428638 100644 --- a/Cython/Compiler/Scanning.py +++ b/Cython/Compiler/Scanning.py @@ -1,4 +1,4 @@ -# cython: infer_types=True, language_level=3, py2_import=True, auto_pickle=False +# cython: infer_types=True, language_level=3, auto_pickle=False # # Cython Scanner # @@ -12,6 +12,7 @@ cython.declare(make_lexicon=object, lexicon=object, import os import platform +from unicodedata import normalize from .. import Utils from ..Plex.Scanners import Scanner @@ -51,25 +52,6 @@ pyx_reserved_words = py_reserved_words + [ ] -class Method(object): - - def __init__(self, name, **kwargs): - self.name = name - self.kwargs = kwargs or None - self.__name__ = name # for Plex tracing - - def __call__(self, stream, text): - method = getattr(stream, self.name) - # self.kwargs is almost always unused => avoid call overhead - return method(text, **self.kwargs) if self.kwargs is not None else method(text) - - def __copy__(self): - return self # immutable, no need to copy - - def __deepcopy__(self, memo): - return self # immutable, no need to copy - - #------------------------------------------------------------------ class CompileTimeScope(object): @@ -154,7 +136,7 @@ class SourceDescriptor(object): _escaped_description = None _cmp_name = '' def __str__(self): - assert False # To catch all places where a descriptor is used directly as a filename + assert False # To catch all places where a descriptor is used directly as a filename def set_file_type_from_name(self, filename): name, ext = os.path.splitext(filename) @@ -360,6 +342,13 @@ class PyrexScanner(Scanner): self.sy = '' self.next() + def normalize_ident(self, text): + try: + text.encode('ascii') # really just name.isascii but supports Python 2 and 3 + except UnicodeEncodeError: + text = normalize('NFKC', text) + self.produce(IDENT, text) + def commentline(self, text): if self.parse_comments: self.produce('commentline', text) @@ -463,7 +452,7 @@ class PyrexScanner(Scanner): systring = self.context.intern_ustring(systring) self.sy = sy self.systring = systring - if False: # debug_scanner: + if False: # debug_scanner: _, line, col = self.position() if not self.systring or self.sy == self.systring: t = self.sy diff --git a/Cython/Compiler/StringEncoding.py b/Cython/Compiler/StringEncoding.py index c37e8aab7..192fc3de3 100644 --- a/Cython/Compiler/StringEncoding.py +++ b/Cython/Compiler/StringEncoding.py @@ -138,6 +138,24 @@ class EncodedString(_unicode): def as_utf8_string(self): return bytes_literal(self.utf8encode(), 'utf8') + def as_c_string_literal(self): + # first encodes the string then produces a c string literal + if self.encoding is None: + s = self.as_utf8_string() + else: + s = bytes_literal(self.byteencode(), self.encoding) + return s.as_c_string_literal() + + if not hasattr(_unicode, "isascii"): + def isascii(self): + # not defined for Python3.7+ since the class already has it + try: + self.encode("ascii") + except UnicodeEncodeError: + return False + else: + return True + def string_contains_surrogates(ustring): """ @@ -211,6 +229,11 @@ class BytesLiteral(_bytes): value = split_string_literal(escape_byte_string(self)) return '"%s"' % value + if not hasattr(_bytes, "isascii"): + def isascii(self): + # already defined for Python3.7+ + return True + def bytes_literal(s, encoding): assert isinstance(s, bytes) @@ -226,6 +249,12 @@ def encoded_string(s, encoding): s.encoding = encoding return s +def encoded_string_or_bytes_literal(s, encoding): + if isinstance(s, bytes): + return bytes_literal(s, encoding) + else: + return encoded_string(s, encoding) + char_from_escape_sequence = { r'\a' : u'\a', @@ -291,7 +320,7 @@ def escape_byte_string(s): """ s = _replace_specials(s) try: - return s.decode("ASCII") # trial decoding: plain ASCII => done + return s.decode("ASCII") # trial decoding: plain ASCII => done except UnicodeDecodeError: pass if IS_PYTHON3: @@ -324,7 +353,7 @@ def split_string_literal(s, limit=2000): while start < len(s): end = start + limit if len(s) > end-4 and '\\' in s[end-4:end]: - end -= 4 - s[end-4:end].find('\\') # just before the backslash + end -= 4 - s[end-4:end].find('\\') # just before the backslash while s[end-1] == '\\': end -= 1 if end == start: diff --git a/Cython/Compiler/Symtab.py b/Cython/Compiler/Symtab.py index 7361a55ae..712a06a7a 100644 --- a/Cython/Compiler/Symtab.py +++ b/Cython/Compiler/Symtab.py @@ -42,6 +42,28 @@ def c_safe_identifier(cname): cname = Naming.pyrex_prefix + cname return cname +def punycodify_name(cname, mangle_with=None): + # if passed the mangle_with should be a byte string + # modified from PEP489 + try: + cname.encode('ascii') + except UnicodeEncodeError: + cname = cname.encode('punycode').replace(b'-', b'_').decode('ascii') + if mangle_with: + # sometimes it necessary to mangle unicode names alone where + # they'll be inserted directly into C, because the punycode + # transformation can turn them into invalid identifiers + cname = "%s_%s" % (mangle_with, cname) + elif cname.startswith(Naming.pyrex_prefix): + # a punycode name could also be a valid ascii variable name so + # change the prefix to distinguish + cname = cname.replace(Naming.pyrex_prefix, + Naming.pyunicode_identifier_prefix, 1) + + return cname + + + class BufferAux(object): writable_needed = False @@ -87,6 +109,7 @@ class Entry(object): # doc_cname string or None C const holding the docstring # getter_cname string C func for getting property # setter_cname string C func for setting or deleting property + # is_cproperty boolean Is an inline property of an external type # is_self_arg boolean Is the "self" arg of an exttype method # is_arg boolean Is the arg of a method # is_local boolean Is a local variable @@ -134,6 +157,8 @@ class Entry(object): # cf_used boolean Entry is used # is_fused_specialized boolean Whether this entry of a cdef or def function # is a specialization + # is_cgetter boolean Is a c-level getter function + # is_cpp_optional boolean Entry should be declared as std::optional (cpp_locals directive) # TODO: utility_code and utility_code_definition serves the same purpose... @@ -160,6 +185,7 @@ class Entry(object): is_cpp_class = 0 is_const = 0 is_property = 0 + is_cproperty = 0 doc_cname = None getter_cname = None setter_cname = None @@ -203,6 +229,8 @@ class Entry(object): error_on_uninitialized = False cf_used = True outer_entry = None + is_cgetter = False + is_cpp_optional = False def __init__(self, name, cname, type, pos = None, init = None): self.name = name @@ -238,6 +266,16 @@ class Entry(object): else: return NotImplemented + @property + def cf_is_reassigned(self): + return len(self.cf_assignments) > 1 + + def make_cpp_optional(self): + assert self.type.is_cpp_class + self.is_cpp_optional = True + assert not self.utility_code # we're not overwriting anything? + self.utility_code_definition = Code.UtilityCode.load_cached("OptionalLocals", "CppSupport.cpp") + class InnerEntry(Entry): """ @@ -262,6 +300,7 @@ class InnerEntry(Entry): self.cf_assignments = outermost_entry.cf_assignments self.cf_references = outermost_entry.cf_references self.overloaded_alternatives = outermost_entry.overloaded_alternatives + self.is_cpp_optional = outermost_entry.is_cpp_optional self.inner_entries.append(self) def __getattr__(self, name): @@ -308,7 +347,7 @@ class Scope(object): is_py_class_scope = 0 is_c_class_scope = 0 is_closure_scope = 0 - is_genexpr_scope = 0 + is_comprehension_scope = 0 is_passthrough = 0 is_cpp_class_scope = 0 is_property_scope = 0 @@ -347,7 +386,6 @@ class Scope(object): self.defined_c_classes = [] self.imported_c_classes = {} self.cname_to_entry = {} - self.string_to_entry = {} self.identifier_to_entry = {} self.num_to_entry = {} self.obj_to_entry = {} @@ -358,11 +396,11 @@ class Scope(object): def __deepcopy__(self, memo): return self - def merge_in(self, other, merge_unused=True, whitelist=None): + def merge_in(self, other, merge_unused=True, allowlist=None): # Use with care... entries = [] for name, entry in other.entries.items(): - if not whitelist or name in whitelist: + if not allowlist or name in allowlist: if entry.used or merge_unused: entries.append((name, entry)) @@ -390,7 +428,7 @@ class Scope(object): def mangle(self, prefix, name = None): if name: - return "%s%s%s" % (prefix, self.scope_prefix, name) + return punycodify_name("%s%s%s" % (prefix, self.scope_prefix, name)) else: return self.parent_scope.mangle(prefix, self.name) @@ -440,11 +478,12 @@ class Scope(object): # Create new entry, and add to dictionary if # name is not None. Reports a warning if already # declared. - if type.is_buffer and not isinstance(self, LocalScope): # and not is_type: + if type.is_buffer and not isinstance(self, LocalScope): # and not is_type: error(pos, 'Buffer types only allowed as function local variables') if not self.in_cinclude and cname and re.match("^_[_A-Z]+$", cname): - # See http://www.gnu.org/software/libc/manual/html_node/Reserved-Names.html#Reserved-Names + # See https://www.gnu.org/software/libc/manual/html_node/Reserved-Names.html#Reserved-Names warning(pos, "'%s' is a reserved name in C." % cname, -1) + entries = self.entries if name and name in entries and not shadow: old_entry = entries[name] @@ -487,8 +526,7 @@ class Scope(object): entries[name] = entry if type.is_memoryviewslice: - from . import MemoryView - entry.init = MemoryView.memslice_entry_init + entry.init = type.default_value entry.scope = self entry.visibility = visibility @@ -563,8 +601,10 @@ class Scope(object): cname = self.mangle(Naming.type_prefix, name) entry = self.lookup_here(name) if not entry: + in_cpp = self.is_cpp() type = PyrexTypes.CStructOrUnionType( - name, kind, scope, typedef_flag, cname, packed) + name, kind, scope, typedef_flag, cname, packed, + in_cpp = in_cpp) entry = self.declare_type(name, type, pos, cname, visibility = visibility, api = api, defining = scope is not None) @@ -650,12 +690,12 @@ class Scope(object): error(pos, "'%s' previously declared as '%s'" % ( entry.name, entry.visibility)) - def declare_enum(self, name, pos, cname, typedef_flag, - visibility = 'private', api = 0, create_wrapper = 0): + def declare_enum(self, name, pos, cname, scoped, typedef_flag, + visibility='private', api=0, create_wrapper=0, doc=None): if name: if not cname: if (self.in_cinclude or visibility == 'public' - or visibility == 'extern' or api): + or visibility == 'extern' or api): cname = name else: cname = self.mangle(Naming.type_prefix, name) @@ -663,13 +703,21 @@ class Scope(object): namespace = self.outer_scope.lookup(self.name).type else: namespace = None - type = PyrexTypes.CEnumType(name, cname, typedef_flag, namespace) + + if scoped: + type = PyrexTypes.CppScopedEnumType(name, cname, namespace, doc=doc) + else: + type = PyrexTypes.CEnumType(name, cname, typedef_flag, namespace, doc=doc) else: type = PyrexTypes.c_anon_enum_type entry = self.declare_type(name, type, pos, cname = cname, visibility = visibility, api = api) + if scoped: + entry.utility_code = Code.UtilityCode.load_cached("EnumClassDecl", "CppSupport.cpp") + self.use_entry_utility_code(entry) entry.create_wrapper = create_wrapper entry.enum_values = [] + self.sue_entries.append(entry) return entry @@ -685,10 +733,13 @@ class Scope(object): cname = name else: cname = self.mangle(Naming.var_prefix, name) - if type.is_cpp_class and visibility != 'extern': - type.check_nullary_constructor(pos) entry = self.declare(name, cname, type, pos, visibility) entry.is_variable = 1 + if type.is_cpp_class and visibility != 'extern': + if self.directives['cpp_locals']: + entry.make_cpp_optional() + else: + type.check_nullary_constructor(pos) if in_pxd and visibility != 'extern': entry.defined_in_pxd = 1 entry.used = 1 @@ -698,6 +749,7 @@ class Scope(object): return entry def declare_builtin(self, name, pos): + name = self.mangle_class_private_name(name) return self.outer_scope.declare_builtin(name, pos) def _declare_pyfunction(self, name, pos, visibility='extern', entry=None): @@ -719,7 +771,7 @@ class Scope(object): entry.type = py_object_type elif entry.type is not py_object_type: return self._declare_pyfunction(name, pos, visibility=visibility, entry=entry) - else: # declare entry stub + else: # declare entry stub self.declare_var(name, py_object_type, pos, visibility=visibility) entry = self.declare_var(None, py_object_type, pos, cname=name, visibility='private') @@ -736,7 +788,7 @@ class Scope(object): qualified_name = self.qualify_name(lambda_name) entry = self.declare(None, func_cname, py_object_type, pos, 'private') - entry.name = lambda_name + entry.name = EncodedString(lambda_name) entry.qualified_name = qualified_name entry.pymethdef_cname = pymethdef_cname entry.func_cname = func_cname @@ -769,7 +821,8 @@ class Scope(object): entry.cname = cname entry.func_cname = cname if visibility != 'private' and visibility != entry.visibility: - warning(pos, "Function '%s' previously declared as '%s', now as '%s'" % (name, entry.visibility, visibility), 1) + warning(pos, "Function '%s' previously declared as '%s', now as '%s'" % ( + name, entry.visibility, visibility), 1) if overridable != entry.is_overridable: warning(pos, "Function '%s' previously declared as '%s'" % ( name, 'cpdef' if overridable else 'cdef'), 1) @@ -786,7 +839,7 @@ class Scope(object): # it's safe to allow signature overrides for alt_entry in entry.all_alternatives(): if not alt_entry.cname or cname == alt_entry.cname: - break # cname not unique! + break # cname not unique! else: can_override = True if can_override: @@ -830,6 +883,23 @@ class Scope(object): type.entry = entry return entry + def declare_cgetter(self, name, return_type, pos=None, cname=None, + visibility="private", modifiers=(), defining=False, **cfunc_type_config): + assert all( + k in ('exception_value', 'exception_check', 'nogil', 'with_gil', 'is_const_method', 'is_static_method') + for k in cfunc_type_config + ) + cfunc_type = PyrexTypes.CFuncType( + return_type, + [PyrexTypes.CFuncTypeArg("self", self.parent_type, None)], + **cfunc_type_config) + entry = self.declare_cfunction( + name, cfunc_type, pos, cname=None, visibility=visibility, modifiers=modifiers, defining=defining) + entry.is_cgetter = True + if cname is not None: + entry.func_cname = cname + return entry + def add_cfunction(self, name, type, pos, cname, visibility, modifiers, inherited=False): # Add a C function entry without giving it a func_cname. entry = self.declare(name, cname, type, pos, visibility) @@ -875,12 +945,33 @@ class Scope(object): def lookup(self, name): # Look up name in this scope or an enclosing one. # Return None if not found. - return (self.lookup_here(name) - or (self.outer_scope and self.outer_scope.lookup(name)) - or None) + + mangled_name = self.mangle_class_private_name(name) + entry = (self.lookup_here(name) # lookup here also does mangling + or (self.outer_scope and self.outer_scope.lookup(mangled_name)) + or None) + if entry: + return entry + + # look up the original name in the outer scope + # Not strictly Python behaviour but see https://github.com/cython/cython/issues/3544 + entry = (self.outer_scope and self.outer_scope.lookup(name)) or None + if entry and entry.is_pyglobal: + self._emit_class_private_warning(entry.pos, name) + return entry def lookup_here(self, name): # Look up in this scope only, return None if not found. + + entry = self.entries.get(self.mangle_class_private_name(name), None) + if entry: + return entry + # Also check the unmangled name in the current scope + # (even if mangling should give us something else). + # This is to support things like global __foo which makes a declaration for __foo + return self.entries.get(name, None) + + def lookup_here_unmangled(self, name): return self.entries.get(name, None) def lookup_target(self, name): @@ -888,6 +979,10 @@ class Scope(object): # variable if not found. entry = self.lookup_here(name) if not entry: + entry = self.lookup_here_unmangled(name) + if entry and entry.is_pyglobal: + self._emit_class_private_warning(entry.pos, name) + if not entry: entry = self.declare_var(name, py_object_type, None) return entry @@ -897,6 +992,7 @@ class Scope(object): if entry.type.is_fused and self.fused_to_specific: return entry.type.specialize(self.fused_to_specific) return entry.type + return None def lookup_operator(self, operator, operands): if operands[0].type.is_cpp_class: @@ -904,8 +1000,7 @@ class Scope(object): method = obj_type.scope.lookup("operator%s" % operator) if method is not None: arg_types = [arg.type for arg in operands[1:]] - res = PyrexTypes.best_match([arg.type for arg in operands[1:]], - method.all_alternatives()) + res = PyrexTypes.best_match(arg_types, method.all_alternatives()) if res is not None: return res function = self.lookup("operator%s" % operator) @@ -915,7 +1010,7 @@ class Scope(object): # look-up nonmember methods listed within a class method_alternatives = [] - if len(operands)==2: # binary operators only + if len(operands) == 2: # binary operators only for n in range(2): if operands[n].type.is_cpp_class: obj_type = operands[n].type @@ -939,6 +1034,11 @@ class Scope(object): operands = [FakeOperand(pos, type=type) for type in types] return self.lookup_operator(operator, operands) + def _emit_class_private_warning(self, pos, name): + warning(pos, "Global name %s matched from within class scope " + "in contradiction to to Python 'class private name' rules. " + "This may change in a future release." % name, 1) + def use_utility_code(self, new_code): self.global_scope().use_utility_code(new_code) @@ -1027,8 +1127,7 @@ class BuiltinScope(Scope): # If python_equiv == "*", the Python equivalent has the same name # as the entry, otherwise it has the name specified by python_equiv. name = EncodedString(name) - entry = self.declare_cfunction(name, type, None, cname, visibility='extern', - utility_code=utility_code) + entry = self.declare_cfunction(name, type, None, cname, visibility='extern', utility_code=utility_code) if python_equiv: if python_equiv == "*": python_equiv = name @@ -1055,10 +1154,12 @@ class BuiltinScope(Scope): entry = self.declare_type(name, type, None, visibility='extern') entry.utility_code = utility_code - var_entry = Entry(name = entry.name, - type = self.lookup('type').type, # make sure "type" is the first type declared... - pos = entry.pos, - cname = entry.type.typeptr_cname) + var_entry = Entry( + name=entry.name, + type=self.lookup('type').type, # make sure "type" is the first type declared... + pos=entry.pos, + cname=entry.type.typeptr_cname, + ) var_entry.qualified_name = self.qualify_name(name) var_entry.is_variable = 1 var_entry.is_cglobal = 1 @@ -1104,7 +1205,7 @@ class BuiltinScope(Scope): "True": ["Py_True", py_object_type], } -const_counter = 1 # As a temporary solution for compiling code in pxds +const_counter = 1 # As a temporary solution for compiling code in pxds class ModuleScope(Scope): # module_name string Python name of the module @@ -1116,7 +1217,6 @@ class ModuleScope(Scope): # utility_code_list [UtilityCode] Queuing utility codes for forwarding to Code.py # c_includes {key: IncludeCode} C headers or verbatim code to be generated # See process_include() for more documentation - # string_to_entry {string : Entry} Map string const to entry # identifier_to_entry {string : Entry} Map identifier string const to entry # context Context # parent_module Scope Parent in the import namespace @@ -1239,7 +1339,7 @@ class ModuleScope(Scope): entry = self.declare(None, None, py_object_type, pos, 'private') if Options.cache_builtins and name not in Code.uncachable_builtins: entry.is_builtin = 1 - entry.is_const = 1 # cached + entry.is_const = 1 # cached entry.name = name entry.cname = Naming.builtin_prefix + name self.cached_builtins.append(entry) @@ -1364,7 +1464,7 @@ class ModuleScope(Scope): entry = self.lookup_here(name) if entry: if entry.is_pyglobal and entry.as_module is scope: - return entry # Already declared as the same module + return entry # Already declared as the same module if not (entry.is_pyglobal and not entry.as_module): # SAGE -- I put this here so Pyrex # cimport's work across directories. @@ -1388,7 +1488,7 @@ class ModuleScope(Scope): # object type, and not declared with cdef, it will live # in the module dictionary, otherwise it will be a C # global variable. - if not visibility in ('private', 'public', 'extern'): + if visibility not in ('private', 'public', 'extern'): error(pos, "Module-level variable cannot be declared %s" % visibility) if not is_cdef: if type is unspecified_type: @@ -1505,7 +1605,7 @@ class ModuleScope(Scope): if entry and not shadow: type = entry.type if not (entry.is_type and type.is_extension_type): - entry = None # Will cause redeclaration and produce an error + entry = None # Will cause redeclaration and produce an error else: scope = type.scope if typedef_flag and (not scope or scope.defined): @@ -1740,10 +1840,11 @@ class LocalScope(Scope): Scope.__init__(self, name, outer_scope, parent_scope) def mangle(self, prefix, name): - return prefix + name + return punycodify_name(prefix + name) def declare_arg(self, name, type, pos): # Add an entry for an argument of a function. + name = self.mangle_class_private_name(name) cname = self.mangle(Naming.var_prefix, name) entry = self.declare(name, cname, type, pos, 'private') entry.is_variable = 1 @@ -1757,6 +1858,7 @@ class LocalScope(Scope): def declare_var(self, name, type, pos, cname = None, visibility = 'private', api = 0, in_pxd = 0, is_cdef = 0): + name = self.mangle_class_private_name(name) # Add an entry for a local variable. if visibility in ('public', 'readonly'): error(pos, "Local variable cannot be declared %s" % visibility) @@ -1793,10 +1895,11 @@ class LocalScope(Scope): def lookup(self, name): # Look up name in this scope or an enclosing one. # Return None if not found. + entry = Scope.lookup(self, name) if entry is not None: entry_scope = entry.scope - while entry_scope.is_genexpr_scope: + while entry_scope.is_comprehension_scope: entry_scope = entry_scope.outer_scope if entry_scope is not self and entry_scope.is_closure_scope: if hasattr(entry.scope, "scope_class"): @@ -1824,19 +1927,21 @@ class LocalScope(Scope): elif entry.in_closure: entry.original_cname = entry.cname entry.cname = "%s->%s" % (Naming.cur_scope_cname, entry.cname) + if entry.type.is_cpp_class and entry.scope.directives['cpp_locals']: + entry.make_cpp_optional() -class GeneratorExpressionScope(Scope): - """Scope for generator expressions and comprehensions. As opposed - to generators, these can be easily inlined in some cases, so all +class ComprehensionScope(Scope): + """Scope for comprehensions (but not generator expressions, which use ClosureScope). + As opposed to generators, these can be easily inlined in some cases, so all we really need is a scope that holds the loop variable(s). """ - is_genexpr_scope = True + is_comprehension_scope = True def __init__(self, outer_scope): parent_scope = outer_scope # TODO: also ignore class scopes? - while parent_scope.is_genexpr_scope: + while parent_scope.is_comprehension_scope: parent_scope = parent_scope.parent_scope name = parent_scope.global_scope().next_id(Naming.genexpr_id_ref) Scope.__init__(self, name, outer_scope, parent_scope) @@ -1845,7 +1950,7 @@ class GeneratorExpressionScope(Scope): # Class/ExtType scopes are filled at class creation time, i.e. from the # module init function or surrounding function. - while outer_scope.is_genexpr_scope or outer_scope.is_c_class_scope or outer_scope.is_py_class_scope: + while outer_scope.is_comprehension_scope or outer_scope.is_c_class_scope or outer_scope.is_py_class_scope: outer_scope = outer_scope.outer_scope self.var_entries = outer_scope.var_entries # keep declarations outside outer_scope.subscopes.add(self) @@ -1860,7 +1965,7 @@ class GeneratorExpressionScope(Scope): # if the outer scope defines a type for this variable, inherit it outer_entry = self.outer_scope.lookup(name) if outer_entry and outer_entry.is_variable: - type = outer_entry.type # may still be 'unspecified_type' ! + type = outer_entry.type # may still be 'unspecified_type' ! # the parent scope needs to generate code for the variable, but # this scope must hold its name exclusively cname = '%s%s' % (self.genexp_prefix, self.parent_scope.mangle(Naming.var_prefix, name or self.next_id())) @@ -1954,6 +2059,14 @@ class ClassScope(Scope): # declared in the class # doc string or None Doc string + def mangle_class_private_name(self, name): + # a few utilitycode names need to specifically be ignored + if name and name.lower().startswith("__pyx_"): + return name + if name and name.startswith('__') and not name.endswith('__'): + name = EncodedString('_%s%s' % (self.class_name.lstrip('_'), name)) + return name + def __init__(self, name, outer_scope): Scope.__init__(self, name, outer_scope, outer_scope) self.class_name = name @@ -1987,22 +2100,10 @@ class PyClassScope(ClassScope): is_py_class_scope = 1 - def mangle_class_private_name(self, name): - return self.mangle_special_name(name) - - def mangle_special_name(self, name): - if name and name.startswith('__') and not name.endswith('__'): - name = EncodedString('_%s%s' % (self.class_name.lstrip('_'), name)) - return name - - def lookup_here(self, name): - name = self.mangle_special_name(name) - return ClassScope.lookup_here(self, name) - def declare_var(self, name, type, pos, cname = None, visibility = 'private', api = 0, in_pxd = 0, is_cdef = 0): - name = self.mangle_special_name(name) + name = self.mangle_class_private_name(name) if type is unspecified_type: type = py_object_type # Add an entry for a class attribute. @@ -2043,7 +2144,7 @@ class PyClassScope(ClassScope): class CClassScope(ClassScope): # Namespace of an extension type. # - # parent_type CClassType + # parent_type PyExtensionType # #typeobj_cname string or None # #objstruct_cname string # method_table_cname string @@ -2062,7 +2163,7 @@ class CClassScope(ClassScope): has_pyobject_attrs = False has_memoryview_attrs = False - has_cpp_class_attrs = False + has_cpp_constructable_attrs = False has_cyclic_pyobject_attrs = False defined = False implemented = False @@ -2087,6 +2188,22 @@ class CClassScope(ClassScope): return not self.parent_type.is_gc_simple return False + def needs_trashcan(self): + # If the trashcan directive is explicitly set to False, + # unconditionally disable the trashcan. + directive = self.directives.get('trashcan') + if directive is False: + return False + # If the directive is set to True and the class has Python-valued + # C attributes, then it should use the trashcan in tp_dealloc. + if directive and self.has_cyclic_pyobject_attrs: + return True + # Use the trashcan if the base class uses it + base_type = self.parent_type.base_type + if base_type and base_type.scope is not None: + return base_type.scope.needs_trashcan() + return self.parent_type.builtin_trashcan + def needs_tp_clear(self): """ Do we need to generate an implementation for the tp_clear slot? Can @@ -2116,6 +2233,7 @@ class CClassScope(ClassScope): def declare_var(self, name, type, pos, cname = None, visibility = 'private', api = 0, in_pxd = 0, is_cdef = 0): + name = self.mangle_class_private_name(name) if is_cdef: # Add an entry for an attribute. if self.defined: @@ -2130,16 +2248,20 @@ class CClassScope(ClassScope): cname = name if visibility == 'private': cname = c_safe_identifier(cname) - if type.is_cpp_class and visibility != 'extern': - type.check_nullary_constructor(pos) - self.use_utility_code(Code.UtilityCode("#include <new>")) + cname = punycodify_name(cname, Naming.unicode_structmember_prefix) entry = self.declare(name, cname, type, pos, visibility) entry.is_variable = 1 self.var_entries.append(entry) + if type.is_cpp_class and visibility != 'extern': + if self.directives['cpp_locals']: + entry.make_cpp_optional() + else: + type.check_nullary_constructor(pos) if type.is_memoryviewslice: self.has_memoryview_attrs = True - elif type.is_cpp_class: - self.has_cpp_class_attrs = True + elif type.needs_cpp_construction: + self.use_utility_code(Code.UtilityCode("#include <new>")) + self.has_cpp_constructable_attrs = True elif type.is_pyobject and (self.is_closure_class_scope or name != '__weakref__'): self.has_pyobject_attrs = True if (not type.is_builtin_type @@ -2169,10 +2291,11 @@ class CClassScope(ClassScope): cname=cname, visibility=visibility, api=api, in_pxd=in_pxd, is_cdef=is_cdef) entry.is_member = 1 - entry.is_pyglobal = 1 # xxx: is_pyglobal changes behaviour in so many places that - # I keep it in for now. is_member should be enough - # later on + # xxx: is_pyglobal changes behaviour in so many places that I keep it in for now. + # is_member should be enough later on + entry.is_pyglobal = 1 self.namespace_cname = "(PyObject *)%s" % self.parent_type.typeptr_cname + return entry def declare_pyfunction(self, name, pos, allow_redefine=False): @@ -2220,6 +2343,7 @@ class CClassScope(ClassScope): def declare_cfunction(self, name, type, pos, cname=None, visibility='private', api=0, in_pxd=0, defining=0, modifiers=(), utility_code=None, overridable=False): + name = self.mangle_class_private_name(name) if get_special_method_signature(name) and not self.parent_type.is_builtin_type: error(pos, "Special methods must be declared with 'def', not 'cdef'") args = type.args @@ -2231,7 +2355,7 @@ class CClassScope(ClassScope): (args[0].type, name, self.parent_type)) entry = self.lookup_here(name) if cname is None: - cname = c_safe_identifier(name) + cname = punycodify_name(c_safe_identifier(name), Naming.unicode_vtabentry_prefix) if entry: if not entry.is_cfunction: warning(pos, "'%s' redeclared " % name, 0) @@ -2246,13 +2370,14 @@ class CClassScope(ClassScope): entry.type = entry.type.with_with_gil(type.with_gil) elif type.compatible_signature_with(entry.type, as_cmethod = 1) and type.nogil == entry.type.nogil: if (self.defined and not in_pxd - and not type.same_c_signature_as_resolved_type(entry.type, as_cmethod = 1, as_pxd_definition = 1)): + and not type.same_c_signature_as_resolved_type( + entry.type, as_cmethod=1, as_pxd_definition=1)): # TODO(robertwb): Make this an error. warning(pos, "Compatible but non-identical C method '%s' not redeclared " "in definition part of extension type '%s'. " "This may cause incorrect vtables to be generated." % ( - name, self.class_name), 2) + name, self.class_name), 2) warning(entry.pos, "Previous declaration is here", 2) entry = self.add_cfunction(name, type, pos, cname, visibility='ignore', modifiers=modifiers) else: @@ -2272,8 +2397,7 @@ class CClassScope(ClassScope): if u'inline' in modifiers: entry.is_inline_cmethod = True - if (self.parent_type.is_final_type or entry.is_inline_cmethod or - self.directives.get('final')): + if self.parent_type.is_final_type or entry.is_inline_cmethod or self.directives.get('final'): entry.is_final_cmethod = True entry.final_func_cname = entry.func_cname @@ -2282,8 +2406,8 @@ class CClassScope(ClassScope): def add_cfunction(self, name, type, pos, cname, visibility, modifiers, inherited=False): # Add a cfunction entry without giving it a func_cname. prev_entry = self.lookup_here(name) - entry = ClassScope.add_cfunction(self, name, type, pos, cname, - visibility, modifiers, inherited=inherited) + entry = ClassScope.add_cfunction( + self, name, type, pos, cname, visibility, modifiers, inherited=inherited) entry.is_cmethod = 1 entry.prev_entry = prev_entry return entry @@ -2292,8 +2416,8 @@ class CClassScope(ClassScope): # overridden methods of builtin types still have their Python # equivalent that must be accessible to support bound methods name = EncodedString(name) - entry = self.declare_cfunction(name, type, None, cname, visibility='extern', - utility_code=utility_code) + entry = self.declare_cfunction( + name, type, pos=None, cname=cname, visibility='extern', utility_code=utility_code) var_entry = Entry(name, name, py_object_type) var_entry.qualified_name = name var_entry.is_variable = 1 @@ -2303,18 +2427,44 @@ class CClassScope(ClassScope): entry.as_variable = var_entry return entry - def declare_property(self, name, doc, pos): + def declare_property(self, name, doc, pos, ctype=None, property_scope=None): entry = self.lookup_here(name) if entry is None: - entry = self.declare(name, name, py_object_type, pos, 'private') - entry.is_property = 1 + entry = self.declare(name, name, py_object_type if ctype is None else ctype, pos, 'private') + entry.is_property = True + if ctype is not None: + entry.is_cproperty = True entry.doc = doc - entry.scope = PropertyScope(name, - outer_scope = self.global_scope(), parent_scope = self) - entry.scope.parent_type = self.parent_type + if property_scope is None: + entry.scope = PropertyScope(name, class_scope=self) + else: + entry.scope = property_scope self.property_entries.append(entry) return entry + def declare_cproperty(self, name, type, cfunc_name, doc=None, pos=None, visibility='extern', + nogil=False, with_gil=False, exception_value=None, exception_check=False, + utility_code=None): + """Internal convenience method to declare a C property function in one go. + """ + property_entry = self.declare_property(name, doc=doc, ctype=type, pos=pos) + cfunc_entry = property_entry.scope.declare_cfunction( + name=name, + type=PyrexTypes.CFuncType( + type, + [PyrexTypes.CFuncTypeArg("self", self.parent_type, pos=None)], + nogil=nogil, + with_gil=with_gil, + exception_value=exception_value, + exception_check=exception_check, + ), + cname=cfunc_name, + utility_code=utility_code, + visibility=visibility, + pos=pos, + ) + return property_entry, cfunc_entry + def declare_inherited_c_attributes(self, base_scope): # Declare entries for all the C attributes of an # inherited type, with cnames modified appropriately @@ -2343,9 +2493,9 @@ class CClassScope(ClassScope): is_builtin = var_entry and var_entry.is_builtin if not is_builtin: cname = adapt(cname) - entry = self.add_cfunction(base_entry.name, base_entry.type, - base_entry.pos, cname, - base_entry.visibility, base_entry.func_modifiers, inherited=True) + entry = self.add_cfunction( + base_entry.name, base_entry.type, base_entry.pos, cname, + base_entry.visibility, base_entry.func_modifiers, inherited=True) entry.is_inherited = 1 if base_entry.is_final_cmethod: entry.is_final_cmethod = True @@ -2409,7 +2559,7 @@ class CppClassScope(Scope): class_name = self.name.split('::')[-1] if name in (class_name, '__init__') and cname is None: cname = "%s__init__%s" % (Naming.func_prefix, class_name) - name = '<init>' + name = EncodedString('<init>') type.return_type = PyrexTypes.CVoidType() # This is called by the actual constructor, but need to support # arguments that cannot by called by value. @@ -2423,7 +2573,7 @@ class CppClassScope(Scope): type.args = [maybe_ref(arg) for arg in type.args] elif name == '__dealloc__' and cname is None: cname = "%s__dealloc__%s" % (Naming.func_prefix, class_name) - name = '<del>' + name = EncodedString('<del>') type.return_type = PyrexTypes.CVoidType() if name in ('<init>', '<del>') and type.nogil: for base in self.type.base_classes: @@ -2453,19 +2603,18 @@ class CppClassScope(Scope): # Declare entries for all the C++ attributes of an # inherited type, with cnames modified appropriately # to work with this type. - for base_entry in \ - base_scope.inherited_var_entries + base_scope.var_entries: - #constructor/destructor is not inherited - if base_entry.name in ("<init>", "<del>"): - continue - #print base_entry.name, self.entries - if base_entry.name in self.entries: - base_entry.name # FIXME: is there anything to do in this case? - entry = self.declare(base_entry.name, base_entry.cname, - base_entry.type, None, 'extern') - entry.is_variable = 1 - entry.is_inherited = 1 - self.inherited_var_entries.append(entry) + for base_entry in base_scope.inherited_var_entries + base_scope.var_entries: + #constructor/destructor is not inherited + if base_entry.name in ("<init>", "<del>"): + continue + #print base_entry.name, self.entries + if base_entry.name in self.entries: + base_entry.name # FIXME: is there anything to do in this case? + entry = self.declare(base_entry.name, base_entry.cname, + base_entry.type, None, 'extern') + entry.is_variable = 1 + entry.is_inherited = 1 + self.inherited_var_entries.append(entry) for base_entry in base_scope.cfunc_entries: entry = self.declare_cfunction(base_entry.name, base_entry.type, base_entry.pos, base_entry.cname, @@ -2507,6 +2656,22 @@ class CppClassScope(Scope): return scope +class CppScopedEnumScope(Scope): + # Namespace of a ScopedEnum + + def __init__(self, name, outer_scope): + Scope.__init__(self, name, outer_scope, None) + + def declare_var(self, name, type, pos, + cname=None, visibility='extern'): + # Add an entry for an attribute. + if not cname: + cname = name + entry = self.declare(name, cname, type, pos, visibility) + entry.is_variable = True + return entry + + class PropertyScope(Scope): # Scope holding the __get__, __set__ and __del__ methods for # a property of an extension type. @@ -2515,6 +2680,31 @@ class PropertyScope(Scope): is_property_scope = 1 + def __init__(self, name, class_scope): + # outer scope is None for some internal properties + outer_scope = class_scope.global_scope() if class_scope.outer_scope else None + Scope.__init__(self, name, outer_scope, parent_scope=class_scope) + self.parent_type = class_scope.parent_type + self.directives = class_scope.directives + + def declare_cfunction(self, name, type, pos, *args, **kwargs): + """Declare a C property function. + """ + if type.return_type.is_void: + error(pos, "C property method cannot return 'void'") + + if type.args and type.args[0].type is py_object_type: + # Set 'self' argument type to extension type. + type.args[0].type = self.parent_scope.parent_type + elif len(type.args) != 1: + error(pos, "C property method must have a single (self) argument") + elif not (type.args[0].type.is_pyobject or type.args[0].type is self.parent_scope.parent_type): + error(pos, "C property method must have a single (object) argument") + + entry = Scope.declare_cfunction(self, name, type, pos, *args, **kwargs) + entry.is_cproperty = True + return entry + def declare_pyfunction(self, name, pos, allow_redefine=False): # Add an entry for a method. signature = get_property_accessor_signature(name) @@ -2529,23 +2719,27 @@ class PropertyScope(Scope): return None -class CConstScope(Scope): +class CConstOrVolatileScope(Scope): - def __init__(self, const_base_type_scope): + def __init__(self, base_type_scope, is_const=0, is_volatile=0): Scope.__init__( self, - 'const_' + const_base_type_scope.name, - const_base_type_scope.outer_scope, - const_base_type_scope.parent_scope) - self.const_base_type_scope = const_base_type_scope + 'cv_' + base_type_scope.name, + base_type_scope.outer_scope, + base_type_scope.parent_scope) + self.base_type_scope = base_type_scope + self.is_const = is_const + self.is_volatile = is_volatile def lookup_here(self, name): - entry = self.const_base_type_scope.lookup_here(name) + entry = self.base_type_scope.lookup_here(name) if entry is not None: entry = copy.copy(entry) - entry.type = PyrexTypes.c_const_type(entry.type) + entry.type = PyrexTypes.c_const_or_volatile_type( + entry.type, self.is_const, self.is_volatile) return entry + class TemplateScope(Scope): def __init__(self, name, outer_scope): Scope.__init__(self, name, outer_scope, None) diff --git a/Cython/Compiler/Tests/TestBuffer.py b/Cython/Compiler/Tests/TestBuffer.py index fcb22b72f..2f653d0ff 100644 --- a/Cython/Compiler/Tests/TestBuffer.py +++ b/Cython/Compiler/Tests/TestBuffer.py @@ -21,7 +21,7 @@ class TestBufferParsing(CythonTest): def test_basic(self): t = self.parse(u"cdef object[float, 4, ndim=2, foo=foo] x") bufnode = t.stats[0].base_type - self.assert_(isinstance(bufnode, TemplatedTypeNode)) + self.assertTrue(isinstance(bufnode, TemplatedTypeNode)) self.assertEqual(2, len(bufnode.positional_args)) # print bufnode.dump() # should put more here... @@ -46,7 +46,7 @@ class TestBufferOptions(CythonTest): def nonfatal_error(self, error): # We're passing self as context to transform to trap this self.error = error - self.assert_(self.expect_error) + self.assertTrue(self.expect_error) def parse_opts(self, opts, expect_error=False): assert opts != "" @@ -55,14 +55,14 @@ class TestBufferOptions(CythonTest): root = self.fragment(s, pipeline=[NormalizeTree(self), PostParse(self)]).root if not expect_error: vardef = root.stats[0].body.stats[0] - assert isinstance(vardef, CVarDefNode) # use normal assert as this is to validate the test code + assert isinstance(vardef, CVarDefNode) # use normal assert as this is to validate the test code buftype = vardef.base_type - self.assert_(isinstance(buftype, TemplatedTypeNode)) - self.assert_(isinstance(buftype.base_type_node, CSimpleBaseTypeNode)) + self.assertTrue(isinstance(buftype, TemplatedTypeNode)) + self.assertTrue(isinstance(buftype.base_type_node, CSimpleBaseTypeNode)) self.assertEqual(u"object", buftype.base_type_node.name) return buftype else: - self.assert_(len(root.stats[0].body.stats) == 0) + self.assertTrue(len(root.stats[0].body.stats) == 0) def non_parse(self, expected_err, opts): self.parse_opts(opts, expect_error=True) @@ -71,14 +71,14 @@ class TestBufferOptions(CythonTest): def __test_basic(self): buf = self.parse_opts(u"unsigned short int, 3") - self.assert_(isinstance(buf.dtype_node, CSimpleBaseTypeNode)) - self.assert_(buf.dtype_node.signed == 0 and buf.dtype_node.longness == -1) + self.assertTrue(isinstance(buf.dtype_node, CSimpleBaseTypeNode)) + self.assertTrue(buf.dtype_node.signed == 0 and buf.dtype_node.longness == -1) self.assertEqual(3, buf.ndim) def __test_dict(self): buf = self.parse_opts(u"ndim=3, dtype=unsigned short int") - self.assert_(isinstance(buf.dtype_node, CSimpleBaseTypeNode)) - self.assert_(buf.dtype_node.signed == 0 and buf.dtype_node.longness == -1) + self.assertTrue(isinstance(buf.dtype_node, CSimpleBaseTypeNode)) + self.assertTrue(buf.dtype_node.signed == 0 and buf.dtype_node.longness == -1) self.assertEqual(3, buf.ndim) def __test_ndim(self): @@ -94,12 +94,12 @@ class TestBufferOptions(CythonTest): cdef object[ndim=ndim, dtype=int] y """, pipeline=[NormalizeTree(self), PostParse(self)]).root stats = t.stats[0].body.stats - self.assert_(stats[0].base_type.ndim == 3) - self.assert_(stats[1].base_type.ndim == 3) + self.assertTrue(stats[0].base_type.ndim == 3) + self.assertTrue(stats[1].base_type.ndim == 3) # add exotic and impossible combinations as they come along... + if __name__ == '__main__': import unittest unittest.main() - diff --git a/Cython/Compiler/Tests/TestCmdLine.py b/Cython/Compiler/Tests/TestCmdLine.py index abc7c0a89..5953112dc 100644 --- a/Cython/Compiler/Tests/TestCmdLine.py +++ b/Cython/Compiler/Tests/TestCmdLine.py @@ -1,4 +1,4 @@ - +import os import sys from unittest import TestCase try: @@ -9,19 +9,25 @@ except ImportError: from .. import Options from ..CmdLine import parse_command_line +from .Utils import backup_Options, restore_Options, check_global_options + class CmdLineParserTest(TestCase): def setUp(self): - backup = {} - for name, value in vars(Options).items(): - backup[name] = value - self._options_backup = backup + self._options_backup = backup_Options() def tearDown(self): + restore_Options(self._options_backup) + + def check_default_global_options(self, white_list=[]): + self.assertEqual(check_global_options(self._options_backup, white_list), "") + + def check_default_options(self, options, white_list=[]): + default_options = Options.CompilationOptions(Options.default_options) no_value = object() - for name, orig_value in self._options_backup.items(): - if getattr(Options, name, no_value) != orig_value: - setattr(Options, name, orig_value) + for name in default_options.__dict__.keys(): + if name not in white_list: + self.assertEqual(getattr(options, name, no_value), getattr(default_options, name), msg="error in option " + name) def test_short_options(self): options, sources = parse_command_line([ @@ -97,6 +103,397 @@ class CmdLineParserTest(TestCase): self.assertEqual(Options.annotate_coverage_xml, 'cov.xml') self.assertTrue(options.gdb_debug) self.assertEqual(options.output_dir, '/gdb/outdir') + self.assertEqual(options.compiler_directives['wraparound'], False) + + def test_embed_before_positional(self): + options, sources = parse_command_line([ + '--embed', + 'source.pyx', + ]) + self.assertEqual(sources, ['source.pyx']) + self.assertEqual(Options.embed, 'main') + + def test_two_embeds(self): + options, sources = parse_command_line([ + '--embed', '--embed=huhu', + 'source.pyx', + ]) + self.assertEqual(sources, ['source.pyx']) + self.assertEqual(Options.embed, 'huhu') + + def test_two_embeds2(self): + options, sources = parse_command_line([ + '--embed=huhu', '--embed', + 'source.pyx', + ]) + self.assertEqual(sources, ['source.pyx']) + self.assertEqual(Options.embed, 'main') + + def test_no_annotate(self): + options, sources = parse_command_line([ + '--embed=huhu', 'source.pyx' + ]) + self.assertFalse(Options.annotate) + + def test_annotate_short(self): + options, sources = parse_command_line([ + '-a', + 'source.pyx', + ]) + self.assertEqual(Options.annotate, 'default') + + def test_annotate_long(self): + options, sources = parse_command_line([ + '--annotate', + 'source.pyx', + ]) + self.assertEqual(Options.annotate, 'default') + + def test_annotate_fullc(self): + options, sources = parse_command_line([ + '--annotate-fullc', + 'source.pyx', + ]) + self.assertEqual(Options.annotate, 'fullc') + + def test_short_w(self): + options, sources = parse_command_line([ + '-w', 'my_working_path', + 'source.pyx' + ]) + self.assertEqual(options.working_path, 'my_working_path') + self.check_default_global_options() + self.check_default_options(options, ['working_path']) + + def test_short_o(self): + options, sources = parse_command_line([ + '-o', 'my_output', + 'source.pyx' + ]) + self.assertEqual(options.output_file, 'my_output') + self.check_default_global_options() + self.check_default_options(options, ['output_file']) + + def test_short_z(self): + options, sources = parse_command_line([ + '-z', 'my_preimport', + 'source.pyx' + ]) + self.assertEqual(Options.pre_import, 'my_preimport') + self.check_default_global_options(['pre_import']) + self.check_default_options(options) + + def test_convert_range(self): + options, sources = parse_command_line([ + '--convert-range', + 'source.pyx' + ]) + self.assertEqual(Options.convert_range, True) + self.check_default_global_options(['convert_range']) + self.check_default_options(options) + + def test_line_directives(self): + options, sources = parse_command_line([ + '--line-directives', + 'source.pyx' + ]) + self.assertEqual(options.emit_linenums, True) + self.check_default_global_options() + self.check_default_options(options, ['emit_linenums']) + + def test_no_c_in_traceback(self): + options, sources = parse_command_line([ + '--no-c-in-traceback', + 'source.pyx' + ]) + self.assertEqual(options.c_line_in_traceback, False) + self.check_default_global_options() + self.check_default_options(options, ['c_line_in_traceback']) + + def test_gdb(self): + options, sources = parse_command_line([ + '--gdb', + 'source.pyx' + ]) + self.assertEqual(options.gdb_debug, True) + self.assertEqual(options.output_dir, os.curdir) + self.check_default_global_options() + self.check_default_options(options, ['gdb_debug', 'output_dir']) + + def test_3str(self): + options, sources = parse_command_line([ + '--3str', + 'source.pyx' + ]) + self.assertEqual(options.language_level, '3str') + self.check_default_global_options() + self.check_default_options(options, ['language_level']) + + def test_capi_reexport_cincludes(self): + options, sources = parse_command_line([ + '--capi-reexport-cincludes', + 'source.pyx' + ]) + self.assertEqual(options.capi_reexport_cincludes, True) + self.check_default_global_options() + self.check_default_options(options, ['capi_reexport_cincludes']) + + def test_fast_fail(self): + options, sources = parse_command_line([ + '--fast-fail', + 'source.pyx' + ]) + self.assertEqual(Options.fast_fail, True) + self.check_default_global_options(['fast_fail']) + self.check_default_options(options) + + def test_cimport_from_pyx(self): + options, sources = parse_command_line([ + '--cimport-from-pyx', + 'source.pyx' + ]) + self.assertEqual(Options.cimport_from_pyx, True) + self.check_default_global_options(['cimport_from_pyx']) + self.check_default_options(options) + + def test_Werror(self): + options, sources = parse_command_line([ + '-Werror', + 'source.pyx' + ]) + self.assertEqual(Options.warning_errors, True) + self.check_default_global_options(['warning_errors']) + self.check_default_options(options) + + def test_warning_errors(self): + options, sources = parse_command_line([ + '--warning-errors', + 'source.pyx' + ]) + self.assertEqual(Options.warning_errors, True) + self.check_default_global_options(['warning_errors']) + self.check_default_options(options) + + def test_Wextra(self): + options, sources = parse_command_line([ + '-Wextra', + 'source.pyx' + ]) + self.assertEqual(options.compiler_directives, Options.extra_warnings) + self.check_default_global_options() + self.check_default_options(options, ['compiler_directives']) + + def test_warning_extra(self): + options, sources = parse_command_line([ + '--warning-extra', + 'source.pyx' + ]) + self.assertEqual(options.compiler_directives, Options.extra_warnings) + self.check_default_global_options() + self.check_default_options(options, ['compiler_directives']) + + def test_old_style_globals(self): + options, sources = parse_command_line([ + '--old-style-globals', + 'source.pyx' + ]) + self.assertEqual(Options.old_style_globals, True) + self.check_default_global_options(['old_style_globals']) + self.check_default_options(options) + + def test_directive_multiple(self): + options, source = parse_command_line([ + '-X', 'cdivision=True', + '-X', 'c_string_type=bytes', + 'source.pyx' + ]) + self.assertEqual(options.compiler_directives['cdivision'], True) + self.assertEqual(options.compiler_directives['c_string_type'], 'bytes') + self.check_default_global_options() + self.check_default_options(options, ['compiler_directives']) + + def test_directive_multiple_v2(self): + options, source = parse_command_line([ + '-X', 'cdivision=True,c_string_type=bytes', + 'source.pyx' + ]) + self.assertEqual(options.compiler_directives['cdivision'], True) + self.assertEqual(options.compiler_directives['c_string_type'], 'bytes') + self.check_default_global_options() + self.check_default_options(options, ['compiler_directives']) + + def test_directive_value_yes(self): + options, source = parse_command_line([ + '-X', 'cdivision=YeS', + 'source.pyx' + ]) + self.assertEqual(options.compiler_directives['cdivision'], True) + self.check_default_global_options() + self.check_default_options(options, ['compiler_directives']) + + def test_directive_value_no(self): + options, source = parse_command_line([ + '-X', 'cdivision=no', + 'source.pyx' + ]) + self.assertEqual(options.compiler_directives['cdivision'], False) + self.check_default_global_options() + self.check_default_options(options, ['compiler_directives']) + + def test_directive_value_invalid(self): + self.assertRaises(ValueError, parse_command_line, [ + '-X', 'cdivision=sadfasd', + 'source.pyx' + ]) + + def test_directive_key_invalid(self): + self.assertRaises(ValueError, parse_command_line, [ + '-X', 'abracadabra', + 'source.pyx' + ]) + + def test_directive_no_value(self): + self.assertRaises(ValueError, parse_command_line, [ + '-X', 'cdivision', + 'source.pyx' + ]) + + def test_compile_time_env_short(self): + options, source = parse_command_line([ + '-E', 'MYSIZE=10', + 'source.pyx' + ]) + self.assertEqual(options.compile_time_env['MYSIZE'], 10) + self.check_default_global_options() + self.check_default_options(options, ['compile_time_env']) + + def test_compile_time_env_long(self): + options, source = parse_command_line([ + '--compile-time-env', 'MYSIZE=10', + 'source.pyx' + ]) + self.assertEqual(options.compile_time_env['MYSIZE'], 10) + self.check_default_global_options() + self.check_default_options(options, ['compile_time_env']) + + def test_compile_time_env_multiple(self): + options, source = parse_command_line([ + '-E', 'MYSIZE=10', '-E', 'ARRSIZE=11', + 'source.pyx' + ]) + self.assertEqual(options.compile_time_env['MYSIZE'], 10) + self.assertEqual(options.compile_time_env['ARRSIZE'], 11) + self.check_default_global_options() + self.check_default_options(options, ['compile_time_env']) + + def test_compile_time_env_multiple_v2(self): + options, source = parse_command_line([ + '-E', 'MYSIZE=10,ARRSIZE=11', + 'source.pyx' + ]) + self.assertEqual(options.compile_time_env['MYSIZE'], 10) + self.assertEqual(options.compile_time_env['ARRSIZE'], 11) + self.check_default_global_options() + self.check_default_options(options, ['compile_time_env']) + + def test_option_first(self): + options, sources = parse_command_line(['-V', 'file.pyx']) + self.assertEqual(sources, ['file.pyx']) + + def test_file_inbetween(self): + options, sources = parse_command_line(['-V', 'file.pyx', '-a']) + self.assertEqual(sources, ['file.pyx']) + + def test_option_trailing(self): + options, sources = parse_command_line(['file.pyx', '-V']) + self.assertEqual(sources, ['file.pyx']) + + def test_multiple_files(self): + options, sources = parse_command_line([ + 'file1.pyx', '-V', + 'file2.pyx', '-a', + 'file3.pyx' + ]) + self.assertEqual(sources, ['file1.pyx', 'file2.pyx', 'file3.pyx']) + + def test_debug_flags(self): + options, sources = parse_command_line([ + '--debug-disposal-code', '--debug-coercion', + 'file3.pyx' + ]) + from Cython.Compiler import DebugFlags + for name in ['debug_disposal_code', 'debug_temp_alloc', 'debug_coercion']: + self.assertEqual(getattr(DebugFlags, name), name in ['debug_disposal_code', 'debug_coercion']) + setattr(DebugFlags, name, 0) # restore original value + + def test_gdb_overwrites_gdb_outdir(self): + options, sources = parse_command_line([ + '--gdb-outdir=my_dir', '--gdb', + 'file3.pyx' + ]) + self.assertEqual(options.gdb_debug, True) + self.assertEqual(options.output_dir, os.curdir) + self.check_default_global_options() + self.check_default_options(options, ['gdb_debug', 'output_dir']) + + def test_gdb_first(self): + options, sources = parse_command_line([ + '--gdb', '--gdb-outdir=my_dir', + 'file3.pyx' + ]) + self.assertEqual(options.gdb_debug, True) + self.assertEqual(options.output_dir, 'my_dir') + self.check_default_global_options() + self.check_default_options(options, ['gdb_debug', 'output_dir']) + + def test_coverage_overwrites_annotation(self): + options, sources = parse_command_line([ + '--annotate-fullc', '--annotate-coverage=my.xml', + 'file3.pyx' + ]) + self.assertEqual(Options.annotate, True) + self.assertEqual(Options.annotate_coverage_xml, 'my.xml') + self.check_default_global_options(['annotate', 'annotate_coverage_xml']) + self.check_default_options(options) + + def test_coverage_first(self): + options, sources = parse_command_line([ + '--annotate-coverage=my.xml', '--annotate-fullc', + 'file3.pyx' + ]) + self.assertEqual(Options.annotate, 'fullc') + self.assertEqual(Options.annotate_coverage_xml, 'my.xml') + self.check_default_global_options(['annotate', 'annotate_coverage_xml']) + self.check_default_options(options) + + def test_annotate_first_fullc_second(self): + options, sources = parse_command_line([ + '--annotate', '--annotate-fullc', + 'file3.pyx' + ]) + self.assertEqual(Options.annotate, 'fullc') + self.check_default_global_options(['annotate']) + self.check_default_options(options) + + def test_annotate_fullc_first(self): + options, sources = parse_command_line([ + '--annotate-fullc', '--annotate', + 'file3.pyx' + ]) + self.assertEqual(Options.annotate, 'default') + self.check_default_global_options(['annotate']) + self.check_default_options(options) + + def test_warning_extra_dont_overwrite(self): + options, sources = parse_command_line([ + '-X', 'cdivision=True', + '--warning-extra', + '-X', 'c_string_type=bytes', + 'source.pyx' + ]) + self.assertTrue(len(options.compiler_directives), len(Options.extra_warnings) + 1) + self.check_default_global_options() + self.check_default_options(options, ['compiler_directives']) def test_errors(self): def error(*args): @@ -116,3 +513,4 @@ class CmdLineParserTest(TestCase): error('--verbose=1') error('--verbose=1') error('--cleanup') + error('--debug-disposal-code-wrong-name', 'file3.pyx') diff --git a/Cython/Compiler/Tests/TestGrammar.py b/Cython/Compiler/Tests/TestGrammar.py index 3dddc960b..f80ec22d3 100644 --- a/Cython/Compiler/Tests/TestGrammar.py +++ b/Cython/Compiler/Tests/TestGrammar.py @@ -27,7 +27,15 @@ VALID_UNDERSCORE_LITERALS = [ '1e1_0', '.1_4', '.1_4e1', + '0b_0', + '0x_f', + '0o_5', + '1_00_00j', + '1_00_00.5j', + '1_00_00e5_1j', '.1_4j', + '(1_2.5+3_3j)', + '(.5_6j)', ] # Copied from CPython's test_grammar.py @@ -36,22 +44,29 @@ INVALID_UNDERSCORE_LITERALS = [ '0_', '42_', '1.4j_', + '0x_', '0b1_', '0xf_', '0o5_', + '0 if 1_Else 1', # Underscores in the base selector: '0_b0', '0_xf', '0_o5', - # Underscore right after the base selector: - '0b_0', - '0x_f', - '0o_5', # Old-style octal, still disallowed: - #'0_7', - #'09_99', - # Special case with exponent: - '0 if 1_Else 1', + # FIXME: still need to support PY_VERSION_HEX < 3 + '0_7', + '09_99', + # Multiple consecutive underscores: + '4_______2', + '0.1__4', + '0.1__4j', + '0b1001__0100', + '0xffff__ffff', + '0x___', + '0o5__77', + '1e1__0', + '1e1__0j', # Underscore right before a dot: '1_.4', '1_.4j', @@ -59,24 +74,24 @@ INVALID_UNDERSCORE_LITERALS = [ '1._4', '1._4j', '._5', + '._5j', # Underscore right after a sign: '1.0e+_1', - # Multiple consecutive underscores: - '4_______2', - '0.1__4', - '0b1001__0100', - '0xffff__ffff', - '0o5__77', - '1e1__0', + '1.0e+_1j', # Underscore right before j: '1.4_j', '1.4e5_j', # Underscore right before e: '1_e1', '1.4_e1', + '1.4_e1j', # Underscore right after e: '1e_1', '1.4e_1', + '1.4e_1j', + # Complex cases with parens: + '(1+1.5_j_)', + '(1+1.5_j)', # Whitespace in literals '1_ 2', '1 _2', @@ -117,11 +132,15 @@ class TestGrammar(CythonTest): # Add/MulNode() -> literal is first or second operand literal_node = literal_node.operand2 if i % 2 else literal_node.operand1 if 'j' in literal or 'J' in literal: - assert isinstance(literal_node, ExprNodes.ImagNode) + if '+' in literal: + # FIXME: tighten this test + assert isinstance(literal_node, ExprNodes.AddNode), (literal, literal_node) + else: + assert isinstance(literal_node, ExprNodes.ImagNode), (literal, literal_node) elif '.' in literal or 'e' in literal or 'E' in literal and not ('0x' in literal or '0X' in literal): - assert isinstance(literal_node, ExprNodes.FloatNode) + assert isinstance(literal_node, ExprNodes.FloatNode), (literal, literal_node) else: - assert isinstance(literal_node, ExprNodes.IntNode) + assert isinstance(literal_node, ExprNodes.IntNode), (literal, literal_node) if __name__ == "__main__": diff --git a/Cython/Compiler/Tests/TestMemView.py b/Cython/Compiler/Tests/TestMemView.py index 837bb8088..1d04a17fc 100644 --- a/Cython/Compiler/Tests/TestMemView.py +++ b/Cython/Compiler/Tests/TestMemView.py @@ -48,16 +48,16 @@ class TestMemviewParsing(CythonTest): def test_basic(self): t = self.parse(u"cdef int[:] x") memv_node = t.stats[0].base_type - self.assert_(isinstance(memv_node, MemoryViewSliceTypeNode)) + self.assertTrue(isinstance(memv_node, MemoryViewSliceTypeNode)) # we also test other similar declarations (buffers, anonymous C arrays) # since the parsing has to distinguish between them. - def disable_test_no_buf_arg(self): # TODO + def disable_test_no_buf_arg(self): # TODO self.not_parseable(u"Expected ']'", u"cdef extern foo(object[int, ndim=2])") - def disable_test_parse_sizeof(self): # TODO + def disable_test_parse_sizeof(self): # TODO self.parse(u"sizeof(int[NN])") self.parse(u"sizeof(int[])") self.parse(u"sizeof(int[][NN])") diff --git a/Cython/Compiler/Tests/TestParseTreeTransforms.py b/Cython/Compiler/Tests/TestParseTreeTransforms.py index 91e0e2f1d..e6889f8f2 100644 --- a/Cython/Compiler/Tests/TestParseTreeTransforms.py +++ b/Cython/Compiler/Tests/TestParseTreeTransforms.py @@ -3,7 +3,7 @@ import os from Cython.TestUtils import TransformTest from Cython.Compiler.ParseTreeTransforms import * from Cython.Compiler.Nodes import * -from Cython.Compiler import Main, Symtab +from Cython.Compiler import Main, Symtab, Options class TestNormalizeTree(TransformTest): @@ -87,9 +87,9 @@ class TestNormalizeTree(TransformTest): def test_pass_eliminated(self): t = self.run_pipeline([NormalizeTree(None)], u"pass") - self.assert_(len(t.stats) == 0) + self.assertTrue(len(t.stats) == 0) -class TestWithTransform(object): # (TransformTest): # Disabled! +class TestWithTransform(object): # (TransformTest): # Disabled! def test_simplified(self): t = self.run_pipeline([WithTransform(None)], u""" @@ -177,8 +177,8 @@ class TestInterpretCompilerDirectives(TransformTest): def setUp(self): super(TestInterpretCompilerDirectives, self).setUp() - compilation_options = Main.CompilationOptions(Main.default_options) - ctx = compilation_options.create_context() + compilation_options = Options.CompilationOptions(Options.default_options) + ctx = Main.Context.from_options(compilation_options) transform = InterpretCompilerDirectives(ctx, ctx.compiler_directives) transform.module_scope = Symtab.ModuleScope('__main__', None, ctx) diff --git a/Cython/Compiler/Tests/TestSignatureMatching.py b/Cython/Compiler/Tests/TestSignatureMatching.py index 166bb225b..57647c873 100644 --- a/Cython/Compiler/Tests/TestSignatureMatching.py +++ b/Cython/Compiler/Tests/TestSignatureMatching.py @@ -47,7 +47,7 @@ class SignatureMatcherTest(unittest.TestCase): self.assertMatches(function_types[1], [pt.c_long_type, pt.c_int_type], functions) def test_cpp_reference_cpp_class(self): - classes = [ cppclasstype("Test%d"%i, []) for i in range(2) ] + classes = [ cppclasstype("Test%d" % i, []) for i in range(2) ] function_types = [ cfunctype(pt.CReferenceType(classes[0])), cfunctype(pt.CReferenceType(classes[1])), @@ -58,7 +58,7 @@ class SignatureMatcherTest(unittest.TestCase): self.assertMatches(function_types[1], [classes[1]], functions) def test_cpp_reference_cpp_class_and_int(self): - classes = [ cppclasstype("Test%d"%i, []) for i in range(2) ] + classes = [ cppclasstype("Test%d" % i, []) for i in range(2) ] function_types = [ cfunctype(pt.CReferenceType(classes[0]), pt.c_int_type), cfunctype(pt.CReferenceType(classes[0]), pt.c_long_type), diff --git a/Cython/Compiler/Tests/TestTreeFragment.py b/Cython/Compiler/Tests/TestTreeFragment.py index 3f15b7457..9ee8da547 100644 --- a/Cython/Compiler/Tests/TestTreeFragment.py +++ b/Cython/Compiler/Tests/TestTreeFragment.py @@ -23,7 +23,7 @@ class TestTreeFragments(CythonTest): T = self.fragment(u"y + y").substitute({"y": NameNode(pos=None, name="x")}) self.assertEqual("x", T.stats[0].expr.operand1.name) self.assertEqual("x", T.stats[0].expr.operand2.name) - self.assert_(T.stats[0].expr.operand1 is not T.stats[0].expr.operand2) + self.assertTrue(T.stats[0].expr.operand1 is not T.stats[0].expr.operand2) def test_substitution(self): F = self.fragment(u"x = 4") @@ -35,7 +35,7 @@ class TestTreeFragments(CythonTest): F = self.fragment(u"PASS") pass_stat = PassStatNode(pos=None) T = F.substitute({"PASS" : pass_stat}) - self.assert_(isinstance(T.stats[0], PassStatNode), T) + self.assertTrue(isinstance(T.stats[0], PassStatNode), T) def test_pos_is_transferred(self): F = self.fragment(u""" @@ -55,9 +55,9 @@ class TestTreeFragments(CythonTest): """) T = F.substitute(temps=[u"TMP"]) s = T.body.stats - self.assert_(isinstance(s[0].expr, TempRefNode)) - self.assert_(isinstance(s[1].rhs, TempRefNode)) - self.assert_(s[0].expr.handle is s[1].rhs.handle) + self.assertTrue(isinstance(s[0].expr, TempRefNode)) + self.assertTrue(isinstance(s[1].rhs, TempRefNode)) + self.assertTrue(s[0].expr.handle is s[1].rhs.handle) if __name__ == "__main__": import unittest diff --git a/Cython/Compiler/Tests/TestTypes.py b/Cython/Compiler/Tests/TestTypes.py index f2f6f3773..4693dd806 100644 --- a/Cython/Compiler/Tests/TestTypes.py +++ b/Cython/Compiler/Tests/TestTypes.py @@ -17,3 +17,59 @@ class TestMethodDispatcherTransform(unittest.TestCase): cenum = PT.CEnumType("E", "cenum", typedef_flag=False) assert_widest(PT.c_int_type, cenum, PT.c_int_type) + + +class TestTypeIdentifiers(unittest.TestCase): + + TEST_DATA = [ + ("char*", "char__ptr"), + ("char *", "char__ptr"), + ("char **", "char__ptr__ptr"), + ("_typedef", "_typedef"), + ("__typedef", "__dundertypedef"), + ("___typedef", "__dunder_typedef"), + ("____typedef", "__dunder__dundertypedef"), + ("_____typedef", "__dunder__dunder_typedef"), + ("const __typedef", "__const___dundertypedef"), + ("int[42]", "int__lArr42__rArr"), + ("int[:]", "int__lArr__D__rArr"), + ("int[:,:]", "int__lArr__D__comma___D__rArr"), + ("int[:,:,:]", "int__lArr__D__comma___D__comma___D__rArr"), + ("int[:,:,...]", "int__lArr__D__comma___D__comma___EL__rArr"), + ("std::vector", "std__in_vector"), + ("std::vector&&", "std__in_vector__fwref"), + ("const std::vector", "__const_std__in_vector"), + ("const std::vector&", "__const_std__in_vector__ref"), + ("const_std", "const_std"), + ] + + def test_escape_special_type_characters(self): + test_func = PT._escape_special_type_characters # keep test usage visible for IDEs + function_name = "_escape_special_type_characters" + self._test_escape(function_name) + + def test_type_identifier_for_declaration(self): + test_func = PT.type_identifier_from_declaration # keep test usage visible for IDEs + function_name = test_func.__name__ + self._test_escape(function_name) + + # differences due to whitespace removal + test_data = [ + ("const &std::vector", "const__refstd__in_vector"), + ("const &std::vector<int>", "const__refstd__in_vector__lAngint__rAng"), + ("const &&std::vector", "const__fwrefstd__in_vector"), + ("const &&&std::vector", "const__fwref__refstd__in_vector"), + ("const &&std::vector", "const__fwrefstd__in_vector"), + ("void (*func)(int x, float y)", + "975d51__void__lParen__ptrfunc__rParen__lParenint__space_x__comma_float__space_y__rParen__etc"), + ("float ** (*func)(int x, int[:] y)", + "31883a__float__ptr__ptr__lParen__ptrfunc__rParen__lParenint__space_x__comma_int__lArr__D__rArry__rParen__etc"), + ] + self._test_escape(function_name, test_data) + + def _test_escape(self, func_name, test_data=TEST_DATA): + escape = getattr(PT, func_name) + for declaration, expected in test_data: + escaped_value = escape(declaration) + self.assertEqual(escaped_value, expected, "%s('%s') == '%s' != '%s'" % ( + func_name, declaration, escaped_value, expected)) diff --git a/Cython/Compiler/Tests/TestUtilityLoad.py b/Cython/Compiler/Tests/TestUtilityLoad.py index 3d1906ca0..44ae461ab 100644 --- a/Cython/Compiler/Tests/TestUtilityLoad.py +++ b/Cython/Compiler/Tests/TestUtilityLoad.py @@ -22,14 +22,11 @@ class TestUtilityLoader(unittest.TestCase): cls = Code.UtilityCode def test_load_as_string(self): - got = strip_2tup(self.cls.load_as_string(self.name)) - self.assertEqual(got, self.expected) - got = strip_2tup(self.cls.load_as_string(self.name, self.filename)) self.assertEqual(got, self.expected) def test_load(self): - utility = self.cls.load(self.name) + utility = self.cls.load(self.name, from_file=self.filename) got = strip_2tup((utility.proto, utility.impl)) self.assertEqual(got, self.expected) @@ -37,10 +34,6 @@ class TestUtilityLoader(unittest.TestCase): got = strip_2tup((required.proto, required.impl)) self.assertEqual(got, self.required) - utility = self.cls.load(self.name, from_file=self.filename) - got = strip_2tup((utility.proto, utility.impl)) - self.assertEqual(got, self.expected) - utility = self.cls.load_cached(self.name, from_file=self.filename) got = strip_2tup((utility.proto, utility.impl)) self.assertEqual(got, self.expected) @@ -59,11 +52,11 @@ class TestTempitaUtilityLoader(TestUtilityLoader): cls = Code.TempitaUtilityCode def test_load_as_string(self): - got = strip_2tup(self.cls.load_as_string(self.name, context=self.context)) + got = strip_2tup(self.cls.load_as_string(self.name, self.filename, context=self.context)) self.assertEqual(got, self.expected_tempita) def test_load(self): - utility = self.cls.load(self.name, context=self.context) + utility = self.cls.load(self.name, self.filename, context=self.context) got = strip_2tup((utility.proto, utility.impl)) self.assertEqual(got, self.expected_tempita) diff --git a/Cython/Compiler/Tests/Utils.py b/Cython/Compiler/Tests/Utils.py new file mode 100644 index 000000000..a158ecc50 --- /dev/null +++ b/Cython/Compiler/Tests/Utils.py @@ -0,0 +1,36 @@ +import copy + +from .. import Options + + +def backup_Options(): + backup = {} + for name, value in vars(Options).items(): + # we need a deep copy of _directive_defaults, because they can be changed + if name == '_directive_defaults': + value = copy.deepcopy(value) + backup[name] = value + return backup + + +def restore_Options(backup): + no_value = object() + for name, orig_value in backup.items(): + if getattr(Options, name, no_value) != orig_value: + setattr(Options, name, orig_value) + # strip Options from new keys that might have been added: + for name in vars(Options).keys(): + if name not in backup: + delattr(Options, name) + + +def check_global_options(expected_options, white_list=[]): + """ + returns error message of "" if check Ok + """ + no_value = object() + for name, orig_value in expected_options.items(): + if name not in white_list: + if getattr(Options, name, no_value) != orig_value: + return "error in option " + name + return "" diff --git a/Cython/Compiler/TreeFragment.py b/Cython/Compiler/TreeFragment.py index b85da8191..cef4469b5 100644 --- a/Cython/Compiler/TreeFragment.py +++ b/Cython/Compiler/TreeFragment.py @@ -29,8 +29,7 @@ class StringParseContext(Main.Context): include_directories = [] if compiler_directives is None: compiler_directives = {} - # TODO: see if "language_level=3" also works for our internal code here. - Main.Context.__init__(self, include_directories, compiler_directives, cpp=cpp, language_level=2) + Main.Context.__init__(self, include_directories, compiler_directives, cpp=cpp, language_level='3str') self.module_name = name def find_module(self, module_name, relative_to=None, pos=None, need_pxd=1, absolute_fallback=True): @@ -179,7 +178,7 @@ class TemplateTransform(VisitorTransform): if pos is None: pos = node.pos return ApplyPositionAndCopy(pos)(sub) else: - return self.visit_Node(node) # make copy as usual + return self.visit_Node(node) # make copy as usual def visit_NameNode(self, node): temphandle = self.tempmap.get(node.name) @@ -235,7 +234,7 @@ class TreeFragment(object): fmt_pxds[key] = fmt(value) mod = t = parse_from_strings(name, fmt_code, fmt_pxds, level=level, initial_pos=initial_pos) if level is None: - t = t.body # Make sure a StatListNode is at the top + t = t.body # Make sure a StatListNode is at the top if not isinstance(t, StatListNode): t = StatListNode(pos=mod.pos, stats=[t]) for transform in pipeline: diff --git a/Cython/Compiler/TypeInference.py b/Cython/Compiler/TypeInference.py index c7ffee7d2..4ae3ab155 100644 --- a/Cython/Compiler/TypeInference.py +++ b/Cython/Compiler/TypeInference.py @@ -140,7 +140,6 @@ class MarkParallelAssignments(EnvTransform): '+', sequence.args[0], sequence.args[2])) - if not is_special: # A for-loop basically translates to subsequent calls to # __getitem__(), so using an IndexNode here allows us to @@ -178,7 +177,7 @@ class MarkParallelAssignments(EnvTransform): return node def visit_FromCImportStatNode(self, node): - pass # Can't be assigned to... + return node # Can't be assigned to... def visit_FromImportStatNode(self, node): for name, target in node.items: @@ -308,10 +307,10 @@ class MarkOverflowingArithmetic(CythonTransform): def visit_SimpleCallNode(self, node): if node.function.is_name and node.function.name == 'abs': - # Overflows for minimum value of fixed size ints. - return self.visit_dangerous_node(node) + # Overflows for minimum value of fixed size ints. + return self.visit_dangerous_node(node) else: - return self.visit_neutral_node(node) + return self.visit_neutral_node(node) visit_UnopNode = visit_neutral_node @@ -359,10 +358,17 @@ class SimpleAssignmentTypeInferer(object): Note: in order to support cross-closure type inference, this must be applies to nested scopes in top-down order. """ - def set_entry_type(self, entry, entry_type): - entry.type = entry_type + def set_entry_type(self, entry, entry_type, scope): for e in entry.all_entries(): e.type = entry_type + if e.type.is_memoryviewslice: + # memoryview slices crash if they don't get initialized + e.init = e.type.default_value + if e.type.is_cpp_class: + if scope.directives['cpp_locals']: + e.make_cpp_optional() + else: + e.type.check_nullary_constructor(entry.pos) def infer_types(self, scope): enabled = scope.directives['infer_types'] @@ -370,12 +376,12 @@ class SimpleAssignmentTypeInferer(object): if enabled == True: spanning_type = aggressive_spanning_type - elif enabled is None: # safe mode + elif enabled is None: # safe mode spanning_type = safe_spanning_type else: for entry in scope.entries.values(): if entry.type is unspecified_type: - self.set_entry_type(entry, py_object_type) + self.set_entry_type(entry, py_object_type, scope) return # Set of assignments @@ -404,7 +410,7 @@ class SimpleAssignmentTypeInferer(object): else: entry = node.entry node_type = spanning_type( - types, entry.might_overflow, entry.pos, scope) + types, entry.might_overflow, scope) node.inferred_type = node_type def infer_name_node_type_partial(node): @@ -413,7 +419,7 @@ class SimpleAssignmentTypeInferer(object): if not types: return entry = node.entry - return spanning_type(types, entry.might_overflow, entry.pos, scope) + return spanning_type(types, entry.might_overflow, scope) def inferred_types(entry): has_none = False @@ -488,9 +494,9 @@ class SimpleAssignmentTypeInferer(object): types = inferred_types(entry) if types and all(types): entry_type = spanning_type( - types, entry.might_overflow, entry.pos, scope) + types, entry.might_overflow, scope) inferred.add(entry) - self.set_entry_type(entry, entry_type) + self.set_entry_type(entry, entry_type, scope) def reinfer(): dirty = False @@ -498,9 +504,9 @@ class SimpleAssignmentTypeInferer(object): for assmt in entry.cf_assignments: assmt.infer_type() types = inferred_types(entry) - new_type = spanning_type(types, entry.might_overflow, entry.pos, scope) + new_type = spanning_type(types, entry.might_overflow, scope) if new_type != entry.type: - self.set_entry_type(entry, new_type) + self.set_entry_type(entry, new_type, scope) dirty = True return dirty @@ -530,22 +536,20 @@ def find_spanning_type(type1, type2): return PyrexTypes.c_double_type return result_type -def simply_type(result_type, pos): +def simply_type(result_type): if result_type.is_reference: result_type = result_type.ref_base_type - if result_type.is_const: - result_type = result_type.const_base_type - if result_type.is_cpp_class: - result_type.check_nullary_constructor(pos) + if result_type.is_cv_qualified: + result_type = result_type.cv_base_type if result_type.is_array: result_type = PyrexTypes.c_ptr_type(result_type.base_type) return result_type -def aggressive_spanning_type(types, might_overflow, pos, scope): - return simply_type(reduce(find_spanning_type, types), pos) +def aggressive_spanning_type(types, might_overflow, scope): + return simply_type(reduce(find_spanning_type, types)) -def safe_spanning_type(types, might_overflow, pos, scope): - result_type = simply_type(reduce(find_spanning_type, types), pos) +def safe_spanning_type(types, might_overflow, scope): + result_type = simply_type(reduce(find_spanning_type, types)) if result_type.is_pyobject: # In theory, any specific Python type is always safe to # infer. However, inferring str can cause some existing code @@ -577,6 +581,8 @@ def safe_spanning_type(types, might_overflow, pos, scope): # used, won't arise in pure Python, and there shouldn't be side # effects, so I'm declaring this safe. return result_type + elif result_type.is_memoryviewslice: + return result_type # TODO: double complex should be OK as well, but we need # to make sure everything is supported. elif (result_type.is_int or result_type.is_enum) and not might_overflow: diff --git a/Cython/Compiler/TypeSlots.py b/Cython/Compiler/TypeSlots.py index 137ea4eba..c260ada1d 100644 --- a/Cython/Compiler/TypeSlots.py +++ b/Cython/Compiler/TypeSlots.py @@ -9,6 +9,8 @@ from . import Naming from . import PyrexTypes from .Errors import error +import copy + invisible = ['__cinit__', '__dealloc__', '__richcmp__', '__nonzero__', '__bool__'] @@ -23,6 +25,7 @@ class Signature(object): # fixed_arg_format string # ret_format string # error_value string + # use_fastcall boolean # # The formats are strings made up of the following # characters: @@ -86,20 +89,24 @@ class Signature(object): 'z': "-1", } - def __init__(self, arg_format, ret_format): - self.has_dummy_arg = 0 - self.has_generic_args = 0 + # Use METH_FASTCALL instead of METH_VARARGS + use_fastcall = False + + def __init__(self, arg_format, ret_format, nogil=False): + self.has_dummy_arg = False + self.has_generic_args = False if arg_format[:1] == '-': - self.has_dummy_arg = 1 + self.has_dummy_arg = True arg_format = arg_format[1:] if arg_format[-1:] == '*': - self.has_generic_args = 1 + self.has_generic_args = True arg_format = arg_format[:-1] self.fixed_arg_format = arg_format self.ret_format = ret_format self.error_value = self.error_value_map.get(ret_format, None) self.exception_check = ret_format != 'r' and self.error_value is not None self.is_staticmethod = False + self.nogil = nogil def __repr__(self): return '<Signature[%s(%s%s)]>' % ( @@ -149,7 +156,8 @@ class Signature(object): exc_value = self.exception_value() return PyrexTypes.CFuncType( ret_type, args, exception_value=exc_value, - exception_check=self.exception_check) + exception_check=self.exception_check, + nogil=self.nogil) def method_flags(self): if self.ret_format == "O": @@ -157,17 +165,50 @@ class Signature(object): if self.has_dummy_arg: full_args = "O" + full_args if full_args in ["O", "T"]: - if self.has_generic_args: - return [method_varargs, method_keywords] - else: + if not self.has_generic_args: return [method_noargs] + elif self.use_fastcall: + return [method_fastcall, method_keywords] + else: + return [method_varargs, method_keywords] elif full_args in ["OO", "TO"] and not self.has_generic_args: return [method_onearg] if self.is_staticmethod: - return [method_varargs, method_keywords] + if self.use_fastcall: + return [method_fastcall, method_keywords] + else: + return [method_varargs, method_keywords] + return None + + def method_function_type(self): + # Return the C function type + mflags = self.method_flags() + kw = "WithKeywords" if (method_keywords in mflags) else "" + for m in mflags: + if m == method_noargs or m == method_onearg: + return "PyCFunction" + if m == method_varargs: + return "PyCFunction" + kw + if m == method_fastcall: + return "__Pyx_PyCFunction_FastCall" + kw return None + def with_fastcall(self): + # Return a copy of this Signature with use_fastcall=True + sig = copy.copy(self) + sig.use_fastcall = True + return sig + + @property + def fastvar(self): + # Used to select variants of functions, one dealing with METH_VARARGS + # and one dealing with __Pyx_METH_FASTCALL + if self.use_fastcall: + return "FASTCALL" + else: + return "VARARGS" + class SlotDescriptor(object): # Abstract base class for type slot descriptors. @@ -180,13 +221,20 @@ class SlotDescriptor(object): # ifdef Full #ifdef string that slot is wrapped in. Using this causes py3, py2 and flags to be ignored.) def __init__(self, slot_name, dynamic=False, inherited=False, - py3=True, py2=True, ifdef=None): + py3=True, py2=True, ifdef=None, is_binop=False): self.slot_name = slot_name self.is_initialised_dynamically = dynamic self.is_inherited = inherited self.ifdef = ifdef self.py3 = py3 self.py2 = py2 + self.is_binop = is_binop + + def slot_code(self, scope): + raise NotImplemented() + + def spec_value(self, scope): + return self.slot_code(scope) def preprocessor_guard_code(self): ifdef = self.ifdef @@ -201,6 +249,19 @@ class SlotDescriptor(object): guard = ("#if PY_MAJOR_VERSION >= 3") return guard + def generate_spec(self, scope, code): + if self.is_initialised_dynamically: + return + value = self.spec_value(scope) + if value == "0": + return + preprocessor_guard = self.preprocessor_guard_code() + if preprocessor_guard: + code.putln(preprocessor_guard) + code.putln("{Py_%s, (void *)%s}," % (self.slot_name, value)) + if preprocessor_guard: + code.putln("#endif") + def generate(self, scope, code): preprocessor_guard = self.preprocessor_guard_code() if preprocessor_guard: @@ -215,7 +276,7 @@ class SlotDescriptor(object): # PyPy currently has a broken PyType_Ready() that fails to # inherit some slots. To work around this, we explicitly # set inherited slots here, but only in PyPy since CPython - # handles this better than we do. + # handles this better than we do (except for buffer slots in type specs). inherited_value = value current_scope = scope while (inherited_value == "0" @@ -225,7 +286,9 @@ class SlotDescriptor(object): current_scope = current_scope.parent_type.base_type.scope inherited_value = self.slot_code(current_scope) if inherited_value != "0": - code.putln("#if CYTHON_COMPILING_IN_PYPY") + # we always need inherited buffer slots for the type spec + is_buffer_slot = int(self.slot_name in ("bf_getbuffer", "bf_releasebuffer")) + code.putln("#if CYTHON_COMPILING_IN_PYPY || %d" % is_buffer_slot) code.putln("%s, /*%s*/" % (inherited_value, self.slot_name)) code.putln("#else") end_pypy_guard = True @@ -248,14 +311,20 @@ class SlotDescriptor(object): def generate_dynamic_init_code(self, scope, code): if self.is_initialised_dynamically: - value = self.slot_code(scope) - if value != "0": - code.putln("%s.%s = %s;" % ( - scope.parent_type.typeobj_cname, - self.slot_name, - value - ) - ) + self.generate_set_slot_code( + self.slot_code(scope), scope, code) + + def generate_set_slot_code(self, value, scope, code): + if value == "0": + return + + if scope.parent_type.typeptr_cname: + target = "%s->%s" % (scope.parent_type.typeptr_cname, self.slot_name) + else: + assert scope.parent_type.typeobj_cname + target = "%s.%s" % (scope.parent_type.typeobj_cname, self.slot_name) + + code.putln("%s = %s;" % (target, value)) class FixedSlot(SlotDescriptor): @@ -360,30 +429,61 @@ class GCClearReferencesSlot(GCDependentSlot): class ConstructorSlot(InternalMethodSlot): # Descriptor for tp_new and tp_dealloc. - def __init__(self, slot_name, method, **kargs): + def __init__(self, slot_name, method=None, **kargs): InternalMethodSlot.__init__(self, slot_name, **kargs) self.method = method - def slot_code(self, scope): - entry = scope.lookup_here(self.method) - if (self.slot_name != 'tp_new' - and scope.parent_type.base_type + def _needs_own(self, scope): + if (scope.parent_type.base_type and not scope.has_pyobject_attrs and not scope.has_memoryview_attrs - and not scope.has_cpp_class_attrs - and not (entry and entry.is_special)): + and not scope.has_cpp_constructable_attrs + and not (self.slot_name == 'tp_new' and scope.parent_type.vtabslot_cname)): + entry = scope.lookup_here(self.method) if self.method else None + if not (entry and entry.is_special): + return False + # Unless we can safely delegate to the parent, all types need a tp_new(). + return True + + def _parent_slot_function(self, scope): + parent_type_scope = scope.parent_type.base_type.scope + if scope.parent_scope is parent_type_scope.parent_scope: + entry = scope.parent_scope.lookup_here(scope.parent_type.base_type.name) + if entry.visibility != 'extern': + return self.slot_code(parent_type_scope) + return None + + def slot_code(self, scope): + if not self._needs_own(scope): # if the type does not have object attributes, it can # delegate GC methods to its parent - iff the parent # functions are defined in the same module - parent_type_scope = scope.parent_type.base_type.scope - if scope.parent_scope is parent_type_scope.parent_scope: - entry = scope.parent_scope.lookup_here(scope.parent_type.base_type.name) - if entry.visibility != 'extern': - return self.slot_code(parent_type_scope) - if entry and not entry.is_special: - return "0" + slot_code = self._parent_slot_function(scope) + return slot_code or '0' return InternalMethodSlot.slot_code(self, scope) + def spec_value(self, scope): + slot_function = self.slot_code(scope) + if self.slot_name == "tp_dealloc" and slot_function != scope.mangle_internal("tp_dealloc"): + # Not used => inherit from base type. + return "0" + return slot_function + + def generate_dynamic_init_code(self, scope, code): + if self.slot_code(scope) != '0': + return + # If we don't have our own slot function and don't know the + # parent function statically, copy it dynamically. + base_type = scope.parent_type.base_type + if base_type.typeptr_cname: + src = '%s->%s' % (base_type.typeptr_cname, self.slot_name) + elif base_type.is_extension_type and base_type.typeobj_cname: + src = '%s.%s' % (base_type.typeobj_cname, self.slot_name) + else: + return + + self.generate_set_slot_code(src, scope, code) + class SyntheticSlot(InternalMethodSlot): # Type slot descriptor for a synthesized method which @@ -404,6 +504,20 @@ class SyntheticSlot(InternalMethodSlot): else: return self.default_value + def spec_value(self, scope): + return self.slot_code(scope) + + +class BinopSlot(SyntheticSlot): + def __init__(self, signature, slot_name, left_method, **kargs): + assert left_method.startswith('__') + right_method = '__r' + left_method[2:] + SyntheticSlot.__init__( + self, slot_name, [left_method, right_method], "0", is_binop=True, **kargs) + # MethodSlot causes special method registration. + self.left_slot = MethodSlot(signature, "", left_method) + self.right_slot = MethodSlot(signature, "", right_method) + class RichcmpSlot(MethodSlot): def slot_code(self, scope): @@ -434,6 +548,10 @@ class TypeFlagsSlot(SlotDescriptor): value += "|Py_TPFLAGS_HAVE_GC" return value + def generate_spec(self, scope, code): + # Flags are stored in the PyType_Spec, not in a PyType_Slot. + return + class DocStringSlot(SlotDescriptor): # Descriptor for the docstring slot. @@ -444,7 +562,7 @@ class DocStringSlot(SlotDescriptor): return "0" if doc.is_unicode: doc = doc.as_utf8_string() - return doc.as_c_string_literal() + return "PyDoc_STR(%s)" % doc.as_c_string_literal() class SuiteSlot(SlotDescriptor): @@ -487,6 +605,13 @@ class SuiteSlot(SlotDescriptor): if self.ifdef: code.putln("#endif") + def generate_spec(self, scope, code): + if self.slot_name == "tp_as_buffer": + # Cannot currently support the buffer protocol in the limited C-API. + return + for slot in self.sub_slots: + slot.generate_spec(scope, code) + substructures = [] # List of all SuiteSlot instances class MethodTableSlot(SlotDescriptor): @@ -503,8 +628,43 @@ class MemberTableSlot(SlotDescriptor): # Slot descriptor for the table of Python-accessible attributes. def slot_code(self, scope): + # Only used in specs. return "0" + def get_member_specs(self, scope): + return [ + get_slot_by_name("tp_dictoffset").members_slot_value(scope), + #get_slot_by_name("tp_weaklistoffset").spec_value(scope), + ] + + def is_empty(self, scope): + for member_entry in self.get_member_specs(scope): + if member_entry: + return False + return True + + def substructure_cname(self, scope): + return "%s%s_%s" % (Naming.pyrex_prefix, self.slot_name, scope.class_name) + + def generate_substructure_spec(self, scope, code): + if self.is_empty(scope): + return + from .Code import UtilityCode + code.globalstate.use_utility_code(UtilityCode.load_cached("IncludeStructmemberH", "ModuleSetupCode.c")) + + ext_type = scope.parent_type + code.putln("static struct PyMemberDef %s[] = {" % self.substructure_cname(scope)) + for member_entry in self.get_member_specs(scope): + if member_entry: + code.putln(member_entry) + code.putln("{NULL, 0, 0, 0, NULL}") + code.putln("};") + + def spec_value(self, scope): + if self.is_empty(scope): + return "0" + return self.substructure_cname(scope) + class GetSetSlot(SlotDescriptor): # Slot descriptor for the table of attribute get & set methods. @@ -520,13 +680,13 @@ class BaseClassSlot(SlotDescriptor): # Slot descriptor for the base class slot. def __init__(self, name): - SlotDescriptor.__init__(self, name, dynamic = 1) + SlotDescriptor.__init__(self, name, dynamic=True) def generate_dynamic_init_code(self, scope, code): base_type = scope.parent_type.base_type if base_type: - code.putln("%s.%s = %s;" % ( - scope.parent_type.typeobj_cname, + code.putln("%s->%s = %s;" % ( + scope.parent_type.typeptr_cname, self.slot_name, base_type.typeptr_cname)) @@ -551,6 +711,13 @@ class DictOffsetSlot(SlotDescriptor): else: return "0" + def members_slot_value(self, scope): + dict_offset = self.slot_code(scope) + if dict_offset == "0": + return None + return '{"__dictoffset__", T_PYSSIZET, %s, READONLY, NULL},' % dict_offset + + # The following dictionary maps __xxx__ method names to slot descriptors. @@ -592,7 +759,7 @@ def get_base_slot_function(scope, slot): # This is useful for enabling the compiler to optimize calls # that recursively climb the class hierarchy. base_type = scope.parent_type.base_type - if scope.parent_scope is base_type.scope.parent_scope: + if base_type and scope.parent_scope is base_type.scope.parent_scope: parent_slot = slot.slot_code(base_type.scope) if parent_slot != '0': entry = scope.parent_scope.lookup_here(scope.parent_type.base_type.name) @@ -621,6 +788,11 @@ def get_slot_by_name(slot_name): assert False, "Slot not found: %s" % slot_name +def get_slot_by_method_name(method_name): + # For now, only search the type struct, no referenced sub-structs. + return method_name_to_slot[method_name] + + def get_slot_code_by_name(scope, slot_name): slot = get_slot_by_name(slot_name) return slot.slot_code(scope) @@ -668,7 +840,7 @@ ssizessizeargfunc = Signature("Tzz", "O") # typedef PyObject *(*ssizessizeargfu intobjargproc = Signature("TiO", 'r') # typedef int(*intobjargproc)(PyObject *, int, PyObject *); ssizeobjargproc = Signature("TzO", 'r') # typedef int(*ssizeobjargproc)(PyObject *, Py_ssize_t, PyObject *); intintobjargproc = Signature("TiiO", 'r') # typedef int(*intintobjargproc)(PyObject *, int, int, PyObject *); -ssizessizeobjargproc = Signature("TzzO", 'r') # typedef int(*ssizessizeobjargproc)(PyObject *, Py_ssize_t, Py_ssize_t, PyObject *); +ssizessizeobjargproc = Signature("TzzO", 'r') # typedef int(*ssizessizeobjargproc)(PyObject *, Py_ssize_t, Py_ssize_t, PyObject *); intintargproc = Signature("Tii", 'r') ssizessizeargproc = Signature("Tzz", 'r') @@ -725,40 +897,40 @@ property_accessor_signatures = { # #------------------------------------------------------------------------------------------ -PyNumberMethods_Py3_GUARD = "PY_MAJOR_VERSION < 3 || (CYTHON_COMPILING_IN_PYPY && PY_VERSION_HEX < 0x03050000)" +PyNumberMethods_Py2only_GUARD = "PY_MAJOR_VERSION < 3 || (CYTHON_COMPILING_IN_PYPY && PY_VERSION_HEX < 0x03050000)" PyNumberMethods = ( - MethodSlot(binaryfunc, "nb_add", "__add__"), - MethodSlot(binaryfunc, "nb_subtract", "__sub__"), - MethodSlot(binaryfunc, "nb_multiply", "__mul__"), - MethodSlot(binaryfunc, "nb_divide", "__div__", ifdef = PyNumberMethods_Py3_GUARD), - MethodSlot(binaryfunc, "nb_remainder", "__mod__"), - MethodSlot(binaryfunc, "nb_divmod", "__divmod__"), - MethodSlot(ternaryfunc, "nb_power", "__pow__"), + BinopSlot(binaryfunc, "nb_add", "__add__"), + BinopSlot(binaryfunc, "nb_subtract", "__sub__"), + BinopSlot(binaryfunc, "nb_multiply", "__mul__"), + BinopSlot(binaryfunc, "nb_divide", "__div__", ifdef = PyNumberMethods_Py2only_GUARD), + BinopSlot(binaryfunc, "nb_remainder", "__mod__"), + BinopSlot(binaryfunc, "nb_divmod", "__divmod__"), + BinopSlot(ternaryfunc, "nb_power", "__pow__"), MethodSlot(unaryfunc, "nb_negative", "__neg__"), MethodSlot(unaryfunc, "nb_positive", "__pos__"), MethodSlot(unaryfunc, "nb_absolute", "__abs__"), - MethodSlot(inquiry, "nb_nonzero", "__nonzero__", py3 = ("nb_bool", "__bool__")), + MethodSlot(inquiry, "nb_bool", "__bool__", py2 = ("nb_nonzero", "__nonzero__")), MethodSlot(unaryfunc, "nb_invert", "__invert__"), - MethodSlot(binaryfunc, "nb_lshift", "__lshift__"), - MethodSlot(binaryfunc, "nb_rshift", "__rshift__"), - MethodSlot(binaryfunc, "nb_and", "__and__"), - MethodSlot(binaryfunc, "nb_xor", "__xor__"), - MethodSlot(binaryfunc, "nb_or", "__or__"), - EmptySlot("nb_coerce", ifdef = PyNumberMethods_Py3_GUARD), + BinopSlot(binaryfunc, "nb_lshift", "__lshift__"), + BinopSlot(binaryfunc, "nb_rshift", "__rshift__"), + BinopSlot(binaryfunc, "nb_and", "__and__"), + BinopSlot(binaryfunc, "nb_xor", "__xor__"), + BinopSlot(binaryfunc, "nb_or", "__or__"), + EmptySlot("nb_coerce", ifdef = PyNumberMethods_Py2only_GUARD), MethodSlot(unaryfunc, "nb_int", "__int__", fallback="__long__"), MethodSlot(unaryfunc, "nb_long", "__long__", fallback="__int__", py3 = "<RESERVED>"), MethodSlot(unaryfunc, "nb_float", "__float__"), - MethodSlot(unaryfunc, "nb_oct", "__oct__", ifdef = PyNumberMethods_Py3_GUARD), - MethodSlot(unaryfunc, "nb_hex", "__hex__", ifdef = PyNumberMethods_Py3_GUARD), + MethodSlot(unaryfunc, "nb_oct", "__oct__", ifdef = PyNumberMethods_Py2only_GUARD), + MethodSlot(unaryfunc, "nb_hex", "__hex__", ifdef = PyNumberMethods_Py2only_GUARD), # Added in release 2.0 MethodSlot(ibinaryfunc, "nb_inplace_add", "__iadd__"), MethodSlot(ibinaryfunc, "nb_inplace_subtract", "__isub__"), MethodSlot(ibinaryfunc, "nb_inplace_multiply", "__imul__"), - MethodSlot(ibinaryfunc, "nb_inplace_divide", "__idiv__", ifdef = PyNumberMethods_Py3_GUARD), + MethodSlot(ibinaryfunc, "nb_inplace_divide", "__idiv__", ifdef = PyNumberMethods_Py2only_GUARD), MethodSlot(ibinaryfunc, "nb_inplace_remainder", "__imod__"), - MethodSlot(ibinaryfunc, "nb_inplace_power", "__ipow__"), # actually ternaryfunc!!! + MethodSlot(ibinaryfunc, "nb_inplace_power", "__ipow__"), # actually ternaryfunc!!! MethodSlot(ibinaryfunc, "nb_inplace_lshift", "__ilshift__"), MethodSlot(ibinaryfunc, "nb_inplace_rshift", "__irshift__"), MethodSlot(ibinaryfunc, "nb_inplace_and", "__iand__"), @@ -767,8 +939,8 @@ PyNumberMethods = ( # Added in release 2.2 # The following require the Py_TPFLAGS_HAVE_CLASS flag - MethodSlot(binaryfunc, "nb_floor_divide", "__floordiv__"), - MethodSlot(binaryfunc, "nb_true_divide", "__truediv__"), + BinopSlot(binaryfunc, "nb_floor_divide", "__floordiv__"), + BinopSlot(binaryfunc, "nb_true_divide", "__truediv__"), MethodSlot(ibinaryfunc, "nb_inplace_floor_divide", "__ifloordiv__"), MethodSlot(ibinaryfunc, "nb_inplace_true_divide", "__itruediv__"), @@ -776,21 +948,21 @@ PyNumberMethods = ( MethodSlot(unaryfunc, "nb_index", "__index__"), # Added in release 3.5 - MethodSlot(binaryfunc, "nb_matrix_multiply", "__matmul__", ifdef="PY_VERSION_HEX >= 0x03050000"), + BinopSlot(binaryfunc, "nb_matrix_multiply", "__matmul__", ifdef="PY_VERSION_HEX >= 0x03050000"), MethodSlot(ibinaryfunc, "nb_inplace_matrix_multiply", "__imatmul__", ifdef="PY_VERSION_HEX >= 0x03050000"), ) PySequenceMethods = ( MethodSlot(lenfunc, "sq_length", "__len__"), - EmptySlot("sq_concat"), # nb_add used instead - EmptySlot("sq_repeat"), # nb_multiply used instead + EmptySlot("sq_concat"), # nb_add used instead + EmptySlot("sq_repeat"), # nb_multiply used instead SyntheticSlot("sq_item", ["__getitem__"], "0"), #EmptySlot("sq_item"), # mp_subscript used instead MethodSlot(ssizessizeargfunc, "sq_slice", "__getslice__"), - EmptySlot("sq_ass_item"), # mp_ass_subscript used instead + EmptySlot("sq_ass_item"), # mp_ass_subscript used instead SyntheticSlot("sq_ass_slice", ["__setslice__", "__delslice__"], "0"), MethodSlot(cmpfunc, "sq_contains", "__contains__"), - EmptySlot("sq_inplace_concat"), # nb_inplace_add used instead - EmptySlot("sq_inplace_repeat"), # nb_inplace_multiply used instead + EmptySlot("sq_inplace_concat"), # nb_inplace_add used instead + EmptySlot("sq_inplace_repeat"), # nb_inplace_multiply used instead ) PyMappingMethods = ( @@ -844,8 +1016,8 @@ slot_table = ( MethodSlot(callfunc, "tp_call", "__call__"), MethodSlot(reprfunc, "tp_str", "__str__"), - SyntheticSlot("tp_getattro", ["__getattr__","__getattribute__"], "0"), #"PyObject_GenericGetAttr"), - SyntheticSlot("tp_setattro", ["__setattr__", "__delattr__"], "0"), #"PyObject_GenericSetAttr"), + SyntheticSlot("tp_getattro", ["__getattr__","__getattribute__"], "0"), #"PyObject_GenericGetAttr"), + SyntheticSlot("tp_setattro", ["__setattr__", "__delattr__"], "0"), #"PyObject_GenericSetAttr"), SuiteSlot(PyBufferProcs, "PyBufferProcs", "tp_as_buffer"), @@ -866,17 +1038,17 @@ slot_table = ( MemberTableSlot("tp_members"), GetSetSlot("tp_getset"), - BaseClassSlot("tp_base"), #EmptySlot("tp_base"), + BaseClassSlot("tp_base"), #EmptySlot("tp_base"), EmptySlot("tp_dict"), SyntheticSlot("tp_descr_get", ["__get__"], "0"), SyntheticSlot("tp_descr_set", ["__set__", "__delete__"], "0"), - DictOffsetSlot("tp_dictoffset"), + DictOffsetSlot("tp_dictoffset", ifdef="!CYTHON_USE_TYPE_SPECS"), # otherwise set via "__dictoffset__" member MethodSlot(initproc, "tp_init", "__init__"), - EmptySlot("tp_alloc"), #FixedSlot("tp_alloc", "PyType_GenericAlloc"), - InternalMethodSlot("tp_new"), + EmptySlot("tp_alloc"), #FixedSlot("tp_alloc", "PyType_GenericAlloc"), + ConstructorSlot("tp_new", "__cinit__"), EmptySlot("tp_free"), EmptySlot("tp_is_gc"), @@ -890,6 +1062,8 @@ slot_table = ( EmptySlot("tp_finalize", ifdef="PY_VERSION_HEX >= 0x030400a1"), EmptySlot("tp_vectorcall", ifdef="PY_VERSION_HEX >= 0x030800b1"), EmptySlot("tp_print", ifdef="PY_VERSION_HEX >= 0x030800b4 && PY_VERSION_HEX < 0x03090000"), + # PyPy specific extension - only here to avoid C compiler warnings. + EmptySlot("tp_pypy_flags", ifdef="CYTHON_COMPILING_IN_PYPY && PYPY_VERSION_NUM+0 >= 0x06000000"), ) #------------------------------------------------------------------------------------------ @@ -920,5 +1094,6 @@ MethodSlot(descrdelfunc, "", "__delete__") method_noargs = "METH_NOARGS" method_onearg = "METH_O" method_varargs = "METH_VARARGS" +method_fastcall = "__Pyx_METH_FASTCALL" # Actually VARARGS on versions < 3.7 method_keywords = "METH_KEYWORDS" method_coexist = "METH_COEXIST" diff --git a/Cython/Compiler/UtilNodes.py b/Cython/Compiler/UtilNodes.py index c41748ace..0d5db9d33 100644 --- a/Cython/Compiler/UtilNodes.py +++ b/Cython/Compiler/UtilNodes.py @@ -45,7 +45,7 @@ class TempRefNode(AtomicExprNode): def calculate_result_code(self): result = self.handle.temp - if result is None: result = "<error>" # might be called and overwritten + if result is None: result = "<error>" # might be called and overwritten return result def generate_result_code(self, code): @@ -122,8 +122,7 @@ class ResultRefNode(AtomicExprNode): self.may_hold_none = may_hold_none if expression is not None: self.pos = expression.pos - if hasattr(expression, "type"): - self.type = expression.type + self.type = getattr(expression, "type", None) if pos is not None: self.pos = pos if type is not None: @@ -144,13 +143,14 @@ class ResultRefNode(AtomicExprNode): def update_expression(self, expression): self.expression = expression - if hasattr(expression, "type"): - self.type = expression.type + type = getattr(expression, "type", None) + if type: + self.type = type def analyse_types(self, env): if self.expression is not None: if not self.expression.type: - self.expression = self.expression.analyse_types(env) + self.expression = self.expression.analyse_types(env) self.type = self.expression.type return self @@ -175,7 +175,7 @@ class ResultRefNode(AtomicExprNode): return self.expression.may_be_none() if self.type is not None: return self.type.is_pyobject - return True # play safe + return True # play it safe def is_simple(self): return True @@ -354,6 +354,12 @@ class TempResultFromStatNode(ExprNodes.ExprNode): self.body = self.body.analyse_expressions(env) return self + def may_be_none(self): + return self.result_ref.may_be_none() + def generate_result_code(self, code): self.result_ref.result_code = self.result() self.body.generate_execution_code(code) + + def generate_function_definitions(self, env, code): + self.body.generate_function_definitions(env, code) diff --git a/Cython/Compiler/UtilityCode.py b/Cython/Compiler/UtilityCode.py index 98e9ab5bf..870abf3e5 100644 --- a/Cython/Compiler/UtilityCode.py +++ b/Cython/Compiler/UtilityCode.py @@ -131,7 +131,7 @@ class CythonUtilityCode(Code.UtilityCodeBase): p = [] for t in pipeline: p.append(t) - if isinstance(p, ParseTreeTransforms.AnalyseDeclarationsTransform): + if isinstance(t, ParseTreeTransforms.AnalyseDeclarationsTransform): break pipeline = p @@ -196,10 +196,10 @@ class CythonUtilityCode(Code.UtilityCodeBase): Load a utility code as a string. Returns (proto, implementation) """ util = cls.load(util_code_name, from_file, **kwargs) - return util.proto, util.impl # keep line numbers => no lstrip() + return util.proto, util.impl # keep line numbers => no lstrip() def declare_in_scope(self, dest_scope, used=False, cython_scope=None, - whitelist=None): + allowlist=None): """ Declare all entries from the utility code in dest_scope. Code will only be included for used entries. If module_name is given, declare the @@ -218,7 +218,7 @@ class CythonUtilityCode(Code.UtilityCodeBase): entry.used = used original_scope = tree.scope - dest_scope.merge_in(original_scope, merge_unused=True, whitelist=whitelist) + dest_scope.merge_in(original_scope, merge_unused=True, allowlist=allowlist) tree.scope = dest_scope for dep in self.requires: @@ -227,6 +227,27 @@ class CythonUtilityCode(Code.UtilityCodeBase): return original_scope + @staticmethod + def filter_inherited_directives(current_directives): + """ + Cython utility code should usually only pick up a few directives from the + environment (those that intentionally control its function) and ignore most + other compiler directives. This function provides a sensible default list + of directives to copy. + """ + from .Options import _directive_defaults + utility_code_directives = dict(_directive_defaults) + inherited_directive_names = ( + 'binding', 'always_allow_keywords', 'allow_none_for_extension_args', + 'auto_pickle', 'ccomplex', + 'c_string_type', 'c_string_encoding', + 'optimize.inline_defnode_calls', 'optimize.unpack_method_calls', + 'optimize.unpack_method_calls_in_pyinit', 'optimize.use_switch') + for name in inherited_directive_names: + if name in current_directives: + utility_code_directives[name] = current_directives[name] + return utility_code_directives + def declare_declarations_in_scope(declaration_string, env, private_type=True, *args, **kwargs): diff --git a/Cython/Compiler/Visitor.pxd b/Cython/Compiler/Visitor.pxd index d5d5692aa..6b8110383 100644 --- a/Cython/Compiler/Visitor.pxd +++ b/Cython/Compiler/Visitor.pxd @@ -1,4 +1,4 @@ -from __future__ import absolute_import +# cython: language_level=3 cimport cython diff --git a/Cython/Compiler/Visitor.py b/Cython/Compiler/Visitor.py index bd5655f5b..5d35c8aac 100644 --- a/Cython/Compiler/Visitor.py +++ b/Cython/Compiler/Visitor.py @@ -569,6 +569,17 @@ class MethodDispatcherTransform(EnvTransform): ### dispatch to specific handlers def _find_handler(self, match_name, has_kwargs): + try: + match_name.encode('ascii') + except UnicodeEncodeError: + # specifically when running the Cython compiler under Python 2 + # getattr can't take a unicode string. + # Classes with unicode names won't have specific handlers and thus it + # should be OK to return None. + # Doing the test here ensures that the same code gets run on + # Python 2 and 3 + return None + call_type = has_kwargs and 'general' or 'simple' handler = getattr(self, '_handle_%s_%s' % (call_type, match_name), None) if handler is None: @@ -663,7 +674,7 @@ class MethodDispatcherTransform(EnvTransform): "method_%s_%s" % (type_name, attr_name), kwargs) if method_handler is None: if (attr_name in TypeSlots.method_name_to_slot - or attr_name == '__new__'): + or attr_name in ['__new__', '__class__']): method_handler = self._find_handler( "slot%s" % attr_name, kwargs) if method_handler is None: @@ -821,6 +832,10 @@ class PrintTree(TreeVisitor): result += "(type=%s, name=\"%s\")" % (repr(node.type), node.name) elif isinstance(node, Nodes.DefNode): result += "(name=\"%s\")" % node.name + elif isinstance(node, ExprNodes.AttributeNode): + result += "(type=%s, attribute=\"%s\")" % (repr(node.type), node.attribute) + elif isinstance(node, (ExprNodes.ConstNode, ExprNodes.PyConstNode)): + result += "(type=%s, value=%r)" % (repr(node.type), node.value) elif isinstance(node, ExprNodes.ExprNode): t = node.type result += "(type=%s)" % repr(t) diff --git a/Cython/Coverage.py b/Cython/Coverage.py index 269896e13..0316664ef 100644 --- a/Cython/Coverage.py +++ b/Cython/Coverage.py @@ -14,7 +14,7 @@ from collections import defaultdict from coverage.plugin import CoveragePlugin, FileTracer, FileReporter # requires coverage.py 4.0+ from coverage.files import canonical_filename -from .Utils import find_root_package_dir, is_package_dir, open_source_file +from .Utils import find_root_package_dir, is_package_dir, is_cython_generated_file, open_source_file from . import __version__ @@ -57,10 +57,19 @@ class Plugin(CoveragePlugin): _c_files_map = None # map from parsed C files to their content _parsed_c_files = None + # map from traced files to lines that are excluded from coverage + _excluded_lines_map = None + # list of regex patterns for lines to exclude + _excluded_line_patterns = () def sys_info(self): return [('Cython version', __version__)] + def configure(self, config): + # Entry point for coverage "configurer". + # Read the regular expressions from the coverage config that match lines to be excluded from coverage. + self._excluded_line_patterns = config.get_option("report:exclude_lines") + def file_tracer(self, filename): """ Try to find a C source file for a file path found by the tracer. @@ -108,7 +117,13 @@ class Plugin(CoveragePlugin): rel_file_path, code = self._read_source_lines(c_file, filename) if code is None: return None # no source found - return CythonModuleReporter(c_file, filename, rel_file_path, code) + return CythonModuleReporter( + c_file, + filename, + rel_file_path, + code, + self._excluded_lines_map.get(rel_file_path, frozenset()) + ) def _find_source_files(self, filename): basename, ext = os.path.splitext(filename) @@ -150,12 +165,11 @@ class Plugin(CoveragePlugin): py_source_file = os.path.splitext(c_file)[0] + '.py' if not os.path.exists(py_source_file): py_source_file = None - - try: - with open(c_file, 'rb') as f: - if b'/* Generated by Cython ' not in f.read(30): - return None, None # not a Cython file - except (IOError, OSError): + if not is_cython_generated_file(c_file, if_not_found=False): + if py_source_file and os.path.exists(c_file): + # if we did not generate the C file, + # then we probably also shouldn't care about the .py file. + py_source_file = None c_file = None return c_file, py_source_file @@ -218,10 +232,16 @@ class Plugin(CoveragePlugin): r'(?:struct|union|enum|class)' r'(\s+[^:]+|)\s*:' ).match + if self._excluded_line_patterns: + line_is_excluded = re.compile("|".join(["(?:%s)" % regex for regex in self._excluded_line_patterns])).search + else: + line_is_excluded = lambda line: False code_lines = defaultdict(dict) executable_lines = defaultdict(set) current_filename = None + if self._excluded_lines_map is None: + self._excluded_lines_map = defaultdict(set) with open(c_file) as lines: lines = iter(lines) @@ -242,6 +262,9 @@ class Plugin(CoveragePlugin): code_line = match.group(1).rstrip() if not_executable(code_line): break + if line_is_excluded(code_line): + self._excluded_lines_map[filename].add(lineno) + break code_lines[filename][lineno] = code_line break elif match_comment_end(comment_line): @@ -298,11 +321,12 @@ class CythonModuleReporter(FileReporter): """ Provide detailed trace information for one source file to coverage.py. """ - def __init__(self, c_file, source_file, rel_file_path, code): + def __init__(self, c_file, source_file, rel_file_path, code, excluded_lines): super(CythonModuleReporter, self).__init__(source_file) self.name = rel_file_path self.c_file = c_file self._code = code + self._excluded_lines = excluded_lines def lines(self): """ @@ -310,6 +334,12 @@ class CythonModuleReporter(FileReporter): """ return set(self._code) + def excluded_lines(self): + """ + Return set of line numbers that are excluded from coverage. + """ + return self._excluded_lines + def _iter_source_tokens(self): current_line = 1 for line_no, code_line in sorted(self._code.items()): @@ -345,4 +375,6 @@ class CythonModuleReporter(FileReporter): def coverage_init(reg, options): - reg.add_file_tracer(Plugin()) + plugin = Plugin() + reg.add_configurer(plugin) + reg.add_file_tracer(plugin) diff --git a/Cython/Debugger/Cygdb.py b/Cython/Debugger/Cygdb.py index 45f31ce6f..596e5e11b 100644 --- a/Cython/Debugger/Cygdb.py +++ b/Cython/Debugger/Cygdb.py @@ -22,7 +22,9 @@ import logging logger = logging.getLogger(__name__) -def make_command_file(path_to_debug_info, prefix_code='', no_import=False): + +def make_command_file(path_to_debug_info, prefix_code='', + no_import=False, skip_interpreter=False): if not no_import: pattern = os.path.join(path_to_debug_info, 'cython_debug', @@ -45,17 +47,23 @@ def make_command_file(path_to_debug_info, prefix_code='', no_import=False): set print pretty on python - # Activate virtualenv, if we were launched from one - import os - virtualenv = os.getenv('VIRTUAL_ENV') - if virtualenv: - path_to_activate_this_py = os.path.join(virtualenv, 'bin', 'activate_this.py') - print("gdb command file: Activating virtualenv: %s; path_to_activate_this_py: %s" % ( - virtualenv, path_to_activate_this_py)) - with open(path_to_activate_this_py) as f: - exec(f.read(), dict(__file__=path_to_activate_this_py)) - - from Cython.Debugger import libcython, libpython + try: + # Activate virtualenv, if we were launched from one + import os + virtualenv = os.getenv('VIRTUAL_ENV') + if virtualenv: + path_to_activate_this_py = os.path.join(virtualenv, 'bin', 'activate_this.py') + print("gdb command file: Activating virtualenv: %s; path_to_activate_this_py: %s" % ( + virtualenv, path_to_activate_this_py)) + with open(path_to_activate_this_py) as f: + exec(f.read(), dict(__file__=path_to_activate_this_py)) + from Cython.Debugger import libcython, libpython + except Exception as ex: + from traceback import print_exc + print("There was an error in Python code originating from the file ''' + str(__file__) + '''") + print("It used the Python interpreter " + str(sys.executable)) + print_exc() + exit(1) end ''')) @@ -64,27 +72,33 @@ def make_command_file(path_to_debug_info, prefix_code='', no_import=False): # f.write("file %s\n" % sys.executable) pass else: - path = os.path.join(path_to_debug_info, "cython_debug", "interpreter") - interpreter_file = open(path) - try: - interpreter = interpreter_file.read() - finally: - interpreter_file.close() - f.write("file %s\n" % interpreter) - f.write('\n'.join('cy import %s\n' % fn for fn in debug_files)) - f.write(textwrap.dedent('''\ - python - import sys + if not skip_interpreter: + # Point Cygdb to the interpreter that was used to generate + # the debugging information. + path = os.path.join(path_to_debug_info, "cython_debug", "interpreter") + interpreter_file = open(path) try: - gdb.lookup_type('PyModuleObject') - except RuntimeError: - sys.stderr.write( - 'Python was not compiled with debug symbols (or it was ' - 'stripped). Some functionality may not work (properly).\\n') - end - - source .cygdbinit - ''')) + interpreter = interpreter_file.read() + finally: + interpreter_file.close() + f.write("file %s\n" % interpreter) + + f.write('\n'.join('cy import %s\n' % fn for fn in debug_files)) + + if not skip_interpreter: + f.write(textwrap.dedent('''\ + python + import sys + try: + gdb.lookup_type('PyModuleObject') + except RuntimeError: + sys.stderr.write( + "''' + interpreter + ''' was not compiled with debug symbols (or it was " + "stripped). Some functionality may not work (properly).\\n") + end + ''')) + + f.write("source .cygdbinit") finally: f.close() @@ -109,6 +123,10 @@ def main(path_to_debug_info=None, gdb_argv=None, no_import=False): parser.add_option("--verbose", "-v", dest="verbosity", action="count", default=0, help="Verbose mode. Multiple -v options increase the verbosity") + parser.add_option("--skip-interpreter", + dest="skip_interpreter", default=False, action="store_true", + help="Do not automatically point GDB to the same interpreter " + "used to generate debugging information") (options, args) = parser.parse_args() if path_to_debug_info is None: @@ -130,12 +148,16 @@ def main(path_to_debug_info=None, gdb_argv=None, no_import=False): logging_level = logging.DEBUG logging.basicConfig(level=logging_level) + skip_interpreter = options.skip_interpreter + logger.info("verbosity = %r", options.verbosity) logger.debug("options = %r; args = %r", options, args) logger.debug("Done parsing command-line options. path_to_debug_info = %r, gdb_argv = %r", path_to_debug_info, gdb_argv) - tempfilename = make_command_file(path_to_debug_info, no_import=no_import) + tempfilename = make_command_file(path_to_debug_info, + no_import=no_import, + skip_interpreter=skip_interpreter) logger.info("Launching %s with command file: %s and gdb_argv: %s", options.gdb, tempfilename, gdb_argv) with open(tempfilename) as tempfile: diff --git a/Cython/Debugger/DebugWriter.py b/Cython/Debugger/DebugWriter.py index 876a3a216..8b1fb75b0 100644 --- a/Cython/Debugger/DebugWriter.py +++ b/Cython/Debugger/DebugWriter.py @@ -5,8 +5,8 @@ import sys import errno try: - from lxml import etree - have_lxml = True + from lxml import etree + have_lxml = True except ImportError: have_lxml = False try: diff --git a/Cython/Debugger/Tests/TestLibCython.py b/Cython/Debugger/Tests/TestLibCython.py index 13560646f..0d8a3e613 100644 --- a/Cython/Debugger/Tests/TestLibCython.py +++ b/Cython/Debugger/Tests/TestLibCython.py @@ -56,13 +56,13 @@ def test_gdb(): stdout, _ = p.communicate() try: internal_python_version = list(map(int, stdout.decode('ascii', 'ignore').split())) - if internal_python_version < [2, 6]: + if internal_python_version < [2, 7]: have_gdb = False except ValueError: have_gdb = False if not have_gdb: - warnings.warn('Skipping gdb tests, need gdb >= 7.2 with Python >= 2.6') + warnings.warn('Skipping gdb tests, need gdb >= 7.2 with Python >= 2.7') return have_gdb @@ -99,6 +99,7 @@ class DebuggerTestCase(unittest.TestCase): opts = dict( test_directory=self.tempdir, module='codefile', + module_path=self.destfile, ) optimization_disabler = build_ext.Optimization() @@ -131,10 +132,11 @@ class DebuggerTestCase(unittest.TestCase): ) cython_compile_testcase.run_distutils( + test_directory=opts['test_directory'], + module=opts['module'], + workdir=opts['test_directory'], incdir=None, - workdir=self.tempdir, extra_extension_args={'extra_objects':['cfuncs.o']}, - **opts ) finally: optimization_disabler.restore_state() diff --git a/Cython/Debugger/Tests/codefile b/Cython/Debugger/Tests/codefile index 6b4c6b6ad..ee587cbb1 100644 --- a/Cython/Debugger/Tests/codefile +++ b/Cython/Debugger/Tests/codefile @@ -37,14 +37,13 @@ def outer(): def inner(): b = 2 # access closed over variables - print a, b + print(a, b) return inner - outer()() spam() -print "bye!" +print("bye!") def use_ham(): ham() diff --git a/Cython/Debugger/Tests/test_libcython_in_gdb.py b/Cython/Debugger/Tests/test_libcython_in_gdb.py index bd7608d60..bb06c2905 100644 --- a/Cython/Debugger/Tests/test_libcython_in_gdb.py +++ b/Cython/Debugger/Tests/test_libcython_in_gdb.py @@ -134,7 +134,7 @@ class TestDebugInformationClasses(DebugTestCase): self.assertEqual(self.spam_func.arguments, ['a']) self.assertEqual(self.spam_func.step_into_functions, - set(['puts', 'some_c_function'])) + {'puts', 'some_c_function'}) expected_lineno = test_libcython.source_to_lineno['def spam(a=0):'] self.assertEqual(self.spam_func.lineno, expected_lineno) @@ -177,12 +177,13 @@ class TestBreak(DebugTestCase): assert step_result.rstrip().endswith(nextline) -class TestKilled(DebugTestCase): - - def test_abort(self): - gdb.execute("set args -c 'import os; os.abort()'") - output = gdb.execute('cy run', to_string=True) - assert 'abort' in output.lower() +# I removed this testcase, because it will never work, because +# gdb.execute(..., to_string=True) does not capture stdout and stderr of python. +# class TestKilled(DebugTestCase): +# def test_abort(self): +# gdb.execute("set args -c 'import os;print(123456789);os.abort()'") +# output = gdb.execute('cy run', to_string=True) +# assert 'abort' in output.lower() class DebugStepperTestCase(DebugTestCase): @@ -322,6 +323,61 @@ class TestPrint(DebugTestCase): self.break_and_run('c = 2') result = gdb.execute('cy print b', to_string=True) self.assertEqual('b = (int) 1\n', result) + result = gdb.execute('cy print python_var', to_string=True) + self.assertEqual('python_var = 13\n', result) + result = gdb.execute('cy print c_var', to_string=True) + self.assertEqual('c_var = (int) 12\n', result) + +correct_result_test_list_inside_func = '''\ + 14 int b, c + 15 + 16 b = c = d = 0 + 17 + 18 b = 1 +> 19 c = 2 + 20 int(10) + 21 puts("spam") + 22 os.path.join("foo", "bar") + 23 some_c_function() +''' +correct_result_test_list_outside_func = '''\ + 5 void some_c_function() + 6 + 7 import os + 8 + 9 cdef int c_var = 12 +> 10 python_var = 13 + 11 + 12 def spam(a=0): + 13 cdef: + 14 int b, c +''' + + +class TestList(DebugTestCase): + def workaround_for_coding_style_checker(self, correct_result_wrong_whitespace): + correct_result = "" + for line in correct_result_test_list_inside_func.split("\n"): + if len(line) < 10 and len(line) > 0: + line += " "*4 + correct_result += line + "\n" + correct_result = correct_result[:-1] + + def test_list_inside_func(self): + self.break_and_run('c = 2') + result = gdb.execute('cy list', to_string=True) + # We don't want to fail because of some trailing whitespace, + # so we remove trailing whitespaces with the following line + result = "\n".join([line.rstrip() for line in result.split("\n")]) + self.assertEqual(correct_result_test_list_inside_func, result) + + def test_list_outside_func(self): + self.break_and_run('python_var = 13') + result = gdb.execute('cy list', to_string=True) + # We don't want to fail because of some trailing whitespace, + # so we remove trailing whitespaces with the following line + result = "\n".join([line.rstrip() for line in result.split("\n")]) + self.assertEqual(correct_result_test_list_outside_func, result) class TestUpDown(DebugTestCase): @@ -362,6 +418,7 @@ class TestExec(DebugTestCase): # test normal behaviour self.assertEqual("[0]", self.eval_command('[a]')) + return #The test after this return freezes gdb, so I temporarily removed it. # test multiline code result = gdb.execute(textwrap.dedent('''\ cy exec diff --git a/Cython/Debugger/libcython.py b/Cython/Debugger/libcython.py index 23153789b..dd634cc35 100644 --- a/Cython/Debugger/libcython.py +++ b/Cython/Debugger/libcython.py @@ -11,7 +11,6 @@ except NameError: import sys import textwrap -import traceback import functools import itertools import collections @@ -69,19 +68,6 @@ _filesystemencoding = sys.getfilesystemencoding() or 'UTF-8' # decorators -def dont_suppress_errors(function): - "*sigh*, readline" - @functools.wraps(function) - def wrapper(*args, **kwargs): - try: - return function(*args, **kwargs) - except Exception: - traceback.print_exc() - raise - - return wrapper - - def default_selected_gdb_frame(err=True): def decorator(function): @functools.wraps(function) @@ -252,7 +238,9 @@ class CythonBase(object): filename = lineno = lexer = None if self.is_cython_function(frame): filename = self.get_cython_function(frame).module.filename - lineno = self.get_cython_lineno(frame) + filename_and_lineno = self.get_cython_lineno(frame) + assert filename == filename_and_lineno[0] + lineno = filename_and_lineno[1] if pygments: lexer = pygments.lexers.CythonLexer(stripall=False) elif self.is_python_function(frame): @@ -334,7 +322,7 @@ class CythonBase(object): func_name = cyfunc.name func_cname = cyfunc.cname - func_args = [] # [(arg, f(arg)) for arg in cyfunc.arguments] + func_args = [] # [(arg, f(arg)) for arg in cyfunc.arguments] else: source_desc, lineno = self.get_source_desc(frame) func_name = frame.name() @@ -392,7 +380,7 @@ class CythonBase(object): result = {} seen = set() - for k, v in pyobject_dict.items(): + for k, v in pyobject_dict.iteritems(): result[k.proxyval(seen)] = v return result @@ -410,7 +398,7 @@ class CythonBase(object): def is_initialized(self, cython_func, local_name): cyvar = cython_func.locals[local_name] - cur_lineno = self.get_cython_lineno() + cur_lineno = self.get_cython_lineno()[1] if '->' in cyvar.cname: # Closed over free variable @@ -695,6 +683,7 @@ class CyImport(CythonCommand): command_class = gdb.COMMAND_STATUS completer_class = gdb.COMPLETE_FILENAME + @libpython.dont_suppress_errors def invoke(self, args, from_tty): if isinstance(args, BYTES): args = args.decode(_filesystemencoding) @@ -742,11 +731,12 @@ class CyImport(CythonCommand): funcarg.tag for funcarg in function.find('Arguments')) for marker in module.find('LineNumberMapping'): - cython_lineno = int(marker.attrib['cython_lineno']) + src_lineno = int(marker.attrib['src_lineno']) + src_path = marker.attrib['src_path'] c_linenos = list(map(int, marker.attrib['c_linenos'].split())) - cython_module.lineno_cy2c[cython_lineno] = min(c_linenos) + cython_module.lineno_cy2c[src_path, src_lineno] = min(c_linenos) for c_lineno in c_linenos: - cython_module.lineno_c2cy[c_lineno] = cython_lineno + cython_module.lineno_c2cy[c_lineno] = (src_path, src_lineno) class CyBreak(CythonCommand): @@ -784,8 +774,8 @@ class CyBreak(CythonCommand): else: cython_module = self.get_cython_function().module - if lineno in cython_module.lineno_cy2c: - c_lineno = cython_module.lineno_cy2c[lineno] + if (cython_module.filename, lineno) in cython_module.lineno_cy2c: + c_lineno = cython_module.lineno_cy2c[cython_module.filename, lineno] breakpoint = '%s:%s' % (cython_module.c_filename, c_lineno) gdb.execute('break ' + breakpoint) else: @@ -841,6 +831,7 @@ class CyBreak(CythonCommand): if func.pf_cname: gdb.execute('break %s' % func.pf_cname) + @libpython.dont_suppress_errors def invoke(self, function_names, from_tty): if isinstance(function_names, BYTES): function_names = function_names.decode(_filesystemencoding) @@ -859,7 +850,7 @@ class CyBreak(CythonCommand): else: self._break_funcname(funcname) - @dont_suppress_errors + @libpython.dont_suppress_errors def complete(self, text, word): # Filter init-module functions (breakpoints can be set using # modulename:linenumber). @@ -904,7 +895,7 @@ class CythonInfo(CythonBase, libpython.PythonInfo): # stepping through Python code, but it would not step back into Cython- # related code. The C level should be dispatched to the 'step' command. if self.is_cython_function(frame): - return self.get_cython_lineno(frame) + return self.get_cython_lineno(frame)[1] return super(CythonInfo, self).lineno(frame) def get_source_line(self, frame): @@ -944,6 +935,7 @@ class CyStep(CythonExecutionControlCommand, libpython.PythonStepperMixin): name = 'cy -step' stepinto = True + @libpython.dont_suppress_errors def invoke(self, args, from_tty): if self.is_python_function(): self.python_step(self.stepinto) @@ -973,7 +965,7 @@ class CyRun(CythonExecutionControlCommand): name = 'cy run' - invoke = CythonExecutionControlCommand.run + invoke = libpython.dont_suppress_errors(CythonExecutionControlCommand.run) class CyCont(CythonExecutionControlCommand): @@ -983,7 +975,7 @@ class CyCont(CythonExecutionControlCommand): """ name = 'cy cont' - invoke = CythonExecutionControlCommand.cont + invoke = libpython.dont_suppress_errors(CythonExecutionControlCommand.cont) class CyFinish(CythonExecutionControlCommand): @@ -992,7 +984,7 @@ class CyFinish(CythonExecutionControlCommand): """ name = 'cy finish' - invoke = CythonExecutionControlCommand.finish + invoke = libpython.dont_suppress_errors(CythonExecutionControlCommand.finish) class CyUp(CythonCommand): @@ -1002,6 +994,7 @@ class CyUp(CythonCommand): name = 'cy up' _command = 'up' + @libpython.dont_suppress_errors def invoke(self, *args): try: gdb.execute(self._command, to_string=True) @@ -1036,6 +1029,7 @@ class CySelect(CythonCommand): name = 'cy select' + @libpython.dont_suppress_errors def invoke(self, stackno, from_tty): try: stackno = int(stackno) @@ -1062,6 +1056,7 @@ class CyBacktrace(CythonCommand): command_class = gdb.COMMAND_STACK completer_class = gdb.COMPLETE_NONE + @libpython.dont_suppress_errors @require_running_program def invoke(self, args, from_tty): # get the first frame @@ -1095,6 +1090,7 @@ class CyList(CythonCommand): command_class = gdb.COMMAND_FILES completer_class = gdb.COMPLETE_NONE + @libpython.dont_suppress_errors # @dispatch_on_frame(c_command='list') def invoke(self, _, from_tty): sd, lineno = self.get_source_desc() @@ -1111,8 +1107,28 @@ class CyPrint(CythonCommand): name = 'cy print' command_class = gdb.COMMAND_DATA - def invoke(self, name, from_tty, max_name_length=None): - if self.is_python_function(): + @libpython.dont_suppress_errors + def invoke(self, name, from_tty): + global_python_dict = self.get_cython_globals_dict() + module_globals = self.get_cython_function().module.globals + + if name in global_python_dict: + value = global_python_dict[name].get_truncated_repr(libpython.MAX_OUTPUT_LEN) + print('%s = %s' % (name, value)) + #This also would work, but beacause the output of cy exec is not captured in gdb.execute, TestPrint would fail + #self.cy.exec_.invoke("print('"+name+"','=', type(" + name + "), "+name+", flush=True )", from_tty) + elif name in module_globals: + cname = module_globals[name].cname + try: + value = gdb.parse_and_eval(cname) + except RuntimeError: + print("unable to get value of %s" % name) + else: + if not value.is_optimized_out: + self.print_gdb_value(name, value) + else: + print("%s is optimized out" % name) + elif self.is_python_function(): return gdb.execute('py-print ' + name) elif self.is_cython_function(): value = self.cy.cy_cvalue.invoke(name.lstrip('*')) @@ -1122,7 +1138,7 @@ class CyPrint(CythonCommand): else: break - self.print_gdb_value(name, value, max_name_length) + self.print_gdb_value(name, value) else: gdb.execute('print ' + name) @@ -1146,6 +1162,7 @@ class CyLocals(CythonCommand): command_class = gdb.COMMAND_STACK completer_class = gdb.COMPLETE_NONE + @libpython.dont_suppress_errors @dispatch_on_frame(c_command='info locals', python_command='py-locals') def invoke(self, args, from_tty): cython_function = self.get_cython_function() @@ -1173,6 +1190,7 @@ class CyGlobals(CyLocals): command_class = gdb.COMMAND_STACK completer_class = gdb.COMPLETE_NONE + @libpython.dont_suppress_errors @dispatch_on_frame(c_command='info variables', python_command='py-globals') def invoke(self, args, from_tty): global_python_dict = self.get_cython_globals_dict() @@ -1189,6 +1207,7 @@ class CyGlobals(CyLocals): seen = set() print('Python globals:') + for k, v in sorted(global_python_dict.items(), key=sortkey): v = v.get_truncated_repr(libpython.MAX_OUTPUT_LEN) seen.add(k) @@ -1219,7 +1238,9 @@ class EvaluateOrExecuteCodeMixin(object): cython_func = self.get_cython_function() for name, cyvar in cython_func.locals.items(): - if cyvar.type == PythonObject and self.is_initialized(cython_func, name): + if (cyvar.type == PythonObject + and self.is_initialized(cython_func, name)): + try: val = gdb.parse_and_eval(cyvar.cname) except RuntimeError: @@ -1247,8 +1268,8 @@ class EvaluateOrExecuteCodeMixin(object): def _find_first_cython_or_python_frame(self): frame = gdb.selected_frame() while frame: - if (self.is_cython_function(frame) or - self.is_python_function(frame)): + if (self.is_cython_function(frame) + or self.is_python_function(frame)): frame.select() return frame @@ -1295,10 +1316,11 @@ class CyExec(CythonCommand, libpython.PyExec, EvaluateOrExecuteCodeMixin): command_class = gdb.COMMAND_STACK completer_class = gdb.COMPLETE_NONE + @libpython.dont_suppress_errors def invoke(self, expr, from_tty): expr, input_type = self.readcode(expr) executor = libpython.PythonCodeExecutor() - executor.xdecref(self.evalcode(expr, executor.Py_single_input)) + executor.xdecref(self.evalcode(expr, executor.Py_file_input)) class CySet(CythonCommand): @@ -1317,6 +1339,7 @@ class CySet(CythonCommand): command_class = gdb.COMMAND_DATA completer_class = gdb.COMPLETE_NONE + @libpython.dont_suppress_errors @require_cython_frame def invoke(self, expr, from_tty): name_and_expr = expr.split('=', 1) @@ -1340,6 +1363,7 @@ class CyCName(gdb.Function, CythonBase): print $cy_cname("module.function") """ + @libpython.dont_suppress_errors @require_cython_frame @gdb_function_value_to_unicode def invoke(self, cyname, frame=None): @@ -1371,6 +1395,7 @@ class CyCValue(CyCName): Get the value of a Cython variable. """ + @libpython.dont_suppress_errors @require_cython_frame @gdb_function_value_to_unicode def invoke(self, cyname, frame=None): @@ -1391,9 +1416,10 @@ class CyLine(gdb.Function, CythonBase): Get the current Cython line. """ + @libpython.dont_suppress_errors @require_cython_frame def invoke(self): - return self.get_cython_lineno() + return self.get_cython_lineno()[1] class CyEval(gdb.Function, CythonBase, EvaluateOrExecuteCodeMixin): @@ -1401,6 +1427,7 @@ class CyEval(gdb.Function, CythonBase, EvaluateOrExecuteCodeMixin): Evaluate Python code in the nearest Python or Cython frame and return """ + @libpython.dont_suppress_errors @gdb_function_value_to_unicode def invoke(self, python_expression): input_type = libpython.PythonCodeExecutor.Py_eval_input diff --git a/Cython/Debugger/libpython.py b/Cython/Debugger/libpython.py index fea626dd7..30713e0d2 100644 --- a/Cython/Debugger/libpython.py +++ b/Cython/Debugger/libpython.py @@ -1,9 +1,10 @@ #!/usr/bin/python -# NOTE: this file is taken from the Python source distribution +# NOTE: Most of this file is taken from the Python source distribution # It can be found under Tools/gdb/libpython.py. It is shipped with Cython # because it's not installed as a python module, and because changes are only # merged into new python versions (v3.2+). +# We added some of our code below the "## added, not in CPython" comment. ''' From gdb 7 onwards, gdb's build can be configured --with-python, allowing gdb @@ -105,6 +106,8 @@ hexdigits = "0123456789abcdef" ENCODING = locale.getpreferredencoding() +FRAME_INFO_OPTIMIZED_OUT = '(frame information optimized out)' +UNABLE_READ_INFO_PYTHON_FRAME = 'Unable to read information on python frame' EVALFRAME = '_PyEval_EvalFrameDefault' class NullPyObjectPtr(RuntimeError): @@ -276,12 +279,13 @@ class PyObjectPtr(object): def safe_tp_name(self): try: - return self.type().field('tp_name').string() - except NullPyObjectPtr: - # NULL tp_name? - return 'unknown' - except RuntimeError: - # Can't even read the object at all? + ob_type = self.type() + tp_name = ob_type.field('tp_name') + return tp_name.string() + # NullPyObjectPtr: NULL tp_name? + # RuntimeError: Can't even read the object at all? + # UnicodeDecodeError: Failed to decode tp_name bytestring + except (NullPyObjectPtr, RuntimeError, UnicodeDecodeError): return 'unknown' def proxyval(self, visited): @@ -355,7 +359,9 @@ class PyObjectPtr(object): try: tp_name = t.field('tp_name').string() tp_flags = int(t.field('tp_flags')) - except RuntimeError: + # RuntimeError: NULL pointers + # UnicodeDecodeError: string() fails to decode the bytestring + except (RuntimeError, UnicodeDecodeError): # Handle any kind of error e.g. NULL ptrs by simply using the base # class return cls @@ -622,8 +628,11 @@ class PyCFunctionObjectPtr(PyObjectPtr): _typename = 'PyCFunctionObject' def proxyval(self, visited): - m_ml = self.field('m_ml') # m_ml is a (PyMethodDef*) - ml_name = m_ml['ml_name'].string() + m_ml = self.field('m_ml') # m_ml is a (PyMethodDef*) + try: + ml_name = m_ml['ml_name'].string() + except UnicodeDecodeError: + ml_name = '<ml_name:UnicodeDecodeError>' pyop_m_self = self.pyop_field('m_self') if pyop_m_self.is_null(): @@ -736,7 +745,7 @@ class PyDictObjectPtr(PyObjectPtr): else: offset = 8 * dk_size - ent_addr = keys['dk_indices']['as_1'].address + ent_addr = keys['dk_indices'].address ent_addr = ent_addr.cast(_type_unsigned_char_ptr()) + offset ent_ptr_t = gdb.lookup_type('PyDictKeyEntry').pointer() ent_addr = ent_addr.cast(ent_ptr_t) @@ -918,7 +927,7 @@ class PyFrameObjectPtr(PyObjectPtr): def filename(self): '''Get the path of the current Python source file, as a string''' if self.is_optimized_out(): - return '(frame information optimized out)' + return FRAME_INFO_OPTIMIZED_OUT return self.co_filename.proxyval(set()) def current_line_num(self): @@ -934,35 +943,50 @@ class PyFrameObjectPtr(PyObjectPtr): if long(f_trace) != 0: # we have a non-NULL f_trace: return self.f_lineno - else: - #try: + + try: return self.co.addr2line(self.f_lasti) - #except ValueError: - # return self.f_lineno + except Exception: + # bpo-34989: addr2line() is a complex function, it can fail in many + # ways. For example, it fails with a TypeError on "FakeRepr" if + # gdb fails to load debug symbols. Use a catch-all "except + # Exception" to make the whole function safe. The caller has to + # handle None anyway for optimized Python. + return None def current_line(self): '''Get the text of the current source line as a string, with a trailing newline character''' if self.is_optimized_out(): - return '(frame information optimized out)' + return FRAME_INFO_OPTIMIZED_OUT + + lineno = self.current_line_num() + if lineno is None: + return '(failed to get frame line number)' + filename = self.filename() try: - f = open(os_fsencode(filename), 'r') + with open(os_fsencode(filename), 'r') as fp: + lines = fp.readlines() except IOError: return None - with f: - all_lines = f.readlines() - # Convert from 1-based current_line_num to 0-based list offset: - return all_lines[self.current_line_num()-1] + + try: + # Convert from 1-based current_line_num to 0-based list offset + return lines[lineno - 1] + except IndexError: + return None def write_repr(self, out, visited): if self.is_optimized_out(): - out.write('(frame information optimized out)') + out.write(FRAME_INFO_OPTIMIZED_OUT) return - out.write('Frame 0x%x, for file %s, line %i, in %s (' + lineno = self.current_line_num() + lineno = str(lineno) if lineno is not None else "?" + out.write('Frame 0x%x, for file %s, line %s, in %s (' % (self.as_address(), self.co_filename.proxyval(visited), - self.current_line_num(), + lineno, self.co_name.proxyval(visited))) first = True for pyop_name, pyop_value in self.iter_locals(): @@ -978,12 +1002,14 @@ class PyFrameObjectPtr(PyObjectPtr): def print_traceback(self): if self.is_optimized_out(): - sys.stdout.write(' (frame information optimized out)\n') + sys.stdout.write(' %s\n' % FRAME_INFO_OPTIMIZED_OUT) return visited = set() - sys.stdout.write(' File "%s", line %i, in %s\n' + lineno = self.current_line_num() + lineno = str(lineno) if lineno is not None else "?" + sys.stdout.write(' File "%s", line %s, in %s\n' % (self.co_filename.proxyval(visited), - self.current_line_num(), + lineno, self.co_name.proxyval(visited))) class PySetObjectPtr(PyObjectPtr): @@ -1091,11 +1117,6 @@ class PyBytesObjectPtr(PyObjectPtr): out.write(byte) out.write(quote) - -class PyStringObjectPtr(PyBytesObjectPtr): - _typename = 'PyStringObject' - - class PyTupleObjectPtr(PyObjectPtr): _typename = 'PyTupleObject' @@ -1166,7 +1187,7 @@ class PyUnicodeObjectPtr(PyObjectPtr): def proxyval(self, visited): global _is_pep393 if _is_pep393 is None: - fields = gdb.lookup_type('PyUnicodeObject').target().fields() + fields = gdb.lookup_type('PyUnicodeObject').fields() _is_pep393 = 'data' in [f.name for f in fields] if _is_pep393: # Python 3.3 and newer @@ -1285,8 +1306,8 @@ class PyUnicodeObjectPtr(PyObjectPtr): # If sizeof(Py_UNICODE) is 2 here (in gdb), join # surrogate pairs before calling _unichr_is_printable. if (i < len(proxy) - and 0xD800 <= ord(ch) < 0xDC00 \ - and 0xDC00 <= ord(proxy[i]) <= 0xDFFF): + and 0xD800 <= ord(ch) < 0xDC00 + and 0xDC00 <= ord(proxy[i]) <= 0xDFFF): ch2 = proxy[i] ucs = ch + ch2 i += 1 @@ -1351,13 +1372,13 @@ class wrapperobject(PyObjectPtr): try: name = self.field('descr')['d_base']['name'].string() return repr(name) - except (NullPyObjectPtr, RuntimeError): + except (NullPyObjectPtr, RuntimeError, UnicodeDecodeError): return '<unknown name>' def safe_tp_name(self): try: return self.field('self')['ob_type']['tp_name'].string() - except (NullPyObjectPtr, RuntimeError): + except (NullPyObjectPtr, RuntimeError, UnicodeDecodeError): return '<unknown tp_name>' def safe_self_addresss(self): @@ -1380,7 +1401,7 @@ class wrapperobject(PyObjectPtr): def int_from_int(gdbval): - return int(str(gdbval)) + return int(gdbval) def stringify(val): @@ -1551,8 +1572,8 @@ class Frame(object): if not caller: return False - if caller in ('_PyCFunction_FastCallDict', - '_PyCFunction_FastCallKeywords'): + if (caller.startswith('cfunction_vectorcall_') or + caller == 'cfunction_call'): arg_name = 'func' # Within that frame: # "func" is the local containing the PyObject* of the @@ -1563,15 +1584,22 @@ class Frame(object): # Use the prettyprinter for the func: func = frame.read_var(arg_name) return str(func) + except ValueError: + return ('PyCFunction invocation (unable to read %s: ' + 'missing debuginfos?)' % arg_name) except RuntimeError: return 'PyCFunction invocation (unable to read %s)' % arg_name if caller == 'wrapper_call': + arg_name = 'wp' try: - func = frame.read_var('wp') + func = frame.read_var(arg_name) return str(func) + except ValueError: + return ('<wrapper_call invocation (unable to read %s: ' + 'missing debuginfos?)>' % arg_name) except RuntimeError: - return '<wrapper_call invocation>' + return '<wrapper_call invocation (unable to read %s)>' % arg_name # This frame isn't worth reporting: return False @@ -1581,7 +1609,7 @@ class Frame(object): # This assumes the _POSIX_THREADS version of Python/ceval_gil.h: name = self._gdbframe.name() if name: - return 'pthread_cond_timedwait' in name + return (name == 'take_gil') def is_gc_collect(self): '''Is this frame "collect" within the garbage-collector?''' @@ -1725,11 +1753,14 @@ class PyList(gdb.Command): pyop = frame.get_pyop() if not pyop or pyop.is_optimized_out(): - print('Unable to read information on python frame') + print(UNABLE_READ_INFO_PYTHON_FRAME) return filename = pyop.filename() lineno = pyop.current_line_num() + if lineno is None: + print('Unable to read python frame line number') + return if start is None: start = lineno - 5 @@ -1882,7 +1913,7 @@ class PyPrint(gdb.Command): pyop_frame = frame.get_pyop() if not pyop_frame: - print('Unable to read information on python frame') + print(UNABLE_READ_INFO_PYTHON_FRAME) return pyop_var, scope = pyop_frame.get_var_by_name(name) @@ -1899,9 +1930,9 @@ PyPrint() class PyLocals(gdb.Command): 'Look up the given python variable name, and print it' - def __init__(self, command="py-locals"): + def __init__(self): gdb.Command.__init__ (self, - command, + "py-locals", gdb.COMMAND_DATA, gdb.COMPLETE_NONE) @@ -1916,22 +1947,14 @@ class PyLocals(gdb.Command): pyop_frame = frame.get_pyop() if not pyop_frame: - print('Unable to read information on python frame') + print(UNABLE_READ_INFO_PYTHON_FRAME) return - namespace = self.get_namespace(pyop_frame) - namespace = [(name.proxyval(set()), val) for name, val in namespace] - - if namespace: - name, val = max(namespace, key=lambda item: len(item[0])) - max_name_length = len(name) - - for name, pyop_value in namespace: - value = pyop_value.get_truncated_repr(MAX_OUTPUT_LEN) - print('%-*s = %s' % (max_name_length, name, value)) - - def get_namespace(self, pyop_frame): - return pyop_frame.iter_locals() + for pyop_name, pyop_value in pyop_frame.iter_locals(): + print('%s = %s' % ( + pyop_name.proxyval(set()), + pyop_value.get_truncated_repr(MAX_OUTPUT_LEN), + )) PyLocals() @@ -1943,24 +1966,80 @@ PyLocals() import re import warnings import tempfile +import functools import textwrap import itertools +import traceback -class PyGlobals(PyLocals): + +def dont_suppress_errors(function): + "*sigh*, readline" + @functools.wraps(function) + def wrapper(*args, **kwargs): + try: + return function(*args, **kwargs) + except Exception: + traceback.print_exc() + raise + + return wrapper + +class PyGlobals(gdb.Command): 'List all the globals in the currently select Python frame' + def __init__(self): + gdb.Command.__init__ (self, + "py-globals", + gdb.COMMAND_DATA, + gdb.COMPLETE_NONE) + + @dont_suppress_errors + def invoke(self, args, from_tty): + name = str(args) + + frame = Frame.get_selected_python_frame() + if not frame: + print('Unable to locate python frame') + return + + pyop_frame = frame.get_pyop() + if not pyop_frame: + print(UNABLE_READ_INFO_PYTHON_FRAME) + return + + for pyop_name, pyop_value in pyop_frame.iter_locals(): + print('%s = %s' + % (pyop_name.proxyval(set()), + pyop_value.get_truncated_repr(MAX_OUTPUT_LEN))) def get_namespace(self, pyop_frame): return pyop_frame.iter_globals() -PyGlobals("py-globals") +PyGlobals() +# This function used to be a part of CPython's libpython.py (as a member function of frame). +# It isn't anymore, so I copied it. +def is_evalframeex(frame): + '''Is this a PyEval_EvalFrameEx frame?''' + if frame._gdbframe.name() == 'PyEval_EvalFrameEx': + ''' + I believe we also need to filter on the inline + struct frame_id.inline_depth, only regarding frames with + an inline depth of 0 as actually being this function + + So we reject those with type gdb.INLINE_FRAME + ''' + if frame._gdbframe.type() == gdb.NORMAL_FRAME: + # We have a PyEval_EvalFrameEx frame: + return True + + return False class PyNameEquals(gdb.Function): def _get_pycurframe_attr(self, attr): frame = Frame(gdb.selected_frame()) - if frame.is_evalframeex(): + if is_evalframeex(frame): pyframe = frame.get_pyop() if pyframe is None: warnings.warn("Use a Python debug build, Python breakpoints " @@ -1971,6 +2050,7 @@ class PyNameEquals(gdb.Function): return None + @dont_suppress_errors def invoke(self, funcname): attr = self._get_pycurframe_attr('co_name') return attr is not None and attr == funcname.string() @@ -1980,6 +2060,7 @@ PyNameEquals("pyname_equals") class PyModEquals(PyNameEquals): + @dont_suppress_errors def invoke(self, modname): attr = self._get_pycurframe_attr('co_filename') if attr is not None: @@ -2003,6 +2084,7 @@ class PyBreak(gdb.Command): py-break func """ + @dont_suppress_errors def invoke(self, funcname, from_tty): if '.' in funcname: modname, dot, funcname = funcname.rpartition('.') @@ -2457,6 +2539,7 @@ class PyStep(ExecutionControlCommandBase, PythonStepperMixin): stepinto = True + @dont_suppress_errors def invoke(self, args, from_tty): self.python_step(stepinto=self.stepinto) @@ -2470,18 +2553,18 @@ class PyNext(PyStep): class PyFinish(ExecutionControlCommandBase): "Execute until function returns to a caller." - invoke = ExecutionControlCommandBase.finish + invoke = dont_suppress_errors(ExecutionControlCommandBase.finish) class PyRun(ExecutionControlCommandBase): "Run the program." - invoke = ExecutionControlCommandBase.run + invoke = dont_suppress_errors(ExecutionControlCommandBase.run) class PyCont(ExecutionControlCommandBase): - invoke = ExecutionControlCommandBase.cont + invoke = dont_suppress_errors(ExecutionControlCommandBase.cont) def _pointervalue(gdbval): @@ -2574,7 +2657,7 @@ class PythonCodeExecutor(object): return pointer def free(self, pointer): - gdb.parse_and_eval("free((void *) %d)" % pointer) + gdb.parse_and_eval("(void) free((void *) %d)" % pointer) def incref(self, pointer): "Increment the reference count of a Python object in the inferior." @@ -2693,6 +2776,7 @@ class FixGdbCommand(gdb.Command): pass # warnings.resetwarnings() + @dont_suppress_errors def invoke(self, args, from_tty): self.fix_gdb() try: @@ -2726,7 +2810,10 @@ class PyExec(gdb.Command): lines = [] while True: try: - line = input('>') + if sys.version_info[0] == 2: + line = raw_input() + else: + line = input('>') except EOFError: break else: @@ -2737,6 +2824,7 @@ class PyExec(gdb.Command): return '\n'.join(lines), PythonCodeExecutor.Py_file_input + @dont_suppress_errors def invoke(self, expr, from_tty): expr, input_type = self.readcode(expr) executor = PythonCodeExecutor() diff --git a/Cython/Distutils/old_build_ext.py b/Cython/Distutils/old_build_ext.py index aa2a1cf22..3595d80e0 100644 --- a/Cython/Distutils/old_build_ext.py +++ b/Cython/Distutils/old_build_ext.py @@ -8,7 +8,6 @@ Note that this module is deprecated. Use cythonize() instead. __revision__ = "$Id:$" -import inspect import sys import os from distutils.errors import DistutilsPlatformError @@ -16,7 +15,6 @@ from distutils.dep_util import newer, newer_group from distutils import log from distutils.command import build_ext as _build_ext from distutils import sysconfig -import warnings try: @@ -25,22 +23,30 @@ except ImportError: basestring = str +# FIXME: the below does not work as intended since importing 'Cython.Distutils' already +# imports this module through 'Cython/Distutils/build_ext.py', so the condition is +# always false and never prints the warning. +""" +import inspect +import warnings + def _check_stack(path): - try: - for frame in inspect.getouterframes(inspect.currentframe(), 0): - if path in frame[1].replace(os.sep, '/'): - return True - except Exception: - pass - return False + try: + for frame in inspect.getouterframes(inspect.currentframe(), 0): + if path in frame[1].replace(os.sep, '/'): + return True + except Exception: + pass + return False + if (not _check_stack('setuptools/extensions.py') - and not _check_stack('pyximport/pyxbuild.py') - and not _check_stack('Cython/Distutils/build_ext.py')): + and not _check_stack('pyximport/pyxbuild.py') + and not _check_stack('Cython/Distutils/build_ext.py')): warnings.warn( "Cython.Distutils.old_build_ext does not properly handle dependencies " "and is deprecated.") - +""" extension_name_re = _build_ext.extension_name_re @@ -163,7 +169,7 @@ class old_build_ext(_build_ext.build_ext): # _build_ext.build_ext.__setattr__(self, name, value) self.__dict__[name] = value - def finalize_options (self): + def finalize_options(self): _build_ext.build_ext.finalize_options(self) if self.cython_include_dirs is None: self.cython_include_dirs = [] @@ -185,14 +191,11 @@ class old_build_ext(_build_ext.build_ext): _build_ext.build_ext.run(self) - def build_extensions(self): - # First, sanity-check the 'extensions' list - self.check_extensions_list(self.extensions) - + def check_extensions_list(self, extensions): + # Note: might get called multiple times. + _build_ext.build_ext.check_extensions_list(self, extensions) for ext in self.extensions: ext.sources = self.cython_sources(ext.sources, ext) - # Call original build_extensions - _build_ext.build_ext.build_extensions(self) def cython_sources(self, sources, extension): """ @@ -201,17 +204,6 @@ class old_build_ext(_build_ext.build_ext): found, and return a modified 'sources' list with Cython source files replaced by the generated C (or C++) files. """ - try: - from Cython.Compiler.Main \ - import CompilationOptions, \ - default_options as cython_default_options, \ - compile as cython_compile - from Cython.Compiler.Errors import PyrexError - except ImportError: - e = sys.exc_info()[1] - print("failed to import Cython: %s" % e) - raise DistutilsPlatformError("Cython does not appear to be installed") - new_sources = [] cython_sources = [] cython_targets = {} @@ -252,10 +244,10 @@ class old_build_ext(_build_ext.build_ext): # 2. Add in any (unique) paths from the extension # cython_include_dirs (if Cython.Distutils.extension is used). # 3. Add in any (unique) paths from the extension include_dirs - includes = self.cython_include_dirs + includes = list(self.cython_include_dirs) try: for i in extension.cython_include_dirs: - if not i in includes: + if i not in includes: includes.append(i) except AttributeError: pass @@ -264,19 +256,18 @@ class old_build_ext(_build_ext.build_ext): # result extension.include_dirs = list(extension.include_dirs) for i in extension.include_dirs: - if not i in includes: + if i not in includes: includes.append(i) # Set up Cython compiler directives: # 1. Start with the command line option. # 2. Add in any (unique) entries from the extension # cython_directives (if Cython.Distutils.extension is used). - directives = self.cython_directives + directives = dict(self.cython_directives) if hasattr(extension, "cython_directives"): directives.update(extension.cython_directives) - # Set the target_ext to '.c'. Cython will change this to '.cpp' if - # needed. + # Set the target file extension for C/C++ mode. if cplus: target_ext = '.cpp' else: @@ -314,6 +305,17 @@ class old_build_ext(_build_ext.build_ext): if not cython_sources: return new_sources + try: + from Cython.Compiler.Main \ + import CompilationOptions, \ + default_options as cython_default_options, \ + compile as cython_compile + from Cython.Compiler.Errors import PyrexError + except ImportError: + e = sys.exc_info()[1] + print("failed to import Cython: %s" % e) + raise DistutilsPlatformError("Cython does not appear to be installed") + module_name = extension.name for source in cython_sources: diff --git a/Cython/Includes/Deprecated/python.pxd b/Cython/Includes/Deprecated/python.pxd deleted file mode 100644 index 56236e925..000000000 --- a/Cython/Includes/Deprecated/python.pxd +++ /dev/null @@ -1,2 +0,0 @@ -# Present for backwards compatibility -from cpython cimport * diff --git a/Cython/Includes/Deprecated/python_bool.pxd b/Cython/Includes/Deprecated/python_bool.pxd deleted file mode 100644 index 9a6d253f4..000000000 --- a/Cython/Includes/Deprecated/python_bool.pxd +++ /dev/null @@ -1,2 +0,0 @@ -# Present for backwards compatibility -from cpython.bool cimport * diff --git a/Cython/Includes/Deprecated/python_buffer.pxd b/Cython/Includes/Deprecated/python_buffer.pxd deleted file mode 100644 index 2baeaae00..000000000 --- a/Cython/Includes/Deprecated/python_buffer.pxd +++ /dev/null @@ -1,2 +0,0 @@ -# Present for backwards compatibility -from cpython.buffer cimport * diff --git a/Cython/Includes/Deprecated/python_bytes.pxd b/Cython/Includes/Deprecated/python_bytes.pxd deleted file mode 100644 index 87af662de..000000000 --- a/Cython/Includes/Deprecated/python_bytes.pxd +++ /dev/null @@ -1,2 +0,0 @@ -# Present for backwards compatibility -from cpython.bytes cimport * diff --git a/Cython/Includes/Deprecated/python_cobject.pxd b/Cython/Includes/Deprecated/python_cobject.pxd deleted file mode 100644 index ed32c6b87..000000000 --- a/Cython/Includes/Deprecated/python_cobject.pxd +++ /dev/null @@ -1,2 +0,0 @@ -# Present for backwards compatibility -from cpython.cobject cimport * diff --git a/Cython/Includes/Deprecated/python_complex.pxd b/Cython/Includes/Deprecated/python_complex.pxd deleted file mode 100644 index 0a780b3b2..000000000 --- a/Cython/Includes/Deprecated/python_complex.pxd +++ /dev/null @@ -1,2 +0,0 @@ -# Present for backwards compatibility -from cpython.complex cimport * diff --git a/Cython/Includes/Deprecated/python_dict.pxd b/Cython/Includes/Deprecated/python_dict.pxd deleted file mode 100644 index 05b5f4796..000000000 --- a/Cython/Includes/Deprecated/python_dict.pxd +++ /dev/null @@ -1,2 +0,0 @@ -# Present for backwards compatibility -from cpython.dict cimport * diff --git a/Cython/Includes/Deprecated/python_exc.pxd b/Cython/Includes/Deprecated/python_exc.pxd deleted file mode 100644 index 6eb236bcc..000000000 --- a/Cython/Includes/Deprecated/python_exc.pxd +++ /dev/null @@ -1,2 +0,0 @@ -# Present for backwards compatibility -from cpython.exc cimport * diff --git a/Cython/Includes/Deprecated/python_float.pxd b/Cython/Includes/Deprecated/python_float.pxd deleted file mode 100644 index 7e133ef9b..000000000 --- a/Cython/Includes/Deprecated/python_float.pxd +++ /dev/null @@ -1,2 +0,0 @@ -# Present for backwards compatibility -from cpython.float cimport * diff --git a/Cython/Includes/Deprecated/python_function.pxd b/Cython/Includes/Deprecated/python_function.pxd deleted file mode 100644 index 1461c4e63..000000000 --- a/Cython/Includes/Deprecated/python_function.pxd +++ /dev/null @@ -1,2 +0,0 @@ -# Present for backwards compatibility -from cpython.function cimport * diff --git a/Cython/Includes/Deprecated/python_getargs.pxd b/Cython/Includes/Deprecated/python_getargs.pxd deleted file mode 100644 index 3852d6a6a..000000000 --- a/Cython/Includes/Deprecated/python_getargs.pxd +++ /dev/null @@ -1,2 +0,0 @@ -# Present for backwards compatibility -from cpython.getargs cimport * diff --git a/Cython/Includes/Deprecated/python_instance.pxd b/Cython/Includes/Deprecated/python_instance.pxd deleted file mode 100644 index 99cb5a909..000000000 --- a/Cython/Includes/Deprecated/python_instance.pxd +++ /dev/null @@ -1,2 +0,0 @@ -# Present for backwards compatibility -from cpython.instance cimport * diff --git a/Cython/Includes/Deprecated/python_int.pxd b/Cython/Includes/Deprecated/python_int.pxd deleted file mode 100644 index c1fd5178d..000000000 --- a/Cython/Includes/Deprecated/python_int.pxd +++ /dev/null @@ -1,2 +0,0 @@ -# Present for backwards compatibility -from cpython.int cimport * diff --git a/Cython/Includes/Deprecated/python_iterator.pxd b/Cython/Includes/Deprecated/python_iterator.pxd deleted file mode 100644 index e09aad279..000000000 --- a/Cython/Includes/Deprecated/python_iterator.pxd +++ /dev/null @@ -1,2 +0,0 @@ -# Present for backwards compatibility -from cpython.iterator cimport * diff --git a/Cython/Includes/Deprecated/python_list.pxd b/Cython/Includes/Deprecated/python_list.pxd deleted file mode 100644 index 64febcf96..000000000 --- a/Cython/Includes/Deprecated/python_list.pxd +++ /dev/null @@ -1,2 +0,0 @@ -# Present for backwards compatibility -from cpython.list cimport * diff --git a/Cython/Includes/Deprecated/python_long.pxd b/Cython/Includes/Deprecated/python_long.pxd deleted file mode 100644 index 1a24380c4..000000000 --- a/Cython/Includes/Deprecated/python_long.pxd +++ /dev/null @@ -1,2 +0,0 @@ -# Present for backwards compatibility -from cpython.long cimport * diff --git a/Cython/Includes/Deprecated/python_mapping.pxd b/Cython/Includes/Deprecated/python_mapping.pxd deleted file mode 100644 index cd01bee01..000000000 --- a/Cython/Includes/Deprecated/python_mapping.pxd +++ /dev/null @@ -1,2 +0,0 @@ -# Present for backwards compatibility -from cpython.mapping cimport * diff --git a/Cython/Includes/Deprecated/python_mem.pxd b/Cython/Includes/Deprecated/python_mem.pxd deleted file mode 100644 index d74429ea3..000000000 --- a/Cython/Includes/Deprecated/python_mem.pxd +++ /dev/null @@ -1,2 +0,0 @@ -# Present for backwards compatibility -from cpython.mem cimport * diff --git a/Cython/Includes/Deprecated/python_method.pxd b/Cython/Includes/Deprecated/python_method.pxd deleted file mode 100644 index e7da5154e..000000000 --- a/Cython/Includes/Deprecated/python_method.pxd +++ /dev/null @@ -1,2 +0,0 @@ -# Present for backwards compatibility -from cpython.method cimport * diff --git a/Cython/Includes/Deprecated/python_module.pxd b/Cython/Includes/Deprecated/python_module.pxd deleted file mode 100644 index 6310c0247..000000000 --- a/Cython/Includes/Deprecated/python_module.pxd +++ /dev/null @@ -1,2 +0,0 @@ -# Present for backwards compatibility -from cpython.module cimport * diff --git a/Cython/Includes/Deprecated/python_number.pxd b/Cython/Includes/Deprecated/python_number.pxd deleted file mode 100644 index ae67da1c3..000000000 --- a/Cython/Includes/Deprecated/python_number.pxd +++ /dev/null @@ -1,2 +0,0 @@ -# Present for backwards compatibility -from cpython.number cimport * diff --git a/Cython/Includes/Deprecated/python_object.pxd b/Cython/Includes/Deprecated/python_object.pxd deleted file mode 100644 index 3981bfa44..000000000 --- a/Cython/Includes/Deprecated/python_object.pxd +++ /dev/null @@ -1,2 +0,0 @@ -# Present for backwards compatibility -from cpython.object cimport * diff --git a/Cython/Includes/Deprecated/python_oldbuffer.pxd b/Cython/Includes/Deprecated/python_oldbuffer.pxd deleted file mode 100644 index e03e66a2e..000000000 --- a/Cython/Includes/Deprecated/python_oldbuffer.pxd +++ /dev/null @@ -1,2 +0,0 @@ -# Present for backwards compatibility -from cpython.oldbuffer cimport * diff --git a/Cython/Includes/Deprecated/python_pycapsule.pxd b/Cython/Includes/Deprecated/python_pycapsule.pxd deleted file mode 100644 index fe9cf8f8d..000000000 --- a/Cython/Includes/Deprecated/python_pycapsule.pxd +++ /dev/null @@ -1,2 +0,0 @@ -# Present for backwards compatibility -from cpython.pycapsule cimport * diff --git a/Cython/Includes/Deprecated/python_ref.pxd b/Cython/Includes/Deprecated/python_ref.pxd deleted file mode 100644 index 944741819..000000000 --- a/Cython/Includes/Deprecated/python_ref.pxd +++ /dev/null @@ -1,2 +0,0 @@ -# Present for backwards compatibility -from cpython.ref cimport * diff --git a/Cython/Includes/Deprecated/python_sequence.pxd b/Cython/Includes/Deprecated/python_sequence.pxd deleted file mode 100644 index fdef5b63e..000000000 --- a/Cython/Includes/Deprecated/python_sequence.pxd +++ /dev/null @@ -1,2 +0,0 @@ -# Present for backwards compatibility -from cpython.sequence cimport * diff --git a/Cython/Includes/Deprecated/python_set.pxd b/Cython/Includes/Deprecated/python_set.pxd deleted file mode 100644 index a2feb9371..000000000 --- a/Cython/Includes/Deprecated/python_set.pxd +++ /dev/null @@ -1,2 +0,0 @@ -# Present for backwards compatibility -from cpython.set cimport * diff --git a/Cython/Includes/Deprecated/python_string.pxd b/Cython/Includes/Deprecated/python_string.pxd deleted file mode 100644 index 24c818338..000000000 --- a/Cython/Includes/Deprecated/python_string.pxd +++ /dev/null @@ -1,2 +0,0 @@ -# Present for backwards compatibility -from cpython.string cimport * diff --git a/Cython/Includes/Deprecated/python_tuple.pxd b/Cython/Includes/Deprecated/python_tuple.pxd deleted file mode 100644 index 190713b02..000000000 --- a/Cython/Includes/Deprecated/python_tuple.pxd +++ /dev/null @@ -1,2 +0,0 @@ -# Present for backwards compatibility -from cpython.tuple cimport * diff --git a/Cython/Includes/Deprecated/python_type.pxd b/Cython/Includes/Deprecated/python_type.pxd deleted file mode 100644 index 3ac47d1b3..000000000 --- a/Cython/Includes/Deprecated/python_type.pxd +++ /dev/null @@ -1,2 +0,0 @@ -# Present for backwards compatibility -from cpython.type cimport * diff --git a/Cython/Includes/Deprecated/python_unicode.pxd b/Cython/Includes/Deprecated/python_unicode.pxd deleted file mode 100644 index 2b488b2dc..000000000 --- a/Cython/Includes/Deprecated/python_unicode.pxd +++ /dev/null @@ -1,2 +0,0 @@ -# Present for backwards compatibility -from cpython.unicode cimport * diff --git a/Cython/Includes/Deprecated/python_version.pxd b/Cython/Includes/Deprecated/python_version.pxd deleted file mode 100644 index c27ca4df9..000000000 --- a/Cython/Includes/Deprecated/python_version.pxd +++ /dev/null @@ -1,2 +0,0 @@ -# Present for backwards compatibility -from cpython.version cimport * diff --git a/Cython/Includes/Deprecated/python_weakref.pxd b/Cython/Includes/Deprecated/python_weakref.pxd deleted file mode 100644 index 1f84f1a17..000000000 --- a/Cython/Includes/Deprecated/python_weakref.pxd +++ /dev/null @@ -1,2 +0,0 @@ -# Present for backwards compatibility -from cpython.weakref cimport * diff --git a/Cython/Includes/Deprecated/stdio.pxd b/Cython/Includes/Deprecated/stdio.pxd deleted file mode 100644 index 41a4aebf1..000000000 --- a/Cython/Includes/Deprecated/stdio.pxd +++ /dev/null @@ -1,2 +0,0 @@ -# Present for backwards compatibility -from libc.stdio cimport * diff --git a/Cython/Includes/Deprecated/stdlib.pxd b/Cython/Includes/Deprecated/stdlib.pxd deleted file mode 100644 index 499511cde..000000000 --- a/Cython/Includes/Deprecated/stdlib.pxd +++ /dev/null @@ -1,2 +0,0 @@ -# Present for backwards compatibility -from libc.stdlib cimport * diff --git a/Cython/Includes/Deprecated/stl.pxd b/Cython/Includes/Deprecated/stl.pxd deleted file mode 100644 index 22248d265..000000000 --- a/Cython/Includes/Deprecated/stl.pxd +++ /dev/null @@ -1,91 +0,0 @@ -cdef extern from "<vector>" namespace std: - - cdef cppclass vector[TYPE]: - #constructors - __init__() - __init__(vector&) - __init__(int) - __init__(int, TYPE&) - __init__(iterator, iterator) - #operators - TYPE& __getitem__(int) - TYPE& __setitem__(int, TYPE&) - vector __new__(vector&) - bool __eq__(vector&, vector&) - bool __ne__(vector&, vector&) - bool __lt__(vector&, vector&) - bool __gt__(vector&, vector&) - bool __le__(vector&, vector&) - bool __ge__(vector&, vector&) - #others - void assign(int, TYPE) - #void assign(iterator, iterator) - TYPE& at(int) - TYPE& back() - iterator begin() - int capacity() - void clear() - bool empty() - iterator end() - iterator erase(iterator) - iterator erase(iterator, iterator) - TYPE& front() - iterator insert(iterator, TYPE&) - void insert(iterator, int, TYPE&) - void insert(iterator, iterator) - int max_size() - void pop_back() - void push_back(TYPE&) - iterator rbegin() - iterator rend() - void reserve(int) - void resize(int) - void resize(int, TYPE&) #void resize(size_type num, const TYPE& = TYPE()) - int size() - void swap(container&) - -cdef extern from "<deque>" namespace std: - - cdef cppclass deque[TYPE]: - #constructors - __init__() - __init__(deque&) - __init__(int) - __init__(int, TYPE&) - __init__(iterator, iterator) - #operators - TYPE& operator[]( size_type index ); - const TYPE& operator[]( size_type index ) const; - deque __new__(deque&); - bool __eq__(deque&, deque&); - bool __ne__(deque&, deque&); - bool __lt__(deque&, deque&); - bool __gt__(deque&, deque&); - bool __le__(deque&, deque&); - bool __ge__(deque&, deque&); - #others - void assign(int, TYPE&) - void assign(iterator, iterator) - TYPE& at(int) - TYPE& back() - iterator begin() - void clear() - bool empty() - iterator end() - iterator erase(iterator) - iterator erase(iterator, iterator) - TYPE& front() - iterator insert(iterator, TYPE&) - void insert(iterator, int, TYPE&) - void insert(iterator, iterator, iterator) - int max_size() - void pop_back() - void pop_front() - void push_back(TYPE&) - void push_front(TYPE&) - iterator rbegin() - iterator rend() - void resize(int) - void resize(int, TYPE&) - int size() - void swap(container&) diff --git a/Cython/Includes/cpython/__init__.pxd b/Cython/Includes/cpython/__init__.pxd index c81f4e665..7ad2684aa 100644 --- a/Cython/Includes/cpython/__init__.pxd +++ b/Cython/Includes/cpython/__init__.pxd @@ -179,6 +179,9 @@ from cpython.bytes cimport * # Python >= 3.0 from cpython.pycapsule cimport * +# Python >= 3.7 +from cpython.contextvars cimport * + ################################################################# # END OF DEPRECATED SECTION ################################################################# diff --git a/Cython/Includes/cpython/array.pxd b/Cython/Includes/cpython/array.pxd index 19230a0a8..8431f7b66 100644 --- a/Cython/Includes/cpython/array.pxd +++ b/Cython/Includes/cpython/array.pxd @@ -46,8 +46,19 @@ : 2012-05-02 andreasvc : (see revision control) """ -from libc.string cimport strcat, strncat, \ - memset, memchr, memcmp, memcpy, memmove + +cdef extern from *: + """ + #if CYTHON_COMPILING_IN_PYPY + #ifdef _MSC_VER + #pragma message ("This module uses CPython specific internals of 'array.array', which are not available in PyPy.") + #else + #warning This module uses CPython specific internals of 'array.array', which are not available in PyPy. + #endif + #endif + """ + +from libc.string cimport memset, memcpy from cpython.object cimport Py_SIZE from cpython.ref cimport PyTypeObject, Py_TYPE diff --git a/Cython/Includes/cpython/bytes.pxd b/Cython/Includes/cpython/bytes.pxd index ea72c6aae..a7d1d09ac 100644 --- a/Cython/Includes/cpython/bytes.pxd +++ b/Cython/Includes/cpython/bytes.pxd @@ -68,6 +68,10 @@ cdef extern from "Python.h": # Return value: New reference. # Identical to PyBytes_FromFormat() except that it takes exactly two arguments. + bytes PyBytes_FromObject(object o) + # Return value: New reference. + # Return the bytes representation of object o that implements the buffer protocol. + Py_ssize_t PyBytes_Size(object string) except -1 # Return the length of the string in string object string. diff --git a/Cython/Includes/cpython/complex.pxd b/Cython/Includes/cpython/complex.pxd index f5ba33957..3fa145008 100644 --- a/Cython/Includes/cpython/complex.pxd +++ b/Cython/Includes/cpython/complex.pxd @@ -14,9 +14,14 @@ cdef extern from "Python.h": ctypedef class __builtin__.complex [object PyComplexObject]: cdef Py_complex cval - # not making these available to keep them read-only: - #cdef double imag "cval.imag" - #cdef double real "cval.real" + + @property + cdef inline double real(self): + return self.cval.real + + @property + cdef inline double imag(self): + return self.cval.imag # PyTypeObject PyComplex_Type # This instance of PyTypeObject represents the Python complex diff --git a/Cython/Includes/cpython/contextvars.pxd b/Cython/Includes/cpython/contextvars.pxd new file mode 100644 index 000000000..cc73cb365 --- /dev/null +++ b/Cython/Includes/cpython/contextvars.pxd @@ -0,0 +1,140 @@ +from cpython.object cimport PyObject +from cpython.ref cimport Py_XDECREF + +cdef extern from "Python.h": + # Defining PyContextVar_Get() below to always return the default value for Py<3.7 + # to make the inline functions sort-of work. + """ + #if PY_VERSION_HEX < 0x030700b1 && !defined(PyContextVar_Get) + #define PyContextVar_Get(var, d, v) \ + ((d) ? \ + ((void)(var), Py_INCREF(d), (v)[0] = (d), 0) : \ + ((v)[0] = NULL, 0) \ + ) + #endif + """ + + ############################################################################ + # Context Variables Objects + ############################################################################ + + # PyContext + # The C structure used to represent a `contextvars.Context` object. + + # PyContextVar + # The C structure used to represent a `contextvars.ContextVar` object. + + # PyContextToken + # The C structure used to represent a `contextvars.Token` object. + + # PyTypeObject PyContext_Type + # Type object representing the `contextvars.Context` type. + + # PyTypeObject PyContextVar_Type + # Type object representing the `contextvars.ContextVar` type. + + # PyTypeObject PyContextToken_Type + # Type object representing the `contextvars.Token` type. + + bint PyContext_CheckExact(object obj) + # Return `true` if `obj` is of type `PyContext_Type`. + # `obj` must not be NULL. This function always succeeds. + + bint PyContextVar_CheckExact(object obj) + # Return `true` if `obj` is of type `PyContextVar_Type`. + # `obj` must not be NULL. This function always succeeds. + + bint PyContextToken_CheckExact(object obj) + # Return `true` if `obj` is of type `PyContextToken_Type`. + # `obj` must not be NULL. This function always succeeds. + + object PyContext_New() + # Return value: New reference. + # Create a new empty context object. + # Returns NULL if an error has occurred. + + object PyContext_Copy(object ctx) + # Return value: New reference. + # Create a shallow copy of the passed `ctx` context object. + # Returns NULL if an error has occurred. + + object PyContext_CopyCurrent() + # Return value: New reference. + # Create a shallow copy of the current thread context. + # Returns NULL if an error has occurred. + + int PyContext_Enter(object ctx) except -1 + # Set `ctx` as the current context for the current thread. + # Returns 0 on success, and -1 on error. + + int PyContext_Exit(object ctx) except -1 + # Deactivate the `ctx` context and restore the previous context + # as the current context for the current thread. + # Returns 0 on success, and -1 on error. + + object PyContextVar_New(const char* name, PyObject* default_value) + # Return value: New reference. + # Create a new ContextVar object. The `name` parameter is used + # for introspection and debug purposes. The `default_value` parameter + # may optionally specify the default value for the context variable. + # If an error has occurred, this function returns NULL. + + object PyContextVar_New_with_default "PyContextVar_New" (const char* name, object default_value) + # A different declaration of PyContextVar_New that requires a default value + # to be passed on call. + + int PyContextVar_Get(object var, PyObject* default_value, PyObject** value) except -1 + # Get the value of a context variable. + # Returns -1 if an error has occurred during lookup, and 0 if no error + # occurred, whether or not a value was found. + # + # If the context variable was found, `value` will be a pointer to it. + # If the context variable was not found, `value` will point to: + # + # • `default_value`, if not NULL; + # • the default value of `var`, if not NULL; + # • NULL + int PyContextVar_Get_with_default "PyContextVar_Get" (object var, object default_value, PyObject** value) except -1 + # A different declaration of PyContextVar_Get that requires a default value + # to be passed on call. + + object PyContextVar_Set(object var, object value) + # Return value: New reference. + # Set the value of `var` to `value` in the current context. + # Returns a token object for this value change, or NULL if an error has occurred. + + int PyContextVar_Reset(object var, object token) except -1 + # Reset the state of the `var` context variable to that it was in + # before `PyContextVar_Set()` that returned `token` was called. + # This function returns 0 on success and -1 on error. + + +cdef inline object get_value(var, default_value=None): + """Return a new reference to the value of the context variable, + or the default value of the context variable, + or None if no such value or default was found. + """ + cdef PyObject *value + PyContextVar_Get(var, NULL, &value) + if value is NULL: + # context variable does not have a default + pyvalue = default_value + else: + # value or default value of context variable + pyvalue = <object>value + Py_XDECREF(value) # PyContextVar_Get() returned an owned reference as 'PyObject*' + return pyvalue + + +cdef inline object get_value_no_default(var, default_value=None): + """Return a new reference to the value of the context variable, + or the provided default value if no such value was found. + + Ignores the default value of the context variable, if any. + """ + cdef PyObject *value + PyContextVar_Get(var, <PyObject*>default_value, &value) + # value of context variable or 'default_value' + pyvalue = <object>value + Py_XDECREF(value) # PyContextVar_Get() returned an owned reference as 'PyObject*' + return pyvalue diff --git a/Cython/Includes/cpython/datetime.pxd b/Cython/Includes/cpython/datetime.pxd index cd0f90719..7d6ee29f3 100644 --- a/Cython/Includes/cpython/datetime.pxd +++ b/Cython/Includes/cpython/datetime.pxd @@ -1,22 +1,161 @@ from cpython.object cimport PyObject +from cpython.version cimport PY_VERSION_HEX cdef extern from "Python.h": ctypedef struct PyTypeObject: pass cdef extern from "datetime.h": + """ + /* Backport for Python 2.x */ + #if PY_MAJOR_VERSION < 3 + #ifndef PyDateTime_DELTA_GET_DAYS + #define PyDateTime_DELTA_GET_DAYS(o) (((PyDateTime_Delta*)o)->days) + #endif + #ifndef PyDateTime_DELTA_GET_SECONDS + #define PyDateTime_DELTA_GET_SECONDS(o) (((PyDateTime_Delta*)o)->seconds) + #endif + #ifndef PyDateTime_DELTA_GET_MICROSECONDS + #define PyDateTime_DELTA_GET_MICROSECONDS(o) (((PyDateTime_Delta*)o)->microseconds) + #endif + #endif + + /* Backport for Python < 3.6 */ + #if PY_VERSION_HEX < 0x030600a4 + #ifndef PyDateTime_TIME_GET_FOLD + #define PyDateTime_TIME_GET_FOLD(o) ((void)(o), 0) + #endif + #ifndef PyDateTime_DATE_GET_FOLD + #define PyDateTime_DATE_GET_FOLD(o) ((void)(o), 0) + #endif + #endif + + /* Backport for Python < 3.6 */ + #if PY_VERSION_HEX < 0x030600a4 + #define __Pyx_DateTime_DateTimeWithFold(year, month, day, hour, minute, second, microsecond, tz, fold) \ + ((void)(fold), PyDateTimeAPI->DateTime_FromDateAndTime(year, month, day, hour, minute, second, \ + microsecond, tz, PyDateTimeAPI->DateTimeType)) + #define __Pyx_DateTime_TimeWithFold(hour, minute, second, microsecond, tz, fold) \ + ((void)(fold), PyDateTimeAPI->Time_FromTime(hour, minute, second, microsecond, tz, PyDateTimeAPI->TimeType)) + #else /* For Python 3.6+ so that we can pass tz */ + #define __Pyx_DateTime_DateTimeWithFold(year, month, day, hour, minute, second, microsecond, tz, fold) \ + PyDateTimeAPI->DateTime_FromDateAndTimeAndFold(year, month, day, hour, minute, second, \ + microsecond, tz, fold, PyDateTimeAPI->DateTimeType) + #define __Pyx_DateTime_TimeWithFold(hour, minute, second, microsecond, tz, fold) \ + PyDateTimeAPI->Time_FromTimeAndFold(hour, minute, second, microsecond, tz, fold, PyDateTimeAPI->TimeType) + #endif + + /* Backport for Python < 3.7 */ + #if PY_VERSION_HEX < 0x030700b1 + #define __Pyx_TimeZone_UTC NULL + #define __Pyx_TimeZone_FromOffsetAndName(offset, name) ((void)(offset), (void)(name), (PyObject*)NULL) + #else + #define __Pyx_TimeZone_UTC PyDateTime_TimeZone_UTC + #define __Pyx_TimeZone_FromOffsetAndName(offset, name) PyTimeZone_FromOffsetAndName(offset, name) + #endif + + /* Backport for Python < 3.10 */ + #if PY_VERSION_HEX < 0x030a00a1 + #ifndef PyDateTime_TIME_GET_TZINFO + #define PyDateTime_TIME_GET_TZINFO(o) \ + ((((PyDateTime_Time*)o)->hastzinfo) ? ((PyDateTime_Time*)o)->tzinfo : Py_None) + #endif + #ifndef PyDateTime_DATE_GET_TZINFO + #define PyDateTime_DATE_GET_TZINFO(o) \ + ((((PyDateTime_DateTime*)o)->hastzinfo) ? ((PyDateTime_DateTime*)o)->tzinfo : Py_None) + #endif + #endif + """ ctypedef extern class datetime.date[object PyDateTime_Date]: - pass + @property + cdef inline int year(self): + return PyDateTime_GET_YEAR(self) + + @property + cdef inline int month(self): + return PyDateTime_GET_MONTH(self) + + @property + cdef inline int day(self): + return PyDateTime_GET_DAY(self) ctypedef extern class datetime.time[object PyDateTime_Time]: - pass + @property + cdef inline int hour(self): + return PyDateTime_TIME_GET_HOUR(self) + + @property + cdef inline int minute(self): + return PyDateTime_TIME_GET_MINUTE(self) + + @property + cdef inline int second(self): + return PyDateTime_TIME_GET_SECOND(self) + + @property + cdef inline int microsecond(self): + return PyDateTime_TIME_GET_MICROSECOND(self) + + @property + cdef inline object tzinfo(self): + return <object>PyDateTime_TIME_GET_TZINFO(self) + + @property + cdef inline int fold(self): + # For Python < 3.6 this returns 0 no matter what + return PyDateTime_TIME_GET_FOLD(self) ctypedef extern class datetime.datetime[object PyDateTime_DateTime]: - pass + @property + cdef inline int year(self): + return PyDateTime_GET_YEAR(self) + + @property + cdef inline int month(self): + return PyDateTime_GET_MONTH(self) + + @property + cdef inline int day(self): + return PyDateTime_GET_DAY(self) + + @property + cdef inline int hour(self): + return PyDateTime_DATE_GET_HOUR(self) + + @property + cdef inline int minute(self): + return PyDateTime_DATE_GET_MINUTE(self) + + @property + cdef inline int second(self): + return PyDateTime_DATE_GET_SECOND(self) + + @property + cdef inline int microsecond(self): + return PyDateTime_DATE_GET_MICROSECOND(self) + + @property + cdef inline object tzinfo(self): + return <object>PyDateTime_DATE_GET_TZINFO(self) + + @property + cdef inline int fold(self): + # For Python < 3.6 this returns 0 no matter what + return PyDateTime_DATE_GET_FOLD(self) ctypedef extern class datetime.timedelta[object PyDateTime_Delta]: - pass + @property + cdef inline int day(self): + return PyDateTime_DELTA_GET_DAYS(self) + + @property + cdef inline int second(self): + return PyDateTime_DELTA_GET_SECONDS(self) + + @property + cdef inline int microsecond(self): + return PyDateTime_DELTA_GET_MICROSECONDS(self) ctypedef extern class datetime.tzinfo[object PyDateTime_TZInfo]: pass @@ -25,10 +164,12 @@ cdef extern from "datetime.h": pass ctypedef struct PyDateTime_Time: + unsigned char fold char hastzinfo PyObject *tzinfo ctypedef struct PyDateTime_DateTime: + unsigned char fold char hastzinfo PyObject *tzinfo @@ -47,14 +188,27 @@ cdef extern from "datetime.h": PyTypeObject *TZInfoType # constructors - object (*Date_FromDate)(int, int, int, PyTypeObject*) - object (*DateTime_FromDateAndTime)(int, int, int, int, int, int, int, object, PyTypeObject*) - object (*Time_FromTime)(int, int, int, int, object, PyTypeObject*) - object (*Delta_FromDelta)(int, int, int, int, PyTypeObject*) + date (*Date_FromDate)(int, int, int, PyTypeObject*) + datetime (*DateTime_FromDateAndTime)(int, int, int, int, int, int, int, object, PyTypeObject*) + time (*Time_FromTime)(int, int, int, int, object, PyTypeObject*) + timedelta (*Delta_FromDelta)(int, int, int, int, PyTypeObject*) # constructors for the DB API - object (*DateTime_FromTimestamp)(object, object, object) - object (*Date_FromTimestamp)(object, object) + datetime (*DateTime_FromTimestamp)(PyObject*, object, PyObject*) + date (*Date_FromTimestamp)(PyObject*, object) + + # We cannot use the following because they do not compile in older Python versions. + # Instead, we use datetime.h's macros here that we can backport in C. + + # Python 3.7+ constructors + object (*TimeZone_FromTimeZone)(object offset, PyObject *name) + + # Python 3.7+ singletons + PyObject *TimeZone_UTC + + # Python 3.6+ PEP 495 constructors + datetime (*DateTime_FromDateAndTimeAndFold)(int, int, int, int, int, int, int, object, int, PyTypeObject*) + time (*Time_FromTimeAndFold)(int, int, int ,int, object, int, PyTypeObject*) # Check type of the object. bint PyDate_Check(object op) @@ -82,21 +236,45 @@ cdef extern from "datetime.h": int PyDateTime_DATE_GET_MINUTE(object o) int PyDateTime_DATE_GET_SECOND(object o) int PyDateTime_DATE_GET_MICROSECOND(object o) + int PyDateTime_DATE_GET_FOLD(object o) + PyObject* PyDateTime_DATE_GET_TZINFO(object o) # returns a borrowed reference # Getters for time (C macros). int PyDateTime_TIME_GET_HOUR(object o) int PyDateTime_TIME_GET_MINUTE(object o) int PyDateTime_TIME_GET_SECOND(object o) int PyDateTime_TIME_GET_MICROSECOND(object o) + int PyDateTime_TIME_GET_FOLD(object o) + PyObject* PyDateTime_TIME_GET_TZINFO(object o) # returns a borrowed reference # Getters for timedelta (C macros). int PyDateTime_DELTA_GET_DAYS(object o) int PyDateTime_DELTA_GET_SECONDS(object o) int PyDateTime_DELTA_GET_MICROSECONDS(object o) + # Constructors + object PyTimeZone_FromOffset(object offset) + object PyTimeZone_FromOffsetAndName(object offset, object name) + + # The above macros is Python 3.7+ so we use these instead + object __Pyx_TimeZone_FromOffsetAndName(object offset, PyObject* name) + + # Constructors for the DB API + datetime PyDateTime_FromTimeStamp(object args) + date PyDate_FromTimeStamp(object args) + + # PEP 495 constructors but patched above to allow passing tz + datetime __Pyx_DateTime_DateTimeWithFold(int, int, int, int, int, int, int, object, int) + datetime __Pyx_DateTime_TimeWithFold(int, int, int ,int, object, int) + # PyDateTime CAPI object. PyDateTime_CAPI *PyDateTimeAPI + PyObject* PyDateTime_TimeZone_UTC + + # PyDateTime_TimeZone_UTC is Python 3.7+ so instead we use the following macro + PyObject* __Pyx_TimeZone_UTC + void PyDateTime_IMPORT() # Datetime C API initialization function. @@ -106,42 +284,57 @@ cdef inline void import_datetime(): # Create date object using DateTime CAPI factory function. # Note, there are no range checks for any of the arguments. -cdef inline object date_new(int year, int month, int day): +cdef inline date date_new(int year, int month, int day): return PyDateTimeAPI.Date_FromDate(year, month, day, PyDateTimeAPI.DateType) # Create time object using DateTime CAPI factory function # Note, there are no range checks for any of the arguments. -cdef inline object time_new(int hour, int minute, int second, int microsecond, object tz): - return PyDateTimeAPI.Time_FromTime(hour, minute, second, microsecond, tz, PyDateTimeAPI.TimeType) +cdef inline time time_new(int hour, int minute, int second, int microsecond, object tz, int fold=0): + return __Pyx_DateTime_TimeWithFold(hour, minute, second, microsecond, tz, fold) # Create datetime object using DateTime CAPI factory function. # Note, there are no range checks for any of the arguments. -cdef inline object datetime_new(int year, int month, int day, int hour, int minute, int second, int microsecond, object tz): - return PyDateTimeAPI.DateTime_FromDateAndTime(year, month, day, hour, minute, second, microsecond, tz, PyDateTimeAPI.DateTimeType) +cdef inline datetime datetime_new(int year, int month, int day, int hour, int minute, int second, int microsecond, object tz, int fold=0): + return __Pyx_DateTime_DateTimeWithFold(year, month, day, hour, minute, second, microsecond, tz, fold) # Create timedelta object using DateTime CAPI factory function. # Note, there are no range checks for any of the arguments. -cdef inline object timedelta_new(int days, int seconds, int useconds): +cdef inline timedelta timedelta_new(int days, int seconds, int useconds): return PyDateTimeAPI.Delta_FromDelta(days, seconds, useconds, 1, PyDateTimeAPI.DeltaType) +# Create timedelta object using DateTime CAPI factory function. +cdef inline object timezone_new(object offset, object name=None): + if PY_VERSION_HEX < 0x030700b1: + raise RuntimeError('Time zones are not available from the C-API.') + return __Pyx_TimeZone_FromOffsetAndName(offset, <PyObject*>name if name is not None else NULL) + +# Create datetime object using DB API constructor. +cdef inline datetime datetime_from_timestamp(timestamp, tz=None): + return PyDateTimeAPI.DateTime_FromTimestamp( + <PyObject*>PyDateTimeAPI.DateTimeType, (timestamp, tz) if tz is not None else (timestamp,), NULL) + +# Create date object using DB API constructor. +cdef inline date date_from_timestamp(timestamp): + return PyDateTimeAPI.Date_FromTimestamp(<PyObject*>PyDateTimeAPI.DateType, (timestamp,)) + # More recognizable getters for date/time/datetime/timedelta. # There are no setters because datetime.h hasn't them. # This is because of immutable nature of these objects by design. # If you would change time/date/datetime/timedelta object you need to recreate. +# Get UTC singleton +cdef inline object get_utc(): + if PY_VERSION_HEX < 0x030700b1: + raise RuntimeError('Time zones are not available from the C-API.') + return <object>__Pyx_TimeZone_UTC + # Get tzinfo of time cdef inline object time_tzinfo(object o): - if (<PyDateTime_Time*>o).hastzinfo: - return <object>(<PyDateTime_Time*>o).tzinfo - else: - return None + return <object>PyDateTime_TIME_GET_TZINFO(o) # Get tzinfo of datetime cdef inline object datetime_tzinfo(object o): - if (<PyDateTime_DateTime*>o).hastzinfo: - return <object>(<PyDateTime_DateTime*>o).tzinfo - else: - return None + return <object>PyDateTime_DATE_GET_TZINFO(o) # Get year of date cdef inline int date_year(object o): @@ -183,6 +376,11 @@ cdef inline int time_second(object o): cdef inline int time_microsecond(object o): return PyDateTime_TIME_GET_MICROSECOND(o) +# Get fold of time +cdef inline int time_fold(object o): + # For Python < 3.6 this returns 0 no matter what + return PyDateTime_TIME_GET_FOLD(o) + # Get hour of datetime cdef inline int datetime_hour(object o): return PyDateTime_DATE_GET_HOUR(o) @@ -199,6 +397,11 @@ cdef inline int datetime_second(object o): cdef inline int datetime_microsecond(object o): return PyDateTime_DATE_GET_MICROSECOND(o) +# Get fold of datetime +cdef inline int datetime_fold(object o): + # For Python < 3.6 this returns 0 no matter what + return PyDateTime_DATE_GET_FOLD(o) + # Get days of timedelta cdef inline int timedelta_days(object o): return (<PyDateTime_Delta*>o).days @@ -210,3 +413,14 @@ cdef inline int timedelta_seconds(object o): # Get microseconds of timedelta cdef inline int timedelta_microseconds(object o): return (<PyDateTime_Delta*>o).microseconds + +cdef inline double total_seconds(timedelta obj): + # Mirrors the "timedelta.total_seconds()" method. + # Note that this implementation is not guaranteed to give *exactly* the same + # result as the original method, due to potential differences in floating point rounding. + cdef: + double days, seconds, micros + days = <double>PyDateTime_DELTA_GET_DAYS(obj) + seconds = <double>PyDateTime_DELTA_GET_SECONDS(obj) + micros = <double>PyDateTime_DELTA_GET_MICROSECONDS(obj) + return days * 24 * 3600 + seconds + micros / 1_000_000 diff --git a/Cython/Includes/cpython/descr.pxd b/Cython/Includes/cpython/descr.pxd new file mode 100644 index 000000000..5075f0bbd --- /dev/null +++ b/Cython/Includes/cpython/descr.pxd @@ -0,0 +1,26 @@ +from .object cimport PyObject, PyTypeObject + +cdef extern from "Python.h": + ctypedef object (*wrapperfunc)(self, args, void* wrapped) + ctypedef object (*wrapperfunc_kwds)(self, args, void* wrapped, kwds) + + struct wrapperbase: + char* name + int offset + void* function + wrapperfunc wrapper + char* doc + int flags + PyObject* name_strobj + + int PyWrapperFlag_KEYWORDS + + ctypedef class __builtin__.wrapper_descriptor [object PyWrapperDescrObject]: + cdef type d_type + cdef d_name + cdef wrapperbase* d_base + cdef void* d_wrapped + + object PyDescr_NewWrapper(PyTypeObject* cls, wrapperbase* wrapper, void* wrapped) + + int PyDescr_IsData(descr) diff --git a/Cython/Includes/cpython/dict.pxd b/Cython/Includes/cpython/dict.pxd index 16dd5e145..979dd392a 100644 --- a/Cython/Includes/cpython/dict.pxd +++ b/Cython/Includes/cpython/dict.pxd @@ -1,6 +1,13 @@ from .object cimport PyObject +from .pyport cimport uint64_t cdef extern from "Python.h": + # On Python 2, PyDict_GetItemWithError is called _PyDict_GetItemWithError + """ + #if PY_MAJOR_VERSION <= 2 + #define PyDict_GetItemWithError _PyDict_GetItemWithError + #endif + """ ############################################################################ # 7.4.1 Dictionary Objects @@ -72,11 +79,25 @@ cdef extern from "Python.h": # NULL if the key key is not present, but without setting an # exception. + PyObject* PyDict_GetItemWithError(object p, object key) except? NULL + # Return value: Borrowed reference. + # Variant of PyDict_GetItem() that does not suppress exceptions. Return + # NULL with an exception set if an exception occurred. Return NULL + # without an exception set if the key wasn’t present. + PyObject* PyDict_GetItemString(object p, const char *key) # Return value: Borrowed reference. # This is the same as PyDict_GetItem(), but key is specified as a # char*, rather than a PyObject*. + PyObject* PyDict_SetDefault(object p, object key, object default) except NULL + # Return value: Borrowed reference. + # This is the same as the Python-level dict.setdefault(). If present, it + # returns the value corresponding to key from the dictionary p. If the key + # is not in the dict, it is inserted with value defaultobj and defaultobj + # is returned. This function evaluates the hash function of key only once, + # instead of evaluating it independently for the lookup and the insertion. + list PyDict_Items(object p) # Return value: New reference. # Return a PyListObject containing all the items from the diff --git a/Cython/Includes/cpython/fileobject.pxd b/Cython/Includes/cpython/fileobject.pxd new file mode 100644 index 000000000..e52cd33f5 --- /dev/null +++ b/Cython/Includes/cpython/fileobject.pxd @@ -0,0 +1,57 @@ +""" +From https://docs.python.org/3.9/c-api/file.html + +These APIs are a minimal emulation of the Python 2 C API for built-in file objects, +which used to rely on the buffered I/O (FILE*) support from the C standard library. +In Python 3, files and streams use the new io module, which defines several layers +over the low-level unbuffered I/O of the operating system. The functions described +below are convenience C wrappers over these new APIs, and meant mostly for internal +error reporting in the interpreter; + +third-party code is advised to access the io APIs instead. +""" + +cdef extern from "Python.h": + + ########################################################################### + # File Objects + ########################################################################### + + object PyFile_FromFd(int fd, const char *name, const char *mode, int buffering, + const char *encoding, const char *errors, const char *newline, int closefd) + # Return value: New reference. + # Create a Python file object from the file descriptor of an already + # opened file fd. The arguments name, encoding, errors and newline can be + # NULL to use the defaults; buffering can be -1 to use the default. name + # is ignored and kept for backward compatibility. Return NULL on failure. + # For a more comprehensive description of the arguments, please refer to + # the io.open() function documentation. + + # Warning: Since Python streams have their own buffering layer, mixing + # them with OS-level file descriptors can produce various issues (such as + # unexpected ordering of data). + + # Changed in version 3.2: Ignore name attribute. + + object PyFile_GetLine(object p, int n) + # Return value: New reference. + # Equivalent to p.readline([n]), this function reads one line from the + # object p. p may be a file object or any object with a readline() + # method. If n is 0, exactly one line is read, regardless of the length of + # the line. If n is greater than 0, no more than n bytes will be read from + # the file; a partial line can be returned. In both cases, an empty string + # is returned if the end of the file is reached immediately. If n is less + # than 0, however, one line is read regardless of length, but EOFError is + # raised if the end of the file is reached immediately. + + int PyFile_WriteObject(object obj, object p, int flags) except? -1 + # Write object obj to file object p. The only supported flag for flags + # is Py_PRINT_RAW; if given, the str() of the object is written instead of + # the repr(). Return 0 on success or -1 on failure; the appropriate + # exception will be set. + + int PyFile_WriteString(const char *s, object p) except? -1 + # Write string s to file object p. Return 0 on success or -1 on failure; + # the appropriate exception will be set. + + enum: Py_PRINT_RAW diff --git a/Cython/Includes/cpython/float.pxd b/Cython/Includes/cpython/float.pxd index 65328f31e..7c567a80f 100644 --- a/Cython/Includes/cpython/float.pxd +++ b/Cython/Includes/cpython/float.pxd @@ -1,4 +1,11 @@ cdef extern from "Python.h": + """ + #if PY_MAJOR_VERSION >= 3 + #define __Pyx_PyFloat_FromString(obj) PyFloat_FromString(obj) + #else + #define __Pyx_PyFloat_FromString(obj) PyFloat_FromString(obj, NULL) + #endif + """ ############################################################################ # 7.2.3 @@ -21,7 +28,7 @@ cdef extern from "Python.h": # Return true if its argument is a PyFloatObject, but not a # subtype of PyFloatObject. - object PyFloat_FromString(object str, char **pend) + object PyFloat_FromString "__Pyx_PyFloat_FromString" (object str) # Return value: New reference. # Create a PyFloatObject object based on the string value in str, # or NULL on failure. The pend argument is ignored. It remains diff --git a/Cython/Includes/cpython/list.pxd b/Cython/Includes/cpython/list.pxd index c6a29535c..8ee9c891c 100644 --- a/Cython/Includes/cpython/list.pxd +++ b/Cython/Includes/cpython/list.pxd @@ -42,17 +42,19 @@ cdef extern from "Python.h": int PyList_SetItem(object list, Py_ssize_t index, object item) except -1 # Set the item at index index in list to item. Return 0 on success - # or -1 on failure. Note: This function ``steals'' a reference to - # item and discards a reference to an item already in the list at - # the affected position. + # or -1 on failure. + # + # WARNING: This function _steals_ a reference to item and discards a + # reference to an item already in the list at the affected position. void PyList_SET_ITEM(object list, Py_ssize_t i, object o) # Macro form of PyList_SetItem() without error checking. This is # normally only used to fill in new lists where there is no - # previous content. Note: This function ``steals'' a reference to - # item, and, unlike PyList_SetItem(), does not discard a reference - # to any item that it being replaced; any reference in list at - # position i will be *leaked*. + # previous content. + # + # WARNING: This function _steals_ a reference to item, and, unlike + # PyList_SetItem(), does not discard a reference to any item that + # it being replaced; any reference in list at position i will be *leaked*. int PyList_Insert(object list, Py_ssize_t index, object item) except -1 # Insert the item item into list list in front of index diff --git a/Cython/Includes/cpython/long.pxd b/Cython/Includes/cpython/long.pxd index eb8140d41..f65cd0073 100644 --- a/Cython/Includes/cpython/long.pxd +++ b/Cython/Includes/cpython/long.pxd @@ -89,7 +89,7 @@ cdef extern from "Python.h": # Return a C long representation of the contents of pylong. If # pylong is greater than LONG_MAX, an OverflowError is raised. - # long PyLong_AsLongAndOverflow(object pylong, int *overflow) except? -1 + long PyLong_AsLongAndOverflow(object pylong, int *overflow) except? -1 # Return a C long representation of the contents of pylong. If pylong is # greater than LONG_MAX or less than LONG_MIN, set *overflow to 1 or -1, # respectively, and return -1; otherwise, set *overflow to 0. If any other @@ -97,7 +97,7 @@ cdef extern from "Python.h": # be returned and *overflow will be 0. # New in version 2.7. - # PY_LONG_LONG PyLong_AsLongLongAndOverflow(object pylong, int *overflow) except? -1 + PY_LONG_LONG PyLong_AsLongLongAndOverflow(object pylong, int *overflow) except? -1 # Return a C long long representation of the contents of pylong. If pylong # is greater than PY_LLONG_MAX or less than PY_LLONG_MIN, set *overflow to # 1 or -1, respectively, and return -1; otherwise, set *overflow to 0. If diff --git a/Cython/Includes/cpython/marshal.pxd b/Cython/Includes/cpython/marshal.pxd new file mode 100644 index 000000000..6b6d6f480 --- /dev/null +++ b/Cython/Includes/cpython/marshal.pxd @@ -0,0 +1,66 @@ +from libc.stdio cimport FILE + +cdef extern from "Python.h": + + ########################################################################### + # Data marshalling support + ########################################################################### + + const int Py_MARSHAL_VERSION + + void PyMarshal_WriteLongToFile(long value, FILE *file, int version) + # Marshal a long integer, value, to file. This will only write the + # least-significant 32 bits of value, regardless of the size of the native + # long type. version indicates the file format. + + void PyMarshal_WriteObjectToFile(object value, FILE *file, int version) + # Marshal a Python object, value, to file. version indicates the file + # format. + + bytes PyMarshal_WriteObjectToString(object value, int version) + # Return value: New reference. + # Return a bytes object containing the marshalled representation of value. + # version indicates the file format. + + long PyMarshal_ReadLongFromFile(FILE *file) except? -1 + # Return a C long from the data stream in a FILE* opened for reading. Only + # a 32-bit value can be read in using this function, regardless of the + # native size of long. + + # On error, sets the appropriate exception (EOFError) and returns -1. + + int PyMarshal_ReadShortFromFile(FILE *file) except? -1 + # Return a C short from the data stream in a FILE* opened for reading. Only + # a 16-bit value can be read in using this function, regardless of the + # native size of short. + + # On error, sets the appropriate exception (EOFError) and returns -1. + + object PyMarshal_ReadObjectFromFile(FILE *file) + # Return value: New reference. + # Return a Python object from the data stream in a FILE* opened for + # reading. + + # On error, sets the appropriate exception (EOFError, ValueError or + # TypeError) and returns NULL. + + object PyMarshal_ReadLastObjectFromFile(FILE *file) + # Return value: New reference. + # Return a Python object from the data stream in a FILE* opened for + # reading. Unlike PyMarshal_ReadObjectFromFile(), this function assumes + # that no further objects will be read from the file, allowing it to + # aggressively load file data into memory so that the de-serialization can + # operate from data in memory, rather than reading a byte at a time from the + # file. Only use these variant if you are certain that you won’t be reading + # anything else from the file. + + # On error, sets the appropriate exception (EOFError, ValueError or + # TypeError) and returns NULL. + + object PyMarshal_ReadObjectFromString(const char *data, Py_ssize_t len) + # Return value: New reference. + # Return a Python object from the data stream in a byte buffer containing + # len bytes pointed to by data. + + # On error, sets the appropriate exception (EOFError, ValueError or + # TypeError) and returns NULL. diff --git a/Cython/Includes/cpython/module.pxd b/Cython/Includes/cpython/module.pxd index 8eb323b01..ea4c3817e 100644 --- a/Cython/Includes/cpython/module.pxd +++ b/Cython/Includes/cpython/module.pxd @@ -145,6 +145,12 @@ cdef extern from "Python.h": bint PyModule_CheckExact(object p) # Return true if p is a module object, but not a subtype of PyModule_Type. + object PyModule_NewObject(object name) + # Return a new module object with the __name__ attribute set to name. + # The module’s __name__, __doc__, __package__, and __loader__ + # attributes are filled in (all but __name__ are set to None); the caller + # is responsible for providing a __file__ attribute. + object PyModule_New(const char *name) # Return value: New reference. # Return a new module object with the __name__ attribute set to @@ -160,21 +166,35 @@ cdef extern from "Python.h": # use other PyModule_*() and PyObject_*() functions rather than # directly manipulate a module's __dict__. + object PyModule_GetNameObject(object module) + # Return module’s __name__ value. If the module does not provide one, or if + # it is not a string, SystemError is raised and NULL is returned. + char* PyModule_GetName(object module) except NULL - # Return module's __name__ value. If the module does not provide - # one, or if it is not a string, SystemError is raised and NULL is - # returned. + # Similar to PyModule_GetNameObject() but return the name encoded + # to 'utf-8'. + + void* PyModule_GetState(object module) + # Return the “state” of the module, that is, a pointer to the block of + # memory allocated at module creation time, or NULL. + # See PyModuleDef.m_size. + + object PyModule_GetFilenameObject(object module) + # Return the name of the file from which module was loaded using module’s + # __file__ attribute. If this is not defined, or if it is not a unicode + # string, raise SystemError and return NULL; otherwise return a reference + # to a Unicode object. char* PyModule_GetFilename(object module) except NULL - # Return the name of the file from which module was loaded using - # module's __file__ attribute. If this is not defined, or if it is - # not a string, raise SystemError and return NULL. + # Similar to PyModule_GetFilenameObject() but return the filename encoded + # to ‘utf-8’. int PyModule_AddObject(object module, const char *name, object value) except -1 # Add an object to module as name. This is a convenience function - # which can be used from the module's initialization - # function. This steals a reference to value. Return -1 on error, - # 0 on success. + # which can be used from the module's initialization function. + # Return -1 on error, 0 on success. + # + # WARNING: This _steals_ a reference to value. int PyModule_AddIntConstant(object module, const char *name, long value) except -1 # Add an integer constant to module as name. This convenience diff --git a/Cython/Includes/cpython/object.pxd b/Cython/Includes/cpython/object.pxd index 5a8116639..aa625523d 100644 --- a/Cython/Includes/cpython/object.pxd +++ b/Cython/Includes/cpython/object.pxd @@ -45,6 +45,8 @@ cdef extern from "Python.h": newfunc tp_new destructor tp_dealloc + destructor tp_del + destructor tp_finalize traverseproc tp_traverse inquiry tp_clear freefunc tp_free @@ -63,6 +65,8 @@ cdef extern from "Python.h": descrgetfunc tp_descr_get descrsetfunc tp_descr_set + unsigned int tp_version_tag + ctypedef struct PyObject: Py_ssize_t ob_refcnt PyTypeObject *ob_type @@ -128,6 +132,17 @@ cdef extern from "Python.h": # failure. This is the equivalent of the Python statement "del # o.attr_name". + object PyObject_GenericGetDict(object o, void *context) + # Return value: New reference. + # A generic implementation for the getter of a __dict__ descriptor. It + # creates the dictionary if necessary. + # New in version 3.3. + + int PyObject_GenericSetDict(object o, object value, void *context) except -1 + # A generic implementation for the setter of a __dict__ descriptor. This + # implementation does not allow the dictionary to be deleted. + # New in version 3.3. + int Py_LT, Py_LE, Py_EQ, Py_NE, Py_GT, Py_GE object PyObject_RichCompare(object o1, object o2, int opid) @@ -177,6 +192,14 @@ cdef extern from "Python.h": # equivalent of the Python expression "str(o)". Called by the # str() built-in function and by the print statement. + object PyObject_Bytes(object o) + # Return value: New reference. + # Compute a bytes representation of object o. Return NULL on + # failure and a bytes object on success. This is equivalent to + # the Python expression bytes(o), when o is not an integer. + # Unlike bytes(o), a TypeError is raised when o is an integer + # instead of a zero-initialized bytes object. + object PyObject_Unicode(object o) # Return value: New reference. # Compute a Unicode string representation of object o. Returns the @@ -320,6 +343,13 @@ cdef extern from "Python.h": # returned. On error, -1 is returned. This is the equivalent to # the Python expression "len(o)". + Py_ssize_t PyObject_LengthHint(object o, Py_ssize_t default) except -1 + # Return an estimated length for the object o. First try to return its + # actual length, then an estimate using __length_hint__(), and finally + # return the default value. On error, return -1. This is the equivalent to + # the Python expression "operator.length_hint(o, default)". + # New in version 3.4. + object PyObject_GetItem(object o, object key) # Return value: New reference. # Return element of o corresponding to the object key or NULL on @@ -397,3 +427,4 @@ cdef extern from "Python.h": long Py_TPFLAGS_DEFAULT_EXTERNAL long Py_TPFLAGS_DEFAULT_CORE long Py_TPFLAGS_DEFAULT + long Py_TPFLAGS_HAVE_FINALIZE diff --git a/Cython/Includes/cpython/pyport.pxd b/Cython/Includes/cpython/pyport.pxd new file mode 100644 index 000000000..fec59c9c8 --- /dev/null +++ b/Cython/Includes/cpython/pyport.pxd @@ -0,0 +1,8 @@ +cdef extern from "Python.h": + ctypedef int int32_t + ctypedef int int64_t + ctypedef unsigned int uint32_t + ctypedef unsigned int uint64_t + + const Py_ssize_t PY_SSIZE_T_MIN + const Py_ssize_t PY_SSIZE_T_MAX diff --git a/Cython/Includes/cpython/pystate.pxd b/Cython/Includes/cpython/pystate.pxd index 1af630793..ee8856b20 100644 --- a/Cython/Includes/cpython/pystate.pxd +++ b/Cython/Includes/cpython/pystate.pxd @@ -84,6 +84,9 @@ cdef extern from "Python.h": # PyGILState_Release on the same thread. void PyGILState_Release(PyGILState_STATE) + # Return 1 if the current thread holds the GIL and 0 otherwise. + int PyGILState_Check() + # Routines for advanced debuggers, requested by David Beazley. # Don't use unless you know what you are doing! PyInterpreterState * PyInterpreterState_Head() diff --git a/Cython/Includes/cpython/time.pxd b/Cython/Includes/cpython/time.pxd new file mode 100644 index 000000000..076abd931 --- /dev/null +++ b/Cython/Includes/cpython/time.pxd @@ -0,0 +1,51 @@ +""" +Cython implementation of (parts of) the standard library time module. +""" + +from libc.stdint cimport int64_t +from cpython.exc cimport PyErr_SetFromErrno + +cdef extern from "Python.h": + ctypedef int64_t _PyTime_t + _PyTime_t _PyTime_GetSystemClock() nogil + double _PyTime_AsSecondsDouble(_PyTime_t t) nogil + +from libc.time cimport ( + tm, + time_t, + localtime as libc_localtime, +) + + +cdef inline double time() nogil: + cdef: + _PyTime_t tic + + tic = _PyTime_GetSystemClock() + return _PyTime_AsSecondsDouble(tic) + + +cdef inline int _raise_from_errno() except -1 with gil: + PyErr_SetFromErrno(RuntimeError) + return <int> -1 # Let the C compiler know that this function always raises. + + +cdef inline tm localtime() nogil except *: + """ + Analogue to the stdlib time.localtime. The returned struct + has some entries that the stdlib version does not: tm_gmtoff, tm_zone + """ + cdef: + time_t tic = <time_t>time() + tm* result + + result = libc_localtime(&tic) + if result is NULL: + _raise_from_errno() + # Fix 0-based date values (and the 1900-based year). + # See tmtotuple() in https://github.com/python/cpython/blob/master/Modules/timemodule.c + result.tm_year += 1900 + result.tm_mon += 1 + result.tm_wday = (result.tm_wday + 6) % 7 + result.tm_yday += 1 + return result[0] diff --git a/Cython/Includes/cpython/tuple.pxd b/Cython/Includes/cpython/tuple.pxd index 09c46e0b4..b0d718047 100644 --- a/Cython/Includes/cpython/tuple.pxd +++ b/Cython/Includes/cpython/tuple.pxd @@ -47,13 +47,15 @@ cdef extern from "Python.h": int PyTuple_SetItem(object p, Py_ssize_t pos, object o) except -1 # Insert a reference to object o at position pos of the tuple - # pointed to by p. Return 0 on success. Note: This function - # ``steals'' a reference to o. + # pointed to by p. Return 0 on success. + # + # WARNING: This function _steals_ a reference to o. void PyTuple_SET_ITEM(object p, Py_ssize_t pos, object o) # Like PyTuple_SetItem(), but does no error checking, and should - # only be used to fill in brand new tuples. Note: This function - # ``steals'' a reference to o. + # only be used to fill in brand new tuples. + # + # WARNING: This function _steals_ a reference to o. int _PyTuple_Resize(PyObject **p, Py_ssize_t newsize) except -1 # Can be used to resize a tuple. newsize will be the new length of diff --git a/Cython/Includes/cpython/type.pxd b/Cython/Includes/cpython/type.pxd index a1d094e37..928a748cd 100644 --- a/Cython/Includes/cpython/type.pxd +++ b/Cython/Includes/cpython/type.pxd @@ -23,6 +23,11 @@ cdef extern from "Python.h": # of the standard type object. Return false in all other # cases. + void PyType_Modified(type type) + # Invalidate the internal lookup cache for the type and all of its + # subtypes. This function must be called after any manual modification + # of the attributes or base classes of the type. + bint PyType_HasFeature(object o, int feature) # Return true if the type object o sets the feature feature. Type # features are denoted by single bit flags. diff --git a/Cython/Includes/cpython/unicode.pxd b/Cython/Includes/cpython/unicode.pxd index 061be0905..ba11f5736 100644 --- a/Cython/Includes/cpython/unicode.pxd +++ b/Cython/Includes/cpython/unicode.pxd @@ -103,6 +103,14 @@ cdef extern from *: # when u is NULL. unicode PyUnicode_FromUnicode(Py_UNICODE *u, Py_ssize_t size) + # Similar to PyUnicode_FromUnicode(), but u points to UTF-8 encoded + # bytes + unicode PyUnicode_FromStringAndSize(const char *u, Py_ssize_t size) + + # Similar to PyUnicode_FromUnicode(), but u points to null-terminated + # UTF-8 encoded bytes. The size is determined with strlen(). + unicode PyUnicode_FromString(const char *u) + # Create a Unicode Object from the given Unicode code point ordinal. # # The ordinal must be in range(0x10000) on narrow Python builds @@ -226,7 +234,7 @@ cdef extern from *: # equal, and greater than, respectively. It is best to pass only ASCII-encoded # strings, but the function interprets the input string as ISO-8859-1 if it # contains non-ASCII characters. - int PyUnicode_CompareWithASCIIString(object uni, char *string) except? -1 + int PyUnicode_CompareWithASCIIString(object uni, const char *string) # Rich compare two unicode strings and return one of the following: # diff --git a/Cython/Includes/libc/complex.pxd b/Cython/Includes/libc/complex.pxd new file mode 100644 index 000000000..7cd740cb8 --- /dev/null +++ b/Cython/Includes/libc/complex.pxd @@ -0,0 +1,35 @@ +cdef extern from "<complex.h>" nogil: + # Trigonometric functions. + double complex cacos(double complex z) + double complex casin(double complex z) + double complex catan(double complex z) + double complex ccos(double complex z) + double complex csin(double complex z) + double complex ctan(double complex z) + + # Hyperbolic functions. + double complex cacosh(double complex z) + double complex casinh(double complex z) + double complex catanh(double complex z) + double complex ccosh(double complex z) + double complex csinh(double complex z) + double complex ctanh(double complex z) + + # Exponential and logarithmic functions. + double complex cexp(double complex z) + double complex clog(double complex z) + double complex clog10(double complex z) + + # Power functions. + double complex cpow(double complex x, double complex y) + double complex csqrt(double complex z) + + # Absolute value, conjugates, and projection. + double cabs(double complex z) + double carg(double complex z) + double complex conj(double complex z) + double complex cproj(double complex z) + + # Decomposing complex values. + double cimag(double complex z) + double creal(double complex z) diff --git a/Cython/Includes/libc/math.pxd b/Cython/Includes/libc/math.pxd index b002670b2..4a9858c2a 100644 --- a/Cython/Includes/libc/math.pxd +++ b/Cython/Includes/libc/math.pxd @@ -23,81 +23,178 @@ cdef extern from "<math.h>" nogil: const float HUGE_VALF const long double HUGE_VALL + # All C99 functions in alphabetical order double acos(double x) + float acosf(float) + double acosh(double x) + float acoshf(float) + long double acoshl(long double) + long double acosl(long double) double asin(double x) + float asinf(float) + double asinh(double x) + float asinhf(float) + long double asinhl(long double) + long double asinl(long double) double atan(double x) double atan2(double y, double x) - double cos(double x) - double sin(double x) - double tan(double x) - - double cosh(double x) - double sinh(double x) - double tanh(double x) - double acosh(double x) - double asinh(double x) + float atan2f(float, float) + long double atan2l(long double, long double) + float atanf(float) double atanh(double x) - - double hypot(double x, double y) - - double exp(double x) - double exp2(double x) - double expm1(double x) - double log(double x) - double logb(double x) - double log2(double x) - double log10(double x) - double log1p(double x) - int ilogb(double x) - - double lgamma(double x) - double tgamma(double x) - - double frexp(double x, int* exponent) - double ldexp(double x, int exponent) - - double modf(double x, double* iptr) - double fmod(double x, double y) - double remainder(double x, double y) - double remquo(double x, double y, int *quot) - double pow(double x, double y) - double sqrt(double x) + float atanhf(float) + long double atanhl(long double) + long double atanl(long double) double cbrt(double x) - - double fabs(double x) + float cbrtf(float) + long double cbrtl(long double) double ceil(double x) - double floor(double x) - double trunc(double x) - double rint(double x) - double round(double x) - double nearbyint(double x) - double nextafter(double, double) - double nexttoward(double, long double) - - long long llrint(double) - long lrint(double) - long long llround(double) - long lround(double) - + float ceilf(float) + long double ceill(long double) double copysign(double, double) float copysignf(float, float) long double copysignl(long double, long double) - + double cos(double x) + float cosf(float) + double cosh(double x) + float coshf(float) + long double coshl(long double) + long double cosl(long double) double erf(double) - float erff(float) - long double erfl(long double) double erfc(double) float erfcf(float) long double erfcl(long double) - + float erff(float) + long double erfl(long double) + double exp(double x) + double exp2(double x) + float exp2f(float) + long double exp2l(long double) + float expf(float) + long double expl(long double) + double expm1(double x) + float expm1f(float) + long double expm1l(long double) + double fabs(double x) + float fabsf(float) + long double fabsl(long double) double fdim(double x, double y) + float fdimf(float, float) + long double fdiml(long double, long double) + double floor(double x) + float floorf(float) + long double floorl(long double) double fma(double x, double y, double z) + float fmaf(float, float, float) + long double fmal(long double, long double, long double) double fmax(double x, double y) + float fmaxf(float, float) + long double fmaxl(long double, long double) double fmin(double x, double y) + float fminf(float, float) + long double fminl(long double, long double) + double fmod(double x, double y) + float fmodf(float, float) + long double fmodl(long double, long double) + double frexp(double x, int* exponent) + float frexpf(float, int* exponent) + long double frexpl(long double, int*) + double hypot(double x, double y) + float hypotf(float, float) + long double hypotl(long double, long double) + int ilogb(double x) + int ilogbf(float) + int ilogbl(long double) + double ldexp(double x, int exponent) + float ldexpf(float, int exponent) + long double ldexpl(long double, int exponent) + double lgamma(double x) + float lgammaf(float) + long double lgammal(long double) + long long llrint(double) + long long llrintf(float) + long long llrintl(long double) + long long llround(double) + long long llroundf(float) + long long llroundl(long double) + double log(double x) + double log10(double x) + float log10f(float) + long double log10l(long double) + double log1p(double x) + float log1pf(float) + long double log1pl(long double) + double log2(double x) + float log2f(float) + long double log2l(long double) + double logb(double x) + float logbf(float) + long double logbl(long double) + float logf(float) + long double logl(long double) + long lrint(double) + long lrintf(float) + long lrintl(long double) + long lround(double) + long lroundf(float) + long lroundl(long double) + double modf(double x, double* iptr) + float modff(float, float* iptr) + long double modfl(long double, long double* iptr) + double nan(const char*) + float nanf(const char*) + long double nanl(const char*) + double nearbyint(double x) + float nearbyintf(float) + long double nearbyintl(long double) + double nextafter(double, double) + float nextafterf(float, float) + long double nextafterl(long double, long double) + double nexttoward(double, long double) + float nexttowardf(float, long double) + long double nexttowardl(long double, long double) + double pow(double x, double y) + float powf(float, float) + long double powl(long double, long double) + double remainder(double x, double y) + float remainderf(float, float) + long double remainderl(long double, long double) + double remquo(double x, double y, int* quot) + float remquof(float, float, int* quot) + long double remquol(long double, long double, int* quot) + double rint(double x) + float rintf(float) + long double rintl(long double) + double round(double x) + float roundf(float) + long double roundl(long double) double scalbln(double x, long n) + float scalblnf(float, long) + long double scalblnl(long double, long) double scalbn(double x, int n) - - double nan(const char*) + float scalbnf(float, int) + long double scalbnl(long double, int) + double sin(double x) + float sinf(float) + double sinh(double x) + float sinhf(float) + long double sinhl(long double) + long double sinl(long double) + double sqrt(double x) + float sqrtf(float) + long double sqrtl(long double) + double tan(double x) + float tanf(float) + double tanh(double x) + float tanhf(float) + long double tanhl(long double) + long double tanl(long double) + double tgamma(double x) + float tgammaf(float) + long double tgammal(long double) + double trunc(double x) + float truncf(float) + long double truncl(long double) int isinf(long double) # -1 / 0 / 1 bint isfinite(long double) diff --git a/Cython/Includes/libc/time.pxd b/Cython/Includes/libc/time.pxd index 3aa15a2ee..318212eea 100644 --- a/Cython/Includes/libc/time.pxd +++ b/Cython/Includes/libc/time.pxd @@ -1,4 +1,4 @@ -# http://en.wikipedia.org/wiki/C_date_and_time_functions +# https://en.wikipedia.org/wiki/C_date_and_time_functions from libc.stddef cimport wchar_t @@ -20,8 +20,9 @@ cdef extern from "<time.h>" nogil: int tm_wday int tm_yday int tm_isdst - char *tm_zone - long tm_gmtoff + # GNU specific extensions + #char *tm_zone + #long tm_gmtoff int daylight # global state long timezone diff --git a/Cython/Includes/libcpp/algorithm.pxd b/Cython/Includes/libcpp/algorithm.pxd index ec7c3835b..712ea1757 100644 --- a/Cython/Includes/libcpp/algorithm.pxd +++ b/Cython/Includes/libcpp/algorithm.pxd @@ -1,43 +1,254 @@ from libcpp cimport bool +from libcpp.utility cimport pair +from libc.stddef import ptrdiff_t cdef extern from "<algorithm>" namespace "std" nogil: - # Sorting and searching - bool binary_search[Iter, T](Iter first, Iter last, const T& value) - bool binary_search[Iter, T, Compare](Iter first, Iter last, const T& value, - Compare comp) + # Non-modifying sequence operations + bool all_of[Iter, Pred](Iter first, Iter last, Pred pred) except + + bool all_of[ExecutionPolicy, Iter, Pred](ExecutionPolicy&& policy, Iter first, Iter last, Pred pred) except + + bool any_of[Iter, Pred](Iter first, Iter last, Pred pred) except + + bool any_of[ExecutionPolicy, Iter, Pred](ExecutionPolicy&& policy, Iter first, Iter last, Pred pred) except + + bool none_of[Iter, Pred](Iter first, Iter last, Pred pred) except + + bool none_of[ExecutionPolicy, Iter, Pred](ExecutionPolicy&& policy, Iter first, Iter last, Pred pred) except + - Iter lower_bound[Iter, T](Iter first, Iter last, const T& value) - Iter lower_bound[Iter, T, Compare](Iter first, Iter last, const T& value, - Compare comp) + void for_each[Iter, UnaryFunction](Iter first, Iter last, UnaryFunction f) except + # actually returns f + void for_each[ExecutionPolicy, Iter, UnaryFunction](ExecutionPolicy&& policy, Iter first, Iter last, UnaryFunction f) except + # actually returns f - Iter upper_bound[Iter, T](Iter first, Iter last, const T& value) - Iter upper_bound[Iter, T, Compare](Iter first, Iter last, const T& value, - Compare comp) + ptrdiff_t count[Iter, T](Iter first, Iter last, const T& value) except + + ptrdiff_t count[ExecutionPolicy, Iter, T](ExecutionPolicy&& policy, Iter first, Iter last, const T& value) except + + ptrdiff_t count_if[Iter, Pred](Iter first, Iter last, Pred pred) except + + ptrdiff_t count_if[ExecutionPolicy, Iter, Pred](ExecutionPolicy&& policy, Iter first, Iter last, Pred pred) except + - void partial_sort[Iter](Iter first, Iter middle, Iter last) - void partial_sort[Iter, Compare](Iter first, Iter middle, Iter last, - Compare comp) + pair[Iter1, Iter2] mismatch[Iter1, Iter2]( + Iter1 first1, Iter1 last1, Iter2 first2) except + # other overloads are tricky + pair[Iter1, Iter2] mismatch[ExecutionPolicy, Iter1, Iter2]( + ExecutionPolicy&& policy, Iter1 first1, Iter1 last1, Iter2 first2) except + - void sort[Iter](Iter first, Iter last) - void sort[Iter, Compare](Iter first, Iter last, Compare comp) + Iter find[Iter, T](Iter first, Iter last, const T& value) except + + Iter find[ExecutionPolicy, Iter, T](ExecutionPolicy&& policy, Iter first, Iter last, const T& value) except + - # Removing duplicates - Iter unique[Iter](Iter first, Iter last) - Iter unique[Iter, BinaryPredicate](Iter first, Iter last, BinaryPredicate p) + Iter find_if[Iter, Pred](Iter first, Iter last, Pred pred) except + + Iter find_if[ExecutionPolicy, Iter, Pred](ExecutionPolicy&& policy, Iter first, Iter last, Pred pred) except + + Iter find_if_not[Iter, Pred](Iter first, Iter last, Pred pred) except + + Iter find_if_not[ExecutionPolicy, Iter, Pred](ExecutionPolicy&& policy, Iter first, Iter last, Pred pred) except + - # Binary heaps (priority queues) - void make_heap[Iter](Iter first, Iter last) - void make_heap[Iter, Compare](Iter first, Iter last, Compare comp) + Iter1 find_end[Iter1, Iter2](Iter1 first1, Iter1 last1, Iter2 first2, Iter2 last2) except + + Iter1 find_end[ExecutionPolicy, Iter1, Iter2](ExecutionPolicy&& policy, Iter1 first1, Iter1 last1, Iter2 first2, Iter2 last2) except + + Iter1 find_end[Iter1, Iter2, BinaryPred]( + Iter1 first1, Iter1 last1, Iter2 first2, Iter2 last2, BinaryPred pred) except + + Iter1 find_end[ExecutionPolicy, Iter1, Iter2, BinaryPred]( + ExecutionPolicy&& policy, Iter1 first1, Iter1 last1, Iter2 first2, Iter2 last2, BinaryPred pred) except + - void pop_heap[Iter](Iter first, Iter last) - void pop_heap[Iter, Compare](Iter first, Iter last, Compare comp) - void push_heap[Iter](Iter first, Iter last) - void push_heap[Iter, Compare](Iter first, Iter last, Compare comp) + Iter1 find_first_of[Iter1, Iter2](Iter1 first1, Iter1 last1, Iter2 first2, Iter2 last2) except + + Iter1 find_first_of[Iter1, Iter2, BinaryPred]( + Iter1 first1, Iter1 last1, Iter2 first2, Iter2 last2, BinaryPred pred) except + + Iter1 find_first_of[ExecutionPolicy, Iter1, Iter2](ExecutionPolicy&& policy, Iter1 first1, Iter1 last1, Iter2 first2, Iter2 last2) except + + Iter1 find_first_of[ExecutionPolicy, Iter1, Iter2, BinaryPred]( + ExecutionPolicy&& policy, Iter1 first1, Iter1 last1, Iter2 first2, Iter2 last2, BinaryPred pred) except + - void sort_heap[Iter](Iter first, Iter last) - void sort_heap[Iter, Compare](Iter first, Iter last, Compare comp) + Iter adjacent_find[Iter](Iter first, Iter last) except + + Iter adjacent_find[ExecutionPolicy, Iter](ExecutionPolicy&& policy, Iter first, Iter last) except + + Iter adjacent_find[Iter, BinaryPred](Iter first, Iter last, BinaryPred pred) except + + Iter adjacent_find[ExecutionPolicy, Iter, BinaryPred](ExecutionPolicy&& policy, Iter first, Iter last, BinaryPred pred) except + + + Iter1 search[Iter1, Iter2](Iter1 first1, Iter1 last1, Iter2 first2, Iter2 last2) except + + Iter1 search[ExecutionPolicy, Iter1, Iter2](ExecutionPolicy&& policy, Iter1 first1, Iter1 last1, Iter2 first2, Iter2 last2) except + + Iter1 search[Iter1, Iter2, BinaryPred]( + Iter1 first1, Iter1 last1, Iter2 first2, Iter2 last2, BinaryPred pred) except + + Iter1 search[ExecutionPolicy, Iter1, Iter2, BinaryPred]( + ExecutionPolicy&& policy, Iter1 first1, Iter1 last1, Iter2 first2, Iter2 last2, BinaryPred pred) except + + Iter search_n[Iter, Size, T](Iter first1, Iter last1, Size count, const T& value) except + + Iter search_n[ExecutionPolicy, Iter, Size, T](ExecutionPolicy&& policy, Iter first1, Iter last1, Size count, const T& value) except + + Iter search_n[Iter, Size, T, BinaryPred]( + Iter first1, Iter last1, Size count, const T& value, BinaryPred pred) except + + Iter search_n[ExecutionPolicy, Iter, Size, T, BinaryPred]( + ExecutionPolicy&& policy, Iter first1, Iter last1, Size count, const T& value, BinaryPred pred) except + + + # Modifying sequence operations + OutputIt copy[InputIt, OutputIt](InputIt first, InputIt last, OutputIt d_first) except + + OutputIt copy[ExecutionPolicy, InputIt, OutputIt](ExecutionPolicy&& policy, InputIt first, InputIt last, OutputIt d_first) except + + OutputIt copy_if[InputIt, OutputIt, Pred](InputIt first, InputIt last, OutputIt d_first, Pred pred) except + + OutputIt copy_if[ExecutionPolicy, InputIt, OutputIt, Pred](ExecutionPolicy&& policy, InputIt first, InputIt last, OutputIt d_first, Pred pred) except + + OutputIt copy_n[InputIt, Size, OutputIt](InputIt first, Size count, OutputIt result) except + + OutputIt copy_n[ExecutionPolicy, InputIt, Size, OutputIt](ExecutionPolicy&& policy, InputIt first, Size count, OutputIt result) except + + Iter2 copy_backward[Iter1, Iter2](Iter1 first, Iter1 last, Iter2 d_last) except + + Iter2 copy_backward[ExecutionPolicy, Iter1, Iter2](ExecutionPolicy&& policy, Iter1 first, Iter1 last, Iter2 d_last) except + + + OutputIt move[InputIt, OutputIt](InputIt first, InputIt last, OutputIt d_first) except + + OutputIt move[ExecutionPolicy, InputIt, OutputIt](ExecutionPolicy&& policy, InputIt first, InputIt last, OutputIt d_first) except + + Iter2 move_backward[Iter1, Iter2](Iter1 first, Iter1 last, Iter2 d_last) except + + Iter2 move_backward[ExecutionPolicy, Iter1, Iter2](ExecutionPolicy&& policy, Iter1 first, Iter1 last, Iter2 d_last) except + + + void fill[Iter, T](Iter first, Iter last, const T& value) except + + void fill[ExecutionPolicy, Iter, T](ExecutionPolicy&& policy, Iter first, Iter last, const T& value) except + + Iter fill_n[Iter, Size, T](Iter first, Size count, const T& value) except + + Iter fill_n[ExecutionPolicy, Iter, Size, T](ExecutionPolicy&& policy, Iter first, Size count, const T& value) except + + + OutputIt transform[InputIt, OutputIt, UnaryOp]( + InputIt first1, InputIt last1, OutputIt d_first, UnaryOp unary_op) except + + + # This overload is ambiguos with the next one. We just let C++ disambiguate from the arguments + # OutputIt transform[ExecutionPolicy, InputIt, OutputIt, UnaryOp]( + # ExecutionPolicy&& policy, InputIt first1, InputIt last1, OutputIt d_first, UnaryOp unary_op) except + + + OutputIt transform[InputIt1, InputIt2, OutputIt, BinaryOp]( + InputIt1 first1, InputIt1 last1, InputIt2 first2, OutputIt d_first, BinaryOp binary_op) except + + + OutputIt transform[ExecutionPolicy, InputIt1, InputIt2, OutputIt, BinaryOp]( + ExecutionPolicy&& policy, InputIt1 first1, InputIt1 last1, InputIt2 first2, OutputIt d_first, BinaryOp binary_op) except + + + void generate[Iter, Generator](Iter first, Iter last, Generator g) except + + void generate[ExecutionPolicy, Iter, Generator](ExecutionPolicy&& policy, Iter first, Iter last, Generator g) except + + void generate_n[Iter, Size, Generator](Iter first, Size count, Generator g) except + + void generate_n[ExecutionPolicy, Iter, Size, Generator](ExecutionPolicy&& policy, Iter first, Size count, Generator g) except + + + Iter remove[Iter, T](Iter first, Iter last, const T& value) except + + Iter remove[ExecutionPolicy, Iter, T](ExecutionPolicy&& policy, Iter first, Iter last, const T& value) except + + Iter remove_if[Iter, UnaryPred](Iter first, Iter last, UnaryPred pred) except + + Iter remove_if[ExecutionPolicy, Iter, UnaryPred](ExecutionPolicy&& policy, Iter first, Iter last, UnaryPred pred) except + + OutputIt remove_copy[InputIt, OutputIt, T](InputIt first, InputIt last, OutputIt d_first, const T& value) except + + OutputIt remove_copy[ExecutionPolicy, InputIt, OutputIt, T](ExecutionPolicy&& policy, InputIt first, InputIt last, OutputIt d_first, const T& value) except + + OutputIt remove_copy_if[InputIt, OutputIt, UnaryPred]( + InputIt first, InputIt last, OutputIt d_first, UnaryPred pred) except + + OutputIt remove_copy_if[ExecutionPolicy, InputIt, OutputIt, UnaryPred]( + ExecutionPolicy&& policy, InputIt first, InputIt last, OutputIt d_first, UnaryPred pred) except + + + void replace[Iter, T](Iter first, Iter last, const T& old_value, const T& new_value) except + + void replace[ExecutionPolicy, Iter, T](ExecutionPolicy&& policy, Iter first, Iter last, const T& old_value, const T& new_value) except + + void replace_if[Iter, UnaryPred, T](Iter first, Iter last, UnaryPred pred, const T& new_value) except + + OutputIt replace_copy[InputIt, OutputIt, T]( + InputIt first, InputIt last, OutputIt d_first, const T& old_value, const T& new_value) except + + void replace_if[ExecutionPolicy, Iter, UnaryPred, T](ExecutionPolicy&& policy, Iter first, Iter last, UnaryPred pred, const T& new_value) except + + + OutputIt replace_copy[ExecutionPolicy, InputIt, OutputIt, T]( + ExecutionPolicy&& policy, InputIt first, InputIt last, OutputIt d_first, const T& old_value, const T& new_value) except + + OutputIt replace_copy_if[InputIt, OutputIt, UnaryPred, T]( + InputIt first, InputIt last, OutputIt d_first, UnaryPred pred, const T& new_value) except + + OutputIt replace_copy_if[ExecutionPolicy, InputIt, OutputIt, UnaryPred, T]( + ExecutionPolicy&& policy, InputIt first, InputIt last, OutputIt d_first, UnaryPred pred, const T& new_value) except + + + void swap[T](T& a, T& b) except + # array overload also works + Iter2 swap_ranges[Iter1, Iter2](Iter1 first1, Iter1 last1, Iter2 first2) except + + void iter_swap[Iter](Iter a, Iter b) except + + + void reverse[Iter](Iter first, Iter last) except + + void reverse[ExecutionPolicy, Iter](ExecutionPolicy&& policy, Iter first, Iter last) except + + OutputIt reverse_copy[InputIt, OutputIt](InputIt first, InputIt last, OutputIt d_first) except + + OutputIt reverse_copy[ExecutionPolicy, InputIt, OutputIt](ExecutionPolicy&& policy, InputIt first, InputIt last, OutputIt d_first) except + + + Iter rotate[Iter](Iter first, Iter n_first, Iter last) except + + Iter rotate[ExecutionPolicy, Iter](ExecutionPolicy&& policy, Iter first, Iter n_first, Iter last) except + + OutputIt rotate_copy[InputIt, OutputIt](InputIt first, InputIt n_first, InputIt last, OutputIt d_first) except + + OutputIt rotate_copy[ExecutionPolicy, InputIt, OutputIt](ExecutionPolicy&& policy, InputIt first, InputIt n_first, InputIt last, OutputIt d_first) except + + + Iter unique[Iter](Iter first, Iter last) except + + Iter unique[ExecutionPolicy, Iter](ExecutionPolicy&& policy, Iter first, Iter last) except + + Iter unique[Iter, BinaryPred](Iter first, Iter last, BinaryPred p) except + + Iter unique[ExecutionPolicy, Iter, BinaryPred](ExecutionPolicy&& policy, Iter first, Iter last, BinaryPred p) except + + OutputIt unique_copy[InputIt, OutputIt](InputIt first, InputIt last, OutputIt d_first) except + + OutputIt unique_copy[ExecutionPolicy, InputIt, OutputIt](ExecutionPolicy&& policy, InputIt first, InputIt last, OutputIt d_first) except + + OutputIt unique_copy[InputIt, OutputIt, BinaryPred]( + InputIt first, InputIt last, OutputIt d_first, BinaryPred pred) except + + OutputIt unique_copy[ExecutionPolicy, InputIt, OutputIt, BinaryPred]( + ExecutionPolicy&& policy, InputIt first, InputIt last, OutputIt d_first, BinaryPred pred) except + + + # Partitioning operations + bool is_partitioned[Iter, Pred](Iter first, Iter last, Pred p) except + + bool is_partitioned[ExecutionPolicy, Iter, Pred](ExecutionPolicy&& policy, Iter first, Iter last, Pred p) except + + Iter partition[Iter, Pred](Iter first, Iter last, Pred p) except + + Iter partition[ExecutionPolicy, Iter, Pred](ExecutionPolicy&& policy, Iter first, Iter last, Pred p) except + + pair[OutputIt1, OutputIt2] partition_copy[InputIt, OutputIt1, OutputIt2, Pred]( + InputIt first, InputIt last, OutputIt1 d_first_true, OutputIt2 d_first_false, Pred p) except + + pair[OutputIt1, OutputIt2] partition_copy[ExecutionPolicy, InputIt, OutputIt1, OutputIt2, Pred]( + ExecutionPolicy&& policy, InputIt first, InputIt last, OutputIt1 d_first_true, OutputIt2 d_first_false, Pred p) except + + + Iter stable_partition[Iter, Pred](Iter first, Iter last, Pred p) except + + Iter stable_partition[ExecutionPolicy, Iter, Pred](ExecutionPolicy&& policy, Iter first, Iter last, Pred p) except + + Iter partition_point[Iter, Pred](Iter first, Iter last, Pred p) except + + Iter partition_point[ExecutionPolicy, Iter, Pred](ExecutionPolicy&& policy, Iter first, Iter last, Pred p) except + + + # Sorting operations + bool is_sorted[Iter](Iter first, Iter last) except + + bool is_sorted[ExecutionPolicy, Iter](ExecutionPolicy&& policy, Iter first, Iter last) except + + bool is_sorted[Iter, Compare](Iter first, Iter last, Compare comp) except + + bool is_sorted[ExecutionPolicy, Iter, Compare](ExecutionPolicy&& policy, Iter first, Iter last, Compare comp) except + + + Iter is_sorted_until[Iter](Iter first, Iter last) except + + Iter is_sorted_until[ExecutionPolicy, Iter](ExecutionPolicy&& policy, Iter first, Iter last) except + + Iter is_sorted_until[Iter, Compare](Iter first, Iter last, Compare comp) except + + Iter is_sorted_until[ExecutionPolicy, Iter, Compare](ExecutionPolicy&& policy, Iter first, Iter last, Compare comp) except + + + void sort[Iter](Iter first, Iter last) except + + void sort[ExecutionPolicy, Iter](ExecutionPolicy&& policy, Iter first, Iter last) except + + void sort[Iter, Compare](Iter first, Iter last, Compare comp) except + + void sort[ExecutionPolicy, Iter, Compare](ExecutionPolicy&& policy, Iter first, Iter last, Compare comp) except + + + void partial_sort[Iter](Iter first, Iter middle, Iter last) except + + void partial_sort[ExecutionPolicy, Iter](ExecutionPolicy&& policy, Iter first, Iter middle, Iter last) except + + void partial_sort[Iter, Compare](Iter first, Iter middle, Iter last, Compare comp) except + + void partial_sort[ExecutionPolicy, Iter, Compare](ExecutionPolicy&& policy, Iter first, Iter middle, Iter last, Compare comp) except + + + OutputIt partial_sort_copy[InputIt, OutputIt]( + InputIt first, InputIt last, OutputIt d_first, OutputIt d_last) except + + OutputIt partial_sort_copy[ExecutionPolicy, InputIt, OutputIt]( + ExecutionPolicy&& policy, InputIt first, InputIt last, OutputIt d_first, OutputIt d_last) except + + OutputIt partial_sort_copy[InputIt, OutputIt, Compare]( + InputIt first, InputIt last, OutputIt d_first, OutputIt d_last, Compare comp) except + + OutputIt partial_sort_copy[ExecutionPolicy, InputIt, OutputIt, Compare]( + ExecutionPolicy&& policy, InputIt first, InputIt last, OutputIt d_first, OutputIt d_last, Compare comp) except + + + void stable_sort[Iter](Iter first, Iter last) except + + void stable_sort[ExecutionPolicy, Iter](ExecutionPolicy&& policy, Iter first, Iter last) except + + void stable_sort[Iter, Compare](Iter first, Iter last, Compare comp) except + + void stable_sort[ExecutionPolicy, Iter, Compare](ExecutionPolicy&& policy, Iter first, Iter last, Compare comp) except + + + void nth_element[Iter](Iter first, Iter nth, Iter last) except + + void nth_element[ExecutionPolicy, Iter](ExecutionPolicy&& policy, Iter first, Iter nth, Iter last) except + + void nth_element[Iter, Compare](Iter first, Iter nth, Iter last, Compare comp) except + + void nth_element[ExecutionPolicy, Iter, Compare](ExecutionPolicy&& policy, Iter first, Iter nth, Iter last, Compare comp) except + + + # Binary search operations (on sorted ranges) + Iter lower_bound[Iter, T](Iter first, Iter last, const T& value) except + + Iter lower_bound[ExecutionPolicy, Iter, T](ExecutionPolicy&& policy, Iter first, Iter last, const T& value) except + + Iter lower_bound[Iter, T, Compare](Iter first, Iter last, const T& value, Compare comp) except + + Iter lower_bound[ExecutionPolicy, Iter, T, Compare](ExecutionPolicy&& policy, Iter first, Iter last, const T& value, Compare comp) except + + + Iter upper_bound[Iter, T](Iter first, Iter last, const T& value) except + + Iter upper_bound[ExecutionPolicy, Iter, T](ExecutionPolicy&& policy, Iter first, Iter last, const T& value) except + + Iter upper_bound[Iter, T, Compare](Iter first, Iter last, const T& value, Compare comp) except + + Iter upper_bound[ExecutionPolicy, Iter, T, Compare](ExecutionPolicy&& policy, Iter first, Iter last, const T& value, Compare comp) except + + + bool binary_search[Iter, T](Iter first, Iter last, const T& value) except + + bool binary_search[ExecutionPolicy, Iter, T](ExecutionPolicy&& policy, Iter first, Iter last, const T& value) except + + bool binary_search[Iter, T, Compare](Iter first, Iter last, const T& value, Compare comp) except + + bool binary_search[ExecutionPolicy, Iter, T, Compare](ExecutionPolicy&& policy, Iter first, Iter last, const T& value, Compare comp) except + + + # Other operations on sorted ranges + + # Set operations (on sorted ranges) + + # Heap operations + void make_heap[Iter](Iter first, Iter last) except + + void make_heap[Iter, Compare](Iter first, Iter last, Compare comp) except + + + void push_heap[Iter](Iter first, Iter last) except + + void push_heap[Iter, Compare](Iter first, Iter last, Compare comp) except + + + void pop_heap[Iter](Iter first, Iter last) except + + void pop_heap[Iter, Compare](Iter first, Iter last, Compare comp) except + + + void sort_heap[Iter](Iter first, Iter last) except + + void sort_heap[Iter, Compare](Iter first, Iter last, Compare comp) except + + + # Minimum/maximum operations + Iter min_element[Iter](Iter first, Iter last) except + + Iter min_element[ExecutionPolicy, Iter](ExecutionPolicy&& policy, Iter first, Iter last) except + + Iter max_element[Iter](Iter first, Iter last) except + + Iter max_element[ExecutionPolicy, Iter](ExecutionPolicy&& policy, Iter first, Iter last) except + + + # Comparison operations + + # Permutation operations - # Copy - OutputIter copy[InputIter,OutputIter](InputIter,InputIter,OutputIter) diff --git a/Cython/Includes/libcpp/atomic.pxd b/Cython/Includes/libcpp/atomic.pxd new file mode 100644 index 000000000..3efd35aef --- /dev/null +++ b/Cython/Includes/libcpp/atomic.pxd @@ -0,0 +1,60 @@ + +cdef extern from "<atomic>" namespace "std" nogil: + + cdef enum memory_order: + memory_order_relaxed + memory_order_consume + memory_order_acquire + memory_order_release + memory_order_acq_rel + memory_order_seq_cst + + cdef cppclass atomic[T]: + atomic() + atomic(T) + + bint is_lock_free() + void store(T) + void store(T, memory_order) + T load() + T load(memory_order) + T exchange(T) + T exchange(T, memory_order) + + bint compare_exchange_weak(T&, T, memory_order, memory_order) + bint compare_exchange_weak(T&, T, memory_order) + bint compare_exchange_weak(T&, T) + bint compare_exchange_strong(T&, T, memory_order, memory_order) + bint compare_exchange_strong(T&, T, memory_order) + bint compare_exchange_strong(T&, T) + + T fetch_add(T, memory_order) + T fetch_add(T) + T fetch_sub(T, memory_order) + T fetch_sub(T) + T fetch_and(T, memory_order) + T fetch_and(T) + T fetch_or(T, memory_order) + T fetch_or(T) + T fetch_xor(T, memory_order) + T fetch_xor(T) + + T operator++() + T operator++(int) + T operator--() + T operator--(int) + + # modify-in-place operators not yet supported by Cython: + # T operator+=(T) + # T operator-=(T) + # T operator&=(T) + # T operator|=(T) + # T operator^=(T) + + bint operator==(atomic[T]&, atomic[T]&) + bint operator==(atomic[T]&, T&) + bint operator==(T&, atomic[T]&) + bint operator!=(atomic[T]&, atomic[T]&) + bint operator!=(atomic[T]&, T&) + bint operator!=(T&, atomic[T]&) + diff --git a/Cython/Includes/libcpp/deque.pxd b/Cython/Includes/libcpp/deque.pxd index 9e2b2291d..37fb8be3e 100644 --- a/Cython/Includes/libcpp/deque.pxd +++ b/Cython/Includes/libcpp/deque.pxd @@ -52,9 +52,9 @@ cdef extern from "<deque>" namespace "std" nogil: bint operator>(deque&, deque&) bint operator<=(deque&, deque&) bint operator>=(deque&, deque&) - void assign(size_t, T&) - void assign(input_iterator, input_iterator) - T& at(size_t) + void assign(size_t, T&) except + + void assign(input_iterator, input_iterator) except + + T& at(size_t) except + T& back() iterator begin() const_iterator const_begin "begin"() @@ -62,25 +62,25 @@ cdef extern from "<deque>" namespace "std" nogil: bint empty() iterator end() const_iterator const_end "end"() - iterator erase(iterator) - iterator erase(iterator, iterator) + iterator erase(iterator) except + + iterator erase(iterator, iterator) except + T& front() - iterator insert(iterator, T&) - void insert(iterator, size_t, T&) - void insert(iterator, input_iterator, input_iterator) + iterator insert(iterator, T&) except + + void insert(iterator, size_t, T&) except + + void insert(iterator, input_iterator, input_iterator) except + size_t max_size() void pop_back() void pop_front() - void push_back(T&) - void push_front(T&) + void push_back(T&) except + + void push_front(T&) except + reverse_iterator rbegin() #const_reverse_iterator rbegin() reverse_iterator rend() #const_reverse_iterator rend() - void resize(size_t) - void resize(size_t, T&) + void resize(size_t) except + + void resize(size_t, T&) except + size_t size() void swap(deque&) # C++11 methods - void shrink_to_fit() + void shrink_to_fit() except + diff --git a/Cython/Includes/libcpp/execution.pxd b/Cython/Includes/libcpp/execution.pxd new file mode 100644 index 000000000..09c86258d --- /dev/null +++ b/Cython/Includes/libcpp/execution.pxd @@ -0,0 +1,15 @@ + +cdef extern from "<execution>" namespace "std::execution" nogil: + cdef cppclass sequenced_policy: + pass + cdef cppclass parallel_policy: + pass + cdef cppclass parallel_unsequenced_policy: + pass + cdef cppclass unsequenced_policy: + pass + + const sequenced_policy seq "std::execution::seq" + const parallel_policy par "std::execution::par" + const parallel_unsequenced_policy par_unseq "std::execution::par_unseq" + const unsequenced_policy unseq "std::execution::unseq" diff --git a/Cython/Includes/libcpp/functional.pxd b/Cython/Includes/libcpp/functional.pxd index 94cbd9e1d..4786d39eb 100644 --- a/Cython/Includes/libcpp/functional.pxd +++ b/Cython/Includes/libcpp/functional.pxd @@ -1,3 +1,5 @@ +from libcpp cimport bool + cdef extern from "<functional>" namespace "std" nogil: cdef cppclass function[T]: function() except + @@ -10,4 +12,10 @@ cdef extern from "<functional>" namespace "std" nogil: function operator=(void*) function operator=[U](U) - bint operator bool() + bool operator bool() + + # Comparisons + cdef cppclass greater[T=*]: + # https://github.com/cython/cython/issues/3193 + greater() except + + bool operator()(const T& lhs, const T& rhs) except + diff --git a/Cython/Includes/libcpp/iterator.pxd b/Cython/Includes/libcpp/iterator.pxd index e0f8bd8d6..0b50c586d 100644 --- a/Cython/Includes/libcpp/iterator.pxd +++ b/Cython/Includes/libcpp/iterator.pxd @@ -1,6 +1,8 @@ #Basic reference: http://www.cplusplus.com/reference/iterator/ #Most of these classes are in fact empty structs +from libc.stddef import ptrdiff_t + cdef extern from "<iterator>" namespace "std" nogil: cdef cppclass iterator[Category,T,Distance,Pointer,Reference]: pass @@ -29,4 +31,4 @@ cdef extern from "<iterator>" namespace "std" nogil: ##insert_iterator<Container> inserter (Container& x, typename Container::iterator it) insert_iterator[CONTAINER] inserter[CONTAINER,ITERATOR](CONTAINER &, ITERATOR) - + ptrdiff_t distance[It](It first, It last) diff --git a/Cython/Includes/libcpp/list.pxd b/Cython/Includes/libcpp/list.pxd index b5b0410ad..1a391de12 100644 --- a/Cython/Includes/libcpp/list.pxd +++ b/Cython/Includes/libcpp/list.pxd @@ -39,7 +39,7 @@ cdef extern from "<list>" namespace "std" nogil: bint operator>(list&, list&) bint operator<=(list&, list&) bint operator>=(list&, list&) - void assign(size_t, T&) + void assign(size_t, T&) except + T& back() iterator begin() const_iterator const_begin "begin"() @@ -53,22 +53,22 @@ cdef extern from "<list>" namespace "std" nogil: iterator insert(iterator, T&) void insert(iterator, size_t, T&) size_t max_size() - void merge(list&) + void merge(list&) except + #void merge(list&, BinPred) void pop_back() void pop_front() - void push_back(T&) - void push_front(T&) + void push_back(T&) except + + void push_front(T&) except + reverse_iterator rbegin() const_reverse_iterator const_rbegin "rbegin"() - void remove(T&) + void remove(T&) except + #void remove_if(UnPred) reverse_iterator rend() const_reverse_iterator const_rend "rend"() - void resize(size_t, T&) + void resize(size_t, T&) except + void reverse() size_t size() - void sort() + void sort() except + #void sort(BinPred) void splice(iterator, list&) void splice(iterator, list&, iterator) diff --git a/Cython/Includes/libcpp/memory.pxd b/Cython/Includes/libcpp/memory.pxd index 2151c1ec7..c477c93fe 100644 --- a/Cython/Includes/libcpp/memory.pxd +++ b/Cython/Includes/libcpp/memory.pxd @@ -59,6 +59,7 @@ cdef extern from "<memory>" namespace "std" nogil: shared_ptr(shared_ptr[T]&, T*) shared_ptr(unique_ptr[T]&) #shared_ptr(weak_ptr[T]&) # Not Supported + shared_ptr[T]& operator=[Y](const shared_ptr[Y]& ptr) # Modifiers void reset() diff --git a/Cython/Includes/libcpp/numeric.pxd b/Cython/Includes/libcpp/numeric.pxd new file mode 100644 index 000000000..d61b3a406 --- /dev/null +++ b/Cython/Includes/libcpp/numeric.pxd @@ -0,0 +1,23 @@ +cdef extern from "<numeric>" namespace "std" nogil: + T inner_product[InputIt1, InputIt2, T](InputIt1 first1, InputIt1 last1, InputIt2 first2, T init) + + T inner_product[InputIt1, InputIt2, T, BinaryOperation1, BinaryOperation2](InputIt1 first1, InputIt1 last1, + InputIt2 first2, T init, + BinaryOperation1 op1, + BinaryOperation2 op2) + + void iota[ForwardIt, T](ForwardIt first, ForwardIt last, T value) + + T accumulate[InputIt, T](InputIt first, InputIt last, T init) + + T accumulate[InputIt, T, BinaryOperation](InputIt first, InputIt last, T init, BinaryOperation op) + + void adjacent_difference[InputIt, OutputIt](InputIt in_first, InputIt in_last, OutputIt out_first) + + void adjacent_difference[InputIt, OutputIt, BinaryOperation](InputIt in_first, InputIt in_last, OutputIt out_first, + BinaryOperation op) + + void partial_sum[InputIt, OutputIt](InputIt in_first, OutputIt in_last, OutputIt out_first) + + void partial_sum[InputIt, OutputIt, BinaryOperation](InputIt in_first, InputIt in_last, OutputIt out_first, + BinaryOperation op) diff --git a/Cython/Includes/libcpp/set.pxd b/Cython/Includes/libcpp/set.pxd index 1069be746..272509f78 100644 --- a/Cython/Includes/libcpp/set.pxd +++ b/Cython/Includes/libcpp/set.pxd @@ -59,3 +59,63 @@ cdef extern from "<set>" namespace "std" nogil: iterator upper_bound(const T&) const_iterator const_upper_bound "upper_bound"(const T&) #value_compare value_comp() + + cdef cppclass multiset[T]: + ctypedef T value_type + + cppclass iterator: + T& operator*() + iterator operator++() + iterator operator--() + bint operator==(iterator) + bint operator!=(iterator) + cppclass reverse_iterator: + T& operator*() + iterator operator++() + iterator operator--() + bint operator==(reverse_iterator) + bint operator!=(reverse_iterator) + cppclass const_iterator(iterator): + pass + cppclass const_reverse_iterator(reverse_iterator): + pass + + multiset() except + + multiset(multiset&) except + + #multiset(key_compare&) + #multiset& operator=(multiset&) + bint operator==(multiset&, multiset&) + bint operator!=(multiset&, multiset&) + bint operator<(multiset&, multiset&) + bint operator>(multiset&, multiset&) + bint operator<=(multiset&, multiset&) + bint operator>=(multiset&, multiset&) + iterator begin() + const_iterator const_begin "begin"() + void clear() + size_t count(const T&) + bint empty() + iterator end() + const_iterator const_end "end"() + pair[iterator, iterator] equal_range(const T&) + #pair[const_iterator, const_iterator] equal_range(T&) + iterator erase(iterator) + iterator erase(iterator, iterator) + size_t erase(T&) + iterator find(T&) + const_iterator const_find "find"(T&) + pair[iterator, bint] insert(const T&) except + + iterator insert(iterator, const T&) except + + void insert(iterator, iterator) except + + #key_compare key_comp() + iterator lower_bound(T&) + const_iterator const_lower_bound "lower_bound"(T&) + size_t max_size() + reverse_iterator rbegin() + const_reverse_iterator const_rbegin "rbegin"() + reverse_iterator rend() + const_reverse_iterator const_rend "rend"() + size_t size() + void swap(multiset&) + iterator upper_bound(const T&) + const_iterator const_upper_bound "upper_bound"(const T&) diff --git a/Cython/Includes/libcpp/unordered_set.pxd b/Cython/Includes/libcpp/unordered_set.pxd index 5aa241752..2eba3a1c0 100644 --- a/Cython/Includes/libcpp/unordered_set.pxd +++ b/Cython/Includes/libcpp/unordered_set.pxd @@ -21,14 +21,9 @@ cdef extern from "<unordered_set>" namespace "std" nogil: pass unordered_set() except + unordered_set(unordered_set&) except + - #unordered_set(key_compare&) #unordered_set& operator=(unordered_set&) bint operator==(unordered_set&, unordered_set&) bint operator!=(unordered_set&, unordered_set&) - bint operator<(unordered_set&, unordered_set&) - bint operator>(unordered_set&, unordered_set&) - bint operator<=(unordered_set&, unordered_set&) - bint operator>=(unordered_set&, unordered_set&) iterator begin() const_iterator const_begin "begin"() void clear() @@ -45,10 +40,7 @@ cdef extern from "<unordered_set>" namespace "std" nogil: const_iterator const_find "find"(T&) pair[iterator, bint] insert(T&) iterator insert(iterator, T&) - #key_compare key_comp() iterator insert(iterator, iterator) - iterator lower_bound(T&) - const_iterator const_lower_bound "lower_bound"(T&) size_t max_size() reverse_iterator rbegin() const_reverse_iterator const_rbegin "rbegin"() @@ -56,8 +48,65 @@ cdef extern from "<unordered_set>" namespace "std" nogil: const_reverse_iterator const_rend "rend"() size_t size() void swap(unordered_set&) - iterator upper_bound(T&) - const_iterator const_upper_bound "upper_bound"(T&) + #value_compare value_comp() + void max_load_factor(float) + float max_load_factor() + void rehash(size_t) + void reserve(size_t) + size_t bucket_count() + size_t max_bucket_count() + size_t bucket_size(size_t) + size_t bucket(const T&) + + cdef cppclass unordered_multiset[T,HASH=*,PRED=*,ALLOCATOR=*]: + ctypedef T value_type + + cppclass iterator: + T& operator*() + iterator operator++() + iterator operator--() + bint operator==(iterator) + bint operator!=(iterator) + cppclass reverse_iterator: + T& operator*() + iterator operator++() + iterator operator--() + bint operator==(reverse_iterator) + bint operator!=(reverse_iterator) + cppclass const_iterator(iterator): + pass + cppclass const_reverse_iterator(reverse_iterator): + pass + + unordered_multiset() except + + unordered_multiset(unordered_multiset&) except + + #unordered_multiset& operator=(unordered_multiset&) + bint operator==(unordered_multiset&, unordered_multiset&) + bint operator!=(unordered_multiset&, unordered_multiset&) + iterator begin() + const_iterator const_begin "begin"() + void clear() + size_t count(T&) + bint empty() + iterator end() + const_iterator const_end "end"() + pair[iterator, iterator] equal_range(T&) + pair[const_iterator, const_iterator] const_equal_range "equal_range"(T&) + iterator erase(iterator) + iterator erase(iterator, iterator) + size_t erase(T&) + iterator find(T&) + const_iterator const_find "find"(T&) + pair[iterator, bint] insert(T&) + iterator insert(iterator, T&) + iterator insert(iterator, iterator) + size_t max_size() + reverse_iterator rbegin() + const_reverse_iterator const_rbegin "rbegin"() + reverse_iterator rend() + const_reverse_iterator const_rend "rend"() + size_t size() + void swap(unordered_set&) #value_compare value_comp() void max_load_factor(float) float max_load_factor() diff --git a/Cython/Includes/libcpp/utility.pxd b/Cython/Includes/libcpp/utility.pxd index ec8ba4018..e0df69b16 100644 --- a/Cython/Includes/libcpp/utility.pxd +++ b/Cython/Includes/libcpp/utility.pxd @@ -16,7 +16,8 @@ cdef extern from "<utility>" namespace "std" nogil: cdef extern from * namespace "cython_std" nogil: """ - #if __cplusplus > 199711L + #if __cplusplus >= 201103L || (defined(_MSC_VER) && _MSC_VER >= 1600) + // move should be defined for these versions of MSVC, but __cplusplus isn't set usefully #include <type_traits> namespace cython_std { diff --git a/Cython/Includes/libcpp/vector.pxd b/Cython/Includes/libcpp/vector.pxd index 9b007dd0c..7ce0fa7a6 100644 --- a/Cython/Includes/libcpp/vector.pxd +++ b/Cython/Includes/libcpp/vector.pxd @@ -76,7 +76,7 @@ cdef extern from "<vector>" namespace "std" nogil: const_reverse_iterator const_rbegin "crbegin"() reverse_iterator rend() const_reverse_iterator const_rend "crend"() - void reserve(size_type) + void reserve(size_type) except + void resize(size_type) except + void resize(size_type, T&) except + size_type size() @@ -85,4 +85,4 @@ cdef extern from "<vector>" namespace "std" nogil: # C++11 methods T* data() const T* const_data "data"() - void shrink_to_fit() + void shrink_to_fit() except + diff --git a/Cython/Includes/numpy/__init__.pxd b/Cython/Includes/numpy/__init__.pxd index 15700c05e..3d9aff71b 100644 --- a/Cython/Includes/numpy/__init__.pxd +++ b/Cython/Includes/numpy/__init__.pxd @@ -1,30 +1,31 @@ # NumPy static imports for Cython # -# If any of the PyArray_* functions are called, import_array must be -# called first. -# -# This also defines backwards-compatibility buffer acquisition -# code for use in Python 2.x (or Python <= 2.5 when NumPy starts -# implementing PEP-3118 directly). +# NOTE: Do not make incompatible local changes to this file without contacting the NumPy project. +# This file is maintained by the NumPy project at +# https://github.com/numpy/numpy/tree/master/numpy # -# Because of laziness, the format string of the buffer is statically -# allocated. Increase the size if this is not enough, or submit a -# patch to do this properly. +# If any of the PyArray_* functions are called, import_array must be +# called first. This is done automatically by Cython 3.0+ if a call +# is not detected inside of the module. # # Author: Dag Sverre Seljebotn # -DEF _buffer_format_string_len = 255 - -cimport cpython.buffer as pybuf from cpython.ref cimport Py_INCREF -from cpython.mem cimport PyObject_Malloc, PyObject_Free -from cpython.object cimport PyObject, PyTypeObject -from cpython.type cimport type +from cpython.object cimport PyObject, PyTypeObject, PyObject_TypeCheck cimport libc.stdio as stdio + +cdef extern from *: + # Leave a marker that the NumPy declarations came from Cython and not from NumPy itself. + # See https://github.com/cython/cython/issues/3573 + """ + /* Using NumPy API declarations from "Cython/Includes/numpy/" */ + """ + + cdef extern from "Python.h": - ctypedef int Py_intptr_t + ctypedef Py_ssize_t Py_intptr_t cdef extern from "numpy/arrayobject.h": ctypedef Py_intptr_t npy_intp @@ -219,7 +220,7 @@ cdef extern from "numpy/arrayobject.h": cdef int type_num cdef int itemsize "elsize" cdef int alignment - cdef dict fields + cdef object fields cdef tuple names # Use PyDataType_HASSUBARRAY to test whether this field is # valid (the pointer can be NULL). Most users should access @@ -242,104 +243,56 @@ cdef extern from "numpy/arrayobject.h": ctypedef class numpy.ndarray [object PyArrayObject, check_size ignore]: cdef __cythonbufferdefaults__ = {"mode": "strided"} - cdef: - # Only taking a few of the most commonly used and stable fields. - # One should use PyArray_* macros instead to access the C fields. - char *data - int ndim "nd" - npy_intp *shape "dimensions" - npy_intp *strides - dtype descr # deprecated since NumPy 1.7 ! - PyObject* base - - # Note: This syntax (function definition in pxd files) is an - # experimental exception made for __getbuffer__ and __releasebuffer__ - # -- the details of this may change. - def __getbuffer__(ndarray self, Py_buffer* info, int flags): - # This implementation of getbuffer is geared towards Cython - # requirements, and does not yet fulfill the PEP. - # In particular strided access is always provided regardless - # of flags - - cdef int i, ndim - cdef int endian_detector = 1 - cdef bint little_endian = ((<char*>&endian_detector)[0] != 0) - - ndim = PyArray_NDIM(self) - - if ((flags & pybuf.PyBUF_C_CONTIGUOUS == pybuf.PyBUF_C_CONTIGUOUS) - and not PyArray_CHKFLAGS(self, NPY_ARRAY_C_CONTIGUOUS)): - raise ValueError(u"ndarray is not C contiguous") - - if ((flags & pybuf.PyBUF_F_CONTIGUOUS == pybuf.PyBUF_F_CONTIGUOUS) - and not PyArray_CHKFLAGS(self, NPY_ARRAY_F_CONTIGUOUS)): - raise ValueError(u"ndarray is not Fortran contiguous") - - info.buf = PyArray_DATA(self) - info.ndim = ndim - if sizeof(npy_intp) != sizeof(Py_ssize_t): - # Allocate new buffer for strides and shape info. - # This is allocated as one block, strides first. - info.strides = <Py_ssize_t*>PyObject_Malloc(sizeof(Py_ssize_t) * 2 * <size_t>ndim) - info.shape = info.strides + ndim - for i in range(ndim): - info.strides[i] = PyArray_STRIDES(self)[i] - info.shape[i] = PyArray_DIMS(self)[i] - else: - info.strides = <Py_ssize_t*>PyArray_STRIDES(self) - info.shape = <Py_ssize_t*>PyArray_DIMS(self) - info.suboffsets = NULL - info.itemsize = PyArray_ITEMSIZE(self) - info.readonly = not PyArray_ISWRITEABLE(self) - - cdef int t - cdef char* f = NULL - cdef dtype descr = <dtype>PyArray_DESCR(self) - cdef int offset - - info.obj = self - - if not PyDataType_HASFIELDS(descr): - t = descr.type_num - if ((descr.byteorder == c'>' and little_endian) or - (descr.byteorder == c'<' and not little_endian)): - raise ValueError(u"Non-native byte order not supported") - if t == NPY_BYTE: f = "b" - elif t == NPY_UBYTE: f = "B" - elif t == NPY_SHORT: f = "h" - elif t == NPY_USHORT: f = "H" - elif t == NPY_INT: f = "i" - elif t == NPY_UINT: f = "I" - elif t == NPY_LONG: f = "l" - elif t == NPY_ULONG: f = "L" - elif t == NPY_LONGLONG: f = "q" - elif t == NPY_ULONGLONG: f = "Q" - elif t == NPY_FLOAT: f = "f" - elif t == NPY_DOUBLE: f = "d" - elif t == NPY_LONGDOUBLE: f = "g" - elif t == NPY_CFLOAT: f = "Zf" - elif t == NPY_CDOUBLE: f = "Zd" - elif t == NPY_CLONGDOUBLE: f = "Zg" - elif t == NPY_OBJECT: f = "O" - else: - raise ValueError(u"unknown dtype code in numpy.pxd (%d)" % t) - info.format = f - return - else: - info.format = <char*>PyObject_Malloc(_buffer_format_string_len) - info.format[0] = c'^' # Native data types, manual alignment - offset = 0 - f = _util_dtypestring(descr, info.format + 1, - info.format + _buffer_format_string_len, - &offset) - f[0] = c'\0' # Terminate format string - - def __releasebuffer__(ndarray self, Py_buffer* info): - if PyArray_HASFIELDS(self): - PyObject_Free(info.format) - if sizeof(npy_intp) != sizeof(Py_ssize_t): - PyObject_Free(info.strides) - # info.shape was stored after info.strides in the same block + # NOTE: no field declarations since direct access is deprecated since NumPy 1.7 + # Instead, we use properties that map to the corresponding C-API functions. + + @property + cdef inline PyObject* base(self) nogil: + """Returns a borrowed reference to the object owning the data/memory. + """ + return PyArray_BASE(self) + + @property + cdef inline dtype descr(self): + """Returns an owned reference to the dtype of the array. + """ + return <dtype>PyArray_DESCR(self) + + @property + cdef inline int ndim(self) nogil: + """Returns the number of dimensions in the array. + """ + return PyArray_NDIM(self) + + @property + cdef inline npy_intp *shape(self) nogil: + """Returns a pointer to the dimensions/shape of the array. + The number of elements matches the number of dimensions of the array (ndim). + Can return NULL for 0-dimensional arrays. + """ + return PyArray_DIMS(self) + + @property + cdef inline npy_intp *strides(self) nogil: + """Returns a pointer to the strides of the array. + The number of elements matches the number of dimensions of the array (ndim). + """ + return PyArray_STRIDES(self) + + @property + cdef inline npy_intp size(self) nogil: + """Returns the total size (in number of elements) of the array. + """ + return PyArray_SIZE(self) + + @property + cdef inline char* data(self) nogil: + """The pointer to the data buffer as a char*. + This is provided for legacy reasons to avoid direct struct field access. + For new code that needs this access, you probably want to cast the result + of `PyArray_DATA()` instead, which returns a 'void*'. + """ + return PyArray_BYTES(self) ctypedef unsigned char npy_bool @@ -416,103 +369,110 @@ cdef extern from "numpy/arrayobject.h": int len int _import_array() except -1 + # A second definition so _import_array isn't marked as used when we use it here. + # Do not use - subject to change any time. + int __pyx_import_array "_import_array"() except -1 # # Macros from ndarrayobject.h # - bint PyArray_CHKFLAGS(ndarray m, int flags) - bint PyArray_IS_C_CONTIGUOUS(ndarray arr) - bint PyArray_IS_F_CONTIGUOUS(ndarray arr) - bint PyArray_ISCONTIGUOUS(ndarray m) - bint PyArray_ISWRITEABLE(ndarray m) - bint PyArray_ISALIGNED(ndarray m) - - int PyArray_NDIM(ndarray) - bint PyArray_ISONESEGMENT(ndarray) - bint PyArray_ISFORTRAN(ndarray) - int PyArray_FORTRANIF(ndarray) - - void* PyArray_DATA(ndarray) - char* PyArray_BYTES(ndarray) - npy_intp* PyArray_DIMS(ndarray) - npy_intp* PyArray_STRIDES(ndarray) - npy_intp PyArray_DIM(ndarray, size_t) - npy_intp PyArray_STRIDE(ndarray, size_t) - - PyObject *PyArray_BASE(ndarray) # returns borrowed reference! - PyArray_Descr *PyArray_DESCR(ndarray) # returns borrowed reference to dtype! - int PyArray_FLAGS(ndarray) - npy_intp PyArray_ITEMSIZE(ndarray) - int PyArray_TYPE(ndarray arr) + bint PyArray_CHKFLAGS(ndarray m, int flags) nogil + bint PyArray_IS_C_CONTIGUOUS(ndarray arr) nogil + bint PyArray_IS_F_CONTIGUOUS(ndarray arr) nogil + bint PyArray_ISCONTIGUOUS(ndarray m) nogil + bint PyArray_ISWRITEABLE(ndarray m) nogil + bint PyArray_ISALIGNED(ndarray m) nogil + + int PyArray_NDIM(ndarray) nogil + bint PyArray_ISONESEGMENT(ndarray) nogil + bint PyArray_ISFORTRAN(ndarray) nogil + int PyArray_FORTRANIF(ndarray) nogil + + void* PyArray_DATA(ndarray) nogil + char* PyArray_BYTES(ndarray) nogil + + npy_intp* PyArray_DIMS(ndarray) nogil + npy_intp* PyArray_STRIDES(ndarray) nogil + npy_intp PyArray_DIM(ndarray, size_t) nogil + npy_intp PyArray_STRIDE(ndarray, size_t) nogil + + PyObject *PyArray_BASE(ndarray) nogil # returns borrowed reference! + PyArray_Descr *PyArray_DESCR(ndarray) nogil # returns borrowed reference to dtype! + PyArray_Descr *PyArray_DTYPE(ndarray) nogil # returns borrowed reference to dtype! NP 1.7+ alias for descr. + int PyArray_FLAGS(ndarray) nogil + void PyArray_CLEARFLAGS(ndarray, int flags) nogil # Added in NumPy 1.7 + void PyArray_ENABLEFLAGS(ndarray, int flags) nogil # Added in NumPy 1.7 + npy_intp PyArray_ITEMSIZE(ndarray) nogil + int PyArray_TYPE(ndarray arr) nogil object PyArray_GETITEM(ndarray arr, void *itemptr) int PyArray_SETITEM(ndarray arr, void *itemptr, object obj) - bint PyTypeNum_ISBOOL(int) - bint PyTypeNum_ISUNSIGNED(int) - bint PyTypeNum_ISSIGNED(int) - bint PyTypeNum_ISINTEGER(int) - bint PyTypeNum_ISFLOAT(int) - bint PyTypeNum_ISNUMBER(int) - bint PyTypeNum_ISSTRING(int) - bint PyTypeNum_ISCOMPLEX(int) - bint PyTypeNum_ISPYTHON(int) - bint PyTypeNum_ISFLEXIBLE(int) - bint PyTypeNum_ISUSERDEF(int) - bint PyTypeNum_ISEXTENDED(int) - bint PyTypeNum_ISOBJECT(int) - - bint PyDataType_ISBOOL(dtype) - bint PyDataType_ISUNSIGNED(dtype) - bint PyDataType_ISSIGNED(dtype) - bint PyDataType_ISINTEGER(dtype) - bint PyDataType_ISFLOAT(dtype) - bint PyDataType_ISNUMBER(dtype) - bint PyDataType_ISSTRING(dtype) - bint PyDataType_ISCOMPLEX(dtype) - bint PyDataType_ISPYTHON(dtype) - bint PyDataType_ISFLEXIBLE(dtype) - bint PyDataType_ISUSERDEF(dtype) - bint PyDataType_ISEXTENDED(dtype) - bint PyDataType_ISOBJECT(dtype) - bint PyDataType_HASFIELDS(dtype) - bint PyDataType_HASSUBARRAY(dtype) - - bint PyArray_ISBOOL(ndarray) - bint PyArray_ISUNSIGNED(ndarray) - bint PyArray_ISSIGNED(ndarray) - bint PyArray_ISINTEGER(ndarray) - bint PyArray_ISFLOAT(ndarray) - bint PyArray_ISNUMBER(ndarray) - bint PyArray_ISSTRING(ndarray) - bint PyArray_ISCOMPLEX(ndarray) - bint PyArray_ISPYTHON(ndarray) - bint PyArray_ISFLEXIBLE(ndarray) - bint PyArray_ISUSERDEF(ndarray) - bint PyArray_ISEXTENDED(ndarray) - bint PyArray_ISOBJECT(ndarray) - bint PyArray_HASFIELDS(ndarray) - - bint PyArray_ISVARIABLE(ndarray) - - bint PyArray_SAFEALIGNEDCOPY(ndarray) - bint PyArray_ISNBO(char) # works on ndarray.byteorder - bint PyArray_IsNativeByteOrder(char) # works on ndarray.byteorder - bint PyArray_ISNOTSWAPPED(ndarray) - bint PyArray_ISBYTESWAPPED(ndarray) - - bint PyArray_FLAGSWAP(ndarray, int) - - bint PyArray_ISCARRAY(ndarray) - bint PyArray_ISCARRAY_RO(ndarray) - bint PyArray_ISFARRAY(ndarray) - bint PyArray_ISFARRAY_RO(ndarray) - bint PyArray_ISBEHAVED(ndarray) - bint PyArray_ISBEHAVED_RO(ndarray) - - - bint PyDataType_ISNOTSWAPPED(dtype) - bint PyDataType_ISBYTESWAPPED(dtype) + bint PyTypeNum_ISBOOL(int) nogil + bint PyTypeNum_ISUNSIGNED(int) nogil + bint PyTypeNum_ISSIGNED(int) nogil + bint PyTypeNum_ISINTEGER(int) nogil + bint PyTypeNum_ISFLOAT(int) nogil + bint PyTypeNum_ISNUMBER(int) nogil + bint PyTypeNum_ISSTRING(int) nogil + bint PyTypeNum_ISCOMPLEX(int) nogil + bint PyTypeNum_ISPYTHON(int) nogil + bint PyTypeNum_ISFLEXIBLE(int) nogil + bint PyTypeNum_ISUSERDEF(int) nogil + bint PyTypeNum_ISEXTENDED(int) nogil + bint PyTypeNum_ISOBJECT(int) nogil + + bint PyDataType_ISBOOL(dtype) nogil + bint PyDataType_ISUNSIGNED(dtype) nogil + bint PyDataType_ISSIGNED(dtype) nogil + bint PyDataType_ISINTEGER(dtype) nogil + bint PyDataType_ISFLOAT(dtype) nogil + bint PyDataType_ISNUMBER(dtype) nogil + bint PyDataType_ISSTRING(dtype) nogil + bint PyDataType_ISCOMPLEX(dtype) nogil + bint PyDataType_ISPYTHON(dtype) nogil + bint PyDataType_ISFLEXIBLE(dtype) nogil + bint PyDataType_ISUSERDEF(dtype) nogil + bint PyDataType_ISEXTENDED(dtype) nogil + bint PyDataType_ISOBJECT(dtype) nogil + bint PyDataType_HASFIELDS(dtype) nogil + bint PyDataType_HASSUBARRAY(dtype) nogil + + bint PyArray_ISBOOL(ndarray) nogil + bint PyArray_ISUNSIGNED(ndarray) nogil + bint PyArray_ISSIGNED(ndarray) nogil + bint PyArray_ISINTEGER(ndarray) nogil + bint PyArray_ISFLOAT(ndarray) nogil + bint PyArray_ISNUMBER(ndarray) nogil + bint PyArray_ISSTRING(ndarray) nogil + bint PyArray_ISCOMPLEX(ndarray) nogil + bint PyArray_ISPYTHON(ndarray) nogil + bint PyArray_ISFLEXIBLE(ndarray) nogil + bint PyArray_ISUSERDEF(ndarray) nogil + bint PyArray_ISEXTENDED(ndarray) nogil + bint PyArray_ISOBJECT(ndarray) nogil + bint PyArray_HASFIELDS(ndarray) nogil + + bint PyArray_ISVARIABLE(ndarray) nogil + + bint PyArray_SAFEALIGNEDCOPY(ndarray) nogil + bint PyArray_ISNBO(char) nogil # works on ndarray.byteorder + bint PyArray_IsNativeByteOrder(char) nogil # works on ndarray.byteorder + bint PyArray_ISNOTSWAPPED(ndarray) nogil + bint PyArray_ISBYTESWAPPED(ndarray) nogil + + bint PyArray_FLAGSWAP(ndarray, int) nogil + + bint PyArray_ISCARRAY(ndarray) nogil + bint PyArray_ISCARRAY_RO(ndarray) nogil + bint PyArray_ISFARRAY(ndarray) nogil + bint PyArray_ISFARRAY_RO(ndarray) nogil + bint PyArray_ISBEHAVED(ndarray) nogil + bint PyArray_ISBEHAVED_RO(ndarray) nogil + + + bint PyDataType_ISNOTSWAPPED(dtype) nogil + bint PyDataType_ISBYTESWAPPED(dtype) nogil bint PyArray_DescrCheck(object) @@ -532,10 +492,11 @@ cdef extern from "numpy/arrayobject.h": bint PyArray_IsPythonScalar(object) bint PyArray_IsAnyScalar(object) bint PyArray_CheckAnyScalar(object) + ndarray PyArray_GETCONTIGUOUS(ndarray) - bint PyArray_SAMESHAPE(ndarray, ndarray) - npy_intp PyArray_SIZE(ndarray) - npy_intp PyArray_NBYTES(ndarray) + bint PyArray_SAMESHAPE(ndarray, ndarray) nogil + npy_intp PyArray_SIZE(ndarray) nogil + npy_intp PyArray_NBYTES(ndarray) nogil object PyArray_FROM_O(object) object PyArray_FROM_OF(object m, int flags) @@ -548,16 +509,16 @@ cdef extern from "numpy/arrayobject.h": npy_intp PyArray_REFCOUNT(object) object PyArray_ContiguousFromAny(op, int, int min_depth, int max_depth) unsigned char PyArray_EquivArrTypes(ndarray a1, ndarray a2) - bint PyArray_EquivByteorders(int b1, int b2) + bint PyArray_EquivByteorders(int b1, int b2) nogil object PyArray_SimpleNew(int nd, npy_intp* dims, int typenum) object PyArray_SimpleNewFromData(int nd, npy_intp* dims, int typenum, void* data) #object PyArray_SimpleNewFromDescr(int nd, npy_intp* dims, dtype descr) object PyArray_ToScalar(void* data, ndarray arr) - void* PyArray_GETPTR1(ndarray m, npy_intp i) - void* PyArray_GETPTR2(ndarray m, npy_intp i, npy_intp j) - void* PyArray_GETPTR3(ndarray m, npy_intp i, npy_intp j, npy_intp k) - void* PyArray_GETPTR4(ndarray m, npy_intp i, npy_intp j, npy_intp k, npy_intp l) + void* PyArray_GETPTR1(ndarray m, npy_intp i) nogil + void* PyArray_GETPTR2(ndarray m, npy_intp i, npy_intp j) nogil + void* PyArray_GETPTR3(ndarray m, npy_intp i, npy_intp j, npy_intp k) nogil + void* PyArray_GETPTR4(ndarray m, npy_intp i, npy_intp j, npy_intp k, npy_intp l) nogil void PyArray_XDECREF_ERR(ndarray) # Cannot be supported due to out arg @@ -685,7 +646,7 @@ cdef extern from "numpy/arrayobject.h": object PyArray_Choose (ndarray, object, ndarray, NPY_CLIPMODE) int PyArray_Sort (ndarray, int, NPY_SORTKIND) object PyArray_ArgSort (ndarray, int, NPY_SORTKIND) - object PyArray_SearchSorted (ndarray, object, NPY_SEARCHSIDE, PyObject*) + object PyArray_SearchSorted (ndarray, object, NPY_SEARCHSIDE, PyObject *) object PyArray_ArgMax (ndarray, int, ndarray) object PyArray_ArgMin (ndarray, int, ndarray) object PyArray_Reshape (ndarray, object) @@ -838,72 +799,67 @@ cdef inline tuple PyDataType_SHAPE(dtype d): else: return () -cdef inline char* _util_dtypestring(dtype descr, char* f, char* end, int* offset) except NULL: - # Recursive utility function used in __getbuffer__ to get format - # string. The new location in the format string is returned. - - cdef dtype child - cdef int endian_detector = 1 - cdef bint little_endian = ((<char*>&endian_detector)[0] != 0) - cdef tuple fields - - for childname in descr.names: - fields = descr.fields[childname] - child, new_offset = fields - - if (end - f) - <int>(new_offset - offset[0]) < 15: - raise RuntimeError(u"Format string allocated too short, see comment in numpy.pxd") - - if ((child.byteorder == c'>' and little_endian) or - (child.byteorder == c'<' and not little_endian)): - raise ValueError(u"Non-native byte order not supported") - # One could encode it in the format string and have Cython - # complain instead, BUT: < and > in format strings also imply - # standardized sizes for datatypes, and we rely on native in - # order to avoid reencoding data types based on their size. - # - # A proper PEP 3118 exporter for other clients than Cython - # must deal properly with this! - - # Output padding bytes - while offset[0] < new_offset: - f[0] = 120 # "x"; pad byte - f += 1 - offset[0] += 1 - - offset[0] += child.itemsize - - if not PyDataType_HASFIELDS(child): - t = child.type_num - if end - f < 5: - raise RuntimeError(u"Format string allocated too short.") - - # Until ticket #99 is fixed, use integers to avoid warnings - if t == NPY_BYTE: f[0] = 98 #"b" - elif t == NPY_UBYTE: f[0] = 66 #"B" - elif t == NPY_SHORT: f[0] = 104 #"h" - elif t == NPY_USHORT: f[0] = 72 #"H" - elif t == NPY_INT: f[0] = 105 #"i" - elif t == NPY_UINT: f[0] = 73 #"I" - elif t == NPY_LONG: f[0] = 108 #"l" - elif t == NPY_ULONG: f[0] = 76 #"L" - elif t == NPY_LONGLONG: f[0] = 113 #"q" - elif t == NPY_ULONGLONG: f[0] = 81 #"Q" - elif t == NPY_FLOAT: f[0] = 102 #"f" - elif t == NPY_DOUBLE: f[0] = 100 #"d" - elif t == NPY_LONGDOUBLE: f[0] = 103 #"g" - elif t == NPY_CFLOAT: f[0] = 90; f[1] = 102; f += 1 # Zf - elif t == NPY_CDOUBLE: f[0] = 90; f[1] = 100; f += 1 # Zd - elif t == NPY_CLONGDOUBLE: f[0] = 90; f[1] = 103; f += 1 # Zg - elif t == NPY_OBJECT: f[0] = 79 #"O" - else: - raise ValueError(u"unknown dtype code in numpy.pxd (%d)" % t) - f += 1 - else: - # Cython ignores struct boundary information ("T{...}"), - # so don't output it - f = _util_dtypestring(child, f, end, offset) - return f + +cdef extern from "numpy/ndarrayobject.h": + PyTypeObject PyTimedeltaArrType_Type + PyTypeObject PyDatetimeArrType_Type + ctypedef int64_t npy_timedelta + ctypedef int64_t npy_datetime + +cdef extern from "numpy/ndarraytypes.h": + ctypedef struct PyArray_DatetimeMetaData: + NPY_DATETIMEUNIT base + int64_t num + +cdef extern from "numpy/arrayscalars.h": + + # abstract types + ctypedef class numpy.generic [object PyObject]: + pass + ctypedef class numpy.number [object PyObject]: + pass + ctypedef class numpy.integer [object PyObject]: + pass + ctypedef class numpy.signedinteger [object PyObject]: + pass + ctypedef class numpy.unsignedinteger [object PyObject]: + pass + ctypedef class numpy.inexact [object PyObject]: + pass + ctypedef class numpy.floating [object PyObject]: + pass + ctypedef class numpy.complexfloating [object PyObject]: + pass + ctypedef class numpy.flexible [object PyObject]: + pass + ctypedef class numpy.character [object PyObject]: + pass + + ctypedef struct PyDatetimeScalarObject: + # PyObject_HEAD + npy_datetime obval + PyArray_DatetimeMetaData obmeta + + ctypedef struct PyTimedeltaScalarObject: + # PyObject_HEAD + npy_timedelta obval + PyArray_DatetimeMetaData obmeta + + ctypedef enum NPY_DATETIMEUNIT: + NPY_FR_Y + NPY_FR_M + NPY_FR_W + NPY_FR_D + NPY_FR_B + NPY_FR_h + NPY_FR_m + NPY_FR_s + NPY_FR_ms + NPY_FR_us + NPY_FR_ns + NPY_FR_ps + NPY_FR_fs + NPY_FR_as # @@ -1032,7 +988,7 @@ cdef inline object get_array_base(ndarray arr): # Cython code. cdef inline int import_array() except -1: try: - _import_array() + __pyx_import_array() except Exception: raise ImportError("numpy.core.multiarray failed to import") @@ -1047,3 +1003,57 @@ cdef inline int import_ufunc() except -1: _import_umath() except Exception: raise ImportError("numpy.core.umath failed to import") + + +cdef inline bint is_timedelta64_object(object obj): + """ + Cython equivalent of `isinstance(obj, np.timedelta64)` + + Parameters + ---------- + obj : object + + Returns + ------- + bool + """ + return PyObject_TypeCheck(obj, &PyTimedeltaArrType_Type) + + +cdef inline bint is_datetime64_object(object obj): + """ + Cython equivalent of `isinstance(obj, np.datetime64)` + + Parameters + ---------- + obj : object + + Returns + ------- + bool + """ + return PyObject_TypeCheck(obj, &PyDatetimeArrType_Type) + + +cdef inline npy_datetime get_datetime64_value(object obj) nogil: + """ + returns the int64 value underlying scalar numpy datetime64 object + + Note that to interpret this as a datetime, the corresponding unit is + also needed. That can be found using `get_datetime64_unit`. + """ + return (<PyDatetimeScalarObject*>obj).obval + + +cdef inline npy_timedelta get_timedelta64_value(object obj) nogil: + """ + returns the int64 value underlying scalar numpy timedelta64 object + """ + return (<PyTimedeltaScalarObject*>obj).obval + + +cdef inline NPY_DATETIMEUNIT get_datetime64_unit(object obj) nogil: + """ + returns the unit part of the dtype for a numpy datetime64 object. + """ + return <NPY_DATETIMEUNIT>(<PyDatetimeScalarObject*>obj).obmeta.base diff --git a/Cython/Includes/posix/dlfcn.pxd b/Cython/Includes/posix/dlfcn.pxd index cff5bea15..bf61997f3 100644 --- a/Cython/Includes/posix/dlfcn.pxd +++ b/Cython/Includes/posix/dlfcn.pxd @@ -1,5 +1,5 @@ # POSIX dynamic linking/loading interface. -# http://pubs.opengroup.org/onlinepubs/9699919799/basedefs/dlfcn.h.html +# https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/dlfcn.h.html cdef extern from "<dlfcn.h>" nogil: void *dlopen(const char *, int) diff --git a/Cython/Includes/posix/mman.pxd b/Cython/Includes/posix/mman.pxd index c810f431b..9bcad66db 100644 --- a/Cython/Includes/posix/mman.pxd +++ b/Cython/Includes/posix/mman.pxd @@ -1,4 +1,4 @@ -# http://pubs.opengroup.org/onlinepubs/009695399/basedefs/sys/mman.h.html +# https://pubs.opengroup.org/onlinepubs/009695399/basedefs/sys/mman.h.html from posix.types cimport off_t, mode_t diff --git a/Cython/Includes/posix/resource.pxd b/Cython/Includes/posix/resource.pxd index 9f55c6ab4..ddcf4e1be 100644 --- a/Cython/Includes/posix/resource.pxd +++ b/Cython/Includes/posix/resource.pxd @@ -1,4 +1,4 @@ -# http://pubs.opengroup.org/onlinepubs/009695399/basedefs/sys/resource.h.html +# https://pubs.opengroup.org/onlinepubs/009695399/basedefs/sys/resource.h.html from posix.time cimport timeval from posix.types cimport id_t diff --git a/Cython/Includes/posix/stdio.pxd b/Cython/Includes/posix/stdio.pxd index 53913fdf4..38b815559 100644 --- a/Cython/Includes/posix/stdio.pxd +++ b/Cython/Includes/posix/stdio.pxd @@ -1,5 +1,5 @@ # POSIX additions to <stdio.h>. -# http://pubs.opengroup.org/onlinepubs/9699919799/basedefs/stdio.h.html +# https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/stdio.h.html from libc.stdio cimport FILE from libc.stddef cimport wchar_t diff --git a/Cython/Includes/posix/stdlib.pxd b/Cython/Includes/posix/stdlib.pxd index 513de938a..188e2e501 100644 --- a/Cython/Includes/posix/stdlib.pxd +++ b/Cython/Includes/posix/stdlib.pxd @@ -1,5 +1,5 @@ # POSIX additions to <stdlib.h> -# http://pubs.opengroup.org/onlinepubs/9699919799/basedefs/stdlib.h.html +# https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/stdlib.h.html cdef extern from "<stdlib.h>" nogil: void _Exit(int) diff --git a/Cython/Includes/posix/time.pxd b/Cython/Includes/posix/time.pxd index 6bc81bfea..a90cab577 100644 --- a/Cython/Includes/posix/time.pxd +++ b/Cython/Includes/posix/time.pxd @@ -1,4 +1,4 @@ -# http://pubs.opengroup.org/onlinepubs/009695399/basedefs/sys/time.h.html +# https://pubs.opengroup.org/onlinepubs/009695399/basedefs/sys/time.h.html from posix.types cimport suseconds_t, time_t, clockid_t, timer_t from posix.signal cimport sigevent diff --git a/Cython/Includes/posix/wait.pxd b/Cython/Includes/posix/wait.pxd index d18cff9cf..7be7c49b6 100644 --- a/Cython/Includes/posix/wait.pxd +++ b/Cython/Includes/posix/wait.pxd @@ -1,4 +1,4 @@ -# http://pubs.opengroup.org/onlinepubs/009695399/basedefs/sys/wait.h.html +# https://pubs.opengroup.org/onlinepubs/009695399/basedefs/sys/wait.h.html from posix.types cimport pid_t, id_t from posix.signal cimport siginfo_t diff --git a/Cython/Plex/Actions.pxd b/Cython/Plex/Actions.pxd index 34660a2d9..cd884ced8 100644 --- a/Cython/Plex/Actions.pxd +++ b/Cython/Plex/Actions.pxd @@ -1,25 +1,26 @@ +# cython: language_level=3 cdef class Action: - cdef perform(self, token_stream, text) - cpdef same_as(self, other) + cdef perform(self, token_stream, text) cdef class Return(Action): - cdef object value - cdef perform(self, token_stream, text) - cpdef same_as(self, other) + cdef object value + cdef perform(self, token_stream, text) cdef class Call(Action): - cdef object function - cdef perform(self, token_stream, text) - cpdef same_as(self, other) + cdef object function + cdef perform(self, token_stream, text) + +cdef class Method(Action): + cdef str name + cdef dict kwargs cdef class Begin(Action): - cdef object state_name - cdef perform(self, token_stream, text) - cpdef same_as(self, other) + cdef object state_name + cdef perform(self, token_stream, text) cdef class Ignore(Action): - cdef perform(self, token_stream, text) + cdef perform(self, token_stream, text) cdef class Text(Action): - cdef perform(self, token_stream, text) + cdef perform(self, token_stream, text) diff --git a/Cython/Plex/Actions.py b/Cython/Plex/Actions.py index c88176e71..725278ddf 100644 --- a/Cython/Plex/Actions.py +++ b/Cython/Plex/Actions.py @@ -1,18 +1,20 @@ +# cython: language_level=3str # cython: auto_pickle=False -#======================================================================= -# -# Python Lexical Analyser -# -# Actions for use in token specifications -# -#======================================================================= +""" +Python Lexical Analyser + +Actions for use in token specifications +""" class Action(object): def perform(self, token_stream, text): pass # abstract - def same_as(self, other): - return self is other + def __copy__(self): + return self # immutable, no need to copy + + def __deepcopy__(self, memo): + return self # immutable, no need to copy class Return(Action): @@ -27,11 +29,8 @@ class Return(Action): def perform(self, token_stream, text): return self.value - def same_as(self, other): - return isinstance(other, Return) and self.value == other.value - def __repr__(self): - return "Return(%s)" % repr(self.value) + return "Return(%r)" % self.value class Call(Action): @@ -48,8 +47,27 @@ class Call(Action): def __repr__(self): return "Call(%s)" % self.function.__name__ - def same_as(self, other): - return isinstance(other, Call) and self.function is other.function + +class Method(Action): + """ + Plex action that calls a specific method on the token stream, + passing the matched text and any provided constant keyword arguments. + """ + + def __init__(self, name, **kwargs): + self.name = name + self.kwargs = kwargs or None + + def perform(self, token_stream, text): + method = getattr(token_stream, self.name) + # self.kwargs is almost always unused => avoid call overhead + return method(text, **self.kwargs) if self.kwargs is not None else method(text) + + def __repr__(self): + kwargs = ( + ', '.join(sorted(['%s=%r' % item for item in self.kwargs.items()])) + if self.kwargs is not None else '') + return "Method(%s%s%s)" % (self.name, ', ' if kwargs else '', kwargs) class Begin(Action): @@ -68,9 +86,6 @@ class Begin(Action): def __repr__(self): return "Begin(%s)" % self.state_name - def same_as(self, other): - return isinstance(other, Begin) and self.state_name == other.state_name - class Ignore(Action): """ @@ -87,7 +102,6 @@ class Ignore(Action): IGNORE = Ignore() -#IGNORE.__doc__ = Ignore.__doc__ class Text(Action): @@ -105,6 +119,3 @@ class Text(Action): TEXT = Text() -#TEXT.__doc__ = Text.__doc__ - - diff --git a/Cython/Plex/DFA.pxd b/Cython/Plex/DFA.pxd new file mode 100644 index 000000000..226d07567 --- /dev/null +++ b/Cython/Plex/DFA.pxd @@ -0,0 +1,30 @@ +# cython: auto_pickle=False + +cimport cython + +from . cimport Machines +from .Transitions cimport TransitionMap + + +@cython.final +cdef class StateMap: + cdef Machines.FastMachine new_machine + cdef dict old_to_new_dict + cdef dict new_to_old_dict + + cdef old_to_new(self, dict old_state_set) + + @cython.locals(state=Machines.Node) + cdef highest_priority_action(self, dict state_set) + + cdef make_key(self, dict state_set) + + +@cython.locals(new_machine=Machines.FastMachine, transitions=TransitionMap) +cpdef nfa_to_dfa(Machines.Machine old_machine, debug=*) + +cdef set_epsilon_closure(dict state_set) +cdef dict epsilon_closure(Machines.Node state) + +@cython.locals(state_set_2=dict, state2=Machines.Node) +cdef add_to_epsilon_closure(dict state_set, Machines.Node state) diff --git a/Cython/Plex/DFA.py b/Cython/Plex/DFA.py index 76324621f..66dc4a379 100644 --- a/Cython/Plex/DFA.py +++ b/Cython/Plex/DFA.py @@ -1,11 +1,9 @@ -#======================================================================= -# -# Python Lexical Analyser -# -# Converting NFA to DFA -# -#======================================================================= +# cython: auto_cpdef=True +""" +Python Lexical Analyser +Converting NFA to DFA +""" from __future__ import absolute_import from . import Machines @@ -29,12 +27,14 @@ def nfa_to_dfa(old_machine, debug=None): # is reached. new_machine = Machines.FastMachine() state_map = StateMap(new_machine) + # Seed the process using the initial states of the old machine. # Make the corresponding new states into initial states of the new # machine with the same names. for (key, old_state) in old_machine.initial_states.items(): new_state = state_map.old_to_new(epsilon_closure(old_state)) new_machine.make_initial_state(key, new_state) + # Tricky bit here: we add things to the end of this list while we're # iterating over it. The iteration stops when closure is achieved. for new_state in new_machine.states: @@ -45,6 +45,7 @@ def nfa_to_dfa(old_machine, debug=None): transitions.add_set(event, set_epsilon_closure(old_target_states)) for event, old_states in transitions.items(): new_machine.add_transitions(new_state, event, state_map.old_to_new(old_states)) + if debug: debug.write("\n===== State Mapping =====\n") state_map.dump(debug) @@ -95,14 +96,11 @@ class StateMap(object): Helper class used by nfa_to_dfa() to map back and forth between sets of states from the old machine and states of the new machine. """ - new_machine = None # Machine - old_to_new_dict = None # {(old_state,...) : new_state} - new_to_old_dict = None # {id(new_state) : old_state_set} def __init__(self, new_machine): - self.new_machine = new_machine - self.old_to_new_dict = {} - self.new_to_old_dict = {} + self.new_machine = new_machine # Machine + self.old_to_new_dict = {} # {(old_state,...) : new_state} + self.new_to_old_dict = {} # {id(new_state) : old_state_set} def old_to_new(self, old_state_set): """ @@ -119,8 +117,6 @@ class StateMap(object): new_state = self.new_machine.new_state(action) self.old_to_new_dict[key] = new_state self.new_to_old_dict[id(new_state)] = old_state_set - #for old_state in old_state_set.keys(): - #new_state.merge_actions(old_state) return new_state def highest_priority_action(self, state_set): @@ -133,13 +129,6 @@ class StateMap(object): best_priority = priority return best_action - # def old_to_new_set(self, old_state_set): - # """ - # Return the new state corresponding to a set of old states as - # a singleton set. - # """ - # return {self.old_to_new(old_state_set):1} - def new_to_old(self, new_state): """Given a new state, return a set of corresponding old states.""" return self.new_to_old_dict[id(new_state)] @@ -149,9 +138,7 @@ class StateMap(object): Convert a set of states into a uniquified sorted tuple suitable for use as a dictionary key. """ - lst = list(state_set) - lst.sort() - return tuple(lst) + return tuple(sorted(state_set)) def dump(self, file): from .Transitions import state_set_str @@ -160,5 +147,3 @@ class StateMap(object): old_state_set = self.new_to_old_dict[id(new_state)] file.write(" State %s <-- %s\n" % ( new_state['number'], state_set_str(old_state_set))) - - diff --git a/Cython/Plex/Errors.py b/Cython/Plex/Errors.py index f460100d7..fa10374f8 100644 --- a/Cython/Plex/Errors.py +++ b/Cython/Plex/Errors.py @@ -1,10 +1,8 @@ -#======================================================================= -# -# Python Lexical Analyser -# -# Exception classes -# -#======================================================================= +""" +Python Lexical Analyser + +Exception classes +""" class PlexError(Exception): @@ -19,10 +17,6 @@ class PlexValueError(PlexError, ValueError): pass -class InvalidRegex(PlexError): - pass - - class InvalidToken(PlexError): def __init__(self, token_number, message): PlexError.__init__(self, "Token number %d: %s" % (token_number, message)) diff --git a/Cython/Plex/Lexicons.py b/Cython/Plex/Lexicons.py index 787f5854b..438e44bda 100644 --- a/Cython/Plex/Lexicons.py +++ b/Cython/Plex/Lexicons.py @@ -1,15 +1,10 @@ -#======================================================================= -# -# Python Lexical Analyser -# -# Lexical Analyser Specification -# -#======================================================================= +""" +Python Lexical Analyser +Lexical Analyser Specification +""" from __future__ import absolute_import -import types - from . import Actions from . import DFA from . import Errors @@ -114,17 +109,14 @@ class Lexicon(object): machine = None # Machine tables = None # StateTableMachine - def __init__(self, specifications, debug=None, debug_flags=7, timings=None): + def __init__(self, specifications, debug=None, debug_flags=7): if not isinstance(specifications, list): raise Errors.InvalidScanner("Scanner definition is not a list") - if timings: - from .Timing import time - total_time = 0.0 - time1 = time() nfa = Machines.Machine() default_initial_state = nfa.new_initial_state('') token_number = 1 + for spec in specifications: if isinstance(spec, State): user_initial_state = nfa.new_initial_state(spec.name) @@ -140,33 +132,22 @@ class Lexicon(object): raise Errors.InvalidToken( token_number, "Expected a token definition (tuple) or State instance") - if timings: - time2 = time() - total_time = total_time + (time2 - time1) - time3 = time() + if debug and (debug_flags & 1): debug.write("\n============= NFA ===========\n") nfa.dump(debug) + dfa = DFA.nfa_to_dfa(nfa, debug=(debug_flags & 3) == 3 and debug) - if timings: - time4 = time() - total_time = total_time + (time4 - time3) + if debug and (debug_flags & 2): debug.write("\n============= DFA ===========\n") dfa.dump(debug) - if timings: - timings.write("Constructing NFA : %5.2f\n" % (time2 - time1)) - timings.write("Converting to DFA: %5.2f\n" % (time4 - time3)) - timings.write("TOTAL : %5.2f\n" % total_time) + self.machine = dfa def add_token_to_machine(self, machine, initial_state, token_spec, token_number): try: (re, action_spec) = self.parse_token_definition(token_spec) - # Disabled this -- matching empty strings can be useful - #if re.nullable: - # raise Errors.InvalidToken( - # token_number, "Pattern can match 0 input symbols") if isinstance(action_spec, Actions.Action): action = action_spec else: @@ -188,6 +169,7 @@ class Lexicon(object): raise Errors.InvalidToken("Token definition is not a tuple") if len(token_spec) != 2: raise Errors.InvalidToken("Wrong number of items in token definition") + pattern, action = token_spec if not isinstance(pattern, Regexps.RE): raise Errors.InvalidToken("Pattern is not an RE instance") @@ -195,6 +177,3 @@ class Lexicon(object): def get_initial_state(self, name): return self.machine.get_initial_state(name) - - - diff --git a/Cython/Plex/Machines.pxd b/Cython/Plex/Machines.pxd new file mode 100644 index 000000000..8f785a07c --- /dev/null +++ b/Cython/Plex/Machines.pxd @@ -0,0 +1,33 @@ +cimport cython + +from .Actions cimport Action +from .Transitions cimport TransitionMap + +cdef long maxint + + +@cython.final +cdef class Machine: + cdef readonly list states + cdef readonly dict initial_states + cdef readonly Py_ssize_t next_state_number + + cpdef new_state(self) + cpdef new_initial_state(self, name) + + +@cython.final +cdef class Node: + cdef readonly TransitionMap transitions + cdef readonly Action action + cdef public dict epsilon_closure + cdef readonly Py_ssize_t number + cdef readonly long action_priority + + +@cython.final +cdef class FastMachine: + cdef readonly dict initial_states + cdef readonly dict new_state_template + cdef readonly list states + cdef readonly Py_ssize_t next_number diff --git a/Cython/Plex/Machines.py b/Cython/Plex/Machines.py index 398850976..baf3f1918 100644 --- a/Cython/Plex/Machines.py +++ b/Cython/Plex/Machines.py @@ -1,42 +1,33 @@ -#======================================================================= -# -# Python Lexical Analyser -# -# Classes for building NFAs and DFAs -# -#======================================================================= +# cython: auto_pickle=False +""" +Python Lexical Analyser +Classes for building NFAs and DFAs +""" from __future__ import absolute_import -import sys - +import cython from .Transitions import TransitionMap -try: - from sys import maxsize as maxint -except ImportError: - from sys import maxint +maxint = 2**31-1 # sentinel value -try: - unichr -except NameError: - unichr = chr +if not cython.compiled: + try: + unichr + except NameError: + unichr = chr LOWEST_PRIORITY = -maxint class Machine(object): """A collection of Nodes representing an NFA or DFA.""" - states = None # [Node] - next_state_number = 1 - initial_states = None # {(name, bol): Node} - def __init__(self): - self.states = [] - self.initial_states = {} + self.states = [] # [Node] + self.initial_states = {} # {(name, bol): Node} + self.next_state_number = 1 def __del__(self): - #print "Destroying", self ### for state in self.states: state.destroy() @@ -72,21 +63,17 @@ class Machine(object): class Node(object): """A state of an NFA or DFA.""" - transitions = None # TransitionMap - action = None # Action - action_priority = None # integer - number = 0 # for debug output - epsilon_closure = None # used by nfa_to_dfa() def __init__(self): # Preinitialise the list of empty transitions, because # the nfa-to-dfa algorithm needs it - #self.transitions = {'':[]} - self.transitions = TransitionMap() - self.action_priority = LOWEST_PRIORITY + self.transitions = TransitionMap() # TransitionMap + self.action_priority = LOWEST_PRIORITY # integer + self.action = None # Action + self.number = 0 # for debug output + self.epsilon_closure = None # used by nfa_to_dfa() def destroy(self): - #print "Destroying", self ### self.transitions = None self.action = None self.epsilon_closure = None @@ -133,23 +120,23 @@ class Node(object): def __lt__(self, other): return self.number < other.number + def __hash__(self): + # Prevent overflowing hash values due to arbitrarily large unsigned addresses. + return id(self) & maxint + class FastMachine(object): """ FastMachine is a deterministic machine represented in a way that allows fast scanning. """ - initial_states = None # {state_name:state} - states = None # [state] where state = {event:state, 'else':state, 'action':Action} - next_number = 1 # for debugging - - new_state_template = { - '': None, 'bol': None, 'eol': None, 'eof': None, 'else': None - } - def __init__(self): - self.initial_states = {} - self.states = [] + self.initial_states = {} # {state_name:state} + self.states = [] # [state] where state = {event:state, 'else':state, 'action':Action} + self.next_number = 1 # for debugging + self.new_state_template = { + '': None, 'bol': None, 'eol': None, 'eof': None, 'else': None + } def __del__(self): for state in self.states: @@ -167,6 +154,7 @@ class FastMachine(object): def make_initial_state(self, name, state): self.initial_states[name] = state + @cython.locals(code0=cython.long, code1=cython.long, maxint=cython.long, state=dict) def add_transitions(self, state, event, new_state, maxint=maxint): if type(event) is tuple: code0, code1 = event @@ -218,9 +206,7 @@ class FastMachine(object): if char_list: ranges = self.chars_to_ranges(char_list) ranges_to_state[ranges] = state - ranges_list = ranges_to_state.keys() - ranges_list.sort() - for ranges in ranges_list: + for ranges in sorted(ranges_to_state): key = self.ranges_to_string(ranges) state = ranges_to_state[ranges] file.write(" %s --> State %d\n" % (key, state['number'])) @@ -229,6 +215,7 @@ class FastMachine(object): if state: file.write(" %s --> State %d\n" % (key, state['number'])) + @cython.locals(char_list=list, i=cython.Py_ssize_t, n=cython.Py_ssize_t, c1=cython.long, c2=cython.long) def chars_to_ranges(self, char_list): char_list.sort() i = 0 diff --git a/Cython/Plex/Regexps.py b/Cython/Plex/Regexps.py index 41816c939..99d8c994a 100644 --- a/Cython/Plex/Regexps.py +++ b/Cython/Plex/Regexps.py @@ -1,21 +1,16 @@ -#======================================================================= -# -# Python Lexical Analyser -# -# Regular Expressions -# -#======================================================================= +""" +Python Lexical Analyser +Regular Expressions +""" from __future__ import absolute_import import types -try: - from sys import maxsize as maxint -except ImportError: - from sys import maxint from . import Errors +maxint = 2**31-1 # sentinel value + # # Constants # @@ -186,37 +181,6 @@ class RE(object): # These are the basic REs from which all others are built. # -## class Char(RE): -## """ -## Char(c) is an RE which matches the character |c|. -## """ - -## nullable = 0 - -## def __init__(self, char): -## self.char = char -## self.match_nl = char == '\n' - -## def build_machine(self, m, initial_state, final_state, match_bol, nocase): -## c = self.char -## if match_bol and c != BOL: -## s1 = self.build_opt(m, initial_state, BOL) -## else: -## s1 = initial_state -## if c == '\n' or c == EOF: -## s1 = self.build_opt(m, s1, EOL) -## if len(c) == 1: -## code = ord(self.char) -## s1.add_transition((code, code+1), final_state) -## if nocase and is_letter_code(code): -## code2 = other_case_code(code) -## s1.add_transition((code2, code2+1), final_state) -## else: -## s1.add_transition(c, final_state) - -## def calc_str(self): -## return "Char(%s)" % repr(self.char) - def Char(c): """ @@ -428,6 +392,7 @@ class SwitchCase(RE): name = "Case" return "%s(%s)" % (name, self.re) + # # Composite RE constructors # ------------------------- @@ -469,7 +434,6 @@ def Any(s): """ Any(s) is an RE which matches any character in the string |s|. """ - #result = apply(Alt, tuple(map(Char, s))) result = CodeRanges(chars_to_ranges(s)) result.str = "Any(%s)" % repr(s) return result @@ -549,6 +513,7 @@ def Case(re): """ return SwitchCase(re, nocase=0) + # # RE Constants # @@ -573,4 +538,3 @@ Eof.__doc__ = \ Eof is an RE which matches the end of the file. """ Eof.str = "Eof" - diff --git a/Cython/Plex/Scanners.pxd b/Cython/Plex/Scanners.pxd index 6e75f55e6..c6cb19b40 100644 --- a/Cython/Plex/Scanners.pxd +++ b/Cython/Plex/Scanners.pxd @@ -28,13 +28,11 @@ cdef class Scanner: cdef public level - @cython.final @cython.locals(input_state=long) - cdef next_char(self) + cdef inline next_char(self) @cython.locals(action=Action) cpdef tuple read(self) - @cython.final - cdef tuple scan_a_token(self) + cdef inline tuple scan_a_token(self) ##cdef tuple position(self) # used frequently by Parsing.py @cython.final @@ -44,7 +42,5 @@ cdef class Scanner: trace=bint, discard=Py_ssize_t, data=unicode, buffer=unicode) cdef run_machine_inlined(self) - @cython.final - cdef begin(self, state) - @cython.final - cdef produce(self, value, text = *) + cdef inline begin(self, state) + cdef inline produce(self, value, text = *) diff --git a/Cython/Plex/Scanners.py b/Cython/Plex/Scanners.py index 88f7e2da3..e850e0cc9 100644 --- a/Cython/Plex/Scanners.py +++ b/Cython/Plex/Scanners.py @@ -1,18 +1,15 @@ +# cython: language_level=3str # cython: auto_pickle=False -#======================================================================= -# -# Python Lexical Analyser -# -# -# Scanning an input stream -# -#======================================================================= +""" +Python Lexical Analyser +Scanning an input stream +""" from __future__ import absolute_import import cython -cython.declare(BOL=object, EOL=object, EOF=object, NOT_FOUND=object) +cython.declare(BOL=object, EOL=object, EOF=object, NOT_FOUND=object) # noqa:E402 from . import Errors from .Regexps import BOL, EOL, EOF @@ -173,26 +170,28 @@ class Scanner(object): buf_len = len(buffer) b_action, b_cur_pos, b_cur_line, b_cur_line_start, b_cur_char, b_input_state, b_next_pos = \ None, 0, 0, 0, u'', 0, 0 + trace = self.trace while 1: - if trace: #TRACE# - print("State %d, %d/%d:%s -->" % ( #TRACE# - state['number'], input_state, cur_pos, repr(cur_char))) #TRACE# + if trace: + print("State %d, %d/%d:%s -->" % ( + state['number'], input_state, cur_pos, repr(cur_char))) + # Begin inlined self.save_for_backup() - #action = state.action #@slow - action = state['action'] #@fast + action = state['action'] if action is not None: b_action, b_cur_pos, b_cur_line, b_cur_line_start, b_cur_char, b_input_state, b_next_pos = \ action, cur_pos, cur_line, cur_line_start, cur_char, input_state, next_pos # End inlined self.save_for_backup() + c = cur_char - #new_state = state.new_state(c) #@slow - new_state = state.get(c, NOT_FOUND) #@fast - if new_state is NOT_FOUND: #@fast - new_state = c and state.get('else') #@fast + new_state = state.get(c, NOT_FOUND) + if new_state is NOT_FOUND: + new_state = c and state.get('else') + if new_state: - if trace: #TRACE# - print("State %d" % new_state['number']) #TRACE# + if trace: + print("State %d" % new_state['number']) state = new_state # Begin inlined: self.next_char() if input_state == 1: @@ -240,8 +239,8 @@ class Scanner(object): cur_char = u'' # End inlined self.next_char() else: # not new_state - if trace: #TRACE# - print("blocked") #TRACE# + if trace: + print("blocked") # Begin inlined: action = self.back_up() if b_action is not None: (action, cur_pos, cur_line, cur_line_start, @@ -252,15 +251,16 @@ class Scanner(object): action = None break # while 1 # End inlined: action = self.back_up() + self.cur_pos = cur_pos self.cur_line = cur_line self.cur_line_start = cur_line_start self.cur_char = cur_char self.input_state = input_state self.next_pos = next_pos - if trace: #TRACE# - if action is not None: #TRACE# - print("Doing %s" % action) #TRACE# + if trace: + if action is not None: + print("Doing %s" % action) return action def next_char(self): @@ -306,7 +306,8 @@ class Scanner(object): return (self.name, self.start_line, self.start_col) def get_position(self): - """Python accessible wrapper around position(), only for error reporting. + """ + Python accessible wrapper around position(), only for error reporting. """ return self.position() @@ -336,3 +337,4 @@ class Scanner(object): Override this method if you want something to be done at end of file. """ + pass diff --git a/Cython/Plex/Timing.py b/Cython/Plex/Timing.py deleted file mode 100644 index 5c3692693..000000000 --- a/Cython/Plex/Timing.py +++ /dev/null @@ -1,23 +0,0 @@ -# -# Get time in platform-dependent way -# - -from __future__ import absolute_import - -import os -from sys import platform, exit, stderr - -if platform == 'mac': - import MacOS - def time(): - return MacOS.GetTicks() / 60.0 - timekind = "real" -elif hasattr(os, 'times'): - def time(): - t = os.times() - return t[0] + t[1] - timekind = "cpu" -else: - stderr.write( - "Don't know how to get time on platform %s\n" % repr(platform)) - exit(1) diff --git a/Cython/Plex/Traditional.py b/Cython/Plex/Traditional.py deleted file mode 100644 index ec7252dae..000000000 --- a/Cython/Plex/Traditional.py +++ /dev/null @@ -1,158 +0,0 @@ -#======================================================================= -# -# Python Lexical Analyser -# -# Traditional Regular Expression Syntax -# -#======================================================================= - -from __future__ import absolute_import - -from .Regexps import Alt, Seq, Rep, Rep1, Opt, Any, AnyBut, Bol, Eol, Char -from .Errors import PlexError - - -class RegexpSyntaxError(PlexError): - pass - - -def re(s): - """ - Convert traditional string representation of regular expression |s| - into Plex representation. - """ - return REParser(s).parse_re() - - -class REParser(object): - def __init__(self, s): - self.s = s - self.i = -1 - self.end = 0 - self.next() - - def parse_re(self): - re = self.parse_alt() - if not self.end: - self.error("Unexpected %s" % repr(self.c)) - return re - - def parse_alt(self): - """Parse a set of alternative regexps.""" - re = self.parse_seq() - if self.c == '|': - re_list = [re] - while self.c == '|': - self.next() - re_list.append(self.parse_seq()) - re = Alt(*re_list) - return re - - def parse_seq(self): - """Parse a sequence of regexps.""" - re_list = [] - while not self.end and not self.c in "|)": - re_list.append(self.parse_mod()) - return Seq(*re_list) - - def parse_mod(self): - """Parse a primitive regexp followed by *, +, ? modifiers.""" - re = self.parse_prim() - while not self.end and self.c in "*+?": - if self.c == '*': - re = Rep(re) - elif self.c == '+': - re = Rep1(re) - else: # self.c == '?' - re = Opt(re) - self.next() - return re - - def parse_prim(self): - """Parse a primitive regexp.""" - c = self.get() - if c == '.': - re = AnyBut("\n") - elif c == '^': - re = Bol - elif c == '$': - re = Eol - elif c == '(': - re = self.parse_alt() - self.expect(')') - elif c == '[': - re = self.parse_charset() - self.expect(']') - else: - if c == '\\': - c = self.get() - re = Char(c) - return re - - def parse_charset(self): - """Parse a charset. Does not include the surrounding [].""" - char_list = [] - invert = 0 - if self.c == '^': - invert = 1 - self.next() - if self.c == ']': - char_list.append(']') - self.next() - while not self.end and self.c != ']': - c1 = self.get() - if self.c == '-' and self.lookahead(1) != ']': - self.next() - c2 = self.get() - for a in range(ord(c1), ord(c2) + 1): - char_list.append(chr(a)) - else: - char_list.append(c1) - chars = ''.join(char_list) - if invert: - return AnyBut(chars) - else: - return Any(chars) - - def next(self): - """Advance to the next char.""" - s = self.s - i = self.i = self.i + 1 - if i < len(s): - self.c = s[i] - else: - self.c = '' - self.end = 1 - - def get(self): - if self.end: - self.error("Premature end of string") - c = self.c - self.next() - return c - - def lookahead(self, n): - """Look ahead n chars.""" - j = self.i + n - if j < len(self.s): - return self.s[j] - else: - return '' - - def expect(self, c): - """ - Expect to find character |c| at current position. - Raises an exception otherwise. - """ - if self.c == c: - self.next() - else: - self.error("Missing %s" % repr(c)) - - def error(self, mess): - """Raise exception to signal syntax error in regexp.""" - raise RegexpSyntaxError("Syntax error in regexp %s at position %d: %s" % ( - repr(self.s), self.i, mess)) - - - diff --git a/Cython/Plex/Transitions.pxd b/Cython/Plex/Transitions.pxd new file mode 100644 index 000000000..53dd4d58e --- /dev/null +++ b/Cython/Plex/Transitions.pxd @@ -0,0 +1,22 @@ +cimport cython + +cdef long maxint + +@cython.final +cdef class TransitionMap: + cdef list map + cdef dict special + + @cython.locals(i=cython.Py_ssize_t, j=cython.Py_ssize_t) + cpdef add(self, event, new_state) + + @cython.locals(i=cython.Py_ssize_t, j=cython.Py_ssize_t) + cpdef add_set(self, event, new_set) + + @cython.locals(i=cython.Py_ssize_t, n=cython.Py_ssize_t, else_set=cython.bint) + cpdef iteritems(self) + + @cython.locals(map=list, lo=cython.Py_ssize_t, mid=cython.Py_ssize_t, hi=cython.Py_ssize_t) + cdef split(self, long code) + + cdef get_special(self, event) diff --git a/Cython/Plex/Transitions.py b/Cython/Plex/Transitions.py index 383381794..f58dd538e 100644 --- a/Cython/Plex/Transitions.py +++ b/Cython/Plex/Transitions.py @@ -1,15 +1,11 @@ -# -# Plex - Transition Maps -# -# This version represents state sets directly as dicts for speed. -# +# cython: auto_pickle=False +""" +Plex - Transition Maps -from __future__ import absolute_import +This version represents state sets directly as dicts for speed. +""" -try: - from sys import maxsize as maxint -except ImportError: - from sys import maxint +maxint = 2**31-1 # sentinel value class TransitionMap(object): @@ -40,24 +36,19 @@ class TransitionMap(object): kept separately in a dictionary. """ - map = None # The list of codes and states - special = None # Mapping for special events - def __init__(self, map=None, special=None): if not map: map = [-maxint, {}, maxint] if not special: special = {} - self.map = map - self.special = special - #self.check() ### + self.map = map # The list of codes and states + self.special = special # Mapping for special events - def add(self, event, new_state, - TupleType=tuple): + def add(self, event, new_state): """ Add transition to |new_state| on |event|. """ - if type(event) is TupleType: + if type(event) is tuple: code0, code1 = event i = self.split(code0) j = self.split(code1) @@ -68,12 +59,11 @@ class TransitionMap(object): else: self.get_special(event)[new_state] = 1 - def add_set(self, event, new_set, - TupleType=tuple): + def add_set(self, event, new_set): """ Add transitions to the states in |new_set| on |event|. """ - if type(event) is TupleType: + if type(event) is tuple: code0, code1 = event i = self.split(code0) j = self.split(code1) @@ -84,15 +74,13 @@ class TransitionMap(object): else: self.get_special(event).update(new_set) - def get_epsilon(self, - none=None): + def get_epsilon(self): """ Return the mapping for epsilon, or None. """ - return self.special.get('', none) + return self.special.get('') - def iteritems(self, - len=len): + def iteritems(self): """ Return the mapping as an iterable of ((code1, code2), state_set) and (special_event, state_set) pairs. @@ -119,8 +107,7 @@ class TransitionMap(object): # ------------------- Private methods -------------------- - def split(self, code, - len=len, maxint=maxint): + def split(self, code): """ Search the list for the position of the split point for |code|, inserting a new split point if necessary. Returns index |i| such @@ -132,6 +119,7 @@ class TransitionMap(object): # Special case: code == map[-1] if code == maxint: return hi + # General case lo = 0 # loop invariant: map[lo] <= code < map[hi] and hi - lo >= 2 @@ -147,7 +135,6 @@ class TransitionMap(object): return lo else: map[hi:hi] = [code, map[hi - 1].copy()] - #self.check() ### return hi def get_special(self, event): @@ -243,9 +230,5 @@ class TransitionMap(object): # State set manipulation functions # -#def merge_state_sets(set1, set2): -# for state in set2.keys(): -# set1[state] = 1 - def state_set_str(set): return "[%s]" % ','.join(["S%d" % state.number for state in set]) diff --git a/Cython/Plex/__init__.py b/Cython/Plex/__init__.py index 81a066f78..83bb9239a 100644 --- a/Cython/Plex/__init__.py +++ b/Cython/Plex/__init__.py @@ -1,10 +1,6 @@ -#======================================================================= -# -# Python Lexical Analyser -# -#======================================================================= - """ +Python Lexical Analyser + The Plex module provides lexical analysers with similar capabilities to GNU Flex. The following classes and functions are exported; see the attached docstrings for more information. @@ -29,10 +25,10 @@ see the attached docstrings for more information. Actions for associating with patterns when creating a Lexicon. """ - +# flake8: noqa:F401 from __future__ import absolute_import -from .Actions import TEXT, IGNORE, Begin +from .Actions import TEXT, IGNORE, Begin, Method from .Lexicons import Lexicon, State from .Regexps import RE, Seq, Alt, Rep1, Empty, Str, Any, AnyBut, AnyChar, Range from .Regexps import Opt, Rep, Bol, Eol, Eof, Case, NoCase diff --git a/Cython/Runtime/refnanny.pyx b/Cython/Runtime/refnanny.pyx index d4b873fe9..5b68923f4 100644 --- a/Cython/Runtime/refnanny.pyx +++ b/Cython/Runtime/refnanny.pyx @@ -1,6 +1,6 @@ # cython: language_level=3, auto_pickle=False -from cpython.ref cimport PyObject, Py_INCREF, Py_DECREF, Py_XDECREF, Py_XINCREF +from cpython.ref cimport PyObject, Py_INCREF, Py_CLEAR, Py_XDECREF, Py_XINCREF from cpython.exc cimport PyErr_Fetch, PyErr_Restore from cpython.pystate cimport PyThreadState_Get @@ -10,6 +10,9 @@ loglevel = 0 reflog = [] cdef log(level, action, obj, lineno): + if reflog is None: + # can happen during finalisation + return if loglevel >= level: reflog.append((lineno, action, id(obj))) @@ -29,7 +32,7 @@ cdef class Context(object): self.refs = {} # id -> (count, [lineno]) self.errors = [] - cdef regref(self, obj, lineno, bint is_null): + cdef regref(self, obj, Py_ssize_t lineno, bint is_null): log(LOG_ALL, u'regref', u"<NULL>" if is_null else obj, lineno) if is_null: self.errors.append(f"NULL argument on line {lineno}") @@ -39,7 +42,7 @@ cdef class Context(object): self.refs[id_] = (count + 1, linenumbers) linenumbers.append(lineno) - cdef bint delref(self, obj, lineno, bint is_null) except -1: + cdef bint delref(self, obj, Py_ssize_t lineno, bint is_null) except -1: # returns whether it is ok to do the decref operation log(LOG_ALL, u'delref', u"<NULL>" if is_null else obj, lineno) if is_null: @@ -50,12 +53,11 @@ cdef class Context(object): if count == 0: self.errors.append(f"Too many decrefs on line {lineno}, reference acquired on lines {linenumbers!r}") return False - elif count == 1: + if count == 1: del self.refs[id_] - return True else: self.refs[id_] = (count - 1, linenumbers) - return True + return True cdef end(self): if self.refs: @@ -63,121 +65,117 @@ cdef class Context(object): for count, linenos in self.refs.itervalues(): msg += f"\n ({count}) acquired on lines: {u', '.join([f'{x}' for x in linenos])}" self.errors.append(msg) - if self.errors: - return u"\n".join([u'REFNANNY: '+error for error in self.errors]) - else: - return None + return u"\n".join([f'REFNANNY: {error}' for error in self.errors]) if self.errors else None + -cdef void report_unraisable(object e=None): +cdef void report_unraisable(filename, Py_ssize_t lineno, object e=None): try: if e is None: import sys e = sys.exc_info()[1] - print(f"refnanny raised an exception: {e}") - except: - pass # We absolutely cannot exit with an exception + print(f"refnanny raised an exception from {filename}:{lineno}: {e}") + finally: + return # We absolutely cannot exit with an exception + # All Python operations must happen after any existing # exception has been fetched, in case we are called from # exception-handling code. -cdef PyObject* SetupContext(char* funcname, int lineno, char* filename) except NULL: +cdef PyObject* SetupContext(char* funcname, Py_ssize_t lineno, char* filename) except NULL: if Context is None: # Context may be None during finalize phase. # In that case, we don't want to be doing anything fancy # like caching and resetting exceptions. return NULL cdef (PyObject*) type = NULL, value = NULL, tb = NULL, result = NULL - PyThreadState_Get() + PyThreadState_Get() # Check that we hold the GIL PyErr_Fetch(&type, &value, &tb) try: ctx = Context(funcname, lineno, filename) Py_INCREF(ctx) result = <PyObject*>ctx except Exception, e: - report_unraisable(e) + report_unraisable(filename, lineno, e) PyErr_Restore(type, value, tb) return result -cdef void GOTREF(PyObject* ctx, PyObject* p_obj, int lineno): +cdef void GOTREF(PyObject* ctx, PyObject* p_obj, Py_ssize_t lineno): if ctx == NULL: return cdef (PyObject*) type = NULL, value = NULL, tb = NULL PyErr_Fetch(&type, &value, &tb) try: - try: - if p_obj is NULL: - (<Context>ctx).regref(None, lineno, True) - else: - (<Context>ctx).regref(<object>p_obj, lineno, False) - except: - report_unraisable() + (<Context>ctx).regref( + <object>p_obj if p_obj is not NULL else None, + lineno, + is_null=p_obj is NULL, + ) except: - # __Pyx_GetException may itself raise errors - pass - PyErr_Restore(type, value, tb) + report_unraisable((<Context>ctx).filename, lineno=(<Context>ctx).start) + finally: + PyErr_Restore(type, value, tb) + return # swallow any exceptions -cdef int GIVEREF_and_report(PyObject* ctx, PyObject* p_obj, int lineno): +cdef bint GIVEREF_and_report(PyObject* ctx, PyObject* p_obj, Py_ssize_t lineno): if ctx == NULL: return 1 cdef (PyObject*) type = NULL, value = NULL, tb = NULL cdef bint decref_ok = False PyErr_Fetch(&type, &value, &tb) try: - try: - if p_obj is NULL: - decref_ok = (<Context>ctx).delref(None, lineno, True) - else: - decref_ok = (<Context>ctx).delref(<object>p_obj, lineno, False) - except: - report_unraisable() + decref_ok = (<Context>ctx).delref( + <object>p_obj if p_obj is not NULL else None, + lineno, + is_null=p_obj is NULL, + ) except: - # __Pyx_GetException may itself raise errors - pass - PyErr_Restore(type, value, tb) - return decref_ok + report_unraisable((<Context>ctx).filename, lineno=(<Context>ctx).start) + finally: + PyErr_Restore(type, value, tb) + return decref_ok # swallow any exceptions -cdef void GIVEREF(PyObject* ctx, PyObject* p_obj, int lineno): +cdef void GIVEREF(PyObject* ctx, PyObject* p_obj, Py_ssize_t lineno): GIVEREF_and_report(ctx, p_obj, lineno) -cdef void INCREF(PyObject* ctx, PyObject* obj, int lineno): +cdef void INCREF(PyObject* ctx, PyObject* obj, Py_ssize_t lineno): Py_XINCREF(obj) - PyThreadState_Get() + PyThreadState_Get() # Check that we hold the GIL GOTREF(ctx, obj, lineno) -cdef void DECREF(PyObject* ctx, PyObject* obj, int lineno): +cdef void DECREF(PyObject* ctx, PyObject* obj, Py_ssize_t lineno): if GIVEREF_and_report(ctx, obj, lineno): Py_XDECREF(obj) - PyThreadState_Get() + PyThreadState_Get() # Check that we hold the GIL cdef void FinishContext(PyObject** ctx): if ctx == NULL or ctx[0] == NULL: return cdef (PyObject*) type = NULL, value = NULL, tb = NULL cdef object errors = None cdef Context context - PyThreadState_Get() + PyThreadState_Get() # Check that we hold the GIL PyErr_Fetch(&type, &value, &tb) try: - try: - context = <Context>ctx[0] - errors = context.end() - if errors: - print(f"{context.filename.decode('latin1')}: {context.name.decode('latin1')}()") - print(errors) - context = None - except: - report_unraisable() + context = <Context>ctx[0] + errors = context.end() + if errors: + print(f"{context.filename.decode('latin1')}: {context.name.decode('latin1')}()") + print(errors) + context = None except: - # __Pyx_GetException may itself raise errors - pass - Py_XDECREF(ctx[0]) - ctx[0] = NULL - PyErr_Restore(type, value, tb) + report_unraisable( + context.filename if context is not None else None, + lineno=context.start if context is not None else 0, + ) + finally: + Py_CLEAR(ctx[0]) + PyErr_Restore(type, value, tb) + return # swallow any exceptions ctypedef struct RefNannyAPIStruct: - void (*INCREF)(PyObject*, PyObject*, int) - void (*DECREF)(PyObject*, PyObject*, int) - void (*GOTREF)(PyObject*, PyObject*, int) - void (*GIVEREF)(PyObject*, PyObject*, int) - PyObject* (*SetupContext)(char*, int, char*) except NULL + void (*INCREF)(PyObject*, PyObject*, Py_ssize_t) + void (*DECREF)(PyObject*, PyObject*, Py_ssize_t) + void (*GOTREF)(PyObject*, PyObject*, Py_ssize_t) + void (*GIVEREF)(PyObject*, PyObject*, Py_ssize_t) + PyObject* (*SetupContext)(char*, Py_ssize_t, char*) except NULL void (*FinishContext)(PyObject**) cdef RefNannyAPIStruct api diff --git a/Cython/Shadow.py b/Cython/Shadow.py index e265b04a7..22d8a8211 100644 --- a/Cython/Shadow.py +++ b/Cython/Shadow.py @@ -1,7 +1,7 @@ # cython.* namespace for pure mode. from __future__ import absolute_import -__version__ = "0.29.24" +__version__ = "3.0.0a9" try: from __builtin__ import basestring @@ -71,7 +71,7 @@ def index_type(base_type, item): else: # int[8] etc. assert int(item) == item # array size must be a plain integer - array(base_type, item) + return array(base_type, item) # END shameless copy @@ -116,12 +116,12 @@ returns = wraparound = boundscheck = initializedcheck = nonecheck = \ exceptval = lambda _=None, check=True: _EmptyDecoratorAndManager() overflowcheck = lambda _: _EmptyDecoratorAndManager() -optimization = _Optimization() +optimize = _Optimization() -overflowcheck.fold = optimization.use_switch = \ - optimization.unpack_method_calls = lambda arg: _EmptyDecoratorAndManager() +overflowcheck.fold = optimize.use_switch = \ + optimize.unpack_method_calls = lambda arg: _EmptyDecoratorAndManager() -final = internal = type_version_tag = no_gc_clear = no_gc = _empty_decorator +final = internal = type_version_tag = no_gc_clear = no_gc = total_ordering = _empty_decorator binding = lambda _: _empty_decorator @@ -146,27 +146,33 @@ def compile(f): # Special functions def cdiv(a, b): - q = a / b - if q < 0: - q += 1 - return q + if a < 0: + a = -a + b = -b + if b < 0: + return (a + b + 1) // b + return a // b def cmod(a, b): r = a % b - if (a*b) < 0: + if (a * b) < 0 and r: r -= b return r # Emulated language constructs -def cast(type, *args, **kwargs): +def cast(t, *args, **kwargs): kwargs.pop('typecheck', None) assert not kwargs - if hasattr(type, '__call__'): - return type(*args) - else: - return args[0] + + if isinstance(t, typedef): + return t(*args) + elif isinstance(t, type): # Doesn't work with old-style classes of Python 2.x + if len(args) != 1 or not (args[0] is None or isinstance(args[0], t)): + return t(*args) + + return args[0] def sizeof(arg): return 1 @@ -178,14 +184,19 @@ def typeof(arg): def address(arg): return pointer(type(arg))([arg]) -def declare(type=None, value=_Unspecified, **kwds): - if type not in (None, object) and hasattr(type, '__call__'): - if value is not _Unspecified: - return type(value) - else: - return type() +def _is_value_type(t): + if isinstance(t, typedef): + return _is_value_type(t._basetype) + + return isinstance(t, type) and issubclass(t, (StructType, UnionType, ArrayType)) + +def declare(t=None, value=_Unspecified, **kwds): + if value is not _Unspecified: + return cast(t, value) + elif _is_value_type(t): + return t() else: - return value + return None class _nogil(object): """Support for 'with nogil' statement and @nogil decorator. @@ -258,24 +269,45 @@ class PointerType(CythonType): class ArrayType(PointerType): - def __init__(self): - self._items = [None] * self._n + def __init__(self, value=None): + if value is None: + self._items = [None] * self._n + else: + super(ArrayType, self).__init__(value) class StructType(CythonType): - def __init__(self, cast_from=_Unspecified, **data): - if cast_from is not _Unspecified: - # do cast - if len(data) > 0: - raise ValueError('Cannot accept keyword arguments when casting.') - if type(cast_from) is not type(self): - raise ValueError('Cannot cast from %s'%cast_from) - for key, value in cast_from.__dict__.items(): - setattr(self, key, value) + def __init__(self, *posargs, **data): + if not (posargs or data): + return + if posargs and data: + raise ValueError('Cannot accept both positional and keyword arguments.') + + # Allow 'cast_from' as single positional or keyword argument. + if data and len(data) == 1 and 'cast_from' in data: + cast_from = data.pop('cast_from') + elif len(posargs) == 1 and type(posargs[0]) is type(self): + cast_from, posargs = posargs[0], () + elif posargs: + for key, arg in zip(self._members, posargs): + setattr(self, key, arg) + return else: for key, value in data.items(): + if key not in self._members: + raise ValueError("Invalid struct attribute for %s: %s" % ( + self.__class__.__name__, key)) setattr(self, key, value) + return + + # do cast + if data: + raise ValueError('Cannot accept keyword arguments when casting.') + if type(cast_from) is not type(self): + raise ValueError('Cannot cast from %s' % cast_from) + for key, value in cast_from.__dict__.items(): + setattr(self, key, value) def __setattr__(self, key, value): if key in self._members: @@ -296,7 +328,7 @@ class UnionType(CythonType): elif type(cast_from) is type(self): datadict = cast_from.__dict__ else: - raise ValueError('Cannot cast from %s'%cast_from) + raise ValueError('Cannot cast from %s' % cast_from) else: datadict = data if len(datadict) > 1: @@ -393,10 +425,34 @@ py_complex = typedef(complex, "double complex") # Predefined types -int_types = ['char', 'short', 'Py_UNICODE', 'int', 'Py_UCS4', 'long', 'longlong', 'Py_ssize_t', 'size_t'] -float_types = ['longdouble', 'double', 'float'] -complex_types = ['longdoublecomplex', 'doublecomplex', 'floatcomplex', 'complex'] -other_types = ['bint', 'void', 'Py_tss_t'] +int_types = [ + 'char', + 'short', + 'Py_UNICODE', + 'int', + 'Py_UCS4', + 'long', + 'longlong', + 'Py_hash_t', + 'Py_ssize_t', + 'size_t', +] +float_types = [ + 'longdouble', + 'double', + 'float', +] +complex_types = [ + 'longdoublecomplex', + 'doublecomplex', + 'floatcomplex', + 'complex', +] +other_types = [ + 'bint', + 'void', + 'Py_tss_t', +] to_repr = { 'longlong': 'long long', @@ -469,6 +525,26 @@ class CythonDotParallel(object): # def threadsavailable(self): # return 1 -import sys + +class CythonCImports(object): + """ + Simplistic module mock to make cimports sort-of work in Python code. + """ + def __init__(self, module): + self.__path__ = [] + self.__file__ = None + self.__name__ = module + self.__package__ = module + + def __getattr__(self, item): + if item.startswith('__') and item.endswith('__'): + raise AttributeError(item) + return __import__(item) + + +import math, sys sys.modules['cython.parallel'] = CythonDotParallel() -del sys +sys.modules['cython.cimports'] = CythonCImports('cython.cimports') +sys.modules['cython.cimports.libc'] = CythonCImports('cython.cimports.libc') +sys.modules['cython.cimports.libc.math'] = math +del math, sys diff --git a/Cython/StringIOTree.py b/Cython/StringIOTree.py index d8239efed..a58fcc935 100644 --- a/Cython/StringIOTree.py +++ b/Cython/StringIOTree.py @@ -39,6 +39,7 @@ try: from cStringIO import StringIO except ImportError: from io import StringIO +import sys class StringIOTree(object): @@ -106,3 +107,44 @@ class StringIOTree(object): def allmarkers(self): children = self.prepended_children return [m for c in children for m in c.allmarkers()] + self.markers + + # Print the result of allmarkers in a nice human-readable form. Use it only for debugging. + # Prints e.g. + # /path/to/source.pyx: + # cython line 2 maps to 3299-3343 + # cython line 4 maps to 2236-2245 2306 3188-3201 + # /path/to/othersource.pyx: + # cython line 3 maps to 1234-1270 + # ... + # Note: In the example above, 3343 maps to line 2, 3344 does not. + def print_hr_allmarkers(self): + from collections import defaultdict + markers = self.allmarkers() + totmap = defaultdict(lambda: defaultdict(list)) + for c_lineno, (cython_desc, cython_lineno) in enumerate(markers): + if cython_lineno > 0 and cython_desc.filename is not None: + totmap[cython_desc.filename][cython_lineno].append(c_lineno + 1) + reprstr = "" + if totmap == 0: + reprstr += "allmarkers is empty\n" + try: + sorted(totmap.items()) + except: + print(totmap) + print(totmap.items()) + for cython_path, filemap in sorted(totmap.items()): + reprstr += cython_path + ":\n" + for cython_lineno, c_linenos in sorted(filemap.items()): + reprstr += "\tcython line " + str(cython_lineno) + " maps to " + i = 0 + while i < len(c_linenos): + reprstr += str(c_linenos[i]) + flag = False + while i+1 < len(c_linenos) and c_linenos[i+1] == c_linenos[i]+1: + i += 1 + flag = True + if flag: + reprstr += "-" + str(c_linenos[i]) + " " + i += 1 + reprstr += "\n" + sys.stdout.write(reprstr) diff --git a/Cython/Tempita/_tempita.py b/Cython/Tempita/_tempita.py index 22a7d233b..c72e76fe2 100644 --- a/Cython/Tempita/_tempita.py +++ b/Cython/Tempita/_tempita.py @@ -1,3 +1,5 @@ +# cython: language_level=3str + """ A small templating language @@ -144,9 +146,8 @@ class Template(object): def from_filename(cls, filename, namespace=None, encoding=None, default_inherit=None, get_template=get_file_template): - f = open(filename, 'rb') - c = f.read() - f.close() + with open(filename, 'rb') as f: + c = f.read() if encoding: c = c.decode(encoding) return cls(content=c, name=filename, namespace=namespace, @@ -335,7 +336,7 @@ class Template(object): if not isinstance(value, basestring_): value = coerce_text(value) if (is_unicode(value) - and self.default_encoding): + and self.default_encoding): value = value.encode(self.default_encoding) except Exception as e: e.args = (self._add_line_info(e.args[0], pos),) @@ -723,7 +724,7 @@ def trim_lex(tokens): else: next_chunk = tokens[i + 1] if (not isinstance(next_chunk, basestring_) - or not isinstance(prev, basestring_)): + or not isinstance(prev, basestring_)): continue prev_ok = not prev or trail_whitespace_re.search(prev) if i == 1 and not prev.strip(): @@ -735,7 +736,7 @@ def trim_lex(tokens): or (i == len(tokens) - 2 and not next_chunk.strip()))): if prev: if ((i == 1 and not prev.strip()) - or prev_ok == 'last'): + or prev_ok == 'last'): tokens[i - 1] = '' else: m = trail_whitespace_re.search(prev) @@ -887,7 +888,7 @@ def parse_cond(tokens, name, context): 'Missing {{endif}}', position=start, name=name) if (isinstance(tokens[0], tuple) - and tokens[0][0] == 'endif'): + and tokens[0][0] == 'endif'): return ('cond', start) + tuple(pieces), tokens[1:] next_chunk, tokens = parse_one_cond(tokens, name, context) pieces.append(next_chunk) @@ -949,7 +950,7 @@ def parse_for(tokens, name, context): 'No {{endfor}}', position=pos, name=name) if (isinstance(tokens[0], tuple) - and tokens[0][0] == 'endfor'): + and tokens[0][0] == 'endfor'): return ('for', pos, vars, expr, content), tokens[1:] next_chunk, tokens = parse_expr(tokens, name, context) content.append(next_chunk) @@ -1009,7 +1010,7 @@ def parse_def(tokens, name, context): 'Missing {{enddef}}', position=start, name=name) if (isinstance(tokens[0], tuple) - and tokens[0][0] == 'enddef'): + and tokens[0][0] == 'enddef'): return ('def', start, func_name, sig, content), tokens[1:] next_chunk, tokens = parse_expr(tokens, name, context) content.append(next_chunk) @@ -1072,7 +1073,7 @@ def parse_signature(sig_text, name, pos): raise TemplateError('Invalid signature: (%s)' % sig_text, position=pos, name=name) if (not nest_count and - (tok_type == tokenize.ENDMARKER or (tok_type == tokenize.OP and tok_string == ','))): + (tok_type == tokenize.ENDMARKER or (tok_type == tokenize.OP and tok_string == ','))): default_expr = isolate_expression(sig_text, start_pos, end_pos) defaults[var_name] = default_expr sig_args.append(var_name) @@ -1162,9 +1163,8 @@ def fill_command(args=None): template_content = sys.stdin.read() template_name = '<stdin>' else: - f = open(template_name, 'rb') - template_content = f.read() - f.close() + with open(template_name, 'rb') as f: + template_content = f.read() if options.use_html: TemplateClass = HTMLTemplate else: @@ -1172,9 +1172,8 @@ def fill_command(args=None): template = TemplateClass(template_content, name=template_name) result = template.substitute(vars) if options.output: - f = open(options.output, 'wb') - f.write(result) - f.close() + with open(options.output, 'wb') as f: + f.write(result) else: sys.stdout.write(result) diff --git a/Cython/TestUtils.py b/Cython/TestUtils.py index 9d6eb67fc..56f1dfa2e 100644 --- a/Cython/TestUtils.py +++ b/Cython/TestUtils.py @@ -2,7 +2,11 @@ from __future__ import absolute_import import os import unittest +import shlex +import sys import tempfile +import textwrap +from io import open from .Compiler import Errors from .CodeWriter import CodeWriter @@ -177,41 +181,97 @@ class TreeAssertVisitor(VisitorTransform): if TreePath.find_first(node, path) is not None: Errors.error( node.pos, - "Unexpected path '%s' found in result tree" % path) + "Unexpected path '%s' found in result tree" % path) self.visitchildren(node) return node visit_Node = VisitorTransform.recurse_to_children -def unpack_source_tree(tree_file, dir=None): - if dir is None: - dir = tempfile.mkdtemp() - header = [] - cur_file = None - f = open(tree_file) - try: - lines = f.readlines() - finally: - f.close() - del f +def unpack_source_tree(tree_file, workdir, cython_root): + programs = { + 'PYTHON': [sys.executable], + 'CYTHON': [sys.executable, os.path.join(cython_root, 'cython.py')], + 'CYTHONIZE': [sys.executable, os.path.join(cython_root, 'cythonize.py')] + } + + if workdir is None: + workdir = tempfile.mkdtemp() + header, cur_file = [], None + with open(tree_file, 'rb') as f: + try: + for line in f: + if line[:5] == b'#####': + filename = line.strip().strip(b'#').strip().decode('utf8').replace('/', os.path.sep) + path = os.path.join(workdir, filename) + if not os.path.exists(os.path.dirname(path)): + os.makedirs(os.path.dirname(path)) + if cur_file is not None: + to_close, cur_file = cur_file, None + to_close.close() + cur_file = open(path, 'wb') + elif cur_file is not None: + cur_file.write(line) + elif line.strip() and not line.lstrip().startswith(b'#'): + if line.strip() not in (b'"""', b"'''"): + command = shlex.split(line.decode('utf8')) + if not command: continue + # In Python 3: prog, *args = command + prog, args = command[0], command[1:] + try: + header.append(programs[prog]+args) + except KeyError: + header.append(command) + finally: + if cur_file is not None: + cur_file.close() + return workdir, header + + +def write_file(file_path, content, dedent=False, encoding=None): + r"""Write some content (text or bytes) to the file + at `file_path` without translating `'\n'` into `os.linesep`. + + The default encoding is `'utf-8'`. + """ + if isinstance(content, bytes): + mode = "wb" + + # binary mode doesn't take an encoding and newline arguments + newline = None + default_encoding = None + else: + mode = "w" + + # any "\n" characters written are not translated + # to the system default line separator, os.linesep + newline = "\n" + default_encoding = "utf-8" + + if encoding is None: + encoding = default_encoding + + if dedent: + content = textwrap.dedent(content) + + with open(file_path, mode=mode, encoding=encoding, newline=newline) as f: + f.write(content) + + +def write_newer_file(file_path, newer_than, content, dedent=False, encoding=None): + r""" + Write `content` to the file `file_path` without translating `'\n'` + into `os.linesep` and make sure it is newer than the file `newer_than`. + + The default encoding is `'utf-8'` (same as for `write_file`). + """ + write_file(file_path, content, dedent=dedent, encoding=encoding) + try: - for line in lines: - if line[:5] == '#####': - filename = line.strip().strip('#').strip().replace('/', os.path.sep) - path = os.path.join(dir, filename) - if not os.path.exists(os.path.dirname(path)): - os.makedirs(os.path.dirname(path)) - if cur_file is not None: - f, cur_file = cur_file, None - f.close() - cur_file = open(path, 'w') - elif cur_file is not None: - cur_file.write(line) - elif line.strip() and not line.lstrip().startswith('#'): - if line.strip() not in ('"""', "'''"): - header.append(line) - finally: - if cur_file is not None: - cur_file.close() - return dir, ''.join(header) + other_time = os.path.getmtime(newer_than) + except OSError: + # Support writing a fresh file (which is always newer than a non-existant one) + other_time = None + + while other_time is None or other_time >= os.path.getmtime(file_path): + write_file(file_path, content, dedent=dedent, encoding=encoding) diff --git a/Cython/Tests/TestCodeWriter.py b/Cython/Tests/TestCodeWriter.py index 42e457da2..c3026cb1d 100644 --- a/Cython/Tests/TestCodeWriter.py +++ b/Cython/Tests/TestCodeWriter.py @@ -19,9 +19,9 @@ class TestCodeWriter(CythonTest): def test_print(self): self.t(u""" - print x, y - print x + y ** 2 - print x, y, z, + print(x + y ** 2) + print(x, y, z) + print(x + y, x + y * z, x * (y + z)) """) def test_if(self): @@ -47,6 +47,20 @@ class TestCodeWriter(CythonTest): pass """) + def test_cdef(self): + self.t(u""" + cdef f(x, y, z): + pass + cdef public void (x = 34, y = 54, z): + pass + cdef f(int *x, void *y, Value *z): + pass + cdef f(int **x, void **y, Value **z): + pass + cdef inline f(int &x, Value &z): + pass + """) + def test_longness_and_signedness(self): self.t(u"def f(unsigned long long long long long int y):\n pass") @@ -65,18 +79,50 @@ class TestCodeWriter(CythonTest): def test_for_loop(self): self.t(u""" for x, y, z in f(g(h(34) * 2) + 23): - print x, y, z + print(x, y, z) + else: + print(43) + """) + self.t(u""" + for abc in (1, 2, 3): + print(x, y, z) else: - print 43 + print(43) + """) + + def test_while_loop(self): + self.t(u""" + while True: + while True: + while True: + continue """) def test_inplace_assignment(self): self.t(u"x += 43") + def test_cascaded_assignment(self): + self.t(u"x = y = z = abc = 43") + def test_attribute(self): self.t(u"a.x") + def test_return_none(self): + self.t(u""" + def f(x, y, z): + return + cdef f(x, y, z): + return + def f(x, y, z): + return None + cdef f(x, y, z): + return None + def f(x, y, z): + return 1234 + cdef f(x, y, z): + return 1234 + """) + if __name__ == "__main__": import unittest unittest.main() - diff --git a/Cython/Tests/TestCythonUtils.py b/Cython/Tests/TestCythonUtils.py index 2641900c0..878562512 100644 --- a/Cython/Tests/TestCythonUtils.py +++ b/Cython/Tests/TestCythonUtils.py @@ -5,7 +5,6 @@ from ..Utils import build_hex_version class TestCythonUtils(unittest.TestCase): def test_build_hex_version(self): self.assertEqual('0x001D00A1', build_hex_version('0.29a1')) - self.assertEqual('0x001D00A1', build_hex_version('0.29a1')) self.assertEqual('0x001D03C4', build_hex_version('0.29.3rc4')) self.assertEqual('0x001D00F0', build_hex_version('0.29')) self.assertEqual('0x040000F0', build_hex_version('4.0')) diff --git a/Cython/Tests/TestJediTyper.py b/Cython/Tests/TestJediTyper.py index 253adef17..ede99b3a8 100644 --- a/Cython/Tests/TestJediTyper.py +++ b/Cython/Tests/TestJediTyper.py @@ -11,7 +11,7 @@ from contextlib import contextmanager from tempfile import NamedTemporaryFile from Cython.Compiler.ParseTreeTransforms import NormalizeTree, InterpretCompilerDirectives -from Cython.Compiler import Main, Symtab, Visitor +from Cython.Compiler import Main, Symtab, Visitor, Options from Cython.TestUtils import TransformTest TOOLS_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', 'Tools')) @@ -210,8 +210,8 @@ class TestTypeInjection(TestJediTyper): """ def setUp(self): super(TestTypeInjection, self).setUp() - compilation_options = Main.CompilationOptions(Main.default_options) - ctx = compilation_options.create_context() + compilation_options = Options.CompilationOptions(Options.default_options) + ctx = Main.Context.from_options(compilation_options) transform = InterpretCompilerDirectives(ctx, ctx.compiler_directives) transform.module_scope = Symtab.ModuleScope('__main__', None, ctx) self.declarations_finder = DeclarationsFinder() diff --git a/Cython/Tests/TestTestUtils.py b/Cython/Tests/TestTestUtils.py new file mode 100644 index 000000000..140cb7c40 --- /dev/null +++ b/Cython/Tests/TestTestUtils.py @@ -0,0 +1,70 @@ +# -*- coding: utf-8 -*- + +import os.path +import unittest +import tempfile +import textwrap +import shutil + +from ..TestUtils import write_file, write_newer_file + + +class TestTestUtils(unittest.TestCase): + def setUp(self): + super(TestTestUtils, self).setUp() + self.temp_dir = tempfile.mkdtemp() + + def tearDown(self): + if self.temp_dir and os.path.isdir(self.temp_dir): + shutil.rmtree(self.temp_dir) + super(TestTestUtils, self).tearDown() + + def _test_path(self, filename): + return os.path.join(self.temp_dir, filename) + + def _test_write_file(self, content, expected, **kwargs): + file_path = self._test_path("abcfile") + write_file(file_path, content, **kwargs) + assert os.path.isfile(file_path) + + with open(file_path, 'rb') as f: + found = f.read() + assert found == expected, (repr(expected), repr(found)) + + def test_write_file_text(self): + text = u"abcüöä" + self._test_write_file(text, text.encode('utf8')) + + def test_write_file_dedent(self): + text = u""" + A horse is a horse, + of course, of course, + And no one can talk to a horse + of course + """ + self._test_write_file(text, textwrap.dedent(text).encode('utf8'), dedent=True) + + def test_write_file_bytes(self): + self._test_write_file(b"ab\0c", b"ab\0c") + + def test_write_newer_file(self): + file_path_1 = self._test_path("abcfile1.txt") + file_path_2 = self._test_path("abcfile2.txt") + write_file(file_path_1, "abc") + assert os.path.isfile(file_path_1) + write_newer_file(file_path_2, file_path_1, "xyz") + assert os.path.isfile(file_path_2) + assert os.path.getmtime(file_path_2) > os.path.getmtime(file_path_1) + + def test_write_newer_file_same(self): + file_path = self._test_path("abcfile.txt") + write_file(file_path, "abc") + mtime = os.path.getmtime(file_path) + write_newer_file(file_path, file_path, "xyz") + assert os.path.getmtime(file_path) > mtime + + def test_write_newer_file_fresh(self): + file_path = self._test_path("abcfile.txt") + assert not os.path.exists(file_path) + write_newer_file(file_path, file_path, "xyz") + assert os.path.isfile(file_path) diff --git a/Cython/Tests/xmlrunner.py b/Cython/Tests/xmlrunner.py index 665f3c241..a02d9d8c0 100644 --- a/Cython/Tests/xmlrunner.py +++ b/Cython/Tests/xmlrunner.py @@ -27,12 +27,12 @@ class TestSequenceFunctions(unittest.TestCase): def test_choice(self): element = random.choice(self.seq) - self.assert_(element in self.seq) + self.assertTrue(element in self.seq) def test_sample(self): self.assertRaises(ValueError, random.sample, self.seq, 20) for element in random.sample(self.seq, 5): - self.assert_(element in self.seq) + self.assertTrue(element in self.seq) if __name__ == '__main__': unittest.main(testRunner=xmlrunner.XMLTestRunner(output='test-reports')) @@ -109,8 +109,7 @@ class _XMLTestResult(_TextTestResult): self.elapsed_times = elapsed_times self.output_patched = False - def _prepare_callback(self, test_info, target_list, verbose_str, - short_str): + def _prepare_callback(self, test_info, target_list, verbose_str, short_str): """Append a _TestInfo to the given target list and sets a callback method to be called by stopTest method. """ @@ -125,7 +124,7 @@ class _XMLTestResult(_TextTestResult): self.start_time = self.stop_time = 0 if self.showAll: - self.stream.writeln('(%.3fs) %s' % \ + self.stream.writeln('(%.3fs) %s' % (test_info.get_elapsed_time(), verbose_str)) elif self.dots: self.stream.write(short_str) @@ -300,8 +299,7 @@ class _XMLTestResult(_TextTestResult): "Generates the XML reports to a given XMLTestRunner object." all_results = self._get_info_by_testcase() - if type(test_runner.output) == str and not \ - os.path.exists(test_runner.output): + if isinstance(test_runner.output, str) and not os.path.exists(test_runner.output): os.makedirs(test_runner.output) for suite, tests in all_results.items(): @@ -321,7 +319,7 @@ class _XMLTestResult(_TextTestResult): xml_content = doc.toprettyxml(indent='\t') if type(test_runner.output) is str: - report_file = open('%s%sTEST-%s.xml' % \ + report_file = open('%s%sTEST-%s.xml' % (test_runner.output, os.sep, suite), 'w') try: report_file.write(xml_content) @@ -348,7 +346,7 @@ class XMLTestRunner(TextTestRunner): """Create the TestResult object which will be used to store information about the executed tests. """ - return _XMLTestResult(self.stream, self.descriptions, \ + return _XMLTestResult(self.stream, self.descriptions, self.verbosity, self.elapsed_times) def run(self, test): diff --git a/Cython/Utility/AsyncGen.c b/Cython/Utility/AsyncGen.c index 4e952185b..f0e7a3dd4 100644 --- a/Cython/Utility/AsyncGen.c +++ b/Cython/Utility/AsyncGen.c @@ -11,6 +11,7 @@ typedef struct { PyObject *ag_finalizer; int ag_hooks_inited; int ag_closed; + int ag_running_async; } __pyx_PyAsyncGenObject; static PyTypeObject *__pyx__PyAsyncGenWrappedValueType = 0; @@ -18,11 +19,11 @@ static PyTypeObject *__pyx__PyAsyncGenASendType = 0; static PyTypeObject *__pyx__PyAsyncGenAThrowType = 0; static PyTypeObject *__pyx_AsyncGenType = 0; -#define __Pyx_AsyncGen_CheckExact(obj) (Py_TYPE(obj) == __pyx_AsyncGenType) +#define __Pyx_AsyncGen_CheckExact(obj) __Pyx_IS_TYPE(obj, __pyx_AsyncGenType) #define __pyx_PyAsyncGenASend_CheckExact(o) \ - (Py_TYPE(o) == __pyx__PyAsyncGenASendType) + __Pyx_IS_TYPE(o, __pyx__PyAsyncGenASendType) #define __pyx_PyAsyncGenAThrow_CheckExact(o) \ - (Py_TYPE(o) == __pyx__PyAsyncGenAThrowType) + __Pyx_IS_TYPE(o, __pyx__PyAsyncGenAThrowType) static PyObject *__Pyx_async_gen_anext(PyObject *o); static CYTHON_INLINE PyObject *__Pyx_async_gen_asend_iternext(PyObject *o); @@ -42,10 +43,11 @@ static __pyx_CoroutineObject *__Pyx_AsyncGen_New( gen->ag_finalizer = NULL; gen->ag_closed = 0; gen->ag_hooks_inited = 0; + gen->ag_running_async = 0; return __Pyx__Coroutine_NewInit((__pyx_CoroutineObject*)gen, body, code, closure, name, qualname, module_name); } -static int __pyx_AsyncGen_init(void); +static int __pyx_AsyncGen_init(PyObject *module); static void __Pyx_PyAsyncGen_Fini(void); //////////////////// AsyncGenerator.cleanup //////////////////// @@ -127,6 +129,8 @@ static PyObject *__Pyx_async_gen_athrow_new(__pyx_PyAsyncGenObject *, PyObject * static const char *__Pyx_NON_INIT_CORO_MSG = "can't send non-None value to a just-started coroutine"; static const char *__Pyx_ASYNC_GEN_IGNORED_EXIT_MSG = "async generator ignored GeneratorExit"; +static const char *__Pyx_ASYNC_GEN_CANNOT_REUSE_SEND_MSG = "cannot reuse already awaited __anext__()/asend()"; +static const char *__Pyx_ASYNC_GEN_CANNOT_REUSE_CLOSE_MSG = "cannot reuse already awaited aclose()/athrow()"; typedef enum { __PYX_AWAITABLE_STATE_INIT, /* new awaitable, has not yet been iterated */ @@ -178,7 +182,7 @@ static __pyx_PyAsyncGenASend *__Pyx_ag_asend_freelist[_PyAsyncGen_MAXFREELIST]; static int __Pyx_ag_asend_freelist_free = 0; #define __pyx__PyAsyncGenWrappedValue_CheckExact(o) \ - (Py_TYPE(o) == __pyx__PyAsyncGenWrappedValueType) + __Pyx_IS_TYPE(o, __pyx__PyAsyncGenWrappedValueType) static int @@ -253,14 +257,15 @@ static PyObject * __Pyx_async_gen_anext(PyObject *g) { __pyx_PyAsyncGenObject *o = (__pyx_PyAsyncGenObject*) g; - if (__Pyx_async_gen_init_hooks(o)) { + if (unlikely(__Pyx_async_gen_init_hooks(o))) { return NULL; } return __Pyx_async_gen_asend_new(o, NULL); } static PyObject * -__Pyx_async_gen_anext_method(PyObject *g, CYTHON_UNUSED PyObject *arg) { +__Pyx_async_gen_anext_method(PyObject *g, PyObject *arg) { + CYTHON_UNUSED_VAR(arg); return __Pyx_async_gen_anext(g); } @@ -268,7 +273,7 @@ __Pyx_async_gen_anext_method(PyObject *g, CYTHON_UNUSED PyObject *arg) { static PyObject * __Pyx_async_gen_asend(__pyx_PyAsyncGenObject *o, PyObject *arg) { - if (__Pyx_async_gen_init_hooks(o)) { + if (unlikely(__Pyx_async_gen_init_hooks(o))) { return NULL; } return __Pyx_async_gen_asend_new(o, arg); @@ -276,9 +281,10 @@ __Pyx_async_gen_asend(__pyx_PyAsyncGenObject *o, PyObject *arg) static PyObject * -__Pyx_async_gen_aclose(__pyx_PyAsyncGenObject *o, CYTHON_UNUSED PyObject *arg) +__Pyx_async_gen_aclose(__pyx_PyAsyncGenObject *o, PyObject *arg) { - if (__Pyx_async_gen_init_hooks(o)) { + CYTHON_UNUSED_VAR(arg); + if (unlikely(__Pyx_async_gen_init_hooks(o))) { return NULL; } return __Pyx_async_gen_athrow_new(o, NULL); @@ -288,7 +294,7 @@ __Pyx_async_gen_aclose(__pyx_PyAsyncGenObject *o, CYTHON_UNUSED PyObject *arg) static PyObject * __Pyx_async_gen_athrow(__pyx_PyAsyncGenObject *o, PyObject *args) { - if (__Pyx_async_gen_init_hooks(o)) { + if (unlikely(__Pyx_async_gen_init_hooks(o))) { return NULL; } return __Pyx_async_gen_athrow_new(o, args); @@ -296,7 +302,8 @@ __Pyx_async_gen_athrow(__pyx_PyAsyncGenObject *o, PyObject *args) static PyObject * -__Pyx_async_gen_self_method(PyObject *g, CYTHON_UNUSED PyObject *arg) { +__Pyx_async_gen_self_method(PyObject *g, PyObject *arg) { + CYTHON_UNUSED_VAR(arg); return __Pyx_NewRef(g); } @@ -313,11 +320,15 @@ static PyGetSetDef __Pyx_async_gen_getsetlist[] = { static PyMemberDef __Pyx_async_gen_memberlist[] = { //REMOVED: {(char*) "ag_frame", T_OBJECT, offsetof(__pyx_PyAsyncGenObject, ag_frame), READONLY}, - {(char*) "ag_running", T_BOOL, offsetof(__pyx_CoroutineObject, is_running), READONLY, NULL}, + {(char*) "ag_running", T_BOOL, offsetof(__pyx_PyAsyncGenObject, ag_running_async), READONLY, NULL}, //REMOVED: {(char*) "ag_code", T_OBJECT, offsetof(__pyx_PyAsyncGenObject, ag_code), READONLY}, //ADDED: "ag_await" {(char*) "ag_await", T_OBJECT, offsetof(__pyx_CoroutineObject, yieldfrom), READONLY, (char*) PyDoc_STR("object being awaited on, or None")}, + {(char *) "__module__", T_OBJECT, offsetof(__pyx_CoroutineObject, gi_modulename), 0, 0}, +#if CYTHON_USE_TYPE_SPECS + {(char *) "__weaklistoffset__", T_PYSSIZET, offsetof(__pyx_CoroutineObject, gi_weakreflist), READONLY, 0}, +#endif {0, 0, 0, 0, 0} /* Sentinel */ }; @@ -346,6 +357,31 @@ static PyMethodDef __Pyx_async_gen_methods[] = { }; +#if CYTHON_USE_TYPE_SPECS +static PyType_Slot __pyx_AsyncGenType_slots[] = { + {Py_tp_dealloc, (void *)__Pyx_Coroutine_dealloc}, + {Py_am_aiter, (void *)PyObject_SelfIter}, + {Py_am_anext, (void *)__Pyx_async_gen_anext}, + {Py_tp_repr, (void *)__Pyx_async_gen_repr}, + {Py_tp_traverse, (void *)__Pyx_async_gen_traverse}, + {Py_tp_methods, (void *)__Pyx_async_gen_methods}, + {Py_tp_members, (void *)__Pyx_async_gen_memberlist}, + {Py_tp_getset, (void *)__Pyx_async_gen_getsetlist}, +#if CYTHON_USE_TP_FINALIZE + {Py_tp_finalize, (void *)__Pyx_Coroutine_del}, +#endif + {0, 0}, +}; + +static PyType_Spec __pyx_AsyncGenType_spec = { + __PYX_TYPE_MODULE_PREFIX "async_generator", + sizeof(__pyx_PyAsyncGenObject), + 0, + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_HAVE_FINALIZE, /*tp_flags*/ + __pyx_AsyncGenType_slots +}; +#else /* CYTHON_USE_TYPE_SPECS */ + #if CYTHON_USE_ASYNC_SLOTS static __Pyx_PyAsyncMethodsStruct __Pyx_async_gen_as_async = { 0, /* am_await */ @@ -360,7 +396,7 @@ static PyTypeObject __pyx_AsyncGenType_type = { sizeof(__pyx_PyAsyncGenObject), /* tp_basicsize */ 0, /* tp_itemsize */ (destructor)__Pyx_Coroutine_dealloc, /* tp_dealloc */ - 0, /* tp_print */ + 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ #if CYTHON_USE_ASYNC_SLOTS @@ -427,7 +463,11 @@ static PyTypeObject __pyx_AsyncGenType_type = { #if PY_VERSION_HEX >= 0x030800b4 && PY_VERSION_HEX < 0x03090000 0, /*tp_print*/ #endif +#if CYTHON_COMPILING_IN_PYPY && PYPY_VERSION_NUM+0 >= 0x06000000 + 0, /*tp_pypy_flags*/ +#endif }; +#endif /* CYTHON_USE_TYPE_SPECS */ static int @@ -439,14 +479,14 @@ __Pyx_PyAsyncGen_ClearFreeLists(void) __pyx__PyAsyncGenWrappedValue *o; o = __Pyx_ag_value_freelist[--__Pyx_ag_value_freelist_free]; assert(__pyx__PyAsyncGenWrappedValue_CheckExact(o)); - PyObject_GC_Del(o); + __Pyx_PyHeapTypeObject_GC_Del(o); } while (__Pyx_ag_asend_freelist_free) { __pyx_PyAsyncGenASend *o; o = __Pyx_ag_asend_freelist[--__Pyx_ag_asend_freelist_free]; - assert(Py_TYPE(o) == __pyx__PyAsyncGenASendType); - PyObject_GC_Del(o); + assert(__Pyx_IS_TYPE(o, __pyx__PyAsyncGenASendType)); + __Pyx_PyHeapTypeObject_GC_Del(o); } return ret; @@ -471,6 +511,7 @@ __Pyx_async_gen_unwrap_value(__pyx_PyAsyncGenObject *gen, PyObject *result) gen->ag_closed = 1; } + gen->ag_running_async = 0; return NULL; } @@ -478,6 +519,7 @@ __Pyx_async_gen_unwrap_value(__pyx_PyAsyncGenObject *gen, PyObject *result) /* async yield */ __Pyx_ReturnWithStopIteration(((__pyx__PyAsyncGenWrappedValue*)result)->agw_val); Py_DECREF(result); + gen->ag_running_async = 0; return NULL; } @@ -494,11 +536,11 @@ __Pyx_async_gen_asend_dealloc(__pyx_PyAsyncGenASend *o) PyObject_GC_UnTrack((PyObject *)o); Py_CLEAR(o->ags_gen); Py_CLEAR(o->ags_sendval); - if (__Pyx_ag_asend_freelist_free < _PyAsyncGen_MAXFREELIST) { + if (likely(__Pyx_ag_asend_freelist_free < _PyAsyncGen_MAXFREELIST)) { assert(__pyx_PyAsyncGenASend_CheckExact(o)); __Pyx_ag_asend_freelist[__Pyx_ag_asend_freelist_free++] = o; } else { - PyObject_GC_Del(o); + __Pyx_PyHeapTypeObject_GC_Del(o); } } @@ -518,17 +560,25 @@ __Pyx_async_gen_asend_send(PyObject *g, PyObject *arg) PyObject *result; if (unlikely(o->ags_state == __PYX_AWAITABLE_STATE_CLOSED)) { - PyErr_SetNone(PyExc_StopIteration); + PyErr_SetString(PyExc_RuntimeError, __Pyx_ASYNC_GEN_CANNOT_REUSE_SEND_MSG); return NULL; } if (o->ags_state == __PYX_AWAITABLE_STATE_INIT) { + if (unlikely(o->ags_gen->ag_running_async)) { + PyErr_SetString( + PyExc_RuntimeError, + "anext(): asynchronous generator is already running"); + return NULL; + } + if (arg == NULL || arg == Py_None) { arg = o->ags_sendval ? o->ags_sendval : Py_None; } o->ags_state = __PYX_AWAITABLE_STATE_ITER; } + o->ags_gen->ag_running_async = 1; result = __Pyx_Coroutine_Send((PyObject*)o->ags_gen, arg); result = __Pyx_async_gen_unwrap_value(o->ags_gen, result); @@ -553,7 +603,7 @@ __Pyx_async_gen_asend_throw(__pyx_PyAsyncGenASend *o, PyObject *args) PyObject *result; if (unlikely(o->ags_state == __PYX_AWAITABLE_STATE_CLOSED)) { - PyErr_SetNone(PyExc_StopIteration); + PyErr_SetString(PyExc_RuntimeError, __Pyx_ASYNC_GEN_CANNOT_REUSE_SEND_MSG); return NULL; } @@ -569,9 +619,10 @@ __Pyx_async_gen_asend_throw(__pyx_PyAsyncGenASend *o, PyObject *args) static PyObject * -__Pyx_async_gen_asend_close(PyObject *g, CYTHON_UNUSED PyObject *args) +__Pyx_async_gen_asend_close(PyObject *g, PyObject *args) { __pyx_PyAsyncGenASend *o = (__pyx_PyAsyncGenASend*) g; + CYTHON_UNUSED_VAR(args); o->ags_state = __PYX_AWAITABLE_STATE_CLOSED; Py_RETURN_NONE; } @@ -586,6 +637,26 @@ static PyMethodDef __Pyx_async_gen_asend_methods[] = { }; +#if CYTHON_USE_TYPE_SPECS +static PyType_Slot __pyx__PyAsyncGenASendType_slots[] = { + {Py_tp_dealloc, (void *)__Pyx_async_gen_asend_dealloc}, + {Py_am_await, (void *)PyObject_SelfIter}, + {Py_tp_traverse, (void *)__Pyx_async_gen_asend_traverse}, + {Py_tp_methods, (void *)__Pyx_async_gen_asend_methods}, + {Py_tp_iter, (void *)PyObject_SelfIter}, + {Py_tp_iternext, (void *)__Pyx_async_gen_asend_iternext}, + {0, 0}, +}; + +static PyType_Spec __pyx__PyAsyncGenASendType_spec = { + __PYX_TYPE_MODULE_PREFIX "async_generator_asend", + sizeof(__pyx_PyAsyncGenASend), + 0, + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /*tp_flags*/ + __pyx__PyAsyncGenASendType_slots +}; +#else /* CYTHON_USE_TYPE_SPECS */ + #if CYTHON_USE_ASYNC_SLOTS static __Pyx_PyAsyncMethodsStruct __Pyx_async_gen_asend_as_async = { PyObject_SelfIter, /* am_await */ @@ -594,7 +665,6 @@ static __Pyx_PyAsyncMethodsStruct __Pyx_async_gen_asend_as_async = { }; #endif - static PyTypeObject __pyx__PyAsyncGenASendType_type = { PyVarObject_HEAD_INIT(0, 0) "async_generator_asend", /* tp_name */ @@ -602,7 +672,7 @@ static PyTypeObject __pyx__PyAsyncGenASendType_type = { 0, /* tp_itemsize */ /* methods */ (destructor)__Pyx_async_gen_asend_dealloc, /* tp_dealloc */ - 0, /* tp_print */ + 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ #if CYTHON_USE_ASYNC_SLOTS @@ -662,20 +732,24 @@ static PyTypeObject __pyx__PyAsyncGenASendType_type = { #if PY_VERSION_HEX >= 0x030800b4 && PY_VERSION_HEX < 0x03090000 0, /*tp_print*/ #endif +#if CYTHON_COMPILING_IN_PYPY && PYPY_VERSION_NUM+0 >= 0x06000000 + 0, /*tp_pypy_flags*/ +#endif }; +#endif /* CYTHON_USE_TYPE_SPECS */ static PyObject * __Pyx_async_gen_asend_new(__pyx_PyAsyncGenObject *gen, PyObject *sendval) { __pyx_PyAsyncGenASend *o; - if (__Pyx_ag_asend_freelist_free) { + if (likely(__Pyx_ag_asend_freelist_free)) { __Pyx_ag_asend_freelist_free--; o = __Pyx_ag_asend_freelist[__Pyx_ag_asend_freelist_free]; _Py_NewReference((PyObject *)o); } else { o = PyObject_GC_New(__pyx_PyAsyncGenASend, __pyx__PyAsyncGenASendType); - if (o == NULL) { + if (unlikely(o == NULL)) { return NULL; } } @@ -701,11 +775,11 @@ __Pyx_async_gen_wrapped_val_dealloc(__pyx__PyAsyncGenWrappedValue *o) { PyObject_GC_UnTrack((PyObject *)o); Py_CLEAR(o->agw_val); - if (__Pyx_ag_value_freelist_free < _PyAsyncGen_MAXFREELIST) { + if (likely(__Pyx_ag_value_freelist_free < _PyAsyncGen_MAXFREELIST)) { assert(__pyx__PyAsyncGenWrappedValue_CheckExact(o)); __Pyx_ag_value_freelist[__Pyx_ag_value_freelist_free++] = o; } else { - PyObject_GC_Del(o); + __Pyx_PyHeapTypeObject_GC_Del(o); } } @@ -719,6 +793,22 @@ __Pyx_async_gen_wrapped_val_traverse(__pyx__PyAsyncGenWrappedValue *o, } +#if CYTHON_USE_TYPE_SPECS +static PyType_Slot __pyx__PyAsyncGenWrappedValueType_slots[] = { + {Py_tp_dealloc, (void *)__Pyx_async_gen_wrapped_val_dealloc}, + {Py_tp_traverse, (void *)__Pyx_async_gen_wrapped_val_traverse}, + {0, 0}, +}; + +static PyType_Spec __pyx__PyAsyncGenWrappedValueType_spec = { + __PYX_TYPE_MODULE_PREFIX "async_generator_wrapped_value", + sizeof(__pyx__PyAsyncGenWrappedValue), + 0, + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /*tp_flags*/ + __pyx__PyAsyncGenWrappedValueType_slots +}; +#else /* CYTHON_USE_TYPE_SPECS */ + static PyTypeObject __pyx__PyAsyncGenWrappedValueType_type = { PyVarObject_HEAD_INIT(0, 0) "async_generator_wrapped_value", /* tp_name */ @@ -726,7 +816,7 @@ static PyTypeObject __pyx__PyAsyncGenWrappedValueType_type = { 0, /* tp_itemsize */ /* methods */ (destructor)__Pyx_async_gen_wrapped_val_dealloc, /* tp_dealloc */ - 0, /* tp_print */ + 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ 0, /* tp_as_async */ @@ -777,7 +867,11 @@ static PyTypeObject __pyx__PyAsyncGenWrappedValueType_type = { #if PY_VERSION_HEX >= 0x030800b4 && PY_VERSION_HEX < 0x03090000 0, /*tp_print*/ #endif +#if CYTHON_COMPILING_IN_PYPY && PYPY_VERSION_NUM+0 >= 0x06000000 + 0, /*tp_pypy_flags*/ +#endif }; +#endif /* CYTHON_USE_TYPE_SPECS */ static PyObject * @@ -787,7 +881,7 @@ __Pyx__PyAsyncGenValueWrapperNew(PyObject *val) __pyx__PyAsyncGenWrappedValue *o; assert(val); - if (__Pyx_ag_value_freelist_free) { + if (likely(__Pyx_ag_value_freelist_free)) { __Pyx_ag_value_freelist_free--; o = __Pyx_ag_value_freelist[__Pyx_ag_value_freelist_free]; assert(__pyx__PyAsyncGenWrappedValue_CheckExact(o)); @@ -815,7 +909,7 @@ __Pyx_async_gen_athrow_dealloc(__pyx_PyAsyncGenAThrow *o) PyObject_GC_UnTrack((PyObject *)o); Py_CLEAR(o->agt_gen); Py_CLEAR(o->agt_args); - PyObject_GC_Del(o); + __Pyx_PyHeapTypeObject_GC_Del(o); } @@ -832,34 +926,56 @@ static PyObject * __Pyx_async_gen_athrow_send(__pyx_PyAsyncGenAThrow *o, PyObject *arg) { __pyx_CoroutineObject *gen = (__pyx_CoroutineObject*)o->agt_gen; - PyObject *retval; + PyObject *retval, *exc_type; + + if (unlikely(o->agt_state == __PYX_AWAITABLE_STATE_CLOSED)) { + PyErr_SetString(PyExc_RuntimeError, __Pyx_ASYNC_GEN_CANNOT_REUSE_CLOSE_MSG); + return NULL; + } - if (o->agt_state == __PYX_AWAITABLE_STATE_CLOSED) { + if (unlikely(gen->resume_label == -1)) { + // already run past the end + o->agt_state = __PYX_AWAITABLE_STATE_CLOSED; PyErr_SetNone(PyExc_StopIteration); return NULL; } if (o->agt_state == __PYX_AWAITABLE_STATE_INIT) { - if (o->agt_gen->ag_closed) { - PyErr_SetNone(PyExc_StopIteration); + if (unlikely(o->agt_gen->ag_running_async)) { + o->agt_state = __PYX_AWAITABLE_STATE_CLOSED; + if (o->agt_args == NULL) { + PyErr_SetString( + PyExc_RuntimeError, + "aclose(): asynchronous generator is already running"); + } else { + PyErr_SetString( + PyExc_RuntimeError, + "athrow(): asynchronous generator is already running"); + } return NULL; } - if (arg != Py_None) { + if (unlikely(o->agt_gen->ag_closed)) { + o->agt_state = __PYX_AWAITABLE_STATE_CLOSED; + PyErr_SetNone(__Pyx_PyExc_StopAsyncIteration); + return NULL; + } + + if (unlikely(arg != Py_None)) { PyErr_SetString(PyExc_RuntimeError, __Pyx_NON_INIT_CORO_MSG); return NULL; } o->agt_state = __PYX_AWAITABLE_STATE_ITER; + o->agt_gen->ag_running_async = 1; if (o->agt_args == NULL) { /* aclose() mode */ o->agt_gen->ag_closed = 1; retval = __Pyx__Coroutine_Throw((PyObject*)gen, - /* Do not close generator when - PyExc_GeneratorExit is passed */ - PyExc_GeneratorExit, NULL, NULL, NULL, 0); + /* Do not close generator when PyExc_GeneratorExit is passed */ + PyExc_GeneratorExit, NULL, NULL, NULL, 0); if (retval && __pyx__PyAsyncGenWrappedValue_CheckExact(retval)) { Py_DECREF(retval); @@ -870,14 +986,13 @@ __Pyx_async_gen_athrow_send(__pyx_PyAsyncGenAThrow *o, PyObject *arg) PyObject *tb = NULL; PyObject *val = NULL; - if (!PyArg_UnpackTuple(o->agt_args, "athrow", 1, 3, - &typ, &val, &tb)) { + if (unlikely(!PyArg_UnpackTuple(o->agt_args, "athrow", 1, 3, &typ, &val, &tb))) { return NULL; } retval = __Pyx__Coroutine_Throw((PyObject*)gen, - /* Do not close generator when PyExc_GeneratorExit is passed */ - typ, val, tb, o->agt_args, 0); + /* Do not close generator when PyExc_GeneratorExit is passed */ + typ, val, tb, o->agt_args, 0); retval = __Pyx_async_gen_unwrap_value(o->agt_gen, retval); } if (retval == NULL) { @@ -894,7 +1009,7 @@ __Pyx_async_gen_athrow_send(__pyx_PyAsyncGenAThrow *o, PyObject *arg) } else { /* aclose() mode */ if (retval) { - if (__pyx__PyAsyncGenWrappedValue_CheckExact(retval)) { + if (unlikely(__pyx__PyAsyncGenWrappedValue_CheckExact(retval))) { Py_DECREF(retval); goto yield_close; } @@ -908,26 +1023,26 @@ __Pyx_async_gen_athrow_send(__pyx_PyAsyncGenAThrow *o, PyObject *arg) } yield_close: + o->agt_gen->ag_running_async = 0; + o->agt_state = __PYX_AWAITABLE_STATE_CLOSED; PyErr_SetString( PyExc_RuntimeError, __Pyx_ASYNC_GEN_IGNORED_EXIT_MSG); return NULL; check_error: - if (PyErr_ExceptionMatches(__Pyx_PyExc_StopAsyncIteration)) { - o->agt_state = __PYX_AWAITABLE_STATE_CLOSED; + o->agt_gen->ag_running_async = 0; + o->agt_state = __PYX_AWAITABLE_STATE_CLOSED; + exc_type = PyErr_Occurred(); + if (__Pyx_PyErr_GivenExceptionMatches2(exc_type, __Pyx_PyExc_StopAsyncIteration, PyExc_GeneratorExit)) { if (o->agt_args == NULL) { // when aclose() is called we don't want to propagate - // StopAsyncIteration; just raise StopIteration, signalling - // that 'aclose()' is done. + // StopAsyncIteration or GeneratorExit; just raise + // StopIteration, signalling that this 'aclose()' await + // is done. PyErr_Clear(); PyErr_SetNone(PyExc_StopIteration); } } - else if (PyErr_ExceptionMatches(PyExc_GeneratorExit)) { - o->agt_state = __PYX_AWAITABLE_STATE_CLOSED; - PyErr_Clear(); /* ignore these errors */ - PyErr_SetNone(PyExc_StopIteration); - } return NULL; } @@ -937,13 +1052,8 @@ __Pyx_async_gen_athrow_throw(__pyx_PyAsyncGenAThrow *o, PyObject *args) { PyObject *retval; - if (o->agt_state == __PYX_AWAITABLE_STATE_INIT) { - PyErr_SetString(PyExc_RuntimeError, __Pyx_NON_INIT_CORO_MSG); - return NULL; - } - - if (o->agt_state == __PYX_AWAITABLE_STATE_CLOSED) { - PyErr_SetNone(PyExc_StopIteration); + if (unlikely(o->agt_state == __PYX_AWAITABLE_STATE_CLOSED)) { + PyErr_SetString(PyExc_RuntimeError, __Pyx_ASYNC_GEN_CANNOT_REUSE_CLOSE_MSG); return NULL; } @@ -951,12 +1061,24 @@ __Pyx_async_gen_athrow_throw(__pyx_PyAsyncGenAThrow *o, PyObject *args) if (o->agt_args) { return __Pyx_async_gen_unwrap_value(o->agt_gen, retval); } else { - /* aclose() mode */ - if (retval && __pyx__PyAsyncGenWrappedValue_CheckExact(retval)) { + // aclose() mode + PyObject *exc_type; + if (unlikely(retval && __pyx__PyAsyncGenWrappedValue_CheckExact(retval))) { + o->agt_gen->ag_running_async = 0; + o->agt_state = __PYX_AWAITABLE_STATE_CLOSED; Py_DECREF(retval); PyErr_SetString(PyExc_RuntimeError, __Pyx_ASYNC_GEN_IGNORED_EXIT_MSG); return NULL; } + exc_type = PyErr_Occurred(); + if (__Pyx_PyErr_GivenExceptionMatches2(exc_type, __Pyx_PyExc_StopAsyncIteration, PyExc_GeneratorExit)) { + // when aclose() is called we don't want to propagate + // StopAsyncIteration or GeneratorExit; just raise + // StopIteration, signalling that this 'aclose()' await + // is done. + PyErr_Clear(); + PyErr_SetNone(PyExc_StopIteration); + } return retval; } } @@ -970,9 +1092,10 @@ __Pyx_async_gen_athrow_iternext(__pyx_PyAsyncGenAThrow *o) static PyObject * -__Pyx_async_gen_athrow_close(PyObject *g, CYTHON_UNUSED PyObject *args) +__Pyx_async_gen_athrow_close(PyObject *g, PyObject *args) { __pyx_PyAsyncGenAThrow *o = (__pyx_PyAsyncGenAThrow*) g; + CYTHON_UNUSED_VAR(args); o->agt_state = __PYX_AWAITABLE_STATE_CLOSED; Py_RETURN_NONE; } @@ -987,6 +1110,27 @@ static PyMethodDef __Pyx_async_gen_athrow_methods[] = { }; +#if CYTHON_USE_TYPE_SPECS +static PyType_Slot __pyx__PyAsyncGenAThrowType_slots[] = { + {Py_tp_dealloc, (void *)__Pyx_async_gen_athrow_dealloc}, + {Py_am_await, (void *)PyObject_SelfIter}, + {Py_tp_traverse, (void *)__Pyx_async_gen_athrow_traverse}, + {Py_tp_iter, (void *)PyObject_SelfIter}, + {Py_tp_iternext, (void *)__Pyx_async_gen_athrow_iternext}, + {Py_tp_methods, (void *)__Pyx_async_gen_athrow_methods}, + {Py_tp_getattro, (void *)__Pyx_PyObject_GenericGetAttrNoDict}, + {0, 0}, +}; + +static PyType_Spec __pyx__PyAsyncGenAThrowType_spec = { + __PYX_TYPE_MODULE_PREFIX "async_generator_athrow", + sizeof(__pyx_PyAsyncGenAThrow), + 0, + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /*tp_flags*/ + __pyx__PyAsyncGenAThrowType_slots +}; +#else /* CYTHON_USE_TYPE_SPECS */ + #if CYTHON_USE_ASYNC_SLOTS static __Pyx_PyAsyncMethodsStruct __Pyx_async_gen_athrow_as_async = { PyObject_SelfIter, /* am_await */ @@ -995,14 +1139,13 @@ static __Pyx_PyAsyncMethodsStruct __Pyx_async_gen_athrow_as_async = { }; #endif - static PyTypeObject __pyx__PyAsyncGenAThrowType_type = { PyVarObject_HEAD_INIT(0, 0) "async_generator_athrow", /* tp_name */ sizeof(__pyx_PyAsyncGenAThrow), /* tp_basicsize */ 0, /* tp_itemsize */ (destructor)__Pyx_async_gen_athrow_dealloc, /* tp_dealloc */ - 0, /* tp_print */ + 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ #if CYTHON_USE_ASYNC_SLOTS @@ -1062,7 +1205,11 @@ static PyTypeObject __pyx__PyAsyncGenAThrowType_type = { #if PY_VERSION_HEX >= 0x030800b4 && PY_VERSION_HEX < 0x03090000 0, /*tp_print*/ #endif +#if CYTHON_COMPILING_IN_PYPY && PYPY_VERSION_NUM+0 >= 0x06000000 + 0, /*tp_pypy_flags*/ +#endif }; +#endif /* CYTHON_USE_TYPE_SPECS */ static PyObject * @@ -1070,7 +1217,7 @@ __Pyx_async_gen_athrow_new(__pyx_PyAsyncGenObject *gen, PyObject *args) { __pyx_PyAsyncGenAThrow *o; o = PyObject_GC_New(__pyx_PyAsyncGenAThrow, __pyx__PyAsyncGenAThrowType); - if (o == NULL) { + if (unlikely(o == NULL)) { return NULL; } o->agt_gen = gen; @@ -1085,26 +1232,42 @@ __Pyx_async_gen_athrow_new(__pyx_PyAsyncGenObject *gen, PyObject *args) /* ---------- global type sharing ------------ */ -static int __pyx_AsyncGen_init(void) { +static int __pyx_AsyncGen_init(PyObject *module) { +#if CYTHON_USE_TYPE_SPECS + __pyx_AsyncGenType = __Pyx_FetchCommonTypeFromSpec(module, &__pyx_AsyncGenType_spec, NULL); +#else + (void) module; // on Windows, C-API functions can't be used in slots statically __pyx_AsyncGenType_type.tp_getattro = __Pyx_PyObject_GenericGetAttrNoDict; - __pyx__PyAsyncGenWrappedValueType_type.tp_getattro = __Pyx_PyObject_GenericGetAttrNoDict; - __pyx__PyAsyncGenAThrowType_type.tp_getattro = __Pyx_PyObject_GenericGetAttrNoDict; - __pyx__PyAsyncGenASendType_type.tp_getattro = __Pyx_PyObject_GenericGetAttrNoDict; - __pyx_AsyncGenType = __Pyx_FetchCommonType(&__pyx_AsyncGenType_type); +#endif if (unlikely(!__pyx_AsyncGenType)) return -1; +#if CYTHON_USE_TYPE_SPECS + __pyx__PyAsyncGenAThrowType = __Pyx_FetchCommonTypeFromSpec(module, &__pyx__PyAsyncGenAThrowType_spec, NULL); +#else + __pyx__PyAsyncGenAThrowType_type.tp_getattro = __Pyx_PyObject_GenericGetAttrNoDict; __pyx__PyAsyncGenAThrowType = __Pyx_FetchCommonType(&__pyx__PyAsyncGenAThrowType_type); +#endif if (unlikely(!__pyx__PyAsyncGenAThrowType)) return -1; +#if CYTHON_USE_TYPE_SPECS + __pyx__PyAsyncGenWrappedValueType = __Pyx_FetchCommonTypeFromSpec(module, &__pyx__PyAsyncGenWrappedValueType_spec, NULL); +#else + __pyx__PyAsyncGenWrappedValueType_type.tp_getattro = __Pyx_PyObject_GenericGetAttrNoDict; __pyx__PyAsyncGenWrappedValueType = __Pyx_FetchCommonType(&__pyx__PyAsyncGenWrappedValueType_type); +#endif if (unlikely(!__pyx__PyAsyncGenWrappedValueType)) return -1; +#if CYTHON_USE_TYPE_SPECS + __pyx__PyAsyncGenASendType = __Pyx_FetchCommonTypeFromSpec(module, &__pyx__PyAsyncGenASendType_spec, NULL); +#else + __pyx__PyAsyncGenASendType_type.tp_getattro = __Pyx_PyObject_GenericGetAttrNoDict; __pyx__PyAsyncGenASendType = __Pyx_FetchCommonType(&__pyx__PyAsyncGenASendType_type); +#endif if (unlikely(!__pyx__PyAsyncGenASendType)) return -1; diff --git a/Cython/Utility/Buffer.c b/Cython/Utility/Buffer.c index 3c7105fa3..3ea9a1b90 100644 --- a/Cython/Utility/Buffer.c +++ b/Cython/Utility/Buffer.c @@ -111,6 +111,7 @@ typedef struct { #if PY_MAJOR_VERSION < 3 static int __Pyx_GetBuffer(PyObject *obj, Py_buffer *view, int flags) { + __Pyx_TypeName obj_type_name; if (PyObject_CheckBuffer(obj)) return PyObject_GetBuffer(obj, view, flags); {{for type_ptr, getbuffer, releasebuffer in types}} @@ -119,7 +120,11 @@ static int __Pyx_GetBuffer(PyObject *obj, Py_buffer *view, int flags) { {{endif}} {{endfor}} - PyErr_Format(PyExc_TypeError, "'%.200s' does not have the buffer interface", Py_TYPE(obj)->tp_name); + obj_type_name = __Pyx_PyType_GetName(Py_TYPE(obj)); + PyErr_Format(PyExc_TypeError, + "'" __Pyx_FMT_TYPENAME "' does not have the buffer interface", + obj_type_name); + __Pyx_DECREF_TypeName(obj_type_name); return -1; } @@ -224,8 +229,8 @@ fail:; // the format string; the access mode/flags is checked by the // exporter. See: // -// http://docs.python.org/3/library/struct.html -// http://legacy.python.org/dev/peps/pep-3118/#additions-to-the-struct-string-syntax +// https://docs.python.org/3/library/struct.html +// https://www.python.org/dev/peps/pep-3118/#additions-to-the-struct-string-syntax // // The alignment code is copied from _struct.c in Python. @@ -372,7 +377,8 @@ typedef struct { char c; void *x; } __Pyx_st_void_p; typedef struct { char c; PY_LONG_LONG x; } __Pyx_st_longlong; #endif -static size_t __Pyx_BufFmt_TypeCharToAlignment(char ch, CYTHON_UNUSED int is_complex) { +static size_t __Pyx_BufFmt_TypeCharToAlignment(char ch, int is_complex) { + CYTHON_UNUSED_VAR(is_complex); switch (ch) { case '?': case 'c': case 'b': case 'B': case 's': case 'p': return 1; case 'h': case 'H': return sizeof(__Pyx_st_short) - sizeof(short); @@ -406,7 +412,8 @@ typedef struct { void *x; char c; } __Pyx_pad_void_p; typedef struct { PY_LONG_LONG x; char c; } __Pyx_pad_longlong; #endif -static size_t __Pyx_BufFmt_TypeCharToPadding(char ch, CYTHON_UNUSED int is_complex) { +static size_t __Pyx_BufFmt_TypeCharToPadding(char ch, int is_complex) { + CYTHON_UNUSED_VAR(is_complex); switch (ch) { case '?': case 'c': case 'b': case 'B': case 's': case 'p': return 1; case 'h': case 'H': return sizeof(__Pyx_pad_short) - sizeof(short); diff --git a/Cython/Utility/Builtins.c b/Cython/Utility/Builtins.c index 1ffb3bceb..30b7145a4 100644 --- a/Cython/Utility/Builtins.c +++ b/Cython/Utility/Builtins.c @@ -20,47 +20,7 @@ static PyObject* __Pyx_Globals(void); /*proto*/ // access requires a rewrite as a dedicated class. static PyObject* __Pyx_Globals(void) { - Py_ssize_t i; - PyObject *names; - PyObject *globals = $moddict_cname; - Py_INCREF(globals); - names = PyObject_Dir($module_cname); - if (!names) - goto bad; - for (i = PyList_GET_SIZE(names)-1; i >= 0; i--) { -#if CYTHON_COMPILING_IN_PYPY - PyObject* name = PySequence_ITEM(names, i); - if (!name) - goto bad; -#else - PyObject* name = PyList_GET_ITEM(names, i); -#endif - if (!PyDict_Contains(globals, name)) { - PyObject* value = __Pyx_GetAttr($module_cname, name); - if (!value) { -#if CYTHON_COMPILING_IN_PYPY - Py_DECREF(name); -#endif - goto bad; - } - if (PyDict_SetItem(globals, name, value) < 0) { -#if CYTHON_COMPILING_IN_PYPY - Py_DECREF(name); -#endif - Py_DECREF(value); - goto bad; - } - } -#if CYTHON_COMPILING_IN_PYPY - Py_DECREF(name); -#endif - } - Py_DECREF(names); - return globals; -bad: - Py_XDECREF(names); - Py_XDECREF(globals); - return NULL; + return __Pyx_NewRef($moddict_cname); } //////////////////// PyExecGlobals.proto //////////////////// @@ -68,17 +28,11 @@ bad: static PyObject* __Pyx_PyExecGlobals(PyObject*); //////////////////// PyExecGlobals //////////////////// -//@requires: Globals +//@substitute: naming //@requires: PyExec static PyObject* __Pyx_PyExecGlobals(PyObject* code) { - PyObject* result; - PyObject* globals = __Pyx_Globals(); - if (unlikely(!globals)) - return NULL; - result = __Pyx_PyExec2(code, globals); - Py_DECREF(globals); - return result; + return __Pyx_PyExec2(code, $moddict_cname); } //////////////////// PyExec.proto //////////////////// @@ -100,9 +54,13 @@ static PyObject* __Pyx_PyExec3(PyObject* o, PyObject* globals, PyObject* locals) if (!globals || globals == Py_None) { globals = $moddict_cname; - } else if (!PyDict_Check(globals)) { - PyErr_Format(PyExc_TypeError, "exec() arg 2 must be a dict, not %.200s", - Py_TYPE(globals)->tp_name); + } else if (unlikely(!PyDict_Check(globals))) { + __Pyx_TypeName globals_type_name = + __Pyx_PyType_GetName(Py_TYPE(globals)); + PyErr_Format(PyExc_TypeError, + "exec() arg 2 must be a dict, not " __Pyx_FMT_TYPENAME, + globals_type_name); + __Pyx_DECREF_TypeName(globals_type_name); goto bad; } if (!locals || locals == Py_None) { @@ -110,17 +68,17 @@ static PyObject* __Pyx_PyExec3(PyObject* o, PyObject* globals, PyObject* locals) } if (__Pyx_PyDict_GetItemStr(globals, PYIDENT("__builtins__")) == NULL) { - if (PyDict_SetItem(globals, PYIDENT("__builtins__"), PyEval_GetBuiltins()) < 0) + if (unlikely(PyDict_SetItem(globals, PYIDENT("__builtins__"), PyEval_GetBuiltins()) < 0)) goto bad; } if (PyCode_Check(o)) { - if (__Pyx_PyCode_HasFreeVars((PyCodeObject *)o)) { + if (unlikely(__Pyx_PyCode_HasFreeVars((PyCodeObject *)o))) { PyErr_SetString(PyExc_TypeError, "code object passed to exec() may not contain free variables"); goto bad; } - #if CYTHON_COMPILING_IN_PYPY || PY_VERSION_HEX < 0x030200B1 + #if PY_VERSION_HEX < 0x030200B1 || (CYTHON_COMPILING_IN_PYPY && PYPY_VERSION_NUM < 0x07030400) result = PyEval_EvalCode((PyCodeObject *)o, globals, locals); #else result = PyEval_EvalCode(o, globals, locals); @@ -134,16 +92,18 @@ static PyObject* __Pyx_PyExec3(PyObject* o, PyObject* globals, PyObject* locals) if (PyUnicode_Check(o)) { cf.cf_flags = PyCF_SOURCE_IS_UTF8; s = PyUnicode_AsUTF8String(o); - if (!s) goto bad; + if (unlikely(!s)) goto bad; o = s; #if PY_MAJOR_VERSION >= 3 - } else if (!PyBytes_Check(o)) { + } else if (unlikely(!PyBytes_Check(o))) { #else - } else if (!PyString_Check(o)) { + } else if (unlikely(!PyString_Check(o))) { #endif + __Pyx_TypeName o_type_name = __Pyx_PyType_GetName(Py_TYPE(o)); PyErr_Format(PyExc_TypeError, - "exec: arg 1 must be string, bytes or code object, got %.200s", - Py_TYPE(o)->tp_name); + "exec: arg 1 must be string, bytes or code object, got " __Pyx_FMT_TYPENAME, + o_type_name); + __Pyx_DECREF_TypeName(o_type_name); goto bad; } #if PY_MAJOR_VERSION >= 3 @@ -170,7 +130,7 @@ bad: static CYTHON_INLINE PyObject *__Pyx_GetAttr3(PyObject *, PyObject *, PyObject *); /*proto*/ //////////////////// GetAttr3 //////////////////// -//@requires: ObjectHandling.c::GetAttr +//@requires: ObjectHandling.c::PyObjectGetAttrStr //@requires: Exceptions.c::PyThreadStateGet //@requires: Exceptions.c::PyErrFetchRestore //@requires: Exceptions.c::PyErrExceptionMatches @@ -186,7 +146,17 @@ static PyObject *__Pyx_GetAttr3Default(PyObject *d) { } static CYTHON_INLINE PyObject *__Pyx_GetAttr3(PyObject *o, PyObject *n, PyObject *d) { - PyObject *r = __Pyx_GetAttr(o, n); + PyObject *r; +#if CYTHON_USE_TYPE_SLOTS + if (likely(PyString_Check(n))) { + r = __Pyx_PyObject_GetAttrStrNoError(o, n); + if (unlikely(!r) && likely(!PyErr_Occurred())) { + r = __Pyx_NewRef(d); + } + return r; + } +#endif + r = PyObject_GetAttr(o, n); return (likely(r)) ? r : __Pyx_GetAttr3Default(d); } @@ -205,7 +175,7 @@ static CYTHON_INLINE int __Pyx_HasAttr(PyObject *o, PyObject *n) { return -1; } r = __Pyx_GetAttr(o, n); - if (unlikely(!r)) { + if (!r) { PyErr_Clear(); return 0; } else { @@ -219,11 +189,12 @@ static CYTHON_INLINE int __Pyx_HasAttr(PyObject *o, PyObject *n) { static PyObject* __Pyx_Intern(PyObject* s); /* proto */ //////////////////// Intern //////////////////// +//@requires: ObjectHandling.c::RaiseUnexpectedTypeError static PyObject* __Pyx_Intern(PyObject* s) { - if (!(likely(PyString_CheckExact(s)))) { - PyErr_Format(PyExc_TypeError, "Expected %.16s, got %.200s", "str", Py_TYPE(s)->tp_name); - return 0; + if (unlikely(!PyString_CheckExact(s))) { + __Pyx_RaiseUnexpectedTypeError("str", s); + return NULL; } Py_INCREF(s); #if PY_MAJOR_VERSION >= 3 @@ -332,8 +303,11 @@ static long __Pyx__PyObject_Ord(PyObject* c) { #endif } else { // FIXME: support character buffers - but CPython doesn't support them either + __Pyx_TypeName c_type_name = __Pyx_PyType_GetName(Py_TYPE(c)); PyErr_Format(PyExc_TypeError, - "ord() expected string of length 1, but %.200s found", Py_TYPE(c)->tp_name); + "ord() expected string of length 1, but " __Pyx_FMT_TYPENAME " found", + c_type_name); + __Pyx_DECREF_TypeName(c_type_name); return (long)(Py_UCS4)-1; } PyErr_Format(PyExc_TypeError, @@ -422,9 +396,6 @@ static CYTHON_INLINE PyObject* __Pyx_PyDict_IterItems(PyObject* d) { //////////////////// py_dict_viewkeys.proto //////////////////// -#if PY_VERSION_HEX < 0x02070000 -#error This module uses dict views, which require Python 2.7 or later -#endif static CYTHON_INLINE PyObject* __Pyx_PyDict_ViewKeys(PyObject* d); /*proto*/ //////////////////// py_dict_viewkeys //////////////////// @@ -438,9 +409,6 @@ static CYTHON_INLINE PyObject* __Pyx_PyDict_ViewKeys(PyObject* d) { //////////////////// py_dict_viewvalues.proto //////////////////// -#if PY_VERSION_HEX < 0x02070000 -#error This module uses dict views, which require Python 2.7 or later -#endif static CYTHON_INLINE PyObject* __Pyx_PyDict_ViewValues(PyObject* d); /*proto*/ //////////////////// py_dict_viewvalues //////////////////// @@ -454,9 +422,6 @@ static CYTHON_INLINE PyObject* __Pyx_PyDict_ViewValues(PyObject* d) { //////////////////// py_dict_viewitems.proto //////////////////// -#if PY_VERSION_HEX < 0x02070000 -#error This module uses dict views, which require Python 2.7 or later -#endif static CYTHON_INLINE PyObject* __Pyx_PyDict_ViewItems(PyObject* d); /*proto*/ //////////////////// py_dict_viewitems //////////////////// diff --git a/Cython/Utility/CConvert.pyx b/Cython/Utility/CConvert.pyx index 5969f6a58..4ae66162e 100644 --- a/Cython/Utility/CConvert.pyx +++ b/Cython/Utility/CConvert.pyx @@ -6,19 +6,20 @@ cdef extern from *: PyTypeObject *Py_TYPE(obj) bint PyMapping_Check(obj) object PyErr_Format(exc, const char *format, ...) + int __Pyx_RaiseUnexpectedTypeError(const char *expected, object obj) except 0 @cname("{{funcname}}") cdef {{struct_type}} {{funcname}}(obj) except *: cdef {{struct_type}} result if not PyMapping_Check(obj): - PyErr_Format(TypeError, b"Expected %.16s, got %.200s", b"a mapping", Py_TYPE(obj).tp_name) + __Pyx_RaiseUnexpectedTypeError(b"a mapping", obj) {{for member in var_entries:}} try: value = obj['{{member.name}}'] except KeyError: raise ValueError("No value specified for struct attribute '{{member.name}}'") - result.{{member.cname}} = value + result.{{member.name}} = value {{endfor}} return result @@ -31,13 +32,14 @@ cdef extern from *: PyTypeObject *Py_TYPE(obj) bint PyMapping_Check(obj) object PyErr_Format(exc, const char *format, ...) + int __Pyx_RaiseUnexpectedTypeError(const char *expected, object obj) except 0 @cname("{{funcname}}") cdef {{struct_type}} {{funcname}}(obj) except *: cdef {{struct_type}} result cdef Py_ssize_t length if not PyMapping_Check(obj): - PyErr_Format(TypeError, b"Expected %.16s, got %.200s", b"a mapping", Py_TYPE(obj).tp_name) + __Pyx_RaiseUnexpectedTypeError(b"a mapping", obj) last_found = None length = len(obj) diff --git a/Cython/Utility/Capsule.c b/Cython/Utility/Capsule.c index cc4fe0d88..14c744cdf 100644 --- a/Cython/Utility/Capsule.c +++ b/Cython/Utility/Capsule.c @@ -6,15 +6,7 @@ static CYTHON_INLINE PyObject *__pyx_capsule_create(void *p, const char *sig); //////////////// Capsule //////////////// static CYTHON_INLINE PyObject * -__pyx_capsule_create(void *p, CYTHON_UNUSED const char *sig) +__pyx_capsule_create(void *p, const char *sig) { - PyObject *cobj; - -#if PY_VERSION_HEX >= 0x02070000 - cobj = PyCapsule_New(p, sig, NULL); -#else - cobj = PyCObject_FromVoidPtr(p, NULL); -#endif - - return cobj; + return PyCapsule_New(p, sig, NULL); } diff --git a/Cython/Utility/CommonStructures.c b/Cython/Utility/CommonStructures.c index c7945feb4..80055a330 100644 --- a/Cython/Utility/CommonStructures.c +++ b/Cython/Utility/CommonStructures.c @@ -1,51 +1,129 @@ /////////////// FetchCommonType.proto /////////////// +#if !CYTHON_USE_TYPE_SPECS static PyTypeObject* __Pyx_FetchCommonType(PyTypeObject* type); +#else +static PyTypeObject* __Pyx_FetchCommonTypeFromSpec(PyObject *module, PyType_Spec *spec, PyObject *bases); +#endif /////////////// FetchCommonType /////////////// +//@requires:ExtensionTypes.c::FixUpExtensionType -static PyTypeObject* __Pyx_FetchCommonType(PyTypeObject* type) { - PyObject* fake_module; - PyTypeObject* cached_type = NULL; +static PyObject *__Pyx_FetchSharedCythonABIModule(void) { + PyObject *abi_module = PyImport_AddModule((char*) __PYX_ABI_MODULE_NAME); + if (!abi_module) return NULL; + Py_INCREF(abi_module); + return abi_module; +} - fake_module = PyImport_AddModule((char*) "_cython_" CYTHON_ABI); - if (!fake_module) return NULL; - Py_INCREF(fake_module); +static int __Pyx_VerifyCachedType(PyObject *cached_type, + const char *name, + Py_ssize_t basicsize, + Py_ssize_t expected_basicsize) { + if (!PyType_Check(cached_type)) { + PyErr_Format(PyExc_TypeError, + "Shared Cython type %.200s is not a type object", name); + return -1; + } + if (basicsize != expected_basicsize) { + PyErr_Format(PyExc_TypeError, + "Shared Cython type %.200s has the wrong size, try recompiling", + name); + return -1; + } + return 0; +} + +#if !CYTHON_USE_TYPE_SPECS +static PyTypeObject* __Pyx_FetchCommonType(PyTypeObject* type) { + PyObject* abi_module; + PyTypeObject *cached_type = NULL; - cached_type = (PyTypeObject*) PyObject_GetAttrString(fake_module, type->tp_name); + abi_module = __Pyx_FetchSharedCythonABIModule(); + if (!abi_module) return NULL; + cached_type = (PyTypeObject*) PyObject_GetAttrString(abi_module, type->tp_name); if (cached_type) { - if (!PyType_Check((PyObject*)cached_type)) { - PyErr_Format(PyExc_TypeError, - "Shared Cython type %.200s is not a type object", - type->tp_name); + if (__Pyx_VerifyCachedType( + (PyObject *)cached_type, + type->tp_name, + cached_type->tp_basicsize, + type->tp_basicsize) < 0) { goto bad; } - if (cached_type->tp_basicsize != type->tp_basicsize) { - PyErr_Format(PyExc_TypeError, - "Shared Cython type %.200s has the wrong size, try recompiling", - type->tp_name); + goto done; + } + + if (!PyErr_ExceptionMatches(PyExc_AttributeError)) goto bad; + PyErr_Clear(); + if (PyType_Ready(type) < 0) goto bad; + if (PyObject_SetAttrString(abi_module, type->tp_name, (PyObject *)type) < 0) + goto bad; + Py_INCREF(type); + cached_type = type; + +done: + Py_DECREF(abi_module); + // NOTE: always returns owned reference, or NULL on error + return cached_type; + +bad: + Py_XDECREF(cached_type); + cached_type = NULL; + goto done; +} +#else + +static PyTypeObject *__Pyx_FetchCommonTypeFromSpec(PyObject *module, PyType_Spec *spec, PyObject *bases) { + PyObject *abi_module, *cached_type = NULL; + + abi_module = __Pyx_FetchSharedCythonABIModule(); + if (!abi_module) return NULL; + + cached_type = PyObject_GetAttrString(abi_module, spec->name); + if (cached_type) { + Py_ssize_t basicsize; +#if CYTHON_COMPILING_IN_LIMITED_API + PyObject *py_basicsize; + py_basicsize = PyObject_GetAttrString(cached_type, "__basicsize__"); + if (unlikely(!py_basicsize)) goto bad; + basicsize = PyLong_AsSsize_t(py_basicsize); + Py_DECREF(py_basicsize); + py_basicsize = 0; + if (unlikely(basicsize == (Py_ssize_t)-1) && PyErr_Occurred()) goto bad; +#else + basicsize = likely(PyType_Check(cached_type)) ? ((PyTypeObject*) cached_type)->tp_basicsize : -1; +#endif + if (__Pyx_VerifyCachedType( + cached_type, + spec->name, + basicsize, + spec->basicsize) < 0) { goto bad; } - } else { - if (!PyErr_ExceptionMatches(PyExc_AttributeError)) goto bad; - PyErr_Clear(); - if (PyType_Ready(type) < 0) goto bad; - if (PyObject_SetAttrString(fake_module, type->tp_name, (PyObject*) type) < 0) - goto bad; - Py_INCREF(type); - cached_type = type; + goto done; } + if (!PyErr_ExceptionMatches(PyExc_AttributeError)) goto bad; + PyErr_Clear(); + // We pass the ABI module reference to avoid keeping the user module alive by foreign type usages. + (void) module; + cached_type = __Pyx_PyType_FromModuleAndSpec(abi_module, spec, bases); + if (unlikely(!cached_type)) goto bad; + if (unlikely(__Pyx_fix_up_extension_type_from_spec(spec, (PyTypeObject *) cached_type) < 0)) goto bad; + if (PyObject_SetAttrString(abi_module, spec->name, cached_type) < 0) goto bad; + done: - Py_DECREF(fake_module); + Py_DECREF(abi_module); // NOTE: always returns owned reference, or NULL on error - return cached_type; + assert(cached_type == NULL || PyType_Check(cached_type)); + return (PyTypeObject *) cached_type; bad: Py_XDECREF(cached_type); cached_type = NULL; goto done; } +#endif /////////////// FetchCommonPointer.proto /////////////// @@ -56,31 +134,27 @@ static void* __Pyx_FetchCommonPointer(void* pointer, const char* name); static void* __Pyx_FetchCommonPointer(void* pointer, const char* name) { -#if PY_VERSION_HEX >= 0x02070000 - PyObject* fake_module = NULL; + PyObject* abi_module = NULL; PyObject* capsule = NULL; void* value = NULL; - fake_module = PyImport_AddModule((char*) "_cython_" CYTHON_ABI); - if (!fake_module) return NULL; - Py_INCREF(fake_module); + abi_module = PyImport_AddModule((char*) __PYX_ABI_MODULE_NAME); + if (!abi_module) return NULL; + Py_INCREF(abi_module); - capsule = PyObject_GetAttrString(fake_module, name); + capsule = PyObject_GetAttrString(abi_module, name); if (!capsule) { if (!PyErr_ExceptionMatches(PyExc_AttributeError)) goto bad; PyErr_Clear(); capsule = PyCapsule_New(pointer, name, NULL); if (!capsule) goto bad; - if (PyObject_SetAttrString(fake_module, name, capsule) < 0) + if (PyObject_SetAttrString(abi_module, name, capsule) < 0) goto bad; } value = PyCapsule_GetPointer(capsule, name); bad: Py_XDECREF(capsule); - Py_DECREF(fake_module); + Py_DECREF(abi_module); return value; -#else - return pointer; -#endif } diff --git a/Cython/Utility/Coroutine.c b/Cython/Utility/Coroutine.c index b71fced3a..1332c9824 100644 --- a/Cython/Utility/Coroutine.c +++ b/Cython/Utility/Coroutine.c @@ -5,12 +5,15 @@ static CYTHON_INLINE PyObject* __Pyx_Generator_Yield_From(__pyx_CoroutineObject //////////////////// GeneratorYieldFrom //////////////////// //@requires: Generator -static void __PyxPyIter_CheckErrorAndDecref(PyObject *source) { +#if CYTHON_USE_TYPE_SLOTS +static void __Pyx_PyIter_CheckErrorAndDecref(PyObject *source) { + __Pyx_TypeName source_type_name = __Pyx_PyType_GetName(Py_TYPE(source)); PyErr_Format(PyExc_TypeError, - "iter() returned non-iterator of type '%.100s'", - Py_TYPE(source)->tp_name); + "iter() returned non-iterator of type '" __Pyx_FMT_TYPENAME "'", source_type_name); + __Pyx_DECREF_TypeName(source_type_name); Py_DECREF(source); } +#endif static CYTHON_INLINE PyObject* __Pyx_Generator_Yield_From(__pyx_CoroutineObject *gen, PyObject *source) { PyObject *source_gen, *retval; @@ -29,7 +32,7 @@ static CYTHON_INLINE PyObject* __Pyx_Generator_Yield_From(__pyx_CoroutineObject if (unlikely(!source_gen)) return NULL; if (unlikely(!PyIter_Check(source_gen))) { - __PyxPyIter_CheckErrorAndDecref(source_gen); + __Pyx_PyIter_CheckErrorAndDecref(source_gen); return NULL; } } else @@ -41,11 +44,7 @@ static CYTHON_INLINE PyObject* __Pyx_Generator_Yield_From(__pyx_CoroutineObject return NULL; } // source_gen is now the iterator, make the first next() call -#if CYTHON_USE_TYPE_SLOTS - retval = Py_TYPE(source_gen)->tp_iternext(source_gen); -#else - retval = PyIter_Next(source_gen); -#endif + retval = __Pyx_PyObject_GetIterNextFunc(source_gen)(source_gen); } if (likely(retval)) { gen->yieldfrom = source_gen; @@ -74,11 +73,7 @@ static PyObject* __Pyx__Coroutine_Yield_From_Generic(__pyx_CoroutineObject *gen, if (__Pyx_Coroutine_Check(source_gen)) { retval = __Pyx_Generator_Next(source_gen); } else { -#if CYTHON_USE_TYPE_SLOTS - retval = Py_TYPE(source_gen)->tp_iternext(source_gen); -#else - retval = PyIter_Next(source_gen); -#endif + retval = __Pyx_PyObject_GetIterNextFunc(source_gen)(source_gen); } if (retval) { gen->yieldfrom = source_gen; @@ -136,13 +131,13 @@ static CYTHON_INLINE PyObject *__Pyx_Coroutine_GetAwaitableIter(PyObject *o) { static void __Pyx_Coroutine_AwaitableIterError(PyObject *source) { #if PY_VERSION_HEX >= 0x030600B3 || defined(_PyErr_FormatFromCause) - _PyErr_FormatFromCause( - PyExc_TypeError, - "'async for' received an invalid object " - "from __anext__: %.100s", - Py_TYPE(source)->tp_name); + __Pyx_TypeName source_type_name = __Pyx_PyType_GetName(Py_TYPE(source)); + _PyErr_FormatFromCause(PyExc_TypeError, + "'async for' received an invalid object from __anext__: " __Pyx_FMT_TYPENAME, source_type_name); + __Pyx_DECREF_TypeName(source_type_name); #elif PY_MAJOR_VERSION >= 3 PyObject *exc, *val, *val2, *tb; + __Pyx_TypeName source_type_name = __Pyx_PyType_GetName(Py_TYPE(source)); assert(PyErr_Occurred()); PyErr_Fetch(&exc, &val, &tb); PyErr_NormalizeException(&exc, &val, &tb); @@ -152,11 +147,9 @@ static void __Pyx_Coroutine_AwaitableIterError(PyObject *source) { } Py_DECREF(exc); assert(!PyErr_Occurred()); - PyErr_Format( - PyExc_TypeError, - "'async for' received an invalid object " - "from __anext__: %.100s", - Py_TYPE(source)->tp_name); + PyErr_Format(PyExc_TypeError, + "'async for' received an invalid object from __anext__: " __Pyx_FMT_TYPENAME, source_type_name); + __Pyx_DECREF_TypeName(source_type_name); PyErr_Fetch(&exc, &val2, &tb); PyErr_NormalizeException(&exc, &val2, &tb); @@ -207,9 +200,10 @@ static PyObject *__Pyx__Coroutine_GetAwaitableIter(PyObject *obj) { goto bad; } if (unlikely(!PyIter_Check(res))) { + __Pyx_TypeName res_type_name = __Pyx_PyType_GetName(Py_TYPE(res)); PyErr_Format(PyExc_TypeError, - "__await__() returned non-iterator of type '%.100s'", - Py_TYPE(res)->tp_name); + "__await__() returned non-iterator of type '" __Pyx_FMT_TYPENAME "'", res_type_name); + __Pyx_DECREF_TypeName(res_type_name); Py_CLEAR(res); } else { int is_coroutine = 0; @@ -229,9 +223,12 @@ static PyObject *__Pyx__Coroutine_GetAwaitableIter(PyObject *obj) { } return res; slot_error: - PyErr_Format(PyExc_TypeError, - "object %.100s can't be used in 'await' expression", - Py_TYPE(obj)->tp_name); + { + __Pyx_TypeName obj_type_name = __Pyx_PyType_GetName(Py_TYPE(obj)); + PyErr_Format(PyExc_TypeError, + "object " __Pyx_FMT_TYPENAME " can't be used in 'await' expression", obj_type_name); + __Pyx_DECREF_TypeName(obj_type_name); + } bad: return NULL; } @@ -247,6 +244,7 @@ static CYTHON_INLINE PyObject *__Pyx_Coroutine_AsyncIterNext(PyObject *o); /*pro //@requires: ObjectHandling.c::PyObjectCallMethod0 static PyObject *__Pyx_Coroutine_GetAsyncIter_Generic(PyObject *obj) { + __Pyx_TypeName obj_type_name; #if PY_VERSION_HEX < 0x030500B1 { PyObject *iter = __Pyx_PyObject_CallMethod0(obj, PYIDENT("__aiter__")); @@ -261,8 +259,10 @@ static PyObject *__Pyx_Coroutine_GetAsyncIter_Generic(PyObject *obj) { if ((0)) (void) __Pyx_PyObject_CallMethod0(obj, PYIDENT("__aiter__")); #endif - PyErr_Format(PyExc_TypeError, "'async for' requires an object with __aiter__ method, got %.100s", - Py_TYPE(obj)->tp_name); + obj_type_name = __Pyx_PyType_GetName(Py_TYPE(obj)); + PyErr_Format(PyExc_TypeError, + "'async for' requires an object with __aiter__ method, got " __Pyx_FMT_TYPENAME, obj_type_name); + __Pyx_DECREF_TypeName(obj_type_name); return NULL; } @@ -295,8 +295,12 @@ static PyObject *__Pyx__Coroutine_AsyncIterNext(PyObject *obj) { // FIXME: for the sake of a nicely conforming exception message, assume any AttributeError meant '__anext__' if (PyErr_ExceptionMatches(PyExc_AttributeError)) #endif - PyErr_Format(PyExc_TypeError, "'async for' requires an object with __anext__ method, got %.100s", - Py_TYPE(obj)->tp_name); + { + __Pyx_TypeName obj_type_name = __Pyx_PyType_GetName(Py_TYPE(obj)); + PyErr_Format(PyExc_TypeError, + "'async for' requires an object with __anext__ method, got " __Pyx_FMT_TYPENAME, obj_type_name); + __Pyx_DECREF_TypeName(obj_type_name); + } return NULL; } @@ -326,12 +330,13 @@ static void __Pyx_Generator_Replace_StopIteration(int in_async_gen); /*proto*/ //////////////////// pep479 //////////////////// //@requires: Exceptions.c::GetException -static void __Pyx_Generator_Replace_StopIteration(CYTHON_UNUSED int in_async_gen) { +static void __Pyx_Generator_Replace_StopIteration(int in_async_gen) { PyObject *exc, *val, *tb, *cur_exc; __Pyx_PyThreadState_declare #ifdef __Pyx_StopAsyncIteration_USED int is_async_stopiteration = 0; #endif + CYTHON_MAYBE_UNUSED_VAR(in_async_gen); cur_exc = PyErr_Occurred(); if (likely(!__Pyx_PyErr_GivenExceptionMatches(cur_exc, PyExc_StopIteration))) { @@ -362,7 +367,8 @@ static void __Pyx_Generator_Replace_StopIteration(CYTHON_UNUSED int in_async_gen //////////////////// CoroutineBase.proto //////////////////// //@substitute: naming -typedef PyObject *(*__pyx_coroutine_body_t)(PyObject *, PyThreadState *, PyObject *); +struct __pyx_CoroutineObject; +typedef PyObject *(*__pyx_coroutine_body_t)(struct __pyx_CoroutineObject *, PyThreadState *, PyObject *); #if CYTHON_USE_EXC_INFO_STACK // See https://bugs.python.org/issue25612 @@ -376,7 +382,7 @@ typedef struct { } __Pyx_ExcInfoStruct; #endif -typedef struct { +typedef struct __pyx_CoroutineObject { PyObject_HEAD __pyx_coroutine_body_t body; PyObject *closure; @@ -439,15 +445,15 @@ static CYTHON_INLINE void __Pyx_Coroutine_ResetFrameBackpointer(__Pyx_ExcInfoStr #define __Pyx_Coroutine_USED static PyTypeObject *__pyx_CoroutineType = 0; static PyTypeObject *__pyx_CoroutineAwaitType = 0; -#define __Pyx_Coroutine_CheckExact(obj) (Py_TYPE(obj) == __pyx_CoroutineType) +#define __Pyx_Coroutine_CheckExact(obj) __Pyx_IS_TYPE(obj, __pyx_CoroutineType) // __Pyx_Coroutine_Check(obj): see override for IterableCoroutine below #define __Pyx_Coroutine_Check(obj) __Pyx_Coroutine_CheckExact(obj) -#define __Pyx_CoroutineAwait_CheckExact(obj) (Py_TYPE(obj) == __pyx_CoroutineAwaitType) +#define __Pyx_CoroutineAwait_CheckExact(obj) __Pyx_IS_TYPE(obj, __pyx_CoroutineAwaitType) #define __Pyx_Coroutine_New(body, code, closure, name, qualname, module_name) \ __Pyx__Coroutine_New(__pyx_CoroutineType, body, code, closure, name, qualname, module_name) -static int __pyx_Coroutine_init(void); /*proto*/ +static int __pyx_Coroutine_init(PyObject *module); /*proto*/ static PyObject *__Pyx__Coroutine_await(PyObject *coroutine); /*proto*/ typedef struct { @@ -463,13 +469,13 @@ static PyObject *__Pyx_CoroutineAwait_Throw(__pyx_CoroutineAwaitObject *self, Py #define __Pyx_Generator_USED static PyTypeObject *__pyx_GeneratorType = 0; -#define __Pyx_Generator_CheckExact(obj) (Py_TYPE(obj) == __pyx_GeneratorType) +#define __Pyx_Generator_CheckExact(obj) __Pyx_IS_TYPE(obj, __pyx_GeneratorType) #define __Pyx_Generator_New(body, code, closure, name, qualname, module_name) \ __Pyx__Coroutine_New(__pyx_GeneratorType, body, code, closure, name, qualname, module_name) static PyObject *__Pyx_Generator_Next(PyObject *self); -static int __pyx_Generator_init(void); /*proto*/ +static int __pyx_Generator_init(PyObject *module); /*proto*/ //////////////////// AsyncGen //////////////////// @@ -485,10 +491,13 @@ static int __pyx_Generator_init(void); /*proto*/ //@requires: Exceptions.c::RaiseException //@requires: Exceptions.c::SaveResetException //@requires: ObjectHandling.c::PyObjectCallMethod1 +//@requires: ObjectHandling.c::PyObjectCallNoArg +//@requires: ObjectHandling.c::PyObjectFastCall //@requires: ObjectHandling.c::PyObjectGetAttrStr +//@requires: ObjectHandling.c::PyObjectGetAttrStrNoError //@requires: CommonStructures.c::FetchCommonType +//@requires: ModuleSetupCode.c::IncludeStructmemberH -#include <structmember.h> #include <frameobject.h> #define __Pyx_Coroutine_Undelegate(gen) Py_CLEAR((gen)->yieldfrom) @@ -499,9 +508,10 @@ static int __pyx_Generator_init(void); /*proto*/ // Returns 0 if no exception or StopIteration is set. // If any other exception is set, returns -1 and leaves // pvalue unchanged. -static int __Pyx_PyGen__FetchStopIterationValue(CYTHON_UNUSED PyThreadState *$local_tstate_cname, PyObject **pvalue) { +static int __Pyx_PyGen__FetchStopIterationValue(PyThreadState *$local_tstate_cname, PyObject **pvalue) { PyObject *et, *ev, *tb; PyObject *value = NULL; + CYTHON_UNUSED_VAR($local_tstate_cname); __Pyx_ErrFetch(&et, &ev, &tb); @@ -520,7 +530,7 @@ static int __Pyx_PyGen__FetchStopIterationValue(CYTHON_UNUSED PyThreadState *$lo value = Py_None; } #if PY_VERSION_HEX >= 0x030300A0 - else if (Py_TYPE(ev) == (PyTypeObject*)PyExc_StopIteration) { + else if (likely(__Pyx_IS_TYPE(ev, (PyTypeObject*)PyExc_StopIteration))) { value = ((PyStopIterationObject *)ev)->value; Py_INCREF(value); Py_DECREF(ev); @@ -606,8 +616,9 @@ void __Pyx_Coroutine_ExceptionClear(__Pyx_ExcInfoStruct *exc_state) { } #define __Pyx_Coroutine_AlreadyRunningError(gen) (__Pyx__Coroutine_AlreadyRunningError(gen), (PyObject*)NULL) -static void __Pyx__Coroutine_AlreadyRunningError(CYTHON_UNUSED __pyx_CoroutineObject *gen) { +static void __Pyx__Coroutine_AlreadyRunningError(__pyx_CoroutineObject *gen) { const char *msg; + CYTHON_MAYBE_UNUSED_VAR(gen); if ((0)) { #ifdef __Pyx_Coroutine_USED } else if (__Pyx_Coroutine_Check((PyObject*)gen)) { @@ -624,8 +635,9 @@ static void __Pyx__Coroutine_AlreadyRunningError(CYTHON_UNUSED __pyx_CoroutineOb } #define __Pyx_Coroutine_NotStartedError(gen) (__Pyx__Coroutine_NotStartedError(gen), (PyObject*)NULL) -static void __Pyx__Coroutine_NotStartedError(CYTHON_UNUSED PyObject *gen) { +static void __Pyx__Coroutine_NotStartedError(PyObject *gen) { const char *msg; + CYTHON_MAYBE_UNUSED_VAR(gen); if ((0)) { #ifdef __Pyx_Coroutine_USED } else if (__Pyx_Coroutine_Check(gen)) { @@ -642,7 +654,9 @@ static void __Pyx__Coroutine_NotStartedError(CYTHON_UNUSED PyObject *gen) { } #define __Pyx_Coroutine_AlreadyTerminatedError(gen, value, closing) (__Pyx__Coroutine_AlreadyTerminatedError(gen, value, closing), (PyObject*)NULL) -static void __Pyx__Coroutine_AlreadyTerminatedError(CYTHON_UNUSED PyObject *gen, PyObject *value, CYTHON_UNUSED int closing) { +static void __Pyx__Coroutine_AlreadyTerminatedError(PyObject *gen, PyObject *value, int closing) { + CYTHON_MAYBE_UNUSED_VAR(gen); + CYTHON_MAYBE_UNUSED_VAR(closing); #ifdef __Pyx_Coroutine_USED if (!closing && __Pyx_Coroutine_Check(gen)) { // `self` is an exhausted coroutine: raise an error, @@ -705,7 +719,7 @@ PyObject *__Pyx_Coroutine_SendEx(__pyx_CoroutineObject *self, PyObject *value, i exc_state = &self->gi_exc_state; if (exc_state->exc_type) { - #if CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_PYSTON + #if CYTHON_COMPILING_IN_PYPY // FIXME: what to do in PyPy? #else // Generators always return to their most recent caller, not @@ -739,7 +753,7 @@ PyObject *__Pyx_Coroutine_SendEx(__pyx_CoroutineObject *self, PyObject *value, i #endif self->is_running = 1; - retval = self->body((PyObject *) self, tstate, value); + retval = self->body(self, tstate, value); self->is_running = 0; #if CYTHON_USE_EXC_INFO_STACK @@ -761,7 +775,7 @@ static CYTHON_INLINE void __Pyx_Coroutine_ResetFrameBackpointer(__Pyx_ExcInfoStr PyObject *exc_tb = exc_state->exc_traceback; if (likely(exc_tb)) { -#if CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_PYSTON +#if CYTHON_COMPILING_IN_PYPY // FIXME: what to do in PyPy? #else PyTracebackObject *tb = (PyTracebackObject *) exc_tb; @@ -772,7 +786,8 @@ static CYTHON_INLINE void __Pyx_Coroutine_ResetFrameBackpointer(__Pyx_ExcInfoStr } static CYTHON_INLINE -PyObject *__Pyx_Coroutine_MethodReturn(CYTHON_UNUSED PyObject* gen, PyObject *retval) { +PyObject *__Pyx_Coroutine_MethodReturn(PyObject* gen, PyObject *retval) { + CYTHON_MAYBE_UNUSED_VAR(gen); if (unlikely(!retval)) { __Pyx_PyThreadState_declare __Pyx_PyThreadState_assign @@ -867,7 +882,7 @@ static PyObject *__Pyx_Coroutine_Send(PyObject *self, PyObject *value) { #endif { if (value == Py_None) - ret = Py_TYPE(yf)->tp_iternext(yf); + ret = __Pyx_PyObject_GetIterNextFunc(yf)(yf); else ret = __Pyx_PyObject_CallMethod1(yf, PYIDENT("send"), value); } @@ -921,16 +936,15 @@ static int __Pyx_Coroutine_CloseIter(__pyx_CoroutineObject *gen, PyObject *yf) { { PyObject *meth; gen->is_running = 1; - meth = __Pyx_PyObject_GetAttrStr(yf, PYIDENT("close")); + meth = __Pyx_PyObject_GetAttrStrNoError(yf, PYIDENT("close")); if (unlikely(!meth)) { - if (!PyErr_ExceptionMatches(PyExc_AttributeError)) { + if (unlikely(PyErr_Occurred())) { PyErr_WriteUnraisable(yf); } - PyErr_Clear(); } else { - retval = PyObject_CallFunction(meth, NULL); + retval = __Pyx_PyObject_CallNoArg(meth); Py_DECREF(meth); - if (!retval) + if (unlikely(!retval)) err = -1; } gen->is_running = 0; @@ -966,7 +980,7 @@ static PyObject *__Pyx_Generator_Next(PyObject *self) { ret = __Pyx_Coroutine_Send(yf, Py_None); } else #endif - ret = Py_TYPE(yf)->tp_iternext(yf); + ret = __Pyx_PyObject_GetIterNextFunc(yf)(yf); gen->is_running = 0; //Py_DECREF(yf); if (likely(ret)) { @@ -977,7 +991,8 @@ static PyObject *__Pyx_Generator_Next(PyObject *self) { return __Pyx_Coroutine_SendEx(gen, Py_None, 0); } -static PyObject *__Pyx_Coroutine_Close_Method(PyObject *self, CYTHON_UNUSED PyObject *arg) { +static PyObject *__Pyx_Coroutine_Close_Method(PyObject *self, PyObject *arg) { + CYTHON_UNUSED_VAR(arg); return __Pyx_Coroutine_Close(self); } @@ -1068,23 +1083,23 @@ static PyObject *__Pyx__Coroutine_Throw(PyObject *self, PyObject *typ, PyObject ret = __Pyx__Coroutine_Throw(((__pyx_CoroutineAwaitObject*)yf)->coroutine, typ, val, tb, args, close_on_genexit); #endif } else { - PyObject *meth = __Pyx_PyObject_GetAttrStr(yf, PYIDENT("throw")); + PyObject *meth = __Pyx_PyObject_GetAttrStrNoError(yf, PYIDENT("throw")); if (unlikely(!meth)) { Py_DECREF(yf); - if (!PyErr_ExceptionMatches(PyExc_AttributeError)) { + if (unlikely(PyErr_Occurred())) { gen->is_running = 0; return NULL; } - PyErr_Clear(); __Pyx_Coroutine_Undelegate(gen); gen->is_running = 0; goto throw_here; } if (likely(args)) { - ret = PyObject_CallObject(meth, args); + ret = __Pyx_PyObject_Call(meth, args, NULL); } else { // "tb" or even "val" might be NULL, but that also correctly terminates the argument list - ret = PyObject_CallFunctionObjArgs(meth, typ, val, tb, NULL); + PyObject *cargs[4] = {NULL, typ, val, tb}; + ret = __Pyx_PyObject_FastCall(meth, cargs+1, 3 | __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET); } Py_DECREF(meth); } @@ -1105,7 +1120,7 @@ static PyObject *__Pyx_Coroutine_Throw(PyObject *self, PyObject *args) { PyObject *val = NULL; PyObject *tb = NULL; - if (!PyArg_UnpackTuple(args, (char *)"throw", 1, 3, &typ, &val, &tb)) + if (unlikely(!PyArg_UnpackTuple(args, (char *)"throw", 1, 3, &typ, &val, &tb))) return NULL; return __Pyx__Coroutine_Throw(self, typ, val, tb, args, 1); @@ -1156,10 +1171,10 @@ static void __Pyx_Coroutine_dealloc(PyObject *self) { // Generator is paused or unstarted, so we need to close PyObject_GC_Track(self); #if PY_VERSION_HEX >= 0x030400a1 && CYTHON_USE_TP_FINALIZE - if (PyObject_CallFinalizerFromDealloc(self)) + if (unlikely(PyObject_CallFinalizerFromDealloc(self))) #else Py_TYPE(gen)->tp_del(self); - if (Py_REFCNT(self) > 0) + if (unlikely(Py_REFCNT(self) > 0)) #endif { // resurrected. :( @@ -1177,7 +1192,7 @@ static void __Pyx_Coroutine_dealloc(PyObject *self) { } #endif __Pyx_Coroutine_clear(self); - PyObject_GC_Del(gen); + __Pyx_PyHeapTypeObject_GC_Del(gen); } static void __Pyx_Coroutine_del(PyObject *self) { @@ -1275,7 +1290,7 @@ static void __Pyx_Coroutine_del(PyObject *self) { // Undo the temporary resurrection; can't use DECREF here, it would // cause a recursive call. assert(Py_REFCNT(self) > 0); - if (--self->ob_refcnt == 0) { + if (likely(--self->ob_refcnt == 0)) { // this is the normal path out return; } @@ -1308,9 +1323,10 @@ static void __Pyx_Coroutine_del(PyObject *self) { } static PyObject * -__Pyx_Coroutine_get_name(__pyx_CoroutineObject *self, CYTHON_UNUSED void *context) +__Pyx_Coroutine_get_name(__pyx_CoroutineObject *self, void *context) { PyObject *name = self->gi_name; + CYTHON_UNUSED_VAR(context); // avoid NULL pointer dereference during garbage collection if (unlikely(!name)) name = Py_None; Py_INCREF(name); @@ -1318,10 +1334,9 @@ __Pyx_Coroutine_get_name(__pyx_CoroutineObject *self, CYTHON_UNUSED void *contex } static int -__Pyx_Coroutine_set_name(__pyx_CoroutineObject *self, PyObject *value, CYTHON_UNUSED void *context) +__Pyx_Coroutine_set_name(__pyx_CoroutineObject *self, PyObject *value, void *context) { - PyObject *tmp; - + CYTHON_UNUSED_VAR(context); #if PY_MAJOR_VERSION >= 3 if (unlikely(value == NULL || !PyUnicode_Check(value))) #else @@ -1332,17 +1347,16 @@ __Pyx_Coroutine_set_name(__pyx_CoroutineObject *self, PyObject *value, CYTHON_UN "__name__ must be set to a string object"); return -1; } - tmp = self->gi_name; Py_INCREF(value); - self->gi_name = value; - Py_XDECREF(tmp); + __Pyx_Py_XDECREF_SET(self->gi_name, value); return 0; } static PyObject * -__Pyx_Coroutine_get_qualname(__pyx_CoroutineObject *self, CYTHON_UNUSED void *context) +__Pyx_Coroutine_get_qualname(__pyx_CoroutineObject *self, void *context) { PyObject *name = self->gi_qualname; + CYTHON_UNUSED_VAR(context); // avoid NULL pointer dereference during garbage collection if (unlikely(!name)) name = Py_None; Py_INCREF(name); @@ -1350,10 +1364,9 @@ __Pyx_Coroutine_get_qualname(__pyx_CoroutineObject *self, CYTHON_UNUSED void *co } static int -__Pyx_Coroutine_set_qualname(__pyx_CoroutineObject *self, PyObject *value, CYTHON_UNUSED void *context) +__Pyx_Coroutine_set_qualname(__pyx_CoroutineObject *self, PyObject *value, void *context) { - PyObject *tmp; - + CYTHON_UNUSED_VAR(context); #if PY_MAJOR_VERSION >= 3 if (unlikely(value == NULL || !PyUnicode_Check(value))) #else @@ -1364,18 +1377,16 @@ __Pyx_Coroutine_set_qualname(__pyx_CoroutineObject *self, PyObject *value, CYTHO "__qualname__ must be set to a string object"); return -1; } - tmp = self->gi_qualname; Py_INCREF(value); - self->gi_qualname = value; - Py_XDECREF(tmp); + __Pyx_Py_XDECREF_SET(self->gi_qualname, value); return 0; } - static PyObject * -__Pyx_Coroutine_get_frame(__pyx_CoroutineObject *self, CYTHON_UNUSED void *context) +__Pyx_Coroutine_get_frame(__pyx_CoroutineObject *self, void *context) { PyObject *frame = self->gi_frame; + CYTHON_UNUSED_VAR(context); if (!frame) { if (unlikely(!self->gi_code)) { // Avoid doing something stupid, e.g. during garbage collection. @@ -1445,7 +1456,7 @@ static __pyx_CoroutineObject *__Pyx__Coroutine_NewInit( static void __Pyx_CoroutineAwait_dealloc(PyObject *self) { PyObject_GC_UnTrack(self); Py_CLEAR(((__pyx_CoroutineAwaitObject*)self)->coroutine); - PyObject_GC_Del(self); + __Pyx_PyHeapTypeObject_GC_Del(self); } static int __Pyx_CoroutineAwait_traverse(__pyx_CoroutineAwaitObject *self, visitproc visit, void *arg) { @@ -1470,7 +1481,8 @@ static PyObject *__Pyx_CoroutineAwait_Throw(__pyx_CoroutineAwaitObject *self, Py return __Pyx_Coroutine_Throw(self->coroutine, args); } -static PyObject *__Pyx_CoroutineAwait_Close(__pyx_CoroutineAwaitObject *self, CYTHON_UNUSED PyObject *arg) { +static PyObject *__Pyx_CoroutineAwait_Close(__pyx_CoroutineAwaitObject *self, PyObject *arg) { + CYTHON_UNUSED_VAR(arg); return __Pyx_Coroutine_Close(self->coroutine); } @@ -1480,7 +1492,10 @@ static PyObject *__Pyx_CoroutineAwait_self(PyObject *self) { } #if !CYTHON_COMPILING_IN_PYPY -static PyObject *__Pyx_CoroutineAwait_no_new(CYTHON_UNUSED PyTypeObject *type, CYTHON_UNUSED PyObject *args, CYTHON_UNUSED PyObject *kwargs) { +static PyObject *__Pyx_CoroutineAwait_no_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) { + CYTHON_UNUSED_VAR(type); + CYTHON_UNUSED_VAR(args); + CYTHON_UNUSED_VAR(kwargs); PyErr_SetString(PyExc_TypeError, "cannot instantiate type, use 'await coroutine' instead"); return NULL; } @@ -1496,9 +1511,32 @@ static PyMethodDef __pyx_CoroutineAwait_methods[] = { {0, 0, 0, 0} }; +#if CYTHON_USE_TYPE_SPECS +static PyType_Slot __pyx_CoroutineAwaitType_slots[] = { + {Py_tp_dealloc, (void *)__Pyx_CoroutineAwait_dealloc}, + {Py_tp_traverse, (void *)__Pyx_CoroutineAwait_traverse}, + {Py_tp_clear, (void *)__Pyx_CoroutineAwait_clear}, +#if !CYTHON_COMPILING_IN_PYPY + {Py_tp_new, (void *)__Pyx_CoroutineAwait_no_new}, +#endif + {Py_tp_methods, (void *)__pyx_CoroutineAwait_methods}, + {Py_tp_iter, (void *)__Pyx_CoroutineAwait_self}, + {Py_tp_iternext, (void *)__Pyx_CoroutineAwait_Next}, + {0, 0}, +}; + +static PyType_Spec __pyx_CoroutineAwaitType_spec = { + __PYX_TYPE_MODULE_PREFIX "coroutine_wrapper", + sizeof(__pyx_CoroutineAwaitObject), + 0, + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /*tp_flags*/ + __pyx_CoroutineAwaitType_slots +}; +#else /* CYTHON_USE_TYPE_SPECS */ + static PyTypeObject __pyx_CoroutineAwaitType_type = { PyVarObject_HEAD_INIT(0, 0) - "coroutine_wrapper", /*tp_name*/ + __PYX_TYPE_MODULE_PREFIX "coroutine_wrapper", /*tp_name*/ sizeof(__pyx_CoroutineAwaitObject), /*tp_basicsize*/ 0, /*tp_itemsize*/ (destructor) __Pyx_CoroutineAwait_dealloc,/*tp_dealloc*/ @@ -1557,7 +1595,11 @@ static PyTypeObject __pyx_CoroutineAwaitType_type = { #if PY_VERSION_HEX >= 0x030800b4 && PY_VERSION_HEX < 0x03090000 0, /*tp_print*/ #endif +#if CYTHON_COMPILING_IN_PYPY && PYPY_VERSION_NUM+0 >= 0x06000000 + 0, /*tp_pypy_flags*/ +#endif }; +#endif /* CYTHON_USE_TYPE_SPECS */ #if PY_VERSION_HEX < 0x030500B1 || defined(__Pyx_IterableCoroutine_USED) || CYTHON_USE_ASYNC_SLOTS static CYTHON_INLINE PyObject *__Pyx__Coroutine_await(PyObject *coroutine) { @@ -1571,7 +1613,8 @@ static CYTHON_INLINE PyObject *__Pyx__Coroutine_await(PyObject *coroutine) { #endif #if PY_VERSION_HEX < 0x030500B1 -static PyObject *__Pyx_Coroutine_await_method(PyObject *coroutine, CYTHON_UNUSED PyObject *arg) { +static PyObject *__Pyx_Coroutine_await_method(PyObject *coroutine, PyObject *arg) { + CYTHON_UNUSED_VAR(arg); return __Pyx__Coroutine_await(coroutine); } #endif @@ -1619,7 +1662,10 @@ static PyMemberDef __pyx_Coroutine_memberlist[] = { {(char*) "cr_await", T_OBJECT, offsetof(__pyx_CoroutineObject, yieldfrom), READONLY, (char*) PyDoc_STR("object being awaited, or None")}, {(char*) "cr_code", T_OBJECT, offsetof(__pyx_CoroutineObject, gi_code), READONLY, NULL}, - {(char *) "__module__", T_OBJECT, offsetof(__pyx_CoroutineObject, gi_modulename), PY_WRITE_RESTRICTED, 0}, + {(char *) "__module__", T_OBJECT, offsetof(__pyx_CoroutineObject, gi_modulename), 0, 0}, +#if CYTHON_USE_TYPE_SPECS + {(char *) "__weaklistoffset__", T_PYSSIZET, offsetof(__pyx_CoroutineObject, gi_weakreflist), READONLY, 0}, +#endif {0, 0, 0, 0, 0} }; @@ -1633,6 +1679,30 @@ static PyGetSetDef __pyx_Coroutine_getsets[] = { {0, 0, 0, 0, 0} }; +#if CYTHON_USE_TYPE_SPECS +static PyType_Slot __pyx_CoroutineType_slots[] = { + {Py_tp_dealloc, (void *)__Pyx_Coroutine_dealloc}, + {Py_am_await, (void *)&__Pyx_Coroutine_await}, + {Py_tp_traverse, (void *)__Pyx_Coroutine_traverse}, + {Py_tp_methods, (void *)__pyx_Coroutine_methods}, + {Py_tp_members, (void *)__pyx_Coroutine_memberlist}, + {Py_tp_getset, (void *)__pyx_Coroutine_getsets}, + {Py_tp_getattro, (void *) __Pyx_PyObject_GenericGetAttrNoDict}, +#if CYTHON_USE_TP_FINALIZE + {Py_tp_finalize, (void *)__Pyx_Coroutine_del}, +#endif + {0, 0}, +}; + +static PyType_Spec __pyx_CoroutineType_spec = { + __PYX_TYPE_MODULE_PREFIX "coroutine", + sizeof(__pyx_CoroutineObject), + 0, + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_HAVE_FINALIZE, /*tp_flags*/ + __pyx_CoroutineType_slots +}; +#else /* CYTHON_USE_TYPE_SPECS */ + #if CYTHON_USE_ASYNC_SLOTS static __Pyx_PyAsyncMethodsStruct __pyx_Coroutine_as_async = { __Pyx_Coroutine_await, /*am_await*/ @@ -1643,7 +1713,7 @@ static __Pyx_PyAsyncMethodsStruct __pyx_Coroutine_as_async = { static PyTypeObject __pyx_CoroutineType_type = { PyVarObject_HEAD_INIT(0, 0) - "coroutine", /*tp_name*/ + __PYX_TYPE_MODULE_PREFIX "coroutine", /*tp_name*/ sizeof(__pyx_CoroutineObject), /*tp_basicsize*/ 0, /*tp_itemsize*/ (destructor) __Pyx_Coroutine_dealloc,/*tp_dealloc*/ @@ -1714,21 +1784,34 @@ static PyTypeObject __pyx_CoroutineType_type = { #if PY_VERSION_HEX >= 0x030800b4 && PY_VERSION_HEX < 0x03090000 0, /*tp_print*/ #endif +#if CYTHON_COMPILING_IN_PYPY && PYPY_VERSION_NUM+0 >= 0x06000000 + 0, /*tp_pypy_flags*/ +#endif }; +#endif /* CYTHON_USE_TYPE_SPECS */ -static int __pyx_Coroutine_init(void) { +static int __pyx_Coroutine_init(PyObject *module) { // on Windows, C-API functions can't be used in slots statically +#if CYTHON_USE_TYPE_SPECS + __pyx_CoroutineType = __Pyx_FetchCommonTypeFromSpec(module, &__pyx_CoroutineType_spec, NULL); +#else + (void) module; __pyx_CoroutineType_type.tp_getattro = __Pyx_PyObject_GenericGetAttrNoDict; __pyx_CoroutineType = __Pyx_FetchCommonType(&__pyx_CoroutineType_type); +#endif if (unlikely(!__pyx_CoroutineType)) return -1; #ifdef __Pyx_IterableCoroutine_USED - if (unlikely(__pyx_IterableCoroutine_init() == -1)) + if (unlikely(__pyx_IterableCoroutine_init(module) == -1)) return -1; #endif +#if CYTHON_USE_TYPE_SPECS + __pyx_CoroutineAwaitType = __Pyx_FetchCommonTypeFromSpec(module, &__pyx_CoroutineAwaitType_spec, NULL); +#else __pyx_CoroutineAwaitType = __Pyx_FetchCommonType(&__pyx_CoroutineAwaitType_type); +#endif if (unlikely(!__pyx_CoroutineAwaitType)) return -1; return 0; @@ -1742,21 +1825,47 @@ static int __pyx_Coroutine_init(void) { static PyTypeObject *__pyx_IterableCoroutineType = 0; #undef __Pyx_Coroutine_Check -#define __Pyx_Coroutine_Check(obj) (__Pyx_Coroutine_CheckExact(obj) || (Py_TYPE(obj) == __pyx_IterableCoroutineType)) +#define __Pyx_Coroutine_Check(obj) (__Pyx_Coroutine_CheckExact(obj) || __Pyx_IS_TYPE(obj, __pyx_IterableCoroutineType)) #define __Pyx_IterableCoroutine_New(body, code, closure, name, qualname, module_name) \ __Pyx__Coroutine_New(__pyx_IterableCoroutineType, body, code, closure, name, qualname, module_name) -static int __pyx_IterableCoroutine_init(void);/*proto*/ +static int __pyx_IterableCoroutine_init(PyObject *module);/*proto*/ //////////////////// IterableCoroutine //////////////////// //@requires: Coroutine //@requires: CommonStructures.c::FetchCommonType +#if CYTHON_USE_TYPE_SPECS +static PyType_Slot __pyx_IterableCoroutineType_slots[] = { + {Py_tp_dealloc, (void *)__Pyx_Coroutine_dealloc}, + {Py_am_await, (void *)&__Pyx_Coroutine_await}, + {Py_tp_traverse, (void *)__Pyx_Coroutine_traverse}, + {Py_tp_iter, (void *)__Pyx_Coroutine_await}, + {Py_tp_iternext, (void *)__Pyx_Generator_Next}, + {Py_tp_methods, (void *)__pyx_Coroutine_methods}, + {Py_tp_members, (void *)__pyx_Coroutine_memberlist}, + {Py_tp_getset, (void *)__pyx_Coroutine_getsets}, + {Py_tp_getattro, (void *) __Pyx_PyObject_GenericGetAttrNoDict}, +#if CYTHON_USE_TP_FINALIZE + {Py_tp_finalize, (void *)__Pyx_Coroutine_del}, +#endif + {0, 0}, +}; + +static PyType_Spec __pyx_IterableCoroutineType_spec = { + __PYX_TYPE_MODULE_PREFIX "iterable_coroutine", + sizeof(__pyx_CoroutineObject), + 0, + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_HAVE_FINALIZE, /*tp_flags*/ + __pyx_IterableCoroutineType_slots +}; +#else /* CYTHON_USE_TYPE_SPECS */ + static PyTypeObject __pyx_IterableCoroutineType_type = { PyVarObject_HEAD_INIT(0, 0) - "iterable_coroutine", /*tp_name*/ + __PYX_TYPE_MODULE_PREFIX "iterable_coroutine", /*tp_name*/ sizeof(__pyx_CoroutineObject), /*tp_basicsize*/ 0, /*tp_itemsize*/ (destructor) __Pyx_Coroutine_dealloc,/*tp_dealloc*/ @@ -1825,12 +1934,21 @@ static PyTypeObject __pyx_IterableCoroutineType_type = { #if PY_VERSION_HEX >= 0x030800b4 && PY_VERSION_HEX < 0x03090000 0, /*tp_print*/ #endif +#if CYTHON_COMPILING_IN_PYPY && PYPY_VERSION_NUM+0 >= 0x06000000 + 0, /*tp_pypy_flags*/ +#endif }; +#endif /* CYTHON_USE_TYPE_SPECS */ -static int __pyx_IterableCoroutine_init(void) { +static int __pyx_IterableCoroutine_init(PyObject *module) { +#if CYTHON_USE_TYPE_SPECS + __pyx_IterableCoroutineType = __Pyx_FetchCommonTypeFromSpec(module, &__pyx_IterableCoroutineType_spec, NULL); +#else + (void) module; __pyx_IterableCoroutineType_type.tp_getattro = __Pyx_PyObject_GenericGetAttrNoDict; __pyx_IterableCoroutineType = __Pyx_FetchCommonType(&__pyx_IterableCoroutineType_type); +#endif if (unlikely(!__pyx_IterableCoroutineType)) return -1; return 0; @@ -1857,6 +1975,10 @@ static PyMemberDef __pyx_Generator_memberlist[] = { {(char*) "gi_yieldfrom", T_OBJECT, offsetof(__pyx_CoroutineObject, yieldfrom), READONLY, (char*) PyDoc_STR("object being iterated by 'yield from', or None")}, {(char*) "gi_code", T_OBJECT, offsetof(__pyx_CoroutineObject, gi_code), READONLY, NULL}, + {(char *) "__module__", T_OBJECT, offsetof(__pyx_CoroutineObject, gi_modulename), 0, 0}, +#if CYTHON_USE_TYPE_SPECS + {(char *) "__weaklistoffset__", T_PYSSIZET, offsetof(__pyx_CoroutineObject, gi_weakreflist), READONLY, 0}, +#endif {0, 0, 0, 0, 0} }; @@ -1870,16 +1992,41 @@ static PyGetSetDef __pyx_Generator_getsets[] = { {0, 0, 0, 0, 0} }; +#if CYTHON_USE_TYPE_SPECS +static PyType_Slot __pyx_GeneratorType_slots[] = { + {Py_tp_dealloc, (void *)__Pyx_Coroutine_dealloc}, + {Py_tp_traverse, (void *)__Pyx_Coroutine_traverse}, + {Py_tp_iter, (void *)PyObject_SelfIter}, + {Py_tp_iternext, (void *)__Pyx_Generator_Next}, + {Py_tp_methods, (void *)__pyx_Generator_methods}, + {Py_tp_members, (void *)__pyx_Generator_memberlist}, + {Py_tp_getset, (void *)__pyx_Generator_getsets}, + {Py_tp_getattro, (void *) __Pyx_PyObject_GenericGetAttrNoDict}, +#if CYTHON_USE_TP_FINALIZE + {Py_tp_finalize, (void *)__Pyx_Coroutine_del}, +#endif + {0, 0}, +}; + +static PyType_Spec __pyx_GeneratorType_spec = { + __PYX_TYPE_MODULE_PREFIX "generator", + sizeof(__pyx_CoroutineObject), + 0, + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_HAVE_FINALIZE, /*tp_flags*/ + __pyx_GeneratorType_slots +}; +#else /* CYTHON_USE_TYPE_SPECS */ + static PyTypeObject __pyx_GeneratorType_type = { PyVarObject_HEAD_INIT(0, 0) - "generator", /*tp_name*/ + __PYX_TYPE_MODULE_PREFIX "generator", /*tp_name*/ sizeof(__pyx_CoroutineObject), /*tp_basicsize*/ 0, /*tp_itemsize*/ (destructor) __Pyx_Coroutine_dealloc,/*tp_dealloc*/ 0, /*tp_print*/ 0, /*tp_getattr*/ 0, /*tp_setattr*/ - 0, /*tp_compare / tp_as_async*/ + 0, /*tp_as_async*/ 0, /*tp_repr*/ 0, /*tp_as_number*/ 0, /*tp_as_sequence*/ @@ -1933,14 +2080,22 @@ static PyTypeObject __pyx_GeneratorType_type = { #if PY_VERSION_HEX >= 0x030800b4 && PY_VERSION_HEX < 0x03090000 0, /*tp_print*/ #endif +#if CYTHON_COMPILING_IN_PYPY && PYPY_VERSION_NUM+0 >= 0x06000000 + 0, /*tp_pypy_flags*/ +#endif }; +#endif /* CYTHON_USE_TYPE_SPECS */ -static int __pyx_Generator_init(void) { +static int __pyx_Generator_init(PyObject *module) { +#if CYTHON_USE_TYPE_SPECS + __pyx_GeneratorType = __Pyx_FetchCommonTypeFromSpec(module, &__pyx_GeneratorType_spec, NULL); +#else + (void) module; // on Windows, C-API functions can't be used in slots statically __pyx_GeneratorType_type.tp_getattro = __Pyx_PyObject_GenericGetAttrNoDict; __pyx_GeneratorType_type.tp_iter = PyObject_SelfIter; - __pyx_GeneratorType = __Pyx_FetchCommonType(&__pyx_GeneratorType_type); +#endif if (unlikely(!__pyx_GeneratorType)) { return -1; } @@ -1967,7 +2122,7 @@ static void __Pyx__ReturnWithStopIteration(PyObject* value); /*proto*/ static void __Pyx__ReturnWithStopIteration(PyObject* value) { PyObject *exc, *args; -#if CYTHON_COMPILING_IN_CPYTHON || CYTHON_COMPILING_IN_PYSTON +#if CYTHON_COMPILING_IN_CPYTHON __Pyx_PyThreadState_declare if ((PY_VERSION_HEX >= 0x03030000 && PY_VERSION_HEX < 0x030500B1) || unlikely(PyTuple_Check(value) || PyExceptionInstance_Check(value))) { @@ -2096,7 +2251,7 @@ static int __Pyx_patch_abc(void) { if (CYTHON_REGISTER_ABCS && !abc_patched) { PyObject *module; module = PyImport_ImportModule((PY_MAJOR_VERSION >= 3) ? "collections.abc" : "collections"); - if (!module) { + if (unlikely(!module)) { PyErr_WriteUnraisable(NULL); if (unlikely(PyErr_WarnEx(PyExc_RuntimeWarning, ((PY_MAJOR_VERSION >= 3) ? @@ -2273,11 +2428,15 @@ old_types.add(_cython_generator_type) #define __Pyx_StopAsyncIteration_USED static PyObject *__Pyx_PyExc_StopAsyncIteration; -static int __pyx_StopAsyncIteration_init(void); /*proto*/ +static int __pyx_StopAsyncIteration_init(PyObject *module); /*proto*/ //////////////////// StopAsyncIteration //////////////////// #if PY_VERSION_HEX < 0x030500B1 +#if CYTHON_USE_TYPE_SPECS +#error Using async coroutines with type specs requires Python 3.5 or later. +#else + static PyTypeObject __Pyx__PyExc_StopAsyncIteration_type = { PyVarObject_HEAD_INIT(0, 0) "StopAsyncIteration", /*tp_name*/ @@ -2329,11 +2488,16 @@ static PyTypeObject __Pyx__PyExc_StopAsyncIteration_type = { #if PY_VERSION_HEX >= 0x030400a1 0, /*tp_finalize*/ #endif +#if CYTHON_COMPILING_IN_PYPY && PYPY_VERSION_NUM+0 >= 0x06000000 + 0, /*tp_pypy_flags*/ +#endif }; #endif +#endif -static int __pyx_StopAsyncIteration_init(void) { +static int __pyx_StopAsyncIteration_init(PyObject *module) { #if PY_VERSION_HEX >= 0x030500B1 + (void) module; __Pyx_PyExc_StopAsyncIteration = PyExc_StopAsyncIteration; #else PyObject *builtins = PyEval_GetBuiltins(); @@ -2351,10 +2515,11 @@ static int __pyx_StopAsyncIteration_init(void) { __Pyx__PyExc_StopAsyncIteration_type.tp_dictoffset = ((PyTypeObject*)PyExc_BaseException)->tp_dictoffset; __Pyx__PyExc_StopAsyncIteration_type.tp_base = (PyTypeObject*)PyExc_Exception; + (void) module; __Pyx_PyExc_StopAsyncIteration = (PyObject*) __Pyx_FetchCommonType(&__Pyx__PyExc_StopAsyncIteration_type); if (unlikely(!__Pyx_PyExc_StopAsyncIteration)) return -1; - if (builtins && unlikely(PyMapping_SetItemString(builtins, (char*) "StopAsyncIteration", __Pyx_PyExc_StopAsyncIteration) < 0)) + if (likely(builtins) && unlikely(PyMapping_SetItemString(builtins, (char*) "StopAsyncIteration", __Pyx_PyExc_StopAsyncIteration) < 0)) return -1; #endif return 0; diff --git a/Cython/Utility/CpdefEnums.pyx b/Cython/Utility/CpdefEnums.pyx index 148d776c2..aa19f45ed 100644 --- a/Cython/Utility/CpdefEnums.pyx +++ b/Cython/Utility/CpdefEnums.pyx @@ -6,10 +6,7 @@ cdef extern from *: int PY_VERSION_HEX cdef object __Pyx_OrderedDict -if PY_VERSION_HEX >= 0x02070000: - from collections import OrderedDict as __Pyx_OrderedDict -else: - __Pyx_OrderedDict = dict +from collections import OrderedDict as __Pyx_OrderedDict @cython.internal cdef class __Pyx_EnumMeta(type): @@ -23,8 +20,7 @@ cdef class __Pyx_EnumMeta(type): # @cython.internal cdef object __Pyx_EnumBase -class __Pyx_EnumBase(int): - __metaclass__ = __Pyx_EnumMeta +class __Pyx_EnumBase(int, metaclass=__Pyx_EnumMeta): def __new__(cls, value, name=None): for v in cls: if v == value: @@ -55,12 +51,38 @@ if PY_VERSION_HEX >= 0x03040000: ('{{item}}', {{item}}), {{endfor}} ])) + {{if enum_doc is not None}} + {{name}}.__doc__ = {{ repr(enum_doc) }} + {{endif}} + {{for item in items}} __Pyx_globals['{{item}}'] = {{name}}.{{item}} {{endfor}} else: class {{name}}(__Pyx_EnumBase): - pass + {{ repr(enum_doc) if enum_doc is not None else 'pass' }} {{for item in items}} __Pyx_globals['{{item}}'] = {{name}}({{item}}, '{{item}}') {{endfor}} + +#################### CppScopedEnumType #################### +#@requires: EnumBase +cdef dict __Pyx_globals = globals() + +if PY_VERSION_HEX >= 0x03040000: + # create new IntEnum() + __Pyx_globals["{{name}}"] = __Pyx_EnumBase('{{name}}', __Pyx_OrderedDict([ + {{for item in items}} + ('{{item}}', <{{underlying_type}}>({{name}}.{{item}})), + {{endfor}} + ])) + +else: + __Pyx_globals["{{name}}"] = type('{{name}}', (__Pyx_EnumBase,), {}) + {{for item in items}} + __Pyx_globals["{{name}}"](<{{underlying_type}}>({{name}}.{{item}}), '{{item}}') + {{endfor}} + +{{if enum_doc is not None}} +__Pyx_globals["{{name}}"].__doc__ = {{ repr(enum_doc) }} +{{endif}} diff --git a/Cython/Utility/CppConvert.pyx b/Cython/Utility/CppConvert.pyx index 03360e510..35f4c50ef 100644 --- a/Cython/Utility/CppConvert.pyx +++ b/Cython/Utility/CppConvert.pyx @@ -5,8 +5,8 @@ cdef extern from *: cdef cppclass string "{{type}}": - string() - string(char* c_str, size_t size) + string() except + + string(char* c_str, size_t size) except + cdef const char* __Pyx_PyObject_AsStringAndSize(object, Py_ssize_t*) except NULL @cname("{{cname}}") @@ -39,7 +39,7 @@ cdef inline object {{cname.replace("PyObject", py_type, 1)}}(const string& s): cdef extern from *: cdef cppclass vector "std::vector" [T]: - void push_back(T&) + void push_back(T&) except + @cname("{{cname}}") cdef vector[X] {{cname}}(object o) except *: @@ -52,20 +52,38 @@ cdef vector[X] {{cname}}(object o) except *: #################### vector.to_py #################### cdef extern from *: - cdef cppclass vector "const std::vector" [T]: + cdef cppclass vector "std::vector" [T]: size_t size() T& operator[](size_t) +cdef extern from "Python.h": + void Py_INCREF(object) + list PyList_New(Py_ssize_t size) + void PyList_SET_ITEM(object list, Py_ssize_t i, object o) + cdef Py_ssize_t PY_SSIZE_T_MAX + @cname("{{cname}}") -cdef object {{cname}}(vector[X]& v): - return [v[i] for i in range(v.size())] +cdef object {{cname}}(const vector[X]& v): + if v.size() > <size_t> PY_SSIZE_T_MAX: + raise MemoryError() + + o = PyList_New(<Py_ssize_t> v.size()) + + cdef Py_ssize_t i + cdef object item + for i in range(v.size()): + item = v[i] + Py_INCREF(item) + PyList_SET_ITEM(o, i, item) + + return o #################### list.from_py #################### cdef extern from *: cdef cppclass cpp_list "std::list" [T]: - void push_back(T&) + void push_back(T&) except + @cname("{{cname}}") cdef cpp_list[X] {{cname}}(object o) except *: @@ -87,14 +105,32 @@ cdef extern from *: bint operator!=(const_iterator) const_iterator begin() const_iterator end() + size_t size() + +cdef extern from "Python.h": + void Py_INCREF(object) + list PyList_New(Py_ssize_t size) + void PyList_SET_ITEM(object list, Py_ssize_t i, object o) + cdef Py_ssize_t PY_SSIZE_T_MAX @cname("{{cname}}") cdef object {{cname}}(const cpp_list[X]& v): - o = [] + if v.size() > <size_t> PY_SSIZE_T_MAX: + raise MemoryError() + + o = PyList_New(<Py_ssize_t> v.size()) + + cdef object item + cdef Py_ssize_t i = 0 cdef cpp_list[X].const_iterator iter = v.begin() + while iter != v.end(): - o.append(cython.operator.dereference(iter)) + item = cython.operator.dereference(iter) + Py_INCREF(item) + PyList_SET_ITEM(o, i, item) cython.operator.preincrement(iter) + i += 1 + return o @@ -102,7 +138,7 @@ cdef object {{cname}}(const cpp_list[X]& v): cdef extern from *: cdef cppclass set "std::{{maybe_unordered}}set" [T]: - void insert(T&) + void insert(T&) except + @cname("{{cname}}") cdef set[X] {{cname}}(object o) except *: @@ -127,19 +163,14 @@ cdef extern from *: @cname("{{cname}}") cdef object {{cname}}(const cpp_set[X]& s): - o = set() - cdef cpp_set[X].const_iterator iter = s.begin() - while iter != s.end(): - o.add(cython.operator.dereference(iter)) - cython.operator.preincrement(iter) - return o + return {v for v in s} #################### pair.from_py #################### cdef extern from *: cdef cppclass pair "std::pair" [T, U]: - pair() - pair(T&, U&) + pair() except + + pair(T&, U&) except + @cname("{{cname}}") cdef pair[X,Y] {{cname}}(object o) except *: @@ -163,19 +194,23 @@ cdef object {{cname}}(const pair[X,Y]& p): cdef extern from *: cdef cppclass pair "std::pair" [T, U]: - pair(T&, U&) + pair(T&, U&) except + cdef cppclass map "std::{{maybe_unordered}}map" [T, U]: - void insert(pair[T, U]&) + void insert(pair[T, U]&) except + cdef cppclass vector "std::vector" [T]: pass + int PY_MAJOR_VERSION @cname("{{cname}}") cdef map[X,Y] {{cname}}(object o) except *: - cdef dict d = o cdef map[X,Y] m - for key, value in d.iteritems(): - m.insert(pair[X,Y](<X>key, <Y>value)) + if PY_MAJOR_VERSION < 3: + for key, value in o.iteritems(): + m.insert(pair[X,Y](<X>key, <Y>value)) + else: + for key, value in o.items(): + m.insert(pair[X,Y](<X>key, <Y>value)) return m diff --git a/Cython/Utility/CppSupport.cpp b/Cython/Utility/CppSupport.cpp index b8fcff064..4e883539f 100644 --- a/Cython/Utility/CppSupport.cpp +++ b/Cython/Utility/CppSupport.cpp @@ -56,3 +56,43 @@ auto __Pyx_pythran_to_python(T &&value) -> decltype(to_python( using returnable_type = typename pythonic::returnable<typename std::remove_cv<typename std::remove_reference<T>::type>::type>::type; return to_python(returnable_type{std::forward<T>(value)}); } + +#define __Pyx_PythranShapeAccessor(x) (pythonic::builtins::getattr(pythonic::types::attr::SHAPE{}, x)) + +////////////// MoveIfSupported.proto ////////////////// + +#if __cplusplus >= 201103L || (defined(_MSC_VER) && _MSC_VER >= 1600) + // move should be defined for these versions of MSVC, but __cplusplus isn't set usefully + #include <utility> + #define __PYX_STD_MOVE_IF_SUPPORTED(x) std::move(x) +#else + #define __PYX_STD_MOVE_IF_SUPPORTED(x) x +#endif + +////////////// EnumClassDecl.proto ////////////////// + +#if defined (_MSC_VER) + #if PY_VERSION_HEX >= 0x03040000 && PY_VERSION_HEX < 0x03050000 + #define __PYX_ENUM_CLASS_DECL + #else + #define __PYX_ENUM_CLASS_DECL enum + #endif +#else + #define __PYX_ENUM_CLASS_DECL enum +#endif + +////////////// OptionalLocals.proto //////////////// +//@proto_block: utility_code_proto_before_types + +#if defined(CYTHON_USE_BOOST_OPTIONAL) + // fallback mode - std::optional is preferred but this gives + // people with a less up-to-date compiler a chance + #include <boost/optional.hpp> + #define __Pyx_Optional_Type boost::optional +#else + #include <optional> + // since std::optional is a C++17 features, a templated using declaration should be safe + // (although it could be replaced with a define) + template <typename T> + using __Pyx_Optional_Type = std::optional<T>; +#endif diff --git a/Cython/Utility/CythonFunction.c b/Cython/Utility/CythonFunction.c index c9f6dacaf..4e3847529 100644 --- a/Cython/Utility/CythonFunction.c +++ b/Cython/Utility/CythonFunction.c @@ -1,16 +1,25 @@ //////////////////// CythonFunctionShared.proto //////////////////// -#define __Pyx_CyFunction_USED 1 +#define __Pyx_CyFunction_USED #define __Pyx_CYFUNCTION_STATICMETHOD 0x01 #define __Pyx_CYFUNCTION_CLASSMETHOD 0x02 #define __Pyx_CYFUNCTION_CCLASS 0x04 +#define __Pyx_CYFUNCTION_COROUTINE 0x08 #define __Pyx_CyFunction_GetClosure(f) \ (((__pyx_CyFunctionObject *) (f))->func_closure) -#define __Pyx_CyFunction_GetClassObj(f) \ - (((__pyx_CyFunctionObject *) (f))->func_classobj) + +#if PY_VERSION_HEX < 0x030900B1 + #define __Pyx_CyFunction_GetClassObj(f) \ + (((__pyx_CyFunctionObject *) (f))->func_classobj) +#else + #define __Pyx_CyFunction_GetClassObj(f) \ + ((PyObject*) ((PyCMethodObject *) (f))->mm_class) +#endif +#define __Pyx_CyFunction_SetClassObj(f, classobj) \ + __Pyx__CyFunction_SetClassObj((__pyx_CyFunctionObject *) (f), (classobj)) #define __Pyx_CyFunction_Defaults(type, f) \ ((type *)(((__pyx_CyFunctionObject *) (f))->defaults)) @@ -19,7 +28,15 @@ typedef struct { +#if PY_VERSION_HEX < 0x030900B1 PyCFunctionObject func; +#else + // PEP-573: PyCFunctionObject + mm_class + PyCMethodObject func; +#endif +#if CYTHON_BACKPORT_VECTORCALL + __pyx_vectorcallfunc func_vectorcall; +#endif #if PY_VERSION_HEX < 0x030500A0 PyObject *func_weakreflist; #endif @@ -30,9 +47,10 @@ typedef struct { PyObject *func_globals; PyObject *func_code; PyObject *func_closure; +#if PY_VERSION_HEX < 0x030900B1 // No-args super() class cell PyObject *func_classobj; - +#endif // Dynamic default args and annotations void *defaults; int defaults_pyobjects; @@ -44,18 +62,26 @@ typedef struct { PyObject *defaults_kwdict; /* Const kwonly defaults dict */ PyObject *(*defaults_getter)(PyObject *); PyObject *func_annotations; /* function annotations dict */ + + // Coroutine marker + PyObject *func_is_coroutine; } __pyx_CyFunctionObject; +#if !CYTHON_USE_MODULE_STATE static PyTypeObject *__pyx_CyFunctionType = 0; +#endif -#define __Pyx_CyFunction_Check(obj) (__Pyx_TypeCheck(obj, __pyx_CyFunctionType)) +#define __Pyx_CyFunction_Check(obj) __Pyx_TypeCheck(obj, __pyx_CyFunctionType) +#define __Pyx_IsCyOrPyCFunction(obj) __Pyx_TypeCheck2(obj, __pyx_CyFunctionType, &PyCFunction_Type) +#define __Pyx_CyFunction_CheckExact(obj) __Pyx_IS_TYPE(obj, __pyx_CyFunctionType) static PyObject *__Pyx_CyFunction_Init(__pyx_CyFunctionObject* op, PyMethodDef *ml, int flags, PyObject* qualname, - PyObject *self, + PyObject *closure, PyObject *module, PyObject *globals, PyObject* code); +static CYTHON_INLINE void __Pyx__CyFunction_SetClassObj(__pyx_CyFunctionObject* f, PyObject* classobj); static CYTHON_INLINE void *__Pyx_CyFunction_InitDefaults(PyObject *m, size_t size, int pyobjects); @@ -67,25 +93,51 @@ static CYTHON_INLINE void __Pyx_CyFunction_SetAnnotationsDict(PyObject *m, PyObject *dict); -static int __pyx_CyFunction_init(void); +static int __pyx_CyFunction_init(PyObject *module); +#if CYTHON_METH_FASTCALL +static PyObject * __Pyx_CyFunction_Vectorcall_NOARGS(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames); +static PyObject * __Pyx_CyFunction_Vectorcall_O(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames); +static PyObject * __Pyx_CyFunction_Vectorcall_FASTCALL_KEYWORDS(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames); +static PyObject * __Pyx_CyFunction_Vectorcall_FASTCALL_KEYWORDS_METHOD(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames); +#if CYTHON_BACKPORT_VECTORCALL +#define __Pyx_CyFunction_func_vectorcall(f) (((__pyx_CyFunctionObject*)f)->func_vectorcall) +#else +#define __Pyx_CyFunction_func_vectorcall(f) (((PyCFunctionObject*)f)->vectorcall) +#endif +#endif //////////////////// CythonFunctionShared //////////////////// //@substitute: naming //@requires: CommonStructures.c::FetchCommonType -////@requires: ObjectHandling.c::PyObjectGetAttrStr - -#include <structmember.h> +//@requires: ObjectHandling.c::PyMethodNew +//@requires: ObjectHandling.c::PyVectorcallFastCallDict +//@requires: ModuleSetupCode.c::IncludeStructmemberH +//@requires: ObjectHandling.c::PyObjectGetAttrStr + +static CYTHON_INLINE void __Pyx__CyFunction_SetClassObj(__pyx_CyFunctionObject* f, PyObject* classobj) { +#if PY_VERSION_HEX < 0x030900B1 + __Pyx_Py_XDECREF_SET( + __Pyx_CyFunction_GetClassObj(f), + ((classobj) ? __Pyx_NewRef(classobj) : NULL)); +#else + __Pyx_Py_XDECREF_SET( + // assigning to "mm_class", which is a "PyTypeObject*" + ((PyCMethodObject *) (f))->mm_class, + (PyTypeObject*)((classobj) ? __Pyx_NewRef(classobj) : NULL)); +#endif +} static PyObject * -__Pyx_CyFunction_get_doc(__pyx_CyFunctionObject *op, CYTHON_UNUSED void *closure) +__Pyx_CyFunction_get_doc(__pyx_CyFunctionObject *op, void *closure) { + CYTHON_UNUSED_VAR(closure); if (unlikely(op->func_doc == NULL)) { - if (op->func.m_ml->ml_doc) { + if (((PyCFunctionObject*)op)->m_ml->ml_doc) { #if PY_MAJOR_VERSION >= 3 - op->func_doc = PyUnicode_FromString(op->func.m_ml->ml_doc); + op->func_doc = PyUnicode_FromString(((PyCFunctionObject*)op)->m_ml->ml_doc); #else - op->func_doc = PyString_FromString(op->func.m_ml->ml_doc); + op->func_doc = PyString_FromString(((PyCFunctionObject*)op)->m_ml->ml_doc); #endif if (unlikely(op->func_doc == NULL)) return NULL; @@ -99,27 +151,27 @@ __Pyx_CyFunction_get_doc(__pyx_CyFunctionObject *op, CYTHON_UNUSED void *closure } static int -__Pyx_CyFunction_set_doc(__pyx_CyFunctionObject *op, PyObject *value, CYTHON_UNUSED void *context) +__Pyx_CyFunction_set_doc(__pyx_CyFunctionObject *op, PyObject *value, void *context) { - PyObject *tmp = op->func_doc; + CYTHON_UNUSED_VAR(context); if (value == NULL) { // Mark as deleted value = Py_None; } Py_INCREF(value); - op->func_doc = value; - Py_XDECREF(tmp); + __Pyx_Py_XDECREF_SET(op->func_doc, value); return 0; } static PyObject * -__Pyx_CyFunction_get_name(__pyx_CyFunctionObject *op, CYTHON_UNUSED void *context) +__Pyx_CyFunction_get_name(__pyx_CyFunctionObject *op, void *context) { + CYTHON_UNUSED_VAR(context); if (unlikely(op->func_name == NULL)) { #if PY_MAJOR_VERSION >= 3 - op->func_name = PyUnicode_InternFromString(op->func.m_ml->ml_name); + op->func_name = PyUnicode_InternFromString(((PyCFunctionObject*)op)->m_ml->ml_name); #else - op->func_name = PyString_InternFromString(op->func.m_ml->ml_name); + op->func_name = PyString_InternFromString(((PyCFunctionObject*)op)->m_ml->ml_name); #endif if (unlikely(op->func_name == NULL)) return NULL; @@ -129,10 +181,9 @@ __Pyx_CyFunction_get_name(__pyx_CyFunctionObject *op, CYTHON_UNUSED void *contex } static int -__Pyx_CyFunction_set_name(__pyx_CyFunctionObject *op, PyObject *value, CYTHON_UNUSED void *context) +__Pyx_CyFunction_set_name(__pyx_CyFunctionObject *op, PyObject *value, void *context) { - PyObject *tmp; - + CYTHON_UNUSED_VAR(context); #if PY_MAJOR_VERSION >= 3 if (unlikely(value == NULL || !PyUnicode_Check(value))) #else @@ -143,25 +194,23 @@ __Pyx_CyFunction_set_name(__pyx_CyFunctionObject *op, PyObject *value, CYTHON_UN "__name__ must be set to a string object"); return -1; } - tmp = op->func_name; Py_INCREF(value); - op->func_name = value; - Py_XDECREF(tmp); + __Pyx_Py_XDECREF_SET(op->func_name, value); return 0; } static PyObject * -__Pyx_CyFunction_get_qualname(__pyx_CyFunctionObject *op, CYTHON_UNUSED void *context) +__Pyx_CyFunction_get_qualname(__pyx_CyFunctionObject *op, void *context) { + CYTHON_UNUSED_VAR(context); Py_INCREF(op->func_qualname); return op->func_qualname; } static int -__Pyx_CyFunction_set_qualname(__pyx_CyFunctionObject *op, PyObject *value, CYTHON_UNUSED void *context) +__Pyx_CyFunction_set_qualname(__pyx_CyFunctionObject *op, PyObject *value, void *context) { - PyObject *tmp; - + CYTHON_UNUSED_VAR(context); #if PY_MAJOR_VERSION >= 3 if (unlikely(value == NULL || !PyUnicode_Check(value))) #else @@ -172,28 +221,15 @@ __Pyx_CyFunction_set_qualname(__pyx_CyFunctionObject *op, PyObject *value, CYTHO "__qualname__ must be set to a string object"); return -1; } - tmp = op->func_qualname; Py_INCREF(value); - op->func_qualname = value; - Py_XDECREF(tmp); + __Pyx_Py_XDECREF_SET(op->func_qualname, value); return 0; } static PyObject * -__Pyx_CyFunction_get_self(__pyx_CyFunctionObject *m, CYTHON_UNUSED void *closure) -{ - PyObject *self; - - self = m->func_closure; - if (self == NULL) - self = Py_None; - Py_INCREF(self); - return self; -} - -static PyObject * -__Pyx_CyFunction_get_dict(__pyx_CyFunctionObject *op, CYTHON_UNUSED void *context) +__Pyx_CyFunction_get_dict(__pyx_CyFunctionObject *op, void *context) { + CYTHON_UNUSED_VAR(context); if (unlikely(op->func_dict == NULL)) { op->func_dict = PyDict_New(); if (unlikely(op->func_dict == NULL)) @@ -204,10 +240,9 @@ __Pyx_CyFunction_get_dict(__pyx_CyFunctionObject *op, CYTHON_UNUSED void *contex } static int -__Pyx_CyFunction_set_dict(__pyx_CyFunctionObject *op, PyObject *value, CYTHON_UNUSED void *context) +__Pyx_CyFunction_set_dict(__pyx_CyFunctionObject *op, PyObject *value, void *context) { - PyObject *tmp; - + CYTHON_UNUSED_VAR(context); if (unlikely(value == NULL)) { PyErr_SetString(PyExc_TypeError, "function's dictionary may not be deleted"); @@ -218,31 +253,33 @@ __Pyx_CyFunction_set_dict(__pyx_CyFunctionObject *op, PyObject *value, CYTHON_UN "setting function's dictionary to a non-dict"); return -1; } - tmp = op->func_dict; Py_INCREF(value); - op->func_dict = value; - Py_XDECREF(tmp); + __Pyx_Py_XDECREF_SET(op->func_dict, value); return 0; } static PyObject * -__Pyx_CyFunction_get_globals(__pyx_CyFunctionObject *op, CYTHON_UNUSED void *context) +__Pyx_CyFunction_get_globals(__pyx_CyFunctionObject *op, void *context) { + CYTHON_UNUSED_VAR(context); Py_INCREF(op->func_globals); return op->func_globals; } static PyObject * -__Pyx_CyFunction_get_closure(CYTHON_UNUSED __pyx_CyFunctionObject *op, CYTHON_UNUSED void *context) +__Pyx_CyFunction_get_closure(__pyx_CyFunctionObject *op, void *context) { + CYTHON_UNUSED_VAR(op); + CYTHON_UNUSED_VAR(context); Py_INCREF(Py_None); return Py_None; } static PyObject * -__Pyx_CyFunction_get_code(__pyx_CyFunctionObject *op, CYTHON_UNUSED void *context) +__Pyx_CyFunction_get_code(__pyx_CyFunctionObject *op, void *context) { PyObject* result = (op->func_code) ? op->func_code : Py_None; + CYTHON_UNUSED_VAR(context); Py_INCREF(result); return result; } @@ -273,29 +310,28 @@ __Pyx_CyFunction_init_defaults(__pyx_CyFunctionObject *op) { } static int -__Pyx_CyFunction_set_defaults(__pyx_CyFunctionObject *op, PyObject* value, CYTHON_UNUSED void *context) { - PyObject* tmp; +__Pyx_CyFunction_set_defaults(__pyx_CyFunctionObject *op, PyObject* value, void *context) { + CYTHON_UNUSED_VAR(context); if (!value) { // del => explicit None to prevent rebuilding value = Py_None; - } else if (value != Py_None && !PyTuple_Check(value)) { + } else if (unlikely(value != Py_None && !PyTuple_Check(value))) { PyErr_SetString(PyExc_TypeError, "__defaults__ must be set to a tuple object"); return -1; } Py_INCREF(value); - tmp = op->defaults_tuple; - op->defaults_tuple = value; - Py_XDECREF(tmp); + __Pyx_Py_XDECREF_SET(op->defaults_tuple, value); return 0; } static PyObject * -__Pyx_CyFunction_get_defaults(__pyx_CyFunctionObject *op, CYTHON_UNUSED void *context) { +__Pyx_CyFunction_get_defaults(__pyx_CyFunctionObject *op, void *context) { PyObject* result = op->defaults_tuple; + CYTHON_UNUSED_VAR(context); if (unlikely(!result)) { if (op->defaults_getter) { - if (__Pyx_CyFunction_init_defaults(op) < 0) return NULL; + if (unlikely(__Pyx_CyFunction_init_defaults(op) < 0)) return NULL; result = op->defaults_tuple; } else { result = Py_None; @@ -306,29 +342,28 @@ __Pyx_CyFunction_get_defaults(__pyx_CyFunctionObject *op, CYTHON_UNUSED void *co } static int -__Pyx_CyFunction_set_kwdefaults(__pyx_CyFunctionObject *op, PyObject* value, CYTHON_UNUSED void *context) { - PyObject* tmp; +__Pyx_CyFunction_set_kwdefaults(__pyx_CyFunctionObject *op, PyObject* value, void *context) { + CYTHON_UNUSED_VAR(context); if (!value) { // del => explicit None to prevent rebuilding value = Py_None; - } else if (value != Py_None && !PyDict_Check(value)) { + } else if (unlikely(value != Py_None && !PyDict_Check(value))) { PyErr_SetString(PyExc_TypeError, "__kwdefaults__ must be set to a dict object"); return -1; } Py_INCREF(value); - tmp = op->defaults_kwdict; - op->defaults_kwdict = value; - Py_XDECREF(tmp); + __Pyx_Py_XDECREF_SET(op->defaults_kwdict, value); return 0; } static PyObject * -__Pyx_CyFunction_get_kwdefaults(__pyx_CyFunctionObject *op, CYTHON_UNUSED void *context) { +__Pyx_CyFunction_get_kwdefaults(__pyx_CyFunctionObject *op, void *context) { PyObject* result = op->defaults_kwdict; + CYTHON_UNUSED_VAR(context); if (unlikely(!result)) { if (op->defaults_getter) { - if (__Pyx_CyFunction_init_defaults(op) < 0) return NULL; + if (unlikely(__Pyx_CyFunction_init_defaults(op) < 0)) return NULL; result = op->defaults_kwdict; } else { result = Py_None; @@ -339,25 +374,24 @@ __Pyx_CyFunction_get_kwdefaults(__pyx_CyFunctionObject *op, CYTHON_UNUSED void * } static int -__Pyx_CyFunction_set_annotations(__pyx_CyFunctionObject *op, PyObject* value, CYTHON_UNUSED void *context) { - PyObject* tmp; +__Pyx_CyFunction_set_annotations(__pyx_CyFunctionObject *op, PyObject* value, void *context) { + CYTHON_UNUSED_VAR(context); if (!value || value == Py_None) { value = NULL; - } else if (!PyDict_Check(value)) { + } else if (unlikely(!PyDict_Check(value))) { PyErr_SetString(PyExc_TypeError, "__annotations__ must be set to a dict object"); return -1; } Py_XINCREF(value); - tmp = op->func_annotations; - op->func_annotations = value; - Py_XDECREF(tmp); + __Pyx_Py_XDECREF_SET(op->func_annotations, value); return 0; } static PyObject * -__Pyx_CyFunction_get_annotations(__pyx_CyFunctionObject *op, CYTHON_UNUSED void *context) { +__Pyx_CyFunction_get_annotations(__pyx_CyFunctionObject *op, void *context) { PyObject* result = op->func_annotations; + CYTHON_UNUSED_VAR(context); if (unlikely(!result)) { result = PyDict_New(); if (unlikely(!result)) return NULL; @@ -367,10 +401,44 @@ __Pyx_CyFunction_get_annotations(__pyx_CyFunctionObject *op, CYTHON_UNUSED void return result; } +static PyObject * +__Pyx_CyFunction_get_is_coroutine(__pyx_CyFunctionObject *op, void *context) { + int is_coroutine; + CYTHON_UNUSED_VAR(context); + if (op->func_is_coroutine) { + return __Pyx_NewRef(op->func_is_coroutine); + } + + is_coroutine = op->flags & __Pyx_CYFUNCTION_COROUTINE; +#if PY_VERSION_HEX >= 0x03050000 + if (is_coroutine) { + PyObject *module, *fromlist, *marker = PYIDENT("_is_coroutine"); + fromlist = PyList_New(1); + if (unlikely(!fromlist)) return NULL; + Py_INCREF(marker); + PyList_SET_ITEM(fromlist, 0, marker); + module = PyImport_ImportModuleLevelObject(PYIDENT("asyncio.coroutines"), NULL, NULL, fromlist, 0); + Py_DECREF(fromlist); + if (unlikely(!module)) goto ignore; + op->func_is_coroutine = __Pyx_PyObject_GetAttrStr(module, marker); + Py_DECREF(module); + if (likely(op->func_is_coroutine)) { + return __Pyx_NewRef(op->func_is_coroutine); + } +ignore: + PyErr_Clear(); + } +#endif + + op->func_is_coroutine = __Pyx_PyBool_FromLong(is_coroutine); + return __Pyx_NewRef(op->func_is_coroutine); +} + //#if PY_VERSION_HEX >= 0x030400C1 //static PyObject * -//__Pyx_CyFunction_get_signature(__pyx_CyFunctionObject *op, CYTHON_UNUSED void *context) { +//__Pyx_CyFunction_get_signature(__pyx_CyFunctionObject *op, void *context) { // PyObject *inspect_module, *signature_class, *signature; +// CYTHON_UNUSED_VAR(context); // // from inspect import Signature // inspect_module = PyImport_ImportModuleLevelObject(PYIDENT("inspect"), NULL, NULL, NULL, 0); // if (unlikely(!inspect_module)) @@ -398,7 +466,6 @@ static PyGetSetDef __pyx_CyFunction_getsets[] = { {(char *) "func_name", (getter)__Pyx_CyFunction_get_name, (setter)__Pyx_CyFunction_set_name, 0, 0}, {(char *) "__name__", (getter)__Pyx_CyFunction_get_name, (setter)__Pyx_CyFunction_set_name, 0, 0}, {(char *) "__qualname__", (getter)__Pyx_CyFunction_get_qualname, (setter)__Pyx_CyFunction_set_qualname, 0, 0}, - {(char *) "__self__", (getter)__Pyx_CyFunction_get_self, 0, 0, 0}, {(char *) "func_dict", (getter)__Pyx_CyFunction_get_dict, (setter)__Pyx_CyFunction_set_dict, 0, 0}, {(char *) "__dict__", (getter)__Pyx_CyFunction_get_dict, (setter)__Pyx_CyFunction_set_dict, 0, 0}, {(char *) "func_globals", (getter)__Pyx_CyFunction_get_globals, 0, 0, 0}, @@ -411,6 +478,7 @@ static PyGetSetDef __pyx_CyFunction_getsets[] = { {(char *) "__defaults__", (getter)__Pyx_CyFunction_get_defaults, (setter)__Pyx_CyFunction_set_defaults, 0, 0}, {(char *) "__kwdefaults__", (getter)__Pyx_CyFunction_get_kwdefaults, (setter)__Pyx_CyFunction_set_kwdefaults, 0, 0}, {(char *) "__annotations__", (getter)__Pyx_CyFunction_get_annotations, (setter)__Pyx_CyFunction_set_annotations, 0, 0}, + {(char *) "_is_coroutine", (getter)__Pyx_CyFunction_get_is_coroutine, 0, 0, 0}, //#if PY_VERSION_HEX >= 0x030400C1 // {(char *) "__signature__", (getter)__Pyx_CyFunction_get_signature, 0, 0, 0}, //#endif @@ -418,18 +486,34 @@ static PyGetSetDef __pyx_CyFunction_getsets[] = { }; static PyMemberDef __pyx_CyFunction_members[] = { - {(char *) "__module__", T_OBJECT, offsetof(PyCFunctionObject, m_module), PY_WRITE_RESTRICTED, 0}, + {(char *) "__module__", T_OBJECT, offsetof(PyCFunctionObject, m_module), 0, 0}, +#if CYTHON_USE_TYPE_SPECS + {(char *) "__dictoffset__", T_PYSSIZET, offsetof(__pyx_CyFunctionObject, func_dict), READONLY, 0}, +#if CYTHON_METH_FASTCALL +#if CYTHON_BACKPORT_VECTORCALL + {(char *) "__vectorcalloffset__", T_PYSSIZET, offsetof(__pyx_CyFunctionObject, func_vectorcall), READONLY, 0}, +#else + {(char *) "__vectorcalloffset__", T_PYSSIZET, offsetof(PyCFunctionObject, vectorcall), READONLY, 0}, +#endif +#endif +#if PY_VERSION_HEX < 0x030500A0 + {(char *) "__weaklistoffset__", T_PYSSIZET, offsetof(__pyx_CyFunctionObject, func_weakreflist), READONLY, 0}, +#else + {(char *) "__weaklistoffset__", T_PYSSIZET, offsetof(PyCFunctionObject, m_weakreflist), READONLY, 0}, +#endif +#endif {0, 0, 0, 0, 0} }; static PyObject * -__Pyx_CyFunction_reduce(__pyx_CyFunctionObject *m, CYTHON_UNUSED PyObject *args) +__Pyx_CyFunction_reduce(__pyx_CyFunctionObject *m, PyObject *args) { + CYTHON_UNUSED_VAR(args); #if PY_MAJOR_VERSION >= 3 Py_INCREF(m->func_qualname); return m->func_qualname; #else - return PyString_FromString(m->func.m_ml->ml_name); + return PyString_FromString(((PyCFunctionObject*)m)->m_ml->ml_name); #endif } @@ -442,27 +526,32 @@ static PyMethodDef __pyx_CyFunction_methods[] = { #if PY_VERSION_HEX < 0x030500A0 #define __Pyx_CyFunction_weakreflist(cyfunc) ((cyfunc)->func_weakreflist) #else -#define __Pyx_CyFunction_weakreflist(cyfunc) ((cyfunc)->func.m_weakreflist) +#define __Pyx_CyFunction_weakreflist(cyfunc) (((PyCFunctionObject*)cyfunc)->m_weakreflist) #endif static PyObject *__Pyx_CyFunction_Init(__pyx_CyFunctionObject *op, PyMethodDef *ml, int flags, PyObject* qualname, PyObject *closure, PyObject *module, PyObject* globals, PyObject* code) { + PyCFunctionObject *cf = (PyCFunctionObject*) op; if (unlikely(op == NULL)) return NULL; op->flags = flags; __Pyx_CyFunction_weakreflist(op) = NULL; - op->func.m_ml = ml; - op->func.m_self = (PyObject *) op; + cf->m_ml = ml; + cf->m_self = (PyObject *) op; Py_XINCREF(closure); op->func_closure = closure; Py_XINCREF(module); - op->func.m_module = module; + cf->m_module = module; op->func_dict = NULL; op->func_name = NULL; Py_INCREF(qualname); op->func_qualname = qualname; op->func_doc = NULL; +#if PY_VERSION_HEX < 0x030900B1 op->func_classobj = NULL; +#else + ((PyCMethodObject*)op)->mm_class = NULL; +#endif op->func_globals = globals; Py_INCREF(op->func_globals); Py_XINCREF(code); @@ -475,6 +564,32 @@ static PyObject *__Pyx_CyFunction_Init(__pyx_CyFunctionObject *op, PyMethodDef * op->defaults_kwdict = NULL; op->defaults_getter = NULL; op->func_annotations = NULL; + op->func_is_coroutine = NULL; +#if CYTHON_METH_FASTCALL + switch (ml->ml_flags & (METH_VARARGS | METH_FASTCALL | METH_NOARGS | METH_O | METH_KEYWORDS | METH_METHOD)) { + case METH_NOARGS: + __Pyx_CyFunction_func_vectorcall(op) = __Pyx_CyFunction_Vectorcall_NOARGS; + break; + case METH_O: + __Pyx_CyFunction_func_vectorcall(op) = __Pyx_CyFunction_Vectorcall_O; + break; + // case METH_FASTCALL is not used + case METH_METHOD | METH_FASTCALL | METH_KEYWORDS: + __Pyx_CyFunction_func_vectorcall(op) = __Pyx_CyFunction_Vectorcall_FASTCALL_KEYWORDS_METHOD; + break; + case METH_FASTCALL | METH_KEYWORDS: + __Pyx_CyFunction_func_vectorcall(op) = __Pyx_CyFunction_Vectorcall_FASTCALL_KEYWORDS; + break; + // case METH_VARARGS is not used + case METH_VARARGS | METH_KEYWORDS: + __Pyx_CyFunction_func_vectorcall(op) = NULL; + break; + default: + PyErr_SetString(PyExc_SystemError, "Bad call flags for CyFunction"); + Py_DECREF(op); + return NULL; + } +#endif return (PyObject *) op; } @@ -482,17 +597,26 @@ static int __Pyx_CyFunction_clear(__pyx_CyFunctionObject *m) { Py_CLEAR(m->func_closure); - Py_CLEAR(m->func.m_module); + Py_CLEAR(((PyCFunctionObject*)m)->m_module); Py_CLEAR(m->func_dict); Py_CLEAR(m->func_name); Py_CLEAR(m->func_qualname); Py_CLEAR(m->func_doc); Py_CLEAR(m->func_globals); Py_CLEAR(m->func_code); - Py_CLEAR(m->func_classobj); +#if PY_VERSION_HEX < 0x030900B1 + Py_CLEAR(__Pyx_CyFunction_GetClassObj(m)); +#else + { + PyObject *cls = (PyObject*) ((PyCMethodObject *) (m))->mm_class; + ((PyCMethodObject *) (m))->mm_class = NULL; + Py_XDECREF(cls); + } +#endif Py_CLEAR(m->defaults_tuple); Py_CLEAR(m->defaults_kwdict); Py_CLEAR(m->func_annotations); + Py_CLEAR(m->func_is_coroutine); if (m->defaults) { PyObject **pydefaults = __Pyx_CyFunction_Defaults(PyObject *, m); @@ -513,7 +637,7 @@ static void __Pyx__CyFunction_dealloc(__pyx_CyFunctionObject *m) if (__Pyx_CyFunction_weakreflist(m) != NULL) PyObject_ClearWeakRefs((PyObject *) m); __Pyx_CyFunction_clear(m); - PyObject_GC_Del(m); + __Pyx_PyHeapTypeObject_GC_Del(m); } static void __Pyx_CyFunction_dealloc(__pyx_CyFunctionObject *m) @@ -525,16 +649,17 @@ static void __Pyx_CyFunction_dealloc(__pyx_CyFunctionObject *m) static int __Pyx_CyFunction_traverse(__pyx_CyFunctionObject *m, visitproc visit, void *arg) { Py_VISIT(m->func_closure); - Py_VISIT(m->func.m_module); + Py_VISIT(((PyCFunctionObject*)m)->m_module); Py_VISIT(m->func_dict); Py_VISIT(m->func_name); Py_VISIT(m->func_qualname); Py_VISIT(m->func_doc); Py_VISIT(m->func_globals); Py_VISIT(m->func_code); - Py_VISIT(m->func_classobj); + Py_VISIT(__Pyx_CyFunction_GetClassObj(m)); Py_VISIT(m->defaults_tuple); Py_VISIT(m->defaults_kwdict); + Py_VISIT(m->func_is_coroutine); if (m->defaults) { PyObject **pydefaults = __Pyx_CyFunction_Defaults(PyObject *, m); @@ -547,28 +672,6 @@ static int __Pyx_CyFunction_traverse(__pyx_CyFunctionObject *m, visitproc visit, return 0; } -static PyObject *__Pyx_CyFunction_descr_get(PyObject *func, PyObject *obj, PyObject *type) -{ -#if PY_MAJOR_VERSION < 3 - __pyx_CyFunctionObject *m = (__pyx_CyFunctionObject *) func; - - if (m->flags & __Pyx_CYFUNCTION_STATICMETHOD) { - Py_INCREF(func); - return func; - } - - if (m->flags & __Pyx_CYFUNCTION_CLASSMETHOD) { - if (type == NULL) - type = (PyObject *)(Py_TYPE(obj)); - return __Pyx_PyMethod_New(func, type, (PyObject *)(Py_TYPE(type))); - } - - if (obj == Py_None) - obj = NULL; -#endif - return __Pyx_PyMethod_New(func, obj, type); -} - static PyObject* __Pyx_CyFunction_repr(__pyx_CyFunctionObject *op) { @@ -628,10 +731,7 @@ static PyObject * __Pyx_CyFunction_CallMethod(PyObject *func, PyObject *self, Py } break; default: - PyErr_SetString(PyExc_SystemError, "Bad call flags in " - "__Pyx_CyFunction_Call. METH_OLDARGS is no " - "longer supported!"); - + PyErr_SetString(PyExc_SystemError, "Bad call flags for CyFunction"); return NULL; } PyErr_Format(PyExc_TypeError, "%.200s() takes no keyword arguments", @@ -646,6 +746,22 @@ static CYTHON_INLINE PyObject *__Pyx_CyFunction_Call(PyObject *func, PyObject *a static PyObject *__Pyx_CyFunction_CallAsMethod(PyObject *func, PyObject *args, PyObject *kw) { PyObject *result; __pyx_CyFunctionObject *cyfunc = (__pyx_CyFunctionObject *) func; + +#if CYTHON_METH_FASTCALL + // Prefer vectorcall if available. This is not the typical case, as + // CPython would normally use vectorcall directly instead of tp_call. + __pyx_vectorcallfunc vc = __Pyx_CyFunction_func_vectorcall(cyfunc); + if (vc) { +#if CYTHON_ASSUME_SAFE_MACROS + return __Pyx_PyVectorcall_FastCallDict(func, vc, &PyTuple_GET_ITEM(args, 0), (size_t)PyTuple_GET_SIZE(args), kw); +#else + // avoid unused function warning + (void) &__Pyx_PyVectorcall_FastCallDict; + return PyVectorcall_Call(func, args, kw); +#endif + } +#endif + if ((cyfunc->flags & __Pyx_CYFUNCTION_CCLASS) && !(cyfunc->flags & __Pyx_CYFUNCTION_STATICMETHOD)) { Py_ssize_t argc; PyObject *new_args; @@ -671,19 +787,198 @@ static PyObject *__Pyx_CyFunction_CallAsMethod(PyObject *func, PyObject *args, P return result; } +#if CYTHON_METH_FASTCALL +// Check that kwnames is empty (if you want to allow keyword arguments, +// simply pass kwnames=NULL) and figure out what to do with "self". +// Return value: +// 1: self = args[0] +// 0: self = cyfunc->func.m_self +// -1: error +static CYTHON_INLINE int __Pyx_CyFunction_Vectorcall_CheckArgs(__pyx_CyFunctionObject *cyfunc, Py_ssize_t nargs, PyObject *kwnames) +{ + int ret = 0; + if ((cyfunc->flags & __Pyx_CYFUNCTION_CCLASS) && !(cyfunc->flags & __Pyx_CYFUNCTION_STATICMETHOD)) { + if (unlikely(nargs < 1)) { + PyErr_Format(PyExc_TypeError, "%.200s() needs an argument", + ((PyCFunctionObject*)cyfunc)->m_ml->ml_name); + return -1; + } + ret = 1; + } + if (unlikely(kwnames) && unlikely(PyTuple_GET_SIZE(kwnames))) { + PyErr_Format(PyExc_TypeError, + "%.200s() takes no keyword arguments", ((PyCFunctionObject*)cyfunc)->m_ml->ml_name); + return -1; + } + return ret; +} + +static PyObject * __Pyx_CyFunction_Vectorcall_NOARGS(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames) +{ + __pyx_CyFunctionObject *cyfunc = (__pyx_CyFunctionObject *)func; + PyMethodDef* def = ((PyCFunctionObject*)cyfunc)->m_ml; +#if CYTHON_BACKPORT_VECTORCALL + Py_ssize_t nargs = (Py_ssize_t)nargsf; +#else + Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); +#endif + PyObject *self; + switch (__Pyx_CyFunction_Vectorcall_CheckArgs(cyfunc, nargs, kwnames)) { + case 1: + self = args[0]; + args += 1; + nargs -= 1; + break; + case 0: + self = ((PyCFunctionObject*)cyfunc)->m_self; + break; + default: + return NULL; + } + + if (unlikely(nargs != 0)) { + PyErr_Format(PyExc_TypeError, + "%.200s() takes no arguments (%" CYTHON_FORMAT_SSIZE_T "d given)", + def->ml_name, nargs); + return NULL; + } + return def->ml_meth(self, NULL); +} + +static PyObject * __Pyx_CyFunction_Vectorcall_O(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames) +{ + __pyx_CyFunctionObject *cyfunc = (__pyx_CyFunctionObject *)func; + PyMethodDef* def = ((PyCFunctionObject*)cyfunc)->m_ml; +#if CYTHON_BACKPORT_VECTORCALL + Py_ssize_t nargs = (Py_ssize_t)nargsf; +#else + Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); +#endif + PyObject *self; + switch (__Pyx_CyFunction_Vectorcall_CheckArgs(cyfunc, nargs, kwnames)) { + case 1: + self = args[0]; + args += 1; + nargs -= 1; + break; + case 0: + self = ((PyCFunctionObject*)cyfunc)->m_self; + break; + default: + return NULL; + } + + if (unlikely(nargs != 1)) { + PyErr_Format(PyExc_TypeError, + "%.200s() takes exactly one argument (%" CYTHON_FORMAT_SSIZE_T "d given)", + def->ml_name, nargs); + return NULL; + } + return def->ml_meth(self, args[0]); +} + +static PyObject * __Pyx_CyFunction_Vectorcall_FASTCALL_KEYWORDS(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames) +{ + __pyx_CyFunctionObject *cyfunc = (__pyx_CyFunctionObject *)func; + PyMethodDef* def = ((PyCFunctionObject*)cyfunc)->m_ml; +#if CYTHON_BACKPORT_VECTORCALL + Py_ssize_t nargs = (Py_ssize_t)nargsf; +#else + Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); +#endif + PyObject *self; + switch (__Pyx_CyFunction_Vectorcall_CheckArgs(cyfunc, nargs, NULL)) { + case 1: + self = args[0]; + args += 1; + nargs -= 1; + break; + case 0: + self = ((PyCFunctionObject*)cyfunc)->m_self; + break; + default: + return NULL; + } + + return ((_PyCFunctionFastWithKeywords)(void(*)(void))def->ml_meth)(self, args, nargs, kwnames); +} + +static PyObject * __Pyx_CyFunction_Vectorcall_FASTCALL_KEYWORDS_METHOD(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames) +{ + __pyx_CyFunctionObject *cyfunc = (__pyx_CyFunctionObject *)func; + PyMethodDef* def = ((PyCFunctionObject*)cyfunc)->m_ml; + PyTypeObject *cls = (PyTypeObject *) __Pyx_CyFunction_GetClassObj(cyfunc); +#if CYTHON_BACKPORT_VECTORCALL + Py_ssize_t nargs = (Py_ssize_t)nargsf; +#else + Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); +#endif + PyObject *self; + switch (__Pyx_CyFunction_Vectorcall_CheckArgs(cyfunc, nargs, NULL)) { + case 1: + self = args[0]; + args += 1; + nargs -= 1; + break; + case 0: + self = ((PyCFunctionObject*)cyfunc)->m_self; + break; + default: + return NULL; + } + + return ((__Pyx_PyCMethod)(void(*)(void))def->ml_meth)(self, cls, args, nargs, kwnames); +} +#endif + +#if CYTHON_USE_TYPE_SPECS +static PyType_Slot __pyx_CyFunctionType_slots[] = { + {Py_tp_dealloc, (void *)__Pyx_CyFunction_dealloc}, + {Py_tp_repr, (void *)__Pyx_CyFunction_repr}, + {Py_tp_call, (void *)__Pyx_CyFunction_CallAsMethod}, + {Py_tp_traverse, (void *)__Pyx_CyFunction_traverse}, + {Py_tp_clear, (void *)__Pyx_CyFunction_clear}, + {Py_tp_methods, (void *)__pyx_CyFunction_methods}, + {Py_tp_members, (void *)__pyx_CyFunction_members}, + {Py_tp_getset, (void *)__pyx_CyFunction_getsets}, + {Py_tp_descr_get, (void *)__Pyx_PyMethod_New}, + {0, 0}, +}; + +static PyType_Spec __pyx_CyFunctionType_spec = { + __PYX_TYPE_MODULE_PREFIX "cython_function_or_method", + sizeof(__pyx_CyFunctionObject), + 0, +#ifdef Py_TPFLAGS_METHOD_DESCRIPTOR + Py_TPFLAGS_METHOD_DESCRIPTOR | +#endif +#ifdef _Py_TPFLAGS_HAVE_VECTORCALL + _Py_TPFLAGS_HAVE_VECTORCALL | +#endif + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_BASETYPE, /*tp_flags*/ + __pyx_CyFunctionType_slots +}; +#else /* CYTHON_USE_TYPE_SPECS */ + static PyTypeObject __pyx_CyFunctionType_type = { PyVarObject_HEAD_INIT(0, 0) - "cython_function_or_method", /*tp_name*/ + __PYX_TYPE_MODULE_PREFIX "cython_function_or_method", /*tp_name*/ sizeof(__pyx_CyFunctionObject), /*tp_basicsize*/ 0, /*tp_itemsize*/ (destructor) __Pyx_CyFunction_dealloc, /*tp_dealloc*/ +#if !CYTHON_METH_FASTCALL 0, /*tp_print*/ +#elif CYTHON_BACKPORT_VECTORCALL + (printfunc)offsetof(__pyx_CyFunctionObject, func_vectorcall), /*tp_vectorcall_offset backported into tp_print*/ +#else + offsetof(PyCFunctionObject, vectorcall), /*tp_vectorcall_offset*/ +#endif 0, /*tp_getattr*/ 0, /*tp_setattr*/ #if PY_MAJOR_VERSION < 3 0, /*tp_compare*/ #else - 0, /*reserved*/ + 0, /*tp_as_async*/ #endif (reprfunc) __Pyx_CyFunction_repr, /*tp_repr*/ 0, /*tp_as_number*/ @@ -695,7 +990,13 @@ static PyTypeObject __pyx_CyFunctionType_type = { 0, /*tp_getattro*/ 0, /*tp_setattro*/ 0, /*tp_as_buffer*/ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /*tp_flags*/ +#ifdef Py_TPFLAGS_METHOD_DESCRIPTOR + Py_TPFLAGS_METHOD_DESCRIPTOR | +#endif +#ifdef _Py_TPFLAGS_HAVE_VECTORCALL + _Py_TPFLAGS_HAVE_VECTORCALL | +#endif + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_BASETYPE, /*tp_flags*/ 0, /*tp_doc*/ (traverseproc) __Pyx_CyFunction_traverse, /*tp_traverse*/ (inquiry) __Pyx_CyFunction_clear, /*tp_clear*/ @@ -712,7 +1013,7 @@ static PyTypeObject __pyx_CyFunctionType_type = { __pyx_CyFunction_getsets, /*tp_getset*/ 0, /*tp_base*/ 0, /*tp_dict*/ - __Pyx_CyFunction_descr_get, /*tp_descr_get*/ + __Pyx_PyMethod_New, /*tp_descr_get*/ 0, /*tp_descr_set*/ offsetof(__pyx_CyFunctionObject, func_dict),/*tp_dictoffset*/ 0, /*tp_init*/ @@ -736,11 +1037,20 @@ static PyTypeObject __pyx_CyFunctionType_type = { #if PY_VERSION_HEX >= 0x030800b4 && PY_VERSION_HEX < 0x03090000 0, /*tp_print*/ #endif +#if CYTHON_COMPILING_IN_PYPY && PYPY_VERSION_NUM+0 >= 0x06000000 + 0, /*tp_pypy_flags*/ +#endif }; +#endif /* CYTHON_USE_TYPE_SPECS */ -static int __pyx_CyFunction_init(void) { +static int __pyx_CyFunction_init(PyObject *module) { +#if CYTHON_USE_TYPE_SPECS + __pyx_CyFunctionType = __Pyx_FetchCommonTypeFromSpec(module, &__pyx_CyFunctionType_spec, NULL); +#else + (void) module; __pyx_CyFunctionType = __Pyx_FetchCommonType(&__pyx_CyFunctionType_type); +#endif if (unlikely(__pyx_CyFunctionType == NULL)) { return -1; } @@ -820,8 +1130,7 @@ static int __Pyx_CyFunction_InitClassCell(PyObject *cyfunctions, PyObject *class if (unlikely(!m)) return -1; #endif - Py_INCREF(classobj); - m->func_classobj = classobj; + __Pyx_CyFunction_SetClassObj(m, classobj); #if !(CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS) Py_DECREF((PyObject*)m); #endif @@ -835,7 +1144,6 @@ static int __Pyx_CyFunction_InitClassCell(PyObject *cyfunctions, PyObject *class typedef struct { __pyx_CyFunctionObject func; PyObject *__signatures__; - PyObject *type; PyObject *self; } __pyx_FusedFunctionObject; @@ -845,8 +1153,10 @@ static PyObject *__pyx_FusedFunction_New(PyMethodDef *ml, int flags, PyObject *code); static int __pyx_FusedFunction_clear(__pyx_FusedFunctionObject *self); +#if !CYTHON_USE_MODULE_STATE static PyTypeObject *__pyx_FusedFunctionType = NULL; -static int __pyx_FusedFunction_init(void); +#endif +static int __pyx_FusedFunction_init(PyObject *module); #define __Pyx_FusedFunction_USED @@ -867,7 +1177,6 @@ __pyx_FusedFunction_New(PyMethodDef *ml, int flags, if (likely(op)) { __pyx_FusedFunctionObject *fusedfunc = (__pyx_FusedFunctionObject *) op; fusedfunc->__signatures__ = NULL; - fusedfunc->type = NULL; fusedfunc->self = NULL; PyObject_GC_Track(op); } @@ -879,7 +1188,6 @@ __pyx_FusedFunction_dealloc(__pyx_FusedFunctionObject *self) { PyObject_GC_UnTrack(self); Py_CLEAR(self->self); - Py_CLEAR(self->type); Py_CLEAR(self->__signatures__); __Pyx__CyFunction_dealloc((__pyx_CyFunctionObject *) self); } @@ -890,7 +1198,6 @@ __pyx_FusedFunction_traverse(__pyx_FusedFunctionObject *self, void *arg) { Py_VISIT(self->self); - Py_VISIT(self->type); Py_VISIT(self->__signatures__); return __Pyx_CyFunction_traverse((__pyx_CyFunctionObject *) self, visit, arg); } @@ -899,7 +1206,6 @@ static int __pyx_FusedFunction_clear(__pyx_FusedFunctionObject *self) { Py_CLEAR(self->self); - Py_CLEAR(self->type); Py_CLEAR(self->__signatures__); return __Pyx_CyFunction_clear((__pyx_CyFunctionObject *) self); } @@ -929,7 +1235,7 @@ __pyx_FusedFunction_descr_get(PyObject *self, PyObject *obj, PyObject *type) ((PyCFunctionObject *) func)->m_module, ((__pyx_CyFunctionObject *) func)->func_globals, ((__pyx_CyFunctionObject *) func)->func_code); - if (!meth) + if (unlikely(!meth)) return NULL; // defaults needs copying fully rather than just copying the pointer @@ -939,9 +1245,10 @@ __pyx_FusedFunction_descr_get(PyObject *self, PyObject *obj, PyObject *type) PyObject **pydefaults; int i; - if (!__Pyx_CyFunction_InitDefaults((PyObject*)meth, - func->func.defaults_size, - func->func.defaults_pyobjects)) { + if (unlikely(!__Pyx_CyFunction_InitDefaults( + (PyObject*)meth, + func->func.defaults_size, + func->func.defaults_pyobjects))) { Py_XDECREF((PyObject*)meth); return NULL; } @@ -952,15 +1259,11 @@ __pyx_FusedFunction_descr_get(PyObject *self, PyObject *obj, PyObject *type) Py_XINCREF(pydefaults[i]); } - Py_XINCREF(func->func.func_classobj); - meth->func.func_classobj = func->func.func_classobj; + __Pyx_CyFunction_SetClassObj(meth, __Pyx_CyFunction_GetClassObj(func)); Py_XINCREF(func->__signatures__); meth->__signatures__ = func->__signatures__; - Py_XINCREF(type); - meth->type = type; - Py_XINCREF(func->func.defaults_tuple); meth->func.defaults_tuple = func->func.defaults_tuple; @@ -974,12 +1277,18 @@ __pyx_FusedFunction_descr_get(PyObject *self, PyObject *obj, PyObject *type) } static PyObject * -_obj_to_str(PyObject *obj) +_obj_to_string(PyObject *obj) { - if (PyType_Check(obj)) + if (PyUnicode_CheckExact(obj)) + return __Pyx_NewRef(obj); +#if PY_MAJOR_VERSION == 2 + else if (PyString_Check(obj)) + return PyUnicode_FromEncodedObject(obj, NULL, "strict"); +#endif + else if (PyType_Check(obj)) return PyObject_GetAttr(obj, PYIDENT("__name__")); else - return PyObject_Str(obj); + return PyObject_Unicode(obj); } static PyObject * @@ -989,65 +1298,55 @@ __pyx_FusedFunction_getitem(__pyx_FusedFunctionObject *self, PyObject *idx) PyObject *unbound_result_func; PyObject *result_func = NULL; - if (self->__signatures__ == NULL) { + if (unlikely(self->__signatures__ == NULL)) { PyErr_SetString(PyExc_TypeError, "Function is not fused"); return NULL; } if (PyTuple_Check(idx)) { - PyObject *list = PyList_New(0); Py_ssize_t n = PyTuple_GET_SIZE(idx); - PyObject *sep = NULL; + PyObject *list = PyList_New(n); int i; if (unlikely(!list)) return NULL; for (i = 0; i < n; i++) { - int ret; PyObject *string; #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS PyObject *item = PyTuple_GET_ITEM(idx, i); #else PyObject *item = PySequence_ITEM(idx, i); if (unlikely(!item)) goto __pyx_err; #endif - string = _obj_to_str(item); + string = _obj_to_string(item); #if !(CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS) Py_DECREF(item); #endif if (unlikely(!string)) goto __pyx_err; - ret = PyList_Append(list, string); - Py_DECREF(string); - if (unlikely(ret < 0)) goto __pyx_err; + PyList_SET_ITEM(list, i, string); } - sep = PyUnicode_FromString("|"); - if (likely(sep)) - signature = PyUnicode_Join(sep, list); -__pyx_err: -; + signature = PyUnicode_Join(PYUNICODE("|"), list); +__pyx_err:; Py_DECREF(list); - Py_XDECREF(sep); } else { - signature = _obj_to_str(idx); + signature = _obj_to_string(idx); } - if (!signature) + if (unlikely(!signature)) return NULL; unbound_result_func = PyObject_GetItem(self->__signatures__, signature); - if (unbound_result_func) { - if (self->self || self->type) { + if (likely(unbound_result_func)) { + if (self->self) { __pyx_FusedFunctionObject *unbound = (__pyx_FusedFunctionObject *) unbound_result_func; // TODO: move this to InitClassCell - Py_CLEAR(unbound->func.func_classobj); - Py_XINCREF(self->func.func_classobj); - unbound->func.func_classobj = self->func.func_classobj; + __Pyx_CyFunction_SetClassObj(unbound, __Pyx_CyFunction_GetClassObj(self)); result_func = __pyx_FusedFunction_descr_get(unbound_result_func, - self->self, self->type); + self->self, self->self); } else { result_func = unbound_result_func; Py_INCREF(result_func); @@ -1067,7 +1366,7 @@ __pyx_FusedFunction_callfunction(PyObject *func, PyObject *args, PyObject *kw) int static_specialized = (cyfunc->flags & __Pyx_CYFUNCTION_STATICMETHOD && !((__pyx_FusedFunctionObject *) func)->__signatures__); - if (cyfunc->flags & __Pyx_CYFUNCTION_CCLASS && !static_specialized) { + if ((cyfunc->flags & __Pyx_CYFUNCTION_CCLASS) && !static_specialized) { return __Pyx_CyFunction_CallAsMethod(func, args, kw); } else { return __Pyx_CyFunction_Call(func, args, kw); @@ -1088,23 +1387,21 @@ __pyx_FusedFunction_call(PyObject *func, PyObject *args, PyObject *kw) PyObject *new_args = NULL; __pyx_FusedFunctionObject *new_func = NULL; PyObject *result = NULL; - PyObject *self = NULL; int is_staticmethod = binding_func->func.flags & __Pyx_CYFUNCTION_STATICMETHOD; - int is_classmethod = binding_func->func.flags & __Pyx_CYFUNCTION_CLASSMETHOD; if (binding_func->self) { // Bound method call, put 'self' in the args tuple + PyObject *self; Py_ssize_t i; new_args = PyTuple_New(argc + 1); - if (!new_args) + if (unlikely(!new_args)) return NULL; self = binding_func->self; -#if !(CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS) - Py_INCREF(self); -#endif + Py_INCREF(self); PyTuple_SET_ITEM(new_args, 0, self); + self = NULL; for (i = 0; i < argc; i++) { #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS @@ -1117,36 +1414,8 @@ __pyx_FusedFunction_call(PyObject *func, PyObject *args, PyObject *kw) } args = new_args; - } else if (binding_func->type) { - // Unbound method call - if (argc < 1) { - PyErr_SetString(PyExc_TypeError, "Need at least one argument, 0 given."); - return NULL; - } -#if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS - self = PyTuple_GET_ITEM(args, 0); -#else - self = PySequence_ITEM(args, 0); if (unlikely(!self)) return NULL; -#endif } - if (self && !is_classmethod && !is_staticmethod) { - int is_instance = PyObject_IsInstance(self, binding_func->type); - if (unlikely(!is_instance)) { - PyErr_Format(PyExc_TypeError, - "First argument should be of type %.200s, got %.200s.", - ((PyTypeObject *) binding_func->type)->tp_name, - Py_TYPE(self)->tp_name); - goto bad; - } else if (unlikely(is_instance == -1)) { - goto bad; - } - } -#if !(CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS) - Py_XDECREF(self); - self = NULL; -#endif - if (binding_func->__signatures__) { PyObject *tup; if (is_staticmethod && binding_func->func.flags & __Pyx_CYFUNCTION_CCLASS) { @@ -1169,23 +1438,31 @@ __pyx_FusedFunction_call(PyObject *func, PyObject *args, PyObject *kw) if (unlikely(!new_func)) goto bad; - Py_XINCREF(binding_func->func.func_classobj); - Py_CLEAR(new_func->func.func_classobj); - new_func->func.func_classobj = binding_func->func.func_classobj; + __Pyx_CyFunction_SetClassObj(new_func, __Pyx_CyFunction_GetClassObj(binding_func)); func = (PyObject *) new_func; } result = __pyx_FusedFunction_callfunction(func, args, kw); bad: -#if !(CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS) - Py_XDECREF(self); -#endif Py_XDECREF(new_args); Py_XDECREF((PyObject *) new_func); return result; } +static PyObject * +__Pyx_FusedFunction_get_self(__pyx_FusedFunctionObject *m, void *closure) +{ + PyObject *self = m->self; + CYTHON_UNUSED_VAR(closure); + if (unlikely(!self)) { + PyErr_SetString(PyExc_AttributeError, "'function' object has no attribute '__self__'"); + } else { + Py_INCREF(self); + } + return self; +} + static PyMemberDef __pyx_FusedFunction_members[] = { {(char *) "__signatures__", T_OBJECT, @@ -1195,6 +1472,38 @@ static PyMemberDef __pyx_FusedFunction_members[] = { {0, 0, 0, 0, 0}, }; +static PyGetSetDef __pyx_FusedFunction_getsets[] = { + {(char *) "__self__", (getter)__Pyx_FusedFunction_get_self, 0, 0, 0}, + // __doc__ is None for the fused function type, but we need it to be + // a descriptor for the instance's __doc__, so rebuild the descriptor in our subclass + // (all other descriptors are inherited) + {(char *) "__doc__", (getter)__Pyx_CyFunction_get_doc, (setter)__Pyx_CyFunction_set_doc, 0, 0}, + {0, 0, 0, 0, 0} +}; + +#if CYTHON_USE_TYPE_SPECS +static PyType_Slot __pyx_FusedFunctionType_slots[] = { + {Py_tp_dealloc, (void *)__pyx_FusedFunction_dealloc}, + {Py_tp_call, (void *)__pyx_FusedFunction_call}, + {Py_tp_traverse, (void *)__pyx_FusedFunction_traverse}, + {Py_tp_clear, (void *)__pyx_FusedFunction_clear}, + {Py_tp_members, (void *)__pyx_FusedFunction_members}, + {Py_tp_getset, (void *)__pyx_FusedFunction_getsets}, + {Py_tp_descr_get, (void *)__pyx_FusedFunction_descr_get}, + {Py_mp_subscript, (void *)__pyx_FusedFunction_getitem}, + {0, 0}, +}; + +static PyType_Spec __pyx_FusedFunctionType_spec = { + __PYX_TYPE_MODULE_PREFIX "fused_cython_function", + sizeof(__pyx_FusedFunctionObject), + 0, + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_BASETYPE, /*tp_flags*/ + __pyx_FusedFunctionType_slots +}; + +#else /* !CYTHON_USE_TYPE_SPECS */ + static PyMappingMethods __pyx_FusedFunction_mapping_methods = { 0, (binaryfunc) __pyx_FusedFunction_getitem, @@ -1203,7 +1512,7 @@ static PyMappingMethods __pyx_FusedFunction_mapping_methods = { static PyTypeObject __pyx_FusedFunctionType_type = { PyVarObject_HEAD_INIT(0, 0) - "fused_cython_function", /*tp_name*/ + __PYX_TYPE_MODULE_PREFIX "fused_cython_function", /*tp_name*/ sizeof(__pyx_FusedFunctionObject), /*tp_basicsize*/ 0, /*tp_itemsize*/ (destructor) __pyx_FusedFunction_dealloc, /*tp_dealloc*/ @@ -1213,7 +1522,7 @@ static PyTypeObject __pyx_FusedFunctionType_type = { #if PY_MAJOR_VERSION < 3 0, /*tp_compare*/ #else - 0, /*reserved*/ + 0, /*tp_as_async*/ #endif 0, /*tp_repr*/ 0, /*tp_as_number*/ @@ -1235,9 +1544,7 @@ static PyTypeObject __pyx_FusedFunctionType_type = { 0, /*tp_iternext*/ 0, /*tp_methods*/ __pyx_FusedFunction_members, /*tp_members*/ - // __doc__ is None for the fused function type, but we need it to be - // a descriptor for the instance's __doc__, so rebuild descriptors in our subclass - __pyx_CyFunction_getsets, /*tp_getset*/ + __pyx_FusedFunction_getsets, /*tp_getset*/ // NOTE: tp_base may be changed later during module initialisation when importing CyFunction across modules. &__pyx_CyFunctionType_type, /*tp_base*/ 0, /*tp_dict*/ @@ -1265,13 +1572,27 @@ static PyTypeObject __pyx_FusedFunctionType_type = { #if PY_VERSION_HEX >= 0x030800b4 && PY_VERSION_HEX < 0x03090000 0, /*tp_print*/ #endif +#if CYTHON_COMPILING_IN_PYPY && PYPY_VERSION_NUM+0 >= 0x06000000 + 0, /*tp_pypy_flags*/ +#endif }; +#endif -static int __pyx_FusedFunction_init(void) { +static int __pyx_FusedFunction_init(PyObject *module) { +#if CYTHON_USE_TYPE_SPECS + PyObject *bases = PyTuple_Pack(1, __pyx_CyFunctionType); + if (unlikely(!bases)) { + return -1; + } + __pyx_FusedFunctionType = __Pyx_FetchCommonTypeFromSpec(module, &__pyx_FusedFunctionType_spec, bases); + Py_DECREF(bases); +#else + (void) module; // Set base from __Pyx_FetchCommonTypeFromSpec, in case it's different from the local static value. __pyx_FusedFunctionType_type.tp_base = __pyx_CyFunctionType; __pyx_FusedFunctionType = __Pyx_FetchCommonType(&__pyx_FusedFunctionType_type); - if (__pyx_FusedFunctionType == NULL) { +#endif + if (unlikely(__pyx_FusedFunctionType == NULL)) { return -1; } return 0; @@ -1291,16 +1612,16 @@ static PyObject* __Pyx_Method_ClassMethod(PyObject *method) { return PyClassMethod_New(method); } #else -#if CYTHON_COMPILING_IN_PYSTON || CYTHON_COMPILING_IN_PYPY - // special C-API function only in Pyston and PyPy >= 5.9 +#if CYTHON_COMPILING_IN_PYPY + // special C-API function only in PyPy >= 5.9 if (PyMethodDescr_Check(method)) #else #if PY_MAJOR_VERSION == 2 // PyMethodDescr_Type is not exposed in the CPython C-API in Py2. static PyTypeObject *methoddescr_type = NULL; - if (methoddescr_type == NULL) { + if (unlikely(methoddescr_type == NULL)) { PyObject *meth = PyObject_GetAttrString((PyObject*)&PyList_Type, "append"); - if (!meth) return NULL; + if (unlikely(!meth)) return NULL; methoddescr_type = Py_TYPE(meth); Py_DECREF(meth); } diff --git a/Cython/Utility/Embed.c b/Cython/Utility/Embed.c index 8f7e8f46e..ec48bfa04 100644 --- a/Cython/Utility/Embed.c +++ b/Cython/Utility/Embed.c @@ -6,7 +6,7 @@ #if PY_MAJOR_VERSION < 3 int %(main_method)s(int argc, char** argv) { -#elif defined(WIN32) || defined(MS_WINDOWS) +#elif defined(_WIN32) || defined(WIN32) || defined(MS_WINDOWS) int %(wmain_method)s(int argc, wchar_t **argv) { #else static int __Pyx_main(int argc, wchar_t **argv) { @@ -24,32 +24,23 @@ static int __Pyx_main(int argc, wchar_t **argv) { #endif if (argc && argv) Py_SetProgramName(argv[0]); + + #if PY_MAJOR_VERSION < 3 + if (PyImport_AppendInittab("%(module_name)s", init%(module_name)s) < 0) return 1; + #else + if (PyImport_AppendInittab("%(module_name)s", PyInit_%(module_name)s) < 0) return 1; + #endif + Py_Initialize(); if (argc && argv) PySys_SetArgv(argc, argv); + { /* init module '%(module_name)s' as '__main__' */ PyObject* m = NULL; %(module_is_main)s = 1; - #if PY_MAJOR_VERSION < 3 - init%(module_name)s(); - #elif CYTHON_PEP489_MULTI_PHASE_INIT - m = PyInit_%(module_name)s(); - if (!PyModule_Check(m)) { - PyModuleDef *mdef = (PyModuleDef *) m; - PyObject *modname = PyUnicode_FromString("__main__"); - m = NULL; - if (modname) { - // FIXME: not currently calling PyModule_FromDefAndSpec() here because we do not have a module spec! - // FIXME: not currently setting __file__, __path__, __spec__, ... - m = PyModule_NewObject(modname); - Py_DECREF(modname); - if (m) PyModule_ExecDef(m, mdef); - } - } - #else - m = PyInit_%(module_name)s(); - #endif - if (PyErr_Occurred()) { + m = PyImport_ImportModule("%(module_name)s"); + + if (!m && PyErr_Occurred()) { PyErr_Print(); /* This exits with the right code if SystemExit. */ #if PY_MAJOR_VERSION < 3 if (Py_FlushLine()) PyErr_Clear(); @@ -68,9 +59,11 @@ static int __Pyx_main(int argc, wchar_t **argv) { } -#if PY_MAJOR_VERSION >= 3 && !defined(WIN32) && !defined(MS_WINDOWS) +#if PY_MAJOR_VERSION >= 3 && !defined(_WIN32) && !defined(WIN32) && !defined(MS_WINDOWS) #include <locale.h> +#if PY_VERSION_HEX < 0x03050000 + static wchar_t* __Pyx_char2wchar(char* arg) { @@ -175,6 +168,8 @@ oom: return NULL; } +#endif + int %(main_method)s(int argc, char **argv) { @@ -197,7 +192,12 @@ int res = 0; setlocale(LC_ALL, ""); for (i = 0; i < argc; i++) { - argv_copy2[i] = argv_copy[i] = __Pyx_char2wchar(argv[i]); + argv_copy2[i] = argv_copy[i] = +#if PY_VERSION_HEX < 0x03050000 + __Pyx_char2wchar(argv[i]); +#else + Py_DecodeLocale(argv[i], NULL); +#endif if (!argv_copy[i]) res = 1; /* failure, but continue to simplify cleanup */ } setlocale(LC_ALL, oldloc); diff --git a/Cython/Utility/Exceptions.c b/Cython/Utility/Exceptions.c index ebbf65e07..aab060604 100644 --- a/Cython/Utility/Exceptions.c +++ b/Cython/Utility/Exceptions.c @@ -6,6 +6,23 @@ // __Pyx_GetException() +/////////////// ErrOccurredWithGIL.proto /////////////// +static CYTHON_INLINE int __Pyx_ErrOccurredWithGIL(void); /* proto */ + +/////////////// ErrOccurredWithGIL /////////////// +static CYTHON_INLINE int __Pyx_ErrOccurredWithGIL(void) { + int err; + #ifdef WITH_THREAD + PyGILState_STATE _save = PyGILState_Ensure(); + #endif + err = !!PyErr_Occurred(); + #ifdef WITH_THREAD + PyGILState_Release(_save); + #endif + return err; +} + + /////////////// PyThreadStateGet.proto /////////////// //@substitute: naming @@ -127,9 +144,9 @@ static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, PyObject // has changed quite a lot between the two versions. #if PY_MAJOR_VERSION < 3 -static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, - CYTHON_UNUSED PyObject *cause) { +static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, PyObject *cause) { __Pyx_PyThreadState_declare + CYTHON_UNUSED_VAR(cause); /* 'cause' is only used in Py3 */ Py_XINCREF(type); if (!value || value == Py_None) @@ -284,7 +301,7 @@ static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, PyObject PyErr_SetObject(type, value); if (tb) { -#if CYTHON_COMPILING_IN_PYPY +#if CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_LIMITED_API PyObject *tmp_type, *tmp_value, *tmp_tb; PyErr_Fetch(&tmp_type, &tmp_value, &tmp_tb); Py_INCREF(tb); @@ -590,9 +607,9 @@ static void __Pyx_WriteUnraisable(const char *name, int clineno, //@requires: PyErrFetchRestore //@requires: PyThreadStateGet -static void __Pyx_WriteUnraisable(const char *name, CYTHON_UNUSED int clineno, - CYTHON_UNUSED int lineno, CYTHON_UNUSED const char *filename, - int full_traceback, CYTHON_UNUSED int nogil) { +static void __Pyx_WriteUnraisable(const char *name, int clineno, + int lineno, const char *filename, + int full_traceback, int nogil) { PyObject *old_exc, *old_val, *old_tb; PyObject *ctx; __Pyx_PyThreadState_declare @@ -605,6 +622,11 @@ static void __Pyx_WriteUnraisable(const char *name, CYTHON_UNUSED int clineno, else state = (PyGILState_STATE)-1; #endif #endif + CYTHON_UNUSED_VAR(clineno); + CYTHON_UNUSED_VAR(lineno); + CYTHON_UNUSED_VAR(filename); + CYTHON_MAYBE_UNUSED_VAR(nogil); + __Pyx_PyThreadState_assign __Pyx_ErrFetch(&old_exc, &old_val, &old_tb); if (full_traceback) { @@ -641,7 +663,7 @@ static int __Pyx_CLineForTraceback(PyThreadState *tstate, int c_line);/*proto*/ #endif /////////////// CLineInTraceback /////////////// -//@requires: ObjectHandling.c::PyObjectGetAttrStr +//@requires: ObjectHandling.c::PyObjectGetAttrStrNoError //@requires: ObjectHandling.c::PyDictVersioning //@requires: PyErrFetchRestore //@substitute: naming @@ -670,7 +692,7 @@ static int __Pyx_CLineForTraceback(CYTHON_NCP_UNUSED PyThreadState *tstate, int } else #endif { - PyObject *use_cline_obj = __Pyx_PyObject_GetAttrStr(${cython_runtime_cname}, PYIDENT("cline_in_traceback")); + PyObject *use_cline_obj = __Pyx_PyObject_GetAttrStrNoError(${cython_runtime_cname}, PYIDENT("cline_in_traceback")); if (use_cline_obj) { use_cline = PyObject_Not(use_cline_obj) ? Py_False : Py_True; Py_DECREF(use_cline_obj); @@ -705,6 +727,17 @@ static void __Pyx_AddTraceback(const char *funcname, int c_line, #include "frameobject.h" #include "traceback.h" +#if CYTHON_COMPILING_IN_LIMITED_API +static void __Pyx_AddTraceback(const char *funcname, int c_line, + int py_line, const char *filename) { + if (c_line) { + // Avoid "unused" warning as long as we don't use this. + (void) $cfilenm_cname; + c_line = __Pyx_CLineForTraceback(__Pyx_PyThreadState_Current, c_line); + } + _PyTraceback_Add(funcname, filename, c_line ? -c_line : py_line); +} +#else static PyCodeObject* __Pyx_CreateCodeObjectForTraceback( const char *funcname, int c_line, int py_line, const char *filename) { @@ -735,6 +768,7 @@ static PyCodeObject* __Pyx_CreateCodeObjectForTraceback( if (!py_funcname) goto bad; py_code = __Pyx_PyCode_New( 0, /*int argcount,*/ + 0, /*int posonlyargcount,*/ 0, /*int kwonlyargcount,*/ 0, /*int nlocals,*/ 0, /*int stacksize,*/ @@ -790,3 +824,4 @@ bad: Py_XDECREF(py_code); Py_XDECREF(py_frame); } +#endif diff --git a/Cython/Utility/ExtensionTypes.c b/Cython/Utility/ExtensionTypes.c index 1b39c9e42..7e497eeb8 100644 --- a/Cython/Utility/ExtensionTypes.c +++ b/Cython/Utility/ExtensionTypes.c @@ -1,12 +1,107 @@ -/////////////// PyType_Ready.proto /////////////// +/////////////// FixUpExtensionType.proto /////////////// -static int __Pyx_PyType_Ready(PyTypeObject *t); +#if CYTHON_USE_TYPE_SPECS +static int __Pyx_fix_up_extension_type_from_spec(PyType_Spec *spec, PyTypeObject *type); /*proto*/ +#endif -/////////////// PyType_Ready /////////////// +/////////////// FixUpExtensionType /////////////// +//@requires:ModuleSetupCode.c::IncludeStructmemberH +//@requires:StringTools.c::IncludeStringH -// Wrapper around PyType_Ready() with some runtime checks and fixes -// to deal with multiple inheritance. -static int __Pyx_PyType_Ready(PyTypeObject *t) { +#if CYTHON_USE_TYPE_SPECS +static int __Pyx_fix_up_extension_type_from_spec(PyType_Spec *spec, PyTypeObject *type) { +#if PY_VERSION_HEX > 0x030900B1 || CYTHON_COMPILING_IN_LIMITED_API + (void) spec; + (void) type; +#else + // Set tp_weakreflist, tp_dictoffset, tp_vectorcalloffset + // Copied and adapted from https://bugs.python.org/issue38140 + const PyType_Slot *slot = spec->slots; + while (slot && slot->slot && slot->slot != Py_tp_members) + slot++; + if (slot && slot->slot == Py_tp_members) { + int changed = 0; +#if !(PY_VERSION_HEX <= 0x030900b1 && CYTHON_COMPILING_IN_CPYTHON) + const +#endif + PyMemberDef *memb = (PyMemberDef*) slot->pfunc; + while (memb && memb->name) { + if (memb->name[0] == '_' && memb->name[1] == '_') { +#if PY_VERSION_HEX < 0x030900b1 + if (strcmp(memb->name, "__weaklistoffset__") == 0) { + // The PyMemberDef must be a Py_ssize_t and readonly. + assert(memb->type == T_PYSSIZET); + assert(memb->flags == READONLY); + type->tp_weaklistoffset = memb->offset; + // FIXME: is it even worth calling PyType_Modified() here? + changed = 1; + } + else if (strcmp(memb->name, "__dictoffset__") == 0) { + // The PyMemberDef must be a Py_ssize_t and readonly. + assert(memb->type == T_PYSSIZET); + assert(memb->flags == READONLY); + type->tp_dictoffset = memb->offset; + // FIXME: is it even worth calling PyType_Modified() here? + changed = 1; + } +#if CYTHON_METH_FASTCALL + else if (strcmp(memb->name, "__vectorcalloffset__") == 0) { + // The PyMemberDef must be a Py_ssize_t and readonly. + assert(memb->type == T_PYSSIZET); + assert(memb->flags == READONLY); +#if PY_VERSION_HEX >= 0x030800b4 + type->tp_vectorcall_offset = memb->offset; +#else + type->tp_print = (printfunc) memb->offset; +#endif + // FIXME: is it even worth calling PyType_Modified() here? + changed = 1; + } +#endif +#else + if ((0)); +#endif +#if PY_VERSION_HEX <= 0x030900b1 && CYTHON_COMPILING_IN_CPYTHON + else if (strcmp(memb->name, "__module__") == 0) { + // PyType_FromSpec() in CPython <= 3.9b1 overwrites this field with a constant string. + // See https://bugs.python.org/issue40703 + PyObject *descr; + // The PyMemberDef must be an object and normally readable, possibly writable. + assert(memb->type == T_OBJECT); + assert(memb->flags == 0 || memb->flags == READONLY); + descr = PyDescr_NewMember(type, memb); + if (unlikely(!descr)) + return -1; + if (unlikely(PyDict_SetItem(type->tp_dict, PyDescr_NAME(descr), descr) < 0)) { + Py_DECREF(descr); + return -1; + } + Py_DECREF(descr); + changed = 1; + } +#endif + } + memb++; + } + if (changed) + PyType_Modified(type); + } +#endif + return 0; +} +#endif + + +/////////////// ValidateBasesTuple.proto /////////////// + +#if CYTHON_COMPILING_IN_CPYTHON || CYTHON_COMPILING_IN_LIMITED_API || CYTHON_USE_TYPE_SPECS +static int __Pyx_validate_bases_tuple(const char *type_name, Py_ssize_t dictoffset, PyObject *bases); /*proto*/ +#endif + +/////////////// ValidateBasesTuple /////////////// + +#if CYTHON_COMPILING_IN_CPYTHON || CYTHON_COMPILING_IN_LIMITED_API || CYTHON_USE_TYPE_SPECS +static int __Pyx_validate_bases_tuple(const char *type_name, Py_ssize_t dictoffset, PyObject *bases) { // Loop over all bases (except the first) and check that those // really are heap types. Otherwise, it would not be safe to // subclass them. @@ -17,52 +112,95 @@ static int __Pyx_PyType_Ready(PyTypeObject *t) { // tp_dictoffset (i.e. there is no __dict__ attribute in the object // structure), we need to check that none of the base classes sets // it either. - int r; - PyObject *bases = t->tp_bases; - if (bases) + Py_ssize_t i, n = PyTuple_GET_SIZE(bases); + for (i = 1; i < n; i++) /* Skip first base */ { - Py_ssize_t i, n = PyTuple_GET_SIZE(bases); - for (i = 1; i < n; i++) /* Skip first base */ - { - PyObject *b0 = PyTuple_GET_ITEM(bases, i); - PyTypeObject *b; + PyObject *b0 = PyTuple_GET_ITEM(bases, i); + PyTypeObject *b; #if PY_MAJOR_VERSION < 3 - /* Disallow old-style classes */ - if (PyClass_Check(b0)) - { - PyErr_Format(PyExc_TypeError, "base class '%.200s' is an old-style class", - PyString_AS_STRING(((PyClassObject*)b0)->cl_name)); - return -1; - } + /* Disallow old-style classes */ + if (PyClass_Check(b0)) + { + PyErr_Format(PyExc_TypeError, "base class '%.200s' is an old-style class", + PyString_AS_STRING(((PyClassObject*)b0)->cl_name)); + return -1; + } #endif - b = (PyTypeObject*)b0; - if (!PyType_HasFeature(b, Py_TPFLAGS_HEAPTYPE)) - { - PyErr_Format(PyExc_TypeError, "base class '%.200s' is not a heap type", - b->tp_name); - return -1; - } - if (t->tp_dictoffset == 0 && b->tp_dictoffset) - { - PyErr_Format(PyExc_TypeError, - "extension type '%.200s' has no __dict__ slot, but base type '%.200s' has: " - "either add 'cdef dict __dict__' to the extension type " - "or add '__slots__ = [...]' to the base type", - t->tp_name, b->tp_name); - return -1; - } + b = (PyTypeObject*) b0; + if (!__Pyx_PyType_HasFeature(b, Py_TPFLAGS_HEAPTYPE)) + { + __Pyx_TypeName b_name = __Pyx_PyType_GetName(b); + PyErr_Format(PyExc_TypeError, + "base class '" __Pyx_FMT_TYPENAME "' is not a heap type", b_name); + __Pyx_DECREF_TypeName(b_name); + return -1; + } + if (dictoffset == 0 && b->tp_dictoffset) + { + __Pyx_TypeName b_name = __Pyx_PyType_GetName(b); + PyErr_Format(PyExc_TypeError, + "extension type '%.200s' has no __dict__ slot, " + "but base type '" __Pyx_FMT_TYPENAME "' has: " + "either add 'cdef dict __dict__' to the extension type " + "or add '__slots__ = [...]' to the base type", + type_name, b_name); + __Pyx_DECREF_TypeName(b_name); + return -1; } } + return 0; +} +#endif + + +/////////////// PyType_Ready.proto /////////////// + +static int __Pyx_PyType_Ready(PyTypeObject *t);/*proto*/ + +/////////////// PyType_Ready /////////////// +//@requires: ObjectHandling.c::PyObjectCallMethod0 +//@requires: ValidateBasesTuple + +// Wrapper around PyType_Ready() with some runtime checks and fixes +// to deal with multiple inheritance. +static int __Pyx_PyType_Ready(PyTypeObject *t) { + +// FIXME: is this really suitable for CYTHON_COMPILING_IN_LIMITED_API? +#if CYTHON_USE_TYPE_SPECS || !(CYTHON_COMPILING_IN_CPYTHON || CYTHON_COMPILING_IN_LIMITED_API) || defined(PYSTON_MAJOR_VERSION) + // avoid C warning about unused helper function + (void)__Pyx_PyObject_CallMethod0; +#if CYTHON_USE_TYPE_SPECS + (void)__Pyx_validate_bases_tuple; +#endif + + return PyType_Ready(t); + +#else + int r; + if (t->tp_bases && unlikely(__Pyx_validate_bases_tuple(t->tp_name, t->tp_dictoffset, t->tp_bases) == -1)) + return -1; #if PY_VERSION_HEX >= 0x03050000 { // Make sure GC does not pick up our non-heap type as heap type with this hack! // For details, see https://github.com/cython/cython/issues/3603 - PyObject *ret, *py_status; int gc_was_enabled; - PyObject *gc = PyImport_Import(PYUNICODE("gc")); + #if PY_VERSION_HEX >= 0x030A00b1 + // finally added in Py3.10 :) + gc_was_enabled = PyGC_Disable(); + (void)__Pyx_PyObject_CallMethod0; + + #else + // Call gc.disable() as a backwards compatible fallback, but only if needed. + PyObject *ret, *py_status; + PyObject *gc = NULL; + #if PY_VERSION_HEX >= 0x030700a1 && (!CYTHON_COMPILING_IN_PYPY || PYPY_VERSION_NUM+0 >= 0x07030400) + // https://foss.heptapod.net/pypy/pypy/-/issues/3385 + gc = PyImport_GetModule(PYUNICODE("gc")); + #endif + if (unlikely(!gc)) gc = PyImport_Import(PYUNICODE("gc")); if (unlikely(!gc)) return -1; - py_status = PyObject_CallMethodObjArgs(gc, PYUNICODE("isenabled"), NULL); + py_status = __Pyx_PyObject_CallMethod0(gc, PYUNICODE("isenabled")); if (unlikely(!py_status)) { Py_DECREF(gc); return -1; @@ -70,7 +208,7 @@ static int __Pyx_PyType_Ready(PyTypeObject *t) { gc_was_enabled = __Pyx_PyObject_IsTrue(py_status); Py_DECREF(py_status); if (gc_was_enabled > 0) { - ret = PyObject_CallMethodObjArgs(gc, PYUNICODE("disable"), NULL); + ret = __Pyx_PyObject_CallMethod0(gc, PYUNICODE("disable")); if (unlikely(!ret)) { Py_DECREF(gc); return -1; @@ -80,6 +218,7 @@ static int __Pyx_PyType_Ready(PyTypeObject *t) { Py_DECREF(gc); return -1; } + #endif // As of https://bugs.python.org/issue22079 // PyType_Ready enforces that all bases of a non-heap type are @@ -89,6 +228,9 @@ static int __Pyx_PyType_Ready(PyTypeObject *t) { // Other than this check, the Py_TPFLAGS_HEAPTYPE flag is unused // in PyType_Ready(). t->tp_flags |= Py_TPFLAGS_HEAPTYPE; +#else + // avoid C warning about unused helper function + (void)__Pyx_PyObject_CallMethod0; #endif r = PyType_Ready(t); @@ -96,29 +238,79 @@ static int __Pyx_PyType_Ready(PyTypeObject *t) { #if PY_VERSION_HEX >= 0x03050000 t->tp_flags &= ~Py_TPFLAGS_HEAPTYPE; + #if PY_VERSION_HEX >= 0x030A00b1 + if (gc_was_enabled) + PyGC_Enable(); + #else if (gc_was_enabled) { - PyObject *t, *v, *tb; - PyErr_Fetch(&t, &v, &tb); - ret = PyObject_CallMethodObjArgs(gc, PYUNICODE("enable"), NULL); + PyObject *tp, *v, *tb; + PyErr_Fetch(&tp, &v, &tb); + ret = __Pyx_PyObject_CallMethod0(gc, PYUNICODE("enable")); if (likely(ret || r == -1)) { Py_XDECREF(ret); // do not overwrite exceptions raised by PyType_Ready() above - PyErr_Restore(t, v, tb); + PyErr_Restore(tp, v, tb); } else { // PyType_Ready() succeeded, but gc.enable() failed. - Py_XDECREF(t); + Py_XDECREF(tp); Py_XDECREF(v); Py_XDECREF(tb); r = -1; } } Py_DECREF(gc); + #endif } #endif return r; +#endif } + +/////////////// PyTrashcan.proto /////////////// + +// These macros are taken from https://github.com/python/cpython/pull/11841 +// Unlike the Py_TRASHCAN_SAFE_BEGIN/Py_TRASHCAN_SAFE_END macros, they +// allow dealing correctly with subclasses. + +// This requires CPython version >= 2.7.4 +// (or >= 3.2.4 but we don't support such old Python 3 versions anyway) +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x02070400 +#define __Pyx_TRASHCAN_BEGIN_CONDITION(op, cond) \ + do { \ + PyThreadState *_tstate = NULL; \ + // If "cond" is false, then _tstate remains NULL and the deallocator + // is run normally without involving the trashcan + if (cond) { \ + _tstate = PyThreadState_GET(); \ + if (_tstate->trash_delete_nesting >= PyTrash_UNWIND_LEVEL) { \ + // Store the object (to be deallocated later) and jump past + // Py_TRASHCAN_END, skipping the body of the deallocator + _PyTrash_thread_deposit_object((PyObject*)(op)); \ + break; \ + } \ + ++_tstate->trash_delete_nesting; \ + } + // The body of the deallocator is here. +#define __Pyx_TRASHCAN_END \ + if (_tstate) { \ + --_tstate->trash_delete_nesting; \ + if (_tstate->trash_delete_later && _tstate->trash_delete_nesting <= 0) \ + _PyTrash_thread_destroy_chain(); \ + } \ + } while (0); + +#define __Pyx_TRASHCAN_BEGIN(op, dealloc) __Pyx_TRASHCAN_BEGIN_CONDITION(op, \ + Py_TYPE(op)->tp_dealloc == (destructor)(dealloc)) + +#else +// The trashcan is a no-op on other Python implementations +// or old CPython versions +#define __Pyx_TRASHCAN_BEGIN(op, dealloc) +#define __Pyx_TRASHCAN_END +#endif + /////////////// CallNextTpDealloc.proto /////////////// static void __Pyx_call_next_tp_dealloc(PyObject* obj, destructor current_tp_dealloc); @@ -174,18 +366,21 @@ static void __Pyx_call_next_tp_clear(PyObject* obj, inquiry current_tp_clear) { /////////////// SetupReduce.proto /////////////// +#if !CYTHON_COMPILING_IN_LIMITED_API static int __Pyx_setup_reduce(PyObject* type_obj); +#endif /////////////// SetupReduce /////////////// //@requires: ObjectHandling.c::PyObjectGetAttrStrNoError //@requires: ObjectHandling.c::PyObjectGetAttrStr //@substitute: naming +#if !CYTHON_COMPILING_IN_LIMITED_API static int __Pyx_setup_reduce_is_named(PyObject* meth, PyObject* name) { int ret; PyObject *name_attr; - name_attr = __Pyx_PyObject_GetAttrStr(meth, PYIDENT("__name__")); + name_attr = __Pyx_PyObject_GetAttrStrNoError(meth, PYIDENT("__name__")); if (likely(name_attr)) { ret = PyObject_RichCompareBool(name_attr, name, Py_EQ); } else { @@ -244,7 +439,7 @@ static int __Pyx_setup_reduce(PyObject* type_obj) { goto __PYX_BAD; } - setstate = __Pyx_PyObject_GetAttrStr(type_obj, PYIDENT("__setstate__")); + setstate = __Pyx_PyObject_GetAttrStrNoError(type_obj, PYIDENT("__setstate__")); if (!setstate) PyErr_Clear(); if (!setstate || __Pyx_setup_reduce_is_named(setstate, PYIDENT("__setstate_cython__"))) { setstate_cython = __Pyx_PyObject_GetAttrStrNoError(type_obj, PYIDENT("__setstate_cython__")); @@ -263,8 +458,13 @@ static int __Pyx_setup_reduce(PyObject* type_obj) { goto __PYX_GOOD; __PYX_BAD: - if (!PyErr_Occurred()) - PyErr_Format(PyExc_RuntimeError, "Unable to initialize pickling for %s", ((PyTypeObject*)type_obj)->tp_name); + if (!PyErr_Occurred()) { + __Pyx_TypeName type_obj_name = + __Pyx_PyType_GetName((PyTypeObject*)type_obj); + PyErr_Format(PyExc_RuntimeError, + "Unable to initialize pickling for " __Pyx_FMT_TYPENAME, type_obj_name); + __Pyx_DECREF_TypeName(type_obj_name); + } ret = -1; __PYX_GOOD: #if !CYTHON_USE_PYTYPE_LOOKUP @@ -278,3 +478,58 @@ __PYX_GOOD: Py_XDECREF(setstate_cython); return ret; } +#endif + + +/////////////// BinopSlot /////////////// + +static CYTHON_INLINE PyObject *{{func_name}}_maybe_call_slot(PyTypeObject* type, PyObject *left, PyObject *right {{extra_arg_decl}}) { + {{slot_type}} slot; +#if CYTHON_USE_TYPE_SLOTS || PY_MAJOR_VERSION < 3 || CYTHON_COMPILING_IN_PYPY + slot = type->tp_as_number ? type->tp_as_number->{{slot_name}} : NULL; +#else + slot = ({{slot_type}}) PyType_GetSlot(type, Py_{{slot_name}}); +#endif + return slot ? slot(left, right {{extra_arg}}) : __Pyx_NewRef(Py_NotImplemented); +} + +static PyObject *{{func_name}}(PyObject *left, PyObject *right {{extra_arg_decl}}) { + int maybe_self_is_left, maybe_self_is_right = 0; + maybe_self_is_left = Py_TYPE(left) == Py_TYPE(right) +#if CYTHON_USE_TYPE_SLOTS + || (Py_TYPE(left)->tp_as_number && Py_TYPE(left)->tp_as_number->{{slot_name}} == &{{func_name}}) +#endif + || __Pyx_TypeCheck(left, {{type_cname}}); + // Optimize for the common case where the left operation is defined (and successful). + if (!({{overloads_left}})) { + maybe_self_is_right = Py_TYPE(left) == Py_TYPE(right) +#if CYTHON_USE_TYPE_SLOTS + || (Py_TYPE(right)->tp_as_number && Py_TYPE(right)->tp_as_number->{{slot_name}} == &{{func_name}}) +#endif + || __Pyx_TypeCheck(right, {{type_cname}}); + } + if (maybe_self_is_left) { + PyObject *res; + if (maybe_self_is_right && {{overloads_right}} && !({{overloads_left}})) { + res = {{call_right}}; + if (res != Py_NotImplemented) return res; + Py_DECREF(res); + // Don't bother calling it again. + maybe_self_is_right = 0; + } + res = {{call_left}}; + if (res != Py_NotImplemented) return res; + Py_DECREF(res); + } + if (({{overloads_left}})) { + maybe_self_is_right = Py_TYPE(left) == Py_TYPE(right) +#if CYTHON_USE_TYPE_SLOTS + || (Py_TYPE(right)->tp_as_number && Py_TYPE(right)->tp_as_number->{{slot_name}} == &{{func_name}}) +#endif + || PyType_IsSubtype(Py_TYPE(right), {{type_cname}}); + } + if (maybe_self_is_right) { + return {{call_right}}; + } + return __Pyx_NewRef(Py_NotImplemented); +} diff --git a/Cython/Utility/FunctionArguments.c b/Cython/Utility/FunctionArguments.c index 8333d9366..1882f826f 100644 --- a/Cython/Utility/FunctionArguments.c +++ b/Cython/Utility/FunctionArguments.c @@ -2,7 +2,7 @@ #define __Pyx_ArgTypeTest(obj, type, none_allowed, name, exact) \ - ((likely((Py_TYPE(obj) == type) | (none_allowed && (obj == Py_None)))) ? 1 : \ + ((likely(__Pyx_IS_TYPE(obj, type) | (none_allowed && (obj == Py_None)))) ? 1 : \ __Pyx__ArgTypeTest(obj, type, name, exact)) static int __Pyx__ArgTypeTest(PyObject *obj, PyTypeObject *type, const char *name, int exact); /*proto*/ @@ -11,6 +11,8 @@ static int __Pyx__ArgTypeTest(PyObject *obj, PyTypeObject *type, const char *nam static int __Pyx__ArgTypeTest(PyObject *obj, PyTypeObject *type, const char *name, int exact) { + __Pyx_TypeName type_name; + __Pyx_TypeName obj_type_name; if (unlikely(!type)) { PyErr_SetString(PyExc_SystemError, "Missing type object"); return 0; @@ -23,9 +25,13 @@ static int __Pyx__ArgTypeTest(PyObject *obj, PyTypeObject *type, const char *nam else { if (likely(__Pyx_TypeCheck(obj, type))) return 1; } + type_name = __Pyx_PyType_GetName(type); + obj_type_name = __Pyx_PyType_GetName(Py_TYPE(obj)); PyErr_Format(PyExc_TypeError, - "Argument '%.200s' has incorrect type (expected %.200s, got %.200s)", - name, type->tp_name, Py_TYPE(obj)->tp_name); + "Argument '%.200s' has incorrect type (expected " __Pyx_FMT_TYPENAME + ", got " __Pyx_FMT_TYPENAME ")", name, type_name, obj_type_name); + __Pyx_DECREF_TypeName(type_name); + __Pyx_DECREF_TypeName(obj_type_name); return 0; } @@ -111,22 +117,28 @@ static void __Pyx_RaiseMappingExpectedError(PyObject* arg); /*proto*/ //////////////////// RaiseMappingExpected //////////////////// static void __Pyx_RaiseMappingExpectedError(PyObject* arg) { - PyErr_Format(PyExc_TypeError, "'%.200s' object is not a mapping", Py_TYPE(arg)->tp_name); + __Pyx_TypeName arg_type_name = __Pyx_PyType_GetName(Py_TYPE(arg)); + PyErr_Format(PyExc_TypeError, + "'" __Pyx_FMT_TYPENAME "' object is not a mapping", arg_type_name); + __Pyx_DECREF_TypeName(arg_type_name); } //////////////////// KeywordStringCheck.proto //////////////////// -static int __Pyx_CheckKeywordStrings(PyObject *kwdict, const char* function_name, int kw_allowed); /*proto*/ +static int __Pyx_CheckKeywordStrings(PyObject *kw, const char* function_name, int kw_allowed); /*proto*/ //////////////////// KeywordStringCheck //////////////////// -// __Pyx_CheckKeywordStrings raises an error if non-string keywords -// were passed to a function, or if any keywords were passed to a -// function that does not accept them. +// __Pyx_CheckKeywordStrings raises an error if non-string keywords +// were passed to a function, or if any keywords were passed to a +// function that does not accept them. +// +// The "kw" argument is either a dict (for METH_VARARGS) or a tuple +// (for METH_FASTCALL). static int __Pyx_CheckKeywordStrings( - PyObject *kwdict, + PyObject *kw, const char* function_name, int kw_allowed) { @@ -134,18 +146,37 @@ static int __Pyx_CheckKeywordStrings( Py_ssize_t pos = 0; #if CYTHON_COMPILING_IN_PYPY /* PyPy appears to check keywords at call time, not at unpacking time => not much to do here */ - if (!kw_allowed && PyDict_Next(kwdict, &pos, &key, 0)) + if (!kw_allowed && PyDict_Next(kw, &pos, &key, 0)) goto invalid_keyword; return 1; #else - while (PyDict_Next(kwdict, &pos, &key, 0)) { + if (CYTHON_METH_FASTCALL && likely(PyTuple_Check(kw))) { + if (unlikely(PyTuple_GET_SIZE(kw) == 0)) + return 1; + if (!kw_allowed) { + key = PyTuple_GET_ITEM(kw, 0); + goto invalid_keyword; + } +#if PY_VERSION_HEX < 0x03090000 + // On CPython >= 3.9, the FASTCALL protocol guarantees that keyword + // names are strings (see https://bugs.python.org/issue37540) + for (pos = 0; pos < PyTuple_GET_SIZE(kw); pos++) { + key = PyTuple_GET_ITEM(kw, pos); + if (unlikely(!PyUnicode_Check(key))) + goto invalid_keyword_type; + } +#endif + return 1; + } + + while (PyDict_Next(kw, &pos, &key, 0)) { #if PY_MAJOR_VERSION < 3 if (unlikely(!PyString_Check(key))) #endif if (unlikely(!PyUnicode_Check(key))) goto invalid_keyword_type; } - if ((!kw_allowed) && unlikely(key)) + if (!kw_allowed && unlikely(key)) goto invalid_keyword; return 1; invalid_keyword_type: @@ -154,11 +185,12 @@ invalid_keyword_type: return 0; #endif invalid_keyword: - PyErr_Format(PyExc_TypeError, #if PY_MAJOR_VERSION < 3 + PyErr_Format(PyExc_TypeError, "%.200s() got an unexpected keyword argument '%.200s'", function_name, PyString_AsString(key)); #else + PyErr_Format(PyExc_TypeError, "%s() got an unexpected keyword argument '%U'", function_name, key); #endif @@ -168,17 +200,22 @@ invalid_keyword: //////////////////// ParseKeywords.proto //////////////////// -static int __Pyx_ParseOptionalKeywords(PyObject *kwds, PyObject **argnames[], \ - PyObject *kwds2, PyObject *values[], Py_ssize_t num_pos_args, \ +static int __Pyx_ParseOptionalKeywords(PyObject *kwds, PyObject *const *kwvalues, + PyObject **argnames[], + PyObject *kwds2, PyObject *values[], Py_ssize_t num_pos_args, const char* function_name); /*proto*/ //////////////////// ParseKeywords //////////////////// //@requires: RaiseDoubleKeywords // __Pyx_ParseOptionalKeywords copies the optional/unknown keyword -// arguments from the kwds dict into kwds2. If kwds2 is NULL, unknown +// arguments from kwds into the dict kwds2. If kwds2 is NULL, unknown // keywords will raise an invalid keyword error. // +// When not using METH_FASTCALL, kwds is a dict and kwvalues is NULL. +// Otherwise, kwds is a tuple with keyword names and kwvalues is a C +// array with the corresponding values. +// // Three kinds of errors are checked: 1) non-string keywords, 2) // unexpected keywords and 3) overlap with positional arguments. // @@ -190,6 +227,7 @@ static int __Pyx_ParseOptionalKeywords(PyObject *kwds, PyObject **argnames[], \ static int __Pyx_ParseOptionalKeywords( PyObject *kwds, + PyObject *const *kwvalues, PyObject **argnames[], PyObject *kwds2, PyObject *values[], @@ -200,8 +238,20 @@ static int __Pyx_ParseOptionalKeywords( Py_ssize_t pos = 0; PyObject*** name; PyObject*** first_kw_arg = argnames + num_pos_args; + int kwds_is_tuple = CYTHON_METH_FASTCALL && likely(PyTuple_Check(kwds)); + + while (1) { + if (kwds_is_tuple) { + if (pos >= PyTuple_GET_SIZE(kwds)) break; + key = PyTuple_GET_ITEM(kwds, pos); + value = kwvalues[pos]; + pos++; + } + else + { + if (!PyDict_Next(kwds, &pos, &key, &value)) break; + } - while (PyDict_Next(kwds, &pos, &key, &value)) { name = first_kw_arg; while (*name && (**name != key)) name++; if (*name) { @@ -237,12 +287,13 @@ static int __Pyx_ParseOptionalKeywords( #endif if (likely(PyUnicode_Check(key))) { while (*name) { - int cmp = (**name == key) ? 0 : + int cmp = ( #if !CYTHON_COMPILING_IN_PYPY && PY_MAJOR_VERSION >= 3 (__Pyx_PyUnicode_GET_LENGTH(**name) != __Pyx_PyUnicode_GET_LENGTH(key)) ? 1 : #endif // In Py2, we may need to convert the argument name from str to unicode for comparison. - PyUnicode_Compare(**name, key); + PyUnicode_Compare(**name, key) + ); if (cmp < 0 && unlikely(PyErr_Occurred())) goto bad; if (cmp == 0) { values[name-argnames] = value; @@ -284,11 +335,12 @@ invalid_keyword_type: "%.200s() keywords must be strings", function_name); goto bad; invalid_keyword: - PyErr_Format(PyExc_TypeError, #if PY_MAJOR_VERSION < 3 + PyErr_Format(PyExc_TypeError, "%.200s() got an unexpected keyword argument '%.200s'", function_name, PyString_AsString(key)); #else + PyErr_Format(PyExc_TypeError, "%s() got an unexpected keyword argument '%U'", function_name, key); #endif @@ -314,7 +366,7 @@ static int __Pyx_MergeKeywords(PyObject *kwdict, PyObject *source_mapping) { if (unlikely(!iter)) { // slow fallback: try converting to dict, then iterate PyObject *args; - if (!PyErr_ExceptionMatches(PyExc_AttributeError)) goto bad; + if (unlikely(!PyErr_ExceptionMatches(PyExc_AttributeError))) goto bad; PyErr_Clear(); args = PyTuple_Pack(1, source_mapping); if (likely(args)) { @@ -350,3 +402,74 @@ bad: Py_XDECREF(iter); return -1; } + + +/////////////// fastcall.proto /////////////// + +// We define various functions and macros with two variants: +//..._FASTCALL and ..._VARARGS + +// The first is used when METH_FASTCALL is enabled and the second is used +// otherwise. If the Python implementation does not support METH_FASTCALL +// (because it's an old version of CPython or it's not CPython at all), +// then the ..._FASTCALL macros simply alias ..._VARARGS + +#define __Pyx_Arg_VARARGS(args, i) PyTuple_GET_ITEM(args, i) +#define __Pyx_NumKwargs_VARARGS(kwds) PyDict_Size(kwds) +#define __Pyx_KwValues_VARARGS(args, nargs) NULL +#define __Pyx_GetKwValue_VARARGS(kw, kwvalues, s) __Pyx_PyDict_GetItemStrWithError(kw, s) +#define __Pyx_KwargsAsDict_VARARGS(kw, kwvalues) PyDict_Copy(kw) +#if CYTHON_METH_FASTCALL + #define __Pyx_Arg_FASTCALL(args, i) args[i] + #define __Pyx_NumKwargs_FASTCALL(kwds) PyTuple_GET_SIZE(kwds) + #define __Pyx_KwValues_FASTCALL(args, nargs) (&args[nargs]) + static CYTHON_INLINE PyObject * __Pyx_GetKwValue_FASTCALL(PyObject *kwnames, PyObject *const *kwvalues, PyObject *s); + #define __Pyx_KwargsAsDict_FASTCALL(kw, kwvalues) _PyStack_AsDict(kwvalues, kw) +#else + #define __Pyx_Arg_FASTCALL __Pyx_Arg_VARARGS + #define __Pyx_NumKwargs_FASTCALL __Pyx_NumKwargs_VARARGS + #define __Pyx_KwValues_FASTCALL __Pyx_KwValues_VARARGS + #define __Pyx_GetKwValue_FASTCALL __Pyx_GetKwValue_VARARGS + #define __Pyx_KwargsAsDict_FASTCALL __Pyx_KwargsAsDict_VARARGS +#endif + +#if CYTHON_COMPILING_IN_CPYTHON +#define __Pyx_ArgsSlice_VARARGS(args, start, stop) __Pyx_PyTuple_FromArray(&__Pyx_Arg_VARARGS(args, start), stop - start) +#define __Pyx_ArgsSlice_FASTCALL(args, start, stop) __Pyx_PyTuple_FromArray(&__Pyx_Arg_FASTCALL(args, start), stop - start) +#else +/* Not CPython, so certainly no METH_FASTCALL support */ +#define __Pyx_ArgsSlice_VARARGS(args, start, stop) PyTuple_GetSlice(args, start, stop) +#define __Pyx_ArgsSlice_FASTCALL(args, start, stop) PyTuple_GetSlice(args, start, stop) +#endif + + +/////////////// fastcall /////////////// +//@requires: ObjectHandling.c::TupleAndListFromArray +//@requires: StringTools.c::UnicodeEquals + +#if CYTHON_METH_FASTCALL +// kwnames: tuple with names of keyword arguments +// kwvalues: C array with values of keyword arguments +// s: str with the keyword name to look for +static CYTHON_INLINE PyObject * __Pyx_GetKwValue_FASTCALL(PyObject *kwnames, PyObject *const *kwvalues, PyObject *s) +{ + // Search the kwnames array for s and return the corresponding value. + // We do two loops: a first one to compare pointers (which will find a + // match if the name in kwnames is interned, given that s is interned + // by Cython). A second loop compares the actual strings. + Py_ssize_t i, n = PyTuple_GET_SIZE(kwnames); + for (i = 0; i < n; i++) + { + if (s == PyTuple_GET_ITEM(kwnames, i)) return kwvalues[i]; + } + for (i = 0; i < n; i++) + { + int eq = __Pyx_PyUnicode_Equals(s, PyTuple_GET_ITEM(kwnames, i), Py_EQ); + if (unlikely(eq != 0)) { + if (unlikely(eq < 0)) return NULL; // error + return kwvalues[i]; + } + } + return NULL; // not found (no exception set) +} +#endif diff --git a/Cython/Utility/ImportExport.c b/Cython/Utility/ImportExport.c index e3c0cafaf..9082c1fb2 100644 --- a/Cython/Utility/ImportExport.c +++ b/Cython/Utility/ImportExport.c @@ -9,6 +9,147 @@ #endif +/////////////// ImportDottedModule.proto /////////////// + +static PyObject *__Pyx_ImportDottedModule(PyObject *name, PyObject *parts_tuple); /*proto*/ + +/////////////// ImportDottedModule /////////////// +//@requires: Import + +#if PY_MAJOR_VERSION >= 3 +static PyObject *__Pyx__ImportDottedModule_Error(PyObject *name, PyObject *parts_tuple, Py_ssize_t count) { + PyObject *partial_name = NULL, *slice = NULL, *sep = NULL; + if (unlikely(PyErr_Occurred())) { + PyErr_Clear(); + } + if (likely(PyTuple_GET_SIZE(parts_tuple) == count)) { + partial_name = name; + } else { + slice = PySequence_GetSlice(parts_tuple, 0, count); + if (unlikely(!slice)) + goto bad; + sep = PyUnicode_FromStringAndSize(".", 1); + if (unlikely(!sep)) + goto bad; + partial_name = PyUnicode_Join(sep, slice); + } + + PyErr_Format( +#if PY_MAJOR_VERSION < 3 + PyExc_ImportError, + "No module named '%s'", PyString_AS_STRING(partial_name)); +#else +#if PY_VERSION_HEX >= 0x030600B1 + PyExc_ModuleNotFoundError, +#else + PyExc_ImportError, +#endif + "No module named '%U'", partial_name); +#endif + +bad: + Py_XDECREF(sep); + Py_XDECREF(slice); + Py_XDECREF(partial_name); + return NULL; +} +#endif + +#if PY_MAJOR_VERSION >= 3 +static PyObject *__Pyx__ImportDottedModule_Lookup(PyObject *name) { + PyObject *imported_module; +#if PY_VERSION_HEX < 0x030700A1 || (CYTHON_COMPILING_IN_PYPY && PYPY_VERSION_NUM < 0x07030400) + PyObject *modules = PyImport_GetModuleDict(); + if (unlikely(!modules)) + return NULL; + imported_module = __Pyx_PyDict_GetItemStr(modules, name); + Py_XINCREF(imported_module); +#else + imported_module = PyImport_GetModule(name); +#endif + return imported_module; +} +#endif + +static PyObject *__Pyx__ImportDottedModule(PyObject *name, PyObject *parts_tuple) { +#if PY_MAJOR_VERSION < 3 + PyObject *module, *from_list, *star = PYIDENT("*"); + CYTHON_UNUSED_VAR(parts_tuple); + from_list = PyList_New(1); + if (unlikely(!from_list)) + return NULL; + Py_INCREF(star); + PyList_SET_ITEM(from_list, 0, star); + module = __Pyx_Import(name, from_list, 0); + Py_DECREF(from_list); + return module; +#else + Py_ssize_t i, nparts; + PyObject *imported_module; + PyObject *module = __Pyx_Import(name, NULL, 0); + if (!parts_tuple || unlikely(!module)) + return module; + + // Look up module in sys.modules, which is safer than the attribute lookups below. + imported_module = __Pyx__ImportDottedModule_Lookup(name); + if (likely(imported_module)) { + Py_DECREF(module); + return imported_module; + } + PyErr_Clear(); + + nparts = PyTuple_GET_SIZE(parts_tuple); + for (i=1; i < nparts && module; i++) { + PyObject *part, *submodule; +#if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + part = PyTuple_GET_ITEM(parts_tuple, i); +#else + part = PySequence_ITEM(parts_tuple, i); +#endif + submodule = __Pyx_PyObject_GetAttrStrNoError(module, part); +#if !(CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS) + Py_DECREF(part); +#endif + Py_DECREF(module); + module = submodule; + } + if (likely(module)) + return module; + return __Pyx__ImportDottedModule_Error(name, parts_tuple, i); +#endif +} + +static PyObject *__Pyx_ImportDottedModule(PyObject *name, PyObject *parts_tuple) { +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030400B1 + PyObject *module = __Pyx__ImportDottedModule_Lookup(name); + if (likely(module)) { + // CPython guards against thread-concurrent initialisation in importlib. + // In this case, we let PyImport_ImportModuleLevelObject() handle the locking. + PyObject *spec = __Pyx_PyObject_GetAttrStrNoError(module, PYIDENT("__spec__")); + if (likely(spec)) { + PyObject *unsafe = __Pyx_PyObject_GetAttrStrNoError(spec, PYIDENT("_initializing")); + if (likely(!unsafe || !__Pyx_PyObject_IsTrue(unsafe))) { + Py_DECREF(spec); + spec = NULL; + } + Py_XDECREF(unsafe); + } + if (likely(!spec)) { + // Not in initialisation phase => use modules as is. + PyErr_Clear(); + return module; + } + Py_DECREF(spec); + Py_DECREF(module); + } else if (PyErr_Occurred()) { + PyErr_Clear(); + } +#endif + + return __Pyx__ImportDottedModule(name, parts_tuple); +} + + /////////////// Import.proto /////////////// static PyObject *__Pyx_Import(PyObject *name, PyObject *from_list, int level); /*proto*/ @@ -18,30 +159,23 @@ static PyObject *__Pyx_Import(PyObject *name, PyObject *from_list, int level); / //@substitute: naming static PyObject *__Pyx_Import(PyObject *name, PyObject *from_list, int level) { - PyObject *empty_list = 0; PyObject *module = 0; - PyObject *global_dict = 0; PyObject *empty_dict = 0; - PyObject *list; + PyObject *empty_list = 0; #if PY_MAJOR_VERSION < 3 PyObject *py_import; py_import = __Pyx_PyObject_GetAttrStr($builtins_cname, PYIDENT("__import__")); - if (!py_import) + if (unlikely(!py_import)) goto bad; - #endif - if (from_list) - list = from_list; - else { + if (!from_list) { empty_list = PyList_New(0); - if (!empty_list) + if (unlikely(!empty_list)) goto bad; - list = empty_list; + from_list = empty_list; } - global_dict = PyModule_GetDict($module_cname); - if (!global_dict) - goto bad; + #endif empty_dict = PyDict_New(); - if (!empty_dict) + if (unlikely(!empty_dict)) goto bad; { #if PY_MAJOR_VERSION >= 3 @@ -49,10 +183,15 @@ static PyObject *__Pyx_Import(PyObject *name, PyObject *from_list, int level) { // Avoid C compiler warning if strchr() evaluates to false at compile time. if ((1) && (strchr(__Pyx_MODULE_NAME, '.'))) { /* try package relative import first */ + #if CYTHON_COMPILING_IN_LIMITED_API + module = PyImport_ImportModuleLevelObject( + name, empty_dict, empty_dict, from_list, 1); + #else module = PyImport_ImportModuleLevelObject( - name, global_dict, empty_dict, list, 1); - if (!module) { - if (!PyErr_ExceptionMatches(PyExc_ImportError)) + name, $moddict_cname, empty_dict, from_list, 1); + #endif + if (unlikely(!module)) { + if (unlikely(!PyErr_ExceptionMatches(PyExc_ImportError))) goto bad; PyErr_Clear(); } @@ -63,23 +202,28 @@ static PyObject *__Pyx_Import(PyObject *name, PyObject *from_list, int level) { if (!module) { #if PY_MAJOR_VERSION < 3 PyObject *py_level = PyInt_FromLong(level); - if (!py_level) + if (unlikely(!py_level)) goto bad; module = PyObject_CallFunctionObjArgs(py_import, - name, global_dict, empty_dict, list, py_level, (PyObject *)NULL); + name, $moddict_cname, empty_dict, from_list, py_level, (PyObject *)NULL); Py_DECREF(py_level); #else + #if CYTHON_COMPILING_IN_LIMITED_API module = PyImport_ImportModuleLevelObject( - name, global_dict, empty_dict, list, level); + name, empty_dict, empty_dict, from_list, level); + #else + module = PyImport_ImportModuleLevelObject( + name, $moddict_cname, empty_dict, from_list, level); + #endif #endif } } bad: + Py_XDECREF(empty_dict); + Py_XDECREF(empty_list); #if PY_MAJOR_VERSION < 3 Py_XDECREF(py_import); #endif - Py_XDECREF(empty_list); - Py_XDECREF(empty_dict); return module; } @@ -333,7 +477,7 @@ static PyTypeObject *__Pyx_ImportType(PyObject *module, const char *module_name, PyObject *result = 0; char warning[200]; Py_ssize_t basicsize; -#ifdef Py_LIMITED_API +#if CYTHON_COMPILING_IN_LIMITED_API PyObject *py_basicsize; #endif @@ -346,7 +490,7 @@ static PyTypeObject *__Pyx_ImportType(PyObject *module, const char *module_name, module_name, class_name); goto bad; } -#ifndef Py_LIMITED_API +#if !CYTHON_COMPILING_IN_LIMITED_API basicsize = ((PyTypeObject *)result)->tp_basicsize; #else py_basicsize = PyObject_GetAttrString(result, "__basicsize__"); @@ -414,7 +558,6 @@ static int __Pyx_ImportFunction(PyObject *module, const char *funcname, void (** PyModule_GetName(module), funcname); goto bad; } -#if PY_VERSION_HEX >= 0x02070000 if (!PyCapsule_IsValid(cobj, sig)) { PyErr_Format(PyExc_TypeError, "C function %.200s.%.200s has wrong signature (expected %.500s, got %.500s)", @@ -422,21 +565,6 @@ static int __Pyx_ImportFunction(PyObject *module, const char *funcname, void (** goto bad; } tmp.p = PyCapsule_GetPointer(cobj, sig); -#else - {const char *desc, *s1, *s2; - desc = (const char *)PyCObject_GetDesc(cobj); - if (!desc) - goto bad; - s1 = desc; s2 = sig; - while (*s1 != '\0' && *s1 == *s2) { s1++; s2++; } - if (*s1 != *s2) { - PyErr_Format(PyExc_TypeError, - "C function %.200s.%.200s has wrong signature (expected %.500s, got %.500s)", - PyModule_GetName(module), funcname, sig, desc); - goto bad; - } - tmp.p = PyCObject_AsVoidPtr(cobj);} -#endif *f = tmp.fp; if (!(*f)) goto bad; @@ -474,11 +602,7 @@ static int __Pyx_ExportFunction(const char *name, void (*f)(void), const char *s goto bad; } tmp.fp = f; -#if PY_VERSION_HEX >= 0x02070000 cobj = PyCapsule_New(tmp.p, sig, 0); -#else - cobj = PyCObject_FromVoidPtrAndDesc(tmp.p, (void *)sig, 0); -#endif if (!cobj) goto bad; if (PyDict_SetItemString(d, name, cobj) < 0) @@ -515,7 +639,6 @@ static int __Pyx_ImportVoidPtr(PyObject *module, const char *name, void **p, con PyModule_GetName(module), name); goto bad; } -#if PY_VERSION_HEX >= 0x02070000 if (!PyCapsule_IsValid(cobj, sig)) { PyErr_Format(PyExc_TypeError, "C variable %.200s.%.200s has wrong signature (expected %.500s, got %.500s)", @@ -523,21 +646,6 @@ static int __Pyx_ImportVoidPtr(PyObject *module, const char *name, void **p, con goto bad; } *p = PyCapsule_GetPointer(cobj, sig); -#else - {const char *desc, *s1, *s2; - desc = (const char *)PyCObject_GetDesc(cobj); - if (!desc) - goto bad; - s1 = desc; s2 = sig; - while (*s1 != '\0' && *s1 == *s2) { s1++; s2++; } - if (*s1 != *s2) { - PyErr_Format(PyExc_TypeError, - "C variable %.200s.%.200s has wrong signature (expected %.500s, got %.500s)", - PyModule_GetName(module), name, sig, desc); - goto bad; - } - *p = PyCObject_AsVoidPtr(cobj);} -#endif if (!(*p)) goto bad; Py_DECREF(d); @@ -569,11 +677,7 @@ static int __Pyx_ExportVoidPtr(PyObject *name, void *p, const char *sig) { if (__Pyx_PyObject_SetAttrStr($module_cname, PYIDENT("$api_name"), d) < 0) goto bad; } -#if PY_VERSION_HEX >= 0x02070000 cobj = PyCapsule_New(p, sig, 0); -#else - cobj = PyCObject_FromVoidPtrAndDesc(p, (void *)sig, 0); -#endif if (!cobj) goto bad; if (PyDict_SetItem(d, name, cobj) < 0) @@ -590,19 +694,19 @@ bad: /////////////// SetVTable.proto /////////////// -static int __Pyx_SetVtable(PyObject *dict, void *vtable); /*proto*/ +static int __Pyx_SetVtable(PyTypeObject* typeptr , void* vtable); /*proto*/ /////////////// SetVTable /////////////// -static int __Pyx_SetVtable(PyObject *dict, void *vtable) { -#if PY_VERSION_HEX >= 0x02070000 +static int __Pyx_SetVtable(PyTypeObject *type, void *vtable) { PyObject *ob = PyCapsule_New(vtable, 0, 0); + if (unlikely(!ob)) + goto bad; +#if CYTHON_COMPILING_IN_LIMITED_API + if (unlikely(PyObject_SetAttr((PyObject *) type, PYIDENT("__pyx_vtable__"), ob) < 0)) #else - PyObject *ob = PyCObject_FromVoidPtr(vtable, 0); + if (unlikely(PyDict_SetItem(type->tp_dict, PYIDENT("__pyx_vtable__"), ob) < 0)) #endif - if (!ob) - goto bad; - if (PyDict_SetItem(dict, PYIDENT("__pyx_vtable__"), ob) < 0) goto bad; Py_DECREF(ob); return 0; @@ -614,20 +718,20 @@ bad: /////////////// GetVTable.proto /////////////// -static void* __Pyx_GetVtable(PyObject *dict); /*proto*/ +static void* __Pyx_GetVtable(PyTypeObject *type); /*proto*/ /////////////// GetVTable /////////////// -static void* __Pyx_GetVtable(PyObject *dict) { +static void* __Pyx_GetVtable(PyTypeObject *type) { void* ptr; - PyObject *ob = PyObject_GetItem(dict, PYIDENT("__pyx_vtable__")); +#if CYTHON_COMPILING_IN_LIMITED_API + PyObject *ob = PyObject_GetAttr((PyObject *)type, PYIDENT("__pyx_vtable__")); +#else + PyObject *ob = PyObject_GetItem(type->tp_dict, PYIDENT("__pyx_vtable__")); +#endif if (!ob) goto bad; -#if PY_VERSION_HEX >= 0x02070000 ptr = PyCapsule_GetPointer(ob, 0); -#else - ptr = PyCObject_AsVoidPtr(ob); -#endif if (!ptr && !PyErr_Occurred()) PyErr_SetString(PyExc_RuntimeError, "invalid vtable found for imported type"); Py_DECREF(ob); @@ -641,13 +745,19 @@ bad: /////////////// MergeVTables.proto /////////////// //@requires: GetVTable +// TODO: find a way to make this work with the Limited API! +#if !CYTHON_COMPILING_IN_LIMITED_API static int __Pyx_MergeVtables(PyTypeObject *type); /*proto*/ +#endif /////////////// MergeVTables /////////////// +#if !CYTHON_COMPILING_IN_LIMITED_API static int __Pyx_MergeVtables(PyTypeObject *type) { int i; void** base_vtables; + __Pyx_TypeName tp_base_name; + __Pyx_TypeName base_name; void* unknown = (void*)-1; PyObject* bases = type->tp_bases; int base_depth = 0; @@ -667,13 +777,13 @@ static int __Pyx_MergeVtables(PyTypeObject *type) { // instance struct is so extended. (It would be good to also do this // check when a multiple-base class is created in pure Python as well.) for (i = 1; i < PyTuple_GET_SIZE(bases); i++) { - void* base_vtable = __Pyx_GetVtable(((PyTypeObject*)PyTuple_GET_ITEM(bases, i))->tp_dict); + void* base_vtable = __Pyx_GetVtable(((PyTypeObject*)PyTuple_GET_ITEM(bases, i))); if (base_vtable != NULL) { int j; PyTypeObject* base = type->tp_base; for (j = 0; j < base_depth; j++) { if (base_vtables[j] == unknown) { - base_vtables[j] = __Pyx_GetVtable(base->tp_dict); + base_vtables[j] = __Pyx_GetVtable(base); base_vtables[j + 1] = unknown; } if (base_vtables[j] == base_vtable) { @@ -690,13 +800,16 @@ static int __Pyx_MergeVtables(PyTypeObject *type) { free(base_vtables); return 0; bad: - PyErr_Format( - PyExc_TypeError, - "multiple bases have vtable conflict: '%s' and '%s'", - type->tp_base->tp_name, ((PyTypeObject*)PyTuple_GET_ITEM(bases, i))->tp_name); + tp_base_name = __Pyx_PyType_GetName(type->tp_base); + base_name = __Pyx_PyType_GetName((PyTypeObject*)PyTuple_GET_ITEM(bases, i)); + PyErr_Format(PyExc_TypeError, + "multiple bases have vtable conflict: '" __Pyx_FMT_TYPENAME "' and '" __Pyx_FMT_TYPENAME "'", tp_base_name, base_name); + __Pyx_DECREF_TypeName(tp_base_name); + __Pyx_DECREF_TypeName(base_name); free(base_vtables); return -1; } +#endif /////////////// ImportNumPyArray.proto /////////////// diff --git a/Cython/Utility/MemoryView.pyx b/Cython/Utility/MemoryView.pyx index 6ca5fab9b..1496755f1 100644 --- a/Cython/Utility/MemoryView.pyx +++ b/Cython/Utility/MemoryView.pyx @@ -1,5 +1,8 @@ #################### View.MemoryView #################### +# cython: language_level=3str +# cython: binding=False + # This utility provides cython.array and cython.view.memoryview from __future__ import absolute_import @@ -8,8 +11,12 @@ cimport cython # from cpython cimport ... cdef extern from "Python.h": + ctypedef struct PyObject int PyIndex_Check(object) object PyLong_FromVoidPtr(void *) + PyObject *PyExc_IndexError + PyObject *PyExc_ValueError + PyObject *PyExc_MemoryError cdef extern from "pythread.h": ctypedef void *PyThread_type_lock @@ -49,7 +56,7 @@ cdef extern from *: Py_ssize_t suboffsets[{{max_dims}}] void __PYX_INC_MEMVIEW({{memviewslice_name}} *memslice, int have_gil) - void __PYX_XDEC_MEMVIEW({{memviewslice_name}} *memslice, int have_gil) + void __PYX_XCLEAR_MEMVIEW({{memviewslice_name}} *memslice, int have_gil) ctypedef struct __pyx_buffer "Py_buffer": PyObject *obj @@ -94,9 +101,6 @@ cdef extern from "<stdlib.h>": void free(void *) nogil void *memcpy(void *dest, void *src, size_t n) nogil - - - # ### cython.array class # @@ -123,17 +127,16 @@ cdef class array: mode="c", bint allocate_buffer=True): cdef int idx - cdef Py_ssize_t i, dim - cdef PyObject **p + cdef Py_ssize_t dim self.ndim = <int> len(shape) self.itemsize = itemsize if not self.ndim: - raise ValueError("Empty shape tuple for cython.array") + raise ValueError, "Empty shape tuple for cython.array" if itemsize <= 0: - raise ValueError("itemsize <= 0 for cython.array") + raise ValueError, "itemsize <= 0 for cython.array" if not isinstance(format, bytes): format = format.encode('ASCII') @@ -145,65 +148,58 @@ cdef class array: self._strides = self._shape + self.ndim if not self._shape: - raise MemoryError("unable to allocate shape and strides.") + raise MemoryError, "unable to allocate shape and strides." # cdef Py_ssize_t dim, stride for idx, dim in enumerate(shape): if dim <= 0: - raise ValueError("Invalid shape in axis %d: %d." % (idx, dim)) + raise ValueError, f"Invalid shape in axis {idx}: {dim}." self._shape[idx] = dim cdef char order - if mode == 'fortran': - order = b'F' - self.mode = u'fortran' - elif mode == 'c': + if mode == 'c': order = b'C' self.mode = u'c' + elif mode == 'fortran': + order = b'F' + self.mode = u'fortran' else: - raise ValueError("Invalid mode, expected 'c' or 'fortran', got %s" % mode) + raise ValueError, f"Invalid mode, expected 'c' or 'fortran', got {mode}" - self.len = fill_contig_strides_array(self._shape, self._strides, - itemsize, self.ndim, order) + self.len = fill_contig_strides_array(self._shape, self._strides, itemsize, self.ndim, order) self.free_data = allocate_buffer self.dtype_is_object = format == b'O' - if allocate_buffer: - # use malloc() for backwards compatibility - # in case external code wants to change the data pointer - self.data = <char *>malloc(self.len) - if not self.data: - raise MemoryError("unable to allocate array data.") - if self.dtype_is_object: - p = <PyObject **> self.data - for i in range(self.len / itemsize): - p[i] = Py_None - Py_INCREF(Py_None) + if allocate_buffer: + _allocate_buffer(self) @cname('getbuffer') def __getbuffer__(self, Py_buffer *info, int flags): cdef int bufmode = -1 - if self.mode == u"c": - bufmode = PyBUF_C_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS - elif self.mode == u"fortran": - bufmode = PyBUF_F_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS - if not (flags & bufmode): - raise ValueError("Can only create a buffer that is contiguous in memory.") + if flags & (PyBUF_C_CONTIGUOUS | PyBUF_F_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS): + if self.mode == u"c": + bufmode = PyBUF_C_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS + elif self.mode == u"fortran": + bufmode = PyBUF_F_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS + if not (flags & bufmode): + raise ValueError, "Can only create a buffer that is contiguous in memory." info.buf = self.data info.len = self.len - info.ndim = self.ndim - info.shape = self._shape - info.strides = self._strides - info.suboffsets = NULL - info.itemsize = self.itemsize - info.readonly = 0 - if flags & PyBUF_FORMAT: - info.format = self.format + if flags & PyBUF_STRIDES: + info.ndim = self.ndim + info.shape = self._shape + info.strides = self._strides else: - info.format = NULL + info.ndim = 1 + info.shape = &self.len if flags & PyBUF_ND else NULL + info.strides = NULL + info.suboffsets = NULL + info.itemsize = self.itemsize + info.readonly = 0 + info.format = self.format if flags & PyBUF_FORMAT else NULL info.obj = self __pyx_getbuffer = capsule(<void *> &__pyx_array_getbuffer, "getbuffer(obj, view, flags)") @@ -211,10 +207,9 @@ cdef class array: def __dealloc__(array self): if self.callback_free_data != NULL: self.callback_free_data(self.data) - elif self.free_data: + elif self.free_data and self.data is not NULL: if self.dtype_is_object: - refcount_objects_in_slice(self.data, self._shape, - self._strides, self.ndim, False) + refcount_objects_in_slice(self.data, self._shape, self._strides, self.ndim, inc=False) free(self.data) PyObject_Free(self._shape) @@ -240,16 +235,35 @@ cdef class array: self.memview[item] = value +@cname("__pyx_array_allocate_buffer") +cdef int _allocate_buffer(array self) except -1: + # use malloc() for backwards compatibility + # in case external code wants to change the data pointer + cdef Py_ssize_t i + cdef PyObject **p + + self.free_data = True + self.data = <char *>malloc(self.len) + if not self.data: + raise MemoryError, "unable to allocate array data." + + if self.dtype_is_object: + p = <PyObject **> self.data + for i in range(self.len // self.itemsize): + p[i] = Py_None + Py_INCREF(Py_None) + return 0 + + @cname("__pyx_array_new") -cdef array array_cwrapper(tuple shape, Py_ssize_t itemsize, char *format, - char *mode, char *buf): +cdef array array_cwrapper(tuple shape, Py_ssize_t itemsize, char *format, char *c_mode, char *buf): cdef array result + cdef str mode = "fortran" if c_mode[0] == b'f' else "c" # this often comes from a constant C string. - if buf == NULL: - result = array(shape, itemsize, format, mode.decode('ASCII')) + if buf is NULL: + result = array.__new__(array, shape, itemsize, format, mode) else: - result = array(shape, itemsize, format, mode.decode('ASCII'), - allocate_buffer=False) + result = array.__new__(array, shape, itemsize, format, mode, allocate_buffer=False) result.data = buf return result @@ -327,7 +341,7 @@ cdef PyThread_type_lock[THREAD_LOCKS_PREALLOCATED] __pyx_memoryview_thread_locks @cname('__pyx_memoryview') -cdef class memoryview(object): +cdef class memoryview: cdef object obj cdef object _size @@ -415,7 +429,7 @@ cdef class memoryview(object): def __setitem__(memoryview self, object index, object value): if self.view.readonly: - raise TypeError("Cannot assign to read-only memoryview") + raise TypeError, "Cannot assign to read-only memoryview" have_slices, index = _unellipsify(index, self.view.ndim) @@ -441,10 +455,10 @@ cdef class memoryview(object): cdef setitem_slice_assignment(self, dst, src): cdef {{memviewslice_name}} dst_slice cdef {{memviewslice_name}} src_slice + cdef {{memviewslice_name}} msrc = get_slice_from_memview(src, &src_slice)[0] + cdef {{memviewslice_name}} mdst = get_slice_from_memview(dst, &dst_slice)[0] - memoryview_copy_contents(get_slice_from_memview(src, &src_slice)[0], - get_slice_from_memview(dst, &dst_slice)[0], - src.ndim, dst.ndim, self.dtype_is_object) + memoryview_copy_contents(msrc, mdst, src.ndim, dst.ndim, self.dtype_is_object) cdef setitem_slice_assign_scalar(self, memoryview dst, value): cdef int array[128] @@ -492,7 +506,7 @@ cdef class memoryview(object): try: result = struct.unpack(self.view.format, bytesitem) except struct.error: - raise ValueError("Unable to convert item to object") + raise ValueError, "Unable to convert item to object" else: if len(self.view.format) == 1: return result[0] @@ -517,7 +531,7 @@ cdef class memoryview(object): @cname('getbuffer') def __getbuffer__(self, Py_buffer *info, int flags): if flags & PyBUF_WRITABLE and self.view.readonly: - raise ValueError("Cannot create writable memory view from read-only memoryview") + raise ValueError, "Cannot create writable memory view from read-only memoryview" if flags & PyBUF_ND: info.shape = self.view.shape @@ -557,6 +571,9 @@ cdef class memoryview(object): @property def base(self): + return self._get_base() + + cdef _get_base(self): return self.obj @property @@ -567,7 +584,7 @@ cdef class memoryview(object): def strides(self): if self.view.strides == NULL: # Note: we always ask for strides, so if this is not set it's a bug - raise ValueError("Buffer view does not expose strides") + raise ValueError, "Buffer view does not expose strides" return tuple([stride for stride in self.view.strides[:self.view.ndim]]) @@ -668,39 +685,34 @@ cdef tuple _unellipsify(object index, int ndim): Replace all ellipses with full slices and fill incomplete indices with full slices. """ - if not isinstance(index, tuple): - tup = (index,) - else: - tup = index + cdef Py_ssize_t idx + tup = <tuple>index if isinstance(index, tuple) else (index,) - result = [] + result = [slice(None)] * ndim have_slices = False seen_ellipsis = False - for idx, item in enumerate(tup): + idx = 0 + for item in tup: if item is Ellipsis: if not seen_ellipsis: - result.extend([slice(None)] * (ndim - len(tup) + 1)) + idx += ndim - len(tup) seen_ellipsis = True - else: - result.append(slice(None)) have_slices = True else: - if not isinstance(item, slice) and not PyIndex_Check(item): - raise TypeError("Cannot index with type '%s'" % type(item)) - - have_slices = have_slices or isinstance(item, slice) - result.append(item) - - nslices = ndim - len(result) - if nslices: - result.extend([slice(None)] * nslices) - + if isinstance(item, slice): + have_slices = True + elif not PyIndex_Check(item): + raise TypeError, f"Cannot index with type '{type(item)}'" + result[idx] = item + idx += 1 + + nslices = ndim - idx return have_slices or nslices, tuple(result) cdef assert_direct_dimensions(Py_ssize_t *suboffsets, int ndim): for suboffset in suboffsets[:ndim]: if suboffset >= 0: - raise ValueError("Indirect dimensions not supported") + raise ValueError, "Indirect dimensions not supported" # ### Slicing a memoryview @@ -740,15 +752,16 @@ cdef memoryview memview_slice(memoryview memview, object indices): # may not be as expected" cdef {{memviewslice_name}} *p_dst = &dst cdef int *p_suboffset_dim = &suboffset_dim - cdef Py_ssize_t start, stop, step + cdef Py_ssize_t start, stop, step, cindex cdef bint have_start, have_stop, have_step for dim, index in enumerate(indices): if PyIndex_Check(index): + cindex = index slice_memviewslice( p_dst, p_src.shape[dim], p_src.strides[dim], p_src.suboffsets[dim], dim, new_ndim, p_suboffset_dim, - index, 0, 0, # start, stop, step + cindex, 0, 0, # start, stop, step 0, 0, 0, # have_{start,stop,step} False) elif index is None: @@ -829,13 +842,16 @@ cdef int slice_memviewslice( if start < 0: start += shape if not 0 <= start < shape: - _err_dim(IndexError, "Index out of bounds (axis %d)", dim) + _err_dim(PyExc_IndexError, "Index out of bounds (axis %d)", dim) else: # index is a slice - negative_step = have_step != 0 and step < 0 - - if have_step and step == 0: - _err_dim(ValueError, "Step may not be zero (axis %d)", dim) + if have_step: + negative_step = step < 0 + if step == 0: + _err_dim(PyExc_ValueError, "Step may not be zero (axis %d)", dim) + else: + negative_step = False + step = 1 # check our bounds and set defaults if have_start: @@ -867,9 +883,6 @@ cdef int slice_memviewslice( else: stop = shape - if not have_step: - step = 1 - # len = ceil( (stop - start) / step ) with cython.cdivision(True): new_shape = (stop - start) // step @@ -885,7 +898,7 @@ cdef int slice_memviewslice( dst.shape[new_ndim] = new_shape dst.suboffsets[new_ndim] = suboffset - # Add the slicing or idexing offsets to the right suboffset or base data * + # Add the slicing or indexing offsets to the right suboffset or base data * if suboffset_dim[0] < 0: dst.data += start * stride else: @@ -896,7 +909,7 @@ cdef int slice_memviewslice( if new_ndim == 0: dst.data = (<char **> dst.data)[0] + suboffset else: - _err_dim(IndexError, "All dimensions preceding dimension %d " + _err_dim(PyExc_IndexError, "All dimensions preceding dimension %d " "must be indexed and not sliced", dim) else: suboffset_dim[0] = new_ndim @@ -914,7 +927,7 @@ cdef char *pybuffer_index(Py_buffer *view, char *bufp, Py_ssize_t index, cdef char *resultp if view.ndim == 0: - shape = view.len / itemsize + shape = view.len // itemsize stride = itemsize else: shape = view.shape[dim] @@ -925,10 +938,10 @@ cdef char *pybuffer_index(Py_buffer *view, char *bufp, Py_ssize_t index, if index < 0: index += view.shape[dim] if index < 0: - raise IndexError("Out of bounds on buffer access (axis %d)" % dim) + raise IndexError, f"Out of bounds on buffer access (axis {dim})" if index >= shape: - raise IndexError("Out of bounds on buffer access (axis %d)" % dim) + raise IndexError, f"Out of bounds on buffer access (axis {dim})" resultp = bufp + index * stride if suboffset >= 0: @@ -940,7 +953,7 @@ cdef char *pybuffer_index(Py_buffer *view, char *bufp, Py_ssize_t index, ### Transposing a memoryviewslice # @cname('__pyx_memslice_transpose') -cdef int transpose_memslice({{memviewslice_name}} *memslice) nogil except 0: +cdef int transpose_memslice({{memviewslice_name}} *memslice) nogil except -1: cdef int ndim = memslice.memview.view.ndim cdef Py_ssize_t *shape = memslice.shape @@ -948,15 +961,15 @@ cdef int transpose_memslice({{memviewslice_name}} *memslice) nogil except 0: # reverse strides and shape cdef int i, j - for i in range(ndim / 2): + for i in range(ndim // 2): j = ndim - 1 - i strides[i], strides[j] = strides[j], strides[i] shape[i], shape[j] = shape[j], shape[i] if memslice.suboffsets[i] >= 0 or memslice.suboffsets[j] >= 0: - _err(ValueError, "Cannot transpose memoryview with indirect dimensions") + _err(PyExc_ValueError, "Cannot transpose memoryview with indirect dimensions") - return 1 + return 0 # ### Creating new memoryview objects from slices and memoryviews @@ -974,7 +987,7 @@ cdef class _memoryviewslice(memoryview): cdef int (*to_dtype_func)(char *, object) except 0 def __dealloc__(self): - __PYX_XDEC_MEMVIEW(&self.from_slice, 1) + __PYX_XCLEAR_MEMVIEW(&self.from_slice, 1) cdef convert_item_to_object(self, char *itemp): if self.to_object_func != NULL: @@ -988,8 +1001,7 @@ cdef class _memoryviewslice(memoryview): else: memoryview.assign_item_from_object(self, itemp, value) - @property - def base(self): + cdef _get_base(self): return self.from_object __pyx_getbuffer = capsule(<void *> &__pyx_memoryview_getbuffer, "getbuffer(obj, view, flags)") @@ -1010,12 +1022,12 @@ cdef memoryview_fromslice({{memviewslice_name}} memviewslice, # assert 0 < ndim <= memviewslice.memview.view.ndim, ( # ndim, memviewslice.memview.view.ndim) - result = _memoryviewslice(None, 0, dtype_is_object) + result = _memoryviewslice.__new__(_memoryviewslice, None, 0, dtype_is_object) result.from_slice = memviewslice __PYX_INC_MEMVIEW(&memviewslice, 1) - result.from_object = (<memoryview> memviewslice.memview).base + result.from_object = (<memoryview> memviewslice.memview)._get_base() result.typeinfo = memviewslice.memview.typeinfo result.view = memviewslice.memview.view @@ -1107,10 +1119,7 @@ cdef memoryview_copy_from_slice(memoryview memview, {{memviewslice_name}} *memvi ### Copy the contents of a memoryview slices # cdef Py_ssize_t abs_py_ssize_t(Py_ssize_t arg) nogil: - if arg < 0: - return -arg - else: - return arg + return -arg if arg < 0 else arg @cname('__pyx_get_best_slice_order') cdef char get_best_order({{memviewslice_name}} *mslice, int ndim) nogil: @@ -1221,7 +1230,7 @@ cdef void *copy_data_to_temp({{memviewslice_name}} *src, result = malloc(size) if not result: - _err(MemoryError, NULL) + _err_no_memory() # tmpslice[0] = src tmpslice.data = <char *> result @@ -1230,8 +1239,7 @@ cdef void *copy_data_to_temp({{memviewslice_name}} *src, tmpslice.shape[i] = src.shape[i] tmpslice.suboffsets[i] = -1 - fill_contig_strides_array(&tmpslice.shape[0], &tmpslice.strides[0], itemsize, - ndim, order) + fill_contig_strides_array(&tmpslice.shape[0], &tmpslice.strides[0], itemsize, ndim, order) # We need to broadcast strides again for i in range(ndim): @@ -1250,19 +1258,20 @@ cdef void *copy_data_to_temp({{memviewslice_name}} *src, @cname('__pyx_memoryview_err_extents') cdef int _err_extents(int i, Py_ssize_t extent1, Py_ssize_t extent2) except -1 with gil: - raise ValueError("got differing extents in dimension %d (got %d and %d)" % - (i, extent1, extent2)) + raise ValueError, f"got differing extents in dimension {i} (got {extent1} and {extent2})" @cname('__pyx_memoryview_err_dim') -cdef int _err_dim(object error, char *msg, int dim) except -1 with gil: - raise error(msg.decode('ascii') % dim) +cdef int _err_dim(PyObject *error, str msg, int dim) except -1 with gil: + raise <object>error, msg % dim @cname('__pyx_memoryview_err') -cdef int _err(object error, char *msg) except -1 with gil: - if msg != NULL: - raise error(msg.decode('ascii')) - else: - raise error +cdef int _err(PyObject *error, str msg) except -1 with gil: + raise <object>error, msg + +@cname('__pyx_memoryview_err_no_memory') +cdef int _err_no_memory() except -1 with gil: + raise MemoryError + @cname('__pyx_memoryview_copy_contents') cdef int memoryview_copy_contents({{memviewslice_name}} src, @@ -1297,7 +1306,7 @@ cdef int memoryview_copy_contents({{memviewslice_name}} src, _err_extents(i, dst.shape[i], src.shape[i]) if src.suboffsets[i] >= 0: - _err_dim(ValueError, "Dimension %d is not direct", i) + _err_dim(PyExc_ValueError, "Dimension %d is not direct", i) if slices_overlap(&src, &dst, ndim, itemsize): # slices overlap, copy to temp, copy temp to dst @@ -1317,9 +1326,9 @@ cdef int memoryview_copy_contents({{memviewslice_name}} src, if direct_copy: # Contiguous slices with same order - refcount_copying(&dst, dtype_is_object, ndim, False) + refcount_copying(&dst, dtype_is_object, ndim, inc=False) memcpy(dst.data, src.data, slice_get_size(&src, ndim)) - refcount_copying(&dst, dtype_is_object, ndim, True) + refcount_copying(&dst, dtype_is_object, ndim, inc=True) free(tmpdata) return 0 @@ -1329,9 +1338,9 @@ cdef int memoryview_copy_contents({{memviewslice_name}} src, transpose_memslice(&src) transpose_memslice(&dst) - refcount_copying(&dst, dtype_is_object, ndim, False) + refcount_copying(&dst, dtype_is_object, ndim, inc=False) copy_strided_to_strided(&src, &dst, ndim, itemsize) - refcount_copying(&dst, dtype_is_object, ndim, True) + refcount_copying(&dst, dtype_is_object, ndim, inc=True) free(tmpdata) return 0 @@ -1359,13 +1368,10 @@ cdef void broadcast_leading({{memviewslice_name}} *mslice, # @cname('__pyx_memoryview_refcount_copying') -cdef void refcount_copying({{memviewslice_name}} *dst, bint dtype_is_object, - int ndim, bint inc) nogil: - # incref or decref the objects in the destination slice if the dtype is - # object +cdef void refcount_copying({{memviewslice_name}} *dst, bint dtype_is_object, int ndim, bint inc) nogil: + # incref or decref the objects in the destination slice if the dtype is object if dtype_is_object: - refcount_objects_in_slice_with_gil(dst.data, dst.shape, - dst.strides, ndim, inc) + refcount_objects_in_slice_with_gil(dst.data, dst.shape, dst.strides, ndim, inc) @cname('__pyx_memoryview_refcount_objects_in_slice_with_gil') cdef void refcount_objects_in_slice_with_gil(char *data, Py_ssize_t *shape, @@ -1377,6 +1383,7 @@ cdef void refcount_objects_in_slice_with_gil(char *data, Py_ssize_t *shape, cdef void refcount_objects_in_slice(char *data, Py_ssize_t *shape, Py_ssize_t *strides, int ndim, bint inc): cdef Py_ssize_t i + cdef Py_ssize_t stride = strides[0] for i in range(shape[0]): if ndim == 1: @@ -1385,10 +1392,9 @@ cdef void refcount_objects_in_slice(char *data, Py_ssize_t *shape, else: Py_DECREF((<PyObject **> data)[0]) else: - refcount_objects_in_slice(data, shape + 1, strides + 1, - ndim - 1, inc) + refcount_objects_in_slice(data, shape + 1, strides + 1, ndim - 1, inc) - data += strides[0] + data += stride # ### Scalar to slice assignment @@ -1397,10 +1403,9 @@ cdef void refcount_objects_in_slice(char *data, Py_ssize_t *shape, cdef void slice_assign_scalar({{memviewslice_name}} *dst, int ndim, size_t itemsize, void *item, bint dtype_is_object) nogil: - refcount_copying(dst, dtype_is_object, ndim, False) - _slice_assign_scalar(dst.data, dst.shape, dst.strides, ndim, - itemsize, item) - refcount_copying(dst, dtype_is_object, ndim, True) + refcount_copying(dst, dtype_is_object, ndim, inc=False) + _slice_assign_scalar(dst.data, dst.shape, dst.strides, ndim, itemsize, item) + refcount_copying(dst, dtype_is_object, ndim, inc=True) @cname('__pyx_memoryview__slice_assign_scalar') @@ -1417,8 +1422,7 @@ cdef void _slice_assign_scalar(char *data, Py_ssize_t *shape, data += stride else: for i in range(extent): - _slice_assign_scalar(data, shape + 1, strides + 1, - ndim - 1, itemsize, item) + _slice_assign_scalar(data, shape + 1, strides + 1, ndim - 1, itemsize, item) data += stride @@ -1464,6 +1468,7 @@ cdef bytes format_from_typeinfo(__Pyx_TypeInfo *type): cdef __Pyx_StructField *field cdef __pyx_typeinfo_string fmt cdef bytes part, result + cdef Py_ssize_t i if type.typegroup == 'S': assert type.fields != NULL @@ -1485,10 +1490,9 @@ cdef bytes format_from_typeinfo(__Pyx_TypeInfo *type): result = alignment.join(parts) + b'}' else: fmt = __Pyx_TypeInfoToFormat(type) + result = fmt.string if type.arraysize[0]: - extents = [unicode(type.arraysize[i]) for i in range(type.ndim)] - result = (u"(%s)" % u','.join(extents)).encode('ascii') + fmt.string - else: - result = fmt.string + extents = [f"{type.arraysize[i]}" for i in range(type.ndim)] + result = f"({u','.join(extents)})".encode('ascii') + result return result diff --git a/Cython/Utility/MemoryView_C.c b/Cython/Utility/MemoryView_C.c index 0a5d8ee2c..07ed24d20 100644 --- a/Cython/Utility/MemoryView_C.c +++ b/Cython/Utility/MemoryView_C.c @@ -113,9 +113,9 @@ static CYTHON_INLINE int __pyx_sub_acquisition_count_locked( #define __pyx_get_slice_count_pointer(memview) (memview->acquisition_count_aligned_p) #define __pyx_get_slice_count(memview) (*__pyx_get_slice_count_pointer(memview)) #define __PYX_INC_MEMVIEW(slice, have_gil) __Pyx_INC_MEMVIEW(slice, have_gil, __LINE__) -#define __PYX_XDEC_MEMVIEW(slice, have_gil) __Pyx_XDEC_MEMVIEW(slice, have_gil, __LINE__) +#define __PYX_XCLEAR_MEMVIEW(slice, have_gil) __Pyx_XCLEAR_MEMVIEW(slice, have_gil, __LINE__) static CYTHON_INLINE void __Pyx_INC_MEMVIEW({{memviewslice_name}} *, int, int); -static CYTHON_INLINE void __Pyx_XDEC_MEMVIEW({{memviewslice_name}} *, int, int); +static CYTHON_INLINE void __Pyx_XCLEAR_MEMVIEW({{memviewslice_name}} *, int, int); /////////////// MemviewSliceIndex.proto /////////////// @@ -230,8 +230,9 @@ fail: } static int -__pyx_check_suboffsets(Py_buffer *buf, int dim, CYTHON_UNUSED int ndim, int spec) +__pyx_check_suboffsets(Py_buffer *buf, int dim, int ndim, int spec) { + CYTHON_UNUSED_VAR(ndim); // Todo: without PyBUF_INDIRECT we may not have suboffset information, i.e., the // ptr may not be set to NULL but may be uninitialized? if (spec & __Pyx_MEMVIEW_DIRECT) { @@ -487,47 +488,49 @@ __pyx_sub_acquisition_count_locked(__pyx_atomic_int *acquisition_count, static CYTHON_INLINE void __Pyx_INC_MEMVIEW({{memviewslice_name}} *memslice, int have_gil, int lineno) { - int first_time; + __pyx_atomic_int_type old_acquisition_count; struct {{memview_struct_name}} *memview = memslice->memview; - if (unlikely(!memview || (PyObject *) memview == Py_None)) - return; /* allow uninitialized memoryview assignment */ - - if (unlikely(__pyx_get_slice_count(memview) < 0)) - __pyx_fatalerror("Acquisition count is %d (line %d)", - __pyx_get_slice_count(memview), lineno); - - first_time = __pyx_add_acquisition_count(memview) == 0; + if (unlikely(!memview || (PyObject *) memview == Py_None)) { + // Allow uninitialized memoryview assignment and do not ref-count None. + return; + } - if (unlikely(first_time)) { - if (have_gil) { - Py_INCREF((PyObject *) memview); + old_acquisition_count = __pyx_add_acquisition_count(memview); + if (unlikely(old_acquisition_count <= 0)) { + if (likely(old_acquisition_count == 0)) { + // First acquisition => keep the memoryview object alive. + if (have_gil) { + Py_INCREF((PyObject *) memview); + } else { + PyGILState_STATE _gilstate = PyGILState_Ensure(); + Py_INCREF((PyObject *) memview); + PyGILState_Release(_gilstate); + } } else { - PyGILState_STATE _gilstate = PyGILState_Ensure(); - Py_INCREF((PyObject *) memview); - PyGILState_Release(_gilstate); + __pyx_fatalerror("Acquisition count is %d (line %d)", + __pyx_get_slice_count(memview), lineno); } } } -static CYTHON_INLINE void __Pyx_XDEC_MEMVIEW({{memviewslice_name}} *memslice, +static CYTHON_INLINE void __Pyx_XCLEAR_MEMVIEW({{memviewslice_name}} *memslice, int have_gil, int lineno) { - int last_time; + __pyx_atomic_int_type old_acquisition_count; struct {{memview_struct_name}} *memview = memslice->memview; if (unlikely(!memview || (PyObject *) memview == Py_None)) { - // we do not ref-count None + // Do not ref-count None. memslice->memview = NULL; return; } - if (unlikely(__pyx_get_slice_count(memview) <= 0)) - __pyx_fatalerror("Acquisition count is %d (line %d)", - __pyx_get_slice_count(memview), lineno); - - last_time = __pyx_sub_acquisition_count(memview) == 1; + old_acquisition_count = __pyx_sub_acquisition_count(memview); memslice->data = NULL; - - if (unlikely(last_time)) { + if (likely(old_acquisition_count > 1)) { + // Still other slices out there => we do not own the reference. + memslice->memview = NULL; + } else if (likely(old_acquisition_count == 1)) { + // Last slice => discard owned Python reference to memoryview object. if (have_gil) { Py_CLEAR(memslice->memview); } else { @@ -536,7 +539,8 @@ static CYTHON_INLINE void __Pyx_XDEC_MEMVIEW({{memviewslice_name}} *memslice, PyGILState_Release(_gilstate); } } else { - memslice->memview = NULL; + __pyx_fatalerror("Acquisition count is %d (line %d)", + __pyx_get_slice_count(memview), lineno); } } @@ -774,7 +778,7 @@ static CYTHON_INLINE PyObject *{{get_function}}(const char *itemp) { {{if from_py_function}} static CYTHON_INLINE int {{set_function}}(const char *itemp, PyObject *obj) { {{dtype}} value = {{from_py_function}}(obj); - if ({{error_condition}}) + if (unlikely({{error_condition}})) return 0; *({{dtype}} *) itemp = value; return 1; @@ -925,7 +929,7 @@ __pyx_fill_slice_{{dtype_name}}({{type_decl}} *p, Py_ssize_t extent, Py_ssize_t ////////// FillStrided1DScalar ////////// /* Fill a slice with a scalar value. The dimension is direct and strided or contiguous */ -/* This can be used as a callback for the memoryview object to efficienty assign a scalar */ +/* This can be used as a callback for the memoryview object to efficiently assign a scalar */ /* Currently unused */ static void __pyx_fill_slice_{{dtype_name}}({{type_decl}} *p, Py_ssize_t extent, Py_ssize_t stride, diff --git a/Cython/Utility/ModuleSetupCode.c b/Cython/Utility/ModuleSetupCode.c index 01c63eac2..dcd081742 100644 --- a/Cython/Utility/ModuleSetupCode.c +++ b/Cython/Utility/ModuleSetupCode.c @@ -1,3 +1,16 @@ +/////////////// InitLimitedAPI /////////////// + +#if defined(CYTHON_LIMITED_API) && 0 /* disabled: enabling Py_LIMITED_API needs more work */ + #ifndef Py_LIMITED_API + #if CYTHON_LIMITED_API+0 > 0x03030000 + #define Py_LIMITED_API CYTHON_LIMITED_API + #else + #define Py_LIMITED_API 0x03030000 + #endif + #endif +#endif + + /////////////// CModulePreamble /////////////// #include <stddef.h> /* For offsetof */ @@ -5,7 +18,7 @@ #define offsetof(type, member) ( (size_t) & ((type*)0) -> member ) #endif -#if !defined(WIN32) && !defined(MS_WINDOWS) +#if !defined(_WIN32) && !defined(WIN32) && !defined(MS_WINDOWS) #ifndef __stdcall #define __stdcall #endif @@ -29,9 +42,7 @@ #ifndef HAVE_LONG_LONG // CPython has required PY_LONG_LONG support for years, even if HAVE_LONG_LONG is not defined for us - #if PY_VERSION_HEX >= 0x02070000 - #define HAVE_LONG_LONG - #endif + #define HAVE_LONG_LONG #endif #ifndef PY_LONG_LONG @@ -44,11 +55,13 @@ #ifdef PYPY_VERSION #define CYTHON_COMPILING_IN_PYPY 1 - #define CYTHON_COMPILING_IN_PYSTON 0 #define CYTHON_COMPILING_IN_CPYTHON 0 + #define CYTHON_COMPILING_IN_LIMITED_API 0 #undef CYTHON_USE_TYPE_SLOTS #define CYTHON_USE_TYPE_SLOTS 0 + #undef CYTHON_USE_TYPE_SPECS + #define CYTHON_USE_TYPE_SPECS 0 #undef CYTHON_USE_PYTYPE_LOOKUP #define CYTHON_USE_PYTYPE_LOOKUP 0 #if PY_VERSION_HEX < 0x03050000 @@ -73,10 +86,19 @@ #define CYTHON_UNPACK_METHODS 0 #undef CYTHON_FAST_THREAD_STATE #define CYTHON_FAST_THREAD_STATE 0 + #undef CYTHON_FAST_GIL + #define CYTHON_FAST_GIL 0 + #undef CYTHON_METH_FASTCALL + #define CYTHON_METH_FASTCALL 0 #undef CYTHON_FAST_PYCALL #define CYTHON_FAST_PYCALL 0 + #ifndef CYTHON_PEP487_INIT_SUBCLASS + #define CYTHON_PEP487_INIT_SUBCLASS (PY_MAJOR_VERSION >= 3) + #endif #undef CYTHON_PEP489_MULTI_PHASE_INIT #define CYTHON_PEP489_MULTI_PHASE_INIT 0 + #undef CYTHON_USE_MODULE_STATE + #define CYTHON_USE_MODULE_STATE 0 #undef CYTHON_USE_TP_FINALIZE #define CYTHON_USE_TP_FINALIZE 0 #undef CYTHON_USE_DICT_VERSIONS @@ -84,44 +106,54 @@ #undef CYTHON_USE_EXC_INFO_STACK #define CYTHON_USE_EXC_INFO_STACK 0 -#elif defined(PYSTON_VERSION) +#elif defined(CYTHON_LIMITED_API) + // EXPERIMENTAL !! #define CYTHON_COMPILING_IN_PYPY 0 - #define CYTHON_COMPILING_IN_PYSTON 1 #define CYTHON_COMPILING_IN_CPYTHON 0 + #define CYTHON_COMPILING_IN_LIMITED_API 1 - #ifndef CYTHON_USE_TYPE_SLOTS - #define CYTHON_USE_TYPE_SLOTS 1 - #endif + #undef CYTHON_USE_TYPE_SLOTS + #define CYTHON_USE_TYPE_SLOTS 0 + #undef CYTHON_USE_TYPE_SPECS + #define CYTHON_USE_TYPE_SPECS 1 #undef CYTHON_USE_PYTYPE_LOOKUP #define CYTHON_USE_PYTYPE_LOOKUP 0 #undef CYTHON_USE_ASYNC_SLOTS #define CYTHON_USE_ASYNC_SLOTS 0 #undef CYTHON_USE_PYLIST_INTERNALS #define CYTHON_USE_PYLIST_INTERNALS 0 - #ifndef CYTHON_USE_UNICODE_INTERNALS - #define CYTHON_USE_UNICODE_INTERNALS 1 + #undef CYTHON_USE_UNICODE_INTERNALS + #define CYTHON_USE_UNICODE_INTERNALS 0 + #ifndef CYTHON_USE_UNICODE_WRITER + #define CYTHON_USE_UNICODE_WRITER 1 #endif - #undef CYTHON_USE_UNICODE_WRITER - #define CYTHON_USE_UNICODE_WRITER 0 #undef CYTHON_USE_PYLONG_INTERNALS #define CYTHON_USE_PYLONG_INTERNALS 0 #ifndef CYTHON_AVOID_BORROWED_REFS #define CYTHON_AVOID_BORROWED_REFS 0 #endif - #ifndef CYTHON_ASSUME_SAFE_MACROS - #define CYTHON_ASSUME_SAFE_MACROS 1 - #endif - #ifndef CYTHON_UNPACK_METHODS - #define CYTHON_UNPACK_METHODS 1 - #endif + #undef CYTHON_ASSUME_SAFE_MACROS + #define CYTHON_ASSUME_SAFE_MACROS 0 + #undef CYTHON_UNPACK_METHODS + #define CYTHON_UNPACK_METHODS 0 #undef CYTHON_FAST_THREAD_STATE #define CYTHON_FAST_THREAD_STATE 0 + #undef CYTHON_FAST_GIL + #define CYTHON_FAST_GIL 0 + #undef CYTHON_METH_FASTCALL + #define CYTHON_METH_FASTCALL 0 #undef CYTHON_FAST_PYCALL #define CYTHON_FAST_PYCALL 0 + #ifndef CYTHON_PEP487_INIT_SUBCLASS + #define CYTHON_PEP487_INIT_SUBCLASS 1 + #endif #undef CYTHON_PEP489_MULTI_PHASE_INIT #define CYTHON_PEP489_MULTI_PHASE_INIT 0 - #undef CYTHON_USE_TP_FINALIZE - #define CYTHON_USE_TP_FINALIZE 0 + #undef CYTHON_USE_MODULE_STATE + #define CYTHON_USE_MODULE_STATE 1 + #ifndef CYTHON_USE_TP_FINALIZE + #define CYTHON_USE_TP_FINALIZE 1 + #endif #undef CYTHON_USE_DICT_VERSIONS #define CYTHON_USE_DICT_VERSIONS 0 #undef CYTHON_USE_EXC_INFO_STACK @@ -129,17 +161,16 @@ #else #define CYTHON_COMPILING_IN_PYPY 0 - #define CYTHON_COMPILING_IN_PYSTON 0 #define CYTHON_COMPILING_IN_CPYTHON 1 + #define CYTHON_COMPILING_IN_LIMITED_API 0 #ifndef CYTHON_USE_TYPE_SLOTS #define CYTHON_USE_TYPE_SLOTS 1 #endif - #if PY_VERSION_HEX < 0x02070000 - // looks like calling _PyType_Lookup() isn't safe in Py<=2.6/3.1 - #undef CYTHON_USE_PYTYPE_LOOKUP - #define CYTHON_USE_PYTYPE_LOOKUP 0 - #elif !defined(CYTHON_USE_PYTYPE_LOOKUP) + #ifndef CYTHON_USE_TYPE_SPECS + #define CYTHON_USE_TYPE_SPECS 0 + #endif + #ifndef CYTHON_USE_PYTYPE_LOOKUP #define CYTHON_USE_PYTYPE_LOOKUP 1 #endif #if PY_MAJOR_VERSION < 3 @@ -148,10 +179,7 @@ #elif !defined(CYTHON_USE_ASYNC_SLOTS) #define CYTHON_USE_ASYNC_SLOTS 1 #endif - #if PY_VERSION_HEX < 0x02070000 - #undef CYTHON_USE_PYLONG_INTERNALS - #define CYTHON_USE_PYLONG_INTERNALS 0 - #elif !defined(CYTHON_USE_PYLONG_INTERNALS) + #ifndef CYTHON_USE_PYLONG_INTERNALS #define CYTHON_USE_PYLONG_INTERNALS 1 #endif #ifndef CYTHON_USE_PYLIST_INTERNALS @@ -178,20 +206,48 @@ #ifndef CYTHON_FAST_THREAD_STATE #define CYTHON_FAST_THREAD_STATE 1 #endif + #ifndef CYTHON_FAST_GIL + // Py3<3.5.2 does not support _PyThreadState_UncheckedGet(). + #define CYTHON_FAST_GIL (PY_MAJOR_VERSION < 3 || PY_VERSION_HEX >= 0x03060000) + #endif + #ifndef CYTHON_METH_FASTCALL + // CPython 3.6 introduced METH_FASTCALL but with slightly different + // semantics. It became stable starting from CPython 3.7. + #define CYTHON_METH_FASTCALL (PY_VERSION_HEX >= 0x030700A1) + #endif #ifndef CYTHON_FAST_PYCALL #define CYTHON_FAST_PYCALL 1 #endif - #ifndef CYTHON_PEP489_MULTI_PHASE_INIT - #define CYTHON_PEP489_MULTI_PHASE_INIT (PY_VERSION_HEX >= 0x03050000) + #ifndef CYTHON_PEP487_INIT_SUBCLASS + #define CYTHON_PEP487_INIT_SUBCLASS 1 #endif - #ifndef CYTHON_USE_TP_FINALIZE - #define CYTHON_USE_TP_FINALIZE (PY_VERSION_HEX >= 0x030400a1) + #if PY_VERSION_HEX < 0x03050000 + #undef CYTHON_PEP489_MULTI_PHASE_INIT + #define CYTHON_PEP489_MULTI_PHASE_INIT 0 + #elif !defined(CYTHON_PEP489_MULTI_PHASE_INIT) + #define CYTHON_PEP489_MULTI_PHASE_INIT 1 + #endif + #ifndef CYTHON_USE_MODULE_STATE + // EXPERIMENTAL !! + #define CYTHON_USE_MODULE_STATE 0 #endif - #ifndef CYTHON_USE_DICT_VERSIONS - #define CYTHON_USE_DICT_VERSIONS (PY_VERSION_HEX >= 0x030600B1) + #if PY_VERSION_HEX < 0x030400a1 + #undef CYTHON_USE_TP_FINALIZE + #define CYTHON_USE_TP_FINALIZE 0 + #elif !defined(CYTHON_USE_TP_FINALIZE) + #define CYTHON_USE_TP_FINALIZE 1 #endif - #ifndef CYTHON_USE_EXC_INFO_STACK - #define CYTHON_USE_EXC_INFO_STACK (PY_VERSION_HEX >= 0x030700A3) + #if PY_VERSION_HEX < 0x030600B1 + #undef CYTHON_USE_DICT_VERSIONS + #define CYTHON_USE_DICT_VERSIONS 0 + #elif !defined(CYTHON_USE_DICT_VERSIONS) + #define CYTHON_USE_DICT_VERSIONS 1 + #endif + #if PY_VERSION_HEX < 0x030700A3 + #undef CYTHON_USE_EXC_INFO_STACK + #define CYTHON_USE_EXC_INFO_STACK 0 + #elif !defined(CYTHON_USE_EXC_INFO_STACK) + #define CYTHON_USE_EXC_INFO_STACK 1 #endif #endif @@ -199,6 +255,13 @@ #define CYTHON_FAST_PYCCALL (CYTHON_FAST_PYCALL && PY_VERSION_HEX >= 0x030600B1) #endif +#if !defined(CYTHON_VECTORCALL) +#define CYTHON_VECTORCALL (CYTHON_FAST_PYCCALL && PY_VERSION_HEX >= 0x030800B1) +#endif + +/* Whether to use METH_FASTCALL with a fake backported implementation of vectorcall */ +#define CYTHON_BACKPORT_VECTORCALL (CYTHON_METH_FASTCALL && PY_VERSION_HEX < 0x030800B1) + #if CYTHON_USE_PYLONG_INTERNALS #include "longintrepr.h" /* These short defines can easily conflict with other code */ @@ -247,14 +310,18 @@ # endif #endif -#ifndef CYTHON_MAYBE_UNUSED_VAR +#ifndef CYTHON_UNUSED_VAR # if defined(__cplusplus) - template<class T> void CYTHON_MAYBE_UNUSED_VAR( const T& ) { } + template<class T> void CYTHON_UNUSED_VAR( const T& ) { } # else -# define CYTHON_MAYBE_UNUSED_VAR(x) (void)(x) +# define CYTHON_UNUSED_VAR(x) (void)(x) # endif #endif +#ifndef CYTHON_MAYBE_UNUSED_VAR + #define CYTHON_MAYBE_UNUSED_VAR(x) CYTHON_UNUSED_VAR(x) +#endif + #ifndef CYTHON_NCP_UNUSED # if CYTHON_COMPILING_IN_CPYTHON # define CYTHON_NCP_UNUSED @@ -268,15 +335,31 @@ #ifdef _MSC_VER #ifndef _MSC_STDINT_H_ #if _MSC_VER < 1300 - typedef unsigned char uint8_t; - typedef unsigned int uint32_t; + typedef unsigned char uint8_t; + typedef unsigned short uint16_t; + typedef unsigned int uint32_t; #else - typedef unsigned __int8 uint8_t; - typedef unsigned __int32 uint32_t; + typedef unsigned __int8 uint8_t; + typedef unsigned __int16 uint16_t; + typedef unsigned __int32 uint32_t; + #endif + #endif + #if _MSC_VER < 1300 + #ifdef _WIN64 + typedef unsigned long long __pyx_uintptr_t; + #else + typedef unsigned int __pyx_uintptr_t; + #endif + #else + #ifdef _WIN64 + typedef unsigned __int64 __pyx_uintptr_t; + #else + typedef unsigned __int32 __pyx_uintptr_t; #endif #endif #else - #include <stdint.h> + #include <stdint.h> + typedef uintptr_t __pyx_uintptr_t; #endif @@ -340,7 +423,7 @@ #endif #endif -// Work around clang bug http://stackoverflow.com/questions/21847816/c-invoke-nested-template-class-destructor +// Work around clang bug https://stackoverflow.com/questions/21847816/c-invoke-nested-template-class-destructor template<typename T> void __Pyx_call_destructor(T& x) { x.~T(); @@ -376,19 +459,29 @@ class __Pyx_FakeReference { #if PY_MAJOR_VERSION < 3 #define __Pyx_BUILTIN_MODULE_NAME "__builtin__" - #define __Pyx_PyCode_New(a, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos) \ - PyCode_New(a+k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos) #define __Pyx_DefaultClassType PyClass_Type + #define __Pyx_PyCode_New(a, p, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos) \ + PyCode_New(a+k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos) #else #define __Pyx_BUILTIN_MODULE_NAME "builtins" -#if PY_VERSION_HEX >= 0x030800A4 && PY_VERSION_HEX < 0x030800B2 - #define __Pyx_PyCode_New(a, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos) \ - PyCode_New(a, 0, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos) + #define __Pyx_DefaultClassType PyType_Type +#if PY_VERSION_HEX >= 0x030800B2 + #define __Pyx_PyCode_New(a, p, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos) \ + PyCode_NewWithPosOnlyArgs(a, p, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos) +#elif PY_VERSION_HEX >= 0x030800A4 + // TODO: remove this special case once Py3.8 is released. + #define __Pyx_PyCode_New(a, p, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos) \ + PyCode_New(a, p, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos) #else - #define __Pyx_PyCode_New(a, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos) \ + #define __Pyx_PyCode_New(a, p, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos) \ PyCode_New(a, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos) #endif - #define __Pyx_DefaultClassType PyType_Type +#endif + +#if PY_VERSION_HEX >= 0x030900A4 || defined(Py_IS_TYPE) + #define __Pyx_IS_TYPE(ob, type) Py_IS_TYPE(ob, type) +#else + #define __Pyx_IS_TYPE(ob, type) (((const PyObject*)ob)->ob_type == (type)) #endif #ifndef Py_TPFLAGS_CHECKTYPES @@ -426,11 +519,41 @@ class __Pyx_FakeReference { #define __Pyx_PyCFunctionFast _PyCFunctionFast #define __Pyx_PyCFunctionFastWithKeywords _PyCFunctionFastWithKeywords #endif -#if CYTHON_FAST_PYCCALL -#define __Pyx_PyFastCFunction_Check(func) \ - ((PyCFunction_Check(func) && (METH_FASTCALL == (PyCFunction_GET_FLAGS(func) & ~(METH_CLASS | METH_STATIC | METH_COEXIST | METH_KEYWORDS | METH_STACKLESS))))) + +#if CYTHON_METH_FASTCALL + #define __Pyx_METH_FASTCALL METH_FASTCALL + #define __Pyx_PyCFunction_FastCall __Pyx_PyCFunctionFast + #define __Pyx_PyCFunction_FastCallWithKeywords __Pyx_PyCFunctionFastWithKeywords +#else + #define __Pyx_METH_FASTCALL METH_VARARGS + #define __Pyx_PyCFunction_FastCall PyCFunction + #define __Pyx_PyCFunction_FastCallWithKeywords PyCFunctionWithKeywords +#endif + +#if CYTHON_VECTORCALL + #define __pyx_vectorcallfunc vectorcallfunc + #define __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET PY_VECTORCALL_ARGUMENTS_OFFSET + #define __Pyx_PyVectorcall_NARGS(n) PyVectorcall_NARGS((size_t)(n)) +#elif CYTHON_BACKPORT_VECTORCALL + typedef PyObject *(*__pyx_vectorcallfunc)(PyObject *callable, PyObject *const *args, + size_t nargsf, PyObject *kwnames); + #define __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET ((size_t)1 << (8 * sizeof(size_t) - 1)) + #define __Pyx_PyVectorcall_NARGS(n) ((Py_ssize_t)(((size_t)(n)) & ~__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)) +#else + #define __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET 0 + #define __Pyx_PyVectorcall_NARGS(n) ((Py_ssize_t)(n)) +#endif + +// PEP-573: PyCFunction holds reference to defining class (PyCMethodObject) +#if PY_VERSION_HEX < 0x030900B1 + #define __Pyx_PyType_FromModuleAndSpec(m, s, b) ((void)m, PyType_FromSpecWithBases(s, b)) + typedef PyObject *(*__Pyx_PyCMethod)(PyObject *, PyTypeObject *, PyObject *const *, size_t, PyObject *); #else -#define __Pyx_PyFastCFunction_Check(func) 0 + #define __Pyx_PyType_FromModuleAndSpec(m, s, b) PyType_FromModuleAndSpec(m, s, b) + #define __Pyx_PyCMethod PyCMethod +#endif +#ifndef METH_METHOD + #define METH_METHOD 0x200 #endif #if CYTHON_COMPILING_IN_PYPY && !defined(PyObject_Malloc) @@ -445,16 +568,17 @@ class __Pyx_FakeReference { #define PyMem_RawFree(p) PyMem_Free(p) #endif -#if CYTHON_COMPILING_IN_PYSTON - // special C-API functions only in Pyston - #define __Pyx_PyCode_HasFreeVars(co) PyCode_HasFreeVars(co) - #define __Pyx_PyFrame_SetLineNumber(frame, lineno) PyFrame_SetLineNumber(frame, lineno) +#if CYTHON_COMPILING_IN_LIMITED_API + #define __Pyx_PyCode_HasFreeVars(co) (PyCode_GetNumFree(co) > 0) + #define __Pyx_PyFrame_SetLineNumber(frame, lineno) #else #define __Pyx_PyCode_HasFreeVars(co) (PyCode_GetNumFree(co) > 0) #define __Pyx_PyFrame_SetLineNumber(frame, lineno) (frame)->f_lineno = (lineno) #endif -#if !CYTHON_FAST_THREAD_STATE || PY_VERSION_HEX < 0x02070000 +#if CYTHON_COMPILING_IN_LIMITED_API + #define __Pyx_PyThreadState_Current PyThreadState_Get() +#elif !CYTHON_FAST_THREAD_STATE #define __Pyx_PyThreadState_Current PyThreadState_GET() #elif PY_VERSION_HEX >= 0x03060000 //#elif PY_VERSION_HEX >= 0x03050200 @@ -466,6 +590,18 @@ class __Pyx_FakeReference { #define __Pyx_PyThreadState_Current _PyThreadState_Current #endif +#if CYTHON_COMPILING_IN_LIMITED_API +static CYTHON_INLINE void *__Pyx_PyModule_GetState(PyObject *op) +{ + void *result; + + result = PyModule_GetState(op); + if (!result) + Py_FatalError("Couldn't find the module state"); + return result; +} +#endif + // TSS (Thread Specific Storage) API #if PY_VERSION_HEX < 0x030700A2 && !defined(PyThread_tss_create) && !defined(Py_tss_NEEDS_INIT) #include "pythread.h" @@ -496,7 +632,7 @@ static CYTHON_INLINE int PyThread_tss_set(Py_tss_t *key, void *value) { static CYTHON_INLINE void * PyThread_tss_get(Py_tss_t *key) { return PyThread_get_key_value(*key); } -// PyThread_delete_key_value(key) is equalivalent to PyThread_set_key_value(key, NULL) +// PyThread_delete_key_value(key) is equivalent to PyThread_set_key_value(key, NULL) // PyThread_ReInitTLS() is a no-op #endif /* TSS (Thread Specific Storage) API */ @@ -514,14 +650,85 @@ static CYTHON_INLINE void * PyThread_tss_get(Py_tss_t *key) { #define __Pyx_PyNumber_InPlaceDivide(x,y) PyNumber_InPlaceDivide(x,y) #endif -#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030500A1 && CYTHON_USE_UNICODE_INTERNALS -#define __Pyx_PyDict_GetItemStr(dict, name) _PyDict_GetItem_KnownHash(dict, name, ((PyASCIIObject *) name)->hash) +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX > 0x030600B4 && CYTHON_USE_UNICODE_INTERNALS +// _PyDict_GetItem_KnownHash() exists since CPython 3.5, but it was +// dropping exceptions. Since 3.6, exceptions are kept. +#define __Pyx_PyDict_GetItemStrWithError(dict, name) _PyDict_GetItem_KnownHash(dict, name, ((PyASCIIObject *) name)->hash) +static CYTHON_INLINE PyObject * __Pyx_PyDict_GetItemStr(PyObject *dict, PyObject *name) { + PyObject *res = __Pyx_PyDict_GetItemStrWithError(dict, name); + if (res == NULL) PyErr_Clear(); + return res; +} +#elif PY_MAJOR_VERSION >= 3 && (!CYTHON_COMPILING_IN_PYPY || PYPY_VERSION_NUM >= 0x07020000) +#define __Pyx_PyDict_GetItemStrWithError PyDict_GetItemWithError +#define __Pyx_PyDict_GetItemStr PyDict_GetItem #else -#define __Pyx_PyDict_GetItemStr(dict, name) PyDict_GetItem(dict, name) +static CYTHON_INLINE PyObject * __Pyx_PyDict_GetItemStrWithError(PyObject *dict, PyObject *name) { + // This is tricky - we should return a borrowed reference but not swallow non-KeyError exceptions. 8-| + // But: this function is only used in Py2 and older PyPys, + // and currently only for argument parsing and other non-correctness-critical lookups + // and we know that 'name' is an interned 'str' with pre-calculated hash value (only comparisons can fail), + // thus, performance matters more than correctness here, especially in the "not found" case. +#if CYTHON_COMPILING_IN_PYPY + // So we ignore any exceptions in old PyPys ... + return PyDict_GetItem(dict, name); +#else + // and hack together a stripped-down and modified PyDict_GetItem() in CPython 2. + PyDictEntry *ep; + PyDictObject *mp = (PyDictObject*) dict; + long hash = ((PyStringObject *) name)->ob_shash; + assert(hash != -1); /* hash values of interned strings are always initialised */ + ep = (mp->ma_lookup)(mp, name, hash); + if (ep == NULL) { + // error occurred + return NULL; + } + // found or not found + return ep->me_value; #endif +} +#define __Pyx_PyDict_GetItemStr PyDict_GetItem +#endif + +/* Type slots */ -/* new Py3.3 unicode type (PEP 393) */ -#if PY_VERSION_HEX > 0x03030000 && defined(PyUnicode_KIND) +#if CYTHON_USE_TYPE_SLOTS + #define __Pyx_PyType_GetFlags(tp) (((PyTypeObject *)tp)->tp_flags) + #define __Pyx_PyType_HasFeature(type, feature) ((__Pyx_PyType_GetFlags(type) & (feature)) != 0) + #define __Pyx_PyObject_GetIterNextFunc(obj) (Py_TYPE(obj)->tp_iternext) +#else + #define __Pyx_PyType_GetFlags(tp) (PyType_GetFlags((PyTypeObject *)tp)) + #define __Pyx_PyType_HasFeature(type, feature) PyType_HasFeature(type, feature) + #define __Pyx_PyObject_GetIterNextFunc(obj) PyIter_Next +#endif + +#if CYTHON_USE_TYPE_SPECS && PY_VERSION_HEX >= 0x03080000 +// In Py3.8+, instances of heap types need to decref their type on deallocation. +// https://bugs.python.org/issue35810 +#define __Pyx_PyHeapTypeObject_GC_Del(obj) { \ + PyTypeObject *type = Py_TYPE(obj); \ + assert(__Pyx_PyType_HasFeature(type, Py_TPFLAGS_HEAPTYPE)); \ + PyObject_GC_Del(obj); \ + Py_DECREF(type); \ +} +#else +#define __Pyx_PyHeapTypeObject_GC_Del(obj) PyObject_GC_Del(obj) +#endif + +#if CYTHON_COMPILING_IN_LIMITED_API + #define CYTHON_PEP393_ENABLED 1 + #define __Pyx_PyUnicode_READY(op) (0) + #define __Pyx_PyUnicode_GET_LENGTH(u) PyUnicode_GetLength(u) + #define __Pyx_PyUnicode_READ_CHAR(u, i) PyUnicode_ReadChar(u, i) + #define __Pyx_PyUnicode_MAX_CHAR_VALUE(u) ((void)u, 1114111) + #define __Pyx_PyUnicode_KIND(u) ((void)u, (0)) + // __Pyx_PyUnicode_DATA() and __Pyx_PyUnicode_READ() must go together, e.g. for iteration. + #define __Pyx_PyUnicode_DATA(u) ((void*)u) + #define __Pyx_PyUnicode_READ(k, d, i) ((void)k, PyUnicode_ReadChar((PyObject*)(d), i)) + //#define __Pyx_PyUnicode_WRITE(k, d, i, ch) /* not available */ + #define __Pyx_PyUnicode_IS_TRUE(u) (0 != PyUnicode_GetLength(u)) +#elif PY_VERSION_HEX > 0x03030000 && defined(PyUnicode_KIND) + /* new Py3.3 unicode type (PEP 393) */ #define CYTHON_PEP393_ENABLED 1 #if defined(PyUnicode_IS_READY) @@ -535,7 +742,7 @@ static CYTHON_INLINE void * PyThread_tss_get(Py_tss_t *key) { #define __Pyx_PyUnicode_GET_LENGTH(u) PyUnicode_GET_LENGTH(u) #define __Pyx_PyUnicode_READ_CHAR(u, i) PyUnicode_READ_CHAR(u, i) #define __Pyx_PyUnicode_MAX_CHAR_VALUE(u) PyUnicode_MAX_CHAR_VALUE(u) - #define __Pyx_PyUnicode_KIND(u) PyUnicode_KIND(u) + #define __Pyx_PyUnicode_KIND(u) ((int)PyUnicode_KIND(u)) #define __Pyx_PyUnicode_DATA(u) PyUnicode_DATA(u) #define __Pyx_PyUnicode_READ(k, d, i) PyUnicode_READ(k, d, i) #define __Pyx_PyUnicode_WRITE(k, d, i, ch) PyUnicode_WRITE(k, d, i, ch) @@ -559,9 +766,9 @@ static CYTHON_INLINE void * PyThread_tss_get(Py_tss_t *key) { #define __Pyx_PyUnicode_GET_LENGTH(u) PyUnicode_GET_SIZE(u) #define __Pyx_PyUnicode_READ_CHAR(u, i) ((Py_UCS4)(PyUnicode_AS_UNICODE(u)[i])) #define __Pyx_PyUnicode_MAX_CHAR_VALUE(u) ((sizeof(Py_UNICODE) == 2) ? 65535 : 1114111) - #define __Pyx_PyUnicode_KIND(u) (sizeof(Py_UNICODE)) + #define __Pyx_PyUnicode_KIND(u) ((int)sizeof(Py_UNICODE)) #define __Pyx_PyUnicode_DATA(u) ((void*)PyUnicode_AS_UNICODE(u)) - /* (void)(k) => avoid unused variable warning due to macro: */ + // (void)(k) => avoid unused variable warning due to macro: #define __Pyx_PyUnicode_READ(k, d, i) ((void)(k), (Py_UCS4)(((Py_UNICODE*)d)[i])) #define __Pyx_PyUnicode_WRITE(k, d, i, ch) (((void)(k)), ((Py_UNICODE*)d)[i] = ch) #define __Pyx_PyUnicode_IS_TRUE(u) (0 != PyUnicode_GET_SIZE(u)) @@ -576,16 +783,20 @@ static CYTHON_INLINE void * PyThread_tss_get(Py_tss_t *key) { PyNumber_Add(a, b) : __Pyx_PyUnicode_Concat(a, b)) #endif -#if CYTHON_COMPILING_IN_PYPY && !defined(PyUnicode_Contains) - #define PyUnicode_Contains(u, s) PySequence_Contains(u, s) -#endif - -#if CYTHON_COMPILING_IN_PYPY && !defined(PyByteArray_Check) - #define PyByteArray_Check(obj) PyObject_TypeCheck(obj, &PyByteArray_Type) -#endif - -#if CYTHON_COMPILING_IN_PYPY && !defined(PyObject_Format) - #define PyObject_Format(obj, fmt) PyObject_CallMethod(obj, "__format__", "O", fmt) +#if CYTHON_COMPILING_IN_PYPY + #if !defined(PyUnicode_DecodeUnicodeEscape) + #define PyUnicode_DecodeUnicodeEscape(s, size, errors) PyUnicode_Decode(s, size, "unicode_escape", errors) + #endif + #if !defined(PyUnicode_Contains) || (PY_MAJOR_VERSION == 2 && PYPY_VERSION_NUM < 0x07030500) + #undef PyUnicode_Contains + #define PyUnicode_Contains(u, s) PySequence_Contains(u, s) + #endif + #if !defined(PyByteArray_Check) + #define PyByteArray_Check(obj) PyObject_TypeCheck(obj, &PyByteArray_Type) + #endif + #if !defined(PyObject_Format) + #define PyObject_Format(obj, fmt) PyObject_CallMethod(obj, "__format__", "O", fmt) + #endif #endif // ("..." % x) must call PyNumber_Remainder() if x is a string subclass that implements "__rmod__()". @@ -622,10 +833,16 @@ static CYTHON_INLINE void * PyThread_tss_get(Py_tss_t *key) { #define __Pyx_PyBaseString_CheckExact(obj) (PyString_CheckExact(obj) || PyUnicode_CheckExact(obj)) #endif -#ifndef PySet_CheckExact - #define PySet_CheckExact(obj) (Py_TYPE(obj) == &PySet_Type) +#if CYTHON_COMPILING_IN_CPYTHON + #define __Pyx_PySequence_ListKeepNew(obj) \ + (likely(PyList_CheckExact(obj) && Py_REFCNT(obj) == 1) ? __Pyx_NewRef(obj) : PySequence_List(obj)) +#else + #define __Pyx_PySequence_ListKeepNew(obj) PySequence_List(obj) #endif +#ifndef PySet_CheckExact + #define PySet_CheckExact(obj) __Pyx_IS_TYPE(obj, &PySet_Type) +#endif #if PY_VERSION_HEX >= 0x030900A4 #define __Pyx_SET_REFCNT(obj, refcnt) Py_SET_REFCNT(obj, refcnt) @@ -673,16 +890,10 @@ static CYTHON_INLINE void * PyThread_tss_get(Py_tss_t *key) { #if PY_VERSION_HEX < 0x030200A4 typedef long Py_hash_t; #define __Pyx_PyInt_FromHash_t PyInt_FromLong - #define __Pyx_PyInt_AsHash_t PyInt_AsLong + #define __Pyx_PyInt_AsHash_t __Pyx_PyIndex_AsHash_t #else #define __Pyx_PyInt_FromHash_t PyInt_FromSsize_t - #define __Pyx_PyInt_AsHash_t PyInt_AsSsize_t -#endif - -#if PY_MAJOR_VERSION >= 3 - #define __Pyx_PyMethod_New(func, self, klass) ((self) ? ((void)(klass), PyMethod_New(func, self)) : __Pyx_NewRef(func)) -#else - #define __Pyx_PyMethod_New(func, self, klass) PyMethod_New(func, self, klass) + #define __Pyx_PyInt_AsHash_t __Pyx_PyIndex_AsSsize_t #endif // backport of PyAsyncMethods from Py3.5 to older Py3.x versions @@ -706,6 +917,11 @@ static CYTHON_INLINE void * PyThread_tss_get(Py_tss_t *key) { #endif +/////////////// IncludeStructmemberH.proto /////////////// + +#include <structmember.h> + + /////////////// SmallCodeConfig.proto /////////////// #ifndef CYTHON_SMALL_CODE @@ -746,14 +962,18 @@ static CYTHON_INLINE void * PyThread_tss_get(Py_tss_t *key) { #if CYTHON_COMPILING_IN_CPYTHON #define __Pyx_TypeCheck(obj, type) __Pyx_IsSubtype(Py_TYPE(obj), (PyTypeObject *)type) +#define __Pyx_TypeCheck2(obj, type1, type2) __Pyx_IsAnySubtype2(Py_TYPE(obj), (PyTypeObject *)type1, (PyTypeObject *)type2) static CYTHON_INLINE int __Pyx_IsSubtype(PyTypeObject *a, PyTypeObject *b);/*proto*/ +static CYTHON_INLINE int __Pyx_IsAnySubtype2(PyTypeObject *cls, PyTypeObject *a, PyTypeObject *b);/*proto*/ static CYTHON_INLINE int __Pyx_PyErr_GivenExceptionMatches(PyObject *err, PyObject *type);/*proto*/ static CYTHON_INLINE int __Pyx_PyErr_GivenExceptionMatches2(PyObject *err, PyObject *type1, PyObject *type2);/*proto*/ #else #define __Pyx_TypeCheck(obj, type) PyObject_TypeCheck(obj, (PyTypeObject *)type) +#define __Pyx_TypeCheck2(obj, type1, type2) (PyObject_TypeCheck(obj, (PyTypeObject *)type1) || PyObject_TypeCheck(obj, (PyTypeObject *)type2)) #define __Pyx_PyErr_GivenExceptionMatches(err, type) PyErr_GivenExceptionMatches(err, type) #define __Pyx_PyErr_GivenExceptionMatches2(err, type1, type2) (PyErr_GivenExceptionMatches(err, type1) || PyErr_GivenExceptionMatches(err, type2)) #endif +#define __Pyx_PyErr_ExceptionMatches2(err1, err2) __Pyx_PyErr_GivenExceptionMatches2(__Pyx_PyErr_Occurred(), err1, err2) #define __Pyx_PyException_Check(obj) __Pyx_TypeCheck(obj, PyExc_Exception) @@ -788,6 +1008,24 @@ static CYTHON_INLINE int __Pyx_IsSubtype(PyTypeObject *a, PyTypeObject *b) { return __Pyx_InBases(a, b); } +static CYTHON_INLINE int __Pyx_IsAnySubtype2(PyTypeObject *cls, PyTypeObject *a, PyTypeObject *b) { + PyObject *mro; + if (cls == a || cls == b) return 1; + mro = cls->tp_mro; + if (likely(mro)) { + Py_ssize_t i, n; + n = PyTuple_GET_SIZE(mro); + for (i = 0; i < n; i++) { + PyObject *base = PyTuple_GET_ITEM(mro, i); + if (base == (PyObject *)a || base == (PyObject *)b) + return 1; + } + return 0; + } + // should only get here for incompletely initialised types, i.e. never under normal usage patterns + return __Pyx_InBases(cls, a) || __Pyx_InBases(cls, b); +} + #if PY_MAJOR_VERSION == 2 static int __Pyx_inner_PyErr_GivenExceptionMatches2(PyObject *err, PyObject* exc_type1, PyObject* exc_type2) { @@ -818,11 +1056,11 @@ static int __Pyx_inner_PyErr_GivenExceptionMatches2(PyObject *err, PyObject* exc } #else static CYTHON_INLINE int __Pyx_inner_PyErr_GivenExceptionMatches2(PyObject *err, PyObject* exc_type1, PyObject *exc_type2) { - int res = exc_type1 ? __Pyx_IsSubtype((PyTypeObject*)err, (PyTypeObject*)exc_type1) : 0; - if (!res) { - res = __Pyx_IsSubtype((PyTypeObject*)err, (PyTypeObject*)exc_type2); + if (exc_type1) { + return __Pyx_IsAnySubtype2((PyTypeObject*)err, (PyTypeObject*)exc_type1, (PyTypeObject*)exc_type2); + } else { + return __Pyx_IsSubtype((PyTypeObject*)err, (PyTypeObject*)exc_type2); } - return res; } #endif @@ -883,7 +1121,7 @@ static CYTHON_INLINE int __Pyx_PyErr_GivenExceptionMatches2(PyObject *err, PyObj /////////////// MathInitCode /////////////// -#if defined(WIN32) || defined(MS_WINDOWS) +#if defined(_WIN32) || defined(WIN32) || defined(MS_WINDOWS) #define _USE_MATH_DEFINES #endif #include <math.h> @@ -957,12 +1195,21 @@ static CYTHON_SMALL_CODE int __Pyx_check_single_interpreter(void) { return 0; } -static CYTHON_SMALL_CODE int __Pyx_copy_spec_to_module(PyObject *spec, PyObject *moddict, const char* from_name, const char* to_name, int allow_none) { +#if CYTHON_COMPILING_IN_LIMITED_API +static CYTHON_SMALL_CODE int __Pyx_copy_spec_to_module(PyObject *spec, PyObject *module, const char* from_name, const char* to_name, int allow_none) +#else +static CYTHON_SMALL_CODE int __Pyx_copy_spec_to_module(PyObject *spec, PyObject *moddict, const char* from_name, const char* to_name, int allow_none) +#endif +{ PyObject *value = PyObject_GetAttrString(spec, from_name); int result = 0; if (likely(value)) { if (allow_none || value != Py_None) { +#if CYTHON_COMPILING_IN_LIMITED_API + result = PyModule_AddObject(module, to_name, value); +#else result = PyDict_SetItemString(moddict, to_name, value); +#endif } Py_DECREF(value); } else if (PyErr_ExceptionMatches(PyExc_AttributeError)) { @@ -973,8 +1220,9 @@ static CYTHON_SMALL_CODE int __Pyx_copy_spec_to_module(PyObject *spec, PyObject return result; } -static CYTHON_SMALL_CODE PyObject* ${pymodule_create_func_cname}(PyObject *spec, CYTHON_UNUSED PyModuleDef *def) { +static CYTHON_SMALL_CODE PyObject* ${pymodule_create_func_cname}(PyObject *spec, PyModuleDef *def) { PyObject *module = NULL, *moddict, *modname; + CYTHON_UNUSED_VAR(def); // For now, we only have exactly one module instance. if (__Pyx_check_single_interpreter()) @@ -989,9 +1237,13 @@ static CYTHON_SMALL_CODE PyObject* ${pymodule_create_func_cname}(PyObject *spec, Py_DECREF(modname); if (unlikely(!module)) goto bad; +#if CYTHON_COMPILING_IN_LIMITED_API + moddict = module; +#else moddict = PyModule_GetDict(module); if (unlikely(!moddict)) goto bad; // moddict is a borrowed reference +#endif if (unlikely(__Pyx_copy_spec_to_module(spec, moddict, "loader", "__loader__", 1) < 0)) goto bad; if (unlikely(__Pyx_copy_spec_to_module(spec, moddict, "origin", "__file__", 1) < 0)) goto bad; @@ -1008,6 +1260,7 @@ bad: /////////////// CodeObjectCache.proto /////////////// +#if !CYTHON_COMPILING_IN_LIMITED_API typedef struct { PyCodeObject* code_object; int code_line; @@ -1024,11 +1277,13 @@ static struct __Pyx_CodeObjectCache __pyx_code_cache = {0,0,NULL}; static int __pyx_bisect_code_objects(__Pyx_CodeObjectCacheEntry* entries, int count, int code_line); static PyCodeObject *__pyx_find_code_object(int code_line); static void __pyx_insert_code_object(int code_line, PyCodeObject* code_object); +#endif /////////////// CodeObjectCache /////////////// // Note that errors are simply ignored in the code below. // This is just a cache, if a lookup or insertion fails - so what? +#if !CYTHON_COMPILING_IN_LIMITED_API static int __pyx_bisect_code_objects(__Pyx_CodeObjectCacheEntry* entries, int count, int code_line) { int start = 0, mid = 0, end = count - 1; if (end >= 0 && code_line > entries[end].code_line) { @@ -1109,9 +1364,11 @@ static void __pyx_insert_code_object(int code_line, PyCodeObject* code_object) { __pyx_code_cache.count++; Py_INCREF(code_object); } +#endif /////////////// CodeObjectCache.cleanup /////////////// + #if !CYTHON_COMPILING_IN_LIMITED_API if (__pyx_code_cache.entries) { __Pyx_CodeObjectCacheEntry* entries = __pyx_code_cache.entries; int i, count = __pyx_code_cache.count; @@ -1123,6 +1380,7 @@ static void __pyx_insert_code_object(int code_line, PyCodeObject* code_object) { } PyMem_Free(entries); } + #endif /////////////// CheckBinaryVersion.proto /////////////// @@ -1169,11 +1427,11 @@ static CYTHON_INLINE int __Pyx_Is_Little_Endian(void) #if CYTHON_REFNANNY typedef struct { - void (*INCREF)(void*, PyObject*, int); - void (*DECREF)(void*, PyObject*, int); - void (*GOTREF)(void*, PyObject*, int); - void (*GIVEREF)(void*, PyObject*, int); - void* (*SetupContext)(const char*, int, const char*); + void (*INCREF)(void*, PyObject*, Py_ssize_t); + void (*DECREF)(void*, PyObject*, Py_ssize_t); + void (*GOTREF)(void*, PyObject*, Py_ssize_t); + void (*GIVEREF)(void*, PyObject*, Py_ssize_t); + void* (*SetupContext)(const char*, Py_ssize_t, const char*); void (*FinishContext)(void**); } __Pyx_RefNannyAPIStruct; static __Pyx_RefNannyAPIStruct *__Pyx_RefNanny = NULL; @@ -1183,28 +1441,35 @@ static CYTHON_INLINE int __Pyx_Is_Little_Endian(void) #define __Pyx_RefNannySetupContext(name, acquire_gil) \ if (acquire_gil) { \ PyGILState_STATE __pyx_gilstate_save = PyGILState_Ensure(); \ - __pyx_refnanny = __Pyx_RefNanny->SetupContext((name), __LINE__, __FILE__); \ + __pyx_refnanny = __Pyx_RefNanny->SetupContext((name), (__LINE__), (__FILE__)); \ PyGILState_Release(__pyx_gilstate_save); \ } else { \ - __pyx_refnanny = __Pyx_RefNanny->SetupContext((name), __LINE__, __FILE__); \ + __pyx_refnanny = __Pyx_RefNanny->SetupContext((name), (__LINE__), (__FILE__)); \ + } + #define __Pyx_RefNannyFinishContextNogil() { \ + PyGILState_STATE __pyx_gilstate_save = PyGILState_Ensure(); \ + __Pyx_RefNannyFinishContext(); \ + PyGILState_Release(__pyx_gilstate_save); \ } #else #define __Pyx_RefNannySetupContext(name, acquire_gil) \ - __pyx_refnanny = __Pyx_RefNanny->SetupContext((name), __LINE__, __FILE__) + __pyx_refnanny = __Pyx_RefNanny->SetupContext((name), (__LINE__), (__FILE__)) + #define __Pyx_RefNannyFinishContextNogil() __Pyx_RefNannyFinishContext() #endif #define __Pyx_RefNannyFinishContext() \ __Pyx_RefNanny->FinishContext(&__pyx_refnanny) - #define __Pyx_INCREF(r) __Pyx_RefNanny->INCREF(__pyx_refnanny, (PyObject *)(r), __LINE__) - #define __Pyx_DECREF(r) __Pyx_RefNanny->DECREF(__pyx_refnanny, (PyObject *)(r), __LINE__) - #define __Pyx_GOTREF(r) __Pyx_RefNanny->GOTREF(__pyx_refnanny, (PyObject *)(r), __LINE__) - #define __Pyx_GIVEREF(r) __Pyx_RefNanny->GIVEREF(__pyx_refnanny, (PyObject *)(r), __LINE__) - #define __Pyx_XINCREF(r) do { if((r) != NULL) {__Pyx_INCREF(r); }} while(0) - #define __Pyx_XDECREF(r) do { if((r) != NULL) {__Pyx_DECREF(r); }} while(0) - #define __Pyx_XGOTREF(r) do { if((r) != NULL) {__Pyx_GOTREF(r); }} while(0) - #define __Pyx_XGIVEREF(r) do { if((r) != NULL) {__Pyx_GIVEREF(r);}} while(0) + #define __Pyx_INCREF(r) __Pyx_RefNanny->INCREF(__pyx_refnanny, (PyObject *)(r), (__LINE__)) + #define __Pyx_DECREF(r) __Pyx_RefNanny->DECREF(__pyx_refnanny, (PyObject *)(r), (__LINE__)) + #define __Pyx_GOTREF(r) __Pyx_RefNanny->GOTREF(__pyx_refnanny, (PyObject *)(r), (__LINE__)) + #define __Pyx_GIVEREF(r) __Pyx_RefNanny->GIVEREF(__pyx_refnanny, (PyObject *)(r), (__LINE__)) + #define __Pyx_XINCREF(r) do { if((r) == NULL); else {__Pyx_INCREF(r); }} while(0) + #define __Pyx_XDECREF(r) do { if((r) == NULL); else {__Pyx_DECREF(r); }} while(0) + #define __Pyx_XGOTREF(r) do { if((r) == NULL); else {__Pyx_GOTREF(r); }} while(0) + #define __Pyx_XGIVEREF(r) do { if((r) == NULL); else {__Pyx_GIVEREF(r);}} while(0) #else #define __Pyx_RefNannyDeclarations #define __Pyx_RefNannySetupContext(name, acquire_gil) + #define __Pyx_RefNannyFinishContextNogil() #define __Pyx_RefNannyFinishContext() #define __Pyx_INCREF(r) Py_INCREF(r) #define __Pyx_DECREF(r) Py_DECREF(r) @@ -1216,6 +1481,10 @@ static CYTHON_INLINE int __Pyx_Is_Little_Endian(void) #define __Pyx_XGIVEREF(r) #endif /* CYTHON_REFNANNY */ +#define __Pyx_Py_XDECREF_SET(r, v) do { \ + PyObject *tmp = (PyObject *) r; \ + r = v; Py_XDECREF(tmp); \ + } while (0) #define __Pyx_XDECREF_SET(r, v) do { \ PyObject *tmp = (PyObject *) r; \ r = v; __Pyx_XDECREF(tmp); \ @@ -1275,7 +1544,8 @@ static int __Pyx_RegisterCleanup(void); /*proto*/ //@substitute: naming #if PY_MAJOR_VERSION < 3 || CYTHON_COMPILING_IN_PYPY -static PyObject* ${cleanup_cname}_atexit(PyObject *module, CYTHON_UNUSED PyObject *unused) { +static PyObject* ${cleanup_cname}_atexit(PyObject *module, PyObject *unused) { + CYTHON_UNUSED_VAR(unused); ${cleanup_cname}(module); Py_INCREF(Py_None); return Py_None; } @@ -1363,6 +1633,8 @@ __Pyx_FastGilFuncInit(); /////////////// FastGil.proto /////////////// //@proto_block: utility_code_proto_before_types +#if CYTHON_FAST_GIL + struct __Pyx_FastGilVtab { PyGILState_STATE (*Fast_PyGILState_Ensure)(void); void (*Fast_PyGILState_Release)(PyGILState_STATE oldstate); @@ -1397,6 +1669,14 @@ static void __Pyx_FastGilFuncInit(void); #endif #endif +#else +#define __Pyx_PyGILState_Ensure PyGILState_Ensure +#define __Pyx_PyGILState_Release PyGILState_Release +#define __Pyx_FastGIL_Remember() +#define __Pyx_FastGIL_Forget() +#define __Pyx_FastGilFuncInit() +#endif + /////////////// FastGil /////////////// //@requires: CommonStructures.c::FetchCommonPointer // The implementations of PyGILState_Ensure/Release calls PyThread_get_key_value @@ -1406,15 +1686,13 @@ static void __Pyx_FastGilFuncInit(void); // To make optimal use of this thread local, we attempt to share it between // modules. -#define __Pyx_FastGIL_ABI_module "_cython_" CYTHON_ABI +#if CYTHON_FAST_GIL + +#define __Pyx_FastGIL_ABI_module __PYX_ABI_MODULE_NAME #define __Pyx_FastGIL_PyCapsuleName "FastGilFuncs" #define __Pyx_FastGIL_PyCapsule \ __Pyx_FastGIL_ABI_module "." __Pyx_FastGIL_PyCapsuleName -#if PY_VERSION_HEX < 0x02070000 - #undef CYTHON_THREAD_LOCAL -#endif - #ifdef CYTHON_THREAD_LOCAL #include "pythread.h" @@ -1502,17 +1780,13 @@ static void __Pyx_FastGilFuncInit0(void) { #else static void __Pyx_FastGilFuncInit0(void) { - CYTHON_UNUSED void* force_use = (void*)&__Pyx_FetchCommonPointer; + CYTHON_UNUSED_VAR(&__Pyx_FetchCommonPointer); } #endif static void __Pyx_FastGilFuncInit(void) { -#if PY_VERSION_HEX >= 0x02070000 struct __Pyx_FastGilVtab* shared = (struct __Pyx_FastGilVtab*)PyCapsule_Import(__Pyx_FastGIL_PyCapsule, 1); -#else - struct __Pyx_FastGilVtab* shared = NULL; -#endif if (shared) { __Pyx_FastGilFuncs = *shared; } else { @@ -1520,3 +1794,22 @@ static void __Pyx_FastGilFuncInit(void) { __Pyx_FastGilFuncInit0(); } } + +#endif + +///////////////////// UtilityCodePragmas ///////////////////////// + +#if _MSC_VER +#pragma warning( push ) +/* Warning 4127: conditional expression is constant + * Cython uses constant conditional expressions to allow in inline functions to be optimized at + * compile-time, so this warning is not useful + */ +#pragma warning( disable : 4127 ) +#endif + +///////////////////// UtilityCodePragmasEnd ////////////////////// + +#if _MSV_VER +#pragma warning( pop ) /* undo whatever Cython has done to warnings */ +#endif diff --git a/Cython/Utility/NumpyImportArray.c b/Cython/Utility/NumpyImportArray.c new file mode 100644 index 000000000..aa8a18fea --- /dev/null +++ b/Cython/Utility/NumpyImportArray.c @@ -0,0 +1,21 @@ +///////////////////////// NumpyImportArray.init //////////////////// + +// comment below is deliberately kept in the generated C file to +// help users debug where this came from: +/* + * Cython has automatically inserted a call to _import_array since + * you didn't include one when you cimported numpy. To disable this + * add the line + * <void>numpy._import_array + */ +#ifdef NPY_NDARRAYOBJECT_H /* numpy headers have been included */ +// NO_IMPORT_ARRAY is Numpy's mechanism for indicating that import_array is handled elsewhere +#if !NO_IMPORT_ARRAY /* https://docs.scipy.org/doc/numpy-1.17.0/reference/c-api.array.html#c.NO_IMPORT_ARRAY */ +if (unlikely(_import_array() == -1)) { + PyErr_SetString(PyExc_ImportError, "numpy.core.multiarray failed to import " + "(auto-generated because you didn't call 'numpy.import_array()' after cimporting numpy; " + "use '<void>numpy._import_array' to disable if you are certain you don't need it)."); + {{ err_goto }}; +} +#endif +#endif diff --git a/Cython/Utility/ObjectHandling.c b/Cython/Utility/ObjectHandling.c index fb99e22e9..074bcd8e7 100644 --- a/Cython/Utility/ObjectHandling.c +++ b/Cython/Utility/ObjectHandling.c @@ -133,7 +133,7 @@ static int __Pyx_unpack_tuple2_generic(PyObject* tuple, PyObject** pvalue1, PyOb if (unlikely(!iter)) goto bad; if (decref_tuple) { Py_DECREF(tuple); tuple = NULL; } - iternext = Py_TYPE(iter)->tp_iternext; + iternext = __Pyx_PyObject_GetIterNextFunc(iter); value1 = iternext(iter); if (unlikely(!value1)) { index = 0; goto unpacking_failed; } value2 = iternext(iter); if (unlikely(!value2)) { index = 1; goto unpacking_failed; } if (!has_known_size && unlikely(__Pyx_IternextUnpackEndCheck(iternext(iter), 2))) goto bad; @@ -185,8 +185,10 @@ static PyObject *__Pyx_PyIter_Next2Default(PyObject* defval) { } static void __Pyx_PyIter_Next_ErrorNoIterator(PyObject *iterator) { + __Pyx_TypeName iterator_type_name = __Pyx_PyType_GetName(Py_TYPE(iterator)); PyErr_Format(PyExc_TypeError, - "%.200s object is not an iterator", Py_TYPE(iterator)->tp_name); + __Pyx_FMT_TYPENAME " object is not an iterator", iterator_type_name); + __Pyx_DECREF_TypeName(iterator_type_name); } // originally copied from Py3's builtin_next() @@ -199,10 +201,8 @@ static CYTHON_INLINE PyObject *__Pyx_PyIter_Next2(PyObject* iterator, PyObject* next = iternext(iterator); if (likely(next)) return next; - #if PY_VERSION_HEX >= 0x02070000 if (unlikely(iternext == &_PyObject_NextNotImplemented)) return NULL; - #endif #else // Since the slot was set, assume that PyIter_Next() will likely succeed, and properly fail otherwise. // Note: PyIter_Next() crashes in CPython if "tp_iternext" is NULL. @@ -275,24 +275,21 @@ static CYTHON_INLINE int __Pyx_IterFinish(void) { /////////////// ObjectGetItem.proto /////////////// #if CYTHON_USE_TYPE_SLOTS -static CYTHON_INLINE PyObject *__Pyx_PyObject_GetItem(PyObject *obj, PyObject* key);/*proto*/ +static CYTHON_INLINE PyObject *__Pyx_PyObject_GetItem(PyObject *obj, PyObject *key);/*proto*/ #else #define __Pyx_PyObject_GetItem(obj, key) PyObject_GetItem(obj, key) #endif /////////////// ObjectGetItem /////////////// // //@requires: GetItemInt - added in IndexNode as it uses templating. +//@requires: PyObjectGetAttrStrNoError +//@requires: PyObjectCallOneArg #if CYTHON_USE_TYPE_SLOTS -static PyObject *__Pyx_PyObject_GetIndex(PyObject *obj, PyObject* index) { +static PyObject *__Pyx_PyObject_GetIndex(PyObject *obj, PyObject *index) { + // Get element from sequence object `obj` at index `index`. PyObject *runerr; Py_ssize_t key_value; - PySequenceMethods *m = Py_TYPE(obj)->tp_as_sequence; - if (unlikely(!(m && m->sq_item))) { - PyErr_Format(PyExc_TypeError, "'%.200s' object is not subscriptable", Py_TYPE(obj)->tp_name); - return NULL; - } - key_value = __Pyx_PyIndex_AsSsize_t(index); if (likely(key_value != -1 || !(runerr = PyErr_Occurred()))) { return __Pyx_GetItemInt_Fast(obj, key_value, 0, 1, 1); @@ -300,18 +297,46 @@ static PyObject *__Pyx_PyObject_GetIndex(PyObject *obj, PyObject* index) { // Error handling code -- only manage OverflowError differently. if (PyErr_GivenExceptionMatches(runerr, PyExc_OverflowError)) { + __Pyx_TypeName index_type_name = __Pyx_PyType_GetName(Py_TYPE(index)); PyErr_Clear(); - PyErr_Format(PyExc_IndexError, "cannot fit '%.200s' into an index-sized integer", Py_TYPE(index)->tp_name); + PyErr_Format(PyExc_IndexError, + "cannot fit '" __Pyx_FMT_TYPENAME "' into an index-sized integer", index_type_name); + __Pyx_DECREF_TypeName(index_type_name); } return NULL; } -static PyObject *__Pyx_PyObject_GetItem(PyObject *obj, PyObject* key) { - PyMappingMethods *m = Py_TYPE(obj)->tp_as_mapping; - if (likely(m && m->mp_subscript)) { - return m->mp_subscript(obj, key); +static PyObject *__Pyx_PyObject_GetItem_Slow(PyObject *obj, PyObject *key) { + __Pyx_TypeName obj_type_name; + // Handles less common slow-path checks for GetItem + if (likely(PyType_Check(obj))) { + PyObject *meth = __Pyx_PyObject_GetAttrStrNoError(obj, PYIDENT("__class_getitem__")); + if (meth) { + PyObject *result = __Pyx_PyObject_CallOneArg(meth, key); + Py_DECREF(meth); + return result; + } } - return __Pyx_PyObject_GetIndex(obj, key); + + obj_type_name = __Pyx_PyType_GetName(Py_TYPE(obj)); + PyErr_Format(PyExc_TypeError, + "'" __Pyx_FMT_TYPENAME "' object is not subscriptable", obj_type_name); + __Pyx_DECREF_TypeName(obj_type_name); + return NULL; +} + +static PyObject *__Pyx_PyObject_GetItem(PyObject *obj, PyObject *key) { + PyTypeObject *tp = Py_TYPE(obj); + PyMappingMethods *mm = tp->tp_as_mapping; + PySequenceMethods *sm = tp->tp_as_sequence; + + if (likely(mm && mm->mp_subscript)) { + return mm->mp_subscript(obj, key); + } + if (likely(sm && sm->sq_item)) { + return __Pyx_PyObject_GetIndex(obj, key); + } + return __Pyx_PyObject_GetItem_Slow(obj, key); } #endif @@ -358,6 +383,7 @@ static PyObject *__Pyx_PyDict_GetItem(PyObject *d, PyObject* key) { #endif /////////////// GetItemInt.proto /////////////// +//@substitute: tempita #define __Pyx_GetItemInt(o, i, type, is_signed, to_py_func, is_list, wraparound, boundscheck) \ (__Pyx_fits_Py_ssize_t(i, type, is_signed) ? \ @@ -380,10 +406,11 @@ static CYTHON_INLINE PyObject *__Pyx_GetItemInt_Fast(PyObject *o, Py_ssize_t i, int is_list, int wraparound, int boundscheck); /////////////// GetItemInt /////////////// +//@substitute: tempita static PyObject *__Pyx_GetItemInt_Generic(PyObject *o, PyObject* j) { PyObject *r; - if (!j) return NULL; + if (unlikely(!j)) return NULL; r = PyObject_GetItem(o, j); Py_DECREF(j); return r; @@ -431,10 +458,18 @@ static CYTHON_INLINE PyObject *__Pyx_GetItemInt_Fast(PyObject *o, Py_ssize_t i, } } else { // inlined PySequence_GetItem() + special cased length overflow - PySequenceMethods *m = Py_TYPE(o)->tp_as_sequence; - if (likely(m && m->sq_item)) { - if (wraparound && unlikely(i < 0) && likely(m->sq_length)) { - Py_ssize_t l = m->sq_length(o); + PyMappingMethods *mm = Py_TYPE(o)->tp_as_mapping; + PySequenceMethods *sm = Py_TYPE(o)->tp_as_sequence; + if (mm && mm->mp_subscript) { + PyObject *r, *key = PyInt_FromSsize_t(i); + if (unlikely(!key)) return NULL; + r = mm->mp_subscript(o, key); + Py_DECREF(key); + return r; + } + if (likely(sm && sm->sq_item)) { + if (wraparound && unlikely(i < 0) && likely(sm->sq_length)) { + Py_ssize_t l = sm->sq_length(o); if (likely(l >= 0)) { i += l; } else { @@ -444,7 +479,7 @@ static CYTHON_INLINE PyObject *__Pyx_GetItemInt_Fast(PyObject *o, Py_ssize_t i, PyErr_Clear(); } } - return m->sq_item(o, i); + return sm->sq_item(o, i); } } #else @@ -471,7 +506,7 @@ static CYTHON_INLINE int __Pyx_SetItemInt_Fast(PyObject *o, Py_ssize_t i, PyObje static int __Pyx_SetItemInt_Generic(PyObject *o, PyObject *j, PyObject *v) { int r; - if (!j) return -1; + if (unlikely(!j)) return -1; r = PyObject_SetItem(o, j, v); Py_DECREF(j); return r; @@ -491,10 +526,19 @@ static CYTHON_INLINE int __Pyx_SetItemInt_Fast(PyObject *o, Py_ssize_t i, PyObje } } else { // inlined PySequence_SetItem() + special cased length overflow - PySequenceMethods *m = Py_TYPE(o)->tp_as_sequence; - if (likely(m && m->sq_ass_item)) { - if (wraparound && unlikely(i < 0) && likely(m->sq_length)) { - Py_ssize_t l = m->sq_length(o); + PyMappingMethods *mm = Py_TYPE(o)->tp_as_mapping; + PySequenceMethods *sm = Py_TYPE(o)->tp_as_sequence; + if (mm && mm->mp_ass_subscript) { + int r; + PyObject *key = PyInt_FromSsize_t(i); + if (unlikely(!key)) return -1; + r = mm->mp_ass_subscript(o, key, v); + Py_DECREF(key); + return r; + } + if (likely(sm && sm->sq_ass_item)) { + if (wraparound && unlikely(i < 0) && likely(sm->sq_length)) { + Py_ssize_t l = sm->sq_length(o); if (likely(l >= 0)) { i += l; } else { @@ -504,7 +548,7 @@ static CYTHON_INLINE int __Pyx_SetItemInt_Fast(PyObject *o, Py_ssize_t i, PyObje PyErr_Clear(); } } - return m->sq_ass_item(o, i, v); + return sm->sq_ass_item(o, i, v); } } #else @@ -537,24 +581,29 @@ static CYTHON_INLINE int __Pyx_DelItemInt_Fast(PyObject *o, Py_ssize_t i, static int __Pyx_DelItem_Generic(PyObject *o, PyObject *j) { int r; - if (!j) return -1; + if (unlikely(!j)) return -1; r = PyObject_DelItem(o, j); Py_DECREF(j); return r; } static CYTHON_INLINE int __Pyx_DelItemInt_Fast(PyObject *o, Py_ssize_t i, - CYTHON_UNUSED int is_list, CYTHON_NCP_UNUSED int wraparound) { + int is_list, CYTHON_NCP_UNUSED int wraparound) { #if !CYTHON_USE_TYPE_SLOTS if (is_list || PySequence_Check(o)) { return PySequence_DelItem(o, i); } #else // inlined PySequence_DelItem() + special cased length overflow - PySequenceMethods *m = Py_TYPE(o)->tp_as_sequence; - if (likely(m && m->sq_ass_item)) { - if (wraparound && unlikely(i < 0) && likely(m->sq_length)) { - Py_ssize_t l = m->sq_length(o); + PyMappingMethods *mm = Py_TYPE(o)->tp_as_mapping; + PySequenceMethods *sm = Py_TYPE(o)->tp_as_sequence; + if ((!is_list) && mm && mm->mp_ass_subscript) { + PyObject *key = PyInt_FromSsize_t(i); + return likely(key) ? mm->mp_ass_subscript(o, key, (PyObject *)NULL) : -1; + } + if (likely(sm && sm->sq_ass_item)) { + if (wraparound && unlikely(i < 0) && likely(sm->sq_length)) { + Py_ssize_t l = sm->sq_length(o); if (likely(l >= 0)) { i += l; } else { @@ -564,7 +613,7 @@ static CYTHON_INLINE int __Pyx_DelItemInt_Fast(PyObject *o, Py_ssize_t i, PyErr_Clear(); } } - return m->sq_ass_item(o, i, (PyObject *)NULL); + return sm->sq_ass_item(o, i, (PyObject *)NULL); } #endif return __Pyx_DelItem_Generic(o, PyInt_FromSsize_t(i)); @@ -599,7 +648,8 @@ static CYTHON_INLINE int __Pyx_PyObject_SetSlice(PyObject* obj, PyObject* value, {{endif}} Py_ssize_t cstart, Py_ssize_t cstop, PyObject** _py_start, PyObject** _py_stop, PyObject** _py_slice, - int has_cstart, int has_cstop, CYTHON_UNUSED int wraparound) { + int has_cstart, int has_cstop, int wraparound) { + __Pyx_TypeName obj_type_name; #if CYTHON_USE_TYPE_SLOTS PyMappingMethods* mp; #if PY_MAJOR_VERSION < 3 @@ -643,6 +693,8 @@ static CYTHON_INLINE int __Pyx_PyObject_SetSlice(PyObject* obj, PyObject* value, return ms->sq_ass_slice(obj, cstart, cstop, value); {{endif}} } +#else + CYTHON_UNUSED_VAR(wraparound); #endif mp = Py_TYPE(obj)->tp_as_mapping; @@ -651,6 +703,8 @@ static CYTHON_INLINE int __Pyx_PyObject_SetSlice(PyObject* obj, PyObject* value, {{else}} if (likely(mp && mp->mp_ass_subscript)) {{endif}} +#else + CYTHON_UNUSED_VAR(wraparound); #endif { {{if access == 'Get'}}PyObject*{{else}}int{{endif}} result; @@ -702,19 +756,70 @@ static CYTHON_INLINE int __Pyx_PyObject_SetSlice(PyObject* obj, PyObject* value, } return result; } + obj_type_name = __Pyx_PyType_GetName(Py_TYPE(obj)); PyErr_Format(PyExc_TypeError, {{if access == 'Get'}} - "'%.200s' object is unsliceable", Py_TYPE(obj)->tp_name); + "'" __Pyx_FMT_TYPENAME "' object is unsliceable", obj_type_name); {{else}} - "'%.200s' object does not support slice %.10s", - Py_TYPE(obj)->tp_name, value ? "assignment" : "deletion"); + "'" __Pyx_FMT_TYPENAME "' object does not support slice %.10s", + obj_type_name, value ? "assignment" : "deletion"); {{endif}} + __Pyx_DECREF_TypeName(obj_type_name); bad: return {{if access == 'Get'}}NULL{{else}}-1{{endif}}; } +/////////////// TupleAndListFromArray.proto /////////////// + +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyList_FromArray(PyObject *const *src, Py_ssize_t n); +static CYTHON_INLINE PyObject* __Pyx_PyTuple_FromArray(PyObject *const *src, Py_ssize_t n); +#endif + +/////////////// TupleAndListFromArray /////////////// +//@substitute: naming + +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE void __Pyx_copy_object_array(PyObject *const *CYTHON_RESTRICT src, PyObject** CYTHON_RESTRICT dest, Py_ssize_t length) { + PyObject *v; + Py_ssize_t i; + for (i = 0; i < length; i++) { + v = dest[i] = src[i]; + Py_INCREF(v); + } +} + +static CYTHON_INLINE PyObject * +__Pyx_PyTuple_FromArray(PyObject *const *src, Py_ssize_t n) +{ + PyObject *res; + if (n <= 0) { + Py_INCREF($empty_tuple); + return $empty_tuple; + } + res = PyTuple_New(n); + if (unlikely(res == NULL)) return NULL; + __Pyx_copy_object_array(src, ((PyTupleObject*)res)->ob_item, n); + return res; +} + +static CYTHON_INLINE PyObject * +__Pyx_PyList_FromArray(PyObject *const *src, Py_ssize_t n) +{ + PyObject *res; + if (n <= 0) { + return PyList_New(0); + } + res = PyList_New(n); + if (unlikely(res == NULL)) return NULL; + __Pyx_copy_object_array(src, ((PyListObject*)res)->ob_item, n); + return res; +} +#endif + + /////////////// SliceTupleAndList.proto /////////////// #if CYTHON_COMPILING_IN_CPYTHON @@ -726,6 +831,8 @@ static CYTHON_INLINE PyObject* __Pyx_PyTuple_GetSlice(PyObject* src, Py_ssize_t #endif /////////////// SliceTupleAndList /////////////// +//@requires: TupleAndListFromArray +//@substitute: tempita #if CYTHON_COMPILING_IN_CPYTHON static CYTHON_INLINE void __Pyx_crop_slice(Py_ssize_t* _start, Py_ssize_t* _stop, Py_ssize_t* _length) { @@ -746,32 +853,12 @@ static CYTHON_INLINE void __Pyx_crop_slice(Py_ssize_t* _start, Py_ssize_t* _stop *_stop = stop; } -static CYTHON_INLINE void __Pyx_copy_object_array(PyObject** CYTHON_RESTRICT src, PyObject** CYTHON_RESTRICT dest, Py_ssize_t length) { - PyObject *v; - Py_ssize_t i; - for (i = 0; i < length; i++) { - v = dest[i] = src[i]; - Py_INCREF(v); - } -} - {{for type in ['List', 'Tuple']}} static CYTHON_INLINE PyObject* __Pyx_Py{{type}}_GetSlice( PyObject* src, Py_ssize_t start, Py_ssize_t stop) { - PyObject* dest; Py_ssize_t length = Py{{type}}_GET_SIZE(src); __Pyx_crop_slice(&start, &stop, &length); - if (unlikely(length <= 0)) - return Py{{type}}_New(0); - - dest = Py{{type}}_New(length); - if (unlikely(!dest)) - return NULL; - __Pyx_copy_object_array( - ((Py{{type}}Object*)src)->ob_item + start, - ((Py{{type}}Object*)dest)->ob_item, - length); - return dest; + return __Pyx_Py{{type}}_FromArray(((Py{{type}}Object*)src)->ob_item + start, length); } {{endfor}} #endif @@ -913,10 +1000,14 @@ static PyObject *__Pyx_CreateClass(PyObject *bases, PyObject *dict, PyObject *na PyObject *result; PyObject *metaclass; - if (PyDict_SetItem(dict, PYIDENT("__module__"), modname) < 0) + if (unlikely(PyDict_SetItem(dict, PYIDENT("__module__"), modname) < 0)) return NULL; - if (PyDict_SetItem(dict, PYIDENT("__qualname__"), qualname) < 0) +#if PY_VERSION_HEX >= 0x03030000 + if (unlikely(PyDict_SetItem(dict, PYIDENT("__qualname__"), qualname) < 0)) return NULL; +#else + CYTHON_MAYBE_UNUSED_VAR(qualname); +#endif /* Python2 __metaclass__ */ metaclass = __Pyx_PyDict_GetItemStr(dict, PYIDENT("__metaclass__")); @@ -937,6 +1028,94 @@ static PyObject *__Pyx_CreateClass(PyObject *bases, PyObject *dict, PyObject *na return result; } +/////////////// Py3UpdateBases.proto /////////////// + +static PyObject* __Pyx_PEP560_update_bases(PyObject *bases); /* proto */ + +/////////////// Py3UpdateBases ///////////////////// +//@requires: PyObjectCallOneArg +//@requires: PyObjectGetAttrStrNoError + +/* Shamelessly adapted from cpython/bltinmodule.c update_bases */ +static PyObject* +__Pyx_PEP560_update_bases(PyObject *bases) +{ + Py_ssize_t i, j, size_bases; + PyObject *base, *meth, *new_base, *result, *new_bases = NULL; + /*assert(PyTuple_Check(bases));*/ + + size_bases = PyTuple_GET_SIZE(bases); + for (i = 0; i < size_bases; i++) { + // original code in CPython: base = args[i]; + base = PyTuple_GET_ITEM(bases, i); + if (PyType_Check(base)) { + if (new_bases) { + // If we already have made a replacement, then we append every normal base, + // otherwise just skip it. + if (PyList_Append(new_bases, base) < 0) { + goto error; + } + } + continue; + } + // original code in CPython: + // if (_PyObject_LookupAttrId(base, &PyId___mro_entries__, &meth) < 0) { + meth = __Pyx_PyObject_GetAttrStrNoError(base, PYIDENT("__mro_entries__")); + if (!meth && PyErr_Occurred()) { + goto error; + } + if (!meth) { + if (new_bases) { + if (PyList_Append(new_bases, base) < 0) { + goto error; + } + } + continue; + } + new_base = __Pyx_PyObject_CallOneArg(meth, bases); + Py_DECREF(meth); + if (!new_base) { + goto error; + } + if (!PyTuple_Check(new_base)) { + PyErr_SetString(PyExc_TypeError, + "__mro_entries__ must return a tuple"); + Py_DECREF(new_base); + goto error; + } + if (!new_bases) { + // If this is a first successful replacement, create new_bases list and + // copy previously encountered bases. + if (!(new_bases = PyList_New(i))) { + goto error; + } + for (j = 0; j < i; j++) { + // original code in CPython: base = args[j]; + base = PyTuple_GET_ITEM(bases, j); + PyList_SET_ITEM(new_bases, j, base); + Py_INCREF(base); + } + } + j = PyList_GET_SIZE(new_bases); + if (PyList_SetSlice(new_bases, j, j, new_base) < 0) { + goto error; + } + Py_DECREF(new_base); + } + if (!new_bases) { + // unlike the CPython implementation, always return a new reference + Py_INCREF(bases); + return bases; + } + result = PyList_AsTuple(new_bases); + Py_DECREF(new_bases); + return result; + +error: + Py_XDECREF(new_bases); + return NULL; +} + /////////////// Py3ClassCreate.proto /////////////// static PyObject *__Pyx_Py3MetaclassPrepare(PyObject *metaclass, PyObject *bases, PyObject *name, PyObject *qualname, @@ -945,27 +1124,27 @@ static PyObject *__Pyx_Py3ClassCreate(PyObject *metaclass, PyObject *name, PyObj PyObject *mkw, int calculate_metaclass, int allow_py2_metaclass); /*proto*/ /////////////// Py3ClassCreate /////////////// -//@requires: PyObjectGetAttrStr +//@substitute: naming +//@requires: PyObjectGetAttrStrNoError //@requires: CalculateMetaclass +//@requires: PyObjectFastCall +//@requires: PyObjectCall2Args +//@requires: PyObjectLookupSpecial +// only in fallback code: +//@requires: GetBuiltinName static PyObject *__Pyx_Py3MetaclassPrepare(PyObject *metaclass, PyObject *bases, PyObject *name, PyObject *qualname, PyObject *mkw, PyObject *modname, PyObject *doc) { PyObject *ns; if (metaclass) { - PyObject *prep = __Pyx_PyObject_GetAttrStr(metaclass, PYIDENT("__prepare__")); + PyObject *prep = __Pyx_PyObject_GetAttrStrNoError(metaclass, PYIDENT("__prepare__")); if (prep) { - PyObject *pargs = PyTuple_Pack(2, name, bases); - if (unlikely(!pargs)) { - Py_DECREF(prep); - return NULL; - } - ns = PyObject_Call(prep, pargs, mkw); + PyObject *pargs[3] = {NULL, name, bases}; + ns = __Pyx_PyObject_FastCallDict(prep, pargs+1, 2 | __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET, mkw); Py_DECREF(prep); - Py_DECREF(pargs); } else { - if (unlikely(!PyErr_ExceptionMatches(PyExc_AttributeError))) + if (unlikely(PyErr_Occurred())) return NULL; - PyErr_Clear(); ns = PyDict_New(); } } else { @@ -977,7 +1156,11 @@ static PyObject *__Pyx_Py3MetaclassPrepare(PyObject *metaclass, PyObject *bases, /* Required here to emulate assignment order */ if (unlikely(PyObject_SetItem(ns, PYIDENT("__module__"), modname) < 0)) goto bad; +#if PY_VERSION_HEX >= 0x03030000 if (unlikely(PyObject_SetItem(ns, PYIDENT("__qualname__"), qualname) < 0)) goto bad; +#else + CYTHON_MAYBE_UNUSED_VAR(qualname); +#endif if (unlikely(doc && PyObject_SetItem(ns, PYIDENT("__doc__"), doc) < 0)) goto bad; return ns; bad: @@ -985,11 +1168,164 @@ bad: return NULL; } +#if PY_VERSION_HEX < 0x030600A4 && CYTHON_PEP487_INIT_SUBCLASS +// https://www.python.org/dev/peps/pep-0487/ +static int __Pyx_SetNamesPEP487(PyObject *type_obj) { + PyTypeObject *type = (PyTypeObject*) type_obj; + PyObject *names_to_set, *key, *value, *set_name, *tmp; + Py_ssize_t i = 0; + +#if CYTHON_USE_TYPE_SLOTS + names_to_set = PyDict_Copy(type->tp_dict); +#else + { + PyObject *d = PyObject_GetAttr(type_obj, PYIDENT("__dict__")); + names_to_set = NULL; + if (likely(d)) { + // d may not be a dict, e.g. PyDictProxy in PyPy2. + PyObject *names_to_set = PyDict_New(); + int ret = likely(names_to_set) ? PyDict_Update(names_to_set, d) : -1; + Py_DECREF(d); + if (unlikely(ret < 0)) + Py_CLEAR(names_to_set); + } + } +#endif + if (unlikely(names_to_set == NULL)) + goto bad; + + while (PyDict_Next(names_to_set, &i, &key, &value)) { + set_name = __Pyx_PyObject_LookupSpecialNoError(value, PYIDENT("__set_name__")); + if (unlikely(set_name != NULL)) { + tmp = __Pyx_PyObject_Call2Args(set_name, type_obj, key); + Py_DECREF(set_name); + if (unlikely(tmp == NULL)) { + __Pyx_TypeName value_type_name = + __Pyx_PyType_GetName(Py_TYPE(value)); + __Pyx_TypeName type_name = __Pyx_PyType_GetName(type); + PyErr_Format(PyExc_RuntimeError, +#if PY_MAJOR_VERSION >= 3 + "Error calling __set_name__ on '" __Pyx_FMT_TYPENAME "' instance %R " "in '" __Pyx_FMT_TYPENAME "'", + value_type_name, key, type_name); +#else + "Error calling __set_name__ on '" __Pyx_FMT_TYPENAME "' instance %.100s in '" __Pyx_FMT_TYPENAME "'", + value_type_name, + PyString_Check(key) ? PyString_AS_STRING(key) : "?", + type_name); +#endif + goto bad; + } else { + Py_DECREF(tmp); + } + } + else if (unlikely(PyErr_Occurred())) { + goto bad; + } + } + + Py_DECREF(names_to_set); + return 0; +bad: + Py_XDECREF(names_to_set); + return -1; +} + +static PyObject *__Pyx_InitSubclassPEP487(PyObject *type_obj, PyObject *mkw) { +#if CYTHON_USE_TYPE_SLOTS && CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS +// Stripped-down version of "super(type_obj, type_obj).__init_subclass__(**mkw)" in CPython 3.8. + PyTypeObject *type = (PyTypeObject*) type_obj; + PyObject *mro = type->tp_mro; + Py_ssize_t i, nbases; + if (unlikely(!mro)) goto done; + + // avoid "unused" warning + (void) &__Pyx_GetBuiltinName; + + Py_INCREF(mro); + nbases = PyTuple_GET_SIZE(mro); + + // Skip over the type itself and 'object'. + assert(PyTuple_GET_ITEM(mro, 0) == type_obj); + for (i = 1; i < nbases-1; i++) { + PyObject *base, *dict, *meth; + base = PyTuple_GET_ITEM(mro, i); + dict = ((PyTypeObject *)base)->tp_dict; + meth = __Pyx_PyDict_GetItemStrWithError(dict, PYIDENT("__init_subclass__")); + if (unlikely(meth)) { + descrgetfunc f = Py_TYPE(meth)->tp_descr_get; + PyObject *res; + Py_INCREF(meth); + if (likely(f)) { + res = f(meth, NULL, type_obj); + Py_DECREF(meth); + if (unlikely(!res)) goto bad; + meth = res; + } + res = __Pyx_PyObject_FastCallDict(meth, NULL, 0, mkw); + Py_DECREF(meth); + if (unlikely(!res)) goto bad; + Py_DECREF(res); + goto done; + } else if (unlikely(PyErr_Occurred())) { + goto bad; + } + } + +done: + Py_XDECREF(mro); + return type_obj; + +bad: + Py_XDECREF(mro); + Py_DECREF(type_obj); + return NULL; + +// CYTHON_USE_TYPE_SLOTS && CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS +#else +// Generic fallback: "super(type_obj, type_obj).__init_subclass__(**mkw)", as used in CPython 3.8. + PyObject *super_type, *super, *func, *res; + +#if CYTHON_COMPILING_IN_PYPY && !defined(PySuper_Type) + super_type = __Pyx_GetBuiltinName(PYIDENT("super")); +#else + super_type = (PyObject*) &PySuper_Type; + // avoid "unused" warning + (void) &__Pyx_GetBuiltinName; +#endif + super = likely(super_type) ? __Pyx_PyObject_Call2Args(super_type, type_obj, type_obj) : NULL; +#if CYTHON_COMPILING_IN_PYPY && !defined(PySuper_Type) + Py_XDECREF(super_type); +#endif + if (unlikely(!super)) { + Py_CLEAR(type_obj); + goto done; + } + func = __Pyx_PyObject_GetAttrStrNoError(super, PYIDENT("__init_subclass__")); + Py_DECREF(super); + if (likely(!func)) { + if (unlikely(PyErr_Occurred())) + Py_CLEAR(type_obj); + goto done; + } + res = __Pyx_PyObject_FastCallDict(func, NULL, 0, mkw); + Py_DECREF(func); + if (unlikely(!res)) + Py_CLEAR(type_obj); + Py_XDECREF(res); +done: + return type_obj; +#endif +} + +// PY_VERSION_HEX < 0x030600A4 && CYTHON_PEP487_INIT_SUBCLASS +#endif + static PyObject *__Pyx_Py3ClassCreate(PyObject *metaclass, PyObject *name, PyObject *bases, PyObject *dict, PyObject *mkw, int calculate_metaclass, int allow_py2_metaclass) { - PyObject *result, *margs; + PyObject *result; PyObject *owned_metaclass = NULL; + PyObject *margs[4] = {NULL, name, bases, dict}; if (allow_py2_metaclass) { /* honour Python2 __metaclass__ for backward compatibility */ owned_metaclass = PyObject_GetItem(dict, PYIDENT("__metaclass__")); @@ -1008,14 +1344,28 @@ static PyObject *__Pyx_Py3ClassCreate(PyObject *metaclass, PyObject *name, PyObj return NULL; owned_metaclass = metaclass; } - margs = PyTuple_Pack(3, name, bases, dict); - if (unlikely(!margs)) { - result = NULL; - } else { - result = PyObject_Call(metaclass, margs, mkw); - Py_DECREF(margs); - } + result = __Pyx_PyObject_FastCallDict(metaclass, margs+1, 3 | __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET, +#if PY_VERSION_HEX < 0x030600A4 + // Before PEP-487, type(a,b,c) did not accept any keyword arguments, so guard at least against that case. + (metaclass == (PyObject*)&PyType_Type) ? NULL : mkw +#else + mkw +#endif + ); Py_XDECREF(owned_metaclass); + +#if PY_VERSION_HEX < 0x030600A4 && CYTHON_PEP487_INIT_SUBCLASS + if (likely(result) && likely(PyType_Check(result))) { + if (unlikely(__Pyx_SetNamesPEP487(result) < 0)) { + Py_CLEAR(result); + } else { + result = __Pyx_InitSubclassPEP487(result, mkw); + } + } +#else + // avoid "unused" warning + (void) &__Pyx_GetBuiltinName; +#endif return result; } @@ -1026,14 +1376,21 @@ static CYTHON_INLINE int __Pyx_TypeTest(PyObject *obj, PyTypeObject *type); /*pr /////////////// ExtTypeTest /////////////// static CYTHON_INLINE int __Pyx_TypeTest(PyObject *obj, PyTypeObject *type) { + __Pyx_TypeName obj_type_name; + __Pyx_TypeName type_name; if (unlikely(!type)) { PyErr_SetString(PyExc_SystemError, "Missing type object"); return 0; } if (likely(__Pyx_TypeCheck(obj, type))) return 1; - PyErr_Format(PyExc_TypeError, "Cannot convert %.200s to %.200s", - Py_TYPE(obj)->tp_name, type->tp_name); + obj_type_name = __Pyx_PyType_GetName(Py_TYPE(obj)); + type_name = __Pyx_PyType_GetName(type); + PyErr_Format(PyExc_TypeError, + "Cannot convert " __Pyx_FMT_TYPENAME " to " __Pyx_FMT_TYPENAME, + obj_type_name, type_name); + __Pyx_DECREF_TypeName(obj_type_name); + __Pyx_DECREF_TypeName(type_name); return 0; } @@ -1101,12 +1458,12 @@ static CYTHON_INLINE PyObject* __Pyx_PyBoolOrNull_FromLong(long b) { static PyObject *__Pyx_GetBuiltinName(PyObject *name); /*proto*/ /////////////// GetBuiltinName /////////////// -//@requires: PyObjectGetAttrStr +//@requires: PyObjectGetAttrStrNoError //@substitute: naming static PyObject *__Pyx_GetBuiltinName(PyObject *name) { - PyObject* result = __Pyx_PyObject_GetAttrStr($builtins_cname, name); - if (unlikely(!result)) { + PyObject* result = __Pyx_PyObject_GetAttrStrNoError($builtins_cname, name); + if (unlikely(!result) && !PyErr_Occurred()) { PyErr_Format(PyExc_NameError, #if PY_MAJOR_VERSION >= 3 "name '%U' is not defined", name); @@ -1123,29 +1480,27 @@ static PyObject *__Pyx_GetBuiltinName(PyObject *name) { static PyObject *__Pyx__GetNameInClass(PyObject *nmspace, PyObject *name); /*proto*/ /////////////// GetNameInClass /////////////// -//@requires: PyObjectGetAttrStr //@requires: GetModuleGlobalName -//@requires: Exceptions.c::PyThreadStateGet -//@requires: Exceptions.c::PyErrFetchRestore -//@requires: Exceptions.c::PyErrExceptionMatches - -static PyObject *__Pyx_GetGlobalNameAfterAttributeLookup(PyObject *name) { - PyObject *result; - __Pyx_PyThreadState_declare - __Pyx_PyThreadState_assign - if (unlikely(!__Pyx_PyErr_ExceptionMatches(PyExc_AttributeError))) - return NULL; - __Pyx_PyErr_Clear(); - __Pyx_GetModuleGlobalNameUncached(result, name); - return result; -} static PyObject *__Pyx__GetNameInClass(PyObject *nmspace, PyObject *name) { PyObject *result; - result = __Pyx_PyObject_GetAttrStr(nmspace, name); - if (!result) { - result = __Pyx_GetGlobalNameAfterAttributeLookup(name); + PyObject *dict; + assert(PyType_Check(nmspace)); +#if CYTHON_USE_TYPE_SLOTS + dict = ((PyTypeObject*)nmspace)->tp_dict; + Py_XINCREF(dict); +#else + dict = PyObject_GetAttr(nmspace, PYIDENT("__dict__")); +#endif + if (likely(dict)) { + result = PyObject_GetItem(dict, name); + Py_DECREF(dict); + if (result) { + return result; + } } + PyErr_Clear(); + __Pyx_GetModuleGlobalNameUncached(result, name); return result; } @@ -1163,6 +1518,30 @@ static PyObject *__Pyx__GetNameInClass(PyObject *nmspace, PyObject *name) { #define __Pyx_SetNameInClass(ns, name, value) PyObject_SetItem(ns, name, value) #endif +/////////////// SetNewInClass.proto /////////////// + +static int __Pyx_SetNewInClass(PyObject *ns, PyObject *name, PyObject *value); + +/////////////// SetNewInClass /////////////// +//@requires: SetNameInClass + +// Special-case setting __new__: if it's a Cython function, wrap it in a +// staticmethod. This is similar to what Python does for a Python function +// called __new__. +static int __Pyx_SetNewInClass(PyObject *ns, PyObject *name, PyObject *value) { +#ifdef __Pyx_CyFunction_USED + int ret; + if (__Pyx_CyFunction_Check(value)) { + PyObject *staticnew = PyStaticMethod_New(value); + if (unlikely(!staticnew)) return -1; + ret = __Pyx_SetNameInClass(ns, name, staticnew); + Py_DECREF(staticnew); + return ret; + } +#endif + return __Pyx_SetNameInClass(ns, name, value); +} + /////////////// GetModuleGlobalName.proto /////////////// //@requires: PyDictVersioning @@ -1200,6 +1579,7 @@ static CYTHON_INLINE PyObject *__Pyx__GetModuleGlobalName(PyObject *name) #endif { PyObject *result; +// FIXME: clean up the macro guard order here: limited API first, then borrowed refs, then cpython #if !CYTHON_AVOID_BORROWED_REFS #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030500A1 // Identifier names are always interned and have a pre-calculated hash value. @@ -1210,6 +1590,14 @@ static CYTHON_INLINE PyObject *__Pyx__GetModuleGlobalName(PyObject *name) } else if (unlikely(PyErr_Occurred())) { return NULL; } +#elif CYTHON_COMPILING_IN_LIMITED_API + if (unlikely(!$module_cname)) { + return NULL; + } + result = PyObject_GetAttr($module_cname, name); + if (likely(result)) { + return result; + } #else result = PyDict_GetItem($moddict_cname, name); __PYX_UPDATE_DICT_CACHE($moddict_cname, result, *dict_cached_value, *dict_version) @@ -1247,16 +1635,31 @@ static CYTHON_INLINE PyObject *__Pyx_GetAttr(PyObject *o, PyObject *n) { return PyObject_GetAttr(o, n); } + /////////////// PyObjectLookupSpecial.proto /////////////// + +#if CYTHON_USE_PYTYPE_LOOKUP && CYTHON_USE_TYPE_SLOTS +#define __Pyx_PyObject_LookupSpecialNoError(obj, attr_name) __Pyx__PyObject_LookupSpecial(obj, attr_name, 0) +#define __Pyx_PyObject_LookupSpecial(obj, attr_name) __Pyx__PyObject_LookupSpecial(obj, attr_name, 1) + +static CYTHON_INLINE PyObject* __Pyx__PyObject_LookupSpecial(PyObject* obj, PyObject* attr_name, int with_error); /*proto*/ + +#else +#define __Pyx_PyObject_LookupSpecialNoError(o,n) __Pyx_PyObject_GetAttrStrNoError(o,n) +#define __Pyx_PyObject_LookupSpecial(o,n) __Pyx_PyObject_GetAttrStr(o,n) +#endif + +/////////////// PyObjectLookupSpecial /////////////// //@requires: PyObjectGetAttrStr +//@requires: PyObjectGetAttrStrNoError #if CYTHON_USE_PYTYPE_LOOKUP && CYTHON_USE_TYPE_SLOTS -static CYTHON_INLINE PyObject* __Pyx_PyObject_LookupSpecial(PyObject* obj, PyObject* attr_name) { +static CYTHON_INLINE PyObject* __Pyx__PyObject_LookupSpecial(PyObject* obj, PyObject* attr_name, int with_error) { PyObject *res; PyTypeObject *tp = Py_TYPE(obj); #if PY_MAJOR_VERSION < 3 if (unlikely(PyInstance_Check(obj))) - return __Pyx_PyObject_GetAttrStr(obj, attr_name); + return with_error ? __Pyx_PyObject_GetAttrStr(obj, attr_name) : __Pyx_PyObject_GetAttrStrNoError(obj, attr_name); #endif // adapted from CPython's special_lookup() in ceval.c res = _PyType_Lookup(tp, attr_name); @@ -1267,13 +1670,11 @@ static CYTHON_INLINE PyObject* __Pyx_PyObject_LookupSpecial(PyObject* obj, PyObj } else { res = f(res, obj, (PyObject *)tp); } - } else { + } else if (with_error) { PyErr_SetObject(PyExc_AttributeError, attr_name); } return res; } -#else -#define __Pyx_PyObject_LookupSpecial(o,n) __Pyx_PyObject_GetAttrStr(o,n) #endif @@ -1292,19 +1693,21 @@ static CYTHON_INLINE PyObject* __Pyx_PyObject_GenericGetAttrNoDict(PyObject* obj #if CYTHON_USE_TYPE_SLOTS && CYTHON_USE_PYTYPE_LOOKUP && PY_VERSION_HEX < 0x03070000 static PyObject *__Pyx_RaiseGenericGetAttributeError(PyTypeObject *tp, PyObject *attr_name) { + __Pyx_TypeName type_name = __Pyx_PyType_GetName(tp); PyErr_Format(PyExc_AttributeError, #if PY_MAJOR_VERSION >= 3 - "'%.50s' object has no attribute '%U'", - tp->tp_name, attr_name); + "'" __Pyx_FMT_TYPENAME "' object has no attribute '%U'", + type_name, attr_name); #else - "'%.50s' object has no attribute '%.400s'", - tp->tp_name, PyString_AS_STRING(attr_name)); + "'" __Pyx_FMT_TYPENAME "' object has no attribute '%.400s'", + type_name, PyString_AS_STRING(attr_name)); #endif + __Pyx_DECREF_TypeName(type_name); return NULL; } static CYTHON_INLINE PyObject* __Pyx_PyObject_GenericGetAttrNoDict(PyObject* obj, PyObject* attr_name) { - // Copied and adapted from _PyObject_GenericGetAttrWithDict() in CPython 2.6/3.7. + // Copied and adapted from _PyObject_GenericGetAttrWithDict() in CPython 3.6/3.7. // To be used in the "tp_getattro" slot of extension types that have no instance dict and cannot be subclassed. PyObject *descr; PyTypeObject *tp = Py_TYPE(obj); @@ -1456,6 +1859,7 @@ static int __Pyx_PyObject_GetMethod(PyObject *obj, PyObject *name, PyObject **me static int __Pyx_PyObject_GetMethod(PyObject *obj, PyObject *name, PyObject **method) { PyObject *attr; #if CYTHON_UNPACK_METHODS && CYTHON_COMPILING_IN_CPYTHON && CYTHON_USE_PYTYPE_LOOKUP + __Pyx_TypeName type_name; // Copied from _PyObject_GetMethod() in CPython 3.7 PyTypeObject *tp = Py_TYPE(obj); PyObject *descr; @@ -1476,12 +1880,14 @@ static int __Pyx_PyObject_GetMethod(PyObject *obj, PyObject *name, PyObject **me descr = _PyType_Lookup(tp, name); if (likely(descr != NULL)) { Py_INCREF(descr); +#if defined(Py_TPFLAGS_METHOD_DESCRIPTOR) && Py_TPFLAGS_METHOD_DESCRIPTOR + if (__Pyx_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)) +#elif PY_MAJOR_VERSION >= 3 // Repeating the condition below accommodates for MSVC's inability to test macros inside of macro expansions. -#if PY_MAJOR_VERSION >= 3 #ifdef __Pyx_CyFunction_USED - if (likely(PyFunction_Check(descr) || (Py_TYPE(descr) == &PyMethodDescr_Type) || __Pyx_CyFunction_Check(descr))) + if (likely(PyFunction_Check(descr) || __Pyx_IS_TYPE(descr, &PyMethodDescr_Type) || __Pyx_CyFunction_Check(descr))) #else - if (likely(PyFunction_Check(descr) || (Py_TYPE(descr) == &PyMethodDescr_Type))) + if (likely(PyFunction_Check(descr) || __Pyx_IS_TYPE(descr, &PyMethodDescr_Type))) #endif #else // "PyMethodDescr_Type" is not part of the C-API in Py2. @@ -1527,19 +1933,21 @@ static int __Pyx_PyObject_GetMethod(PyObject *obj, PyObject *name, PyObject **me goto try_unpack; } - if (descr != NULL) { + if (likely(descr != NULL)) { *method = descr; return 0; } + type_name = __Pyx_PyType_GetName(tp); PyErr_Format(PyExc_AttributeError, #if PY_MAJOR_VERSION >= 3 - "'%.50s' object has no attribute '%U'", - tp->tp_name, name); + "'" __Pyx_FMT_TYPENAME "' object has no attribute '%U'", + type_name, name); #else - "'%.50s' object has no attribute '%.400s'", - tp->tp_name, PyString_AS_STRING(name)); + "'" __Pyx_FMT_TYPENAME "' object has no attribute '%.400s'", + type_name, PyString_AS_STRING(name)); #endif + __Pyx_DECREF_TypeName(type_name); return 0; // Generic fallback implementation using normal attribute lookup. @@ -1587,7 +1995,7 @@ static int __Pyx_TryUnpackUnboundCMethod(__Pyx_CachedCFunction* target) { target->method = method; #if CYTHON_COMPILING_IN_CPYTHON #if PY_MAJOR_VERSION >= 3 - // method dscriptor type isn't exported in Py2.x, cannot easily check the type there + // method descriptor type isn't exported in Py2.x, cannot easily check the type there if (likely(__Pyx_TypeCheck(method, &PyMethodDescr_Type))) #endif { @@ -1667,13 +2075,13 @@ static CYTHON_INLINE PyObject* __Pyx_CallUnboundCMethod1(__Pyx_CachedCFunction* // Not using #ifdefs for PY_VERSION_HEX to avoid C compiler warnings about unused functions. if (flag == METH_O) { return (*(cfunc->func))(self, arg); - } else if (PY_VERSION_HEX >= 0x030600B1 && flag == METH_FASTCALL) { - if (PY_VERSION_HEX >= 0x030700A0) { + } else if ((PY_VERSION_HEX >= 0x030600B1) && flag == METH_FASTCALL) { + if ((PY_VERSION_HEX >= 0x030700A0)) { return (*(__Pyx_PyCFunctionFast)(void*)(PyCFunction)cfunc->func)(self, &arg, 1); } else { return (*(__Pyx_PyCFunctionFastWithKeywords)(void*)(PyCFunction)cfunc->func)(self, &arg, 1, NULL); } - } else if (PY_VERSION_HEX >= 0x030700A0 && flag == (METH_FASTCALL | METH_KEYWORDS)) { + } else if ((PY_VERSION_HEX >= 0x030700A0) && flag == (METH_FASTCALL | METH_KEYWORDS)) { return (*(__Pyx_PyCFunctionFastWithKeywords)(void*)(PyCFunction)cfunc->func)(self, &arg, 1, NULL); } } @@ -1785,6 +2193,104 @@ bad: } +/////////////// PyObjectFastCall.proto /////////////// + +#define __Pyx_PyObject_FastCall(func, args, nargs) __Pyx_PyObject_FastCallDict(func, args, (size_t)(nargs), NULL) +static CYTHON_INLINE PyObject* __Pyx_PyObject_FastCallDict(PyObject *func, PyObject **args, size_t nargs, PyObject *kwargs); /*proto*/ + +/////////////// PyObjectFastCall /////////////// +//@requires: PyObjectCall +//@requires: PyFunctionFastCall +//@requires: PyObjectCallMethO +//@substitute: naming + +static PyObject* __Pyx_PyObject_FastCall_fallback(PyObject *func, PyObject **args, size_t nargs, PyObject *kwargs) { + PyObject *argstuple; + PyObject *result; + size_t i; + + argstuple = PyTuple_New((Py_ssize_t)nargs); + if (unlikely(!argstuple)) return NULL; + for (i = 0; i < nargs; i++) { + Py_INCREF(args[i]); + PyTuple_SET_ITEM(argstuple, (Py_ssize_t)i, args[i]); + } + result = __Pyx_PyObject_Call(func, argstuple, kwargs); + Py_DECREF(argstuple); + return result; +} + +static CYTHON_INLINE PyObject* __Pyx_PyObject_FastCallDict(PyObject *func, PyObject **args, size_t _nargs, PyObject *kwargs) { + // Special fast paths for 0 and 1 arguments + // NOTE: in many cases, this is called with a constant value for nargs + // which is known at compile-time. So the branches below will typically + // be optimized away. + Py_ssize_t nargs = __Pyx_PyVectorcall_NARGS(_nargs); +#if CYTHON_COMPILING_IN_CPYTHON + if (nargs == 0 && kwargs == NULL) { +#ifdef __Pyx_CyFunction_USED + if (__Pyx_IsCyOrPyCFunction(func)) +#else + if (PyCFunction_Check(func)) +#endif + { + if (likely(PyCFunction_GET_FLAGS(func) & METH_NOARGS)) { + return __Pyx_PyObject_CallMethO(func, NULL); + } + } + } + else if (nargs == 1 && kwargs == NULL) { + if (PyCFunction_Check(func)) + { + if (likely(PyCFunction_GET_FLAGS(func) & METH_O)) { + return __Pyx_PyObject_CallMethO(func, args[0]); + } + } + } +#endif + + #if PY_VERSION_HEX < 0x030800B1 + #if CYTHON_FAST_PYCCALL + if (PyCFunction_Check(func)) { + if (kwargs) { + return _PyCFunction_FastCallDict(func, args, nargs, kwargs); + } else { + return _PyCFunction_FastCallKeywords(func, args, nargs, NULL); + } + } + #if PY_VERSION_HEX >= 0x030700A1 + if (!kwargs && __Pyx_IS_TYPE(func, &PyMethodDescr_Type)) { + return _PyMethodDescr_FastCallKeywords(func, args, nargs, NULL); + } + #endif + #endif + #if CYTHON_FAST_PYCALL + if (PyFunction_Check(func)) { + return __Pyx_PyFunction_FastCallDict(func, args, nargs, kwargs); + } + #endif + #endif + + #if CYTHON_VECTORCALL + vectorcallfunc f = _PyVectorcall_Function(func); + if (f) { + return f(func, args, (size_t)nargs, kwargs); + } + #elif defined(__Pyx_CyFunction_USED) && CYTHON_BACKPORT_VECTORCALL + // exclude fused functions for now + if (__Pyx_CyFunction_CheckExact(func)) { + __pyx_vectorcallfunc f = __Pyx_CyFunction_func_vectorcall(func); + if (f) return f(func, args, (size_t)nargs, kwargs); + } + #endif + + if (nargs == 0) { + return __Pyx_PyObject_Call(func, $empty_tuple, kwargs); + } + return __Pyx_PyObject_FastCall_fallback(func, args, (size_t)nargs, kwargs); +} + + /////////////// PyObjectCallMethod0.proto /////////////// static PyObject* __Pyx_PyObject_CallMethod0(PyObject* obj, PyObject* method_name); /*proto*/ @@ -1839,59 +2345,6 @@ static PyObject* __Pyx_PyObject_CallMethod1(PyObject* obj, PyObject* method_name } -/////////////// PyObjectCallMethod2.proto /////////////// - -static PyObject* __Pyx_PyObject_CallMethod2(PyObject* obj, PyObject* method_name, PyObject* arg1, PyObject* arg2); /*proto*/ - -/////////////// PyObjectCallMethod2 /////////////// -//@requires: PyObjectCall -//@requires: PyFunctionFastCall -//@requires: PyCFunctionFastCall -//@requires: PyObjectCall2Args - -static PyObject* __Pyx_PyObject_Call3Args(PyObject* function, PyObject* arg1, PyObject* arg2, PyObject* arg3) { - #if CYTHON_FAST_PYCALL - if (PyFunction_Check(function)) { - PyObject *args[3] = {arg1, arg2, arg3}; - return __Pyx_PyFunction_FastCall(function, args, 3); - } - #endif - #if CYTHON_FAST_PYCCALL - if (__Pyx_PyFastCFunction_Check(function)) { - PyObject *args[3] = {arg1, arg2, arg3}; - return __Pyx_PyFunction_FastCall(function, args, 3); - } - #endif - - args = PyTuple_New(3); - if (unlikely(!args)) goto done; - Py_INCREF(arg1); - PyTuple_SET_ITEM(args, 0, arg1); - Py_INCREF(arg2); - PyTuple_SET_ITEM(args, 1, arg2); - Py_INCREF(arg3); - PyTuple_SET_ITEM(args, 2, arg3); - - result = __Pyx_PyObject_Call(function, args, NULL); - Py_DECREF(args); - return result; -} - -static PyObject* __Pyx_PyObject_CallMethod2(PyObject* obj, PyObject* method_name, PyObject* arg1, PyObject* arg2) { - PyObject *args, *method = NULL, *result = NULL; - int is_method = __Pyx_PyObject_GetMethod(obj, method_name, &method); - if (likely(is_method)) { - result = __Pyx_PyObject_Call3Args(method, obj, arg1, arg2); - Py_DECREF(method); - return result; - } - if (unlikely(!method)) return NULL; - result = __Pyx_PyObject_Call2Args(method, arg1, arg2); - Py_DECREF(method); - return result; -} - - /////////////// tp_new.proto /////////////// #define __Pyx_tp_new(type_obj, args) __Pyx_tp_new_kwargs(type_obj, args, NULL) @@ -1963,14 +2416,12 @@ static CYTHON_INLINE PyObject* __Pyx_PyObject_CallMethO(PyObject *func, PyObject /////////////// PyFunctionFastCall.proto /////////////// #if CYTHON_FAST_PYCALL + +#if !CYTHON_VECTORCALL #define __Pyx_PyFunction_FastCall(func, args, nargs) \ __Pyx_PyFunction_FastCallDict((func), (args), (nargs), NULL) -// let's assume that the non-public C-API function might still change during the 3.6 beta phase -#if 1 || PY_VERSION_HEX < 0x030600B1 static PyObject *__Pyx_PyFunction_FastCallDict(PyObject *func, PyObject **args, Py_ssize_t nargs, PyObject *kwargs); -#else -#define __Pyx_PyFunction_FastCallDict(func, args, nargs, kwargs) _PyFunction_FastCallDict(func, args, nargs, kwargs) #endif // Backport from Python 3 @@ -1982,7 +2433,7 @@ static PyObject *__Pyx_PyFunction_FastCallDict(PyObject *func, PyObject **args, // ((char *)(foo) \ // + Py_BUILD_ASSERT_EXPR(offsetof(struct foo, string) == 0)) // -// Written by Rusty Russell, public domain, http://ccodearchive.net/ +// Written by Rusty Russell, public domain, https://ccodearchive.net/ #define __Pyx_BUILD_ASSERT_EXPR(cond) \ (sizeof(char [1 - 2*!(cond)]) - 1) @@ -2011,8 +2462,7 @@ static PyObject *__Pyx_PyFunction_FastCallDict(PyObject *func, PyObject **args, /////////////// PyFunctionFastCall /////////////// // copied from CPython 3.6 ceval.c -#if CYTHON_FAST_PYCALL - +#if CYTHON_FAST_PYCALL && !CYTHON_VECTORCALL static PyObject* __Pyx_PyFunction_FastCallNoKw(PyCodeObject *co, PyObject **args, Py_ssize_t na, PyObject *globals) { PyFrameObject *f; @@ -2048,7 +2498,6 @@ static PyObject* __Pyx_PyFunction_FastCallNoKw(PyCodeObject *co, PyObject **args } -#if 1 || PY_VERSION_HEX < 0x030600B1 static PyObject *__Pyx_PyFunction_FastCallDict(PyObject *func, PyObject **args, Py_ssize_t nargs, PyObject *kwargs) { PyCodeObject *co = (PyCodeObject *)PyFunction_GET_CODE(func); PyObject *globals = PyFunction_GET_GLOBALS(func); @@ -2069,7 +2518,7 @@ static PyObject *__Pyx_PyFunction_FastCallDict(PyObject *func, PyObject **args, assert(kwargs == NULL || PyDict_Check(kwargs)); nk = kwargs ? PyDict_Size(kwargs) : 0; - if (Py_EnterRecursiveCall((char*)" while calling a Python object")) { + if (unlikely(Py_EnterRecursiveCall((char*)" while calling a Python object"))) { return NULL; } @@ -2158,83 +2607,19 @@ done: Py_LeaveRecursiveCall(); return result; } -#endif /* CPython < 3.6 */ -#endif /* CYTHON_FAST_PYCALL */ - - -/////////////// PyCFunctionFastCall.proto /////////////// - -#if CYTHON_FAST_PYCCALL -static CYTHON_INLINE PyObject *__Pyx_PyCFunction_FastCall(PyObject *func, PyObject **args, Py_ssize_t nargs); -#else -#define __Pyx_PyCFunction_FastCall(func, args, nargs) (assert(0), NULL) -#endif - -/////////////// PyCFunctionFastCall /////////////// - -#if CYTHON_FAST_PYCCALL -static CYTHON_INLINE PyObject * __Pyx_PyCFunction_FastCall(PyObject *func_obj, PyObject **args, Py_ssize_t nargs) { - PyCFunctionObject *func = (PyCFunctionObject*)func_obj; - PyCFunction meth = PyCFunction_GET_FUNCTION(func); - PyObject *self = PyCFunction_GET_SELF(func); - int flags = PyCFunction_GET_FLAGS(func); - - assert(PyCFunction_Check(func)); - assert(METH_FASTCALL == (flags & ~(METH_CLASS | METH_STATIC | METH_COEXIST | METH_KEYWORDS | METH_STACKLESS))); - assert(nargs >= 0); - assert(nargs == 0 || args != NULL); - - /* _PyCFunction_FastCallDict() must not be called with an exception set, - because it may clear it (directly or indirectly) and so the - caller loses its exception */ - assert(!PyErr_Occurred()); - - if ((PY_VERSION_HEX < 0x030700A0) || unlikely(flags & METH_KEYWORDS)) { - return (*((__Pyx_PyCFunctionFastWithKeywords)(void*)meth)) (self, args, nargs, NULL); - } else { - return (*((__Pyx_PyCFunctionFast)(void*)meth)) (self, args, nargs); - } -} -#endif /* CYTHON_FAST_PYCCALL */ +#endif /* CYTHON_FAST_PYCALL && !CYTHON_VECTORCALL */ /////////////// PyObjectCall2Args.proto /////////////// -static CYTHON_UNUSED PyObject* __Pyx_PyObject_Call2Args(PyObject* function, PyObject* arg1, PyObject* arg2); /*proto*/ +static CYTHON_INLINE PyObject* __Pyx_PyObject_Call2Args(PyObject* function, PyObject* arg1, PyObject* arg2); /*proto*/ /////////////// PyObjectCall2Args /////////////// -//@requires: PyObjectCall -//@requires: PyFunctionFastCall -//@requires: PyCFunctionFastCall +//@requires: PyObjectFastCall -static CYTHON_UNUSED PyObject* __Pyx_PyObject_Call2Args(PyObject* function, PyObject* arg1, PyObject* arg2) { - PyObject *args, *result = NULL; - #if CYTHON_FAST_PYCALL - if (PyFunction_Check(function)) { - PyObject *args[2] = {arg1, arg2}; - return __Pyx_PyFunction_FastCall(function, args, 2); - } - #endif - #if CYTHON_FAST_PYCCALL - if (__Pyx_PyFastCFunction_Check(function)) { - PyObject *args[2] = {arg1, arg2}; - return __Pyx_PyCFunction_FastCall(function, args, 2); - } - #endif - - args = PyTuple_New(2); - if (unlikely(!args)) goto done; - Py_INCREF(arg1); - PyTuple_SET_ITEM(args, 0, arg1); - Py_INCREF(arg2); - PyTuple_SET_ITEM(args, 1, arg2); - - Py_INCREF(function); - result = __Pyx_PyObject_Call(function, args, NULL); - Py_DECREF(args); - Py_DECREF(function); -done: - return result; +static CYTHON_INLINE PyObject* __Pyx_PyObject_Call2Args(PyObject* function, PyObject* arg1, PyObject* arg2) { + PyObject *args[3] = {NULL, arg1, arg2}; + return __Pyx_PyObject_FastCall(function, args+1, 2 | __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET); } @@ -2243,88 +2628,97 @@ done: static CYTHON_INLINE PyObject* __Pyx_PyObject_CallOneArg(PyObject *func, PyObject *arg); /*proto*/ /////////////// PyObjectCallOneArg /////////////// -//@requires: PyObjectCallMethO -//@requires: PyObjectCall -//@requires: PyFunctionFastCall -//@requires: PyCFunctionFastCall +//@requires: PyObjectFastCall -#if CYTHON_COMPILING_IN_CPYTHON -static PyObject* __Pyx__PyObject_CallOneArg(PyObject *func, PyObject *arg) { - PyObject *result; - PyObject *args = PyTuple_New(1); - if (unlikely(!args)) return NULL; - Py_INCREF(arg); - PyTuple_SET_ITEM(args, 0, arg); - result = __Pyx_PyObject_Call(func, args, NULL); - Py_DECREF(args); - return result; -} - -static CYTHON_INLINE PyObject* __Pyx_PyObject_CallOneArg(PyObject *func, PyObject *arg) { -#if CYTHON_FAST_PYCALL - if (PyFunction_Check(func)) { - return __Pyx_PyFunction_FastCall(func, &arg, 1); - } -#endif - if (likely(PyCFunction_Check(func))) { - if (likely(PyCFunction_GET_FLAGS(func) & METH_O)) { - // fast and simple case that we are optimising for - return __Pyx_PyObject_CallMethO(func, arg); -#if CYTHON_FAST_PYCCALL - } else if (__Pyx_PyFastCFunction_Check(func)) { - return __Pyx_PyCFunction_FastCall(func, &arg, 1); -#endif - } - } - return __Pyx__PyObject_CallOneArg(func, arg); -} -#else static CYTHON_INLINE PyObject* __Pyx_PyObject_CallOneArg(PyObject *func, PyObject *arg) { - PyObject *result; - PyObject *args = PyTuple_Pack(1, arg); - if (unlikely(!args)) return NULL; - result = __Pyx_PyObject_Call(func, args, NULL); - Py_DECREF(args); - return result; + PyObject *args[2] = {NULL, arg}; + return __Pyx_PyObject_FastCall(func, args+1, 1 | __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET); } -#endif /////////////// PyObjectCallNoArg.proto /////////////// -//@requires: PyObjectCall -//@substitute: naming -#if CYTHON_COMPILING_IN_CPYTHON static CYTHON_INLINE PyObject* __Pyx_PyObject_CallNoArg(PyObject *func); /*proto*/ -#else -#define __Pyx_PyObject_CallNoArg(func) __Pyx_PyObject_Call(func, $empty_tuple, NULL) -#endif /////////////// PyObjectCallNoArg /////////////// -//@requires: PyObjectCallMethO -//@requires: PyObjectCall -//@requires: PyFunctionFastCall -//@substitute: naming +//@requires: PyObjectFastCall -#if CYTHON_COMPILING_IN_CPYTHON static CYTHON_INLINE PyObject* __Pyx_PyObject_CallNoArg(PyObject *func) { -#if CYTHON_FAST_PYCALL - if (PyFunction_Check(func)) { - return __Pyx_PyFunction_FastCall(func, NULL, 0); - } -#endif -#ifdef __Pyx_CyFunction_USED - if (likely(PyCFunction_Check(func) || __Pyx_CyFunction_Check(func))) -#else - if (likely(PyCFunction_Check(func))) + PyObject *arg = NULL; + return __Pyx_PyObject_FastCall(func, (&arg)+1, 0 | __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET); +} + + +/////////////// PyVectorcallFastCallDict.proto /////////////// + +#if CYTHON_METH_FASTCALL +static CYTHON_INLINE PyObject *__Pyx_PyVectorcall_FastCallDict(PyObject *func, __pyx_vectorcallfunc vc, PyObject *const *args, size_t nargs, PyObject *kw); #endif - { - if (likely(PyCFunction_GET_FLAGS(func) & METH_NOARGS)) { - // fast and simple case that we are optimising for - return __Pyx_PyObject_CallMethO(func, NULL); - } + +/////////////// PyVectorcallFastCallDict /////////////// + +#if CYTHON_METH_FASTCALL +// Slow path when kw is non-empty +static PyObject *__Pyx_PyVectorcall_FastCallDict_kw(PyObject *func, __pyx_vectorcallfunc vc, PyObject *const *args, size_t nargs, PyObject *kw) +{ + // Code based on _PyObject_FastCallDict() and _PyStack_UnpackDict() from CPython + PyObject *res = NULL; + PyObject *kwnames; + PyObject **newargs; + PyObject **kwvalues; + Py_ssize_t i, pos; + size_t j; + PyObject *key, *value; + unsigned long keys_are_strings; + Py_ssize_t nkw = PyDict_GET_SIZE(kw); + + // Copy positional arguments + newargs = (PyObject **)PyMem_Malloc((nargs + (size_t)nkw) * sizeof(args[0])); + if (unlikely(newargs == NULL)) { + PyErr_NoMemory(); + return NULL; } - return __Pyx_PyObject_Call(func, $empty_tuple, NULL); + for (j = 0; j < nargs; j++) newargs[j] = args[j]; + + // Copy keyword arguments + kwnames = PyTuple_New(nkw); + if (unlikely(kwnames == NULL)) { + PyMem_Free(newargs); + return NULL; + } + kwvalues = newargs + nargs; + pos = i = 0; + keys_are_strings = Py_TPFLAGS_UNICODE_SUBCLASS; + while (PyDict_Next(kw, &pos, &key, &value)) { + keys_are_strings &= Py_TYPE(key)->tp_flags; + Py_INCREF(key); + Py_INCREF(value); + PyTuple_SET_ITEM(kwnames, i, key); + kwvalues[i] = value; + i++; + } + if (unlikely(!keys_are_strings)) { + PyErr_SetString(PyExc_TypeError, "keywords must be strings"); + goto cleanup; + } + + // The actual call + res = vc(func, newargs, nargs, kwnames); + +cleanup: + Py_DECREF(kwnames); + for (i = 0; i < nkw; i++) + Py_DECREF(kwvalues[i]); + PyMem_Free(newargs); + return res; +} + +static CYTHON_INLINE PyObject *__Pyx_PyVectorcall_FastCallDict(PyObject *func, __pyx_vectorcallfunc vc, PyObject *const *args, size_t nargs, PyObject *kw) +{ + if (likely(kw == NULL) || PyDict_GET_SIZE(kw) == 0) { + return vc(func, args, nargs, NULL); + } + return __Pyx_PyVectorcall_FastCallDict_kw(func, vc, args, nargs, kw); } #endif @@ -2341,10 +2735,9 @@ static PyObject* __Pyx_PyNumber_InPlaceMatrixMultiply(PyObject* x, PyObject* y); #endif /////////////// MatrixMultiply /////////////// -//@requires: PyObjectGetAttrStr +//@requires: PyObjectGetAttrStrNoError //@requires: PyObjectCallOneArg -//@requires: PyFunctionFastCall -//@requires: PyCFunctionFastCall +//@requires: PyObjectCall2Args #if PY_VERSION_HEX < 0x03050000 static PyObject* __Pyx_PyObject_CallMatrixMethod(PyObject* method, PyObject* arg) { @@ -2354,34 +2747,9 @@ static PyObject* __Pyx_PyObject_CallMatrixMethod(PyObject* method, PyObject* arg if (likely(PyMethod_Check(method))) { PyObject *self = PyMethod_GET_SELF(method); if (likely(self)) { - PyObject *args; PyObject *function = PyMethod_GET_FUNCTION(method); - #if CYTHON_FAST_PYCALL - if (PyFunction_Check(function)) { - PyObject *args[2] = {self, arg}; - result = __Pyx_PyFunction_FastCall(function, args, 2); - goto done; - } - #endif - #if CYTHON_FAST_PYCCALL - if (__Pyx_PyFastCFunction_Check(function)) { - PyObject *args[2] = {self, arg}; - result = __Pyx_PyCFunction_FastCall(function, args, 2); - goto done; - } - #endif - args = PyTuple_New(2); - if (unlikely(!args)) goto done; - Py_INCREF(self); - PyTuple_SET_ITEM(args, 0, self); - Py_INCREF(arg); - PyTuple_SET_ITEM(args, 1, arg); - Py_INCREF(function); - Py_DECREF(method); method = NULL; - result = __Pyx_PyObject_Call(function, args, NULL); - Py_DECREF(args); - Py_DECREF(function); - return result; + result = __Pyx_PyObject_Call2Args(function, self, arg); + goto done; } } #endif @@ -2391,21 +2759,21 @@ done: return result; } -#define __Pyx_TryMatrixMethod(x, y, py_method_name) { \ - PyObject *func = __Pyx_PyObject_GetAttrStr(x, py_method_name); \ +#define __Pyx_TryMatrixMethod(x, y, py_method_name) { \ + PyObject *func = __Pyx_PyObject_GetAttrStrNoError(x, py_method_name); \ if (func) { \ PyObject *result = __Pyx_PyObject_CallMatrixMethod(func, y); \ if (result != Py_NotImplemented) \ return result; \ Py_DECREF(result); \ - } else { \ - if (!PyErr_ExceptionMatches(PyExc_AttributeError)) \ - return NULL; \ - PyErr_Clear(); \ + } else if (unlikely(PyErr_Occurred())) { \ + return NULL; \ } \ } static PyObject* __Pyx__PyNumber_MatrixMultiply(PyObject* x, PyObject* y, const char* op_name) { + __Pyx_TypeName x_type_name; + __Pyx_TypeName y_type_name; int right_is_subtype = PyObject_IsSubclass((PyObject*)Py_TYPE(y), (PyObject*)Py_TYPE(x)); if (unlikely(right_is_subtype == -1)) return NULL; @@ -2418,11 +2786,13 @@ static PyObject* __Pyx__PyNumber_MatrixMultiply(PyObject* x, PyObject* y, const if (!right_is_subtype) { __Pyx_TryMatrixMethod(y, x, PYIDENT("__rmatmul__")) } + x_type_name = __Pyx_PyType_GetName(Py_TYPE(x)); + y_type_name = __Pyx_PyType_GetName(Py_TYPE(y)); PyErr_Format(PyExc_TypeError, - "unsupported operand type(s) for %.2s: '%.100s' and '%.100s'", - op_name, - Py_TYPE(x)->tp_name, - Py_TYPE(y)->tp_name); + "unsupported operand type(s) for %.2s: '" __Pyx_FMT_TYPENAME "' and '" + __Pyx_FMT_TYPENAME "'", op_name, x_type_name, y_type_name); + __Pyx_DECREF_TypeName(x_type_name); + __Pyx_DECREF_TypeName(y_type_name); return NULL; } @@ -2493,3 +2863,240 @@ static CYTHON_INLINE int __Pyx_object_dict_version_matches(PyObject* obj, PY_UIN return obj_dict_version == __Pyx_get_object_dict_version(obj); } #endif + + +/////////////// PyMethodNew.proto /////////////// + +#if PY_MAJOR_VERSION >= 3 +// This should be an actual function (not a macro), such that we can put it +// directly in a tp_descr_get slot. +static PyObject *__Pyx_PyMethod_New(PyObject *func, PyObject *self, PyObject *typ) { + CYTHON_UNUSED_VAR(typ); + if (!self) + return __Pyx_NewRef(func); + return PyMethod_New(func, self); +} +#else + #define __Pyx_PyMethod_New PyMethod_New +#endif + +/////////////// UnicodeConcatInPlace.proto //////////////// + +# if CYTHON_COMPILING_IN_CPYTHON && PY_MAJOR_VERSION >= 3 +// __Pyx_PyUnicode_ConcatInPlace may modify the first argument 'left' +// However, unlike `PyUnicode_Append` it will never NULL it. +// It behaves like a regular function - returns a new reference and NULL on error + #if CYTHON_REFNANNY + #define __Pyx_PyUnicode_ConcatInPlace(left, right) __Pyx_PyUnicode_ConcatInPlaceImpl(&left, right, __pyx_refnanny) + #else + #define __Pyx_PyUnicode_ConcatInPlace(left, right) __Pyx_PyUnicode_ConcatInPlaceImpl(&left, right) + #endif + // __Pyx_PyUnicode_ConcatInPlace is slightly odd because it has the potential to modify the input + // argument (but only in cases where no user should notice). Therefore, it needs to keep Cython's + // refnanny informed. + static CYTHON_INLINE PyObject *__Pyx_PyUnicode_ConcatInPlaceImpl(PyObject **p_left, PyObject *right + #if CYTHON_REFNANNY + , void* __pyx_refnanny + #endif + ); /* proto */ +#else +#define __Pyx_PyUnicode_ConcatInPlace __Pyx_PyUnicode_Concat +#endif +#define __Pyx_PyUnicode_ConcatInPlaceSafe(left, right) ((unlikely((left) == Py_None) || unlikely((right) == Py_None)) ? \ + PyNumber_InPlaceAdd(left, right) : __Pyx_PyUnicode_ConcatInPlace(left, right)) + +/////////////// UnicodeConcatInPlace //////////////// +//@substitute: naming + +# if CYTHON_COMPILING_IN_CPYTHON && PY_MAJOR_VERSION >= 3 +// copied directly from unicode_object.c "unicode_modifiable +// removing _PyUnicode_HASH since it's a macro we don't have +// - this is OK because trying PyUnicode_Resize on a non-modifyable +// object will still work, it just won't happen in place +static int +__Pyx_unicode_modifiable(PyObject *unicode) +{ + if (Py_REFCNT(unicode) != 1) + return 0; + if (!PyUnicode_CheckExact(unicode)) + return 0; + if (PyUnicode_CHECK_INTERNED(unicode)) + return 0; + return 1; +} + +static CYTHON_INLINE PyObject *__Pyx_PyUnicode_ConcatInPlaceImpl(PyObject **p_left, PyObject *right + #if CYTHON_REFNANNY + , void* __pyx_refnanny + #endif + ) { + // heavily based on PyUnicode_Append + PyObject *left = *p_left; + Py_ssize_t left_len, right_len, new_len; + + if (unlikely(__Pyx_PyUnicode_READY(left) == -1)) + return NULL; + if (unlikely(__Pyx_PyUnicode_READY(right) == -1)) + return NULL; + + // Shortcuts + left_len = PyUnicode_GET_LENGTH(left); + if (left_len == 0) { + Py_INCREF(right); + return right; + } + right_len = PyUnicode_GET_LENGTH(right); + if (right_len == 0) { + Py_INCREF(left); + return left; + } + if (unlikely(left_len > PY_SSIZE_T_MAX - right_len)) { + PyErr_SetString(PyExc_OverflowError, + "strings are too large to concat"); + return NULL; + } + new_len = left_len + right_len; + + if (__Pyx_unicode_modifiable(left) + && PyUnicode_CheckExact(right) + && PyUnicode_KIND(right) <= PyUnicode_KIND(left) + // Don't resize for ascii += latin1. Convert ascii to latin1 requires + // to change the structure size, but characters are stored just after + // the structure, and so it requires to move all characters which is + // not so different than duplicating the string. + && !(PyUnicode_IS_ASCII(left) && !PyUnicode_IS_ASCII(right))) { + + __Pyx_GIVEREF(*p_left); + if (unlikely(PyUnicode_Resize(p_left, new_len) != 0)) { + // on failure PyUnicode_Resize does not deallocate the the input + // so left will remain unchanged - simply undo the giveref + __Pyx_GOTREF(*p_left); + return NULL; + } + __Pyx_INCREF(*p_left); + + // copy 'right' into the newly allocated area of 'left' + _PyUnicode_FastCopyCharacters(*p_left, left_len, right, 0, right_len); + return *p_left; + } else { + return __Pyx_PyUnicode_Concat(left, right); + } + } +#endif + +////////////// StrConcatInPlace.proto /////////////////////// +//@requires: UnicodeConcatInPlace + +#if PY_MAJOR_VERSION >= 3 + // allow access to the more efficient versions where we know str_type is unicode + #define __Pyx_PyStr_Concat __Pyx_PyUnicode_Concat + #define __Pyx_PyStr_ConcatInPlace __Pyx_PyUnicode_ConcatInPlace +#else + #define __Pyx_PyStr_Concat PyNumber_Add + #define __Pyx_PyStr_ConcatInPlace PyNumber_InPlaceAdd +#endif +#define __Pyx_PyStr_ConcatSafe(a, b) ((unlikely((a) == Py_None) || unlikely((b) == Py_None)) ? \ + PyNumber_Add(a, b) : __Pyx_PyStr_Concat(a, b)) +#define __Pyx_PyStr_ConcatInPlaceSafe(a, b) ((unlikely((a) == Py_None) || unlikely((b) == Py_None)) ? \ + PyNumber_InPlaceAdd(a, b) : __Pyx_PyStr_ConcatInPlace(a, b)) + +/////////////// FormatTypeName.proto /////////////// + +#if CYTHON_COMPILING_IN_LIMITED_API +typedef PyObject *__Pyx_TypeName; +#define __Pyx_FMT_TYPENAME "%U" +static __Pyx_TypeName __Pyx_PyType_GetName(PyTypeObject* tp); /*proto*/ +#define __Pyx_DECREF_TypeName(obj) Py_XDECREF(obj) +#else +typedef const char *__Pyx_TypeName; +#define __Pyx_FMT_TYPENAME "%.200s" +#define __Pyx_PyType_GetName(tp) ((tp)->tp_name) +#define __Pyx_DECREF_TypeName(obj) +#endif + +/////////////// FormatTypeName /////////////// + +#if CYTHON_COMPILING_IN_LIMITED_API +static __Pyx_TypeName +__Pyx_PyType_GetName(PyTypeObject* tp) +{ + PyObject *name = __Pyx_PyObject_GetAttrStr((PyObject *)tp, + PYIDENT("__name__")); + if (unlikely(name == NULL) || unlikely(!PyUnicode_Check(name))) { + PyErr_Clear(); + Py_XSETREF(name, __Pyx_NewRef(PYIDENT("?"))); + } + return name; +} +#endif + + +/////////////// RaiseUnexpectedTypeError.proto /////////////// + +static int __Pyx_RaiseUnexpectedTypeError(const char *expected, PyObject *obj); /*proto*/ + +/////////////// RaiseUnexpectedTypeError /////////////// + +static int +__Pyx_RaiseUnexpectedTypeError(const char *expected, PyObject *obj) +{ + __Pyx_TypeName obj_type_name = __Pyx_PyType_GetName(Py_TYPE(obj)); + PyErr_Format(PyExc_TypeError, "Expected %s, got " __Pyx_FMT_TYPENAME, + expected, obj_type_name); + __Pyx_DECREF_TypeName(obj_type_name); + return 0; +} + + +/////////////// RaiseUnboundLocalError.proto /////////////// +static CYTHON_INLINE void __Pyx_RaiseUnboundLocalError(const char *varname);/*proto*/ + +/////////////// RaiseUnboundLocalError /////////////// +static CYTHON_INLINE void __Pyx_RaiseUnboundLocalError(const char *varname) { + PyErr_Format(PyExc_UnboundLocalError, "local variable '%s' referenced before assignment", varname); +} + + +/////////////// RaiseClosureNameError.proto /////////////// +static CYTHON_INLINE void __Pyx_RaiseClosureNameError(const char *varname);/*proto*/ + +/////////////// RaiseClosureNameError /////////////// +static CYTHON_INLINE void __Pyx_RaiseClosureNameError(const char *varname) { + PyErr_Format(PyExc_NameError, "free variable '%s' referenced before assignment in enclosing scope", varname); +} + + +/////////////// RaiseUnboundMemoryviewSliceNogil.proto /////////////// +static void __Pyx_RaiseUnboundMemoryviewSliceNogil(const char *varname);/*proto*/ + +/////////////// RaiseUnboundMemoryviewSliceNogil /////////////// +//@requires: RaiseUnboundLocalError + +// Don't inline the function, it should really never be called in production +static void __Pyx_RaiseUnboundMemoryviewSliceNogil(const char *varname) { + #ifdef WITH_THREAD + PyGILState_STATE gilstate = PyGILState_Ensure(); + #endif + __Pyx_RaiseUnboundLocalError(varname); + #ifdef WITH_THREAD + PyGILState_Release(gilstate); + #endif +} + +//////////////// RaiseCppGlobalNameError.proto /////////////////////// +static CYTHON_INLINE void __Pyx_RaiseCppGlobalNameError(const char *varname); /*proto*/ + +/////////////// RaiseCppGlobalNameError ////////////////////////////// +static CYTHON_INLINE void __Pyx_RaiseCppGlobalNameError(const char *varname) { + PyErr_Format(PyExc_NameError, "C++ global '%s' is not initialized", varname); +} + +//////////////// RaiseCppAttributeError.proto /////////////////////// +static CYTHON_INLINE void __Pyx_RaiseCppAttributeError(const char *varname); /*proto*/ + +/////////////// RaiseCppAttributeError ////////////////////////////// +static CYTHON_INLINE void __Pyx_RaiseCppAttributeError(const char *varname) { + PyErr_Format(PyExc_AttributeError, "C++ attribute '%s' is not initialized", varname); +} + + diff --git a/Cython/Utility/Optimize.c b/Cython/Utility/Optimize.c index 35f3a67c9..e03b414e5 100644 --- a/Cython/Utility/Optimize.c +++ b/Cython/Utility/Optimize.c @@ -94,7 +94,7 @@ static CYTHON_INLINE PyObject* __Pyx_PyList_Pop(PyObject* L); /*proto*/ //@requires: ObjectHandling.c::PyObjectCallMethod0 static CYTHON_INLINE PyObject* __Pyx__PyObject_Pop(PyObject* L) { - if (Py_TYPE(L) == &PySet_Type) { + if (__Pyx_IS_TYPE(L, &PySet_Type)) { return PySet_Pop(L); } return __Pyx_PyObject_CallMethod0(L, PYIDENT("pop")); @@ -190,7 +190,7 @@ static PyObject* __Pyx_PyDict_GetItemDefault(PyObject* d, PyObject* key, PyObjec static PyObject* __Pyx_PyDict_GetItemDefault(PyObject* d, PyObject* key, PyObject* default_value) { PyObject* value; -#if PY_MAJOR_VERSION >= 3 && !CYTHON_COMPILING_IN_PYPY +#if PY_MAJOR_VERSION >= 3 && (!CYTHON_COMPILING_IN_PYPY || PYPY_VERSION_NUM >= 0x07020000) value = PyDict_GetItemWithError(d, key); if (unlikely(!value)) { if (unlikely(PyErr_Occurred())) @@ -227,8 +227,9 @@ static CYTHON_INLINE PyObject *__Pyx_PyDict_SetDefault(PyObject *d, PyObject *ke /////////////// dict_setdefault /////////////// static CYTHON_INLINE PyObject *__Pyx_PyDict_SetDefault(PyObject *d, PyObject *key, PyObject *default_value, - CYTHON_UNUSED int is_safe_type) { + int is_safe_type) { PyObject* value; + CYTHON_MAYBE_UNUSED_VAR(is_safe_type); #if PY_VERSION_HEX >= 0x030400A0 // we keep the method call at the end to avoid "unused" C compiler warnings if ((1)) { @@ -238,7 +239,7 @@ static CYTHON_INLINE PyObject *__Pyx_PyDict_SetDefault(PyObject *d, PyObject *ke #else if (is_safe_type == 1 || (is_safe_type == -1 && /* the following builtins presumably have repeatably safe and fast hash functions */ -#if PY_MAJOR_VERSION >= 3 && !CYTHON_COMPILING_IN_PYPY +#if PY_MAJOR_VERSION >= 3 && (!CYTHON_COMPILING_IN_PYPY || PYPY_VERSION_NUM >= 0x07020000) (PyUnicode_CheckExact(key) || PyString_CheckExact(key) || PyLong_CheckExact(key)))) { value = PyDict_GetItemWithError(d, key); if (unlikely(!value)) { @@ -591,51 +592,366 @@ static double __Pyx__PyObject_AsDouble(PyObject* obj); /* proto */ PyFloat_AsDouble(obj) : __Pyx__PyObject_AsDouble(obj)) #else #define __Pyx_PyObject_AsDouble(obj) \ -((likely(PyFloat_CheckExact(obj))) ? \ - PyFloat_AS_DOUBLE(obj) : __Pyx__PyObject_AsDouble(obj)) +((likely(PyFloat_CheckExact(obj))) ? PyFloat_AS_DOUBLE(obj) : \ + likely(PyLong_CheckExact(obj)) ? \ + PyLong_AsDouble(obj) : __Pyx__PyObject_AsDouble(obj)) #endif /////////////// pyobject_as_double /////////////// +//@requires: pybytes_as_double +//@requires: pyunicode_as_double +//@requires: ObjectHandling.c::PyObjectCallOneArg static double __Pyx__PyObject_AsDouble(PyObject* obj) { - PyObject* float_value; + if (PyUnicode_CheckExact(obj)) { + return __Pyx_PyUnicode_AsDouble(obj); + } else if (PyBytes_CheckExact(obj)) { + return __Pyx_PyBytes_AsDouble(obj); + } else if (PyByteArray_CheckExact(obj)) { + return __Pyx_PyByteArray_AsDouble(obj); + } else { + PyObject* float_value; #if !CYTHON_USE_TYPE_SLOTS - float_value = PyNumber_Float(obj); if ((0)) goto bad; + float_value = PyNumber_Float(obj); if ((0)) goto bad; + // avoid "unused" warnings + (void)__Pyx_PyObject_CallOneArg; #else - PyNumberMethods *nb = Py_TYPE(obj)->tp_as_number; - if (likely(nb) && likely(nb->nb_float)) { - float_value = nb->nb_float(obj); - if (likely(float_value) && unlikely(!PyFloat_Check(float_value))) { - PyErr_Format(PyExc_TypeError, - "__float__ returned non-float (type %.200s)", - Py_TYPE(float_value)->tp_name); - Py_DECREF(float_value); - goto bad; + PyNumberMethods *nb = Py_TYPE(obj)->tp_as_number; + if (likely(nb) && likely(nb->nb_float)) { + float_value = nb->nb_float(obj); + if (likely(float_value) && unlikely(!PyFloat_Check(float_value))) { + __Pyx_TypeName float_value_type_name = __Pyx_PyType_GetName(Py_TYPE(float_value)); + PyErr_Format(PyExc_TypeError, + "__float__ returned non-float (type " __Pyx_FMT_TYPENAME ")", + float_value_type_name); + __Pyx_DECREF_TypeName(float_value_type_name); + Py_DECREF(float_value); + goto bad; + } + } else { + float_value = __Pyx_PyObject_CallOneArg((PyObject*)&PyFloat_Type, obj); } - } else if (PyUnicode_CheckExact(obj) || PyBytes_CheckExact(obj)) { -#if PY_MAJOR_VERSION >= 3 - float_value = PyFloat_FromString(obj); -#else - float_value = PyFloat_FromString(obj, 0); #endif + if (likely(float_value)) { + double value = PyFloat_AS_DOUBLE(float_value); + Py_DECREF(float_value); + return value; + } + } +bad: + return (double)-1; +} + + +/////////////// pystring_as_double.proto /////////////// +//@requires: pyunicode_as_double +//@requires: pybytes_as_double + +static CYTHON_INLINE double __Pyx_PyString_AsDouble(PyObject *obj) { + #if PY_MAJOR_VERSION >= 3 + (void)__Pyx_PyBytes_AsDouble; + return __Pyx_PyUnicode_AsDouble(obj); + #else + (void)__Pyx_PyUnicode_AsDouble; + return __Pyx_PyBytes_AsDouble(obj); + #endif +} + + +/////////////// pyunicode_as_double.proto /////////////// + +static CYTHON_INLINE double __Pyx_PyUnicode_AsDouble(PyObject *obj);/*proto*/ + +/////////////// pyunicode_as_double.proto /////////////// +//@requires: pybytes_as_double + +#if PY_MAJOR_VERSION >= 3 && !CYTHON_COMPILING_IN_PYPY +static const char* __Pyx__PyUnicode_AsDouble_Copy(const void* data, const int kind, char* buffer, Py_ssize_t start, Py_ssize_t end) { + int last_was_punctuation; + Py_ssize_t i; + // number must not start with punctuation + last_was_punctuation = 1; + for (i=start; i <= end; i++) { + Py_UCS4 chr = PyUnicode_READ(kind, data, i); + int is_punctuation = (chr == '_') | (chr == '.'); + *buffer = (char)chr; + // reject sequences of '_' and '.' + buffer += (chr != '_'); + if (unlikely(chr > 127)) goto parse_failure; + if (unlikely(last_was_punctuation & is_punctuation)) goto parse_failure; + last_was_punctuation = is_punctuation; + } + if (unlikely(last_was_punctuation)) goto parse_failure; + *buffer = '\0'; + return buffer; + +parse_failure: + return NULL; +} + +static double __Pyx__PyUnicode_AsDouble_inf_nan(const void* data, int kind, Py_ssize_t start, Py_ssize_t length) { + int matches = 1; + Py_UCS4 chr; + Py_UCS4 sign = PyUnicode_READ(kind, data, start); + int is_signed = (sign == '-') | (sign == '+'); + start += is_signed; + length -= is_signed; + + switch (PyUnicode_READ(kind, data, start)) { + #ifdef Py_NAN + case 'n': + case 'N': + if (unlikely(length != 3)) goto parse_failure; + chr = PyUnicode_READ(kind, data, start+1); + matches &= (chr == 'a') | (chr == 'A'); + chr = PyUnicode_READ(kind, data, start+2); + matches &= (chr == 'n') | (chr == 'N'); + if (unlikely(!matches)) goto parse_failure; + return (sign == '-') ? -Py_NAN : Py_NAN; + #endif + case 'i': + case 'I': + if (unlikely(length < 3)) goto parse_failure; + chr = PyUnicode_READ(kind, data, start+1); + matches &= (chr == 'n') | (chr == 'N'); + chr = PyUnicode_READ(kind, data, start+2); + matches &= (chr == 'f') | (chr == 'F'); + if (likely(length == 3 && matches)) + return (sign == '-') ? -Py_HUGE_VAL : Py_HUGE_VAL; + if (unlikely(length != 8)) goto parse_failure; + chr = PyUnicode_READ(kind, data, start+3); + matches &= (chr == 'i') | (chr == 'I'); + chr = PyUnicode_READ(kind, data, start+4); + matches &= (chr == 'n') | (chr == 'N'); + chr = PyUnicode_READ(kind, data, start+5); + matches &= (chr == 'i') | (chr == 'I'); + chr = PyUnicode_READ(kind, data, start+6); + matches &= (chr == 't') | (chr == 'T'); + chr = PyUnicode_READ(kind, data, start+7); + matches &= (chr == 'y') | (chr == 'Y'); + if (unlikely(!matches)) goto parse_failure; + return (sign == '-') ? -Py_HUGE_VAL : Py_HUGE_VAL; + case '.': case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': + break; + default: + goto parse_failure; + } + return 0.0; +parse_failure: + return -1.0; +} + +static double __Pyx_PyUnicode_AsDouble_WithSpaces(PyObject *obj) { + double value; + const char *last; + char *end; + Py_ssize_t start, length = PyUnicode_GET_LENGTH(obj); + const int kind = PyUnicode_KIND(obj); + const void* data = PyUnicode_DATA(obj); + + // strip spaces at start and end + start = 0; + while (Py_UNICODE_ISSPACE(PyUnicode_READ(kind, data, start))) + start++; + while (start < length - 1 && Py_UNICODE_ISSPACE(PyUnicode_READ(kind, data, length - 1))) + length--; + length -= start; + if (unlikely(length <= 0)) goto fallback; + + // parse NaN / inf + value = __Pyx__PyUnicode_AsDouble_inf_nan(data, kind, start, length); + if (unlikely(value == -1.0)) goto fallback; + if (value != 0.0) return value; + + if (length < 40) { + char number[40]; + last = __Pyx__PyUnicode_AsDouble_Copy(data, kind, number, start, start + length); + if (unlikely(!last)) goto fallback; + value = PyOS_string_to_double(number, &end, NULL); } else { - PyObject* args = PyTuple_New(1); - if (unlikely(!args)) goto bad; - PyTuple_SET_ITEM(args, 0, obj); - float_value = PyObject_Call((PyObject*)&PyFloat_Type, args, 0); - PyTuple_SET_ITEM(args, 0, 0); - Py_DECREF(args); + char *number = (char*) PyMem_Malloc((length + 1) * sizeof(char)); + if (unlikely(!number)) goto fallback; + last = __Pyx__PyUnicode_AsDouble_Copy(data, kind, number, start, start + length); + if (unlikely(!last)) { + PyMem_Free(number); + goto fallback; + } + value = PyOS_string_to_double(number, &end, NULL); + PyMem_Free(number); + } + if (likely(end == last) || (value == (double)-1 && PyErr_Occurred())) { + return value; + } +fallback: + return __Pyx_SlowPyString_AsDouble(obj); +} +#endif + +static CYTHON_INLINE double __Pyx_PyUnicode_AsDouble(PyObject *obj) { + // Currently not optimised for Py2.7. +#if PY_MAJOR_VERSION >= 3 && !CYTHON_COMPILING_IN_PYPY + if (unlikely(__Pyx_PyUnicode_READY(obj) == -1)) + return (double)-1; + if (likely(PyUnicode_IS_ASCII(obj))) { + const char *s; + Py_ssize_t length; + s = PyUnicode_AsUTF8AndSize(obj, &length); + return __Pyx__PyBytes_AsDouble(obj, s, length); } + return __Pyx_PyUnicode_AsDouble_WithSpaces(obj); +#else + return __Pyx_SlowPyString_AsDouble(obj); +#endif +} + + +/////////////// pybytes_as_double.proto /////////////// + +static double __Pyx_SlowPyString_AsDouble(PyObject *obj);/*proto*/ +static double __Pyx__PyBytes_AsDouble(PyObject *obj, const char* start, Py_ssize_t length);/*proto*/ + +static CYTHON_INLINE double __Pyx_PyBytes_AsDouble(PyObject *obj) { + return __Pyx__PyBytes_AsDouble(obj, PyBytes_AS_STRING(obj), PyBytes_GET_SIZE(obj)); +} +static CYTHON_INLINE double __Pyx_PyByteArray_AsDouble(PyObject *obj) { + return __Pyx__PyBytes_AsDouble(obj, PyByteArray_AS_STRING(obj), PyByteArray_GET_SIZE(obj)); +} + + +/////////////// pybytes_as_double /////////////// + +static double __Pyx_SlowPyString_AsDouble(PyObject *obj) { + PyObject *float_value; +#if PY_MAJOR_VERSION >= 3 + float_value = PyFloat_FromString(obj); +#else + float_value = PyFloat_FromString(obj, 0); #endif if (likely(float_value)) { double value = PyFloat_AS_DOUBLE(float_value); Py_DECREF(float_value); return value; } -bad: return (double)-1; } +static const char* __Pyx__PyBytes_AsDouble_Copy(const char* start, char* buffer, Py_ssize_t length) { + // number must not start with punctuation + int last_was_punctuation = 1; + Py_ssize_t i; + for (i=0; i < length; i++) { + char chr = start[i]; + int is_punctuation = (chr == '_') | (chr == '.') | (chr == 'e') | (chr == 'E'); + *buffer = chr; + buffer += (chr != '_'); + // reject sequences of '_' and '.' + if (unlikely(last_was_punctuation & is_punctuation)) goto parse_failure; + last_was_punctuation = is_punctuation; + } + if (unlikely(last_was_punctuation)) goto parse_failure; + *buffer = '\0'; + return buffer; + +parse_failure: + return NULL; +} + +static double __Pyx__PyBytes_AsDouble_inf_nan(const char* start, Py_ssize_t length) { + int matches = 1; + char sign = start[0]; + int is_signed = (sign == '+') | (sign == '-'); + start += is_signed; + length -= is_signed; + + switch (start[0]) { + #ifdef Py_NAN + case 'n': + case 'N': + if (unlikely(length != 3)) goto parse_failure; + matches &= (start[1] == 'a' || start[1] == 'A'); + matches &= (start[2] == 'n' || start[2] == 'N'); + if (unlikely(!matches)) goto parse_failure; + return (sign == '-') ? -Py_NAN : Py_NAN; + #endif + case 'i': + case 'I': + if (unlikely(length < 3)) goto parse_failure; + matches &= (start[1] == 'n' || start[1] == 'N'); + matches &= (start[2] == 'f' || start[2] == 'F'); + if (likely(length == 3 && matches)) + return (sign == '-') ? -Py_HUGE_VAL : Py_HUGE_VAL; + if (unlikely(length != 8)) goto parse_failure; + matches &= (start[3] == 'i' || start[3] == 'I'); + matches &= (start[4] == 'n' || start[4] == 'N'); + matches &= (start[5] == 'i' || start[5] == 'I'); + matches &= (start[6] == 't' || start[6] == 'T'); + matches &= (start[7] == 'y' || start[7] == 'Y'); + if (unlikely(!matches)) goto parse_failure; + return (sign == '-') ? -Py_HUGE_VAL : Py_HUGE_VAL; + case '.': case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': + break; + default: + goto parse_failure; + } + return 0.0; +parse_failure: + return -1.0; +} + +static CYTHON_INLINE int __Pyx__PyBytes_AsDouble_IsSpace(char ch) { + // see Py_ISSPACE() in CPython + // https://github.com/python/cpython/blob/master/Python/pyctype.c + return (ch == 0x20) | !((ch < 0x9) | (ch > 0xd)); +} + +static CYTHON_UNUSED double __Pyx__PyBytes_AsDouble(PyObject *obj, const char* start, Py_ssize_t length) { + double value; + Py_ssize_t i, digits; + const char *last = start + length; + char *end; + + // strip spaces at start and end + while (__Pyx__PyBytes_AsDouble_IsSpace(*start)) + start++; + while (start < last - 1 && __Pyx__PyBytes_AsDouble_IsSpace(last[-1])) + last--; + length = last - start; + if (unlikely(length <= 0)) goto fallback; + + // parse NaN / inf + value = __Pyx__PyBytes_AsDouble_inf_nan(start, length); + if (unlikely(value == -1.0)) goto fallback; + if (value != 0.0) return value; + + // look for underscores + digits = 0; + for (i=0; i < length; digits += start[i++] != '_'); + + if (likely(digits == length)) { + value = PyOS_string_to_double(start, &end, NULL); + } else if (digits < 40) { + char number[40]; + last = __Pyx__PyBytes_AsDouble_Copy(start, number, length); + if (unlikely(!last)) goto fallback; + value = PyOS_string_to_double(number, &end, NULL); + } else { + char *number = (char*) PyMem_Malloc((digits + 1) * sizeof(char)); + if (unlikely(!number)) goto fallback; + last = __Pyx__PyBytes_AsDouble_Copy(start, number, length); + if (unlikely(!last)) { + PyMem_Free(number); + goto fallback; + } + value = PyOS_string_to_double(number, &end, NULL); + PyMem_Free(number); + } + if (likely(end == last) || (value == (double)-1 && PyErr_Occurred())) { + return value; + } +fallback: + return __Pyx_SlowPyString_AsDouble(obj); +} + /////////////// PyNumberPow2.proto /////////////// @@ -648,7 +964,7 @@ static PyObject* __Pyx__PyNumber_PowerOf2(PyObject *two, PyObject *exp, PyObject static PyObject* __Pyx__PyNumber_PowerOf2(PyObject *two, PyObject *exp, PyObject *none, int inplace) { // in CPython, 1<<N is substantially faster than 2**N -// see http://bugs.python.org/issue21420 +// see https://bugs.python.org/issue21420 #if !CYTHON_COMPILING_IN_PYPY Py_ssize_t shiftby; #if PY_MAJOR_VERSION < 3 @@ -722,7 +1038,9 @@ return_compare = ( ) }} -static CYTHON_INLINE {{c_ret_type}} __Pyx_PyInt_{{'' if ret_type.is_pyobject else 'Bool'}}{{op}}{{order}}(PyObject *op1, PyObject *op2, CYTHON_UNUSED long intval, CYTHON_UNUSED long inplace) { +static CYTHON_INLINE {{c_ret_type}} __Pyx_PyInt_{{'' if ret_type.is_pyobject else 'Bool'}}{{op}}{{order}}(PyObject *op1, PyObject *op2, long intval, long inplace) { + CYTHON_MAYBE_UNUSED_VAR(intval); + CYTHON_UNUSED_VAR(inplace); if (op1 == op2) { {{return_true if op == 'Eq' else return_false}}; } @@ -776,7 +1094,11 @@ static CYTHON_INLINE {{c_ret_type}} __Pyx_PyInt_{{'' if ret_type.is_pyobject els if (PyFloat_CheckExact({{pyval}})) { const long {{'a' if order == 'CObj' else 'b'}} = intval; +#if CYTHON_COMPILING_IN_LIMITED_API + double {{ival}} = __pyx_PyFloat_AsDouble({{pyval}}); +#else double {{ival}} = PyFloat_AS_DOUBLE({{pyval}}); +#endif {{return_compare('(double)a', '(double)b', c_op)}} } @@ -807,29 +1129,27 @@ static {{c_ret_type}} __Pyx_PyInt_{{'' if ret_type.is_pyobject else 'Bool'}}{{op {{py: return_false = 'Py_RETURN_FALSE' if ret_type.is_pyobject else 'return 0'}} {{py: slot_name = {'TrueDivide': 'true_divide', 'FloorDivide': 'floor_divide'}.get(op, op.lower()) }} {{py: cfunc_name = '__Pyx_PyInt_%s%s%s' % ('' if ret_type.is_pyobject else 'Bool', op, order)}} -{{py: zerodiv_check = lambda operand, _cfunc_name=cfunc_name: '%s_ZeroDivisionError(%s)' % (_cfunc_name, operand)}} {{py: c_op = { - 'Add': '+', 'Subtract': '-', 'Remainder': '%', 'TrueDivide': '/', 'FloorDivide': '/', + 'Add': '+', 'Subtract': '-', 'Multiply': '*', 'Remainder': '%', 'TrueDivide': '/', 'FloorDivide': '/', 'Or': '|', 'Xor': '^', 'And': '&', 'Rshift': '>>', 'Lshift': '<<', 'Eq': '==', 'Ne': '!=', }[op] }} +{{py: +def zerodiv_check(operand, optype='integer', _is_mod=op == 'Remainder', _needs_check=(order == 'CObj' and c_op in '%/')): + return ((( + 'if (unlikely(zerodivision_check && ((%s) == 0))) {' + ' PyErr_SetString(PyExc_ZeroDivisionError, "%s division%s by zero");' + ' return NULL;' + '}') % (operand, optype, ' or modulo' if _is_mod else '') + ) if _needs_check else '') +}} -{{if op in ('TrueDivide', 'FloorDivide', 'Remainder')}} -#if PY_MAJOR_VERSION < 3 || CYTHON_USE_PYLONG_INTERNALS -#define {{zerodiv_check('operand')}} \ - if (unlikely(zerodivision_check && ((operand) == 0))) { \ - PyErr_SetString(PyExc_ZeroDivisionError, "integer division{{if op == 'Remainder'}} or modulo{{endif}} by zero"); \ - return NULL; \ - } -#endif -{{endif}} - -static {{c_ret_type}} {{cfunc_name}}(PyObject *op1, PyObject *op2, CYTHON_UNUSED long intval, int inplace, int zerodivision_check) { - // Prevent "unused" warnings. - (void)inplace; - (void)zerodivision_check; +static {{c_ret_type}} {{cfunc_name}}(PyObject *op1, PyObject *op2, long intval, int inplace, int zerodivision_check) { + CYTHON_MAYBE_UNUSED_VAR(intval); + CYTHON_MAYBE_UNUSED_VAR(inplace); + CYTHON_UNUSED_VAR(zerodivision_check); {{if op in ('Eq', 'Ne')}} if (op1 == op2) { @@ -844,6 +1164,7 @@ static {{c_ret_type}} {{cfunc_name}}(PyObject *op1, PyObject *op2, CYTHON_UNUSED long x; {{endif}} long {{ival}} = PyInt_AS_LONG({{pyval}}); + {{zerodiv_check('b')}} {{if op in ('Eq', 'Ne')}} if (a {{c_op}} b) { @@ -859,21 +1180,18 @@ static {{c_ret_type}} {{cfunc_name}}(PyObject *op1, PyObject *op2, CYTHON_UNUSED return PyInt_FromLong(x); return PyLong_Type.tp_as_number->nb_{{slot_name}}(op1, op2); {{elif c_op == '%'}} - {{zerodiv_check('b')}} // see ExprNodes.py :: mod_int_utility_code x = a % b; x += ((x != 0) & ((x ^ b) < 0)) * b; return PyInt_FromLong(x); {{elif op == 'TrueDivide'}} - {{zerodiv_check('b')}} if (8 * sizeof(long) <= 53 || likely(labs({{ival}}) <= ((PY_LONG_LONG)1 << 53))) { return PyFloat_FromDouble((double)a / (double)b); } // let Python do the rounding return PyInt_Type.tp_as_number->nb_{{slot_name}}(op1, op2); {{elif op == 'FloorDivide'}} - // INT_MIN / -1 is the only case that overflows, b == 0 is an error case - {{zerodiv_check('b')}} + // INT_MIN / -1 is the only case that overflows if (unlikely(b == -1 && ((unsigned long)a) == 0-(unsigned long)a)) return PyInt_Type.tp_as_number->nb_{{slot_name}}(op1, op2); else { @@ -889,6 +1207,19 @@ static {{c_ret_type}} {{cfunc_name}}(PyObject *op1, PyObject *op2, CYTHON_UNUSED if (likely(b < (long) (sizeof(long)*8) && a == (a << b) >> b) || !a) { return PyInt_FromLong(a {{c_op}} b); } + {{elif c_op == '*'}} +#ifdef HAVE_LONG_LONG + if (sizeof(PY_LONG_LONG) > sizeof(long)) { + PY_LONG_LONG result = (PY_LONG_LONG)a {{c_op}} (PY_LONG_LONG)b; + return (result >= LONG_MIN && result <= LONG_MAX) ? + PyInt_FromLong((long)result) : PyLong_FromLongLong(result); + } +#endif +#if CYTHON_USE_TYPE_SLOTS + return PyInt_Type.tp_as_number->nb_{{slot_name}}(op1, op2); +#else + return PyNumber_{{op}}(op1, op2); +#endif {{else}} // other operations are safe, no overflow return PyInt_FromLong(a {{c_op}} b); @@ -908,6 +1239,38 @@ static {{c_ret_type}} {{cfunc_name}}(PyObject *op1, PyObject *op2, CYTHON_UNUSED {{endif}} const digit* digits = ((PyLongObject*){{pyval}})->ob_digit; const Py_ssize_t size = Py_SIZE({{pyval}}); + {{if c_op == '&'}} + // special case for &-ing arbitrarily large numbers with known single digit operands + if ((intval & PyLong_MASK) == intval) { + long result = 0; + if(likely(size)) { + result = intval & (likely(size>0) ? digits[0] : (PyLong_MASK - digits[0] + 1)); + } + return PyLong_FromLong(result); + } + {{endif}} + // special cases for 0: + - * % / // | ^ & >> << + if (unlikely(size == 0)) { + {{if order == 'CObj' and c_op in '%/'}} + // division by zero! + {{zerodiv_check('0')}} + {{elif order == 'CObj' and c_op in '+-|^>><<'}} + // x == x+0 == x-0 == x|0 == x^0 == x>>0 == x<<0 + return __Pyx_NewRef(op1); + {{elif order == 'CObj' and c_op in '*&'}} + // 0 == x*0 == x&0 + return __Pyx_NewRef(op2); + {{elif order == 'ObjC' and c_op in '+|^'}} + // x == 0+x == 0|x == 0^x + return __Pyx_NewRef(op2); + {{elif order == 'ObjC' and c_op == '-'}} + // -x == 0-x + return PyLong_FromLong(-intval); + {{elif order == 'ObjC' and (c_op in '*%&>><<' or op == 'FloorDivide')}} + // 0 == 0*x == 0%x == 0&x == 0>>x == 0<<x == 0//x + return __Pyx_NewRef(op1); + {{endif}} + } // handle most common case first to avoid indirect branch and optimise branch prediction if (likely(__Pyx_sst_abs(size) <= 1)) { {{ival}} = likely(size) ? digits[0] : 0; @@ -917,15 +1280,15 @@ static {{c_ret_type}} {{cfunc_name}}(PyObject *op1, PyObject *op2, CYTHON_UNUSED {{for _size in range(2, 5)}} {{for _case in (-_size, _size)}} case {{_case}}: - if (8 * sizeof(long) - 1 > {{_size}} * PyLong_SHIFT{{if op == 'TrueDivide'}} && {{_size-1}} * PyLong_SHIFT < 53{{endif}}) { + if (8 * sizeof(long) - 1 > {{_size}} * PyLong_SHIFT{{if c_op == '*'}}+30{{endif}}{{if op == 'TrueDivide'}} && {{_size-1}} * PyLong_SHIFT < 53{{endif}}) { {{ival}} = {{'-' if _case < 0 else ''}}(long) {{pylong_join(_size, 'digits')}}; break; {{if op not in ('Eq', 'Ne', 'TrueDivide')}} -#ifdef HAVE_LONG_LONG - } else if (8 * sizeof(PY_LONG_LONG) - 1 > {{_size}} * PyLong_SHIFT) { + #ifdef HAVE_LONG_LONG + } else if (8 * sizeof(PY_LONG_LONG) - 1 > {{_size}} * PyLong_SHIFT{{if c_op == '*'}}+30{{endif}}) { ll{{ival}} = {{'-' if _case < 0 else ''}}(PY_LONG_LONG) {{pylong_join(_size, 'digits', 'unsigned PY_LONG_LONG')}}; goto long_long; -#endif + #endif {{endif}} } // if size doesn't fit into a long or PY_LONG_LONG anymore, fall through to default @@ -954,20 +1317,25 @@ static {{c_ret_type}} {{cfunc_name}}(PyObject *op1, PyObject *op2, CYTHON_UNUSED {{return_false}}; } {{else}} - {{if c_op == '%'}} - {{zerodiv_check('b')}} + {{if c_op == '*'}} + (void)a; (void)b; + #ifdef HAVE_LONG_LONG + ll{{ival}} = {{ival}}; + goto long_long; + #else + return PyLong_Type.tp_as_number->nb_{{slot_name}}(op1, op2); + #endif + {{elif c_op == '%'}} // see ExprNodes.py :: mod_int_utility_code x = a % b; x += ((x != 0) & ((x ^ b) < 0)) * b; {{elif op == 'TrueDivide'}} - {{zerodiv_check('b')}} if ((8 * sizeof(long) <= 53 || likely(labs({{ival}}) <= ((PY_LONG_LONG)1 << 53))) || __Pyx_sst_abs(size) <= 52 / PyLong_SHIFT) { return PyFloat_FromDouble((double)a / (double)b); } return PyLong_Type.tp_as_number->nb_{{slot_name}}(op1, op2); {{elif op == 'FloorDivide'}} - {{zerodiv_check('b')}} { long q, r; // see ExprNodes.py :: div_int_utility_code @@ -1020,10 +1388,14 @@ static {{c_ret_type}} {{cfunc_name}}(PyObject *op1, PyObject *op2, CYTHON_UNUSED } #endif - {{if c_op in '+-' or op in ('TrueDivide', 'Eq', 'Ne')}} + {{if c_op in '+-*' or op in ('TrueDivide', 'Eq', 'Ne')}} if (PyFloat_CheckExact({{pyval}})) { const long {{'a' if order == 'CObj' else 'b'}} = intval; +#if CYTHON_COMPILING_IN_LIMITED_API + double {{ival}} = __pyx_PyFloat_AsDouble({{pyval}}); +#else double {{ival}} = PyFloat_AS_DOUBLE({{pyval}}); +#endif {{if op in ('Eq', 'Ne')}} if ((double)a {{c_op}} (double)b) { {{return_true}}; @@ -1032,12 +1404,7 @@ static {{c_ret_type}} {{cfunc_name}}(PyObject *op1, PyObject *op2, CYTHON_UNUSED } {{else}} double result; - {{if op == 'TrueDivide'}} - if (unlikely(zerodivision_check && b == 0)) { - PyErr_SetString(PyExc_ZeroDivisionError, "float division by zero"); - return NULL; - } - {{endif}} + {{zerodiv_check('b', 'float')}} // copied from floatobject.c in Py3.5: PyFPE_START_PROTECT("{{op.lower() if not op.endswith('Divide') else 'divide'}}", return NULL) result = ((double)a) {{c_op}} (double)b; @@ -1078,27 +1445,27 @@ static {{c_ret_type}} __Pyx_PyFloat_{{'' if ret_type.is_pyobject else 'Bool'}}{{ {{py: return_false = 'Py_RETURN_FALSE' if ret_type.is_pyobject else 'return 0'}} {{py: pyval, fval = ('op2', 'b') if order == 'CObj' else ('op1', 'a') }} {{py: cfunc_name = '__Pyx_PyFloat_%s%s%s' % ('' if ret_type.is_pyobject else 'Bool', op, order) }} -{{py: zerodiv_check = lambda operand, _cfunc_name=cfunc_name: '%s_ZeroDivisionError(%s)' % (_cfunc_name, operand)}} {{py: c_op = { 'Add': '+', 'Subtract': '-', 'TrueDivide': '/', 'Divide': '/', 'Remainder': '%', 'Eq': '==', 'Ne': '!=', }[op] }} - -{{if order == 'CObj' and c_op in '%/'}} -#define {{zerodiv_check('operand')}} if (unlikely(zerodivision_check && ((operand) == 0))) { \ - PyErr_SetString(PyExc_ZeroDivisionError, "float division{{if op == 'Remainder'}} or modulo{{endif}} by zero"); \ - return NULL; \ -} -{{endif}} +{{py: +def zerodiv_check(operand, _is_mod=op == 'Remainder', _needs_check=(order == 'CObj' and c_op in '%/')): + return ((( + 'if (unlikely(zerodivision_check && ((%s) == 0.0))) {' + ' PyErr_SetString(PyExc_ZeroDivisionError, "float division%s by zero");' + ' return NULL;' + '}') % (operand, ' or modulo' if _is_mod else '') + ) if _needs_check else '') +}} static {{c_ret_type}} {{cfunc_name}}(PyObject *op1, PyObject *op2, double floatval, int inplace, int zerodivision_check) { const double {{'a' if order == 'CObj' else 'b'}} = floatval; double {{fval}}{{if op not in ('Eq', 'Ne')}}, result{{endif}}; // Prevent "unused" warnings. - (void)inplace; - (void)zerodivision_check; + (void)inplace; (void)zerodivision_check; {{if op in ('Eq', 'Ne')}} if (op1 == op2) { @@ -1107,14 +1474,18 @@ static {{c_ret_type}} {{cfunc_name}}(PyObject *op1, PyObject *op2, double floatv {{endif}} if (likely(PyFloat_CheckExact({{pyval}}))) { +#if CYTHON_COMPILING_IN_LIMITED_API + {{fval}} = __pyx_PyFloat_AsDouble({{pyval}}); +#else {{fval}} = PyFloat_AS_DOUBLE({{pyval}}); - {{if order == 'CObj' and c_op in '%/'}}{{zerodiv_check(fval)}}{{endif}} +#endif + {{zerodiv_check(fval)}} } else #if PY_MAJOR_VERSION < 3 if (likely(PyInt_CheckExact({{pyval}}))) { {{fval}} = (double) PyInt_AS_LONG({{pyval}}); - {{if order == 'CObj' and c_op in '%/'}}{{zerodiv_check(fval)}}{{endif}} + {{zerodiv_check(fval)}} } else #endif @@ -1123,7 +1494,7 @@ static {{c_ret_type}} {{cfunc_name}}(PyObject *op1, PyObject *op2, double floatv const digit* digits = ((PyLongObject*){{pyval}})->ob_digit; const Py_ssize_t size = Py_SIZE({{pyval}}); switch (size) { - case 0: {{if order == 'CObj' and c_op in '%/'}}{{zerodiv_check('0')}}{{else}}{{fval}} = 0.0;{{endif}} break; + case 0: {{fval}} = 0.0; {{zerodiv_check(fval)}} break; case -1: {{fval}} = -(double) digits[0]; break; case 1: {{fval}} = (double) digits[0]; break; {{for _size in (2, 3, 4)}} @@ -1155,7 +1526,11 @@ static {{c_ret_type}} {{cfunc_name}}(PyObject *op1, PyObject *op2, double floatv {{else}} {{fval}} = PyLong_AsDouble({{pyval}}); if (unlikely({{fval}} == -1.0 && PyErr_Occurred())) return NULL; - {{if order == 'CObj' and c_op in '%/'}}{{zerodiv_check(fval)}}{{endif}} + {{if zerodiv_check(fval)}} + #if !CYTHON_USE_PYLONG_INTERNALS + {{zerodiv_check(fval)}} + #endif + {{endif}} {{endif}} } } else { @@ -1177,7 +1552,6 @@ static {{c_ret_type}} {{cfunc_name}}(PyObject *op1, PyObject *op2, double floatv } {{else}} // copied from floatobject.c in Py3.5: - {{if order == 'CObj' and c_op in '%/'}}{{zerodiv_check('b')}}{{endif}} PyFPE_START_PROTECT("{{op.lower() if not op.endswith('Divide') else 'divide'}}", return NULL) {{if c_op == '%'}} result = fmod(a, b); diff --git a/Cython/Utility/Overflow.c b/Cython/Utility/Overflow.c index 6dff81cc1..2e58ee54a 100644 --- a/Cython/Utility/Overflow.c +++ b/Cython/Utility/Overflow.c @@ -46,6 +46,24 @@ static int __Pyx_check_twos_complement(void) { #define __Pyx_div_no_overflow(a, b, overflow) ((a) / (b)) #define __Pyx_div_const_no_overflow(a, b, overflow) ((a) / (b)) +#if defined(__has_builtin) +# if __has_builtin(__builtin_add_overflow) && !defined(__ibmxl__) +# define __PYX_HAVE_BUILTIN_OVERFLOW +# endif +#elif defined(__GNUC__) && (__GNUC__ >= 5) && (!defined(__INTEL_COMPILER) || (__INTEL_COMPILER >= 1800)) +# define __PYX_HAVE_BUILTIN_OVERFLOW +#endif + +#if defined(__GNUC__) +# define __Pyx_is_constant(x) (__builtin_constant_p(x)) +#elif defined(__has_builtin) +# if __has_builtin(__builtin_constant_p) +# define __Pyx_is_constant(x) (__builtin_constant_p(x)) +# endif +#else +# define __Pyx_is_constant(x) (0) +#endif + /////////////// Common.init /////////////// //@substitute: naming @@ -56,6 +74,8 @@ if (unlikely(__Pyx_check_twos_complement())) { /////////////// BaseCaseUnsigned.proto /////////////// +{{if UINT == "long long"}}#ifdef HAVE_LONG_LONG{{endif}} + static CYTHON_INLINE {{UINT}} __Pyx_add_{{NAME}}_checking_overflow({{UINT}} a, {{UINT}} b, int *overflow); static CYTHON_INLINE {{UINT}} __Pyx_sub_{{NAME}}_checking_overflow({{UINT}} a, {{UINT}} b, int *overflow); static CYTHON_INLINE {{UINT}} __Pyx_mul_{{NAME}}_checking_overflow({{UINT}} a, {{UINT}} b, int *overflow); @@ -64,11 +84,41 @@ static CYTHON_INLINE {{UINT}} __Pyx_div_{{NAME}}_checking_overflow({{UINT}} a, { // Use these when b is known at compile time. #define __Pyx_add_const_{{NAME}}_checking_overflow __Pyx_add_{{NAME}}_checking_overflow #define __Pyx_sub_const_{{NAME}}_checking_overflow __Pyx_sub_{{NAME}}_checking_overflow +#if defined(__PYX_HAVE_BUILTIN_OVERFLOW) +#define __Pyx_mul_const_{{NAME}}_checking_overflow __Pyx_mul_{{NAME}}_checking_overflow +#else static CYTHON_INLINE {{UINT}} __Pyx_mul_const_{{NAME}}_checking_overflow({{UINT}} a, {{UINT}} constant, int *overflow); +#endif #define __Pyx_div_const_{{NAME}}_checking_overflow __Pyx_div_{{NAME}}_checking_overflow +{{if UINT == "long long"}}#endif{{endif}} + /////////////// BaseCaseUnsigned /////////////// +{{if UINT == "long long"}}#ifdef HAVE_LONG_LONG{{endif}} + +#if defined(__PYX_HAVE_BUILTIN_OVERFLOW) + +static CYTHON_INLINE {{UINT}} __Pyx_add_{{NAME}}_checking_overflow({{UINT}} a, {{UINT}} b, int *overflow) { + {{UINT}} result; + *overflow |= __builtin_add_overflow(a, b, &result); + return result; +} + +static CYTHON_INLINE {{UINT}} __Pyx_sub_{{NAME}}_checking_overflow({{UINT}} a, {{UINT}} b, int *overflow) { + {{UINT}} result; + *overflow |= __builtin_sub_overflow(a, b, &result); + return result; +} + +static CYTHON_INLINE {{UINT}} __Pyx_mul_{{NAME}}_checking_overflow({{UINT}} a, {{UINT}} b, int *overflow) { + {{UINT}} result; + *overflow |= __builtin_mul_overflow(a, b, &result); + return result; +} + +#else + static CYTHON_INLINE {{UINT}} __Pyx_add_{{NAME}}_checking_overflow({{UINT}} a, {{UINT}} b, int *overflow) { {{UINT}} r = a + b; *overflow |= r < a; @@ -82,7 +132,12 @@ static CYTHON_INLINE {{UINT}} __Pyx_sub_{{NAME}}_checking_overflow({{UINT}} a, { } static CYTHON_INLINE {{UINT}} __Pyx_mul_{{NAME}}_checking_overflow({{UINT}} a, {{UINT}} b, int *overflow) { - if ((sizeof({{UINT}}) < sizeof(unsigned long))) { + // if we have a constant, use the constant version + if (__Pyx_is_constant(b)) { + return __Pyx_mul_const_{{NAME}}_checking_overflow(a, b, overflow); + } else if (__Pyx_is_constant(a)) { + return __Pyx_mul_const_{{NAME}}_checking_overflow(b, a, overflow); + } else if ((sizeof({{UINT}}) < sizeof(unsigned long))) { unsigned long big_r = ((unsigned long) a) * ((unsigned long) b); {{UINT}} r = ({{UINT}}) big_r; *overflow |= big_r != r; @@ -95,21 +150,27 @@ static CYTHON_INLINE {{UINT}} __Pyx_mul_{{NAME}}_checking_overflow({{UINT}} a, { return r; #endif } else { - {{UINT}} prod = a * b; - double dprod = ((double) a) * ((double) b); - // Overflow results in an error of at least 2^sizeof(UINT), - // whereas rounding represents an error on the order of 2^(sizeof(UINT)-53). - *overflow |= fabs(dprod - prod) > (__PYX_MAX({{UINT}}) / 2); - return prod; + return __Pyx_mul_const_{{NAME}}_checking_overflow(a, b, overflow); } } static CYTHON_INLINE {{UINT}} __Pyx_mul_const_{{NAME}}_checking_overflow({{UINT}} a, {{UINT}} b, int *overflow) { - if (b > 1) { - *overflow |= a > __PYX_MAX({{UINT}}) / b; + // note that deliberately the overflow check is written such that it divides by b; this + // function is used when b is a constant thus the compiler should be able to eliminate the + // (very slow on most CPUs!) division operation + {{UINT}} prod; + if (__Pyx_is_constant(a) && !__Pyx_is_constant(b)) { + // if a is a compile-time constant and b isn't, swap them + {{UINT}} temp = b; + b = a; + a = temp; } - return a * b; + prod = a * b; + if (b != 0) + *overflow |= a > (__PYX_MAX({{UINT}}) / b); + return prod; } +#endif // __PYX_HAVE_BUILTIN_OVERFLOW static CYTHON_INLINE {{UINT}} __Pyx_div_{{NAME}}_checking_overflow({{UINT}} a, {{UINT}} b, int *overflow) { @@ -120,9 +181,13 @@ static CYTHON_INLINE {{UINT}} __Pyx_div_{{NAME}}_checking_overflow({{UINT}} a, { return a / b; } +{{if UINT == "long long"}}#endif{{endif}} + /////////////// BaseCaseSigned.proto /////////////// +{{if INT == "long long"}}#ifdef HAVE_LONG_LONG{{endif}} + static CYTHON_INLINE {{INT}} __Pyx_add_{{NAME}}_checking_overflow({{INT}} a, {{INT}} b, int *overflow); static CYTHON_INLINE {{INT}} __Pyx_sub_{{NAME}}_checking_overflow({{INT}} a, {{INT}} b, int *overflow); static CYTHON_INLINE {{INT}} __Pyx_mul_{{NAME}}_checking_overflow({{INT}} a, {{INT}} b, int *overflow); @@ -130,13 +195,43 @@ static CYTHON_INLINE {{INT}} __Pyx_div_{{NAME}}_checking_overflow({{INT}} a, {{I // Use when b is known at compile time. -static CYTHON_INLINE {{INT}} __Pyx_add_const_{{NAME}}_checking_overflow({{INT}} a, {{INT}} b, int *overflow); -static CYTHON_INLINE {{INT}} __Pyx_sub_const_{{NAME}}_checking_overflow({{INT}} a, {{INT}} b, int *overflow); +#define __Pyx_add_const_{{NAME}}_checking_overflow __Pyx_add_{{NAME}}_checking_overflow +#define __Pyx_sub_const_{{NAME}}_checking_overflow __Pyx_sub_{{NAME}}_checking_overflow +#if defined(__PYX_HAVE_BUILTIN_OVERFLOW) +#define __Pyx_mul_const_{{NAME}}_checking_overflow __Pyx_mul_{{NAME}}_checking_overflow +#else static CYTHON_INLINE {{INT}} __Pyx_mul_const_{{NAME}}_checking_overflow({{INT}} a, {{INT}} constant, int *overflow); +#endif #define __Pyx_div_const_{{NAME}}_checking_overflow __Pyx_div_{{NAME}}_checking_overflow +{{if INT == "long long"}}#endif{{endif}} + /////////////// BaseCaseSigned /////////////// +{{if INT == "long long"}}#ifdef HAVE_LONG_LONG{{endif}} + +#if defined(__PYX_HAVE_BUILTIN_OVERFLOW) + +static CYTHON_INLINE {{INT}} __Pyx_add_{{NAME}}_checking_overflow({{INT}} a, {{INT}} b, int *overflow) { + {{INT}} result; + *overflow |= __builtin_add_overflow(a, b, &result); + return result; +} + +static CYTHON_INLINE {{INT}} __Pyx_sub_{{NAME}}_checking_overflow({{INT}} a, {{INT}} b, int *overflow) { + {{INT}} result; + *overflow |= __builtin_sub_overflow(a, b, &result); + return result; +} + +static CYTHON_INLINE {{INT}} __Pyx_mul_{{NAME}}_checking_overflow({{INT}} a, {{INT}} b, int *overflow) { + {{INT}} result; + *overflow |= __builtin_mul_overflow(a, b, &result); + return result; +} + +#else + static CYTHON_INLINE {{INT}} __Pyx_add_{{NAME}}_checking_overflow({{INT}} a, {{INT}} b, int *overflow) { if ((sizeof({{INT}}) < sizeof(long))) { long big_r = ((long) a) + ((long) b); @@ -151,40 +246,33 @@ static CYTHON_INLINE {{INT}} __Pyx_add_{{NAME}}_checking_overflow({{INT}} a, {{I return r; #endif } else { - // Signed overflow undefined, but unsigned overflow is well defined. - {{INT}} r = ({{INT}}) ((unsigned {{INT}}) a + (unsigned {{INT}}) b); + // Signed overflow undefined, but unsigned overflow is well defined. Casting is + // implementation-defined, but we assume two's complement (see __Pyx_check_twos_complement + // above), and arithmetic in two's-complement is the same as unsigned arithmetic. + unsigned {{INT}} r = (unsigned {{INT}}) a + (unsigned {{INT}}) b; // Overflow happened if the operands have the same sign, but the result // has opposite sign. - // sign(a) == sign(b) != sign(r) - {{INT}} sign_a = __PYX_SIGN_BIT({{INT}}) & a; - {{INT}} sign_b = __PYX_SIGN_BIT({{INT}}) & b; - {{INT}} sign_r = __PYX_SIGN_BIT({{INT}}) & r; - *overflow |= (sign_a == sign_b) & (sign_a != sign_r); - return r; - } -} - -static CYTHON_INLINE {{INT}} __Pyx_add_const_{{NAME}}_checking_overflow({{INT}} a, {{INT}} b, int *overflow) { - if (b > 0) { - *overflow |= a > __PYX_MAX({{INT}}) - b; - } else if (b < 0) { - *overflow |= a < __PYX_MIN({{INT}}) - b; + *overflow |= (((unsigned {{INT}})a ^ r) & ((unsigned {{INT}})b ^ r)) >> (8 * sizeof({{INT}}) - 1); + return ({{INT}}) r; } - return a + b; } static CYTHON_INLINE {{INT}} __Pyx_sub_{{NAME}}_checking_overflow({{INT}} a, {{INT}} b, int *overflow) { - *overflow |= b == __PYX_MIN({{INT}}); - return __Pyx_add_{{NAME}}_checking_overflow(a, -b, overflow); -} - -static CYTHON_INLINE {{INT}} __Pyx_sub_const_{{NAME}}_checking_overflow({{INT}} a, {{INT}} b, int *overflow) { - *overflow |= b == __PYX_MIN({{INT}}); - return __Pyx_add_const_{{NAME}}_checking_overflow(a, -b, overflow); + // Compilers don't handle widening as well in the subtraction case, so don't bother + unsigned {{INT}} r = (unsigned {{INT}}) a - (unsigned {{INT}}) b; + // Overflow happened if the operands differing signs, and the result + // has opposite sign to a. + *overflow |= (((unsigned {{INT}})a ^ (unsigned {{INT}})b) & ((unsigned {{INT}})a ^ r)) >> (8 * sizeof({{INT}}) - 1); + return ({{INT}}) r; } static CYTHON_INLINE {{INT}} __Pyx_mul_{{NAME}}_checking_overflow({{INT}} a, {{INT}} b, int *overflow) { - if ((sizeof({{INT}}) < sizeof(long))) { + // if we have a constant, use the constant version + if (__Pyx_is_constant(b)) { + return __Pyx_mul_const_{{NAME}}_checking_overflow(a, b, overflow); + } else if (__Pyx_is_constant(a)) { + return __Pyx_mul_const_{{NAME}}_checking_overflow(b, a, overflow); + } else if ((sizeof({{INT}}) < sizeof(long))) { long big_r = ((long) a) * ((long) b); {{INT}} r = ({{INT}}) big_r; *overflow |= big_r != r; @@ -197,16 +285,20 @@ static CYTHON_INLINE {{INT}} __Pyx_mul_{{NAME}}_checking_overflow({{INT}} a, {{I return ({{INT}}) r; #endif } else { - {{INT}} prod = a * b; - double dprod = ((double) a) * ((double) b); - // Overflow results in an error of at least 2^sizeof(INT), - // whereas rounding represents an error on the order of 2^(sizeof(INT)-53). - *overflow |= fabs(dprod - prod) > (__PYX_MAX({{INT}}) / 2); - return prod; + return __Pyx_mul_const_{{NAME}}_checking_overflow(a, b, overflow); } } static CYTHON_INLINE {{INT}} __Pyx_mul_const_{{NAME}}_checking_overflow({{INT}} a, {{INT}} b, int *overflow) { + // note that deliberately all these comparisons are written such that they divide by b; this + // function is used when b is a constant thus the compiler should be able to eliminate the + // (very slow on most CPUs!) division operations + if (__Pyx_is_constant(a) && !__Pyx_is_constant(b)) { + // if a is a compile-time constant and b isn't, swap them + {{INT}} temp = b; + b = a; + a = temp; + } if (b > 1) { *overflow |= a > __PYX_MAX({{INT}}) / b; *overflow |= a < __PYX_MIN({{INT}}) / b; @@ -216,18 +308,21 @@ static CYTHON_INLINE {{INT}} __Pyx_mul_const_{{NAME}}_checking_overflow({{INT}} *overflow |= a > __PYX_MIN({{INT}}) / b; *overflow |= a < __PYX_MAX({{INT}}) / b; } - return a * b; + return ({{INT}}) (((unsigned {{INT}})a) * ((unsigned {{INT}}) b)); } +#endif // defined(__PYX_HAVE_BUILTIN_OVERFLOW) static CYTHON_INLINE {{INT}} __Pyx_div_{{NAME}}_checking_overflow({{INT}} a, {{INT}} b, int *overflow) { if (b == 0) { *overflow |= 1; return 0; } - *overflow |= (a == __PYX_MIN({{INT}})) & (b == -1); - return a / b; + *overflow |= a == __PYX_MIN({{INT}}) && b == -1; + return ({{INT}}) ((unsigned {{INT}}) a / (unsigned {{INT}}) b); } +{{if INT == "long long"}}#endif{{endif}} + /////////////// SizeCheck.init /////////////// //@substitute: naming @@ -265,24 +360,24 @@ static CYTHON_INLINE {{TYPE}} __Pyx_{{BINOP}}_{{NAME}}_checking_overflow({{TYPE} return __Pyx_{{BINOP}}_no_overflow(a, b, overflow); } else if (__PYX_IS_UNSIGNED({{TYPE}})) { if ((sizeof({{TYPE}}) == sizeof(unsigned int))) { - return __Pyx_{{BINOP}}_unsigned_int_checking_overflow(a, b, overflow); + return ({{TYPE}}) __Pyx_{{BINOP}}_unsigned_int_checking_overflow(a, b, overflow); } else if ((sizeof({{TYPE}}) == sizeof(unsigned long))) { - return __Pyx_{{BINOP}}_unsigned_long_checking_overflow(a, b, overflow); + return ({{TYPE}}) __Pyx_{{BINOP}}_unsigned_long_checking_overflow(a, b, overflow); #ifdef HAVE_LONG_LONG } else if ((sizeof({{TYPE}}) == sizeof(unsigned PY_LONG_LONG))) { - return __Pyx_{{BINOP}}_unsigned_long_long_checking_overflow(a, b, overflow); + return ({{TYPE}}) __Pyx_{{BINOP}}_unsigned_long_long_checking_overflow(a, b, overflow); #endif } else { abort(); return 0; /* handled elsewhere */ } } else { if ((sizeof({{TYPE}}) == sizeof(int))) { - return __Pyx_{{BINOP}}_int_checking_overflow(a, b, overflow); + return ({{TYPE}}) __Pyx_{{BINOP}}_int_checking_overflow(a, b, overflow); } else if ((sizeof({{TYPE}}) == sizeof(long))) { - return __Pyx_{{BINOP}}_long_checking_overflow(a, b, overflow); + return ({{TYPE}}) __Pyx_{{BINOP}}_long_checking_overflow(a, b, overflow); #ifdef HAVE_LONG_LONG } else if ((sizeof({{TYPE}}) == sizeof(PY_LONG_LONG))) { - return __Pyx_{{BINOP}}_long_long_checking_overflow(a, b, overflow); + return ({{TYPE}}) __Pyx_{{BINOP}}_long_long_checking_overflow(a, b, overflow); #endif } else { abort(); return 0; /* handled elsewhere */ @@ -293,19 +388,24 @@ static CYTHON_INLINE {{TYPE}} __Pyx_{{BINOP}}_{{NAME}}_checking_overflow({{TYPE} /////////////// LeftShift.proto /////////////// static CYTHON_INLINE {{TYPE}} __Pyx_lshift_{{NAME}}_checking_overflow({{TYPE}} a, {{TYPE}} b, int *overflow) { - *overflow |= + int overflow_check = #if {{SIGNED}} - (b < 0) | + (a < 0) || (b < 0) || #endif - (b > ({{TYPE}}) (8 * sizeof({{TYPE}}))) | (a > (__PYX_MAX({{TYPE}}) >> b)); - return a << b; + // the following must be a _logical_ OR as the RHS is undefined if the LHS is true + (b >= ({{TYPE}}) (8 * sizeof({{TYPE}}))) || (a > (__PYX_MAX({{TYPE}}) >> b)); + if (overflow_check) { + *overflow |= 1; + return 0; + } else { + return a << b; + } } #define __Pyx_lshift_const_{{NAME}}_checking_overflow __Pyx_lshift_{{NAME}}_checking_overflow /////////////// UnaryNegOverflows.proto /////////////// -//FIXME: shouldn't the macro name be prefixed by "__Pyx_" ? Too late now, I guess... // from intobject.c -#define UNARY_NEG_WOULD_OVERFLOW(x) \ +#define __Pyx_UNARY_NEG_WOULD_OVERFLOW(x) \ (((x) < 0) & ((unsigned long)(x) == 0-(unsigned long)(x))) diff --git a/Cython/Utility/Profile.c b/Cython/Utility/Profile.c index 47f04b1c7..906502078 100644 --- a/Cython/Utility/Profile.c +++ b/Cython/Utility/Profile.c @@ -6,7 +6,7 @@ // but maybe some other profilers don't. #ifndef CYTHON_PROFILE -#if CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_PYSTON +#if CYTHON_COMPILING_IN_LIMITED_API || CYTHON_COMPILING_IN_PYPY #define CYTHON_PROFILE 0 #else #define CYTHON_PROFILE 1 @@ -214,12 +214,12 @@ if (CYTHON_TRACE_NOGIL) { \ int ret = 0; \ PyThreadState *tstate; \ - PyGILState_STATE state = PyGILState_Ensure(); \ + PyGILState_STATE state = __Pyx_PyGILState_Ensure(); \ tstate = __Pyx_PyThreadState_Current; \ if (__Pyx_IsTracing(tstate, 0, 0) && tstate->c_tracefunc && $frame_cname->f_trace) { \ ret = __Pyx_call_line_trace_func(tstate, $frame_cname, lineno); \ } \ - PyGILState_Release(state); \ + __Pyx_PyGILState_Release(state); \ if (unlikely(ret)) goto_error; \ } \ } else { \ diff --git a/Cython/Utility/StringTools.c b/Cython/Utility/StringTools.c index a7559ec03..6776eb11b 100644 --- a/Cython/Utility/StringTools.c +++ b/Cython/Utility/StringTools.c @@ -7,15 +7,78 @@ #include <string> + +//////////////////// ssize_strlen.proto //////////////////// + +static CYTHON_INLINE Py_ssize_t __Pyx_ssize_strlen(const char *s);/*proto*/ + +//////////////////// ssize_strlen //////////////////// +//@requires: IncludeStringH + +static CYTHON_INLINE Py_ssize_t __Pyx_ssize_strlen(const char *s) { + size_t len = strlen(s); + if (unlikely(len > PY_SSIZE_T_MAX)) { + PyErr_SetString(PyExc_OverflowError, "byte string is too long"); + return -1; + } + return (Py_ssize_t) len; +} + + +//////////////////// ssize_pyunicode_strlen.proto //////////////////// + +static CYTHON_INLINE Py_ssize_t __Pyx_Py_UNICODE_ssize_strlen(const Py_UNICODE *u);/*proto*/ + +//////////////////// ssize_pyunicode_strlen //////////////////// + +static CYTHON_INLINE Py_ssize_t __Pyx_Py_UNICODE_ssize_strlen(const Py_UNICODE *u) { + size_t len = __Pyx_Py_UNICODE_strlen(u); + if (unlikely(len > PY_SSIZE_T_MAX)) { + PyErr_SetString(PyExc_OverflowError, "Py_UNICODE string is too long"); + return -1; + } + return (Py_ssize_t) len; +} + + //////////////////// InitStrings.proto //////////////////// +#if CYTHON_COMPILING_IN_LIMITED_API +static int __Pyx_InitString(__Pyx_StringTabEntry t, PyObject **str); /*proto*/ +#else static int __Pyx_InitStrings(__Pyx_StringTabEntry *t); /*proto*/ +#endif //////////////////// InitStrings //////////////////// +#if PY_MAJOR_VERSION >= 3 +static int __Pyx_InitString(__Pyx_StringTabEntry t, PyObject **str) { + if (t.is_unicode | t.is_str) { + if (t.intern) { + *str = PyUnicode_InternFromString(t.s); + } else if (t.encoding) { + *str = PyUnicode_Decode(t.s, t.n - 1, t.encoding, NULL); + } else { + *str = PyUnicode_FromStringAndSize(t.s, t.n - 1); + } + } else { + *str = PyBytes_FromStringAndSize(t.s, t.n - 1); + } + if (!*str) + return -1; + // initialise cached hash value + if (PyObject_Hash(*str) == -1) + return -1; + return 0; +} +#endif + +#if !CYTHON_COMPILING_IN_LIMITED_API static int __Pyx_InitStrings(__Pyx_StringTabEntry *t) { while (t->p) { - #if PY_MAJOR_VERSION < 3 + #if PY_MAJOR_VERSION >= 3 /* Python 3+ has unicode identifiers */ + __Pyx_InitString(*t, t->p); + #else if (t->is_unicode) { *t->p = PyUnicode_DecodeUTF8(t->s, t->n - 1, NULL); } else if (t->intern) { @@ -23,28 +86,17 @@ static int __Pyx_InitStrings(__Pyx_StringTabEntry *t) { } else { *t->p = PyString_FromStringAndSize(t->s, t->n - 1); } - #else /* Python 3+ has unicode identifiers */ - if (t->is_unicode | t->is_str) { - if (t->intern) { - *t->p = PyUnicode_InternFromString(t->s); - } else if (t->encoding) { - *t->p = PyUnicode_Decode(t->s, t->n - 1, t->encoding, NULL); - } else { - *t->p = PyUnicode_FromStringAndSize(t->s, t->n - 1); - } - } else { - *t->p = PyBytes_FromStringAndSize(t->s, t->n - 1); - } - #endif if (!*t->p) return -1; // initialise cached hash value if (PyObject_Hash(*t->p) == -1) return -1; + #endif ++t; } return 0; } +#endif //////////////////// BytesContains.proto //////////////////// @@ -183,7 +235,7 @@ static CYTHON_INLINE int __Pyx_PyUnicode_Equals(PyObject* s1, PyObject* s2, int //@requires: BytesEquals static CYTHON_INLINE int __Pyx_PyUnicode_Equals(PyObject* s1, PyObject* s2, int equals) { -#if CYTHON_COMPILING_IN_PYPY +#if CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_LIMITED_API return PyObject_RichCompareBool(s1, s2, equals); #else #if PY_MAJOR_VERSION < 3 @@ -294,7 +346,7 @@ static CYTHON_INLINE int __Pyx_PyBytes_Equals(PyObject* s1, PyObject* s2, int eq //@requires: IncludeStringH static CYTHON_INLINE int __Pyx_PyBytes_Equals(PyObject* s1, PyObject* s2, int equals) { -#if CYTHON_COMPILING_IN_PYPY +#if CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_LIMITED_API return PyObject_RichCompareBool(s1, s2, equals); #else if (s1 == s2) { @@ -591,6 +643,8 @@ static CYTHON_INLINE PyObject* __Pyx_PyUnicode_Substring( stop = length; if (stop <= start) return __Pyx_NewRef($empty_unicode); + if (start == 0 && stop == length) + return __Pyx_NewRef(text); #if CYTHON_PEP393_ENABLED return PyUnicode_FromKindAndData(PyUnicode_KIND(text), PyUnicode_1BYTE_DATA(text) + start*PyUnicode_KIND(text), stop-start); @@ -835,25 +889,29 @@ static PyObject* __Pyx_PyUnicode_Join(PyObject* value_tuple, Py_ssize_t value_co //@substitute: naming static PyObject* __Pyx_PyUnicode_Join(PyObject* value_tuple, Py_ssize_t value_count, Py_ssize_t result_ulength, - CYTHON_UNUSED Py_UCS4 max_char) { + Py_UCS4 max_char) { #if CYTHON_USE_UNICODE_INTERNALS && CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS PyObject *result_uval; - int result_ukind; + int result_ukind, kind_shift; Py_ssize_t i, char_pos; void *result_udata; + CYTHON_MAYBE_UNUSED_VAR(max_char); #if CYTHON_PEP393_ENABLED // Py 3.3+ (post PEP-393) result_uval = PyUnicode_New(result_ulength, max_char); if (unlikely(!result_uval)) return NULL; result_ukind = (max_char <= 255) ? PyUnicode_1BYTE_KIND : (max_char <= 65535) ? PyUnicode_2BYTE_KIND : PyUnicode_4BYTE_KIND; + kind_shift = (result_ukind == PyUnicode_4BYTE_KIND) ? 2 : result_ukind - 1; result_udata = PyUnicode_DATA(result_uval); #else // Py 2.x/3.2 (pre PEP-393) result_uval = PyUnicode_FromUnicode(NULL, result_ulength); if (unlikely(!result_uval)) return NULL; result_ukind = sizeof(Py_UNICODE); + kind_shift = (result_ukind == 4) ? 2 : result_ukind - 1; result_udata = PyUnicode_AS_UNICODE(result_uval); #endif + assert(kind_shift == 2 || kind_shift == 1 || kind_shift == 0); char_pos = 0; for (i=0; i < value_count; i++) { @@ -866,12 +924,12 @@ static PyObject* __Pyx_PyUnicode_Join(PyObject* value_tuple, Py_ssize_t value_co ulength = __Pyx_PyUnicode_GET_LENGTH(uval); if (unlikely(!ulength)) continue; - if (unlikely(char_pos + ulength < 0)) + if (unlikely((PY_SSIZE_T_MAX >> kind_shift) - ulength < char_pos)) goto overflow; ukind = __Pyx_PyUnicode_KIND(uval); udata = __Pyx_PyUnicode_DATA(uval); if (!CYTHON_PEP393_ENABLED || ukind == result_ukind) { - memcpy((char *)result_udata + char_pos * result_ukind, udata, (size_t) (ulength * result_ukind)); + memcpy((char *)result_udata + (char_pos << kind_shift), udata, (size_t) (ulength << kind_shift)); } else { #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030300F0 || defined(_PyUnicode_FastCopyCharacters) _PyUnicode_FastCopyCharacters(result_uval, char_pos, uval, 0, ulength); @@ -893,8 +951,9 @@ bad: return NULL; #else // non-CPython fallback - result_ulength++; - value_count++; + CYTHON_UNUSED_VAR(max_char); + CYTHON_UNUSED_VAR(result_ulength); + CYTHON_UNUSED_VAR(value_count); return PyUnicode_Join($empty_unicode, value_tuple); #endif } @@ -1130,11 +1189,12 @@ static PyObject* __Pyx_PyObject_Format(PyObject* obj, PyObject* format_spec) { likely(PyString_CheckExact(s)) ? PyUnicode_FromEncodedObject(s, NULL, "strict") : \ PyObject_Format(s, f)) #elif CYTHON_USE_TYPE_SLOTS - // Py3 nicely returns unicode strings from str() which makes this quite efficient for builtin types + // Py3 nicely returns unicode strings from str() and repr(), which makes this quite efficient for builtin types. + // In Py3.8+, tp_str() delegates to tp_repr(), so we call tp_repr() directly here. #define __Pyx_PyObject_FormatSimple(s, f) ( \ likely(PyUnicode_CheckExact(s)) ? (Py_INCREF(s), s) : \ - likely(PyLong_CheckExact(s)) ? PyLong_Type.tp_str(s) : \ - likely(PyFloat_CheckExact(s)) ? PyFloat_Type.tp_str(s) : \ + likely(PyLong_CheckExact(s)) ? PyLong_Type.tp_repr(s) : \ + likely(PyFloat_CheckExact(s)) ? PyFloat_Type.tp_repr(s) : \ PyObject_Format(s, f)) #else #define __Pyx_PyObject_FormatSimple(s, f) ( \ @@ -1193,3 +1253,22 @@ static CYTHON_INLINE PyObject* __Pyx_PyUnicode_Unicode(PyObject *obj) { #define __Pyx_PyObject_Unicode(obj) \ (likely(PyUnicode_CheckExact(obj)) ? __Pyx_NewRef(obj) : PyObject_Unicode(obj)) #endif + + +//////////////////// PyStr_Str.proto //////////////////// + +static CYTHON_INLINE PyObject* __Pyx_PyStr_Str(PyObject *obj);/*proto*/ + +//////////////////// PyStr_Str //////////////////// + +static CYTHON_INLINE PyObject* __Pyx_PyStr_Str(PyObject *obj) { + if (unlikely(obj == Py_None)) + obj = PYIDENT("None"); + return __Pyx_NewRef(obj); +} + + +//////////////////// PyObject_Str.proto //////////////////// + +#define __Pyx_PyObject_Str(obj) \ + (likely(PyString_CheckExact(obj)) ? __Pyx_NewRef(obj) : PyObject_Str(obj)) diff --git a/Cython/Utility/TestCythonScope.pyx b/Cython/Utility/TestCythonScope.pyx index f585be298..7da7665c3 100644 --- a/Cython/Utility/TestCythonScope.pyx +++ b/Cython/Utility/TestCythonScope.pyx @@ -1,6 +1,11 @@ ########## TestClass ########## # These utilities are for testing purposes +# The "cythonscope" test calls METH_O functions with their (self, arg) signature. +# cython: always_allow_keywords=False + +from __future__ import print_function + cdef extern from *: cdef object __pyx_test_dep(object) @@ -12,32 +17,32 @@ cdef class TestClass(object): self.value = value def __str__(self): - return 'TestClass(%d)' % self.value + return f'TestClass({self.value})' cdef cdef_method(self, int value): - print 'Hello from cdef_method', value + print('Hello from cdef_method', value) cpdef cpdef_method(self, int value): - print 'Hello from cpdef_method', value + print('Hello from cpdef_method', value) def def_method(self, int value): - print 'Hello from def_method', value + print('Hello from def_method', value) @cname('cdef_cname') cdef cdef_cname_method(self, int value): - print "Hello from cdef_cname_method", value + print("Hello from cdef_cname_method", value) @cname('cpdef_cname') cpdef cpdef_cname_method(self, int value): - print "Hello from cpdef_cname_method", value + print("Hello from cpdef_cname_method", value) @cname('def_cname') def def_cname_method(self, int value): - print "Hello from def_cname_method", value + print("Hello from def_cname_method", value) @cname('__pyx_test_call_other_cy_util') cdef test_call(obj): - print 'test_call' + print('test_call') __pyx_test_dep(obj) @cname('__pyx_TestClass_New') @@ -46,19 +51,20 @@ cdef _testclass_new(int value): ########### TestDep ########## +from __future__ import print_function + @cname('__pyx_test_dep') cdef test_dep(obj): - print 'test_dep', obj + print('test_dep', obj) ########## TestScope ########## @cname('__pyx_testscope') cdef object _testscope(int value): - return "hello from cython scope, value=%d" % value + return f"hello from cython scope, value={value}" ########## View.TestScope ########## @cname('__pyx_view_testscope') cdef object _testscope(int value): - return "hello from cython.view scope, value=%d" % value - + return f"hello from cython.view scope, value={value}" diff --git a/Cython/Utility/TypeConversion.c b/Cython/Utility/TypeConversion.c index 7ccb67263..444788071 100644 --- a/Cython/Utility/TypeConversion.c +++ b/Cython/Utility/TypeConversion.c @@ -68,9 +68,9 @@ static CYTHON_INLINE PyObject* __Pyx_PyUnicode_FromString(const char*); #define __Pyx_PyBytes_AsString(s) ((const char*) PyBytes_AS_STRING(s)) #define __Pyx_PyBytes_AsSString(s) ((const signed char*) PyBytes_AS_STRING(s)) #define __Pyx_PyBytes_AsUString(s) ((const unsigned char*) PyBytes_AS_STRING(s)) -#define __Pyx_PyObject_AsWritableString(s) ((char*) __Pyx_PyObject_AsString(s)) -#define __Pyx_PyObject_AsWritableSString(s) ((signed char*) __Pyx_PyObject_AsString(s)) -#define __Pyx_PyObject_AsWritableUString(s) ((unsigned char*) __Pyx_PyObject_AsString(s)) +#define __Pyx_PyObject_AsWritableString(s) ((char*)(__pyx_uintptr_t) __Pyx_PyObject_AsString(s)) +#define __Pyx_PyObject_AsWritableSString(s) ((signed char*)(__pyx_uintptr_t) __Pyx_PyObject_AsString(s)) +#define __Pyx_PyObject_AsWritableUString(s) ((unsigned char*)(__pyx_uintptr_t) __Pyx_PyObject_AsString(s)) #define __Pyx_PyObject_AsSString(s) ((const signed char*) __Pyx_PyObject_AsString(s)) #define __Pyx_PyObject_AsUString(s) ((const unsigned char*) __Pyx_PyObject_AsString(s)) #define __Pyx_PyObject_FromCString(s) __Pyx_PyObject_FromString((const char*)s) @@ -80,11 +80,21 @@ static CYTHON_INLINE PyObject* __Pyx_PyUnicode_FromString(const char*); #define __Pyx_PyUnicode_FromCString(s) __Pyx_PyUnicode_FromString((const char*)s) // There used to be a Py_UNICODE_strlen() in CPython 3.x, but it is deprecated since Py3.3. -static CYTHON_INLINE size_t __Pyx_Py_UNICODE_strlen(const Py_UNICODE *u) { +#if CYTHON_COMPILING_IN_LIMITED_API +static CYTHON_INLINE size_t __Pyx_Py_UNICODE_strlen(const wchar_t *u) +{ + const wchar_t *u_end = u; + while (*u_end++) ; + return (size_t)(u_end - u - 1); +} +#else +static CYTHON_INLINE size_t __Pyx_Py_UNICODE_strlen(const Py_UNICODE *u) +{ const Py_UNICODE *u_end = u; while (*u_end++) ; return (size_t)(u_end - u - 1); } +#endif #define __Pyx_PyUnicode_FromUnicode(u) PyUnicode_FromUnicode(u, __Pyx_Py_UNICODE_strlen(u)) #define __Pyx_PyUnicode_FromUnicodeAndLength PyUnicode_FromUnicode @@ -102,6 +112,7 @@ static CYTHON_INLINE PyObject* __Pyx_PyNumber_IntOrLong(PyObject* x); static CYTHON_INLINE Py_ssize_t __Pyx_PyIndex_AsSsize_t(PyObject*); static CYTHON_INLINE PyObject * __Pyx_PyInt_FromSize_t(size_t); +static CYTHON_INLINE Py_hash_t __Pyx_PyIndex_AsHash_t(PyObject*); #if CYTHON_ASSUME_SAFE_MACROS #define __pyx_PyFloat_AsDouble(x) (PyFloat_CheckExact(x) ? PyFloat_AS_DOUBLE(x) : PyFloat_AsDouble(x)) @@ -115,7 +126,8 @@ static CYTHON_INLINE PyObject * __Pyx_PyInt_FromSize_t(size_t); #else #define __Pyx_PyNumber_Int(x) (PyInt_CheckExact(x) ? __Pyx_NewRef(x) : PyNumber_Int(x)) #endif -#define __Pyx_PyNumber_Float(x) (PyFloat_CheckExact(x) ? __Pyx_NewRef(x) : PyNumber_Float(x)) +// __Pyx_PyNumber_Float is now in it's own section since it has dependencies (needed to make +// string conversion work the same in all circumstances) #if PY_MAJOR_VERSION < 3 && __PYX_DEFAULT_STRING_ENCODING_IS_ASCII static int __Pyx_sys_getdefaultencoding_not_ascii; @@ -270,7 +282,7 @@ static CYTHON_INLINE const char* __Pyx_PyObject_AsStringAndSize(PyObject* o, Py_ } else #endif /* __PYX_DEFAULT_STRING_ENCODING_IS_ASCII || __PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT */ -#if (!CYTHON_COMPILING_IN_PYPY) || (defined(PyByteArray_AS_STRING) && defined(PyByteArray_GET_SIZE)) +#if (!CYTHON_COMPILING_IN_PYPY && !CYTHON_COMPILING_IN_LIMITED_API) || (defined(PyByteArray_AS_STRING) && defined(PyByteArray_GET_SIZE)) if (PyByteArray_Check(o)) { *length = PyByteArray_GET_SIZE(o); return PyByteArray_AS_STRING(o); @@ -303,23 +315,27 @@ static CYTHON_INLINE int __Pyx_PyObject_IsTrueAndDecref(PyObject* x) { } static PyObject* __Pyx_PyNumber_IntOrLongWrongResultType(PyObject* result, const char* type_name) { + __Pyx_TypeName result_type_name = __Pyx_PyType_GetName(Py_TYPE(result)); #if PY_MAJOR_VERSION >= 3 if (PyLong_Check(result)) { // CPython issue #17576: warn if 'result' not of exact type int. if (PyErr_WarnFormat(PyExc_DeprecationWarning, 1, - "__int__ returned non-int (type %.200s). " - "The ability to return an instance of a strict subclass of int " - "is deprecated, and may be removed in a future version of Python.", - Py_TYPE(result)->tp_name)) { + "__int__ returned non-int (type " __Pyx_FMT_TYPENAME "). " + "The ability to return an instance of a strict subclass of int is deprecated, " + "and may be removed in a future version of Python.", + result_type_name)) { + __Pyx_DECREF_TypeName(result_type_name); Py_DECREF(result); return NULL; } + __Pyx_DECREF_TypeName(result_type_name); return result; } #endif PyErr_Format(PyExc_TypeError, - "__%.4s__ returned non-%.4s (type %.200s)", - type_name, type_name, Py_TYPE(result)->tp_name); + "__%.4s__ returned non-%.4s (type " __Pyx_FMT_TYPENAME ")", + type_name, type_name, result_type_name); + __Pyx_DECREF_TypeName(result_type_name); Py_DECREF(result); return NULL; } @@ -420,6 +436,25 @@ static CYTHON_INLINE Py_ssize_t __Pyx_PyIndex_AsSsize_t(PyObject* b) { } +static CYTHON_INLINE Py_hash_t __Pyx_PyIndex_AsHash_t(PyObject* o) { + if (sizeof(Py_hash_t) == sizeof(Py_ssize_t)) { + return (Py_hash_t) __Pyx_PyIndex_AsSsize_t(o); +#if PY_MAJOR_VERSION < 3 + } else if (likely(PyInt_CheckExact(o))) { + return PyInt_AS_LONG(o); +#endif + } else { + Py_ssize_t ival; + PyObject *x; + x = PyNumber_Index(o); + if (!x) return -1; + ival = PyInt_AsLong(x); + Py_DECREF(x); + return ival; + } +} + + static CYTHON_INLINE PyObject * __Pyx_PyBool_FromLong(long b) { return b ? __Pyx_NewRef(Py_True) : __Pyx_NewRef(Py_False); } @@ -429,6 +464,54 @@ static CYTHON_INLINE PyObject * __Pyx_PyInt_FromSize_t(size_t ival) { return PyInt_FromSize_t(ival); } +/////////////// pynumber_float.proto /////////////// + +static CYTHON_INLINE PyObject* __Pyx__PyNumber_Float(PyObject* obj); /* proto */ +#define __Pyx_PyNumber_Float(x) (PyFloat_CheckExact(x) ? __Pyx_NewRef(x) : __Pyx__PyNumber_Float(x)) + +/////////////// pynumber_float /////////////// +//@requires: Optimize.c::pybytes_as_double +//@requires: Optimize.c::pyunicode_as_double + +static CYTHON_INLINE PyObject* __Pyx__PyNumber_Float(PyObject* obj) { + // 'obj is PyFloat' is handled in the calling macro + double val; + if (PyLong_CheckExact(obj)) { +#if CYTHON_USE_PYLONG_INTERNALS + const digit* digits = ((PyLongObject*)obj)->ob_digit; + switch (Py_SIZE(obj)) { + case 0: + val = 0.0; + goto no_error; + // single digit PyLong values always cast safely to double + case 1: + val = (double) digits[0]; + goto no_error; + case -1: + val = (double) - (sdigit) digits[0]; + goto no_error; + default: + val = PyLong_AsDouble(obj); + } +#else + val = PyLong_AsDouble(obj); +#endif + } else if (PyUnicode_CheckExact(obj)) { + val = __Pyx_PyUnicode_AsDouble(obj); + } else if (PyBytes_CheckExact(obj)) { + val = __Pyx_PyBytes_AsDouble(obj); + } else if (PyByteArray_CheckExact(obj)) { + val = __Pyx_PyByteArray_AsDouble(obj); + } else { + return PyNumber_Float(obj); + } + + if (unlikely(val == -1 && PyErr_Occurred())) { + return NULL; + } +no_error: + return PyFloat_FromDouble(val); +} /////////////// GCCDiagnostics.proto /////////////// @@ -471,7 +554,10 @@ static {{struct_type_decl}} {{funcname}}(PyObject * o) { {{struct_type_decl}} result; if (!PyTuple_Check(o) || PyTuple_GET_SIZE(o) != {{size}}) { - PyErr_Format(PyExc_TypeError, "Expected %.16s of size %d, got %.200s", "a tuple", {{size}}, Py_TYPE(o)->tp_name); + __Pyx_TypeName o_type_name = __Pyx_PyType_GetName(Py_TYPE(o)); + PyErr_Format(PyExc_TypeError, + "Expected a tuple of size %d, got " __Pyx_FMT_TYPENAME, {{size}}, o_type_name); + __Pyx_DECREF_TypeName(o_type_name); goto bad; } @@ -764,10 +850,10 @@ static CYTHON_INLINE PyObject* {{TO_PY_FUNCTION}}({{TYPE}} value, Py_ssize_t wid } } while (unlikely(remaining != 0)); - if (last_one_off) { - assert(*dpos == '0'); - dpos++; - } + // Correct dpos by 1 if we read an excess digit. + assert(!last_one_off || *dpos == '0'); + dpos += last_one_off; + length = end - dpos; ulength = length; prepend_sign = 0; @@ -866,7 +952,7 @@ static CYTHON_INLINE {{TYPE}} {{FROM_PY_FUNCTION}}(PyObject *x) { const int is_unsigned = neg_one > const_zero; #if PY_MAJOR_VERSION < 3 if (likely(PyInt_Check(x))) { - if (sizeof({{TYPE}}) < sizeof(long)) { + if ((sizeof({{TYPE}}) < sizeof(long))) { __PYX_VERIFY_RETURN_INT({{TYPE}}, long, PyInt_AS_LONG(x)) } else { long val = PyInt_AS_LONG(x); @@ -886,10 +972,10 @@ static CYTHON_INLINE {{TYPE}} {{FROM_PY_FUNCTION}}(PyObject *x) { case 1: __PYX_VERIFY_RETURN_INT({{TYPE}}, digit, digits[0]) {{for _size in (2, 3, 4)}} case {{_size}}: - if (8 * sizeof({{TYPE}}) > {{_size-1}} * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > {{_size}} * PyLong_SHIFT) { + if ((8 * sizeof({{TYPE}}) > {{_size-1}} * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > {{_size}} * PyLong_SHIFT)) { __PYX_VERIFY_RETURN_INT({{TYPE}}, unsigned long, {{pylong_join(_size, 'digits')}}) - } else if (8 * sizeof({{TYPE}}) >= {{_size}} * PyLong_SHIFT) { + } else if ((8 * sizeof({{TYPE}}) >= {{_size}} * PyLong_SHIFT)) { return ({{TYPE}}) {{pylong_join(_size, 'digits', TYPE)}}; } } @@ -911,10 +997,10 @@ static CYTHON_INLINE {{TYPE}} {{FROM_PY_FUNCTION}}(PyObject *x) { goto raise_neg_overflow; } #endif - if (sizeof({{TYPE}}) <= sizeof(unsigned long)) { + if ((sizeof({{TYPE}}) <= sizeof(unsigned long))) { __PYX_VERIFY_RETURN_INT_EXC({{TYPE}}, unsigned long, PyLong_AsUnsignedLong(x)) #ifdef HAVE_LONG_LONG - } else if (sizeof({{TYPE}}) <= sizeof(unsigned PY_LONG_LONG)) { + } else if ((sizeof({{TYPE}}) <= sizeof(unsigned PY_LONG_LONG))) { __PYX_VERIFY_RETURN_INT_EXC({{TYPE}}, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) #endif } @@ -929,10 +1015,10 @@ static CYTHON_INLINE {{TYPE}} {{FROM_PY_FUNCTION}}(PyObject *x) { {{for _size in (2, 3, 4)}} {{for _case in (-_size, _size)}} case {{_case}}: - if (8 * sizeof({{TYPE}}){{' - 1' if _case < 0 else ''}} > {{_size-1}} * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > {{_size}} * PyLong_SHIFT) { + if ((8 * sizeof({{TYPE}}){{' - 1' if _case < 0 else ''}} > {{_size-1}} * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > {{_size}} * PyLong_SHIFT)) { __PYX_VERIFY_RETURN_INT({{TYPE}}, {{'long' if _case < 0 else 'unsigned long'}}, {{'-(long) ' if _case < 0 else ''}}{{pylong_join(_size, 'digits')}}) - } else if (8 * sizeof({{TYPE}}) - 1 > {{_size}} * PyLong_SHIFT) { + } else if ((8 * sizeof({{TYPE}}) - 1 > {{_size}} * PyLong_SHIFT)) { return ({{TYPE}}) ({{'((%s)-1)*' % TYPE if _case < 0 else ''}}{{pylong_join(_size, 'digits', TYPE)}}); } } @@ -941,18 +1027,18 @@ static CYTHON_INLINE {{TYPE}} {{FROM_PY_FUNCTION}}(PyObject *x) { {{endfor}} } #endif - if (sizeof({{TYPE}}) <= sizeof(long)) { + if ((sizeof({{TYPE}}) <= sizeof(long))) { __PYX_VERIFY_RETURN_INT_EXC({{TYPE}}, long, PyLong_AsLong(x)) #ifdef HAVE_LONG_LONG - } else if (sizeof({{TYPE}}) <= sizeof(PY_LONG_LONG)) { + } else if ((sizeof({{TYPE}}) <= sizeof(PY_LONG_LONG))) { __PYX_VERIFY_RETURN_INT_EXC({{TYPE}}, PY_LONG_LONG, PyLong_AsLongLong(x)) #endif } } { -#if CYTHON_COMPILING_IN_PYPY && !defined(_PyLong_AsByteArray) +#if (CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_LIMITED_API) && !defined(_PyLong_AsByteArray) PyErr_SetString(PyExc_RuntimeError, - "_PyLong_AsByteArray() not available in PyPy, cannot convert large numbers"); + "_PyLong_AsByteArray() not available, cannot convert large numbers"); #else {{TYPE}} val; PyObject *v = __Pyx_PyNumber_IntOrLong(x); diff --git a/Cython/Utils.py b/Cython/Utils.py index d59d67d78..e6c98f583 100644 --- a/Cython/Utils.py +++ b/Cython/Utils.py @@ -1,7 +1,7 @@ -# -# Cython -- Things that don't belong -# anywhere else in particular -# +""" +Cython -- Things that don't belong + anywhere else in particular +""" from __future__ import absolute_import @@ -20,31 +20,43 @@ import sys import re import io import codecs +import glob import shutil import tempfile from contextlib import contextmanager +from . import __version__ as cython_version + +PACKAGE_FILES = ("__init__.py", "__init__.pyc", "__init__.pyx", "__init__.pxd") + modification_time = os.path.getmtime _function_caches = [] + + def clear_function_caches(): for cache in _function_caches: cache.clear() + def cached_function(f): cache = {} _function_caches.append(cache) uncomputed = object() + def wrapper(*args): res = cache.get(args, uncomputed) if res is uncomputed: res = cache[args] = f(*args) return res + wrapper.uncached = f return wrapper + def cached_method(f): cache_name = '__%s_cache' % f.__name__ + def wrapper(self, *args): cache = getattr(self, cache_name, None) if cache is None: @@ -54,8 +66,10 @@ def cached_method(f): return cache[args] res = cache[args] = f(self, *args) return res + return wrapper + def replace_suffix(path, newsuf): base, _ = os.path.splitext(path) return base + newsuf @@ -81,6 +95,9 @@ def castrate_file(path, st): # failed compilation. # Also sets access and modification times back to # those specified by st (a stat struct). + if not is_cython_generated_file(path, allow_failed=True, if_not_found=False): + return + try: f = open_new_file(path) except EnvironmentError: @@ -92,6 +109,31 @@ def castrate_file(path, st): if st: os.utime(path, (st.st_atime, st.st_mtime-1)) + +def is_cython_generated_file(path, allow_failed=False, if_not_found=True): + failure_marker = b"#error Do not use this file, it is the result of a failed Cython compilation." + file_content = None + if os.path.exists(path): + try: + with open(path, "rb") as f: + file_content = f.read(len(failure_marker)) + except (OSError, IOError): + pass # Probably just doesn't exist any more + + if file_content is None: + # file does not exist (yet) + return if_not_found + + return ( + # Cython C file? + file_content.startswith(b"/* Generated by Cython ") or + # Cython output file after previous failures? + (allow_failed and file_content == failure_marker) or + # Let's allow overwriting empty files as well. They might have resulted from previous failures. + not file_content + ) + + def file_newer_than(path, time): ftime = modification_time(path) return ftime > time @@ -134,24 +176,31 @@ def find_root_package_dir(file_path): else: return dir + @cached_function -def check_package_dir(dir, package_names): +def check_package_dir(dir_path, package_names): + namespace = True for dirname in package_names: - dir = os.path.join(dir, dirname) - if not is_package_dir(dir): - return None - return dir + dir_path = os.path.join(dir_path, dirname) + has_init = contains_init(dir_path) + if has_init: + namespace = False + return dir_path, namespace + @cached_function -def is_package_dir(dir_path): - for filename in ("__init__.py", - "__init__.pyc", - "__init__.pyx", - "__init__.pxd"): +def contains_init(dir_path): + for filename in PACKAGE_FILES: path = os.path.join(dir_path, filename) if path_exists(path): return 1 + +def is_package_dir(dir_path): + if contains_init(dir_path): + return 1 + + @cached_function def path_exists(path): # try on the filesystem first @@ -176,6 +225,40 @@ def path_exists(path): pass return False + +_parse_file_version = re.compile(r".*[.]cython-([0-9]+)[.][^./\\]+$").findall + + +@cached_function +def find_versioned_file(directory, filename, suffix, + _current_version=int(re.sub(r"^([0-9]+)[.]([0-9]+).*", r"\1\2", cython_version))): + """ + Search a directory for versioned pxd files, e.g. "lib.cython-30.pxd" for a Cython 3.0+ version. + + @param directory: the directory to search + @param filename: the filename without suffix + @param suffix: the filename extension including the dot, e.g. ".pxd" + @return: the file path if found, or None + """ + assert not suffix or suffix[:1] == '.' + path_prefix = os.path.join(directory, filename) + + matching_files = glob.glob(path_prefix + ".cython-*" + suffix) + path = path_prefix + suffix + if not os.path.exists(path): + path = None + best_match = (-1, path) # last resort, if we do not have versioned .pxd files + + for path in matching_files: + versions = _parse_file_version(path) + if versions: + int_version = int(versions[0]) + # Let's assume no duplicates. + if best_match[0] < int_version <= _current_version: + best_match = (int_version, path) + return best_match[1] + + # file name encodings def decode_filename(filename): @@ -189,12 +272,13 @@ def decode_filename(filename): pass return filename + # support for source file encoding detection _match_file_encoding = re.compile(br"(\w*coding)[:=]\s*([-\w.]+)").search -def detect_opened_file_encoding(f): +def detect_opened_file_encoding(f, default='UTF-8'): # PEPs 263 and 3120 # Most of the time the first two lines fall in the first couple of hundred chars, # and this bulk read/split is much faster. @@ -206,6 +290,7 @@ def detect_opened_file_encoding(f): lines = start.split(b"\n") if not data: break + m = _match_file_encoding(lines[0]) if m and m.group(1) != b'c_string_encoding': return m.group(2).decode('iso8859-1') @@ -213,7 +298,7 @@ def detect_opened_file_encoding(f): m = _match_file_encoding(lines[1]) if m: return m.group(2).decode('iso8859-1') - return "UTF-8" + return default def skip_bom(f): @@ -358,6 +443,37 @@ def captured_fd(stream=2, encoding=None): os.close(orig_stream) +def get_encoding_candidates(): + candidates = [sys.getdefaultencoding()] + for stream in (sys.stdout, sys.stdin, sys.__stdout__, sys.__stdin__): + encoding = getattr(stream, 'encoding', None) + # encoding might be None (e.g. somebody redirects stdout): + if encoding is not None and encoding not in candidates: + candidates.append(encoding) + return candidates + + +def prepare_captured(captured): + captured_bytes = captured.strip() + if not captured_bytes: + return None + for encoding in get_encoding_candidates(): + try: + return captured_bytes.decode(encoding) + except UnicodeDecodeError: + pass + # last resort: print at least the readable ascii parts correctly. + return captured_bytes.decode('latin-1') + + +def print_captured(captured, output, header_line=None): + captured = prepare_captured(captured) + if captured: + if header_line: + output.write(header_line) + output.write(captured) + + def print_bytes(s, header_text=None, end=b'\n', file=sys.stdout, flush=True): if header_text: file.write(header_text) # note: text! => file.write() instead of out.write() @@ -372,33 +488,41 @@ def print_bytes(s, header_text=None, end=b'\n', file=sys.stdout, flush=True): if flush: out.flush() + class LazyStr: def __init__(self, callback): self.callback = callback + def __str__(self): return self.callback() + def __repr__(self): return self.callback() + def __add__(self, right): return self.callback() + right + def __radd__(self, left): return left + self.callback() class OrderedSet(object): - def __init__(self, elements=()): - self._list = [] - self._set = set() - self.update(elements) - def __iter__(self): - return iter(self._list) - def update(self, elements): - for e in elements: - self.add(e) - def add(self, e): - if e not in self._set: - self._list.append(e) - self._set.add(e) + def __init__(self, elements=()): + self._list = [] + self._set = set() + self.update(elements) + + def __iter__(self): + return iter(self._list) + + def update(self, elements): + for e in elements: + self.add(e) + + def add(self, e): + if e not in self._set: + self._list.append(e) + self._set.add(e) # Class decorator that adds a metaclass and recreates the class with it. @@ -420,7 +544,7 @@ def add_metaclass(metaclass): def raise_error_if_module_name_forbidden(full_module_name): - #it is bad idea to call the pyx-file cython.pyx, so fail early + # it is bad idea to call the pyx-file cython.pyx, so fail early if full_module_name == 'cython' or full_module_name.startswith('cython.'): raise ValueError('cython is a special module, cannot be used as a module name') diff --git a/Demos/benchmarks/bpnn3.py b/Demos/benchmarks/bpnn3.py index 362bc2703..8498bd898 100644 --- a/Demos/benchmarks/bpnn3.py +++ b/Demos/benchmarks/bpnn3.py @@ -1,7 +1,7 @@ #!/usr/bin/python # Back-Propagation Neural Networks # -# Written in Python. See http://www.python.org/ +# Written in Python. See https://www.python.org/ # # Neil Schemenauer <nascheme@enme.ucalgary.ca> @@ -29,7 +29,7 @@ class NN(object): # print 'class NN' def __init__(self, ni, nh, no): # number of input, hidden, and output nodes - self.ni = ni + 1 # +1 for bias node + self.ni = ni + 1 # +1 for bias node self.nh = nh self.no = no @@ -67,7 +67,7 @@ class NN(object): for j in range(self.nh): sum = 0.0 for i in range(self.ni): - sum = sum + self.ai[i] * self.wi[i][j] + sum = sum + self.ai[i] * self.wi[i][j] self.ah[j] = 1.0/(1.0+math.exp(-sum)) # output activations diff --git a/Demos/benchmarks/chaos.py b/Demos/benchmarks/chaos.py index 36bd1bcd3..d770ff41c 100644 --- a/Demos/benchmarks/chaos.py +++ b/Demos/benchmarks/chaos.py @@ -130,7 +130,7 @@ class Spline(object): I = ii break else: - I = dom[1] - 1 + I = dom[1] - 1 return I def __len__(self): diff --git a/Demos/benchmarks/meteor_contest.py b/Demos/benchmarks/meteor_contest.py index 7eb6ca299..7610d5c08 100644 --- a/Demos/benchmarks/meteor_contest.py +++ b/Demos/benchmarks/meteor_contest.py @@ -64,7 +64,7 @@ def get_senh(board, cti): def get_puzzle(w=w, h=h): - board = [E*x + S*y + (y%2) for y in range(h) for x in range(w)] + board = [E*x + S*y + (y % 2) for y in range(h) for x in range(w)] cti = dict((board[i], i) for i in range(len(board))) idos = [[E, E, E, SE], # incremental direction offsets @@ -152,4 +152,3 @@ if __name__ == "__main__": options, args = parser.parse_args() util.run_benchmark(options, options.num_runs, main) - diff --git a/Demos/benchmarks/nqueens.py b/Demos/benchmarks/nqueens.py index 87e63df11..918f9e589 100644 --- a/Demos/benchmarks/nqueens.py +++ b/Demos/benchmarks/nqueens.py @@ -43,7 +43,7 @@ def permutations(iterable): else: return -# From http://code.activestate.com/recipes/576647/ +# From https://code.activestate.com/recipes/576647/ @cython.locals(queen_count=int, i=int, vec=list) def n_queens(queen_count): """N-Queens solver. diff --git a/Demos/benchmarks/richards.py b/Demos/benchmarks/richards.py index 913ec5daf..76a92c3a0 100644 --- a/Demos/benchmarks/richards.py +++ b/Demos/benchmarks/richards.py @@ -333,7 +333,7 @@ class WorkTask(Task): pkt.ident = dest pkt.datum = 0 - for i in BUFSIZE_RANGE: # range(BUFSIZE) + for i in BUFSIZE_RANGE: # range(BUFSIZE) w.count += 1 if w.count > 26: w.count = 1 @@ -382,9 +382,9 @@ class Richards(object): wkq = Packet(wkq , I_DEVB, K_DEV) HandlerTask(I_HANDLERB, 3000, wkq, TaskState().waitingWithPacket(), HandlerTaskRec()) - wkq = None; - DeviceTask(I_DEVA, 4000, wkq, TaskState().waiting(), DeviceTaskRec()); - DeviceTask(I_DEVB, 5000, wkq, TaskState().waiting(), DeviceTaskRec()); + wkq = None + DeviceTask(I_DEVA, 4000, wkq, TaskState().waiting(), DeviceTaskRec()) + DeviceTask(I_DEVB, 5000, wkq, TaskState().waiting(), DeviceTaskRec()) schedule() diff --git a/Demos/benchmarks/spectralnorm.py b/Demos/benchmarks/spectralnorm.py index 7b56b05f6..f476c6bcd 100644 --- a/Demos/benchmarks/spectralnorm.py +++ b/Demos/benchmarks/spectralnorm.py @@ -11,28 +11,28 @@ from time import time import util import optparse -def eval_A (i, j): +def eval_A(i, j): return 1.0 / ((i + j) * (i + j + 1) / 2 + i + 1) -def eval_A_times_u (u): +def eval_A_times_u(u): return [ part_A_times_u(i,u) for i in range(len(u)) ] -def eval_At_times_u (u): +def eval_At_times_u(u): return [ part_At_times_u(i,u) for i in range(len(u)) ] -def eval_AtA_times_u (u): - return eval_At_times_u (eval_A_times_u (u)) +def eval_AtA_times_u(u): + return eval_At_times_u(eval_A_times_u(u)) def part_A_times_u(i, u): partial_sum = 0 for j, u_j in enumerate(u): - partial_sum += eval_A (i, j) * u_j + partial_sum += eval_A(i, j) * u_j return partial_sum def part_At_times_u(i, u): partial_sum = 0 for j, u_j in enumerate(u): - partial_sum += eval_A (j, i) * u_j + partial_sum += eval_A(j, i) * u_j return partial_sum DEFAULT_N = 130 @@ -43,13 +43,13 @@ def main(n): t0 = time() u = [1] * DEFAULT_N - for dummy in range (10): - v = eval_AtA_times_u (u) - u = eval_AtA_times_u (v) + for dummy in range(10): + v = eval_AtA_times_u(u) + u = eval_AtA_times_u(v) vBv = vv = 0 - for ue, ve in zip (u, v): + for ue, ve in zip(u, v): vBv += ue * ve vv += ve * ve tk = time() diff --git a/Demos/callback/run_cheese.py b/Demos/callback/run_cheese.py index e04910e7e..f0feb3c30 100644 --- a/Demos/callback/run_cheese.py +++ b/Demos/callback/run_cheese.py @@ -4,5 +4,3 @@ def report_cheese(name): print("Found cheese: " + name) cheese.find(report_cheese) - - diff --git a/Demos/freeze/README.rst b/Demos/freeze/README.rst index 31c4b4c12..20383ef28 100644 --- a/Demos/freeze/README.rst +++ b/Demos/freeze/README.rst @@ -106,6 +106,6 @@ Cython 0.11.2 (or newer, assuming the API does not change) SEE ALSO ======== -* `Python <http://www.python.org>`_ +* `Python <https://www.python.org/>`_ * `Cython <http://www.cython.org>`_ -* `freeze.py <http://wiki.python.org/moin/Freeze>`_ +* `freeze.py <https://wiki.python.org/moin/Freeze>`_ diff --git a/Demos/pyprimes.py b/Demos/pyprimes.py index 7c5242244..5725a8e29 100644 --- a/Demos/pyprimes.py +++ b/Demos/pyprimes.py @@ -5,9 +5,9 @@ def primes(kmax): while k < kmax: i = 0 while i < k and n % p[i] != 0: - i = i + 1 + i += 1 if i == k: p.append(n) - k = k + 1 - n = n + 1 + k += 1 + n += 1 return p diff --git a/Doc/s5/cython-ep2008.txt b/Doc/s5/cython-ep2008.txt index a29ca35d6..aeb56ae66 100644 --- a/Doc/s5/cython-ep2008.txt +++ b/Doc/s5/cython-ep2008.txt @@ -51,7 +51,7 @@ Cython is * an Open-Source project - * http://cython.org + * https://cython.org/ * a Python compiler (almost) @@ -115,7 +115,7 @@ Major Cython Developers * many, *many* others - see - * http://cython.org/ + * https://cython.org/ * the mailing list archives of Cython and Pyrex @@ -398,4 +398,4 @@ Cython \... use it, and join the project! - http://cython.org/ + https://cython.org/ diff --git a/Doc/s5/ui/default/cython-logo64.png b/Doc/s5/ui/default/cython-logo64.png Binary files differindex 7ff4f6e92..c24c9e6d5 100644 --- a/Doc/s5/ui/default/cython-logo64.png +++ b/Doc/s5/ui/default/cython-logo64.png diff --git a/Doc/s5/ui/default/iepngfix.htc b/Doc/s5/ui/default/iepngfix.htc index 4d90c87a9..5fd31ff9e 100644 --- a/Doc/s5/ui/default/iepngfix.htc +++ b/Doc/s5/ui/default/iepngfix.htc @@ -3,7 +3,7 @@ <script>
-// IE5.5+ PNG Alpha Fix v1.0 by Angus Turnbull http://www.twinhelix.com
+// IE5.5+ PNG Alpha Fix v1.0 by Angus Turnbull https://www.twinhelix.com/
// Free usage permitted as long as this notice remains intact.
// This must be a path to a blank image. That's all the configuration you need here.
diff --git a/Doc/s5/ui/default/slides.js b/Doc/s5/ui/default/slides.js index 452203586..6f7d08a25 100644 --- a/Doc/s5/ui/default/slides.js +++ b/Doc/s5/ui/default/slides.js @@ -1,6 +1,6 @@ // S5 v1.1 slides.js -- released into the Public Domain // -// Please see http://www.meyerweb.com/eric/tools/s5/credits.html for information +// Please see https://meyerweb.com/eric/tools/s5/credits.html for information // about all the wonderful and talented contributors to this code! var undef; diff --git a/LICENSE.txt b/LICENSE.txt index d9a10c0d8..4f6f63985 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -1,6 +1,6 @@ Apache License Version 2.0, January 2004 - http://www.apache.org/licenses/ + https://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION diff --git a/MANIFEST.in b/MANIFEST.in index c25865da9..9b26a2632 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -5,6 +5,7 @@ include pylintrc include setup.py include setupegg.py include bin/* +include *requirements*.txt include cython.py cythonize.py cygdb.py recursive-include Cython *.pyx *.pxd include Cython/Parser/Grammar Cython/Parser/__init__.py @@ -3,14 +3,20 @@ PYTHON?=python TESTOPTS?= REPO = git://github.com/cython/cython.git VERSION?=$(shell sed -ne 's|^__version__\s*=\s*"\([^"]*\)".*|\1|p' Cython/Shadow.py) +PARALLEL?=$(shell ${PYTHON} -c 'import sys; print("-j5" if sys.version_info >= (3,5) else "")' || true) -MANYLINUX_IMAGE_X86_64=quay.io/pypa/manylinux1_x86_64 -MANYLINUX_IMAGE_686=quay.io/pypa/manylinux1_i686 +MANYLINUX1_IMAGE_X86_64=quay.io/pypa/manylinux1_x86_64 +MANYLINUX1_IMAGE_686=quay.io/pypa/manylinux1_i686 +MANYLINUX_IMAGE_X86_64=quay.io/pypa/manylinux_2_24_x86_64 +MANYLINUX_IMAGE_686=quay.io/pypa/manylinux_2_24_i686 all: local local: - ${PYTHON} setup.py build_ext --inplace + ${PYTHON} setup.py build_ext --inplace $(PARALLEL) + +plocal: + ${PYTHON} setup.py build_ext --inplace --cython-profile $(PARALLEL) sdist: dist/$(PACKAGENAME)-$(VERSION).tar.gz @@ -44,6 +50,7 @@ clean: @rm -f *.pyd */*.pyd */*/*.pyd @rm -f *~ */*~ */*/*~ @rm -f core */core + @rm -f Cython/*.c @rm -f Cython/Compiler/*.c @rm -f Cython/Plex/*.c @rm -f Cython/Tempita/*.c @@ -56,6 +63,9 @@ testclean: test: testclean ${PYTHON} runtests.py -vv ${TESTOPTS} +checks: + ${PYTHON} runtests.py -vv --no-unit --no-doctest --no-file --no-pyregr --no-examples + s5: $(MAKE) -C Doc/s5 slides @@ -64,14 +74,25 @@ wheel_manylinux: wheel_manylinux64 wheel_manylinux32 wheel_manylinux32 wheel_manylinux64: dist/$(PACKAGENAME)-$(VERSION).tar.gz echo "Building wheels for $(PACKAGENAME) $(VERSION)" mkdir -p wheelhouse_$(subst wheel_,,$@) - time docker run --rm -t \ - -v $(shell pwd):/io \ - -e CFLAGS="-O3 -g0 -mtune=generic -pipe -fPIC" \ - -e LDFLAGS="$(LDFLAGS) -fPIC" \ - -e WHEELHOUSE=wheelhouse_$(subst wheel_,,$@) \ - $(if $(patsubst %32,,$@),$(MANYLINUX_IMAGE_X86_64),$(MANYLINUX_IMAGE_686)) \ - bash -c 'for PYBIN in /opt/python/*/bin; do \ - $$PYBIN/python -V; \ - { $$PYBIN/pip wheel -w /io/$$WHEELHOUSE /io/$< & } ; \ - done; wait; \ - for whl in /io/$$WHEELHOUSE/$(PACKAGENAME)-$(VERSION)-*-linux_*.whl; do auditwheel repair $$whl -w /io/$$WHEELHOUSE; done' + for dockerimage in $(if $(patsubst %32,,$@),$(MANYLINUX1_IMAGE_X86_64) $(MANYLINUX_IMAGE_X86_64),$(MANYLINUX1_IMAGE_686) $(MANYLINUX_IMAGE_686)); do \ + time docker run --rm -t \ + -v $(shell pwd):/io \ + -e CFLAGS="-O3 -g0 -mtune=generic -pipe -fPIC" \ + -e LDFLAGS="$(LDFLAGS) -fPIC" \ + -e WHEELHOUSE=wheelhouse_$(subst wheel_,,$@) \ + "$$dockerimage" \ + bash -c '\ + rm -fr /opt/python/*pypy* ; \ + for cpdir in /opt/python/*27* ; do \ + if [ -d "$$cpdir" ]; \ + then rm -fr /opt/python/*3[78912]; \ + else rm -fr /opt/python/*{27*,3[456]*}; \ + fi; break; \ + done ; \ + ls /opt/python/ ; \ + for PYBIN in /opt/python/*/bin; do \ + $$PYBIN/python -V; \ + { $$PYBIN/pip wheel -w /io/$$WHEELHOUSE /io/$< & } ; \ + done; wait; \ + for whl in /io/$$WHEELHOUSE/$(PACKAGENAME)-$(VERSION)-*-linux_*.whl; do auditwheel repair $$whl -w /io/$$WHEELHOUSE; done' ; \ + done diff --git a/README.rst b/README.rst index fe4b5958a..0f56f5661 100644 --- a/README.rst +++ b/README.rst @@ -1,4 +1,4 @@ -Welcome to Cython! +Welcome to Cython! ================== Cython is a language that makes writing C extensions for @@ -14,20 +14,24 @@ code from Cython code. This makes Cython the ideal language for wrapping external C libraries, and for fast C modules that speed up the execution of Python code. -* Official website: http://cython.org/ -* Documentation: http://docs.cython.org/en/latest/ +* Official website: https://cython.org/ +* Documentation: https://docs.cython.org/ * Github repository: https://github.com/cython/cython * Wiki: https://github.com/cython/cython/wiki +You can **support the Cython project** via +`Github Sponsors <https://github.com/users/scoder/sponsorship>`_ or +`Tidelift <https://tidelift.com/subscription/pkg/pypi-cython>`_. + Installation: ------------- -If you already have a C compiler, just do:: +If you already have a C compiler, just run following command:: pip install Cython -otherwise, see `the installation page <http://docs.cython.org/en/latest/src/quickstart/install.html>`_. +otherwise, see `the installation page <https://docs.cython.org/en/latest/src/quickstart/install.html>`_. License: @@ -45,6 +49,77 @@ Contributing: Want to contribute to the Cython project? Here is some `help to get you started <https://github.com/cython/cython/blob/master/docs/CONTRIBUTING.rst>`_. +We are currently building the next great Cython edition: +`Cython 3.0 <https://github.com/cython/cython/milestone/58>`_. +You can help us make the life of Python 3.x users easier. + + +Differences to other Python compilers +------------------------------------- + +Started as a project in the early 2000s, Cython has outlived +`most other attempts <https://wiki.python.org/moin/PythonImplementations#Compilers>`_ +at producing static compilers for the Python language. + +Similar projects that have a relevance today include: + +* `PyPy <https://www.pypy.org/>`_, a Python implementation with a JIT compiler. + + * Pros: JIT compilation with runtime optimisations, fully language compliant, + good integration with external C/C++ code + * Cons: non-CPython runtime, relatively large resource usage of the runtime, + limited compatibility with CPython extensions, non-obvious performance results + +* `Numba <http://numba.pydata.org/>`_, a Python extension that features a + JIT compiler for a subset of the language, based on the LLVM compiler + infrastructure (probably best known for its ``clang`` C compiler). + It mostly targets numerical code that uses NumPy. + + * Pros: JIT compilation with runtime optimisations + * Cons: limited language support, relatively large runtime dependency (LLVM), + non-obvious performance results + +* `Pythran <https://pythran.readthedocs.io/>`_, a static Python-to-C++ + extension compiler for a subset of the language, mostly targeted + at numerical computation. Pythran can be (and is probably best) used + as an additional + `backend for NumPy code <https://cython.readthedocs.io/en/latest/src/userguide/numpy_pythran.html>`_ + in Cython. + +* `mypyc <https://mypyc.readthedocs.io/>`_, a static Python-to-C extension + compiler, based on the `mypy <http://www.mypy-lang.org/>`_ static Python + analyser. Like Cython's + `pure Python mode <https://cython.readthedocs.io/en/latest/src/tutorial/pure.html>`_, + mypyc can make use of PEP-484 type annotations to optimise code for static types. + + * Pros: good support for language and PEP-484 typing, good type inference, + reasonable performance gains + * Cons: no support for low-level optimisations and typing, + opinionated Python type interpretation, reduced Python compatibility + and introspection after compilation + +* `Nuitka <https://nuitka.net/>`_, a static Python-to-C extension compiler. + + * Pros: highly language compliant, reasonable performance gains, + support for static application linking (similar to + `cython_freeze <https://github.com/cython/cython/blob/master/bin/cython_freeze>`_) + * Cons: no support for low-level optimisations and typing + +In comparison to the above, Cython provides + +* fast, efficient and highly compliant support for almost all + Python language features, including dynamic features and introspection +* full runtime compatibility with all still-in-use and future versions + of CPython +* "generate once, compile everywhere" C code generation that allows for + reproducible performance results and testing +* C compile time adaptation to the target platform and Python version +* support for other C-API implementations, including PyPy and Pyston +* seamless integration with C/C++ code +* broad support for manual optimisation and tuning down to the C level +* a large user base with thousands of libraries, packages and tools +* almost two decades of bug fixing and static code optimisations + Get the full source history: ---------------------------- @@ -63,21 +138,21 @@ The following is from Pyrex: This is a development version of Pyrex, a language for writing Python extension modules. -For more info, see: +For more info, take a look at: * Doc/About.html for a description of the language * INSTALL.txt for installation instructions * USAGE.txt for usage instructions * Demos for usage examples -Comments, suggestions, bug reports, etc. are +Comments, suggestions, bug reports, etc. are most welcome! Copyright stuff: Pyrex is free of restrictions. You may use, redistribute, modify and distribute modified versions. -The latest version of Pyrex can be found `here <http://www.cosc.canterbury.ac.nz/~greg/python/Pyrex/>`_. +The latest version of Pyrex can be found `here <https://www.cosc.canterbury.ac.nz/~greg/python/Pyrex/>`_. | Greg Ewing, Computer Science Dept | University of Canterbury diff --git a/Tools/BUILD.bazel b/Tools/BUILD.bazel index e69de29bb..8b1378917 100644 --- a/Tools/BUILD.bazel +++ b/Tools/BUILD.bazel @@ -0,0 +1 @@ + diff --git a/Tools/ci-run.sh b/Tools/ci-run.sh new file mode 100644 index 000000000..e4f39eba9 --- /dev/null +++ b/Tools/ci-run.sh @@ -0,0 +1,130 @@ +#!/usr/bin/bash + +GCC_VERSION=${GCC_VERSION:=8} + +# Set up compilers +if [ "${OS_NAME##ubuntu*}" == "" -a "$TEST_CODE_STYLE" != "1" ]; then + echo "Installing requirements [apt]" + sudo apt-add-repository -y "ppa:ubuntu-toolchain-r/test" + sudo apt update -y -q + sudo apt install -y -q ccache gdb python-dbg python3-dbg gcc-$GCC_VERSION || exit 1 + if [ -z "${BACKEND##*cpp*}" ]; then + sudo apt install -y -q g++-$GCC_VERSION || exit 1 + fi + sudo /usr/sbin/update-ccache-symlinks + echo "/usr/lib/ccache" >> $GITHUB_PATH # export ccache to path + + sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-$GCC_VERSION 60 $(if [ -z "${BACKEND##*cpp*}" ]; then echo " --slave /usr/bin/g++ g++ /usr/bin/g++-$GCC_VERSION"; fi) + + export CC="gcc" + if [ -z "${BACKEND##*cpp*}" ]; then + sudo update-alternatives --set g++ /usr/bin/g++-$GCC_VERSION + export CXX="g++" + fi +fi +if [ "${OS_NAME##macos*}" == "" ]; then + export CC="clang -Wno-deprecated-declarations" + export CXX="clang++ -stdlib=libc++ -Wno-deprecated-declarations" +fi + +# Set up miniconda +if [ "$STACKLESS" == "true" ]; then + echo "Installing stackless python" + #conda install --quiet --yes nomkl --file=test-requirements.txt --file=test-requirements-cpython.txt + conda config --add channels stackless + conda install --quiet --yes stackless || exit 1 +fi + +# Log versions in use +echo "====================" +echo "|VERSIONS INSTALLED|" +echo "====================" +python -c 'import sys; print("Python %s" % (sys.version,))' +if [ "$CC" ]; then + which ${CC%% *} + ${CC%% *} --version +fi +if [ "$CXX" ]; then + which ${CXX%% *} + ${CXX%% *} --version +fi +echo "====================" + +# Install python requirements +echo "Installing requirements [python]" +if [ -z "${PYTHON_VERSION##2.7}" ]; then + pip install wheel || exit 1 + pip install -r test-requirements-27.txt || exit 1 +elif [ -z "${PYTHON_VERSION##3.[45]*}" ]; then + python -m pip install wheel || exit 1 + python -m pip install -r test-requirements-34.txt || exit 1 +else + python -m pip install -U pip setuptools wheel || exit 1 + + if [ -n "${PYTHON_VERSION##*-dev}" ]; then + python -m pip install -r test-requirements.txt || exit 1 + + if [ "${PYTHON_VERSION##pypy*}" -a "${PYTHON_VERSION##3.[4789]*}" ]; then + python -m pip install -r test-requirements-cpython.txt || exit 1 + fi + fi +fi + +if [ "$TEST_CODE_STYLE" == "1" ]; then + STYLE_ARGS="--no-unit --no-doctest --no-file --no-pyregr --no-examples"; + python -m pip install -r doc-requirements.txt || exit 1 +else + STYLE_ARGS="--no-code-style"; + + # Install more requirements + if [ -n "${PYTHON_VERSION##*-dev}" ]; then + if [ -z "${BACKEND##*cpp*}" ]; then + echo "WARNING: Currently not installing pythran due to compatibility issues" + # python -m pip install pythran==0.9.5 || exit 1 + fi + + if [ "$BACKEND" != "cpp" -a -n "${PYTHON_VERSION##pypy*}" -a -n "${PYTHON_VERSION##2*}" -a -n "${PYTHON_VERSION##*3.4}" ]; then + python -m pip install mypy || exit 1 + fi + fi +fi + +# Run tests +ccache -s 2>/dev/null || true +export PATH="/usr/lib/ccache:$PATH" + +if [ "$NO_CYTHON_COMPILE" != "1" -a -n "${PYTHON_VERSION##pypy*}" ]; then + CFLAGS="-O2 -ggdb -Wall -Wextra $(python -c 'import sys; print("-fno-strict-aliasing" if sys.version_info[0] == 2 else "")')" \ + python setup.py build_ext -i \ + $(if [ "$COVERAGE" == "1" ]; then echo " --cython-coverage"; fi) \ + $(if [ "$CYTHON_COMPILE_ALL" == "1" ]; then echo " --cython-compile-all"; fi) \ + $(python -c 'import sys; print("-j5" if sys.version_info >= (3,5) else "")') \ + || exit 1 + if [ -z "$COVERAGE" -a -z "$STACKLESS" -a -z "$LIMITED_API" -a -z "$CYTHON_COMPILE_ALL" -a -z "$EXTRA_CFLAGS" -a -n "${BACKEND//*cpp*}" ]; then + python setup.py bdist_wheel || exit 1 + fi +fi + +if [ "$TEST_CODE_STYLE" == "1" ]; then + make -C docs html || exit 1 +elif [ -n "${PYTHON_VERSION##pypy*}" ]; then + # Run the debugger tests in python-dbg if available (but don't fail, because they currently do fail) + PYTHON_DBG="python$( python -c 'import sys; print("%d.%d" % sys.version_info[:2])' )-dbg" + if $PYTHON_DBG -V >&2; then CFLAGS="-O0 -ggdb" $PYTHON_DBG runtests.py -vv --no-code-style Debugger --backends=$BACKEND; fi; +fi + +export CFLAGS="-O0 -ggdb -Wall -Wextra $EXTRA_CFLAGS" +python runtests.py \ + -vv $STYLE_ARGS \ + -x Debugger \ + --backends=$BACKEND \ + $LIMITED_API \ + $EXCLUDE \ + $(if [ "$COVERAGE" == "1" ]; then echo " --coverage --coverage-html --cython-only"; fi) \ + $(if [ -z "$TEST_CODE_STYLE" ]; then echo " -j7 "; fi) + +EXIT_CODE=$? + +ccache -s 2>/dev/null || true + +exit $EXIT_CODE diff --git a/Tools/cython-mode.el b/Tools/cython-mode.el index e4be99f5b..058dd7b20 100644 --- a/Tools/cython-mode.el +++ b/Tools/cython-mode.el @@ -103,7 +103,7 @@ (defgroup cython nil "Major mode for editing and compiling Cython files" :group 'languages :prefix "cython-" - :link '(url-link :tag "Homepage" "http://cython.org")) + :link '(url-link :tag "Homepage" "https://cython.org/")) ;;;###autoload (defcustom cython-default-compile-format "cython -a %s" diff --git a/Tools/dump_github_issues.py b/Tools/dump_github_issues.py new file mode 100644 index 000000000..daec51c50 --- /dev/null +++ b/Tools/dump_github_issues.py @@ -0,0 +1,142 @@ +""" +Dump the GitHub issues of the current project to a file (.json.gz). + +Usage: python3 Tools/dump_github_issues.py +""" + +import configparser +import gzip +import json +import os.path + +from datetime import datetime +from urllib.request import urlopen + +GIT_CONFIG_FILE = ".git/config" + + +class RateLimitReached(Exception): + pass + + +def gen_urls(repo): + i = 0 + while True: + yield f"https://api.github.com/repos/{repo}/issues?state=all&per_page=100&page={i}" + i += 1 + + +def read_rate_limit(): + with urlopen("https://api.github.com/rate_limit") as p: + return json.load(p) + + +def parse_rate_limit(limits): + limits = limits['resources']['core'] + return limits['limit'], limits['remaining'], datetime.fromtimestamp(limits['reset']) + + +def load_url(url): + with urlopen(url) as p: + data = json.load(p) + if isinstance(data, dict) and 'rate limit' in data.get('message', ''): + raise RateLimitReached() + + assert isinstance(data, list), type(data) + return data or None # None indicates empty last page + + +def join_list_data(lists): + result = [] + for data in lists: + if not data: + break + result.extend(data) + return result + + +def output_filename(repo): + timestamp = datetime.now() + return f"github_issues_{repo.replace('/', '_')}_{timestamp.strftime('%Y%m%d_%H%M%S')}.json.gz" + + +def write_gzjson(file_name, data, indent=2): + with gzip.open(file_name, "wt", encoding='utf-8') as gz: + json.dump(data, gz, indent=indent) + + +def find_origin_url(git_config=GIT_CONFIG_FILE): + assert os.path.exists(git_config) + parser = configparser.ConfigParser() + parser.read(git_config) + return parser.get('remote "origin"', 'url') + + +def parse_repo_name(git_url): + if git_url.endswith('.git'): + git_url = git_url[:-4] + return '/'.join(git_url.split('/')[-2:]) + + +def dump_issues(repo): + """Main entry point.""" + print(f"Reading issues from repo '{repo}'") + urls = gen_urls(repo) + try: + paged_data = map(load_url, urls) + issues = join_list_data(paged_data) + except RateLimitReached: + limit, remaining, reset_time = parse_rate_limit(read_rate_limit()) + print(f"FAILURE: Rate limits ({limit}) reached, remaining: {remaining}, reset at {reset_time}") + return + + filename = output_filename(repo) + print(f"Writing {len(issues)} to {filename}") + write_gzjson(filename, issues) + + +### TESTS + +def test_join_list_data(): + assert join_list_data([]) == [] + assert join_list_data([[1,2]]) == [1,2] + assert join_list_data([[1,2], [3]]) == [1,2,3] + assert join_list_data([[0], [1,2], [3]]) == [0,1,2,3] + assert join_list_data([[0], [1,2], [[[]],[]]]) == [0,1,2,[[]],[]] + + +def test_output_filename(): + filename = output_filename("re/po") + import re + assert re.match(r"github_issues_re_po_[0-9]{8}_[0-9]{6}\.json", filename) + + +def test_find_origin_url(): + assert find_origin_url() + + +def test_parse_repo_name(): + assert parse_repo_name("https://github.com/cython/cython") == "cython/cython" + assert parse_repo_name("git+ssh://git@github.com/cython/cython.git") == "cython/cython" + assert parse_repo_name("git+ssh://git@github.com/fork/cython.git") == "fork/cython" + + +def test_write_gzjson(): + import tempfile + with tempfile.NamedTemporaryFile() as tmp: + write_gzjson(tmp.name, [{}]) + + # test JSON format + with gzip.open(tmp.name) as f: + assert json.load(f) == [{}] + + # test indentation + with gzip.open(tmp.name) as f: + assert f.read() == b'[\n {}\n]' + + +### MAIN + +if __name__ == '__main__': + repo_name = parse_repo_name(find_origin_url()) + dump_issues(repo_name) diff --git a/Tools/rules.bzl b/Tools/rules.bzl index cd3eed58f..c59af6a99 100644 --- a/Tools/rules.bzl +++ b/Tools/rules.bzl @@ -11,8 +11,8 @@ load("@cython//Tools:rules.bzl", "pyx_library") pyx_library(name = 'mylib', srcs = ['a.pyx', 'a.pxd', 'b.py', 'pkg/__init__.py', 'pkg/c.pyx'], - py_deps = ['//py_library/dep'], - data = ['//other/data'], + # python library deps passed to py_library + deps = ['//py_library/dep'] ) The __init__.py file must be in your srcs list so that Cython can resolve diff --git a/appveyor.yml b/appveyor.yml index 370b0072a..8148e517c 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -5,17 +5,24 @@ environment: global: # SDK v7.0 MSVC Express 2008's SetEnv.cmd script will fail if the # /E:ON and /V:ON options are not enabled in the batch script interpreter - # See: http://stackoverflow.com/a/13751649/163740 + # See: https://stackoverflow.com/questions/11267463/compiling-python-modules-on-windows-x64/13751649#13751649 WITH_ENV: "cmd /E:ON /V:ON /C .\\appveyor\\run_with_env.cmd" + BACKEND: c + PARALLEL: "-j4" + EXTRA_CFLAGS: "" matrix: - PYTHON: "C:\\Python27" PYTHON_VERSION: "2.7" PYTHON_ARCH: "32" + PYTHONIOENCODING: "utf-8" + PARALLEL: "" - PYTHON: "C:\\Python27-x64" PYTHON_VERSION: "2.7" PYTHON_ARCH: "64" + PYTHONIOENCODING: "utf-8" + PARALLEL: "" - PYTHON: "C:\\Python39" PYTHON_VERSION: "3.9" @@ -32,14 +39,31 @@ environment: - PYTHON: "C:\\Python38-x64" PYTHON_VERSION: "3.8" PYTHON_ARCH: "64" + EXTRA_CFLAGS: "-DCYTHON_USE_TYPE_SPECS=1" + + - PYTHON: "C:\\Python38-x64" + PYTHON_VERSION: "3.8" + PYTHON_ARCH: "64" + BACKEND: c,cpp - PYTHON: "C:\\Python37" PYTHON_VERSION: "3.7" PYTHON_ARCH: "32" + BACKEND: c,cpp + + - PYTHON: "C:\\Python37-x64" + PYTHON_VERSION: "3.7" + PYTHON_ARCH: "64" - PYTHON: "C:\\Python37-x64" PYTHON_VERSION: "3.7" PYTHON_ARCH: "64" + EXTRA_CFLAGS: "-DCYTHON_USE_TYPE_SPECS=1" + + - PYTHON: "C:\\Python37-x64" + PYTHON_VERSION: "3.7" + PYTHON_ARCH: "64" + BACKEND: cpp - PYTHON: "C:\\Python36" PYTHON_VERSION: "3.6" @@ -60,10 +84,19 @@ environment: - PYTHON: "C:\\Python34" PYTHON_VERSION: "3.4" PYTHON_ARCH: "32" + PARALLEL: "" - PYTHON: "C:\\Python34-x64" PYTHON_VERSION: "3.4" PYTHON_ARCH: "64" + PARALLEL: "" + + - PYTHON: "C:\\Python27-x64" + PYTHON_VERSION: "2.7" + PYTHON_ARCH: "64" + BACKEND: cpp + PYTHONIOENCODING: "utf-8" + PARALLEL: "" clone_depth: 5 @@ -84,14 +117,15 @@ install: build: off build_script: - - "%WITH_ENV% %PYTHON%\\python.exe setup.py build_ext --inplace" + - "%WITH_ENV% %PYTHON%\\python.exe setup.py build_ext --inplace %PARALLEL%" - "%WITH_ENV% %PYTHON%\\python.exe setup.py bdist_wheel" test: off test_script: - "%PYTHON%\\Scripts\\pip.exe install -r test-requirements.txt" - - "set CFLAGS=" - - "%WITH_ENV% %PYTHON%\\python.exe runtests.py -vv --no-cpp --no-code-style -j5" + - "%PYTHON%\\Scripts\\pip.exe install win_unicode_console" + - "set CFLAGS=/Od /W3 %EXTRA_CFLAGS%" + - "%WITH_ENV% %PYTHON%\\python.exe runtests.py -vv --backend=%BACKEND% --no-code-style -j5" artifacts: - path: dist\* diff --git a/appveyor/install.ps1 b/appveyor/install.ps1 index d91b3f772..6dc32f40a 100644 --- a/appveyor/install.ps1 +++ b/appveyor/install.ps1 @@ -1,6 +1,6 @@ # Sample script to install Python and pip under Windows # Authors: Olivier Grisel and Kyle Kastner -# License: CC0 1.0 Universal: http://creativecommons.org/publicdomain/zero/1.0/ +# License: CC0 1.0 Universal: https://creativecommons.org/publicdomain/zero/1.0/ $PYTHON_BASE_URL = "https://www.python.org/ftp/python/" $GET_PIP_URL = "https://bootstrap.pypa.io/get-pip.py" diff --git a/bin/cython-generate-lexicon.py b/bin/cython-generate-lexicon.py new file mode 100755 index 000000000..e28441585 --- /dev/null +++ b/bin/cython-generate-lexicon.py @@ -0,0 +1,132 @@ +#!/usr/bin/env python3 + +# +# Updates Cython's Lexicon.py with the unicode characters that are accepted as +# identifiers. Should be run with the most recent version of Python possible +# to ensure that Lexicon is as complete as possible. +# +# Python3 only (it relies on str.isidentifier which is a Python 3 addition) +# +# Run with either +# --overwrite to update the existing Lexicon.py file +# --here to create a copy of Lexicon.py in the current directory + +import functools +import re +import os +import sys + +# Make sure we import the right Cython +cythonpath, _ = os.path.split(os.path.realpath(__file__)) # bin directory +cythonpath, _ = os.path.split(cythonpath) +if os.path.exists(os.path.join(cythonpath, "Cython")): + sys.path.insert(0, cythonpath) + print("Found (and using) local cython directory") +# else we aren't in a development directory + +from Cython.Compiler import Lexicon + + +def main(): + arg = '--overwrite' + if len(sys.argv) == 2: + arg = sys.argv[1] + if len(sys.argv) > 2 or arg not in ['--overwrite','--here']: + print("""Call the script with either: + --overwrite to update the existing Lexicon.py file (default) + --here to create an version of Lexicon.py in the current directory +""") + return + + generated_code = ( + f"# generated with:\n" + f"# {sys.implementation.name} {sys.version.splitlines()[0].strip()}\n" + "\n" + f"{generate_character_sets()}\n" + ) + + print("Reading file", Lexicon.__file__) + with open(Lexicon.__file__, 'r') as f: + parts = re.split(r"(# (?:BEGIN|END) GENERATED CODE\n?)", f.read()) + + if len(parts) not in (4,5) or ' GENERATED CODE' not in parts[1] or ' GENERATED CODE' not in parts[3]: + print("Warning: generated code section not found - code not inserted") + return + + parts[2] = generated_code + output = "".join(parts) + + if arg == "--here": + outfile = "Lexicon.py" + else: + assert arg == "--overwrite" + outfile = Lexicon.__file__ + + print("Writing to file", outfile) + with open(outfile, 'w') as f: + f.write(output) + + +# The easiest way to generate an appropriate character set is just to use the str.isidentifier method +# An alternative approach for getting character sets is at https://stackoverflow.com/a/49332214/4657412 +@functools.lru_cache() +def get_start_characters_as_number(): + return [ i for i in range(sys.maxunicode) if str.isidentifier(chr(i)) ] + + +def get_continue_characters_as_number(): + return [ i for i in range(sys.maxunicode) if str.isidentifier('a'+chr(i)) ] + + +def get_continue_not_start_as_number(): + start = get_start_characters_as_number() + cont = get_continue_characters_as_number() + assert set(start) <= set(cont), \ + "We assume that all identifier start characters are also continuation characters." + return sorted(set(cont).difference(start)) + + +def to_ranges(char_num_list): + # Convert the large lists of character digits to + # list of characters + # a list pairs of characters representing closed ranges + char_num_list = sorted(char_num_list) + first_good_val = char_num_list[0] + + single_chars = [] + ranges = [] + for n in range(1, len(char_num_list)): + if char_num_list[n]-1 != char_num_list[n-1]: + # discontinuous + if first_good_val == char_num_list[n-1]: + single_chars.append(chr(char_num_list[n-1])) + else: + ranges.append(chr(first_good_val) + chr(char_num_list[n-1])) + first_good_val = char_num_list[n] + + return ''.join(single_chars), ''.join(ranges) + + +def make_split_strings(chars, splitby=60, indent=" "): + lines = [f'u"{chars[i:i+splitby]}"' for i in range(0, len(chars), splitby)] + return indent + f"\n{indent}".join(lines) + + +def generate_character_sets(): + declarations = [] + for char_type, char_generator in [ + ("unicode_start_ch", get_start_characters_as_number), + ("unicode_continuation_ch", get_continue_not_start_as_number), + ]: + for set_type, chars in zip(("any", "range"), to_ranges(char_generator())): + declarations.append( + f"{char_type}_{set_type} = (\n" + f"{make_split_strings(chars)}\n" + f")\n" + ) + + return "".join(declarations) + + +if __name__ == "__main__": + main() diff --git a/bin/cythonrun b/bin/cythonrun index 1c6195492..f09e1d58d 100755 --- a/bin/cythonrun +++ b/bin/cythonrun @@ -9,7 +9,7 @@ Basic usage: python cythonrun somefile.py [ARGS] """ -from Cython.Build.BuildExecutable import build, build_and_run +from Cython.Build.BuildExecutable import build_and_run if __name__ == '__main__': import sys diff --git a/doc-requirements.txt b/doc-requirements.txt index 45eac3cfc..b4eb56d9f 100644 --- a/doc-requirements.txt +++ b/doc-requirements.txt @@ -1,3 +1,4 @@ sphinx==3.5.3 sphinx-issues==1.2.0 +sphinx-tabs==3.0.0 jupyter diff --git a/docs/CONTRIBUTING.rst b/docs/CONTRIBUTING.rst index 57cd10499..2dc0e96f0 100644 --- a/docs/CONTRIBUTING.rst +++ b/docs/CONTRIBUTING.rst @@ -5,13 +5,20 @@ If you are looking for a good way to contribute to the Cython project, please * have a look at the `Cython Hacker Guide <https://github.com/cython/cython/wiki/HackerGuide>`_, especially the section on `getting started <https://github.com/cython/cython/wiki/HackerGuide#getting-started>`_. -* look through the `issues that need help <https://github.com/cython/cython/issues?q=is%3Aissue+is%3Aopen+view+label%3A%22help+wanted%22>`_. -* look through the `issues that are a good entry point for beginners <https://github.com/cython/cython/issues?q=is%3Aissue+is%3Aopen+view+label%3A%22good+first+issue%22>`_. +* look through the `issues that need help <https://github.com/cython/cython/labels/help%20wanted>`_. +* look through the `issues that are a good entry point for beginners <https://github.com/cython/cython/labels/good%20first%20issue>`_. * ask on the `core developers mailing list <https://mail.python.org/mailman/listinfo/cython-devel>`_ for guidance. +Note that some (but not all) "good first issue"s also require an understanding of C +and a bit of the CPython C-API – usually those that also have the ``Code Generation`` +label. We generally consider a ticket a "good first issue" if it has a limited scope +that new contributors will have to learn about, e.g. only needs changes to the parser, +the type analysis or the code generation, but does not require changes all across the +compiler pipeline. + If you have code that you want to contribute, please make sure that it -* includes tests in the `tests/` directory (see the `Hacker Guide on Testing <https://github.com/cython/cython/wiki/HackerGuide#the-test-suite>`_) +* includes tests in the ``tests/`` directory (see the `Hacker Guide on Testing <https://github.com/cython/cython/wiki/HackerGuide#the-test-suite>`_) * comes in form of a pull request -We use `travis <https://travis-ci.org/cython/cython>`_ and `appveyor <https://ci.appveyor.com/project/cython/cython>`_ for cross-platform testing, including pull requests. +We use `github actions <https://github.com/cython/cython/actions>`_, `travis <https://travis-ci.org/cython/cython>`_ and `appveyor <https://ci.appveyor.com/project/cython/cython>`_ for cross-platform testing, including pull requests. diff --git a/docs/README b/docs/README index 7b3f61b5a..db5a80c91 100644 --- a/docs/README +++ b/docs/README @@ -11,7 +11,7 @@ On windows systems, you only need Sphinx. Open PowerShell and type:: You can then see the documentation by opening in a browser ``cython/docs/build/html/index.html``. The current Cython documentation files are hosted at -https://cython.readthedocs.io/en/latest/ +https://cython.readthedocs.io/ Notes @@ -17,7 +17,7 @@ Eventually, it seems all of the old users manual could be whittled down into independent tutorial topics. Much discussion of what we'd like to see is at -http://www.mail-archive.com/cython-dev@codespeak.net/msg06945.html +https://www.mail-archive.com/cython-dev@codespeak.net/msg06945.html There is currently a huge amount of redundancy, but no one section has it all. diff --git a/docs/_static/css/tabs.css b/docs/_static/css/tabs.css new file mode 100644 index 000000000..80691775f --- /dev/null +++ b/docs/_static/css/tabs.css @@ -0,0 +1,60 @@ +.sphinx-tabs { + margin-bottom: 1rem; +} + +[role="tablist"] { + border-bottom: 0px solid #a0b3bf; +} + +.sphinx-tabs-tab { + position: relative; + font-family: 'Helvetica Neue',Arial,Helvetica,sans-serif; + color: black; + line-height: 24px; + margin: 0; + font-size: 16px; + font-weight: 400; + background-color: rgba(255, 255, 255, 0); + border-radius: 5px 5px 5px 5px; + border: 0; + padding: 0.5rem 1.6rem; + margin-bottom: 0; +} + +.sphinx-tabs-tab[aria-selected="true"] { + border: 2px solid gray; + border-bottom: 2px solid gray; + margin: -2px; + background-color: #efefef; + z-index: 999; /* render on top*/ +} + +.sphinx-tabs-tab[aria-selected="false"] { + border: 2px solid #dddddd; + border-bottom: 2px solid #dddddd; + margin: -2px; + background-color: white; + } + +.sphinx-tabs-tab:focus { + z-index: 1; + outline-offset: 1px; +} + +.sphinx-tabs-panel { + position: relative; + padding: 0rem; + border: 0px solid #a0b3bf; + margin: 0px 0px 0px 0px; + border-radius: 0 0 5px 5px; + border-top: 0; + background: white; +} + +.sphinx-tabs-panel.code-tab { + padding: 0.4rem; +} + +.sphinx-tab img { + margin-bottom: 24 px; +} diff --git a/docs/_templates/layout.html b/docs/_templates/layout.html index a071c96c8..814971239 100644 --- a/docs/_templates/layout.html +++ b/docs/_templates/layout.html @@ -1,7 +1,18 @@ {% extends "!layout.html" %} +{% block relbar1 %} +{{ super() }} +{% if pagename != "src/donating" %} +<div style="width: 100%; height: 3em; vertical-align: middle; text-align: center; font-weight: bold; background-color: lightgray; border: solid red 1px"> + <p style="font-size: 110%">🤝 Like the tool? Help making it better! <a href="{{ pathto("src/donating") }}">Your donation helps!</a> 🤝</p> +</div> +{% endif %} +{% endblock %} + + {% block footer %} {{ super() }} +{# ## Removed to avoid tracking our users. <script type="text/javascript"> var gaJsHost = (("https:" == document.location.protocol) ? "https://ssl." : "http://www."); document.write(unescape("%3Cscript src='" + gaJsHost + "google-analytics.com/ga.js' type='text/javascript'%3E%3C/script%3E")); @@ -11,4 +22,5 @@ try { var pageTracker = _gat._getTracker("UA-6139100-3"); pageTracker._trackPageview(); } catch(err) {}</script> +#} {% endblock %} diff --git a/docs/conf.py b/docs/conf.py index a84e0b928..10b0567c4 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -21,11 +21,6 @@ YEAR = datetime.date.today().strftime('%Y') # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. sys.path.insert(0, os.path.abspath('..')) -sys.path.append(os.path.abspath('sphinxext')) - -# Import support for ipython console session syntax highlighting (lives -# in the sphinxext directory defined above) -import ipython_console_highlighting # -- General configuration ----------------------------------------------------- @@ -39,12 +34,12 @@ highlight_language = 'cython' # Add any Sphinx extension module names here, as strings. They can be extensions # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. extensions = [ - 'ipython_console_highlighting', - 'cython_highlighting', 'sphinx.ext.imgmath', 'sphinx.ext.todo', 'sphinx.ext.intersphinx', - 'sphinx.ext.autodoc' + 'sphinx.ext.autodoc', + 'sphinx_issues', # if this is missing, pip install sphinx-issues + 'sphinx_tabs.tabs', # if this is missing, pip install sphinx-tabs ] try: import rst2pdf @@ -132,6 +127,16 @@ intersphinx_mapping = {'python': ('https://docs.python.org/3/', None)} # If true, keep warnings as "system message" paragraphs in the built documents. #keep_warnings = False +# The output image format. The default is 'png'. It should be either 'png' or 'svg'. +imgmath_image_format = "svg" + +# For sphinx-issues + +issues_github_path = "cython/cython" + +# For sphinx-tabs + +sphinx_tabs_disable_tab_closing = True # -- Options for HTML output --------------------------------------------------- @@ -174,6 +179,11 @@ html_favicon = "_static/favicon.ico" # so a file named "default.css" will overwrite the builtin "default.css". html_static_path = ['_static'] +# Overwriting css from extensions +html_context = { + 'css_files': ['_static/css/tabs.css'], +} + # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. #html_last_updated_fmt = '%b %d, %Y' diff --git a/docs/examples/Cython Magics.ipynb b/docs/examples/Cython Magics.ipynb index 0a8c8f56f..9bbf934c8 100644 --- a/docs/examples/Cython Magics.ipynb +++ b/docs/examples/Cython Magics.ipynb @@ -356,7 +356,7 @@ "cell_type": "markdown",
"metadata": {},
"source": [
- "You can similarly use the `-I/--include` flag to add include directories to the search path, and `-c/--compile-args` to add extra flags that are passed to Cython via the `extra_compile_args` of the distutils `Extension` class. Please see [the Cython docs on C library usage](http://docs.cython.org/src/tutorial/clibraries.html) for more details on the use of these flags."
+ "You can similarly use the `-I/--include` flag to add include directories to the search path, and `-c/--compile-args` to add extra flags that are passed to Cython via the `extra_compile_args` of the distutils `Extension` class. Please see [the Cython docs on C library usage](https://docs.cython.org/src/tutorial/clibraries.html) for more details on the use of these flags."
]
}
],
diff --git a/docs/examples/README.rst b/docs/examples/README.rst index 02c21a5fe..f998f4f33 100644 --- a/docs/examples/README.rst +++ b/docs/examples/README.rst @@ -1,3 +1,5 @@ -This example directory is organized like the ``Cython/docs/src/`` directory,
-with one directory per ``.rst`` file. All files in this directory are tested
-in the :file:`runtests.py` with the mode `compile`.
+:orphan: + +This example directory is organized like the ``Cython/docs/src/`` directory, +with one directory per ``.rst`` file. All files in this directory are tested +in the :file:`runtests.py` with the mode `compile`. diff --git a/docs/examples/not_in_docs/great_circle/p1.py b/docs/examples/not_in_docs/great_circle/p1.py index c0694a235..e60b9723f 100644 --- a/docs/examples/not_in_docs/great_circle/p1.py +++ b/docs/examples/not_in_docs/great_circle/p1.py @@ -1,7 +1,7 @@ import math def great_circle(lon1, lat1, lon2, lat2): - radius = 3956 # miles + radius = 3956 # miles x = math.pi/180.0 a = (90.0 - lat1)*x diff --git a/docs/examples/quickstart/build/hello.pyx b/docs/examples/quickstart/build/hello.pyx index 47fc8d1cf..da1b827ac 100644 --- a/docs/examples/quickstart/build/hello.pyx +++ b/docs/examples/quickstart/build/hello.pyx @@ -1,2 +1,2 @@ -def say_hello_to(name):
- print("Hello %s!" % name)
+def say_hello_to(name): + print("Hello %s!" % name) diff --git a/docs/examples/quickstart/build/setup.py b/docs/examples/quickstart/build/setup.py index 4fb8c8154..fe959a106 100644 --- a/docs/examples/quickstart/build/setup.py +++ b/docs/examples/quickstart/build/setup.py @@ -1,5 +1,8 @@ -from distutils.core import setup
-from Cython.Build import cythonize
-
-setup(name='Hello world app',
- ext_modules=cythonize("hello.pyx"))
+from setuptools import setup +from Cython.Build import cythonize + +setup( + name='Hello world app', + ext_modules=cythonize("hello.pyx"), + zip_safe=False, +) diff --git a/docs/examples/quickstart/cythonize/cdef_keyword.py b/docs/examples/quickstart/cythonize/cdef_keyword.py new file mode 100644 index 000000000..6c0ee3e68 --- /dev/null +++ b/docs/examples/quickstart/cythonize/cdef_keyword.py @@ -0,0 +1,4 @@ +@cython.cfunc +@cython.exceptval(-2, check=True) +def f(x: cython.double) -> cython.double: + return x ** 2 - x diff --git a/docs/examples/quickstart/cythonize/cdef_keyword.pyx b/docs/examples/quickstart/cythonize/cdef_keyword.pyx index 16503ee89..bc7d893fa 100644 --- a/docs/examples/quickstart/cythonize/cdef_keyword.pyx +++ b/docs/examples/quickstart/cythonize/cdef_keyword.pyx @@ -1,2 +1,4 @@ -cdef double f(double x) except? -2:
- return x ** 2 - x
+ + +cdef double f(double x) except? -2: + return x ** 2 - x diff --git a/docs/examples/quickstart/cythonize/integrate.py b/docs/examples/quickstart/cythonize/integrate.py index 8d420b923..80d6d13a7 100644 --- a/docs/examples/quickstart/cythonize/integrate.py +++ b/docs/examples/quickstart/cythonize/integrate.py @@ -1,10 +1,10 @@ -def f(x):
- return x ** 2 - x
-
-
-def integrate_f(a, b, N):
- s = 0
- dx = (b - a) / N
- for i in range(N):
- s += f(a + i * dx)
- return s * dx
+def f(x): + return x ** 2 - x + + +def integrate_f(a, b, N): + s = 0 + dx = (b - a) / N + for i in range(N): + s += f(a + i * dx) + return s * dx diff --git a/docs/examples/quickstart/cythonize/integrate_cy.py b/docs/examples/quickstart/cythonize/integrate_cy.py new file mode 100644 index 000000000..592ce8db7 --- /dev/null +++ b/docs/examples/quickstart/cythonize/integrate_cy.py @@ -0,0 +1,13 @@ +def f(x: cython.double): + return x ** 2 - x + + +def integrate_f(a: cython.double, b: cython.double, N: cython.int): + i: cython.int + s: cython.double + dx: cython.double + s = 0 + dx = (b - a) / N + for i in range(N): + s += f(a + i * dx) + return s * dx diff --git a/docs/examples/quickstart/cythonize/integrate_cy.pyx b/docs/examples/quickstart/cythonize/integrate_cy.pyx index 0bb0cd548..0e20a6c33 100644 --- a/docs/examples/quickstart/cythonize/integrate_cy.pyx +++ b/docs/examples/quickstart/cythonize/integrate_cy.pyx @@ -1,12 +1,13 @@ -def f(double x):
- return x ** 2 - x
-
-
-def integrate_f(double a, double b, int N):
- cdef int i
- cdef double s, dx
- s = 0
- dx = (b - a) / N
- for i in range(N):
- s += f(a + i * dx)
- return s * dx
+def f(double x): + return x ** 2 - x + + +def integrate_f(double a, double b, int N): + cdef int i + cdef double s + cdef double dx + s = 0 + dx = (b - a) / N + for i in range(N): + s += f(a + i * dx) + return s * dx diff --git a/docs/examples/tutorial/array/clone.pyx b/docs/examples/tutorial/array/clone.pyx index e2bac0e4a..2eb803499 100644 --- a/docs/examples/tutorial/array/clone.pyx +++ b/docs/examples/tutorial/array/clone.pyx @@ -1,8 +1,8 @@ -from cpython cimport array
-import array
-
-cdef array.array int_array_template = array.array('i', [])
-cdef array.array newarray
-
-# create an array with 3 elements with same type as template
-newarray = array.clone(int_array_template, 3, zero=False)
+from cpython cimport array +import array + +cdef array.array int_array_template = array.array('i', []) +cdef array.array newarray + +# create an array with 3 elements with same type as template +newarray = array.clone(int_array_template, 3, zero=False) diff --git a/docs/examples/tutorial/array/overhead.pyx b/docs/examples/tutorial/array/overhead.pyx index e385bff3f..88bc97500 100644 --- a/docs/examples/tutorial/array/overhead.pyx +++ b/docs/examples/tutorial/array/overhead.pyx @@ -1,15 +1,15 @@ -from cpython cimport array
-import array
-
-cdef array.array a = array.array('i', [1, 2, 3])
-cdef int[:] ca = a
-
-cdef int overhead(object a):
- cdef int[:] ca = a
- return ca[0]
-
-cdef int no_overhead(int[:] ca):
- return ca[0]
-
-print(overhead(a)) # new memory view will be constructed, overhead
-print(no_overhead(ca)) # ca is already a memory view, so no overhead
+from cpython cimport array +import array + +cdef array.array a = array.array('i', [1, 2, 3]) +cdef int[:] ca = a + +cdef int overhead(object a): + cdef int[:] ca = a + return ca[0] + +cdef int no_overhead(int[:] ca): + return ca[0] + +print(overhead(a)) # new memory view will be constructed, overhead +print(no_overhead(ca)) # ca is already a memory view, so no overhead diff --git a/docs/examples/tutorial/array/resize.pyx b/docs/examples/tutorial/array/resize.pyx index a11fbde7b..7b92958b4 100644 --- a/docs/examples/tutorial/array/resize.pyx +++ b/docs/examples/tutorial/array/resize.pyx @@ -1,10 +1,10 @@ -from cpython cimport array
-import array
-
-cdef array.array a = array.array('i', [1, 2, 3])
-cdef array.array b = array.array('i', [4, 5, 6])
-
-# extend a with b, resize as needed
-array.extend(a, b)
-# resize a, leaving just original three elements
-array.resize(a, len(a) - len(b))
+from cpython cimport array +import array + +cdef array.array a = array.array('i', [1, 2, 3]) +cdef array.array b = array.array('i', [4, 5, 6]) + +# extend a with b, resize as needed +array.extend(a, b) +# resize a, leaving just original three elements +array.resize(a, len(a) - len(b)) diff --git a/docs/examples/tutorial/array/safe_usage.pyx b/docs/examples/tutorial/array/safe_usage.pyx index 61d6b39eb..15107ae92 100644 --- a/docs/examples/tutorial/array/safe_usage.pyx +++ b/docs/examples/tutorial/array/safe_usage.pyx @@ -1,6 +1,6 @@ -from cpython cimport array
-import array
-cdef array.array a = array.array('i', [1, 2, 3])
-cdef int[:] ca = a
-
-print(ca[0])
+from cpython cimport array +import array +cdef array.array a = array.array('i', [1, 2, 3]) +cdef int[:] ca = a + +print(ca[0]) diff --git a/docs/examples/tutorial/array/unsafe_usage.pyx b/docs/examples/tutorial/array/unsafe_usage.pyx index 2aefeb102..d1f498c68 100644 --- a/docs/examples/tutorial/array/unsafe_usage.pyx +++ b/docs/examples/tutorial/array/unsafe_usage.pyx @@ -1,11 +1,11 @@ -from cpython cimport array
-import array
-
-cdef array.array a = array.array('i', [1, 2, 3])
-
-# access underlying pointer:
-print(a.data.as_ints[0])
-
-from libc.string cimport memset
-
-memset(a.data.as_voidptr, 0, len(a) * sizeof(int))
+from cpython cimport array +import array + +cdef array.array a = array.array('i', [1, 2, 3]) + +# access underlying pointer: +print(a.data.as_ints[0]) + +from libc.string cimport memset + +memset(a.data.as_voidptr, 0, len(a) * sizeof(int)) diff --git a/docs/examples/tutorial/cdef_classes/integrate.py b/docs/examples/tutorial/cdef_classes/integrate.py new file mode 100644 index 000000000..cd02554e5 --- /dev/null +++ b/docs/examples/tutorial/cdef_classes/integrate.py @@ -0,0 +1,17 @@ +from cython.cimports.sin_of_square import Function, SinOfSquareFunction + +def integrate(f: Function, a: float, b: float, N: cython.int): + i: cython.int + + if f is None: + raise ValueError("f cannot be None") + + s: float = 0 + dx: float = (b - a) / N + + for i in range(N): + s += f.evaluate(a + i * dx) + + return s * dx + +print(integrate(SinOfSquareFunction(), 0, 1, 10000)) diff --git a/docs/examples/tutorial/cdef_classes/integrate.pyx b/docs/examples/tutorial/cdef_classes/integrate.pyx index a3bbcbfec..ad4c8540b 100644 --- a/docs/examples/tutorial/cdef_classes/integrate.pyx +++ b/docs/examples/tutorial/cdef_classes/integrate.pyx @@ -1,14 +1,17 @@ -from sin_of_square cimport Function, SinOfSquareFunction
-
-def integrate(Function f, double a, double b, int N):
- cdef int i
- cdef double s, dx
- if f is None:
- raise ValueError("f cannot be None")
- s = 0
- dx = (b - a) / N
- for i in range(N):
- s += f.evaluate(a + i * dx)
- return s * dx
-
-print(integrate(SinOfSquareFunction(), 0, 1, 10000))
+from sin_of_square cimport Function, SinOfSquareFunction + +def integrate(Function f, double a, double b, int N): + cdef int i + cdef double s, dx + if f is None: + raise ValueError("f cannot be None") + + s = 0 + dx = (b - a) / N + + for i in range(N): + s += f.evaluate(a + i * dx) + + return s * dx + +print(integrate(SinOfSquareFunction(), 0, 1, 10000)) diff --git a/docs/examples/tutorial/cdef_classes/math_function.py b/docs/examples/tutorial/cdef_classes/math_function.py index 21281cc9b..1a6ee896c 100644 --- a/docs/examples/tutorial/cdef_classes/math_function.py +++ b/docs/examples/tutorial/cdef_classes/math_function.py @@ -1,7 +1,7 @@ -class MathFunction(object):
- def __init__(self, name, operator):
- self.name = name
- self.operator = operator
-
- def __call__(self, *operands):
- return self.operator(*operands)
+class MathFunction(object): + def __init__(self, name, operator): + self.name = name + self.operator = operator + + def __call__(self, *operands): + return self.operator(*operands) diff --git a/docs/examples/tutorial/cdef_classes/math_function_2.py b/docs/examples/tutorial/cdef_classes/math_function_2.py new file mode 100644 index 000000000..ba5917639 --- /dev/null +++ b/docs/examples/tutorial/cdef_classes/math_function_2.py @@ -0,0 +1,5 @@ +@cython.cclass +class Function: + @cython.ccall + def evaluate(self, x: float) -> float: + return 0 diff --git a/docs/examples/tutorial/cdef_classes/math_function_2.pyx b/docs/examples/tutorial/cdef_classes/math_function_2.pyx index 1793ef689..a4bdb7aa2 100644 --- a/docs/examples/tutorial/cdef_classes/math_function_2.pyx +++ b/docs/examples/tutorial/cdef_classes/math_function_2.pyx @@ -1,3 +1,5 @@ -cdef class Function:
- cpdef double evaluate(self, double x) except *:
- return 0
+ +cdef class Function: + + cpdef double evaluate(self, double x) except *: + return 0 diff --git a/docs/examples/tutorial/cdef_classes/nonecheck.py b/docs/examples/tutorial/cdef_classes/nonecheck.py new file mode 100644 index 000000000..dccb97435 --- /dev/null +++ b/docs/examples/tutorial/cdef_classes/nonecheck.py @@ -0,0 +1,20 @@ +# cython: nonecheck=True +# ^^^ Turns on nonecheck globally + +import cython + +@cython.cclass +class MyClass: + pass + +# Turn off nonecheck locally for the function +@cython.nonecheck(False) +def func(): + obj: MyClass = None + try: + # Turn nonecheck on again for a block + with cython.nonecheck(True): + print(obj.myfunc()) # Raises exception + except AttributeError: + pass + print(obj.myfunc()) # Hope for a crash! diff --git a/docs/examples/tutorial/cdef_classes/nonecheck.pyx b/docs/examples/tutorial/cdef_classes/nonecheck.pyx index b9e12c8d5..92c8fa42b 100644 --- a/docs/examples/tutorial/cdef_classes/nonecheck.pyx +++ b/docs/examples/tutorial/cdef_classes/nonecheck.pyx @@ -1,19 +1,20 @@ -# cython: nonecheck=True
-# ^^^ Turns on nonecheck globally
-
-import cython
-
-cdef class MyClass:
- pass
-
-# Turn off nonecheck locally for the function
-@cython.nonecheck(False)
-def func():
- cdef MyClass obj = None
- try:
- # Turn nonecheck on again for a block
- with cython.nonecheck(True):
- print(obj.myfunc()) # Raises exception
- except AttributeError:
- pass
- print(obj.myfunc()) # Hope for a crash!
+# cython: nonecheck=True +# ^^^ Turns on nonecheck globally + +import cython + + +cdef class MyClass: + pass + +# Turn off nonecheck locally for the function +@cython.nonecheck(False) +def func(): + cdef MyClass obj = None + try: + # Turn nonecheck on again for a block + with cython.nonecheck(True): + print(obj.myfunc()) # Raises exception + except AttributeError: + pass + print(obj.myfunc()) # Hope for a crash! diff --git a/docs/examples/tutorial/cdef_classes/sin_of_square.py b/docs/examples/tutorial/cdef_classes/sin_of_square.py new file mode 100644 index 000000000..1904ea934 --- /dev/null +++ b/docs/examples/tutorial/cdef_classes/sin_of_square.py @@ -0,0 +1,13 @@ +from cython.cimports.libc.math import sin + +@cython.cclass +class Function: + @cython.ccall + def evaluate(self, x: float) -> float: + return 0 + +@cython.cclass +class SinOfSquareFunction(Function): + @cython.ccall + def evaluate(self, x: float) -> float: + return sin(x ** 2) diff --git a/docs/examples/tutorial/cdef_classes/sin_of_square.pyx b/docs/examples/tutorial/cdef_classes/sin_of_square.pyx index 7aab96056..67af294b5 100644 --- a/docs/examples/tutorial/cdef_classes/sin_of_square.pyx +++ b/docs/examples/tutorial/cdef_classes/sin_of_square.pyx @@ -1,9 +1,13 @@ -from libc.math cimport sin
-
-cdef class Function:
- cpdef double evaluate(self, double x) except *:
- return 0
-
-cdef class SinOfSquareFunction(Function):
- cpdef double evaluate(self, double x) except *:
- return sin(x ** 2)
+from libc.math cimport sin + + +cdef class Function: + + cpdef double evaluate(self, double x) except *: + return 0 + + +cdef class SinOfSquareFunction(Function): + + cpdef double evaluate(self, double x) except *: + return sin(x ** 2) diff --git a/docs/examples/tutorial/cdef_classes/wave_function.py b/docs/examples/tutorial/cdef_classes/wave_function.py new file mode 100644 index 000000000..7ff59a762 --- /dev/null +++ b/docs/examples/tutorial/cdef_classes/wave_function.py @@ -0,0 +1,22 @@ +from cython.cimports.sin_of_square import Function + +@cython.cclass +class WaveFunction(Function): + + # Not available in Python-space: + offset: float + + # Available in Python-space: + freq = cython.declare(cython.double, visibility='public') + + # Available in Python-space, but only for reading: + scale = cython.declare(cython.double, visibility='readonly') + + # Available in Python-space: + @property + def period(self): + return 1.0 / self.freq + + @period.setter + def period(self, value): + self.freq = 1.0 / value diff --git a/docs/examples/tutorial/cdef_classes/wave_function.pyx b/docs/examples/tutorial/cdef_classes/wave_function.pyx index aa35d954e..34b144667 100644 --- a/docs/examples/tutorial/cdef_classes/wave_function.pyx +++ b/docs/examples/tutorial/cdef_classes/wave_function.pyx @@ -1,21 +1,22 @@ -from sin_of_square cimport Function
-
-cdef class WaveFunction(Function):
-
- # Not available in Python-space:
- cdef double offset
-
- # Available in Python-space:
- cdef public double freq
-
- # Available in Python-space, but only for reading:
- cdef readonly double scale
-
- # Available in Python-space:
- @property
- def period(self):
- return 1.0 / self.freq
-
- @period.setter
- def period(self, value):
- self.freq = 1.0 / value
+from sin_of_square cimport Function + + +cdef class WaveFunction(Function): + + # Not available in Python-space: + cdef double offset + + # Available in Python-space: + cdef public double freq + + # Available in Python-space, but only for reading: + cdef readonly double scale + + # Available in Python-space: + @property + def period(self): + return 1.0 / self.freq + + @period.setter + def period(self, value): + self.freq = 1.0 / value diff --git a/docs/examples/tutorial/clibraries/cqueue.pxd b/docs/examples/tutorial/clibraries/cqueue.pxd index 13a07d317..a657ae331 100644 --- a/docs/examples/tutorial/clibraries/cqueue.pxd +++ b/docs/examples/tutorial/clibraries/cqueue.pxd @@ -1,5 +1,3 @@ -# cqueue.pxd
-
cdef extern from "c-algorithms/src/queue.h":
ctypedef struct Queue:
pass
diff --git a/docs/examples/tutorial/clibraries/queue.py b/docs/examples/tutorial/clibraries/queue.py new file mode 100644 index 000000000..45529fa94 --- /dev/null +++ b/docs/examples/tutorial/clibraries/queue.py @@ -0,0 +1,8 @@ +from cython.cimports import cqueue + +@cython.cclass +class Queue: + _c_queue = cython.declare(cython.pointer(cqueue.Queue)) + + def __cinit__(self): + self._c_queue = cqueue.queue_new() diff --git a/docs/examples/tutorial/clibraries/queue.pyx b/docs/examples/tutorial/clibraries/queue.pyx index 5363ee4f5..654c07b8d 100644 --- a/docs/examples/tutorial/clibraries/queue.pyx +++ b/docs/examples/tutorial/clibraries/queue.pyx @@ -1,9 +1,8 @@ -# queue.pyx
-
-cimport cqueue
-
-cdef class Queue:
- cdef cqueue.Queue* _c_queue
-
- def __cinit__(self):
- self._c_queue = cqueue.queue_new()
+cimport cqueue + + +cdef class Queue: + cdef cqueue.Queue* _c_queue + + def __cinit__(self): + self._c_queue = cqueue.queue_new() diff --git a/docs/examples/tutorial/clibraries/queue2.py b/docs/examples/tutorial/clibraries/queue2.py new file mode 100644 index 000000000..de6d58a99 --- /dev/null +++ b/docs/examples/tutorial/clibraries/queue2.py @@ -0,0 +1,10 @@ +from cython.cimports import cqueue + +@cython.cclass +class Queue: + _c_queue = cython.declare(cython.pointer(cqueue.Queue)) + + def __cinit__(self): + self._c_queue = cqueue.queue_new() + if self._c_queue is cython.NULL: + raise MemoryError() diff --git a/docs/examples/tutorial/clibraries/queue2.pyx b/docs/examples/tutorial/clibraries/queue2.pyx index 9278fbf4b..5dca04c22 100644 --- a/docs/examples/tutorial/clibraries/queue2.pyx +++ b/docs/examples/tutorial/clibraries/queue2.pyx @@ -1,11 +1,10 @@ -# queue.pyx
-
-cimport cqueue
-
-cdef class Queue:
- cdef cqueue.Queue* _c_queue
-
- def __cinit__(self):
- self._c_queue = cqueue.queue_new()
- if self._c_queue is NULL:
- raise MemoryError()
+cimport cqueue + + +cdef class Queue: + cdef cqueue.Queue* _c_queue + + def __cinit__(self): + self._c_queue = cqueue.queue_new() + if self._c_queue is NULL: + raise MemoryError() diff --git a/docs/examples/tutorial/clibraries/queue3.py b/docs/examples/tutorial/clibraries/queue3.py new file mode 100644 index 000000000..79f341254 --- /dev/null +++ b/docs/examples/tutorial/clibraries/queue3.py @@ -0,0 +1,68 @@ +from cython.cimports import cqueue +from cython import cast + +@cython.cclass +class Queue: + """A queue class for C integer values. + + >>> q = Queue() + >>> q.append(5) + >>> q.peek() + 5 + >>> q.pop() + 5 + """ + _c_queue = cython.declare(cython.pointer(cqueue.Queue)) + def __cinit__(self): + self._c_queue = cqueue.queue_new() + if self._c_queue is cython.NULL: + raise MemoryError() + + def __dealloc__(self): + if self._c_queue is not cython.NULL: + cqueue.queue_free(self._c_queue) + + @cython.ccall + def append(self, value: cython.int): + if not cqueue.queue_push_tail(self._c_queue, + cast(cython.p_void, cast(cython.Py_ssize_t, value))): + raise MemoryError() + + # The `cpdef` feature is obviously not available for the original "extend()" + # method, as the method signature is incompatible with Python argument + # types (Python does not have pointers). However, we can rename + # the C-ish "extend()" method to e.g. "extend_ints()", and write + # a new "extend()" method that provides a suitable Python interface by + # accepting an arbitrary Python iterable. + @cython.ccall + def extend(self, values): + for value in values: + self.append(value) + + @cython.cfunc + def extend_ints(self, values: cython.p_int, count: cython.size_t): + value: cython.int + for value in values[:count]: # Slicing pointer to limit the iteration boundaries. + self.append(value) + + @cython.ccall + @cython.exceptval(-1, check=True) + def peek(self) -> cython.int: + value: cython.int = cast(cython.Py_ssize_t, cqueue.queue_peek_head(self._c_queue)) + + if value == 0: + # this may mean that the queue is empty, + # or that it happens to contain a 0 value + if cqueue.queue_is_empty(self._c_queue): + raise IndexError("Queue is empty") + return value + + @cython.ccall + @cython.exceptval(-1, check=True) + def pop(self) -> cython.int: + if cqueue.queue_is_empty(self._c_queue): + raise IndexError("Queue is empty") + return cast(cython.Py_ssize_t, cqueue.queue_pop_head(self._c_queue)) + + def __bool__(self): + return not cqueue.queue_is_empty(self._c_queue) diff --git a/docs/examples/tutorial/clibraries/queue3.pyx b/docs/examples/tutorial/clibraries/queue3.pyx index cc84cf172..c15c48e15 100644 --- a/docs/examples/tutorial/clibraries/queue3.pyx +++ b/docs/examples/tutorial/clibraries/queue3.pyx @@ -1,7 +1,7 @@ -# queue.pyx - cimport cqueue + + cdef class Queue: """A queue class for C integer values. @@ -22,6 +22,7 @@ cdef class Queue: if self._c_queue is not NULL: cqueue.queue_free(self._c_queue) + cpdef append(self, int value): if not cqueue.queue_push_tail(self._c_queue, <void*> <Py_ssize_t> value): @@ -33,15 +34,19 @@ cdef class Queue: # the C-ish "extend()" method to e.g. "extend_ints()", and write # a new "extend()" method that provides a suitable Python interface by # accepting an arbitrary Python iterable. + cpdef extend(self, values): for value in values: self.append(value) + cdef extend_ints(self, int* values, size_t count): cdef int value for value in values[:count]: # Slicing pointer to limit the iteration boundaries. self.append(value) + + cpdef int peek(self) except? -1: cdef int value = <Py_ssize_t> cqueue.queue_peek_head(self._c_queue) @@ -52,6 +57,8 @@ cdef class Queue: raise IndexError("Queue is empty") return value + + cpdef int pop(self) except? -1: if cqueue.queue_is_empty(self._c_queue): raise IndexError("Queue is empty") diff --git a/docs/examples/tutorial/clibraries/test_queue.py b/docs/examples/tutorial/clibraries/test_queue.py index 5390a82c1..41b267395 100644 --- a/docs/examples/tutorial/clibraries/test_queue.py +++ b/docs/examples/tutorial/clibraries/test_queue.py @@ -1,36 +1,36 @@ -from __future__ import print_function
-
-import time
-
-import queue
-
-Q = queue.Queue()
-
-Q.append(10)
-Q.append(20)
-print(Q.peek())
-print(Q.pop())
-print(Q.pop())
-try:
- print(Q.pop())
-except IndexError as e:
- print("Error message:", e) # Prints "Queue is empty"
-
-i = 10000
-
-values = range(i)
-
-start_time = time.time()
-
-Q.extend(values)
-
-end_time = time.time() - start_time
-
-print("Adding {} items took {:1.3f} msecs.".format(i, 1000 * end_time))
-
-for i in range(41):
- Q.pop()
-
-Q.pop()
-print("The answer is:")
-print(Q.pop())
+from __future__ import print_function + +import time + +import queue + +Q = queue.Queue() + +Q.append(10) +Q.append(20) +print(Q.peek()) +print(Q.pop()) +print(Q.pop()) +try: + print(Q.pop()) +except IndexError as e: + print("Error message:", e) # Prints "Queue is empty" + +i = 10000 + +values = range(i) + +start_time = time.time() + +Q.extend(values) + +end_time = time.time() - start_time + +print("Adding {} items took {:1.3f} msecs.".format(i, 1000 * end_time)) + +for i in range(41): + Q.pop() + +Q.pop() +print("The answer is:") +print(Q.pop()) diff --git a/docs/examples/tutorial/cython_tutorial/primes.py b/docs/examples/tutorial/cython_tutorial/primes.py new file mode 100644 index 000000000..645d9479d --- /dev/null +++ b/docs/examples/tutorial/cython_tutorial/primes.py @@ -0,0 +1,27 @@ +def primes(nb_primes: cython.int): + i: cython.int + p: cython.int[1000] + + if nb_primes > 1000: + nb_primes = 1000 + + if not cython.compiled: # Only if regular Python is running + p = [0] * 1000 # Make p work almost like a C array + + len_p: cython.int = 0 # The current number of elements in p. + n: cython.int = 2 + while len_p < nb_primes: + # Is n prime? + for i in p[:len_p]: + if n % i == 0: + break + + # If no break occurred in the loop, we have a prime. + else: + p[len_p] = n + len_p += 1 + n += 1 + + # Let's copy the result into a Python list: + result_as_list = [prime for prime in p[:len_p]] + return result_as_list diff --git a/docs/examples/tutorial/cython_tutorial/primes.pyx b/docs/examples/tutorial/cython_tutorial/primes.pyx index 96ecdb59a..2f93b17fc 100644 --- a/docs/examples/tutorial/cython_tutorial/primes.pyx +++ b/docs/examples/tutorial/cython_tutorial/primes.pyx @@ -1,9 +1,13 @@ def primes(int nb_primes): cdef int n, i, len_p cdef int p[1000] + if nb_primes > 1000: nb_primes = 1000 + + + len_p = 0 # The current number of elements in p. n = 2 while len_p < nb_primes: @@ -18,6 +22,6 @@ def primes(int nb_primes): len_p += 1 n += 1 - # Let's return the result in a python list: - result_as_list = [prime for prime in p[:len_p]] + # Let's copy the result into a Python list: + result_as_list = [prime for prime in p[:len_p]] return result_as_list diff --git a/docs/examples/tutorial/cython_tutorial/primes_cpp.py b/docs/examples/tutorial/cython_tutorial/primes_cpp.py new file mode 100644 index 000000000..468d00c46 --- /dev/null +++ b/docs/examples/tutorial/cython_tutorial/primes_cpp.py @@ -0,0 +1,22 @@ +# distutils: language=c++ + +import cython +from cython.cimports.libcpp.vector import vector + +def primes(nb_primes: cython.uint): + i: cython.int + p: vector[cython.int] + p.reserve(nb_primes) # allocate memory for 'nb_primes' elements. + + n: cython.int = 2 + while p.size() < nb_primes: # size() for vectors is similar to len() + for i in p: + if n % i == 0: + break + else: + p.push_back(n) # push_back is similar to append() + n += 1 + + # If possible, C values and C++ objects are automatically + # converted to Python objects at need. + return p # so here, the vector will be copied into a Python list. diff --git a/docs/examples/tutorial/cython_tutorial/primes_cpp.pyx b/docs/examples/tutorial/cython_tutorial/primes_cpp.pyx index 57bfe9cc2..afef8bd13 100644 --- a/docs/examples/tutorial/cython_tutorial/primes_cpp.pyx +++ b/docs/examples/tutorial/cython_tutorial/primes_cpp.pyx @@ -1,21 +1,22 @@ -# distutils: language=c++
-
-from libcpp.vector cimport vector
-
-def primes(unsigned int nb_primes):
- cdef int n, i
- cdef vector[int] p
- p.reserve(nb_primes) # allocate memory for 'nb_primes' elements.
-
- n = 2
- while p.size() < nb_primes: # size() for vectors is similar to len()
- for i in p:
- if n % i == 0:
- break
- else:
- p.push_back(n) # push_back is similar to append()
- n += 1
-
- # Vectors are automatically converted to Python
- # lists when converted to Python objects.
- return p
+# distutils: language=c++ + + +from libcpp.vector cimport vector + +def primes(unsigned int nb_primes): + cdef int n, i + cdef vector[int] p + p.reserve(nb_primes) # allocate memory for 'nb_primes' elements. + + n = 2 + while p.size() < nb_primes: # size() for vectors is similar to len() + for i in p: + if n % i == 0: + break + else: + p.push_back(n) # push_back is similar to append() + n += 1 + + # If possible, C values and C++ objects are automatically + # converted to Python objects at need. + return p # so here, the vector will be copied into a Python list. diff --git a/docs/examples/tutorial/cython_tutorial/primes_python.py b/docs/examples/tutorial/cython_tutorial/primes_python.py index f6559d519..845af5bbf 100644 --- a/docs/examples/tutorial/cython_tutorial/primes_python.py +++ b/docs/examples/tutorial/cython_tutorial/primes_python.py @@ -1,4 +1,4 @@ -def primes_python(nb_primes): +def primes(nb_primes): p = [] n = 2 while len(p) < nb_primes: diff --git a/docs/examples/tutorial/cython_tutorial/setup.py b/docs/examples/tutorial/cython_tutorial/setup.py deleted file mode 100644 index 302a08e5f..000000000 --- a/docs/examples/tutorial/cython_tutorial/setup.py +++ /dev/null @@ -1,6 +0,0 @@ -from distutils.core import setup -from Cython.Build import cythonize - -setup( - ext_modules=cythonize("fib.pyx"), -) diff --git a/docs/examples/tutorial/embedding/embedded.pyx b/docs/examples/tutorial/embedding/embedded.pyx new file mode 100644 index 000000000..26704d45f --- /dev/null +++ b/docs/examples/tutorial/embedding/embedded.pyx @@ -0,0 +1,11 @@ +# embedded.pyx + +# The following two lines are for test purposed only, please ignore them. +# distutils: sources = embedded_main.c +# tag: py3only + +TEXT_TO_SAY = 'Hello from Python!' + +cdef public int say_hello_from_python() except -1: + print(TEXT_TO_SAY) + return 0 diff --git a/docs/examples/tutorial/embedding/embedded_main.c b/docs/examples/tutorial/embedding/embedded_main.c new file mode 100644 index 000000000..e14901a5e --- /dev/null +++ b/docs/examples/tutorial/embedding/embedded_main.c @@ -0,0 +1,69 @@ +/* embedded_main.c */ + +/* This include file is automatically generated by Cython for 'public' functions. */ +#include "embedded.h" + +#ifdef __cplusplus +extern "C" { +#endif + +int +main(int argc, char *argv[]) +{ + PyObject *pmodule; + wchar_t *program; + + program = Py_DecodeLocale(argv[0], NULL); + if (program == NULL) { + fprintf(stderr, "Fatal error: cannot decode argv[0], got %d arguments\n", argc); + exit(1); + } + + /* Add a built-in module, before Py_Initialize */ + if (PyImport_AppendInittab("embedded", PyInit_embedded) == -1) { + fprintf(stderr, "Error: could not extend in-built modules table\n"); + exit(1); + } + + /* Pass argv[0] to the Python interpreter */ + Py_SetProgramName(program); + + /* Initialize the Python interpreter. Required. + If this step fails, it will be a fatal error. */ + Py_Initialize(); + + /* Optionally import the module; alternatively, + import can be deferred until the embedded script + imports it. */ + pmodule = PyImport_ImportModule("embedded"); + if (!pmodule) { + PyErr_Print(); + fprintf(stderr, "Error: could not import module 'embedded'\n"); + goto exit_with_error; + } + + /* Now call into your module code. */ + if (say_hello_from_python() < 0) { + PyErr_Print(); + fprintf(stderr, "Error in Python code, exception was printed.\n"); + goto exit_with_error; + } + + /* ... */ + + /* Clean up after using CPython. */ + PyMem_RawFree(program); + Py_Finalize(); + + return 0; + + /* Clean up in the error cases above. */ +exit_with_error: + PyMem_RawFree(program); + Py_Finalize(); + return 1; +} + +#ifdef __cplusplus +} +#endif diff --git a/docs/examples/tutorial/external/atoi.py b/docs/examples/tutorial/external/atoi.py new file mode 100644 index 000000000..250b26a5c --- /dev/null +++ b/docs/examples/tutorial/external/atoi.py @@ -0,0 +1,6 @@ +from cython.cimports.libc.stdlib import atoi + +@cython.cfunc +def parse_charptr_to_py_int(s: cython.p_char): + assert s is not cython.NULL, "byte string value is NULL" + return atoi(s) # note: atoi() has no error detection! diff --git a/docs/examples/tutorial/external/atoi.pyx b/docs/examples/tutorial/external/atoi.pyx index 48643bbf2..ef1219854 100644 --- a/docs/examples/tutorial/external/atoi.pyx +++ b/docs/examples/tutorial/external/atoi.pyx @@ -1,5 +1,6 @@ -from libc.stdlib cimport atoi
-
-cdef parse_charptr_to_py_int(char* s):
- assert s is not NULL, "byte string value is NULL"
- return atoi(s) # note: atoi() has no error detection!
+from libc.stdlib cimport atoi + + +cdef parse_charptr_to_py_int(char* s): + assert s is not NULL, "byte string value is NULL" + return atoi(s) # note: atoi() has no error detection! diff --git a/docs/examples/tutorial/external/cpdef_sin.pyx b/docs/examples/tutorial/external/cpdef_sin.pyx index 47e09f433..eee2326a4 100644 --- a/docs/examples/tutorial/external/cpdef_sin.pyx +++ b/docs/examples/tutorial/external/cpdef_sin.pyx @@ -1,7 +1,7 @@ -"""
->>> sin(0)
-0.0
-"""
-
-cdef extern from "math.h":
- cpdef double sin(double x)
+""" +>>> sin(0) +0.0 +""" + +cdef extern from "math.h": + cpdef double sin(double x) diff --git a/docs/examples/tutorial/external/keyword_args.pyx b/docs/examples/tutorial/external/keyword_args.pyx index 327e4e08b..7c2a786cc 100644 --- a/docs/examples/tutorial/external/keyword_args.pyx +++ b/docs/examples/tutorial/external/keyword_args.pyx @@ -1,2 +1,2 @@ -cdef extern from "string.h":
- char* strstr(const char *haystack, const char *needle)
+cdef extern from "string.h": + char* strstr(const char *haystack, const char *needle) diff --git a/docs/examples/tutorial/external/keyword_args_call.py b/docs/examples/tutorial/external/keyword_args_call.py new file mode 100644 index 000000000..b3b3f5049 --- /dev/null +++ b/docs/examples/tutorial/external/keyword_args_call.py @@ -0,0 +1,7 @@ +from cython.cimports.strstr import strstr + +def main(): + data: cython.p_char = "hfvcakdfagbcffvschvxcdfgccbcfhvgcsnfxjh" + + pos = strstr(needle='akd', haystack=data) + print(pos is not cython.NULL) diff --git a/docs/examples/tutorial/external/keyword_args_call.pyx b/docs/examples/tutorial/external/keyword_args_call.pyx index 4be5f755d..de2b6f2b2 100644 --- a/docs/examples/tutorial/external/keyword_args_call.pyx +++ b/docs/examples/tutorial/external/keyword_args_call.pyx @@ -1,7 +1,7 @@ -cdef extern from "string.h":
- char* strstr(const char *haystack, const char *needle)
-
-cdef char* data = "hfvcakdfagbcffvschvxcdfgccbcfhvgcsnfxjh"
-
-cdef char* pos = strstr(needle='akd', haystack=data)
-print(pos is not NULL)
+cdef extern from "string.h": + char* strstr(const char *haystack, const char *needle) + +cdef char* data = "hfvcakdfagbcffvschvxcdfgccbcfhvgcsnfxjh" + +cdef char* pos = strstr(needle='akd', haystack=data) +print(pos is not NULL) diff --git a/docs/examples/tutorial/external/libc_sin.py b/docs/examples/tutorial/external/libc_sin.py new file mode 100644 index 000000000..f4223253d --- /dev/null +++ b/docs/examples/tutorial/external/libc_sin.py @@ -0,0 +1,5 @@ +from cython.cimports.libc.math import sin + +@cython.cfunc +def f(x: cython.double) -> cython.double: + return sin(x * x) diff --git a/docs/examples/tutorial/external/libc_sin.pyx b/docs/examples/tutorial/external/libc_sin.pyx index 25a4430e3..2de8444d6 100644 --- a/docs/examples/tutorial/external/libc_sin.pyx +++ b/docs/examples/tutorial/external/libc_sin.pyx @@ -1,4 +1,5 @@ -from libc.math cimport sin
-
-cdef double f(double x):
- return sin(x * x)
+from libc.math cimport sin + + +cdef double f(double x): + return sin(x * x) diff --git a/docs/examples/tutorial/external/py_version_hex.py b/docs/examples/tutorial/external/py_version_hex.py new file mode 100644 index 000000000..3b19d0d02 --- /dev/null +++ b/docs/examples/tutorial/external/py_version_hex.py @@ -0,0 +1,4 @@ +from cython.cimports.cpython.version import PY_VERSION_HEX + +# Python version >= 3.2 final ? +print(PY_VERSION_HEX >= 0x030200F0) diff --git a/docs/examples/tutorial/external/py_version_hex.pyx b/docs/examples/tutorial/external/py_version_hex.pyx index d732b00e7..e33f207c1 100644 --- a/docs/examples/tutorial/external/py_version_hex.pyx +++ b/docs/examples/tutorial/external/py_version_hex.pyx @@ -1,4 +1,4 @@ -from cpython.version cimport PY_VERSION_HEX
-
-# Python version >= 3.2 final ?
-print(PY_VERSION_HEX >= 0x030200F0)
+from cpython.version cimport PY_VERSION_HEX + +# Python version >= 3.2 final ? +print(PY_VERSION_HEX >= 0x030200F0) diff --git a/docs/examples/tutorial/external/setup.py b/docs/examples/tutorial/external/setup.py index 653214c84..289bc9534 100644 --- a/docs/examples/tutorial/external/setup.py +++ b/docs/examples/tutorial/external/setup.py @@ -1,13 +1,12 @@ -from distutils.core import setup
-from distutils.extension import Extension
-from Cython.Build import cythonize
-
-ext_modules = [
- Extension("demo",
- sources=["demo.pyx"],
- libraries=["m"] # Unix-like specific
- )
-]
-
-setup(name="Demos",
- ext_modules=cythonize(ext_modules))
+from setuptools import Extension, setup +from Cython.Build import cythonize + +ext_modules = [ + Extension("demo", + sources=["demo.pyx"], + libraries=["m"] # Unix-like specific + ) +] + +setup(name="Demos", + ext_modules=cythonize(ext_modules)) diff --git a/docs/examples/tutorial/external/strstr.pxd b/docs/examples/tutorial/external/strstr.pxd new file mode 100644 index 000000000..7c2a786cc --- /dev/null +++ b/docs/examples/tutorial/external/strstr.pxd @@ -0,0 +1,2 @@ +cdef extern from "string.h": + char* strstr(const char *haystack, const char *needle) diff --git a/docs/examples/tutorial/memory_allocation/malloc.py b/docs/examples/tutorial/memory_allocation/malloc.py new file mode 100644 index 000000000..fb7e82236 --- /dev/null +++ b/docs/examples/tutorial/memory_allocation/malloc.py @@ -0,0 +1,24 @@ +import random +from cython.cimports.libc.stdlib import malloc, free + +def random_noise(number: cython.int = 1): + i: cython.int + # allocate number * sizeof(double) bytes of memory + my_array: cython.p_double = cython.cast(cython.p_double, malloc( + number * cython.sizeof(cython.double))) + if not my_array: + raise MemoryError() + + try: + ran = random.normalvariate + for i in range(number): + my_array[i] = ran(0, 1) + + # ... let's just assume we do some more heavy C calculations here to make up + # for the work that it takes to pack the C double values into Python float + # objects below, right after throwing away the existing objects above. + + return [x for x in my_array[:number]] + finally: + # return the previously allocated memory to the system + free(my_array) diff --git a/docs/examples/tutorial/memory_allocation/malloc.pyx b/docs/examples/tutorial/memory_allocation/malloc.pyx index e01a378e3..6aa583aab 100644 --- a/docs/examples/tutorial/memory_allocation/malloc.pyx +++ b/docs/examples/tutorial/memory_allocation/malloc.pyx @@ -1,23 +1,24 @@ -import random
-from libc.stdlib cimport malloc, free
-
-def random_noise(int number=1):
- cdef int i
- # allocate number * sizeof(double) bytes of memory
- cdef double *my_array = <double *> malloc(number * sizeof(double))
- if not my_array:
- raise MemoryError()
-
- try:
- ran = random.normalvariate
- for i in range(number):
- my_array[i] = ran(0, 1)
-
- # ... let's just assume we do some more heavy C calculations here to make up
- # for the work that it takes to pack the C double values into Python float
- # objects below, right after throwing away the existing objects above.
-
- return [x for x in my_array[:number]]
- finally:
- # return the previously allocated memory to the system
- free(my_array)
+import random +from libc.stdlib cimport malloc, free + +def random_noise(int number=1): + cdef int i + # allocate number * sizeof(double) bytes of memory + cdef double *my_array = <double *> malloc( + number * sizeof(double)) + if not my_array: + raise MemoryError() + + try: + ran = random.normalvariate + for i in range(number): + my_array[i] = ran(0, 1) + + # ... let's just assume we do some more heavy C calculations here to make up + # for the work that it takes to pack the C double values into Python float + # objects below, right after throwing away the existing objects above. + + return [x for x in my_array[:number]] + finally: + # return the previously allocated memory to the system + free(my_array) diff --git a/docs/examples/tutorial/memory_allocation/some_memory.py b/docs/examples/tutorial/memory_allocation/some_memory.py new file mode 100644 index 000000000..31ad63a6e --- /dev/null +++ b/docs/examples/tutorial/memory_allocation/some_memory.py @@ -0,0 +1,27 @@ +from cython.cimports.cpython.mem import PyMem_Malloc, PyMem_Realloc, PyMem_Free + +@cython.cclass +class SomeMemory: + data: cython.p_double + + def __cinit__(self, number: cython.size_t): + # allocate some memory (uninitialised, may contain arbitrary data) + self.data = cython.cast(cython.p_double, PyMem_Malloc( + number * cython.sizeof(cython.double))) + if not self.data: + raise MemoryError() + + def resize(self, new_number: cython.size_t): + # Allocates new_number * sizeof(double) bytes, + # preserving the current content and making a best-effort to + # re-use the original data location. + mem = cython.cast(cython.p_double, PyMem_Realloc( + self.data, new_number * cython.sizeof(cython.double))) + if not mem: + raise MemoryError() + # Only overwrite the pointer if the memory was really reallocated. + # On error (mem is NULL), the originally memory has not been freed. + self.data = mem + + def __dealloc__(self): + PyMem_Free(self.data) # no-op if self.data is NULL diff --git a/docs/examples/tutorial/memory_allocation/some_memory.pyx b/docs/examples/tutorial/memory_allocation/some_memory.pyx index 2e639ac4d..e6bb63b77 100644 --- a/docs/examples/tutorial/memory_allocation/some_memory.pyx +++ b/docs/examples/tutorial/memory_allocation/some_memory.pyx @@ -1,25 +1,27 @@ -from cpython.mem cimport PyMem_Malloc, PyMem_Realloc, PyMem_Free
-
-cdef class SomeMemory:
-
- cdef double* data
-
- def __cinit__(self, size_t number):
- # allocate some memory (uninitialised, may contain arbitrary data)
- self.data = <double*> PyMem_Malloc(number * sizeof(double))
- if not self.data:
- raise MemoryError()
-
- def resize(self, size_t new_number):
- # Allocates new_number * sizeof(double) bytes,
- # preserving the current content and making a best-effort to
- # re-use the original data location.
- mem = <double*> PyMem_Realloc(self.data, new_number * sizeof(double))
- if not mem:
- raise MemoryError()
- # Only overwrite the pointer if the memory was really reallocated.
- # On error (mem is NULL), the originally memory has not been freed.
- self.data = mem
-
- def __dealloc__(self):
- PyMem_Free(self.data) # no-op if self.data is NULL
+from cpython.mem cimport PyMem_Malloc, PyMem_Realloc, PyMem_Free + + +cdef class SomeMemory: + cdef double* data + + def __cinit__(self, size_t number): + # allocate some memory (uninitialised, may contain arbitrary data) + self.data = <double*> PyMem_Malloc( + number * sizeof(double)) + if not self.data: + raise MemoryError() + + def resize(self, size_t new_number): + # Allocates new_number * sizeof(double) bytes, + # preserving the current content and making a best-effort to + # re-use the original data location. + mem = <double*> PyMem_Realloc( + self.data, new_number * sizeof(double)) + if not mem: + raise MemoryError() + # Only overwrite the pointer if the memory was really reallocated. + # On error (mem is NULL), the originally memory has not been freed. + self.data = mem + + def __dealloc__(self): + PyMem_Free(self.data) # no-op if self.data is NULL diff --git a/docs/examples/tutorial/numpy/convolve2.pyx b/docs/examples/tutorial/numpy/convolve2.pyx index be7512fe1..c122aeb3c 100644 --- a/docs/examples/tutorial/numpy/convolve2.pyx +++ b/docs/examples/tutorial/numpy/convolve2.pyx @@ -1,4 +1,4 @@ -# tag: numpy_old +# tag: numpy # You can ignore the previous line. # It's for internal testing of the cython documentation. diff --git a/docs/examples/tutorial/numpy/convolve_py.py b/docs/examples/tutorial/numpy/convolve_py.py index c3cbc5f86..39b276a04 100644 --- a/docs/examples/tutorial/numpy/convolve_py.py +++ b/docs/examples/tutorial/numpy/convolve_py.py @@ -1,43 +1,43 @@ -import numpy as np
-
-
-def naive_convolve(f, g):
- # f is an image and is indexed by (v, w)
- # g is a filter kernel and is indexed by (s, t),
- # it needs odd dimensions
- # h is the output image and is indexed by (x, y),
- # it is not cropped
- if g.shape[0] % 2 != 1 or g.shape[1] % 2 != 1:
- raise ValueError("Only odd dimensions on filter supported")
- # smid and tmid are number of pixels between the center pixel
- # and the edge, ie for a 5x5 filter they will be 2.
- #
- # The output size is calculated by adding smid, tmid to each
- # side of the dimensions of the input image.
- vmax = f.shape[0]
- wmax = f.shape[1]
- smax = g.shape[0]
- tmax = g.shape[1]
- smid = smax // 2
- tmid = tmax // 2
- xmax = vmax + 2 * smid
- ymax = wmax + 2 * tmid
- # Allocate result image.
- h = np.zeros([xmax, ymax], dtype=f.dtype)
- # Do convolution
- for x in range(xmax):
- for y in range(ymax):
- # Calculate pixel value for h at (x,y). Sum one component
- # for each pixel (s, t) of the filter g.
- s_from = max(smid - x, -smid)
- s_to = min((xmax - x) - smid, smid + 1)
- t_from = max(tmid - y, -tmid)
- t_to = min((ymax - y) - tmid, tmid + 1)
- value = 0
- for s in range(s_from, s_to):
- for t in range(t_from, t_to):
- v = x - smid + s
- w = y - tmid + t
- value += g[smid - s, tmid - t] * f[v, w]
- h[x, y] = value
- return h
+import numpy as np + + +def naive_convolve(f, g): + # f is an image and is indexed by (v, w) + # g is a filter kernel and is indexed by (s, t), + # it needs odd dimensions + # h is the output image and is indexed by (x, y), + # it is not cropped + if g.shape[0] % 2 != 1 or g.shape[1] % 2 != 1: + raise ValueError("Only odd dimensions on filter supported") + # smid and tmid are number of pixels between the center pixel + # and the edge, ie for a 5x5 filter they will be 2. + # + # The output size is calculated by adding smid, tmid to each + # side of the dimensions of the input image. + vmax = f.shape[0] + wmax = f.shape[1] + smax = g.shape[0] + tmax = g.shape[1] + smid = smax // 2 + tmid = tmax // 2 + xmax = vmax + 2 * smid + ymax = wmax + 2 * tmid + # Allocate result image. + h = np.zeros([xmax, ymax], dtype=f.dtype) + # Do convolution + for x in range(xmax): + for y in range(ymax): + # Calculate pixel value for h at (x,y). Sum one component + # for each pixel (s, t) of the filter g. + s_from = max(smid - x, -smid) + s_to = min((xmax - x) - smid, smid + 1) + t_from = max(tmid - y, -tmid) + t_to = min((ymax - y) - tmid, tmid + 1) + value = 0 + for s in range(s_from, s_to): + for t in range(t_from, t_to): + v = x - smid + s + w = y - tmid + t + value += g[smid - s, tmid - t] * f[v, w] + h[x, y] = value + return h diff --git a/docs/examples/tutorial/profiling_tutorial/calc_pi.py b/docs/examples/tutorial/profiling_tutorial/calc_pi.py index bc265e560..08c9ec2d4 100644 --- a/docs/examples/tutorial/profiling_tutorial/calc_pi.py +++ b/docs/examples/tutorial/profiling_tutorial/calc_pi.py @@ -1,10 +1,10 @@ -# calc_pi.py
-
-def recip_square(i):
- return 1. / i ** 2
-
-def approx_pi(n=10000000):
- val = 0.
- for k in range(1, n + 1):
- val += recip_square(k)
- return (6 * val) ** .5
+# calc_pi.py + +def recip_square(i): + return 1. / i ** 2 + +def approx_pi(n=10000000): + val = 0. + for k in range(1, n + 1): + val += recip_square(k) + return (6 * val) ** .5 diff --git a/docs/examples/tutorial/profiling_tutorial/calc_pi_2.pyx b/docs/examples/tutorial/profiling_tutorial/calc_pi_2.pyx index dab8d238d..c4c6a1bb9 100644 --- a/docs/examples/tutorial/profiling_tutorial/calc_pi_2.pyx +++ b/docs/examples/tutorial/profiling_tutorial/calc_pi_2.pyx @@ -1,13 +1,13 @@ -# cython: profile=True
-
-# calc_pi.pyx
-
-def recip_square(int i):
- return 1. / i ** 2
-
-def approx_pi(int n=10000000):
- cdef double val = 0.
- cdef int k
- for k in range(1, n + 1):
- val += recip_square(k)
- return (6 * val) ** .5
+# cython: profile=True + +# calc_pi.pyx + +def recip_square(int i): + return 1. / i ** 2 + +def approx_pi(int n=10000000): + cdef double val = 0. + cdef int k + for k in range(1, n + 1): + val += recip_square(k) + return (6 * val) ** .5 diff --git a/docs/examples/tutorial/profiling_tutorial/calc_pi_3.pyx b/docs/examples/tutorial/profiling_tutorial/calc_pi_3.pyx index 0f0bdb18a..8636e659c 100644 --- a/docs/examples/tutorial/profiling_tutorial/calc_pi_3.pyx +++ b/docs/examples/tutorial/profiling_tutorial/calc_pi_3.pyx @@ -1,13 +1,13 @@ -# cython: profile=True
-
-# calc_pi.pyx
-
-cdef inline double recip_square(int i):
- return 1. / (i * i)
-
-def approx_pi(int n=10000000):
- cdef double val = 0.
- cdef int k
- for k in range(1, n + 1):
- val += recip_square(k)
- return (6 * val) ** .5
+# cython: profile=True + +# calc_pi.pyx + +cdef inline double recip_square(int i) except -1.0: + return 1. / (i * i) + +def approx_pi(int n=10000000): + cdef double val = 0. + cdef int k + for k in range(1, n + 1): + val += recip_square(k) + return (6 * val) ** .5 diff --git a/docs/examples/tutorial/profiling_tutorial/calc_pi_4.pyx b/docs/examples/tutorial/profiling_tutorial/calc_pi_4.pyx index ab3f9ea9f..bd4783af1 100644 --- a/docs/examples/tutorial/profiling_tutorial/calc_pi_4.pyx +++ b/docs/examples/tutorial/profiling_tutorial/calc_pi_4.pyx @@ -1,16 +1,16 @@ -# cython: profile=True
-
-# calc_pi.pyx
-
-cimport cython
-
-@cython.profile(False)
-cdef inline double recip_square(int i):
- return 1. / (i * i)
-
-def approx_pi(int n=10000000):
- cdef double val = 0.
- cdef int k
- for k in range(1, n + 1):
- val += recip_square(k)
- return (6 * val) ** .5
+# cython: profile=True + +# calc_pi.pyx + +cimport cython + +@cython.profile(False) +cdef inline double recip_square(int i) except -1.0: + return 1. / (i * i) + +def approx_pi(int n=10000000): + cdef double val = 0. + cdef int k + for k in range(1, n + 1): + val += recip_square(k) + return (6 * val) ** .5 diff --git a/docs/examples/tutorial/profiling_tutorial/often_called.pyx b/docs/examples/tutorial/profiling_tutorial/often_called.pyx index 77c689c9c..3699dce4f 100644 --- a/docs/examples/tutorial/profiling_tutorial/often_called.pyx +++ b/docs/examples/tutorial/profiling_tutorial/often_called.pyx @@ -1,5 +1,5 @@ -cimport cython
-
-@cython.profile(False)
-def my_often_called_function():
- pass
+cimport cython + +@cython.profile(False) +def my_often_called_function(): + pass diff --git a/docs/examples/tutorial/profiling_tutorial/profile.py b/docs/examples/tutorial/profiling_tutorial/profile.py index 1c12bf971..60e90c8ba 100644 --- a/docs/examples/tutorial/profiling_tutorial/profile.py +++ b/docs/examples/tutorial/profiling_tutorial/profile.py @@ -1,10 +1,10 @@ -# profile.py
-
-import pstats, cProfile
-
-import calc_pi
-
-cProfile.runctx("calc_pi.approx_pi()", globals(), locals(), "Profile.prof")
-
-s = pstats.Stats("Profile.prof")
-s.strip_dirs().sort_stats("time").print_stats()
+# profile.py + +import pstats, cProfile + +import calc_pi + +cProfile.runctx("calc_pi.approx_pi()", globals(), locals(), "Profile.prof") + +s = pstats.Stats("Profile.prof") +s.strip_dirs().sort_stats("time").print_stats() diff --git a/docs/examples/tutorial/profiling_tutorial/profile_2.py b/docs/examples/tutorial/profiling_tutorial/profile_2.py index 38fb50104..b0e91b07f 100644 --- a/docs/examples/tutorial/profiling_tutorial/profile_2.py +++ b/docs/examples/tutorial/profiling_tutorial/profile_2.py @@ -1,13 +1,13 @@ -# profile.py
-
-import pstats, cProfile
-
-import pyximport
-pyximport.install()
-
-import calc_pi
-
-cProfile.runctx("calc_pi.approx_pi()", globals(), locals(), "Profile.prof")
-
-s = pstats.Stats("Profile.prof")
-s.strip_dirs().sort_stats("time").print_stats()
+# profile.py + +import pstats, cProfile + +import pyximport +pyximport.install() + +import calc_pi + +cProfile.runctx("calc_pi.approx_pi()", globals(), locals(), "Profile.prof") + +s = pstats.Stats("Profile.prof") +s.strip_dirs().sort_stats("time").print_stats() diff --git a/docs/examples/tutorial/pure/A.py b/docs/examples/tutorial/pure/A.py index 1e0cea950..98d5530c8 100644 --- a/docs/examples/tutorial/pure/A.py +++ b/docs/examples/tutorial/pure/A.py @@ -1,14 +1,14 @@ -def myfunction(x, y=2):
- a = x - y
- return a + x * y
-
-def _helper(a):
- return a + 1
-
-class A:
- def __init__(self, b=0):
- self.a = 3
- self.b = b
-
- def foo(self, x):
- print(x + _helper(1.0))
+def myfunction(x, y=2): + a = x - y + return a + x * y + +def _helper(a): + return a + 1 + +class A: + def __init__(self, b=0): + self.a = 3 + self.b = b + + def foo(self, x): + print(x + _helper(1.0)) diff --git a/docs/examples/tutorial/pure/A_equivalent.pyx b/docs/examples/tutorial/pure/A_equivalent.pyx index 1b256a010..ab9e0081c 100644 --- a/docs/examples/tutorial/pure/A_equivalent.pyx +++ b/docs/examples/tutorial/pure/A_equivalent.pyx @@ -1,15 +1,15 @@ -cpdef int myfunction(int x, int y=2):
- a = x - y
- return a + x * y
-
-cdef double _helper(double a):
- return a + 1
-
-cdef class A:
- cdef public int a, b
- def __init__(self, b=0):
- self.a = 3
- self.b = b
-
- cpdef foo(self, double x):
- print(x + _helper(1.0))
+cpdef int myfunction(int x, int y=2): + a = x - y + return a + x * y + +cdef double _helper(double a): + return a + 1 + +cdef class A: + cdef public int a, b + def __init__(self, b=0): + self.a = 3 + self.b = b + + cpdef foo(self, double x): + print(x + _helper(1.0)) diff --git a/docs/examples/tutorial/pure/annotations.py b/docs/examples/tutorial/pure/annotations.py index 2b8487c0b..09682c352 100644 --- a/docs/examples/tutorial/pure/annotations.py +++ b/docs/examples/tutorial/pure/annotations.py @@ -1,5 +1,5 @@ -import cython
-
-def func(foo: dict, bar: cython.int) -> tuple:
- foo["hello world"] = 3 + bar
- return foo, 5
+import cython + +def func(foo: dict, bar: cython.int) -> tuple: + foo["hello world"] = 3 + bar + return foo, 5 diff --git a/docs/examples/tutorial/pure/c_arrays.py b/docs/examples/tutorial/pure/c_arrays.py index 33067da68..f221b7ae8 100644 --- a/docs/examples/tutorial/pure/c_arrays.py +++ b/docs/examples/tutorial/pure/c_arrays.py @@ -1,15 +1,15 @@ -import cython
-
-
-@cython.locals(counts=cython.int[10], digit=cython.int)
-def count_digits(digits):
- """
- >>> digits = '01112222333334445667788899'
- >>> count_digits(map(int, digits))
- [1, 3, 4, 5, 3, 1, 2, 2, 3, 2]
- """
- counts = [0] * 10
- for digit in digits:
- assert 0 <= digit <= 9
- counts[digit] += 1
- return counts
+import cython + + +@cython.locals(counts=cython.int[10], digit=cython.int) +def count_digits(digits): + """ + >>> digits = '01112222333334445667788899' + >>> count_digits(map(int, digits)) + [1, 3, 4, 5, 3, 1, 2, 2, 3, 2] + """ + counts = [0] * 10 + for digit in digits: + assert 0 <= digit <= 9 + counts[digit] += 1 + return counts diff --git a/docs/examples/tutorial/pure/cclass.py b/docs/examples/tutorial/pure/cclass.py index 61c65183d..7f9cb1a04 100644 --- a/docs/examples/tutorial/pure/cclass.py +++ b/docs/examples/tutorial/pure/cclass.py @@ -1,16 +1,16 @@ -import cython
-
-
-@cython.cclass
-class A:
- cython.declare(a=cython.int, b=cython.int)
- c = cython.declare(cython.int, visibility='public')
- d = cython.declare(cython.int) # private by default.
- e = cython.declare(cython.int, visibility='readonly')
-
- def __init__(self, a, b, c, d=5, e=3):
- self.a = a
- self.b = b
- self.c = c
- self.d = d
- self.e = e
+import cython + + +@cython.cclass +class A: + cython.declare(a=cython.int, b=cython.int) + c = cython.declare(cython.int, visibility='public') + d = cython.declare(cython.int) # private by default. + e = cython.declare(cython.int, visibility='readonly') + + def __init__(self, a, b, c, d=5, e=3): + self.a = a + self.b = b + self.c = c + self.d = d + self.e = e diff --git a/docs/examples/tutorial/pure/compiled_switch.py b/docs/examples/tutorial/pure/compiled_switch.py index 47d62a3e6..a35cac2c6 100644 --- a/docs/examples/tutorial/pure/compiled_switch.py +++ b/docs/examples/tutorial/pure/compiled_switch.py @@ -1,6 +1,6 @@ -import cython
-
-if cython.compiled:
- print("Yep, I'm compiled.")
-else:
- print("Just a lowly interpreted script.")
+import cython + +if cython.compiled: + print("Yep, I'm compiled.") +else: + print("Just a lowly interpreted script.") diff --git a/docs/examples/tutorial/pure/cython_declare.py b/docs/examples/tutorial/pure/cython_declare.py index cf6d58bba..50a4ab8aa 100644 --- a/docs/examples/tutorial/pure/cython_declare.py +++ b/docs/examples/tutorial/pure/cython_declare.py @@ -1,4 +1,4 @@ -import cython
-
-x = cython.declare(cython.int) # cdef int x
-y = cython.declare(cython.double, 0.57721) # cdef double y = 0.57721
+import cython + +x = cython.declare(cython.int) # cdef int x +y = cython.declare(cython.double, 0.57721) # cdef double y = 0.57721 diff --git a/docs/examples/tutorial/pure/cython_declare2.py b/docs/examples/tutorial/pure/cython_declare2.py index 35fae7d7b..ee491d62b 100644 --- a/docs/examples/tutorial/pure/cython_declare2.py +++ b/docs/examples/tutorial/pure/cython_declare2.py @@ -1,3 +1,3 @@ -import cython
-
-cython.declare(x=cython.int, y=cython.double) # cdef int x; cdef double y
+import cython + +cython.declare(x=cython.int, y=cython.double) # cdef int x; cdef double y diff --git a/docs/examples/tutorial/pure/dostuff.py b/docs/examples/tutorial/pure/dostuff.py index 748c31b10..7a88533c5 100644 --- a/docs/examples/tutorial/pure/dostuff.py +++ b/docs/examples/tutorial/pure/dostuff.py @@ -1,5 +1,5 @@ -def dostuff(n):
- t = 0
- for i in range(n):
- t += i
- return t
+def dostuff(n): + t = 0 + for i in range(n): + t += i + return t diff --git a/docs/examples/tutorial/pure/exceptval.py b/docs/examples/tutorial/pure/exceptval.py index 8bf564040..3d991f7c1 100644 --- a/docs/examples/tutorial/pure/exceptval.py +++ b/docs/examples/tutorial/pure/exceptval.py @@ -1,7 +1,7 @@ -import cython
-
-@cython.exceptval(-1)
-def func(x: cython.int) -> cython.int:
- if x < 0:
- raise ValueError("need integer >= 0")
- return x + 1
+import cython + +@cython.exceptval(-1) +def func(x: cython.int) -> cython.int: + if x < 0: + raise ValueError("need integer >= 0") + return x + 1 diff --git a/docs/examples/tutorial/pure/locals.py b/docs/examples/tutorial/pure/locals.py index 8eda7114a..b273a9ebe 100644 --- a/docs/examples/tutorial/pure/locals.py +++ b/docs/examples/tutorial/pure/locals.py @@ -1,6 +1,6 @@ -import cython
-
-@cython.locals(a=cython.long, b=cython.long, n=cython.longlong)
-def foo(a, b, x, y):
- n = a * b
- # ...
+import cython + +@cython.locals(a=cython.long, b=cython.long, n=cython.longlong) +def foo(a, b, x, y): + n = a * b + # ... diff --git a/docs/examples/tutorial/pure/mymodule.py b/docs/examples/tutorial/pure/mymodule.py index 62d4c76ac..83f5cdc28 100644 --- a/docs/examples/tutorial/pure/mymodule.py +++ b/docs/examples/tutorial/pure/mymodule.py @@ -1,10 +1,10 @@ -# mymodule.py
-
-import cython
-
-# override with Python import if not in compiled code
-if not cython.compiled:
- from math import sin
-
-# calls sin() from math.h when compiled with Cython and math.sin() in Python
-print(sin(0))
+# mymodule.py + +import cython + +# override with Python import if not in compiled code +if not cython.compiled: + from math import sin + +# calls sin() from math.h when compiled with Cython and math.sin() in Python +print(sin(0)) diff --git a/docs/examples/tutorial/pure/pep_526.py b/docs/examples/tutorial/pure/pep_526.py index 163d97859..ecb3bfcdf 100644 --- a/docs/examples/tutorial/pure/pep_526.py +++ b/docs/examples/tutorial/pure/pep_526.py @@ -1,22 +1,22 @@ -import cython
-
-def func():
- # Cython types are evaluated as for cdef declarations
- x: cython.int # cdef int x
- y: cython.double = 0.57721 # cdef double y = 0.57721
- z: cython.float = 0.57721 # cdef float z = 0.57721
-
- # Python types shadow Cython types for compatibility reasons
- a: float = 0.54321 # cdef double a = 0.54321
- b: int = 5 # cdef object b = 5
- c: long = 6 # cdef object c = 6
- pass
-
-@cython.cclass
-class A:
- a: cython.int
- b: cython.int
-
- def __init__(self, b=0):
- self.a = 3
- self.b = b
+import cython + +def func(): + # Cython types are evaluated as for cdef declarations + x: cython.int # cdef int x + y: cython.double = 0.57721 # cdef double y = 0.57721 + z: cython.float = 0.57721 # cdef float z = 0.57721 + + # Python types shadow Cython types for compatibility reasons + a: float = 0.54321 # cdef double a = 0.54321 + b: int = 5 # cdef object b = 5 + c: long = 6 # cdef object c = 6 + pass + +@cython.cclass +class A: + a: cython.int + b: cython.int + + def __init__(self, b=0): + self.a = 3 + self.b = b diff --git a/docs/examples/tutorial/pure/py_cimport.py b/docs/examples/tutorial/pure/py_cimport.py new file mode 100644 index 000000000..233ddde7e --- /dev/null +++ b/docs/examples/tutorial/pure/py_cimport.py @@ -0,0 +1,5 @@ + +from cython.cimports.libc import math + +def use_libc_math(): + return math.ceil(5.5) diff --git a/docs/examples/tutorial/string/api_func.pyx b/docs/examples/tutorial/string/api_func.pyx index ec6b27751..c9e05f9e3 100644 --- a/docs/examples/tutorial/string/api_func.pyx +++ b/docs/examples/tutorial/string/api_func.pyx @@ -1,5 +1,5 @@ -from to_unicode cimport _text
-
-def api_func(s):
- text_input = _text(s)
- # ...
+from to_unicode cimport _text + +def api_func(s): + text_input = _text(s) + # ... diff --git a/docs/examples/tutorial/string/arg_memview.pyx b/docs/examples/tutorial/string/arg_memview.pyx index 63a18f943..e2b6d75be 100644 --- a/docs/examples/tutorial/string/arg_memview.pyx +++ b/docs/examples/tutorial/string/arg_memview.pyx @@ -1,5 +1,5 @@ -def process_byte_data(unsigned char[:] data):
- length = data.shape[0]
- first_byte = data[0]
- slice_view = data[1:-1]
- # ...
+def process_byte_data(unsigned char[:] data): + length = data.shape[0] + first_byte = data[0] + slice_view = data[1:-1] + # ... diff --git a/docs/examples/tutorial/string/auto_conversion_1.pyx b/docs/examples/tutorial/string/auto_conversion_1.pyx index 929c03d68..ee2b116e4 100644 --- a/docs/examples/tutorial/string/auto_conversion_1.pyx +++ b/docs/examples/tutorial/string/auto_conversion_1.pyx @@ -1,9 +1,9 @@ -# cython: c_string_type=unicode, c_string_encoding=utf8
-
-cdef char* c_string = 'abcdefg'
-
-# implicit decoding:
-cdef object py_unicode_object = c_string
-
-# explicit conversion to Python bytes:
-py_bytes_object = <bytes>c_string
+# cython: c_string_type=unicode, c_string_encoding=utf8 + +cdef char* c_string = 'abcdefg' + +# implicit decoding: +cdef object py_unicode_object = c_string + +# explicit conversion to Python bytes: +py_bytes_object = <bytes>c_string diff --git a/docs/examples/tutorial/string/auto_conversion_2.pyx b/docs/examples/tutorial/string/auto_conversion_2.pyx index 84436661d..9f7a5ad04 100644 --- a/docs/examples/tutorial/string/auto_conversion_2.pyx +++ b/docs/examples/tutorial/string/auto_conversion_2.pyx @@ -1,12 +1,12 @@ -# cython: c_string_type=str, c_string_encoding=ascii
-
-cdef char* c_string = 'abcdefg'
-
-# implicit decoding in Py3, bytes conversion in Py2:
-cdef object py_str_object = c_string
-
-# explicit conversion to Python bytes:
-py_bytes_object = <bytes>c_string
-
-# explicit conversion to Python unicode:
-py_bytes_object = <unicode>c_string
+# cython: c_string_type=str, c_string_encoding=ascii + +cdef char* c_string = 'abcdefg' + +# implicit decoding in Py3, bytes conversion in Py2: +cdef object py_str_object = c_string + +# explicit conversion to Python bytes: +py_bytes_object = <bytes>c_string + +# explicit conversion to Python unicode: +py_bytes_object = <unicode>c_string diff --git a/docs/examples/tutorial/string/auto_conversion_3.pyx b/docs/examples/tutorial/string/auto_conversion_3.pyx index 509d7e324..b9a1b7517 100644 --- a/docs/examples/tutorial/string/auto_conversion_3.pyx +++ b/docs/examples/tutorial/string/auto_conversion_3.pyx @@ -1,6 +1,6 @@ -# cython: c_string_type=unicode, c_string_encoding=ascii
-
-def func():
- ustring = u'abc'
- cdef char* s = ustring
- return s[0] # returns u'a'
+# cython: c_string_type=unicode, c_string_encoding=ascii + +def func(): + ustring = u'abc' + cdef char* s = ustring + return s[0] # returns u'a' diff --git a/docs/examples/tutorial/string/c_func.pyx b/docs/examples/tutorial/string/c_func.pyx index a456b815b..4763f2671 100644 --- a/docs/examples/tutorial/string/c_func.pyx +++ b/docs/examples/tutorial/string/c_func.pyx @@ -1,22 +1,23 @@ -from libc.stdlib cimport malloc
-from libc.string cimport strcpy, strlen
-
-cdef char* hello_world = 'hello world'
-cdef Py_ssize_t n = strlen(hello_world)
-
-
-cdef char* c_call_returning_a_c_string():
- cdef char* c_string = <char *> malloc((n + 1) * sizeof(char))
- if not c_string:
- raise MemoryError()
- strcpy(c_string, hello_world)
- return c_string
-
-
-cdef void get_a_c_string(char** c_string_ptr, Py_ssize_t *length):
- c_string_ptr[0] = <char *> malloc((n + 1) * sizeof(char))
- if not c_string_ptr[0]:
- raise MemoryError()
-
- strcpy(c_string_ptr[0], hello_world)
- length[0] = n
+from libc.stdlib cimport malloc +from libc.string cimport strcpy, strlen + +cdef char* hello_world = 'hello world' +cdef Py_ssize_t n = strlen(hello_world) + + +cdef char* c_call_returning_a_c_string(): + cdef char* c_string = <char *> malloc((n + 1) * sizeof(char)) + if not c_string: + return NULL # malloc failed + + strcpy(c_string, hello_world) + return c_string + + +cdef void get_a_c_string(char** c_string_ptr, Py_ssize_t *length): + c_string_ptr[0] = <char *> malloc((n + 1) * sizeof(char)) + if not c_string_ptr[0]: + return # malloc failed + + strcpy(c_string_ptr[0], hello_world) + length[0] = n diff --git a/docs/examples/tutorial/string/const.pyx b/docs/examples/tutorial/string/const.pyx index e066c77ac..0b89b9e41 100644 --- a/docs/examples/tutorial/string/const.pyx +++ b/docs/examples/tutorial/string/const.pyx @@ -1,4 +1,4 @@ -cdef extern from "someheader.h":
- ctypedef const char specialChar
- int process_string(const char* s)
- const unsigned char* look_up_cached_string(const unsigned char* key)
+cdef extern from "someheader.h": + ctypedef const char specialChar + int process_string(const char* s) + const unsigned char* look_up_cached_string(const unsigned char* key) diff --git a/docs/examples/tutorial/string/cpp_string.pyx b/docs/examples/tutorial/string/cpp_string.pyx index b6c9e44f9..313e72a67 100644 --- a/docs/examples/tutorial/string/cpp_string.pyx +++ b/docs/examples/tutorial/string/cpp_string.pyx @@ -1,12 +1,12 @@ -# distutils: language = c++
-
-from libcpp.string cimport string
-
-def get_bytes():
- py_bytes_object = b'hello world'
- cdef string s = py_bytes_object
-
- s.append('abc')
- py_bytes_object = s
- return py_bytes_object
-
+# distutils: language = c++ + +from libcpp.string cimport string + +def get_bytes(): + py_bytes_object = b'hello world' + cdef string s = py_bytes_object + + s.append('abc') + py_bytes_object = s + return py_bytes_object + diff --git a/docs/examples/tutorial/string/decode.pyx b/docs/examples/tutorial/string/decode.pyx index c3e2faf68..79fc41835 100644 --- a/docs/examples/tutorial/string/decode.pyx +++ b/docs/examples/tutorial/string/decode.pyx @@ -1,9 +1,9 @@ -from c_func cimport get_a_c_string
-
-cdef char* c_string = NULL
-cdef Py_ssize_t length = 0
-
-# get pointer and length from a C function
-get_a_c_string(&c_string, &length)
-
-ustring = c_string[:length].decode('UTF-8')
+from c_func cimport get_a_c_string + +cdef char* c_string = NULL +cdef Py_ssize_t length = 0 + +# get pointer and length from a C function +get_a_c_string(&c_string, &length) + +ustring = c_string[:length].decode('UTF-8') diff --git a/docs/examples/tutorial/string/decode_cpp_string.pyx b/docs/examples/tutorial/string/decode_cpp_string.pyx index 8f1d01af8..52861c209 100644 --- a/docs/examples/tutorial/string/decode_cpp_string.pyx +++ b/docs/examples/tutorial/string/decode_cpp_string.pyx @@ -1,10 +1,10 @@ -# distutils: language = c++
-
-from libcpp.string cimport string
-
-def get_ustrings():
- cdef string s = string(b'abcdefg')
-
- ustring1 = s.decode('UTF-8')
- ustring2 = s[2:-2].decode('UTF-8')
- return ustring1, ustring2
+# distutils: language = c++ + +from libcpp.string cimport string + +def get_ustrings(): + cdef string s = string(b'abcdefg') + + ustring1 = s.decode('UTF-8') + ustring2 = s[2:-2].decode('UTF-8') + return ustring1, ustring2 diff --git a/docs/examples/tutorial/string/for_bytes.pyx b/docs/examples/tutorial/string/for_bytes.pyx index 69e9202ae..d4d3e1f81 100644 --- a/docs/examples/tutorial/string/for_bytes.pyx +++ b/docs/examples/tutorial/string/for_bytes.pyx @@ -1,6 +1,6 @@ -cdef bytes bytes_string = b"hello to A bytes' world"
-
-cdef char c
-for c in bytes_string:
- if c == 'A':
- print("Found the letter A")
+cdef bytes bytes_string = b"hello to A bytes' world" + +cdef char c +for c in bytes_string: + if c == 'A': + print("Found the letter A") diff --git a/docs/examples/tutorial/string/for_char.pyx b/docs/examples/tutorial/string/for_char.pyx index adc16bcd8..81abae97c 100644 --- a/docs/examples/tutorial/string/for_char.pyx +++ b/docs/examples/tutorial/string/for_char.pyx @@ -1,6 +1,6 @@ -cdef char* c_string = "Hello to A C-string's world"
-
-cdef char c
-for c in c_string[:11]:
- if c == 'A':
- print("Found the letter A")
+cdef char* c_string = "Hello to A C-string's world" + +cdef char c +for c in c_string[:11]: + if c == 'A': + print("Found the letter A") diff --git a/docs/examples/tutorial/string/for_unicode.pyx b/docs/examples/tutorial/string/for_unicode.pyx index aabd562e4..0f8ab0c7d 100644 --- a/docs/examples/tutorial/string/for_unicode.pyx +++ b/docs/examples/tutorial/string/for_unicode.pyx @@ -1,6 +1,6 @@ -cdef unicode ustring = u'Hello world'
-
-# NOTE: no typing required for 'uchar' !
-for uchar in ustring:
- if uchar == u'A':
- print("Found the letter A")
+cdef unicode ustring = u'Hello world' + +# NOTE: no typing required for 'uchar' ! +for uchar in ustring: + if uchar == u'A': + print("Found the letter A") diff --git a/docs/examples/tutorial/string/if_char_in.pyx b/docs/examples/tutorial/string/if_char_in.pyx index 73521b2de..e33e18d59 100644 --- a/docs/examples/tutorial/string/if_char_in.pyx +++ b/docs/examples/tutorial/string/if_char_in.pyx @@ -1,5 +1,5 @@ -cpdef void is_in(Py_UCS4 uchar_val):
- if uchar_val in u'abcABCxY':
- print("The character is in the string.")
- else:
- print("The character is not in the string")
+cpdef void is_in(Py_UCS4 uchar_val): + if uchar_val in u'abcABCxY': + print("The character is in the string.") + else: + print("The character is not in the string") diff --git a/docs/examples/tutorial/string/naive_decode.pyx b/docs/examples/tutorial/string/naive_decode.pyx index 186d2affa..b3c44d9af 100644 --- a/docs/examples/tutorial/string/naive_decode.pyx +++ b/docs/examples/tutorial/string/naive_decode.pyx @@ -1,4 +1,4 @@ -from c_func cimport c_call_returning_a_c_string
-
-cdef char* some_c_string = c_call_returning_a_c_string()
-ustring = some_c_string.decode('UTF-8')
+from c_func cimport c_call_returning_a_c_string + +cdef char* some_c_string = c_call_returning_a_c_string() +ustring = some_c_string.decode('UTF-8') diff --git a/docs/examples/tutorial/string/return_memview.pyx b/docs/examples/tutorial/string/return_memview.pyx index f6233436a..be9b597a4 100644 --- a/docs/examples/tutorial/string/return_memview.pyx +++ b/docs/examples/tutorial/string/return_memview.pyx @@ -1,9 +1,9 @@ -def process_byte_data(unsigned char[:] data):
- # ... process the data, here, dummy processing.
- cdef bint return_all = (data[0] == 108)
-
- if return_all:
- return bytes(data)
- else:
- # example for returning a slice
- return bytes(data[5:7])
+def process_byte_data(unsigned char[:] data): + # ... process the data, here, dummy processing. + cdef bint return_all = (data[0] == 108) + + if return_all: + return bytes(data) + else: + # example for returning a slice + return bytes(data[5:7]) diff --git a/docs/examples/tutorial/string/slicing_c_string.pyx b/docs/examples/tutorial/string/slicing_c_string.pyx index 2e937430e..f8d272e32 100644 --- a/docs/examples/tutorial/string/slicing_c_string.pyx +++ b/docs/examples/tutorial/string/slicing_c_string.pyx @@ -1,15 +1,15 @@ -from libc.stdlib cimport free
-from c_func cimport get_a_c_string
-
-
-def main():
- cdef char* c_string = NULL
- cdef Py_ssize_t length = 0
-
- # get pointer and length from a C function
- get_a_c_string(&c_string, &length)
-
- try:
- py_bytes_string = c_string[:length] # Performs a copy of the data
- finally:
- free(c_string)
+from libc.stdlib cimport free +from c_func cimport get_a_c_string + + +def main(): + cdef char* c_string = NULL + cdef Py_ssize_t length = 0 + + # get pointer and length from a C function + get_a_c_string(&c_string, &length) + + try: + py_bytes_string = c_string[:length] # Performs a copy of the data + finally: + free(c_string) diff --git a/docs/examples/tutorial/string/to_char.pyx b/docs/examples/tutorial/string/to_char.pyx index 5e4a16161..7912ea485 100644 --- a/docs/examples/tutorial/string/to_char.pyx +++ b/docs/examples/tutorial/string/to_char.pyx @@ -1,8 +1,8 @@ -# define a global name for whatever char type is used in the module
-ctypedef unsigned char char_type
-
-cdef char_type[:] _chars(s):
- if isinstance(s, unicode):
- # encode to the specific encoding used inside of the module
- s = (<unicode>s).encode('utf8')
- return s
+# define a global name for whatever char type is used in the module +ctypedef unsigned char char_type + +cdef char_type[:] _chars(s): + if isinstance(s, unicode): + # encode to the specific encoding used inside of the module + s = (<unicode>s).encode('utf8') + return s diff --git a/docs/examples/tutorial/string/to_unicode.pyx b/docs/examples/tutorial/string/to_unicode.pyx index 8ab8f2662..188a8776e 100644 --- a/docs/examples/tutorial/string/to_unicode.pyx +++ b/docs/examples/tutorial/string/to_unicode.pyx @@ -1,22 +1,22 @@ -# to_unicode.pyx
-
-from cpython.version cimport PY_MAJOR_VERSION
-
-cdef unicode _text(s):
- if type(s) is unicode:
- # Fast path for most common case(s).
- return <unicode>s
-
- elif PY_MAJOR_VERSION < 3 and isinstance(s, bytes):
- # Only accept byte strings as text input in Python 2.x, not in Py3.
- return (<bytes>s).decode('ascii')
-
- elif isinstance(s, unicode):
- # We know from the fast path above that 's' can only be a subtype here.
- # An evil cast to <unicode> might still work in some(!) cases,
- # depending on what the further processing does. To be safe,
- # we can always create a copy instead.
- return unicode(s)
-
- else:
- raise TypeError("Could not convert to unicode.")
+# to_unicode.pyx + +from cpython.version cimport PY_MAJOR_VERSION + +cdef unicode _text(s): + if type(s) is unicode: + # Fast path for most common case(s). + return <unicode>s + + elif PY_MAJOR_VERSION < 3 and isinstance(s, bytes): + # Only accept byte strings as text input in Python 2.x, not in Py3. + return (<bytes>s).decode('ascii') + + elif isinstance(s, unicode): + # We know from the fast path above that 's' can only be a subtype here. + # An evil cast to <unicode> might still work in some(!) cases, + # depending on what the further processing does. To be safe, + # we can always create a copy instead. + return unicode(s) + + else: + raise TypeError("Could not convert to unicode.") diff --git a/docs/examples/tutorial/string/try_finally.pyx b/docs/examples/tutorial/string/try_finally.pyx index aea786f5b..4cf943e56 100644 --- a/docs/examples/tutorial/string/try_finally.pyx +++ b/docs/examples/tutorial/string/try_finally.pyx @@ -1,9 +1,9 @@ -from libc.stdlib cimport free
-from c_func cimport c_call_returning_a_c_string
-
-cdef bytes py_string
-cdef char* c_string = c_call_returning_a_c_string()
-try:
- py_string = c_string
-finally:
- free(c_string)
+from libc.stdlib cimport free +from c_func cimport c_call_returning_a_c_string + +cdef bytes py_string +cdef char* c_string = c_call_returning_a_c_string() +try: + py_string = c_string +finally: + free(c_string) diff --git a/docs/examples/tutorial/string/utf_eight.pyx b/docs/examples/tutorial/string/utf_eight.pyx index 7113947f4..654c51ae8 100644 --- a/docs/examples/tutorial/string/utf_eight.pyx +++ b/docs/examples/tutorial/string/utf_eight.pyx @@ -1,15 +1,15 @@ -from libc.stdlib cimport free
-
-cdef unicode tounicode(char* s):
- return s.decode('UTF-8', 'strict')
-
-cdef unicode tounicode_with_length(
- char* s, size_t length):
- return s[:length].decode('UTF-8', 'strict')
-
-cdef unicode tounicode_with_length_and_free(
- char* s, size_t length):
- try:
- return s[:length].decode('UTF-8', 'strict')
- finally:
+from libc.stdlib cimport free + +cdef unicode tounicode(char* s): + return s.decode('UTF-8', 'strict') + +cdef unicode tounicode_with_length( + char* s, size_t length): + return s[:length].decode('UTF-8', 'strict') + +cdef unicode tounicode_with_length_and_free( + char* s, size_t length): + try: + return s[:length].decode('UTF-8', 'strict') + finally: free(s)
\ No newline at end of file diff --git a/docs/examples/userguide/buffer/matrix.pyx b/docs/examples/userguide/buffer/matrix.pyx index abdb2d3c7..ca597c2f2 100644 --- a/docs/examples/userguide/buffer/matrix.pyx +++ b/docs/examples/userguide/buffer/matrix.pyx @@ -1,16 +1,16 @@ -# distutils: language = c++
-
-# matrix.pyx
-
-from libcpp.vector cimport vector
-
-cdef class Matrix:
- cdef unsigned ncols
- cdef vector[float] v
-
- def __cinit__(self, unsigned ncols):
- self.ncols = ncols
-
- def add_row(self):
- """Adds a row, initially zero-filled."""
- self.v.resize(self.v.size() + self.ncols)
+# distutils: language = c++ + +# matrix.pyx + +from libcpp.vector cimport vector + +cdef class Matrix: + cdef unsigned ncols + cdef vector[float] v + + def __cinit__(self, unsigned ncols): + self.ncols = ncols + + def add_row(self): + """Adds a row, initially zero-filled.""" + self.v.resize(self.v.size() + self.ncols) diff --git a/docs/examples/userguide/buffer/matrix_with_buffer.pyx b/docs/examples/userguide/buffer/matrix_with_buffer.pyx index 985991cbe..c355f0fe8 100644 --- a/docs/examples/userguide/buffer/matrix_with_buffer.pyx +++ b/docs/examples/userguide/buffer/matrix_with_buffer.pyx @@ -1,45 +1,45 @@ -# distutils: language = c++
-
-from cpython cimport Py_buffer
-from libcpp.vector cimport vector
-
-cdef class Matrix:
- cdef Py_ssize_t ncols
- cdef Py_ssize_t shape[2]
- cdef Py_ssize_t strides[2]
- cdef vector[float] v
-
- def __cinit__(self, Py_ssize_t ncols):
- self.ncols = ncols
-
- def add_row(self):
- """Adds a row, initially zero-filled."""
- self.v.resize(self.v.size() + self.ncols)
-
- def __getbuffer__(self, Py_buffer *buffer, int flags):
- cdef Py_ssize_t itemsize = sizeof(self.v[0])
-
- self.shape[0] = self.v.size() / self.ncols
- self.shape[1] = self.ncols
-
- # Stride 1 is the distance, in bytes, between two items in a row;
- # this is the distance between two adjacent items in the vector.
- # Stride 0 is the distance between the first elements of adjacent rows.
- self.strides[1] = <Py_ssize_t>( <char *>&(self.v[1])
- - <char *>&(self.v[0]))
- self.strides[0] = self.ncols * self.strides[1]
-
- buffer.buf = <char *>&(self.v[0])
- buffer.format = 'f' # float
- buffer.internal = NULL # see References
- buffer.itemsize = itemsize
- buffer.len = self.v.size() * itemsize # product(shape) * itemsize
- buffer.ndim = 2
- buffer.obj = self
- buffer.readonly = 0
- buffer.shape = self.shape
- buffer.strides = self.strides
- buffer.suboffsets = NULL # for pointer arrays only
-
- def __releasebuffer__(self, Py_buffer *buffer):
- pass
+# distutils: language = c++ + +from cpython cimport Py_buffer +from libcpp.vector cimport vector + +cdef class Matrix: + cdef Py_ssize_t ncols + cdef Py_ssize_t shape[2] + cdef Py_ssize_t strides[2] + cdef vector[float] v + + def __cinit__(self, Py_ssize_t ncols): + self.ncols = ncols + + def add_row(self): + """Adds a row, initially zero-filled.""" + self.v.resize(self.v.size() + self.ncols) + + def __getbuffer__(self, Py_buffer *buffer, int flags): + cdef Py_ssize_t itemsize = sizeof(self.v[0]) + + self.shape[0] = self.v.size() / self.ncols + self.shape[1] = self.ncols + + # Stride 1 is the distance, in bytes, between two items in a row; + # this is the distance between two adjacent items in the vector. + # Stride 0 is the distance between the first elements of adjacent rows. + self.strides[1] = <Py_ssize_t>( <char *>&(self.v[1]) + - <char *>&(self.v[0])) + self.strides[0] = self.ncols * self.strides[1] + + buffer.buf = <char *>&(self.v[0]) + buffer.format = 'f' # float + buffer.internal = NULL # see References + buffer.itemsize = itemsize + buffer.len = self.v.size() * itemsize # product(shape) * itemsize + buffer.ndim = 2 + buffer.obj = self + buffer.readonly = 0 + buffer.shape = self.shape + buffer.strides = self.strides + buffer.suboffsets = NULL # for pointer arrays only + + def __releasebuffer__(self, Py_buffer *buffer): + pass diff --git a/docs/examples/userguide/buffer/view_count.pyx b/docs/examples/userguide/buffer/view_count.pyx index ee8d5085d..8027f3ee9 100644 --- a/docs/examples/userguide/buffer/view_count.pyx +++ b/docs/examples/userguide/buffer/view_count.pyx @@ -1,29 +1,29 @@ -# distutils: language = c++
-
-from cpython cimport Py_buffer
-from libcpp.vector cimport vector
-
-cdef class Matrix:
-
- cdef int view_count
-
- cdef Py_ssize_t ncols
- cdef vector[float] v
- # ...
-
- def __cinit__(self, Py_ssize_t ncols):
- self.ncols = ncols
- self.view_count = 0
-
- def add_row(self):
- if self.view_count > 0:
- raise ValueError("can't add row while being viewed")
- self.v.resize(self.v.size() + self.ncols)
-
- def __getbuffer__(self, Py_buffer *buffer, int flags):
- # ... as before
-
- self.view_count += 1
-
- def __releasebuffer__(self, Py_buffer *buffer):
+# distutils: language = c++ + +from cpython cimport Py_buffer +from libcpp.vector cimport vector + +cdef class Matrix: + + cdef int view_count + + cdef Py_ssize_t ncols + cdef vector[float] v + # ... + + def __cinit__(self, Py_ssize_t ncols): + self.ncols = ncols + self.view_count = 0 + + def add_row(self): + if self.view_count > 0: + raise ValueError("can't add row while being viewed") + self.v.resize(self.v.size() + self.ncols) + + def __getbuffer__(self, Py_buffer *buffer, int flags): + # ... as before + + self.view_count += 1 + + def __releasebuffer__(self, Py_buffer *buffer): self.view_count -= 1
\ No newline at end of file diff --git a/docs/examples/userguide/early_binding_for_speed/rectangle.pyx b/docs/examples/userguide/early_binding_for_speed/rectangle.pyx index ffc5593b0..de70b0263 100644 --- a/docs/examples/userguide/early_binding_for_speed/rectangle.pyx +++ b/docs/examples/userguide/early_binding_for_speed/rectangle.pyx @@ -1,19 +1,19 @@ -cdef class Rectangle:
- cdef int x0, y0
- cdef int x1, y1
-
- def __init__(self, int x0, int y0, int x1, int y1):
- self.x0 = x0
- self.y0 = y0
- self.x1 = x1
- self.y1 = y1
-
- def area(self):
- area = (self.x1 - self.x0) * (self.y1 - self.y0)
- if area < 0:
- area = -area
- return area
-
-def rectArea(x0, y0, x1, y1):
- rect = Rectangle(x0, y0, x1, y1)
- return rect.area()
+cdef class Rectangle: + cdef int x0, y0 + cdef int x1, y1 + + def __init__(self, int x0, int y0, int x1, int y1): + self.x0 = x0 + self.y0 = y0 + self.x1 = x1 + self.y1 = y1 + + def area(self): + area = (self.x1 - self.x0) * (self.y1 - self.y0) + if area < 0: + area = -area + return area + +def rectArea(x0, y0, x1, y1): + rect = Rectangle(x0, y0, x1, y1) + return rect.area() diff --git a/docs/examples/userguide/early_binding_for_speed/rectangle_cdef.pyx b/docs/examples/userguide/early_binding_for_speed/rectangle_cdef.pyx index 5502b4568..1933326d2 100644 --- a/docs/examples/userguide/early_binding_for_speed/rectangle_cdef.pyx +++ b/docs/examples/userguide/early_binding_for_speed/rectangle_cdef.pyx @@ -1,22 +1,22 @@ -cdef class Rectangle:
- cdef int x0, y0
- cdef int x1, y1
-
- def __init__(self, int x0, int y0, int x1, int y1):
- self.x0 = x0
- self.y0 = y0
- self.x1 = x1
- self.y1 = y1
-
- cdef int _area(self):
- area = (self.x1 - self.x0) * (self.y1 - self.y0)
- if area < 0:
- area = -area
- return area
-
- def area(self):
- return self._area()
-
-def rectArea(x0, y0, x1, y1):
- rect = Rectangle(x0, y0, x1, y1)
- return rect.area()
+cdef class Rectangle: + cdef int x0, y0 + cdef int x1, y1 + + def __init__(self, int x0, int y0, int x1, int y1): + self.x0 = x0 + self.y0 = y0 + self.x1 = x1 + self.y1 = y1 + + cdef int _area(self): + cdef int area = (self.x1 - self.x0) * (self.y1 - self.y0) + if area < 0: + area = -area + return area + + def area(self): + return self._area() + +def rectArea(x0, y0, x1, y1): + cdef Rectangle rect = Rectangle(x0, y0, x1, y1) + return rect._area() diff --git a/docs/examples/userguide/early_binding_for_speed/rectangle_cpdef.pyx b/docs/examples/userguide/early_binding_for_speed/rectangle_cpdef.pyx index 01df3759f..f8b7d86a8 100644 --- a/docs/examples/userguide/early_binding_for_speed/rectangle_cpdef.pyx +++ b/docs/examples/userguide/early_binding_for_speed/rectangle_cpdef.pyx @@ -1,19 +1,19 @@ -cdef class Rectangle:
- cdef int x0, y0
- cdef int x1, y1
-
- def __init__(self, int x0, int y0, int x1, int y1):
- self.x0 = x0
- self.y0 = y0
- self.x1 = x1
- self.y1 = y1
-
- cpdef int area(self):
- area = (self.x1 - self.x0) * (self.y1 - self.y0)
- if area < 0:
- area = -area
- return area
-
-def rectArea(x0, y0, x1, y1):
- rect = Rectangle(x0, y0, x1, y1)
- return rect.area()
+cdef class Rectangle: + cdef int x0, y0 + cdef int x1, y1 + + def __init__(self, int x0, int y0, int x1, int y1): + self.x0 = x0 + self.y0 = y0 + self.x1 = x1 + self.y1 = y1 + + cpdef int area(self): + area = (self.x1 - self.x0) * (self.y1 - self.y0) + if area < 0: + area = -area + return area + +def rectArea(x0, y0, x1, y1): + cdef Rectangle rect = Rectangle(x0, y0, x1, y1) + return rect.area() diff --git a/docs/examples/userguide/extension_types/c_property.pyx b/docs/examples/userguide/extension_types/c_property.pyx new file mode 100644 index 000000000..b759ceec1 --- /dev/null +++ b/docs/examples/userguide/extension_types/c_property.pyx @@ -0,0 +1,20 @@ +cdef extern from "complexobject.h": + + struct Py_complex: + double real + double imag + + ctypedef class __builtin__.complex [object PyComplexObject]: + cdef Py_complex cval + + @property + cdef inline double real(self): + return self.cval.real + + @property + cdef inline double imag(self): + return self.cval.imag + + +def cprint(complex c): + print(f"{c.real :.4f}{c.imag :+.4f}j") # uses C calls to the above property methods. diff --git a/docs/examples/userguide/extension_types/dict_animal.pyx b/docs/examples/userguide/extension_types/dict_animal.pyx index ab66900fe..1aa0ccc11 100644 --- a/docs/examples/userguide/extension_types/dict_animal.pyx +++ b/docs/examples/userguide/extension_types/dict_animal.pyx @@ -1,11 +1,11 @@ -cdef class Animal:
-
- cdef int number_of_legs
- cdef dict __dict__
-
- def __cinit__(self, int number_of_legs):
- self.number_of_legs = number_of_legs
-
-
-dog = Animal(4)
-dog.has_tail = True
+cdef class Animal: + + cdef int number_of_legs + cdef dict __dict__ + + def __cinit__(self, int number_of_legs): + self.number_of_legs = number_of_legs + + +dog = Animal(4) +dog.has_tail = True diff --git a/docs/examples/userguide/extension_types/extendable_animal.pyx b/docs/examples/userguide/extension_types/extendable_animal.pyx index fe53218b5..701a93148 100644 --- a/docs/examples/userguide/extension_types/extendable_animal.pyx +++ b/docs/examples/userguide/extension_types/extendable_animal.pyx @@ -1,14 +1,14 @@ -cdef class Animal:
-
- cdef int number_of_legs
-
- def __cinit__(self, int number_of_legs):
- self.number_of_legs = number_of_legs
-
-
-class ExtendableAnimal(Animal): # Note that we use class, not cdef class
- pass
-
-
-dog = ExtendableAnimal(4)
+cdef class Animal: + + cdef int number_of_legs + + def __cinit__(self, int number_of_legs): + self.number_of_legs = number_of_legs + + +class ExtendableAnimal(Animal): # Note that we use class, not cdef class + pass + + +dog = ExtendableAnimal(4) dog.has_tail = True
\ No newline at end of file diff --git a/docs/examples/userguide/extension_types/my_module.pyx b/docs/examples/userguide/extension_types/my_module.pyx index 6ceb057ca..fb0701c12 100644 --- a/docs/examples/userguide/extension_types/my_module.pyx +++ b/docs/examples/userguide/extension_types/my_module.pyx @@ -1,11 +1,11 @@ -from __future__ import print_function
-
-cdef class Shrubbery:
-
- def __init__(self, w, h):
- self.width = w
- self.height = h
-
- def describe(self):
- print("This shrubbery is", self.width,
- "by", self.height, "cubits.")
+from __future__ import print_function + +cdef class Shrubbery: + + def __init__(self, w, h): + self.width = w + self.height = h + + def describe(self): + print("This shrubbery is", self.width, + "by", self.height, "cubits.") diff --git a/docs/examples/userguide/extension_types/python_access.pyx b/docs/examples/userguide/extension_types/python_access.pyx index 13e19091e..6d5225ec0 100644 --- a/docs/examples/userguide/extension_types/python_access.pyx +++ b/docs/examples/userguide/extension_types/python_access.pyx @@ -1,3 +1,3 @@ -cdef class Shrubbery:
- cdef public int width, height
- cdef readonly float depth
+cdef class Shrubbery: + cdef public int width, height + cdef readonly float depth diff --git a/docs/examples/userguide/extension_types/shrubbery.py b/docs/examples/userguide/extension_types/shrubbery.py new file mode 100644 index 000000000..075664527 --- /dev/null +++ b/docs/examples/userguide/extension_types/shrubbery.py @@ -0,0 +1,14 @@ +from __future__ import print_function + +@cython.cclass +class Shrubbery: + width: cython.int + height: cython.int + + def __init__(self, w, h): + self.width = w + self.height = h + + def describe(self): + print("This shrubbery is", self.width, + "by", self.height, "cubits.") diff --git a/docs/examples/userguide/extension_types/shrubbery.pyx b/docs/examples/userguide/extension_types/shrubbery.pyx index 9d2a5481a..b74dfbd1b 100644 --- a/docs/examples/userguide/extension_types/shrubbery.pyx +++ b/docs/examples/userguide/extension_types/shrubbery.pyx @@ -1,12 +1,14 @@ -from __future__ import print_function
-
-cdef class Shrubbery:
- cdef int width, height
-
- def __init__(self, w, h):
- self.width = w
- self.height = h
-
- def describe(self):
- print("This shrubbery is", self.width,
- "by", self.height, "cubits.")
+from __future__ import print_function + + +cdef class Shrubbery: + cdef int width + cdef int height + + def __init__(self, w, h): + self.width = w + self.height = h + + def describe(self): + print("This shrubbery is", self.width, + "by", self.height, "cubits.") diff --git a/docs/examples/userguide/extension_types/shrubbery_2.pyx b/docs/examples/userguide/extension_types/shrubbery_2.pyx index e0d8c45b5..d05d28243 100644 --- a/docs/examples/userguide/extension_types/shrubbery_2.pyx +++ b/docs/examples/userguide/extension_types/shrubbery_2.pyx @@ -1,8 +1,8 @@ -from my_module cimport Shrubbery
-
-cdef Shrubbery another_shrubbery(Shrubbery sh1):
- cdef Shrubbery sh2
- sh2 = Shrubbery()
- sh2.width = sh1.width
- sh2.height = sh1.height
- return sh2
+from my_module cimport Shrubbery + +cdef Shrubbery another_shrubbery(Shrubbery sh1): + cdef Shrubbery sh2 + sh2 = Shrubbery() + sh2.width = sh1.width + sh2.height = sh1.height + return sh2 diff --git a/docs/examples/userguide/extension_types/widen_shrubbery.pyx b/docs/examples/userguide/extension_types/widen_shrubbery.pyx index aff3bd116..a312fbfd9 100644 --- a/docs/examples/userguide/extension_types/widen_shrubbery.pyx +++ b/docs/examples/userguide/extension_types/widen_shrubbery.pyx @@ -1,4 +1,4 @@ -from my_module cimport Shrubbery
-
-cdef widen_shrubbery(Shrubbery sh, extra_width):
- sh.width = sh.width + extra_width
+from my_module cimport Shrubbery + +cdef widen_shrubbery(Shrubbery sh, extra_width): + sh.width = sh.width + extra_width diff --git a/docs/examples/userguide/external_C_code/delorean.pyx b/docs/examples/userguide/external_C_code/delorean.pyx index 52c713616..9c6af9f87 100644 --- a/docs/examples/userguide/external_C_code/delorean.pyx +++ b/docs/examples/userguide/external_C_code/delorean.pyx @@ -1,9 +1,9 @@ -# delorean.pyx
-
-cdef public struct Vehicle:
- int speed
- float power
-
-cdef api void activate(Vehicle *v):
- if v.speed >= 88 and v.power >= 1.21:
- print("Time travel achieved")
\ No newline at end of file +# delorean.pyx + +cdef public struct Vehicle: + int speed + float power + +cdef api void activate(Vehicle *v) except *: + if v.speed >= 88 and v.power >= 1.21: + print("Time travel achieved") diff --git a/docs/examples/userguide/external_C_code/marty.c b/docs/examples/userguide/external_C_code/marty.c index 8096ab19a..d7f5117f7 100644 --- a/docs/examples/userguide/external_C_code/marty.c +++ b/docs/examples/userguide/external_C_code/marty.c @@ -1,13 +1,14 @@ -# marty.c
-#include "delorean_api.h"
-
-Vehicle car;
-
-int main(int argc, char *argv[]) {
- Py_Initialize();
- import_delorean();
- car.speed = atoi(argv[1]);
- car.power = atof(argv[2]);
- activate(&car);
- Py_Finalize();
-}
+# marty.c +#include "delorean_api.h" + +Vehicle car; + +int main(int argc, char *argv[]) { + Py_Initialize(); + import_delorean(); + car.speed = atoi(argv[1]); + car.power = atof(argv[2]); + activate(&car); + /* Error handling left out - call PyErr_Occurred() to test for Python exceptions. */ + Py_Finalize(); +} diff --git a/docs/examples/userguide/external_C_code/platform_adaptation.pyx b/docs/examples/userguide/external_C_code/platform_adaptation.pyx new file mode 100644 index 000000000..0beece8f4 --- /dev/null +++ b/docs/examples/userguide/external_C_code/platform_adaptation.pyx @@ -0,0 +1,14 @@ +cdef extern from *: + """ + #if defined(_WIN32) || defined(MS_WINDOWS) || defined(_MSC_VER) + #include "stdlib.h" + #define myapp_sleep(m) _sleep(m) + #else + #include <unistd.h> + #define myapp_sleep(m) ((void) usleep((m) * 1000)) + #endif + """ + # using "myapp_" prefix in the C code to prevent C naming conflicts + void msleep "myapp_sleep"(int milliseconds) nogil + +msleep(milliseconds=1) diff --git a/docs/examples/userguide/external_C_code/c_code_docstring.pyx b/docs/examples/userguide/external_C_code/verbatim_c_code.pyx index 430e89c48..fb1937166 100644 --- a/docs/examples/userguide/external_C_code/c_code_docstring.pyx +++ b/docs/examples/userguide/external_C_code/verbatim_c_code.pyx @@ -1,9 +1,9 @@ -cdef extern from *:
- """
- /* This is C code which will be put
- * in the .c file output by Cython */
- static long square(long x) {return x * x;}
- #define assign(x, y) ((x) = (y))
- """
- long square(long x)
- void assign(long& x, long y)
+cdef extern from *: + """ + /* This is C code which will be put + * in the .c file output by Cython */ + static long square(long x) {return x * x;} + #define assign(x, y) ((x) = (y)) + """ + long square(long x) + void assign(long& x, long y) diff --git a/docs/examples/userguide/fusedtypes/char_or_float.pyx b/docs/examples/userguide/fusedtypes/char_or_float.pyx index 6db746831..5a525431f 100644 --- a/docs/examples/userguide/fusedtypes/char_or_float.pyx +++ b/docs/examples/userguide/fusedtypes/char_or_float.pyx @@ -1,17 +1,17 @@ -from __future__ import print_function
-
-ctypedef fused char_or_float:
- char
- float
-
-
-cpdef char_or_float plus_one(char_or_float var):
- return var + 1
-
-
-def show_me():
- cdef:
- char a = 127
- float b = 127
- print('char', plus_one(a))
- print('float', plus_one(b))
+from __future__ import print_function + +ctypedef fused char_or_float: + char + float + + +cpdef char_or_float plus_one(char_or_float var): + return var + 1 + + +def show_me(): + cdef: + char a = 127 + float b = 127 + print('char', plus_one(a)) + print('float', plus_one(b)) diff --git a/docs/examples/userguide/language_basics/casting_python.pxd b/docs/examples/userguide/language_basics/casting_python.pxd new file mode 100644 index 000000000..fa3d46030 --- /dev/null +++ b/docs/examples/userguide/language_basics/casting_python.pxd @@ -0,0 +1,2 @@ +cdef extern from *: + ctypedef Py_ssize_t Py_intptr_t diff --git a/docs/examples/userguide/language_basics/casting_python.py b/docs/examples/userguide/language_basics/casting_python.py new file mode 100644 index 000000000..1c02c461c --- /dev/null +++ b/docs/examples/userguide/language_basics/casting_python.py @@ -0,0 +1,22 @@ +from cython.cimports.cpython.ref import PyObject + +def main(): + + python_string = "foo" + + # Note that the variables below are automatically inferred + # as the correct pointer type that is assigned to them. + # They do not need to be typed explicitly. + + ptr = cython.cast(cython.p_void, python_string) + adress_in_c = cython.cast(Py_intptr_t, ptr) + address_from_void = adress_in_c # address_from_void is a python int + + ptr2 = cython.cast(cython.pointer(PyObject), python_string) + address_in_c2 = cython.cast(Py_intptr_t, ptr2) + address_from_PyObject = address_in_c2 # address_from_PyObject is a python int + + assert address_from_void == address_from_PyObject == id(python_string) + + print(cython.cast(object, ptr)) # Prints "foo" + print(cython.cast(object, ptr2)) # prints "foo" diff --git a/docs/examples/userguide/language_basics/casting_python.pyx b/docs/examples/userguide/language_basics/casting_python.pyx index fe84acde2..4cc819ae3 100644 --- a/docs/examples/userguide/language_basics/casting_python.pyx +++ b/docs/examples/userguide/language_basics/casting_python.pyx @@ -1,19 +1,19 @@ -from cpython.ref cimport PyObject
-
-cdef extern from *:
- ctypedef Py_ssize_t Py_intptr_t
-
-python_string = "foo"
-
-cdef void* ptr = <void*>python_string
-cdef Py_intptr_t adress_in_c = <Py_intptr_t>ptr
-address_from_void = adress_in_c # address_from_void is a python int
-
-cdef PyObject* ptr2 = <PyObject*>python_string
-cdef Py_intptr_t address_in_c2 = <Py_intptr_t>ptr2
-address_from_PyObject = address_in_c2 # address_from_PyObject is a python int
-
-assert address_from_void == address_from_PyObject == id(python_string)
-
-print(<object>ptr) # Prints "foo"
-print(<object>ptr2) # prints "foo"
+from cpython.ref cimport PyObject + +cdef extern from *: + ctypedef Py_ssize_t Py_intptr_t + +python_string = "foo" + +cdef void* ptr = <void*>python_string +cdef Py_intptr_t adress_in_c = <Py_intptr_t>ptr +address_from_void = adress_in_c # address_from_void is a python int + +cdef PyObject* ptr2 = <PyObject*>python_string +cdef Py_intptr_t address_in_c2 = <Py_intptr_t>ptr2 +address_from_PyObject = address_in_c2 # address_from_PyObject is a python int + +assert address_from_void == address_from_PyObject == id(python_string) + +print(<object>ptr) # Prints "foo" +print(<object>ptr2) # prints "foo" diff --git a/docs/examples/userguide/language_basics/cdef_block.pyx b/docs/examples/userguide/language_basics/cdef_block.pyx index 4132aeee1..c0c303029 100644 --- a/docs/examples/userguide/language_basics/cdef_block.pyx +++ b/docs/examples/userguide/language_basics/cdef_block.pyx @@ -1,12 +1,12 @@ -from __future__ import print_function
-
-cdef:
- struct Spam:
- int tons
-
- int i
- float a
- Spam *p
-
- void f(Spam *s):
- print(s.tons, "Tons of spam")
+from __future__ import print_function + +cdef: + struct Spam: + int tons + + int i + float a + Spam *p + + void f(Spam *s) except *: + print(s.tons, "Tons of spam") diff --git a/docs/examples/userguide/language_basics/compile_time.pyx b/docs/examples/userguide/language_basics/compile_time.pyx index fcaf75f29..28465f62a 100644 --- a/docs/examples/userguide/language_basics/compile_time.pyx +++ b/docs/examples/userguide/language_basics/compile_time.pyx @@ -1,9 +1,9 @@ -from __future__ import print_function
-
-DEF FavouriteFood = u"spam"
-DEF ArraySize = 42
-DEF OtherArraySize = 2 * ArraySize + 17
-
-cdef int a1[ArraySize]
-cdef int a2[OtherArraySize]
+from __future__ import print_function + +DEF FavouriteFood = u"spam" +DEF ArraySize = 42 +DEF OtherArraySize = 2 * ArraySize + 17 + +cdef int a1[ArraySize] +cdef int a2[OtherArraySize] print("I like", FavouriteFood)
\ No newline at end of file diff --git a/docs/examples/userguide/language_basics/kwargs_1.pyx b/docs/examples/userguide/language_basics/kwargs_1.pyx index e5e18c008..1117c967c 100644 --- a/docs/examples/userguide/language_basics/kwargs_1.pyx +++ b/docs/examples/userguide/language_basics/kwargs_1.pyx @@ -1,6 +1,6 @@ -def f(a, b, *args, c, d = 42, e, **kwds):
- ...
-
-
-# We cannot call f with less verbosity than this.
-foo = f(4, "bar", c=68, e=1.0)
+def f(a, b, *args, c, d = 42, e, **kwds): + ... + + +# We cannot call f with less verbosity than this. +foo = f(4, "bar", c=68, e=1.0) diff --git a/docs/examples/userguide/language_basics/kwargs_2.pyx b/docs/examples/userguide/language_basics/kwargs_2.pyx index a2c639ea6..902df694c 100644 --- a/docs/examples/userguide/language_basics/kwargs_2.pyx +++ b/docs/examples/userguide/language_basics/kwargs_2.pyx @@ -1,5 +1,5 @@ -def g(a, b, *, c, d):
- ...
-
-# We cannot call g with less verbosity than this.
-foo = g(4.0, "something", c=68, d="other")
+def g(a, b, *, c, d): + ... + +# We cannot call g with less verbosity than this. +foo = g(4.0, "something", c=68, d="other") diff --git a/docs/examples/userguide/language_basics/open_file.py b/docs/examples/userguide/language_basics/open_file.py new file mode 100644 index 000000000..ad3ae0374 --- /dev/null +++ b/docs/examples/userguide/language_basics/open_file.py @@ -0,0 +1,19 @@ +from cython.cimports.libc.stdio import FILE, fopen +from cython.cimports.libc.stdlib import malloc, free +from cython.cimports.cpython.exc import PyErr_SetFromErrnoWithFilenameObject + +def open_file(): + p = fopen("spam.txt", "r") # The type of "p" is "FILE*", as returned by fopen(). + + if p is cython.NULL: + PyErr_SetFromErrnoWithFilenameObject(OSError, "spam.txt") + ... + + +def allocating_memory(number=10): + # Note that the type of the variable "my_array" is automatically inferred from the assignment. + my_array = cython.cast(p_double, malloc(number * cython.sizeof(double))) + if not my_array: # same as 'is NULL' above + raise MemoryError() + ... + free(my_array) diff --git a/docs/examples/userguide/language_basics/open_file.pyx b/docs/examples/userguide/language_basics/open_file.pyx index 19eac104e..ad45fc8c4 100644 --- a/docs/examples/userguide/language_basics/open_file.pyx +++ b/docs/examples/userguide/language_basics/open_file.pyx @@ -1,18 +1,18 @@ -from libc.stdio cimport FILE, fopen
-from libc.stdlib cimport malloc, free
-from cpython.exc cimport PyErr_SetFromErrnoWithFilenameObject
-
-def open_file():
- cdef FILE* p
- p = fopen("spam.txt", "r")
- if p is NULL:
- PyErr_SetFromErrnoWithFilenameObject(OSError, "spam.txt")
- ...
-
-
-def allocating_memory(number=10):
- cdef double *my_array = <double *> malloc(number * sizeof(double))
- if not my_array: # same as 'is NULL' above
- raise MemoryError()
- ...
- free(my_array)
+from libc.stdio cimport FILE, fopen +from libc.stdlib cimport malloc, free +from cpython.exc cimport PyErr_SetFromErrnoWithFilenameObject + +def open_file(): + cdef FILE* p + p = fopen("spam.txt", "r") + if p is NULL: + PyErr_SetFromErrnoWithFilenameObject(OSError, "spam.txt") + ... + + +def allocating_memory(number=10): + cdef double *my_array = <double *> malloc(number * sizeof(double)) + if not my_array: # same as 'is NULL' above + raise MemoryError() + ... + free(my_array) diff --git a/docs/examples/userguide/language_basics/optional_subclassing.py b/docs/examples/userguide/language_basics/optional_subclassing.py new file mode 100644 index 000000000..480ae100b --- /dev/null +++ b/docs/examples/userguide/language_basics/optional_subclassing.py @@ -0,0 +1,19 @@ +from __future__ import print_function + +@cython.cclass +class A: + @cython.cfunc + def foo(self): + print("A") + +@cython.cclass +class B(A): + @cython.cfunc + def foo(self, x=None): + print("B", x) + +@cython.cclass +class C(B): + @cython.ccall + def foo(self, x=True, k:cython.int = 3): + print("C", x, k) diff --git a/docs/examples/userguide/language_basics/optional_subclassing.pyx b/docs/examples/userguide/language_basics/optional_subclassing.pyx index f655cadf3..b2a3d4dec 100644 --- a/docs/examples/userguide/language_basics/optional_subclassing.pyx +++ b/docs/examples/userguide/language_basics/optional_subclassing.pyx @@ -1,13 +1,19 @@ -from __future__ import print_function
-
-cdef class A:
- cdef foo(self):
- print("A")
-
-cdef class B(A):
- cdef foo(self, x=None):
- print("B", x)
-
-cdef class C(B):
- cpdef foo(self, x=True, int k=3):
- print("C", x, k)
+from __future__ import print_function + + +cdef class A: + + cdef foo(self): + print("A") + + +cdef class B(A): + + cdef foo(self, x=None): + print("B", x) + + +cdef class C(B): + + cpdef foo(self, x=True, int k=3): + print("C", x, k) diff --git a/docs/examples/userguide/language_basics/override.py b/docs/examples/userguide/language_basics/override.py new file mode 100644 index 000000000..f9e0be83f --- /dev/null +++ b/docs/examples/userguide/language_basics/override.py @@ -0,0 +1,17 @@ +from __future__ import print_function + +@cython.cclass +class A: + @cython.cfunc + def foo(self): + print("A") + +@cython.cclass +class B(A): + @cython.ccall + def foo(self): + print("B") + +class C(B): # NOTE: no cclass decorator + def foo(self): + print("C") diff --git a/docs/examples/userguide/language_basics/override.pyx b/docs/examples/userguide/language_basics/override.pyx index 873a7ec6e..1a7ceefb7 100644 --- a/docs/examples/userguide/language_basics/override.pyx +++ b/docs/examples/userguide/language_basics/override.pyx @@ -1,13 +1,17 @@ -from __future__ import print_function
-
-cdef class A:
- cdef foo(self):
- print("A")
-
-cdef class B(A):
- cpdef foo(self):
- print("B")
-
-class C(B): # NOTE: not cdef class
- def foo(self):
- print("C")
+from __future__ import print_function + + +cdef class A: + + cdef foo(self): + print("A") + + +cdef class B(A): + + cpdef foo(self): + print("B") + +class C(B): # NOTE: not cdef class + def foo(self): + print("C") diff --git a/docs/examples/userguide/language_basics/parameter_refcount.py b/docs/examples/userguide/language_basics/parameter_refcount.py new file mode 100644 index 000000000..2b25915ba --- /dev/null +++ b/docs/examples/userguide/language_basics/parameter_refcount.py @@ -0,0 +1,23 @@ +from __future__ import print_function + +from cython.cimports.cpython.ref import PyObject + +import sys + +python_dict = {"abc": 123} +python_dict_refcount = sys.getrefcount(python_dict) + +@cython.cfunc +def owned_reference(obj: object): + refcount = sys.getrefcount(python_dict) + print('Inside owned_reference: {refcount}'.format(refcount=refcount)) + +@cython.cfunc +def borrowed_reference(obj: cython.pointer(PyObject)): + refcount = obj.ob_refcnt + print('Inside borrowed_reference: {refcount}'.format(refcount=refcount)) + +def main(): + print('Initial refcount: {refcount}'.format(refcount=python_dict_refcount)) + owned_reference(python_dict) + borrowed_reference(cython.cast(cython.pointer(PyObject), python_dict)) diff --git a/docs/examples/userguide/language_basics/parameter_refcount.pyx b/docs/examples/userguide/language_basics/parameter_refcount.pyx new file mode 100644 index 000000000..6fe3ffadd --- /dev/null +++ b/docs/examples/userguide/language_basics/parameter_refcount.pyx @@ -0,0 +1,23 @@ +from __future__ import print_function + +from cpython.ref cimport PyObject + +import sys + +python_dict = {"abc": 123} +python_dict_refcount = sys.getrefcount(python_dict) + + +cdef owned_reference(object obj): + refcount = sys.getrefcount(python_dict) + print('Inside owned_reference: {refcount}'.format(refcount=refcount)) + + +cdef borrowed_reference(PyObject * obj): + refcount = obj.ob_refcnt + print('Inside borrowed_reference: {refcount}'.format(refcount=refcount)) + + +print('Initial refcount: {refcount}'.format(refcount=python_dict_refcount)) +owned_reference(python_dict) +borrowed_reference(<PyObject *>python_dict) diff --git a/docs/examples/userguide/language_basics/struct_union_enum.py b/docs/examples/userguide/language_basics/struct_union_enum.py new file mode 100644 index 000000000..b78c0aa02 --- /dev/null +++ b/docs/examples/userguide/language_basics/struct_union_enum.py @@ -0,0 +1,7 @@ +Grail = cython.struct( + age=cython.int, + volume=cython.float) + +Food = cython.union( + spam=cython.p_char, + eggs=cython.p_float) diff --git a/docs/examples/userguide/language_basics/struct_union_enum.pyx b/docs/examples/userguide/language_basics/struct_union_enum.pyx index ccbc28d42..af9b06d9a 100644 --- a/docs/examples/userguide/language_basics/struct_union_enum.pyx +++ b/docs/examples/userguide/language_basics/struct_union_enum.pyx @@ -1,16 +1,16 @@ -cdef struct Grail:
- int age
- float volume
-
-cdef union Food:
- char *spam
- float *eggs
-
-cdef enum CheeseType:
- cheddar, edam,
- camembert
-
-cdef enum CheeseState:
- hard = 1
- soft = 2
- runny = 3
+cdef struct Grail: + int age + float volume + +cdef union Food: + char *spam + float *eggs + +cdef enum CheeseType: + cheddar, edam, + camembert + +cdef enum CheeseState: + hard = 1 + soft = 2 + runny = 3 diff --git a/docs/examples/userguide/memoryviews/add_one.pyx b/docs/examples/userguide/memoryviews/add_one.pyx index cbe65b069..7de7a1274 100644 --- a/docs/examples/userguide/memoryviews/add_one.pyx +++ b/docs/examples/userguide/memoryviews/add_one.pyx @@ -1,12 +1,12 @@ -import numpy as np
-
-def add_one(int[:,:] buf):
- for x in range(buf.shape[0]):
- for y in range(buf.shape[1]):
- buf[x, y] += 1
-
-# exporting_object must be a Python object
-# implementing the buffer interface, e.g. a numpy array.
-exporting_object = np.zeros((10, 20), dtype=np.intc)
-
-add_one(exporting_object)
+import numpy as np + +def add_one(int[:,:] buf): + for x in range(buf.shape[0]): + for y in range(buf.shape[1]): + buf[x, y] += 1 + +# exporting_object must be a Python object +# implementing the buffer interface, e.g. a numpy array. +exporting_object = np.zeros((10, 20), dtype=np.intc) + +add_one(exporting_object) diff --git a/docs/examples/userguide/memoryviews/copy.pyx b/docs/examples/userguide/memoryviews/copy.pyx index 9f000a3b4..9eb1307bf 100644 --- a/docs/examples/userguide/memoryviews/copy.pyx +++ b/docs/examples/userguide/memoryviews/copy.pyx @@ -1,12 +1,12 @@ -import numpy as np
-
-cdef int[:, :, :] to_view, from_view
-to_view = np.empty((20, 15, 30), dtype=np.intc)
-from_view = np.ones((20, 15, 30), dtype=np.intc)
-
-# copy the elements in from_view to to_view
-to_view[...] = from_view
-# or
-to_view[:] = from_view
-# or
-to_view[:, :, :] = from_view
+import numpy as np + +cdef int[:, :, :] to_view, from_view +to_view = np.empty((20, 15, 30), dtype=np.intc) +from_view = np.ones((20, 15, 30), dtype=np.intc) + +# copy the elements in from_view to to_view +to_view[...] = from_view +# or +to_view[:] = from_view +# or +to_view[:, :, :] = from_view diff --git a/docs/examples/userguide/memoryviews/memory_layout.pyx b/docs/examples/userguide/memoryviews/memory_layout.pyx index 5c2818dc0..8f9d8a23c 100644 --- a/docs/examples/userguide/memoryviews/memory_layout.pyx +++ b/docs/examples/userguide/memoryviews/memory_layout.pyx @@ -1,11 +1,11 @@ -from cython cimport view
-
-# direct access in both dimensions, strided in the first dimension, contiguous in the last
-cdef int[:, ::view.contiguous] a
-
-# contiguous list of pointers to contiguous lists of ints
-cdef int[::view.indirect_contiguous, ::1] b
-
-# direct or indirect in the first dimension, direct in the second dimension
-# strided in both dimensions
-cdef int[::view.generic, :] c
+from cython cimport view + +# direct access in both dimensions, strided in the first dimension, contiguous in the last +cdef int[:, ::view.contiguous] a + +# contiguous list of pointers to contiguous lists of ints +cdef int[::view.indirect_contiguous, ::1] b + +# direct or indirect in the first dimension, direct in the second dimension +# strided in both dimensions +cdef int[::view.generic, :] c diff --git a/docs/examples/userguide/memoryviews/memory_layout_2.pyx b/docs/examples/userguide/memoryviews/memory_layout_2.pyx index 1cb039dd4..71d2cceb2 100644 --- a/docs/examples/userguide/memoryviews/memory_layout_2.pyx +++ b/docs/examples/userguide/memoryviews/memory_layout_2.pyx @@ -1,6 +1,6 @@ -from cython cimport view
-
-# VALID
-cdef int[::view.indirect, ::1, :] a
-cdef int[::view.indirect, :, ::1] b
-cdef int[::view.indirect_contiguous, ::1, :] c
+from cython cimport view + +# VALID +cdef int[::view.indirect, ::1, :] a +cdef int[::view.indirect, :, ::1] b +cdef int[::view.indirect_contiguous, ::1, :] c diff --git a/docs/examples/userguide/memoryviews/memview_to_c.pyx b/docs/examples/userguide/memoryviews/memview_to_c.pyx index c5abc19ac..ad6190cc7 100644 --- a/docs/examples/userguide/memoryviews/memview_to_c.pyx +++ b/docs/examples/userguide/memoryviews/memview_to_c.pyx @@ -1,28 +1,28 @@ -cdef extern from "C_func_file.c":
- # C is include here so that it doesn't need to be compiled externally
- pass
-
-cdef extern from "C_func_file.h":
- void multiply_by_10_in_C(double *, unsigned int)
-
-import numpy as np
-
-def multiply_by_10(arr): # 'arr' is a one-dimensional numpy array
-
- if not arr.flags['C_CONTIGUOUS']:
- arr = np.ascontiguousarray(arr) # Makes a contiguous copy of the numpy array.
-
- cdef double[::1] arr_memview = arr
-
- multiply_by_10_in_C(&arr_memview[0], arr_memview.shape[0])
-
- return arr
-
-
-a = np.ones(5, dtype=np.double)
-print(multiply_by_10(a))
-
-b = np.ones(10, dtype=np.double)
-b = b[::2] # b is not contiguous.
-
-print(multiply_by_10(b)) # but our function still works as expected.
+cdef extern from "C_func_file.c": + # C is include here so that it doesn't need to be compiled externally + pass + +cdef extern from "C_func_file.h": + void multiply_by_10_in_C(double *, unsigned int) + +import numpy as np + +def multiply_by_10(arr): # 'arr' is a one-dimensional numpy array + + if not arr.flags['C_CONTIGUOUS']: + arr = np.ascontiguousarray(arr) # Makes a contiguous copy of the numpy array. + + cdef double[::1] arr_memview = arr + + multiply_by_10_in_C(&arr_memview[0], arr_memview.shape[0]) + + return arr + + +a = np.ones(5, dtype=np.double) +print(multiply_by_10(a)) + +b = np.ones(10, dtype=np.double) +b = b[::2] # b is not contiguous. + +print(multiply_by_10(b)) # but our function still works as expected. diff --git a/docs/examples/userguide/memoryviews/not_none.pyx b/docs/examples/userguide/memoryviews/not_none.pyx index ae3b6c936..f6c0fed8a 100644 --- a/docs/examples/userguide/memoryviews/not_none.pyx +++ b/docs/examples/userguide/memoryviews/not_none.pyx @@ -1,11 +1,11 @@ -import numpy as np
-
-def process_buffer(int[:,:] input_view not None,
- int[:,:] output_view=None):
-
- if output_view is None:
- # Creating a default view, e.g.
- output_view = np.empty_like(input_view)
-
- # process 'input_view' into 'output_view'
- return output_view
+import numpy as np + +def process_buffer(int[:,:] input_view not None, + int[:,:] output_view=None): + + if output_view is None: + # Creating a default view, e.g. + output_view = np.empty_like(input_view) + + # process 'input_view' into 'output_view' + return output_view diff --git a/docs/examples/userguide/memoryviews/np_flag_const.pyx b/docs/examples/userguide/memoryviews/np_flag_const.pyx index 03f0ea4a3..54eb3d338 100644 --- a/docs/examples/userguide/memoryviews/np_flag_const.pyx +++ b/docs/examples/userguide/memoryviews/np_flag_const.pyx @@ -1,7 +1,7 @@ -import numpy as np
-
-cdef const double[:] myslice # const item type => read-only view
-
-a = np.linspace(0, 10, num=50)
-a.setflags(write=False)
-myslice = a
+import numpy as np + +cdef const double[:] myslice # const item type => read-only view + +a = np.linspace(0, 10, num=50) +a.setflags(write=False) +myslice = a diff --git a/docs/examples/userguide/memoryviews/slicing.pyx b/docs/examples/userguide/memoryviews/slicing.pyx index a6134aae2..d7bd896e6 100644 --- a/docs/examples/userguide/memoryviews/slicing.pyx +++ b/docs/examples/userguide/memoryviews/slicing.pyx @@ -1,10 +1,10 @@ -import numpy as np
-
-exporting_object = np.arange(0, 15 * 10 * 20, dtype=np.intc).reshape((15, 10, 20))
-
-cdef int[:, :, :] my_view = exporting_object
-
-# These are all equivalent
-my_view[10]
-my_view[10, :, :]
-my_view[10, ...]
+import numpy as np + +exporting_object = np.arange(0, 15 * 10 * 20, dtype=np.intc).reshape((15, 10, 20)) + +cdef int[:, :, :] my_view = exporting_object + +# These are all equivalent +my_view[10] +my_view[10, :, :] +my_view[10, ...] diff --git a/docs/examples/userguide/memoryviews/transpose.pyx b/docs/examples/userguide/memoryviews/transpose.pyx index 7611529c2..8a53f7140 100644 --- a/docs/examples/userguide/memoryviews/transpose.pyx +++ b/docs/examples/userguide/memoryviews/transpose.pyx @@ -1,6 +1,6 @@ -import numpy as np
-
-array = np.arange(20, dtype=np.intc).reshape((2, 10))
-
-cdef int[:, ::1] c_contig = array
-cdef int[::1, :] f_contig = c_contig.T
+import numpy as np + +array = np.arange(20, dtype=np.intc).reshape((2, 10)) + +cdef int[:, ::1] c_contig = array +cdef int[::1, :] f_contig = c_contig.T diff --git a/docs/examples/userguide/memoryviews/view_string.pyx b/docs/examples/userguide/memoryviews/view_string.pyx index 7aace3ea5..9fdeae053 100644 --- a/docs/examples/userguide/memoryviews/view_string.pyx +++ b/docs/examples/userguide/memoryviews/view_string.pyx @@ -1,9 +1,9 @@ -cdef bint is_y_in(const unsigned char[:] string_view):
- cdef int i
- for i in range(string_view.shape[0]):
- if string_view[i] == b'y':
- return True
- return False
-
-print(is_y_in(b'hello world')) # False
-print(is_y_in(b'hello Cython')) # True
+cdef bint is_y_in(const unsigned char[:] string_view): + cdef int i + for i in range(string_view.shape[0]): + if string_view[i] == b'y': + return True + return False + +print(is_y_in(b'hello world')) # False +print(is_y_in(b'hello Cython')) # True diff --git a/docs/examples/userguide/numpy_tutorial/compute_fused_types.pyx b/docs/examples/userguide/numpy_tutorial/compute_fused_types.pyx index 2fc87907d..af5ef9071 100644 --- a/docs/examples/userguide/numpy_tutorial/compute_fused_types.pyx +++ b/docs/examples/userguide/numpy_tutorial/compute_fused_types.pyx @@ -1,44 +1,44 @@ -# cython: infer_types=True
-import numpy as np
-cimport cython
-
-ctypedef fused my_type:
- int
- double
- long long
-
-
-cdef my_type clip(my_type a, my_type min_value, my_type max_value):
- return min(max(a, min_value), max_value)
-
-
-@cython.boundscheck(False)
-@cython.wraparound(False)
-def compute(my_type[:, ::1] array_1, my_type[:, ::1] array_2, my_type a, my_type b, my_type c):
-
- x_max = array_1.shape[0]
- y_max = array_1.shape[1]
-
- assert tuple(array_1.shape) == tuple(array_2.shape)
-
- if my_type is int:
- dtype = np.intc
- elif my_type is double:
- dtype = np.double
- elif my_type is cython.longlong:
- dtype = np.longlong
-
- result = np.zeros((x_max, y_max), dtype=dtype)
- cdef my_type[:, ::1] result_view = result
-
- cdef my_type tmp
- cdef Py_ssize_t x, y
-
- for x in range(x_max):
- for y in range(y_max):
-
- tmp = clip(array_1[x, y], 2, 10)
- tmp = tmp * a + array_2[x, y] * b
- result_view[x, y] = tmp + c
-
- return result
+# cython: infer_types=True +import numpy as np +cimport cython + +ctypedef fused my_type: + int + double + long long + + +cdef my_type clip(my_type a, my_type min_value, my_type max_value): + return min(max(a, min_value), max_value) + + +@cython.boundscheck(False) +@cython.wraparound(False) +def compute(my_type[:, ::1] array_1, my_type[:, ::1] array_2, my_type a, my_type b, my_type c): + + x_max = array_1.shape[0] + y_max = array_1.shape[1] + + assert tuple(array_1.shape) == tuple(array_2.shape) + + if my_type is int: + dtype = np.intc + elif my_type is double: + dtype = np.double + elif my_type is cython.longlong: + dtype = np.longlong + + result = np.zeros((x_max, y_max), dtype=dtype) + cdef my_type[:, ::1] result_view = result + + cdef my_type tmp + cdef Py_ssize_t x, y + + for x in range(x_max): + for y in range(y_max): + + tmp = clip(array_1[x, y], 2, 10) + tmp = tmp * a + array_2[x, y] * b + result_view[x, y] = tmp + c + + return result diff --git a/docs/examples/userguide/numpy_tutorial/compute_infer_types.pyx b/docs/examples/userguide/numpy_tutorial/compute_infer_types.pyx index 98a683de7..3882c289d 100644 --- a/docs/examples/userguide/numpy_tutorial/compute_infer_types.pyx +++ b/docs/examples/userguide/numpy_tutorial/compute_infer_types.pyx @@ -1,34 +1,34 @@ -# cython: infer_types=True
-import numpy as np
-cimport cython
-
-DTYPE = np.intc
-
-
-cdef int clip(int a, int min_value, int max_value):
- return min(max(a, min_value), max_value)
-
-
-@cython.boundscheck(False)
-@cython.wraparound(False)
-def compute(int[:, ::1] array_1, int[:, ::1] array_2, int a, int b, int c):
-
- x_max = array_1.shape[0]
- y_max = array_1.shape[1]
-
- assert tuple(array_1.shape) == tuple(array_2.shape)
-
- result = np.zeros((x_max, y_max), dtype=DTYPE)
- cdef int[:, ::1] result_view = result
-
- cdef int tmp
- cdef Py_ssize_t x, y
-
- for x in range(x_max):
- for y in range(y_max):
-
- tmp = clip(array_1[x, y], 2, 10)
- tmp = tmp * a + array_2[x, y] * b
- result_view[x, y] = tmp + c
-
- return result
+# cython: infer_types=True +import numpy as np +cimport cython + +DTYPE = np.intc + + +cdef int clip(int a, int min_value, int max_value): + return min(max(a, min_value), max_value) + + +@cython.boundscheck(False) +@cython.wraparound(False) +def compute(int[:, ::1] array_1, int[:, ::1] array_2, int a, int b, int c): + + x_max = array_1.shape[0] + y_max = array_1.shape[1] + + assert tuple(array_1.shape) == tuple(array_2.shape) + + result = np.zeros((x_max, y_max), dtype=DTYPE) + cdef int[:, ::1] result_view = result + + cdef int tmp + cdef Py_ssize_t x, y + + for x in range(x_max): + for y in range(y_max): + + tmp = clip(array_1[x, y], 2, 10) + tmp = tmp * a + array_2[x, y] * b + result_view[x, y] = tmp + c + + return result diff --git a/docs/examples/userguide/numpy_tutorial/compute_memview.pyx b/docs/examples/userguide/numpy_tutorial/compute_memview.pyx index d264e773a..166cd6df3 100644 --- a/docs/examples/userguide/numpy_tutorial/compute_memview.pyx +++ b/docs/examples/userguide/numpy_tutorial/compute_memview.pyx @@ -1,34 +1,34 @@ -import numpy as np
-
-DTYPE = np.intc
-
-
-cdef int clip(int a, int min_value, int max_value):
- return min(max(a, min_value), max_value)
-
-
-def compute(int[:, :] array_1, int[:, :] array_2, int a, int b, int c):
-
- cdef Py_ssize_t x_max = array_1.shape[0]
- cdef Py_ssize_t y_max = array_1.shape[1]
-
- # array_1.shape is now a C array, no it's not possible
- # to compare it simply by using == without a for-loop.
- # To be able to compare it to array_2.shape easily,
- # we convert them both to Python tuples.
- assert tuple(array_1.shape) == tuple(array_2.shape)
-
- result = np.zeros((x_max, y_max), dtype=DTYPE)
- cdef int[:, :] result_view = result
-
- cdef int tmp
- cdef Py_ssize_t x, y
-
- for x in range(x_max):
- for y in range(y_max):
-
- tmp = clip(array_1[x, y], 2, 10)
- tmp = tmp * a + array_2[x, y] * b
- result_view[x, y] = tmp + c
-
- return result
+import numpy as np + +DTYPE = np.intc + + +cdef int clip(int a, int min_value, int max_value): + return min(max(a, min_value), max_value) + + +def compute(int[:, :] array_1, int[:, :] array_2, int a, int b, int c): + + cdef Py_ssize_t x_max = array_1.shape[0] + cdef Py_ssize_t y_max = array_1.shape[1] + + # array_1.shape is now a C array, no it's not possible + # to compare it simply by using == without a for-loop. + # To be able to compare it to array_2.shape easily, + # we convert them both to Python tuples. + assert tuple(array_1.shape) == tuple(array_2.shape) + + result = np.zeros((x_max, y_max), dtype=DTYPE) + cdef int[:, :] result_view = result + + cdef int tmp + cdef Py_ssize_t x, y + + for x in range(x_max): + for y in range(y_max): + + tmp = clip(array_1[x, y], 2, 10) + tmp = tmp * a + array_2[x, y] * b + result_view[x, y] = tmp + c + + return result diff --git a/docs/examples/userguide/numpy_tutorial/compute_prange.pyx b/docs/examples/userguide/numpy_tutorial/compute_prange.pyx index c00d2261b..562c73070 100644 --- a/docs/examples/userguide/numpy_tutorial/compute_prange.pyx +++ b/docs/examples/userguide/numpy_tutorial/compute_prange.pyx @@ -1,53 +1,53 @@ -# tag: openmp
-# You can ignore the previous line.
-# It's for internal testing of the cython documentation.
-
-# distutils: extra_compile_args=-fopenmp
-# distutils: extra_link_args=-fopenmp
-
-import numpy as np
-cimport cython
-from cython.parallel import prange
-
-ctypedef fused my_type:
- int
- double
- long long
-
-
-# We declare our plain c function nogil
-cdef my_type clip(my_type a, my_type min_value, my_type max_value) nogil:
- return min(max(a, min_value), max_value)
-
-
-@cython.boundscheck(False)
-@cython.wraparound(False)
-def compute(my_type[:, ::1] array_1, my_type[:, ::1] array_2, my_type a, my_type b, my_type c):
-
- cdef Py_ssize_t x_max = array_1.shape[0]
- cdef Py_ssize_t y_max = array_1.shape[1]
-
- assert tuple(array_1.shape) == tuple(array_2.shape)
-
- if my_type is int:
- dtype = np.intc
- elif my_type is double:
- dtype = np.double
- elif my_type is cython.longlong:
- dtype = np.longlong
-
- result = np.zeros((x_max, y_max), dtype=dtype)
- cdef my_type[:, ::1] result_view = result
-
- cdef my_type tmp
- cdef Py_ssize_t x, y
-
- # We use prange here.
- for x in prange(x_max, nogil=True):
- for y in range(y_max):
-
- tmp = clip(array_1[x, y], 2, 10)
- tmp = tmp * a + array_2[x, y] * b
- result_view[x, y] = tmp + c
-
- return result
+# tag: openmp +# You can ignore the previous line. +# It's for internal testing of the cython documentation. + +# distutils: extra_compile_args=-fopenmp +# distutils: extra_link_args=-fopenmp + +import numpy as np +cimport cython +from cython.parallel import prange + +ctypedef fused my_type: + int + double + long long + + +# We declare our plain c function nogil +cdef my_type clip(my_type a, my_type min_value, my_type max_value) nogil: + return min(max(a, min_value), max_value) + + +@cython.boundscheck(False) +@cython.wraparound(False) +def compute(my_type[:, ::1] array_1, my_type[:, ::1] array_2, my_type a, my_type b, my_type c): + + cdef Py_ssize_t x_max = array_1.shape[0] + cdef Py_ssize_t y_max = array_1.shape[1] + + assert tuple(array_1.shape) == tuple(array_2.shape) + + if my_type is int: + dtype = np.intc + elif my_type is double: + dtype = np.double + elif my_type is cython.longlong: + dtype = np.longlong + + result = np.zeros((x_max, y_max), dtype=dtype) + cdef my_type[:, ::1] result_view = result + + cdef my_type tmp + cdef Py_ssize_t x, y + + # We use prange here. + for x in prange(x_max, nogil=True): + for y in range(y_max): + + tmp = clip(array_1[x, y], 2, 10) + tmp = tmp * a + array_2[x, y] * b + result_view[x, y] = tmp + c + + return result diff --git a/docs/examples/userguide/numpy_tutorial/compute_py.py b/docs/examples/userguide/numpy_tutorial/compute_py.py index 00bcf244c..4a5f90b4d 100644 --- a/docs/examples/userguide/numpy_tutorial/compute_py.py +++ b/docs/examples/userguide/numpy_tutorial/compute_py.py @@ -1,28 +1,28 @@ -import numpy as np
-
-
-def clip(a, min_value, max_value):
- return min(max(a, min_value), max_value)
-
-
-def compute(array_1, array_2, a, b, c):
- """
- This function must implement the formula
- np.clip(array_1, 2, 10) * a + array_2 * b + c
-
- array_1 and array_2 are 2D.
- """
- x_max = array_1.shape[0]
- y_max = array_1.shape[1]
-
- assert array_1.shape == array_2.shape
-
- result = np.zeros((x_max, y_max), dtype=array_1.dtype)
-
- for x in range(x_max):
- for y in range(y_max):
- tmp = clip(array_1[x, y], 2, 10)
- tmp = tmp * a + array_2[x, y] * b
- result[x, y] = tmp + c
-
- return result
+import numpy as np + + +def clip(a, min_value, max_value): + return min(max(a, min_value), max_value) + + +def compute(array_1, array_2, a, b, c): + """ + This function must implement the formula + np.clip(array_1, 2, 10) * a + array_2 * b + c + + array_1 and array_2 are 2D. + """ + x_max = array_1.shape[0] + y_max = array_1.shape[1] + + assert array_1.shape == array_2.shape + + result = np.zeros((x_max, y_max), dtype=array_1.dtype) + + for x in range(x_max): + for y in range(y_max): + tmp = clip(array_1[x, y], 2, 10) + tmp = tmp * a + array_2[x, y] * b + result[x, y] = tmp + c + + return result diff --git a/docs/examples/userguide/numpy_tutorial/compute_typed.pyx b/docs/examples/userguide/numpy_tutorial/compute_typed.pyx index 8218aa709..cccc1aa3b 100644 --- a/docs/examples/userguide/numpy_tutorial/compute_typed.pyx +++ b/docs/examples/userguide/numpy_tutorial/compute_typed.pyx @@ -1,50 +1,50 @@ -import numpy as np
-
-# We now need to fix a datatype for our arrays. I've used the variable
-# DTYPE for this, which is assigned to the usual NumPy runtime
-# type info object.
-DTYPE = np.intc
-
-# cdef means here that this function is a plain C function (so faster).
-# To get all the benefits, we type the arguments and the return value.
-cdef int clip(int a, int min_value, int max_value):
- return min(max(a, min_value), max_value)
-
-
-def compute(array_1, array_2, int a, int b, int c):
-
- # The "cdef" keyword is also used within functions to type variables. It
- # can only be used at the top indentation level (there are non-trivial
- # problems with allowing them in other places, though we'd love to see
- # good and thought out proposals for it).
- cdef Py_ssize_t x_max = array_1.shape[0]
- cdef Py_ssize_t y_max = array_1.shape[1]
-
- assert array_1.shape == array_2.shape
- assert array_1.dtype == DTYPE
- assert array_2.dtype == DTYPE
-
- result = np.zeros((x_max, y_max), dtype=DTYPE)
-
- # It is very important to type ALL your variables. You do not get any
- # warnings if not, only much slower code (they are implicitly typed as
- # Python objects).
- # For the "tmp" variable, we want to use the same data type as is
- # stored in the array, so we use int because it correspond to np.intc.
- # NB! An important side-effect of this is that if "tmp" overflows its
- # datatype size, it will simply wrap around like in C, rather than raise
- # an error like in Python.
-
- cdef int tmp
-
- # Py_ssize_t is the proper C type for Python array indices.
- cdef Py_ssize_t x, y
-
- for x in range(x_max):
- for y in range(y_max):
-
- tmp = clip(array_1[x, y], 2, 10)
- tmp = tmp * a + array_2[x, y] * b
- result[x, y] = tmp + c
-
- return result
+import numpy as np + +# We now need to fix a datatype for our arrays. I've used the variable +# DTYPE for this, which is assigned to the usual NumPy runtime +# type info object. +DTYPE = np.intc + +# cdef means here that this function is a plain C function (so faster). +# To get all the benefits, we type the arguments and the return value. +cdef int clip(int a, int min_value, int max_value): + return min(max(a, min_value), max_value) + + +def compute(array_1, array_2, int a, int b, int c): + + # The "cdef" keyword is also used within functions to type variables. It + # can only be used at the top indentation level (there are non-trivial + # problems with allowing them in other places, though we'd love to see + # good and thought out proposals for it). + cdef Py_ssize_t x_max = array_1.shape[0] + cdef Py_ssize_t y_max = array_1.shape[1] + + assert array_1.shape == array_2.shape + assert array_1.dtype == DTYPE + assert array_2.dtype == DTYPE + + result = np.zeros((x_max, y_max), dtype=DTYPE) + + # It is very important to type ALL your variables. You do not get any + # warnings if not, only much slower code (they are implicitly typed as + # Python objects). + # For the "tmp" variable, we want to use the same data type as is + # stored in the array, so we use int because it correspond to np.intc. + # NB! An important side-effect of this is that if "tmp" overflows its + # datatype size, it will simply wrap around like in C, rather than raise + # an error like in Python. + + cdef int tmp + + # Py_ssize_t is the proper C type for Python array indices. + cdef Py_ssize_t x, y + + for x in range(x_max): + for y in range(y_max): + + tmp = clip(array_1[x, y], 2, 10) + tmp = tmp * a + array_2[x, y] * b + result[x, y] = tmp + c + + return result diff --git a/docs/examples/userguide/parallelism/breaking_loop.pyx b/docs/examples/userguide/parallelism/breaking_loop.pyx index d11b179d9..2cf562edf 100644 --- a/docs/examples/userguide/parallelism/breaking_loop.pyx +++ b/docs/examples/userguide/parallelism/breaking_loop.pyx @@ -1,13 +1,13 @@ -from cython.parallel import prange
-
-cdef int func(Py_ssize_t n):
- cdef Py_ssize_t i
-
- for i in prange(n, nogil=True):
- if i == 8:
- with gil:
- raise Exception()
- elif i == 4:
- break
- elif i == 2:
- return i
+from cython.parallel import prange + +cdef int func(Py_ssize_t n) except -1: + cdef Py_ssize_t i + + for i in prange(n, nogil=True): + if i == 8: + with gil: + raise Exception() + elif i == 4: + break + elif i == 2: + return i diff --git a/docs/examples/userguide/parallelism/cimport_openmp.pyx b/docs/examples/userguide/parallelism/cimport_openmp.pyx index 235ee10bc..797936fe7 100644 --- a/docs/examples/userguide/parallelism/cimport_openmp.pyx +++ b/docs/examples/userguide/parallelism/cimport_openmp.pyx @@ -1,13 +1,13 @@ -# tag: openmp
-# You can ignore the previous line.
-# It's for internal testing of the Cython documentation.
-
-from cython.parallel cimport parallel
-cimport openmp
-
-cdef int num_threads
-
-openmp.omp_set_dynamic(1)
-with nogil, parallel():
- num_threads = openmp.omp_get_num_threads()
- # ...
+# tag: openmp +# You can ignore the previous line. +# It's for internal testing of the Cython documentation. + +from cython.parallel cimport parallel +cimport openmp + +cdef int num_threads + +openmp.omp_set_dynamic(1) +with nogil, parallel(): + num_threads = openmp.omp_get_num_threads() + # ... diff --git a/docs/examples/userguide/parallelism/setup.py b/docs/examples/userguide/parallelism/setup.py index 0fb6026f7..fe6d0a64c 100644 --- a/docs/examples/userguide/parallelism/setup.py +++ b/docs/examples/userguide/parallelism/setup.py @@ -1,17 +1,16 @@ -from distutils.core import setup
-from distutils.extension import Extension
-from Cython.Build import cythonize
-
-ext_modules = [
- Extension(
- "hello",
- ["hello.pyx"],
- extra_compile_args=['-fopenmp'],
- extra_link_args=['-fopenmp'],
- )
-]
-
-setup(
- name='hello-parallel-world',
- ext_modules=cythonize(ext_modules),
-)
+from setuptools import Extension, setup +from Cython.Build import cythonize + +ext_modules = [ + Extension( + "hello", + ["hello.pyx"], + extra_compile_args=['-fopenmp'], + extra_link_args=['-fopenmp'], + ) +] + +setup( + name='hello-parallel-world', + ext_modules=cythonize(ext_modules), +) diff --git a/docs/examples/userguide/parallelism/simple_sum.pyx b/docs/examples/userguide/parallelism/simple_sum.pyx index 83b862ea6..929a988e5 100644 --- a/docs/examples/userguide/parallelism/simple_sum.pyx +++ b/docs/examples/userguide/parallelism/simple_sum.pyx @@ -1,10 +1,10 @@ -from cython.parallel import prange
-
-cdef int i
-cdef int n = 30
-cdef int sum = 0
-
-for i in prange(n, nogil=True):
- sum += i
-
-print(sum)
+from cython.parallel import prange + +cdef int i +cdef int n = 30 +cdef int sum = 0 + +for i in prange(n, nogil=True): + sum += i + +print(sum) diff --git a/docs/examples/userguide/sharing_declarations/landscaping.pyx b/docs/examples/userguide/sharing_declarations/landscaping.pyx index c54e74fd0..afc999e53 100644 --- a/docs/examples/userguide/sharing_declarations/landscaping.pyx +++ b/docs/examples/userguide/sharing_declarations/landscaping.pyx @@ -1,7 +1,7 @@ -cimport shrubbing
-import shrubbing
-
-def main():
- cdef shrubbing.Shrubbery sh
- sh = shrubbing.standard_shrubbery()
- print("Shrubbery size is", sh.width, 'x', sh.length)
+cimport shrubbing +import shrubbing + +def main(): + cdef shrubbing.Shrubbery sh + sh = shrubbing.standard_shrubbery() + print("Shrubbery size is", sh.width, 'x', sh.length) diff --git a/docs/examples/userguide/sharing_declarations/lunch.pyx b/docs/examples/userguide/sharing_declarations/lunch.pyx index 7bb2d4756..8b0911510 100644 --- a/docs/examples/userguide/sharing_declarations/lunch.pyx +++ b/docs/examples/userguide/sharing_declarations/lunch.pyx @@ -1,4 +1,4 @@ -cimport c_lunch
-
-def eject_tomato(float speed):
- c_lunch.eject_tomato(speed)
+cimport c_lunch + +def eject_tomato(float speed): + c_lunch.eject_tomato(speed) diff --git a/docs/examples/userguide/sharing_declarations/restaurant.pyx b/docs/examples/userguide/sharing_declarations/restaurant.pyx index 0c1dbf5c0..3257c681b 100644 --- a/docs/examples/userguide/sharing_declarations/restaurant.pyx +++ b/docs/examples/userguide/sharing_declarations/restaurant.pyx @@ -1,12 +1,12 @@ -from __future__ import print_function
-cimport dishes
-from dishes cimport spamdish
-
-cdef void prepare(spamdish *d):
- d.oz_of_spam = 42
- d.filler = dishes.sausage
-
-def serve():
- cdef spamdish d
- prepare(&d)
- print(f'{d.oz_of_spam} oz spam, filler no. {d.filler}')
+from __future__ import print_function +cimport dishes +from dishes cimport spamdish + +cdef void prepare(spamdish *d): + d.oz_of_spam = 42 + d.filler = dishes.sausage + +def serve(): + cdef spamdish d + prepare(&d) + print(f'{d.oz_of_spam} oz spam, filler no. {d.filler}') diff --git a/docs/examples/userguide/sharing_declarations/setup.py b/docs/examples/userguide/sharing_declarations/setup.py index 64804f97d..505b53e9d 100644 --- a/docs/examples/userguide/sharing_declarations/setup.py +++ b/docs/examples/userguide/sharing_declarations/setup.py @@ -1,4 +1,4 @@ -from distutils.core import setup
-from Cython.Build import cythonize
-
-setup(ext_modules=cythonize(["landscaping.pyx", "shrubbing.pyx"]))
+from setuptools import setup +from Cython.Build import cythonize + +setup(ext_modules=cythonize(["landscaping.pyx", "shrubbing.pyx"])) diff --git a/docs/examples/userguide/sharing_declarations/shrubbing.pyx b/docs/examples/userguide/sharing_declarations/shrubbing.pyx index a8b70dae2..bb97e7e77 100644 --- a/docs/examples/userguide/sharing_declarations/shrubbing.pyx +++ b/docs/examples/userguide/sharing_declarations/shrubbing.pyx @@ -1,7 +1,7 @@ -cdef class Shrubbery:
- def __cinit__(self, int w, int l):
- self.width = w
- self.length = l
-
-def standard_shrubbery():
- return Shrubbery(3, 7)
+cdef class Shrubbery: + def __cinit__(self, int w, int l): + self.width = w + self.length = l + +def standard_shrubbery(): + return Shrubbery(3, 7) diff --git a/docs/examples/userguide/sharing_declarations/spammery.pyx b/docs/examples/userguide/sharing_declarations/spammery.pyx index f65cf63d7..16cbda06e 100644 --- a/docs/examples/userguide/sharing_declarations/spammery.pyx +++ b/docs/examples/userguide/sharing_declarations/spammery.pyx @@ -1,11 +1,11 @@ -from __future__ import print_function
-
-from volume cimport cube
-
-def menu(description, size):
- print(description, ":", cube(size),
- "cubic metres of spam")
-
-menu("Entree", 1)
-menu("Main course", 3)
-menu("Dessert", 2)
+from __future__ import print_function + +from volume cimport cube + +def menu(description, size): + print(description, ":", cube(size), + "cubic metres of spam") + +menu("Entree", 1) +menu("Main course", 3) +menu("Dessert", 2) diff --git a/docs/examples/userguide/sharing_declarations/volume.pyx b/docs/examples/userguide/sharing_declarations/volume.pyx index 0fbab6fb7..0476b6068 100644 --- a/docs/examples/userguide/sharing_declarations/volume.pyx +++ b/docs/examples/userguide/sharing_declarations/volume.pyx @@ -1,2 +1,2 @@ -cdef float cube(float x):
- return x * x * x
+cdef float cube(float x): + return x * x * x diff --git a/docs/examples/userguide/wrapping_CPlusPlus/Rectangle.pxd b/docs/examples/userguide/wrapping_CPlusPlus/Rectangle.pxd index 68f949122..a26e69b51 100644 --- a/docs/examples/userguide/wrapping_CPlusPlus/Rectangle.pxd +++ b/docs/examples/userguide/wrapping_CPlusPlus/Rectangle.pxd @@ -1,7 +1,7 @@ cdef extern from "Rectangle.cpp":
pass
-# Decalre the class with cdef
+# Declare the class with cdef
cdef extern from "Rectangle.h" namespace "shapes":
cdef cppclass Rectangle:
Rectangle() except +
diff --git a/docs/examples/userguide/wrapping_CPlusPlus/cython_usage.pyx b/docs/examples/userguide/wrapping_CPlusPlus/cython_usage.pyx index 24192bf96..d074fa5ab 100644 --- a/docs/examples/userguide/wrapping_CPlusPlus/cython_usage.pyx +++ b/docs/examples/userguide/wrapping_CPlusPlus/cython_usage.pyx @@ -1,12 +1,12 @@ -# distutils: language = c++
-
-from Rectangle cimport Rectangle
-
-def main():
- rec_ptr = new Rectangle(1, 2, 3, 4) # Instantiate a Rectangle object on the heap
- try:
- rec_area = rec_ptr.getArea()
- finally:
- del rec_ptr # delete heap allocated object
-
- cdef Rectangle rec_stack # Instantiate a Rectangle object on the stack
+# distutils: language = c++ + +from Rectangle cimport Rectangle + +def main(): + rec_ptr = new Rectangle(1, 2, 3, 4) # Instantiate a Rectangle object on the heap + try: + rec_area = rec_ptr.getArea() + finally: + del rec_ptr # delete heap allocated object + + cdef Rectangle rec_stack # Instantiate a Rectangle object on the stack diff --git a/docs/examples/userguide/wrapping_CPlusPlus/function_templates.pyx b/docs/examples/userguide/wrapping_CPlusPlus/function_templates.pyx index 13c75426e..35d064fdd 100644 --- a/docs/examples/userguide/wrapping_CPlusPlus/function_templates.pyx +++ b/docs/examples/userguide/wrapping_CPlusPlus/function_templates.pyx @@ -1,7 +1,7 @@ -# distutils: language = c++
-
-cdef extern from "<algorithm>" namespace "std":
- T max[T](T a, T b)
-
-print(max[long](3, 4))
-print(max(1.5, 2.5)) # simple template argument deduction
+# distutils: language = c++ + +cdef extern from "<algorithm>" namespace "std": + T max[T](T a, T b) + +print(max[long](3, 4)) +print(max(1.5, 2.5)) # simple template argument deduction diff --git a/docs/examples/userguide/wrapping_CPlusPlus/iterate.pyx b/docs/examples/userguide/wrapping_CPlusPlus/iterate.pyx index ea0007e6a..cdce8910f 100644 --- a/docs/examples/userguide/wrapping_CPlusPlus/iterate.pyx +++ b/docs/examples/userguide/wrapping_CPlusPlus/iterate.pyx @@ -1,12 +1,12 @@ -# distutils: language = c++
-
-from libcpp.vector cimport vector
-
-def main():
- cdef vector[int] v = [4, 6, 5, 10, 3]
-
- cdef int value
- for value in v:
- print(value)
-
- return [x*x for x in v if x % 2 == 0]
+# distutils: language = c++ + +from libcpp.vector cimport vector + +def main(): + cdef vector[int] v = [4, 6, 5, 10, 3] + + cdef int value + for value in v: + print(value) + + return [x*x for x in v if x % 2 == 0] diff --git a/docs/examples/userguide/wrapping_CPlusPlus/nested_class.pyx b/docs/examples/userguide/wrapping_CPlusPlus/nested_class.pyx index e53f39b98..c5c764468 100644 --- a/docs/examples/userguide/wrapping_CPlusPlus/nested_class.pyx +++ b/docs/examples/userguide/wrapping_CPlusPlus/nested_class.pyx @@ -1,17 +1,17 @@ -# distutils: language = c++
-
-cdef extern from "<vector>" namespace "std":
- cdef cppclass vector[T]:
- cppclass iterator:
- T operator*()
- iterator operator++()
- bint operator==(iterator)
- bint operator!=(iterator)
- vector()
- void push_back(T&)
- T& operator[](int)
- T& at(int)
- iterator begin()
- iterator end()
-
-cdef vector[int].iterator iter #iter is declared as being of type vector<int>::iterator
+# distutils: language = c++ + +cdef extern from "<vector>" namespace "std": + cdef cppclass vector[T]: + cppclass iterator: + T operator*() + iterator operator++() + bint operator==(iterator) + bint operator!=(iterator) + vector() + void push_back(T&) + T& operator[](int) + T& at(int) + iterator begin() + iterator end() + +cdef vector[int].iterator iter #iter is declared as being of type vector<int>::iterator diff --git a/docs/examples/userguide/wrapping_CPlusPlus/python_to_cpp.pyx b/docs/examples/userguide/wrapping_CPlusPlus/python_to_cpp.pyx index 30bdb7bcb..b4be72c16 100644 --- a/docs/examples/userguide/wrapping_CPlusPlus/python_to_cpp.pyx +++ b/docs/examples/userguide/wrapping_CPlusPlus/python_to_cpp.pyx @@ -1,19 +1,19 @@ -# distutils: language = c++
-
-from libcpp.string cimport string
-from libcpp.vector cimport vector
-
-py_bytes_object = b'The knights who say ni'
-py_unicode_object = u'Those who hear them seldom live to tell the tale.'
-
-cdef string s = py_bytes_object
-print(s) # b'The knights who say ni'
-
-cdef string cpp_string = <string> py_unicode_object.encode('utf-8')
-print(cpp_string) # b'Those who hear them seldom live to tell the tale.'
-
-cdef vector[int] vect = range(1, 10, 2)
-print(vect) # [1, 3, 5, 7, 9]
-
-cdef vector[string] cpp_strings = b'It is a good shrubbery'.split()
-print(cpp_strings[1]) # b'is'
+# distutils: language = c++ + +from libcpp.string cimport string +from libcpp.vector cimport vector + +py_bytes_object = b'The knights who say ni' +py_unicode_object = u'Those who hear them seldom live to tell the tale.' + +cdef string s = py_bytes_object +print(s) # b'The knights who say ni' + +cdef string cpp_string = <string> py_unicode_object.encode('utf-8') +print(cpp_string) # b'Those who hear them seldom live to tell the tale.' + +cdef vector[int] vect = range(1, 10, 2) +print(vect) # [1, 3, 5, 7, 9] + +cdef vector[string] cpp_strings = b'It is a good shrubbery'.split() +print(cpp_strings[1]) # b'is' diff --git a/docs/examples/userguide/wrapping_CPlusPlus/rect_ptr.pyx b/docs/examples/userguide/wrapping_CPlusPlus/rect_ptr.pyx index 508e55dc6..0c48689e7 100644 --- a/docs/examples/userguide/wrapping_CPlusPlus/rect_ptr.pyx +++ b/docs/examples/userguide/wrapping_CPlusPlus/rect_ptr.pyx @@ -1,12 +1,12 @@ -# distutils: language = c++
-
-from Rectangle cimport Rectangle
-
-cdef class PyRectangle:
- cdef Rectangle*c_rect # hold a pointer to the C++ instance which we're wrapping
-
- def __cinit__(self, int x0, int y0, int x1, int y1):
- self.c_rect = new Rectangle(x0, y0, x1, y1)
-
- def __dealloc__(self):
- del self.c_rect
+# distutils: language = c++ + +from Rectangle cimport Rectangle + +cdef class PyRectangle: + cdef Rectangle*c_rect # hold a pointer to the C++ instance which we're wrapping + + def __cinit__(self, int x0, int y0, int x1, int y1): + self.c_rect = new Rectangle(x0, y0, x1, y1) + + def __dealloc__(self): + del self.c_rect diff --git a/docs/examples/userguide/wrapping_CPlusPlus/setup.py b/docs/examples/userguide/wrapping_CPlusPlus/setup.py index 0c89865d6..09009d28d 100644 --- a/docs/examples/userguide/wrapping_CPlusPlus/setup.py +++ b/docs/examples/userguide/wrapping_CPlusPlus/setup.py @@ -1,4 +1,4 @@ -from distutils.core import setup +from setuptools import setup from Cython.Build import cythonize diff --git a/docs/examples/userguide/wrapping_CPlusPlus/templates.pyx b/docs/examples/userguide/wrapping_CPlusPlus/templates.pyx index 8e7383ca2..4ff232b82 100644 --- a/docs/examples/userguide/wrapping_CPlusPlus/templates.pyx +++ b/docs/examples/userguide/wrapping_CPlusPlus/templates.pyx @@ -1,30 +1,30 @@ -# distutils: language = c++
-
-# import dereference and increment operators
-from cython.operator cimport dereference as deref, preincrement as inc
-
-cdef extern from "<vector>" namespace "std":
- cdef cppclass vector[T]:
- cppclass iterator:
- T operator*()
- iterator operator++()
- bint operator==(iterator)
- bint operator!=(iterator)
- vector()
- void push_back(T&)
- T& operator[](int)
- T& at(int)
- iterator begin()
- iterator end()
-
-cdef vector[int] *v = new vector[int]()
-cdef int i
-for i in range(10):
- v.push_back(i)
-
-cdef vector[int].iterator it = v.begin()
-while it != v.end():
- print(deref(it))
- inc(it)
-
-del v
+# distutils: language = c++ + +# import dereference and increment operators +from cython.operator cimport dereference as deref, preincrement as inc + +cdef extern from "<vector>" namespace "std": + cdef cppclass vector[T]: + cppclass iterator: + T operator*() + iterator operator++() + bint operator==(iterator) + bint operator!=(iterator) + vector() + void push_back(T&) + T& operator[](int) + T& at(int) + iterator begin() + iterator end() + +cdef vector[int] *v = new vector[int]() +cdef int i +for i in range(10): + v.push_back(i) + +cdef vector[int].iterator it = v.begin() +while it != v.end(): + print(deref(it)) + inc(it) + +del v diff --git a/docs/examples/userguide/wrapping_CPlusPlus/vector_demo.pyx b/docs/examples/userguide/wrapping_CPlusPlus/vector_demo.pyx index d7fdfc969..f1697e1ec 100644 --- a/docs/examples/userguide/wrapping_CPlusPlus/vector_demo.pyx +++ b/docs/examples/userguide/wrapping_CPlusPlus/vector_demo.pyx @@ -1,15 +1,15 @@ -# distutils: language = c++
-
-from libcpp.vector cimport vector
-
-cdef vector[int] vect
-cdef int i, x
-
-for i in range(10):
- vect.push_back(i)
-
-for i in range(10):
- print(vect[i])
-
-for x in vect:
- print(x)
+# distutils: language = c++ + +from libcpp.vector cimport vector + +cdef vector[int] vect +cdef int i, x + +for i in range(10): + vect.push_back(i) + +for i in range(10): + print(vect[i]) + +for x in vect: + print(x) diff --git a/docs/examples/userguide/wrapping_CPlusPlus/wrapper_vector.pyx b/docs/examples/userguide/wrapping_CPlusPlus/wrapper_vector.pyx index 592e83ad9..4cdf12fc2 100644 --- a/docs/examples/userguide/wrapping_CPlusPlus/wrapper_vector.pyx +++ b/docs/examples/userguide/wrapping_CPlusPlus/wrapper_vector.pyx @@ -1,17 +1,17 @@ -# distutils: language = c++
-
-from libcpp.vector cimport vector
-
-
-cdef class VectorStack:
- cdef vector[int] v
-
- def push(self, x):
- self.v.push_back(x)
-
- def pop(self):
- if self.v.empty():
- raise IndexError()
- x = self.v.back()
- self.v.pop_back()
- return x
+# distutils: language = c++ + +from libcpp.vector cimport vector + + +cdef class VectorStack: + cdef vector[int] v + + def push(self, x): + self.v.push_back(x) + + def pop(self): + if self.v.empty(): + raise IndexError() + x = self.v.back() + self.v.pop_back() + return x diff --git a/docs/index.rst b/docs/index.rst index 3e14d8dad..0103d8d9f 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -2,7 +2,7 @@ Welcome to Cython's Documentation ================================= -Also see the `Cython project homepage <http://cython.org/>`_. +Also see the `Cython project homepage <https://cython.org/>`_. .. toctree:: :maxdepth: 2 @@ -10,4 +10,6 @@ Also see the `Cython project homepage <http://cython.org/>`_. src/quickstart/index src/tutorial/index src/userguide/index + src/reference/index + Contributing <CONTRIBUTING> src/changes diff --git a/docs/make.bat b/docs/make.bat index fb25acc6a..3e719777d 100644 --- a/docs/make.bat +++ b/docs/make.bat @@ -56,7 +56,7 @@ if errorlevel 9009 ( echo.may add the Sphinx directory to PATH.
echo.
echo.If you don't have Sphinx installed, grab it from
- echo.http://sphinx-doc.org/
+ echo.https://sphinx-doc.org/
exit /b 1
)
diff --git a/docs/sphinxext/cython_highlighting.py b/docs/sphinxext/cython_highlighting.py deleted file mode 100644 index 06e11b891..000000000 --- a/docs/sphinxext/cython_highlighting.py +++ /dev/null @@ -1,183 +0,0 @@ -import re - -from pygments.lexer import Lexer, RegexLexer, ExtendedRegexLexer, \ - LexerContext, include, combined, do_insertions, bygroups, using -from pygments.token import Error, Text, \ - Comment, Operator, Keyword, Name, String, Number, Generic, Punctuation -from pygments.util import get_bool_opt, get_list_opt, shebang_matches -from pygments import unistring as uni - -from sphinx import highlighting - - -line_re = re.compile('.*?\n') - -class CythonLexer(RegexLexer): - """ - For `Cython <http://cython.org>`_ source code. - """ - - name = 'Cython' - aliases = ['cython', 'pyx'] - filenames = ['*.pyx', '*.pxd', '*.pxi'] - mimetypes = ['text/x-cython', 'application/x-cython'] - - tokens = { - 'root': [ - (r'\n', Text), - (r'^(\s*)("""(?:.|\n)*?""")', bygroups(Text, String.Doc)), - (r"^(\s*)('''(?:.|\n)*?''')", bygroups(Text, String.Doc)), - (r'[^\S\n]+', Text), - (r'#.*$', Comment), - (r'[]{}:(),;[]', Punctuation), - (r'\\\n', Text), - (r'\\', Text), - (r'(in|is|and|or|not)\b', Operator.Word), - (r'(<)([a-zA-Z0-9.?]+)(>)', - bygroups(Punctuation, Keyword.Type, Punctuation)), - (r'!=|==|<<|>>|[-~+/*%=<>&^|.?]', Operator), - (r'(from)(\d+)(<=)(\s+)(<)(\d+)(:)', - bygroups(Keyword, Number.Integer, Operator, Name, Operator, - Name, Punctuation)), - include('keywords'), - (r'(def|property)(\s+)', bygroups(Keyword, Text), 'funcname'), - (r'(cp?def)(\s+)', bygroups(Keyword, Text), 'cdef'), - (r'(class|struct)(\s+)', bygroups(Keyword, Text), 'classname'), - (r'(from)(\s+)', bygroups(Keyword, Text), 'fromimport'), - (r'(c?import)(\s+)', bygroups(Keyword, Text), 'import'), - include('builtins'), - include('backtick'), - ('(?:[rR]|[uU][rR]|[rR][uU])"""', String, 'tdqs'), - ("(?:[rR]|[uU][rR]|[rR][uU])'''", String, 'tsqs'), - ('(?:[rR]|[uU][rR]|[rR][uU])"', String, 'dqs'), - ("(?:[rR]|[uU][rR]|[rR][uU])'", String, 'sqs'), - ('[uU]?"""', String, combined('stringescape', 'tdqs')), - ("[uU]?'''", String, combined('stringescape', 'tsqs')), - ('[uU]?"', String, combined('stringescape', 'dqs')), - ("[uU]?'", String, combined('stringescape', 'sqs')), - include('name'), - include('numbers'), - ], - 'keywords': [ - (r'(assert|break|by|continue|ctypedef|del|elif|else|except\??|exec|' - r'finally|for|gil|global|if|include|lambda|nogil|pass|print|raise|' - r'return|try|while|yield|as|with)\b', Keyword), - (r'(DEF|IF|ELIF|ELSE)\b', Comment.Preproc), - ], - 'builtins': [ - (r'(?<!\.)(__import__|abs|all|any|apply|basestring|bin|bool|buffer|' - r'bytearray|bytes|callable|chr|classmethod|cmp|coerce|compile|' - r'complex|delattr|dict|dir|divmod|enumerate|eval|execfile|exit|' - r'file|filter|float|frozenset|getattr|globals|hasattr|hash|hex|id|' - r'input|int|intern|isinstance|issubclass|iter|len|list|locals|' - r'long|map|max|min|next|object|oct|open|ord|pow|property|range|' - r'raw_input|reduce|reload|repr|reversed|round|set|setattr|slice|' - r'sorted|staticmethod|str|sum|super|tuple|type|unichr|unicode|' - r'vars|xrange|zip)\b', Name.Builtin), - (r'(?<!\.)(self|None|Ellipsis|NotImplemented|False|True|NULL' - r')\b', Name.Builtin.Pseudo), - (r'(?<!\.)(ArithmeticError|AssertionError|AttributeError|' - r'BaseException|DeprecationWarning|EOFError|EnvironmentError|' - r'Exception|FloatingPointError|FutureWarning|GeneratorExit|IOError|' - r'ImportError|ImportWarning|IndentationError|IndexError|KeyError|' - r'KeyboardInterrupt|LookupError|MemoryError|NameError|' - r'NotImplemented|NotImplementedError|OSError|OverflowError|' - r'OverflowWarning|PendingDeprecationWarning|ReferenceError|' - r'RuntimeError|RuntimeWarning|StandardError|StopIteration|' - r'SyntaxError|SyntaxWarning|SystemError|SystemExit|TabError|' - r'TypeError|UnboundLocalError|UnicodeDecodeError|' - r'UnicodeEncodeError|UnicodeError|UnicodeTranslateError|' - r'UnicodeWarning|UserWarning|ValueError|Warning|ZeroDivisionError' - r')\b', Name.Exception), - ], - 'numbers': [ - (r'(\d+\.?\d*|\d*\.\d+)([eE][+-]?[0-9]+)?', Number.Float), - (r'0\d+', Number.Oct), - (r'0[xX][a-fA-F0-9]+', Number.Hex), - (r'\d+L', Number.Integer.Long), - (r'\d+', Number.Integer) - ], - 'backtick': [ - ('`.*?`', String.Backtick), - ], - 'name': [ - (r'@[a-zA-Z0-9_]+', Name.Decorator), - ('[a-zA-Z_][a-zA-Z0-9_]*', Name), - ], - 'funcname': [ - ('[a-zA-Z_][a-zA-Z0-9_]*', Name.Function, '#pop') - ], - 'cdef': [ - (r'(public|readonly|extern|api|inline)\b', Keyword.Reserved), - (r'(struct|enum|union|class)\b', Keyword), - (r'([a-zA-Z_][a-zA-Z0-9_]*)(\s*)(?=[(:#=]|$)', - bygroups(Name.Function, Text), '#pop'), - (r'([a-zA-Z_][a-zA-Z0-9_]*)(\s*)(,)', - bygroups(Name.Function, Text, Punctuation)), - (r'from\b', Keyword, '#pop'), - (r'as\b', Keyword), - (r':', Punctuation, '#pop'), - (r'(?=["\'])', Text, '#pop'), - (r'[a-zA-Z_][a-zA-Z0-9_]*', Keyword.Type), - (r'.', Text), - ], - 'classname': [ - ('[a-zA-Z_][a-zA-Z0-9_]*', Name.Class, '#pop') - ], - 'import': [ - (r'(\s+)(as)(\s+)', bygroups(Text, Keyword, Text)), - (r'[a-zA-Z_][a-zA-Z0-9_.]*', Name.Namespace), - (r'(\s*)(,)(\s*)', bygroups(Text, Operator, Text)), - (r'', Text, '#pop') # all else: go back - ], - 'fromimport': [ - (r'(\s+)(c?import)\b', bygroups(Text, Keyword), '#pop'), - (r'[a-zA-Z_.][a-zA-Z0-9_.]*', Name.Namespace), - # ``cdef foo from "header"``, or ``for foo from 0 < i < 10`` - (r'', Text, '#pop'), - ], - 'stringescape': [ - (r'\\([\\abfnrtv"\']|\n|N{.*?}|u[a-fA-F0-9]{4}|' - r'U[a-fA-F0-9]{8}|x[a-fA-F0-9]{2}|[0-7]{1,3})', String.Escape) - ], - 'strings': [ - (r'%(\([a-zA-Z0-9]+\))?[-#0 +]*([0-9]+|[*])?(\.([0-9]+|[*]))?' - '[hlL]?[diouxXeEfFgGcrs%]', String.Interpol), - (r'[^\\\'"%\n]+', String), - # quotes, percents and backslashes must be parsed one at a time - (r'[\'"\\]', String), - # unhandled string formatting sign - (r'%', String) - # newlines are an error (use "nl" state) - ], - 'nl': [ - (r'\n', String) - ], - 'dqs': [ - (r'"', String, '#pop'), - (r'\\\\|\\"|\\\n', String.Escape), # included here again for raw strings - include('strings') - ], - 'sqs': [ - (r"'", String, '#pop'), - (r"\\\\|\\'|\\\n", String.Escape), # included here again for raw strings - include('strings') - ], - 'tdqs': [ - (r'"""', String, '#pop'), - include('strings'), - include('nl') - ], - 'tsqs': [ - (r"'''", String, '#pop'), - include('strings'), - include('nl') - ], - } - - ##TODO: fix this, as shebang lines don't make sense for cython. - def analyse_text(text): - return shebang_matches(text, r'pythonw?(2\.\d)?') - -def setup(app): - app.add_lexer('cython', CythonLexer()) diff --git a/docs/sphinxext/ipython_console_highlighting.py b/docs/sphinxext/ipython_console_highlighting.py deleted file mode 100644 index b9b4f5f29..000000000 --- a/docs/sphinxext/ipython_console_highlighting.py +++ /dev/null @@ -1,77 +0,0 @@ -from pygments.lexer import Lexer, do_insertions -from pygments.lexers.agile import PythonConsoleLexer, PythonLexer, \ - PythonTracebackLexer -from pygments.token import Comment, Generic -from sphinx import highlighting -import re - -line_re = re.compile('.*?\n') - -class IPythonConsoleLexer(Lexer): - """ - For IPython console output or doctests, such as: - - Tracebacks are not currently supported. - - .. sourcecode:: ipython - - In [1]: a = 'foo' - - In [2]: a - Out[2]: 'foo' - - In [3]: print a - foo - - In [4]: 1 / 0 - """ - name = 'IPython console session' - aliases = ['ipython'] - mimetypes = ['text/x-ipython-console'] - input_prompt = re.compile("(In \[[0-9]+\]: )|( \.\.\.+:)") - output_prompt = re.compile("(Out\[[0-9]+\]: )|( \.\.\.+:)") - continue_prompt = re.compile(" \.\.\.+:") - tb_start = re.compile("\-+") - - def get_tokens_unprocessed(self, text): - pylexer = PythonLexer(**self.options) - tblexer = PythonTracebackLexer(**self.options) - - curcode = '' - insertions = [] - for match in line_re.finditer(text): - line = match.group() - input_prompt = self.input_prompt.match(line) - continue_prompt = self.continue_prompt.match(line.rstrip()) - output_prompt = self.output_prompt.match(line) - if line.startswith("#"): - insertions.append((len(curcode), - [(0, Comment, line)])) - elif input_prompt is not None: - insertions.append((len(curcode), - [(0, Generic.Prompt, input_prompt.group())])) - curcode += line[input_prompt.end():] - elif continue_prompt is not None: - insertions.append((len(curcode), - [(0, Generic.Prompt, continue_prompt.group())])) - curcode += line[continue_prompt.end():] - elif output_prompt is not None: - insertions.append((len(curcode), - [(0, Generic.Output, output_prompt.group())])) - curcode += line[output_prompt.end():] - else: - if curcode: - for item in do_insertions(insertions, - pylexer.get_tokens_unprocessed(curcode)): - yield item - curcode = '' - insertions = [] - yield match.start(), Generic.Output, line - if curcode: - for item in do_insertions(insertions, - pylexer.get_tokens_unprocessed(curcode)): - yield item - - -def setup(app): - app.add_lexer('ipython', IPythonConsoleLexer()) diff --git a/docs/src/cimport-warning b/docs/src/cimport-warning new file mode 100644 index 000000000..c19819b93 --- /dev/null +++ b/docs/src/cimport-warning @@ -0,0 +1,7 @@ +.. warning:: + + This code uses an external native (non-Python) library through a ``cimport``. + Cython compilation enables this, but there is no support for this from + plain Python. Trying to run this code from Python (without compilation) + will fail when accessing the external library. + This is described in more detail in :ref:`calling-c-functions`. diff --git a/docs/src/donating.rst b/docs/src/donating.rst new file mode 100644 index 000000000..33b9558e6 --- /dev/null +++ b/docs/src/donating.rst @@ -0,0 +1,49 @@ +:orphan: + +🌷️ Thank you for your interest in supporting Cython! 🌷️ +========================================================= + +Managing, maintaining and advancing a project as large as Cython takes +**a lot of time and dedication**. + +**Your support can make a difference** +for a great tool that helps you every day! + +Please consider signing a subscription for continuous project support via + +* `GitHub Sponsors <https://github.com/users/scoder/sponsorship>`_ +* `Tidelift <https://tidelift.com/subscription/pkg/pypi-cython>`_ +* `PayPal <https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=HLS9JEYD4ETB6&source=url>`_ + +or donating via + +* `PayPal <https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=HLS9JEYD4ETB6&source=url>`_ + +Note that PayPal takes 5 - 15% fees for small non-EUR payments, +which is money that *you pay without helping us*. +Consider signing up for a GitHub Sponsors subscription instead, +which is currently free of additional charges. + +Also note that we are not accepting donations in crypto currencies. +Much of the development for Cython is done in a carbon-neutral way +or with compensated and very low emissions. +Crypto currencies do not fit into this ambition. + + +Legal Notice for Donations +-------------------------- + +Any donation that you make to the Cython project is voluntary and +is not a fee for any services, goods, or advantages. By making +a donation to the Cython project, you acknowledge that we have the +right to use the money you donate in any lawful way and for any +lawful purpose we see fit and we are not obligated to disclose +the way and purpose to any party unless required by applicable +law. Although Cython is free software, to the best of our knowledge +the Cython project does not have any tax exempt status. The Cython +project is neither a registered non-profit corporation nor a +registered charity in any country. Your donation may or may not +be tax-deductible; please consult your tax advisor in this matter. +We will not publish or disclose your name and/or e-mail address +without your consent, unless required by applicable law. Your +donation is non-refundable. diff --git a/docs/src/quickstart/build.rst b/docs/src/quickstart/build.rst index 628ed2604..5d9e8a307 100644 --- a/docs/src/quickstart/build.rst +++ b/docs/src/quickstart/build.rst @@ -3,23 +3,23 @@ Building Cython code Cython code must, unlike Python, be compiled. This happens in two stages: - - A ``.pyx`` file is compiled by Cython to a ``.c`` file, containing + - A ``.pyx`` or ``.py`` file is compiled by Cython to a ``.c`` file, containing the code of a Python extension module. - The ``.c`` file is compiled by a C compiler to a ``.so`` file (or ``.pyd`` on Windows) which can be ``import``-ed directly into a Python session. - Distutils or setuptools take care of this part. + `setuptools <https://setuptools.readthedocs.io/>`_ takes care of this part. Although Cython can call them for you in certain cases. - -To understand fully the Cython + distutils/setuptools build process, + +To understand fully the Cython + setuptools build process, one may want to read more about `distributing Python modules <https://docs.python.org/3/distributing/index.html>`_. There are several ways to build Cython code: - - Write a distutils/setuptools ``setup.py``. This is the normal and recommended way. + - Write a setuptools ``setup.py``. This is the normal and recommended way. - Use :ref:`Pyximport<pyximport>`, importing Cython ``.pyx`` files as if they - were ``.py`` files (using distutils to compile and build in the background). + were ``.py`` files (using setuptools to compile and build in the background). This method is easier than writing a ``setup.py``, but is not very flexible. So you'll need to write a ``setup.py`` if, for example, you need certain compilations options. - Run the ``cython`` command-line utility manually to produce the ``.c`` file @@ -30,12 +30,12 @@ There are several ways to build Cython code: both of which allow Cython code inline. This is the easiest way to get started writing Cython code and running it. -Currently, using distutils or setuptools is the most common way Cython files are built and distributed. +Currently, using setuptools is the most common way Cython files are built and distributed. The other methods are described in more detail in the :ref:`compilation` section of the reference manual. -Building a Cython module using distutils ----------------------------------------- +Building a Cython module using setuptools +----------------------------------------- Imagine a simple "hello world" script in a file ``hello.pyx``: @@ -49,11 +49,10 @@ To build, run ``python setup.py build_ext --inplace``. Then simply start a Python session and do ``from hello import say_hello_to`` and use the imported function as you see fit. -One caveat if you use setuptools instead of distutils, the default -action when running ``python setup.py install`` is to create a zipped -``egg`` file which will not work with ``cimport`` for ``pxd`` files -when you try to use them from a dependent package. -To prevent this, include ``zip_safe=False`` in the arguments to ``setup()``. +One caveat: the default action when running ``python setup.py install`` is to +create a zipped ``egg`` file which will not work with ``cimport`` for ``pxd`` +files when you try to use them from a dependent package. To prevent this, +include ``zip_safe=False`` in the arguments to ``setup()``. .. _jupyter-notebook: @@ -64,7 +63,7 @@ Cython can be used conveniently and interactively from a web browser through the Jupyter notebook. To install Jupyter notebook, e.g. into a virtualenv, use pip: -.. sourcecode:: bash +.. code-block:: bash (venv)$ pip install jupyter (venv)$ jupyter notebook @@ -74,14 +73,32 @@ and load the ``Cython`` extension from within the Jupyter notebook:: %load_ext Cython -Then, prefix a cell with the ``%%cython`` marker to compile it:: +Then, prefix a cell with the ``%%cython`` marker to compile it + +.. tabs:: + + .. group-tab:: Pure Python + + .. code-block:: python + + %%cython + + a: cython.int = 0 + for i in range(10): + a += i + print(a) + + + .. group-tab:: Cython + + .. code-block:: python - %%cython + %%cython - cdef int a = 0 - for i in range(10): - a += i - print(a) + cdef int a = 0 + for i in range(10): + a += i + print(a) You can show Cython's code analysis by passing the ``--annotate`` option:: @@ -104,5 +121,6 @@ Using the Sage notebook functions defined in a Cython cell imported into the running session. -.. [Jupyter] http://jupyter.org/ -.. [Sage] W. Stein et al., Sage Mathematics Software, http://www.sagemath.org/ +.. [Jupyter] https://jupyter.org/ +.. + [Sage] W. Stein et al., Sage Mathematics Software, https://www.sagemath.org/ diff --git a/docs/src/quickstart/cythonize.rst b/docs/src/quickstart/cythonize.rst index 22cad0470..09cbb470f 100644 --- a/docs/src/quickstart/cythonize.rst +++ b/docs/src/quickstart/cythonize.rst @@ -1,6 +1,9 @@ Faster code via static typing ============================= +.. include:: + ../two-syntax-variants-used + Cython is a Python compiler. This means that it can compile normal Python code without changes (with a few obvious exceptions of some as-yet unsupported language features, see :ref:`Cython limitations<cython-limitations>`). @@ -33,6 +36,7 @@ Typing Variables Consider the following pure Python code: .. literalinclude:: ../../examples/quickstart/cythonize/integrate.py + :caption: integrate.py Simply compiling this in Cython merely gives a 35% speedup. This is better than nothing, but adding some static types can make a much larger @@ -40,7 +44,17 @@ difference. With additional type declarations, this might look like: -.. literalinclude:: ../../examples/quickstart/cythonize/integrate_cy.pyx +.. tabs:: + + .. group-tab:: Pure Python + + .. literalinclude:: ../../examples/quickstart/cythonize/integrate_cy.py + :caption: integrate_cy.py + + .. group-tab:: Cython + + .. literalinclude:: ../../examples/quickstart/cythonize/integrate_cy.pyx + :caption: integrate_cy.pyx Since the iterator variable ``i`` is typed with C semantics, the for-loop will be compiled to pure C code. Typing ``a``, ``s`` and ``dx`` is important as they are involved @@ -55,27 +69,40 @@ Typing Functions Python function calls can be expensive -- in Cython doubly so because one might need to convert to and from Python objects to do the call. -In our example above, the argument is assumed to be a C double both inside f() +In our example above, the argument is assumed to be a C double both inside ``f()`` and in the call to it, yet a Python ``float`` object must be constructed around the argument in order to pass it. -Therefore Cython provides a syntax for declaring a C-style function, -the cdef keyword: +Therefore, Cython provides a way for declaring a C-style function, +the Cython specific ``cdef`` statement, as well as the ``@cfunc`` decorator to +declare C-style functions in Python syntax. Both approaches are +equivalent and produce the same C code: + +.. tabs:: + + .. group-tab:: Pure Python + + .. literalinclude:: ../../examples/quickstart/cythonize/cdef_keyword.py -.. literalinclude:: ../../examples/quickstart/cythonize/cdef_keyword.pyx + .. group-tab:: Cython + + .. literalinclude:: ../../examples/quickstart/cythonize/cdef_keyword.pyx Some form of except-modifier should usually be added, otherwise Cython will not be able to propagate exceptions raised in the function (or a function it calls). The ``except? -2`` means that an error will be checked for if ``-2`` is returned (though the ``?`` indicates that ``-2`` may also -be used as a valid return value). +be used as a valid return value). The same can be expressed using only Python +syntax with the decorator ``@exceptval(-2, check=True)``. + Alternatively, the slower ``except *`` is always safe. An except clause can be left out if the function returns a Python object or if it is guaranteed that an exception will not be raised -within the function call. +within the function call. Again, Cython provides the decorator ``@exceptval(check=True)`` +providing the same functionality. -A side-effect of cdef is that the function is no longer available from -Python-space, as Python wouldn't know how to call it. It is also no +A side-effect of ``cdef`` (and the ``@cfunc`` decorator) is that the function is no longer +visible from Python-space, as Python wouldn't know how to call it. It is also no longer possible to change :func:`f` at runtime. Using the ``cpdef`` keyword instead of ``cdef``, a Python wrapper is also @@ -84,7 +111,8 @@ typed values directly) and from Python (wrapping values in Python objects). In fact, ``cpdef`` does not just provide a Python wrapper, it also installs logic to allow the method to be overridden by python methods, even when called from within cython. This does add a tiny overhead compared to ``cdef`` -methods. +methods. Again, Cython provides a ``@ccall`` decorator which provides the same +functionality as ``cpdef`` keyword. Speedup: 150 times over pure Python. @@ -118,7 +146,15 @@ This report is invaluable when optimizing a function for speed, and for determining when to :ref:`release the GIL <nogil>`: in general, a ``nogil`` block may contain only "white" code. -.. figure:: htmlreport.png +.. tabs:: + + .. group-tab:: Pure Python + + .. figure:: htmlreport_py.png + + .. group-tab:: Cython + + .. figure:: htmlreport_pyx.png Note that Cython deduces the type of local variables based on their assignments (including as loop variable targets) which can also cut down on the need to @@ -135,4 +171,3 @@ with this language feature. It can be of great help to cut down on the need to t everything, but it also can lead to surprises. Especially if one isn't familiar with arithmetic expressions with c types. A quick overview of those can be found `here <https://www.eskimo.com/~scs/cclass/int/sx4cb.html>`_. - diff --git a/docs/src/quickstart/htmlreport.png b/docs/src/quickstart/htmlreport.png Binary files differdeleted file mode 100644 index cc30cec9f..000000000 --- a/docs/src/quickstart/htmlreport.png +++ /dev/null diff --git a/docs/src/quickstart/htmlreport_py.png b/docs/src/quickstart/htmlreport_py.png Binary files differnew file mode 100755 index 000000000..a42a9d1cc --- /dev/null +++ b/docs/src/quickstart/htmlreport_py.png diff --git a/docs/src/quickstart/htmlreport_pyx.png b/docs/src/quickstart/htmlreport_pyx.png Binary files differnew file mode 100755 index 000000000..bc9cff2f9 --- /dev/null +++ b/docs/src/quickstart/htmlreport_pyx.png diff --git a/docs/src/quickstart/install.rst b/docs/src/quickstart/install.rst index a71adffb5..8b5f4c350 100644 --- a/docs/src/quickstart/install.rst +++ b/docs/src/quickstart/install.rst @@ -7,9 +7,7 @@ Many scientific Python distributions, such as Anaconda [Anaconda]_, Enthought Canopy [Canopy]_, and Sage [Sage]_, bundle Cython and no setup is needed. Note however that if your distribution ships a version of Cython which is too old you can still -use the instructions below to update Cython. Everything in this -tutorial should work with Cython 0.11.2 and newer, unless a footnote -says otherwise. +use the instructions below to update Cython. Unlike most Python software, Cython requires a C compiler to be present on the system. The details of getting a C compiler varies @@ -24,13 +22,16 @@ according to the system used: XCode, which can be retrieved from the Mac OS X's install DVDs or from https://developer.apple.com/. - - **Windows** A popular option is to use the open source MinGW (a + - **Windows** The CPython project recommends building extension modules + (including Cython modules) with the same compiler that Python was + built with. This is usually a specific version of Microsoft Visual + C/C++ (MSVC) - see https://wiki.python.org/moin/WindowsCompilers. + MSVC is the only compiler that Cython is currently tested with on + Windows. A possible alternative is the open source MinGW (a Windows distribution of gcc). See the appendix for instructions for setting up MinGW manually. Enthought Canopy and Python(x,y) bundle MinGW, but some of the configuration steps in the appendix might - still be necessary. Another option is to use Microsoft's Visual C. - One must then use the same version which the installed Python was - compiled with. + still be necessary. .. dagss tried other forms of ReST lists and they didn't look nice .. with rst2latex. @@ -41,7 +42,7 @@ The simplest way of installing Cython is by using ``pip``:: The newest Cython release can always be downloaded from -http://cython.org. Unpack the tarball or zip file, enter the +https://cython.org/. Unpack the tarball or zip file, enter the directory, and then run:: python setup.py install @@ -59,4 +60,4 @@ with .. [Anaconda] https://docs.anaconda.com/anaconda/ .. [Canopy] https://www.enthought.com/product/canopy/ -.. [Sage] W. Stein et al., Sage Mathematics Software, http://www.sagemath.org/ +.. [Sage] W. Stein et al., Sage Mathematics Software, https://www.sagemath.org/ diff --git a/docs/src/quickstart/jupyter.png b/docs/src/quickstart/jupyter.png Binary files differindex 84b3543ad..34b38df6d 100644 --- a/docs/src/quickstart/jupyter.png +++ b/docs/src/quickstart/jupyter.png diff --git a/docs/src/quickstart/overview.rst b/docs/src/quickstart/overview.rst index 1585f89fe..1a378e837 100644 --- a/docs/src/quickstart/overview.rst +++ b/docs/src/quickstart/overview.rst @@ -1,7 +1,7 @@ Cython - an overview ==================== -[Cython] is a programming language that makes writing C extensions +[Cython]_ is a programming language that makes writing C extensions for the Python language as easy as Python itself. It aims to become a superset of the [Python]_ language which gives it high-level, object-oriented, functional, and dynamic programming. Its main feature @@ -44,13 +44,13 @@ thus merges the two worlds into a very broadly applicable programming language. .. [Cython] G. Ewing, R. W. Bradshaw, S. Behnel, D. S. Seljebotn et al., - The Cython compiler, http://cython.org. + The Cython compiler, https://cython.org/. .. [IronPython] Jim Hugunin et al., https://archive.codeplex.com/?p=IronPython. .. [Jython] J. Huginin, B. Warsaw, F. Bock, et al., - Jython: Python for the Java platform, http://www.jython.org. + Jython: Python for the Java platform, https://www.jython.org. .. [PyPy] The PyPy Group, PyPy: a Python implementation written in Python, - http://pypy.org. + https://pypy.org/. .. [Pyrex] G. Ewing, Pyrex: C-Extensions for Python, - http://www.cosc.canterbury.ac.nz/greg.ewing/python/Pyrex/ + https://www.cosc.canterbury.ac.nz/greg.ewing/python/Pyrex/ .. [Python] G. van Rossum et al., The Python programming language, https://www.python.org/. diff --git a/docs/src/reference/directives.rst b/docs/src/reference/directives.rst index f527536a8..46d7fa9c4 100644 --- a/docs/src/reference/directives.rst +++ b/docs/src/reference/directives.rst @@ -1,3 +1,5 @@ +:orphan: + Compiler Directives =================== diff --git a/docs/src/reference/extension_types.rst b/docs/src/reference/extension_types.rst index ea26f38a0..9fe32660f 100644 --- a/docs/src/reference/extension_types.rst +++ b/docs/src/reference/extension_types.rst @@ -1,3 +1,5 @@ +:orphan: + .. highlight:: cython *************** @@ -59,7 +61,7 @@ This section was moved to :ref:`arithmetic_methods`. Rich Comparisons ================ -This section was moved to :ref:`righ_comparisons`. +This section was moved to :ref:`rich_comparisons`. The ``__next__()`` Method ========================= diff --git a/docs/src/reference/language_basics.rst b/docs/src/reference/language_basics.rst index bd8b1e38c..8d7cd0b06 100644 --- a/docs/src/reference/language_basics.rst +++ b/docs/src/reference/language_basics.rst @@ -1,3 +1,5 @@ +:orphan: + .. highlight:: cython diff --git a/docs/src/tutorial/appendix.rst b/docs/src/tutorial/appendix.rst index b0ab0426e..82f225bbf 100644 --- a/docs/src/tutorial/appendix.rst +++ b/docs/src/tutorial/appendix.rst @@ -2,7 +2,7 @@ Appendix: Installing MinGW on Windows ===================================== 1. Download the MinGW installer from - http://www.mingw.org/wiki/HOWTO_Install_the_MinGW_GCC_Compiler_Suite. + https://www.mingw.org/wiki/HOWTO_Install_the_MinGW_GCC_Compiler_Suite. (As of this writing, the download link is a bit difficult to find; it's under "About" in the menu on the left-hand side). You want the file @@ -28,4 +28,65 @@ procedure. Any contributions towards making the Windows install process smoother is welcomed; it is an unfortunate fact that none of the regular Cython developers have convenient access to Windows. +Python 3.8+ +----------- + +Since Python 3.8, the search paths of DLL dependencies has been reset. +(`changelog <https://docs.python.org/3/whatsnew/3.8.html#bpo-36085-whatsnew>`_) + +Only the system paths, the directory containing the DLL or PYD file +are searched for load-time dependencies. +Instead, a new function `os.add_dll_directory() <https://docs.python.org/3.8/library/os.html#os.add_dll_directory>`_ +was added to supply additional search paths. But such a runtime update is not applicable in all situations. + +Unlike MSVC, MinGW has its owned standard libraries such as ``libstdc++-6.dll``, +which are not placed in the system path (such as ``C:\Windows\System32``). +For a C++ example, you can check the dependencies by MSVC tool ``dumpbin``:: + + > dumpbin /dependents my_gnu_extension.cp38-win_amd64.pyd + ... + Dump of file my_gnu_extension.cp38-win_amd64.pyd + + File Type: DLL + + Image has the following dependencies: + + python38.dll + KERNEL32.dll + msvcrt.dll + libgcc_s_seh-1.dll + libstdc++-6.dll + ... + +These standard libraries can be embedded via static linking, by adding the following options to the linker:: + + -static-libgcc -static-libstdc++ -Wl,-Bstatic,--whole-archive -lwinpthread -Wl,--no-whole-archive + +In ``setup.py``, a cross platform config can be added through +extending ``build_ext`` class:: + + from setuptools import setup + from setuptools.command.build_ext import build_ext + + link_args = ['-static-libgcc', + '-static-libstdc++', + '-Wl,-Bstatic,--whole-archive', + '-lwinpthread', + '-Wl,--no-whole-archive'] + + ... # Add extensions + + class Build(build_ext): + def build_extensions(self): + if self.compiler.compiler_type == 'mingw32': + for e in self.extensions: + e.extra_link_args = link_args + super(Build, self).build_extensions() + + setup( + ... + cmdclass={'build_ext': Build}, + ... + ) + .. [WinInst] https://github.com/cython/cython/wiki/CythonExtensionsOnWindows diff --git a/docs/src/tutorial/caveats.rst b/docs/src/tutorial/caveats.rst index 65443ae26..192313162 100644 --- a/docs/src/tutorial/caveats.rst +++ b/docs/src/tutorial/caveats.rst @@ -5,7 +5,6 @@ Since Cython mixes C and Python semantics, some things may be a bit surprising or unintuitive. Work always goes on to make Cython more natural for Python users, so this list may change in the future. - - ``10**-2 == 0``, instead of ``0.01`` like in Python. - Given two typed ``int`` variables ``a`` and ``b``, ``a % b`` has the same sign as the second argument (following Python semantics) rather than having the same sign as the first (as in C). The C behavior can be diff --git a/docs/src/tutorial/cdef_classes.rst b/docs/src/tutorial/cdef_classes.rst index a95b802a8..ec2566dbd 100644 --- a/docs/src/tutorial/cdef_classes.rst +++ b/docs/src/tutorial/cdef_classes.rst @@ -1,5 +1,9 @@ +*********************************** Extension types (aka. cdef classes) -=================================== +*********************************** + +.. include:: + ../two-syntax-variants-used To support object-oriented programming, Cython supports writing normal Python classes exactly as in Python: @@ -24,17 +28,33 @@ single inheritance. Normal Python classes, on the other hand, can inherit from any number of Python classes and extension types, both in Cython code and pure Python code. +.. tabs:: + .. group-tab:: Pure Python + + .. literalinclude:: ../../examples/tutorial/cdef_classes/math_function_2.py + + .. group-tab:: Cython + + .. literalinclude:: ../../examples/tutorial/cdef_classes/math_function_2.pyx + So far our integration example has not been very useful as it only integrates a single hard-coded function. In order to remedy this, with hardly sacrificing speed, we will use a cdef class to represent a function on floating point numbers: -.. literalinclude:: ../../examples/tutorial/cdef_classes/math_function_2.pyx - The directive cpdef makes two versions of the method available; one fast for use from Cython and one slower for use from Python. Then: -.. literalinclude:: ../../examples/tutorial/cdef_classes/sin_of_square.pyx +.. tabs:: + .. group-tab:: Pure Python + + .. literalinclude:: ../../examples/tutorial/cdef_classes/sin_of_square.py + :caption: sin_of_square.py + + .. group-tab:: Cython + + .. literalinclude:: ../../examples/tutorial/cdef_classes/sin_of_square.pyx + :caption: sin_of_square.pyx This does slightly more than providing a python wrapper for a cdef method: unlike a cdef method, a cpdef method is fully overridable by @@ -43,13 +63,24 @@ little calling overhead compared to a cdef method. To make the class definitions visible to other modules, and thus allow for efficient C-level usage and inheritance outside of the module that -implements them, we define them in a :file:`sin_of_square.pxd` file: +implements them, we define them in a ``.pxd`` file with the same name +as the module: .. literalinclude:: ../../examples/tutorial/cdef_classes/sin_of_square.pxd + :caption: sin_of_square.pxd Using this, we can now change our integration example: -.. literalinclude:: ../../examples/tutorial/cdef_classes/integrate.pyx +.. tabs:: + .. group-tab:: Pure Python + + .. literalinclude:: ../../examples/tutorial/cdef_classes/integrate.py + :caption: integrate.py + + .. group-tab:: Cython + + .. literalinclude:: ../../examples/tutorial/cdef_classes/integrate.pyx + :caption: integrate.pyx This is almost as fast as the previous code, however it is much more flexible as the function to integrate can be changed. We can even pass in a new @@ -70,32 +101,50 @@ into a Cython module. Some notes on our new implementation of ``evaluate``: - - The fast method dispatch here only works because ``evaluate`` was - declared in ``Function``. Had ``evaluate`` been introduced in - ``SinOfSquareFunction``, the code would still work, but Cython - would have used the slower Python method dispatch mechanism - instead. +- The fast method dispatch here only works because ``evaluate`` was + declared in ``Function``. Had ``evaluate`` been introduced in + ``SinOfSquareFunction``, the code would still work, but Cython + would have used the slower Python method dispatch mechanism + instead. - - In the same way, had the argument ``f`` not been typed, but only - been passed as a Python object, the slower Python dispatch would - be used. +- In the same way, had the argument ``f`` not been typed, but only + been passed as a Python object, the slower Python dispatch would + be used. - - Since the argument is typed, we need to check whether it is - ``None``. In Python, this would have resulted in an ``AttributeError`` - when the ``evaluate`` method was looked up, but Cython would instead - try to access the (incompatible) internal structure of ``None`` as if - it were a ``Function``, leading to a crash or data corruption. +- Since the argument is typed, we need to check whether it is + ``None``. In Python, this would have resulted in an ``AttributeError`` + when the ``evaluate`` method was looked up, but Cython would instead + try to access the (incompatible) internal structure of ``None`` as if + it were a ``Function``, leading to a crash or data corruption. There is a *compiler directive* ``nonecheck`` which turns on checks for this, at the cost of decreased speed. Here's how compiler directives are used to dynamically switch on or off ``nonecheck``: -.. literalinclude:: ../../examples/tutorial/cdef_classes/nonecheck.pyx +.. tabs:: + .. group-tab:: Pure Python + + .. literalinclude:: ../../examples/tutorial/cdef_classes/nonecheck.py + :caption: nonecheck.py + + .. group-tab:: Cython + + .. literalinclude:: ../../examples/tutorial/cdef_classes/nonecheck.pyx + :caption: nonecheck.pyx Attributes in cdef classes behave differently from attributes in regular classes: - - All attributes must be pre-declared at compile-time - - Attributes are by default only accessible from Cython (typed access) - - Properties can be declared to expose dynamic attributes to Python-space +- All attributes must be pre-declared at compile-time +- Attributes are by default only accessible from Cython (typed access) +- Properties can be declared to expose dynamic attributes to Python-space + +.. tabs:: + .. group-tab:: Pure Python + + .. literalinclude:: ../../examples/tutorial/cdef_classes/wave_function.py + :caption: wave_function.py + + .. group-tab:: Cython -.. literalinclude:: ../../examples/tutorial/cdef_classes/wave_function.pyx + .. literalinclude:: ../../examples/tutorial/cdef_classes/wave_function.pyx + :caption: wave_function.pyx diff --git a/docs/src/tutorial/clibraries.rst b/docs/src/tutorial/clibraries.rst index dd91d9ef2..0dfe1a954 100644 --- a/docs/src/tutorial/clibraries.rst +++ b/docs/src/tutorial/clibraries.rst @@ -5,6 +5,9 @@ Using C libraries ****************** +.. include:: + ../two-syntax-variants-used + Apart from writing fast code, one of the main use cases of Cython is to call external C libraries from Python code. As Cython code compiles down to C code itself, it is actually trivial to call C @@ -24,7 +27,7 @@ decide to use its double ended queue implementation. To make the handling easier, however, you decide to wrap it in a Python extension type that can encapsulate all memory management. -.. [CAlg] Simon Howard, C Algorithms library, http://c-algorithms.sourceforge.net/ +.. [CAlg] Simon Howard, C Algorithms library, https://fragglet.github.io/c-algorithms/ Defining external declarations @@ -33,15 +36,17 @@ Defining external declarations You can download CAlg `here <https://codeload.github.com/fragglet/c-algorithms/zip/master>`_. The C API of the queue implementation, which is defined in the header -file ``c-algorithms/src/queue.h``, essentially looks like this: +file :file:`c-algorithms/src/queue.h`, essentially looks like this: .. literalinclude:: ../../examples/tutorial/clibraries/c-algorithms/src/queue.h :language: C + :caption: queue.h To get started, the first step is to redefine the C API in a ``.pxd`` -file, say, ``cqueue.pxd``: +file, say, :file:`cqueue.pxd`: .. literalinclude:: ../../examples/tutorial/clibraries/cqueue.pxd + :caption: cqueue.pxd Note how these declarations are almost identical to the header file declarations, so you can often just copy them over. However, you do @@ -100,20 +105,33 @@ Writing a wrapper class After declaring our C library's API, we can start to design the Queue class that should wrap the C queue. It will live in a file called -``queue.pyx``. [#]_ +:file:`queue.pyx`/:file:`queue.py`. [#]_ -.. [#] Note that the name of the ``.pyx`` file must be different from - the ``cqueue.pxd`` file with declarations from the C library, +.. [#] Note that the name of the ``.pyx``/``.py`` file must be different from + the :file:`cqueue.pxd` file with declarations from the C library, as both do not describe the same code. A ``.pxd`` file next to - a ``.pyx`` file with the same name defines exported - declarations for code in the ``.pyx`` file. As the - ``cqueue.pxd`` file contains declarations of a regular C - library, there must not be a ``.pyx`` file with the same name + a ``.pyx``/``.py`` file with the same name defines exported + declarations for code in the ``.pyx``/``.py`` file. As the + :file:`cqueue.pxd` file contains declarations of a regular C + library, there must not be a ``.pyx``/``.py`` file with the same name that Cython associates with it. Here is a first start for the Queue class: -.. literalinclude:: ../../examples/tutorial/clibraries/queue.pyx +.. tabs:: + + .. group-tab:: Pure Python + + .. literalinclude:: ../../examples/tutorial/clibraries/queue.py + :caption: queue.py + + .. note:: Currently, Cython contains a bug not allowing using + annotations with types containing pointers (GitHub issue :issue:`4293`). + + .. group-tab:: Cython + + .. literalinclude:: ../../examples/tutorial/clibraries/queue.pyx + :caption: queue.pyx Note that it says ``__cinit__`` rather than ``__init__``. While ``__init__`` is available as well, it is not guaranteed to be run (for @@ -122,10 +140,11 @@ ancestor's constructor). Because not initializing C pointers often leads to hard crashes of the Python interpreter, Cython provides ``__cinit__`` which is *always* called immediately on construction, before CPython even considers calling ``__init__``, and which -therefore is the right place to initialise ``cdef`` fields of the new -instance. However, as ``__cinit__`` is called during object -construction, ``self`` is not fully constructed yet, and one must -avoid doing anything with ``self`` but assigning to ``cdef`` fields. +therefore is the right place to initialise static attributes +(``cdef`` fields) of the new instance. However, as ``__cinit__`` is +called during object construction, ``self`` is not fully constructed yet, +and one must avoid doing anything with ``self`` but assigning to static +attributes (``cdef`` fields). Note also that the above method takes no parameters, although subtypes may want to accept some. A no-arguments ``__cinit__()`` method is a @@ -152,7 +171,17 @@ pointer to the new queue. The Python way to get out of this is to raise a ``MemoryError`` [#]_. We can thus change the init function as follows: -.. literalinclude:: ../../examples/tutorial/clibraries/queue2.pyx +.. tabs:: + + .. group-tab:: Pure Python + + .. literalinclude:: ../../examples/tutorial/clibraries/queue2.py + :caption: queue.py + + .. group-tab:: Cython + + .. literalinclude:: ../../examples/tutorial/clibraries/queue2.pyx + :caption: queue.pyx .. [#] In the specific case of a ``MemoryError``, creating a new exception instance in order to raise it may actually fail because @@ -169,31 +198,60 @@ longer used (i.e. all references to it have been deleted). To this end, CPython provides a callback that Cython makes available as a special method ``__dealloc__()``. In our case, all we have to do is to free the C Queue, but only if we succeeded in initialising it in -the init method:: +the init method: + +.. tabs:: + + .. group-tab:: Pure Python + + .. code-block:: python - def __dealloc__(self): - if self._c_queue is not NULL: - cqueue.queue_free(self._c_queue) + def __dealloc__(self): + if self._c_queue is not cython.NULL: + cqueue.queue_free(self._c_queue) + .. group-tab:: Cython + + .. code-block:: cython + + def __dealloc__(self): + if self._c_queue is not NULL: + cqueue.queue_free(self._c_queue) Compiling and linking ===================== At this point, we have a working Cython module that we can test. To -compile it, we need to configure a ``setup.py`` script for distutils. -Here is the most basic script for compiling a Cython module:: +compile it, we need to configure a ``setup.py`` script for setuptools. +Here is the most basic script for compiling a Cython module + +.. tabs:: + + .. group-tab:: Pure Python + + .. code-block:: python + + from setuptools import Extension, setup + from Cython.Build import cythonize + + setup( + ext_modules = cythonize([Extension("queue", ["queue.py"])]) + ) - from distutils.core import setup - from distutils.extension import Extension - from Cython.Build import cythonize + .. group-tab:: Cython - setup( - ext_modules = cythonize([Extension("queue", ["queue.pyx"])]) - ) + .. code-block:: cython + + from setuptools import Extension, setup + from Cython.Build import cythonize + + setup( + ext_modules = cythonize([Extension("queue", ["queue.pyx"])]) + ) To build against the external C library, we need to make sure Cython finds the necessary libraries. -There are two ways to archive this. First we can tell distutils where to find +There are two ways to archive this. First we can tell setuptools where to find the c-source to compile the :file:`queue.c` implementation automatically. Alternatively, we can build and install C-Alg as system library and dynamically link it. The latter is useful if other applications also use C-Alg. @@ -202,33 +260,69 @@ if other applications also use C-Alg. Static Linking --------------- -To build the c-code automatically we need to include compiler directives in `queue.pyx`:: +To build the c-code automatically we need to include compiler directives in :file:`queue.pyx`/:file:`queue.py` + +.. tabs:: + + .. group-tab:: Pure Python + + .. code-block:: python + + # distutils: sources = c-algorithms/src/queue.c + # distutils: include_dirs = c-algorithms/src/ - # distutils: sources = c-algorithms/src/queue.c - # distutils: include_dirs = c-algorithms/src/ + import cython + from cython.cimports import cqueue - cimport cqueue + @cython.cclass + class Queue: + _c_queue = cython.declare(cython.pointer(cqueue.Queue)) - cdef class Queue: - cdef cqueue.Queue* _c_queue - def __cinit__(self): - self._c_queue = cqueue.queue_new() - if self._c_queue is NULL: - raise MemoryError() + def __cinit__(self): + self._c_queue = cqueue.queue_new() + if self._c_queue is cython.NULL: + raise MemoryError() - def __dealloc__(self): - if self._c_queue is not NULL: - cqueue.queue_free(self._c_queue) + def __dealloc__(self): + if self._c_queue is not cython.NULL: + cqueue.queue_free(self._c_queue) + + .. group-tab:: Cython + + .. code-block:: cython + + # distutils: sources = c-algorithms/src/queue.c + # distutils: include_dirs = c-algorithms/src/ + + + cimport cqueue + + + cdef class Queue: + cdef cqueue.Queue* _c_queue + + def __cinit__(self): + self._c_queue = cqueue.queue_new() + if self._c_queue is NULL: + raise MemoryError() + + def __dealloc__(self): + if self._c_queue is not NULL: + cqueue.queue_free(self._c_queue) The ``sources`` compiler directive gives the path of the C -files that distutils is going to compile and +files that setuptools is going to compile and link (statically) into the resulting extension module. In general all relevant header files should be found in ``include_dirs``. -Now we can build the project using:: +Now we can build the project using: + +.. code-block:: bash $ python setup.py build_ext -i -And test whether our build was successful:: +And test whether our build was successful: + +.. code-block:: bash $ python -c 'import queue; Q = queue.Queue()' @@ -240,14 +334,18 @@ Dynamic linking is useful, if the library we are going to wrap is already installed on the system. To perform dynamic linking we first need to build and install c-alg. -To build c-algorithms on your system:: +To build c-algorithms on your system: + +.. code-block:: bash $ cd c-algorithms $ sh autogen.sh $ ./configure $ make -to install CAlg run:: +to install CAlg run: + +.. code-block:: bash $ make install @@ -262,26 +360,53 @@ Afterwards the file :file:`/usr/local/lib/libcalg.so` should exist. In this approach we need to tell the setup script to link with an external library. To do so we need to extend the setup script to install change the extension setup from -:: +.. tabs:: - ext_modules = cythonize([Extension("queue", ["queue.pyx"])]) + .. group-tab:: Pure Python + + .. code-block:: python + + ext_modules = cythonize([Extension("queue", ["queue.py"])]) + + .. group-tab:: Cython + + .. code-block:: cython + + ext_modules = cythonize([Extension("queue", ["queue.pyx"])]) to -:: +.. tabs:: + + .. group-tab:: Pure Python - ext_modules = cythonize([ - Extension("queue", ["queue.pyx"], - libraries=["calg"]) - ]) + .. code-block:: python -Now we should be able to build the project using:: + ext_modules = cythonize([ + Extension("queue", ["queue.py"], + libraries=["calg"]) + ]) + + .. group-tab:: Cython + + .. code-block:: cython + + ext_modules = cythonize([ + Extension("queue", ["queue.pyx"], + libraries=["calg"]) + ]) + +Now we should be able to build the project using: + +.. code-block:: bash $ python setup.py build_ext -i If the `libcalg` is not installed in a 'normal' location, users can provide the required parameters externally by passing appropriate C compiler -flags, such as:: +flags, such as: + +.. code-block:: bash CFLAGS="-I/usr/local/otherdir/calg/include" \ LDFLAGS="-L/usr/local/otherdir/calg/lib" \ @@ -290,12 +415,16 @@ flags, such as:: Before we run the module, we also need to make sure that `libcalg` is in -the `LD_LIBRARY_PATH` environment variable, e.g. by setting:: +the `LD_LIBRARY_PATH` environment variable, e.g. by setting: + +.. code-block:: bash $ export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib Once we have compiled the module for the first time, we can now import -it and instantiate a new Queue:: +it and instantiate a new Queue: + +.. code-block:: bash $ export PYTHONPATH=. $ python -c 'import queue; Q = queue.Queue()' @@ -313,7 +442,7 @@ practice to look at what interfaces Python offers, e.g. in its queue, it's enough to provide the methods ``append()``, ``peek()`` and ``pop()``, and additionally an ``extend()`` method to add multiple values at once. Also, since we already know that all values will be -coming from C, it's best to provide only ``cdef`` methods for now, and +coming from C, it's best to provide only ``cdef``/``@cfunc`` methods for now, and to give them a straight C interface. In C, it is common for data structures to store data as a ``void*`` to @@ -323,28 +452,76 @@ additional memory allocations through a trick: we cast our ``int`` values to ``void*`` and vice versa, and store the value directly as the pointer value. -Here is a simple implementation for the ``append()`` method:: +Here is a simple implementation for the ``append()`` method: + +.. tabs:: + + .. group-tab:: Pure Python + + .. code-block:: python + + @cython.cfunc + def append(self, value: cython.int): + cqueue.queue_push_tail(self._c_queue, cython.cast(cython.p_void, value)) + + .. group-tab:: Cython - cdef append(self, int value): - cqueue.queue_push_tail(self._c_queue, <void*>value) + .. code-block:: cython + + cdef append(self, int value): + cqueue.queue_push_tail(self._c_queue, <void*>value) Again, the same error handling considerations as for the ``__cinit__()`` method apply, so that we end up with this -implementation instead:: +implementation instead: + +.. tabs:: + + .. group-tab:: Pure Python + + .. code-block:: python + + @cython.cfunc + def append(self, value: cython.int): + if not cqueue.queue_push_tail(self._c_queue, + cython.cast(cython.p_void, value)): + raise MemoryError() + + .. group-tab:: Cython - cdef append(self, int value): - if not cqueue.queue_push_tail(self._c_queue, - <void*>value): - raise MemoryError() + .. code-block:: cython -Adding an ``extend()`` method should now be straight forward:: + cdef append(self, int value): + if not cqueue.queue_push_tail(self._c_queue, + <void*>value): + raise MemoryError() - cdef extend(self, int* values, size_t count): - """Append all ints to the queue. - """ - cdef int value - for value in values[:count]: # Slicing pointer to limit the iteration boundaries. - self.append(value) +Adding an ``extend()`` method should now be straight forward: + +.. tabs:: + + .. group-tab:: Pure Python + + .. code-block:: python + + @cython.cfunc + def extend(self, values: cython.p_int, count: cython.size_t): + """Append all ints to the queue. + """ + value: cython.int + for value in values[:count]: # Slicing pointer to limit the iteration boundaries. + self.append(value) + + .. group-tab:: Cython + + .. code-block:: cython + + cdef extend(self, int* values, size_t count): + """Append all ints to the queue. + """ + cdef int value + for value in values[:count]: # Slicing pointer to limit the iteration boundaries. + self.append(value) This becomes handy when reading values from a C array, for example. @@ -353,13 +530,31 @@ the two methods to get the first element: ``peek()`` and ``pop()``, which provide read-only and destructive read access respectively. To avoid compiler warnings when casting ``void*`` to ``int`` directly, we use an intermediate data type that is big enough to hold a ``void*``. -Here, ``Py_ssize_t``:: +Here, ``Py_ssize_t``: + +.. tabs:: + + .. group-tab:: Pure Python + + .. code-block:: python + + @cython.cfunc + def peek(self) -> cython.int: + return cython.cast(cython.Py_ssize_t, cqueue.queue_peek_head(self._c_queue)) + + @cython.cfunc + def pop(self) -> cython.int: + return cython.cast(cython.Py_ssize_t, cqueue.queue_pop_head(self._c_queue)) - cdef int peek(self): - return <Py_ssize_t>cqueue.queue_peek_head(self._c_queue) + .. group-tab:: Cython - cdef int pop(self): - return <Py_ssize_t>cqueue.queue_pop_head(self._c_queue) + .. code-block:: cython + + cdef int peek(self): + return <Py_ssize_t>cqueue.queue_peek_head(self._c_queue) + + cdef int pop(self): + return <Py_ssize_t>cqueue.queue_pop_head(self._c_queue) Normally, in C, we risk losing data when we convert a larger integer type to a smaller integer type without checking the boundaries, and ``Py_ssize_t`` @@ -380,22 +575,44 @@ from ints, we cannot distinguish anymore if the return value was the queue was ``0``. In Cython code, we want the first case to raise an exception, whereas the second case should simply return ``0``. To deal with this, we need to special case this value, -and check if the queue really is empty or not:: +and check if the queue really is empty or not: + +.. tabs:: + + .. group-tab:: Pure Python + + .. code-block:: python + + @cython.cfunc + @cython.exceptval(-1, check=True) + def int peek(self): + value: cython.int = cython.cast(cython.Py_ssize_t, cqueue.queue_peek_head(self._c_queue)) + if value == 0: + # this may mean that the queue is empty, or + # that it happens to contain a 0 value + if cqueue.queue_is_empty(self._c_queue): + raise IndexError("Queue is empty") + return value + + .. group-tab:: Cython - cdef int peek(self) except? -1: - cdef int value = <Py_ssize_t>cqueue.queue_peek_head(self._c_queue) - if value == 0: - # this may mean that the queue is empty, or - # that it happens to contain a 0 value - if cqueue.queue_is_empty(self._c_queue): - raise IndexError("Queue is empty") - return value + .. code-block:: cython + + cdef int peek(self) except? -1: + cdef int value = <Py_ssize_t>cqueue.queue_peek_head(self._c_queue) + if value == 0: + # this may mean that the queue is empty, or + # that it happens to contain a 0 value + if cqueue.queue_is_empty(self._c_queue): + raise IndexError("Queue is empty") + return value Note how we have effectively created a fast path through the method in the hopefully common cases that the return value is not ``0``. Only that specific case needs an additional check if the queue is empty. -The ``except? -1`` declaration in the method signature falls into the +The ``except? -1`` or ``@cython.exceptval(-1, check=True)`` declaration +in the method signature falls into the same category. If the function was a Python function returning a Python object value, CPython would simply return ``NULL`` internally instead of a Python object to indicate an exception, which would @@ -417,7 +634,8 @@ exception when receiving this exact value. We chose to use ``-1`` as the exception return value as we expect it to be an unlikely value to be put into the queue. The question mark -in the ``except? -1`` declaration indicates that the return value is +in the ``except? -1`` declaration and ``check=True`` in ``@cython.exceptval`` +indicates that the return value is ambiguous (there *may* be a ``-1`` value in the queue, after all) and that an additional exception check using ``PyErr_Occurred()`` is needed in calling code. Without it, Cython code that calls this @@ -430,12 +648,29 @@ values. Now that the ``peek()`` method is implemented, the ``pop()`` method also needs adaptation. Since it removes a value from the queue, however, it is not enough to test if the queue is empty *after* the -removal. Instead, we must test it on entry:: +removal. Instead, we must test it on entry: + +.. tabs:: + + .. group-tab:: Pure Python + + .. code-block:: python + + @cython.cfunc + @cython.exceptval(-1, check=True) + def pop(self) -> cython.int: + if cqueue.queue_is_empty(self._c_queue): + raise IndexError("Queue is empty") + return cython.cast(cython.Py_ssize_t, cqueue.queue_pop_head(self._c_queue)) + + .. group-tab:: Cython - cdef int pop(self) except? -1: - if cqueue.queue_is_empty(self._c_queue): - raise IndexError("Queue is empty") - return <Py_ssize_t>cqueue.queue_pop_head(self._c_queue) + .. code-block:: cython + + cdef int pop(self) except? -1: + if cqueue.queue_is_empty(self._c_queue): + raise IndexError("Queue is empty") + return <Py_ssize_t>cqueue.queue_pop_head(self._c_queue) The return value for exception propagation is declared exactly as for ``peek()``. @@ -450,7 +685,7 @@ code can use either name):: Note that this method returns either ``True`` or ``False`` as we declared the return type of the ``queue_is_empty()`` function as -``bint`` in ``cqueue.pxd``. +``bint`` in :file:`cqueue.pxd`. Testing the result @@ -464,14 +699,14 @@ you can call. C methods are not visible from Python code, and thus not callable from doctests. A quick way to provide a Python API for the class is to change the -methods from ``cdef`` to ``cpdef``. This will let Cython generate two -entry points, one that is callable from normal Python code using the -Python call semantics and Python objects as arguments, and one that is -callable from C code with fast C semantics and without requiring -intermediate argument conversion from or to Python types. Note that ``cpdef`` -methods ensure that they can be appropriately overridden by Python -methods even when they are called from Cython. This adds a tiny overhead -compared to ``cdef`` methods. +methods from ``cdef``/``@cfunc`` to ``cpdef``/``@ccall``. This will +let Cython generate two entry points, one that is callable from normal +Python code using the Python call semantics and Python objects as arguments, +and one that is callable from C code with fast C semantics and without requiring +intermediate argument conversion from or to Python types. Note that +``cpdef``/``@ccall`` methods ensure that they can be appropriately overridden +by Python methods even when they are called from Cython. This adds a tiny overhead +compared to ``cdef``/``@cfunc`` methods. Now that we have both a C-interface and a Python interface for our class, we should make sure that both interfaces are consistent. @@ -482,14 +717,24 @@ C arrays and C memory. Both signatures are incompatible. We will solve this issue by considering that in C, the API could also want to support other input types, e.g. arrays of ``long`` or ``char``, which is usually supported with differently named C API functions such as -``extend_ints()``, ``extend_longs()``, extend_chars()``, etc. This allows +``extend_ints()``, ``extend_longs()``, ``extend_chars()``, etc. This allows us to free the method name ``extend()`` for the duck typed Python method, which can accept arbitrary iterables. The following listing shows the complete implementation that uses -``cpdef`` methods where possible: +``cpdef``/``@ccall`` methods where possible: + +.. tabs:: + + .. group-tab:: Pure Python -.. literalinclude:: ../../examples/tutorial/clibraries/queue3.pyx + .. literalinclude:: ../../examples/tutorial/clibraries/queue3.py + :caption: queue.py + + .. group-tab:: Cython + + .. literalinclude:: ../../examples/tutorial/clibraries/queue3.pyx + :caption: queue.pyx Now we can test our Queue implementation using a python script, for example here :file:`test_queue.py`: @@ -540,29 +785,73 @@ C-API into the callback function. We will use this to pass our Python predicate function. First, we have to define a callback function with the expected -signature that we can pass into the C-API function:: - - cdef int evaluate_predicate(void* context, cqueue.QueueValue value): - "Callback function that can be passed as predicate_func" - try: - # recover Python function object from void* argument - func = <object>context - # call function, convert result into 0/1 for True/False - return bool(func(<int>value)) - except: - # catch any Python errors and return error indicator - return -1 +signature that we can pass into the C-API function: + +.. tabs:: + + .. group-tab:: Pure Python + + .. code-block:: python + + @cython.cfunc + @cython.exceptval(check=False) + def evaluate_predicate(context: cython.p_void, value: cqueue.QueueValue) -> cython.int: + "Callback function that can be passed as predicate_func" + try: + # recover Python function object from void* argument + func = cython.cast(object, context) + # call function, convert result into 0/1 for True/False + return bool(func(cython.cast(int, value))) + except: + # catch any Python errors and return error indicator + return -1 + + .. note:: ``@cfunc`` functions in pure python are defined as ``@exceptval(-1, check=True)`` + by default. Since ``evaluate_predicate()`` should be passed to function as parameter, + we need to turn off exception checking entirely. + + .. group-tab:: Cython + + .. code-block:: cython + + cdef int evaluate_predicate(void* context, cqueue.QueueValue value): + "Callback function that can be passed as predicate_func" + try: + # recover Python function object from void* argument + func = <object>context + # call function, convert result into 0/1 for True/False + return bool(func(<int>value)) + except: + # catch any Python errors and return error indicator + return -1 The main idea is to pass a pointer (a.k.a. borrowed reference) to the function object as the user context argument. We will call the C-API -function as follows:: - - def pop_until(self, python_predicate_function): - result = cqueue.queue_pop_head_until( - self._c_queue, evaluate_predicate, - <void*>python_predicate_function) - if result == -1: - raise RuntimeError("an error occurred") +function as follows: + +.. tabs:: + + .. group-tab:: Pure Python + + .. code-block:: python + + def pop_until(self, python_predicate_function): + result = cqueue.queue_pop_head_until( + self._c_queue, evaluate_predicate, + cython.cast(cython.p_void, python_predicate_function)) + if result == -1: + raise RuntimeError("an error occurred") + + .. group-tab:: Cython + + .. code-block:: cython + + def pop_until(self, python_predicate_function): + result = cqueue.queue_pop_head_until( + self._c_queue, evaluate_predicate, + <void*>python_predicate_function) + if result == -1: + raise RuntimeError("an error occurred") The usual pattern is to first cast the Python object reference into a :c:type:`void*` to pass it into the C-API function, and then cast diff --git a/docs/src/tutorial/cython_tutorial.rst b/docs/src/tutorial/cython_tutorial.rst index f80a8b016..647ec62b2 100644 --- a/docs/src/tutorial/cython_tutorial.rst +++ b/docs/src/tutorial/cython_tutorial.rst @@ -6,6 +6,9 @@ Basic Tutorial ************** +.. include:: + ../two-syntax-variants-used + The Basics of Cython ==================== @@ -18,7 +21,7 @@ serve for now.) The Cython compiler will convert it into C code which makes equivalent calls to the Python/C API. But Cython is much more than that, because parameters and variables can be -declared to have C data types. Code which manipulates Python values and C +declared to have C data types. Code which manipulates :term:`Python values<Python object>` and C values can be freely intermixed, with conversions occurring automatically wherever possible. Reference count maintenance and error checking of Python operations is also automatic, and the full power of Python's exception @@ -40,7 +43,7 @@ Save this code in a file named :file:`helloworld.pyx`. Now we need to create the :file:`setup.py`, which is like a python Makefile (for more information see :ref:`compilation`). Your :file:`setup.py` should look like:: - from distutils.core import setup + from setuptools import setup from Cython.Build import cythonize setup( @@ -49,7 +52,7 @@ see :ref:`compilation`). Your :file:`setup.py` should look like:: To use this to build your Cython file use the commandline options: -.. sourcecode:: text +.. code-block:: text $ python setup.py build_ext --inplace @@ -103,13 +106,18 @@ Now following the steps for the Hello World example we first rename the file to have a `.pyx` extension, lets say :file:`fib.pyx`, then we create the :file:`setup.py` file. Using the file created for the Hello World example, all that you need to change is the name of the Cython filename, and the resulting -module name, doing this we have: +module name, doing this we have:: + + from setuptools import setup + from Cython.Build import cythonize -.. literalinclude:: ../../examples/tutorial/cython_tutorial/setup.py + setup( + ext_modules=cythonize("fib.pyx"), + ) Build the extension with the same command used for the helloworld.pyx: -.. sourcecode:: text +.. code-block:: text $ python setup.py build_ext --inplace @@ -127,29 +135,59 @@ Primes Here's a small example showing some of what can be done. It's a routine for finding prime numbers. You tell it how many primes you want, and it returns them as a Python list. + +.. tabs:: + .. group-tab:: Pure Python -:file:`primes.pyx`: + .. literalinclude:: ../../examples/tutorial/cython_tutorial/primes.py + :linenos: + :caption: primes.py -.. literalinclude:: ../../examples/tutorial/cython_tutorial/primes.pyx - :linenos: + .. group-tab:: Cython + + .. literalinclude:: ../../examples/tutorial/cython_tutorial/primes.pyx + :linenos: + :caption: primes.pyx You'll see that it starts out just like a normal Python function definition, -except that the parameter ``nb_primes`` is declared to be of type ``int`` . This +except that the parameter ``nb_primes`` is declared to be of type ``int``. This means that the object passed will be converted to a C integer (or a ``TypeError.`` will be raised if it can't be). -Now, let's dig into the core of the function:: +Now, let's dig into the core of the function: + +.. tabs:: + .. group-tab:: Pure Python + + .. literalinclude:: ../../examples/tutorial/cython_tutorial/primes.py + :lines: 2,3 + :dedent: + :lineno-start: 2 - cdef int n, i, len_p - cdef int p[1000] + .. literalinclude:: ../../examples/tutorial/cython_tutorial/primes.py + :lines: 11,12 + :dedent: + :lineno-start: 11 + + Lines 2, 3, 11 and 12 use the variable annotations + to define some local C variables. + The result is stored in the C array ``p`` during processing, + and will be copied into a Python list at the end (line 26). -Lines 2 and 3 use the ``cdef`` statement to define some local C variables. -The result is stored in the C array ``p`` during processing, -and will be copied into a Python list at the end (line 22). + .. group-tab:: Cython + + .. literalinclude:: ../../examples/tutorial/cython_tutorial/primes.pyx + :lines: 2,3 + :dedent: + :lineno-start: 2 + + Lines 2 and 3 use the ``cdef`` statement to define some local C variables. + The result is stored in the C array ``p`` during processing, + and will be copied into a Python list at the end (line 26). .. NOTE:: You cannot create very large arrays in this manner, because - they are allocated on the C function call stack, which is a - rather precious and scarce resource. + they are allocated on the C function call :term:`stack<Stack allocation>`, + which is a rather precious and scarce resource. To request larger arrays, or even arrays with a length only known at runtime, you can learn how to make efficient use of @@ -157,61 +195,83 @@ and will be copied into a Python list at the end (line 22). :ref:`Python arrays <array-array>` or :ref:`NumPy arrays <memoryviews>` with Cython. -:: - - if nb_primes > 1000: - nb_primes = 1000 +.. literalinclude:: ../../examples/tutorial/cython_tutorial/primes.pyx + :lines: 5,6 + :dedent: + :lineno-start: 5 As in C, declaring a static array requires knowing the size at compile time. We make sure the user doesn't set a value above 1000 (or we would have a -segmentation fault, just like in C). :: +segmentation fault, just like in C) + +.. tabs:: + .. group-tab:: Pure Python + + .. literalinclude:: ../../examples/tutorial/cython_tutorial/primes.py + :lines: 8,9 + :dedent: + :lineno-start: 8 - len_p = 0 # The number of elements in p - n = 2 - while len_p < nb_primes: + When we run this code from Python, we have to initialize the items in the array. + This is most easily done by filling it with zeros (as seen on line 8-9). + When we compile this with Cython, on the other hand, the array will + behave as in C. It is allocated on the function call stack with a fixed + length of 1000 items that contain arbitrary data from the last time that + memory was used. We will then overwrite those items in our calculation. -Lines 7-9 set up for a loop which will test candidate numbers for primeness -until the required number of primes has been found. :: + .. literalinclude:: ../../examples/tutorial/cython_tutorial/primes.py + :lines: 10-13 + :dedent: + :lineno-start: 10 - # Is n prime? - for i in p[:len_p]: - if n % i == 0: - break + .. group-tab:: Cython -Lines 11-12, which try dividing a candidate by all the primes found so far, + .. literalinclude:: ../../examples/tutorial/cython_tutorial/primes.pyx + :lines: 10-13 + :dedent: + :lineno-start: 10 + +Lines 11-13 set up a while loop which will test numbers-candidates to primes +until the required number of primes has been found. + +.. literalinclude:: ../../examples/tutorial/cython_tutorial/primes.pyx + :lines: 14-17 + :dedent: + :lineno-start: 14 + +Lines 15-16, which try to divide a candidate by all the primes found so far, are of particular interest. Because no Python objects are referred to, the loop is translated entirely into C code, and thus runs very fast. -You will notice the way we iterate over the ``p`` C array. :: +You will notice the way we iterate over the ``p`` C array. - for i in p[:len_p]: +.. literalinclude:: ../../examples/tutorial/cython_tutorial/primes.pyx + :lines: 15 + :dedent: + :lineno-start: 15 The loop gets translated into a fast C loop and works just like iterating over a Python list or NumPy array. If you don't slice the C array with ``[:len_p]``, then Cython will loop over the 1000 elements of the array. -:: - - # If no break occurred in the loop - else: - p[len_p] = n - len_p += 1 - n += 1 +.. literalinclude:: ../../examples/tutorial/cython_tutorial/primes.pyx + :lines: 19-23 + :dedent: + :lineno-start: 19 If no breaks occurred, it means that we found a prime, and the block of code -after the ``else`` line 16 will be executed. We add the prime found to ``p``. +after the ``else`` line 20 will be executed. We add the prime found to ``p``. If you find having an ``else`` after a for-loop strange, just know that it's a lesser known features of the Python language, and that Cython executes it at C speed for you. If the for-else syntax confuses you, see this excellent `blog post <https://shahriar.svbtle.com/pythons-else-clause-in-loops>`_. -:: - - # Let's put the result in a python list: - result_as_list = [prime for prime in p[:len_p]] - return result_as_list +.. literalinclude:: ../../examples/tutorial/cython_tutorial/primes.pyx + :lines: 25-27 + :dedent: + :lineno-start: 25 -In line 22, before returning the result, we need to copy our C array into a +In line 26, before returning the result, we need to copy our C array into a Python list, because Python can't read C arrays. Cython can automatically convert many C types from and to Python types, as described in the documentation on :ref:`type conversion <type-conversion>`, so we can use @@ -225,11 +285,20 @@ Because the variable ``result_as_list`` hasn't been explicitly declared with a t it is assumed to hold a Python object, and from the assignment, Cython also knows that the exact type is a Python list. -Finally, at line 18, a normal -Python return statement returns the result list. +Finally, at line 27, a normal Python return statement returns the result list. + +.. tabs:: + .. group-tab:: Pure Python + + Compiling primes.py with the Cython compiler produces an extension module + which we can try out in the interactive interpreter as follows: -Compiling primes.pyx with the Cython compiler produces an extension module -which we can try out in the interactive interpreter as follows:: + .. group-tab:: Cython + + Compiling primes.pyx with the Cython compiler produces an extension module + which we can try out in the interactive interpreter as follows: + +.. code-block:: python >>> import primes >>> primes.primes(10) @@ -238,12 +307,20 @@ which we can try out in the interactive interpreter as follows:: See, it works! And if you're curious about how much work Cython has saved you, take a look at the C code generated for this module. - Cython has a way to visualise where interaction with Python objects and Python's C-API is taking place. For this, pass the ``annotate=True`` parameter to ``cythonize()``. It produces a HTML file. Let's see: -.. figure:: htmlreport.png +.. tabs:: + .. group-tab:: Pure Python + + .. figure:: htmlreport_py.png + :scale: 90 % + + .. group-tab:: Cython + + .. figure:: htmlreport_pyx.png + :scale: 90 % If a line is white, it means that the code generated doesn't interact with Python, so will run as fast as normal C code. The darker the yellow, the more @@ -262,42 +339,64 @@ Python behavior, the language will perform division checks at runtime, just like Python does. You can deactivate those checks by using the :ref:`compiler directives<compiler-directives>`. -Now let's see if, even if we have division checks, we obtained a boost in speed. -Let's write the same program, but Python-style: +Now let's see if we get a speed increase even if there is a division check. +Let's write the same program, but in Python: .. literalinclude:: ../../examples/tutorial/cython_tutorial/primes_python.py + :caption: primes_python.py / primes_python_compiled.py -It is also possible to take a plain ``.py`` file and to compile it with Cython. -Let's take ``primes_python``, change the function name to ``primes_python_compiled`` and -compile it with Cython (without changing the code). We will also change the name of the -file to ``example_py_cy.py`` to differentiate it from the others. -Now the ``setup.py`` looks like this:: +It is possible to take a plain (unannotated) ``.py`` file and to compile it with Cython. +Let's create a copy of ``primes_python`` and name it ``primes_python_compiled`` +to be able to compare it to the (non-compiled) Python module. +Then we compile that file with Cython, without changing the code. +Now the ``setup.py`` looks like this: - from distutils.core import setup - from Cython.Build import cythonize +.. tabs:: + .. group-tab:: Pure Python - setup( - ext_modules=cythonize(['example.pyx', # Cython code file with primes() function - 'example_py_cy.py'], # Python code file with primes_python_compiled() function - annotate=True), # enables generation of the html annotation file - ) + .. code-block:: python + + from setuptools import setup + from Cython.Build import cythonize + + setup( + ext_modules=cythonize( + ['primes.py', # Cython code file with primes() function + 'primes_python_compiled.py'], # Python code file with primes() function + annotate=True), # enables generation of the html annotation file + ) + + .. group-tab:: Cython + + .. code-block:: python + + from setuptools import setup + from Cython.Build import cythonize + + setup( + ext_modules=cythonize( + ['primes.pyx', # Cython code file with primes() function + 'primes_python_compiled.py'], # Python code file with primes() function + annotate=True), # enables generation of the html annotation file + ) Now we can ensure that those two programs output the same values:: - >>> primes_python(1000) == primes(1000) + >>> import primes, primes_python, primes_python_compiled + >>> primes_python.primes(1000) == primes.primes(1000) True - >>> primes_python_compiled(1000) == primes(1000) + >>> primes_python_compiled.primes(1000) == primes.primes(1000) True It's possible to compare the speed now:: - python -m timeit -s 'from example_py import primes_python' 'primes_python(1000)' + python -m timeit -s 'from primes_python import primes' 'primes(1000)' 10 loops, best of 3: 23 msec per loop - python -m timeit -s 'from example_py_cy import primes_python_compiled' 'primes_python_compiled(1000)' + python -m timeit -s 'from primes_python_compiled import primes' 'primes(1000)' 100 loops, best of 3: 11.9 msec per loop - python -m timeit -s 'from example import primes' 'primes(1000)' + python -m timeit -s 'from primes import primes' 'primes(1000)' 1000 loops, best of 3: 1.65 msec per loop The cythonize version of ``primes_python`` is 2 times faster than the Python one, @@ -325,9 +424,9 @@ Primes with C++ With Cython, it is also possible to take advantage of the C++ language, notably, part of the C++ standard library is directly importable from Cython code. -Let's see what our :file:`primes.pyx` becomes when -using `vector <https://en.cppreference.com/w/cpp/container/vector>`_ from the C++ -standard library. +Let's see what our code becomes when using +`vector <https://en.cppreference.com/w/cpp/container/vector>`_ +from the C++ standard library. .. note:: @@ -338,8 +437,19 @@ standard library. how many elements you are going to put in the vector. For more details see `this page from cppreference <https://en.cppreference.com/w/cpp/container/vector>`_. -.. literalinclude:: ../../examples/tutorial/cython_tutorial/primes_cpp.pyx - :linenos: +.. tabs:: + .. group-tab:: Pure Python + + .. literalinclude:: ../../examples/tutorial/cython_tutorial/primes_cpp.py + :linenos: + + .. include:: + ../cimport-warning + + .. group-tab:: Cython + + .. literalinclude:: ../../examples/tutorial/cython_tutorial/primes_cpp.pyx + :linenos: The first line is a compiler directive. It tells Cython to compile your code to C++. This will enable the use of C++ language features and the C++ standard library. @@ -357,4 +467,3 @@ Language Details For more about the Cython language, see :ref:`language-basics`. To dive right in to using Cython in a numerical computation context, see :ref:`memoryviews`. - diff --git a/docs/src/tutorial/embedding.rst b/docs/src/tutorial/embedding.rst new file mode 100644 index 000000000..3f6325428 --- /dev/null +++ b/docs/src/tutorial/embedding.rst @@ -0,0 +1,77 @@ +.. highlight:: cython + +.. _embedding: + +********************************************** +Embedding Cython modules in C/C++ applications +********************************************** + +**This is a stub documentation page. PRs very welcome.** + +Quick links: + +* `CPython docs <https://docs.python.org/3/extending/embedding.html>`_ + +* `Cython Wiki <https://github.com/cython/cython/wiki/EmbeddingCython>`_ + +* See the ``--embed`` option to the ``cython`` and ``cythonize`` frontends + for generating a C main function and the + `cython_freeze <https://github.com/cython/cython/blob/master/bin/cython_freeze>`_ + script for merging multiple extension modules into one library. + +* `Embedding demo program <https://github.com/cython/cython/tree/master/Demos/embed>`_ + +* See the documentation of the `module init function + <https://docs.python.org/3/extending/extending.html#the-module-s-method-table-and-initialization-function>`_ + in CPython and `PEP 489 <https://www.python.org/dev/peps/pep-0489/>`_ regarding the module + initialisation mechanism in CPython 3.5 and later. + + +Initialising your main module +============================= + +Most importantly, DO NOT call the module init function instead of importing +the module. This is not the right way to initialise an extension module. +(It was always wrong but used to work before, but since Python 3.5, it is +wrong *and* no longer works.) + +For details, see the documentation of the +`module init function <https://docs.python.org/3/extending/extending.html#the-module-s-method-table-and-initialization-function>`_ +in CPython and `PEP 489 <https://www.python.org/dev/peps/pep-0489/>`_ regarding the module +initialisation mechanism in CPython 3.5 and later. + +The `PyImport_AppendInittab() <https://docs.python.org/3/c-api/import.html#c.PyImport_AppendInittab>`_ +function in CPython allows registering statically (or dynamically) linked extension +modules for later imports. An example is given in the documentation of the module +init function that is linked above. + + +Embedding example code +====================== + +The following is a simple example that shows the main steps for embedding a +Cython module (``embedded.pyx``) in Python 3.x. + +First, here is a Cython module that exports a C function to be called by external +code. Note that the ``say_hello_from_python()`` function is declared as ``public`` +to export it as a linker symbol that can be used by other C files, which in this +case is ``embedded_main.c``. + +.. literalinclude:: ../../examples/tutorial/embedding/embedded.pyx + +The C ``main()`` function of your program could look like this: + +.. literalinclude:: ../../examples/tutorial/embedding/embedded_main.c + :linenos: + :language: c + +(Adapted from the `CPython documentation +<https://docs.python.org/3/extending/extending.html#the-module-s-method-table-and-initialization-function>`_.) + +Instead of writing such a ``main()`` function yourself, you can also let +Cython generate one into your module's C file with the ``cython --embed`` +option. Or use the +`cython_freeze <https://github.com/cython/cython/blob/master/bin/cython_freeze>`_ +script to embed multiple modules. See the +`embedding demo program <https://github.com/cython/cython/tree/master/Demos/embed>`_ +for a complete example setup. diff --git a/docs/src/tutorial/external.rst b/docs/src/tutorial/external.rst index b55b96505..d0c5af0a0 100644 --- a/docs/src/tutorial/external.rst +++ b/docs/src/tutorial/external.rst @@ -1,6 +1,9 @@ Calling C functions ==================== +.. include:: + ../two-syntax-variants-used + This tutorial describes shortly what you need to know in order to call C library functions from Cython code. For a longer and more comprehensive tutorial about using external C libraries, wrapping them @@ -15,7 +18,17 @@ For example, let's say you need a low-level way to parse a number from a ``char*`` value. You could use the ``atoi()`` function, as defined by the ``stdlib.h`` header file. This can be done as follows: -.. literalinclude:: ../../examples/tutorial/external/atoi.pyx +.. tabs:: + + .. group-tab:: Pure Python + + .. literalinclude:: ../../examples/tutorial/external/atoi.py + :caption: atoi.py + + .. group-tab:: Cython + + .. literalinclude:: ../../examples/tutorial/external/atoi.pyx + :caption: atoi.pyx You can find a complete list of these standard cimport files in Cython's source package @@ -28,12 +41,33 @@ Cython also has a complete set of declarations for CPython's C-API. For example, to test at C compilation time which CPython version your code is being compiled with, you can do this: -.. literalinclude:: ../../examples/tutorial/external/py_version_hex.pyx +.. tabs:: + + .. group-tab:: Pure Python + + .. literalinclude:: ../../examples/tutorial/external/py_version_hex.py + :caption: py_version_hex.py + + .. group-tab:: Cython + + .. literalinclude:: ../../examples/tutorial/external/py_version_hex.pyx + :caption: py_version_hex.pyx + +.. _libc.math: Cython also provides declarations for the C math library: -.. literalinclude:: ../../examples/tutorial/external/libc_sin.pyx +.. tabs:: + + .. group-tab:: Pure Python + + .. literalinclude:: ../../examples/tutorial/external/libc_sin.py + :caption: libc_sin.py + .. group-tab:: Cython + + .. literalinclude:: ../../examples/tutorial/external/libc_sin.pyx + :caption: libc_sin.pyx Dynamic linking --------------- @@ -41,7 +75,7 @@ Dynamic linking The libc math library is special in that it is not linked by default on some Unix-like systems, such as Linux. In addition to cimporting the declarations, you must configure your build system to link against the -shared library ``m``. For distutils, it is enough to add it to the +shared library ``m``. For setuptools, it is enough to add it to the ``libraries`` parameter of the ``Extension()`` setup: .. literalinclude:: ../../examples/tutorial/external/setup.py @@ -81,6 +115,9 @@ This allows the C declaration to be reused in other Cython modules, while still providing an automatically generated Python wrapper in this specific module. +.. note:: External declarations must be placed in a ``.pxd`` file in Pure + Python mode. + Naming parameters ----------------- @@ -101,7 +138,19 @@ You can now make it clear which of the two arguments does what in your call, thus avoiding any ambiguities and often making your code more readable: -.. literalinclude:: ../../examples/tutorial/external/keyword_args_call.pyx +.. tabs:: + + .. group-tab:: Pure Python + + .. literalinclude:: ../../examples/tutorial/external/keyword_args_call.py + :caption: keyword_args_call.py + .. literalinclude:: ../../examples/tutorial/external/strstr.pxd + :caption: strstr.pxd + + .. group-tab:: Cython + + .. literalinclude:: ../../examples/tutorial/external/keyword_args_call.pyx + :caption: keyword_args_call.pyx Note that changing existing parameter names later is a backwards incompatible API modification, just as for Python code. Thus, if diff --git a/docs/src/tutorial/htmlreport.png b/docs/src/tutorial/htmlreport.png Binary files differdeleted file mode 100644 index 4fc98f3e0..000000000 --- a/docs/src/tutorial/htmlreport.png +++ /dev/null diff --git a/docs/src/tutorial/htmlreport_py.png b/docs/src/tutorial/htmlreport_py.png Binary files differnew file mode 100644 index 000000000..86da8fa30 --- /dev/null +++ b/docs/src/tutorial/htmlreport_py.png diff --git a/docs/src/tutorial/htmlreport_pyx.png b/docs/src/tutorial/htmlreport_pyx.png Binary files differnew file mode 100644 index 000000000..5a9cb89ca --- /dev/null +++ b/docs/src/tutorial/htmlreport_pyx.png diff --git a/docs/src/tutorial/index.rst b/docs/src/tutorial/index.rst index 14bc5d9ee..67ff4ff1c 100644 --- a/docs/src/tutorial/index.rst +++ b/docs/src/tutorial/index.rst @@ -13,10 +13,10 @@ Tutorials profiling_tutorial strings memory_allocation + embedding pure numpy array readings related_work appendix - diff --git a/docs/src/tutorial/memory_allocation.rst b/docs/src/tutorial/memory_allocation.rst index f53c1119a..fa79310de 100644 --- a/docs/src/tutorial/memory_allocation.rst +++ b/docs/src/tutorial/memory_allocation.rst @@ -4,6 +4,9 @@ Memory Allocation ***************** +.. include:: + ../two-syntax-variants-used + Dynamic memory allocation is mostly a non-issue in Python. Everything is an object, and the reference counting system and garbage collector automatically return memory to the system when it is no longer being used. @@ -19,10 +22,10 @@ In some situations, however, these objects can still incur an unacceptable amount of overhead, which can then makes a case for doing manual memory management in C. -Simple C values and structs (such as a local variable ``cdef double x``) are -usually allocated on the stack and passed by value, but for larger and more +Simple C values and structs (such as a local variable ``cdef double x`` / ``x: cython.double``) are +usually :term:`allocated on the stack<Stack allocation>` and passed by value, but for larger and more complicated objects (e.g. a dynamically-sized list of doubles), the memory must -be manually requested and released. C provides the functions :c:func:`malloc`, +be :term:`manually requested and released<Heap allocation>`. C provides the functions :c:func:`malloc`, :c:func:`realloc`, and :c:func:`free` for this purpose, which can be imported in cython from ``clibc.stdlib``. Their signatures are: @@ -34,8 +37,15 @@ in cython from ``clibc.stdlib``. Their signatures are: A very simple example of malloc usage is the following: -.. literalinclude:: ../../examples/tutorial/memory_allocation/malloc.pyx - :linenos: + +.. tabs:: + .. group-tab:: Pure Python + + .. literalinclude:: ../../examples/tutorial/memory_allocation/malloc.py + + .. group-tab:: Cython + + .. literalinclude:: ../../examples/tutorial/memory_allocation/malloc.pyx Note that the C-API functions for allocating memory on the Python heap are generally preferred over the low-level C functions above as the @@ -45,9 +55,20 @@ smaller memory blocks, which speeds up their allocation by avoiding costly operating system calls. The C-API functions can be found in the ``cpython.mem`` standard -declarations file:: +declarations file: + +.. tabs:: + .. group-tab:: Pure Python + + .. code-block:: python - from cpython.mem cimport PyMem_Malloc, PyMem_Realloc, PyMem_Free + from cython.cimports.cpython.mem import PyMem_Malloc, PyMem_Realloc, PyMem_Free + + .. group-tab:: Cython + + .. code-block:: cython + + from cpython.mem cimport PyMem_Malloc, PyMem_Realloc, PyMem_Free Their interface and usage is identical to that of the corresponding low-level C functions. @@ -64,4 +85,11 @@ If a chunk of memory needs a larger lifetime than can be managed by a to a Python object to leverage the Python runtime's memory management, e.g.: -.. literalinclude:: ../../examples/tutorial/memory_allocation/some_memory.pyx +.. tabs:: + .. group-tab:: Pure Python + + .. literalinclude:: ../../examples/tutorial/memory_allocation/some_memory.py + + .. group-tab:: Cython + + .. literalinclude:: ../../examples/tutorial/memory_allocation/some_memory.pyx diff --git a/docs/src/tutorial/numpy.rst b/docs/src/tutorial/numpy.rst index 5fa205976..0a5535da6 100644 --- a/docs/src/tutorial/numpy.rst +++ b/docs/src/tutorial/numpy.rst @@ -28,7 +28,7 @@ systems, it will be :file:`yourmod.pyd`). We run a Python session to test both the Python version (imported from ``.py``-file) and the compiled Cython module. -.. sourcecode:: ipython +.. code-block:: ipythonconsole In [1]: import numpy as np In [2]: import convolve_py @@ -69,7 +69,7 @@ compatibility. Consider this code (*read the comments!*) : After building this and continuing my (very informal) benchmarks, I get: -.. sourcecode:: ipython +.. code-block:: ipythonconsole In [21]: import convolve2 In [22]: %timeit -n2 -r3 convolve2.naive_convolve(f, g) @@ -97,7 +97,7 @@ These are the needed changes:: Usage: -.. sourcecode:: ipython +.. code-block:: ipythonconsole In [18]: import convolve3 In [19]: %timeit -n3 -r100 convolve3.naive_convolve(f, g) @@ -143,7 +143,7 @@ if we try to actually use negative indices with this disabled. The function call overhead now starts to play a role, so we compare the latter two examples with larger N: -.. sourcecode:: ipython +.. code-block:: ipythonconsole In [11]: %timeit -n3 -r100 convolve4.naive_convolve(f, g) 3 loops, best of 100: 5.97 ms per loop @@ -170,6 +170,16 @@ function call.) The actual rules are a bit more complicated but the main message is clear: Do not use typed objects without knowing that they are not set to None. +What typing does not do +======================= + +The main purpose of typing things as :obj:`ndarray` is to allow efficient +indexing of single elements, and to speed up access to a small number of +attributes such as ``.shape``. Typing does not allow Cython to speed +up mathematical operations on the whole array (for example, adding two arrays +together). Typing does not allow Cython to speed up calls to Numpy global +functions or to methods of the array. + More generic code ================== diff --git a/docs/src/tutorial/profiling_tutorial.rst b/docs/src/tutorial/profiling_tutorial.rst index a7cfab0a8..ebe5ab067 100644 --- a/docs/src/tutorial/profiling_tutorial.rst +++ b/docs/src/tutorial/profiling_tutorial.rst @@ -75,8 +75,8 @@ Enabling coverage analysis -------------------------- Since Cython 0.23, line tracing (see above) also enables support for coverage -reporting with the `coverage.py <http://coverage.readthedocs.io/>`_ tool. -To make the coverage analysis understand Cython modules, you also need to enable +reporting with the `coverage.py <https://coverage.readthedocs.io/>`_ tool. To +make the coverage analysis understand Cython modules, you also need to enable Cython's coverage plugin in your ``.coveragerc`` file as follows: .. code-block:: ini @@ -200,18 +200,21 @@ profile script then at all. The script now outputs the following: 1 0.000 0.000 4.406 4.406 <string>:1(<module>) 1 0.000 0.000 0.000 0.000 {method 'disable' of '_lsprof.Profiler' objects} -We gained 1.8 seconds. Not too shabby. Comparing the output to the previous, we -see that recip_square function got faster while the approx_pi function has not -changed a lot. Let's concentrate on the recip_square function a bit more. First -note, that this function is not to be called from code outside of our module; -so it would be wise to turn it into a cdef to reduce call overhead. We should -also get rid of the power operator: it is turned into a pow(i,2) function call by -Cython, but we could instead just write i*i which could be faster. The +We gained 1.8 seconds. Not too shabby. Comparing the output to the previous, we +see that the ``recip_square()`` function got faster while the ``approx_pi()`` +function has not changed a lot. Let's concentrate on the recip_square function +a bit more. First, note that this function is not to be called from code outside +of our module; so it would be wise to turn it into a cdef to reduce call overhead. +We should also get rid of the power operator: it is turned into a pow(i,2) function +call by Cython, but we could instead just write i*i which could be faster. The whole function is also a good candidate for inlining. Let's look at the necessary changes for these ideas: .. literalinclude:: ../../examples/tutorial/profiling_tutorial/calc_pi_3.pyx +Note that the ``except`` declaration is needed in the signature of ``recip_square()`` +in order to propagate division by zero errors. + Now running the profile script yields: .. code-block:: none @@ -254,15 +257,15 @@ Running this shows an interesting result: 1 0.000 0.000 0.000 0.000 {method 'disable' of '_lsprof.Profiler' objects} First note the tremendous speed gain: this version only takes 1/50 of the time -of our first Cython version. Also note that recip_square has vanished from the -table like we wanted. But the most peculiar and import change is that -approx_pi also got much faster. This is a problem with all profiling: calling a -function in a profile run adds a certain overhead to the function call. This +of our first Cython version. Also note that recip_square has vanished from the +table like we wanted. But the most peculiar and important change is that +approx_pi also got much faster. This is a problem with all profiling: calling a +function in a profile run adds a certain overhead to the function call. This overhead is **not** added to the time spent in the called function, but to the -time spent in the **calling** function. In this example, approx_pi didn't need 2.622 +time spent in the **calling** function. In this example, approx_pi didn't need 2.622 seconds in the last run; but it called recip_square 10000000 times, each time taking a -little to set up profiling for it. This adds up to the massive time loss of -around 2.6 seconds. Having disabled profiling for the often called function now +little to set up profiling for it. This adds up to the massive time loss of +around 2.6 seconds. Having disabled profiling for the often called function now reveals realistic timings for approx_pi; we could continue optimizing it now if needed. @@ -274,5 +277,3 @@ faster than calling pow(x,0.5). Even so, the result we achieved here is quite satisfactory: we came up with a solution that is much faster then our original Python version while retaining functionality and readability. - - diff --git a/docs/src/tutorial/pure.rst b/docs/src/tutorial/pure.rst index 775e7719c..f31821d1f 100644 --- a/docs/src/tutorial/pure.rst +++ b/docs/src/tutorial/pure.rst @@ -13,7 +13,7 @@ To go beyond that, Cython provides language constructs to add static typing and cythonic functionalities to a Python module to make it run much faster when compiled, while still allowing it to be interpreted. This is accomplished via an augmenting ``.pxd`` file, via Python -type annotations (following +type :ref:`pep484_type_annotations` (following `PEP 484 <https://www.python.org/dev/peps/pep-0484/>`_ and `PEP 526 <https://www.python.org/dev/peps/pep-0526/>`_), and/or via special functions and decorators available after importing the magic @@ -82,7 +82,7 @@ in the :file:`.pxd`, that is, to be accessible from Python, In the example above, the type of the local variable `a` in `myfunction()` -is not fixed and will thus be a Python object. To statically type it, one +is not fixed and will thus be a :term:`Python object`. To statically type it, one can use Cython's ``@cython.locals`` decorator (see :ref:`magic_attributes`, and :ref:`magic_attributes_pxd`). @@ -153,26 +153,10 @@ Static typing @exceptval(-1, check=False) # cdef int func() except -1: @exceptval(check=True) # cdef int func() except *: @exceptval(-1, check=True) # cdef int func() except? -1: + @exceptval(check=False) # no exception checking/propagation -* Python annotations can be used to declare argument types, as shown in the - following example. To avoid conflicts with other kinds of annotation - usages, this can be disabled with the directive ``annotation_typing=False``. - - .. literalinclude:: ../../examples/tutorial/pure/annotations.py - - This can be combined with the ``@cython.exceptval()`` decorator for non-Python - return types: - - .. literalinclude:: ../../examples/tutorial/pure/exceptval.py - - Since version 0.27, Cython also supports the variable annotations defined - in `PEP 526 <https://www.python.org/dev/peps/pep-0526/>`_. This allows to - declare types of variables in a Python 3.6 compatible way as follows: - - .. literalinclude:: ../../examples/tutorial/pure/pep_526.py - - There is currently no way to express the visibility of object attributes. - + If exception propagation is disabled, any Python exceptions that are raised + inside of the function will be printed and ignored. C types ^^^^^^^ @@ -225,6 +209,23 @@ Here is an example of a :keyword:`cdef` function:: return a == b +cimports +^^^^^^^^ + +The special ``cython.cimports`` package name gives access to cimports +in code that uses Python syntax. Note that this does not mean that C +libraries become available to Python code. It only means that you can +tell Cython what cimports you want to use, without requiring special +syntax. Running such code in plain Python will fail. + +.. literalinclude:: ../../examples/tutorial/pure/py_cimport.py + +Since such code must necessarily refer to the non-existing +``cython.cimports`` 'package', the plain cimport form +``cimport cython.cimports...`` is not available. +You must use the form ``from cython.cimports...``. + + Further Cython functions and declarations ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -242,6 +243,13 @@ Further Cython functions and declarations print(cython.sizeof(cython.longlong)) print(cython.sizeof(n)) +* ``typeof`` returns a string representation of the argument's type for debugging purposes. It can take expressions. + + :: + + cython.declare(n=cython.longlong) + print(cython.typeof(n)) + * ``struct`` can be used to create struct types.:: MyStruct = cython.struct(x=cython.int, y=cython.int, data=cython.double) @@ -289,10 +297,58 @@ can be augmented with the following :file:`.pxd` file :file:`dostuff.pxd`: The :func:`cython.declare()` function can be used to specify types for global variables in the augmenting :file:`.pxd` file. +.. _pep484_type_annotations: + +PEP-484 type annotations +------------------------ + +Python `type hints <https://www.python.org/dev/peps/pep-0484>`_ +can be used to declare argument types, as shown in the +following example. To avoid conflicts with other kinds of annotation +usages, this can be disabled with the directive ``annotation_typing=False``. + + .. literalinclude:: ../../examples/tutorial/pure/annotations.py + +Note the use of ``cython.int`` rather than ``int`` - Cython does not translate +an ``int`` annotation to a C integer by default since the behaviour can be +quite different with respect to overflow and division. + +Annotations can be combined with the ``@cython.exceptval()`` decorator for non-Python +return types: + + .. literalinclude:: ../../examples/tutorial/pure/exceptval.py + +Note that the default exception handling behaviour when returning C numeric types +is to check for ``-1``, and if that was returned, check Python's error indicator +for an exception. This means, if no ``@exceptval`` decorator is provided, and the +return type is a numeric type, then the default with type annotations is +``@exceptval(-1, check=True)``, in order to make sure that exceptions are correctly +and efficiently reported to the caller. Exception propagation can be disabled +explicitly with ``@exceptval(check=False)``, in which case any Python exceptions +raised inside of the function will be printed and ignored. + +Since version 0.27, Cython also supports the variable annotations defined +in `PEP 526 <https://www.python.org/dev/peps/pep-0526/>`_. This allows to +declare types of variables in a Python 3.6 compatible way as follows: + +.. literalinclude:: ../../examples/tutorial/pure/pep_526.py + +There is currently no way to express the visibility of object attributes. + +Cython does not support the full range of annotations described by PEP-484. +For example it does not currently understand features from the ``typing`` module +such as ``Optional[]`` or typed containers such as ``List[str]``. This is partly +because some of these type hints are not relevant for the compilation to +efficient C code. In other cases, however, where the generated C code could +benefit from these type hints but does not currently, help is welcome to +improve the type analysis in Cython. + Tips and Tricks --------------- +.. _calling-c-functions: + Calling C functions ^^^^^^^^^^^^^^^^^^^ diff --git a/docs/src/tutorial/pxd_files.rst b/docs/src/tutorial/pxd_files.rst index 5fc5b0a89..0a22f7a2a 100644 --- a/docs/src/tutorial/pxd_files.rst +++ b/docs/src/tutorial/pxd_files.rst @@ -11,27 +11,25 @@ using the ``cimport`` keyword. ``pxd`` files have many use-cases: - 1. They can be used for sharing external C declarations. - 2. They can contain functions which are well suited for inlining by - the C compiler. Such functions should be marked ``inline``, example: - :: +1. They can be used for sharing external C declarations. +2. They can contain functions which are well suited for inlining by + the C compiler. Such functions should be marked ``inline``, example:: cdef inline int int_min(int a, int b): return b if b < a else a - 3. When accompanying an equally named ``pyx`` file, they +3. When accompanying an equally named ``pyx`` file, they provide a Cython interface to the Cython module so that other Cython modules can communicate with it using a more efficient protocol than the Python one. In our integration example, we might break it up into ``pxd`` files like this: - 1. Add a ``cmath.pxd`` function which defines the C functions available from +1. Add a ``cmath.pxd`` function which defines the C functions available from the C ``math.h`` header file, like ``sin``. Then one would simply do ``from cmath cimport sin`` in ``integrate.pyx``. - 2. Add a ``integrate.pxd`` so that other modules written in Cython - can define fast custom functions to integrate. - :: +2. Add a ``integrate.pxd`` so that other modules written in Cython + can define fast custom functions to integrate:: cdef class Function: cpdef evaluate(self, double x) @@ -41,3 +39,37 @@ In our integration example, we might break it up into ``pxd`` files like this: Note that if you have a cdef class with attributes, the attributes must be declared in the class declaration ``pxd`` file (if you use one), not the ``pyx`` file. The compiler will tell you about this. + + +__init__.pxd +^^^^^^^^^^^^ + +Cython also supports ``__init__.pxd`` files for declarations in package's +namespaces, similar to ``__init__.py`` files in Python. + +Continuing the integration example, we could package the module as follows: + +1. Place the module files in a directory tree as one usually would for + Python: + + .. code-block:: text + + CyIntegration/ + ├── __init__.pyx + ├── __init__.pxd + ├── integrate.pyx + └── integrate.pxd + +2. In ``__init__.pxd``, use ``cimport`` for any declarations that one + would want to be available from the package's main namespace:: + + from CyIntegration cimport integrate + + Other modules would then be able to use ``cimport`` on the package in + order to recursively gain faster, Cython access to the entire package + and the data declared in its modules:: + + cimport CyIntegration + + cpdef do_integration(CyIntegration.integrate.Function f): + return CyIntegration.integrate.integrate(f, 0., 2., 1) diff --git a/docs/src/tutorial/python_division.png b/docs/src/tutorial/python_division.png Binary files differindex 617be942c..a54fdd0b7 100644 --- a/docs/src/tutorial/python_division.png +++ b/docs/src/tutorial/python_division.png diff --git a/docs/src/tutorial/readings.rst b/docs/src/tutorial/readings.rst index a3f09d39e..80ed26e66 100644 --- a/docs/src/tutorial/readings.rst +++ b/docs/src/tutorial/readings.rst @@ -1,7 +1,7 @@ Further reading =============== -The main documentation is located at http://docs.cython.org/. Some +The main documentation is located at https://docs.cython.org/. Some recent features might not have documentation written yet, in such cases some notes can usually be found in the form of a Cython Enhancement Proposal (CEP) on https://github.com/cython/cython/wiki/enhancements. @@ -16,7 +16,7 @@ features for managing it. Finally, don't hesitate to ask questions (or post reports on successes!) on the Cython users mailing list [UserList]_. The Cython developer mailing list, [DevList]_, is also open to everybody, but -focusses on core development issues. Feel free to use it to report a +focuses on core development issues. Feel free to use it to report a clear bug, to ask for guidance if you have time to spare to develop Cython, or if you have suggestions for future development. diff --git a/docs/src/tutorial/related_work.rst b/docs/src/tutorial/related_work.rst index 01cc5b327..af55ae88b 100644 --- a/docs/src/tutorial/related_work.rst +++ b/docs/src/tutorial/related_work.rst @@ -41,8 +41,9 @@ Python modules. .. [ctypes] https://docs.python.org/library/ctypes.html. .. there's also the original ctypes home page: http://python.net/crew/theller/ctypes/ -.. [Pyrex] G. Ewing, Pyrex: C-Extensions for Python, - http://www.cosc.canterbury.ac.nz/greg.ewing/python/Pyrex/ +.. + [Pyrex] G. Ewing, Pyrex: C-Extensions for Python, + https://www.cosc.canterbury.ac.nz/greg.ewing/python/Pyrex/ .. [ShedSkin] M. Dufour, J. Coughlan, ShedSkin, https://github.com/shedskin/shedskin .. [SWIG] David M. Beazley et al., diff --git a/docs/src/tutorial/strings.rst b/docs/src/tutorial/strings.rst index a4ce2b9d0..0a3e348dc 100644 --- a/docs/src/tutorial/strings.rst +++ b/docs/src/tutorial/strings.rst @@ -124,6 +124,9 @@ Python variable:: from c_func cimport c_call_returning_a_c_string cdef char* c_string = c_call_returning_a_c_string() + if c_string is NULL: + ... # handle error + cdef bytes py_string = c_string A type cast to :obj:`object` or :obj:`bytes` will do the same thing:: @@ -441,7 +444,7 @@ characters and is compatible with plain ASCII encoded text that it encodes efficiently. This makes it a very good choice for source code files which usually consist mostly of ASCII characters. -.. _`UTF-8`: http://en.wikipedia.org/wiki/UTF-8 +.. _`UTF-8`: https://en.wikipedia.org/wiki/UTF-8 As an example, putting the following line into a UTF-8 encoded source file will print ``5``, as UTF-8 encodes the letter ``'ö'`` in the two @@ -554,7 +557,7 @@ above character. For more information on this topic, it is worth reading the `Wikipedia article about the UTF-16 encoding`_. -.. _`Wikipedia article about the UTF-16 encoding`: http://en.wikipedia.org/wiki/UTF-16/UCS-2 +.. _`Wikipedia article about the UTF-16 encoding`: https://en.wikipedia.org/wiki/UTF-16/UCS-2 The same properties apply to Cython code that gets compiled for a narrow CPython runtime environment. In most cases, e.g. when diff --git a/docs/src/two-syntax-variants-used b/docs/src/two-syntax-variants-used new file mode 100644 index 000000000..af583a0a9 --- /dev/null +++ b/docs/src/two-syntax-variants-used @@ -0,0 +1,18 @@ +.. note:: + + This page uses two different syntax variants: + + * Cython specific ``cdef`` syntax, which was designed to make type declarations + concise and easily readable from a C/C++ perspective. + + * Pure Python syntax which allows static Cython type declarations in + :ref:`pure Python code <pep484_type_annotations>`, + following `PEP-484 <https://www.python.org/dev/peps/pep-0484/>`_ type hints + and `PEP 526 <https://www.python.org/dev/peps/pep-0526/>`_ variable annotations. + + To make use of C data types in Python syntax, you need to import the special + ``cython`` module in the Python module that you want to compile, e.g. + + .. code-block:: python + + import cython diff --git a/docs/src/userguide/compute_typed_html.jpg b/docs/src/userguide/compute_typed_html.jpg Binary files differindex f8607bdd8..a1e006573 100644 --- a/docs/src/userguide/compute_typed_html.jpg +++ b/docs/src/userguide/compute_typed_html.jpg diff --git a/docs/src/userguide/debugging.rst b/docs/src/userguide/debugging.rst index 06af18bbf..a33ff8dd8 100644 --- a/docs/src/userguide/debugging.rst +++ b/docs/src/userguide/debugging.rst @@ -21,19 +21,20 @@ source, and then running:: make sudo make install +Installing the Cython debugger can be quite tricky. `This installation script and example code <https://gitlab.com/volkerweissmann/cygdb_installation>`_ might be useful. + The debugger will need debug information that the Cython compiler can export. This can be achieved from within the setup script by passing ``gdb_debug=True`` to ``cythonize()``:: - from distutils.core import setup - from distutils.extension import Extension + from setuptools import Extension, setup extensions = [Extension('source', ['source.pyx'])] setup(..., ext_modules=cythonize(extensions, gdb_debug=True)) For development it's often helpful to pass the ``--inplace`` flag to -the ``setup.py`` script, which makes distutils build your project +the ``setup.py`` script, which makes setuptools build your project "in place", i.e., not in a separate `build` directory. When invoking Cython from the command line directly you can have it write diff --git a/docs/src/userguide/extension_types.rst b/docs/src/userguide/extension_types.rst index 63b407871..7bb621fcc 100644 --- a/docs/src/userguide/extension_types.rst +++ b/docs/src/userguide/extension_types.rst @@ -11,7 +11,7 @@ Introduction As well as creating normal user-defined classes with the Python class statement, Cython also lets you create new built-in Python types, known as -extension types. You define an extension type using the :keyword:`cdef` class +:term:`extension types<Extension type>`. You define an extension type using the :keyword:`cdef` class statement. Here's an example: .. literalinclude:: ../../examples/userguide/extension_types/shrubbery.pyx @@ -314,7 +314,7 @@ when it is deleted.:: del shop.cheese print(shop.cheese) -.. sourcecode:: text +.. code-block:: text # Test output We don't have: [] @@ -327,7 +327,8 @@ when it is deleted.:: Subclassing ============= -An extension type may inherit from a built-in type or another extension type:: +If an extension type inherits from other types, the first base class must be +a built-in type or another extension type:: cdef class Parrot: ... @@ -342,7 +343,9 @@ extern extension type. If the base type is defined in another Cython module, it must either be declared as an extern extension type or imported using the :keyword:`cimport` statement. -An extension type can only have one base class (no multiple inheritance). +Multiple inheritance is supported, however the second and subsequent base +classes must be an ordinary Python class (not an extension type or a built-in +type). Cython extension types can also be subclassed in Python. A Python class can inherit from multiple extension types provided that the usual Python rules for @@ -399,7 +402,7 @@ compared to a :keyword:`cdef` method:: print("p2:") p2.describe() -.. sourcecode:: text +.. code-block:: text # Output p1: @@ -504,7 +507,7 @@ It is quite common to want to instantiate an extension class from an existing (pointer to a) data structure, often as returned by external C/C++ functions. As extension classes can only accept Python objects as arguments in their -contructors, this necessitates the use of factory functions. For example, :: +constructors, this necessitates the use of factory functions. For example, :: from libc.stdlib cimport malloc, free @@ -611,10 +614,97 @@ object called :attr:`__weakref__`. For example,:: cdef object __weakref__ -Controlling cyclic garbage collection in CPython -================================================ +Controlling deallocation and garbage collection in CPython +========================================================== -By default each extension type will support the cyclic garbage collector of +.. NOTE:: + + This section only applies to the usual CPython implementation + of Python. Other implementations like PyPy work differently. + +.. _dealloc_intro: + +Introduction +------------ + +First of all, it is good to understand that there are two ways to +trigger deallocation of Python objects in CPython: +CPython uses reference counting for all objects and any object with a +reference count of zero is immediately deallocated. This is the most +common way of deallocating an object. For example, consider :: + + >>> x = "foo" + >>> x = "bar" + +After executing the second line, the string ``"foo"`` is no longer referenced, +so it is deallocated. This is done using the ``tp_dealloc`` slot, which can be +customized in Cython by implementing ``__dealloc__``. + +The second mechanism is the cyclic garbage collector. +This is meant to resolve cyclic reference cycles such as :: + + >>> class Object: + ... pass + >>> def make_cycle(): + ... x = Object() + ... y = [x] + ... x.attr = y + +When calling ``make_cycle``, a reference cycle is created since ``x`` +references ``y`` and vice versa. Even though neither ``x`` or ``y`` +are accessible after ``make_cycle`` returns, both have a reference count +of 1, so they are not immediately deallocated. At regular times, the garbage +collector runs, which will notice the reference cycle +(using the ``tp_traverse`` slot) and break it. +Breaking a reference cycle means taking an object in the cycle +and removing all references from it to other Python objects (we call this +*clearing* an object). Clearing is almost the same as deallocating, except +that the actual object is not yet freed. For ``x`` in the example above, +the attributes of ``x`` would be removed from ``x``. + +Note that it suffices to clear just one object in the reference cycle, +since there is no longer a cycle after clearing one object. Once the cycle +is broken, the usual refcount-based deallocation will actually remove the +objects from memory. Clearing is implemented in the ``tp_clear`` slot. +As we just explained, it is sufficient that one object in the cycle +implements ``tp_clear``. + +.. _trashcan: + +Enabling the deallocation trashcan +---------------------------------- + +In CPython, it is possible to create deeply recursive objects. For example:: + + >>> L = None + >>> for i in range(2**20): + ... L = [L] + +Now imagine that we delete the final ``L``. Then ``L`` deallocates +``L[0]``, which deallocates ``L[0][0]`` and so on until we reach a +recursion depth of ``2**20``. This deallocation is done in C and such +a deep recursion will likely overflow the C call stack, crashing Python. + +CPython invented a mechanism for this called the *trashcan*. It limits the +recursion depth of deallocations by delaying some deallocations. + +By default, Cython extension types do not use the trashcan but it can be +enabled by setting the ``trashcan`` directive to ``True``. For example:: + + cimport cython + @cython.trashcan(True) + cdef class Object: + cdef dict __dict__ + +Trashcan usage is inherited by subclasses +(unless explicitly disabled by ``@cython.trashcan(False)``). +Some builtin types like ``list`` use the trashcan, so subclasses of it +use the trashcan by default. + +Disabling cycle breaking (``tp_clear``) +--------------------------------------- + +By default, each extension type will support the cyclic garbage collector of CPython. If any Python objects can be referenced, Cython will automatically generate the ``tp_traverse`` and ``tp_clear`` slots. This is usually what you want. @@ -622,13 +712,13 @@ want. There is at least one reason why this might not be what you want: If you need to cleanup some external resources in the ``__dealloc__`` special function and your object happened to be in a reference cycle, the garbage collector may -have triggered a call to ``tp_clear`` to drop references. This is the way that -reference cycles are broken so that the garbage can actually be reclaimed. +have triggered a call to ``tp_clear`` to clear the object +(see :ref:`dealloc_intro`). -In that case any object references have vanished by the time when -``__dealloc__`` is called. Now your cleanup code lost access to the objects it -has to clean up. In that case you can disable the cycle breaker ``tp_clear`` -by using the ``no_gc_clear`` decorator :: +In that case, any object references have vanished when ``__dealloc__`` +is called. Now your cleanup code lost access to the objects it has to clean up. +To fix this, you can disable clearing instances of a specific class by using +the ``no_gc_clear`` directive:: @cython.no_gc_clear cdef class DBCursor: @@ -641,17 +731,21 @@ by using the ``no_gc_clear`` decorator :: This example tries to close a cursor via a database connection when the Python object is destroyed. The ``DBConnection`` object is kept alive by the reference from ``DBCursor``. But if a cursor happens to be in a reference cycle, the -garbage collector may effectively "steal" the database connection reference, +garbage collector may delete the database connection reference, which makes it impossible to clean up the cursor. -Using the ``no_gc_clear`` decorator this can not happen anymore because the -references of a cursor object will not be cleared anymore. +If you use ``no_gc_clear``, it is important that any given reference cycle +contains at least one object *without* ``no_gc_clear``. Otherwise, the cycle +cannot be broken, which is a memory leak. + +Disabling cyclic garbage collection +----------------------------------- In rare cases, extension types can be guaranteed not to participate in cycles, but the compiler won't be able to prove this. This would be the case if the class can never reference itself, even indirectly. In that case, you can manually disable cycle collection by using the -``no_gc`` decorator, but beware that doing so when in fact the extension type +``no_gc`` directive, but beware that doing so when in fact the extension type can participate in cycles could cause memory leaks :: @cython.no_gc @@ -730,7 +824,7 @@ built-in complex object.:: because, in the Python header files, the ``PyComplexObject`` struct is declared with: - .. sourcecode:: c + .. code-block:: c typedef struct { ... @@ -805,8 +899,7 @@ write ``dtype.itemsize`` in Cython code which will be compiled into direct access of the C struct field, without going through a C-API equivalent of ``dtype.__getattr__('itemsize')``. -For example we may have an extension -module ``foo_extension``:: +For example, we may have an extension module ``foo_extension``:: cdef class Foo: cdef public int field0, field1, field2; @@ -816,7 +909,9 @@ module ``foo_extension``:: self.field1 = f1 self.field2 = f2 -but a C struct in a file ``foo_nominal.h``:: +but a C struct in a file ``foo_nominal.h``: + +.. code-block:: c typedef struct { PyObject_HEAD @@ -850,6 +945,7 @@ use the C-API equivalent of:: instead of the desired C equivalent of ``return f->f0 + f->f1 + f->f2``. We can alias the fields by using:: + cdef extern from "foo_nominal.h": ctypedef class foo_extension.Foo [object FooStructNominal]: @@ -867,6 +963,19 @@ code. No changes to Python need be made to achieve significant speedups, even though the field names in Python and C are different. Of course, one should make sure the fields are equivalent. +C inline properties +------------------- + +Similar to Python property attributes, Cython provides a way to declare C-level +properties on external extension types. This is often used to shadow Python +attributes through faster C level data access, but can also be used to add certain +functionality to existing types when using them from Cython. The declarations +must use `cdef inline`. + +For example, the above ``complex`` type could also be declared like this: + +.. literalinclude:: ../../examples/userguide/extension_types/c_property.pyx + Implicit importing ------------------ diff --git a/docs/src/userguide/external_C_code.rst b/docs/src/userguide/external_C_code.rst index e6605223d..75ad3fb2f 100644 --- a/docs/src/userguide/external_C_code.rst +++ b/docs/src/userguide/external_C_code.rst @@ -189,19 +189,19 @@ same applies equally to union and enum declarations. +-------------------------+---------------------------------------------+-----------------------------------------------------------------------+ | C code | Possibilities for corresponding Cython Code | Comments | +=========================+=============================================+=======================================================================+ -| .. sourcecode:: c | :: | Cython will refer to the as ``struct Foo`` in the generated C code. | +| .. code-block:: c | :: | Cython will refer to the as ``struct Foo`` in the generated C code. | | | | | | struct Foo { | cdef struct Foo: | | | ... | ... | | | }; | | | +-------------------------+---------------------------------------------+-----------------------------------------------------------------------+ -| .. sourcecode:: c | :: | Cython will refer to the type simply as ``Foo`` in | +| .. code-block:: c | :: | Cython will refer to the type simply as ``Foo`` in | | | | the generated C code. | | typedef struct { | ctypedef struct Foo: | | | ... | ... | | | } Foo; | | | +-------------------------+---------------------------------------------+-----------------------------------------------------------------------+ -| .. sourcecode:: c | :: | If the C header uses both a tag and a typedef with *different* | +| .. code-block:: c | :: | If the C header uses both a tag and a typedef with *different* | | | | names, you can use either form of declaration in Cython | | typedef struct foo { | cdef struct foo: | (although if you need to forward reference the type, | | ... | ... | you'll have to use the first form). | @@ -212,7 +212,7 @@ same applies equally to union and enum declarations. | | ctypedef struct Foo: | | | | ... | | +-------------------------+---------------------------------------------+-----------------------------------------------------------------------+ -| .. sourcecode:: c | :: | If the header uses the *same* name for the tag and typedef, you | +| .. code-block:: c | :: | If the header uses the *same* name for the tag and typedef, you | | | | won't be able to include a :keyword:`ctypedef` for it -- but then, | | typedef struct Foo { | cdef struct Foo: | it's not necessary. | | ... | ... | | @@ -223,6 +223,38 @@ See also use of :ref:`external_extension_types`. Note that in all the cases below, you refer to the type in Cython code simply as :c:type:`Foo`, not ``struct Foo``. +Pointers +-------- +When interacting with a C-api there may be functions that require pointers as arguments. +Pointers are variables that contain a memory address to another variable. + +For example:: + + cdef extern from "<my_lib.h>": + cdef void increase_by_one(int *my_var) + +This function takes a pointer to an integer as argument. Knowing the address of the +integer allows the function to modify the value in place, so that the caller can see +the changes afterwards. In order to get the address from an existing variable, +use the ``&`` operator:: + + cdef int some_int = 42 + cdef int *some_int_pointer = &some_int + increase_by_one(some_int_pointer) + # Or without creating the extra variable + increase_by_one(&some_int) + print(some_int) # prints 44 (== 42+1+1) + +If you want to manipulate the variable the pointer points to, you can access it by +referencing its first element like you would in python ``my_pointer[0]``. For example:: + + cdef void increase_by_one(int *my_var): + my_var[0] += 1 + +For a deeper introduction to pointers, you can read `this tutorial at tutorialspoint +<https://www.tutorialspoint.com/cprogramming/c_pointers.htm>`_. For differences between +Cython and C syntax for manipulating pointers, see :ref:`statements_and_expressions`. + Accessing Python/C API routines --------------------------------- @@ -235,6 +267,10 @@ routines in the Python/C API. For example,:: will allow you to create Python strings containing null bytes. +Note that Cython comes with ready-to-use declarations of (almost) all C-API functions +in the cimportable ``cpython.*`` modules. See the list in +https://github.com/cython/cython/tree/master/Cython/Includes/cpython + Special Types -------------- @@ -329,13 +365,16 @@ are entirely on your own with this feature. If you want to declare a name the C file for it, you can do this using a C name declaration. Consider this an advanced feature, only for the rare cases where everything else fails. + +.. _verbatim_c: + Including verbatim C code ------------------------- For advanced use cases, Cython allows you to directly write C code as "docstring" of a ``cdef extern from`` block: -.. literalinclude:: ../../examples/userguide/external_C_code/c_code_docstring.pyx +.. literalinclude:: ../../examples/userguide/external_C_code/verbatim_c_code.pyx The above is essentially equivalent to having the C code in a file ``header.h`` and writing :: @@ -344,6 +383,11 @@ The above is essentially equivalent to having the C code in a file long square(long x) void assign(long& x, long y) +This feature is commonly used for platform specific adaptations at +compile time, for example: + +.. literalinclude:: ../../examples/userguide/external_C_code/platform_adaptation.pyx + It is also possible to combine a header file and verbatim C code:: cdef extern from "badheader.h": @@ -381,12 +425,12 @@ You can make C types, variables and functions defined in a Cython module accessible to C code that is linked together with the Cython-generated C file, by declaring them with the public keyword:: - cdef public struct Bunny: # public type declaration + cdef public struct Bunny: # a public type declaration int vorpalness - cdef public int spam # public variable declaration + cdef public int spam # a public variable declaration - cdef public void grail(Bunny *) # public function declaration + cdef public void grail(Bunny *) # a public function declaration If there are any public declarations in a Cython module, a header file called :file:`modulename.h` file is generated containing equivalent C declarations for @@ -488,7 +532,7 @@ the call to :func:`import_modulename`, it is likely that this wasn't done. You can use both :keyword:`public` and :keyword:`api` on the same function to make it available by both methods, e.g.:: - cdef public api void belt_and_braces(): + cdef public api void belt_and_braces() except *: ... However, note that you should include either :file:`modulename.h` or @@ -513,8 +557,8 @@ You can declare a whole group of items as :keyword:`public` and/or example,:: cdef public api: - void order_spam(int tons) - char *get_lunch(float tomato_size) + void order_spam(int tons) except * + char *get_lunch(float tomato_size) except NULL This can be a useful thing to do in a ``.pxd`` file (see :ref:`sharing-declarations`) to make the module's public interface @@ -524,7 +568,7 @@ Acquiring and Releasing the GIL --------------------------------- Cython provides facilities for acquiring and releasing the -`Global Interpreter Lock (GIL) <http://docs.python.org/dev/glossary.html#term-global-interpreter-lock>`_. +`Global Interpreter Lock (GIL) <https://docs.python.org/dev/glossary.html#term-global-interpreter-lock>`_. This may be useful when calling from multi-threaded code into (external C) code that may block, or when wanting to use Python from a (native) C thread callback. Releasing the GIL should @@ -549,14 +593,18 @@ You can release the GIL around a section of code using the with nogil: <code to be executed with the GIL released> -Code in the body of the with-statement must not raise exceptions or -manipulate Python objects in any way, and must not call anything that -manipulates Python objects without first re-acquiring the GIL. Cython -validates these operations at compile time, but cannot look into -external C functions, for example. They must be correctly declared -as requiring or not requiring the GIL (see below) in order to make +Code in the body of the with-statement must not manipulate Python objects +in any way, and must not call anything that manipulates Python objects without +first re-acquiring the GIL. Cython validates these operations at compile time, +but cannot look into external C functions, for example. They must be correctly +declared as requiring or not requiring the GIL (see below) in order to make Cython's checks effective. +Since Cython 3.0, some simple Python statements can be used inside of ``nogil`` +sections: ``raise``, ``assert`` and ``print`` (the Py2 statement, not the function). +Since they tend to be lone Python statements, Cython will automatically acquire +and release the GIL around them for convenience. + .. _gil: Acquiring the GIL @@ -580,6 +628,26 @@ The GIL may also be acquired through the ``with gil`` statement:: with gil: <execute this block with the GIL acquired> +.. _gil_conditional: + +Conditional Acquiring / Releasing the GIL +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Sometimes it is helpful to use a condition to decide whether to run a +certain piece of code with or without the GIL. This code would run anyway, +the difference is whether the GIL will be held or released. +The condition must be constant (at compile time). + +This could be useful for profiling, debugging, performance testing, and +for fused types (see :ref:`fused_gil_conditional`).:: + + DEF FREE_GIL = True + + with nogil(FREE_GIL): + <code to be executed with the GIL released> + + with gil(False): + <GIL is still released> + Declaring a function as callable without the GIL -------------------------------------------------- diff --git a/docs/src/userguide/fusedtypes.rst b/docs/src/userguide/fusedtypes.rst index b50bb0efd..1ca18daf6 100644 --- a/docs/src/userguide/fusedtypes.rst +++ b/docs/src/userguide/fusedtypes.rst @@ -63,24 +63,40 @@ Fused types can be used to declare parameters of functions or methods:: cdef cfunc(my_fused_type arg): return arg + 1 -If the you use the same fused type more than once in an argument list, then each -specialization of the fused type must be the same:: +If the same fused type appears more than once in the function arguments, +then they will all have the same specialised type:: cdef cfunc(my_fused_type arg1, my_fused_type arg2): - return cython.typeof(arg1) == cython.typeof(arg2) + # arg1 and arg2 always have the same type here + return arg1 + arg2 -In this case, the type of both parameters is either an int, or a double -(according to the previous examples). However, because these arguments use the -same fused type ``my_fused_type``, both ``arg1`` and ``arg2`` are -specialized to the same type. Therefore this function returns True for every -possible valid invocation. You are allowed to mix fused types however:: +Here, the type of both parameters is either an int, or a double +(according to the previous examples), because they use the same fused type +name ``my_fused_type``. Mixing different fused types (or differently named +fused types) in the arguments will specialise them independently:: def func(A x, B y): ... -where ``A`` and ``B`` are different fused types. This will result in -specialized code paths for all combinations of types contained in ``A`` -and ``B``. +This will result in specialized code paths for all combinations of types +contained in ``A`` and ``B``, e.g.:: + + ctypedef fused my_fused_type: + cython.int + cython.double + + ctypedef fused my_fused_type2: + cython.int + cython.double + + cdef func(my_fused_type a, my_fused_type2 b): + # a and b may have the same or different types here + print("SAME!" if my_fused_type is my_fused_type2 else "NOT SAME!) + return a + b + +Note that a simple typedef to rename the fused type does not currently work here. +See Github issue :issue:`4302`. + Fused types and arrays ---------------------- @@ -249,6 +265,34 @@ to figure out whether a specialization is part of another set of types if bunch_of_types in string_t: print("s is a string!") +.. _fused_gil_conditional: + +Conditional GIL Acquiring / Releasing +===================================== + +Acquiring and releasing the GIL can be controlled by a condition +which is known at compile time (see :ref:`gil_conditional`). + +This is most useful when combined with fused types. +A fused type function may have to handle both cython native types +(e.g. cython.int or cython.double) and python types (e.g. object or bytes). +Conditional Acquiring / Releasing the GIL provides a method for running +the same piece of code either with the GIL released (for cython native types) +and with the GIL held (for python types).:: + + cimport cython + + ctypedef fused double_or_object: + cython.double + object + + def increment(double_or_object x): + with nogil(double_or_object is cython.double): + # Same code handles both cython.double (GIL is released) + # and python object (GIL is not released). + x = x + 1 + return x + __signatures__ ============== diff --git a/docs/src/userguide/glossary.rst b/docs/src/userguide/glossary.rst new file mode 100644 index 000000000..514139633 --- /dev/null +++ b/docs/src/userguide/glossary.rst @@ -0,0 +1,46 @@ +Glossary +======== + +.. glossary:: + + Extension type + "Extension type" can refer to either a Cython class defined with ``cdef class`` or more generally to any Python type that is ultimately implemented as a native C struct (including the built-in types like `int` or `dict`). + + Dynamic allocation or Heap allocation + A C variable allocated with ``malloc`` (in C) or ``new`` (in C++) is + `allocated dynamically/heap allocated <https://en.wikipedia.org/wiki/C_dynamic_memory_allocation>`_. + Its lifetime is until the user deletes it explicitly (with ``free`` in C or ``del`` in C++). + This can happen in a different function than the allocation. + + pointer + A **pointer** is a variable that stores the address of another variable + (i.e. direct address of the memory location). They allow for + dynamic memory allocation and deallocation. They can be used to build + dynamic data structures. `Read more <https://en.wikipedia.org/wiki/Pointer_(computer_programming)#C_pointers>`__. + + Python object + When using Python, the contents of every variable is a Python object + (including Cython extension types). Key features of Python objects are that + they are passed *by reference* and that their lifetime is *managed* automatically + so that they are destroyed when no more references exist to them. + In Cython, they are distinct from C types, which are passed *by value* and whose + lifetime is managed depending on whether they are allocated on the stack or heap. + To explicitly declare a Python object variable in Cython use ``cdef object abc``. + Internally in C, they are referred to as ``PyObject*``. + + Stack allocation + A C variable declared within a function as ``cdef SomeType a`` + is said to be allocated on the stack. + It exists for the duration of the function only. + + Typed memoryview + A useful Cython type for getting quick access to blocks of memory. + A memoryview alone does not actually own any memory. + However, it can be initialized with a Python object that supports the + `buffer protocol`_ (typically "array" types, for example a Numpy array). + The memoryview keeps a reference to that Python object alive + and provides quick access to the memory without needing to go + through the Python API of the object and its ``__getitem__``/``__setitem__`` methods. + For more information, see :ref:`memoryviews`. + +.. _buffer protocol: https://docs.python.org/3/c-api/buffer.html diff --git a/docs/src/userguide/index.rst b/docs/src/userguide/index.rst index cfbee2fbd..9dc3a75a3 100644 --- a/docs/src/userguide/index.rst +++ b/docs/src/userguide/index.rst @@ -16,6 +16,7 @@ Contents: wrapping_CPlusPlus fusedtypes pypy + migrating_to_cy30 limitations pyrex_differences memoryviews @@ -34,3 +35,4 @@ Indices and tables .. toctree:: + glossary diff --git a/docs/src/userguide/language_basics.rst b/docs/src/userguide/language_basics.rst index 0fdd87783..fb78dcf42 100644 --- a/docs/src/userguide/language_basics.rst +++ b/docs/src/userguide/language_basics.rst @@ -11,6 +11,9 @@ Language Basics ***************** +.. include:: + ../two-syntax-variants-used + .. _declaring_data_types: Declaring Data Types @@ -44,72 +47,152 @@ the use of ‘early binding’ programming techniques. C variable and type definitions =============================== -The :keyword:`cdef` statement is used to declare C variables, either local or -module-level:: +C variables can be declared by - cdef int i, j, k - cdef float f, g[42], *h +* using the Cython specific :keyword:`cdef` statement, +* using PEP-484/526 type annotations with C data types or +* using the function ``cython.declare()``. -and C :keyword:`struct`, :keyword:`union` or :keyword:`enum` types: +The :keyword:`cdef` statement and ``declare()`` can define function-local and +module-level variables as well as attributes in classes, but type annotations only +affect local variables and attributes and are ignored at the module level. +This is because type annotations are not Cython specific, so Cython keeps +the variables in the module dict (as Python values) instead of making them +module internal C variables. Use ``declare()`` in Python code to explicitly +define global C variables. -.. literalinclude:: ../../examples/userguide/language_basics/struct_union_enum.pyx +.. tabs:: -See also :ref:`struct-union-enum-styles` + .. group-tab:: Pure Python -.. note:: + .. code-block:: python + + a_global_variable = declare(cython.int) + + def func(): + i: cython.int + j: cython.int + k: cython.int + f: cython.float + g: cython.int[42] + h: cython.p_float + + .. group-tab:: Cython + + .. code-block:: cython + + cdef int a_global_variable + + def func(): + cdef int i, j, k + cdef float f, g[42], *h + +Moreover, C :keyword:`struct`, :keyword:`union` and :keyword:`enum` are supported: + +.. tabs:: + + .. group-tab:: Pure Python + + .. literalinclude:: ../../examples/userguide/language_basics/struct_union_enum.py - Structs can be declared as ``cdef packed struct``, which has - the same effect as the C directive ``#pragma pack(1)``. + .. note:: Currently, Pure Python mode does not support enums. (GitHub issue :issue:`4252`) -Declaring an enum as ``cpdef`` will create a :pep:`435`-style Python wrapper:: + .. group-tab:: Cython - cpdef enum CheeseState: - hard = 1 - soft = 2 - runny = 3 + .. literalinclude:: ../../examples/userguide/language_basics/struct_union_enum.pyx + See also :ref:`struct-union-enum-styles` + .. note:: -There is currently no special syntax for defining a constant, but you can use -an anonymous :keyword:`enum` declaration for this purpose, for example,:: + Structs can be declared as ``cdef packed struct``, which has + the same effect as the C directive ``#pragma pack(1)``. - cdef enum: - tons_of_spam = 3 + Declaring an enum as ``cpdef`` will create a :pep:`435`-style Python wrapper:: + + cpdef enum CheeseState: + hard = 1 + soft = 2 + runny = 3 + + There is currently no special syntax for defining a constant, but you can use + an anonymous :keyword:`enum` declaration for this purpose, for example,:: + + cdef enum: + tons_of_spam = 3 + + .. note:: + the words ``struct``, ``union`` and ``enum`` are used only when + defining a type, not when referring to it. For example, to declare a variable + pointing to a ``Grail`` struct, you would write:: + + cdef Grail *gp + + and not:: + + cdef struct Grail *gp # WRONG .. note:: - the words ``struct``, ``union`` and ``enum`` are used only when - defining a type, not when referring to it. For example, to declare a variable - pointing to a ``Grail`` you would write:: - cdef Grail *gp + There is also support for giving names to types using the + ``ctypedef`` statement or the ``cython.typedef()`` function, e.g. - and not:: + .. tabs:: - cdef struct Grail *gp # WRONG + .. group-tab:: Pure Python - There is also a ``ctypedef`` statement for giving names to types, e.g.:: + .. code-block:: python - ctypedef unsigned long ULong + ULong = cython.typedef(cython.ulong) - ctypedef int* IntPtr + IntPtr = cython.typedef(cython.p_int) + .. group-tab:: Cython -It is also possible to declare functions with :keyword:`cdef`, making them c functions. + .. code-block:: cython -:: + ctypedef unsigned long ULong - cdef int eggs(unsigned long l, float f): - ... + ctypedef int* IntPtr + + +You can create a C function by declaring it with :keyword:`cdef` or by decorating a Python function with ``@cfunc``: + +.. tabs:: + + .. group-tab:: Pure Python + + .. code-block:: python + + @cython.cfunc + def eggs(l: cython.ulong, f: cython.float) -> cython.int: + ... + + .. group-tab:: Cython + + .. code-block:: cython + + cdef int eggs(unsigned long l, float f): + ... You can read more about them in :ref:`python_functions_vs_c_functions`. -You can declare classes with :keyword:`cdef`, making them :ref:`extension-types`. Those will +Classes can be declared as :ref:`extension-types`. Those will have a behavior very close to python classes, but are faster because they use a ``struct`` internally to store attributes. +They are declared with the :keyword:`cdef` keyword or the ``@cclass`` class decorator. Here is a simple example: -.. literalinclude:: ../../examples/userguide/extension_types/shrubbery.pyx +.. tabs:: + + .. group-tab:: Pure Python + + .. literalinclude:: ../../examples/userguide/extension_types/shrubbery.py + + .. group-tab:: Cython + + .. literalinclude:: ../../examples/userguide/extension_types/shrubbery.pyx You can read more about them in :ref:`extension-types`. @@ -119,15 +202,21 @@ You can read more about them in :ref:`extension-types`. Types ----- -Cython uses the normal C syntax for C types, including pointers. It provides +The Cython language uses the normal C syntax for C types, including pointers. It provides all the standard C types, namely ``char``, ``short``, ``int``, ``long``, -``long long`` as well as their ``unsigned`` versions, e.g. ``unsigned int``. +``long long`` as well as their ``unsigned`` versions, +e.g. ``unsigned int`` (``cython.uint`` in Python code). The special ``bint`` type is used for C boolean values (``int`` with 0/non-0 values for False/True) and ``Py_ssize_t`` for (signed) sizes of Python containers. -Pointer types are constructed as in C, by appending a ``*`` to the base type -they point to, e.g. ``int**`` for a pointer to a pointer to a C int. +Pointer types are constructed as in C when using Cython syntax, by appending a ``*`` to the base type +they point to, e.g. ``int**`` for a pointer to a pointer to a C int. In Pure python mode, simple pointer types +use a naming scheme with "p"s instead, separated from the type name with an underscore, e.g. ``cython.pp_int`` for a pointer to +a pointer to a C int. Further pointer types can be constructed with the ``cython.pointer()`` function, +e.g. ``cython.pointer(cython.int)``. + + Arrays use the normal C array syntax, e.g. ``int[10]``, and the size must be known at compile time for stack allocated arrays. Cython doesn't support variable length arrays from C99. Note that Cython uses array access for pointer dereferencing, as ``*x`` is not valid Python syntax, @@ -135,23 +224,53 @@ whereas ``x[0]`` is. Also, the Python types ``list``, ``dict``, ``tuple``, etc. may be used for static typing, as well as any user defined :ref:`extension-types`. -For example:: +For example - cdef list foo = [] +.. tabs:: -This requires an *exact* match of the class, it does not allow -subclasses. This allows Cython to optimize code by accessing -internals of the builtin class. -For this kind of typing, Cython uses internally a C variable of type ``PyObject*``. -The Python types int, long, and float are not available for static -typing and instead interpreted as C ``int``, ``long``, and ``float`` -respectively, as statically typing variables with these Python -types has zero advantages. + .. group-tab:: Pure Python + + .. code-block:: python + + def main(): + foo: list = [] + + .. group-tab:: Cython + + .. code-block:: cython + + cdef list foo = [] + +This requires an *exact* match of the class, it does not allow subclasses. +This allows Cython to optimize code by accessing internals of the builtin class, +which is the main reason for declaring builtin types in the first place. + +For declared builtin types, Cython uses internally a C variable of type ``PyObject*``. + +.. note:: The Python types ``int``, ``long``, and ``float`` are not available for static + typing in ``.pyx`` files and instead interpreted as C ``int``, ``long``, and ``float`` + respectively, as statically typing variables with these Python + types has zero advantages. On the other hand, annotating in Pure Python with + ``int``, ``long``, and ``float`` Python types will be interpreted as + Python object types. Cython provides an accelerated and typed equivalent of a Python tuple, the ``ctuple``. -A ``ctuple`` is assembled from any valid C types. For example:: +A ``ctuple`` is assembled from any valid C types. For example + +.. tabs:: + + .. group-tab:: Pure Python + + .. code-block:: python - cdef (double, int) bar + def main(): + bar: (cython.double, cython.int) + + .. group-tab:: Cython + + .. code-block:: cython + + cdef (double, int) bar They compile down to C-structures and can be used as efficient alternatives to Python tuples. @@ -164,9 +283,9 @@ and is typically what one wants). If you want to use these numeric Python types simply omit the type declaration and let them be objects. -It is also possible to declare :ref:`extension-types` (declared with ``cdef class``). +It is also possible to declare :ref:`extension-types` (declared with ``cdef class`` or the ``@cclass`` decorator). This does allow subclasses. This typing is mostly used to access -``cdef`` methods and attributes of the extension type. +``cdef``/``@cfunc`` methods and attributes of the extension type. The C code uses a variable which is a pointer to a structure of the specific type, something like ``struct MyExtensionTypeObject*``. @@ -177,8 +296,11 @@ Grouping multiple C declarations If you have a series of declarations that all begin with :keyword:`cdef`, you can group them into a :keyword:`cdef` block like this: +.. note:: This is supported only in Cython's ``cdef`` syntax. + .. literalinclude:: ../../examples/userguide/language_basics/cdef_block.pyx + .. _cpdef: .. _cdef: .. _python_functions_vs_c_functions: @@ -188,48 +310,95 @@ Python functions vs. C functions There are two kinds of function definition in Cython: -Python functions are defined using the def statement, as in Python. They take -Python objects as parameters and return Python objects. +Python functions are defined using the :keyword:`def` statement, as in Python. They take +:term:`Python objects<Python object>` as parameters and return Python objects. -C functions are defined using the new :keyword:`cdef` statement. They take +C functions are defined using the :keyword:`cdef` statement in Cython syntax or with the ``@cfunc`` decorator. They take either Python objects or C values as parameters, and can return either Python objects or C values. Within a Cython module, Python functions and C functions can call each other freely, but only Python functions can be called from outside the module by interpreted Python code. So, any functions that you want to "export" from your -Cython module must be declared as Python functions using def. -There is also a hybrid function, called :keyword:`cpdef`. A :keyword:`cpdef` -can be called from anywhere, but uses the faster C calling conventions -when being called from other Cython code. A :keyword:`cpdef` can also be overridden +Cython module must be declared as Python functions using ``def``. +There is also a hybrid function, declared with :keyword:`cpdef` in ``.pyx`` files or with the ``@ccall`` decorator. These functions +can be called from anywhere, but use the faster C calling convention +when being called from other Cython code. They can also be overridden by a Python method on a subclass or an instance attribute, even when called from Cython. If this happens, most performance gains are of course lost and even if it does not, -there is a tiny overhead in calling a :keyword:`cpdef` method from Cython compared to -calling a :keyword:`cdef` method. +there is a tiny overhead in calling such a method from Cython compared to +calling a C method. Parameters of either type of function can be declared to have C data types, -using normal C declaration syntax. For example,:: +using normal C declaration syntax. For example, - def spam(int i, char *s): - ... +.. tabs:: - cdef int eggs(unsigned long l, float f): - ... + .. group-tab:: Pure Python -``ctuples`` may also be used:: + .. code-block:: python - cdef (int, float) chips((long, long, double) t): - ... + def spam(i: cython.int, s: cython.p_char): + ... + + @cython.cfunc + def eggs(l: cython.ulong, f: cython.float) -> cython.int: + ... + + .. group-tab:: Cython + + .. code-block:: cython + + def spam(int i, char *s): + ... + + + cdef int eggs(unsigned long l, float f): + ... +``ctuples`` may also be used + +.. tabs:: + + .. group-tab:: Pure Python + + .. code-block:: python + + @cython.cfunc + def chips(t: (cython.long, cython.long, cython.double)) -> (cython.int, cython.float): + ... + + .. group-tab:: Cython + + .. code-block:: cython + + cdef (int, float) chips((long, long, double) t): + ... When a parameter of a Python function is declared to have a C data type, it is passed in as a Python object and automatically converted to a C value, if possible. In other words, the definition of ``spam`` above is equivalent to -writing:: +writing + +.. tabs:: + + .. group-tab:: Pure Python + + .. code-block:: python + + def spam(python_i, python_s): + i: cython.int = python_i + s: cython.p_char = python_s + ... + + .. group-tab:: Cython + + .. code-block:: cython + + def spam(python_i, python_s): + cdef int i = python_i + cdef char* s = python_s + ... - def spam(python_i, python_s): - cdef int i = python_i - cdef char* s = python_s - ... Automatic conversion is currently only possible for numeric types, string types and structs (composed recursively of any of these types); @@ -242,7 +411,8 @@ with string attributes if they are to be used after the function returns. C functions, on the other hand, can have parameters of any type, since they're passed in directly using a normal C function call. -Functions declared using :keyword:`cdef` with Python object return type, like Python functions, will return a :keyword:`None` +C Functions declared using :keyword:`cdef` or the ``@cfunc`` decorator with a +Python object return type, like Python functions, will return a :keyword:`None` value when execution leaves the function body without an explicit return value. This is in contrast to C/C++, which leaves the return value undefined. In the case of non-Python object return types, the equivalent of zero is returned, for example, 0 for ``int``, :keyword:`False` for ``bint`` and :keyword:`NULL` for pointer types. @@ -256,10 +426,24 @@ Python objects as parameters and return values If no type is specified for a parameter or return value, it is assumed to be a Python object. (Note that this is different from the C convention, where it would default to int.) For example, the following defines a C function that -takes two Python objects as parameters and returns a Python object:: +takes two Python objects as parameters and returns a Python object - cdef spamobjs(x, y): - ... +.. tabs:: + + .. group-tab:: Pure Python + + .. code-block:: python + + @cython.cfunc + def spamobjs(x, y): + ... + + .. group-tab:: Cython + + .. code-block:: cython + + cdef spamobjs(x, y): + ... Reference counting for these objects is performed automatically according to the standard Python/C API rules (i.e. borrowed references are taken as @@ -273,43 +457,90 @@ parameters and a new reference is returned). The name object can also be used to explicitly declare something as a Python object. This can be useful if the name being declared would otherwise be taken -as the name of a type, for example,:: +as the name of a type, for example, - cdef ftang(object int): - ... +.. tabs:: + + .. group-tab:: Pure Python + + .. code-block:: python + + @cython.cfunc + def ftang(int: object): + ... -declares a parameter called int which is a Python object. You can also use + .. group-tab:: Cython + + .. code-block:: cython + + cdef ftang(object int): + ... + +declares a parameter called ``int`` which is a Python object. You can also use object as the explicit return type of a function, e.g.:: cdef object ftang(object int): ... +.. note:: Currently, Cython contains a bug not allowing ``object`` as return annotation in + pure Python from a C function. (GitHub issue :issue:`2529`) + In the interests of clarity, it is probably a good idea to always be explicit about object parameters in C functions. +To create a borrowed reference, specify the parameter type as ``PyObject*``. +Cython won't perform automatic ``Py_INCREF``, or ``Py_DECREF``, e.g.: + +.. tabs:: + + .. group-tab:: Pure Python + + .. literalinclude:: ../../examples/userguide/language_basics/parameter_refcount.py + + .. group-tab:: Cython + + .. literalinclude:: ../../examples/userguide/language_basics/parameter_refcount.pyx + +will display:: + + Initial refcount: 2 + Inside owned_reference: 3 + Inside borrowed_reference: 2 + .. _optional_arguments: Optional Arguments ------------------ -Unlike C, it is possible to use optional arguments in ``cdef`` and ``cpdef`` functions. -There are differences though whether you declare them in a ``.pyx`` +Unlike C, it is possible to use optional arguments in C and ``cpdef``/``@ccall`` functions. +There are differences though whether you declare them in a ``.pyx``/``.py`` file or the corresponding ``.pxd`` file. To avoid repetition (and potential future inconsistencies), default argument values are not visible in the declaration (in ``.pxd`` files) but only in the implementation (in ``.pyx`` files). -When in a ``.pyx`` file, the signature is the same as it is in Python itself: +When in a ``.pyx``/``.py`` file, the signature is the same as it is in Python itself: + +.. tabs:: + + .. group-tab:: Pure Python -.. literalinclude:: ../../examples/userguide/language_basics/optional_subclassing.pyx + .. literalinclude:: ../../examples/userguide/language_basics/optional_subclassing.py + :caption: optional_subclassing.py + + .. group-tab:: Cython + + .. literalinclude:: ../../examples/userguide/language_basics/optional_subclassing.pyx + :caption: optional_subclassing.pyx When in a ``.pxd`` file, the signature is different like this example: ``cdef foo(x=*)``. This is because the program calling the function just needs to know what signatures are possible in C, but doesn't need to know the value of the default arguments.: .. literalinclude:: ../../examples/userguide/language_basics/optional_subclassing.pxd + :caption: optional_subclassing.pxd .. note:: The number of arguments may increase when subclassing, @@ -362,8 +593,8 @@ through defined error return values. For functions that return a Python object ``NULL`` pointer, so any function returning a Python object has a well-defined error return value. -While this is always the case for :keyword:`def` functions, functions -defined as :keyword:`cdef` or :keyword:`cpdef` can return arbitrary C types, +While this is always the case for C functions, functions +defined as C functions or ``cpdef``/``@ccall`` functions can return arbitrary C types, which do not have such a well-defined error return value. Thus, if an exception is detected in such a function, a warning message is printed, the exception is ignored, and the function returns immediately without @@ -371,10 +602,25 @@ propagating the exception to its caller. If you want such a C function to be able to propagate exceptions, you need to declare an exception return value for it as a contract with the caller. -Here is an example:: +Here is an example - cdef int spam() except -1: - ... +.. tabs:: + + .. group-tab:: Pure Python + + .. code-block:: python + + @cython.cfunc + @cython.exceptval(-1) + def spam() -> cython.int: + ... + + .. group-tab:: Cython + + .. code-block:: cython + + cdef int spam() except -1: + ... With this declaration, whenever an exception occurs inside ``spam``, it will immediately return with the value ``-1``. From the caller's side, whenever @@ -392,20 +638,53 @@ returns small results. If all possible return values are legal and you can't reserve one entirely for signalling errors, you can use an alternative -form of exception value declaration:: +form of exception value declaration - cdef int spam() except? -1: - ... +.. tabs:: + + .. group-tab:: Pure Python + + .. code-block:: python + + @cython.cfunc + @cython.exceptval(-1, check=True) + def spam() -> cython.int: + ... + + The keyword argument ``check=True`` indicates that the value ``-1`` _may_ signal an error. -The "?" indicates that the value ``-1`` only signals a possible error. In this -case, Cython generates a call to :c:func:`PyErr_Occurred` if the exception value + .. group-tab:: Cython + + .. code-block:: cython + + cdef int spam() except? -1: + ... + + The ``?`` indicates that the value ``-1`` _may_ signal an error. + +In this case, Cython generates a call to :c:func:`PyErr_Occurred` if the exception value is returned, to make sure it really received an exception and not just a normal result. -There is also a third form of exception value declaration:: +There is also a third form of exception value declaration - cdef int spam() except *: - ... +.. tabs:: + + .. group-tab:: Pure Python + + .. code-block:: python + + @cython.cfunc + @cython.exceptval(check=True) + def spam() -> cython.int: + ... + + .. group-tab:: Cython + + .. code-block:: cython + + cdef int spam() except *: + ... This form causes Cython to generate a call to :c:func:`PyErr_Occurred` after *every* call to spam, regardless of what value it returns. If you have a @@ -418,6 +697,8 @@ An external C++ function that may raise an exception can be declared with:: cdef int spam() except + +.. note:: These declarations are not used in Python code, only in ``.pxd`` and ``.pyx`` files. + See :ref:`wrapping-cplusplus` for more details. Some things to note: @@ -425,7 +706,7 @@ Some things to note: * Exception values can only be declared for functions returning a C integer, enum, float or pointer type, and the value must be a constant expression. Functions that return ``void``, or a struct/union by value, can only use - the ``except *`` form. + the ``except *`` or ``exceptval(check=True)`` form. * The exception value specification is part of the signature of the function. If you're passing a pointer to a function as a parameter or assigning it to a variable, the declared type of the parameter or variable must have @@ -434,6 +715,8 @@ Some things to note: int (*grail)(int, char*) except -1 + .. note:: Pointers to functions are currently not supported by pure Python mode. (GitHub issue :issue:`4279`) + * You don't need to (and shouldn't) declare exception values for functions which return Python objects. Remember that a function with no declared return type implicitly returns a Python object. (Exceptions on such @@ -458,7 +741,15 @@ function or a C function that calls Python/C API routines. To get an exception from a non-Python-aware function such as :func:`fopen`, you will have to check the return value and raise it yourself, for example: -.. literalinclude:: ../../examples/userguide/language_basics/open_file.pyx +.. tabs:: + + .. group-tab:: Pure Python + + .. literalinclude:: ../../examples/userguide/language_basics/open_file.py + + .. group-tab:: Cython + + .. literalinclude:: ../../examples/userguide/language_basics/open_file.pyx .. _overriding_in_extension_types: @@ -466,15 +757,30 @@ Overriding in extension types ----------------------------- -``cpdef`` methods can override ``cdef`` methods: +``cpdef``/``@ccall`` methods can override C methods: + +.. tabs:: -.. literalinclude:: ../../examples/userguide/language_basics/optional_subclassing.pyx + .. group-tab:: Pure Python + + .. literalinclude:: ../../examples/userguide/language_basics/optional_subclassing.py + + .. group-tab:: Cython + + .. literalinclude:: ../../examples/userguide/language_basics/optional_subclassing.pyx When subclassing an extension type with a Python class, -``def`` methods can override ``cpdef`` methods but not ``cdef`` -methods: +Python methods can override ``cpdef``/``@ccall`` methods but not plain C methods: + +.. tabs:: + + .. group-tab:: Pure Python -.. literalinclude:: ../../examples/userguide/language_basics/override.pyx + .. literalinclude:: ../../examples/userguide/language_basics/override.py + + .. group-tab:: Cython + + .. literalinclude:: ../../examples/userguide/language_basics/override.pyx If ``C`` above would be an extension type (``cdef class``), this would not work correctly. @@ -536,13 +842,27 @@ as the C string is needed. If you can't guarantee that the Python string will live long enough, you will need to copy the C string. Cython detects and prevents some mistakes of this kind. For instance, if you -attempt something like:: +attempt something like + +.. tabs:: + + .. group-tab:: Pure Python + + .. code-block:: python + + def main(): + s: cython.p_char + s = pystring1 + pystring2 - cdef char *s - s = pystring1 + pystring2 + .. group-tab:: Cython -then Cython will produce the error message ``Obtaining char* from temporary -Python value``. The reason is that concatenating the two Python strings + .. code-block:: cython + + cdef char *s + s = pystring1 + pystring2 + +then Cython will produce the error message ``Storing unsafe C derivative of temporary +Python reference``. The reason is that concatenating the two Python strings produces a new Python string object that is referenced only by a temporary internal variable that Cython generates. As soon as the statement has finished, the temporary variable will be decrefed and the Python string deallocated, @@ -550,11 +870,26 @@ leaving ``s`` dangling. Since this code could not possibly work, Cython refuses compile it. The solution is to assign the result of the concatenation to a Python -variable, and then obtain the ``char*`` from that, i.e.:: +variable, and then obtain the ``char*`` from that, i.e. + +.. tabs:: + + .. group-tab:: Pure Python + + .. code-block:: python + + def main(): + s: cython.p_char + p = pystring1 + pystring2 + s = p - cdef char *s - p = pystring1 + pystring2 - s = p + .. group-tab:: Cython + + .. code-block:: cython + + cdef char *s + p = pystring1 + pystring2 + s = p It is then your responsibility to hold the reference p for as long as necessary. @@ -569,40 +904,93 @@ be careful what you do. Type Casting ------------ -Where C uses ``"("`` and ``")"``, Cython uses ``"<"`` and ``">"``. For example:: +The Cython language supports type casting in a similar way as C. Where C uses ``"("`` and ``")"``, +Cython uses ``"<"`` and ``">"``. In pure python mode, the ``cython.cast()`` function is used. For example: + +.. tabs:: + + .. group-tab:: Pure Python + + .. code-block:: python + + def main(): + p: cython.p_char + q: cython.p_float + p = cython.cast(cython.p_char, q) + + When casting a C value to a Python object type or vice versa, + Cython will attempt a coercion. Simple examples are casts like ``cast(int, pyobj_value)``, + which convert a Python number to a plain C ``int`` value, or the statement ``cast(bytes, charptr_value)``, + which copies a C ``char*`` string into a new Python bytes object. + + .. note:: Cython will not prevent a redundant cast, but emits a warning for it. + + To get the address of some Python object, use a cast to a pointer type + like ``cast(p_void, ...)`` or ``cast(pointer(PyObject), ...)``. + You can also cast a C pointer back to a Python object reference + with ``cast(object, ...)``, or to a more specific builtin or extension type + (e.g. ``cast(MyExtType, ptr)``). This will increase the reference count of + the object by one, i.e. the cast returns an owned reference. + Here is an example: + + + .. group-tab:: Cython - cdef char *p - cdef float *q - p = <char*>q + .. code-block:: cython -When casting a C value to a Python object type or vice versa, -Cython will attempt a coercion. Simple examples are casts like ``<int>pyobj``, -which converts a Python number to a plain C ``int`` value, or ``<bytes>charptr``, -which copies a C ``char*`` string into a new Python bytes object. + cdef char *p + cdef float *q + p = <char*>q - .. note:: Cython will not prevent a redundant cast, but emits a warning for it. + When casting a C value to a Python object type or vice versa, + Cython will attempt a coercion. Simple examples are casts like ``<int>pyobj_value``, + which convert a Python number to a plain C ``int`` value, or the statement ``<bytes>charptr_value``, + which copies a C ``char*`` string into a new Python bytes object. -To get the address of some Python object, use a cast to a pointer type -like ``<void*>`` or ``<PyObject*>``. -You can also cast a C pointer back to a Python object reference -with ``<object>``, or a more specific builtin or extension type -(e.g. ``<MyExtType>ptr``). This will increase the reference count of -the object by one, i.e. the cast returns an owned reference. -Here is an example: + .. note:: Cython will not prevent a redundant cast, but emits a warning for it. -.. literalinclude:: ../../examples/userguide/language_basics/casting_python.pyx + To get the address of some Python object, use a cast to a pointer type + like ``<void*>`` or ``<PyObject*>``. + You can also cast a C pointer back to a Python object reference + with ``<object>``, or to a more specific builtin or extension type + (e.g. ``<MyExtType>ptr``). This will increase the reference count of + the object by one, i.e. the cast returns an owned reference. + Here is an example: -The precedence of ``<...>`` is such that ``<type>a.b.c`` is interpreted as ``<type>(a.b.c)``. +.. tabs:: + + .. group-tab:: Pure Python + + .. literalinclude:: ../../examples/userguide/language_basics/casting_python.pxd + :caption: casting_python.pxd + .. literalinclude:: ../../examples/userguide/language_basics/casting_python.py + :caption: casting_python.py + + Casting with ``cast(object, ...)`` creates an owned reference. Cython will automatically + perform a ``Py_INCREF`` and ``Py_DECREF`` operation. Casting to + ``cast(pointer(PyObject), ...)`` creates a borrowed reference, leaving the refcount unchanged. + + .. group-tab:: Cython + + .. literalinclude:: ../../examples/userguide/language_basics/casting_python.pyx + :caption: casting_python.pyx + + The precedence of ``<...>`` is such that ``<type>a.b.c`` is interpreted as ``<type>(a.b.c)``. + + Casting to ``<object>`` creates an owned reference. Cython will automatically + perform a ``Py_INCREF`` and ``Py_DECREF`` operation. Casting to + ``<PyObject *>`` creates a borrowed reference, leaving the refcount unchanged. .. _checked_type_casts: Checked Type Casts ------------------ -A cast like ``<MyExtensionType>x`` will cast x to the class +A cast like ``<MyExtensionType>x`` or ``cast(MyExtensionType, x)`` will cast ``x`` to the class ``MyExtensionType`` without any checking at all. -To have a cast checked, use the syntax like: ``<MyExtensionType?>x``. +To have a cast checked, use ``<MyExtensionType?>x`` in Cython syntax +or ``cast(MyExtensionType, x, typecheck=True)``. In this case, Cython will apply a runtime check that raises a ``TypeError`` if ``x`` is not an instance of ``MyExtensionType``. This tests for the exact class for builtin types, @@ -634,17 +1022,37 @@ direct equivalent in Python. * An integer literal is treated as a C constant, and will be truncated to whatever size your C compiler thinks appropriate. - To get a Python integer (of arbitrary precision) cast immediately to - an object (e.g. ``<object>100000000000000000000``). The ``L``, ``LL``, - and ``U`` suffixes have the same meaning as in C. + To get a Python integer (of arbitrary precision), cast immediately to + an object (e.g. ``<object>100000000000000000000`` or ``cast(object, 100000000000000000000)``). The ``L``, ``LL``, + and ``U`` suffixes have the same meaning in Cython syntax as in C. * There is no ``->`` operator in Cython. Instead of ``p->x``, use ``p.x`` * There is no unary ``*`` operator in Cython. Instead of ``*p``, use ``p[0]`` -* There is an ``&`` operator, with the same semantics as in C. -* The null C pointer is called ``NULL``, not ``0`` (and ``NULL`` is a reserved word). -* Type casts are written ``<type>value`` , for example,:: +* There is an ``&`` operator in Cython, with the same semantics as in C. + In pure python mode, use the ``cython.address()`` function instead. +* The null C pointer is called ``NULL``, not ``0``. ``NULL`` is a reserved word in Cython + and special object in pure python mode. +* Type casts are written ``<type>value`` or ``cast(type, value)``, for example, + + .. tabs:: + + .. group-tab:: Pure Python + + .. code-block:: python + + def main(): + p: cython.p_char + q: cython.p_float - cdef char* p, float* q - p = <char*>q + p = cython.cast(cython.p_char, q) + + .. group-tab:: Cython + + .. code-block:: cython + + cdef char* p + cdef float* q + + p = <char*>q Scope rules ----------- @@ -726,6 +1134,8 @@ Python and C, and that Cython uses the Python precedences, not the C ones. Integer for-loops ------------------ +.. note:: This syntax is supported only in Cython files. Use a normal `for-in-range()` loop instead. + Cython recognises the usual Python for-in-range integer loop pattern:: for i in range(n): @@ -874,6 +1284,18 @@ Conditional Compilation Some features are available for conditional compilation and compile-time constants within a Cython source file. +.. note:: + + This feature has very little use cases. Specifically, is is not a good + way to adapt code to platform and environment. Use code generation or + (preferably) C compile time adaptation for this. See, for example, + :ref:`verbatim_c`. + +.. note:: + + Cython currently does not support conditional compilation and compile-time + definitions in Pure Python mode. As it stands, this is unlikely to change. + Compile-Time Definitions ------------------------ diff --git a/docs/src/userguide/limitations.rst b/docs/src/userguide/limitations.rst index 6128c308a..670f01d03 100644 --- a/docs/src/userguide/limitations.rst +++ b/docs/src/userguide/limitations.rst @@ -8,9 +8,12 @@ Limitations This page used to list bugs in Cython that made the semantics of compiled code differ from that in Python. Most of the missing -features have been fixed in Cython 0.15. Note that a -future version 1.0 of Cython is planned to provide full Python -language compatibility. +features have been fixed in Cython 0.15. A future version of +Cython is planned to provide full Python language compatibility. +For now, the issue tracker can provide an overview of deviations +that we are aware of and would like to see fixed. + +https://github.com/cython/cython/labels/Python%20Semantics Below is a list of differences that we will probably not be addressing. Most of these things that fall more into the implementation details rather diff --git a/docs/src/userguide/memoryviews.rst b/docs/src/userguide/memoryviews.rst index 328831e86..ce83a3005 100644 --- a/docs/src/userguide/memoryviews.rst +++ b/docs/src/userguide/memoryviews.rst @@ -20,6 +20,9 @@ A memoryview can be used in any context (function parameters, module-level, cdef class attribute, etc) and can be obtained from nearly any object that exposes writable buffer through the `PEP 3118`_ buffer interface. +.. _`PEP 3118`: https://www.python.org/dev/peps/pep-3118/ + + .. _view_quickstart: Quickstart @@ -56,16 +59,11 @@ A complete 3D view:: cdef int[:,:,:] view3D = exporting_object -A 2D view that restricts the first dimension of a buffer to 100 rows -starting at the second (index 1) and then skips every second (odd) row:: - - cdef int[1:102:2,:] partial_view = exporting_object - -This also works conveniently as function arguments: +They also work conveniently as function arguments: .. code-block:: cython - def process_3d_buffer(int[1:102:2,:] view not None): + def process_3d_buffer(int[:,:,:] view not None): ... The ``not None`` declaration for the argument automatically rejects @@ -234,6 +232,8 @@ NumPy arrays support this interface, as do :ref:`view_cython_arrays`. The data array to themselves be pointers; Cython memoryviews do not yet support this. +.. _`new style buffers`: https://docs.python.org/3/c-api/buffer.html + .. _view_memory_layout: Memory layout @@ -660,6 +660,6 @@ object handling. For the details of how to compile and call functions in C files, see :ref:`using_c_libraries`. -.. _GIL: http://docs.python.org/dev/glossary.html#term-global-interpreter-lock +.. _GIL: https://docs.python.org/dev/glossary.html#term-global-interpreter-lock .. _NumPy: https://docs.scipy.org/doc/numpy/reference/arrays.ndarray.html#memory-layout .. _example: https://docs.scipy.org/doc/numpy/reference/arrays.indexing.html diff --git a/docs/src/userguide/migrating_to_cy30.rst b/docs/src/userguide/migrating_to_cy30.rst new file mode 100644 index 000000000..1dd29a9a2 --- /dev/null +++ b/docs/src/userguide/migrating_to_cy30.rst @@ -0,0 +1,159 @@ +.. highlight:: cython + +.. _cython30: + +********************************* +Migrating from Cython 0.29 to 3.0 +********************************* + +Cython 3.0 is a major revision of the compiler and the language +that comes with some backwards incompatible changes. +This document lists the important ones and explains how to deal with +them in existing code. + + +Python 3 syntax/semantics +========================= + +Cython 3.0 now uses Python 3 syntax and semantics by default, which previously +required setting the ``language_level`` `directive <compiler-directives>` to +either ``3`` or ``3str``. +The new default setting is now ``language_level=3str``, which means Python 3 +semantics, but unprefixed strings are ``str`` objects, i.e. unicode text strings +under Python 3 and byte strings under Python 2.7. + +You can revert your code to the previous (Python 2.x) semantics by setting +``language_level=2``. + +Further semantic changes due to the language level include: + +* ``/``-division uses the true (float) division operator, unless ``cdivision`` is enabled. +* ``print`` is a function, not a statement. +* Python classes that are defined without bases (``class C: ...``) are "new-style" + classes also in Py2.x (if you never heard about "old-style classes", you're probably + happy without them). +* Annotations (type hints) are now stored as strings. + (`PEP 563 <https://github.com/cython/cython/issues/2863>`_) +* ``StopIteration`` handling in generators has been changed according to + `PEP 479 <https://www.python.org/dev/peps/pep-0479/>`_. + + +Python semantics +================ + +Some Python compatibility bugs were fixed, e.g. + +* Subscripting (``x[1]``) now tries the mapping protocol before the sequence protocol. + (https://github.com/cython/cython/issues/1807) +* Exponentiation of integer literals now follows Python semantics and not C semantics. + (https://github.com/cython/cython/issues/2133) + + +Binding functions +================= + +The :ref:`binding directive <compiler-directives>` is now enabled by default. +This makes Cython compiled Python (``def``) functions mostly compatible +with normal (non-compiled) Python functions, regarding signature introspection, +annotations, etc. + +It also makes them bind as methods in Python classes on attribute assignments, +thus the name. +If this is not intended, i.e. if a function is really meant to be a function +and never a method, you can disable the binding (and all other Python function +features) by setting ``binding=False`` or selectively adding a decorator +``@cython.binding(False)``. +In pure Python mode, the decorator was not available in Cython 0.29.16 yet, +but compiled code does not suffer from this. + +We recommend, however, to keep the new function features and instead deal +with the binding issue using the standard Python ``staticmethod()`` builtin. + +:: + + def func(self, b): ... + + class MyClass(object): + binding_method = func + + no_method = staticmethod(func) + + +Namespace packages +================== + +Cython now has support for loading pxd files also from namespace packages +according to `PEP-420 <https://www.python.org/dev/peps/pep-0420/>`_. +This might have an impact on the import path. + + +NumPy C-API +=========== + +Cython used to generate code that depended on the deprecated pre-NumPy-1.7 C-API. +This is no longer the case with Cython 3.0. + +You can now define the macro ``NPY_NO_DEPRECATED_API=NPY_1_7_API_VERSION`` +to get rid of the long-standing build warnings that the compiled C module +uses a deprecated API. Either per file:: + + # distutils: define_macros=NPY_NO_DEPRECATED_API=NPY_1_7_API_VERSION + +or by setting it in your Extensions in ``setup.py``:: + + Extension(... + define_macros=[("NPY_NO_DEPRECATED_API", "NPY_1_7_API_VERSION")] + ) + +One side-effect of the different C-API usage is that your code may now +require a call to the `NumPy C-API initialisation function +<https://docs.scipy.org/doc/numpy-1.17.0/reference/c-api.array.html#importing-the-api>`_ +where it previously got away without doing so. + +In order to reduce the user impact here, Cython 3.0 will now call it +automatically when it sees ``numpy`` being cimported, but the function +not being used. +In the (hopefully rare) cases where this gets in the way, the internal +C-API initialisation can be disabled by faking the use of the function +without actually calling it, e.g. + +:: + + # Explicitly disable the automatic initialisation of NumPy's C-API. + <void>import_array + +Class-private name mangling +=========================== + +Cython has been updated to follow the `Python rules for class-private names +<https://docs.python.org/3/tutorial/classes.html#private-variables>`_ +more closely. Essentially any name that starts with and doesn't end with +``__`` within a class is mangled with the class name. Most user code +should be unaffected -- unlike in Python unmangled global names will +still be matched to ensure it is possible to access C names +beginning with ``__``:: + + cdef extern void __foo() + + class C: # or "cdef class" + def call_foo(self): + return __foo() # still calls the global name + +What will no-longer work is overriding methods starting with ``__`` in +a ``cdef class``:: + + cdef class Base: + cdef __bar(self): + return 1 + + def call_bar(self): + return self.__bar() + + cdef class Derived(Base): + cdef __bar(self): + return 2 + +Here ``Base.__bar`` is mangled to ``_Base__bar`` and ``Derived.__bar`` +to ``_Derived__bar``. Therefore ``call_bar`` will always call +``_Base__bar``. This matches established Python behaviour and applies +for ``def``, ``cdef`` and ``cpdef`` methods and attributes. diff --git a/docs/src/userguide/numpy_pythran.rst b/docs/src/userguide/numpy_pythran.rst index 185f7c654..cb075d729 100644 --- a/docs/src/userguide/numpy_pythran.rst +++ b/docs/src/userguide/numpy_pythran.rst @@ -16,22 +16,22 @@ This can lead to really interesting speedup in some cases, going from 2 up to Please note that this feature is experimental. -Usage example with distutils ----------------------------- +Usage example with setuptools +----------------------------- You first need to install Pythran. See its `documentation -<http://pythran.readthedocs.io/en/latest/>`_ for more information. +<https://pythran.readthedocs.io/>`_ for more information. Then, simply add a ``cython: np_pythran=True`` directive at the top of the Python files that needs to be compiled using Pythran numpy support. -Here is an example of a simple ``setup.py`` file using distutils: +Here is an example of a simple ``setup.py`` file using setuptools: .. code:: - from distutils.core import setup + from setuptools import setup from Cython.Build import cythonize - + setup( name = "My hello app", ext_modules = cythonize('hello_pythran.pyx') diff --git a/docs/src/userguide/numpy_tutorial.rst b/docs/src/userguide/numpy_tutorial.rst index 3d1cd5a74..b74c41509 100644 --- a/docs/src/userguide/numpy_tutorial.rst +++ b/docs/src/userguide/numpy_tutorial.rst @@ -31,7 +31,7 @@ Cython at a glance Cython is a compiler which compiles Python-like code files to C code. Still, ''Cython is not a Python to C translator''. That is, it doesn't take your full -program and "turns it into C" -- rather, the result makes full use of the +program and "turn it into C" -- rather, the result makes full use of the Python runtime environment. A way of looking at it may be that your code is still Python in that it runs within the Python runtime environment, but rather than compiling to interpreted Python bytecode one compiles to native machine @@ -61,11 +61,11 @@ Using Cython consists of these steps: However there are several options to automate these steps: -1. The `SAGE <http://sagemath.org>`_ mathematics software system provides +1. The `SAGE <https://sagemath.org>`_ mathematics software system provides excellent support for using Cython and NumPy from an interactive command line or through a notebook interface (like Maple/Mathematica). See `this documentation - <http://doc.sagemath.org/html/en/developer/coding_in_cython.html>`_. + <https://doc.sagemath.org/html/en/developer/coding_in_cython.html>`_. 2. Cython can be used as an extension within a Jupyter notebook, making it easy to compile and use Cython code with just a ``%%cython`` at the top of a cell. For more information see @@ -73,7 +73,7 @@ However there are several options to automate these steps: 3. A version of pyximport is shipped with Cython, so that you can import pyx-files dynamically into Python and have them compiled automatically (See :ref:`pyximport`). -4. Cython supports distutils so that you can very easily create build scripts +4. Cython supports setuptools so that you can very easily create build scripts which automate the process, this is the preferred method for Cython implemented libraries and packages. See :ref:`Basic setup.py <basic_setup.py>`. @@ -88,7 +88,9 @@ However there are several options to automate these steps: Installation ============= -If you already have a C compiler, just do:: +If you already have a C compiler, just do: + +.. code-block:: bash pip install Cython @@ -97,7 +99,9 @@ otherwise, see :ref:`the installation page <install>`. As of this writing SAGE comes with an older release of Cython than required for this tutorial. So if using SAGE you should download the newest Cython and -then execute :: +then execute : + +.. code-block:: bash $ cd path/to/cython-distro $ path-to-sage/sage -python setup.py install @@ -108,7 +112,9 @@ Manual compilation ==================== As it is always important to know what is going on, I'll describe the manual -method here. First Cython is run:: +method here. First Cython is run: + +.. code-block:: bash $ cython yourmod.pyx @@ -120,7 +126,9 @@ line by line. Then we compile the C file. This may vary according to your system, but the C file should be built like Python was built. Python documentation for writing extensions should have some details. On Linux this often means something -like:: +like: + +.. code-block:: bash $ gcc -shared -pthread -fPIC -fwrapv -O2 -Wall -fno-strict-aliasing -I/usr/include/python2.7 -o yourmod.so yourmod.c @@ -166,7 +174,7 @@ This should be compiled to produce :file:`compute_cy.so` for Linux systems run a Python session to test both the Python version (imported from ``.py``-file) and the compiled Cython module. -.. sourcecode:: ipython +.. code-block:: ipythonconsole In [1]: import numpy as np In [2]: array_1 = np.random.uniform(0, 1000, size=(3000, 2000)).astype(np.intc) @@ -218,7 +226,7 @@ of C code to set up while in :file:`compute_typed.c` a normal C for loop is used After building this and continuing my (very informal) benchmarks, I get: -.. sourcecode:: ipython +.. code-block:: ipythonconsole In [13]: %timeit compute_typed.compute(array_1, array_2, a, b, c) 26.5 s ± 422 ms per loop (mean ± std. dev. of 7 runs, 1 loop each) @@ -287,7 +295,7 @@ Here is how to use them in our code: Let's see how much faster accessing is now. -.. sourcecode:: ipython +.. code-block:: ipythonconsole In [22]: %timeit compute_memview.compute(array_1, array_2, a, b, c) 22.9 ms ± 197 µs per loop (mean ± std. dev. of 7 runs, 10 loops each) @@ -326,7 +334,7 @@ mode in many ways, see :ref:`compiler-directives` for more information. -.. sourcecode:: ipython +.. code-block:: ipythonconsole In [23]: %timeit compute_index.compute(array_1, array_2, a, b, c) 16.8 ms ± 25.4 µs per loop (mean ± std. dev. of 7 runs, 100 loops each) @@ -376,7 +384,7 @@ all about, you can see `this answer on StackOverflow For the sake of giving numbers, here are the speed gains that you should get by declaring the memoryviews as contiguous: -.. sourcecode:: ipython +.. code-block:: ipythonconsole In [23]: %timeit compute_contiguous.compute(array_1, array_2, a, b, c) 11.1 ms ± 30.2 µs per loop (mean ± std. dev. of 7 runs, 100 loops each) @@ -405,7 +413,7 @@ be useful when using fused types. We now do a speed test: -.. sourcecode:: ipython +.. code-block:: ipythonconsole In [24]: %timeit compute_infer_types.compute(array_1, array_2, a, b, c) 11.5 ms ± 261 µs per loop (mean ± std. dev. of 7 runs, 100 loops each) @@ -439,14 +447,14 @@ In this case, our function now works for ints, doubles and floats. We can check that the output type is the right one:: - >>>compute(array_1, array_2, a, b, c).dtype + >>> compute(array_1, array_2, a, b, c).dtype dtype('int32') - >>>compute(array_1.astype(np.double), array_2.astype(np.double), a, b, c).dtype + >>> compute(array_1.astype(np.double), array_2.astype(np.double), a, b, c).dtype dtype('float64') We now do a speed test: -.. sourcecode:: ipython +.. code-block:: ipythonconsole In [25]: %timeit compute_fused_types.compute(array_1, array_2, a, b, c) 11.5 ms ± 258 µs per loop (mean ± std. dev. of 7 runs, 100 loops each) @@ -463,7 +471,9 @@ like the function :func:`prange`. You can see more information about Cython and parallelism in :ref:`parallel`. Since we do elementwise operations, we can easily distribute the work among multiple threads. It's important not to forget to pass the correct arguments to the compiler to enable OpenMP. When using the Jupyter notebook, -you should use the cell magic like this:: +you should use the cell magic like this: + +.. code-block:: ipython %%cython --force # distutils: extra_compile_args=-fopenmp @@ -476,7 +486,7 @@ declare our :func:`clip` function ``nogil``. We can have substantial speed gains for minimal effort: -.. sourcecode:: ipython +.. code-block:: ipythonconsole In [25]: %timeit compute_prange.compute(array_1, array_2, a, b, c) 9.33 ms ± 412 µs per loop (mean ± std. dev. of 7 runs, 100 loops each) @@ -487,8 +497,8 @@ than NumPy! Where to go from here? ====================== -* If you want to learn how to make use of `BLAS <http://www.netlib.org/blas/>`_ - or `LAPACK <http://www.netlib.org/lapack/>`_ with Cython, you can watch +* If you want to learn how to make use of `BLAS <https://www.netlib.org/blas/>`_ + or `LAPACK <https://www.netlib.org/lapack/>`_ with Cython, you can watch `the presentation of Ian Henriksen at SciPy 2015 <https://www.youtube.com/watch?v=R4yB-8tB0J0&t=693s&ab_channel=Enthought>`_. * If you want to learn how to use Pythran as backend in Cython, you diff --git a/docs/src/userguide/parallelism.rst b/docs/src/userguide/parallelism.rst index ad97885e1..e9d473e66 100644 --- a/docs/src/userguide/parallelism.rst +++ b/docs/src/userguide/parallelism.rst @@ -118,7 +118,7 @@ Example with a reduction: .. literalinclude:: ../../examples/userguide/parallelism/simple_sum.pyx -Example with a typed memoryview (e.g. a NumPy array):: +Example with a :term:`typed memoryview<Typed memoryview>` (e.g. a NumPy array):: from cython.parallel import prange diff --git a/docs/src/userguide/pypy.rst b/docs/src/userguide/pypy.rst index cf1fbb24d..893277bda 100644 --- a/docs/src/userguide/pypy.rst +++ b/docs/src/userguide/pypy.rst @@ -2,7 +2,7 @@ Porting Cython code to PyPy =========================== Cython has basic support for cpyext, the layer in -`PyPy <http://pypy.org/>`_ that emulates CPython's C-API. This is +`PyPy <https://pypy.org/>`_ that emulates CPython's C-API. This is achieved by making the generated C code adapt at C compile time, so the generated code will compile in both CPython and PyPy unchanged. diff --git a/docs/src/userguide/pyrex_differences.rst b/docs/src/userguide/pyrex_differences.rst index 8a958ec9a..aa2eb75da 100644 --- a/docs/src/userguide/pyrex_differences.rst +++ b/docs/src/userguide/pyrex_differences.rst @@ -310,7 +310,7 @@ Automatic ``typecheck`` Rather than introducing a new keyword ``typecheck`` as explained in the `Pyrex docs -<http://www.cosc.canterbury.ac.nz/greg.ewing/python/Pyrex/version/Doc/Manual/special_methods.html>`_, +<https://www.cosc.canterbury.ac.nz/greg.ewing/python/Pyrex/version/Doc/Manual/special_methods.html>`_, Cython emits a (non-spoofable and faster) typecheck whenever :func:`isinstance` is used with an extension type as the second parameter. diff --git a/docs/src/userguide/sharing_declarations.rst b/docs/src/userguide/sharing_declarations.rst index 57f41e38d..70e29e2b2 100644 --- a/docs/src/userguide/sharing_declarations.rst +++ b/docs/src/userguide/sharing_declarations.rst @@ -226,3 +226,24 @@ Some things to note about this example: ``egg`` file which will not work with ``cimport`` for ``pxd`` files when you try to use them from a dependent package. To prevent this, include ``zip_safe=False`` in the arguments to ``setup()``. + +.. _versioning: + +Versioning +========== + +``.pxd`` files can be labelled with a minimum Cython version as part of +their file name, similar to the version tagging of ``.so`` files in PEP 3149. +For example a file called :file:`Shrubbing.cython-30.pxd` will only be +found by ``cimport Shrubbing`` on Cython 3.0 and higher. Cython will use the +file tagged with the highest compatible version number. + +Note that versioned files that are distributed across different directories +will not be found. Only the first directory in the Python module search +path in which a matching ``.pxd`` file is found will be considered. + +The purpose of this feature is to allow third-party packages to release +Cython interfaces to their packages that take advantage of the latest Cython +features while not breaking compatibility for users with older versions of Cython. +Users intending to use ``.pxd`` files solely within their own project +need not produce these tagged files. diff --git a/docs/src/userguide/source_files_and_compilation.rst b/docs/src/userguide/source_files_and_compilation.rst index c4f01806d..edc5b1d22 100644 --- a/docs/src/userguide/source_files_and_compilation.rst +++ b/docs/src/userguide/source_files_and_compilation.rst @@ -44,7 +44,7 @@ Compiling with the ``cython`` command One way is to compile it manually with the Cython compiler, e.g.: -.. sourcecode:: text +.. code-block:: text $ cython primes.pyx @@ -53,7 +53,7 @@ compiled with the C compiler using whatever options are appropriate on your platform for generating an extension module. For these options look at the official Python documentation. -The other, and probably better, way is to use the :mod:`distutils` extension +The other, and probably better, way is to use the :mod:`setuptools` extension provided with Cython. The benefit of this method is that it will give the platform specific compilation options, acting like a stripped down autotools. @@ -62,7 +62,9 @@ Compiling with the ``cythonize`` command ---------------------------------------- Run the ``cythonize`` compiler command with your options and list of -``.pyx`` files to generate an extension module. For example:: +``.pyx`` files to generate an extension module. For example: + +.. code-block:: bash $ cythonize -a -i yourmod.pyx @@ -82,7 +84,9 @@ There simpler command line tool ``cython`` only invokes the source code translat In the case of manual compilation, how to compile your ``.c`` files will vary depending on your operating system and compiler. The Python documentation for writing extension modules should have some details for your system. On a Linux -system, for example, it might look similar to this:: +system, for example, it might look similar to this: + +.. code-block:: bash $ gcc -shared -pthread -fPIC -fwrapv -O2 -Wall -fno-strict-aliasing \ -I/usr/include/python3.5 -o yourmod.so yourmod.c @@ -93,7 +97,7 @@ to libraries you want to link with.) After compilation, a ``yourmod.so`` (:file:`yourmod.pyd` for Windows) file is written into the target directory and your module, ``yourmod``, is available for you to import as with any other -Python module. Note that if you are not relying on ``cythonize`` or distutils, +Python module. Note that if you are not relying on ``cythonize`` or setuptools, you will not automatically benefit from the platform specific file extension that CPython generates for disambiguation, such as ``yourmod.cpython-35m-x86_64-linux-gnu.so`` on a regular 64bit Linux installation @@ -103,25 +107,36 @@ of CPython 3.5. Basic setup.py =============== -The distutils extension provided with Cython allows you to pass ``.pyx`` files +The setuptools extension provided with Cython allows you to pass ``.pyx`` files directly to the ``Extension`` constructor in your setup file. If you have a single Cython file that you want to turn into a compiled extension, say with filename :file:`example.pyx` the associated :file:`setup.py` would be:: - from distutils.core import setup + from setuptools import setup from Cython.Build import cythonize setup( ext_modules = cythonize("example.pyx") ) -To understand the :file:`setup.py` more fully look at the official -:mod:`distutils` documentation. To compile the extension for use in the -current directory use: +If your build depends directly on Cython in this way, +then you may also want to inform pip that :mod:`Cython` is required for +:file:`setup.py` to execute, following `PEP 518 +<https://www.python.org/dev/peps/pep-0518/>`, creating a :file:`pyproject.toml` +file containing, at least: + +.. code-block:: ini + + + [build-system] + requires = ["setuptools", "wheel", "Cython"] + +To understand the :file:`setup.py` more fully look at the official `setuptools +documentation`_. To compile the extension for use in the current directory use: -.. sourcecode:: text +.. code-block:: text $ python setup.py build_ext --inplace @@ -131,7 +146,7 @@ Configuring the C-Build If you have include files in non-standard places you can pass an ``include_path`` parameter to ``cythonize``:: - from distutils.core import setup + from setuptools import setup from Cython.Build import cythonize setup( @@ -150,20 +165,38 @@ the necessary include files, e.g. for NumPy:: you have to add the path to NumPy include files. You need to add this path only if you use ``cimport numpy``. -Despite this, you will still get warnings like the -following from the compiler, because Cython is using a deprecated Numpy API:: +Despite this, you may still get warnings like the following from the compiler, +because Cython is not disabling the usage of the old deprecated Numpy API:: .../include/numpy/npy_1_7_deprecated_api.h:15:2: warning: #warning "Using deprecated NumPy API, disable it by " "#defining NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION" [-Wcpp] -For the time being, it is just a warning that you can ignore. +In Cython 3.0, you can get rid of this warning by defining the C macro +``NPY_NO_DEPRECATED_API`` as ``NPY_1_7_API_VERSION`` +in your build, e.g.:: + + # distutils: define_macros=NPY_NO_DEPRECATED_API=NPY_1_7_API_VERSION + +or (see below):: + + Extension( + ..., + define_macros=[("NPY_NO_DEPRECATED_API", "NPY_1_7_API_VERSION")], + ) + +With older Cython releases, setting this macro will fail the C compilation, +because Cython generates code that uses this deprecated C-API. However, the +warning has no negative effects even in recent NumPy versions including 1.18.x. +You can ignore it until you (or your library's users) switch to a newer NumPy +version that removes this long deprecated API, in which case you also need to +use Cython 3.0 or later. Thus, the earlier you switch to Cython 3.0, the +better for your users. If you need to specify compiler options, libraries to link with or other linker options you will need to create ``Extension`` instances manually (note that glob syntax can still be used to specify multiple extensions in one line):: - from distutils.core import setup - from distutils.extension import Extension + from setuptools import Extension, setup from Cython.Build import cythonize extensions = [ @@ -182,11 +215,10 @@ in one line):: ext_modules=cythonize(extensions), ) -Note that when using setuptools, you should import it before Cython as -setuptools may replace the ``Extension`` class in distutils. Otherwise, +Note that when using setuptools, you should import it before Cython, otherwise, both might disagree about the class to use here. -Note also that if you use setuptools instead of distutils, the default +Note also that if you use setuptools instead of :mod:`distutils`, the default action when running ``python setup.py install`` is to create a zipped ``egg`` file which will not work with ``cimport`` for ``pxd`` files when you try to use them from a dependent package. @@ -204,7 +236,7 @@ merges the list of libraries, so this works as expected (similarly with other options, like ``include_dirs`` above). If you have some C files that have been wrapped with Cython and you want to -compile them into your extension, you can define the distutils ``sources`` +compile them into your extension, you can define the setuptools ``sources`` parameter:: # distutils: sources = helper.c, another_helper.c @@ -213,9 +245,8 @@ Note that these sources are added to the list of sources of the current extension module. Spelling this out in the :file:`setup.py` file looks as follows:: - from distutils.core import setup + from setuptools import Extension, setup from Cython.Build import cythonize - from distutils.extension import Extension sourcefiles = ['example.pyx', 'helper.c', 'another_helper.c'] @@ -226,14 +257,14 @@ as follows:: ) The :class:`Extension` class takes many options, and a fuller explanation can -be found in the `distutils documentation`_. Some useful options to know about +be found in the `setuptools documentation`_. Some useful options to know about are ``include_dirs``, ``libraries``, and ``library_dirs`` which specify where to find the ``.h`` and library files when linking to external libraries. -.. _distutils documentation: https://docs.python.org/extending/building.html +.. _setuptools documentation: https://setuptools.readthedocs.io/ Sometimes this is not enough and you need finer customization of the -distutils :class:`Extension`. +setuptools :class:`Extension`. To do this, you can provide a custom function ``create_extension`` to create the final :class:`Extension` object after Cython has processed the sources, dependencies and ``# distutils`` directives but before the @@ -329,11 +360,10 @@ doesn't want to use it just to install your module. Also, the installed version may not be the same one you used, and may not compile your sources correctly. This simply means that the :file:`setup.py` file that you ship with will just -be a normal distutils file on the generated `.c` files, for the basic example +be a normal setuptools file on the generated `.c` files, for the basic example we would have instead:: - from distutils.core import setup - from distutils.extension import Extension + from setuptools import Extension, setup setup( ext_modules = [Extension("example", ["example.c"])] @@ -342,8 +372,7 @@ we would have instead:: This is easy to combine with :func:`cythonize` by changing the file extension of the extension module sources:: - from distutils.core import setup - from distutils.extension import Extension + from setuptools import Extension, setup USE_CYTHON = ... # command line option, try-import, ... @@ -385,14 +414,17 @@ Another option is to make Cython a setup dependency of your system and use Cython's build_ext module which runs ``cythonize`` as part of the build process:: setup( - setup_requires=[ - 'cython>=0.x', - ], extensions = [Extension("*", ["*.pyx"])], - cmdclass={'build_ext': Cython.Build.build_ext}, + cmdclass={'build_ext': Cython.Build.new_build_ext}, ... ) +This depends on pip knowing that :mod:`Cython` is a setup dependency, by having +a :file:`pyproject.toml` file:: + + [build-system] + requires = ["setuptools", "wheel", "Cython"] + If you want to expose the C-level interface of your library for other libraries to cimport from, use package_data to install the ``.pxd`` files, e.g.:: @@ -522,7 +554,7 @@ glob must be on a separate line. Pyximport will check the file date for each of those files before deciding whether to rebuild the module. In order to keep track of the fact that the dependency has been handled, Pyximport updates the modification time of your ".pyx" source file. Future versions may do -something more sophisticated like informing distutils of the dependencies +something more sophisticated like informing setuptools of the dependencies directly. @@ -538,7 +570,7 @@ compiled. Usually the defaults are fine. You might run into problems if you wanted to write your program in half-C, half-Cython and build them into a single library. -Pyximport does not hide the Distutils/GCC warnings and errors generated +Pyximport does not hide the setuptools/GCC warnings and errors generated by the import process. Arguably this will give you better feedback if something went wrong and why. And if nothing went wrong it will give you the warm fuzzy feeling that pyximport really did rebuild your module as it @@ -582,7 +614,7 @@ The Sage notebook allows transparently editing and compiling Cython code simply by typing ``%cython`` at the top of a cell and evaluate it. Variables and functions defined in a Cython cell are imported into the running session. Please check `Sage documentation -<http://www.sagemath.org/doc/>`_ for details. +<https://www.sagemath.org/doc/>`_ for details. You can tailor the behavior of the Cython compiler by specifying the directives below. @@ -623,6 +655,8 @@ You can see them also by typing ```%%cython?`` in IPython or a Jupyter notebook. -a, --annotate Produce a colorized HTML version of the source. +--annotate-fullc Produce a colorized HTML version of the source which includes entire generated C/C++-code. + -+, --cplus Output a C++ rather than C file. -f, --force Force the compilation of a new module, even if the source has been previously compiled. @@ -648,6 +682,7 @@ You can see them also by typing ```%%cython?`` in IPython or a Jupyter notebook. --pgo Enable profile guided optimisation in the C compiler. Compiles the cell twice and executes it in between to generate a runtime profile. --verbose Print debug information like generated .c/.cpp file location and exact gcc/g++ command invoked. + ============================================ ======================================================================================================================================= @@ -659,7 +694,7 @@ Compiler options Compiler options can be set in the :file:`setup.py`, before calling :func:`cythonize`, like this:: - from distutils.core import setup + from setuptools import setup from Cython.Build import cythonize from Cython.Compiler import Options @@ -675,7 +710,6 @@ Here are the options that are available: .. autodata:: Cython.Compiler.Options.docstrings .. autodata:: Cython.Compiler.Options.embed_pos_in_docstring -.. autodata:: Cython.Compiler.Options.emit_code_comments .. pre_import .. autodata:: Cython.Compiler.Options.generate_cleanup_code .. autodata:: Cython.Compiler.Options.clear_to_none @@ -711,7 +745,11 @@ Cython code. Here is the list of currently supported directives: class attribute (hence the name) and will emulate the attributes of Python functions, including introspections like argument names and annotations. - Default is False. + + Default is True. + + .. versionchanged:: 3.0.0 + Default changed from False to True ``boundscheck`` (True / False) If set to False, Cython is free to assume that indexing operations @@ -740,9 +778,12 @@ Cython code. Here is the list of currently supported directives: Default is True. ``initializedcheck`` (True / False) - If set to True, Cython checks that a memoryview is initialized - whenever its elements are accessed or assigned to. Setting this - to False disables these checks. + If set to True, Cython checks that + - a memoryview is initialized whenever its elements are accessed + or assigned to. + - a C++ class is initialized when it is accessed + (only when ``cpp_locals`` is on) + Setting this to False disables these checks. Default is True. ``nonecheck`` (True / False) @@ -788,12 +829,19 @@ Cython code. Here is the list of currently supported directives: False. ``always_allow_keywords`` (True / False) - Avoid the ``METH_NOARGS`` and ``METH_O`` when constructing - functions/methods which take zero or one arguments. Has no effect - on special methods and functions with more than one argument. The - ``METH_NOARGS`` and ``METH_O`` signatures provide faster + When disabled, uses the ``METH_NOARGS`` and ``METH_O`` signatures when + constructing functions/methods which take zero or one arguments. Has no + effect on special methods and functions with more than one argument. The + ``METH_NOARGS`` and ``METH_O`` signatures provide slightly faster calling conventions but disallow the use of keywords. +``c_api_binop_methods`` (True / False) + When enabled, makes the special binary operator methods (``__add__``, etc.) + behave according to the low-level C-API slot semantics, i.e. only a single + method implements both the normal and reversed operator. This used to be + the default in Cython 0.x and was now replaced by Python semantics, i.e. the + default in Cython 3.x and later is ``False``. + ``profile`` (True / False) Write hooks for Python profilers into the compiled C code. Default is False. @@ -803,7 +851,7 @@ Cython code. Here is the list of currently supported directives: into the compiled C code. This also enables profiling. Default is False. Note that the generated module will not actually use line tracing, unless you additionally pass the C macro definition - ``CYTHON_TRACE=1`` to the C compiler (e.g. using the distutils option + ``CYTHON_TRACE=1`` to the C compiler (e.g. using the setuptools option ``define_macros``). Define ``CYTHON_TRACE_NOGIL=1`` to also include ``nogil`` functions and sections. @@ -856,7 +904,22 @@ Cython code. Here is the list of currently supported directives: asyncio before Python 3.5. This directive can be applied in modules or selectively as decorator on an async-def coroutine to make the affected coroutine(s) iterable and thus directly interoperable with yield-from. - + +``annotation_typing`` (True / False) + Uses function argument annotations to determine the type of variables. Default + is True, but can be disabled. Since Python does not enforce types given in + annotations, setting to False gives greater compatibility with Python code. + Must be set globally. + +``emit_code_comments`` (True / False) + Copy the original source code line by line into C code comments in the generated + code file to help with understanding the output. + This is also required for coverage analysis. + +``cpp_locals`` (True / False) + Make C++ variables behave more like Python variables by allowing them to be + "unbound" instead of always default-constructing them at the start of a + function. See :ref:`cpp_locals directive` for more detail. .. _configurable_optimisations: @@ -927,7 +990,9 @@ One can set compiler directives through a special header comment near the top of The comment must appear before any code (but can appear after other comments or whitespace). -One can also pass a directive on the command line by using the -X switch:: +One can also pass a directive on the command line by using the -X switch: + +.. code-block:: bash $ cython -X boundscheck=True ... @@ -964,7 +1029,7 @@ In :file:`setup.py` Compiler directives can also be set in the :file:`setup.py` file by passing a keyword argument to ``cythonize``:: - from distutils.core import setup + from setuptools import setup from Cython.Build import cythonize setup( diff --git a/docs/src/userguide/special_methods.rst b/docs/src/userguide/special_methods.rst index 8bb4d9be4..42ce93498 100644 --- a/docs/src/userguide/special_methods.rst +++ b/docs/src/userguide/special_methods.rst @@ -12,7 +12,7 @@ mention. .. Note:: Everything said on this page applies only to extension types, defined - with the :keyword:`cdef class` statement. It doesn't apply to classes defined with the + with the :keyword:`cdef` class statement. It doesn't apply to classes defined with the Python :keyword:`class` statement, where the normal Python rules apply. .. _declaration: @@ -43,9 +43,11 @@ There are two methods concerned with initialising the object. The :meth:`__cinit__` method is where you should perform basic C-level initialisation of the object, including allocation of any C data structures that your object will own. You need to be careful what you do in the -:meth:`__cinit__` method, because the object may not yet be fully valid Python +:meth:`__cinit__` method, because the object may not yet be a fully valid Python object when it is called. Therefore, you should be careful invoking any Python -operations which might touch the object; in particular, its methods. +operations which might touch the object; in particular, its methods and anything +that could be overridden by subtypes (and thus depend on their subtype state being +initialised already). By the time your :meth:`__cinit__` method is called, memory has been allocated for the object and any C attributes it has have been initialised to 0 or null. (Any @@ -53,12 +55,13 @@ Python attributes have also been initialised to None, but you probably shouldn't rely on that.) Your :meth:`__cinit__` method is guaranteed to be called exactly once. -If your extension type has a base type, the :meth:`__cinit__` method of the base type -is automatically called before your :meth:`__cinit__` method is called; you cannot -explicitly call the inherited :meth:`__cinit__` method. If you need to pass a modified -argument list to the base type, you will have to do the relevant part of the -initialisation in the :meth:`__init__` method instead (where the normal rules for -calling inherited methods apply). +If your extension type has a base type, any existing :meth:`__cinit__` methods in +the base type hierarchy are automatically called before your :meth:`__cinit__` +method. You cannot explicitly call the inherited :meth:`__cinit__` methods, and the +base types are free to choose whether they implement :meth:`__cinit__` at all. +If you need to pass a modified argument list to the base type, you will have to do +the relevant part of the initialisation in the :meth:`__init__` method instead, where +the normal rules for calling inherited methods apply. Any initialisation which cannot safely be done in the :meth:`__cinit__` method should be done in the :meth:`__init__` method. By the time :meth:`__init__` is called, the object is @@ -126,34 +129,51 @@ executed unless they are explicitly called by the subclass. Arithmetic methods ------------------- -Arithmetic operator methods, such as :meth:`__add__`, behave differently from their -Python counterparts. There are no separate "reversed" versions of these -methods (:meth:`__radd__`, etc.) Instead, if the first operand cannot perform the -operation, the same method of the second operand is called, with the operands -in the same order. +Arithmetic operator methods, such as :meth:`__add__`, used to behave differently +from their Python counterparts in Cython 0.x, following the low-level semantics +of the C-API slot functions. Since Cython 3.0, they are called in the same way +as in Python, including the separate "reversed" versions of these methods +(:meth:`__radd__`, etc.). -This means that you can't rely on the first parameter of these methods being -"self" or being the right type, and you should test the types of both operands -before deciding what to do. If you can't handle the combination of types you've -been given, you should return `NotImplemented`. +Previously, if the first operand could not perform the operation, the same method +of the second operand was called, with the operands in the same order. +This means that you could not rely on the first parameter of these methods being +"self" or being the right type, and you needed to test the types of both operands +before deciding what to do. -This also applies to the in-place arithmetic method :meth:`__ipow__`. It doesn't apply -to any of the other in-place methods (:meth:`__iadd__`, etc.) which always -take `self` as the first argument. +If backwards compatibility is needed, the normal operator method (``__add__``, etc.) +can still be implemented to support both variants, applying a type check to the +arguments. The reversed method (``__radd__``, etc.) can always be implemented +with ``self`` as first argument and will be ignored by older Cython versions, whereas +Cython 3.x and later will only call the normal method with the expected argument order, +and otherwise call the reversed method instead. -.. _righ_comparisons: +Alternatively, the old Cython 0.x (or native C-API) behaviour is still available with +the directive ``c_api_binop_methods=True``. + +If you can't handle the combination of types you've been given, you should return +``NotImplemented``. This will let Python's operator implementation first try to apply +the reversed operator to the second operand, and failing that as well, report an +appropriate error to the user. + +This change in behaviour also applies to the in-place arithmetic method :meth:`__ipow__`. +It does not apply to any of the other in-place methods (:meth:`__iadd__`, etc.) +which always take ``self`` as the first argument. + +.. _rich_comparisons: Rich comparisons ----------------- -There are two ways to implement comparison methods. +There are a few ways to implement comparison methods. Depending on the application, one way or the other may be better: -* The first way uses the 6 Python +* Use the 6 Python `special methods <https://docs.python.org/3/reference/datamodel.html#basic-customization>`_ :meth:`__eq__`, :meth:`__lt__`, etc. - This is new since Cython 0.27 and works exactly as in plain Python classes. -* The second way uses a single special method :meth:`__richcmp__`. + This is supported since Cython 0.27 and works exactly as in plain Python classes. + +* Use a single special method :meth:`__richcmp__`. This implements all rich comparison operations in one method. The signature is ``def __richcmp__(self, other, int op)``. The integer argument ``op`` indicates which operation is to be performed @@ -175,6 +195,27 @@ Depending on the application, one way or the other may be better: These constants can be cimported from the ``cpython.object`` module. +* Use the ``@cython.total_ordering`` decorator, which is a low-level + re-implementation of the `functools.total_ordering + <https://docs.python.org/3/library/functools.html#functools.total_ordering>`_ + decorator specifically for ``cdef`` classes. (Normal Python classes can use + the original ``functools`` decorator.) + + .. code-block:: cython + + @cython.total_ordering + cdef class ExtGe: + cdef int x + + def __ge__(self, other): + if not isinstance(other, ExtGe): + return NotImplemented + return self.x >= (<ExtGe>other).x + + def __eq__(self, other): + return isinstance(other, ExtGe) and self.x == (<ExtGe>other).x + + .. _the__next__method: The :meth:`__next__` method @@ -212,13 +253,13 @@ https://docs.python.org/3/reference/datamodel.html#special-method-names +-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ | __dealloc__ |self | | Basic deallocation (no direct Python equivalent) | +-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ -| __cmp__ |x, y | int | 3-way comparison | +| __cmp__ |x, y | int | 3-way comparison (Python 2 only) | +-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ | __str__ |self | object | str(self) | +-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ | __repr__ |self | object | repr(self) | +-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ -| __hash__ |self | int | Hash function | +| __hash__ |self | Py_hash_t | Hash function (returns 32/64 bit integer) | +-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ | __call__ |self, ... | object | self(...) | +-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ @@ -266,47 +307,53 @@ Arithmetic operators https://docs.python.org/3/reference/datamodel.html#emulating-numeric-types -+-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ -| Name | Parameters | Return type | Description | -+=======================+=======================================+=============+=====================================================+ -| __add__ | x, y | object | binary `+` operator | -+-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ -| __sub__ | x, y | object | binary `-` operator | -+-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ -| __mul__ | x, y | object | `*` operator | -+-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ -| __div__ | x, y | object | `/` operator for old-style division | -+-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ -| __floordiv__ | x, y | object | `//` operator | -+-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ -| __truediv__ | x, y | object | `/` operator for new-style division | -+-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ -| __mod__ | x, y | object | `%` operator | -+-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ -| __divmod__ | x, y | object | combined div and mod | -+-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ -| __pow__ | x, y, z | object | `**` operator or pow(x, y, z) | -+-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ -| __neg__ | self | object | unary `-` operator | -+-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ -| __pos__ | self | object | unary `+` operator | -+-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ -| __abs__ | self | object | absolute value | -+-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ -| __nonzero__ | self | int | convert to boolean | -+-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ -| __invert__ | self | object | `~` operator | -+-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ -| __lshift__ | x, y | object | `<<` operator | -+-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ -| __rshift__ | x, y | object | `>>` operator | -+-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ -| __and__ | x, y | object | `&` operator | -+-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ -| __or__ | x, y | object | `|` operator | -+-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ -| __xor__ | x, y | object | `^` operator | -+-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ ++-----------------------------+--------------------+-------------+-----------------------------------------------------+ +| Name | Parameters | Return type | Description | ++=============================+====================+=============+=====================================================+ +| __add__, __radd__ | self, other | object | binary `+` operator | ++-----------------------------+--------------------+-------------+-----------------------------------------------------+ +| __sub__, __rsub__ | self, other | object | binary `-` operator | ++-----------------------------+--------------------+-------------+-----------------------------------------------------+ +| __mul__, __rmul__ | self, other | object | `*` operator | ++-----------------------------+--------------------+-------------+-----------------------------------------------------+ +| __div__, __rdiv__ | self, other | object | `/` operator for old-style division | ++-----------------------------+--------------------+-------------+-----------------------------------------------------+ +| __floordiv__, __rfloordiv__ | self, other | object | `//` operator | ++-----------------------------+--------------------+-------------+-----------------------------------------------------+ +| __truediv__, __rtruediv__ | self, other | object | `/` operator for new-style division | ++-----------------------------+--------------------+-------------+-----------------------------------------------------+ +| __mod__, __rmod__ | self, other | object | `%` operator | ++-----------------------------+--------------------+-------------+-----------------------------------------------------+ +| __divmod__, __rdivmod__ | self, other | object | combined div and mod | ++-----------------------------+--------------------+-------------+-----------------------------------------------------+ +| __pow__, __rpow__ | self, other, [mod] | object | `**` operator or pow(x, y, [mod]) | ++-----------------------------+--------------------+-------------+-----------------------------------------------------+ +| __neg__ | self | object | unary `-` operator | ++-----------------------------+--------------------+-------------+-----------------------------------------------------+ +| __pos__ | self | object | unary `+` operator | ++-----------------------------+--------------------+-------------+-----------------------------------------------------+ +| __abs__ | self | object | absolute value | ++-----------------------------+--------------------+-------------+-----------------------------------------------------+ +| __nonzero__ | self | int | convert to boolean | ++-----------------------------+--------------------+-------------+-----------------------------------------------------+ +| __invert__ | self | object | `~` operator | ++-----------------------------+--------------------+-------------+-----------------------------------------------------+ +| __lshift__, __rlshift__ | self, other | object | `<<` operator | ++-----------------------------+--------------------+-------------+-----------------------------------------------------+ +| __rshift__, __rrshift__ | self, other | object | `>>` operator | ++-----------------------------+--------------------+-------------+-----------------------------------------------------+ +| __and__, __rand__ | self, other | object | `&` operator | ++-----------------------------+--------------------+-------------+-----------------------------------------------------+ +| __or__, __ror__ | self, other | object | `|` operator | ++-----------------------------+--------------------+-------------+-----------------------------------------------------+ +| __xor__, __rxor__ | self, other | object | `^` operator | ++-----------------------------+--------------------+-------------+-----------------------------------------------------+ + +Note that Cython 0.x did not make use of the ``__r...__`` variants and instead +used the bidirectional C slot signature for the regular methods, thus making the +first argument ambiguous (not 'self' typed). +Since Cython 3.0, the operator calls are passed to the respective special methods. +See the section on :ref:`Arithmetic methods <arithmetic_methods>` above. Numeric conversions ^^^^^^^^^^^^^^^^^^^ @@ -351,7 +398,7 @@ https://docs.python.org/3/reference/datamodel.html#emulating-numeric-types +-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ | __imod__ | self, x | object | `%=` operator | +-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ -| __ipow__ | x, y, z | object | `**=` operator | +| __ipow__ | self, y, z | object | `**=` operator | +-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ | __ilshift__ | self, x | object | `<<=` operator | +-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ @@ -372,7 +419,7 @@ https://docs.python.org/3/reference/datamodel.html#emulating-container-types +-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ | Name | Parameters | Return type | Description | +=======================+=======================================+=============+=====================================================+ -| __len__ | self int | | len(self) | +| __len__ | self | Py_ssize_t | len(self) | +-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ | __getitem__ | self, x | object | self[x] | +-----------------------+---------------------------------------+-------------+-----------------------------------------------------+ diff --git a/docs/src/userguide/wrapping_CPlusPlus.rst b/docs/src/userguide/wrapping_CPlusPlus.rst index e12bf38be..b4df14e69 100644 --- a/docs/src/userguide/wrapping_CPlusPlus.rst +++ b/docs/src/userguide/wrapping_CPlusPlus.rst @@ -11,8 +11,8 @@ Overview Cython has native support for most of the C++ language. Specifically: -* C++ objects can be dynamically allocated with ``new`` and ``del`` keywords. -* C++ objects can be stack-allocated. +* C++ objects can be :term:`dynamically allocated<Dynamic allocation or Heap allocation>` with ``new`` and ``del`` keywords. +* C++ objects can be :term:`stack-allocated<Stack allocation>`. * C++ classes can be declared with the new keyword ``cppclass``. * Templated classes and functions are supported. * Overloaded functions are supported. @@ -97,7 +97,7 @@ We use the lines:: pass to include the C++ code from :file:`Rectangle.cpp`. It is also possible to specify to -distutils that :file:`Rectangle.cpp` is a source. To do that, you can add this directive at the +setuptools that :file:`Rectangle.cpp` is a source. To do that, you can add this directive at the top of the ``.pyx`` (not ``.pxd``) file:: # distutils: sources = Rectangle.cpp @@ -136,6 +136,9 @@ a "default" constructor:: def func(): cdef Foo foo ... + +See the section on the :ref:`cpp_locals directive` for a way +to avoid requiring a nullary/default constructor. Note that, like C++, if the class has only one constructor and it is a nullary one, it's not necessary to declare it. @@ -162,7 +165,9 @@ attribute access, you could just implement some properties: Cython initializes C++ class attributes of a cdef class using the nullary constructor. If the class you're wrapping does not have a nullary constructor, you must store a pointer -to the wrapped class and manually allocate and deallocate it. +to the wrapped class and manually allocate and deallocate it. Alternatively, the +:ref:`cpp_locals directive` avoids the need for the pointer and only initializes the +C++ class attribute when it is assigned to. A convenient and safe place to do so is in the `__cinit__` and `__dealloc__` methods which are guaranteed to be called exactly once upon creation and deletion of the Python instance. @@ -331,19 +336,27 @@ arguments) or by an explicit cast, e.g.: The following coercions are available: -+------------------+----------------+-----------------+ -| Python type => | *C++ type* | => Python type | -+==================+================+=================+ -| bytes | std::string | bytes | -+------------------+----------------+-----------------+ -| iterable | std::vector | list | -+------------------+----------------+-----------------+ -| iterable | std::list | list | -+------------------+----------------+-----------------+ -| iterable | std::set | set | -+------------------+----------------+-----------------+ -| iterable (len 2) | std::pair | tuple (len 2) | -+------------------+----------------+-----------------+ ++------------------+------------------------+-----------------+ +| Python type => | *C++ type* | => Python type | ++==================+========================+=================+ +| bytes | std::string | bytes | ++------------------+------------------------+-----------------+ +| iterable | std::vector | list | ++------------------+------------------------+-----------------+ +| iterable | std::list | list | ++------------------+------------------------+-----------------+ +| iterable | std::set | set | ++------------------+------------------------+-----------------+ +| iterable | std::unordered_set | set | ++------------------+------------------------+-----------------+ +| mapping | std::map | dict | ++------------------+------------------------+-----------------+ +| mapping | std::unordered_map | dict | ++------------------+------------------------+-----------------+ +| iterable (len 2) | std::pair | tuple (len 2) | ++------------------+------------------------+-----------------+ +| complex | std::complex | complex | ++------------------+------------------------+-----------------+ All conversions create a new container and copy the data into it. The items in the containers are converted to a corresponding type @@ -454,7 +467,7 @@ Static member method If the Rectangle class has a static member: -.. sourcecode:: c++ +.. code-block:: c++ namespace shapes { class Rectangle { @@ -482,6 +495,33 @@ Note, however, that it is unnecessary to declare the arguments of extern functions as references (const or otherwise) as it has no impact on the caller's syntax. +Scoped Enumerations +------------------- + +Cython supports scoped enumerations (:keyword:`enum class`) in C++ mode:: + + cdef enum class Cheese: + cheddar = 1 + camembert = 2 + +As with "plain" enums, you may access the enumerators as attributes of the type. +Unlike plain enums however, the enumerators are not visible to the +enclosing scope:: + + cdef Cheese c1 = Cheese.cheddar # OK + cdef Cheese c2 = cheddar # ERROR! + +Optionally, you may specify the underlying type of a scoped enumeration. +This is especially important when declaring an external scoped enumeration +with an underlying type:: + + cdef extern from "Foo.h": + cdef enum class Spam(unsigned int): + x = 10 + y = 20 + ... + +Declaring an enum class as ``cpdef`` will create a :pep:`435`-style Python wrapper. ``auto`` Keyword ---------------- @@ -527,7 +567,7 @@ Specify C++ language in setup.py Instead of specifying the language and the sources in the source files, it is possible to declare them in the :file:`setup.py` file:: - from distutils.core import setup + from setuptools import setup from Cython.Build import cythonize setup(ext_modules = cythonize( @@ -553,7 +593,7 @@ recognize the ``language`` option and it needs to be specified as an option to an :class:`Extension` that describes your extension and that is then handled by ``cythonize()`` as follows:: - from distutils.core import setup, Extension + from setuptools import Extension, setup from Cython.Build import cythonize setup(ext_modules = cythonize(Extension( @@ -568,7 +608,7 @@ often preferable (and overrides any global option). Starting with version 0.17, Cython also allows passing external source files into the ``cythonize()`` command this way. Here is a simplified setup.py file:: - from distutils.core import setup + from setuptools import setup from Cython.Build import cythonize setup( @@ -586,14 +626,52 @@ any source code, to compile it in C++ mode and link it statically against the .. note:: When using distutils directives, the paths are relative to the working - directory of the distutils run (which is usually the - project root where the :file:`setup.py` resides). + directory of the setuptools run (which is usually the project root where + the :file:`setup.py` resides). To compile manually (e.g. using ``make``), the ``cython`` command-line utility can be used to generate a C++ ``.cpp`` file, and then compile it into a python extension. C++ mode for the ``cython`` command is turned on with the ``--cplus`` option. +.. _cpp_locals directive: + +``cpp_locals`` directive +======================== + +The ``cpp_locals`` compiler directive is an experimental feature that makes +C++ variables behave like normal Python object variables. With this +directive they are only initialized at their first assignment, and thus +they no longer require a nullary constructor to be stack-allocated. Trying to +access an uninitialized C++ variable will generate an ``UnboundLocalError`` +(or similar) in the same way as a Python variable would. For example:: + + def function(dont_write): + cdef SomeCppClass c # not initialized + if dont_write: + return c.some_cpp_function() # UnboundLocalError + else: + c = SomeCppClass(...) # initialized + return c.some_cpp_function() # OK + +Additionally, the directive avoids initializing temporary C++ objects before +they are assigned, for cases where Cython needs to use such objects in its +own code-generation (often for return values of functions that can throw +exceptions). + +For extra speed, the ``initializedcheck`` directive disables the check for an +unbound-local. With this directive on, accessing a variable that has not +been initialized will trigger undefined behaviour, and it is entirely the user's +responsibility to avoid such access. + +The ``cpp_locals`` directive is currently implemented using ``std::optional`` +and thus requires a C++17 compatible compiler. Defining +``CYTHON_USE_BOOST_OPTIONAL`` (as define for the C++ compiler) uses ``boost::optional`` +instead (but is even more experimental and untested). The directive may +come with a memory and performance cost due to the need to store and check +a boolean that tracks if a variable is initialized, but the C++ compiler should +be able to eliminate the check in most cases. + Caveats and Limitations ======================== @@ -10,7 +10,7 @@ # Profiled execution. profile=no -# Add files or directories to the blacklist. They should be base names, not +# Add files or directories to the ignorelist. They should be base names, not # paths. ignore=.git,.gitmarker diff --git a/pyximport/pyxbuild.py b/pyximport/pyxbuild.py index de4a2241f..842e54502 100644 --- a/pyximport/pyxbuild.py +++ b/pyximport/pyxbuild.py @@ -103,7 +103,7 @@ def pyx_to_dll(filename, ext=None, force_rebuild=0, build_in_temp=False, pyxbuil so_path = obj_build_ext.get_outputs()[0] if obj_build_ext.inplace: # Python distutils get_outputs()[ returns a wrong so_path - # when --inplace ; see http://bugs.python.org/issue5977 + # when --inplace ; see https://bugs.python.org/issue5977 # workaround: so_path = os.path.join(os.path.dirname(filename), os.path.basename(so_path)) @@ -119,9 +119,9 @@ def pyx_to_dll(filename, ext=None, force_rebuild=0, build_in_temp=False, pyxbuil while count < 100: count += 1 r_path = os.path.join(obj_build_ext.build_lib, - basename + '.reload%s'%count) + basename + '.reload%s' % count) try: - import shutil # late import / reload_support is: debugging + import shutil # late import / reload_support is: debugging try: # Try to unlink first --- if the .so file # is mmapped by another process, @@ -140,7 +140,7 @@ def pyx_to_dll(filename, ext=None, force_rebuild=0, build_in_temp=False, pyxbuil break else: # used up all 100 slots - raise ImportError("reload count for %s reached maximum"%org_path) + raise ImportError("reload count for %s reached maximum" % org_path) _reloads[org_path]=(timestamp, so_path, count) return so_path except KeyboardInterrupt: @@ -157,4 +157,3 @@ def pyx_to_dll(filename, ext=None, force_rebuild=0, build_in_temp=False, pyxbuil if __name__=="__main__": pyx_to_dll("dummy.pyx") from . import test - diff --git a/pyximport/pyximport.py b/pyximport/pyximport.py index 5628f301b..7ff2ad219 100644 --- a/pyximport/pyximport.py +++ b/pyximport/pyximport.py @@ -191,7 +191,7 @@ def build_module(name, pyxfilename, pyxbuild_dir=None, inplace=False, language_l reload_support=pyxargs.reload_support) assert os.path.exists(so_path), "Cannot find: %s" % so_path - junkpath = os.path.join(os.path.dirname(so_path), name+"_*") #very dangerous with --inplace ? yes, indeed, trying to eat my files ;) + junkpath = os.path.join(os.path.dirname(so_path), name+"_*") #very dangerous with --inplace ? yes, indeed, trying to eat my files ;) junkstuff = glob.glob(junkpath) for path in junkstuff: if path != so_path: @@ -269,7 +269,7 @@ class PyxImporter(object): pyxbuild_dir=self.pyxbuild_dir, inplace=self.inplace, language_level=self.language_level) - if ty != imp.C_EXTENSION: # only when an extension, check if we have a .pyx next! + if ty != imp.C_EXTENSION: # only when an extension, check if we have a .pyx next! return None # find .pyx fast, when .so/.pyd exist --inplace @@ -350,7 +350,7 @@ class PyImporter(PyxImporter): language_level=language_level) self.uncompilable_modules = {} self.blocked_modules = ['Cython', 'pyxbuild', 'pyximport.pyxbuild', - 'distutils.extension', 'distutils.sysconfig'] + 'distutils'] def find_module(self, fullname, package_path=None): if fullname in sys.modules: diff --git a/pyximport/test/test_pyximport.py b/pyximport/test/test_pyximport.py index b3a4a9058..0b6b7432b 100644 --- a/pyximport/test/test_pyximport.py +++ b/pyximport/test/test_pyximport.py @@ -66,9 +66,9 @@ def make_ext(name, filename): assert len(pyximport._test_files)==1, pyximport._test_files reload(dummy) - time.sleep(1) # sleep a second to get safer mtimes + time.sleep(1) # sleep a second to get safer mtimes open(os.path.join(tempdir, "abc.txt"), "w").write(" ") - print("Here goes the reolad") + print("Here goes the reload") reload(dummy) assert len(pyximport._test_files) == 1, pyximport._test_files diff --git a/pyximport/test/test_reload.py b/pyximport/test/test_reload.py index ba53746f9..8c4c7962b 100644 --- a/pyximport/test/test_reload.py +++ b/pyximport/test/test_reload.py @@ -22,8 +22,8 @@ def test(): import hello assert hello.x == 1 - time.sleep(1) # sleep to make sure that new "hello.pyx" has later - # timestamp than object file. + time.sleep(1) # sleep to make sure that new "hello.pyx" has later + # timestamp than object file. open(hello_file, "w").write("x = 2; print x; after = 'after'\n") reload(hello) diff --git a/runtests.py b/runtests.py index a29bfb806..6622f9c67 100755 --- a/runtests.py +++ b/runtests.py @@ -3,6 +3,7 @@ from __future__ import print_function import atexit +import base64 import os import sys import re @@ -31,6 +32,9 @@ except (ImportError, AttributeError): IS_CPYTHON = True IS_PYPY = False +IS_PY2 = sys.version_info[0] < 3 +CAN_SYMLINK = sys.platform != 'win32' and hasattr(os, 'symlink') + from io import open as io_open try: from StringIO import StringIO @@ -64,11 +68,9 @@ except NameError: basestring = str WITH_CYTHON = True -CY3_DIR = None from distutils.command.build_ext import build_ext as _build_ext from distutils import sysconfig -from distutils import ccompiler _to_clean = [] @atexit.register @@ -96,7 +98,7 @@ def get_distutils_distro(_cache=[]): distutils_distro = Distribution() if sys.platform == 'win32': - # TODO: Figure out why this hackery (see http://thread.gmane.org/gmane.comp.python.cython.devel/8280/). + # TODO: Figure out why this hackery (see https://thread.gmane.org/gmane.comp.python.cython.devel/8280/). config_files = distutils_distro.find_config_files() try: config_files.remove('setup.cfg') @@ -116,7 +118,6 @@ def get_distutils_distro(_cache=[]): EXT_DEP_MODULES = { 'tag:numpy': 'numpy', - 'tag:numpy_old': 'numpy', 'tag:pythran': 'pythran', 'tag:setuptools': 'setuptools.sandbox', 'tag:asyncio': 'asyncio', @@ -236,17 +237,13 @@ def update_linetrace_extension(ext): return ext -def update_old_numpy_extension(ext): - update_numpy_extension(ext, set_api17_macro=False) - - def update_numpy_extension(ext, set_api17_macro=True): import numpy from numpy.distutils.misc_util import get_info ext.include_dirs.append(numpy.get_include()) - if set_api17_macro: + if set_api17_macro and getattr(numpy, '__version__', '') not in ('1.19.0', '1.19.1'): ext.define_macros.append(('NPY_NO_DEPRECATED_API', 'NPY_1_7_API_VERSION')) # We need the npymath library for numpy.math. @@ -255,6 +252,22 @@ def update_numpy_extension(ext, set_api17_macro=True): getattr(ext, attr).extend(value) +def update_gdb_extension(ext, _has_gdb=[None]): + # We should probably also check for Python support. + if not include_debugger: + _has_gdb[0] = False + if _has_gdb[0] is None: + try: + subprocess.check_call(["gdb", "--version"]) + except (IOError, subprocess.CalledProcessError): + _has_gdb[0] = False + else: + _has_gdb[0] = True + if not _has_gdb[0]: + return EXCLUDE_EXT + return ext + + def update_openmp_extension(ext): ext.openmp = True language = ext.language @@ -284,22 +297,61 @@ def update_cpp11_extension(ext): update cpp11 extensions that will run on versions of gcc >4.8 """ gcc_version = get_gcc_version(ext.language) + already_has_std = any(ca for ca in ext.extra_compile_args if "-std" in ca) if gcc_version: compiler_version = gcc_version.group(1) - if float(compiler_version) > 4.8: + if float(compiler_version) > 4.8 and not already_has_std: ext.extra_compile_args.append("-std=c++11") return ext clang_version = get_clang_version(ext.language) if clang_version: - ext.extra_compile_args.append("-std=c++11") + if not already_has_std: + ext.extra_compile_args.append("-std=c++11") + if sys.platform == "darwin": + ext.extra_compile_args.append("-stdlib=libc++") + ext.extra_compile_args.append("-mmacosx-version-min=10.7") + return ext + + return EXCLUDE_EXT + +def update_cpp17_extension(ext): + """ + update cpp17 extensions that will run on versions of gcc >=5.0 + """ + gcc_version = get_gcc_version(ext.language) + if gcc_version: + compiler_version = gcc_version.group(1) + if sys.version_info[0] < 3: + # The Python 2.7 headers contain the 'register' modifier + # which gcc warns about in C++17 mode. + ext.extra_compile_args.append('-Wno-register') + if float(compiler_version) >= 5.0: + ext.extra_compile_args.append("-std=c++17") + return ext + + clang_version = get_clang_version(ext.language) + if clang_version: + ext.extra_compile_args.append("-std=c++17") + if sys.version_info[0] < 3: + # The Python 2.7 headers contain the 'register' modifier + # which clang warns about in C++17 mode. + ext.extra_compile_args.append('-Wno-register') if sys.platform == "darwin": ext.extra_compile_args.append("-stdlib=libc++") - ext.extra_compile_args.append("-mmacosx-version-min=10.7") + ext.extra_compile_args.append("-mmacosx-version-min=10.13") return ext return EXCLUDE_EXT +def require_gcc(version): + def check(ext): + gcc_version = get_gcc_version(ext.language) + if gcc_version: + if float(gcc_version.group(1)) >= float(version): + return ext + return EXCLUDE_EXT + return check def get_cc_version(language): """ @@ -310,7 +362,8 @@ def get_cc_version(language): else: cc = sysconfig.get_config_var('CC') if not cc: - cc = ccompiler.get_default_compiler() + from distutils import ccompiler + cc = ccompiler.get_default_compiler() if not cc: return '' @@ -380,12 +433,15 @@ EXCLUDE_EXT = object() EXT_EXTRAS = { 'tag:numpy' : update_numpy_extension, - 'tag:numpy_old' : update_old_numpy_extension, 'tag:openmp': update_openmp_extension, + 'tag:gdb': update_gdb_extension, 'tag:cpp11': update_cpp11_extension, + 'tag:cpp17': update_cpp17_extension, 'tag:trace' : update_linetrace_extension, 'tag:bytesformat': exclude_extension_in_pyver((3, 3), (3, 4)), # no %-bytes formatting 'tag:no-macos': exclude_extension_on_platform('darwin'), + 'tag:py3only': exclude_extension_in_pyver((2, 7)), + 'tag:cppexecpolicies': require_gcc("9.1") } @@ -393,28 +449,27 @@ EXT_EXTRAS = { VER_DEP_MODULES = { # tests are excluded if 'CurrentPythonVersion OP VersionTuple', i.e. # (2,4) : (operator.lt, ...) excludes ... when PyVer < 2.4.x - (2,7) : (operator.lt, lambda x: x in ['run.withstat_py27', # multi context with statement - 'run.yield_inside_lambda', - 'run.test_dictviews', - 'run.pyclass_special_methods', - 'run.set_literals', - ]), + # The next line should start (3,); but this is a dictionary, so # we can only have one (3,) key. Since 2.7 is supposed to be the # last 2.x release, things would have to change drastically for this # to be unsafe... (2,999): (operator.lt, lambda x: x in ['run.special_methods_T561_py3', 'run.test_raisefrom', + 'run.different_package_names', + 'run.unicode_imports', # encoding problems on appveyor in Py2 'run.reimport_failure', # reimports don't do anything in Py2 ]), (3,): (operator.ge, lambda x: x in ['run.non_future_division', 'compile.extsetslice', 'compile.extdelslice', - 'run.special_methods_T561_py2' + 'run.special_methods_T561_py2', ]), (3,3) : (operator.lt, lambda x: x in ['build.package_compilation', + 'build.cythonize_pep420_namespace', 'run.yield_from_py33', 'pyximport.pyximport_namespace', + 'run.qualname', ]), (3,4): (operator.lt, lambda x: x in ['run.py34_signature', 'run.test_unicode', # taken from Py3.7, difficult to backport @@ -426,6 +481,10 @@ VER_DEP_MODULES = { 'run.mod__spec__', 'run.pep526_variable_annotations', # typing module 'run.test_exceptions', # copied from Py3.7+ + 'run.time_pxd', # _PyTime_GetSystemClock doesn't exist in 3.4 + ]), + (3,7): (operator.lt, lambda x: x in ['run.pycontextvar', + 'run.pep557_dataclasses', # dataclasses module ]), } @@ -483,7 +542,7 @@ def parse_tags(filepath): return tags -list_unchanging_dir = memoize(lambda x: os.listdir(x)) +list_unchanging_dir = memoize(lambda x: os.listdir(x)) # needs lambda to set function attribute @memoize @@ -494,10 +553,23 @@ def _list_pyregr_data_files(test_directory): if is_data_file(filename)] +def import_module_from_file(module_name, file_path, execute=True): + import importlib.util + spec = importlib.util.spec_from_file_location(module_name, file_path) + m = importlib.util.module_from_spec(spec) + if execute: + sys.modules[module_name] = m + spec.loader.exec_module(m) + return m + + def import_ext(module_name, file_path=None): if file_path: - import imp - return imp.load_dynamic(module_name, file_path) + if sys.version_info >= (3, 5): + return import_module_from_file(module_name, file_path) + else: + import imp + return imp.load_dynamic(module_name, file_path) else: try: from importlib import invalidate_caches @@ -529,9 +601,14 @@ class build_ext(_build_ext): class ErrorWriter(object): match_error = re.compile(r'(warning:)?(?:.*:)?\s*([-0-9]+)\s*:\s*([-0-9]+)\s*:\s*(.*)').match - def __init__(self): + def __init__(self, encoding=None): self.output = [] - self.write = self.output.append + self.encoding = encoding + + def write(self, value): + if self.encoding: + value = value.encode('ISO-8859-1').decode(self.encoding) + self.output.append(value) def _collect(self): s = ''.join(self.output) @@ -607,7 +684,8 @@ class TestBuilder(object): with_pyregr, languages, test_bugs, language_level, common_utility_dir, pythran_dir=None, default_mode='run', stats=None, - add_embedded_test=False): + add_embedded_test=False, add_cython_import=False, + add_cpp_locals_extra_tests=False): self.rootdir = rootdir self.workdir = workdir self.selectors = selectors @@ -618,7 +696,7 @@ class TestBuilder(object): self.cleanup_failures = options.cleanup_failures self.with_pyregr = with_pyregr self.cython_only = options.cython_only - self.doctest_selector = re.compile(options.only_pattern).search if options.only_pattern else None + self.test_selector = re.compile(options.only_pattern).search if options.only_pattern else None self.languages = languages self.test_bugs = test_bugs self.fork = options.fork @@ -629,11 +707,15 @@ class TestBuilder(object): self.default_mode = default_mode self.stats = stats self.add_embedded_test = add_embedded_test + self.add_cython_import = add_cython_import + self.capture = options.capture + self.add_cpp_locals_extra_tests = add_cpp_locals_extra_tests def build_suite(self): suite = unittest.TestSuite() filenames = os.listdir(self.rootdir) filenames.sort() + # TODO: parallelise I/O with a thread pool for the different directories once we drop Py2 support for filename in filenames: path = os.path.join(self.rootdir, filename) if os.path.isdir(path) and filename != TEST_SUPPORT_DIR: @@ -648,7 +730,7 @@ class TestBuilder(object): and (sys.version_info < (3, 8) or sys.platform != 'darwin')): # Non-Windows makefile. if [1 for selector in self.selectors if selector("embedded")] \ - and not [1 for selector in self.exclude_selectors if selector("embedded")]: + and not [1 for selector in self.exclude_selectors if selector("embedded")]: suite.addTest(unittest.makeSuite(EmbedTest)) return suite @@ -687,8 +769,13 @@ class TestBuilder(object): mode = 'pyregr' if ext == '.srctree': + if self.cython_only: + # EndToEnd tests always execute arbitrary build and test code + continue if 'cpp' not in tags['tag'] or 'cpp' in self.languages: - suite.addTest(EndToEndTest(filepath, workdir, self.cleanup_workdir, stats=self.stats)) + suite.addTest(EndToEndTest(filepath, workdir, + self.cleanup_workdir, stats=self.stats, + capture=self.capture)) continue # Choose the test suite. @@ -707,7 +794,7 @@ class TestBuilder(object): raise KeyError('Invalid test mode: ' + mode) for test in self.build_tests(test_class, path, workdir, - module, mode == 'error', tags): + module, filepath, mode == 'error', tags): suite.addTest(test) if mode == 'run' and ext == '.py' and not self.cython_only and not filename.startswith('test_'): @@ -718,14 +805,16 @@ class TestBuilder(object): if pyver ] if not min_py_ver or any(sys.version_info >= min_ver for min_ver in min_py_ver): - suite.addTest(PureDoctestTestCase(module, os.path.join(path, filename), tags, stats=self.stats)) + suite.addTest(PureDoctestTestCase(module, filepath, tags, stats=self.stats)) return suite - def build_tests(self, test_class, path, workdir, module, expect_errors, tags): + def build_tests(self, test_class, path, workdir, module, module_path, expect_errors, tags): warning_errors = 'werror' in tags['tag'] expect_warnings = 'warnings' in tags['tag'] + extra_directives_list = [{}] + if expect_errors: if skip_c(tags) and 'cpp' in self.languages: languages = ['cpp'] @@ -740,9 +829,14 @@ class TestBuilder(object): if 'cpp' in languages and 'no-cpp' in tags['tag']: languages = list(languages) languages.remove('cpp') + if (self.add_cpp_locals_extra_tests and 'cpp' in languages and + 'cpp' in tags['tag'] and not 'no-cpp-locals' in tags['tag']): + extra_directives_list.append({'cpp_locals': True}) if not languages: return [] + language_levels = [2, 3] if 'all_language_levels' in tags['tag'] else [None] + pythran_dir = self.pythran_dir if 'pythran' in tags['tag'] and not pythran_dir and 'cpp' in languages: import pythran.config @@ -752,23 +846,36 @@ class TestBuilder(object): pythran_ext = pythran.config.make_extension() pythran_dir = pythran_ext['include_dirs'][0] + add_cython_import = self.add_cython_import and module_path.endswith('.py') + preparse_list = tags.get('preparse', ['id']) - tests = [ self.build_test(test_class, path, workdir, module, tags, language, + tests = [ self.build_test(test_class, path, workdir, module, module_path, + tags, language, language_level, expect_errors, expect_warnings, warning_errors, preparse, - pythran_dir if language == "cpp" else None) + pythran_dir if language == "cpp" else None, + add_cython_import=add_cython_import, + extra_directives=extra_directives) for language in languages - for preparse in preparse_list ] + for preparse in preparse_list + for language_level in language_levels + for extra_directives in extra_directives_list + ] return tests - def build_test(self, test_class, path, workdir, module, tags, language, - expect_errors, expect_warnings, warning_errors, preparse, pythran_dir): + def build_test(self, test_class, path, workdir, module, module_path, tags, language, language_level, + expect_errors, expect_warnings, warning_errors, preparse, pythran_dir, add_cython_import, + extra_directives): language_workdir = os.path.join(workdir, language) if not os.path.exists(language_workdir): os.makedirs(language_workdir) workdir = os.path.join(language_workdir, module) if preparse != 'id': - workdir += '_%s' % str(preparse) - return test_class(path, workdir, module, tags, + workdir += '_%s' % (preparse,) + if language_level: + workdir += '_cy%d' % (language_level,) + if extra_directives: + workdir += ('_directives_'+ '_'.join('%s_%s' % (k, v) for k,v in extra_directives.items())) + return test_class(path, workdir, module, module_path, tags, language=language, preparse=preparse, expect_errors=expect_errors, @@ -778,14 +885,16 @@ class TestBuilder(object): cleanup_sharedlibs=self.cleanup_sharedlibs, cleanup_failures=self.cleanup_failures, cython_only=self.cython_only, - doctest_selector=self.doctest_selector, + test_selector=self.test_selector, fork=self.fork, - language_level=self.language_level, + language_level=language_level or self.language_level, warning_errors=warning_errors, test_determinism=self.test_determinism, common_utility_dir=self.common_utility_dir, pythran_dir=pythran_dir, - stats=self.stats) + stats=self.stats, + add_cython_import=add_cython_import, + ) def skip_c(tags): @@ -816,17 +925,30 @@ def filter_stderr(stderr_bytes): return stderr_bytes +def filter_test_suite(test_suite, selector): + filtered_tests = [] + for test in test_suite._tests: + if isinstance(test, unittest.TestSuite): + filter_test_suite(test, selector) + elif not selector(test.id()): + continue + filtered_tests.append(test) + test_suite._tests[:] = filtered_tests + + class CythonCompileTestCase(unittest.TestCase): - def __init__(self, test_directory, workdir, module, tags, language='c', preparse='id', + def __init__(self, test_directory, workdir, module, module_path, tags, language='c', preparse='id', expect_errors=False, expect_warnings=False, annotate=False, cleanup_workdir=True, - cleanup_sharedlibs=True, cleanup_failures=True, cython_only=False, doctest_selector=None, + cleanup_sharedlibs=True, cleanup_failures=True, cython_only=False, test_selector=None, fork=True, language_level=2, warning_errors=False, test_determinism=False, - common_utility_dir=None, pythran_dir=None, stats=None): + common_utility_dir=None, pythran_dir=None, stats=None, add_cython_import=False, + extra_directives={}): self.test_directory = test_directory self.tags = tags self.workdir = workdir self.module = module + self.module_path = module_path self.language = language self.preparse = preparse self.name = module if self.preparse == "id" else "%s_%s" % (module, preparse) @@ -837,7 +959,7 @@ class CythonCompileTestCase(unittest.TestCase): self.cleanup_sharedlibs = cleanup_sharedlibs self.cleanup_failures = cleanup_failures self.cython_only = cython_only - self.doctest_selector = doctest_selector + self.test_selector = test_selector self.fork = fork self.language_level = language_level self.warning_errors = warning_errors @@ -845,27 +967,53 @@ class CythonCompileTestCase(unittest.TestCase): self.common_utility_dir = common_utility_dir self.pythran_dir = pythran_dir self.stats = stats + self.add_cython_import = add_cython_import + self.extra_directives = extra_directives unittest.TestCase.__init__(self) def shortDescription(self): - return "compiling (%s%s) %s" % (self.language, "/pythran" if self.pythran_dir is not None else "", self.name) + return "compiling (%s%s%s) %s" % ( + self.language, + "/cy2" if self.language_level == 2 else "/cy3" if self.language_level == 3 else "", + "/pythran" if self.pythran_dir is not None else "", + self.description_name() + ) + + def description_name(self): + return self.name def setUp(self): from Cython.Compiler import Options self._saved_options = [ (name, getattr(Options, name)) - for name in ('warning_errors', 'clear_to_none', 'error_on_unknown_names', 'error_on_uninitialized') + for name in ( + 'warning_errors', + 'clear_to_none', + 'error_on_unknown_names', + 'error_on_uninitialized', + # 'cache_builtins', # not currently supported due to incorrect global caching + ) ] self._saved_default_directives = list(Options.get_directive_defaults().items()) Options.warning_errors = self.warning_errors if sys.version_info >= (3, 4): Options._directive_defaults['autotestdict'] = False + Options._directive_defaults.update(self.extra_directives) if not os.path.exists(self.workdir): os.makedirs(self.workdir) if self.workdir not in sys.path: sys.path.insert(0, self.workdir) + if self.add_cython_import: + with open(self.module_path, 'rb') as f: + source = f.read() + if b'cython.cimports.' in source: + from Cython.Shadow import CythonCImports + for name in set(re.findall(br"(cython\.cimports(?:\.\w+)+)", source)): + name = name.decode() + sys.modules[name] = CythonCImports(name) + def tearDown(self): from Cython.Compiler import Options for name, value in self._saved_options: @@ -881,6 +1029,13 @@ class CythonCompileTestCase(unittest.TestCase): del sys.modules[self.module] except KeyError: pass + + # remove any stubs of cimported modules in pure Python mode + if self.add_cython_import: + for name in list(sys.modules): + if name.startswith('cython.cimports.'): + del sys.modules[name] + cleanup = self.cleanup_failures or self.success cleanup_c_files = WITH_CYTHON and self.cleanup_workdir and cleanup cleanup_lib_files = self.cleanup_sharedlibs and cleanup @@ -891,13 +1046,17 @@ class CythonCompileTestCase(unittest.TestCase): shutil.rmtree(self.workdir, ignore_errors=True) else: for rmfile in os.listdir(self.workdir): + ext = os.path.splitext(rmfile)[1] if not cleanup_c_files: - if (rmfile[-2:] in (".c", ".h") or - rmfile[-4:] == ".cpp" or - rmfile.endswith(".html") and rmfile.startswith(self.module)): + # Keep C, C++ files, header files, preprocessed sources + # and assembly sources (typically the .i and .s files + # are intentionally generated when -save-temps is given) + if ext in (".c", ".cpp", ".h", ".i", ".ii", ".s"): + continue + if ext == ".html" and rmfile.startswith(self.module): continue - is_shared_obj = rmfile.endswith(".so") or rmfile.endswith(".dll") + is_shared_obj = ext in (".so", ".dll") if not cleanup_lib_files and is_shared_obj: continue @@ -929,8 +1088,9 @@ class CythonCompileTestCase(unittest.TestCase): def runCompileTest(self): return self.compile( - self.test_directory, self.module, self.workdir, - self.test_directory, self.expect_errors, self.expect_warnings, self.annotate) + self.test_directory, self.module, self.module_path, self.workdir, + self.test_directory, self.expect_errors, self.expect_warnings, self.annotate, + self.add_cython_import) def find_module_source_file(self, source_file): if not os.path.exists(source_file): @@ -955,10 +1115,7 @@ class CythonCompileTestCase(unittest.TestCase): fout.write(preparse_func(fin.read())) else: # use symlink on Unix, copy on Windows - try: - copy = os.symlink - except AttributeError: - copy = shutil.copy + copy = os.symlink if CAN_SYMLINK else shutil.copy join = os.path.join for filename in file_list: @@ -971,21 +1128,32 @@ class CythonCompileTestCase(unittest.TestCase): [filename for filename in file_list if not os.path.isfile(os.path.join(workdir, filename))]) - def split_source_and_output(self, test_directory, module, workdir): - source_file = self.find_module_source_file(os.path.join(test_directory, module) + '.pyx') + def split_source_and_output(self, source_file, workdir, add_cython_import=False): + from Cython.Utils import detect_opened_file_encoding + with io_open(source_file, 'rb') as f: + # encoding is passed to ErrorWriter but not used on the source + # since it is sometimes deliberately wrong + encoding = detect_opened_file_encoding(f, default=None) + with io_open(source_file, 'r', encoding='ISO-8859-1') as source_and_output: error_writer = warnings_writer = None - out = io_open(os.path.join(workdir, module + os.path.splitext(source_file)[1]), + out = io_open(os.path.join(workdir, os.path.basename(source_file)), 'w', encoding='ISO-8859-1') try: for line in source_and_output: - if line.startswith("_ERRORS"): + if line.startswith(u"_ERRORS"): out.close() - out = error_writer = ErrorWriter() - elif line.startswith("_WARNINGS"): + out = error_writer = ErrorWriter(encoding=encoding) + elif line.startswith(u"_WARNINGS"): out.close() - out = warnings_writer = ErrorWriter() + out = warnings_writer = ErrorWriter(encoding=encoding) else: + if add_cython_import and line.strip() and not ( + line.startswith(u'#') or line.startswith(u"from __future__ import ")): + # insert "import cython" statement after any directives or future imports + if line != u"import cython\n": + out.write(u"import cython\n") + add_cython_import = False out.write(line) finally: out.close() @@ -993,18 +1161,16 @@ class CythonCompileTestCase(unittest.TestCase): return (error_writer.geterrors() if error_writer else [], warnings_writer.geterrors() if warnings_writer else []) - def run_cython(self, test_directory, module, targetdir, incdir, annotate, + def run_cython(self, test_directory, module, module_path, targetdir, incdir, annotate, extra_compile_options=None): include_dirs = INCLUDE_DIRS + [os.path.join(test_directory, '..', TEST_SUPPORT_DIR)] if incdir: include_dirs.append(incdir) - if self.preparse == 'id': - source = self.find_module_source_file( - os.path.join(test_directory, module + '.pyx')) - else: - self.copy_files(test_directory, targetdir, [module + '.pyx']) - source = os.path.join(targetdir, module + '.pyx') + if self.preparse != 'id' and test_directory != targetdir: + file_name = os.path.basename(module_path) + self.copy_files(test_directory, targetdir, [file_name]) + module_path = os.path.join(targetdir, file_name) target = os.path.join(targetdir, self.build_target_filename(module)) if extra_compile_options is None: @@ -1017,9 +1183,9 @@ class CythonCompileTestCase(unittest.TestCase): try: CompilationOptions except NameError: - from Cython.Compiler.Main import CompilationOptions + from Cython.Compiler.Options import CompilationOptions from Cython.Compiler.Main import compile as cython_compile - from Cython.Compiler.Main import default_options + from Cython.Compiler.Options import default_options common_utility_include_dir = self.common_utility_dir options = CompilationOptions( @@ -1036,8 +1202,7 @@ class CythonCompileTestCase(unittest.TestCase): common_utility_include_dir = common_utility_include_dir, **extra_compile_options ) - cython_compile(source, options=options, - full_module_name=module) + cython_compile(module_path, options=options, full_module_name=module) def run_distutils(self, test_directory, module, workdir, incdir, extra_extension_args=None): @@ -1075,6 +1240,10 @@ class CythonCompileTestCase(unittest.TestCase): if self.language == 'cpp': # Set the language now as the fixer might need it extension.language = 'c++' + if self.extra_directives.get('cpp_locals'): + extension = update_cpp17_extension(extension) + if extension is EXCLUDE_EXT: + return if 'distutils' in self.tags: from Cython.Build.Dependencies import DistutilsInfo @@ -1106,10 +1275,30 @@ class CythonCompileTestCase(unittest.TestCase): extension = newext or extension if self.language == 'cpp': extension.language = 'c++' + if IS_PY2: + workdir = str(workdir) # work around type check in distutils that disallows unicode strings + build_extension.extensions = [extension] build_extension.build_temp = workdir build_extension.build_lib = workdir - build_extension.run() + + from Cython.Utils import captured_fd, prepare_captured + from distutils.errors import CompileError + + error = None + with captured_fd(2) as get_stderr: + try: + build_extension.run() + except CompileError as exc: + error = str(exc) + stderr = get_stderr() + if stderr: + # The test module name should always be ASCII, but let's not risk encoding failures. + output = b"Compiler output for module " + module.encode('utf-8') + b":\n" + stderr + b"\n" + out = sys.stdout if sys.version_info[0] == 2 else sys.stdout.buffer + out.write(output) + if error is not None: + raise CompileError(u"%s\nCompiler output:\n%s" % (error, prepare_captured(stderr))) finally: os.chdir(cwd) @@ -1131,27 +1320,28 @@ class CythonCompileTestCase(unittest.TestCase): return get_ext_fullpath(module) - def compile(self, test_directory, module, workdir, incdir, - expect_errors, expect_warnings, annotate): + def compile(self, test_directory, module, module_path, workdir, incdir, + expect_errors, expect_warnings, annotate, add_cython_import): expected_errors = expected_warnings = errors = warnings = () - if expect_errors or expect_warnings: + if expect_errors or expect_warnings or add_cython_import: expected_errors, expected_warnings = self.split_source_and_output( - test_directory, module, workdir) + module_path, workdir, add_cython_import) test_directory = workdir + module_path = os.path.join(workdir, os.path.basename(module_path)) if WITH_CYTHON: old_stderr = sys.stderr try: sys.stderr = ErrorWriter() with self.stats.time(self.name, self.language, 'cython'): - self.run_cython(test_directory, module, workdir, incdir, annotate) + self.run_cython(test_directory, module, module_path, workdir, incdir, annotate) errors, warnings = sys.stderr.getall() finally: sys.stderr = old_stderr if self.test_determinism and not expect_errors: workdir2 = workdir + '-again' os.mkdir(workdir2) - self.run_cython(test_directory, module, workdir2, incdir, annotate) + self.run_cython(test_directory, module, module_path, workdir2, incdir, annotate) diffs = [] for file in os.listdir(workdir2): if (open(os.path.join(workdir, file)).read() @@ -1202,11 +1392,14 @@ class CythonCompileTestCase(unittest.TestCase): finally: if show_output: stdout = get_stdout and get_stdout().strip() + stderr = get_stderr and filter_stderr(get_stderr()).strip() + if so_path and not stderr: + # normal success case => ignore non-error compiler output + stdout = None if stdout: print_bytes( stdout, header_text="\n=== C/C++ compiler output: =========\n", end=None, file=sys.__stderr__) - stderr = get_stderr and filter_stderr(get_stderr()).strip() if stderr: print_bytes( stderr, header_text="\n=== C/C++ compiler error output: ===\n", @@ -1240,11 +1433,8 @@ class CythonRunTestCase(CythonCompileTestCase): from Cython.Compiler import Options Options.clear_to_none = False - def shortDescription(self): - if self.cython_only: - return CythonCompileTestCase.shortDescription(self) - else: - return "compiling (%s%s) and running %s" % (self.language, "/pythran" if self.pythran_dir is not None else "", self.name) + def description_name(self): + return self.name if self.cython_only else "and running %s" % self.name def run(self, result=None): if result is None: @@ -1255,8 +1445,7 @@ class CythonRunTestCase(CythonCompileTestCase): try: self.success = False ext_so_path = self.runCompileTest() - # Py2.6 lacks "_TextTestResult.skipped" - failures, errors, skipped = len(result.failures), len(result.errors), len(getattr(result, 'skipped', [])) + failures, errors, skipped = len(result.failures), len(result.errors), len(result.skipped) if not self.cython_only and ext_so_path is not None: self.run_tests(result, ext_so_path) if failures == len(result.failures) and errors == len(result.errors): @@ -1286,8 +1475,8 @@ class CythonRunTestCase(CythonCompileTestCase): else: module = module_or_name tests = doctest.DocTestSuite(module) - if self.doctest_selector is not None: - tests._tests[:] = [test for test in tests._tests if self.doctest_selector(test.id())] + if self.test_selector: + filter_test_suite(tests, self.test_selector) with self.stats.time(self.name, self.language, 'run'): tests.run(result) run_forked_test(result, run_test, self.shortDescription(), self.fork) @@ -1387,9 +1576,13 @@ class PureDoctestTestCase(unittest.TestCase): try: self.setUp() - import imp with self.stats.time(self.name, 'py', 'pyimport'): - m = imp.load_source(loaded_module_name, self.module_path) + if sys.version_info >= (3, 5): + m = import_module_from_file(self.module_name, self.module_path) + else: + import imp + m = imp.load_source(loaded_module_name, self.module_path) + try: with self.stats.time(self.name, 'py', 'pyrun'): doctest.DocTestSuite(m).run(result) @@ -1441,10 +1634,6 @@ class PartialTestResult(_TextTestResult): _TextTestResult.__init__( self, self._StringIO(), True, base_result.dots + base_result.showAll*2) - try: - self.skipped - except AttributeError: - self.skipped = [] # Py2.6 def strip_error_results(self, results): for test_case, error in results: @@ -1469,10 +1658,7 @@ class PartialTestResult(_TextTestResult): if output: result.stream.write(output) result.errors.extend(errors) - try: - result.skipped.extend(skipped) - except AttributeError: - pass # Py2.6 + result.skipped.extend(skipped) result.failures.extend(failures) result.testsRun += tests_run @@ -1485,12 +1671,14 @@ class PartialTestResult(_TextTestResult): class CythonUnitTestCase(CythonRunTestCase): def shortDescription(self): - return "compiling (%s) tests in %s" % (self.language, self.name) + return "compiling (%s) tests in %s" % (self.language, self.description_name()) def run_tests(self, result, ext_so_path): with self.stats.time(self.name, self.language, 'import'): module = import_ext(self.module, ext_so_path) tests = unittest.defaultTestLoader.loadTestsFromModule(module) + if self.test_selector: + filter_test_suite(tests, self.test_selector) with self.stats.time(self.name, self.language, 'run'): tests.run(result) @@ -1578,10 +1766,12 @@ class TestCodeFormat(unittest.TestCase): def runTest(self): import pycodestyle - config_file = os.path.join(self.cython_dir, "tox.ini") + config_file = os.path.join(self.cython_dir, "setup.cfg") if not os.path.exists(config_file): - config_file=os.path.join(os.path.dirname(__file__), "tox.ini") - paths = glob.glob(os.path.join(self.cython_dir, "**/*.py"), recursive=True) + config_file = os.path.join(os.path.dirname(__file__), "setup.cfg") + paths = [] + for codedir in ['Cython', 'Demos', 'docs', 'pyximport', 'tests']: + paths += glob.glob(os.path.join(self.cython_dir, codedir + "/**/*.py"), recursive=True) style = pycodestyle.StyleGuide(config_file=config_file) print("") # Fix the first line of the report. result = style.check_files(paths) @@ -1638,13 +1828,13 @@ def collect_doctests(path, module_prefix, suite, selectors, exclude_selectors): return dirname not in ("Mac", "Distutils", "Plex", "Tempita") def file_matches(filename): filename, ext = os.path.splitext(filename) - blacklist = ['libcython', 'libpython', 'test_libcython_in_gdb', - 'TestLibCython'] + excludelist = ['libcython', 'libpython', 'test_libcython_in_gdb', + 'TestLibCython'] return (ext == '.py' and not '~' in filename and not '#' in filename and not filename.startswith('.') and not - filename in blacklist) + filename in excludelist) import doctest for dirpath, dirnames, filenames in os.walk(path): for dir in list(dirnames): @@ -1681,12 +1871,13 @@ class EndToEndTest(unittest.TestCase): """ cython_root = os.path.dirname(os.path.abspath(__file__)) - def __init__(self, treefile, workdir, cleanup_workdir=True, stats=None): + def __init__(self, treefile, workdir, cleanup_workdir=True, stats=None, capture=True): self.name = os.path.splitext(os.path.basename(treefile))[0] self.treefile = treefile self.workdir = os.path.join(workdir, self.name) self.cleanup_workdir = cleanup_workdir self.stats = stats + self.capture = capture cython_syspath = [self.cython_root] for path in sys.path: if path.startswith(self.cython_root) and path not in cython_syspath: @@ -1702,11 +1893,9 @@ class EndToEndTest(unittest.TestCase): def setUp(self): from Cython.TestUtils import unpack_source_tree - _, self.commands = unpack_source_tree(self.treefile, self.workdir) + _, self.commands = unpack_source_tree(self.treefile, self.workdir, self.cython_root) self.old_dir = os.getcwd() os.chdir(self.workdir) - if self.workdir not in sys.path: - sys.path.insert(0, self.workdir) def tearDown(self): if self.cleanup_workdir: @@ -1727,31 +1916,31 @@ class EndToEndTest(unittest.TestCase): def runTest(self): self.success = False - commands = (self.commands - .replace("CYTHON", "PYTHON %s" % os.path.join(self.cython_root, 'cython.py')) - .replace("PYTHON", sys.executable)) old_path = os.environ.get('PYTHONPATH') env = dict(os.environ) new_path = self.cython_syspath if old_path: - new_path = new_path + os.pathsep + old_path + new_path = new_path + os.pathsep + self.workdir + os.pathsep + old_path env['PYTHONPATH'] = new_path + if not env.get("PYTHONIOENCODING"): + env["PYTHONIOENCODING"] = sys.stdout.encoding or sys.getdefaultencoding() cmd = [] out = [] err = [] - for command_no, command in enumerate(filter(None, commands.splitlines()), 1): + for command_no, command in enumerate(self.commands, 1): with self.stats.time('%s(%d)' % (self.name, command_no), 'c', - 'etoe-build' if ' setup.py ' in command else 'etoe-run'): - p = subprocess.Popen(command, - stderr=subprocess.PIPE, - stdout=subprocess.PIPE, - shell=True, - env=env) - _out, _err = p.communicate() + 'etoe-build' if 'setup.py' in command else 'etoe-run'): + if self.capture: + p = subprocess.Popen(command, stderr=subprocess.PIPE, stdout=subprocess.PIPE, env=env) + _out, _err = p.communicate() + res = p.returncode + else: + p = subprocess.call(command, env=env) + _out, _err = b'', b'' + res = p cmd.append(command) out.append(_out) err.append(_err) - res = p.returncode if res == 0 and b'REFNANNY: ' in _out: res = -1 if res != 0: @@ -1793,13 +1982,10 @@ class EmbedTest(unittest.TestCase): if not os.path.isdir(libdir) or libname not in os.listdir(libdir): # report the error for the original directory libdir = sysconfig.get_config_var('LIBDIR') - cython = 'cython.py' - if sys.version_info[0] >=3 and CY3_DIR: - cython = os.path.join(CY3_DIR, cython) - cython = os.path.abspath(os.path.join('..', '..', cython)) + cython = os.path.abspath(os.path.join('..', '..', 'cython.py')) try: - subprocess.check_call([ + subprocess.check_output([ "make", "PYTHON='%s'" % sys.executable, "CYTHON='%s'" % cython, @@ -1812,17 +1998,43 @@ class EmbedTest(unittest.TestCase): self.assertTrue(True) # :) +def load_listfile(filename): + # just re-use the FileListExclude implementation + fle = FileListExcluder(filename) + return list(fle.excludes) class MissingDependencyExcluder(object): def __init__(self, deps): # deps: { matcher func : module name } self.exclude_matchers = [] - for matcher, mod in deps.items(): + for matcher, module_name in deps.items(): try: - __import__(mod) + module = __import__(module_name) except ImportError: self.exclude_matchers.append(string_selector(matcher)) + print("Test dependency not found: '%s'" % module_name) + else: + version = self.find_dep_version(module_name, module) + print("Test dependency found: '%s' version %s" % (module_name, version)) self.tests_missing_deps = [] + + def find_dep_version(self, name, module): + try: + version = module.__version__ + except AttributeError: + stdlib_dir = os.path.dirname(shutil.__file__) + os.sep + module_path = getattr(module, '__file__', stdlib_dir) # no __file__? => builtin stdlib module + if module_path.startswith(stdlib_dir): + # stdlib module + version = sys.version.partition(' ')[0] + elif '.' in name: + # incrementally look for a parent package with version + name = name.rpartition('.')[0] + return self.find_dep_version(name, __import__(name)) + else: + version = '?.?' + return version + def __call__(self, testname, tags=None): for matcher in self.exclude_matchers: if matcher(testname, tags): @@ -1860,8 +2072,7 @@ class FileListExcluder(object): self.excludes[line.split()[0]] = True def __call__(self, testname, tags=None): - exclude = (testname in self.excludes - or testname.split('.')[-1] in self.excludes) + exclude = any(string_selector(ex)(testname) for ex in self.excludes) if exclude and self.verbose: print("Excluding %s because it's listed in %s" % (testname, self._list_file)) @@ -1903,14 +2114,18 @@ class ShardExcludeSelector(object): # This is an exclude selector so it can override the (include) selectors. # It may not provide uniform distribution (in time or count), but is a # determanistic partition of the tests which is important. + + # Random seed to improve the hash distribution. + _seed = base64.b64decode(b'2ged1EtsGz/GkisJr22UcLeP6n9XIaA5Vby2wM49Wvg=') + def __init__(self, shard_num, shard_count): self.shard_num = shard_num self.shard_count = shard_count - def __call__(self, testname, tags=None, _hash=zlib.crc32, _is_py2=sys.version_info[0] < 3): + def __call__(self, testname, tags=None, _hash=zlib.crc32, _is_py2=IS_PY2): # Cannot use simple hash() here as shard processes might use different hash seeds. # CRC32 is fast and simple, but might return negative values in Py2. - hashval = _hash(testname) & 0x7fffffff if _is_py2 else _hash(testname.encode()) + hashval = _hash(self._seed + testname) & 0x7fffffff if _is_py2 else _hash(self._seed + testname.encode()) return hashval % self.shard_count != self.shard_num @@ -1982,6 +2197,10 @@ def flush_and_terminate(status): def main(): global DISTDIR, WITH_CYTHON + + # Set an environment variable to the top directory + os.environ['CYTHON_PROJECT_DIR'] = os.path.abspath(os.path.dirname(__file__)) + DISTDIR = os.path.join(os.getcwd(), os.path.dirname(sys.argv[0])) from Cython.Compiler import DebugFlags @@ -1993,7 +2212,7 @@ def main(): args.append(arg) from optparse import OptionParser - parser = OptionParser() + parser = OptionParser(usage="usage: %prog [options] [selector ...]") parser.add_option("--no-cleanup", dest="cleanup_workdir", action="store_false", default=True, help="do not delete the generated C files (allows passing --no-cython on next run)") @@ -2017,6 +2236,9 @@ def main(): parser.add_option("--no-cpp", dest="use_cpp", action="store_false", default=True, help="do not test C++ compilation backend") + parser.add_option("--no-cpp-locals", dest="use_cpp_locals", + action="store_false", default=True, + help="do not rerun select C++ tests with cpp_locals directive") parser.add_option("--no-unit", dest="unittests", action="store_false", default=True, help="do not run the unit tests") @@ -2050,6 +2272,9 @@ def main(): parser.add_option("-x", "--exclude", dest="exclude", action="append", metavar="PATTERN", help="exclude tests matching the PATTERN") + parser.add_option("--listfile", dest="listfile", + action="append", + help="specify a file containing a list of tests to run") parser.add_option("-j", "--shard_count", dest="shard_count", metavar="N", type=int, default=1, help="shard this run into several parallel runs") @@ -2115,6 +2340,10 @@ def main(): help="test whether Cython's output is deterministic") parser.add_option("--pythran-dir", dest="pythran_dir", default=None, help="specify Pythran include directory. This will run the C++ tests using Pythran backend for Numpy") + parser.add_option("--no-capture", dest="capture", default=True, action="store_false", + help="do not capture stdout, stderr in srctree tests. Makes pdb.set_trace interactive") + parser.add_option("--limited-api", dest="limited_api", default=False, action="store_true", + help="Compiles Cython using CPython's LIMITED_API") options, cmd_args = parser.parse_args(args) @@ -2141,7 +2370,18 @@ def main(): if options.xml_output_dir: shutil.rmtree(options.xml_output_dir, ignore_errors=True) + if options.listfile: + for listfile in options.listfile: + cmd_args.extend(load_listfile(listfile)) + + if options.capture and not options.for_debugging: + keep_alive_interval = 10 + else: + keep_alive_interval = None if options.shard_count > 1 and options.shard_num == -1: + if "PYTHONIOENCODING" not in os.environ: + # Make sure subprocesses can print() Unicode text. + os.environ["PYTHONIOENCODING"] = sys.stdout.encoding or sys.getdefaultencoding() import multiprocessing pool = multiprocessing.Pool(options.shard_count) tasks = [(options, cmd_args, shard_num) for shard_num in range(options.shard_count)] @@ -2150,7 +2390,7 @@ def main(): # NOTE: create process pool before time stamper thread to avoid forking issues. total_time = time.time() stats = Stats() - with time_stamper_thread(): + with time_stamper_thread(interval=keep_alive_interval): for shard_num, shard_stats, return_code, failure_output in pool.imap_unordered(runtests_callback, tasks): if return_code != 0: error_shards.append(shard_num) @@ -2170,7 +2410,7 @@ def main(): else: return_code = 0 else: - with time_stamper_thread(): + with time_stamper_thread(interval=keep_alive_interval): _, stats, return_code, _ = runtests(options, cmd_args, coverage) if coverage: @@ -2200,20 +2440,30 @@ def time_stamper_thread(interval=10): Print regular time stamps into the build logs to find slow tests. @param interval: time interval in seconds """ + if not interval or interval < 0: + # Do nothing + yield + return + try: _xrange = xrange except NameError: _xrange = range import threading - from datetime import datetime + import datetime from time import sleep interval = _xrange(interval * 4) - now = datetime.now - write = sys.__stderr__.write + now = datetime.datetime.now stop = False + # We capture stderr in some places. + # => make sure we write to the real (original) stderr of the test runner. + stderr = os.dup(2) + def write(s): + os.write(stderr, s if type(s) is bytes else s.encode('ascii')) + def time_stamper(): while True: for _ in interval: @@ -2223,18 +2473,19 @@ def time_stamper_thread(interval=10): write('\n#### %s\n' % now()) thread = threading.Thread(target=time_stamper, name='time_stamper') - thread.setDaemon(True) # Py2.6 ... + thread.setDaemon(True) # Py2 ... thread.start() try: yield finally: stop = True thread.join() + os.close(stderr) def configure_cython(options): global CompilationOptions, pyrex_default_options, cython_compile - from Cython.Compiler.Main import \ + from Cython.Compiler.Options import \ CompilationOptions, \ default_options as pyrex_default_options from Cython.Compiler.Options import _directive_defaults as directive_defaults @@ -2268,6 +2519,23 @@ def runtests_callback(args): def runtests(options, cmd_args, coverage=None): + # faulthandler should be able to provide a limited traceback + # in the event of a segmentation fault. Only available on Python 3.3+ + try: + import faulthandler + except ImportError: + pass # OK - not essential + else: + faulthandler.enable() + + if sys.platform == "win32" and sys.version_info < (3, 6): + # enable Unicode console output, if possible + try: + import win_unicode_console + except ImportError: + pass + else: + win_unicode_console.enable() WITH_CYTHON = options.with_cython ROOTDIR = os.path.abspath(options.root_dir) @@ -2306,7 +2574,7 @@ def runtests(options, cmd_args, coverage=None): options.cleanup_sharedlibs = False options.fork = False if WITH_CYTHON and include_debugger: - from Cython.Compiler.Main import default_options as compiler_default_options + from Cython.Compiler.Options import default_options as compiler_default_options compiler_default_options['gdb_debug'] = True compiler_default_options['output_dir'] = os.getcwd() @@ -2323,6 +2591,10 @@ def runtests(options, cmd_args, coverage=None): sys.path.insert(0, os.path.split(libpath)[0]) CDEFS.append(('CYTHON_REFNANNY', '1')) + if options.limited_api: + CFLAGS.append("-DCYTHON_LIMITED_API=1") + CFLAGS.append('-Wno-unused-function') + if xml_output_dir and options.fork: # doesn't currently work together sys.stderr.write("Disabling forked testing to support XML test output\n") @@ -2381,6 +2653,10 @@ def runtests(options, cmd_args, coverage=None): bug_files = [ ('bugs.txt', True), ('pypy_bugs.txt', IS_PYPY), + ('pypy2_bugs.txt', IS_PYPY and IS_PY2), + ('pypy_crash_bugs.txt', IS_PYPY), + ('pypy_implementation_detail_bugs.txt', IS_PYPY), + ('limited_api_bugs.txt', options.limited_api), ('windows_bugs.txt', sys.platform == 'win32'), ('cygwin_bugs.txt', sys.platform == 'cygwin') ] @@ -2411,8 +2687,8 @@ def runtests(options, cmd_args, coverage=None): sys.stderr.write("Backends: %s\n" % ','.join(backends)) languages = backends - if 'TRAVIS' in os.environ and sys.platform == 'darwin' and 'cpp' in languages: - bugs_file_name = 'travis_macos_cpp_bugs.txt' + if 'CI' in os.environ and sys.platform == 'darwin' and 'cpp' in languages: + bugs_file_name = 'macos_cpp_bugs.txt' exclude_selectors += [ FileListExcluder(os.path.join(ROOTDIR, bugs_file_name), verbose=verbose_excludes) @@ -2440,8 +2716,10 @@ def runtests(options, cmd_args, coverage=None): filetests = TestBuilder(ROOTDIR, WORKDIR, selectors, exclude_selectors, options, options.pyregr, languages, test_bugs, options.language_level, common_utility_dir, - options.pythran_dir, add_embedded_test=True, stats=stats) + options.pythran_dir, add_embedded_test=True, stats=stats, + add_cpp_locals_extra_tests=options.use_cpp_locals) test_suite.addTest(filetests.build_suite()) + if options.examples and languages: examples_workdir = os.path.join(WORKDIR, 'examples') for subdirectory in glob.glob(os.path.join(options.examples_dir, "*/")): @@ -2449,7 +2727,7 @@ def runtests(options, cmd_args, coverage=None): options, options.pyregr, languages, test_bugs, options.language_level, common_utility_dir, options.pythran_dir, - default_mode='compile', stats=stats) + default_mode='compile', stats=stats, add_cython_import=True) test_suite.addTest(filetests.build_suite()) if options.system_pyregr and languages: @@ -2486,10 +2764,7 @@ def runtests(options, cmd_args, coverage=None): else: text_runner_options = {} if options.failfast: - if sys.version_info < (2, 7): - sys.stderr.write("--failfast not supported with Python < 2.7\n") - else: - text_runner_options['failfast'] = True + text_runner_options['failfast'] = True test_runner = unittest.TextTestRunner(verbosity=options.verbosity, **text_runner_options) if options.pyximport_py: diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 000000000..d7aaad869 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,43 @@ +[flake8] +max-complexity = 10 + +[pycodestyle] +exclude = .git,build,__pycache__,venv*,TEST*,tests/run/test*.py,Cython/Debugger/libpython.py +max-line-length = 150 +format = pylint +# See https://pycodestyle.pycqa.org/en/latest/intro.html#configuration +select = + E711, E713, E714, E501, W291, E502, E703, + # indentation + E101, E111, E112, E113, E117 + E121, E125, E129, + # E114, E115, E116, E122, + # whitespace + E211, E223, E224, E227, E228, E242, E261, E273, E274, E275, + # E201, E202, E203, E211, E265 + # E303, E306, + W1, W2, W3 +#ignore = W, E +ignore = + W504, + # W504 line break after binary operator + S001, + # S001 found module formatter + E226, + # E226 missing whitespace around operator[run] + +[coverage:run] +branch = True +parallel = True +concurrency = multiprocessing,thread +include = Cython/* +source = Cython +omit = Test* + +[bdist_wheel] +universal = 1 + +[metadata] +license_files = + LICENSE.txt + COPYING.txt @@ -14,7 +14,7 @@ is_cpython = platform.python_implementation() == 'CPython' # this specifies which versions of python we support, pip >= 9 knows to skip # versions of packages which are not compatible with the running python -PYTHON_REQUIRES = '>=2.6, !=3.0.*, !=3.1.*, !=3.2.*' +PYTHON_REQUIRES = '>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*' if sys.platform == "darwin": # Don't create resource files on OS X tar. @@ -42,8 +42,7 @@ pxd_include_dirs = [ directory for directory, dirs, files in os.walk(os.path.join('Cython', 'Includes')) if '__init__.pyx' in files or '__init__.pxd' in files - or directory == os.path.join('Cython', 'Includes') - or directory == os.path.join('Cython', 'Includes', 'Deprecated')] + or directory == os.path.join('Cython', 'Includes')] pxd_include_patterns = [ p+'/*.pxd' for p in pxd_include_dirs ] + [ @@ -80,11 +79,14 @@ else: scripts = ["cython.py", "cythonize.py", "cygdb.py"] -def compile_cython_modules(profile=False, compile_more=False, cython_with_refnanny=False): +def compile_cython_modules(profile=False, coverage=False, compile_more=False, cython_with_refnanny=False): source_root = os.path.abspath(os.path.dirname(__file__)) compiled_modules = [ "Cython.Plex.Scanners", "Cython.Plex.Actions", + "Cython.Plex.Machines", + "Cython.Plex.Transitions", + "Cython.Plex.DFA", "Cython.Compiler.Scanning", "Cython.Compiler.Visitor", "Cython.Compiler.FlowControl", @@ -133,6 +135,8 @@ def compile_cython_modules(profile=False, compile_more=False, cython_with_refnan defines = [] if cython_with_refnanny: defines.append(('CYTHON_REFNANNY', '1')) + if coverage: + defines.append(('CYTHON_TRACE', '1')) extensions = [] for module in compiled_modules: @@ -157,10 +161,18 @@ def compile_cython_modules(profile=False, compile_more=False, cython_with_refnan from Cython.Distutils.build_ext import new_build_ext from Cython.Compiler.Options import get_directive_defaults - get_directive_defaults()['language_level'] = 2 + get_directive_defaults().update( + language_level=2, + binding=False, + always_allow_keywords=False, + autotestdict=False, + ) if profile: get_directive_defaults()['profile'] = True sys.stderr.write("Enabled profiling for the Cython binary modules\n") + if coverage: + get_directive_defaults()['linetrace'] = True + sys.stderr.write("Enabled line tracing and profiling for the Cython binary modules\n") # not using cythonize() directly to let distutils decide whether building extensions was requested add_command_class("build_ext", new_build_ext) @@ -171,6 +183,10 @@ cython_profile = '--cython-profile' in sys.argv if cython_profile: sys.argv.remove('--cython-profile') +cython_coverage = '--cython-coverage' in sys.argv +if cython_coverage: + sys.argv.remove('--cython-coverage') + try: sys.argv.remove("--cython-compile-all") cython_compile_more = True @@ -189,15 +205,10 @@ try: except ValueError: compile_cython_itself = True -if compile_cython_itself and (is_cpython or cython_compile_more): - compile_cython_modules(cython_profile, cython_compile_more, cython_with_refnanny) - setup_args.update(setuptools_extra_args) -from Cython import __version__ as version - -def dev_status(): +def dev_status(version): if 'b' in version or 'c' in version: # 1b1, 1beta1, 2rc1, ... return 'Development Status :: 4 - Beta' @@ -225,65 +236,74 @@ packages = [ 'pyximport', ] -setup( - name='Cython', - version=version, - url='http://cython.org/', - author='Robert Bradshaw, Stefan Behnel, Dag Seljebotn, Greg Ewing, et al.', - author_email='cython-devel@python.org', - description="The Cython compiler for writing C extensions for the Python language.", - long_description=textwrap.dedent("""\ - The Cython language makes writing C extensions for the Python language as - easy as Python itself. Cython is a source code translator based on Pyrex_, - but supports more cutting edge functionality and optimizations. - - The Cython language is a superset of the Python language (almost all Python - code is also valid Cython code), but Cython additionally supports optional - static typing to natively call C functions, operate with C++ classes and - declare fast C types on variables and class attributes. This allows the - compiler to generate very efficient C code from Cython code. - - This makes Cython the ideal language for writing glue code for external - C/C++ libraries, and for fast C modules that speed up the execution of - Python code. - - Note that for one-time builds, e.g. for CI/testing, on platforms that are not - covered by one of the wheel packages provided on PyPI *and* the pure Python wheel - that we provide is not used, it is substantially faster than a full source build - to install an uncompiled (slower) version of Cython with:: - - pip install Cython --install-option="--no-cython-compile" - - .. _Pyrex: http://www.cosc.canterbury.ac.nz/greg.ewing/python/Pyrex/ - """), - license='Apache', - classifiers=[ - dev_status(), - "Intended Audience :: Developers", - "License :: OSI Approved :: Apache Software License", - "Operating System :: OS Independent", - "Programming Language :: Python", - "Programming Language :: Python :: 2", - "Programming Language :: Python :: 2.6", - "Programming Language :: Python :: 2.7", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.4", - "Programming Language :: Python :: 3.5", - "Programming Language :: Python :: 3.6", - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: Implementation :: CPython", - "Programming Language :: Python :: Implementation :: PyPy", - "Programming Language :: C", - "Programming Language :: Cython", - "Topic :: Software Development :: Code Generators", - "Topic :: Software Development :: Compilers", - "Topic :: Software Development :: Libraries :: Python Modules" - ], - - scripts=scripts, - packages=packages, - py_modules=["cython"], - **setup_args -) + +def run_build(): + if compile_cython_itself and (is_cpython or cython_compile_more): + compile_cython_modules(cython_profile, cython_coverage, cython_compile_more, cython_with_refnanny) + + from Cython import __version__ as version + setup( + name='Cython', + version=version, + url='https://cython.org/', + author='Robert Bradshaw, Stefan Behnel, Dag Seljebotn, Greg Ewing, et al.', + author_email='cython-devel@python.org', + description="The Cython compiler for writing C extensions for the Python language.", + long_description=textwrap.dedent("""\ + The Cython language makes writing C extensions for the Python language as + easy as Python itself. Cython is a source code translator based on Pyrex_, + but supports more cutting edge functionality and optimizations. + + The Cython language is a superset of the Python language (almost all Python + code is also valid Cython code), but Cython additionally supports optional + static typing to natively call C functions, operate with C++ classes and + declare fast C types on variables and class attributes. This allows the + compiler to generate very efficient C code from Cython code. + + This makes Cython the ideal language for writing glue code for external + C/C++ libraries, and for fast C modules that speed up the execution of + Python code. + + Note that for one-time builds, e.g. for CI/testing, on platforms that are not + covered by one of the wheel packages provided on PyPI *and* the pure Python wheel + that we provide is not used, it is substantially faster than a full source build + to install an uncompiled (slower) version of Cython with:: + + pip install Cython --install-option="--no-cython-compile" + + .. _Pyrex: https://www.cosc.canterbury.ac.nz/greg.ewing/python/Pyrex/ + """), + license='Apache', + classifiers=[ + dev_status(version), + "Intended Audience :: Developers", + "License :: OSI Approved :: Apache Software License", + "Operating System :: OS Independent", + "Programming Language :: Python", + "Programming Language :: Python :: 2", + "Programming Language :: Python :: 2.7", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.4", + "Programming Language :: Python :: 3.5", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python :: Implementation :: PyPy", + "Programming Language :: C", + "Programming Language :: Cython", + "Topic :: Software Development :: Code Generators", + "Topic :: Software Development :: Compilers", + "Topic :: Software Development :: Libraries :: Python Modules" + ], + + scripts=scripts, + packages=packages, + py_modules=["cython"], + **setup_args + ) + + +if __name__ == '__main__': + run_build() diff --git a/test-requirements-34.txt b/test-requirements-34.txt new file mode 100644 index 000000000..8697eff4b --- /dev/null +++ b/test-requirements-34.txt @@ -0,0 +1,3 @@ +numpy < 1.19.0 +coverage +pycodestyle diff --git a/test-requirements.txt b/test-requirements.txt index 2cb479eee..27254b8a5 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,3 +1,3 @@ -numpy != 1.19.0 +numpy coverage pycodestyle diff --git a/tests/broken/cdefemptysue.pyx b/tests/broken/cdefemptysue.pyx deleted file mode 100644 index d2dda1b25..000000000 --- a/tests/broken/cdefemptysue.pyx +++ /dev/null @@ -1,14 +0,0 @@ -cdef extern from "cdefemptysue.h": - - cdef struct spam: - pass - - ctypedef union eggs: - pass - - cdef enum ham: - pass - -cdef extern spam s -cdef extern eggs e -cdef extern ham h diff --git a/tests/broken/cdefexternblock.pyx b/tests/broken/cdefexternblock.pyx deleted file mode 100644 index 89fca223f..000000000 --- a/tests/broken/cdefexternblock.pyx +++ /dev/null @@ -1,19 +0,0 @@ -cdef extern from "cheese.h": - - ctypedef int camembert - - struct roquefort: - int x - - char *swiss - - void cheddar() - - class external.runny [object runny_obj]: - cdef int a - def __init__(self): - pass - -cdef runny r -r = x -r.a = 42 diff --git a/tests/buffers/bufaccess.pyx b/tests/buffers/bufaccess.pyx index 8761e6eb9..6b0b4ac30 100644 --- a/tests/buffers/bufaccess.pyx +++ b/tests/buffers/bufaccess.pyx @@ -13,8 +13,6 @@ from cpython.object cimport PyObject from cpython.ref cimport Py_INCREF, Py_DECREF cimport cython -__test__ = {} - import sys #import re exclude = []#re.compile('object').search] @@ -27,8 +25,7 @@ if getattr(sys, 'pypy_version_info', None) is not None: def testcase(func): for e in exclude: if e(func.__name__): - return func - __test__[func.__name__] = func.__doc__ + func.__doc__ = "" # disable the test return func @@ -959,6 +956,8 @@ def addref(*args): def decref(*args): for item in args: Py_DECREF(item) +@cython.binding(False) +@cython.always_allow_keywords(False) def get_refcount(x): return (<PyObject*>x).ob_refcnt @@ -991,15 +990,16 @@ def assign_to_object(object[object] buf, int idx, obj): See comments on printbuf_object above. >>> a, b = [1, 2, 3], [4, 5, 6] - >>> get_refcount(a), get_refcount(b) - (2, 2) + >>> rca1, rcb1 = get_refcount(a), get_refcount(b) + >>> rca1 == rcb1 + True >>> addref(a) >>> A = ObjectMockBuffer(None, [1, a]) # 1, ...,otherwise it thinks nested lists... - >>> get_refcount(a), get_refcount(b) - (3, 2) + >>> get_refcount(a) == rca1+1, get_refcount(b) == rcb1 + (True, True) >>> assign_to_object(A, 1, b) - >>> get_refcount(a), get_refcount(b) - (2, 3) + >>> get_refcount(a) == rca1, get_refcount(b) == rcb1+1 + (True, True) >>> decref(b) """ buf[idx] = obj @@ -1010,15 +1010,14 @@ def assign_temporary_to_object(object[object] buf): See comments on printbuf_object above. >>> a, b = [1, 2, 3], {4:23} - >>> get_refcount(a) - 2 + >>> rc1 = get_refcount(a) >>> addref(a) >>> A = ObjectMockBuffer(None, [b, a]) - >>> get_refcount(a) - 3 + >>> get_refcount(a) == rc1+1 + True >>> assign_temporary_to_object(A) - >>> get_refcount(a) - 2 + >>> get_refcount(a) == rc1 + True >>> printbuf_object(A, (2,)) {4: 23} 2 diff --git a/tests/buffers/buffmt.pyx b/tests/buffers/buffmt.pyx index eba10020d..0a9757270 100644 --- a/tests/buffers/buffmt.pyx +++ b/tests/buffers/buffmt.pyx @@ -3,10 +3,6 @@ import struct # Tests buffer format string parsing. -__test__ = {} -def testcase(func): - __test__[func.__name__] = func.__doc__ - return func from libc cimport stdlib @@ -56,7 +52,6 @@ cdef class MockBuffer: info.format = self.format info.itemsize = self.itemsize -@testcase def _int(fmt): """ >>> _int("i") @@ -78,14 +73,12 @@ def _int(fmt): """ cdef object[int] buf = MockBuffer(fmt, sizeof(int)) -@testcase def _ulong(fmt): """ >>> _ulong("L") """ cdef object[unsigned long] buf = MockBuffer(fmt, sizeof(unsigned long)) -@testcase def wrongsize(): """ >>> wrongsize() @@ -96,7 +89,6 @@ def wrongsize(): """ cdef object[float] buf = MockBuffer("f", 1) -@testcase def _obj(fmt): """ >>> _obj("O") @@ -151,7 +143,6 @@ cdef struct UnpackedStruct4: char d int e, f, g -@testcase def char3int(fmt): """ >>> char3int("ciii") @@ -166,7 +157,7 @@ def char3int(fmt): >>> char3int("c3xiii") >>> char3int("cxxxiii") - Standard alignment (assming int size is 4) + Standard alignment (assuming int size is 4) >>> char3int("=c3xiii") >>> char3int("=ciii") Traceback (most recent call last): @@ -185,7 +176,6 @@ def char3int(fmt): cdef object[Char3Int, ndim=1] buf = obj -@testcase def long_string(fmt): """ >>> long_string("90198s") @@ -194,7 +184,6 @@ def long_string(fmt): cdef object[LongString, ndim=1] buf = obj -@testcase def unpacked_struct(fmt): """ Native formats: @@ -218,7 +207,6 @@ def unpacked_struct(fmt): cdef struct ComplexTest: ComplexFloat a, b, c -@testcase def complex_test(fmt): """ >>> complex_test("ZfZfZf") @@ -236,7 +224,6 @@ def complex_test(fmt): cdef object[ComplexTest] buf1 = obj -@testcase def alignment_string(fmt, exc=None): """ >>> alignment_string("@i") @@ -258,7 +245,6 @@ def alignment_string(fmt, exc=None): print "fail" -@testcase def int_and_long_are_same(): """ >>> int_and_long_are_same() @@ -273,7 +259,6 @@ cdef struct MixedComplex: double real float imag -@testcase def mixed_complex_struct(): """ Triggering a specific execution path for this case. @@ -311,7 +296,6 @@ cdef packed struct PartiallyPackedStruct2: char b int c -@testcase def packed_struct(fmt): """ Assuming int is four bytes: @@ -334,7 +318,6 @@ def packed_struct(fmt): """ cdef object[PackedStruct] buf = MockBuffer(fmt, sizeof(PackedStruct)) -@testcase def partially_packed_struct(fmt): """ Assuming int is four bytes: @@ -362,7 +345,6 @@ def partially_packed_struct(fmt): cdef object[PartiallyPackedStruct] buf = MockBuffer( fmt, sizeof(PartiallyPackedStruct)) -@testcase def partially_packed_struct_2(fmt): """ Assuming int is four bytes: @@ -380,7 +362,7 @@ def partially_packed_struct_2(fmt): Traceback (most recent call last): ... ValueError: Buffer dtype mismatch; next field is at offset 8 but 5 expected - + >>> partially_packed_struct_2("ccici") Traceback (most recent call last): ... @@ -398,7 +380,6 @@ cdef packed struct PackedStructWithCharArrays: char[3] d -@testcase def packed_struct_with_strings(fmt): """ >>> packed_struct_with_strings("T{f:a:i:b:5s:c:3s:d:}") @@ -430,7 +411,6 @@ ctypedef struct PackedStructWithNDArrays: float d -@testcase def packed_struct_with_arrays(fmt): """ >>> packed_struct_with_arrays("T{(16)d:a:(16)d:b:d:c:}") @@ -440,7 +420,6 @@ def packed_struct_with_arrays(fmt): fmt, sizeof(PackedStructWithArrays)) -@testcase def unpacked_struct_with_arrays(fmt): """ >>> if struct.calcsize('P') == 8: # 64 bit @@ -453,7 +432,6 @@ def unpacked_struct_with_arrays(fmt): fmt, sizeof(UnpackedStructWithArrays)) -@testcase def packed_struct_with_ndarrays(fmt): """ >>> packed_struct_with_ndarrays("T{d:a:(2,2)d:b:f:c:f:d:}") diff --git a/tests/buffers/mockbuffers.pxi b/tests/buffers/mockbuffers.pxi index 09395cc51..8703b789b 100644 --- a/tests/buffers/mockbuffers.pxi +++ b/tests/buffers/mockbuffers.pxi @@ -1,5 +1,4 @@ from libc cimport stdlib -from libc cimport stdio cimport cpython.buffer import sys @@ -34,7 +33,7 @@ cdef class MockBuffer: cdef Py_ssize_t x, s, cumprod, itemsize self.label = label self.release_ok = True - self.log = "" + self.log = u"" self.offset = offset self.itemsize = itemsize = self.get_itemsize() self.writable = writable @@ -138,7 +137,7 @@ cdef class MockBuffer: self.received_flags.append(name) if flags & cpython.buffer.PyBUF_WRITABLE and not self.writable: - raise BufferError("Writable buffer requested from read-only mock: %s" % ' | '.join(self.received_flags)) + raise BufferError(f"Writable buffer requested from read-only mock: {' | '.join(self.received_flags)}") buffer.buf = <void*>(<char*>self.buffer + (<int>self.offset * self.itemsize)) buffer.obj = self @@ -152,29 +151,29 @@ cdef class MockBuffer: buffer.itemsize = self.itemsize buffer.internal = NULL if self.label: - msg = "acquired %s" % self.label - print msg - self.log += msg + "\n" + msg = f"acquired {self.label}" + print(msg) + self.log += msg + u"\n" def __releasebuffer__(MockBuffer self, Py_buffer* buffer): if buffer.suboffsets != self.suboffsets: self.release_ok = False if self.label: - msg = "released %s" % self.label - print msg - self.log += msg + "\n" + msg = f"released {self.label}" + print(msg) + self.log += msg + u"\n" def printlog(self): - print self.log[:-1] + print(self.log[:-1]) def resetlog(self): - self.log = "" + self.log = u"" cdef int write(self, char* buf, object value) except -1: raise Exception() cdef get_itemsize(self): - print "ERROR, not subclassed", self.__class__ + print(f"ERROR, not subclassed: {self.__class__}") cdef get_default_format(self): - print "ERROR, not subclassed", self.__class__ + print(f"ERROR, not subclassed {self.__class__}") cdef class CharMockBuffer(MockBuffer): cdef int write(self, char* buf, object value) except -1: @@ -246,10 +245,10 @@ cdef class ErrorBuffer: self.label = label def __getbuffer__(ErrorBuffer self, Py_buffer* buffer, int flags): - raise Exception("acquiring %s" % self.label) + raise Exception(f"acquiring {self.label}") def __releasebuffer__(ErrorBuffer self, Py_buffer* buffer): - raise Exception("releasing %s" % self.label) + raise Exception(f"releasing {self.label}") # # Structs @@ -336,13 +335,12 @@ cdef class LongComplexMockBuffer(MockBuffer): def print_offsets(*args, size, newline=True): - sys.stdout.write(' '.join([str(item // size) for item in args])) - if newline: - sys.stdout.write('\n') + sys.stdout.write(' '.join([str(item // size) for item in args]) + ('\n' if newline else '')) def print_int_offsets(*args, newline=True): print_offsets(*args, size=sizeof(int), newline=newline) + shape_5_3_4_list = [[list(range(k * 12 + j * 4, k * 12 + j * 4 + 4)) for j in range(3)] for k in range(5)] diff --git a/tests/buffers/userbuffer.pyx b/tests/buffers/userbuffer.pyx index b9c871970..df774a07b 100644 --- a/tests/buffers/userbuffer.pyx +++ b/tests/buffers/userbuffer.pyx @@ -1,21 +1,11 @@ -import sys -__doc__ = u"" - -if sys.version_info[:2] == (2, 6): - __doc__ += u""" ->>> memoryview = _memoryview -""" - -__doc__ += u""" +__doc__ = u""" >>> b1 = UserBuffer1() >>> m1 = memoryview(b1) >>> m1.tolist() [0, 1, 2, 3, 4] >>> del m1, b1 -""" -__doc__ += u""" >>> b2 = UserBuffer2() >>> m2 = memoryview(b2) UserBuffer2: getbuffer diff --git a/tests/build/common_include_dir.srctree b/tests/build/common_include_dir.srctree index 0b3e4f36f..2ab4662c0 100644 --- a/tests/build/common_include_dir.srctree +++ b/tests/build/common_include_dir.srctree @@ -16,18 +16,17 @@ PYTHON fake_grep.py -c '#include "common/AddTraceback_impl_.*h"' c.c import sys from Cython.Build.Dependencies import cythonize -import platform import os from distutils.core import setup -# os x on Travis specifically seems to crash with nthreads>0 -osx_on_travis = (platform.system() == "Darwin" and os.getenv("TRAVIS")) +# os x on CI specifically seems to crash with nthreads>0 +osx_on_ci = (sys.platform == "darwin" and os.getenv("CI")) # Test concurrent safety if multiprocessing is available. -# (In particular, TravisCI does not support spawning processes from tests.) +# (In particular, CI providers like Travis and Github Actions do not support spawning processes from tests.) nthreads = 0 -if not (hasattr(sys, 'pypy_version_info') or osx_on_travis): +if not (hasattr(sys, 'pypy_version_info') or osx_on_ci): try: import multiprocessing multiprocessing.Pool(2).close() diff --git a/tests/build/cythonize_options.srctree b/tests/build/cythonize_options.srctree index 56e38be61..fcef9645b 100644 --- a/tests/build/cythonize_options.srctree +++ b/tests/build/cythonize_options.srctree @@ -1,3 +1,5 @@ +# mode: run + PYTHON setup.py build_ext --inplace PYTHON -c "import a" @@ -5,11 +7,35 @@ PYTHON -c "import a" from Cython.Build.Dependencies import cythonize +import sys from distutils.core import setup +try: + # io.StringIO cannot handle 'str' in Py2 + from StringIO import StringIO +except ImportError: + from io import StringIO + +old_stderr = sys.stderr +captured = sys.stderr = StringIO() +try: + setup( + ext_modules = cythonize( + "*.pyx", include_path=['subdir'], + compiler_directives={'cdivision': True}, + show_all_warnings=True, + ), + ) + output = sys.stderr.getvalue() +finally: + sys.stderr = old_stderr + sys.stderr.write(captured.getvalue()) + +assert "Unraisable exception in function" in output, output + -setup( - ext_modules = cythonize("*.pyx", include_path=['subdir'], compiler_directives={'cdivision': True}), -) +######## subdir/x.pxd ######## + +######## subdir/y.pxi ######## ######## a.pyx ######## @@ -22,8 +48,6 @@ def mod_int_c(int a, int b): assert mod_int_c(-1, 10) < 0 - -######## subdir/x.pxd ######## - -######## subdir/y.pxi ######## - +# unraisable exceptions should produce a warning +cdef int no_exc_propagate(): + raise TypeError() diff --git a/tests/build/cythonize_pep420_namespace.srctree b/tests/build/cythonize_pep420_namespace.srctree new file mode 100644 index 000000000..99649376a --- /dev/null +++ b/tests/build/cythonize_pep420_namespace.srctree @@ -0,0 +1,59 @@ +PYTHON setup.py build_ext --inplace +PYTHON -c "import runner" + +######## setup.py ######## + +from Cython.Build.Dependencies import cythonize + +from distutils.core import setup, Extension + +setup( + ext_modules=cythonize([ + Extension("nsp.m1.a", ["nsp/m1/a.pyx"]), + Extension("nsp.m2.b", ["nsp/m2/b.pyx"]), + Extension("nsp.m3.c.d", ["nsp/m3/c/d.pyx"]), + ]), +) + +######## nsp/m1/__init__.py ######## + +######## nsp/m1/a.pyx ######## + +cdef class A: + pass + +######## nsp/m1/a.pxd ######## + +cdef class A: + pass + +######## nsp/m2/__init__.py ######## + +######## nsp/m2/b.pyx ######## + +from nsp.m1.a cimport A +from nsp.m3.c.d cimport D + +cdef class B(A): + pass + +######## nsp/m3/__init__.py ######## + +######## nsp/m3/c/d.pyx ######## + +cdef class D: + pass + +######## nsp/m3/c/d.pxd ######## + +cdef class D: + pass + +######## runner.py ######## + +from nsp.m1.a import A +from nsp.m2.b import B +from nsp.m3.c.d import D + +a = A() +b = B() diff --git a/tests/build/cythonize_with_annotate.srctree b/tests/build/cythonize_with_annotate.srctree new file mode 100644 index 000000000..f529d8620 --- /dev/null +++ b/tests/build/cythonize_with_annotate.srctree @@ -0,0 +1,45 @@ +PYTHON setup.py build_ext --inplace +PYTHON -c "import not_annotated; not_annotated.check()" +PYTHON -c "import default_annotated; default_annotated.check()" +PYTHON -c "import fullc_annotated; fullc_annotated.check()" +######## setup.py ######## + +from Cython.Build.Dependencies import cythonize + +from distutils.core import setup + +setup( + ext_modules = cythonize(["not_annotated.pyx"], language_level=3) + + cythonize(["default_annotated.pyx"], annotate=True, language_level=3) + + cythonize(["fullc_annotated.pyx"], annotate='fullc', language_level=3) +) +######## not_annotated.pyx ######## +# check that html-file doesn't exist: +def check(): + import os.path as os_path + assert not os_path.isfile(__name__+'.html') + + + +######## default_annotated.pyx ######## +# load html-site and check that the marker isn't there: +def check(): + from codecs import open + with open(__name__+'.html', 'r', 'utf8') as html_file: + html = html_file.read() + + from Cython.Compiler.Annotate import AnnotationCCodeWriter + assert (AnnotationCCodeWriter.COMPLETE_CODE_TITLE not in html) # per default no complete c code + + + +######## fullc_annotated.pyx ######## +# load html-site and check that the marker is there: +def check(): + from codecs import open + with open(__name__+'.html', 'r', 'utf8') as html_file: + html = html_file.read() + + from Cython.Compiler.Annotate import AnnotationCCodeWriter + assert (AnnotationCCodeWriter.COMPLETE_CODE_TITLE in html) + diff --git a/tests/build/cythonize_with_annotate_via_Options.srctree b/tests/build/cythonize_with_annotate_via_Options.srctree new file mode 100644 index 000000000..f08875506 --- /dev/null +++ b/tests/build/cythonize_with_annotate_via_Options.srctree @@ -0,0 +1,27 @@ +PYTHON setup.py build_ext --inplace +PYTHON -c "import fullc_annotated; fullc_annotated.check()" + +######## setup.py ######## + +from Cython.Build.Dependencies import cythonize +from Cython.Compiler import Options + +Options.annotate = 'fullc' + +from distutils.core import setup + +setup( + ext_modules = cythonize(["fullc_annotated.pyx"], language_level=3) +) + +######## fullc_annotated.pyx ######## +# load html-site and check that the marker is there: + +def check(): + from codecs import open + with open(__name__+'.html', 'r', 'utf8') as html_file: + html = html_file.read() + + from Cython.Compiler.Annotate import AnnotationCCodeWriter + assert (AnnotationCCodeWriter.COMPLETE_CODE_TITLE in html) + diff --git a/tests/build/cythonize_with_annotate_via_cli.srctree b/tests/build/cythonize_with_annotate_via_cli.srctree new file mode 100644 index 000000000..5ca615cd4 --- /dev/null +++ b/tests/build/cythonize_with_annotate_via_cli.srctree @@ -0,0 +1,36 @@ +CYTHONIZE -i -3 not_annotated.pyx +PYTHON -c "import not_annotated; not_annotated.check()" +CYTHONIZE -i -3 --annotate default_annotated.pyx +PYTHON -c "import default_annotated; default_annotated.check()" +CYTHONIZE -i -3 --annotate-fullc fullc_annotated.pyx +PYTHON -c "import fullc_annotated; fullc_annotated.check()" + +######## not_annotated.pyx ######## +# check that html-file doesn't exist: +def check(): + import os.path as os_path + assert not os_path.isfile(__name__+'.html') + + + +######## default_annotated.pyx ######## +# load html-site and check that the marker isn't there: +def check(): + from codecs import open + with open(__name__+'.html', 'r', 'utf8') as html_file: + html = html_file.read() + + from Cython.Compiler.Annotate import AnnotationCCodeWriter + assert (AnnotationCCodeWriter.COMPLETE_CODE_TITLE not in html) # per default no complete c code + + + +######## fullc_annotated.pyx ######## +# load html-site and check that the marker is there: +def check(): + from codecs import open + with open(__name__+'.html', 'r', 'utf8') as html_file: + html = html_file.read() + + from Cython.Compiler.Annotate import AnnotationCCodeWriter + assert (AnnotationCCodeWriter.COMPLETE_CODE_TITLE in html) diff --git a/tests/build/dotted.filename.modules.pxd b/tests/build/dotted.filename.modules.pxd new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/tests/build/dotted.filename.modules.pxd diff --git a/tests/build/dotted.filename.modules.pyx b/tests/build/dotted.filename.modules.pyx new file mode 100644 index 000000000..681cbf6c8 --- /dev/null +++ b/tests/build/dotted.filename.modules.pyx @@ -0,0 +1,7 @@ +# mode: compile +# tag: warnings + +_WARNINGS=""" +1:0: Dotted filenames ('dotted.filename.modules.pxd') are deprecated. Please use the normal Python package directory layout. +1:0: Dotted filenames ('dotted.filename.modules.pyx') are deprecated. Please use the normal Python package directory layout. +""" diff --git a/tests/build/inline_distutils.srctree b/tests/build/inline_distutils.srctree index 7f735796a..d0b21373d 100644 --- a/tests/build/inline_distutils.srctree +++ b/tests/build/inline_distutils.srctree @@ -38,4 +38,4 @@ namespace A { from my_lib cimport x -print x +print(x) diff --git a/tests/compile/branch_hints.pyx b/tests/compile/branch_hints.pyx new file mode 100644 index 000000000..575ee6cba --- /dev/null +++ b/tests/compile/branch_hints.pyx @@ -0,0 +1,91 @@ +# mode: compile +# tag: if, unlikely + +cimport cython + + +@cython.test_assert_path_exists( + "//IfClauseNode", + "//IfClauseNode[not(@branch_hint)]", +) +def if_simple(x): + if x: + x = 2 + + +@cython.test_assert_path_exists( + "//IfClauseNode", + "//IfClauseNode[not(@branch_hint)]", +) +def if_return(x): + if x: + return 1 + raise TypeError() + + +@cython.test_assert_path_exists( + "//IfClauseNode", + "//IfClauseNode[@branch_hint = 'unlikely']", +) +def if_raise_else(x): + if x: + raise TypeError() + else: + return 1 + + +@cython.test_assert_path_exists( + "//IfClauseNode", + "//IfClauseNode[@branch_hint = 'likely']", +) +def if_else_raise(x): + if x: + return 1 + else: + raise TypeError() + + +@cython.test_assert_path_exists( + "//IfClauseNode", + "//IfClauseNode[@branch_hint = 'unlikely']", +) +def if_raise_else_raise(x): + if x: + raise ValueError() + else: + raise TypeError() + + +@cython.test_assert_path_exists( + "//IfClauseNode", + "//IfClauseNode[@branch_hint = 'unlikely']", +) +@cython.test_fail_if_path_exists( + "//IfClauseNode[@branch_hint = 'likely']", + "//IfClauseNode[not(@branch_hint)]", +) +def if_elif_raise_else_raise(x): + if x: + raise ValueError() + elif not x: + raise AttributeError() + else: + raise TypeError() + + +@cython.test_assert_path_exists( + "//IfClauseNode", + "//IfClauseNode[@branch_hint = 'unlikely']", + "//IfClauseNode[@branch_hint = 'unlikely']//GILStatNode", +) +@cython.test_fail_if_path_exists( + "//IfClauseNode[@branch_hint = 'likely']", + "//IfClauseNode[not(@branch_hint)]", +) +cpdef int nogil_if_raise(int x) nogil except -1: + if x: + raise TypeError() + elif not x: + raise ValueError() + else: + x = 2 diff --git a/tests/compile/buildenv.pyx b/tests/compile/buildenv.pyx index ec2445c59..01f29883a 100644 --- a/tests/compile/buildenv.pyx +++ b/tests/compile/buildenv.pyx @@ -34,14 +34,15 @@ cdef extern from *: # Cython config cdef int CYTHON_COMPILING_IN_CPYTHON + cdef int CYTHON_COMPILING_IN_LIMITED_API cdef int CYTHON_COMPILING_IN_PYPY - cdef int CYTHON_COMPILING_IN_PYSTON cdef int CYTHON_USE_PYLONG_INTERNALS cdef int CYTHON_USE_PYLIST_INTERNALS cdef int CYTHON_USE_UNICODE_INTERNALS cdef int CYTHON_USE_UNICODE_WRITER cdef int CYTHON_AVOID_BORROWED_REFS cdef int CYTHON_ASSUME_SAFE_MACROS + cdef int CYTHON_USE_TYPE_SLOTS cdef int CYTHON_UNPACK_METHODS cdef int CYTHON_FAST_THREAD_STATE cdef int CYTHON_FAST_PYCALL @@ -76,8 +77,8 @@ Python {sys.version_info} PY_VERSION_HEX 0x{PY_VERSION_HEX:X} CYTHON_COMPILING_IN_CPYTHON {CYTHON_COMPILING_IN_CPYTHON} +CYTHON_COMPILING_IN_LIMITED_API {CYTHON_COMPILING_IN_LIMITED_API} CYTHON_COMPILING_IN_PYPY {CYTHON_COMPILING_IN_PYPY} -CYTHON_COMPILING_IN_PYSTON {CYTHON_COMPILING_IN_PYSTON} CYTHON_USE_PYLONG_INTERNALS {CYTHON_USE_PYLONG_INTERNALS} CYTHON_USE_PYLIST_INTERNALS {CYTHON_USE_PYLIST_INTERNALS} @@ -85,6 +86,7 @@ CYTHON_USE_UNICODE_INTERNALS {CYTHON_USE_UNICODE_INTERNALS} CYTHON_USE_UNICODE_WRITER {CYTHON_USE_UNICODE_WRITER} CYTHON_AVOID_BORROWED_REFS {CYTHON_AVOID_BORROWED_REFS} CYTHON_ASSUME_SAFE_MACROS {CYTHON_ASSUME_SAFE_MACROS} +CYTHON_USE_TYPE_SLOTS {CYTHON_USE_TYPE_SLOTS} CYTHON_UNPACK_METHODS {CYTHON_UNPACK_METHODS} CYTHON_FAST_THREAD_STATE {CYTHON_FAST_THREAD_STATE} CYTHON_FAST_PYCALL {CYTHON_FAST_PYCALL} @@ -107,6 +109,15 @@ SIZEOF_VOID_P {SIZEOF_VOID_P} ({sizeof(void*)}) SIZEOF_UINTPTR_T {SIZEOF_UINTPTR_T} ({sizeof(unsigned int *)}) SIZEOF_OFF_T {SIZEOF_OFF_T} +Paths: +sys.executable = {sys.executable} +sys.exec_prefix = {sys.exec_prefix} +sys.base_exec_prefix = {getattr(sys, 'base_exec_prefix', "")} +sys.prefix = {sys.prefix} +sys.path = {sys.path} +PYTHONPATH (env) = {get_env('PYTHONPATH', '')} +PYTHONHOME (env) = {get_env('PYTHONHOME', '')} + Distutils: INCDIR = {sysconfig.get_python_inc()} LIBS = {config_var('LIBS')} diff --git a/tests/compile/builtinbuffer.py b/tests/compile/builtinbuffer.py index c18ec1bf7..833fa9954 100644 --- a/tests/compile/builtinbuffer.py +++ b/tests/compile/builtinbuffer.py @@ -4,4 +4,3 @@ import cython @cython.cclass class BuiltinRef: cython.declare(pybuf = 'Py_buffer') - diff --git a/tests/compile/cascmp.pyx b/tests/compile/cascmp.pyx deleted file mode 100644 index c36997fbb..000000000 --- a/tests/compile/cascmp.pyx +++ /dev/null @@ -1,17 +0,0 @@ -# mode: compile - -cdef void foo(): - cdef int bool, int1=0, int2=0, int3=0, int4=0 - cdef object obj1, obj2, obj3, obj4 - obj1 = 1 - obj2 = 2 - obj3 = 3 - obj4 = 4 - bool = int1 < int2 < int3 - bool = obj1 < obj2 < obj3 - bool = int1 < int2 < obj3 - bool = obj1 < 2 < 3 - bool = obj1 < 2 < 3 < 4 - bool = int1 < (int2 == int3) < int4 - -foo() diff --git a/tests/compile/cast_ctypedef_array_T518.pyx b/tests/compile/cast_ctypedef_array_T518.pyx index a62f4cf4c..6a52374c3 100644 --- a/tests/compile/cast_ctypedef_array_T518.pyx +++ b/tests/compile/cast_ctypedef_array_T518.pyx @@ -1,4 +1,4 @@ -# ticket: 518 +# ticket: t518 # mode: compile cdef extern from "cast_ctypedef_array_T518_helper.h": diff --git a/tests/compile/cdefemptysue.pyx b/tests/compile/cdefemptysue.pyx new file mode 100644 index 000000000..1baf67d36 --- /dev/null +++ b/tests/compile/cdefemptysue.pyx @@ -0,0 +1,43 @@ +# mode: compile +# tag: struct, union, enum, cdefextern + +cdef extern from *: + """ + struct spam { int a; }; + struct flat_spam { int a; }; + typedef struct { int a; } flat_spam_type; + + typedef union { int a; long b; } eggs; + typedef union { int a; long b; } flat_eggs; + + enum ham { TOAST }; + enum flat_ham { FLAT_TOAST }; + """ + + cdef struct spam: + pass + + cdef struct flat_spam: pass + + ctypedef struct flat_spam_type: pass + + ctypedef union eggs: + pass + + ctypedef union flat_eggs: pass + + cdef enum ham: + pass + + cdef enum flat_ham: pass + + +cdef extern spam s +cdef extern flat_spam fs +cdef extern flat_spam_type fst + +cdef extern eggs e +cdef extern flat_eggs fe + +cdef extern ham h +cdef extern flat_ham fh diff --git a/tests/compile/cdefexternblock.pyx b/tests/compile/cdefexternblock.pyx new file mode 100644 index 000000000..865a59345 --- /dev/null +++ b/tests/compile/cdefexternblock.pyx @@ -0,0 +1,23 @@ +# mode: compile +# tag: struct, union, enum, cdefextern + +cdef extern from "cheese.h": + + ctypedef int camembert + + struct roquefort: + int x + + char *swiss + + void cheddar() + + # FIXME: find a real declaration here. + #class external.runny [object runny_obj]: + # cdef int a + # def __init__(self): + # pass + + +#cdef runny r = runny() +#r.a = 42 diff --git a/tests/compile/cimport_package_module_T4.pyx b/tests/compile/cimport_package_module_T4.pyx index 28403f122..717f0bf8f 100644 --- a/tests/compile/cimport_package_module_T4.pyx +++ b/tests/compile/cimport_package_module_T4.pyx @@ -1,4 +1,4 @@ -# ticket: 4 +# ticket: t4 # mode: compile from a cimport b diff --git a/tests/compile/cimportfrom_T248.pyx b/tests/compile/cimportfrom_T248.pyx index 13796baf1..1be252f3c 100644 --- a/tests/compile/cimportfrom_T248.pyx +++ b/tests/compile/cimportfrom_T248.pyx @@ -1,4 +1,4 @@ -# ticket: 248 +# ticket: t248 # mode: compile from ewing8 cimport (Foo, diff --git a/tests/compile/complex_annotations.pyx b/tests/compile/complex_annotations.pyx new file mode 100644 index 000000000..5d95611ac --- /dev/null +++ b/tests/compile/complex_annotations.pyx @@ -0,0 +1,7 @@ +# mode: compile + +# Complex numbers defined in annotations weren't having their utility code imported directly +# leading to compile-errors that the type wasn't defined. The test is intentionally minimal since +# anything more thorough ends up creating the utility code +cdef f(x: complex): + pass diff --git a/tests/compile/complex_decorators.pyx b/tests/compile/complex_decorators.pyx new file mode 100644 index 000000000..e8d65244d --- /dev/null +++ b/tests/compile/complex_decorators.pyx @@ -0,0 +1,10 @@ +# mode: compile + +cimport cython + +# Complex numbers defined in "cython.locals" weren't having their utility code imported directly +# leading to compile-errors that the type wasn't defined. The test is intentionally minimal since +# anything more thorough ends up creating the utility code +@cython.locals(x=complex) +cdef f(x): + pass diff --git a/tests/compile/const_decl.pyx b/tests/compile/const_decl.pyx index 5424c8f90..c47f952df 100644 --- a/tests/compile/const_decl.pyx +++ b/tests/compile/const_decl.pyx @@ -1,12 +1,17 @@ # mode: compile -cdef const_args(const int a, const int *b, const (int*) c, int *const d): +cdef const_args(const int a, const int *b, const (int*) c, int *const d, int **const e, int *const *f): print a print b[0] - b = NULL # OK, the pointer itself is not const - c[0] = 4 # OK, the value is not const - d[0] = 7 # OK, the value is not const + b = NULL # OK, the pointer itself is not const + c[0] = 4 # OK, the value is not const + d[0] = 7 # OK, the value is not const + e[0][0] = 1 # OK, the value is not const + e[0] = NULL # OK, the pointed pointer is not const + f[0][0] = 1 # OK, the value is not const + f = NULL # OK, the pointer is not const def call_const_args(x): cdef int k = x - const_args(x, &k, &k, &k) + cdef int* arr = [x] + const_args(x, &k, &k, &k, &arr, &arr) diff --git a/tests/compile/cpp_enums.h b/tests/compile/cpp_enums.h deleted file mode 100644 index 4ea444ee4..000000000 --- a/tests/compile/cpp_enums.h +++ /dev/null @@ -1,11 +0,0 @@ -enum Enum1 { - Item1, - Item2 -}; - -namespace Namespace1 { - enum Enum2 { - Item3, - Item4 - }; -} diff --git a/tests/compile/cpp_enums.pyx b/tests/compile/cpp_enums.pyx deleted file mode 100644 index b738dee05..000000000 --- a/tests/compile/cpp_enums.pyx +++ /dev/null @@ -1,27 +0,0 @@ -# tag: cpp -# mode: compile - -cdef extern from "cpp_enums.h": - cdef enum Enum1: - Item1 - Item2 - -a = Item1 -b = Item2 - -cdef Enum1 x, y -x = Item1 -y = Item2 - -cdef extern from "cpp_enums.h" namespace "Namespace1": - cdef enum Enum2: - Item3 - Item4 - -c = Item3 -d = Item4 - -cdef Enum2 z, w -z = Item3 -w = Item4 - diff --git a/tests/compile/cpp_rvalue_reference_binding.pyx b/tests/compile/cpp_rvalue_reference_binding.pyx new file mode 100644 index 000000000..492a950ac --- /dev/null +++ b/tests/compile/cpp_rvalue_reference_binding.pyx @@ -0,0 +1,22 @@ +# tag: cpp, cpp11 +# mode: compile + +cdef extern from *: + """ + template <typename T> + void accept(T&& x) { (void) x; } + """ + cdef void accept[T](T&& x) + +cdef int make_int_py() except *: + # might raise Python exception (thus needs a temp) + return 1 + +cdef int make_int_cpp() except +: + # might raise C++ exception (thus needs a temp) + return 1 + +def test_func_arg(): + # won't compile if move() isn't called on the temp: + accept(make_int_py()) + accept(make_int_cpp()) diff --git a/tests/compile/cpp_temp_assignment.pyx b/tests/compile/cpp_temp_assignment.pyx new file mode 100644 index 000000000..58ae39a70 --- /dev/null +++ b/tests/compile/cpp_temp_assignment.pyx @@ -0,0 +1,98 @@ +# tag: cpp,cpp11 +# mode: compile +# tag: no-cpp-locals +# TODO cpp_locals works fine with the standard library that comes with gcc11 +# but not with gcc8. Therefore disable the test for now + +cdef extern from *: + """ + class NoAssignIterator { + public: + explicit NoAssignIterator(int pos) : pos_(pos) {} + NoAssignIterator(NoAssignIterator&) = delete; + NoAssignIterator(NoAssignIterator&&) {} + NoAssignIterator& operator=(NoAssignIterator&) = delete; + NoAssignIterator& operator=(NoAssignIterator&&) { return *this; } + // Default constructor of temp variable is needed by Cython + // as of 3.0a6. + NoAssignIterator() : pos_(0) {} + int operator*() { + return pos_; + } + NoAssignIterator operator++() { + return NoAssignIterator(pos_ + 1); + } + int operator!=(NoAssignIterator other) { + return pos_ != other.pos_; + } + int pos_; + }; + class NoAssign { + public: + NoAssign() {} + NoAssign(NoAssign&) = delete; + NoAssign(NoAssign&&) {} + NoAssign& operator=(NoAssign&) = delete; + NoAssign& operator=(NoAssign&&) { return *this; } + void func() {} + NoAssignIterator begin() { + return NoAssignIterator(0); + } + NoAssignIterator end() { + return NoAssignIterator(2); + } + }; + + NoAssign get_NoAssign_Py() { + return NoAssign(); + } + NoAssign get_NoAssign_Cpp() { + return NoAssign(); + } + + """ + cdef cppclass NoAssignIterator: + int operator*() + NoAssignIterator operator++() + int operator!=(NoAssignIterator) + + cdef cppclass NoAssign: + void func() + NoAssignIterator begin() + NoAssignIterator end() + + # might raise Python exception (thus needs a temp) + NoAssign get_NoAssign_Py() except * + # might raise C++ exception (thus needs a temp) + NoAssign get_NoAssign_Cpp() except + + +cdef internal_cpp_func(NoAssign arg): + pass + +def test_call_to_function(): + # will fail to compile if move constructors aren't used + internal_cpp_func(get_NoAssign_Py()) + internal_cpp_func(get_NoAssign_Cpp()) + +def test_assignment_to_name(): + # will fail if move constructors aren't used + cdef NoAssign value + value = get_NoAssign_Py() + value = get_NoAssign_Cpp() + +def test_assignment_to_scope(): + cdef NoAssign value + value = get_NoAssign_Py() + value = get_NoAssign_Cpp() + def inner(): + value.func() + +cdef class AssignToClassAttr: + cdef NoAssign attr + def __init__(self): + self.attr = get_NoAssign_Py() + self.attr = get_NoAssign_Cpp() + +def test_generator_cpp_iterator_as_temp(): + for i in get_NoAssign_Py(): + yield i diff --git a/tests/compile/cpp_templates.pyx b/tests/compile/cpp_templates.pyx index cab981e38..7952a1610 100644 --- a/tests/compile/cpp_templates.pyx +++ b/tests/compile/cpp_templates.pyx @@ -1,6 +1,6 @@ # tag: cpp # mode: compile -# ticket: 767 +# ticket: t767 cdef extern from "templates.h": cdef cppclass TemplateTest1[T]: diff --git a/tests/compile/cppenum.pyx b/tests/compile/cppenum.pyx new file mode 100644 index 000000000..8431ac83b --- /dev/null +++ b/tests/compile/cppenum.pyx @@ -0,0 +1,31 @@ +# mode: compile +# tag: cpp,cpp11 + + +cpdef enum class Spam: + a, b + c + d + e + f = 42 + + +cpdef enum class Cheese(unsigned int): + x = 1 + y = 2 + + +cdef enum struct parrot_state: + alive = 1 + dead = 0 + + +cdef void eggs(): + cdef Spam s1 + s1 = Spam.a + s2 = Spam.b + + cdef Cheese c1 + c1 = Cheese.x + +eggs() diff --git a/tests/compile/crunchytype.h b/tests/compile/crunchytype.h deleted file mode 100644 index 6ea0e37c0..000000000 --- a/tests/compile/crunchytype.h +++ /dev/null @@ -1,5 +0,0 @@ - -struct CrunchyType { - int number; - PyObject* string; -}; diff --git a/tests/compile/crunchytype.pxd b/tests/compile/crunchytype.pxd index c03e38dad..c1f2b555b 100644 --- a/tests/compile/crunchytype.pxd +++ b/tests/compile/crunchytype.pxd @@ -1,4 +1,10 @@ -cdef extern from "crunchytype.h": +cdef extern from *: + """ + struct CrunchyType { + int number; + PyObject* string; + }; + """ cdef class crunchytype.Crunchy [ object CrunchyType ]: cdef int number cdef object string diff --git a/tests/compile/ctypedef_public_class_T355.pyx b/tests/compile/ctypedef_public_class_T355.pyx index 505f9d67f..ceb2d65e4 100644 --- a/tests/compile/ctypedef_public_class_T355.pyx +++ b/tests/compile/ctypedef_public_class_T355.pyx @@ -1,4 +1,4 @@ -# ticket: 355 +# ticket: t355 # mode: compile ctypedef public class Time [type MyTime_Type, object MyTimeObject]: diff --git a/tests/compile/cython_compiled_folding.pxd b/tests/compile/cython_compiled_folding.pxd new file mode 100644 index 000000000..4c85d4311 --- /dev/null +++ b/tests/compile/cython_compiled_folding.pxd @@ -0,0 +1 @@ +from libc.math cimport sin, cos, sqrt, tan, log diff --git a/tests/compile/cython_compiled_folding.py b/tests/compile/cython_compiled_folding.py new file mode 100644 index 000000000..047b61689 --- /dev/null +++ b/tests/compile/cython_compiled_folding.py @@ -0,0 +1,39 @@ +# mode: compile + +# libc sin, cos and sqrt cimported in the pxd file + +import cython +from cython import compiled + +if not cython.compiled: + from math import sin + +if cython.compiled: + pass +else: + from math import cos + +if "aa" == "bb": + pass +elif cython.compiled: + pass +elif True: + from math import sqrt + +if "aa" == "bb": + pass +elif compiled: + pass +else: + from math import tan + +# log10 isn't defined in the pxd file +from math import log10 + +@cython.test_fail_if_path_exists("//FromImportStatNode//ImportNode") +@cython.test_assert_path_exists("//AddNode") +def import_log(x, y): + if compiled: + return x+y + else: + from math import log diff --git a/tests/compile/ellipsis_T488.pyx b/tests/compile/ellipsis_T488.pyx index d90d21634..804b18497 100644 --- a/tests/compile/ellipsis_T488.pyx +++ b/tests/compile/ellipsis_T488.pyx @@ -1,4 +1,4 @@ -# ticket: 488 +# ticket: t488 # mode: compile #from ... import foo diff --git a/tests/compile/find_pxd.srctree b/tests/compile/find_pxd.srctree index 75d2765c9..23fc5c5c7 100644 --- a/tests/compile/find_pxd.srctree +++ b/tests/compile/find_pxd.srctree @@ -41,7 +41,7 @@ ctypedef int my_type ######## path/numpy/__init__.pxd ######## -# gh-2905: This should be found before Cython/Inlude/numpy/__init__.pxd +# gh-2905: This should be found before Cython/Includes/numpy/__init__.pxd ctypedef int my_type diff --git a/tests/compile/fused_redeclare_T3111.pyx b/tests/compile/fused_redeclare_T3111.pyx index baf932bd4..80e211fad 100644 --- a/tests/compile/fused_redeclare_T3111.pyx +++ b/tests/compile/fused_redeclare_T3111.pyx @@ -22,10 +22,14 @@ def foo(dtype_t[:] a, dtype_t_out[:, :] b): # "__pyxutil:16:4: '___pyx_npy_uint8' redeclared". The remaining warnings are # unrelated to this test. _WARNINGS = """ -20:10: 'cpdef_method' redeclared -31:10: 'cpdef_cname_method' redeclared -446:72: Argument evaluation order in C function call is undefined and may not be as expected -446:72: Argument evaluation order in C function call is undefined and may not be as expected -749:34: Argument evaluation order in C function call is undefined and may not be as expected -749:34: Argument evaluation order in C function call is undefined and may not be as expected +# cpdef redeclaration bug, from TestCythonScope.pyx +25:10: 'cpdef_method' redeclared +36:10: 'cpdef_cname_method' redeclared +# from MemoryView.pyx +987:29: Ambiguous exception value, same as default return value: 0 +987:29: Ambiguous exception value, same as default return value: 0 +1014:46: Ambiguous exception value, same as default return value: 0 +1014:46: Ambiguous exception value, same as default return value: 0 +1104:29: Ambiguous exception value, same as default return value: 0 +1104:29: Ambiguous exception value, same as default return value: 0 """ diff --git a/tests/compile/fused_unused.pyx b/tests/compile/fused_unused.pyx new file mode 100644 index 000000000..9aac0b7fb --- /dev/null +++ b/tests/compile/fused_unused.pyx @@ -0,0 +1,9 @@ +# mode: compile +# tag: fused + +# This previously lead to a crash due to an empty module body. + +ctypedef fused cinteger: + int + long + Py_ssize_t diff --git a/tests/compile/fused_wraparound.pyx b/tests/compile/fused_wraparound.pyx new file mode 100644 index 000000000..e9facf9c1 --- /dev/null +++ b/tests/compile/fused_wraparound.pyx @@ -0,0 +1,22 @@ +# mode: compile +# tag: fused, werror + +""" +Very short test for https://github.com/cython/cython/issues/3492 +(Needs its own file since werror fails on the main fused-test files) +wraparound and boundscheck directives shouldn't break the fused +dispatcher function +""" + +cimport cython + +ctypedef fused fused_t: + str + int + long + complex + +@cython.wraparound(False) +@cython.boundscheck(False) +def func(fused_t a, cython.floating b): + return a, b diff --git a/tests/compile/extern_packed_struct.pyx b/tests/compile/packed_structs.pyx index 6df4d20d0..6e722b46d 100644 --- a/tests/compile/extern_packed_struct.pyx +++ b/tests/compile/packed_structs.pyx @@ -3,3 +3,8 @@ cdef extern from *: cdef packed struct MyStruct: char a + +cdef public packed struct PublicStruct: + int a + unsigned char b + int c diff --git a/tests/compile/tree_assertions.pyx b/tests/compile/tree_assertions.pyx new file mode 100644 index 000000000..e311bfd25 --- /dev/null +++ b/tests/compile/tree_assertions.pyx @@ -0,0 +1,20 @@ +# mode: compile + +# This is a sort of meta test - to test the functionality of "test_assert_path_exists" + +cimport cython + +@cython.test_assert_path_exists("//ReturnStatNode") +def not_in_inner_compiler_directives(): + # used to fail because ReturnStatNode wasn't in *this* CompilerDirectivesNode + with cython.boundscheck(False): + pass + return 1 # should pass + +@cython.test_assert_path_exists("//ReturnStatNode") +def in_inner_compiler_directives(): + # used to fail because ReturnStatNode wasn't in *this* CompilerDirectivesNode + with cython.boundscheck(False): + return 1 + +# it's hard to come up with a corresponding test for fail_if_path_exists.. diff --git a/tests/compile/types_and_names.pyx b/tests/compile/types_and_names.pyx index 8ded94e4d..2637d4ba6 100644 --- a/tests/compile/types_and_names.pyx +++ b/tests/compile/types_and_names.pyx @@ -23,3 +23,16 @@ cdef A a foo(2, 3, [], [], P, P, &P) a.point("something", 3, "anything", [], "an object", P, &P) + +# Test that internally generated names do not conflict. +cdef class A_spec: + pass + +cdef class A_members: + pass + +cdef class A_methods: + pass + +cdef class A_slots: + pass diff --git a/tests/compile/volatile.pyx b/tests/compile/volatile.pyx new file mode 100644 index 000000000..d69d8b355 --- /dev/null +++ b/tests/compile/volatile.pyx @@ -0,0 +1,17 @@ +# mode: compile + +cdef volatile int x = 1 + +cdef const volatile char* greeting1 = "hello world" +cdef volatile const char* greeting2 = "goodbye" + + +cdef extern from "stdlib.h": + volatile void* malloc(size_t) + +cdef volatile long* test(volatile size_t s): + cdef volatile long* arr = <long*><volatile long*>malloc(s) + return arr + + +test(64) diff --git a/tests/compile/weakref_T276.pyx b/tests/compile/weakref_T276.pyx index d56f67303..7fbc52819 100644 --- a/tests/compile/weakref_T276.pyx +++ b/tests/compile/weakref_T276.pyx @@ -1,4 +1,4 @@ -# ticket: 276 +# ticket: t276 # mode: compile __doc__ = u""" diff --git a/tests/errors/bufaccess_noassignT444.pyx b/tests/errors/bufaccess_noassignT444.pyx index 44d2ebd0b..c618106fb 100644 --- a/tests/errors/bufaccess_noassignT444.pyx +++ b/tests/errors/bufaccess_noassignT444.pyx @@ -1,4 +1,4 @@ -# ticket: 444 +# ticket: t444 # mode: error def test(): diff --git a/tests/errors/buffertypedef_T117.pyx b/tests/errors/buffertypedef_T117.pyx index d88a3895b..cefa79939 100644 --- a/tests/errors/buffertypedef_T117.pyx +++ b/tests/errors/buffertypedef_T117.pyx @@ -1,4 +1,4 @@ -# ticket: 117 +# ticket: t117 # mode: error ctypedef object[float] mybuffer diff --git a/tests/errors/callingnonexisting_T307.pyx b/tests/errors/callingnonexisting_T307.pyx index e9409fcab..ac767e581 100644 --- a/tests/errors/callingnonexisting_T307.pyx +++ b/tests/errors/callingnonexisting_T307.pyx @@ -1,4 +1,4 @@ -# ticket: 307 +# ticket: t307 # mode: error nonexisting(3, with_kw_arg=4) diff --git a/tests/errors/cdef_class_properties_decorated.pyx b/tests/errors/cdef_class_properties_decorated.pyx index 7485d8891..f796b7128 100644 --- a/tests/errors/cdef_class_properties_decorated.pyx +++ b/tests/errors/cdef_class_properties_decorated.pyx @@ -1,5 +1,5 @@ # mode: error -# ticket: 264 +# ticket: t264 # tag: property, decorator diff --git a/tests/errors/cdef_func_decorators.pyx b/tests/errors/cdef_func_decorators.pyx new file mode 100644 index 000000000..e249b2e97 --- /dev/null +++ b/tests/errors/cdef_func_decorators.pyx @@ -0,0 +1,39 @@ +# mode: error +# tag: decorator + +from functools import wraps + +@wraps +cdef cant_be_decoratored(): + pass + +@wraps +cpdef also_cant_be_decorated(): + pass + +cdef class C: + @wraps + cdef still_cant_be_decorated(self): + pass + + @property + cdef property_only_works_for_extern_classes(self): + pass + + @wraps + cpdef also_still_cant_be_decorated(self): + pass + + @wraps + @wraps + cdef two_is_just_as_bad_as_one(self): + pass + +_ERRORS = """ +6:0: Cdef functions cannot take arbitrary decorators. +10:0: Cdef functions cannot take arbitrary decorators. +15:4: Cdef functions cannot take arbitrary decorators. +19:4: Cdef functions cannot take arbitrary decorators. +23:4: Cdef functions cannot take arbitrary decorators. +27:4: Cdef functions cannot take arbitrary decorators. +""" diff --git a/tests/errors/cdef_members_T517.pyx b/tests/errors/cdef_members_T517.pyx index 9bd3b111c..34bfd79fd 100644 --- a/tests/errors/cdef_members_T517.pyx +++ b/tests/errors/cdef_members_T517.pyx @@ -1,4 +1,4 @@ -# ticket: 517 +# ticket: t517 # mode: error ctypedef void* VoidP diff --git a/tests/errors/cdefkwargs.pyx b/tests/errors/cdefkwargs.pyx index a4477e2da..2fedeb04a 100644 --- a/tests/errors/cdefkwargs.pyx +++ b/tests/errors/cdefkwargs.pyx @@ -6,10 +6,6 @@ __doc__ = u""" >>> call4() """ -import sys, re -if sys.version_info >= (2,6): - __doc__ = re.sub(u"Error: (.*)exactly(.*)", u"Error: \\1at most\\2", __doc__) - # the calls: def call2(): diff --git a/tests/errors/cfunc_directive_in_pyclass.pyx b/tests/errors/cfunc_directive_in_pyclass.pyx index bb12dc271..d6b373a40 100644 --- a/tests/errors/cfunc_directive_in_pyclass.pyx +++ b/tests/errors/cfunc_directive_in_pyclass.pyx @@ -7,5 +7,5 @@ class Pyclass(object): pass _ERRORS = """ - 6:4: cfunc directive is not allowed here + 5:4: cfunc directive is not allowed here """ diff --git a/tests/errors/cimport_attributes.pyx b/tests/errors/cimport_attributes.pyx index 53e303a42..7037b9aa3 100644 --- a/tests/errors/cimport_attributes.pyx +++ b/tests/errors/cimport_attributes.pyx @@ -1,4 +1,5 @@ # mode: error +# tag: cpp cimport libcpp @@ -24,8 +25,8 @@ print my_map_with_shadow.python_attribute # OK (if such a module existed at ru _ERRORS = u""" -5:12: cimported module has no attribute 'no_such_attribute' -8:16: cimported module has no attribute 'no_such_attribute' -11:12: cimported module has no attribute 'no_such_attribute' -14:15: cimported module has no attribute 'no_such_attribute' +6:12: cimported module has no attribute 'no_such_attribute' +9:16: cimported module has no attribute 'no_such_attribute' +12:12: cimported module has no attribute 'no_such_attribute' +15:15: cimported module has no attribute 'no_such_attribute' """ diff --git a/tests/errors/compile_time_unraisable_T370.pyx b/tests/errors/compile_time_unraisable_T370.pyx index 963f280fb..d22f0c6c0 100644 --- a/tests/errors/compile_time_unraisable_T370.pyx +++ b/tests/errors/compile_time_unraisable_T370.pyx @@ -1,4 +1,4 @@ -# ticket: 370 +# ticket: t370 # mode: error cdef int raiseit(): diff --git a/tests/errors/const_decl_errors.pyx b/tests/errors/const_decl_errors.pyx index 459480adb..29c9896ea 100644 --- a/tests/errors/const_decl_errors.pyx +++ b/tests/errors/const_decl_errors.pyx @@ -10,23 +10,34 @@ cdef const int x = 10 cdef struct S: int member -cdef func(const int a, const int* b, const (int*) c, const S s, int *const d, +cdef func(const int a, const int* b, const (int*) c, const S s, int *const d, int **const e, int *const *f, const S *const t): a = 10 c = NULL b[0] = 100 s.member = 1000 d = NULL + e[0][0] = 1 # ok + e[0] = NULL # ok + e = NULL # nok + f[0][0] = 1 # ok + f[0] = NULL # nok + f = NULL # ok t = &s +cdef volatile object v + _ERRORS = """ -3:5: Const base type cannot be a Python object +3:5: Const/volatile base type cannot be a Python object 8:5: Assignment to const 'x' 15:4: Assignment to const 'a' 16:4: Assignment to const 'c' 17:5: Assignment to const dereference 18:5: Assignment to const attribute 'member' 19:4: Assignment to const 'd' -20:4: Assignment to const 't' +22:4: Assignment to const 'e' +24:5: Assignment to const dereference +26:4: Assignment to const 't' +28:5: Const/volatile base type cannot be a Python object """ diff --git a/tests/errors/cpdef_vars.pyx b/tests/errors/cpdef_vars.pyx index f356decfc..b317d15b0 100644 --- a/tests/errors/cpdef_vars.pyx +++ b/tests/errors/cpdef_vars.pyx @@ -1,4 +1,4 @@ -# tag: warnings +# mode: error cpdef str a = "123" cpdef b = 2 @@ -16,9 +16,9 @@ def func(): return d -_WARNINGS = """ -3:6: cpdef variables will not be supported in Cython 3; currently they are no different from cdef variables -4:6: cpdef variables will not be supported in Cython 3; currently they are no different from cdef variables -7:10: cpdef variables will not be supported in Cython 3; currently they are no different from cdef variables -15:10: cpdef variables will not be supported in Cython 3; currently they are no different from cdef variables +_ERRORS = """ +3:6: Variables cannot be declared with 'cpdef'. Use 'cdef' instead. +4:6: Variables cannot be declared with 'cpdef'. Use 'cdef' instead. +7:10: Variables cannot be declared with 'cpdef'. Use 'cdef' instead. +15:10: Variables cannot be declared with 'cpdef'. Use 'cdef' instead. """ diff --git a/tests/errors/cpp_enum_redeclare.pyx b/tests/errors/cpp_enum_redeclare.pyx new file mode 100644 index 000000000..3d2ae7763 --- /dev/null +++ b/tests/errors/cpp_enum_redeclare.pyx @@ -0,0 +1,13 @@ +# mode: error +# tag: cpp + +cdef enum class Spam: + a + +cdef enum class Spam: + b + +_ERRORS=""" +7:5: 'Spam' redeclared +4:5: Previous declaration is here +""" diff --git a/tests/errors/cpp_object_template.pyx b/tests/errors/cpp_object_template.pyx index a90bdedff..93d6c28da 100644 --- a/tests/errors/cpp_object_template.pyx +++ b/tests/errors/cpp_object_template.pyx @@ -1,4 +1,5 @@ # mode: error +# tag: cpp from libcpp.vector cimport vector @@ -12,6 +13,6 @@ def main(): va.push_back(A()) _ERRORS = u""" -9:16: Python object type 'Python object' cannot be used as a template argument -11:16: Python object type 'A' cannot be used as a template argument +10:16: Python object type 'Python object' cannot be used as a template argument +12:16: Python object type 'A' cannot be used as a template argument """ diff --git a/tests/errors/cpp_rvalue_reference_support.pyx b/tests/errors/cpp_rvalue_reference_support.pyx new file mode 100644 index 000000000..e1e7015c9 --- /dev/null +++ b/tests/errors/cpp_rvalue_reference_support.pyx @@ -0,0 +1,32 @@ +# mode: error +# tag: werror, cpp, cpp11 + +# These tests check for unsupported use of rvalue-references (&&) +# and should be removed or cleaned up when support is added. + +cdef int&& x + +cdef void foo(int&& x): + pass + +cdef int&& bar(): + pass + +cdef extern from *: + """ + void baz(int x, int&& y) {} + + template <typename T> + void qux(const T&& x) {} + """ + cdef void baz(int x, int&& y) + cdef void qux[T](const T&& x) + + +_ERRORS=""" +7:8: C++ rvalue-references cannot be declared +9:13: Rvalue-reference as function argument not supported +12:14: Rvalue-reference as function return type not supported +22:17: Rvalue-reference as function argument not supported +23:20: Rvalue-reference as function argument not supported +""" diff --git a/tests/errors/cppexc_non_extern.pyx b/tests/errors/cppexc_non_extern.pyx new file mode 100644 index 000000000..95427e6e4 --- /dev/null +++ b/tests/errors/cppexc_non_extern.pyx @@ -0,0 +1,22 @@ +# mode: error +# tag: warnings + +cdef inline void handle_exception(): + pass + +# GH 3064 - cppfunc caused invalid code to be generated with +handle_exception +# error to prevent this +cdef test_func1(self) except +handle_exception: + pass + +# warning +cdef test_func2(self) except +: + pass + +_ERRORS = """ +9:16: Only extern functions can throw C++ exceptions. +""" + +_WARNINGS = """ +13:16: Only extern functions can throw C++ exceptions. +""" diff --git a/tests/errors/declareafteruse_T158.pyx b/tests/errors/declareafteruse_T158.pyx index 34e5f097e..a6ff6da13 100644 --- a/tests/errors/declareafteruse_T158.pyx +++ b/tests/errors/declareafteruse_T158.pyx @@ -1,4 +1,4 @@ -# ticket: 158 +# ticket: t158 # mode: error def mult_decl_test(): @@ -52,26 +52,19 @@ cdef int *baz print var[0][0] cdef unsigned long long[100][100] var -# in 0.11.1 these are warnings -FUTURE_ERRORS = u""" -6:13: cdef variable 's' declared after it is used -6:16: cdef variable 'vv' declared after it is used -11:14: cdef variable 'i' declared after it is used -17:14: cdef variable 'i' declared after it is used -23:14: cdef variable 'i' declared after it is used -26:9: cdef variable 's' declared after it is used -32:17: cdef variable 't' declared after it is used -36:13: cdef variable 'r' declared after it is used -42:17: cdef variable 't' declared after it is used -49:10: cdef variable 'baz' declared after it is used -52:24: cdef variable 'var' declared after it is used -""" - -syntax error - _ERRORS = u""" -42:17: cdef variable 't' declared after it is used -49:10: cdef variable 'baz' declared after it is used -52:24: cdef variable 'var' declared after it is used -70:7: Syntax error in simple statement list +5:17: local variable 'vv' referenced before assignment +6:17: local variable 's' referenced before assignment +7:13: cdef variable 's' declared after it is used +7:16: cdef variable 'vv' declared after it is used +12:14: cdef variable 'i' declared after it is used +18:14: cdef variable 'i' declared after it is used +24:14: cdef variable 'i' declared after it is used +27:9: cdef variable 's' declared after it is used +33:17: cdef variable 't' declared after it is used +43:17: cdef variable 't' declared after it is used +50:10: cdef variable 'baz' declared after it is used +53:34: cdef variable 'var' declared after it is used """ +# FIXME not detected +#37:13: cdef variable 'r' declared after it is used diff --git a/tests/errors/duplicate_const.pyx b/tests/errors/duplicate_const.pyx new file mode 100644 index 000000000..1367dd13b --- /dev/null +++ b/tests/errors/duplicate_const.pyx @@ -0,0 +1,13 @@ +# mode: error + +cdef extern from *: + cdef const const int a + cdef const volatile int b + cdef volatile const int c + cdef volatile volatile int d + + +_ERRORS = """ +4:9: Duplicate 'const' +7:9: Duplicate 'volatile' +""" diff --git a/tests/errors/e2_packedstruct_T290.pyx b/tests/errors/e2_packedstruct_T290.pyx index 1a0f40f18..084b36d71 100644 --- a/tests/errors/e2_packedstruct_T290.pyx +++ b/tests/errors/e2_packedstruct_T290.pyx @@ -1,4 +1,4 @@ -# ticket: 290 +# ticket: t290 # mode: error cdef packed foo: diff --git a/tests/errors/e_assert.pyx b/tests/errors/e_assert.pyx new file mode 100644 index 000000000..616d86ff1 --- /dev/null +++ b/tests/errors/e_assert.pyx @@ -0,0 +1,25 @@ +# mode: error +# tag: assert + +def nontrivial_assert_in_nogil(int a, obj): + with nogil: + # NOK + assert obj + assert a*obj + assert obj, "abc" + + # OK + assert a + assert a*a + assert a, "abc" + assert a, u"abc" + assert a, f"123{a}xyz" + + +_ERRORS = """ +7:15: Truth-testing Python object not allowed without gil +8:15: Converting to Python object not allowed without gil +8:16: Operation not allowed without gil +8:16: Truth-testing Python object not allowed without gil +9:15: Truth-testing Python object not allowed without gil +""" diff --git a/tests/errors/e_autotestdict.pyx b/tests/errors/e_autotestdict.pyx index a5d56f3d2..097b09fb2 100644 --- a/tests/errors/e_autotestdict.pyx +++ b/tests/errors/e_autotestdict.pyx @@ -7,5 +7,5 @@ def foo(): pass _ERRORS = u""" -6:0: The autotestdict compiler directive is not allowed in function scope +5:0: The autotestdict compiler directive is not allowed in function scope """ diff --git a/tests/errors/e_binop_and.pyx b/tests/errors/e_binop_and.pyx new file mode 100644 index 000000000..195677bdf --- /dev/null +++ b/tests/errors/e_binop_and.pyx @@ -0,0 +1,14 @@ +# mode: error +# tag: and, binop, warnings + +def test_and(a, b): + return a && b + + +_WARNINGS = """ +5:13: Found the C operator '&&', did you mean the Python operator 'and'? +""" + +_ERRORS = """ +5:13: Syntax error in simple statement list +""" diff --git a/tests/errors/e_binop_or.pyx b/tests/errors/e_binop_or.pyx new file mode 100644 index 000000000..1e4109244 --- /dev/null +++ b/tests/errors/e_binop_or.pyx @@ -0,0 +1,14 @@ +# mode: error +# tag: or, binop, warnings + +def test_or(a, b): + return a || b + + +_WARNINGS = """ +5:13: Found the C operator '||', did you mean the Python operator 'or'? +""" + +_ERRORS = """ +5:13: Syntax error in simple statement list +""" diff --git a/tests/errors/e_cdef_keywords_T241.pyx b/tests/errors/e_cdef_keywords_T241.pyx index 87524ebdd..28a2783fe 100644 --- a/tests/errors/e_cdef_keywords_T241.pyx +++ b/tests/errors/e_cdef_keywords_T241.pyx @@ -1,4 +1,4 @@ -# ticket: 241 +# ticket: t241 # mode: error cdef some_function(x, y): diff --git a/tests/errors/e_cdefemptysue.pyx b/tests/errors/e_cdefemptysue.pyx index f71370f4a..7f218b1fe 100644 --- a/tests/errors/e_cdefemptysue.pyx +++ b/tests/errors/e_cdefemptysue.pyx @@ -8,8 +8,21 @@ ctypedef union eggs: cdef enum ham: pass + + +cdef struct flat_spam: pass + +ctypedef union flat_eggs: pass + +cdef enum flat_ham: pass + + _ERRORS = u""" 3:5: Empty struct or union definition not allowed outside a 'cdef extern from' block 6:0: Empty struct or union definition not allowed outside a 'cdef extern from' block 9:5: Empty enum definition not allowed outside a 'cdef extern from' block + +13:5: Empty struct or union definition not allowed outside a 'cdef extern from' block +15:0: Empty struct or union definition not allowed outside a 'cdef extern from' block +17:5: Empty enum definition not allowed outside a 'cdef extern from' block """ diff --git a/tests/errors/e_cenum_with_type.pyx b/tests/errors/e_cenum_with_type.pyx new file mode 100644 index 000000000..43c0e08fc --- /dev/null +++ b/tests/errors/e_cenum_with_type.pyx @@ -0,0 +1,8 @@ +# mode: error + +cdef enum Spam(int): + a, b + +_ERRORS = u""" +3:14: Expected ':', found '(' +""" diff --git a/tests/errors/e_cpp_references.pyx b/tests/errors/e_cpp_references.pyx new file mode 100644 index 000000000..39ace0a13 --- /dev/null +++ b/tests/errors/e_cpp_references.pyx @@ -0,0 +1,10 @@ +# mode: error +# tag: cpp, cpp11 + +cdef foo(object& x): pass +cdef bar(object&& x): pass + +_ERRORS=""" +4:15: Reference base type cannot be a Python object +5:15: Rvalue-reference base type cannot be a Python object +""" diff --git a/tests/errors/e_decorators.pyx b/tests/errors/e_decorators.pyx index 5abc1fc29..33ef2355d 100644 --- a/tests/errors/e_decorators.pyx +++ b/tests/errors/e_decorators.pyx @@ -1,13 +1,12 @@ # mode: error -_ERRORS = u""" -4:4 Expected a newline after decorator -""" - - class A: pass @A().a def f(): pass + +_ERRORS = u""" +6:4: Expected a newline after decorator +""" diff --git a/tests/errors/e_exttype_total_ordering.pyx b/tests/errors/e_exttype_total_ordering.pyx new file mode 100644 index 000000000..6d5dd34e6 --- /dev/null +++ b/tests/errors/e_exttype_total_ordering.pyx @@ -0,0 +1,178 @@ +# mode: error +# tag: total_ordering, warnings + +cimport cython + + +# Test all combinations with not enough methods. + +@cython.total_ordering +cdef class ExtNoFuncs: + pass + +@cython.total_ordering +cdef class ExtGe: + def __ge__(self, other): + return False + +@cython.total_ordering +cdef class ExtLe: + def __le__(self, other): + return False + +@cython.total_ordering +cdef class ExtLeGe: + def __le__(self, other): + return False + + def __ge__(self, other): + return False + +@cython.total_ordering +cdef class ExtGt: + def __gt__(self, other): + return False + +@cython.total_ordering +cdef class ExtGtGe: + def __gt__(self, other): + return False + + def __ge__(self, other): + return False + +@cython.total_ordering +cdef class ExtGtLe: + def __gt__(self, other): + return False + + def __le__(self, other): + return False + +@cython.total_ordering +cdef class ExtGtLeGe: + def __gt__(self, other): + return False + + def __le__(self, other): + return False + + def __ge__(self, other): + return False + +@cython.total_ordering +cdef class ExtLt: + def __lt__(self, other): + return False + +@cython.total_ordering +cdef class ExtLtGe: + def __lt__(self, other): + return False + + def __ge__(self, other): + return False + +@cython.total_ordering +cdef class ExtLtLe: + def __lt__(self, other): + return False + + def __le__(self, other): + return False + +@cython.total_ordering +cdef class ExtLtLeGe: + def __lt__(self, other): + return False + + def __le__(self, other): + return False + + def __ge__(self, other): + return False + +@cython.total_ordering +cdef class ExtLtGt: + def __lt__(self, other): + return False + + def __gt__(self, other): + return False + +@cython.total_ordering +cdef class ExtLtGtGe: + def __lt__(self, other): + return False + + def __gt__(self, other): + return False + + def __ge__(self, other): + return False + +@cython.total_ordering +cdef class ExtLtGtLe: + def __lt__(self, other): + return False + + def __gt__(self, other): + return False + + def __le__(self, other): + return False + +@cython.total_ordering +cdef class ExtLtGtLeGe: + def __lt__(self, other): + return False + + def __gt__(self, other): + return False + + def __le__(self, other): + return False + + def __ge__(self, other): + return False + +@cython.total_ordering +cdef class ExtNe: + def __ne__(self, other): + return False + +@cython.total_ordering +cdef class ExtEq: + def __eq__(self, other): + return False + +@cython.total_ordering +cdef class ExtEqNe: + def __eq__(self, other): + return False + + def __ne__(self, other): + return False + + +_WARNINGS = """ +10:5: total_ordering directive used, but no comparison and equality methods defined +14:5: total_ordering directive used, but no equality method defined +19:5: total_ordering directive used, but no equality method defined +24:5: total_ordering directive used, but no equality method defined +32:5: total_ordering directive used, but no equality method defined +37:5: total_ordering directive used, but no equality method defined +45:5: total_ordering directive used, but no equality method defined +53:5: total_ordering directive used, but no equality method defined +64:5: total_ordering directive used, but no equality method defined +69:5: total_ordering directive used, but no equality method defined +77:5: total_ordering directive used, but no equality method defined +85:5: total_ordering directive used, but no equality method defined +96:5: total_ordering directive used, but no equality method defined +104:5: total_ordering directive used, but no equality method defined +115:5: total_ordering directive used, but no equality method defined +126:5: total_ordering directive used, but no equality method defined +140:5: total_ordering directive used, but no comparison methods defined +145:5: total_ordering directive used, but no comparison methods defined +150:5: total_ordering directive used, but no comparison methods defined +""" diff --git a/tests/errors/e_int_literals_py2.py b/tests/errors/e_int_literals_py2.py index 9eef36673..750214e5e 100644 --- a/tests/errors/e_int_literals_py2.py +++ b/tests/errors/e_int_literals_py2.py @@ -3,7 +3,7 @@ def int_literals(): a = 1L # ok - b = 10000000000000L # ok + b = 10000000000000L # ok c = 1UL d = 10000000000000UL e = 10000000000000LL diff --git a/tests/errors/e_pure_cimports.pyx b/tests/errors/e_pure_cimports.pyx new file mode 100644 index 000000000..231a95959 --- /dev/null +++ b/tests/errors/e_pure_cimports.pyx @@ -0,0 +1,31 @@ +# mode: error +# tag: pure, import, cimport + +import cython.cimportsy # FIXME: not currently an error? + +import cython.cimports +import cython.cimports.libc +import cython.cimports as cim + +cimport cython.cimports +cimport cython.cimports.libc +cimport cython.cimports as cim +import cython.cimports.libc as cython + + +# ok +import cython.cimports.libc as libc +from cython.cimports import libc +from cython.cimports cimport libc + + +_ERRORS = """ +6:7: Cannot cimport the 'cython.cimports' package directly, only submodules. +7:7: Python cimports must use 'from cython.cimports... import ...' or 'import ... as ...', not just 'import ...' +8:7: Cannot cimport the 'cython.cimports' package directly, only submodules. +10:8: Cannot cimport the 'cython.cimports' package directly, only submodules. +11:8: Python cimports must use 'from cython.cimports... import ...' or 'import ... as ...', not just 'import ...' +12:8: Cannot cimport the 'cython.cimports' package directly, only submodules. +# The following is not an accurate error message, but it's difficult to distinguish this case. And it's rare. +13:7: Python cimports must use 'from cython.cimports... import ...' or 'import ... as ...', not just 'import ...' +""" diff --git a/tests/errors/e_tuple_args_T692.py b/tests/errors/e_tuple_args_T692.py index 715433d01..3ca30519b 100644 --- a/tests/errors/e_tuple_args_T692.py +++ b/tests/errors/e_tuple_args_T692.py @@ -1,4 +1,4 @@ -# ticket: 692 +# ticket: t692 # mode: error def func((a, b)): @@ -9,4 +9,3 @@ _ERRORS = u""" 5:11: undeclared name not builtin: a 5:15: undeclared name not builtin: b """ - diff --git a/tests/errors/fused_types.pyx b/tests/errors/fused_types.pyx index 438e46584..378ac5506 100644 --- a/tests/errors/fused_types.pyx +++ b/tests/errors/fused_types.pyx @@ -1,4 +1,5 @@ # mode: error +# ticket: 1772 cimport cython from cython import fused_type @@ -48,6 +49,18 @@ def outer(cython.floating f): def inner(): cdef cython.floating g + +# Mixing const and non-const type makes fused type ambiguous +cdef fused mix_const_t: + int + const int + +cdef cdef_func_with_mix_const_type(mix_const_t val): + print(val) + +cdef_func_with_mix_const_type(1) + + # This is all valid dtype5 = fused_type(int, long, float) dtype6 = cython.fused_type(int, long) @@ -63,18 +76,40 @@ ctypedef fused fused2: func(x, y) +cdef floating return_type_unfindable1(cython.integral x): + return 1.0 + +cpdef floating return_type_unfindable2(cython.integral x): + return 1.0 + +cdef void contents_unfindable1(cython.integral x): + z: floating = 1 # note: cdef variables also fail with an error but not by the time this test aborts + sz = sizeof(floating) + _ERRORS = u""" -10:15: fused_type does not take keyword arguments -15:33: Type specified multiple times -26:0: Invalid use of fused types, type cannot be specialized -26:4: Not enough types specified to specialize the function, int2_t is still fused +11:15: fused_type does not take keyword arguments +16:33: Type specified multiple times 27:0: Invalid use of fused types, type cannot be specialized 27:4: Not enough types specified to specialize the function, int2_t is still fused -28:16: Call with wrong number of arguments (expected 2, got 1) -29:16: Call with wrong number of arguments (expected 2, got 3) -36:6: Invalid base type for memoryview slice: int * -39:0: Fused lambdas not allowed -42:5: Fused types not allowed here -45:9: Fused types not allowed here +28:0: Invalid use of fused types, type cannot be specialized +28:4: Not enough types specified to specialize the function, int2_t is still fused +29:16: Call with wrong number of arguments (expected 2, got 1) +30:16: Call with wrong number of arguments (expected 2, got 3) +37:6: Invalid base type for memoryview slice: int * +40:0: Fused lambdas not allowed +43:5: Fused types not allowed here +43:21: cdef variable 'x' declared after it is used +46:9: Fused types not allowed here +61:0: Invalid use of fused types, type cannot be specialized +61:29: ambiguous overloaded method +# Possibly duplicates the errors more often than we want +79:5: Return type is a fused type that cannot be determined from the function arguments +82:6: Return type is a fused type that cannot be determined from the function arguments +86:4: 'z' cannot be specialized since its type is not a fused argument to this function +86:4: 'z' cannot be specialized since its type is not a fused argument to this function +86:4: 'z' cannot be specialized since its type is not a fused argument to this function +87:24: Type cannot be specialized since it is not a fused argument to this function +87:24: Type cannot be specialized since it is not a fused argument to this function +87:24: Type cannot be specialized since it is not a fused argument to this function """ diff --git a/tests/errors/missing_baseclass_in_predecl_T262.pyx b/tests/errors/missing_baseclass_in_predecl_T262.pyx index ece07b155..907f072f5 100644 --- a/tests/errors/missing_baseclass_in_predecl_T262.pyx +++ b/tests/errors/missing_baseclass_in_predecl_T262.pyx @@ -1,4 +1,4 @@ -# ticket: 262 +# ticket: t262 # mode: error cdef class Album diff --git a/tests/errors/missing_self_in_cpdef_method_T156.pyx b/tests/errors/missing_self_in_cpdef_method_T156.pyx index 21241a221..e8b0c5369 100644 --- a/tests/errors/missing_self_in_cpdef_method_T156.pyx +++ b/tests/errors/missing_self_in_cpdef_method_T156.pyx @@ -1,4 +1,4 @@ -# ticket: 156 +# ticket: t156 # mode: error cdef class B: diff --git a/tests/errors/missing_self_in_cpdef_method_T165.pyx b/tests/errors/missing_self_in_cpdef_method_T165.pyx index 6a95922d5..89763cd2a 100644 --- a/tests/errors/missing_self_in_cpdef_method_T165.pyx +++ b/tests/errors/missing_self_in_cpdef_method_T165.pyx @@ -1,4 +1,4 @@ -# ticket: 165 +# ticket: t165 # mode: error cdef class A: diff --git a/tests/errors/nogil.pyx b/tests/errors/nogil.pyx index 1d22f9d9b..aa3011d00 100644 --- a/tests/errors/nogil.pyx +++ b/tests/errors/nogil.pyx @@ -8,14 +8,14 @@ cdef void g(int x) nogil: cdef object z z = None -cdef void h(int x) nogil: +cdef void h(int x) nogil: # allowed p() cdef object p() nogil: pass -cdef void r() nogil: - q() +cdef void r() nogil: # allowed + q() # allowed cdef object m(): cdef object x, y = 0, obj @@ -23,11 +23,11 @@ cdef object m(): global fred q() with nogil: - r() + r() # allowed to call plain C functions q() - i = 42 + i = 42 # allowed with type inference obj = None - 17L + 17L # allowed <object>7j help xxx = `"Hello"` @@ -45,14 +45,14 @@ cdef object m(): {x, y} obj and x t(obj) -# f(42) # Cython handles this internally + f(42) x + obj -obj x = y = obj x, y = y, x obj[i] = x obj.fred = x - print obj + print obj # allowed! del fred return obj raise obj # allowed! @@ -90,8 +90,13 @@ def bare_pyvar_name(object x): with nogil: x -# For m(), the important thing is that there are errors on all lines in the range 23-69 -# except these: 29, 34, 44, 56, 58, 60, 62-64 +cdef int fstrings(int x, object obj) nogil except -1: + f"" # allowed + f"a" # allowed + f"a"f"b" # allowed + f"{x}" + f"{obj}" + _ERRORS = u""" 4:5: Function with Python return type cannot be declared nogil @@ -105,14 +110,10 @@ _ERRORS = u""" 31:16: Constructing complex number not allowed without gil 33:8: Assignment of Python object not allowed without gil 33:14: Backquote expression not allowed without gil -33:15: Operation not allowed without gil 34:15: Assignment of Python object not allowed without gil -34:15: Operation not allowed without gil 34:15: Python import not allowed without gil -35:8: Operation not allowed without gil 35:13: Python import not allowed without gil 35:25: Constructing Python list not allowed without gil -35:25: Operation not allowed without gil 36:17: Iterating over Python object not allowed without gil 38:11: Discarding owned Python object not allowed without gil 38:11: Indexing Python object not allowed without gil @@ -137,6 +138,7 @@ _ERRORS = u""" 46:12: Discarding owned Python object not allowed without gil 46:12: Truth-testing Python object not allowed without gil 47:10: Python type test not allowed without gil +48:9: Discarding owned Python object not allowed without gil 49:10: Discarding owned Python object not allowed without gil 49:10: Operation not allowed without gil 50:8: Discarding owned Python object not allowed without gil @@ -151,10 +153,10 @@ _ERRORS = u""" 53:11: Indexing Python object not allowed without gil 54:11: Accessing Python attribute not allowed without gil 54:11: Assignment of Python object not allowed without gil -55:8: Constructing Python tuple not allowed without gil -55:8: Python print statement not allowed without gil + 56:8: Deleting Python object not allowed without gil 57:8: Returning Python object not allowed without gil + 59:11: Truth-testing Python object not allowed without gil 61:14: Truth-testing Python object not allowed without gil 63:8: For-loop using object bounds or target not allowed without gil @@ -162,4 +164,9 @@ _ERRORS = u""" 63:24: Coercion from Python not allowed without the GIL 65:8: Try-except statement not allowed without gil 86:8: For-loop using object bounds or target not allowed without gil + +97:4: Discarding owned Python object not allowed without gil +97:6: String formatting not allowed without gil +98:4: Discarding owned Python object not allowed without gil +98:6: String formatting not allowed without gil """ diff --git a/tests/errors/nogil_conditional.pyx b/tests/errors/nogil_conditional.pyx new file mode 100644 index 000000000..e800eb199 --- /dev/null +++ b/tests/errors/nogil_conditional.pyx @@ -0,0 +1,81 @@ +# cython: remove_unreachable=False +# mode: error + +cdef int f_nogil(int x) nogil: + cdef int y + y = x + 10 + return y + + +def f_gil(x): + y = 0 + y = x + 100 + return y + + +def illegal_gil_usage(): + cdef int res = 0 + with nogil(True): + res = f_gil(res) + + with nogil(True): + res = f_gil(res) + + with gil(False): + res = f_gil(res) + + with nogil(False): + res = f_nogil(res) + + +def foo(a): + return a < 10 + + +def non_constant_condition(int x) -> int: + cdef int res = x + with nogil(x < 10): + res = f_nogil(res) + + with gil(foo(x)): + res = f_gil(res) + + +ctypedef fused number_or_object: + float + object + + +def fused_type(number_or_object x): + with nogil(number_or_object is object): + res = x + 1 + + # This should be fine + with nogil(number_or_object is float): + res = x + 1 + + return res + + +_ERRORS = u""" +19:14: Accessing Python global or builtin not allowed without gil +19:19: Calling gil-requiring function not allowed without gil +19:19: Coercion from Python not allowed without the GIL +19:19: Constructing Python tuple not allowed without gil +19:20: Converting to Python object not allowed without gil +21:13: Trying to release the GIL while it was previously released. +22:18: Accessing Python global or builtin not allowed without gil +22:23: Calling gil-requiring function not allowed without gil +22:23: Coercion from Python not allowed without the GIL +22:23: Constructing Python tuple not allowed without gil +22:24: Converting to Python object not allowed without gil +25:18: Accessing Python global or builtin not allowed without gil +25:23: Calling gil-requiring function not allowed without gil +25:23: Coercion from Python not allowed without the GIL +25:23: Constructing Python tuple not allowed without gil +25:24: Converting to Python object not allowed without gil +37:17: Non-constant condition in a `with nogil(<condition>)` statement +40:16: Non-constant condition in a `with gil(<condition>)` statement +51:8: Assignment of Python object not allowed without gil +51:16: Calling gil-requiring function not allowed without gil +""" diff --git a/tests/errors/nonconst_excval.pyx b/tests/errors/nonconst_excval.pyx new file mode 100644 index 000000000..ad80f2e27 --- /dev/null +++ b/tests/errors/nonconst_excval.pyx @@ -0,0 +1,12 @@ +# mode: error + +import math + +cdef double cfunc(double x) except math.nan: + return x + + +_ERRORS = """ +5:39: Exception value must be constant +5:39: Not allowed in a constant expression +""" diff --git a/tests/errors/notcimportedT418.pyx b/tests/errors/notcimportedT418.pyx index 980d66555..c2821fb83 100644 --- a/tests/errors/notcimportedT418.pyx +++ b/tests/errors/notcimportedT418.pyx @@ -1,4 +1,4 @@ -# ticket: 418 +# ticket: t418 # mode: error import somemod.child diff --git a/tests/errors/pep487_exttype.pyx b/tests/errors/pep487_exttype.pyx new file mode 100644 index 000000000..e9e4f44a2 --- /dev/null +++ b/tests/errors/pep487_exttype.pyx @@ -0,0 +1,13 @@ +# mode: error + +cdef class Imp: + def __init_subclass__(cls, a=None, **kwargs): + super().__init_subclass__(**kwargs) + print(a) + +cdef class ExImp1(Imp): pass +class ExImp2(Imp, a=60): pass + +_ERRORS = u""" +4:4: '__init_subclass__' is not supported by extension class +""" diff --git a/tests/errors/pep492_badsyntax_async2.pyx b/tests/errors/pep492_badsyntax_async2.pyx deleted file mode 100644 index 4ea30dba1..000000000 --- a/tests/errors/pep492_badsyntax_async2.pyx +++ /dev/null @@ -1,11 +0,0 @@ -# mode: error -# tag: pep492, async - -async def foo(): - def foo(a:await list()): - pass - -_ERRORS = """ -5:14: 'await' not supported here -5:14: 'await' not supported here -""" diff --git a/tests/errors/posonly.pyx b/tests/errors/posonly.pyx new file mode 100644 index 000000000..2724f3037 --- /dev/null +++ b/tests/errors/posonly.pyx @@ -0,0 +1,102 @@ +# mode: error +# tag: posonly + +def f(a, b = 5, /, c): + pass + +def f(a = 5, b, /, c): + pass + +def f(a = 5, b, /): + pass + +def f(a, /, a): + pass + +def f(a, /, *, a): + pass + +#def f(a, b/2, c): #D +# pass + +#def f(*args, /): #D +# pass + +#def f(*args, a, /): +# pass + +#def f(**kwargs, /): +# pass + +#def f(/, a = 1): # D +# pass + +#def f(/, a): +# pass + +#def f(/): +# pass + +#def f(*, a, /): +# pass + +#def f(*, /, a): +# pass + +#def f(a, /, c, /): +# pass + +#def f(a, /, c, /, d): +# pass + +#def f(a, /, c, /, d, *, e): +# pass + +#def f(a, *, c, /, d, e): +# pass + +def test_invalid_syntax_lambda(self): + lambda a, b = 5, /, c: None + lambda a = 5, b, /, c: None + lambda a = 5, b, /: None + lambda a, /, a: None + lambda a, /, *, a: None +# lambda *args, /: None +# lambda *args, a, /: None +# lambda **kwargs, /: None +# lambda /, a = 1: None +# lambda /, a: None +# lambda /: None +# lambda *, a, /: None +# lambda *, /, a: None + +async def f(a, b = 5, /, c): + pass + +#def test_multiple_seps(a,/,b,/): +# pass + +_ERRORS = u""" +4:19: Non-default argument following default argument +7:13: Non-default argument following default argument +7:19: Non-default argument following default argument +10:13: Non-default argument following default argument +13:6: Previous declaration is here +13:12: 'a' redeclared +16:6: Previous declaration is here +16:15: 'a' redeclared +59:24: Non-default argument following default argument +60:18: Non-default argument following default argument +60:24: Non-default argument following default argument +61:18: Non-default argument following default argument +62:11: Previous declaration is here +62:17: 'a' redeclared +63:11: Previous declaration is here +63:20: 'a' redeclared +73:25: Non-default argument following default argument +""" + + + + + diff --git a/tests/errors/posonly2.pyx b/tests/errors/posonly2.pyx new file mode 100644 index 000000000..dab079047 --- /dev/null +++ b/tests/errors/posonly2.pyx @@ -0,0 +1,9 @@ +# mode: error +# tag: posonly + +def f(a, b/2, c): + pass + +_ERRORS = u""" +4:11: Syntax error in Python function argument list +""" diff --git a/tests/errors/posonly3.pyx b/tests/errors/posonly3.pyx new file mode 100644 index 000000000..4b4f0db75 --- /dev/null +++ b/tests/errors/posonly3.pyx @@ -0,0 +1,13 @@ +# mode: error +# tag: posonly + +def f(*args, /): + pass + +def f(*args, a, /): + pass + + +_ERRORS = u""" +4:13: Expected ')', found '/' +""" diff --git a/tests/errors/posonly4.pyx b/tests/errors/posonly4.pyx new file mode 100644 index 000000000..d1a1f61f9 --- /dev/null +++ b/tests/errors/posonly4.pyx @@ -0,0 +1,9 @@ +# mode: error +# tag: posonly + +def f(/, a = 1): + pass + +_ERRORS = u""" +4:6: Got zero positional-only arguments despite presence of positional-only specifier '/' +""" diff --git a/tests/errors/posonly5.pyx b/tests/errors/posonly5.pyx new file mode 100644 index 000000000..9e359226a --- /dev/null +++ b/tests/errors/posonly5.pyx @@ -0,0 +1,11 @@ +# mode: error +# tag: posonly + +def test_multiple_seps(a,/,b,/): + pass + +_ERRORS = u""" +4:29: Expected ')', found '/' +""" + + diff --git a/tests/errors/pure_errors.py b/tests/errors/pure_errors.py index 4a9dca53b..570dc006f 100644 --- a/tests/errors/pure_errors.py +++ b/tests/errors/pure_errors.py @@ -53,5 +53,5 @@ def pyfunc(x): # invalid _ERRORS = """ 44:22: Calling gil-requiring function not allowed without gil 45:24: Calling gil-requiring function not allowed without gil -49:0: Python functions cannot be declared 'nogil' +48:0: Python functions cannot be declared 'nogil' """ diff --git a/tests/errors/pxd_cdef_class_declaration_T286.pyx b/tests/errors/pxd_cdef_class_declaration_T286.pyx index 0c968f701..b1a8bc844 100644 --- a/tests/errors/pxd_cdef_class_declaration_T286.pyx +++ b/tests/errors/pxd_cdef_class_declaration_T286.pyx @@ -1,4 +1,4 @@ -# ticket: 286 +# ticket: t286 # mode: error cdef class A: diff --git a/tests/errors/pyobjcastdisallow_T313.pyx b/tests/errors/pyobjcastdisallow_T313.pyx index 5048719d8..30973380f 100644 --- a/tests/errors/pyobjcastdisallow_T313.pyx +++ b/tests/errors/pyobjcastdisallow_T313.pyx @@ -1,4 +1,4 @@ -# ticket: 313 +# ticket: t313 # mode: error a = 3 diff --git a/tests/errors/return_outside_function_T135.pyx b/tests/errors/return_outside_function_T135.pyx index d7432ca50..4788c2a72 100644 --- a/tests/errors/return_outside_function_T135.pyx +++ b/tests/errors/return_outside_function_T135.pyx @@ -1,5 +1,5 @@ # cython: remove_unreachable=False -# ticket: 135 +# ticket: t135 # mode: error def _runtime_True(): diff --git a/tests/errors/reversed_literal_pyobjs.pyx b/tests/errors/reversed_literal_pyobjs.pyx index 8444eaf2b..e5d1cb97d 100644 --- a/tests/errors/reversed_literal_pyobjs.pyx +++ b/tests/errors/reversed_literal_pyobjs.pyx @@ -10,6 +10,7 @@ for i in reversed(range(j, [], 2)): pass for i in reversed(range(j, [], -2)): pass +# code below is no longer a compile-time error (although won't run without an exception) for i in reversed(range({}, j, 2)): pass for i in reversed(range({}, j, -2)): @@ -24,8 +25,4 @@ _ERRORS = """ 7:24: Cannot coerce list to type 'long' 9:27: Cannot coerce list to type 'long' 11:27: Cannot coerce list to type 'long' -13:24: Cannot interpret dict as type 'long' -15:24: Cannot interpret dict as type 'long' -17:27: Cannot interpret dict as type 'long' -19:27: Cannot interpret dict as type 'long' """ diff --git a/tests/errors/tree_assert.pyx b/tests/errors/tree_assert.pyx index 81ae9a1d3..b76990cab 100644 --- a/tests/errors/tree_assert.pyx +++ b/tests/errors/tree_assert.pyx @@ -11,8 +11,8 @@ def test(): _ERRORS = u""" -9:0: Expected path '//ComprehensionNode' not found in result tree -9:0: Expected path '//ComprehensionNode//FuncDefNode' not found in result tree -9:0: Unexpected path '//NameNode' found in result tree -9:0: Unexpected path '//SimpleCallNode' found in result tree +5:0: Expected path '//ComprehensionNode' not found in result tree +5:0: Expected path '//ComprehensionNode//FuncDefNode' not found in result tree +5:0: Unexpected path '//NameNode' found in result tree +5:0: Unexpected path '//SimpleCallNode' found in result tree """ diff --git a/tests/errors/typoT304.pyx b/tests/errors/typoT304.pyx index 7c736c898..2de5573d4 100644 --- a/tests/errors/typoT304.pyx +++ b/tests/errors/typoT304.pyx @@ -1,4 +1,4 @@ -# ticket: 304 +# ticket: t304 # mode: error def f(): diff --git a/tests/errors/unicode_identifiers_e1.pyx b/tests/errors/unicode_identifiers_e1.pyx new file mode 100644 index 000000000..5087a3d43 --- /dev/null +++ b/tests/errors/unicode_identifiers_e1.pyx @@ -0,0 +1,8 @@ +# -*- coding: utf-8 -*- +# mode: error + +★1 = 5 # invalid start symbol + +_ERRORS = u""" +4:0: Unrecognized character +""" diff --git a/tests/errors/unicode_identifiers_e2.pyx b/tests/errors/unicode_identifiers_e2.pyx new file mode 100644 index 000000000..e9e53aa5b --- /dev/null +++ b/tests/errors/unicode_identifiers_e2.pyx @@ -0,0 +1,9 @@ +# -*- coding: utf-8 -*- +# mode: error + +class MyClass₡: # invalid continue symbol + pass + +_ERRORS = u""" +4:13: Unrecognized character +""" diff --git a/tests/errors/unicode_identifiers_e3.pyx b/tests/errors/unicode_identifiers_e3.pyx new file mode 100644 index 000000000..e5cba9caa --- /dev/null +++ b/tests/errors/unicode_identifiers_e3.pyx @@ -0,0 +1,11 @@ +# -*- coding: utf-8 -*- +# mode: error + +def f(): + a = 1 + ́b = 2 # looks like an indentation error but is actually a combining accent as the first letter of column 4 + c = 3 + +_ERRORS = u""" +6:4: Unrecognized character +""" diff --git a/tests/errors/unicode_identifiers_e4.pyx b/tests/errors/unicode_identifiers_e4.pyx new file mode 100644 index 000000000..035eff2c9 --- /dev/null +++ b/tests/errors/unicode_identifiers_e4.pyx @@ -0,0 +1,13 @@ +# -*- coding: utf-8 -*- +# mode: error + +cdef class C: + # these two symbols "\u1e69" and "\u1e9b\u0323" normalize to the same thing + # so the two attributes can't coexist + cdef int ṩomething + cdef double ẛ̣omething + +_ERRORS = u""" +7:13: Previous declaration is here +8:16: 'ṩomething' redeclared +""" diff --git a/tests/errors/uninitialized_lhs.pyx b/tests/errors/uninitialized_lhs.pyx index 5d5ece854..8550761a4 100644 --- a/tests/errors/uninitialized_lhs.pyx +++ b/tests/errors/uninitialized_lhs.pyx @@ -1,7 +1,7 @@ # cython: warn.maybe_uninitialized=True # mode: error # tag: werror -# ticket: 739 +# ticket: t739 def index_lhs(a): cdef object idx diff --git a/tests/errors/w_numpy_arr_as_cppvec_ref.pyx b/tests/errors/w_numpy_arr_as_cppvec_ref.pyx index e7fbaeece..d3a70dbed 100644 --- a/tests/errors/w_numpy_arr_as_cppvec_ref.pyx +++ b/tests/errors/w_numpy_arr_as_cppvec_ref.pyx @@ -1,10 +1,12 @@ # mode: error -# tag: cpp, werror, numpy +# tag: cpp, werror, numpy, no-cpp-locals import numpy as np cimport numpy as np from libcpp.vector cimport vector +np.import_array() + cdef extern from *: void cpp_function_vector1(vector[int]) void cpp_function_vector2(vector[int] &) @@ -24,8 +26,8 @@ def main(): _ERRORS = """ -17:25: Cannot pass Python object as C++ data structure reference (vector[int] &), will pass by copy. -18:25: Cannot pass Python object as C++ data structure reference (vector[int] &), will pass by copy. -19:28: Cannot pass Python object as C++ data structure reference (vector[int] &), will pass by copy. -19:33: Cannot pass Python object as C++ data structure reference (vector[int] &), will pass by copy. +19:25: Cannot pass Python object as C++ data structure reference (vector[int] &), will pass by copy. +20:25: Cannot pass Python object as C++ data structure reference (vector[int] &), will pass by copy. +21:28: Cannot pass Python object as C++ data structure reference (vector[int] &), will pass by copy. +21:33: Cannot pass Python object as C++ data structure reference (vector[int] &), will pass by copy. """ diff --git a/tests/limited_api_bugs.txt b/tests/limited_api_bugs.txt new file mode 100644 index 000000000..b98ff76c0 --- /dev/null +++ b/tests/limited_api_bugs.txt @@ -0,0 +1,11 @@ +# This file contains tests corresponding to unresolved bugs using CPython's +# Limited API which will be skipped in the normal testing run. + +convolve2 +dict_animal +not_none +queue3 +test_queue +memoryview +memview +buffer diff --git a/tests/travis_macos_cpp_bugs.txt b/tests/macos_cpp_bugs.txt index 17c5d4066..17c5d4066 100644 --- a/tests/travis_macos_cpp_bugs.txt +++ b/tests/macos_cpp_bugs.txt diff --git a/tests/memoryview/cfunc_convert_with_memoryview.pyx b/tests/memoryview/cfunc_convert_with_memoryview.pyx new file mode 100644 index 000000000..55bd02205 --- /dev/null +++ b/tests/memoryview/cfunc_convert_with_memoryview.pyx @@ -0,0 +1,51 @@ +# mode: run +# tag: autowrap +# cython: always_allow_keywords=True + +cdef void memoryview_func_a(double [:] x): + x[0] = 1 + +cdef void memoryview_func_b(double [::1] x): + x[0] = 2 + +cdef void memoryview_func_c(int [:] x): + x[0] = 1 + +cdef void memoryview_func_d(int [:] x): + x[0] = 2 + +cdef void memoryview_func_e(int [:,::1] x): + x[0,0] = 4 + +cdef void memoryview_func_f(int [::1,:] x): + x[0,0] = 4 + + +def test_memview_wrapping(): + """ + We're mainly concerned that the code compiles without the names clashing + >>> test_memview_wrapping() + 1.0 + 2.0 + 1 + 2 + """ + cdef a = memoryview_func_a + cdef b = memoryview_func_b + cdef c = memoryview_func_c + cdef d = memoryview_func_d + cdef e = memoryview_func_e + cdef f = memoryview_func_f + cdef double[1] double_arr = [0] + cdef int[1] int_arr = [0] + + a(<double[:1]> double_arr) + print(double_arr[0]) + b(<double[:1:1]> double_arr) + print(double_arr[0]) + c(<int[:1]> int_arr) + print(int_arr[0]) + d(<int[:1:1]> int_arr) + print(int_arr[0]) + # don't call e and f because it's harder without needing extra dependencies + # it's mostly a compile test for them diff --git a/tests/memoryview/cythonarray.pyx b/tests/memoryview/cythonarray.pyx index 1f87a4e18..d0408ff23 100644 --- a/tests/memoryview/cythonarray.pyx +++ b/tests/memoryview/cythonarray.pyx @@ -209,3 +209,80 @@ def test_cyarray_from_carray(): mslice = a print mslice[0, 0], mslice[1, 0], mslice[2, 5] + +class InheritFrom(v.array): + """ + Test is just to confirm it works, not to do anything meaningful with it + (Be aware that itemsize isn't necessarily right) + >>> inst = InheritFrom(shape=(3, 3, 3), itemsize=4, format="i") + """ + pass + + +def test_char_array_in_python_api(*shape): + """ + >>> import sys + >>> if sys.version_info[0] < 3: + ... def bytes(b): return memoryview(b).tobytes() # don't call str() + + >>> arr1d = test_char_array_in_python_api(10) + >>> print(bytes(arr1d).decode('ascii')) + xxxxxxxxxx + >>> len(bytes(arr1d)) + 10 + >>> arr2d = test_char_array_in_python_api(10, 2) + >>> print(bytes(arr2d).decode('ascii')) + xxxxxxxxxxxxxxxxxxxx + >>> len(bytes(arr2d)) + 20 + + # memoryview + >>> len(bytes(memoryview(arr1d))) + 10 + >>> bytes(memoryview(arr1d)) == bytes(arr1d) + True + >>> len(bytes(memoryview(arr2d))) + 20 + >>> bytes(memoryview(arr2d)) == bytes(arr2d) + True + + # BytesIO + >>> from io import BytesIO + >>> BytesIO(arr1d).read() == bytes(arr1d) + True + >>> BytesIO(arr2d).read() == bytes(arr2d) + True + + >>> b = BytesIO() + >>> print(b.write(arr1d)) + 10 + >>> b.getvalue() == bytes(arr1d) or b.getvalue() + True + + >>> b = BytesIO() + >>> print(b.write(arr2d)) + 20 + >>> b.getvalue() == bytes(arr2d) or b.getvalue() + True + + # BufferedWriter (uses PyBUF_SIMPLE, see https://github.com/cython/cython/issues/3775) + >>> from io import BufferedWriter + >>> b = BytesIO() + >>> bw = BufferedWriter(b) + >>> print(bw.write(arr1d)) + 10 + >>> bw.flush() + >>> b.getvalue() == bytes(arr1d) + True + + >>> b = BytesIO() + >>> bw = BufferedWriter(b) + >>> print(bw.write(arr2d)) + 20 + >>> bw.flush() + >>> b.getvalue() == bytes(arr2d) + True + """ + arr = array(shape=shape, itemsize=sizeof(char), format='c', mode='c') + arr[:] = b'x' + return arr diff --git a/tests/memoryview/memoryview.pyx b/tests/memoryview/memoryview.pyx index 15c71d481..273a1fa68 100644 --- a/tests/memoryview/memoryview.pyx +++ b/tests/memoryview/memoryview.pyx @@ -1,5 +1,7 @@ # mode: run +# Test declarations, behaviour and coercions of the memoryview type itself. + u''' >>> f() >>> g() @@ -155,6 +157,7 @@ def assignmvs(): cdef int[:] mv3 mv1 = array((10,), itemsize=sizeof(int), format='i') mv2 = mv1 + mv1 = mv1 mv1 = mv2 mv3 = mv2 @@ -247,7 +250,7 @@ def basic_struct(MyStruct[:] mslice): >>> basic_struct(MyStructMockBuffer(None, [(1, 2, 3, 4, 5)], format="ccqii")) [('a', 1), ('b', 2), ('c', 3), ('d', 4), ('e', 5)] """ - buf = mslice + cdef object buf = mslice print sorted([(k, int(v)) for k, v in buf[0].items()]) def nested_struct(NestedStruct[:] mslice): @@ -259,7 +262,7 @@ def nested_struct(NestedStruct[:] mslice): >>> nested_struct(NestedStructMockBuffer(None, [(1, 2, 3, 4, 5)], format="T{ii}T{2i}i")) 1 2 3 4 5 """ - buf = mslice + cdef object buf = mslice d = buf[0] print d['x']['a'], d['x']['b'], d['y']['a'], d['y']['b'], d['z'] @@ -275,7 +278,7 @@ def packed_struct(PackedStruct[:] mslice): 1 2 """ - buf = mslice + cdef object buf = mslice print buf[0]['a'], buf[0]['b'] def nested_packed_struct(NestedPackedStruct[:] mslice): @@ -289,7 +292,7 @@ def nested_packed_struct(NestedPackedStruct[:] mslice): >>> nested_packed_struct(NestedPackedStructMockBuffer(None, [(1, 2, 3, 4, 5)], format="^c@i^ci@i")) 1 2 3 4 5 """ - buf = mslice + cdef object buf = mslice d = buf[0] print d['a'], d['b'], d['sub']['a'], d['sub']['b'], d['c'] @@ -299,7 +302,7 @@ def complex_dtype(long double complex[:] mslice): >>> complex_dtype(LongComplexMockBuffer(None, [(0, -1)])) -1j """ - buf = mslice + cdef object buf = mslice print buf[0] def complex_inplace(long double complex[:] mslice): @@ -307,7 +310,7 @@ def complex_inplace(long double complex[:] mslice): >>> complex_inplace(LongComplexMockBuffer(None, [(0, -1)])) (1+1j) """ - buf = mslice + cdef object buf = mslice buf[0] = buf[0] + 1 + 2j print buf[0] @@ -318,7 +321,7 @@ def complex_struct_dtype(LongComplex[:] mslice): >>> complex_struct_dtype(LongComplexMockBuffer(None, [(0, -1)])) 0.0 -1.0 """ - buf = mslice + cdef object buf = mslice print buf[0]['real'], buf[0]['imag'] # @@ -356,7 +359,7 @@ def get_int_2d(int[:, :] mslice, int i, int j): ... IndexError: Out of bounds on buffer access (axis 1) """ - buf = mslice + cdef object buf = mslice return buf[i, j] def set_int_2d(int[:, :] mslice, int i, int j, int value): @@ -409,11 +412,48 @@ def set_int_2d(int[:, :] mslice, int i, int j, int value): IndexError: Out of bounds on buffer access (axis 1) """ - buf = mslice + cdef object buf = mslice buf[i, j] = value # +# auto type inference +# (note that for most numeric types "might_overflow" stops the type inference from working well) +# +def type_infer(double[:, :] arg): + """ + >>> type_infer(DoubleMockBuffer(None, range(6), (2,3))) + double + double[:] + double[:] + double[:, :] + """ + a = arg[0,0] + print(cython.typeof(a)) + b = arg[0] + print(cython.typeof(b)) + c = arg[0,:] + print(cython.typeof(c)) + d = arg[:,:] + print(cython.typeof(d)) + +# +# Loop optimization +# +@cython.test_fail_if_path_exists("//CoerceToPyTypeNode") +def memview_iter(double[:, :] arg): + """ + memview_iter(DoubleMockBuffer("C", range(6), (2,3))) + True + """ + cdef double total = 0 + for mview1d in arg: + for val in mview1d: + total += val + if total == 15: + return True + +# # Test all kinds of indexing and flags # @@ -426,7 +466,7 @@ def writable(unsigned short int[:, :, :] mslice): >>> [str(x) for x in R.received_flags] # Py2/3 ['FORMAT', 'ND', 'STRIDES', 'WRITABLE'] """ - buf = mslice + cdef object buf = mslice buf[2, 2, 1] = 23 def strided(int[:] mslice): @@ -441,7 +481,7 @@ def strided(int[:] mslice): >>> A.release_ok True """ - buf = mslice + cdef object buf = mslice return buf[2] def c_contig(int[::1] mslice): @@ -450,7 +490,7 @@ def c_contig(int[::1] mslice): >>> c_contig(A) 2 """ - buf = mslice + cdef object buf = mslice return buf[2] def c_contig_2d(int[:, ::1] mslice): @@ -461,7 +501,7 @@ def c_contig_2d(int[:, ::1] mslice): >>> c_contig_2d(A) 7 """ - buf = mslice + cdef object buf = mslice return buf[1, 3] def f_contig(int[::1, :] mslice): @@ -470,7 +510,7 @@ def f_contig(int[::1, :] mslice): >>> f_contig(A) 2 """ - buf = mslice + cdef object buf = mslice return buf[0, 1] def f_contig_2d(int[::1, :] mslice): @@ -481,7 +521,7 @@ def f_contig_2d(int[::1, :] mslice): >>> f_contig_2d(A) 7 """ - buf = mslice + cdef object buf = mslice return buf[3, 1] def generic(int[::view.generic, ::view.generic] mslice1, @@ -552,7 +592,7 @@ def printbuf_td_cy_int(td_cy_int[:] mslice, shape): ... ValueError: Buffer dtype mismatch, expected 'td_cy_int' but got 'short' """ - buf = mslice + cdef object buf = mslice cdef int i for i in range(shape[0]): print buf[i], @@ -567,7 +607,7 @@ def printbuf_td_h_short(td_h_short[:] mslice, shape): ... ValueError: Buffer dtype mismatch, expected 'td_h_short' but got 'int' """ - buf = mslice + cdef object buf = mslice cdef int i for i in range(shape[0]): print buf[i], @@ -582,7 +622,7 @@ def printbuf_td_h_cy_short(td_h_cy_short[:] mslice, shape): ... ValueError: Buffer dtype mismatch, expected 'td_h_cy_short' but got 'int' """ - buf = mslice + cdef object buf = mslice cdef int i for i in range(shape[0]): print buf[i], @@ -597,7 +637,7 @@ def printbuf_td_h_ushort(td_h_ushort[:] mslice, shape): ... ValueError: Buffer dtype mismatch, expected 'td_h_ushort' but got 'short' """ - buf = mslice + cdef object buf = mslice cdef int i for i in range(shape[0]): print buf[i], @@ -612,7 +652,7 @@ def printbuf_td_h_double(td_h_double[:] mslice, shape): ... ValueError: Buffer dtype mismatch, expected 'td_h_double' but got 'float' """ - buf = mslice + cdef object buf = mslice cdef int i for i in range(shape[0]): print buf[i], @@ -626,6 +666,8 @@ def addref(*args): def decref(*args): for item in args: Py_DECREF(item) +@cython.binding(False) +@cython.always_allow_keywords(False) def get_refcount(x): return (<PyObject*>x).ob_refcnt @@ -647,7 +689,7 @@ def printbuf_object(object[:] mslice, shape): {4: 23} 2 [34, 3] 2 """ - buf = mslice + cdef object buf = mslice cdef int i for i in range(shape[0]): print repr(buf[i]), (<PyObject*>buf[i]).ob_refcnt @@ -668,7 +710,7 @@ def assign_to_object(object[:] mslice, int idx, obj): (2, 3) >>> decref(b) """ - buf = mslice + cdef object buf = mslice buf[idx] = obj def assign_temporary_to_object(object[:] mslice): @@ -695,7 +737,7 @@ def assign_temporary_to_object(object[:] mslice): >>> assign_to_object(A, 1, a) >>> decref(a) """ - buf = mslice + cdef object buf = mslice buf[1] = {3-2: 2+(2*4)-2} @@ -743,7 +785,7 @@ def test_generic_slicing(arg, indirect=False): """ cdef int[::view.generic, ::view.generic, :] _a = arg - a = _a + cdef object a = _a b = a[2:8:2, -4:1:-1, 1:3] print b.shape @@ -826,7 +868,7 @@ def test_direct_slicing(arg): released A """ cdef int[:, :, :] _a = arg - a = _a + cdef object a = _a b = a[2:8:2, -4:1:-1, 1:3] print b.shape @@ -854,7 +896,7 @@ def test_slicing_and_indexing(arg): released A """ cdef int[:, :, :] _a = arg - a = _a + cdef object a = _a b = a[-5:, 1, 1::2] c = b[4:1:-1, ::-1] d = c[2, 1:2] @@ -1099,7 +1141,7 @@ def optimised_index_of_slice(int[:,:,:] arr, int x, int y, int z): def test_assign_from_byteslike(byteslike): - # Once http://python3statement.org is accepted, should be just + # Once https://python3statement.org/ is accepted, should be just # >>> test_assign_from_byteslike(bytes(b'hello')) # b'hello' # ... diff --git a/tests/memoryview/memoryview_cache_builtins.srctree b/tests/memoryview/memoryview_cache_builtins.srctree new file mode 100644 index 000000000..91b1c4753 --- /dev/null +++ b/tests/memoryview/memoryview_cache_builtins.srctree @@ -0,0 +1,21 @@ +PYTHON setup.py build_ext --inplace + +################ setup.py ##################### + +from distutils.core import setup +from Cython.Build import cythonize +from Cython.Compiler import Options + +Options.cache_builtins = False + +setup( + ext_modules = cythonize("mview.pyx") +) + +############### mview.pyx ################ + +# https://github.com/cython/cython/issues/3406 +# Failure was at Cython compilation stage + +def f(double [::1] x): + pass diff --git a/tests/memoryview/memoryview_no_binding_T3613.pyx b/tests/memoryview/memoryview_no_binding_T3613.pyx new file mode 100644 index 000000000..1c736359e --- /dev/null +++ b/tests/memoryview/memoryview_no_binding_T3613.pyx @@ -0,0 +1,21 @@ +# mode: compile +# tag: memoryview + +# cython: binding=False + +# See GH 3613 - when memoryviews were compiled with binding off they ended up in an +# inconsistent state where different directives were applied at different stages +# of the pipeline + +import cython + +def f(double[:] a): + pass + +@cython.binding(False) +def g(double[:] a): + pass + +@cython.binding(True) +def h(double[:] a): + pass diff --git a/tests/memoryview/memoryview_pep489_typing.pyx b/tests/memoryview/memoryview_pep484_typing.pyx index 0a62ff624..66445a1b7 100644 --- a/tests/memoryview/memoryview_pep489_typing.pyx +++ b/tests/memoryview/memoryview_pep484_typing.pyx @@ -1,5 +1,5 @@ # mode: run -# tag: pep489, memoryview +# tag: pep484, memoryview cimport cython diff --git a/tests/memoryview/memoryviewattrs.pyx b/tests/memoryview/memoryviewattrs.pyx index 66bc9da56..7322424c3 100644 --- a/tests/memoryview/memoryviewattrs.pyx +++ b/tests/memoryview/memoryviewattrs.pyx @@ -1,13 +1,6 @@ # mode: run # tag: numpy -__test__ = {} - -def testcase(func): - __test__[func.__name__] = func.__doc__ - return func - - cimport cython from cython.view cimport array @@ -15,7 +8,6 @@ import numpy as np cimport numpy as np -@testcase def test_shape_stride_suboffset(): u''' >>> test_shape_stride_suboffset() @@ -49,7 +41,6 @@ def test_shape_stride_suboffset(): print c_contig.suboffsets[0], c_contig.suboffsets[1], c_contig.suboffsets[2] -@testcase def test_copy_to(): u''' >>> test_copy_to() @@ -72,7 +63,6 @@ def test_copy_to(): print ' '.join(str(to_data[i]) for i in range(2*2*2)) -@testcase def test_overlapping_copy(): """ >>> test_overlapping_copy() @@ -88,7 +78,6 @@ def test_overlapping_copy(): assert slice[i] == 10 - 1 - i -@testcase def test_copy_return_type(): """ >>> test_copy_return_type() @@ -103,7 +92,6 @@ def test_copy_return_type(): print(f_contig[2, 2]) -@testcase def test_partly_overlapping(): """ >>> test_partly_overlapping() @@ -119,7 +107,6 @@ def test_partly_overlapping(): for i in range(5): assert slice2[i] == i + 4 -@testcase @cython.nonecheck(True) def test_nonecheck1(): u''' @@ -131,7 +118,6 @@ def test_nonecheck1(): cdef int[:,:,:] uninitialized print uninitialized.is_c_contig() -@testcase @cython.nonecheck(True) def test_nonecheck2(): u''' @@ -143,7 +129,6 @@ def test_nonecheck2(): cdef int[:,:,:] uninitialized print uninitialized.is_f_contig() -@testcase @cython.nonecheck(True) def test_nonecheck3(): u''' @@ -155,7 +140,6 @@ def test_nonecheck3(): cdef int[:,:,:] uninitialized uninitialized.copy() -@testcase @cython.nonecheck(True) def test_nonecheck4(): u''' @@ -167,7 +151,6 @@ def test_nonecheck4(): cdef int[:,:,:] uninitialized uninitialized.copy_fortran() -@testcase @cython.nonecheck(True) def test_nonecheck5(): u''' @@ -179,7 +162,6 @@ def test_nonecheck5(): cdef int[:,:,:] uninitialized uninitialized._data -@testcase def test_copy_mismatch(): u''' >>> test_copy_mismatch() @@ -193,7 +175,6 @@ def test_copy_mismatch(): mv1[...] = mv2 -@testcase def test_is_contiguous(): u""" >>> test_is_contiguous() @@ -222,7 +203,6 @@ def test_is_contiguous(): print 'strided', strided[::2].is_c_contig() -@testcase def call(): u''' >>> call() @@ -265,7 +245,6 @@ def call(): assert len(mv3) == 3 -@testcase def two_dee(): u''' >>> two_dee() @@ -313,7 +292,6 @@ def two_dee(): print (<long*>mv3._data)[0] , (<long*>mv3._data)[1] , (<long*>mv3._data)[2] , (<long*>mv3._data)[3] -@testcase def fort_two_dee(): u''' >>> fort_two_dee() diff --git a/tests/memoryview/memslice.pyx b/tests/memoryview/memslice.pyx index 827012c1e..d68ee43fe 100644 --- a/tests/memoryview/memslice.pyx +++ b/tests/memoryview/memslice.pyx @@ -1,5 +1,7 @@ # mode: run +# Test the behaviour of memoryview slicing and indexing, contiguity, etc. + # Note: see also bufaccess.pyx from __future__ import unicode_literals @@ -1058,6 +1060,8 @@ def addref(*args): def decref(*args): for item in args: Py_DECREF(item) +@cython.binding(False) +@cython.always_allow_keywords(False) def get_refcount(x): return (<PyObject*>x).ob_refcnt @@ -1577,7 +1581,7 @@ def test_index_slicing_away_direct_indirect(): All dimensions preceding dimension 1 must be indexed and not sliced """ cdef int[:, ::view.indirect, :] a = TestIndexSlicingDirectIndirectDims() - a_obj = a + cdef object a_obj = a print a[1][2][3] print a[1, 2, 3] @@ -1907,11 +1911,7 @@ def test_struct_attributes_format(): """ cdef TestAttrs[10] array cdef TestAttrs[:] struct_memview = array - - if sys.version_info[:2] >= (2, 7): - print builtins.memoryview(struct_memview).format - else: - print "T{i:int_attrib:c:char_attrib:}" + print builtins.memoryview(struct_memview).format # Test padding at the end of structs in the buffer support @@ -2199,7 +2199,7 @@ def test_object_dtype_copying(): 7 8 9 - 2 5 + 5 1 5 """ cdef int i @@ -2220,10 +2220,12 @@ def test_object_dtype_copying(): print m2[i] obj = m2[5] - print get_refcount(obj), obj + refcount1 = get_refcount(obj) + print obj del m2 - print get_refcount(obj), obj + refcount2 = get_refcount(obj) + print refcount1 - refcount2, obj assert unique_refcount == get_refcount(unique), (unique_refcount, get_refcount(unique)) @@ -2310,7 +2312,9 @@ def test_contig_scalar_to_slice_assignment(): """ >>> test_contig_scalar_to_slice_assignment() 14 14 14 14 + 30 14 30 14 20 20 20 20 + 30 30 20 20 """ cdef int[5][10] a cdef int[:, ::1] m = a @@ -2318,9 +2322,15 @@ def test_contig_scalar_to_slice_assignment(): m[...] = 14 print m[0, 0], m[-1, -1], m[3, 2], m[4, 9] + m[:, :1] = 30 + print m[0, 0], m[0, 1], m[1, 0], m[1, 1] + m[:, :] = 20 print m[0, 0], m[-1, -1], m[3, 2], m[4, 9] + m[:1, :] = 30 + print m[0, 0], m[0, 1], m[1, 0], m[1, 1] + @testcase def test_dtype_object_scalar_assignment(): """ diff --git a/tests/memoryview/numpy_memoryview.pyx b/tests/memoryview/numpy_memoryview.pyx index 9b18be615..6860fafeb 100644 --- a/tests/memoryview/numpy_memoryview.pyx +++ b/tests/memoryview/numpy_memoryview.pyx @@ -17,6 +17,10 @@ include "../buffers/mockbuffers.pxi" ctypedef np.int32_t dtype_t +IS_PYPY = hasattr(sys, 'pypy_version_info') +NUMPY_VERSION = tuple(int(v) for v in np.__version__.split('.')[:2]) +print(NUMPY_VERSION) + def get_array(): # We need to type our array to get a __pyx_get_buffer() that typechecks # for np.ndarray and calls __getbuffer__ in numpy.pxd @@ -32,22 +36,13 @@ def ae(*args): if x != args[0]: raise AssertionError(args) -__test__ = {} - -def testcase(f): - __test__[f.__name__] = f.__doc__ - return f - -def testcase_numpy_1_5(f): - major, minor, *rest = np.__version__.split('.') - if (int(major), int(minor)) >= (1, 5): - __test__[f.__name__] = f.__doc__ +def testcase_no_pypy(f, _is_pypy=hasattr(sys, "pypy_version_info")): + if _is_pypy: + f.__doc__ = "" # disable the tests return f - def gc_collect_if_required(): - major, minor, *rest = np.__version__.split('.') - if (int(major), int(minor)) >= (1, 14): + if NUMPY_VERSION >= (1, 14) or IS_PYPY: import gc gc.collect() @@ -56,7 +51,6 @@ def gc_collect_if_required(): ### Test slicing memoryview slices # -@testcase def test_partial_slicing(array): """ >>> test_partial_slicing(a) @@ -72,7 +66,6 @@ def test_partial_slicing(array): ae(b.strides[0], c.strides[0], obj.strides[0]) ae(b.strides[1], c.strides[1], obj.strides[1]) -@testcase def test_ellipsis(array): """ >>> test_ellipsis(a) @@ -114,7 +107,6 @@ def test_ellipsis(array): # ### Test slicing memoryview objects # -@testcase def test_partial_slicing_memoryview(array): """ >>> test_partial_slicing_memoryview(a) @@ -131,7 +123,6 @@ def test_partial_slicing_memoryview(array): ae(b.strides[0], c.strides[0], obj.strides[0]) ae(b.strides[1], c.strides[1], obj.strides[1]) -@testcase def test_ellipsis_memoryview(array): """ >>> test_ellipsis_memoryview(a) @@ -172,7 +163,6 @@ def test_ellipsis_memoryview(array): ae(e.strides[0], e_obj.strides[0]) -@testcase def test_transpose(): """ >>> test_transpose() @@ -186,7 +176,7 @@ def test_transpose(): numpy_obj = np.arange(4 * 3, dtype=np.int32).reshape(4, 3) a = numpy_obj - a_obj = a + cdef object a_obj = a cdef dtype_t[:, :] b = a.T print a.T.shape[0], a.T.shape[1] @@ -203,7 +193,6 @@ def test_transpose(): print a[3, 2], a.T[2, 3], a_obj[3, 2], a_obj.T[2, 3], numpy_obj[3, 2], numpy_obj.T[2, 3] -@testcase def test_transpose_type(a): """ >>> a = np.zeros((5, 10), dtype=np.float64) @@ -216,12 +205,8 @@ def test_transpose_type(a): print m_transpose[6, 4] -@testcase_numpy_1_5 def test_numpy_like_attributes(cyarray): """ - For some reason this fails in numpy 1.4, with shape () and strides (40, 8) - instead of 20, 4 on my machine. Investigate this. - >>> cyarray = create_array(shape=(8, 5), mode="c") >>> test_numpy_like_attributes(cyarray) >>> test_numpy_like_attributes(cyarray.memview) @@ -237,14 +222,13 @@ def test_numpy_like_attributes(cyarray): cdef int[:, :] mslice = numarray assert (<object> mslice).base is numarray -@testcase_numpy_1_5 def test_copy_and_contig_attributes(a): """ >>> a = np.arange(20, dtype=np.int32).reshape(5, 4) >>> test_copy_and_contig_attributes(a) """ cdef np.int32_t[:, :] mslice = a - m = mslice + cdef object m = mslice # object copy # Test object copy attributes assert np.all(a == np.array(m.copy())) @@ -274,7 +258,7 @@ def build_numarray(array array): def index(array array): print build_numarray(array)[3, 2] -@testcase_numpy_1_5 +@testcase_no_pypy def test_coerce_to_numpy(): """ Test coercion to NumPy arrays, especially with automatically @@ -355,6 +339,7 @@ def test_coerce_to_numpy(): 'e': 800, } + smallstructs[idx] = { 'a': 600, 'b': 700 } nestedstructs[idx] = { @@ -412,7 +397,7 @@ def test_coerce_to_numpy(): index(<td_h_ushort[:4, :5]> <td_h_ushort *> h_ushorts) -@testcase_numpy_1_5 +@testcase_no_pypy def test_memslice_getbuffer(): """ >>> test_memslice_getbuffer(); gc_collect_if_required() @@ -451,7 +436,6 @@ cdef packed struct StructArray: int a[4] signed char b[5] -@testcase_numpy_1_5 def test_memslice_structarray(data, dtype): """ >>> def b(s): return s.encode('ascii') @@ -507,7 +491,6 @@ def test_memslice_structarray(data, dtype): print myslice[i].a[j] print myslice[i].b.decode('ASCII') -@testcase_numpy_1_5 def test_structarray_errors(StructArray[:] a): """ >>> dtype = np.dtype([('a', '4i'), ('b', '5b')]) @@ -554,7 +537,6 @@ def stringstructtest(StringStruct[:] view): def stringtest(String[:] view): pass -@testcase_numpy_1_5 def test_string_invalid_dims(): """ >>> def b(s): return s.encode('ascii') @@ -575,7 +557,6 @@ ctypedef struct AttributesStruct: float attrib2 StringStruct attrib3 -@testcase_numpy_1_5 def test_struct_attributes(): """ >>> test_struct_attributes() @@ -631,7 +612,6 @@ cdef class SuboffsetsNoStridesBuffer(Buffer): getbuffer(self, info) info.suboffsets = self._shape -@testcase def test_null_strides(Buffer buffer_obj): """ >>> test_null_strides(Buffer()) @@ -651,7 +631,6 @@ def test_null_strides(Buffer buffer_obj): assert m2[i, j] == buffer_obj.m[i, j], (i, j, m2[i, j], buffer_obj.m[i, j]) assert m3[i, j] == buffer_obj.m[i, j] -@testcase def test_null_strides_error(buffer_obj): """ >>> test_null_strides_error(Buffer()) @@ -725,7 +704,6 @@ ctypedef struct SameTypeAfterArraysStructSimple: double b[16] double c -@testcase def same_type_after_arrays_simple(): """ >>> same_type_after_arrays_simple() @@ -747,7 +725,6 @@ ctypedef struct SameTypeAfterArraysStructComposite: double h[4] int i -@testcase def same_type_after_arrays_composite(): """ >>> same_type_after_arrays_composite() if sys.version_info[:2] >= (3, 5) else None diff --git a/tests/memoryview/numpy_memoryview_readonly.pyx b/tests/memoryview/numpy_memoryview_readonly.pyx index 20b6c7393..f1b289968 100644 --- a/tests/memoryview/numpy_memoryview_readonly.pyx +++ b/tests/memoryview/numpy_memoryview_readonly.pyx @@ -1,10 +1,14 @@ # mode: run # tag: readonly, const, numpy +# ticket: 1772 import numpy as np +cimport cython -def new_array(): - return np.arange(10).astype('float') +def new_array(dtype='float', writeable=True): + array = np.arange(10, dtype=dtype) + array.setflags(write=writeable) + return array ARRAY = new_array() @@ -124,3 +128,45 @@ def test_copy(): rw[1] = 2 rw2[2] = 2 return rw[0], rw[1], rw[2], rw2[0], rw2[1], rw2[2] + + +cdef getmax_floating(const cython.floating[:] x): + """Function with fused type, should work with both ro and rw memoryviews""" + cdef cython.floating max_val = - float('inf') + for val in x: + if val > max_val: + max_val = val + return max_val + + +def test_mmview_const_fused_cdef(): + """Test cdef function with const fused type memory view as argument. + + >>> test_mmview_const_fused_cdef() + """ + cdef float[:] data_rw = new_array(dtype='float32') + assert getmax_floating(data_rw) == 9 + + cdef const float[:] data_ro = new_array(dtype='float32', writeable=False) + assert getmax_floating(data_ro) == 9 + + +def test_mmview_const_fused_def(const cython.floating[:] x): + """Test def function with const fused type memory view as argument. + + With read-write numpy array: + + >>> test_mmview_const_fused_def(new_array('float32', writeable=True)) + 0.0 + >>> test_mmview_const_fused_def(new_array('float64', writeable=True)) + 0.0 + + With read-only numpy array: + + >>> test_mmview_const_fused_def(new_array('float32', writeable=False)) + 0.0 + >>> test_mmview_const_fused_def(new_array('float64', writeable=False)) + 0.0 + """ + cdef cython.floating result = x[0] + return result diff --git a/tests/memoryview/relaxed_strides.pyx b/tests/memoryview/relaxed_strides.pyx index 1743f23d0..6f90f4db5 100644 --- a/tests/memoryview/relaxed_strides.pyx +++ b/tests/memoryview/relaxed_strides.pyx @@ -10,12 +10,12 @@ Thanks to Nathaniel Smith and Sebastian Berg. See also: Mailing list threads: - http://thread.gmane.org/gmane.comp.python.cython.devel/14762 - http://thread.gmane.org/gmane.comp.python.cython.devel/14634 + https://thread.gmane.org/gmane.comp.python.cython.devel/14762 + https://thread.gmane.org/gmane.comp.python.cython.devel/14634 Detailed discussion of the difference between numpy/cython's current definition of "contiguity", and the correct definition: - http://thread.gmane.org/gmane.comp.python.cython.devel/14634/focus=14640 + https://thread.gmane.org/gmane.comp.python.cython.devel/14634/focus=14640 The PR implementing NPY_RELAXED_STRIDES_CHECKING: https://github.com/numpy/numpy/pull/3162 @@ -37,13 +37,6 @@ NUMPY_HAS_RELAXED_STRIDES = ( np.ones((10, 1), order="C").flags.f_contiguous) -def not_py26(f): - import sys - if sys.version_info < (2, 7): - return lambda a: None - return f - - def test_one_sized(array): """ >>> contig = np.ascontiguousarray(np.arange(10, dtype=np.double)[::100]) @@ -81,7 +74,6 @@ def test_zero_sized_multidim_ccontig(array): cdef double[:, :, ::1] a = array return a -@not_py26 def test_zero_sized_multidim_fcontig(array): """ >>> contig = np.ascontiguousarray(np.zeros((4,4,4))[::2, 2:2, ::2]) diff --git a/tests/memoryview/view_return_errors.pyx b/tests/memoryview/view_return_errors.pyx index 8e9443108..6cedef969 100644 --- a/tests/memoryview/view_return_errors.pyx +++ b/tests/memoryview/view_return_errors.pyx @@ -13,7 +13,18 @@ cdef double[:] foo(int i): raise TypeError('dummy') -def propagate(i): +cdef double[:] foo_nogil(int i) nogil: + if i == 1: + raise AttributeError('dummy') + if i == 2: + raise RuntimeError('dummy') + if i == 3: + raise ValueError('dummy') + if i == 4: + raise TypeError('dummy') + + +def propagate(i, bint nogil=False): """ >>> propagate(0) TypeError('Memoryview return value is not initialized') @@ -25,9 +36,20 @@ def propagate(i): ValueError('dummy') >>> propagate(4) TypeError('dummy') + + >>> propagate(0, nogil=True) + TypeError('Memoryview return value is not initialized') + >>> propagate(1, nogil=True) + AttributeError('dummy') + >>> propagate(2, nogil=True) + RuntimeError('dummy') + >>> propagate(3, nogil=True) + ValueError('dummy') + >>> propagate(4, nogil=True) + TypeError('dummy') """ try: - foo(i) + foo_nogil(i) if nogil else foo(i) except Exception as e: print '%s(%r)' % (e.__class__.__name__, e.args[0]) else: diff --git a/tests/pypy2_bugs.txt b/tests/pypy2_bugs.txt new file mode 100644 index 000000000..18fbcd204 --- /dev/null +++ b/tests/pypy2_bugs.txt @@ -0,0 +1,29 @@ +# Specific bugs that only apply to pypy2 + +build.cythonize_script +build.cythonize_script_package +run.initial_file_path +run.reduce_pickle +run.final_in_pxd +run.cdef_multiple_inheritance +run.cdef_multiple_inheritance_nodict +run.extstarargs +run.cpython_capi +run.isnot + +# pypy 2 seems to be preferring .py files to .so files +# https://foss.heptapod.net/pypy/pypy/issues/3185 +run.language_level +run.pure_pxd + +# Silly error with doctest matching slightly different string outputs rather than +# an actual bug but one I can't easily resolve +run.with_gil + + +# looks like a "when does the GC run?" issue - slightly surprised it's OK on pypy3 +memoryview.numpy_memoryview + +# type features that are disabled in PyPy2: +#run.test_genericclass +run.test_subclassinit diff --git a/tests/pypy_bugs.txt b/tests/pypy_bugs.txt index 77b814cf1..3d23c60b3 100644 --- a/tests/pypy_bugs.txt +++ b/tests/pypy_bugs.txt @@ -4,38 +4,64 @@ broken_exception bufaccess -memoryview -memslice +memoryview.memoryview$ sequential_parallel yield_from_pep380 memoryview_inplace_division +run.pycontextvar +run.unicodemethods +run.unicode_imports +run.test_genericclass +run.tp_new +run.test_fstring +run.test_exceptions +run.test_dictviews +run.str_subclass_kwargs +run.special_method_docstrings +run.slice_ptr +compile.min_async +run.cython_includes +run.pyarray +run.test_unicode +run.__getattribute__ +run.__getattribute_subclasses__ +run.__debug__ +run.array_cimport +run.builtin_abs +run.builtincomplex +run.cdef_multiple_inheritance_errors +run.cdivision_CEP_516 +run.cyfunction +run.final_cdef_class +run.index +run.pyclass_special_methods +run.reimport_from_package +run.reimport_from_subinterpreter run.reimport_failure +pkg.cimportfrom +embedded +TestCyCache +run.ext_auto_richcmp +run.coverage_cmd +run.coverage_cmd_src_layout +run.coverage_installed_pkg +run.coverage_api +run.coverage_nogil + +# very little coroutine-related seems to work +run.test_asyncgen +run.test_coroutines_pep492 +run.async_iter_pep492 +run.embedsignatures +run.py35_asyncio_async_def +run.asyncio_generators # gc issue? -memoryview_in_subclasses external_ref_reassignment run.exttype_dealloc # bugs in cpyext run.special_methods_T561 run.special_methods_T561_py2 - -# tests for things that don't exist in cpyext -compile.pylong -run.datetime_pxd -run.datetime_cimport -run.datetime_members -run.extern_builtins_T258 -run.line_trace -run.line_profile_test -run.pstats_profile_test -run.longintrepr - -# refcounting-specific tests -double_dealloc_T796 -run.exceptionrefcount -run.capiimpl -run.refcount_in_meth - diff --git a/tests/pypy_crash_bugs.txt b/tests/pypy_crash_bugs.txt new file mode 100644 index 000000000..779603146 --- /dev/null +++ b/tests/pypy_crash_bugs.txt @@ -0,0 +1,13 @@ +# Bugs that causes hard crashes that we certainly don't +# want to run because it will break the testsuite + +# segfault +run.fastcall +memslice + +# """Fatal RPython error: NotImplementedError +# Aborted (core dumped)""" +run.py35_pep492_interop + +# gc issue? +memoryview_in_subclasses diff --git a/tests/pypy_implementation_detail_bugs.txt b/tests/pypy_implementation_detail_bugs.txt new file mode 100644 index 000000000..ecf0b71dc --- /dev/null +++ b/tests/pypy_implementation_detail_bugs.txt @@ -0,0 +1,45 @@ +# PyPy "bugs" that are probably implementation differences from +# CPython rather than actual bugs. Therefore they aren't targets +# to be fixed (but there *may* be other details in the testfile +# that should be tested on PyPy?) + +run.starargs + +# refcounting-specific tests +double_dealloc_T796 +run.exceptionrefcount +run.capiimpl +run.refcount_in_meth +# Ideally just disable the reference-counting tests on PyPy? +run.fused_types +run.generator_frame_cycle +run.generators_in_refcycles +run.generators_py +run.parallel + +# "sys.getsizeof(object, default) will always return default on PyPy, and +# raise a TypeError if default is not provided." +buildenv + +# tests for things that don't exist in cpyext +compile.pylong +run.datetime_pxd +run.datetime_cimport +run.datetime_members +run.extern_builtins_T258 +run.line_trace +run.line_profile_test +run.pstats_profile_test +run.longintrepr + +# tests probably rely on immediate GC (although maybe the tests could be tweaked so +# only these bits don't run in PyPy?) +buffers.buffer +buffers.userbuffer +memoryview.cythonarray +memoryview.memoryview_pep484_typing +run.cpp_classes +run.cpp_classes_def + +# missing pypy feature? +matrix_multiplier diff --git a/tests/pyximport/pyximport_pyimport.srctree b/tests/pyximport/pyximport_pyimport.srctree index 69bc47988..753cef83f 100644 --- a/tests/pyximport/pyximport_pyimport.srctree +++ b/tests/pyximport/pyximport_pyimport.srctree @@ -8,7 +8,6 @@ import pyximport # blacklist for speed import pyximport.pyxbuild, Cython.Compiler.Pipeline -import distutils.core, distutils.ccompiler, distutils.command.build pyximport.install(pyximport=False, pyimport=True, build_dir=os.path.join(os.path.dirname(__file__), "TEST_TMP")) diff --git a/tests/run/addop.pyx b/tests/run/addop.pyx index 0e345d0e4..d6c3a3f65 100644 --- a/tests/run/addop.pyx +++ b/tests/run/addop.pyx @@ -166,3 +166,20 @@ def add_large_x(x): ... except TypeError: pass """ return 2**30 + x + + +def add0(x): + """ + >>> add0(0) + (0, 0) + >>> add0(1) + (1, 1) + >>> add0(-1) + (-1, -1) + >>> a, b = add0(2**32) + >>> bigint(a) + 4294967296 + >>> bigint(b) + 4294967296 + """ + return x + 0, 0 + x diff --git a/tests/run/always_allow_keywords_T295.pyx b/tests/run/always_allow_keywords_T295.pyx index 694839fcf..8e6e07739 100644 --- a/tests/run/always_allow_keywords_T295.pyx +++ b/tests/run/always_allow_keywords_T295.pyx @@ -1,7 +1,12 @@ -# ticket: 295 +# mode: run +# ticket: t295 cimport cython +import sys +IS_PY2 = sys.version_info[0] == 2 + + def assert_typeerror_no_keywords(func, *args, **kwds): # Python 3.9 produces an slightly different error message # to previous versions, so doctest isn't matching the @@ -14,13 +19,18 @@ def assert_typeerror_no_keywords(func, *args, **kwds): assert False, "call did not raise TypeError" +def func0(): + """ + >>> func0() + >>> func0(**{}) + """ + def func1(arg): """ >>> func1(None) >>> func1(*[None]) - >>> assert_typeerror_no_keywords(func1, arg=None) + >>> func1(arg=None) """ - pass @cython.always_allow_keywords(False) def func2(arg): @@ -29,7 +39,6 @@ def func2(arg): >>> func2(*[None]) >>> assert_typeerror_no_keywords(func2, arg=None) """ - pass @cython.always_allow_keywords(True) def func3(arg): @@ -40,26 +49,132 @@ def func3(arg): """ pass + cdef class A: """ - >>> A().meth1(None) - >>> A().meth1(*[None]) - >>> assert_typeerror_no_keywords(A().meth1, arg=None) - >>> A().meth2(None) - >>> A().meth2(*[None]) - >>> assert_typeerror_no_keywords(A().meth2, arg=None) - >>> A().meth3(None) - >>> A().meth3(*[None]) - >>> A().meth3(arg=None) + >>> class PyA(object): + ... def meth0(self): pass + ... def meth1(self, arg): pass + + >>> PyA().meth0() + >>> PyA.meth0(PyA()) + >>> if not IS_PY2: PyA.meth0(self=PyA()) + >>> try: PyA().meth0(self=PyA()) + ... except TypeError as exc: assert 'multiple' in str(exc), "Unexpected message: %s" % exc + ... else: assert False, "No TypeError when passing 'self' argument twice" + + >>> PyA().meth1(1) + >>> PyA.meth1(PyA(), 1) + >>> PyA.meth1(PyA(), arg=1) + >>> if not IS_PY2: PyA.meth1(self=PyA(), arg=1) """ + @cython.always_allow_keywords(False) + def meth0_nokw(self): + """ + >>> A().meth0_nokw() + >>> A().meth0_nokw(**{}) + >>> try: pass #A.meth0_nokw(self=A()) + ... except TypeError as exc: assert 'needs an argument' in str(exc), "Unexpected message: %s" % exc + ... else: pass #assert False, "No TypeError for missing 'self' positional argument" + """ + + @cython.always_allow_keywords(True) + def meth0_kw(self): + """ + >>> A().meth0_kw() + >>> A().meth0_kw(**{}) + >>> A.meth0_kw(A()) + >>> #A.meth0_kw(self=A()) + >>> try: pass #A().meth0_kw(self=A()) + ... except TypeError as exc: assert 'multiple' in str(exc), "Unexpected message: %s" % exc + ... else: pass #assert False, "No TypeError when passing 'self' argument twice" + """ + + @cython.always_allow_keywords(True) + def meth1_kw(self, arg): + """ + >>> A().meth1_kw(None) + >>> A().meth1_kw(*[None]) + >>> A().meth1_kw(arg=None) + >>> A.meth1_kw(A(), arg=None) + >>> #A.meth1_kw(self=A(), arg=None) + """ + + @cython.always_allow_keywords(False) + def meth1_nokw(self, arg): + """ + >>> A().meth1_nokw(None) + >>> A().meth1_nokw(*[None]) + >>> assert_typeerror_no_keywords(A().meth1_nokw, arg=None) + >>> assert_typeerror_no_keywords(A.meth1_nokw, A(), arg=None) + >>> try: pass # A.meth1_nokw(self=A(), arg=None) + ... except TypeError as exc: assert 'needs an argument' in str(exc), "Unexpected message: %s" % exc + ... else: pass # assert False, "No TypeError for missing 'self' positional argument" + """ + + @cython.always_allow_keywords(False) + def meth2(self, arg): + """ + >>> A().meth2(None) + >>> A().meth2(*[None]) + >>> assert_typeerror_no_keywords(A().meth2, arg=None) + """ + + @cython.always_allow_keywords(True) + def meth3(self, arg): + """ + >>> A().meth3(None) + >>> A().meth3(*[None]) + >>> A().meth3(arg=None) + """ + + +class B(object): + @cython.always_allow_keywords(False) + def meth0_nokw(self): + """ + >>> B().meth0_nokw() + >>> B().meth0_nokw(**{}) + >>> if not IS_PY2: assert_typeerror_no_keywords(B.meth0_nokw, self=B()) + """ + + @cython.always_allow_keywords(True) + def meth0_kw(self): + """ + >>> B().meth0_kw() + >>> B().meth0_kw(**{}) + >>> B.meth0_kw(B()) + >>> if not IS_PY2: B.meth0_kw(self=B()) + >>> try: B().meth0_kw(self=B()) + ... except TypeError as exc: assert 'multiple' in str(exc), "Unexpected message: %s" % exc + ... else: assert False, "No TypeError when passing 'self' argument twice" + """ + + @cython.always_allow_keywords(True) def meth1(self, arg): - pass + """ + >>> B().meth1(None) + >>> B().meth1(*[None]) + >>> B().meth1(arg=None) + >>> B.meth1(B(), arg=None) + >>> if not IS_PY2: B.meth1(self=B(), arg=None) + """ @cython.always_allow_keywords(False) def meth2(self, arg): - pass + """ + >>> B().meth2(None) + >>> B().meth2(*[None]) + >>> B.meth2(B(), None) + >>> if not IS_PY2: B.meth2(self=B(), arg=None) + >>> B().meth2(arg=None) # assert_typeerror_no_keywords(B().meth2, arg=None) -> not a cdef class! + """ @cython.always_allow_keywords(True) def meth3(self, arg): - pass + """ + >>> B().meth3(None) + >>> B().meth3(*[None]) + >>> B().meth3(arg=None) + """ diff --git a/tests/run/annotate_html.pyx b/tests/run/annotate_html.pyx index 765c2e13f..3db7bf190 100644 --- a/tests/run/annotate_html.pyx +++ b/tests/run/annotate_html.pyx @@ -11,6 +11,8 @@ >>> import re >>> assert re.search('<pre .*def.* .*mixed_test.*</pre>', html) +>>> from Cython.Compiler.Annotate import AnnotationCCodeWriter +>>> assert (AnnotationCCodeWriter.COMPLETE_CODE_TITLE not in html) # per default no complete c code """ diff --git a/tests/run/annotation_typing.pyx b/tests/run/annotation_typing.pyx index 2582c3d03..7b988c112 100644 --- a/tests/run/annotation_typing.pyx +++ b/tests/run/annotation_typing.pyx @@ -3,6 +3,7 @@ cimport cython from cython cimport typeof +from cpython.ref cimport PyObject def old_dict_syntax(a: list, b: "int" = 2, c: {'ctype': 'long int'} = 3, d: {'type': 'float'} = 4) -> list: @@ -196,6 +197,37 @@ def call_exception_default(raise_exc=False): return exception_default(raise_exc) +@cython.test_assert_path_exists( + "//CFuncDefNode", + "//CFuncDefNode//DefNode", + "//CFuncDefNode[@return_type]", + "//CFuncDefNode[@return_type.is_int = True]", +) +@cython.ccall +def exception_default_uint(raise_exc : cython.bint = False) -> cython.uint: + """ + >>> print(exception_default_uint(raise_exc=False)) + 10 + >>> exception_default_uint(raise_exc=True) + Traceback (most recent call last): + ValueError: huhu! + """ + if raise_exc: + raise ValueError("huhu!") + return 10 + + +def call_exception_default_uint(raise_exc=False): + """ + >>> print(call_exception_default_uint(raise_exc=False)) + 10 + >>> call_exception_default_uint(raise_exc=True) + Traceback (most recent call last): + ValueError: huhu! + """ + return exception_default_uint(raise_exc) + + class EarlyClass(object): """ >>> a = EarlyClass(1) @@ -231,18 +263,36 @@ def py_float_default(price : float=None, ndigits=4): return price, ndigits +cdef class ClassAttribute: + cls_attr : float = 1. + + +@cython.cfunc +def take_ptr(obj: cython.pointer(PyObject)): + pass + +def call_take_ptr(): + """ + >>> call_take_ptr() # really just a compile-test + """ + python_dict = {"abc": 123} + take_ptr(cython.cast(cython.pointer(PyObject), python_dict)) + + _WARNINGS = """ -8:32: Strings should no longer be used for type declarations. Use 'cython.int' etc. directly. -8:47: Dicts should no longer be used as type annotations. Use 'cython.int' etc. directly. -8:56: Strings should no longer be used for type declarations. Use 'cython.int' etc. directly. -8:77: Dicts should no longer be used as type annotations. Use 'cython.int' etc. directly. -8:85: Python type declaration in signature annotation does not refer to a Python type -8:85: Strings should no longer be used for type declarations. Use 'cython.int' etc. directly. -211:44: Unknown type declaration in annotation, ignoring -218:29: Ambiguous types in annotation, ignoring +9:32: Strings should no longer be used for type declarations. Use 'cython.int' etc. directly. +9:47: Dicts should no longer be used as type annotations. Use 'cython.int' etc. directly. +9:56: Strings should no longer be used for type declarations. Use 'cython.int' etc. directly. +9:77: Dicts should no longer be used as type annotations. Use 'cython.int' etc. directly. +9:85: Python type declaration in signature annotation does not refer to a Python type +9:85: Strings should no longer be used for type declarations. Use 'cython.int' etc. directly. +243:44: Unknown type declaration in annotation, ignoring +250:29: Ambiguous types in annotation, ignoring +267:15: Annotation ignored since class-level attributes must be Python objects. Were you trying to set up an instance attribute? # BUG: -46:6: 'pytypes_cpdef' redeclared +47:6: 'pytypes_cpdef' redeclared 121:0: 'struct_io' redeclared -156:0: 'struct_convert' redeclared -175:0: 'exception_default' redeclared +150:0: 'struct_convert' redeclared +169:0: 'exception_default' redeclared +200:0: 'exception_default_uint' redeclared """ diff --git a/tests/run/args_unpacking_in_closure_T658.pyx b/tests/run/args_unpacking_in_closure_T658.pyx index 98b57743b..b52d689c9 100644 --- a/tests/run/args_unpacking_in_closure_T658.pyx +++ b/tests/run/args_unpacking_in_closure_T658.pyx @@ -1,6 +1,6 @@ # mode: run # tag: closures -# ticket: 658 +# ticket: t658 def outer(int x, *args, **kwargs): """ diff --git a/tests/run/argument_unpacking_closure_T736.py b/tests/run/argument_unpacking_closure_T736.py index 6c83d7026..80c824417 100644 --- a/tests/run/argument_unpacking_closure_T736.py +++ b/tests/run/argument_unpacking_closure_T736.py @@ -1,5 +1,5 @@ # mode: run -# ticket: 736 +# ticket: t736 # tag: default arguments, closure def default_args_for_closure(a=1, b=2): diff --git a/tests/run/arithmetic_analyse_types.pyx b/tests/run/arithmetic_analyse_types.pyx index 7d5e41666..c8c7f7ac8 100644 --- a/tests/run/arithmetic_analyse_types.pyx +++ b/tests/run/arithmetic_analyse_types.pyx @@ -1,4 +1,4 @@ -# ticket: 676 +# ticket: t676 # tag: cpp from cython cimport typeof diff --git a/tests/run/array_cimport.srctree b/tests/run/array_cimport.srctree index 4fefade5f..5951f2604 100644 --- a/tests/run/array_cimport.srctree +++ b/tests/run/array_cimport.srctree @@ -32,6 +32,6 @@ from tt cimport Foo cdef array a = array('i', [1,2,3]) cdef Foo x -print a.data.as_ints[0] +print(a.data.as_ints[0]) x = Foo(a) -print x.obj.data.as_ints[0] +print(x.obj.data.as_ints[0]) diff --git a/tests/run/assert.pyx b/tests/run/assert.pyx index 1e6cc17be..2e3dedf8e 100644 --- a/tests/run/assert.pyx +++ b/tests/run/assert.pyx @@ -2,6 +2,10 @@ cimport cython +@cython.test_assert_path_exists( + '//AssertStatNode', + '//AssertStatNode//RaiseStatNode', +) def f(a, b, int i): """ >>> f(1, 2, 1) @@ -22,7 +26,9 @@ def f(a, b, int i): @cython.test_assert_path_exists( '//AssertStatNode', - '//AssertStatNode//TupleNode') + '//AssertStatNode//RaiseStatNode', + '//AssertStatNode//RaiseStatNode//TupleNode', +) def g(a, b): """ >>> g(1, "works") @@ -38,7 +44,9 @@ def g(a, b): @cython.test_assert_path_exists( '//AssertStatNode', - '//AssertStatNode//TupleNode') + '//AssertStatNode//RaiseStatNode', + '//AssertStatNode//RaiseStatNode//TupleNode', +) def g(a, b): """ >>> g(1, "works") @@ -54,8 +62,9 @@ def g(a, b): @cython.test_assert_path_exists( '//AssertStatNode', - '//AssertStatNode//TupleNode', - '//AssertStatNode//TupleNode//TupleNode') + '//AssertStatNode//RaiseStatNode', + '//AssertStatNode//RaiseStatNode//TupleNode', + '//AssertStatNode//RaiseStatNode//TupleNode//TupleNode',) def assert_with_tuple_arg(a): """ >>> assert_with_tuple_arg(True) @@ -67,9 +76,12 @@ def assert_with_tuple_arg(a): @cython.test_assert_path_exists( - '//AssertStatNode') + '//AssertStatNode', + '//AssertStatNode//RaiseStatNode', +) @cython.test_fail_if_path_exists( - '//AssertStatNode//TupleNode') + '//AssertStatNode//TupleNode', +) def assert_with_str_arg(a): """ >>> assert_with_str_arg(True) diff --git a/tests/run/bad_c_struct_T252.pyx b/tests/run/bad_c_struct_T252.pyx index 247a55202..57b1155c2 100644 --- a/tests/run/bad_c_struct_T252.pyx +++ b/tests/run/bad_c_struct_T252.pyx @@ -1,4 +1,4 @@ -# ticket: 252 +# ticket: t252 cdef cf(default=None): return default diff --git a/tests/run/binop_reverse_methods_GH2056.pyx b/tests/run/binop_reverse_methods_GH2056.pyx new file mode 100644 index 000000000..15ef5b0e7 --- /dev/null +++ b/tests/run/binop_reverse_methods_GH2056.pyx @@ -0,0 +1,170 @@ +cimport cython + +@cython.c_api_binop_methods(False) +@cython.cclass +class Base(object): + """ + >>> Base() + 2 + 'Base.__add__(Base(), 2)' + >>> 2 + Base() + 'Base.__radd__(Base(), 2)' + + >>> Base(implemented=False) + 2 #doctest: +ELLIPSIS + Traceback (most recent call last): + ... + TypeError: unsupported operand type... + >>> 2 + Base(implemented=False) #doctest: +ELLIPSIS + Traceback (most recent call last): + ... + TypeError: unsupported operand type... + + >>> Base() ** 2 + 'Base.__pow__(Base(), 2, None)' + >>> 2 ** Base() + 'Base.__rpow__(Base(), 2, None)' + >>> pow(Base(), 2, 100) + 'Base.__pow__(Base(), 2, 100)' + """ + implemented: cython.bint + + def __init__(self, *, implemented=True): + self.implemented = implemented + + def __add__(self, other): + if (<Base>self).implemented: + return "Base.__add__(%s, %s)" % (self, other) + else: + return NotImplemented + + def __radd__(self, other): + if (<Base>self).implemented: + return "Base.__radd__(%s, %s)" % (self, other) + else: + return NotImplemented + + def __pow__(self, other, mod): + if (<Base>self).implemented: + return "Base.__pow__(%s, %s, %s)" % (self, other, mod) + else: + return NotImplemented + + def __rpow__(self, other, mod): + if (<Base>self).implemented: + return "Base.__rpow__(%s, %s, %s)" % (self, other, mod) + else: + return NotImplemented + + def __repr__(self): + return "%s()" % (self.__class__.__name__) + +@cython.c_api_binop_methods(False) +@cython.cclass +class OverloadLeft(Base): + """ + >>> OverloadLeft() + 2 + 'OverloadLeft.__add__(OverloadLeft(), 2)' + >>> 2 + OverloadLeft() + 'Base.__radd__(OverloadLeft(), 2)' + + >>> OverloadLeft() + Base() + 'OverloadLeft.__add__(OverloadLeft(), Base())' + >>> Base() + OverloadLeft() + 'Base.__add__(Base(), OverloadLeft())' + + >>> OverloadLeft(implemented=False) + Base(implemented=False) #doctest: +ELLIPSIS + Traceback (most recent call last): + ... + TypeError: unsupported operand type... + >>> Base(implemented=False) + OverloadLeft(implemented=False) #doctest: +ELLIPSIS + Traceback (most recent call last): + ... + TypeError: unsupported operand type... + """ + derived_implemented: cython.bint + + def __init__(self, *, implemented=True): + super().__init__(implemented=implemented) + self.derived_implemented = implemented + + def __add__(self, other): + if (<OverloadLeft>self).derived_implemented: + return "OverloadLeft.__add__(%s, %s)" % (self, other) + else: + return NotImplemented + + +@cython.c_api_binop_methods(False) +@cython.cclass +class OverloadRight(Base): + """ + >>> OverloadRight() + 2 + 'Base.__add__(OverloadRight(), 2)' + >>> 2 + OverloadRight() + 'OverloadRight.__radd__(OverloadRight(), 2)' + + >>> OverloadRight() + Base() + 'Base.__add__(OverloadRight(), Base())' + >>> Base() + OverloadRight() + 'OverloadRight.__radd__(OverloadRight(), Base())' + + >>> OverloadRight(implemented=False) + Base(implemented=False) #doctest: +ELLIPSIS + Traceback (most recent call last): + ... + TypeError: unsupported operand type... + >>> Base(implemented=False) + OverloadRight(implemented=False) #doctest: +ELLIPSIS + Traceback (most recent call last): + ... + TypeError: unsupported operand type... + """ + derived_implemented: cython.bint + + def __init__(self, *, implemented=True): + super().__init__(implemented=implemented) + self.derived_implemented = implemented + + def __radd__(self, other): + if (<OverloadRight>self).derived_implemented: + return "OverloadRight.__radd__(%s, %s)" % (self, other) + else: + return NotImplemented + +@cython.c_api_binop_methods(True) +@cython.cclass +class OverloadCApi(Base): + """ + >>> OverloadCApi() + 2 + 'OverloadCApi.__add__(OverloadCApi(), 2)' + >>> 2 + OverloadCApi() + 'OverloadCApi.__add__(2, OverloadCApi())' + + >>> OverloadCApi() + Base() + 'OverloadCApi.__add__(OverloadCApi(), Base())' + >>> Base() + OverloadCApi() + 'OverloadCApi.__add__(Base(), OverloadCApi())' + + >>> OverloadCApi(derived_implemented=False) + 2 #doctest: +ELLIPSIS + Traceback (most recent call last): + ... + TypeError: unsupported operand type... + >>> 2 + OverloadCApi(derived_implemented=False) #doctest: +ELLIPSIS + Traceback (most recent call last): + ... + TypeError: unsupported operand type... + """ + derived_implemented: cython.bint + + def __init__(self, *, derived_implemented=True): + super().__init__(implemented=True) + self.derived_implemented = derived_implemented + + def __add__(self, other): + if isinstance(self, OverloadCApi): + derived_implemented = (<OverloadCApi>self).derived_implemented + else: + derived_implemented = (<OverloadCApi>other).derived_implemented + if derived_implemented: + return "OverloadCApi.__add__(%s, %s)" % (self, other) + else: + return NotImplemented + +# TODO: Test a class that only defines the `__r...__()` methods. diff --git a/tests/run/bint_binop_T145.pyx b/tests/run/bint_binop_T145.pyx index c6df52303..f0f0ef560 100644 --- a/tests/run/bint_binop_T145.pyx +++ b/tests/run/bint_binop_T145.pyx @@ -1,4 +1,4 @@ -# ticket: 145 +# ticket: t145 cimport cython diff --git a/tests/run/bint_property_T354.pyx b/tests/run/bint_property_T354.pyx index 5a461ac76..ef4c623ee 100644 --- a/tests/run/bint_property_T354.pyx +++ b/tests/run/bint_property_T354.pyx @@ -1,4 +1,4 @@ -# ticket: 354 +# ticket: t354 cdef class Test: """ diff --git a/tests/run/bound_builtin_methods_T589.pyx b/tests/run/bound_builtin_methods_T589.pyx index 18a517cbd..1d2fb9f66 100644 --- a/tests/run/bound_builtin_methods_T589.pyx +++ b/tests/run/bound_builtin_methods_T589.pyx @@ -1,4 +1,4 @@ -# ticket: 589 +# ticket: t589 cimport cython diff --git a/tests/run/builtin_abs.pyx b/tests/run/builtin_abs.pyx index ba6351d9b..59f3a93c4 100644 --- a/tests/run/builtin_abs.pyx +++ b/tests/run/builtin_abs.pyx @@ -1,5 +1,5 @@ # mode: run -# ticket: 698 +# ticket: t698 # distutils: extra_compile_args=-fwrapv cdef extern from *: @@ -60,6 +60,27 @@ def int_abs(int a): """ return abs(a) +@cython.overflowcheck(True) +@cython.test_assert_path_exists("//ReturnStatNode//NameNode[@entry.name = 'abs']", + "//ReturnStatNode//NameNode[@entry.cname = 'abs']") +cdef int c_int_abs(int a) nogil except *: + return abs(a) + +def test_c_int_abs(int a): + """ + >>> test_c_int_abs(-5) == 5 + True + >>> test_c_int_abs(-5.1) == 5 + True + >>> test_c_int_abs(-max_int-1) #doctest: +ELLIPSIS + Traceback (most recent call last): + ... + OverflowError: ... + >>> test_c_int_abs(max_int) == abs(max_int) or (max_int, test_c_int_abs(max_int), abs(max_int)) + True + """ + return c_int_abs(a) + @cython.test_assert_path_exists("//ReturnStatNode//NameNode[@entry.name = 'abs']") @cython.test_fail_if_path_exists("//ReturnStatNode//NameNode[@entry.cname = 'abs']", "//ReturnStatNode//NameNode[@entry.cname = 'labs']") @@ -70,6 +91,19 @@ def uint_abs(unsigned int a): """ return abs(a) +@cython.test_assert_path_exists("//ReturnStatNode//NameNode[@entry.name = 'abs']") +@cython.test_fail_if_path_exists("//ReturnStatNode//NameNode[@entry.cname = 'abs']", + "//ReturnStatNode//NameNode[@entry.cname = 'labs']") +cdef unsigned int c_uint_abs(unsigned int a) nogil: + return abs(a) + +def test_c_uint_abs(unsigned int a): + """ + >>> test_c_uint_abs(max_int) == abs(max_int) or (max_int, test_c_uint_abs(max_int), abs(max_int)) + True + """ + return c_uint_abs(a) + @cython.overflowcheck(True) @cython.test_assert_path_exists("//ReturnStatNode//NameNode[@entry.name = 'abs']", "//ReturnStatNode//NameNode[@entry.cname = 'labs']") @@ -88,6 +122,27 @@ def long_abs(long a): """ return abs(a) +@cython.overflowcheck(True) +@cython.test_assert_path_exists("//ReturnStatNode//NameNode[@entry.name = 'abs']", + "//ReturnStatNode//NameNode[@entry.cname = 'labs']") +cdef long c_long_abs(long a) nogil except *: + return abs(a) + +def test_c_long_abs(long a): + """ + >>> test_c_long_abs(-5) == 5 + True + >>> test_c_long_abs(-5.1) == 5 + True + >>> test_c_long_abs(-max_long-1) #doctest: +ELLIPSIS + Traceback (most recent call last): + ... + OverflowError: ... + >>> test_c_long_abs(max_long) == abs(max_long) or (max_long, test_c_long_abs(max_long), abs(max_long)) + True + """ + return c_long_abs(a) + @cython.test_assert_path_exists("//ReturnStatNode//NameNode[@entry.name = 'abs']") @cython.test_fail_if_path_exists("//ReturnStatNode//NameNode[@entry.cname = 'abs']", "//ReturnStatNode//NameNode[@entry.cname = 'labs']") @@ -100,6 +155,21 @@ def ulong_abs(unsigned long a): """ return abs(a) +@cython.test_assert_path_exists("//ReturnStatNode//NameNode[@entry.name = 'abs']") +@cython.test_fail_if_path_exists("//ReturnStatNode//NameNode[@entry.cname = 'abs']", + "//ReturnStatNode//NameNode[@entry.cname = 'labs']") +cdef unsigned long c_ulong_abs(unsigned long a) nogil: + return abs(a) + +def test_c_ulong_abs(unsigned long a): + """ + >>> test_c_ulong_abs(max_long) == abs(max_long) or (max_int, test_c_ulong_abs(max_long), abs(max_long)) + True + >>> test_c_ulong_abs(max_long + 5) == abs(max_long + 5) or (max_long + 5, test_c_ulong_abs(max_long + 5), abs(max_long + 5)) + True + """ + return c_ulong_abs(a) + @cython.overflowcheck(True) @cython.test_assert_path_exists("//ReturnStatNode//NameNode[@entry.name = 'abs']", "//ReturnStatNode//NameNode[@entry.cname = '__Pyx_abs_longlong']") @@ -116,6 +186,25 @@ def long_long_abs(long long a): """ return abs(a) +@cython.overflowcheck(True) +@cython.test_assert_path_exists("//ReturnStatNode//NameNode[@entry.name = 'abs']", + "//ReturnStatNode//NameNode[@entry.cname = '__Pyx_abs_longlong']") +cdef long long c_long_long_abs(long long a) nogil except *: + return abs(a) + +def test_c_long_long_abs(long long a): + """ + >>> test_c_long_long_abs(-(2**33)) == 2**33 + True + >>> test_c_long_long_abs(-max_long_long-1) #doctest: +ELLIPSIS + Traceback (most recent call last): + ... + OverflowError: ... + >>> test_c_long_long_abs(max_long_long) == abs(max_long_long) or (max_long_long, test_c_long_long_abs(max_long_long), abs(max_long_long)) + True + """ + return c_long_long_abs(a) + @cython.test_assert_path_exists("//ReturnStatNode//NameNode[@entry.name = 'abs']", "//ReturnStatNode//NameNode[@entry.cname = 'fabs']") def double_abs(double a): @@ -128,6 +217,20 @@ def double_abs(double a): return abs(a) @cython.test_assert_path_exists("//ReturnStatNode//NameNode[@entry.name = 'abs']", + "//ReturnStatNode//NameNode[@entry.cname = 'fabs']") +cdef double c_double_abs(double a) nogil: + return abs(a) + +def test_c_double_abs(double a): + """ + >>> test_c_double_abs(-5) + 5.0 + >>> test_c_double_abs(-5.5) + 5.5 + """ + return c_double_abs(a) + +@cython.test_assert_path_exists("//ReturnStatNode//NameNode[@entry.name = 'abs']", "//ReturnStatNode//NameNode[@entry.cname = 'fabsf']") def float_abs(float a): """ @@ -139,6 +242,20 @@ def float_abs(float a): return abs(a) @cython.test_assert_path_exists("//ReturnStatNode//NameNode[@entry.name = 'abs']", + "//ReturnStatNode//NameNode[@entry.cname = 'fabsf']") +cdef float c_float_abs(float a) nogil: + return abs(a) + +def test_c_float_abs(float a): + """ + >>> test_c_float_abs(-5) + 5.0 + >>> test_c_float_abs(-5.5) + 5.5 + """ + return c_float_abs(a) + +@cython.test_assert_path_exists("//ReturnStatNode//NameNode[@entry.name = 'abs']", "//ReturnStatNode//NameNode[@entry.cname = '__Pyx_c_abs_double']") def complex_abs(complex a): """ @@ -148,3 +265,17 @@ def complex_abs(complex a): 5.5 """ return abs(a) + +@cython.test_assert_path_exists("//ReturnStatNode//NameNode[@entry.name = 'abs']", + "//ReturnStatNode//NameNode[@entry.cname = '__Pyx_c_abs_double']") +cdef double c_complex_abs(complex a) nogil: + return abs(a) + +def test_c_complex_abs(complex a): + """ + >>> test_c_complex_abs(-5j) + 5.0 + >>> test_c_complex_abs(-5.5j) + 5.5 + """ + return c_complex_abs(a) diff --git a/tests/run/builtin_float.py b/tests/run/builtin_float.py index f2eff2be8..8e135a607 100644 --- a/tests/run/builtin_float.py +++ b/tests/run/builtin_float.py @@ -1,6 +1,20 @@ +# mode: run +# tag: pure3.0 +import cython import sys +def fix_underscores(s): + if sys.version_info < (3, 6) or getattr(sys, 'pypy_version_info', (9, 9)) < (3, 7, 4): + # Py2 float() does not support PEP-515 underscore literals + if isinstance(s, bytes): + if not cython.compiled and b'_' in s: + return s.replace(b'_', b'') + elif '_' in s: + return s.replace('_', '') + return s + + def empty_float(): """ >>> float() @@ -11,24 +25,254 @@ def empty_float(): x = float() return x + def float_conjugate(): """ >>> float_call_conjugate() 1.5 """ - if sys.version_info >= (2,6): - x = 1.5 .conjugate() - else: - x = 1.5 + x = 1.5 .conjugate() return x + def float_call_conjugate(): """ >>> float_call_conjugate() 1.5 """ - if sys.version_info >= (2,6): - x = float(1.5).conjugate() - else: - x = 1.5 + x = float(1.5).conjugate() return x + + +def from_int(i): + """ + >>> from_int(0) + 0.0 + >>> from_int(1) + 1.0 + >>> from_int(-1) + -1.0 + >>> from_int(99) + 99.0 + >>> from_int(-99) + -99.0 + + >>> for exp in (14, 15, 16, 30, 31, 32, 52, 53, 54, 60, 61, 62, 63, 64): + ... for sign in (1, 0, -1): + ... value = (sign or 1) * 2**exp + sign + ... float_value = from_int(value) + ... assert float_value == float(value), "expected %s2**%s+%s == %r, got %r, difference %r" % ( + ... '-' if sign < 0 else '', exp, sign, float(value), float_value, float_value - float(value)) + """ + return float(i) + + +@cython.test_assert_path_exists( + "//CoerceToPyTypeNode", + "//CoerceToPyTypeNode//PythonCapiCallNode", +) +def from_bytes(s: bytes): + """ + >>> from_bytes(b"123") + 123.0 + >>> from_bytes(b"123.25") + 123.25 + >>> from_bytes(fix_underscores(b"98_5_6.2_1")) + 9856.21 + >>> from_bytes(fix_underscores(b"12_4_131_123123_1893798127398123_19238712_128937198237.8222113_519879812387")) + 1.2413112312318938e+47 + >>> from_bytes(b"123E100") + 1.23e+102 + >>> from_bytes(b"12__._3") # doctest: +ELLIPSIS + Traceback (most recent call last): + ValueError: ...12__._3... + >>> from_bytes(b"_12.3") # doctest: +ELLIPSIS + Traceback (most recent call last): + ValueError: ..._12.3... + >>> from_bytes(b"12.3_") # doctest: +ELLIPSIS + Traceback (most recent call last): + ValueError: ...12.3_... + >>> from_bytes(b"na_n") # doctest: +ELLIPSIS + Traceback (most recent call last): + ValueError: ...na_n... + >>> from_bytes(None) # doctest: +ELLIPSIS + Traceback (most recent call last): + TypeError... + """ + return float(s) + + +@cython.test_assert_path_exists( + "//CoerceToPyTypeNode", + "//CoerceToPyTypeNode//PythonCapiCallNode", +) +def from_bytes_literals(): + """ + >>> from_bytes_literals() + (123.0, 123.23, 123.76, 1e+100) + """ + return float(b"123"), float(b"123.23"), float(fix_underscores(b"12_3.7_6")), float(b"1e100") + + +@cython.test_assert_path_exists( + "//CoerceToPyTypeNode", + "//CoerceToPyTypeNode//PythonCapiCallNode", +) +def from_bytearray(s: bytearray): + """ + >>> from_bytearray(bytearray(b"123")) + 123.0 + >>> from_bytearray(bytearray(b"123.25")) + 123.25 + >>> from_bytearray(bytearray(fix_underscores(b"98_5_6.2_1"))) + 9856.21 + >>> from_bytearray(bytearray(fix_underscores(b"12_4_131_123123_1893798127398123_19238712_128937198237.8222113_519879812387"))) + 1.2413112312318938e+47 + >>> from_bytearray(bytearray(b"123E100")) + 1.23e+102 + >>> from_bytearray(bytearray(b"12__._3")) # doctest: +ELLIPSIS + Traceback (most recent call last): + ValueError: ...12__._3... + >>> from_bytearray(bytearray(b"_12.3")) # doctest: +ELLIPSIS + Traceback (most recent call last): + ValueError: ..._12.3... + >>> from_bytearray(bytearray(b"12.3_")) # doctest: +ELLIPSIS + Traceback (most recent call last): + ValueError: ...12.3_... + >>> from_bytearray(bytearray(b"in_f")) # doctest: +ELLIPSIS + Traceback (most recent call last): + ValueError: ...in_f... + >>> from_bytearray(None) # doctest: +ELLIPSIS + Traceback (most recent call last): + TypeError... + """ + return float(s) + + +@cython.test_assert_path_exists( + "//CoerceToPyTypeNode", + "//CoerceToPyTypeNode//PythonCapiCallNode", +) +def from_str(s: 'str'): + """ + >>> from_str("123") + 123.0 + >>> from_str("123.25") + 123.25 + >>> from_str(fix_underscores("3_21.2_5")) + 321.25 + >>> from_str(fix_underscores("12_4_131_123123_1893798127398123_19238712_128937198237.8222113_519879812387")) + 1.2413112312318938e+47 + >>> from_str("123E100") + 1.23e+102 + >>> from_str("12__._3") # doctest: +ELLIPSIS + Traceback (most recent call last): + ValueError: ...12__._3... + >>> from_str("_12.3") # doctest: +ELLIPSIS + Traceback (most recent call last): + ValueError: ..._12.3... + >>> from_str("12.3_") # doctest: +ELLIPSIS + Traceback (most recent call last): + ValueError: ...12.3_... + >>> from_str("n_an") # doctest: +ELLIPSIS + Traceback (most recent call last): + ValueError: ...n_an... + >>> from_str(None) # doctest: +ELLIPSIS + Traceback (most recent call last): + TypeError... + """ + return float(s) + + +@cython.test_assert_path_exists( + "//CoerceToPyTypeNode", + "//CoerceToPyTypeNode//PythonCapiCallNode", +) +def from_str_literals(): + """ + >>> from_str_literals() + (123.0, 123.23, 124.23, 1e+100) + """ + return float("123"), float("123.23"), float(fix_underscores("1_2_4.2_3")), float("1e100") + + +@cython.test_assert_path_exists( + "//CoerceToPyTypeNode", + "//CoerceToPyTypeNode//PythonCapiCallNode", +) +def from_unicode(s: 'unicode'): + """ + >>> from_unicode(u"123") + 123.0 + >>> from_unicode(u"123.25") + 123.25 + >>> from_unicode(fix_underscores(u"12_4.8_5")) + 124.85 + >>> from_unicode(fix_underscores(u"12_4_131_123123_1893798127398123_19238712_128937198237.8222113_519879812387")) + 1.2413112312318938e+47 + >>> from_unicode(u"123E100") + 1.23e+102 + >>> from_unicode(u"123.23\\N{PUNCTUATION SPACE}") + 123.23 + >>> from_unicode(u"\\N{PUNCTUATION SPACE} 123.23 \\N{PUNCTUATION SPACE}") + 123.23 + >>> from_unicode(fix_underscores(u"\\N{PUNCTUATION SPACE} 12_3.2_3 \\N{PUNCTUATION SPACE}")) + 123.23 + >>> from_unicode(u"\\N{PUNCTUATION SPACE} " * 25 + u"123.54 " + u"\\N{PUNCTUATION SPACE} " * 22) # >= 40 chars + 123.54 + >>> from_unicode(fix_underscores(u"\\N{PUNCTUATION SPACE} " * 25 + u"1_23.5_4 " + u"\\N{PUNCTUATION SPACE} " * 22)) + 123.54 + >>> from_unicode(u"\\N{PUNCTUATION SPACE} " + u"123.54 " * 2 + u"\\N{PUNCTUATION SPACE}") # doctest: +ELLIPSIS + Traceback (most recent call last): + ValueError: ...123.54 123.54... + >>> from_unicode(u"\\N{PUNCTUATION SPACE} " * 25 + u"123.54 " * 2 + u"\\N{PUNCTUATION SPACE} " * 22) # doctest: +ELLIPSIS + Traceback (most recent call last): + ValueError: ...123.54 123.54... + >>> from_unicode(u"_12__._3") # doctest: +ELLIPSIS + Traceback (most recent call last): + ValueError: ..._12__._3... + >>> from_unicode(u"_12.3") # doctest: +ELLIPSIS + Traceback (most recent call last): + ValueError: ..._12.3... + >>> from_unicode(u"12.3_") # doctest: +ELLIPSIS + Traceback (most recent call last): + ValueError: ...12.3_... + >>> from_unicode(u"i_nf") # doctest: +ELLIPSIS + Traceback (most recent call last): + ValueError: ...i_nf... + >>> from_unicode(None) # doctest: +ELLIPSIS + Traceback (most recent call last): + TypeError... + """ + return float(s) + + +@cython.test_assert_path_exists( + "//CoerceToPyTypeNode", + "//CoerceToPyTypeNode//PythonCapiCallNode", +) +def from_unicode_literals(): + """ + >>> from_unicode_literals() + (123.0, 123.23, 123.45, 1e+100, 123.23) + """ + return float(u"123"), float(u"123.23"), float(fix_underscores(u"12_3.4_5")), float(u"1e100"), float(u"123.23\N{PUNCTUATION SPACE}") + + +def catch_valueerror(val): + """ + >>> catch_valueerror("foo") + False + >>> catch_valueerror(u"foo") + False + >>> catch_valueerror(b"foo") + False + >>> catch_valueerror(bytearray(b"foo")) + False + >>> catch_valueerror("-1") + -1.0 + """ + try: + return float(val) + except ValueError: + return False diff --git a/tests/run/builtin_methods_return_values.pyx b/tests/run/builtin_methods_return_values.pyx index 50f1427ca..09a25273c 100644 --- a/tests/run/builtin_methods_return_values.pyx +++ b/tests/run/builtin_methods_return_values.pyx @@ -1,6 +1,6 @@ # mode: run # tag: list, set, builtins -# ticket: 688 +# ticket: t688 _set = set diff --git a/tests/run/builtin_subtype_methods_T653.pyx b/tests/run/builtin_subtype_methods_T653.pyx index bcfda81f0..522cffff8 100644 --- a/tests/run/builtin_subtype_methods_T653.pyx +++ b/tests/run/builtin_subtype_methods_T653.pyx @@ -1,6 +1,6 @@ #cython: language_level=2 # mode: run -# ticket: 653 +# ticket: t653 cimport cython diff --git a/tests/run/builtin_subtype_methods_cy3.pyx b/tests/run/builtin_subtype_methods_cy3.pyx index 1cff75fb7..1f54ac598 100644 --- a/tests/run/builtin_subtype_methods_cy3.pyx +++ b/tests/run/builtin_subtype_methods_cy3.pyx @@ -1,6 +1,6 @@ # cython: language_level=3 # mode: run -# ticket: 653 +# ticket: t653 class DictPySubtype(dict): diff --git a/tests/run/builtin_type_inheritance_T608.pyx b/tests/run/builtin_type_inheritance_T608.pyx index 67e97ec1e..1214b6841 100644 --- a/tests/run/builtin_type_inheritance_T608.pyx +++ b/tests/run/builtin_type_inheritance_T608.pyx @@ -1,4 +1,4 @@ -# ticket: 608 +# ticket: t608 cdef class MyInt(int): """ diff --git a/tests/run/builtin_types_class.py b/tests/run/builtin_types_class.py new file mode 100644 index 000000000..f09be80f4 --- /dev/null +++ b/tests/run/builtin_types_class.py @@ -0,0 +1,60 @@ +# mode: run + +from __future__ import print_function + +import cython + +# https://github.com/cython/cython/issues/3954 +# calls to the __class__ attributes of builtin types were optimized to something invalid + +@cython.locals(d=dict) +def test_dict(d): + """ + >>> test_dict({}) + dict + {} + """ + print(d.__class__.__name__) + print(d.__class__()) + +@cython.locals(i=int) +def test_int(i): + """ + >>> test_int(0) + int + 0 + """ + print(i.__class__.__name__) + print(i.__class__()) + +@cython.cclass +class C: + def __str__(self): + return "I'm a C object" + +@cython.locals(c=C) +def test_cdef_class(c): + """ + # This wasn't actually broken but is worth testing anyway + >>> test_cdef_class(C()) + C + I'm a C object + """ + print(c.__class__.__name__) + print(c.__class__()) + +@cython.locals(d=object) +def test_object(o): + """ + >>> test_object({}) + dict + {} + >>> test_object(1) + int + 0 + >>> test_object(C()) + C + I'm a C object + """ + print(o.__class__.__name__) + print(o.__class__()) diff --git a/tests/run/builtin_types_none_T166.pyx b/tests/run/builtin_types_none_T166.pyx index 33cabffa8..276f52724 100644 --- a/tests/run/builtin_types_none_T166.pyx +++ b/tests/run/builtin_types_none_T166.pyx @@ -1,4 +1,4 @@ -# ticket: 166 +# ticket: t166 __doc__ = u""" >>> l = None diff --git a/tests/run/bytearray_iter.py b/tests/run/bytearray_iter.py new file mode 100644 index 000000000..1865f057b --- /dev/null +++ b/tests/run/bytearray_iter.py @@ -0,0 +1,90 @@ +# mode: run +# tag: pure3, pure2 + +import cython + +@cython.test_assert_path_exists("//ForFromStatNode") +@cython.test_fail_if_path_exists("//ForInStatNode") +@cython.locals(x=bytearray) +def basic_bytearray_iter(x): + """ + >>> basic_bytearray_iter(bytearray(b"hello")) + h + e + l + l + o + """ + for a in x: + print(chr(a)) + +@cython.test_assert_path_exists("//ForFromStatNode") +@cython.test_fail_if_path_exists("//ForInStatNode") +@cython.locals(x=bytearray) +def reversed_bytearray_iter(x): + """ + >>> reversed_bytearray_iter(bytearray(b"hello")) + o + l + l + e + h + """ + for a in reversed(x): + print(chr(a)) + +@cython.test_assert_path_exists("//ForFromStatNode") +@cython.test_fail_if_path_exists("//ForInStatNode") +@cython.locals(x=bytearray) +def modifying_bytearray_iter1(x): + """ + >>> modifying_bytearray_iter1(bytearray(b"abcdef")) + a + b + c + 3 + """ + count = 0 + for a in x: + print(chr(a)) + del x[-1] + count += 1 + print(count) + +@cython.test_assert_path_exists("//ForFromStatNode") +@cython.test_fail_if_path_exists("//ForInStatNode") +@cython.locals(x=bytearray) +def modifying_bytearray_iter2(x): + """ + >>> modifying_bytearray_iter2(bytearray(b"abcdef")) + a + c + e + 3 + """ + count = 0 + for a in x: + print(chr(a)) + del x[0] + count += 1 + print(count) + +@cython.test_assert_path_exists("//ForFromStatNode") +@cython.test_fail_if_path_exists("//ForInStatNode") +@cython.locals(x=bytearray) +def modifying_reversed_bytearray_iter(x): + """ + NOTE - I'm not 100% sure how well-defined this behaviour is in Python. + However, for the moment Python and Cython seem to do the same thing. + Testing that it doesn't crash is probably more important than the exact output! + >>> modifying_reversed_bytearray_iter(bytearray(b"abcdef")) + f + f + f + f + f + f + """ + for a in reversed(x): + print(chr(a)) + del x[0] diff --git a/tests/run/c_int_types_T255.pyx b/tests/run/c_int_types_T255.pyx index eec5d5e10..58b7839d9 100644 --- a/tests/run/c_int_types_T255.pyx +++ b/tests/run/c_int_types_T255.pyx @@ -1,4 +1,4 @@ -# ticket: 255 +# ticket: t255 __doc__ = u"" @@ -685,14 +685,13 @@ class MyBadInt2(MyInt2): def test_convert_pyint(x): u""" - >>> test_convert_pyint(None) + >>> test_convert_pyint(None) # doctest: +ELLIPSIS Traceback (most recent call last): - ... - TypeError: an integer is required - >>> test_convert_pyint("123") + TypeError:... int... + >>> test_convert_pyint("123") # doctest: +ELLIPSIS Traceback (most recent call last): ... - TypeError: an integer is required + TypeError:... int... >>> test_convert_pyint(MyBadInt(0)) #doctest: +ELLIPSIS Traceback (most recent call last): ... @@ -733,14 +732,14 @@ class MyBadLong(MyLong): def test_convert_pylong(x): u""" - >>> test_convert_pylong(None) + >>> test_convert_pylong(None) # doctest: +ELLIPSIS Traceback (most recent call last): ... - TypeError: an integer is required - >>> test_convert_pylong("123") + TypeError:... int... + >>> test_convert_pylong("123") # doctest: +ELLIPSIS Traceback (most recent call last): ... - TypeError: an integer is required + TypeError:... int... >>> test_convert_pylong(MyBadLong(0)) #doctest: +ELLIPSIS Traceback (most recent call last): ... diff --git a/tests/run/c_type_methods_T236.pyx b/tests/run/c_type_methods_T236.pyx index 5ac22feae..b1ef5d023 100644 --- a/tests/run/c_type_methods_T236.pyx +++ b/tests/run/c_type_methods_T236.pyx @@ -1,10 +1,8 @@ -# ticket: 236 - -__doc__ = '' +# ticket: t236 import sys -if sys.version_info >= (2,6): - __doc__ += ''' + +__doc__ = ''' >>> float_is_integer(1.0) True >>> float_is_integer(1.1) @@ -19,7 +17,6 @@ True ''' def float_is_integer(float f): - # requires Python 2.6+ return f.is_integer() def int_bit_length(int i): diff --git a/tests/run/callargs.pyx b/tests/run/callargs.pyx index 05f3f639b..c87f57962 100644 --- a/tests/run/callargs.pyx +++ b/tests/run/callargs.pyx @@ -168,15 +168,15 @@ def test_int_kwargs(f): """ >>> test_int_kwargs(e) # doctest: +ELLIPSIS Traceback (most recent call last): - TypeError: ...keywords must be strings + TypeError: ...keywords must be strings... >>> test_int_kwargs(f) # doctest: +ELLIPSIS Traceback (most recent call last): - TypeError: ...keywords must be strings + TypeError: ...keywords must be strings... >>> test_int_kwargs(g) # doctest: +ELLIPSIS Traceback (most recent call last): - TypeError: ...keywords must be strings + TypeError: ...keywords must be strings... >>> test_int_kwargs(h) # doctest: +ELLIPSIS Traceback (most recent call last): - TypeError: ...keywords must be strings + TypeError: ...keywords must be strings... """ f(a=1,b=2,c=3, **{10:20,30:40}) diff --git a/tests/run/cascaded_list_unpacking_T467.pyx b/tests/run/cascaded_list_unpacking_T467.pyx index 3008bddaf..fc0963d0d 100644 --- a/tests/run/cascaded_list_unpacking_T467.pyx +++ b/tests/run/cascaded_list_unpacking_T467.pyx @@ -1,4 +1,4 @@ -# ticket: 467 +# ticket: t467 def simple_parallel_assignment_from_call(): """ diff --git a/tests/run/cascaded_typed_assignments_T466.pyx b/tests/run/cascaded_typed_assignments_T466.pyx index eef9d5b7c..c2081bf91 100644 --- a/tests/run/cascaded_typed_assignments_T466.pyx +++ b/tests/run/cascaded_typed_assignments_T466.pyx @@ -1,5 +1,5 @@ # mode: run -# ticket: 466 +# ticket: t466 # extension to T409 cimport cython diff --git a/tests/run/cascadedassignment.pyx b/tests/run/cascadedassignment.pyx index 5bde089f7..ad606fe14 100644 --- a/tests/run/cascadedassignment.pyx +++ b/tests/run/cascadedassignment.pyx @@ -56,3 +56,15 @@ def test_cascaded_assignment_evaluate_expr(): """ a = b = c = float(expr()) return a, b, c + + +def test_overwrite(): + """ + >>> test_overwrite() + {0: {1: {2: {}}}} + """ + x = a = {} + for i in range(3): + a[i] = a = {} + assert a == {} + return x diff --git a/tests/run/cascmp.pyx b/tests/run/cascmp.pyx new file mode 100644 index 000000000..becae3fc5 --- /dev/null +++ b/tests/run/cascmp.pyx @@ -0,0 +1,38 @@ +# mode: run +# tag: cascade, compare + +def ints_and_objects(): + """ + >>> ints_and_objects() + (0, 1, 0, 1, 1, 0) + """ + cdef int int1=0, int2=0, int3=0, int4=0 + cdef int r1, r2, r3, r4, r5, r6 + cdef object obj1, obj2, obj3, obj4 + obj1 = 1 + obj2 = 2 + obj3 = 3 + obj4 = 4 + r1 = int1 < int2 < int3 + r2 = obj1 < obj2 < obj3 + r3 = int1 < int2 < obj3 + r4 = obj1 < 2 < 3 + r5 = obj1 < 2 < 3 < 4 + r6 = int1 < (int2 == int3) < int4 + return r1, r2, r3, r4, r5, r6 + + +def const_cascade(x): + """ + >>> const_cascade(2) + (True, False, True, False, False, True, False) + """ + return ( + 0 <= 1, + 1 <= 0, + 1 <= 1 <= 2, + 1 <= 0 < 1, + 1 <= 1 <= 0, + 1 <= 1 <= x <= 2 <= 3 > x <= 2 <= 2, + 1 <= 1 <= x <= 1 <= 1 <= x <= 2, + ) diff --git a/tests/run/cclass_assign_attr_GH3100.pyx b/tests/run/cclass_assign_attr_GH3100.pyx new file mode 100644 index 000000000..21785b81a --- /dev/null +++ b/tests/run/cclass_assign_attr_GH3100.pyx @@ -0,0 +1,19 @@ +cdef class Foo: + """ + >>> D = Foo.__dict__ + >>> D["meth"] is D["meth2"] + True + >>> D["classmeth"] is D["classmeth2"] + True + >>> D["staticmeth"] is D["staticmeth2"] + True + """ + def meth(self): pass + @classmethod + def classmeth(cls): pass + @staticmethod + def staticmeth(): pass + + meth2 = meth + classmeth2 = classmeth + staticmeth2 = staticmeth diff --git a/tests/run/cdef_bool_T227.pyx b/tests/run/cdef_bool_T227.pyx index 889963951..be403e695 100644 --- a/tests/run/cdef_bool_T227.pyx +++ b/tests/run/cdef_bool_T227.pyx @@ -1,4 +1,4 @@ -# ticket: 227 +# ticket: t227 from cpython.bool cimport bool diff --git a/tests/run/cdef_class_field.pyx b/tests/run/cdef_class_field.pyx index 9fb745ecb..5012bab89 100644 --- a/tests/run/cdef_class_field.pyx +++ b/tests/run/cdef_class_field.pyx @@ -1,6 +1,6 @@ # mode: run # tag: exttype -# ticket: 677 +# ticket: t677 """ >>> str(Foo(4)) diff --git a/tests/run/cdef_class_property_decorator_T264.pyx b/tests/run/cdef_class_property_decorator_T264.pyx index 421f762bc..b53bd7ec1 100644 --- a/tests/run/cdef_class_property_decorator_T264.pyx +++ b/tests/run/cdef_class_property_decorator_T264.pyx @@ -1,5 +1,5 @@ # mode: run -# ticket: 264 +# ticket: t264 # tag: property, decorator my_property = property diff --git a/tests/run/cdef_decorator_directives_T183.pyx b/tests/run/cdef_decorator_directives_T183.pyx index 84b05a8df..dc362b255 100644 --- a/tests/run/cdef_decorator_directives_T183.pyx +++ b/tests/run/cdef_decorator_directives_T183.pyx @@ -1,4 +1,4 @@ -# ticket: 183 +# ticket: t183 cimport cython diff --git a/tests/run/cdef_locals_decorator_T477.pyx b/tests/run/cdef_locals_decorator_T477.pyx index 2b40d05ad..5af71e748 100644 --- a/tests/run/cdef_locals_decorator_T477.pyx +++ b/tests/run/cdef_locals_decorator_T477.pyx @@ -1,4 +1,4 @@ -# ticket: 477 +# ticket: t477 import cython @cython.locals(x=double) diff --git a/tests/run/cdef_members_T517.pyx b/tests/run/cdef_members_T517.pyx index d69d72ca4..389879f67 100644 --- a/tests/run/cdef_members_T517.pyx +++ b/tests/run/cdef_members_T517.pyx @@ -1,4 +1,4 @@ -# ticket: 517 +# ticket: t517 #cython: embedsignature=True __doc__ = u""" diff --git a/tests/run/cdef_methods_T462.pyx b/tests/run/cdef_methods_T462.pyx index dde15adbc..80bfa4b6f 100644 --- a/tests/run/cdef_methods_T462.pyx +++ b/tests/run/cdef_methods_T462.pyx @@ -1,4 +1,4 @@ -# ticket: 462 +# ticket: t462 cimport cython diff --git a/tests/run/cdef_multiple_inheritance.pyx b/tests/run/cdef_multiple_inheritance.pyx index 4213d35d3..0f8eaef24 100644 --- a/tests/run/cdef_multiple_inheritance.pyx +++ b/tests/run/cdef_multiple_inheritance.pyx @@ -1,3 +1,5 @@ +cimport cython + cdef class CBase(object): cdef int a cdef c_method(self): @@ -9,7 +11,8 @@ class PyBase(object): def py_method(self): return "PyBase" -cdef class Both(CBase, PyBase): +@cython.binding(True) +cdef class BothBound(CBase, PyBase): cdef dict __dict__ """ >>> b = Both() @@ -32,7 +35,7 @@ cdef class Both(CBase, PyBase): def call_c_method(self): return self.c_method() -cdef class BothSub(Both): +cdef class BothSub(BothBound): """ >>> b = BothSub() >>> b.py_method() @@ -43,3 +46,27 @@ cdef class BothSub(Both): 'Both' """ pass + +@cython.binding(False) +cdef class BothUnbound(CBase, PyBase): + cdef dict __dict__ + """ + >>> b = Both() + >>> b.py_method() + 'PyBase' + >>> b.cp_method() + 'Both' + >>> b.call_c_method() + 'Both' + + >>> isinstance(b, CBase) + True + >>> isinstance(b, PyBase) + True + """ + cdef c_method(self): + return "Both" + cpdef cp_method(self): + return "Both" + def call_c_method(self): + return self.c_method() diff --git a/tests/run/cdef_multiple_inheritance_cimport.srctree b/tests/run/cdef_multiple_inheritance_cimport.srctree new file mode 100644 index 000000000..dd56b3e30 --- /dev/null +++ b/tests/run/cdef_multiple_inheritance_cimport.srctree @@ -0,0 +1,44 @@ +# Test for https://github.com/cython/cython/issues/4106 + +PYTHON setup.py build_ext --inplace +PYTHON -c "import sub" + +######## setup.py ######## + +from Cython.Build import cythonize +from distutils.core import setup + +setup( + ext_modules = cythonize("*.pyx"), +) + +######## base.pxd ######## + +cdef class A: + cdef dict __dict__ + cdef int a(self) + +cdef class B(A): + cdef int b(self) + +######## base.pyx ######## + +cdef class A: + cdef int a(self): + return 1 + +class PyA: + pass + +cdef class B(A, PyA): + cdef int b(self): + return 2 + +######## sub.pyx ######## + +from base cimport B +print(B) + +cdef class C(B): + cdef int c(self): + return 3 diff --git a/tests/run/cdef_multiple_inheritance_errors.srctree b/tests/run/cdef_multiple_inheritance_errors.srctree index e6b3426ba..4f49184f4 100644 --- a/tests/run/cdef_multiple_inheritance_errors.srctree +++ b/tests/run/cdef_multiple_inheritance_errors.srctree @@ -48,6 +48,7 @@ cdef class X(Base, Py): pass ######## oldstyle.pyx ######## +# cython: language_level=2 cdef class Base: cdef dict __dict__ @@ -64,31 +65,31 @@ import sys try: import notheaptype - assert False + assert False, "notheaptype" except TypeError as msg: assert str(msg) == "base class 'object' is not a heap type" try: import wrongbase - assert False + assert False, "wrongbase" except TypeError as msg: assert str(msg) == "best base 'str' must be equal to first base 'wrongbase.Base'" try: import badmro - assert False + assert False, "badmro" except TypeError as msg: assert str(msg).startswith("Cannot create a consistent method resolution") try: import nodict - assert False + assert False, "nodict" except TypeError as msg: assert str(msg) == "extension type 'nodict.X' has no __dict__ slot, but base type 'Py' has: either add 'cdef dict __dict__' to the extension type or add '__slots__ = [...]' to the base type" try: # This should work on Python 3 but fail on Python 2 import oldstyle - assert sys.version_info[0] >= 3 + assert sys.version_info[0] >= 3, "oldstyle" except TypeError as msg: assert str(msg) == "base class 'OldStyle' is an old-style class" diff --git a/tests/run/cdef_setitem_T284.pyx b/tests/run/cdef_setitem_T284.pyx index 2c885d5be..389b8c409 100644 --- a/tests/run/cdef_setitem_T284.pyx +++ b/tests/run/cdef_setitem_T284.pyx @@ -1,4 +1,4 @@ -# ticket: 284 +# ticket: t284 def no_cdef(): """ diff --git a/tests/run/cdivision_CEP_516.pyx b/tests/run/cdivision_CEP_516.pyx index c8b24a0e1..fbd2def3a 100644 --- a/tests/run/cdivision_CEP_516.pyx +++ b/tests/run/cdivision_CEP_516.pyx @@ -27,6 +27,9 @@ True >>> [test_cdiv_cmod(a, b) for a, b in v] [(1, 7), (-1, -7), (1, -7), (-1, 7)] +>>> [test_cdiv_cmod(a, b) for a, b in [(4, -4), (4, -2), (4, -1)]] +[(-1, 0), (-2, 0), (-4, 0)] + >>> all([mod_int_py(a,b) == a % b for a in range(-10, 10) for b in range(-10, 10) if b != 0]) True >>> all([div_int_py(a,b) == a // b for a in range(-10, 10) for b in range(-10, 10) if b != 0]) diff --git a/tests/run/cfunc_call_tuple_args_T408.pyx b/tests/run/cfunc_call_tuple_args_T408.pyx index 165329737..e32eb036c 100644 --- a/tests/run/cfunc_call_tuple_args_T408.pyx +++ b/tests/run/cfunc_call_tuple_args_T408.pyx @@ -1,4 +1,4 @@ -# ticket: 408 +# ticket: t408 __doc__ = """ >>> call_with_tuple(1, 1.2, 'test', [1,2,3]) diff --git a/tests/run/cfunc_convert.pyx b/tests/run/cfunc_convert.pyx index 3391cd226..6db0765d4 100644 --- a/tests/run/cfunc_convert.pyx +++ b/tests/run/cfunc_convert.pyx @@ -1,4 +1,5 @@ # mode: run +# tag: autowrap # cython: always_allow_keywords=True cimport cython @@ -82,7 +83,7 @@ def test_global(): cdef long long rad(long long x): cdef long long rad = 1 - for p in range(2, <long long>sqrt(x) + 1): + for p in range(2, <long long>sqrt(<double>x) + 1): # MSVC++ fails without the input cast if x % p == 0: rad *= p while x % p == 0: @@ -229,3 +230,39 @@ def test_cdef_class_params(a, b): TypeError: Argument 'b' has incorrect type (expected cfunc_convert.B, got cfunc_convert.A) """ return (<object>test_cdef_class_params_cfunc)(a, b) + +# There were a few cases where duplicate utility code definitions (i.e. with the same name) +# could be generated, causing C compile errors. This file tests them. + +cdef cfunc_dup_f1(x, r): + return "f1" + +cdef cfunc_dup_f2(x1, r): + return "f2" + +def make_map(): + """ + https://github.com/cython/cython/issues/3716 + This is testing the generation of wrappers for f1 and f2 + >>> for k, f in make_map().items(): + ... print(k == f(0, 0)) # in both cases the functions should just return their name + True + True + + # Test passing of keyword arguments + >>> print(make_map()['f1'](x=1, r=2)) + f1 + >>> make_map()['f1'](x1=1, r=2) # doctest: +ELLIPSIS + Traceback (most recent call last): + TypeError: ... + >>> print(make_map()['f2'](x1=1, r=2)) + f2 + >>> make_map()['f2'](x=1, r=2) # doctest: +ELLIPSIS + Traceback (most recent call last): + TypeError: ... + """ + cdef map = { + "f1": cfunc_dup_f1, + "f2": cfunc_dup_f2, + } + return map diff --git a/tests/run/char_constants_T99.pyx b/tests/run/char_constants_T99.pyx index aa2550b52..241b461a1 100644 --- a/tests/run/char_constants_T99.pyx +++ b/tests/run/char_constants_T99.pyx @@ -1,4 +1,4 @@ -# ticket: 99 +# ticket: t99 cdef char c = 'c' cdef char* s = 'abcdef' diff --git a/tests/run/charcomparisonT412.pyx b/tests/run/charcomparisonT412.pyx index b9316a36c..97afb4769 100644 --- a/tests/run/charcomparisonT412.pyx +++ b/tests/run/charcomparisonT412.pyx @@ -1,4 +1,4 @@ -# ticket: 412 +# ticket: t412 def f(): """ diff --git a/tests/run/charptr_comparison_T582.pyx b/tests/run/charptr_comparison_T582.pyx index 3701921cf..1e5e43cd2 100644 --- a/tests/run/charptr_comparison_T582.pyx +++ b/tests/run/charptr_comparison_T582.pyx @@ -1,4 +1,4 @@ -# ticket: 582 +# ticket: t582 cimport cython diff --git a/tests/run/cimport_cython_T505.pyx b/tests/run/cimport_cython_T505.pyx index 20d2daf3a..69229856f 100644 --- a/tests/run/cimport_cython_T505.pyx +++ b/tests/run/cimport_cython_T505.pyx @@ -1,4 +1,4 @@ -# ticket: 505 +# ticket: t505 cimport cython diff --git a/tests/run/cimport_from_pyx.srctree b/tests/run/cimport_from_pyx.srctree index 602188748..d25e21b10 100644 --- a/tests/run/cimport_from_pyx.srctree +++ b/tests/run/cimport_from_pyx.srctree @@ -15,14 +15,22 @@ setup( ######## a.pyx ######## -from b cimport Bclass, Bfunc, Bstruct, Benum, Benum_value, Btypedef, Py_EQ, Py_NE +from b cimport (Bclass, Bfunc, Bstruct, Benum, Benum_value, Btypedef, Py_EQ, Py_NE, + DecoratedClass, cfuncOutside) cdef Bclass b = Bclass(5) assert Bfunc(&b.value) == b.value +assert b.anotherValue == 6, b.anotherValue assert b.asStruct().value == b.value cdef Btypedef b_type = &b.value cdef Benum b_enum = Benum_value cdef int tmp = Py_EQ +cdef DecoratedClass dc = DecoratedClass() +assert dc.cfuncInClass().value == 5 +assert dc.cpdefInClass() == 1.0 + +assert cfuncOutside().value == 2 + #from c cimport ClassC #cdef ClassC c = ClassC() #print c.value @@ -31,6 +39,8 @@ cdef int tmp = Py_EQ from cpython.object cimport Py_EQ, Py_NE +cimport cython + cdef enum Benum: Benum_value @@ -41,14 +51,34 @@ ctypedef long *Btypedef cdef class Bclass: cdef long value + anotherValue: cython.double def __init__(self, value): self.value = value + self.anotherValue = value + 1 cdef Bstruct asStruct(self): return Bstruct(value=self.value) + cdef double getOtherValue(self): + return self.anotherValue cdef long Bfunc(Btypedef x): return x[0] +@cython.cclass +class DecoratedClass: + @cython.cfunc + @cython.returns(Bstruct) + def cfuncInClass(self): + return Bstruct(value=5) + @cython.ccall + @cython.returns(cython.double) + def cpdefInClass(self): + return 1.0 + +@cython.cfunc +@cython.returns(Bstruct) +def cfuncOutside(): + return Bstruct(value=2) + ######## c.pxd ######## cdef class ClassC: diff --git a/tests/run/cimport_from_sys_path.srctree b/tests/run/cimport_from_sys_path.srctree index e6f619d78..e1541d57c 100644 --- a/tests/run/cimport_from_sys_path.srctree +++ b/tests/run/cimport_from_sys_path.srctree @@ -32,7 +32,7 @@ static int foo(int a) ######## a.pyx ######## from b.other cimport foo -print foo(10) +print(foo(10)) cimport b.other -print b.other.foo(10) +print(b.other.foo(10)) diff --git a/tests/run/class_attribute_init_values_T18.pyx b/tests/run/class_attribute_init_values_T18.pyx index 5e12f665f..7887853df 100644 --- a/tests/run/class_attribute_init_values_T18.pyx +++ b/tests/run/class_attribute_init_values_T18.pyx @@ -1,4 +1,4 @@ -# ticket: 18 +# ticket: t18 __doc__ = u""" >>> f = PyFoo() diff --git a/tests/run/class_func_in_control_structures_T87.pyx b/tests/run/class_func_in_control_structures_T87.pyx index 5c23ceff9..14ee440a0 100644 --- a/tests/run/class_func_in_control_structures_T87.pyx +++ b/tests/run/class_func_in_control_structures_T87.pyx @@ -1,4 +1,4 @@ -# ticket: 87 +# ticket: t87 __doc__ = u""" >>> d = Defined() diff --git a/tests/run/class_scope_del_T684.py b/tests/run/class_scope_del_T684.py index 43368f333..f52b31864 100644 --- a/tests/run/class_scope_del_T684.py +++ b/tests/run/class_scope_del_T684.py @@ -1,6 +1,6 @@ # mode:run # tag: class, scope, del -# ticket: 684 +# ticket: t684 class DelInClass(object): """ diff --git a/tests/run/classdecorators_T336.pyx b/tests/run/classdecorators_T336.pyx index c29c1cbaa..b56e08c07 100644 --- a/tests/run/classdecorators_T336.pyx +++ b/tests/run/classdecorators_T336.pyx @@ -1,4 +1,4 @@ -# ticket: 336 +# ticket: t336 __doc__ = u""" >>> print('\\n'.join(calls)) diff --git a/tests/run/clear_to_null.pyx b/tests/run/clear_to_null.pyx index c8a355570..c54faa583 100644 --- a/tests/run/clear_to_null.pyx +++ b/tests/run/clear_to_null.pyx @@ -2,7 +2,7 @@ Check that Cython generates a tp_clear function that actually clears object references to NULL instead of None. -Discussed here: http://article.gmane.org/gmane.comp.python.cython.devel/14833 +Discussed here: https://article.gmane.org/gmane.comp.python.cython.devel/14833 """ from cpython.ref cimport PyObject, Py_TYPE diff --git a/tests/run/closure_class_T596.pyx b/tests/run/closure_class_T596.pyx index 6a92d9f82..7808a8d9e 100644 --- a/tests/run/closure_class_T596.pyx +++ b/tests/run/closure_class_T596.pyx @@ -1,6 +1,6 @@ # mode: run # tag: closures -# ticket: 596 +# ticket: t596 def simple(a, b): """ diff --git a/tests/run/closure_decorators_T478.pyx b/tests/run/closure_decorators_T478.pyx index e1c5f4918..1bccdcc19 100644 --- a/tests/run/closure_decorators_T478.pyx +++ b/tests/run/closure_decorators_T478.pyx @@ -1,6 +1,6 @@ # mode: run # tag: closures -# ticket: 478 +# ticket: t478 __doc__ = """ >>> Num(13).is_prime() diff --git a/tests/run/closure_inside_cdef_T554.pyx b/tests/run/closure_inside_cdef_T554.pyx index 3a112868d..3406259b5 100644 --- a/tests/run/closure_inside_cdef_T554.pyx +++ b/tests/run/closure_inside_cdef_T554.pyx @@ -1,6 +1,6 @@ # mode: run # tag: closures -# ticket: 554 +# ticket: t554 def call_f(x): """ diff --git a/tests/run/closure_name_mangling_T537.pyx b/tests/run/closure_name_mangling_T537.pyx index 0324109ec..148f9f1ae 100644 --- a/tests/run/closure_name_mangling_T537.pyx +++ b/tests/run/closure_name_mangling_T537.pyx @@ -1,6 +1,6 @@ # mode: run # tag: closures -# ticket: 537 +# ticket: t537 __doc__ = u""" >>> f1 = nested1() diff --git a/tests/run/closure_names.pyx b/tests/run/closure_names.pyx index b6d253983..0ceb320d5 100644 --- a/tests/run/closure_names.pyx +++ b/tests/run/closure_names.pyx @@ -1,6 +1,6 @@ # mode: run # tag: closures -# ticket: gh-1797 +# ticket: 1797 def func(): diff --git a/tests/run/closures_T82.pyx b/tests/run/closures_T82.pyx index 85ce9043e..c8a228a19 100644 --- a/tests/run/closures_T82.pyx +++ b/tests/run/closures_T82.pyx @@ -1,6 +1,6 @@ # mode: run # tag: closures -# ticket: 82 +# ticket: t82 # preparse: id # preparse: def_to_cdef diff --git a/tests/run/cmethod_inline_T474.pyx b/tests/run/cmethod_inline_T474.pyx index ddc33a62b..d09db3bf4 100644 --- a/tests/run/cmethod_inline_T474.pyx +++ b/tests/run/cmethod_inline_T474.pyx @@ -1,5 +1,5 @@ # mode: run -# ticket: 474 +# ticket: t474 cimport cython diff --git a/tests/run/common_utility_types.srctree b/tests/run/common_utility_types.srctree index 1f3d2aea3..35daad505 100644 --- a/tests/run/common_utility_types.srctree +++ b/tests/run/common_utility_types.srctree @@ -31,7 +31,7 @@ print("importing...") import a, b print(type(a.funcA)) -assert type(a.funcA).__name__ == 'cython_function_or_method' +assert type(a.funcA).__name__.endswith('cython_function_or_method') assert type(a.funcA) is type(b.funcB) assert a.funcA.func_globals is a.__dict__ diff --git a/tests/run/complex_cast_T445.pyx b/tests/run/complex_cast_T445.pyx index 62709c9c1..b4069ec9d 100644 --- a/tests/run/complex_cast_T445.pyx +++ b/tests/run/complex_cast_T445.pyx @@ -1,4 +1,4 @@ -# ticket: 445 +# ticket: t445 def complex_double_cast(double x, double complex z): """ diff --git a/tests/run/complex_coercion_sideeffects_T693.pyx b/tests/run/complex_coercion_sideeffects_T693.pyx index 92b1f94a9..18a9575dc 100644 --- a/tests/run/complex_coercion_sideeffects_T693.pyx +++ b/tests/run/complex_coercion_sideeffects_T693.pyx @@ -1,5 +1,5 @@ # mode: run -# ticket: 693 +# ticket: t693 cdef double complex func(double complex x): print "hello" diff --git a/tests/run/complex_int_T446.pyx b/tests/run/complex_int_T446.pyx index a6f6aac2d..16e991c7c 100644 --- a/tests/run/complex_int_T446.pyx +++ b/tests/run/complex_int_T446.pyx @@ -1,9 +1,14 @@ -# ticket: 446 +# ticket: t446 import cython -cdef extern from "complex_int_T446_fix.h": - pass +cdef extern from *: + """ + #if defined _MSC_VER && defined __cplusplus + #define CYTHON_CCOMPLEX 0 + #endif + """ + def test_arith(int complex a, int complex b): """ diff --git a/tests/run/complex_int_T446_fix.h b/tests/run/complex_int_T446_fix.h deleted file mode 100644 index c6d11d15e..000000000 --- a/tests/run/complex_int_T446_fix.h +++ /dev/null @@ -1,3 +0,0 @@ -#if defined _MSC_VER && defined __cplusplus -#define CYTHON_CCOMPLEX 0 -#endif diff --git a/tests/run/complex_numbers_T305.pyx b/tests/run/complex_numbers_T305.pyx index 310b7233e..acbc0a5fa 100644 --- a/tests/run/complex_numbers_T305.pyx +++ b/tests/run/complex_numbers_T305.pyx @@ -1,4 +1,4 @@ -# ticket: 305 +# ticket: t305 from cpython.object cimport Py_EQ, Py_NE diff --git a/tests/run/complex_numbers_T305_long_double.pyx b/tests/run/complex_numbers_T305_long_double.pyx index 891d44271..9bc1e73fb 100644 --- a/tests/run/complex_numbers_T305_long_double.pyx +++ b/tests/run/complex_numbers_T305_long_double.pyx @@ -1,4 +1,4 @@ -# ticket: 305 +# ticket: t305 cimport cython diff --git a/tests/run/complex_numbers_c89_T398.pyx b/tests/run/complex_numbers_c89_T398.pyx index 7d680a167..546d37c31 100644 --- a/tests/run/complex_numbers_c89_T398.pyx +++ b/tests/run/complex_numbers_c89_T398.pyx @@ -1,4 +1,4 @@ -# ticket: 398 +# ticket: t398 cdef extern from "complex_numbers_c89_T398.h": pass include "complex_numbers_T305.pyx" diff --git a/tests/run/complex_numbers_c89_T398_long_double.pyx b/tests/run/complex_numbers_c89_T398_long_double.pyx index 91450e22b..9ac1607e4 100644 --- a/tests/run/complex_numbers_c89_T398_long_double.pyx +++ b/tests/run/complex_numbers_c89_T398_long_double.pyx @@ -1,4 +1,4 @@ -# ticket: 398 +# ticket: t398 cdef extern from "complex_numbers_c89_T398.h": pass include "complex_numbers_T305_long_double.pyx" diff --git a/tests/run/complex_numbers_c99_T398.pyx b/tests/run/complex_numbers_c99_T398.pyx index 762c24b02..332029cab 100644 --- a/tests/run/complex_numbers_c99_T398.pyx +++ b/tests/run/complex_numbers_c99_T398.pyx @@ -1,4 +1,4 @@ -# ticket: 398 +# ticket: t398 cdef extern from "complex_numbers_c99_T398.h": pass include "complex_numbers_T305.pyx" diff --git a/tests/run/complex_numbers_cmath_T2891.pyx b/tests/run/complex_numbers_cmath_T2891.pyx new file mode 100644 index 000000000..a5b1a65ea --- /dev/null +++ b/tests/run/complex_numbers_cmath_T2891.pyx @@ -0,0 +1,15 @@ +# ticket: 2891 +# tag: c, no-cpp + +cdef extern from "complex_numbers_c99_T398.h": pass + +from libc.complex cimport cimag, creal, cabs, carg + + +def test_decomposing(double complex z): + """ + >>> test_decomposing(3+4j) + (3.0, 4.0, 5.0, 0.9272952180016122) + """ + + return (creal(z), cimag(z), cabs(z), carg(z)) diff --git a/tests/run/complex_numbers_cxx_T398.pyx b/tests/run/complex_numbers_cxx_T398.pyx index 9b9b69a73..f5e535ab7 100644 --- a/tests/run/complex_numbers_cxx_T398.pyx +++ b/tests/run/complex_numbers_cxx_T398.pyx @@ -1,4 +1,4 @@ -# ticket: 398 +# ticket: t398 cdef extern from "complex_numbers_cxx_T398.h": pass include "complex_numbers_T305.pyx" diff --git a/tests/run/constant_folding.py b/tests/run/constant_folding.py index 2df8da308..4dd70a625 100644 --- a/tests/run/constant_folding.py +++ b/tests/run/constant_folding.py @@ -136,6 +136,17 @@ def binop_mul_pow(): return (mul_int, mul_large_int, pow_int, pow_large_int) +def binop_pow_negative(): + """ + >>> print_big_ints(binop_pow_negative()) + (4.018775720164609e-06, 8.020807320287816e-38, 0.1) + """ + pow_int = 12 ** -5 + pow_large_int = 1234 ** -12 + pow_expression_int = 10 ** (1-2) + return (pow_int, pow_large_int, pow_expression_int) + + @cython.test_fail_if_path_exists( "//SliceIndexNode", ) diff --git a/tests/run/constants.pyx b/tests/run/constants.pyx index 37772eaef..5f70e3ac2 100644 --- a/tests/run/constants.pyx +++ b/tests/run/constants.pyx @@ -156,8 +156,17 @@ def multiplied_lists_nonconst_left(x): """ return x * [1,2,3] + +@cython.test_fail_if_path_exists("//MulNode") +def multiplied_nonconst_list_const_int(x): + """ + >>> multiplied_nonconst_list_const_int(2) + [1, 2, 3, 1, 2, 3] + """ + return [1,x,3] * 2 + + @cython.test_fail_if_path_exists("//MulNode//ListNode") -@cython.test_assert_path_exists("//MulNode") def multiplied_lists_nonconst_expression(x): """ >>> multiplied_lists_nonconst_expression(5) == [1,2,3] * (5 * 2) diff --git a/tests/run/contains_T455.pyx b/tests/run/contains_T455.pyx index 5caed0b86..8bb087acf 100644 --- a/tests/run/contains_T455.pyx +++ b/tests/run/contains_T455.pyx @@ -1,4 +1,4 @@ -# ticket: 455 +# ticket: t455 def in_sequence(x, seq): """ diff --git a/tests/run/coroutines.py b/tests/run/coroutines.py index d0b9ab9db..3b236a925 100644 --- a/tests/run/coroutines.py +++ b/tests/run/coroutines.py @@ -1,6 +1,6 @@ # cython: language_level=3 # mode: run -# tag: pep492, pure3.5 +# tag: pep492, pure3.5, gh1462, async, await async def test_coroutine_frame(awaitable): @@ -28,3 +28,33 @@ async def test_coroutine_frame(awaitable): """ b = await awaitable return b + + +# gh1462: Using decorators on coroutines. + +def pass_through(func): + return func + + +@pass_through +async def test_pass_through(): + """ + >>> t = test_pass_through() + >>> try: t.send(None) + ... except StopIteration as ex: + ... print(ex.args[0] if ex.args else None) + ... else: print("NOT STOPPED!") + None + """ + + +@pass_through(pass_through) +async def test_pass_through_with_args(): + """ + >>> t = test_pass_through_with_args() + >>> try: t.send(None) + ... except StopIteration as ex: + ... print(ex.args[0] if ex.args else None) + ... else: print("NOT STOPPED!") + None + """ diff --git a/tests/run/coverage_cmd.srctree b/tests/run/coverage_cmd.srctree index 9504cc6f8..70408409c 100644 --- a/tests/run/coverage_cmd.srctree +++ b/tests/run/coverage_cmd.srctree @@ -40,6 +40,13 @@ def func2(a): return a * 2 # 11 +def func3(a): + x = 1 # 15 + a *= 2 # # pragma: no cover + a += x # + return a * 42 # 18 # pragma: no cover + + ######## pkg/coverage_test_pyx.pyx ######## # cython: linetrace=True # distutils: define_macros=CYTHON_TRACE=1 @@ -54,6 +61,13 @@ def func2(int a): return a * 2 # 11 +def func3(int a): + cdef int x = 1 # 15 + a *= 2 # # pragma: no cover + a += x # + return a * 42 # 18 # pragma: no cover + + ######## coverage_test_include_pyx.pyx ######## # cython: linetrace=True # distutils: define_macros=CYTHON_TRACE=1 @@ -152,8 +166,9 @@ def run_report(): missing.append(int(start)) files[os.path.basename(name)] = (statements, missed, covered, missing) - assert 7 not in files['coverage_test_pyx.pyx'][-1], files['coverage_test_pyx.pyx'] - assert 12 not in files['coverage_test_pyx.pyx'][-1], files['coverage_test_pyx.pyx'] + report = files['coverage_test_pyx.pyx'] + assert 7 not in report[-1], report + assert 12 not in report[-1], report def run_xml_report(): @@ -170,35 +185,74 @@ def run_xml_report(): for line in module.findall('lines/line') ) - assert files['pkg/coverage_test_pyx.pyx'][5] > 0, files['pkg/coverage_test_pyx.pyx'] - assert files['pkg/coverage_test_pyx.pyx'][6] > 0, files['pkg/coverage_test_pyx.pyx'] - assert files['pkg/coverage_test_pyx.pyx'][7] > 0, files['pkg/coverage_test_pyx.pyx'] + report = files['pkg/coverage_test_pyx.pyx'] + assert report[5] > 0, report + assert report[6] > 0, report + assert report[7] > 0, report + + +def run_json_report(): + import coverage + if coverage.version_info < (5, 0): + # JSON output comes in coverage 5.0 + return + + stdout = run_coverage_command('json', '-o', '-') + + import json + files = json.loads(stdout.decode("ascii"))['files'] + + for filename in [ + 'pkg/coverage_test_py.py', + 'pkg/coverage_test_pyx.pyx', + ]: + report = files[filename.replace('/', os.sep)] + summary = report['summary'] + assert summary['missing_lines'] == 2, summary + assert summary['excluded_lines'] == 2, summary + assert report['missing_lines'] == [15, 17], report + assert report['excluded_lines'] == [16, 18], report + + assert not frozenset( + report['missing_lines'] + report['excluded_lines'] + ).intersection(report['executed_lines']) def run_html_report(): + from collections import defaultdict + stdout = run_coverage_command('html', '-d', 'html') _parse_lines = re.compile( r'<p[^>]* id=["\'][^0-9"\']*(?P<id>[0-9]+)[^0-9"\']*["\'][^>]*' - r' class=["\'][^"\']*(?P<run>mis|run)[^"\']*["\']').findall + r' class=["\'][^"\']*(?P<run>mis|run|exc)[^"\']*["\']').findall files = {} for file_path in iglob('html/*.html'): with open(file_path) as f: page = f.read() - executed = set() - missing = set() - for line, has_run in _parse_lines(page): - (executed if has_run == 'run' else missing).add(int(line)) - files[file_path] = (executed, missing) + report = defaultdict(set) + for line, state in _parse_lines(page): + report[state].add(int(line)) + files[file_path] = report - executed, missing = [data for path, data in files.items() if 'coverage_test_pyx' in path][0] - assert executed - assert 5 in executed, executed - assert 6 in executed, executed - assert 7 in executed, executed + for filename, report in files.items(): + if "coverage_test_pyx" not in filename: + continue + executed = report["run"] + missing = report["mis"] + excluded = report["exc"] + assert executed + assert 5 in executed, executed + assert 6 in executed, executed + assert 7 in executed, executed + assert 15 in missing, missing + assert 16 in excluded, excluded + assert 17 in missing, missing + assert 18 in excluded, excluded if __name__ == '__main__': run_report() run_xml_report() + run_json_report() run_html_report() diff --git a/tests/run/coverage_nogil.srctree b/tests/run/coverage_nogil.srctree index 053008cc2..d51993ea5 100644 --- a/tests/run/coverage_nogil.srctree +++ b/tests/run/coverage_nogil.srctree @@ -1,5 +1,5 @@ # mode: run -# tag: coverage,trace,nogil +# tag: coverage,trace,nogil,fastgil """ PYTHON setup.py build_ext -i @@ -21,22 +21,34 @@ setup(ext_modules = cythonize([ plugins = Cython.Coverage -######## coverage_test_nogil.pyx ######## -# cython: linetrace=True +######## coverage_test_nogil_fastgil.pyx ######## +# cython: linetrace=True,fast_gil=True # distutils: define_macros=CYTHON_TRACE=1 CYTHON_TRACE_NOGIL=1 +include "_coverage_test_nogil.pxi" + +######## coverage_test_nogil_nofastgil.pyx ######## +# cython: linetrace=True,fast_gil=False +# distutils: define_macros=CYTHON_TRACE=1 CYTHON_TRACE_NOGIL=1 +include "_coverage_test_nogil.pxi" + + +######## _coverage_test_nogil.pxi ######## +# 1 +# 2 +# 3 cdef int func1(int a, int b) nogil: # 4 cdef int x # 5 with gil: # 6 x = 1 # 7 cdef int c = func2(a) + b # 8 return x + c # 9 - - +# 10 +# 11 cdef int func2(int a) with gil: # 12 return a * 2 # 13 - - +# 14 +# 15 def call(int a, int b): # 16 a, b = b, a # 17 with nogil: # 18 @@ -56,11 +68,12 @@ except ImportError: from coverage import coverage -def run_coverage(): +def run_coverage(module_name): + print("Testing module %s" % module_name) cov = coverage() cov.start() - import coverage_test_nogil as module + module = __import__(module_name) module_name = module.__name__ module_path = module_name + '.pyx' assert not any(module.__file__.endswith(ext) @@ -69,7 +82,6 @@ def run_coverage(): assert module.call(1, 2) == (1 * 2) + 2 + 1 cov.stop() - out = StringIO() cov.report(file=out) #cov.report([module], file=out) @@ -77,15 +89,17 @@ def run_coverage(): assert any(module_path in line for line in lines), \ "'%s' not found in coverage report:\n\n%s" % (module_path, out.getvalue()) - mod_file, exec_lines, excl_lines, missing_lines, _ = cov.analysis2(os.path.abspath(module_path)) - assert module_path in mod_file + module_pxi = "_coverage_test_nogil.pxi" + mod_file, exec_lines, excl_lines, missing_lines, _ = cov.analysis2(os.path.abspath(module_pxi)) + assert module_pxi in mod_file executed = set(exec_lines) - set(missing_lines) - # check that everything that runs with the gil owned was executed - assert all(line in executed for line in [12, 13, 16, 17, 18, 20]), '%s / %s' % (exec_lines, missing_lines) + # check that everything that runs with the gil owned was executed (missing due to pxi: 4, 12, 16) + assert all(line in executed for line in [13, 17, 18, 20]), '%s / %s' % (exec_lines, missing_lines) # check that everything that runs in nogil sections was executed - assert all(line in executed for line in [4, 6, 7, 8, 9]), '%s / %s' % (exec_lines, missing_lines) + assert all(line in executed for line in [6, 7, 8, 9]), '%s / %s' % (exec_lines, missing_lines) if __name__ == '__main__': - run_coverage() + for module_name in ["coverage_test_nogil_fastgil", "coverage_test_nogil_nofastgil"]: + run_coverage(module_name) diff --git a/tests/run/cpdef_enums.pxd b/tests/run/cpdef_enums.pxd index 14f64501f..9140e2d5e 100644 --- a/tests/run/cpdef_enums.pxd +++ b/tests/run/cpdef_enums.pxd @@ -11,5 +11,19 @@ cpdef enum PxdEnum: RANK_1 = 37 RANK_2 = 389 +cpdef enum cpdefPxdDocEnum: + """Home is where... + """ + RANK_6 = 159 + +cpdef enum cpdefPxdDocLineEnum: + """Home is where...""" + RANK_7 = 889 + cdef enum PxdSecretEnum: - RANK_3 = 5077 + RANK_8 = 5077 + +cdef enum cdefPxdDocEnum: + """the heart is. + """ + RANK_9 = 2458 diff --git a/tests/run/cpdef_enums.pyx b/tests/run/cpdef_enums.pyx index c8d9d8205..d93e19582 100644 --- a/tests/run/cpdef_enums.pyx +++ b/tests/run/cpdef_enums.pyx @@ -11,6 +11,8 @@ True True >>> FIVE == 5 or FIVE True +>>> ELEVEN == 11 or ELEVEN +True >>> SEVEN # doctest: +ELLIPSIS Traceback (most recent call last): NameError: ...name 'SEVEN' is not defined @@ -29,6 +31,10 @@ True True >>> RANK_2 == 389 or RANK_2 True +>>> RANK_6 == 159 or RANK_6 +True +>>> RANK_7 == 889 or RANK_7 +True >>> RANK_3 # doctest: +ELLIPSIS Traceback (most recent call last): NameError: ...name 'RANK_3' is not defined @@ -48,7 +54,6 @@ Traceback (most recent call last): NameError: ...name 'IntEnum' is not defined """ - cdef extern from *: cpdef enum: # ExternPyx ONE "1" @@ -63,21 +68,30 @@ cpdef enum PyxEnum: THREE = 3 FIVE = 5 +cpdef enum cpdefPyxDocEnum: + """Home is where... + """ + ELEVEN = 11 + +cpdef enum cpdefPyxDocLineEnum: + """Home is where...""" + FOURTEEN = 14 + cdef enum SecretPyxEnum: SEVEN = 7 +cdef enum cdefPyxDocEnum: + """the heart is. + """ + FIVE_AND_SEVEN = 5077 + + def test_as_variable_from_cython(): """ >>> test_as_variable_from_cython() """ - import sys - if sys.version_info >= (2, 7): - assert list(PyxEnum) == [TWO, THREE, FIVE], list(PyxEnum) - assert list(PxdEnum) == [RANK_0, RANK_1, RANK_2], list(PxdEnum) - else: - # No OrderedDict. - assert set(PyxEnum) == {TWO, THREE, FIVE}, list(PyxEnum) - assert set(PxdEnum) == {RANK_0, RANK_1, RANK_2}, list(PxdEnum) + assert list(PyxEnum) == [TWO, THREE, FIVE], list(PyxEnum) + assert list(PxdEnum) == [RANK_0, RANK_1, RANK_2], list(PxdEnum) cdef int verify_pure_c() nogil: cdef int x = TWO @@ -95,3 +109,21 @@ def verify_resolution_GH1533(): """ THREE = 100 return int(PyxEnum.THREE) + + +def check_docs(): + """ + >>> PxdEnum.__doc__ not in ("Home is where...\\n ", "Home is where...") + True + >>> PyxEnum.__doc__ not in ("Home is where...\\n ", "Home is where...") + True + >>> cpdefPyxDocEnum.__doc__ == "Home is where...\\n " + True + >>> cpdefPxdDocEnum.__doc__ == "Home is where...\\n " + True + >>> cpdefPyxDocLineEnum.__doc__ + 'Home is where...' + >>> cpdefPxdDocLineEnum.__doc__ + 'Home is where...' + """ + pass diff --git a/tests/run/cpdef_method_override.pyx b/tests/run/cpdef_method_override.pyx index 97aafe562..aadef67a6 100644 --- a/tests/run/cpdef_method_override.pyx +++ b/tests/run/cpdef_method_override.pyx @@ -1,6 +1,6 @@ # mode: run # tag: cpdef -# ticket: gh-1771 +# ticket: 1771 def _call_method(cls): obj = cls() diff --git a/tests/run/cpdef_nogil.pyx b/tests/run/cpdef_nogil.pyx new file mode 100644 index 000000000..b44e6f551 --- /dev/null +++ b/tests/run/cpdef_nogil.pyx @@ -0,0 +1,19 @@ +# cython: binding=True +# mode: run +# tag: cyfunction + +cpdef int simple() nogil: + """ + >>> simple() + 1 + """ + return 1 + + +cpdef int call_nogil(): + """ + >>> call_nogil() + 1 + """ + with nogil: + return simple() diff --git a/tests/run/cpdef_scoped_enums.pyx b/tests/run/cpdef_scoped_enums.pyx new file mode 100644 index 000000000..8787b7751 --- /dev/null +++ b/tests/run/cpdef_scoped_enums.pyx @@ -0,0 +1,42 @@ +# mode: run +# tag: cpp, cpp11 + +cdef extern from *: + """ + enum class Enum1 { + Item1 = 1, + Item2 = 2 + }; + + enum class Enum2 { + Item4 = 4, + Item5 = 5 + }; + """ + cpdef enum class Enum1: + Item1 + Item2 + + cpdef enum class Enum2: + """Apricots and other fruits. + """ + Item4 + Item5 + + +def test_enum_to_list(): + """ + >>> test_enum_to_list() + """ + assert list(Enum1) == [1, 2] + assert list(Enum2) == [4, 5] + + +def test_enum_doc(): + """ + >>> Enum2.__doc__ == "Apricots and other fruits.\\n " + True + >>> Enum1.__doc__ != "Apricots and other fruits.\\n " + True + """ + pass diff --git a/tests/run/cpdef_scoped_enums_import.srctree b/tests/run/cpdef_scoped_enums_import.srctree new file mode 100644 index 000000000..2d79fb05a --- /dev/null +++ b/tests/run/cpdef_scoped_enums_import.srctree @@ -0,0 +1,71 @@ +# mode: run +# tag: cpp, cpp11 + +""" +PYTHON setup.py build_ext --inplace +PYTHON -c "import runner" +""" + +######## setup.py ######## + +from Cython.Build.Dependencies import cythonize +from distutils.core import setup +setup(ext_modules=cythonize("*.pyx", language='c++')) + +setup( + ext_modules = cythonize([ + "cheese.pyx", + "import_scoped_enum_test.pyx", + "dotted_import_scoped_enum_test.pyx" + ]) +) + +######## cheese.pxd ######## +# distutils: language = c++ +# distutils: extra_compile_args = -std=c++11 + + +cdef extern from * namespace "Namespace": + """ + namespace Namespace { + enum class Cheese { + cheddar = 1, + camembert = 2 + }; + } + """ + cpdef enum class Cheese: + cheddar + camembert + +######## cheese.pyx ######## +# distutils: language = c++ +# distutils: extra_compile_args = -std=c++11 + +pass + +######## import_scoped_enum_test.pyx ######## +# distutils: language = c++ +# distutils: extra_compile_args = -std=c++11 + +from cheese import Cheese +from cheese cimport Cheese + +cdef Cheese c = Cheese.cheddar +assert list(Cheese) == [1, 2] + +######## dotted_import_scoped_enum_test.pyx ######## +# distutils: language = c++ +# distutils: extra_compile_args = -std=c++11 + + +cimport cheese + +cdef cheese.Cheese c = cheese.Cheese.cheddar +assert [cheese.Cheese.cheddar, cheese.Cheese.camembert] == [1, 2] +cdef cheese.Cheese d = int(1) + +######## runner.py ######## + +import import_scoped_enum_test +import dotted_import_scoped_enum_test diff --git a/tests/run/cpdef_temps_T411.pyx b/tests/run/cpdef_temps_T411.pyx index 061d9ef13..66f08331d 100644 --- a/tests/run/cpdef_temps_T411.pyx +++ b/tests/run/cpdef_temps_T411.pyx @@ -1,4 +1,4 @@ -# ticket: 411 +# ticket: t411 cdef class A: """ diff --git a/tests/run/cpp_bool.pyx b/tests/run/cpp_bool.pyx index b6027cd9d..98e281a2e 100644 --- a/tests/run/cpp_bool.pyx +++ b/tests/run/cpp_bool.pyx @@ -1,5 +1,5 @@ # mode: run -# tag: cpp, werror +# tag: cpp, werror, no-cpp-locals from libcpp cimport bool diff --git a/tests/run/cpp_class_attrib.srctree b/tests/run/cpp_class_attrib.srctree new file mode 100644 index 000000000..3ae850839 --- /dev/null +++ b/tests/run/cpp_class_attrib.srctree @@ -0,0 +1,26 @@ +# tag: cpp + +PYTHON setup.py build_ext --inplace +PYTHON -c "import runner" + +######## setup.py ######## + +from Cython.Build.Dependencies import cythonize +from distutils.core import setup +import os + +example_dir = os.path.abspath(os.path.join(os.environ['CYTHON_PROJECT_DIR'], + 'docs/examples/userguide/wrapping_CPlusPlus')) + +ext_modules= cythonize(os.path.join(example_dir, "rect_with_attributes.pyx"), + include_path=[example_dir]) +setup(ext_modules=ext_modules) + +######## runner.py ######## + +import rect_with_attributes + +x0, y0, x1, y1 = 1, 2, 3, 4 +rect_obj = rect_with_attributes.PyRectangle(x0, y0, x1, y1) + +assert rect_obj.x0 == x0 diff --git a/tests/run/cpp_class_redef.pyx b/tests/run/cpp_class_redef.pyx index 8f0ae3ad5..36cd8ea04 100644 --- a/tests/run/cpp_class_redef.pyx +++ b/tests/run/cpp_class_redef.pyx @@ -1,5 +1,5 @@ # mode: run -# tag: cpp, warnings +# tag: cpp, warnings, no-cpp-locals # This gives a warning about the previous .pxd definition, but should not give an error. cdef cppclass Foo: diff --git a/tests/run/cpp_classes.pyx b/tests/run/cpp_classes.pyx index c7654c065..930a292b5 100644 --- a/tests/run/cpp_classes.pyx +++ b/tests/run/cpp_classes.pyx @@ -1,5 +1,5 @@ # mode: run -# tag: cpp, werror +# tag: cpp, werror, no-cpp-locals from libcpp.vector cimport vector @@ -130,6 +130,10 @@ def test_value_call(int w): del sqr +cdef struct StructWithEmpty: + Empty empty + + def get_destructor_count(): return destructor_count @@ -146,6 +150,15 @@ def test_stack_allocation(int w, int h): print rect.method(<int>5) return destructor_count +def test_stack_allocation_in_struct(): + """ + >>> d = test_stack_allocation_in_struct() + >>> get_destructor_count() - d + 1 + """ + cdef StructWithEmpty swe + sizeof(swe.empty) # use it for something + return destructor_count cdef class EmptyHolder: cdef Empty empty @@ -153,6 +166,9 @@ cdef class EmptyHolder: cdef class AnotherEmptyHolder(EmptyHolder): cdef Empty another_empty +cdef class EmptyViaStructHolder: + cdef StructWithEmpty swe + def test_class_member(): """ >>> test_class_member() @@ -183,6 +199,18 @@ def test_derived_class_member(): assert destructor_count - start_destructor_count == 2, \ destructor_count - start_destructor_count +def test_class_in_struct_member(): + """ + >>> test_class_in_struct_member() + """ + start_constructor_count = constructor_count + start_destructor_count = destructor_count + e = EmptyViaStructHolder() + #assert constructor_count - start_constructor_count == 1, \ + # constructor_count - start_constructor_count + del e + assert destructor_count - start_destructor_count == 1, \ + destructor_count - start_destructor_count cdef class TemplateClassMember: cdef vector[int] x diff --git a/tests/run/cpp_classes_def.pyx b/tests/run/cpp_classes_def.pyx index 0377fc1f6..074d85a13 100644 --- a/tests/run/cpp_classes_def.pyx +++ b/tests/run/cpp_classes_def.pyx @@ -1,5 +1,5 @@ # mode: run -# tag: cpp, werror, cpp11 +# tag: cpp, werror, cpp11, no-cpp-locals # cython: experimental_cpp_class_def=True cdef double pi @@ -9,6 +9,7 @@ from libcpp cimport bool from libcpp.memory cimport unique_ptr from libcpp.vector cimport vector from cython.operator cimport dereference as deref +import cython cdef extern from "shapes.h" namespace "shapes": cdef cppclass Shape: @@ -23,6 +24,11 @@ cdef cppclass RegularPolygon(Shape): float area() const: cdef double theta = pi / this.n return this.radius * this.radius * sin(theta) * cos(theta) * this.n + void do_with() except *: + # only a compile test - the file doesn't actually have to exist + # "with" was broken by https://github.com/cython/cython/issues/4212 + with open("doesnt matter") as f: + return def test_Poly(int n, float radius=1): """ diff --git a/tests/run/cpp_const_method.pyx b/tests/run/cpp_const_method.pyx index ef1cada31..d959dbe09 100644 --- a/tests/run/cpp_const_method.pyx +++ b/tests/run/cpp_const_method.pyx @@ -1,5 +1,5 @@ # mode: run -# tag: cpp, werror +# tag: cpp, werror, no-cpp-locals # cython: experimental_cpp_class_def=True from libcpp.vector cimport vector @@ -82,6 +82,6 @@ def test_vector_members(py_a, py_b): cdef vector_members(vector[const Wrapper[int]*] a, const vector[wrapInt*] b): # TODO: Cython-level error. # b[0].set(100) - + # TODO: const_iterator return [x.get() for x in a], b[0].get() diff --git a/tests/run/cpp_enums.pyx b/tests/run/cpp_enums.pyx new file mode 100644 index 000000000..ac5ea9fe7 --- /dev/null +++ b/tests/run/cpp_enums.pyx @@ -0,0 +1,58 @@ +# tag: cpp +# mode: run, no-cpp-locals + +cdef extern from *: + """ + enum Enum1 { + Item1, + Item2 + }; + + """ + cdef enum Enum1: + Item1 + Item2 + +a = Item1 +b = Item2 + +cdef Enum1 x, y +x = Item1 +y = Item2 + + +def compare_enums(): + """ + >>> compare_enums() + (True, True, True, True) + """ + return x == a, a == Item1, b == y, y == Item2 + + +cdef extern from * namespace "Namespace1": + """ + namespace Namespace1 { + enum Enum2 { + Item3, + Item4 + }; + } + """ + cdef enum Enum2: + Item3 + Item4 + +c = Item3 +d = Item4 + +cdef Enum2 z, w +z = Item3 +w = Item4 + + +def compare_namespace_enums(): + """ + >>> compare_namespace_enums() + (True, True, True, True) + """ + return z == c, c == Item3, d == w, d == Item4 diff --git a/tests/run/cpp_exception_declaration_compatibility.srctree b/tests/run/cpp_exception_declaration_compatibility.srctree deleted file mode 100644 index 775761c23..000000000 --- a/tests/run/cpp_exception_declaration_compatibility.srctree +++ /dev/null @@ -1,38 +0,0 @@ -# tag: cpp - -""" -PYTHON setup.py build_ext -i -PYTHON test.py -""" - -############### setup.py ################### -from distutils.core import setup -from Cython.Build import cythonize - -setup( - name="cython_test", - ext_modules=cythonize('*.pyx', language="c++") -) - - -############### test.py ################### - -from cpp_exc import TestClass - -TestClass().test_func() - - -############### cpp_exc.pxd ################### - -cdef inline void handle_exception(): - pass - -cdef class TestClass: - cpdef test_func(self) except +handle_exception - - -############### cpp_exc.pyx ################### - -cdef class TestClass: - cpdef test_func(self) except +handle_exception: - print('test') diff --git a/tests/run/cpp_exceptions_utility_code.pyx b/tests/run/cpp_exceptions_utility_code.pyx new file mode 100644 index 000000000..74f87dfb1 --- /dev/null +++ b/tests/run/cpp_exceptions_utility_code.pyx @@ -0,0 +1,33 @@ +# mode: run +# tag: cpp, werror, no-cpp-locals +# ticket: 3065 + +# This is intentionally in a file on its own. The issue was that it failed to generate utility-code +# and so putting it with the other c++ exception checks wouldn't be a useful test + +cdef extern from *: + """ + #include <stdexcept> + + void cppf(int raiseCpp) { + if (raiseCpp) { + throw std::runtime_error("cpp"); + } else { + PyErr_SetString(PyExc_RuntimeError, "py"); + } + } + """ + void cppf(int) except+* + + +def callcppf(int raiseCpp): + """ + >>> callcppf(0) + py + >>> callcppf(1) + cpp + """ + try: + cppf(raiseCpp) + except RuntimeError as e: + print(e.args[0]) diff --git a/tests/run/cpp_forwarding_ref.pyx b/tests/run/cpp_forwarding_ref.pyx new file mode 100644 index 000000000..b3b044353 --- /dev/null +++ b/tests/run/cpp_forwarding_ref.pyx @@ -0,0 +1,66 @@ +# mode: run +# tag: cpp, cpp11, no-cpp-locals + +from libcpp.utility cimport move + + +cdef extern from *: + """ + #include <utility> + + const char* f(int& x) { + (void) x; + return "lvalue-ref"; + } + + const char* f(int&& x) { + (void) x; + return "rvalue-ref"; + } + + template <typename T> + const char* foo(T&& x) + { + return f(std::forward<T>(x)); + } + """ + const char* foo[T](T&& x) + + +cdef extern from *: + """ + #include <utility> + + template <typename T1> + const char* bar(T1 x, T1 y) { + return "first"; + } + + template <typename T1, typename T2> + const char* bar(T1&& x, T2 y, T2 z) { + return "second"; + } + + """ + const char* bar[T1](T1 x, T1 y) + const char* bar[T1, T2](T1&& x, T2 y, T2 z) + +def test_forwarding_ref(): + """ + >>> test_forwarding_ref() + """ + cdef int x = 1 + assert foo(x) == b"lvalue-ref" + assert foo(<int>(1)) == b"rvalue-ref" + assert foo(move(x)) == b"rvalue-ref" + + +def test_forwarding_ref_overload(): + """ + >>> test_forwarding_ref_overload() + """ + cdef int x = 1 + cdef int y = 2 + cdef int z = 3 + assert bar(x, y) == b"first" + assert bar(x, y, z) == b"second" diff --git a/tests/run/cpp_iterators.pyx b/tests/run/cpp_iterators.pyx index e5c54c9db..716fcb021 100644 --- a/tests/run/cpp_iterators.pyx +++ b/tests/run/cpp_iterators.pyx @@ -1,5 +1,5 @@ # mode: run -# tag: cpp, werror +# tag: cpp, werror, no-cpp-locals from libcpp.deque cimport deque from libcpp.vector cimport vector @@ -25,7 +25,7 @@ def test_vector(py_v): def test_deque_iterator_subtraction(py_v): """ - >>> test_deque_iterator_subtraction([1, 2, 3]) + >>> print(test_deque_iterator_subtraction([1, 2, 3])) 3 """ cdef deque[int] dint @@ -38,7 +38,7 @@ def test_deque_iterator_subtraction(py_v): def test_vector_iterator_subtraction(py_v): """ - >>> test_vector_iterator_subtraction([1, 2, 3]) + >>> print(test_vector_iterator_subtraction([1, 2, 3])) 3 """ cdef vector[int] vint = py_v @@ -140,3 +140,40 @@ def test_iteration_in_generator_reassigned(): if vint is not orig_vint: del vint del orig_vint + +cdef extern from *: + """ + std::vector<int> make_vec1() { + std::vector<int> vint; + vint.push_back(1); + vint.push_back(2); + return vint; + } + """ + cdef vector[int] make_vec1() except + + +cdef vector[int] make_vec2() except *: + return make_vec1() + +cdef vector[int] make_vec3(): + try: + return make_vec1() + except: + pass + +def test_iteration_from_function_call(): + """ + >>> test_iteration_from_function_call() + 1 + 2 + 1 + 2 + 1 + 2 + """ + for i in make_vec1(): + print(i) + for i in make_vec2(): + print(i) + for i in make_vec3(): + print(i) diff --git a/tests/run/cpp_locals_directive.pyx b/tests/run/cpp_locals_directive.pyx new file mode 100644 index 000000000..6c9c89ba5 --- /dev/null +++ b/tests/run/cpp_locals_directive.pyx @@ -0,0 +1,235 @@ +# mode: run +# tag: cpp, cpp17, no-cpp-locals +# no-cpp-locals because the test is already run with it explicitly set + +# cython: cpp_locals=True + +cimport cython + +from libcpp cimport bool as cppbool + +cdef extern from *: + r""" + static void print_C_destructor(); + + class C { + public: + C() = delete; // look! No default constructor + C(int x, bool print_destructor=true) : x(x), print_destructor(print_destructor) {} + C(C&& rhs) : x(rhs.x), print_destructor(rhs.print_destructor) { + rhs.print_destructor = false; // moved-from instances are deleted silently + } + C& operator=(C&& rhs) { + x=rhs.x; + print_destructor=rhs.print_destructor; + rhs.print_destructor = false; // moved-from instances are deleted silently + return *this; + } + C(const C& rhs) = default; + C& operator=(const C& rhs) = default; + ~C() { + if (print_destructor) print_C_destructor(); + } + + int getX() const { return x; } + + private: + int x; + bool print_destructor; + }; + + C make_C(int x) { + return C(x); + } + """ + cdef cppclass C: + C(int) + C(int, cppbool) + int getX() const + C make_C(int) except + # needs a temp to receive + +# this function just makes sure the output from the destructor can be captured by doctest +cdef void print_C_destructor "print_C_destructor" () with gil: + print("~C()") + +def maybe_assign_infer(assign, value, do_print): + """ + >>> maybe_assign_infer(True, 5, True) + 5 + ~C() + >>> maybe_assign_infer(False, 0, True) + Traceback (most recent call last): + ... + UnboundLocalError: local variable 'x' referenced before assignment + >>> maybe_assign_infer(False, 0, False) # no destructor call here + """ + if assign: + x = C(value) + if do_print: + print(x.getX()) + +def maybe_assign_cdef(assign, value): + """ + >>> maybe_assign_cdef(True, 5) + 5 + ~C() + >>> maybe_assign_cdef(False, 0) + Traceback (most recent call last): + ... + UnboundLocalError: local variable 'x' referenced before assignment + """ + cdef C x + if assign: + x = C(value) + print(x.getX()) + +def maybe_assign_annotation(assign, value): + """ + >>> maybe_assign_annotation(True, 5) + 5 + ~C() + >>> maybe_assign_annotation(False, 0) + Traceback (most recent call last): + ... + UnboundLocalError: local variable 'x' referenced before assignment + """ + x: C + if assign: + x = C(value) + print(x.getX()) + +def maybe_assign_directive1(assign, value): + """ + >>> maybe_assign_directive1(True, 5) + 5 + ~C() + >>> maybe_assign_directive1(False, 0) + Traceback (most recent call last): + ... + UnboundLocalError: local variable 'x' referenced before assignment + """ + x = cython.declare(C) + if assign: + x = C(value) + print(x.getX()) + +@cython.locals(x=C) +def maybe_assign_directive2(assign, value): + """ + >>> maybe_assign_directive2(True, 5) + 5 + ~C() + >>> maybe_assign_directive2(False, 0) + Traceback (most recent call last): + ... + UnboundLocalError: local variable 'x' referenced before assignment + """ + if assign: + x = C(value) + print(x.getX()) + +def maybe_assign_nocheck(assign, value): + """ + >>> maybe_assign_nocheck(True, 5) + 5 + ~C() + + # unfortunately it's quite difficult to test not assigning because there's a decent chance it'll crash + """ + if assign: + x = C(value) + with cython.initializedcheck(False): + print(x.getX()) + +def uses_temp(value): + """ + needs a temp to handle the result of make_C - still doesn't use the default constructor + >>> uses_temp(10) + 10 + ~C() + """ + + x = make_C(value) + print(x.getX()) + +# c should not be optional - it isn't easy to check this, but we can at least check it compiles +cdef void has_argument(C c): + print(c.getX()) + +def call_has_argument(): + """ + >>> call_has_argument() + 50 + """ + has_argument(C(50, False)) + +cdef class HoldsC: + """ + >>> inst = HoldsC(True, False) + >>> inst.getCX() + 10 + >>> access_from_function_with_different_directive(inst) + 10 + 10 + >>> inst.getCX() # it was changed in access_from_function_with_different_directive + 20 + >>> inst = HoldsC(False, False) + >>> inst.getCX() + Traceback (most recent call last): + ... + AttributeError: C++ attribute 'value' is not initialized + >>> access_from_function_with_different_directive(inst) + Traceback (most recent call last): + ... + AttributeError: C++ attribute 'value' is not initialized + """ + cdef C value + def __cinit__(self, initialize, print_destructor): + if initialize: + self.value = C(10, print_destructor) + + def getCX(self): + return self.value.getX() + +cdef acceptC(C& c): + return c.getX() + +@cython.cpp_locals(False) +def access_from_function_with_different_directive(HoldsC c): + # doctest is in HoldsC class + print(acceptC(c.value)) # this originally tried to pass a __Pyx_Optional<C> as a C instance + print(c.value.getX()) + c.value = C(20, False) # make sure that we can change it too + +def dont_test_on_pypy(f): + import sys + if not hasattr(sys, "pypy_version_info"): + return f + +@dont_test_on_pypy # non-deterministic destruction +def testHoldsCDestruction(initialize): + """ + >>> testHoldsCDestruction(True) + ~C() + >>> testHoldsCDestruction(False) # no destructor + """ + x = HoldsC(initialize, True) + del x + +cdef C global_var + +def initialize_global_var(): + global global_var + global_var = C(-1, False) + +def read_global_var(): + """ + >>> read_global_var() + Traceback (most recent call last): + ... + NameError: C++ global 'global_var' is not initialized + >>> initialize_global_var() + >>> read_global_var() + -1 + """ + print(global_var.getX()) diff --git a/tests/run/cpp_locals_directive_unused.pyx b/tests/run/cpp_locals_directive_unused.pyx new file mode 100644 index 000000000..429e64beb --- /dev/null +++ b/tests/run/cpp_locals_directive_unused.pyx @@ -0,0 +1,14 @@ +# mode: run +# tag: cpp, cpp17, no-cpp-locals +# no cpp_locals because this test is already run with cpp_locals explicitly set + +# cython: cpp_locals=True + +cdef cppclass C: + C() + +cdef class PyC: + """ + >>> PyC() and None # doesn't really do anything, but should run + """ + cdef C # this limited usage wasn't triggering the creation of utility code diff --git a/tests/run/cpp_move.pyx b/tests/run/cpp_move.pyx index d93afe28d..ca7cb7794 100644 --- a/tests/run/cpp_move.pyx +++ b/tests/run/cpp_move.pyx @@ -1,5 +1,5 @@ # mode: run -# tag: cpp, werror, cpp11 +# tag: cpp, werror, cpp11, no-cpp-locals from libcpp cimport nullptr from libcpp.memory cimport shared_ptr, make_shared diff --git a/tests/run/cpp_nested_classes.pyx b/tests/run/cpp_nested_classes.pyx index 6008b2379..b50f79936 100644 --- a/tests/run/cpp_nested_classes.pyx +++ b/tests/run/cpp_nested_classes.pyx @@ -1,4 +1,4 @@ -# tag: cpp +# tag: cpp, no-cpp-locals cdef extern from "cpp_nested_classes_support.h": cdef cppclass A: @@ -26,6 +26,10 @@ cdef extern from "cpp_nested_classes_support.h": pass +ctypedef A AliasA1 +ctypedef AliasA1 AliasA2 + + def test_nested_classes(): """ >>> test_nested_classes() @@ -47,6 +51,20 @@ def test_nested_typedef(py_x): cdef A.my_int x = py_x assert A.negate(x) == -py_x +def test_typedef_for_nested(py_x): + """ + >>> test_typedef_for_nested(5) + """ + cdef AliasA1.my_int x = py_x + assert A.negate(x) == -py_x + +def test_typedef_for_nested_deep(py_x): + """ + >>> test_typedef_for_nested_deep(5) + """ + cdef AliasA2.my_int x = py_x + assert A.negate(x) == -py_x + def test_typed_nested_typedef(x): """ >>> test_typed_nested_typedef(4) diff --git a/tests/run/cpp_nonstdint.h b/tests/run/cpp_nonstdint.h index 63c977972..0805576b5 100644 --- a/tests/run/cpp_nonstdint.h +++ b/tests/run/cpp_nonstdint.h @@ -53,7 +53,7 @@ class Integral { resize_signed_int(bytes, N, extended, len); } bool res = memcmp(extended, other, len); - delete extended; + delete [] extended; return res; } bool operator!=(const long long val) const diff --git a/tests/run/cpp_nonstdint.pyx b/tests/run/cpp_nonstdint.pyx index 62153d190..238107b79 100644 --- a/tests/run/cpp_nonstdint.pyx +++ b/tests/run/cpp_nonstdint.pyx @@ -1,4 +1,4 @@ -# tag: cpp +# tag: cpp, no-cpp-locals cdef extern from "cpp_nonstdint.h": ctypedef int Int24 diff --git a/tests/run/cpp_operator_exc_handling.pyx b/tests/run/cpp_operator_exc_handling.pyx index 381870c22..67b00e39e 100644 --- a/tests/run/cpp_operator_exc_handling.pyx +++ b/tests/run/cpp_operator_exc_handling.pyx @@ -1,5 +1,5 @@ # mode: run -# tag: cpp, werror +# tag: cpp, werror, no-cpp-locals from cython.operator import (preincrement, predecrement, postincrement, postdecrement) diff --git a/tests/run/cpp_operators_helper.h b/tests/run/cpp_operators_helper.h index ec77d7f62..6c5a545fe 100644 --- a/tests/run/cpp_operators_helper.h +++ b/tests/run/cpp_operators_helper.h @@ -84,10 +84,12 @@ NONMEMBER_BIN_OP2(COMMA) #define REF_BIN_OP(op) int& operator op (int x) { x++; return value; } class RefTestOps { - int value = 0; + int value; public: + RefTestOps() { value = 0; } + REF_UN_OP(-); REF_UN_OP(+); REF_UN_OP(*); diff --git a/tests/run/cpp_scoped_enums.pyx b/tests/run/cpp_scoped_enums.pyx new file mode 100644 index 000000000..7a2bc912f --- /dev/null +++ b/tests/run/cpp_scoped_enums.pyx @@ -0,0 +1,136 @@ +# mode: run +# tag: cpp, cpp11 + +from libcpp.limits cimport numeric_limits + +cdef extern from *: + """ + enum class Enum1 { + Item1, + Item2 + }; + """ + cdef enum class Enum1: + Item1 + Item2 + + +cdef extern from * namespace "Namespace1": + """ + namespace Namespace1 { + enum class Enum2 { + Item1, + Item2 + }; + } + """ + cdef enum class Enum2: + Item1 + Item2 + + +cdef enum class Enum3(int): + a = 1 + b = 2 + + +cdef extern from *: + """ + enum class sorted + { + a = 1, + b = 0 + }; + """ + cdef enum class Enum4 "sorted": + a + b + + +cdef extern from *: + """ + #include <limits> + + enum class LongIntEnum : long int { + val = std::numeric_limits<long int>::max(), + }; + """ + enum class LongIntEnum(long int): + val + + +def test_compare_enums(): + """ + >>> test_compare_enums() + (True, True, False, False) + """ + cdef Enum1 x, y + x = Enum1.Item1 + y = Enum1.Item2 + + return ( + x == Enum1.Item1, + y == Enum1.Item2, + x == Enum1.Item2, + y == Enum1.Item1 + ) + + +def test_compare_namespace_enums(): + """ + >>> test_compare_enums() + (True, True, False, False) + """ + cdef Enum2 z, w + + z = Enum2.Item1 + w = Enum2.Item2 + + return ( + z == Enum2.Item1, + w == Enum2.Item2, + z == Enum2.Item2, + w == Enum2.Item1 + ) + + +def test_coerce_to_from_py_value(object i): + """ + >>> test_coerce_to_from_py_value(1) + (True, False) + + >>> test_coerce_to_from_py_value(2) + (False, True) + + >>> test_coerce_to_from_py_value(3) + (False, False) + + >>> test_coerce_to_from_py_value(11111111111111111111111111111111111111111111) + Traceback (most recent call last): + OverflowError: Python int too large to convert to C long + """ + cdef Enum3 x = i + y = Enum3.b + + return ( + x == Enum3.a, + y == int(i) + ) + + +def test_reserved_cname(): + """ + >>> test_reserved_cname() + True + """ + cdef Enum4 x = Enum4.a + return Enum4.a == int(1) + + +def test_large_enum(): + """ + >>> test_large_enum() + True + """ + long_max = int(numeric_limits[long].max()) + return LongIntEnum.val == long_max diff --git a/tests/run/cpp_smart_ptr.pyx b/tests/run/cpp_smart_ptr.pyx index 7a249fbd3..d71151c5e 100644 --- a/tests/run/cpp_smart_ptr.pyx +++ b/tests/run/cpp_smart_ptr.pyx @@ -1,5 +1,5 @@ # mode: run -# tag: cpp, werror, cpp11 +# tag: cpp, werror, cpp11, no-cpp-locals from libcpp.memory cimport unique_ptr, shared_ptr, default_delete, dynamic_pointer_cast from libcpp cimport nullptr @@ -81,6 +81,15 @@ cdef cppclass C(B): cdef shared_ptr[A] holding_subclass = shared_ptr[A](new C()) + +def test_assignment_to_base_class(): + """ + >>> test_assignment_to_base_class() + """ + cdef shared_ptr[C] derived = shared_ptr[C](new C()) + cdef shared_ptr[A] base = derived + + def test_dynamic_pointer_cast(): """ >>> test_dynamic_pointer_cast() diff --git a/tests/run/cpp_static_method_overload.pyx b/tests/run/cpp_static_method_overload.pyx new file mode 100644 index 000000000..59eec2f44 --- /dev/null +++ b/tests/run/cpp_static_method_overload.pyx @@ -0,0 +1,49 @@ +# mode: run +# tag: cpp, no-cpp-locals + +cdef extern from *: + """ + struct Foo + { + + static const char* bar(int x, int y) { + return "second"; + } + + static const char* bar(int x) { + return "first"; + } + + const char* baz(int x, int y) { + return "second"; + } + + const char* baz(int x) { + return "first"; + } + }; + """ + cppclass Foo: + @staticmethod + const char* bar(int x) + + @staticmethod + const char* bar(int x, int y) + + const char* baz(int x) + const char* baz(int x, int y) + +def test_normal_method_overload(): + """ + >>> test_normal_method_overload() + """ + cdef Foo f + assert f.baz(1) == b"first" + assert f.baz(1, 2) == b"second" + +def test_static_method_overload(): + """ + >>> test_static_method_overload() + """ + assert Foo.bar(1) == b"first" + assert Foo.bar(1, 2) == b"second" diff --git a/tests/run/cpp_stl_algo_execpolicies.pyx b/tests/run/cpp_stl_algo_execpolicies.pyx new file mode 100644 index 000000000..989e42f6c --- /dev/null +++ b/tests/run/cpp_stl_algo_execpolicies.pyx @@ -0,0 +1,53 @@ +# mode: run +# tag: cpp, werror, cpp17, cppexecpolicies, no-cpp-locals + +from libcpp.algorithm cimport is_sorted, sort, stable_sort, nth_element, all_of, count, copy +from libcpp.execution cimport seq +from libcpp.vector cimport vector +from libcpp.functional cimport greater +from libcpp.iterator cimport back_inserter +from libcpp cimport bool + + +def is_sorted_ints(vector[int] values): + """ + Test is_sorted. + + >>> is_sorted_ints([3, 1, 4, 1, 5]) + False + >>> is_sorted_ints([1, 1, 3, 4, 5]) + True + """ + return is_sorted(seq, values.begin(), values.end()) + + +def sort_ints_reverse(vector[int] values): + """Test sort using a standard library comparison function object. + + >>> sort_ints_reverse([5, 7, 4, 2, 8, 6, 1, 9, 0, 3]) + [9, 8, 7, 6, 5, 4, 3, 2, 1, 0] + """ + sort(seq, values.begin(), values.end(), greater[int]()) + return values + + +def count_ones(vector[int] values): + """ + Test count. + + >>> count_ones([1, 2, 1, 2]) + 2 + """ + return count(seq, values.begin(), values.end(), 1) + + +def copy_int(vector[int] values): + """ + Test copy. + + >>> copy_int(range(5)) + [0, 1, 2, 3, 4] + """ + cdef vector[int] out + copy(seq, values.begin(), values.end(), back_inserter(out)) + return out diff --git a/tests/run/cpp_stl_algo_modifying_sequence_ops.pyx b/tests/run/cpp_stl_algo_modifying_sequence_ops.pyx new file mode 100644 index 000000000..0d823e288 --- /dev/null +++ b/tests/run/cpp_stl_algo_modifying_sequence_ops.pyx @@ -0,0 +1,436 @@ +# mode: run +# tag: cpp, werror, cpp11, no-cpp-locals + +from __future__ import print_function + +from cython.operator cimport dereference as deref +from cython.operator cimport preincrement, postincrement +from libcpp cimport bool +from libcpp.algorithm cimport copy, copy_if, copy_n, copy_backward, move, move_backward, fill, fill_n, transform +from libcpp.algorithm cimport generate, generate_n, remove, remove_if, remove_copy, remove_copy_if, replace, replace_if +from libcpp.algorithm cimport replace_copy, replace_copy_if, swap, swap_ranges, iter_swap, reverse, reverse_copy +from libcpp.algorithm cimport rotate, rotate_copy, unique, unique_copy +from libcpp.algorithm cimport sort, upper_bound, min_element, max_element +from libcpp.iterator cimport back_inserter +from libcpp.string cimport string +from libcpp.vector cimport vector + + +def copy_int(vector[int] values): + """ + Test copy. + + >>> copy_int(range(5)) + [0, 1, 2, 3, 4] + """ + cdef vector[int] out + copy(values.begin(), values.end(), back_inserter(out)) + return out + + +cdef bool is_odd(int i): + return i % 2 + + +def copy_int_if_odd(vector[int] values): + """ + Test copy_if. + + >>> copy_int_if_odd(range(5)) + [1, 3] + """ + cdef vector[int] out + copy_if(values.begin(), values.end(), back_inserter(out), is_odd) + return out + + +def copy_int_n(vector[int] values, int count): + """ + Test copy_n. + + >>> copy_int_n(range(5), 2) + [0, 1] + """ + cdef vector[int] out + copy_n(values.begin(), count, back_inserter(out)) + return out + + +def copy_int_backward(vector[int] values): + """ + Test copy_backward. + + >>> copy_int_backward(range(5)) + [0, 0, 0, 0, 1, 2, 3, 4] + """ + out = vector[int](values.size() + 3) + copy_backward(values.begin(), values.end(), out.end()) + return out + + +def move_int(vector[int] values): + """ + Test move. + + >>> move_int(range(5)) + [0, 1, 2, 3, 4] + """ + cdef vector[int] out + move(values.begin(), values.end(), back_inserter(out)) + return out + + +def move_int_backward(vector[int] values): + """ + Test move_backward. + + >>> move_int_backward(range(5)) + [0, 0, 0, 0, 1, 2, 3, 4] + """ + out = vector[int](values.size() + 3) + move_backward(values.begin(), values.end(), out.end()) + return out + + +def fill_int(vector[int] array, int value): + """ + Test fill. + + >>> fill_int(range(5), -1) + [-1, -1, -1, -1, -1] + """ + fill(array.begin(), array.end(), value) + return array + + +def fill_int_n(vector[int] array, int count, int value): + """ + Test fill_n. + + >>> fill_int_n(range(5), 3, -1) + [-1, -1, -1, 3, 4] + """ + fill_n(array.begin(), count, value) + return array + + +cdef int to_ord(unsigned char c): + return c + + +def string_to_ord(string s): + """ + Test transform (unary version). + + >> string_to_ord(b"HELLO") + [72, 69, 76, 76, 79] + """ + cdef vector[int] ordinals + transform(s.begin(), s.end(), back_inserter(ordinals), to_ord) + return ordinals + + +cdef int add_ints(int lhs, int rhs): + return lhs + rhs + + +def add_int_vectors(vector[int] lhs, vector[int] rhs): + """ + Test transform (binary version). + + >>> add_int_vectors([1, 2, 3], [4, 5, 6]) + [5, 7, 9] + """ + transform(lhs.begin(), lhs.end(), rhs.begin(), lhs.begin(), add_ints) + return lhs + + +cdef int i = 0 +cdef int generator(): + return postincrement(i) + + +def generate_ints(int count): + """ + Test generate. + + >> generate_ints(5) + [0, 1, 2, 3, 4] + """ + out = vector[int](count) + generate(out.begin(), out.end(), generator) + return out + + +cdef int j = 0 +cdef int generator2(): + return postincrement(j) + + +def generate_n_ints(int count): + """ + Test generate_n. + + >> generate_n_ints(5) + [0, 1, 2, 3, 4, 0, 0, 0] + """ + out = vector[int](count + 3) + generate_n(out.begin(), count, generator2) + return out + + +def remove_spaces(string s): + """ + Test remove. + + >>> print(remove_spaces(b"Text with some spaces").decode("ascii")) + Textwithsomespaces + """ + s.erase(remove(s.begin(), s.end(), ord(" ")), s.end()) + return s + + +cdef bool is_whitespace(unsigned char c) except -1: + # std::isspace from <cctype> + return chr(c) in " \f\n\r\t\v" + + +def remove_whitespace(string s): + r""" + Test remove_if. + + >>> print(remove_whitespace(b"Text\n with\tsome \t whitespaces\n\n").decode("ascii")) + Textwithsomewhitespaces + """ + s.erase(remove_if(s.begin(), s.end(), &is_whitespace), s.end()) + return s + + +def remove_spaces2(string s): + """ + Test remove_copy. + + >>> print(remove_spaces2(b"Text with some spaces").decode("ascii")) + Textwithsomespaces + """ + cdef string out + remove_copy(s.begin(), s.end(), back_inserter(out), ord(" ")) + return out + + +def remove_whitespace2(string s): + r""" + Test remove_copy_if. + + >>> print(remove_whitespace2(b"Text\n with\tsome \t whitespaces\n\n").decode("ascii")) + Textwithsomewhitespaces + """ + cdef string out + remove_copy_if(s.begin(), s.end(), back_inserter(out), &is_whitespace) + return out + + +def replace_ints(vector[int] values, int old, int new): + """ + Test replace. + + >>> replace_ints([5, 7, 4, 2, 8, 6, 1, 9, 0, 3], 8, 88) + [5, 7, 4, 2, 88, 6, 1, 9, 0, 3] + """ + replace(values.begin(), values.end(), old, new) + return values + + +cdef bool less_than_five(int i): + return i < 5 + + +def replace_ints_less_than_five(vector[int] values, int new): + """ + Test replace_if (using cppreference example that doesn't translate well). + + >>> replace_ints_less_than_five([5, 7, 4, 2, 88, 6, 1, 9, 0, 3], 55) + [5, 7, 55, 55, 88, 6, 55, 9, 55, 55] + """ + replace_if(values.begin(), values.end(), less_than_five, new) + return values + + +def replace_ints2(vector[int] values, int old, int new): + """ + Test replace_copy. + + >>> replace_ints2([5, 7, 4, 2, 8, 6, 1, 9, 0, 3], 8, 88) + [5, 7, 4, 2, 88, 6, 1, 9, 0, 3] + """ + cdef vector[int] out + replace_copy(values.begin(), values.end(), back_inserter(out), old, new) + return out + + +def replace_ints_less_than_five2(vector[int] values, int new): + """ + Test replace_copy_if (using cppreference example that doesn't translate well). + + >>> replace_ints_less_than_five2([5, 7, 4, 2, 88, 6, 1, 9, 0, 3], 55) + [5, 7, 55, 55, 88, 6, 55, 9, 55, 55] + """ + cdef vector[int] out + replace_copy_if(values.begin(), values.end(), back_inserter(out), less_than_five, new) + return out + + +def test_swap_ints(): + """ + >>> test_swap_ints() + 5 3 + 3 5 + """ + cdef int a = 5, b = 3 + print(a, b) + swap(a, b) + print(a, b) + + +def test_swap_vectors(): + """ + >>> test_swap_vectors() + [1, 2, 3] [4, 5, 6] + [4, 5, 6] [1, 2, 3] + """ + cdef vector[int] a = [1, 2, 3], b = [4, 5, 6] + print(a, b) + swap(a, b) + print(a, b) + + +def test_swap_ranges(): + """ + >>> test_swap_ranges() + [1, 2, 3] [4, 5, 6] + [4, 5, 6] [1, 2, 3] + """ + cdef vector[int] a = [1, 2, 3], b = [4, 5, 6] + print(a, b) + swap_ranges(a.begin(), a.end(), b.begin()) + print(a, b) + + +def selection_sort(vector[int] values, reversed=False): + """ + Test iter_swap using cppreference example. Extra "reversed argument tests max_element + + >>> selection_sort([-7, 6, 2, 4, -1, 6, -9, -1, 2, -5, 10, -9, -5, -3, -5, -3, 6, 6, 1, 8]) + [-9, -9, -7, -5, -5, -5, -3, -3, -1, -1, 1, 2, 2, 4, 6, 6, 6, 6, 8, 10] + >>> selection_sort([-7, 6, 2, 4, -1, 6, -9, -1, 2, -5, 10, -9, -5, -3, -5, -3, 6, 6, 1, 8], reversed=True) + [10, 8, 6, 6, 6, 6, 4, 2, 2, 1, -1, -1, -3, -3, -5, -5, -5, -7, -9, -9] + """ + i = values.begin() + end = values.end() + while i < end: + iter_swap(i, min_element(i, end) if not reversed else max_element(i,end)) + preincrement(i) + return values + + +def reverse_ints(vector[int] values): + """ + Test reverse. + + >>> reverse_ints([1, 2, 3]) + [3, 2, 1] + """ + reverse(values.begin(), values.end()) + return values + + +def reverse_ints2(vector[int] values): + """ + Test reverse_copy. + + >>> reverse_ints2([1, 2, 3]) + [3, 2, 1] + """ + cdef vector[int] out + reverse_copy(values.begin(), values.end(), back_inserter(out)) + return out + + +def insertion_sort(vector[int] values): + """ + Test rotate using cppreference example. + + >>> insertion_sort([2, 4, 2, 0, 5, 10, 7, 3, 7, 1]) + [0, 1, 2, 2, 3, 4, 5, 7, 7, 10] + """ + i = values.begin() + while i < values.end(): + rotate(upper_bound(values.begin(), i, deref(i)), i, i + 1) + preincrement(i) + return values + + +def rotate_ints_about_middle(vector[int] values): + """ + Test rotate_copy. + + >>> rotate_ints_about_middle([1, 2, 3, 4, 5]) + [3, 4, 5, 1, 2] + """ + cdef vector[int] out + cdef vector[int].iterator pivot = values.begin() + values.size()/2 + rotate_copy(values.begin(), pivot, values.end(), back_inserter(out)) + return out + + +def unique_ints(vector[int] values): + """ + Test unique. + + >>> unique_ints([1, 2, 3, 1, 2, 3, 3, 4, 5, 4, 5, 6, 7]) + [1, 2, 3, 4, 5, 6, 7] + """ + sort(values.begin(), values.end()) + values.erase(unique(values.begin(), values.end()), values.end()) + return values + + +cdef bool both_space(unsigned char lhs, unsigned char rhs): + return lhs == rhs == ord(' ') + + +def collapse_spaces(string text): + """ + Test unique (predicate version) using cppreference example for unique_copy. + + >>> print(collapse_spaces(b"The string with many spaces!").decode("ascii")) + The string with many spaces! + """ + last = unique(text.begin(), text.end(), &both_space) + text.erase(last, text.end()) + return text + + +def unique_ints2(vector[int] values): + """ + Test unique_copy. + + >>> unique_ints2([1, 2, 3, 1, 2, 3, 3, 4, 5, 4, 5, 6, 7]) + [1, 2, 3, 4, 5, 6, 7] + """ + cdef vector[int] out + sort(values.begin(), values.end()) + unique_copy(values.begin(), values.end(), back_inserter(out)) + return out + + +def collapse_spaces2(string text): + """ + Test unique_copy (predicate version) using cppreference example. + + >>> print(collapse_spaces2(b"The string with many spaces!").decode("ascii")) + The string with many spaces! + """ + cdef string out + unique_copy(text.begin(), text.end(), back_inserter(out), &both_space) + return out diff --git a/tests/run/cpp_stl_algo_nonmodifying_sequence_ops.pyx b/tests/run/cpp_stl_algo_nonmodifying_sequence_ops.pyx new file mode 100644 index 000000000..2b001b5f2 --- /dev/null +++ b/tests/run/cpp_stl_algo_nonmodifying_sequence_ops.pyx @@ -0,0 +1,307 @@ +# mode: run +# tag: cpp, werror, cpp11 + +from cython.operator cimport dereference as deref + +from libcpp cimport bool +from libcpp.algorithm cimport all_of, any_of, none_of, for_each, count, count_if, mismatch, find, find_if, find_if_not +from libcpp.algorithm cimport find_end, find_first_of, adjacent_find, search, search_n +from libcpp.iterator cimport distance +from libcpp.string cimport string +from libcpp.vector cimport vector + + +cdef bool is_odd(int i): + return i % 2 + + +def all_odd(vector[int] values): + """ + Test all_of with is_odd predicate. + + >>> all_odd([3, 5, 7, 11, 13, 17, 19, 23]) + True + >>> all_odd([3, 4]) + False + """ + return all_of(values.begin(), values.end(), is_odd) + + +def any_odd(vector[int] values): + """ + Test any_of with is_odd predicate. + + >>> any_odd([1, 2, 3, 4]) + True + >>> any_odd([2, 4, 6, 8]) + False + """ + return any_of(values.begin(), values.end(), is_odd) + + +def none_odd(vector[int] values): + """ + Test none_of with is_odd predicate. + + >>> none_odd([2, 4, 6, 8]) + True + >>> none_odd([1, 2, 3, 4]) + False + """ + return none_of(values.begin(), values.end(), is_odd) + + +def count_ones(vector[int] values): + """ + Test count. + + >>> count_ones([1, 2, 1, 2]) + 2 + """ + return count(values.begin(), values.end(), 1) + + +cdef void add_one(int &i): + # https://github.com/cython/cython/issues/1863 + (&i)[0] += 1 + + +def increment_ints(vector[int] values): + """ + Test for_each. + + >>> increment_ints([3, 4, 2, 8, 15, 267]) + [4, 5, 3, 9, 16, 268] + """ + for_each(values.begin(), values.end(), &add_one) + return values + + +def count_odd(vector[int] values): + """ + Test count_if with is_odd predicate. + + >>> count_odd([1, 2, 3, 4]) + 2 + >>> count_odd([2, 4, 6, 8]) + 0 + """ + return count_if(values.begin(), values.end(), is_odd) + + +def mirror_ends(string data): + """ + Test mismatch using cppreference example. + + This program determines the longest substring that is simultaneously found at the very beginning of the given string + and at the very end of it, in reverse order (possibly overlapping). + + >>> print(mirror_ends(b'abXYZba').decode('ascii')) + ab + >>> print(mirror_ends(b'abca').decode('ascii')) + a + >>> print(mirror_ends(b'aba').decode('ascii')) + aba + """ + return string(data.begin(), mismatch(data.begin(), data.end(), data.rbegin()).first) + + +def mismatch_ints(vector[int] values1, vector[int] values2): + """ + Test mismatch(first1, last1, first2). + + >>> mismatch_ints([1, 2, 3], [1, 2, 3]) + >>> mismatch_ints([1, 2], [1, 2, 3]) + >>> mismatch_ints([1, 3], [1, 2, 3]) + (3, 2) + """ + result = mismatch(values1.begin(), values1.end(), values2.begin()) + if result.first == values1.end(): + return + return deref(result.first), deref(result.second) + + +def is_int_in(vector[int] values, int target): + """ + Test find. + + >>> is_int_in(range(5), 3) + True + >>> is_int_in(range(5), 10) + False + """ + return find(values.begin(), values.end(), target) != values.end() + + +def find_odd(vector[int] values): + """ + Test find_if using is_odd predicate. + + >>> find_odd([2, 3, 4]) + 3 + >>> find_odd([2, 4, 6]) + """ + result = find_if(values.begin(), values.end(), is_odd) + if result != values.end(): + return deref(result) + else: + return None + + +def find_even(vector[int] values): + """ + Test find_if_not using is_odd predicate. + + >>> find_even([3, 4, 5]) + 4 + >>> find_even([1, 3, 5]) + """ + result = find_if_not(values.begin(), values.end(), is_odd) + if result != values.end(): + return deref(result) + else: + return None + + +def find_last_int_sequence(vector[int] values, vector[int] target): + """ + Test find_end. + + >>> find_last_int_sequence([1, 2, 3, 1, 2, 3], [2, 3]) + 4 + >>> find_last_int_sequence([1, 2, 3], [4, 5]) + """ + result = find_end(values.begin(), values.end(), target.begin(), target.end()) + if result != values.end(): + return distance(values.begin(), result) + else: + return None + + +cdef bool is_equal(int lhs, int rhs): + return lhs == rhs + + +def find_last_int_sequence2(vector[int] values, vector[int] target): + """ + Test find_end (using is_equal predicate). + + >>> find_last_int_sequence2([1, 2, 3, 1, 2, 3], [2, 3]) + 4 + >>> find_last_int_sequence2([1, 2, 3], [4, 5]) + """ + result = find_end(values.begin(), values.end(), target.begin(), target.end(), &is_equal) + if result != values.end(): + return distance(values.begin(), result) + else: + return None + + +def find_first_int_in_set(values, target): + """ + Test find_first_of. + + >>> find_first_int_in_set([1, 2, 3, 4, 5], [3, 5]) + 2 + >>> find_first_int_in_set([1, 2, 3], [4, 5]) + """ + cdef vector[int] v = values + cdef vector[int] t = target + result = find_first_of(v.begin(), v.end(), t.begin(), t.end()) + if result != v.end(): + return distance(v.begin(), result) + else: + return None + + +def find_first_int_in_set2(vector[int] values, vector[int] target): + """ + Test find_first_of with is_equal predicate. + + >>> find_first_int_in_set2([1, 2, 3, 4, 5], [3, 5]) + 2 + >>> find_first_int_in_set2([1, 2, 3], [4, 5]) + """ + result = find_first_of(values.begin(), values.end(), target.begin(), target.end(), is_equal) + if result != values.end(): + return distance(values.begin(), result) + else: + return None + + +def find_adjacent_int(vector[int] values): + """ + Test adjacent_find. + + >>> find_adjacent_int([0, 1, 2, 3, 40, 40, 41, 41, 5]) + 4 + >>> find_adjacent_int(range(5)) + """ + result = adjacent_find(values.begin(), values.end()) + if result != values.end(): + return distance(values.begin(), result) + else: + return None + + +def find_adjacent_int2(vector[int] values): + """ + Test find_adjacent with is_equal predicate. + + >>> find_adjacent_int2([0, 1, 2, 3, 40, 40, 41, 41, 5]) + 4 + >>> find_adjacent_int2(range(5)) + """ + result = adjacent_find(values.begin(), values.end(), is_equal) + if result != values.end(): + return distance(values.begin(), result) + else: + return None + + +def in_quote(string quote, string word): + """ + Test search using cppreference example. + + >>> in_quote(b"why waste time learning, when ignorance is instantaneous?", b"learning") + True + >>> in_quote(b"why waste time learning, when ignorance is instantaneous?", b"lemming") + False + """ + return search(quote.begin(), quote.end(), word.begin(), word.end()) != quote.end() + + +def in_quote2(string quote, string word): + """ + Test search using cppreference example (with is_equal predicate). + + >>> in_quote2(b"why waste time learning, when ignorance is instantaneous?", b"learning") + True + >>> in_quote2(b"why waste time learning, when ignorance is instantaneous?", b"lemming") + False + """ + return search(quote.begin(), quote.end(), word.begin(), word.end(), &is_equal) != quote.end() + + +def consecutive_values(string c, int count, char v): + """ + Test search_n using cppreference example (without std::begin and std::end). + + >>> consecutive_values(b"1001010100010101001010101", 4, ord("0")) + False + >>> consecutive_values(b"1001010100010101001010101", 3, ord("0")) + True + """ + return search_n(c.begin(), c.end(), count, v) != c.end() + + +def consecutive_values2(string c, int count, char v): + """ + Test search_n using cppreference example (with is_equal predicate). + + >>> consecutive_values2(b"1001010100010101001010101", 4, ord("0")) + False + >>> consecutive_values2(b"1001010100010101001010101", 3, ord("0")) + True + """ + return search_n(c.begin(), c.end(), count, v, &is_equal) != c.end() diff --git a/tests/run/cpp_stl_algo_partitioning_ops.pyx b/tests/run/cpp_stl_algo_partitioning_ops.pyx new file mode 100644 index 000000000..1de80d84b --- /dev/null +++ b/tests/run/cpp_stl_algo_partitioning_ops.pyx @@ -0,0 +1,90 @@ +# mode: run +# tag: cpp, werror, cpp11, no-cpp-locals + +from __future__ import print_function + +from libcpp cimport bool +from libcpp.algorithm cimport is_partitioned, partition, partition_copy, stable_partition, partition_point +from libcpp.algorithm cimport for_each, copy, reverse +from libcpp.iterator cimport back_inserter +from libcpp.vector cimport vector + + +cdef bool is_even(int i): + return i % 2 == 0 + + +def test_is_partitioned(): + """ + >>> test_is_partitioned() + False + True + False + """ + cdef vector[int] values = range(10) + print(is_partitioned(values.begin(), values.end(), is_even)) + + partition(values.begin(), values.end(), &is_even) + print(is_partitioned(values.begin(), values.end(), is_even)) + + reverse(values.begin(), values.end()) + print(is_partitioned(values.begin(), values.end(), is_even)) + + +cdef int print_int(int v) except -1: + print(v, end=" ") + + +def print_partition(vector[int] values): + """ + Test partition. + + >> print_partition(range(10)) + 0 8 2 6 4 * 5 3 7 1 9 + """ + it = partition(values.begin(), values.end(), &is_even) + for_each(values.begin(), it, &print_int) + print("*", end=" ") + for_each(it, values.end(), &print_int) + print() + + +def partition_ints_even(vector[int] values): + """ + Test partition_copy. + + >>> partition_ints_even(range(10)) + ([0, 2, 4, 6, 8], [1, 3, 5, 7, 9]) + """ + cdef vector[int] even_values, odd_values + partition_copy(values.begin(), values.end(), back_inserter(even_values), back_inserter(odd_values), &is_even) + return even_values, odd_values + + +cdef bool is_positive(int v): + return v > 0 + + +def partition_ints_positive(vector[int] values): + """ + Test stable_partition. + + >>> partition_ints_positive([0, 0, 3, 0, 2, 4, 5, 0, 7]) + [3, 2, 4, 5, 7, 0, 0, 0, 0] + """ + stable_partition(values.begin(), values.end(), &is_positive) + return values + + +def partition_point_ints_even(vector[int] values): + """ + Test partition_point. + + >>> partition_point_ints_even([0, 8, 2, 6, 4, 5, 3, 7, 1, 9]) + ([0, 8, 2, 6, 4], [5, 3, 7, 1, 9]) + """ + it = partition_point(values.begin(), values.end(), is_even) + cdef vector[int] even_values, odd_values + copy(values.begin(), it, back_inserter(even_values)) + copy(it, values.end(), back_inserter(odd_values)) + return even_values, odd_values diff --git a/tests/run/cpp_stl_algo_sorting_ops.pyx b/tests/run/cpp_stl_algo_sorting_ops.pyx new file mode 100644 index 000000000..ac8d1b586 --- /dev/null +++ b/tests/run/cpp_stl_algo_sorting_ops.pyx @@ -0,0 +1,187 @@ +# mode: run +# tag: cpp, werror, cpp11, no-cpp-locals + +from __future__ import print_function + +from libcpp cimport bool +from libcpp.algorithm cimport is_sorted, is_sorted_until, sort, partial_sort, partial_sort_copy, stable_sort +from libcpp.algorithm cimport nth_element +from libcpp.functional cimport greater +from libcpp.iterator cimport distance +from libcpp.string cimport string +from libcpp.vector cimport vector + + +def is_sorted_ints(vector[int] values): + """ + Test is_sorted. + + >>> is_sorted_ints([3, 1, 4, 1, 5]) + False + >>> is_sorted_ints([1, 1, 3, 4, 5]) + True + """ + return is_sorted(values.begin(), values.end()) + + +def initial_sorted_elements(vector[int] values): + """ + Test is_sorted_until. + + >>> initial_sorted_elements([4, 1, 9, 5, 1, 3]) + 1 + >>> initial_sorted_elements([4, 5, 9, 3, 1, 1]) + 3 + >>> initial_sorted_elements([9, 3, 1, 4, 5, 1]) + 1 + >>> initial_sorted_elements([1, 3, 5, 4, 1, 9]) + 3 + >>> initial_sorted_elements([5, 9, 1, 1, 3, 4]) + 2 + >>> initial_sorted_elements([4, 9, 1, 5, 1, 3]) + 2 + >>> initial_sorted_elements([1, 1, 4, 9, 5, 3]) + 4 + """ + sorted_end = is_sorted_until(values.begin(), values.end()) + return distance(values.begin(), sorted_end) + + +def sort_ints(vector[int] values): + """Test sort using the default operator<. + + >>> sort_ints([5, 7, 4, 2, 8, 6, 1, 9, 0, 3]) + [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + """ + sort(values.begin(), values.end()) + return values + + +def sort_ints_reverse(vector[int] values): + """Test sort using a standard library comparison function object. + + >>> sort_ints_reverse([5, 7, 4, 2, 8, 6, 1, 9, 0, 3]) + [9, 8, 7, 6, 5, 4, 3, 2, 1, 0] + """ + sort(values.begin(), values.end(), greater[int]()) + return values + + +def partial_sort_ints(vector[int] values, int k): + """ + Test partial_sort using the default operator<. + + >>> partial_sort_ints([4, 2, 3, 1, 5], 2)[:2] + [1, 2] + """ + partial_sort(values.begin(), values.begin() + k, values.end()) + return values + + +def partial_sort_ints_reverse(vector[int] values, int k): + """ + Test partial_sort using a standard library comparison function object. + + >>> partial_sort_ints_reverse([4, 2, 3, 1, 5], 2)[:2] + [5, 4] + """ + partial_sort(values.begin(), values.begin() + k, values.end(), greater[int]()) + return values + + +def partial_sort_ints2(vector[int] values, int k): + """ + Test partial_sort_copy using the default operator<. + + >>> partial_sort_ints2([4, 2, 3, 1, 5], 2) + [1, 2] + """ + output = vector[int](2) + partial_sort_copy(values.begin(), values.end(), output.begin(), output.end()) + return output + + +def partial_sort_ints_reverse2(vector[int] values, int k): + """ + Test partial_sort_copy using a standard library comparison function object. + + >>> partial_sort_ints_reverse2([4, 2, 3, 1, 5], 2) + [5, 4] + """ + output = vector[int](2) + partial_sort_copy(values.begin(), values.end(), output.begin(), output.end(), greater[int]()) + return output + + +cdef extern from *: + """ + struct Employee + { + Employee() = default; + Employee(int age, std::string name): age(age), name(name) {} + int age; + std::string name; // Does not participate in comparisons + }; + + bool operator<(const Employee& lhs, const Employee& rhs) + { + return lhs.age < rhs.age; + } + """ + cppclass Employee: + Employee() + Employee(int, string) + int age + string name + +cdef bool Employee_greater(const Employee& lhs, const Employee& rhs): + return lhs.age > rhs.age + +def test_stable_sort(): + """ + Test stable_sort using cppreference example. + + >>> test_stable_sort() + 32, Arthur + 108, Zaphod + 108, Ford + 108, Zaphod + 108, Ford + 32, Arthur + """ + cdef vector[Employee] employees + employees.push_back(Employee(108, <string>b"Zaphod")) + employees.push_back(Employee(32, <string>b"Arthur")) + employees.push_back(Employee(108, <string>b"Ford")) + + stable_sort(employees.begin(), employees.end()) + + for e in employees: + print("%s, %s" % (e.age, <str>(e.name).decode("ascii"))) + + stable_sort(employees.begin(), employees.end(), &Employee_greater) + + for e in employees: + print("%s, %s" % (e.age, <str>(e.name).decode("ascii"))) + + +def second_smallest(vector[int] values): + """ + Test nth_element using the default operator<. + + >>> second_smallest([5, 6, 4, 3, 2, 6, 7, 9, 3]) + 3 + """ + nth_element(values.begin(), values.begin() + 1, values.end()) + return values[1] + + +def second_largest(vector[int] values): + """ + Test nth_element using a standard library comparison function object. + + >>> second_largest([5, 6, 4, 3, 2, 6, 7, 9, 3]) + 7 + """ + nth_element(values.begin(), values.begin() + 1, values.end(), greater[int]()) + return values[1] diff --git a/tests/run/cpp_stl_atomic.pyx b/tests/run/cpp_stl_atomic.pyx new file mode 100644 index 000000000..ba187a455 --- /dev/null +++ b/tests/run/cpp_stl_atomic.pyx @@ -0,0 +1,85 @@ +# mode: run +# tag: cpp, cpp11, werror, no-cpp-locals + +from cython.operator cimport preincrement as incr, dereference as deref +from libc.stdint cimport * + +from libcpp.atomic cimport atomic + +def int_test(int x): + """ + >>> int_test(55) + 3 + >>> int_test(42) + 3 + >>> int_test(100000) + 3 + """ + atom = new atomic[int](x) + try: + atom.store(0) + incr(deref(atom)) + incr(deref(atom)) + incr(deref(atom)) + return atom.load() + finally: + del atom + +ctypedef atomic[int32_t] atomint32_t + +def typedef_test(int x): + """ + >>> typedef_test(55) + 3 + >>> typedef_test(42) + 3 + >>> typedef_test(100000) + 3 + """ + atom = new atomint32_t(x) + try: + atom.store(0) + incr(deref(atom)) + incr(deref(atom)) + incr(deref(atom)) + return atom.load() + finally: + del atom + +def stack_allocation_test(int x): + """ + >>> stack_allocation_test(55) + 3 + >>> stack_allocation_test(42) + 3 + >>> stack_allocation_test(100000) + 3 + """ + cdef atomint32_t atom + atom.store(x) + try: + atom.store(0) + incr(atom) + incr(atom) + incr(atom) + return atom.load() + finally: + pass + +def nogil_int_test(int x): + """ + >>> nogil_int_test(55) + 55 + >>> nogil_int_test(42) + 42 + >>> nogil_int_test(100000) + 100000 + """ + with nogil: + atom = new atomic[int](0) + try: + with nogil: + atom.store(x) + return atom.load() + finally: + del atom diff --git a/tests/run/cpp_stl_conversion.pyx b/tests/run/cpp_stl_conversion.pyx index 3ea404d25..07d56d68b 100644 --- a/tests/run/cpp_stl_conversion.pyx +++ b/tests/run/cpp_stl_conversion.pyx @@ -2,6 +2,8 @@ # tag: cpp, werror, cpp11 import sys +from collections import defaultdict + from libcpp.map cimport map from libcpp.unordered_map cimport unordered_map from libcpp.set cimport set as cpp_set @@ -97,6 +99,23 @@ def test_int_vector(o): cdef vector[int] v = o return v +cdef vector[int] takes_vector(vector[int] x): + return x + +def test_list_literal_to_vector(): + """ + >>> test_list_literal_to_vector() + [1, 2, 3] + """ + return takes_vector([1, 2, 3]) + +def test_tuple_literal_to_vector(): + """ + >>> test_tuple_literal_to_vector() + [1, 2, 3] + """ + return takes_vector((1, 2, 3)) + def test_string_vector(s): """ >>> list(map(normalize, test_string_vector('ab cd ef gh'.encode('ascii')))) @@ -143,10 +162,12 @@ def test_typedef_vector(o): Traceback (most recent call last): ... OverflowError: ... + + "TypeError: an integer is required" on CPython >>> test_typedef_vector([1, 2, None]) #doctest: +ELLIPSIS Traceback (most recent call last): ... - TypeError: an integer is required + TypeError: ...int... """ cdef vector[my_int] v = o return v @@ -193,22 +214,36 @@ def test_unordered_set(o): def test_map(o): """ - >>> test_map({1: 1.0, 2: 0.5, 3: 0.25}) + >>> d = {1: 1.0, 2: 0.5, 3: 0.25} + >>> test_map(d) + {1: 1.0, 2: 0.5, 3: 0.25} + >>> dd = defaultdict(float) + >>> dd.update(d) + >>> test_map(dd) # try with a non-dict {1: 1.0, 2: 0.5, 3: 0.25} """ cdef map[int, double] m = o return m def test_unordered_map(o): - """ - >>> d = test_map({1: 1.0, 2: 0.5, 3: 0.25}) - >>> sorted(d) - [1, 2, 3] - >>> (d[1], d[2], d[3]) - (1.0, 0.5, 0.25) - """ - cdef unordered_map[int, double] m = o - return m + """ + >>> d = {1: 1.0, 2: 0.5, 3: 0.25} + >>> m = test_map(d) + >>> sorted(m) + [1, 2, 3] + >>> (m[1], m[2], m[3]) + (1.0, 0.5, 0.25) + + >>> dd = defaultdict(float) + >>> dd.update(d) + >>> m = test_map(dd) + >>> sorted(m) + [1, 2, 3] + >>> (m[1], m[2], m[3]) + (1.0, 0.5, 0.25) + """ + cdef unordered_map[int, double] m = o + return m def test_nested(o): """ @@ -239,3 +274,14 @@ def test_enum_map(o): """ cdef map[Color, Color] m = o return m + +cdef map[unsigned int, unsigned int] takes_map(map[unsigned int, unsigned int] m): + return m + +def test_dict_literal_to_map(): + """ + >>> test_dict_literal_to_map() + {1: 1} + """ + return takes_map({1: 1}) # https://github.com/cython/cython/pull/4228 + # DictNode could not be converted directly diff --git a/tests/run/cpp_stl_cpp11.pyx b/tests/run/cpp_stl_cpp11.pyx index f4fa4d360..be0b72e33 100644 --- a/tests/run/cpp_stl_cpp11.pyx +++ b/tests/run/cpp_stl_cpp11.pyx @@ -1,5 +1,5 @@ # mode: run -# tag: cpp, werror, cpp11 +# tag: cpp, werror, cpp11, no-cpp-locals import sys from libcpp.unordered_map cimport unordered_map diff --git a/tests/run/cpp_stl_forward_list.pyx b/tests/run/cpp_stl_forward_list.pyx index 8ca081c4e..817e4d710 100644 --- a/tests/run/cpp_stl_forward_list.pyx +++ b/tests/run/cpp_stl_forward_list.pyx @@ -1,5 +1,5 @@ # mode: run -# tag: cpp, werror, cpp11 +# tag: cpp, werror, cpp11, no-cpp-locals from cython.operator cimport dereference as deref from cython.operator cimport preincrement as incr diff --git a/tests/run/cpp_stl_list.pyx b/tests/run/cpp_stl_list.pyx index 59367e2f5..6a3c60ee5 100644 --- a/tests/run/cpp_stl_list.pyx +++ b/tests/run/cpp_stl_list.pyx @@ -1,5 +1,5 @@ # mode: run -# tag: cpp, werror +# tag: cpp, werror, no-cpp-locals from cython.operator cimport dereference as deref from cython.operator cimport preincrement as incr diff --git a/tests/run/cpp_stl_multiset.pyx b/tests/run/cpp_stl_multiset.pyx new file mode 100644 index 000000000..26f3bbe61 --- /dev/null +++ b/tests/run/cpp_stl_multiset.pyx @@ -0,0 +1,106 @@ +# mode: run +# tag: cpp, cpp11 + +# cython: language_level=3 + +from libcpp.set cimport multiset +from libcpp.unordered_set cimport unordered_multiset + +def test_multiset_insert(vals): + """ + >>> test_multiset_insert([1,2,2,3, -1]) + [-1, 1, 2, 2, 3] + """ + cdef multiset[int] ms = multiset[int]() + for v in vals: + ms.insert(v) + return [ item for item in ms ] + +def test_multiset_count(vals, to_find): + """ + >>> test_multiset_count([1,2,2,3, -1], 1) + 1 + >>> test_multiset_count([1,2,2,3, -1], 2) + 2 + """ + cdef multiset[int] ms = multiset[int]() + for v in vals: + ms.insert(v) + return ms.count(to_find) + +def test_multiset_erase(vals, int to_remove): + """ + >>> test_multiset_erase([1,2,2,3, -1], 1) + [-1, 2, 2, 3] + >>> test_multiset_erase([1,2,2,3, -1], 2) # removes both copies of 2 + [-1, 1, 3] + """ + cdef multiset[int] ms = multiset[int]() + for v in vals: + ms.insert(v) + ms.erase(to_remove) + return [ item for item in ms ] + +def test_multiset_find_erase(vals, to_remove): + """ + >>> test_multiset_find_erase([1,2,2,3, -1], 1) + [-1, 2, 2, 3] + >>> test_multiset_find_erase([1,2,2,3, -1], 2) # removes a single copy of 2 + [-1, 1, 2, 3] + """ + cdef multiset[int] ms = multiset[int]() + for v in vals: + ms.insert(v) + it = ms.find(to_remove) + ms.erase(it) + return [ item for item in ms ] + + +def test_unordered_multiset_insert(vals): + """ + >>> test_unordered_multiset_insert([1,2,2,3, -1]) + [-1, 1, 2, 2, 3] + """ + cdef unordered_multiset[int] ms = unordered_multiset[int]() + for v in vals: + ms.insert(v) + return sorted([ item for item in ms ]) + +def test_unordered_multiset_count(vals, to_find): + """ + >>> test_unordered_multiset_count([1,2,2,3, -1], 1) + 1 + >>> test_unordered_multiset_count([1,2,2,3, -1], 2) + 2 + """ + cdef unordered_multiset[int] ms = unordered_multiset[int]() + for v in vals: + ms.insert(v) + return ms.count(to_find) + +def test_unordered_multiset_erase(vals, int to_remove): + """ + >>> test_unordered_multiset_erase([1,2,2,3, -1], 1) + [-1, 2, 2, 3] + >>> test_unordered_multiset_erase([1,2,2,3, -1], 2) # removes both copies of 2 + [-1, 1, 3] + """ + cdef unordered_multiset[int] ms = unordered_multiset[int]() + for v in vals: + ms.insert(v) + ms.erase(to_remove) + return sorted([ item for item in ms ]) + +def test_unordered_multiset_find_erase(vals, to_remove): + """ + >>> test_unordered_multiset_find_erase([1,2,2,3, -1], 1) + [-1, 2, 2, 3] + >>> test_unordered_multiset_find_erase([1,2,2,3, -1], 2) # removes a single copy of 2 + [-1, 1, 2, 3] + """ + cdef unordered_multiset[int] ms = unordered_multiset[int]() + for v in vals: + ms.insert(v) + it = ms.find(to_remove) + ms.erase(it) + return sorted([ item for item in ms ]) diff --git a/tests/run/cpp_stl_numeric_ops.pyx b/tests/run/cpp_stl_numeric_ops.pyx new file mode 100644 index 000000000..52f3c58ec --- /dev/null +++ b/tests/run/cpp_stl_numeric_ops.pyx @@ -0,0 +1,147 @@ +# mode: run +# tag: cpp, werror, cpp11 + +from libcpp.numeric cimport inner_product, iota, accumulate, adjacent_difference, partial_sum +from libcpp.vector cimport vector +from libcpp cimport bool + +# Subtracts two integers. +cdef int subtract_integers(int lhs, int rhs): + return lhs - rhs + +# Adds two integers. +cdef int add_integers(int lhs, int rhs): + return lhs + rhs + +# Multiplies two integers. +cdef int multiply_integers(int lhs, int rhs): + return lhs * rhs + +# Determines equality for two integers. +# If lhs == rhs, returns true. Returns false otherwise. +cdef bool is_equal(int lhs, int rhs): + return lhs == rhs + +def test_inner_product(vector[int] v1, vector[int] v2, int init): + """ + Test inner_product with integer values. + >>> test_inner_product([1, 2, 3], [1, 2, 3], 1) + 15 + """ + return inner_product(v1.begin(), v1.end(), v2.begin(), init) + + +def test_inner_product_with_zero(vector[int] v1, vector[int] v2, int init): + """ + Test inner_product with a zero value in the container. + >>> test_inner_product_with_zero([1, 2, 0], [1, 1, 1], 0) + 3 + """ + return inner_product(v1.begin(), v1.end(), v2.begin(), init) + +def test_inner_product_with_bin_op(vector[int] v1, vector[int] v2, int init): + """ + Test inner_product with two binary operations. In this case, + Looks at number of pairwise matches between v1 and v2. + [5, 1, 2, 3, 4] + [5, 4, 2, 3, 1] + There are 3 matches (5, 2, 3). So, 1 + 1 + 1 = 3. + + >>> test_inner_product_with_bin_op([5, 1, 2, 3, 4], [5, 4, 2, 3, 1], 0) + 3 + """ + return inner_product(v1.begin(), v1.end(), v2.begin(), init, add_integers, is_equal) + +def test_iota(vector[int] v, int value): + """ + Test iota with beginning value of 0. + >>> test_iota(range(6), 0) + [0, 1, 2, 3, 4, 5] + """ + iota(v.begin(), v.end(), value) + return v + +def test_iota_negative_init(vector[int] v, int value): + """ + Test iota with a negative beginning value. + >>> test_iota_negative_init(range(7), -4) + [-4, -3, -2, -1, 0, 1, 2] + """ + iota(v.begin(), v.end(), value) + return v + +def test_accumulate(vector[int] v, int init): + """ + Test accumulate. + 0 + 1 = 1 + 1 + 2 = 3 + 3 + 3 = 6 + >>> test_accumulate([1, 2, 3], 0) + 6 + """ + return accumulate(v.begin(), v.end(), init) + +def test_accumulate_with_subtraction(vector[int] v, int init): + """ + Test accumulate with subtraction. Note that accumulate is a fold-left operation. + 0 - 1 = -1 + -1 - 2 = -3 + -3 - 3 = -6 + >>> test_accumulate_with_subtraction([1, 2, 3], 0) + -6 + """ + return accumulate(v.begin(), v.end(), init, subtract_integers) + +def test_adjacent_difference(vector[int] v): + """ + Test adjacent_difference with integer values. + 2 - 0, -> 2 + 4 - 2, -> 2 + 6 - 4, -> 2 + 8 - 6, -> 2 + 10 - 8, -> 2 + 12 - 10 -> 2 + >>> test_adjacent_difference([2, 4, 6, 8, 10, 12]) + [2, 2, 2, 2, 2, 2] + """ + adjacent_difference(v.begin(), v.end(), v.begin()) + return v + +def test_adjacent_difference_with_bin_op(vector[int] v): + """ + Test adjacent_difference with a binary operation. + 0 + 1 -> 1 + 1 + 2 -> 3 + 2 + 4 -> 6 + 4 + 5 -> 9 + 5 + 6 -> 11 + >>> test_adjacent_difference_with_bin_op([1, 2, 4, 5, 6]) + [1, 3, 6, 9, 11] + """ + adjacent_difference(v.begin(), v.end(), v.begin(), add_integers) + return v + +def test_partial_sum(vector[int] v): + """ + Test partial_sum with integer values. + 2 + 0 -> 2 + 2 + 2 -> 4 + 4 + 2 -> 6 + 6 + 2 -> 8 + 8 + 2 -> 10 + 10 + 2 -> 12 + >>> test_partial_sum([2, 2, 2, 2, 2, 2]) + [2, 4, 6, 8, 10, 12] + """ + partial_sum(v.begin(), v.end(), v.begin()) + return v + +def test_partial_sum_with_bin_op(vector[int] v): + """ + Test partial_sum with a binary operation. + Using multiply_integers, partial_sum will calculate the first 5 powers of 2. + >>> test_partial_sum_with_bin_op([2, 2, 2, 2, 2]) + [2, 4, 8, 16, 32] + """ + partial_sum(v.begin(), v.end(), v.begin(), multiply_integers) + return v diff --git a/tests/run/cpp_stl_string.pyx b/tests/run/cpp_stl_string.pyx index 521085bfc..ff7757b08 100644 --- a/tests/run/cpp_stl_string.pyx +++ b/tests/run/cpp_stl_string.pyx @@ -377,6 +377,34 @@ def test_iteration(string s): """ return [c for c in s] +def test_to_string(x): + """ + >>> print(test_to_string(5)) + si=5 sl=5 ss=5 sss=5 + >>> print(test_to_string(-5)) + si=-5 sl=-5 ss=5 sss=-5 + """ + si = to_string(<int>x).decode('ascii') + sl = to_string(<long>x).decode('ascii') + ss = to_string(<size_t>abs(x)).decode('ascii') + sss = to_string(<ssize_t>x).decode('ascii') + return f"si={si} sl={sl} ss={ss} sss={sss}" + +def test_stoi(char *a): + """ + >>> test_stoi(b'5') + 5 + """ + cdef string s = string(a) + return stoi(s) + +def test_stof(char *a): + """ + >>> test_stof(b'5.5') + 5.5 + """ + cdef string s = string(a) + return stof(s) def test_to_string(x): """ diff --git a/tests/run/cpp_stl_vector.pyx b/tests/run/cpp_stl_vector.pyx index 5c943e423..c42cb96b8 100644 --- a/tests/run/cpp_stl_vector.pyx +++ b/tests/run/cpp_stl_vector.pyx @@ -1,5 +1,5 @@ # mode: run -# tag: cpp, werror +# tag: cpp, werror, no-cpp-locals from cython.operator cimport dereference as d from cython.operator cimport preincrement as incr diff --git a/tests/run/cpp_template_functions.pyx b/tests/run/cpp_template_functions.pyx index dce882879..19fa78413 100644 --- a/tests/run/cpp_template_functions.pyx +++ b/tests/run/cpp_template_functions.pyx @@ -1,5 +1,5 @@ # mode: run -# tag: cpp, warnings +# tag: cpp, warnings, no-cpp-locals cimport cython from libcpp.pair cimport pair diff --git a/tests/run/cpp_template_ref_args.pyx b/tests/run/cpp_template_ref_args.pyx index c98c077ef..754122454 100644 --- a/tests/run/cpp_template_ref_args.pyx +++ b/tests/run/cpp_template_ref_args.pyx @@ -1,4 +1,4 @@ -# tag: cpp +# tag: cpp, no-cpp-locals from libcpp.vector cimport vector diff --git a/tests/run/cpp_template_subclasses.pyx b/tests/run/cpp_template_subclasses.pyx index 0ebcd4c6e..3e7f3b506 100644 --- a/tests/run/cpp_template_subclasses.pyx +++ b/tests/run/cpp_template_subclasses.pyx @@ -1,5 +1,5 @@ # mode: run -# tag: cpp, werror +# tag: cpp, werror, no-cpp-locals from cython.operator import dereference as deref from libcpp.pair cimport pair diff --git a/tests/run/cpp_type_inference.pyx b/tests/run/cpp_type_inference.pyx index 95631ebfb..f75836a42 100644 --- a/tests/run/cpp_type_inference.pyx +++ b/tests/run/cpp_type_inference.pyx @@ -12,6 +12,11 @@ cdef extern from "shapes.h" namespace "shapes": cdef cppclass Square(Shape): Square(int) + cdef cppclass Empty(Shape): + Empty() + + cdef Empty make_Empty "shapes::Empty"() + from cython cimport typeof from cython.operator cimport dereference as d @@ -48,3 +53,35 @@ def test_derived_types(int size, bint round): ptr = new Square(size) print typeof(ptr) del ptr + +def test_stack_allocated(bint b=True): + """ + >>> test_stack_allocated() + """ + e1 = Empty() + e2 = Empty() + e = e1 if b else e2 + assert typeof(e1) == "Empty", typeof(e1) + assert typeof(e2) == "Empty", typeof(e2) + assert typeof(e) == "Empty", typeof(e) + +cdef extern from *: + """ + template <typename T> + struct MyTemplate {}; + """ + cdef cppclass MyTemplate[T]: + MyTemplate() + +def test_template_types(): + """ + >>> test_template_types() + """ + t_stack = MyTemplate[int]() + assert typeof(t_stack) == "MyTemplate[int]", typeof(t_stack) + + t_ptr = new MyTemplate[double]() + try: + assert typeof(t_ptr) == "MyTemplate[double] *", typeof(t_ptr) + finally: + del t_ptr diff --git a/tests/run/crashT245.pyx b/tests/run/crashT245.pyx index 862deca37..0ef9cb447 100644 --- a/tests/run/crashT245.pyx +++ b/tests/run/crashT245.pyx @@ -1,4 +1,4 @@ -# ticket: 245 +# ticket: t245 cimport crashT245_pxd diff --git a/tests/run/ctuple.pyx b/tests/run/ctuple.pyx index 235265bb1..4053df42b 100644 --- a/tests/run/ctuple.pyx +++ b/tests/run/ctuple.pyx @@ -142,6 +142,17 @@ cpdef (int, double) cpdef_ctuple_return_type(int x, double y): return x, y +def cast_to_ctuple(*o): + """ + >>> cast_to_ctuple(1, 2.) + (1, 2.0) + """ + cdef int x + cdef double y + x, y = <(int, double)>o + return x, y + + @cython.infer_types(True) def test_type_inference(): """ diff --git a/tests/run/ctypedef_int_types_T333.pyx b/tests/run/ctypedef_int_types_T333.pyx index 3f1a99f69..b0a47d484 100644 --- a/tests/run/ctypedef_int_types_T333.pyx +++ b/tests/run/ctypedef_int_types_T333.pyx @@ -1,4 +1,4 @@ -# ticket: 333 +# ticket: t333 #cython: autotestdict=True # ------------------------------------------------------------------- diff --git a/tests/run/cyfunction.pyx b/tests/run/cyfunction.pyx index 3811b2694..619d11f55 100644 --- a/tests/run/cyfunction.pyx +++ b/tests/run/cyfunction.pyx @@ -271,13 +271,13 @@ def test_annotations(a: "test", b: "other" = 2, c: 123 = 4) -> "ret": >>> isinstance(test_annotations.__annotations__, dict) True >>> sorted(test_annotations.__annotations__.items()) - [('a', 'test'), ('b', 'other'), ('c', 123), ('return', 'ret')] + [('a', "'test'"), ('b', "'other'"), ('c', '123'), ('return', "'ret'")] >>> def func_b(): return 42 >>> def func_c(): return 99 >>> inner = test_annotations(1, func_b, func_c) >>> sorted(inner.__annotations__.items()) - [('return', 99), ('x', 'banana'), ('y', 42)] + [('return', 'c()'), ('x', "'banana'"), ('y', 'b()')] >>> inner.__annotations__ = {234: 567} >>> inner.__annotations__ @@ -293,14 +293,14 @@ def test_annotations(a: "test", b: "other" = 2, c: 123 = 4) -> "ret": >>> inner = test_annotations(1, func_b, func_c) >>> sorted(inner.__annotations__.items()) - [('return', 99), ('x', 'banana'), ('y', 42)] + [('return', 'c()'), ('x', "'banana'"), ('y', 'b()')] >>> inner.__annotations__['abc'] = 66 >>> sorted(inner.__annotations__.items()) - [('abc', 66), ('return', 99), ('x', 'banana'), ('y', 42)] + [('abc', 66), ('return', 'c()'), ('x', "'banana'"), ('y', 'b()')] >>> inner = test_annotations(1, func_b, func_c) >>> sorted(inner.__annotations__.items()) - [('return', 99), ('x', 'banana'), ('y', 42)] + [('return', 'c()'), ('x', "'banana'"), ('y', 'b()')] """ def inner(x: "banana", y: b()) -> c(): return x,y @@ -376,6 +376,18 @@ class TestUnboundMethod: def meth(self): pass +class TestStaticmethod(object): + """ + >>> x = TestStaticmethod() + >>> x.staticmeth(42) + 42 + >>> x.staticmeth.__get__(42)() + 42 + """ + @staticmethod + def staticmeth(arg): return arg + + cdef class TestOptimisedBuiltinMethod: """ >>> obj = TestOptimisedBuiltinMethod() @@ -391,3 +403,22 @@ cdef class TestOptimisedBuiltinMethod: def call(self, arg, obj=None): (obj or self).append(arg+1) # optimistically optimised => uses fast fallback method call + + +def do_nothing(f): + """Dummy decorator for `test_firstlineno_decorated_function`""" + return f + + +@do_nothing +@do_nothing +def test_firstlineno_decorated_function(): + """ + check that `test_firstlineno_decorated_function` starts 5 lines below `do_nothing` + + >>> test_firstlineno_decorated_function() + 5 + """ + l1 = do_nothing.__code__.co_firstlineno + l2 = test_firstlineno_decorated_function.__code__.co_firstlineno + return l2 - l1 diff --git a/tests/run/cyfunction_METH_O_GH1728.pyx b/tests/run/cyfunction_METH_O_GH1728.pyx index 621fc565f..5b3b0779b 100644 --- a/tests/run/cyfunction_METH_O_GH1728.pyx +++ b/tests/run/cyfunction_METH_O_GH1728.pyx @@ -8,9 +8,9 @@ cdef class TestMethodOneArg: def call_meth(x): """ - >>> call_meth(TestMethodOneArg()) + >>> call_meth(TestMethodOneArg()) # doctest: +ELLIPSIS Traceback (most recent call last): ... - TypeError: meth() takes exactly one argument (0 given) + TypeError: meth() takes exactly ... argument (0 given) """ return x.meth() diff --git a/tests/run/cyfunction_defaults.pyx b/tests/run/cyfunction_defaults.pyx index 2fe592a23..157b8aa7e 100644 --- a/tests/run/cyfunction_defaults.pyx +++ b/tests/run/cyfunction_defaults.pyx @@ -251,3 +251,51 @@ def test_func_default_scope_local(): return arg print i # genexprs don't leak return func + +cdef class C: + def f1(self, a, b=1, c=[]): + pass + def f2(self, a, b=1,/, c=[1]): + pass + def f3(self, a, /, b=1, *, c=[1]): + pass + cpdef f4(self, a, char*c=NULL): + pass + cpdef f5(self, a, str s = "123"): + pass + cpdef f6(self, a, int s = 4): + pass + cpdef f7(self, a, dict s = {'a':22}): + pass + cpdef f8(self, a, list s = [15]): + pass + + +def check_defaults_on_methods_for_introspection(): + """ + >>> C.f1.__defaults__ + (1, []) + >>> C.f1.__kwdefaults__ + >>> C.f2.__defaults__ + (1, [1]) + >>> C.f2.__kwdefaults__ + >>> C.f3.__defaults__ + (1,) + >>> C.f3.__kwdefaults__ + {'c': [1]} + >>> C.f4.__defaults__ + >>> C.f4.__kwdefaults__ + >>> C.f5.__defaults__ + ('123',) + >>> C.f5.__kwdefaults__ + >>> C.f6.__defaults__ + (4,) + >>> C.f6.__kwdefaults__ + >>> C.f7.__defaults__ + ({'a': 22},) + >>> C.f7.__kwdefaults__ + >>> C.f8.__defaults__ + ([15],) + >>> C.f8.__kwdefaults__ + """ + pass diff --git a/tests/run/cython3.pyx b/tests/run/cython3.pyx index c7ad56f87..9dae55301 100644 --- a/tests/run/cython3.pyx +++ b/tests/run/cython3.pyx @@ -16,6 +16,7 @@ x = u'abc' >>> except_as_deletes True + >>> no_match_does_not_touch_target True """ @@ -25,11 +26,24 @@ IS_PY2 = sys.version_info[0] < 3 if not IS_PY2: __doc__ = __doc__.replace(" u'", " '") + def locals_function(a, b=2): x = 'abc' return locals() +### "new style" classes + +class T: + """ + >>> t = T() + >>> isinstance(t, T) + True + >>> isinstance(T, type) # not a Py2 old style class! + True + """ + + ### true division def truediv(x): @@ -340,7 +354,7 @@ def unicode_literals(): def non_ascii_unprefixed_str(): - u""" + """ >>> s = non_ascii_unprefixed_str() >>> isinstance(s, bytes) False @@ -353,7 +367,7 @@ def non_ascii_unprefixed_str(): def non_ascii_raw_str(): - u""" + """ >>> s = non_ascii_raw_str() >>> isinstance(s, bytes) False @@ -366,7 +380,7 @@ def non_ascii_raw_str(): def non_ascii_raw_prefixed_unicode(): - u""" + """ >>> s = non_ascii_raw_prefixed_unicode() >>> isinstance(s, bytes) False @@ -604,15 +618,15 @@ def annotation_syntax(a: "test new test", b : "other" = 2, *args: "ARGS", **kwar >>> len(annotation_syntax.__annotations__) 5 >>> print(annotation_syntax.__annotations__['a']) - test new test + 'test new test' >>> print(annotation_syntax.__annotations__['b']) - other + 'other' >>> print(annotation_syntax.__annotations__['args']) - ARGS + 'ARGS' >>> print(annotation_syntax.__annotations__['kwargs']) - KWARGS + 'KWARGS' >>> print(annotation_syntax.__annotations__['return']) - ret + 'ret' """ result : int = a + b @@ -636,19 +650,17 @@ async def async_def_annotations(x: 'int') -> 'float': >>> ret, arg = sorted(async_def_annotations.__annotations__.items()) >>> print(ret[0]); print(ret[1]) return - float + 'float' >>> print(arg[0]); print(arg[1]) x - int + 'int' """ return float(x) -''' def repr_returns_str(x) -> str: """ >>> repr_returns_str(123) '123' """ return repr(x) -''' diff --git a/tests/run/cython3_no_unicode_literals.pyx b/tests/run/cython3_no_unicode_literals.pyx index c9690b305..99e8b3f1a 100644 --- a/tests/run/cython3_no_unicode_literals.pyx +++ b/tests/run/cython3_no_unicode_literals.pyx @@ -140,6 +140,12 @@ def str_type_is_str(): cdef str s = 'abc' return str, s +def strip_wrapped_string(s): + # PEP 563 translates an annotation of "test new test" to '"test new test"' + # but choice of string delimiters is a bit arbitrary + # this function handles that + assert s[0] == s[-1] # delimiters on either end are the same + return s[1:-1] # strip them def annotation_syntax(a: "test new test", b : "other" = 2, *args: "ARGS", **kwargs: "KWARGS") -> "ret": """ @@ -150,15 +156,15 @@ def annotation_syntax(a: "test new test", b : "other" = 2, *args: "ARGS", **kwar >>> len(annotation_syntax.__annotations__) 5 - >>> annotation_syntax.__annotations__['a'] + >>> strip_wrapped_string(annotation_syntax.__annotations__['a']) 'test new test' - >>> annotation_syntax.__annotations__['b'] + >>> strip_wrapped_string(annotation_syntax.__annotations__['b']) 'other' - >>> annotation_syntax.__annotations__['args'] + >>> strip_wrapped_string(annotation_syntax.__annotations__['args']) 'ARGS' - >>> annotation_syntax.__annotations__['kwargs'] + >>> strip_wrapped_string(annotation_syntax.__annotations__['kwargs']) 'KWARGS' - >>> annotation_syntax.__annotations__['return'] + >>> strip_wrapped_string(annotation_syntax.__annotations__['return']) 'ret' """ result : int = a + b diff --git a/tests/run/cython_includes.pyx b/tests/run/cython_includes.pyx index 5aab63f99..af91f6f9e 100644 --- a/tests/run/cython_includes.pyx +++ b/tests/run/cython_includes.pyx @@ -15,10 +15,12 @@ cimport cpython.ceval cimport cpython.cobject cimport cpython.codecs cimport cpython.complex +cimport cpython.contextvars cimport cpython.conversion cimport cpython.datetime cimport cpython.dict cimport cpython.exc +cimport cpython.fileobject cimport cpython.float cimport cpython.function cimport cpython.genobject @@ -31,6 +33,7 @@ cimport cpython.list cimport cpython.long cimport cpython.longintrepr cimport cpython.mapping +cimport cpython.marshal cimport cpython.mem cimport cpython.memoryview cimport cpython.method diff --git a/tests/run/datetime_cimport.pyx b/tests/run/datetime_cimport.pyx index e7e95206f..2fe90a397 100644 --- a/tests/run/datetime_cimport.pyx +++ b/tests/run/datetime_cimport.pyx @@ -1,10 +1,12 @@ # coding: utf-8 from cpython.datetime cimport import_datetime -from cpython.datetime cimport date, time, datetime, timedelta, PyDateTime_IMPORT +from cpython.datetime cimport date, time, datetime, timedelta, timezone_new, PyDateTime_IMPORT + +import sys import_datetime() - + def test_date(int year, int month, int day): ''' >>> val = test_date(2012, 12, 31) @@ -40,3 +42,20 @@ def test_timedelta(int days, int seconds, int useconds): ''' val = timedelta(days, seconds, useconds) return val + +def test_timezone(int days, int seconds, int useconds, str name): + ''' + >>> val = test_timezone(0, 3600, 0, 'CET') + >>> print(val) + True + ''' + try: + val = timezone_new(timedelta(days, seconds, useconds), name) + except RuntimeError: + if sys.version_info < (3, 7): + return True + else: + # It's only supposed to raise on Python < 3.7 + return False + else: + return True diff --git a/tests/run/datetime_members.pyx b/tests/run/datetime_members.pyx index 7d44019ad..1202a5ec2 100644 --- a/tests/run/datetime_members.pyx +++ b/tests/run/datetime_members.pyx @@ -1,11 +1,17 @@ +# mode: run +# tag: datetime + +import sys + from cpython.datetime cimport import_datetime from cpython.datetime cimport time_new, date_new, datetime_new, timedelta_new +from cpython.datetime cimport datetime, time from cpython.datetime cimport time_tzinfo, datetime_tzinfo -from cpython.datetime cimport time_hour, time_minute, time_second, time_microsecond +from cpython.datetime cimport time_hour, time_minute, time_second, time_microsecond, time_tzinfo, time_fold from cpython.datetime cimport date_day, date_month, date_year from cpython.datetime cimport datetime_day, datetime_month, datetime_year from cpython.datetime cimport datetime_hour, datetime_minute, datetime_second, \ - datetime_microsecond + datetime_microsecond, datetime_tzinfo, datetime_fold from cpython.datetime cimport timedelta_days, timedelta_seconds, timedelta_microseconds import_datetime() @@ -20,31 +26,41 @@ def test_date(int year, int month, int day): o.month == date_month(o), \ o.day == date_day(o) -def test_datetime(int year, int month, int day, - int hour, int minute, int second, int microsecond): +def test_datetime(int year, int month, int day, int hour, + int minute, int second, int microsecond, int fold): ''' - >>> test_datetime(2012, 12, 31, 12, 30, 59, 12345) - (True, True, True, True, True, True, True) + >>> test_datetime(2012, 12, 31, 12, 30, 59, 12345, 0) + (True, True, True, True, True, True, True, True, True) + >>> test_datetime(2012, 12, 11, 12, 30, 59, 3322, 1 if sys.version_info >= (3, 7) else 0) + (True, True, True, True, True, True, True, True, True) ''' - o = datetime_new(year, month, day, hour, minute, second, microsecond, None) + o = datetime_new( + year, month, day, hour, minute, second, microsecond, None, fold + ) return o.year == datetime_year(o), \ o.month == datetime_month(o), \ o.day == datetime_day(o), \ o.hour == datetime_hour(o), \ o.minute == datetime_minute(o), \ o.second == datetime_second(o), \ - o.microsecond == datetime_microsecond(o) + o.microsecond == datetime_microsecond(o), \ + o.tzinfo == datetime_tzinfo(o), \ + o.fold == datetime_fold(o) -def test_time(int hour, int minute, int second, int microsecond): +def test_time(int hour, int minute, int second, int microsecond, int fold): ''' - >>> test_time(12, 30, 59, 12345) - (True, True, True, True) + >>> test_time(12, 30, 59, 12345, 0) + (True, True, True, True, True, True) + >>> test_time(12, 30, 43, 5432, 1 if sys.version_info >= (3, 7) else 0) + (True, True, True, True, True, True) ''' - o = time_new(hour, minute, second, microsecond, None) + o = time_new(hour, minute, second, microsecond, None, fold) return o.hour == time_hour(o), \ o.minute == time_minute(o), \ o.second == time_second(o), \ - o.microsecond == time_microsecond(o) + o.microsecond == time_microsecond(o), \ + o.tzinfo == time_tzinfo(o), \ + o.fold == time_fold(o) def test_timedelta(int days, int seconds, int microseconds): ''' @@ -55,4 +71,3 @@ def test_timedelta(int days, int seconds, int microseconds): return o.days == timedelta_days(o), \ o.seconds == timedelta_seconds(o), \ o.microseconds == timedelta_microseconds(o) - diff --git a/tests/run/datetime_pxd.pyx b/tests/run/datetime_pxd.pyx index 64c4980db..f2dbfe144 100644 --- a/tests/run/datetime_pxd.pyx +++ b/tests/run/datetime_pxd.pyx @@ -1,10 +1,8 @@ # coding: utf-8 -#cimport cpython.datetime as cy_datetime -#from datetime import time, date, datetime, timedelta, tzinfo +cimport cython - -from cpython.datetime cimport import_datetime +from cpython.datetime cimport import_datetime, timedelta from cpython.datetime cimport time_new, date_new, datetime_new, timedelta_new from cpython.datetime cimport time_tzinfo, datetime_tzinfo from cpython.datetime cimport time_hour, time_minute, time_second, time_microsecond @@ -12,8 +10,20 @@ from cpython.datetime cimport date_day, date_month, date_year from cpython.datetime cimport datetime_day, datetime_month, datetime_year from cpython.datetime cimport datetime_hour, datetime_minute, datetime_second, \ datetime_microsecond +from cpython.datetime cimport datetime, total_seconds +from cpython.datetime cimport date_from_timestamp, get_utc, datetime_from_timestamp + +# These were added in Python 2.7.5, make sure that their backport works. +from cpython.datetime cimport ( + timedelta as timedelta_ext_type, + PyDateTime_DELTA_GET_DAYS, + PyDateTime_DELTA_GET_SECONDS, + PyDateTime_DELTA_GET_MICROSECONDS, +) import datetime as py_datetime +import time as py_time +import sys import_datetime() @@ -37,7 +47,23 @@ class FixedOffset(py_datetime.tzinfo): def dst(self, dt): return ZERO - + + +def do_timedelta_macros(timedelta_ext_type delta): + """ + >>> delta = py_datetime.timedelta(days=13, hours=7, seconds=31, microseconds=993322) + >>> (delta.days, delta.seconds, delta.microseconds) + (13, 25231, 993322) + >>> do_timedelta_macros(delta) + (13, 25231, 993322) + """ + return ( + PyDateTime_DELTA_GET_DAYS(delta), + PyDateTime_DELTA_GET_SECONDS(delta), + PyDateTime_DELTA_GET_MICROSECONDS(delta), + ) + + def do_date(int year, int month, int day): """ >>> do_date(2012, 12, 31) @@ -46,7 +72,7 @@ def do_date(int year, int month, int day): v = date_new(year, month, day) return type(v) is py_datetime.date, v.year == year, v.month == month, v.day == day -def do_datetime(int year, int month, int day, +def do_datetime(int year, int month, int day, int hour, int minute, int second, int microsecond): """ >>> do_datetime(2012, 12, 31, 12, 23, 0, 0) @@ -69,7 +95,7 @@ def do_time(int hour, int minute, int second, int microsecond): def do_time_tzinfo(int hour, int minute, int second, int microsecond, object tz): """ - >>> tz = FixedOffset(60*3, 'Moscow') + >>> tz = FixedOffset(60*3, 'Moscow') >>> do_time_tzinfo(12, 23, 0, 0, tz) (True, True, True, True, True, True) """ @@ -79,10 +105,10 @@ def do_time_tzinfo(int hour, int minute, int second, int microsecond, object tz) v.microsecond == microsecond, v.tzinfo is tz -def do_datetime_tzinfo(int year, int month, int day, +def do_datetime_tzinfo(int year, int month, int day, int hour, int minute, int second, int microsecond, object tz): """ - >>> tz = FixedOffset(60*3, 'Moscow') + >>> tz = FixedOffset(60*3, 'Moscow') >>> do_datetime_tzinfo(2012, 12, 31, 12, 23, 0, 0, tz) (True, True, True, True, True, True, True, True, True) """ @@ -90,35 +116,35 @@ def do_datetime_tzinfo(int year, int month, int day, return type(v) is py_datetime.datetime, v.year == year, v.month == month, v.day == day, \ v.hour == hour, v.minute == minute, v.second == second, \ v.microsecond == microsecond, v.tzinfo is tz - + def do_time_tzinfo2(int hour, int minute, int second, int microsecond, object tz): """ - >>> tz = FixedOffset(60*3, 'Moscow') + >>> tz = FixedOffset(60*3, 'Moscow') >>> do_time_tzinfo2(12, 23, 0, 0, tz) (True, True, True, True, True, True, True, True) """ v = time_new(hour, minute, second, microsecond, None) v1 = time_new( - time_hour(v), - time_minute(v), - time_second(v), - time_microsecond(v), + time_hour(v), + time_minute(v), + time_second(v), + time_microsecond(v), tz) r1 = (v1.tzinfo == tz) r2 = (tz == time_tzinfo(v1)) v2 = time_new( - time_hour(v1), - time_minute(v1), - time_second(v1), - time_microsecond(v1), + time_hour(v1), + time_minute(v1), + time_second(v1), + time_microsecond(v1), None) r3 = (v2.tzinfo == None) r4 = (None == time_tzinfo(v2)) v3 = time_new( - time_hour(v2), - time_minute(v2), - time_second(v2), - time_microsecond(v2), + time_hour(v2), + time_minute(v2), + time_second(v2), + time_microsecond(v2), tz) r5 = (v3.tzinfo == tz) r6 = (tz == time_tzinfo(v3)) @@ -130,44 +156,124 @@ def do_time_tzinfo2(int hour, int minute, int second, int microsecond, object tz def do_datetime_tzinfo2(int year, int month, int day, int hour, int minute, int second, int microsecond, object tz): """ - >>> tz = FixedOffset(60*3, 'Moscow') + >>> tz = FixedOffset(60*3, 'Moscow') >>> do_datetime_tzinfo2(2012, 12, 31, 12, 23, 0, 0, tz) (True, True, True, True, True, True, True, True) """ v = datetime_new(year, month, day, hour, minute, second, microsecond, None) v1 = datetime_new( - datetime_year(v), - datetime_month(v), - datetime_day(v), - datetime_hour(v), - datetime_minute(v), - datetime_second(v), - datetime_microsecond(v), + datetime_year(v), + datetime_month(v), + datetime_day(v), + datetime_hour(v), + datetime_minute(v), + datetime_second(v), + datetime_microsecond(v), tz) r1 = (v1.tzinfo == tz) r2 = (tz == datetime_tzinfo(v1)) v2 = datetime_new( - datetime_year(v1), - datetime_month(v1), - datetime_day(v1), - datetime_hour(v1), - datetime_minute(v1), - datetime_second(v1), - datetime_microsecond(v1), + datetime_year(v1), + datetime_month(v1), + datetime_day(v1), + datetime_hour(v1), + datetime_minute(v1), + datetime_second(v1), + datetime_microsecond(v1), None) r3 = (v2.tzinfo == None) r4 = (None == datetime_tzinfo(v2)) v3 = datetime_new( - datetime_year(v2), - datetime_month(v2), - datetime_day(v2), - datetime_hour(v2), - datetime_minute(v2), - datetime_second(v2), - datetime_microsecond(v2), + datetime_year(v2), + datetime_month(v2), + datetime_day(v2), + datetime_hour(v2), + datetime_minute(v2), + datetime_second(v2), + datetime_microsecond(v2), tz) r5 = (v3.tzinfo == tz) r6 = (tz == datetime_tzinfo(v3)) r7 = (v2 == v) r8 = (v3 == v1) return r1, r2, r3, r4, r5, r6, r7, r8 + + +def test_timedelta_total_seconds(): + """ + >>> cytotal, pytotal = test_timedelta_total_seconds() + >>> assert cytotal == pytotal, (cytotal, pytotal) + >>> cytotal == pytotal + True + """ + cdef: + datetime now = py_datetime.datetime.now() + timedelta td = now - py_datetime.datetime(1970, 1, 1) + + pytd = now - py_datetime.datetime(1970, 1, 1) + + return total_seconds(td), pytd.total_seconds() + + +@cython.test_fail_if_path_exists( + "//CoerceFromPyTypeNode", + "//AttributeNode", +) +def test_datetime_attrs_inlined(datetime dt): + # GH#3737 + """ + >>> from datetime import datetime + >>> py_dt = datetime(2020, 8, 18, 4, 9) + >>> dt = test_datetime_attrs_inlined(py_dt) + >>> dt[:5] + (2020, 8, 18, 4, 9) + >>> dt[5] == py_dt.second or (dt[5], py_dt.second) + True + >>> dt[6] == py_dt.microsecond or (dt[6], py_dt.microsecond) + True + """ + return ( + dt.year, + dt.month, + dt.day, + dt.hour, + dt.minute, + dt.second, + dt.microsecond, + ) + +def test_date_from_timestamp(): + """ + >>> from datetime import datetime + >>> tp, dt = test_date_from_timestamp() + >>> tp == dt + True + """ + tp = date_from_timestamp(1518185542) + dt = py_datetime.date(2018, 2, 9) + return tp, dt + +def test_get_utc(): + """ + >>> from datetime import datetime + >>> test_get_utc() + True + """ + try: + get_utc() + except RuntimeError: + if sys.version_info >= (3, 7): + raise # get_utc() is only supposed to raise on Python < 3.7 + return True + +def test_datetime_from_timestamp(): + """ + >>> from datetime import datetime + >>> tp, dt = test_datetime_from_timestamp() + >>> tp == dt + True + """ + time = py_time.time() + tp = datetime_from_timestamp(time) + dt = py_datetime.datetime.fromtimestamp(time) + return tp, dt diff --git a/tests/run/decorators_T593.pyx b/tests/run/decorators_T593.pyx index 9ac3508b8..824e5fb2a 100644 --- a/tests/run/decorators_T593.pyx +++ b/tests/run/decorators_T593.pyx @@ -1,5 +1,5 @@ # mode: run -# ticket: 593 +# ticket: t593 # tag: property, decorator """ @@ -110,8 +110,8 @@ class Base(type): class Bar(metaclass=Base): """ - >>> Bar._order - ['__module__', '__qualname__', '__doc__', 'bar'] + >>> [n for n in Bar._order if n not in {"__qualname__", "__annotations__"}] + ['__module__', '__doc__', 'bar'] """ @property def bar(self): diff --git a/tests/run/decorators_py_T593.py b/tests/run/decorators_py_T593.py index 98cffa79f..339e575de 100644 --- a/tests/run/decorators_py_T593.py +++ b/tests/run/decorators_py_T593.py @@ -1,5 +1,5 @@ # mode: run -# ticket: 593 +# ticket: t593 # tag: property, decorator """ diff --git a/tests/run/default_args_T674.py b/tests/run/default_args_T674.py index 1ca9381bc..7acf84048 100644 --- a/tests/run/default_args_T674.py +++ b/tests/run/default_args_T674.py @@ -1,5 +1,5 @@ # mode: run -# ticket: 674 +# ticket: t674 def test_inner(a): """ diff --git a/tests/run/dict_getitem.pyx b/tests/run/dict_getitem.pyx index 7a17663ba..f8e0210b8 100644 --- a/tests/run/dict_getitem.pyx +++ b/tests/run/dict_getitem.pyx @@ -146,3 +146,17 @@ def getitem_not_none(dict d not None, key): KeyError: (1, 2) """ return d[key] + + +def getitem_int_key(d, int key): + """ + >>> d = {-1: 10} + >>> getitem_int_key(d, -1) # dict + 10 + >>> class D(dict): pass + >>> d = D({-1: 10}) + >>> getitem_int_key(d, -1) # D + 10 + """ + # Based on GH-1807: must check Mapping protocol first, even for integer "index" keys. + return d[key] diff --git a/tests/run/different_package_names.srctree b/tests/run/different_package_names.srctree new file mode 100644 index 000000000..f70699012 --- /dev/null +++ b/tests/run/different_package_names.srctree @@ -0,0 +1,43 @@ +# mode: run +# tag: import,cimport,packages + +PYTHON setup.py build_ext --inplace +PYTHON -c "import pkg_py" +PYTHON -c "import pkg_py.pkg_pyx" +PYTHON -c "import pkg_py.pkg_pyx.module as module; module.run_test()" + +######## setup.py ######## + +from distutils.core import setup +from Cython.Build import cythonize + +setup( + ext_modules=cythonize('**/*.pyx', language_level=3), +) + + +######## pkg_py/__init__.py ######## + +TYPE = 'py' + +######## pkg_py/pkg_pyx/__init__.pyx ######## + +TYPE = 'pyx' + +######## pkg_py/pkg_pyx/pkg_pxd/__init__.pxd ######## + +# Not what Python would consider a package, but Cython can use it for cimports. +from libc.math cimport fabs + +######## pkg_py/pkg_pyx/module.pyx ######## + +from pkg_py.pkg_pyx.pkg_pxd cimport fabs + +def run_test(): + import pkg_py + assert pkg_py.TYPE == 'py' + + import pkg_py.pkg_pyx + assert pkg_py.pkg_pyx.TYPE == 'pyx' + + assert fabs(-2.0) == 2.0 diff --git a/tests/run/division_T384.pyx b/tests/run/division_T384.pyx index 301dc3a61..5da5475fd 100644 --- a/tests/run/division_T384.pyx +++ b/tests/run/division_T384.pyx @@ -1,4 +1,4 @@ -# ticket: 384 +# ticket: t384 """ >>> test(3) diff --git a/tests/run/duplicate_keyword_in_call.py b/tests/run/duplicate_keyword_in_call.py index e3d041d76..aba5772c3 100644 --- a/tests/run/duplicate_keyword_in_call.py +++ b/tests/run/duplicate_keyword_in_call.py @@ -1,6 +1,6 @@ # mode: run # tag: kwargs, call -# ticket: 717 +# ticket: t717 def f(**kwargs): return sorted(kwargs.items()) diff --git a/tests/run/duplicate_utilitycode_from_pyx.srctree b/tests/run/duplicate_utilitycode_from_pyx.srctree new file mode 100644 index 000000000..c539af7a6 --- /dev/null +++ b/tests/run/duplicate_utilitycode_from_pyx.srctree @@ -0,0 +1,29 @@ + + +PYTHON setup.py build_ext --inplace +PYTHON -c "import modb; modb.ClassB()" + +#################### moda.pyx #################### + +cdef class ClassA: + cdef int[2] a + +#################### modb.pyx ##################### + +from moda cimport ClassA + +cdef class ClassB(ClassA): + cdef int[2] b + +###################### setup.py ################### + +from setuptools import setup +from Cython.Build import cythonize +import Cython.Compiler.Options + +Cython.Compiler.Options.cimport_from_pyx = True + +setup( + ext_modules = cythonize(["moda.pyx", "modb.pyx"], + compiler_directives={'language_level': 3}) +) diff --git a/tests/run/dynamic_args.pyx b/tests/run/dynamic_args.pyx index 900671028..1205b06b0 100644 --- a/tests/run/dynamic_args.pyx +++ b/tests/run/dynamic_args.pyx @@ -1,5 +1,5 @@ # mode: run -# ticket: 674 +# ticket: t674 cdef class Foo: cdef str name diff --git a/tests/run/ellipsis_T488.pyx b/tests/run/ellipsis_T488.pyx index e7892dbc9..ab04b317c 100644 --- a/tests/run/ellipsis_T488.pyx +++ b/tests/run/ellipsis_T488.pyx @@ -1,4 +1,4 @@ -# ticket: 488 +# ticket: t488 """ >>> test() diff --git a/tests/run/embedsignatures.pyx b/tests/run/embedsignatures.pyx index 147f7afdd..233359127 100644 --- a/tests/run/embedsignatures.pyx +++ b/tests/run/embedsignatures.pyx @@ -1,10 +1,16 @@ #cython: embedsignature=True, annotation_typing=False +# signatures here are a little fragile - when they are +# generated during the build process gives slightly +# different (but equivalent) forms - therefore tests +# may need changing occasionally to reflect behaviour +# and this isn't necessarily a bug + import sys if sys.version_info >= (3, 4): def funcdoc(f): - if not f.__text_signature__: + if not getattr(f, "__text_signature__", None): return f.__doc__ doc = '%s%s' % (f.__name__, f.__text_signature__) if f.__doc__: @@ -83,6 +89,9 @@ __doc__ = ur""" >>> print (Ext.n.__doc__) Ext.n(self, a: int, b: float = 1.0, *args: tuple, **kwargs: dict) -> (None, True) + >>> print (Ext.o.__doc__) + Ext.o(self, a, b=1, /, c=5, *args, **kwargs) + >>> print (Ext.get_int.__doc__) Ext.get_int(self) -> int @@ -265,6 +274,9 @@ cdef class Ext: def n(self, a: int, b: float = 1.0, *args: tuple, **kwargs: dict) -> (None, True): pass + def o(self, a, b=1, /, c=5, *args, **kwargs): + pass + cpdef int get_int(self): return 0 @@ -441,16 +453,16 @@ Foo.m01(self, a: ...) -> Ellipsis Foo.m02(self, a: True, b: False) -> bool >>> print(Foo.m03.__doc__) -Foo.m03(self, a: 42, b: 42, c: -42) -> int +Foo.m03(self, a: 42, b: +42, c: -42) -> int >>> print(Foo.m04.__doc__) -Foo.m04(self, a: 3.14, b: 3.14, c: -3.14) -> float +Foo.m04(self, a: 3.14, b: +3.14, c: -3.14) -> float >>> print(Foo.m05.__doc__) Foo.m05(self, a: 1 + 2j, b: +2j, c: -2j) -> complex >>> print(Foo.m06.__doc__) -Foo.m06(self, a: 'abc', b: b'abc', c: u'abc') -> (str, bytes, unicode) +Foo.m06(self, a: 'abc', b: b'abc', c: 'abc') -> (str, bytes, unicode) >>> print(Foo.m07.__doc__) Foo.m07(self, a: [1, 2, 3], b: []) -> list @@ -459,7 +471,7 @@ Foo.m07(self, a: [1, 2, 3], b: []) -> list Foo.m08(self, a: (1, 2, 3), b: ()) -> tuple >>> print(Foo.m09.__doc__) -Foo.m09(self, a: {1, 2, 3}, b: set()) -> set +Foo.m09(self, a: {1, 2, 3}, b: {i for i in ()}) -> set >>> print(Foo.m10.__doc__) Foo.m10(self, a: {1: 1, 2: 2, 3: 3}, b: {}) -> dict diff --git a/tests/run/empty_for_loop_T208.pyx b/tests/run/empty_for_loop_T208.pyx index 88a134f25..70b3be512 100644 --- a/tests/run/empty_for_loop_T208.pyx +++ b/tests/run/empty_for_loop_T208.pyx @@ -1,4 +1,4 @@ -# ticket: 208 +# ticket: t208 def go_py_empty(): """ diff --git a/tests/run/enumerate_T316.pyx b/tests/run/enumerate_T316.pyx index f148e1a22..89755b76b 100644 --- a/tests/run/enumerate_T316.pyx +++ b/tests/run/enumerate_T316.pyx @@ -1,4 +1,4 @@ -# ticket: 316 +# ticket: t316 cimport cython diff --git a/tests/run/exceptionrefcount.pyx b/tests/run/exceptionrefcount.pyx index d4ce39fd9..1d1a6742f 100644 --- a/tests/run/exceptionrefcount.pyx +++ b/tests/run/exceptionrefcount.pyx @@ -27,8 +27,11 @@ __doc__ = u""" >>> run_test(50, test_finally) """ +cimport cython from cpython.ref cimport PyObject +@cython.binding(False) +@cython.always_allow_keywords(False) def get_refcount(obj): return (<PyObject*>obj).ob_refcnt diff --git a/tests/run/existing_output_files.srctree b/tests/run/existing_output_files.srctree new file mode 100644 index 000000000..5f6f03355 --- /dev/null +++ b/tests/run/existing_output_files.srctree @@ -0,0 +1,179 @@ +PYTHON test.py + +######## test.py ######## + +from __future__ import print_function + +import os.path +from Cython.Utils import is_cython_generated_file +from Cython.Compiler.Errors import CompileError +from Cython.Build.Dependencies import cythonize + +# Make sure the source files are newer than the .c files, so that cythonize() regenerates them. +files = {} +for source_file in sorted(os.listdir(os.getcwd())): + if 'module' in source_file and (source_file.endswith(".pyx") or source_file.endswith(".py")): + c_file = files[source_file] = os.path.splitext(source_file)[0] + ".c" + os.utime(source_file, None) + assert not os.path.exists(c_file) or os.path.getmtime(source_file) >= os.path.getmtime(c_file) + +for source_file, c_file in files.items(): + print("Testing:", source_file, c_file) + assert is_cython_generated_file(c_file, allow_failed=True, if_not_found=True) + + # cythonizing should (re)generate the file + cythonize(source_file, language_level=3) + assert is_cython_generated_file(c_file, if_not_found=False) + assert os.path.getmtime(source_file) <= os.path.getmtime(c_file) + + # calling cythonize again should not rewrite the file + # (not asserting this here, but at least it shouldn't fail) + cythonize(source_file, language_level=3) + assert is_cython_generated_file(c_file, if_not_found=False) + assert os.path.getmtime(source_file) <= os.path.getmtime(c_file) + + +# But overwriting an unknown file should fail, even when requested multiple times. +for source_file in [ + "refuse_to_overwrite.pyx", + "refuse_to_overwrite.py", + "compile_failure.pyx", + "refuse_to_overwrite_header.pyx", + "refuse_to_overwrite_api_header.pyx", +]: + if 'api_header' in source_file: + target_file = os.path.splitext(source_file)[0] + "_api.h" + elif 'header' in source_file: + target_file = os.path.splitext(source_file)[0] + ".h" + else: + target_file = os.path.splitext(source_file)[0] + ".c" + + for _ in range(3): + os.utime(source_file, None) + assert not is_cython_generated_file(target_file) + try: + print("Testing:", source_file) + cythonize(source_file, language_level=3) + except CompileError: + print("REFUSED to overwrite %s, OK" % target_file) + assert not is_cython_generated_file(target_file) + else: + assert False, "FAILURE: Existing output file was overwritten for source file %s" % source_file + + +######## pymodule.c ######## +#error Do not use this file, it is the result of a failed Cython compilation. + +######## pymodule.py ######## +""" +Overwriting a failed .py file result works +""" + +######## cymodule.c ######## +#error Do not use this file, it is the result of a failed Cython compilation. + +######## cymodule.pyx ######## +""" +Overwriting a failed .pyx file result works +""" + +######## overwritten_cymodule.c ######## +/* Generated by Cython 0.8.15 */ + +######## overwritten_cymodule.pyx ######## +""" +Overwriting an outdated .c file works +""" + + +######## new_cymodule.pyx ######## +""" +Creating a new .c file works +""" + +######## new_pymodule.py ######## +""" +Creating a new .c file works +""" + + +######## refuse_to_overwrite.c ######## +static int external_function(int x) { + return x + 1; +} + +######## refuse_to_overwrite.py ######## +""" +Do not overwrite an unknown output file +""" + +######## refuse_to_overwrite.pyx ######## +""" +Do not overwrite an unknown output file +""" + + +######## compile_failure.c ######## +static int external_function(int x) { + return x + 1; +} + +######## compile_failure.pyx ######## +""" +Do not overwrite an unknown output file even on compile failures. +""" + +Not Python syntax! + + + +######## write_module_header.pyx ######## + +cdef public int func(): + return 1 + + +######## overwrite_module_header.c ######## +/* Generated by Cython 0.8.15 */ + +######## overwrite_module_header.pyx ######## + +cdef public int func(): + return 1 + + +######## refuse_to_overwrite_header.h ######## +static int external_function(int x) { + return x + 1; +} + +######## refuse_to_overwrite_header.pyx ######## + +cdef public int func(): + return 1 + + +######## write_module_api_header.pyx ######## + +cdef api int func(): + return 1 + + +######## overwrite_module_api_header.c ######## +/* Generated by Cython 0.8.15 */ + +######## overwrite_module_api_header.pyx ######## + +cdef public int func(): + return 1 + + +######## refuse_to_overwrite_api_header_api.h ######## +static int external_function(int x) { + return x + 1; +} + +######## refuse_to_overwrite_api_header.pyx ######## + +cdef api int func(): + return 1 diff --git a/tests/run/ext_attr_getter.srctree b/tests/run/ext_attr_getter.srctree index 6fdf7503c..76f059ed7 100644 --- a/tests/run/ext_attr_getter.srctree +++ b/tests/run/ext_attr_getter.srctree @@ -1,17 +1,60 @@ +# mode: run +# tag: cgetter, property + +""" PYTHON setup.py build_ext --inplace -PYTHON -c "import runner" +PYTHON run_failure_tests.py +PYTHON runner.py +""" ######## setup.py ######## from Cython.Build.Dependencies import cythonize from distutils.core import setup -# force the build order -setup(ext_modules= cythonize("foo_extension.pyx")) +# Enforce the right build order +setup(ext_modules = cythonize("foo_extension.pyx", language_level=3)) +setup(ext_modules = cythonize("getter[0-9].pyx", language_level=3)) + + +######## run_failure_tests.py ######## + +import glob +import sys + +from Cython.Build.Dependencies import cythonize +from Cython.Compiler.Errors import CompileError + +# Run the failure tests +failed_tests = [] +passed_tests = [] + +def run_test(name): + title = name + with open(name, 'r') as f: + for line in f: + if 'TEST' in line: + title = line.partition('TEST:')[2].strip() + break + sys.stderr.write("\n### TESTING: %s\n" % title) + + try: + cythonize(name, language_level=3) + except CompileError as e: + sys.stderr.write("\nOK: got expected exception\n") + passed_tests.append(name) + else: + sys.stderr.write("\nFAIL: compilation did not detect the error\n") + failed_tests.append(name) -setup(ext_modules = cythonize("getter*.pyx")) +for name in sorted(glob.glob("getter_fail*.pyx")): + run_test(name) -######## foo_nominal.h ######## +assert not failed_tests, "Failed tests: %s" % failed_tests +assert passed_tests # check that tests were found at all + + +######## foo.h ######## #include <Python.h> @@ -24,30 +67,77 @@ typedef struct { int f0; int f1; int f2; + int v[10]; } FooStructNominal; +typedef struct { + PyObject_HEAD +} FooStructOpaque; + + +#define PyFoo_GET0M(a) (((FooStructNominal*)a)->f0) +#define PyFoo_GET1M(a) (((FooStructNominal*)a)->f1) +#define PyFoo_GET2M(a) (((FooStructNominal*)a)->f2) + +int PyFoo_Get0F(FooStructOpaque *f) +{ + return PyFoo_GET0M(f); +} + +int PyFoo_Get1F(FooStructOpaque *f) +{ + return PyFoo_GET1M(f); +} + +int PyFoo_Get2F(FooStructOpaque *f) +{ + return PyFoo_GET2M(f); +} + +int *PyFoo_GetV(FooStructOpaque *f) +{ + return ((FooStructNominal*)f)->v; +} + #ifdef __cplusplus } #endif + ######## foo_extension.pyx ######## cdef class Foo: - cdef public int field0, field1, field2; - - def __init__(self, f0, f1, f2): - self.field0 = f0 - self.field1 = f1 - self.field2 = f2 + cdef public int _field0, _field1, _field2; + cdef public int _vector[10]; -cdef get_field0(Foo f): - return f.field0 + @property + def field0(self): + return self._field0 -cdef get_field1(Foo f): - return f.field1 + @property + def field1(self): + return self._field1 -cdef get_field2(Foo f): - return f.field2 + @property + def field2(self): + return self._field2 + + def __init__(self, f0, f1, f2, vec=None): + if vec is None: + vec = () + if not isinstance(vec, tuple): + raise ValueError("v must be None or a tuple") + self._field0 = f0 + self._field1 = f1 + self._field2 = f2 + i = 0 + for v in vec: + self._vector[i] = v + if i > 9: + break + i += 1 + for j in range(i,10): + self._vector[j] = 0 # A pure-python class that disallows direct access to fields class OpaqueFoo(Foo): @@ -69,7 +159,7 @@ class OpaqueFoo(Foo): # Access base Foo fields from C via aliased field names -cdef extern from "foo_nominal.h": +cdef extern from "foo.h": ctypedef class foo_extension.Foo [object FooStructNominal]: cdef: @@ -78,13 +168,164 @@ cdef extern from "foo_nominal.h": int field2 "f2" def sum(Foo f): - # the f.__getattr__('field0') is replaced in c by f->f0 + # Note - not a cdef function but compiling the f.__getattr__('field0') + # notices the alias and replaces the __getattr__ in c by f->f0 anyway return f.field0 + f.field1 + f.field2 +def check_pyobj(Foo f): + # compare the c code to the check_pyobj in getter2.pyx + return bool(f.field1) + + +######## getter.pxd ######## + +# Access base Foo fields from C via getter functions + + +cdef extern from "foo.h": + ctypedef class foo_extension.Foo [object FooStructOpaque, check_size ignore]: + @property + cdef inline int fieldM0(self): + return PyFoo_GET0M(self) + + @property + cdef inline int fieldF1(self) except -123: + return PyFoo_Get1F(self) + + @property + cdef inline int fieldM2(self): + return PyFoo_GET2M(self) + + @property + cdef inline int *vector(self): + return PyFoo_GetV(self) + + @property + cdef inline int meaning_of_life(self) except -99: + cdef int ret = 21 + ret *= 2 + return ret + + int PyFoo_GET0M(Foo); # this is actually a macro ! + int PyFoo_Get1F(Foo); + int PyFoo_GET2M(Foo); # this is actually a macro ! + int *PyFoo_GetV(Foo); + + +######## getter1.pyx ######## + +cimport getter + +def sum(getter.Foo f): + # Note - not a cdef function but compiling the f.__getattr__('field0') + # notices the getter and replaces the __getattr__ in c by PyFoo_GET anyway + return f.fieldM0 + f.fieldF1 + f.fieldM2 + +def check_10(getter.Foo f): + return f.fieldF1 != 10 + +def vec0(getter.Foo f): + return f.vector[0] + +def check_binop(getter.Foo f): + return f.fieldF1 / 10 + + +######## getter2.pyx ######## + +cimport getter + +def check_pyobj(getter.Foo f): + return bool(f.fieldF1) + +def check_unary(getter.Foo f): + return -f.fieldF1 + +def check_meaning_of_life(getter.Foo f): + return f.meaning_of_life + + +######## getter_fail_classmethod.pyx ######## + +# TEST: Make sure not all decorators are accepted. + +cdef extern from "foo.h": + ctypedef class foo_extension.Foo [object FooStructOpaque]: + @property + @classmethod + cdef inline int field0(cls): + print('in classmethod of Foo') + + +######## getter_fail_dot_getter.pyx ######## + +# TEST: Make sure not all decorators are accepted. + +cdef extern from "foo.h": + ctypedef class foo_extension.Foo [object FooStructOpaque]: + @property + cdef inline int field0(self): + pass + + @field0.getter + cdef inline void field1(self): + pass + + +######## getter_fail_no_inline.pyx ######## + +# TEST: Properties must be declared "inline". + +cdef extern from "foo.h": + ctypedef class foo_extension.Foo [object FooStructOpaque]: + @property + cdef int field0(self): + pass + + +######## getter_fail_void.pyx ######## + +# TEST: Properties must have a non-void return type. + +cdef extern from "foo.h": + ctypedef class foo_extension.Foo [object FooStructOpaque]: + @property + cdef void field0(self): + pass + + +######## getter_fail_no_args.pyx ######## + +# TEST: Properties must have the right signature. + +cdef extern from "foo.h": + ctypedef class foo_extension.Foo [object FooStructOpaque]: + @property + cdef int field0(): + pass + + +######## getter_fail_too_many_args.pyx ######## + +# TEST: Properties must have the right signature. + +cdef extern from "foo.h": + ctypedef class foo_extension.Foo [object FooStructOpaque]: + @property + cdef int field0(x, y): + pass + + ######## runner.py ######## -import foo_extension, getter0 +import warnings +import foo_extension, getter0, getter1, getter2 +def sum(f): + # pure python field access, but code is identical to cython cdef sum + return f.field0 + f.field1 + f.field2 + +# Baseline test: if this fails something else is wrong foo = foo_extension.Foo(23, 123, 1023) assert foo.field0 == 23 @@ -92,18 +333,36 @@ assert foo.field1 == 123 assert foo.field2 == 1023 ret = getter0.sum(foo) -assert ret == foo.field0 + foo.field1 + foo.field2 +assert ret == sum(foo) + +# Aliasing test. Check 'cdef int field0 "f0" works as advertised: +# - C can access the fields through the aliases +# - Python cannot access the fields at all opaque_foo = foo_extension.OpaqueFoo(23, 123, 1023) -# C can access the fields through the aliases opaque_ret = getter0.sum(opaque_foo) assert opaque_ret == ret + +val = getter2.check_pyobj(opaque_foo) +assert val is True +val = getter2.check_unary(opaque_foo) +assert val == -123 +val = getter2.check_meaning_of_life(opaque_foo) +assert val == 42 + try: - # Python cannot access the fields f0 = opaque_ret.field0 assert False except AttributeError as e: pass +# Getter test. Check C-level getter works as advertised: +# - C accesses the fields through getter calls (maybe macros) +# - Python accesses the fields through attribute lookup + +opaque_foo = foo_extension.OpaqueFoo(23, 123, 1023, (1, 2, 3)) + +opaque_ret = getter1.sum(opaque_foo) +assert opaque_ret == ret diff --git a/tests/run/ext_instance_type_T232.pyx b/tests/run/ext_instance_type_T232.pyx index 9538a9b96..c089b1603 100644 --- a/tests/run/ext_instance_type_T232.pyx +++ b/tests/run/ext_instance_type_T232.pyx @@ -1,4 +1,4 @@ -# ticket: 232 +# ticket: t232 cdef class MyExt: cdef object attr diff --git a/tests/run/extended_unpacking_T235.pyx b/tests/run/extended_unpacking_T235.pyx index 2388e2587..8d38120a1 100644 --- a/tests/run/extended_unpacking_T235.pyx +++ b/tests/run/extended_unpacking_T235.pyx @@ -1,4 +1,4 @@ -# ticket: 235 +# ticket: t235 __doc__ = u""" >>> class FakeSeq(object): diff --git a/tests/run/extended_unpacking_T409.pyx b/tests/run/extended_unpacking_T409.pyx index 1198a15f3..7c3b71d51 100644 --- a/tests/run/extended_unpacking_T409.pyx +++ b/tests/run/extended_unpacking_T409.pyx @@ -1,4 +1,4 @@ -# ticket: 409 +# ticket: t409 def simple(): """ diff --git a/tests/run/extern_builtins_T258.pyx b/tests/run/extern_builtins_T258.pyx index d732931dd..1c6ea9843 100644 --- a/tests/run/extern_builtins_T258.pyx +++ b/tests/run/extern_builtins_T258.pyx @@ -1,4 +1,4 @@ -# ticket: 258 +# ticket: t258 cdef extern from "Python.h": diff --git a/tests/run/extstarargs.pyx b/tests/run/extstarargs.pyx index 041603865..b1dcd4957 100644 --- a/tests/run/extstarargs.pyx +++ b/tests/run/extstarargs.pyx @@ -1,124 +1,169 @@ -__doc__ = u""" - >>> s = Silly(1,2,3, 'test') - >>> (spam,grail,swallow,creosote,onlyt,onlyk,tk) = ( - ... s.spam,s.grail,s.swallow,s.creosote,s.onlyt,s.onlyk,s.tk) - - >>> spam(1,2,3) - (1, 2, 3) - >>> spam(1,2) - Traceback (most recent call last): - TypeError: spam() takes exactly 3 positional arguments (2 given) - >>> spam(1,2,3,4) - Traceback (most recent call last): - TypeError: spam() takes exactly 3 positional arguments (4 given) - >>> spam(1,2,3, a=1) #doctest: +ELLIPSIS - Traceback (most recent call last): - TypeError: spam() got an unexpected keyword argument 'a' - - >>> grail(1,2,3) - (1, 2, 3, ()) - >>> grail(1,2,3,4) - (1, 2, 3, (4,)) - >>> grail(1,2,3,4,5,6,7,8,9) - (1, 2, 3, (4, 5, 6, 7, 8, 9)) - >>> grail(1,2) - Traceback (most recent call last): - TypeError: grail() takes at least 3 positional arguments (2 given) - >>> grail(1,2,3, a=1) #doctest: +ELLIPSIS - Traceback (most recent call last): - TypeError: grail() got an unexpected keyword argument 'a' - - >>> swallow(1,2,3) - (1, 2, 3, ()) - >>> swallow(1,2,3,4) - Traceback (most recent call last): - TypeError: swallow() takes exactly 3 positional arguments (4 given) - >>> swallow(1,2,3, a=1, b=2) - (1, 2, 3, (('a', 1), ('b', 2))) - >>> swallow(1,2,3, x=1) - Traceback (most recent call last): - TypeError: swallow() got multiple values for keyword argument 'x' - - >>> creosote(1,2,3) - (1, 2, 3, (), ()) - >>> creosote(1,2,3,4) - (1, 2, 3, (4,), ()) - >>> creosote(1,2,3, a=1) - (1, 2, 3, (), (('a', 1),)) - >>> creosote(1,2,3,4, a=1, b=2) - (1, 2, 3, (4,), (('a', 1), ('b', 2))) - >>> creosote(1,2,3,4, x=1) - Traceback (most recent call last): - TypeError: creosote() got multiple values for keyword argument 'x' - - >>> onlyt(1) - (1,) - >>> onlyt(1,2) - (1, 2) - >>> onlyt(a=1) - Traceback (most recent call last): - TypeError: onlyt() got an unexpected keyword argument 'a' - >>> onlyt(1, a=2) - Traceback (most recent call last): - TypeError: onlyt() got an unexpected keyword argument 'a' - - >>> onlyk(a=1) - (('a', 1),) - >>> onlyk(a=1, b=2) - (('a', 1), ('b', 2)) - >>> onlyk(1) - Traceback (most recent call last): - TypeError: onlyk() takes exactly 0 positional arguments (1 given) - >>> onlyk(1, 2) - Traceback (most recent call last): - TypeError: onlyk() takes exactly 0 positional arguments (2 given) - >>> onlyk(1, a=1, b=2) - Traceback (most recent call last): - TypeError: onlyk() takes exactly 0 positional arguments (1 given) - - >>> tk(a=1) - (('a', 1),) - >>> tk(a=1, b=2) - (('a', 1), ('b', 2)) - >>> tk(1) - (1,) - >>> tk(1, 2) - (1, 2) - >>> tk(1, a=1, b=2) - (1, ('a', 1), ('b', 2)) -""" - -import sys, re -if sys.version_info >= (2,6): - __doc__ = re.sub(u"(ELLIPSIS[^>]*Error: )[^\n]*\n", u"\\1...\n", __doc__) +cimport cython cdef sorteditems(d): - l = list(d.items()) - l.sort() - return tuple(l) + return tuple(sorted(d.items())) + cdef class Silly: def __init__(self, *a): - pass + """ + >>> s = Silly(1,2,3, 'test') + """ def spam(self, x, y, z): + """ + >>> s = Silly() + >>> s.spam(1,2,3) + (1, 2, 3) + >>> s.spam(1,2) + Traceback (most recent call last): + TypeError: spam() takes exactly 3 positional arguments (2 given) + >>> s.spam(1,2,3,4) + Traceback (most recent call last): + TypeError: spam() takes exactly 3 positional arguments (4 given) + >>> s.spam(1,2,3, a=1) + Traceback (most recent call last): + TypeError: spam() got an unexpected keyword argument 'a' + """ return (x, y, z) def grail(self, x, y, z, *a): + """ + >>> s = Silly() + >>> s.grail(1,2,3) + (1, 2, 3, ()) + >>> s.grail(1,2,3,4) + (1, 2, 3, (4,)) + >>> s.grail(1,2,3,4,5,6,7,8,9) + (1, 2, 3, (4, 5, 6, 7, 8, 9)) + >>> s.grail(1,2) + Traceback (most recent call last): + TypeError: grail() takes at least 3 positional arguments (2 given) + >>> s.grail(1,2,3, a=1) + Traceback (most recent call last): + TypeError: grail() got an unexpected keyword argument 'a' + """ return (x, y, z, a) def swallow(self, x, y, z, **k): + """ + >>> s = Silly() + >>> s.swallow(1,2,3) + (1, 2, 3, ()) + >>> s.swallow(1,2,3,4) + Traceback (most recent call last): + TypeError: swallow() takes exactly 3 positional arguments (4 given) + >>> s.swallow(1,2,3, a=1, b=2) + (1, 2, 3, (('a', 1), ('b', 2))) + >>> s.swallow(1,2,3, x=1) + Traceback (most recent call last): + TypeError: swallow() got multiple values for keyword argument 'x' + """ return (x, y, z, sorteditems(k)) def creosote(self, x, y, z, *a, **k): + """ + >>> s = Silly() + >>> s.creosote(1,2,3) + (1, 2, 3, (), ()) + >>> s.creosote(1,2,3,4) + (1, 2, 3, (4,), ()) + >>> s.creosote(1,2,3, a=1) + (1, 2, 3, (), (('a', 1),)) + >>> s.creosote(1,2,3,4, a=1, b=2) + (1, 2, 3, (4,), (('a', 1), ('b', 2))) + >>> s.creosote(1,2,3,4, x=1) + Traceback (most recent call last): + TypeError: creosote() got multiple values for keyword argument 'x' + """ return (x, y, z, a, sorteditems(k)) def onlyt(self, *a): + """ + >>> s = Silly() + >>> s.onlyt(1) + (1,) + >>> s.onlyt(1,2) + (1, 2) + >>> s.onlyt(a=1) + Traceback (most recent call last): + TypeError: onlyt() got an unexpected keyword argument 'a' + >>> s.onlyt(1, a=2) + Traceback (most recent call last): + TypeError: onlyt() got an unexpected keyword argument 'a' + """ + return a + + @cython.binding(False) # passthrough of exact same tuple can't work with binding + def onlyt_nobinding(self, *a): + """ + >>> s = Silly() + >>> s.onlyt_nobinding(1) + (1,) + >>> s.onlyt_nobinding(1,2) + (1, 2) + >>> s.onlyt_nobinding(a=1) + Traceback (most recent call last): + TypeError: onlyt_nobinding() got an unexpected keyword argument 'a' + >>> s.onlyt_nobinding(1, a=2) + Traceback (most recent call last): + TypeError: onlyt_nobinding() got an unexpected keyword argument 'a' + >>> test_no_copy_args(s.onlyt_nobinding) + True + """ return a def onlyk(self, **k): + """ + >>> s = Silly() + >>> s.onlyk(a=1) + (('a', 1),) + >>> s.onlyk(a=1, b=2) + (('a', 1), ('b', 2)) + >>> s.onlyk(1) + Traceback (most recent call last): + TypeError: onlyk() takes exactly 0 positional arguments (1 given) + >>> s.onlyk(1, 2) + Traceback (most recent call last): + TypeError: onlyk() takes exactly 0 positional arguments (2 given) + >>> s.onlyk(1, a=1, b=2) + Traceback (most recent call last): + TypeError: onlyk() takes exactly 0 positional arguments (1 given) + """ return sorteditems(k) def tk(self, *a, **k): + """ + >>> s = Silly() + >>> s.tk(a=1) + (('a', 1),) + >>> s.tk(a=1, b=2) + (('a', 1), ('b', 2)) + >>> s.tk(1) + (1,) + >>> s.tk(1, 2) + (1, 2) + >>> s.tk(1, a=1, b=2) + (1, ('a', 1), ('b', 2)) + """ return a + sorteditems(k) + + @cython.binding(False) # passthrough of exact same tuple can't work with binding + def t_kwonly(self, *a, k): + """ + >>> s = Silly() + >>> test_no_copy_args(s.t_kwonly, k=None) + True + """ + return a + + +def test_no_copy_args(func, **kw): + """ + func is a function such that func(*args, **kw) returns args. + We test that no copy is made of the args tuple. + This tests both the caller side and the callee side. + """ + args = (1, 2, 3) + return func(*args, **kw) is args diff --git a/tests/run/exttype.pyx b/tests/run/exttype.pyx index 61d0c169a..c14c1e4ed 100644 --- a/tests/run/exttype.pyx +++ b/tests/run/exttype.pyx @@ -1,6 +1,52 @@ +# mode: run +# tag: exttype, tpnew + +from __future__ import print_function + +from cpython.object cimport PyTypeObject + cdef gobble(a, b): - print a, b + print(a, b) + + +def tp_new_ptr(exttype): + assert isinstance(exttype, type) + tp = <PyTypeObject*> exttype + return <unsigned long long><void*>tp.tp_new + + +cdef class Empty: + """ + >>> n = Empty() + >>> isinstance(n, Empty) + True + >>> tp_new_ptr(Empty) != 0 + True + """ + + +cdef class EmptySubclass(Empty): + """ + >>> n = EmptySubclass() + >>> isinstance(n, EmptySubclass) + True + >>> tp_new_ptr(EmptySubclass) != 0 + True + >>> tp_new_ptr(EmptySubclass) == tp_new_ptr(Empty) + True + """ + + +cdef class CInit: + """ + >>> c = CInit() + >>> isinstance(c, CInit) + True + """ + def __cinit__(self): + assert self is not None + cdef class Spam: """ @@ -21,6 +67,7 @@ cdef class Spam: def eat(self): gobble(self.eggs, self.ham) + def f(Spam spam): """ >>> s = Spam(12) diff --git a/tests/run/exttype_total_ordering.pyx b/tests/run/exttype_total_ordering.pyx new file mode 100644 index 000000000..c5f71b8b7 --- /dev/null +++ b/tests/run/exttype_total_ordering.pyx @@ -0,0 +1,1020 @@ +# mode: run +# tag: total_ordering + +from __future__ import print_function + +""" + >>> class PyTotalOrdering: + ... def __init__(self, value): + ... self.value = value + ... def __eq__(self, other): + ... return self.value == other.value + ... def __lt__(self, other): + ... return self.value < other.value + >>> test_all_comp(functools.total_ordering(PyTotalOrdering)) + True +""" + +cimport cython +import functools +import operator + +COMPARISONS = [ + # Don't test equals, the directive doesn't add that. + # ('==', operator.__eq__), + ('!=', operator.__ne__), + ('<', operator.__lt__), + ('>', operator.__gt__), + ('<=', operator.__le__), + ('>=', operator.__ge__), +] + +def test_all_comp(cls): + """Check every combination of comparison operators.""" + a, b, c = 10, 15, 20 + succeeded = True + for comp, func in COMPARISONS: + for left in [cls(a), cls(b), cls(c)]: + for right in [ValueHolder(a), ValueHolder(b), ValueHolder(c)]: + expected = func(left.value, right.value) + try: + result = func(left, right) + # repeat to rule out deallocation bugs (and assert determinism) + for _ in range(10): + assert result == func(left, right) + except TypeError: + print("TypeError:", left.value, comp, right.value) + succeeded = False + else: + if expected != result: + print( + left.value, comp, right.value, + "expected:", expected, "got:", result + ) + succeeded = False + return succeeded + +class ValueHolder: + """Has a value, but can't compare.""" + def __init__(self, value): + self.value = value + + + +cdef class ExtTypeNoTotalOrdering: + """ + >>> a = ExtTypeNoTotalOrdering(5) + >>> b = ExtTypeNoTotalOrdering(10) + >>> a == b + False + >>> a != b # Added in Python 3, but Cython backports + True + >>> a < b + True + >>> b < a + False + >>> a > b + False + >>> b > a + True + >>> import sys + >>> try: _ = a >= b + ... except TypeError: + ... assert sys.version_info[0] >= 3 + ... else: + ... assert sys.version_info[0] < 3 + >>> try: _ = a <= b + ... except TypeError: + ... assert sys.version_info[0] >= 3 + ... else: + ... assert sys.version_info[0] < 3 + """ + cdef public int value + def __init__(self, val): + self.value = val + + def __lt__(self, other): + return self.value < other.value + + def __eq__(self, other): + return self.value == other.value + +# Every combination of methods which is valid. + +@cython.total_ordering +cdef class ExtTypeTotalOrderingNeGe: + """ + >>> test_all_comp(ExtTypeTotalOrderingNeGe) + True + """ + cdef public int value + def __init__(self, val): + self.value = val + + def __ne__(self, other): + return self.value != other.value + + def __ge__(self, other): + return self.value >= other.value + +@cython.total_ordering +cdef class ExtTypeTotalOrderingNeLe: + """ + >>> test_all_comp(ExtTypeTotalOrderingNeLe) + True + """ + cdef public int value + def __init__(self, val): + self.value = val + + def __ne__(self, other): + return self.value != other.value + + def __le__(self, other): + return self.value <= other.value + +@cython.total_ordering +cdef class ExtTypeTotalOrderingNeLeGe: + """ + >>> test_all_comp(ExtTypeTotalOrderingNeLeGe) + True + """ + cdef public int value + def __init__(self, val): + self.value = val + + def __ne__(self, other): + return self.value != other.value + + def __le__(self, other): + return self.value <= other.value + + def __ge__(self, other): + return self.value >= other.value + +@cython.total_ordering +cdef class ExtTypeTotalOrderingNeGt: + """ + >>> test_all_comp(ExtTypeTotalOrderingNeGt) + True + """ + cdef public int value + def __init__(self, val): + self.value = val + + def __ne__(self, other): + return self.value != other.value + + def __gt__(self, other): + return self.value > other.value + +@cython.total_ordering +cdef class ExtTypeTotalOrderingNeGtGe: + """ + >>> test_all_comp(ExtTypeTotalOrderingNeGtGe) + True + """ + cdef public int value + def __init__(self, val): + self.value = val + + def __ne__(self, other): + return self.value != other.value + + def __gt__(self, other): + return self.value > other.value + + def __ge__(self, other): + return self.value >= other.value + +@cython.total_ordering +cdef class ExtTypeTotalOrderingNeGtLe: + """ + >>> test_all_comp(ExtTypeTotalOrderingNeGtLe) + True + """ + cdef public int value + def __init__(self, val): + self.value = val + + def __ne__(self, other): + return self.value != other.value + + def __gt__(self, other): + return self.value > other.value + + def __le__(self, other): + return self.value <= other.value + +@cython.total_ordering +cdef class ExtTypeTotalOrderingNeGtLeGe: + """ + >>> test_all_comp(ExtTypeTotalOrderingNeGtLeGe) + True + """ + cdef public int value + def __init__(self, val): + self.value = val + + def __ne__(self, other): + return self.value != other.value + + def __gt__(self, other): + return self.value > other.value + + def __le__(self, other): + return self.value <= other.value + + def __ge__(self, other): + return self.value >= other.value + +@cython.total_ordering +cdef class ExtTypeTotalOrderingNeLt: + """ + >>> test_all_comp(ExtTypeTotalOrderingNeLt) + True + """ + cdef public int value + def __init__(self, val): + self.value = val + + def __ne__(self, other): + return self.value != other.value + + def __lt__(self, other): + return self.value < other.value + +@cython.total_ordering +cdef class ExtTypeTotalOrderingNeLtGe: + """ + >>> test_all_comp(ExtTypeTotalOrderingNeLtGe) + True + """ + cdef public int value + def __init__(self, val): + self.value = val + + def __ne__(self, other): + return self.value != other.value + + def __lt__(self, other): + return self.value < other.value + + def __ge__(self, other): + return self.value >= other.value + +@cython.total_ordering +cdef class ExtTypeTotalOrderingNeLtLe: + """ + >>> test_all_comp(ExtTypeTotalOrderingNeLtLe) + True + """ + cdef public int value + def __init__(self, val): + self.value = val + + def __ne__(self, other): + return self.value != other.value + + def __lt__(self, other): + return self.value < other.value + + def __le__(self, other): + return self.value <= other.value + +@cython.total_ordering +cdef class ExtTypeTotalOrderingNeLtLeGe: + """ + >>> test_all_comp(ExtTypeTotalOrderingNeLtLeGe) + True + """ + cdef public int value + def __init__(self, val): + self.value = val + + def __ne__(self, other): + return self.value != other.value + + def __lt__(self, other): + return self.value < other.value + + def __le__(self, other): + return self.value <= other.value + + def __ge__(self, other): + return self.value >= other.value + +@cython.total_ordering +cdef class ExtTypeTotalOrderingNeLtGt: + """ + >>> test_all_comp(ExtTypeTotalOrderingNeLtGt) + True + """ + cdef public int value + def __init__(self, val): + self.value = val + + def __ne__(self, other): + return self.value != other.value + + def __lt__(self, other): + return self.value < other.value + + def __gt__(self, other): + return self.value > other.value + +@cython.total_ordering +cdef class ExtTypeTotalOrderingNeLtGtGe: + """ + >>> test_all_comp(ExtTypeTotalOrderingNeLtGtGe) + True + """ + cdef public int value + def __init__(self, val): + self.value = val + + def __ne__(self, other): + return self.value != other.value + + def __lt__(self, other): + return self.value < other.value + + def __gt__(self, other): + return self.value > other.value + + def __ge__(self, other): + return self.value >= other.value + +@cython.total_ordering +cdef class ExtTypeTotalOrderingNeLtGtLe: + """ + >>> test_all_comp(ExtTypeTotalOrderingNeLtGtLe) + True + """ + cdef public int value + def __init__(self, val): + self.value = val + + def __ne__(self, other): + return self.value != other.value + + def __lt__(self, other): + return self.value < other.value + + def __gt__(self, other): + return self.value > other.value + + def __le__(self, other): + return self.value <= other.value + +@cython.total_ordering +cdef class ExtTypeTotalOrderingNeLtGtLeGe: + """ + >>> test_all_comp(ExtTypeTotalOrderingNeLtGtLeGe) + True + """ + cdef public int value + def __init__(self, val): + self.value = val + + def __ne__(self, other): + return self.value != other.value + + def __lt__(self, other): + return self.value < other.value + + def __gt__(self, other): + return self.value > other.value + + def __le__(self, other): + return self.value <= other.value + + def __ge__(self, other): + return self.value >= other.value + +@cython.total_ordering +cdef class ExtTypeTotalOrderingEqGe: + """ + >>> test_all_comp(ExtTypeTotalOrderingEqGe) + True + """ + cdef public int value + def __init__(self, val): + self.value = val + + def __eq__(self, other): + return self.value == other.value + + def __ge__(self, other): + return self.value >= other.value + +@cython.total_ordering +cdef class ExtTypeTotalOrderingEqLe: + """ + >>> test_all_comp(ExtTypeTotalOrderingEqLe) + True + """ + cdef public int value + def __init__(self, val): + self.value = val + + def __eq__(self, other): + return self.value == other.value + + def __le__(self, other): + return self.value <= other.value + +@cython.total_ordering +cdef class ExtTypeTotalOrderingEqLeGe: + """ + >>> test_all_comp(ExtTypeTotalOrderingEqLeGe) + True + """ + cdef public int value + def __init__(self, val): + self.value = val + + def __eq__(self, other): + return self.value == other.value + + def __le__(self, other): + return self.value <= other.value + + def __ge__(self, other): + return self.value >= other.value + +@cython.total_ordering +cdef class ExtTypeTotalOrderingEqGt: + """ + >>> test_all_comp(ExtTypeTotalOrderingEqGt) + True + """ + cdef public int value + def __init__(self, val): + self.value = val + + def __eq__(self, other): + return self.value == other.value + + def __gt__(self, other): + return self.value > other.value + +@cython.total_ordering +cdef class ExtTypeTotalOrderingEqGtGe: + """ + >>> test_all_comp(ExtTypeTotalOrderingEqGtGe) + True + """ + cdef public int value + def __init__(self, val): + self.value = val + + def __eq__(self, other): + return self.value == other.value + + def __gt__(self, other): + return self.value > other.value + + def __ge__(self, other): + return self.value >= other.value + +@cython.total_ordering +cdef class ExtTypeTotalOrderingEqGtLe: + """ + >>> test_all_comp(ExtTypeTotalOrderingEqGtLe) + True + """ + cdef public int value + def __init__(self, val): + self.value = val + + def __eq__(self, other): + return self.value == other.value + + def __gt__(self, other): + return self.value > other.value + + def __le__(self, other): + return self.value <= other.value + +@cython.total_ordering +cdef class ExtTypeTotalOrderingEqGtLeGe: + """ + >>> test_all_comp(ExtTypeTotalOrderingEqGtLeGe) + True + """ + cdef public int value + def __init__(self, val): + self.value = val + + def __eq__(self, other): + return self.value == other.value + + def __gt__(self, other): + return self.value > other.value + + def __le__(self, other): + return self.value <= other.value + + def __ge__(self, other): + return self.value >= other.value + +@cython.total_ordering +cdef class ExtTypeTotalOrderingEqLt: + """ + >>> test_all_comp(ExtTypeTotalOrderingEqLt) + True + """ + cdef public int value + def __init__(self, val): + self.value = val + + def __eq__(self, other): + return self.value == other.value + + def __lt__(self, other): + return self.value < other.value + +@cython.total_ordering +cdef class ExtTypeTotalOrderingEqLtGe: + """ + >>> test_all_comp(ExtTypeTotalOrderingEqLtGe) + True + """ + cdef public int value + def __init__(self, val): + self.value = val + + def __eq__(self, other): + return self.value == other.value + + def __lt__(self, other): + return self.value < other.value + + def __ge__(self, other): + return self.value >= other.value + +@cython.total_ordering +cdef class ExtTypeTotalOrderingEqLtLe: + """ + >>> test_all_comp(ExtTypeTotalOrderingEqLtLe) + True + """ + cdef public int value + def __init__(self, val): + self.value = val + + def __eq__(self, other): + return self.value == other.value + + def __lt__(self, other): + return self.value < other.value + + def __le__(self, other): + return self.value <= other.value + +@cython.total_ordering +cdef class ExtTypeTotalOrderingEqLtLeGe: + """ + >>> test_all_comp(ExtTypeTotalOrderingEqLtLeGe) + True + """ + cdef public int value + def __init__(self, val): + self.value = val + + def __eq__(self, other): + return self.value == other.value + + def __lt__(self, other): + return self.value < other.value + + def __le__(self, other): + return self.value <= other.value + + def __ge__(self, other): + return self.value >= other.value + +@cython.total_ordering +cdef class ExtTypeTotalOrderingEqLtGt: + """ + >>> test_all_comp(ExtTypeTotalOrderingEqLtGt) + True + """ + cdef public int value + def __init__(self, val): + self.value = val + + def __eq__(self, other): + return self.value == other.value + + def __lt__(self, other): + return self.value < other.value + + def __gt__(self, other): + return self.value > other.value + +@cython.total_ordering +cdef class ExtTypeTotalOrderingEqLtGtGe: + """ + >>> test_all_comp(ExtTypeTotalOrderingEqLtGtGe) + True + """ + cdef public int value + def __init__(self, val): + self.value = val + + def __eq__(self, other): + return self.value == other.value + + def __lt__(self, other): + return self.value < other.value + + def __gt__(self, other): + return self.value > other.value + + def __ge__(self, other): + return self.value >= other.value + +@cython.total_ordering +cdef class ExtTypeTotalOrderingEqLtGtLe: + """ + >>> test_all_comp(ExtTypeTotalOrderingEqLtGtLe) + True + """ + cdef public int value + def __init__(self, val): + self.value = val + + def __eq__(self, other): + return self.value == other.value + + def __lt__(self, other): + return self.value < other.value + + def __gt__(self, other): + return self.value > other.value + + def __le__(self, other): + return self.value <= other.value + +@cython.total_ordering +cdef class ExtTypeTotalOrderingEqLtGtLeGe: + """ + >>> test_all_comp(ExtTypeTotalOrderingEqLtGtLeGe) + True + """ + cdef public int value + def __init__(self, val): + self.value = val + + def __eq__(self, other): + return self.value == other.value + + def __lt__(self, other): + return self.value < other.value + + def __gt__(self, other): + return self.value > other.value + + def __le__(self, other): + return self.value <= other.value + + def __ge__(self, other): + return self.value >= other.value + +@cython.total_ordering +cdef class ExtTypeTotalOrderingEqNeGe: + """ + >>> test_all_comp(ExtTypeTotalOrderingEqNeGe) + True + """ + cdef public int value + def __init__(self, val): + self.value = val + + def __eq__(self, other): + return self.value == other.value + + def __ne__(self, other): + return self.value != other.value + + def __ge__(self, other): + return self.value >= other.value + +@cython.total_ordering +cdef class ExtTypeTotalOrderingEqNeLe: + """ + >>> test_all_comp(ExtTypeTotalOrderingEqNeLe) + True + """ + cdef public int value + def __init__(self, val): + self.value = val + + def __eq__(self, other): + return self.value == other.value + + def __ne__(self, other): + return self.value != other.value + + def __le__(self, other): + return self.value <= other.value + +@cython.total_ordering +cdef class ExtTypeTotalOrderingEqNeLeGe: + """ + >>> test_all_comp(ExtTypeTotalOrderingEqNeLeGe) + True + """ + cdef public int value + def __init__(self, val): + self.value = val + + def __eq__(self, other): + return self.value == other.value + + def __ne__(self, other): + return self.value != other.value + + def __le__(self, other): + return self.value <= other.value + + def __ge__(self, other): + return self.value >= other.value + +@cython.total_ordering +cdef class ExtTypeTotalOrderingEqNeGt: + """ + >>> test_all_comp(ExtTypeTotalOrderingEqNeGt) + True + """ + cdef public int value + def __init__(self, val): + self.value = val + + def __eq__(self, other): + return self.value == other.value + + def __ne__(self, other): + return self.value != other.value + + def __gt__(self, other): + return self.value > other.value + +@cython.total_ordering +cdef class ExtTypeTotalOrderingEqNeGtGe: + """ + >>> test_all_comp(ExtTypeTotalOrderingEqNeGtGe) + True + """ + cdef public int value + def __init__(self, val): + self.value = val + + def __eq__(self, other): + return self.value == other.value + + def __ne__(self, other): + return self.value != other.value + + def __gt__(self, other): + return self.value > other.value + + def __ge__(self, other): + return self.value >= other.value + +@cython.total_ordering +cdef class ExtTypeTotalOrderingEqNeGtLe: + """ + >>> test_all_comp(ExtTypeTotalOrderingEqNeGtLe) + True + """ + cdef public int value + def __init__(self, val): + self.value = val + + def __eq__(self, other): + return self.value == other.value + + def __ne__(self, other): + return self.value != other.value + + def __gt__(self, other): + return self.value > other.value + + def __le__(self, other): + return self.value <= other.value + +@cython.total_ordering +cdef class ExtTypeTotalOrderingEqNeGtLeGe: + """ + >>> test_all_comp(ExtTypeTotalOrderingEqNeGtLeGe) + True + """ + cdef public int value + def __init__(self, val): + self.value = val + + def __eq__(self, other): + return self.value == other.value + + def __ne__(self, other): + return self.value != other.value + + def __gt__(self, other): + return self.value > other.value + + def __le__(self, other): + return self.value <= other.value + + def __ge__(self, other): + return self.value >= other.value + +@cython.total_ordering +cdef class ExtTypeTotalOrderingEqNeLt: + """ + >>> test_all_comp(ExtTypeTotalOrderingEqNeLt) + True + """ + cdef public int value + def __init__(self, val): + self.value = val + + def __eq__(self, other): + return self.value == other.value + + def __ne__(self, other): + return self.value != other.value + + def __lt__(self, other): + return self.value < other.value + +@cython.total_ordering +cdef class ExtTypeTotalOrderingEqNeLtGe: + """ + >>> test_all_comp(ExtTypeTotalOrderingEqNeLtGe) + True + """ + cdef public int value + def __init__(self, val): + self.value = val + + def __eq__(self, other): + return self.value == other.value + + def __ne__(self, other): + return self.value != other.value + + def __lt__(self, other): + return self.value < other.value + + def __ge__(self, other): + return self.value >= other.value + +@cython.total_ordering +cdef class ExtTypeTotalOrderingEqNeLtLe: + """ + >>> test_all_comp(ExtTypeTotalOrderingEqNeLtLe) + True + """ + cdef public int value + def __init__(self, val): + self.value = val + + def __eq__(self, other): + return self.value == other.value + + def __ne__(self, other): + return self.value != other.value + + def __lt__(self, other): + return self.value < other.value + + def __le__(self, other): + return self.value <= other.value + +@cython.total_ordering +cdef class ExtTypeTotalOrderingEqNeLtLeGe: + """ + >>> test_all_comp(ExtTypeTotalOrderingEqNeLtLeGe) + True + """ + cdef public int value + def __init__(self, val): + self.value = val + + def __eq__(self, other): + return self.value == other.value + + def __ne__(self, other): + return self.value != other.value + + def __lt__(self, other): + return self.value < other.value + + def __le__(self, other): + return self.value <= other.value + + def __ge__(self, other): + return self.value >= other.value + +@cython.total_ordering +cdef class ExtTypeTotalOrderingEqNeLtGt: + """ + >>> test_all_comp(ExtTypeTotalOrderingEqNeLtGt) + True + """ + cdef public int value + def __init__(self, val): + self.value = val + + def __eq__(self, other): + return self.value == other.value + + def __ne__(self, other): + return self.value != other.value + + def __lt__(self, other): + return self.value < other.value + + def __gt__(self, other): + return self.value > other.value + +@cython.total_ordering +cdef class ExtTypeTotalOrderingEqNeLtGtGe: + """ + >>> test_all_comp(ExtTypeTotalOrderingEqNeLtGtGe) + True + """ + cdef public int value + def __init__(self, val): + self.value = val + + def __eq__(self, other): + return self.value == other.value + + def __ne__(self, other): + return self.value != other.value + + def __lt__(self, other): + return self.value < other.value + + def __gt__(self, other): + return self.value > other.value + + def __ge__(self, other): + return self.value >= other.value + +@cython.total_ordering +cdef class ExtTypeTotalOrderingEqNeLtGtLe: + """ + >>> test_all_comp(ExtTypeTotalOrderingEqNeLtGtLe) + True + """ + cdef public int value + def __init__(self, val): + self.value = val + + def __eq__(self, other): + return self.value == other.value + + def __ne__(self, other): + return self.value != other.value + + def __lt__(self, other): + return self.value < other.value + + def __gt__(self, other): + return self.value > other.value + + def __le__(self, other): + return self.value <= other.value + +@cython.total_ordering +cdef class ExtTypeTotalOrderingEqNeLtGtLeGe: + """ + >>> test_all_comp(ExtTypeTotalOrderingEqNeLtGtLeGe) + True + """ + cdef public int value + def __init__(self, val): + self.value = val + + def __eq__(self, other): + return self.value == other.value + + def __ne__(self, other): + return self.value != other.value + + def __lt__(self, other): + return self.value < other.value + + def __gt__(self, other): + return self.value > other.value + + def __le__(self, other): + return self.value <= other.value + + def __ge__(self, other): + return self.value >= other.value diff --git a/tests/run/fastcall.pyx b/tests/run/fastcall.pyx index ec9f5ed5b..a770fa9e3 100644 --- a/tests/run/fastcall.pyx +++ b/tests/run/fastcall.pyx @@ -1,6 +1,8 @@ # mode: run # tag: METH_FASTCALL +cimport cython + import sys import struct from collections import deque @@ -63,3 +65,64 @@ cdef class SelfCast: """ def index_of_self(self, list orbit not None): return orbit.index(self) + + +cdef extern from *: + int PyCFunction_GET_FLAGS(op) + + +def has_fastcall(meth): + """ + Given a builtin_function_or_method or cyfunction ``meth``, + return whether it uses ``METH_FASTCALL``. + """ + # Hardcode METH_FASTCALL constant equal to 0x80 for simplicity + return bool(PyCFunction_GET_FLAGS(meth) & 0x80) + + +def assert_fastcall(meth): + """ + Assert that ``meth`` uses ``METH_FASTCALL`` if the Python + implementation supports it. + """ + # getattr uses METH_FASTCALL on CPython >= 3.7 + if has_fastcall(getattr) and not has_fastcall(meth): + raise AssertionError(f"{meth} does not use METH_FASTCALL") + + +@cython.binding(False) +def fastcall_function(**kw): + """ + >>> assert_fastcall(fastcall_function) + """ + return kw + +@cython.binding(True) +def fastcall_cyfunction(**kw): + """ + >>> assert_fastcall(fastcall_cyfunction) + """ + return kw + +cdef class Dummy: + @cython.binding(False) + def fastcall_method(self, x, *args, **kw): + """ + >>> assert_fastcall(Dummy().fastcall_method) + """ + return tuple(args) + tuple(kw) + +cdef class CyDummy: + @cython.binding(True) + def fastcall_method(self, x, *args, **kw): + """ + >>> assert_fastcall(CyDummy.fastcall_method) + """ + return tuple(args) + tuple(kw) + +class PyDummy: + def fastcall_method(self, x, *args, **kw): + """ + >>> assert_fastcall(PyDummy.fastcall_method) + """ + return tuple(args) + tuple(kw) diff --git a/tests/run/file_encoding_T740.py b/tests/run/file_encoding_T740.py index f61973353..a71b04be9 100644 --- a/tests/run/file_encoding_T740.py +++ b/tests/run/file_encoding_T740.py @@ -1,6 +1,6 @@ # encoding: koi8-r # mode: run -# ticket: 740 +# ticket: t740 """ >>> wtf 'wtf' diff --git a/tests/run/final_method_T586.pyx b/tests/run/final_method_T586.pyx index 8fa1ba25c..f50bb8aa8 100644 --- a/tests/run/final_method_T586.pyx +++ b/tests/run/final_method_T586.pyx @@ -1,5 +1,5 @@ # mode: run -# ticket: 568 +# ticket: t568 cimport cython diff --git a/tests/run/float_floor_division_T260.pyx b/tests/run/float_floor_division_T260.pyx index 66f6f83f7..58d13a157 100644 --- a/tests/run/float_floor_division_T260.pyx +++ b/tests/run/float_floor_division_T260.pyx @@ -1,4 +1,4 @@ -# ticket: 260 +# ticket: t260 def floor_div_float(double a, double b): """ diff --git a/tests/run/float_len_T480.pyx b/tests/run/float_len_T480.pyx index efb456c54..4f623030a 100644 --- a/tests/run/float_len_T480.pyx +++ b/tests/run/float_len_T480.pyx @@ -1,4 +1,4 @@ -# ticket: 480 +# ticket: t480 def f(x): return x diff --git a/tests/run/for_from_float_T254.pyx b/tests/run/for_from_float_T254.pyx index 0ea153a9c..bf07f42b6 100644 --- a/tests/run/for_from_float_T254.pyx +++ b/tests/run/for_from_float_T254.pyx @@ -1,4 +1,4 @@ -# ticket: 254 +# ticket: t254 def double_target(a, b): """ diff --git a/tests/run/for_from_pyvar_loop_T601.pyx b/tests/run/for_from_pyvar_loop_T601.pyx index 2d5890d9b..949fcef06 100644 --- a/tests/run/for_from_pyvar_loop_T601.pyx +++ b/tests/run/for_from_pyvar_loop_T601.pyx @@ -1,4 +1,4 @@ -# ticket: 601 +# ticket: t601 cdef unsigned long size2(): return 3 diff --git a/tests/run/for_in_break_continue_T533.pyx b/tests/run/for_in_break_continue_T533.pyx index 0baa9fa49..32f646aaf 100644 --- a/tests/run/for_in_break_continue_T533.pyx +++ b/tests/run/for_in_break_continue_T533.pyx @@ -1,4 +1,4 @@ -# ticket: 533 +# ticket: t533 def for_in(): """ diff --git a/tests/run/for_in_range_T372.pyx b/tests/run/for_in_range_T372.pyx index bff686295..11ef4b592 100644 --- a/tests/run/for_in_range_T372.pyx +++ b/tests/run/for_in_range_T372.pyx @@ -1,5 +1,5 @@ # mode: run -# ticket: 372 +# ticket: t372 cimport cython diff --git a/tests/run/fstring.pyx b/tests/run/fstring.pyx index 45bfaf5e3..88666574c 100644 --- a/tests/run/fstring.pyx +++ b/tests/run/fstring.pyx @@ -18,6 +18,65 @@ min_long = LONG_MIN @cython.test_fail_if_path_exists( + "//JoinedStrNode", +) +@cython.test_assert_path_exists( + "//AddNode", +) +def concat_strings(a, b): + """ + >>> concat_strings("", "") + x + <BLANKLINE> + x + x + x + xx + >>> concat_strings("a", "") + ax + a + x + ax + ax + axx + >>> concat_strings("", "b") + x + b + xb + xb + xb + xxb + >>> concat_strings("a", "b") + ax + ab + xb + axb + axb + axxb + >>> concat_strings("".join(["a", "b"]), "") # fresh temp string left + abx + ab + x + abx + abx + abxx + >>> concat_strings("", "".join(["a", "b"])) # fresh temp string right + x + ab + xab + xab + xab + xxab + """ + print(f"{a}x") + print(f"{a}{b}") + print(f"x{b}") + print(f"{a+'x'}{b}") # fresh temp string left + print(f"{a}{'x'+b}") # fresh temp string right + print(f"{a+'x'}{'x'+b}") # fresh temp strings right and left + + +@cython.test_fail_if_path_exists( "//FormattedValueNode", "//JoinedStrNode", "//AddNode", @@ -519,6 +578,27 @@ def percent_s_unicode(u, int i): return u"%s-%d" % (u, i) +@cython.test_assert_path_exists( + "//FormattedValueNode", +) +def sideeffect(l): + """ + >>> class Listish(list): + ... def __format__(self, format_spec): + ... self.append("format called") + ... return repr(self) + ... def append(self, item): + ... list.append(self, item) + ... return self + + >>> l = Listish() + >>> sideeffect(l) if getattr(sys, 'pypy_version_info', ())[:2] != (7,3) else [123, 'format called'] # 7.3.4, 7.3.5 + [123, 'format called'] + """ + f"{l.append(123)}" # unused f-string ! + return list(l) + + ######################################## # await inside f-string diff --git a/tests/run/funcexc_iter_T228.pyx b/tests/run/funcexc_iter_T228.pyx index 0c5bde250..4b81166f6 100644 --- a/tests/run/funcexc_iter_T228.pyx +++ b/tests/run/funcexc_iter_T228.pyx @@ -1,4 +1,4 @@ -# ticket: 228 +# ticket: t228 __doc__ = u""" >>> def py_iterator(): diff --git a/tests/run/function_as_method_T494.pyx b/tests/run/function_as_method_T494.pyx index 34728bc8c..92deafc6d 100644 --- a/tests/run/function_as_method_T494.pyx +++ b/tests/run/function_as_method_T494.pyx @@ -1,4 +1,4 @@ -# ticket: 494 +# ticket: t494 # cython: binding=True __doc__ = """ @@ -12,3 +12,39 @@ class A: def foo(self): return self is not None + +# assignment of functions used in a "static method" type way behaves differently +# in Python2 and 3 +import sys +if sys.version_info[0] == 2: + __doc__ = """>>> B.plus1(1) #doctest: +IGNORE_EXCEPTION_DETAIL +Traceback (most recent call last): + ... +TypeError: unbound +""" +else: + __doc__ = """>>> B.plus1(1) +2 +""" + +# with binding==False assignment of functions always worked - doesn't match Python +# behaviour but ensures Cython behaviour stays consistent +__doc__ += """ +>>> B.plus1_nobind(1) +2 +""" + +cimport cython + +def f_plus(a): + return a + 1 + +@cython.binding(False) +def f_plus_nobind(a): + return a+1 + +cdef class B: + plus1 = f_plus + plus1_nobind = f_plus_nobind + + diff --git a/tests/run/function_as_method_py_T494.py b/tests/run/function_as_method_py_T494.py index 49d5f27ce..5317d8848 100644 --- a/tests/run/function_as_method_py_T494.py +++ b/tests/run/function_as_method_py_T494.py @@ -1,4 +1,4 @@ -# ticket: 494 +# ticket: t494 __doc__ = """ >>> A.foo = foo @@ -11,3 +11,35 @@ class A: def foo(self): return self is not None + + +# assignment of functions used in a "static method" type way behaves differently +# in Python2 and 3 +import sys +if sys.version_info[0] == 2: + __doc__ = u""" +>>> B.plus1(1) #doctest: +IGNORE_EXCEPTION_DETAIL +Traceback (most recent call last): + ... +TypeError: unbound +>>> C.plus1(1) #doctest: +IGNORE_EXCEPTION_DETAIL +Traceback (most recent call last): + ... +TypeError: unbound +""" +else: + __doc__ = u""" +>>> B.plus1(1) +2 +>>> C.plus1(1) +2 +""" + +def f_plus(a): + return a + 1 + +class B: + plus1 = f_plus + +class C(object): + plus1 = f_plus diff --git a/tests/run/function_binding_T494.pyx b/tests/run/function_binding_T494.pyx index b41221e2c..86c64c9c0 100644 --- a/tests/run/function_binding_T494.pyx +++ b/tests/run/function_binding_T494.pyx @@ -1,4 +1,4 @@ -# ticket: 494 +# ticket: t494 cimport cython diff --git a/tests/run/function_self.py b/tests/run/function_self.py new file mode 100644 index 000000000..e4796c46b --- /dev/null +++ b/tests/run/function_self.py @@ -0,0 +1,91 @@ +# mode: run +# tag: pure2.7 + +# cython: binding=True + +import cython +import sys + +def regular(x): + """ + >>> hasattr(regular, "__self__") + False + >>> nested = regular(10) + >>> hasattr(nested, "__self__") + False + """ + def nested(y): + return x+y + return nested + +@cython.locals(x=cython.floating) +def fused(x): + """ + >>> nested = fused(10.) + >>> hasattr(nested, "__self__") + False + + #>>> hasattr(fused, "__self__") # FIXME this fails for fused functions + #False + >>> fused.__self__ # but this is OK + Traceback (most recent call last): + ... + AttributeError: 'function' object has no attribute '__self__' + """ + def nested_in_fused(y): + return x+y + return nested_in_fused + +# FIXME - doesn't currently work at all +#def get_nested_fused(x): +# @cython.locals(x=cython.floating) +# def nested_fused(y): +# return x+y +# return nested_fused + +class C: + """ + >>> c = C() + >>> c.regular.__self__ is c + True + >>> c.fused.__self__ is c + True + """ + def regular(self): + pass + + @cython.locals(x=cython.floating) + def fused(self, x): + return x + +__doc__ = "" +if sys.version_info[0] > 2 or cython.compiled: + __doc__ += """ + >>> hasattr(C.regular, "__self__") # __self__==None on pure-python 2 + False + >>> C.fused.__self__ # returns None on pure-python 2 + Traceback (most recent call last): + ... + AttributeError: 'function' object has no attribute '__self__' + """ + +if cython.compiled: + __doc__ = """ + >>> fused['double'].__self__ + Traceback (most recent call last): + ... + AttributeError: 'function' object has no attribute '__self__' + + >>> C.fused['double'].__self__ + Traceback (most recent call last): + ... + AttributeError: 'function' object has no attribute '__self__' + + >>> c = C() + >>> c.fused['double'].__self__ is c + True + + # The PR that changed __self__ also changed how __doc__ is set up slightly + >>> fused['double'].__doc__ == fused.__doc__ and isinstance(fused.__doc__, str) + True + """ diff --git a/tests/run/fused_bound_functions.py b/tests/run/fused_bound_functions.py new file mode 100644 index 000000000..650bc51ff --- /dev/null +++ b/tests/run/fused_bound_functions.py @@ -0,0 +1,151 @@ +# mode: run +# tag: pure3.0 +# cython: binding=True + +""" +Test that fused functions can be used in the same way as CyFunctions with respect to +assigning them to class attributes. Previously they enforced extra type/argument checks +beyond those which CyFunctions did. +""" + +import cython + +MyFusedClass = cython.fused_type( + float, + 'Cdef', + object) + +def fused_func(x: MyFusedClass): + return (type(x).__name__, cython.typeof(x)) + +IntOrFloat = cython.fused_type(int, float) + +def fused_func_0(x: IntOrFloat = 0): + """ + Fused functions can legitimately take 0 arguments + >>> fused_func_0() + ('int', 'int') + + # subscripted in module __doc__ conditionally + """ + return (type(x).__name__, cython.typeof(x)) + +def regular_func(x): + return (type(x).__name__, cython.typeof(x)) + +def regular_func_0(): + return + +@cython.cclass +class Cdef: + __doc__ = """ + >>> c = Cdef() + + # functions are callable with an instance of c + >>> c.fused_func() + ('Cdef', 'Cdef') + >>> c.regular_func() + ('Cdef', '{typeofCdef}') + >>> c.fused_in_class(1.5) + ('float', 'float') + + # Fused functions are callable without an instance + # (This applies to everything in Py3 - see __doc__ below) + >>> Cdef.fused_func(1.5) + ('float', 'float') + >>> Cdef.fused_in_class(c, 1.5) + ('float', 'float') + >>> Cdef.fused_func_0() + ('int', 'int') + + # Functions not expecting an argument don't work with an instance + >>> c.regular_func_0() # doctest: +ELLIPSIS + Traceback (most recent call last): + TypeError: regular_func_0() takes ... arguments ...1... given... + """.format(typeofCdef = 'Python object' if cython.compiled else 'Cdef') + + if cython.compiled: + __doc__ += """ + + # fused_func_0 does not accept a "Cdef" instance + >>> c.fused_func_0() + Traceback (most recent call last): + TypeError: No matching signature found + + # subscripting requires fused methods (so not pure Python) + >>> Cdef.fused_func_0['float']() + ('float', 'float') + >>> c.fused_func_0['float']() # doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + TypeError: (Exception looks quite different in Python2 and 3 so no way to match both) + """ + fused_func = fused_func + fused_func_0 = fused_func_0 + regular_func = regular_func + regular_func_0 = regular_func_0 + + def fused_in_class(self, x: MyFusedClass): + return (type(x).__name__, cython.typeof(x)) + + def regular_in_class(self): + return type(self).__name__ + +class Regular(object): + __doc__ = """ + >>> c = Regular() + + # Functions are callable with an instance of C + >>> c.fused_func() + ('Regular', '{typeofRegular}') + >>> c.regular_func() + ('Regular', '{typeofRegular}') + + # Fused functions are callable without an instance + # (This applies to everything in Py3 - see __doc__ below) + >>> Regular.fused_func(1.5) + ('float', 'float') + >>> Regular.fused_func_0() + ('int', 'int') + + # Functions not expecting an argument don't work with an instance + >>> c.regular_func_0() # doctest: +ELLIPSIS + Traceback (most recent call last): + TypeError: regular_func_0() takes ... arguments ...1... given... + """.format(typeofRegular = "Python object" if cython.compiled else 'Regular') + if cython.compiled: + __doc__ += """ + # fused_func_0 does not accept a "Regular" instance + >>> c.fused_func_0() + Traceback (most recent call last): + TypeError: No matching signature found + + # subscripting requires fused methods (so not pure Python) + >>> c.fused_func_0['float']() # doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + TypeError: (Exception looks quite different in Python2 and 3 so no way to match both) + >>> Regular.fused_func_0['float']() + ('float', 'float') + """ + + fused_func = fused_func + fused_func_0 = fused_func_0 + regular_func = regular_func + regular_func_0 = regular_func_0 + +import sys +if sys.version_info[0] > 2: + # extra Py3 only tests - shows that functions added to a class can be called + # with an type as the first argument + __doc__ = """ + >>> Cdef.regular_func(1.5) + ('float', '{typeoffloat}') + >>> Regular.regular_func(1.5) + ('float', '{typeoffloat}') + >>> Cdef.regular_func_0() + >>> Regular.regular_func_0() + """.format(typeoffloat='Python object' if cython.compiled else 'float') +if cython.compiled: + __doc__ += """ + >>> fused_func_0['float']() + ('float', 'float') + """ diff --git a/tests/run/fused_cpdef.pyx b/tests/run/fused_cpdef.pyx index 0b63c8b98..4a614e0f4 100644 --- a/tests/run/fused_cpdef.pyx +++ b/tests/run/fused_cpdef.pyx @@ -1,13 +1,17 @@ +# cython: language_level=3 +# mode: run + cimport cython +import sys, io cy = __import__("cython") cpdef func1(self, cython.integral x): - print "%s," % (self,), + print(f"{self},", end=' ') if cython.integral is int: - print 'x is int', x, cython.typeof(x) + print('x is int', x, cython.typeof(x)) else: - print 'x is long', x, cython.typeof(x) + print('x is long', x, cython.typeof(x)) class A(object): @@ -16,6 +20,18 @@ class A(object): def __str__(self): return "A" +cdef class B: + cpdef int meth(self, cython.integral x): + print(f"{self},", end=' ') + if cython.integral is int: + print('x is int', x, cython.typeof(x)) + else: + print('x is long', x, cython.typeof(x)) + return 0 + + def __str__(self): + return "B" + pyfunc = func1 def test_fused_cpdef(): @@ -32,23 +48,71 @@ def test_fused_cpdef(): A, x is long 2 long A, x is long 2 long A, x is long 2 long + <BLANKLINE> + B, x is long 2 long """ func1[int](None, 2) func1[long](None, 2) func1(None, 2) - print + print() pyfunc[cy.int](None, 2) pyfunc(None, 2) - print + print() A.meth[cy.int](A(), 2) A.meth(A(), 2) A().meth[cy.long](2) A().meth(2) + print() + + B().meth(2) + + +midimport_run = io.StringIO() +if sys.version_info.major < 3: + # Monkey-patch midimport_run.write to accept non-unicode strings under Python 2. + midimport_run.write = lambda c: io.StringIO.write(midimport_run, unicode(c)) + +realstdout = sys.stdout +sys.stdout = midimport_run + +try: + # Run `test_fused_cpdef()` during import and save the result for + # `test_midimport_run()`. + test_fused_cpdef() +except Exception as e: + midimport_run.write(f"{e!r}\n") +finally: + sys.stdout = realstdout + +def test_midimport_run(): + # At one point, dynamically calling fused cpdef functions during import + # would fail because the type signature-matching indices weren't + # yet initialized. + # (See Compiler.FusedNode.FusedCFuncDefNode._fused_signature_index, + # GH-3366.) + """ + >>> test_midimport_run() + None, x is int 2 int + None, x is long 2 long + None, x is long 2 long + <BLANKLINE> + None, x is int 2 int + None, x is long 2 long + <BLANKLINE> + A, x is int 2 int + A, x is long 2 long + A, x is long 2 long + A, x is long 2 long + <BLANKLINE> + B, x is long 2 long + """ + print(midimport_run.getvalue(), end='') + def assert_raise(func, *args): try: @@ -70,23 +134,31 @@ def test_badcall(): assert_raise(A.meth) assert_raise(A().meth[cy.int]) assert_raise(A.meth[cy.int]) + assert_raise(B().meth, 1, 2, 3) + +def test_nomatch(): + """ + >>> func1(None, ()) + Traceback (most recent call last): + TypeError: No matching signature found + """ ctypedef long double long_double cpdef multiarg(cython.integral x, cython.floating y): if cython.integral is int: - print "x is an int,", + print("x is an int,", end=' ') else: - print "x is a long,", + print("x is a long,", end=' ') if cython.floating is long_double: - print "y is a long double:", + print("y is a long double:", end=' ') elif float is cython.floating: - print "y is a float:", + print("y is a float:", end=' ') else: - print "y is a double:", + print("y is a double:", end=' ') - print x, y + print(x, y) def test_multiarg(): """ @@ -104,3 +176,13 @@ def test_multiarg(): multiarg[int, float](1, 2.0) multiarg[cy.int, cy.float](1, 2.0) multiarg(4, 5.0) + +def test_ambiguousmatch(): + """ + >>> multiarg(5, ()) + Traceback (most recent call last): + TypeError: Function call with ambiguous argument types + >>> multiarg((), 2.0) + Traceback (most recent call last): + TypeError: Function call with ambiguous argument types + """ diff --git a/tests/run/fused_cpp.pyx b/tests/run/fused_cpp.pyx index c5a0d7d34..9f3bb5104 100644 --- a/tests/run/fused_cpp.pyx +++ b/tests/run/fused_cpp.pyx @@ -2,6 +2,8 @@ cimport cython from libcpp.vector cimport vector +from libcpp.typeinfo cimport type_info +from cython.operator cimport typeid def test_cpp_specialization(cython.floating element): """ @@ -14,3 +16,28 @@ def test_cpp_specialization(cython.floating element): cdef vector[cython.floating] *v = new vector[cython.floating]() v.push_back(element) print cython.typeof(v), cython.typeof(element), v.at(0) + +cdef fused C: + int + object + +cdef const type_info* tidint = &typeid(int) +def typeid_call(C x): + """ + For GH issue 3203 + >>> typeid_call(1) + True + """ + cdef const type_info* a = &typeid(C) + return a[0] == tidint[0] + +cimport cython + +def typeid_call2(cython.integral x): + """ + For GH issue 3203 + >>> typeid_call2[int](1) + True + """ + cdef const type_info* a = &typeid(cython.integral) + return a[0] == tidint[0] diff --git a/tests/run/fused_def.pyx b/tests/run/fused_def.pyx index f0e1be4c8..0f5ec1bbd 100644 --- a/tests/run/fused_def.pyx +++ b/tests/run/fused_def.pyx @@ -54,9 +54,13 @@ f = 5.6 i = 9 -def opt_func(fused_t obj, cython.floating myf = 1.2, cython.integral myi = 7): +def opt_func(fused_t obj, cython.floating myf = 1.2, cython.integral myi = 7, + another_opt = 2, yet_another_opt=3): """ - Test runtime dispatch, indexing of various kinds and optional arguments + Test runtime dispatch, indexing of various kinds and optional arguments. + Use 5 arguments because at one point the optional argument from the + 5th argument was overwriting that of the __pyx_fused dispatcher. + https://github.com/cython/cython/issues/3511 >>> opt_func("spam", f, i) str object double long @@ -121,9 +125,9 @@ def opt_func(fused_t obj, cython.floating myf = 1.2, cython.integral myi = 7): >>> opt_func() Traceback (most recent call last): TypeError: Expected at least 1 argument, got 0 - >>> opt_func("abc", f, i, 5) # doctest: +ELLIPSIS + >>> opt_func("abc", f, i, 5, 5, 5) # doctest: +ELLIPSIS Traceback (most recent call last): - TypeError: ...at most 3... + TypeError: ...at most 5... >>> opt_func[ExtClassA, cy.float, cy.long](object(), f) Traceback (most recent call last): TypeError: Argument 'obj' has incorrect type (expected fused_def.ExtClassA, got object) @@ -139,7 +143,7 @@ def run_cyfunction_check(): fused_cython_function 1 """ - print(type(opt_func).__name__) + print(type(opt_func).__name__.rsplit('.', 1)[-1]) print(__Pyx_CyFunction_Check(opt_func)) # should be True def test_opt_func(): @@ -154,19 +158,19 @@ def test_opt_func(): def test_opt_func_introspection(): """ >>> opt_func.__defaults__ - (1.2, 7) + (1.2, 7, 2, 3) >>> opt_func.__kwdefaults__ >>> opt_func.__annotations__ {} >>> opt_func[str, float, int].__defaults__ - (1.2, 7) + (1.2, 7, 2, 3) >>> opt_func[str, float, int].__kwdefaults__ >>> opt_func[str, float, int].__annotations__ {} >>> opt_func[str, cy.double, cy.long].__defaults__ - (1.2, 7) + (1.2, 7, 2, 3) >>> opt_func[str, cy.double, cy.long].__kwdefaults__ >>> opt_func[str, cy.double, cy.long].__annotations__ {} diff --git a/tests/run/fused_types.pyx b/tests/run/fused_types.pyx index 4e96bab3a..3a0009bdd 100644 --- a/tests/run/fused_types.pyx +++ b/tests/run/fused_types.pyx @@ -1,4 +1,5 @@ # mode: run +# ticket: t1772 cimport cython from cython.view cimport array @@ -20,6 +21,7 @@ ctypedef double *p_double ctypedef int *p_int fused_type3 = cython.fused_type(int, double) fused_composite = cython.fused_type(fused_type2, fused_type3) +just_float = cython.fused_type(float) def test_pure(): """ @@ -363,6 +365,22 @@ def test_fused_memslice_dtype_repeated_2(cython.floating[:] array1, cython.float """ print cython.typeof(array1), cython.typeof(array2), cython.typeof(array3) +def test_fused_const_memslice_dtype_repeated(const cython.floating[:] array1, cython.floating[:] array2): + """Test fused types memory view with one being const + + >>> sorted(test_fused_const_memslice_dtype_repeated.__signatures__) + ['double', 'float'] + + >>> test_fused_const_memslice_dtype_repeated(get_array(8, 'd'), get_array(8, 'd')) + const double[:] double[:] + >>> test_fused_const_memslice_dtype_repeated(get_array(4, 'f'), get_array(4, 'f')) + const float[:] float[:] + >>> test_fused_const_memslice_dtype_repeated(get_array(8, 'd'), get_array(4, 'f')) + Traceback (most recent call last): + ValueError: Buffer dtype mismatch, expected 'double' but got 'float' + """ + print cython.typeof(array1), cython.typeof(array2) + def test_cython_numeric(cython.numeric arg): """ Test to see whether complex numbers have their utility code declared @@ -388,6 +406,18 @@ def test_index_fused_args(cython.floating f, ints_t i): """ _test_index_fused_args[cython.floating, ints_t](f, i) +cdef _test_index_const_fused_args(const cython.floating f, const ints_t i): + print(cython.typeof(f), cython.typeof(i)) + +def test_index_const_fused_args(const cython.floating f, const ints_t i): + """Test indexing function implementation with const fused type args + + >>> import cython + >>> test_index_const_fused_args[cython.double, cython.int](2.0, 3) + ('const double', 'const int') + """ + _test_index_const_fused_args[cython.floating, ints_t](f, i) + def test_composite(fused_composite x): """ @@ -404,6 +434,60 @@ def test_composite(fused_composite x): return 2 * x +cdef cdef_func_const_fused_arg(const cython.floating val, + const fused_type1 * ptr_to_const, + const (cython.floating *) const_ptr): + print(val, cython.typeof(val)) + print(ptr_to_const[0], cython.typeof(ptr_to_const[0])) + print(const_ptr[0], cython.typeof(const_ptr[0])) + + ptr_to_const = NULL # pointer is not const, value is const + const_ptr[0] = 0.0 # pointer is const, value is not const + +def test_cdef_func_with_const_fused_arg(): + """Test cdef function with const fused type argument + + >>> test_cdef_func_with_const_fused_arg() + (0.0, 'const float') + (1, 'const int') + (2.0, 'float') + """ + cdef float arg0 = 0.0 + cdef int arg1 = 1 + cdef float arg2 = 2.0 + cdef_func_const_fused_arg(arg0, &arg1, &arg2) + + +cdef in_check_1(just_float x): + return just_float in floating + +cdef in_check_2(just_float x, floating y): + # the "floating" on the right-hand side of the in statement should not be specialized + # - the test should still work. + return just_float in floating + +cdef in_check_3(floating x): + # the floating on the left-hand side of the in statement should be specialized + # but the one of the right-hand side should not (so that the test can still work). + return floating in floating + +def test_fused_in_check(): + """ + It should be possible to use fused types on in "x in ...fused_type" statements + even if that type is specialized in the function. + + >>> test_fused_in_check() + True + True + True + True + """ + print(in_check_1(1.0)) + print(in_check_2(1.0, 2.0)) + print(in_check_2[float, double](1.0, 2.0)) + print(in_check_3[float](1.0)) + + ### see GH3642 - presence of cdef inside "unrelated" caused a type to be incorrectly inferred cdef unrelated(cython.floating x): cdef cython.floating t = 1 diff --git a/tests/run/generator_expressions_and_locals.pyx b/tests/run/generator_expressions_and_locals.pyx index 7a87164ff..a239f9b29 100644 --- a/tests/run/generator_expressions_and_locals.pyx +++ b/tests/run/generator_expressions_and_locals.pyx @@ -1,6 +1,6 @@ # mode: run # tag: genexpr, locals -# ticket: 715 +# ticket: t715 def genexpr_not_in_locals(): """ diff --git a/tests/run/generators.pyx b/tests/run/generators.pyx index 9faa9d4cf..6314ee8f0 100644 --- a/tests/run/generators.pyx +++ b/tests/run/generators.pyx @@ -504,6 +504,26 @@ def test_generator_abc(): yield 1 +def test_generator_frame(a=1): + """ + >>> gen = test_generator_frame() + >>> import types + >>> isinstance(gen.gi_frame, types.FrameType) or gen.gi_frame + True + >>> gen.gi_frame is gen.gi_frame # assert that it's cached + True + >>> gen.gi_frame.f_code is not None + True + >>> code_obj = gen.gi_frame.f_code + >>> code_obj.co_argcount + 1 + >>> code_obj.co_varnames + ('a', 'b') + """ + b = a + 1 + yield b + + # GH Issue 3265 - **kwds could cause a crash in some cases due to not # handling NULL pointers (in testing it shows as a REFNANNY error). # This was on creation of the generator and diff --git a/tests/run/generators_GH1731.pyx b/tests/run/generators_GH1731.pyx index 99cae90ca..77b0877f2 100644 --- a/tests/run/generators_GH1731.pyx +++ b/tests/run/generators_GH1731.pyx @@ -1,5 +1,5 @@ # mode: run -# ticket: gh1731 +# ticket: 1731 def cygen(): diff --git a/tests/run/genexpr_T491.pyx b/tests/run/genexpr_T491.pyx index 7fa6c1754..7640b3518 100644 --- a/tests/run/genexpr_T491.pyx +++ b/tests/run/genexpr_T491.pyx @@ -1,4 +1,4 @@ -# ticket: 491 +# ticket: t491 def test_genexpr(): """ diff --git a/tests/run/genexpr_T715.pyx b/tests/run/genexpr_T715.pyx index 2c6f5d6c8..028a93128 100644 --- a/tests/run/genexpr_T715.pyx +++ b/tests/run/genexpr_T715.pyx @@ -1,5 +1,5 @@ # mode: run -# ticket: 715 +# ticket: t715 # tag: genexpr, comprehension def t715(*items): diff --git a/tests/run/genexpr_iterable_lookup_T600.pyx b/tests/run/genexpr_iterable_lookup_T600.pyx index 220098a1f..945652717 100644 --- a/tests/run/genexpr_iterable_lookup_T600.pyx +++ b/tests/run/genexpr_iterable_lookup_T600.pyx @@ -1,5 +1,5 @@ # mode: run -# ticket: 600 +# ticket: t600 # tag: genexpr # cython: language_level=3 diff --git a/tests/run/hash_T326.pyx b/tests/run/hash_T326.pyx index 11184837f..bc9566879 100644 --- a/tests/run/hash_T326.pyx +++ b/tests/run/hash_T326.pyx @@ -1,5 +1,5 @@ # mode: run -# ticket: 326 +# ticket: t326 # tag: hash diff --git a/tests/run/if_and_or.pyx b/tests/run/if_and_or.pyx new file mode 100644 index 000000000..6f2534078 --- /dev/null +++ b/tests/run/if_and_or.pyx @@ -0,0 +1,119 @@ +# mode: run +# tag: if, and, or + +def if_x(x): + """ + >>> if_x(0) + 2 + >>> if_x(1) + 1 + """ + if x: + return 1 + else: + return 2 + +def if_not(x): + """ + >>> if_not(0) + 1 + >>> if_not(1) + 2 + """ + if not x: + return 1 + else: + return 2 + + +def if_and(a, b): + """ + >>> if_and(3, 0) + 2 + >>> if_and(0, 3) + 2 + >>> if_and(0, 0) + 2 + >>> if_and(3, 3) + 1 + """ + if a and b: + return 1 + else: + return 2 + + +def if_not_and(a, b): + """ + >>> if_not_and(3, 0) + 1 + >>> if_not_and(0, 3) + 1 + >>> if_not_and(0, 0) + 1 + >>> if_not_and(3, 3) + 2 + """ + if not (a and b): + return 1 + else: + return 2 + + +def if_or(a, b): + """ + >>> if_or(3, 0) + 1 + >>> if_or(0, 3) + 1 + >>> if_or(0, 0) + 2 + >>> if_or(3, 3) + 1 + """ + if a or b: + return 1 + else: + return 2 + + +def if_not_or(a, b): + """ + >>> if_not_or(3, 0) + 2 + >>> if_not_or(0, 3) + 2 + >>> if_not_or(0, 0) + 1 + >>> if_not_or(3, 3) + 2 + """ + if not (a or b): + return 1 + else: + return 2 + + +def if_and_or(a, b, c, d): + """ + >>> if_and_or(3, 0, 0, 3) + 1 + >>> if_and_or(0, 3, 0, 3) + 1 + >>> if_and_or(0, 3, 3, 0) + 1 + >>> if_and_or(0, 3, 3, 0) + 1 + >>> if_and_or(0, 0, 0, 0) + 2 + >>> if_and_or(0, 3, 0, 0) + 2 + >>> if_and_or(0, 0, 3, 0) + 2 + >>> if_and_or(0, 0, 0, 3) + 2 + """ + if (a or b) and (c or d): + return 1 + else: + return 2 diff --git a/tests/run/ifelseexpr_T267.pyx b/tests/run/ifelseexpr_T267.pyx index 24dabc442..973c1979a 100644 --- a/tests/run/ifelseexpr_T267.pyx +++ b/tests/run/ifelseexpr_T267.pyx @@ -1,6 +1,6 @@ # mode: run # tag: condexpr -# ticket: 267 +# ticket: t267 cimport cython diff --git a/tests/run/import_error_T734.py b/tests/run/import_error_T734.py index 4138fba1c..efcd79944 100644 --- a/tests/run/import_error_T734.py +++ b/tests/run/import_error_T734.py @@ -1,5 +1,5 @@ # mode: run -# ticket: 734 +# ticket: t734 def test_import_error(): """ diff --git a/tests/run/importas.pyx b/tests/run/importas.pyx index c57b057a2..6b3c98304 100644 --- a/tests/run/importas.pyx +++ b/tests/run/importas.pyx @@ -1,4 +1,14 @@ +# mode: run +# tag: all_language_levels + __doc__ = u""" +>>> try: sys +... except NameError: pass +... else: print("sys was defined!") +>>> try: distutils +... except NameError: pass +... else: print("distutils was defined!") + >>> import sys as sous >>> import distutils.core as corey >>> from copy import deepcopy as copey diff --git a/tests/run/importfrom.pyx b/tests/run/importfrom.pyx index 52bc67611..a3e5de78d 100644 --- a/tests/run/importfrom.pyx +++ b/tests/run/importfrom.pyx @@ -68,6 +68,10 @@ def typed_imports(): try: from sys import version_info as maxunicode except TypeError, e: + if getattr(sys, "pypy_version_info", None): + # translate message + if e.args[0].startswith("int() argument must be"): + e = "an integer is required" print(e) try: diff --git a/tests/run/in_list_with_side_effects_T544.pyx b/tests/run/in_list_with_side_effects_T544.pyx index 3bb3954c3..4a75f206a 100644 --- a/tests/run/in_list_with_side_effects_T544.pyx +++ b/tests/run/in_list_with_side_effects_T544.pyx @@ -1,4 +1,4 @@ -# ticket: 544 +# ticket: t544 def count(i=[0]): i[0] += 1 diff --git a/tests/run/include_multiple_modules.srctree b/tests/run/include_multiple_modules.srctree new file mode 100644 index 000000000..0bd768301 --- /dev/null +++ b/tests/run/include_multiple_modules.srctree @@ -0,0 +1,31 @@ +PYTHON setup.py build_ext --inplace + +############# setup.py ############# + +from Cython.Build.Dependencies import cythonize +from distutils.core import setup + +setup( + ext_modules = cythonize(["a.pyx", "b.pyx", "include_both.pyx"]), + ) + +############# a.pyx ############### + +cdef public f(): + pass + +############# b.pyx ############### + +cdef public g(): + pass + +############# include_both.pyx #### + +# This is just checking that a and b don't duplicate any names +# and thus it's possible to include them both in one place + +cdef extern from "a.h": + pass + +cdef extern from "b.h": + pass diff --git a/tests/run/initial_file_path.srctree b/tests/run/initial_file_path.srctree index a55df688b..e90cf6866 100644 --- a/tests/run/initial_file_path.srctree +++ b/tests/run/initial_file_path.srctree @@ -29,8 +29,8 @@ except ImportError as e: traceback.print_exc() def test(): - print "FILE: ", initial_file - print "PATH: ", initial_path + print("FILE: ", initial_file) + print("PATH: ", initial_path) assert initial_path[0].endswith('my_test_package'), initial_path assert initial_file.endswith('__init__.py'), initial_file assert import_error is None, import_error @@ -51,8 +51,8 @@ except ImportError as e: traceback.print_exc() def test(): - print "FILE: ", initial_file - print "PATH: ", initial_path + print("FILE: ", initial_file) + print("PATH: ", initial_path) assert initial_path[0].endswith('another'), initial_path assert initial_file.endswith('__init__.py'), initial_file assert import_error is None, import_error diff --git a/tests/run/inlinepxd.pyx b/tests/run/inlinepxd.pyx index 3d724f7d3..65c596c36 100644 --- a/tests/run/inlinepxd.pyx +++ b/tests/run/inlinepxd.pyx @@ -1,3 +1,8 @@ +# mode: run +# tag: inline, pxd + +# cython: wraparound = False + __doc__ = u""" >>> f() 3 @@ -28,3 +33,12 @@ def i(): def j(): return my_add3(2, 4) + +def test_wraparound(): + """ + >>> test_wraparound() + 1.0 + """ + # the wraparound directive from this scope should not affect the inline pxd + a = [ 0.0, 1.0 ] + return inlinepxd_support.index(a) diff --git a/tests/run/inlinepxd_support.pxd b/tests/run/inlinepxd_support.pxd index c2941e2b3..863f4eaf1 100644 --- a/tests/run/inlinepxd_support.pxd +++ b/tests/run/inlinepxd_support.pxd @@ -1,3 +1,9 @@ cdef inline int my_add(int a, int b=1, int c=0): return a + b + c + +cdef inline index(list L): + # This function should *not* be affected by directives set in the outer scope, such as "wraparound". + # See https://github.com/cython/cython/issues/1071 + return L[-1] + diff --git a/tests/run/int128.pyx b/tests/run/int128.pyx index cb18ccbd8..8e31ee141 100644 --- a/tests/run/int128.pyx +++ b/tests/run/int128.pyx @@ -117,3 +117,73 @@ def signed_conversion(x): """ cdef int128_t n = x return n + + +def get_int_distribution(shuffle=True): + """ + >>> L = get_int_distribution() + >>> bigint(L[0]) + 682 + >>> bigint(L[ len(L) // 2 ]) + 5617771410183435 + >>> bigint(L[-1]) + 52818775009509558395695966805 + >>> len(L) + 66510 + """ + # Large integers that cover 1-4 (30 bits) or 1-7 (15 bits) PyLong digits. + # Uses only integer calculations to avoid rounding issues. + pow2 = [2**exp for exp in range(98)] + ints = [ + n // 3 + for i in range(11, len(pow2) - 1) + # Take a low but growing number of integers from each power-of-2 range. + for n in range(pow2[i], pow2[i+1], pow2[i - 8] - 1) + ] + return ints * 3 # longer list, but keeps median in the middle + + +def intsum(L): + """ + >>> L = get_int_distribution() + >>> bigint(intsum(L)) + 61084913298497804284622382871263 + >>> bigint(sum(L)) + 61084913298497804284622382871263 + + >>> from random import shuffle + >>> shuffle(L) + >>> bigint(intsum(L)) + 61084913298497804284622382871263 + """ + cdef uint128_t i, x = 0 + for i in L: + x += i + return x + + +def intxor(L): + """ + >>> L = get_int_distribution() + >>> bigint(intxor(L)) + 31773794341658093722410838161 + >>> bigint(intxor(L * 2)) + 0 + >>> import operator + >>> from functools import reduce + >>> bigint(reduce(operator.xor, L)) + 31773794341658093722410838161 + >>> bigint(reduce(operator.xor, L * 2)) + 0 + + >>> from random import shuffle + >>> shuffle(L) + >>> bigint(intxor(L)) + 31773794341658093722410838161 + >>> bigint(intxor(L * 2)) + 0 + """ + cdef uint128_t i, x = 0 + for i in L: + x ^= i + return x diff --git a/tests/run/int_float_builtins_as_casts_T400.pyx b/tests/run/int_float_builtins_as_casts_T400.pyx index 6d02eff36..99d1796ed 100644 --- a/tests/run/int_float_builtins_as_casts_T400.pyx +++ b/tests/run/int_float_builtins_as_casts_T400.pyx @@ -1,4 +1,4 @@ -# ticket: 400 +# ticket: t400 cimport cython diff --git a/tests/run/int_float_builtins_as_casts_T400_long_double.pyx b/tests/run/int_float_builtins_as_casts_T400_long_double.pyx index d7e61f43b..776434ee8 100644 --- a/tests/run/int_float_builtins_as_casts_T400_long_double.pyx +++ b/tests/run/int_float_builtins_as_casts_T400_long_double.pyx @@ -1,4 +1,4 @@ -# ticket: 400 +# ticket: t400 cimport cython diff --git a/tests/run/intern_T431.pyx b/tests/run/intern_T431.pyx index 5851d9741..6b7ef0516 100644 --- a/tests/run/intern_T431.pyx +++ b/tests/run/intern_T431.pyx @@ -1,4 +1,4 @@ -# ticket: 431 +# ticket: t431 __doc__ = u""" >>> s == s_interned diff --git a/tests/run/ipow_crash_T562.pyx b/tests/run/ipow_crash_T562.pyx index 6fea958b5..7b074ea5d 100644 --- a/tests/run/ipow_crash_T562.pyx +++ b/tests/run/ipow_crash_T562.pyx @@ -1,4 +1,4 @@ -# ticket: 562 +# ticket: t562 class IPOW: """ diff --git a/tests/run/isnot.pyx b/tests/run/isnot.pyx index 9baabbc7e..f7efd017d 100644 --- a/tests/run/isnot.pyx +++ b/tests/run/isnot.pyx @@ -3,12 +3,17 @@ cimport cython +# Use a single global object for identity checks. +# PyPy can optimise away integer objects, for example, and may fail the 'is' test. +obj = object() + + @cython.test_fail_if_path_exists('//NotNode') def is_not(a, b): """ >>> is_not(1, 2) True - >>> x = 1 + >>> x = obj >>> is_not(x, x) False """ @@ -20,7 +25,7 @@ def not_is_not(a, b): """ >>> not_is_not(1, 2) False - >>> x = 1 + >>> x = obj >>> not_is_not(x, x) True """ @@ -32,7 +37,7 @@ def not_is(a, b): """ >>> not_is(1, 2) True - >>> x = 1 + >>> x = obj >>> not_is(x, x) False """ diff --git a/tests/run/iterdict.pyx b/tests/run/iterdict.pyx index 0e8eaba1c..e2d697e0f 100644 --- a/tests/run/iterdict.pyx +++ b/tests/run/iterdict.pyx @@ -555,3 +555,19 @@ def for_in_iteritems_of_expression(*args, **kwargs): for k, v in dict(*args, **kwargs).iteritems(): result.append((k, v)) return result + + +cdef class NotADict: + """ + >>> NotADict().listvalues() # doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + ... + TypeError: descriptor 'values' for 'mappingproxy' objects doesn't apply to a 'iterdict.NotADict' object + """ + cdef long v + def __cinit__(self): + self.v = 1 + itervalues = type(object.__dict__).values + + def listvalues(self): + return [v for v in self.itervalues()] diff --git a/tests/run/knuth_man_or_boy_test.pyx b/tests/run/knuth_man_or_boy_test.pyx index 068cec524..d2b5c8825 100644 --- a/tests/run/knuth_man_or_boy_test.pyx +++ b/tests/run/knuth_man_or_boy_test.pyx @@ -46,9 +46,13 @@ def compute(val): def a(in_k, x1, x2, x3, x4, x5): """ >>> import sys - >>> sys.setrecursionlimit(1350) + >>> old_limit = sys.getrecursionlimit() + >>> sys.setrecursionlimit(1350 if not getattr(sys, 'pypy_version_info', None) else 2700) + >>> a(10, 1, -1, -1, 1, 0) -67 + + >>> sys.setrecursionlimit(old_limit) """ k = [in_k] def b(): diff --git a/tests/run/kwargproblems.pyx b/tests/run/kwargproblems.pyx index 88e3ac53a..7984fbb08 100644 --- a/tests/run/kwargproblems.pyx +++ b/tests/run/kwargproblems.pyx @@ -4,7 +4,7 @@ def test(**kw): >>> d = {1 : 2} >>> test(**d) # doctest: +ELLIPSIS Traceback (most recent call last): - TypeError: ...keywords must be strings + TypeError: ...keywords must be strings... >>> d {1: 2} >>> d = {} diff --git a/tests/run/kwargs_passthrough.pyx b/tests/run/kwargs_passthrough.pyx index 576306efd..c09b6cba4 100644 --- a/tests/run/kwargs_passthrough.pyx +++ b/tests/run/kwargs_passthrough.pyx @@ -138,6 +138,18 @@ def wrap_modify_mix(f): >>> wrapped(a=2, test=3) CALLED (2, 1) + + >>> def py_modify(**kwargs): + ... print(sorted(kwargs.items())) + ... kwargs['new'] = len(kwargs) + ... return kwargs + + >>> wrapped_modify = wrap_modify_mix(py_modify) + >>> sorted(wrapped_modify(a=1).items()) + CALLED + [('a', 1)] + [('a', 1), ('test', 1)] + [('a', 1), ('new', 2), ('test', 1)] """ def wrapper(*args, **kwargs): print("CALLED") diff --git a/tests/run/lambda_T195.pyx b/tests/run/lambda_T195.pyx index fdbde9982..bfae08fea 100644 --- a/tests/run/lambda_T195.pyx +++ b/tests/run/lambda_T195.pyx @@ -1,6 +1,6 @@ # mode: run # tag: lambda -# ticket: 195 +# ticket: t195 __doc__ = u""" #>>> py_identity = lambda x:x diff --git a/tests/run/lambda_T723.pyx b/tests/run/lambda_T723.pyx index e746a3f58..39ae66851 100644 --- a/tests/run/lambda_T723.pyx +++ b/tests/run/lambda_T723.pyx @@ -1,5 +1,5 @@ # mode: run -# ticket: 723 +# ticket: t723 # tag: lambda def t723(a): diff --git a/tests/run/lambda_class_T605.pyx b/tests/run/lambda_class_T605.pyx index 82e1ff8a7..2efe77e7c 100644 --- a/tests/run/lambda_class_T605.pyx +++ b/tests/run/lambda_class_T605.pyx @@ -1,6 +1,6 @@ # mode: run # tag: lambda -# ticket: 605 +# ticket: t605 cdef int cdef_CONST = 123 CONST = 456 diff --git a/tests/run/lambda_module_T603.pyx b/tests/run/lambda_module_T603.pyx index 245f86937..c92fce5df 100644 --- a/tests/run/lambda_module_T603.pyx +++ b/tests/run/lambda_module_T603.pyx @@ -1,6 +1,6 @@ # mode: run # tag: lambda -# ticket: 603 +# ticket: t603 # Module scope lambda functions diff --git a/tests/run/large_consts_T237.pyx b/tests/run/large_consts_T237.pyx index f11a24541..7c448401f 100644 --- a/tests/run/large_consts_T237.pyx +++ b/tests/run/large_consts_T237.pyx @@ -1,4 +1,4 @@ -# ticket: 237 +# ticket: t237 #def add_large_c(): # cdef unsigned long long val = 2**30 + 2**30 # return val diff --git a/tests/run/letnode_T766.pyx b/tests/run/letnode_T766.pyx index 337808ee4..0f2d53e0b 100644 --- a/tests/run/letnode_T766.pyx +++ b/tests/run/letnode_T766.pyx @@ -1,5 +1,5 @@ # mode: run -# ticket: 766 +# ticket: t766 # tag: letnode def test_letnode_range(int n): diff --git a/tests/run/libc_math.pyx b/tests/run/libc_math.pyx index c3d768e82..9bb93678c 100644 --- a/tests/run/libc_math.pyx +++ b/tests/run/libc_math.pyx @@ -2,8 +2,8 @@ from libc.math cimport (M_E, M_LOG2E, M_LOG10E, M_LN2, M_LN10, M_PI, M_PI_2, M_PI_4, M_1_PI, M_2_PI, M_2_SQRTPI, M_SQRT2, M_SQRT1_2) -from libc.math cimport (acos, asin, atan, atan2, cos, sin, tan, cosh, sinh, - tanh, acosh, asinh, atanh, exp, log, log10, pow, sqrt) +from libc.math cimport (acos, asin, atan, atan2, cos, modf, sin, sinf, sinl, + tan, cosh, sinh, tanh, acosh, asinh, atanh, exp, log, log10, pow, sqrt) cimport libc.math as libc_math @@ -34,3 +34,21 @@ def test_sin(x): [True, True, True, True, True, True, True, True, True, True] """ return sin(x) + + +def test_sin_kwarg(x): + """ + >>> test_sin_kwarg(0) + 0.0 + """ + return sin(x=x) + + +def test_modf(x): + """ + >>> test_modf(2.5) + (0.5, 2.0) + """ + cdef double i + cdef double f = modf(x, &i) + return (f, i) diff --git a/tests/run/libcpp_algo.pyx b/tests/run/libcpp_algo.pyx index 40285cbd1..758da7705 100644 --- a/tests/run/libcpp_algo.pyx +++ b/tests/run/libcpp_algo.pyx @@ -1,12 +1,13 @@ +# mode: run # tag: cpp from libcpp cimport bool -from libcpp.algorithm cimport make_heap, sort_heap, sort, partial_sort +from libcpp.algorithm cimport make_heap, sort_heap from libcpp.vector cimport vector # XXX should use std::greater, but I don't know how to wrap that. -cdef inline bool greater(int x, int y): +cdef inline bool greater(const int &x, const int &y): return x > y @@ -27,33 +28,3 @@ def heapsort(l, bool reverse=False): sort_heap(v.begin(), v.end()) return v - - -def partialsort(l, int k, reverse=False): - """ - >>> partialsort([4, 2, 3, 1, 5], k=2)[:2] - [1, 2] - >>> partialsort([4, 2, 3, 1, 5], k=2, reverse=True)[:2] - [5, 4] - """ - cdef vector[int] v = l - if reverse: - partial_sort(v.begin(), v.begin() + k, v.end(), &greater) - else: - partial_sort(v.begin(), v.begin() + k, v.end()) - return v - - -def stdsort(l, reverse=False): - """ - >>> stdsort([3, 2, 1, 4, 5]) - [1, 2, 3, 4, 5] - >>> stdsort([3, 2, 1, 4, 5], reverse=True) - [5, 4, 3, 2, 1] - """ - cdef vector[int] v = l - if reverse: - sort(v.begin(), v.end(), &greater) - else: - sort(v.begin(), v.end()) - return v diff --git a/tests/run/libcpp_all.pyx b/tests/run/libcpp_all.pyx index 2930807d9..6633adb7a 100644 --- a/tests/run/libcpp_all.pyx +++ b/tests/run/libcpp_all.pyx @@ -1,9 +1,10 @@ -# tag: cpp +# tag: cpp, no-cpp-locals import cython cimport libcpp +# cimport libcpp.atomic cimport libcpp.deque cimport libcpp.list cimport libcpp.map @@ -15,6 +16,7 @@ cimport libcpp.vector cimport libcpp.complex cimport libcpp.limits +# from libcpp.atomic cimport * from libcpp.deque cimport * from libcpp.list cimport * from libcpp.map cimport * @@ -26,6 +28,7 @@ from libcpp.vector cimport * from libcpp.complex cimport * from libcpp.limits cimport * +# cdef libcpp.atomic.atomc[int] a1 = atomic[int]() cdef libcpp.deque.deque[int] d1 = deque[int]() cdef libcpp.list.list[int] l1 = list[int]() cdef libcpp.map.map[int,int] m1 = map[int,int]() diff --git a/tests/run/line_profile_test.srctree b/tests/run/line_profile_test.srctree index 5ea4ce665..80e4f2ae3 100644 --- a/tests/run/line_profile_test.srctree +++ b/tests/run/line_profile_test.srctree @@ -109,6 +109,9 @@ def cy_generator(int n): x = 1 for i in range(n): yield x + 2 + # waste some time to avoid 0 runtimes (line profiler cannot handle those) + while <object>(i + x) < n + 10: + i += 2 @cython.binding(True) diff --git a/tests/run/list_comp_in_closure_T598.pyx b/tests/run/list_comp_in_closure_T598.pyx index 3f418add1..45b572ac0 100644 --- a/tests/run/list_comp_in_closure_T598.pyx +++ b/tests/run/list_comp_in_closure_T598.pyx @@ -1,6 +1,6 @@ # mode: run # tag: closures -# ticket: 598 +# ticket: t598 # cython: language_level=3 def list_comp_in_closure(): diff --git a/tests/run/list_pop.pyx b/tests/run/list_pop.pyx index b1379f199..3e7c5bdf5 100644 --- a/tests/run/list_pop.pyx +++ b/tests/run/list_pop.pyx @@ -206,7 +206,7 @@ def crazy_pop(L): """ >>> crazy_pop(list(range(10))) # doctest: +ELLIPSIS Traceback (most recent call last): - TypeError: pop... at most ... argument... + TypeError: pop... argument... >>> crazy_pop(A()) (1, 2, 3) """ diff --git a/tests/run/locals_T732.pyx b/tests/run/locals_T732.pyx index d134948af..6ce8c7447 100644 --- a/tests/run/locals_T732.pyx +++ b/tests/run/locals_T732.pyx @@ -1,5 +1,5 @@ # mode: run -# ticket: 731 +# ticket: t731 # tag: locals, vars, dir cimport cython @@ -23,8 +23,8 @@ def test_class_locals_and_dir(): >>> klass = test_class_locals_and_dir() >>> 'visible' in klass.locs and 'not_visible' not in klass.locs True - >>> klass.names - ['__module__', '__qualname__', 'visible'] + >>> [n for n in klass.names if n not in {"__qualname__", "__annotations__"}] + ['__module__', 'visible'] """ not_visible = 1234 class Foo: diff --git a/tests/run/locals_expressions_T430.pyx b/tests/run/locals_expressions_T430.pyx index a0f8a0d62..a0e9dff3e 100644 --- a/tests/run/locals_expressions_T430.pyx +++ b/tests/run/locals_expressions_T430.pyx @@ -1,4 +1,4 @@ -# ticket: 430 +# ticket: t430 __doc__ = u""" >>> sorted( get_locals(1,2,3, k=5) .items()) diff --git a/tests/run/locals_rebind_T429.pyx b/tests/run/locals_rebind_T429.pyx index 9b3f06a5a..e84a13947 100644 --- a/tests/run/locals_rebind_T429.pyx +++ b/tests/run/locals_rebind_T429.pyx @@ -1,4 +1,4 @@ -# ticket: 429 +# ticket: t429 __doc__ = u""" >>> sorted( get_locals(1,2,3, k=5) .items()) diff --git a/tests/run/lvalue_refs.pyx b/tests/run/lvalue_refs.pyx index d42f2407e..c70744533 100644 --- a/tests/run/lvalue_refs.pyx +++ b/tests/run/lvalue_refs.pyx @@ -1,4 +1,4 @@ -# tag: cpp +# tag: cpp, no-cpp-locals from libcpp.vector cimport vector @@ -28,3 +28,15 @@ def test_lvalue_ref_assignment(): assert bar[0] == &baz[0][0] assert bar[0][0] == bongle + +# not *strictly* lvalue refs but this file seems the closest applicable place for it. +# GH 3754 - std::vector operator[] returns a reference, and this causes problems if +# the reference is passed into Cython __Pyx_GetItemInt +def test_ref_used_for_indexing(): + """ + >>> test_ref_used_for_indexing() + 'looked up correctly' + """ + cdef vector[int] idx = [1,2,3] + d = {1: "looked up correctly", 2:"oops"} + return d[idx[0]] diff --git a/tests/run/metaclass.pyx b/tests/run/metaclass.pyx index d5464fb17..9e4db4c8b 100644 --- a/tests/run/metaclass.pyx +++ b/tests/run/metaclass.pyx @@ -69,8 +69,8 @@ class Py3ClassMCOnly(object, metaclass=Py3MetaclassPlusAttr): 321 >>> obj.metaclass_was_here True - >>> obj._order - ['__module__', '__qualname__', '__doc__', 'bar', 'metaclass_was_here'] + >>> [n for n in obj._order if n not in {"__qualname__", "__annotations__"}] + ['__module__', '__doc__', 'bar', 'metaclass_was_here'] """ bar = 321 @@ -81,8 +81,8 @@ class Py3InheritedMetaclass(Py3ClassMCOnly): 345 >>> obj.metaclass_was_here True - >>> obj._order - ['__module__', '__qualname__', '__doc__', 'bar', 'metaclass_was_here'] + >>> [n for n in obj._order if n not in {"__qualname__", "__annotations__"}] + ['__module__', '__doc__', 'bar', 'metaclass_was_here'] """ bar = 345 @@ -109,8 +109,8 @@ class Py3Foo(object, metaclass=Py3Base, foo=123): 123 >>> obj.bar 321 - >>> obj._order - ['__module__', '__qualname__', '__doc__', 'bar', 'foo'] + >>> [n for n in obj._order if n not in {"__qualname__", "__annotations__"}] + ['__module__', '__doc__', 'bar', 'foo'] """ bar = 321 @@ -122,8 +122,8 @@ class Py3FooInherited(Py3Foo, foo=567): 567 >>> obj.bar 321 - >>> obj._order - ['__module__', '__qualname__', '__doc__', 'bar', 'foo'] + >>> [n for n in obj._order if n not in {"__qualname__", "__annotations__"}] + ['__module__', '__doc__', 'bar', 'foo'] """ bar = 321 diff --git a/tests/run/method_module_name_T422.pyx b/tests/run/method_module_name_T422.pyx index fbefa2144..731cd1b30 100644 --- a/tests/run/method_module_name_T422.pyx +++ b/tests/run/method_module_name_T422.pyx @@ -1,4 +1,4 @@ -# ticket: 422 +# ticket: t422 """ >>> Foo.incr.__module__ is not None diff --git a/tests/run/methodmangling_T5.py b/tests/run/methodmangling_T5.py index 1cfa85310..9e2b7c63a 100644 --- a/tests/run/methodmangling_T5.py +++ b/tests/run/methodmangling_T5.py @@ -1,5 +1,10 @@ # mode: run -# ticket: 5 +# ticket: t5 + +# A small number of extra tests checking: +# 1) this works correctly with pure-Python-mode decorators - methodmangling_pure.py. +# 2) this works correctly with cdef classes - methodmangling_cdef.pyx +# 3) with "error_on_unknown_names" - methodmangling_unknown_names.py class CyTest(object): """ @@ -15,8 +20,23 @@ class CyTest(object): >>> '__x' in dir(cy) False + >>> cy._CyTest__y + 2 + + >>> '_CyTest___more_than_two' in dir(cy) + True + >>> '___more_than_two' in dir(cy) + False + >>> '___more_than_two_special___' in dir(cy) + True """ __x = 1 + ___more_than_two = 3 + ___more_than_two_special___ = 4 + + def __init__(self): + self.__y = 2 + def __private(self): return 8 def get(self): @@ -88,8 +108,285 @@ class _UnderscoreTest(object): 1 >>> ut.get() 1 + >>> ut._UnderscoreTest__UnderscoreNested().ret1() + 1 + >>> ut._UnderscoreTest__UnderscoreNested.__name__ + '__UnderscoreNested' + >>> ut._UnderscoreTest__prop + 1 """ __x = 1 def get(self): return self.__x + + class __UnderscoreNested(object): + def ret1(self): + return 1 + + @property + def __prop(self): + return self.__x + +class C: + error = """Traceback (most recent call last): +... +TypeError: +""" + __doc__ = """ +>>> instance = C() + +Instance methods have their arguments mangled +>>> instance.method1(__arg=1) # doctest: +IGNORE_EXCEPTION_DETAIL +{error} +>>> instance.method1(_C__arg=1) +1 +>>> instance.method2(__arg=1) # doctest: +IGNORE_EXCEPTION_DETAIL +{error} +>>> instance.method2(_C__arg=1) +1 + +Works when optional argument isn't passed +>>> instance.method2() +None + +Where args are in the function's **kwargs dict, names aren't mangled +>>> instance.method3(__arg=1) # doctest: +1 +>>> instance.method3(_C__arg=1) # doctest: +IGNORE_EXCEPTION_DETAIL +Traceback (most recent call last): +... +KeyError: + +Lambda functions behave in the same way: +>>> instance.method_lambda(__arg=1) # doctest: +IGNORE_EXCEPTION_DETAIL +{error} +>>> instance.method_lambda(_C__arg=1) +1 + +Class methods - have their arguments mangled +>>> instance.class_meth(__arg=1) # doctest: +IGNORE_EXCEPTION_DETAIL +{error} +>>> instance.class_meth(_C__arg=1) +1 +>>> C.class_meth(__arg=1) # doctest: +IGNORE_EXCEPTION_DETAIL +{error} +>>> C.class_meth(_C__arg=1) +1 + +Static methods - have their arguments mangled +>>> instance.static_meth(__arg=1) # doctest: +IGNORE_EXCEPTION_DETAIL +{error} +>>> instance.static_meth(_C__arg=1) +1 +>>> C.static_meth(__arg=1) # doctest: +IGNORE_EXCEPTION_DETAIL +{error} +>>> C.static_meth(_C__arg=1) +1 + +Functions assigned to the class don't have their arguments mangled +>>> instance.class_assigned_function(__arg=1) +1 +>>> instance.class_assigned_function(_C__arg=1) # doctest: +IGNORE_EXCEPTION_DETAIL +{error} + +Functions assigned to an instance don't have their arguments mangled +>>> instance.instance_assigned_function = free_function2 +>>> instance.instance_assigned_function(__arg=1) +1 +>>> instance.instance_assigned_function(_C__arg=1) # doctest: +IGNORE_EXCEPTION_DETAIL +{error} + +Locals are reported as mangled +>>> list(sorted(k for k in instance.get_locals(1).keys())) +['_C__arg', 'self'] +""".format(error=error) + + def method1(self, __arg): + print(__arg) + + def method2(self, __arg=None): + # __arg is optional + print(__arg) + + def method3(self, **kwargs): + print(kwargs['__arg']) + + method_lambda = lambda self, __arg: __arg + + def get_locals(self, __arg): + return locals() + + @classmethod + def class_meth(cls, __arg): + print(__arg) + + @staticmethod + def static_meth(__arg, dummy_arg=None): + # dummy_arg is to mask https://github.com/cython/cython/issues/3090 + print(__arg) + +def free_function1(x, __arg): + print(__arg) + +def free_function2(__arg, dummy_arg=None): + # dummy_arg is to mask https://github.com/cython/cython/issues/3090 + print(__arg) + +C.class_assigned_function = free_function1 + +__global_arg = True + +_D__arg1 = None +_D__global_arg = False # define these because otherwise Cython gives a compile-time error + # while Python gives a runtime error (which is difficult to test) +def can_find_global_arg(): + """ + >>> can_find_global_arg() + True + """ + return __global_arg + +def cant_find_global_arg(): + """ + Gets _D_global_arg instead + >>> cant_find_global_arg() + False + """ + class D: + def f(self): + return __global_arg + return D().f() + +class CMultiplyNested: + def f1(self, __arg, name=None, return_closure=False): + """ + >>> inst = CMultiplyNested() + >>> for name in [None, '__arg', '_CMultiplyNested__arg', '_D__arg']: + ... try: + ... print(inst.f1(1,name)) + ... except TypeError: + ... print("TypeError") # not concerned about exact details + ... # now test behaviour is the same in closures + ... closure = inst.f1(1, return_closure=True) + ... try: + ... if name is None: + ... print(closure(2)) + ... else: + ... print(closure(**{ name: 2})) + ... except TypeError: + ... print("TypeError") + 2 + 2 + TypeError + TypeError + TypeError + TypeError + 2 + 2 + """ + class D: + def g(self, __arg): + return __arg + if return_closure: + return D().g + if name is not None: + return D().g(**{ name: 2 }) + else: + return D().g(2) + + def f2(self, __arg1): + """ + This finds the global name '_D__arg1' + It's tested in this way because without the global + Python gives a runtime error and Cython a compile error + >>> print(CMultiplyNested().f2(1)) + None + """ + class D: + def g(self): + return __arg1 + return D().g() + + def f3(self, arg, name): + """ + >>> inst = CMultiplyNested() + >>> inst.f3(1, None) + 2 + >>> inst.f3(1, '__arg') # doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + ... + TypeError: + >>> inst.f3(1, '_CMultiplyNested__arg') + 2 + """ + def g(__arg, dummy=1): + return __arg + if name is not None: + return g(**{ name: 2}) + else: + return g(2) + + def f4(self, __arg): + """ + >>> CMultiplyNested().f4(1) + 1 + """ + def g(): + return __arg + return g() + + def f5(self, __arg): + """ + Default values are found in the outer scope correcly + >>> CMultiplyNested().f5(1) + 1 + """ + def g(x=__arg): + return x + return g() + + def f6(self, __arg1): + """ + This will find the global name _D__arg1 + >>> print(CMultiplyNested().f6(1)) + None + """ + class D: + def g(self, x=__arg1): + return x + return D().g() + + def f7(self, __arg): + """ + Lookup works in generator expressions + >>> list(CMultiplyNested().f7(1)) + [1] + """ + return (__arg for x in range(1)) + +class __NameWithDunder: + """ + >>> __NameWithDunder.__name__ + '__NameWithDunder' + """ + pass + +class Inherits(__NameWithDunder): + """ + Compile check that it can find the base class + >>> x = Inherits() + """ + pass + +def regular_function(__x, dummy=None): + # as before, dummy stops Cython creating a 1 arg, non-keyword call + return __x + +class CallsRegularFunction: + def call(self): + """ + >>> CallsRegularFunction().call() + 1 + """ + return regular_function(__x=1) # __x shouldn't be mangled as an argument elsewhere diff --git a/tests/run/methodmangling_cdef.pxd b/tests/run/methodmangling_cdef.pxd new file mode 100644 index 000000000..58a9130a4 --- /dev/null +++ b/tests/run/methodmangling_cdef.pxd @@ -0,0 +1,3 @@ +cdef class InPxd: + cdef public int __y + cdef int __private_cdef(self) diff --git a/tests/run/methodmangling_cdef.pyx b/tests/run/methodmangling_cdef.pyx new file mode 100644 index 000000000..83f02fa6a --- /dev/null +++ b/tests/run/methodmangling_cdef.pyx @@ -0,0 +1,95 @@ +# mode: run + +def call_cdt_private_cdef(CDefTest o): + return o._CDefTest__private_cdef() + +cdef __c_func(): + return "cdef function" + +cdef __c_var = "Shouldn't see this" + +cdef class CDefTest: + """ + >>> cd = CDefTest() + >>> '_CDefTest__private' in dir(cd) + True + >>> cd._CDefTest__private() + 8 + >>> call_cdt_private_cdef(cd) + 8 + >>> '__private' in dir(cd) + False + >>> '_CDefTest__x' in dir(cd) + True + + >>> '__x' in dir(cd) + False + >>> cd._CDefTest__y + 2 + """ + __x = 1 + cdef public int __y + + def __init__(self): + self.__y = 2 + + def __private(self): return 8 + + cdef __private_cdef(self): return 8 + + def get(self): + """ + >>> CDefTest().get() + (1, 1, 8) + """ + return self._CDefTest__x, self.__x, self.__private() + + def get_inner(self): + """ + >>> CDefTest().get_inner() + (1, 1, 8) + """ + def get(o): + return o._CDefTest__x, o.__x, o.__private() + return get(self) + + def get_c_func(self): + """ + Should still be able to access C function with __names + >>> CDefTest().get_c_func() + 'cdef function' + """ + return __c_func() + + def get_c_func2(self): + """ + Should find mangled name before C __name + >>> CDefTest().get_c_func2() + 'lambda' + """ + _CDefTest__c_func = lambda: "lambda" + return __c_func() + + def get_c_var(self): + """ + >>> CDefTest().get_c_var() + 'c var' + """ + global __c_var + __c_var = "c var" + return __c_var + +def call_inpdx_private_cdef(InPxd o): + return o._InPxd__private_cdef() + +cdef class InPxd: + """ + >>> InPxd()._InPxd__y + 2 + >>> call_inpdx_private_cdef(InPxd()) + 8 + """ + def __init__(self): + self.__y = 2 + + cdef int __private_cdef(self): return 8 diff --git a/tests/run/methodmangling_pure.py b/tests/run/methodmangling_pure.py new file mode 100644 index 000000000..10b811e2f --- /dev/null +++ b/tests/run/methodmangling_pure.py @@ -0,0 +1,76 @@ +# mode: run +# cython: language_level=3 + +# This file tests that methodmangling is applied correctly to +# pure Python decorated classes. + +import cython + +if cython.compiled: + # don't run in Python mode since a significant number of the tests + # are only for Cython features + + def declare(**kwargs): + return kwargs['__x'] + + class RegularClass: + @cython.locals(__x=cython.int) + def f1(self, __x, dummy=None): + """ + Is the locals decorator correctly applied + >>> c = RegularClass() + >>> c.f1(1) + 1 + >>> c.f1("a") + Traceback (most recent call last): + ... + TypeError: an integer is required + >>> c.f1(_RegularClass__x = 1) + 1 + """ + return __x + + def f2(self, x): + """ + Is the locals decorator correctly applied + >>> c = RegularClass() + >>> c.f2(1) + 1 + >>> c.f2("a") + Traceback (most recent call last): + ... + TypeError: an integer is required + """ + __x = cython.declare(cython.int, x) + + return __x + + def f3(self, x): + """ + Is the locals decorator correctly applied + >>> c = RegularClass() + >>> c.f3(1) + 1 + >>> c.f3("a") + Traceback (most recent call last): + ... + TypeError: an integer is required + """ + cython.declare(__x=cython.int) + __x = x + + return __x + + def f4(self, x): + """ + We shouldn't be tripped up by a function called + "declare" that is nothing to do with cython + >>> RegularClass().f4(1) + 1 + """ + return declare(__x=x) +else: + __doc__ = """ + >>> True + True + """ # stops Python2 from failing diff --git a/tests/run/methodmangling_unknown_names.py b/tests/run/methodmangling_unknown_names.py new file mode 100644 index 000000000..dae1f5c22 --- /dev/null +++ b/tests/run/methodmangling_unknown_names.py @@ -0,0 +1,25 @@ +# mode: run +# tag: allow_unknown_names, pure2.0, pure3.0 + +class Test(object): + def run(self): + """ + >>> Test().run() + NameError1 + NameError2 + found mangled + """ + try: + print(__something) + except NameError: + print("NameError1") # correct - shouldn't exist + globals()['__something'] = 'found unmangled' + try: + print(__something) + except NameError: + print("NameError2") # correct - shouldn't exist + globals()['_Test__something'] = 'found mangled' + try: + print(__something) # should print this + except NameError: + print("NameError3") diff --git a/tests/run/modop.pyx b/tests/run/modop.pyx index c7b4c7d57..82e43d095 100644 --- a/tests/run/modop.pyx +++ b/tests/run/modop.pyx @@ -9,7 +9,7 @@ def modobj(obj2, obj3): '5' >>> modobj(1, 0) # doctest: +ELLIPSIS Traceback (most recent call last): - ZeroDivisionError: integer division... + ZeroDivisionError: integer... modulo by zero """ obj1 = obj2 % obj3 return obj1 @@ -19,11 +19,22 @@ def mod_10_obj(int2): """ >>> mod_10_obj(0) # doctest: +ELLIPSIS Traceback (most recent call last): - ZeroDivisionError: integer division... + ZeroDivisionError: ... modulo by zero + >>> 10 % 1 + 0 + >>> mod_10_obj(1) + 0 >>> mod_10_obj(3) 1 + >>> 10 % -1 + 0 + >>> mod_10_obj(-1) + 0 + >>> mod_10_obj(-10) + 0 """ - return 10 % int2 + int1 = 10 % int2 + return int1 def mod_obj_10(int2): @@ -168,6 +179,53 @@ def mod_obj_17(int2): return int1 +def mod_int_17(int int2): + """ + >>> 0 % 17 + 0 + >>> mod_int_17(0) + 0 + >>> 1 % 17 + 1 + >>> mod_int_17(1) + 1 + >>> (-1) % 17 + 16 + >>> mod_int_17(-1) + 16 + >>> 9 % 17 + 9 + >>> mod_int_17(16) + 16 + >>> 17 % 17 + 0 + >>> mod_int_17(17) + 0 + >>> (-17) % 17 + 0 + >>> mod_int_17(-17) + 0 + >>> (-18) % 17 + 16 + >>> mod_int_17(-18) + 16 + >>> 10002 % 17 + 6 + >>> mod_int_17(10002) + 6 + >>> int((2**25) % 17) + 2 + >>> int(mod_int_17(2**25)) + 2 + >>> int((-2**25) % 17) + 15 + >>> int(mod_int_17(-2**25)) + 15 + """ + int1 = int2 % 17 + return int1 + + def mod_obj_m2(int2): """ >>> 0 % -2 diff --git a/tests/run/mulop.pyx b/tests/run/mulop.pyx new file mode 100644 index 000000000..0fba7dba9 --- /dev/null +++ b/tests/run/mulop.pyx @@ -0,0 +1,166 @@ +# mode: run +# tag: multiply + +import sys +IS_PY2 = sys.version_info[0] < 3 + + +def print_long(x): + if IS_PY2: + x = str(x).rstrip('L') + print(x) + + +def mul_10_obj(x): + """ + >>> mul_10_obj(0) + 0 + >>> mul_10_obj(10) + 100 + >>> mul_10_obj(-10) + -100 + >>> 10 * (2**14) + 163840 + >>> mul_10_obj(2**14) + 163840 + >>> mul_10_obj(-2**14) + -163840 + >>> print_long(10 * (2**29)) + 5368709120 + >>> print_long(mul_10_obj(2**29)) + 5368709120 + >>> print_long(mul_10_obj(-2**29)) + -5368709120 + >>> print_long(10 * (2**30)) + 10737418240 + >>> print_long(mul_10_obj(2**30)) + 10737418240 + >>> print_long(mul_10_obj(-2**30)) + -10737418240 + >>> print_long(10 * (2**63)) + 92233720368547758080 + >>> print_long(mul_10_obj(2**63)) + 92233720368547758080 + >>> print_long(mul_10_obj(-2**63)) + -92233720368547758080 + >>> print_long(10 * (2**128)) + 3402823669209384634633746074317682114560 + >>> print_long(mul_10_obj(2**128)) + 3402823669209384634633746074317682114560 + >>> print_long(mul_10_obj(-2**128)) + -3402823669209384634633746074317682114560 + """ + result = 10 * x + return result + + +def mul_obj_10(x): + """ + >>> mul_obj_10(0) + 0 + >>> mul_obj_10(10) + 100 + >>> mul_obj_10(-10) + -100 + >>> 10 * (2**14) + 163840 + >>> mul_obj_10(2**14) + 163840 + >>> mul_obj_10(-2**14) + -163840 + >>> print_long(10 * (2**29)) + 5368709120 + >>> print_long(mul_obj_10(2**29)) + 5368709120 + >>> print_long(mul_obj_10(-2**29)) + -5368709120 + >>> print_long(10 * (2**30)) + 10737418240 + >>> print_long(mul_obj_10(2**30)) + 10737418240 + >>> print_long(mul_obj_10(-2**30)) + -10737418240 + >>> print_long(10 * (2**63)) + 92233720368547758080 + >>> print_long(mul_obj_10(2**63)) + 92233720368547758080 + >>> print_long(mul_obj_10(-2**63)) + -92233720368547758080 + >>> print_long(10 * (2**128)) + 3402823669209384634633746074317682114560 + >>> print_long(mul_obj_10(2**128)) + 3402823669209384634633746074317682114560 + >>> print_long(mul_obj_10(-2**128)) + -3402823669209384634633746074317682114560 + """ + result = x * 10 + return result + + +def mul_bigint_obj(x): + """ + >>> mul_bigint_obj(0) + 0 + >>> print_long(mul_bigint_obj(1)) + 536870912 + >>> print_long(mul_bigint_obj(2)) + 1073741824 + >>> print_long(mul_bigint_obj(2**29)) + 288230376151711744 + >>> print_long(mul_bigint_obj(-2**29)) + -288230376151711744 + >>> print_long(mul_bigint_obj(2**30)) + 576460752303423488 + >>> print_long(mul_bigint_obj(-2**30)) + -576460752303423488 + >>> print_long(mul_bigint_obj(2**59)) + 309485009821345068724781056 + >>> print_long(mul_bigint_obj(-2**59)) + -309485009821345068724781056 + """ + result = (2**29) * x + return result + + +def mul_obj_float(x): + """ + >>> mul_obj_float(-0.0) + -0.0 + >>> mul_obj_float(0) + 0.0 + >>> mul_obj_float(1.0) + 2.0 + >>> mul_obj_float(-2.0) + -4.0 + >>> mul_obj_float(-0.5) + -1.0 + """ + result = x * 2.0 + return result + + +def mul_float_obj(x): + """ + >>> mul_float_obj(0) + 0.0 + >>> mul_float_obj(2) + 4.0 + >>> mul_float_obj(-2) + -4.0 + >>> 2.0 * (2**30-1) + 2147483646.0 + >>> mul_float_obj(2**30-1) + 2147483646.0 + >>> mul_float_obj(-(2**30-1)) + -2147483646.0 + >>> mul_float_obj(-0.0) + -0.0 + >>> mul_float_obj(1.0) + 2.0 + >>> mul_float_obj(-2.0) + -4.0 + >>> mul_float_obj(-0.5) + -1.0 + """ + result = 2.0 * x + return result diff --git a/tests/run/no_gc_clear.pyx b/tests/run/no_gc_clear.pyx index 643ec4c6e..9afca8af8 100644 --- a/tests/run/no_gc_clear.pyx +++ b/tests/run/no_gc_clear.pyx @@ -3,7 +3,7 @@ Check that the @cython.no_gc_clear decorator disables generation of the tp_clear slot so that __dealloc__ will still see the original reference contents. -Discussed here: http://article.gmane.org/gmane.comp.python.cython.devel/14986 +Discussed here: https://article.gmane.org/gmane.comp.python.cython.devel/14986 """ cimport cython diff --git a/tests/run/nogil.pyx b/tests/run/nogil.pyx index c6f6d8b53..6649ee071 100644 --- a/tests/run/nogil.pyx +++ b/tests/run/nogil.pyx @@ -28,10 +28,10 @@ cdef int g(int x) nogil: y = x + 42 return y -cdef int with_gil_func() except 0 with gil: +cdef int with_gil_func() except -1 with gil: raise Exception("error!") -cdef int nogil_func() nogil except 0: +cdef int nogil_func() nogil except -1: with_gil_func() def test_nogil_exception_propagation(): diff --git a/tests/run/nogil_conditional.pyx b/tests/run/nogil_conditional.pyx new file mode 100644 index 000000000..eba22d5b2 --- /dev/null +++ b/tests/run/nogil_conditional.pyx @@ -0,0 +1,271 @@ +# mode: run + +try: + from StringIO import StringIO +except ImportError: + from io import StringIO + + +def test(int x): + """ + >>> test(0) + 110 + """ + with nogil(True): + x = f_nogil(x) + with gil(True): + x = f_gil(x) + return x + + +cdef int f_nogil(int x) nogil: + cdef int y + y = x + 10 + return y + + +def f_gil(x): + y = 0 + y = x + 100 + return y + + +cdef int with_gil_func() except? -1 with gil: + raise Exception("error!") + + +cdef int nogil_func() nogil except? -1: + with_gil_func() + + +def test_nogil_exception_propagation(): + """ + >>> test_nogil_exception_propagation() + Traceback (most recent call last): + ... + Exception: error! + """ + with nogil: + with gil: + with nogil(True): + nogil_func() + + +cdef int write_unraisable() nogil: + with gil: + raise ValueError() + + +def test_unraisable(): + """ + >>> print(test_unraisable()) # doctest: +ELLIPSIS + ValueError + Exception...ignored... + """ + import sys + old_stderr = sys.stderr + stderr = sys.stderr = StringIO() + try: + write_unraisable() + finally: + sys.stderr = old_stderr + return stderr.getvalue().strip() + + +def test_nested(): + """ + >>> test_nested() + 240 + """ + cdef int res = 0 + + with nogil(True): + res = f_nogil(res) + with gil(1 < 2): + res = f_gil(res) + with nogil: + res = f_nogil(res) + + with gil: + res = f_gil(res) + with nogil(True): + res = f_nogil(res) + with nogil: + res = f_nogil(res) + + return res + + +DEF FREE_GIL = True +DEF FREE_GIL_FALSE = False + + +def test_nested_condition_false(): + """ + >>> test_nested_condition_false() + 220 + """ + cdef int res = 0 + + with gil(FREE_GIL_FALSE): + res = f_gil(res) + with nogil(False): + res = f_gil(res) + + with nogil(FREE_GIL): + res = f_nogil(res) + with gil(False): + res = f_nogil(res) + + return res + +def test_try_finally(): + """ + >>> test_try_finally() + 113 + """ + cdef int res = 0 + + try: + with nogil(True): + try: + res = f_nogil(res) + with gil(1 < 2): + try: + res = f_gil(res) + finally: + res += 1 + finally: + res = res + 1 + finally: + res += 1 + + return res + + +ctypedef fused number_or_object: + int + float + object + + +def test_fused(number_or_object x) -> number_or_object: + """ + >>> test_fused[int](1) + 2 + >>> test_fused[float](1.0) + 2.0 + >>> test_fused[object](1) + 2 + >>> test_fused[object](1.0) + 2.0 + """ + cdef number_or_object res = x + + with nogil(number_or_object is not object): + res = res + 1 + + return res + + +ctypedef fused int_or_object: + int + object + + +def test_fused_object(int_or_object x): + """ + >>> test_fused_object[object]("spam") + 456 + >>> test_fused_object[int](1000) + 1000 + """ + cdef int res = 0 + + if int_or_object is object: + with nogil(False): + res += len(x) + + try: + with nogil(int_or_object is object): + try: + with gil(int_or_object is object): + res = f_gil(res) + with gil: + res = f_gil(res) + with gil(False): + res = f_nogil(res) + + with gil(int_or_object is not object): + res = f_nogil(res) + with nogil(False): + res = f_nogil(res) + + res = f_nogil(res) + finally: + res = res + 1 + + with nogil(int_or_object is not object): + res = f_gil(res) + + with gil(int_or_object is not object): + res = f_gil(res) + + with nogil(int_or_object is object): + res = f_nogil(res) + + finally: + res += 1 + else: + res = x + + return res + + +def test_fused_int(int_or_object x): + """ + >>> test_fused_int[object]("spam") + 4 + >>> test_fused_int[int](1000) + 1452 + """ + cdef int res = 0 + + if int_or_object is int: + res += x + + try: + with nogil(int_or_object is int): + try: + with gil(int_or_object is int): + res = f_gil(res) + with gil: + res = f_gil(res) + with gil(False): + res = f_nogil(res) + + with gil(int_or_object is not int): + res = f_nogil(res) + with nogil(False): + res = f_nogil(res) + + res = f_nogil(res) + finally: + res = res + 1 + + with nogil(int_or_object is not int): + res = f_gil(res) + + with gil(int_or_object is not int): + res = f_gil(res) + + with nogil(int_or_object is int): + res = f_nogil(res) + + finally: + res += 1 + else: + with nogil(False): + res = len(x) + + return res diff --git a/tests/run/non_dict_kwargs_T470.pyx b/tests/run/non_dict_kwargs_T470.pyx index b60d56210..be6f8c67b 100644 --- a/tests/run/non_dict_kwargs_T470.pyx +++ b/tests/run/non_dict_kwargs_T470.pyx @@ -1,5 +1,5 @@ # mode: run -# ticket: 470 +# ticket: t470 def func(**kwargs): diff --git a/tests/run/numpy_ValueError_T172.pyx b/tests/run/numpy_ValueError_T172.pyx index 66558eefc..233048ce7 100644 --- a/tests/run/numpy_ValueError_T172.pyx +++ b/tests/run/numpy_ValueError_T172.pyx @@ -1,4 +1,4 @@ -# ticket: 172 +# ticket: t172 # tag: numpy __doc__ = u""" diff --git a/tests/run/numpy_attributes.pyx b/tests/run/numpy_attributes.pyx new file mode 100644 index 000000000..c77346a49 --- /dev/null +++ b/tests/run/numpy_attributes.pyx @@ -0,0 +1,87 @@ +# mode: run +# tag: numpy + +cimport cython + +import numpy as np +cimport numpy as cnp + +cnp.import_array() + + +@cython.test_assert_path_exists( + "//ReturnStatNode", + "//ReturnStatNode//IndexNode", + "//ReturnStatNode//IndexNode//SimpleCallNode", +) +@cython.test_fail_if_path_exists( + "//ReturnStatNode//AttributeNode", +) +def access_shape(): + """ + >>> print(access_shape()) + 10 + """ + cdef cnp.ndarray[double, ndim=2, mode='c'] array_in = \ + 1e10 * np.ones((10, 10)) + + return array_in.shape[0] + + +@cython.test_assert_path_exists( + "//ReturnStatNode", + "//ReturnStatNode//SimpleCallNode", +) +@cython.test_fail_if_path_exists( + "//ReturnStatNode//AttributeNode", +) +def access_size(): + """ + >>> print(access_size()) + 100 + """ + cdef cnp.ndarray[double, ndim=2, mode='c'] array_in = \ + 1e10 * np.ones((10, 10)) + + return array_in.size + + +@cython.test_assert_path_exists( + "//ReturnStatNode", + "//ReturnStatNode//IndexNode", + "//ReturnStatNode//IndexNode//SimpleCallNode", +) +@cython.test_fail_if_path_exists( + "//ReturnStatNode//AttributeNode", +) +def access_strides(): + """ + >>> x, y = access_strides() + >>> print(x) + 80 + >>> print(y) + 8 + """ + cdef cnp.ndarray[double, ndim=2, mode='c'] array_in = \ + 1e10 * np.ones((10, 10), dtype=np.float64) + + return (array_in.strides[0], array_in.strides[1]) + + +@cython.test_assert_path_exists( + "//ReturnStatNode", + "//ReturnStatNode//PrimaryCmpNode", + "//ReturnStatNode//PrimaryCmpNode//SimpleCallNode", +) +@cython.test_fail_if_path_exists( + "//ReturnStatNode//AttributeNode", +) +def access_data(): + """ + >>> access_data() + True + """ + cdef cnp.ndarray[double, ndim=2, mode='c'] array_in = \ + 1e10 * np.ones((10, 10), dtype=np.float64) + + return array_in.data is not NULL diff --git a/tests/run/numpy_bufacc_T155.pyx b/tests/run/numpy_bufacc_T155.pyx index 927eddb29..2c637ebc4 100644 --- a/tests/run/numpy_bufacc_T155.pyx +++ b/tests/run/numpy_bufacc_T155.pyx @@ -1,5 +1,5 @@ -# ticket: 155 -# tag: numpy_old +# ticket: t155 +# tag: numpy """ >>> myfunc() @@ -17,4 +17,3 @@ def myfunc(): A[i, :] /= 2 return A[0,0] -include "numpy_common.pxi" diff --git a/tests/run/numpy_cimport.pyx b/tests/run/numpy_cimport.pyx index 7c23d7725..8aaea8597 100644 --- a/tests/run/numpy_cimport.pyx +++ b/tests/run/numpy_cimport.pyx @@ -6,4 +6,3 @@ True """ cimport numpy as np -include "numpy_common.pxi" diff --git a/tests/run/numpy_cimport_1.pyx b/tests/run/numpy_cimport_1.pyx new file mode 100644 index 000000000..05754f591 --- /dev/null +++ b/tests/run/numpy_cimport_1.pyx @@ -0,0 +1,25 @@ +# mode: run +# tag: warnings, numpy + +cimport numpy as np +# np.import_array not called - should generate warning + +cdef extern from *: + """ + static void** _check_array_api(void) { + return PyArray_API; /* should be non NULL */ + } + """ + void** _check_array_api() + +def check_array_api(): + """ + >>> check_array_api() + True + """ + return _check_array_api() != NULL + + +_WARNINGS = """ +4:8: 'numpy.import_array()' has been added automatically since 'numpy' was cimported but 'numpy.import_array' was not called. +""" diff --git a/tests/run/numpy_cimport_2.pyx b/tests/run/numpy_cimport_2.pyx new file mode 100644 index 000000000..cdf8783c9 --- /dev/null +++ b/tests/run/numpy_cimport_2.pyx @@ -0,0 +1,25 @@ +# mode: run +# tag: warnings, numpy + +cimport numpy as np +np.import_array() +# np.import_array is called - no warning necessary + +cdef extern from *: + """ + static void** _check_array_api(void) { + return PyArray_API; /* should be non NULL */ + } + """ + void** _check_array_api() + +def check_array_api(): + """ + >>> check_array_api() + True + """ + return _check_array_api() != NULL + + +_WARNINGS = """ +""" diff --git a/tests/run/numpy_cimport_3.pyx b/tests/run/numpy_cimport_3.pyx new file mode 100644 index 000000000..b5b24661d --- /dev/null +++ b/tests/run/numpy_cimport_3.pyx @@ -0,0 +1,8 @@ +# mode: compile +# tag: warnings, numpy + +import numpy as np +# Numpy is only imported - no warning necessary + +_WARNINGS = """ +""" diff --git a/tests/run/numpy_cimport_4.pyx b/tests/run/numpy_cimport_4.pyx new file mode 100644 index 000000000..44e112054 --- /dev/null +++ b/tests/run/numpy_cimport_4.pyx @@ -0,0 +1,24 @@ +# mode: run +# tag: warnings, numpy + +cimport numpy +<void>numpy.import_array # dummy call should stop Cython auto-generating call to import_array + +cdef extern from *: + """ + static void** _check_array_api(void) { + return PyArray_API; /* should be non NULL if initialized */ + } + """ + void** _check_array_api() + +def check_array_api(): + """ + >>> check_array_api() + True + """ + return _check_array_api() == NULL # not initialized + + +_WARNINGS = """ +""" diff --git a/tests/run/numpy_cimport_5.pyx b/tests/run/numpy_cimport_5.pyx new file mode 100644 index 000000000..2768f2dae --- /dev/null +++ b/tests/run/numpy_cimport_5.pyx @@ -0,0 +1,25 @@ +# mode: run +# tag: warnings, numpy + +from numpy cimport ndarray +# np.import_array not called - should generate warning + +cdef extern from *: + """ + static void** _check_array_api(void) { + return PyArray_API; /* should be non NULL */ + } + """ + void** _check_array_api() + +def check_array_api(): + """ + >>> check_array_api() + True + """ + return _check_array_api() != NULL + + +_WARNINGS = """ +4:0: 'numpy.import_array()' has been added automatically since 'numpy' was cimported but 'numpy.import_array' was not called. +""" diff --git a/tests/run/numpy_cimport_6.pyx b/tests/run/numpy_cimport_6.pyx new file mode 100644 index 000000000..5cda5246d --- /dev/null +++ b/tests/run/numpy_cimport_6.pyx @@ -0,0 +1,25 @@ +# mode: run +# tag: warnings, numpy + +from numpy cimport ndarray, import_array +import_array() +# np.import_array is called - no warning necessary + +cdef extern from *: + """ + static void** _check_array_api(void) { + return PyArray_API; /* should be non NULL */ + } + """ + void** _check_array_api() + +def check_array_api(): + """ + >>> check_array_api() + True + """ + return _check_array_api() != NULL + + +_WARNINGS = """ +""" diff --git a/tests/run/numpy_common.pxi b/tests/run/numpy_common.pxi deleted file mode 100644 index 615bf701a..000000000 --- a/tests/run/numpy_common.pxi +++ /dev/null @@ -1,10 +0,0 @@ -# hack to avoid C compiler warnings about unused functions in the NumPy header files - -from numpy cimport import_array # , import_umath - -cdef extern from *: - bint FALSE "0" - -if FALSE: - import_array() -# import_umath() diff --git a/tests/run/numpy_parallel.pyx b/tests/run/numpy_parallel.pyx index 7dcf2b27a..96a60be14 100644 --- a/tests/run/numpy_parallel.pyx +++ b/tests/run/numpy_parallel.pyx @@ -1,10 +1,9 @@ -# tag: numpy_old +# tag: numpy # tag: openmp cimport cython from cython.parallel import prange cimport numpy as np -include "numpy_common.pxi" @cython.boundscheck(False) @@ -22,7 +21,7 @@ def test_parallel_numpy_arrays(): 3 4 """ - cdef Py_ssize_t i + cdef Py_ssize_t i, length cdef np.ndarray[np.int_t] x try: @@ -33,10 +32,10 @@ def test_parallel_numpy_arrays(): return x = numpy.zeros(10, dtype=numpy.int) + length = x.shape[0] - for i in prange(x.shape[0], nogil=True): + for i in prange(length, nogil=True): x[i] = i - 5 for i in x: - print i - + print(i) diff --git a/tests/run/numpy_pythran.pyx b/tests/run/numpy_pythran.pyx index 363d9ea85..8bc5fa5f2 100644 --- a/tests/run/numpy_pythran.pyx +++ b/tests/run/numpy_pythran.pyx @@ -55,3 +55,13 @@ def calculate_tax(cnp.ndarray[double, ndim=1] d): np.sum(seg3 * prog_seg3 + 939.57) + np.sum(seg4 * prog_seg4) ) / np.sum(d) + +def access_shape(): + """ + >>> access_shape() + 10 + """ + cdef cnp.ndarray[double, ndim=2, mode='c'] array_in = \ + 1e10 * np.ones((10, 10)) + + return array_in.shape[0] diff --git a/tests/run/numpy_subarray.pyx b/tests/run/numpy_subarray.pyx index c5d82d4b1..d032a3a01 100644 --- a/tests/run/numpy_subarray.pyx +++ b/tests/run/numpy_subarray.pyx @@ -1,4 +1,4 @@ -# tag: numpy_old +# tag: numpy cimport numpy as np cimport cython diff --git a/tests/run/numpy_test.pyx b/tests/run/numpy_test.pyx index 662dd4ad0..d2718a463 100644 --- a/tests/run/numpy_test.pyx +++ b/tests/run/numpy_test.pyx @@ -1,30 +1,19 @@ -# tag: numpy_old -# cannot be named "numpy" in order to not clash with the numpy module! +# tag: numpy cimport numpy as np cimport cython import re -import sys - -# initialise NumPy C-API -np.import_array() def little_endian(): cdef int endian_detector = 1 return (<char*>&endian_detector)[0] != 0 -__test__ = {} def testcase(f): - __test__[f.__name__] = f.__doc__ - return f - -def testcase_have_buffer_interface(f): - major, minor, *rest = np.__version__.split('.') - if (int(major), int(minor)) >= (1, 5) and sys.version_info[:2] >= (2, 6): - __test__[f.__name__] = f.__doc__ + # testcase decorator now does nothing (following changes to doctest) + # but is a useful indicator of what functions are designed as tests return f if little_endian(): @@ -272,8 +261,6 @@ try: except: __doc__ = u"" -__test__[__name__] = __doc__ - def assert_dtype_sizes(): assert sizeof(np.int8_t) == 1 @@ -684,7 +671,6 @@ def get_Foo_array(): data[5].b = 9.0 return np.asarray(<Foo[:]>data).copy() -@testcase_have_buffer_interface def test_fused_ndarray(fused_ndarray a): """ >>> import cython @@ -733,9 +719,6 @@ cpdef test_fused_cpdef_ndarray(fused_ndarray a): else: print b[5] -testcase_have_buffer_interface(test_fused_cpdef_ndarray) - -@testcase_have_buffer_interface def test_fused_cpdef_ndarray_cdef_call(): """ >>> test_fused_cpdef_ndarray_cdef_call() diff --git a/tests/run/overflow_check.pxi b/tests/run/overflow_check.pxi index 9bfd068a2..0b9844c89 100644 --- a/tests/run/overflow_check.pxi +++ b/tests/run/overflow_check.pxi @@ -1,14 +1,15 @@ cimport cython cdef object two = 2 -cdef int size_in_bits = sizeof(INT) * 8 +cdef int size_in_bits_ = sizeof(INT) * 8 cdef bint is_signed_ = not ((<INT>-1) > 0) -cdef INT max_value_ = <INT>(two ** (size_in_bits - is_signed_) - 1) +cdef INT max_value_ = <INT>(two ** (size_in_bits_ - is_signed_) - 1) cdef INT min_value_ = ~max_value_ cdef INT half_ = max_value_ // <INT>2 # Python visible. +size_in_bits = size_in_bits_ is_signed = is_signed_ max_value = max_value_ min_value = min_value_ @@ -230,6 +231,17 @@ def test_lshift(INT a, int b): """ >>> test_lshift(1, 10) 1024 + >>> test_lshift(1, size_in_bits - 2) == 1 << (size_in_bits - 2) + True + >>> test_lshift(0, size_in_bits - 1) + 0 + >>> test_lshift(1, size_in_bits - 1) == 1 << (size_in_bits - 1) if not is_signed else True + True + >>> if is_signed: expect_overflow(test_lshift, 1, size_in_bits - 1) + >>> expect_overflow(test_lshift, 0, size_in_bits) + >>> expect_overflow(test_lshift, 1, size_in_bits) + >>> expect_overflow(test_lshift, 0, size_in_bits + 1) + >>> expect_overflow(test_lshift, 1, size_in_bits + 1) >>> expect_overflow(test_lshift, 1, 100) >>> expect_overflow(test_lshift, max_value, 1) >>> test_lshift(max_value, 0) == max_value diff --git a/tests/run/packedstruct_T290.pyx b/tests/run/packedstruct_T290.pyx index 203008ca7..d9381567e 100644 --- a/tests/run/packedstruct_T290.pyx +++ b/tests/run/packedstruct_T290.pyx @@ -1,4 +1,4 @@ -# ticket: 290 +# ticket: t290 """ >>> f() diff --git a/tests/run/parallel_swap_assign_T425.pyx b/tests/run/parallel_swap_assign_T425.pyx index fb7d1ac29..b6e7e233d 100644 --- a/tests/run/parallel_swap_assign_T425.pyx +++ b/tests/run/parallel_swap_assign_T425.pyx @@ -1,4 +1,4 @@ -# ticket: 425 +# ticket: t425 cimport cython diff --git a/tests/run/pep448_extended_unpacking.pyx b/tests/run/pep448_extended_unpacking.pyx index 61414f423..cb6501f07 100644 --- a/tests/run/pep448_extended_unpacking.pyx +++ b/tests/run/pep448_extended_unpacking.pyx @@ -464,6 +464,10 @@ def unpack_dict_simple(it): return {**it} +@cython.test_assert_path_exists('//MergedDictNode') +@cython.test_fail_if_path_exists( + '//MergedDictNode//MergedDictNode', +) def unpack_dict_from_iterable(it): """ >>> d = unpack_dict_from_iterable(dict(a=1, b=2, c=3)) @@ -536,3 +540,28 @@ def unpack_dict_keep_originals(a, b, c): True """ return {**a, **b, 2: 4, **c} + + +@cython.test_assert_path_exists( + '//MergedDictNode', + '//MergedDictNode//MergedDictNode', + '//MergedDictNode//MergedDictNode//DictNode', +) +def unpack_in_call(f): + """ + >>> def f(a=1, test=2, **kwargs): + ... return a, test, sorted(kwargs.items()) + >>> wrapped = unpack_in_call(f) + >>> wrapped(1) + (1, 1, [('more', 2)]) + >>> wrapped(test='overwritten') + (1, 1, [('more', 2)]) + >>> wrapped(b=3) + (1, 1, [('b', 3), ('more', 2)]) + >>> wrapped(more=4) + Traceback (most recent call last): + TypeError: function() got multiple values for keyword argument 'more' + """ + def wrapper(*args, **kwargs): + return f(*args, more=2, **{**kwargs, 'test': 1}) + return wrapper diff --git a/tests/run/pep448_test_extcall.pyx b/tests/run/pep448_test_extcall.pyx index 03121556c..6fd6bf1b7 100644 --- a/tests/run/pep448_test_extcall.pyx +++ b/tests/run/pep448_test_extcall.pyx @@ -224,10 +224,10 @@ def call_g_positional(): def call_nonseq_positional1(): """ - >>> call_nonseq_positional1() + >>> call_nonseq_positional1() # doctest: +ELLIPSIS Traceback (most recent call last): ... - TypeError: 'Nothing' object is not iterable + TypeError: ...Nothing... # TypeError: g() argument after * must be a sequence, not Nothing """ @@ -237,10 +237,10 @@ def call_nonseq_positional1(): def call_nonseq_positional2(): """ - >>> call_nonseq_positional2() + >>> call_nonseq_positional2() # doctest: +ELLIPSIS Traceback (most recent call last): ... - TypeError: 'Nothing' object is not iterable + TypeError: ...Nothing... # TypeError: g() argument after * must be a sequence, not Nothing """ @@ -326,7 +326,7 @@ def errors_non_string_kwarg(): """ >>> errors_non_string_kwarg() # doctest: +ELLIPSIS Traceback (most recent call last): - TypeError: ...keywords must be strings + TypeError: ...keywords must be strings... """ f(**{1:2}) @@ -463,10 +463,10 @@ def call_builtin_empty_dict(): def call_builtin_nonempty_dict(): """ - >>> call_builtin_nonempty_dict() + >>> call_builtin_nonempty_dict() # doctest: +ELLIPSIS Traceback (most recent call last): ... - TypeError: id() takes no keyword arguments + TypeError: id() ... keyword argument... """ return id(1, **{'foo': 1}) @@ -474,7 +474,7 @@ def call_builtin_nonempty_dict(): ''' Cython: currently just passes empty kwargs into f() while CPython keeps the content # A corner case of keyword dictionary items being deleted during -# the function call setup. See <http://bugs.python.org/issue2016>. +# the function call setup. See <https://bugs.python.org/issue2016>. def call_kwargs_modified_while_building(): """ diff --git a/tests/run/pep526_variable_annotations.py b/tests/run/pep526_variable_annotations.py index 9a43f35f6..c5a6d97fd 100644 --- a/tests/run/pep526_variable_annotations.py +++ b/tests/run/pep526_variable_annotations.py @@ -152,10 +152,20 @@ def iter_declared_dict_arg(d : Dict[float, float]): return s +def literal_list_ptr(): + """ + >>> literal_list_ptr() + 4 + """ + a : cython.p_int = [1, 2, 3, 4, 5] + return a[3] + + _WARNINGS = """ 37:19: Unknown type declaration in annotation, ignoring 38:12: Unknown type declaration in annotation, ignoring 39:18: Unknown type declaration in annotation, ignoring +73:11: Annotation ignored since class-level attributes must be Python objects. Were you trying to set up an instance attribute? 73:19: Unknown type declaration in annotation, ignoring # FIXME: these are sort-of evaluated now, so the warning is misleading 126:21: Unknown type declaration in annotation, ignoring diff --git a/tests/run/pep557_dataclasses.py b/tests/run/pep557_dataclasses.py new file mode 100644 index 000000000..288c71ed2 --- /dev/null +++ b/tests/run/pep557_dataclasses.py @@ -0,0 +1,54 @@ +# mode: run +# tag: pep557, pure3.7 + +import dataclasses +from typing import Sequence + + +@dataclasses.dataclass +class Color: + """ + >>> list(Color.__dataclass_fields__.keys()) + ['red', 'green', 'blue', 'alpha'] + >>> Color(1, 2, 3) + Color(red=1, green=2, blue=3, alpha=255) + >>> Color(1, 2, 3, 4) + Color(red=1, green=2, blue=3, alpha=4) + >>> Color(green=1, blue=2, red=3, alpha=40) + Color(red=3, green=1, blue=2, alpha=40) + """ + red: int + green: int + blue: int + alpha: int = 255 + + +@dataclasses.dataclass +class NamedColor(Color): + """ + >>> list(NamedColor.__dataclass_fields__.keys()) + ['red', 'green', 'blue', 'alpha', 'names'] + >>> NamedColor(1, 2, 3) + NamedColor(red=1, green=2, blue=3, alpha=255, names=[]) + >>> NamedColor(1, 2, 3, 4) + NamedColor(red=1, green=2, blue=3, alpha=4, names=[]) + >>> NamedColor(green=1, blue=2, red=3, alpha=40) + NamedColor(red=3, green=1, blue=2, alpha=40, names=[]) + >>> NamedColor(1, 2, 3, names=["blackish", "very dark cyan"]) + NamedColor(red=1, green=2, blue=3, alpha=255, names=['blackish', 'very dark cyan']) + """ + names: Sequence[str] = dataclasses.field(default_factory=list) + + +@dataclasses.dataclass(frozen=True) +class IceCream: + """ + >>> IceCream("vanilla") + IceCream(flavour='vanilla', num_toppings=2) + >>> IceCream("vanilla") == IceCream("vanilla", num_toppings=3) + False + >>> IceCream("vanilla") == IceCream("vanilla", num_toppings=2) + True + """ + flavour: str + num_toppings: int = 2 diff --git a/tests/run/pep563_annotations.py b/tests/run/pep563_annotations.py new file mode 100644 index 000000000..0db1ba52c --- /dev/null +++ b/tests/run/pep563_annotations.py @@ -0,0 +1,40 @@ +# mode: run +# tag: pep563, pure3.7 + +from __future__ import annotations + +def f(a: 1+2==3, b: list, c: this_cant_evaluate, d: "Hello from inside a string") -> "Return me!": + """ + The absolute exact strings aren't reproducible according to the PEP, + so be careful to avoid being too specific + >>> stypes = (type(""), type(u"")) # Python 2 is a bit awkward here + >>> eval(f.__annotations__['a']) + True + >>> isinstance(f.__annotations__['a'], stypes) + True + >>> print(f.__annotations__['b']) + list + >>> print(f.__annotations__['c']) + this_cant_evaluate + >>> isinstance(eval(f.__annotations__['d']), stypes) + True + >>> print(f.__annotations__['return'][1:-1]) # First and last could be either " or ' + Return me! + >>> f.__annotations__['return'][0] == f.__annotations__['return'][-1] + True + """ + pass + + +def empty_decorator(cls): + return cls + + +@empty_decorator +class DecoratedStarship(object): + """ + >>> sorted(DecoratedStarship.__annotations__.items()) + [('captain', 'str'), ('damage', 'cython.int')] + """ + captain: str = 'Picard' # instance variable with default + damage: cython.int # instance variable without default diff --git a/tests/run/posonly.py b/tests/run/posonly.py new file mode 100644 index 000000000..0b4c999ea --- /dev/null +++ b/tests/run/posonly.py @@ -0,0 +1,568 @@ +# cython: always_allow_keywords=True +# mode: run +# tag: posonly, pure3.8 + +import cython +import sys +import pickle + +def test_optional_posonly_args1(a, b=10, /, c=100): + """ + >>> test_optional_posonly_args1(1, 2, 3) + 6 + >>> test_optional_posonly_args1(1, 2, c=3) + 6 + >>> test_optional_posonly_args1(1, b=2, c=3) # doctest: +ELLIPSIS + Traceback (most recent call last): + TypeError: test_optional_posonly_args1() got ... keyword argument... 'b' + >>> test_optional_posonly_args1(1, 2) + 103 + >>> test_optional_posonly_args1(1, b=2) # doctest: +ELLIPSIS + Traceback (most recent call last): + TypeError: test_optional_posonly_args1() got ... keyword argument... 'b' + """ + return a + b + c + +def test_optional_posonly_args2(a=1, b=10, /, c=100): + """ + >>> test_optional_posonly_args2(1, 2, 3) + 6 + >>> test_optional_posonly_args2(1, 2, c=3) + 6 + >>> test_optional_posonly_args2(1, b=2, c=3) # doctest: +ELLIPSIS + Traceback (most recent call last): + TypeError: test_optional_posonly_args2() got ... keyword argument... 'b' + >>> test_optional_posonly_args2(1, 2) + 103 + >>> test_optional_posonly_args2(1, b=2) # doctest: +ELLIPSIS + Traceback (most recent call last): + TypeError: test_optional_posonly_args2() got ... keyword argument... 'b' + >>> test_optional_posonly_args2(1, c=2) + 13 + """ + return a + b + c + +# TODO: this causes a line that is too long for old versions of Clang +#def many_args(a1,a2,a3,a4,a5,a6,a7,a8,a9,a10,a11,a12,a13,a14,a15,a16,a17,a18,a19,a20,a21, +# a22,a23,a24,a25,a26,a27,a28,a29,a30,a31,a32,a33,a34,a35,a36,a37,a38,a39,a40, +# a41,a42,a43,a44,a45,a46,a47,a48,a49,a50,a51,a52,a53,a54,a55,a56,a57,a58,a59, +# a60,a61,a62,a63,a64,a65,a66,a67,a68,a69,a70,a71,a72,a73,a74,a75,a76,a77,a78, +# a79,a80,a81,a82,a83,a84,a85,a86,a87,a88,a89,a90,a91,a92,a93,a94,a95,a96,a97, +# a98,a99,a100,a101,a102,a103,a104,a105,a106,a107,a108,a109,a110,a111,a112, +# a113,a114,a115,a116,a117,a118,a119,a120,a121,a122,a123,a124,a125,a126,a127, +# a128,a129,a130,a131,a132,a133,a134,a135,a136,a137,a138,a139,a140,a141,a142, +# a143,a144,a145,a146,a147,a148,a149,a150,a151,a152,a153,a154,a155,a156,a157, +# a158,a159,a160,a161,a162,a163,a164,a165,a166,a167,a168,a169,a170,a171,a172, +# a173,a174,a175,a176,a177,a178,a179,a180,a181,a182,a183,a184,a185,a186,a187, +# a188,a189,a190,a191,a192,a193,a194,a195,a196,a197,a198,a199,a200,a201,a202, +# a203,a204,a205,a206,a207,a208,a209,a210,a211,a212,a213,a214,a215,a216,a217, +# a218,a219,a220,a221,a222,a223,a224,a225,a226,a227,a228,a229,a230,a231,a232, +# a233,a234,a235,a236,a237,a238,a239,a240,a241,a242,a243,a244,a245,a246,a247, +# a248,a249,a250,a251,a252,a253,a254,a255,a256,a257,a258,a259,a260,a261,a262, +# a263,a264,a265,a266,a267,a268,a269,a270,a271,a272,a273,a274,a275,a276,a277, +# a278,a279,a280,a281,a282,a283,a284,a285,a286,a287,a288,a289,a290,a291,a292, +# a293,a294,a295,a296,a297,a298,a299,/,b,c=42,*,d): +# """ +# >>> many_args(*range(299),b=1,c=2,d=3) +# (298, 1, 2, 3) +# >>> many_args(*range(299),b=1,d=3) +# (298, 1, 42, 3) +# >>> many_args(*range(300),d=3) +# (298, 299, 42, 3) +# """ +# return (a299, b, c, d) + +#TODO: update this test for Python 3.8 final +@cython.binding(True) +def func_introspection1(a, b, c, /, d, e=1, *, f, g=2): + """ + >>> if sys.version_info[0] < 3: + ... assert func_introspection2.__code__.co_argcount == 7, func_introspection2.__code__.co_argcount + ... else: + ... assert func_introspection2.__code__.co_argcount == 5, func_introspection2.__code__.co_argcount + >>> func_introspection1.__defaults__ + (1,) + """ + +@cython.binding(True) +def func_introspection2(a, b, c=1, /, d=2, e=3, *, f, g=4): + """ + >>> if sys.version_info[0] < 3: + ... assert func_introspection2.__code__.co_argcount == 7, func_introspection2.__code__.co_argcount + ... else: + ... assert func_introspection2.__code__.co_argcount == 5, func_introspection2.__code__.co_argcount + >>> func_introspection2.__defaults__ + (1, 2, 3) + """ + +def test_pos_only_call_via_unpacking(a, b, /): + """ + >>> test_pos_only_call_via_unpacking(*[1,2]) + 3 + """ + return a + b + +def test_use_positional_as_keyword1(a, /): + """ + >>> test_use_positional_as_keyword1(1) + >>> test_use_positional_as_keyword1(a=1) # doctest: +ELLIPSIS + Traceback (most recent call last): + TypeError: test_use_positional_as_keyword1() ... keyword arguments... + """ + +def test_use_positional_as_keyword2(a, /, b): + """ + >>> test_use_positional_as_keyword2(1, 2) + >>> test_use_positional_as_keyword2(1, b=2) + >>> test_use_positional_as_keyword2(a=1, b=2) # doctest: +ELLIPSIS + Traceback (most recent call last): + TypeError: test_use_positional_as_keyword2() ... positional...arguments... + """ + +def test_use_positional_as_keyword3(a, b, /): + """ + >>> test_use_positional_as_keyword3(1, 2) + >>> test_use_positional_as_keyword3(a=1, b=2) # doctest:+ELLIPSIS + Traceback (most recent call last): + TypeError: test_use_positional_as_keyword3() got ... keyword argument... + """ + +def test_positional_only_and_arg_invalid_calls(a, b, /, c): + """ + >>> test_positional_only_and_arg_invalid_calls(1, 2, 3) + >>> test_positional_only_and_arg_invalid_calls(1, 2, c=3) + >>> test_positional_only_and_arg_invalid_calls(1, 2) # doctest: +ELLIPSIS + Traceback (most recent call last): + TypeError: test_positional_only_and_arg_invalid_calls() ... positional argument... + >>> test_positional_only_and_arg_invalid_calls(1) # doctest: +ELLIPSIS + Traceback (most recent call last): + TypeError: test_positional_only_and_arg_invalid_calls() ... positional arguments... + >>> test_positional_only_and_arg_invalid_calls(1,2,3,4) # doctest: +ELLIPSIS + Traceback (most recent call last): + TypeError: test_positional_only_and_arg_invalid_calls() takes ... positional arguments ...4 ...given... + """ + +def test_positional_only_and_optional_arg_invalid_calls(a, b, /, c=3): + """ + >>> test_positional_only_and_optional_arg_invalid_calls(1, 2) + >>> test_positional_only_and_optional_arg_invalid_calls(1) # doctest: +ELLIPSIS + Traceback (most recent call last): + TypeError: test_positional_only_and_optional_arg_invalid_calls() ... positional argument... + >>> test_positional_only_and_optional_arg_invalid_calls() # doctest: +ELLIPSIS + Traceback (most recent call last): + TypeError: test_positional_only_and_optional_arg_invalid_calls() ... positional arguments... + >>> test_positional_only_and_optional_arg_invalid_calls(1, 2, 3, 4) # doctest: +ELLIPSIS + Traceback (most recent call last): + TypeError: test_positional_only_and_optional_arg_invalid_calls() takes ... positional arguments ...4 ...given... + """ + +def test_positional_only_and_kwonlyargs_invalid_calls(a, b, /, c, *, d, e): + """ + >>> test_positional_only_and_kwonlyargs_invalid_calls(1, 2, 3, d=1, e=2) + >>> test_positional_only_and_kwonlyargs_invalid_calls(1, 2, 3, e=2) # doctest: +ELLIPSIS + Traceback (most recent call last): + TypeError: test_positional_only_and_kwonlyargs_invalid_calls() ... keyword-only argument...d... + >>> test_positional_only_and_kwonlyargs_invalid_calls(1, 2, 3) # doctest: +ELLIPSIS + Traceback (most recent call last): + TypeError: test_positional_only_and_kwonlyargs_invalid_calls() ... keyword-only argument...d... + >>> test_positional_only_and_kwonlyargs_invalid_calls(1, 2) # doctest: +ELLIPSIS + Traceback (most recent call last): + TypeError: test_positional_only_and_kwonlyargs_invalid_calls() ... positional argument... + >>> test_positional_only_and_kwonlyargs_invalid_calls(1) # doctest: +ELLIPSIS + Traceback (most recent call last): + TypeError: test_positional_only_and_kwonlyargs_invalid_calls() ... positional arguments... + >>> test_positional_only_and_kwonlyargs_invalid_calls() # doctest: +ELLIPSIS + Traceback (most recent call last): + TypeError: test_positional_only_and_kwonlyargs_invalid_calls() ... positional arguments... + >>> test_positional_only_and_kwonlyargs_invalid_calls(1, 2, 3, 4, 5, 6, d=7, e=8) # doctest: +ELLIPSIS + Traceback (most recent call last): + TypeError: test_positional_only_and_kwonlyargs_invalid_calls() takes ... positional arguments ... + >>> test_positional_only_and_kwonlyargs_invalid_calls(1, 2, 3, d=1, e=4, f=56) # doctest: +ELLIPSIS + Traceback (most recent call last): + TypeError: test_positional_only_and_kwonlyargs_invalid_calls() got an unexpected keyword argument 'f' + """ + +def test_positional_only_invalid_calls(a, b, /): + """ + >>> test_positional_only_invalid_calls(1, 2) + >>> test_positional_only_invalid_calls(1) # doctest: +ELLIPSIS + Traceback (most recent call last): + TypeError: test_positional_only_invalid_calls() ... positional argument... + >>> test_positional_only_invalid_calls() # doctest: +ELLIPSIS + Traceback (most recent call last): + TypeError: test_positional_only_invalid_calls() ... positional arguments... + >>> test_positional_only_invalid_calls(1, 2, 3) # doctest: +ELLIPSIS + Traceback (most recent call last): + TypeError: test_positional_only_invalid_calls() takes ... positional arguments ...3 ...given... + """ + +def test_positional_only_with_optional_invalid_calls(a, b=2, /): + """ + >>> test_positional_only_with_optional_invalid_calls(1) + >>> test_positional_only_with_optional_invalid_calls() # doctest: +ELLIPSIS + Traceback (most recent call last): + TypeError: test_positional_only_with_optional_invalid_calls() ... positional argument... + >>> test_positional_only_with_optional_invalid_calls(1, 2, 3) # doctest: +ELLIPSIS + Traceback (most recent call last): + TypeError: test_positional_only_with_optional_invalid_calls() takes ... positional arguments ...3 ...given... + """ + +def test_no_standard_args_usage(a, b, /, *, c): + """ + >>> test_no_standard_args_usage(1, 2, c=3) + >>> test_no_standard_args_usage(1, b=2, c=3) # doctest: +ELLIPSIS + Traceback (most recent call last): + TypeError: test_no_standard_args_usage() ... positional... arguments... + """ + +#def test_change_default_pos_only(): +# TODO: probably remove this, since __defaults__ is not writable in Cython? +# """ +# >>> test_change_default_pos_only() +# True +# True +# """ +# def f(a, b=2, /, c=3): +# return a + b + c +# +# print((2,3) == f.__defaults__) +# f.__defaults__ = (1, 2, 3) +# print(f(1, 2, 3) == 6) + +def test_lambdas(): + """ + >>> test_lambdas() + 3 + 3 + 3 + 3 + 3 + """ + x = lambda a, /, b: a + b + print(x(1,2)) + print(x(1,b=2)) + + x = lambda a, /, b=2: a + b + print(x(1)) + + x = lambda a, b, /: a + b + print(x(1, 2)) + + x = lambda a, b, /, : a + b + print(x(1, 2)) + +class TestPosonlyMethods(object): + """ + >>> TestPosonlyMethods().f(1,2) + (1, 2) + >>> TestPosonlyMethods.f(TestPosonlyMethods(), 1, 2) + (1, 2) + >>> try: + ... TestPosonlyMethods.f(1,2) + ... except TypeError: + ... print("Got type error") + Got type error + >>> TestPosonlyMethods().f(1, b=2) # doctest: +ELLIPSIS + Traceback (most recent call last): + TypeError: ...f() got ... keyword argument... 'b' + """ + def f(self, a, b, /): + return a, b + +class TestMangling(object): + """ + >>> TestMangling().f() + 42 + >>> TestMangling().f2() + 42 + + #>>> TestMangling().f3() + #(42, 43) + #>>> TestMangling().f4() + #(42, 43, 44) + + >>> TestMangling().f2(1) + 1 + + #>>> TestMangling().f3(1, _TestMangling__b=2) + #(1, 2) + #>>> TestMangling().f4(1, _TestMangling__b=2, _TestMangling__c=3) + #(1, 2, 3) + """ + def f(self, *, __a=42): + return __a + + def f2(self, __a=42, /): + return __a + +# FIXME: https://github.com/cython/cython/issues/1382 +# def f3(self, __a=42, /, __b=43): +# return (__a, __b) + +# def f4(self, __a=42, /, __b=43, *, __c=44): +# return (__a, __b, __c) + +def test_module_function(a, b, /): + """ + >>> test_module_function(1, 2) + >>> test_module_function() # doctest: +ELLIPSIS + Traceback (most recent call last): + TypeError: test_module_function() ... positional arguments... + """ + +def test_closures1(x,y): + """ + >>> test_closures1(1,2)(3,4) + 10 + >>> test_closures1(1,2)(3) # doctest: +ELLIPSIS + Traceback (most recent call last): + TypeError: ...g() ... positional argument... + >>> test_closures1(1,2)(3,4,5) # doctest: +ELLIPSIS + Traceback (most recent call last): + TypeError: ...g() ... positional argument... + """ + def g(x2, /, y2): + return x + y + x2 + y2 + return g + +def test_closures2(x, /, y): + """ + >>> test_closures2(1,2)(3,4) + 10 + """ + def g(x2,y2): + return x + y + x2 + y2 + return g + + +def test_closures3(x, /, y): + """ + >>> test_closures3(1,2)(3,4) + 10 + >>> test_closures3(1,2)(3) # doctest: +ELLIPSIS + Traceback (most recent call last): + TypeError: ...g() ... positional argument... + >>> test_closures3(1,2)(3,4,5) # doctest: +ELLIPSIS + Traceback (most recent call last): + TypeError: ...g() ... positional argument... + """ + def g(x2, /, y2): + return x + y + x2 + y2 + return g + + +def test_same_keyword_as_positional_with_kwargs(something, /, **kwargs): + """ + >>> test_same_keyword_as_positional_with_kwargs(42, something=42) + (42, {'something': 42}) + >>> test_same_keyword_as_positional_with_kwargs(something=42) # doctest: +ELLIPSIS + Traceback (most recent call last): + TypeError: test_same_keyword_as_positional_with_kwargs() ... positional argument... + >>> test_same_keyword_as_positional_with_kwargs(42) + (42, {}) + """ + return (something, kwargs) + +def test_serialization1(a, b, /): + """ + >>> pickled_posonly = pickle.dumps(test_serialization1) + >>> unpickled_posonly = pickle.loads(pickled_posonly) + >>> unpickled_posonly(1, 2) + (1, 2) + >>> unpickled_posonly(a=1, b=2) # doctest: +ELLIPSIS + Traceback (most recent call last): + TypeError: test_serialization1() got ... keyword argument... + """ + return (a, b) + +def test_serialization2(a, /, b): + """ + >>> pickled_optional = pickle.dumps(test_serialization2) + >>> unpickled_optional = pickle.loads(pickled_optional) + >>> unpickled_optional(1, 2) + (1, 2) + >>> unpickled_optional(a=1, b=2) # doctest: +ELLIPSIS + Traceback (most recent call last): + TypeError: test_serialization2() ... positional... arguments... + """ + return (a, b) + +def test_serialization3(a=1, /, b=2): + """ + >>> pickled_defaults = pickle.dumps(test_serialization3) + >>> unpickled_defaults = pickle.loads(pickled_defaults) + >>> unpickled_defaults(1, 2) + (1, 2) + >>> unpickled_defaults(a=1, b=2) # doctest: +ELLIPSIS + Traceback (most recent call last): + TypeError: test_serialization3() got ... keyword argument... 'a' + """ + return (a, b) + + +async def test_async(a=1, /, b=2): + """ + >>> test_async(a=1, b=2) # doctest: +ELLIPSIS + Traceback (most recent call last): + TypeError: test_async() got ... keyword argument... 'a' + """ + return a, b + + +def test_async_call(*args, **kwargs): + """ + >>> test_async_call(1, 2) + >>> test_async_call(1, b=2) + >>> test_async_call(1) + >>> test_async_call() + """ + if sys.version_info < (3, 6): + return + try: + coro = test_async(*args, **kwargs) + coro.send(None) + except StopIteration as e: + result = e.value + assert result == (1, 2), result + + +def test_generator(a=1, /, b=2): + """ + >>> test_generator(a=1, b=2) # doctest: +ELLIPSIS + Traceback (most recent call last): + TypeError: test_generator() got ... keyword argument... 'a' + >>> gen = test_generator(1, 2) + >>> next(gen) + (1, 2) + >>> gen = test_generator(1, b=2) + >>> next(gen) + (1, 2) + >>> gen = test_generator(1) + >>> next(gen) + (1, 2) + >>> gen = test_generator() + >>> next(gen) + (1, 2) + """ + yield a, b + +def f_call_1_0_0(a,/): + """ + >>> f_call_1_0_0(1) + (1,) + """ + return (a,) + +def f_call_1_1_0(a, /, b): + """ + >>> f_call_1_1_0(1,2) + (1, 2) + """ + return (a,b) + +def f_call_1_1_1(a, /, b, *, c): + """ + >>> f_call_1_1_1(1,2,c=3) + (1, 2, 3) + """ + return (a,b,c) + +def f_call_1_1_1_star(a, /, b, *args, c): + """ + >>> f_call_1_1_1_star(1,2,c=3) + (1, 2, (), 3) + >>> f_call_1_1_1_star(1,2,3,4,5,6,7,8,c=9) + (1, 2, (3, 4, 5, 6, 7, 8), 9) + """ + return (a,b,args,c) + +def f_call_1_1_1_kwds(a, /, b, *, c, **kwds): + """ + >>> f_call_1_1_1_kwds(1,2,c=3) + (1, 2, 3, {}) + >>> f_call_1_1_1_kwds(1,2,c=3,d=4,e=5) == (1, 2, 3, {'d': 4, 'e': 5}) + True + """ + return (a,b,c,kwds) + +def f_call_1_1_1_star_kwds(a, /, b, *args, c, **kwds): + """ + >>> f_call_1_1_1_star_kwds(1,2,c=3,d=4,e=5) == (1, 2, (), 3, {'d': 4, 'e': 5}) + True + >>> f_call_1_1_1_star_kwds(1,2,3,4,c=5,d=6,e=7) == (1, 2, (3, 4), 5, {'d': 6, 'e': 7}) + True + """ + return (a,b,args,c,kwds) + +def f_call_one_optional_kwd(a, /, *, b=2): + """ + >>> f_call_one_optional_kwd(1) + (1, 2) + >>> f_call_one_optional_kwd(1, b=3) + (1, 3) + """ + return (a,b) + +def f_call_posonly_stararg(a, /, *args): + """ + >>> f_call_posonly_stararg(1) + (1, ()) + >>> f_call_posonly_stararg(1, 2, 3, 4) + (1, (2, 3, 4)) + """ + return (a,args) + +def f_call_posonly_kwarg(a, /, **kw): + """ + >>> f_call_posonly_kwarg(1) + (1, {}) + >>> all_args = f_call_posonly_kwarg(1, b=2, c=3, d=4) + >>> all_args == (1, {'b': 2, 'c': 3, 'd': 4}) or all_args + True + """ + return (a,kw) + +def f_call_posonly_stararg_kwarg(a, /, *args, **kw): + """ + >>> f_call_posonly_stararg_kwarg(1) + (1, (), {}) + >>> f_call_posonly_stararg_kwarg(1, 2) + (1, (2,), {}) + >>> all_args = f_call_posonly_stararg_kwarg(1, b=3, c=4) + >>> all_args == (1, (), {'b': 3, 'c': 4}) or all_args + True + >>> all_args = f_call_posonly_stararg_kwarg(1, 2, b=3, c=4) + >>> all_args == (1, (2,), {'b': 3, 'c': 4}) or all_args + True + """ + return (a,args,kw) + +def test_empty_kwargs(a, b, /): + """ + >>> test_empty_kwargs(1, 2) + (1, 2) + >>> test_empty_kwargs(1, 2, **{}) + (1, 2) + >>> test_empty_kwargs(1, 2, **{'c': 3}) + Traceback (most recent call last): + TypeError: test_empty_kwargs() got an unexpected keyword argument 'c' + """ + return (a,b) + + +@cython.cclass +class TestExtensionClass: + """ + >>> t = TestExtensionClass() + >>> t.f(1,2) + (1, 2, 3) + >>> t.f(1,2,4) + (1, 2, 4) + >>> t.f(1, 2, c=4) + (1, 2, 4) + >>> t.f(1, 2, 5, c=6) # doctest: +ELLIPSIS + Traceback (most recent call last): + TypeError: ...f() got multiple values for ...argument 'c' + """ + def f(self, a, b, /, c=3): + return (a,b,c) diff --git a/tests/run/powop.pyx b/tests/run/powop.pyx index 414774562..3f9fdd077 100644 --- a/tests/run/powop.pyx +++ b/tests/run/powop.pyx @@ -123,9 +123,9 @@ def optimised_pow2(n): 0.5 >>> optimised_pow2(0.5) == 2 ** 0.5 True - >>> optimised_pow2('test') + >>> optimised_pow2('test') # doctest: +ELLIPSIS Traceback (most recent call last): - TypeError: unsupported operand type(s) for ** or pow(): 'int' and 'str' + TypeError: ...operand... **... """ if isinstance(n, (int, long)) and 0 <= n < 1000: assert isinstance(2.0 ** n, float), 'float %s' % n @@ -153,9 +153,9 @@ def optimised_pow2_inplace(n): 0.5 >>> optimised_pow2_inplace(0.5) == 2 ** 0.5 True - >>> optimised_pow2_inplace('test') #doctest: +ELLIPSIS + >>> optimised_pow2_inplace('test') # doctest: +ELLIPSIS Traceback (most recent call last): - TypeError: unsupported operand type(s) for ...: 'int' and 'str' + TypeError: ...operand... **... """ x = 2 x **= n diff --git a/tests/run/property_decorator_T593.py b/tests/run/property_decorator_T593.py index 45ddb57a8..d34db8951 100644 --- a/tests/run/property_decorator_T593.py +++ b/tests/run/property_decorator_T593.py @@ -1,5 +1,5 @@ # mode: run -# ticket: 593 +# ticket: t593 # tag: property, decorator my_property = property diff --git a/tests/run/pstats_profile_test.pyx b/tests/run/pstats_profile_test.pyx index da17f08c5..944bdd9fb 100644 --- a/tests/run/pstats_profile_test.pyx +++ b/tests/run/pstats_profile_test.pyx @@ -1,7 +1,7 @@ # tag: pstats # cython: profile = True -__doc__ = u""" +u""" >>> import os, tempfile, cProfile as profile, pstats >>> statsfile = tempfile.mkstemp()[1] >>> profile.runctx("test_profile(100)", locals(), globals(), statsfile) @@ -12,9 +12,7 @@ __doc__ = u""" >>> short_stats['f_cdef'] 100 >>> short_stats['f_cpdef'] - 200 - >>> short_stats['f_cpdef (wrapper)'] - 100 + 300 >>> short_stats['f_inline'] 100 >>> short_stats['f_inline_prof'] @@ -49,10 +47,8 @@ __doc__ = u""" 200 >>> short_stats['m_cdef'] 100 - >>> short_stats['m_cpdef'] - 200 - >>> short_stats['m_cpdef (wrapper)'] - 100 + >>> short_stats['m_cpdef'] - (200 if CPDEF_METHODS_COUNT_TWICE else 0) # FIXME! + 300 >>> try: ... os.unlink(statsfile) @@ -60,10 +56,10 @@ __doc__ = u""" ... pass >>> sorted(callees(s, 'test_profile')) #doctest: +NORMALIZE_WHITESPACE - ['f_cdef', 'f_cpdef', 'f_cpdef (wrapper)', 'f_def', + ['f_cdef', 'f_cpdef', 'f_def', 'f_inline', 'f_inline_prof', 'f_raise', - 'm_cdef', 'm_cpdef', 'm_cpdef (wrapper)', 'm_def', + 'm_cdef', 'm_cpdef', 'm_def', 'withgil_prof'] >>> profile.runctx("test_generators()", locals(), globals(), statsfile) @@ -121,6 +117,15 @@ __doc__ = u""" cimport cython + +# FIXME: With type specs, cpdef methods are currently counted twice. +# https://github.com/cython/cython/issues/2137 +cdef extern from *: + int CYTHON_USE_TYPE_SPECS + +CPDEF_METHODS_COUNT_TWICE = CYTHON_USE_TYPE_SPECS + + def callees(pstats, target_caller): pstats.calc_callees() for (_, _, caller), callees in pstats.all_callees.items(): @@ -129,10 +134,11 @@ def callees(pstats, target_caller): if 'pyx' in file: yield callee + def test_profile(long N): cdef long i, n = 0 cdef A a = A() - for i from 0 <= i < N: + for i in range(N): n += f_def(i) n += f_cdef(i) n += f_cpdef(i) diff --git a/tests/run/pstats_profile_test_pycfunc.pyx b/tests/run/pstats_profile_test_pycfunc.pyx new file mode 100644 index 000000000..717811082 --- /dev/null +++ b/tests/run/pstats_profile_test_pycfunc.pyx @@ -0,0 +1,228 @@ +# tag: pstats +# cython: profile = True +# cython: binding = False + +__doc__ = u""" + >>> import os, tempfile, cProfile as profile, pstats + >>> statsfile = tempfile.mkstemp()[1] + >>> profile.runctx("test_profile(100)", locals(), globals(), statsfile) + >>> s = pstats.Stats(statsfile) + >>> short_stats = dict([(k[2], v[1]) for k,v in s.stats.items()]) + >>> short_stats['f_def'] + 100 + >>> short_stats['f_cdef'] + 100 + >>> short_stats['f_cpdef'] + 200 + >>> short_stats['f_cpdef (wrapper)'] + 100 + >>> short_stats['f_inline'] + 100 + >>> short_stats['f_inline_prof'] + 100 + >>> short_stats['f_noprof'] + Traceback (most recent call last): + ... + KeyError: 'f_noprof' + >>> short_stats['f_raise'] + 100 + + >>> short_stats['withgil_prof'] + 100 + >>> short_stats['withgil_noprof'] + Traceback (most recent call last): + ... + KeyError: 'withgil_noprof' + + >>> short_stats['nogil_prof'] + Traceback (most recent call last): + ... + KeyError: 'nogil_prof' + >>> short_stats['nogil_noprof'] + Traceback (most recent call last): + ... + KeyError: 'nogil_noprof' + + >>> short_stats['f_raise'] + 100 + + >>> short_stats['m_def'] + 200 + >>> short_stats['m_cdef'] + 100 + >>> short_stats['m_cpdef'] + 200 + >>> short_stats['m_cpdef (wrapper)'] + 100 + + >>> try: + ... os.unlink(statsfile) + ... except: + ... pass + + >>> sorted(callees(s, 'test_profile')) #doctest: +NORMALIZE_WHITESPACE + ['f_cdef', 'f_cpdef', 'f_cpdef (wrapper)', 'f_def', + 'f_inline', 'f_inline_prof', + 'f_raise', + 'm_cdef', 'm_cpdef', 'm_cpdef (wrapper)', 'm_def', + 'withgil_prof'] + + >>> profile.runctx("test_generators()", locals(), globals(), statsfile) + >>> s = pstats.Stats(statsfile) + >>> short_stats = dict([(k[2], v[1]) for k,v in s.stats.items()]) + >>> short_stats['generator'] + 3 + + >>> short_stats['generator_exception'] + 2 + + >>> short_stats['genexpr'] + 11 + + >>> sorted(callees(s, 'test_generators')) + ['call_generator', 'call_generator_exception', 'generator_expr'] + + >>> list(callees(s, 'call_generator')) + ['generator'] + + >>> list(callees(s, 'generator')) + [] + + >>> list(callees(s, 'generator_exception')) + [] + + >>> list(callees(s, 'generator_expr')) + ['genexpr'] + + >>> list(callees(s, 'genexpr')) + [] + + >>> def python_generator(): + ... yield 1 + ... yield 2 + >>> def call_python_generator(): + ... list(python_generator()) + + >>> profile.runctx("call_python_generator()", locals(), globals(), statsfile) + >>> python_stats = pstats.Stats(statsfile) + >>> python_stats_dict = dict([(k[2], v[1]) for k,v in python_stats.stats.items()]) + + >>> profile.runctx("call_generator()", locals(), globals(), statsfile) + >>> cython_stats = pstats.Stats(statsfile) + >>> cython_stats_dict = dict([(k[2], v[1]) for k,v in cython_stats.stats.items()]) + + >>> python_stats_dict['python_generator'] == cython_stats_dict['generator'] + True + + >>> try: + ... os.unlink(statsfile) + ... except: + ... pass +""" + +cimport cython + +def callees(pstats, target_caller): + pstats.calc_callees() + for (_, _, caller), callees in pstats.all_callees.items(): + if caller == target_caller: + for (file, line, callee) in callees.keys(): + if 'pyx' in file: + yield callee + +def test_profile(long N): + cdef long i, n = 0 + cdef A a = A() + for i from 0 <= i < N: + n += f_def(i) + n += f_cdef(i) + n += f_cpdef(i) + n += (<object>f_cpdef)(i) + n += f_inline(i) + n += f_inline_prof(i) + n += f_noprof(i) + n += nogil_noprof(i) + n += nogil_prof(i) + n += withgil_noprof(i) + n += withgil_prof(i) + n += a.m_def(i) + n += (<object>a).m_def(i) + n += a.m_cpdef(i) + n += (<object>a).m_cpdef(i) + n += a.m_cdef(i) + try: + n += f_raise(i+2) + except RuntimeError: + pass + return n + +def f_def(long a): + return a + +cdef long f_cdef(long a): + return a + +cpdef long f_cpdef(long a): + return a + +cdef inline long f_inline(long a): + return a + +@cython.profile(True) +cdef inline long f_inline_prof(long a): + return a + +@cython.profile(False) +cdef int f_noprof(long a): + return a + +cdef long f_raise(long) except -2: + raise RuntimeError + +@cython.profile(False) +cdef int withgil_noprof(long a) with gil: + return (a) +@cython.profile(True) +cdef int withgil_prof(long a) with gil: + return (a) + +@cython.profile(False) +cdef int nogil_noprof(long a) nogil: + return a +@cython.profile(True) +cdef int nogil_prof(long a) nogil: + return a + +cdef class A(object): + def m_def(self, long a): + return a + cpdef m_cpdef(self, long a): + return a + cdef m_cdef(self, long a): + return a + +def test_generators(): + call_generator() + call_generator_exception() + generator_expr() + +def call_generator(): + list(generator()) + +def generator(): + yield 1 + yield 2 + +def call_generator_exception(): + try: + list(generator_exception()) + except ValueError: + pass + +def generator_exception(): + yield 1 + raise ValueError(2) + +def generator_expr(): + e = (x for x in range(10)) + return sum(e) diff --git a/tests/run/ptr_warning_T714.pyx b/tests/run/ptr_warning_T714.pyx index 3ec8b5b12..3bd330525 100644 --- a/tests/run/ptr_warning_T714.pyx +++ b/tests/run/ptr_warning_T714.pyx @@ -1,6 +1,6 @@ # mode: run # tag: werror -# ticket: 714 +# ticket: t714 def test_ptr(): """ diff --git a/tests/run/public_fused_types.srctree b/tests/run/public_fused_types.srctree index 54c7aa148..2474e0eaa 100644 --- a/tests/run/public_fused_types.srctree +++ b/tests/run/public_fused_types.srctree @@ -196,8 +196,8 @@ import a as a_mod def ae(result, expected): "assert equals" if result != expected: - print 'result :', result - print 'expected:', expected + print('result :', result) + print('expected:', expected) assert result == expected @@ -227,7 +227,7 @@ ae(myobj.cpdef_method[cy.int, cy.float](10, 10.0), (10, 10.0)) d = {'obj': obj, 'myobj': myobj, 'ae': ae} -exec s in d +exec(s, d) # Test def methods # ae(obj.def_method(12, 14.9), 26) diff --git a/tests/run/pure.pyx b/tests/run/pure.pyx index 57a575a33..23e41c77b 100644 --- a/tests/run/pure.pyx +++ b/tests/run/pure.pyx @@ -25,10 +25,12 @@ def test_declare(n): (100, 100) >>> test_declare(100.5) (100, 100) - >>> test_declare(None) + + # CPython: "TypeError: an integer is required" + >>> test_declare(None) # doctest: +ELLIPSIS Traceback (most recent call last): ... - TypeError: an integer is required + TypeError: ...int... """ x = cython.declare(cython.int) y = cython.declare(cython.int, n) diff --git a/tests/run/pure_cdef_class_property_decorator_T264.pxd b/tests/run/pure_cdef_class_property_decorator_T264.pxd index 4c1d879cf..e5e616a00 100644 --- a/tests/run/pure_cdef_class_property_decorator_T264.pxd +++ b/tests/run/pure_cdef_class_property_decorator_T264.pxd @@ -1,5 +1,5 @@ # mode: run -# ticket: 264 +# ticket: t264 # tag: property, decorator cdef class Prop: diff --git a/tests/run/pure_cdef_class_property_decorator_T264.py b/tests/run/pure_cdef_class_property_decorator_T264.py index 70f299b4f..8ca2fc7b2 100644 --- a/tests/run/pure_cdef_class_property_decorator_T264.py +++ b/tests/run/pure_cdef_class_property_decorator_T264.py @@ -1,5 +1,5 @@ # mode: run -# ticket: 264 +# ticket: t264 # tag: property, decorator class Prop(object): diff --git a/tests/run/pure_fused.pxd b/tests/run/pure_fused.pxd new file mode 100644 index 000000000..c00105087 --- /dev/null +++ b/tests/run/pure_fused.pxd @@ -0,0 +1,9 @@ +cimport cython + +ctypedef fused NotInPy: + int + float + +cdef class TestCls: + @cython.locals(loc = NotInPy) + cpdef cpfunc(self, NotInPy arg) diff --git a/tests/run/pure_fused.py b/tests/run/pure_fused.py new file mode 100644 index 000000000..35a9d27d7 --- /dev/null +++ b/tests/run/pure_fused.py @@ -0,0 +1,64 @@ +# mode: run +# tag: fused, pure3.6 + +#cython: annotation_typing=True + +import cython + +InPy = cython.fused_type(cython.int, cython.float) + +class TestCls: + # although annotations as strings isn't recommended and generates a warning + # it does allow the test to run on more (pure) Python versions + def func1(self, arg: 'NotInPy'): + """ + >>> TestCls().func1(1.0) + 'float' + >>> TestCls().func1(2) + 'int' + """ + loc: 'NotInPy' = arg + return cython.typeof(arg) + + if cython.compiled: + @cython.locals(arg=NotInPy, loc=NotInPy) # NameError for 'NotInPy' in pure Python + def func2(self, arg): + """ + >>> TestCls().func2(1.0) + 'float' + >>> TestCls().func2(2) + 'int' + """ + loc = arg + return cython.typeof(arg) + + def cpfunc(self, arg): + """ + >>> TestCls().cpfunc(1.0) + 'float' + >>> TestCls().cpfunc(2) + 'int' + """ + loc = arg + return cython.typeof(arg) + + def func1_inpy(self, arg: InPy): + """ + >>> TestCls().func1_inpy(1.0) + 'float' + >>> TestCls().func1_inpy(2) + 'int' + """ + loc: InPy = arg + return cython.typeof(arg) + + @cython.locals(arg = InPy, loc = InPy) + def func2_inpy(self, arg): + """ + >>> TestCls().func2_inpy(1.0) + 'float' + >>> TestCls().func2_inpy(2) + 'int' + """ + loc = arg + return cython.typeof(arg) diff --git a/tests/run/pure_pxd.srctree b/tests/run/pure_pxd.srctree index 59c71cdf6..fcba49ac0 100644 --- a/tests/run/pure_pxd.srctree +++ b/tests/run/pure_pxd.srctree @@ -55,6 +55,17 @@ def func(a, b, c): """ return a + b + c +def sum_generator_expression(a): + # GH-3477 - closure variables incorrectly captured in functions transformed to cdef + return sum(i for i in range(a)) + +def run_sum_generator_expression(a): + """ + >>> run_sum_generator_expression(5) + 10 + """ + return sum_generator_expression(a) + def test(module): import os.path @@ -95,3 +106,6 @@ cdef class TypedMethod: cpdef int func(x, int y, z) except? -1 # argument names should not matter, types should + + +cdef int sum_generator_expression(int a) diff --git a/tests/run/pure_py.py b/tests/run/pure_py.py index 89139155c..55c90ddb2 100644 --- a/tests/run/pure_py.py +++ b/tests/run/pure_py.py @@ -36,10 +36,6 @@ def test_declare(n): (100, 100) >>> test_declare(100.5) (100, 100) - >>> test_declare(None) #doctest: +ELLIPSIS - Traceback (most recent call last): - ... - TypeError: ... """ x = cython.declare(cython.int) y = cython.declare(cython.int, n) @@ -144,16 +140,22 @@ def test_with_nogil(nogil, should_raise=False): MyUnion = cython.union(n=cython.int, x=cython.double) MyStruct = cython.struct(is_integral=cython.bint, data=MyUnion) MyStruct2 = cython.typedef(MyStruct[2]) +MyStruct3 = cython.typedef(MyStruct[3]) def test_struct(n, x): """ >>> test_struct(389, 1.64493) - (389, 1.64493) + (389, 1.64493, False) """ - a = cython.declare(MyStruct2) + a = cython.declare(MyStruct3) a[0] = MyStruct(is_integral=True, data=MyUnion(n=n)) a[1] = MyStruct(is_integral=False, data={'x': x}) - return a[0].data.n, a[1].data.x + if sys.version_info >= (3, 6): + # dict is ordered => struct creation via keyword arguments above was deterministic! + a[2] = MyStruct(False, MyUnion(x=x)) + else: + a[2] = MyStruct(is_integral=False, data=MyUnion(x=x)) + return a[0].data.n, a[1].data.x, a[2].is_integral import cython as cy from cython import declare, cast, locals, address, typedef, p_void, compiled @@ -221,6 +223,8 @@ def test_declare_c_types(n): @cython.ccall @cython.returns(cython.double) def c_call(x): + if x == -2.0: + raise RuntimeError("huhu!") return x @@ -239,6 +243,10 @@ def call_ccall(x): 1.0 >>> (is_compiled and 1) or result 1 + + >>> call_ccall(-2) + Traceback (most recent call last): + RuntimeError: huhu! """ ret = c_call(x) return ret, cython.typeof(ret) @@ -248,6 +256,8 @@ def call_ccall(x): @cython.inline @cython.returns(cython.double) def cdef_inline(x): + if x == -2.0: + raise RuntimeError("huhu!") return x + 1 @@ -262,6 +272,10 @@ def call_cdef_inline(x): 'int' >>> result == 2.0 or result True + + >>> call_cdef_inline(-2) + Traceback (most recent call last): + RuntimeError: huhu! """ ret = cdef_inline(x) return ret, cython.typeof(ret) @@ -396,6 +410,23 @@ def ccall_except_check_always(x): return x+1 +@cython.test_fail_if_path_exists("//CFuncDeclaratorNode//IntNode[@value = '-1']") +@cython.test_assert_path_exists("//CFuncDeclaratorNode") +@cython.ccall +@cython.returns(cython.long) +@cython.exceptval(check=False) +def ccall_except_no_check(x): + """ + >>> ccall_except_no_check(41) + 42 + >>> try: _ = ccall_except_no_check(0) # no exception propagated! + ... except ValueError: assert not is_compiled + """ + if x == 0: + raise ValueError + return x+1 + + @cython.final @cython.cclass class CClass(object): @@ -422,3 +453,132 @@ class TestUnboundMethod: True """ def meth(self): pass + +@cython.cclass +class Foo: + a = cython.declare(cython.double) + b = cython.declare(cython.double) + c = cython.declare(cython.double) + + @cython.locals(a=cython.double, b=cython.double, c=cython.double) + def __init__(self, a, b, c): + self.a = a + self.b = b + self.c = c + +@cython.cclass +class EmptyClass(object): + def __init__(self, *args): + pass + +def same_type_cast(): + """ + >>> same_type_cast() + True + """ + + f = EmptyClass() + return f is cython.cast(EmptyClass, f) + +def multi_args_init_cast(): + """ + >>> multi_args_init_cast() + True + """ + f = Foo(10, 20, 30) + return cython.cast(Foo, f) is f + +def multi_args_init_declare(): + """ + >>> multi_args_init_declare() is None + True + """ + f = cython.declare(Foo) + + if cython.compiled: + f = None + + return f + +EmptyClassSyn = cython.typedef(EmptyClass) + +def empty_declare(): + """ + >>> empty_declare() + [] + """ + + r0 = cython.declare(EmptyClass) + r1 = cython.declare(EmptyClassSyn) + r2 = cython.declare(MyStruct) + r3 = cython.declare(MyUnion) + r4 = cython.declare(MyStruct2) + r5 = cython.declare(cython.int[2]) + + if cython.compiled: + r0 = None + r1 = None + + res = [ + r0 is None, + r1 is None, + r2 is not None, + r3 is not None, + r4 is not None, + r5 is not None + ] + + r2.is_integral = True + assert( r2.is_integral == True ) + + r3.x = 12.3 + assert( r3.x == 12.3 ) + + #It generates a correct C code, but raises an exception when interpreted + if cython.compiled: + r4[0].is_integral = True + assert( r4[0].is_integral == True ) + + r5[0] = 42 + assert ( r5[0] == 42 ) + + return [i for i, x in enumerate(res) if not x] + +def same_declare(): + """ + >>> same_declare() + True + """ + + f = EmptyClass() + f2 = cython.declare(EmptyClass, f) + return f2 is f + +def none_cast(): + """ + >>> none_cast() is None + True + """ + + f = None + return cython.cast(EmptyClass, f) + +def none_declare(): + """ + >>> none_declare() is None + True + """ + + f = None + f2 = cython.declare(Foo, f) + return f2 + +def array_init_with_list(): + """ + >>> array_init_with_list() + [10, 42] + """ + x = cython.declare(cython.int[20], list(range(20))) + x[12] = 42 + + return [x[10], x[12]] diff --git a/tests/run/pure_py_cimports.py b/tests/run/pure_py_cimports.py new file mode 100644 index 000000000..57985cc43 --- /dev/null +++ b/tests/run/pure_py_cimports.py @@ -0,0 +1,13 @@ +# mode: run +# tag: pure, import, cimport + +from cython.cimports.libc import math +from cython.cimports.libc.math import ceil + + +def libc_math_ceil(x): + """ + >>> libc_math_ceil(1.5) + [2, 2] + """ + return [int(n) for n in [ceil(x), math.ceil(x)]] diff --git a/tests/run/pure_pyx_cimports.pyx b/tests/run/pure_pyx_cimports.pyx new file mode 100644 index 000000000..17133684d --- /dev/null +++ b/tests/run/pure_pyx_cimports.pyx @@ -0,0 +1,19 @@ +# mode: run +# tag: pure, import, cimport + +cimport cython.cimports.libc.math as libc_math1 + +from cython.cimports.libc import math as libc_math2 +from cython.cimports.libc.math import ceil as math_ceil + +#from cython.cimports cimport libc # FIXME: currently crashes during analysis when submodule cannot be found +from cython.cimports.libc cimport math +from cython.cimports.libc.math cimport ceil + + +def libc_math_ceil(x): + """ + >>> libc_math_ceil(1.5) + [2, 2, 2, 2, 2] + """ + return [int(n) for n in [ceil(x), math.ceil(x), libc_math1.ceil(x), libc_math2.ceil(x), math_ceil(x)]] diff --git a/tests/run/pxd_argument_names.srctree b/tests/run/pxd_argument_names.srctree index 885643609..94262ef92 100644 --- a/tests/run/pxd_argument_names.srctree +++ b/tests/run/pxd_argument_names.srctree @@ -1,5 +1,5 @@ # mode: run -# ticket: gh1888 +# ticket: 1888 PYTHON setup.py build_ext --inplace PYTHON -c "import a; a.test()" diff --git a/tests/run/py34_signature.pyx b/tests/run/py34_signature.pyx index 6bfaec677..14780ff7b 100644 --- a/tests/run/py34_signature.pyx +++ b/tests/run/py34_signature.pyx @@ -101,19 +101,17 @@ cpdef cp1(a, b): """ -# Currently broken, see GH #1864 cpdef cp2(a, b=True): """ >>> def py_cp2(a, b=True): pass - #>>> signatures_match(cp2, py_cp2) + >>> signatures_match(cp2, py_cp2) """ -# Currently broken, see GH #1864 cpdef cp3(a=1, b=True): """ >>> def py_cp3(a=1, b=True): pass - #>>> signatures_match(cp3, py_cp3) + >>> signatures_match(cp3, py_cp3) """ diff --git a/tests/run/py35_asyncio_async_def.srctree b/tests/run/py35_asyncio_async_def.srctree index 9da5560b3..bbfe8bbe6 100644 --- a/tests/run/py35_asyncio_async_def.srctree +++ b/tests/run/py35_asyncio_async_def.srctree @@ -1,5 +1,5 @@ # mode: run -# tag: asyncio, gh1685 +# tag: asyncio, gh1685, gh2273 PYTHON setup.py build_ext -i PYTHON main.py @@ -19,6 +19,7 @@ setup( import asyncio import cy_test +import py_test from contextlib import closing async def main(): @@ -31,6 +32,12 @@ with closing(asyncio.get_event_loop()) as loop: print("Running Cython coroutine ...") loop.run_until_complete(cy_test.say()) +assert asyncio.iscoroutinefunction(cy_test.cy_async_def_example) == True +assert asyncio.iscoroutinefunction(cy_test.cy_async_def_example) == True +assert asyncio.iscoroutinefunction(py_test.py_async_def_example) == True +assert asyncio.iscoroutinefunction(py_test.py_async_def_example) == True +assert asyncio.iscoroutinefunction(cy_test.cy_def_example) == False +assert asyncio.iscoroutinefunction(py_test.py_def_example) == False ######## cy_test.pyx ######## @@ -51,8 +58,19 @@ async def cb(): await asyncio.sleep(0.5) print("done!") +async def cy_async_def_example(): + return 1 + +def cy_def_example(): + return 1 ######## py_test.py ######## async def py_async(): print("- and this one is from Python") + +async def py_async_def_example(): + return 1 + +def py_def_example(): + return 1 diff --git a/tests/run/py_hash_t.pyx b/tests/run/py_hash_t.pyx index f18a8c3ac..3b20584d0 100644 --- a/tests/run/py_hash_t.pyx +++ b/tests/run/py_hash_t.pyx @@ -2,12 +2,30 @@ cimport cython +class IntLike(object): + def __init__(self, value): + self.value = value + def __index__(self): + return self.value + + def assign_py_hash_t(x): """ >>> assign_py_hash_t(12) 12 >>> assign_py_hash_t(-12) -12 + + >>> assign_py_hash_t(IntLike(-3)) + -3 + >>> assign_py_hash_t(IntLike(1 << 100)) # doctest: +ELLIPSIS + Traceback (most recent call last): + ... + OverflowError: ... + >>> assign_py_hash_t(IntLike(1.5)) # doctest: +ELLIPSIS + Traceback (most recent call last): + ... + TypeError: __index__ ... (type ...float...) """ cdef Py_hash_t h = x return h diff --git a/tests/run/py_ucs4_type.pyx b/tests/run/py_ucs4_type.pyx index 7193319c6..afd45fca3 100644 --- a/tests/run/py_ucs4_type.pyx +++ b/tests/run/py_ucs4_type.pyx @@ -132,15 +132,24 @@ def unicode_type_methods(Py_UCS4 uchar): uchar.isupper(), ] -@cython.test_assert_path_exists('//PythonCapiCallNode') -@cython.test_fail_if_path_exists('//SimpleCallNode') +#@cython.test_assert_path_exists('//PythonCapiCallNode') +#@cython.test_fail_if_path_exists('//SimpleCallNode') def unicode_methods(Py_UCS4 uchar): """ - >>> unicode_methods(ord('A')) == ['a', 'A', 'A'] + >>> unicode_methods(ord('A')) == ['a', 'A', 'A'] or unicode_methods(ord('A')) + True + >>> unicode_methods(ord('a')) == ['a', 'A', 'A'] or unicode_methods(ord('a')) True - >>> unicode_methods(ord('a')) == ['a', 'A', 'A'] + >>> unicode_methods(0x1E9E) == [u'\\xdf', u'\\u1e9e', u'\\u1e9e'] or unicode_methods(0x1E9E) + True + >>> unicode_methods(0x0130) in ( + ... [u'i\\u0307', u'\\u0130', u'\\u0130'], # Py3 + ... [u'i', u'\\u0130', u'\\u0130'], # Py2 + ... ) or unicode_methods(0x0130) True """ + # \u1E9E == 'LATIN CAPITAL LETTER SHARP S' + # \u0130 == 'LATIN CAPITAL LETTER I WITH DOT ABOVE' return [ # character conversion uchar.lower(), @@ -149,11 +158,11 @@ def unicode_methods(Py_UCS4 uchar): ] -@cython.test_assert_path_exists('//PythonCapiCallNode') -@cython.test_fail_if_path_exists( - '//SimpleCallNode', - '//CoerceFromPyTypeNode', -) +#@cython.test_assert_path_exists('//PythonCapiCallNode') +#@cython.test_fail_if_path_exists( +# '//SimpleCallNode', +# '//CoerceFromPyTypeNode', +#) def unicode_method_return_type(Py_UCS4 uchar): """ >>> unicode_method_return_type(ord('A')) @@ -366,5 +375,5 @@ def uchar_lookup_in_dict(obj, Py_UCS4 uchar): _WARNINGS = """ -364:16: Item lookup of unicode character codes now always converts to a Unicode string. Use an explicit C integer cast to get back the previous integer lookup behaviour. +373:16: Item lookup of unicode character codes now always converts to a Unicode string. Use an explicit C integer cast to get back the previous integer lookup behaviour. """ diff --git a/tests/run/py_unicode_strings.pyx b/tests/run/py_unicode_strings.pyx index 09ccebc3c..ba4df1087 100644 --- a/tests/run/py_unicode_strings.pyx +++ b/tests/run/py_unicode_strings.pyx @@ -1,15 +1,15 @@ +# mode: run # tag: py_unicode_strings import sys -cimport cython -from libc.string cimport memcpy, strcpy +from libc.string cimport memcpy -cdef bint Py_UNICODE_equal(const Py_UNICODE* u1, const Py_UNICODE* u2): - while u1[0] != 0 and u2[0] != 0 and u1[0] == u2[0]: - u1 += 1 - u2 += 1 - return u1[0] == u2[0] +cdef assert_Py_UNICODE_equal(const Py_UNICODE* u1, const Py_UNICODE* u2): + cdef size_t i = 0 + while u1[i] != 0 and u2[i] != 0 and u1[i] == u2[i]: + i += 1 + assert u1[i] == u2[i], f"Mismatch at position {i}: {<long>u1[i]} != {<long>u2[i]} ({u1!r} != {u2!r})" ctypedef Py_UNICODE* LPWSTR @@ -48,9 +48,10 @@ def test_c_to_python(): assert c_pu_str[1:7] == uobj[1:7] assert c_wstr[1:7] == uobj[1:7] - assert c_pu_arr[1] == uobj[1] - assert c_pu_str[1] == uobj[1] - assert c_wstr[1] == uobj[1] + cdef Py_UNICODE ch = uobj[1] # Py_UCS4 is unsigned, Py_UNICODE is usually signed. + assert c_pu_arr[1] == ch + assert c_pu_str[1] == ch + assert c_wstr[1] == ch assert len(c_pu_str) == 8 assert len(c_pu_arr) == 8 @@ -81,20 +82,20 @@ def test_python_to_c(): """ cdef unicode u - assert Py_UNICODE_equal(c_pu_arr, uobj) - assert Py_UNICODE_equal(c_pu_str, uobj) - assert Py_UNICODE_equal(c_pu_str, <LPWSTR>uobj) + assert_Py_UNICODE_equal(c_pu_arr, uobj) + assert_Py_UNICODE_equal(c_pu_str, uobj) + assert_Py_UNICODE_equal(c_pu_str, <LPWSTR>uobj) u = uobj[1:] - assert Py_UNICODE_equal(c_pu_str + 1, u) - assert Py_UNICODE_equal(c_wstr + 1, u) + assert_Py_UNICODE_equal(c_pu_str + 1, u) + assert_Py_UNICODE_equal(c_wstr + 1, u) u = uobj[:1] - assert Py_UNICODE_equal(<Py_UNICODE*>u"u", u) + assert_Py_UNICODE_equal(<Py_UNICODE*>u"u", u) u = uobj[1:7] - assert Py_UNICODE_equal(<Py_UNICODE*>u"nicode", u) + assert_Py_UNICODE_equal(<Py_UNICODE*>u"nicode", u) u = uobj[1] - assert Py_UNICODE_equal(<Py_UNICODE*>u"n", u) + assert_Py_UNICODE_equal(<Py_UNICODE*>u"n", u) - assert Py_UNICODE_equal(uwide_literal, <Py_UNICODE*>c_pu_wide_literal) + assert_Py_UNICODE_equal(uwide_literal, <Py_UNICODE*>c_pu_wide_literal) assert len(u"abc\0") == 4 assert len(<Py_UNICODE*>u"abc\0") == 3 diff --git a/tests/run/py_unicode_type.pyx b/tests/run/py_unicode_type.pyx index d8d172bc9..0d33be927 100644 --- a/tests/run/py_unicode_type.pyx +++ b/tests/run/py_unicode_type.pyx @@ -123,8 +123,8 @@ def unicode_type_methods(Py_UNICODE uchar): uchar.isupper(), ] -@cython.test_assert_path_exists('//PythonCapiCallNode') -@cython.test_fail_if_path_exists('//SimpleCallNode') +#@cython.test_assert_path_exists('//PythonCapiCallNode') +#@cython.test_fail_if_path_exists('//SimpleCallNode') def unicode_methods(Py_UNICODE uchar): """ >>> unicode_methods(ord('A')) == ['a', 'A', 'A'] diff --git a/tests/run/pyclass_annotations_pep526.py b/tests/run/pyclass_annotations_pep526.py new file mode 100644 index 000000000..154dc9cf6 --- /dev/null +++ b/tests/run/pyclass_annotations_pep526.py @@ -0,0 +1,59 @@ +# cython: language_level=3 +# mode: run +# tag: pure3.7, pep526, pep484 + +from __future__ import annotations + +import sys + +try: + from typing import ClassVar +except ImportError: # Py<=3.5 + ClassVar = {int: int} + +class NotAStr: + pass + +class PyAnnotatedClass: + """ + >>> PyAnnotatedClass.__annotations__["CLASS_VAR"] + 'ClassVar[int]' + >>> PyAnnotatedClass.__annotations__["obj"] + 'str' + >>> PyAnnotatedClass.__annotations__["literal"] + "'int'" + >>> PyAnnotatedClass.__annotations__["recurse"] + "'PyAnnotatedClass'" + >>> PyAnnotatedClass.__annotations__["default"] + 'bool' + >>> PyAnnotatedClass.CLASS_VAR + 1 + >>> PyAnnotatedClass.default + False + >>> PyAnnotatedClass.obj + Traceback (most recent call last): + ... + AttributeError: type object 'PyAnnotatedClass' has no attribute 'obj' + """ + CLASS_VAR: ClassVar[int] = 1 + obj: str + literal: "int" + recurse: "PyAnnotatedClass" + default: bool = False + # https://github.com/cython/cython/issues/4196 and https://github.com/cython/cython/issues/4198 + not_object: float = 0.1 # Shouldn't try to create a c attribute + lie_about_type: str = NotAStr # Shouldn't generate a runtime type-check + + +class PyVanillaClass: + """ + Before Py3.10, unannotated classes did not have '__annotations__'. + + >>> try: + ... a = PyVanillaClass.__annotations__ + ... except AttributeError: + ... assert sys.version_info < (3, 10) + ... else: + ... assert sys.version_info >= (3, 10) + ... assert a == {} + """ diff --git a/tests/run/pyclass_scope_T671.py b/tests/run/pyclass_scope_T671.py index 911268e22..00a10de11 100644 --- a/tests/run/pyclass_scope_T671.py +++ b/tests/run/pyclass_scope_T671.py @@ -1,5 +1,5 @@ # mode: run -# ticket: 671 +# ticket: t671 A = 1234 diff --git a/tests/run/pycontextvar.pyx b/tests/run/pycontextvar.pyx new file mode 100644 index 000000000..bf2d0876d --- /dev/null +++ b/tests/run/pycontextvar.pyx @@ -0,0 +1,45 @@ +# mode: run + +from cpython.contextvars cimport ( + PyContextVar_New, PyContextVar_New_with_default, + get_value, get_value_no_default, +) + +NOTHING = object() +CVAR = PyContextVar_New("cvar", NULL) +CVAR_WITH_DEFAULT = PyContextVar_New_with_default("cvar_wd", "DEFAULT") + + +import contextvars +PYCVAR = contextvars.ContextVar("pycvar") + + +def test_get_value(var, default=NOTHING): + """ + >>> test_get_value(CVAR) + >>> test_get_value(CVAR, "default") + 'default' + >>> test_get_value(PYCVAR) + >>> test_get_value(PYCVAR, "default") + 'default' + >>> test_get_value(CVAR_WITH_DEFAULT) + 'DEFAULT' + >>> test_get_value(CVAR_WITH_DEFAULT, "default") + 'DEFAULT' + """ + return get_value(var, default) if default is not NOTHING else get_value(var) + + +def test_get_value_no_default(var, default=NOTHING): + """ + >>> test_get_value_no_default(CVAR) + >>> test_get_value_no_default(CVAR, "default") + 'default' + >>> test_get_value_no_default(PYCVAR) + >>> test_get_value_no_default(PYCVAR, "default") + 'default' + >>> test_get_value_no_default(CVAR_WITH_DEFAULT) + >>> test_get_value_no_default(CVAR_WITH_DEFAULT, "default") + 'default' + """ + return get_value_no_default(var, default) if default is not NOTHING else get_value_no_default(var) diff --git a/tests/run/pyfunction_redefine_T489.pyx b/tests/run/pyfunction_redefine_T489.pyx index cf393407f..5931ff1b6 100644 --- a/tests/run/pyfunction_redefine_T489.pyx +++ b/tests/run/pyfunction_redefine_T489.pyx @@ -1,4 +1,4 @@ -# ticket: 489 +# ticket: t489 """ >>> xxx diff --git a/tests/run/pyintop.pyx b/tests/run/pyintop.pyx index c101542d0..a0bb1e9d9 100644 --- a/tests/run/pyintop.pyx +++ b/tests/run/pyintop.pyx @@ -25,6 +25,8 @@ def or_obj(obj2, obj3): @cython.test_fail_if_path_exists('//IntBinopNode') def or_int(obj2): """ + >>> or_int(0) + 16 >>> or_int(1) 17 >>> or_int(16) @@ -47,6 +49,8 @@ def xor_obj(obj2, obj3): @cython.test_fail_if_path_exists('//IntBinopNode') def xor_int(obj2): """ + >>> xor_int(0) + 16 >>> xor_int(2) 18 >>> xor_int(16) @@ -69,10 +73,14 @@ def and_obj(obj2, obj3): @cython.test_fail_if_path_exists('//IntBinopNode') def and_int(obj2): """ + >>> and_int(0) + 0 >>> and_int(1) 0 >>> and_int(18) 16 + >>> and_int(-1) + 16 """ obj1 = obj2 & 0x10 return obj1 @@ -98,9 +106,30 @@ def rshift_obj(obj2, obj3): return obj1 +@cython.test_assert_path_exists('//IntBinopNode') +def rshift_int_obj(obj3): + """ + >>> rshift_int_obj(3) + 0 + >>> rshift_int_obj(2) + 0 + >>> rshift_int_obj(1) + 1 + >>> rshift_int_obj(0) + 2 + >>> rshift_int_obj(-1) + Traceback (most recent call last): + ValueError: negative shift count + """ + obj1 = 2 >> obj3 + return obj1 + + @cython.test_fail_if_path_exists('//IntBinopNode') def rshift_int(obj2): """ + >>> rshift_int(0) + 0 >>> rshift_int(2) 0 @@ -226,6 +255,8 @@ def mixed_obj(obj2, obj3): ) def mixed_int(obj2): """ + >>> mixed_int(0) + 16 >>> mixed_int(2) 18 >>> mixed_int(16) diff --git a/tests/run/pyobjcast_T313.pyx b/tests/run/pyobjcast_T313.pyx index fb4ef80a0..0e5fc7e8e 100644 --- a/tests/run/pyobjcast_T313.pyx +++ b/tests/run/pyobjcast_T313.pyx @@ -1,4 +1,4 @@ -# ticket: 313 +# ticket: t313 # Ensure casting still works to void* """ diff --git a/tests/run/r_docstrings.pyx b/tests/run/r_docstrings.pyx index b01a8a4ed..4ee3f8735 100644 --- a/tests/run/r_docstrings.pyx +++ b/tests/run/r_docstrings.pyx @@ -14,10 +14,21 @@ doctest = u"""# Python 3 gets all of these right ... >>> C.__doc__ '\\n This is a class docstring.\\n ' + >>> C.docstring_copy_C + '\\n This is a class docstring.\\n ' + >>> CS.docstring_copy_C + '\\n This is a class docstring.\\n ' + >>> CS.__doc__ '\\n This is a subclass docstring.\\n ' + >>> CS.docstring_copy_CS + '\\n This is a subclass docstring.\\n ' + >>> CSS.docstring_copy_CS + '\\n This is a subclass docstring.\\n ' >>> print(CSS.__doc__) None + >>> CSS.docstring_copy_CSS + 'A module docstring' >>> T.__doc__ '\\n This is an extension type docstring.\\n ' @@ -34,22 +45,38 @@ Compare with standard Python: >>> Pyf.__doc__ '\\n This is a function docstring.\\n ' - >>> class PyC: + >>> class PyC(object): ... ''' ... This is a class docstring. ... ''' - >>> class PyCS(C): + ... docstring_copy_C = __doc__ + >>> class PyCS(PyC): ... ''' ... This is a subclass docstring. ... ''' - >>> class PyCSS(CS): - ... pass + ... docstring_copy_CS = __doc__ + >>> class PyCSS(PyCS): + ... docstring_copy_CSS = __doc__ >>> PyC.__doc__ '\\n This is a class docstring.\\n ' + >>> PyC.docstring_copy_C + '\\n This is a class docstring.\\n ' + >>> PyCS.docstring_copy_C + '\\n This is a class docstring.\\n ' + >>> PyCSS.docstring_copy_C + '\\n This is a class docstring.\\n ' + >>> PyCS.__doc__ '\\n This is a subclass docstring.\\n ' + >>> PyCS.docstring_copy_CS + '\\n This is a subclass docstring.\\n ' + >>> PyCSS.docstring_copy_CS + '\\n This is a subclass docstring.\\n ' + >>> PyCSS.__doc__ + >>> PyCSS.docstring_copy_CSS + 'A module docstring' """ __test__ = {"test_docstrings" : doctest} @@ -59,18 +86,24 @@ def f(): This is a function docstring. """ -class C: + +class C(object): """ This is a class docstring. """ + docstring_copy_C = __doc__ + class CS(C): """ This is a subclass docstring. """ + docstring_copy_CS = __doc__ + class CSS(CS): - pass + docstring_copy_CSS = __doc__ + cdef class T: """ diff --git a/tests/run/raise_memory_error_T650.pyx b/tests/run/raise_memory_error_T650.pyx index 76b1ef4ab..8bced08e0 100644 --- a/tests/run/raise_memory_error_T650.pyx +++ b/tests/run/raise_memory_error_T650.pyx @@ -1,4 +1,4 @@ -# ticket: 650 +# ticket: t650 cimport cython diff --git a/tests/run/range_optimisation_T203.pyx b/tests/run/range_optimisation_T203.pyx index 880d8952d..bf8c0c978 100644 --- a/tests/run/range_optimisation_T203.pyx +++ b/tests/run/range_optimisation_T203.pyx @@ -1,4 +1,4 @@ -# ticket: 203 +# ticket: t203 cdef int get_bound(int m): print u"get_bound(%s)"%m diff --git a/tests/run/refcount_in_meth.pyx b/tests/run/refcount_in_meth.pyx index ed21b437b..af6212779 100644 --- a/tests/run/refcount_in_meth.pyx +++ b/tests/run/refcount_in_meth.pyx @@ -12,8 +12,10 @@ True True """ +cimport cython from cpython.ref cimport PyObject +@cython.always_allow_keywords(False) def get_refcount(obj): return (<PyObject*>obj).ob_refcnt diff --git a/tests/run/reimport_from_package.srctree b/tests/run/reimport_from_package.srctree index 027d26de5..91e4b9d0a 100644 --- a/tests/run/reimport_from_package.srctree +++ b/tests/run/reimport_from_package.srctree @@ -17,8 +17,8 @@ setup( import sys import a -assert a in sys.modules.values(), list(sys.modules) -assert sys.modules['a'] is a, list(sys.modules) +assert a in sys.modules.values(), sorted(sys.modules) +assert sys.modules['a'] is a, sorted(sys.modules) from atest.package import module @@ -33,8 +33,8 @@ assert 'atest.package.module' in sys.modules import a import atest.package.module as module -assert module in sys.modules.values(), list(sys.modules) -assert sys.modules['atest.package.module'] is module, list(sys.modules) +assert module in sys.modules.values(), sorted(sys.modules) +assert sys.modules['atest.package.module'] is module, sorted(sys.modules) if sys.version_info >= (3, 5): from . import pymodule diff --git a/tests/run/relative_cimport.srctree b/tests/run/relative_cimport.srctree index 0184fe464..2989f9ac5 100644 --- a/tests/run/relative_cimport.srctree +++ b/tests/run/relative_cimport.srctree @@ -3,6 +3,7 @@ PYTHON setup.py build_ext --inplace PYTHON -c "from pkg.b import test; assert test() == (1, 2)" +PYTHON -c "from pkg.b_py2 import test; assert test() == (1, 2)" PYTHON -c "from pkg.sub.c import test; assert test() == (1, 2)" ######## setup.py ######## @@ -42,7 +43,23 @@ cdef class test_pxd: from . cimport a from .a cimport test_pxd -cimport a as implicitly_relative_a + +assert a.test_pxd is test_pxd + +def test(): + cdef test_pxd obj = test_pxd() + obj.x = 1 + obj.y = 2 + return (obj.x, obj.y) + + +######## pkg/b_py2.pyx ######## + +# cython: language_level=2 + +from . cimport a +from .a cimport test_pxd +cimport a as implicitly_relative_a # <-- Py2 "feature" assert a.test_pxd is test_pxd assert implicitly_relative_a.test_pxd is test_pxd diff --git a/tests/run/relativeimport_T542.srctree b/tests/run/relativeimport_T542.srctree index 232c5f06f..27020174f 100644 --- a/tests/run/relativeimport_T542.srctree +++ b/tests/run/relativeimport_T542.srctree @@ -26,7 +26,8 @@ try: except ImportError: pass else: - assert False, "absolute import succeeded" + import sys + assert False, "absolute import succeeded: %s" % sorted(sys.modules) import relimport.a import relimport.bmod diff --git a/tests/run/reraise.py b/tests/run/reraise.py index de7aad7bd..43becf59d 100644 --- a/tests/run/reraise.py +++ b/tests/run/reraise.py @@ -31,6 +31,6 @@ def test_reraise_error(): ... else: print("FAILED") """ import sys - if hasattr(sys, 'exc_clear'): # Py2 + if hasattr(sys, 'exc_clear'): # Py2 sys.exc_clear() raise diff --git a/tests/run/self_in_ext_type_closure.pyx b/tests/run/self_in_ext_type_closure.pyx index efb116659..ae45cce76 100644 --- a/tests/run/self_in_ext_type_closure.pyx +++ b/tests/run/self_in_ext_type_closure.pyx @@ -1,5 +1,5 @@ # mode: run -# ticket: 742 +# ticket: t742 import cython diff --git a/tests/run/seq_mul.py b/tests/run/seq_mul.py new file mode 100644 index 000000000..2c4a41e65 --- /dev/null +++ b/tests/run/seq_mul.py @@ -0,0 +1,148 @@ +# mode: run +# tag: list, mulop, pure3.0 + +import cython + + +@cython.test_fail_if_path_exists("//MulNode") +@cython.test_assert_path_exists("//ListNode[@mult_factor]") +def cint_times_list(n: cython.int): + """ + >>> cint_times_list(3) + [] + [None, None, None] + [3, 3, 3] + [1, 2, 3, 1, 2, 3, 1, 2, 3] + """ + a = n * [] + b = n * [None] + c = n * [n] + d = n * [1, 2, 3] + + print(a) + print(b) + print(c) + print(d) + + +@cython.test_fail_if_path_exists("//MulNode") +@cython.test_assert_path_exists("//ListNode[@mult_factor]") +def list_times_cint(n: cython.int): + """ + >>> list_times_cint(3) + [] + [None, None, None] + [3, 3, 3] + [1, 2, 3, 1, 2, 3, 1, 2, 3] + """ + a = [] * n + b = [None] * n + c = [n] * n + d = [1, 2, 3] * n + + print(a) + print(b) + print(c) + print(d) + + +@cython.test_fail_if_path_exists("//MulNode") +@cython.test_assert_path_exists("//TupleNode[@mult_factor]") +def cint_times_tuple(n: cython.int): + """ + >>> cint_times_tuple(3) + () + (None, None, None) + (3, 3, 3) + (1, 2, 3, 1, 2, 3, 1, 2, 3) + """ + a = n * () + b = n * (None,) + c = n * (n,) + d = n * (1, 2, 3) + + print(a) + print(b) + print(c) + print(d) + + +@cython.test_fail_if_path_exists("//MulNode") +@cython.test_assert_path_exists("//TupleNode[@mult_factor]") +def tuple_times_cint(n: cython.int): + """ + >>> tuple_times_cint(3) + () + (None, None, None) + (3, 3, 3) + (1, 2, 3, 1, 2, 3, 1, 2, 3) + """ + a = () * n + b = (None,) * n + c = (n,) * n + d = (1, 2, 3) * n + + print(a) + print(b) + print(c) + print(d) + + +# TODO: enable in Cython 3.1 when we can infer unsafe C int operations as PyLong +#@cython.test_fail_if_path_exists("//MulNode") +def list_times_pyint(n: cython.longlong): + """ + >>> list_times_cint(3) + [] + [None, None, None] + [3, 3, 3] + [1, 2, 3, 1, 2, 3, 1, 2, 3] + """ + py_n = n + 1 # might overflow => should be inferred as Python long! + + a = [] * py_n + b = [None] * py_n + c = py_n * [n] + d = py_n * [1, 2, 3] + + print(a) + print(b) + print(c) + print(d) + + +@cython.cfunc +def sideeffect(x) -> cython.int: + global _sideeffect_value + _sideeffect_value += 1 + return _sideeffect_value + x + + +def reset_sideeffect(): + global _sideeffect_value + _sideeffect_value = 0 + + +@cython.test_fail_if_path_exists("//MulNode") +@cython.test_assert_path_exists("//ListNode[@mult_factor]") +def complicated_cint_times_list(n: cython.int): + """ + >>> complicated_cint_times_list(3) + [] + [None, None, None, None] + [3, 3, 3, 3] + [1, 2, 3, 1, 2, 3, 1, 2, 3, 1, 2, 3] + """ + reset_sideeffect() + a = [] * sideeffect((lambda: n)()) + reset_sideeffect() + b = sideeffect((lambda: n)()) * [None] + reset_sideeffect() + c = [n] * sideeffect((lambda: n)()) + reset_sideeffect() + d = sideeffect((lambda: n)()) * [1, 2, 3] + + print(a) + print(b) + print(c) + print(d) diff --git a/tests/run/sequential_parallel.pyx b/tests/run/sequential_parallel.pyx index b383b9cb5..3d8e1efff 100644 --- a/tests/run/sequential_parallel.pyx +++ b/tests/run/sequential_parallel.pyx @@ -656,7 +656,7 @@ def outer_parallel_section(): sum += inner_parallel_section() return sum -cdef int nogil_cdef_except_clause() nogil except 0: +cdef int nogil_cdef_except_clause() nogil except -1: return 1 cdef void nogil_cdef_except_star() nogil except *: diff --git a/tests/run/set_item.pyx b/tests/run/set_item.pyx new file mode 100644 index 000000000..4086351ca --- /dev/null +++ b/tests/run/set_item.pyx @@ -0,0 +1,75 @@ +# mode: run +# tag: list, dict, setitem, delitem + +def set_item(obj, key, value): + """ + >>> set_item([1, 2, 3], 1, -1) + [1, -1, 3] + >>> set_item([1, 2, 3], -1, -1) + [1, 2, -1] + >>> set_item({}, 'abc', 5) + {'abc': 5} + >>> set_item({}, -1, 5) + {-1: 5} + >>> class D(dict): pass + >>> set_item(D({}), 'abc', 5) + {'abc': 5} + >>> set_item(D({}), -1, 5) + {-1: 5} + """ + obj[key] = value + return obj + + +def set_item_int(obj, int key, value): + """ + >>> set_item_int([1, 2, 3], 1, -1) + [1, -1, 3] + >>> set_item_int([1, 2, 3], -1, -1) + [1, 2, -1] + >>> set_item_int({}, 1, 5) + {1: 5} + >>> set_item_int({}, -1, 5) + {-1: 5} + >>> class D(dict): pass + >>> set_item_int(D({}), 1, 5) + {1: 5} + >>> set_item_int(D({}), -1, 5) + {-1: 5} + """ + obj[key] = value + return obj + + +def del_item(obj, key): + """ + >>> del_item([1, 2, 3], 1) + [1, 3] + >>> del_item([1, 2, 3], -3) + [2, 3] + >>> class D(dict): pass + >>> del_item({'abc': 1, 'def': 2}, 'abc') + {'def': 2} + >>> del_item(D({'abc': 1, 'def': 2}), 'abc') + {'def': 2} + >>> del_item(D({-1: 1, -2: 2}), -1) + {-2: 2} + """ + del obj[key] + return obj + + +def del_item_int(obj, int key): + """ + >>> del_item_int([1, 2, 3], 1) + [1, 3] + >>> del_item_int([1, 2, 3], -3) + [2, 3] + >>> class D(dict): pass + >>> del_item_int(D({-1: 1, 1: 2}), 1) + {-1: 1} + >>> del_item_int(D({-1: 1, -2: 2}), -1) + {-2: 2} + """ + del obj[key] + return obj diff --git a/tests/run/set_new.py b/tests/run/set_new.py new file mode 100644 index 000000000..d1c2c4acb --- /dev/null +++ b/tests/run/set_new.py @@ -0,0 +1,21 @@ +""" +>>> X = make_class_with_new(cynew) +>>> X.__new__ is cynew +True +>>> X().__new__ is cynew +True +>>> def pynew(cls): return object.__new__(cls) +>>> X = make_class_with_new(pynew) +>>> X.__new__ is pynew +True +>>> X().__new__ is pynew +True +""" + +def make_class_with_new(n): + class X(object): + __new__ = n + return X + +def cynew(cls): + return object.__new__(cls) diff --git a/tests/run/short_circuit_T404.pyx b/tests/run/short_circuit_T404.pyx index 09affaf7a..dbb24820e 100644 --- a/tests/run/short_circuit_T404.pyx +++ b/tests/run/short_circuit_T404.pyx @@ -1,4 +1,4 @@ -# ticket: 404 +# ticket: t404 cdef long foo(long x): print "foo(%s)" % x diff --git a/tests/run/special_methods_T561.pyx b/tests/run/special_methods_T561.pyx index 350d452f4..fc381a243 100644 --- a/tests/run/special_methods_T561.pyx +++ b/tests/run/special_methods_T561.pyx @@ -1,6 +1,6 @@ # mode: run -# ticket: 561 -# ticket: 3 +# ticket: t561 +# ticket: t3 # The patch in #561 changes code generation for most special methods # to remove the Cython-generated wrapper and let PyType_Ready() @@ -56,30 +56,6 @@ __doc__ = u""" >>> g11 = object.__getattribute__(GetAttribute(), '__getattribute__') >>> g11('attr') GetAttribute getattribute 'attr' - >>> # If you define either setattr or delattr, you get wrapper objects - >>> # for both methods. (This behavior is unchanged by #561.) - >>> sa_setattr = SetAttr().__setattr__ - >>> sa_setattr('foo', 'bar') - SetAttr setattr 'foo' 'bar' - >>> sa_delattr = SetAttr().__delattr__ - >>> sa_delattr('foo') - Traceback (most recent call last): - ... - AttributeError: 'special_methods_T561.SetAttr' object has no attribute 'foo' - >>> da_setattr = DelAttr().__setattr__ - >>> da_setattr('foo', 'bar') - Traceback (most recent call last): - ... - AttributeError: 'special_methods_T561.DelAttr' object has no attribute 'foo' - >>> da_delattr = DelAttr().__delattr__ - >>> da_delattr('foo') - DelAttr delattr 'foo' - >>> sda_setattr = SetDelAttr().__setattr__ - >>> sda_setattr('foo', 'bar') - SetDelAttr setattr 'foo' 'bar' - >>> sda_delattr = SetDelAttr().__delattr__ - >>> sda_delattr('foo') - SetDelAttr delattr 'foo' >>> # If you define either set or delete, you get wrapper objects >>> # for both methods. (This behavior is unchanged by #561.) >>> s_set = Set().__set__ @@ -119,6 +95,39 @@ if sys.version_info >= (2,5): VS __index__ 0 """ +cdef extern from *: + # type specs require a bug fix in Py3.8+ for some of these tests. + const int CYTHON_USE_TYPE_SPECS + +if not CYTHON_USE_TYPE_SPECS or sys.version_info >= (3,8): + __doc__ += u""" + >>> # If you define either setattr or delattr, you get wrapper objects + >>> # for both methods. (This behavior is unchanged by #561.) + >>> sa_setattr = SetAttr().__setattr__ + >>> sa_setattr('foo', 'bar') + SetAttr setattr 'foo' 'bar' + >>> sa_delattr = SetAttr().__delattr__ + >>> sa_delattr('foo') + Traceback (most recent call last): + ... + AttributeError: 'special_methods_T561.SetAttr' object has no attribute 'foo' + >>> da_setattr = DelAttr().__setattr__ + >>> da_setattr('foo', 'bar') + Traceback (most recent call last): + ... + AttributeError: 'special_methods_T561.DelAttr' object has no attribute 'foo' + >>> da_delattr = DelAttr().__delattr__ + >>> da_delattr('foo') + DelAttr delattr 'foo' + >>> sda_setattr = SetDelAttr().__setattr__ + >>> sda_setattr('foo', 'bar') + SetDelAttr setattr 'foo' 'bar' + >>> sda_delattr = SetDelAttr().__delattr__ + >>> sda_delattr('foo') + SetDelAttr delattr 'foo' +""" + + cdef class VerySpecial: """ >>> vs0 = VerySpecial(0) diff --git a/tests/run/special_methods_T561_py2.pyx b/tests/run/special_methods_T561_py2.pyx index 285f820f6..9781376b3 100644 --- a/tests/run/special_methods_T561_py2.pyx +++ b/tests/run/special_methods_T561_py2.pyx @@ -1,4 +1,4 @@ -# ticket: 561 +# ticket: t561 # tag: py2 # This file tests the behavior of special methods under Python 2 # after #561. (Only methods whose behavior differs between Python 2 and 3 diff --git a/tests/run/special_methods_T561_py3.pyx b/tests/run/special_methods_T561_py3.pyx index 932002254..0a5f53547 100644 --- a/tests/run/special_methods_T561_py3.pyx +++ b/tests/run/special_methods_T561_py3.pyx @@ -1,4 +1,4 @@ -# ticket: 561 +# ticket: t561 # tag: py3 # This file tests the behavior of special methods under Python 3 # after #561. (Only methods whose behavior differs between Python 2 and 3 diff --git a/tests/run/ssize_t_T399.pyx b/tests/run/ssize_t_T399.pyx index 4a8f65d68..265a3c924 100644 --- a/tests/run/ssize_t_T399.pyx +++ b/tests/run/ssize_t_T399.pyx @@ -1,4 +1,4 @@ -# ticket: 399 +# ticket: t399 __doc__ = u""" >>> test(-2) diff --git a/tests/run/starargs.pyx b/tests/run/starargs.pyx index 5e17faa47..7da9c6442 100644 --- a/tests/run/starargs.pyx +++ b/tests/run/starargs.pyx @@ -1,7 +1,6 @@ cdef sorteditems(d): - l = list(d.items()) - l.sort() - return tuple(l) + return tuple(sorted(d.items())) + def spam(x, y, z): """ @@ -79,6 +78,8 @@ def onlyt(*a): >>> onlyt(1, a=2) Traceback (most recent call last): TypeError: onlyt() got an unexpected keyword argument 'a' + >>> test_no_copy_args(onlyt) + True """ return a @@ -114,3 +115,20 @@ def tk(*a, **k): (1, ('a', 1), ('b', 2)) """ return a + sorteditems(k) + +def t_kwonly(*a, k): + """ + >>> test_no_copy_args(t_kwonly, k=None) + True + """ + return a + + +def test_no_copy_args(func, **kw): + """ + func is a function such that func(*args, **kw) returns args. + We test that no copy is made of the args tuple. + This tests both the caller side and the callee side. + """ + args = (1, 2, 3) + return func(*args, **kw) is args diff --git a/tests/run/starred_target_T664.pyx b/tests/run/starred_target_T664.pyx index 0d291c648..5a88d5ab2 100644 --- a/tests/run/starred_target_T664.pyx +++ b/tests/run/starred_target_T664.pyx @@ -1,4 +1,4 @@ -# ticket: 664 +# ticket: t664 def assign(): """ diff --git a/tests/run/static_methods.pxd b/tests/run/static_methods.pxd index 4dc199955..383808149 100644 --- a/tests/run/static_methods.pxd +++ b/tests/run/static_methods.pxd @@ -1,3 +1,6 @@ cdef class FromPxd: @staticmethod cdef static_cdef(int* x) + + @staticmethod + cdef static_cdef_with_implicit_object(obj) diff --git a/tests/run/static_methods.pyx b/tests/run/static_methods.pyx index ba7df379d..6c485fac7 100644 --- a/tests/run/static_methods.pyx +++ b/tests/run/static_methods.pyx @@ -87,6 +87,10 @@ cdef class FromPxd: cdef static_cdef(int* x): return 'pxd_cdef', x[0] + @staticmethod + cdef static_cdef_with_implicit_object(obj): + return obj+1 + def call_static_pxd_cdef(int x): """ >>> call_static_pxd_cdef(2) @@ -94,3 +98,11 @@ def call_static_pxd_cdef(int x): """ cdef int *x_ptr = &x return FromPxd.static_cdef(x_ptr) + +def call_static_pxd_cdef_with_implicit_object(int x): + """ + # https://github.com/cython/cython/issues/3174 + >>> call_static_pxd_cdef_with_implicit_object(2) + 3 + """ + return FromPxd.static_cdef_with_implicit_object(x) diff --git a/tests/run/staticmethod.pyx b/tests/run/staticmethod.pyx index ceaf616fb..5ede6a642 100644 --- a/tests/run/staticmethod.pyx +++ b/tests/run/staticmethod.pyx @@ -1,35 +1,17 @@ -__doc__ = u""" ->>> class1.plus1(1) -2 ->>> class2.plus1(1) -2 ->>> class3.plus1(1) -2 ->>> class4.plus1(1) -2 ->>> class4().plus1(1) -2 ->>> class4.bplus1(1) -2 ->>> class4().bplus1(1) -2 -""" - cimport cython -def f_plus(a): - return a + 1 class class1: - plus1 = f_plus - -class class2(object): - plus1 = f_plus - -cdef class class3: - plus1 = f_plus - -class class4: + u""" + >>> class1.plus1(1) + 2 + >>> class1().plus1(1) + 2 + >>> class1.bplus1(1) + 2 + >>> class1().bplus1(1) + 2 + """ @staticmethod def plus1(a): return a + 1 @@ -49,14 +31,14 @@ def nested_class(): >>> obj.plus1(1) 2 """ - class class5(object): + class class2(object): def __new__(cls): # implicit staticmethod return object.__new__(cls) @staticmethod def plus1(a): return a + 1 - return class5 + return class2 cdef class BaseClass(object): diff --git a/tests/run/str_char_coercion_T412.pyx b/tests/run/str_char_coercion_T412.pyx index 6a470e2b7..c1c0ba709 100644 --- a/tests/run/str_char_coercion_T412.pyx +++ b/tests/run/str_char_coercion_T412.pyx @@ -1,4 +1,4 @@ -# ticket: 412 +# ticket: t412 cdef int i = 'x' cdef char c = 'x' diff --git a/tests/run/str_subclass_kwargs.pyx b/tests/run/str_subclass_kwargs.pyx new file mode 100644 index 000000000..94c6712f4 --- /dev/null +++ b/tests/run/str_subclass_kwargs.pyx @@ -0,0 +1,21 @@ +def test_str_subclass_kwargs(k=None): + """ + Test passing keywords with names that are not of type ``str`` + but a subclass: + + >>> class StrSubclass(str): + ... pass + >>> class StrNoCompare(str): + ... def __eq__(self, other): + ... raise RuntimeError("do not compare me") + ... def __hash__(self): + ... return hash(str(self)) + >>> kwargs = {StrSubclass('k'): 'value'} + >>> test_str_subclass_kwargs(**kwargs) + 'value' + >>> kwargs = {StrNoCompare('k'): 'value'} + >>> test_str_subclass_kwargs(**kwargs) + Traceback (most recent call last): + RuntimeError: do not compare me + """ + return k diff --git a/tests/run/strfunction.pyx b/tests/run/strfunction.pyx index dc6adaf79..d4a1c95d4 100644 --- a/tests/run/strfunction.pyx +++ b/tests/run/strfunction.pyx @@ -5,6 +5,8 @@ __doc__ = u""" 'test' """ +cimport cython + s = str z = str('test') @@ -39,3 +41,33 @@ def sub(string): #def csub(string): # return csubs(string) + + +@cython.test_fail_if_path_exists("//SimpleCallNode") +@cython.test_assert_path_exists("//PythonCapiCallNode") +def typed(str s): + """ + >>> print(typed(None)) + None + >>> type(typed(None)) is type(typed(None)) + True + >>> print(typed('abc')) + abc + >>> type(typed('abc')) is type(typed('abc')) + True + """ + return str(s) + + +@cython.test_fail_if_path_exists( + "//SimpleCallNode", + "//PythonCapiCallNode", +) +def typed_not_none(str s not None): + """ + >>> print(typed('abc')) + abc + >>> type(typed('abc')) is type(typed('abc')) + True + """ + return str(s) diff --git a/tests/run/struct_conversion.pyx b/tests/run/struct_conversion.pyx index 26bd62686..8ef652a5d 100644 --- a/tests/run/struct_conversion.pyx +++ b/tests/run/struct_conversion.pyx @@ -28,10 +28,9 @@ def test_constructor_kwds(x, y, color): """ >>> sorted(test_constructor_kwds(1.25, 2.5, 128).items()) [('color', 128), ('x', 1.25), ('y', 2.5)] - >>> test_constructor_kwds(1.25, 2.5, None) + >>> test_constructor_kwds(1.25, 2.5, None) # doctest: +ELLIPSIS Traceback (most recent call last): - ... - TypeError: an integer is required + TypeError:... int... """ cdef Point p = Point(x=x, y=y, color=color) return p @@ -41,10 +40,9 @@ def return_constructor_kwds(double x, y, color): """ >>> sorted(return_constructor_kwds(1.25, 2.5, 128).items()) [('color', 128), ('x', 1.25), ('y', 2.5)] - >>> return_constructor_kwds(1.25, 2.5, None) + >>> return_constructor_kwds(1.25, 2.5, None) # doctest: +ELLIPSIS Traceback (most recent call last): - ... - TypeError: an integer is required + TypeError:... int... """ return Point(x=x, y=y, color=color) @@ -169,3 +167,19 @@ def test_nested_obj_to_struct(NestedStruct nested): nested.mystruct.s.decode('UTF-8'), nested.d) +cdef struct OverriddenCname: + int x "not_x" + +def test_obj_to_struct_cnames(OverriddenCname s): + """ + >>> test_obj_to_struct_cnames({ 'x': 1 }) + 1 + """ + print(s.x) + +def test_struct_to_obj_cnames(): + """ + >>> test_struct_to_obj_cnames() + {'x': 2} + """ + return OverriddenCname(2) diff --git a/tests/run/subop.pyx b/tests/run/subop.pyx index 027ec07bb..fbb92331f 100644 --- a/tests/run/subop.pyx +++ b/tests/run/subop.pyx @@ -191,3 +191,27 @@ def sub_large_x(x): ... except TypeError: pass """ return 2**30 - x + + +def sub0(x): + """ + >>> sub0(0) + (0, 0) + >>> sub0(1) + (1, -1) + >>> sub0(-1) + (-1, 1) + >>> sub0(99) + (99, -99) + >>> a, b = sub0(2**32) + >>> bigint(a) + 4294967296 + >>> bigint(b) + -4294967296 + >>> a, b = sub0(-2**32) + >>> bigint(a) + -4294967296 + >>> bigint(b) + 4294967296 + """ + return x - 0, 0 - x diff --git a/tests/run/temp_alloc_T409.pyx b/tests/run/temp_alloc_T409.pyx index 383e1ef6f..425b7064b 100644 --- a/tests/run/temp_alloc_T409.pyx +++ b/tests/run/temp_alloc_T409.pyx @@ -1,4 +1,4 @@ -# ticket: 409 +# ticket: t409 # Extracted from sage/plot/plot3d/index_face_set.pyx:502 # Turns out to be a bug in implementation of PEP 3132 (Extended Iterable Unpacking) diff --git a/tests/run/temp_sideeffects_T654.pyx b/tests/run/temp_sideeffects_T654.pyx index 8a00f7e5b..fe8f03fee 100644 --- a/tests/run/temp_sideeffects_T654.pyx +++ b/tests/run/temp_sideeffects_T654.pyx @@ -1,4 +1,4 @@ -# ticket: 654 +# ticket: t654 # function call arguments diff --git a/tests/run/test_asyncgen.py b/tests/run/test_asyncgen.py index ec1599434..410d4fe0b 100644 --- a/tests/run/test_asyncgen.py +++ b/tests/run/test_asyncgen.py @@ -247,16 +247,6 @@ class AsyncGenTest(unittest.TestCase): else: self.assertTrue(False) - if sys.version_info < (2, 7): - def assertIn(self, x, container): - self.assertTrue(x in container) - - def assertIs(self, x, y): - self.assertTrue(x is y) - - assertRaises = assertRaisesRegex - - def compare_generators(self, sync_gen, async_gen): def sync_iterate(g): res = [] @@ -273,19 +263,26 @@ class AsyncGenTest(unittest.TestCase): def async_iterate(g): res = [] while True: + an = g.__anext__() try: - next(g.__anext__()) + while True: + try: + next(an) + except StopIteration as ex: + if ex.args: + res.append(ex.args[0]) + break + else: + res.append('EMPTY StopIteration') + break + except StopAsyncIteration: + raise + except Exception as ex: + res.append(str(type(ex))) + break except StopAsyncIteration: res.append('STOP') break - except StopIteration as ex: - if ex.args: - res.append(ex.args[0]) - else: - res.append('EMPTY StopIteration') - break - except Exception as ex: - res.append(str(type(ex))) return res sync_gen_result = sync_iterate(sync_gen) @@ -313,19 +310,22 @@ class AsyncGenTest(unittest.TestCase): g = gen() ai = g.__aiter__() - self.assertEqual(next(ai.__anext__()), ('result',)) + + an = ai.__anext__() + self.assertEqual(next(an), ('result',)) try: - next(ai.__anext__()) + next(an) except StopIteration as ex: self.assertEqual(ex.args[0], 123) else: self.fail('StopIteration was not raised') - self.assertEqual(next(ai.__anext__()), ('result',)) + an = ai.__anext__() + self.assertEqual(next(an), ('result',)) try: - next(ai.__anext__()) + next(an) except StopAsyncIteration as ex: self.assertFalse(ex.args) else: @@ -349,10 +349,12 @@ class AsyncGenTest(unittest.TestCase): g = gen() ai = g.__aiter__() - self.assertEqual(next(ai.__anext__()), ('result',)) + + an = ai.__anext__() + self.assertEqual(next(an), ('result',)) try: - next(ai.__anext__()) + next(an) except StopIteration as ex: self.assertEqual(ex.args[0], 123) else: @@ -459,6 +461,37 @@ class AsyncGenTest(unittest.TestCase): "non-None value .* async generator"): gen().__anext__().send(100) + def test_async_gen_exception_11(self): + def sync_gen(): + yield 10 + yield 20 + + def sync_gen_wrapper(): + yield 1 + sg = sync_gen() + sg.send(None) + try: + sg.throw(GeneratorExit()) + except GeneratorExit: + yield 2 + yield 3 + + async def async_gen(): + yield 10 + yield 20 + + async def async_gen_wrapper(): + yield 1 + asg = async_gen() + await asg.asend(None) + try: + await asg.athrow(GeneratorExit()) + except GeneratorExit: + yield 2 + yield 3 + + self.compare_generators(sync_gen_wrapper(), async_gen_wrapper()) + def test_async_gen_api_01(self): async def gen(): yield 123 @@ -752,17 +785,13 @@ class AsyncGenAsyncioTest(unittest.TestCase): gen = foo() it = gen.__aiter__() self.assertEqual(await it.__anext__(), 1) - t = self.loop.create_task(it.__anext__()) - await asyncio.sleep(0.01) await gen.aclose() - return t - t = self.loop.run_until_complete(run()) + self.loop.run_until_complete(run()) self.assertEqual(DONE, 1) # Silence ResourceWarnings fut.cancel() - t.cancel() self.loop.run_until_complete(asyncio.sleep(0.01)) @needs_py36_asyncio @@ -785,7 +814,7 @@ class AsyncGenAsyncioTest(unittest.TestCase): await g.__anext__() del g - await asyncio.sleep(0.1) + await asyncio.sleep(0.2) self.loop.run_until_complete(run()) self.assertEqual(DONE, 1) @@ -860,6 +889,33 @@ class AsyncGenAsyncioTest(unittest.TestCase): self.loop.run_until_complete(run()) self.assertEqual(DONE, 10) + def test_async_gen_asyncio_aclose_12(self): + DONE = 0 + + async def target(): + await asyncio.sleep(0.01) + 1 / ZERO + + async def foo(): + nonlocal DONE + task = self.loop.create_task(target()) + try: + yield 1 + finally: + try: + await task + except ZeroDivisionError: + DONE = 1 + + async def run(): + gen = foo() + it = gen.__aiter__() + await it.__anext__() + await gen.aclose() + + self.loop.run_until_complete(run()) + self.assertEqual(DONE, 1) + def test_async_gen_asyncio_asend_01(self): DONE = 0 @@ -1162,47 +1218,156 @@ class AsyncGenAsyncioTest(unittest.TestCase): self.loop.run_until_complete(asyncio.sleep(0.1)) - self.loop.run_until_complete(self.loop.shutdown_asyncgens()) - self.assertEqual(finalized, 2) # Silence warnings t1.cancel() t2.cancel() - self.loop.run_until_complete(asyncio.sleep(0.1)) - @needs_py36_asyncio - def test_async_gen_asyncio_shutdown_02(self): - logged = 0 + with self.assertRaises(asyncio.CancelledError): + self.loop.run_until_complete(t1) + with self.assertRaises(asyncio.CancelledError): + self.loop.run_until_complete(t2) - def logger(loop, context): - nonlocal logged - self.assertIn('asyncgen', context) - expected = 'an error occurred during closing of asynchronous' - if expected in context['message']: - logged += 1 + self.loop.run_until_complete(self.loop.shutdown_asyncgens()) - async def waiter(timeout): - try: - await asyncio.sleep(timeout) - yield 1 - finally: - 1 / ZERO + self.assertEqual(finalized, 2) - async def wait(): - async for _ in waiter(1): + """ + def test_async_gen_expression_01(self): + async def arange(n): + for i in range(n): + await asyncio.sleep(0.01) + yield i + + def make_arange(n): + # This syntax is legal starting with Python 3.7 + return (i * 2 async for i in arange(n)) + + async def run(): + return [i async for i in make_arange(10)] + + res = self.loop.run_until_complete(run()) + self.assertEqual(res, [i * 2 for i in range(10)]) + + def test_async_gen_expression_02(self): + async def wrap(n): + await asyncio.sleep(0.01) + return n + + def make_arange(n): + # This syntax is legal starting with Python 3.7 + return (i * 2 for i in range(n) if await wrap(i)) + + async def run(): + return [i async for i in make_arange(10)] + + res = self.loop.run_until_complete(run()) + self.assertEqual(res, [i * 2 for i in range(1, 10)]) + """ + + def test_asyncgen_nonstarted_hooks_are_cancellable(self): + # See https://bugs.python.org/issue38013 + messages = [] + + def exception_handler(loop, context): + messages.append(context) + + async def async_iterate(): + yield 1 + yield 2 + + async def main(): + # loop = asyncio.get_running_loop() + loop = self.loop + loop.set_exception_handler(exception_handler) + + async for i in async_iterate(): + break + + # asyncio.run(main()) + self.loop.run_until_complete(main()) + + self.assertEqual([], messages) + + def test_async_gen_await_same_anext_coro_twice(self): + async def async_iterate(): + yield 1 + yield 2 + + async def run(): + it = async_iterate() + nxt = it.__anext__() + await nxt + with self.assertRaisesRegex( + RuntimeError, + r"cannot reuse already awaited __anext__\(\)/asend\(\)" + ): + await nxt + + await it.aclose() # prevent unfinished iterator warning + + self.loop.run_until_complete(run()) + + def test_async_gen_await_same_aclose_coro_twice(self): + async def async_iterate(): + yield 1 + yield 2 + + async def run(): + it = async_iterate() + nxt = it.aclose() + await nxt + with self.assertRaisesRegex( + RuntimeError, + r"cannot reuse already awaited aclose\(\)/athrow\(\)" + ): + await nxt + + self.loop.run_until_complete(run()) + + def test_async_gen_aclose_twice_with_different_coros(self): + # Regression test for https://bugs.python.org/issue39606 + async def async_iterate(): + yield 1 + yield 2 + + async def run(): + it = async_iterate() + await it.aclose() + await it.aclose() + + self.loop.run_until_complete(run()) + + def test_async_gen_aclose_after_exhaustion(self): + # Regression test for https://bugs.python.org/issue39606 + async def async_iterate(): + yield 1 + yield 2 + + async def run(): + it = async_iterate() + async for _ in it: pass + await it.aclose() - t = self.loop.create_task(wait()) - self.loop.run_until_complete(asyncio.sleep(0.1)) + self.loop.run_until_complete(run()) - self.loop.set_exception_handler(logger) - self.loop.run_until_complete(self.loop.shutdown_asyncgens()) + """ + def test_async_gen_aclose_compatible_with_get_stack(self): + async def async_generator(): + yield object() - self.assertEqual(logged, 1) + async def run(): + ag = async_generator() + self.loop.create_task(ag.aclose()) + tasks = asyncio.all_tasks() + for task in tasks: + # No AttributeError raised + task.get_stack() + + self.loop.run_until_complete(run()) + """ - # Silence warnings - t.cancel() - self.loop.run_until_complete(asyncio.sleep(0.1)) if __name__ == "__main__": unittest.main() diff --git a/tests/run/test_coroutines_pep492.pyx b/tests/run/test_coroutines_pep492.pyx index a010b701f..2841d97af 100644 --- a/tests/run/test_coroutines_pep492.pyx +++ b/tests/run/test_coroutines_pep492.pyx @@ -70,6 +70,12 @@ except ImportError: return (<PyObject*>obj).ob_refcnt +def no_pypy(f): + import platform + if platform.python_implementation() == 'PyPy': + return unittest.skip("excluded in PyPy") + + # compiled exec() def exec(code_string, l, g): from Cython.Shadow import inline @@ -109,7 +115,7 @@ class AsyncYield(object): def run_async(coro): #assert coro.__class__ is types.GeneratorType - assert coro.__class__.__name__ in ('coroutine', '_GeneratorWrapper'), coro.__class__.__name__ + assert coro.__class__.__name__.rsplit('.', 1)[-1] in ('coroutine', '_GeneratorWrapper'), coro.__class__.__name__ buffer = [] result = None @@ -123,7 +129,7 @@ def run_async(coro): def run_async__await__(coro): - assert coro.__class__.__name__ in ('coroutine', '_GeneratorWrapper'), coro.__class__.__name__ + assert coro.__class__.__name__.rsplit('.', 1)[-1] in ('coroutine', '_GeneratorWrapper'), coro.__class__.__name__ aw = coro.__await__() buffer = [] result = None @@ -149,17 +155,6 @@ def silence_coro_gc(): gc.collect() -def min_py27(method): - return None if sys.version_info < (2, 7) else method - - -def ignore_py26(manager): - @contextlib.contextmanager - def dummy(): - yield - return dummy() if sys.version_info < (2, 7) else manager - - @contextlib.contextmanager def captured_stderr(): try: @@ -213,9 +208,10 @@ class AsyncBadSyntaxTest(unittest.TestCase): pass """, - """async def foo(a:await something()): - pass - """, + #"""async def foo(a:await something()): + # pass + #""", # No longer an error with pep-563 (although still nonsense) + # Some other similar tests have also been commented out """async def foo(): def bar(): @@ -413,9 +409,9 @@ class AsyncBadSyntaxTest(unittest.TestCase): pass """, - """async def foo(a:await b): - pass - """, + #"""async def foo(a:await b): + # pass + #""", """def baz(): async def foo(a=await b): @@ -628,9 +624,9 @@ class AsyncBadSyntaxTest(unittest.TestCase): pass """, - """async def foo(a:await b): - pass - """, + #"""async def foo(a:await b): + # pass + #""", """def baz(): async def foo(a=await b): @@ -905,7 +901,7 @@ class CoroutineTest(unittest.TestCase): raise StopIteration with silence_coro_gc(): - self.assertRegex(repr(foo()), '^<coroutine object.* at 0x.*>$') + self.assertRegex(repr(foo()), '^<[^\s]*coroutine object.* at 0x.*>$') def test_func_4(self): async def foo(): @@ -1091,7 +1087,7 @@ class CoroutineTest(unittest.TestCase): c.close() def test_func_15(self): - # See http://bugs.python.org/issue25887 for details + # See https://bugs.python.org/issue25887 for details async def spammer(): return 'spam' @@ -1108,7 +1104,7 @@ class CoroutineTest(unittest.TestCase): reader(spammer_coro).send(None) def test_func_16(self): - # See http://bugs.python.org/issue25887 for details + # See https://bugs.python.org/issue25887 for details @types_coroutine def nop(): @@ -1139,7 +1135,7 @@ class CoroutineTest(unittest.TestCase): reader.throw(Exception('wat')) def test_func_17(self): - # See http://bugs.python.org/issue25887 for details + # See https://bugs.python.org/issue25887 for details async def coroutine(): return 'spam' @@ -1162,7 +1158,7 @@ class CoroutineTest(unittest.TestCase): coro.close() def test_func_18(self): - # See http://bugs.python.org/issue25887 for details + # See https://bugs.python.org/issue25887 for details async def coroutine(): return 'spam' @@ -1831,7 +1827,7 @@ class CoroutineTest(unittest.TestCase): buffer = [] async def test1(): - with ignore_py26(self.assertWarnsRegex(DeprecationWarning, "legacy")): + with self.assertWarnsRegex(DeprecationWarning, "legacy"): async for i1, i2 in AsyncIter(): buffer.append(i1 + i2) @@ -1845,7 +1841,7 @@ class CoroutineTest(unittest.TestCase): buffer = [] async def test2(): nonlocal buffer - with ignore_py26(self.assertWarnsRegex(DeprecationWarning, "legacy")): + with self.assertWarnsRegex(DeprecationWarning, "legacy"): async for i in AsyncIter(): buffer.append(i[0]) if i[0] == 20: @@ -1864,7 +1860,7 @@ class CoroutineTest(unittest.TestCase): buffer = [] async def test3(): nonlocal buffer - with ignore_py26(self.assertWarnsRegex(DeprecationWarning, "legacy")): + with self.assertWarnsRegex(DeprecationWarning, "legacy"): async for i in AsyncIter(): if i[0] > 20: continue @@ -2081,7 +2077,6 @@ class CoroutineTest(unittest.TestCase): self.assertEqual(CNT, 0) # old-style pre-Py3.5.2 protocol - no longer supported - @min_py27 def __test_for_9(self): # Test that DeprecationWarning can safely be converted into # an exception (__aiter__ should not have a chance to raise @@ -2099,7 +2094,6 @@ class CoroutineTest(unittest.TestCase): run_async(foo()) # old-style pre-Py3.5.2 protocol - no longer supported - @min_py27 def __test_for_10(self): # Test that DeprecationWarning can safely be converted into # an exception. @@ -2418,6 +2412,7 @@ class CoroutineTest(unittest.TestCase): finally: aw.close() + @no_pypy def test_fatal_coro_warning(self): # Issue 27811 async def func(): pass diff --git a/tests/run/test_exceptions.pyx b/tests/run/test_exceptions.pyx index d47c9c4db..15585a6b3 100644 --- a/tests/run/test_exceptions.pyx +++ b/tests/run/test_exceptions.pyx @@ -26,7 +26,7 @@ except ImportError: no_tracing = unittest.skip("For nested functions, Cython generates a C call without recursion checks.") -cpython_only = unittest.skip("Tests for _testcapi make no sense here.") +cpython_only = unittest.skip("Tests for _testcapi or Python error messages make no sense here.") class NaiveException(Exception): @@ -143,6 +143,7 @@ class ExceptionTests(unittest.TestCase): self.raise_catch(StopAsyncIteration, "StopAsyncIteration") + @cpython_only def testSyntaxErrorMessage(self): # make sure the right exception message is raised for each of # these code fragments @@ -165,6 +166,7 @@ class ExceptionTests(unittest.TestCase): ckmsg(s, "'continue' not properly in loop") ckmsg("continue\n", "'continue' not properly in loop") + @cpython_only def testSyntaxErrorMissingParens(self): def ckmsg(src, msg, exception=SyntaxError): try: @@ -193,6 +195,7 @@ class ExceptionTests(unittest.TestCase): s = '''if True:\n print()\n\texec "mixed tabs and spaces"''' ckmsg(s, "inconsistent use of tabs and spaces in indentation", TabError) + @cpython_only def testSyntaxErrorOffset(self): def check(src, lineno, offset): with self.assertRaises(SyntaxError) as cm: diff --git a/tests/run/test_fstring.pyx b/tests/run/test_fstring.pyx index 309696c28..d8db6e888 100644 --- a/tests/run/test_fstring.pyx +++ b/tests/run/test_fstring.pyx @@ -3,14 +3,17 @@ # tag: allow_unknown_names, f_strings, pep498 import ast +import os import types import decimal import unittest -import contextlib import sys IS_PY2 = sys.version_info[0] < 3 -IS_PY26 = sys.version_info[:2] < (2, 7) +if IS_PY2: + # Define `ascii` as `repr` for Python2 as it functions + # the same way in that version. + ascii = repr from Cython.Build.Inline import cython_inline from Cython.TestUtils import CythonTest @@ -56,32 +59,14 @@ class TestCase(CythonTest): def assertEqual(self, first, second, msg=None): # strip u'' string prefixes in Py2 if first != second and isinstance(first, unicode): - stripped_first = first.replace("u'", "'").replace('u"', '"') + stripped_first = first.replace(u"u'", u"'").replace(u'u"', u'"') if stripped_first == second: - first = stripped_first - elif stripped_first.decode('unicode_escape') == second: - first = stripped_first.decode('unicode_escape') + first = second + elif u'\\' in stripped_first and stripped_first.encode('utf8').decode('unicode_escape') == second: + first = second super(TestCase, self).assertEqual(first, second, msg) - if IS_PY26: - @contextlib.contextmanager - def assertRaises(self, exc): - try: - yield - except exc: - pass - else: - assert False, "exception '%s' not raised" % exc - - def assertIn(self, value, collection): - self.assertTrue(value in collection) - def test__format__lookup(self): - if IS_PY26: - return - elif IS_PY2: - raise unittest.SkipTest("Py3-only") - # Make sure __format__ is looked up on the type, not the instance. class X: def __format__(self, spec): @@ -109,7 +94,7 @@ class TestCase(CythonTest): self.assertEqual(type(y).__format__(y, ''), 'class') def __test_ast(self): - # Inspired by http://bugs.python.org/issue24975 + # Inspired by https://bugs.python.org/issue24975 class X: def __init__(self): self.called = False @@ -132,14 +117,263 @@ f'{a * x()}'""" # Make sure x was called. self.assertTrue(x.called) + def __test_ast_line_numbers(self): + expr = """ +a = 10 +f'{a * x()}'""" + t = ast.parse(expr) + self.assertEqual(type(t), ast.Module) + self.assertEqual(len(t.body), 2) + # check `a = 10` + self.assertEqual(type(t.body[0]), ast.Assign) + self.assertEqual(t.body[0].lineno, 2) + # check `f'...'` + self.assertEqual(type(t.body[1]), ast.Expr) + self.assertEqual(type(t.body[1].value), ast.JoinedStr) + self.assertEqual(len(t.body[1].value.values), 1) + self.assertEqual(type(t.body[1].value.values[0]), ast.FormattedValue) + self.assertEqual(t.body[1].lineno, 3) + self.assertEqual(t.body[1].value.lineno, 3) + self.assertEqual(t.body[1].value.values[0].lineno, 3) + # check the binop location + binop = t.body[1].value.values[0].value + self.assertEqual(type(binop), ast.BinOp) + self.assertEqual(type(binop.left), ast.Name) + self.assertEqual(type(binop.op), ast.Mult) + self.assertEqual(type(binop.right), ast.Call) + self.assertEqual(binop.lineno, 3) + self.assertEqual(binop.left.lineno, 3) + self.assertEqual(binop.right.lineno, 3) + self.assertEqual(binop.col_offset, 3) + self.assertEqual(binop.left.col_offset, 3) + self.assertEqual(binop.right.col_offset, 7) + + def __test_ast_line_numbers_multiple_formattedvalues(self): + expr = """ +f'no formatted values' +f'eggs {a * x()} spam {b + y()}'""" + t = ast.parse(expr) + self.assertEqual(type(t), ast.Module) + self.assertEqual(len(t.body), 2) + # check `f'no formatted value'` + self.assertEqual(type(t.body[0]), ast.Expr) + self.assertEqual(type(t.body[0].value), ast.JoinedStr) + self.assertEqual(t.body[0].lineno, 2) + # check `f'...'` + self.assertEqual(type(t.body[1]), ast.Expr) + self.assertEqual(type(t.body[1].value), ast.JoinedStr) + self.assertEqual(len(t.body[1].value.values), 4) + self.assertEqual(type(t.body[1].value.values[0]), ast.Constant) + self.assertEqual(type(t.body[1].value.values[0].value), str) + self.assertEqual(type(t.body[1].value.values[1]), ast.FormattedValue) + self.assertEqual(type(t.body[1].value.values[2]), ast.Constant) + self.assertEqual(type(t.body[1].value.values[2].value), str) + self.assertEqual(type(t.body[1].value.values[3]), ast.FormattedValue) + self.assertEqual(t.body[1].lineno, 3) + self.assertEqual(t.body[1].value.lineno, 3) + self.assertEqual(t.body[1].value.values[0].lineno, 3) + self.assertEqual(t.body[1].value.values[1].lineno, 3) + self.assertEqual(t.body[1].value.values[2].lineno, 3) + self.assertEqual(t.body[1].value.values[3].lineno, 3) + # check the first binop location + binop1 = t.body[1].value.values[1].value + self.assertEqual(type(binop1), ast.BinOp) + self.assertEqual(type(binop1.left), ast.Name) + self.assertEqual(type(binop1.op), ast.Mult) + self.assertEqual(type(binop1.right), ast.Call) + self.assertEqual(binop1.lineno, 3) + self.assertEqual(binop1.left.lineno, 3) + self.assertEqual(binop1.right.lineno, 3) + self.assertEqual(binop1.col_offset, 8) + self.assertEqual(binop1.left.col_offset, 8) + self.assertEqual(binop1.right.col_offset, 12) + # check the second binop location + binop2 = t.body[1].value.values[3].value + self.assertEqual(type(binop2), ast.BinOp) + self.assertEqual(type(binop2.left), ast.Name) + self.assertEqual(type(binop2.op), ast.Add) + self.assertEqual(type(binop2.right), ast.Call) + self.assertEqual(binop2.lineno, 3) + self.assertEqual(binop2.left.lineno, 3) + self.assertEqual(binop2.right.lineno, 3) + self.assertEqual(binop2.col_offset, 23) + self.assertEqual(binop2.left.col_offset, 23) + self.assertEqual(binop2.right.col_offset, 27) + + def __test_ast_line_numbers_nested(self): + expr = """ +a = 10 +f'{a * f"-{x()}-"}'""" + t = ast.parse(expr) + self.assertEqual(type(t), ast.Module) + self.assertEqual(len(t.body), 2) + # check `a = 10` + self.assertEqual(type(t.body[0]), ast.Assign) + self.assertEqual(t.body[0].lineno, 2) + # check `f'...'` + self.assertEqual(type(t.body[1]), ast.Expr) + self.assertEqual(type(t.body[1].value), ast.JoinedStr) + self.assertEqual(len(t.body[1].value.values), 1) + self.assertEqual(type(t.body[1].value.values[0]), ast.FormattedValue) + self.assertEqual(t.body[1].lineno, 3) + self.assertEqual(t.body[1].value.lineno, 3) + self.assertEqual(t.body[1].value.values[0].lineno, 3) + # check the binop location + binop = t.body[1].value.values[0].value + self.assertEqual(type(binop), ast.BinOp) + self.assertEqual(type(binop.left), ast.Name) + self.assertEqual(type(binop.op), ast.Mult) + self.assertEqual(type(binop.right), ast.JoinedStr) + self.assertEqual(binop.lineno, 3) + self.assertEqual(binop.left.lineno, 3) + self.assertEqual(binop.right.lineno, 3) + self.assertEqual(binop.col_offset, 3) + self.assertEqual(binop.left.col_offset, 3) + self.assertEqual(binop.right.col_offset, 7) + # check the nested call location + self.assertEqual(len(binop.right.values), 3) + self.assertEqual(type(binop.right.values[0]), ast.Constant) + self.assertEqual(type(binop.right.values[0].value), str) + self.assertEqual(type(binop.right.values[1]), ast.FormattedValue) + self.assertEqual(type(binop.right.values[2]), ast.Constant) + self.assertEqual(type(binop.right.values[2].value), str) + self.assertEqual(binop.right.values[0].lineno, 3) + self.assertEqual(binop.right.values[1].lineno, 3) + self.assertEqual(binop.right.values[2].lineno, 3) + call = binop.right.values[1].value + self.assertEqual(type(call), ast.Call) + self.assertEqual(call.lineno, 3) + self.assertEqual(call.col_offset, 11) + + def __test_ast_line_numbers_duplicate_expression(self): + """Duplicate expression + + NOTE: this is currently broken, always sets location of the first + expression. + """ + expr = """ +a = 10 +f'{a * x()} {a * x()} {a * x()}' +""" + t = ast.parse(expr) + self.assertEqual(type(t), ast.Module) + self.assertEqual(len(t.body), 2) + # check `a = 10` + self.assertEqual(type(t.body[0]), ast.Assign) + self.assertEqual(t.body[0].lineno, 2) + # check `f'...'` + self.assertEqual(type(t.body[1]), ast.Expr) + self.assertEqual(type(t.body[1].value), ast.JoinedStr) + self.assertEqual(len(t.body[1].value.values), 5) + self.assertEqual(type(t.body[1].value.values[0]), ast.FormattedValue) + self.assertEqual(type(t.body[1].value.values[1]), ast.Constant) + self.assertEqual(type(t.body[1].value.values[1].value), str) + self.assertEqual(type(t.body[1].value.values[2]), ast.FormattedValue) + self.assertEqual(type(t.body[1].value.values[3]), ast.Constant) + self.assertEqual(type(t.body[1].value.values[3].value), str) + self.assertEqual(type(t.body[1].value.values[4]), ast.FormattedValue) + self.assertEqual(t.body[1].lineno, 3) + self.assertEqual(t.body[1].value.lineno, 3) + self.assertEqual(t.body[1].value.values[0].lineno, 3) + self.assertEqual(t.body[1].value.values[1].lineno, 3) + self.assertEqual(t.body[1].value.values[2].lineno, 3) + self.assertEqual(t.body[1].value.values[3].lineno, 3) + self.assertEqual(t.body[1].value.values[4].lineno, 3) + # check the first binop location + binop = t.body[1].value.values[0].value + self.assertEqual(type(binop), ast.BinOp) + self.assertEqual(type(binop.left), ast.Name) + self.assertEqual(type(binop.op), ast.Mult) + self.assertEqual(type(binop.right), ast.Call) + self.assertEqual(binop.lineno, 3) + self.assertEqual(binop.left.lineno, 3) + self.assertEqual(binop.right.lineno, 3) + self.assertEqual(binop.col_offset, 3) + self.assertEqual(binop.left.col_offset, 3) + self.assertEqual(binop.right.col_offset, 7) + # check the second binop location + binop = t.body[1].value.values[2].value + self.assertEqual(type(binop), ast.BinOp) + self.assertEqual(type(binop.left), ast.Name) + self.assertEqual(type(binop.op), ast.Mult) + self.assertEqual(type(binop.right), ast.Call) + self.assertEqual(binop.lineno, 3) + self.assertEqual(binop.left.lineno, 3) + self.assertEqual(binop.right.lineno, 3) + self.assertEqual(binop.col_offset, 3) # FIXME: this is wrong + self.assertEqual(binop.left.col_offset, 3) # FIXME: this is wrong + self.assertEqual(binop.right.col_offset, 7) # FIXME: this is wrong + # check the third binop location + binop = t.body[1].value.values[4].value + self.assertEqual(type(binop), ast.BinOp) + self.assertEqual(type(binop.left), ast.Name) + self.assertEqual(type(binop.op), ast.Mult) + self.assertEqual(type(binop.right), ast.Call) + self.assertEqual(binop.lineno, 3) + self.assertEqual(binop.left.lineno, 3) + self.assertEqual(binop.right.lineno, 3) + self.assertEqual(binop.col_offset, 3) # FIXME: this is wrong + self.assertEqual(binop.left.col_offset, 3) # FIXME: this is wrong + self.assertEqual(binop.right.col_offset, 7) # FIXME: this is wrong + + def __test_ast_line_numbers_multiline_fstring(self): + # See bpo-30465 for details. + expr = """ +a = 10 +f''' + {a + * + x()} +non-important content +''' +""" + t = ast.parse(expr) + self.assertEqual(type(t), ast.Module) + self.assertEqual(len(t.body), 2) + # check `a = 10` + self.assertEqual(type(t.body[0]), ast.Assign) + self.assertEqual(t.body[0].lineno, 2) + # check `f'...'` + self.assertEqual(type(t.body[1]), ast.Expr) + self.assertEqual(type(t.body[1].value), ast.JoinedStr) + self.assertEqual(len(t.body[1].value.values), 3) + self.assertEqual(type(t.body[1].value.values[0]), ast.Constant) + self.assertEqual(type(t.body[1].value.values[0].value), str) + self.assertEqual(type(t.body[1].value.values[1]), ast.FormattedValue) + self.assertEqual(type(t.body[1].value.values[2]), ast.Constant) + self.assertEqual(type(t.body[1].value.values[2].value), str) + self.assertEqual(t.body[1].lineno, 3) + self.assertEqual(t.body[1].value.lineno, 3) + self.assertEqual(t.body[1].value.values[0].lineno, 3) + self.assertEqual(t.body[1].value.values[1].lineno, 3) + self.assertEqual(t.body[1].value.values[2].lineno, 3) + self.assertEqual(t.body[1].col_offset, 0) + self.assertEqual(t.body[1].value.col_offset, 0) + self.assertEqual(t.body[1].value.values[0].col_offset, 0) + self.assertEqual(t.body[1].value.values[1].col_offset, 0) + self.assertEqual(t.body[1].value.values[2].col_offset, 0) + # NOTE: the following lineno information and col_offset is correct for + # expressions within FormattedValues. + binop = t.body[1].value.values[1].value + self.assertEqual(type(binop), ast.BinOp) + self.assertEqual(type(binop.left), ast.Name) + self.assertEqual(type(binop.op), ast.Mult) + self.assertEqual(type(binop.right), ast.Call) + self.assertEqual(binop.lineno, 4) + self.assertEqual(binop.left.lineno, 4) + self.assertEqual(binop.right.lineno, 6) + self.assertEqual(binop.col_offset, 4) + self.assertEqual(binop.left.col_offset, 4) + self.assertEqual(binop.right.col_offset, 7) + def test_docstring(self): def f(): f'''Not a docstring''' - self.assertTrue(f.__doc__ is None) + self.assertIsNone(f.__doc__) def g(): '''Not a docstring''' \ f'' - self.assertTrue(g.__doc__ is None) + self.assertIsNone(g.__doc__) def __test_literal_eval(self): with self.assertRaisesRegex(ValueError, 'malformed node or string'): @@ -175,9 +409,27 @@ f'{a * x()}'""" ]) def test_mismatched_parens(self): - self.assertAllRaise(SyntaxError, 'f-string: mismatched', + self.assertAllRaise(SyntaxError, r"f-string: closing parenthesis '\}' " + r"does not match opening parenthesis '\('", ["f'{((}'", ]) + self.assertAllRaise(SyntaxError, r"f-string: closing parenthesis '\)' " + r"does not match opening parenthesis '\['", + ["f'{a[4)}'", + ]) + self.assertAllRaise(SyntaxError, r"f-string: closing parenthesis '\]' " + r"does not match opening parenthesis '\('", + ["f'{a(4]}'", + ]) + self.assertAllRaise(SyntaxError, r"f-string: closing parenthesis '\}' " + r"does not match opening parenthesis '\['", + ["f'{a[4}'", + ]) + self.assertAllRaise(SyntaxError, r"f-string: closing parenthesis '\}' " + r"does not match opening parenthesis '\('", + ["f'{a(4}'", + ]) + self.assertRaises(SyntaxError, eval, "f'{" + "("*500 + "}'") def test_double_braces(self): self.assertEqual(f'{{', '{') @@ -255,7 +507,9 @@ f'{a * x()}'""" ["f'{1#}'", # error because the expression becomes "(1#)" "f'{3(#)}'", "f'{#}'", - "f'{)#}'", # When wrapped in parens, this becomes + ]) + self.assertAllRaise(SyntaxError, r"f-string: unmatched '\)'", + ["f'{)#}'", # When wrapped in parens, this becomes # '()#)'. Make sure that doesn't compile. ]) @@ -271,29 +525,35 @@ f'{a * x()}'""" width = 1 # Test around 256. - for i in range(250, 260): - self.assertEqual(cy_eval(build_fstr(i), x=x, width=width), (x+' ')*i) + # for i in range(250, 260): + # self.assertEqual(eval(build_fstr(i)), (x+' ')*i) + self.assertEqual( + cy_eval('[' + ', '.join(build_fstr(i) for i in range(250, 260)) + ']', x=x, width=width), + [(x+' ')*i for i in range(250, 260)], + ) # Test concatenating 2 largs fstrings. + # self.assertEqual(eval(build_fstr(255)*256), (x+' ')*(255*256)) self.assertEqual(cy_eval(build_fstr(255)*3, x=x, width=width), (x+' ')*(255*3)) # CPython uses 255*256 s = build_fstr(253, '{x:{width}} ') + # self.assertEqual(eval(s), (x+' ')*254) self.assertEqual(cy_eval(s, x=x, width=width), (x+' ')*254) # Test lots of expressions and constants, concatenated. s = "f'{1}' 'x' 'y'" * 1024 + # self.assertEqual(eval(s), '1xy' * 1024) self.assertEqual(cy_eval(s, x=x, width=width), '1xy' * 1024) def test_format_specifier_expressions(self): width = 10 precision = 4 value = decimal.Decimal('12.34567') - if not IS_PY26: - self.assertEqual(f'result: {value:{width}.{precision}}', 'result: 12.35') - self.assertEqual(f'result: {value:{width!r}.{precision}}', 'result: 12.35') - self.assertEqual(f'result: {value:{width:0}.{precision:1}}', 'result: 12.35') - self.assertEqual(f'result: {value:{1}{0:0}.{precision:1}}', 'result: 12.35') - self.assertEqual(f'result: {value:{ 1}{ 0:0}.{ precision:1}}', 'result: 12.35') + self.assertEqual(f'result: {value:{width}.{precision}}', 'result: 12.35') + self.assertEqual(f'result: {value:{width!r}.{precision}}', 'result: 12.35') + self.assertEqual(f'result: {value:{width:0}.{precision:1}}', 'result: 12.35') + self.assertEqual(f'result: {value:{1}{0:0}.{precision:1}}', 'result: 12.35') + self.assertEqual(f'result: {value:{ 1}{ 0:0}.{ precision:1}}', 'result: 12.35') self.assertEqual(f'{10:#{1}0x}', ' 0xa') self.assertEqual(f'{10:{"#"}1{0}{"x"}}', ' 0xa') self.assertEqual(f'{-10:-{"#"}1{0}x}', ' -0xa') @@ -306,14 +566,13 @@ f'{a * x()}'""" # This looks like a nested format spec. ]) - self.assertAllRaise(SyntaxError, "invalid syntax", + self.assertAllRaise(SyntaxError, "f-string: invalid syntax", [# Invalid syntax inside a nested spec. "f'{4:{/5}}'", ]) # CYTHON: The nesting restriction seems rather arbitrary. Ignoring it for now and instead test that it works. - if not IS_PY26: - self.assertEqual(f'result: {value:{width:{0}}.{precision:1}}', 'result: 12.35') + self.assertEqual(f'result: {value:{width:{0}}.{precision:1}}', 'result: 12.35') #self.assertAllRaise(SyntaxError, "f-string: expressions nested too deeply", # [# Can't nest format specifiers. # "f'result: {value:{width:{0}}.{precision:1}}'", @@ -371,7 +630,7 @@ f'{a * x()}'""" ]) # Different error message is raised for other whitespace characters. - self.assertAllRaise(SyntaxError, 'invalid character in identifier', + self.assertAllRaise(SyntaxError, r"invalid non-printable character U\+00A0", ["f'''{\xa0}'''", #"\xa0", ]) @@ -383,12 +642,12 @@ f'{a * x()}'""" # are added around it. But we shouldn't go from an invalid # expression to a valid one. The added parens are just # supposed to allow whitespace (including newlines). - self.assertAllRaise(SyntaxError, 'invalid syntax', + self.assertAllRaise(SyntaxError, 'f-string: invalid syntax', ["f'{,}'", "f'{,}'", # this is (,), which is an error ]) - self.assertAllRaise(SyntaxError, "f-string: expecting '}'", + self.assertAllRaise(SyntaxError, r"f-string: unmatched '\)'", ["f'{3)+(4}'", ]) @@ -502,7 +761,7 @@ f'{a * x()}'""" # lambda doesn't work without parens, because the colon # makes the parser think it's a format_spec - self.assertAllRaise(SyntaxError, 'unexpected EOF while parsing', + self.assertAllRaise(SyntaxError, 'f-string: invalid syntax', ["f'{lambda x:x}'", ]) @@ -511,9 +770,11 @@ f'{a * x()}'""" # a function into a generator def fn(y): f'y:{yield y*2}' + f'{yield}' g = fn(4) self.assertEqual(next(g), 8) + self.assertEqual(next(g), None) def test_yield_send(self): def fn(x): @@ -630,8 +891,7 @@ f'{a * x()}'""" self.assertEqual(f'{f"{y}"*3}', '555') def test_invalid_string_prefixes(self): - self.assertAllRaise(SyntaxError, 'unexpected EOF while parsing', - ["fu''", + single_quote_cases = ["fu''", "uf''", "Fu''", "fU''", @@ -652,8 +912,10 @@ f'{a * x()}'""" "bf''", "bF''", "Bf''", - "BF''", - ]) + "BF''",] + double_quote_cases = [case.replace("'", '"') for case in single_quote_cases] + self.assertAllRaise(SyntaxError, 'unexpected EOF while parsing', + single_quote_cases + double_quote_cases) def test_leading_trailing_spaces(self): self.assertEqual(f'{ 3}', '3') @@ -676,12 +938,17 @@ f'{a * x()}'""" self.assertEqual(f'{3!=4!s}', 'True') self.assertEqual(f'{3!=4!s:.3}', 'Tru') + def test_equal_equal(self): + # Because an expression ending in = has special meaning, + # there's a special test for ==. Make sure it works. + + self.assertEqual(f'{0==1}', 'False') + def test_conversions(self): self.assertEqual(f'{3.14:10.10}', ' 3.14') - if not IS_PY26: - self.assertEqual(f'{3.14!s:10.10}', '3.14 ') - self.assertEqual(f'{3.14!r:10.10}', '3.14 ') - self.assertEqual(f'{3.14!a:10.10}', '3.14 ') + self.assertEqual(f'{3.14!s:10.10}', '3.14 ') + self.assertEqual(f'{3.14!r:10.10}', '3.14 ') + self.assertEqual(f'{3.14!a:10.10}', '3.14 ') self.assertEqual(f'{"a"}', 'a') self.assertEqual(f'{"a"!r}', "'a'") @@ -816,12 +1083,6 @@ f'{a * x()}'""" self.assertEqual('{d[a]}'.format(d=d), 'string') self.assertEqual('{d[0]}'.format(d=d), 'integer') - def test_invalid_expressions(self): - self.assertAllRaise(SyntaxError, 'invalid syntax', - [r"f'{a[4)}'", - r"f'{a(4]}'", - ]) - def test_errors(self): # see issue 26287 exc = ValueError if sys.version_info < (3, 4) else TypeError @@ -834,6 +1095,16 @@ f'{a * x()}'""" r"f'{1000:j}'", ]) + def __test_filename_in_syntaxerror(self): + # see issue 38964 + with temp_cwd() as cwd: + file_path = os.path.join(cwd, 't.py') + with open(file_path, 'w') as f: + f.write('f"{a b}"') # This generates a SyntaxError + _, _, stderr = assert_python_failure(file_path, + PYTHONIOENCODING='ascii') + self.assertIn(file_path.encode('ascii', 'backslashreplace'), stderr) + def test_loop(self): for i in range(1000): self.assertEqual(f'i:{i}', 'i:' + str(i)) @@ -855,5 +1126,133 @@ f'{a * x()}'""" self.assertEqual(cy_eval('f"\\\n"'), '') self.assertEqual(cy_eval('f"\\\r"'), '') + def test_debug_conversion(self): + x = 'A string' + self.assertEqual(f'{x=}', 'x=' + repr(x)) + self.assertEqual(f'{x =}', 'x =' + repr(x)) + self.assertEqual(f'{x=!s}', 'x=' + str(x)) + self.assertEqual(f'{x=!r}', 'x=' + repr(x)) + self.assertEqual(f'{x=!a}', 'x=' + ascii(x)) + + x = 2.71828 + self.assertEqual(f'{x=:.2f}', 'x=' + format(x, '.2f')) + self.assertEqual(f'{x=:}', 'x=' + format(x, '')) + self.assertEqual(f'{x=!r:^20}', 'x=' + format(repr(x), '^20')) + self.assertEqual(f'{x=!s:^20}', 'x=' + format(str(x), '^20')) + self.assertEqual(f'{x=!a:^20}', 'x=' + format(ascii(x), '^20')) + + x = 9 + self.assertEqual(f'{3*x+15=}', '3*x+15=42') + + # There is code in ast.c that deals with non-ascii expression values. So, + # use a unicode identifier to trigger that. + tenπ = 31.4 + self.assertEqual(f'{tenπ=:.2f}', 'tenπ=31.40') + + # Also test with Unicode in non-identifiers. + if not IS_PY2: # unicode representation looks different right now - not sure if that's a good thing + self.assertEqual(f'{"Σ"=}', '"Σ"=\'Σ\'') + + # Make sure nested fstrings still work. + self.assertEqual(f'{f"{3.1415=:.1f}":*^20}', '*****3.1415=3.1*****') + + # Make sure text before and after an expression with = works + # correctly. + pi = 'π' + if not IS_PY2: # unicode representation looks different right now - not sure if that's a good thing + self.assertEqual(f'alpha α {pi=} ω omega', u"alpha α pi='π' ω omega") + + # Check multi-line expressions. + self.assertEqual(f'''{ +3 +=}''', '\n3\n=3') + + # Since = is handled specially, make sure all existing uses of + # it still work. + + self.assertEqual(f'{0==1}', 'False') + self.assertEqual(f'{0!=1}', 'True') + self.assertEqual(f'{0<=1}', 'True') + self.assertEqual(f'{0>=1}', 'False') + # Walrus not implemented yet, skip + # self.assertEqual(f'{(x:="5")}', '5') + # self.assertEqual(x, '5') + # self.assertEqual(f'{(x:=5)}', '5') + # self.assertEqual(x, 5) + self.assertEqual(f'{"="}', '=') + + x = 20 + # This isn't an assignment expression, it's 'x', with a format + # spec of '=10'. See test_walrus: you need to use parens. + self.assertEqual(f'{x:=10}', ' 20') + + # Test named function parameters, to make sure '=' parsing works + # there. + def f(a): + nonlocal x + oldx = x + x = a + return oldx + x = 0 + self.assertEqual(f'{f(a="3=")}', '0') + self.assertEqual(x, '3=') + self.assertEqual(f'{f(a=4)}', '3=') + self.assertEqual(x, 4) + + # Make sure __format__ is being called. + class C: + def __format__(self, s): + return f'FORMAT-{s}' + def __repr__(self): + return 'REPR' + + self.assertEqual(f'{C()=}', 'C()=REPR') + self.assertEqual(f'{C()=!r}', 'C()=REPR') + self.assertEqual(f'{C()=:}', 'C()=FORMAT-') + self.assertEqual(f'{C()=: }', 'C()=FORMAT- ') + self.assertEqual(f'{C()=:x}', 'C()=FORMAT-x') + self.assertEqual(f'{C()=!r:*^20}', 'C()=********REPR********') + + self.assertRaises(SyntaxError, eval, "f'{C=]'") + + # Make sure leading and following text works. + x = 'foo' + self.assertEqual(f'X{x=}Y', 'Xx='+repr(x)+'Y') + + # Make sure whitespace around the = works. + self.assertEqual(f'X{x =}Y', 'Xx ='+repr(x)+'Y') + self.assertEqual(f'X{x= }Y', 'Xx= '+repr(x)+'Y') + self.assertEqual(f'X{x = }Y', 'Xx = '+repr(x)+'Y') + + # These next lines contains tabs. Backslash escapes don't + # work in f-strings. + # patchcheck doesn't like these tabs. So the only way to test + # this will be to dynamically created and exec the f-strings. But + # that's such a hassle I'll save it for another day. For now, convert + # the tabs to spaces just to shut up patchcheck. + #self.assertEqual(f'X{x =}Y', 'Xx\t='+repr(x)+'Y') + #self.assertEqual(f'X{x = }Y', 'Xx\t=\t'+repr(x)+'Y') + + + def test_walrus(self): + x = 20 + # This isn't an assignment expression, it's 'x', with a format + # spec of '=10'. + self.assertEqual(f'{x:=10}', ' 20') + + # Note to anyone going to enable these: please have a look to the test + # above this one for more walrus cases to enable. + """ + # This is an assignment expression, which requires parens. + self.assertEqual(f'{(x:=10)}', '10') + self.assertEqual(x, 10) + """ + + def test_invalid_syntax_error_message(self): + # with self.assertRaisesRegex(SyntaxError, "f-string: invalid syntax"): + # compile("f'{a $ b}'", "?", "exec") + self.assertAllRaise(CompileError, "f-string: invalid syntax", ["f'{a $ b}'"]) + + if __name__ == '__main__': unittest.main() diff --git a/tests/run/test_genericclass.py b/tests/run/test_genericclass.py new file mode 100644 index 000000000..e52fc2dc7 --- /dev/null +++ b/tests/run/test_genericclass.py @@ -0,0 +1,289 @@ +# mode: run +# tag: pure3.7 +# cython: language_level=3 + +# COPIED FROM CPython 3.7 + +import contextlib +import unittest +import sys + +class TestMROEntry(unittest.TestCase): + def test_mro_entry_signature(self): + tested = [] + class B: ... + class C: + def __mro_entries__(self, *args, **kwargs): + tested.extend([args, kwargs]) + return (C,) + c = C() + self.assertEqual(tested, []) + class D(B, c): ... + self.assertEqual(tested[0], ((B, c),)) + self.assertEqual(tested[1], {}) + + def test_mro_entry(self): + tested = [] + class A: ... + class B: ... + class C: + def __mro_entries__(self, bases): + tested.append(bases) + return (self.__class__,) + c = C() + self.assertEqual(tested, []) + class D(A, c, B): ... + self.assertEqual(tested[-1], (A, c, B)) + self.assertEqual(D.__bases__, (A, C, B)) + self.assertEqual(D.__orig_bases__, (A, c, B)) + self.assertEqual(D.__mro__, (D, A, C, B, object)) + d = D() + class E(d): ... + self.assertEqual(tested[-1], (d,)) + self.assertEqual(E.__bases__, (D,)) + + def test_mro_entry_none(self): + tested = [] + class A: ... + class B: ... + class C: + def __mro_entries__(self, bases): + tested.append(bases) + return () + c = C() + self.assertEqual(tested, []) + class D(A, c, B): ... + self.assertEqual(tested[-1], (A, c, B)) + self.assertEqual(D.__bases__, (A, B)) + self.assertEqual(D.__orig_bases__, (A, c, B)) + self.assertEqual(D.__mro__, (D, A, B, object)) + class E(c): ... + self.assertEqual(tested[-1], (c,)) + if sys.version_info[0] > 2: + # not all of it works on Python 2 + self.assertEqual(E.__bases__, (object,)) + self.assertEqual(E.__orig_bases__, (c,)) + if sys.version_info[0] > 2: + # not all of it works on Python 2 + self.assertEqual(E.__mro__, (E, object)) + + def test_mro_entry_with_builtins(self): + tested = [] + class A: ... + class C: + def __mro_entries__(self, bases): + tested.append(bases) + return (dict,) + c = C() + self.assertEqual(tested, []) + class D(A, c): ... + self.assertEqual(tested[-1], (A, c)) + self.assertEqual(D.__bases__, (A, dict)) + self.assertEqual(D.__orig_bases__, (A, c)) + self.assertEqual(D.__mro__, (D, A, dict, object)) + + def test_mro_entry_with_builtins_2(self): + tested = [] + class C: + def __mro_entries__(self, bases): + tested.append(bases) + return (C,) + c = C() + self.assertEqual(tested, []) + class D(c, dict): ... + self.assertEqual(tested[-1], (c, dict)) + self.assertEqual(D.__bases__, (C, dict)) + self.assertEqual(D.__orig_bases__, (c, dict)) + self.assertEqual(D.__mro__, (D, C, dict, object)) + + def test_mro_entry_errors(self): + class C_too_many: + def __mro_entries__(self, bases, something, other): + return () + c = C_too_many() + with self.assertRaises(TypeError): + class D(c): ... + class C_too_few: + def __mro_entries__(self): + return () + d = C_too_few() + with self.assertRaises(TypeError): + class D(d): ... + + def test_mro_entry_errors_2(self): + class C_not_callable: + __mro_entries__ = "Surprise!" + c = C_not_callable() + with self.assertRaises(TypeError): + class D(c): ... + class C_not_tuple: + def __mro_entries__(self): + return object + c = C_not_tuple() + with self.assertRaises(TypeError): + class D(c): ... + + def test_mro_entry_metaclass(self): + meta_args = [] + class Meta(type): + def __new__(mcls, name, bases, ns): + meta_args.extend([mcls, name, bases, ns]) + return super().__new__(mcls, name, bases, ns) + class A: ... + class C: + def __mro_entries__(self, bases): + return (A,) + c = C() + class D(c, metaclass=Meta): + x = 1 + self.assertEqual(meta_args[0], Meta) + self.assertEqual(meta_args[1], 'D') + self.assertEqual(meta_args[2], (A,)) + self.assertEqual(meta_args[3]['x'], 1) + self.assertEqual(D.__bases__, (A,)) + self.assertEqual(D.__orig_bases__, (c,)) + self.assertEqual(D.__mro__, (D, A, object)) + self.assertEqual(D.__class__, Meta) + + @unittest.skipIf(sys.version_info < (3, 7), "'type' checks for __mro_entries__ not implemented") + def test_mro_entry_type_call(self): + # Substitution should _not_ happen in direct type call + class C: + def __mro_entries__(self, bases): + return () + c = C() + with self.assertRaisesRegex(TypeError, + "MRO entry resolution; " + "use types.new_class()"): + type('Bad', (c,), {}) + + +class TestClassGetitem(unittest.TestCase): + # BEGIN - Additional tests from cython + def test_no_class_getitem(self): + class C: ... + with self.assertRaises(TypeError): + C[int] + + # END - Additional tests from cython + + def test_class_getitem(self): + getitem_args = [] + class C: + def __class_getitem__(*args, **kwargs): + getitem_args.extend([args, kwargs]) + return None + C[int, str] + self.assertEqual(getitem_args[0], (C, (int, str))) + self.assertEqual(getitem_args[1], {}) + + def test_class_getitem_format(self): + class C: + def __class_getitem__(cls, item): + return f'C[{item.__name__}]' + self.assertEqual(C[int], 'C[int]') + self.assertEqual(C[C], 'C[C]') + + def test_class_getitem_inheritance(self): + class C: + def __class_getitem__(cls, item): + return f'{cls.__name__}[{item.__name__}]' + class D(C): ... + self.assertEqual(D[int], 'D[int]') + self.assertEqual(D[D], 'D[D]') + + def test_class_getitem_inheritance_2(self): + class C: + def __class_getitem__(cls, item): + return 'Should not see this' + class D(C): + def __class_getitem__(cls, item): + return f'{cls.__name__}[{item.__name__}]' + self.assertEqual(D[int], 'D[int]') + self.assertEqual(D[D], 'D[D]') + + def test_class_getitem_classmethod(self): + class C: + @classmethod + def __class_getitem__(cls, item): + return f'{cls.__name__}[{item.__name__}]' + class D(C): ... + self.assertEqual(D[int], 'D[int]') + self.assertEqual(D[D], 'D[D]') + + @unittest.skipIf(sys.version_info < (3, 6), "__init_subclass__() requires Py3.6+ (PEP 487)") + def test_class_getitem_patched(self): + class C: + def __init_subclass__(cls): + def __class_getitem__(cls, item): + return f'{cls.__name__}[{item.__name__}]' + cls.__class_getitem__ = classmethod(__class_getitem__) + class D(C): ... + self.assertEqual(D[int], 'D[int]') + self.assertEqual(D[D], 'D[D]') + + def test_class_getitem_with_builtins(self): + class A(dict): + called_with = None + + def __class_getitem__(cls, item): + cls.called_with = item + class B(A): + pass + self.assertIs(B.called_with, None) + B[int] + self.assertIs(B.called_with, int) + + def test_class_getitem_errors(self): + class C_too_few: + def __class_getitem__(cls): + return None + with self.assertRaises(TypeError): + C_too_few[int] + class C_too_many: + def __class_getitem__(cls, one, two): + return None + with self.assertRaises(TypeError): + C_too_many[int] + + def test_class_getitem_errors_2(self): + class C: + def __class_getitem__(cls, item): + return None + with self.assertRaises(TypeError): + C()[int] + class E: ... + e = E() + e.__class_getitem__ = lambda cls, item: 'This will not work' + with self.assertRaises(TypeError): + e[int] + class C_not_callable: + __class_getitem__ = "Surprise!" + with self.assertRaises(TypeError): + C_not_callable[int] + + def test_class_getitem_metaclass(self): + class Meta(type): + def __class_getitem__(cls, item): + return f'{cls.__name__}[{item.__name__}]' + self.assertEqual(Meta[int], 'Meta[int]') + + def test_class_getitem_with_metaclass(self): + class Meta(type): pass + class C(metaclass=Meta): + def __class_getitem__(cls, item): + return f'{cls.__name__}[{item.__name__}]' + self.assertEqual(C[int], 'C[int]') + + def test_class_getitem_metaclass_first(self): + class Meta(type): + def __getitem__(cls, item): + return 'from metaclass' + class C(metaclass=Meta): + def __class_getitem__(cls, item): + return 'from __class_getitem__' + self.assertEqual(C[int], 'from metaclass') + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/run/test_genericclass_exttype.pyx b/tests/run/test_genericclass_exttype.pyx new file mode 100644 index 000000000..b8cdd0c47 --- /dev/null +++ b/tests/run/test_genericclass_exttype.pyx @@ -0,0 +1,96 @@ +# mode: run +# cython: language_level=3 + +import unittest +import sys + + +cdef class UnSupport: pass + +cdef class Unpack: + para_list = [] + def __class_getitem__(*args, **kwargs): + Unpack.para_list.extend([args, kwargs]) + +cdef class Format: + def __class_getitem__(cls, item): + return f'{cls.__name__}[{item.__name__}]' + +cdef class ExFormat(Format): pass + +cdef class Override: + def __class_getitem__(cls, item): + return 'Should not see this' + +cdef class Covered(Override): + def __class_getitem__(cls, item): + return f'{cls.__name__}[{item.__name__}]' + +cdef class Decorated: + @classmethod + def __class_getitem__(cls, item): + return f'{cls.__name__}[{item.__name__}]' + +cdef class ExDecorated(Decorated): pass + +cdef class Invalid1: + def __class_getitem__(cls): pass + +cdef class Invalid2: + def __class_getitem__(cls, item1, item2): pass + +cdef class Invalid3: + cdef dict __dict__ + def __init__(self): + self.__class_getitem__ = lambda cls, items: 'This will not work' + +cdef class Invalid4: + __class_getitem__ = "Surprise!" + + +class TestClassGetitem(unittest.TestCase): + # BEGIN - Additional tests from cython + def test_no_class_getitem(self): + with self.assertRaises(TypeError): + UnSupport[int] + + # END - Additional tests from cython + + def test_class_getitem(self): + Unpack[int, str] + self.assertEqual(Unpack.para_list[0], (Unpack, (int, str))) + self.assertEqual(Unpack.para_list[1], {}) + + def test_class_getitem_format(self): + self.assertEqual(Format[int], 'Format[int]') + self.assertEqual(Format[Format], 'Format[Format]') + + def test_class_getitem_inheritance(self): + self.assertEqual(ExFormat[int], 'ExFormat[int]') + self.assertEqual(ExFormat[ExFormat], 'ExFormat[ExFormat]') + + def test_class_getitem_inheritance_2(self): + self.assertEqual(Covered[int], 'Covered[int]') + self.assertEqual(Covered[Covered], 'Covered[Covered]') + + def test_class_getitem_classmethod(self): + self.assertEqual(ExDecorated[int], 'ExDecorated[int]') + self.assertEqual(ExDecorated[ExDecorated], 'ExDecorated[ExDecorated]') + + def test_class_getitem_errors(self): + with self.assertRaises(TypeError): + Invalid1[int] + with self.assertRaises(TypeError): + Invalid2[int] + + def test_class_getitem_errors_2(self): + with self.assertRaises(TypeError): + Format()[int] + with self.assertRaises(TypeError): + Invalid3()[int] + with self.assertRaises(TypeError): + Invalid4[int] + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/run/test_grammar.py b/tests/run/test_grammar.py index 3bdadbefa..c41b75f55 100644 --- a/tests/run/test_grammar.py +++ b/tests/run/test_grammar.py @@ -1,9 +1,26 @@ -### COPIED FROM CPython 3.5 - ADDED PART FOLLOWS ### +### COPIED FROM CPython 3.9 - ADDED PART FOLLOWS ### # cython: language_level=3 +import cython import contextlib from tempfile import NamedTemporaryFile -from Cython.Compiler.Main import compile as cython_compile +from Cython.Compiler.Main import compile as cython_compile, CompileError +from Cython.Build.Inline import cython_inline + + +@contextlib.contextmanager +def hidden_stderr(): + try: + from StringIO import StringIO + except ImportError: + from io import StringIO + + old_stderr = sys.stderr + try: + sys.stderr = StringIO() + yield + finally: + sys.stderr = old_stderr def _compile(code): @@ -11,36 +28,44 @@ def _compile(code): f.write(code.encode('utf8')) f.flush() - try: - from StringIO import StringIO - except ImportError: - from io import StringIO - - old_stderr = sys.stderr - try: - sys.stderr = StringIO() + with hidden_stderr(): result = cython_compile(f.name, language_level=3) - finally: - sys.stderr = old_stderr return result -def check_syntax_error(test, code): +def check_syntax_error(test, code, msg=None): result = _compile(code) assert not result.c_file -def compile(code, name, what): - assert what == 'exec' - result = _compile(code) - if not result.c_file: - raise SyntaxError('unexpected EOF') # see usage of compile() below +def check_syntax_warning(test, code, *args): + # ignore for now + return _compile(code) -def exec(code): - result = _compile(code) - if not result.c_file: - raise SyntaxError('unexpected EOF') # see usage of compile() below +if cython.compiled: + def compile(code, name, what): + assert what == 'exec' + result = _compile(code) + if not result.c_file: + raise SyntaxError('unexpected EOF') # see usage of compile() below + + def exec(code): + result = _compile(code) + if not result.c_file: + raise SyntaxError('unexpected EOF') # see usage of compile() below + + def eval(code): + try: + with hidden_stderr(): + return cython_inline(code) + except CompileError as exc: + raise SyntaxError(str(exc)) + + +def use_old_parser(): + # FIXME: currently disabling new PEG parser tests. + return True import unittest @@ -72,16 +97,109 @@ skip = unittest.skip # Python test set -- part 1, grammar. # This just tests whether the parser accepts them all. -#from test.support import check_syntax_error +#from test.support import check_syntax_error, check_syntax_warning, use_old_parser import inspect import unittest import sys +import warnings # testing import * from sys import * +# different import patterns to check that __annotations__ does not interfere +# with import machinery +#import test.ann_module as ann_module +#import typing +#from collections import ChainMap +#from test import ann_module2 +#import test + +# These are shared with test_tokenize and other test modules. +# +# Note: since several test cases filter out floats by looking for "e" and ".", +# don't add hexadecimal literals that contain "e" or "E". +VALID_UNDERSCORE_LITERALS = [ + '0_0_0', + '4_2', + '1_0000_0000', + '0b1001_0100', + '0xffff_ffff', + '0o5_7_7', + '1_00_00.5', + '1_00_00.5e5', + '1_00_00e5_1', + '1e1_0', + '.1_4', + '.1_4e1', + '0b_0', + '0x_f', + '0o_5', + '1_00_00j', + '1_00_00.5j', + '1_00_00e5_1j', + '.1_4j', + '(1_2.5+3_3j)', + '(.5_6j)', +] +INVALID_UNDERSCORE_LITERALS = [ + # Trailing underscores: + '0_', + '42_', + '1.4j_', + '0x_', + '0b1_', + '0xf_', + '0o5_', + '0 if 1_Else 1', + # Underscores in the base selector: + '0_b0', + '0_xf', + '0_o5', + # Old-style octal, still disallowed: + '0_7', + '09_99', + # Multiple consecutive underscores: + '4_______2', + '0.1__4', + '0.1__4j', + '0b1001__0100', + '0xffff__ffff', + '0x___', + '0o5__77', + '1e1__0', + '1e1__0j', + # Underscore right before a dot: + '1_.4', + '1_.4j', + # Underscore right after a dot: + '1._4', + '1._4j', + '._5', + '._5j', + # Underscore right after a sign: + '1.0e+_1', + '1.0e+_1j', + # Underscore right before j: + '1.4_j', + '1.4e5_j', + # Underscore right before e: + '1_e1', + '1.4_e1', + '1.4_e1j', + # Underscore right after e: + '1e_1', + '1.4e_1', + '1.4e_1j', + # Complex cases with parens: + '(1+1.5_j_)', + '(1+1.5_j)', +] + class TokenTests(unittest.TestCase): + #from test.support import check_syntax_error + check_syntax_error = check_syntax_error + def test_backslash(self): # Backslash means line continuation: x = 1 \ @@ -158,6 +276,40 @@ class TokenTests(unittest.TestCase): self.assertEqual(1 if 0else 0, 0) self.assertRaises(SyntaxError, eval, "0 if 1Else 0") + @skip("Done more efficiently in TestGrammar") + def test_underscore_literals(self): + for lit in VALID_UNDERSCORE_LITERALS: + self.assertEqual(eval(lit), eval(lit.replace('_', ''))) + for lit in INVALID_UNDERSCORE_LITERALS: + self.assertRaises(SyntaxError, eval, lit) + # Sanity check: no literal begins with an underscore + self.assertRaises(NameError, eval, "_0") + + def test_bad_numerical_literals(self): + check = self.check_syntax_error + check("0b12", "invalid digit '2' in binary literal") + check("0b1_2", "invalid digit '2' in binary literal") + check("0b2", "invalid digit '2' in binary literal") + check("0b1_", "invalid binary literal") + check("0b", "invalid binary literal") + check("0o18", "invalid digit '8' in octal literal") + check("0o1_8", "invalid digit '8' in octal literal") + check("0o8", "invalid digit '8' in octal literal") + check("0o1_", "invalid octal literal") + check("0o", "invalid octal literal") + check("0x1_", "invalid hexadecimal literal") + check("0x", "invalid hexadecimal literal") + check("1_", "invalid decimal literal") + # FIXME: not currently a syntax error + """ + check("012", + "leading zeros in decimal integer literals are not permitted; " + "use an 0o prefix for octal integers") + """ + check("1.2_", "invalid decimal literal") + check("1e2_", "invalid decimal literal") + check("1e+", "invalid decimal literal") + def test_string_literals(self): x = ''; y = ""; self.assertTrue(len(x) == 0 and x == y) x = '\''; y = "'"; self.assertTrue(len(x) == 1 and x == y and ord(x) == 39) @@ -201,7 +353,8 @@ the \'lazy\' dog.\n\ def test_ellipsis(self): x = ... self.assertTrue(x is Ellipsis) - self.assertRaises(SyntaxError, eval, ".. .") + # FIXME: why is this not rejected ??? + #self.assertRaises(SyntaxError, eval, ".. .") def test_eof_error(self): samples = ("def foo(", "\ndef foo(", "def foo(\n") @@ -210,7 +363,7 @@ the \'lazy\' dog.\n\ compile(s, "<test>", "exec") self.assertIn("unexpected EOF", str(cm.exception)) -var_annot_global: int # a global annotated is necessary for test_var_annot +var_annot_global: int # a global annotated is necessary for test_var_annot # custom namespace for testing __annotations__ @@ -225,6 +378,18 @@ class CNS: class GrammarTests(unittest.TestCase): + #from test.support import check_syntax_error, check_syntax_warning + check_syntax_error, check_syntax_warning = check_syntax_error, check_syntax_warning + + if not hasattr(unittest.TestCase, 'subTest'): + @contextlib.contextmanager + def subTest(self, source=None, case=None, **kwargs): + try: + yield + except Exception: + print(source or case) + raise + # single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE # XXX can't test in a script -- this rule is only used when interactive @@ -250,7 +415,7 @@ class GrammarTests(unittest.TestCase): my_lst[one()-1]: int = 5 self.assertEqual(my_lst, [5]) - @skip("Bug: global vs. local declarations do not currently raise an error") + @skip("Cython Bug: global vs. local declarations do not currently raise an error") def test_var_annot_syntax_errors(self): # parser pass check_syntax_error(self, "def f: int") @@ -271,7 +436,6 @@ class GrammarTests(unittest.TestCase): " global x\n" " x: int\n") - @skip("Class annotations not implemented") def test_var_annot_basic_semantics(self): # execution order with self.assertRaises(ZeroDivisionError): @@ -289,21 +453,21 @@ class GrammarTests(unittest.TestCase): def f_OK(): x: 1/0 f_OK() - - ### The following are compile time errors in Cython. - - #def fbad(): - # x: int - # print(x) - #with self.assertRaises(UnboundLocalError): - # fbad() - #def f2bad(): - # (no_such_global): int - # print(no_such_global) - #try: - # f2bad() - #except Exception as e: - # self.assertIs(type(e), NameError) + # Compile-time errors in Cython: + """ + def fbad(): + x: int + print(x) + with self.assertRaises(UnboundLocalError): + fbad() + def f2bad(): + (no_such_global): int + print(no_such_global) + try: + f2bad() + except Exception as e: + self.assertIs(type(e), NameError) + """ # class semantics class C: @@ -312,7 +476,8 @@ class GrammarTests(unittest.TestCase): z = 2 def __init__(self, x): self.x: int = x - self.assertEqual(C.__annotations__, {'_C__foo': int, 's': str}) + + self.assertEqual(C.__annotations__, {'_C__foo': 'int', 's': 'str'}) with self.assertRaises(NameError): class CBad: no_such_name_defined.attr: int = 0 @@ -321,7 +486,7 @@ class GrammarTests(unittest.TestCase): x: int x.y: list = [] - @skip("Class annotations not implemented") + @skip("Not currently supported: https://github.com/cython/cython/issues/3839") def test_var_annot_metaclass_semantics(self): class CMeta(type): @classmethod @@ -403,9 +568,23 @@ class GrammarTests(unittest.TestCase): exec('X: str', {}, CNS2()) self.assertEqual(nonloc_ns['__annotations__']['x'], str) + @skip("Depends on 3-args compiled exec()") + def test_var_annot_rhs(self): + ns = {} + exec('x: tuple = 1, 2', ns) + self.assertEqual(ns['x'], (1, 2)) + stmt = ('def f():\n' + ' x: int = yield') + exec(stmt, ns) + self.assertEqual(list(ns['f']()), [None]) + + ns = {"a": 1, 'b': (2, 3, 4), "c":5, "Tuple": typing.Tuple} + exec('x: Tuple[int, ...] = a,*b,c', ns) + self.assertEqual(ns['x'], (1, 2, 3, 4, 5)) + def test_funcdef(self): ### [decorators] 'def' NAME parameters ['->' test] ':' suite - ### decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE + ### decorator: '@' namedexpr_test NEWLINE ### decorators: decorator+ ### parameters: '(' [typedargslist] ')' ### typedargslist: ((tfpdef ['=' test] ',')* @@ -544,9 +723,10 @@ class GrammarTests(unittest.TestCase): pos2key2dict(1,2,k2=100,tokwarg1=100,tokwarg2=200) pos2key2dict(1,2,tokwarg1=100,tokwarg2=200, k2=100) - self.assertRaises(SyntaxError, eval, "def f(*): pass") - self.assertRaises(SyntaxError, eval, "def f(*,): pass") - self.assertRaises(SyntaxError, eval, "def f(*, **kwds): pass") + # FIXME: currently does not raise an error + #self.assertRaises(SyntaxError, eval, "def f(*): pass") + #self.assertRaises(SyntaxError, eval, "def f(*,): pass") + #self.assertRaises(SyntaxError, eval, "def f(*, **kwds): pass") # keyword arguments after *arglist def f(*args, **kwargs): @@ -566,39 +746,75 @@ class GrammarTests(unittest.TestCase): # argument annotation tests def f(x) -> list: pass - self.assertEqual(f.__annotations__, {'return': list}) + self.assertEqual(f.__annotations__, {'return': 'list'}) def f(x: int): pass - self.assertEqual(f.__annotations__, {'x': int}) + self.assertEqual(f.__annotations__, {'x': 'int'}) + def f(x: int, /): pass + self.assertEqual(f.__annotations__, {'x': 'int'}) + def f(x: int = 34, /): pass + self.assertEqual(f.__annotations__, {'x': 'int'}) def f(*x: str): pass - self.assertEqual(f.__annotations__, {'x': str}) + self.assertEqual(f.__annotations__, {'x': 'str'}) def f(**x: float): pass - self.assertEqual(f.__annotations__, {'x': float}) + self.assertEqual(f.__annotations__, {'x': 'float'}) def f(x, y: 1+2): pass - self.assertEqual(f.__annotations__, {'y': 3}) + self.assertEqual(f.__annotations__, {'y': '1 + 2'}) + def f(x, y: 1+2, /): pass + self.assertEqual(f.__annotations__, {'y': '1 + 2'}) def f(a, b: 1, c: 2, d): pass - self.assertEqual(f.__annotations__, {'b': 1, 'c': 2}) + self.assertEqual(f.__annotations__, {'b': '1', 'c': '2'}) + def f(a, b: 1, /, c: 2, d): pass + self.assertEqual(f.__annotations__, {'b': '1', 'c': '2'}) def f(a, b: 1, c: 2, d, e: 3 = 4, f=5, *g: 6): pass self.assertEqual(f.__annotations__, - {'b': 1, 'c': 2, 'e': 3, 'g': 6}) + {'b': '1', 'c': '2', 'e': '3', 'g': '6'}) def f(a, b: 1, c: 2, d, e: 3 = 4, f=5, *g: 6, h: 7, i=8, j: 9 = 10, **k: 11) -> 12: pass self.assertEqual(f.__annotations__, - {'b': 1, 'c': 2, 'e': 3, 'g': 6, 'h': 7, 'j': 9, - 'k': 11, 'return': 12}) + {'b': '1', 'c': '2', 'e': '3', 'g': '6', 'h': '7', 'j': '9', + 'k': '11', 'return': '12'}) + # FIXME: compile failure on positional-only argument declaration + """ + def f(a, b: 1, c: 2, d, e: 3 = 4, f: int = 5, /, *g: 6, h: 7, i=8, j: 9 = 10, + **k: 11) -> 12: pass + self.assertEqual(f.__annotations__, + {'b': 1, 'c': 2, 'e': 3, 'f': int, 'g': 6, 'h': 7, 'j': 9, + 'k': 11, 'return': 12}) + """ # Check for issue #20625 -- annotations mangling class Spam: def f(self, *, __kw: 1): pass class Ham(Spam): pass - self.assertEqual(Spam.f.__annotations__, {'_Spam__kw': 1}) - self.assertEqual(Ham.f.__annotations__, {'_Spam__kw': 1}) + # FIXME: not currently mangled + """ + self.assertEqual(Spam.f.__annotations__, {'_Spam__kw': '1'}) + self.assertEqual(Ham.f.__annotations__, {'_Spam__kw': '1'}) + """ # Check for SF Bug #1697248 - mixing decorators and a return annotation def null(x): return x @null def f(x) -> list: pass - self.assertEqual(f.__annotations__, {'return': list}) - - # test MAKE_CLOSURE with a variety of oparg's + self.assertEqual(f.__annotations__, {'return': 'list'}) + + # Test expressions as decorators (PEP 614): + # FIXME: implement PEP 614 + """ + @False or null + def f(x): pass + @d := null + def f(x): pass + @lambda f: null(f) + def f(x): pass + @[..., null, ...][1] + def f(x): pass + @null(null)(null) + def f(x): pass + @[null][0].__call__.__call__ + def f(x): pass + """ + + # test closures with a variety of opargs closure = 1 def f(): return closure def f(x=1): return closure @@ -626,7 +842,7 @@ class GrammarTests(unittest.TestCase): ### lambdef: 'lambda' [varargslist] ':' test l1 = lambda : 0 self.assertEqual(l1(), 0) - l2 = lambda : a[d] # XXX just testing the expression + l2 = lambda : a[d] # XXX just testing the expression l3 = lambda : [2 < x for x in [-1, 3, 0]] self.assertEqual(l3(), [0, 1, 0]) l4 = lambda x = lambda y = lambda z=1 : z : y() : x() @@ -703,11 +919,13 @@ class GrammarTests(unittest.TestCase): for case in cases: source = case.format(keyword) with self.subTest(source=source): - with self.assertRaisesRegex(SyntaxError, custom_msg): + #with self.assertRaisesRegex(SyntaxError, custom_msg): + with self.assertRaises(SyntaxError): exec(source) source = source.replace("foo", "(foo.)") with self.subTest(source=source): - with self.assertRaisesRegex(SyntaxError, "invalid syntax"): + #with self.assertRaisesRegex(SyntaxError, "invalid syntax"): + with self.assertRaises(SyntaxError): exec(source) def test_del_stmt(self): @@ -719,6 +937,24 @@ class GrammarTests(unittest.TestCase): del abc del x, y, (z, xyz) + x, y, z = "xyz" + del x + del y, + del (z) + del () + + a, b, c, d, e, f, g = "abcdefg" + del a, (b, c), (d, (e, f)) + + a, b, c, d, e, f, g = "abcdefg" + del a, [b, c], (d, [e, f]) + + abcd = list("abcd") + del abcd[1:2] + + # FIXME: currently fails to compile + #compile("del a, (b[0].c, (d.e, f.g[1:2])), [h.i.j], ()", "<testcase>", "exec") + def test_pass_stmt(self): # 'pass' pass @@ -848,6 +1084,59 @@ class GrammarTests(unittest.TestCase): break self.assertEqual(count, 0) + def test_continue_in_finally(self): + count = 0 + while count < 2: + count += 1 + try: + pass + finally: + continue + break + self.assertEqual(count, 2) + + count = 0 + while count < 2: + count += 1 + try: + break + finally: + continue + self.assertEqual(count, 2) + + count = 0 + while count < 2: + count += 1 + try: + 1/0 + finally: + continue + break + self.assertEqual(count, 2) + + for count in [0, 1]: + try: + pass + finally: + continue + break + self.assertEqual(count, 1) + + for count in [0, 1]: + try: + break + finally: + continue + self.assertEqual(count, 1) + + for count in [0, 1]: + try: + 1/0 + finally: + continue + break + self.assertEqual(count, 1) + def test_return_in_finally(self): def g1(): try: @@ -870,6 +1159,62 @@ class GrammarTests(unittest.TestCase): return 4 self.assertEqual(g3(), 4) + @skip("FIXME: currently crashes because the iterable is cleaned up on 'return', not on loop exit") + def test_break_in_finally_after_return(self): + # See issue #37830 + def g1(x): + for count in [0, 1]: + count2 = 0 + while count2 < 20: + count2 += 10 + try: + return count + count2 + finally: + if x: + break + return 'end', count, count2 + self.assertEqual(g1(False), 10) + self.assertEqual(g1(True), ('end', 1, 10)) + + def g2(x): + for count in [0, 1]: + for count2 in [10, 20]: + try: + return count + count2 + finally: + if x: + break + return 'end', count, count2 + self.assertEqual(g2(False), 10) + self.assertEqual(g2(True), ('end', 1, 10)) + + @skip("FIXME: currently crashes because the iterable is cleaned up on 'return', not on loop exit") + def test_continue_in_finally_after_return(self): + # See issue #37830 + def g1(x): + count = 0 + while count < 100: + count += 1 + try: + return count + finally: + if x: + continue + return 'end', count + self.assertEqual(g1(False), 1) + self.assertEqual(g1(True), ('end', 100)) + + def g2(x): + for count in [0, 1]: + try: + return count + finally: + if x: + continue + return 'end', count + self.assertEqual(g2(False), 0) + self.assertEqual(g2(True), ('end', 1)) + def test_yield(self): # Allowed as standalone statement def g(): yield 1 @@ -895,7 +1240,7 @@ class GrammarTests(unittest.TestCase): def g(): f((yield from ()), 1) # Do not require parenthesis for tuple unpacking def g(): rest = 4, 5, 6; yield 1, 2, 3, *rest - self.assertEquals(list(g()), [(1, 2, 3, 4, 5, 6)]) + self.assertEqual(list(g()), [(1, 2, 3, 4, 5, 6)]) check_syntax_error(self, "def g(): f(yield 1)") check_syntax_error(self, "def g(): f(yield 1, 1)") check_syntax_error(self, "def g(): f(yield from ())") @@ -907,23 +1252,15 @@ class GrammarTests(unittest.TestCase): check_syntax_error(self, "class foo:yield 1") check_syntax_error(self, "class foo:yield from ()") # Check annotation refleak on SyntaxError - check_syntax_error(self, "def g(a:(yield)): pass") + #check_syntax_error(self, "def g(a:(yield)): pass") # no longer a syntax error with PEP-563 - @skip("DeprecationWarning not implemented") + @skip("Not currently a syntax error") def test_yield_in_comprehensions(self): # Check yield in comprehensions def g(): [x for x in [(yield 1)]] def g(): [x for x in [(yield from ())]] - def check(code, warntext): - with self.assertWarnsRegex(DeprecationWarning, warntext): - compile(code, '<test string>', 'exec') - import warnings - with warnings.catch_warnings(): - warnings.filterwarnings('error', category=DeprecationWarning) - with self.assertRaisesRegex(SyntaxError, warntext): - compile(code, '<test string>', 'exec') - + check = self.check_syntax_error check("def g(): [(yield x) for x in ()]", "'yield' inside list comprehension") check("def g(): [x for x in () if not (yield x)]", @@ -1014,6 +1351,15 @@ class GrammarTests(unittest.TestCase): else: self.fail("AssertionError not raised by 'assert False'") + self.check_syntax_warning('assert(x, "msg")', + 'assertion is always true') + # FIXME: currently fails to compile + """ + with warnings.catch_warnings(): + warnings.simplefilter('error', SyntaxWarning) + compile('assert x, "msg"', '<testcase>', 'exec') + """ + ### compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | funcdef | classdef # Tested below @@ -1076,7 +1422,7 @@ class GrammarTests(unittest.TestCase): def test_try(self): ### try_stmt: 'try' ':' suite (except_clause ':' suite)+ ['else' ':' suite] ### | 'try' ':' suite 'finally' ':' suite - ### except_clause: 'except' [expr ['as' expr]] + ### except_clause: 'except' [expr ['as' NAME]] try: 1/0 except ZeroDivisionError: @@ -1094,6 +1440,9 @@ class GrammarTests(unittest.TestCase): except (EOFError, TypeError, ZeroDivisionError) as msg: pass try: pass finally: pass + with self.assertRaises(SyntaxError): + compile("try:\n pass\nexcept Exception as a.b:\n pass", "?", "exec") + compile("try:\n pass\nexcept Exception as a[b]:\n pass", "?", "exec") def test_suite(self): # simple_stmt | NEWLINE INDENT NEWLINE* (stmt NEWLINE*)+ DEDENT @@ -1132,11 +1481,116 @@ class GrammarTests(unittest.TestCase): if 1 > 1: pass if 1 <= 1: pass if 1 >= 1: pass - if 1 is 1: pass - if 1 is not 1: pass + if x is x: pass + if x is not x: pass if 1 in (): pass if 1 not in (): pass - if 1 < 1 > 1 == 1 >= 1 <= 1 != 1 in 1 not in 1 is 1 is not 1: pass + if 1 < 1 > 1 == 1 >= 1 <= 1 != 1 in 1 not in x is x is not x: pass + + def test_comparison_is_literal(self): + def check(test, msg='"is" with a literal'): + self.check_syntax_warning(test, msg) + + check('x is 1') + check('x is "thing"') + check('1 is x') + check('x is y is 1') + check('x is not 1', '"is not" with a literal') + + # FIXME: this fails to compile + """ + with warnings.catch_warnings(): + warnings.simplefilter('error', SyntaxWarning) + compile('x is None', '<testcase>', 'exec') + compile('x is False', '<testcase>', 'exec') + compile('x is True', '<testcase>', 'exec') + compile('x is ...', '<testcase>', 'exec') + """ + + def test_warn_missed_comma(self): + # FIXME: would be nice if this could actually raise a compile time warning as well + def check(test): + self.check_syntax_warning(test, msg) + + msg=r'is not callable; perhaps you missed a comma\?' + check('[(1, 2) (3, 4)]') + check('[(x, y) (3, 4)]') + check('[[1, 2] (3, 4)]') + check('[{1, 2} (3, 4)]') + check('[{1: 2} (3, 4)]') + check('[[i for i in range(5)] (3, 4)]') + check('[{i for i in range(5)} (3, 4)]') + check('[(i for i in range(5)) (3, 4)]') + check('[{i: i for i in range(5)} (3, 4)]') + check('[f"{x}" (3, 4)]') + check('[f"x={x}" (3, 4)]') + check('["abc" (3, 4)]') + check('[b"abc" (3, 4)]') + check('[123 (3, 4)]') + check('[12.3 (3, 4)]') + check('[12.3j (3, 4)]') + check('[None (3, 4)]') + check('[True (3, 4)]') + check('[... (3, 4)]') + + msg=r'is not subscriptable; perhaps you missed a comma\?' + check('[{1, 2} [i, j]]') + check('[{i for i in range(5)} [i, j]]') + check('[(i for i in range(5)) [i, j]]') + check('[(lambda x, y: x) [i, j]]') + check('[123 [i, j]]') + check('[12.3 [i, j]]') + check('[12.3j [i, j]]') + check('[None [i, j]]') + check('[True [i, j]]') + check('[... [i, j]]') + + msg=r'indices must be integers or slices, not tuple; perhaps you missed a comma\?' + check('[(1, 2) [i, j]]') + check('[(x, y) [i, j]]') + check('[[1, 2] [i, j]]') + check('[[i for i in range(5)] [i, j]]') + check('[f"{x}" [i, j]]') + check('[f"x={x}" [i, j]]') + check('["abc" [i, j]]') + check('[b"abc" [i, j]]') + + msg=r'indices must be integers or slices, not tuple;' + check('[[1, 2] [3, 4]]') + msg=r'indices must be integers or slices, not list;' + check('[[1, 2] [[3, 4]]]') + check('[[1, 2] [[i for i in range(5)]]]') + msg=r'indices must be integers or slices, not set;' + check('[[1, 2] [{3, 4}]]') + check('[[1, 2] [{i for i in range(5)}]]') + msg=r'indices must be integers or slices, not dict;' + check('[[1, 2] [{3: 4}]]') + check('[[1, 2] [{i: i for i in range(5)}]]') + msg=r'indices must be integers or slices, not generator;' + check('[[1, 2] [(i for i in range(5))]]') + msg=r'indices must be integers or slices, not function;' + check('[[1, 2] [(lambda x, y: x)]]') + msg=r'indices must be integers or slices, not str;' + check('[[1, 2] [f"{x}"]]') + check('[[1, 2] [f"x={x}"]]') + check('[[1, 2] ["abc"]]') + msg=r'indices must be integers or slices, not' + check('[[1, 2] [b"abc"]]') + check('[[1, 2] [12.3]]') + check('[[1, 2] [12.3j]]') + check('[[1, 2] [None]]') + check('[[1, 2] [...]]') + + """ + with warnings.catch_warnings(): + warnings.simplefilter('error', SyntaxWarning) + compile('[(lambda x, y: x) (3, 4)]', '<testcase>', 'exec') + compile('[[1, 2] [i]]', '<testcase>', 'exec') + compile('[[1, 2] [0]]', '<testcase>', 'exec') + compile('[[1, 2] [True]]', '<testcase>', 'exec') + compile('[[1, 2] [1:2]]', '<testcase>', 'exec') + compile('[{(1, 2): 3} [i, j]]', '<testcase>', 'exec') + """ def test_binary_mask_ops(self): x = 1 & 1 @@ -1244,13 +1698,30 @@ class GrammarTests(unittest.TestCase): def meth2(self, arg): pass def meth3(self, a1, a2): pass - # decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE + # decorator: '@' namedexpr_test NEWLINE # decorators: decorator+ # decorated: decorators (classdef | funcdef) def class_decorator(x): return x @class_decorator class G: pass + # Test expressions as decorators (PEP 614): + # FIXME: implement PEP 614 + """ + @False or class_decorator + class H: pass + @d := class_decorator + class I: pass + @lambda c: class_decorator(c) + class J: pass + @[..., class_decorator, ...][1] + class K: pass + @class_decorator(class_decorator)(class_decorator) + class L: pass + @[class_decorator][0].__call__.__call__ + class M: pass + """ + def test_dictcomps(self): # dictorsetmaker: ( (test ':' test (comp_for | # (',' test ':' test)* [','])) | @@ -1347,7 +1818,7 @@ class GrammarTests(unittest.TestCase): self.assertEqual(sum(b), sum([x for x in range(10)])) self.assertEqual(sum(x**2 for x in range(10)), sum([x**2 for x in range(10)])) - self.assertEqual(sum(x*x for x in range(10) if x%2), sum([x*x for x in range(10) if x%2])) + self.assertEqual(sum(x*x for x in range(10) if x % 2), sum([x*x for x in range(10) if x % 2])) self.assertEqual(sum(x for x in (y for y in range(10))), sum([x for x in range(10)])) self.assertEqual(sum(x for x in (y for y in (z for z in range(10)))), sum([x for x in range(10)])) self.assertEqual(sum(x for x in [y for y in (z for z in range(10))]), sum([x for x in range(10)])) @@ -1358,6 +1829,8 @@ class GrammarTests(unittest.TestCase): def test_comprehension_specials(self): # test for outmost iterable precomputation + # FIXME: https://github.com/cython/cython/issues/1159 + """ x = 10; g = (i for i in range(x)); x = 5 self.assertEqual(len(list(g)), 10) @@ -1365,6 +1838,7 @@ class GrammarTests(unittest.TestCase): x = 10; t = False; g = ((i,j) for i in range(x) if t for j in range(x)) x = 5; t = True; self.assertEqual([(i,j) for i in range(10) for j in range(5)], list(g)) + """ # Grammar allows multiple adjacent 'if's in listcomps and genexps, # even though it's silly. Make sure it works (ifelse broke this.) @@ -1395,11 +1869,75 @@ class GrammarTests(unittest.TestCase): with manager() as x, manager(): pass + if not use_old_parser(): + test_cases = [ + """if 1: + with ( + manager() + ): + pass + """, + """if 1: + with ( + manager() as x + ): + pass + """, + """if 1: + with ( + manager() as (x, y), + manager() as z, + ): + pass + """, + """if 1: + with ( + manager(), + manager() + ): + pass + """, + """if 1: + with ( + manager() as x, + manager() as y + ): + pass + """, + """if 1: + with ( + manager() as x, + manager() + ): + pass + """, + """if 1: + with ( + manager() as x, + manager() as y, + manager() as z, + ): + pass + """, + """if 1: + with ( + manager() as x, + manager() as y, + manager(), + ): + pass + """, + ] + for case in test_cases: + with self.subTest(case=case): + compile(case, "<string>", "exec") + + def test_if_else_expr(self): # Test ifelse expressions in various cases def _checkeval(msg, ret): "helper to check that evaluation of expressions is done correctly" - print(x) + print(msg) return ret # the next line is not allowed anymore @@ -1426,9 +1964,11 @@ class GrammarTests(unittest.TestCase): self.assertEqual(16 // (4 // 2), 8) self.assertEqual((16 // 4) // 2, 2) self.assertEqual(16 // 4 // 2, 2) - self.assertTrue(False is (2 is 3)) - self.assertFalse((False is 2) is 3) - self.assertFalse(False is 2 is 3) + x = 2 + y = 3 + self.assertTrue(False is (x is y)) + self.assertFalse((False is x) is y) + self.assertFalse(False is x is y) def test_matrix_mul(self): # This is not intended to be a comprehensive test, rather just to be few @@ -1516,54 +2056,5 @@ class GrammarTests(unittest.TestCase): foo().send(None) -### END OF COPY ### - -GrammarTests.assertRaisesRegex = lambda self, exc, msg: self.assertRaises(exc) - -if sys.version_info < (2, 7): - def assertRaises(self, exc_type, func=None, *args, **kwargs): - if func is not None: - return unittest.TestCase.assertRaises(self, exc_type, func, *args, **kwargs) - @contextlib.contextmanager - def assertRaisesCM(): - class Result(object): - exception = exc_type("unexpected EOF") # see usage above - try: - yield Result() - except exc_type: - self.assertTrue(True) - else: - self.assertTrue(False) - return assertRaisesCM() - GrammarTests.assertRaises = assertRaises - TokenTests.assertRaises = assertRaises - - -if not hasattr(unittest.TestCase, 'subTest'): - @contextlib.contextmanager - def subTest(self, source, **kwargs): - try: - yield - except Exception: - print(source) - raise - GrammarTests.subTest = subTest - - -if not hasattr(unittest.TestCase, 'assertIn'): - def assertIn(self, member, container, msg=None): - self.assertTrue(member in container, msg) - TokenTests.assertIn = assertIn - - -# FIXME: disabling some tests for real Cython bugs here -del GrammarTests.test_comprehension_specials # iterable pre-calculation in generator expression -del GrammarTests.test_funcdef # annotation mangling - -# this test is difficult to enable in Py2.6 -if sys.version_info < (2,7): - del GrammarTests.test_former_statements_refer_to_builtins - - if __name__ == '__main__': unittest.main() diff --git a/tests/run/test_subclassinit.py b/tests/run/test_subclassinit.py new file mode 100644 index 000000000..7661c6062 --- /dev/null +++ b/tests/run/test_subclassinit.py @@ -0,0 +1,316 @@ +# mode: run +# tag: pure3.6 +# cython: language_level=3str + +import sys +HAS_NATIVE_SUPPORT = sys.version_info >= (3, 6) +IS_PY2 = sys.version_info[0] == 2 + +import re +import types +import unittest + +ZERO = 0 + +skip_if_not_native = unittest.skipIf(not HAS_NATIVE_SUPPORT, "currently requires Python 3.6+") + + +class Test(unittest.TestCase): + if not hasattr(unittest.TestCase, 'assertRegex'): + def assertRegex(self, value, regex): + self.assertTrue(re.search(regex, str(value)), + "'%s' did not match '%s'" % (value, regex)) + + if not hasattr(unittest.TestCase, 'assertCountEqual'): + def assertCountEqual(self, first, second): + self.assertEqual(set(first), set(second)) + self.assertEqual(len(first), len(second)) + + def test_init_subclass(self): + class A: + initialized = False + + def __init_subclass__(cls): + if HAS_NATIVE_SUPPORT: + super().__init_subclass__() + cls.initialized = True + + class B(A): + pass + + self.assertFalse(A.initialized) + self.assertTrue(B.initialized) + + def test_init_subclass_dict(self): + class A(dict): + initialized = False + + def __init_subclass__(cls): + if HAS_NATIVE_SUPPORT: + super().__init_subclass__() + cls.initialized = True + + class B(A): + pass + + self.assertFalse(A.initialized) + self.assertTrue(B.initialized) + + def test_init_subclass_kwargs(self): + class A: + def __init_subclass__(cls, **kwargs): + cls.kwargs = kwargs + + class B(A, x=3): + pass + + self.assertEqual(B.kwargs, dict(x=3)) + + def test_init_subclass_error(self): + class A: + def __init_subclass__(cls): + raise RuntimeError + + with self.assertRaises(RuntimeError): + class B(A): + pass + + def test_init_subclass_wrong(self): + class A: + def __init_subclass__(cls, whatever): + pass + + with self.assertRaises(TypeError): + class B(A): + pass + + def test_init_subclass_skipped(self): + class BaseWithInit: + def __init_subclass__(cls, **kwargs): + if HAS_NATIVE_SUPPORT: + super().__init_subclass__(**kwargs) + cls.initialized = cls + + class BaseWithoutInit(BaseWithInit): + pass + + class A(BaseWithoutInit): + pass + + self.assertIs(A.initialized, A) + self.assertIs(BaseWithoutInit.initialized, BaseWithoutInit) + + def test_init_subclass_diamond(self): + class Base: + def __init_subclass__(cls, **kwargs): + if HAS_NATIVE_SUPPORT: + super().__init_subclass__(**kwargs) + cls.calls = [] + + class Left(Base): + pass + + class Middle: + def __init_subclass__(cls, middle, **kwargs): + super().__init_subclass__(**kwargs) + cls.calls += [middle] + + class Right(Base): + def __init_subclass__(cls, right="right", **kwargs): + super().__init_subclass__(**kwargs) + cls.calls += [right] + + class A(Left, Middle, Right, middle="middle"): + pass + + self.assertEqual(A.calls, ["right", "middle"]) + self.assertEqual(Left.calls, []) + self.assertEqual(Right.calls, []) + + def test_set_name(self): + class Descriptor: + def __set_name__(self, owner, name): + self.owner = owner + self.name = name + + class A: + d = Descriptor() + + self.assertEqual(A.d.name, "d") + self.assertIs(A.d.owner, A) + + @skip_if_not_native + def test_set_name_metaclass(self): + class Meta(type): + def __new__(cls, name, bases, ns): + ret = super().__new__(cls, name, bases, ns) + self.assertEqual(ret.d.name, "d") + self.assertIs(ret.d.owner, ret) + return 0 + + class Descriptor: + def __set_name__(self, owner, name): + self.owner = owner + self.name = name + + class A(metaclass=Meta): + d = Descriptor() + self.assertEqual(A, 0) + + def test_set_name_error(self): + class Descriptor: + def __set_name__(self, owner, name): + 1 / ZERO + + with self.assertRaises(RuntimeError) as cm: + class NotGoingToWork: + attr = Descriptor() + + exc = cm.exception + self.assertRegex(str(exc), r'\bNotGoingToWork\b') + self.assertRegex(str(exc), r'\battr\b') + self.assertRegex(str(exc), r'\bDescriptor\b') + if HAS_NATIVE_SUPPORT: + self.assertIsInstance(exc.__cause__, ZeroDivisionError) + + def test_set_name_wrong(self): + class Descriptor: + def __set_name__(self): + pass + + with self.assertRaises(RuntimeError) as cm: + class NotGoingToWork: + attr = Descriptor() + + exc = cm.exception + self.assertRegex(str(exc), r'\bNotGoingToWork\b') + self.assertRegex(str(exc), r'\battr\b') + self.assertRegex(str(exc), r'\bDescriptor\b') + if HAS_NATIVE_SUPPORT: + self.assertIsInstance(exc.__cause__, TypeError) + + def test_set_name_lookup(self): + resolved = [] + class NonDescriptor: + def __getattr__(self, name): + resolved.append(name) + + class A: + d = NonDescriptor() + + self.assertNotIn('__set_name__', resolved, + '__set_name__ is looked up in instance dict') + + @skip_if_not_native + def test_set_name_init_subclass(self): + class Descriptor: + def __set_name__(self, owner, name): + self.owner = owner + self.name = name + + class Meta(type): + def __new__(cls, name, bases, ns): + self = super().__new__(cls, name, bases, ns) + self.meta_owner = self.owner + self.meta_name = self.name + return self + + class A: + def __init_subclass__(cls): + cls.owner = cls.d.owner + cls.name = cls.d.name + + class B(A, metaclass=Meta): + d = Descriptor() + + self.assertIs(B.owner, B) + self.assertEqual(B.name, 'd') + self.assertIs(B.meta_owner, B) + self.assertEqual(B.name, 'd') + + def test_set_name_modifying_dict(self): + notified = [] + class Descriptor: + def __set_name__(self, owner, name): + setattr(owner, name + 'x', None) + notified.append(name) + + class A: + a = Descriptor() + b = Descriptor() + c = Descriptor() + d = Descriptor() + e = Descriptor() + + self.assertCountEqual(notified, ['a', 'b', 'c', 'd', 'e']) + + def test_errors(self): + class MyMeta(type): + pass + + with self.assertRaises(TypeError): + class MyClass(metaclass=MyMeta, otherarg=1): + pass + + if not IS_PY2: + with self.assertRaises(TypeError): + types.new_class("MyClass", (object,), + dict(metaclass=MyMeta, otherarg=1)) + types.prepare_class("MyClass", (object,), + dict(metaclass=MyMeta, otherarg=1)) + + class MyMeta(type): + def __init__(self, name, bases, namespace, otherarg): + super().__init__(name, bases, namespace) + + with self.assertRaises(TypeError): + class MyClass(metaclass=MyMeta, otherarg=1): + pass + + class MyMeta(type): + def __new__(cls, name, bases, namespace, otherarg): + return super().__new__(cls, name, bases, namespace) + + def __init__(self, name, bases, namespace, otherarg): + super().__init__(name, bases, namespace) + self.otherarg = otherarg + + class MyClass(metaclass=MyMeta, otherarg=1): + pass + + self.assertEqual(MyClass.otherarg, 1) + + @skip_if_not_native + def test_errors_changed_pep487(self): + # These tests failed before Python 3.6, PEP 487 + class MyMeta(type): + def __new__(cls, name, bases, namespace): + return super().__new__(cls, name=name, bases=bases, + dict=namespace) + + with self.assertRaises(TypeError): + class MyClass(metaclass=MyMeta): + pass + + class MyMeta(type): + def __new__(cls, name, bases, namespace, otherarg): + self = super().__new__(cls, name, bases, namespace) + self.otherarg = otherarg + return self + + class MyClass(metaclass=MyMeta, otherarg=1): + pass + + self.assertEqual(MyClass.otherarg, 1) + + def test_type(self): + t = type('NewClass', (object,), {}) + self.assertIsInstance(t, type) + self.assertEqual(t.__name__, 'NewClass') + + with self.assertRaises(TypeError): + type(name='NewClass', bases=(object,), dict={}) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/run/test_unicode.pyx b/tests/run/test_unicode.pyx index 4616acf2c..2386b5124 100644 --- a/tests/run/test_unicode.pyx +++ b/tests/run/test_unicode.pyx @@ -846,7 +846,6 @@ class UnicodeTest(CommonTest, self.assertEqual('finnish'.capitalize(), 'FInnish') else: self.assertEqual('finnish'.capitalize(), 'Finnish') - self.assertEqual('A\u0345\u03a3'.capitalize(), 'A\u0345\u03c2') def test_title(self): diff --git a/tests/run/time_pxd.pyx b/tests/run/time_pxd.pyx new file mode 100644 index 000000000..fe252895f --- /dev/null +++ b/tests/run/time_pxd.pyx @@ -0,0 +1,64 @@ +# mode: run +# tag: py3only,pytime + +import time + +from cpython cimport time as ctime + + +def test_time(): + """ + >>> tic1, tic2, tic3 = test_time() + >>> tic1 <= tic3 # sanity check + True + >>> tic1 <= tic2 + True + >>> tic2 <= tic3 + True + """ + # check that ctime.time() matches time.time() to within call-time tolerance + tic1 = time.time() + tic2 = ctime.time() + tic3 = time.time() + + return tic1, tic2, tic3 + + +def test_localtime(): + """ + >>> ltp, ltc = test_localtime() + >>> ltp.tm_year == ltc['tm_year'] or (ltp.tm_year, ltc['tm_year']) + True + >>> ltp.tm_mon == ltc['tm_mon'] or (ltp.tm_mon, ltc['tm_mon']) + True + >>> ltp.tm_mday == ltc['tm_mday'] or (ltp.tm_mday, ltc['tm_mday']) + True + >>> ltp.tm_hour == ltc['tm_hour'] or (ltp.tm_hour, ltc['tm_hour']) + True + >>> ltp.tm_min == ltc['tm_min'] or (ltp.tm_min, ltc['tm_min']) + True + >>> ltp.tm_sec == ltc['tm_sec'] or (ltp.tm_sec, ltc['tm_sec']) + True + >>> ltp.tm_wday == ltc['tm_wday'] or (ltp.tm_wday, ltc['tm_wday']) + True + >>> ltp.tm_yday == ltc['tm_yday'] or (ltp.tm_yday, ltc['tm_yday']) + True + >>> ltp.tm_isdst == ltc['tm_isdst'] or (ltp.tm_isdst, ltc['tm_isdst']) + True + """ + ltp = time.localtime() + ltc = ctime.localtime() + + i = 0 + while ltp.tm_sec != ltc.tm_sec: + # If the time.localtime call is just before the end of a second and the + # ctime.localtime call is just after the beginning of the next second, + # re-call. This should not occur twice in a row. + time.sleep(0.1) + ltp = time.localtime() + ltc = ctime.localtime() + i += 1 + if i > 10: + break + + return ltp, ltc diff --git a/tests/run/tp_new.pyx b/tests/run/tp_new.pyx index c25a84289..45b52259f 100644 --- a/tests/run/tp_new.pyx +++ b/tests/run/tp_new.pyx @@ -1,4 +1,6 @@ -# ticket: 808 +# mode: run +# tag: exttype, tpnew +# ticket: t808 cimport cython diff --git a/tests/run/tp_new_T454.pyx b/tests/run/tp_new_T454.pyx index 893ef9b1b..d3401442c 100644 --- a/tests/run/tp_new_T454.pyx +++ b/tests/run/tp_new_T454.pyx @@ -1,4 +1,4 @@ -# ticket: 454 +# ticket: t454 cimport cython diff --git a/tests/run/trashcan.pyx b/tests/run/trashcan.pyx new file mode 100644 index 000000000..29331fdf6 --- /dev/null +++ b/tests/run/trashcan.pyx @@ -0,0 +1,148 @@ +# mode: run + +cimport cython + + +# Count number of times an object was deallocated twice. This should remain 0. +cdef int double_deallocations = 0 +def assert_no_double_deallocations(): + global double_deallocations + err = double_deallocations + double_deallocations = 0 + assert not err + + +# Compute x = f(f(f(...(None)...))) nested n times and throw away the result. +# The real test happens when exiting this function: then a big recursive +# deallocation of x happens. We are testing two things in the tests below: +# that Python does not crash and that no double deallocation happens. +# See also https://github.com/python/cpython/pull/11841 +def recursion_test(f, int n=2**20): + x = None + cdef int i + for i in range(n): + x = f(x) + + +@cython.trashcan(True) +cdef class Recurse: + """ + >>> recursion_test(Recurse) + >>> assert_no_double_deallocations() + """ + cdef public attr + cdef int deallocated + + def __cinit__(self, x): + self.attr = x + + def __dealloc__(self): + # Check that we're not being deallocated twice + global double_deallocations + double_deallocations += self.deallocated + self.deallocated = 1 + + +cdef class RecurseSub(Recurse): + """ + >>> recursion_test(RecurseSub) + >>> assert_no_double_deallocations() + """ + cdef int subdeallocated + + def __dealloc__(self): + # Check that we're not being deallocated twice + global double_deallocations + double_deallocations += self.subdeallocated + self.subdeallocated = 1 + + +@cython.freelist(4) +@cython.trashcan(True) +cdef class RecurseFreelist: + """ + >>> recursion_test(RecurseFreelist) + >>> recursion_test(RecurseFreelist, 1000) + >>> assert_no_double_deallocations() + """ + cdef public attr + cdef int deallocated + + def __cinit__(self, x): + self.attr = x + + def __dealloc__(self): + # Check that we're not being deallocated twice + global double_deallocations + double_deallocations += self.deallocated + self.deallocated = 1 + + +# Subclass of list => uses trashcan by default +# As long as https://github.com/python/cpython/pull/11841 is not fixed, +# this does lead to double deallocations, so we skip that check. +cdef class RecurseList(list): + """ + >>> RecurseList(42) + [42] + >>> recursion_test(RecurseList) + """ + def __init__(self, x): + super().__init__((x,)) + + +# Some tests where the trashcan is NOT used. When the trashcan is not used +# in a big recursive deallocation, the __dealloc__s of the base classes are +# only run after the __dealloc__s of the subclasses. +# We use this to detect trashcan usage. +cdef int base_deallocated = 0 +cdef int trashcan_used = 0 +def assert_no_trashcan_used(): + global base_deallocated, trashcan_used + err = trashcan_used + trashcan_used = base_deallocated = 0 + assert not err + + +cdef class Base: + def __dealloc__(self): + global base_deallocated + base_deallocated = 1 + + +# Trashcan disabled by default +cdef class Sub1(Base): + """ + >>> recursion_test(Sub1, 100) + >>> assert_no_trashcan_used() + """ + cdef public attr + + def __cinit__(self, x): + self.attr = x + + def __dealloc__(self): + global base_deallocated, trashcan_used + trashcan_used += base_deallocated + + +@cython.trashcan(True) +cdef class Middle(Base): + cdef public foo + + +# Trashcan disabled explicitly +@cython.trashcan(False) +cdef class Sub2(Middle): + """ + >>> recursion_test(Sub2, 1000) + >>> assert_no_trashcan_used() + """ + cdef public attr + + def __cinit__(self, x): + self.attr = x + + def __dealloc__(self): + global base_deallocated, trashcan_used + trashcan_used += base_deallocated diff --git a/tests/run/tuple.pyx b/tests/run/tuple.pyx index 797fb43dc..5e0dbe784 100644 --- a/tests/run/tuple.pyx +++ b/tests/run/tuple.pyx @@ -117,9 +117,9 @@ def tuple_of_args_tuple(*args): @cython.test_fail_if_path_exists('//SimpleCallNode//SimpleCallNode') def tuple_of_object(ob): """ - >>> tuple(type(1)) + >>> tuple(type(1)) # doctest: +ELLIPSIS Traceback (most recent call last): - TypeError: 'type' object is not iterable + TypeError: ...type... >>> sorted(tuple(set([1, 2, 3]))) [1, 2, 3] """ diff --git a/tests/run/tuple_constants.pyx b/tests/run/tuple_constants.pyx index 55992d933..f60d5d818 100644 --- a/tests/run/tuple_constants.pyx +++ b/tests/run/tuple_constants.pyx @@ -2,6 +2,9 @@ cimport cython module_level_tuple = (1,2,3) +second_module_level_tuple = (1,2,3) # should be deduplicated to be the same as the first +string_module_level_tuple = ("1", "2") +string_module_level_tuple2 = ("1", "2") def return_module_level_tuple(): """ @@ -10,6 +13,32 @@ def return_module_level_tuple(): """ return module_level_tuple +def test_deduplicated_tuples(): + """ + >>> test_deduplicated_tuples() + """ + assert (module_level_tuple is second_module_level_tuple) + assert (module_level_tuple is (1,2,3)) # also deduplicated with a function tuple + assert (string_module_level_tuple is string_module_level_tuple2) + assert (string_module_level_tuple is ("1", "2")) + +def func1(arg1, arg2): + pass + +def func2(arg1, arg2): + pass + +def test_deduplicated_args(): + """ + >>> test_deduplicated_args() + """ + # This is a concern because in large modules *a lot* of similar code objects + # are generated often with the same argument names. Therefore it's worth ensuring that + # they are correctly deduplicated + import sys + if not hasattr(sys, "pypy_version_info"): # test doesn't work on PyPy (which is probably fair enough) + assert func1.__code__.co_varnames is func2.__code__.co_varnames + @cython.test_assert_path_exists("//TupleNode", "//TupleNode[@is_literal = true]") @cython.test_fail_if_path_exists("//TupleNode[@is_literal = false]") diff --git a/tests/run/tupleunpack_T298.pyx b/tests/run/tupleunpack_T298.pyx index 7763b1fc7..7e3150243 100644 --- a/tests/run/tupleunpack_T298.pyx +++ b/tests/run/tupleunpack_T298.pyx @@ -1,4 +1,4 @@ -# ticket: 298 +# ticket: t298 """ >>> func() diff --git a/tests/run/tupleunpack_T712.pyx b/tests/run/tupleunpack_T712.pyx index 24d750849..73b3c1446 100644 --- a/tests/run/tupleunpack_T712.pyx +++ b/tests/run/tupleunpack_T712.pyx @@ -1,5 +1,5 @@ # mode: run -# ticket: 712 +# ticket: t712 def single_from_string(): """ diff --git a/tests/run/type_inference.pyx b/tests/run/type_inference.pyx index d2bef6431..df77f6bd9 100644 --- a/tests/run/type_inference.pyx +++ b/tests/run/type_inference.pyx @@ -276,6 +276,9 @@ def cascade(): assert typeof(e) == "double" def cascaded_assignment(): + """ + >>> cascaded_assignment() + """ a = b = c = d = 1.0 assert typeof(a) == "double" assert typeof(b) == "double" @@ -284,6 +287,36 @@ def cascaded_assignment(): e = a + b + c + d assert typeof(e) == "double" + +def unpacking(x): + """ + >>> unpacking(0) + """ + a, b, c, (d, e) = x, 1, 2.0, [3, [5, 6]] + assert typeof(a) == "Python object", typeof(a) + assert typeof(b) == "long", typeof(b) + assert typeof(c) == "double", typeof(c) + assert typeof(d) == "long", typeof(d) + assert typeof(e) == "list object", typeof(e) + + +def star_unpacking(*x): + """ + >>> star_unpacking(1, 2) + """ + a, b = x + c, *d = x + *e, f = x + *g, g = x # re-assignment + assert typeof(a) == "Python object", typeof(a) + assert typeof(b) == "Python object", typeof(b) + assert typeof(c) == "Python object", typeof(c) + assert typeof(d) == "list object", typeof(d) + assert typeof(e) == "list object", typeof(e) + assert typeof(f) == "Python object", typeof(f) + assert typeof(g) == "Python object", typeof(f) + + def increment(): """ >>> increment() @@ -756,3 +789,16 @@ def test_bound_methods(): default_arg = o.default_arg assert default_arg(2) == 12, default_arg(2) assert default_arg(2, 3) == 15, default_arg(2, 2) + +def test_builtin_max(): + """ + # builtin max is slightly complicated because it gets transformed to EvalWithTempExprNode + # See https://github.com/cython/cython/issues/4155 + >>> test_builtin_max() + """ + class C: + a = 2 + def get_max(self): + a = max(self.a, self.a) + assert typeof(a) == "Python object", typeof(a) + C().get_max() diff --git a/tests/run/type_inference_T768.pyx b/tests/run/type_inference_T768.pyx index cb7b9eccb..8f1f2e61e 100644 --- a/tests/run/type_inference_T768.pyx +++ b/tests/run/type_inference_T768.pyx @@ -1,5 +1,5 @@ # mode: run -# ticket: 768 +# ticket: t768 from cython cimport typeof def type_inference_del_int(): diff --git a/tests/run/type_inference_T768_cpp.pyx b/tests/run/type_inference_T768_cpp.pyx index 56fad4dab..295351914 100644 --- a/tests/run/type_inference_T768_cpp.pyx +++ b/tests/run/type_inference_T768_cpp.pyx @@ -1,6 +1,6 @@ # mode: run # tag: cpp -# ticket: 768 +# ticket: t768 from cython cimport typeof cdef extern from "shapes.h" namespace "shapes": diff --git a/tests/run/type_slots_int_long_T287.pyx b/tests/run/type_slots_int_long_T287.pyx index 2426225b5..7b66125b9 100644 --- a/tests/run/type_slots_int_long_T287.pyx +++ b/tests/run/type_slots_int_long_T287.pyx @@ -1,4 +1,4 @@ -# ticket: 287 +# ticket: t287 __doc__ = u""" >>> print( "%d" % Int() ) diff --git a/tests/run/typeddefaultargT373.pyx b/tests/run/typeddefaultargT373.pyx index 98ac85c3a..d817c97c7 100644 --- a/tests/run/typeddefaultargT373.pyx +++ b/tests/run/typeddefaultargT373.pyx @@ -1,4 +1,4 @@ -# ticket: 373 +# ticket: t373 import math diff --git a/tests/run/typedfieldbug_T303.pyx b/tests/run/typedfieldbug_T303.pyx index 4298a1cce..e1790c574 100644 --- a/tests/run/typedfieldbug_T303.pyx +++ b/tests/run/typedfieldbug_T303.pyx @@ -1,5 +1,5 @@ # mode: run -# ticket: 303 +# ticket: t303 __doc__ = """ >>> try: readonly() diff --git a/tests/run/typetest_T417.pyx b/tests/run/typetest_T417.pyx index 7d7b24902..622b35416 100644 --- a/tests/run/typetest_T417.pyx +++ b/tests/run/typetest_T417.pyx @@ -1,4 +1,4 @@ -# ticket: 417 +# ticket: t417 #cython: autotestdict=True cdef class Foo: diff --git a/tests/run/unbound_special_methods.pyx b/tests/run/unbound_special_methods.pyx index e16c7b2d5..1d3a75450 100644 --- a/tests/run/unbound_special_methods.pyx +++ b/tests/run/unbound_special_methods.pyx @@ -14,11 +14,11 @@ text = u'ab jd sdflk as sa sadas asdas fsdf ' "//AttributeNode[@entry.cname = 'PyUnicode_Contains']") def unicode_contains(unicode s, substring): """ - >>> unicode_contains(text, 'fl') + >>> unicode_contains(text, u'fl') True - >>> unicode_contains(text, 'XYZ') + >>> unicode_contains(text, u'XYZ') False - >>> unicode_contains(None, 'XYZ') + >>> unicode_contains(None, u'XYZ') Traceback (most recent call last): AttributeError: 'NoneType' object has no attribute '__contains__' """ @@ -32,11 +32,11 @@ def unicode_contains(unicode s, substring): "//NameNode[@entry.cname = 'PyUnicode_Contains']") def unicode_contains_unbound(unicode s, substring): """ - >>> unicode_contains_unbound(text, 'fl') + >>> unicode_contains_unbound(text, u'fl') True - >>> unicode_contains_unbound(text, 'XYZ') + >>> unicode_contains_unbound(text, u'XYZ') False - >>> unicode_contains_unbound(None, 'XYZ') # doctest: +ELLIPSIS + >>> unicode_contains_unbound(None, u'XYZ') # doctest: +ELLIPSIS Traceback (most recent call last): TypeError: descriptor '__contains__' requires a '...' object but received a 'NoneType' """ @@ -46,17 +46,17 @@ def unicode_contains_unbound(unicode s, substring): cdef class UnicodeSubclass(unicode): """ >>> u = UnicodeSubclass(text) - >>> 'fl' in u + >>> u'fl' in u False - >>> 'XYZ' in u + >>> u'XYZ' in u True - >>> u.method('fl') + >>> u.method(u'fl') False - >>> u.method('XYZ') + >>> u.method(u'XYZ') True - >>> u.operator('fl') + >>> u.operator(u'fl') False - >>> u.operator('XYZ') + >>> u.operator(u'XYZ') True """ def __contains__(self, substring): diff --git a/tests/run/unicode_formatting.pyx b/tests/run/unicode_formatting.pyx index fa51f8b94..cddcdf136 100644 --- a/tests/run/unicode_formatting.pyx +++ b/tests/run/unicode_formatting.pyx @@ -38,21 +38,21 @@ def mix_format(a, int b, list c): class PySubtype(unicode): def __rmod__(self, other): - return f'PyRMOD({other})' + return f'PyRMOD({self}, {other})' cdef class ExtSubtype(unicode): - def __mod__(one, other): - return f'ExtMOD({one}, {other})' + def __rmod__(self, other): + return f'ExtRMOD({self}, {other})' def subtypes(): """ >>> py, ext = subtypes() >>> print(py) - PyRMOD(-%s-) + PyRMOD(PySub, -%s-) >>> print(ext) - ExtMOD(-%s-, ExtSub) + ExtRMOD(ExtSub, -%s-) """ return [ '-%s-' % PySubtype("PySub"), diff --git a/tests/run/unicode_identifiers.pxd b/tests/run/unicode_identifiers.pxd new file mode 100644 index 000000000..06469e6c7 --- /dev/null +++ b/tests/run/unicode_identifiers.pxd @@ -0,0 +1,10 @@ +# -*- coding: utf-8 -*- +# cython: language_level=3 + +cdef Fα1() +cdef class Γναμε2: + cdef public int α + cdef boring_cdef(self) + cdef εxciting_cdef(self) + cpdef boring_cpdef(self) + cpdef εxciting_cpdef(self) diff --git a/tests/run/unicode_identifiers.pyx b/tests/run/unicode_identifiers.pyx new file mode 100644 index 000000000..767e3283e --- /dev/null +++ b/tests/run/unicode_identifiers.pyx @@ -0,0 +1,243 @@ +# -*- coding: utf-8 -*- +# mode: run +# tag: pep3131, traceback + +# cython: language_level=3 + +# Code with unicode identifiers can be compiled with Cython running either Python 2 or 3. +# However Python access to unicode identifiers is only possible in Python 3. In Python 2 +# it's only really safe to use the unicode identifiers for purely Cython interfaces +# (although this isn't enforced...). Therefore the majority of the doctests are +# Python3 only and only a limited set are run in Python2. +# This is controlled by putting the Python3 only tests in the module __doc__ attribute +# Most of the individual function and class docstrings are only present as a compile test + +cimport cython + +import sys + + +if sys.version_info[0] > 2: + __doc__ = u""" + >>> f()() + 2 + >>> f().__name__ + 'nεsted' + + The test is mainly to see if the traceback is generated correctly + >>> print_traceback_name() + unicode_identifiers.Fα1 + + Just check that a cpdef function is callable + >>> Fα3() + 1 + + >>> Γναμε2.ναμε3 + 1 + >>> x = Γναμε2() + >>> print(x.α) + 100 + >>> x.α = 200 + >>> print(x.α) + 200 + + >>> B().Ƒ() + >>> C().Ƒ() + + Test generation of locals() + >>> sorted(Γναμε2().boring_function(1,2).keys()) + ['self', 'somevalue', 'x', 'ναμε5', 'ναμε6'] + + >>> Γναμε2().boring_cpdef() - Γναμε2().εxciting_cpdef() + 0 + >>> function_taking_fancy_argument(Γναμε2()).ναμε3 + 1 + >>> metho_function_taking_fancy_argument(Γναμε2()).ναμε3 + 1 + >>> NormalClassΓΓ().ναμε + 10 + >>> NormalClassΓΓ().εxciting_function(None).__qualname__ + 'NormalClassΓΓ.εxciting_function.<locals>.nestεd' + + Do kwargs work? + >>> unicode_kwarg(αrγ=5) + 5 + >>> unicode_kwarg_from_cy() + 1 + + Normalization of attributes + (The cdef class version is testable in Python 2 too) + >>> NormalizeAttrPy().get() + 5 + """ +else: + __doc__ = "" + +global_ναμε1 = None +cdef double global_ναμε2 = 1.2 + +def f(): + """docstring""" + ναμε2 = 2 + def nεsted(): + return ναμε2 + return nεsted + +# Ƒ is notably awkward because its punycode starts with "2" causing +# C compile errors. Therefore try a few different variations... +cdef class A: + cdef int ναμε + def __init__(self): + self.ναμε = 1 + cdef Ƒ(self): + return self.ναμε == 1 + def regular_function(self): + """ + Can use unicode cdef functions and (private) attributes internally + >>> A().regular_function() + True + """ + return self.Ƒ() +cdef class B: + cpdef Ƒ(self): + pass +cdef class C: + def Ƒ(self): + pass +cdef class D: + cdef int Ƒ + +def regular_function(): + """ + Unicode names can be used internally on python2 + >>> regular_function() + 10 + """ + cdef int variableƑ = 5 + ναμε2 = 2 + return variableƑ*ναμε2 + +cdef Fα1(): + """docstring""" + ναμε2 = 2 + raise RuntimeError() # forces generation of a traceback + +def print_traceback_name(): + try: + Fα1() + except RuntimeError as e: + import traceback + # get the name of one level up in the traceback + print(traceback.extract_tb(e.__traceback__,2)[1][2]) + + +def Fα2(): + """docstring""" + def nested_normal(): + """docstring""" + pass + def nεstεd_uni(): + """docstring""" + pass + return nested_normal, nεstεd_uni + +cpdef Fα3(): + """docstring""" + return 1 + +cdef class Γναμε2: + """ + docstring + """ + ναμε3 = 1 + + def __init__(self): + self.α = 100 + def boring_function(self,x,ναμε5): + """docstring""" + ναμε6 = ναμε5 + somevalue = global_ναμε1 == self.ναμε3 + return locals() + def εxciting_function(self,y): + """docstring""" + def nestεd(): + pass + return nestεd + + cdef boring_cdef(self): + """docstring""" + pass + cdef εxciting_cdef(self): + """docstring""" + pass + + cpdef boring_cpdef(self): + """docstring""" + return 2 + cpdef εxciting_cpdef(self): + """docstring""" + return 2 + +cdef class Derived(Γναμε2): + pass + +cdef Γναμε2 global_ναμε3 = Γναμε2() + + +@cython.always_allow_keywords(False) # METH_O signature +def metho_function_taking_fancy_argument(Γναμε2 αrγ): + return αrγ + +@cython.always_allow_keywords(True) +def function_taking_fancy_argument(Γναμε2 αrγ): + return αrγ + + +class NormalClassΓΓ(Γναμε2): + """ + docstring + """ + def __init__(self): + self.ναμε = 10 + + def boring_function(self,x,ναμε5): + """docstring""" + ναμε6 = ναμε5 + somevalue = global_ναμε1 == self.ναμε3 + return locals() + def εxciting_function(self,y): + """docstring""" + def nestεd(): + pass + return nestεd + +def unicode_kwarg(*, αrγ): + return αrγ + +def unicode_kwarg_from_cy(): + return unicode_kwarg(αrγ=1) + +class NormalizeAttrPy: + """Python normalizes identifier names before they are used; + therefore fi and fi should access the same attribute""" + def __init__(self): + self.fi = 5 # note unicode ligature symbol + def get(self): + return self.fi + +cdef class NormalizeAttrCdef: + """Python normalizes identifier names before they are used; + therefore fi and fi should access the same attribute + >>> NormalizeAttrCdef().get() + 5 + """ + cdef int fi # note unicode ligature symbol + def __init__(self): + self.fi = 5 + def get(self): + return self.fi + +if sys.version_info[0]<=2: + # These symbols are causing problems for doctest + del NormalClassΓΓ + del globals()[u'Γναμε2'.encode('utf-8')] diff --git a/tests/run/unicode_identifiers_import.pyx b/tests/run/unicode_identifiers_import.pyx new file mode 100644 index 000000000..17e091697 --- /dev/null +++ b/tests/run/unicode_identifiers_import.pyx @@ -0,0 +1,31 @@ +# -*- coding: utf-8 -*- +# cython: language_level = 3 +# mode: compile +# tag: pep3131 + +# compile only test since there's no way to get +# it to import another test module at runtime + +# this test looks at [c]importing unicode stuff +from unicode_identifiers cimport Fα1, Γναμε2 +cimport unicode_identifiers +from unicode_identifiers cimport Γναμε2 as Γναμε3 + +from unicode_identifiers import NormalClassΓΓ +from unicode_identifiers import NormalClassΓΓ as NörmalCläss + + +cdef class C(unicode_identifiers.Γναμε2): + pass + +cdef class D(Γναμε2): + pass + +cdef class E(Γναμε3): + pass + +def f(): + Fα1() + unicode_identifiers.Fα1() + + diff --git a/tests/run/unicode_identifiers_normalization.srctree b/tests/run/unicode_identifiers_normalization.srctree new file mode 100644 index 000000000..764384455 --- /dev/null +++ b/tests/run/unicode_identifiers_normalization.srctree @@ -0,0 +1,83 @@ +# -*- coding: utf-8 -*- +# mode: run +# tag: pure3.0, pep3131 + +PYTHON build_tests.py +# show behaviour in Python mode +PYTHON -m doctest test0.py +PYTHON -m doctest test1.py +PYTHON -m doctest test2.py + +PYTHON setup.py build_ext --inplace +# test in Cython mode +PYTHON -c "import doctest; import test0 as m; exit(doctest.testmod(m)[0])" +PYTHON -c "import doctest; import test1 as m; exit(doctest.testmod(m)[0])" +PYTHON -c "import doctest; import test2 as m; exit(doctest.testmod(m)[0])" + +########## setup.py ######### + +from Cython.Build.Dependencies import cythonize +from distutils.core import setup + +setup( + ext_modules = cythonize("test*.py"), +) + +######### build_tests.py ######## +# -*- coding: utf-8 -*- +from __future__ import unicode_literals +import sys +import unicodedata + +# a few pairs of unicode strings that should be equivalent after normalization +string_pairs = [("fi", "fi"), # ligature and two letters + ("a\u0301", '\u00e1'), # a with acute accent with combining character or as 1 character + ("α\u0334\u0362", "α\u0362\u0334") # alpha with a pair of combining characters + # in a different order. No single character to normalize to + ] + +# Show that the pairs genuinely aren't equal before normalization +for sp in string_pairs: + assert sp[0] != sp[1] + assert unicodedata.normalize('NFKC', sp[0]) == unicodedata.normalize('NFKC', sp[1]) + +# some code that accesses the identifiers through the two different names +# contains doctests +example_code = [ +""" +class C: + ''' + >>> C().get() + True + ''' + def __init__(self): + self.{0} = True + def get(self): + return self.{1} +""", """ +def pass_through({0}): + ''' + >>> pass_through(True) + True + ''' + return {1} +""", """ +import cython +{0} = True +def test(): + ''' + >>> test() + True + ''' + return {1} +"""] + +from io import open + +for idx, (code, strings) in enumerate(zip(example_code, string_pairs)): + with open("test{0}.py".format(idx), "w", encoding="utf8") as f: + code = code.format(*strings) + f.write("# -*- coding: utf-8 -*-\n") + # The code isn't Py2 compatible. Only write actual code in Py3+. + if sys.version_info[0] > 2: + f.write(code) diff --git a/tests/run/unicode_imports.srctree b/tests/run/unicode_imports.srctree new file mode 100644 index 000000000..262f4949a --- /dev/null +++ b/tests/run/unicode_imports.srctree @@ -0,0 +1,105 @@ +# -*- coding: utf-8 -*- +# tag: py3, pep489 + +PYTHON setup.py build_ext --inplace +PYTHON -m mydoctest + +########### mydoctest.py ####### + +import sys +if sys.version_info < (3, 5): + # The module is only Cythonized and not build for these versions + # so don't run the tests + exit() + +import doctest +import from_py +val = doctest.testmod(from_py)[0] +import from_cy +val += doctest.testmod(from_cy)[0] + +exit(val) + +########### setup.py ######## + +# -*- coding: utf-8 -*- + +from __future__ import unicode_literals + +import sys +from Cython.Build import cythonize + +files = ["mymoð.pyx", "from_cy.pyx"] + + +# For Python 2 and Python <= 3.4 just run pyx->c; +# don't compile the C file +modules = cythonize(files) + +if sys.version_info >= (3, 5): + from distutils.core import setup + + setup( + ext_modules = modules + ) + +############ mymoð.pyx ######### + +def f(): + return True + +cdef public api void cdef_func(): + pass + +############ pxd_moð.pxd ########## + +cdef struct S: + int x + +cdef public api void cdef_func() # just to test generation of headers + +############ from_py.py ######### + +# -*- coding: utf-8 -*- + +import mymoð +from mymoð import f + +__doc__ = """ +>>> mymoð.f() +True +>>> f() +True +""" + +######### from_cy.pyx ########## + +# -*- coding: utf-8 -*- + +import mymoð + +from mymoð import f + +cimport pxd_moð +from pxd_moð cimport S + + +def test_imported(): + """ + >>> test_imported() + True + """ + return mymoð.f() and f() # True and True + + +def test_cimported(): + """ + >>> test_cimported() + 3 + """ + cdef pxd_moð.S v1 + v1.x = 1 + cdef S v2 + v2.x = 2 + return v1.x + v2.x + diff --git a/tests/run/unicodeliterals.pyx b/tests/run/unicodeliterals.pyx index bcc34d213..1449f5534 100644 --- a/tests/run/unicodeliterals.pyx +++ b/tests/run/unicodeliterals.pyx @@ -102,11 +102,7 @@ __doc__ = br""" True >>> ustring_in_constant_tuple == ('a', u'abc', u'\\N{SNOWMAN}', u'x' * 3, u'\\N{SNOWMAN}' * 4 + u'O') or ustring_in_constant_tuple # unescaped by Python True -""" -if sys.version_info >= (2,6,5): - # this doesn't work well in older Python versions - __doc__ += u"""\ >>> expected = u'\U00101234' # unescaped by Cython >>> if wide_literal == expected: print(True) ... else: print(repr(wide_literal), repr(expected), sys.maxunicode) diff --git a/tests/run/unsigned_char_ptr_bytes_conversion_T359.pyx b/tests/run/unsigned_char_ptr_bytes_conversion_T359.pyx index fb611511b..09df291f5 100644 --- a/tests/run/unsigned_char_ptr_bytes_conversion_T359.pyx +++ b/tests/run/unsigned_char_ptr_bytes_conversion_T359.pyx @@ -1,4 +1,4 @@ -# ticket: 359 +# ticket: t359 cdef unsigned char* some_c_unstring = 'test toast taste' diff --git a/tests/run/unsignedbehaviour_T184.pyx b/tests/run/unsignedbehaviour_T184.pyx index 9cbf65fc1..4dcb86398 100644 --- a/tests/run/unsignedbehaviour_T184.pyx +++ b/tests/run/unsignedbehaviour_T184.pyx @@ -1,4 +1,4 @@ -# ticket: 184 +# ticket: t184 """ >>> c_call() diff --git a/tests/run/versioned_pxds.srctree b/tests/run/versioned_pxds.srctree new file mode 100644 index 000000000..fb46cb26f --- /dev/null +++ b/tests/run/versioned_pxds.srctree @@ -0,0 +1,79 @@ +# mode: run +# tag: pxd + +""" +PYTHON setup.py build_ext --inplace +PYTHON -c "import runner" +""" + +######## setup.py ######## + +from Cython.Build.Dependencies import cythonize + +from distutils.core import setup, Extension + +setup( + ext_modules=cythonize([ + Extension("pkg.m1.a", ["pkg/m1/a.pyx"]), + Extension("pkg.m2.b", ["pkg/m2/b.pyx"]) + ]), +) + +######## pkg/__init__.py ######## + +######## pkg/m1/__init__.py ######## + + +######## pkg/m1/a.pyx ######## + +cdef class A: + def __init__(self): + self.x = 5 + +######## pkg/m1/a.pxd ######## + +to be ignored if there is a more specific file + +######## pkg/m1/a.cython-2.pxd ######## + +very outdated, not to be picked up + +######## pkg/m1/a.cython-20.pxd ######## + +outdated, not to be picked up + +######## pkg/m1/a.cython-29.pxd ######## + +# closest version should get found! + +cdef class A: + cdef public float x + +######## pkg/m1/a.cython-300000.pxd ######## + +Invalid distant future syntax right here! + +######## pkg/m1/a.cython-100000.pxd ######## + +Invalid future syntax right here! + + +######## pkg/m2/__init__.py ######## + +######## pkg/m2/b.pyx ######## + +from pkg.m1.a cimport A + +cdef class B(A): + pass + +######## runner.py ######## + +from pkg.m1.a import A +from pkg.m2.b import B + +a = A() +b = B() + +assert a.x == 5 +assert isinstance(a.x, float), type(a.x) diff --git a/tests/run/with_gil.pyx b/tests/run/with_gil.pyx index 46e9e6c2a..6fee3f192 100644 --- a/tests/run/with_gil.pyx +++ b/tests/run/with_gil.pyx @@ -1,3 +1,6 @@ +# mode: run +# tag: nogil, withgil + """ Test the 'with gil:' statement. """ @@ -460,6 +463,35 @@ def test_nogil_try_finally_error_label(): print e.args[0] +def void_with_python_objects(): + """ + >>> void_with_python_objects() + """ + with nogil: + _void_with_python_objects() + + +cdef void _void_with_python_objects() nogil: + c = 123 + with gil: + obj1 = [123] + obj2 = [456] + + +def void_with_py_arg_reassigned(x): + """ + >>> void_with_py_arg_reassigned(123) + """ + with nogil: + _void_with_py_arg_reassigned(x) + + +cdef void _void_with_py_arg_reassigned(x) nogil: + c = 123 + with gil: + x = [456] + + cdef void test_timing_callback() with gil: pass diff --git a/tests/run/with_gil_automatic.pyx b/tests/run/with_gil_automatic.pyx new file mode 100644 index 000000000..425dbbce7 --- /dev/null +++ b/tests/run/with_gil_automatic.pyx @@ -0,0 +1,138 @@ +# mode: run +# tag: nogil +# cython: language_level=2 + +cimport cython + + +#### print + +@cython.test_assert_path_exists( + "//GILStatNode", + "//GILStatNode//GILStatNode", + "//GILStatNode//GILStatNode//PrintStatNode", +) +def test_print_in_nogil_section(x): + """ + >>> test_print_in_nogil_section(123) + --123-- + """ + with nogil: + print f"--{x}--" + + +@cython.test_assert_path_exists( + "//GILStatNode", + "//GILStatNode//PrintStatNode", +) +@cython.test_fail_if_path_exists( + "//GILStatNode//GILStatNode", +) +cpdef int test_print_in_nogil_func(x) nogil except -1: + """ + >>> _ = test_print_in_nogil_func(123) + --123-- + """ + print f"--{x}--" + + +#### raise + +@cython.test_assert_path_exists( + "//GILStatNode", + "//GILStatNode//GILStatNode", + "//GILStatNode//GILStatNode//RaiseStatNode", +) +def test_raise_in_nogil_section(x): + """ + >>> try: test_raise_in_nogil_section(123) + ... except ValueError as exc: print(exc) + ... else: print("NOT RAISED !") + --123-- + """ + with nogil: + raise ValueError(f"--{x}--") + + +@cython.test_assert_path_exists( + "//GILStatNode", + "//GILStatNode//RaiseStatNode", +) +@cython.test_fail_if_path_exists( + "//GILStatNode//GILStatNode", +) +cpdef int test_raise_in_nogil_func(x) nogil except -1: + """ + >>> test_raise_in_nogil_func(123) + Traceback (most recent call last): + ValueError: --123-- + """ + raise ValueError(f"--{x}--") + + +#### assert + +@cython.test_assert_path_exists( + "//GILStatNode", + "//GILStatNode//AssertStatNode", + "//GILStatNode//AssertStatNode//GILStatNode", + "//GILStatNode//AssertStatNode//GILStatNode//RaiseStatNode", +) +def assert_in_nogil_section(int x): + """ + >>> assert_in_nogil_section(123) + >>> assert_in_nogil_section(0) + Traceback (most recent call last): + AssertionError + """ + with nogil: + assert x + + +@cython.test_assert_path_exists( + "//GILStatNode", + "//GILStatNode//AssertStatNode", + "//GILStatNode//AssertStatNode//GILStatNode", + "//GILStatNode//AssertStatNode//GILStatNode//RaiseStatNode", +) +def assert_in_nogil_section_ustring(int x): + """ + >>> assert_in_nogil_section_string(123) + >>> assert_in_nogil_section_string(0) + Traceback (most recent call last): + AssertionError: failed! + """ + with nogil: + assert x, u"failed!" + + +@cython.test_assert_path_exists( + "//GILStatNode", + "//GILStatNode//AssertStatNode", + "//GILStatNode//AssertStatNode//GILStatNode", + "//GILStatNode//AssertStatNode//GILStatNode//RaiseStatNode", +) +def assert_in_nogil_section_string(int x): + """ + >>> assert_in_nogil_section_string(123) + >>> assert_in_nogil_section_string(0) + Traceback (most recent call last): + AssertionError: failed! + """ + with nogil: + assert x, "failed!" + + +@cython.test_assert_path_exists( + "//AssertStatNode", + "//AssertStatNode//GILStatNode", + "//AssertStatNode//GILStatNode//RaiseStatNode", +) +cpdef int assert_in_nogil_func(int x) nogil except -1: + """ + >>> _ = assert_in_nogil_func(123) + >>> assert_in_nogil_func(0) + Traceback (most recent call last): + AssertionError: failed! + """ + assert x, "failed!" diff --git a/tests/run/with_statement_module_level_T536.pyx b/tests/run/with_statement_module_level_T536.pyx index 506c362f9..a18925f46 100644 --- a/tests/run/with_statement_module_level_T536.pyx +++ b/tests/run/with_statement_module_level_T536.pyx @@ -1,4 +1,4 @@ -# ticket: 536 +# ticket: t536 __doc__ = """ >>> inner_result diff --git a/tests/run/yield_from_pep380.pyx b/tests/run/yield_from_pep380.pyx index a1b7500d5..d73144c9e 100644 --- a/tests/run/yield_from_pep380.pyx +++ b/tests/run/yield_from_pep380.pyx @@ -4,7 +4,7 @@ Test suite for PEP 380 implementation adapted from original tests written by Greg Ewing -see <http://www.cosc.canterbury.ac.nz/greg.ewing/python/yield-from/YieldFrom-Python3.1.2-rev5.zip> +see <https://www.cosc.canterbury.ac.nz/greg.ewing/python/yield-from/YieldFrom-Python3.1.2-rev5.zip> """ import sys diff --git a/tests/windows_bugs.txt b/tests/windows_bugs.txt index ea5b23902..efca82325 100644 --- a/tests/windows_bugs.txt +++ b/tests/windows_bugs.txt @@ -24,3 +24,9 @@ queue queue2 queue3 lunch + +# "C linkage function cannot return C++ class" (uses public C++ cdef function) +cpp_template_subclasses + +# MSVC lacks "complex.h" +complex_numbers_cmath_T2891 @@ -10,9 +10,3 @@ envlist = py26, py27, py32, py33, py34, pypy setenv = CFLAGS=-O0 -ggdb commands = {envpython} runtests.py -vv - -[pycodestyle] -ignore = W, E -select = E711, E714, E501, W291 -max-line-length = 150 -format = pylint |
