diff options
| -rw-r--r-- | .github/workflows/ci.yaml | 6 | ||||
| -rw-r--r-- | doc/development_guide/testing.rst | 21 | ||||
| -rw-r--r-- | setup.cfg | 2 | ||||
| -rw-r--r-- | tests/conftest.py | 36 |
4 files changed, 60 insertions, 5 deletions
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 700fd8f31..5d83b9180 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -476,7 +476,7 @@ jobs: . venv/bin/activate pytest --benchmark-disable tests/ - pytest-primer-stlib: + pytest-primer-stdlib: name: Run primer tests on stdlib Python ${{ matrix.python-version }} (Linux) runs-on: ubuntu-latest needs: prepare-tests-linux @@ -508,7 +508,7 @@ jobs: run: | . venv/bin/activate pip install -e . - pytest -m primer_stdlib -n auto + pytest -m primer_stdlib --primer-stdlib -n auto pytest-primer-external: name: Run primer tests on external libs Python ${{ matrix.python-version }} (Linux) @@ -542,4 +542,4 @@ jobs: run: | . venv/bin/activate pip install -e . - pytest -m primer_external -n auto + pytest -m primer_external --primer-external -n auto diff --git a/doc/development_guide/testing.rst b/doc/development_guide/testing.rst index d2fe7e785..ffcba3b00 100644 --- a/doc/development_guide/testing.rst +++ b/doc/development_guide/testing.rst @@ -87,7 +87,7 @@ current environment in order to have faster feedback. Run from Pylint root direc python tests/test_functional.py -You can use all the options you would use for pytest, for example ``-k "test_functional[len_checks]"``. +You can use all the options you would use for pytest_, for example ``-k "test_functional[len_checks]"``. Furthermore, if required the .txt file with expected messages can be regenerated based on the the current output by appending ``--update-functional-output`` to the command line:: @@ -138,6 +138,25 @@ and should exit with exit code 2 the ``.out`` file should be named ``bad_configu The content of the ``.out`` file should have a similar pattern as a normal Pylint output. Note that the module name should be ``{abspath}`` and the file name ``{relpath}``. +Primer tests +------------------------------------------- + +Pylint also uses what we refer to as ``primer`` tests. These are tests that are run automatically +in our Continuous Integration and check whether any changes in Pylint lead to crashes or fatal errors +on the ``stdlib`` and a selection of external repositories. + +To run the ``primer`` tests you can add either ``--primer-stdlib`` or ``--primer-external`` to the +pytest_ command. If you want to only run the ``primer`` you can add either of their marks, for example:: + + pytest -m primer_external --primer-external + +The list of repositories is created on the basis of three criteria: 1) projects need to use a diverse +range of language features, 2) projects need to be well maintained and 3) projects should not have a codebase +that is too repetitive. This guarantees a good balance between speed of our CI and finding potential bugs. + +You can find the latest list of repositories and any relevant code for these tests in the ``tests/primer`` +directory. + .. _tox: https://tox.readthedocs.io/en/latest/ .. _pytest: https://pytest.readthedocs.io/en/latest/ .. _astroid: https://github.com/pycqa/astroid @@ -69,7 +69,7 @@ test = pytest [tool:pytest] testpaths = tests python_files = *test_*.py -addopts = --strict-markers -m "not primer_stdlib and not primer_external" +addopts = --strict-markers markers = primer_stdlib: Checks for crashes and errors when running pylint on stdlib primer_external: Checks for crashes and errors when running pylint on external libs diff --git a/tests/conftest.py b/tests/conftest.py index 7a9c062db..406e93efd 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -56,3 +56,39 @@ def disable(): @pytest.fixture(scope="module") def reporter(): return MinimalTestReporter + + +def pytest_addoption(parser) -> None: + parser.addoption( + "--primer-stdlib", + action="store_true", + default=False, + help="Run primer stdlib tests", + ) + parser.addoption( + "--primer-external", + action="store_true", + default=False, + help="Run primer external tests", + ) + + +def pytest_collection_modifyitems(config, items) -> None: + """Convert command line options to markers""" + # Add skip_primer_stdlib mark + if not config.getoption("--primer-external"): + skip_primer_external = pytest.mark.skip( + reason="need --primer-external option to run" + ) + for item in items: + if "primer_external" in item.keywords: + item.add_marker(skip_primer_external) + + # Add skip_primer_stdlib mark + if not config.getoption("--primer-stdlib"): + skip_primer_stdlib = pytest.mark.skip( + reason="need --primer-stdlib option to run" + ) + for item in items: + if "primer_stdlib" in item.keywords: + item.add_marker(skip_primer_stdlib) |