diff options
author | Pierre Sassoulas <pierre.sassoulas@gmail.com> | 2021-11-21 16:26:13 +0100 |
---|---|---|
committer | GitHub <noreply@github.com> | 2021-11-21 16:26:13 +0100 |
commit | 3455dc2bf9c436e6961fe74f04d03eb08e4bdb99 (patch) | |
tree | b3861ef918636f220a8052e07485832680061a2e | |
parent | 0f6a55c7f6f481acdcf57fbe68d3da80d77b9d89 (diff) | |
download | pylint-git-3455dc2bf9c436e6961fe74f04d03eb08e4bdb99.tar.gz |
Add accetpance tests in CI for python 3.8+ instead of launching manually at release (#5353)
* [test] Add acceptance tests in the continuous integration
* Disable duplicate-code from the acceptance tests
* Rename acceptance to primer stdlib for clarity
-rw-r--r-- | .github/workflows/ci.yaml | 34 | ||||
-rw-r--r-- | setup.cfg | 4 | ||||
-rw-r--r-- | tbump.toml | 4 | ||||
-rw-r--r-- | tests/acceptance/test_stdlib.py | 54 | ||||
-rw-r--r-- | tests/primer/test_primer_stdlib.py | 65 |
5 files changed, 101 insertions, 60 deletions
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index e4c17eae4..218346f30 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -475,3 +475,37 @@ jobs: run: | . venv/bin/activate pytest --benchmark-disable tests/ + + pytest-primer-stlib: + name: Run primer tests on stdlib Python ${{ matrix.python-version }} (Linux) + runs-on: ubuntu-latest + needs: prepare-tests-linux + strategy: + matrix: + python-version: [3.8, 3.9, "3.10"] + steps: + - name: Check out code from GitHub + uses: actions/checkout@v2.3.5 + - name: Set up Python ${{ matrix.python-version }} + id: python + uses: actions/setup-python@v2.2.2 + with: + python-version: ${{ matrix.python-version }} + - name: Restore Python virtual environment + id: cache-venv + uses: actions/cache@v2.1.6 + with: + path: venv + key: + ${{ runner.os }}-${{ steps.python.outputs.python-version }}-${{ + needs.prepare-tests-linux.outputs.python-key }} + - name: Fail job if Python cache restore failed + if: steps.cache-venv.outputs.cache-hit != 'true' + run: | + echo "Failed to restore Python venv from cache" + exit 1 + - name: Run pytest + run: | + . venv/bin/activate + pip install -e . + pytest -m primer_stdlib -n auto @@ -69,9 +69,9 @@ test = pytest [tool:pytest] testpaths = tests python_files = *test_*.py -addopts = -m "not acceptance" +addopts = -m "not primer_stdlib" markers = - acceptance: + primer_stdlib: Checks for crashes and errors when running pylint on stdlib benchmark: Baseline of pylint performance, if this regress something serious happened [isort] diff --git a/tbump.toml b/tbump.toml index 75904bca1..57d910f1e 100644 --- a/tbump.toml +++ b/tbump.toml @@ -36,10 +36,6 @@ cmd = "pip3 install copyrite;copyrite --contribution-threshold 1 --change-thresh name = "Apply pre-commit" cmd = "pre-commit run --all-files||echo 'Hack so this command does not fail'" -[[before_commit]] -name = "Acceptance tests" -cmd = "pytest -m acceptance" - # Or run some commands after the git tag and the branch # have been pushed: # [[after_push]] diff --git a/tests/acceptance/test_stdlib.py b/tests/acceptance/test_stdlib.py deleted file mode 100644 index c5af893b4..000000000 --- a/tests/acceptance/test_stdlib.py +++ /dev/null @@ -1,54 +0,0 @@ -# Licensed under the GPL: https://www.gnu.org/licenses/old-licenses/gpl-2.0.html -# For details: https://github.com/PyCQA/pylint/blob/main/LICENSE - -import contextlib -import io -import os -import sys - -import pytest - -import pylint.lint - - -def is_module(filename): - return filename.endswith(".py") - - -def is_package(filename, location): - return os.path.exists(os.path.join(location, filename, "__init__.py")) - - -@contextlib.contextmanager -def _patch_stdout(out): - sys.stdout = out - try: - yield - finally: - sys.stdout = sys.__stdout__ - - -LIB_DIRS = [os.path.dirname(os.__file__)] -MODULES_TO_CHECK = [ - (location, module) - for location in LIB_DIRS - for module in os.listdir(location) - if is_module(module) or is_package(module, location) -] -MODULES_NAMES = [m[1] for m in MODULES_TO_CHECK] - - -@pytest.mark.acceptance -@pytest.mark.parametrize( - ("test_module_location", "test_module_name"), MODULES_TO_CHECK, ids=MODULES_NAMES -) -def test_libmodule(test_module_location, test_module_name): - os.chdir(test_module_location) - with _patch_stdout(io.StringIO()): - try: - pylint.lint.Run([test_module_name, "--enable=all", "--ignore=test"]) - except SystemExit as ex: - assert ex.code != 32 - return - - assert False, "shouldn't get there" diff --git a/tests/primer/test_primer_stdlib.py b/tests/primer/test_primer_stdlib.py new file mode 100644 index 000000000..fdc16adcd --- /dev/null +++ b/tests/primer/test_primer_stdlib.py @@ -0,0 +1,65 @@ +# Licensed under the GPL: https://www.gnu.org/licenses/old-licenses/gpl-2.0.html +# For details: https://github.com/PyCQA/pylint/blob/main/LICENSE + +import contextlib +import io +import os +import sys + +import pytest +from pytest import CaptureFixture + +import pylint.lint + + +def is_module(filename: str) -> bool: + return filename.endswith(".py") + + +def is_package(filename: str, location: str) -> bool: + return os.path.exists(os.path.join(location, filename, "__init__.py")) + + +@contextlib.contextmanager +def _patch_stdout(out): + sys.stdout = out + try: + yield + finally: + sys.stdout = sys.__stdout__ + + +LIB_DIRS = [os.path.dirname(os.__file__)] +MODULES_TO_CHECK = [ + (location, module) + for location in LIB_DIRS + for module in os.listdir(location) + if is_module(module) or is_package(module, location) +] +MODULES_NAMES = [m[1] for m in MODULES_TO_CHECK] + + +@pytest.mark.primer_stdlib +@pytest.mark.parametrize( + ("test_module_location", "test_module_name"), MODULES_TO_CHECK, ids=MODULES_NAMES +) +def test_lib_module_no_crash( + test_module_location: str, test_module_name: str, capsys: CaptureFixture +) -> None: + """Test that pylint does not produces any crashes or fatal errors on stdlib modules""" + os.chdir(test_module_location) + with _patch_stdout(io.StringIO()): + try: + # We want to test all the code we can + enables = ["--enable-all-extensions", "--enable=all"] + # Duplicate code takes too long and is relatively safe + # We don't want to lint the test directory which are repetitive + disables = ["--disable=duplicate-code", "--ignore=test"] + pylint.lint.Run([test_module_name] + enables + disables) + except SystemExit as ex: + out, err = capsys.readouterr() + assert not err, err + assert not out + msg = f"Encountered {{}} during primer stlib test for {test_module_name}" + assert ex.code != 32, msg.format("a crash") + assert ex.code % 2 == 0, msg.format("a message of category 'fatal'") |