diff options
| -rw-r--r-- | .github/workflows/ci.yaml | 68 | ||||
| -rw-r--r-- | .github/workflows/primer-test.yaml | 167 | ||||
| -rw-r--r-- | doc/development_guide/testing.rst | 9 | ||||
| -rw-r--r-- | setup.cfg | 3 | ||||
| -rw-r--r-- | tests/conftest.py | 6 | ||||
| -rw-r--r-- | tests/primer/packages_to_lint.json | 59 | ||||
| -rw-r--r-- | tests/primer/packages_to_lint_batch_one.json | 33 | ||||
| -rw-r--r-- | tests/primer/packages_to_lint_batch_two.json | 28 | ||||
| -rw-r--r-- | tests/primer/test_primer_external.py | 38 |
9 files changed, 274 insertions, 137 deletions
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 5d83b9180..1028e8d09 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -475,71 +475,3 @@ jobs: run: | . venv/bin/activate pytest --benchmark-disable tests/ - - pytest-primer-stdlib: - name: Run primer tests on stdlib Python ${{ matrix.python-version }} (Linux) - runs-on: ubuntu-latest - needs: prepare-tests-linux - strategy: - matrix: - python-version: [3.8, 3.9, "3.10"] - steps: - - name: Check out code from GitHub - uses: actions/checkout@v2.3.5 - - name: Set up Python ${{ matrix.python-version }} - id: python - uses: actions/setup-python@v2.3.0 - with: - python-version: ${{ matrix.python-version }} - - name: Restore Python virtual environment - id: cache-venv - uses: actions/cache@v2.1.6 - with: - path: venv - key: - ${{ runner.os }}-${{ steps.python.outputs.python-version }}-${{ - needs.prepare-tests-linux.outputs.python-key }} - - name: Fail job if Python cache restore failed - if: steps.cache-venv.outputs.cache-hit != 'true' - run: | - echo "Failed to restore Python venv from cache" - exit 1 - - name: Run pytest - run: | - . venv/bin/activate - pip install -e . - pytest -m primer_stdlib --primer-stdlib -n auto - - pytest-primer-external: - name: Run primer tests on external libs Python ${{ matrix.python-version }} (Linux) - runs-on: ubuntu-latest - needs: prepare-tests-linux - strategy: - matrix: - python-version: [3.8, 3.9, "3.10"] - steps: - - name: Check out code from GitHub - uses: actions/checkout@v2.3.5 - - name: Set up Python ${{ matrix.python-version }} - id: python - uses: actions/setup-python@v2.2.2 - with: - python-version: ${{ matrix.python-version }} - - name: Restore Python virtual environment - id: cache-venv - uses: actions/cache@v2.1.6 - with: - path: venv - key: - ${{ runner.os }}-${{ steps.python.outputs.python-version }}-${{ - needs.prepare-tests-linux.outputs.python-key }} - - name: Fail job if Python cache restore failed - if: steps.cache-venv.outputs.cache-hit != 'true' - run: | - echo "Failed to restore Python venv from cache" - exit 1 - - name: Run pytest - run: | - . venv/bin/activate - pip install -e . - pytest -m primer_external --primer-external -n auto diff --git a/.github/workflows/primer-test.yaml b/.github/workflows/primer-test.yaml new file mode 100644 index 000000000..e80833fb5 --- /dev/null +++ b/.github/workflows/primer-test.yaml @@ -0,0 +1,167 @@ +name: Primer tests + +on: + push: + branches: + - main + - 2.* + pull_request: + paths: + - "pylint/**" + - "tests/primer/**" + +env: + CACHE_VERSION: 3 + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + prepare-tests-linux: + name: Prepare tests for Python ${{ matrix.python-version }} (Linux) + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [3.8, 3.9, "3.10"] + outputs: + python-key: ${{ steps.generate-python-key.outputs.key }} + steps: + - name: Check out code from GitHub + uses: actions/checkout@v2.4.0 + with: + fetch-depth: 0 + - name: Set up Python ${{ matrix.python-version }} + id: python + uses: actions/setup-python@v2.3.0 + with: + python-version: ${{ matrix.python-version }} + - name: Generate partial Python venv restore key + id: generate-python-key + run: >- + echo "::set-output name=key::venv-${{ env.CACHE_VERSION }}-${{ + hashFiles('setup.cfg', 'requirements_test.txt', 'requirements_test_min.txt') + }}" + - name: Restore Python virtual environment + id: cache-venv + uses: actions/cache@v2.1.6 + with: + path: venv + key: >- + ${{ runner.os }}-${{ steps.python.outputs.python-version }}-${{ + steps.generate-python-key.outputs.key }} + restore-keys: | + ${{ runner.os }}-${{ steps.python.outputs.python-version }}-venv-${{ env.CACHE_VERSION }}- + - name: Create Python virtual environment + if: steps.cache-venv.outputs.cache-hit != 'true' + run: | + python -m venv venv + . venv/bin/activate + python -m pip install -U pip setuptools wheel + pip install -U -r requirements_test.txt + + pytest-primer-stdlib: + name: Run primer tests on stdlib Python ${{ matrix.python-version }} (Linux) + runs-on: ubuntu-latest + needs: prepare-tests-linux + strategy: + matrix: + python-version: [3.8, 3.9, "3.10"] + steps: + - name: Check out code from GitHub + uses: actions/checkout@v2.3.5 + - name: Set up Python ${{ matrix.python-version }} + id: python + uses: actions/setup-python@v2.3.0 + with: + python-version: ${{ matrix.python-version }} + - name: Restore Python virtual environment + id: cache-venv + uses: actions/cache@v2.1.6 + with: + path: venv + key: + ${{ runner.os }}-${{ steps.python.outputs.python-version }}-${{ + needs.prepare-tests-linux.outputs.python-key }} + - name: Fail job if Python cache restore failed + if: steps.cache-venv.outputs.cache-hit != 'true' + run: | + echo "Failed to restore Python venv from cache" + exit 1 + - name: Run pytest + run: | + . venv/bin/activate + pip install -e . + pytest -m primer_stdlib --primer-stdlib -n auto + + pytest-primer-external-batch-one: + name: + Run primer tests batch one on external libs Python ${{ matrix.python-version }} + (Linux) + runs-on: ubuntu-latest + needs: prepare-tests-linux + strategy: + matrix: + python-version: [3.8, 3.9, "3.10"] + steps: + - name: Check out code from GitHub + uses: actions/checkout@v2.3.5 + - name: Set up Python ${{ matrix.python-version }} + id: python + uses: actions/setup-python@v2.2.2 + with: + python-version: ${{ matrix.python-version }} + - name: Restore Python virtual environment + id: cache-venv + uses: actions/cache@v2.1.6 + with: + path: venv + key: + ${{ runner.os }}-${{ steps.python.outputs.python-version }}-${{ + needs.prepare-tests-linux.outputs.python-key }} + - name: Fail job if Python cache restore failed + if: steps.cache-venv.outputs.cache-hit != 'true' + run: | + echo "Failed to restore Python venv from cache" + exit 1 + - name: Run pytest + run: | + . venv/bin/activate + pip install -e . + pytest -m primer_external_batch_one --primer-external -n auto + + pytest-primer-external-batch-two: + name: + Run primer tests batch two on external libs Python ${{ matrix.python-version }} + (Linux) + runs-on: ubuntu-latest + needs: prepare-tests-linux + strategy: + matrix: + python-version: [3.8, 3.9, "3.10"] + steps: + - name: Check out code from GitHub + uses: actions/checkout@v2.3.5 + - name: Set up Python ${{ matrix.python-version }} + id: python + uses: actions/setup-python@v2.2.2 + with: + python-version: ${{ matrix.python-version }} + - name: Restore Python virtual environment + id: cache-venv + uses: actions/cache@v2.1.6 + with: + path: venv + key: + ${{ runner.os }}-${{ steps.python.outputs.python-version }}-${{ + needs.prepare-tests-linux.outputs.python-key }} + - name: Fail job if Python cache restore failed + if: steps.cache-venv.outputs.cache-hit != 'true' + run: | + echo "Failed to restore Python venv from cache" + exit 1 + - name: Run pytest + run: | + . venv/bin/activate + pip install -e . + pytest -m primer_external_batch_two --primer-external -n auto diff --git a/doc/development_guide/testing.rst b/doc/development_guide/testing.rst index 3cdee7c90..85d1043ef 100644 --- a/doc/development_guide/testing.rst +++ b/doc/development_guide/testing.rst @@ -148,7 +148,14 @@ on the ``stdlib`` and a selection of external repositories. To run the ``primer`` tests you can add either ``--primer-stdlib`` or ``--primer-external`` to the pytest_ command. If you want to only run the ``primer`` you can add either of their marks, for example:: - pytest -m primer_external --primer-external + pytest -m primer_stdlib --primer-stdlib + +The external ``primer`` has been split up in two marks to speed up our Continuous Integration. You can run +either of the two batches or run them both:: + + pytest -m primer_external_batch_one --primer-external # Runs batch one + pytest -m primer_external_batch_two --primer-external # Runs batch two + pytest -m "primer_external_batch_one or primer_external_batch_two" --primer-external # Runs both batches The list of repositories is created on the basis of three criteria: 1) projects need to use a diverse range of language features, 2) projects need to be well maintained and 3) projects should not have a codebase @@ -72,7 +72,8 @@ python_files = *test_*.py addopts = --strict-markers markers = primer_stdlib: Checks for crashes and errors when running pylint on stdlib - primer_external: Checks for crashes and errors when running pylint on external libs + primer_external_batch_one: Checks for crashes and errors when running pylint on external libs (batch one) + primer_external_batch_two: Checks for crashes and errors when running pylint on external libs (batch two) benchmark: Baseline of pylint performance, if this regress something serious happened [isort] diff --git a/tests/conftest.py b/tests/conftest.py index 406e93efd..bdd3e4516 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -75,13 +75,15 @@ def pytest_addoption(parser) -> None: def pytest_collection_modifyitems(config, items) -> None: """Convert command line options to markers""" - # Add skip_primer_stdlib mark + # Add skip_primer_external mark if not config.getoption("--primer-external"): skip_primer_external = pytest.mark.skip( reason="need --primer-external option to run" ) for item in items: - if "primer_external" in item.keywords: + if "primer_external_batch_one" in item.keywords: + item.add_marker(skip_primer_external) + if "primer_external_batch_two" in item.keywords: item.add_marker(skip_primer_external) # Add skip_primer_stdlib mark diff --git a/tests/primer/packages_to_lint.json b/tests/primer/packages_to_lint.json deleted file mode 100644 index 507fa3af3..000000000 --- a/tests/primer/packages_to_lint.json +++ /dev/null @@ -1,59 +0,0 @@ -{ - "flask": { - "url": "https://github.com/pallets/flask.git", - "branch": "main", - "directories": ["src/flask"] - }, - "pytest": { - "url": "https://github.com/pytest-dev/pytest.git", - "branch": "main", - "directories": ["src/_pytest"] - }, - "psycopg": { - "url": "https://github.com/psycopg/psycopg.git", - "branch": "master", - "directories": ["psycopg/psycopg"] - }, - "keras": { - "url": "https://github.com/keras-team/keras.git", - "branch": "master", - "directories": ["keras"] - }, - "sentry": { - "url": "https://github.com/getsentry/sentry.git", - "branch": "master", - "directories": ["src/sentry"] - }, - "django": { - "url": "https://github.com/django/django.git", - "branch": "main", - "directories": ["django"] - }, - "pandas": { - "url": "https://github.com/pandas-dev/pandas.git", - "branch": "master", - "directories": ["pandas"], - "pylint_additional_args": ["--ignore-patterns=\"test_"] - }, - "black": { - "url": "https://github.com/psf/black.git", - "branch": "main", - "directories": ["src/black/", "src/blackd/", "src/black_primer/", "src/blib2to3/"] - }, - "home-assistant": { - "url": "https://github.com/home-assistant/core.git", - "branch": "dev", - "directories": ["homeassistant"] - }, - "graph-explorer": { - "url": "https://github.com/vimeo/graph-explorer.git", - "branch": "master", - "directories": ["graph_explorer"], - "pylintrc_relpath": ".pylintrc" - }, - "pygame": { - "url": "https://github.com/pygame/pygame.git", - "branch": "main", - "directories": ["src_py"] - } -} diff --git a/tests/primer/packages_to_lint_batch_one.json b/tests/primer/packages_to_lint_batch_one.json new file mode 100644 index 000000000..95768b82e --- /dev/null +++ b/tests/primer/packages_to_lint_batch_one.json @@ -0,0 +1,33 @@ +{ + "black": { + "branch": "main", + "directories": ["src/black/", "src/blackd/", "src/black_primer/", "src/blib2to3/"], + "url": "https://github.com/psf/black.git" + }, + "django": { + "branch": "main", + "directories": ["django"], + "url": "https://github.com/django/django.git" + }, + "flask": { + "branch": "main", + "directories": ["src/flask"], + "url": "https://github.com/pallets/flask.git" + }, + "graph-explorer": { + "branch": "master", + "directories": ["graph_explorer"], + "pylintrc_relpath": ".pylintrc", + "url": "https://github.com/vimeo/graph-explorer.git" + }, + "home-assistant": { + "branch": "dev", + "directories": ["homeassistant"], + "url": "https://github.com/home-assistant/core.git" + }, + "keras": { + "branch": "master", + "directories": ["keras"], + "url": "https://github.com/keras-team/keras.git" + } +} diff --git a/tests/primer/packages_to_lint_batch_two.json b/tests/primer/packages_to_lint_batch_two.json new file mode 100644 index 000000000..636d155ff --- /dev/null +++ b/tests/primer/packages_to_lint_batch_two.json @@ -0,0 +1,28 @@ +{ + "pandas": { + "branch": "master", + "directories": ["pandas"], + "pylint_additional_args": ["--ignore-patterns=\"test_"], + "url": "https://github.com/pandas-dev/pandas.git" + }, + "psycopg": { + "branch": "master", + "directories": ["psycopg/psycopg"], + "url": "https://github.com/psycopg/psycopg.git" + }, + "pygame": { + "branch": "main", + "directories": ["src_py"], + "url": "https://github.com/pygame/pygame.git" + }, + "pytest": { + "branch": "main", + "directories": ["src/_pytest"], + "url": "https://github.com/pytest-dev/pytest.git" + }, + "sentry": { + "branch": "master", + "directories": ["src/sentry"], + "url": "https://github.com/getsentry/sentry.git" + } +} diff --git a/tests/primer/test_primer_external.py b/tests/primer/test_primer_external.py index a3fb9b66c..12206b0fc 100644 --- a/tests/primer/test_primer_external.py +++ b/tests/primer/test_primer_external.py @@ -24,16 +24,42 @@ def get_packages_to_lint_from_json( return result -PACKAGE_TO_LINT_JSON = Path(__file__).parent / "packages_to_lint.json" -PACKAGES_TO_LINT = get_packages_to_lint_from_json(PACKAGE_TO_LINT_JSON) -"""Dictionary of external packages used during the primer test""" +PACKAGE_TO_LINT_JSON_BATCH_ONE = ( + Path(__file__).parent / "packages_to_lint_batch_one.json" +) +PACKAGES_TO_LINT_BATCH_ONE = get_packages_to_lint_from_json( + PACKAGE_TO_LINT_JSON_BATCH_ONE +) +"""Dictionary of external packages used during the primer test in batch one""" + +PACKAGE_TO_LINT_JSON_BATCH_TWO = ( + Path(__file__).parent / "packages_to_lint_batch_two.json" +) +PACKAGES_TO_LINT_BATCH_TWO = get_packages_to_lint_from_json( + PACKAGE_TO_LINT_JSON_BATCH_TWO +) +"""Dictionary of external packages used during the primer test in batch two""" class TestPrimer: @staticmethod - @pytest.mark.primer_external - @pytest.mark.parametrize("package", PACKAGES_TO_LINT.values(), ids=PACKAGES_TO_LINT) - def test_primer_external_packages_no_crash( + @pytest.mark.primer_external_batch_one + @pytest.mark.parametrize( + "package", PACKAGES_TO_LINT_BATCH_ONE.values(), ids=PACKAGES_TO_LINT_BATCH_ONE + ) + def test_primer_external_packages_no_crash_batch_one( + package: PackageToLint, + caplog: LogCaptureFixture, + ) -> None: + __tracebackhide__ = True # pylint: disable=unused-variable + TestPrimer._primer_test(package, caplog) + + @staticmethod + @pytest.mark.primer_external_batch_two + @pytest.mark.parametrize( + "package", PACKAGES_TO_LINT_BATCH_TWO.values(), ids=PACKAGES_TO_LINT_BATCH_TWO + ) + def test_primer_external_packages_no_crash_batch_two( package: PackageToLint, caplog: LogCaptureFixture, ) -> None: |