Add accetpance tests in CI for python 3.8+ instead of launching manually at release (#5353)

* [test] Add acceptance tests in the continuous integration

* Disable duplicate-code from the acceptance tests

* Rename acceptance to primer stdlib for clarity

5 files changed, 101 insertions, 60 deletions

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index e4c17eae4..218346f30 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -475,3 +475,37 @@ jobs:
         run: |
           . venv/bin/activate
           pytest --benchmark-disable tests/
+
+  pytest-primer-stlib:
+    name: Run primer tests on stdlib Python ${{ matrix.python-version }} (Linux)
+    runs-on: ubuntu-latest
+    needs: prepare-tests-linux
+    strategy:
+      matrix:
+        python-version: [3.8, 3.9, "3.10"]
+    steps:
+      - name: Check out code from GitHub
+        uses: actions/checkout@v2.3.5
+      - name: Set up Python ${{ matrix.python-version }}
+        id: python
+        uses: actions/setup-python@v2.2.2
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Restore Python virtual environment
+        id: cache-venv
+        uses: actions/cache@v2.1.6
+        with:
+          path: venv
+          key:
+            ${{ runner.os }}-${{ steps.python.outputs.python-version }}-${{
+            needs.prepare-tests-linux.outputs.python-key }}
+      - name: Fail job if Python cache restore failed
+        if: steps.cache-venv.outputs.cache-hit != 'true'
+        run: |
+          echo "Failed to restore Python venv from cache"
+          exit 1
+      - name: Run pytest
+        run: |
+          . venv/bin/activate
+          pip install -e .
+          pytest -m primer_stdlib -n auto
diff --git a/setup.cfg b/setup.cfg
index 4d0761b67..c19718df0 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -69,9 +69,9 @@ test = pytest
 [tool:pytest]
 testpaths = tests
 python_files = *test_*.py
-addopts = -m "not acceptance"
+addopts = -m "not primer_stdlib"
 markers =
-    acceptance:
+    primer_stdlib: Checks for crashes and errors when running pylint on stdlib
     benchmark: Baseline of pylint performance, if this regress something serious happened
 
 [isort]
diff --git a/tbump.toml b/tbump.toml
index 75904bca1..57d910f1e 100644
--- a/tbump.toml
+++ b/tbump.toml
@@ -36,10 +36,6 @@ cmd = "pip3 install copyrite;copyrite --contribution-threshold 1 --change-thresh
 name = "Apply pre-commit"
 cmd = "pre-commit run --all-files||echo 'Hack so this command does not fail'"
 
-[[before_commit]]
-name = "Acceptance tests"
-cmd = "pytest -m acceptance"
-
 # Or run some commands after the git tag and the branch
 # have been pushed:
 #  [[after_push]]
diff --git a/tests/acceptance/test_stdlib.py b/tests/acceptance/test_stdlib.py
deleted file mode 100644
index c5af893b4..000000000
--- a/tests/acceptance/test_stdlib.py
+++ /dev/null
@@ -1,54 +0,0 @@
-# Licensed under the GPL: https://www.gnu.org/licenses/old-licenses/gpl-2.0.html
-# For details: https://github.com/PyCQA/pylint/blob/main/LICENSE
-
-import contextlib
-import io
-import os
-import sys
-
-import pytest
-
-import pylint.lint
-
-
-def is_module(filename):
-    return filename.endswith(".py")
-
-
-def is_package(filename, location):
-    return os.path.exists(os.path.join(location, filename, "__init__.py"))
-
-
-@contextlib.contextmanager
-def _patch_stdout(out):
-    sys.stdout = out
-    try:
-        yield
-    finally:
-        sys.stdout = sys.__stdout__
-
-
-LIB_DIRS = [os.path.dirname(os.__file__)]
-MODULES_TO_CHECK = [
-    (location, module)
-    for location in LIB_DIRS
-    for module in os.listdir(location)
-    if is_module(module) or is_package(module, location)
-]
-MODULES_NAMES = [m[1] for m in MODULES_TO_CHECK]
-
-
-@pytest.mark.acceptance
-@pytest.mark.parametrize(
-    ("test_module_location", "test_module_name"), MODULES_TO_CHECK, ids=MODULES_NAMES
-)
-def test_libmodule(test_module_location, test_module_name):
-    os.chdir(test_module_location)
-    with _patch_stdout(io.StringIO()):
-        try:
-            pylint.lint.Run([test_module_name, "--enable=all", "--ignore=test"])
-        except SystemExit as ex:
-            assert ex.code != 32
-            return
-
-        assert False, "shouldn't get there"
diff --git a/tests/primer/test_primer_stdlib.py b/tests/primer/test_primer_stdlib.py
new file mode 100644
index 000000000..fdc16adcd
--- /dev/null
+++ b/tests/primer/test_primer_stdlib.py
@@ -0,0 +1,65 @@
+# Licensed under the GPL: https://www.gnu.org/licenses/old-licenses/gpl-2.0.html
+# For details: https://github.com/PyCQA/pylint/blob/main/LICENSE
+
+import contextlib
+import io
+import os
+import sys
+
+import pytest
+from pytest import CaptureFixture
+
+import pylint.lint
+
+
+def is_module(filename: str) -> bool:
+    return filename.endswith(".py")
+
+
+def is_package(filename: str, location: str) -> bool:
+    return os.path.exists(os.path.join(location, filename, "__init__.py"))
+
+
+@contextlib.contextmanager
+def _patch_stdout(out):
+    sys.stdout = out
+    try:
+        yield
+    finally:
+        sys.stdout = sys.__stdout__
+
+
+LIB_DIRS = [os.path.dirname(os.__file__)]
+MODULES_TO_CHECK = [
+    (location, module)
+    for location in LIB_DIRS
+    for module in os.listdir(location)
+    if is_module(module) or is_package(module, location)
+]
+MODULES_NAMES = [m[1] for m in MODULES_TO_CHECK]
+
+
+@pytest.mark.primer_stdlib
+@pytest.mark.parametrize(
+    ("test_module_location", "test_module_name"), MODULES_TO_CHECK, ids=MODULES_NAMES
+)
+def test_lib_module_no_crash(
+    test_module_location: str, test_module_name: str, capsys: CaptureFixture
+) -> None:
+    """Test that pylint does not produces any crashes or fatal errors on stdlib modules"""
+    os.chdir(test_module_location)
+    with _patch_stdout(io.StringIO()):
+        try:
+            # We want to test all the code we can
+            enables = ["--enable-all-extensions", "--enable=all"]
+            # Duplicate code takes too long and is relatively safe
+            # We don't want to lint the test directory which are repetitive
+            disables = ["--disable=duplicate-code", "--ignore=test"]
+            pylint.lint.Run([test_module_name] + enables + disables)
+        except SystemExit as ex:
+            out, err = capsys.readouterr()
+            assert not err, err
+            assert not out
+            msg = f"Encountered {{}} during primer stlib test for {test_module_name}"
+            assert ex.code != 32, msg.format("a crash")
+            assert ex.code % 2 == 0, msg.format("a message of category 'fatal'")