3 files changed, 128 insertions, 2 deletions
diff --git a/tests/primer/packages_to_lint.json b/tests/primer/packages_to_lint.json
new file mode 100644
index 000000000..507fa3af3
--- /dev/null
+++ b/tests/primer/packages_to_lint.json
@@ -0,0 +1,59 @@
+{
+  "flask": {
+    "url": "https://github.com/pallets/flask.git",
+    "branch": "main",
+    "directories": ["src/flask"]
+  },
+  "pytest": {
+    "url": "https://github.com/pytest-dev/pytest.git",
+    "branch": "main",
+    "directories": ["src/_pytest"]
+  },
+  "psycopg": {
+    "url": "https://github.com/psycopg/psycopg.git",
+    "branch": "master",
+    "directories": ["psycopg/psycopg"]
+  },
+  "keras": {
+    "url": "https://github.com/keras-team/keras.git",
+    "branch": "master",
+    "directories": ["keras"]
+  },
+  "sentry": {
+    "url": "https://github.com/getsentry/sentry.git",
+    "branch": "master",
+    "directories": ["src/sentry"]
+  },
+  "django": {
+    "url": "https://github.com/django/django.git",
+    "branch": "main",
+    "directories": ["django"]
+  },
+  "pandas": {
+    "url": "https://github.com/pandas-dev/pandas.git",
+    "branch": "master",
+    "directories": ["pandas"],
+    "pylint_additional_args": ["--ignore-patterns=\"test_"]
+  },
+  "black": {
+    "url": "https://github.com/psf/black.git",
+    "branch": "main",
+    "directories": ["src/black/", "src/blackd/", "src/black_primer/", "src/blib2to3/"]
+  },
+  "home-assistant": {
+    "url": "https://github.com/home-assistant/core.git",
+    "branch": "dev",
+    "directories": ["homeassistant"]
+  },
+  "graph-explorer": {
+    "url": "https://github.com/vimeo/graph-explorer.git",
+    "branch": "master",
+    "directories": ["graph_explorer"],
+    "pylintrc_relpath": ".pylintrc"
+  },
+  "pygame": {
+    "url": "https://github.com/pygame/pygame.git",
+    "branch": "main",
+    "directories": ["src_py"]
+  }
+}
diff --git a/tests/primer/test_primer_external.py b/tests/primer/test_primer_external.py
new file mode 100644
index 000000000..e6ba0df67
--- /dev/null
+++ b/tests/primer/test_primer_external.py
@@ -0,0 +1,66 @@
+# Licensed under the GPL: https://www.gnu.org/licenses/old-licenses/gpl-2.0.html
+# For details: https://github.com/PyCQA/pylint/blob/main/LICENSE
+import json
+import logging
+import subprocess
+from pathlib import Path
+from typing import Dict, Union
+
+import pytest
+from pytest import LogCaptureFixture
+
+from pylint.testutils.primer import PackageToLint
+
+PRIMER_DIRECTORY = Path(".pylint_primer_tests/").resolve()
+
+
+def get_packages_to_lint_from_json(
+    json_path: Union[Path, str]
+) -> Dict[str, PackageToLint]:
+    result: Dict[str, PackageToLint] = {}
+    with open(json_path, encoding="utf8") as f:
+        for name, package_data in json.load(f).items():
+            result[name] = PackageToLint(**package_data)
+    return result
+
+
+PACKAGE_TO_LINT_JSON = Path(__file__).parent / "packages_to_lint.json"
+PACKAGES_TO_LINT = get_packages_to_lint_from_json(PACKAGE_TO_LINT_JSON)
+"""Dictionary of external packages used during the primer test"""
+
+
+class TestPrimer:
+    @staticmethod
+    @pytest.mark.primer
+    @pytest.mark.parametrize("package", PACKAGES_TO_LINT.values(), ids=PACKAGES_TO_LINT)
+    def test_primer_external_packages_no_crash(
+        package: PackageToLint,
+        caplog: LogCaptureFixture,
+    ) -> None:
+        __tracebackhide__ = True  # pylint: disable=unused-variable
+        TestPrimer._primer_test(package, caplog)
+
+    @staticmethod
+    def _primer_test(package: PackageToLint, caplog: LogCaptureFixture) -> None:
+        """Runs pylint over external packages to check for crashes and fatal messages
+
+        We only check for crashes (bit-encoded exit code 32) and fatal messages
+        (bit-encoded exit code 1). We assume that these external repositories do not
+        have any fatal errors in their code so that any fatal errors are pylint false
+        positives
+        """
+        caplog.set_level(logging.INFO)
+        package.lazy_clone()
+
+        try:
+            # We want to test all the code we can
+            enables = ["--enable-all-extensions", "--enable=all"]
+            # Duplicate code takes too long and is relatively safe
+            disables = ["--disable=duplicate-code"]
+            command = ["pylint"] + enables + disables + package.pylint_args
+            logging.info("Launching primer:\n%s", " ".join(command))
+            subprocess.run(command, check=True)
+        except subprocess.CalledProcessError as ex:
+            msg = f"Encountered {{}} during primer test for {package}"
+            assert ex.returncode != 32, msg.format("a crash")
+            assert ex.returncode % 2 == 0, msg.format("a message of category 'fatal'")
diff --git a/tests/primer/test_primer_stdlib.py b/tests/primer/test_primer_stdlib.py
index fdc16adcd..48264e6df 100644
--- a/tests/primer/test_primer_stdlib.py
+++ b/tests/primer/test_primer_stdlib.py
@@ -39,14 +39,15 @@ MODULES_TO_CHECK = [
 MODULES_NAMES = [m[1] for m in MODULES_TO_CHECK]
 
 
-@pytest.mark.primer_stdlib
+@pytest.mark.primer
 @pytest.mark.parametrize(
     ("test_module_location", "test_module_name"), MODULES_TO_CHECK, ids=MODULES_NAMES
 )
-def test_lib_module_no_crash(
+def test_primer_stdlib_no_crash(
     test_module_location: str, test_module_name: str, capsys: CaptureFixture
 ) -> None:
     """Test that pylint does not produces any crashes or fatal errors on stdlib modules"""
+    __tracebackhide__ = True  # pylint: disable=unused-variable
     os.chdir(test_module_location)
     with _patch_stdout(io.StringIO()):
         try: