Fail "new primer" runs on fatal errors, like the "old primer" (#6746)

1 files changed, 19 insertions, 1 deletions

diff --git a/tests/primer/primer_tool.py b/tests/primer/primer_tool.py
index 643c1bff8..45c4d1b10 100644
--- a/tests/primer/primer_tool.py
+++ b/tests/primer/primer_tool.py
@@ -7,7 +7,9 @@ from __future__ import annotations
 import argparse
 import json
 import sys
+import warnings
 from io import StringIO
+from itertools import chain
 from pathlib import Path
 from typing import Dict, List, Union
 
@@ -134,6 +136,22 @@ class Primer:
         ) as f:
             json.dump(packages, f)
 
+        # Fail loudly (and fail CI pipelines) if any fatal errors are found,
+        # unless they are astroid-errors, in which case just warn.
+        # This is to avoid introducing a dependency on bleeding-edge astroid
+        # for pylint CI pipelines generally, even though we want to use astroid main
+        # for the purpose of diffing emitted messages and generating PR comments.
+        messages = list(chain.from_iterable(packages.values()))
+        astroid_errors = [msg for msg in messages if msg["symbol"] == "astroid-error"]
+        other_fatal_msgs = [
+            msg
+            for msg in messages
+            if msg["type"] == "fatal" and msg["symbol"] != "astroid-error"
+        ]
+        if astroid_errors:
+            warnings.warn(f"Fatal errors traced to astroid:  {astroid_errors}")
+        assert not other_fatal_msgs, other_fatal_msgs
+
     def _handle_compare_command(self) -> None:
         with open(self.config.base_file, encoding="utf-8") as f:
             main_dict: PackageMessages = json.load(f)
@@ -246,7 +264,7 @@ class Primer:
             arguments += [f"--rcfile={data.pylintrc_relpath}"]
         output = StringIO()
         reporter = JSONReporter(output)
-        Run(arguments, reporter=reporter, do_exit=False)
+        Run(arguments, reporter=reporter, exit=False)
         return json.loads(output.getvalue())
 
     @staticmethod