Permit to lint to the end in case of crash on a file (#4810)

* Permit to lint to the end in case of crash on a file

And add pre-filled issue template so it's easier to open an issue in pylinthome

* Print full traceback when a crash occur

See discussion here: https://github.com/PyCQA/pylint/pull/4810\#issuecomment-897735720

3 files changed, 85 insertions, 3 deletions

diff --git a/pylint/lint/pylinter.py b/pylint/lint/pylinter.py
index 3202ea8a4..cd5afbc28 100644
--- a/pylint/lint/pylinter.py
+++ b/pylint/lint/pylinter.py
@@ -13,6 +13,7 @@ import warnings
 from io import TextIOWrapper
 
 import astroid
+from astroid import AstroidError
 
 from pylint import checkers, config, exceptions, interfaces, reporters
 from pylint.constants import MAIN_CHECKER_NAME, MSG_TYPES
@@ -23,7 +24,11 @@ from pylint.lint.report_functions import (
     report_messages_stats,
     report_total_messages_stats,
 )
-from pylint.lint.utils import fix_import_path
+from pylint.lint.utils import (
+    fix_import_path,
+    get_fatal_error_message,
+    prepare_crash_report,
+)
 from pylint.message import MessageDefinitionStore, MessagesHandlerMixIn
 from pylint.reporters.ureports import nodes as report_nodes
 from pylint.utils import ASTWalker, FileState, utils
@@ -165,6 +170,8 @@ class PyLinter(
     priority = 0
     level = 0
     msgs = MSGS
+    # Will be used like this : datetime.now().strftime(crash_file_path)
+    crash_file_path: str = "pylint-crash-%Y-%m-%d-%H.txt"
 
     @staticmethod
     def make_options():
@@ -960,7 +967,6 @@ class PyLinter(
 
         files_or_modules is either a string or list of strings presenting modules to check.
         """
-
         self.initialize()
 
         if not isinstance(files_or_modules, (list, tuple)):
@@ -1014,7 +1020,24 @@ class PyLinter(
         """
         with self._astroid_module_checker() as check_astroid_module:
             for name, filepath, modname in file_descrs:
-                self._check_file(get_ast, check_astroid_module, name, filepath, modname)
+                error = None
+                try:
+                    self._check_file(
+                        get_ast, check_astroid_module, name, filepath, modname
+                    )
+                except Exception as ex:  # pylint: disable=broad-except
+                    error = ex
+                    template_path = prepare_crash_report(
+                        error, filepath, self.crash_file_path
+                    )
+                if error is not None:
+                    msg = get_fatal_error_message(filepath, template_path)
+                    if isinstance(error, AstroidError):
+                        symbol = "astroid-error"
+                        msg = (filepath, msg)
+                    else:
+                        symbol = "fatal"
+                    self.add_message(symbol, args=msg)
 
     def _check_file(self, get_ast, check_astroid_module, name, filepath, modname):
         """Check a file using the passed utility functions (get_ast and check_astroid_module)
diff --git a/pylint/lint/utils.py b/pylint/lint/utils.py
index 1f201d864..6c5390cb7 100644
--- a/pylint/lint/utils.py
+++ b/pylint/lint/utils.py
@@ -3,7 +3,12 @@
 
 import contextlib
 import sys
+import traceback
+from datetime import datetime
+from pathlib import Path
+from typing import Union
 
+from pylint.config import PYLINT_HOME
 from pylint.lint.expand_modules import get_python_path
 
 
@@ -11,6 +16,58 @@ class ArgumentPreprocessingError(Exception):
     """Raised if an error occurs during argument preprocessing."""
 
 
+def prepare_crash_report(
+    ex: Exception, filepath: str, crash_file_path: Union[Path, str]
+) -> Path:
+    issue_template_path = (
+        Path(PYLINT_HOME) / datetime.now().strftime(str(crash_file_path))
+    ).resolve()
+    with open(filepath, encoding="utf8") as f:
+        file_content = f.read()
+    template = ""
+    if not issue_template_path.exists():
+        template = """\
+First, please verify that the bug is not already filled:
+https://github.com/PyCQA/pylint/issues/
+
+Then create a new crash issue:
+https://github.com/PyCQA/pylint/issues/new?assignees=&labels=crash%2Cneeds+triage&template=BUG-REPORT.yml
+
+"""
+    template += f"""\
+
+Issue title:
+Crash ``{ex}`` (if possible, be more specific about what made pylint crash)
+Content:
+When parsing the following file:
+
+<!--
+ If sharing the code is not an option, please state so,
+ but providing only the stacktrace would still be helpful.
+ -->
+
+```python
+{file_content}
+```
+
+pylint crashed with a ``{ex.__class__.__name__}`` and with the following stacktrace:
+```
+"""
+    with open(issue_template_path, "a", encoding="utf8") as f:
+        f.write(template)
+        traceback.print_exc(file=f)
+        f.write("```\n")
+    return issue_template_path
+
+
+def get_fatal_error_message(filepath: str, issue_template_path: Path) -> str:
+    return (
+        f"Fatal error while checking '{filepath}'. "
+        f"Please open an issue in our bug tracker so we address this. "
+        f"There is a pre-filled template that you can use in '{issue_template_path}'."
+    )
+
+
 def preprocess_options(args, search_for):
     """look for some options (keys of <search_for>) which have to be processed
     before others
diff --git a/pylint/utils/ast_walker.py b/pylint/utils/ast_walker.py
index 143267399..756889251 100644
--- a/pylint/utils/ast_walker.py
+++ b/pylint/utils/ast_walker.py
@@ -2,6 +2,7 @@
 # For details: https://github.com/PyCQA/pylint/blob/main/LICENSE
 
 import collections
+import traceback
 
 from astroid import nodes
 
@@ -81,5 +82,6 @@ class ASTWalker:
             if self.exception_msg is False:
                 file = getattr(astroid.root(), "file", None)
                 print(f"Exception on node {repr(astroid)} in file '{file}'")
+                traceback.print_exc()
                 self.exception_msg = True
             raise