Produce a score of 0 for fatal errors and add fatal to score evaluation (#5521)

Co-authored-by: Daniël van Noord <13665637+DanielNoord@users.noreply.github.com>

7 files changed, 33 insertions, 10 deletions

diff --git a/ChangeLog b/ChangeLog
index 265ac67a4..36b75c95d 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -54,6 +54,13 @@ Release date: TBA
 * The ``PyLinter`` class will now be initialiazed with a ``TextReporter``
   as its reporter if none is provided.
 
+* Fatal errors now emit a score of 0.0 regardless of whether the linted module
+  contained any statements
+
+  Closes #5451
+
+* ``fatal`` was added to the variables permitted in score evaluation expressions.
+
 * Fix false positive - Allow unpacking of ``self`` in a subclass of ``typing.NamedTuple``.
 
   Closes #5312
diff --git a/doc/faq.rst b/doc/faq.rst
index 6a401bf25..a2c0f2252 100644
--- a/doc/faq.rst
+++ b/doc/faq.rst
@@ -252,7 +252,7 @@ default value by changing the mixin-class-rgx option.
 Even though the final rating Pylint renders is nominally out of ten, there's no
 lower bound on it. By default, the formula to calculate score is ::
 
-    10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)
+    0 if fatal else 10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)
 
 However, this option can be changed in the Pylint rc file. If having negative
 values really bugs you, you can set the formula to be the maximum of 0 and the
diff --git a/doc/whatsnew/2.13.rst b/doc/whatsnew/2.13.rst
index a0f9a4a70..9c7d93d2c 100644
--- a/doc/whatsnew/2.13.rst
+++ b/doc/whatsnew/2.13.rst
@@ -68,7 +68,14 @@ Other Changes
 
   Closes #5065
 
-* The ``PyLinter`` class will now be initialiazed with a ``TextReporter``
+* Fatal errors now emit a score of 0.0 regardless of whether the linted module
+  contained any statements
+
+  Closes #5451
+
+* ``fatal`` was added to the variables permitted in score evaluation expressions.
+
+* The ``PyLinter`` class will now be initialized with a ``TextReporter``
   as its reporter if none is provided.
 
 * The ``testutils`` for unittests now accept ``end_lineno`` and ``end_column``. Tests
diff --git a/examples/pylintrc b/examples/pylintrc
index 07e870b58..2662898dc 100644
--- a/examples/pylintrc
+++ b/examples/pylintrc
@@ -90,11 +90,11 @@ enable=c-extension-no-member
 [REPORTS]
 
 # Python expression which should return a score less than or equal to 10. You
-# have access to the variables 'error', 'warning', 'refactor', and 'convention'
+# have access to the variables 'fatal', 'error', 'warning', 'refactor', 'convention' and 'info'
 # which contain the number of messages in each category, as well as 'statement'
 # which is the total number of statements analyzed. This score is used by the
 # global evaluation report (RP0004).
-evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)
+evaluation=0 if fatal else 10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)
 
 # Template used to display messages. This is a python new-style format string
 # used to format the message information. See doc for all details.
diff --git a/pylint/lint/pylinter.py b/pylint/lint/pylinter.py
index 3617d9141..451394925 100644
--- a/pylint/lint/pylinter.py
+++ b/pylint/lint/pylinter.py
@@ -764,7 +764,7 @@ class PyLinter(
                         self.enable(msg.msgid)
                         self.fail_on_symbols.append(msg.symbol)
                     elif msg.msgid[0] in fail_on_cats:
-                        # message starts with a cateogry value, flag (but do not enable) it
+                        # message starts with a category value, flag (but do not enable) it
                         self.fail_on_symbols.append(msg.symbol)
 
     def any_fail_on_issues(self):
@@ -1315,6 +1315,7 @@ class PyLinter(
         evaluation = self.config.evaluation
         try:
             stats_dict = {
+                "fatal": self.stats.fatal,
                 "error": self.stats.error,
                 "warning": self.stats.warning,
                 "refactor": self.stats.refactor,
diff --git a/pylintrc b/pylintrc
index 7e983510e..2682bfc1a 100644
--- a/pylintrc
+++ b/pylintrc
@@ -98,11 +98,11 @@ files-output=no
 reports=no
 
 # Python expression which should return a note less than 10 (10 is the highest
-# note). You have access to the variables errors warning, statement which
-# respectively contain the number of errors / warnings messages and the total
-# number of statements analyzed. This is used by the global evaluation report
-# (RP0004).
-evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)
+# note). You have access to the variables 'fatal', 'error', 'warning', 'refactor', 'convention'
+# and 'info', which contain the number of messages in each category, as
+# well as 'statement', which is the total number of statements analyzed. This
+# score is used by the global evaluation report (RP0004).
+evaluation=0 if fatal else 10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)
 
 # Template used to display messages. This is a python new-style format string
 # used to format the message information. See doc for all details
diff --git a/tests/test_self.py b/tests/test_self.py
index ebdb91443..27bf17cf6 100644
--- a/tests/test_self.py
+++ b/tests/test_self.py
@@ -1154,6 +1154,14 @@ class TestRunTC:
         # and errors that are generated they don't affect the exit code.
         self._runtest([path, "--fail-under=-10"] + args, code=expected)
 
+    def test_one_module_fatal_error(self):
+        """
+        Fatal errors in one of several modules linted still exits non-zero.
+        """
+        valid_path = join(HERE, "conftest.py")
+        invalid_path = join(HERE, "garbagePath.py")
+        self._runtest([valid_path, invalid_path], code=1)
+
     @pytest.mark.parametrize(
         "args, expected",
         [