Add support for --fail-under flag

Add a --fail-under <score> flag, also configurable in a `.pylintrc`file

If the final score is more than the specified score, it's considered a
success and pylint exits with exit code 0. Otherwise, it's considered
a failure and pylint exits with its current exit code based on the
messages issued.

Close #2242

7 files changed, 72 insertions, 5 deletions

diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt
index 4d900531a..606d4d0df 100644
--- a/CONTRIBUTORS.txt
+++ b/CONTRIBUTORS.txt
@@ -349,4 +349,6 @@ contributors:
 
 * Enji Cooper: contributor
 
-* Bastien Vallet: contributor
\ No newline at end of file
+* Bastien Vallet: contributor
+
+* Pek Chhan: contributor
diff --git a/ChangeLog b/ChangeLog
index 8cfec00e6..fc7b8a1d1 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -7,11 +7,15 @@ What's New in Pylint 2.5.0?
 
 Release date: TBA
 
+* Add a --fail-under <score> flag, also configurable in a .pylintrc file. If the final score is more than the specified score, it's considered a success and pylint exits with exitcode 0. Otherwise, it's considered a failure and pylint exits with its current exitcode based on the messages issued.
+
+  Close #2242
+
 * Don't emit ``line-too-long`` for multilines when a
   `pylint:disable=line-too-long` comment stands at their end
 
   Close #2957
-  
+
 * Do not exempt bare except from ``undefined-variable`` and similar checks
 
   If a node was wrapped in a ``TryExcept``, ``pylint`` was taking a hint
diff --git a/doc/whatsnew/2.5.rst b/doc/whatsnew/2.5.rst
index f73d3c2b6..a764799ce 100644
--- a/doc/whatsnew/2.5.rst
+++ b/doc/whatsnew/2.5.rst
@@ -51,3 +51,5 @@ separated list of regexes, that if a name matches will be exempt of naming-check
 separated list of regexes, that if a name matches will be always marked as a blacklisted name.
 
 * Mutable ``collections.*`` are now flagged as dangerous defaults.
+
+* Add new --fail-under flag for setting the threshold for the score to fail overall tests. If the score is over the fail-under threshold, pylint will complete SystemExit with value 0 to indicate no errors.
diff --git a/pylint/lint.py b/pylint/lint.py
index b84f77f6a..c85d512ec 100644
--- a/pylint/lint.py
+++ b/pylint/lint.py
@@ -372,6 +372,15 @@ class PyLinter(
                 },
             ),
             (
+                "fail-under",
+                {
+                    "default": 10,
+                    "type": "int",
+                    "metavar": "<score>",
+                    "help": "Specify a score threshold to be exceeded before program exits with error.",
+                },
+            ),
+            (
                 "confidence",
                 {
                     "type": "multiple_choice",
@@ -1234,20 +1243,23 @@ class PyLinter(
 
             if self.config.reports:
                 self.reporter.display_reports(sect)
-            self._report_evaluation()
+            score_value = self._report_evaluation()
             # save results if persistent run
             if self.config.persistent:
                 config.save_results(self.stats, self.file_state.base_name)
         else:
             self.reporter.on_close(self.stats, {})
+            score_value = None
+        return score_value
 
     def _report_evaluation(self):
         """make the global evaluation report"""
         # check with at least check 1 statements (usually 0 when there is a
         # syntax error preventing pylint from further processing)
+        note = None
         previous_stats = config.load_results(self.file_state.base_name)
         if self.stats["statement"] == 0:
-            return
+            return note
 
         # get a global note for the code
         evaluation = self.config.evaluation
@@ -1265,6 +1277,7 @@ class PyLinter(
         if self.config.score:
             sect = report_nodes.EvaluationSection(msg)
             self.reporter.display_reports(sect)
+        return note
 
 
 def check_parallel(linter, jobs, files):
@@ -1756,11 +1769,13 @@ group are mutually exclusive.",
         # behaviour
         with fix_import_path(args):
             linter.check(args)
-            linter.generate_reports()
+            score_value = linter.generate_reports()
         if do_exit:
             if linter.config.exit_zero:
                 sys.exit(0)
             else:
+                if score_value and score_value > linter.config.fail_under:
+                    sys.exit(0)
                 sys.exit(self.linter.msg_status)
 
     def cb_set_rcfile(self, name, value):
diff --git a/tests/input/fail_under_minus6.py b/tests/input/fail_under_minus6.py
new file mode 100644
index 000000000..7984ffd1b
--- /dev/null
+++ b/tests/input/fail_under_minus6.py
@@ -0,0 +1,16 @@
+"""
+	Pylint score:  -6
+"""
+import os
+
+import nonexistent
+
+
+def loop():
+	count = 0
+	for i in range(5):
+		count += 1
+	print(count)
+
+path =   '/tmp'
+os.path.exists(path)
diff --git a/tests/input/fail_under_plus6.py b/tests/input/fail_under_plus6.py
new file mode 100644
index 000000000..640088a3d
--- /dev/null
+++ b/tests/input/fail_under_plus6.py
@@ -0,0 +1,8 @@
+""""
+    pylint score:  +6
+"""
+
+import os
+
+path = '/tmp'
+os.path.exists(path)
diff --git a/tests/unittest_fail_threshold.py b/tests/unittest_fail_threshold.py
new file mode 100644
index 000000000..89a9fa866
--- /dev/null
+++ b/tests/unittest_fail_threshold.py
@@ -0,0 +1,20 @@
+import sys
+from os.path import abspath, dirname, join
+
+from pylint.lint import Run
+
+HERE = abspath(dirname(__file__))
+
+
+def test_fail_under_plus():
+    try:
+        run = Run(["--fail-under", "-1", join(HERE, "input", "fail_under_plus6.py")])
+    except SystemExit as ex:
+        assert ex.code == 0
+
+
+def test_fail_under_minus():
+    try:
+        run = Run(["--fail-under", "-1", join(HERE, "input", "fail_under_minus6.py")])
+    except SystemExit as ex:
+        assert ex.code != 0