Multiple `output-format` support (#4492)

* implement MultiReporter
* implement ,-separated output-format with :outputs
* add tests for new output-format
* add docs, changelog, whatsnew, contributor entry


Co-authored-by: Pierre Sassoulas <pierre.sassoulas@gmail.com>

8 files changed, 360 insertions, 22 deletions

diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt
index 0e6a1d4d4..d4ebc2443 100644
--- a/CONTRIBUTORS.txt
+++ b/CONTRIBUTORS.txt
@@ -491,3 +491,5 @@ contributors:
   - Added ignore_signatures to duplicate checker
 
 * Jacob Walls: contributor
+
+* ruro: contributor
diff --git a/ChangeLog b/ChangeLog
index 905873ebf..858d0e117 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -72,6 +72,11 @@ modules are added.
 
   Closes #2309
 
+* Allow comma-separated list in ``output-format`` and separate output files for
+  each specified format.
+
+  Closes #1798
+
 
 What's New in Pylint 2.8.2?
 ===========================
diff --git a/doc/user_guide/output.rst b/doc/user_guide/output.rst
index b329d3057..4d375fdc7 100644
--- a/doc/user_guide/output.rst
+++ b/doc/user_guide/output.rst
@@ -6,6 +6,15 @@ The default format for the output is raw text. You can change this by passing
 pylint the ``--output-format=<value>`` option. Possible values are: json,
 parseable, colorized and msvs (visual studio).
 
+Multiple output formats can be used at the same time by passing
+``--output-format`` a comma-separated list of formats. To change the output file
+for an individual format, specify it after a semicolon. For example, you can
+save a json report to ``somefile`` and print a colorized report to stdout at the
+same time with :
+::
+
+  --output-format=json:somefile,colorized
+
 Moreover you can customize the exact way information are displayed using the
 `--msg-template=<format string>` option. The `format string` uses the
 `Python new format syntax`_ and the following fields are available :
diff --git a/doc/whatsnew/2.9.rst b/doc/whatsnew/2.9.rst
index 644e9a648..ad91ed5d8 100644
--- a/doc/whatsnew/2.9.rst
+++ b/doc/whatsnew/2.9.rst
@@ -41,3 +41,6 @@ Other Changes
   of overridden functions. It aims to separate the functionality of ``arguments-differ``.
 
 * Fix incompatibility with Python 3.6.0 caused by ``typing.Counter`` and ``typing.NoReturn`` usage
+
+* Allow comma-separated list in ``output-format`` and separate output files for
+  each specified format.  Each output file can be defined after a semicolon for example : ``--output-format=json:myfile.json,colorized``
diff --git a/pylint/lint/pylinter.py b/pylint/lint/pylinter.py
index 392b188b4..fc7cee346 100644
--- a/pylint/lint/pylinter.py
+++ b/pylint/lint/pylinter.py
@@ -43,6 +43,14 @@ def _read_stdin():
     return sys.stdin.read()
 
 
+def _load_reporter_by_class(reporter_class: str) -> type:
+    qname = reporter_class
+    module_part = astroid.modutils.get_module_part(qname)
+    module = astroid.modutils.load_module_from_name(module_part)
+    class_name = qname.split(".")[-1]
+    return getattr(module, class_name)
+
+
 # Python Linter class #########################################################
 
 MSGS = {
@@ -451,7 +459,7 @@ class PyLinter(
         messages store / checkers / reporter / astroid manager"""
         self.msgs_store = MessageDefinitionStore()
         self.reporter = None
-        self._reporter_name = None
+        self._reporter_names = None
         self._reporters = {}
         self._checkers = collections.defaultdict(list)
         self._pragma_lineno = {}
@@ -502,7 +510,7 @@ class PyLinter(
         # Make sure to load the default reporter, because
         # the option has been set before the plugins had been loaded.
         if not self.reporter:
-            self._load_reporter()
+            self._load_reporters()
 
     def load_plugin_modules(self, modnames):
         """take a list of module names which are pylint plugins and load
@@ -527,25 +535,49 @@ class PyLinter(
             if hasattr(module, "load_configuration"):
                 module.load_configuration(self)
 
-    def _load_reporter(self):
-        name = self._reporter_name.lower()
-        if name in self._reporters:
-            self.set_reporter(self._reporters[name]())
+    def _load_reporters(self) -> None:
+        sub_reporters = []
+        output_files = []
+        with contextlib.ExitStack() as stack:
+            for reporter_name in self._reporter_names.split(","):
+                reporter_name, *reporter_output = reporter_name.split(":", 1)
+
+                reporter = self._load_reporter_by_name(reporter_name)
+                sub_reporters.append(reporter)
+
+                if reporter_output:
+                    (reporter_output,) = reporter_output
+
+                    # pylint: disable=consider-using-with
+                    output_file = stack.enter_context(open(reporter_output, "w"))
+
+                    reporter.set_output(output_file)
+                    output_files.append(output_file)
+
+            # Extend the lifetime of all opened output files
+            close_output_files = stack.pop_all().close
+
+        if len(sub_reporters) > 1 or output_files:
+            self.set_reporter(
+                reporters.MultiReporter(
+                    sub_reporters,
+                    close_output_files,
+                )
+            )
         else:
-            try:
-                reporter_class = self._load_reporter_class()
-            except (ImportError, AttributeError) as e:
-                raise exceptions.InvalidReporterError(name) from e
-            else:
-                self.set_reporter(reporter_class())
+            self.set_reporter(sub_reporters[0])
 
-    def _load_reporter_class(self):
-        qname = self._reporter_name
-        module_part = astroid.modutils.get_module_part(qname)
-        module = astroid.modutils.load_module_from_name(module_part)
-        class_name = qname.split(".")[-1]
-        reporter_class = getattr(module, class_name)
-        return reporter_class
+    def _load_reporter_by_name(self, reporter_name: str) -> reporters.BaseReporter:
+        name = reporter_name.lower()
+        if name in self._reporters:
+            return self._reporters[name]()
+
+        try:
+            reporter_class = _load_reporter_by_class(reporter_name)
+        except (ImportError, AttributeError) as e:
+            raise exceptions.InvalidReporterError(name) from e
+        else:
+            return reporter_class()
 
     def set_reporter(self, reporter):
         """set the reporter used to display messages and reports"""
@@ -575,11 +607,11 @@ class PyLinter(
                     meth(value)
                 return  # no need to call set_option, disable/enable methods do it
         elif optname == "output-format":
-            self._reporter_name = value
+            self._reporter_names = value
             # If the reporters are already available, load
             # the reporter class.
             if self._reporters:
-                self._load_reporter()
+                self._load_reporters()
 
         try:
             checkers.BaseTokenChecker.set_option(self, optname, value, action, optdict)
diff --git a/pylint/reporters/__init__.py b/pylint/reporters/__init__.py
index d5433f227..1721be7b6 100644
--- a/pylint/reporters/__init__.py
+++ b/pylint/reporters/__init__.py
@@ -26,6 +26,7 @@ from pylint import utils
 from pylint.reporters.base_reporter import BaseReporter
 from pylint.reporters.collecting_reporter import CollectingReporter
 from pylint.reporters.json_reporter import JSONReporter
+from pylint.reporters.multi_reporter import MultiReporter
 from pylint.reporters.reports_handler_mix_in import ReportsHandlerMixIn
 
 
@@ -34,4 +35,10 @@ def initialize(linter):
     utils.register_plugins(linter, __path__[0])
 
 
-__all__ = ["BaseReporter", "ReportsHandlerMixIn", "JSONReporter", "CollectingReporter"]
+__all__ = [
+    "BaseReporter",
+    "ReportsHandlerMixIn",
+    "JSONReporter",
+    "CollectingReporter",
+    "MultiReporter",
+]
diff --git a/pylint/reporters/multi_reporter.py b/pylint/reporters/multi_reporter.py
new file mode 100644
index 000000000..5723dc160
--- /dev/null
+++ b/pylint/reporters/multi_reporter.py
@@ -0,0 +1,102 @@
+# Licensed under the GPL: https://www.gnu.org/licenses/old-licenses/gpl-2.0.html
+# For details: https://github.com/PyCQA/pylint/blob/master/LICENSE
+
+
+import os
+from typing import IO, Any, AnyStr, Callable, List, Mapping, Optional, Union
+
+from pylint.interfaces import IReporter
+from pylint.reporters.base_reporter import BaseReporter
+from pylint.reporters.ureports.nodes import BaseLayout
+
+AnyFile = IO[AnyStr]
+AnyPath = Union[str, bytes, os.PathLike]
+PyLinter = Any
+
+
+class MultiReporter:
+    """Reports messages and layouts in plain text"""
+
+    __implements__ = IReporter
+    name = "_internal_multi_reporter"
+    # Note: do not register this reporter with linter.register_reporter as it is
+    #       not intended to be used directly like a regular reporter, but is
+    #       instead used to implement the
+    #       `--output-format=json:somefile.json,colorized`
+    #       multiple output formats feature
+
+    extension = ""
+
+    def __init__(
+        self,
+        sub_reporters: List[BaseReporter],
+        close_output_files: Callable[[], None],
+        output: Optional[AnyFile] = None,
+    ):
+        self._sub_reporters = sub_reporters
+        self.close_output_files = close_output_files
+
+        self._path_strip_prefix = os.getcwd() + os.sep
+        self._linter: Optional[PyLinter] = None
+
+        self.set_output(output)
+
+    def __del__(self):
+        self.close_output_files()
+
+    @property
+    def path_strip_prefix(self) -> str:
+        return self._path_strip_prefix
+
+    @property
+    def linter(self) -> Optional[PyLinter]:
+        return self._linter
+
+    @linter.setter
+    def linter(self, value: PyLinter) -> None:
+        self._linter = value
+        for rep in self._sub_reporters:
+            rep.linter = value
+
+    def handle_message(self, msg: str) -> None:
+        """Handle a new message triggered on the current file."""
+        for rep in self._sub_reporters:
+            rep.handle_message(msg)
+
+    # pylint: disable=no-self-use
+    def set_output(self, output: Optional[AnyFile] = None) -> None:
+        """set output stream"""
+        # MultiReporter doesn't have it's own output. This method is only
+        # provided for API parity with BaseReporter and should not be called
+        # with non-None values for 'output'.
+        if output is not None:
+            raise NotImplementedError("MultiReporter does not support direct output.")
+
+    def writeln(self, string: str = "") -> None:
+        """write a line in the output buffer"""
+        for rep in self._sub_reporters:
+            rep.writeln(string)
+
+    def display_reports(self, layout: BaseLayout) -> None:
+        """display results encapsulated in the layout tree"""
+        for rep in self._sub_reporters:
+            rep.display_reports(layout)
+
+    def display_messages(self, layout: BaseLayout) -> None:
+        """hook for displaying the messages of the reporter"""
+        for rep in self._sub_reporters:
+            rep.display_messages(layout)
+
+    def on_set_current_module(self, module: str, filepath: Optional[AnyPath]) -> None:
+        """hook called when a module starts to be analysed"""
+        for rep in self._sub_reporters:
+            rep.on_set_current_module(module, filepath)
+
+    def on_close(
+        self,
+        stats: Mapping[Any, Any],
+        previous_stats: Mapping[Any, Any],
+    ) -> None:
+        """hook called when a module finished analyzing"""
+        for rep in self._sub_reporters:
+            rep.on_close(stats, previous_stats)
diff --git a/tests/unittest_reporting.py b/tests/unittest_reporting.py
index abd912b27..554f325cf 100644
--- a/tests/unittest_reporting.py
+++ b/tests/unittest_reporting.py
@@ -15,12 +15,16 @@
 # pylint: disable=redefined-outer-name
 
 import warnings
+from contextlib import redirect_stdout
 from io import StringIO
+from json import dumps
 
 import pytest
 
 from pylint import checkers
+from pylint.interfaces import IReporter
 from pylint.lint import PyLinter
+from pylint.reporters import BaseReporter
 from pylint.reporters.text import ParseableTextReporter, TextReporter
 
 
@@ -73,6 +77,180 @@ def test_parseable_output_regression():
     )
 
 
+class NopReporter(BaseReporter):
+    __implements__ = IReporter
+    name = "nop-reporter"
+    extension = ""
+
+    def __init__(self, output=None):
+        super().__init__(output)
+        print("A NopReporter was initialized.", file=self.out)
+
+    def writeln(self, string=""):
+        pass
+
+    def _display(self, layout):
+        pass
+
+
+def test_multi_format_output(tmp_path):
+    text = StringIO(newline=None)
+    json = tmp_path / "somefile.json"
+
+    source_file = tmp_path / "somemodule.py"
+    source_file.write_text('NOT_EMPTY = "This module is not empty"\n')
+    escaped_source_file = dumps(str(source_file))
+
+    nop_format = NopReporter.__module__ + "." + NopReporter.__name__
+    formats = ",".join(["json:" + str(json), "text", nop_format])
+
+    with redirect_stdout(text):
+        linter = PyLinter()
+        linter.set_option("persistent", False)
+        linter.set_option("output-format", formats)
+        linter.set_option("reports", True)
+        linter.set_option("score", True)
+        linter.load_default_plugins()
+
+        assert linter.reporter.linter is linter
+        with pytest.raises(NotImplementedError):
+            linter.reporter.set_output(text)
+
+        linter.open()
+        linter.check_single_file("somemodule", source_file, "somemodule")
+        linter.add_message("line-too-long", line=1, args=(1, 2))
+        linter.generate_reports()
+        linter.reporter.writeln("direct output")
+
+        # Ensure the output files are flushed and closed
+        linter.reporter.close_output_files()
+        del linter.reporter
+
+    with open(json) as f:
+        assert (
+            f.read() == "[\n"
+            "    {\n"
+            '        "type": "convention",\n'
+            '        "module": "somemodule",\n'
+            '        "obj": "",\n'
+            '        "line": 1,\n'
+            '        "column": 0,\n'
+            f'        "path": {escaped_source_file},\n'
+            '        "symbol": "missing-module-docstring",\n'
+            '        "message": "Missing module docstring",\n'
+            '        "message-id": "C0114"\n'
+            "    },\n"
+            "    {\n"
+            '        "type": "convention",\n'
+            '        "module": "somemodule",\n'
+            '        "obj": "",\n'
+            '        "line": 1,\n'
+            '        "column": 0,\n'
+            f'        "path": {escaped_source_file},\n'
+            '        "symbol": "line-too-long",\n'
+            '        "message": "Line too long (1/2)",\n'
+            '        "message-id": "C0301"\n'
+            "    }\n"
+            "]\n"
+            "direct output\n"
+        )
+
+    assert (
+        text.getvalue() == "A NopReporter was initialized.\n"
+        "************* Module somemodule\n"
+        f"{source_file}:1:0: C0114: Missing module docstring (missing-module-docstring)\n"
+        f"{source_file}:1:0: C0301: Line too long (1/2) (line-too-long)\n"
+        "\n"
+        "\n"
+        "Report\n"
+        "======\n"
+        "1 statements analysed.\n"
+        "\n"
+        "Statistics by type\n"
+        "------------------\n"
+        "\n"
+        "+---------+-------+-----------+-----------+------------+---------+\n"
+        "|type     |number |old number |difference |%documented |%badname |\n"
+        "+=========+=======+===========+===========+============+=========+\n"
+        "|module   |1      |NC         |NC         |0.00        |0.00     |\n"
+        "+---------+-------+-----------+-----------+------------+---------+\n"
+        "|class    |0      |NC         |NC         |0           |0        |\n"
+        "+---------+-------+-----------+-----------+------------+---------+\n"
+        "|method   |0      |NC         |NC         |0           |0        |\n"
+        "+---------+-------+-----------+-----------+------------+---------+\n"
+        "|function |0      |NC         |NC         |0           |0        |\n"
+        "+---------+-------+-----------+-----------+------------+---------+\n"
+        "\n"
+        "\n"
+        "\n"
+        "Raw metrics\n"
+        "-----------\n"
+        "\n"
+        "+----------+-------+------+---------+-----------+\n"
+        "|type      |number |%     |previous |difference |\n"
+        "+==========+=======+======+=========+===========+\n"
+        "|code      |2      |66.67 |NC       |NC         |\n"
+        "+----------+-------+------+---------+-----------+\n"
+        "|docstring |0      |0.00  |NC       |NC         |\n"
+        "+----------+-------+------+---------+-----------+\n"
+        "|comment   |0      |0.00  |NC       |NC         |\n"
+        "+----------+-------+------+---------+-----------+\n"
+        "|empty     |1      |33.33 |NC       |NC         |\n"
+        "+----------+-------+------+---------+-----------+\n"
+        "\n"
+        "\n"
+        "\n"
+        "Duplication\n"
+        "-----------\n"
+        "\n"
+        "+-------------------------+------+---------+-----------+\n"
+        "|                         |now   |previous |difference |\n"
+        "+=========================+======+=========+===========+\n"
+        "|nb duplicated lines      |0     |NC       |NC         |\n"
+        "+-------------------------+------+---------+-----------+\n"
+        "|percent duplicated lines |0.000 |NC       |NC         |\n"
+        "+-------------------------+------+---------+-----------+\n"
+        "\n"
+        "\n"
+        "\n"
+        "Messages by category\n"
+        "--------------------\n"
+        "\n"
+        "+-----------+-------+---------+-----------+\n"
+        "|type       |number |previous |difference |\n"
+        "+===========+=======+=========+===========+\n"
+        "|convention |2      |NC       |NC         |\n"
+        "+-----------+-------+---------+-----------+\n"
+        "|refactor   |0      |NC       |NC         |\n"
+        "+-----------+-------+---------+-----------+\n"
+        "|warning    |0      |NC       |NC         |\n"
+        "+-----------+-------+---------+-----------+\n"
+        "|error      |0      |NC       |NC         |\n"
+        "+-----------+-------+---------+-----------+\n"
+        "\n"
+        "\n"
+        "\n"
+        "Messages\n"
+        "--------\n"
+        "\n"
+        "+-------------------------+------------+\n"
+        "|message id               |occurrences |\n"
+        "+=========================+============+\n"
+        "|missing-module-docstring |1           |\n"
+        "+-------------------------+------------+\n"
+        "|line-too-long            |1           |\n"
+        "+-------------------------+------------+\n"
+        "\n"
+        "\n"
+        "\n"
+        "\n"
+        "-------------------------------------\n"
+        "Your code has been rated at -10.00/10\n"
+        "\n"
+        "direct output\n"
+    )
+
+
 def test_display_results_is_renamed():
     class CustomReporter(TextReporter):
         def _display(self, layout):