From 18669a4cba5baceb3dbf5d41cfdc0874e174952a Mon Sep 17 00:00:00 2001 From: ruro Date: Mon, 24 May 2021 21:37:49 +0300 Subject: Multiple `output-format` support (#4492) * implement MultiReporter * implement ,-separated output-format with :outputs * add tests for new output-format * add docs, changelog, whatsnew, contributor entry Co-authored-by: Pierre Sassoulas --- CONTRIBUTORS.txt | 2 + ChangeLog | 5 ++ doc/user_guide/output.rst | 9 ++ doc/whatsnew/2.9.rst | 3 + pylint/lint/pylinter.py | 74 ++++++++++----- pylint/reporters/__init__.py | 9 +- pylint/reporters/multi_reporter.py | 102 +++++++++++++++++++++ tests/unittest_reporting.py | 178 +++++++++++++++++++++++++++++++++++++ 8 files changed, 360 insertions(+), 22 deletions(-) create mode 100644 pylint/reporters/multi_reporter.py diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt index 0e6a1d4d4..d4ebc2443 100644 --- a/CONTRIBUTORS.txt +++ b/CONTRIBUTORS.txt @@ -491,3 +491,5 @@ contributors: - Added ignore_signatures to duplicate checker * Jacob Walls: contributor + +* ruro: contributor diff --git a/ChangeLog b/ChangeLog index 905873ebf..858d0e117 100644 --- a/ChangeLog +++ b/ChangeLog @@ -72,6 +72,11 @@ modules are added. Closes #2309 +* Allow comma-separated list in ``output-format`` and separate output files for + each specified format. + + Closes #1798 + What's New in Pylint 2.8.2? =========================== diff --git a/doc/user_guide/output.rst b/doc/user_guide/output.rst index b329d3057..4d375fdc7 100644 --- a/doc/user_guide/output.rst +++ b/doc/user_guide/output.rst @@ -6,6 +6,15 @@ The default format for the output is raw text. You can change this by passing pylint the ``--output-format=`` option. Possible values are: json, parseable, colorized and msvs (visual studio). +Multiple output formats can be used at the same time by passing +``--output-format`` a comma-separated list of formats. To change the output file +for an individual format, specify it after a semicolon. For example, you can +save a json report to ``somefile`` and print a colorized report to stdout at the +same time with : +:: + + --output-format=json:somefile,colorized + Moreover you can customize the exact way information are displayed using the `--msg-template=` option. The `format string` uses the `Python new format syntax`_ and the following fields are available : diff --git a/doc/whatsnew/2.9.rst b/doc/whatsnew/2.9.rst index 644e9a648..ad91ed5d8 100644 --- a/doc/whatsnew/2.9.rst +++ b/doc/whatsnew/2.9.rst @@ -41,3 +41,6 @@ Other Changes of overridden functions. It aims to separate the functionality of ``arguments-differ``. * Fix incompatibility with Python 3.6.0 caused by ``typing.Counter`` and ``typing.NoReturn`` usage + +* Allow comma-separated list in ``output-format`` and separate output files for + each specified format. Each output file can be defined after a semicolon for example : ``--output-format=json:myfile.json,colorized`` diff --git a/pylint/lint/pylinter.py b/pylint/lint/pylinter.py index 392b188b4..fc7cee346 100644 --- a/pylint/lint/pylinter.py +++ b/pylint/lint/pylinter.py @@ -43,6 +43,14 @@ def _read_stdin(): return sys.stdin.read() +def _load_reporter_by_class(reporter_class: str) -> type: + qname = reporter_class + module_part = astroid.modutils.get_module_part(qname) + module = astroid.modutils.load_module_from_name(module_part) + class_name = qname.split(".")[-1] + return getattr(module, class_name) + + # Python Linter class ######################################################### MSGS = { @@ -451,7 +459,7 @@ class PyLinter( messages store / checkers / reporter / astroid manager""" self.msgs_store = MessageDefinitionStore() self.reporter = None - self._reporter_name = None + self._reporter_names = None self._reporters = {} self._checkers = collections.defaultdict(list) self._pragma_lineno = {} @@ -502,7 +510,7 @@ class PyLinter( # Make sure to load the default reporter, because # the option has been set before the plugins had been loaded. if not self.reporter: - self._load_reporter() + self._load_reporters() def load_plugin_modules(self, modnames): """take a list of module names which are pylint plugins and load @@ -527,25 +535,49 @@ class PyLinter( if hasattr(module, "load_configuration"): module.load_configuration(self) - def _load_reporter(self): - name = self._reporter_name.lower() - if name in self._reporters: - self.set_reporter(self._reporters[name]()) + def _load_reporters(self) -> None: + sub_reporters = [] + output_files = [] + with contextlib.ExitStack() as stack: + for reporter_name in self._reporter_names.split(","): + reporter_name, *reporter_output = reporter_name.split(":", 1) + + reporter = self._load_reporter_by_name(reporter_name) + sub_reporters.append(reporter) + + if reporter_output: + (reporter_output,) = reporter_output + + # pylint: disable=consider-using-with + output_file = stack.enter_context(open(reporter_output, "w")) + + reporter.set_output(output_file) + output_files.append(output_file) + + # Extend the lifetime of all opened output files + close_output_files = stack.pop_all().close + + if len(sub_reporters) > 1 or output_files: + self.set_reporter( + reporters.MultiReporter( + sub_reporters, + close_output_files, + ) + ) else: - try: - reporter_class = self._load_reporter_class() - except (ImportError, AttributeError) as e: - raise exceptions.InvalidReporterError(name) from e - else: - self.set_reporter(reporter_class()) + self.set_reporter(sub_reporters[0]) - def _load_reporter_class(self): - qname = self._reporter_name - module_part = astroid.modutils.get_module_part(qname) - module = astroid.modutils.load_module_from_name(module_part) - class_name = qname.split(".")[-1] - reporter_class = getattr(module, class_name) - return reporter_class + def _load_reporter_by_name(self, reporter_name: str) -> reporters.BaseReporter: + name = reporter_name.lower() + if name in self._reporters: + return self._reporters[name]() + + try: + reporter_class = _load_reporter_by_class(reporter_name) + except (ImportError, AttributeError) as e: + raise exceptions.InvalidReporterError(name) from e + else: + return reporter_class() def set_reporter(self, reporter): """set the reporter used to display messages and reports""" @@ -575,11 +607,11 @@ class PyLinter( meth(value) return # no need to call set_option, disable/enable methods do it elif optname == "output-format": - self._reporter_name = value + self._reporter_names = value # If the reporters are already available, load # the reporter class. if self._reporters: - self._load_reporter() + self._load_reporters() try: checkers.BaseTokenChecker.set_option(self, optname, value, action, optdict) diff --git a/pylint/reporters/__init__.py b/pylint/reporters/__init__.py index d5433f227..1721be7b6 100644 --- a/pylint/reporters/__init__.py +++ b/pylint/reporters/__init__.py @@ -26,6 +26,7 @@ from pylint import utils from pylint.reporters.base_reporter import BaseReporter from pylint.reporters.collecting_reporter import CollectingReporter from pylint.reporters.json_reporter import JSONReporter +from pylint.reporters.multi_reporter import MultiReporter from pylint.reporters.reports_handler_mix_in import ReportsHandlerMixIn @@ -34,4 +35,10 @@ def initialize(linter): utils.register_plugins(linter, __path__[0]) -__all__ = ["BaseReporter", "ReportsHandlerMixIn", "JSONReporter", "CollectingReporter"] +__all__ = [ + "BaseReporter", + "ReportsHandlerMixIn", + "JSONReporter", + "CollectingReporter", + "MultiReporter", +] diff --git a/pylint/reporters/multi_reporter.py b/pylint/reporters/multi_reporter.py new file mode 100644 index 000000000..5723dc160 --- /dev/null +++ b/pylint/reporters/multi_reporter.py @@ -0,0 +1,102 @@ +# Licensed under the GPL: https://www.gnu.org/licenses/old-licenses/gpl-2.0.html +# For details: https://github.com/PyCQA/pylint/blob/master/LICENSE + + +import os +from typing import IO, Any, AnyStr, Callable, List, Mapping, Optional, Union + +from pylint.interfaces import IReporter +from pylint.reporters.base_reporter import BaseReporter +from pylint.reporters.ureports.nodes import BaseLayout + +AnyFile = IO[AnyStr] +AnyPath = Union[str, bytes, os.PathLike] +PyLinter = Any + + +class MultiReporter: + """Reports messages and layouts in plain text""" + + __implements__ = IReporter + name = "_internal_multi_reporter" + # Note: do not register this reporter with linter.register_reporter as it is + # not intended to be used directly like a regular reporter, but is + # instead used to implement the + # `--output-format=json:somefile.json,colorized` + # multiple output formats feature + + extension = "" + + def __init__( + self, + sub_reporters: List[BaseReporter], + close_output_files: Callable[[], None], + output: Optional[AnyFile] = None, + ): + self._sub_reporters = sub_reporters + self.close_output_files = close_output_files + + self._path_strip_prefix = os.getcwd() + os.sep + self._linter: Optional[PyLinter] = None + + self.set_output(output) + + def __del__(self): + self.close_output_files() + + @property + def path_strip_prefix(self) -> str: + return self._path_strip_prefix + + @property + def linter(self) -> Optional[PyLinter]: + return self._linter + + @linter.setter + def linter(self, value: PyLinter) -> None: + self._linter = value + for rep in self._sub_reporters: + rep.linter = value + + def handle_message(self, msg: str) -> None: + """Handle a new message triggered on the current file.""" + for rep in self._sub_reporters: + rep.handle_message(msg) + + # pylint: disable=no-self-use + def set_output(self, output: Optional[AnyFile] = None) -> None: + """set output stream""" + # MultiReporter doesn't have it's own output. This method is only + # provided for API parity with BaseReporter and should not be called + # with non-None values for 'output'. + if output is not None: + raise NotImplementedError("MultiReporter does not support direct output.") + + def writeln(self, string: str = "") -> None: + """write a line in the output buffer""" + for rep in self._sub_reporters: + rep.writeln(string) + + def display_reports(self, layout: BaseLayout) -> None: + """display results encapsulated in the layout tree""" + for rep in self._sub_reporters: + rep.display_reports(layout) + + def display_messages(self, layout: BaseLayout) -> None: + """hook for displaying the messages of the reporter""" + for rep in self._sub_reporters: + rep.display_messages(layout) + + def on_set_current_module(self, module: str, filepath: Optional[AnyPath]) -> None: + """hook called when a module starts to be analysed""" + for rep in self._sub_reporters: + rep.on_set_current_module(module, filepath) + + def on_close( + self, + stats: Mapping[Any, Any], + previous_stats: Mapping[Any, Any], + ) -> None: + """hook called when a module finished analyzing""" + for rep in self._sub_reporters: + rep.on_close(stats, previous_stats) diff --git a/tests/unittest_reporting.py b/tests/unittest_reporting.py index abd912b27..554f325cf 100644 --- a/tests/unittest_reporting.py +++ b/tests/unittest_reporting.py @@ -15,12 +15,16 @@ # pylint: disable=redefined-outer-name import warnings +from contextlib import redirect_stdout from io import StringIO +from json import dumps import pytest from pylint import checkers +from pylint.interfaces import IReporter from pylint.lint import PyLinter +from pylint.reporters import BaseReporter from pylint.reporters.text import ParseableTextReporter, TextReporter @@ -73,6 +77,180 @@ def test_parseable_output_regression(): ) +class NopReporter(BaseReporter): + __implements__ = IReporter + name = "nop-reporter" + extension = "" + + def __init__(self, output=None): + super().__init__(output) + print("A NopReporter was initialized.", file=self.out) + + def writeln(self, string=""): + pass + + def _display(self, layout): + pass + + +def test_multi_format_output(tmp_path): + text = StringIO(newline=None) + json = tmp_path / "somefile.json" + + source_file = tmp_path / "somemodule.py" + source_file.write_text('NOT_EMPTY = "This module is not empty"\n') + escaped_source_file = dumps(str(source_file)) + + nop_format = NopReporter.__module__ + "." + NopReporter.__name__ + formats = ",".join(["json:" + str(json), "text", nop_format]) + + with redirect_stdout(text): + linter = PyLinter() + linter.set_option("persistent", False) + linter.set_option("output-format", formats) + linter.set_option("reports", True) + linter.set_option("score", True) + linter.load_default_plugins() + + assert linter.reporter.linter is linter + with pytest.raises(NotImplementedError): + linter.reporter.set_output(text) + + linter.open() + linter.check_single_file("somemodule", source_file, "somemodule") + linter.add_message("line-too-long", line=1, args=(1, 2)) + linter.generate_reports() + linter.reporter.writeln("direct output") + + # Ensure the output files are flushed and closed + linter.reporter.close_output_files() + del linter.reporter + + with open(json) as f: + assert ( + f.read() == "[\n" + " {\n" + ' "type": "convention",\n' + ' "module": "somemodule",\n' + ' "obj": "",\n' + ' "line": 1,\n' + ' "column": 0,\n' + f' "path": {escaped_source_file},\n' + ' "symbol": "missing-module-docstring",\n' + ' "message": "Missing module docstring",\n' + ' "message-id": "C0114"\n' + " },\n" + " {\n" + ' "type": "convention",\n' + ' "module": "somemodule",\n' + ' "obj": "",\n' + ' "line": 1,\n' + ' "column": 0,\n' + f' "path": {escaped_source_file},\n' + ' "symbol": "line-too-long",\n' + ' "message": "Line too long (1/2)",\n' + ' "message-id": "C0301"\n' + " }\n" + "]\n" + "direct output\n" + ) + + assert ( + text.getvalue() == "A NopReporter was initialized.\n" + "************* Module somemodule\n" + f"{source_file}:1:0: C0114: Missing module docstring (missing-module-docstring)\n" + f"{source_file}:1:0: C0301: Line too long (1/2) (line-too-long)\n" + "\n" + "\n" + "Report\n" + "======\n" + "1 statements analysed.\n" + "\n" + "Statistics by type\n" + "------------------\n" + "\n" + "+---------+-------+-----------+-----------+------------+---------+\n" + "|type |number |old number |difference |%documented |%badname |\n" + "+=========+=======+===========+===========+============+=========+\n" + "|module |1 |NC |NC |0.00 |0.00 |\n" + "+---------+-------+-----------+-----------+------------+---------+\n" + "|class |0 |NC |NC |0 |0 |\n" + "+---------+-------+-----------+-----------+------------+---------+\n" + "|method |0 |NC |NC |0 |0 |\n" + "+---------+-------+-----------+-----------+------------+---------+\n" + "|function |0 |NC |NC |0 |0 |\n" + "+---------+-------+-----------+-----------+------------+---------+\n" + "\n" + "\n" + "\n" + "Raw metrics\n" + "-----------\n" + "\n" + "+----------+-------+------+---------+-----------+\n" + "|type |number |% |previous |difference |\n" + "+==========+=======+======+=========+===========+\n" + "|code |2 |66.67 |NC |NC |\n" + "+----------+-------+------+---------+-----------+\n" + "|docstring |0 |0.00 |NC |NC |\n" + "+----------+-------+------+---------+-----------+\n" + "|comment |0 |0.00 |NC |NC |\n" + "+----------+-------+------+---------+-----------+\n" + "|empty |1 |33.33 |NC |NC |\n" + "+----------+-------+------+---------+-----------+\n" + "\n" + "\n" + "\n" + "Duplication\n" + "-----------\n" + "\n" + "+-------------------------+------+---------+-----------+\n" + "| |now |previous |difference |\n" + "+=========================+======+=========+===========+\n" + "|nb duplicated lines |0 |NC |NC |\n" + "+-------------------------+------+---------+-----------+\n" + "|percent duplicated lines |0.000 |NC |NC |\n" + "+-------------------------+------+---------+-----------+\n" + "\n" + "\n" + "\n" + "Messages by category\n" + "--------------------\n" + "\n" + "+-----------+-------+---------+-----------+\n" + "|type |number |previous |difference |\n" + "+===========+=======+=========+===========+\n" + "|convention |2 |NC |NC |\n" + "+-----------+-------+---------+-----------+\n" + "|refactor |0 |NC |NC |\n" + "+-----------+-------+---------+-----------+\n" + "|warning |0 |NC |NC |\n" + "+-----------+-------+---------+-----------+\n" + "|error |0 |NC |NC |\n" + "+-----------+-------+---------+-----------+\n" + "\n" + "\n" + "\n" + "Messages\n" + "--------\n" + "\n" + "+-------------------------+------------+\n" + "|message id |occurrences |\n" + "+=========================+============+\n" + "|missing-module-docstring |1 |\n" + "+-------------------------+------------+\n" + "|line-too-long |1 |\n" + "+-------------------------+------------+\n" + "\n" + "\n" + "\n" + "\n" + "-------------------------------------\n" + "Your code has been rated at -10.00/10\n" + "\n" + "direct output\n" + ) + + def test_display_results_is_renamed(): class CustomReporter(TextReporter): def _display(self, layout): -- cgit v1.2.1