From a655e78fa8e9c51c6455806c1f04ef3f839c0526 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20van=20Noord?= <13665637+DanielNoord@users.noreply.github.com> Date: Fri, 1 Oct 2021 11:15:44 +0200 Subject: Move ``merge_stats`` to ``checkerstats.py`` --- pylint/lint/parallel.py | 29 ++++------------------------- pylint/utils/__init__.py | 2 ++ pylint/utils/checkerstats.py | 30 ++++++++++++++++++++++++++++++ 3 files changed, 36 insertions(+), 25 deletions(-) create mode 100644 pylint/utils/checkerstats.py diff --git a/pylint/lint/parallel.py b/pylint/lint/parallel.py index 1896fc95f..b770b7bce 100644 --- a/pylint/lint/parallel.py +++ b/pylint/lint/parallel.py @@ -3,15 +3,13 @@ import collections import functools -from typing import TYPE_CHECKING, Any, DefaultDict, Dict, Iterable, List, Tuple, Union +from typing import Any, DefaultDict, Iterable, List, Tuple from pylint import reporters from pylint.lint.utils import _patch_sys_path from pylint.message import Message -from pylint.typing import CheckerStats, FileItem - -if TYPE_CHECKING: - from typing import Counter # typing.Counter added in Python 3.6.1 +from pylint.typing import FileItem +from pylint.utils import merge_stats try: import multiprocessing @@ -35,25 +33,6 @@ def _get_new_args(message): return (message.msg_id, message.symbol, location, message.msg, message.confidence) -def _merge_stats(stats: List[CheckerStats]): - merged: CheckerStats = {} - by_msg: "Counter[str]" = collections.Counter() - for stat in stats: - message_stats: Union["Counter[str]", Dict] = stat.pop("by_msg", {}) # type: ignore - by_msg.update(message_stats) - - for key, item in stat.items(): - if key not in merged: - merged[key] = item - elif isinstance(item, dict): - merged[key].update(item) # type: ignore - else: - merged[key] = merged[key] + item # type: ignore - - merged["by_msg"] = by_msg - return merged - - def _worker_initialize(linter, arguments=None): global _worker_linter # pylint: disable=global-statement _worker_linter = linter @@ -167,7 +146,7 @@ def check_parallel(linter, jobs, files: Iterable[FileItem], arguments=None): pool.join() _merge_mapreduce_data(linter, all_mapreduce_data) - linter.stats = _merge_stats([linter.stats] + all_stats) + linter.stats = merge_stats([linter.stats] + all_stats) # Insert stats data to local checkers. for checker in linter.get_checkers(): diff --git a/pylint/utils/__init__.py b/pylint/utils/__init__.py index 77cb08b76..a0658bf99 100644 --- a/pylint/utils/__init__.py +++ b/pylint/utils/__init__.py @@ -44,6 +44,7 @@ main pylint class """ from pylint.utils.ast_walker import ASTWalker +from pylint.utils.checkerstats import merge_stats from pylint.utils.file_state import FileState from pylint.utils.utils import ( HAS_ISORT_5, @@ -83,4 +84,5 @@ __all__ = [ "normalize_text", "register_plugins", "tokenize_module", + "merge_stats", ] diff --git a/pylint/utils/checkerstats.py b/pylint/utils/checkerstats.py new file mode 100644 index 000000000..42af08167 --- /dev/null +++ b/pylint/utils/checkerstats.py @@ -0,0 +1,30 @@ +# Licensed under the GPL: https://www.gnu.org/licenses/old-licenses/gpl-2.0.html +# For details: https://github.com/PyCQA/pylint/blob/main/LICENSE + +import collections +from typing import TYPE_CHECKING, Dict, List, Union + +from pylint.typing import CheckerStats + +if TYPE_CHECKING: + from typing import Counter # typing.Counter added in Python 3.6.1 + + +def merge_stats(stats: List[CheckerStats]): + """Used to merge two stats objects into a new one when pylint is run in parallel mode""" + merged: CheckerStats = {} + by_msg: "Counter[str]" = collections.Counter() + for stat in stats: + message_stats: Union["Counter[str]", Dict] = stat.pop("by_msg", {}) # type: ignore + by_msg.update(message_stats) + + for key, item in stat.items(): + if key not in merged: + merged[key] = item + elif isinstance(item, dict): + merged[key].update(item) # type: ignore + else: + merged[key] = merged[key] + item # type: ignore + + merged["by_msg"] = by_msg + return merged -- cgit v1.2.1