diff options
-rw-r--r-- | pylint/checkers/__init__.py | 79 | ||||
-rw-r--r-- | pylint/checkers/base.py | 112 | ||||
-rw-r--r-- | pylint/checkers/base_checker.py | 2 | ||||
-rw-r--r-- | pylint/checkers/design_analysis.py | 4 | ||||
-rw-r--r-- | pylint/checkers/imports.py | 22 | ||||
-rw-r--r-- | pylint/checkers/raw_metrics.py | 64 | ||||
-rw-r--r-- | pylint/checkers/similar.py | 24 | ||||
-rw-r--r-- | pylint/config/__init__.py | 8 | ||||
-rw-r--r-- | pylint/lint/parallel.py | 6 | ||||
-rw-r--r-- | pylint/lint/pylinter.py | 52 | ||||
-rw-r--r-- | pylint/lint/report_functions.py | 33 | ||||
-rw-r--r-- | pylint/message/message_handler_mix_in.py | 22 | ||||
-rw-r--r-- | pylint/reporters/base_reporter.py | 6 | ||||
-rw-r--r-- | pylint/reporters/multi_reporter.py | 6 | ||||
-rw-r--r-- | pylint/reporters/reports_handler_mix_in.py | 23 | ||||
-rw-r--r-- | pylint/testutils/unittest_linter.py | 9 | ||||
-rw-r--r-- | pylint/typing.py | 11 | ||||
-rw-r--r-- | pylint/utils/__init__.py | 4 | ||||
-rw-r--r-- | pylint/utils/checkerstats.py | 30 | ||||
-rw-r--r-- | pylint/utils/linterstats.py | 365 | ||||
-rw-r--r-- | tests/lint/unittest_lint.py | 6 | ||||
-rw-r--r-- | tests/test_check_parallel.py | 251 | ||||
-rw-r--r-- | tests/test_import_graph.py | 4 | ||||
-rw-r--r-- | tests/test_regr.py | 6 |
24 files changed, 734 insertions, 415 deletions
diff --git a/pylint/checkers/__init__.py b/pylint/checkers/__init__.py index 9e7ff6eed..273b085a0 100644 --- a/pylint/checkers/__init__.py +++ b/pylint/checkers/__init__.py @@ -47,35 +47,84 @@ messages nor reports. XXX not true, emit a 07 report ! """ -from typing import Iterable, List, Union +import sys +from typing import List, Optional, Tuple, Union from pylint.checkers.base_checker import BaseChecker, BaseTokenChecker from pylint.checkers.deprecated import DeprecatedMixin from pylint.checkers.mapreduce_checker import MapReduceMixin -from pylint.typing import CheckerStats -from pylint.utils import diff_string, register_plugins +from pylint.utils import LinterStats, diff_string, register_plugins + +if sys.version_info >= (3, 8): + from typing import Literal +else: + from typing_extensions import Literal def table_lines_from_stats( - stats: CheckerStats, - old_stats: CheckerStats, - columns: Iterable[str], + stats: LinterStats, + old_stats: Optional[LinterStats], + stat_type: Literal["duplicated_lines", "message_types"], ) -> List[str]: """get values listed in <columns> from <stats> and <old_stats>, and return a formatted list of values, designed to be given to a ureport.Table object """ lines: List[str] = [] - for m_type in columns: - new: Union[int, str] = stats[m_type] # type: ignore - old: Union[int, str, None] = old_stats.get(m_type) # type: ignore - if old is not None: - diff_str = diff_string(old, new) + if stat_type == "duplicated_lines": + new: List[Tuple[str, Union[str, int, float]]] = [ + ("nb_duplicated_lines", stats.duplicated_lines["nb_duplicated_lines"]), + ( + "percent_duplicated_lines", + stats.duplicated_lines["percent_duplicated_lines"], + ), + ] + if old_stats: + old: List[Tuple[str, Union[str, int, float]]] = [ + ( + "nb_duplicated_lines", + old_stats.duplicated_lines["nb_duplicated_lines"], + ), + ( + "percent_duplicated_lines", + old_stats.duplicated_lines["percent_duplicated_lines"], + ), + ] + else: + old = [("nb_duplicated_lines", "NC"), ("percent_duplicated_lines", "NC")] + elif stat_type == "message_types": + new = [ + ("convention", stats.convention), + ("refactor", stats.refactor), + ("warning", stats.warning), + ("error", stats.error), + ] + if old_stats: + old = [ + ("convention", old_stats.convention), + ("refactor", old_stats.refactor), + ("warning", old_stats.warning), + ("error", old_stats.error), + ] else: - old, diff_str = "NC", "NC" - new = f"{new:.3f}" if isinstance(new, float) else str(new) - old = f"{old:.3f}" if isinstance(old, float) else str(old) - lines.extend((m_type.replace("_", " "), new, old, diff_str)) + old = [ + ("convention", "NC"), + ("refactor", "NC"), + ("warning", "NC"), + ("error", "NC"), + ] + + for index, _ in enumerate(new): + new_value = new[index][1] + old_value = old[index][1] + diff_str = ( + diff_string(old_value, new_value) + if isinstance(old_value, float) + else old_value + ) + new_str = f"{new_value:.3f}" if isinstance(new_value, float) else str(new_value) + old_str = f"{old_value:.3f}" if isinstance(old_value, float) else str(old_value) + lines.extend((new[index][0].replace("_", " "), new_str, old_str, diff_str)) return lines diff --git a/pylint/checkers/base.py b/pylint/checkers/base.py index fe685d03b..124881b77 100644 --- a/pylint/checkers/base.py +++ b/pylint/checkers/base.py @@ -68,12 +68,12 @@ import collections import itertools import re import sys -from typing import Any, Dict, Iterator, Optional, Pattern, Union +from typing import Any, Dict, Iterator, Optional, Pattern, cast import astroid from astroid import nodes -from pylint import checkers, exceptions, interfaces +from pylint import checkers, interfaces from pylint import utils as lint_utils from pylint.checkers import utils from pylint.checkers.utils import ( @@ -83,7 +83,12 @@ from pylint.checkers.utils import ( is_property_setter, ) from pylint.reporters.ureports import nodes as reporter_nodes -from pylint.typing import CheckerStats +from pylint.utils import LinterStats + +if sys.version_info >= (3, 8): + from typing import Literal +else: + from typing_extensions import Literal class NamingStyle: @@ -389,8 +394,8 @@ def _has_abstract_methods(node): def report_by_type_stats( sect, - stats: CheckerStats, - old_stats: CheckerStats, + stats: LinterStats, + old_stats: Optional[LinterStats], ): """make a report of @@ -400,38 +405,28 @@ def report_by_type_stats( # percentage of different types documented and/or with a bad name nice_stats: Dict[str, Dict[str, str]] = {} for node_type in ("module", "class", "method", "function"): - try: - total: int = stats[node_type] # type: ignore - except KeyError as e: - raise exceptions.EmptyReportError() from e + node_type = cast(Literal["function", "class", "method", "module"], node_type) + total = stats.get_node_count(node_type) nice_stats[node_type] = {} if total != 0: - try: - undocumented_node: int = stats["undocumented_" + node_type] # type: ignore - documented = total - undocumented_node - percent = (documented * 100.0) / total - nice_stats[node_type]["percent_documented"] = f"{percent:.2f}" - except KeyError: - nice_stats[node_type]["percent_documented"] = "NC" - try: - badname_node: int = stats["badname_" + node_type] # type: ignore - percent = (badname_node * 100.0) / total - nice_stats[node_type]["percent_badname"] = f"{percent:.2f}" - except KeyError: - nice_stats[node_type]["percent_badname"] = "NC" + undocumented_node = stats.get_undocumented(node_type) + documented = total - undocumented_node + percent = (documented * 100.0) / total + nice_stats[node_type]["percent_documented"] = f"{percent:.2f}" + badname_node = stats.get_bad_names(node_type) + percent = (badname_node * 100.0) / total + nice_stats[node_type]["percent_badname"] = f"{percent:.2f}" lines = ["type", "number", "old number", "difference", "%documented", "%badname"] for node_type in ("module", "class", "method", "function"): - new = stats[node_type] - old: Optional[Union[str, int]] = old_stats.get(node_type, None) # type: ignore - if old is not None: - diff_str = lint_utils.diff_string(old, new) - else: - old, diff_str = "NC", "NC" + node_type = cast(Literal["function", "class", "method", "module"], node_type) + new = stats.get_node_count(node_type) + old = old_stats.get_node_count(node_type) if old_stats else None + diff_str = lint_utils.diff_string(old, new) if old else None lines += [ node_type, str(new), - str(old), - diff_str, + str(old) if old else "NC", + diff_str if diff_str else "NC", nice_stats[node_type].get("percent_documented", "0"), nice_stats[node_type].get("percent_badname", "0"), ] @@ -1086,13 +1081,12 @@ class BasicChecker(_BasicChecker): def __init__(self, linter): super().__init__(linter) - self.stats: CheckerStats = {} self._tryfinallys = None def open(self): """initialize visit variables and statistics""" self._tryfinallys = [] - self.stats = self.linter.add_stats(module=0, function=0, method=0, class_=0) + self.linter.stats.reset_node_count() @utils.check_messages("using-constant-test", "missing-parentheses-for-call-in-test") def visit_if(self, node: nodes.If) -> None: @@ -1163,13 +1157,13 @@ class BasicChecker(_BasicChecker): def visit_module(self, _: nodes.Module) -> None: """check module name, docstring and required arguments""" - self.stats["module"] += 1 # type: ignore + self.linter.stats.node_count["module"] += 1 def visit_classdef(self, _: nodes.ClassDef) -> None: """check module name, docstring and redefinition increment branch counter """ - self.stats["class"] += 1 # type: ignore + self.linter.stats.node_count["klass"] += 1 @utils.check_messages( "pointless-statement", "pointless-string-statement", "expression-not-assigned" @@ -1308,7 +1302,10 @@ class BasicChecker(_BasicChecker): """check function name, docstring, arguments, redefinition, variable names, max locals """ - self.stats["method" if node.is_method() else "function"] += 1 # type: ignore + if node.is_method(): + self.linter.stats.node_count["method"] += 1 + else: + self.linter.stats.node_count["function"] += 1 self._check_dangerous_default(node) visit_asyncfunctiondef = visit_functiondef @@ -1846,19 +1843,7 @@ class NameChecker(_BasicChecker): self._non_ascii_rgx_compiled = re.compile("[^\u0000-\u007F]") def open(self): - self.stats = self.linter.add_stats( - badname_module=0, - badname_class=0, - badname_function=0, - badname_method=0, - badname_attr=0, - badname_const=0, - badname_variable=0, - badname_inlinevar=0, - badname_argument=0, - badname_class_attribute=0, - badname_class_const=0, - ) + self.linter.stats.reset_bad_names() for group in self.config.name_group: for name_type in group.split(":"): self._name_group[name_type] = f"group_{group}" @@ -2044,7 +2029,7 @@ class NameChecker(_BasicChecker): ) self.add_message(warning, node=node, args=args, confidence=confidence) - self.stats["badname_" + node_type] += 1 # type: ignore + self.linter.stats.increase_bad_name(node_type, 1) def _name_allowed_by_regex(self, name: str) -> bool: return name in self.config.good_names or any( @@ -2074,7 +2059,7 @@ class NameChecker(_BasicChecker): if self._name_allowed_by_regex(name=name): return if self._name_disallowed_by_regex(name=name): - self.stats["badname_" + node_type] += 1 + self.linter.stats.increase_bad_name(node_type, 1) self.add_message("disallowed-name", node=node, args=name) return regexp = self._name_regexps[node_type] @@ -2168,12 +2153,7 @@ class DocStringChecker(_BasicChecker): ) def open(self): - self.stats = self.linter.add_stats( - undocumented_module=0, - undocumented_function=0, - undocumented_method=0, - undocumented_class=0, - ) + self.linter.stats.reset_undocumented() @utils.check_messages("missing-docstring", "empty-docstring") def visit_module(self, node: nodes.Module) -> None: @@ -2210,17 +2190,21 @@ class DocStringChecker(_BasicChecker): overridden = True break self._check_docstring( - ftype, node, report_missing=not overridden, confidence=confidence + ftype, node, report_missing=not overridden, confidence=confidence # type: ignore ) elif isinstance(node.parent.frame(), nodes.Module): - self._check_docstring(ftype, node) + self._check_docstring(ftype, node) # type: ignore else: return visit_asyncfunctiondef = visit_functiondef def _check_docstring( - self, node_type, node, report_missing=True, confidence=interfaces.HIGH + self, + node_type: Literal["class", "function", "method", "module"], + node, + report_missing=True, + confidence=interfaces.HIGH, ): """check the node has a non empty docstring""" docstring = node.doc @@ -2240,7 +2224,10 @@ class DocStringChecker(_BasicChecker): if node_type != "module" and max_lines > -1 and lines < max_lines: return - self.stats["undocumented_" + node_type] += 1 + if node_type == "class": + self.linter.stats.undocumented["klass"] += 1 + else: + self.linter.stats.undocumented[node_type] += 1 if ( node.body and isinstance(node.body[0], nodes.Expr) @@ -2262,7 +2249,10 @@ class DocStringChecker(_BasicChecker): message = "missing-function-docstring" self.add_message(message, node=node, confidence=confidence) elif not docstring.strip(): - self.stats["undocumented_" + node_type] += 1 + if node_type == "class": + self.linter.stats.undocumented["klass"] += 1 + else: + self.linter.stats.undocumented[node_type] += 1 self.add_message( "empty-docstring", node=node, args=(node_type,), confidence=confidence ) diff --git a/pylint/checkers/base_checker.py b/pylint/checkers/base_checker.py index 938ae792e..328adcb4d 100644 --- a/pylint/checkers/base_checker.py +++ b/pylint/checkers/base_checker.py @@ -27,7 +27,6 @@ from pylint.constants import _MSG_ORDER, WarningScope from pylint.exceptions import InvalidMessageError from pylint.interfaces import Confidence, IRawChecker, ITokenChecker, implements from pylint.message.message_definition import MessageDefinition -from pylint.typing import CheckerStats from pylint.utils import get_rst_section, get_rst_title @@ -55,7 +54,6 @@ class BaseChecker(OptionsProviderMixIn): self.name = self.name.lower() super().__init__() self.linter = linter - self.stats: CheckerStats = {} def __gt__(self, other): """Permit to sort a list of Checker by name.""" diff --git a/pylint/checkers/design_analysis.py b/pylint/checkers/design_analysis.py index 828537793..947e22078 100644 --- a/pylint/checkers/design_analysis.py +++ b/pylint/checkers/design_analysis.py @@ -37,7 +37,6 @@ from pylint import utils from pylint.checkers import BaseChecker from pylint.checkers.utils import check_messages from pylint.interfaces import IAstroidChecker -from pylint.typing import CheckerStats MSGS = { # pylint: disable=consider-using-namedtuple-or-dataclass "R0901": ( @@ -407,14 +406,13 @@ class MisdesignChecker(BaseChecker): def __init__(self, linter=None): super().__init__(linter) - self.stats: CheckerStats = {} self._returns = None self._branches = None self._stmts = None def open(self): """initialize visit variables""" - self.stats = self.linter.add_stats() + self.linter.stats.reset_node_count() self._returns = [] self._branches = defaultdict(int) self._stmts = [] diff --git a/pylint/checkers/imports.py b/pylint/checkers/imports.py index abecc1712..3e0b39d26 100644 --- a/pylint/checkers/imports.py +++ b/pylint/checkers/imports.py @@ -70,7 +70,6 @@ from pylint.graph import DotBackend, get_cycles from pylint.interfaces import IAstroidChecker from pylint.lint import PyLinter from pylint.reporters.ureports.nodes import Paragraph, Section, VerbatimText -from pylint.typing import CheckerStats from pylint.utils import IsortDriver, get_global_option @@ -170,26 +169,27 @@ def _repr_tree_defs(data, indent_str=None): return "\n".join(lines) -def _dependencies_graph(filename: str, dep_info: Dict[str, List[str]]) -> str: +def _dependencies_graph(filename: str, dep_info: Dict[str, Set[str]]) -> str: """write dependencies as a dot (graphviz) file""" done = {} printer = DotBackend(os.path.splitext(os.path.basename(filename))[0], rankdir="LR") printer.emit('URL="." node[shape="box"]') for modname, dependencies in sorted(dep_info.items()): + sorted_dependencies = sorted(dependencies) done[modname] = 1 printer.emit_node(modname) - for depmodname in dependencies: + for depmodname in sorted_dependencies: if depmodname not in done: done[depmodname] = 1 printer.emit_node(depmodname) for depmodname, dependencies in sorted(dep_info.items()): - for modname in dependencies: + for modname in sorted(dependencies): printer.emit_edge(modname, depmodname) return printer.generate(filename) def _make_graph( - filename: str, dep_info: Dict[str, List[str]], sect: Section, gtype: str + filename: str, dep_info: Dict[str, Set[str]], sect: Section, gtype: str ): """generate a dependencies graph and add some information about it in the report's section @@ -428,7 +428,6 @@ class ImportsChecker(DeprecatedMixin, BaseChecker): self, linter: Optional[PyLinter] = None ): # pylint: disable=super-init-not-called # See https://github.com/PyCQA/pylint/issues/4941 BaseChecker.__init__(self, linter) - self.stats: CheckerStats = {} self.import_graph: collections.defaultdict = collections.defaultdict(set) self._imports_stack: List[Tuple[Any, Any]] = [] self._first_non_import_node = None @@ -470,9 +469,8 @@ class ImportsChecker(DeprecatedMixin, BaseChecker): def open(self): """called before visiting project (i.e set of modules)""" - self.linter.add_stats(dependencies={}) - self.linter.add_stats(cycles=[]) - self.stats = self.linter.stats + self.linter.stats.dependencies = {} + self.linter.stats = self.linter.stats self.import_graph = collections.defaultdict(set) self._module_pkg = {} # mapping of modules to the pkg they belong in self._excluded_edges = collections.defaultdict(set) @@ -859,7 +857,7 @@ class ImportsChecker(DeprecatedMixin, BaseChecker): self._module_pkg[context_name] = context_name.rsplit(".", 1)[0] # handle dependencies - dependencies_stat: Dict[str, Union[Set]] = self.stats["dependencies"] # type: ignore + dependencies_stat: Dict[str, Union[Set]] = self.linter.stats.dependencies importedmodnames = dependencies_stat.setdefault(importedmodname, set()) if context_name not in importedmodnames: importedmodnames.add(context_name) @@ -935,7 +933,7 @@ class ImportsChecker(DeprecatedMixin, BaseChecker): def _report_dependencies_graph(self, sect, _, _dummy): """write dependencies as a dot (graphviz) file""" - dep_info = self.stats["dependencies"] + dep_info = self.linter.stats.dependencies if not dep_info or not ( self.config.import_graph or self.config.ext_import_graph @@ -955,7 +953,7 @@ class ImportsChecker(DeprecatedMixin, BaseChecker): def _filter_dependencies_graph(self, internal): """build the internal or the external dependency graph""" graph = collections.defaultdict(set) - for importee, importers in self.stats["dependencies"].items(): + for importee, importers in self.linter.stats.dependencies.items(): for importer in importers: package = self._module_pkg.get(importer, importer) is_inside = importee.startswith(package) diff --git a/pylint/checkers/raw_metrics.py b/pylint/checkers/raw_metrics.py index c8db0d238..42d23dab3 100644 --- a/pylint/checkers/raw_metrics.py +++ b/pylint/checkers/raw_metrics.py @@ -15,38 +15,43 @@ # Licensed under the GPL: https://www.gnu.org/licenses/old-licenses/gpl-2.0.html # For details: https://github.com/PyCQA/pylint/blob/main/LICENSE +import sys import tokenize -from typing import Any, Optional, Union +from typing import Any, Optional, cast from pylint.checkers import BaseTokenChecker -from pylint.exceptions import EmptyReportError from pylint.interfaces import ITokenChecker from pylint.reporters.ureports.nodes import Table -from pylint.typing import CheckerStats -from pylint.utils import diff_string +from pylint.utils import LinterStats, diff_string + +if sys.version_info >= (3, 8): + from typing import Literal +else: + from typing_extensions import Literal def report_raw_stats( sect, - stats: CheckerStats, - old_stats: CheckerStats, -): + stats: LinterStats, + old_stats: Optional[LinterStats], +) -> None: """calculate percentage of code / doc / comment / empty""" - total_lines: int = stats["total_lines"] # type: ignore - if not total_lines: - raise EmptyReportError() + total_lines = stats.code_type_count["total"] sect.description = f"{total_lines} lines have been analyzed" lines = ["type", "number", "%", "previous", "difference"] for node_type in ("code", "docstring", "comment", "empty"): - key = node_type + "_lines" - total: int = stats[key] # type: ignore - percent = float(total * 100) / total_lines - old: Optional[Union[int, str]] = old_stats.get(key, None) # type: ignore - if old is not None: - diff_str = diff_string(old, total) - else: - old, diff_str = "NC", "NC" - lines += [node_type, str(total), f"{percent:.2f}", str(old), diff_str] + node_type = cast(Literal["code", "docstring", "comment", "empty"], node_type) + total = stats.code_type_count[node_type] + percent = float(total * 100) / total_lines if total_lines else None + old = old_stats.code_type_count[node_type] if old_stats else None + diff_str = diff_string(old, total) if old else None + lines += [ + node_type, + str(total), + f"{percent:.2f}" if percent is not None else "NC", + str(old) if old else "NC", + diff_str if diff_str else "NC", + ] sect.append(Table(children=lines, cols=5, rheaders=1)) @@ -72,17 +77,10 @@ class RawMetricsChecker(BaseTokenChecker): def __init__(self, linter): super().__init__(linter) - self.stats: CheckerStats = {} def open(self): """init statistics""" - self.stats = self.linter.add_stats( - total_lines=0, - code_lines=0, - empty_lines=0, - docstring_lines=0, - comment_lines=0, - ) + self.linter.stats.reset_code_count() def process_tokens(self, tokens): """update stats""" @@ -90,8 +88,8 @@ class RawMetricsChecker(BaseTokenChecker): tokens = list(tokens) while i < len(tokens): i, lines_number, line_type = get_type(tokens, i) - self.stats["total_lines"] += lines_number - self.stats[line_type] += lines_number + self.linter.stats.code_type_count["total"] += lines_number + self.linter.stats.code_type_count[line_type] += lines_number JUNK = (tokenize.NL, tokenize.INDENT, tokenize.NEWLINE, tokenize.ENDMARKER) @@ -108,16 +106,16 @@ def get_type(tokens, start_index): pos = tokens[i][3] if line_type is None: if tok_type == tokenize.STRING: - line_type = "docstring_lines" + line_type = "docstring" elif tok_type == tokenize.COMMENT: - line_type = "comment_lines" + line_type = "comment" elif tok_type in JUNK: pass else: - line_type = "code_lines" + line_type = "code" i += 1 if line_type is None: - line_type = "empty_lines" + line_type = "empty" elif i < len(tokens) and tokens[i][0] == tokenize.NEWLINE: i += 1 return i, pos[0] - start[0] + 1, line_type diff --git a/pylint/checkers/similar.py b/pylint/checkers/similar.py index 9ff5a973b..f37453e36 100644 --- a/pylint/checkers/similar.py +++ b/pylint/checkers/similar.py @@ -73,8 +73,7 @@ from astroid import nodes from pylint.checkers import BaseChecker, MapReduceMixin, table_lines_from_stats from pylint.interfaces import IRawChecker from pylint.reporters.ureports.nodes import Table -from pylint.typing import CheckerStats -from pylint.utils import decoding_stream +from pylint.utils import LinterStats, decoding_stream DEFAULT_MIN_SIMILARITY_LINE = 4 @@ -724,14 +723,12 @@ MSGS = { def report_similarities( sect, - stats: CheckerStats, - old_stats: CheckerStats, -): + stats: LinterStats, + old_stats: Optional[LinterStats], +) -> None: """make a layout with some stats about duplication""" lines = ["", "now", "previous", "difference"] - lines += table_lines_from_stats( - stats, old_stats, ("nb_duplicated_lines", "percent_duplicated_lines") - ) + lines += table_lines_from_stats(stats, old_stats, "duplicated_lines") sect.append(Table(children=lines, cols=4, rheaders=1, cheaders=1)) @@ -809,7 +806,6 @@ class SimilarChecker(BaseChecker, Similar, MapReduceMixin): ignore_imports=self.config.ignore_imports, ignore_signatures=self.config.ignore_signatures, ) - self.stats: CheckerStats = {} def set_option(self, optname, value, action=None, optdict=None): """method called to set an option (registered in the options list) @@ -831,9 +827,7 @@ class SimilarChecker(BaseChecker, Similar, MapReduceMixin): def open(self): """init the checkers: reset linesets and statistics information""" self.linesets = [] - self.stats = self.linter.add_stats( - nb_duplicated_lines=0, percent_duplicated_lines=0 - ) + self.linter.stats.reset_duplicated_lines() def process_module(self, node: nodes.Module) -> None: """process a module @@ -849,7 +843,7 @@ class SimilarChecker(BaseChecker, Similar, MapReduceMixin): """compute and display similarities on closing (i.e. end of parsing)""" total = sum(len(lineset) for lineset in self.linesets) duplicated = 0 - stats = self.stats + stats = self.linter.stats for num, couples in self._compute_sims(): msg = [] lineset = start_line = end_line = None @@ -863,8 +857,8 @@ class SimilarChecker(BaseChecker, Similar, MapReduceMixin): self.add_message("R0801", args=(len(couples), "\n".join(msg))) duplicated += num * (len(couples) - 1) - stats["nb_duplicated_lines"] = duplicated - stats["percent_duplicated_lines"] = total and duplicated * 100.0 / total + stats.nb_duplicated_lines += int(duplicated) + stats.percent_duplicated_lines += float(total and duplicated * 100.0 / total) def get_map_data(self): """Passthru override""" diff --git a/pylint/config/__init__.py b/pylint/config/__init__.py index d6e2c1f25..6bf30d3b4 100644 --- a/pylint/config/__init__.py +++ b/pylint/config/__init__.py @@ -49,6 +49,7 @@ from pylint.config.option import Option from pylint.config.option_manager_mixin import OptionsManagerMixIn from pylint.config.option_parser import OptionParser from pylint.config.options_provider_mixin import OptionsProviderMixIn, UnsupportedAction +from pylint.utils import LinterStats __all__ = [ "ConfigurationMixIn", @@ -117,9 +118,12 @@ def load_results(base): data_file = _get_pdata_path(base, 1) try: with open(data_file, "rb") as stream: - return pickle.load(stream) + data = pickle.load(stream) + if not isinstance(data, LinterStats): + raise TypeError + return data except Exception: # pylint: disable=broad-except - return {} + return None def save_results(results, base): diff --git a/pylint/lint/parallel.py b/pylint/lint/parallel.py index b770b7bce..f148e913f 100644 --- a/pylint/lint/parallel.py +++ b/pylint/lint/parallel.py @@ -9,7 +9,7 @@ from pylint import reporters from pylint.lint.utils import _patch_sys_path from pylint.message import Message from pylint.typing import FileItem -from pylint.utils import merge_stats +from pylint.utils import LinterStats, merge_stats try: import multiprocessing @@ -48,7 +48,9 @@ def _worker_initialize(linter, arguments=None): def _worker_check_single_file( file_item: FileItem, -) -> Tuple[int, Any, str, Any, List[Tuple[Any, ...]], Any, Any, DefaultDict[Any, List]]: +) -> Tuple[ + int, Any, str, Any, List[Tuple[Any, ...]], LinterStats, Any, DefaultDict[Any, List] +]: if not _worker_linter: raise Exception("Worker linter not yet initialised") _worker_linter.open() diff --git a/pylint/lint/pylinter.py b/pylint/lint/pylinter.py index 053f5dcb5..728b4855e 100644 --- a/pylint/lint/pylinter.py +++ b/pylint/lint/pylinter.py @@ -32,8 +32,8 @@ from pylint.lint.utils import ( ) from pylint.message import MessageDefinitionStore, MessagesHandlerMixIn from pylint.reporters.ureports import nodes as report_nodes -from pylint.typing import CheckerStats, FileItem, ModuleDescriptionDict -from pylint.utils import ASTWalker, FileState, utils +from pylint.typing import FileItem, ModuleDescriptionDict +from pylint.utils import ASTWalker, FileState, LinterStats, ModuleStats, utils from pylint.utils.pragma_parser import ( OPTION_PO, InvalidPragmaError, @@ -502,9 +502,9 @@ class PyLinter( self._ignore_file = False # visit variables self.file_state = FileState() - self.current_name = None + self.current_name: Optional[str] = None self.current_file = None - self.stats: CheckerStats = {} + self.stats = LinterStats() self.fail_on_symbols = [] # init options self._external_opts = options @@ -731,10 +731,8 @@ class PyLinter( self.fail_on_symbols.append(msg.symbol) def any_fail_on_issues(self): - return ( - self.stats - and self.stats.get("by_msg") is not None - and any(x in self.fail_on_symbols for x in self.stats["by_msg"]) + return self.stats and any( + x in self.fail_on_symbols for x in self.stats.by_msg.keys() ) def disable_noerror_messages(self): @@ -1101,10 +1099,9 @@ class PyLinter( self.reporter.on_set_current_module(modname, filepath) self.current_name = modname self.current_file = filepath or modname - self.stats["by_module"][modname] = {} # type: ignore # Refactor of PyLinter.stats necessary - self.stats["by_module"][modname]["statement"] = 0 # type: ignore - for msg_cat in MSG_TYPES.values(): - self.stats["by_module"][modname][msg_cat] = 0 # type: ignore + self.stats.by_module[modname] = ModuleStats( + convention=0, error=0, fatal=0, info=0, refactor=0, statement=0, warning=0 + ) @contextlib.contextmanager def _astroid_module_checker(self): @@ -1136,7 +1133,7 @@ class PyLinter( ) # notify global end - self.stats["statement"] = walker.nbstatements + self.stats.statement = walker.nbstatements for checker in reversed(_checkers): checker.close() @@ -1181,7 +1178,7 @@ class PyLinter( ast_node, walker, rawcheckers, tokencheckers ) - self.stats["by_module"][self.current_name]["statement"] = ( + self.stats.by_module[self.current_name]["statement"] = ( walker.nbstatements - before_check_statements ) @@ -1231,7 +1228,7 @@ class PyLinter( def open(self): """initialize counters""" - self.stats = {"by_module": {}, "by_msg": {}} + self.stats = LinterStats() MANAGER.always_load_extensions = self.config.unsafe_load_any_extension MANAGER.max_inferable_values = self.config.limit_inference_results MANAGER.extension_package_whitelist.update(self.config.extension_pkg_allow_list) @@ -1239,8 +1236,7 @@ class PyLinter( MANAGER.extension_package_whitelist.update( self.config.extension_pkg_whitelist ) - for msg_cat in MSG_TYPES.values(): - self.stats[msg_cat] = 0 + self.stats.reset_message_count() def generate_reports(self): """close the whole package /module, it's time to make reports ! @@ -1266,7 +1262,7 @@ class PyLinter( if self.config.persistent: config.save_results(self.stats, self.file_state.base_name) else: - self.reporter.on_close(self.stats, {}) + self.reporter.on_close(self.stats, LinterStats()) score_value = None return score_value @@ -1276,21 +1272,29 @@ class PyLinter( # syntax error preventing pylint from further processing) note = None previous_stats = config.load_results(self.file_state.base_name) - if self.stats["statement"] == 0: + if self.stats.statement == 0: return note # get a global note for the code evaluation = self.config.evaluation try: - note = eval(evaluation, {}, self.stats) # pylint: disable=eval-used + stats_dict = { + "error": self.stats.error, + "warning": self.stats.warning, + "refactor": self.stats.refactor, + "convention": self.stats.convention, + "statement": self.stats.statement, + } + note = eval(evaluation, {}, stats_dict) # pylint: disable=eval-used except Exception as ex: # pylint: disable=broad-except msg = f"An exception occurred while rating: {ex}" else: - self.stats["global_note"] = note + self.stats.global_note = note msg = f"Your code has been rated at {note:.2f}/10" - pnote = previous_stats.get("global_note") - if pnote is not None: - msg += f" (previous run: {pnote:.2f}/10, {note - pnote:+.2f})" + if previous_stats: + pnote = previous_stats.global_note + if pnote is not None: + msg += f" (previous run: {pnote:.2f}/10, {note - pnote:+.2f})" if self.config.score: sect = report_nodes.EvaluationSection(msg) diff --git a/pylint/lint/report_functions.py b/pylint/lint/report_functions.py index 201b28b7b..5e95354c4 100644 --- a/pylint/lint/report_functions.py +++ b/pylint/lint/report_functions.py @@ -2,37 +2,32 @@ # For details: https://github.com/PyCQA/pylint/blob/main/LICENSE import collections -from typing import DefaultDict, Dict, List, Tuple, Union +from typing import DefaultDict, Dict, Union from pylint import checkers, exceptions from pylint.reporters.ureports.nodes import Table -from pylint.typing import CheckerStats +from pylint.utils import LinterStats def report_total_messages_stats( sect, - stats: CheckerStats, - previous_stats: CheckerStats, + stats: LinterStats, + previous_stats: LinterStats, ): """make total errors / warnings report""" lines = ["type", "number", "previous", "difference"] - lines += checkers.table_lines_from_stats( - stats, previous_stats, ("convention", "refactor", "warning", "error") - ) + lines += checkers.table_lines_from_stats(stats, previous_stats, "message_types") sect.append(Table(children=lines, cols=4, rheaders=1)) def report_messages_stats( sect, - stats: CheckerStats, - _: CheckerStats, + stats: LinterStats, + _: LinterStats, ): """make messages type report""" - if not stats["by_msg"]: - # don't print this report when we didn't detected any errors - raise exceptions.EmptyReportError() - by_msg_stats: Dict[str, int] = stats["by_msg"] # type: ignore - in_order: List[Tuple[int, str]] = sorted( + by_msg_stats = stats.by_msg + in_order = sorted( (value, msg_id) for msg_id, value in by_msg_stats.items() if not msg_id.startswith("I") @@ -46,11 +41,11 @@ def report_messages_stats( def report_messages_by_module_stats( sect, - stats: CheckerStats, - _: CheckerStats, + stats: LinterStats, + _: LinterStats, ): """make errors / warnings by modules report""" - module_stats: Dict[str, Dict[str, int]] = stats["by_module"] # type: ignore + module_stats = stats.by_module if len(module_stats) == 1: # don't print this report when we are analysing a single module raise exceptions.EmptyReportError() @@ -58,9 +53,9 @@ def report_messages_by_module_stats( dict ) for m_type in ("fatal", "error", "warning", "refactor", "convention"): - total: int = stats[m_type] # type: ignore + total = stats.get_global_message_count(m_type) for module in module_stats.keys(): - mod_total = module_stats[module][m_type] + mod_total = stats.get_module_message_count(module, m_type) percent = 0 if total == 0 else float(mod_total * 100) / total by_mod[module][m_type] = percent sorted_result = [] diff --git a/pylint/message/message_handler_mix_in.py b/pylint/message/message_handler_mix_in.py index 662a086cb..db8f6d0be 100644 --- a/pylint/message/message_handler_mix_in.py +++ b/pylint/message/message_handler_mix_in.py @@ -332,23 +332,13 @@ class MessagesHandlerMixIn: # update stats msg_cat = MSG_TYPES[message_definition.msgid[0]] self.msg_status |= MSG_TYPES_STATUS[message_definition.msgid[0]] - if self.stats is None: - # pylint: disable=fixme - # TODO self.stats should make sense, - # class should make sense as soon as instantiated - # This is not true for Linter and Reporter at least - # pylint: enable=fixme - self.stats = { - msg_cat: 0, - "by_module": {self.current_name: {msg_cat: 0}}, - "by_msg": {}, - } - self.stats[msg_cat] += 1 # type: ignore - self.stats["by_module"][self.current_name][msg_cat] += 1 # type: ignore + + self.stats.increase_single_message_count(msg_cat, 1) + self.stats.increase_single_module_message_count(self.current_name, msg_cat, 1) try: - self.stats["by_msg"][message_definition.symbol] += 1 # type: ignore + self.stats.by_msg[message_definition.symbol] += 1 except KeyError: - self.stats["by_msg"][message_definition.symbol] = 1 # type: ignore + self.stats.by_msg[message_definition.symbol] = 1 # Interpolate arguments into message string msg = message_definition.msg if args: @@ -369,7 +359,7 @@ class MessagesHandlerMixIn: Message( message_definition.msgid, message_definition.symbol, - (abspath, path, module, obj, line or 1, col_offset or 0), + (abspath, path, module, obj, line or 1, col_offset or 0), # type: ignore msg, confidence, ) diff --git a/pylint/reporters/base_reporter.py b/pylint/reporters/base_reporter.py index 244935a6b..21549c943 100644 --- a/pylint/reporters/base_reporter.py +++ b/pylint/reporters/base_reporter.py @@ -8,7 +8,7 @@ from warnings import warn from pylint.message import Message from pylint.reporters.ureports.nodes import Text -from pylint.typing import CheckerStats +from pylint.utils import LinterStats if TYPE_CHECKING: from pylint.lint.pylinter import PyLinter @@ -81,7 +81,7 @@ class BaseReporter: def on_close( self, - stats: CheckerStats, - previous_stats: CheckerStats, + stats: LinterStats, + previous_stats: LinterStats, ) -> None: """Hook called when a module finished analyzing.""" diff --git a/pylint/reporters/multi_reporter.py b/pylint/reporters/multi_reporter.py index b3b67d87a..a6be1c1b8 100644 --- a/pylint/reporters/multi_reporter.py +++ b/pylint/reporters/multi_reporter.py @@ -8,7 +8,7 @@ from typing import IO, TYPE_CHECKING, Any, AnyStr, Callable, List, Optional from pylint.interfaces import IReporter from pylint.message import Message from pylint.reporters.base_reporter import BaseReporter -from pylint.typing import CheckerStats +from pylint.utils import LinterStats if TYPE_CHECKING: from pylint.reporters.ureports.nodes import Section @@ -97,8 +97,8 @@ class MultiReporter: def on_close( self, - stats: CheckerStats, - previous_stats: CheckerStats, + stats: LinterStats, + previous_stats: LinterStats, ) -> None: """hook called when a module finished analyzing""" for rep in self._sub_reporters: diff --git a/pylint/reporters/reports_handler_mix_in.py b/pylint/reporters/reports_handler_mix_in.py index 4c302dbf4..14631740d 100644 --- a/pylint/reporters/reports_handler_mix_in.py +++ b/pylint/reporters/reports_handler_mix_in.py @@ -2,12 +2,12 @@ # For details: https://github.com/PyCQA/pylint/blob/main/LICENSE import collections -from typing import TYPE_CHECKING, Any, Callable, DefaultDict, Dict, List, Tuple +from typing import TYPE_CHECKING, Callable, DefaultDict, Dict, List, Optional, Tuple from pylint.exceptions import EmptyReportError from pylint.interfaces import IChecker from pylint.reporters.ureports.nodes import Section -from pylint.typing import CheckerStats +from pylint.utils import LinterStats if TYPE_CHECKING: from pylint.lint.pylinter import PyLinter @@ -59,11 +59,11 @@ class ReportsHandlerMixIn: def make_reports( # type: ignore # ReportsHandlerMixIn is always mixed with PyLinter self: "PyLinter", - stats: CheckerStats, - old_stats: CheckerStats, + stats: LinterStats, + old_stats: Optional[LinterStats], ) -> Section: """render registered reports""" - sect = Section("Report", f"{self.stats['statement']} statements analysed.") + sect = Section("Report", f"{self.stats.statement} statements analysed.") for checker in self.report_order(): for reportid, r_title, r_cb in self._reports[checker]: if not self.report_is_enabled(reportid): @@ -76,16 +76,3 @@ class ReportsHandlerMixIn: report_sect.report_id = reportid sect.append(report_sect) return sect - - def add_stats( # type: ignore # ReportsHandlerMixIn is always mixed with PyLinter - self: "PyLinter", **kwargs: Any - ) -> CheckerStats: - """add some stats entries to the statistic dictionary - raise an AssertionError if there is a key conflict - """ - for key, value in kwargs.items(): - if key[-1] == "_": - key = key[:-1] - assert key not in self.stats - self.stats[key] = value - return self.stats diff --git a/pylint/testutils/unittest_linter.py b/pylint/testutils/unittest_linter.py index c6c187abb..2d9e42e65 100644 --- a/pylint/testutils/unittest_linter.py +++ b/pylint/testutils/unittest_linter.py @@ -8,7 +8,7 @@ from astroid import nodes from pylint.interfaces import Confidence from pylint.testutils.global_test_linter import linter from pylint.testutils.output_line import MessageTest -from pylint.typing import CheckerStats +from pylint.utils import LinterStats class UnittestLinter: @@ -18,7 +18,7 @@ class UnittestLinter: def __init__(self): self._messages = [] - self.stats: CheckerStats = {} + self.stats = LinterStats() def release_messages(self): try: @@ -42,11 +42,6 @@ class UnittestLinter: def is_message_enabled(*unused_args, **unused_kwargs): return True - def add_stats(self, **kwargs): - for name, value in kwargs.items(): - self.stats[name] = value - return self.stats - @property def options_providers(self): return linter.options_providers diff --git a/pylint/typing.py b/pylint/typing.py index b0171d0d8..2d8e74763 100644 --- a/pylint/typing.py +++ b/pylint/typing.py @@ -3,10 +3,7 @@ """A collection of typing utilities.""" import sys -from typing import TYPE_CHECKING, Dict, List, NamedTuple, Union - -if TYPE_CHECKING: - from typing import Counter # typing.Counter added in Python 3.6.1 +from typing import NamedTuple, Union if sys.version_info >= (3, 8): from typing import Literal, TypedDict @@ -46,12 +43,6 @@ class ErrorDescriptionDict(TypedDict): ex: Union[ImportError, SyntaxError] -# The base type of the "stats" attribute of a checker -CheckerStats = Dict[ - str, Union[int, "Counter[str]", List, Dict[str, Union[int, str, Dict[str, int]]]] -] - - class MessageLocationTuple(NamedTuple): """Tuple with information about the location of a to-be-displayed message""" diff --git a/pylint/utils/__init__.py b/pylint/utils/__init__.py index a0658bf99..73f2e0fa0 100644 --- a/pylint/utils/__init__.py +++ b/pylint/utils/__init__.py @@ -44,8 +44,8 @@ main pylint class """ from pylint.utils.ast_walker import ASTWalker -from pylint.utils.checkerstats import merge_stats from pylint.utils.file_state import FileState +from pylint.utils.linterstats import LinterStats, ModuleStats, merge_stats from pylint.utils.utils import ( HAS_ISORT_5, IsortDriver, @@ -85,4 +85,6 @@ __all__ = [ "register_plugins", "tokenize_module", "merge_stats", + "LinterStats", + "ModuleStats", ] diff --git a/pylint/utils/checkerstats.py b/pylint/utils/checkerstats.py deleted file mode 100644 index 42af08167..000000000 --- a/pylint/utils/checkerstats.py +++ /dev/null @@ -1,30 +0,0 @@ -# Licensed under the GPL: https://www.gnu.org/licenses/old-licenses/gpl-2.0.html -# For details: https://github.com/PyCQA/pylint/blob/main/LICENSE - -import collections -from typing import TYPE_CHECKING, Dict, List, Union - -from pylint.typing import CheckerStats - -if TYPE_CHECKING: - from typing import Counter # typing.Counter added in Python 3.6.1 - - -def merge_stats(stats: List[CheckerStats]): - """Used to merge two stats objects into a new one when pylint is run in parallel mode""" - merged: CheckerStats = {} - by_msg: "Counter[str]" = collections.Counter() - for stat in stats: - message_stats: Union["Counter[str]", Dict] = stat.pop("by_msg", {}) # type: ignore - by_msg.update(message_stats) - - for key, item in stat.items(): - if key not in merged: - merged[key] = item - elif isinstance(item, dict): - merged[key].update(item) # type: ignore - else: - merged[key] = merged[key] + item # type: ignore - - merged["by_msg"] = by_msg - return merged diff --git a/pylint/utils/linterstats.py b/pylint/utils/linterstats.py new file mode 100644 index 000000000..f46054455 --- /dev/null +++ b/pylint/utils/linterstats.py @@ -0,0 +1,365 @@ +# Licensed under the GPL: https://www.gnu.org/licenses/old-licenses/gpl-2.0.html +# For details: https://github.com/PyCQA/pylint/blob/main/LICENSE + +import sys +from typing import Dict, List, Optional, Set, cast + +if sys.version_info >= (3, 8): + from typing import Literal, TypedDict +else: + from typing_extensions import Literal, TypedDict + + +class BadNames(TypedDict): + """TypedDict to store counts of node types with bad names""" + + argument: int + attr: int + klass: int + class_attribute: int + class_const: int + const: int + inlinevar: int + function: int + method: int + module: int + variable: int + + +class CodeTypeCount(TypedDict): + """TypedDict to store counts of lines of code types""" + + code: int + comment: int + docstring: int + empty: int + total: int + + +class DuplicatedLines(TypedDict): + """TypedDict to store counts of lines of duplicated code""" + + nb_duplicated_lines: int + percent_duplicated_lines: float + + +class NodeCount(TypedDict): + """TypedDict to store counts of different types of nodes""" + + function: int + klass: int + method: int + module: int + + +class UndocumentedNodes(TypedDict): + """TypedDict to store counts of undocumented node types""" + + function: int + klass: int + method: int + module: int + + +class ModuleStats(TypedDict): + """TypedDict to store counts of types of messages and statements""" + + convention: int + error: int + fatal: int + info: int + refactor: int + statement: int + warning: int + + +# pylint: disable-next=too-many-instance-attributes +class LinterStats: + """Class used to linter stats""" + + def __init__( + self, + bad_names: Optional[BadNames] = None, + by_module: Optional[Dict[str, ModuleStats]] = None, + by_msg: Optional[Dict[str, int]] = None, + code_type_count: Optional[CodeTypeCount] = None, + dependencies: Optional[Dict[str, Set[str]]] = None, + duplicated_lines: Optional[DuplicatedLines] = None, + node_count: Optional[NodeCount] = None, + undocumented: Optional[UndocumentedNodes] = None, + ) -> None: + self.bad_names = bad_names or BadNames( + argument=0, + attr=0, + klass=0, + class_attribute=0, + class_const=0, + const=0, + inlinevar=0, + function=0, + method=0, + module=0, + variable=0, + ) + self.by_module: Dict[str, ModuleStats] = by_module or {} + self.by_msg: Dict[str, int] = by_msg or {} + self.code_type_count = code_type_count or CodeTypeCount( + code=0, comment=0, docstring=0, empty=0, total=0 + ) + + self.dependencies: Dict[str, Set[str]] = dependencies or {} + self.duplicated_lines = duplicated_lines or DuplicatedLines( + nb_duplicated_lines=0, percent_duplicated_lines=0.0 + ) + self.node_count = node_count or NodeCount( + function=0, klass=0, method=0, module=0 + ) + self.undocumented = undocumented or UndocumentedNodes( + function=0, klass=0, method=0, module=0 + ) + + self.convention = 0 + self.error = 0 + self.fatal = 0 + self.info = 0 + self.refactor = 0 + self.statement = 0 + self.warning = 0 + + self.global_note = 0 + self.nb_duplicated_lines = 0 + self.percent_duplicated_lines = 0.0 + + def __str__(self) -> str: + return f"""{self.bad_names} + {sorted(self.by_module.items())} + {sorted(self.by_msg.items())} + {self.code_type_count} + {sorted(self.dependencies.items())} + {self.duplicated_lines} + {self.undocumented} + {self.convention} + {self.error} + {self.fatal} + {self.info} + {self.refactor} + {self.statement} + {self.warning} + {self.global_note} + {self.nb_duplicated_lines} + {self.percent_duplicated_lines}""" + + def get_bad_names( + self, + node_name: Literal[ + "argument", + "attr", + "class", + "class_attribute", + "class_const", + "const", + "inlinevar", + "function", + "method", + "module", + "variable", + ], + ) -> int: + """Get a bad names node count""" + if node_name == "class": + return self.bad_names.get("klass", 0) + return self.bad_names.get(node_name, 0) + + def increase_bad_name(self, node_name: str, increase: int) -> None: + """Increase a bad names node count""" + if node_name not in { + "argument", + "attr", + "class", + "class_attribute", + "class_const", + "const", + "inlinevar", + "function", + "method", + "module", + "variable", + }: + raise ValueError("Node type not part of the bad_names stat") + + node_name = cast( + Literal[ + "argument", + "attr", + "class", + "class_attribute", + "class_const", + "const", + "inlinevar", + "function", + "method", + "module", + "variable", + ], + node_name, + ) + if node_name == "class": + self.bad_names["klass"] += increase + else: + self.bad_names[node_name] += increase + + def reset_bad_names(self) -> None: + """Resets the bad_names attribute""" + self.bad_names = BadNames( + argument=0, + attr=0, + klass=0, + class_attribute=0, + class_const=0, + const=0, + inlinevar=0, + function=0, + method=0, + module=0, + variable=0, + ) + + def get_code_count( + self, type_name: Literal["code", "comment", "docstring", "empty", "total"] + ) -> int: + """Get a code type count""" + return self.code_type_count.get(type_name, 0) + + def reset_code_count(self) -> None: + """Resets the code_type_count attribute""" + self.code_type_count = CodeTypeCount( + code=0, comment=0, docstring=0, empty=0, total=0 + ) + + def reset_duplicated_lines(self) -> None: + """Resets the duplicated_lines attribute""" + self.duplicated_lines = DuplicatedLines( + nb_duplicated_lines=0, percent_duplicated_lines=0.0 + ) + + def get_node_count( + self, node_name: Literal["function", "class", "method", "module"] + ) -> int: + """Get a node count while handling some extra conditions""" + if node_name == "class": + return self.node_count.get("klass", 0) + return self.node_count.get(node_name, 0) + + def reset_node_count(self) -> None: + """Resets the node count attribute""" + self.node_count = NodeCount(function=0, klass=0, method=0, module=0) + + def get_undocumented( + self, node_name: Literal["function", "class", "method", "module"] + ) -> float: + """Get a undocumented node count""" + if node_name == "class": + return self.undocumented["klass"] + return self.undocumented[node_name] + + def reset_undocumented(self) -> None: + """Resets the undocumented attribute""" + self.undocumented = UndocumentedNodes(function=0, klass=0, method=0, module=0) + + def get_global_message_count(self, type_name: str) -> int: + """Get a global message count""" + return getattr(self, type_name, 0) + + def get_module_message_count(self, modname: str, type_name: str) -> int: + """Get a module message count""" + return getattr(self.by_module[modname], type_name, 0) + + def increase_single_message_count(self, type_name: str, increase: int) -> None: + """Increase the message type count of an individual message type""" + setattr(self, type_name, getattr(self, type_name) + increase) + + def increase_single_module_message_count( + self, + modname: str, + type_name: Literal["convention", "error", "fatal", "info", "refactor"], + increase: int, + ) -> None: + """Increase the message type count of an individual message type of a module""" + self.by_module[modname][type_name] = ( + self.by_module[modname][type_name] + increase + ) + + def reset_message_count(self) -> None: + """Resets the message type count of the stats object""" + self.convention = 0 + self.error = 0 + self.fatal = 0 + self.info = 0 + self.refactor = 0 + self.warning = 0 + + +def merge_stats(stats: List[LinterStats]): + """Used to merge multiple stats objects into a new one when pylint is run in parallel mode""" + merged = LinterStats() + for stat in stats: + merged.bad_names["argument"] += stat.bad_names["argument"] + merged.bad_names["attr"] += stat.bad_names["attr"] + merged.bad_names["klass"] += stat.bad_names["klass"] + merged.bad_names["class_attribute"] += stat.bad_names["class_attribute"] + merged.bad_names["class_const"] += stat.bad_names["class_const"] + merged.bad_names["const"] += stat.bad_names["const"] + merged.bad_names["inlinevar"] += stat.bad_names["inlinevar"] + merged.bad_names["function"] += stat.bad_names["function"] + merged.bad_names["method"] += stat.bad_names["method"] + merged.bad_names["module"] += stat.bad_names["module"] + merged.bad_names["variable"] += stat.bad_names["variable"] + + for mod_key, mod_value in stat.by_module.items(): + merged.by_module[mod_key] = mod_value + + for msg_key, msg_value in stat.by_msg.items(): + try: + merged.by_msg[msg_key] += msg_value + except KeyError: + merged.by_msg[msg_key] = msg_value + + merged.code_type_count["code"] += stat.code_type_count["code"] + merged.code_type_count["comment"] += stat.code_type_count["comment"] + merged.code_type_count["docstring"] += stat.code_type_count["docstring"] + merged.code_type_count["empty"] += stat.code_type_count["empty"] + merged.code_type_count["total"] += stat.code_type_count["total"] + + for dep_key, dep_value in stat.dependencies.items(): + try: + merged.dependencies[dep_key].update(dep_value) + except KeyError: + merged.dependencies[dep_key] = dep_value + + merged.duplicated_lines["nb_duplicated_lines"] += stat.duplicated_lines[ + "nb_duplicated_lines" + ] + merged.duplicated_lines["percent_duplicated_lines"] += stat.duplicated_lines[ + "percent_duplicated_lines" + ] + + merged.node_count["function"] += stat.node_count["function"] + merged.node_count["klass"] += stat.node_count["klass"] + merged.node_count["method"] += stat.node_count["method"] + merged.node_count["module"] += stat.node_count["module"] + + merged.undocumented["function"] += stat.undocumented["function"] + merged.undocumented["klass"] += stat.undocumented["klass"] + merged.undocumented["method"] += stat.undocumented["method"] + merged.undocumented["module"] += stat.undocumented["module"] + + merged.convention += stat.convention + merged.error += stat.error + merged.fatal += stat.fatal + merged.info += stat.info + merged.refactor += stat.refactor + merged.statement += stat.statement + merged.warning += stat.warning + + merged.global_note += stat.global_note + return merged diff --git a/tests/lint/unittest_lint.py b/tests/lint/unittest_lint.py index 277799676..79437e987 100644 --- a/tests/lint/unittest_lint.py +++ b/tests/lint/unittest_lint.py @@ -48,7 +48,7 @@ from io import StringIO from os import chdir, getcwd from os.path import abspath, basename, dirname, isdir, join, sep from shutil import rmtree -from typing import Dict, Iterable, Iterator, List, Optional, Tuple +from typing import Iterable, Iterator, List, Optional, Tuple import platformdirs import pytest @@ -870,7 +870,7 @@ def test_by_module_statement_value(init_linter: PyLinter) -> None: linter = init_linter linter.check([os.path.join(os.path.dirname(__file__), "data")]) - by_module_stats: Dict[str, Dict[str, int]] = linter.stats["by_module"] # type: ignore + by_module_stats = linter.stats.by_module for module, module_stats in by_module_stats.items(): linter2 = init_linter @@ -881,4 +881,4 @@ def test_by_module_statement_value(init_linter: PyLinter) -> None: # Check that the by_module "statement" is equal to the global "statement" # computed for that module - assert module_stats["statement"] == linter2.stats["statement"] + assert module_stats["statement"] == linter2.stats.statement diff --git a/tests/test_check_parallel.py b/tests/test_check_parallel.py index c0fbe00fd..be250d892 100644 --- a/tests/test_check_parallel.py +++ b/tests/test_check_parallel.py @@ -10,7 +10,6 @@ # pylint: disable=protected-access,missing-function-docstring,no-self-use -import collections import os from typing import List @@ -25,7 +24,8 @@ from pylint.lint.parallel import _worker_check_single_file as worker_check_singl from pylint.lint.parallel import _worker_initialize as worker_initialize from pylint.lint.parallel import check_parallel from pylint.testutils import GenericTestReporter as Reporter -from pylint.typing import CheckerStats, FileItem +from pylint.typing import FileItem +from pylint.utils import LinterStats, ModuleStats def _gen_file_data(idx: int = 0) -> FileItem: @@ -103,11 +103,10 @@ class ParallelTestChecker(BaseChecker): super().__init__(linter) self.data: List[str] = [] self.linter = linter - self.stats: CheckerStats = {} def open(self) -> None: """init the checkers: reset statistics information""" - self.stats = self.linter.add_stats() + self.linter.stats.reset_node_count() self.data = [] def close(self) -> None: @@ -202,26 +201,24 @@ class TestCheckParallelFramework: no_errors_status = 0 assert no_errors_status == msg_status assert { - "by_module": { - "--test-file_data-name-0--": { - "convention": 0, - "error": 0, - "fatal": 0, - "info": 0, - "refactor": 0, - "statement": 18, - "warning": 0, - } - }, - "by_msg": {}, - "convention": 0, - "error": 0, - "fatal": 0, - "info": 0, - "refactor": 0, - "statement": 18, - "warning": 0, - } == stats + "--test-file_data-name-0--": { + "convention": 0, + "error": 0, + "fatal": 0, + "info": 0, + "refactor": 0, + "statement": 18, + "warning": 0, + } + } == stats.by_module + assert {} == stats.by_msg + assert stats.convention == 0 + assert stats.error == 0 + assert stats.fatal == 0 + assert stats.info == 0 + assert stats.refactor == 0 + assert stats.statement == 18 + assert stats.warning == 0 def test_worker_check_sequential_checker(self) -> None: """Same as test_worker_check_single_file_no_checkers with SequentialTestChecker""" @@ -248,26 +245,24 @@ class TestCheckParallelFramework: no_errors_status = 0 assert no_errors_status == msg_status assert { - "by_module": { - "--test-file_data-name-0--": { - "convention": 0, - "error": 0, - "fatal": 0, - "info": 0, - "refactor": 0, - "statement": 18, - "warning": 0, - } - }, - "by_msg": {}, - "convention": 0, - "error": 0, - "fatal": 0, - "info": 0, - "refactor": 0, - "statement": 18, - "warning": 0, - } == stats + "--test-file_data-name-0--": { + "convention": 0, + "error": 0, + "fatal": 0, + "info": 0, + "refactor": 0, + "statement": 18, + "warning": 0, + } + } == stats.by_module + assert {} == stats.by_msg + assert stats.convention == 0 + assert stats.error == 0 + assert stats.fatal == 0 + assert stats.info == 0 + assert stats.refactor == 0 + assert stats.statement == 18 + assert stats.warning == 0 class TestCheckParallel: @@ -299,52 +294,48 @@ class TestCheckParallel: "checkers registered" ) assert { - "by_module": { - "--test-file_data-name-0--": { - "convention": 0, - "error": 0, - "fatal": 0, - "info": 0, - "refactor": 0, - "statement": 18, - "warning": 0, - } - }, - "by_msg": {}, - "convention": 0, - "error": 0, - "fatal": 0, - "info": 0, - "refactor": 0, - "statement": 18, - "warning": 0, - } == linter.stats + "--test-file_data-name-0--": { + "convention": 0, + "error": 0, + "fatal": 0, + "info": 0, + "refactor": 0, + "statement": 18, + "warning": 0, + } + } == linter.stats.by_module + assert linter.stats.by_msg == {} + assert linter.stats.convention == 0 + assert linter.stats.error == 0 + assert linter.stats.fatal == 0 + assert linter.stats.info == 0 + assert linter.stats.refactor == 0 + assert linter.stats.statement == 18 + assert linter.stats.warning == 0 # now run the regular mode of checking files and check that, in this proc, we # collect the right data filepath = [single_file_container[0][1]] # get the filepath element linter.check(filepath) assert { - "by_module": { - "input.similar1": { # module is the only change from previous - "convention": 0, - "error": 0, - "fatal": 0, - "info": 0, - "refactor": 0, - "statement": 18, - "warning": 0, - } - }, - "by_msg": {}, - "convention": 0, - "error": 0, - "fatal": 0, - "info": 0, - "refactor": 0, - "statement": 18, - "warning": 0, - } == linter.stats + "input.similar1": { # module is the only change from previous + "convention": 0, + "error": 0, + "fatal": 0, + "info": 0, + "refactor": 0, + "statement": 18, + "warning": 0, + } + } == linter.stats.by_module + assert linter.stats.by_msg == {} + assert linter.stats.convention == 0 + assert linter.stats.error == 0 + assert linter.stats.fatal == 0 + assert linter.stats.info == 0 + assert linter.stats.refactor == 0 + assert linter.stats.statement == 18 + assert linter.stats.warning == 0 def test_invoke_single_job(self) -> None: """Tests basic checkers functionality using just a single workderdo @@ -365,26 +356,24 @@ class TestCheckParallel: ) assert { - "by_module": { - "--test-file_data-name-0--": { - "convention": 0, - "error": 0, - "fatal": 0, - "info": 0, - "refactor": 0, - "statement": 18, - "warning": 0, - } - }, - "by_msg": collections.Counter(), - "convention": 0, - "error": 0, - "fatal": 0, - "info": 0, - "refactor": 0, - "statement": 18, - "warning": 0, - } == linter.stats + "--test-file_data-name-0--": { + "convention": 0, + "error": 0, + "fatal": 0, + "info": 0, + "refactor": 0, + "statement": 18, + "warning": 0, + } + } == linter.stats.by_module + assert linter.stats.by_msg == {} + assert linter.stats.convention == 0 + assert linter.stats.error == 0 + assert linter.stats.fatal == 0 + assert linter.stats.info == 0 + assert linter.stats.refactor == 0 + assert linter.stats.statement == 18 + assert linter.stats.warning == 0 assert linter.msg_status == 0, "We expect a single-file check to exit cleanly" @pytest.mark.parametrize( @@ -425,30 +414,30 @@ class TestCheckParallel: # define the stats we expect to get back from the runs, these should only vary # with the number of files. - expected_stats = { - "by_module": { + expected_stats = LinterStats( + by_module={ # pylint: disable-next=consider-using-f-string "--test-file_data-name-%d--" - % idx: { - "convention": 0, - "error": 0, - "fatal": 0, - "info": 0, - "refactor": 0, - "statement": 18, - "warning": 0, - } + % idx: ModuleStats( + convention=0, + error=0, + fatal=0, + info=0, + refactor=0, + statement=18, + warning=0, + ) for idx in range(num_files) - }, - "by_msg": {}, - "convention": 0, - "error": 0, - "fatal": 0, - "info": 0, - "refactor": 0, - "statement": 18 * num_files, - "warning": 0, - } + } + ) + expected_stats.by_msg = {} + expected_stats.convention = 0 + expected_stats.error = 0 + expected_stats.fatal = 0 + expected_stats.info = 0 + expected_stats.refactor = 0 + expected_stats.statement = 18 * num_files + expected_stats.warning = 0 file_infos = _gen_file_datas(num_files) @@ -482,11 +471,11 @@ class TestCheckParallel: stats_check_parallel = linter.stats assert linter.msg_status == 0, "We should not fail the lint" - assert ( - stats_single_proc == stats_check_parallel + assert str(stats_single_proc) == str( + stats_check_parallel ), "Single-proc and check_parallel() should return the same thing" - assert ( - stats_check_parallel == expected_stats + assert str(stats_check_parallel) == str( + expected_stats ), "The lint is returning unexpected results, has something changed?" @pytest.mark.parametrize( @@ -551,6 +540,6 @@ class TestCheckParallel: arguments=None, ) stats_check_parallel = linter.stats - assert ( - stats_single_proc["by_msg"] == stats_check_parallel["by_msg"] + assert str(stats_single_proc.by_msg) == str( + stats_check_parallel.by_msg ), "Single-proc and check_parallel() should return the same thing" diff --git a/tests/test_import_graph.py b/tests/test_import_graph.py index 42e3baf56..6035601bf 100644 --- a/tests/test_import_graph.py +++ b/tests/test_import_graph.py @@ -48,7 +48,7 @@ POSSIBLE_DOT_FILENAMES = ["foo.dot", "foo.gv", "tests/regrtest_data/foo.dot"] @pytest.mark.parametrize("dest", POSSIBLE_DOT_FILENAMES, indirect=True) def test_dependencies_graph(dest: str) -> None: """DOC files are correctly generated, and the graphname is the basename""" - imports._dependencies_graph(dest, {"labas": ["hoho", "yep"], "hoho": ["yep"]}) + imports._dependencies_graph(dest, {"labas": {"hoho", "yep"}, "hoho": {"yep"}}) with open(dest, encoding="utf-8") as stream: assert ( stream.read().strip() @@ -75,7 +75,7 @@ URL="." node[shape="box"] def test_missing_graphviz(filename: str) -> None: """Raises if graphviz is not installed, and defaults to png if no extension given""" with pytest.raises(RuntimeError, match=r"Cannot generate `graph\.png`.*"): - imports._dependencies_graph(filename, {"a": ["b", "c"], "b": ["c"]}) + imports._dependencies_graph(filename, {"a": {"b", "c"}, "b": {"c"}}) @pytest.fixture diff --git a/tests/test_regr.py b/tests/test_regr.py index 07385a64d..6dadcf2c6 100644 --- a/tests/test_regr.py +++ b/tests/test_regr.py @@ -114,12 +114,12 @@ def modify_path() -> Iterator: def test_check_package___init__(finalize_linter: PyLinter) -> None: filename = ["package.__init__"] finalize_linter.check(filename) - checked = list(finalize_linter.stats["by_module"].keys()) # type: ignore # Refactor of PyLinter.stats necessary - assert checked == filename + checked = list(finalize_linter.stats.by_module.keys()) + assert sorted(checked) == sorted(filename) os.chdir(join(REGR_DATA, "package")) finalize_linter.check(["__init__"]) - checked = list(finalize_linter.stats["by_module"].keys()) # type: ignore + checked = list(finalize_linter.stats.by_module.keys()) assert checked == ["__init__"] |