summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorDaniƫl van Noord <13665637+DanielNoord@users.noreply.github.com>2021-10-07 09:40:31 +0200
committerGitHub <noreply@github.com>2021-10-07 09:40:31 +0200
commit16c09cb48a3c66f453faccdf380aea6969d61202 (patch)
tree9cd442a7c64913f9815f6bf140ed39d0e24ac5fd
parentd98c29d888fbfd11aa42da593c6cfc6812434ee0 (diff)
downloadpylint-git-16c09cb48a3c66f453faccdf380aea6969d61202.tar.gz
Refactor ``LinterStats`` (#5074)
* Refactor ``self.stats`` on linter and checker This adds a new class ``LinterStats`` which is used to store all kinds of stats during a run of ``pylint``. Tests have been changed slightly to be able to use the new class. Co-authored-by: Marc Mueller <30130371+cdce8p@users.noreply.github.com>
-rw-r--r--pylint/checkers/__init__.py79
-rw-r--r--pylint/checkers/base.py112
-rw-r--r--pylint/checkers/base_checker.py2
-rw-r--r--pylint/checkers/design_analysis.py4
-rw-r--r--pylint/checkers/imports.py22
-rw-r--r--pylint/checkers/raw_metrics.py64
-rw-r--r--pylint/checkers/similar.py24
-rw-r--r--pylint/config/__init__.py8
-rw-r--r--pylint/lint/parallel.py6
-rw-r--r--pylint/lint/pylinter.py52
-rw-r--r--pylint/lint/report_functions.py33
-rw-r--r--pylint/message/message_handler_mix_in.py22
-rw-r--r--pylint/reporters/base_reporter.py6
-rw-r--r--pylint/reporters/multi_reporter.py6
-rw-r--r--pylint/reporters/reports_handler_mix_in.py23
-rw-r--r--pylint/testutils/unittest_linter.py9
-rw-r--r--pylint/typing.py11
-rw-r--r--pylint/utils/__init__.py4
-rw-r--r--pylint/utils/checkerstats.py30
-rw-r--r--pylint/utils/linterstats.py365
-rw-r--r--tests/lint/unittest_lint.py6
-rw-r--r--tests/test_check_parallel.py251
-rw-r--r--tests/test_import_graph.py4
-rw-r--r--tests/test_regr.py6
24 files changed, 734 insertions, 415 deletions
diff --git a/pylint/checkers/__init__.py b/pylint/checkers/__init__.py
index 9e7ff6eed..273b085a0 100644
--- a/pylint/checkers/__init__.py
+++ b/pylint/checkers/__init__.py
@@ -47,35 +47,84 @@ messages nor reports. XXX not true, emit a 07 report !
"""
-from typing import Iterable, List, Union
+import sys
+from typing import List, Optional, Tuple, Union
from pylint.checkers.base_checker import BaseChecker, BaseTokenChecker
from pylint.checkers.deprecated import DeprecatedMixin
from pylint.checkers.mapreduce_checker import MapReduceMixin
-from pylint.typing import CheckerStats
-from pylint.utils import diff_string, register_plugins
+from pylint.utils import LinterStats, diff_string, register_plugins
+
+if sys.version_info >= (3, 8):
+ from typing import Literal
+else:
+ from typing_extensions import Literal
def table_lines_from_stats(
- stats: CheckerStats,
- old_stats: CheckerStats,
- columns: Iterable[str],
+ stats: LinterStats,
+ old_stats: Optional[LinterStats],
+ stat_type: Literal["duplicated_lines", "message_types"],
) -> List[str]:
"""get values listed in <columns> from <stats> and <old_stats>,
and return a formatted list of values, designed to be given to a
ureport.Table object
"""
lines: List[str] = []
- for m_type in columns:
- new: Union[int, str] = stats[m_type] # type: ignore
- old: Union[int, str, None] = old_stats.get(m_type) # type: ignore
- if old is not None:
- diff_str = diff_string(old, new)
+ if stat_type == "duplicated_lines":
+ new: List[Tuple[str, Union[str, int, float]]] = [
+ ("nb_duplicated_lines", stats.duplicated_lines["nb_duplicated_lines"]),
+ (
+ "percent_duplicated_lines",
+ stats.duplicated_lines["percent_duplicated_lines"],
+ ),
+ ]
+ if old_stats:
+ old: List[Tuple[str, Union[str, int, float]]] = [
+ (
+ "nb_duplicated_lines",
+ old_stats.duplicated_lines["nb_duplicated_lines"],
+ ),
+ (
+ "percent_duplicated_lines",
+ old_stats.duplicated_lines["percent_duplicated_lines"],
+ ),
+ ]
+ else:
+ old = [("nb_duplicated_lines", "NC"), ("percent_duplicated_lines", "NC")]
+ elif stat_type == "message_types":
+ new = [
+ ("convention", stats.convention),
+ ("refactor", stats.refactor),
+ ("warning", stats.warning),
+ ("error", stats.error),
+ ]
+ if old_stats:
+ old = [
+ ("convention", old_stats.convention),
+ ("refactor", old_stats.refactor),
+ ("warning", old_stats.warning),
+ ("error", old_stats.error),
+ ]
else:
- old, diff_str = "NC", "NC"
- new = f"{new:.3f}" if isinstance(new, float) else str(new)
- old = f"{old:.3f}" if isinstance(old, float) else str(old)
- lines.extend((m_type.replace("_", " "), new, old, diff_str))
+ old = [
+ ("convention", "NC"),
+ ("refactor", "NC"),
+ ("warning", "NC"),
+ ("error", "NC"),
+ ]
+
+ for index, _ in enumerate(new):
+ new_value = new[index][1]
+ old_value = old[index][1]
+ diff_str = (
+ diff_string(old_value, new_value)
+ if isinstance(old_value, float)
+ else old_value
+ )
+ new_str = f"{new_value:.3f}" if isinstance(new_value, float) else str(new_value)
+ old_str = f"{old_value:.3f}" if isinstance(old_value, float) else str(old_value)
+ lines.extend((new[index][0].replace("_", " "), new_str, old_str, diff_str))
return lines
diff --git a/pylint/checkers/base.py b/pylint/checkers/base.py
index fe685d03b..124881b77 100644
--- a/pylint/checkers/base.py
+++ b/pylint/checkers/base.py
@@ -68,12 +68,12 @@ import collections
import itertools
import re
import sys
-from typing import Any, Dict, Iterator, Optional, Pattern, Union
+from typing import Any, Dict, Iterator, Optional, Pattern, cast
import astroid
from astroid import nodes
-from pylint import checkers, exceptions, interfaces
+from pylint import checkers, interfaces
from pylint import utils as lint_utils
from pylint.checkers import utils
from pylint.checkers.utils import (
@@ -83,7 +83,12 @@ from pylint.checkers.utils import (
is_property_setter,
)
from pylint.reporters.ureports import nodes as reporter_nodes
-from pylint.typing import CheckerStats
+from pylint.utils import LinterStats
+
+if sys.version_info >= (3, 8):
+ from typing import Literal
+else:
+ from typing_extensions import Literal
class NamingStyle:
@@ -389,8 +394,8 @@ def _has_abstract_methods(node):
def report_by_type_stats(
sect,
- stats: CheckerStats,
- old_stats: CheckerStats,
+ stats: LinterStats,
+ old_stats: Optional[LinterStats],
):
"""make a report of
@@ -400,38 +405,28 @@ def report_by_type_stats(
# percentage of different types documented and/or with a bad name
nice_stats: Dict[str, Dict[str, str]] = {}
for node_type in ("module", "class", "method", "function"):
- try:
- total: int = stats[node_type] # type: ignore
- except KeyError as e:
- raise exceptions.EmptyReportError() from e
+ node_type = cast(Literal["function", "class", "method", "module"], node_type)
+ total = stats.get_node_count(node_type)
nice_stats[node_type] = {}
if total != 0:
- try:
- undocumented_node: int = stats["undocumented_" + node_type] # type: ignore
- documented = total - undocumented_node
- percent = (documented * 100.0) / total
- nice_stats[node_type]["percent_documented"] = f"{percent:.2f}"
- except KeyError:
- nice_stats[node_type]["percent_documented"] = "NC"
- try:
- badname_node: int = stats["badname_" + node_type] # type: ignore
- percent = (badname_node * 100.0) / total
- nice_stats[node_type]["percent_badname"] = f"{percent:.2f}"
- except KeyError:
- nice_stats[node_type]["percent_badname"] = "NC"
+ undocumented_node = stats.get_undocumented(node_type)
+ documented = total - undocumented_node
+ percent = (documented * 100.0) / total
+ nice_stats[node_type]["percent_documented"] = f"{percent:.2f}"
+ badname_node = stats.get_bad_names(node_type)
+ percent = (badname_node * 100.0) / total
+ nice_stats[node_type]["percent_badname"] = f"{percent:.2f}"
lines = ["type", "number", "old number", "difference", "%documented", "%badname"]
for node_type in ("module", "class", "method", "function"):
- new = stats[node_type]
- old: Optional[Union[str, int]] = old_stats.get(node_type, None) # type: ignore
- if old is not None:
- diff_str = lint_utils.diff_string(old, new)
- else:
- old, diff_str = "NC", "NC"
+ node_type = cast(Literal["function", "class", "method", "module"], node_type)
+ new = stats.get_node_count(node_type)
+ old = old_stats.get_node_count(node_type) if old_stats else None
+ diff_str = lint_utils.diff_string(old, new) if old else None
lines += [
node_type,
str(new),
- str(old),
- diff_str,
+ str(old) if old else "NC",
+ diff_str if diff_str else "NC",
nice_stats[node_type].get("percent_documented", "0"),
nice_stats[node_type].get("percent_badname", "0"),
]
@@ -1086,13 +1081,12 @@ class BasicChecker(_BasicChecker):
def __init__(self, linter):
super().__init__(linter)
- self.stats: CheckerStats = {}
self._tryfinallys = None
def open(self):
"""initialize visit variables and statistics"""
self._tryfinallys = []
- self.stats = self.linter.add_stats(module=0, function=0, method=0, class_=0)
+ self.linter.stats.reset_node_count()
@utils.check_messages("using-constant-test", "missing-parentheses-for-call-in-test")
def visit_if(self, node: nodes.If) -> None:
@@ -1163,13 +1157,13 @@ class BasicChecker(_BasicChecker):
def visit_module(self, _: nodes.Module) -> None:
"""check module name, docstring and required arguments"""
- self.stats["module"] += 1 # type: ignore
+ self.linter.stats.node_count["module"] += 1
def visit_classdef(self, _: nodes.ClassDef) -> None:
"""check module name, docstring and redefinition
increment branch counter
"""
- self.stats["class"] += 1 # type: ignore
+ self.linter.stats.node_count["klass"] += 1
@utils.check_messages(
"pointless-statement", "pointless-string-statement", "expression-not-assigned"
@@ -1308,7 +1302,10 @@ class BasicChecker(_BasicChecker):
"""check function name, docstring, arguments, redefinition,
variable names, max locals
"""
- self.stats["method" if node.is_method() else "function"] += 1 # type: ignore
+ if node.is_method():
+ self.linter.stats.node_count["method"] += 1
+ else:
+ self.linter.stats.node_count["function"] += 1
self._check_dangerous_default(node)
visit_asyncfunctiondef = visit_functiondef
@@ -1846,19 +1843,7 @@ class NameChecker(_BasicChecker):
self._non_ascii_rgx_compiled = re.compile("[^\u0000-\u007F]")
def open(self):
- self.stats = self.linter.add_stats(
- badname_module=0,
- badname_class=0,
- badname_function=0,
- badname_method=0,
- badname_attr=0,
- badname_const=0,
- badname_variable=0,
- badname_inlinevar=0,
- badname_argument=0,
- badname_class_attribute=0,
- badname_class_const=0,
- )
+ self.linter.stats.reset_bad_names()
for group in self.config.name_group:
for name_type in group.split(":"):
self._name_group[name_type] = f"group_{group}"
@@ -2044,7 +2029,7 @@ class NameChecker(_BasicChecker):
)
self.add_message(warning, node=node, args=args, confidence=confidence)
- self.stats["badname_" + node_type] += 1 # type: ignore
+ self.linter.stats.increase_bad_name(node_type, 1)
def _name_allowed_by_regex(self, name: str) -> bool:
return name in self.config.good_names or any(
@@ -2074,7 +2059,7 @@ class NameChecker(_BasicChecker):
if self._name_allowed_by_regex(name=name):
return
if self._name_disallowed_by_regex(name=name):
- self.stats["badname_" + node_type] += 1
+ self.linter.stats.increase_bad_name(node_type, 1)
self.add_message("disallowed-name", node=node, args=name)
return
regexp = self._name_regexps[node_type]
@@ -2168,12 +2153,7 @@ class DocStringChecker(_BasicChecker):
)
def open(self):
- self.stats = self.linter.add_stats(
- undocumented_module=0,
- undocumented_function=0,
- undocumented_method=0,
- undocumented_class=0,
- )
+ self.linter.stats.reset_undocumented()
@utils.check_messages("missing-docstring", "empty-docstring")
def visit_module(self, node: nodes.Module) -> None:
@@ -2210,17 +2190,21 @@ class DocStringChecker(_BasicChecker):
overridden = True
break
self._check_docstring(
- ftype, node, report_missing=not overridden, confidence=confidence
+ ftype, node, report_missing=not overridden, confidence=confidence # type: ignore
)
elif isinstance(node.parent.frame(), nodes.Module):
- self._check_docstring(ftype, node)
+ self._check_docstring(ftype, node) # type: ignore
else:
return
visit_asyncfunctiondef = visit_functiondef
def _check_docstring(
- self, node_type, node, report_missing=True, confidence=interfaces.HIGH
+ self,
+ node_type: Literal["class", "function", "method", "module"],
+ node,
+ report_missing=True,
+ confidence=interfaces.HIGH,
):
"""check the node has a non empty docstring"""
docstring = node.doc
@@ -2240,7 +2224,10 @@ class DocStringChecker(_BasicChecker):
if node_type != "module" and max_lines > -1 and lines < max_lines:
return
- self.stats["undocumented_" + node_type] += 1
+ if node_type == "class":
+ self.linter.stats.undocumented["klass"] += 1
+ else:
+ self.linter.stats.undocumented[node_type] += 1
if (
node.body
and isinstance(node.body[0], nodes.Expr)
@@ -2262,7 +2249,10 @@ class DocStringChecker(_BasicChecker):
message = "missing-function-docstring"
self.add_message(message, node=node, confidence=confidence)
elif not docstring.strip():
- self.stats["undocumented_" + node_type] += 1
+ if node_type == "class":
+ self.linter.stats.undocumented["klass"] += 1
+ else:
+ self.linter.stats.undocumented[node_type] += 1
self.add_message(
"empty-docstring", node=node, args=(node_type,), confidence=confidence
)
diff --git a/pylint/checkers/base_checker.py b/pylint/checkers/base_checker.py
index 938ae792e..328adcb4d 100644
--- a/pylint/checkers/base_checker.py
+++ b/pylint/checkers/base_checker.py
@@ -27,7 +27,6 @@ from pylint.constants import _MSG_ORDER, WarningScope
from pylint.exceptions import InvalidMessageError
from pylint.interfaces import Confidence, IRawChecker, ITokenChecker, implements
from pylint.message.message_definition import MessageDefinition
-from pylint.typing import CheckerStats
from pylint.utils import get_rst_section, get_rst_title
@@ -55,7 +54,6 @@ class BaseChecker(OptionsProviderMixIn):
self.name = self.name.lower()
super().__init__()
self.linter = linter
- self.stats: CheckerStats = {}
def __gt__(self, other):
"""Permit to sort a list of Checker by name."""
diff --git a/pylint/checkers/design_analysis.py b/pylint/checkers/design_analysis.py
index 828537793..947e22078 100644
--- a/pylint/checkers/design_analysis.py
+++ b/pylint/checkers/design_analysis.py
@@ -37,7 +37,6 @@ from pylint import utils
from pylint.checkers import BaseChecker
from pylint.checkers.utils import check_messages
from pylint.interfaces import IAstroidChecker
-from pylint.typing import CheckerStats
MSGS = { # pylint: disable=consider-using-namedtuple-or-dataclass
"R0901": (
@@ -407,14 +406,13 @@ class MisdesignChecker(BaseChecker):
def __init__(self, linter=None):
super().__init__(linter)
- self.stats: CheckerStats = {}
self._returns = None
self._branches = None
self._stmts = None
def open(self):
"""initialize visit variables"""
- self.stats = self.linter.add_stats()
+ self.linter.stats.reset_node_count()
self._returns = []
self._branches = defaultdict(int)
self._stmts = []
diff --git a/pylint/checkers/imports.py b/pylint/checkers/imports.py
index abecc1712..3e0b39d26 100644
--- a/pylint/checkers/imports.py
+++ b/pylint/checkers/imports.py
@@ -70,7 +70,6 @@ from pylint.graph import DotBackend, get_cycles
from pylint.interfaces import IAstroidChecker
from pylint.lint import PyLinter
from pylint.reporters.ureports.nodes import Paragraph, Section, VerbatimText
-from pylint.typing import CheckerStats
from pylint.utils import IsortDriver, get_global_option
@@ -170,26 +169,27 @@ def _repr_tree_defs(data, indent_str=None):
return "\n".join(lines)
-def _dependencies_graph(filename: str, dep_info: Dict[str, List[str]]) -> str:
+def _dependencies_graph(filename: str, dep_info: Dict[str, Set[str]]) -> str:
"""write dependencies as a dot (graphviz) file"""
done = {}
printer = DotBackend(os.path.splitext(os.path.basename(filename))[0], rankdir="LR")
printer.emit('URL="." node[shape="box"]')
for modname, dependencies in sorted(dep_info.items()):
+ sorted_dependencies = sorted(dependencies)
done[modname] = 1
printer.emit_node(modname)
- for depmodname in dependencies:
+ for depmodname in sorted_dependencies:
if depmodname not in done:
done[depmodname] = 1
printer.emit_node(depmodname)
for depmodname, dependencies in sorted(dep_info.items()):
- for modname in dependencies:
+ for modname in sorted(dependencies):
printer.emit_edge(modname, depmodname)
return printer.generate(filename)
def _make_graph(
- filename: str, dep_info: Dict[str, List[str]], sect: Section, gtype: str
+ filename: str, dep_info: Dict[str, Set[str]], sect: Section, gtype: str
):
"""generate a dependencies graph and add some information about it in the
report's section
@@ -428,7 +428,6 @@ class ImportsChecker(DeprecatedMixin, BaseChecker):
self, linter: Optional[PyLinter] = None
): # pylint: disable=super-init-not-called # See https://github.com/PyCQA/pylint/issues/4941
BaseChecker.__init__(self, linter)
- self.stats: CheckerStats = {}
self.import_graph: collections.defaultdict = collections.defaultdict(set)
self._imports_stack: List[Tuple[Any, Any]] = []
self._first_non_import_node = None
@@ -470,9 +469,8 @@ class ImportsChecker(DeprecatedMixin, BaseChecker):
def open(self):
"""called before visiting project (i.e set of modules)"""
- self.linter.add_stats(dependencies={})
- self.linter.add_stats(cycles=[])
- self.stats = self.linter.stats
+ self.linter.stats.dependencies = {}
+ self.linter.stats = self.linter.stats
self.import_graph = collections.defaultdict(set)
self._module_pkg = {} # mapping of modules to the pkg they belong in
self._excluded_edges = collections.defaultdict(set)
@@ -859,7 +857,7 @@ class ImportsChecker(DeprecatedMixin, BaseChecker):
self._module_pkg[context_name] = context_name.rsplit(".", 1)[0]
# handle dependencies
- dependencies_stat: Dict[str, Union[Set]] = self.stats["dependencies"] # type: ignore
+ dependencies_stat: Dict[str, Union[Set]] = self.linter.stats.dependencies
importedmodnames = dependencies_stat.setdefault(importedmodname, set())
if context_name not in importedmodnames:
importedmodnames.add(context_name)
@@ -935,7 +933,7 @@ class ImportsChecker(DeprecatedMixin, BaseChecker):
def _report_dependencies_graph(self, sect, _, _dummy):
"""write dependencies as a dot (graphviz) file"""
- dep_info = self.stats["dependencies"]
+ dep_info = self.linter.stats.dependencies
if not dep_info or not (
self.config.import_graph
or self.config.ext_import_graph
@@ -955,7 +953,7 @@ class ImportsChecker(DeprecatedMixin, BaseChecker):
def _filter_dependencies_graph(self, internal):
"""build the internal or the external dependency graph"""
graph = collections.defaultdict(set)
- for importee, importers in self.stats["dependencies"].items():
+ for importee, importers in self.linter.stats.dependencies.items():
for importer in importers:
package = self._module_pkg.get(importer, importer)
is_inside = importee.startswith(package)
diff --git a/pylint/checkers/raw_metrics.py b/pylint/checkers/raw_metrics.py
index c8db0d238..42d23dab3 100644
--- a/pylint/checkers/raw_metrics.py
+++ b/pylint/checkers/raw_metrics.py
@@ -15,38 +15,43 @@
# Licensed under the GPL: https://www.gnu.org/licenses/old-licenses/gpl-2.0.html
# For details: https://github.com/PyCQA/pylint/blob/main/LICENSE
+import sys
import tokenize
-from typing import Any, Optional, Union
+from typing import Any, Optional, cast
from pylint.checkers import BaseTokenChecker
-from pylint.exceptions import EmptyReportError
from pylint.interfaces import ITokenChecker
from pylint.reporters.ureports.nodes import Table
-from pylint.typing import CheckerStats
-from pylint.utils import diff_string
+from pylint.utils import LinterStats, diff_string
+
+if sys.version_info >= (3, 8):
+ from typing import Literal
+else:
+ from typing_extensions import Literal
def report_raw_stats(
sect,
- stats: CheckerStats,
- old_stats: CheckerStats,
-):
+ stats: LinterStats,
+ old_stats: Optional[LinterStats],
+) -> None:
"""calculate percentage of code / doc / comment / empty"""
- total_lines: int = stats["total_lines"] # type: ignore
- if not total_lines:
- raise EmptyReportError()
+ total_lines = stats.code_type_count["total"]
sect.description = f"{total_lines} lines have been analyzed"
lines = ["type", "number", "%", "previous", "difference"]
for node_type in ("code", "docstring", "comment", "empty"):
- key = node_type + "_lines"
- total: int = stats[key] # type: ignore
- percent = float(total * 100) / total_lines
- old: Optional[Union[int, str]] = old_stats.get(key, None) # type: ignore
- if old is not None:
- diff_str = diff_string(old, total)
- else:
- old, diff_str = "NC", "NC"
- lines += [node_type, str(total), f"{percent:.2f}", str(old), diff_str]
+ node_type = cast(Literal["code", "docstring", "comment", "empty"], node_type)
+ total = stats.code_type_count[node_type]
+ percent = float(total * 100) / total_lines if total_lines else None
+ old = old_stats.code_type_count[node_type] if old_stats else None
+ diff_str = diff_string(old, total) if old else None
+ lines += [
+ node_type,
+ str(total),
+ f"{percent:.2f}" if percent is not None else "NC",
+ str(old) if old else "NC",
+ diff_str if diff_str else "NC",
+ ]
sect.append(Table(children=lines, cols=5, rheaders=1))
@@ -72,17 +77,10 @@ class RawMetricsChecker(BaseTokenChecker):
def __init__(self, linter):
super().__init__(linter)
- self.stats: CheckerStats = {}
def open(self):
"""init statistics"""
- self.stats = self.linter.add_stats(
- total_lines=0,
- code_lines=0,
- empty_lines=0,
- docstring_lines=0,
- comment_lines=0,
- )
+ self.linter.stats.reset_code_count()
def process_tokens(self, tokens):
"""update stats"""
@@ -90,8 +88,8 @@ class RawMetricsChecker(BaseTokenChecker):
tokens = list(tokens)
while i < len(tokens):
i, lines_number, line_type = get_type(tokens, i)
- self.stats["total_lines"] += lines_number
- self.stats[line_type] += lines_number
+ self.linter.stats.code_type_count["total"] += lines_number
+ self.linter.stats.code_type_count[line_type] += lines_number
JUNK = (tokenize.NL, tokenize.INDENT, tokenize.NEWLINE, tokenize.ENDMARKER)
@@ -108,16 +106,16 @@ def get_type(tokens, start_index):
pos = tokens[i][3]
if line_type is None:
if tok_type == tokenize.STRING:
- line_type = "docstring_lines"
+ line_type = "docstring"
elif tok_type == tokenize.COMMENT:
- line_type = "comment_lines"
+ line_type = "comment"
elif tok_type in JUNK:
pass
else:
- line_type = "code_lines"
+ line_type = "code"
i += 1
if line_type is None:
- line_type = "empty_lines"
+ line_type = "empty"
elif i < len(tokens) and tokens[i][0] == tokenize.NEWLINE:
i += 1
return i, pos[0] - start[0] + 1, line_type
diff --git a/pylint/checkers/similar.py b/pylint/checkers/similar.py
index 9ff5a973b..f37453e36 100644
--- a/pylint/checkers/similar.py
+++ b/pylint/checkers/similar.py
@@ -73,8 +73,7 @@ from astroid import nodes
from pylint.checkers import BaseChecker, MapReduceMixin, table_lines_from_stats
from pylint.interfaces import IRawChecker
from pylint.reporters.ureports.nodes import Table
-from pylint.typing import CheckerStats
-from pylint.utils import decoding_stream
+from pylint.utils import LinterStats, decoding_stream
DEFAULT_MIN_SIMILARITY_LINE = 4
@@ -724,14 +723,12 @@ MSGS = {
def report_similarities(
sect,
- stats: CheckerStats,
- old_stats: CheckerStats,
-):
+ stats: LinterStats,
+ old_stats: Optional[LinterStats],
+) -> None:
"""make a layout with some stats about duplication"""
lines = ["", "now", "previous", "difference"]
- lines += table_lines_from_stats(
- stats, old_stats, ("nb_duplicated_lines", "percent_duplicated_lines")
- )
+ lines += table_lines_from_stats(stats, old_stats, "duplicated_lines")
sect.append(Table(children=lines, cols=4, rheaders=1, cheaders=1))
@@ -809,7 +806,6 @@ class SimilarChecker(BaseChecker, Similar, MapReduceMixin):
ignore_imports=self.config.ignore_imports,
ignore_signatures=self.config.ignore_signatures,
)
- self.stats: CheckerStats = {}
def set_option(self, optname, value, action=None, optdict=None):
"""method called to set an option (registered in the options list)
@@ -831,9 +827,7 @@ class SimilarChecker(BaseChecker, Similar, MapReduceMixin):
def open(self):
"""init the checkers: reset linesets and statistics information"""
self.linesets = []
- self.stats = self.linter.add_stats(
- nb_duplicated_lines=0, percent_duplicated_lines=0
- )
+ self.linter.stats.reset_duplicated_lines()
def process_module(self, node: nodes.Module) -> None:
"""process a module
@@ -849,7 +843,7 @@ class SimilarChecker(BaseChecker, Similar, MapReduceMixin):
"""compute and display similarities on closing (i.e. end of parsing)"""
total = sum(len(lineset) for lineset in self.linesets)
duplicated = 0
- stats = self.stats
+ stats = self.linter.stats
for num, couples in self._compute_sims():
msg = []
lineset = start_line = end_line = None
@@ -863,8 +857,8 @@ class SimilarChecker(BaseChecker, Similar, MapReduceMixin):
self.add_message("R0801", args=(len(couples), "\n".join(msg)))
duplicated += num * (len(couples) - 1)
- stats["nb_duplicated_lines"] = duplicated
- stats["percent_duplicated_lines"] = total and duplicated * 100.0 / total
+ stats.nb_duplicated_lines += int(duplicated)
+ stats.percent_duplicated_lines += float(total and duplicated * 100.0 / total)
def get_map_data(self):
"""Passthru override"""
diff --git a/pylint/config/__init__.py b/pylint/config/__init__.py
index d6e2c1f25..6bf30d3b4 100644
--- a/pylint/config/__init__.py
+++ b/pylint/config/__init__.py
@@ -49,6 +49,7 @@ from pylint.config.option import Option
from pylint.config.option_manager_mixin import OptionsManagerMixIn
from pylint.config.option_parser import OptionParser
from pylint.config.options_provider_mixin import OptionsProviderMixIn, UnsupportedAction
+from pylint.utils import LinterStats
__all__ = [
"ConfigurationMixIn",
@@ -117,9 +118,12 @@ def load_results(base):
data_file = _get_pdata_path(base, 1)
try:
with open(data_file, "rb") as stream:
- return pickle.load(stream)
+ data = pickle.load(stream)
+ if not isinstance(data, LinterStats):
+ raise TypeError
+ return data
except Exception: # pylint: disable=broad-except
- return {}
+ return None
def save_results(results, base):
diff --git a/pylint/lint/parallel.py b/pylint/lint/parallel.py
index b770b7bce..f148e913f 100644
--- a/pylint/lint/parallel.py
+++ b/pylint/lint/parallel.py
@@ -9,7 +9,7 @@ from pylint import reporters
from pylint.lint.utils import _patch_sys_path
from pylint.message import Message
from pylint.typing import FileItem
-from pylint.utils import merge_stats
+from pylint.utils import LinterStats, merge_stats
try:
import multiprocessing
@@ -48,7 +48,9 @@ def _worker_initialize(linter, arguments=None):
def _worker_check_single_file(
file_item: FileItem,
-) -> Tuple[int, Any, str, Any, List[Tuple[Any, ...]], Any, Any, DefaultDict[Any, List]]:
+) -> Tuple[
+ int, Any, str, Any, List[Tuple[Any, ...]], LinterStats, Any, DefaultDict[Any, List]
+]:
if not _worker_linter:
raise Exception("Worker linter not yet initialised")
_worker_linter.open()
diff --git a/pylint/lint/pylinter.py b/pylint/lint/pylinter.py
index 053f5dcb5..728b4855e 100644
--- a/pylint/lint/pylinter.py
+++ b/pylint/lint/pylinter.py
@@ -32,8 +32,8 @@ from pylint.lint.utils import (
)
from pylint.message import MessageDefinitionStore, MessagesHandlerMixIn
from pylint.reporters.ureports import nodes as report_nodes
-from pylint.typing import CheckerStats, FileItem, ModuleDescriptionDict
-from pylint.utils import ASTWalker, FileState, utils
+from pylint.typing import FileItem, ModuleDescriptionDict
+from pylint.utils import ASTWalker, FileState, LinterStats, ModuleStats, utils
from pylint.utils.pragma_parser import (
OPTION_PO,
InvalidPragmaError,
@@ -502,9 +502,9 @@ class PyLinter(
self._ignore_file = False
# visit variables
self.file_state = FileState()
- self.current_name = None
+ self.current_name: Optional[str] = None
self.current_file = None
- self.stats: CheckerStats = {}
+ self.stats = LinterStats()
self.fail_on_symbols = []
# init options
self._external_opts = options
@@ -731,10 +731,8 @@ class PyLinter(
self.fail_on_symbols.append(msg.symbol)
def any_fail_on_issues(self):
- return (
- self.stats
- and self.stats.get("by_msg") is not None
- and any(x in self.fail_on_symbols for x in self.stats["by_msg"])
+ return self.stats and any(
+ x in self.fail_on_symbols for x in self.stats.by_msg.keys()
)
def disable_noerror_messages(self):
@@ -1101,10 +1099,9 @@ class PyLinter(
self.reporter.on_set_current_module(modname, filepath)
self.current_name = modname
self.current_file = filepath or modname
- self.stats["by_module"][modname] = {} # type: ignore # Refactor of PyLinter.stats necessary
- self.stats["by_module"][modname]["statement"] = 0 # type: ignore
- for msg_cat in MSG_TYPES.values():
- self.stats["by_module"][modname][msg_cat] = 0 # type: ignore
+ self.stats.by_module[modname] = ModuleStats(
+ convention=0, error=0, fatal=0, info=0, refactor=0, statement=0, warning=0
+ )
@contextlib.contextmanager
def _astroid_module_checker(self):
@@ -1136,7 +1133,7 @@ class PyLinter(
)
# notify global end
- self.stats["statement"] = walker.nbstatements
+ self.stats.statement = walker.nbstatements
for checker in reversed(_checkers):
checker.close()
@@ -1181,7 +1178,7 @@ class PyLinter(
ast_node, walker, rawcheckers, tokencheckers
)
- self.stats["by_module"][self.current_name]["statement"] = (
+ self.stats.by_module[self.current_name]["statement"] = (
walker.nbstatements - before_check_statements
)
@@ -1231,7 +1228,7 @@ class PyLinter(
def open(self):
"""initialize counters"""
- self.stats = {"by_module": {}, "by_msg": {}}
+ self.stats = LinterStats()
MANAGER.always_load_extensions = self.config.unsafe_load_any_extension
MANAGER.max_inferable_values = self.config.limit_inference_results
MANAGER.extension_package_whitelist.update(self.config.extension_pkg_allow_list)
@@ -1239,8 +1236,7 @@ class PyLinter(
MANAGER.extension_package_whitelist.update(
self.config.extension_pkg_whitelist
)
- for msg_cat in MSG_TYPES.values():
- self.stats[msg_cat] = 0
+ self.stats.reset_message_count()
def generate_reports(self):
"""close the whole package /module, it's time to make reports !
@@ -1266,7 +1262,7 @@ class PyLinter(
if self.config.persistent:
config.save_results(self.stats, self.file_state.base_name)
else:
- self.reporter.on_close(self.stats, {})
+ self.reporter.on_close(self.stats, LinterStats())
score_value = None
return score_value
@@ -1276,21 +1272,29 @@ class PyLinter(
# syntax error preventing pylint from further processing)
note = None
previous_stats = config.load_results(self.file_state.base_name)
- if self.stats["statement"] == 0:
+ if self.stats.statement == 0:
return note
# get a global note for the code
evaluation = self.config.evaluation
try:
- note = eval(evaluation, {}, self.stats) # pylint: disable=eval-used
+ stats_dict = {
+ "error": self.stats.error,
+ "warning": self.stats.warning,
+ "refactor": self.stats.refactor,
+ "convention": self.stats.convention,
+ "statement": self.stats.statement,
+ }
+ note = eval(evaluation, {}, stats_dict) # pylint: disable=eval-used
except Exception as ex: # pylint: disable=broad-except
msg = f"An exception occurred while rating: {ex}"
else:
- self.stats["global_note"] = note
+ self.stats.global_note = note
msg = f"Your code has been rated at {note:.2f}/10"
- pnote = previous_stats.get("global_note")
- if pnote is not None:
- msg += f" (previous run: {pnote:.2f}/10, {note - pnote:+.2f})"
+ if previous_stats:
+ pnote = previous_stats.global_note
+ if pnote is not None:
+ msg += f" (previous run: {pnote:.2f}/10, {note - pnote:+.2f})"
if self.config.score:
sect = report_nodes.EvaluationSection(msg)
diff --git a/pylint/lint/report_functions.py b/pylint/lint/report_functions.py
index 201b28b7b..5e95354c4 100644
--- a/pylint/lint/report_functions.py
+++ b/pylint/lint/report_functions.py
@@ -2,37 +2,32 @@
# For details: https://github.com/PyCQA/pylint/blob/main/LICENSE
import collections
-from typing import DefaultDict, Dict, List, Tuple, Union
+from typing import DefaultDict, Dict, Union
from pylint import checkers, exceptions
from pylint.reporters.ureports.nodes import Table
-from pylint.typing import CheckerStats
+from pylint.utils import LinterStats
def report_total_messages_stats(
sect,
- stats: CheckerStats,
- previous_stats: CheckerStats,
+ stats: LinterStats,
+ previous_stats: LinterStats,
):
"""make total errors / warnings report"""
lines = ["type", "number", "previous", "difference"]
- lines += checkers.table_lines_from_stats(
- stats, previous_stats, ("convention", "refactor", "warning", "error")
- )
+ lines += checkers.table_lines_from_stats(stats, previous_stats, "message_types")
sect.append(Table(children=lines, cols=4, rheaders=1))
def report_messages_stats(
sect,
- stats: CheckerStats,
- _: CheckerStats,
+ stats: LinterStats,
+ _: LinterStats,
):
"""make messages type report"""
- if not stats["by_msg"]:
- # don't print this report when we didn't detected any errors
- raise exceptions.EmptyReportError()
- by_msg_stats: Dict[str, int] = stats["by_msg"] # type: ignore
- in_order: List[Tuple[int, str]] = sorted(
+ by_msg_stats = stats.by_msg
+ in_order = sorted(
(value, msg_id)
for msg_id, value in by_msg_stats.items()
if not msg_id.startswith("I")
@@ -46,11 +41,11 @@ def report_messages_stats(
def report_messages_by_module_stats(
sect,
- stats: CheckerStats,
- _: CheckerStats,
+ stats: LinterStats,
+ _: LinterStats,
):
"""make errors / warnings by modules report"""
- module_stats: Dict[str, Dict[str, int]] = stats["by_module"] # type: ignore
+ module_stats = stats.by_module
if len(module_stats) == 1:
# don't print this report when we are analysing a single module
raise exceptions.EmptyReportError()
@@ -58,9 +53,9 @@ def report_messages_by_module_stats(
dict
)
for m_type in ("fatal", "error", "warning", "refactor", "convention"):
- total: int = stats[m_type] # type: ignore
+ total = stats.get_global_message_count(m_type)
for module in module_stats.keys():
- mod_total = module_stats[module][m_type]
+ mod_total = stats.get_module_message_count(module, m_type)
percent = 0 if total == 0 else float(mod_total * 100) / total
by_mod[module][m_type] = percent
sorted_result = []
diff --git a/pylint/message/message_handler_mix_in.py b/pylint/message/message_handler_mix_in.py
index 662a086cb..db8f6d0be 100644
--- a/pylint/message/message_handler_mix_in.py
+++ b/pylint/message/message_handler_mix_in.py
@@ -332,23 +332,13 @@ class MessagesHandlerMixIn:
# update stats
msg_cat = MSG_TYPES[message_definition.msgid[0]]
self.msg_status |= MSG_TYPES_STATUS[message_definition.msgid[0]]
- if self.stats is None:
- # pylint: disable=fixme
- # TODO self.stats should make sense,
- # class should make sense as soon as instantiated
- # This is not true for Linter and Reporter at least
- # pylint: enable=fixme
- self.stats = {
- msg_cat: 0,
- "by_module": {self.current_name: {msg_cat: 0}},
- "by_msg": {},
- }
- self.stats[msg_cat] += 1 # type: ignore
- self.stats["by_module"][self.current_name][msg_cat] += 1 # type: ignore
+
+ self.stats.increase_single_message_count(msg_cat, 1)
+ self.stats.increase_single_module_message_count(self.current_name, msg_cat, 1)
try:
- self.stats["by_msg"][message_definition.symbol] += 1 # type: ignore
+ self.stats.by_msg[message_definition.symbol] += 1
except KeyError:
- self.stats["by_msg"][message_definition.symbol] = 1 # type: ignore
+ self.stats.by_msg[message_definition.symbol] = 1
# Interpolate arguments into message string
msg = message_definition.msg
if args:
@@ -369,7 +359,7 @@ class MessagesHandlerMixIn:
Message(
message_definition.msgid,
message_definition.symbol,
- (abspath, path, module, obj, line or 1, col_offset or 0),
+ (abspath, path, module, obj, line or 1, col_offset or 0), # type: ignore
msg,
confidence,
)
diff --git a/pylint/reporters/base_reporter.py b/pylint/reporters/base_reporter.py
index 244935a6b..21549c943 100644
--- a/pylint/reporters/base_reporter.py
+++ b/pylint/reporters/base_reporter.py
@@ -8,7 +8,7 @@ from warnings import warn
from pylint.message import Message
from pylint.reporters.ureports.nodes import Text
-from pylint.typing import CheckerStats
+from pylint.utils import LinterStats
if TYPE_CHECKING:
from pylint.lint.pylinter import PyLinter
@@ -81,7 +81,7 @@ class BaseReporter:
def on_close(
self,
- stats: CheckerStats,
- previous_stats: CheckerStats,
+ stats: LinterStats,
+ previous_stats: LinterStats,
) -> None:
"""Hook called when a module finished analyzing."""
diff --git a/pylint/reporters/multi_reporter.py b/pylint/reporters/multi_reporter.py
index b3b67d87a..a6be1c1b8 100644
--- a/pylint/reporters/multi_reporter.py
+++ b/pylint/reporters/multi_reporter.py
@@ -8,7 +8,7 @@ from typing import IO, TYPE_CHECKING, Any, AnyStr, Callable, List, Optional
from pylint.interfaces import IReporter
from pylint.message import Message
from pylint.reporters.base_reporter import BaseReporter
-from pylint.typing import CheckerStats
+from pylint.utils import LinterStats
if TYPE_CHECKING:
from pylint.reporters.ureports.nodes import Section
@@ -97,8 +97,8 @@ class MultiReporter:
def on_close(
self,
- stats: CheckerStats,
- previous_stats: CheckerStats,
+ stats: LinterStats,
+ previous_stats: LinterStats,
) -> None:
"""hook called when a module finished analyzing"""
for rep in self._sub_reporters:
diff --git a/pylint/reporters/reports_handler_mix_in.py b/pylint/reporters/reports_handler_mix_in.py
index 4c302dbf4..14631740d 100644
--- a/pylint/reporters/reports_handler_mix_in.py
+++ b/pylint/reporters/reports_handler_mix_in.py
@@ -2,12 +2,12 @@
# For details: https://github.com/PyCQA/pylint/blob/main/LICENSE
import collections
-from typing import TYPE_CHECKING, Any, Callable, DefaultDict, Dict, List, Tuple
+from typing import TYPE_CHECKING, Callable, DefaultDict, Dict, List, Optional, Tuple
from pylint.exceptions import EmptyReportError
from pylint.interfaces import IChecker
from pylint.reporters.ureports.nodes import Section
-from pylint.typing import CheckerStats
+from pylint.utils import LinterStats
if TYPE_CHECKING:
from pylint.lint.pylinter import PyLinter
@@ -59,11 +59,11 @@ class ReportsHandlerMixIn:
def make_reports( # type: ignore # ReportsHandlerMixIn is always mixed with PyLinter
self: "PyLinter",
- stats: CheckerStats,
- old_stats: CheckerStats,
+ stats: LinterStats,
+ old_stats: Optional[LinterStats],
) -> Section:
"""render registered reports"""
- sect = Section("Report", f"{self.stats['statement']} statements analysed.")
+ sect = Section("Report", f"{self.stats.statement} statements analysed.")
for checker in self.report_order():
for reportid, r_title, r_cb in self._reports[checker]:
if not self.report_is_enabled(reportid):
@@ -76,16 +76,3 @@ class ReportsHandlerMixIn:
report_sect.report_id = reportid
sect.append(report_sect)
return sect
-
- def add_stats( # type: ignore # ReportsHandlerMixIn is always mixed with PyLinter
- self: "PyLinter", **kwargs: Any
- ) -> CheckerStats:
- """add some stats entries to the statistic dictionary
- raise an AssertionError if there is a key conflict
- """
- for key, value in kwargs.items():
- if key[-1] == "_":
- key = key[:-1]
- assert key not in self.stats
- self.stats[key] = value
- return self.stats
diff --git a/pylint/testutils/unittest_linter.py b/pylint/testutils/unittest_linter.py
index c6c187abb..2d9e42e65 100644
--- a/pylint/testutils/unittest_linter.py
+++ b/pylint/testutils/unittest_linter.py
@@ -8,7 +8,7 @@ from astroid import nodes
from pylint.interfaces import Confidence
from pylint.testutils.global_test_linter import linter
from pylint.testutils.output_line import MessageTest
-from pylint.typing import CheckerStats
+from pylint.utils import LinterStats
class UnittestLinter:
@@ -18,7 +18,7 @@ class UnittestLinter:
def __init__(self):
self._messages = []
- self.stats: CheckerStats = {}
+ self.stats = LinterStats()
def release_messages(self):
try:
@@ -42,11 +42,6 @@ class UnittestLinter:
def is_message_enabled(*unused_args, **unused_kwargs):
return True
- def add_stats(self, **kwargs):
- for name, value in kwargs.items():
- self.stats[name] = value
- return self.stats
-
@property
def options_providers(self):
return linter.options_providers
diff --git a/pylint/typing.py b/pylint/typing.py
index b0171d0d8..2d8e74763 100644
--- a/pylint/typing.py
+++ b/pylint/typing.py
@@ -3,10 +3,7 @@
"""A collection of typing utilities."""
import sys
-from typing import TYPE_CHECKING, Dict, List, NamedTuple, Union
-
-if TYPE_CHECKING:
- from typing import Counter # typing.Counter added in Python 3.6.1
+from typing import NamedTuple, Union
if sys.version_info >= (3, 8):
from typing import Literal, TypedDict
@@ -46,12 +43,6 @@ class ErrorDescriptionDict(TypedDict):
ex: Union[ImportError, SyntaxError]
-# The base type of the "stats" attribute of a checker
-CheckerStats = Dict[
- str, Union[int, "Counter[str]", List, Dict[str, Union[int, str, Dict[str, int]]]]
-]
-
-
class MessageLocationTuple(NamedTuple):
"""Tuple with information about the location of a to-be-displayed message"""
diff --git a/pylint/utils/__init__.py b/pylint/utils/__init__.py
index a0658bf99..73f2e0fa0 100644
--- a/pylint/utils/__init__.py
+++ b/pylint/utils/__init__.py
@@ -44,8 +44,8 @@ main pylint class
"""
from pylint.utils.ast_walker import ASTWalker
-from pylint.utils.checkerstats import merge_stats
from pylint.utils.file_state import FileState
+from pylint.utils.linterstats import LinterStats, ModuleStats, merge_stats
from pylint.utils.utils import (
HAS_ISORT_5,
IsortDriver,
@@ -85,4 +85,6 @@ __all__ = [
"register_plugins",
"tokenize_module",
"merge_stats",
+ "LinterStats",
+ "ModuleStats",
]
diff --git a/pylint/utils/checkerstats.py b/pylint/utils/checkerstats.py
deleted file mode 100644
index 42af08167..000000000
--- a/pylint/utils/checkerstats.py
+++ /dev/null
@@ -1,30 +0,0 @@
-# Licensed under the GPL: https://www.gnu.org/licenses/old-licenses/gpl-2.0.html
-# For details: https://github.com/PyCQA/pylint/blob/main/LICENSE
-
-import collections
-from typing import TYPE_CHECKING, Dict, List, Union
-
-from pylint.typing import CheckerStats
-
-if TYPE_CHECKING:
- from typing import Counter # typing.Counter added in Python 3.6.1
-
-
-def merge_stats(stats: List[CheckerStats]):
- """Used to merge two stats objects into a new one when pylint is run in parallel mode"""
- merged: CheckerStats = {}
- by_msg: "Counter[str]" = collections.Counter()
- for stat in stats:
- message_stats: Union["Counter[str]", Dict] = stat.pop("by_msg", {}) # type: ignore
- by_msg.update(message_stats)
-
- for key, item in stat.items():
- if key not in merged:
- merged[key] = item
- elif isinstance(item, dict):
- merged[key].update(item) # type: ignore
- else:
- merged[key] = merged[key] + item # type: ignore
-
- merged["by_msg"] = by_msg
- return merged
diff --git a/pylint/utils/linterstats.py b/pylint/utils/linterstats.py
new file mode 100644
index 000000000..f46054455
--- /dev/null
+++ b/pylint/utils/linterstats.py
@@ -0,0 +1,365 @@
+# Licensed under the GPL: https://www.gnu.org/licenses/old-licenses/gpl-2.0.html
+# For details: https://github.com/PyCQA/pylint/blob/main/LICENSE
+
+import sys
+from typing import Dict, List, Optional, Set, cast
+
+if sys.version_info >= (3, 8):
+ from typing import Literal, TypedDict
+else:
+ from typing_extensions import Literal, TypedDict
+
+
+class BadNames(TypedDict):
+ """TypedDict to store counts of node types with bad names"""
+
+ argument: int
+ attr: int
+ klass: int
+ class_attribute: int
+ class_const: int
+ const: int
+ inlinevar: int
+ function: int
+ method: int
+ module: int
+ variable: int
+
+
+class CodeTypeCount(TypedDict):
+ """TypedDict to store counts of lines of code types"""
+
+ code: int
+ comment: int
+ docstring: int
+ empty: int
+ total: int
+
+
+class DuplicatedLines(TypedDict):
+ """TypedDict to store counts of lines of duplicated code"""
+
+ nb_duplicated_lines: int
+ percent_duplicated_lines: float
+
+
+class NodeCount(TypedDict):
+ """TypedDict to store counts of different types of nodes"""
+
+ function: int
+ klass: int
+ method: int
+ module: int
+
+
+class UndocumentedNodes(TypedDict):
+ """TypedDict to store counts of undocumented node types"""
+
+ function: int
+ klass: int
+ method: int
+ module: int
+
+
+class ModuleStats(TypedDict):
+ """TypedDict to store counts of types of messages and statements"""
+
+ convention: int
+ error: int
+ fatal: int
+ info: int
+ refactor: int
+ statement: int
+ warning: int
+
+
+# pylint: disable-next=too-many-instance-attributes
+class LinterStats:
+ """Class used to linter stats"""
+
+ def __init__(
+ self,
+ bad_names: Optional[BadNames] = None,
+ by_module: Optional[Dict[str, ModuleStats]] = None,
+ by_msg: Optional[Dict[str, int]] = None,
+ code_type_count: Optional[CodeTypeCount] = None,
+ dependencies: Optional[Dict[str, Set[str]]] = None,
+ duplicated_lines: Optional[DuplicatedLines] = None,
+ node_count: Optional[NodeCount] = None,
+ undocumented: Optional[UndocumentedNodes] = None,
+ ) -> None:
+ self.bad_names = bad_names or BadNames(
+ argument=0,
+ attr=0,
+ klass=0,
+ class_attribute=0,
+ class_const=0,
+ const=0,
+ inlinevar=0,
+ function=0,
+ method=0,
+ module=0,
+ variable=0,
+ )
+ self.by_module: Dict[str, ModuleStats] = by_module or {}
+ self.by_msg: Dict[str, int] = by_msg or {}
+ self.code_type_count = code_type_count or CodeTypeCount(
+ code=0, comment=0, docstring=0, empty=0, total=0
+ )
+
+ self.dependencies: Dict[str, Set[str]] = dependencies or {}
+ self.duplicated_lines = duplicated_lines or DuplicatedLines(
+ nb_duplicated_lines=0, percent_duplicated_lines=0.0
+ )
+ self.node_count = node_count or NodeCount(
+ function=0, klass=0, method=0, module=0
+ )
+ self.undocumented = undocumented or UndocumentedNodes(
+ function=0, klass=0, method=0, module=0
+ )
+
+ self.convention = 0
+ self.error = 0
+ self.fatal = 0
+ self.info = 0
+ self.refactor = 0
+ self.statement = 0
+ self.warning = 0
+
+ self.global_note = 0
+ self.nb_duplicated_lines = 0
+ self.percent_duplicated_lines = 0.0
+
+ def __str__(self) -> str:
+ return f"""{self.bad_names}
+ {sorted(self.by_module.items())}
+ {sorted(self.by_msg.items())}
+ {self.code_type_count}
+ {sorted(self.dependencies.items())}
+ {self.duplicated_lines}
+ {self.undocumented}
+ {self.convention}
+ {self.error}
+ {self.fatal}
+ {self.info}
+ {self.refactor}
+ {self.statement}
+ {self.warning}
+ {self.global_note}
+ {self.nb_duplicated_lines}
+ {self.percent_duplicated_lines}"""
+
+ def get_bad_names(
+ self,
+ node_name: Literal[
+ "argument",
+ "attr",
+ "class",
+ "class_attribute",
+ "class_const",
+ "const",
+ "inlinevar",
+ "function",
+ "method",
+ "module",
+ "variable",
+ ],
+ ) -> int:
+ """Get a bad names node count"""
+ if node_name == "class":
+ return self.bad_names.get("klass", 0)
+ return self.bad_names.get(node_name, 0)
+
+ def increase_bad_name(self, node_name: str, increase: int) -> None:
+ """Increase a bad names node count"""
+ if node_name not in {
+ "argument",
+ "attr",
+ "class",
+ "class_attribute",
+ "class_const",
+ "const",
+ "inlinevar",
+ "function",
+ "method",
+ "module",
+ "variable",
+ }:
+ raise ValueError("Node type not part of the bad_names stat")
+
+ node_name = cast(
+ Literal[
+ "argument",
+ "attr",
+ "class",
+ "class_attribute",
+ "class_const",
+ "const",
+ "inlinevar",
+ "function",
+ "method",
+ "module",
+ "variable",
+ ],
+ node_name,
+ )
+ if node_name == "class":
+ self.bad_names["klass"] += increase
+ else:
+ self.bad_names[node_name] += increase
+
+ def reset_bad_names(self) -> None:
+ """Resets the bad_names attribute"""
+ self.bad_names = BadNames(
+ argument=0,
+ attr=0,
+ klass=0,
+ class_attribute=0,
+ class_const=0,
+ const=0,
+ inlinevar=0,
+ function=0,
+ method=0,
+ module=0,
+ variable=0,
+ )
+
+ def get_code_count(
+ self, type_name: Literal["code", "comment", "docstring", "empty", "total"]
+ ) -> int:
+ """Get a code type count"""
+ return self.code_type_count.get(type_name, 0)
+
+ def reset_code_count(self) -> None:
+ """Resets the code_type_count attribute"""
+ self.code_type_count = CodeTypeCount(
+ code=0, comment=0, docstring=0, empty=0, total=0
+ )
+
+ def reset_duplicated_lines(self) -> None:
+ """Resets the duplicated_lines attribute"""
+ self.duplicated_lines = DuplicatedLines(
+ nb_duplicated_lines=0, percent_duplicated_lines=0.0
+ )
+
+ def get_node_count(
+ self, node_name: Literal["function", "class", "method", "module"]
+ ) -> int:
+ """Get a node count while handling some extra conditions"""
+ if node_name == "class":
+ return self.node_count.get("klass", 0)
+ return self.node_count.get(node_name, 0)
+
+ def reset_node_count(self) -> None:
+ """Resets the node count attribute"""
+ self.node_count = NodeCount(function=0, klass=0, method=0, module=0)
+
+ def get_undocumented(
+ self, node_name: Literal["function", "class", "method", "module"]
+ ) -> float:
+ """Get a undocumented node count"""
+ if node_name == "class":
+ return self.undocumented["klass"]
+ return self.undocumented[node_name]
+
+ def reset_undocumented(self) -> None:
+ """Resets the undocumented attribute"""
+ self.undocumented = UndocumentedNodes(function=0, klass=0, method=0, module=0)
+
+ def get_global_message_count(self, type_name: str) -> int:
+ """Get a global message count"""
+ return getattr(self, type_name, 0)
+
+ def get_module_message_count(self, modname: str, type_name: str) -> int:
+ """Get a module message count"""
+ return getattr(self.by_module[modname], type_name, 0)
+
+ def increase_single_message_count(self, type_name: str, increase: int) -> None:
+ """Increase the message type count of an individual message type"""
+ setattr(self, type_name, getattr(self, type_name) + increase)
+
+ def increase_single_module_message_count(
+ self,
+ modname: str,
+ type_name: Literal["convention", "error", "fatal", "info", "refactor"],
+ increase: int,
+ ) -> None:
+ """Increase the message type count of an individual message type of a module"""
+ self.by_module[modname][type_name] = (
+ self.by_module[modname][type_name] + increase
+ )
+
+ def reset_message_count(self) -> None:
+ """Resets the message type count of the stats object"""
+ self.convention = 0
+ self.error = 0
+ self.fatal = 0
+ self.info = 0
+ self.refactor = 0
+ self.warning = 0
+
+
+def merge_stats(stats: List[LinterStats]):
+ """Used to merge multiple stats objects into a new one when pylint is run in parallel mode"""
+ merged = LinterStats()
+ for stat in stats:
+ merged.bad_names["argument"] += stat.bad_names["argument"]
+ merged.bad_names["attr"] += stat.bad_names["attr"]
+ merged.bad_names["klass"] += stat.bad_names["klass"]
+ merged.bad_names["class_attribute"] += stat.bad_names["class_attribute"]
+ merged.bad_names["class_const"] += stat.bad_names["class_const"]
+ merged.bad_names["const"] += stat.bad_names["const"]
+ merged.bad_names["inlinevar"] += stat.bad_names["inlinevar"]
+ merged.bad_names["function"] += stat.bad_names["function"]
+ merged.bad_names["method"] += stat.bad_names["method"]
+ merged.bad_names["module"] += stat.bad_names["module"]
+ merged.bad_names["variable"] += stat.bad_names["variable"]
+
+ for mod_key, mod_value in stat.by_module.items():
+ merged.by_module[mod_key] = mod_value
+
+ for msg_key, msg_value in stat.by_msg.items():
+ try:
+ merged.by_msg[msg_key] += msg_value
+ except KeyError:
+ merged.by_msg[msg_key] = msg_value
+
+ merged.code_type_count["code"] += stat.code_type_count["code"]
+ merged.code_type_count["comment"] += stat.code_type_count["comment"]
+ merged.code_type_count["docstring"] += stat.code_type_count["docstring"]
+ merged.code_type_count["empty"] += stat.code_type_count["empty"]
+ merged.code_type_count["total"] += stat.code_type_count["total"]
+
+ for dep_key, dep_value in stat.dependencies.items():
+ try:
+ merged.dependencies[dep_key].update(dep_value)
+ except KeyError:
+ merged.dependencies[dep_key] = dep_value
+
+ merged.duplicated_lines["nb_duplicated_lines"] += stat.duplicated_lines[
+ "nb_duplicated_lines"
+ ]
+ merged.duplicated_lines["percent_duplicated_lines"] += stat.duplicated_lines[
+ "percent_duplicated_lines"
+ ]
+
+ merged.node_count["function"] += stat.node_count["function"]
+ merged.node_count["klass"] += stat.node_count["klass"]
+ merged.node_count["method"] += stat.node_count["method"]
+ merged.node_count["module"] += stat.node_count["module"]
+
+ merged.undocumented["function"] += stat.undocumented["function"]
+ merged.undocumented["klass"] += stat.undocumented["klass"]
+ merged.undocumented["method"] += stat.undocumented["method"]
+ merged.undocumented["module"] += stat.undocumented["module"]
+
+ merged.convention += stat.convention
+ merged.error += stat.error
+ merged.fatal += stat.fatal
+ merged.info += stat.info
+ merged.refactor += stat.refactor
+ merged.statement += stat.statement
+ merged.warning += stat.warning
+
+ merged.global_note += stat.global_note
+ return merged
diff --git a/tests/lint/unittest_lint.py b/tests/lint/unittest_lint.py
index 277799676..79437e987 100644
--- a/tests/lint/unittest_lint.py
+++ b/tests/lint/unittest_lint.py
@@ -48,7 +48,7 @@ from io import StringIO
from os import chdir, getcwd
from os.path import abspath, basename, dirname, isdir, join, sep
from shutil import rmtree
-from typing import Dict, Iterable, Iterator, List, Optional, Tuple
+from typing import Iterable, Iterator, List, Optional, Tuple
import platformdirs
import pytest
@@ -870,7 +870,7 @@ def test_by_module_statement_value(init_linter: PyLinter) -> None:
linter = init_linter
linter.check([os.path.join(os.path.dirname(__file__), "data")])
- by_module_stats: Dict[str, Dict[str, int]] = linter.stats["by_module"] # type: ignore
+ by_module_stats = linter.stats.by_module
for module, module_stats in by_module_stats.items():
linter2 = init_linter
@@ -881,4 +881,4 @@ def test_by_module_statement_value(init_linter: PyLinter) -> None:
# Check that the by_module "statement" is equal to the global "statement"
# computed for that module
- assert module_stats["statement"] == linter2.stats["statement"]
+ assert module_stats["statement"] == linter2.stats.statement
diff --git a/tests/test_check_parallel.py b/tests/test_check_parallel.py
index c0fbe00fd..be250d892 100644
--- a/tests/test_check_parallel.py
+++ b/tests/test_check_parallel.py
@@ -10,7 +10,6 @@
# pylint: disable=protected-access,missing-function-docstring,no-self-use
-import collections
import os
from typing import List
@@ -25,7 +24,8 @@ from pylint.lint.parallel import _worker_check_single_file as worker_check_singl
from pylint.lint.parallel import _worker_initialize as worker_initialize
from pylint.lint.parallel import check_parallel
from pylint.testutils import GenericTestReporter as Reporter
-from pylint.typing import CheckerStats, FileItem
+from pylint.typing import FileItem
+from pylint.utils import LinterStats, ModuleStats
def _gen_file_data(idx: int = 0) -> FileItem:
@@ -103,11 +103,10 @@ class ParallelTestChecker(BaseChecker):
super().__init__(linter)
self.data: List[str] = []
self.linter = linter
- self.stats: CheckerStats = {}
def open(self) -> None:
"""init the checkers: reset statistics information"""
- self.stats = self.linter.add_stats()
+ self.linter.stats.reset_node_count()
self.data = []
def close(self) -> None:
@@ -202,26 +201,24 @@ class TestCheckParallelFramework:
no_errors_status = 0
assert no_errors_status == msg_status
assert {
- "by_module": {
- "--test-file_data-name-0--": {
- "convention": 0,
- "error": 0,
- "fatal": 0,
- "info": 0,
- "refactor": 0,
- "statement": 18,
- "warning": 0,
- }
- },
- "by_msg": {},
- "convention": 0,
- "error": 0,
- "fatal": 0,
- "info": 0,
- "refactor": 0,
- "statement": 18,
- "warning": 0,
- } == stats
+ "--test-file_data-name-0--": {
+ "convention": 0,
+ "error": 0,
+ "fatal": 0,
+ "info": 0,
+ "refactor": 0,
+ "statement": 18,
+ "warning": 0,
+ }
+ } == stats.by_module
+ assert {} == stats.by_msg
+ assert stats.convention == 0
+ assert stats.error == 0
+ assert stats.fatal == 0
+ assert stats.info == 0
+ assert stats.refactor == 0
+ assert stats.statement == 18
+ assert stats.warning == 0
def test_worker_check_sequential_checker(self) -> None:
"""Same as test_worker_check_single_file_no_checkers with SequentialTestChecker"""
@@ -248,26 +245,24 @@ class TestCheckParallelFramework:
no_errors_status = 0
assert no_errors_status == msg_status
assert {
- "by_module": {
- "--test-file_data-name-0--": {
- "convention": 0,
- "error": 0,
- "fatal": 0,
- "info": 0,
- "refactor": 0,
- "statement": 18,
- "warning": 0,
- }
- },
- "by_msg": {},
- "convention": 0,
- "error": 0,
- "fatal": 0,
- "info": 0,
- "refactor": 0,
- "statement": 18,
- "warning": 0,
- } == stats
+ "--test-file_data-name-0--": {
+ "convention": 0,
+ "error": 0,
+ "fatal": 0,
+ "info": 0,
+ "refactor": 0,
+ "statement": 18,
+ "warning": 0,
+ }
+ } == stats.by_module
+ assert {} == stats.by_msg
+ assert stats.convention == 0
+ assert stats.error == 0
+ assert stats.fatal == 0
+ assert stats.info == 0
+ assert stats.refactor == 0
+ assert stats.statement == 18
+ assert stats.warning == 0
class TestCheckParallel:
@@ -299,52 +294,48 @@ class TestCheckParallel:
"checkers registered"
)
assert {
- "by_module": {
- "--test-file_data-name-0--": {
- "convention": 0,
- "error": 0,
- "fatal": 0,
- "info": 0,
- "refactor": 0,
- "statement": 18,
- "warning": 0,
- }
- },
- "by_msg": {},
- "convention": 0,
- "error": 0,
- "fatal": 0,
- "info": 0,
- "refactor": 0,
- "statement": 18,
- "warning": 0,
- } == linter.stats
+ "--test-file_data-name-0--": {
+ "convention": 0,
+ "error": 0,
+ "fatal": 0,
+ "info": 0,
+ "refactor": 0,
+ "statement": 18,
+ "warning": 0,
+ }
+ } == linter.stats.by_module
+ assert linter.stats.by_msg == {}
+ assert linter.stats.convention == 0
+ assert linter.stats.error == 0
+ assert linter.stats.fatal == 0
+ assert linter.stats.info == 0
+ assert linter.stats.refactor == 0
+ assert linter.stats.statement == 18
+ assert linter.stats.warning == 0
# now run the regular mode of checking files and check that, in this proc, we
# collect the right data
filepath = [single_file_container[0][1]] # get the filepath element
linter.check(filepath)
assert {
- "by_module": {
- "input.similar1": { # module is the only change from previous
- "convention": 0,
- "error": 0,
- "fatal": 0,
- "info": 0,
- "refactor": 0,
- "statement": 18,
- "warning": 0,
- }
- },
- "by_msg": {},
- "convention": 0,
- "error": 0,
- "fatal": 0,
- "info": 0,
- "refactor": 0,
- "statement": 18,
- "warning": 0,
- } == linter.stats
+ "input.similar1": { # module is the only change from previous
+ "convention": 0,
+ "error": 0,
+ "fatal": 0,
+ "info": 0,
+ "refactor": 0,
+ "statement": 18,
+ "warning": 0,
+ }
+ } == linter.stats.by_module
+ assert linter.stats.by_msg == {}
+ assert linter.stats.convention == 0
+ assert linter.stats.error == 0
+ assert linter.stats.fatal == 0
+ assert linter.stats.info == 0
+ assert linter.stats.refactor == 0
+ assert linter.stats.statement == 18
+ assert linter.stats.warning == 0
def test_invoke_single_job(self) -> None:
"""Tests basic checkers functionality using just a single workderdo
@@ -365,26 +356,24 @@ class TestCheckParallel:
)
assert {
- "by_module": {
- "--test-file_data-name-0--": {
- "convention": 0,
- "error": 0,
- "fatal": 0,
- "info": 0,
- "refactor": 0,
- "statement": 18,
- "warning": 0,
- }
- },
- "by_msg": collections.Counter(),
- "convention": 0,
- "error": 0,
- "fatal": 0,
- "info": 0,
- "refactor": 0,
- "statement": 18,
- "warning": 0,
- } == linter.stats
+ "--test-file_data-name-0--": {
+ "convention": 0,
+ "error": 0,
+ "fatal": 0,
+ "info": 0,
+ "refactor": 0,
+ "statement": 18,
+ "warning": 0,
+ }
+ } == linter.stats.by_module
+ assert linter.stats.by_msg == {}
+ assert linter.stats.convention == 0
+ assert linter.stats.error == 0
+ assert linter.stats.fatal == 0
+ assert linter.stats.info == 0
+ assert linter.stats.refactor == 0
+ assert linter.stats.statement == 18
+ assert linter.stats.warning == 0
assert linter.msg_status == 0, "We expect a single-file check to exit cleanly"
@pytest.mark.parametrize(
@@ -425,30 +414,30 @@ class TestCheckParallel:
# define the stats we expect to get back from the runs, these should only vary
# with the number of files.
- expected_stats = {
- "by_module": {
+ expected_stats = LinterStats(
+ by_module={
# pylint: disable-next=consider-using-f-string
"--test-file_data-name-%d--"
- % idx: {
- "convention": 0,
- "error": 0,
- "fatal": 0,
- "info": 0,
- "refactor": 0,
- "statement": 18,
- "warning": 0,
- }
+ % idx: ModuleStats(
+ convention=0,
+ error=0,
+ fatal=0,
+ info=0,
+ refactor=0,
+ statement=18,
+ warning=0,
+ )
for idx in range(num_files)
- },
- "by_msg": {},
- "convention": 0,
- "error": 0,
- "fatal": 0,
- "info": 0,
- "refactor": 0,
- "statement": 18 * num_files,
- "warning": 0,
- }
+ }
+ )
+ expected_stats.by_msg = {}
+ expected_stats.convention = 0
+ expected_stats.error = 0
+ expected_stats.fatal = 0
+ expected_stats.info = 0
+ expected_stats.refactor = 0
+ expected_stats.statement = 18 * num_files
+ expected_stats.warning = 0
file_infos = _gen_file_datas(num_files)
@@ -482,11 +471,11 @@ class TestCheckParallel:
stats_check_parallel = linter.stats
assert linter.msg_status == 0, "We should not fail the lint"
- assert (
- stats_single_proc == stats_check_parallel
+ assert str(stats_single_proc) == str(
+ stats_check_parallel
), "Single-proc and check_parallel() should return the same thing"
- assert (
- stats_check_parallel == expected_stats
+ assert str(stats_check_parallel) == str(
+ expected_stats
), "The lint is returning unexpected results, has something changed?"
@pytest.mark.parametrize(
@@ -551,6 +540,6 @@ class TestCheckParallel:
arguments=None,
)
stats_check_parallel = linter.stats
- assert (
- stats_single_proc["by_msg"] == stats_check_parallel["by_msg"]
+ assert str(stats_single_proc.by_msg) == str(
+ stats_check_parallel.by_msg
), "Single-proc and check_parallel() should return the same thing"
diff --git a/tests/test_import_graph.py b/tests/test_import_graph.py
index 42e3baf56..6035601bf 100644
--- a/tests/test_import_graph.py
+++ b/tests/test_import_graph.py
@@ -48,7 +48,7 @@ POSSIBLE_DOT_FILENAMES = ["foo.dot", "foo.gv", "tests/regrtest_data/foo.dot"]
@pytest.mark.parametrize("dest", POSSIBLE_DOT_FILENAMES, indirect=True)
def test_dependencies_graph(dest: str) -> None:
"""DOC files are correctly generated, and the graphname is the basename"""
- imports._dependencies_graph(dest, {"labas": ["hoho", "yep"], "hoho": ["yep"]})
+ imports._dependencies_graph(dest, {"labas": {"hoho", "yep"}, "hoho": {"yep"}})
with open(dest, encoding="utf-8") as stream:
assert (
stream.read().strip()
@@ -75,7 +75,7 @@ URL="." node[shape="box"]
def test_missing_graphviz(filename: str) -> None:
"""Raises if graphviz is not installed, and defaults to png if no extension given"""
with pytest.raises(RuntimeError, match=r"Cannot generate `graph\.png`.*"):
- imports._dependencies_graph(filename, {"a": ["b", "c"], "b": ["c"]})
+ imports._dependencies_graph(filename, {"a": {"b", "c"}, "b": {"c"}})
@pytest.fixture
diff --git a/tests/test_regr.py b/tests/test_regr.py
index 07385a64d..6dadcf2c6 100644
--- a/tests/test_regr.py
+++ b/tests/test_regr.py
@@ -114,12 +114,12 @@ def modify_path() -> Iterator:
def test_check_package___init__(finalize_linter: PyLinter) -> None:
filename = ["package.__init__"]
finalize_linter.check(filename)
- checked = list(finalize_linter.stats["by_module"].keys()) # type: ignore # Refactor of PyLinter.stats necessary
- assert checked == filename
+ checked = list(finalize_linter.stats.by_module.keys())
+ assert sorted(checked) == sorted(filename)
os.chdir(join(REGR_DATA, "package"))
finalize_linter.check(["__init__"])
- checked = list(finalize_linter.stats["by_module"].keys()) # type: ignore
+ checked = list(finalize_linter.stats.by_module.keys())
assert checked == ["__init__"]