diff options
Diffstat (limited to 'pylint/lint/parallel.py')
| -rw-r--r-- | pylint/lint/parallel.py | 49 |
1 files changed, 34 insertions, 15 deletions
diff --git a/pylint/lint/parallel.py b/pylint/lint/parallel.py index eaacb710d..6dbeed983 100644 --- a/pylint/lint/parallel.py +++ b/pylint/lint/parallel.py @@ -3,7 +3,18 @@ import collections import functools -from typing import Any, DefaultDict, Iterable, List, Tuple +from typing import ( + TYPE_CHECKING, + Any, + DefaultDict, + Iterable, + List, + Sequence, + Tuple, + Union, +) + +import dill from pylint import reporters from pylint.lint.utils import _patch_sys_path @@ -16,6 +27,9 @@ try: except ImportError: multiprocessing = None # type: ignore[assignment] +if TYPE_CHECKING: + from pylint.lint import PyLinter + # PyLinter object used by worker processes when checking files using multiprocessing # should only be used by the worker processes _worker_linter = None @@ -33,9 +47,16 @@ def _get_new_args(message): return (message.msg_id, message.symbol, location, message.msg, message.confidence) -def _worker_initialize(linter, arguments=None): +def _worker_initialize( + linter: bytes, arguments: Union[None, str, Sequence[str]] = None +) -> None: + """Function called to initialize a worker for a Process within a multiprocessing Pool + + :param linter: A linter-class (PyLinter) instance pickled with dill + :param arguments: File or module name(s) to lint and to be added to sys.path + """ global _worker_linter # pylint: disable=global-statement - _worker_linter = linter + _worker_linter = dill.loads(linter) # On the worker process side the messages are just collected and passed back to # parent process as _worker_check_file function's return value @@ -97,26 +118,24 @@ def _merge_mapreduce_data(linter, all_mapreduce_data): checker.reduce_map_data(linter, collated_map_reduce_data[checker.name]) -def check_parallel(linter, jobs, files: Iterable[FileItem], arguments=None): +def check_parallel( + linter: "PyLinter", + jobs: int, + files: Iterable[FileItem], + arguments: Union[None, str, Sequence[str]] = None, +) -> None: """Use the given linter to lint the files with given amount of workers (jobs) This splits the work filestream-by-filestream. If you need to do work across multiple files, as in the similarity-checker, then inherit from MapReduceMixin and - implement the map/reduce mixin functionality""" - # The reporter does not need to be passed to worker processes, i.e. the reporter does - original_reporter = linter.reporter - linter.reporter = None - + implement the map/reduce mixin functionality. + """ # The linter is inherited by all the pool's workers, i.e. the linter # is identical to the linter object here. This is required so that # a custom PyLinter object can be used. initializer = functools.partial(_worker_initialize, arguments=arguments) pool = multiprocessing.Pool( # pylint: disable=consider-using-with - jobs, initializer=initializer, initargs=[linter] + jobs, initializer=initializer, initargs=[dill.dumps(linter)] ) - # ...and now when the workers have inherited the linter, the actual reporter - # can be set back here on the parent process so that results get stored into - # correct reporter - linter.set_reporter(original_reporter) linter.open() try: all_stats = [] @@ -141,7 +160,7 @@ def check_parallel(linter, jobs, files: Iterable[FileItem], arguments=None): msg = Message( msg[0], msg[1], MessageLocationTuple(*msg[2]), msg[3], msg[4] ) - linter.reporter.handle_message(msg) # type: ignore[attr-defined] # linter.set_reporter() call above makes linter have a reporter attr + linter.reporter.handle_message(msg) all_stats.append(stats) all_mapreduce_data[worker_idx].append(mapreduce_data) linter.msg_status |= msg_status |