Support Implicit Namespace Packages (PEP 420) (#8153)

Co-authored-by: Andreas Finkler <3929834+DudeNr33@users.noreply.github.com>

10 files changed, 152 insertions, 32 deletions

diff --git a/pylint/config/argument.py b/pylint/config/argument.py
index b0ff4d5de..fd01a9b5f 100644
--- a/pylint/config/argument.py
+++ b/pylint/config/argument.py
@@ -88,6 +88,16 @@ def _path_transformer(value: str) -> str:
     return os.path.expandvars(os.path.expanduser(value))
 
 
+def _paths_csv_transformer(value: str) -> Sequence[str]:
+    """Transforms a comma separated list of paths while expanding user and
+    variables.
+    """
+    paths: list[str] = []
+    for path in _csv_transformer(value):
+        paths.append(os.path.expandvars(os.path.expanduser(path)))
+    return paths
+
+
 def _py_version_transformer(value: str) -> tuple[int, ...]:
     """Transforms a version string into a version tuple."""
     try:
@@ -138,6 +148,7 @@ _TYPE_TRANSFORMERS: dict[str, Callable[[str], _ArgumentTypes]] = {
     "confidence": _confidence_transformer,
     "non_empty_string": _non_empty_string_transformer,
     "path": _path_transformer,
+    "paths_csv": _paths_csv_transformer,
     "py_version": _py_version_transformer,
     "regexp": _regex_transformer,
     "regexp_csv": _regexp_csv_transfomer,
diff --git a/pylint/config/option.py b/pylint/config/option.py
index 95248d6b1..5a425f34d 100644
--- a/pylint/config/option.py
+++ b/pylint/config/option.py
@@ -117,6 +117,7 @@ VALIDATORS: dict[str, Callable[[Any, str, Any], Any] | Callable[[Any], Any]] = {
     "string": utils._unquote,
     "int": int,
     "float": float,
+    "paths_csv": _csv_validator,
     "regexp": lambda pattern: re.compile(pattern or ""),
     "regexp_csv": _regexp_csv_validator,
     "regexp_paths_csv": _regexp_paths_csv_validator,
@@ -163,6 +164,7 @@ def _validate(value: Any, optdict: Any, name: str = "") -> Any:
 # pylint: disable=no-member
 class Option(optparse.Option):
     TYPES = optparse.Option.TYPES + (
+        "paths_csv",
         "regexp",
         "regexp_csv",
         "regexp_paths_csv",
@@ -175,6 +177,7 @@ class Option(optparse.Option):
     )
     ATTRS = optparse.Option.ATTRS + ["hide", "level"]
     TYPE_CHECKER = copy.copy(optparse.Option.TYPE_CHECKER)
+    TYPE_CHECKER["paths_csv"] = _csv_validator
     TYPE_CHECKER["regexp"] = _regexp_validator
     TYPE_CHECKER["regexp_csv"] = _regexp_csv_validator
     TYPE_CHECKER["regexp_paths_csv"] = _regexp_paths_csv_validator
diff --git a/pylint/lint/__init__.py b/pylint/lint/__init__.py
index 86186ebd4..573d9c262 100644
--- a/pylint/lint/__init__.py
+++ b/pylint/lint/__init__.py
@@ -18,6 +18,7 @@ import sys
 
 from pylint.config.exceptions import ArgumentPreprocessingError
 from pylint.lint.caching import load_results, save_results
+from pylint.lint.expand_modules import discover_package_path
 from pylint.lint.parallel import check_parallel
 from pylint.lint.pylinter import PyLinter
 from pylint.lint.report_functions import (
@@ -26,7 +27,12 @@ from pylint.lint.report_functions import (
     report_total_messages_stats,
 )
 from pylint.lint.run import Run
-from pylint.lint.utils import _patch_sys_path, fix_import_path
+from pylint.lint.utils import (
+    _augment_sys_path,
+    _patch_sys_path,
+    augmented_sys_path,
+    fix_import_path,
+)
 
 __all__ = [
     "check_parallel",
@@ -38,6 +44,9 @@ __all__ = [
     "ArgumentPreprocessingError",
     "_patch_sys_path",
     "fix_import_path",
+    "_augment_sys_path",
+    "augmented_sys_path",
+    "discover_package_path",
     "save_results",
     "load_results",
 ]
diff --git a/pylint/lint/base_options.py b/pylint/lint/base_options.py
index 1c37eac2f..8540340cd 100644
--- a/pylint/lint/base_options.py
+++ b/pylint/lint/base_options.py
@@ -344,6 +344,17 @@ def _make_linter_options(linter: PyLinter) -> Options:
             },
         ),
         (
+            "source-roots",
+            {
+                "type": "paths_csv",
+                "metavar": "<path>[,<path>...]",
+                "default": (),
+                "help": "Add paths to the list of the source roots. The source root is an absolute "
+                "path or a path relative to the current working directory used to "
+                "determine a package namespace for modules located under the source root.",
+            },
+        ),
+        (
             "recursive",
             {
                 "type": "yn",
diff --git a/pylint/lint/expand_modules.py b/pylint/lint/expand_modules.py
index 8259e25ad..bb25986e4 100644
--- a/pylint/lint/expand_modules.py
+++ b/pylint/lint/expand_modules.py
@@ -6,6 +6,7 @@ from __future__ import annotations
 
 import os
 import sys
+import warnings
 from collections.abc import Sequence
 from re import Pattern
 
@@ -24,14 +25,31 @@ def _modpath_from_file(filename: str, is_namespace: bool, path: list[str]) -> li
 
 
 def get_python_path(filepath: str) -> str:
-    """TODO This get the python path with the (bad) assumption that there is always
-    an __init__.py.
+    # TODO: Remove deprecated function
+    warnings.warn(
+        "get_python_path has been deprecated because assumption that there's always an __init__.py "
+        "is not true since python 3.3 and is causing problems, particularly with PEP 420."
+        "Use discover_package_path and pass source root(s).",
+        DeprecationWarning,
+        stacklevel=2,
+    )
+    return discover_package_path(filepath, [])
 
-    This is not true since python 3.3 and is causing problem.
-    """
-    dirname = os.path.realpath(os.path.expanduser(filepath))
+
+def discover_package_path(modulepath: str, source_roots: Sequence[str]) -> str:
+    """Discover package path from one its modules and source roots."""
+    dirname = os.path.realpath(os.path.expanduser(modulepath))
     if not os.path.isdir(dirname):
         dirname = os.path.dirname(dirname)
+
+    # Look for a source root that contains the module directory
+    for source_root in source_roots:
+        source_root = os.path.realpath(os.path.expanduser(source_root))
+        if os.path.commonpath([source_root, dirname]) == source_root:
+            return source_root
+
+    # Fall back to legacy discovery by looking for __init__.py upwards as
+    # it's the only way given that source root was not found or was not provided
     while True:
         if not os.path.exists(os.path.join(dirname, "__init__.py")):
             return dirname
@@ -64,6 +82,7 @@ def _is_ignored_file(
 # pylint: disable = too-many-locals, too-many-statements
 def expand_modules(
     files_or_modules: Sequence[str],
+    source_roots: Sequence[str],
     ignore_list: list[str],
     ignore_list_re: list[Pattern[str]],
     ignore_list_paths_re: list[Pattern[str]],
@@ -81,8 +100,8 @@ def expand_modules(
             something, ignore_list, ignore_list_re, ignore_list_paths_re
         ):
             continue
-        module_path = get_python_path(something)
-        additional_search_path = [".", module_path] + path
+        module_package_path = discover_package_path(something, source_roots)
+        additional_search_path = [".", module_package_path] + path
         if os.path.exists(something):
             # this is a file or a directory
             try:
diff --git a/pylint/lint/parallel.py b/pylint/lint/parallel.py
index 9730914b7..544a256d3 100644
--- a/pylint/lint/parallel.py
+++ b/pylint/lint/parallel.py
@@ -13,7 +13,7 @@ from typing import TYPE_CHECKING, Any
 import dill
 
 from pylint import reporters
-from pylint.lint.utils import _patch_sys_path
+from pylint.lint.utils import _augment_sys_path
 from pylint.message import Message
 from pylint.typing import FileItem
 from pylint.utils import LinterStats, merge_stats
@@ -37,12 +37,12 @@ _worker_linter: PyLinter | None = None
 
 
 def _worker_initialize(
-    linter: bytes, arguments: None | str | Sequence[str] = None
+    linter: bytes, extra_packages_paths: Sequence[str] | None = None
 ) -> None:
     """Function called to initialize a worker for a Process within a concurrent Pool.
 
     :param linter: A linter-class (PyLinter) instance pickled with dill
-    :param arguments: File or module name(s) to lint and to be added to sys.path
+    :param extra_packages_paths: Extra entries to be added to sys.path
     """
     global _worker_linter  # pylint: disable=global-statement
     _worker_linter = dill.loads(linter)
@@ -53,8 +53,8 @@ def _worker_initialize(
     _worker_linter.set_reporter(reporters.CollectingReporter())
     _worker_linter.open()
 
-    # Patch sys.path so that each argument is importable just like in single job mode
-    _patch_sys_path(arguments or ())
+    if extra_packages_paths:
+        _augment_sys_path(extra_packages_paths)
 
 
 def _worker_check_single_file(
@@ -130,7 +130,7 @@ def check_parallel(
     linter: PyLinter,
     jobs: int,
     files: Iterable[FileItem],
-    arguments: None | str | Sequence[str] = None,
+    extra_packages_paths: Sequence[str] | None = None,
 ) -> None:
     """Use the given linter to lint the files with given amount of workers (jobs).
 
@@ -140,7 +140,9 @@ def check_parallel(
     # The linter is inherited by all the pool's workers, i.e. the linter
     # is identical to the linter object here. This is required so that
     # a custom PyLinter object can be used.
-    initializer = functools.partial(_worker_initialize, arguments=arguments)
+    initializer = functools.partial(
+        _worker_initialize, extra_packages_paths=extra_packages_paths
+    )
     with ProcessPoolExecutor(
         max_workers=jobs, initializer=initializer, initargs=(dill.dumps(linter),)
     ) as executor:
diff --git a/pylint/lint/pylinter.py b/pylint/lint/pylinter.py
index 4d1bede9a..5b749d5b2 100644
--- a/pylint/lint/pylinter.py
+++ b/pylint/lint/pylinter.py
@@ -36,7 +36,11 @@ from pylint.constants import (
 from pylint.interfaces import HIGH
 from pylint.lint.base_options import _make_linter_options
 from pylint.lint.caching import load_results, save_results
-from pylint.lint.expand_modules import _is_ignored_file, expand_modules
+from pylint.lint.expand_modules import (
+    _is_ignored_file,
+    discover_package_path,
+    expand_modules,
+)
 from pylint.lint.message_state_handler import _MessageStateHandler
 from pylint.lint.parallel import check_parallel
 from pylint.lint.report_functions import (
@@ -46,7 +50,7 @@ from pylint.lint.report_functions import (
 )
 from pylint.lint.utils import (
     _is_relative_to,
-    fix_import_path,
+    augmented_sys_path,
     get_fatal_error_message,
     prepare_crash_report,
 )
@@ -675,6 +679,13 @@ class PyLinter(
                     "Missing filename required for --from-stdin"
                 )
 
+        extra_packages_paths = list(
+            {
+                discover_package_path(file_or_module, self.config.source_roots)
+                for file_or_module in files_or_modules
+            }
+        )
+
         # TODO: Move the parallel invocation into step 5 of the checking process
         if not self.config.from_stdin and self.config.jobs > 1:
             original_sys_path = sys.path[:]
@@ -682,13 +693,13 @@ class PyLinter(
                 self,
                 self.config.jobs,
                 self._iterate_file_descrs(files_or_modules),
-                files_or_modules,  # this argument patches sys.path
+                extra_packages_paths,
             )
             sys.path = original_sys_path
             return
 
         # 3) Get all FileItems
-        with fix_import_path(files_or_modules):
+        with augmented_sys_path(extra_packages_paths):
             if self.config.from_stdin:
                 fileitems = self._get_file_descr_from_stdin(files_or_modules[0])
                 data: str | None = _read_stdin()
@@ -697,7 +708,7 @@ class PyLinter(
                 data = None
 
         # The contextmanager also opens all checkers and sets up the PyLinter class
-        with fix_import_path(files_or_modules):
+        with augmented_sys_path(extra_packages_paths):
             with self._astroid_module_checker() as check_astroid_module:
                 # 4) Get the AST for each FileItem
                 ast_per_fileitem = self._get_asts(fileitems, data)
@@ -884,10 +895,13 @@ class PyLinter(
             if self.should_analyze_file(name, filepath, is_argument=is_arg):
                 yield FileItem(name, filepath, descr["basename"])
 
-    def _expand_files(self, modules: Sequence[str]) -> dict[str, ModuleDescriptionDict]:
+    def _expand_files(
+        self, files_or_modules: Sequence[str]
+    ) -> dict[str, ModuleDescriptionDict]:
         """Get modules and errors from a list of modules and handle errors."""
         result, errors = expand_modules(
-            modules,
+            files_or_modules,
+            self.config.source_roots,
             self.config.ignore,
             self.config.ignore_patterns,
             self._ignore_paths,
diff --git a/pylint/lint/utils.py b/pylint/lint/utils.py
index d4ad131f3..98fb8087a 100644
--- a/pylint/lint/utils.py
+++ b/pylint/lint/utils.py
@@ -7,12 +7,13 @@ from __future__ import annotations
 import contextlib
 import sys
 import traceback
+import warnings
 from collections.abc import Iterator, Sequence
 from datetime import datetime
 from pathlib import Path
 
 from pylint.config import PYLINT_HOME
-from pylint.lint.expand_modules import get_python_path
+from pylint.lint.expand_modules import discover_package_path
 
 
 def prepare_crash_report(ex: Exception, filepath: str, crash_file_path: str) -> Path:
@@ -73,14 +74,26 @@ def get_fatal_error_message(filepath: str, issue_template_path: Path) -> str:
 
 
 def _patch_sys_path(args: Sequence[str]) -> list[str]:
+    # TODO: Remove deprecated function
+    warnings.warn(
+        "_patch_sys_path has been deprecated because it relies on auto-magic package path "
+        "discovery which is implemented by get_python_path that is deprecated. "
+        "Use _augment_sys_path and pass additional sys.path entries as an argument obtained from "
+        "discover_package_path.",
+        DeprecationWarning,
+        stacklevel=2,
+    )
+    return _augment_sys_path([discover_package_path(arg, []) for arg in args])
+
+
+def _augment_sys_path(additional_paths: Sequence[str]) -> list[str]:
     original = list(sys.path)
     changes = []
     seen = set()
-    for arg in args:
-        path = get_python_path(arg)
-        if path not in seen:
-            changes.append(path)
-            seen.add(path)
+    for additional_path in additional_paths:
+        if additional_path not in seen:
+            changes.append(additional_path)
+            seen.add(additional_path)
 
     sys.path[:] = changes + sys.path
     return original
@@ -95,7 +108,23 @@ def fix_import_path(args: Sequence[str]) -> Iterator[None]:
     We avoid adding duplicate directories to sys.path.
     `sys.path` is reset to its original value upon exiting this context.
     """
-    original = _patch_sys_path(args)
+    # TODO: Remove deprecated function
+    warnings.warn(
+        "fix_import_path has been deprecated because it relies on auto-magic package path "
+        "discovery which is implemented by get_python_path that is deprecated. "
+        "Use augmented_sys_path and pass additional sys.path entries as an argument obtained from "
+        "discover_package_path.",
+        DeprecationWarning,
+        stacklevel=2,
+    )
+    with augmented_sys_path([discover_package_path(arg, []) for arg in args]):
+        yield
+
+
+@contextlib.contextmanager
+def augmented_sys_path(additional_paths: Sequence[str]) -> Iterator[None]:
+    """Augment 'sys.path' by adding non-existent entries from additional_paths."""
+    original = _augment_sys_path(additional_paths)
     try:
         yield
     finally:
diff --git a/pylint/pyreverse/main.py b/pylint/pyreverse/main.py
index 8a2055c41..13669d5b4 100644
--- a/pylint/pyreverse/main.py
+++ b/pylint/pyreverse/main.py
@@ -13,7 +13,8 @@ from typing import NoReturn
 from pylint import constants
 from pylint.config.arguments_manager import _ArgumentsManager
 from pylint.config.arguments_provider import _ArgumentsProvider
-from pylint.lint.utils import fix_import_path
+from pylint.lint import discover_package_path
+from pylint.lint.utils import augmented_sys_path
 from pylint.pyreverse import writer
 from pylint.pyreverse.diadefslib import DiadefsHandler
 from pylint.pyreverse.inspector import Linker, project_from_files
@@ -203,6 +204,17 @@ OPTIONS: Options = (
             "help": "set the output directory path.",
         },
     ),
+    (
+        "source-roots",
+        {
+            "type": "paths_csv",
+            "metavar": "<path>[,<path>...]",
+            "default": (),
+            "help": "Add paths to the list of the source roots. The source root is an absolute "
+            "path or a path relative to the current working directory used to "
+            "determine a package namespace for modules located under the source root.",
+        },
+    ),
 )
 
 
@@ -235,7 +247,10 @@ class Run(_ArgumentsManager, _ArgumentsProvider):
         if not args:
             print(self.help())
             return 1
-        with fix_import_path(args):
+        extra_packages_paths = list(
+            {discover_package_path(arg, self.config.source_roots) for arg in args}
+        )
+        with augmented_sys_path(extra_packages_paths):
             project = project_from_files(
                 args,
                 project_name=self.config.project,
diff --git a/pylint/testutils/pyreverse.py b/pylint/testutils/pyreverse.py
index fc20b5453..7a61ff5fe 100644
--- a/pylint/testutils/pyreverse.py
+++ b/pylint/testutils/pyreverse.py
@@ -67,6 +67,7 @@ class PyreverseConfig(
 
 
 class TestFileOptions(TypedDict):
+    source_roots: list[str]
     output_formats: list[str]
     command_line_args: list[str]
 
@@ -97,7 +98,11 @@ def get_functional_test_files(
             test_files.append(
                 FunctionalPyreverseTestfile(
                     source=path,
-                    options={"output_formats": ["mmd"], "command_line_args": []},
+                    options={
+                        "source_roots": [],
+                        "output_formats": ["mmd"],
+                        "command_line_args": [],
+                    },
                 )
             )
     return test_files
@@ -106,7 +111,9 @@ def get_functional_test_files(
 def _read_config(config_file: Path) -> TestFileOptions:
     config = configparser.ConfigParser()
     config.read(str(config_file))
+    source_roots = config.get("testoptions", "source_roots", fallback=None)
     return {
+        "source_roots": source_roots.split(",") if source_roots else [],
         "output_formats": config.get(
             "testoptions", "output_formats", fallback="mmd"
         ).split(","),