Support Implicit Namespace Packages (PEP 420) (#8153)

Co-authored-by: Andreas Finkler <3929834+DudeNr33@users.noreply.github.com>

16 files changed, 147 insertions, 22 deletions

diff --git a/tests/lint/unittest_expand_modules.py b/tests/lint/unittest_expand_modules.py
index 1c3f23b00..a8c46498c 100644
--- a/tests/lint/unittest_expand_modules.py
+++ b/tests/lint/unittest_expand_modules.py
@@ -151,6 +151,7 @@ class TestExpandModules(CheckerTestCase):
         ignore_list_re: list[re.Pattern[str]] = []
         modules, errors = expand_modules(
             files_or_modules,
+            [],
             ignore_list,
             ignore_list_re,
             self.linter.config.ignore_paths,
@@ -180,6 +181,7 @@ class TestExpandModules(CheckerTestCase):
         ignore_list_re: list[re.Pattern[str]] = []
         modules, errors = expand_modules(
             files_or_modules,
+            [],
             ignore_list,
             ignore_list_re,
             self.linter.config.ignore_paths,
@@ -209,6 +211,7 @@ class TestExpandModules(CheckerTestCase):
         ignore_list_re: list[re.Pattern[str]] = []
         modules, errors = expand_modules(
             files_or_modules,
+            [],
             ignore_list,
             ignore_list_re,
             self.linter.config.ignore_paths,
diff --git a/tests/lint/unittest_lint.py b/tests/lint/unittest_lint.py
index 3c91eba7f..23cde4967 100644
--- a/tests/lint/unittest_lint.py
+++ b/tests/lint/unittest_lint.py
@@ -39,8 +39,7 @@ from pylint.constants import (
     _warn_about_old_home,
 )
 from pylint.exceptions import InvalidMessageError
-from pylint.lint import PyLinter
-from pylint.lint.utils import fix_import_path
+from pylint.lint import PyLinter, expand_modules
 from pylint.message import Message
 from pylint.reporters import text
 from pylint.testutils import create_files
@@ -117,8 +116,17 @@ def fake_path() -> Iterator[list[str]]:
     sys.path[:] = orig
 
 
+def test_deprecated() -> None:
+    """Test that fix_import_path() and get_python_path() are deprecated"""
+    with tempdir():
+        create_files(["__init__.py"])
+        with pytest.deprecated_call():
+            with lint.fix_import_path([""]):
+                expand_modules.get_python_path("__init__.py")
+
+
 def test_no_args(fake_path: list[str]) -> None:
-    with lint.fix_import_path([]):
+    with lint.augmented_sys_path([]):
         assert sys.path == fake_path
     assert sys.path == fake_path
 
@@ -131,8 +139,12 @@ def test_one_arg(fake_path: list[str], case: list[str]) -> None:
         create_files(["a/b/__init__.py"])
         expected = [join(chroot, "a")] + fake_path
 
+        extra_sys_paths = [
+            expand_modules.discover_package_path(arg, []) for arg in case
+        ]
+
         assert sys.path == fake_path
-        with lint.fix_import_path(case):
+        with lint.augmented_sys_path(extra_sys_paths):
             assert sys.path == expected
         assert sys.path == fake_path
 
@@ -151,8 +163,12 @@ def test_two_similar_args(fake_path: list[str], case: list[str]) -> None:
         create_files(["a/b/__init__.py", "a/c/__init__.py"])
         expected = [join(chroot, "a")] + fake_path
 
+        extra_sys_paths = [
+            expand_modules.discover_package_path(arg, []) for arg in case
+        ]
+
         assert sys.path == fake_path
-        with lint.fix_import_path(case):
+        with lint.augmented_sys_path(extra_sys_paths):
             assert sys.path == expected
         assert sys.path == fake_path
 
@@ -173,8 +189,12 @@ def test_more_args(fake_path: list[str], case: list[str]) -> None:
             for suffix in (sep.join(("a", "b")), "a", sep.join(("a", "e")))
         ] + fake_path
 
+        extra_sys_paths = [
+            expand_modules.discover_package_path(arg, []) for arg in case
+        ]
+
         assert sys.path == fake_path
-        with lint.fix_import_path(case):
+        with lint.augmented_sys_path(extra_sys_paths):
             assert sys.path == expected
         assert sys.path == fake_path
 
@@ -1188,6 +1208,39 @@ def test_recursive_ignore(ignore_parameter: str, ignore_parameter_value: str) ->
     assert module in linted_file_paths
 
 
+def test_recursive_implicit_namespace() -> None:
+    run = Run(
+        [
+            "--verbose",
+            "--recursive",
+            "y",
+            "--source-roots",
+            join(REGRTEST_DATA_DIR, "pep420", "basic", "project"),
+            join(REGRTEST_DATA_DIR, "pep420", "basic"),
+        ],
+        exit=False,
+    )
+    run.linter.set_reporter(testutils.GenericTestReporter())
+    run.linter.check([join(REGRTEST_DATA_DIR, "pep420", "basic")])
+    assert run.linter.file_state.base_name == "namespace.package"
+
+
+def test_recursive_implicit_namespace_wrapper() -> None:
+    run = Run(
+        [
+            "--recursive",
+            "y",
+            "--source-roots",
+            join(REGRTEST_DATA_DIR, "pep420", "wrapper", "project"),
+            join(REGRTEST_DATA_DIR, "pep420", "wrapper"),
+        ],
+        exit=False,
+    )
+    run.linter.set_reporter(testutils.GenericTestReporter())
+    run.linter.check([join(REGRTEST_DATA_DIR, "pep420", "wrapper")])
+    assert run.linter.reporter.messages == []
+
+
 def test_relative_imports(initialized_linter: PyLinter) -> None:
     """Regression test for https://github.com/PyCQA/pylint/issues/3651"""
     linter = initialized_linter
@@ -1235,8 +1288,10 @@ print(submodule1)
 """
             )
         os.chdir("namespace")
+        extra_sys_paths = [expand_modules.discover_package_path(tmpdir, [])]
+
         # Add the parent directory to sys.path
-        with fix_import_path([tmpdir]):
+        with lint.augmented_sys_path(extra_sys_paths):
             linter.check(["submodule2.py"])
     assert not linter.stats.by_msg
 
@@ -1257,6 +1312,7 @@ def test_lint_namespace_package_under_dir_on_path(initialized_linter: PyLinter)
     with tempdir() as tmpdir:
         create_files(["namespace_on_path/submodule1.py"])
         os.chdir(tmpdir)
-        with fix_import_path([tmpdir]):
+        extra_sys_paths = [expand_modules.discover_package_path(tmpdir, [])]
+        with lint.augmented_sys_path(extra_sys_paths):
             linter.check(["namespace_on_path"])
     assert linter.file_state.base_name == "namespace_on_path"
diff --git a/tests/primer/packages_to_prime.json b/tests/primer/packages_to_prime.json
index 547c3ea58..fa1457773 100644
--- a/tests/primer/packages_to_prime.json
+++ b/tests/primer/packages_to_prime.json
@@ -61,7 +61,8 @@
   },
   "poetry-core": {
     "branch": "main",
-    "directories": ["src/poetry/core"],
-    "url": "https://github.com/python-poetry/poetry-core"
+    "directories": ["src"],
+    "url": "https://github.com/python-poetry/poetry-core",
+    "pylint_additional_args": ["--recursive=y", "--source-roots=src"]
   }
 }
diff --git a/tests/pyreverse/conftest.py b/tests/pyreverse/conftest.py
index a37e4bde1..02a19e613 100644
--- a/tests/pyreverse/conftest.py
+++ b/tests/pyreverse/conftest.py
@@ -9,7 +9,7 @@ from collections.abc import Callable
 import pytest
 from astroid.nodes.scoped_nodes import Module
 
-from pylint.lint import fix_import_path
+from pylint.lint import augmented_sys_path, discover_package_path
 from pylint.pyreverse.inspector import Project, project_from_files
 from pylint.testutils.pyreverse import PyreverseConfig
 from pylint.typing import GetProjectCallable
@@ -74,7 +74,7 @@ def get_project() -> GetProjectCallable:
         def _astroid_wrapper(func: Callable[[str], Module], modname: str) -> Module:
             return func(modname)
 
-        with fix_import_path([module]):
+        with augmented_sys_path([discover_package_path(module, [])]):
             project = project_from_files([module], _astroid_wrapper, project_name=name)
         return project
 
diff --git a/tests/pyreverse/functional/class_diagrams/namespaces/pep420/pep420.dot b/tests/pyreverse/functional/class_diagrams/namespaces/pep420/pep420.dot
new file mode 100644
index 000000000..b01a46f38
--- /dev/null
+++ b/tests/pyreverse/functional/class_diagrams/namespaces/pep420/pep420.dot
@@ -0,0 +1,5 @@
+digraph "classes" {
+rankdir=BT
+charset="utf-8"
+"namespaces.pep420.pep420.PEP420" [color="black", fontcolor="black", label=<{PEP420|<br ALIGN="LEFT"/>|}>, shape="record", style="solid"];
+}
diff --git a/tests/pyreverse/functional/class_diagrams/namespaces/pep420/pep420.mmd b/tests/pyreverse/functional/class_diagrams/namespaces/pep420/pep420.mmd
new file mode 100644
index 000000000..4deda423b
--- /dev/null
+++ b/tests/pyreverse/functional/class_diagrams/namespaces/pep420/pep420.mmd
@@ -0,0 +1,3 @@
+classDiagram
+  class PEP420 {
+  }
diff --git a/tests/pyreverse/functional/class_diagrams/namespaces/pep420/pep420.puml b/tests/pyreverse/functional/class_diagrams/namespaces/pep420/pep420.puml
new file mode 100644
index 000000000..5819158ae
--- /dev/null
+++ b/tests/pyreverse/functional/class_diagrams/namespaces/pep420/pep420.puml
@@ -0,0 +1,5 @@
+@startuml classes
+set namespaceSeparator none
+class "PEP420" as namespaces.pep420.pep420.PEP420 {
+}
+@enduml
diff --git a/tests/pyreverse/functional/class_diagrams/namespaces/pep420/pep420.py b/tests/pyreverse/functional/class_diagrams/namespaces/pep420/pep420.py
new file mode 100644
index 000000000..f2d568c47
--- /dev/null
+++ b/tests/pyreverse/functional/class_diagrams/namespaces/pep420/pep420.py
@@ -0,0 +1,2 @@
+class PEP420:
+    pass
diff --git a/tests/pyreverse/functional/class_diagrams/namespaces/pep420/pep420.rc b/tests/pyreverse/functional/class_diagrams/namespaces/pep420/pep420.rc
new file mode 100644
index 000000000..0ae3403a9
--- /dev/null
+++ b/tests/pyreverse/functional/class_diagrams/namespaces/pep420/pep420.rc
@@ -0,0 +1,3 @@
+[testoptions]
+source_roots=../..
+output_formats=mmd,dot,puml
diff --git a/tests/pyreverse/test_main.py b/tests/pyreverse/test_main.py
index 4cc0573d6..1e6196270 100644
--- a/tests/pyreverse/test_main.py
+++ b/tests/pyreverse/test_main.py
@@ -16,7 +16,7 @@ import pytest
 from _pytest.capture import CaptureFixture
 from _pytest.fixtures import SubRequest
 
-from pylint.lint import fix_import_path
+from pylint.lint import augmented_sys_path, discover_package_path
 from pylint.pyreverse import main
 
 TEST_DATA_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "data"))
@@ -61,7 +61,7 @@ def test_project_root_in_sys_path() -> None:
     """Test the context manager adds the project root directory to sys.path.
     This should happen when pyreverse is run from any directory
     """
-    with fix_import_path([TEST_DATA_DIR]):
+    with augmented_sys_path([discover_package_path(TEST_DATA_DIR, [])]):
         assert sys.path == [PROJECT_ROOT_DIR]
 
 
diff --git a/tests/pyreverse/test_pyreverse_functional.py b/tests/pyreverse/test_pyreverse_functional.py
index 15fd1978b..aab500b81 100644
--- a/tests/pyreverse/test_pyreverse_functional.py
+++ b/tests/pyreverse/test_pyreverse_functional.py
@@ -1,7 +1,7 @@
 # Licensed under the GPL: https://www.gnu.org/licenses/old-licenses/gpl-2.0.html
 # For details: https://github.com/PyCQA/pylint/blob/main/LICENSE
 # Copyright (c) https://github.com/PyCQA/pylint/blob/main/CONTRIBUTORS.txt
-
+import os
 from pathlib import Path
 
 import pytest
@@ -13,7 +13,8 @@ from pylint.testutils.pyreverse import (
 )
 
 FUNCTIONAL_DIR = Path(__file__).parent / "functional"
-CLASS_DIAGRAM_TESTS = get_functional_test_files(FUNCTIONAL_DIR / "class_diagrams")
+CLASS_DIAGRAMS_DIR = FUNCTIONAL_DIR / "class_diagrams"
+CLASS_DIAGRAM_TESTS = get_functional_test_files(CLASS_DIAGRAMS_DIR)
 CLASS_DIAGRAM_TEST_IDS = [testfile.source.stem for testfile in CLASS_DIAGRAM_TESTS]
 
 
@@ -24,9 +25,23 @@ CLASS_DIAGRAM_TEST_IDS = [testfile.source.stem for testfile in CLASS_DIAGRAM_TES
 )
 def test_class_diagrams(testfile: FunctionalPyreverseTestfile, tmp_path: Path) -> None:
     input_file = testfile.source
+    input_path = os.path.dirname(input_file)
+    if testfile.options["source_roots"]:
+        source_roots = ",".join(
+            [
+                os.path.realpath(
+                    os.path.expanduser(os.path.join(input_path, source_root))
+                )
+                for source_root in testfile.options["source_roots"]
+            ]
+        )
+    else:
+        source_roots = ""
     for output_format in testfile.options["output_formats"]:
         with pytest.raises(SystemExit) as sys_exit:
             args = ["-o", f"{output_format}", "-d", str(tmp_path)]
+            if source_roots:
+                args += ["--source-roots", source_roots]
             args.extend(testfile.options["command_line_args"])
             args += [str(input_file)]
             Run(args)
diff --git a/tests/regrtest_data/pep420/basic/project/namespace/package/__init__.py b/tests/regrtest_data/pep420/basic/project/namespace/package/__init__.py
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/tests/regrtest_data/pep420/basic/project/namespace/package/__init__.py
diff --git a/tests/regrtest_data/pep420/wrapper/project/namespace/package/logging/__init__.py b/tests/regrtest_data/pep420/wrapper/project/namespace/package/logging/__init__.py
new file mode 100644
index 000000000..8bd4f6df2
--- /dev/null
+++ b/tests/regrtest_data/pep420/wrapper/project/namespace/package/logging/__init__.py
@@ -0,0 +1,5 @@
+"""
+A drop-in replacement for `logging`.
+"""
+
+from .wrapper import *  # noqa: F403
diff --git a/tests/regrtest_data/pep420/wrapper/project/namespace/package/logging/wrapper/__init__.py b/tests/regrtest_data/pep420/wrapper/project/namespace/package/logging/wrapper/__init__.py
new file mode 100644
index 000000000..50a2ac814
--- /dev/null
+++ b/tests/regrtest_data/pep420/wrapper/project/namespace/package/logging/wrapper/__init__.py
@@ -0,0 +1,5 @@
+"""
+A wrapper for original logging module
+"""
+
+from logging import *  # noqa: F403
diff --git a/tests/test_check_parallel.py b/tests/test_check_parallel.py
index 96e67517e..9134d485b 100644
--- a/tests/test_check_parallel.py
+++ b/tests/test_check_parallel.py
@@ -10,6 +10,7 @@ from __future__ import annotations
 
 import argparse
 import os
+import sys
 from concurrent.futures import ProcessPoolExecutor
 from concurrent.futures.process import BrokenProcessPool
 from pickle import PickleError
@@ -21,7 +22,7 @@ from astroid import nodes
 import pylint.interfaces
 import pylint.lint.parallel
 from pylint.checkers import BaseRawFileChecker
-from pylint.lint import PyLinter
+from pylint.lint import PyLinter, augmented_sys_path
 from pylint.lint.parallel import _worker_check_single_file as worker_check_single_file
 from pylint.lint.parallel import _worker_initialize as worker_initialize
 from pylint.lint.parallel import check_parallel
@@ -173,6 +174,14 @@ class TestCheckParallelFramework:
         worker_initialize(linter=dill.dumps(linter))
         assert isinstance(pylint.lint.parallel._worker_linter, type(linter))
 
+    def test_worker_initialize_with_package_paths(self) -> None:
+        linter = PyLinter(reporter=Reporter())
+        with augmented_sys_path([]):
+            worker_initialize(
+                linter=dill.dumps(linter), extra_packages_paths=["fake-path"]
+            )
+            assert "fake-path" in sys.path
+
     @pytest.mark.needs_two_cores
     def test_worker_initialize_pickling(self) -> None:
         """Test that we can pickle objects that standard pickling in multiprocessing can't.
@@ -324,7 +333,6 @@ class TestCheckParallel:
             linter,
             jobs=1,
             files=iter(single_file_container),
-            arguments=["--enable", "R9999"],
         )
         assert len(linter.get_checkers()) == 2, (
             "We should only have the 'main' and 'sequential-checker' "
@@ -390,7 +398,9 @@ class TestCheckParallel:
         # Invoke the lint process in a multi-process way, although we only specify one
         # job.
         check_parallel(
-            linter, jobs=1, files=iter(single_file_container), arguments=None
+            linter,
+            jobs=1,
+            files=iter(single_file_container),
         )
 
         assert {
@@ -504,7 +514,6 @@ class TestCheckParallel:
                     linter,
                     jobs=num_jobs,
                     files=file_infos,
-                    arguments=None,
                 )
                 stats_check_parallel = linter.stats
                 assert linter.msg_status == 0, "We should not fail the lint"
@@ -572,7 +581,6 @@ class TestCheckParallel:
                     linter,
                     jobs=num_jobs,
                     files=file_infos,
-                    arguments=None,
                 )
                 stats_check_parallel = linter.stats
         assert str(stats_single_proc.by_msg) == str(
@@ -600,5 +608,5 @@ class TestCheckParallel:
                 files=iter(single_file_container),
                 # This will trigger an exception in the initializer for the parallel jobs
                 # because arguments has to be an Iterable.
-                arguments=1,  # type: ignore[arg-type]
+                extra_packages_paths=1,  # type: ignore[arg-type]
             )
diff --git a/tests/test_deprecation.py b/tests/test_deprecation.py
index d30e69b85..ed57c7306 100644
--- a/tests/test_deprecation.py
+++ b/tests/test_deprecation.py
@@ -11,6 +11,7 @@ from typing import Any
 import pytest
 from astroid import nodes
 
+from pylint import lint
 from pylint.checkers import BaseChecker
 from pylint.checkers.mapreduce_checker import MapReduceMixin
 from pylint.config import load_results, save_results
@@ -106,3 +107,16 @@ def test_collectblocklines() -> None:
     state = FileState("foo", MessageDefinitionStore())
     with pytest.warns(DeprecationWarning):
         state.collect_block_lines(MessageDefinitionStore(), nodes.Module("foo"))
+
+
+def test_patch_sys_path() -> None:
+    """Test that _patch_sys_path() is deprecated"""
+    with pytest.deprecated_call():
+        lint._patch_sys_path([])
+
+
+def test_fix_import_path() -> None:
+    """Test that fix_import_path() is deprecated"""
+    with pytest.deprecated_call():
+        with lint.fix_import_path([]):
+            pass