Deduplicate module file paths to prevent redundant scans. (#7747)

* Use dict for 'expand_modules' result rather than list.
With 'path' as the key, we get deduplication for free
and do not need to reprocess the list for deduplication later.
* Fix deduplication to account for CLI args marker.
* Fix corner case with CLI arg flag handling during deduplication. 
* Add 'deduplication' to custom Pyenchant dict.

Closes #6242
Closes #4053

Co-authored-by: Eric McDonald <emcd@users.noreply.github.com>
Co-authored-by: Pierre Sassoulas <pierre.sassoulas@gmail.com>

1 files changed, 63 insertions, 18 deletions

diff --git a/tests/lint/unittest_expand_modules.py b/tests/lint/unittest_expand_modules.py
index 1883db543..88f058b1e 100644
--- a/tests/lint/unittest_expand_modules.py
+++ b/tests/lint/unittest_expand_modules.py
@@ -45,6 +45,14 @@ this_file_from_init = {
     "path": EXPAND_MODULES,
 }
 
+this_file_from_init_deduplicated = {
+    "basename": "lint",
+    "basepath": INIT_PATH,
+    "isarg": True,
+    "name": "lint.unittest_expand_modules",
+    "path": EXPAND_MODULES,
+}
+
 unittest_lint = {
     "basename": "lint",
     "basepath": INIT_PATH,
@@ -77,7 +85,6 @@ test_caching = {
     "path": str(TEST_DIRECTORY / "lint/test_caching.py"),
 }
 
-
 init_of_package = {
     "basename": "lint",
     "basepath": INIT_PATH,
@@ -87,6 +94,20 @@ init_of_package = {
 }
 
 
+def _list_expected_package_modules(
+    deduplicating: bool = False,
+) -> tuple[dict[str, object], ...]:
+    """Generates reusable list of modules for our package."""
+    return (
+        init_of_package,
+        test_caching,
+        test_pylinter,
+        test_utils,
+        this_file_from_init_deduplicated if deduplicating else this_file_from_init,
+        unittest_lint,
+    )
+
+
 class TestExpandModules(CheckerTestCase):
     """Test the expand_modules function while allowing options to be set."""
 
@@ -102,23 +123,19 @@ class TestExpandModules(CheckerTestCase):
     @pytest.mark.parametrize(
         "files_or_modules,expected",
         [
-            ([__file__], [this_file]),
+            ([__file__], {this_file["path"]: this_file}),
             (
                 [str(Path(__file__).parent)],
-                [
-                    init_of_package,
-                    test_caching,
-                    test_pylinter,
-                    test_utils,
-                    this_file_from_init,
-                    unittest_lint,
-                ],
+                {
+                    module["path"]: module  # pylint: disable=unsubscriptable-object
+                    for module in _list_expected_package_modules()
+                },
             ),
         ],
     )
     @set_config(ignore_paths="")
     def test_expand_modules(
-        self, files_or_modules: list[str], expected: list[ModuleDescriptionDict]
+        self, files_or_modules: list[str], expected: dict[str, ModuleDescriptionDict]
     ) -> None:
         """Test expand_modules with the default value of ignore-paths."""
         ignore_list: list[str] = []
@@ -129,25 +146,54 @@ class TestExpandModules(CheckerTestCase):
             ignore_list_re,
             self.linter.config.ignore_paths,
         )
-        modules.sort(key=lambda d: d["name"])
         assert modules == expected
         assert not errors
 
     @pytest.mark.parametrize(
         "files_or_modules,expected",
         [
-            ([__file__], []),
+            ([__file__, __file__], {this_file["path"]: this_file}),
+            (
+                [EXPAND_MODULES, str(Path(__file__).parent), EXPAND_MODULES],
+                {
+                    module["path"]: module  # pylint: disable=unsubscriptable-object
+                    for module in _list_expected_package_modules(deduplicating=True)
+                },
+            ),
+        ],
+    )
+    @set_config(ignore_paths="")
+    def test_expand_modules_deduplication(
+        self, files_or_modules: list[str], expected: dict[str, ModuleDescriptionDict]
+    ) -> None:
+        """Test expand_modules deduplication."""
+        ignore_list: list[str] = []
+        ignore_list_re: list[re.Pattern[str]] = []
+        modules, errors = expand_modules(
+            files_or_modules,
+            ignore_list,
+            ignore_list_re,
+            self.linter.config.ignore_paths,
+        )
+        assert modules == expected
+        assert not errors
+
+    @pytest.mark.parametrize(
+        "files_or_modules,expected",
+        [
+            ([__file__], {}),
             (
                 [str(Path(__file__).parent)],
-                [
-                    init_of_package,
-                ],
+                {
+                    module["path"]: module  # pylint: disable=unsubscriptable-object
+                    for module in (init_of_package,)
+                },
             ),
         ],
     )
     @set_config(ignore_paths=".*/lint/.*")
     def test_expand_modules_with_ignore(
-        self, files_or_modules: list[str], expected: list[ModuleDescriptionDict]
+        self, files_or_modules: list[str], expected: dict[str, ModuleDescriptionDict]
     ) -> None:
         """Test expand_modules with a non-default value of ignore-paths."""
         ignore_list: list[str] = []
@@ -158,6 +204,5 @@ class TestExpandModules(CheckerTestCase):
             ignore_list_re,
             self.linter.config.ignore_paths,
         )
-        modules.sort(key=lambda d: d["name"])
         assert modules == expected
         assert not errors