[functional tests] Display all violations for disorganized directories

2 files changed, 21 insertions, 7 deletions

diff --git a/pylint/testutils/functional/find_functional_tests.py b/pylint/testutils/functional/find_functional_tests.py
index 200cee7ec..f71e3090a 100644
--- a/pylint/testutils/functional/find_functional_tests.py
+++ b/pylint/testutils/functional/find_functional_tests.py
@@ -71,17 +71,27 @@ def _check_functional_tests_structure(directory: Path) -> None:
         f"{directory} contains too many functional tests files "
         + f"({len(files)} > {REASONABLY_DISPLAYABLE_VERTICALLY})."
     )
-
+    directory_does_not_exists: list[tuple[Path, Path]] = []
+    misplaced_file: list[Path] = []
     for file in files:
         possible_dir = file.parent / file.stem.split("_")[0]
-        assert not possible_dir.exists(), f"{file} should go in {possible_dir}."
-
+        if possible_dir.exists():
+            directory_does_not_exists.append((file, possible_dir))
         # Exclude some directories as they follow a different structure
         if (
             not len(file.parent.stem) == 1  # First letter sub-directories
             and file.parent.stem not in IGNORED_PARENT_DIRS
             and file.parent.parent.stem not in IGNORED_PARENT_PARENT_DIRS
         ):
-            assert file.stem.startswith(
-                file.parent.stem
-            ), f"{file} should not go in {file.parent}"
+            if not file.stem.startswith(file.parent.stem):
+                misplaced_file.append(file)
+    if directory_does_not_exists or misplaced_file:
+        msg = "The following functional tests are disorganized:\n"
+        for file, possible_dir in directory_does_not_exists:
+            msg += f"- {file} should go in {possible_dir}\n"
+        for file in misplaced_file:
+            msg += (
+                f"- {file} should go in a directory that starts with the first letters"
+                f" of '{file.stem}' (not '{file.parent.stem}')\n"
+            )
+        raise AssertionError(msg)
diff --git a/tests/testutils/test_functional_testutils.py b/tests/testutils/test_functional_testutils.py
index e0055f1b0..e12fd7a84 100644
--- a/tests/testutils/test_functional_testutils.py
+++ b/tests/testutils/test_functional_testutils.py
@@ -41,7 +41,11 @@ def test_parsing_of_pylintrc_init_hook() -> None:
 
 def test_get_functional_test_files_from_directory() -> None:
     """Test that we correctly check the functional test directory structures."""
-    with pytest.raises(AssertionError, match="using_dir.py should not go in"):
+    match = (
+        "using_dir.py should go in a directory that starts with the "
+        "first letters of 'using_dir'"
+    )
+    with pytest.raises(AssertionError, match=match):
         get_functional_test_files_from_directory(DATA_DIRECTORY / "u")