[refactor] Permit to get the expected exit code in configuration test framework

6 files changed, 49 insertions, 15 deletions

diff --git a/pylint/testutils/configuration_test.py b/pylint/testutils/configuration_test.py
index 2a33d9388..ea61d771e 100644
--- a/pylint/testutils/configuration_test.py
+++ b/pylint/testutils/configuration_test.py
@@ -7,7 +7,7 @@ import json
 import logging
 import unittest
 from pathlib import Path
-from typing import Any, Dict, Tuple, Union
+from typing import Any, Dict, List, Tuple, Union
 from unittest.mock import Mock
 
 from pylint.lint import Run
@@ -22,13 +22,9 @@ def get_expected_or_default(
     tested_configuration_file: str, suffix: str, default: ConfigurationValue
 ) -> str:
     """Return the expected value from the file if it exists, or the given default."""
-
-    def get_path_according_to_suffix() -> Path:
-        path = Path(tested_configuration_file)
-        return path.parent / f"{path.stem}.{suffix}"
-
     expected = default
-    expected_result_path = get_path_according_to_suffix()
+    path = Path(tested_configuration_file)
+    expected_result_path = path.parent / f"{path.stem}.{suffix}"
     if expected_result_path.exists():
         with open(expected_result_path, encoding="utf8") as f:
             expected = f.read()
@@ -74,18 +70,56 @@ def get_expected_output(
     configuration_path: str, user_specific_path: Path
 ) -> Tuple[int, str]:
     """Get the expected output of a functional test."""
-    output = get_expected_or_default(configuration_path, suffix="out", default="")
-    if output:
+
+    def get_related_files(
+        tested_configuration_file: str, suffix_filter: str
+    ) -> List[Path]:
+        path = Path(tested_configuration_file)
+        return [
+            p
+            for p in path.parent.iterdir()
+            if path.stem in str(p) and str(p).endswith(suffix_filter)
+        ]
+
+    exit_code = 0
+    msg = (
+        "we expect a single file of the form "
+        "'filename_dot_expected_error_code_dot_out.32.out'"
+    )
+    possible_out_files = get_related_files(configuration_path, suffix_filter="out")
+    if len(possible_out_files) > 1:
+        logging.error(
+            "Too much .out files for %s %s.",
+            configuration_path,
+            msg,
+        )
+        return -1, "out file is broken"
+    if not possible_out_files:
         # logging is helpful to see what the expected exit code is and why.
         # The output of the program is checked during the test so printing
         # messes with the result.
-        logging.info(
-            "Output exists for %s so the expected exit code is 2", configuration_path
-        )
-        exit_code = 2
-    else:
         logging.info(".out file does not exists, so the expected exit code is 0")
-        exit_code = 0
+        return 0, ""
+    path = possible_out_files[0]
+    try:
+        exit_code = int(str(path.stem).rsplit(".", maxsplit=1)[-1])
+    except Exception as e:  # pylint: disable=broad-except
+        logging.error(
+            "Wrong format for .out file name for %s %s: %s",
+            configuration_path,
+            msg,
+            e,
+        )
+        return -1, "out file is broken"
+
+    output = get_expected_or_default(
+        configuration_path, suffix=f"{exit_code}.out", default=""
+    )
+    logging.info(
+        "Output exists for %s so the expected exit code is %s",
+        configuration_path,
+        exit_code,
+    )
     return exit_code, output.format(
         abspath=configuration_path,
         relpath=Path(configuration_path).relative_to(user_specific_path),
diff --git a/tests/config/functional/toml/issue_3181/top_level_list_of_disable.out b/tests/config/functional/toml/issue_3181/top_level_list_of_disable.2.out
index b12d21046..b12d21046 100644
--- a/tests/config/functional/toml/issue_3181/top_level_list_of_disable.out
+++ b/tests/config/functional/toml/issue_3181/top_level_list_of_disable.2.out
diff --git a/tests/config/functional/toml/issue_4580/empty_list.out b/tests/config/functional/toml/issue_4580/empty_list.2.out
index c25f99b30..c25f99b30 100644
--- a/tests/config/functional/toml/issue_4580/empty_list.out
+++ b/tests/config/functional/toml/issue_4580/empty_list.2.out
diff --git a/tests/config/functional/toml/issue_4580/invalid_data_for_basic.out b/tests/config/functional/toml/issue_4580/invalid_data_for_basic.2.out
index 2654d87f0..2654d87f0 100644
--- a/tests/config/functional/toml/issue_4580/invalid_data_for_basic.out
+++ b/tests/config/functional/toml/issue_4580/invalid_data_for_basic.2.out
diff --git a/tests/config/functional/toml/issue_4580/top_level_disable.out b/tests/config/functional/toml/issue_4580/top_level_disable.2.out
index a63759758..a63759758 100644
--- a/tests/config/functional/toml/issue_4580/top_level_disable.out
+++ b/tests/config/functional/toml/issue_4580/top_level_disable.2.out
diff --git a/tests/config/functional/toml/issue_4746/loaded_plugin_does_not_exists.out b/tests/config/functional/toml/issue_4746/loaded_plugin_does_not_exists.2.out
index a6837722a..a6837722a 100644
--- a/tests/config/functional/toml/issue_4746/loaded_plugin_does_not_exists.out
+++ b/tests/config/functional/toml/issue_4746/loaded_plugin_does_not_exists.2.out