[refactor] Permit to get the expected exit code in configuration test framework

14 files changed, 78 insertions, 18 deletions

diff --git a/pylint/testutils/configuration_test.py b/pylint/testutils/configuration_test.py
index 2a33d9388..809fd4eb5 100644
--- a/pylint/testutils/configuration_test.py
+++ b/pylint/testutils/configuration_test.py
@@ -7,7 +7,7 @@ import json
 import logging
 import unittest
 from pathlib import Path
-from typing import Any, Dict, Tuple, Union
+from typing import Any, Dict, List, Tuple, Union
 from unittest.mock import Mock
 
 from pylint.lint import Run
@@ -19,16 +19,14 @@ PylintConfiguration = Dict[str, ConfigurationValue]
 
 
 def get_expected_or_default(
-    tested_configuration_file: str, suffix: str, default: ConfigurationValue
+    tested_configuration_file: Union[str, Path],
+    suffix: str,
+    default: ConfigurationValue,
 ) -> str:
     """Return the expected value from the file if it exists, or the given default."""
-
-    def get_path_according_to_suffix() -> Path:
-        path = Path(tested_configuration_file)
-        return path.parent / f"{path.stem}.{suffix}"
-
     expected = default
-    expected_result_path = get_path_according_to_suffix()
+    path = Path(tested_configuration_file)
+    expected_result_path = path.parent / f"{path.stem}.{suffix}"
     if expected_result_path.exists():
         with open(expected_result_path, encoding="utf8") as f:
             expected = f.read()
@@ -70,22 +68,61 @@ def get_expected_configuration(
     return result
 
 
+def get_related_files(
+    tested_configuration_file: Union[str, Path], suffix_filter: str
+) -> List[Path]:
+    """Return all the file related to a test conf file endind with a suffix."""
+    conf_path = Path(tested_configuration_file)
+    return [
+        p
+        for p in conf_path.parent.iterdir()
+        if str(p.stem).startswith(conf_path.stem) and str(p).endswith(suffix_filter)
+    ]
+
+
 def get_expected_output(
-    configuration_path: str, user_specific_path: Path
+    configuration_path: Union[str, Path], user_specific_path: Path
 ) -> Tuple[int, str]:
     """Get the expected output of a functional test."""
-    output = get_expected_or_default(configuration_path, suffix="out", default="")
-    if output:
+    exit_code = 0
+    msg = (
+        "we expect a single file of the form 'filename.32.out' where 'filename' represents "
+        "the name of the configuration file, and '32' the expected error code."
+    )
+    possible_out_files = get_related_files(configuration_path, suffix_filter="out")
+    if len(possible_out_files) > 1:
+        logging.error(
+            "Too much .out files for %s %s.",
+            configuration_path,
+            msg,
+        )
+        return -1, "out file is broken"
+    if not possible_out_files:
         # logging is helpful to see what the expected exit code is and why.
         # The output of the program is checked during the test so printing
         # messes with the result.
-        logging.info(
-            "Output exists for %s so the expected exit code is 2", configuration_path
-        )
-        exit_code = 2
-    else:
         logging.info(".out file does not exists, so the expected exit code is 0")
-        exit_code = 0
+        return 0, ""
+    path = possible_out_files[0]
+    try:
+        exit_code = int(str(path.stem).rsplit(".", maxsplit=1)[-1])
+    except Exception as e:  # pylint: disable=broad-except
+        logging.error(
+            "Wrong format for .out file name for %s %s: %s",
+            configuration_path,
+            msg,
+            e,
+        )
+        return -1, "out file is broken"
+
+    output = get_expected_or_default(
+        configuration_path, suffix=f"{exit_code}.out", default=""
+    )
+    logging.info(
+        "Output exists for %s so the expected exit code is %s",
+        configuration_path,
+        exit_code,
+    )
     return exit_code, output.format(
         abspath=configuration_path,
         relpath=Path(configuration_path).relative_to(user_specific_path),
diff --git a/tests/config/functional/toml/issue_3181/toml_decode_error.1.out b/tests/config/functional/toml/issue_3181/toml_decode_error.1.out
index ce4be6dad..545acbd2d 100644
--- a/tests/config/functional/toml/issue_3181/toml_decode_error.1.out
+++ b/tests/config/functional/toml/issue_3181/toml_decode_error.1.out
@@ -1,2 +1,2 @@
 ************* Module {abspath}
-{relpath}:1:0: F0010: error while code parsing: Found invalid character in key name: '*'. Try quoting the key name. (line 3 column 2 char 48) (parse-error)
+{relpath}:1:0: F0011: error while parsing the configuration: Found invalid character in key name: '*'. Try quoting the key name. (line 3 column 2 char 48) (config-parse-error)
diff --git a/tests/config/functional/toml/issue_3181/top_level_list_of_disable.out b/tests/config/functional/toml/issue_3181/top_level_list_of_disable.2.out
index b12d21046..b12d21046 100644
--- a/tests/config/functional/toml/issue_3181/top_level_list_of_disable.out
+++ b/tests/config/functional/toml/issue_3181/top_level_list_of_disable.2.out
diff --git a/tests/config/functional/toml/issue_4580/empty_list.out b/tests/config/functional/toml/issue_4580/empty_list.2.out
index c25f99b30..c25f99b30 100644
--- a/tests/config/functional/toml/issue_4580/empty_list.out
+++ b/tests/config/functional/toml/issue_4580/empty_list.2.out
diff --git a/tests/config/functional/toml/issue_4580/invalid_data_for_basic.out b/tests/config/functional/toml/issue_4580/invalid_data_for_basic.2.out
index 2654d87f0..2654d87f0 100644
--- a/tests/config/functional/toml/issue_4580/invalid_data_for_basic.out
+++ b/tests/config/functional/toml/issue_4580/invalid_data_for_basic.2.out
diff --git a/tests/config/functional/toml/issue_4580/top_level_disable.out b/tests/config/functional/toml/issue_4580/top_level_disable.2.out
index a63759758..a63759758 100644
--- a/tests/config/functional/toml/issue_4580/top_level_disable.out
+++ b/tests/config/functional/toml/issue_4580/top_level_disable.2.out
diff --git a/tests/config/functional/toml/issue_4746/loaded_plugin_does_not_exists.out b/tests/config/functional/toml/issue_4746/loaded_plugin_does_not_exists.2.out
index a6837722a..a6837722a 100644
--- a/tests/config/functional/toml/issue_4746/loaded_plugin_does_not_exists.out
+++ b/tests/config/functional/toml/issue_4746/loaded_plugin_does_not_exists.2.out
diff --git a/tests/testutils/data/t.3.out b/tests/testutils/data/t.3.out
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/tests/testutils/data/t.3.out
diff --git a/tests/testutils/data/t.out b/tests/testutils/data/t.out
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/tests/testutils/data/t.out
diff --git a/tests/testutils/data/t.toml b/tests/testutils/data/t.toml
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/tests/testutils/data/t.toml
diff --git a/tests/testutils/data/u.out b/tests/testutils/data/u.out
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/tests/testutils/data/u.out
diff --git a/tests/testutils/data/u.toml b/tests/testutils/data/u.toml
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/tests/testutils/data/u.toml
diff --git a/tests/testutils/data/v.toml b/tests/testutils/data/v.toml
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/tests/testutils/data/v.toml
diff --git a/tests/testutils/test_configuration_test.py b/tests/testutils/test_configuration_test.py
new file mode 100644
index 000000000..195c4415c
--- /dev/null
+++ b/tests/testutils/test_configuration_test.py
@@ -0,0 +1,23 @@
+import logging
+from pathlib import Path
+
+from pytest import LogCaptureFixture
+
+from pylint.testutils.configuration_test import get_expected_output
+
+HERE = Path(__file__).parent
+USER_SPECIFIC_PATH = HERE.parent.parent
+DATA_DIRECTORY = HERE / "data"
+
+
+def test_get_expected_output(caplog: LogCaptureFixture) -> None:
+    caplog.set_level(logging.INFO)
+    exit_code, _ = get_expected_output(DATA_DIRECTORY / "t.toml", USER_SPECIFIC_PATH)
+    assert "Too much .out files" in str(caplog.text)
+    assert exit_code == -1
+    exit_code, _ = get_expected_output(DATA_DIRECTORY / "u.toml", USER_SPECIFIC_PATH)
+    assert exit_code == -1
+    assert "Wrong format for .out file name" in str(caplog.text)
+    exit_code, _ = get_expected_output(DATA_DIRECTORY / "v.toml", USER_SPECIFIC_PATH)
+    assert exit_code == 0
+    assert ".out file does not exists" in str(caplog.text)