[testutil] Add proper test for the functional tests helpers

19 files changed, 135 insertions, 39 deletions

diff --git a/pylint/testutils/lint_module_test.py b/pylint/testutils/lint_module_test.py
index e139af12b..e1036170a 100644
--- a/pylint/testutils/lint_module_test.py
+++ b/pylint/testutils/lint_module_test.py
@@ -266,7 +266,7 @@ class LintModuleTest:
         expected_messages: MessageCounter,
         actual_output: list[OutputLine],
     ) -> str:
-        msg = [f'Wrong results for file "{self._test_file.base}":']
+        msg = [f'Wrong message(s) raised for "{Path(self._test_file.source).name}":']
         missing, unexpected = self.multiset_difference(
             expected_messages, actual_messages
         )
@@ -289,7 +289,7 @@ class LintModuleTest:
     ) -> str:
         missing = set(expected_lines) - set(received_lines)
         unexpected = set(received_lines) - set(expected_lines)
-        error_msg = f"Wrong output for '{self._test_file.base}.txt':"
+        error_msg = f'Wrong output for "{Path(self._test_file.expected_output).name}":'
         sort_by_line_number = operator.attrgetter("lineno")
         if missing:
             error_msg += "\n- Missing lines:\n"
diff --git a/pylint/testutils/output_line.py b/pylint/testutils/output_line.py
index fb05c8775..8158d514a 100644
--- a/pylint/testutils/output_line.py
+++ b/pylint/testutils/output_line.py
@@ -13,7 +13,6 @@ from astroid import nodes
 from pylint.constants import PY38_PLUS
 from pylint.interfaces import UNDEFINED, Confidence
 from pylint.message.message import Message
-from pylint.testutils.constants import UPDATE_OPTION
 
 _T = TypeVar("_T")
 
@@ -139,13 +138,7 @@ class OutputLine(NamedTuple):
                 )
             raise IndexError
         except Exception:  # pylint: disable=broad-except
-            warnings.warn(
-                "Expected 'msg-symbolic-name:42:27:MyClass.my_function:The message:"
-                f"CONFIDENCE' but we got '{':'.join(row)}'. Try updating the expected"
-                f" output with:\npython tests/test_functional.py {UPDATE_OPTION}",
-                UserWarning,
-                stacklevel=2,
-            )
+            # We need this to not fail for the update script to work.
             return cls("", 0, 0, None, None, "", "", "")
 
     def to_csv(self) -> tuple[str, str, str, str, str, str, str, str]:
diff --git a/tests/testutils/data/functional/broken_output_ok_test/exec_used.py b/tests/testutils/data/functional/broken_output_ok_test/exec_used.py
new file mode 100644
index 000000000..73d629d95
--- /dev/null
+++ b/tests/testutils/data/functional/broken_output_ok_test/exec_used.py
@@ -0,0 +1,7 @@
+"""This is an example for a functional test with
+- an output that is broken
+- the functional test itself is right
+
+So it should be updated.
+"""
+exec('a = 42')  # [exec-used]
diff --git a/tests/testutils/data/functional/broken_output_ok_test/exec_used.txt b/tests/testutils/data/functional/broken_output_ok_test/exec_used.txt
new file mode 100644
index 000000000..d74b20893
--- /dev/null
+++ b/tests/testutils/data/functional/broken_output_ok_test/exec_used.txt
@@ -0,0 +1 @@
+exec-used:UNDEFINED
diff --git a/tests/testutils/data/functional/broken_output_wrong_test/exec_used.py b/tests/testutils/data/functional/broken_output_wrong_test/exec_used.py
new file mode 100644
index 000000000..8b6c06a57
--- /dev/null
+++ b/tests/testutils/data/functional/broken_output_wrong_test/exec_used.py
@@ -0,0 +1,7 @@
+"""This is an example for a functional test with
+- an output that is broken
+- the functional test itself is wrong
+
+So it cannot be updated.
+"""
+exec('a = 42')
diff --git a/tests/testutils/data/functional/broken_output_wrong_test/exec_used.txt b/tests/testutils/data/functional/broken_output_wrong_test/exec_used.txt
new file mode 100644
index 000000000..d74b20893
--- /dev/null
+++ b/tests/testutils/data/functional/broken_output_wrong_test/exec_used.txt
@@ -0,0 +1 @@
+exec-used:UNDEFINED
diff --git a/tests/testutils/data/functional/no_output_ok_test/exec_used.py b/tests/testutils/data/functional/no_output_ok_test/exec_used.py
new file mode 100644
index 000000000..59d19fec0
--- /dev/null
+++ b/tests/testutils/data/functional/no_output_ok_test/exec_used.py
@@ -0,0 +1,7 @@
+"""This is an example for a functional test with
+- an output that does not exist
+- the functional test itself is right
+
+So it should be created
+"""
+exec('a = 42')  # [exec-used]
diff --git a/tests/testutils/data/functional/no_output_wrong_test/exec_used.py b/tests/testutils/data/functional/no_output_wrong_test/exec_used.py
new file mode 100644
index 000000000..d3b318ab7
--- /dev/null
+++ b/tests/testutils/data/functional/no_output_wrong_test/exec_used.py
@@ -0,0 +1,7 @@
+"""This is an example for a functional test with
+- an output that does not exist
+- the functional test itself is wrong
+
+So it cannot be created.
+"""
+exec('a = 42')
diff --git a/tests/testutils/data/functional/ok_output_ok_test/exec_used.py b/tests/testutils/data/functional/ok_output_ok_test/exec_used.py
new file mode 100644
index 000000000..049213dd4
--- /dev/null
+++ b/tests/testutils/data/functional/ok_output_ok_test/exec_used.py
@@ -0,0 +1,7 @@
+"""This is an example for a functional test with
+- an output that is right
+- the functional test itself is right
+
+Nothing should be done.
+"""
+exec('a = 42')  # [exec-used]
diff --git a/tests/testutils/data/functional/ok_output_ok_test/exec_used.txt b/tests/testutils/data/functional/ok_output_ok_test/exec_used.txt
new file mode 100644
index 000000000..6d0f82baf
--- /dev/null
+++ b/tests/testutils/data/functional/ok_output_ok_test/exec_used.txt
@@ -0,0 +1 @@
+exec-used:7:0:7:14::Use of exec:UNDEFINED
diff --git a/tests/testutils/data/functional/ok_output_wrong_test/exec_used.py b/tests/testutils/data/functional/ok_output_wrong_test/exec_used.py
new file mode 100644
index 000000000..3f8637228
--- /dev/null
+++ b/tests/testutils/data/functional/ok_output_wrong_test/exec_used.py
@@ -0,0 +1,7 @@
+"""This is an example for a functional test with
+- an output that is right
+- the functional test itself is wrong
+
+Nothing should be done.
+"""
+exec('a = 42')
diff --git a/tests/testutils/data/functional/ok_output_wrong_test/exec_used.txt b/tests/testutils/data/functional/ok_output_wrong_test/exec_used.txt
new file mode 100644
index 000000000..6d0f82baf
--- /dev/null
+++ b/tests/testutils/data/functional/ok_output_wrong_test/exec_used.txt
@@ -0,0 +1 @@
+exec-used:7:0:7:14::Use of exec:UNDEFINED
diff --git a/tests/testutils/data/functional/wrong_output_ok_test/exec_used.py b/tests/testutils/data/functional/wrong_output_ok_test/exec_used.py
new file mode 100644
index 000000000..0874f0a15
--- /dev/null
+++ b/tests/testutils/data/functional/wrong_output_ok_test/exec_used.py
@@ -0,0 +1,7 @@
+"""This is an example for a functional test with
+- an output that is wrong (but not broken)
+- the functional test itself is right
+
+So it needs to be updated.
+"""
+exec('a = 42')  # [exec-used]
diff --git a/tests/testutils/data/functional/wrong_output_ok_test/exec_used.txt b/tests/testutils/data/functional/wrong_output_ok_test/exec_used.txt
new file mode 100644
index 000000000..a737b8d51
--- /dev/null
+++ b/tests/testutils/data/functional/wrong_output_ok_test/exec_used.txt
@@ -0,0 +1 @@
+missing-docstring:5:0:1:1::Missing docstring in file:HIGH
diff --git a/tests/testutils/data/functional/wrong_output_wrong_test/exec_used.py b/tests/testutils/data/functional/wrong_output_wrong_test/exec_used.py
new file mode 100644
index 000000000..fc35114e8
--- /dev/null
+++ b/tests/testutils/data/functional/wrong_output_wrong_test/exec_used.py
@@ -0,0 +1,7 @@
+"""This is an example for a functional test with
+- an output that is wrong (but not broken)
+- the functional test itself is wrong
+
+So it can't be updated.
+"""
+exec('a = 42')
diff --git a/tests/testutils/data/functional/wrong_output_wrong_test/exec_used.txt b/tests/testutils/data/functional/wrong_output_wrong_test/exec_used.txt
new file mode 100644
index 000000000..a737b8d51
--- /dev/null
+++ b/tests/testutils/data/functional/wrong_output_wrong_test/exec_used.txt
@@ -0,0 +1 @@
+missing-docstring:5:0:1:1::Missing docstring in file:HIGH
diff --git a/tests/testutils/test_functional_testutils.py b/tests/testutils/test_functional_testutils.py
index 68dad697d..e0055f1b0 100644
--- a/tests/testutils/test_functional_testutils.py
+++ b/tests/testutils/test_functional_testutils.py
@@ -42,7 +42,7 @@ def test_parsing_of_pylintrc_init_hook() -> None:
 def test_get_functional_test_files_from_directory() -> None:
     """Test that we correctly check the functional test directory structures."""
     with pytest.raises(AssertionError, match="using_dir.py should not go in"):
-        get_functional_test_files_from_directory(DATA_DIRECTORY)
+        get_functional_test_files_from_directory(DATA_DIRECTORY / "u")
 
 
 def test_minimal_messages_config_enabled(pytest_config: MagicMock) -> None:
diff --git a/tests/testutils/test_lint_module_output_update.py b/tests/testutils/test_lint_module_output_update.py
index 2e387a118..22da7b0f7 100644
--- a/tests/testutils/test_lint_module_output_update.py
+++ b/tests/testutils/test_lint_module_output_update.py
@@ -6,15 +6,20 @@
 
 from __future__ import annotations
 
+import shutil
 from collections.abc import Callable
 from pathlib import Path
 
 import pytest
 
-from pylint.constants import PY38_PLUS
-from pylint.testutils import FunctionalTestFile
+from pylint.constants import IS_PYPY, PY38_PLUS, PY39_PLUS
+from pylint.testutils import FunctionalTestFile, LintModuleTest
 from pylint.testutils.functional import LintModuleOutputUpdate
 
+FIXTURE_DIRECTORY = Path(__file__).parent / "data/functional"
+DIRECTORIES = list(FIXTURE_DIRECTORY.iterdir())
+DIRECTORIES_NAMES = [d.name for d in DIRECTORIES]
+
 
 @pytest.fixture()
 def lint_module_fixture(
@@ -76,4 +81,65 @@ def test_lint_module_output_update_remove_useless_txt(
     expected_output_file.write_text("", encoding="utf8")
     filename.write_text("", encoding="utf8")
     lmou.runTest()
-    assert not (expected_output_file).exists()
+    assert not expected_output_file.exists()
+
+
+@pytest.mark.skipif(
+    not PY38_PLUS or (IS_PYPY and not PY39_PLUS),
+    reason="Requires accurate 'end_col' value to update output",
+)
+@pytest.mark.parametrize("directory_path", DIRECTORIES, ids=DIRECTORIES_NAMES)
+def test_update_of_functional_output(directory_path: Path, tmp_path: Path) -> None:
+    """Functional test for the functional tests helper."""
+
+    def _check_expected_output(ftf: FunctionalTestFile) -> None:
+        new_output_path = ftf.expected_output
+        assert Path(
+            new_output_path
+        ).exists(), "The expected output file does not exists"
+        with open(new_output_path, encoding="utf8") as f:
+            new_output = f.read()
+        assert (
+            new_output == "exec-used:7:0:7:14::Use of exec:UNDEFINED\n"
+        ), f"The content was wrongly updated in {new_output_path}"
+
+    new_path = tmp_path / directory_path.name
+    shutil.copytree(directory_path, new_path)
+    for filename in new_path.iterdir():
+        if filename.suffix != ".py":
+            continue
+        new_path_str = str(new_path)
+        ftf = FunctionalTestFile(directory=new_path_str, filename=filename.name)
+        # Standard functional test helper
+        lint_module = LintModuleTest(ftf)
+        # Functional test helper that automatically update the output
+        lint_module_output_update = LintModuleOutputUpdate(ftf)
+        if "wrong_test" in new_path_str:
+            expected = r'Wrong message\(s\) raised for "exec_used.py"'
+            with pytest.raises(AssertionError, match=expected):
+                lint_module.runTest()
+            # When the tests are wrong we do not update the output at all
+            # and the test should fail
+            with pytest.raises(AssertionError, match=expected):
+                lint_module_output_update.runTest()
+        elif "ok_test" in new_path_str:
+            if any(f"{x}_output" in new_path_str for x in ("wrong", "no", "broken")):
+                with pytest.raises(
+                    AssertionError, match='Wrong output for "exec_used.txt"'
+                ):
+                    lint_module.runTest()
+            elif "ok_output" in new_path_str:
+                lint_module.runTest()
+                _check_expected_output(ftf)
+            else:
+                raise AssertionError(f"Unhandled test case: {new_path_str}")
+
+            # When the tests are ok we update the output whatever it's state
+            # was originally
+            lint_module_output_update.runTest()
+            _check_expected_output(ftf)
+        else:
+            raise AssertionError(
+                f"Do not pollute '{FIXTURE_DIRECTORY}' with unrelated "
+                f"or badly named test files."
+            )
diff --git a/tests/testutils/test_output_line.py b/tests/testutils/test_output_line.py
index 5b2bf1a1b..a7e677157 100644
--- a/tests/testutils/test_output_line.py
+++ b/tests/testutils/test_output_line.py
@@ -134,31 +134,6 @@ def test_output_line_to_csv(confidence: Confidence, message: _MessageCallable) -
     )
 
 
-def test_output_line_from_csv_error() -> None:
-    """Test that errors are correctly raised for incorrect OutputLine's."""
-    # Test a csv-string which does not have a number for line and column
-    with pytest.warns(
-        UserWarning,
-        match="msg-symbolic-name:42:27:MyClass.my_function:The message",
-    ):
-        OutputLine.from_csv("'missing-docstring', 'line', 'column', 'obj', 'msg'", True)
-    # Test a tuple which does not have a number for line and column
-    with pytest.warns(
-        UserWarning, match="we got 'missing-docstring:line:column:obj:msg'"
-    ):
-        csv = ("missing-docstring", "line", "column", "obj", "msg")
-        OutputLine.from_csv(csv, True)
-    # Test a csv-string that is too long
-    with pytest.warns(
-        UserWarning,
-        match="msg-symbolic-name:42:27:MyClass.my_function:The message",
-    ):
-        OutputLine.from_csv(
-            "'missing-docstring', 1, 2, 'obj', 'msg', 'func', 'message', 'conf', 'too_long'",
-            True,
-        )
-
-
 @pytest.mark.parametrize(
     "confidence,expected_confidence", [[None, "UNDEFINED"], ["INFERENCE", "INFERENCE"]]
 )