Create a framework of functional tests for configuration files (#5287)

* Migrate old unittest to the new framework for testing.

* Add a regression test for #4746 : This permits to introduce an example
of configuration file with an error.

* Proper import for pytest import of CaptureFixture

Co-authored-by: Daniël van Noord <13665637+DanielNoord@users.noreply.github.com>

16 files changed, 192 insertions, 96 deletions

diff --git a/tests/config/conftest.py b/tests/config/conftest.py
new file mode 100644
index 000000000..5a6778af2
--- /dev/null
+++ b/tests/config/conftest.py
@@ -0,0 +1,10 @@
+from pathlib import Path
+
+import pytest
+
+HERE = Path(__file__).parent
+
+
+@pytest.fixture()
+def file_to_lint_path() -> str:
+    return str(HERE / "file_to_lint.py")
diff --git a/tests/config/functional/ini/pylintrc_with_message_control.ini b/tests/config/functional/ini/pylintrc_with_message_control.ini
new file mode 100644
index 000000000..95d99036f
--- /dev/null
+++ b/tests/config/functional/ini/pylintrc_with_message_control.ini
@@ -0,0 +1,5 @@
+# Check that we can read the "regular" INI .pylintrc file
+[messages control]
+disable = logging-not-lazy,logging-format-interpolation
+jobs = 10
+reports = yes
diff --git a/tests/config/functional/ini/pylintrc_with_message_control.result.json b/tests/config/functional/ini/pylintrc_with_message_control.result.json
new file mode 100644
index 000000000..21938c319
--- /dev/null
+++ b/tests/config/functional/ini/pylintrc_with_message_control.result.json
@@ -0,0 +1,7 @@
+{
+  "functional_append": {
+    "disable": [["logging-not-lazy"], ["logging-format-interpolation"]]
+  },
+  "jobs": 10,
+  "reports": true
+}
diff --git a/tests/config/functional/setup_cfg/setup_cfg_with_message_control.cfg b/tests/config/functional/setup_cfg/setup_cfg_with_message_control.cfg
new file mode 100644
index 000000000..d5e4c564c
--- /dev/null
+++ b/tests/config/functional/setup_cfg/setup_cfg_with_message_control.cfg
@@ -0,0 +1,5 @@
+# setup.cfg is an INI file where section names are prefixed with "pylint."
+[pylint.messages control]
+disable = logging-not-lazy,logging-format-interpolation
+jobs = 10
+reports = yes
diff --git a/tests/config/functional/setup_cfg/setup_cfg_with_message_control.result.json b/tests/config/functional/setup_cfg/setup_cfg_with_message_control.result.json
new file mode 100644
index 000000000..21938c319
--- /dev/null
+++ b/tests/config/functional/setup_cfg/setup_cfg_with_message_control.result.json
@@ -0,0 +1,7 @@
+{
+  "functional_append": {
+    "disable": [["logging-not-lazy"], ["logging-format-interpolation"]]
+  },
+  "jobs": 10,
+  "reports": true
+}
diff --git a/tests/config/functional/toml/issue_4746/loaded_plugin_does_not_exists.out b/tests/config/functional/toml/issue_4746/loaded_plugin_does_not_exists.out
new file mode 100644
index 000000000..a6837722a
--- /dev/null
+++ b/tests/config/functional/toml/issue_4746/loaded_plugin_does_not_exists.out
@@ -0,0 +1,2 @@
+************* Module {abspath}
+{relpath}:1:0: E0013: Plugin 'pylint_websockets' is impossible to load, is it installed ? ('No module named 'pylint_websockets'') (bad-plugin-value)
diff --git a/tests/config/functional/toml/issue_4746/loaded_plugin_does_not_exists.result.json b/tests/config/functional/toml/issue_4746/loaded_plugin_does_not_exists.result.json
new file mode 100644
index 000000000..0b6175c4a
--- /dev/null
+++ b/tests/config/functional/toml/issue_4746/loaded_plugin_does_not_exists.result.json
@@ -0,0 +1,3 @@
+{
+  "load_plugins": ["pylint_websockets"]
+}
diff --git a/tests/config/functional/toml/issue_4746/loaded_plugin_does_not_exists.toml b/tests/config/functional/toml/issue_4746/loaded_plugin_does_not_exists.toml
new file mode 100644
index 000000000..c015a9448
--- /dev/null
+++ b/tests/config/functional/toml/issue_4746/loaded_plugin_does_not_exists.toml
@@ -0,0 +1,3 @@
+# The pylint_websockets plugin does not exist and therefore this toml is invalid
+[tool.pylint.MASTER]
+load-plugins = 'pylint_websockets'
diff --git a/tests/config/functional/toml/rich_types.result.json b/tests/config/functional/toml/rich_types.result.json
new file mode 100644
index 000000000..21938c319
--- /dev/null
+++ b/tests/config/functional/toml/rich_types.result.json
@@ -0,0 +1,7 @@
+{
+  "functional_append": {
+    "disable": [["logging-not-lazy"], ["logging-format-interpolation"]]
+  },
+  "jobs": 10,
+  "reports": true
+}
diff --git a/tests/config/functional/toml/rich_types.toml b/tests/config/functional/toml/rich_types.toml
new file mode 100644
index 000000000..91178390e
--- /dev/null
+++ b/tests/config/functional/toml/rich_types.toml
@@ -0,0 +1,11 @@
+# Check that we can read a TOML file where lists, integers and
+# booleans are expressed as such (and not as strings), using TOML
+# type system.
+
+[tool.pylint."messages control"]
+disable = [
+    "logging-not-lazy",
+    "logging-format-interpolation",
+]
+jobs = 10
+reports = true
diff --git a/tests/config/functional/toml/toml_with_enable.result.json b/tests/config/functional/toml/toml_with_enable.result.json
new file mode 100644
index 000000000..0bdbc840d
--- /dev/null
+++ b/tests/config/functional/toml/toml_with_enable.result.json
@@ -0,0 +1,9 @@
+{
+  "functional_append": {
+    "disable": [["logging-not-lazy"], ["logging-format-interpolation"]],
+    "enable": [["suppressed-message"], ["locally-disabled"]]
+  },
+  "functional_remove": {
+    "disable": [["suppressed-message"], ["locally-disabled"]]
+  }
+}
diff --git a/tests/config/functional/toml/toml_with_enable.toml b/tests/config/functional/toml/toml_with_enable.toml
new file mode 100644
index 000000000..a1e7b65af
--- /dev/null
+++ b/tests/config/functional/toml/toml_with_enable.toml
@@ -0,0 +1,5 @@
+# Check that we can add or remove value in list
+# (This is mostly a check for the functional test themselves)
+[tool.pylint."messages control"]
+disable = "logging-not-lazy,logging-format-interpolation"
+enable = "locally-disabled,suppressed-message"
diff --git a/tests/config/functional/toml/toml_with_message_control.result.json b/tests/config/functional/toml/toml_with_message_control.result.json
new file mode 100644
index 000000000..21938c319
--- /dev/null
+++ b/tests/config/functional/toml/toml_with_message_control.result.json
@@ -0,0 +1,7 @@
+{
+  "functional_append": {
+    "disable": [["logging-not-lazy"], ["logging-format-interpolation"]]
+  },
+  "jobs": 10,
+  "reports": true
+}
diff --git a/tests/config/functional/toml/toml_with_message_control.toml b/tests/config/functional/toml/toml_with_message_control.toml
new file mode 100644
index 000000000..0e58d8918
--- /dev/null
+++ b/tests/config/functional/toml/toml_with_message_control.toml
@@ -0,0 +1,7 @@
+# Check that we can read a TOML file where lists and integers are
+# expressed as strings.
+
+[tool.pylint."messages control"]
+disable = "logging-not-lazy,logging-format-interpolation"
+jobs = "10"
+reports = "yes"
diff --git a/tests/config/test_config.py b/tests/config/test_config.py
index 8ce5d9b42..f89e62416 100644
--- a/tests/config/test_config.py
+++ b/tests/config/test_config.py
@@ -1,33 +1,9 @@
-# pylint: disable=missing-module-docstring, missing-function-docstring, protected-access
 import os
-import unittest.mock
 from pathlib import Path
-from typing import Optional, Set, Union
+from typing import Optional, Set
 
-import pylint.lint
 from pylint.lint.run import Run
-
-# We use an external file and not __file__ or pylint warning in this file
-# makes the tests fails because the exit code changes
-FILE_TO_LINT = str(Path(__file__).parent / "file_to_lint.py")
-
-
-def get_runner_from_config_file(
-    config_file: Union[str, Path], expected_exit_code: int = 0
-) -> Run:
-    """Initialize pylint with the given configuration file and return the Run"""
-    args = ["--rcfile", str(config_file), FILE_TO_LINT]
-    # If we used `pytest.raises(SystemExit)`, the `runner` variable
-    # would not be accessible outside the `with` block.
-    with unittest.mock.patch("sys.exit") as mocked_exit:
-        # Do not actually run checks, that could be slow. Do not mock
-        # `Pylinter.check`: it calls `Pylinter.initialize` which is
-        # needed to properly set up messages inclusion/exclusion
-        # in `_msg_states`, used by `is_message_enabled`.
-        with unittest.mock.patch("pylint.lint.pylinter.check_parallel"):
-            runner = pylint.lint.Run(args)
-    mocked_exit.assert_called_once_with(expected_exit_code)
-    return runner
+from pylint.testutils.configuration_test import run_using_a_configuration_file
 
 
 def check_configuration_file_reader(
@@ -46,74 +22,7 @@ def check_configuration_file_reader(
     assert bool(runner.linter.config.reports) == expected_reports_truthey
 
 
-def test_can_read_ini(tmp_path: Path) -> None:
-    # Check that we can read the "regular" INI .pylintrc file
-    config_file = tmp_path / ".pylintrc"
-    config_file.write_text(
-        """
-[messages control]
-disable = logging-not-lazy,logging-format-interpolation
-jobs = 10
-reports = yes
-"""
-    )
-    run = get_runner_from_config_file(config_file)
-    check_configuration_file_reader(run)
-
-
-def test_can_read_setup_cfg(tmp_path: Path) -> None:
-    # Check that we can read a setup.cfg (which is an INI file where
-    # section names are prefixed with "pylint."
-    config_file = tmp_path / "setup.cfg"
-    config_file.write_text(
-        """
-[pylint.messages control]
-disable = logging-not-lazy,logging-format-interpolation
-jobs = 10
-reports = yes
-"""
-    )
-    run = get_runner_from_config_file(config_file)
-    check_configuration_file_reader(run)
-
-
-def test_can_read_toml(tmp_path: Path) -> None:
-    # Check that we can read a TOML file where lists and integers are
-    # expressed as strings.
-    config_file = tmp_path / "pyproject.toml"
-    config_file.write_text(
-        """
-[tool.pylint."messages control"]
-disable = "logging-not-lazy,logging-format-interpolation"
-jobs = "10"
-reports = "yes"
-"""
-    )
-    run = get_runner_from_config_file(config_file)
-    check_configuration_file_reader(run)
-
-
-def test_can_read_toml_rich_types(tmp_path: Path) -> None:
-    # Check that we can read a TOML file where lists, integers and
-    # booleans are expressed as such (and not as strings), using TOML
-    # type system.
-    config_file = tmp_path / "pyproject.toml"
-    config_file.write_text(
-        """
-[tool.pylint."messages control"]
-disable = [
-    "logging-not-lazy",
-    "logging-format-interpolation",
-]
-jobs = 10
-reports = true
-"""
-    )
-    run = get_runner_from_config_file(config_file)
-    check_configuration_file_reader(run)
-
-
-def test_can_read_toml_env_variable(tmp_path: Path) -> None:
+def test_can_read_toml_env_variable(tmp_path: Path, file_to_lint_path: str) -> None:
     """We can read and open a properly formatted toml file."""
     config_file = tmp_path / "pyproject.toml"
     config_file.write_text(
@@ -126,5 +35,8 @@ reports = "yes"
     )
     env_var = "tmp_path_env"
     os.environ[env_var] = str(config_file)
-    run = get_runner_from_config_file(f"${env_var}")
-    check_configuration_file_reader(run)
+    mock_exit, _, runner = run_using_a_configuration_file(
+        f"${env_var}", file_to_lint_path
+    )
+    mock_exit.assert_called_once_with(0)
+    check_configuration_file_reader(runner)
diff --git a/tests/config/test_functional_config_loading.py b/tests/config/test_functional_config_loading.py
new file mode 100644
index 000000000..452d7cc90
--- /dev/null
+++ b/tests/config/test_functional_config_loading.py
@@ -0,0 +1,96 @@
+# Licensed under the GPL: https://www.gnu.org/licenses/old-licenses/gpl-2.0.html
+# For details: https://github.com/PyCQA/pylint/blob/main/LICENSE
+
+"""
+This launches the configuration functional tests. This permits to test configuration
+files by providing a file with the appropriate extension in the ``tests/config/functional``
+directory.
+
+Let's say you have a regression_list_crash.toml file to test. Then if there is an error in the conf,
+add ``regression_list_crash.out`` alongside your file with the expected output of pylint in it. Use
+``{relpath}`` and ``{abspath}`` for the path of the file. The exit code will have to be 2 (error)
+if this file exists.
+
+You must also define a ``regression_list_crash.result.json`` if you want to check the parsed configuration.
+This file will be loaded as a dict and will override the default value of the default pylint
+configuration. If you need to append or remove a value use the special key ``"functional_append"``
+and ``"functional_remove":``. Check the existing code for examples.
+"""
+
+# pylint: disable=redefined-outer-name
+import logging
+from pathlib import Path
+
+import pytest
+from pytest import CaptureFixture, LogCaptureFixture
+
+from pylint.testutils.configuration_test import (
+    PylintConfiguration,
+    get_expected_configuration,
+    get_expected_output,
+    run_using_a_configuration_file,
+)
+
+HERE = Path(__file__).parent
+FUNCTIONAL_DIR = HERE / "functional"
+# We use string then recast to path so we can use -k in pytest.
+# Otherwise we get 'configuration_path0' as a test name. The path is relative to the functional
+# directory because otherwise the string would be very lengthy.
+ACCEPTED_CONFIGURATION_EXTENSIONS = ("toml", "ini", "cfg")
+CONFIGURATION_PATHS = [
+    str(path.relative_to(FUNCTIONAL_DIR))
+    for ext in ACCEPTED_CONFIGURATION_EXTENSIONS
+    for path in FUNCTIONAL_DIR.rglob(f"*.{ext}")
+]
+
+
+@pytest.fixture()
+def default_configuration(
+    tmp_path: Path, file_to_lint_path: str
+) -> PylintConfiguration:
+    empty_pylintrc = tmp_path / "pylintrc"
+    empty_pylintrc.write_text("")
+    mock_exit, _, runner = run_using_a_configuration_file(
+        str(empty_pylintrc), file_to_lint_path
+    )
+    mock_exit.assert_called_once_with(0)
+    return runner.linter.config.__dict__
+
+
+@pytest.mark.parametrize("configuration_path", CONFIGURATION_PATHS)
+def test_functional_config_loading(
+    configuration_path: str,
+    default_configuration: PylintConfiguration,
+    file_to_lint_path: str,
+    capsys: CaptureFixture,
+    caplog: LogCaptureFixture,
+):
+    """Functional tests for configurations."""
+    # logging is helpful to see what's expected and why. The output of the
+    # program is checked during the test so printing messes with the result.
+    caplog.set_level(logging.INFO)
+    configuration_path = str(FUNCTIONAL_DIR / configuration_path)
+    msg = f"Wrong result with configuration {configuration_path}"
+    expected_code, expected_output = get_expected_output(configuration_path)
+    expected_loaded_configuration = get_expected_configuration(
+        configuration_path, default_configuration
+    )
+    mock_exit, _, runner = run_using_a_configuration_file(
+        configuration_path, file_to_lint_path
+    )
+    mock_exit.assert_called_once_with(expected_code)
+    out, err = capsys.readouterr()
+    # rstrip() applied so we can have a final newline in the expected test file
+    assert expected_output.rstrip() == out.rstrip(), msg
+    assert sorted(expected_loaded_configuration.keys()) == sorted(
+        runner.linter.config.__dict__.keys()
+    ), msg
+    for key, expected_value in expected_loaded_configuration.items():
+        key_msg = f"{msg} for key '{key}':"
+        if isinstance(expected_value, list):
+            assert sorted(expected_value) == sorted(
+                runner.linter.config.__dict__[key]
+            ), key_msg
+        else:
+            assert expected_value == runner.linter.config.__dict__[key], key_msg
+    assert not err, msg