From 6626c239e513348ee7ed6133868c86f82bf26b67 Mon Sep 17 00:00:00 2001 From: Pierre Sassoulas Date: Wed, 10 Nov 2021 09:27:15 +0100 Subject: Create a framework of functional tests for configuration files Also migrate three standard unittest to the new framework for testing. --- pylint/testutils/configuration_test.py | 85 ++++++++++++++++++++++ .../ini/pylintrc_with_message_control.ini | 5 ++ .../ini/pylintrc_with_message_control.result.json | 7 ++ .../setup_cfg/setup_cfg_with_message_control.cfg | 5 ++ .../setup_cfg_with_message_control.result.json | 7 ++ .../toml/toml_with_message_control.result.json | 7 ++ .../functional/toml/toml_with_message_control.toml | 7 ++ tests/config/test_config.py | 47 ------------ tests/config/test_functional_config_loading.py | 79 ++++++++++++++++++++ 9 files changed, 202 insertions(+), 47 deletions(-) create mode 100644 pylint/testutils/configuration_test.py create mode 100644 tests/config/functional/ini/pylintrc_with_message_control.ini create mode 100644 tests/config/functional/ini/pylintrc_with_message_control.result.json create mode 100644 tests/config/functional/setup_cfg/setup_cfg_with_message_control.cfg create mode 100644 tests/config/functional/setup_cfg/setup_cfg_with_message_control.result.json create mode 100644 tests/config/functional/toml/toml_with_message_control.result.json create mode 100644 tests/config/functional/toml/toml_with_message_control.toml create mode 100644 tests/config/test_functional_config_loading.py diff --git a/pylint/testutils/configuration_test.py b/pylint/testutils/configuration_test.py new file mode 100644 index 000000000..00d828b08 --- /dev/null +++ b/pylint/testutils/configuration_test.py @@ -0,0 +1,85 @@ +# Licensed under the GPL: https://www.gnu.org/licenses/old-licenses/gpl-2.0.html +# For details: https://github.com/PyCQA/pylint/blob/main/LICENSE + +"""Utility function for configuration testing.""" +import copy +import json +import unittest +from pathlib import Path +from typing import Any, Dict, Tuple, Union + +from pylint.lint import Run + +USER_SPECIFIC = str(Path(__file__).parent.parent.parent) + + +def get_expected_or_default(pyproject_toml_path: str, suffix: str, default: Any) -> str: + def get_path_according_to_suffix() -> Path: + path = Path(pyproject_toml_path) + return path.parent / f"{path.stem}.{suffix}" + + expected = default + expected_result_path = get_path_according_to_suffix() + if expected_result_path.exists(): + with open(expected_result_path, encoding="utf8") as f: + expected = f.read() + return expected + + +EXPECTED_CONF_APPEND_KEY = "functional_append" +EXPECTED_CONF_REMOVE_KEY = "functional_remove" +EXPECTED_CONF_SPECIAL_KEYS = [EXPECTED_CONF_APPEND_KEY, EXPECTED_CONF_REMOVE_KEY] + + +def get_expected_configuration( + configuration_path: str, default_configuration: Dict[str, Any] +) -> Dict[str, Any]: + """Get the expected parsed configuration of a configuration functional test""" + result = copy.deepcopy(default_configuration) + config_as_json = get_expected_or_default( + configuration_path, suffix="result.json", default="{}" + ) + to_override = json.loads(config_as_json) + for key, value in to_override.items(): + if key not in EXPECTED_CONF_SPECIAL_KEYS: + result[key] = value + if EXPECTED_CONF_APPEND_KEY in to_override: + for key, value in to_override[EXPECTED_CONF_APPEND_KEY].items(): + result[key] += value + if EXPECTED_CONF_REMOVE_KEY in to_override: + for key, value in to_override[EXPECTED_CONF_REMOVE_KEY].items(): + result[key] -= value + return result + + +def get_expected_output(configuration_path: str) -> Tuple[int, str]: + """Get the expected output of a functional test.""" + + def get_relative_path(path: str) -> str: + """Get the relative path we want without user specific information""" + return path.replace(USER_SPECIFIC, "")[1:] + + output = get_expected_or_default(configuration_path, suffix="out", default="") + exit_code = 2 if output else 0 + return exit_code, output.format( + abspath=configuration_path, relpath=get_relative_path(configuration_path) + ) + + +def run_using_a_configuration_file( + configuration_path: Union[Path, str], file_to_lint: str = __file__ +) -> Tuple[unittest.mock.Mock, unittest.mock.Mock, Run]: + """Simulate a run with a configuration without really launching the checks.""" + configuration_path = str(configuration_path) + args = ["--rcfile", configuration_path, file_to_lint] + # If we used `pytest.raises(SystemExit)`, the `runner` variable + # would not be accessible outside the `with` block. + with unittest.mock.patch("sys.exit") as mocked_exit: + # Do not actually run checks, that could be slow. Do not mock + # `Pylinter.check`: it calls `Pylinter.initialize` which is + # needed to properly set up messages inclusion/exclusion + # in `_msg_states`, used by `is_message_enabled`. + check = "pylint.lint.pylinter.check_parallel" + with unittest.mock.patch(check) as mocked_check_parallel: + runner = Run(args) + return mocked_exit, mocked_check_parallel, runner diff --git a/tests/config/functional/ini/pylintrc_with_message_control.ini b/tests/config/functional/ini/pylintrc_with_message_control.ini new file mode 100644 index 000000000..95d99036f --- /dev/null +++ b/tests/config/functional/ini/pylintrc_with_message_control.ini @@ -0,0 +1,5 @@ +# Check that we can read the "regular" INI .pylintrc file +[messages control] +disable = logging-not-lazy,logging-format-interpolation +jobs = 10 +reports = yes diff --git a/tests/config/functional/ini/pylintrc_with_message_control.result.json b/tests/config/functional/ini/pylintrc_with_message_control.result.json new file mode 100644 index 000000000..21938c319 --- /dev/null +++ b/tests/config/functional/ini/pylintrc_with_message_control.result.json @@ -0,0 +1,7 @@ +{ + "functional_append": { + "disable": [["logging-not-lazy"], ["logging-format-interpolation"]] + }, + "jobs": 10, + "reports": true +} diff --git a/tests/config/functional/setup_cfg/setup_cfg_with_message_control.cfg b/tests/config/functional/setup_cfg/setup_cfg_with_message_control.cfg new file mode 100644 index 000000000..d5e4c564c --- /dev/null +++ b/tests/config/functional/setup_cfg/setup_cfg_with_message_control.cfg @@ -0,0 +1,5 @@ +# setup.cfg is an INI file where section names are prefixed with "pylint." +[pylint.messages control] +disable = logging-not-lazy,logging-format-interpolation +jobs = 10 +reports = yes diff --git a/tests/config/functional/setup_cfg/setup_cfg_with_message_control.result.json b/tests/config/functional/setup_cfg/setup_cfg_with_message_control.result.json new file mode 100644 index 000000000..21938c319 --- /dev/null +++ b/tests/config/functional/setup_cfg/setup_cfg_with_message_control.result.json @@ -0,0 +1,7 @@ +{ + "functional_append": { + "disable": [["logging-not-lazy"], ["logging-format-interpolation"]] + }, + "jobs": 10, + "reports": true +} diff --git a/tests/config/functional/toml/toml_with_message_control.result.json b/tests/config/functional/toml/toml_with_message_control.result.json new file mode 100644 index 000000000..21938c319 --- /dev/null +++ b/tests/config/functional/toml/toml_with_message_control.result.json @@ -0,0 +1,7 @@ +{ + "functional_append": { + "disable": [["logging-not-lazy"], ["logging-format-interpolation"]] + }, + "jobs": 10, + "reports": true +} diff --git a/tests/config/functional/toml/toml_with_message_control.toml b/tests/config/functional/toml/toml_with_message_control.toml new file mode 100644 index 000000000..0e58d8918 --- /dev/null +++ b/tests/config/functional/toml/toml_with_message_control.toml @@ -0,0 +1,7 @@ +# Check that we can read a TOML file where lists and integers are +# expressed as strings. + +[tool.pylint."messages control"] +disable = "logging-not-lazy,logging-format-interpolation" +jobs = "10" +reports = "yes" diff --git a/tests/config/test_config.py b/tests/config/test_config.py index 8ce5d9b42..723d4636e 100644 --- a/tests/config/test_config.py +++ b/tests/config/test_config.py @@ -46,53 +46,6 @@ def check_configuration_file_reader( assert bool(runner.linter.config.reports) == expected_reports_truthey -def test_can_read_ini(tmp_path: Path) -> None: - # Check that we can read the "regular" INI .pylintrc file - config_file = tmp_path / ".pylintrc" - config_file.write_text( - """ -[messages control] -disable = logging-not-lazy,logging-format-interpolation -jobs = 10 -reports = yes -""" - ) - run = get_runner_from_config_file(config_file) - check_configuration_file_reader(run) - - -def test_can_read_setup_cfg(tmp_path: Path) -> None: - # Check that we can read a setup.cfg (which is an INI file where - # section names are prefixed with "pylint." - config_file = tmp_path / "setup.cfg" - config_file.write_text( - """ -[pylint.messages control] -disable = logging-not-lazy,logging-format-interpolation -jobs = 10 -reports = yes -""" - ) - run = get_runner_from_config_file(config_file) - check_configuration_file_reader(run) - - -def test_can_read_toml(tmp_path: Path) -> None: - # Check that we can read a TOML file where lists and integers are - # expressed as strings. - config_file = tmp_path / "pyproject.toml" - config_file.write_text( - """ -[tool.pylint."messages control"] -disable = "logging-not-lazy,logging-format-interpolation" -jobs = "10" -reports = "yes" -""" - ) - run = get_runner_from_config_file(config_file) - check_configuration_file_reader(run) - - def test_can_read_toml_rich_types(tmp_path: Path) -> None: # Check that we can read a TOML file where lists, integers and # booleans are expressed as such (and not as strings), using TOML diff --git a/tests/config/test_functional_config_loading.py b/tests/config/test_functional_config_loading.py new file mode 100644 index 000000000..5d3696c2f --- /dev/null +++ b/tests/config/test_functional_config_loading.py @@ -0,0 +1,79 @@ +# Licensed under the GPL: https://www.gnu.org/licenses/old-licenses/gpl-2.0.html +# For details: https://github.com/PyCQA/pylint/blob/main/LICENSE + +""" +This launch the configuration functional tests. This permit to test configuration +files by providing a file with the appropriate extension in the ``tests/config/functional`` +directory. + +Let's say you have a regression_list_crash.toml file to test. Then if there is an error in the conf, +add ``regression_list_crash.out`` alongside your file with the expected output of pylint in it. Use +``{relpath}`` and ``{abspath}`` for the path of the file. The exit code will have to be 2 (error) +if this file exists. + +You must also define a ``regression_list_crash.result.json`` if you want to check the parsed configuration. +This file will be loaded as a dict and will override the default value of the default pylint +configuration. If you need to append or remove a value use the special key ``"functional_append"`` +and ``"functional_remove":``. Check the existing code for example. +""" + +# pylint: disable=redefined-outer-name + +from pathlib import Path +from typing import Any, Dict + +import pytest +from _pytest.capture import CaptureFixture + +from pylint.testutils.configuration_test import ( + get_expected_configuration, + get_expected_output, + run_using_a_configuration_file, +) + +HERE = Path(__file__).parent +FUNCTIONAL_DIR = HERE / "functional" +# We use string then recast to path so we can use -k in pytest otherwise +# we get 'configuration_path0' as a test name. Relative to the functional +# directory because otherwise the string is very lengthy +CONFIGURATION_PATHS = [ + str(path.relative_to(FUNCTIONAL_DIR)) + for ext in ("toml", "ini", "cfg") + for path in FUNCTIONAL_DIR.rglob(f"*.{ext}") +] +FILE_TO_LINT = str(HERE / "file_to_lint.py") + + +@pytest.fixture() +def default_configuration(tmp_path: Path) -> Dict[str, Any]: + empty_pylintrc = tmp_path / "pylintrc" + empty_pylintrc.write_text("") + mock_exit, _, runner = run_using_a_configuration_file( + str(empty_pylintrc), FILE_TO_LINT + ) + mock_exit.assert_called_once_with(0) + return runner.linter.config.__dict__ + + +@pytest.mark.parametrize("configuration_path", CONFIGURATION_PATHS) +def test_functional_config_loading( + configuration_path: str, + default_configuration: Dict[str, Any], + capsys: CaptureFixture, +): + """Functional tests for configurations.""" + configuration_path = str(FUNCTIONAL_DIR / configuration_path) + msg = f"Wrong result with configuration {configuration_path}" + expected_code, expected_output = get_expected_output(configuration_path) + expected_loaded_configuration = get_expected_configuration( + configuration_path, default_configuration + ) + mock_exit, _, runner = run_using_a_configuration_file( + configuration_path, FILE_TO_LINT + ) + mock_exit.assert_called_once_with(expected_code) + out, err = capsys.readouterr() + # rstrip() applied so we can have a final newline in the expected test file + assert expected_output.rstrip() == out.rstrip(), msg + assert expected_loaded_configuration == runner.linter.config.__dict__ + assert not err, msg -- cgit v1.2.1