diff options
Diffstat (limited to 'tests')
17 files changed, 132 insertions, 29 deletions
diff --git a/tests/testutils/data/functional/broken_output_ok_test/exec_used.py b/tests/testutils/data/functional/broken_output_ok_test/exec_used.py new file mode 100644 index 000000000..73d629d95 --- /dev/null +++ b/tests/testutils/data/functional/broken_output_ok_test/exec_used.py @@ -0,0 +1,7 @@ +"""This is an example for a functional test with +- an output that is broken +- the functional test itself is right + +So it should be updated. +""" +exec('a = 42') # [exec-used] diff --git a/tests/testutils/data/functional/broken_output_ok_test/exec_used.txt b/tests/testutils/data/functional/broken_output_ok_test/exec_used.txt new file mode 100644 index 000000000..d74b20893 --- /dev/null +++ b/tests/testutils/data/functional/broken_output_ok_test/exec_used.txt @@ -0,0 +1 @@ +exec-used:UNDEFINED diff --git a/tests/testutils/data/functional/broken_output_wrong_test/exec_used.py b/tests/testutils/data/functional/broken_output_wrong_test/exec_used.py new file mode 100644 index 000000000..8b6c06a57 --- /dev/null +++ b/tests/testutils/data/functional/broken_output_wrong_test/exec_used.py @@ -0,0 +1,7 @@ +"""This is an example for a functional test with +- an output that is broken +- the functional test itself is wrong + +So it cannot be updated. +""" +exec('a = 42') diff --git a/tests/testutils/data/functional/broken_output_wrong_test/exec_used.txt b/tests/testutils/data/functional/broken_output_wrong_test/exec_used.txt new file mode 100644 index 000000000..d74b20893 --- /dev/null +++ b/tests/testutils/data/functional/broken_output_wrong_test/exec_used.txt @@ -0,0 +1 @@ +exec-used:UNDEFINED diff --git a/tests/testutils/data/functional/no_output_ok_test/exec_used.py b/tests/testutils/data/functional/no_output_ok_test/exec_used.py new file mode 100644 index 000000000..59d19fec0 --- /dev/null +++ b/tests/testutils/data/functional/no_output_ok_test/exec_used.py @@ -0,0 +1,7 @@ +"""This is an example for a functional test with +- an output that does not exist +- the functional test itself is right + +So it should be created +""" +exec('a = 42') # [exec-used] diff --git a/tests/testutils/data/functional/no_output_wrong_test/exec_used.py b/tests/testutils/data/functional/no_output_wrong_test/exec_used.py new file mode 100644 index 000000000..d3b318ab7 --- /dev/null +++ b/tests/testutils/data/functional/no_output_wrong_test/exec_used.py @@ -0,0 +1,7 @@ +"""This is an example for a functional test with +- an output that does not exist +- the functional test itself is wrong + +So it cannot be created. +""" +exec('a = 42') diff --git a/tests/testutils/data/functional/ok_output_ok_test/exec_used.py b/tests/testutils/data/functional/ok_output_ok_test/exec_used.py new file mode 100644 index 000000000..049213dd4 --- /dev/null +++ b/tests/testutils/data/functional/ok_output_ok_test/exec_used.py @@ -0,0 +1,7 @@ +"""This is an example for a functional test with +- an output that is right +- the functional test itself is right + +Nothing should be done. +""" +exec('a = 42') # [exec-used] diff --git a/tests/testutils/data/functional/ok_output_ok_test/exec_used.txt b/tests/testutils/data/functional/ok_output_ok_test/exec_used.txt new file mode 100644 index 000000000..6d0f82baf --- /dev/null +++ b/tests/testutils/data/functional/ok_output_ok_test/exec_used.txt @@ -0,0 +1 @@ +exec-used:7:0:7:14::Use of exec:UNDEFINED diff --git a/tests/testutils/data/functional/ok_output_wrong_test/exec_used.py b/tests/testutils/data/functional/ok_output_wrong_test/exec_used.py new file mode 100644 index 000000000..3f8637228 --- /dev/null +++ b/tests/testutils/data/functional/ok_output_wrong_test/exec_used.py @@ -0,0 +1,7 @@ +"""This is an example for a functional test with +- an output that is right +- the functional test itself is wrong + +Nothing should be done. +""" +exec('a = 42') diff --git a/tests/testutils/data/functional/ok_output_wrong_test/exec_used.txt b/tests/testutils/data/functional/ok_output_wrong_test/exec_used.txt new file mode 100644 index 000000000..6d0f82baf --- /dev/null +++ b/tests/testutils/data/functional/ok_output_wrong_test/exec_used.txt @@ -0,0 +1 @@ +exec-used:7:0:7:14::Use of exec:UNDEFINED diff --git a/tests/testutils/data/functional/wrong_output_ok_test/exec_used.py b/tests/testutils/data/functional/wrong_output_ok_test/exec_used.py new file mode 100644 index 000000000..0874f0a15 --- /dev/null +++ b/tests/testutils/data/functional/wrong_output_ok_test/exec_used.py @@ -0,0 +1,7 @@ +"""This is an example for a functional test with +- an output that is wrong (but not broken) +- the functional test itself is right + +So it needs to be updated. +""" +exec('a = 42') # [exec-used] diff --git a/tests/testutils/data/functional/wrong_output_ok_test/exec_used.txt b/tests/testutils/data/functional/wrong_output_ok_test/exec_used.txt new file mode 100644 index 000000000..a737b8d51 --- /dev/null +++ b/tests/testutils/data/functional/wrong_output_ok_test/exec_used.txt @@ -0,0 +1 @@ +missing-docstring:5:0:1:1::Missing docstring in file:HIGH diff --git a/tests/testutils/data/functional/wrong_output_wrong_test/exec_used.py b/tests/testutils/data/functional/wrong_output_wrong_test/exec_used.py new file mode 100644 index 000000000..fc35114e8 --- /dev/null +++ b/tests/testutils/data/functional/wrong_output_wrong_test/exec_used.py @@ -0,0 +1,7 @@ +"""This is an example for a functional test with +- an output that is wrong (but not broken) +- the functional test itself is wrong + +So it can't be updated. +""" +exec('a = 42') diff --git a/tests/testutils/data/functional/wrong_output_wrong_test/exec_used.txt b/tests/testutils/data/functional/wrong_output_wrong_test/exec_used.txt new file mode 100644 index 000000000..a737b8d51 --- /dev/null +++ b/tests/testutils/data/functional/wrong_output_wrong_test/exec_used.txt @@ -0,0 +1 @@ +missing-docstring:5:0:1:1::Missing docstring in file:HIGH diff --git a/tests/testutils/test_functional_testutils.py b/tests/testutils/test_functional_testutils.py index 68dad697d..e0055f1b0 100644 --- a/tests/testutils/test_functional_testutils.py +++ b/tests/testutils/test_functional_testutils.py @@ -42,7 +42,7 @@ def test_parsing_of_pylintrc_init_hook() -> None: def test_get_functional_test_files_from_directory() -> None: """Test that we correctly check the functional test directory structures.""" with pytest.raises(AssertionError, match="using_dir.py should not go in"): - get_functional_test_files_from_directory(DATA_DIRECTORY) + get_functional_test_files_from_directory(DATA_DIRECTORY / "u") def test_minimal_messages_config_enabled(pytest_config: MagicMock) -> None: diff --git a/tests/testutils/test_lint_module_output_update.py b/tests/testutils/test_lint_module_output_update.py index 2e387a118..22da7b0f7 100644 --- a/tests/testutils/test_lint_module_output_update.py +++ b/tests/testutils/test_lint_module_output_update.py @@ -6,15 +6,20 @@ from __future__ import annotations +import shutil from collections.abc import Callable from pathlib import Path import pytest -from pylint.constants import PY38_PLUS -from pylint.testutils import FunctionalTestFile +from pylint.constants import IS_PYPY, PY38_PLUS, PY39_PLUS +from pylint.testutils import FunctionalTestFile, LintModuleTest from pylint.testutils.functional import LintModuleOutputUpdate +FIXTURE_DIRECTORY = Path(__file__).parent / "data/functional" +DIRECTORIES = list(FIXTURE_DIRECTORY.iterdir()) +DIRECTORIES_NAMES = [d.name for d in DIRECTORIES] + @pytest.fixture() def lint_module_fixture( @@ -76,4 +81,65 @@ def test_lint_module_output_update_remove_useless_txt( expected_output_file.write_text("", encoding="utf8") filename.write_text("", encoding="utf8") lmou.runTest() - assert not (expected_output_file).exists() + assert not expected_output_file.exists() + + +@pytest.mark.skipif( + not PY38_PLUS or (IS_PYPY and not PY39_PLUS), + reason="Requires accurate 'end_col' value to update output", +) +@pytest.mark.parametrize("directory_path", DIRECTORIES, ids=DIRECTORIES_NAMES) +def test_update_of_functional_output(directory_path: Path, tmp_path: Path) -> None: + """Functional test for the functional tests helper.""" + + def _check_expected_output(ftf: FunctionalTestFile) -> None: + new_output_path = ftf.expected_output + assert Path( + new_output_path + ).exists(), "The expected output file does not exists" + with open(new_output_path, encoding="utf8") as f: + new_output = f.read() + assert ( + new_output == "exec-used:7:0:7:14::Use of exec:UNDEFINED\n" + ), f"The content was wrongly updated in {new_output_path}" + + new_path = tmp_path / directory_path.name + shutil.copytree(directory_path, new_path) + for filename in new_path.iterdir(): + if filename.suffix != ".py": + continue + new_path_str = str(new_path) + ftf = FunctionalTestFile(directory=new_path_str, filename=filename.name) + # Standard functional test helper + lint_module = LintModuleTest(ftf) + # Functional test helper that automatically update the output + lint_module_output_update = LintModuleOutputUpdate(ftf) + if "wrong_test" in new_path_str: + expected = r'Wrong message\(s\) raised for "exec_used.py"' + with pytest.raises(AssertionError, match=expected): + lint_module.runTest() + # When the tests are wrong we do not update the output at all + # and the test should fail + with pytest.raises(AssertionError, match=expected): + lint_module_output_update.runTest() + elif "ok_test" in new_path_str: + if any(f"{x}_output" in new_path_str for x in ("wrong", "no", "broken")): + with pytest.raises( + AssertionError, match='Wrong output for "exec_used.txt"' + ): + lint_module.runTest() + elif "ok_output" in new_path_str: + lint_module.runTest() + _check_expected_output(ftf) + else: + raise AssertionError(f"Unhandled test case: {new_path_str}") + + # When the tests are ok we update the output whatever it's state + # was originally + lint_module_output_update.runTest() + _check_expected_output(ftf) + else: + raise AssertionError( + f"Do not pollute '{FIXTURE_DIRECTORY}' with unrelated " + f"or badly named test files." + ) diff --git a/tests/testutils/test_output_line.py b/tests/testutils/test_output_line.py index 5b2bf1a1b..a7e677157 100644 --- a/tests/testutils/test_output_line.py +++ b/tests/testutils/test_output_line.py @@ -134,31 +134,6 @@ def test_output_line_to_csv(confidence: Confidence, message: _MessageCallable) - ) -def test_output_line_from_csv_error() -> None: - """Test that errors are correctly raised for incorrect OutputLine's.""" - # Test a csv-string which does not have a number for line and column - with pytest.warns( - UserWarning, - match="msg-symbolic-name:42:27:MyClass.my_function:The message", - ): - OutputLine.from_csv("'missing-docstring', 'line', 'column', 'obj', 'msg'", True) - # Test a tuple which does not have a number for line and column - with pytest.warns( - UserWarning, match="we got 'missing-docstring:line:column:obj:msg'" - ): - csv = ("missing-docstring", "line", "column", "obj", "msg") - OutputLine.from_csv(csv, True) - # Test a csv-string that is too long - with pytest.warns( - UserWarning, - match="msg-symbolic-name:42:27:MyClass.my_function:The message", - ): - OutputLine.from_csv( - "'missing-docstring', 1, 2, 'obj', 'msg', 'func', 'message', 'conf', 'too_long'", - True, - ) - - @pytest.mark.parametrize( "confidence,expected_confidence", [[None, "UNDEFINED"], ["INFERENCE", "INFERENCE"]] ) |