Add ``--minimal-messages-config`` option for functional tests (#6246)

* Add ``--minimal-messages-config`` option for functional tests

* Add test file option to exclude from --minimal-messages-config run.

* Fix ``too-many-branches`` false negative

* Fix ``access-member-before-definition`` false negative

* Fix ``consider-using-min-builtin`` and ``consider-using-max-builtin`` false negatives

* Fix ``consider-using-with`` false negative

* Exclude ``deprecated-module`` tests from ``--minimal-messages-config`` run.
Have to find out how the list of deprecated modules is built first.

* Update doc/whatsnew/2.14.rst

* Ignore deprecated module false negatives

* Fix ``deprecated-typing-alias`` false negative

* Skip tests for ``fixme`` message, don't know yet how enable/disable affects TokenChecker/RawChecker

* Exclude functional test for various logging checks, as the output string depends on what messages are enabled.

* Fix false negative for ``nan-comparison``

* Fix false negative for ``non-ascii-module-import``

* Fix false negative for ``non-ascii-file-name``

* Skip tests for logging messages

* Fix false negative for ``undefined-variable`` in ``metaclass`` argument

* Exclude test for ``catching-non-exception``. ``exc._all_bases_known`` is ``False`` when enabling just this message.

* Exclude ``ungrouped_imports_suppression``. ``useless-suppression`` is triggered too often in this case.

* Fix false negative for ``use-a-generator``

* Exclude functional tests for ``useless-suppression`` from ``--minimal-messages-config``

* Exclude another deprecation check from ``--minimal-messages-config`` run

* Update tests/testutils/data/m/minimal_messages_config.py

Co-authored-by: Pierre Sassoulas <pierre.sassoulas@gmail.com>

Co-authored-by: Pierre Sassoulas <pierre.sassoulas@gmail.com>

23 files changed, 103 insertions, 0 deletions

diff --git a/tests/conftest.py b/tests/conftest.py
index dd0b1195e..71d37d9c5 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -75,6 +75,17 @@ def pytest_addoption(parser) -> None:
         default=False,
         help="Run primer external tests",
     )
+    parser.addoption(
+        "--minimal-messages-config",
+        action="store_true",
+        default=False,
+        help=(
+            "Disable all messages that are not explicitly expected when running functional tests. "
+            "This is useful for finding problems with the @only_required_for_messages / @check_messages "
+            "decorator, but can also produce false negatives if a functional test file only tests for "
+            "false positive of messages and thus does not declare which messages are expected."
+        ),
+    )
 
 
 def pytest_collection_modifyitems(config, items) -> None:
diff --git a/tests/functional/d/deprecated/deprecated_module_py3.rc b/tests/functional/d/deprecated/deprecated_module_py3.rc
index 375b36140..4489289c1 100644
--- a/tests/functional/d/deprecated/deprecated_module_py3.rc
+++ b/tests/functional/d/deprecated/deprecated_module_py3.rc
@@ -1,2 +1,5 @@
 [typing]
 py-version=3.6
+
+[testoptions]
+exclude_from_minimal_messages_config=true
diff --git a/tests/functional/d/deprecated/deprecated_module_py310.rc b/tests/functional/d/deprecated/deprecated_module_py310.rc
index b29fd450a..397f15a46 100644
--- a/tests/functional/d/deprecated/deprecated_module_py310.rc
+++ b/tests/functional/d/deprecated/deprecated_module_py310.rc
@@ -1,2 +1,3 @@
 [testoptions]
 min_pyver = 3.10
+exclude_from_minimal_messages_config=true
diff --git a/tests/functional/d/deprecated/deprecated_module_py33.rc b/tests/functional/d/deprecated/deprecated_module_py33.rc
new file mode 100644
index 000000000..10388685f
--- /dev/null
+++ b/tests/functional/d/deprecated/deprecated_module_py33.rc
@@ -0,0 +1,2 @@
+[testoptions]
+exclude_from_minimal_messages_config=true
diff --git a/tests/functional/d/deprecated/deprecated_module_py36.rc b/tests/functional/d/deprecated/deprecated_module_py36.rc
new file mode 100644
index 000000000..10388685f
--- /dev/null
+++ b/tests/functional/d/deprecated/deprecated_module_py36.rc
@@ -0,0 +1,2 @@
+[testoptions]
+exclude_from_minimal_messages_config=true
diff --git a/tests/functional/d/deprecated/deprecated_module_py39.rc b/tests/functional/d/deprecated/deprecated_module_py39.rc
index 16b75eea7..1782cf624 100644
--- a/tests/functional/d/deprecated/deprecated_module_py39.rc
+++ b/tests/functional/d/deprecated/deprecated_module_py39.rc
@@ -1,2 +1,3 @@
 [testoptions]
 min_pyver=3.9
+exclude_from_minimal_messages_config=true
diff --git a/tests/functional/d/deprecated/deprecated_module_py39_earlier_pyversion.rc b/tests/functional/d/deprecated/deprecated_module_py39_earlier_pyversion.rc
index 8118783a1..115bd6b23 100644
--- a/tests/functional/d/deprecated/deprecated_module_py39_earlier_pyversion.rc
+++ b/tests/functional/d/deprecated/deprecated_module_py39_earlier_pyversion.rc
@@ -3,3 +3,4 @@ py-version=3.8
 
 [testoptions]
 min_pyver=3.9
+exclude_from_minimal_messages_config=true
diff --git a/tests/functional/d/deprecated/deprecated_relative_import/dot_relative_import.rc b/tests/functional/d/deprecated/deprecated_relative_import/dot_relative_import.rc
new file mode 100644
index 000000000..10388685f
--- /dev/null
+++ b/tests/functional/d/deprecated/deprecated_relative_import/dot_relative_import.rc
@@ -0,0 +1,2 @@
+[testoptions]
+exclude_from_minimal_messages_config=true
diff --git a/tests/functional/d/deprecated/deprecated_relative_import/subpackage/dot_dot_relative_import.rc b/tests/functional/d/deprecated/deprecated_relative_import/subpackage/dot_dot_relative_import.rc
new file mode 100644
index 000000000..10388685f
--- /dev/null
+++ b/tests/functional/d/deprecated/deprecated_relative_import/subpackage/dot_dot_relative_import.rc
@@ -0,0 +1,2 @@
+[testoptions]
+exclude_from_minimal_messages_config=true
diff --git a/tests/functional/f/fixme.rc b/tests/functional/f/fixme.rc
index be1b23458..181b05d02 100644
--- a/tests/functional/f/fixme.rc
+++ b/tests/functional/f/fixme.rc
@@ -3,3 +3,6 @@
 notes=XXX,TODO,./TODO
 # Regular expression of note tags to take in consideration.
 notes-rgx=FIXME(?!.*ISSUE-\d+)|TO.*DO
+
+[testoptions]
+exclude_from_minimal_messages_config=true
diff --git a/tests/functional/l/logging_format_interpolation_py36.rc b/tests/functional/l/logging_format_interpolation_py36.rc
new file mode 100644
index 000000000..10388685f
--- /dev/null
+++ b/tests/functional/l/logging_format_interpolation_py36.rc
@@ -0,0 +1,2 @@
+[testoptions]
+exclude_from_minimal_messages_config=true
diff --git a/tests/functional/l/logging_fstring_interpolation_py36.rc b/tests/functional/l/logging_fstring_interpolation_py36.rc
new file mode 100644
index 000000000..10388685f
--- /dev/null
+++ b/tests/functional/l/logging_fstring_interpolation_py36.rc
@@ -0,0 +1,2 @@
+[testoptions]
+exclude_from_minimal_messages_config=true
diff --git a/tests/functional/l/logging_fstring_interpolation_py37.rc b/tests/functional/l/logging_fstring_interpolation_py37.rc
index 484b38f34..7a78f5277 100644
--- a/tests/functional/l/logging_fstring_interpolation_py37.rc
+++ b/tests/functional/l/logging_fstring_interpolation_py37.rc
@@ -1,5 +1,6 @@
 [testoptions]
 min_pyver=3.7
+exclude_from_minimal_messages_config=true
 
 [LOGGING]
 logging-format-style=new
diff --git a/tests/functional/l/logging_not_lazy_module.rc b/tests/functional/l/logging_not_lazy_module.rc
index c241cd06b..747998294 100644
--- a/tests/functional/l/logging_not_lazy_module.rc
+++ b/tests/functional/l/logging_not_lazy_module.rc
@@ -1,2 +1,5 @@
 [LOGGING]
 logging-modules=logging,my.logging
+
+[testoptions]
+exclude_from_minimal_messages_config=true
diff --git a/tests/functional/l/logging_not_lazy_with_logger.rc b/tests/functional/l/logging_not_lazy_with_logger.rc
new file mode 100644
index 000000000..10388685f
--- /dev/null
+++ b/tests/functional/l/logging_not_lazy_with_logger.rc
@@ -0,0 +1,2 @@
+[testoptions]
+exclude_from_minimal_messages_config=true
diff --git a/tests/functional/r/renamed_import_logging_not_lazy.rc b/tests/functional/r/renamed_import_logging_not_lazy.rc
new file mode 100644
index 000000000..10388685f
--- /dev/null
+++ b/tests/functional/r/renamed_import_logging_not_lazy.rc
@@ -0,0 +1,2 @@
+[testoptions]
+exclude_from_minimal_messages_config=true
diff --git a/tests/functional/t/try_except_raise_crash.rc b/tests/functional/t/try_except_raise_crash.rc
index a17bb22da..562828352 100644
--- a/tests/functional/t/try_except_raise_crash.rc
+++ b/tests/functional/t/try_except_raise_crash.rc
@@ -1,2 +1,3 @@
 [testoptions]
 min_pyver=3.7
+exclude_from_minimal_messages_config=true
diff --git a/tests/functional/u/ungrouped_imports_suppression.rc b/tests/functional/u/ungrouped_imports_suppression.rc
new file mode 100644
index 000000000..10388685f
--- /dev/null
+++ b/tests/functional/u/ungrouped_imports_suppression.rc
@@ -0,0 +1,2 @@
+[testoptions]
+exclude_from_minimal_messages_config=true
diff --git a/tests/functional/u/useless/useless_suppression.rc b/tests/functional/u/useless/useless_suppression.rc
new file mode 100644
index 000000000..10388685f
--- /dev/null
+++ b/tests/functional/u/useless/useless_suppression.rc
@@ -0,0 +1,2 @@
+[testoptions]
+exclude_from_minimal_messages_config=true
diff --git a/tests/testutils/data/m/minimal_messages_config.py b/tests/testutils/data/m/minimal_messages_config.py
new file mode 100644
index 000000000..ac0b70ec7
--- /dev/null
+++ b/tests/testutils/data/m/minimal_messages_config.py
@@ -0,0 +1,10 @@
+"""
+Dummy test file to test the --minimal-messages-config option.
+
+This file is tested in ``test_minimal_messages_config_enabled`` and
+``test_minimal_messages_config_excluded_file``.
+"""
+
+f = open("foo.txt")  # [consider-using-with, unspecified-encoding]
+
+print("%d" % 1)  # [consider-using-f-string]
diff --git a/tests/testutils/data/m/minimal_messages_excluded.py b/tests/testutils/data/m/minimal_messages_excluded.py
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/tests/testutils/data/m/minimal_messages_excluded.py
diff --git a/tests/testutils/data/m/minimal_messages_excluded.rc b/tests/testutils/data/m/minimal_messages_excluded.rc
new file mode 100644
index 000000000..10388685f
--- /dev/null
+++ b/tests/testutils/data/m/minimal_messages_excluded.rc
@@ -0,0 +1,2 @@
+[testoptions]
+exclude_from_minimal_messages_config=true
diff --git a/tests/testutils/test_functional_testutils.py b/tests/testutils/test_functional_testutils.py
index f37ffb1ca..1a1e99e2e 100644
--- a/tests/testutils/test_functional_testutils.py
+++ b/tests/testutils/test_functional_testutils.py
@@ -5,8 +5,10 @@
 """Tests for the functional test framework."""
 
 from pathlib import Path
+from unittest.mock import MagicMock
 
 import pytest
+from _pytest.outcomes import Skipped
 
 from pylint import testutils
 from pylint.testutils.functional import (
@@ -18,6 +20,18 @@ HERE = Path(__file__).parent
 DATA_DIRECTORY = HERE / "data"
 
 
+@pytest.fixture(name="pytest_config")
+def pytest_config_fixture() -> MagicMock:
+    def _mock_getoption(option):
+        if option == "minimal_messages_config":
+            return True
+        return False
+
+    config = MagicMock()
+    config.getoption.side_effect = _mock_getoption
+    return config
+
+
 def test_parsing_of_pylintrc_init_hook() -> None:
     """Test that we correctly parse an init-hook in a settings file."""
     with pytest.raises(RuntimeError):
@@ -29,3 +43,35 @@ def test_get_functional_test_files_from_directory() -> None:
     """Test that we correctly check the functional test directory structures."""
     with pytest.raises(AssertionError, match="using_dir.py should not go in"):
         get_functional_test_files_from_directory(DATA_DIRECTORY)
+
+
+def test_minimal_messages_config_enabled(pytest_config) -> None:
+    """Test that all messages not targeted in the functional test are disabled
+    when running with --minimal-messages-config.
+    """
+    test_file = FunctionalTestFile(
+        str(DATA_DIRECTORY / "m"), "minimal_messages_config.py"
+    )
+    mod_test = testutils.LintModuleTest(test_file, pytest_config)
+    assert all(
+        mod_test._linter.is_message_enabled(msgid)
+        for msgid in (
+            "consider-using-with",
+            "unspecified-encoding",
+            "consider-using-f-string",
+        )
+    )
+    assert not mod_test._linter.is_message_enabled("unused-import")
+
+
+def test_minimal_messages_config_excluded_file(pytest_config) -> None:
+    """Test that functional test files can be excluded from the run with
+    --minimal-messages-config if they set the exclude_from_minimal_messages_config
+    option in their rcfile.
+    """
+    test_file = FunctionalTestFile(
+        str(DATA_DIRECTORY / "m"), "minimal_messages_excluded.py"
+    )
+    mod_test = testutils.LintModuleTest(test_file, pytest_config)
+    with pytest.raises(Skipped):
+        mod_test.setUp()