Add ``--minimal-messages-config`` option for functional tests (#6246)

* Add ``--minimal-messages-config`` option for functional tests

* Add test file option to exclude from --minimal-messages-config run.

* Fix ``too-many-branches`` false negative

* Fix ``access-member-before-definition`` false negative

* Fix ``consider-using-min-builtin`` and ``consider-using-max-builtin`` false negatives

* Fix ``consider-using-with`` false negative

* Exclude ``deprecated-module`` tests from ``--minimal-messages-config`` run.
Have to find out how the list of deprecated modules is built first.

* Update doc/whatsnew/2.14.rst

* Ignore deprecated module false negatives

* Fix ``deprecated-typing-alias`` false negative

* Skip tests for ``fixme`` message, don't know yet how enable/disable affects TokenChecker/RawChecker

* Exclude functional test for various logging checks, as the output string depends on what messages are enabled.

* Fix false negative for ``nan-comparison``

* Fix false negative for ``non-ascii-module-import``

* Fix false negative for ``non-ascii-file-name``

* Skip tests for logging messages

* Fix false negative for ``undefined-variable`` in ``metaclass`` argument

* Exclude test for ``catching-non-exception``. ``exc._all_bases_known`` is ``False`` when enabling just this message.

* Exclude ``ungrouped_imports_suppression``. ``useless-suppression`` is triggered too often in this case.

* Fix false negative for ``use-a-generator``

* Exclude functional tests for ``useless-suppression`` from ``--minimal-messages-config``

* Exclude another deprecation check from ``--minimal-messages-config`` run

* Update tests/testutils/data/m/minimal_messages_config.py

Co-authored-by: Pierre Sassoulas <pierre.sassoulas@gmail.com>

Co-authored-by: Pierre Sassoulas <pierre.sassoulas@gmail.com>

33 files changed, 137 insertions, 11 deletions

diff --git a/ChangeLog b/ChangeLog
index d962bb7b9..70fe39fc6 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -140,6 +140,9 @@ Release date: TBA
 
 * The concept of checker priority has been removed.
 
+* Add a new command line option ``--minimal-messages-config`` for ``pytest``, which disables all
+  irrelevant messages when running the functional tests.
+
 * ``duplicate-argument-name`` now only raises once for each set of duplicated arguments.
 
 * Fix bug where specifically enabling just ``await-outside-async`` was not possible.
diff --git a/pylint/checkers/base/comparison_checker.py b/pylint/checkers/base/comparison_checker.py
index 1db50761b..19000b28e 100644
--- a/pylint/checkers/base/comparison_checker.py
+++ b/pylint/checkers/base/comparison_checker.py
@@ -241,6 +241,7 @@ class ComparisonChecker(_BasicChecker):
         "literal-comparison",
         "comparison-with-itself",
         "comparison-with-callable",
+        "nan-comparison",
     )
     def visit_compare(self, node: nodes.Compare) -> None:
         self._check_callable_comparison(node)
diff --git a/pylint/checkers/classes/class_checker.py b/pylint/checkers/classes/class_checker.py
index 05b198c43..81ff73ff0 100644
--- a/pylint/checkers/classes/class_checker.py
+++ b/pylint/checkers/classes/class_checker.py
@@ -1511,7 +1511,9 @@ a metaclass class method.",
 
         self._check_protected_attribute_access(node)
 
-    @check_messages("assigning-non-slot", "invalid-class-object")
+    @check_messages(
+        "assigning-non-slot", "invalid-class-object", "access-member-before-definition"
+    )
     def visit_assignattr(self, node: nodes.AssignAttr) -> None:
         if isinstance(
             node.assign_type(), nodes.AugAssign
diff --git a/pylint/checkers/design_analysis.py b/pylint/checkers/design_analysis.py
index ea1be4de7..e343a4a6b 100644
--- a/pylint/checkers/design_analysis.py
+++ b/pylint/checkers/design_analysis.py
@@ -609,7 +609,7 @@ class MisdesignChecker(BaseChecker):
         self._inc_branch(node, 2)
         self._inc_all_stmts(2)
 
-    @check_messages("too-many-boolean-expressions")
+    @check_messages("too-many-boolean-expressions", "too-many-branches")
     def visit_if(self, node: nodes.If) -> None:
         """Increments the branches counter and checks boolean expressions."""
         self._check_boolean_expressions(node)
diff --git a/pylint/checkers/non_ascii_names.py b/pylint/checkers/non_ascii_names.py
index d7892dbbc..f6ac3e569 100644
--- a/pylint/checkers/non_ascii_names.py
+++ b/pylint/checkers/non_ascii_names.py
@@ -94,7 +94,7 @@ class NonAsciiNameChecker(base_checker.BaseChecker):
 
             self.add_message(msg, node=node, args=args, confidence=interfaces.HIGH)
 
-    @utils.check_messages("non-ascii-name")
+    @utils.check_messages("non-ascii-name", "non-ascii-file-name")
     def visit_module(self, node: nodes.Module) -> None:
         self._check_name("file", node.name.split(".")[-1], node)
 
@@ -166,11 +166,11 @@ class NonAsciiNameChecker(base_checker.BaseChecker):
             name = alias or module_name
             self._check_name("module", name, node)
 
-    @utils.check_messages("non-ascii-name")
+    @utils.check_messages("non-ascii-name", "non-ascii-module-import")
     def visit_import(self, node: nodes.Import) -> None:
         self._check_module_import(node)
 
-    @utils.check_messages("non-ascii-name")
+    @utils.check_messages("non-ascii-name", "non-ascii-module-import")
     def visit_importfrom(self, node: nodes.ImportFrom) -> None:
         self._check_module_import(node)
 
diff --git a/pylint/checkers/refactoring/refactoring_checker.py b/pylint/checkers/refactoring/refactoring_checker.py
index 530e15243..79ae3ce90 100644
--- a/pylint/checkers/refactoring/refactoring_checker.py
+++ b/pylint/checkers/refactoring/refactoring_checker.py
@@ -661,7 +661,7 @@ class RefactoringChecker(checkers.BaseTokenChecker):
         if node.name and isinstance(node.name, nodes.AssignName):
             self._check_redefined_argument_from_local(node.name)
 
-    @utils.check_messages("redefined-argument-from-local")
+    @utils.check_messages("redefined-argument-from-local", "consider-using-with")
     def visit_with(self, node: nodes.With) -> None:
         for var, names in node.items:
             if isinstance(var, nodes.Name):
@@ -776,6 +776,8 @@ class RefactoringChecker(checkers.BaseTokenChecker):
         "no-else-break",
         "no-else-continue",
         "consider-using-get",
+        "consider-using-min-builtin",
+        "consider-using-max-builtin",
     )
     def visit_if(self, node: nodes.If) -> None:
         self._check_simplifiable_if(node)
@@ -1023,6 +1025,7 @@ class RefactoringChecker(checkers.BaseTokenChecker):
         "consider-using-with",
         "use-list-literal",
         "use-dict-literal",
+        "use-a-generator",
     )
     def visit_call(self, node: nodes.Call) -> None:
         self._check_raising_stopiteration_in_generator_next_call(node)
diff --git a/pylint/checkers/variables.py b/pylint/checkers/variables.py
index 77dd6fabc..a4858ec91 100644
--- a/pylint/checkers/variables.py
+++ b/pylint/checkers/variables.py
@@ -1133,6 +1133,7 @@ class VariablesChecker(BaseChecker):
         "invalid-all-object",
         "invalid-all-format",
         "unused-variable",
+        "undefined-variable",
     )
     def leave_module(self, node: nodes.Module) -> None:
         """Leave module: check globals."""
diff --git a/pylint/extensions/typing.py b/pylint/extensions/typing.py
index 84b4e5962..258f4a4b2 100644
--- a/pylint/extensions/typing.py
+++ b/pylint/extensions/typing.py
@@ -307,7 +307,7 @@ class TypingChecker(BaseChecker):
             )
         )
 
-    @check_messages("consider-using-alias")
+    @check_messages("consider-using-alias", "deprecated-typing-alias")
     def leave_module(self, node: nodes.Module) -> None:
         """After parsing of module is complete, add messages for
         'consider-using-alias' check.
diff --git a/pylint/testutils/functional/test_file.py b/pylint/testutils/functional/test_file.py
index a27f344e8..4fb0f9573 100644
--- a/pylint/testutils/functional/test_file.py
+++ b/pylint/testutils/functional/test_file.py
@@ -32,6 +32,7 @@ class TestFileOptions(TypedDict):
     requires: list[str]
     except_implementations: list[str]
     exclude_platforms: list[str]
+    exclude_from_minimal_messages_config: bool
 
 
 # mypy need something literal, we can't create this dynamically from TestFileOptions
@@ -42,6 +43,7 @@ POSSIBLE_TEST_OPTIONS = {
     "requires",
     "except_implementations",
     "exclude_platforms",
+    "exclude_from_minimal_messages_config",
 }
 
 
@@ -67,6 +69,7 @@ class FunctionalTestFile:
             "requires": [],
             "except_implementations": [],
             "exclude_platforms": [],
+            "exclude_from_minimal_messages_config": False,
         }
         self._parse_options()
 
diff --git a/pylint/testutils/lint_module_test.py b/pylint/testutils/lint_module_test.py
index 6d6639a1a..5ada1f3f0 100644
--- a/pylint/testutils/lint_module_test.py
+++ b/pylint/testutils/lint_module_test.py
@@ -56,19 +56,23 @@ class LintModuleTest:
         except NoFileError:
             pass
 
+        self._test_file = test_file
+        self._check_end_position = (
+            sys.version_info >= self._test_file.options["min_pyver_end_position"]
+        )
         try:
             args = [test_file.source]
         except NoFileError:
             # If we're still raising NoFileError the actual source file doesn't exist
             args = [""]
+        if config and config.getoption("minimal_messages_config"):
+            with self._open_source_file() as f:
+                messages_to_enable = {msg[1] for msg in self.get_expected_messages(f)}
+            args.extend(["--disable=all", f"--enable={','.join(messages_to_enable)}"])
         _config_initialization(
             self._linter, args_list=args, config_file=rc_file, reporter=_test_reporter
         )
-        self._test_file = test_file
         self._config = config
-        self._check_end_position = (
-            sys.version_info >= self._test_file.options["min_pyver_end_position"]
-        )
 
     def setUp(self) -> None:
         if self._should_be_skipped_due_to_version():
@@ -92,6 +96,12 @@ class LintModuleTest:
         if excluded_platforms:
             if sys.platform.lower() in excluded_platforms:
                 pytest.skip(f"Test cannot run on platform {sys.platform!r}")
+        if (
+            self._config
+            and self._config.getoption("minimal_messages_config")
+            and self._test_file.options["exclude_from_minimal_messages_config"]
+        ):
+            pytest.skip("Test excluded from --minimal-messages-config")
 
     def runTest(self) -> None:
         self._runTest()
diff --git a/tests/conftest.py b/tests/conftest.py
index dd0b1195e..71d37d9c5 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -75,6 +75,17 @@ def pytest_addoption(parser) -> None:
         default=False,
         help="Run primer external tests",
     )
+    parser.addoption(
+        "--minimal-messages-config",
+        action="store_true",
+        default=False,
+        help=(
+            "Disable all messages that are not explicitly expected when running functional tests. "
+            "This is useful for finding problems with the @only_required_for_messages / @check_messages "
+            "decorator, but can also produce false negatives if a functional test file only tests for "
+            "false positive of messages and thus does not declare which messages are expected."
+        ),
+    )
 
 
 def pytest_collection_modifyitems(config, items) -> None:
diff --git a/tests/functional/d/deprecated/deprecated_module_py3.rc b/tests/functional/d/deprecated/deprecated_module_py3.rc
index 375b36140..4489289c1 100644
--- a/tests/functional/d/deprecated/deprecated_module_py3.rc
+++ b/tests/functional/d/deprecated/deprecated_module_py3.rc
@@ -1,2 +1,5 @@
 [typing]
 py-version=3.6
+
+[testoptions]
+exclude_from_minimal_messages_config=true
diff --git a/tests/functional/d/deprecated/deprecated_module_py310.rc b/tests/functional/d/deprecated/deprecated_module_py310.rc
index b29fd450a..397f15a46 100644
--- a/tests/functional/d/deprecated/deprecated_module_py310.rc
+++ b/tests/functional/d/deprecated/deprecated_module_py310.rc
@@ -1,2 +1,3 @@
 [testoptions]
 min_pyver = 3.10
+exclude_from_minimal_messages_config=true
diff --git a/tests/functional/d/deprecated/deprecated_module_py33.rc b/tests/functional/d/deprecated/deprecated_module_py33.rc
new file mode 100644
index 000000000..10388685f
--- /dev/null
+++ b/tests/functional/d/deprecated/deprecated_module_py33.rc
@@ -0,0 +1,2 @@
+[testoptions]
+exclude_from_minimal_messages_config=true
diff --git a/tests/functional/d/deprecated/deprecated_module_py36.rc b/tests/functional/d/deprecated/deprecated_module_py36.rc
new file mode 100644
index 000000000..10388685f
--- /dev/null
+++ b/tests/functional/d/deprecated/deprecated_module_py36.rc
@@ -0,0 +1,2 @@
+[testoptions]
+exclude_from_minimal_messages_config=true
diff --git a/tests/functional/d/deprecated/deprecated_module_py39.rc b/tests/functional/d/deprecated/deprecated_module_py39.rc
index 16b75eea7..1782cf624 100644
--- a/tests/functional/d/deprecated/deprecated_module_py39.rc
+++ b/tests/functional/d/deprecated/deprecated_module_py39.rc
@@ -1,2 +1,3 @@
 [testoptions]
 min_pyver=3.9
+exclude_from_minimal_messages_config=true
diff --git a/tests/functional/d/deprecated/deprecated_module_py39_earlier_pyversion.rc b/tests/functional/d/deprecated/deprecated_module_py39_earlier_pyversion.rc
index 8118783a1..115bd6b23 100644
--- a/tests/functional/d/deprecated/deprecated_module_py39_earlier_pyversion.rc
+++ b/tests/functional/d/deprecated/deprecated_module_py39_earlier_pyversion.rc
@@ -3,3 +3,4 @@ py-version=3.8
 
 [testoptions]
 min_pyver=3.9
+exclude_from_minimal_messages_config=true
diff --git a/tests/functional/d/deprecated/deprecated_relative_import/dot_relative_import.rc b/tests/functional/d/deprecated/deprecated_relative_import/dot_relative_import.rc
new file mode 100644
index 000000000..10388685f
--- /dev/null
+++ b/tests/functional/d/deprecated/deprecated_relative_import/dot_relative_import.rc
@@ -0,0 +1,2 @@
+[testoptions]
+exclude_from_minimal_messages_config=true
diff --git a/tests/functional/d/deprecated/deprecated_relative_import/subpackage/dot_dot_relative_import.rc b/tests/functional/d/deprecated/deprecated_relative_import/subpackage/dot_dot_relative_import.rc
new file mode 100644
index 000000000..10388685f
--- /dev/null
+++ b/tests/functional/d/deprecated/deprecated_relative_import/subpackage/dot_dot_relative_import.rc
@@ -0,0 +1,2 @@
+[testoptions]
+exclude_from_minimal_messages_config=true
diff --git a/tests/functional/f/fixme.rc b/tests/functional/f/fixme.rc
index be1b23458..181b05d02 100644
--- a/tests/functional/f/fixme.rc
+++ b/tests/functional/f/fixme.rc
@@ -3,3 +3,6 @@
 notes=XXX,TODO,./TODO
 # Regular expression of note tags to take in consideration.
 notes-rgx=FIXME(?!.*ISSUE-\d+)|TO.*DO
+
+[testoptions]
+exclude_from_minimal_messages_config=true
diff --git a/tests/functional/l/logging_format_interpolation_py36.rc b/tests/functional/l/logging_format_interpolation_py36.rc
new file mode 100644
index 000000000..10388685f
--- /dev/null
+++ b/tests/functional/l/logging_format_interpolation_py36.rc
@@ -0,0 +1,2 @@
+[testoptions]
+exclude_from_minimal_messages_config=true
diff --git a/tests/functional/l/logging_fstring_interpolation_py36.rc b/tests/functional/l/logging_fstring_interpolation_py36.rc
new file mode 100644
index 000000000..10388685f
--- /dev/null
+++ b/tests/functional/l/logging_fstring_interpolation_py36.rc
@@ -0,0 +1,2 @@
+[testoptions]
+exclude_from_minimal_messages_config=true
diff --git a/tests/functional/l/logging_fstring_interpolation_py37.rc b/tests/functional/l/logging_fstring_interpolation_py37.rc
index 484b38f34..7a78f5277 100644
--- a/tests/functional/l/logging_fstring_interpolation_py37.rc
+++ b/tests/functional/l/logging_fstring_interpolation_py37.rc
@@ -1,5 +1,6 @@
 [testoptions]
 min_pyver=3.7
+exclude_from_minimal_messages_config=true
 
 [LOGGING]
 logging-format-style=new
diff --git a/tests/functional/l/logging_not_lazy_module.rc b/tests/functional/l/logging_not_lazy_module.rc
index c241cd06b..747998294 100644
--- a/tests/functional/l/logging_not_lazy_module.rc
+++ b/tests/functional/l/logging_not_lazy_module.rc
@@ -1,2 +1,5 @@
 [LOGGING]
 logging-modules=logging,my.logging
+
+[testoptions]
+exclude_from_minimal_messages_config=true
diff --git a/tests/functional/l/logging_not_lazy_with_logger.rc b/tests/functional/l/logging_not_lazy_with_logger.rc
new file mode 100644
index 000000000..10388685f
--- /dev/null
+++ b/tests/functional/l/logging_not_lazy_with_logger.rc
@@ -0,0 +1,2 @@
+[testoptions]
+exclude_from_minimal_messages_config=true
diff --git a/tests/functional/r/renamed_import_logging_not_lazy.rc b/tests/functional/r/renamed_import_logging_not_lazy.rc
new file mode 100644
index 000000000..10388685f
--- /dev/null
+++ b/tests/functional/r/renamed_import_logging_not_lazy.rc
@@ -0,0 +1,2 @@
+[testoptions]
+exclude_from_minimal_messages_config=true
diff --git a/tests/functional/t/try_except_raise_crash.rc b/tests/functional/t/try_except_raise_crash.rc
index a17bb22da..562828352 100644
--- a/tests/functional/t/try_except_raise_crash.rc
+++ b/tests/functional/t/try_except_raise_crash.rc
@@ -1,2 +1,3 @@
 [testoptions]
 min_pyver=3.7
+exclude_from_minimal_messages_config=true
diff --git a/tests/functional/u/ungrouped_imports_suppression.rc b/tests/functional/u/ungrouped_imports_suppression.rc
new file mode 100644
index 000000000..10388685f
--- /dev/null
+++ b/tests/functional/u/ungrouped_imports_suppression.rc
@@ -0,0 +1,2 @@
+[testoptions]
+exclude_from_minimal_messages_config=true
diff --git a/tests/functional/u/useless/useless_suppression.rc b/tests/functional/u/useless/useless_suppression.rc
new file mode 100644
index 000000000..10388685f
--- /dev/null
+++ b/tests/functional/u/useless/useless_suppression.rc
@@ -0,0 +1,2 @@
+[testoptions]
+exclude_from_minimal_messages_config=true
diff --git a/tests/testutils/data/m/minimal_messages_config.py b/tests/testutils/data/m/minimal_messages_config.py
new file mode 100644
index 000000000..ac0b70ec7
--- /dev/null
+++ b/tests/testutils/data/m/minimal_messages_config.py
@@ -0,0 +1,10 @@
+"""
+Dummy test file to test the --minimal-messages-config option.
+
+This file is tested in ``test_minimal_messages_config_enabled`` and
+``test_minimal_messages_config_excluded_file``.
+"""
+
+f = open("foo.txt")  # [consider-using-with, unspecified-encoding]
+
+print("%d" % 1)  # [consider-using-f-string]
diff --git a/tests/testutils/data/m/minimal_messages_excluded.py b/tests/testutils/data/m/minimal_messages_excluded.py
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/tests/testutils/data/m/minimal_messages_excluded.py
diff --git a/tests/testutils/data/m/minimal_messages_excluded.rc b/tests/testutils/data/m/minimal_messages_excluded.rc
new file mode 100644
index 000000000..10388685f
--- /dev/null
+++ b/tests/testutils/data/m/minimal_messages_excluded.rc
@@ -0,0 +1,2 @@
+[testoptions]
+exclude_from_minimal_messages_config=true
diff --git a/tests/testutils/test_functional_testutils.py b/tests/testutils/test_functional_testutils.py
index f37ffb1ca..1a1e99e2e 100644
--- a/tests/testutils/test_functional_testutils.py
+++ b/tests/testutils/test_functional_testutils.py
@@ -5,8 +5,10 @@
 """Tests for the functional test framework."""
 
 from pathlib import Path
+from unittest.mock import MagicMock
 
 import pytest
+from _pytest.outcomes import Skipped
 
 from pylint import testutils
 from pylint.testutils.functional import (
@@ -18,6 +20,18 @@ HERE = Path(__file__).parent
 DATA_DIRECTORY = HERE / "data"
 
 
+@pytest.fixture(name="pytest_config")
+def pytest_config_fixture() -> MagicMock:
+    def _mock_getoption(option):
+        if option == "minimal_messages_config":
+            return True
+        return False
+
+    config = MagicMock()
+    config.getoption.side_effect = _mock_getoption
+    return config
+
+
 def test_parsing_of_pylintrc_init_hook() -> None:
     """Test that we correctly parse an init-hook in a settings file."""
     with pytest.raises(RuntimeError):
@@ -29,3 +43,35 @@ def test_get_functional_test_files_from_directory() -> None:
     """Test that we correctly check the functional test directory structures."""
     with pytest.raises(AssertionError, match="using_dir.py should not go in"):
         get_functional_test_files_from_directory(DATA_DIRECTORY)
+
+
+def test_minimal_messages_config_enabled(pytest_config) -> None:
+    """Test that all messages not targeted in the functional test are disabled
+    when running with --minimal-messages-config.
+    """
+    test_file = FunctionalTestFile(
+        str(DATA_DIRECTORY / "m"), "minimal_messages_config.py"
+    )
+    mod_test = testutils.LintModuleTest(test_file, pytest_config)
+    assert all(
+        mod_test._linter.is_message_enabled(msgid)
+        for msgid in (
+            "consider-using-with",
+            "unspecified-encoding",
+            "consider-using-f-string",
+        )
+    )
+    assert not mod_test._linter.is_message_enabled("unused-import")
+
+
+def test_minimal_messages_config_excluded_file(pytest_config) -> None:
+    """Test that functional test files can be excluded from the run with
+    --minimal-messages-config if they set the exclude_from_minimal_messages_config
+    option in their rcfile.
+    """
+    test_file = FunctionalTestFile(
+        str(DATA_DIRECTORY / "m"), "minimal_messages_excluded.py"
+    )
+    mod_test = testutils.LintModuleTest(test_file, pytest_config)
+    with pytest.raises(Skipped):
+        mod_test.setUp()