Use an empty pylintrc for tests using project's pylintrc implicitely

We don't want to use the project's  pylintrc during tests, because
it means that a change in our config could break tests.
But we want to see if the changes to the default break tests.

Create a private '_Run' class in pylint.testutil._run

Co-authored-by: Daniël van Noord <13665637+DanielNoord@users.noreply.github.com>

14 files changed, 121 insertions, 76 deletions

diff --git a/pylint/testutils/_run.py b/pylint/testutils/_run.py
new file mode 100644
index 000000000..0ad68868f
--- /dev/null
+++ b/pylint/testutils/_run.py
@@ -0,0 +1,45 @@
+# Licensed under the GPL: https://www.gnu.org/licenses/old-licenses/gpl-2.0.html
+# For details: https://github.com/PyCQA/pylint/blob/main/LICENSE
+# Copyright (c) https://github.com/PyCQA/pylint/blob/main/CONTRIBUTORS.txt
+
+"""Classes and functions used to mimic normal pylint runs.
+
+This module is considered private and can change at any time.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+from typing import Any
+
+from pylint.lint import Run as LintRun
+from pylint.lint.run import UNUSED_PARAM_SENTINEL
+from pylint.reporters.base_reporter import BaseReporter
+from pylint.testutils.lint_module_test import PYLINTRC
+
+
+def _add_rcfile_default_pylintrc(args: list[str]) -> list[str]:
+    """Add a default pylintrc with the rcfile option in a list of pylint args."""
+    if not any("--rcfile" in arg for arg in args):
+        args.insert(0, f"--rcfile={PYLINTRC}")
+    return args
+
+
+class _Run(LintRun):
+
+    """Like Run, but we're using an explicitly set empty pylintrc.
+
+    We don't want to use the project's pylintrc during tests, because
+    it means that a change in our config could break tests.
+    But we want to see if the changes to the default break tests.
+    """
+
+    def __init__(
+        self,
+        args: Sequence[str],
+        reporter: BaseReporter | None = None,
+        exit: bool = True,  # pylint: disable=redefined-builtin
+        do_exit: Any = UNUSED_PARAM_SENTINEL,
+    ) -> None:
+        args = _add_rcfile_default_pylintrc(list(args))
+        super().__init__(args, reporter, exit, do_exit)
diff --git a/tests/benchmark/test_baseline_benchmarks.py b/tests/benchmark/test_baseline_benchmarks.py
index 5431fcf47..e96d58bbe 100644
--- a/tests/benchmark/test_baseline_benchmarks.py
+++ b/tests/benchmark/test_baseline_benchmarks.py
@@ -15,8 +15,9 @@ import pytest
 from astroid import nodes
 
 from pylint.checkers import BaseRawFileChecker
-from pylint.lint import PyLinter, Run, check_parallel
+from pylint.lint import PyLinter, check_parallel
 from pylint.testutils import GenericTestReporter as Reporter
+from pylint.testutils._run import _Run as Run
 from pylint.typing import FileItem
 from pylint.utils import register_plugins
 
@@ -302,14 +303,9 @@ class TestEstablishBaselineBenchmarks:
         ), f"Expected no errors to be thrown: {pprint.pformat(linter.reporter.messages)}"
 
     def test_baseline_benchmark_j1_all_checks_single_file(self, benchmark):
-        """Runs a single file, with -j1, against all plug-ins.
-
-        ... that's the intent at least.
-        """
-        # Just 1 file, but all Checkers/Extensions
-        fileinfos = [self.empty_filepath]
-
-        runner = benchmark(Run, fileinfos, reporter=Reporter(), exit=False)
+        """Runs a single file, with -j1, against all checkers/Extensions."""
+        args = [self.empty_filepath, "--enable=all", "--enable-all-extensions"]
+        runner = benchmark(Run, args, reporter=Reporter(), exit=False)
         assert runner.linter.config.jobs == 1
         print("len(runner.linter._checkers)", len(runner.linter._checkers))
         assert len(runner.linter._checkers) > 1, "Should have more than 'master'"
diff --git a/tests/checkers/unittest_refactoring.py b/tests/checkers/unittest_refactoring.py
index 37a40a472..d40e0eb92 100644
--- a/tests/checkers/unittest_refactoring.py
+++ b/tests/checkers/unittest_refactoring.py
@@ -6,8 +6,8 @@ import os
 
 import pytest
 
-from pylint.lint import Run
 from pylint.reporters.text import TextReporter
+from pylint.testutils._run import _Run as Run
 
 PARENT_DIR = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
 REGR_DATA = os.path.join(PARENT_DIR, "regrtest_data")
@@ -16,7 +16,10 @@ REGR_DATA = os.path.join(PARENT_DIR, "regrtest_data")
 @pytest.mark.timeout(8)
 def test_process_tokens() -> None:
     with pytest.raises(SystemExit) as cm:
-        Run([os.path.join(REGR_DATA, "very_long_line.py")], reporter=TextReporter())
+        Run(
+            [os.path.join(REGR_DATA, "very_long_line.py"), "--disable=C"],
+            reporter=TextReporter(),
+        )
     assert cm.value.code == 0
 
 
@@ -28,6 +31,7 @@ def test_issue_5724() -> None:
             [
                 os.path.join(REGR_DATA, "issue_5724.py"),
                 "--enable=missing-final-newline",
+                "--disable=C",
             ],
             reporter=TextReporter(),
         )
diff --git a/tests/config/test_argparse_config.py b/tests/config/test_argparse_config.py
index 901bb5470..3bad5e8fa 100644
--- a/tests/config/test_argparse_config.py
+++ b/tests/config/test_argparse_config.py
@@ -11,7 +11,7 @@ import pytest
 
 from pylint.config.arguments_manager import _ArgumentsManager
 from pylint.config.exceptions import UnrecognizedArgumentAction
-from pylint.lint import Run
+from pylint.testutils._run import _Run as Run
 
 HERE = abspath(dirname(__file__))
 REGRTEST_DATA_DIR = join(HERE, "..", "regrtest_data")
@@ -64,7 +64,7 @@ class TestDeprecationOptions:
         run = Run([EMPTY_MODULE, "--ignore=test,test_two"], exit=False)
         assert run.linter.config.ignore == ["test", "test_two"]
         assert run.linter.config.ignore == run.linter.config.black_list
-        assert run.linter.config.ignore_patterns == [re.compile("^\\.#")]
+        assert run.linter.config.ignore_patterns == (re.compile("^\\.#"),)
         assert run.linter.config.ignore_patterns == run.linter.config.black_list_re
 
 
diff --git a/tests/config/test_config.py b/tests/config/test_config.py
index 712d291ed..fefdb220d 100644
--- a/tests/config/test_config.py
+++ b/tests/config/test_config.py
@@ -11,7 +11,8 @@ import pytest
 from pytest import CaptureFixture
 
 from pylint.config.exceptions import _UnrecognizedOptionError
-from pylint.lint import Run
+from pylint.lint import Run as LintRun
+from pylint.testutils._run import _Run as Run
 from pylint.testutils.configuration_test import run_using_a_configuration_file
 
 HERE = Path(__file__).parent.absolute()
@@ -20,7 +21,7 @@ EMPTY_MODULE = REGRTEST_DATA_DIR / "empty.py"
 
 
 def check_configuration_file_reader(
-    runner: Run,
+    runner: LintRun,
     expected_disabled: set[str] | None = None,
     expected_jobs: int = 10,
     expected_reports_truthey: bool = True,
diff --git a/tests/conftest.py b/tests/conftest.py
index 829b9bacc..7a47cf8ad 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -16,10 +16,12 @@ from pylint.lint import PyLinter
 from pylint.lint.run import _cpu_count
 from pylint.testutils import MinimalTestReporter
 
+HERE = Path(__file__).parent
+
 
 @pytest.fixture()
-def tests_directory():
-    return Path(__file__).parent
+def tests_directory() -> Path:
+    return HERE
 
 
 @pytest.fixture
diff --git a/tests/data/empty_pylintrc b/tests/data/empty_pylintrc
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/tests/data/empty_pylintrc
diff --git a/tests/lint/test_utils.py b/tests/lint/test_utils.py
index 891ee0020..6cc79f18b 100644
--- a/tests/lint/test_utils.py
+++ b/tests/lint/test_utils.py
@@ -7,8 +7,8 @@ from pathlib import Path, PosixPath
 
 import pytest
 
-from pylint.lint import Run
 from pylint.lint.utils import get_fatal_error_message, prepare_crash_report
+from pylint.testutils._run import _Run as Run
 
 
 def test_prepare_crash_report(tmp_path: PosixPath) -> None:
diff --git a/tests/lint/unittest_lint.py b/tests/lint/unittest_lint.py
index 039a13f48..0fb17062e 100644
--- a/tests/lint/unittest_lint.py
+++ b/tests/lint/unittest_lint.py
@@ -32,10 +32,11 @@ from pylint.constants import (
     OLD_DEFAULT_PYLINT_HOME,
 )
 from pylint.exceptions import InvalidMessageError
-from pylint.lint import PyLinter, Run
+from pylint.lint import PyLinter
 from pylint.message import Message
 from pylint.reporters import text
 from pylint.testutils import create_files
+from pylint.testutils._run import _Run as Run
 from pylint.typing import MessageLocationTuple
 from pylint.utils import FileState, print_full_documentation, tokenize_module
 
diff --git a/tests/message/unittest_message_id_store.py b/tests/message/unittest_message_id_store.py
index 69ff5f50b..530086f41 100644
--- a/tests/message/unittest_message_id_store.py
+++ b/tests/message/unittest_message_id_store.py
@@ -9,10 +9,10 @@ from pathlib import Path
 
 import pytest
 
-from pylint import lint
 from pylint.exceptions import InvalidMessageError, UnknownMessageError
 from pylint.message.message_definition import MessageDefinition
 from pylint.message.message_id_store import MessageIdStore
+from pylint.testutils._run import _Run as Run
 
 EMPTY_FILE = str(Path(__file__).parent.parent.resolve() / "regrtest_data" / "empty.py")
 
@@ -106,13 +106,9 @@ def test_exclusivity_of_msgids() -> None:
         "is unique for each checker. You can use 'script/get_unused_message_id_category.py' "
         "to get an unique id."
     )
+    runner = Run(["--enable-all-extensions", EMPTY_FILE], exit=False)
 
-    runner = lint.Run(
-        ["--enable-all-extensions", EMPTY_FILE],
-        exit=False,
-    )
-
-    # Some pairs are hard-coded as they are pre-existing and non-exclusive
+    # Some pairs are hard-coded as they are pre-existing and non-exclusive,
     # and we don't want to rename them for backwards compatibility
     checker_id_pairs = {
         "00": ("master", "miscellaneous"),
diff --git a/tests/primer/test_primer_stdlib.py b/tests/primer/test_primer_stdlib.py
index 59599ae00..6cae6fd36 100644
--- a/tests/primer/test_primer_stdlib.py
+++ b/tests/primer/test_primer_stdlib.py
@@ -10,7 +10,7 @@ import sys
 import pytest
 from pytest import CaptureFixture
 
-import pylint.lint
+from pylint.testutils._run import _Run as Run
 
 
 def is_module(filename: str) -> bool:
@@ -57,7 +57,7 @@ def test_primer_stdlib_no_crash(
             # Duplicate code takes too long and is relatively safe
             # We don't want to lint the test directory which are repetitive
             disables = ["--disable=duplicate-code", "--ignore=test"]
-            pylint.lint.Run([test_module_name] + enables + disables)
+            Run([test_module_name] + enables + disables)
         except SystemExit as ex:
             out, err = capsys.readouterr()
             assert not err, err
diff --git a/tests/profile/test_profile_against_externals.py b/tests/profile/test_profile_against_externals.py
index 81819476e..33b25efbc 100644
--- a/tests/profile/test_profile_against_externals.py
+++ b/tests/profile/test_profile_against_externals.py
@@ -11,8 +11,8 @@ import pprint
 
 import pytest
 
-from pylint.lint import Run
 from pylint.testutils import GenericTestReporter as Reporter
+from pylint.testutils._run import _Run as Run
 
 
 def _get_py_files(scanpath):
diff --git a/tests/test_self.py b/tests/test_self.py
index 70963fe04..233a7d3c1 100644
--- a/tests/test_self.py
+++ b/tests/test_self.py
@@ -30,11 +30,12 @@ from py._path.local import LocalPath  # type: ignore[import]
 
 from pylint import extensions, modify_sys_path
 from pylint.constants import MAIN_CHECKER_NAME, MSG_TYPES_STATUS
-from pylint.lint import Run
 from pylint.lint.pylinter import PyLinter
 from pylint.message import Message
 from pylint.reporters import JSONReporter
 from pylint.reporters.text import BaseReporter, ColorizedTextReporter, TextReporter
+from pylint.testutils._run import _add_rcfile_default_pylintrc
+from pylint.testutils._run import _Run as Run
 from pylint.testutils.utils import _patch_streams
 from pylint.utils import utils
 
@@ -130,6 +131,7 @@ class TestRunTC:
     ) -> None:
         if out is None:
             out = StringIO()
+        args = _add_rcfile_default_pylintrc(args)
         pylint_code = self._run_pylint(args, reporter=reporter, out=out)
         if reporter:
             output = reporter.out.getvalue()
@@ -144,7 +146,7 @@ class TestRunTC:
 
     @staticmethod
     def _run_pylint(args: list[str], out: TextIO, reporter: Any = None) -> int:
-        args = args + ["--persistent=no"]
+        args = _add_rcfile_default_pylintrc(args + ["--persistent=no"])
         with _patch_streams(out):
             with pytest.raises(SystemExit) as cm:
                 with warnings.catch_warnings():
@@ -160,6 +162,7 @@ class TestRunTC:
 
     def _test_output(self, args: list[str], expected_output: str) -> None:
         out = StringIO()
+        args = _add_rcfile_default_pylintrc(args)
         self._run_pylint(args, out=out)
         actual_output = self._clean_paths(out.getvalue())
         expected_output = self._clean_paths(expected_output)
@@ -172,6 +175,7 @@ class TestRunTC:
         the ``args`` passed to this method!) and check the file content afterwards.
         """
         out = StringIO()
+        args = _add_rcfile_default_pylintrc(args)
         self._run_pylint(args, out=out)
         cmdline_output = out.getvalue()
         file_output = self._clean_paths(Path(filename).read_text(encoding="utf-8"))
@@ -183,6 +187,7 @@ class TestRunTC:
 
     def test_pkginfo(self) -> None:
         """Make pylint check 'pylint.__pkginfo__.py'."""
+        # Disable invalid-name because of invalid argument names
         args = ["pylint.__pkginfo__", "--disable=invalid-name"]
         self._runtest(args, reporter=TextReporter(StringIO()), code=0)
 
@@ -1091,13 +1096,13 @@ class TestRunTC:
         path = join(HERE, "regrtest_data", "fail_on.py")
         # We set fail-under to be something very low so that even with the warnings
         # and errors that are generated they don't affect the exit code.
-        self._runtest([path, "--fail-under=-10"] + args, code=expected)
+        self._runtest([path, "--fail-under=-10", "--disable=C"] + args, code=expected)
 
     def test_one_module_fatal_error(self):
         """Fatal errors in one of several modules linted still exits non-zero."""
         valid_path = join(HERE, "conftest.py")
         invalid_path = join(HERE, "garbagePath.py")
-        self._runtest([valid_path, invalid_path], code=1)
+        self._runtest([valid_path, invalid_path, "--disable=C"], code=1)
 
     @pytest.mark.parametrize(
         "args, expected",
@@ -1197,7 +1202,10 @@ class TestRunTC:
         Reported in https://github.com/PyCQA/pylint/issues/5437
         """
         with pytest.raises(SystemExit) as ex:
-            Run(["--ignore-paths", "test", join(HERE, "regrtest_data", "empty.py")])
+            args = _add_rcfile_default_pylintrc(
+                ["--ignore-paths", "test", join(HERE, "regrtest_data", "empty.py")]
+            )
+            Run(args)
         assert ex.value.code == 0
 
     @staticmethod
@@ -1284,7 +1292,7 @@ class TestCallbackOptions:
     )
     def test_output_of_callback_options(command: list[str], expected: str) -> None:
         """Test whether certain strings are in the output of a callback command."""
-
+        command = _add_rcfile_default_pylintrc(command)
         process = subprocess.run(
             [sys.executable, "-m", "pylint"] + command,
             capture_output=True,
@@ -1294,47 +1302,44 @@ class TestCallbackOptions:
         assert expected in process.stdout
 
     @staticmethod
-    def test_help_msg() -> None:
+    @pytest.mark.parametrize(
+        "args,expected,error",
+        [
+            [["--help-msg", "W0101"], ":unreachable (W0101)", False],
+            [["--help-msg", "WX101"], "No such message id", False],
+            [["--help-msg"], "--help-msg: expected at least one argumen", True],
+        ],
+    )
+    def test_help_msg(args: list[str], expected: str, error: bool) -> None:
         """Test the --help-msg flag."""
-
-        process = subprocess.run(
-            [sys.executable, "-m", "pylint", "--help-msg", "W0101"],
-            capture_output=True,
-            encoding="utf-8",
-            check=False,
-        )
-        assert ":unreachable (W0101)" in process.stdout
-
-        process = subprocess.run(
-            [sys.executable, "-m", "pylint", "--help-msg", "WX101"],
-            capture_output=True,
-            encoding="utf-8",
-            check=False,
-        )
-        assert "No such message id" in process.stdout
-
+        args = _add_rcfile_default_pylintrc(args)
         process = subprocess.run(
-            [sys.executable, "-m", "pylint", "--help-msg"],
+            [sys.executable, "-m", "pylint"] + args,
             capture_output=True,
             encoding="utf-8",
             check=False,
         )
-        assert "--help-msg: expected at least one argumen" in process.stderr
+        if error:
+            result = process.stderr
+        else:
+            result = process.stdout
+        assert expected in result
 
     @staticmethod
     def test_generate_rcfile() -> None:
         """Test the --generate-rcfile flag."""
+        args = _add_rcfile_default_pylintrc(["--generate-rcfile"])
         process = subprocess.run(
-            [sys.executable, "-m", "pylint", "--generate-rcfile"],
+            [sys.executable, "-m", "pylint"] + args,
             capture_output=True,
             encoding="utf-8",
             check=False,
         )
         assert "[MASTER]" in process.stdout
         assert "profile" not in process.stdout
-
+        args = _add_rcfile_default_pylintrc(["--generate-rcfile"])
         process_two = subprocess.run(
-            [sys.executable, "-m", "pylint", "--generate-rcfile"],
+            [sys.executable, "-m", "pylint"] + args,
             capture_output=True,
             encoding="utf-8",
             check=False,
@@ -1367,14 +1372,14 @@ class TestCallbackOptions:
     @staticmethod
     def test_generate_toml_config() -> None:
         """Test the --generate-toml-config flag."""
-        process = subprocess.run(
+        args = _add_rcfile_default_pylintrc(
             [
-                sys.executable,
-                "-m",
-                "pylint",
                 "--preferred-modules=a:b",
                 "--generate-toml-config",
-            ],
+            ]
+        )
+        process = subprocess.run(
+            [sys.executable, "-m", "pylint"] + args,
             capture_output=True,
             encoding="utf-8",
             check=False,
@@ -1384,13 +1389,7 @@ class TestCallbackOptions:
         assert 'preferred-modules = ["a:b"]' in process.stdout
 
         process_two = subprocess.run(
-            [
-                sys.executable,
-                "-m",
-                "pylint",
-                "--preferred-modules=a:b",
-                "--generate-toml-config",
-            ],
+            [sys.executable, "-m", "pylint"] + args,
             capture_output=True,
             encoding="utf-8",
             check=False,
@@ -1400,17 +1399,18 @@ class TestCallbackOptions:
     @staticmethod
     def test_generate_toml_config_disable_symbolic_names() -> None:
         """Test that --generate-toml-config puts symbolic names in the --disable option."""
-        out = StringIO()
-        with _patch_streams(out):
+        output_stream = StringIO()
+        with _patch_streams(output_stream):
             with pytest.raises(SystemExit):
                 with warnings.catch_warnings():
                     warnings.simplefilter("ignore")
                     Run(["--generate-toml-config"])
 
-        bytes_out = BytesIO(out.getvalue().encode("utf-8"))
+        out = output_stream.getvalue()
+        bytes_out = BytesIO(out.encode("utf-8"))
         content = tomllib.load(bytes_out)
         messages = content["tool"]["pylint"]["messages control"]["disable"]
-        assert "invalid-name" in messages, out.getvalue()
+        assert "useless-suppression" in messages, out
 
     @staticmethod
     def test_errors_only() -> None:
diff --git a/tests/test_similar.py b/tests/test_similar.py
index 8d8f8d785..e4923b74c 100644
--- a/tests/test_similar.py
+++ b/tests/test_similar.py
@@ -13,7 +13,7 @@ from typing import TextIO
 
 import pytest
 
-from pylint.lint import Run
+from pylint.testutils._run import _Run as Run
 from pylint.testutils.utils import _patch_streams
 
 HERE = abspath(dirname(__file__))
@@ -143,6 +143,6 @@ class TestSimilarCodeChecker:
         """Tests enabling ignore-imports works correctly."""
         path = join(DATA, "ignore_imports")
         self._runtest(
-            [path, "-e=duplicate-code", "-d=unused-import", "--ignore-imports=y"],
+            [path, "-e=duplicate-code", "-d=unused-import,C", "--ignore-imports=y"],
             code=0,
         )