SERVER-73479 resmoke should error locally when attempting to run tests that are disabled

6 files changed, 75 insertions, 7 deletions

diff --git a/buildscripts/resmokelib/config.py b/buildscripts/resmokelib/config.py
index f285c6b21de..7ec8e0d4947 100644
--- a/buildscripts/resmokelib/config.py
+++ b/buildscripts/resmokelib/config.py
@@ -65,6 +65,7 @@ DEFAULTS = {
     "exclude_with_any_tags": None,
     "flow_control": None,
     "flow_control_tickets": None,
+    "force_excluded_tests": False,
     "fuzz_mongod_configs": None,
     "config_fuzz_seed": None,
     "genny_executable": None,
@@ -349,6 +350,9 @@ EVERGREEN_VERSION_ID = None
 # If set, then any jstests that have any of the specified tags will be excluded from the suite(s).
 EXCLUDE_WITH_ANY_TAGS = None
 
+# Allow test files passed as positional args to run even if they are excluded on the suite config.
+FORCE_EXCLUDED_TESTS = None
+
 # A tag which is implicited excluded. This is useful for temporarily disabling a test.
 EXCLUDED_TAG = "__TEMPORARILY_DISABLED__"
 
diff --git a/buildscripts/resmokelib/configure_resmoke.py b/buildscripts/resmokelib/configure_resmoke.py
index d140bfa36f7..786483edd51 100644
--- a/buildscripts/resmokelib/configure_resmoke.py
+++ b/buildscripts/resmokelib/configure_resmoke.py
@@ -364,6 +364,9 @@ or explicitly pass --installDir to the run subcommand of buildscripts/resmoke.py
     _config.EVERGREEN_VARIANT_NAME = config.pop("variant_name")
     _config.EVERGREEN_VERSION_ID = config.pop("version_id")
 
+    # Force invalid suite config
+    _config.FORCE_EXCLUDED_TESTS = config.pop("force_excluded_tests")
+
     # Archival options. Archival is enabled only when running on evergreen.
     if not _config.EVERGREEN_TASK_ID:
         _config.ARCHIVE_FILE = None
diff --git a/buildscripts/resmokelib/run/__init__.py b/buildscripts/resmokelib/run/__init__.py
index 0d88780163d..2219dfe8f16 100644
--- a/buildscripts/resmokelib/run/__init__.py
+++ b/buildscripts/resmokelib/run/__init__.py
@@ -440,6 +440,11 @@ class TestRunner(Subcommand):
             self._resmoke_logger.error("Failed to parse YAML suite definition: %s", str(err))
             self.list_suites()
             self.exit(1)
+        except errors.ResmokeError as err:
+            self._resmoke_logger.error(
+                "Cannot run excluded test in suite config. Use '--force-excluded-tests' to override: %s",
+                str(err))
+            self.exit(1)
 
     def _log_suite_config(self, suite):
         sb = [
@@ -702,6 +707,11 @@ class RunPlugin(PluginInterface):
                   " specified tags will be excluded from any suites that are run."
                   " The tag '{}' is implicitly part of this list.".format(config.EXCLUDED_TAG)))
 
+        parser.add_argument(
+            "--force-excluded-tests", dest="force_excluded_tests", action="store_true",
+            help=("Allows running tests in a suite config's excluded test roots"
+                  " when passed as positional arg(s)."))
+
         parser.add_argument("--genny", dest="genny_executable", metavar="PATH",
                             help="The path to the genny executable for resmoke to use.")
 
diff --git a/buildscripts/resmokelib/suitesconfig.py b/buildscripts/resmokelib/suitesconfig.py
index a00e973f06d..de2a79c17fe 100644
--- a/buildscripts/resmokelib/suitesconfig.py
+++ b/buildscripts/resmokelib/suitesconfig.py
@@ -3,6 +3,7 @@ import collections
 import os
 from threading import Lock
 from typing import Dict, List
+import buildscripts.resmokelib.logging.loggers as loggers
 
 import buildscripts.resmokelib.utils.filesystem as fs
 from buildscripts.resmokelib import config as _config
@@ -101,10 +102,20 @@ def get_suites(suite_names_or_paths, test_files):
     suites = []
     for suite_filename in suite_names_or_paths:
         suite_config = _get_suite_config(suite_filename)
+        suite = _suite.Suite(suite_filename, suite_config)
         if suite_roots:
             # Override the suite's default test files with those passed in from the command line.
-            suite_config.update(suite_roots)
-        suite = _suite.Suite(suite_filename, suite_config)
+            override_suite_config = suite_config.copy()
+            override_suite_config.update(suite_roots)
+            override_suite = _suite.Suite(suite_filename, override_suite_config)
+            for test in override_suite.tests:
+                if test in suite.excluded:
+                    if _config.FORCE_EXCLUDED_TESTS:
+                        loggers.ROOT_EXECUTOR_LOGGER.warning("Will forcibly run excluded test: %s",
+                                                             test)
+                    else:
+                        raise errors.ResmokeError(f"'{test}' excluded in '{suite.get_name()}'")
+            suite = override_suite
         suites.append(suite)
     return suites
 
diff --git a/buildscripts/tests/resmoke_end2end/suites/resmoke_suite_with_excludes.yml b/buildscripts/tests/resmoke_end2end/suites/resmoke_suite_with_excludes.yml
new file mode 100644
index 00000000000..d30395543bc
--- /dev/null
+++ b/buildscripts/tests/resmoke_end2end/suites/resmoke_suite_with_excludes.yml
@@ -0,0 +1,13 @@
+test_kind: js_test
+
+selector:
+  roots:
+  - buildscripts/tests/resmoke_end2end/testfiles/*.js
+  exclude_files:
+  - buildscripts/tests/resmoke_end2end/testfiles/one.js
+
+
+executor:
+  config:
+    shell_options:
+      nodb: ''
diff --git a/buildscripts/tests/resmoke_end2end/test_resmoke.py b/buildscripts/tests/resmoke_end2end/test_resmoke.py
index 145f79f388d..ee0be723577 100644
--- a/buildscripts/tests/resmoke_end2end/test_resmoke.py
+++ b/buildscripts/tests/resmoke_end2end/test_resmoke.py
@@ -430,7 +430,7 @@ class TestSetParameters(_ResmokeSelftest):
 
 def execute_resmoke(resmoke_args):
     return subprocess.run([sys.executable, "buildscripts/resmoke.py", "run"] + resmoke_args,
-                          text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).stdout
+                          text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 
 
 class TestExceptionExtraction(unittest.TestCase):
@@ -438,7 +438,7 @@ class TestExceptionExtraction(unittest.TestCase):
         resmoke_args = [
             "--suites=buildscripts/tests/resmoke_end2end/suites/resmoke_failing_python.yml",
         ]
-        output = execute_resmoke(resmoke_args)
+        output = execute_resmoke(resmoke_args).stdout
 
         expected = "The following tests failed (with exit code):\n        buildscripts/tests/resmoke_end2end/failtestfiles/python_failure.py (1 DB Exception)\n            [LAST Part of Exception]"
         assert expected in output
@@ -447,7 +447,7 @@ class TestExceptionExtraction(unittest.TestCase):
         resmoke_args = [
             "--suites=buildscripts/tests/resmoke_end2end/suites/resmoke_failing_javascript.yml",
         ]
-        output = execute_resmoke(resmoke_args)
+        output = execute_resmoke(resmoke_args).stdout
 
         expected = "The following tests failed (with exit code):\n        buildscripts/tests/resmoke_end2end/failtestfiles/js_failure.js (253 Failure executing JS file)\n            uncaught exception: Error: [true] != [false] are not equal"
         assert expected in output
@@ -456,7 +456,7 @@ class TestExceptionExtraction(unittest.TestCase):
         resmoke_args = [
             "--suites=buildscripts/tests/resmoke_end2end/suites/resmoke_fixture_error.yml",
         ]
-        output = execute_resmoke(resmoke_args)
+        output = execute_resmoke(resmoke_args).stdout
 
         expected = "The following tests had errors:\n    job0_fixture_setup_0\n        Traceback (most recent call last):\n"
         assert expected in output
@@ -465,7 +465,34 @@ class TestExceptionExtraction(unittest.TestCase):
         resmoke_args = [
             "--suites=buildscripts/tests/resmoke_end2end/suites/resmoke_hook_error.yml",
         ]
-        output = execute_resmoke(resmoke_args)
+        output = execute_resmoke(resmoke_args).stdout
 
         expected = "The following tests had errors:\n    buildscripts/tests/resmoke_end2end/failtestfiles/js_failure.js\n        Traceback (most recent call last):\n"
         assert expected in output
+
+
+class TestForceExcludedTest(unittest.TestCase):
+    def test_no_force_exclude(self):
+        resmoke_args = [
+            "--suites=buildscripts/tests/resmoke_end2end/suites/resmoke_suite_with_excludes.yml",
+            "buildscripts/tests/resmoke_end2end/testfiles/one.js",
+        ]
+
+        result = execute_resmoke(resmoke_args)
+
+        expected = "Cannot run excluded test in suite config. Use '--force-excluded-tests' to override:"
+        assert expected in result.stdout
+        assert result.returncode == 1
+
+    def test_with_force_exclude(self):
+        resmoke_args = [
+            "--suites=buildscripts/tests/resmoke_end2end/suites/resmoke_suite_with_excludes.yml",
+            "--force-excluded-tests",
+            "--dryRun",
+            "tests",
+            "buildscripts/tests/resmoke_end2end/testfiles/one.js",
+        ]
+
+        result = execute_resmoke(resmoke_args)
+
+        assert result.returncode == 0