SERVER-54420 don't run feature flag tests on the multiversion ff variant

4 files changed, 29 insertions, 9 deletions

diff --git a/buildscripts/resmokelib/config.py b/buildscripts/resmokelib/config.py
index cb14c081ee7..de93f1840f1 100644
--- a/buildscripts/resmokelib/config.py
+++ b/buildscripts/resmokelib/config.py
@@ -90,6 +90,7 @@ DEFAULTS = {
     "report_failure_status": "fail",
     "report_file": None,
     "run_all_feature_flag_tests": False,
+    "run_all_feature_flags_no_tests": False,
     "additional_feature_flags": None,
     "seed": int(time.time() * 256),  # Taken from random.py code in Python 2.7.
     "service_executor": None,
@@ -378,6 +379,9 @@ INSTALL_DIR = None
 # Whether to run tests for feature flags.
 RUN_ALL_FEATURE_FLAG_TESTS = None
 
+# Whether to run the server with feature flags. Defaults to true if `RUN_ALL_FEATURE_FLAG_TESTS` is true.
+RUN_ALL_FEATURE_FLAGS = None
+
 # List of enabled feature flags.
 ENABLED_FEATURE_FLAGS = []
 
diff --git a/buildscripts/resmokelib/configure_resmoke.py b/buildscripts/resmokelib/configure_resmoke.py
index 34eb2295b2e..d32a1e1e758 100644
--- a/buildscripts/resmokelib/configure_resmoke.py
+++ b/buildscripts/resmokelib/configure_resmoke.py
@@ -48,7 +48,7 @@ def _validate_options(parser, args):
             "Cannot use --replayFile with additional test files listed on the command line invocation."
         )
 
-    if args.run_all_feature_flag_tests:
+    if args.run_all_feature_flag_tests or args.run_all_feature_flags_no_tests:
         if not os.path.isfile(ALL_FEATURE_FLAG_FILE):
             parser.error(
                 "To run tests with all feature flags, the %s file must exist and be placed in"
@@ -156,6 +156,12 @@ def _update_config_vars(values):  # pylint: disable=too-many-statements,too-many
 
     def setup_feature_flags():
         _config.RUN_ALL_FEATURE_FLAG_TESTS = config.pop("run_all_feature_flag_tests")
+        _config.RUN_ALL_FEATURE_FLAGS = config.pop("run_all_feature_flags_no_tests")
+
+        # Running all feature flag tests implies running the fixtures with feature flags.
+        if _config.RUN_ALL_FEATURE_FLAG_TESTS:
+            _config.RUN_ALL_FEATURE_FLAGS = True
+
         all_feature_flags = []
         enabled_feature_flags = []
         try:
@@ -163,10 +169,10 @@ def _update_config_vars(values):  # pylint: disable=too-many-statements,too-many
         except FileNotFoundError:
             # If we ask resmoke to run with all feature flags, the feature flags file
             # needs to exist.
-            if _config.RUN_ALL_FEATURE_FLAG_TESTS:
+            if _config.RUN_ALL_FEATURE_FLAGS:
                 raise
 
-        if _config.RUN_ALL_FEATURE_FLAG_TESTS:
+        if _config.RUN_ALL_FEATURE_FLAGS:
             enabled_feature_flags = all_feature_flags[:]
 
         # Specify additional feature flags from the command line.
@@ -192,8 +198,12 @@ def _update_config_vars(values):  # pylint: disable=too-many-statements,too-many
     _config.EXCLUDE_WITH_ANY_TAGS.extend(
         utils.default_if_none(_tags_from_list(config.pop("exclude_with_any_tags")), []))
 
-    # Don't run tests with feature flags that are not enabled.
-    _config.EXCLUDE_WITH_ANY_TAGS.extend(not_enabled_feature_flags)
+    if _config.RUN_ALL_FEATURE_FLAGS and not _config.RUN_ALL_FEATURE_FLAG_TESTS:
+        # Don't run any feature flag tests.
+        _config.EXCLUDE_WITH_ANY_TAGS.extend(all_feature_flags)
+    else:
+        # Don't run tests with feature flags that are not enabled.
+        _config.EXCLUDE_WITH_ANY_TAGS.extend(not_enabled_feature_flags)
 
     _config.FAIL_FAST = not config.pop("continue_on_failure")
     _config.FLOW_CONTROL = config.pop("flow_control")
diff --git a/buildscripts/resmokelib/run/__init__.py b/buildscripts/resmokelib/run/__init__.py
index c7edacd52cb..0d9a3432ea4 100644
--- a/buildscripts/resmokelib/run/__init__.py
+++ b/buildscripts/resmokelib/run/__init__.py
@@ -846,6 +846,12 @@ class RunPlugin(PluginInterface):
             "Run MongoDB servers with all feature flags enabled and only run tests tags with these feature flags"
         )
 
+        parser.add_argument(
+            "--runAllFeatureFlagsNoTests", dest="run_all_feature_flags_no_tests",
+            action="store_true", help=
+            "Run MongoDB servers with all feature flags enabled but don't run any tests tagged with these feature flags; used for multiversion suites"
+        )
+
         parser.add_argument("--additionalFeatureFlags", dest="additional_feature_flags",
                             action="append", metavar="featureFlag1, featureFlag2, ...",
                             help="Additional feature flags")
@@ -1208,6 +1214,8 @@ def to_local_args(input_args=None):  # pylint: disable=too-many-branches,too-man
         setattr(parsed_args, "additional_feature_flags", config.ENABLED_FEATURE_FLAGS)
         del parsed_args.run_all_feature_flag_tests
 
+    del parsed_args.run_all_feature_flags_no_tests
+
     # The top-level parser has one subparser that contains all subcommand parsers.
     command_subparser = [
         action for action in parser._actions  # pylint: disable=protected-access
diff --git a/etc/evergreen.yml b/etc/evergreen.yml
index 8c6473a1cd0..419f126dc22 100644
--- a/etc/evergreen.yml
+++ b/etc/evergreen.yml
@@ -9757,11 +9757,9 @@ buildvariants:
   cron: "0 12 * * *" # Every day starting at 12:00
   expansions:
     <<: *enterprise-rhel-80-64-bit-multiversion-expansions-template
-    # TODO (SERVER-55857): Remove 'requires_fcv_51' once tests are filtered out using the
-    # 'REQUIRES_FCV_TAG' list in 'multiversion_constants.py'.
+    # No feature flag tests since they aren't compatible with the older binaries.
     test_flags: >-
-      --excludeWithAnyTags=requires_fcv_51,multiversion_incompatible
-      --runAllFeatureFlagTests
+      --runAllFeatureFlagsNoTests
 
 - name: rhel-80-64-bit-nossl
   display_name: "RHEL 8.0 Shared Library (No SSL)"