SERVER-47509: Have resmoke better cope with multiple set parameters.

10 files changed, 243 insertions, 32 deletions

diff --git a/buildscripts/resmokelib/config.py b/buildscripts/resmokelib/config.py
index 1e89bcd599d..58b21e4bb4a 100644
--- a/buildscripts/resmokelib/config.py
+++ b/buildscripts/resmokelib/config.py
@@ -65,9 +65,9 @@ DEFAULTS = {
     "logger_file": None,
     "mongo_executable": None,
     "mongod_executable": None,
-    "mongod_set_parameters": None,
+    "mongod_set_parameters": [],
     "mongos_executable": None,
-    "mongos_set_parameters": None,
+    "mongos_set_parameters": [],
     "mrlog": None,
     "no_journal": False,
     "num_clients_per_fixture": 1,
@@ -330,13 +330,13 @@ MONGO_EXECUTABLE = None
 MONGOD_EXECUTABLE = None
 
 # The --setParameter options passed to mongod.
-MONGOD_SET_PARAMETERS = None
+MONGOD_SET_PARAMETERS = []
 
 # The path to the mongos executable used by resmoke.py.
 MONGOS_EXECUTABLE = None
 
 # The --setParameter options passed to mongos.
-MONGOS_SET_PARAMETERS = None
+MONGOS_SET_PARAMETERS = []
 
 # If true, then all mongod's started by resmoke.py and by the mongo shell will not have journaling
 # enabled.
diff --git a/buildscripts/resmokelib/configure_resmoke.py b/buildscripts/resmokelib/configure_resmoke.py
index 622f3cef3ae..b8ab910f716 100644
--- a/buildscripts/resmokelib/configure_resmoke.py
+++ b/buildscripts/resmokelib/configure_resmoke.py
@@ -1,5 +1,6 @@
 """Configure the command line input for the resmoke 'run' subcommand."""
 
+import collections
 import configparser
 import datetime
 import os
@@ -25,7 +26,7 @@ def validate_and_update_config(parser, args):
 def _validate_options(parser, args):
     """Do preliminary validation on the options and error on any invalid options."""
 
-    if not 'shell_port' in args or not 'shell_conn_strin' in args:
+    if not 'shell_port' in args or not 'shell_conn_string' in args:
         return
 
     if args.shell_port is not None and args.shell_conn_string is not None:
@@ -42,8 +43,32 @@ def _validate_options(parser, args):
             "Cannot use --replayFile with additional test files listed on the command line invocation."
         )
 
+    def get_set_param_errors(process_params):
+        agg_set_params = collections.defaultdict(list)
+        for set_param in process_params:
+            for key, value in utils.load_yaml(set_param).items():
+                agg_set_params[key] += [value]
 
-def _validate_config(parser):
+        errors = []
+        for key, values in agg_set_params.items():
+            if len(values) > 1:
+                errors.append(f"setParameter has multiple values. Key: {key} Values: {values}")
+
+        return errors
+
+    config = vars(args)
+    mongod_set_param_errors = get_set_param_errors(config.get('mongod_set_parameters') or [])
+    mongos_set_param_errors = get_set_param_errors(config.get('mongos_set_parameters') or [])
+    error_msgs = {}
+    if mongod_set_param_errors:
+        error_msgs["mongodSetParameters"] = mongod_set_param_errors
+    if mongos_set_param_errors:
+        error_msgs["mongosSetParameters"] = mongos_set_param_errors
+    if error_msgs:
+        parser.error(str(error_msgs))
+
+
+def _validate_config(parser):  # pylint: disable=too-many-branches
     """Do validation on the config settings."""
 
     if _config.REPEAT_TESTS_MAX:
@@ -140,11 +165,18 @@ def _update_config_vars(values):  # pylint: disable=too-many-statements,too-many
 
     _config.DBTEST_EXECUTABLE = _expand_user(config.pop("dbtest_executable"))
     _config.MONGO_EXECUTABLE = _expand_user(config.pop("mongo_executable"))
+
+    def _merge_set_params(param_list):
+        ret = {}
+        for set_param in param_list:
+            ret.update(utils.load_yaml(set_param))
+        return utils.dump_yaml(ret)
+
     _config.MONGOD_EXECUTABLE = _expand_user(config.pop("mongod_executable"))
-    _config.MONGOD_SET_PARAMETERS = config.pop("mongod_set_parameters")
+    _config.MONGOD_SET_PARAMETERS = _merge_set_params(config.pop("mongod_set_parameters"))
     _config.MONGOS_EXECUTABLE = _expand_user(config.pop("mongos_executable"))
+    _config.MONGOS_SET_PARAMETERS = _merge_set_params(config.pop("mongos_set_parameters"))
 
-    _config.MONGOS_SET_PARAMETERS = config.pop("mongos_set_parameters")
     _config.MRLOG = config.pop("mrlog")
     _config.NO_JOURNAL = config.pop("no_journal")
     _config.NUM_CLIENTS_PER_FIXTURE = config.pop("num_clients_per_fixture")
diff --git a/buildscripts/resmokelib/run/__init__.py b/buildscripts/resmokelib/run/__init__.py
index cf5a74faddc..9a8d17db6d6 100644
--- a/buildscripts/resmokelib/run/__init__.py
+++ b/buildscripts/resmokelib/run/__init__.py
@@ -673,7 +673,7 @@ class RunPlugin(PluginInterface):
                             help="The path to the mongod executable for resmoke.py to use.")
 
         parser.add_argument(
-            "--mongodSetParameters", dest="mongod_set_parameters",
+            "--mongodSetParameters", dest="mongod_set_parameters", action="append",
             metavar="{key1: value1, key2: value2, ..., keyN: valueN}",
             help=("Passes one or more --setParameter options to all mongod processes"
                   " started by resmoke.py. The argument is specified as bracketed YAML -"
@@ -683,7 +683,7 @@ class RunPlugin(PluginInterface):
                             help="The path to the mongos executable for resmoke.py to use.")
 
         parser.add_argument(
-            "--mongosSetParameters", dest="mongos_set_parameters",
+            "--mongosSetParameters", dest="mongos_set_parameters", action="append",
             metavar="{key1: value1, key2: value2, ..., keyN: valueN}",
             help=("Passes one or more --setParameter options to all mongos processes"
                   " started by resmoke.py. The argument is specified as bracketed YAML -"
diff --git a/buildscripts/tests/resmoke_end2end/suites/resmoke_no_mongod.yml b/buildscripts/tests/resmoke_end2end/suites/resmoke_no_mongod.yml
index 92af90962dd..5076c1c018a 100644
--- a/buildscripts/tests/resmoke_end2end/suites/resmoke_no_mongod.yml
+++ b/buildscripts/tests/resmoke_end2end/suites/resmoke_no_mongod.yml
@@ -2,7 +2,8 @@ test_kind: js_test
 
 selector:
   roots:
-  - buildscripts/tests/resmoke_end2end/testfiles/*.js
+  - buildscripts/tests/resmoke_end2end/testfiles/one.js
+  - buildscripts/tests/resmoke_end2end/testfiles/two.js
 
 executor:
   config:
diff --git a/buildscripts/tests/resmoke_end2end/suites/resmoke_selftest_set_parameters.yml b/buildscripts/tests/resmoke_end2end/suites/resmoke_selftest_set_parameters.yml
new file mode 100644
index 00000000000..45548678628
--- /dev/null
+++ b/buildscripts/tests/resmoke_end2end/suites/resmoke_selftest_set_parameters.yml
@@ -0,0 +1,21 @@
+test_kind: js_test
+
+selector:
+  roots:
+  - jstests/resmoke_selftest/end2end/**.js
+
+executor:
+  archive:
+    tests: true
+  config:
+    shell_options:
+      global_vars:
+        TestData:
+          outputLocation: ./output.json # This is a template value replaced at runtime.
+  fixture:
+    class: MongoDFixture
+    mongod_options:
+      set_parameters:
+        enableTestCommands: 1
+        logComponentVerbosity: {storage: 2}
+        testingDiagnosticsEnabled: false
diff --git a/buildscripts/tests/resmoke_end2end/suites/resmoke_selftest_set_parameters_sharding.yml b/buildscripts/tests/resmoke_end2end/suites/resmoke_selftest_set_parameters_sharding.yml
new file mode 100644
index 00000000000..d5a0f893269
--- /dev/null
+++ b/buildscripts/tests/resmoke_end2end/suites/resmoke_selftest_set_parameters_sharding.yml
@@ -0,0 +1,19 @@
+test_kind: js_test
+
+selector:
+  roots:
+  - jstests/resmoke_selftest/end2end/**.js
+
+executor:
+  archive:
+    tests: true
+  config:
+    shell_options:
+      global_vars:
+        TestData:
+          outputLocation: ./output.json # This is a template value replaced at runtime.
+  fixture:
+    class: ShardedClusterFixture
+    mongod_options:
+      set_parameters:
+        enableTestCommands: 1
diff --git a/buildscripts/tests/resmoke_end2end/test_resmoke.py b/buildscripts/tests/resmoke_end2end/test_resmoke.py
index a1f6a57e432..48b095f7dc3 100644
--- a/buildscripts/tests/resmoke_end2end/test_resmoke.py
+++ b/buildscripts/tests/resmoke_end2end/test_resmoke.py
@@ -7,6 +7,7 @@ import os.path
 import sys
 import time
 import unittest
+import yaml
 
 from buildscripts.resmokelib import core
 from buildscripts.resmokelib.utils import rmtree
@@ -17,16 +18,14 @@ from buildscripts.resmokelib.utils import rmtree
 class _ResmokeSelftest(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
+        cls.end2end_root = "buildscripts/tests/resmoke_end2end"
         cls.test_dir = os.path.normpath("/data/db/selftest")
         cls.resmoke_const_args = ["run", "--dbpathPrefix={}".format(cls.test_dir)]
+        cls.suites_root = os.path.join(cls.end2end_root, "suites")
+        cls.testfiles_root = os.path.join(cls.end2end_root, "testfiles")
+        cls.report_file = os.path.join(cls.test_dir, "reports.json")
 
     def setUp(self):
-        #self.test_dir = os.path.normpath("/data/db/selftest")
-        self.end2end_root = "buildscripts/tests/resmoke_end2end"
-        self.suites_root = f"{self.end2end_root}/suites"
-        self.testfiles_root = f"{self.end2end_root}/testfiles"
-        self.report_file = os.path.join(self.test_dir, "reports.json")
-
         self.logger = logging.getLogger(self._testMethodName)
         self.logger.setLevel(logging.DEBUG)
         handler = logging.StreamHandler(sys.stdout)
@@ -248,9 +247,140 @@ class TestTestSelection(_ResmokeSelftest):
         self.assertEqual(6 * [f"{self.testfiles_root}/two.js"], self.get_tests_run())
 
     def test_disallow_mixing_replay_and_positional(self):
+        self.create_file_in_test_dir("replay", f"{self.testfiles_root}/two.js\n" * 3)
+
         # Additionally can assert on the error message.
-        self.assertEqual(1, self.execute_resmoke(["--replay=foo", "jstests/filename.js"]).wait())
-        self.assertEqual(1, self.execute_resmoke(["@foo", "jstests/filename.js"]).wait())
+        self.assertEqual(
+            2,
+            self.execute_resmoke([f"--replay={self.test_dir}/replay",
+                                  "jstests/filename.js"]).wait())
+
+        # When multiple positional arguments are presented, they're all treated as test files. Technically errors on file `@<testdir>/replay` not existing. It's not a requirement that this invocation errors in this less specific way.
+        self.assertEqual(
+            1,
+            self.execute_resmoke([f"@{self.test_dir}/replay", "jstests/filename.js"]).wait())
+        self.assertEqual(
+            1,
+            self.execute_resmoke([f"{self.testfiles_root}/one.js",
+                                  f"@{self.test_dir}/replay"]).wait())
+
+
+class TestSetParameters(_ResmokeSelftest):
+    def setUp(self):
+        self.shell_output_file = f"{self.test_dir}/output.json"
+        try:
+            os.remove(self.shell_output_file)
+        except OSError:
+            pass
+
+        super().setUp()
+
+    def parse_output_json(self):
+        # Parses the outputted json.
+        with open(self.shell_output_file) as fd:
+            return json.load(fd)
 
-        # Technically errors on file `@foo` not existing. Only the first positional argument can be interpreted as a `@replay_file`. This is a limitation not a feature, this test just serves as documentation.
-        self.assertEqual(1, self.execute_resmoke([f"{self.testfiles_root}/one.js", "@foo"]).wait())
+    def generate_suite(self, suite_output_path, template_file):
+        """Read the template file, substitute the `outputLocation` and rewrite to the file `suite_output_path`."""
+
+        with open(os.path.normpath(template_file), "r") as template_suite_fd:
+            suite = yaml.safe_load(template_suite_fd)
+
+        try:
+            os.remove(suite_output_path)
+        except FileNotFoundError:
+            pass
+
+        suite["executor"]["config"]["shell_options"]["global_vars"]["TestData"][
+            "outputLocation"] = self.shell_output_file
+        with open(os.path.normpath(suite_output_path), "w") as fd:
+            yaml.dump(suite, fd, default_flow_style=False)
+
+    def generate_suite_and_execute_resmoke(self, suite_template, resmoke_args):
+        """Generates a resmoke suite with the appropriate `outputLocation` and runs resmoke against that suite with the `fixture_info` test. Input `resmoke_args` are appended to the run command."""
+
+        suite_file = f"{self.test_dir}/suite.yml"
+        self.generate_suite(suite_file, suite_template)
+
+        self.logger.info(
+            "Running test. Template suite: {suite_template} Rewritten suite: {self.suite_file} Resmoke Args: {resmoke_args} Test output file: {self.shell_output_file}."
+        )
+
+        resmoke_process = core.programs.make_process(self.logger, [
+            sys.executable, "buildscripts/resmoke.py", "run", f"--suites={suite_file}",
+            f"{self.testfiles_root}/fixture_info.js"
+        ] + resmoke_args)
+        resmoke_process.start()
+
+        return resmoke_process
+
+    def test_suite_set_parameters(self):
+        self.generate_suite_and_execute_resmoke(
+            f"{self.suites_root}/resmoke_selftest_set_parameters.yml", []).wait()
+
+        set_params = self.parse_output_json()
+        self.assertEqual("1", set_params["enableTestCommands"])
+        self.assertEqual("false", set_params["testingDiagnosticsEnabled"])
+        self.assertEqual("{'storage': 2}", set_params["logComponentVerbosity"])
+
+    def test_cli_set_parameters(self):
+        self.generate_suite_and_execute_resmoke(
+            f"{self.suites_root}/resmoke_selftest_set_parameters.yml",
+            ["""--mongodSetParameter={"enableFlowControl": false, "flowControlMaxSamples": 500}"""
+             ]).wait()
+
+        set_params = self.parse_output_json()
+        self.assertEqual("1", set_params["enableTestCommands"])
+        self.assertEqual("false", set_params["enableFlowControl"])
+        self.assertEqual("500", set_params["flowControlMaxSamples"])
+
+    def test_override_set_parameters(self):
+        self.generate_suite_and_execute_resmoke(
+            f"{self.suites_root}/resmoke_selftest_set_parameters.yml",
+            ["""--mongodSetParameter={"testingDiagnosticsEnabled": true}"""]).wait()
+
+        set_params = self.parse_output_json()
+        self.assertEqual("true", set_params["testingDiagnosticsEnabled"])
+        self.assertEqual("{'storage': 2}", set_params["logComponentVerbosity"])
+
+    def test_merge_cli_set_parameters(self):
+        self.generate_suite_and_execute_resmoke(
+            f"{self.suites_root}/resmoke_selftest_set_parameters.yml", [
+                """--mongodSetParameter={"enableFlowControl": false}""",
+                """--mongodSetParameter={"flowControlMaxSamples": 500}"""
+            ]).wait()
+
+        set_params = self.parse_output_json()
+        self.assertEqual("false", set_params["testingDiagnosticsEnabled"])
+        self.assertEqual("{'storage': 2}", set_params["logComponentVerbosity"])
+        self.assertEqual("false", set_params["enableFlowControl"])
+        self.assertEqual("500", set_params["flowControlMaxSamples"])
+
+    def test_merge_error_cli_set_parameters(self):
+        self.assertEqual(
+            2,
+            self.generate_suite_and_execute_resmoke(
+                f"{self.suites_root}/resmoke_selftest_set_parameters.yml", [
+                    """--mongodSetParameter={"enableFlowControl": false}""",
+                    """--mongodSetParameter={"enableFlowControl": true}"""
+                ]).wait())
+
+    def test_mongos_set_parameter(self):
+        self.generate_suite_and_execute_resmoke(
+            f"{self.suites_root}/resmoke_selftest_set_parameters_sharding.yml", [
+                """--mongosSetParameter={"maxTimeMSForHedgedReads": 100}""",
+                """--mongosSetParameter={"mongosShutdownTimeoutMillisForSignaledShutdown": 1000}"""
+            ]).wait()
+
+        set_params = self.parse_output_json()
+        self.assertEqual("100", set_params["maxTimeMSForHedgedReads"])
+        self.assertEqual("1000", set_params["mongosShutdownTimeoutMillisForSignaledShutdown"])
+
+    def test_merge_error_cli_mongos_set_parameter(self):
+        self.assertEqual(
+            2,
+            self.generate_suite_and_execute_resmoke(
+                f"{self.suites_root}/resmoke_selftest_set_parameters_sharding.yml", [
+                    """--mongosSetParameter={"maxTimeMSForHedgedReads": 100}""",
+                    """--mongosSetParameter={"maxTimeMSForHedgedReads": 1000}"""
+                ]).wait())
diff --git a/buildscripts/tests/resmoke_end2end/testfiles/fixture_info.js b/buildscripts/tests/resmoke_end2end/testfiles/fixture_info.js
new file mode 100644
index 00000000000..fb925299a74
--- /dev/null
+++ b/buildscripts/tests/resmoke_end2end/testfiles/fixture_info.js
@@ -0,0 +1,6 @@
+(function() {
+"use strict";
+
+writeFile(TestData.outputLocation,
+          tojson(db.adminCommand("getCmdLineOpts")["parsed"]["setParameter"]));
+}());
diff --git a/buildscripts/tests/resmoke_end2end/testfiles/fixture_info_mongos.js b/buildscripts/tests/resmoke_end2end/testfiles/fixture_info_mongos.js
new file mode 100644
index 00000000000..0837b88d06a
--- /dev/null
+++ b/buildscripts/tests/resmoke_end2end/testfiles/fixture_info_mongos.js
@@ -0,0 +1,9 @@
+(function() {
+"use strict";
+
+print("DEBUG BUILDINFO")
+printjson(db.adminCommand("buildInfo"))
+
+writeFile(TestData.outputLocation,
+          tojson(db.adminCommand("getCmdLineOpts")["parsed"]["setParameter"]));
+}());
diff --git a/etc/evergreen.yml b/etc/evergreen.yml
index 340340b4b93..3e7765c6344 100644
--- a/etc/evergreen.yml
+++ b/etc/evergreen.yml
@@ -10040,10 +10040,8 @@ buildvariants:
     repo_edition: enterprise
     scons_cache_scope: shared
     large_distro_name: rhel62-large
-    # TODO (SERVER-47509): Re-enable index build commit-quorum when resmoke handles multiple
-    # mongodSetParameters.
     test_flags: >-
-        --mongodSetParameters="{maxNumberOfTransactionOperationsInSingleOplogEntry: 2, enableIndexBuildCommitQuorum: false}"
+        --mongodSetParameters="{maxNumberOfTransactionOperationsInSingleOplogEntry: 2}"
         --excludeWithAnyTags=exclude_from_large_txns
   tasks:
   - name: compile_TG
@@ -10127,12 +10125,10 @@ buildvariants:
     # works correctly.
     # TODO (SERVER-47022): Re-enable oplog constraint enforcement once we set the application mode
     # correctly after rollbackViaRefetch.
-    # TODO (SERVER-47509): Re-enable index build commit-quorum when resmoke handles multiple
-    # mongodSetParameters.
     test_flags: >-
       --majorityReadConcern=off
       --excludeWithAnyTags=requires_majority_read_concern,uses_prepare_transaction,uses_multi_shard_transaction,uses_atclustertime
-      --mongodSetParameters="{oplogApplicationEnforcesSteadyStateConstraints: false, enableIndexBuildCommitQuorum: false}"
+      --mongodSetParameters="{oplogApplicationEnforcesSteadyStateConstraints: false}"
     compile_flags: >-
       -j$(grep -c ^processor /proc/cpuinfo)
       --ssl
@@ -10360,7 +10356,6 @@ buildvariants:
     scons_cache_scope: shared
     build_mongoreplay: false
     large_distro_name: rhel62-large
-  # Additions of tasks already defining '--mongodSetParameters' will override this test suite's parameter settings (SERVER-47509).
   tasks:
   - name: compile_TG
     distros:
@@ -11860,12 +11855,10 @@ buildvariants:
     # We also relax oplog constraints so that applying oplog entries after a rollbackViaRefetch works correctly.
     # TODO (SERVER-47022): Re-enable oplog constraint enforcement once we set the application mode
     # correctly after rollbackViaRefetch.
-    # TODO (SERVER-47509): Re-enable index build commit-quorum when resmoke handles multiple
-    # mongodSetParameters.
     test_flags: >-
       --storageEngine=ephemeralForTest
-      --excludeWithAnyTags=requires_persistence,requires_fsync,SERVER-21420,SERVER-21658,requires_journaling,requires_wiredtiger,uses_transactions,uses_speculative_majority,requires_snapshot_read,requires_majority_read_concern,uses_change_streams,requires_sharding,incompatible_with_eft
-      --mongodSetParameters="{oplogApplicationEnforcesSteadyStateConstraints: false, enableIndexBuildCommitQuorum: false}"
+      --excludeWithAnyTags=requires_persistence,requires_fsync,SERVER-21420,SERVER-21658,requires_journaling,requires_wiredtiger,uses_transactions,requires_document_locking,uses_speculative_majority,requires_snapshot_read,requires_majority_read_concern,uses_change_streams,requires_sharding,incompatible_with_eft
+      --mongodSetParameters="{oplogApplicationEnforcesSteadyStateConstraints: false}"
     compile_flags: -j$(grep -c ^processor /proc/cpuinfo) --dbg=off --opt=on --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars
     multiversion_platform: rhel62
     multiversion_edition: targeted