SERVER-47509 resmoke accepts multiple "mongodSetParameters" options but only uses the last one.

This commit also cherry-picks the following.
SERVER-39448 Make resmoke pass TestData.setParametersMongos to mongo shell as a JavaScript object
(cherry picked from commit f998d1f6bd1d74a815e1bbe6e984c8e73da8398d)

4 files changed, 62 insertions, 22 deletions

diff --git a/buildscripts/resmokeconfig/suites/ssl.yml b/buildscripts/resmokeconfig/suites/ssl.yml
index 890928c491e..a5dae795063 100644
--- a/buildscripts/resmokeconfig/suites/ssl.yml
+++ b/buildscripts/resmokeconfig/suites/ssl.yml
@@ -16,5 +16,9 @@ executor:
       readMode: commands
       global_vars:
         TestData:
-          setParameters: "logComponentVerbosity={network:2}"
-          setParametersMongos: "logComponentVerbosity={network:2}"
+          setParameters:
+            logComponentVerbosity:
+              network: 2
+          setParametersMongos:
+            logComponentVerbosity:
+              network: 2
diff --git a/buildscripts/resmokelib/config.py b/buildscripts/resmokelib/config.py
index f0886877bfc..bbb60bd9b96 100644
--- a/buildscripts/resmokelib/config.py
+++ b/buildscripts/resmokelib/config.py
@@ -52,9 +52,9 @@ DEFAULTS = {
     "jobs": 1,
     "mongo_executable": None,
     "mongod_executable": None,
-    "mongod_set_parameters": None,
+    "mongod_set_parameters": [],
     "mongos_executable": None,
-    "mongos_set_parameters": None,
+    "mongos_set_parameters": [],
     "no_journal": False,
     "num_clients_per_fixture": 1,
     "perf_report_file": None,
@@ -270,13 +270,13 @@ MONGO_EXECUTABLE = None
 MONGOD_EXECUTABLE = None
 
 # The --setParameter options passed to mongod.
-MONGOD_SET_PARAMETERS = None
+MONGOD_SET_PARAMETERS = []  # type: ignore
 
 # The path to the mongos executable used by resmoke.py.
 MONGOS_EXECUTABLE = None
 
 # The --setParameter options passed to mongos.
-MONGOS_SET_PARAMETERS = None
+MONGOS_SET_PARAMETERS = []  # type: ignore
 
 # If true, then all mongod's started by resmoke.py and by the mongo shell will not have journaling
 # enabled.
diff --git a/buildscripts/resmokelib/core/programs.py b/buildscripts/resmokelib/core/programs.py
index de0eaa3c49f..5c540e6203c 100644
--- a/buildscripts/resmokelib/core/programs.py
+++ b/buildscripts/resmokelib/core/programs.py
@@ -213,12 +213,18 @@ def mongo_shell_program(  # pylint: disable=too-many-branches,too-many-locals,to
     # Initialize setParameters for mongod and mongos, to be passed to the shell via TestData. Since
     # they are dictionaries, they will be converted to JavaScript objects when passed to the shell
     # by the _format_shell_vars() function.
-    mongod_set_parameters = {}
+    mongod_set_parameters = test_data.get("setParameters", {}).copy()
+    mongos_set_parameters = test_data.get("setParametersMongos", {}).copy()
+
+    # Propagate additional setParameters to mongod processes spawned by the mongo shell. Command
+    # line options to resmoke.py override the YAML configuration.
     if config.MONGOD_SET_PARAMETERS is not None:
-        if "setParameters" in test_data:
-            raise ValueError("setParameters passed via TestData can only be set from either the"
-                             " command line or the suite YAML, not both")
-        mongod_set_parameters = utils.load_yaml(config.MONGOD_SET_PARAMETERS)
+        mongod_set_parameters.update(utils.load_yaml(config.MONGOD_SET_PARAMETERS))
+
+    # Propagate additional setParameters to mongos processes spawned by the mongo shell. Command
+    # line options to resmoke.py override the YAML configuration.
+    if config.MONGOS_SET_PARAMETERS is not None:
+        mongos_set_parameters.update(utils.load_yaml(config.MONGOS_SET_PARAMETERS))
 
     # If the 'logComponentVerbosity' setParameter for mongod was not already specified, we set its
     # value to a default.
@@ -226,13 +232,7 @@ def mongo_shell_program(  # pylint: disable=too-many-branches,too-many-locals,to
                                      default_mongod_log_component_verbosity())
 
     test_data["setParameters"] = mongod_set_parameters
-
-    if config.MONGOS_SET_PARAMETERS is not None:
-        if "setParametersMongos" in test_data:
-            raise ValueError("setParametersMongos passed via TestData can only be set from either"
-                             " the command line or the suite YAML, not both")
-        mongos_set_parameters = utils.load_yaml(config.MONGOS_SET_PARAMETERS)
-        test_data["setParametersMongos"] = mongos_set_parameters
+    test_data["setParametersMongos"] = mongos_set_parameters
 
     # There's a periodic background thread that checks for and aborts expired transactions.
     # "transactionLifetimeLimitSeconds" specifies for how long a transaction can run before expiring
diff --git a/buildscripts/resmokelib/parser.py b/buildscripts/resmokelib/parser.py
index d209d70e230..605ed9f7742 100644
--- a/buildscripts/resmokelib/parser.py
+++ b/buildscripts/resmokelib/parser.py
@@ -106,7 +106,7 @@ def _make_parser():  # pylint: disable=too-many-statements
     parser.add_option("--mongod", dest="mongod_executable", metavar="PATH",
                       help="The path to the mongod executable for resmoke.py to use.")
 
-    parser.add_option("--mongodSetParameters", dest="mongod_set_parameters",
+    parser.add_option("--mongodSetParameters", action="append", dest="mongod_set_parameters",
                       metavar="{key1: value1, key2: value2, ..., keyN: valueN}",
                       help=("Passes one or more --setParameter options to all mongod processes"
                             " started by resmoke.py. The argument is specified as bracketed YAML -"
@@ -115,7 +115,7 @@ def _make_parser():  # pylint: disable=too-many-statements
     parser.add_option("--mongos", dest="mongos_executable", metavar="PATH",
                       help="The path to the mongos executable for resmoke.py to use.")
 
-    parser.add_option("--mongosSetParameters", dest="mongos_set_parameters",
+    parser.add_option("--mongosSetParameters", action="append", dest="mongos_set_parameters",
                       metavar="{key1: value1, key2: value2, ..., keyN: valueN}",
                       help=("Passes one or more --setParameter options to all mongos processes"
                             " started by resmoke.py. The argument is specified as bracketed YAML -"
@@ -329,6 +329,34 @@ def _validate_options(parser, options, args):
                      " test(s) under those suite configuration(s)".format(
                          options.executor_file, " ".join(args)))
 
+    def get_set_param_errors(process_params):
+        """Return the list of errors to indicate parameters with multiple values."""
+        agg_set_params = collections.defaultdict(list)
+        for set_param in process_params:
+            for key, value in utils.load_yaml(set_param).items():
+                agg_set_params[key] += [value]
+
+        errors = []
+        for key, values in agg_set_params.items():
+            if len(values) == 1:
+                for left, _ in enumerate(values):
+                    for right in range(left + 1, len(values)):
+                        if values[left] != values[right]:
+                            errors.append(
+                                "setParameter has multiple values. Key: {} Values: {}".format(
+                                    key, values))
+        return errors
+
+    mongod_set_param_errors = get_set_param_errors(options.mongod_set_parameters or [])
+    mongos_set_param_errors = get_set_param_errors(options.mongos_set_parameters or [])
+    error_msgs = {}
+    if mongod_set_param_errors:
+        error_msgs["mongodSetParameters"] = mongod_set_param_errors
+    if mongos_set_param_errors:
+        error_msgs["mongosSetParameters"] = mongos_set_param_errors
+    if error_msgs:
+        parser.error(str(error_msgs))
+
 
 def validate_benchmark_options():
     """Error out early if any options are incompatible with benchmark test suites.
@@ -348,6 +376,14 @@ def validate_benchmark_options():
             "results. Please use --jobs=1" % _config.JOBS)
 
 
+def _merge_set_params(param_list):
+    """Merge the setParameters."""
+    ret = {}
+    for set_param in param_list:
+        ret.update(utils.load_yaml(set_param))
+    return utils.dump_yaml(ret)
+
+
 def _update_config_vars(values):  # pylint: disable=too-many-statements
     """Update the variables of the config module."""
 
@@ -377,9 +413,9 @@ def _update_config_vars(values):  # pylint: disable=too-many-statements
     _config.MAJORITY_READ_CONCERN = config.pop("majority_read_concern") == "on"
     _config.MONGO_EXECUTABLE = _expand_user(config.pop("mongo_executable"))
     _config.MONGOD_EXECUTABLE = _expand_user(config.pop("mongod_executable"))
-    _config.MONGOD_SET_PARAMETERS = config.pop("mongod_set_parameters")
+    _config.MONGOD_SET_PARAMETERS = _merge_set_params(config.pop("mongod_set_parameters"))
     _config.MONGOS_EXECUTABLE = _expand_user(config.pop("mongos_executable"))
-    _config.MONGOS_SET_PARAMETERS = config.pop("mongos_set_parameters")
+    _config.MONGOS_SET_PARAMETERS = _merge_set_params(config.pop("mongos_set_parameters"))
     _config.NO_JOURNAL = config.pop("no_journal")
     _config.NO_PREALLOC_JOURNAL = config.pop("prealloc_journal") == "off"
     _config.NUM_CLIENTS_PER_FIXTURE = config.pop("num_clients_per_fixture")