SERVER-61460 correctly merge --setParameter in resmoke

(cherry picked from commit 876e0930d208a9b29c2aa02b291a0d57db678f8f)

4 files changed, 105 insertions, 3 deletions

diff --git a/buildscripts/resmokelib/testing/fixtures/fixturelib.py b/buildscripts/resmokelib/testing/fixtures/fixturelib.py
index e163340f3a6..b29a72850c6 100644
--- a/buildscripts/resmokelib/testing/fixtures/fixturelib.py
+++ b/buildscripts/resmokelib/testing/fixtures/fixturelib.py
@@ -1,4 +1,5 @@
 """Facade wrapping the resmokelib dependencies used by fixtures."""
+from typing import Dict
 
 from buildscripts.resmokelib import config
 from buildscripts.resmokelib import core
@@ -6,6 +7,7 @@ from buildscripts.resmokelib import errors
 from buildscripts.resmokelib import utils
 from buildscripts.resmokelib import logging
 from buildscripts.resmokelib.core import network
+from buildscripts.resmokelib.utils.dictionary import merge_dicts
 from buildscripts.resmokelib.utils.history import make_historic as _make_historic
 from buildscripts.resmokelib.testing.fixtures import _builder
 from buildscripts.resmokelib.multiversionconstants import LAST_LTS_MONGOD_BINARY, LAST_LTS_MONGOS_BINARY
@@ -95,6 +97,23 @@ class FixtureLib(object):
         """Return the next available port that fixture can use."""
         return network.PortAllocator.next_fixture_port(job_num)
 
+    SET_PARAMETERS_KEY = "set_parameters"
+
+    def merge_mongo_option_dicts(self, original: Dict, override: Dict):
+        """
+        Merge mongod/s options such that --setParameter is merged recursively.
+
+        Values from `original` are replaced in-place with those of `override` where they exist.
+        """
+        original_set_parameters = original.get(self.SET_PARAMETERS_KEY, {})
+        override_set_parameters = override.get(self.SET_PARAMETERS_KEY, {})
+
+        merged_set_parameters = merge_dicts(original_set_parameters, override_set_parameters)
+        original.update(override)
+        original[self.SET_PARAMETERS_KEY] = merged_set_parameters
+
+        return original
+
 
 class _FixtureConfig(object):  # pylint: disable=too-many-instance-attributes
     """Class that stores fixture configuration info."""
diff --git a/buildscripts/resmokelib/testing/fixtures/shardedcluster.py b/buildscripts/resmokelib/testing/fixtures/shardedcluster.py
index 5c68f437659..821cfc59dc6 100644
--- a/buildscripts/resmokelib/testing/fixtures/shardedcluster.py
+++ b/buildscripts/resmokelib/testing/fixtures/shardedcluster.py
@@ -298,7 +298,8 @@ class ShardedClusterFixture(interface.Fixture):  # pylint: disable=too-many-inst
         replset_config_options["configsvr"] = True
 
         mongod_options = self.mongod_options.copy()
-        mongod_options.update(
+        mongod_options = self.fixturelib.merge_mongo_option_dicts(
+            mongod_options,
             self.fixturelib.make_historic(configsvr_options.pop("mongod_options", {})))
         mongod_options["configsvr"] = ""
         mongod_options["dbpath"] = os.path.join(self._dbpath_prefix, "config")
@@ -340,8 +341,8 @@ class ShardedClusterFixture(interface.Fixture):  # pylint: disable=too-many-inst
                                                     num_rs_nodes_per_shard]
 
         mongod_options = self.mongod_options.copy()
-        mongod_options.update(
-            self.fixturelib.make_historic(shard_options.pop("mongod_options", {})))
+        mongod_options = self.fixturelib.merge_mongo_option_dicts(
+            mongod_options, self.fixturelib.make_historic(shard_options.pop("mongod_options", {})))
         mongod_options["shardsvr"] = ""
         mongod_options["dbpath"] = os.path.join(self._dbpath_prefix, "shard{}".format(index))
         mongod_options["replSet"] = ShardedClusterFixture._SHARD_REPLSET_NAME_PREFIX + str(index)
diff --git a/buildscripts/resmokelib/utils/dictionary.py b/buildscripts/resmokelib/utils/dictionary.py
new file mode 100644
index 00000000000..2037ca7ff5b
--- /dev/null
+++ b/buildscripts/resmokelib/utils/dictionary.py
@@ -0,0 +1,17 @@
+"""Utility functions for working with Dict-type structures."""
+from typing import MutableMapping
+
+
+def merge_dicts(dict1, dict2):
+    """Recursively merges dict2 into dict1."""
+    if not (isinstance(dict1, MutableMapping) and isinstance(dict2, MutableMapping)):
+        return dict2
+
+    for k in dict2.keys():
+        if dict2[k] is None:
+            dict1.pop(k)
+        elif k in dict1:
+            dict1[k] = merge_dicts(dict1[k], dict2[k])
+        else:
+            dict1[k] = dict2[k]
+    return dict1
diff --git a/buildscripts/tests/resmokelib/testing/fixtures/test_fixturelib.py b/buildscripts/tests/resmokelib/testing/fixtures/test_fixturelib.py
new file mode 100644
index 00000000000..1fcc848dc90
--- /dev/null
+++ b/buildscripts/tests/resmokelib/testing/fixtures/test_fixturelib.py
@@ -0,0 +1,65 @@
+"""Unittest for the resmokelib.testing.fixturelib.utils module"""
+
+import copy
+import unittest
+
+# pylint: disable=missing-docstring,protected-access
+from buildscripts.resmokelib.testing.fixtures.fixturelib import FixtureLib
+
+
+class TestMergeMongoOptionDicts(unittest.TestCase):
+    def setUp(self) -> None:
+        self.under_test = FixtureLib()
+
+    def test_merge_empty(self):  # pylint: disable=no-self-use
+        original = {
+            "dbpath": "value0", self.under_test.SET_PARAMETERS_KEY: {
+                "param1": "value1",
+                "param2": "value2",
+            }
+        }
+
+        override = {}
+        merged = self.under_test.merge_mongo_option_dicts(copy.deepcopy(original), override)
+
+        self.assertDictEqual(merged, original)
+
+    def test_merge_non_params(self):  # pylint: disable=no-self-use
+        non_param1_key = "non_param1"
+        non_param2_key = "non_param2"
+        original = {
+            non_param1_key: "value0", non_param2_key: {"nested_param1": "value0", },
+            self.under_test.SET_PARAMETERS_KEY: {"param1": "value1", }
+        }
+
+        override = {
+            non_param1_key: "value1",
+            non_param2_key: "value1",
+        }
+
+        self.under_test.merge_mongo_option_dicts(original, override)
+
+        expected = {
+            non_param1_key: "value1", non_param2_key: "value1",
+            self.under_test.SET_PARAMETERS_KEY: {"param1": "value1", }
+        }
+        self.assertEqual(original, expected)
+
+    def test_merge_params(self):  # pylint: disable=no-self-use
+        original = {
+            "dbpath": "value", self.under_test.SET_PARAMETERS_KEY: {
+                "param1": "value",
+                "param2": {"param3": "value", },
+            }
+        }
+
+        override = {self.under_test.SET_PARAMETERS_KEY: {"param2": {"param3": {"param4": "value"}}}}
+        self.under_test.merge_mongo_option_dicts(original, override)
+
+        expected = {
+            "dbpath": "value", self.under_test.SET_PARAMETERS_KEY: {
+                "param1": "value", "param2": {"param3": {"param4": "value"}}
+            }
+        }
+
+        self.assertDictEqual(original, expected)