SERVER-61460 correctly merge --setParameter in resmoke

(cherry picked from commit 876e0930d208a9b29c2aa02b291a0d57db678f8f)

5 files changed, 108 insertions, 21 deletions

diff --git a/buildscripts/resmokelib/suitesconfig.py b/buildscripts/resmokelib/suitesconfig.py
index bc254dd3bde..e82cd697e8b 100644
--- a/buildscripts/resmokelib/suitesconfig.py
+++ b/buildscripts/resmokelib/suitesconfig.py
@@ -10,6 +10,7 @@ from buildscripts.resmokelib import errors
 from buildscripts.resmokelib import utils
 from buildscripts.resmokelib.testing import suite as _suite
 from buildscripts.resmokelib.utils import load_yaml_file
+from buildscripts.resmokelib.utils.dictionary import merge_dicts
 
 SuiteName = str
 
@@ -39,8 +40,7 @@ def get_named_suites() -> List[SuiteName]:
 
 def get_suite_files() -> Dict[str, str]:
     """Get the physical files defining these suites for parsing comments."""
-    return MatrixSuiteConfig.merge_dicts(ExplicitSuiteConfig.get_suite_files(),
-                                         MatrixSuiteConfig.get_suite_files())
+    return merge_dicts(ExplicitSuiteConfig.get_suite_files(), MatrixSuiteConfig.get_suite_files())
 
 
 def create_test_membership_map(fail_on_missing_selector=False, test_kind=None):
@@ -237,7 +237,7 @@ class MatrixSuiteConfig(SuiteConfigInterface):
 
         if override_names:
             for override_name in override_names:
-                cls.merge_dicts(res, overrides[override_name])
+                merge_dicts(res, overrides[override_name])
 
         return res
 
@@ -291,20 +291,6 @@ class MatrixSuiteConfig(SuiteConfigInterface):
         return cls._all_mappings
 
     @classmethod
-    def merge_dicts(cls, dict1, dict2):
-        """Recursively merges dict2 into dict1."""
-        if not isinstance(dict1, dict) or not isinstance(dict2, dict):
-            return dict2
-        for k in dict2:
-            if dict2[k] is None:
-                dict1.pop(k)
-            elif k in dict1:
-                dict1[k] = cls.merge_dicts(dict1[k], dict2[k])
-            else:
-                dict1[k] = dict2[k]
-        return dict1
-
-    @classmethod
     def __get_suite_files_in_dir(cls, target_dir):
         """Get the physical files defining these suites for parsing comments."""
         root = os.path.abspath(target_dir)
diff --git a/buildscripts/resmokelib/testing/fixtures/fixturelib.py b/buildscripts/resmokelib/testing/fixtures/fixturelib.py
index d5c715b6b37..ef298296562 100644
--- a/buildscripts/resmokelib/testing/fixtures/fixturelib.py
+++ b/buildscripts/resmokelib/testing/fixtures/fixturelib.py
@@ -1,4 +1,5 @@
 """Facade wrapping the resmokelib dependencies used by fixtures."""
+from typing import Dict
 
 from buildscripts.resmokelib import config
 from buildscripts.resmokelib import core
@@ -6,6 +7,7 @@ from buildscripts.resmokelib import errors
 from buildscripts.resmokelib import utils
 from buildscripts.resmokelib import logging
 from buildscripts.resmokelib.core import network
+from buildscripts.resmokelib.utils.dictionary import merge_dicts
 from buildscripts.resmokelib.utils.history import make_historic as _make_historic
 from buildscripts.resmokelib.testing.fixtures import _builder
 
@@ -94,6 +96,23 @@ class FixtureLib:
         """Return the next available port that fixture can use."""
         return network.PortAllocator.next_fixture_port(job_num)
 
+    SET_PARAMETERS_KEY = "set_parameters"
+
+    def merge_mongo_option_dicts(self, original: Dict, override: Dict):
+        """
+        Merge mongod/s options such that --setParameter is merged recursively.
+
+        Values from `original` are replaced in-place with those of `override` where they exist.
+        """
+        original_set_parameters = original.get(self.SET_PARAMETERS_KEY, {})
+        override_set_parameters = override.get(self.SET_PARAMETERS_KEY, {})
+
+        merged_set_parameters = merge_dicts(original_set_parameters, override_set_parameters)
+        original.update(override)
+        original[self.SET_PARAMETERS_KEY] = merged_set_parameters
+
+        return original
+
 
 class _FixtureConfig(object):  # pylint: disable=too-many-instance-attributes
     """Class that stores fixture configuration info."""
diff --git a/buildscripts/resmokelib/testing/fixtures/shardedcluster.py b/buildscripts/resmokelib/testing/fixtures/shardedcluster.py
index 354dfd290f0..60d0750e7e7 100644
--- a/buildscripts/resmokelib/testing/fixtures/shardedcluster.py
+++ b/buildscripts/resmokelib/testing/fixtures/shardedcluster.py
@@ -8,7 +8,6 @@ import pymongo
 import pymongo.errors
 
 import buildscripts.resmokelib.testing.fixtures.interface as interface
-import buildscripts.resmokelib.utils.registry as registry
 
 
 class ShardedClusterFixture(interface.Fixture):  # pylint: disable=too-many-instance-attributes
@@ -281,7 +280,8 @@ class ShardedClusterFixture(interface.Fixture):  # pylint: disable=too-many-inst
         replset_config_options["configsvr"] = True
 
         mongod_options = self.mongod_options.copy()
-        mongod_options.update(
+        mongod_options = self.fixturelib.merge_mongo_option_dicts(
+            mongod_options,
             self.fixturelib.make_historic(configsvr_options.pop("mongod_options", {})))
         mongod_options["configsvr"] = ""
         mongod_options["dbpath"] = os.path.join(self._dbpath_prefix, "config")
@@ -321,8 +321,8 @@ class ShardedClusterFixture(interface.Fixture):  # pylint: disable=too-many-inst
         replset_config_options["configsvr"] = False
 
         mongod_options = self.mongod_options.copy()
-        mongod_options.update(
-            self.fixturelib.make_historic(shard_options.pop("mongod_options", {})))
+        mongod_options = self.fixturelib.merge_mongo_option_dicts(
+            mongod_options, self.fixturelib.make_historic(shard_options.pop("mongod_options", {})))
         mongod_options["shardsvr"] = ""
         mongod_options["dbpath"] = os.path.join(self._dbpath_prefix, "shard{}".format(index))
         mongod_options["replSet"] = self._SHARD_REPLSET_NAME_PREFIX + str(index)
diff --git a/buildscripts/resmokelib/utils/dictionary.py b/buildscripts/resmokelib/utils/dictionary.py
new file mode 100644
index 00000000000..2037ca7ff5b
--- /dev/null
+++ b/buildscripts/resmokelib/utils/dictionary.py
@@ -0,0 +1,17 @@
+"""Utility functions for working with Dict-type structures."""
+from typing import MutableMapping
+
+
+def merge_dicts(dict1, dict2):
+    """Recursively merges dict2 into dict1."""
+    if not (isinstance(dict1, MutableMapping) and isinstance(dict2, MutableMapping)):
+        return dict2
+
+    for k in dict2.keys():
+        if dict2[k] is None:
+            dict1.pop(k)
+        elif k in dict1:
+            dict1[k] = merge_dicts(dict1[k], dict2[k])
+        else:
+            dict1[k] = dict2[k]
+    return dict1
diff --git a/buildscripts/tests/resmokelib/testing/fixtures/test_fixturelib.py b/buildscripts/tests/resmokelib/testing/fixtures/test_fixturelib.py
new file mode 100644
index 00000000000..1fcc848dc90
--- /dev/null
+++ b/buildscripts/tests/resmokelib/testing/fixtures/test_fixturelib.py
@@ -0,0 +1,65 @@
+"""Unittest for the resmokelib.testing.fixturelib.utils module"""
+
+import copy
+import unittest
+
+# pylint: disable=missing-docstring,protected-access
+from buildscripts.resmokelib.testing.fixtures.fixturelib import FixtureLib
+
+
+class TestMergeMongoOptionDicts(unittest.TestCase):
+    def setUp(self) -> None:
+        self.under_test = FixtureLib()
+
+    def test_merge_empty(self):  # pylint: disable=no-self-use
+        original = {
+            "dbpath": "value0", self.under_test.SET_PARAMETERS_KEY: {
+                "param1": "value1",
+                "param2": "value2",
+            }
+        }
+
+        override = {}
+        merged = self.under_test.merge_mongo_option_dicts(copy.deepcopy(original), override)
+
+        self.assertDictEqual(merged, original)
+
+    def test_merge_non_params(self):  # pylint: disable=no-self-use
+        non_param1_key = "non_param1"
+        non_param2_key = "non_param2"
+        original = {
+            non_param1_key: "value0", non_param2_key: {"nested_param1": "value0", },
+            self.under_test.SET_PARAMETERS_KEY: {"param1": "value1", }
+        }
+
+        override = {
+            non_param1_key: "value1",
+            non_param2_key: "value1",
+        }
+
+        self.under_test.merge_mongo_option_dicts(original, override)
+
+        expected = {
+            non_param1_key: "value1", non_param2_key: "value1",
+            self.under_test.SET_PARAMETERS_KEY: {"param1": "value1", }
+        }
+        self.assertEqual(original, expected)
+
+    def test_merge_params(self):  # pylint: disable=no-self-use
+        original = {
+            "dbpath": "value", self.under_test.SET_PARAMETERS_KEY: {
+                "param1": "value",
+                "param2": {"param3": "value", },
+            }
+        }
+
+        override = {self.under_test.SET_PARAMETERS_KEY: {"param2": {"param3": {"param4": "value"}}}}
+        self.under_test.merge_mongo_option_dicts(original, override)
+
+        expected = {
+            "dbpath": "value", self.under_test.SET_PARAMETERS_KEY: {
+                "param1": "value", "param2": {"param3": {"param4": "value"}}
+            }
+        }
+
+        self.assertDictEqual(original, expected)