SERVER-41020 Add tests that fuzz storage engine parameters

5 files changed, 162 insertions, 2 deletions

diff --git a/buildscripts/resmokelib/config.py b/buildscripts/resmokelib/config.py
index 27dfe5a828a..a8d16931527 100644
--- a/buildscripts/resmokelib/config.py
+++ b/buildscripts/resmokelib/config.py
@@ -58,6 +58,8 @@ DEFAULTS = {
     "exclude_with_any_tags": None,
     "flow_control": None,
     "flow_control_tickets": None,
+    "fuzz_mongod_configs": False,
+    "config_fuzz_seed": None,
     "genny_executable": None,
     "include_with_any_tags": None,
     "install_dir": None,
@@ -303,6 +305,9 @@ EXCLUDED_TAG = "__TEMPORARILY_DISABLED__"
 # If true, then a test failure or error will cause resmoke.py to exit and not run any more tests.
 FAIL_FAST = None
 
+FUZZ_MONGOD_CONFIGS = False
+CONFIG_FUZZ_SEED = None
+
 # Executable file for genny, passed in as a command line arg.
 GENNY_EXECUTABLE = None
 
diff --git a/buildscripts/resmokelib/configure_resmoke.py b/buildscripts/resmokelib/configure_resmoke.py
index 8dc953bb535..3707a0178db 100644
--- a/buildscripts/resmokelib/configure_resmoke.py
+++ b/buildscripts/resmokelib/configure_resmoke.py
@@ -8,11 +8,13 @@ import os.path
 import distutils.spawn
 import sys
 import platform
+import random
 
 import pymongo.uri_parser
 
 from buildscripts.resmokelib import config as _config
 from buildscripts.resmokelib import utils
+from buildscripts.resmokelib import mongod_fuzzer_configs
 
 
 def validate_and_update_config(parser, args):
@@ -181,6 +183,15 @@ def _update_config_vars(values):  # pylint: disable=too-many-statements,too-many
 
     _config.MONGOD_EXECUTABLE = _expand_user(config.pop("mongod_executable"))
     _config.MONGOD_SET_PARAMETERS = _merge_set_params(config.pop("mongod_set_parameters"))
+    _config.FUZZ_MONGOD_CONFIGS = config.pop("fuzz_mongod_configs")
+    _config.CONFIG_FUZZ_SEED = config.pop("config_fuzz_seed")
+
+    if _config.FUZZ_MONGOD_CONFIGS:
+        if not _config.CONFIG_FUZZ_SEED:
+            _config.CONFIG_FUZZ_SEED = random.randrange(sys.maxsize)
+        _config.MONGOD_SET_PARAMETERS, _config.WT_ENGINE_CONFIG = mongod_fuzzer_configs \
+            .fuzz_set_parameters(_config.CONFIG_FUZZ_SEED, _config.MONGOD_SET_PARAMETERS)
+
     _config.MONGOS_EXECUTABLE = _expand_user(config.pop("mongos_executable"))
     _config.MONGOS_SET_PARAMETERS = _merge_set_params(config.pop("mongos_set_parameters"))
 
@@ -236,7 +247,9 @@ def _update_config_vars(values):  # pylint: disable=too-many-statements,too-many
 
     # Wiredtiger options.
     _config.WT_COLL_CONFIG = config.pop("wt_coll_config")
-    _config.WT_ENGINE_CONFIG = config.pop("wt_engine_config")
+    wt_engine_config = config.pop("wt_engine_config")
+    if wt_engine_config:  # prevents fuzzed wt_engine_config from being overwritten unless user specifies it
+        _config.WT_ENGINE_CONFIG = config.pop("wt_engine_config")
     _config.WT_INDEX_CONFIG = config.pop("wt_index_config")
 
     # Benchmark/Benchrun options.
diff --git a/buildscripts/resmokelib/mongod_fuzzer_configs.py b/buildscripts/resmokelib/mongod_fuzzer_configs.py
new file mode 100644
index 00000000000..3277cb6a5b8
--- /dev/null
+++ b/buildscripts/resmokelib/mongod_fuzzer_configs.py
@@ -0,0 +1,71 @@
+"""Generator functions for all parameters that we fuzz when invoked with --fuzzMongodConfigs."""
+
+import random
+from buildscripts.resmokelib import utils
+
+
+def generate_eviction_configs(rng):
+    """Generate random configurations for wiredTigerEngineConfigString parameter."""
+    eviction_checkpoint_target = rng.randint(1, 99)
+    eviction_target = rng.randint(50, 95)
+    eviction_trigger = rng.randint(eviction_target + 1, 99)
+    eviction_dirty_target = rng.randint(5, 50)
+    eviction_dirty_trigger = rng.randint(eviction_dirty_target, 75)
+
+    close_idle_time_secs = rng.randint(1, 100)
+    close_handle_minimum = rng.randint(0, 1000)
+    close_scan_interval = rng.randint(1, 100)
+
+    return "eviction_checkpoint_target={0},eviction_dirty_target={1},eviction_dirty_trigger={2},"\
+           "eviction_target={3},eviction_trigger={4},file_manager=(close_handle_minimum={5},"\
+           "close_idle_time={6},close_scan_interval={7})".format(eviction_checkpoint_target,
+                                                                 eviction_dirty_target,
+                                                                 eviction_dirty_trigger,
+                                                                 eviction_target,
+                                                                 eviction_trigger,
+                                                                 close_handle_minimum,
+                                                                 close_idle_time_secs,
+                                                                 close_scan_interval)
+
+
+def generate_flow_control_parameters(rng):
+    """Generate parameters related to flow control and returns a dictionary."""
+    configs = {}
+    configs["enableFlowControl"] = rng.choice([True, False])
+    if not configs["enableFlowControl"]:
+        return configs
+
+    configs["flowControlTargetLagSeconds"] = rng.randint(1, 1000)
+    configs["flowControlThresholdLagPercentage"] = rng.random()
+    configs["flowControlMaxSamples"] = rng.randint(1, 1000 * 1000)
+    configs["flowControlSamplePeriod"] = rng.randint(1, 1000 * 1000)
+    configs["flowControlMinTicketsPerSecond"] = rng.randint(1, 10 * 1000)
+
+    return configs
+
+
+def generate_independent_parameters(rng):
+    """Return a dictionary with values for each independent parameter."""
+    ret = {}
+    ret["wiredTigerCursorCacheSize"] = rng.randint(-100, 100)
+    ret["wiredTigerSessionCloseIdleTimeSecs"] = rng.randint(0, 300)
+    ret["wiredTigerConcurrentWriteTransactions"] = rng.randint(16, 256)
+    ret["wiredTigerConcurrentReadTransactions"] = rng.randint(16, 256)
+
+    return ret
+
+
+def fuzz_set_parameters(seed, user_provided_params):
+    """Randomly generate mongod configurations and wiredTigerConnectionString."""
+    rng = random.Random(seed)
+
+    ret = {}
+    params = [generate_flow_control_parameters(rng), generate_independent_parameters(rng)]
+    for dct in params:
+        for key, value in dct.items():
+            ret[key] = value
+
+    for key, value in utils.load_yaml(user_provided_params).items():
+        ret[key] = value
+
+    return utils.dump_yaml(ret), generate_eviction_configs(rng)
diff --git a/buildscripts/resmokelib/run/__init__.py b/buildscripts/resmokelib/run/__init__.py
index 6026d5951e0..7603debfcce 100644
--- a/buildscripts/resmokelib/run/__init__.py
+++ b/buildscripts/resmokelib/run/__init__.py
@@ -167,7 +167,18 @@ class TestRunner(Subcommand):  # pylint: disable=too-many-instance-attributes
         self._resmoke_logger.info("verbatim resmoke.py invocation: %s",
                                   " ".join([shlex.quote(arg) for arg in sys.argv]))
 
-        if config.EVERGREEN_TASK_ID:
+        if config.FUZZ_MONGOD_CONFIGS:
+            local_args = to_local_args()
+            local_args = strip_fuzz_config_params(local_args)
+            self._resmoke_logger.info(
+                "resmoke.py invocation for local usage: %s %s %s",
+                os.path.join("buildscripts", "resmoke.py"), " ".join(local_args),
+                "--fuzzMongodConfigs --configFuzzSeed=" + str(config.CONFIG_FUZZ_SEED))
+            self._resmoke_logger.info("Fuzzed mongodSetParameters:\n%s",
+                                      config.MONGOD_SET_PARAMETERS)
+            self._resmoke_logger.info("Fuzzed wiredTigerConnectionString: %s",
+                                      config.WT_ENGINE_CONFIG)
+        elif config.EVERGREEN_TASK_ID:
             local_args = to_local_args()
             self._resmoke_logger.info("resmoke.py invocation for local usage: %s %s",
                                       os.path.join("buildscripts", "resmoke.py"),
@@ -672,6 +683,12 @@ class RunPlugin(PluginInterface):
         parser.add_argument("--mongod", dest="mongod_executable", metavar="PATH",
                             help="The path to the mongod executable for resmoke.py to use.")
 
+        parser.add_argument("--fuzzMongodConfigs", dest="fuzz_mongod_configs", action="store_true",
+                            help="Will randomly choose storage configs that were not specified.")
+
+        parser.add_argument("--configFuzzSeed", dest="config_fuzz_seed", metavar="PATH",
+                            help="Sets the seed used by storage config fuzzer")
+
         parser.add_argument(
             "--mongodSetParameters", dest="mongod_set_parameters", action="append",
             metavar="{key1: value1, key2: value2, ..., keyN: valueN}",
@@ -1131,3 +1148,13 @@ def to_local_args(input_args=None):  # pylint: disable=too-many-branches,too-man
 
     return ["run"] + [arg for arg in (suites_arg, storage_engine_arg) if arg is not None
                       ] + other_local_args + positional_args
+
+
+def strip_fuzz_config_params(input_args):
+    """Delete fuzz related command line args because we have to add the --fuzzSeed manually."""
+    ret = []
+    for arg in input_args:
+        if "--fuzzMongodConfigs" not in arg and "-fuzzSeed" not in arg:
+            ret.append(arg)
+
+    return ret
diff --git a/etc/evergreen.yml b/etc/evergreen.yml
index dd3ecdbbd8d..de9b2d854f1 100644
--- a/etc/evergreen.yml
+++ b/etc/evergreen.yml
@@ -5751,6 +5751,42 @@ tasks:
       resmoke_args: --suites=core --storageEngine=wiredTiger
 
 - <<: *task_template
+  name: config_fuzzer_jsCore
+  tags: []
+  commands:
+  - func: "do setup"
+  - func: "run tests"
+    vars:
+      resmoke_args: --suites=core --storageEngine=wiredTiger --fuzzMongodConfigs
+
+- <<: *task_template
+  name: config_fuzzer_concurrency
+  tags: []
+  commands:
+  - func: "do setup"
+  - func: "run tests"
+    vars:
+      resmoke_args: --suites=concurrency --storageEngine=wiredTiger --fuzzMongodConfigs
+
+- <<: *task_template
+  name: config_fuzzer_concurrency_replication
+  tags: []
+  commands:
+  - func: "do setup"
+  - func: "run tests"
+    vars:
+      resmoke_args: --suites=concurrency_replication --storageEngine=wiredTiger --fuzzMongodConfigs
+
+- <<: *task_template
+  name: config_fuzzer_replica_sets_jscore_passthrough
+  tags: []
+  commands:
+  - func: "do setup"
+  - func: "run tests"
+    vars:
+      resmoke_args: --suites=replica_sets_jscore_passthrough --storageEngine=wiredTiger --fuzzMongodConfigs
+
+- <<: *task_template
   name: jsCore_ese
   tags: ["jscore", "encrypt"]
   depends_on:
@@ -11105,6 +11141,14 @@ buildvariants:
   - name: auth_audit_gen
   - name: auth_gen
   - name: causally_consistent_jscore_txns_passthrough
+  - name: config_fuzzer_concurrency
+  - name: config_fuzzer_concurrency_replication
+    distros:
+    - rhel70-large
+  - name: config_fuzzer_jsCore
+  - name: config_fuzzer_replica_sets_jscore_passthrough
+    distros:
+    - rhel70-large
   - name: .encrypt !.sharding !.replica_sets !.aggregation !.jscore
   - name: external_auth
   - name: external_auth_aws