SERVER-38110 SERVER-38113: Generate resmoke config and evg config for a sub-suite

20 files changed, 477 insertions, 640 deletions

diff --git a/.gitignore b/.gitignore
index efb56426b11..a1564ef2888 100644
--- a/.gitignore
+++ b/.gitignore
@@ -169,3 +169,4 @@ src/mongo/embedded/java/jar/build/
 local.properties
 
 compile_commands.json
+generated_resmoke_config
diff --git a/buildscripts/evergreen_gen_fuzzer_tests.py b/buildscripts/evergreen_gen_fuzzer_tests.py
index c22df9c972d..161802fa4e4 100755
--- a/buildscripts/evergreen_gen_fuzzer_tests.py
+++ b/buildscripts/evergreen_gen_fuzzer_tests.py
@@ -9,14 +9,15 @@ import os
 
 from collections import namedtuple
 
-import yaml
-
 from shrub.config import Configuration
 from shrub.command import CommandDefinition
 from shrub.task import TaskDependency
 from shrub.variant import DisplayTaskDefinition
 from shrub.variant import TaskSpec
 
+import util.read_config as read_config
+import util.taskname as taskname
+
 CONFIG_DIRECTORY = "generated_resmoke_config"
 
 ConfigOptions = namedtuple("ConfigOptions", [
@@ -35,34 +36,6 @@ ConfigOptions = namedtuple("ConfigOptions", [
 ])
 
 
-def _get_config_value(attrib, cmd_line_options, config_file_data, required=False, default=None):
-    """
-    Get the configuration value to use.
-
-    First use command line options, then config file option, then the default. If required is
-    true, throw an exception if the value is not found.
-
-    :param attrib: Attribute to search for.
-    :param cmd_line_options: Command line options.
-    :param config_file_data: Config file data.
-    :param required: Is this option required.
-    :param default: Default value if option is not found.
-    :return: value to use for this option.
-    """
-    value = getattr(cmd_line_options, attrib, None)
-    if value:
-        return value
-
-    value = config_file_data.get(attrib)
-    if value:
-        return value
-
-    if required:
-        raise ValueError("{0} must be specified".format(attrib))
-
-    return default
-
-
 def _get_config_options(cmd_line_options, config_file):  # pylint: disable=too-many-locals
     """
     Get the configuration to use.
@@ -73,32 +46,33 @@ def _get_config_options(cmd_line_options, config_file):  # pylint: disable=too-m
     :param config_file: config file to use.
     :return: ConfigOptions to use.
     """
-    config_file_data = {}
-    if config_file:
-        with open(config_file) as file_handle:
-            config_file_data = yaml.load(file_handle)
+    config_file_data = read_config.read_config_file(config_file)
 
     num_files = int(
-        _get_config_value("num_files", cmd_line_options, config_file_data, required=True))
+        read_config.get_config_value("num_files", cmd_line_options, config_file_data,
+                                     required=True))
     num_tasks = int(
-        _get_config_value("num_tasks", cmd_line_options, config_file_data, required=True))
-    resmoke_args = _get_config_value("resmoke_args", cmd_line_options, config_file_data, default="")
-    npm_command = _get_config_value("npm_command", cmd_line_options, config_file_data,
-                                    default="jstestfuzz")
-    jstestfuzz_vars = _get_config_value("jstestfuzz_vars", cmd_line_options, config_file_data,
-                                        default="")
-    name = _get_config_value("name", cmd_line_options, config_file_data, required=True)
-    variant = _get_config_value("build_variant", cmd_line_options, config_file_data, required=True)
-    continue_on_failure = _get_config_value("continue_on_failure", cmd_line_options,
-                                            config_file_data, default="false")
-    resmoke_jobs_max = _get_config_value("resmoke_jobs_max", cmd_line_options, config_file_data,
-                                         default="0")
-    should_shuffle = _get_config_value("should_shuffle", cmd_line_options, config_file_data,
-                                       default="false")
-    timeout_secs = _get_config_value("timeout_secs", cmd_line_options, config_file_data,
-                                     default="1800")
-    use_multiversion = _get_config_value("task_path_suffix", cmd_line_options, config_file_data,
-                                         default=False)
+        read_config.get_config_value("num_tasks", cmd_line_options, config_file_data,
+                                     required=True))
+    resmoke_args = read_config.get_config_value("resmoke_args", cmd_line_options, config_file_data,
+                                                default="")
+    npm_command = read_config.get_config_value("npm_command", cmd_line_options, config_file_data,
+                                               default="jstestfuzz")
+    jstestfuzz_vars = read_config.get_config_value("jstestfuzz_vars", cmd_line_options,
+                                                   config_file_data, default="")
+    name = read_config.get_config_value("name", cmd_line_options, config_file_data, required=True)
+    variant = read_config.get_config_value("build_variant", cmd_line_options, config_file_data,
+                                           required=True)
+    continue_on_failure = read_config.get_config_value("continue_on_failure", cmd_line_options,
+                                                       config_file_data, default="false")
+    resmoke_jobs_max = read_config.get_config_value("resmoke_jobs_max", cmd_line_options,
+                                                    config_file_data, default="0")
+    should_shuffle = read_config.get_config_value("should_shuffle", cmd_line_options,
+                                                  config_file_data, default="false")
+    timeout_secs = read_config.get_config_value("timeout_secs", cmd_line_options, config_file_data,
+                                                default="1800")
+    use_multiversion = read_config.get_config_value("task_path_suffix", cmd_line_options,
+                                                    config_file_data, default=False)
 
     return ConfigOptions(num_files, num_tasks, resmoke_args, npm_command, jstestfuzz_vars, name,
                          variant, continue_on_failure, resmoke_jobs_max, should_shuffle,
@@ -131,7 +105,8 @@ def _generate_evg_tasks(options):
     task_specs = []
 
     for task_index in range(options.num_tasks):
-        name = _name_task(options.name, task_index, options.num_tasks) + "_" + options.variant
+        name = taskname.name_generated_task(options.name, task_index, options.num_tasks,
+                                            options.variant)
         task_names.append(name)
         task_specs.append(TaskSpec(name))
         task = evg_config.task(name)
diff --git a/buildscripts/generate_resmoke_suites.py b/buildscripts/generate_resmoke_suites.py
index 6268de9f258..dba1922a584 100755
--- a/buildscripts/generate_resmoke_suites.py
+++ b/buildscripts/generate_resmoke_suites.py
@@ -14,29 +14,49 @@ import itertools
 import logging
 import os
 import sys
-
 from collections import defaultdict
 from collections import namedtuple
 from operator import itemgetter
 
-from jinja2 import Template
+import yaml
+
+from shrub.config import Configuration
+from shrub.command import CommandDefinition
+from shrub.task import TaskDependency
+from shrub.variant import DisplayTaskDefinition
+from shrub.variant import TaskSpec
 
 from client.github import GithubApi
 
 import client.evergreen as evergreen
+import util.read_config as read_config
+import util.taskname as taskname
 import util.testname as testname
 import util.time as timeutil
 
 LOGGER = logging.getLogger(__name__)
 
-TEMPLATES_DIR = "buildscripts/templates/generate_resmoke_suites"
-TEST_SUITE_DIR = "buildscripts/resmokeconfig/suites"
-
 MAX_RUNTIME_KEY = "max_runtime"
+TEST_SUITE_DIR = os.path.join("buildscripts", "resmokeconfig", "suites")
+CONFIG_DIR = "generated_resmoke_config"
+
+HEADER_TEMPLATE = """# DO NOT EDIT THIS FILE. All manual edits will be lost.
+# This file was generated by {file} from
+# {suite_file}.
+"""
 
 CommitRange = namedtuple("CommitRange", ["start", "end"])
 ProjectTarget = namedtuple("ProjectTarget", ["owner", "project", "branch"])
 Dependencies = namedtuple("Dependencies", ["evergreen", "github"])
+ConfigOptions = namedtuple("ConfigOptions", [
+    "max_sub_suites",
+    "resmoke_args",
+    "resmoke_jobs_max",
+    "run_multiple_jobs",
+    "suite",
+    "task",
+    "variant",
+])
 
 
 def enable_logging():
@@ -49,6 +69,35 @@ def enable_logging():
     )
 
 
+def get_config_options(cmd_line_options, config_file):
+    """
+    Get the configuration to use for generated tests.
+
+    Command line options override config file options.
+
+    :param cmd_line_options: Command line options specified.
+    :param config_file: config file to use.
+    :return: ConfigOptions to use.
+    """
+    config_file_data = read_config.read_config_file(config_file)
+
+    max_sub_suites = read_config.get_config_value("max_sub_suites", cmd_line_options,
+                                                  config_file_data)
+    resmoke_args = read_config.get_config_value("resmoke_args", cmd_line_options, config_file_data,
+                                                default="")
+    resmoke_jobs_max = read_config.get_config_value("resmoke_jobs_max", cmd_line_options,
+                                                    config_file_data)
+    run_multiple_jobs = read_config.get_config_value("run_multiple_jobs", cmd_line_options,
+                                                     config_file_data, default="true")
+    task = read_config.get_config_value("task", cmd_line_options, config_file_data, required=True)
+    suite = read_config.get_config_value("suite", cmd_line_options, config_file_data, default=task)
+    variant = read_config.get_config_value("build_variant", cmd_line_options, config_file_data,
+                                           required=True)
+
+    return ConfigOptions(max_sub_suites, resmoke_args, resmoke_jobs_max, run_multiple_jobs, suite,
+                         task, variant)
+
+
 def get_start_and_end_commit_since_date(github_api, target, start_date):
     """Get the first and last commits on the given branch from the start date specified."""
 
@@ -247,32 +296,88 @@ def divide_tests_into_suites_by_maxtime(tests, sorted_tests, max_time_seconds, m
     return suites
 
 
-def get_misc_model(test_list, extra_model_data=None):
-    """Build a model that will run any missing tests."""
-    model = {
-        "is_misc": True,
-        "excluded_tests": test_list,
-    }
+def generate_subsuite_file(source_suite_name, target_suite_name, roots=None, excludes=None):
+    """
+    Read and evaluate the yaml suite file.
+
+    Override selector.roots and selector.excludes with the provided values. Write the results to
+    target_suite_name.
+    """
+    source_file = os.path.join(TEST_SUITE_DIR, source_suite_name + ".yml")
+    with open(source_file, "r") as fstream:
+        suite_config = yaml.load(fstream)
+
+    with open(os.path.join(CONFIG_DIR, target_suite_name + ".yml"), 'w') as out:
+        out.write(HEADER_TEMPLATE.format(file=__file__, suite_file=source_file))
+        if roots:
+            suite_config['selector']['roots'] = roots
+        if excludes:
+            suite_config['selector']['exclude_files'] = excludes
+        out.write(yaml.dump(suite_config, default_flow_style=False, Dumper=yaml.SafeDumper))
+
+
+def render_suite(suites, suite_name):
+    """Render the given suites into yml files that can be used by resmoke.py."""
+    for idx, suite in enumerate(suites):
+        suite.name = taskname.name_generated_task(suite_name, idx, len(suites))
+        generate_subsuite_file(suite_name, suite.name, roots=suite.tests)
+
 
-    if extra_model_data:
-        model.update(extra_model_data)
+def render_misc_suite(test_list, suite_name):
+    """Render a misc suite to run any tests that might be added to the directory."""
+    subsuite_name = "{0}_{1}".format(suite_name, "misc")
+    generate_subsuite_file(suite_name, subsuite_name, excludes=test_list)
 
-    return model
 
+def prepare_directory_for_suite(directory):
+    """Ensure that dir exists."""
+    if not os.path.exists(directory):
+        os.makedirs(directory)
 
-def render_template(model, task, index):
-    """Render the specified model as a yml file in the test suites directory."""
-    template_file = "{dir}/{task}.yml.j2".format(dir=TEMPLATES_DIR, task=task)
-    target_file = "{dir}/{task}_{index}.yml".format(dir=TEST_SUITE_DIR, task=task, index=index)
 
-    render(model, template_file, target_file)
+def generate_evg_config(suites, options):
+    """Generate evergreen configuration for the given suites."""
+    evg_config = Configuration()
 
+    task_names = []
+    task_specs = []
 
-def render(model, source, destination):
-    """Render the specified model with the template at `source` to the file `destination`."""
-    with open(source, "r") as inp, open(destination, "w") as out:
-        template = Template(inp.read(), trim_blocks=True)
-        out.write(template.render(model))
+    def generate_task(sub_suite_name, sub_task_name):
+        """Generate evergreen config for a resmoke task."""
+        task_names.append(sub_task_name)
+        task_specs.append(TaskSpec(sub_task_name))
+        task = evg_config.task(sub_task_name)
+
+        target_suite_file = os.path.join(CONFIG_DIR, sub_suite_name)
+
+        run_tests_vars = {
+            "resmoke_args": "--suites={0} {1}".format(target_suite_file, options.resmoke_args),
+            "run_multiple_jobs": options.run_multiple_jobs,
+        }
+
+        if options.resmoke_jobs_max:
+            run_tests_vars["resmoke_jobs_max"] = options.resmoke_jobs_max
+
+        commands = [
+            CommandDefinition().function("do setup"),
+            CommandDefinition().function("run tests").vars(run_tests_vars)
+        ]
+        task.dependency(TaskDependency("compile")).commands(commands)
+
+    for idx, suite in enumerate(suites):
+        sub_task_name = taskname.name_generated_task(options.task, idx, len(suites),
+                                                     options.variant)
+        generate_task(suite.name, sub_task_name)
+
+    # Add the misc suite
+    misc_suite_name = "{0}_misc".format(options.suite)
+    generate_task(misc_suite_name, "{0}_misc_{1}".format(options.task, options.variant))
+
+    dt = DisplayTaskDefinition(options.task).execution_tasks(task_names) \
+        .execution_task("{0}_gen".format(options.task))
+    evg_config.variant(options.variant).tasks(task_specs).display_task(dt)
+
+    return evg_config
 
 
 class Suite(object):
@@ -283,6 +388,7 @@ class Suite(object):
         self.tests = []
         self.total_runtime = 0
         self.variant_runtime = defaultdict(int)
+        self._name = None
 
     def add_test(self, test_name, test_data):
         """Add the given test to this suite."""
@@ -304,19 +410,6 @@ class Suite(object):
 
         return len(self.tests)
 
-    def get_model(self, extra_model_data=None):
-        """Get a model of this suite that can be used to render a yml file."""
-
-        model = {"test_names": self.tests, "variants": []}
-        for variant in self.variant_runtime:
-            model["variants"].append(
-                {"name": variant, "runtime": self.variant_runtime[variant] / 60})
-
-        if extra_model_data:
-            model.update(extra_model_data)
-
-        return model
-
 
 class Main(object):
     """Orchestrate the execution of generate_resmoke_suites."""
@@ -325,6 +418,7 @@ class Main(object):
         """Initialize the object."""
         self.deps = deps
         self.options = {}
+        self.config_options = {}
         self.commit_range = None
         self.test_list = []
 
@@ -332,8 +426,10 @@ class Main(object):
         """Parse the command line options and return the parsed data."""
         parser = argparse.ArgumentParser(description=self.main.__doc__)
 
+        parser.add_argument("--expansion-file", dest="expansion_file", type=str,
+                            help="Location of expansions file generated by evergreen.")
         parser.add_argument("--analysis-duration", dest="duration_days", default=14,
-                            help="Number of days to analyze.")
+                            help="Number of days to analyze.", type=int)
         parser.add_argument("--branch", dest="branch", default="master",
                             help="Branch of project to analyze.")
         parser.add_argument("--end-commit", dest="end_commit", help="End analysis at this commit.")
@@ -350,9 +446,19 @@ class Main(object):
                             help="Comma-separated list of Evergreeen build variants to analyze.")
         parser.add_argument("--max-sub-suites", dest="max_sub_suites", type=int,
                             help="Max number of suites to divide into.")
+        parser.add_argument("--resmoke-args", dest="resmoke_args",
+                            help="Arguments to pass to resmoke calls.")
+        parser.add_argument("--resmoke-jobs_max", dest="resmoke_jobs_max",
+                            help="Number of resmoke jobs to invoke.")
+        parser.add_argument("--suite", dest="suite",
+                            help="Name of suite being split(defaults to task_name)")
+        parser.add_argument("--run-multiple-jobs", dest="run_multiple_jobs",
+                            help="Should resmoke run multiple jobs")
+        parser.add_argument("--task_name", dest="task", help="Name of task to split.")
+        parser.add_argument("--variant", dest="build_variant",
+                            help="Build variant being run against.")
         parser.add_argument("--verbose", dest="verbose", action="store_true", default=False,
                             help="Enable verbose logging.")
-        parser.add_argument("task", nargs=1, help="task to analyze.")
 
         options = parser.parse_args()
 
@@ -360,6 +466,8 @@ class Main(object):
             if not options.start_commit or not options.end_commit:
                 parser.error("--start-commit and --end-commit must both be specified")
 
+        self.config_options = get_config_options(options, options.expansion_file)
+
         return options
 
     def get_data(self, target, start_date, task, variants):
@@ -376,32 +484,17 @@ class Main(object):
         return divide_tests_into_suites_by_maxtime(tests, self.test_list, execution_time_secs,
                                                    self.options.max_sub_suites)
 
-    def render_suites(self, suites, task):
-        """Render the given suites into yml files that can be used by resmoke.py."""
-        for idx, suite in enumerate(suites):
-            render_template(suite.get_model(self.extra_model_data()), task, idx)
-
-    def render_misc_suite(self, task):
-        """Render a misc suite to run any tests that might be added to the task directory."""
-        model = get_misc_model(self.test_list, self.extra_model_data())
-        source = "{dir}/{task}.yml.j2".format(dir=TEMPLATES_DIR, task=task)
-        target = "{dir}/{task}_misc.yml".format(dir=TEST_SUITE_DIR, task=task)
-
-        render(model, source, target)
-
-    def extra_model_data(self):
-        """Build extra data to include in the model."""
-        return {
-            "options": self.options,
-            "start_commit": self.commit_range.start,
-            "end_commit": self.commit_range.end,
-        }
+    def write_evergreen_configuration(self, suites, task):
+        """Generate the evergreen configuration for the new suite and write it to disk."""
+        evg_config = generate_evg_config(suites, self.config_options)
+
+        with open(os.path.join(CONFIG_DIR, task + ".json"), "w") as file_handle:
+            file_handle.write(evg_config.to_json())
 
     def main(self):
         """Generate resmoke suites that run within a specified target execution time."""
 
         options = self.parse_commandline()
-
         self.options = options
 
         if options.verbose:
@@ -411,20 +504,21 @@ class Main(object):
             self.commit_range = CommitRange(options.start_commit, options.end_commit)
 
         LOGGER.debug("Starting execution for options %s", options)
-        task = options.task[0]
-
         today = datetime.datetime.utcnow().replace(microsecond=0)
         start_date = today - datetime.timedelta(days=options.duration_days)
-
         target = ProjectTarget(options.owner, options.project, options.branch)
 
-        data = self.get_data(target, start_date, task, options.variants)
+        prepare_directory_for_suite(CONFIG_DIR)
+
+        data = self.get_data(target, start_date, self.config_options.task, options.variants)
         suites = self.calculate_suites(data, options.execution_time_minutes * 60)
 
-        LOGGER.debug("Creating %d suites", len(suites))
+        LOGGER.debug("Creating %d suites for %s", len(suites), self.config_options.task)
+
+        render_suite(suites, self.config_options.suite)
+        render_misc_suite(self.test_list, self.config_options.suite)
 
-        self.render_suites(suites, task)
-        self.render_misc_suite(task)
+        self.write_evergreen_configuration(suites, self.config_options.task)
 
 
 if __name__ == "__main__":
diff --git a/buildscripts/resmokeconfig/suites/buildscripts_test.yml b/buildscripts/resmokeconfig/suites/buildscripts_test.yml
index 4886acb7b43..3184bb1fed9 100644
--- a/buildscripts/resmokeconfig/suites/buildscripts_test.yml
+++ b/buildscripts/resmokeconfig/suites/buildscripts_test.yml
@@ -8,7 +8,6 @@ selector:
   - buildscripts/tests/resmokelib/test_archival.py # Requires boto3.
   - buildscripts/tests/resmokelib/test_selector.py # Test assumes POSIX path.
   - buildscripts/tests/test_aws_ec2.py # Requires boto3.
-  - buildscripts/tests/test_evergreen_gen_fuzzer_tests.py # Requires mock.
   - buildscripts/tests/test_remote_operations.py # Requires ssh to be enabled locally.
   - buildscripts/tests/test_update_test_lifecycle.py # Test assumes POSIX path.
 
diff --git a/buildscripts/templates/generate_resmoke_suites/replica_sets_auth.yml.j2 b/buildscripts/templates/generate_resmoke_suites/replica_sets_auth.yml.j2
deleted file mode 100644
index e2f89521be6..00000000000
--- a/buildscripts/templates/generate_resmoke_suites/replica_sets_auth.yml.j2
+++ /dev/null
@@ -1,50 +0,0 @@
-# This file was generated by buildscripts/generate_resmoke_suites.py and manual edits should also be
-# made to replica_sets_auth.yml in order to ensure running the full test suite locally matches the
-# behavior of running parts of the test suite in Evergreen. It was generated against commit range:
-#   {{ start_commit }} - {{ end_commit }}
-# with the following options:
-#   {{ options }}
-
-# Section that is ignored by resmoke.py.
-config_variables:
-- &keyFile jstests/libs/authTestsKey
-- &keyFileData Thiskeyisonlyforrunningthesuitewithauthenticationdontuseitinanytestsdirectly
-
-test_kind: js_test
-
-selector:
-{% if variants is defined %}
-  # The following tests take approximately
-{% for variant in variants %}
-  #   {{ variant.runtime }} minutes to run on {{ variant.name }}
-{% endfor %}
-{% endif %}
-  roots:
-{% for test in test_names %}
-  - {{ test }}
-{% endfor %}
-{% if is_misc is defined %}
-  - jstests/replsets/*.js
-  exclude_files:
-  # Skip any tests that run with auth explicitly.
-  - jstests/replsets/*[aA]uth*.js
-  # Also skip tests that require a ScopedThread, because ScopedThreads don't inherit credentials.
-  - jstests/replsets/interrupted_batch_insert.js
-{% if excluded_tests is defined %}
-{% for test in excluded_tests %}
-  - {{ test }}
-{% endfor %}
-{% endif %}
-{% endif %}
-
-executor:
-  config:
-    shell_options:
-      global_vars:
-        TestData:
-          auth: true
-          authMechanism: SCRAM-SHA-1
-          keyFile: *keyFile
-          keyFileData: *keyFileData
-      nodb: ''
-      readMode: commands
diff --git a/buildscripts/templates/generate_resmoke_suites/replica_sets_ese.yml.j2 b/buildscripts/templates/generate_resmoke_suites/replica_sets_ese.yml.j2
deleted file mode 100644
index 041cbfe22f8..00000000000
--- a/buildscripts/templates/generate_resmoke_suites/replica_sets_ese.yml.j2
+++ /dev/null
@@ -1,42 +0,0 @@
-# This file was generated by buildscripts/generate_resmoke_suites.py and manual edits should also be
-# made to replica_sets_ese.yml in order to ensure running the full test suite locally matches the
-# behavior of running parts of the test suite in Evergreen. It was generated against commit range:
-#   {{ start_commit }} - {{ end_commit }}
-# with the following options:
-#   {{ options }}
-
-# Section that is ignored by resmoke.py.
-config_variables:
-- &keyFile src/mongo/db/modules/enterprise/jstests/encryptdb/libs/ekf2
-
-test_kind: js_test
-
-selector:
-{% if variants is defined %}
-  # The following tests take approximately
-{% for variant in variants %}
-  #   {{ variant.runtime }} minutes to run on {{ variant.name }}
-{% endfor %}
-{% endif %}
-  roots:
-{% for test in test_names %}
-  - {{ test }}
-{% endfor %}
-{% if is_misc is defined %}
-  - jstests/replsets/*.js
-{% if excluded_tests is defined %}
-{% for test in excluded_tests %}
-  - {{ test }}
-{% endfor %}
-{% endif %}
-{% endif %}
-
-executor:
-  config:
-    shell_options:
-      nodb: ''
-      global_vars:
-        TestData:
-          enableEncryption: ''
-          encryptionKeyFile: *keyFile
-      readMode: commands
diff --git a/buildscripts/templates/generate_resmoke_suites/sharding.yml.j2 b/buildscripts/templates/generate_resmoke_suites/sharding.yml.j2
deleted file mode 100644
index f680444cbbc..00000000000
--- a/buildscripts/templates/generate_resmoke_suites/sharding.yml.j2
+++ /dev/null
@@ -1,34 +0,0 @@
-# This file was generated by buildscripts/generate_resmoke_suites.py and manual edits should also be
-# made to sharding.yml in order to ensure running the full test suite locally matches the behavior
-# of running parts of the test suite in Evergreen. It was generated against commit range:
-#   {{ start_commit }} - {{ end_commit }}
-# with the following options:
-#   {{ options }}
-test_kind: js_test
-
-selector:
-{% if variants is defined %}
-  # The following tests take approximately
-{% for variant in variants %}
-  #   {{ variant.runtime }} minutes to run on {{ variant.name }}
-{% endfor %}
-{% endif %}
-  roots:
-{% for test in test_names %}
-  - {{ test }}
-{% endfor %}
-{% if is_misc is defined %}
-  - jstests/sharding/*.js
-{% endif %}
-{% if excluded_tests is defined %}
-  exclude_files:
-{% for test in excluded_tests %}
-  - {{ test }}
-{% endfor %}
-{% endif %}
-
-executor:
-  config:
-    shell_options:
-      nodb: ''
-      readMode: commands
diff --git a/buildscripts/templates/generate_resmoke_suites/sharding_auth.yml.j2 b/buildscripts/templates/generate_resmoke_suites/sharding_auth.yml.j2
deleted file mode 100644
index f7beb4884f1..00000000000
--- a/buildscripts/templates/generate_resmoke_suites/sharding_auth.yml.j2
+++ /dev/null
@@ -1,71 +0,0 @@
-# This file was generated by buildscripts/generate_resmoke_suites.py and manual edits should also be
-# made to sharding_auth.yml in order to ensure running the full test suite locally matches the
-# behavior of running parts of the test suite in Evergreen. It was generated against commit range:
-#   {{ start_commit }} - {{ end_commit }}
-# with the following options:
-#   {{ options }}
-
-# Section that is ignored by resmoke.py.
-config_variables:
-- &keyFile jstests/libs/authTestsKey
-- &keyFileData Thiskeyisonlyforrunningthesuitewithauthenticationdontuseitinanytestsdirectly
-
-test_kind: js_test
-
-selector:
-{% if variants is defined %}
-  # The following tests take approximately
-{% for variant in variants %}
-  #   {{ variant.runtime }} minutes to run on {{ variant.name }}
-{% endfor %}
-{% endif %}
-  roots:
-{% for test in test_names %}
-  - {{ test }}
-{% endfor %}
-{% if is_misc is defined %}
-  - jstests/sharding/*.js
-  exclude_files:
-  # Skip any tests that run with auth explicitly.
-  - jstests/sharding/*[aA]uth*.js
-  - jstests/sharding/advance_cluster_time_action_type.js
-  - jstests/sharding/aggregation_currentop.js # SERVER-19318
-  - jstests/sharding/kill_sessions.js
-  # Skip these additional tests when running with auth enabled.
-  - jstests/sharding/parallel.js
-  # Skip these tests that run with enableTestCommands off.
-  - jstests/sharding/shard_config_db_collections.js
-  # Skip the testcases that do not have auth bypass when running ops in parallel.
-  - jstests/sharding/cleanup_orphaned_cmd_during_movechunk.js         # SERVER-21713
-  - jstests/sharding/cleanup_orphaned_cmd_during_movechunk_hashed.js  # SERVER-21713
-  - jstests/sharding/migration_ignore_interrupts_1.js                 # SERVER-21713
-  - jstests/sharding/migration_ignore_interrupts_2.js                 # SERVER-21713
-  - jstests/sharding/migration_ignore_interrupts_3.js                 # SERVER-21713
-  - jstests/sharding/migration_ignore_interrupts_4.js                 # SERVER-21713
-  - jstests/sharding/migration_move_chunk_after_receive.js            # SERVER-21713
-  - jstests/sharding/migration_server_status.js                       # SERVER-21713
-  - jstests/sharding/migration_sets_fromMigrate_flag.js               # SERVER-21713
-  - jstests/sharding/migration_with_source_ops.js                     # SERVER-21713
-  - jstests/sharding/movechunk_interrupt_at_primary_stepdown.js       # SERVER-21713
-  - jstests/sharding/movechunk_parallel.js                            # SERVER-21713
-  - jstests/sharding/migration_critical_section_concurrency.js        # SERVER-21713
-  # Runs with auth enabled.
-  - jstests/sharding/mongod_returns_no_cluster_time_without_keys.js
-{% if excluded_tests is defined %}
-{% for test in excluded_tests %}
-  - {{ test }}
-{% endfor %}
-{% endif %}
-{% endif %}
-
-executor:
-  config:
-    shell_options:
-      global_vars:
-        TestData:
-          auth: true
-          authMechanism: SCRAM-SHA-1
-          keyFile: *keyFile
-          keyFileData: *keyFileData
-      nodb: ''
-      readMode: commands
diff --git a/buildscripts/templates/generate_resmoke_suites/sharding_auth_audit.yml.j2 b/buildscripts/templates/generate_resmoke_suites/sharding_auth_audit.yml.j2
deleted file mode 100644
index 8c7a770fd5e..00000000000
--- a/buildscripts/templates/generate_resmoke_suites/sharding_auth_audit.yml.j2
+++ /dev/null
@@ -1,73 +0,0 @@
-# This file was generated by buildscripts/generate_resmoke_suites.py and manual edits should also be
-# made to sharding_auth_audit.yml in order to ensure running the full test
-# suite locally matches the behavior of running parts of the test suite in Evergreen. It was
-# generated against commit range:
-#   {{ start_commit }} - {{ end_commit }}
-# with the following options:
-#   {{ options }}
-
-# Section that is ignored by resmoke.py.
-config_variables:
-- &keyFile jstests/libs/authTestsKey
-- &keyFileData Thiskeyisonlyforrunningthesuitewithauthenticationdontuseitinanytestsdirectly
-
-test_kind: js_test
-
-selector:
-{% if variants is defined %}
-  # The following tests take approximately
-{% for variant in variants %}
-  #   {{ variant.runtime }} minutes to run on {{ variant.name }}
-{% endfor %}
-{% endif %}
-  roots:
-{% for test in test_names %}
-  - {{ test }}
-{% endfor %}
-{% if is_misc is defined %}
-  - jstests/sharding/*.js
-  exclude_files:
-  # Skip any tests that run with auth explicitly.
-  - jstests/sharding/*[aA]uth*.js
-  - jstests/sharding/advance_cluster_time_action_type.js
-  - jstests/sharding/aggregation_currentop.js # SERVER-19318
-  - jstests/sharding/kill_sessions.js
-  # Skip these additional tests when running with auth enabled.
-  - jstests/sharding/parallel.js
-  # Skip these tests that run with enableTestCommands off.
-  - jstests/sharding/shard_config_db_collections.js
-  # Skip the testcases that do not have auth bypass when running ops in parallel.
-  - jstests/sharding/cleanup_orphaned_cmd_during_movechunk.js         # SERVER-21713
-  - jstests/sharding/cleanup_orphaned_cmd_during_movechunk_hashed.js  # SERVER-21713
-  - jstests/sharding/migration_with_source_ops.js                     # SERVER-21713
-  - jstests/sharding/migration_sets_fromMigrate_flag.js               # SERVER-21713
-  - jstests/sharding/migration_ignore_interrupts_1.js                 # SERVER-21713
-  - jstests/sharding/migration_ignore_interrupts_2.js                 # SERVER-21713
-  - jstests/sharding/migration_ignore_interrupts_3.js                 # SERVER-21713
-  - jstests/sharding/migration_ignore_interrupts_4.js                 # SERVER-21713
-  - jstests/sharding/movechunk_interrupt_at_primary_stepdown.js       # SERVER-21713
-  - jstests/sharding/movechunk_parallel.js                            # SERVER-21713
-  - jstests/sharding/migration_server_status.js                       # SERVER-21713
-  - jstests/sharding/migration_move_chunk_after_receive.js            # SERVER-21713
-  - jstests/sharding/migration_critical_section_concurrency.js        # SERVER-21713
-  # Runs with auth enabled.
-  - jstests/sharding/mongod_returns_no_cluster_time_without_keys.js
-{% if excluded_tests is defined %}
-{% for test in excluded_tests %}
-  - {{ test }}
-{% endfor %}
-{% endif %}
-{% endif %}
-
-executor:
-  config:
-    shell_options:
-      global_vars:
-        TestData:
-          auditDestination: 'console'
-          auth: true
-          authMechanism: SCRAM-SHA-1
-          keyFile: *keyFile
-          keyFileData: *keyFileData
-      nodb: ''
-      readMode: commands
diff --git a/buildscripts/templates/generate_resmoke_suites/sharding_ese.yml.j2 b/buildscripts/templates/generate_resmoke_suites/sharding_ese.yml.j2
deleted file mode 100644
index 16cd54389e4..00000000000
--- a/buildscripts/templates/generate_resmoke_suites/sharding_ese.yml.j2
+++ /dev/null
@@ -1,43 +0,0 @@
-# This file was generated by buildscripts/generate_resmoke_suites.py and manual edits should also be
-# made to sharding_ese.yml in order to ensure running the full test suite locally matches the
-# behavior of running parts of the test suite in Evergreen. It was generated against commit range:
-#   {{ start_commit }} - {{ end_commit }}
-# with the following options:
-#   {{ options }}
-
-# Section that is ignored by resmoke.py.
-config_variables:
-- &keyFile src/mongo/db/modules/enterprise/jstests/encryptdb/libs/ekf2
-
-test_kind: js_test
-
-selector:
-{% if variants is defined %}
-  # The following tests take approximately
-{% for variant in variants %}
-  #   {{ variant.runtime }} minutes to run on {{ variant.name }}
-{% endfor %}
-{% endif %}
-  roots:
-{% for test in test_names %}
-  - {{ test }}
-{% endfor %}
-{% if is_misc is defined %}
-  - jstests/sharding/*.js
-  exclude_files:
-{% if excluded_tests is defined %}
-{% for test in excluded_tests %}
-  - {{ test }}
-{% endfor %}
-{% endif %}
-{% endif %}
-
-executor:
-  config:
-    shell_options:
-      nodb: ''
-      global_vars:
-        TestData:
-          enableEncryption: ''
-          encryptionKeyFile: *keyFile
-      readMode: commands
diff --git a/buildscripts/templates/generate_resmoke_suites/sharding_last_stable_mongos_and_mixed_shards.yml.j2 b/buildscripts/templates/generate_resmoke_suites/sharding_last_stable_mongos_and_mixed_shards.yml.j2
deleted file mode 100644
index 76176065e79..00000000000
--- a/buildscripts/templates/generate_resmoke_suites/sharding_last_stable_mongos_and_mixed_shards.yml.j2
+++ /dev/null
@@ -1,97 +0,0 @@
-# This file was generated by buildscripts/generate_resmoke_suites.py and manual edits should also be
-# made to sharding_last_stable_mongos_and_mixed_shards.yml in order to ensure running the full test
-# suite locally matches the behavior of running parts of the test suite in Evergreen. It was
-# generated against commit range:
-#   {{ start_commit }} - {{ end_commit }}
-# with the following options:
-#   {{ options }}
-
-test_kind: js_test
-
-selector:
-{% if variants is defined %}
-  # The following tests take approximately
-{% for variant in variants %}
-  #   {{ variant.runtime }} minutes to run on {{ variant.name }}
-{% endfor %}
-{% endif %}
-  roots:
-{% for test in test_names %}
-  - {{ test }}
-{% endfor %}
-{% if is_misc is defined %}
-  - jstests/sharding/*.js
-  exclude_files:
-  # Will always fail on last-stable. In order for the test to succeed, the setFCV
-  # command has to reach the shards. Since the cluster will already be
-  # running in fully downgraded version, the config server won't forward the
-  # command to the shards - it'll just return success immediately.
-  - jstests/sharding/max_time_ms_sharded_new_commands.js
-  # Requires fix to SERVER-31689
-  - jstests/sharding/aggregation_currentop.js
-  # SERVER-33683: We added a restriction on using an aggregation within a transaction against
-  # mongos. This should be removed and the test can be adjusted and re-added to this passthrough.
-  - jstests/sharding/aggregations_in_session.js
-  # New waitForClusterTime
-  - jstests/sharding/auth_slaveok_routing.js
-  # This test should not be run with a mixed cluster environment.
-  - jstests/sharding/nonreplicated_uuids_on_shardservers.js
-  # Enable when SERVER-33538 is backported.
-  - jstests/sharding/mapReduce_outSharded_checkUUID.js
-  # Will always fail because we can't downgrade FCV before the last-stable binary mongos connects,
-  # meaning that either the test will stall, or mongos will crash due to connecting to an upgraded
-  # FCV cluster.
-  - jstests/sharding/mongos_wait_csrs_initiate.js
-  # Enable if SERVER-34971 is backported or 4.2 becomes last-stable
-  - jstests/sharding/update_replace_id.js
-  - jstests/sharding/stale_mongos_updates_and_removes.js
-  - jstests/sharding/geo_near_sharded.js
-  # Enable when 4.2 becomes last-stable.
-  - jstests/sharding/collation_targeting.js
-  - jstests/sharding/collation_targeting_inherited.js
-  - jstests/sharding/geo_near_random1.js
-  - jstests/sharding/geo_near_random2.js
-  - jstests/sharding/restart_transactions.js
-  - jstests/sharding/shard7.js
-  - jstests/sharding/shard_collection_existing_zones.js
-  - jstests/sharding/snapshot_cursor_commands_mongos.js
-  - jstests/sharding/transactions_error_labels.js
-  - jstests/sharding/transactions_implicit_abort.js
-  - jstests/sharding/transactions_multi_writes.js
-  - jstests/sharding/transactions_read_concerns.js
-  - jstests/sharding/transactions_recover_decision_from_local_participant.js
-  - jstests/sharding/transactions_reject_writes_for_moved_chunks.js
-  - jstests/sharding/transactions_snapshot_errors_first_statement.js
-  - jstests/sharding/transactions_snapshot_errors_subsequent_statements.js
-  - jstests/sharding/transactions_stale_database_version_errors.js
-  - jstests/sharding/transactions_stale_shard_version_errors.js
-  - jstests/sharding/transactions_target_at_point_in_time.js
-  - jstests/sharding/transactions_view_resolution.js
-  - jstests/sharding/transactions_writes_not_retryable.js
-  - jstests/sharding/txn_agg.js
-  - jstests/sharding/txn_basic_two_phase_commit.js
-  - jstests/sharding/txn_coordinator_commands_basic_requirements.js
-  - jstests/sharding/txn_writes_during_movechunk.js
-  - jstests/sharding/update_sharded.js
-  - jstests/sharding/shard_existing_coll_chunk_count.js
-  - jstests/sharding/failcommand_failpoint_not_parallel.js
-  - jstests/sharding/transactions_expiration.js
-  # Enable if SERVER-20865 is backported or 4.2 becomes last-stable
-  - jstests/sharding/sharding_statistics_server_status.js
-
-{% if excluded_tests is defined %}
-{% for test in excluded_tests %}
-  - {{ test }}
-{% endfor %}
-{% endif %}
-{% endif %}
-
-executor:
-  config:
-    shell_options:
-      global_vars:
-        TestData:
-          mongosBinVersion: 'last-stable'
-          shardMixedBinVersions: true
-          skipCheckingUUIDsConsistentAcrossCluster: true
-      nodb: ''
diff --git a/buildscripts/tests/test_evergreen_gen_fuzzer_tests.py b/buildscripts/tests/test_evergreen_gen_fuzzer_tests.py
index 1956aa97519..cddb2379ea6 100644
--- a/buildscripts/tests/test_evergreen_gen_fuzzer_tests.py
+++ b/buildscripts/tests/test_evergreen_gen_fuzzer_tests.py
@@ -10,42 +10,6 @@ from buildscripts import evergreen_gen_fuzzer_tests as gft
 # pylint: disable=missing-docstring,protected-access
 
 
-class TestGetConfigValue(unittest.TestCase):
-    def test_undefined_values_return_none(self):
-        value = gft._get_config_value("unknown", {}, {})
-
-        self.assertEqual(None, value)
-
-    def test_default_can_be_specified(self):
-        value = gft._get_config_value("option", {}, {}, default="default")
-
-        self.assertEqual("default", value)
-
-    def test_exception_throw_for_missing_required(self):
-        self.assertRaises(ValueError, gft._get_config_value, "missing", {}, {}, required=True)
-
-    def test_config_file_value_is_used(self):
-        value = gft._get_config_value("option", {}, {"option": "value 0"}, default="default",
-                                      required=True)
-        self.assertEqual("value 0", value)
-
-    def test_cmdline_value_is_used(self):
-        cmdline_mock = mock.Mock
-        cmdline_mock.option = "cmdline value"
-        value = gft._get_config_value("option", cmdline_mock, {"option": "value 0"},
-                                      default="default", required=True)
-
-        self.assertEqual("cmdline value", value)
-
-
-class TestNameTask(unittest.TestCase):
-    def test_name_task_with_width_one(self):
-        self.assertEqual("name_3", gft._name_task("name", 3, 10))
-
-    def test_name_task_with_width_four(self):
-        self.assertEqual("task_3141", gft._name_task("task", 3141, 5000))
-
-
 class TestGenerateEvgTasks(unittest.TestCase):
     @staticmethod
     def _create_options_mock():
diff --git a/buildscripts/tests/test_generate_resmoke_suites.py b/buildscripts/tests/test_generate_resmoke_suites.py
index efb69f061ed..e5501f19548 100644
--- a/buildscripts/tests/test_generate_resmoke_suites.py
+++ b/buildscripts/tests/test_generate_resmoke_suites.py
@@ -3,11 +3,16 @@
 from __future__ import absolute_import
 
 import datetime
+import math
+import os
 import unittest
+import yaml
 
-from mock import patch, Mock
+from mock import patch, mock_open, call, Mock
 
 from buildscripts import generate_resmoke_suites as grs
+from generate_resmoke_suites import render_suite, render_misc_suite, \
+    prepare_directory_for_suite
 
 # pylint: disable=missing-docstring,invalid-name,unused-argument,no-self-use
 
@@ -251,8 +256,8 @@ class OrganizeExecutionsByTestTest(unittest.TestCase):
         self.assertEquals(len(tests), 0)
 
     @patch("buildscripts.generate_resmoke_suites.os")
-    def test_only_test_executions(self, os):
-        os.path.isfile.return_value = True
+    def test_only_test_executions(self, mock_os):
+        mock_os.path.isfile.return_value = True
 
         executions = [{
             "revision": "revision1",
@@ -279,8 +284,8 @@ class OrganizeExecutionsByTestTest(unittest.TestCase):
         self.assertEquals(tests["test3.js"]["variant1"], 3)
 
     @patch("buildscripts.generate_resmoke_suites.os")
-    def test_mix_of_test_and_hook_executions(self, os):
-        os.path.isfile.return_value = True
+    def test_mix_of_test_and_hook_executions(self, mock_os):
+        mock_os.path.isfile.return_value = True
 
         executions = [
             {
@@ -323,8 +328,8 @@ class OrganizeExecutionsByTestTest(unittest.TestCase):
         self.assertEquals(tests["test3.js"]["variant1"], 6)
 
     @patch("buildscripts.generate_resmoke_suites.os")
-    def test_multiple_revisions_for_same_test(self, os):
-        os.path.isfile.return_value = True
+    def test_multiple_revisions_for_same_test(self, mock_os):
+        mock_os.path.isfile.return_value = True
 
         executions = [
             {
@@ -373,8 +378,8 @@ class OrganizeExecutionsByTestTest(unittest.TestCase):
         self.assertEquals(tests["test3.js"]["variant1"], 3)
 
     @patch("buildscripts.generate_resmoke_suites.os")
-    def test_non_files_are_not_included(self, os):
-        os.path.isfile.return_value = False
+    def test_non_files_are_not_included(self, mock_os):
+        mock_os.path.isfile.return_value = False
 
         executions = [{
             "revision": "revision1",
@@ -520,62 +525,150 @@ class SuiteTest(unittest.TestCase):
         self.assertEqual(suite.get_test_count(), 3)
         self.assertEqual(suite.get_runtime(), 29)
 
-    def test_model_generation(self):
-        suite = grs.Suite()
-        suite.add_test('test1', {
-            "max_runtime": 10 * 60,
-            "variant1": 5 * 60,
-            "variant2": 10 * 60,
-            "variant3": 7 * 60,
-        })
-        suite.add_test('test2', {
-            "max_runtime": 12 * 60,
-            "variant1": 12 * 60,
-            "variant2": 8 * 60,
-            "variant3": 6 * 60,
-        })
-        suite.add_test('test3', {
-            "max_runtime": 7 * 60,
-            "variant1": 6 * 60,
-            "variant2": 6 * 60,
-            "variant3": 7 * 60,
-        })
-
-        model = suite.get_model()
 
-        self.assertEqual(model["test_names"], ["test1", "test2", "test3"])
-        self.assertIn({"runtime": 23, "name": "variant1"}, model["variants"])
-        self.assertIn({"runtime": 24, "name": "variant2"}, model["variants"])
-        self.assertIn({"runtime": 20, "name": "variant3"}, model["variants"])
+def create_suite(count=3, start=0):
+    """ Create a suite with count tests."""
+    suite = grs.Suite()
+    for i in range(start, start + count):
+        suite.add_test('test{}'.format(i), {})
+    return suite
 
 
-class GetMiscModelTest(unittest.TestCase):
-    def test_model_with_test_in_same_dir(self):
-        test_list = [
-            "dir0/subdir0/test0",
-            "dir0/subdir0/test1",
-            "dir0/subdir0/test2",
-            "dir0/subdir0/test3",
-        ]
+class RenderSuites(unittest.TestCase):
+    EXPECTED_FORMAT = """selector:
+  excludes:
+  - fixed
+  roots:
+  - test{}
+  - test{}
+  - test{}
+"""
 
-        model = grs.get_misc_model(test_list)
+    def _test(self, size):
 
-        self.assertIn("is_misc", model)
-
-        self.assertIn("excluded_tests", model)
-        self.assertEqual(len(model["excluded_tests"]), 4)
-        self.assertIn("dir0/subdir0/test0", model["excluded_tests"])
-        self.assertIn("dir0/subdir0/test1", model["excluded_tests"])
-        self.assertIn("dir0/subdir0/test2", model["excluded_tests"])
-        self.assertIn("dir0/subdir0/test3", model["excluded_tests"])
+        suites = [create_suite(start=3 * i) for i in range(size)]
+        expected = [
+            self.EXPECTED_FORMAT.format(*range(3 * i, 3 * (i + 1))) for i in range(len(suites))
+        ]
 
-    def test_model_includes_extra_data(self):
-        test_list = ["dir0/subdir0/test0"]
-        extra_data = {
-            "extra": "data",
-        }
+        m = mock_open(read_data=yaml.dump({'selector': {'roots': [], 'excludes': ['fixed']}}))
+        with patch('generate_resmoke_suites.open', m, create=True):
+            render_suite(suites, 'suite_name')
+        handle = m()
+
+        # The other writes are for the headers.
+        self.assertEquals(len(suites) * 2, handle.write.call_count)
+        handle.write.assert_has_calls([call(e) for e in expected], any_order=True)
+        calls = [
+            call(os.path.join(grs.TEST_SUITE_DIR, 'suite_name.yml'), 'r')
+            for _ in range(len(suites))
+        ]
+        m.assert_has_calls(calls, any_order=True)
+        filename = os.path.join(grs.CONFIG_DIR, 'suite_name_{{:0{}}}.yml'.format(
+            int(math.ceil(math.log10(size)))))
+        calls = [call(filename.format(i), 'w') for i in range(size)]
+        m.assert_has_calls(calls, any_order=True)
+
+    def test_1_suite(self):
+        self._test(1)
+
+    def test_11_suites(self):
+        self._test(11)
+
+    def test_101_suites(self):
+        self._test(101)
+
+
+class RenderMiscSuites(unittest.TestCase):
+    def test_single_suite(self):
+
+        test_list = ['test{}'.format(i) for i in range(10)]
+        m = mock_open(read_data=yaml.dump({'selector': {'roots': []}}))
+        with patch('generate_resmoke_suites.open', m, create=True):
+            render_misc_suite(test_list, 'suite_name')
+        handle = m()
+
+        # The other writes are for the headers.
+        self.assertEquals(2, handle.write.call_count)
+        handle.write.assert_any_call("""selector:
+  exclude_files:
+  - test0
+  - test1
+  - test2
+  - test3
+  - test4
+  - test5
+  - test6
+  - test7
+  - test8
+  - test9
+  roots: []
+""")
+        calls = [call(os.path.join(grs.TEST_SUITE_DIR, 'suite_name.yml'), 'r')]
+        m.assert_has_calls(calls, any_order=True)
+        filename = os.path.join(grs.CONFIG_DIR, 'suite_name_misc.yml')
+        calls = [call(filename, 'w')]
+        m.assert_has_calls(calls, any_order=True)
+
+
+class PrepareDirectoryForSuite(unittest.TestCase):
+    def test_no_directory(self):
+        with patch('generate_resmoke_suites.os') as mock_os:
+            mock_os.path.exists.return_value = False
+            prepare_directory_for_suite('tmp')
+
+        mock_os.makedirs.assert_called_once_with('tmp')
+
+
+class GenerateEvgConfigTest(unittest.TestCase):
+    @staticmethod
+    def generate_mock_suites(count):
+        suites = []
+        for idx in range(count):
+            suite = Mock()
+            suite.name = "suite {0}".format(idx)
+            suites.append(suite)
 
-        model = grs.get_misc_model(test_list, extra_data)
+        return suites
 
-        self.assertIn("extra", model)
-        self.assertEqual(model["extra"], "data")
+    @staticmethod
+    def generate_mock_options():
+        options = Mock()
+        options.resmoke_args = "resmoke_args"
+        options.run_multiple_jobs = "true"
+        options.variant = "buildvariant"
+        options.suite = "suite"
+        options.task = "suite"
+
+        return options
+
+    def test_evg_config_is_created(self):
+        options = self.generate_mock_options()
+        suites = self.generate_mock_suites(3)
+
+        config = grs.generate_evg_config(suites, options).to_map()
+
+        self.assertEqual(len(config["tasks"]), len(suites) + 1)
+        command1 = config["tasks"][0]["commands"][1]
+        self.assertIn(options.resmoke_args, command1["vars"]["resmoke_args"])
+        self.assertIn(options.run_multiple_jobs, command1["vars"]["run_multiple_jobs"])
+        self.assertEqual("run tests", command1["func"])
+
+    def test_evg_config_is_created_with_diff_task_and_suite(self):
+        options = self.generate_mock_options()
+        options.task = "task"
+        suites = self.generate_mock_suites(3)
+
+        config = grs.generate_evg_config(suites, options).to_map()
+
+        self.assertEqual(len(config["tasks"]), len(suites) + 1)
+        display_task = config["buildvariants"][0]["display_tasks"][0]
+        self.assertEqual(options.task, display_task["name"])
+        self.assertEqual(len(suites) + 2, len(display_task["execution_tasks"]))
+        self.assertIn(options.task + "_gen", display_task["execution_tasks"])
+        self.assertIn(options.task + "_misc_" + options.variant, display_task["execution_tasks"])
+
+        task = config["tasks"][0]
+        self.assertIn(options.variant, task["name"])
+        self.assertIn(task["name"], display_task["execution_tasks"])
+        self.assertIn(options.suite, task["commands"][1]["vars"]["resmoke_args"])
diff --git a/buildscripts/tests/util/test_read_config.py b/buildscripts/tests/util/test_read_config.py
new file mode 100644
index 00000000000..c3887fcd8ec
--- /dev/null
+++ b/buildscripts/tests/util/test_read_config.py
@@ -0,0 +1,38 @@
+"""Unit tests for the util/read_config.py file."""
+
+from __future__ import absolute_import
+
+import unittest
+import mock
+
+from buildscripts.util import read_config as read_config
+
+# pylint: disable=missing-docstring,protected-access
+
+
+class TestGetConfigValue(unittest.TestCase):
+    def test_undefined_values_return_none(self):
+        value = read_config.get_config_value("unknown", {}, {})
+
+        self.assertEqual(None, value)
+
+    def test_default_can_be_specified(self):
+        value = read_config.get_config_value("option", {}, {}, default="default")
+
+        self.assertEqual("default", value)
+
+    def test_exception_throw_for_missing_required(self):
+        self.assertRaises(KeyError, read_config.get_config_value, "missing", {}, {}, required=True)
+
+    def test_config_file_value_is_used(self):
+        value = read_config.get_config_value("option", {}, {"option": "value 0"}, default="default",
+                                             required=True)
+        self.assertEqual("value 0", value)
+
+    def test_cmdline_value_is_used(self):
+        cmdline_mock = mock.Mock
+        cmdline_mock.option = "cmdline value"
+        value = read_config.get_config_value("option", cmdline_mock, {"option": "value 0"},
+                                             default="default", required=True)
+
+        self.assertEqual("cmdline value", value)
diff --git a/buildscripts/tests/util/test_taskname.py b/buildscripts/tests/util/test_taskname.py
new file mode 100644
index 00000000000..cf2811c7bca
--- /dev/null
+++ b/buildscripts/tests/util/test_taskname.py
@@ -0,0 +1,17 @@
+"""Unit tests for the util/taskname.py script."""
+
+from __future__ import absolute_import
+
+import unittest
+
+from buildscripts.util import taskname as taskname
+
+# pylint: disable=missing-docstring,protected-access
+
+
+class TestNameTask(unittest.TestCase):
+    def test_name_task_with_width_one(self):
+        self.assertEqual("name_3_var", taskname.name_generated_task("name", 3, 10, "var"))
+
+    def test_name_task_with_width_four(self):
+        self.assertEqual("task_3141_var", taskname.name_generated_task("task", 3141, 5000, "var"))
diff --git a/buildscripts/util/read_config.py b/buildscripts/util/read_config.py
new file mode 100644
index 00000000000..aad2b9a150c
--- /dev/null
+++ b/buildscripts/util/read_config.py
@@ -0,0 +1,44 @@
+"""Functions to read configuration files."""
+import yaml
+
+
+def get_config_value(attrib, cmd_line_options, config_file_data, required=False, default=None):
+    """
+    Get the configuration value to use.
+
+    First use command line options, then config file option, then the default. If required is
+    true, throw an exception if the value is not found.
+
+    :param attrib: Attribute to search for.
+    :param cmd_line_options: Command line options.
+    :param config_file_data: Config file data.
+    :param required: Is this option required.
+    :param default: Default value if option is not found.
+    :return: value to use for this option.
+    """
+    value = getattr(cmd_line_options, attrib, None)
+    if value is not None:
+        return value
+
+    if attrib in config_file_data:
+        return config_file_data[attrib]
+
+    if required:
+        raise KeyError("{0} must be specified".format(attrib))
+
+    return default
+
+
+def read_config_file(config_file):
+    """
+    Read the yaml config file specified.
+
+    :param config_file: path to config file.
+    :return: Object representing contents of config file.
+    """
+    config_file_data = {}
+    if config_file:
+        with open(config_file) as file_handle:
+            config_file_data = yaml.load(file_handle)
+
+    return config_file_data
diff --git a/buildscripts/util/taskname.py b/buildscripts/util/taskname.py
new file mode 100644
index 00000000000..ee9852e2295
--- /dev/null
+++ b/buildscripts/util/taskname.py
@@ -0,0 +1,20 @@
+"""Functions for working with resmoke task names."""
+
+import math
+
+
+def name_generated_task(parent_name, task_index, total_tasks, variant=None):
+    """
+    Create a zero-padded sub-task name.
+
+    :param parent_name: Name of the parent task.
+    :param task_index: Index of this sub-task.
+    :param total_tasks: Total number of sub-tasks being generated.
+    :return: Zero-padded name of sub-task.
+    """
+    suffix = ""
+    if variant:
+        suffix = "_{0}".format(variant)
+
+    index_width = int(math.ceil(math.log10(total_tasks)))
+    return "{0}_{1}{2}".format(parent_name, str(task_index).zfill(index_width), suffix)
diff --git a/etc/evergreen.yml b/etc/evergreen.yml
index d3b911b4e3b..e2e9024ce27 100644
--- a/etc/evergreen.yml
+++ b/etc/evergreen.yml
@@ -1846,7 +1846,7 @@ functions:
           set -o verbose
 
           ${activate_virtualenv}
-          $python -m pip install git+https://github.com/evergreen-ci/shrub.py
+          $python -m pip install -r etc/pip/evgtest-requirements.txt
           $python buildscripts/evergreen_gen_fuzzer_tests.py --expansion-file expansions.yml
 
     - command: archive.targz_pack
diff --git a/etc/pip/components/resmoke.req b/etc/pip/components/resmoke.req
index 9bcc7c6f684..8417ea6b721 100644
--- a/etc/pip/components/resmoke.req
+++ b/etc/pip/components/resmoke.req
@@ -1,3 +1,4 @@
 mock; python_version < "3"
 PyKMIP == 0.4.0; python_version < "3" # It's now 0.8.0. We're far enough back to have API conflicts.
 jinja2
+shrub.py
\ No newline at end of file
diff --git a/etc/pip/constraints.txt b/etc/pip/constraints.txt
index 27662e16b20..e84f5015f43 100644
--- a/etc/pip/constraints.txt
+++ b/etc/pip/constraints.txt
@@ -7,9 +7,9 @@
 # Common requirements
 asn1crypto==0.24.0
 astroid==1.6.5
-boto3==1.9.23
-botocore==1.12.23
-certifi==2018.10.15
+boto3==1.9.57
+botocore==1.12.57
+certifi==2018.11.29
 cffi==1.11.5
 chardet==3.0.4
 cryptography==2.3
@@ -21,28 +21,29 @@ Jinja2==2.10
 jira==2.0.0
 jmespath==0.9.3
 lazy-object-proxy==1.3.1
-MarkupSafe==1.0
+MarkupSafe==1.1.0
 mccabe==0.6.1
 oauthlib==2.1.0
-pbr==4.3.0
-psutil==5.4.7
+pbr==5.1.1
+psutil==5.4.8
 pycparser==2.19
 pydocstyle==2.1.1
-PyJWT==1.6.4
+PyJWT==1.7.0
 pylint==1.9.3
 pymongo==3.5.1
-python-dateutil==2.7.3
+python-dateutil==2.7.5
 PyYAML==3.13
-regex==2018.8.29
-requests==2.19.1
+regex==2018.11.22
+requests==2.20.1
 requests-oauth==0.4.1
 requests-oauthlib==1.0.0
 requests-toolbelt==0.8.0
 s3transfer==0.1.13
+shrub.py==0.1.0
 six==1.11.0
 snowballstemmer==1.2.1
 typing==3.6.6
-urllib3==1.23
+urllib3==1.24.1
 wrapt==1.10.11
 yapf==0.21.0