SERVER-38110: Generate resmoke config YAML for a sub-suite

10 files changed, 196 insertions, 535 deletions

diff --git a/.gitignore b/.gitignore
index efb56426b11..a1564ef2888 100644
--- a/.gitignore
+++ b/.gitignore
@@ -169,3 +169,4 @@ src/mongo/embedded/java/jar/build/
 local.properties
 
 compile_commands.json
+generated_resmoke_config
diff --git a/buildscripts/generate_resmoke_suites.py b/buildscripts/generate_resmoke_suites.py
index 6268de9f258..14ae68b04a5 100755
--- a/buildscripts/generate_resmoke_suites.py
+++ b/buildscripts/generate_resmoke_suites.py
@@ -10,27 +10,35 @@ from __future__ import absolute_import
 
 import argparse
 import datetime
+import glob
 import itertools
 import logging
+import math
 import os
 import sys
+import yaml
 
 from collections import defaultdict
 from collections import namedtuple
 from operator import itemgetter
 
-from jinja2 import Template
-
 from client.github import GithubApi
 
 import client.evergreen as evergreen
 import util.testname as testname
 import util.time as timeutil
+from evergreen_gen_fuzzer_tests import CONFIG_DIRECTORY
 
 LOGGER = logging.getLogger(__name__)
 
-TEMPLATES_DIR = "buildscripts/templates/generate_resmoke_suites"
 TEST_SUITE_DIR = "buildscripts/resmokeconfig/suites"
+GENERATED_SUITES_DIR = "buildscripts/resmokeconfig/generated"
+
+HEADER_TEMPLATE = """# DO NOT EDIT THIS FILE. All manual edits will be lost.
+# This file was generated by {file} from
+# {suite_file}.
+"""
+
 
 MAX_RUNTIME_KEY = "max_runtime"
 
@@ -247,32 +255,66 @@ def divide_tests_into_suites_by_maxtime(tests, sorted_tests, max_time_seconds, m
     return suites
 
 
-def get_misc_model(test_list, extra_model_data=None):
-    """Build a model that will run any missing tests."""
-    model = {
-        "is_misc": True,
-        "excluded_tests": test_list,
-    }
+def get_suite_filename(suite_name):
+    """ Get the source suite filename"""
+    return "{dir}/{suite_name}.yml".format(dir=TEST_SUITE_DIR, suite_name=suite_name)
+
+
+def get_subsuite_filename(directory, suite_name, index, index_width=None):
+    """Generate filename for sub-suite from directory, suite_name and index. zero fill index if
+    index_width is not None,"""
+    filled_index = index
+    if index_width is not None:
+        filled_index = str(index).zfill(index_width)
+    return "{dir}/{suite_name}_{index}.yml".format(dir=directory,
+                                                   suite_name=suite_name,
+                                                   index=filled_index)
+
+
+def generate_subsuite_file(suite_file, target_file, roots=None, excludes=None):
+    """ Read and evaluate the yaml suite file. Over ride selector.roots and selector.excludes
+    with the provided values. Write the result to target_file."""
+    with open(suite_file, "r") as fstream:
+        suite_config = yaml.load(fstream)
+
+    with open(target_file, 'w') as out:
+        out.write(HEADER_TEMPLATE.format(file=__file__, suite_file=suite_file))
+        if roots:
+            suite_config['selector']['roots'] = roots
+        if excludes:
+            suite_config['selector']['exclude_files'] = excludes
+        out.write(yaml.dump(suite_config, default_flow_style=False, Dumper=yaml.SafeDumper))
+
+
+def render_suite(suites, suite_name, dir):
+    """Render the given suites into yml files that can be used by resmoke.py."""
 
-    if extra_model_data:
-        model.update(extra_model_data)
+    suite_file = get_suite_filename(suite_name)
 
-    return model
+    index_width = int(math.ceil(math.log10(len(suites))))
+    for idx, suite in enumerate(suites):
+        subsuite_file = get_subsuite_filename(dir, suite_name, idx, index_width=index_width)
+        generate_subsuite_file(suite_file, subsuite_file, roots=suite.tests)
 
 
-def render_template(model, task, index):
-    """Render the specified model as a yml file in the test suites directory."""
-    template_file = "{dir}/{task}.yml.j2".format(dir=TEMPLATES_DIR, task=task)
-    target_file = "{dir}/{task}_{index}.yml".format(dir=TEST_SUITE_DIR, task=task, index=index)
+def render_misc_suite(test_list, suite_name, dir):
+    """Render a misc suite to run any tests that might be added to the directory."""
+    suite_file = get_suite_filename(suite_name)
 
-    render(model, template_file, target_file)
+    subsuite_file = get_subsuite_filename(dir, suite_name, 'misc')
+    generate_subsuite_file(suite_file, subsuite_file, excludes=test_list)
 
 
-def render(model, source, destination):
-    """Render the specified model with the template at `source` to the file `destination`."""
-    with open(source, "r") as inp, open(destination, "w") as out:
-        template = Template(inp.read(), trim_blocks=True)
-        out.write(template.render(model))
+def prepare_directory_for_suite(suite_name, dir):
+    """Ensure that dir exists and that it does not contain any suite related files."""
+    if os.path.exists(dir):
+        suite_files_glob = get_subsuite_filename(dir, suite_name, '[0-9]*')
+        misc_file_glob = get_subsuite_filename(dir, suite_name, 'misc')
+        for suite_file in glob.glob(suite_files_glob) + glob.glob(misc_file_glob):
+            os.remove(suite_file)
+        LOGGER.debug("Removing %s", dir)
+    else:
+        os.makedirs(dir)
 
 
 class Suite(object):
@@ -304,19 +346,6 @@ class Suite(object):
 
         return len(self.tests)
 
-    def get_model(self, extra_model_data=None):
-        """Get a model of this suite that can be used to render a yml file."""
-
-        model = {"test_names": self.tests, "variants": []}
-        for variant in self.variant_runtime:
-            model["variants"].append(
-                {"name": variant, "runtime": self.variant_runtime[variant] / 60})
-
-        if extra_model_data:
-            model.update(extra_model_data)
-
-        return model
-
 
 class Main(object):
     """Orchestrate the execution of generate_resmoke_suites."""
@@ -333,7 +362,7 @@ class Main(object):
         parser = argparse.ArgumentParser(description=self.main.__doc__)
 
         parser.add_argument("--analysis-duration", dest="duration_days", default=14,
-                            help="Number of days to analyze.")
+                            help="Number of days to analyze.", type=int)
         parser.add_argument("--branch", dest="branch", default="master",
                             help="Branch of project to analyze.")
         parser.add_argument("--end-commit", dest="end_commit", help="End analysis at this commit.")
@@ -352,7 +381,7 @@ class Main(object):
                             help="Max number of suites to divide into.")
         parser.add_argument("--verbose", dest="verbose", action="store_true", default=False,
                             help="Enable verbose logging.")
-        parser.add_argument("task", nargs=1, help="task to analyze.")
+        parser.add_argument("suites", nargs='+', help="A list of suites to analyze.")
 
         options = parser.parse_args()
 
@@ -376,27 +405,6 @@ class Main(object):
         return divide_tests_into_suites_by_maxtime(tests, self.test_list, execution_time_secs,
                                                    self.options.max_sub_suites)
 
-    def render_suites(self, suites, task):
-        """Render the given suites into yml files that can be used by resmoke.py."""
-        for idx, suite in enumerate(suites):
-            render_template(suite.get_model(self.extra_model_data()), task, idx)
-
-    def render_misc_suite(self, task):
-        """Render a misc suite to run any tests that might be added to the task directory."""
-        model = get_misc_model(self.test_list, self.extra_model_data())
-        source = "{dir}/{task}.yml.j2".format(dir=TEMPLATES_DIR, task=task)
-        target = "{dir}/{task}_misc.yml".format(dir=TEST_SUITE_DIR, task=task)
-
-        render(model, source, target)
-
-    def extra_model_data(self):
-        """Build extra data to include in the model."""
-        return {
-            "options": self.options,
-            "start_commit": self.commit_range.start,
-            "end_commit": self.commit_range.end,
-        }
-
     def main(self):
         """Generate resmoke suites that run within a specified target execution time."""
 
@@ -411,20 +419,20 @@ class Main(object):
             self.commit_range = CommitRange(options.start_commit, options.end_commit)
 
         LOGGER.debug("Starting execution for options %s", options)
-        task = options.task[0]
-
         today = datetime.datetime.utcnow().replace(microsecond=0)
         start_date = today - datetime.timedelta(days=options.duration_days)
-
         target = ProjectTarget(options.owner, options.project, options.branch)
 
-        data = self.get_data(target, start_date, task, options.variants)
-        suites = self.calculate_suites(data, options.execution_time_minutes * 60)
+        for suite in options.suites:
+            prepare_directory_for_suite(suite, CONFIG_DIRECTORY)
+
+            data = self.get_data(target, start_date, suite, options.variants)
+            suites = self.calculate_suites(data, options.execution_time_minutes * 60)
 
-        LOGGER.debug("Creating %d suites", len(suites))
+            LOGGER.debug("Creating %d suites for %s", len(suites), suite)
 
-        self.render_suites(suites, task)
-        self.render_misc_suite(task)
+            render_suite(suites, suite, CONFIG_DIRECTORY)
+            render_misc_suite(self.test_list, suite, CONFIG_DIRECTORY)
 
 
 if __name__ == "__main__":
diff --git a/buildscripts/templates/generate_resmoke_suites/replica_sets_auth.yml.j2 b/buildscripts/templates/generate_resmoke_suites/replica_sets_auth.yml.j2
deleted file mode 100644
index e2f89521be6..00000000000
--- a/buildscripts/templates/generate_resmoke_suites/replica_sets_auth.yml.j2
+++ /dev/null
@@ -1,50 +0,0 @@
-# This file was generated by buildscripts/generate_resmoke_suites.py and manual edits should also be
-# made to replica_sets_auth.yml in order to ensure running the full test suite locally matches the
-# behavior of running parts of the test suite in Evergreen. It was generated against commit range:
-#   {{ start_commit }} - {{ end_commit }}
-# with the following options:
-#   {{ options }}
-
-# Section that is ignored by resmoke.py.
-config_variables:
-- &keyFile jstests/libs/authTestsKey
-- &keyFileData Thiskeyisonlyforrunningthesuitewithauthenticationdontuseitinanytestsdirectly
-
-test_kind: js_test
-
-selector:
-{% if variants is defined %}
-  # The following tests take approximately
-{% for variant in variants %}
-  #   {{ variant.runtime }} minutes to run on {{ variant.name }}
-{% endfor %}
-{% endif %}
-  roots:
-{% for test in test_names %}
-  - {{ test }}
-{% endfor %}
-{% if is_misc is defined %}
-  - jstests/replsets/*.js
-  exclude_files:
-  # Skip any tests that run with auth explicitly.
-  - jstests/replsets/*[aA]uth*.js
-  # Also skip tests that require a ScopedThread, because ScopedThreads don't inherit credentials.
-  - jstests/replsets/interrupted_batch_insert.js
-{% if excluded_tests is defined %}
-{% for test in excluded_tests %}
-  - {{ test }}
-{% endfor %}
-{% endif %}
-{% endif %}
-
-executor:
-  config:
-    shell_options:
-      global_vars:
-        TestData:
-          auth: true
-          authMechanism: SCRAM-SHA-1
-          keyFile: *keyFile
-          keyFileData: *keyFileData
-      nodb: ''
-      readMode: commands
diff --git a/buildscripts/templates/generate_resmoke_suites/replica_sets_ese.yml.j2 b/buildscripts/templates/generate_resmoke_suites/replica_sets_ese.yml.j2
deleted file mode 100644
index 041cbfe22f8..00000000000
--- a/buildscripts/templates/generate_resmoke_suites/replica_sets_ese.yml.j2
+++ /dev/null
@@ -1,42 +0,0 @@
-# This file was generated by buildscripts/generate_resmoke_suites.py and manual edits should also be
-# made to replica_sets_ese.yml in order to ensure running the full test suite locally matches the
-# behavior of running parts of the test suite in Evergreen. It was generated against commit range:
-#   {{ start_commit }} - {{ end_commit }}
-# with the following options:
-#   {{ options }}
-
-# Section that is ignored by resmoke.py.
-config_variables:
-- &keyFile src/mongo/db/modules/enterprise/jstests/encryptdb/libs/ekf2
-
-test_kind: js_test
-
-selector:
-{% if variants is defined %}
-  # The following tests take approximately
-{% for variant in variants %}
-  #   {{ variant.runtime }} minutes to run on {{ variant.name }}
-{% endfor %}
-{% endif %}
-  roots:
-{% for test in test_names %}
-  - {{ test }}
-{% endfor %}
-{% if is_misc is defined %}
-  - jstests/replsets/*.js
-{% if excluded_tests is defined %}
-{% for test in excluded_tests %}
-  - {{ test }}
-{% endfor %}
-{% endif %}
-{% endif %}
-
-executor:
-  config:
-    shell_options:
-      nodb: ''
-      global_vars:
-        TestData:
-          enableEncryption: ''
-          encryptionKeyFile: *keyFile
-      readMode: commands
diff --git a/buildscripts/templates/generate_resmoke_suites/sharding.yml.j2 b/buildscripts/templates/generate_resmoke_suites/sharding.yml.j2
deleted file mode 100644
index f680444cbbc..00000000000
--- a/buildscripts/templates/generate_resmoke_suites/sharding.yml.j2
+++ /dev/null
@@ -1,34 +0,0 @@
-# This file was generated by buildscripts/generate_resmoke_suites.py and manual edits should also be
-# made to sharding.yml in order to ensure running the full test suite locally matches the behavior
-# of running parts of the test suite in Evergreen. It was generated against commit range:
-#   {{ start_commit }} - {{ end_commit }}
-# with the following options:
-#   {{ options }}
-test_kind: js_test
-
-selector:
-{% if variants is defined %}
-  # The following tests take approximately
-{% for variant in variants %}
-  #   {{ variant.runtime }} minutes to run on {{ variant.name }}
-{% endfor %}
-{% endif %}
-  roots:
-{% for test in test_names %}
-  - {{ test }}
-{% endfor %}
-{% if is_misc is defined %}
-  - jstests/sharding/*.js
-{% endif %}
-{% if excluded_tests is defined %}
-  exclude_files:
-{% for test in excluded_tests %}
-  - {{ test }}
-{% endfor %}
-{% endif %}
-
-executor:
-  config:
-    shell_options:
-      nodb: ''
-      readMode: commands
diff --git a/buildscripts/templates/generate_resmoke_suites/sharding_auth.yml.j2 b/buildscripts/templates/generate_resmoke_suites/sharding_auth.yml.j2
deleted file mode 100644
index f7beb4884f1..00000000000
--- a/buildscripts/templates/generate_resmoke_suites/sharding_auth.yml.j2
+++ /dev/null
@@ -1,71 +0,0 @@
-# This file was generated by buildscripts/generate_resmoke_suites.py and manual edits should also be
-# made to sharding_auth.yml in order to ensure running the full test suite locally matches the
-# behavior of running parts of the test suite in Evergreen. It was generated against commit range:
-#   {{ start_commit }} - {{ end_commit }}
-# with the following options:
-#   {{ options }}
-
-# Section that is ignored by resmoke.py.
-config_variables:
-- &keyFile jstests/libs/authTestsKey
-- &keyFileData Thiskeyisonlyforrunningthesuitewithauthenticationdontuseitinanytestsdirectly
-
-test_kind: js_test
-
-selector:
-{% if variants is defined %}
-  # The following tests take approximately
-{% for variant in variants %}
-  #   {{ variant.runtime }} minutes to run on {{ variant.name }}
-{% endfor %}
-{% endif %}
-  roots:
-{% for test in test_names %}
-  - {{ test }}
-{% endfor %}
-{% if is_misc is defined %}
-  - jstests/sharding/*.js
-  exclude_files:
-  # Skip any tests that run with auth explicitly.
-  - jstests/sharding/*[aA]uth*.js
-  - jstests/sharding/advance_cluster_time_action_type.js
-  - jstests/sharding/aggregation_currentop.js # SERVER-19318
-  - jstests/sharding/kill_sessions.js
-  # Skip these additional tests when running with auth enabled.
-  - jstests/sharding/parallel.js
-  # Skip these tests that run with enableTestCommands off.
-  - jstests/sharding/shard_config_db_collections.js
-  # Skip the testcases that do not have auth bypass when running ops in parallel.
-  - jstests/sharding/cleanup_orphaned_cmd_during_movechunk.js         # SERVER-21713
-  - jstests/sharding/cleanup_orphaned_cmd_during_movechunk_hashed.js  # SERVER-21713
-  - jstests/sharding/migration_ignore_interrupts_1.js                 # SERVER-21713
-  - jstests/sharding/migration_ignore_interrupts_2.js                 # SERVER-21713
-  - jstests/sharding/migration_ignore_interrupts_3.js                 # SERVER-21713
-  - jstests/sharding/migration_ignore_interrupts_4.js                 # SERVER-21713
-  - jstests/sharding/migration_move_chunk_after_receive.js            # SERVER-21713
-  - jstests/sharding/migration_server_status.js                       # SERVER-21713
-  - jstests/sharding/migration_sets_fromMigrate_flag.js               # SERVER-21713
-  - jstests/sharding/migration_with_source_ops.js                     # SERVER-21713
-  - jstests/sharding/movechunk_interrupt_at_primary_stepdown.js       # SERVER-21713
-  - jstests/sharding/movechunk_parallel.js                            # SERVER-21713
-  - jstests/sharding/migration_critical_section_concurrency.js        # SERVER-21713
-  # Runs with auth enabled.
-  - jstests/sharding/mongod_returns_no_cluster_time_without_keys.js
-{% if excluded_tests is defined %}
-{% for test in excluded_tests %}
-  - {{ test }}
-{% endfor %}
-{% endif %}
-{% endif %}
-
-executor:
-  config:
-    shell_options:
-      global_vars:
-        TestData:
-          auth: true
-          authMechanism: SCRAM-SHA-1
-          keyFile: *keyFile
-          keyFileData: *keyFileData
-      nodb: ''
-      readMode: commands
diff --git a/buildscripts/templates/generate_resmoke_suites/sharding_auth_audit.yml.j2 b/buildscripts/templates/generate_resmoke_suites/sharding_auth_audit.yml.j2
deleted file mode 100644
index 8c7a770fd5e..00000000000
--- a/buildscripts/templates/generate_resmoke_suites/sharding_auth_audit.yml.j2
+++ /dev/null
@@ -1,73 +0,0 @@
-# This file was generated by buildscripts/generate_resmoke_suites.py and manual edits should also be
-# made to sharding_auth_audit.yml in order to ensure running the full test
-# suite locally matches the behavior of running parts of the test suite in Evergreen. It was
-# generated against commit range:
-#   {{ start_commit }} - {{ end_commit }}
-# with the following options:
-#   {{ options }}
-
-# Section that is ignored by resmoke.py.
-config_variables:
-- &keyFile jstests/libs/authTestsKey
-- &keyFileData Thiskeyisonlyforrunningthesuitewithauthenticationdontuseitinanytestsdirectly
-
-test_kind: js_test
-
-selector:
-{% if variants is defined %}
-  # The following tests take approximately
-{% for variant in variants %}
-  #   {{ variant.runtime }} minutes to run on {{ variant.name }}
-{% endfor %}
-{% endif %}
-  roots:
-{% for test in test_names %}
-  - {{ test }}
-{% endfor %}
-{% if is_misc is defined %}
-  - jstests/sharding/*.js
-  exclude_files:
-  # Skip any tests that run with auth explicitly.
-  - jstests/sharding/*[aA]uth*.js
-  - jstests/sharding/advance_cluster_time_action_type.js
-  - jstests/sharding/aggregation_currentop.js # SERVER-19318
-  - jstests/sharding/kill_sessions.js
-  # Skip these additional tests when running with auth enabled.
-  - jstests/sharding/parallel.js
-  # Skip these tests that run with enableTestCommands off.
-  - jstests/sharding/shard_config_db_collections.js
-  # Skip the testcases that do not have auth bypass when running ops in parallel.
-  - jstests/sharding/cleanup_orphaned_cmd_during_movechunk.js         # SERVER-21713
-  - jstests/sharding/cleanup_orphaned_cmd_during_movechunk_hashed.js  # SERVER-21713
-  - jstests/sharding/migration_with_source_ops.js                     # SERVER-21713
-  - jstests/sharding/migration_sets_fromMigrate_flag.js               # SERVER-21713
-  - jstests/sharding/migration_ignore_interrupts_1.js                 # SERVER-21713
-  - jstests/sharding/migration_ignore_interrupts_2.js                 # SERVER-21713
-  - jstests/sharding/migration_ignore_interrupts_3.js                 # SERVER-21713
-  - jstests/sharding/migration_ignore_interrupts_4.js                 # SERVER-21713
-  - jstests/sharding/movechunk_interrupt_at_primary_stepdown.js       # SERVER-21713
-  - jstests/sharding/movechunk_parallel.js                            # SERVER-21713
-  - jstests/sharding/migration_server_status.js                       # SERVER-21713
-  - jstests/sharding/migration_move_chunk_after_receive.js            # SERVER-21713
-  - jstests/sharding/migration_critical_section_concurrency.js        # SERVER-21713
-  # Runs with auth enabled.
-  - jstests/sharding/mongod_returns_no_cluster_time_without_keys.js
-{% if excluded_tests is defined %}
-{% for test in excluded_tests %}
-  - {{ test }}
-{% endfor %}
-{% endif %}
-{% endif %}
-
-executor:
-  config:
-    shell_options:
-      global_vars:
-        TestData:
-          auditDestination: 'console'
-          auth: true
-          authMechanism: SCRAM-SHA-1
-          keyFile: *keyFile
-          keyFileData: *keyFileData
-      nodb: ''
-      readMode: commands
diff --git a/buildscripts/templates/generate_resmoke_suites/sharding_ese.yml.j2 b/buildscripts/templates/generate_resmoke_suites/sharding_ese.yml.j2
deleted file mode 100644
index 16cd54389e4..00000000000
--- a/buildscripts/templates/generate_resmoke_suites/sharding_ese.yml.j2
+++ /dev/null
@@ -1,43 +0,0 @@
-# This file was generated by buildscripts/generate_resmoke_suites.py and manual edits should also be
-# made to sharding_ese.yml in order to ensure running the full test suite locally matches the
-# behavior of running parts of the test suite in Evergreen. It was generated against commit range:
-#   {{ start_commit }} - {{ end_commit }}
-# with the following options:
-#   {{ options }}
-
-# Section that is ignored by resmoke.py.
-config_variables:
-- &keyFile src/mongo/db/modules/enterprise/jstests/encryptdb/libs/ekf2
-
-test_kind: js_test
-
-selector:
-{% if variants is defined %}
-  # The following tests take approximately
-{% for variant in variants %}
-  #   {{ variant.runtime }} minutes to run on {{ variant.name }}
-{% endfor %}
-{% endif %}
-  roots:
-{% for test in test_names %}
-  - {{ test }}
-{% endfor %}
-{% if is_misc is defined %}
-  - jstests/sharding/*.js
-  exclude_files:
-{% if excluded_tests is defined %}
-{% for test in excluded_tests %}
-  - {{ test }}
-{% endfor %}
-{% endif %}
-{% endif %}
-
-executor:
-  config:
-    shell_options:
-      nodb: ''
-      global_vars:
-        TestData:
-          enableEncryption: ''
-          encryptionKeyFile: *keyFile
-      readMode: commands
diff --git a/buildscripts/templates/generate_resmoke_suites/sharding_last_stable_mongos_and_mixed_shards.yml.j2 b/buildscripts/templates/generate_resmoke_suites/sharding_last_stable_mongos_and_mixed_shards.yml.j2
deleted file mode 100644
index 76176065e79..00000000000
--- a/buildscripts/templates/generate_resmoke_suites/sharding_last_stable_mongos_and_mixed_shards.yml.j2
+++ /dev/null
@@ -1,97 +0,0 @@
-# This file was generated by buildscripts/generate_resmoke_suites.py and manual edits should also be
-# made to sharding_last_stable_mongos_and_mixed_shards.yml in order to ensure running the full test
-# suite locally matches the behavior of running parts of the test suite in Evergreen. It was
-# generated against commit range:
-#   {{ start_commit }} - {{ end_commit }}
-# with the following options:
-#   {{ options }}
-
-test_kind: js_test
-
-selector:
-{% if variants is defined %}
-  # The following tests take approximately
-{% for variant in variants %}
-  #   {{ variant.runtime }} minutes to run on {{ variant.name }}
-{% endfor %}
-{% endif %}
-  roots:
-{% for test in test_names %}
-  - {{ test }}
-{% endfor %}
-{% if is_misc is defined %}
-  - jstests/sharding/*.js
-  exclude_files:
-  # Will always fail on last-stable. In order for the test to succeed, the setFCV
-  # command has to reach the shards. Since the cluster will already be
-  # running in fully downgraded version, the config server won't forward the
-  # command to the shards - it'll just return success immediately.
-  - jstests/sharding/max_time_ms_sharded_new_commands.js
-  # Requires fix to SERVER-31689
-  - jstests/sharding/aggregation_currentop.js
-  # SERVER-33683: We added a restriction on using an aggregation within a transaction against
-  # mongos. This should be removed and the test can be adjusted and re-added to this passthrough.
-  - jstests/sharding/aggregations_in_session.js
-  # New waitForClusterTime
-  - jstests/sharding/auth_slaveok_routing.js
-  # This test should not be run with a mixed cluster environment.
-  - jstests/sharding/nonreplicated_uuids_on_shardservers.js
-  # Enable when SERVER-33538 is backported.
-  - jstests/sharding/mapReduce_outSharded_checkUUID.js
-  # Will always fail because we can't downgrade FCV before the last-stable binary mongos connects,
-  # meaning that either the test will stall, or mongos will crash due to connecting to an upgraded
-  # FCV cluster.
-  - jstests/sharding/mongos_wait_csrs_initiate.js
-  # Enable if SERVER-34971 is backported or 4.2 becomes last-stable
-  - jstests/sharding/update_replace_id.js
-  - jstests/sharding/stale_mongos_updates_and_removes.js
-  - jstests/sharding/geo_near_sharded.js
-  # Enable when 4.2 becomes last-stable.
-  - jstests/sharding/collation_targeting.js
-  - jstests/sharding/collation_targeting_inherited.js
-  - jstests/sharding/geo_near_random1.js
-  - jstests/sharding/geo_near_random2.js
-  - jstests/sharding/restart_transactions.js
-  - jstests/sharding/shard7.js
-  - jstests/sharding/shard_collection_existing_zones.js
-  - jstests/sharding/snapshot_cursor_commands_mongos.js
-  - jstests/sharding/transactions_error_labels.js
-  - jstests/sharding/transactions_implicit_abort.js
-  - jstests/sharding/transactions_multi_writes.js
-  - jstests/sharding/transactions_read_concerns.js
-  - jstests/sharding/transactions_recover_decision_from_local_participant.js
-  - jstests/sharding/transactions_reject_writes_for_moved_chunks.js
-  - jstests/sharding/transactions_snapshot_errors_first_statement.js
-  - jstests/sharding/transactions_snapshot_errors_subsequent_statements.js
-  - jstests/sharding/transactions_stale_database_version_errors.js
-  - jstests/sharding/transactions_stale_shard_version_errors.js
-  - jstests/sharding/transactions_target_at_point_in_time.js
-  - jstests/sharding/transactions_view_resolution.js
-  - jstests/sharding/transactions_writes_not_retryable.js
-  - jstests/sharding/txn_agg.js
-  - jstests/sharding/txn_basic_two_phase_commit.js
-  - jstests/sharding/txn_coordinator_commands_basic_requirements.js
-  - jstests/sharding/txn_writes_during_movechunk.js
-  - jstests/sharding/update_sharded.js
-  - jstests/sharding/shard_existing_coll_chunk_count.js
-  - jstests/sharding/failcommand_failpoint_not_parallel.js
-  - jstests/sharding/transactions_expiration.js
-  # Enable if SERVER-20865 is backported or 4.2 becomes last-stable
-  - jstests/sharding/sharding_statistics_server_status.js
-
-{% if excluded_tests is defined %}
-{% for test in excluded_tests %}
-  - {{ test }}
-{% endfor %}
-{% endif %}
-{% endif %}
-
-executor:
-  config:
-    shell_options:
-      global_vars:
-        TestData:
-          mongosBinVersion: 'last-stable'
-          shardMixedBinVersions: true
-          skipCheckingUUIDsConsistentAcrossCluster: true
-      nodb: ''
diff --git a/buildscripts/tests/test_generate_resmoke_suites.py b/buildscripts/tests/test_generate_resmoke_suites.py
index efb69f061ed..76f3bc00de9 100644
--- a/buildscripts/tests/test_generate_resmoke_suites.py
+++ b/buildscripts/tests/test_generate_resmoke_suites.py
@@ -3,11 +3,16 @@
 from __future__ import absolute_import
 
 import datetime
+import math
 import unittest
+import yaml
 
-from mock import patch, Mock
+from mock import patch, mock_open, call
 
 from buildscripts import generate_resmoke_suites as grs
+from generate_resmoke_suites import render_suite, render_misc_suite, \
+    prepare_directory_for_suite
+
 
 # pylint: disable=missing-docstring,invalid-name,unused-argument,no-self-use
 
@@ -520,62 +525,119 @@ class SuiteTest(unittest.TestCase):
         self.assertEqual(suite.get_test_count(), 3)
         self.assertEqual(suite.get_runtime(), 29)
 
-    def test_model_generation(self):
-        suite = grs.Suite()
-        suite.add_test('test1', {
-            "max_runtime": 10 * 60,
-            "variant1": 5 * 60,
-            "variant2": 10 * 60,
-            "variant3": 7 * 60,
-        })
-        suite.add_test('test2', {
-            "max_runtime": 12 * 60,
-            "variant1": 12 * 60,
-            "variant2": 8 * 60,
-            "variant3": 6 * 60,
-        })
-        suite.add_test('test3', {
-            "max_runtime": 7 * 60,
-            "variant1": 6 * 60,
-            "variant2": 6 * 60,
-            "variant3": 7 * 60,
-        })
-
-        model = suite.get_model()
-
-        self.assertEqual(model["test_names"], ["test1", "test2", "test3"])
-        self.assertIn({"runtime": 23, "name": "variant1"}, model["variants"])
-        self.assertIn({"runtime": 24, "name": "variant2"}, model["variants"])
-        self.assertIn({"runtime": 20, "name": "variant3"}, model["variants"])
-
-
-class GetMiscModelTest(unittest.TestCase):
-    def test_model_with_test_in_same_dir(self):
-        test_list = [
-            "dir0/subdir0/test0",
-            "dir0/subdir0/test1",
-            "dir0/subdir0/test2",
-            "dir0/subdir0/test3",
-        ]
-
-        model = grs.get_misc_model(test_list)
-
-        self.assertIn("is_misc", model)
-
-        self.assertIn("excluded_tests", model)
-        self.assertEqual(len(model["excluded_tests"]), 4)
-        self.assertIn("dir0/subdir0/test0", model["excluded_tests"])
-        self.assertIn("dir0/subdir0/test1", model["excluded_tests"])
-        self.assertIn("dir0/subdir0/test2", model["excluded_tests"])
-        self.assertIn("dir0/subdir0/test3", model["excluded_tests"])
-
-    def test_model_includes_extra_data(self):
-        test_list = ["dir0/subdir0/test0"]
-        extra_data = {
-            "extra": "data",
-        }
-
-        model = grs.get_misc_model(test_list, extra_data)
 
-        self.assertIn("extra", model)
-        self.assertEqual(model["extra"], "data")
+def create_suite(count=3, start=0):
+    """ Create a suite with count tests."""
+    suite = grs.Suite()
+    for i in range(start, start + count):
+        suite.add_test('test{}'.format(i), {})
+    return suite
+
+
+class RenderSuites(unittest.TestCase):
+    EXPECTED_FORMAT = """selector:
+  excludes:
+  - fixed
+  roots:
+  - test{}
+  - test{}
+  - test{}
+"""
+
+    def _test(self, size):
+
+        suites = [create_suite(start=3*i) for i in range(size)]
+        expected = [self.EXPECTED_FORMAT .format(*range(3 * i, 3 * (i+1)))
+                    for i in range(len(suites))]
+
+        m = mock_open(read_data=yaml.dump({'selector': {'roots': [], 'excludes': ['fixed']}}))
+        with patch('generate_resmoke_suites.open', m, create=True):
+            render_suite(suites, 'suite_name', 'tmp')
+        handle = m()
+
+        # The other writes are for the headers.
+        self.assertEquals(len(suites) * 2, handle.write.call_count)
+        handle.write.assert_has_calls([call(e) for e in expected], any_order=True)
+        calls = [call('buildscripts/resmokeconfig/suites/suite_name.yml', 'r')
+                 for _ in range(len(suites))]
+        m.assert_has_calls(calls, any_order=True)
+        filename = 'tmp/suite_name_{{:0{}}}.yml'.format(int(math.ceil(math.log10(size))))
+        calls = [call(filename.format(i), 'w') for i in range(size)]
+        m.assert_has_calls(calls, any_order=True)
+
+    def test_1_suite(self):
+        self._test(1)
+
+    def test_11_suites(self):
+        self._test(11)
+
+    def test_101_suites(self):
+        self._test(101)
+
+
+class RenderMiscSuites(unittest.TestCase):
+
+    def test_single_suite(self):
+
+        test_list = ['test{}'.format(i) for i in range(10)]
+        m = mock_open(read_data=yaml.dump({'selector': {'roots': []}}))
+        with patch('generate_resmoke_suites.open', m, create=True):
+            render_misc_suite(test_list, 'suite_name', 'tmp')
+        handle = m()
+
+        # The other writes are for the headers.
+        self.assertEquals(2, handle.write.call_count)
+        handle.write.assert_any_call("""selector:
+  exclude_files:
+  - test0
+  - test1
+  - test2
+  - test3
+  - test4
+  - test5
+  - test6
+  - test7
+  - test8
+  - test9
+  roots: []
+""")
+        calls = [call('buildscripts/resmokeconfig/suites/suite_name.yml', 'r')]
+        m.assert_has_calls(calls, any_order=True)
+        filename = 'tmp/suite_name_misc.yml'
+        calls = [call(filename, 'w')]
+        m.assert_has_calls(calls, any_order=True)
+
+
+class PrepareDirectoryForSuite(unittest.TestCase):
+
+    def test_no_directory(self):
+        with patch('generate_resmoke_suites.os') as mock_os,\
+             patch('generate_resmoke_suites.glob.glob') as mock_glob:
+            mock_os.path.exists.return_value = False
+            prepare_directory_for_suite('suite_name', 'tmp')
+
+        mock_os.makedirs.assert_called_once_with('tmp')
+        mock_glob.assert_not_called()
+
+    def _test(self, matched=None):
+        if matched is None:
+            matched = []
+        with patch('generate_resmoke_suites.os') as mock_os, \
+                patch('generate_resmoke_suites.glob.glob') as mock_glob:
+            mock_os.path.exists.return_value = True
+            mock_glob.side_effect = (matched, [])
+            prepare_directory_for_suite('suite_name', 'tmp')
+
+        mock_glob.assert_has_calls([call('tmp/suite_name_[0-9]*.yml'),
+                                      call('tmp/suite_name_misc.yml')])
+        if matched:
+            mock_os.remove.assert_has_calls([call(filename) for filename in matched])
+        else:
+            mock_os.remove.assert_not_called()
+        mock_os.makedirs.assert_not_called()
+
+    def test_empty_directory(self):
+        self._test()
+
+    def test_old_suite_files(self):
+        self._test(matched=['tmp/suite_name_{}.yml'.format(i) for i in range(3)])