SERVER-38112: Add ability to dynamically split up long running tasks in evergreen

3 files changed, 378 insertions, 234 deletions

diff --git a/buildscripts/generate_resmoke_suites.py b/buildscripts/evergreen_generate_resmoke_tasks.py
index 063be009840..35a72dbc74d 100755
--- a/buildscripts/generate_resmoke_suites.py
+++ b/buildscripts/evergreen_generate_resmoke_tasks.py
@@ -11,6 +11,7 @@ from __future__ import absolute_import
 import argparse
 import datetime
 import logging
+import math
 import os
 import sys
 from collections import defaultdict
@@ -21,6 +22,7 @@ import yaml
 
 from shrub.config import Configuration
 from shrub.command import CommandDefinition
+from shrub.operations import CmdTimeoutUpdate
 from shrub.task import TaskDependency
 from shrub.variant import DisplayTaskDefinition
 from shrub.variant import TaskSpec
@@ -51,10 +53,13 @@ ConfigOptions = namedtuple("ConfigOptions", [
     "project",
     "resmoke_args",
     "resmoke_jobs_max",
+    "target_resmoke_time",
     "run_multiple_jobs",
     "suite",
     "task",
     "variant",
+    "use_large_distro",
+    "large_distro_name",
 ])
 
 
@@ -90,15 +95,23 @@ def get_config_options(cmd_line_options, config_file):
                                                 default="")
     resmoke_jobs_max = read_config.get_config_value("resmoke_jobs_max", cmd_line_options,
                                                     config_file_data)
+    target_resmoke_time = int(
+        read_config.get_config_value("target_resmoke_time", cmd_line_options, config_file_data,
+                                     default=60))
     run_multiple_jobs = read_config.get_config_value("run_multiple_jobs", cmd_line_options,
                                                      config_file_data, default="true")
     task = read_config.get_config_value("task", cmd_line_options, config_file_data, required=True)
     suite = read_config.get_config_value("suite", cmd_line_options, config_file_data, default=task)
     variant = read_config.get_config_value("build_variant", cmd_line_options, config_file_data,
                                            required=True)
+    use_large_distro = read_config.get_config_value("use_large_distro", cmd_line_options,
+                                                    config_file_data, default=False)
+    large_distro_name = read_config.get_config_value("large_distro_name", cmd_line_options,
+                                                     config_file_data)
 
     return ConfigOptions(fallback_num_sub_suites, max_sub_suites, project, resmoke_args,
-                         resmoke_jobs_max, run_multiple_jobs, suite, task, variant)
+                         resmoke_jobs_max, target_resmoke_time, run_multiple_jobs, suite, task,
+                         variant, use_large_distro, large_distro_name)
 
 
 def divide_remaining_tests_among_suites(remaining_tests_runtimes, suites):
@@ -191,32 +204,46 @@ def generate_evg_config(suites, options):
     task_names = []
     task_specs = []
 
-    def generate_task(sub_suite_name, sub_task_name):
+    def generate_task(sub_suite_name, sub_task_name, max_test_runtime=None,
+                      expected_suite_runtime=None):
         """Generate evergreen config for a resmoke task."""
         task_names.append(sub_task_name)
-        task_specs.append(TaskSpec(sub_task_name))
+        spec = TaskSpec(sub_task_name)
+        if options.use_large_distro:
+            spec.distro(options.large_distro_name)
+        task_specs.append(spec)
         task = evg_config.task(sub_task_name)
 
         target_suite_file = os.path.join(CONFIG_DIR, sub_suite_name)
 
         run_tests_vars = {
-            "resmoke_args": "--suites={0} {1}".format(target_suite_file, options.resmoke_args),
+            "resmoke_args": "--suites={0}.yml {1}".format(target_suite_file, options.resmoke_args),
             "run_multiple_jobs": options.run_multiple_jobs,
+            "task": options.task,
         }
 
         if options.resmoke_jobs_max:
             run_tests_vars["resmoke_jobs_max"] = options.resmoke_jobs_max
 
-        commands = [
+        commands = []
+        if max_test_runtime or expected_suite_runtime:
+            cmd_timeout = CmdTimeoutUpdate()
+            if max_test_runtime:
+                cmd_timeout.timeout(int(math.ceil(max_test_runtime * 3)))
+            if expected_suite_runtime:
+                cmd_timeout.exec_timeout(int(math.ceil(expected_suite_runtime * 3)))
+            commands.append(cmd_timeout.validate().resolve())
+
+        commands += [
             CommandDefinition().function("do setup"),
-            CommandDefinition().function("run tests").vars(run_tests_vars)
+            CommandDefinition().function("run generated tests").vars(run_tests_vars)
         ]
         task.dependency(TaskDependency("compile")).commands(commands)
 
     for idx, suite in enumerate(suites):
         sub_task_name = taskname.name_generated_task(options.task, idx, len(suites),
                                                      options.variant)
-        generate_task(suite.name, sub_task_name)
+        generate_task(suite.name, sub_task_name, suite.max_runtime, suite.get_runtime())
 
     # Add the misc suite
     misc_suite_name = "{0}_misc".format(options.suite)
@@ -298,6 +325,7 @@ class Suite(object):
         """Initialize the object."""
         self.tests = []
         self.total_runtime = 0
+        self.max_runtime = 0
 
     def add_test(self, test_file, runtime):
         """Add the given test to this suite."""
@@ -305,6 +333,9 @@ class Suite(object):
         self.tests.append(test_file)
         self.total_runtime += runtime
 
+        if runtime > self.max_runtime:
+            self.max_runtime = runtime
+
     def get_runtime(self):
         """Get the current average runtime of all the tests currently in this suite."""
 
@@ -334,7 +365,7 @@ class Main(object):
                             help="Location of expansions file generated by evergreen.")
         parser.add_argument("--analysis-duration", dest="duration_days", default=14,
                             help="Number of days to analyze.", type=int)
-        parser.add_argument("--execution-time", dest="execution_time_minutes", default=60, type=int,
+        parser.add_argument("--execution-time", dest="target_resmoke_time", type=int,
                             help="Target execution time (in minutes).")
         parser.add_argument("--max-sub-suites", dest="max_sub_suites", type=int,
                             help="Max number of suites to divide into.")
@@ -353,6 +384,10 @@ class Main(object):
         parser.add_argument("--task-name", dest="task", help="Name of task to split.")
         parser.add_argument("--variant", dest="build_variant",
                             help="Build variant being run against.")
+        parser.add_argument("--use-large-distro", dest="use_large_distro",
+                            help="Should subtasks use large distros.")
+        parser.add_argument("--large_distro_name", dest="large_distro_name",
+                            help="Name of large distro.")
         parser.add_argument("--verbose", dest="verbose", action="store_true", default=False,
                             help="Enable verbose logging.")
 
@@ -366,8 +401,8 @@ class Main(object):
         try:
             evg_stats = self.get_evg_stats(self.config_options.project, start_date, end_date,
                                            self.config_options.task, self.config_options.variant)
-            return self.calculate_suites_from_evg_stats(evg_stats,
-                                                        self.options.execution_time_minutes * 60)
+            target_execution_time_secs = self.config_options.target_resmoke_time * 60
+            return self.calculate_suites_from_evg_stats(evg_stats, target_execution_time_secs)
         except requests.HTTPError as err:
             if err.response.status_code == requests.codes.SERVICE_UNAVAILABLE:
                 # Evergreen may return a 503 when the service is degraded.
@@ -391,11 +426,16 @@ class Main(object):
     def calculate_suites_from_evg_stats(self, data, execution_time_secs):
         """Divide tests into suites that can be run in less than the specified execution time."""
         test_stats = TestStats(data)
-        tests_runtimes = test_stats.get_tests_runtimes()
+        tests_runtimes = self.filter_existing_tests(test_stats.get_tests_runtimes())
         self.test_list = [info[0] for info in tests_runtimes]
         return divide_tests_into_suites(tests_runtimes, execution_time_secs,
                                         self.options.max_sub_suites)
 
+    @staticmethod
+    def filter_existing_tests(tests_runtimes):
+        """Filter out tests that do not exist in the filesystem."""
+        return [info for info in tests_runtimes if os.path.exists(info[0])]
+
     def calculate_fallback_suites(self):
         """Divide tests into a fixed number of suites."""
         num_suites = self.config_options.fallback_num_sub_suites
diff --git a/buildscripts/tests/test_generate_resmoke_suites.py b/buildscripts/tests/test_evergreen_generate_resmoke_tasks.py
index a6e78ce1d30..dd4db4f34a9 100644
--- a/buildscripts/tests/test_generate_resmoke_suites.py
+++ b/buildscripts/tests/test_evergreen_generate_resmoke_tasks.py
@@ -12,8 +12,8 @@ import yaml
 
 from mock import patch, mock_open, call, Mock
 
-from buildscripts import generate_resmoke_suites as grs
-from buildscripts.generate_resmoke_suites import render_suite, render_misc_suite, \
+from buildscripts import evergreen_generate_resmoke_tasks as grt
+from buildscripts.evergreen_generate_resmoke_tasks import render_suite, render_misc_suite, \
     prepare_directory_for_suite
 
 # pylint: disable=missing-docstring,invalid-name,unused-argument,no-self-use
@@ -28,7 +28,7 @@ class TestTestStats(unittest.TestCase):
             self._make_evg_result("dir/test2.js", 1, 30),
             self._make_evg_result("dir/test1.js", 2, 25),
         ]
-        test_stats = grs.TestStats(evg_results)
+        test_stats = grt.TestStats(evg_results)
         expected_runtimes = [
             ("dir/test2.js", 30),
             ("dir/test1.js", 20),
@@ -44,7 +44,7 @@ class TestTestStats(unittest.TestCase):
             self._make_evg_result("test3:CleanEveryN", 10, 30),
             self._make_evg_result("test3:CheckReplDBHash", 10, 35),
         ]
-        test_stats = grs.TestStats(evg_results)
+        test_stats = grt.TestStats(evg_results)
         expected_runtimes = [
             ("dir/test3.js", 42.5),
             ("dir/test2.js", 30),
@@ -78,19 +78,19 @@ class DivideRemainingTestsAmongSuitesTest(unittest.TestCase):
         return tests_runtimes
 
     def test_each_suite_gets_one_test(self):
-        suites = [grs.Suite(), grs.Suite(), grs.Suite()]
+        suites = [grt.Suite(), grt.Suite(), grt.Suite()]
         tests_runtimes = self.generate_tests_runtimes(3)
 
-        grs.divide_remaining_tests_among_suites(tests_runtimes, suites)
+        grt.divide_remaining_tests_among_suites(tests_runtimes, suites)
 
         for suite in suites:
             self.assertEqual(suite.get_test_count(), 1)
 
     def test_each_suite_gets_at_least_one_test(self):
-        suites = [grs.Suite(), grs.Suite(), grs.Suite()]
+        suites = [grt.Suite(), grt.Suite(), grt.Suite()]
         tests_runtimes = self.generate_tests_runtimes(5)
 
-        grs.divide_remaining_tests_among_suites(tests_runtimes, suites)
+        grt.divide_remaining_tests_among_suites(tests_runtimes, suites)
 
         total_tests = 0
         for suite in suites:
@@ -109,7 +109,7 @@ class DivideTestsIntoSuitesByMaxtimeTest(unittest.TestCase):
             ("test3", 3),
         ]
 
-        suites = grs.divide_tests_into_suites(tests_runtimes, max_time)
+        suites = grt.divide_tests_into_suites(tests_runtimes, max_time)
         self.assertEqual(len(suites), 1)
         self.assertEqual(suites[0].get_test_count(), 3)
         self.assertEqual(suites[0].get_runtime(), 12)
@@ -122,7 +122,7 @@ class DivideTestsIntoSuitesByMaxtimeTest(unittest.TestCase):
             ("test3", 3),
         ]
 
-        suites = grs.divide_tests_into_suites(tests_runtimes, max_time)
+        suites = grt.divide_tests_into_suites(tests_runtimes, max_time)
         self.assertEqual(len(suites), 3)
 
     def test_if_test_is_greater_than_max_it_goes_alone(self):
@@ -133,7 +133,7 @@ class DivideTestsIntoSuitesByMaxtimeTest(unittest.TestCase):
             ("test3", 3),
         ]
 
-        suites = grs.divide_tests_into_suites(tests_runtimes, max_time)
+        suites = grt.divide_tests_into_suites(tests_runtimes, max_time)
         self.assertEqual(len(suites), 2)
         self.assertEqual(suites[0].get_test_count(), 1)
         self.assertEqual(suites[0].get_runtime(), 15)
@@ -149,7 +149,7 @@ class DivideTestsIntoSuitesByMaxtimeTest(unittest.TestCase):
             ("test5", 3),
         ]
 
-        suites = grs.divide_tests_into_suites(tests_runtimes, max_time, max_suites=max_suites)
+        suites = grt.divide_tests_into_suites(tests_runtimes, max_time, max_suites=max_suites)
         self.assertEqual(len(suites), max_suites)
         total_tests = 0
         for suite in suites:
@@ -159,7 +159,7 @@ class DivideTestsIntoSuitesByMaxtimeTest(unittest.TestCase):
 
 class SuiteTest(unittest.TestCase):
     def test_adding_tests_increases_count_and_runtime(self):
-        suite = grs.Suite()
+        suite = grt.Suite()
         suite.add_test('test1', 10)
         suite.add_test('test2', 12)
         suite.add_test('test3', 7)
@@ -170,7 +170,7 @@ class SuiteTest(unittest.TestCase):
 
 def create_suite(count=3, start=0):
     """ Create a suite with count tests."""
-    suite = grs.Suite()
+    suite = grt.Suite()
     for i in range(start, start + count):
         suite.add_test('test{}'.format(i), 1)
     return suite
@@ -194,7 +194,7 @@ class RenderSuites(unittest.TestCase):
         ]
 
         m = mock_open(read_data=yaml.dump({'selector': {'roots': [], 'excludes': ['fixed']}}))
-        with patch('buildscripts.generate_resmoke_suites.open', m, create=True):
+        with patch('buildscripts.evergreen_generate_resmoke_tasks.open', m, create=True):
             render_suite(suites, 'suite_name')
         handle = m()
 
@@ -202,11 +202,11 @@ class RenderSuites(unittest.TestCase):
         self.assertEquals(len(suites) * 2, handle.write.call_count)
         handle.write.assert_has_calls([call(e) for e in expected], any_order=True)
         calls = [
-            call(os.path.join(grs.TEST_SUITE_DIR, 'suite_name.yml'), 'r')
+            call(os.path.join(grt.TEST_SUITE_DIR, 'suite_name.yml'), 'r')
             for _ in range(len(suites))
         ]
         m.assert_has_calls(calls, any_order=True)
-        filename = os.path.join(grs.CONFIG_DIR, 'suite_name_{{:0{}}}.yml'.format(
+        filename = os.path.join(grt.CONFIG_DIR, 'suite_name_{{:0{}}}.yml'.format(
             int(math.ceil(math.log10(size)))))
         calls = [call(filename.format(i), 'w') for i in range(size)]
         m.assert_has_calls(calls, any_order=True)
@@ -226,7 +226,7 @@ class RenderMiscSuites(unittest.TestCase):
 
         test_list = ['test{}'.format(i) for i in range(10)]
         m = mock_open(read_data=yaml.dump({'selector': {'roots': []}}))
-        with patch('buildscripts.generate_resmoke_suites.open', m, create=True):
+        with patch('buildscripts.evergreen_generate_resmoke_tasks.open', m, create=True):
             render_misc_suite(test_list, 'suite_name')
         handle = m()
 
@@ -246,16 +246,16 @@ class RenderMiscSuites(unittest.TestCase):
   - test9
   roots: []
 """)
-        calls = [call(os.path.join(grs.TEST_SUITE_DIR, 'suite_name.yml'), 'r')]
+        calls = [call(os.path.join(grt.TEST_SUITE_DIR, 'suite_name.yml'), 'r')]
         m.assert_has_calls(calls, any_order=True)
-        filename = os.path.join(grs.CONFIG_DIR, 'suite_name_misc.yml')
+        filename = os.path.join(grt.CONFIG_DIR, 'suite_name_misc.yml')
         calls = [call(filename, 'w')]
         m.assert_has_calls(calls, any_order=True)
 
 
 class PrepareDirectoryForSuite(unittest.TestCase):
     def test_no_directory(self):
-        with patch('buildscripts.generate_resmoke_suites.os') as mock_os:
+        with patch('buildscripts.evergreen_generate_resmoke_tasks.os') as mock_os:
             mock_os.path.exists.return_value = False
             prepare_directory_for_suite('tmp')
 
@@ -269,6 +269,8 @@ class GenerateEvgConfigTest(unittest.TestCase):
         for idx in range(count):
             suite = Mock()
             suite.name = "suite {0}".format(idx)
+            suite.max_runtime = 5.28
+            suite.get_runtime = lambda: 100.874
             suites.append(suite)
 
         return suites
@@ -281,6 +283,7 @@ class GenerateEvgConfigTest(unittest.TestCase):
         options.variant = "buildvariant"
         options.suite = "suite"
         options.task = "suite"
+        options.use_large_distro = None
 
         return options
 
@@ -288,20 +291,20 @@ class GenerateEvgConfigTest(unittest.TestCase):
         options = self.generate_mock_options()
         suites = self.generate_mock_suites(3)
 
-        config = grs.generate_evg_config(suites, options).to_map()
+        config = grt.generate_evg_config(suites, options).to_map()
 
         self.assertEqual(len(config["tasks"]), len(suites) + 1)
-        command1 = config["tasks"][0]["commands"][1]
+        command1 = config["tasks"][0]["commands"][2]
         self.assertIn(options.resmoke_args, command1["vars"]["resmoke_args"])
         self.assertIn(options.run_multiple_jobs, command1["vars"]["run_multiple_jobs"])
-        self.assertEqual("run tests", command1["func"])
+        self.assertEqual("run generated tests", command1["func"])
 
     def test_evg_config_is_created_with_diff_task_and_suite(self):
         options = self.generate_mock_options()
         options.task = "task"
         suites = self.generate_mock_suites(3)
 
-        config = grs.generate_evg_config(suites, options).to_map()
+        config = grt.generate_evg_config(suites, options).to_map()
 
         self.assertEqual(len(config["tasks"]), len(suites) + 1)
         display_task = config["buildvariants"][0]["display_tasks"][0]
@@ -313,7 +316,20 @@ class GenerateEvgConfigTest(unittest.TestCase):
         task = config["tasks"][0]
         self.assertIn(options.variant, task["name"])
         self.assertIn(task["name"], display_task["execution_tasks"])
-        self.assertIn(options.suite, task["commands"][1]["vars"]["resmoke_args"])
+        self.assertIn(options.suite, task["commands"][2]["vars"]["resmoke_args"])
+
+    def test_evg_config_can_use_large_distro(self):
+        options = self.generate_mock_options()
+        options.use_large_distro = "true"
+        options.large_distro_name = "large distro name"
+
+        suites = self.generate_mock_suites(3)
+
+        config = grt.generate_evg_config(suites, options).to_map()
+
+        self.assertEqual(len(config["tasks"]), len(suites) + 1)
+        self.assertEqual(options.large_distro_name,
+                         config["buildvariants"][0]["tasks"][0]["distros"][0])
 
 
 class MainTest(unittest.TestCase):
@@ -323,18 +339,19 @@ class MainTest(unittest.TestCase):
             "test_file": "test{}.js".format(i), "avg_duration_pass": 60, "num_pass": 1
         } for i in range(100)]
 
-        main = grs.Main(evg)
+        main = grt.Main(evg)
         main.options = Mock()
-        main.options.execution_time_minutes = 10
-        main.config_options = grs.ConfigOptions(2, 15, "project", "", 1, True, "task", "suite",
-                                                "variant")
+        main.config_options = grt.ConfigOptions(2, 15, "project", "", 1, 10, True, "task", "suite",
+                                                "variant", False, "")
 
-        suites = main.calculate_suites(_DATE, _DATE)
+        with patch('os.path.exists') as exists_mock:
+            exists_mock.return_value = True
+            suites = main.calculate_suites(_DATE, _DATE)
 
-        # There are 100 tests taking 1 minute, with a target of 10 min we expect 10 suites.
-        self.assertEqual(10, len(suites))
-        for suite in suites:
-            self.assertEqual(10, len(suite.tests))
+            # There are 100 tests taking 1 minute, with a target of 10 min we expect 10 suites.
+            self.assertEqual(10, len(suites))
+            for suite in suites:
+                self.assertEqual(10, len(suite.tests))
 
     def test_calculate_suites_fallback(self):
         response = Mock()
@@ -342,11 +359,11 @@ class MainTest(unittest.TestCase):
         evg = Mock()
         evg.test_stats.side_effect = requests.HTTPError(response=response)
 
-        main = grs.Main(evg)
+        main = grt.Main(evg)
         main.options = Mock()
         main.options.execution_time_minutes = 10
-        main.config_options = grs.ConfigOptions(2, 15, "project", "", 1, True, "task", "suite",
-                                                "variant")
+        main.config_options = grt.ConfigOptions(2, 15, "project", "", 1, 30, True, "task", "suite",
+                                                "variant", False, "")
         main.list_tests = Mock(return_value=["test{}.js".format(i) for i in range(100)])
 
         suites = main.calculate_suites(_DATE, _DATE)
@@ -361,12 +378,28 @@ class MainTest(unittest.TestCase):
         evg = Mock()
         evg.test_stats.side_effect = requests.HTTPError(response=response)
 
-        main = grs.Main(evg)
+        main = grt.Main(evg)
         main.options = Mock()
         main.options.execution_time_minutes = 10
-        main.config_options = grs.ConfigOptions(2, 15, "project", "", 1, True, "task", "suite",
-                                                "variant")
+        main.config_options = grt.ConfigOptions(2, 15, "project", "", 1, 30, True, "task", "suite",
+                                                "variant", False, "")
         main.list_tests = Mock(return_value=["test{}.js".format(i) for i in range(100)])
 
         with self.assertRaises(requests.HTTPError):
             main.calculate_suites(_DATE, _DATE)
+
+    def test_filter_missing_files(self):
+        tests_runtimes = [
+            ("dir1/file1.js", 20.32),
+            ("dir2/file2.js", 24.32),
+            ("dir1/file3.js", 36.32),
+        ]
+
+        with patch("os.path.exists") as exists_mock:
+            exists_mock.side_effect = [False, True, True]
+            filtered_list = grt.Main.filter_existing_tests(tests_runtimes)
+
+            self.assertEqual(2, len(filtered_list))
+            self.assertNotIn(tests_runtimes[0], filtered_list)
+            self.assertIn(tests_runtimes[2], filtered_list)
+            self.assertIn(tests_runtimes[1], filtered_list)
diff --git a/etc/evergreen.yml b/etc/evergreen.yml
index d8c6133951d..e1e764a9aaf 100644
--- a/etc/evergreen.yml
+++ b/etc/evergreen.yml
@@ -734,6 +734,20 @@ functions:
           fi
         done
 
+  "configure evergreen api credentials" : &configure_evergreen_api_credentials
+    command: shell.exec
+    type: test
+    params:
+      working_dir: src
+      silent: true
+      script: |
+        # Create the Evergreen API credentials
+        cat > .evergreen.yml <<END_OF_CREDS
+        api_server_host: https://evergreen.mongodb.com/api
+        api_key: "${evergreen_api_key}"
+        user: "${evergreen_api_user}"
+        END_OF_CREDS
+
   "git get project" : &git_get_project
     command: git.get_project
     params:
@@ -1467,149 +1481,248 @@ functions:
           if [[ ! -f $venv2_bin/python2 ]]; then ln -sfv "$py2_exe" "$venv2_bin/python2"; fi
           if [[ ! -f $venv2_bin/python3 ]]; then ln -sfv "$py3_exe" "$venv2_bin/python3"; fi
 
-  "run tests" :
-    - *install_pip_requirements
-    - *determine_task_timeout
-    - *update_task_timeout_expansions
-    - *update_task_timeout
-    - command: expansions.update
-      params:
-        updates:
-        - key: aws_key_remote
-          value: ${mongodatafiles_aws_key}
-        - key: aws_profile_remote
-          value: mongodata_aws
-        - key: aws_secret_remote
-          value: ${mongodatafiles_aws_secret}
-    - *set_up_remote_credentials
-    - *determine_resmoke_jobs
-    - *update_resmoke_jobs_expansions
-    - command: shell.exec
-      type: test
-      params:
-        working_dir: src
-        shell: bash
-        script: |
-          set -o errexit
-          set -o verbose
-
-          if [[ ${disable_unit_tests|false} = "false" && ! -f ${skip_tests|/dev/null} ]]; then
+  "execute resmoke tests" : &execute_resmoke_tests
+    command: shell.exec
+    type: test
+    params:
+      working_dir: src
+      shell: bash
+      script: |
+        set -o errexit
+        set -o verbose
 
-          # activate the virtualenv if it has been set up
-          ${activate_virtualenv}
+        if [[ ${disable_unit_tests|false} = "false" && ! -f ${skip_tests|/dev/null} ]]; then
 
-          # Set the TMPDIR environment variable to be a directory in the task's working
-          # directory so that temporary files created by processes spawned by resmoke.py get
-          # cleaned up after the task completes. This also ensures the spawned processes
-          # aren't impacted by limited space in the mount point for the /tmp directory.
-          export TMPDIR="${workdir}/tmp"
-          mkdir -p $TMPDIR
+        # activate the virtualenv if it has been set up
+        ${activate_virtualenv}
 
-          if [ -f /proc/self/coredump_filter ]; then
-            # Set the shell process (and its children processes) to dump ELF headers (bit 4),
-            # anonymous shared mappings (bit 1), and anonymous private mappings (bit 0).
-            echo 0x13 > /proc/self/coredump_filter
-
-            if [ -f /sbin/sysctl ]; then
-              # Check that the core pattern is set explicitly on our distro image instead
-              # of being the OS's default value. This ensures that coredump names are consistent
-              # across distros and can be picked up by Evergreen.
-              core_pattern=$(/sbin/sysctl -n "kernel.core_pattern")
-              if [ "$core_pattern" = "dump_%e.%p.core" ]; then
-                echo "Enabling coredumps"
-                ulimit -c unlimited
-              fi
+        # Set the TMPDIR environment variable to be a directory in the task's working
+        # directory so that temporary files created by processes spawned by resmoke.py get
+        # cleaned up after the task completes. This also ensures the spawned processes
+        # aren't impacted by limited space in the mount point for the /tmp directory.
+        export TMPDIR="${workdir}/tmp"
+        mkdir -p $TMPDIR
+
+        if [ -f /proc/self/coredump_filter ]; then
+          # Set the shell process (and its children processes) to dump ELF headers (bit 4),
+          # anonymous shared mappings (bit 1), and anonymous private mappings (bit 0).
+          echo 0x13 > /proc/self/coredump_filter
+
+          if [ -f /sbin/sysctl ]; then
+            # Check that the core pattern is set explicitly on our distro image instead
+            # of being the OS's default value. This ensures that coredump names are consistent
+            # across distros and can be picked up by Evergreen.
+            core_pattern=$(/sbin/sysctl -n "kernel.core_pattern")
+            if [ "$core_pattern" = "dump_%e.%p.core" ]; then
+              echo "Enabling coredumps"
+              ulimit -c unlimited
             fi
           fi
+        fi
 
-          extra_args="$extra_args --jobs=${resmoke_jobs|1}"
+        extra_args="$extra_args --jobs=${resmoke_jobs|1}"
 
-          if [ ${should_shuffle|true} = true ]; then
-            extra_args="$extra_args --shuffle"
-          fi
+        if [ ${should_shuffle|true} = true ]; then
+          extra_args="$extra_args --shuffle"
+        fi
 
-          if [ ${continue_on_failure|true} = true ]; then
-            extra_args="$extra_args --continueOnFailure"
-          fi
+        if [ ${continue_on_failure|true} = true ]; then
+          extra_args="$extra_args --continueOnFailure"
+        fi
 
-          # Default storageEngineCacheSizeGB to 1. Override on individual test config if needed.
-          # resmoke will assign to the appropriate parameter on storage engines that support it.
-          set +o errexit
-          echo "${resmoke_args}" | grep -q storageEngineCacheSizeGB
-          if [ $? -eq 1 ]; then
-            extra_args="$extra_args --storageEngineCacheSizeGB=1"
-          fi
-          set -o errexit
+        # Default storageEngineCacheSizeGB to 1. Override on individual test config if needed.
+        # resmoke will assign to the appropriate parameter on storage engines that support it.
+        set +o errexit
+        echo "${resmoke_args}" | grep -q storageEngineCacheSizeGB
+        if [ $? -eq 1 ]; then
+          extra_args="$extra_args --storageEngineCacheSizeGB=1"
+        fi
+        set -o errexit
 
 
-          # Reduce the JSHeapLimit for the serial_run task task on Code Coverage builder variant.
-          if [[ "${build_variant}" = "enterprise-rhel-62-64-bit-coverage" && "${task_name}" = "serial_run" ]]; then
-            extra_args="$extra_args --mongodSetParameter {'jsHeapLimitMB':10}"
-          fi
+        # Reduce the JSHeapLimit for the serial_run task task on Code Coverage builder variant.
+        if [[ "${build_variant}" = "enterprise-rhel-62-64-bit-coverage" && "${task_name}" = "serial_run" ]]; then
+          extra_args="$extra_args --mongodSetParameter {'jsHeapLimitMB':10}"
+        fi
 
-          path_value="$PATH"
-          if [ ${variant_path_suffix} ]; then
-            path_value="$path_value:${variant_path_suffix}"
-          fi
-          if [ ${task_path_suffix} ]; then
-            path_value="$path_value:${task_path_suffix}"
-          fi
+        path_value="$PATH"
+        if [ ${variant_path_suffix} ]; then
+          path_value="$path_value:${variant_path_suffix}"
+        fi
+        if [ ${task_path_suffix} ]; then
+          path_value="$path_value:${task_path_suffix}"
+        fi
 
-          if [ "${is_patch}" = "true" ]; then
-            extra_args="$extra_args --tagFile=etc/test_lifecycle.yml --patchBuild"
-          else
-            extra_args="$extra_args --tagFile=etc/test_retrial.yml"
-          fi
+        if [ "${is_patch}" = "true" ]; then
+          extra_args="$extra_args --tagFile=etc/test_lifecycle.yml --patchBuild"
+        else
+          extra_args="$extra_args --tagFile=etc/test_retrial.yml"
+        fi
 
-          # The "resmoke_wrapper" expansion is used by the 'burn_in_tests' task to wrap the resmoke.py
-          # invocation. It doesn't set any environment variables and should therefore come last in
-          # this list of expansions.
-          set +o errexit
-          PATH="$path_value"                              \
-              AWS_PROFILE=${aws_profile_remote}           \
-              ${gcov_environment}                         \
-              ${lang_environment}                         \
-              ${san_options}                              \
-              ${san_symbolizer}                           \
-              ${snmp_config_path}                         \
-              ${resmoke_wrapper}                          \
-              $python buildscripts/evergreen_run_tests.py \
-                  ${resmoke_args}                         \
-                  $extra_args                             \
-                  ${test_flags}                           \
-                  --log=buildlogger                       \
-                  --staggerJobs=on                        \
-                  --buildId=${build_id}                   \
-                  --distroId=${distro_id}                 \
-                  --executionNumber=${execution}          \
-                  --projectName=${project}                \
-                  --gitRevision=${revision}               \
-                  --revisionOrderId=${revision_order_id}  \
-                  --taskId=${task_id}                     \
-                  --taskName=${task_name}                 \
-                  --variantName=${build_variant}          \
-                  --versionId=${version_id}               \
-                  --archiveFile=archive.json              \
-                  --reportFile=report.json                \
-                  --perfReportFile=perf.json
-          resmoke_exit_code=$?
+        # The "resmoke_wrapper" expansion is used by the 'burn_in_tests' task to wrap the resmoke.py
+        # invocation. It doesn't set any environment variables and should therefore come last in
+        # this list of expansions.
+        set +o errexit
+        PATH="$path_value"                              \
+            AWS_PROFILE=${aws_profile_remote}           \
+            ${gcov_environment}                         \
+            ${lang_environment}                         \
+            ${san_options}                              \
+            ${san_symbolizer}                           \
+            ${snmp_config_path}                         \
+            ${resmoke_wrapper}                          \
+            $python buildscripts/evergreen_run_tests.py \
+                ${resmoke_args}                         \
+                $extra_args                             \
+                ${test_flags}                           \
+                --log=buildlogger                       \
+                --staggerJobs=on                        \
+                --buildId=${build_id}                   \
+                --distroId=${distro_id}                 \
+                --executionNumber=${execution}          \
+                --projectName=${project}                \
+                --gitRevision=${revision}               \
+                --revisionOrderId=${revision_order_id}  \
+                --taskId=${task_id}                     \
+                --taskName=${task_name}                 \
+                --variantName=${build_variant}          \
+                --versionId=${version_id}               \
+                --archiveFile=archive.json              \
+                --reportFile=report.json                \
+                --perfReportFile=perf.json
+        resmoke_exit_code=$?
+        set -o errexit
+
+        # 74 is exit code for IOError on POSIX systems, which is raised when the machine is
+        # shutting down.
+        #
+        # 75 is exit code resmoke.py uses when the log output would be incomplete due to failing
+        # to communicate with logkeeper.
+        if [[ $resmoke_exit_code = 74 || $resmoke_exit_code = 75 ]]; then
+          echo $resmoke_exit_code > run_tests_infrastructure_failure
+          exit 0
+        elif [ $resmoke_exit_code != 0 ]; then
+          # On failure save the resmoke exit code.
+          echo $resmoke_exit_code > resmoke_error_code
+        fi
+        exit $resmoke_exit_code
+        fi # end if [[ ${disable_unit_tests} && ! -f ${skip_tests|/dev/null} ]]
+
+  "retrieve generated test configuration": &retrieve_generated_test_configuration
+    command: s3.get
+    params:
+      aws_key: ${aws_key}
+      aws_secret: ${aws_secret}
+      bucket: mciuploads
+      remote_file: ${project}/${build_variant}/${revision}/generate_tasks/${task}_gen-${build_id}.tgz
+      local_file: "generate_tasks_config.tgz"
+
+  "extract generated test configuration": &extract_generated_test_configuration
+    command: shell.exec
+    type: test
+    params:
+      shell: bash
+      script: |
+        set -o verbose
+        set -o errexit
+
+        target_dir="src/generated_resmoke_config"
+        mkdir -p $target_dir
+        mv generate_tasks_config.tgz $target_dir
+
+        cd $target_dir
+        tar xzf generate_tasks_config.tgz
+        ls
+
+  "generate resmoke tasks":
+    - *git_get_project
+    - *configure_evergreen_api_credentials
+    - command: expansions.write
+      params:
+        file: src/expansions.yml
+
+    - command: shell.exec
+      params:
+        working_dir: src
+        script: |
           set -o errexit
+          set -o verbose
 
-          # 74 is exit code for IOError on POSIX systems, which is raised when the machine is
-          # shutting down.
-          #
-          # 75 is exit code resmoke.py uses when the log output would be incomplete due to failing
-          # to communicate with logkeeper.
-          if [[ $resmoke_exit_code = 74 || $resmoke_exit_code = 75 ]]; then
-            echo $resmoke_exit_code > run_tests_infrastructure_failure
-            exit 0
-          elif [ $resmoke_exit_code != 0 ]; then
-            # On failure save the resmoke exit code.
-            echo $resmoke_exit_code > resmoke_error_code
-          fi
-          exit $resmoke_exit_code
-          fi # end if [[ ${disable_unit_tests} && ! -f ${skip_tests|/dev/null} ]]
+          ${activate_virtualenv}
+          $python -m pip install -r etc/pip/evgtest-requirements.txt
+          $python buildscripts/evergreen_generate_resmoke_tasks.py --expansion-file expansions.yml
+
+    - command: archive.targz_pack
+      params:
+        target: generate_tasks_config.tgz
+        source_dir: src/generated_resmoke_config
+        include:
+          - "*"
+
+    - command: s3.put
+      params:
+        aws_key: ${aws_key}
+        aws_secret: ${aws_secret}
+        local_file: generate_tasks_config.tgz
+        remote_file: ${project}/${build_variant}/${revision}/generate_tasks/${task_name}-${build_id}.tgz
+        bucket: mciuploads
+        permissions: public-read
+        content_type: ${content_type|application/gzip}
+        display_name: Generated Task Config - Execution ${execution}
+
+    - command: generate.tasks
+      params:
+        files:
+          - src/generated_resmoke_config/${task}.json
+
+  "run generated tests" :
+  - *install_pip_requirements
+  - *retrieve_generated_test_configuration
+  - *extract_generated_test_configuration
+  - command: expansions.update
+    params:
+      updates:
+      - key: aws_key_remote
+        value: ${mongodatafiles_aws_key}
+      - key: aws_profile_remote
+        value: mongodata_aws
+      - key: aws_secret_remote
+        value: ${mongodatafiles_aws_secret}
+  - *set_up_remote_credentials
+  - *determine_resmoke_jobs
+  - *update_resmoke_jobs_expansions
+  - *execute_resmoke_tests
+    # The existence of the "run_tests_infrastructure_failure" file indicates this failure isn't
+    # directly actionable. We use type=setup rather than type=system or type=test for this command
+    # because we don't intend for any human to look at this failure.
+  - command: shell.exec
+    type: setup
+    params:
+      working_dir: src
+      script: |
+        set -o verbose
+        if [ -f run_tests_infrastructure_failure ]; then
+          exit $(cat run_tests_infrastructure_failure)
+        fi
 
+  "run tests" :
+    - *install_pip_requirements
+    - *determine_task_timeout
+    - *update_task_timeout_expansions
+    - *update_task_timeout
+    - command: expansions.update
+      params:
+        updates:
+        - key: aws_key_remote
+          value: ${mongodatafiles_aws_key}
+        - key: aws_profile_remote
+          value: mongodata_aws
+        - key: aws_secret_remote
+          value: ${mongodatafiles_aws_secret}
+    - *set_up_remote_credentials
+    - *determine_resmoke_jobs
+    - *update_resmoke_jobs_expansions
+    - *execute_resmoke_tests
       # The existence of the "run_tests_infrastructure_failure" file indicates this failure isn't
       # directly actionable. We use type=setup rather than type=system or type=test for this command
       # because we don't intend for any human to look at this failure.
@@ -6267,6 +6380,16 @@ tasks:
     vars:
       resmoke_args: --suites=sasl --storageEngine=wiredTiger
 
+- name: sharding_gen
+  depends_on: []
+  commands:
+  - func: "generate resmoke tasks"
+    vars:
+      task: sharding
+      use_large_distro: "true"
+      resmoke_args: --storageEngine=wiredTiger
+      fallback_num_sub_suites: 15
+
 - <<: *task_template
   name: sharding
   commands:
@@ -8232,6 +8355,7 @@ tasks:
   depends_on: []
   commands:
   - func: "git get project"
+  - func: "configure evergreen api credentials"
   - command: shell.exec
     timeout_secs: 14400 # Timeout if there is no output for 4 hours
     type: test
@@ -8257,13 +8381,6 @@ tasks:
         $(echo "${testlifecycle_jira_key_certificate}" | sed  's/^/  /')
         END_OF_CREDS
 
-        # Create the Evergreen API credentials
-        cat > .evergreen.yml <<END_OF_CREDS
-        api_server_host: https://evergreen.mongodb.com/api
-        api_key: "${evergreen_api_key}"
-        user: "${evergreen_api_user}"
-        END_OF_CREDS
-
         set -o verbose
 
         ${activate_virtualenv}
@@ -11430,11 +11547,12 @@ buildvariants:
     build_mongoreplay: true
     jstestfuzz_num_generated_files: 40
     jstestfuzz_concurrent_num_files: 10
+    target_resmoke_time: 20
+    large_distro_name: rhel62-large
   display_tasks:
   - *dbtest
   - *replica_sets_auth
   - *replica_sets_ese
-  - *sharding
   - *sharding_auth
   - *sharding_auth_audit
   - *sharding_ese
@@ -11679,54 +11797,7 @@ buildvariants:
   - name: sharded_multi_stmt_txn_jscore_passthrough
     distros:
     - rhel62-large
-  - name: sharding_0
-    distros:
-    - rhel62-large
-  - name: sharding_1
-    distros:
-    - rhel62-large
-  - name: sharding_2
-    distros:
-    - rhel62-large
-  - name: sharding_3
-    distros:
-    - rhel62-large
-  - name: sharding_4
-    distros:
-    - rhel62-large
-  - name: sharding_5
-    distros:
-    - rhel62-large
-  - name: sharding_6
-    distros:
-    - rhel62-large
-  - name: sharding_7
-    distros:
-    - rhel62-large
-  - name: sharding_8
-    distros:
-    - rhel62-large
-  - name: sharding_9
-    distros:
-    - rhel62-large
-  - name: sharding_10
-    distros:
-    - rhel62-large
-  - name: sharding_11
-    distros:
-    - rhel62-large
-  - name: sharding_12
-    distros:
-    - rhel62-large
-  - name: sharding_13
-    distros:
-    - rhel62-large
-  - name: sharding_14
-    distros:
-    - rhel62-large
-  - name: sharding_misc
-    distros:
-    - rhel62-large
+  - name: sharding_gen
   - name: sharding_auth_0
     distros:
     - rhel62-large