SERVER-39311 Enable burn_in repetition count: at least 2 executions, with a maximum of 1000 excutions or 10 minutes of execution time

3 files changed, 323 insertions, 256 deletions

diff --git a/buildscripts/burn_in_tests.py b/buildscripts/burn_in_tests.py
index 398184d4f96..d39d3c0eef9 100644
--- a/buildscripts/burn_in_tests.py
+++ b/buildscripts/burn_in_tests.py
@@ -47,6 +47,9 @@ SUITE_FILES = ["with_server"]
 SUPPORTED_TEST_KINDS = ("fsm_workload_test", "js_test", "json_schema_test",
                         "multi_stmt_txn_passthrough", "parallel_fsm_workload_test")
 
+BURN_IN_TESTS_GEN_TASK = "burn_in_tests_gen"
+BURN_IN_TESTS_TASK = "burn_in_tests"
+
 
 def parse_command_line():
     """Parse command line options."""
@@ -65,9 +68,13 @@ def parse_command_line():
                       help="The base commit to compare to for determining changes.")
 
     parser.add_option("--buildVariant", dest="buildvariant", default=None,
-                      help=("The buildvariant the tasks will execute on. Required when"
+                      help=("The buildvariant to select the tasks. Required when"
                             " generating the JSON file with test executor information"))
 
+    parser.add_option("--runBuildVariant", dest="run_buildvariant", default=None,
+                      help=("The buildvariant the tasks will execute on. If not specied then tasks"
+                            " will execute on the the buildvariant specied in --buildVariant."))
+
     parser.add_option("--distro", dest="distro", default=None,
                       help=("The distro the tasks will execute on. Can only be specified"
                             " with --generateTasksFile."))
@@ -119,6 +126,14 @@ def parse_command_line():
     return options, args
 
 
+def check_variant(buildvariant, parser):
+    """Check if the buildvariant is found in the evergreen file."""
+    evg_conf = evergreen.parse_evergreen_file(EVERGREEN_FILE)
+    if not evg_conf.get_variant(buildvariant):
+        parser.error("Buildvariant '{}' not found in {}, select from:\n\t{}".format(
+            buildvariant, EVERGREEN_FILE, "\n\t".join(sorted(evg_conf.variant_names))))
+
+
 def validate_options(parser, options):
     """Validate command line options."""
 
@@ -139,10 +154,10 @@ def validate_options(parser, options):
         parser.error("Must specify --buildVariant to find changed tests")
 
     if options.buildvariant:
-        evg_conf = evergreen.parse_evergreen_file(EVERGREEN_FILE)
-        if not evg_conf.get_variant(options.buildvariant):
-            parser.error("Buildvariant '{}' not found in {}, select from:\n\t{}".format(
-                options.buildvariant, EVERGREEN_FILE, "\n\t".join(sorted(evg_conf.variant_names))))
+        check_variant(options.buildvariant, parser)
+
+    if options.run_buildvariant:
+        check_variant(options.run_buildvariant, parser)
 
 
 def find_last_activated_task(revisions, variant, branch_name):
@@ -428,15 +443,19 @@ def _sub_task_name(variant, task, task_num):
     return "burn_in:{}_{}_{}".format(variant, task, task_num)
 
 
+def _get_run_buildvariant(options):
+    """Return the build variant to execute the tasks on."""
+    if options.run_buildvariant:
+        return options.run_buildvariant
+    return options.buildvariant
+
+
 def create_generate_tasks_file(options, tests_by_task):
     """Create the Evergreen generate.tasks file."""
 
-    if not tests_by_task:
-        return
-
     evg_config = Configuration()
     task_specs = []
-    task_names = ["burn_in_tests_gen"]
+    task_names = [BURN_IN_TESTS_GEN_TASK]
     for task in sorted(tests_by_task):
         multiversion_path = tests_by_task[task].get("use_multiversion")
         for test_num, test in enumerate(tests_by_task[task]["tests"]):
@@ -461,8 +480,8 @@ def create_generate_tasks_file(options, tests_by_task):
             commands.append(CommandDefinition().function("run tests").vars(run_tests_vars))
             evg_sub_task.commands(commands)
 
-    display_task = DisplayTaskDefinition("burn_in_tests").execution_tasks(task_names)
-    evg_config.variant(options.buildvariant).tasks(task_specs).display_task(display_task)
+    display_task = DisplayTaskDefinition(BURN_IN_TESTS_TASK).execution_tasks(task_names)
+    evg_config.variant(_get_run_buildvariant(options)).tasks(task_specs).display_task(display_task)
 
     _write_json_file(evg_config.to_map(), options.generate_tasks_file)
 
diff --git a/buildscripts/tests/ciconfig/test_evergreen.py b/buildscripts/tests/ciconfig/test_evergreen.py
index d7b345ca144..fd57a449f79 100644
--- a/buildscripts/tests/ciconfig/test_evergreen.py
+++ b/buildscripts/tests/ciconfig/test_evergreen.py
@@ -71,7 +71,7 @@ class TestEvergreenProjectConfig(unittest.TestCase):
         self.assertIn("amazon", self.conf.distro_names)
 
 
-class TestTask(unittest.TestCase):  #pylint: disable=too-many-public-methods
+class TestTask(unittest.TestCase):  # pylint: disable=too-many-public-methods
     """Unit tests for the Task class."""
 
     def test_from_dict(self):
@@ -147,19 +147,6 @@ class TestTask(unittest.TestCase):  #pylint: disable=too-many-public-methods
         self.assertIsNone(task.multiversion_setup_command)
         self.assertIsNone(task.multiversion_path)
 
-    def test_get_vars_suite_name_run_tests(self):
-        suite_name = "core"
-        task_commands = [{
-            "func": "run tests",
-            "vars": {"resmoke_args": "--suites={} --shellWriteMode=commands".format(suite_name)}
-        }]
-
-        task_dict = {"name": "jsCore", "commands": task_commands}
-        task = _evergreen.Task(task_dict)
-
-        with self.assertRaises(ValueError):
-            task.get_vars_suite_name(task_commands[0]["vars"])
-
     def test_resmoke_args_gen(self):
         task_name = "core"
         task_commands = [{
@@ -252,6 +239,46 @@ class TestTask(unittest.TestCase):  #pylint: disable=too-many-public-methods
         self.assertDictEqual(task_commands[0], task.generate_resmoke_tasks_command)
         self.assertEqual(task_name, task.get_vars_task_name(task_commands[0]["vars"]))
 
+    def test_gen_resmoke_multiversion(self):
+        multiversion_path = "/data/multiversion"
+        task_name = "core"
+        task_commands = [{
+            "func": "generate resmoke tasks", "vars": {
+                "task": task_name, "resmoke_args": "--shellWriteMode=commands",
+                "use_multiversion": multiversion_path
+            }
+        }]
+        task_dict = {"name": "jsCore", "commands": task_commands}
+        task = _evergreen.Task(task_dict)
+
+        self.assertTrue(task.is_multiversion_task)
+        self.assertEqual(multiversion_path, task.multiversion_path)
+
+    def test_gen_resmoke_no_multiversion(self):
+        task_name = "core"
+        task_commands = [{
+            "func": "generate resmoke tasks",
+            "vars": {"task": task_name, "resmoke_args": "--shellWriteMode=commands"}
+        }]
+        task_dict = {"name": "jsCore", "commands": task_commands}
+        task = _evergreen.Task(task_dict)
+
+        self.assertFalse(task.is_multiversion_task)
+        self.assertIsNone(task.multiversion_path)
+
+    def test_get_vars_suite_name_run_tests(self):
+        suite_name = "core"
+        task_commands = [{
+            "func": "run tests",
+            "vars": {"resmoke_args": "--suites={} --shellWriteMode=commands".format(suite_name)}
+        }]
+
+        task_dict = {"name": "jsCore", "commands": task_commands}
+        task = _evergreen.Task(task_dict)
+
+        with self.assertRaises(ValueError):
+            task.get_vars_suite_name(task_commands[0]["vars"])
+
     def test_get_vars_suite_name_generate_resmoke_tasks(self):
         task_name = "jsCore"
         suite_name = "core"
@@ -274,7 +301,7 @@ class TestTask(unittest.TestCase):  #pylint: disable=too-many-public-methods
         with self.assertRaises(TypeError):
             task.get_vars_suite_name(None)
 
-    def test_get_vars_suite_name_gen_resmoke_notasksuite(self):
+    def test_get_vars_suite_name_gen_resmoke_notask_nosuite(self):
         task_name = "jsCore"
         task_commands = [{
             "func": "generate resmoke tasks", "vars": {"resmoke_args": "--shellWriteMode=commands"}
@@ -285,33 +312,6 @@ class TestTask(unittest.TestCase):  #pylint: disable=too-many-public-methods
         with self.assertRaises(ValueError):
             task.get_vars_suite_name(task_commands[0]["vars"])
 
-    def test_gen_resmoke_multiversion(self):
-        multiversion_path = "/data/multiversion"
-        task_name = "core"
-        task_commands = [{
-            "func": "generate resmoke tasks", "vars": {
-                "task": task_name, "resmoke_args": "--shellWriteMode=commands",
-                "use_multiversion": multiversion_path
-            }
-        }]
-        task_dict = {"name": "jsCore", "commands": task_commands}
-        task = _evergreen.Task(task_dict)
-
-        self.assertTrue(task.is_multiversion_task)
-        self.assertEqual(multiversion_path, task.multiversion_path)
-
-    def test_gen_resmoke_no_multiversion(self):
-        task_name = "core"
-        task_commands = [{
-            "func": "generate resmoke tasks",
-            "vars": {"task": task_name, "resmoke_args": "--shellWriteMode=commands"}
-        }]
-        task_dict = {"name": "jsCore", "commands": task_commands}
-        task = _evergreen.Task(task_dict)
-
-        self.assertFalse(task.is_multiversion_task)
-        self.assertIsNone(task.multiversion_path)
-
     def test_get_vars_task_name(self):
         task_name = "task_name"
         task_commands = [{
diff --git a/buildscripts/tests/test_burn_in_tests.py b/buildscripts/tests/test_burn_in_tests.py
index 0ab6bec59c3..4f04795df04 100644
--- a/buildscripts/tests/test_burn_in_tests.py
+++ b/buildscripts/tests/test_burn_in_tests.py
@@ -20,21 +20,118 @@ EVG_CLIENT = "buildscripts.client.evergreen"
 GIT = "buildscripts.git"
 RESMOKELIB = "buildscripts.resmokelib"
 
+GENERATE_RESMOKE_TASKS_NAME = "this_is_a_gen_task"
+GET_GENERATE_RESMOKE_TASKS_NAME = lambda _: GENERATE_RESMOKE_TASKS_NAME
+GENERATE_RESMOKE_TASKS_COMMAND = {
+    "func": "generate resmoke tasks", "vars": {
+        "task": GENERATE_RESMOKE_TASKS_NAME, "suite": "suite3",
+        "resmoke_args": "--shellWriteMode=commands"
+    }
+}
 
-class ValidateOptions(unittest.TestCase):
-    class ParserError(Exception):
-        pass
+GENERATE_RESMOKE_TASKS_COMMAND2 = {
+    "func": "generate resmoke tasks",
+    "vars": {"task": GENERATE_RESMOKE_TASKS_NAME, "resmoke_args": "--shellWriteMode=commands"}
+}
 
-    @staticmethod
-    def _raise_parser_error(err):
-        raise ValidateOptions.ParserError(err)
+MULTIVERSION_PATH = "/data/multiversion"
+GENERATE_RESMOKE_TASKS_MULTIVERSION_COMMAND = {
+    "func": "generate resmoke tasks", "vars": {
+        "task": GENERATE_RESMOKE_TASKS_NAME, "resmoke_args": "--shellWriteMode=commands",
+        "use_multiversion": MULTIVERSION_PATH
+    }
+}
+
+MULTIVERSION_COMMAND = {"func": "do multiversion setup"}
+RUN_TESTS_MULTIVERSION_COMMAND = {
+    "func": "run tests", "vars": {
+        "task": GENERATE_RESMOKE_TASKS_NAME, "resmoke_args": "--shellWriteMode=commands",
+        "task_path_suffix": MULTIVERSION_PATH
+    }
+}
 
-    @staticmethod
-    def _mock_parser():
-        parser = Mock()
-        parser.error = ValidateOptions._raise_parser_error
-        return parser
 
+def tasks_mock(  #pylint: disable=too-many-arguments
+        tasks, generate_resmoke_tasks_command=None, get_vars_task_name=None, run_tests_command=None,
+        multiversion_path=None, multiversion_setup_command=None):
+    task_list = Mock()
+    task_list.tasks = []
+    for idx, task in enumerate(tasks):
+        task_list.tasks.append(Mock())
+        task_list.tasks[idx].is_generate_resmoke_task = generate_resmoke_tasks_command is not None
+        task_list.tasks[idx].is_run_tests_task = run_tests_command is not None
+        task_list.tasks[idx].is_multiversion_task = multiversion_path is not None
+        task_list.tasks[idx].generate_resmoke_tasks_command = generate_resmoke_tasks_command
+        task_list.tasks[idx].run_tests_command = run_tests_command
+        task_list.tasks[idx].get_vars_task_name = get_vars_task_name
+        task_list.tasks[idx].name = task["name"]
+        resmoke_args = task.get("combined_resmoke_args")
+        task_list.tasks[idx].combined_resmoke_args = resmoke_args
+        task_list.tasks[idx].resmoke_suite = evg.ResmokeArgs.get_arg(
+            resmoke_args, "suites") if resmoke_args else None
+        task_list.tasks[idx].multiversion_path = multiversion_path
+        task_list.tasks[idx].multiversion_setup_command = multiversion_setup_command
+
+    return task_list
+
+
+VARIANTS = {
+    "variantall":
+        tasks_mock([{"name": "task1", "combined_resmoke_args": "--suites=suite1 var1arg1"},
+                    {"name": "task2", "combined_resmoke_args": "--suites=suite1 var1arg2"},
+                    {"name": "task3", "combined_resmoke_args": "--suites=suite1 var1arg3"}]),
+    "variant1":
+        tasks_mock([{"name": "task1", "combined_resmoke_args": "--suites=suite1 var1arg1"},
+                    {"name": "task2"}]),
+    "variant2":
+        tasks_mock([{"name": "task2", "combined_resmoke_args": "var2arg1"},
+                    {"name": "task3", "combined_resmoke_args": "--suites=suite3 var2arg3"}]),
+    "variant3":
+        tasks_mock([{"name": "task2", "combined_resmoke_args": "var3arg1"}]),
+    "variant4":
+        tasks_mock([]),
+    "variant_multiversion":
+        tasks_mock(
+            [{"name": "multiversion_task", "combined_resmoke_args": "--suites=suite3 vararg"}],
+            run_tests_command=RUN_TESTS_MULTIVERSION_COMMAND,
+            multiversion_setup_command=RUN_TESTS_MULTIVERSION_COMMAND,
+            multiversion_path=MULTIVERSION_PATH),
+    "variant_generate_tasks":
+        tasks_mock([{
+            "name": GENERATE_RESMOKE_TASKS_NAME, "combined_resmoke_args": "--suites=suite3 vararg"
+        }], generate_resmoke_tasks_command=GENERATE_RESMOKE_TASKS_COMMAND,
+                   get_vars_task_name=GET_GENERATE_RESMOKE_TASKS_NAME),
+    "variant_generate_tasks_no_suite":
+        tasks_mock([{
+            "name": GENERATE_RESMOKE_TASKS_NAME, "combined_resmoke_args": "--suites=suite3 vararg"
+        }], generate_resmoke_tasks_command=GENERATE_RESMOKE_TASKS_COMMAND2,
+                   get_vars_task_name=GET_GENERATE_RESMOKE_TASKS_NAME),
+    "variant_generate_tasks_diff_names":
+        tasks_mock([{
+            "name": "gen_task_name_different_from_vars_task_name",
+            "combined_resmoke_args": "--suites=suite3 vararg"
+        }], generate_resmoke_tasks_command=GENERATE_RESMOKE_TASKS_COMMAND,
+                   get_vars_task_name=GET_GENERATE_RESMOKE_TASKS_NAME),
+    "variant_generate_tasks_multiversion":
+        tasks_mock([{
+            "name": GENERATE_RESMOKE_TASKS_NAME, "combined_resmoke_args": "--suites=suite3 vararg"
+        }], generate_resmoke_tasks_command=GENERATE_RESMOKE_TASKS_MULTIVERSION_COMMAND,
+                   get_vars_task_name=GET_GENERATE_RESMOKE_TASKS_NAME,
+                   multiversion_path=MULTIVERSION_PATH),
+}
+
+EVERGREEN_CONF = Mock()
+EVERGREEN_CONF.get_variant = VARIANTS.get
+EVERGREEN_CONF.variant_names = VARIANTS.keys()
+
+
+def _mock_parser():
+    parser = Mock()
+    parser.error = Mock()
+    return parser
+
+
+class TestValidateOptions(unittest.TestCase):
     @staticmethod
     def _mock_options():
         options = Mock()
@@ -43,109 +140,132 @@ class ValidateOptions(unittest.TestCase):
         options.repeat_tests_min = None
         options.repeat_tests_secs = None
         options.buildvariant = None
+        options.run_buildvariant = None
         options.test_list_file = None
         return options
 
     def test_validate_options_listfile_buildvariant(self):
-        parser = self._mock_parser()
+        mock_parser = _mock_parser()
         options = self._mock_options()
         options.test_list_file = "list_file.json"
         options.buildvariant = "variant1"
-        with patch(EVG_CI + ".parse_evergreen_file", return_value=CreateTaskList.evergreen_conf):
-            burn_in.validate_options(parser, options)
+        with patch(EVG_CI + ".parse_evergreen_file", return_value=EVERGREEN_CONF):
+            burn_in.validate_options(mock_parser, options)
+            mock_parser.error.assert_not_called()
 
     def test_validate_options_nolistfile_buildvariant(self):
-        parser = self._mock_parser()
+        mock_parser = _mock_parser()
         options = self._mock_options()
         options.buildvariant = "variant1"
-        with patch(EVG_CI + ".parse_evergreen_file", return_value=CreateTaskList.evergreen_conf):
-            burn_in.validate_options(parser, options)
+        with patch(EVG_CI + ".parse_evergreen_file", return_value=EVERGREEN_CONF):
+            burn_in.validate_options(mock_parser, options)
+            mock_parser.error.assert_not_called()
 
     def test_validate_options_listfile_nobuildvariant(self):
-        parser = self._mock_parser()
+        mock_parser = _mock_parser()
         options = self._mock_options()
         options.test_list_file = "list_file.json"
-        with patch(EVG_CI + ".parse_evergreen_file", return_value=CreateTaskList.evergreen_conf):
-            burn_in.validate_options(parser, options)
+        with patch(EVG_CI + ".parse_evergreen_file", return_value=EVERGREEN_CONF):
+            burn_in.validate_options(mock_parser, options)
+            mock_parser.error.assert_not_called()
 
     def test_validate_options_no_listfile_no_buildvariant(self):
-        parser = self._mock_parser()
+        mock_parser = _mock_parser()
         options = self._mock_options()
-        with patch(EVG_CI + ".parse_evergreen_file", return_value=CreateTaskList.evergreen_conf):
-            with self.assertRaises(self.ParserError):
-                burn_in.validate_options(parser, options)
+        with patch(EVG_CI + ".parse_evergreen_file", return_value=EVERGREEN_CONF):
+            burn_in.validate_options(mock_parser, options)
+            mock_parser.error.assert_called()
 
     def test_validate_options_buildvariant(self):
-        parser = self._mock_parser()
+        mock_parser = _mock_parser()
         options = self._mock_options()
         options.buildvariant = "variant1"
-        with patch(EVG_CI + ".parse_evergreen_file", return_value=CreateTaskList.evergreen_conf):
-            burn_in.validate_options(parser, options)
+        with patch(EVG_CI + ".parse_evergreen_file", return_value=EVERGREEN_CONF):
+            burn_in.validate_options(mock_parser, options)
+            mock_parser.error.assert_not_called()
+
+    def test_validate_options_run_buildvariant(self):
+        mock_parser = _mock_parser()
+        options = self._mock_options()
+        options.buildvariant = "variant1"
+        options.run_buildvariant = "variant1"
+        with patch(EVG_CI + ".parse_evergreen_file", return_value=EVERGREEN_CONF):
+            burn_in.validate_options(mock_parser, options)
+            mock_parser.error.assert_not_called()
 
     def test_validate_options_bad_buildvariant(self):
-        parser = self._mock_parser()
+        mock_parser = _mock_parser()
         options = self._mock_options()
         options.buildvariant = "badvariant1"
-        with patch(EVG_CI + ".parse_evergreen_file", return_value=CreateTaskList.evergreen_conf):
-            with self.assertRaises(self.ParserError):
-                burn_in.validate_options(parser, options)
+        with patch(EVG_CI + ".parse_evergreen_file", return_value=EVERGREEN_CONF):
+            burn_in.validate_options(mock_parser, options)
+            mock_parser.error.assert_called()
+
+    def test_validate_options_bad_run_buildvariant(self):
+        mock_parser = _mock_parser()
+        options = self._mock_options()
+        options.run_buildvariant = "badvariant1"
+        with patch(EVG_CI + ".parse_evergreen_file", return_value=EVERGREEN_CONF):
+            burn_in.validate_options(mock_parser, options)
+            mock_parser.error.assert_called()
 
     def test_validate_options_tests_max_no_tests_secs(self):
-        parser = self._mock_parser()
+        mock_parser = _mock_parser()
         options = self._mock_options()
         options.repeat_tests_max = 3
-        with patch(EVG_CI + ".parse_evergreen_file", return_value=CreateTaskList.evergreen_conf):
-            with self.assertRaises(self.ParserError):
-                burn_in.validate_options(parser, options)
+        with patch(EVG_CI + ".parse_evergreen_file", return_value=EVERGREEN_CONF):
+            burn_in.validate_options(mock_parser, options)
+            mock_parser.error.assert_called()
 
     def test_validate_options_tests_min_no_tests_secs(self):
-        parser = self._mock_parser()
+        mock_parser = _mock_parser()
         options = self._mock_options()
         options.repeat_tests_min = 3
-        with patch(EVG_CI + ".parse_evergreen_file", return_value=CreateTaskList.evergreen_conf):
-            with self.assertRaises(self.ParserError):
-                burn_in.validate_options(parser, options)
+        with patch(EVG_CI + ".parse_evergreen_file", return_value=EVERGREEN_CONF):
+            burn_in.validate_options(mock_parser, options)
+            mock_parser.error.assert_called()
 
     def test_validate_options_tests_min_gt_tests_max(self):
-        parser = self._mock_parser()
+        mock_parser = _mock_parser()
         options = self._mock_options()
         options.repeat_tests_min = 3
         options.repeat_tests_max = 2
-        with patch(EVG_CI + ".parse_evergreen_file", return_value=CreateTaskList.evergreen_conf):
-            with self.assertRaises(self.ParserError):
-                burn_in.validate_options(parser, options)
+        with patch(EVG_CI + ".parse_evergreen_file", return_value=EVERGREEN_CONF):
+            burn_in.validate_options(mock_parser, options)
+            mock_parser.error.assert_called()
 
     def test_validate_options_tests_secs(self):
-        parser = self._mock_parser()
+        mock_parser = _mock_parser()
         options = self._mock_options()
         options.buildvariant = "variant1"
         options.repeat_tests_min = 2
         options.repeat_tests_max = 1000
         options.repeat_tests_secs = 3
-        with patch(EVG_CI + ".parse_evergreen_file", return_value=CreateTaskList.evergreen_conf):
-            burn_in.validate_options(parser, options)
+        with patch(EVG_CI + ".parse_evergreen_file", return_value=EVERGREEN_CONF):
+            burn_in.validate_options(mock_parser, options)
+            mock_parser.error.assert_not_called()
 
     def test_validate_options_tests_secs_and_tests_num(self):
-        parser = self._mock_parser()
+        mock_parser = _mock_parser()
         options = self._mock_options()
         options.buildvariant = "variant1"
         options.repeat_tests_num = 1
         options.repeat_tests_min = 1
         options.repeat_tests_max = 3
         options.repeat_tests_secs = 3
-        with patch(EVG_CI + ".parse_evergreen_file", return_value=CreateTaskList.evergreen_conf):
-            with self.assertRaises(self.ParserError):
-                burn_in.validate_options(parser, options)
+        with patch(EVG_CI + ".parse_evergreen_file", return_value=EVERGREEN_CONF):
+            burn_in.validate_options(mock_parser, options)
+            mock_parser.error.assert_called()
 
     def test_validate_options_tests_secs_no_buildvariant(self):
-        parser = self._mock_parser()
+        mock_parser = _mock_parser()
         options = self._mock_options()
         options.repeat_tests_min = 1
         options.repeat_tests_max = 3
         options.repeat_tests_secs = 3
-        with patch(EVG_CI + ".parse_evergreen_file", return_value=CreateTaskList.evergreen_conf):
-            with self.assertRaises(self.ParserError):
-                burn_in.validate_options(parser, options)
+        with patch(EVG_CI + ".parse_evergreen_file", return_value=EVERGREEN_CONF):
+            burn_in.validate_options(mock_parser, options)
+            mock_parser.error.assert_called()
 
 
 class TestGetResmokeRepeatOptions(unittest.TestCase):
@@ -219,6 +339,41 @@ class TestGetResmokeRepeatOptions(unittest.TestCase):
         self.assertEqual(repeat_options, "--repeatSuites={}".format(burn_in.REPEAT_SUITES))
 
 
+class TestCheckVariant(unittest.TestCase):
+    @staticmethod
+    def test_check_variant():
+        mock_parser = _mock_parser()
+        buildvariant = "variant1"
+        with patch(EVG_CI + ".parse_evergreen_file", return_value=EVERGREEN_CONF):
+            burn_in.check_variant(buildvariant, mock_parser)
+            mock_parser.error.assert_not_called()
+
+    @staticmethod
+    def test_check_variant_badvariant():
+        mock_parser = _mock_parser()
+        buildvariant = "badvariant"
+        with patch(EVG_CI + ".parse_evergreen_file", return_value=EVERGREEN_CONF):
+            burn_in.check_variant(buildvariant, mock_parser)
+            mock_parser.error.assert_called()
+
+
+class TestGetRunBuildvariant(unittest.TestCase):
+    def test__get_run_buildvariant_rb(self):
+        run_buildvariant = "variant1"
+        buildvariant = "variant2"
+        options = Mock()
+        options.run_buildvariant = run_buildvariant
+        options.buildvariant = buildvariant
+        self.assertEqual(run_buildvariant, burn_in._get_run_buildvariant(options))
+
+    def test__get_run_buildvariant_bv(self):
+        buildvariant = "variant2"
+        options = Mock()
+        options.run_buildvariant = None
+        options.buildvariant = buildvariant
+        self.assertEqual(buildvariant, burn_in._get_run_buildvariant(options))
+
+
 class TestGetTaskName(unittest.TestCase):
     def test__get_task_name(self):
         name = "mytask"
@@ -317,6 +472,7 @@ class TestCreateGenerateTasksFile(unittest.TestCase):
     def _options_mock():
         options = Mock()
         options.buildvariant = None
+        options.run_buildvariant = None
         options.distro = None
         return options
 
@@ -386,10 +542,10 @@ class TestCreateGenerateTasksFile(unittest.TestCase):
             self.assertEqual(len(evg_config["buildvariants"][0]["tasks"]), 7)
             self.assertEqual(len(evg_config["buildvariants"][0]["display_tasks"]), 1)
             display_task = evg_config["buildvariants"][0]["display_tasks"][0]
-            self.assertEqual(display_task["name"], "burn_in_tests")
+            self.assertEqual(display_task["name"], burn_in.BURN_IN_TESTS_TASK)
             execution_tasks = display_task["execution_tasks"]
             self.assertEqual(len(execution_tasks), 8)
-            self.assertEqual(execution_tasks[0], "burn_in_tests_gen")
+            self.assertEqual(execution_tasks[0], burn_in.BURN_IN_TESTS_GEN_TASK)
             self.assertEqual(execution_tasks[1], "burn_in:myvariant_task1_0")
             self.assertEqual(execution_tasks[2], "burn_in:myvariant_task1_1")
             self.assertEqual(execution_tasks[3], "burn_in:myvariant_task2_0")
@@ -415,12 +571,22 @@ class TestCreateGenerateTasksFile(unittest.TestCase):
                     self.assertEqual(task["distros"][0], options.distro)
 
     def test_create_generate_tasks_file_no_tasks(self):
+        variant = "myvariant"
         options = self._options_mock()
-        options.buildvariant = "myvariant"
+        options.buildvariant = variant
         tests_by_task = {}
         with patch(BURN_IN + "._write_json_file") as mock_write_json:
             burn_in.create_generate_tasks_file(options, tests_by_task)
-            self.assertEqual(mock_write_json.call.call_count, 0)
+            evg_config = mock_write_json.call_args_list[0][0][0]
+            self.assertEqual(len(evg_config), 1)
+            self.assertEqual(len(evg_config["buildvariants"]), 1)
+            self.assertEqual(evg_config["buildvariants"][0]["name"], variant)
+            display_tasks = evg_config["buildvariants"][0]["display_tasks"]
+            self.assertEqual(len(display_tasks), 1)
+            self.assertEqual(display_tasks[0]["name"], burn_in.BURN_IN_TESTS_TASK)
+            execution_tasks = display_tasks[0]["execution_tasks"]
+            self.assertEqual(len(execution_tasks), 1)
+            self.assertEqual(execution_tasks[0], burn_in.BURN_IN_TESTS_GEN_TASK)
 
 
 class UpdateReportDataTests(unittest.TestCase):
@@ -657,122 +823,14 @@ class CreateExecutorList(unittest.TestCase):
         self.assertEqual(mock_suite_class.call_count, 0)
 
 
-def tasks_mock(  #pylint: disable=too-many-arguments
-        tasks, generate_resmoke_tasks_command=None, get_vars_task_name=None, run_tests_command=None,
-        multiversion_path=None, multiversion_setup_command=None):
-    task_list = Mock()
-    task_list.tasks = []
-    for idx, task in enumerate(tasks):
-        task_list.tasks.append(Mock())
-        task_list.tasks[idx].is_generate_resmoke_task = generate_resmoke_tasks_command is not None
-        task_list.tasks[idx].is_run_tests_task = run_tests_command is not None
-        task_list.tasks[idx].is_multiversion_task = multiversion_path is not None
-        task_list.tasks[idx].generate_resmoke_tasks_command = generate_resmoke_tasks_command
-        task_list.tasks[idx].run_tests_command = run_tests_command
-        task_list.tasks[idx].get_vars_task_name = get_vars_task_name
-        task_list.tasks[idx].name = task["name"]
-        resmoke_args = task.get("combined_resmoke_args")
-        task_list.tasks[idx].combined_resmoke_args = resmoke_args
-        task_list.tasks[idx].resmoke_suite = evg.ResmokeArgs.get_arg(
-            resmoke_args, "suites") if resmoke_args else None
-        task_list.tasks[idx].multiversion_path = multiversion_path
-        task_list.tasks[idx].multiversion_setup_command = multiversion_setup_command
-
-    return task_list
-
-
 class CreateTaskList(unittest.TestCase):
-
-    GENERATE_RESMOKE_TASKS_NAME = "this_is_a_gen_task"
-    GET_GENERATE_RESMOKE_TASKS_NAME = lambda _: CreateTaskList.GENERATE_RESMOKE_TASKS_NAME
-    GENERATE_RESMOKE_TASKS_COMMAND = {
-        "func": "generate resmoke tasks", "vars": {
-            "task": GENERATE_RESMOKE_TASKS_NAME, "suite": "suite3",
-            "resmoke_args": "--shellWriteMode=commands"
-        }
-    }
-
-    GENERATE_RESMOKE_TASKS_COMMAND2 = {
-        "func": "generate resmoke tasks",
-        "vars": {"task": GENERATE_RESMOKE_TASKS_NAME, "resmoke_args": "--shellWriteMode=commands"}
-    }
-
-    MULTIVERSION_PATH = "/data/multiversion"
-    GENERATE_RESMOKE_TASKS_MULTIVERSION_COMMAND = {
-        "func": "generate resmoke tasks", "vars": {
-            "task": GENERATE_RESMOKE_TASKS_NAME, "resmoke_args": "--shellWriteMode=commands",
-            "use_multiversion": MULTIVERSION_PATH
-        }
-    }
-
-    MULTIVERSION_COMMAND = {"func": "do multiversion setup"}
-    RUN_TESTS_MULTIVERSION_COMMAND = {
-        "func": "run tests", "vars": {
-            "task": GENERATE_RESMOKE_TASKS_NAME, "resmoke_args": "--shellWriteMode=commands",
-            "task_path_suffix": MULTIVERSION_PATH
-        }
-    }
-
-    VARIANTS = {
-        "variantall":
-            tasks_mock([{"name": "task1", "combined_resmoke_args": "--suites=suite1 var1arg1"},
-                        {"name": "task2", "combined_resmoke_args": "--suites=suite1 var1arg2"},
-                        {"name": "task3", "combined_resmoke_args": "--suites=suite1 var1arg3"}]),
-        "variant1":
-            tasks_mock([{"name": "task1", "combined_resmoke_args": "--suites=suite1 var1arg1"},
-                        {"name": "task2"}]),
-        "variant2":
-            tasks_mock([{"name": "task2", "combined_resmoke_args": "var2arg1"},
-                        {"name": "task3", "combined_resmoke_args": "--suites=suite3 var2arg3"}]),
-        "variant3":
-            tasks_mock([{"name": "task2", "combined_resmoke_args": "var3arg1"}]),
-        "variant4":
-            tasks_mock([]),
-        "variant_multiversion":
-            tasks_mock(
-                [{"name": "multiversion_task", "combined_resmoke_args": "--suites=suite3 vararg"}],
-                run_tests_command=RUN_TESTS_MULTIVERSION_COMMAND,
-                multiversion_setup_command=RUN_TESTS_MULTIVERSION_COMMAND,
-                multiversion_path=MULTIVERSION_PATH),
-        "variant_generate_tasks":
-            tasks_mock([{
-                "name": GENERATE_RESMOKE_TASKS_NAME,
-                "combined_resmoke_args": "--suites=suite3 vararg"
-            }], generate_resmoke_tasks_command=GENERATE_RESMOKE_TASKS_COMMAND,
-                       get_vars_task_name=GET_GENERATE_RESMOKE_TASKS_NAME),
-        "variant_generate_tasks_diff_names":
-            tasks_mock([{
-                "name": "gen_task_name_different_from_vars_task_name",
-                "combined_resmoke_args": "--suites=suite3 vararg"
-            }], generate_resmoke_tasks_command=GENERATE_RESMOKE_TASKS_COMMAND,
-                       get_vars_task_name=GET_GENERATE_RESMOKE_TASKS_NAME),
-        "variant_generate_tasks_no_suite":
-            tasks_mock([{
-                "name": GENERATE_RESMOKE_TASKS_NAME,
-                "combined_resmoke_args": "--suites=suite3 vararg"
-            }], generate_resmoke_tasks_command=GENERATE_RESMOKE_TASKS_COMMAND2,
-                       get_vars_task_name=GET_GENERATE_RESMOKE_TASKS_NAME),
-        "variant_generate_tasks_multiversion":
-            tasks_mock([{
-                "name": GENERATE_RESMOKE_TASKS_NAME,
-                "combined_resmoke_args": "--suites=suite3 vararg"
-            }], generate_resmoke_tasks_command=GENERATE_RESMOKE_TASKS_MULTIVERSION_COMMAND,
-                       get_vars_task_name=GET_GENERATE_RESMOKE_TASKS_NAME,
-                       multiversion_path=MULTIVERSION_PATH),
-    }
-
-    evergreen_conf = Mock()
-    evergreen_conf.get_variant = VARIANTS.get
-    evergreen_conf.variant_names = VARIANTS.keys()
-
     def test_create_task_list(self):
         variant = "variantall"
         suites = [SUITE1, SUITE2, SUITE3]
         exclude_suites = []
         suite_list = _create_executor_list(suites, exclude_suites)
-        task_list = burn_in.create_task_list(self.evergreen_conf, variant, suite_list,
-                                             exclude_suites)
-        self.assertEqual(len(task_list), len(self.VARIANTS["variantall"].tasks))
+        task_list = burn_in.create_task_list(EVERGREEN_CONF, variant, suite_list, exclude_suites)
+        self.assertEqual(len(task_list), len(VARIANTS["variantall"].tasks))
         self.assertIn("task1", task_list)
         self.assertEqual(task_list["task1"]["resmoke_args"], "--suites=suite1 var1arg1")
         self.assertEqual(task_list["task1"]["tests"], SUITE1.tests)
@@ -791,66 +849,58 @@ class CreateTaskList(unittest.TestCase):
         suites = [SUITE1, SUITE2, SUITE3]
         exclude_suites = []
         suite_list = _create_executor_list(suites, exclude_suites)
-        task_list = burn_in.create_task_list(self.evergreen_conf, variant, suite_list,
-                                             exclude_suites)
-        self.assertEqual(len(task_list), len(self.VARIANTS["variant_multiversion"].tasks))
-        self.assertEqual(task_list["multiversion_task"]["use_multiversion"], self.MULTIVERSION_PATH)
+        task_list = burn_in.create_task_list(EVERGREEN_CONF, variant, suite_list, exclude_suites)
+        self.assertEqual(len(task_list), len(VARIANTS["variant_multiversion"].tasks))
+        self.assertEqual(task_list["multiversion_task"]["use_multiversion"], MULTIVERSION_PATH)
 
     def test_create_task_list_gen_tasks(self):
         variant = "variant_generate_tasks"
         suites = [SUITE3]
         exclude_suites = []
         suite_list = _create_executor_list(suites, exclude_suites)
-        task_list = burn_in.create_task_list(self.evergreen_conf, variant, suite_list,
-                                             exclude_suites)
-        self.assertEqual(len(task_list), len(self.VARIANTS["variant_generate_tasks"].tasks))
-        self.assertIn(self.GENERATE_RESMOKE_TASKS_NAME, task_list)
-        self.assertEqual(task_list[self.GENERATE_RESMOKE_TASKS_NAME]["tests"], SUITE3.tests)
-        self.assertIsNone(task_list[self.GENERATE_RESMOKE_TASKS_NAME]["use_multiversion"])
+        task_list = burn_in.create_task_list(EVERGREEN_CONF, variant, suite_list, exclude_suites)
+        self.assertEqual(len(task_list), len(VARIANTS["variant_generate_tasks"].tasks))
+        self.assertIn(GENERATE_RESMOKE_TASKS_NAME, task_list)
+        self.assertEqual(task_list[GENERATE_RESMOKE_TASKS_NAME]["tests"], SUITE3.tests)
+        self.assertIsNone(task_list[GENERATE_RESMOKE_TASKS_NAME]["use_multiversion"])
 
     def test_create_task_list_gen_tasks_diff_task_names(self):
         variant = "variant_generate_tasks_diff_names"
         suites = [SUITE3]
         exclude_suites = []
         suite_list = _create_executor_list(suites, exclude_suites)
-        task_list = burn_in.create_task_list(self.evergreen_conf, variant, suite_list,
-                                             exclude_suites)
-        self.assertEqual(len(task_list), len(self.VARIANTS["variant_generate_tasks"].tasks))
-        self.assertIn(self.GENERATE_RESMOKE_TASKS_NAME, task_list)
-        self.assertEqual(task_list[self.GENERATE_RESMOKE_TASKS_NAME]["tests"], SUITE3.tests)
-        self.assertIsNone(task_list[self.GENERATE_RESMOKE_TASKS_NAME]["use_multiversion"])
+        task_list = burn_in.create_task_list(EVERGREEN_CONF, variant, suite_list, exclude_suites)
+        self.assertEqual(len(task_list), len(VARIANTS["variant_generate_tasks"].tasks))
+        self.assertIn(GENERATE_RESMOKE_TASKS_NAME, task_list)
+        self.assertEqual(task_list[GENERATE_RESMOKE_TASKS_NAME]["tests"], SUITE3.tests)
+        self.assertIsNone(task_list[GENERATE_RESMOKE_TASKS_NAME]["use_multiversion"])
 
     def test_create_task_list_gen_tasks_multiversion(self):
         variant = "variant_generate_tasks_multiversion"
         suites = [SUITE3]
         exclude_suites = []
         suite_list = _create_executor_list(suites, exclude_suites)
-        task_list = burn_in.create_task_list(self.evergreen_conf, variant, suite_list,
-                                             exclude_suites)
-        self.assertEqual(
-            len(task_list), len(self.VARIANTS["variant_generate_tasks_multiversion"].tasks))
-        self.assertEqual(task_list[self.GENERATE_RESMOKE_TASKS_NAME]["use_multiversion"],
-                         self.MULTIVERSION_PATH)
+        task_list = burn_in.create_task_list(EVERGREEN_CONF, variant, suite_list, exclude_suites)
+        self.assertEqual(len(task_list), len(VARIANTS["variant_generate_tasks_multiversion"].tasks))
+        self.assertEqual(task_list[GENERATE_RESMOKE_TASKS_NAME]["use_multiversion"],
+                         MULTIVERSION_PATH)
 
     def test_create_task_list_gen_tasks_no_suite(self):
         variant = "variant_generate_tasks_no_suite"
         suites = [SUITE3]
         exclude_suites = []
         suite_list = _create_executor_list(suites, exclude_suites)
-        task_list = burn_in.create_task_list(self.evergreen_conf, variant, suite_list,
-                                             exclude_suites)
-        self.assertEqual(
-            len(task_list), len(self.VARIANTS["variant_generate_tasks_no_suite"].tasks))
-        self.assertIn(self.GENERATE_RESMOKE_TASKS_NAME, task_list)
-        self.assertEqual(task_list[self.GENERATE_RESMOKE_TASKS_NAME]["tests"], SUITE3.tests)
+        task_list = burn_in.create_task_list(EVERGREEN_CONF, variant, suite_list, exclude_suites)
+        self.assertEqual(len(task_list), len(VARIANTS["variant_generate_tasks_no_suite"].tasks))
+        self.assertIn(GENERATE_RESMOKE_TASKS_NAME, task_list)
+        self.assertEqual(task_list[GENERATE_RESMOKE_TASKS_NAME]["tests"], SUITE3.tests)
 
     def test_create_task_list_no_excludes(self):
         variant = "variant1"
         suites = [SUITE1, SUITE2]
         exclude_suites = []
         suite_list = _create_executor_list(suites, exclude_suites)
-        task_list = burn_in.create_task_list(self.evergreen_conf, variant, suite_list,
-                                             exclude_suites)
+        task_list = burn_in.create_task_list(EVERGREEN_CONF, variant, suite_list, exclude_suites)
         self.assertEqual(len(task_list), 1)
         self.assertIn("task1", task_list)
         self.assertEqual(task_list["task1"]["resmoke_args"], "--suites=suite1 var1arg1")
@@ -863,8 +913,7 @@ class CreateTaskList(unittest.TestCase):
         suites = [SUITE1, SUITE2, SUITE3]
         suite_list = _create_executor_list(suites, [])
         exclude_suites = ["suite2"]
-        task_list = burn_in.create_task_list(self.evergreen_conf, variant, suite_list,
-                                             exclude_suites)
+        task_list = burn_in.create_task_list(EVERGREEN_CONF, variant, suite_list, exclude_suites)
         self.assertEqual(len(task_list), 1)
         self.assertIn("task3", task_list)
         self.assertEqual(task_list["task3"]["resmoke_args"], "--suites=suite3 var2arg3")
@@ -876,8 +925,7 @@ class CreateTaskList(unittest.TestCase):
         variant = "variant2"
         suite_list = {}
         exclude_suites = ["suite2"]
-        task_list = burn_in.create_task_list(self.evergreen_conf, variant, suite_list,
-                                             exclude_suites)
+        task_list = burn_in.create_task_list(EVERGREEN_CONF, variant, suite_list, exclude_suites)
         self.assertEqual(len(task_list), 0)
         self.assertEqual(task_list, {})
 
@@ -893,7 +941,7 @@ class CreateTaskList(unittest.TestCase):
         suite_list = _create_executor_list(suites, [])
         with patch("sys.exit", _raise_bad_variant):
             with self.assertRaises(BadVariant):
-                burn_in.create_task_list(self.evergreen_conf, variant, suite_list, [])
+                burn_in.create_task_list(EVERGREEN_CONF, variant, suite_list, [])
 
 
 class FindChangedTests(unittest.TestCase):