diff options
author | Jonathan Abrahams <jonathan@mongodb.com> | 2019-03-18 13:14:22 -0400 |
---|---|---|
committer | Jonathan Abrahams <jonathan@mongodb.com> | 2019-03-18 13:14:22 -0400 |
commit | bd487118cc1f8d233881950f81006c9da89377b9 (patch) | |
tree | 46ac3f0b14508ffc1e563904b1e2b97f76d02b85 | |
parent | c90d961fbe008f81455ceaf53a4c8284eb76bbc5 (diff) | |
download | mongo-bd487118cc1f8d233881950f81006c9da89377b9.tar.gz |
Revert "SERVER-39311 Enable burn_in repetition count: at least 2 executions, with a maximum of 1000 excutions or 10 minutes of execution time"
This reverts commit df4243e71b7c69c4da59c90e657f0749d6c9c979.
-rw-r--r-- | buildscripts/burn_in_tests.py | 31 | ||||
-rw-r--r-- | buildscripts/tests/ciconfig/test_evergreen.py | 84 | ||||
-rw-r--r-- | buildscripts/tests/test_burn_in_tests.py | 436 | ||||
-rw-r--r-- | etc/evergreen.yml | 89 |
4 files changed, 262 insertions, 378 deletions
diff --git a/buildscripts/burn_in_tests.py b/buildscripts/burn_in_tests.py index 906aa618e7a..398184d4f96 100644 --- a/buildscripts/burn_in_tests.py +++ b/buildscripts/burn_in_tests.py @@ -65,13 +65,9 @@ def parse_command_line(): help="The base commit to compare to for determining changes.") parser.add_option("--buildVariant", dest="buildvariant", default=None, - help=("The buildvariant to select the tasks. Required when" + help=("The buildvariant the tasks will execute on. Required when" " generating the JSON file with test executor information")) - parser.add_option("--runBuildVariant", dest="run_buildvariant", default=None, - help=("The buildvariant the tasks will execute on. If not specied then tasks" - " will execute on the the buildvariant specied in --buildVariant.")) - parser.add_option("--distro", dest="distro", default=None, help=("The distro the tasks will execute on. Can only be specified" " with --generateTasksFile.")) @@ -123,14 +119,6 @@ def parse_command_line(): return options, args -def check_variant(buildvariant, parser): - """Check if the buildvariant is found in the evergreen file.""" - evg_conf = evergreen.parse_evergreen_file(EVERGREEN_FILE) - if not evg_conf.get_variant(buildvariant): - parser.error("Buildvariant '{}' not found in {}, select from:\n\t{}".format( - buildvariant, EVERGREEN_FILE, "\n\t".join(sorted(evg_conf.variant_names)))) - - def validate_options(parser, options): """Validate command line options.""" @@ -151,10 +139,10 @@ def validate_options(parser, options): parser.error("Must specify --buildVariant to find changed tests") if options.buildvariant: - check_variant(options.buildvariant, parser) - - if options.run_buildvariant: - check_variant(options.run_buildvariant, parser) + evg_conf = evergreen.parse_evergreen_file(EVERGREEN_FILE) + if not evg_conf.get_variant(options.buildvariant): + parser.error("Buildvariant '{}' not found in {}, select from:\n\t{}".format( + options.buildvariant, EVERGREEN_FILE, "\n\t".join(sorted(evg_conf.variant_names)))) def find_last_activated_task(revisions, variant, branch_name): @@ -440,13 +428,6 @@ def _sub_task_name(variant, task, task_num): return "burn_in:{}_{}_{}".format(variant, task, task_num) -def _get_run_buildvariant(options): - """Return the build variant to execute the tasks on.""" - if options.run_buildvariant: - return options.run_buildvariant - return options.buildvariant - - def create_generate_tasks_file(options, tests_by_task): """Create the Evergreen generate.tasks file.""" @@ -481,7 +462,7 @@ def create_generate_tasks_file(options, tests_by_task): evg_sub_task.commands(commands) display_task = DisplayTaskDefinition("burn_in_tests").execution_tasks(task_names) - evg_config.variant(_get_run_buildvariant(options)).tasks(task_specs).display_task(display_task) + evg_config.variant(options.buildvariant).tasks(task_specs).display_task(display_task) _write_json_file(evg_config.to_map(), options.generate_tasks_file) diff --git a/buildscripts/tests/ciconfig/test_evergreen.py b/buildscripts/tests/ciconfig/test_evergreen.py index fd57a449f79..d7b345ca144 100644 --- a/buildscripts/tests/ciconfig/test_evergreen.py +++ b/buildscripts/tests/ciconfig/test_evergreen.py @@ -71,7 +71,7 @@ class TestEvergreenProjectConfig(unittest.TestCase): self.assertIn("amazon", self.conf.distro_names) -class TestTask(unittest.TestCase): # pylint: disable=too-many-public-methods +class TestTask(unittest.TestCase): #pylint: disable=too-many-public-methods """Unit tests for the Task class.""" def test_from_dict(self): @@ -147,6 +147,19 @@ class TestTask(unittest.TestCase): # pylint: disable=too-many-public-methods self.assertIsNone(task.multiversion_setup_command) self.assertIsNone(task.multiversion_path) + def test_get_vars_suite_name_run_tests(self): + suite_name = "core" + task_commands = [{ + "func": "run tests", + "vars": {"resmoke_args": "--suites={} --shellWriteMode=commands".format(suite_name)} + }] + + task_dict = {"name": "jsCore", "commands": task_commands} + task = _evergreen.Task(task_dict) + + with self.assertRaises(ValueError): + task.get_vars_suite_name(task_commands[0]["vars"]) + def test_resmoke_args_gen(self): task_name = "core" task_commands = [{ @@ -239,46 +252,6 @@ class TestTask(unittest.TestCase): # pylint: disable=too-many-public-methods self.assertDictEqual(task_commands[0], task.generate_resmoke_tasks_command) self.assertEqual(task_name, task.get_vars_task_name(task_commands[0]["vars"])) - def test_gen_resmoke_multiversion(self): - multiversion_path = "/data/multiversion" - task_name = "core" - task_commands = [{ - "func": "generate resmoke tasks", "vars": { - "task": task_name, "resmoke_args": "--shellWriteMode=commands", - "use_multiversion": multiversion_path - } - }] - task_dict = {"name": "jsCore", "commands": task_commands} - task = _evergreen.Task(task_dict) - - self.assertTrue(task.is_multiversion_task) - self.assertEqual(multiversion_path, task.multiversion_path) - - def test_gen_resmoke_no_multiversion(self): - task_name = "core" - task_commands = [{ - "func": "generate resmoke tasks", - "vars": {"task": task_name, "resmoke_args": "--shellWriteMode=commands"} - }] - task_dict = {"name": "jsCore", "commands": task_commands} - task = _evergreen.Task(task_dict) - - self.assertFalse(task.is_multiversion_task) - self.assertIsNone(task.multiversion_path) - - def test_get_vars_suite_name_run_tests(self): - suite_name = "core" - task_commands = [{ - "func": "run tests", - "vars": {"resmoke_args": "--suites={} --shellWriteMode=commands".format(suite_name)} - }] - - task_dict = {"name": "jsCore", "commands": task_commands} - task = _evergreen.Task(task_dict) - - with self.assertRaises(ValueError): - task.get_vars_suite_name(task_commands[0]["vars"]) - def test_get_vars_suite_name_generate_resmoke_tasks(self): task_name = "jsCore" suite_name = "core" @@ -301,7 +274,7 @@ class TestTask(unittest.TestCase): # pylint: disable=too-many-public-methods with self.assertRaises(TypeError): task.get_vars_suite_name(None) - def test_get_vars_suite_name_gen_resmoke_notask_nosuite(self): + def test_get_vars_suite_name_gen_resmoke_notasksuite(self): task_name = "jsCore" task_commands = [{ "func": "generate resmoke tasks", "vars": {"resmoke_args": "--shellWriteMode=commands"} @@ -312,6 +285,33 @@ class TestTask(unittest.TestCase): # pylint: disable=too-many-public-methods with self.assertRaises(ValueError): task.get_vars_suite_name(task_commands[0]["vars"]) + def test_gen_resmoke_multiversion(self): + multiversion_path = "/data/multiversion" + task_name = "core" + task_commands = [{ + "func": "generate resmoke tasks", "vars": { + "task": task_name, "resmoke_args": "--shellWriteMode=commands", + "use_multiversion": multiversion_path + } + }] + task_dict = {"name": "jsCore", "commands": task_commands} + task = _evergreen.Task(task_dict) + + self.assertTrue(task.is_multiversion_task) + self.assertEqual(multiversion_path, task.multiversion_path) + + def test_gen_resmoke_no_multiversion(self): + task_name = "core" + task_commands = [{ + "func": "generate resmoke tasks", + "vars": {"task": task_name, "resmoke_args": "--shellWriteMode=commands"} + }] + task_dict = {"name": "jsCore", "commands": task_commands} + task = _evergreen.Task(task_dict) + + self.assertFalse(task.is_multiversion_task) + self.assertIsNone(task.multiversion_path) + def test_get_vars_task_name(self): task_name = "task_name" task_commands = [{ diff --git a/buildscripts/tests/test_burn_in_tests.py b/buildscripts/tests/test_burn_in_tests.py index 5df69720804..0ab6bec59c3 100644 --- a/buildscripts/tests/test_burn_in_tests.py +++ b/buildscripts/tests/test_burn_in_tests.py @@ -20,118 +20,21 @@ EVG_CLIENT = "buildscripts.client.evergreen" GIT = "buildscripts.git" RESMOKELIB = "buildscripts.resmokelib" -GENERATE_RESMOKE_TASKS_NAME = "this_is_a_gen_task" -GET_GENERATE_RESMOKE_TASKS_NAME = lambda _: GENERATE_RESMOKE_TASKS_NAME -GENERATE_RESMOKE_TASKS_COMMAND = { - "func": "generate resmoke tasks", "vars": { - "task": GENERATE_RESMOKE_TASKS_NAME, "suite": "suite3", - "resmoke_args": "--shellWriteMode=commands" - } -} - -GENERATE_RESMOKE_TASKS_COMMAND2 = { - "func": "generate resmoke tasks", - "vars": {"task": GENERATE_RESMOKE_TASKS_NAME, "resmoke_args": "--shellWriteMode=commands"} -} - -MULTIVERSION_PATH = "/data/multiversion" -GENERATE_RESMOKE_TASKS_MULTIVERSION_COMMAND = { - "func": "generate resmoke tasks", "vars": { - "task": GENERATE_RESMOKE_TASKS_NAME, "resmoke_args": "--shellWriteMode=commands", - "use_multiversion": MULTIVERSION_PATH - } -} - -MULTIVERSION_COMMAND = {"func": "do multiversion setup"} -RUN_TESTS_MULTIVERSION_COMMAND = { - "func": "run tests", "vars": { - "task": GENERATE_RESMOKE_TASKS_NAME, "resmoke_args": "--shellWriteMode=commands", - "task_path_suffix": MULTIVERSION_PATH - } -} - - -def tasks_mock( #pylint: disable=too-many-arguments - tasks, generate_resmoke_tasks_command=None, get_vars_task_name=None, run_tests_command=None, - multiversion_path=None, multiversion_setup_command=None): - task_list = Mock() - task_list.tasks = [] - for idx, task in enumerate(tasks): - task_list.tasks.append(Mock()) - task_list.tasks[idx].is_generate_resmoke_task = generate_resmoke_tasks_command is not None - task_list.tasks[idx].is_run_tests_task = run_tests_command is not None - task_list.tasks[idx].is_multiversion_task = multiversion_path is not None - task_list.tasks[idx].generate_resmoke_tasks_command = generate_resmoke_tasks_command - task_list.tasks[idx].run_tests_command = run_tests_command - task_list.tasks[idx].get_vars_task_name = get_vars_task_name - task_list.tasks[idx].name = task["name"] - resmoke_args = task.get("combined_resmoke_args") - task_list.tasks[idx].combined_resmoke_args = resmoke_args - task_list.tasks[idx].resmoke_suite = evg.ResmokeArgs.get_arg( - resmoke_args, "suites") if resmoke_args else None - task_list.tasks[idx].multiversion_path = multiversion_path - task_list.tasks[idx].multiversion_setup_command = multiversion_setup_command - - return task_list - - -VARIANTS = { - "variantall": - tasks_mock([{"name": "task1", "combined_resmoke_args": "--suites=suite1 var1arg1"}, - {"name": "task2", "combined_resmoke_args": "--suites=suite1 var1arg2"}, - {"name": "task3", "combined_resmoke_args": "--suites=suite1 var1arg3"}]), - "variant1": - tasks_mock([{"name": "task1", "combined_resmoke_args": "--suites=suite1 var1arg1"}, - {"name": "task2"}]), - "variant2": - tasks_mock([{"name": "task2", "combined_resmoke_args": "var2arg1"}, - {"name": "task3", "combined_resmoke_args": "--suites=suite3 var2arg3"}]), - "variant3": - tasks_mock([{"name": "task2", "combined_resmoke_args": "var3arg1"}]), - "variant4": - tasks_mock([]), - "variant_multiversion": - tasks_mock( - [{"name": "multiversion_task", "combined_resmoke_args": "--suites=suite3 vararg"}], - run_tests_command=RUN_TESTS_MULTIVERSION_COMMAND, - multiversion_setup_command=RUN_TESTS_MULTIVERSION_COMMAND, - multiversion_path=MULTIVERSION_PATH), - "variant_generate_tasks": - tasks_mock([{ - "name": GENERATE_RESMOKE_TASKS_NAME, "combined_resmoke_args": "--suites=suite3 vararg" - }], generate_resmoke_tasks_command=GENERATE_RESMOKE_TASKS_COMMAND, - get_vars_task_name=GET_GENERATE_RESMOKE_TASKS_NAME), - "variant_generate_tasks_no_suite": - tasks_mock([{ - "name": GENERATE_RESMOKE_TASKS_NAME, "combined_resmoke_args": "--suites=suite3 vararg" - }], generate_resmoke_tasks_command=GENERATE_RESMOKE_TASKS_COMMAND2, - get_vars_task_name=GET_GENERATE_RESMOKE_TASKS_NAME), - "variant_generate_tasks_diff_names": - tasks_mock([{ - "name": "gen_task_name_different_from_vars_task_name", - "combined_resmoke_args": "--suites=suite3 vararg" - }], generate_resmoke_tasks_command=GENERATE_RESMOKE_TASKS_COMMAND, - get_vars_task_name=GET_GENERATE_RESMOKE_TASKS_NAME), - "variant_generate_tasks_multiversion": - tasks_mock([{ - "name": GENERATE_RESMOKE_TASKS_NAME, "combined_resmoke_args": "--suites=suite3 vararg" - }], generate_resmoke_tasks_command=GENERATE_RESMOKE_TASKS_MULTIVERSION_COMMAND, - get_vars_task_name=GET_GENERATE_RESMOKE_TASKS_NAME, - multiversion_path=MULTIVERSION_PATH), -} - -EVERGREEN_CONF = Mock() -EVERGREEN_CONF.get_variant = VARIANTS.get -EVERGREEN_CONF.variant_names = VARIANTS.keys() +class ValidateOptions(unittest.TestCase): + class ParserError(Exception): + pass -def _mock_parser(): - parser = Mock() - parser.error = Mock() - return parser + @staticmethod + def _raise_parser_error(err): + raise ValidateOptions.ParserError(err) + @staticmethod + def _mock_parser(): + parser = Mock() + parser.error = ValidateOptions._raise_parser_error + return parser -class TestValidateOptions(unittest.TestCase): @staticmethod def _mock_options(): options = Mock() @@ -140,132 +43,109 @@ class TestValidateOptions(unittest.TestCase): options.repeat_tests_min = None options.repeat_tests_secs = None options.buildvariant = None - options.run_buildvariant = None options.test_list_file = None return options def test_validate_options_listfile_buildvariant(self): - mock_parser = _mock_parser() + parser = self._mock_parser() options = self._mock_options() options.test_list_file = "list_file.json" options.buildvariant = "variant1" - with patch(EVG_CI + ".parse_evergreen_file", return_value=EVERGREEN_CONF): - burn_in.validate_options(mock_parser, options) - mock_parser.error.assert_not_called() + with patch(EVG_CI + ".parse_evergreen_file", return_value=CreateTaskList.evergreen_conf): + burn_in.validate_options(parser, options) def test_validate_options_nolistfile_buildvariant(self): - mock_parser = _mock_parser() + parser = self._mock_parser() options = self._mock_options() options.buildvariant = "variant1" - with patch(EVG_CI + ".parse_evergreen_file", return_value=EVERGREEN_CONF): - burn_in.validate_options(mock_parser, options) - mock_parser.error.assert_not_called() + with patch(EVG_CI + ".parse_evergreen_file", return_value=CreateTaskList.evergreen_conf): + burn_in.validate_options(parser, options) def test_validate_options_listfile_nobuildvariant(self): - mock_parser = _mock_parser() + parser = self._mock_parser() options = self._mock_options() options.test_list_file = "list_file.json" - with patch(EVG_CI + ".parse_evergreen_file", return_value=EVERGREEN_CONF): - burn_in.validate_options(mock_parser, options) - mock_parser.error.assert_not_called() + with patch(EVG_CI + ".parse_evergreen_file", return_value=CreateTaskList.evergreen_conf): + burn_in.validate_options(parser, options) def test_validate_options_no_listfile_no_buildvariant(self): - mock_parser = _mock_parser() + parser = self._mock_parser() options = self._mock_options() - with patch(EVG_CI + ".parse_evergreen_file", return_value=EVERGREEN_CONF): - burn_in.validate_options(mock_parser, options) - mock_parser.error.assert_called() + with patch(EVG_CI + ".parse_evergreen_file", return_value=CreateTaskList.evergreen_conf): + with self.assertRaises(self.ParserError): + burn_in.validate_options(parser, options) def test_validate_options_buildvariant(self): - mock_parser = _mock_parser() + parser = self._mock_parser() options = self._mock_options() options.buildvariant = "variant1" - with patch(EVG_CI + ".parse_evergreen_file", return_value=EVERGREEN_CONF): - burn_in.validate_options(mock_parser, options) - mock_parser.error.assert_not_called() - - def test_validate_options_run_buildvariant(self): - mock_parser = _mock_parser() - options = self._mock_options() - options.buildvariant = "variant1" - options.run_buildvariant = "variant1" - with patch(EVG_CI + ".parse_evergreen_file", return_value=EVERGREEN_CONF): - burn_in.validate_options(mock_parser, options) - mock_parser.error.assert_not_called() + with patch(EVG_CI + ".parse_evergreen_file", return_value=CreateTaskList.evergreen_conf): + burn_in.validate_options(parser, options) def test_validate_options_bad_buildvariant(self): - mock_parser = _mock_parser() + parser = self._mock_parser() options = self._mock_options() options.buildvariant = "badvariant1" - with patch(EVG_CI + ".parse_evergreen_file", return_value=EVERGREEN_CONF): - burn_in.validate_options(mock_parser, options) - mock_parser.error.assert_called() - - def test_validate_options_bad_run_buildvariant(self): - mock_parser = _mock_parser() - options = self._mock_options() - options.run_buildvariant = "badvariant1" - with patch(EVG_CI + ".parse_evergreen_file", return_value=EVERGREEN_CONF): - burn_in.validate_options(mock_parser, options) - mock_parser.error.assert_called() + with patch(EVG_CI + ".parse_evergreen_file", return_value=CreateTaskList.evergreen_conf): + with self.assertRaises(self.ParserError): + burn_in.validate_options(parser, options) def test_validate_options_tests_max_no_tests_secs(self): - mock_parser = _mock_parser() + parser = self._mock_parser() options = self._mock_options() options.repeat_tests_max = 3 - with patch(EVG_CI + ".parse_evergreen_file", return_value=EVERGREEN_CONF): - burn_in.validate_options(mock_parser, options) - mock_parser.error.assert_called() + with patch(EVG_CI + ".parse_evergreen_file", return_value=CreateTaskList.evergreen_conf): + with self.assertRaises(self.ParserError): + burn_in.validate_options(parser, options) def test_validate_options_tests_min_no_tests_secs(self): - mock_parser = _mock_parser() + parser = self._mock_parser() options = self._mock_options() options.repeat_tests_min = 3 - with patch(EVG_CI + ".parse_evergreen_file", return_value=EVERGREEN_CONF): - burn_in.validate_options(mock_parser, options) - mock_parser.error.assert_called() + with patch(EVG_CI + ".parse_evergreen_file", return_value=CreateTaskList.evergreen_conf): + with self.assertRaises(self.ParserError): + burn_in.validate_options(parser, options) def test_validate_options_tests_min_gt_tests_max(self): - mock_parser = _mock_parser() + parser = self._mock_parser() options = self._mock_options() options.repeat_tests_min = 3 options.repeat_tests_max = 2 - with patch(EVG_CI + ".parse_evergreen_file", return_value=EVERGREEN_CONF): - burn_in.validate_options(mock_parser, options) - mock_parser.error.assert_called() + with patch(EVG_CI + ".parse_evergreen_file", return_value=CreateTaskList.evergreen_conf): + with self.assertRaises(self.ParserError): + burn_in.validate_options(parser, options) def test_validate_options_tests_secs(self): - mock_parser = _mock_parser() + parser = self._mock_parser() options = self._mock_options() options.buildvariant = "variant1" options.repeat_tests_min = 2 options.repeat_tests_max = 1000 options.repeat_tests_secs = 3 - with patch(EVG_CI + ".parse_evergreen_file", return_value=EVERGREEN_CONF): - burn_in.validate_options(mock_parser, options) - mock_parser.error.assert_not_called() + with patch(EVG_CI + ".parse_evergreen_file", return_value=CreateTaskList.evergreen_conf): + burn_in.validate_options(parser, options) def test_validate_options_tests_secs_and_tests_num(self): - mock_parser = _mock_parser() + parser = self._mock_parser() options = self._mock_options() options.buildvariant = "variant1" options.repeat_tests_num = 1 options.repeat_tests_min = 1 options.repeat_tests_max = 3 options.repeat_tests_secs = 3 - with patch(EVG_CI + ".parse_evergreen_file", return_value=EVERGREEN_CONF): - burn_in.validate_options(mock_parser, options) - mock_parser.error.assert_called() + with patch(EVG_CI + ".parse_evergreen_file", return_value=CreateTaskList.evergreen_conf): + with self.assertRaises(self.ParserError): + burn_in.validate_options(parser, options) def test_validate_options_tests_secs_no_buildvariant(self): - mock_parser = _mock_parser() + parser = self._mock_parser() options = self._mock_options() options.repeat_tests_min = 1 options.repeat_tests_max = 3 options.repeat_tests_secs = 3 - with patch(EVG_CI + ".parse_evergreen_file", return_value=EVERGREEN_CONF): - burn_in.validate_options(mock_parser, options) - mock_parser.error.assert_called() + with patch(EVG_CI + ".parse_evergreen_file", return_value=CreateTaskList.evergreen_conf): + with self.assertRaises(self.ParserError): + burn_in.validate_options(parser, options) class TestGetResmokeRepeatOptions(unittest.TestCase): @@ -339,41 +219,6 @@ class TestGetResmokeRepeatOptions(unittest.TestCase): self.assertEqual(repeat_options, "--repeatSuites={}".format(burn_in.REPEAT_SUITES)) -class TestCheckVariant(unittest.TestCase): - @staticmethod - def test_check_variant(): - mock_parser = _mock_parser() - buildvariant = "variant1" - with patch(EVG_CI + ".parse_evergreen_file", return_value=EVERGREEN_CONF): - burn_in.check_variant(buildvariant, mock_parser) - mock_parser.error.assert_not_called() - - @staticmethod - def test_check_variant_badvariant(): - mock_parser = _mock_parser() - buildvariant = "badvariant" - with patch(EVG_CI + ".parse_evergreen_file", return_value=EVERGREEN_CONF): - burn_in.check_variant(buildvariant, mock_parser) - mock_parser.error.assert_called() - - -class TestGetRunBuildvariant(unittest.TestCase): - def test__get_run_buildvariant_rb(self): - run_buildvariant = "variant1" - buildvariant = "variant2" - options = Mock() - options.run_buildvariant = run_buildvariant - options.buildvariant = buildvariant - self.assertEqual(run_buildvariant, burn_in._get_run_buildvariant(options)) - - def test__get_run_buildvariant_bv(self): - buildvariant = "variant2" - options = Mock() - options.run_buildvariant = None - options.buildvariant = buildvariant - self.assertEqual(buildvariant, burn_in._get_run_buildvariant(options)) - - class TestGetTaskName(unittest.TestCase): def test__get_task_name(self): name = "mytask" @@ -472,7 +317,6 @@ class TestCreateGenerateTasksFile(unittest.TestCase): def _options_mock(): options = Mock() options.buildvariant = None - options.run_buildvariant = None options.distro = None return options @@ -813,14 +657,122 @@ class CreateExecutorList(unittest.TestCase): self.assertEqual(mock_suite_class.call_count, 0) +def tasks_mock( #pylint: disable=too-many-arguments + tasks, generate_resmoke_tasks_command=None, get_vars_task_name=None, run_tests_command=None, + multiversion_path=None, multiversion_setup_command=None): + task_list = Mock() + task_list.tasks = [] + for idx, task in enumerate(tasks): + task_list.tasks.append(Mock()) + task_list.tasks[idx].is_generate_resmoke_task = generate_resmoke_tasks_command is not None + task_list.tasks[idx].is_run_tests_task = run_tests_command is not None + task_list.tasks[idx].is_multiversion_task = multiversion_path is not None + task_list.tasks[idx].generate_resmoke_tasks_command = generate_resmoke_tasks_command + task_list.tasks[idx].run_tests_command = run_tests_command + task_list.tasks[idx].get_vars_task_name = get_vars_task_name + task_list.tasks[idx].name = task["name"] + resmoke_args = task.get("combined_resmoke_args") + task_list.tasks[idx].combined_resmoke_args = resmoke_args + task_list.tasks[idx].resmoke_suite = evg.ResmokeArgs.get_arg( + resmoke_args, "suites") if resmoke_args else None + task_list.tasks[idx].multiversion_path = multiversion_path + task_list.tasks[idx].multiversion_setup_command = multiversion_setup_command + + return task_list + + class CreateTaskList(unittest.TestCase): + + GENERATE_RESMOKE_TASKS_NAME = "this_is_a_gen_task" + GET_GENERATE_RESMOKE_TASKS_NAME = lambda _: CreateTaskList.GENERATE_RESMOKE_TASKS_NAME + GENERATE_RESMOKE_TASKS_COMMAND = { + "func": "generate resmoke tasks", "vars": { + "task": GENERATE_RESMOKE_TASKS_NAME, "suite": "suite3", + "resmoke_args": "--shellWriteMode=commands" + } + } + + GENERATE_RESMOKE_TASKS_COMMAND2 = { + "func": "generate resmoke tasks", + "vars": {"task": GENERATE_RESMOKE_TASKS_NAME, "resmoke_args": "--shellWriteMode=commands"} + } + + MULTIVERSION_PATH = "/data/multiversion" + GENERATE_RESMOKE_TASKS_MULTIVERSION_COMMAND = { + "func": "generate resmoke tasks", "vars": { + "task": GENERATE_RESMOKE_TASKS_NAME, "resmoke_args": "--shellWriteMode=commands", + "use_multiversion": MULTIVERSION_PATH + } + } + + MULTIVERSION_COMMAND = {"func": "do multiversion setup"} + RUN_TESTS_MULTIVERSION_COMMAND = { + "func": "run tests", "vars": { + "task": GENERATE_RESMOKE_TASKS_NAME, "resmoke_args": "--shellWriteMode=commands", + "task_path_suffix": MULTIVERSION_PATH + } + } + + VARIANTS = { + "variantall": + tasks_mock([{"name": "task1", "combined_resmoke_args": "--suites=suite1 var1arg1"}, + {"name": "task2", "combined_resmoke_args": "--suites=suite1 var1arg2"}, + {"name": "task3", "combined_resmoke_args": "--suites=suite1 var1arg3"}]), + "variant1": + tasks_mock([{"name": "task1", "combined_resmoke_args": "--suites=suite1 var1arg1"}, + {"name": "task2"}]), + "variant2": + tasks_mock([{"name": "task2", "combined_resmoke_args": "var2arg1"}, + {"name": "task3", "combined_resmoke_args": "--suites=suite3 var2arg3"}]), + "variant3": + tasks_mock([{"name": "task2", "combined_resmoke_args": "var3arg1"}]), + "variant4": + tasks_mock([]), + "variant_multiversion": + tasks_mock( + [{"name": "multiversion_task", "combined_resmoke_args": "--suites=suite3 vararg"}], + run_tests_command=RUN_TESTS_MULTIVERSION_COMMAND, + multiversion_setup_command=RUN_TESTS_MULTIVERSION_COMMAND, + multiversion_path=MULTIVERSION_PATH), + "variant_generate_tasks": + tasks_mock([{ + "name": GENERATE_RESMOKE_TASKS_NAME, + "combined_resmoke_args": "--suites=suite3 vararg" + }], generate_resmoke_tasks_command=GENERATE_RESMOKE_TASKS_COMMAND, + get_vars_task_name=GET_GENERATE_RESMOKE_TASKS_NAME), + "variant_generate_tasks_diff_names": + tasks_mock([{ + "name": "gen_task_name_different_from_vars_task_name", + "combined_resmoke_args": "--suites=suite3 vararg" + }], generate_resmoke_tasks_command=GENERATE_RESMOKE_TASKS_COMMAND, + get_vars_task_name=GET_GENERATE_RESMOKE_TASKS_NAME), + "variant_generate_tasks_no_suite": + tasks_mock([{ + "name": GENERATE_RESMOKE_TASKS_NAME, + "combined_resmoke_args": "--suites=suite3 vararg" + }], generate_resmoke_tasks_command=GENERATE_RESMOKE_TASKS_COMMAND2, + get_vars_task_name=GET_GENERATE_RESMOKE_TASKS_NAME), + "variant_generate_tasks_multiversion": + tasks_mock([{ + "name": GENERATE_RESMOKE_TASKS_NAME, + "combined_resmoke_args": "--suites=suite3 vararg" + }], generate_resmoke_tasks_command=GENERATE_RESMOKE_TASKS_MULTIVERSION_COMMAND, + get_vars_task_name=GET_GENERATE_RESMOKE_TASKS_NAME, + multiversion_path=MULTIVERSION_PATH), + } + + evergreen_conf = Mock() + evergreen_conf.get_variant = VARIANTS.get + evergreen_conf.variant_names = VARIANTS.keys() + def test_create_task_list(self): variant = "variantall" suites = [SUITE1, SUITE2, SUITE3] exclude_suites = [] suite_list = _create_executor_list(suites, exclude_suites) - task_list = burn_in.create_task_list(EVERGREEN_CONF, variant, suite_list, exclude_suites) - self.assertEqual(len(task_list), len(VARIANTS["variantall"].tasks)) + task_list = burn_in.create_task_list(self.evergreen_conf, variant, suite_list, + exclude_suites) + self.assertEqual(len(task_list), len(self.VARIANTS["variantall"].tasks)) self.assertIn("task1", task_list) self.assertEqual(task_list["task1"]["resmoke_args"], "--suites=suite1 var1arg1") self.assertEqual(task_list["task1"]["tests"], SUITE1.tests) @@ -839,58 +791,66 @@ class CreateTaskList(unittest.TestCase): suites = [SUITE1, SUITE2, SUITE3] exclude_suites = [] suite_list = _create_executor_list(suites, exclude_suites) - task_list = burn_in.create_task_list(EVERGREEN_CONF, variant, suite_list, exclude_suites) - self.assertEqual(len(task_list), len(VARIANTS["variant_multiversion"].tasks)) - self.assertEqual(task_list["multiversion_task"]["use_multiversion"], MULTIVERSION_PATH) + task_list = burn_in.create_task_list(self.evergreen_conf, variant, suite_list, + exclude_suites) + self.assertEqual(len(task_list), len(self.VARIANTS["variant_multiversion"].tasks)) + self.assertEqual(task_list["multiversion_task"]["use_multiversion"], self.MULTIVERSION_PATH) def test_create_task_list_gen_tasks(self): variant = "variant_generate_tasks" suites = [SUITE3] exclude_suites = [] suite_list = _create_executor_list(suites, exclude_suites) - task_list = burn_in.create_task_list(EVERGREEN_CONF, variant, suite_list, exclude_suites) - self.assertEqual(len(task_list), len(VARIANTS["variant_generate_tasks"].tasks)) - self.assertIn(GENERATE_RESMOKE_TASKS_NAME, task_list) - self.assertEqual(task_list[GENERATE_RESMOKE_TASKS_NAME]["tests"], SUITE3.tests) - self.assertIsNone(task_list[GENERATE_RESMOKE_TASKS_NAME]["use_multiversion"]) + task_list = burn_in.create_task_list(self.evergreen_conf, variant, suite_list, + exclude_suites) + self.assertEqual(len(task_list), len(self.VARIANTS["variant_generate_tasks"].tasks)) + self.assertIn(self.GENERATE_RESMOKE_TASKS_NAME, task_list) + self.assertEqual(task_list[self.GENERATE_RESMOKE_TASKS_NAME]["tests"], SUITE3.tests) + self.assertIsNone(task_list[self.GENERATE_RESMOKE_TASKS_NAME]["use_multiversion"]) def test_create_task_list_gen_tasks_diff_task_names(self): variant = "variant_generate_tasks_diff_names" suites = [SUITE3] exclude_suites = [] suite_list = _create_executor_list(suites, exclude_suites) - task_list = burn_in.create_task_list(EVERGREEN_CONF, variant, suite_list, exclude_suites) - self.assertEqual(len(task_list), len(VARIANTS["variant_generate_tasks"].tasks)) - self.assertIn(GENERATE_RESMOKE_TASKS_NAME, task_list) - self.assertEqual(task_list[GENERATE_RESMOKE_TASKS_NAME]["tests"], SUITE3.tests) - self.assertIsNone(task_list[GENERATE_RESMOKE_TASKS_NAME]["use_multiversion"]) + task_list = burn_in.create_task_list(self.evergreen_conf, variant, suite_list, + exclude_suites) + self.assertEqual(len(task_list), len(self.VARIANTS["variant_generate_tasks"].tasks)) + self.assertIn(self.GENERATE_RESMOKE_TASKS_NAME, task_list) + self.assertEqual(task_list[self.GENERATE_RESMOKE_TASKS_NAME]["tests"], SUITE3.tests) + self.assertIsNone(task_list[self.GENERATE_RESMOKE_TASKS_NAME]["use_multiversion"]) def test_create_task_list_gen_tasks_multiversion(self): variant = "variant_generate_tasks_multiversion" suites = [SUITE3] exclude_suites = [] suite_list = _create_executor_list(suites, exclude_suites) - task_list = burn_in.create_task_list(EVERGREEN_CONF, variant, suite_list, exclude_suites) - self.assertEqual(len(task_list), len(VARIANTS["variant_generate_tasks_multiversion"].tasks)) - self.assertEqual(task_list[GENERATE_RESMOKE_TASKS_NAME]["use_multiversion"], - MULTIVERSION_PATH) + task_list = burn_in.create_task_list(self.evergreen_conf, variant, suite_list, + exclude_suites) + self.assertEqual( + len(task_list), len(self.VARIANTS["variant_generate_tasks_multiversion"].tasks)) + self.assertEqual(task_list[self.GENERATE_RESMOKE_TASKS_NAME]["use_multiversion"], + self.MULTIVERSION_PATH) def test_create_task_list_gen_tasks_no_suite(self): variant = "variant_generate_tasks_no_suite" suites = [SUITE3] exclude_suites = [] suite_list = _create_executor_list(suites, exclude_suites) - task_list = burn_in.create_task_list(EVERGREEN_CONF, variant, suite_list, exclude_suites) - self.assertEqual(len(task_list), len(VARIANTS["variant_generate_tasks_no_suite"].tasks)) - self.assertIn(GENERATE_RESMOKE_TASKS_NAME, task_list) - self.assertEqual(task_list[GENERATE_RESMOKE_TASKS_NAME]["tests"], SUITE3.tests) + task_list = burn_in.create_task_list(self.evergreen_conf, variant, suite_list, + exclude_suites) + self.assertEqual( + len(task_list), len(self.VARIANTS["variant_generate_tasks_no_suite"].tasks)) + self.assertIn(self.GENERATE_RESMOKE_TASKS_NAME, task_list) + self.assertEqual(task_list[self.GENERATE_RESMOKE_TASKS_NAME]["tests"], SUITE3.tests) def test_create_task_list_no_excludes(self): variant = "variant1" suites = [SUITE1, SUITE2] exclude_suites = [] suite_list = _create_executor_list(suites, exclude_suites) - task_list = burn_in.create_task_list(EVERGREEN_CONF, variant, suite_list, exclude_suites) + task_list = burn_in.create_task_list(self.evergreen_conf, variant, suite_list, + exclude_suites) self.assertEqual(len(task_list), 1) self.assertIn("task1", task_list) self.assertEqual(task_list["task1"]["resmoke_args"], "--suites=suite1 var1arg1") @@ -903,7 +863,8 @@ class CreateTaskList(unittest.TestCase): suites = [SUITE1, SUITE2, SUITE3] suite_list = _create_executor_list(suites, []) exclude_suites = ["suite2"] - task_list = burn_in.create_task_list(EVERGREEN_CONF, variant, suite_list, exclude_suites) + task_list = burn_in.create_task_list(self.evergreen_conf, variant, suite_list, + exclude_suites) self.assertEqual(len(task_list), 1) self.assertIn("task3", task_list) self.assertEqual(task_list["task3"]["resmoke_args"], "--suites=suite3 var2arg3") @@ -915,7 +876,8 @@ class CreateTaskList(unittest.TestCase): variant = "variant2" suite_list = {} exclude_suites = ["suite2"] - task_list = burn_in.create_task_list(EVERGREEN_CONF, variant, suite_list, exclude_suites) + task_list = burn_in.create_task_list(self.evergreen_conf, variant, suite_list, + exclude_suites) self.assertEqual(len(task_list), 0) self.assertEqual(task_list, {}) @@ -931,7 +893,7 @@ class CreateTaskList(unittest.TestCase): suite_list = _create_executor_list(suites, []) with patch("sys.exit", _raise_bad_variant): with self.assertRaises(BadVariant): - burn_in.create_task_list(EVERGREEN_CONF, variant, suite_list, []) + burn_in.create_task_list(self.evergreen_conf, variant, suite_list, []) class FindChangedTests(unittest.TestCase): diff --git a/etc/evergreen.yml b/etc/evergreen.yml index 41680992205..9b4cce36b25 100644 --- a/etc/evergreen.yml +++ b/etc/evergreen.yml @@ -305,13 +305,13 @@ variables: tasks: - name: compile_all_run_unittests_TG requires: - - name: burn_in_tests_gen + - name: burn_in_tests distros: - windows-64-vs2017-compile - name: compile_benchmarks distros: - windows-64-vs2017-compile - - name: burn_in_tests_gen + - name: burn_in_tests - name: aggregation_multiversion_fuzzer_gen - name: aggregation_wildcard_fuzzer_gen - name: audit @@ -3317,7 +3317,7 @@ timeout: ## - Non standard jstests, i.e., jstestfuzz ## Note that the task name supports glob patterns. test_lifecycle_excluded_tasks: -- burn_in_tests* +- burn_in_tests - compile* - benchmarks* - dbtest* @@ -4613,10 +4613,6 @@ tasks: diff -w requirements.txt.python2.old requirements.txt.python2.new diff -w requirements.txt.python3.old requirements.txt.python3.new -### This task is deprecated, but left in here in case of need to run burn_in_tests -### instead of the generated task: -### - Rename burn_in_tests_gen task to burn_in_tests_gen_UNUSED -### - Rename burn_in_tests_gen to burn_in_tests on the build variants - <<: *task_template name: burn_in_tests depends_on: @@ -4659,61 +4655,6 @@ tasks: task_path_suffix: /data/multiversion:$HOME resmoke_wrapper: $python buildscripts/burn_in_tests.py --testListFile=jstests/new_tests.json resmoke_args: --repeatSuites=2 -### - -- name: burn_in_tests_gen - commands: - - func: "git get project" - # The repository is cloned in a directory distinct from src for the modified test detection - # because the extraction of the artifacts performed in the 'do setup' causes - # 'git diff --name-only' to see all tests as modified on Windows (git 1.9.5). See SERVER-30634. - vars: - git_project_directory: burn_in_tests_clonedir - - command: shell.exec - params: - working_dir: burn_in_tests_clonedir - shell: bash - script: | - set -o errexit - set -o verbose - mkdir ../src - ${activate_virtualenv} - $python -m pip install -r etc/pip/evgtest-requirements.txt - # If this is a scheduled build, we check for changes against the last scheduled commit. - if [ "${is_patch}" != "true" ]; then - burn_in_args="--checkEvergreen" - fi - # Capture a list of new and modified tests. The expansion macro burn_in_tests_build_variant - # is used to for finding the associated tasks from a different build varaint than the - # burn_in_tests_gen task executes on. - build_variant_opts="--buildVariant=${build_variant}" - if [ -n "${burn_in_tests_build_variant|}" ]; then - build_variant_opts="--buildVariant=${burn_in_tests_build_variant} --runBuildVariant=${build_variant}" - fi - # Increase the burn_in repetition from 2 to 1000 executions or 10 minutes - burn_in_args="$burn_in_args --repeatTestsMin=2 --repeatTestsMax=1000 --repeatTestsSecs=600" - # Evergreen executable is in $HOME. - PATH=$PATH:$HOME $python buildscripts/burn_in_tests.py --branch=${branch_name} $build_variant_opts --distro=${distro_id} --generateTasksFile=../src/burn_in_tests_gen.json --noExec $burn_in_args - - command: archive.targz_pack - params: - target: src/burn_in_tests_gen.tgz - source_dir: src - include: - - burn_in_tests_gen.json - - command: s3.put - params: - aws_key: ${aws_key} - aws_secret: ${aws_secret} - local_file: src/burn_in_tests_gen.tgz - remote_file: ${project}/${build_variant}/${revision}/burn_in_tests_gen/burn_in_tests_gen-${build_id}.tgz - bucket: mciuploads - permissions: public-read - content_type: ${content_type|application/gzip} - display_name: Burn_in_tests Task Config - Execution ${execution} - - command: generate.tasks - params: - files: - - src/burn_in_tests_gen.json - <<: *benchmark_template name: benchmarks_orphaned @@ -7839,10 +7780,10 @@ buildvariants: # configurations to run the tests under. Do not add more tasks to this list. - name: compile_TG requires: - - name: burn_in_tests_gen + - name: burn_in_tests distros: - rhel62-large - - name: burn_in_tests_gen + - name: burn_in_tests - name: linux-64-lsm stepback: false @@ -9510,14 +9451,14 @@ buildvariants: tasks: - name: compile_TG requires: - - name: burn_in_tests_gen + - name: burn_in_tests - name: verify_pip distros: - windows-64-vs2017-compile - name: compile_benchmarks distros: - windows-64-vs2017-compile - - name: burn_in_tests_gen + - name: burn_in_tests - name: verify_pip - name: audit - name: auth_audit_gen @@ -10588,7 +10529,7 @@ buildvariants: tasks: - name: compile_all_run_unittests_TG requires: - - name: burn_in_tests_gen + - name: burn_in_tests - name: verify_pip - name: lint distros: @@ -10598,7 +10539,7 @@ buildvariants: - rhel62-large - name: lint - name: lint_yaml - - name: burn_in_tests_gen + - name: burn_in_tests distros: - rhel62-large - name: verify_pip @@ -11008,10 +10949,10 @@ buildvariants: # not add more tasks to this list. - name: compile_TG requires: - - name: burn_in_tests_gen + - name: burn_in_tests distros: - rhel62-large - - name: burn_in_tests_gen + - name: burn_in_tests - name: enterprise-rhel-62-64-bit-coverage display_name: "~ Enterprise RHEL 6.2 DEBUG Code Coverage" @@ -13057,10 +12998,10 @@ buildvariants: - rhel62-large - name: compile_TG requires: - - name: burn_in_tests_gen + - name: burn_in_tests distros: - rhel62-large - - name: burn_in_tests_gen + - name: burn_in_tests - name: jsCore - name: rhel-62-64-bit-mobile @@ -13208,10 +13149,10 @@ buildvariants: # to this list. - name: compile_TG requires: - - name: burn_in_tests_gen + - name: burn_in_tests distros: - rhel62-large - - name: burn_in_tests_gen + - name: burn_in_tests - name: jsCore_mobile - name: macos-mobile |