diff options
author | Jonathan Abrahams <jonathan@mongodb.com> | 2019-02-25 13:13:42 -0500 |
---|---|---|
committer | Jonathan Abrahams <jonathan@mongodb.com> | 2019-03-18 11:56:38 -0400 |
commit | df4243e71b7c69c4da59c90e657f0749d6c9c979 (patch) | |
tree | c5848bdb340cad7d567e715e568d721c3c47cabd | |
parent | 39cd4943c968f63d42d0b81909110f7b5161645b (diff) | |
download | mongo-df4243e71b7c69c4da59c90e657f0749d6c9c979.tar.gz |
SERVER-39311 Enable burn_in repetition count: at least 2 executions, with a maximum of 1000 excutions or 10 minutes of execution time
-rw-r--r-- | buildscripts/burn_in_tests.py | 31 | ||||
-rw-r--r-- | buildscripts/tests/ciconfig/test_evergreen.py | 84 | ||||
-rw-r--r-- | buildscripts/tests/test_burn_in_tests.py | 436 | ||||
-rw-r--r-- | etc/evergreen.yml | 89 |
4 files changed, 378 insertions, 262 deletions
diff --git a/buildscripts/burn_in_tests.py b/buildscripts/burn_in_tests.py index 398184d4f96..906aa618e7a 100644 --- a/buildscripts/burn_in_tests.py +++ b/buildscripts/burn_in_tests.py @@ -65,9 +65,13 @@ def parse_command_line(): help="The base commit to compare to for determining changes.") parser.add_option("--buildVariant", dest="buildvariant", default=None, - help=("The buildvariant the tasks will execute on. Required when" + help=("The buildvariant to select the tasks. Required when" " generating the JSON file with test executor information")) + parser.add_option("--runBuildVariant", dest="run_buildvariant", default=None, + help=("The buildvariant the tasks will execute on. If not specied then tasks" + " will execute on the the buildvariant specied in --buildVariant.")) + parser.add_option("--distro", dest="distro", default=None, help=("The distro the tasks will execute on. Can only be specified" " with --generateTasksFile.")) @@ -119,6 +123,14 @@ def parse_command_line(): return options, args +def check_variant(buildvariant, parser): + """Check if the buildvariant is found in the evergreen file.""" + evg_conf = evergreen.parse_evergreen_file(EVERGREEN_FILE) + if not evg_conf.get_variant(buildvariant): + parser.error("Buildvariant '{}' not found in {}, select from:\n\t{}".format( + buildvariant, EVERGREEN_FILE, "\n\t".join(sorted(evg_conf.variant_names)))) + + def validate_options(parser, options): """Validate command line options.""" @@ -139,10 +151,10 @@ def validate_options(parser, options): parser.error("Must specify --buildVariant to find changed tests") if options.buildvariant: - evg_conf = evergreen.parse_evergreen_file(EVERGREEN_FILE) - if not evg_conf.get_variant(options.buildvariant): - parser.error("Buildvariant '{}' not found in {}, select from:\n\t{}".format( - options.buildvariant, EVERGREEN_FILE, "\n\t".join(sorted(evg_conf.variant_names)))) + check_variant(options.buildvariant, parser) + + if options.run_buildvariant: + check_variant(options.run_buildvariant, parser) def find_last_activated_task(revisions, variant, branch_name): @@ -428,6 +440,13 @@ def _sub_task_name(variant, task, task_num): return "burn_in:{}_{}_{}".format(variant, task, task_num) +def _get_run_buildvariant(options): + """Return the build variant to execute the tasks on.""" + if options.run_buildvariant: + return options.run_buildvariant + return options.buildvariant + + def create_generate_tasks_file(options, tests_by_task): """Create the Evergreen generate.tasks file.""" @@ -462,7 +481,7 @@ def create_generate_tasks_file(options, tests_by_task): evg_sub_task.commands(commands) display_task = DisplayTaskDefinition("burn_in_tests").execution_tasks(task_names) - evg_config.variant(options.buildvariant).tasks(task_specs).display_task(display_task) + evg_config.variant(_get_run_buildvariant(options)).tasks(task_specs).display_task(display_task) _write_json_file(evg_config.to_map(), options.generate_tasks_file) diff --git a/buildscripts/tests/ciconfig/test_evergreen.py b/buildscripts/tests/ciconfig/test_evergreen.py index d7b345ca144..fd57a449f79 100644 --- a/buildscripts/tests/ciconfig/test_evergreen.py +++ b/buildscripts/tests/ciconfig/test_evergreen.py @@ -71,7 +71,7 @@ class TestEvergreenProjectConfig(unittest.TestCase): self.assertIn("amazon", self.conf.distro_names) -class TestTask(unittest.TestCase): #pylint: disable=too-many-public-methods +class TestTask(unittest.TestCase): # pylint: disable=too-many-public-methods """Unit tests for the Task class.""" def test_from_dict(self): @@ -147,19 +147,6 @@ class TestTask(unittest.TestCase): #pylint: disable=too-many-public-methods self.assertIsNone(task.multiversion_setup_command) self.assertIsNone(task.multiversion_path) - def test_get_vars_suite_name_run_tests(self): - suite_name = "core" - task_commands = [{ - "func": "run tests", - "vars": {"resmoke_args": "--suites={} --shellWriteMode=commands".format(suite_name)} - }] - - task_dict = {"name": "jsCore", "commands": task_commands} - task = _evergreen.Task(task_dict) - - with self.assertRaises(ValueError): - task.get_vars_suite_name(task_commands[0]["vars"]) - def test_resmoke_args_gen(self): task_name = "core" task_commands = [{ @@ -252,6 +239,46 @@ class TestTask(unittest.TestCase): #pylint: disable=too-many-public-methods self.assertDictEqual(task_commands[0], task.generate_resmoke_tasks_command) self.assertEqual(task_name, task.get_vars_task_name(task_commands[0]["vars"])) + def test_gen_resmoke_multiversion(self): + multiversion_path = "/data/multiversion" + task_name = "core" + task_commands = [{ + "func": "generate resmoke tasks", "vars": { + "task": task_name, "resmoke_args": "--shellWriteMode=commands", + "use_multiversion": multiversion_path + } + }] + task_dict = {"name": "jsCore", "commands": task_commands} + task = _evergreen.Task(task_dict) + + self.assertTrue(task.is_multiversion_task) + self.assertEqual(multiversion_path, task.multiversion_path) + + def test_gen_resmoke_no_multiversion(self): + task_name = "core" + task_commands = [{ + "func": "generate resmoke tasks", + "vars": {"task": task_name, "resmoke_args": "--shellWriteMode=commands"} + }] + task_dict = {"name": "jsCore", "commands": task_commands} + task = _evergreen.Task(task_dict) + + self.assertFalse(task.is_multiversion_task) + self.assertIsNone(task.multiversion_path) + + def test_get_vars_suite_name_run_tests(self): + suite_name = "core" + task_commands = [{ + "func": "run tests", + "vars": {"resmoke_args": "--suites={} --shellWriteMode=commands".format(suite_name)} + }] + + task_dict = {"name": "jsCore", "commands": task_commands} + task = _evergreen.Task(task_dict) + + with self.assertRaises(ValueError): + task.get_vars_suite_name(task_commands[0]["vars"]) + def test_get_vars_suite_name_generate_resmoke_tasks(self): task_name = "jsCore" suite_name = "core" @@ -274,7 +301,7 @@ class TestTask(unittest.TestCase): #pylint: disable=too-many-public-methods with self.assertRaises(TypeError): task.get_vars_suite_name(None) - def test_get_vars_suite_name_gen_resmoke_notasksuite(self): + def test_get_vars_suite_name_gen_resmoke_notask_nosuite(self): task_name = "jsCore" task_commands = [{ "func": "generate resmoke tasks", "vars": {"resmoke_args": "--shellWriteMode=commands"} @@ -285,33 +312,6 @@ class TestTask(unittest.TestCase): #pylint: disable=too-many-public-methods with self.assertRaises(ValueError): task.get_vars_suite_name(task_commands[0]["vars"]) - def test_gen_resmoke_multiversion(self): - multiversion_path = "/data/multiversion" - task_name = "core" - task_commands = [{ - "func": "generate resmoke tasks", "vars": { - "task": task_name, "resmoke_args": "--shellWriteMode=commands", - "use_multiversion": multiversion_path - } - }] - task_dict = {"name": "jsCore", "commands": task_commands} - task = _evergreen.Task(task_dict) - - self.assertTrue(task.is_multiversion_task) - self.assertEqual(multiversion_path, task.multiversion_path) - - def test_gen_resmoke_no_multiversion(self): - task_name = "core" - task_commands = [{ - "func": "generate resmoke tasks", - "vars": {"task": task_name, "resmoke_args": "--shellWriteMode=commands"} - }] - task_dict = {"name": "jsCore", "commands": task_commands} - task = _evergreen.Task(task_dict) - - self.assertFalse(task.is_multiversion_task) - self.assertIsNone(task.multiversion_path) - def test_get_vars_task_name(self): task_name = "task_name" task_commands = [{ diff --git a/buildscripts/tests/test_burn_in_tests.py b/buildscripts/tests/test_burn_in_tests.py index 0ab6bec59c3..5df69720804 100644 --- a/buildscripts/tests/test_burn_in_tests.py +++ b/buildscripts/tests/test_burn_in_tests.py @@ -20,21 +20,118 @@ EVG_CLIENT = "buildscripts.client.evergreen" GIT = "buildscripts.git" RESMOKELIB = "buildscripts.resmokelib" +GENERATE_RESMOKE_TASKS_NAME = "this_is_a_gen_task" +GET_GENERATE_RESMOKE_TASKS_NAME = lambda _: GENERATE_RESMOKE_TASKS_NAME +GENERATE_RESMOKE_TASKS_COMMAND = { + "func": "generate resmoke tasks", "vars": { + "task": GENERATE_RESMOKE_TASKS_NAME, "suite": "suite3", + "resmoke_args": "--shellWriteMode=commands" + } +} -class ValidateOptions(unittest.TestCase): - class ParserError(Exception): - pass +GENERATE_RESMOKE_TASKS_COMMAND2 = { + "func": "generate resmoke tasks", + "vars": {"task": GENERATE_RESMOKE_TASKS_NAME, "resmoke_args": "--shellWriteMode=commands"} +} - @staticmethod - def _raise_parser_error(err): - raise ValidateOptions.ParserError(err) +MULTIVERSION_PATH = "/data/multiversion" +GENERATE_RESMOKE_TASKS_MULTIVERSION_COMMAND = { + "func": "generate resmoke tasks", "vars": { + "task": GENERATE_RESMOKE_TASKS_NAME, "resmoke_args": "--shellWriteMode=commands", + "use_multiversion": MULTIVERSION_PATH + } +} + +MULTIVERSION_COMMAND = {"func": "do multiversion setup"} +RUN_TESTS_MULTIVERSION_COMMAND = { + "func": "run tests", "vars": { + "task": GENERATE_RESMOKE_TASKS_NAME, "resmoke_args": "--shellWriteMode=commands", + "task_path_suffix": MULTIVERSION_PATH + } +} - @staticmethod - def _mock_parser(): - parser = Mock() - parser.error = ValidateOptions._raise_parser_error - return parser +def tasks_mock( #pylint: disable=too-many-arguments + tasks, generate_resmoke_tasks_command=None, get_vars_task_name=None, run_tests_command=None, + multiversion_path=None, multiversion_setup_command=None): + task_list = Mock() + task_list.tasks = [] + for idx, task in enumerate(tasks): + task_list.tasks.append(Mock()) + task_list.tasks[idx].is_generate_resmoke_task = generate_resmoke_tasks_command is not None + task_list.tasks[idx].is_run_tests_task = run_tests_command is not None + task_list.tasks[idx].is_multiversion_task = multiversion_path is not None + task_list.tasks[idx].generate_resmoke_tasks_command = generate_resmoke_tasks_command + task_list.tasks[idx].run_tests_command = run_tests_command + task_list.tasks[idx].get_vars_task_name = get_vars_task_name + task_list.tasks[idx].name = task["name"] + resmoke_args = task.get("combined_resmoke_args") + task_list.tasks[idx].combined_resmoke_args = resmoke_args + task_list.tasks[idx].resmoke_suite = evg.ResmokeArgs.get_arg( + resmoke_args, "suites") if resmoke_args else None + task_list.tasks[idx].multiversion_path = multiversion_path + task_list.tasks[idx].multiversion_setup_command = multiversion_setup_command + + return task_list + + +VARIANTS = { + "variantall": + tasks_mock([{"name": "task1", "combined_resmoke_args": "--suites=suite1 var1arg1"}, + {"name": "task2", "combined_resmoke_args": "--suites=suite1 var1arg2"}, + {"name": "task3", "combined_resmoke_args": "--suites=suite1 var1arg3"}]), + "variant1": + tasks_mock([{"name": "task1", "combined_resmoke_args": "--suites=suite1 var1arg1"}, + {"name": "task2"}]), + "variant2": + tasks_mock([{"name": "task2", "combined_resmoke_args": "var2arg1"}, + {"name": "task3", "combined_resmoke_args": "--suites=suite3 var2arg3"}]), + "variant3": + tasks_mock([{"name": "task2", "combined_resmoke_args": "var3arg1"}]), + "variant4": + tasks_mock([]), + "variant_multiversion": + tasks_mock( + [{"name": "multiversion_task", "combined_resmoke_args": "--suites=suite3 vararg"}], + run_tests_command=RUN_TESTS_MULTIVERSION_COMMAND, + multiversion_setup_command=RUN_TESTS_MULTIVERSION_COMMAND, + multiversion_path=MULTIVERSION_PATH), + "variant_generate_tasks": + tasks_mock([{ + "name": GENERATE_RESMOKE_TASKS_NAME, "combined_resmoke_args": "--suites=suite3 vararg" + }], generate_resmoke_tasks_command=GENERATE_RESMOKE_TASKS_COMMAND, + get_vars_task_name=GET_GENERATE_RESMOKE_TASKS_NAME), + "variant_generate_tasks_no_suite": + tasks_mock([{ + "name": GENERATE_RESMOKE_TASKS_NAME, "combined_resmoke_args": "--suites=suite3 vararg" + }], generate_resmoke_tasks_command=GENERATE_RESMOKE_TASKS_COMMAND2, + get_vars_task_name=GET_GENERATE_RESMOKE_TASKS_NAME), + "variant_generate_tasks_diff_names": + tasks_mock([{ + "name": "gen_task_name_different_from_vars_task_name", + "combined_resmoke_args": "--suites=suite3 vararg" + }], generate_resmoke_tasks_command=GENERATE_RESMOKE_TASKS_COMMAND, + get_vars_task_name=GET_GENERATE_RESMOKE_TASKS_NAME), + "variant_generate_tasks_multiversion": + tasks_mock([{ + "name": GENERATE_RESMOKE_TASKS_NAME, "combined_resmoke_args": "--suites=suite3 vararg" + }], generate_resmoke_tasks_command=GENERATE_RESMOKE_TASKS_MULTIVERSION_COMMAND, + get_vars_task_name=GET_GENERATE_RESMOKE_TASKS_NAME, + multiversion_path=MULTIVERSION_PATH), +} + +EVERGREEN_CONF = Mock() +EVERGREEN_CONF.get_variant = VARIANTS.get +EVERGREEN_CONF.variant_names = VARIANTS.keys() + + +def _mock_parser(): + parser = Mock() + parser.error = Mock() + return parser + + +class TestValidateOptions(unittest.TestCase): @staticmethod def _mock_options(): options = Mock() @@ -43,109 +140,132 @@ class ValidateOptions(unittest.TestCase): options.repeat_tests_min = None options.repeat_tests_secs = None options.buildvariant = None + options.run_buildvariant = None options.test_list_file = None return options def test_validate_options_listfile_buildvariant(self): - parser = self._mock_parser() + mock_parser = _mock_parser() options = self._mock_options() options.test_list_file = "list_file.json" options.buildvariant = "variant1" - with patch(EVG_CI + ".parse_evergreen_file", return_value=CreateTaskList.evergreen_conf): - burn_in.validate_options(parser, options) + with patch(EVG_CI + ".parse_evergreen_file", return_value=EVERGREEN_CONF): + burn_in.validate_options(mock_parser, options) + mock_parser.error.assert_not_called() def test_validate_options_nolistfile_buildvariant(self): - parser = self._mock_parser() + mock_parser = _mock_parser() options = self._mock_options() options.buildvariant = "variant1" - with patch(EVG_CI + ".parse_evergreen_file", return_value=CreateTaskList.evergreen_conf): - burn_in.validate_options(parser, options) + with patch(EVG_CI + ".parse_evergreen_file", return_value=EVERGREEN_CONF): + burn_in.validate_options(mock_parser, options) + mock_parser.error.assert_not_called() def test_validate_options_listfile_nobuildvariant(self): - parser = self._mock_parser() + mock_parser = _mock_parser() options = self._mock_options() options.test_list_file = "list_file.json" - with patch(EVG_CI + ".parse_evergreen_file", return_value=CreateTaskList.evergreen_conf): - burn_in.validate_options(parser, options) + with patch(EVG_CI + ".parse_evergreen_file", return_value=EVERGREEN_CONF): + burn_in.validate_options(mock_parser, options) + mock_parser.error.assert_not_called() def test_validate_options_no_listfile_no_buildvariant(self): - parser = self._mock_parser() + mock_parser = _mock_parser() options = self._mock_options() - with patch(EVG_CI + ".parse_evergreen_file", return_value=CreateTaskList.evergreen_conf): - with self.assertRaises(self.ParserError): - burn_in.validate_options(parser, options) + with patch(EVG_CI + ".parse_evergreen_file", return_value=EVERGREEN_CONF): + burn_in.validate_options(mock_parser, options) + mock_parser.error.assert_called() def test_validate_options_buildvariant(self): - parser = self._mock_parser() + mock_parser = _mock_parser() options = self._mock_options() options.buildvariant = "variant1" - with patch(EVG_CI + ".parse_evergreen_file", return_value=CreateTaskList.evergreen_conf): - burn_in.validate_options(parser, options) + with patch(EVG_CI + ".parse_evergreen_file", return_value=EVERGREEN_CONF): + burn_in.validate_options(mock_parser, options) + mock_parser.error.assert_not_called() + + def test_validate_options_run_buildvariant(self): + mock_parser = _mock_parser() + options = self._mock_options() + options.buildvariant = "variant1" + options.run_buildvariant = "variant1" + with patch(EVG_CI + ".parse_evergreen_file", return_value=EVERGREEN_CONF): + burn_in.validate_options(mock_parser, options) + mock_parser.error.assert_not_called() def test_validate_options_bad_buildvariant(self): - parser = self._mock_parser() + mock_parser = _mock_parser() options = self._mock_options() options.buildvariant = "badvariant1" - with patch(EVG_CI + ".parse_evergreen_file", return_value=CreateTaskList.evergreen_conf): - with self.assertRaises(self.ParserError): - burn_in.validate_options(parser, options) + with patch(EVG_CI + ".parse_evergreen_file", return_value=EVERGREEN_CONF): + burn_in.validate_options(mock_parser, options) + mock_parser.error.assert_called() + + def test_validate_options_bad_run_buildvariant(self): + mock_parser = _mock_parser() + options = self._mock_options() + options.run_buildvariant = "badvariant1" + with patch(EVG_CI + ".parse_evergreen_file", return_value=EVERGREEN_CONF): + burn_in.validate_options(mock_parser, options) + mock_parser.error.assert_called() def test_validate_options_tests_max_no_tests_secs(self): - parser = self._mock_parser() + mock_parser = _mock_parser() options = self._mock_options() options.repeat_tests_max = 3 - with patch(EVG_CI + ".parse_evergreen_file", return_value=CreateTaskList.evergreen_conf): - with self.assertRaises(self.ParserError): - burn_in.validate_options(parser, options) + with patch(EVG_CI + ".parse_evergreen_file", return_value=EVERGREEN_CONF): + burn_in.validate_options(mock_parser, options) + mock_parser.error.assert_called() def test_validate_options_tests_min_no_tests_secs(self): - parser = self._mock_parser() + mock_parser = _mock_parser() options = self._mock_options() options.repeat_tests_min = 3 - with patch(EVG_CI + ".parse_evergreen_file", return_value=CreateTaskList.evergreen_conf): - with self.assertRaises(self.ParserError): - burn_in.validate_options(parser, options) + with patch(EVG_CI + ".parse_evergreen_file", return_value=EVERGREEN_CONF): + burn_in.validate_options(mock_parser, options) + mock_parser.error.assert_called() def test_validate_options_tests_min_gt_tests_max(self): - parser = self._mock_parser() + mock_parser = _mock_parser() options = self._mock_options() options.repeat_tests_min = 3 options.repeat_tests_max = 2 - with patch(EVG_CI + ".parse_evergreen_file", return_value=CreateTaskList.evergreen_conf): - with self.assertRaises(self.ParserError): - burn_in.validate_options(parser, options) + with patch(EVG_CI + ".parse_evergreen_file", return_value=EVERGREEN_CONF): + burn_in.validate_options(mock_parser, options) + mock_parser.error.assert_called() def test_validate_options_tests_secs(self): - parser = self._mock_parser() + mock_parser = _mock_parser() options = self._mock_options() options.buildvariant = "variant1" options.repeat_tests_min = 2 options.repeat_tests_max = 1000 options.repeat_tests_secs = 3 - with patch(EVG_CI + ".parse_evergreen_file", return_value=CreateTaskList.evergreen_conf): - burn_in.validate_options(parser, options) + with patch(EVG_CI + ".parse_evergreen_file", return_value=EVERGREEN_CONF): + burn_in.validate_options(mock_parser, options) + mock_parser.error.assert_not_called() def test_validate_options_tests_secs_and_tests_num(self): - parser = self._mock_parser() + mock_parser = _mock_parser() options = self._mock_options() options.buildvariant = "variant1" options.repeat_tests_num = 1 options.repeat_tests_min = 1 options.repeat_tests_max = 3 options.repeat_tests_secs = 3 - with patch(EVG_CI + ".parse_evergreen_file", return_value=CreateTaskList.evergreen_conf): - with self.assertRaises(self.ParserError): - burn_in.validate_options(parser, options) + with patch(EVG_CI + ".parse_evergreen_file", return_value=EVERGREEN_CONF): + burn_in.validate_options(mock_parser, options) + mock_parser.error.assert_called() def test_validate_options_tests_secs_no_buildvariant(self): - parser = self._mock_parser() + mock_parser = _mock_parser() options = self._mock_options() options.repeat_tests_min = 1 options.repeat_tests_max = 3 options.repeat_tests_secs = 3 - with patch(EVG_CI + ".parse_evergreen_file", return_value=CreateTaskList.evergreen_conf): - with self.assertRaises(self.ParserError): - burn_in.validate_options(parser, options) + with patch(EVG_CI + ".parse_evergreen_file", return_value=EVERGREEN_CONF): + burn_in.validate_options(mock_parser, options) + mock_parser.error.assert_called() class TestGetResmokeRepeatOptions(unittest.TestCase): @@ -219,6 +339,41 @@ class TestGetResmokeRepeatOptions(unittest.TestCase): self.assertEqual(repeat_options, "--repeatSuites={}".format(burn_in.REPEAT_SUITES)) +class TestCheckVariant(unittest.TestCase): + @staticmethod + def test_check_variant(): + mock_parser = _mock_parser() + buildvariant = "variant1" + with patch(EVG_CI + ".parse_evergreen_file", return_value=EVERGREEN_CONF): + burn_in.check_variant(buildvariant, mock_parser) + mock_parser.error.assert_not_called() + + @staticmethod + def test_check_variant_badvariant(): + mock_parser = _mock_parser() + buildvariant = "badvariant" + with patch(EVG_CI + ".parse_evergreen_file", return_value=EVERGREEN_CONF): + burn_in.check_variant(buildvariant, mock_parser) + mock_parser.error.assert_called() + + +class TestGetRunBuildvariant(unittest.TestCase): + def test__get_run_buildvariant_rb(self): + run_buildvariant = "variant1" + buildvariant = "variant2" + options = Mock() + options.run_buildvariant = run_buildvariant + options.buildvariant = buildvariant + self.assertEqual(run_buildvariant, burn_in._get_run_buildvariant(options)) + + def test__get_run_buildvariant_bv(self): + buildvariant = "variant2" + options = Mock() + options.run_buildvariant = None + options.buildvariant = buildvariant + self.assertEqual(buildvariant, burn_in._get_run_buildvariant(options)) + + class TestGetTaskName(unittest.TestCase): def test__get_task_name(self): name = "mytask" @@ -317,6 +472,7 @@ class TestCreateGenerateTasksFile(unittest.TestCase): def _options_mock(): options = Mock() options.buildvariant = None + options.run_buildvariant = None options.distro = None return options @@ -657,122 +813,14 @@ class CreateExecutorList(unittest.TestCase): self.assertEqual(mock_suite_class.call_count, 0) -def tasks_mock( #pylint: disable=too-many-arguments - tasks, generate_resmoke_tasks_command=None, get_vars_task_name=None, run_tests_command=None, - multiversion_path=None, multiversion_setup_command=None): - task_list = Mock() - task_list.tasks = [] - for idx, task in enumerate(tasks): - task_list.tasks.append(Mock()) - task_list.tasks[idx].is_generate_resmoke_task = generate_resmoke_tasks_command is not None - task_list.tasks[idx].is_run_tests_task = run_tests_command is not None - task_list.tasks[idx].is_multiversion_task = multiversion_path is not None - task_list.tasks[idx].generate_resmoke_tasks_command = generate_resmoke_tasks_command - task_list.tasks[idx].run_tests_command = run_tests_command - task_list.tasks[idx].get_vars_task_name = get_vars_task_name - task_list.tasks[idx].name = task["name"] - resmoke_args = task.get("combined_resmoke_args") - task_list.tasks[idx].combined_resmoke_args = resmoke_args - task_list.tasks[idx].resmoke_suite = evg.ResmokeArgs.get_arg( - resmoke_args, "suites") if resmoke_args else None - task_list.tasks[idx].multiversion_path = multiversion_path - task_list.tasks[idx].multiversion_setup_command = multiversion_setup_command - - return task_list - - class CreateTaskList(unittest.TestCase): - - GENERATE_RESMOKE_TASKS_NAME = "this_is_a_gen_task" - GET_GENERATE_RESMOKE_TASKS_NAME = lambda _: CreateTaskList.GENERATE_RESMOKE_TASKS_NAME - GENERATE_RESMOKE_TASKS_COMMAND = { - "func": "generate resmoke tasks", "vars": { - "task": GENERATE_RESMOKE_TASKS_NAME, "suite": "suite3", - "resmoke_args": "--shellWriteMode=commands" - } - } - - GENERATE_RESMOKE_TASKS_COMMAND2 = { - "func": "generate resmoke tasks", - "vars": {"task": GENERATE_RESMOKE_TASKS_NAME, "resmoke_args": "--shellWriteMode=commands"} - } - - MULTIVERSION_PATH = "/data/multiversion" - GENERATE_RESMOKE_TASKS_MULTIVERSION_COMMAND = { - "func": "generate resmoke tasks", "vars": { - "task": GENERATE_RESMOKE_TASKS_NAME, "resmoke_args": "--shellWriteMode=commands", - "use_multiversion": MULTIVERSION_PATH - } - } - - MULTIVERSION_COMMAND = {"func": "do multiversion setup"} - RUN_TESTS_MULTIVERSION_COMMAND = { - "func": "run tests", "vars": { - "task": GENERATE_RESMOKE_TASKS_NAME, "resmoke_args": "--shellWriteMode=commands", - "task_path_suffix": MULTIVERSION_PATH - } - } - - VARIANTS = { - "variantall": - tasks_mock([{"name": "task1", "combined_resmoke_args": "--suites=suite1 var1arg1"}, - {"name": "task2", "combined_resmoke_args": "--suites=suite1 var1arg2"}, - {"name": "task3", "combined_resmoke_args": "--suites=suite1 var1arg3"}]), - "variant1": - tasks_mock([{"name": "task1", "combined_resmoke_args": "--suites=suite1 var1arg1"}, - {"name": "task2"}]), - "variant2": - tasks_mock([{"name": "task2", "combined_resmoke_args": "var2arg1"}, - {"name": "task3", "combined_resmoke_args": "--suites=suite3 var2arg3"}]), - "variant3": - tasks_mock([{"name": "task2", "combined_resmoke_args": "var3arg1"}]), - "variant4": - tasks_mock([]), - "variant_multiversion": - tasks_mock( - [{"name": "multiversion_task", "combined_resmoke_args": "--suites=suite3 vararg"}], - run_tests_command=RUN_TESTS_MULTIVERSION_COMMAND, - multiversion_setup_command=RUN_TESTS_MULTIVERSION_COMMAND, - multiversion_path=MULTIVERSION_PATH), - "variant_generate_tasks": - tasks_mock([{ - "name": GENERATE_RESMOKE_TASKS_NAME, - "combined_resmoke_args": "--suites=suite3 vararg" - }], generate_resmoke_tasks_command=GENERATE_RESMOKE_TASKS_COMMAND, - get_vars_task_name=GET_GENERATE_RESMOKE_TASKS_NAME), - "variant_generate_tasks_diff_names": - tasks_mock([{ - "name": "gen_task_name_different_from_vars_task_name", - "combined_resmoke_args": "--suites=suite3 vararg" - }], generate_resmoke_tasks_command=GENERATE_RESMOKE_TASKS_COMMAND, - get_vars_task_name=GET_GENERATE_RESMOKE_TASKS_NAME), - "variant_generate_tasks_no_suite": - tasks_mock([{ - "name": GENERATE_RESMOKE_TASKS_NAME, - "combined_resmoke_args": "--suites=suite3 vararg" - }], generate_resmoke_tasks_command=GENERATE_RESMOKE_TASKS_COMMAND2, - get_vars_task_name=GET_GENERATE_RESMOKE_TASKS_NAME), - "variant_generate_tasks_multiversion": - tasks_mock([{ - "name": GENERATE_RESMOKE_TASKS_NAME, - "combined_resmoke_args": "--suites=suite3 vararg" - }], generate_resmoke_tasks_command=GENERATE_RESMOKE_TASKS_MULTIVERSION_COMMAND, - get_vars_task_name=GET_GENERATE_RESMOKE_TASKS_NAME, - multiversion_path=MULTIVERSION_PATH), - } - - evergreen_conf = Mock() - evergreen_conf.get_variant = VARIANTS.get - evergreen_conf.variant_names = VARIANTS.keys() - def test_create_task_list(self): variant = "variantall" suites = [SUITE1, SUITE2, SUITE3] exclude_suites = [] suite_list = _create_executor_list(suites, exclude_suites) - task_list = burn_in.create_task_list(self.evergreen_conf, variant, suite_list, - exclude_suites) - self.assertEqual(len(task_list), len(self.VARIANTS["variantall"].tasks)) + task_list = burn_in.create_task_list(EVERGREEN_CONF, variant, suite_list, exclude_suites) + self.assertEqual(len(task_list), len(VARIANTS["variantall"].tasks)) self.assertIn("task1", task_list) self.assertEqual(task_list["task1"]["resmoke_args"], "--suites=suite1 var1arg1") self.assertEqual(task_list["task1"]["tests"], SUITE1.tests) @@ -791,66 +839,58 @@ class CreateTaskList(unittest.TestCase): suites = [SUITE1, SUITE2, SUITE3] exclude_suites = [] suite_list = _create_executor_list(suites, exclude_suites) - task_list = burn_in.create_task_list(self.evergreen_conf, variant, suite_list, - exclude_suites) - self.assertEqual(len(task_list), len(self.VARIANTS["variant_multiversion"].tasks)) - self.assertEqual(task_list["multiversion_task"]["use_multiversion"], self.MULTIVERSION_PATH) + task_list = burn_in.create_task_list(EVERGREEN_CONF, variant, suite_list, exclude_suites) + self.assertEqual(len(task_list), len(VARIANTS["variant_multiversion"].tasks)) + self.assertEqual(task_list["multiversion_task"]["use_multiversion"], MULTIVERSION_PATH) def test_create_task_list_gen_tasks(self): variant = "variant_generate_tasks" suites = [SUITE3] exclude_suites = [] suite_list = _create_executor_list(suites, exclude_suites) - task_list = burn_in.create_task_list(self.evergreen_conf, variant, suite_list, - exclude_suites) - self.assertEqual(len(task_list), len(self.VARIANTS["variant_generate_tasks"].tasks)) - self.assertIn(self.GENERATE_RESMOKE_TASKS_NAME, task_list) - self.assertEqual(task_list[self.GENERATE_RESMOKE_TASKS_NAME]["tests"], SUITE3.tests) - self.assertIsNone(task_list[self.GENERATE_RESMOKE_TASKS_NAME]["use_multiversion"]) + task_list = burn_in.create_task_list(EVERGREEN_CONF, variant, suite_list, exclude_suites) + self.assertEqual(len(task_list), len(VARIANTS["variant_generate_tasks"].tasks)) + self.assertIn(GENERATE_RESMOKE_TASKS_NAME, task_list) + self.assertEqual(task_list[GENERATE_RESMOKE_TASKS_NAME]["tests"], SUITE3.tests) + self.assertIsNone(task_list[GENERATE_RESMOKE_TASKS_NAME]["use_multiversion"]) def test_create_task_list_gen_tasks_diff_task_names(self): variant = "variant_generate_tasks_diff_names" suites = [SUITE3] exclude_suites = [] suite_list = _create_executor_list(suites, exclude_suites) - task_list = burn_in.create_task_list(self.evergreen_conf, variant, suite_list, - exclude_suites) - self.assertEqual(len(task_list), len(self.VARIANTS["variant_generate_tasks"].tasks)) - self.assertIn(self.GENERATE_RESMOKE_TASKS_NAME, task_list) - self.assertEqual(task_list[self.GENERATE_RESMOKE_TASKS_NAME]["tests"], SUITE3.tests) - self.assertIsNone(task_list[self.GENERATE_RESMOKE_TASKS_NAME]["use_multiversion"]) + task_list = burn_in.create_task_list(EVERGREEN_CONF, variant, suite_list, exclude_suites) + self.assertEqual(len(task_list), len(VARIANTS["variant_generate_tasks"].tasks)) + self.assertIn(GENERATE_RESMOKE_TASKS_NAME, task_list) + self.assertEqual(task_list[GENERATE_RESMOKE_TASKS_NAME]["tests"], SUITE3.tests) + self.assertIsNone(task_list[GENERATE_RESMOKE_TASKS_NAME]["use_multiversion"]) def test_create_task_list_gen_tasks_multiversion(self): variant = "variant_generate_tasks_multiversion" suites = [SUITE3] exclude_suites = [] suite_list = _create_executor_list(suites, exclude_suites) - task_list = burn_in.create_task_list(self.evergreen_conf, variant, suite_list, - exclude_suites) - self.assertEqual( - len(task_list), len(self.VARIANTS["variant_generate_tasks_multiversion"].tasks)) - self.assertEqual(task_list[self.GENERATE_RESMOKE_TASKS_NAME]["use_multiversion"], - self.MULTIVERSION_PATH) + task_list = burn_in.create_task_list(EVERGREEN_CONF, variant, suite_list, exclude_suites) + self.assertEqual(len(task_list), len(VARIANTS["variant_generate_tasks_multiversion"].tasks)) + self.assertEqual(task_list[GENERATE_RESMOKE_TASKS_NAME]["use_multiversion"], + MULTIVERSION_PATH) def test_create_task_list_gen_tasks_no_suite(self): variant = "variant_generate_tasks_no_suite" suites = [SUITE3] exclude_suites = [] suite_list = _create_executor_list(suites, exclude_suites) - task_list = burn_in.create_task_list(self.evergreen_conf, variant, suite_list, - exclude_suites) - self.assertEqual( - len(task_list), len(self.VARIANTS["variant_generate_tasks_no_suite"].tasks)) - self.assertIn(self.GENERATE_RESMOKE_TASKS_NAME, task_list) - self.assertEqual(task_list[self.GENERATE_RESMOKE_TASKS_NAME]["tests"], SUITE3.tests) + task_list = burn_in.create_task_list(EVERGREEN_CONF, variant, suite_list, exclude_suites) + self.assertEqual(len(task_list), len(VARIANTS["variant_generate_tasks_no_suite"].tasks)) + self.assertIn(GENERATE_RESMOKE_TASKS_NAME, task_list) + self.assertEqual(task_list[GENERATE_RESMOKE_TASKS_NAME]["tests"], SUITE3.tests) def test_create_task_list_no_excludes(self): variant = "variant1" suites = [SUITE1, SUITE2] exclude_suites = [] suite_list = _create_executor_list(suites, exclude_suites) - task_list = burn_in.create_task_list(self.evergreen_conf, variant, suite_list, - exclude_suites) + task_list = burn_in.create_task_list(EVERGREEN_CONF, variant, suite_list, exclude_suites) self.assertEqual(len(task_list), 1) self.assertIn("task1", task_list) self.assertEqual(task_list["task1"]["resmoke_args"], "--suites=suite1 var1arg1") @@ -863,8 +903,7 @@ class CreateTaskList(unittest.TestCase): suites = [SUITE1, SUITE2, SUITE3] suite_list = _create_executor_list(suites, []) exclude_suites = ["suite2"] - task_list = burn_in.create_task_list(self.evergreen_conf, variant, suite_list, - exclude_suites) + task_list = burn_in.create_task_list(EVERGREEN_CONF, variant, suite_list, exclude_suites) self.assertEqual(len(task_list), 1) self.assertIn("task3", task_list) self.assertEqual(task_list["task3"]["resmoke_args"], "--suites=suite3 var2arg3") @@ -876,8 +915,7 @@ class CreateTaskList(unittest.TestCase): variant = "variant2" suite_list = {} exclude_suites = ["suite2"] - task_list = burn_in.create_task_list(self.evergreen_conf, variant, suite_list, - exclude_suites) + task_list = burn_in.create_task_list(EVERGREEN_CONF, variant, suite_list, exclude_suites) self.assertEqual(len(task_list), 0) self.assertEqual(task_list, {}) @@ -893,7 +931,7 @@ class CreateTaskList(unittest.TestCase): suite_list = _create_executor_list(suites, []) with patch("sys.exit", _raise_bad_variant): with self.assertRaises(BadVariant): - burn_in.create_task_list(self.evergreen_conf, variant, suite_list, []) + burn_in.create_task_list(EVERGREEN_CONF, variant, suite_list, []) class FindChangedTests(unittest.TestCase): diff --git a/etc/evergreen.yml b/etc/evergreen.yml index 9b4cce36b25..41680992205 100644 --- a/etc/evergreen.yml +++ b/etc/evergreen.yml @@ -305,13 +305,13 @@ variables: tasks: - name: compile_all_run_unittests_TG requires: - - name: burn_in_tests + - name: burn_in_tests_gen distros: - windows-64-vs2017-compile - name: compile_benchmarks distros: - windows-64-vs2017-compile - - name: burn_in_tests + - name: burn_in_tests_gen - name: aggregation_multiversion_fuzzer_gen - name: aggregation_wildcard_fuzzer_gen - name: audit @@ -3317,7 +3317,7 @@ timeout: ## - Non standard jstests, i.e., jstestfuzz ## Note that the task name supports glob patterns. test_lifecycle_excluded_tasks: -- burn_in_tests +- burn_in_tests* - compile* - benchmarks* - dbtest* @@ -4613,6 +4613,10 @@ tasks: diff -w requirements.txt.python2.old requirements.txt.python2.new diff -w requirements.txt.python3.old requirements.txt.python3.new +### This task is deprecated, but left in here in case of need to run burn_in_tests +### instead of the generated task: +### - Rename burn_in_tests_gen task to burn_in_tests_gen_UNUSED +### - Rename burn_in_tests_gen to burn_in_tests on the build variants - <<: *task_template name: burn_in_tests depends_on: @@ -4655,6 +4659,61 @@ tasks: task_path_suffix: /data/multiversion:$HOME resmoke_wrapper: $python buildscripts/burn_in_tests.py --testListFile=jstests/new_tests.json resmoke_args: --repeatSuites=2 +### + +- name: burn_in_tests_gen + commands: + - func: "git get project" + # The repository is cloned in a directory distinct from src for the modified test detection + # because the extraction of the artifacts performed in the 'do setup' causes + # 'git diff --name-only' to see all tests as modified on Windows (git 1.9.5). See SERVER-30634. + vars: + git_project_directory: burn_in_tests_clonedir + - command: shell.exec + params: + working_dir: burn_in_tests_clonedir + shell: bash + script: | + set -o errexit + set -o verbose + mkdir ../src + ${activate_virtualenv} + $python -m pip install -r etc/pip/evgtest-requirements.txt + # If this is a scheduled build, we check for changes against the last scheduled commit. + if [ "${is_patch}" != "true" ]; then + burn_in_args="--checkEvergreen" + fi + # Capture a list of new and modified tests. The expansion macro burn_in_tests_build_variant + # is used to for finding the associated tasks from a different build varaint than the + # burn_in_tests_gen task executes on. + build_variant_opts="--buildVariant=${build_variant}" + if [ -n "${burn_in_tests_build_variant|}" ]; then + build_variant_opts="--buildVariant=${burn_in_tests_build_variant} --runBuildVariant=${build_variant}" + fi + # Increase the burn_in repetition from 2 to 1000 executions or 10 minutes + burn_in_args="$burn_in_args --repeatTestsMin=2 --repeatTestsMax=1000 --repeatTestsSecs=600" + # Evergreen executable is in $HOME. + PATH=$PATH:$HOME $python buildscripts/burn_in_tests.py --branch=${branch_name} $build_variant_opts --distro=${distro_id} --generateTasksFile=../src/burn_in_tests_gen.json --noExec $burn_in_args + - command: archive.targz_pack + params: + target: src/burn_in_tests_gen.tgz + source_dir: src + include: + - burn_in_tests_gen.json + - command: s3.put + params: + aws_key: ${aws_key} + aws_secret: ${aws_secret} + local_file: src/burn_in_tests_gen.tgz + remote_file: ${project}/${build_variant}/${revision}/burn_in_tests_gen/burn_in_tests_gen-${build_id}.tgz + bucket: mciuploads + permissions: public-read + content_type: ${content_type|application/gzip} + display_name: Burn_in_tests Task Config - Execution ${execution} + - command: generate.tasks + params: + files: + - src/burn_in_tests_gen.json - <<: *benchmark_template name: benchmarks_orphaned @@ -7780,10 +7839,10 @@ buildvariants: # configurations to run the tests under. Do not add more tasks to this list. - name: compile_TG requires: - - name: burn_in_tests + - name: burn_in_tests_gen distros: - rhel62-large - - name: burn_in_tests + - name: burn_in_tests_gen - name: linux-64-lsm stepback: false @@ -9451,14 +9510,14 @@ buildvariants: tasks: - name: compile_TG requires: - - name: burn_in_tests + - name: burn_in_tests_gen - name: verify_pip distros: - windows-64-vs2017-compile - name: compile_benchmarks distros: - windows-64-vs2017-compile - - name: burn_in_tests + - name: burn_in_tests_gen - name: verify_pip - name: audit - name: auth_audit_gen @@ -10529,7 +10588,7 @@ buildvariants: tasks: - name: compile_all_run_unittests_TG requires: - - name: burn_in_tests + - name: burn_in_tests_gen - name: verify_pip - name: lint distros: @@ -10539,7 +10598,7 @@ buildvariants: - rhel62-large - name: lint - name: lint_yaml - - name: burn_in_tests + - name: burn_in_tests_gen distros: - rhel62-large - name: verify_pip @@ -10949,10 +11008,10 @@ buildvariants: # not add more tasks to this list. - name: compile_TG requires: - - name: burn_in_tests + - name: burn_in_tests_gen distros: - rhel62-large - - name: burn_in_tests + - name: burn_in_tests_gen - name: enterprise-rhel-62-64-bit-coverage display_name: "~ Enterprise RHEL 6.2 DEBUG Code Coverage" @@ -12998,10 +13057,10 @@ buildvariants: - rhel62-large - name: compile_TG requires: - - name: burn_in_tests + - name: burn_in_tests_gen distros: - rhel62-large - - name: burn_in_tests + - name: burn_in_tests_gen - name: jsCore - name: rhel-62-64-bit-mobile @@ -13149,10 +13208,10 @@ buildvariants: # to this list. - name: compile_TG requires: - - name: burn_in_tests + - name: burn_in_tests_gen distros: - rhel62-large - - name: burn_in_tests + - name: burn_in_tests_gen - name: jsCore_mobile - name: macos-mobile |