diff options
| -rwxr-xr-x | buildscripts/evergreen_generate_resmoke_tasks.py | 162 | ||||
| -rw-r--r-- | buildscripts/tests/test_evergreen_generate_resmoke_tasks.py | 26 | ||||
| -rw-r--r-- | etc/pip/components/resmoke.req | 2 | ||||
| -rw-r--r-- | etc/pip/constraints.txt | 8 |
4 files changed, 140 insertions, 58 deletions
diff --git a/buildscripts/evergreen_generate_resmoke_tasks.py b/buildscripts/evergreen_generate_resmoke_tasks.py index 35a72dbc74d..729c81ecfe9 100755 --- a/buildscripts/evergreen_generate_resmoke_tasks.py +++ b/buildscripts/evergreen_generate_resmoke_tasks.py @@ -60,6 +60,10 @@ ConfigOptions = namedtuple("ConfigOptions", [ "variant", "use_large_distro", "large_distro_name", + "use_multiversion", + "is_patch", + "depends_on", + "requires", ]) @@ -73,6 +77,13 @@ def enable_logging(): ) +def split_if_exists(str_to_split): + """Split the given string on ',' if it is not None.""" + if str_to_split: + return str_to_split.split(',') + return None + + def get_config_options(cmd_line_options, config_file): """ Get the configuration to use for generated tests. @@ -83,6 +94,7 @@ def get_config_options(cmd_line_options, config_file): :param config_file: config file to use. :return: ConfigOptions to use. """ + # pylint: disable=too-many-locals config_file_data = read_config.read_config_file(config_file) fallback_num_sub_suites = read_config.get_config_value( @@ -108,10 +120,18 @@ def get_config_options(cmd_line_options, config_file): config_file_data, default=False) large_distro_name = read_config.get_config_value("large_distro_name", cmd_line_options, config_file_data) + use_multiversion = read_config.get_config_value("use_multiversion", cmd_line_options, + config_file_data) + is_patch = read_config.get_config_value("is_patch", cmd_line_options, config_file_data) + depends_on = split_if_exists( + read_config.get_config_value("depends_on", cmd_line_options, config_file_data)) + requires = split_if_exists( + read_config.get_config_value("requires", cmd_line_options, config_file_data)) return ConfigOptions(fallback_num_sub_suites, max_sub_suites, project, resmoke_args, resmoke_jobs_max, target_resmoke_time, run_multiple_jobs, suite, task, - variant, use_large_distro, large_distro_name) + variant, use_large_distro, large_distro_name, use_multiversion, is_patch, + depends_on, requires) def divide_remaining_tests_among_suites(remaining_tests_runtimes, suites): @@ -173,6 +193,9 @@ def generate_subsuite_file(source_suite_name, target_suite_name, roots=None, exc out.write(HEADER_TEMPLATE.format(file=__file__, suite_file=source_file)) if roots: suite_config['selector']['roots'] = roots + + if 'exclude_files' in suite_config['selector']: + del suite_config['selector']['exclude_files'] if excludes: suite_config['selector']['exclude_files'] = excludes out.write(yaml.dump(suite_config, default_flow_style=False, Dumper=yaml.SafeDumper)) @@ -197,35 +220,38 @@ def prepare_directory_for_suite(directory): os.makedirs(directory) -def generate_evg_config(suites, options): - """Generate evergreen configuration for the given suites.""" - evg_config = Configuration() - - task_names = [] - task_specs = [] +class EvergreenConfigGenerator(object): + """Generate evergreen configurations.""" - def generate_task(sub_suite_name, sub_task_name, max_test_runtime=None, - expected_suite_runtime=None): - """Generate evergreen config for a resmoke task.""" - task_names.append(sub_task_name) - spec = TaskSpec(sub_task_name) - if options.use_large_distro: - spec.distro(options.large_distro_name) - task_specs.append(spec) - task = evg_config.task(sub_task_name) + def __init__(self, suites, options): + """Create new EvergreenConfigGenerator object.""" + self.suites = suites + self.options = options + self.evg_config = Configuration() + self.task_specs = [] + self.task_names = [] + + def _set_task_distro(self, task_spec): + if self.options.use_large_distro and self.options.large_distro_name: + task_spec.distro(self.options.large_distro_name) + + def _get_run_tests_vars(self, suite_file): + variables = { + "resmoke_args": "--suites={0}.yml {1}".format(suite_file, self.options.resmoke_args), + "run_multiple_jobs": self.options.run_multiple_jobs, + "task": self.options.task, + } - target_suite_file = os.path.join(CONFIG_DIR, sub_suite_name) + if self.options.resmoke_jobs_max: + variables["resmoke_jobs_max"] = self.options.resmoke_jobs_max - run_tests_vars = { - "resmoke_args": "--suites={0}.yml {1}".format(target_suite_file, options.resmoke_args), - "run_multiple_jobs": options.run_multiple_jobs, - "task": options.task, - } + if self.options.use_multiversion: + variables["task_path_suffix"] = self.options.use_multiversion - if options.resmoke_jobs_max: - run_tests_vars["resmoke_jobs_max"] = options.resmoke_jobs_max + return variables - commands = [] + @staticmethod + def _add_timeout_command(commands, max_test_runtime, expected_suite_runtime): if max_test_runtime or expected_suite_runtime: cmd_timeout = CmdTimeoutUpdate() if max_test_runtime: @@ -234,26 +260,69 @@ def generate_evg_config(suites, options): cmd_timeout.exec_timeout(int(math.ceil(expected_suite_runtime * 3))) commands.append(cmd_timeout.validate().resolve()) - commands += [ - CommandDefinition().function("do setup"), - CommandDefinition().function("run generated tests").vars(run_tests_vars) - ] - task.dependency(TaskDependency("compile")).commands(commands) + def _add_dependencies(self, task): + task.dependency(TaskDependency("compile")) + if not self.options.is_patch: + # Don't worry about task dependencies in patch builds, only mainline. + if self.options.depends_on: + for dep in self.options.depends_on: + task.dependency(TaskDependency(dep)) + if self.options.requires: + for dep in self.options.requires: + task.requires(TaskDependency(dep)) + + return task + + def _generate_task(self, sub_suite_name, sub_task_name, max_test_runtime=None, + expected_suite_runtime=None): + """Generate evergreen config for a resmoke task.""" + spec = TaskSpec(sub_task_name) + self._set_task_distro(spec) + self.task_specs.append(spec) + + self.task_names.append(sub_task_name) + task = self.evg_config.task(sub_task_name) - for idx, suite in enumerate(suites): - sub_task_name = taskname.name_generated_task(options.task, idx, len(suites), - options.variant) - generate_task(suite.name, sub_task_name, suite.max_runtime, suite.get_runtime()) + target_suite_file = os.path.join(CONFIG_DIR, sub_suite_name) + run_tests_vars = self._get_run_tests_vars(target_suite_file) + + commands = [] + self._add_timeout_command(commands, max_test_runtime, expected_suite_runtime) + commands.append(CommandDefinition().function("do setup")) + if self.options.use_multiversion: + commands.append(CommandDefinition().function("do multiversion setup")) + commands.append(CommandDefinition().function("run generated tests").vars(run_tests_vars)) + + self._add_dependencies(task).commands(commands) + + def _generate_all_tasks(self): + for idx, suite in enumerate(self.suites): + sub_task_name = taskname.name_generated_task(self.options.task, idx, len(self.suites), + self.options.variant) + self._generate_task(suite.name, sub_task_name, suite.max_runtime, suite.get_runtime()) + + # Add the misc suite + misc_suite_name = "{0}_misc".format(self.options.suite) + self._generate_task(misc_suite_name, "{0}_misc_{1}".format(self.options.task, + self.options.variant)) + + def _generate_display_task(self): + dt = DisplayTaskDefinition(self.options.task)\ + .execution_tasks(self.task_names) \ + .execution_task("{0}_gen".format(self.options.task)) + return dt - # Add the misc suite - misc_suite_name = "{0}_misc".format(options.suite) - generate_task(misc_suite_name, "{0}_misc_{1}".format(options.task, options.variant)) + def _generate_variant(self): + self._generate_all_tasks() - dt = DisplayTaskDefinition(options.task).execution_tasks(task_names) \ - .execution_task("{0}_gen".format(options.task)) - evg_config.variant(options.variant).tasks(task_specs).display_task(dt) + self.evg_config.variant(self.options.variant)\ + .tasks(self.task_specs)\ + .display_task(self._generate_display_task()) - return evg_config + def generate_config(self): + """Generate evergreen configuration.""" + self._generate_variant() + return self.evg_config class TestStats(object): @@ -386,8 +455,15 @@ class Main(object): help="Build variant being run against.") parser.add_argument("--use-large-distro", dest="use_large_distro", help="Should subtasks use large distros.") - parser.add_argument("--large_distro_name", dest="large_distro_name", + parser.add_argument("--large-distro-name", dest="large_distro_name", help="Name of large distro.") + parser.add_argument("--use-multiversion", dest="use_multiversion", + help="Task path suffix for multiversion generated tasks.") + parser.add_argument("--is-patch", dest="is_patch", help="Is this part of a patch build.") + parser.add_argument("--depends-on", dest="depends_on", + help="Generate depends on for these tasks.") + parser.add_argument("--requires", dest="requires", + help="Generate requires for these tasks.") parser.add_argument("--verbose", dest="verbose", action="store_true", default=False, help="Enable verbose logging.") @@ -451,7 +527,7 @@ class Main(object): def write_evergreen_configuration(self, suites, task): """Generate the evergreen configuration for the new suite and write it to disk.""" - evg_config = generate_evg_config(suites, self.config_options) + evg_config = EvergreenConfigGenerator(suites, self.config_options).generate_config() with open(os.path.join(CONFIG_DIR, task + ".json"), "w") as file_handle: file_handle.write(evg_config.to_json()) diff --git a/buildscripts/tests/test_evergreen_generate_resmoke_tasks.py b/buildscripts/tests/test_evergreen_generate_resmoke_tasks.py index dd4db4f34a9..ccf33ca66e6 100644 --- a/buildscripts/tests/test_evergreen_generate_resmoke_tasks.py +++ b/buildscripts/tests/test_evergreen_generate_resmoke_tasks.py @@ -262,7 +262,7 @@ class PrepareDirectoryForSuite(unittest.TestCase): mock_os.makedirs.assert_called_once_with('tmp') -class GenerateEvgConfigTest(unittest.TestCase): +class EvergreenConfigGeneratorTest(unittest.TestCase): @staticmethod def generate_mock_suites(count): suites = [] @@ -284,6 +284,8 @@ class GenerateEvgConfigTest(unittest.TestCase): options.suite = "suite" options.task = "suite" options.use_large_distro = None + options.use_multiversion = False + options.is_patch = True return options @@ -291,7 +293,7 @@ class GenerateEvgConfigTest(unittest.TestCase): options = self.generate_mock_options() suites = self.generate_mock_suites(3) - config = grt.generate_evg_config(suites, options).to_map() + config = grt.EvergreenConfigGenerator(suites, options).generate_config().to_map() self.assertEqual(len(config["tasks"]), len(suites) + 1) command1 = config["tasks"][0]["commands"][2] @@ -304,7 +306,7 @@ class GenerateEvgConfigTest(unittest.TestCase): options.task = "task" suites = self.generate_mock_suites(3) - config = grt.generate_evg_config(suites, options).to_map() + config = grt.EvergreenConfigGenerator(suites, options).generate_config().to_map() self.assertEqual(len(config["tasks"]), len(suites) + 1) display_task = config["buildvariants"][0]["display_tasks"][0] @@ -325,7 +327,7 @@ class GenerateEvgConfigTest(unittest.TestCase): suites = self.generate_mock_suites(3) - config = grt.generate_evg_config(suites, options).to_map() + config = grt.EvergreenConfigGenerator(suites, options).generate_config().to_map() self.assertEqual(len(config["tasks"]), len(suites) + 1) self.assertEqual(options.large_distro_name, @@ -333,6 +335,13 @@ class GenerateEvgConfigTest(unittest.TestCase): class MainTest(unittest.TestCase): + @staticmethod + def get_mock_options(): + options = Mock() + options.target_resmoke_time = 10 + options.fallback_num_sub_suites = 2 + return options + def test_calculate_suites(self): evg = Mock() evg.test_stats.return_value = [{ @@ -341,8 +350,7 @@ class MainTest(unittest.TestCase): main = grt.Main(evg) main.options = Mock() - main.config_options = grt.ConfigOptions(2, 15, "project", "", 1, 10, True, "task", "suite", - "variant", False, "") + main.config_options = self.get_mock_options() with patch('os.path.exists') as exists_mock: exists_mock.return_value = True @@ -362,8 +370,7 @@ class MainTest(unittest.TestCase): main = grt.Main(evg) main.options = Mock() main.options.execution_time_minutes = 10 - main.config_options = grt.ConfigOptions(2, 15, "project", "", 1, 30, True, "task", "suite", - "variant", False, "") + main.config_options = self.get_mock_options() main.list_tests = Mock(return_value=["test{}.js".format(i) for i in range(100)]) suites = main.calculate_suites(_DATE, _DATE) @@ -381,8 +388,7 @@ class MainTest(unittest.TestCase): main = grt.Main(evg) main.options = Mock() main.options.execution_time_minutes = 10 - main.config_options = grt.ConfigOptions(2, 15, "project", "", 1, 30, True, "task", "suite", - "variant", False, "") + main.config_options = self.get_mock_options() main.list_tests = Mock(return_value=["test{}.js".format(i) for i in range(100)]) with self.assertRaises(requests.HTTPError): diff --git a/etc/pip/components/resmoke.req b/etc/pip/components/resmoke.req index 8417ea6b721..f9bd3ae9adc 100644 --- a/etc/pip/components/resmoke.req +++ b/etc/pip/components/resmoke.req @@ -1,4 +1,4 @@ mock; python_version < "3" PyKMIP == 0.4.0; python_version < "3" # It's now 0.8.0. We're far enough back to have API conflicts. jinja2 -shrub.py
\ No newline at end of file +shrub.py == 0.2.0
\ No newline at end of file diff --git a/etc/pip/constraints.txt b/etc/pip/constraints.txt index 86d48613080..db5f2a5bd97 100644 --- a/etc/pip/constraints.txt +++ b/etc/pip/constraints.txt @@ -7,8 +7,8 @@ # Common requirements asn1crypto==0.24.0 astroid==1.6.5 -boto3==1.9.65 -botocore==1.12.65 +boto3==1.9.69 +botocore==1.12.69 certifi==2018.11.29 cffi==1.11.5 chardet==3.0.4 @@ -39,7 +39,7 @@ requests-oauth==0.4.1 requests-oauthlib==1.0.0 requests-toolbelt==0.8.0 s3transfer==0.1.13 -shrub.py==0.1.0 +shrub.py==0.2.0 six==1.12.0 snowballstemmer==1.2.1 typing==3.6.6 @@ -62,7 +62,7 @@ singledispatch==3.4.0.3; python_version < "3" # Python3 requirements mypy==0.580; python_version > "3" -typed-ast==1.1.0; python_version > "3" +typed-ast==1.1.1; python_version > "3" # Platform-specific components pypiwin32==219; sys_platform == "win32" and python_version < "3" |