diff options
| -rwxr-xr-x | buildscripts/evergreen_generate_resmoke_tasks.py | 49 | ||||
| -rw-r--r-- | buildscripts/tests/test_evergreen_generate_resmoke_tasks.py | 19 | ||||
| -rw-r--r-- | etc/evergreen.yml | 124 |
3 files changed, 156 insertions, 36 deletions
diff --git a/buildscripts/evergreen_generate_resmoke_tasks.py b/buildscripts/evergreen_generate_resmoke_tasks.py index 9ecd1e10f5d..a0d1893cdcf 100755 --- a/buildscripts/evergreen_generate_resmoke_tasks.py +++ b/buildscripts/evergreen_generate_resmoke_tasks.py @@ -51,22 +51,23 @@ HEADER_TEMPLATE = """# DO NOT EDIT THIS FILE. All manual edits will be lost. """ ConfigOptions = namedtuple("ConfigOptions", [ + "build_id", + "depends_on", "fallback_num_sub_suites", + "is_patch", + "large_distro_name", "max_sub_suites", "project", + "repeat_suites", "resmoke_args", "resmoke_jobs_max", - "target_resmoke_time", "run_multiple_jobs", "suite", + "target_resmoke_time", "task", - "variant", "use_large_distro", - "large_distro_name", "use_multiversion", - "is_patch", - "depends_on", - "build_id", + "variant", ]) @@ -132,11 +133,14 @@ def get_config_options(cmd_line_options, config_file): depends_on = split_if_exists( read_config.get_config_value("depends_on", cmd_line_options, config_file_data)) build_id = read_config.get_config_value("build_id", cmd_line_options, config_file_data) + repeat_suites = int( + read_config.get_config_value("resmoke_repeat_suites", cmd_line_options, config_file_data, + default=1)) - return ConfigOptions(fallback_num_sub_suites, max_sub_suites, project, resmoke_args, - resmoke_jobs_max, target_resmoke_time, run_multiple_jobs, suite, task, - variant, use_large_distro, large_distro_name, use_multiversion, is_patch, - depends_on, build_id) + return ConfigOptions(build_id, depends_on, fallback_num_sub_suites, is_patch, large_distro_name, + max_sub_suites, project, repeat_suites, resmoke_args, resmoke_jobs_max, + run_multiple_jobs, suite, target_resmoke_time, task, use_large_distro, + use_multiversion, variant) def divide_remaining_tests_among_suites(remaining_tests_runtimes, suites): @@ -281,9 +285,16 @@ class EvergreenConfigGenerator(object): if self.options.use_large_distro and self.options.large_distro_name: task_spec.distro(self.options.large_distro_name) + def _generate_resmoke_args(self, suite_file): + resmoke_args = "--suite={0}.yml {1}".format(suite_file, self.options.resmoke_args) + if self.options.repeat_suites and "repeatSuites" not in resmoke_args: + resmoke_args += " --repeatSuites={0} ".format(self.options.repeat_suites) + + return resmoke_args + def _get_run_tests_vars(self, suite_file): variables = { - "resmoke_args": "--suites={0}.yml {1}".format(suite_file, self.options.resmoke_args), + "resmoke_args": self._generate_resmoke_args(suite_file), "run_multiple_jobs": self.options.run_multiple_jobs, "task": self.options.task, } @@ -296,14 +307,20 @@ class EvergreenConfigGenerator(object): return variables - @staticmethod - def _add_timeout_command(commands, max_test_runtime, expected_suite_runtime): + def _add_timeout_command(self, commands, max_test_runtime, expected_suite_runtime): + repeat_factor = self.options.repeat_suites if max_test_runtime or expected_suite_runtime: cmd_timeout = CmdTimeoutUpdate() if max_test_runtime: - cmd_timeout.timeout(calculate_timeout(max_test_runtime, 3)) + timeout = calculate_timeout(max_test_runtime, 3) * repeat_factor + LOGGER.debug("Setting timeout to: %d (max=%d, repeat=%d)", timeout, + max_test_runtime, repeat_factor) + cmd_timeout.timeout(timeout) if expected_suite_runtime: - cmd_timeout.exec_timeout(calculate_timeout(expected_suite_runtime, 3)) + exec_timeout = calculate_timeout(expected_suite_runtime, 3) * repeat_factor + LOGGER.debug("Setting exec_timeout to: %d (runtime=%d, repeat=%d)", exec_timeout, + expected_suite_runtime, repeat_factor) + cmd_timeout.exec_timeout(exec_timeout) commands.append(cmd_timeout.validate().resolve()) @staticmethod @@ -523,6 +540,8 @@ class Main(object): parser.add_argument("--is-patch", dest="is_patch", help="Is this part of a patch build.") parser.add_argument("--depends-on", dest="depends_on", help="Generate depends on for these tasks.") + parser.add_argument("--repeat-suites", dest="resmoke_repeat_suites", + help="Repeat each suite the specified number of times.") parser.add_argument("--verbose", dest="verbose", action="store_true", default=False, help="Enable verbose logging.") diff --git a/buildscripts/tests/test_evergreen_generate_resmoke_tasks.py b/buildscripts/tests/test_evergreen_generate_resmoke_tasks.py index af6fdadf9d1..dbac0cd1802 100644 --- a/buildscripts/tests/test_evergreen_generate_resmoke_tasks.py +++ b/buildscripts/tests/test_evergreen_generate_resmoke_tasks.py @@ -335,6 +335,7 @@ class EvergreenConfigGeneratorTest(unittest.TestCase): options.use_large_distro = None options.use_multiversion = False options.is_patch = True + options.repeat_suites = 1 return options @@ -434,6 +435,24 @@ class EvergreenConfigGeneratorTest(unittest.TestCase): cfg_generator._add_dependencies(cfg_mock) self.assertEqual(4, cfg_mock.dependency.call_count) + def test_evg_config_has_timeouts_for_repeated_suites(self): + options = self.generate_mock_options() + options.repeat_suites = 5 + suites = self.generate_mock_suites(3) + + config = grt.EvergreenConfigGenerator(suites, options, Mock()).generate_config().to_map() + + self.assertEqual(len(config["tasks"]), len(suites) + 1) + command1 = config["tasks"][0]["commands"][2] + self.assertIn(" --repeatSuites=5 ", command1["vars"]["resmoke_args"]) + self.assertIn(options.resmoke_args, command1["vars"]["resmoke_args"]) + timeout_cmd = config["tasks"][0]["commands"][0] + self.assertEqual("timeout.update", timeout_cmd["command"]) + expected_timeout = grt.calculate_timeout(suites[0].max_runtime, 3) * 5 + self.assertEqual(expected_timeout, timeout_cmd["params"]["timeout_secs"]) + expected_exec_timeout = grt.calculate_timeout(suites[0].get_runtime(), 3) * 5 + self.assertEqual(expected_exec_timeout, timeout_cmd["params"]["exec_timeout_secs"]) + class NormalizeTestNameTest(unittest.TestCase): def test_unix_names(self): diff --git a/etc/evergreen.yml b/etc/evergreen.yml index d2f87cb0669..49b0e1e8afb 100644 --- a/etc/evergreen.yml +++ b/etc/evergreen.yml @@ -5977,6 +5977,18 @@ tasks: resmoke_args: --suites=parallel --storageEngine=wiredTiger resmoke_jobs_max: 1 +- name: parallel_gen + depends_on: + - name: jsCore + commands: + - func: "generate resmoke tasks" + vars: + task: parallel + depends_on: jsCore + resmoke_args: --storageEngine=wiredTiger + fallback_num_sub_suites: 1 + resmoke_jobs_max: 1 + - <<: *task_template name: parallel_compatibility depends_on: @@ -6706,6 +6718,17 @@ tasks: vars: resmoke_args: --suites=logical_session_cache_sharding_default_refresh_jscore_passthrough --storageEngine=wiredTiger +- name: logical_session_cache_sharding_default_refresh_jscore_passthrough_gen + depends_on: + - name: jsCore + commands: + - func: "generate resmoke tasks" + vars: + task: logical_session_cache_sharding_default_refresh_jscore_passthrough + depends_on: jsCore + resmoke_args: --storageEngine=wiredTiger + fallback_num_sub_suites: 1 + - <<: *task_template name: logical_session_cache_sharding_100ms_refresh_jscore_passthrough depends_on: @@ -6716,6 +6739,17 @@ tasks: vars: resmoke_args: --suites=logical_session_cache_sharding_100ms_refresh_jscore_passthrough --storageEngine=wiredTiger +- name: logical_session_cache_sharding_100ms_refresh_jscore_passthrough_gen + depends_on: + - name: jsCore + commands: + - func: "generate resmoke tasks" + vars: + task: logical_session_cache_sharding_100ms_refresh_jscore_passthrough + depends_on: jsCore + resmoke_args: --storageEngine=wiredTiger + fallback_num_sub_suites: 1 + - <<: *task_template name: logical_session_cache_sharding_1sec_refresh_jscore_passthrough depends_on: @@ -6747,6 +6781,17 @@ tasks: vars: resmoke_args: --suites=logical_session_cache_sharding_10sec_refresh_jscore_passthrough --storageEngine=wiredTiger +- name: logical_session_cache_sharding_10sec_refresh_jscore_passthrough_gen + depends_on: + - name: jsCore + commands: + - func: "generate resmoke tasks" + vars: + task: logical_session_cache_sharding_10sec_refresh_jscore_passthrough + depends_on: jsCore + resmoke_args: --storageEngine=wiredTiger + fallback_num_sub_suites: 1 + - <<: *task_template name: logical_session_cache_standalone_default_refresh_jscore_passthrough depends_on: @@ -6757,6 +6802,17 @@ tasks: vars: resmoke_args: --suites=logical_session_cache_standalone_default_refresh_jscore_passthrough --storageEngine=wiredTiger +- name: logical_session_cache_standalone_default_refresh_jscore_passthrough_gen + depends_on: + - name: jsCore + commands: + - func: "generate resmoke tasks" + vars: + task: logical_session_cache_standalone_default_refresh_jscore_passthrough + depends_on: jsCore + resmoke_args: --storageEngine=wiredTiger + fallback_num_sub_suites: 1 + - <<: *task_template name: logical_session_cache_standalone_100ms_refresh_jscore_passthrough depends_on: @@ -6767,6 +6823,17 @@ tasks: vars: resmoke_args: --suites=logical_session_cache_standalone_100ms_refresh_jscore_passthrough --storageEngine=wiredTiger +- name: logical_session_cache_standalone_100ms_refresh_jscore_passthrough_gen + depends_on: + - name: jsCore + commands: + - func: "generate resmoke tasks" + vars: + task: logical_session_cache_standalone_100ms_refresh_jscore_passthrough + depends_on: jsCore + resmoke_args: --storageEngine=wiredTiger + fallback_num_sub_suites: 1 + - <<: *task_template name: logical_session_cache_standalone_1sec_refresh_jscore_passthrough depends_on: @@ -6798,6 +6865,17 @@ tasks: vars: resmoke_args: --suites=logical_session_cache_standalone_10sec_refresh_jscore_passthrough --storageEngine=wiredTiger +- name: logical_session_cache_standalone_10sec_refresh_jscore_passthrough_gen + depends_on: + - name: jsCore + commands: + - func: "generate resmoke tasks" + vars: + task: logical_session_cache_standalone_10sec_refresh_jscore_passthrough + depends_on: jsCore + resmoke_args: --storageEngine=wiredTiger + fallback_num_sub_suites: 1 + - <<: *task_template name: retryable_writes_jscore_stepdown_passthrough depends_on: @@ -7701,6 +7779,10 @@ buildvariants: --excludeWithAnyTags=requires_http_client --repeatSuites=10 --shuffle + resmoke_repeat_suites: 10 + # TODO: There are currently 2 ways of repeating suites, one for regular suites and one for + # generated suites. Once everything is converted to generated suites, we should remove the + # '--repeatSuites=10' from the test_flags. This will be done in SERVER-38817. scons_cache_scope: shared tooltags: "" build_mongoreplay: true @@ -7716,19 +7798,19 @@ buildvariants: - name: integration_tests_sharded - name: jsCore - name: jsCore_txns - - name: logical_session_cache_replication_100ms_refresh_jscore_passthrough - - name: logical_session_cache_replication_1sec_refresh_jscore_passthrough - - name: logical_session_cache_replication_10sec_refresh_jscore_passthrough - - name: logical_session_cache_replication_default_refresh_jscore_passthrough - - name: logical_session_cache_sharding_100ms_refresh_jscore_passthrough - - name: logical_session_cache_sharding_1sec_refresh_jscore_passthrough - - name: logical_session_cache_sharding_10sec_refresh_jscore_passthrough - - name: logical_session_cache_sharding_default_refresh_jscore_passthrough - - name: logical_session_cache_standalone_100ms_refresh_jscore_passthrough - - name: logical_session_cache_standalone_1sec_refresh_jscore_passthrough - - name: logical_session_cache_standalone_10sec_refresh_jscore_passthrough - - name: logical_session_cache_standalone_default_refresh_jscore_passthrough - - name: parallel + - name: logical_session_cache_replication_100ms_refresh_jscore_passthrough_gen + - name: logical_session_cache_replication_1sec_refresh_jscore_passthrough_gen + - name: logical_session_cache_replication_10sec_refresh_jscore_passthrough_gen + - name: logical_session_cache_replication_default_refresh_jscore_passthrough_gen + - name: logical_session_cache_sharding_100ms_refresh_jscore_passthrough_gen + - name: logical_session_cache_sharding_1sec_refresh_jscore_passthrough_gen + - name: logical_session_cache_sharding_10sec_refresh_jscore_passthrough_gen + - name: logical_session_cache_sharding_default_refresh_jscore_passthrough_gen + - name: logical_session_cache_standalone_100ms_refresh_jscore_passthrough_gen + - name: logical_session_cache_standalone_1sec_refresh_jscore_passthrough_gen + - name: logical_session_cache_standalone_10sec_refresh_jscore_passthrough_gen + - name: logical_session_cache_standalone_default_refresh_jscore_passthrough_gen + - name: parallel_gen - name: concurrency - name: concurrency_replication distros: @@ -7743,17 +7825,17 @@ buildvariants: distros: - rhel62-large #- name: concurrency_sharded_replication_local_read_write_multi_stmt_txn # Removed until SERVER-38499 is resolved. - distros: - - rhel62-large + # distros: + # - rhel62-large #- name: concurrency_sharded_replication_local_read_write_multi_stmt_txn_with_balancer # Removed until SERVER-38499 is resolved. - distros: - - rhel62-large + # distros: + # - rhel62-large #- name: concurrency_sharded_replication_multi_stmt_txn # Removed until SERVER-38499 is resolved. - distros: - - rhel62-large + # distros: + # - rhel62-large #- name: concurrency_sharded_replication_multi_stmt_txn_with_balancer # Removed until SERVER-38499 is resolved. - distros: - - rhel62-large + # distros: + # - rhel62-large - name: concurrency_sharded_replication_with_balancer distros: - rhel62-large |