diff options
author | David Bradford <david.bradford@mongodb.com> | 2018-12-14 15:30:39 -0500 |
---|---|---|
committer | David Bradford <david.bradford@mongodb.com> | 2018-12-14 15:30:39 -0500 |
commit | dcc81686d19bce6a7795ae440092f78579aebf91 (patch) | |
tree | 304681993f262dffea3f939e29c1c8b3fde206ac | |
parent | ac39ac47d776bbd0f442f945d4ae6091b73cfb03 (diff) | |
download | mongo-dcc81686d19bce6a7795ae440092f78579aebf91.tar.gz |
SERVER-38112: Add ability to dynamically split up long running tasks in evergreen
-rwxr-xr-x | buildscripts/evergreen_generate_resmoke_tasks.py (renamed from buildscripts/generate_resmoke_suites.py) | 62 | ||||
-rw-r--r-- | buildscripts/tests/test_evergreen_generate_resmoke_tasks.py (renamed from buildscripts/tests/test_generate_resmoke_suites.py) | 115 | ||||
-rw-r--r-- | etc/evergreen.yml | 435 |
3 files changed, 378 insertions, 234 deletions
diff --git a/buildscripts/generate_resmoke_suites.py b/buildscripts/evergreen_generate_resmoke_tasks.py index 063be009840..35a72dbc74d 100755 --- a/buildscripts/generate_resmoke_suites.py +++ b/buildscripts/evergreen_generate_resmoke_tasks.py @@ -11,6 +11,7 @@ from __future__ import absolute_import import argparse import datetime import logging +import math import os import sys from collections import defaultdict @@ -21,6 +22,7 @@ import yaml from shrub.config import Configuration from shrub.command import CommandDefinition +from shrub.operations import CmdTimeoutUpdate from shrub.task import TaskDependency from shrub.variant import DisplayTaskDefinition from shrub.variant import TaskSpec @@ -51,10 +53,13 @@ ConfigOptions = namedtuple("ConfigOptions", [ "project", "resmoke_args", "resmoke_jobs_max", + "target_resmoke_time", "run_multiple_jobs", "suite", "task", "variant", + "use_large_distro", + "large_distro_name", ]) @@ -90,15 +95,23 @@ def get_config_options(cmd_line_options, config_file): default="") resmoke_jobs_max = read_config.get_config_value("resmoke_jobs_max", cmd_line_options, config_file_data) + target_resmoke_time = int( + read_config.get_config_value("target_resmoke_time", cmd_line_options, config_file_data, + default=60)) run_multiple_jobs = read_config.get_config_value("run_multiple_jobs", cmd_line_options, config_file_data, default="true") task = read_config.get_config_value("task", cmd_line_options, config_file_data, required=True) suite = read_config.get_config_value("suite", cmd_line_options, config_file_data, default=task) variant = read_config.get_config_value("build_variant", cmd_line_options, config_file_data, required=True) + use_large_distro = read_config.get_config_value("use_large_distro", cmd_line_options, + config_file_data, default=False) + large_distro_name = read_config.get_config_value("large_distro_name", cmd_line_options, + config_file_data) return ConfigOptions(fallback_num_sub_suites, max_sub_suites, project, resmoke_args, - resmoke_jobs_max, run_multiple_jobs, suite, task, variant) + resmoke_jobs_max, target_resmoke_time, run_multiple_jobs, suite, task, + variant, use_large_distro, large_distro_name) def divide_remaining_tests_among_suites(remaining_tests_runtimes, suites): @@ -191,32 +204,46 @@ def generate_evg_config(suites, options): task_names = [] task_specs = [] - def generate_task(sub_suite_name, sub_task_name): + def generate_task(sub_suite_name, sub_task_name, max_test_runtime=None, + expected_suite_runtime=None): """Generate evergreen config for a resmoke task.""" task_names.append(sub_task_name) - task_specs.append(TaskSpec(sub_task_name)) + spec = TaskSpec(sub_task_name) + if options.use_large_distro: + spec.distro(options.large_distro_name) + task_specs.append(spec) task = evg_config.task(sub_task_name) target_suite_file = os.path.join(CONFIG_DIR, sub_suite_name) run_tests_vars = { - "resmoke_args": "--suites={0} {1}".format(target_suite_file, options.resmoke_args), + "resmoke_args": "--suites={0}.yml {1}".format(target_suite_file, options.resmoke_args), "run_multiple_jobs": options.run_multiple_jobs, + "task": options.task, } if options.resmoke_jobs_max: run_tests_vars["resmoke_jobs_max"] = options.resmoke_jobs_max - commands = [ + commands = [] + if max_test_runtime or expected_suite_runtime: + cmd_timeout = CmdTimeoutUpdate() + if max_test_runtime: + cmd_timeout.timeout(int(math.ceil(max_test_runtime * 3))) + if expected_suite_runtime: + cmd_timeout.exec_timeout(int(math.ceil(expected_suite_runtime * 3))) + commands.append(cmd_timeout.validate().resolve()) + + commands += [ CommandDefinition().function("do setup"), - CommandDefinition().function("run tests").vars(run_tests_vars) + CommandDefinition().function("run generated tests").vars(run_tests_vars) ] task.dependency(TaskDependency("compile")).commands(commands) for idx, suite in enumerate(suites): sub_task_name = taskname.name_generated_task(options.task, idx, len(suites), options.variant) - generate_task(suite.name, sub_task_name) + generate_task(suite.name, sub_task_name, suite.max_runtime, suite.get_runtime()) # Add the misc suite misc_suite_name = "{0}_misc".format(options.suite) @@ -298,6 +325,7 @@ class Suite(object): """Initialize the object.""" self.tests = [] self.total_runtime = 0 + self.max_runtime = 0 def add_test(self, test_file, runtime): """Add the given test to this suite.""" @@ -305,6 +333,9 @@ class Suite(object): self.tests.append(test_file) self.total_runtime += runtime + if runtime > self.max_runtime: + self.max_runtime = runtime + def get_runtime(self): """Get the current average runtime of all the tests currently in this suite.""" @@ -334,7 +365,7 @@ class Main(object): help="Location of expansions file generated by evergreen.") parser.add_argument("--analysis-duration", dest="duration_days", default=14, help="Number of days to analyze.", type=int) - parser.add_argument("--execution-time", dest="execution_time_minutes", default=60, type=int, + parser.add_argument("--execution-time", dest="target_resmoke_time", type=int, help="Target execution time (in minutes).") parser.add_argument("--max-sub-suites", dest="max_sub_suites", type=int, help="Max number of suites to divide into.") @@ -353,6 +384,10 @@ class Main(object): parser.add_argument("--task-name", dest="task", help="Name of task to split.") parser.add_argument("--variant", dest="build_variant", help="Build variant being run against.") + parser.add_argument("--use-large-distro", dest="use_large_distro", + help="Should subtasks use large distros.") + parser.add_argument("--large_distro_name", dest="large_distro_name", + help="Name of large distro.") parser.add_argument("--verbose", dest="verbose", action="store_true", default=False, help="Enable verbose logging.") @@ -366,8 +401,8 @@ class Main(object): try: evg_stats = self.get_evg_stats(self.config_options.project, start_date, end_date, self.config_options.task, self.config_options.variant) - return self.calculate_suites_from_evg_stats(evg_stats, - self.options.execution_time_minutes * 60) + target_execution_time_secs = self.config_options.target_resmoke_time * 60 + return self.calculate_suites_from_evg_stats(evg_stats, target_execution_time_secs) except requests.HTTPError as err: if err.response.status_code == requests.codes.SERVICE_UNAVAILABLE: # Evergreen may return a 503 when the service is degraded. @@ -391,11 +426,16 @@ class Main(object): def calculate_suites_from_evg_stats(self, data, execution_time_secs): """Divide tests into suites that can be run in less than the specified execution time.""" test_stats = TestStats(data) - tests_runtimes = test_stats.get_tests_runtimes() + tests_runtimes = self.filter_existing_tests(test_stats.get_tests_runtimes()) self.test_list = [info[0] for info in tests_runtimes] return divide_tests_into_suites(tests_runtimes, execution_time_secs, self.options.max_sub_suites) + @staticmethod + def filter_existing_tests(tests_runtimes): + """Filter out tests that do not exist in the filesystem.""" + return [info for info in tests_runtimes if os.path.exists(info[0])] + def calculate_fallback_suites(self): """Divide tests into a fixed number of suites.""" num_suites = self.config_options.fallback_num_sub_suites diff --git a/buildscripts/tests/test_generate_resmoke_suites.py b/buildscripts/tests/test_evergreen_generate_resmoke_tasks.py index a6e78ce1d30..dd4db4f34a9 100644 --- a/buildscripts/tests/test_generate_resmoke_suites.py +++ b/buildscripts/tests/test_evergreen_generate_resmoke_tasks.py @@ -12,8 +12,8 @@ import yaml from mock import patch, mock_open, call, Mock -from buildscripts import generate_resmoke_suites as grs -from buildscripts.generate_resmoke_suites import render_suite, render_misc_suite, \ +from buildscripts import evergreen_generate_resmoke_tasks as grt +from buildscripts.evergreen_generate_resmoke_tasks import render_suite, render_misc_suite, \ prepare_directory_for_suite # pylint: disable=missing-docstring,invalid-name,unused-argument,no-self-use @@ -28,7 +28,7 @@ class TestTestStats(unittest.TestCase): self._make_evg_result("dir/test2.js", 1, 30), self._make_evg_result("dir/test1.js", 2, 25), ] - test_stats = grs.TestStats(evg_results) + test_stats = grt.TestStats(evg_results) expected_runtimes = [ ("dir/test2.js", 30), ("dir/test1.js", 20), @@ -44,7 +44,7 @@ class TestTestStats(unittest.TestCase): self._make_evg_result("test3:CleanEveryN", 10, 30), self._make_evg_result("test3:CheckReplDBHash", 10, 35), ] - test_stats = grs.TestStats(evg_results) + test_stats = grt.TestStats(evg_results) expected_runtimes = [ ("dir/test3.js", 42.5), ("dir/test2.js", 30), @@ -78,19 +78,19 @@ class DivideRemainingTestsAmongSuitesTest(unittest.TestCase): return tests_runtimes def test_each_suite_gets_one_test(self): - suites = [grs.Suite(), grs.Suite(), grs.Suite()] + suites = [grt.Suite(), grt.Suite(), grt.Suite()] tests_runtimes = self.generate_tests_runtimes(3) - grs.divide_remaining_tests_among_suites(tests_runtimes, suites) + grt.divide_remaining_tests_among_suites(tests_runtimes, suites) for suite in suites: self.assertEqual(suite.get_test_count(), 1) def test_each_suite_gets_at_least_one_test(self): - suites = [grs.Suite(), grs.Suite(), grs.Suite()] + suites = [grt.Suite(), grt.Suite(), grt.Suite()] tests_runtimes = self.generate_tests_runtimes(5) - grs.divide_remaining_tests_among_suites(tests_runtimes, suites) + grt.divide_remaining_tests_among_suites(tests_runtimes, suites) total_tests = 0 for suite in suites: @@ -109,7 +109,7 @@ class DivideTestsIntoSuitesByMaxtimeTest(unittest.TestCase): ("test3", 3), ] - suites = grs.divide_tests_into_suites(tests_runtimes, max_time) + suites = grt.divide_tests_into_suites(tests_runtimes, max_time) self.assertEqual(len(suites), 1) self.assertEqual(suites[0].get_test_count(), 3) self.assertEqual(suites[0].get_runtime(), 12) @@ -122,7 +122,7 @@ class DivideTestsIntoSuitesByMaxtimeTest(unittest.TestCase): ("test3", 3), ] - suites = grs.divide_tests_into_suites(tests_runtimes, max_time) + suites = grt.divide_tests_into_suites(tests_runtimes, max_time) self.assertEqual(len(suites), 3) def test_if_test_is_greater_than_max_it_goes_alone(self): @@ -133,7 +133,7 @@ class DivideTestsIntoSuitesByMaxtimeTest(unittest.TestCase): ("test3", 3), ] - suites = grs.divide_tests_into_suites(tests_runtimes, max_time) + suites = grt.divide_tests_into_suites(tests_runtimes, max_time) self.assertEqual(len(suites), 2) self.assertEqual(suites[0].get_test_count(), 1) self.assertEqual(suites[0].get_runtime(), 15) @@ -149,7 +149,7 @@ class DivideTestsIntoSuitesByMaxtimeTest(unittest.TestCase): ("test5", 3), ] - suites = grs.divide_tests_into_suites(tests_runtimes, max_time, max_suites=max_suites) + suites = grt.divide_tests_into_suites(tests_runtimes, max_time, max_suites=max_suites) self.assertEqual(len(suites), max_suites) total_tests = 0 for suite in suites: @@ -159,7 +159,7 @@ class DivideTestsIntoSuitesByMaxtimeTest(unittest.TestCase): class SuiteTest(unittest.TestCase): def test_adding_tests_increases_count_and_runtime(self): - suite = grs.Suite() + suite = grt.Suite() suite.add_test('test1', 10) suite.add_test('test2', 12) suite.add_test('test3', 7) @@ -170,7 +170,7 @@ class SuiteTest(unittest.TestCase): def create_suite(count=3, start=0): """ Create a suite with count tests.""" - suite = grs.Suite() + suite = grt.Suite() for i in range(start, start + count): suite.add_test('test{}'.format(i), 1) return suite @@ -194,7 +194,7 @@ class RenderSuites(unittest.TestCase): ] m = mock_open(read_data=yaml.dump({'selector': {'roots': [], 'excludes': ['fixed']}})) - with patch('buildscripts.generate_resmoke_suites.open', m, create=True): + with patch('buildscripts.evergreen_generate_resmoke_tasks.open', m, create=True): render_suite(suites, 'suite_name') handle = m() @@ -202,11 +202,11 @@ class RenderSuites(unittest.TestCase): self.assertEquals(len(suites) * 2, handle.write.call_count) handle.write.assert_has_calls([call(e) for e in expected], any_order=True) calls = [ - call(os.path.join(grs.TEST_SUITE_DIR, 'suite_name.yml'), 'r') + call(os.path.join(grt.TEST_SUITE_DIR, 'suite_name.yml'), 'r') for _ in range(len(suites)) ] m.assert_has_calls(calls, any_order=True) - filename = os.path.join(grs.CONFIG_DIR, 'suite_name_{{:0{}}}.yml'.format( + filename = os.path.join(grt.CONFIG_DIR, 'suite_name_{{:0{}}}.yml'.format( int(math.ceil(math.log10(size))))) calls = [call(filename.format(i), 'w') for i in range(size)] m.assert_has_calls(calls, any_order=True) @@ -226,7 +226,7 @@ class RenderMiscSuites(unittest.TestCase): test_list = ['test{}'.format(i) for i in range(10)] m = mock_open(read_data=yaml.dump({'selector': {'roots': []}})) - with patch('buildscripts.generate_resmoke_suites.open', m, create=True): + with patch('buildscripts.evergreen_generate_resmoke_tasks.open', m, create=True): render_misc_suite(test_list, 'suite_name') handle = m() @@ -246,16 +246,16 @@ class RenderMiscSuites(unittest.TestCase): - test9 roots: [] """) - calls = [call(os.path.join(grs.TEST_SUITE_DIR, 'suite_name.yml'), 'r')] + calls = [call(os.path.join(grt.TEST_SUITE_DIR, 'suite_name.yml'), 'r')] m.assert_has_calls(calls, any_order=True) - filename = os.path.join(grs.CONFIG_DIR, 'suite_name_misc.yml') + filename = os.path.join(grt.CONFIG_DIR, 'suite_name_misc.yml') calls = [call(filename, 'w')] m.assert_has_calls(calls, any_order=True) class PrepareDirectoryForSuite(unittest.TestCase): def test_no_directory(self): - with patch('buildscripts.generate_resmoke_suites.os') as mock_os: + with patch('buildscripts.evergreen_generate_resmoke_tasks.os') as mock_os: mock_os.path.exists.return_value = False prepare_directory_for_suite('tmp') @@ -269,6 +269,8 @@ class GenerateEvgConfigTest(unittest.TestCase): for idx in range(count): suite = Mock() suite.name = "suite {0}".format(idx) + suite.max_runtime = 5.28 + suite.get_runtime = lambda: 100.874 suites.append(suite) return suites @@ -281,6 +283,7 @@ class GenerateEvgConfigTest(unittest.TestCase): options.variant = "buildvariant" options.suite = "suite" options.task = "suite" + options.use_large_distro = None return options @@ -288,20 +291,20 @@ class GenerateEvgConfigTest(unittest.TestCase): options = self.generate_mock_options() suites = self.generate_mock_suites(3) - config = grs.generate_evg_config(suites, options).to_map() + config = grt.generate_evg_config(suites, options).to_map() self.assertEqual(len(config["tasks"]), len(suites) + 1) - command1 = config["tasks"][0]["commands"][1] + command1 = config["tasks"][0]["commands"][2] self.assertIn(options.resmoke_args, command1["vars"]["resmoke_args"]) self.assertIn(options.run_multiple_jobs, command1["vars"]["run_multiple_jobs"]) - self.assertEqual("run tests", command1["func"]) + self.assertEqual("run generated tests", command1["func"]) def test_evg_config_is_created_with_diff_task_and_suite(self): options = self.generate_mock_options() options.task = "task" suites = self.generate_mock_suites(3) - config = grs.generate_evg_config(suites, options).to_map() + config = grt.generate_evg_config(suites, options).to_map() self.assertEqual(len(config["tasks"]), len(suites) + 1) display_task = config["buildvariants"][0]["display_tasks"][0] @@ -313,7 +316,20 @@ class GenerateEvgConfigTest(unittest.TestCase): task = config["tasks"][0] self.assertIn(options.variant, task["name"]) self.assertIn(task["name"], display_task["execution_tasks"]) - self.assertIn(options.suite, task["commands"][1]["vars"]["resmoke_args"]) + self.assertIn(options.suite, task["commands"][2]["vars"]["resmoke_args"]) + + def test_evg_config_can_use_large_distro(self): + options = self.generate_mock_options() + options.use_large_distro = "true" + options.large_distro_name = "large distro name" + + suites = self.generate_mock_suites(3) + + config = grt.generate_evg_config(suites, options).to_map() + + self.assertEqual(len(config["tasks"]), len(suites) + 1) + self.assertEqual(options.large_distro_name, + config["buildvariants"][0]["tasks"][0]["distros"][0]) class MainTest(unittest.TestCase): @@ -323,18 +339,19 @@ class MainTest(unittest.TestCase): "test_file": "test{}.js".format(i), "avg_duration_pass": 60, "num_pass": 1 } for i in range(100)] - main = grs.Main(evg) + main = grt.Main(evg) main.options = Mock() - main.options.execution_time_minutes = 10 - main.config_options = grs.ConfigOptions(2, 15, "project", "", 1, True, "task", "suite", - "variant") + main.config_options = grt.ConfigOptions(2, 15, "project", "", 1, 10, True, "task", "suite", + "variant", False, "") - suites = main.calculate_suites(_DATE, _DATE) + with patch('os.path.exists') as exists_mock: + exists_mock.return_value = True + suites = main.calculate_suites(_DATE, _DATE) - # There are 100 tests taking 1 minute, with a target of 10 min we expect 10 suites. - self.assertEqual(10, len(suites)) - for suite in suites: - self.assertEqual(10, len(suite.tests)) + # There are 100 tests taking 1 minute, with a target of 10 min we expect 10 suites. + self.assertEqual(10, len(suites)) + for suite in suites: + self.assertEqual(10, len(suite.tests)) def test_calculate_suites_fallback(self): response = Mock() @@ -342,11 +359,11 @@ class MainTest(unittest.TestCase): evg = Mock() evg.test_stats.side_effect = requests.HTTPError(response=response) - main = grs.Main(evg) + main = grt.Main(evg) main.options = Mock() main.options.execution_time_minutes = 10 - main.config_options = grs.ConfigOptions(2, 15, "project", "", 1, True, "task", "suite", - "variant") + main.config_options = grt.ConfigOptions(2, 15, "project", "", 1, 30, True, "task", "suite", + "variant", False, "") main.list_tests = Mock(return_value=["test{}.js".format(i) for i in range(100)]) suites = main.calculate_suites(_DATE, _DATE) @@ -361,12 +378,28 @@ class MainTest(unittest.TestCase): evg = Mock() evg.test_stats.side_effect = requests.HTTPError(response=response) - main = grs.Main(evg) + main = grt.Main(evg) main.options = Mock() main.options.execution_time_minutes = 10 - main.config_options = grs.ConfigOptions(2, 15, "project", "", 1, True, "task", "suite", - "variant") + main.config_options = grt.ConfigOptions(2, 15, "project", "", 1, 30, True, "task", "suite", + "variant", False, "") main.list_tests = Mock(return_value=["test{}.js".format(i) for i in range(100)]) with self.assertRaises(requests.HTTPError): main.calculate_suites(_DATE, _DATE) + + def test_filter_missing_files(self): + tests_runtimes = [ + ("dir1/file1.js", 20.32), + ("dir2/file2.js", 24.32), + ("dir1/file3.js", 36.32), + ] + + with patch("os.path.exists") as exists_mock: + exists_mock.side_effect = [False, True, True] + filtered_list = grt.Main.filter_existing_tests(tests_runtimes) + + self.assertEqual(2, len(filtered_list)) + self.assertNotIn(tests_runtimes[0], filtered_list) + self.assertIn(tests_runtimes[2], filtered_list) + self.assertIn(tests_runtimes[1], filtered_list) diff --git a/etc/evergreen.yml b/etc/evergreen.yml index d8c6133951d..e1e764a9aaf 100644 --- a/etc/evergreen.yml +++ b/etc/evergreen.yml @@ -734,6 +734,20 @@ functions: fi done + "configure evergreen api credentials" : &configure_evergreen_api_credentials + command: shell.exec + type: test + params: + working_dir: src + silent: true + script: | + # Create the Evergreen API credentials + cat > .evergreen.yml <<END_OF_CREDS + api_server_host: https://evergreen.mongodb.com/api + api_key: "${evergreen_api_key}" + user: "${evergreen_api_user}" + END_OF_CREDS + "git get project" : &git_get_project command: git.get_project params: @@ -1467,149 +1481,248 @@ functions: if [[ ! -f $venv2_bin/python2 ]]; then ln -sfv "$py2_exe" "$venv2_bin/python2"; fi if [[ ! -f $venv2_bin/python3 ]]; then ln -sfv "$py3_exe" "$venv2_bin/python3"; fi - "run tests" : - - *install_pip_requirements - - *determine_task_timeout - - *update_task_timeout_expansions - - *update_task_timeout - - command: expansions.update - params: - updates: - - key: aws_key_remote - value: ${mongodatafiles_aws_key} - - key: aws_profile_remote - value: mongodata_aws - - key: aws_secret_remote - value: ${mongodatafiles_aws_secret} - - *set_up_remote_credentials - - *determine_resmoke_jobs - - *update_resmoke_jobs_expansions - - command: shell.exec - type: test - params: - working_dir: src - shell: bash - script: | - set -o errexit - set -o verbose - - if [[ ${disable_unit_tests|false} = "false" && ! -f ${skip_tests|/dev/null} ]]; then + "execute resmoke tests" : &execute_resmoke_tests + command: shell.exec + type: test + params: + working_dir: src + shell: bash + script: | + set -o errexit + set -o verbose - # activate the virtualenv if it has been set up - ${activate_virtualenv} + if [[ ${disable_unit_tests|false} = "false" && ! -f ${skip_tests|/dev/null} ]]; then - # Set the TMPDIR environment variable to be a directory in the task's working - # directory so that temporary files created by processes spawned by resmoke.py get - # cleaned up after the task completes. This also ensures the spawned processes - # aren't impacted by limited space in the mount point for the /tmp directory. - export TMPDIR="${workdir}/tmp" - mkdir -p $TMPDIR + # activate the virtualenv if it has been set up + ${activate_virtualenv} - if [ -f /proc/self/coredump_filter ]; then - # Set the shell process (and its children processes) to dump ELF headers (bit 4), - # anonymous shared mappings (bit 1), and anonymous private mappings (bit 0). - echo 0x13 > /proc/self/coredump_filter - - if [ -f /sbin/sysctl ]; then - # Check that the core pattern is set explicitly on our distro image instead - # of being the OS's default value. This ensures that coredump names are consistent - # across distros and can be picked up by Evergreen. - core_pattern=$(/sbin/sysctl -n "kernel.core_pattern") - if [ "$core_pattern" = "dump_%e.%p.core" ]; then - echo "Enabling coredumps" - ulimit -c unlimited - fi + # Set the TMPDIR environment variable to be a directory in the task's working + # directory so that temporary files created by processes spawned by resmoke.py get + # cleaned up after the task completes. This also ensures the spawned processes + # aren't impacted by limited space in the mount point for the /tmp directory. + export TMPDIR="${workdir}/tmp" + mkdir -p $TMPDIR + + if [ -f /proc/self/coredump_filter ]; then + # Set the shell process (and its children processes) to dump ELF headers (bit 4), + # anonymous shared mappings (bit 1), and anonymous private mappings (bit 0). + echo 0x13 > /proc/self/coredump_filter + + if [ -f /sbin/sysctl ]; then + # Check that the core pattern is set explicitly on our distro image instead + # of being the OS's default value. This ensures that coredump names are consistent + # across distros and can be picked up by Evergreen. + core_pattern=$(/sbin/sysctl -n "kernel.core_pattern") + if [ "$core_pattern" = "dump_%e.%p.core" ]; then + echo "Enabling coredumps" + ulimit -c unlimited fi fi + fi - extra_args="$extra_args --jobs=${resmoke_jobs|1}" + extra_args="$extra_args --jobs=${resmoke_jobs|1}" - if [ ${should_shuffle|true} = true ]; then - extra_args="$extra_args --shuffle" - fi + if [ ${should_shuffle|true} = true ]; then + extra_args="$extra_args --shuffle" + fi - if [ ${continue_on_failure|true} = true ]; then - extra_args="$extra_args --continueOnFailure" - fi + if [ ${continue_on_failure|true} = true ]; then + extra_args="$extra_args --continueOnFailure" + fi - # Default storageEngineCacheSizeGB to 1. Override on individual test config if needed. - # resmoke will assign to the appropriate parameter on storage engines that support it. - set +o errexit - echo "${resmoke_args}" | grep -q storageEngineCacheSizeGB - if [ $? -eq 1 ]; then - extra_args="$extra_args --storageEngineCacheSizeGB=1" - fi - set -o errexit + # Default storageEngineCacheSizeGB to 1. Override on individual test config if needed. + # resmoke will assign to the appropriate parameter on storage engines that support it. + set +o errexit + echo "${resmoke_args}" | grep -q storageEngineCacheSizeGB + if [ $? -eq 1 ]; then + extra_args="$extra_args --storageEngineCacheSizeGB=1" + fi + set -o errexit - # Reduce the JSHeapLimit for the serial_run task task on Code Coverage builder variant. - if [[ "${build_variant}" = "enterprise-rhel-62-64-bit-coverage" && "${task_name}" = "serial_run" ]]; then - extra_args="$extra_args --mongodSetParameter {'jsHeapLimitMB':10}" - fi + # Reduce the JSHeapLimit for the serial_run task task on Code Coverage builder variant. + if [[ "${build_variant}" = "enterprise-rhel-62-64-bit-coverage" && "${task_name}" = "serial_run" ]]; then + extra_args="$extra_args --mongodSetParameter {'jsHeapLimitMB':10}" + fi - path_value="$PATH" - if [ ${variant_path_suffix} ]; then - path_value="$path_value:${variant_path_suffix}" - fi - if [ ${task_path_suffix} ]; then - path_value="$path_value:${task_path_suffix}" - fi + path_value="$PATH" + if [ ${variant_path_suffix} ]; then + path_value="$path_value:${variant_path_suffix}" + fi + if [ ${task_path_suffix} ]; then + path_value="$path_value:${task_path_suffix}" + fi - if [ "${is_patch}" = "true" ]; then - extra_args="$extra_args --tagFile=etc/test_lifecycle.yml --patchBuild" - else - extra_args="$extra_args --tagFile=etc/test_retrial.yml" - fi + if [ "${is_patch}" = "true" ]; then + extra_args="$extra_args --tagFile=etc/test_lifecycle.yml --patchBuild" + else + extra_args="$extra_args --tagFile=etc/test_retrial.yml" + fi - # The "resmoke_wrapper" expansion is used by the 'burn_in_tests' task to wrap the resmoke.py - # invocation. It doesn't set any environment variables and should therefore come last in - # this list of expansions. - set +o errexit - PATH="$path_value" \ - AWS_PROFILE=${aws_profile_remote} \ - ${gcov_environment} \ - ${lang_environment} \ - ${san_options} \ - ${san_symbolizer} \ - ${snmp_config_path} \ - ${resmoke_wrapper} \ - $python buildscripts/evergreen_run_tests.py \ - ${resmoke_args} \ - $extra_args \ - ${test_flags} \ - --log=buildlogger \ - --staggerJobs=on \ - --buildId=${build_id} \ - --distroId=${distro_id} \ - --executionNumber=${execution} \ - --projectName=${project} \ - --gitRevision=${revision} \ - --revisionOrderId=${revision_order_id} \ - --taskId=${task_id} \ - --taskName=${task_name} \ - --variantName=${build_variant} \ - --versionId=${version_id} \ - --archiveFile=archive.json \ - --reportFile=report.json \ - --perfReportFile=perf.json - resmoke_exit_code=$? + # The "resmoke_wrapper" expansion is used by the 'burn_in_tests' task to wrap the resmoke.py + # invocation. It doesn't set any environment variables and should therefore come last in + # this list of expansions. + set +o errexit + PATH="$path_value" \ + AWS_PROFILE=${aws_profile_remote} \ + ${gcov_environment} \ + ${lang_environment} \ + ${san_options} \ + ${san_symbolizer} \ + ${snmp_config_path} \ + ${resmoke_wrapper} \ + $python buildscripts/evergreen_run_tests.py \ + ${resmoke_args} \ + $extra_args \ + ${test_flags} \ + --log=buildlogger \ + --staggerJobs=on \ + --buildId=${build_id} \ + --distroId=${distro_id} \ + --executionNumber=${execution} \ + --projectName=${project} \ + --gitRevision=${revision} \ + --revisionOrderId=${revision_order_id} \ + --taskId=${task_id} \ + --taskName=${task_name} \ + --variantName=${build_variant} \ + --versionId=${version_id} \ + --archiveFile=archive.json \ + --reportFile=report.json \ + --perfReportFile=perf.json + resmoke_exit_code=$? + set -o errexit + + # 74 is exit code for IOError on POSIX systems, which is raised when the machine is + # shutting down. + # + # 75 is exit code resmoke.py uses when the log output would be incomplete due to failing + # to communicate with logkeeper. + if [[ $resmoke_exit_code = 74 || $resmoke_exit_code = 75 ]]; then + echo $resmoke_exit_code > run_tests_infrastructure_failure + exit 0 + elif [ $resmoke_exit_code != 0 ]; then + # On failure save the resmoke exit code. + echo $resmoke_exit_code > resmoke_error_code + fi + exit $resmoke_exit_code + fi # end if [[ ${disable_unit_tests} && ! -f ${skip_tests|/dev/null} ]] + + "retrieve generated test configuration": &retrieve_generated_test_configuration + command: s3.get + params: + aws_key: ${aws_key} + aws_secret: ${aws_secret} + bucket: mciuploads + remote_file: ${project}/${build_variant}/${revision}/generate_tasks/${task}_gen-${build_id}.tgz + local_file: "generate_tasks_config.tgz" + + "extract generated test configuration": &extract_generated_test_configuration + command: shell.exec + type: test + params: + shell: bash + script: | + set -o verbose + set -o errexit + + target_dir="src/generated_resmoke_config" + mkdir -p $target_dir + mv generate_tasks_config.tgz $target_dir + + cd $target_dir + tar xzf generate_tasks_config.tgz + ls + + "generate resmoke tasks": + - *git_get_project + - *configure_evergreen_api_credentials + - command: expansions.write + params: + file: src/expansions.yml + + - command: shell.exec + params: + working_dir: src + script: | set -o errexit + set -o verbose - # 74 is exit code for IOError on POSIX systems, which is raised when the machine is - # shutting down. - # - # 75 is exit code resmoke.py uses when the log output would be incomplete due to failing - # to communicate with logkeeper. - if [[ $resmoke_exit_code = 74 || $resmoke_exit_code = 75 ]]; then - echo $resmoke_exit_code > run_tests_infrastructure_failure - exit 0 - elif [ $resmoke_exit_code != 0 ]; then - # On failure save the resmoke exit code. - echo $resmoke_exit_code > resmoke_error_code - fi - exit $resmoke_exit_code - fi # end if [[ ${disable_unit_tests} && ! -f ${skip_tests|/dev/null} ]] + ${activate_virtualenv} + $python -m pip install -r etc/pip/evgtest-requirements.txt + $python buildscripts/evergreen_generate_resmoke_tasks.py --expansion-file expansions.yml + + - command: archive.targz_pack + params: + target: generate_tasks_config.tgz + source_dir: src/generated_resmoke_config + include: + - "*" + + - command: s3.put + params: + aws_key: ${aws_key} + aws_secret: ${aws_secret} + local_file: generate_tasks_config.tgz + remote_file: ${project}/${build_variant}/${revision}/generate_tasks/${task_name}-${build_id}.tgz + bucket: mciuploads + permissions: public-read + content_type: ${content_type|application/gzip} + display_name: Generated Task Config - Execution ${execution} + + - command: generate.tasks + params: + files: + - src/generated_resmoke_config/${task}.json + + "run generated tests" : + - *install_pip_requirements + - *retrieve_generated_test_configuration + - *extract_generated_test_configuration + - command: expansions.update + params: + updates: + - key: aws_key_remote + value: ${mongodatafiles_aws_key} + - key: aws_profile_remote + value: mongodata_aws + - key: aws_secret_remote + value: ${mongodatafiles_aws_secret} + - *set_up_remote_credentials + - *determine_resmoke_jobs + - *update_resmoke_jobs_expansions + - *execute_resmoke_tests + # The existence of the "run_tests_infrastructure_failure" file indicates this failure isn't + # directly actionable. We use type=setup rather than type=system or type=test for this command + # because we don't intend for any human to look at this failure. + - command: shell.exec + type: setup + params: + working_dir: src + script: | + set -o verbose + if [ -f run_tests_infrastructure_failure ]; then + exit $(cat run_tests_infrastructure_failure) + fi + "run tests" : + - *install_pip_requirements + - *determine_task_timeout + - *update_task_timeout_expansions + - *update_task_timeout + - command: expansions.update + params: + updates: + - key: aws_key_remote + value: ${mongodatafiles_aws_key} + - key: aws_profile_remote + value: mongodata_aws + - key: aws_secret_remote + value: ${mongodatafiles_aws_secret} + - *set_up_remote_credentials + - *determine_resmoke_jobs + - *update_resmoke_jobs_expansions + - *execute_resmoke_tests # The existence of the "run_tests_infrastructure_failure" file indicates this failure isn't # directly actionable. We use type=setup rather than type=system or type=test for this command # because we don't intend for any human to look at this failure. @@ -6267,6 +6380,16 @@ tasks: vars: resmoke_args: --suites=sasl --storageEngine=wiredTiger +- name: sharding_gen + depends_on: [] + commands: + - func: "generate resmoke tasks" + vars: + task: sharding + use_large_distro: "true" + resmoke_args: --storageEngine=wiredTiger + fallback_num_sub_suites: 15 + - <<: *task_template name: sharding commands: @@ -8232,6 +8355,7 @@ tasks: depends_on: [] commands: - func: "git get project" + - func: "configure evergreen api credentials" - command: shell.exec timeout_secs: 14400 # Timeout if there is no output for 4 hours type: test @@ -8257,13 +8381,6 @@ tasks: $(echo "${testlifecycle_jira_key_certificate}" | sed 's/^/ /') END_OF_CREDS - # Create the Evergreen API credentials - cat > .evergreen.yml <<END_OF_CREDS - api_server_host: https://evergreen.mongodb.com/api - api_key: "${evergreen_api_key}" - user: "${evergreen_api_user}" - END_OF_CREDS - set -o verbose ${activate_virtualenv} @@ -11430,11 +11547,12 @@ buildvariants: build_mongoreplay: true jstestfuzz_num_generated_files: 40 jstestfuzz_concurrent_num_files: 10 + target_resmoke_time: 20 + large_distro_name: rhel62-large display_tasks: - *dbtest - *replica_sets_auth - *replica_sets_ese - - *sharding - *sharding_auth - *sharding_auth_audit - *sharding_ese @@ -11679,54 +11797,7 @@ buildvariants: - name: sharded_multi_stmt_txn_jscore_passthrough distros: - rhel62-large - - name: sharding_0 - distros: - - rhel62-large - - name: sharding_1 - distros: - - rhel62-large - - name: sharding_2 - distros: - - rhel62-large - - name: sharding_3 - distros: - - rhel62-large - - name: sharding_4 - distros: - - rhel62-large - - name: sharding_5 - distros: - - rhel62-large - - name: sharding_6 - distros: - - rhel62-large - - name: sharding_7 - distros: - - rhel62-large - - name: sharding_8 - distros: - - rhel62-large - - name: sharding_9 - distros: - - rhel62-large - - name: sharding_10 - distros: - - rhel62-large - - name: sharding_11 - distros: - - rhel62-large - - name: sharding_12 - distros: - - rhel62-large - - name: sharding_13 - distros: - - rhel62-large - - name: sharding_14 - distros: - - rhel62-large - - name: sharding_misc - distros: - - rhel62-large + - name: sharding_gen - name: sharding_auth_0 distros: - rhel62-large |