diff options
| -rwxr-xr-x | buildscripts/burn_in_tests.py | 67 | ||||
| -rw-r--r-- | buildscripts/evergreen_burn_in_tests.py | 4 | ||||
| -rw-r--r-- | buildscripts/tests/test_burn_in_tests.py | 18 | ||||
| -rw-r--r-- | etc/evergreen.yml | 10 | ||||
| -rw-r--r-- | etc/evergreen_nightly.yml | 4 | ||||
| -rw-r--r-- | etc/evergreen_yml_components/definitions.yml | 98 | ||||
| -rw-r--r-- | etc/evergreen_yml_components/variants/task_generation.yml | 1 | ||||
| -rw-r--r-- | evergreen/generate_version.sh | 4 | ||||
| -rw-r--r-- | evergreen/generate_version_burn_in.sh | 17 | ||||
| -rwxr-xr-x | evergreen/prelude.sh | 1 | ||||
| -rw-r--r-- | evergreen/prelude_mongo_task_generator.sh | 24 |
11 files changed, 168 insertions, 80 deletions
diff --git a/buildscripts/burn_in_tests.py b/buildscripts/burn_in_tests.py index 27da0996b63..84c7d98aab0 100755 --- a/buildscripts/burn_in_tests.py +++ b/buildscripts/burn_in_tests.py @@ -13,6 +13,7 @@ from typing import Optional, Set, Tuple, List, Dict, NamedTuple import click import yaml from git import Repo +from pydantic import BaseModel import structlog from structlog.stdlib import LoggerFactory @@ -352,7 +353,8 @@ def create_task_list_for_tests(changed_tests: Set[str], build_variant: str, def create_tests_by_task(build_variant: str, evg_conf: EvergreenProjectConfig, - changed_tests: Set[str], install_dir: str) -> Dict[str, TaskInfo]: + changed_tests: Set[str], + install_dir: Optional[str]) -> Dict[str, TaskInfo]: """ Create a list of tests by task. @@ -367,7 +369,11 @@ def create_tests_by_task(build_variant: str, evg_conf: EvergreenProjectConfig, exclude_tests.append(f"{ENTERPRISE_MODULE_PATH}/**/*") changed_tests = filter_tests(changed_tests, exclude_tests) - buildscripts.resmokelib.parser.set_run_options(f"--installDir={shlex.quote(install_dir)}") + run_options = "" + if install_dir is not None: + run_options = f"--installDir={shlex.quote(install_dir)}" + buildscripts.resmokelib.parser.set_run_options(run_options) + if changed_tests: return create_task_list_for_tests(changed_tests, build_variant, evg_conf, exclude_suites, exclude_tasks) @@ -408,7 +414,7 @@ def _configure_logging(verbose: bool): logging.basicConfig( format="[%(asctime)s - %(name)s - %(levelname)s] %(message)s", level=level, - stream=sys.stdout, + stream=sys.stderr, ) for log_name in EXTERNAL_LOGGERS: logging.getLogger(log_name).setLevel(logging.WARNING) @@ -537,11 +543,45 @@ class LocalBurnInExecutor(BurnInExecutor): run_tests(tests_by_task, resmoke_cmd) +class DiscoveredTask(BaseModel): + """ + Model for a discovered task to run. + + * task_name: Name of discovered task. + * test_list: List of tests to run under discovered task. + """ + + task_name: str + test_list: List[str] + + +class DiscoveredTaskList(BaseModel): + """Model for a list of discovered tasks.""" + + discovered_tasks: List[DiscoveredTask] + + +class YamlBurnInExecutor(BurnInExecutor): + """A burn-in executor that outputs discovered tasks as YAML.""" + + def execute(self, tests_by_task: Dict[str, TaskInfo]) -> None: + """ + Report the given tasks and their tests to stdout. + + :param tests_by_task: Dictionary of tasks to run with tests to run in each. + """ + discovered_tasks = DiscoveredTaskList(discovered_tasks=[ + DiscoveredTask(task_name=task_name, test_list=task_info.tests) + for task_name, task_info in tests_by_task.items() + ]) + print(yaml.safe_dump(discovered_tasks.dict())) + + class BurnInOrchestrator: """Orchestrate the execution of burn_in_tests.""" def __init__(self, change_detector: FileChangeDetector, burn_in_executor: BurnInExecutor, - evg_conf: EvergreenProjectConfig) -> None: + evg_conf: EvergreenProjectConfig, install_dir: Optional[str]) -> None: """ Create a new orchestrator. @@ -552,8 +592,9 @@ class BurnInOrchestrator: self.change_detector = change_detector self.burn_in_executor = burn_in_executor self.evg_conf = evg_conf + self.install_dir = install_dir - def burn_in(self, repos: List[Repo], build_variant: str, install_dir: str) -> None: + def burn_in(self, repos: List[Repo], build_variant: str) -> None: """ Execute burn in tests for the given git repositories. @@ -564,7 +605,7 @@ class BurnInOrchestrator: LOGGER.info("Found changed tests", files=changed_tests) tests_by_task = create_tests_by_task(build_variant, self.evg_conf, changed_tests, - install_dir) + self.install_dir) LOGGER.debug("tests and tasks found", tests_by_task=tests_by_task) self.burn_in_executor.execute(tests_by_task) @@ -584,18 +625,20 @@ class BurnInOrchestrator: help="The maximum number of times to repeat tests if time option is specified.") @click.option("--repeat-tests-secs", "repeat_tests_secs", default=None, type=int, metavar="SECONDS", help="Repeat tests for the given time (in secs).") +@click.option("--yaml", "use_yaml", is_flag=True, default=False, + help="Output discovered tasks in YAML. Tests will not be run.") @click.option("--verbose", "verbose", default=False, is_flag=True, help="Enable extra logging.") @click.option( "--origin-rev", "origin_rev", default=None, help="The revision in the mongo repo that changes will be compared against if specified.") -@click.option("--install-dir", "install_dir", required=True, type=str, +@click.option("--install-dir", "install_dir", type=str, help="Path to bin directory of a testable installation") @click.argument("resmoke_args", nargs=-1, type=click.UNPROCESSED) # pylint: disable=too-many-arguments,too-many-locals def main(build_variant: str, no_exec: bool, repeat_tests_num: Optional[int], repeat_tests_min: Optional[int], repeat_tests_max: Optional[int], repeat_tests_secs: Optional[int], resmoke_args: str, verbose: bool, - origin_rev: Optional[str], install_dir: str) -> None: + origin_rev: Optional[str], install_dir: Optional[str], use_yaml: bool) -> None: """ Run new or changed tests in repeated mode to validate their stability. @@ -639,11 +682,13 @@ def main(build_variant: str, no_exec: bool, repeat_tests_num: Optional[int], change_detector = LocalFileChangeDetector(origin_rev) executor = LocalBurnInExecutor(resmoke_args, repeat_config) - if no_exec: + if use_yaml: + executor = YamlBurnInExecutor() + elif no_exec: executor = NopBurnInExecutor() - burn_in_orchestrator = BurnInOrchestrator(change_detector, executor, evg_conf) - burn_in_orchestrator.burn_in(repos, build_variant, install_dir) + burn_in_orchestrator = BurnInOrchestrator(change_detector, executor, evg_conf, install_dir) + burn_in_orchestrator.burn_in(repos, build_variant) if __name__ == "__main__": diff --git a/buildscripts/evergreen_burn_in_tests.py b/buildscripts/evergreen_burn_in_tests.py index 37a0d2874ff..a628a9a5d20 100644 --- a/buildscripts/evergreen_burn_in_tests.py +++ b/buildscripts/evergreen_burn_in_tests.py @@ -436,8 +436,8 @@ def burn_in(task_id: str, build_variant: str, generate_config: GenerateConfig, change_detector = EvergreenFileChangeDetector(task_id, evg_api, os.environ) executor = GenerateBurnInExecutor(generate_config, repeat_config, evg_api, generate_tasks_file) - burn_in_orchestrator = BurnInOrchestrator(change_detector, executor, evg_conf) - burn_in_orchestrator.burn_in(repos, build_variant, install_dir) + burn_in_orchestrator = BurnInOrchestrator(change_detector, executor, evg_conf, install_dir) + burn_in_orchestrator.burn_in(repos, build_variant) @click.command() diff --git a/buildscripts/tests/test_burn_in_tests.py b/buildscripts/tests/test_burn_in_tests.py index c51f8c60ecf..369131eff60 100644 --- a/buildscripts/tests/test_burn_in_tests.py +++ b/buildscripts/tests/test_burn_in_tests.py @@ -4,12 +4,14 @@ from __future__ import absolute_import import collections import datetime +from io import StringIO import os import sys import subprocess import unittest from mock import Mock, patch, MagicMock +import yaml import buildscripts.burn_in_tests as under_test from buildscripts.ciconfig.evergreen import parse_evergreen_file, VariantTask @@ -556,3 +558,19 @@ class TestLocalFileChangeDetector(unittest.TestCase): self.assertIn(file_list[2], found_tests) self.assertNotIn(file_list[1], found_tests) self.assertEqual(2, len(found_tests)) + + +class TestYamlBurnInExecutor(unittest.TestCase): + @patch('sys.stdout', new_callable=StringIO) + def test_found_tasks_should_be_reported_as_yaml(self, stdout): + n_tasks = 5 + n_tests = 3 + tests_by_task = create_tests_by_task_mock(n_tasks, n_tests) + + yaml_executor = under_test.YamlBurnInExecutor() + yaml_executor.execute(tests_by_task) + + yaml_raw = stdout.getvalue() + results = yaml.safe_load(yaml_raw) + self.assertEqual(n_tasks, len(results["discovered_tasks"])) + self.assertEqual(n_tests, len(results["discovered_tasks"][0]["test_list"])) diff --git a/etc/evergreen.yml b/etc/evergreen.yml index ef5e180275a..c1571745fd1 100644 --- a/etc/evergreen.yml +++ b/etc/evergreen.yml @@ -643,7 +643,7 @@ buildvariants: target_resmoke_time: 10 max_sub_suites: 5 large_distro_name: rhel80-medium - burn_in_tag_buildvariants: enterprise-rhel-80-64-bit-inmem linux-64-duroff enterprise-rhel-80-64-bit-multiversion + burn_in_tag_buildvariants: enterprise-rhel-80-64-bit-dynamic-required num_scons_link_jobs_available: 0.99 test_flags: >- --mongodSetParameters="{roundtripBsonColumnOnValidate: true}" @@ -1306,7 +1306,9 @@ buildvariants: idle_timeout_factor: 1.5 exec_timeout_factor: 1.5 large_distro_name: rhel80-medium - burn_in_tag_buildvariants: enterprise-rhel-80-64-bit-inmem linux-64-duroff enterprise-rhel-80-64-bit-multiversion + burn_in_tag_buildvariants: enterprise-rhel-80-64-bit-dynamic-required + burn_in_tag_compile_distro: rhel80-xlarge + burn_in_tag_compile_task_group_name: compile_and_archive_dist_test_TG num_scons_link_jobs_available: 0.99 tasks: - name: compile_test_and_package_parallel_core_stream_TG @@ -1579,7 +1581,9 @@ buildvariants: target_resmoke_time: 10 max_sub_suites: 5 large_distro_name: rhel80-medium - burn_in_tag_buildvariants: enterprise-rhel-80-64-bit-inmem linux-64-duroff enterprise-rhel-80-64-bit-multiversion + burn_in_tag_buildvariants: enterprise-rhel-80-64-bit-dynamic-required + burn_in_tag_compile_distro: rhel80-xlarge + burn_in_tag_compile_task_group_name: compile_and_archive_dist_test_TG num_scons_link_jobs_available: 0.99 test_flags: >- --mongodSetParameters="{internalQueryForceClassicEngine: true}" diff --git a/etc/evergreen_nightly.yml b/etc/evergreen_nightly.yml index 5e8a5058866..abd31a0e91b 100644 --- a/etc/evergreen_nightly.yml +++ b/etc/evergreen_nightly.yml @@ -184,7 +184,9 @@ buildvariants: idle_timeout_factor: 1.5 exec_timeout_factor: 1.5 large_distro_name: rhel80-medium - burn_in_tag_buildvariants: enterprise-rhel-80-64-bit-inmem linux-64-duroff enterprise-rhel-80-64-bit-multiversion + burn_in_tag_buildvariants: enterprise-rhel-80-64-bit-dynamic-required + burn_in_tag_compile_distro: rhel80-xlarge + burn_in_tag_compile_task_group_name: compile_and_archive_dist_test_TG num_scons_link_jobs_available: 0.99 tasks: - name: compile_test_and_package_parallel_core_stream_TG diff --git a/etc/evergreen_yml_components/definitions.yml b/etc/evergreen_yml_components/definitions.yml index 6b27f051064..9176c6b804e 100644 --- a/etc/evergreen_yml_components/definitions.yml +++ b/etc/evergreen_yml_components/definitions.yml @@ -107,6 +107,19 @@ variables: vars: resmoke_args: --help +- &gen_burn_in_task_template + name: gen_burn_in_task_template + depends_on: + - name: version_gen + variant: generate-tasks-for-version + - name: version_burn_in_gen + variant: generate-tasks-for-version + - name: archive_dist_test_debug + commands: + - func: "generate resmoke tasks" + vars: + resmoke_args: --help + - &benchmark_template name: benchmark_template depends_on: @@ -1073,7 +1086,7 @@ functions: files: - src/generated_resmoke_config/*.json - "generate burn in tags": + "generate version burn in": - *f_expansions_write - *configure_evergreen_api_credentials - command: subprocess.exec @@ -1081,27 +1094,29 @@ functions: params: binary: bash args: - - "./src/evergreen/burn_in_tests_generate.sh" + - "./src/evergreen/generate_version_burn_in.sh" - command: archive.targz_pack params: - target: burn_in_tags_gen.tgz - source_dir: src/generated_burn_in_tags_config + target: generate_tasks_config.tgz + source_dir: src/generated_resmoke_config include: - "*" - command: s3.put params: aws_key: ${aws_key} aws_secret: ${aws_secret} - local_file: burn_in_tags_gen.tgz - remote_file: ${project}/${build_variant}/${revision}/burn_in_tags_gen/burn_in_tags_gen-${build_id}.tgz + local_file: generate_tasks_config.tgz + remote_file: ${project}/${revision}/generate_tasks/generated-burn-in-config-${version_id}.tgz bucket: mciuploads permissions: public-read - content_type: application/gzip - display_name: Burn_in_tags Task Config - Execution ${execution} + content_type: ${content_type|application/gzip} + display_name: Generated Burn In Task Config - Execution ${execution} + optional: true - command: generate.tasks params: + optional: true files: - - src/generated_burn_in_tags_config/burn_in_tags_gen.json + - src/generated_resmoke_config/*.json "generate resmoke tasks": - *fetch_artifacts @@ -3274,46 +3289,11 @@ tasks: args: - "src/evergreen/check_idl_compat.sh" -- name: burn_in_tests_gen +- <<: *gen_burn_in_task_template + name: burn_in_tests_gen tags: [] commands: - - command: manifest.load - - func: "git get project and add git tag" - - *f_expansions_write - - *kill_processes - - *cleanup_environment - - func: "set up venv" - - func: "upload pip requirements" - - func: "configure evergreen api credentials" - - command: subprocess.exec - type: test - params: - binary: bash - args: - - "src/evergreen/burn_in_tests.sh" - env: - BURN_IN_TESTS: ${burn_in_tests} - - command: archive.targz_pack - params: - target: src/burn_in_tests_gen.tgz - source_dir: src - include: - - burn_in_tests_gen.json - - - command: s3.put - params: - aws_key: ${aws_key} - aws_secret: ${aws_secret} - local_file: src/burn_in_tests_gen.tgz - remote_file: ${project}/${build_variant}/${revision}/burn_in_tests_gen/burn_in_tests_gen-${build_id}.tgz - bucket: mciuploads - permissions: public-read - content_type: application/gzip - display_name: Burn_in_tests Task Config - Execution ${execution} - - command: generate.tasks - params: - files: - - src/burn_in_tests_gen.json + - func: "generate resmoke tasks" - <<: *benchmark_template name: benchmarks_orphaned @@ -4325,27 +4305,25 @@ tasks: commands: - func: "generate resmoke tasks" -- name: burn_in_tags_gen +- <<: *gen_burn_in_task_template + name: burn_in_tags_gen tags: [] - depends_on: - - name: archive_dist_test + commands: + - func: "generate resmoke tasks" + +- name: version_gen commands: - command: manifest.load - - func: "git get project and add git tag" + - *git_get_project - *f_expansions_write + - *add_git_tag - *kill_processes - *cleanup_environment - func: "set up venv" - func: "upload pip requirements" - - func: "configure evergreen api credentials" - - func: "generate burn in tags" - vars: - max_revisions: 25 - repeat_tests_secs: 600 - repeat_tests_min: 2 - repeat_tests_max: 1000 + - func: "generate version" -- name: version_gen +- name: version_burn_in_gen commands: - command: manifest.load - *git_get_project @@ -4355,7 +4333,7 @@ tasks: - *cleanup_environment - func: "set up venv" - func: "upload pip requirements" - - func: "generate version" + - func: "generate version burn in" - name: version_expansions_gen commands: diff --git a/etc/evergreen_yml_components/variants/task_generation.yml b/etc/evergreen_yml_components/variants/task_generation.yml index b45a68d1ff8..e5cdf11b1c7 100644 --- a/etc/evergreen_yml_components/variants/task_generation.yml +++ b/etc/evergreen_yml_components/variants/task_generation.yml @@ -9,4 +9,5 @@ buildvariants: - rhel80-medium tasks: - name: version_gen + - name: version_burn_in_gen - name: version_expansions_gen diff --git a/evergreen/generate_version.sh b/evergreen/generate_version.sh index b51cc809f6c..5a0ad5bb8d6 100644 --- a/evergreen/generate_version.sh +++ b/evergreen/generate_version.sh @@ -6,9 +6,7 @@ cd src set -o errexit set -o verbose -curl -L https://github.com/mongodb/mongo-task-generator/releases/download/v0.5.3/mongo-task-generator --output mongo-task-generator -chmod +x mongo-task-generator - +setup_mongo_task_generator activate_venv PATH=$PATH:$HOME:/ ./mongo-task-generator \ --expansion-file ../expansions.yml \ diff --git a/evergreen/generate_version_burn_in.sh b/evergreen/generate_version_burn_in.sh new file mode 100644 index 00000000000..8c37474c86e --- /dev/null +++ b/evergreen/generate_version_burn_in.sh @@ -0,0 +1,17 @@ +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" +. "$DIR/prelude.sh" + +cd src + +set -o errexit +set -o verbose + +setup_mongo_task_generator +activate_venv +PATH=$PATH:$HOME:/ ./mongo-task-generator \ + --expansion-file ../expansions.yml \ + --evg-auth-file ./.evergreen.yml \ + --evg-project-file ${evergreen_config_file_path} \ + --generate-sub-tasks-config etc/generate_subtasks_config.yml \ + --burn-in \ + $@ diff --git a/evergreen/prelude.sh b/evergreen/prelude.sh index 3844afaead5..72de6ac13bb 100755 --- a/evergreen/prelude.sh +++ b/evergreen/prelude.sh @@ -11,6 +11,7 @@ evergreen_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)" . "$evergreen_dir/prelude_python.sh" . "$evergreen_dir/prelude_venv.sh" . "$evergreen_dir/prelude_db_contrib_tool.sh" +. "$evergreen_dir/prelude_mongo_task_generator.sh" expansions_yaml="$evergreen_dir/../../expansions.yml" expansions_default_yaml="$evergreen_dir/../etc/expansions.default.yml" diff --git a/evergreen/prelude_mongo_task_generator.sh b/evergreen/prelude_mongo_task_generator.sh new file mode 100644 index 00000000000..511ba44c7cd --- /dev/null +++ b/evergreen/prelude_mongo_task_generator.sh @@ -0,0 +1,24 @@ +function setup_mongo_task_generator { + if [ ! -f mongo-task-generator ]; then + curl -L https://github.com/mongodb/mongo-task-generator/releases/download/v0.6.2/mongo-task-generator --output mongo-task-generator + chmod +x mongo-task-generator + fi +} + +## Comment above and uncomment below to test unreleased mongo-task-generator changes that are +## pushed to the `<branch-name>` of the `git@github.com:<user-name>/mongo-task-generator.git` +## repo +#function setup_mongo_task_generator { +# if [ ! -f mongo-task-generator ]; then +# +# curl https://sh.rustup.rs -sSf | sh -s -- -y +# source "$HOME/.cargo/env" +# git clone git@github.com:<user-name>/mongo-task-generator.git unreleased-mongo-task-generator +# pushd unreleased-mongo-task-generator +# git checkout <branch-name> +# cargo build --release --locked +# generator_path="$(pwd)/target/release/mongo-task-generator" +# popd +# cp "$generator_path" mongo-task-generator +# fi +#} |