diff options
Diffstat (limited to 'etc/evergreen_yml_components/definitions.yml')
| -rw-r--r-- | etc/evergreen_yml_components/definitions.yml | 337 |
1 files changed, 231 insertions, 106 deletions
diff --git a/etc/evergreen_yml_components/definitions.yml b/etc/evergreen_yml_components/definitions.yml index 289e48cbcc9..e03094af84d 100644 --- a/etc/evergreen_yml_components/definitions.yml +++ b/etc/evergreen_yml_components/definitions.yml @@ -1,6 +1,4 @@ #################################################### -# Static Definitions # -#################################################### # This file contains static Evergreen config definitions used by other YAML files. # It is not run directly by any Evergreen project. # @@ -110,7 +108,7 @@ variables: - &benchmark_template name: benchmark_template depends_on: - - name: compile_benchmarks + - name: compile_upload_benchmarks commands: - func: "do benchmark setup" - func: "run tests" @@ -118,7 +116,7 @@ variables: resmoke_args: --help resmoke_jobs_max: 1 - func: "send benchmark results" - - func: "analyze benchmark results" +# - func: "analyze benchmark results" vars: suite: benchmark_suite @@ -498,12 +496,19 @@ functions: args: - "src/evergreen/functions/binaries_extract.sh" - "check binary version": &check_binary_version - command: subprocess.exec + "get version expansions": &get_version_expansions + command: s3.get params: - binary: bash - args: - - "src/evergreen/functions/binary_version_check.sh" + aws_key: ${aws_key} + aws_secret: ${aws_secret} + remote_file: ${project}/${version_id}/version_expansions.yml + bucket: mciuploads + local_file: src/version_expansions.yml + + "apply version expansions": &apply_version_expansions + command: expansions.update + params: + file: src/version_expansions.yml "fetch benchmarks": &fetch_benchmarks command: s3.get @@ -783,7 +788,7 @@ functions: remote_file: ${project}/${build_variant}/${revision}/pip-requirements-${task_id}-${execution}.txt bucket: mciuploads permissions: public-read - content_type: atext-plain + content_type: text/plain display_name: Pip Requirements "send benchmark results": @@ -834,7 +839,6 @@ functions: - *adjust_venv - *fetch_binaries - *extract_binaries - - *check_binary_version - *get_buildnumber - *f_expansions_write - *set_up_credentials @@ -1177,6 +1181,8 @@ functions: - "src/evergreen/scons_lint.sh" "scons compile": + - *get_version_expansions + - *apply_version_expansions - *f_expansions_write - command: subprocess.exec type: test @@ -1185,6 +1191,24 @@ functions: args: - "src/evergreen/scons_compile.sh" + "generate version expansions": + - *f_expansions_write + - command: subprocess.exec + params: + binary: bash + args: + - "src/evergreen/functions/version_expansions_generate.sh" + - command: s3.put + params: + aws_key: ${aws_key} + aws_secret: ${aws_secret} + local_file: src/version_expansions.yml + remote_file: ${project}/${version_id}/version_expansions.yml + bucket: mciuploads + permissions: public-read + content_type: application/x-yaml + display_name: version expansions + "generate compile expansions": - *f_expansions_write - command: subprocess.exec @@ -2045,9 +2069,29 @@ functions: remote_file: ${project}/${build_variant}/${revision}/local-resmoke-invocation-${task_id}-${execution}.txt bucket: mciuploads permissions: public-read - content_type: atext-plain + content_type: text/plain display_name: Resmoke.py Invocation for Local Usage + "antithesis image build": + - command: subprocess.exec + params: + binary: bash + args: + - "./src/evergreen/antithesis_image_build.sh" + + "antithesis image push": + - command: subprocess.exec + params: + binary: bash + args: + - "./src/evergreen/antithesis_image_push.sh" + + "antithesis dry run": + - command: subprocess.exec + params: + binary: bash + args: + - "./src/evergreen/antithesis_dry_run.sh" # Pre task steps pre: @@ -2094,7 +2138,9 @@ tasks: ## compile - build all scons targets except unittests ## - name: compile_dist_test tags: [] - depends_on: [] + depends_on: + - name: version_expansions_gen + variant: generate-tasks-for-version commands: - func: "scons compile" vars: @@ -2104,6 +2150,37 @@ tasks: task_compile_flags: >- PREFIX=dist-test +- name: compile_upload_benchmarks + tags: [] + depends_on: + - name: version_expansions_gen + variant: generate-tasks-for-version + commands: + - func: "scons compile" + vars: + targets: install-benchmarks + compiling_for_test: true + - command: archive.targz_pack + params: + target: "benchmarks.tgz" + source_dir: "src" + include: + - "./build/benchmarks.txt" + - "./build/**_bm" + - "./build/**_bm.gcno" + - "./build/**_bm.exe" + - "./build/**_bm.pdb" + - command: s3.put + params: + aws_key: ${aws_key} + aws_secret: ${aws_secret} + local_file: benchmarks.tgz + remote_file: ${project}/${build_variant}/${revision}/benchmarks/${build_id}.tgz + bucket: mciuploads + permissions: public-read + content_type: application/tar + display_name: Benchmarks + - name: determine_patch_tests commands: - *f_expansions_write @@ -2119,7 +2196,6 @@ tasks: depends_on: - name: compile_dist_test commands: - - *f_expansions_write - func: "scons compile" vars: targets: >- @@ -2175,7 +2251,6 @@ tasks: - "./etc/repo_config.yaml" - "./etc/scons/**" - "buildscripts/**" - - "compile_expansions.yml" - "all_feature_flags.txt" # Must correspond to the definition in buildscripts/idl/lib.py. - "jstests/**" - "patch_files.txt" @@ -2240,6 +2315,9 @@ tasks: - name: compile_ninja tags: [] + depends_on: + - name: version_expansions_gen + variant: generate-tasks-for-version commands: - func: "scons compile" vars: @@ -2260,6 +2338,9 @@ tasks: - name: compile_ninja_next tags: [] + depends_on: + - name: version_expansions_gen + variant: generate-tasks-for-version commands: - func: "scons compile" vars: @@ -2281,6 +2362,9 @@ tasks: - name: compile_build_tools_next tags: [] + depends_on: + - name: version_expansions_gen + variant: generate-tasks-for-version commands: - func: "scons compile" vars: @@ -2291,6 +2375,9 @@ tasks: - name: libdeps_graph_linting tags: [] + depends_on: + - name: version_expansions_gen + variant: generate-tasks-for-version commands: - *f_expansions_write - command: subprocess.exec @@ -2303,6 +2390,7 @@ tasks: vars: task_compile_flags: >- --link-model=dynamic + --force-macos-dynamic-link --build-tools=next targets: generate-libdeps-graph @@ -2350,8 +2438,10 @@ tasks: - name: clang_tidy tags: [] exec_timeout_secs: 3600 # 1 hour timeout for the task overall + depends_on: + - name: version_expansions_gen + variant: generate-tasks-for-version commands: - - *f_expansions_write - func: "scons compile" vars: targets: generated-sources compiledb @@ -2398,7 +2488,7 @@ tasks: ## run_unittests with UndoDB live-record ## #- name: run_unittests_with_recording # depends_on: -# - name: compile_unittests_for_recorded_unittest +# - name: compile_unittests_for_recorded_unittest # commands: # - *f_expansions_write # - func: "run diskstats" @@ -2427,6 +2517,9 @@ tasks: ##compile_and_archive_libfuzzertests - build libfuzzertests ## - name: compile_and_archive_libfuzzertests tags: [] + depends_on: + - name: version_expansions_gen + variant: generate-tasks-for-version commands: - func: "scons compile" vars: @@ -2457,6 +2550,9 @@ tasks: - name: server_discovery_and_monitoring_json_test tags: [] + depends_on: + - name: version_expansions_gen + variant: generate-tasks-for-version commands: - func: "scons compile" vars: @@ -2469,6 +2565,9 @@ tasks: - name: server_selection_json_test tags: [] + depends_on: + - name: version_expansions_gen + variant: generate-tasks-for-version commands: - func: "scons compile" vars: @@ -2516,6 +2615,9 @@ tasks: - name: compile_visibility_test tags: [] + depends_on: + - name: version_expansions_gen + variant: generate-tasks-for-version commands: - func: "scons compile" vars: @@ -2525,6 +2627,7 @@ tasks: --dbg=on --opt=on --link-model=dynamic + --force-macos-dynamic-link --visibility-support=on - command: s3.put @@ -2543,7 +2646,12 @@ tasks: - name: embedded_sdk_build_cdriver tags: [] + depends_on: + - name: version_expansions_gen + variant: generate-tasks-for-version commands: + - *get_version_expansions + - *apply_version_expansions - func: f_expansions_write - command: subprocess.exec params: @@ -2578,6 +2686,8 @@ tasks: depends_on: - name: embedded_sdk_install_dev commands: + - *get_version_expansions + - *apply_version_expansions - *f_expansions_write - command: subprocess.exec params: @@ -2610,6 +2720,7 @@ tasks: task_compile_flags: *embedded_sdk_compile_flags task_compile_flags_extra: >- --link-model=dynamic + --force-macos-dynamic-link # Unlike static builds, dynamic builds have no need to # constrain the number of link jobs. Unfortunately, --jlink=1 # means one link job, not 100%. So this is a bit gross but set @@ -2621,6 +2732,8 @@ tasks: depends_on: - name: embedded_sdk_install_tests commands: + - *get_version_expansions + - *apply_version_expansions - *f_expansions_write - command: subprocess.exec type: test @@ -2647,6 +2760,8 @@ tasks: depends_on: - name: embedded_sdk_install_tests commands: + - *get_version_expansions + - *apply_version_expansions - *f_expansions_write - command: subprocess.exec type: test @@ -2665,6 +2780,8 @@ tasks: depends_on: - name: embedded_sdk_run_tests commands: + - *get_version_expansions + - *apply_version_expansions # A second put, this time to -latest, to give devs a reasonable # way to get the most recent build. - command: s3.put @@ -2684,6 +2801,8 @@ tasks: depends_on: - name: embedded_sdk_run_tests commands: + - *get_version_expansions + - *apply_version_expansions # A second put, this time to -latest, to give devs a reasonable # way to get the most recent build. - command: s3.put @@ -2700,6 +2819,9 @@ tasks: - name: stitch_support_create_lib tags: [] + depends_on: + - name: version_expansions_gen + variant: generate-tasks-for-version commands: - *f_expansions_write - func: "scons compile" @@ -2750,6 +2872,8 @@ tasks: depends_on: - name: stitch_support_install_tests commands: + - func: "get version expansions" + - func: "apply version expansions" - *f_expansions_write - command: subprocess.exec type: test @@ -2760,6 +2884,9 @@ tasks: - name: crypt_create_lib tags: [] + depends_on: + - name: version_expansions_gen + variant: generate-tasks-for-version commands: - *f_expansions_write - func: "scons compile" @@ -2793,6 +2920,9 @@ tasks: - name: crypt_create_debug_lib tags: [] + depends_on: + - name: version_expansions_gen + variant: generate-tasks-for-version commands: - *f_expansions_write - func: "scons compile" @@ -2863,6 +2993,8 @@ tasks: depends_on: - name: crypt_install_tests commands: + - func: "get version expansions" + - func: "apply version expansions" - *f_expansions_write - command: subprocess.exec type: test @@ -2871,52 +3003,6 @@ tasks: args: - "src/evergreen/crypt_run_tests.sh" -- name: compile_benchmarks - tags: [] - depends_on: [] - commands: - - command: manifest.load - - func: "git get project and add git tag" - - *f_expansions_write - - *kill_processes - - *cleanup_environment - - func: "set up venv" - - func: "upload pip requirements" - - func: "get buildnumber" - - func: "f_expansions_write" - - func: "set up credentials" - - func: "use WiredTiger develop" # noop if ${use_wt_develop} is not "true" - - func: "set up win mount script" - - func: "generate compile expansions" - # Then we load the generated version data into the agent so we can use it in task definitions - - func: "apply compile expansions" - - func: "scons compile" - vars: - targets: install-benchmarks - compiling_for_test: true - - func: "attach scons logs" - - func: "send scons cedar report" - - command: archive.targz_pack - params: - target: "benchmarks.tgz" - source_dir: "src" - include: - - "./build/benchmarks.txt" - - "./build/**_bm" - - "./build/**_bm.gcno" - - "./build/**_bm.exe" - - "./build/**_bm.pdb" - - command: s3.put - params: - aws_key: ${aws_key} - aws_secret: ${aws_secret} - local_file: benchmarks.tgz - remote_file: ${project}/${build_variant}/${revision}/benchmarks/${build_id}.tgz - bucket: mciuploads - permissions: public-read - content_type: application/tar - display_name: Benchmarks - ## lint ## - name: lint_pylinters tags: ["lint"] @@ -3136,7 +3222,7 @@ tasks: exec_timeout_secs: 14400 # 4 hour timeout. resmoke_jobs_max: 1 - func: "send benchmark results" - - func: "analyze benchmark results" +# - func: "analyze benchmark results" vars: suite: benchmarks @@ -3150,7 +3236,7 @@ tasks: suite: benchmarks_sharding resmoke_jobs_max: 1 - func: "send benchmark results" - - func: "analyze benchmark results" +# - func: "analyze benchmark results" # Disabled under SERVER-64949. # - <<: *benchmark_template @@ -4131,8 +4217,19 @@ tasks: - func: "run tests" - <<: *task_template + name: aggregation_column_store_index_passthrough + # TODO SERVER-61644 Remove the 'feature_flag_guarded' tag and ensure this suite is run on all + # variants. + tags: ["aggregation", "sbe_only", "feature_flag_guarded"] + depends_on: + - name: aggregation + commands: + - func: "do setup" + - func: "run tests" + +- <<: *task_template name: audit - tags: [] + tags: ["audit", "patch_build"] commands: - func: "do setup" - func: "run tests" @@ -4175,9 +4272,21 @@ tasks: - func: "upload pip requirements" - func: "generate version" +- name: version_expansions_gen + commands: + - command: manifest.load + - *git_get_project + - *f_expansions_write + - *add_git_tag + - *kill_processes + - *cleanup_environment + - func: "set up venv" + - func: "upload pip requirements" + - func: "generate version expansions" + - <<: *gen_task_template name: auth_audit_gen - tags: ["auth", "audit"] + tags: ["auth", "audit", "patch_build"] commands: - func: "generate resmoke tasks" @@ -4401,7 +4510,7 @@ tasks: - <<: *task_template name: ese - tags: ["encrypt"] + tags: ["encrypt", "patch_build"] commands: - func: "do setup" - func: "run tests" @@ -4525,9 +4634,7 @@ tasks: - "src/evergreen/external_auth_pip.sh" - func: "run tests" vars: - # TODO SERVER-64323 - # restore concurrency for this suite when the issue on windows is resolved. - resmoke_jobs_max: 1 + resmoke_jobs_max: ${external_auth_jobs_max|4} resmoke_args: --excludeWithAnyTags=requires_domain_controller - <<: *task_template @@ -5622,7 +5729,7 @@ tasks: - <<: *task_template name: sasl - tags: [] + tags: ["sasl", "patch_build"] commands: - func: "do setup" - func: "run tests" @@ -5746,7 +5853,7 @@ tasks: - <<: *gen_task_template name: ssl_gen - tags: ["encrypt", "ssl"] + tags: ["encrypt", "ssl", "patch_build"] commands: - func: "generate resmoke tasks" vars: @@ -5754,7 +5861,7 @@ tasks: - <<: *gen_task_template name: sslSpecial_gen - tags: ["encrypt", "ssl"] + tags: ["encrypt", "ssl", "patch_build"] commands: - func: "generate resmoke tasks" vars: @@ -5762,7 +5869,7 @@ tasks: - <<: *gen_task_template name: ssl_x509_gen - tags: ["encrypt", "ssl"] + tags: ["encrypt", "ssl", "patch_build"] commands: - func: "generate resmoke tasks" vars: @@ -5843,12 +5950,16 @@ tasks: tags: ["multiversion_passthrough", "multiversion", "no_version_combination"] commands: - func: "generate resmoke tasks" + vars: + multiversion_exclude_tags_version: last_continuous - <<: *gen_task_template name: retryable_writes_downgrade_last_lts_gen tags: ["multiversion_passthrough", "multiversion", "no_version_combination"] commands: - func: "generate resmoke tasks" + vars: + multiversion_exclude_tags_version: last_lts - <<: *gen_task_template name: sharded_retryable_writes_downgrade_gen @@ -5977,7 +6088,7 @@ tasks: - <<: *task_template name: client_encrypt - tags: ["ssl", "encrypt"] + tags: ["ssl", "encrypt", "patch_build"] commands: - func: "do setup" - func: "run tests" @@ -5986,7 +6097,7 @@ tasks: - <<: *task_template name: fle - tags: ["encrypt"] + tags: ["encrypt", "patch_build"] commands: - func: "do setup" - func: "load aws test credentials" @@ -5996,7 +6107,7 @@ tasks: - <<: *task_template name: fle2_query_analysis - tags: ["encrypt"] + tags: ["encrypt", "patch_build"] commands: - func: "do setup" - func: "load aws test credentials" @@ -6004,13 +6115,27 @@ tasks: - <<: *task_template name: fle2 - tags: ["encrypt"] + tags: ["encrypt", "patch_build"] commands: - func: "do setup" - func: "run tests" - <<: *task_template name: fle2_sharding + tags: ["encrypt", "patch_build"] + commands: + - func: "do setup" + - func: "run tests" + +- <<: *task_template + name: fle2_high_cardinality + tags: ["encrypt"] + commands: + - func: "do setup" + - func: "run tests" + +- <<: *task_template + name: fle2_sharding_high_cardinality tags: ["encrypt"] commands: - func: "do setup" @@ -6018,7 +6143,7 @@ tasks: - <<: *task_template name: ocsp - tags: ["ssl", "encrypt", "ocsp"] + tags: ["ssl", "encrypt", "ocsp", "patch_build"] commands: - func: "do setup" - func: "run tests" @@ -6334,7 +6459,6 @@ tasks: - *kill_processes - *cleanup_environment - *set_up_venv - - func: "fetch packages" - func: "set up remote credentials" vars: private_key_file: ~/.ssh/kitchen.pem @@ -7242,16 +7366,16 @@ tasks: args: - "./src/evergreen/feature_flag_tags_check.sh" -- name: antithesis_image_build +- name: antithesis_image_push tags: ["antithesis"] # this is not patchable to avoid hitting the docker registry excessively. # When iterating on this task, feel free to make this patchable for # testing purposes. Your image changes will be pushed with the # evergreen-patch tag, so as to not clobber the waterfall. Use the # antithesis_image_tag build parameter to override this if required. - patchable: false depends_on: - name: archive_dist_test_debug + exec_timeout_secs: 7200 commands: - *f_expansions_write - func: "git get project no modules" @@ -7259,13 +7383,6 @@ tasks: - func: "kill processes" - func: "cleanup environment" - func: "set up venv" - - command: s3.get - params: - aws_key: ${aws_key} - aws_secret: ${aws_secret} - remote_file: ${project}/${build_variant}/antithesis_last_push.txt - local_file: antithesis_last_push.txt - bucket: mciuploads - func: "do setup" - command: s3.get params: @@ -7274,22 +7391,9 @@ tasks: remote_file: ${mongo_debugsymbols} bucket: mciuploads local_file: src/mongo-debugsymbols.tgz - - command: subprocess.exec - params: - binary: bash - args: - - "./src/evergreen/antithesis_image_build.sh" - - command: s3.put - params: - optional: true - aws_key: ${aws_key} - aws_secret: ${aws_secret} - local_file: antithesis_next_push.txt - remote_file: ${project}/${build_variant}/antithesis_last_push.txt - bucket: mciuploads - permissions: private - content_type: text/plain - display_name: Last Push Date (seconds since epoch) + - func: "antithesis image build" + - func: "antithesis dry run" + - func: "antithesis image push" - name: generate_buildid_to_debug_symbols_mapping tags: ["symbolizer"] @@ -7317,6 +7421,11 @@ task_groups: - compile_dist_test - <<: *compile_task_group_template + name: compile_upload_benchmarks_TG + tasks: + - compile_upload_benchmarks + +- <<: *compile_task_group_template name: compile_and_archive_dist_test_TG tasks: - compile_dist_test @@ -7401,6 +7510,21 @@ task_groups: - package - <<: *compile_task_group_template + name: compile_test_benchmark_and_package_serial_TG + tasks: + - compile_dist_test + - archive_dist_test + - archive_dist_test_debug + - compile_unittests + - run_unittests + - compile_dbtest + - run_dbtest + - archive_dbtest + - compile_upload_benchmarks + - compile_all + - package + +- <<: *compile_task_group_template name: compile_and_test_TG tasks: - compile_dist_test @@ -7545,6 +7669,7 @@ task_groups: tags: ["stitch"] tasks: - "stitch_support_create_lib" + - <<: *stitch_support_task_group_template name: stitch_support_lib_build_and_test tags: ["stitch"] |