diff options
author | Richard Samuels <richard.l.samuels@gmail.com> | 2021-05-18 16:19:17 -0400 |
---|---|---|
committer | Evergreen Agent <no-reply@evergreen.mongodb.com> | 2021-05-27 18:26:29 +0000 |
commit | 6135750dab50a54e40929913fa043d33f2d6f26e (patch) | |
tree | b6ee72b6b72ed152d2c35b71517ef524d7ff301d | |
parent | d7c69eb4c0453f77047fcc95ed4ab76a6580718c (diff) | |
download | mongo-6135750dab50a54e40929913fa043d33f2d6f26e.tar.gz |
SERVER-57047 Remove requirement to specify workdir when subprocess.exec'ing shell scripts
-rw-r--r-- | etc/evergreen.yml | 170 | ||||
-rw-r--r-- | evergreen/functions/venv_setup.sh | 9 | ||||
-rwxr-xr-x | evergreen/prelude.sh | 13 | ||||
-rw-r--r-- | evergreen/prelude_venv.sh | 5 | ||||
-rw-r--r-- | evergreen/prelude_workdir.sh | 26 |
5 files changed, 39 insertions, 184 deletions
diff --git a/etc/evergreen.yml b/etc/evergreen.yml index 1bf79e297b9..ed82ce59cd8 100644 --- a/etc/evergreen.yml +++ b/etc/evergreen.yml @@ -383,8 +383,6 @@ functions: binary: bash args: - "src/evergreen/functions/files_remove.sh" - env: - workdir: ${workdir} "configure evergreen api credentials": &configure_evergreen_api_credentials command: subprocess.exec @@ -393,8 +391,6 @@ functions: binary: bash args: - "src/evergreen/functions/evergreen_api_credentials_configure.sh" - env: - workdir: ${workdir} "configure selected tests credentials": &configure_selected_tests_credentials command: subprocess.exec @@ -405,12 +401,10 @@ functions: args: - "src/evergreen/functions/selected_tests_credentials_configure.sh" env: - workdir: ${workdir} project: ${project} selected_tests_auth_user: ${selected_tests_auth_user} selected_tests_auth_token: ${selected_tests_auth_token} - "git get project": &git_get_project - command: git.get_project params: @@ -471,8 +465,6 @@ functions: binary: bash args: - "src/evergreen/functions/binaries_extract.sh" - env: - workdir: ${workdir} "check binary version": &check_binary_version command: subprocess.exec @@ -480,8 +472,6 @@ functions: binary: bash args: - "src/evergreen/functions/binary_version_check.sh" - env: - workdir: ${workdir} "fetch benchmarks": &fetch_benchmarks command: s3.get @@ -570,8 +560,6 @@ functions: binary: bash args: - "./src/evergreen/functions/system_resource_info_collect.sh" - env: - workdir: ${workdir} # Run a monitor process as a background, system task to periodically # display how many threads interesting processes are using. @@ -583,8 +571,6 @@ functions: binary: bash args: - "./src/evergreen/functions/process_threads_monitor.sh" - env: - workdir: ${workdir} "set up credentials": &set_up_credentials command: subprocess.exec @@ -593,8 +579,6 @@ functions: silent: true args: - "./src/evergreen/functions/credentials_setup.sh" - env: - workdir: ${workdir} "set up win mount script": - *f_expansions_write @@ -604,8 +588,6 @@ functions: silent: true args: - "./src/evergreen/functions/win_mount_script_setup.sh" - env: - workdir: ${workdir} "set up notary client credentials": - *f_expansions_write @@ -615,8 +597,6 @@ functions: silent: true args: - "./src/evergreen/functions/notary_client_credentials_setup.sh" - env: - workdir: ${workdir} "f_remote_credentials_setup_exec": &set_up_remote_credentials command: subprocess.exec @@ -625,7 +605,6 @@ functions: args: - "./src/evergreen/functions/remote_credentials_setup.sh" env: - workdir: ${workdir} private_key_remote_bash_var: ${private_key_remote} "set up remote credentials": @@ -640,8 +619,6 @@ functions: silent: true args: - "./src/evergreen/functions/bf_suggestion_service_call.sh" - env: - workdir: ${workdir} "upload debugsymbols": &upload_debugsymbols command: s3.put @@ -670,8 +647,6 @@ functions: binary: bash args: - "./src/evergreen/functions/wiredtiger_develop_use.sh" - env: - workdir: ${workdir} "shared scons cache pruning": - *f_expansions_write @@ -681,8 +656,6 @@ functions: binary: bash args: - "./src/evergreen/functions/shared_scons_cache_pruning.sh" - env: - workdir: ${workdir} "umount shared scons directory": - *f_expansions_write @@ -691,8 +664,6 @@ functions: binary: bash args: - "./src/evergreen/functions/shared_scons_directory_umount.sh" - env: - workdir: ${workdir} "get all modified patch files": - *f_expansions_write @@ -701,8 +672,6 @@ functions: binary: bash args: - "./src/evergreen/functions/modified_patch_files_get_all.sh" - env: - workdir: ${workdir} # This function should only be called from patch-build-only tasks. "get added and modified patch files": @@ -712,8 +681,6 @@ functions: binary: bash args: - "./src/evergreen/functions/added_and_modified_patch_files_get.sh" - env: - workdir: ${workdir} "determine resmoke jobs": &determine_resmoke_jobs command: subprocess.exec @@ -721,8 +688,6 @@ functions: binary: bash args: - "./src/evergreen/functions/resmoke_jobs_determine.sh" - env: - workdir: ${workdir} "update resmoke jobs expansions": &update_resmoke_jobs_expansions command: expansions.update @@ -736,8 +701,6 @@ functions: binary: bash args: - "./src/evergreen/functions/task_timeout_determine.sh" - env: - workdir: ${workdir} "update task timeout expansions": &update_task_timeout_expansions command: expansions.update @@ -786,8 +749,8 @@ functions: args: - "src/evergreen/functions/venv_setup.sh" env: - workdir: ${workdir} pip_dir: ${pip_dir} + workdir: ${workdir} "upload pip requirements": &upload_pip_requirements command: s3.put @@ -852,8 +815,6 @@ functions: binary: bash args: - "./src/evergreen/multiversion_setup.sh" - env: - workdir: ${workdir} "do multiversion setup": - *f_expansions_write @@ -866,8 +827,6 @@ functions: binary: bash args: - "./src/evergreen/compiled_binaries_get.sh" - env: - workdir: ${workdir} "generate powercycle tasks": - *f_expansions_write @@ -879,8 +838,6 @@ functions: binary: bash args: - "./src/evergreen/powercycle_tasks_generate.sh" - env: - workdir: ${workdir} - command: archive.targz_pack params: @@ -917,8 +874,6 @@ functions: binary: bash args: - "./src/evergreen/powercycle_sentinel_run.sh" - env: - workdir: ${workdir} "execute resmoke tests": &execute_resmoke_tests command: subprocess.exec @@ -927,8 +882,6 @@ functions: binary: bash args: - "./src/evergreen/resmoke_tests_execute.sh" - env: - workdir: ${workdir} "retrieve generated test configuration": &retrieve_generated_test_configuration command: s3.get @@ -960,8 +913,6 @@ functions: binary: bash args: - "./src/evergreen/selected_tests_generate.sh" - env: - workdir: ${workdir} - command: archive.targz_pack params: target: generate_tasks_config.tgz @@ -994,8 +945,6 @@ functions: binary: bash args: - "./src/evergreen/burn_in_tests_generate.sh" - env: - workdir: ${workdir} - command: archive.targz_pack params: target: burn_in_tags_gen.tgz @@ -1032,8 +981,6 @@ functions: binary: bash args: - "./src/evergreen/randomized_multiversion_tasks_generate.sh" - env: - workdir: ${workdir} - *do_multiversion_setup - command: subprocess.exec @@ -1041,8 +988,6 @@ functions: binary: bash args: - "./src/evergreen/randomized_multiversion_tasks_generate_exclude_tags.sh" - env: - workdir: ${workdir} - command: archive.targz_pack params: target: generate_tasks_config.tgz @@ -1091,8 +1036,6 @@ functions: binary: bash args: - "./src/evergreen/resmoke_tasks_generate.sh" - env: - workdir: ${workdir} - command: archive.targz_pack params: @@ -1202,8 +1145,6 @@ functions: binary: bash args: - "src/evergreen/scons_lint.sh" - env: - workdir: ${workdir} "scons compile": - *f_expansions_write @@ -1213,9 +1154,6 @@ functions: binary: bash args: - "src/evergreen/scons_compile.sh" - env: - workdir: ${workdir} - patch_compile_flags: ${patch_compile_flags} "generate compile expansions": - *f_expansions_write @@ -1224,8 +1162,6 @@ functions: binary: bash args: - "src/evergreen/functions/compile_expansions_generate.sh" - env: - workdir: ${workdir} "apply compile expansions": - command: expansions.update @@ -1250,8 +1186,6 @@ functions: binary: bash args: - "./src/evergreen/do_jepsen_setup/nodes.sh" - env: - workdir: ${workdir} - command: subprocess.exec params: binary: bash @@ -1267,15 +1201,11 @@ functions: binary: bash args: - "./src/evergreen/jepsen_test_run.sh" - env: - workdir: ${workdir} - command: subprocess.exec params: binary: bash args: - "./src/evergreen/jepsen_test_fail.sh" - env: - workdir: ${workdir} "load aws test credentials": - *f_expansions_write @@ -1285,8 +1215,6 @@ functions: silent: true args: - "./src/evergreen/functions/aws_test_credentials_load.sh" - env: - workdir: ${workdir} "generate explicit multiversion tasks": @@ -1308,8 +1236,6 @@ functions: binary: bash args: - "./src/evergreen/explicit_multiversion_tasks_generate.sh" - env: - workdir: ${workdir} - command: archive.targz_pack params: @@ -1363,8 +1289,6 @@ functions: binary: bash args: - "./src/evergreen/implicit_multiversions_tasks_generate.sh" - env: - workdir: ${workdir} - command: archive.targz_pack params: @@ -1416,8 +1340,6 @@ functions: binary: bash args: - "./src/evergreen/fuzzer_tasks_generate.sh" - env: - workdir: ${workdir} - command: archive.targz_pack params: @@ -1455,8 +1377,6 @@ functions: binary: bash args: - "./src/evergreen/jstestfuzz_setup.sh" - env: - workdir: ${workdir} "lint fuzzer sanity patch": - *f_expansions_write @@ -1466,8 +1386,6 @@ functions: binary: bash args: - "./src/evergreen/lint_fuzzer_sanity_patch.sh" - env: - workdir: ${workdir} "lint fuzzer sanity all": - *f_expansions_write @@ -1477,8 +1395,6 @@ functions: binary: bash args: - "./src/evergreen/lint_fuzzer_sanity_all.sh" - env: - workdir: ${workdir} # Used by generator "run jstestfuzz": @@ -1495,16 +1411,12 @@ functions: binary: bash args: - "./src/evergreen/jstestfuzz_run.sh" - env: - workdir: ${workdir} # TODO SERVER-57161 remove this touch command - command: subprocess.exec params: binary: touch args: - "./src/jstestfuzz/out/placeholder.js" - env: - workdir: ${workdir} - command: archive.targz_pack params: target: "jstests.tgz" @@ -1531,8 +1443,6 @@ functions: binary: bash args: - "./src/evergreen/idl_tests_run.sh" - env: - workdir: ${workdir} "run powercycle test": - *f_expansions_write @@ -1542,8 +1452,6 @@ functions: binary: bash args: - "./src/evergreen/powercycle_run_test.sh" - env: - workdir: ${workdir} - command: expansions.update params: @@ -1562,16 +1470,12 @@ functions: binary: bash args: - "./src/evergreen/powercycle_ssh_failure_exit.sh" - env: - workdir: ${workdir} - command: subprocess.exec type: test params: binary: bash args: - "./src/evergreen/powercycle_exit.sh" - env: - workdir: ${workdir} "run packager.py": - *f_expansions_write @@ -1580,8 +1484,6 @@ functions: binary: bash args: - "./src/evergreen/packager.py_run.sh" - env: - workdir: ${workdir} "do snmp setup": command: subprocess.exec @@ -1620,8 +1522,6 @@ functions: binary: bash args: - "./src/evergreen/kitchen_run.sh" - env: - workdir: ${workdir} "set up EC2 instance": &set_up_ec2_instance @@ -1646,8 +1546,6 @@ functions: binary: bash args: - "./src/evergreen/powercycle_check_host.sh" - env: - workdir: ${workdir} - command: expansions.update params: @@ -1659,8 +1557,6 @@ functions: binary: bash args: - "./src/evergreen/powercycle_setup_host.sh" - env: - workdir: ${workdir} ### Process & archive remote EC2 artifacts ### "save powercycle artifacts": &save_powercycle_artifacts @@ -1669,8 +1565,6 @@ functions: binary: bash args: - "./src/evergreen/powercycle_save_artifacts.sh" - env: - workdir: ${workdir} "archive remote EC2 artifacts": &archive_remote_ec2_artifacts command: s3.put @@ -1711,8 +1605,6 @@ functions: binary: bash args: - "./src/evergreen/local_client_logs_tar.sh" - env: - workdir: ${workdir} "archive local client logs": &archive_local_client_logs command: s3.put @@ -1866,8 +1758,6 @@ functions: binary: bash args: - "./src/evergreen/failed_unittests_gather.sh" - env: - workdir: ${workdir} "tar failed unittests": &tar_failed_unittests command: archive.targz_pack @@ -1933,8 +1823,6 @@ functions: binary: bash args: - "./src/evergreen/hang_analyzer.sh" - env: - workdir: ${workdir} "wait for resmoke to shutdown": command: subprocess.exec @@ -2084,8 +1972,6 @@ functions: binary: bash args: - "./src/evergreen/scons_splunk.sh" - env: - workdir: ${workdir} "attach report": command: attach.results @@ -2107,8 +1993,6 @@ functions: binary: bash args: - "./src/evergreen/wiki_page.sh" - env: - workdir: ${workdir} - command: attach.artifacts params: files: @@ -2206,8 +2090,6 @@ tasks: binary: bash args: - "src/evergreen/gen_feature_flags.sh" - env: - workdir: ${workdir} - command: archive.targz_pack params: @@ -2292,8 +2174,6 @@ tasks: binary: bash args: - "./src/evergreen/ninja_compile.sh" - env: - workdir: ${workdir} - name: compile_ninja_next commands: @@ -2313,8 +2193,6 @@ tasks: binary: bash args: - "./src/evergreen/ninja_compile.sh" - env: - workdir: ${workdir} - name: compile_build_tools_next commands: @@ -2333,8 +2211,6 @@ tasks: binary: bash args: - "src/evergreen/libdeps_setup.sh" - env: - workdir: ${workdir} - func: "scons compile" vars: @@ -2349,8 +2225,6 @@ tasks: binary: bash args: - "src/evergreen/libdeps_run.sh" - env: - workdir: ${workdir} - command: s3.put params: @@ -2551,8 +2425,6 @@ tasks: binary: bash args: - "src/evergreen/embedded_sdk_build_cdriver.sh" - env: - workdir: ${workdir} - name: embedded_sdk_install_dev depends_on: @@ -2586,8 +2458,6 @@ tasks: binary: bash args: - "src/evergreen/embedded_sdk_s3_tar.sh" - env: - workdir: ${workdir} # Upload it so we can download from EVG. - command: s3.put @@ -2630,8 +2500,6 @@ tasks: binary: bash args: - "src/evergreen/embedded_sdk_tests_s3_tar.sh" - env: - workdir: ${workdir} # Upload it so we can download from EVG. - command: s3.put @@ -2657,15 +2525,11 @@ tasks: binary: bash args: - "src/evergreen/embedded_sdk_run_tests.sh" - env: - workdir: ${workdir} - command: subprocess.exec params: binary: bash args: - "src/evergreen/embedded_sdk_run_tests_post.sh" - env: - workdir: ${workdir} - name: embedded_sdk_s3_put_latest depends_on: @@ -2724,8 +2588,6 @@ tasks: binary: bash args: - "src/evergreen/stitch_support_create_lib_tar.sh" - env: - workdir: ${workdir} - command: s3.put params: aws_key: ${aws_key} @@ -2765,8 +2627,6 @@ tasks: binary: bash args: - "src/evergreen/stitch_support_run_tests.sh" - env: - workdir: ${workdir} - name: compile_benchmarks depends_on: [] @@ -2890,8 +2750,6 @@ tasks: binary: bash args: - "src/evergreen/lint_yaml.sh" - env: - workdir: ${workdir} - name: lint_shellscripts commands: @@ -2931,8 +2789,6 @@ tasks: binary: bash args: - "src/evergreen/check_idl_compat.sh" - env: - workdir: ${workdir} - name: burn_in_tests_gen commands: @@ -2949,8 +2805,6 @@ tasks: binary: bash args: - "src/evergreen/burn_in_tests.sh" - env: - workdir: ${workdir} - command: archive.targz_pack params: target: src/burn_in_tests_gen.tgz @@ -2989,8 +2843,6 @@ tasks: binary: bash args: - "src/evergreen/burn_in_tests_multiversion.sh" - env: - workdir: ${workdir} - command: archive.targz_pack params: @@ -4478,8 +4330,6 @@ tasks: binary: bash args: - "src/evergreen/external_auth_pip.sh" - env: - workdir: ${workdir} - func: "run tests" vars: resmoke_args: --suites=external_auth --excludeWithAnyTags=requires_domain_controller --storageEngine=wiredTiger @@ -4496,15 +4346,11 @@ tasks: silent: true args: - "src/evergreen/external_auth_aws_setup.sh" - env: - workdir: ${workdir} - command: subprocess.exec params: binary: bash args: - "src/evergreen/external_auth_aws_pip.sh" - env: - workdir: ${workdir} - func: "run tests" vars: resmoke_args: --suites=external_auth_aws --storageEngine=wiredTiger @@ -6277,16 +6123,12 @@ tasks: silent: true args: - "src/evergreen/blackduck_setup.sh" - env: - workdir: ${workdir} - command: subprocess.exec type: test params: binary: bash args: - "src/evergreen/blackduck_hub.sh" - env: - workdir: ${workdir} - name: tla_plus commands: @@ -6361,8 +6203,6 @@ tasks: binary: bash args: - "src/evergreen/package.sh" - env: - workdir: ${workdir} - command: s3.put params: aws_key: ${aws_key} @@ -6512,8 +6352,6 @@ tasks: binary: bash args: - "./src/evergreen/packages_publish.sh" - env: - workdir: ${workdir} - name: push tags: ["publish"] @@ -6577,8 +6415,6 @@ tasks: binary: bash args: - "./src/evergreen/notary_client_run.sh" - env: - workdir: ${workdir} # Put the binaries tarball/zipfile - command: s3.put @@ -7127,8 +6963,6 @@ tasks: binary: bash args: - "./src/evergreen/commit_message_validate.sh" - env: - workdir: ${workdir} - name: check_for_todos exec_timeout_secs: 600 # 10 minute timeout @@ -7144,8 +6978,6 @@ tasks: binary: bash args: - "./src/evergreen/todos_check.sh" - env: - workdir: ${workdir} - <<: *task_template name: mqlrun diff --git a/evergreen/functions/venv_setup.sh b/evergreen/functions/venv_setup.sh index 7eb6c3d14a2..0a901296aee 100644 --- a/evergreen/functions/venv_setup.sh +++ b/evergreen/functions/venv_setup.sh @@ -1,10 +1,11 @@ # exit immediately if virtualenv is not found set -o errexit -evergreen_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" -. "$evergreen_dir/../prelude_python.sh" +evergreen_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)/.." +. "$evergreen_dir/prelude_workdir.sh" +. "$evergreen_dir/prelude_python.sh" -python_loc=$(which ${python:-/opt/mongodbtoolchain/v3/bin/python3}) +python_loc=$(which ${python}) venv_dir="${workdir}/venv" if [ -d "$venv_dir" ]; then exit 0 @@ -34,7 +35,7 @@ toolchain_txt="$pip_dir/toolchain-requirements.txt" # the whole prelude cannot be imported because it requires pyyaml to be # installed, which happens just below. -. "$evergreen_dir/../prelude_venv.sh" +. "$evergreen_dir/prelude_venv.sh" activate_venv echo "Upgrading pip to 21.0.1" diff --git a/evergreen/prelude.sh b/evergreen/prelude.sh index 86f41a53c44..a44b69e7a0a 100755 --- a/evergreen/prelude.sh +++ b/evergreen/prelude.sh @@ -2,20 +2,12 @@ if [[ "$0" == *"/evergreen/prelude.sh" ]]; then echo "ERROR: do not execute this script. source it instead. ie: . prelude.sh" exit 1 fi +set -o errexit # path the directory that contains this script. evergreen_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" -# bootstrapping python assumes that the user has not cd'd before the prelude. -# Ensure that here. -calculated_workdir=$(cd "$evergreen_dir/../.." && echo $PWD) -if [ "$PWD" != "$calculated_workdir" ]; then - echo "ERROR: Your script changed directory before loading prelude.sh. Don't do that" - echo "\$PWD: $PWD" - echo "\$calculated_workdir: $calculated_workdir" - exit 1 -fi - +. "$evergreen_dir/prelude_workdir.sh" . "$evergreen_dir/prelude_python.sh" . "$evergreen_dir/prelude_venv.sh" @@ -79,3 +71,4 @@ function set_sudo { set -o errexit fi } +set +o errexit diff --git a/evergreen/prelude_venv.sh b/evergreen/prelude_venv.sh index 953f4e63d02..728e097ffe0 100644 --- a/evergreen/prelude_venv.sh +++ b/evergreen/prelude_venv.sh @@ -13,7 +13,10 @@ function activate_venv { fi python=python else - python=${python:-/opt/mongodbtoolchain/v3/bin/python3} + if [ -z "$python" ]; then + echo "\$python is unset. This should never happen" + fi + python=${python} fi if [ "Windows_NT" = "$OS" ]; then diff --git a/evergreen/prelude_workdir.sh b/evergreen/prelude_workdir.sh new file mode 100644 index 00000000000..2e39ba1f3b8 --- /dev/null +++ b/evergreen/prelude_workdir.sh @@ -0,0 +1,26 @@ +calculated_workdir=$(cd "$evergreen_dir/../.." && echo "$PWD") +pwd_cygpath="$PWD" +if [ "Windows_NT" = "$OS" ]; then + calculated_workdir=$(cygpath -w "$calculated_workdir") + pwd_cygpath=$(cygpath -w "$pwd_cygpath") +fi +if [ -z "$workdir" ]; then + workdir="$calculated_workdir" + +# skip this test on Windows. The directories will never match due to the many +# different path types present on Windows+Cygwin +elif [ "$workdir" != "$calculated_workdir" ] && [ "Windows_NT" != "$OS" ]; then + # if you move the checkout directory (ex: simple project config project), + # then this assertion will fail in the future. You need to update + # calculated_workdir, and all the relative directories in this file. + echo "\$workdir was specified, but didn't match \$calculated_workdir. Did the directory structure change? Update prelude.sh" + echo "\$workdir: $workdir" + echo "\$calculated_workdir: $calculated_workdir" + exit 1 +fi +if [ "$pwd_cygpath" != "$calculated_workdir" ]; then + echo "ERROR: Your script changed directory before loading prelude.sh. Don't do that" + echo "\$PWD: $PWD" + echo "\$calculated_workdir: $calculated_workdir" + exit 1 +fi |