diff options
| -rw-r--r-- | buildscripts/evergreen_expansions2bash.py | 46 | ||||
| -rwxr-xr-x | buildscripts/generate_compile_expansions.py | 5 | ||||
| -rwxr-xr-x | buildscripts/generate_compile_expansions_shared_cache.py | 9 | ||||
| -rw-r--r-- | etc/evergreen.yml | 129 | ||||
| -rw-r--r-- | etc/expansions.default.yml | 35 | ||||
| -rwxr-xr-x | evergreen/prelude.sh | 106 | ||||
| -rwxr-xr-x | evergreen/scons_compile.sh | 77 | ||||
| -rwxr-xr-x | evergreen/scons_lint.sh | 24 |
8 files changed, 319 insertions, 112 deletions
diff --git a/buildscripts/evergreen_expansions2bash.py b/buildscripts/evergreen_expansions2bash.py new file mode 100644 index 00000000000..ab1a0e09449 --- /dev/null +++ b/buildscripts/evergreen_expansions2bash.py @@ -0,0 +1,46 @@ +"""Convert Evergreen's expansions.yml to an eval-able shell script.""" +import sys +import platform +from shlex import quote +from typing import Any + +import yaml +import click + + +def _error(msg: str) -> None: + print(f"___expansions_error={quote(msg)}") + sys.exit(1) + + +@click.command() +@click.argument("expansions_file", type=str) +@click.argument("defaults_file", type=str) +def _main(expansions_file, defaults_file): + try: + with open(defaults_file) as fh: + defaults = yaml.safe_load(fh) + with open(expansions_file) as fh: + expansions = yaml.safe_load(fh) + + # inject defaults into expansions + for key, value in defaults.items(): + if key not in expansions: + expansions[key] = str(value) + + if not isinstance(expansions, dict): + _error("ERROR: expected to read a dictionary. Has the output format " + "of expansions.write changed?") + + if not expansions: + _error("ERROR: found 0 expansions. This is almost certainly wrong.") + + for key, value in expansions.items(): + print(f"{key}={quote(value)}; ", end="") + + except Exception as ex: # pylint: disable=broad-except + _error(ex) + + +if __name__ == "__main__": + _main() # pylint: disable=no-value-for-parameter diff --git a/buildscripts/generate_compile_expansions.py b/buildscripts/generate_compile_expansions.py index 653f01502cb..cd3b7c63b0c 100755 --- a/buildscripts/generate_compile_expansions.py +++ b/buildscripts/generate_compile_expansions.py @@ -11,6 +11,7 @@ import json import os import re import sys +import shlex import yaml VERSION_JSON = "version.json" @@ -94,8 +95,8 @@ def generate_scons_cache_expansions(): scons_cache_mode = "nolinked" if os.getenv("USE_SCONS_CACHE") not in (None, False, "false", ""): - expansions["scons_cache_args"] = "--cache={0} --cache-dir='{1}'".format( - scons_cache_mode, default_cache_path) + expansions["scons_cache_args"] = "--cache={0} --cache-dir={1}".format( + scons_cache_mode, shlex.quote(default_cache_path)) return expansions diff --git a/buildscripts/generate_compile_expansions_shared_cache.py b/buildscripts/generate_compile_expansions_shared_cache.py index 685eedb629f..4f31341c806 100755 --- a/buildscripts/generate_compile_expansions_shared_cache.py +++ b/buildscripts/generate_compile_expansions_shared_cache.py @@ -11,6 +11,7 @@ import json import os import re import sys +import shlex import yaml VERSION_JSON = "version.json" @@ -99,8 +100,8 @@ def generate_scons_cache_expansions(): shared_mount_root = '/efs' default_cache_path = os.path.join(shared_mount_root, system_uuid, "scons-cache") expansions["scons_cache_path"] = default_cache_path - expansions["scons_cache_args"] = "--cache={0} --cache-dir='{1}'".format( - scons_cache_mode, default_cache_path) + expansions["scons_cache_args"] = "--cache={0} --cache-dir={1}".format( + scons_cache_mode, shlex.quote(default_cache_path)) # Local shared cache - host-based elif os.getenv("SCONS_CACHE_SCOPE") == "local": @@ -112,8 +113,8 @@ def generate_scons_cache_expansions(): default_cache_path = os.path.join(default_cache_path_base, system_uuid) expansions["scons_cache_path"] = default_cache_path - expansions["scons_cache_args"] = "--cache={0} --cache-dir='{1}'".format( - scons_cache_mode, default_cache_path) + expansions["scons_cache_args"] = "--cache={0} --cache-dir={1}".format( + scons_cache_mode, shlex.quote(default_cache_path)) # No cache else: # Anything else is 'none' diff --git a/etc/evergreen.yml b/etc/evergreen.yml index e0773a93087..c9eb6b9b696 100644 --- a/etc/evergreen.yml +++ b/etc/evergreen.yml @@ -382,7 +382,6 @@ variables: ####################################### functions: - "remove files": &remove_files command: shell.exec params: @@ -433,7 +432,7 @@ functions: "git get project": &git_get_project command: git.get_project params: - directory: ${git_project_directory|src} + directory: src revisions: # for each module include revision as <module_name> : ${<module_name>_rev} enterprise: ${enterprise_rev} wtdevelop: ${wtdevelop_rev} @@ -1097,8 +1096,6 @@ functions: # exit immediately if virtualenv is not found set -o errexit - virtualenv_loc=$(which ${virtualenv|virtualenv}) - python_loc=$(which ${python|/opt/mongodbtoolchain/v3/bin/python3}) venv_dir="${workdir}/venv" "$python_loc" -m venv --system-site-packages "$venv_dir" @@ -1712,112 +1709,32 @@ functions: - "./src/evergreen/check_run_tests_infrastructure_failure.sh" "scons lint": - - command: shell.exec + - command: expansions.write + params: + file: expansions.yml + - command: subprocess.exec type: test params: - working_dir: src - shell: bash - script: | - set -o errexit - set -o verbose - - ${activate_virtualenv} - export MYPY="$( - if which cygpath 2>/dev/null; then - PATH+=":$(cypath "${workdir}")/venv_3/Scripts" - else - PATH+=":${workdir}/venv_3/bin" - fi - PATH+=':/opt/mongodbtoolchain/v3/bin' - which mypy - )" - echo "Found mypy executable at '$MYPY'" - export extra_flags="" - if [[ ${is_patch|false} == "true" ]]; then - extra_flags="--lint-scope=changed" - fi - - ${compile_env|} python3 ./buildscripts/scons.py ${compile_flags|} $extra_flags --stack-size=1024 GITDIFFFLAGS="${revision}" REVISION="${revision|}" ENTERPRISE_REV="${enterprise_rev|}" ${targets} + binary: bash + args: + - "src/evergreen/scons_lint.sh" + env: + python: ${python} + workdir: ${workdir} "scons compile": - command: shell.exec - type: test - params: - working_dir: src - shell: bash - script: | - set -o errexit - set -o verbose - - rm -rf ${install_directory|/data/mongo-install-directory} - - # Use hardlinks to reduce the disk space impact of installing - # all of the binaries and associated debug info. - - # The expansion here is a workaround to let us set a different install-action - # for tasks that don't support the one we set here. A better plan would be - # to support install-action for Ninja builds directly. - # TODO: https://jira.mongodb.org/browse/SERVER-48203 - extra_args="--install-action=${task_install_action|hardlink}" - - # By default, limit link jobs to one quarter of our overall -j - # concurrency unless locally overridden. We do this because in - # static link environments, the memory consumption of each - # link job is so high that without constraining the number of - # links we are likely to OOM or thrash the machine. Dynamic - # builds, where htis is not a concern, override this value. - echo "Changing SCons to run with --jlink=${num_scons_link_jobs_available|0.25}" - extra_args="$extra_args --jlink=${num_scons_link_jobs_available|0.25}" - - if [ "${scons_cache_scope|}" = "shared" ]; then - extra_args="$extra_args --cache-debug=scons_cache.log" - fi - - # Conditionally enable scons time debugging - if [ "${show_scons_timings|true}" = "true" ]; then - extra_args="$extra_args --debug=time" - fi - - # Build packages where the upload tasks expect them - if [ -n "${git_project_directory|}" ]; then - extra_args="$extra_args PKGDIR=${git_project_directory}" - else - extra_args="$extra_args PKGDIR=${workdir}/src" - fi - - # If we are doing a patch build or we are building a non-push - # build on the waterfall, then we don't need the --release - # flag. Otherwise, this is potentially a build that "leaves - # the building", so we do want that flag. The non --release - # case should auto enale the faster decider when - # applicable. Furthermore, for the non --release cases we can - # accelerate the build slightly for situations where we invoke - # SCons multiple times on the same machine by allowing SCons - # to assume that implicit dependencies are cacheable across - # runs. - if [ "${is_patch|false}" = "true" ] || [ -z "${push_bucket|}" ] || [ "${compiling_for_test|false}" = "true" ]; then - extra_args="$extra_args --implicit-cache --build-fast-and-loose=on" - else - extra_args="$extra_args --release" - fi - - if [ "Windows_NT" = "$OS" ]; then - vcvars="$(vswhere -latest -property installationPath | tr '\\' '/' | dos2unix.exe)/VC/Auxiliary/Build/" - export PATH="$(echo "$(cd "$vcvars" && cmd /C "vcvarsall.bat amd64 && C:/cygwin/bin/bash -c 'echo \$PATH'")" | tail -n +6)":$PATH - fi - - ${activate_virtualenv} - - ${compile_env|} $python ./buildscripts/scons.py \ - ${compile_flags|} ${task_compile_flags|} ${task_compile_flags_extra|} \ - ${scons_cache_args|} $extra_args \ - ${targets} MONGO_VERSION=${version} ${patch_compile_flags|} || exit_status=$? - - # If compile fails we do not run any tests - if [[ $exit_status -ne 0 ]]; then - touch ${skip_tests} - fi - exit $exit_status + - command: expansions.write + params: + file: expansions.yml + - command: subprocess.exec + type: test + params: + binary: bash + args: + - "src/evergreen/scons_compile.sh" + env: + python: ${python} + workdir: ${workdir} "generate compile expansions": command: shell.exec diff --git a/etc/expansions.default.yml b/etc/expansions.default.yml new file mode 100644 index 00000000000..43797f8dd98 --- /dev/null +++ b/etc/expansions.default.yml @@ -0,0 +1,35 @@ +report_file: src/report.json +archive_file: src/archive.json +decompress: tar xzvf +private_key_file: /dev/null +aws_profile_remote: default +resmoke_jobs_factor: 1 +resmoke_jobs_max: 0 +timeout_secs: 0 +exec_timeout_secs: 0 +python: /opt/mongodbtoolchain/v3/bin/python3 +multiversion_edition: base +multiversion_platform: linux_x86_64 +multiversion_architecture: x86_64 +disable_unit_tests: false +skip_tests: /dev/null +resmoke_jobs: 1 +should_shuffle: true +continue_on_failure: true +install_dir: dist-test/bin +is_patch: false +install_directory: /data/mongo-install-directory +task_install_action: hardlink +num_scons_link_jobs_available: 0.25 +show_scons_timings: true +compiling_for_test: false +scons_cache_mode: nolinked +is_commit_queue: false +use_scons_cache: false +npm_command: jstestfuzz +connection_attempts: 25 +tar: tar +cmake_path: /opt/cmake/bin/cmake +content_type: application/x-gzip +jstestfuzz_concurrent_num_files: 10 +curator_release: "latest" diff --git a/evergreen/prelude.sh b/evergreen/prelude.sh new file mode 100755 index 00000000000..823b29449af --- /dev/null +++ b/evergreen/prelude.sh @@ -0,0 +1,106 @@ +if [[ "$0" == *"/evergreen/prelude.sh" ]]; then + echo "ERROR: do not execute this script. source it instead. ie: . prelude.sh" + exit 1 +fi + +# path the directory that contains this script. +evergreen_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" + +# bootstrapping python assumes that the user has not cd'd before the prelude. +# Ensure that here. +calculated_workdir=$(cd "$evergreen_dir/../.." && echo $PWD) +if [ "$PWD" != "$calculated_workdir" ]; then + echo "ERROR: Your script changed directory before loading prelude.sh. Don't do that" + echo "\$PWD: $PWD" + echo "\$calculated_workdir: $calculated_workdir" + exit 1 +fi + +function activate_venv { + set -e + # check if virtualenv is set up + if [ -d "${workdir}/venv" ]; then + if [ "Windows_NT" = "$OS" ]; then + # Need to quote the path on Windows to preserve the separator. + . "${workdir}/venv/Scripts/activate" 2>/tmp/activate_error.log + else + . ${workdir}/venv/bin/activate 2>/tmp/activate_error.log + fi + if [ $? -ne 0 ]; then + echo "Failed to activate virtualenv: $(cat /tmp/activate_error.log)" + fi + python=python + else + python=${python:-/opt/mongodbtoolchain/v3/bin/python3} + fi + + if [ "Windows_NT" = "$OS" ]; then + export PYTHONPATH="$PYTHONPATH;$(cygpath -w ${workdir}/src)" + else + export PYTHONPATH="$PYTHONPATH:${workdir}/src" + fi + + echo "python set to $(which $python)" + set +e +} + +expansions_yaml="$evergreen_dir/../../expansions.yml" +expansions_default_yaml="$evergreen_dir/../etc/expansions.default.yml" +script="$evergreen_dir/../buildscripts/evergreen_expansions2bash.py" +if [ "Windows_NT" = "$OS" ]; then + expansions_yaml=$(cygpath -w "$expansions_yaml") + expansions_default_yaml=$(cygpath -w "$expansions_default_yaml") + script=$(cygpath -w "$script") +fi + +eval $(activate_venv >/dev/null && $python "$script" "$expansions_yaml" "$expansions_default_yaml") +if [ -n "$___expansions_error" ]; then + echo $___expansions_error + exit 1 +fi +unset expansions_yaml +unset expansions_default_yaml +unset script +unset evergreen_dir + +function add_nodejs_to_path { + # Add node and npm binaries to PATH + if [ "Windows_NT" = "$OS" ]; then + # An "npm" directory might not have been created in %APPDATA% by the Windows installer. + # Work around the issue by specifying a different %APPDATA% path. + # See: https://github.com/nodejs/node-v0.x-archive/issues/8141 + export APPDATA=${workdir}/npm-app-data + export PATH="$PATH:/cygdrive/c/Program Files (x86)/nodejs" # Windows location + # TODO: this is to work around BUILD-8652 + cd "$(pwd -P | sed 's,cygdrive/c/,cygdrive/z/,')" + else + export PATH="$PATH:/opt/node/bin" + fi +} + +function posix_workdir { + if [ "Windows_NT" = "$OS" ]; then + echo $(cygpath -u "${workdir}") + else + echo ${workdir} + fi +} + +function set_sudo { + set -o >/tmp/settings.log + set +o errexit + grep errexit /tmp/settings.log | grep on + errexit_on=$? + # Set errexit "off". + set +o errexit + sudo= + # Use sudo, if it is supported. + sudo date >/dev/null 2>&1 + if [ $? -eq 0 ]; then + sudo=sudo + fi + # Set errexit "on", if previously enabled. + if [ $errexit_on -eq 0 ]; then + set -o errexit + fi +} diff --git a/evergreen/scons_compile.sh b/evergreen/scons_compile.sh new file mode 100755 index 00000000000..a69c07b29e1 --- /dev/null +++ b/evergreen/scons_compile.sh @@ -0,0 +1,77 @@ +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" +. "$DIR/prelude.sh" + +cd src + +set -o errexit +set -o verbose + +rm -rf ${install_directory} + +# Use hardlinks to reduce the disk space impact of installing +# all of the binaries and associated debug info. + +# The expansion here is a workaround to let us set a different install-action +# for tasks that don't support the one we set here. A better plan would be +# to support install-action for Ninja builds directly. +# TODO: https://jira.mongodb.org/browse/SERVER-48203 +extra_args="--install-action=${task_install_action}" + +# By default, limit link jobs to one quarter of our overall -j +# concurrency unless locally overridden. We do this because in +# static link environments, the memory consumption of each +# link job is so high that without constraining the number of +# links we are likely to OOM or thrash the machine. Dynamic +# builds, where htis is not a concern, override this value. +echo "Changing SCons to run with --jlink=${num_scons_link_jobs_available}" +extra_args="$extra_args --jlink=${num_scons_link_jobs_available}" + +if [ "${scons_cache_scope}" = "shared" ]; then + extra_args="$extra_args --cache-debug=scons_cache.log" +fi + +# Conditionally enable scons time debugging +if [ "${show_scons_timings}" = "true" ]; then + extra_args="$extra_args --debug=time" +fi + +# Build packages where the upload tasks expect them +if [ -n "${git_project_directory}" ]; then + extra_args="$extra_args PKGDIR='${git_project_directory}'" +else + extra_args="$extra_args PKGDIR='${workdir}/src'" +fi + +# If we are doing a patch build or we are building a non-push +# build on the waterfall, then we don't need the --release +# flag. Otherwise, this is potentially a build that "leaves +# the building", so we do want that flag. The non --release +# case should auto enale the faster decider when +# applicable. Furthermore, for the non --release cases we can +# accelerate the build slightly for situations where we invoke +# SCons multiple times on the same machine by allowing SCons +# to assume that implicit dependencies are cacheable across +# runs. +if [ "${is_patch}" = "true" ] || [ -z "${push_bucket}" ] || [ "${compiling_for_test}" = "true" ]; then + extra_args="$extra_args --implicit-cache --build-fast-and-loose=on" +else + extra_args="$extra_args --release" +fi + +if [ "Windows_NT" = "$OS" ]; then + vcvars="$(vswhere -latest -property installationPath | tr '\\' '/' | dos2unix.exe)/VC/Auxiliary/Build/" + export PATH="$(echo "$(cd "$vcvars" && cmd /C "vcvarsall.bat amd64 && C:/cygwin/bin/bash -c 'echo \$PATH'")" | tail -n +6)":$PATH +fi + +activate_venv + +eval ${compile_env} $python ./buildscripts/scons.py \ + ${compile_flags} ${task_compile_flags} ${task_compile_flags_extra} \ + ${scons_cache_args} $extra_args \ + ${targets} MONGO_VERSION=${version} ${patch_compile_flags} || exit_status=$? + +# If compile fails we do not run any tests +if [[ $exit_status -ne 0 ]]; then + touch ${skip_tests} +fi +exit $exit_status diff --git a/evergreen/scons_lint.sh b/evergreen/scons_lint.sh new file mode 100755 index 00000000000..708ef4168a5 --- /dev/null +++ b/evergreen/scons_lint.sh @@ -0,0 +1,24 @@ +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" +. "$DIR/prelude.sh" + +cd src + +set -o errexit +set -o verbose + +activate_venv +export MYPY="$( + if which cygpath 2>/dev/null; then + PATH+=":$(cypath "${workdir}")/venv_3/Scripts" + else + PATH+=":${workdir}/venv_3/bin" + fi + PATH+=':/opt/mongodbtoolchain/v3/bin' + which mypy +)" +echo "Found mypy executable at '$MYPY'" +export extra_flags="" +if [[ ${is_patch} == "true" ]]; then + extra_flags="--lint-scope=changed" +fi +eval ${compile_env} python3 ./buildscripts/scons.py ${compile_flags} $extra_flags --stack-size=1024 GITDIFFFLAGS="${revision}" REVISION="${revision}" ENTERPRISE_REV="${enterprise_rev}" ${targets} |