SERVER-55300 Implement expansions handling for standalone shell

3 files changed, 207 insertions, 0 deletions

diff --git a/evergreen/prelude.sh b/evergreen/prelude.sh
new file mode 100755
index 00000000000..823b29449af
--- /dev/null
+++ b/evergreen/prelude.sh
@@ -0,0 +1,106 @@
+if [[ "$0" == *"/evergreen/prelude.sh" ]]; then
+  echo "ERROR: do not execute this script. source it instead. ie: . prelude.sh"
+  exit 1
+fi
+
+# path the directory that contains this script.
+evergreen_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
+
+# bootstrapping python assumes that the user has not cd'd before the prelude.
+# Ensure that here.
+calculated_workdir=$(cd "$evergreen_dir/../.." && echo $PWD)
+if [ "$PWD" != "$calculated_workdir" ]; then
+  echo "ERROR: Your script changed directory before loading prelude.sh. Don't do that"
+  echo "\$PWD: $PWD"
+  echo "\$calculated_workdir: $calculated_workdir"
+  exit 1
+fi
+
+function activate_venv {
+  set -e
+  # check if virtualenv is set up
+  if [ -d "${workdir}/venv" ]; then
+    if [ "Windows_NT" = "$OS" ]; then
+      # Need to quote the path on Windows to preserve the separator.
+      . "${workdir}/venv/Scripts/activate" 2>/tmp/activate_error.log
+    else
+      . ${workdir}/venv/bin/activate 2>/tmp/activate_error.log
+    fi
+    if [ $? -ne 0 ]; then
+      echo "Failed to activate virtualenv: $(cat /tmp/activate_error.log)"
+    fi
+    python=python
+  else
+    python=${python:-/opt/mongodbtoolchain/v3/bin/python3}
+  fi
+
+  if [ "Windows_NT" = "$OS" ]; then
+    export PYTHONPATH="$PYTHONPATH;$(cygpath -w ${workdir}/src)"
+  else
+    export PYTHONPATH="$PYTHONPATH:${workdir}/src"
+  fi
+
+  echo "python set to $(which $python)"
+  set +e
+}
+
+expansions_yaml="$evergreen_dir/../../expansions.yml"
+expansions_default_yaml="$evergreen_dir/../etc/expansions.default.yml"
+script="$evergreen_dir/../buildscripts/evergreen_expansions2bash.py"
+if [ "Windows_NT" = "$OS" ]; then
+  expansions_yaml=$(cygpath -w "$expansions_yaml")
+  expansions_default_yaml=$(cygpath -w "$expansions_default_yaml")
+  script=$(cygpath -w "$script")
+fi
+
+eval $(activate_venv >/dev/null && $python "$script" "$expansions_yaml" "$expansions_default_yaml")
+if [ -n "$___expansions_error" ]; then
+  echo $___expansions_error
+  exit 1
+fi
+unset expansions_yaml
+unset expansions_default_yaml
+unset script
+unset evergreen_dir
+
+function add_nodejs_to_path {
+  # Add node and npm binaries to PATH
+  if [ "Windows_NT" = "$OS" ]; then
+    # An "npm" directory might not have been created in %APPDATA% by the Windows installer.
+    # Work around the issue by specifying a different %APPDATA% path.
+    # See: https://github.com/nodejs/node-v0.x-archive/issues/8141
+    export APPDATA=${workdir}/npm-app-data
+    export PATH="$PATH:/cygdrive/c/Program Files (x86)/nodejs" # Windows location
+    # TODO: this is to work around BUILD-8652
+    cd "$(pwd -P | sed 's,cygdrive/c/,cygdrive/z/,')"
+  else
+    export PATH="$PATH:/opt/node/bin"
+  fi
+}
+
+function posix_workdir {
+  if [ "Windows_NT" = "$OS" ]; then
+    echo $(cygpath -u "${workdir}")
+  else
+    echo ${workdir}
+  fi
+}
+
+function set_sudo {
+  set -o >/tmp/settings.log
+  set +o errexit
+  grep errexit /tmp/settings.log | grep on
+  errexit_on=$?
+  # Set errexit "off".
+  set +o errexit
+  sudo=
+  # Use sudo, if it is supported.
+  sudo date >/dev/null 2>&1
+  if [ $? -eq 0 ]; then
+    sudo=sudo
+  fi
+  # Set errexit "on", if previously enabled.
+  if [ $errexit_on -eq 0 ]; then
+    set -o errexit
+  fi
+}
diff --git a/evergreen/scons_compile.sh b/evergreen/scons_compile.sh
new file mode 100755
index 00000000000..a69c07b29e1
--- /dev/null
+++ b/evergreen/scons_compile.sh
@@ -0,0 +1,77 @@
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
+. "$DIR/prelude.sh"
+
+cd src
+
+set -o errexit
+set -o verbose
+
+rm -rf ${install_directory}
+
+# Use hardlinks to reduce the disk space impact of installing
+# all of the binaries and associated debug info.
+
+# The expansion here is a workaround to let us set a different install-action
+# for tasks that don't support the one we set here. A better plan would be
+# to support install-action for Ninja builds directly.
+# TODO: https://jira.mongodb.org/browse/SERVER-48203
+extra_args="--install-action=${task_install_action}"
+
+# By default, limit link jobs to one quarter of our overall -j
+# concurrency unless locally overridden. We do this because in
+# static link environments, the memory consumption of each
+# link job is so high that without constraining the number of
+# links we are likely to OOM or thrash the machine. Dynamic
+# builds, where htis is not a concern, override this value.
+echo "Changing SCons to run with --jlink=${num_scons_link_jobs_available}"
+extra_args="$extra_args --jlink=${num_scons_link_jobs_available}"
+
+if [ "${scons_cache_scope}" = "shared" ]; then
+  extra_args="$extra_args --cache-debug=scons_cache.log"
+fi
+
+# Conditionally enable scons time debugging
+if [ "${show_scons_timings}" = "true" ]; then
+  extra_args="$extra_args --debug=time"
+fi
+
+# Build packages where the upload tasks expect them
+if [ -n "${git_project_directory}" ]; then
+  extra_args="$extra_args PKGDIR='${git_project_directory}'"
+else
+  extra_args="$extra_args PKGDIR='${workdir}/src'"
+fi
+
+# If we are doing a patch build or we are building a non-push
+# build on the waterfall, then we don't need the --release
+# flag. Otherwise, this is potentially a build that "leaves
+# the building", so we do want that flag. The non --release
+# case should auto enale the faster decider when
+# applicable. Furthermore, for the non --release cases we can
+# accelerate the build slightly for situations where we invoke
+# SCons multiple times on the same machine by allowing SCons
+# to assume that implicit dependencies are cacheable across
+# runs.
+if [ "${is_patch}" = "true" ] || [ -z "${push_bucket}" ] || [ "${compiling_for_test}" = "true" ]; then
+  extra_args="$extra_args --implicit-cache --build-fast-and-loose=on"
+else
+  extra_args="$extra_args --release"
+fi
+
+if [ "Windows_NT" = "$OS" ]; then
+  vcvars="$(vswhere -latest -property installationPath | tr '\\' '/' | dos2unix.exe)/VC/Auxiliary/Build/"
+  export PATH="$(echo "$(cd "$vcvars" && cmd /C "vcvarsall.bat amd64 && C:/cygwin/bin/bash -c 'echo \$PATH'")" | tail -n +6)":$PATH
+fi
+
+activate_venv
+
+eval ${compile_env} $python ./buildscripts/scons.py \
+  ${compile_flags} ${task_compile_flags} ${task_compile_flags_extra} \
+  ${scons_cache_args} $extra_args \
+  ${targets} MONGO_VERSION=${version} ${patch_compile_flags} || exit_status=$?
+
+# If compile fails we do not run any tests
+if [[ $exit_status -ne 0 ]]; then
+  touch ${skip_tests}
+fi
+exit $exit_status
diff --git a/evergreen/scons_lint.sh b/evergreen/scons_lint.sh
new file mode 100755
index 00000000000..708ef4168a5
--- /dev/null
+++ b/evergreen/scons_lint.sh
@@ -0,0 +1,24 @@
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
+. "$DIR/prelude.sh"
+
+cd src
+
+set -o errexit
+set -o verbose
+
+activate_venv
+export MYPY="$(
+  if which cygpath 2>/dev/null; then
+    PATH+=":$(cypath "${workdir}")/venv_3/Scripts"
+  else
+    PATH+=":${workdir}/venv_3/bin"
+  fi
+  PATH+=':/opt/mongodbtoolchain/v3/bin'
+  which mypy
+)"
+echo "Found mypy executable at '$MYPY'"
+export extra_flags=""
+if [[ ${is_patch} == "true" ]]; then
+  extra_flags="--lint-scope=changed"
+fi
+eval ${compile_env} python3 ./buildscripts/scons.py ${compile_flags} $extra_flags --stack-size=1024 GITDIFFFLAGS="${revision}" REVISION="${revision}" ENTERPRISE_REV="${enterprise_rev}" ${targets}