SERVER-55626 Extract shell scripts from server yaml part 3

24 files changed, 416 insertions, 27 deletions

diff --git a/evergreen/functions/added_and_modified_patch_files_get.sh b/evergreen/functions/added_and_modified_patch_files_get.sh
new file mode 100755
index 00000000000..9b230f3b081
--- /dev/null
+++ b/evergreen/functions/added_and_modified_patch_files_get.sh
@@ -0,0 +1,14 @@
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
+. "$DIR/../prelude.sh"
+
+cd src
+
+set -o verbose
+set -o errexit
+
+git diff HEAD --name-only --line-prefix="${workdir}/src/" --diff-filter=d >>modified_and_created_patch_files.txt
+if [ -d src/mongo/db/modules/enterprise ]; then
+  pushd src/mongo/db/modules/enterprise
+  git diff HEAD --name-only --line-prefix="${workdir}/src/src/mongo/db/modules/enterprise/" --diff-filter=d >>~1/modified_and_created_patch_files.txt
+  popd
+fi
diff --git a/evergreen/functions/bf_suggestion_service_call.sh b/evergreen/functions/bf_suggestion_service_call.sh
new file mode 100755
index 00000000000..d022bf38cf7
--- /dev/null
+++ b/evergreen/functions/bf_suggestion_service_call.sh
@@ -0,0 +1,22 @@
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
+. "$DIR/../prelude.sh"
+
+cd src
+
+report_file="report.json"
+# Check if the report file exists and has failures.
+if [ -f $report_file ] && grep -Eq "\"failures\": [1-9]" $report_file; then
+  # Calling the BF Suggestion server endpoint to start feature extraction.
+  payload="{\"task_id\": \"${task_id}\", \"execution\": ${execution}}"
+  echo "Sending task info to the BF suggestion service"
+  # The --user option is passed through stdin to avoid showing in process list.
+  user_option="--user ${bfsuggestion_user}:${bfsuggestion_password}"
+  curl --header "Content-Type: application/json" \
+    --data "$payload" \
+    --max-time 10 \
+    --silent \
+    --show-error \
+    --config - \
+    https://bfsuggestion.corp.mongodb.com/tasks <<<$user_option
+  echo "Request to BF Suggestion service status: $?"
+fi
diff --git a/evergreen/functions/binaries_extract.sh b/evergreen/functions/binaries_extract.sh
new file mode 100755
index 00000000000..efa7089ec3f
--- /dev/null
+++ b/evergreen/functions/binaries_extract.sh
@@ -0,0 +1,7 @@
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
+. "$DIR/../prelude.sh"
+
+cd src
+
+set -o errexit
+${decompress} mongo-binaries.tgz
diff --git a/evergreen/functions/binary_version_check.sh b/evergreen/functions/binary_version_check.sh
new file mode 100755
index 00000000000..f5f1a74487c
--- /dev/null
+++ b/evergreen/functions/binary_version_check.sh
@@ -0,0 +1,16 @@
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
+. "$DIR/../prelude.sh"
+
+cd src
+
+set -o errexit
+mongo_binary=dist-test/bin/mongo${exe}
+activate_venv
+bin_ver=$($python -c "import yaml; print(yaml.safe_load(open('compile_expansions.yml'))['version']);" | tr -d '[ \r\n]')
+# Due to SERVER-23810, we cannot use $mongo_binary --quiet --nodb --eval "version();"
+mongo_ver=$($mongo_binary --version | perl -pe '/version v([^\"]*)/; $_ = $1;' | tr -d '[ \r\n]')
+# The versions must match
+if [ "$bin_ver" != "$mongo_ver" ]; then
+  echo "The mongo version is $mongo_ver, expected version is $bin_ver"
+  exit 1
+fi
diff --git a/evergreen/process_code_coverage_data.sh b/evergreen/functions/code_coverage_data_process.sh
index c23114884ff..c23114884ff 100755
--- a/evergreen/process_code_coverage_data.sh
+++ b/evergreen/functions/code_coverage_data_process.sh
diff --git a/evergreen/functions/credentials_setup.sh b/evergreen/functions/credentials_setup.sh
new file mode 100755
index 00000000000..4f0605a45bf
--- /dev/null
+++ b/evergreen/functions/credentials_setup.sh
@@ -0,0 +1,12 @@
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
+. "$DIR/../prelude.sh"
+
+cd src
+
+cat >mci.buildlogger <<END_OF_CREDS
+slavename='${slave}'
+passwd='${passwd}'
+builder='MCI_${build_variant}'
+build_num=${builder_num}
+build_phase='${task_name}_${execution}'
+END_OF_CREDS
diff --git a/evergreen/functions/evergreen_api_credentials_configure.sh b/evergreen/functions/evergreen_api_credentials_configure.sh
new file mode 100755
index 00000000000..c42f637a2c9
--- /dev/null
+++ b/evergreen/functions/evergreen_api_credentials_configure.sh
@@ -0,0 +1,11 @@
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
+. "$DIR/../prelude.sh"
+
+cd src
+
+# Create the Evergreen API credentials
+cat >.evergreen.yml <<END_OF_CREDS
+api_server_host: https://evergreen.mongodb.com/api
+api_key: "${evergreen_api_key}"
+user: "${evergreen_api_user}"
+END_OF_CREDS
diff --git a/evergreen/functions/files_remove.sh b/evergreen/functions/files_remove.sh
new file mode 100755
index 00000000000..a2c905cdedd
--- /dev/null
+++ b/evergreen/functions/files_remove.sh
@@ -0,0 +1,12 @@
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
+. "$DIR/../prelude.sh"
+
+if [ -z "${files}" ]; then
+  exit 0
+fi
+for file in ${files}; do
+  if [ -f "$file" ]; then
+    echo "Removing file $file"
+    rm -f $file
+  fi
+done
diff --git a/evergreen/cleanup_fuse_watchdog.sh b/evergreen/functions/fuse_watchdog_cleanup.sh
index dd6dd587515..dd6dd587515 100755
--- a/evergreen/cleanup_fuse_watchdog.sh
+++ b/evergreen/functions/fuse_watchdog_cleanup.sh
diff --git a/evergreen/functions/modified_patch_files_get_all.sh b/evergreen/functions/modified_patch_files_get_all.sh
new file mode 100755
index 00000000000..5e79cc6ce0d
--- /dev/null
+++ b/evergreen/functions/modified_patch_files_get_all.sh
@@ -0,0 +1,19 @@
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
+. "$DIR/../prelude.sh"
+
+cd src
+
+set -o verbose
+set -o errexit
+
+# For patch builds gather the modified patch files.
+if [ "${is_patch}" = "true" ]; then
+  # Get list of patched files
+  git diff HEAD --name-only >>patch_files.txt
+  if [ -d src/mongo/db/modules/enterprise ]; then
+    pushd src/mongo/db/modules/enterprise
+    # Update the patch_files.txt in the mongo repo.
+    git diff HEAD --name-only >>~1/patch_files.txt
+    popd
+  fi
+fi
diff --git a/evergreen/functions/notary_client_credentials_setup.sh b/evergreen/functions/notary_client_credentials_setup.sh
new file mode 100755
index 00000000000..a1dab18b8cc
--- /dev/null
+++ b/evergreen/functions/notary_client_credentials_setup.sh
@@ -0,0 +1,14 @@
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
+. "$DIR/../prelude.sh"
+
+cd src
+
+set -o errexit
+
+cat <<EOF >notary_env.sh
+export NOTARY_TOKEN=${signing_auth_token_50}
+export BARQUE_USERNAME=${barque_user}
+export BARQUE_API_KEY=${barque_api_key}
+EOF
+
+echo "${signing_auth_token_50}" >signing_auth_token
diff --git a/evergreen/functions/process_threads_monitor.sh b/evergreen/functions/process_threads_monitor.sh
new file mode 100755
index 00000000000..c28269a1552
--- /dev/null
+++ b/evergreen/functions/process_threads_monitor.sh
@@ -0,0 +1,46 @@
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
+. "$DIR/../prelude.sh"
+
+proc_list="(java|lein|mongo|python|_test$|_test\.exe$)"
+if [ "Windows_NT" = "$OS" ]; then
+  get_pids() {
+    proc_pids=$(tasklist /fo:csv \
+      | awk -F'","' '{x=$1; gsub("\"","",x); print $2, x}' \
+      | grep -iE $1 \
+      | cut -f1 -d ' ')
+  }
+  get_process_info() {
+    proc_name=""
+    proc_info=$(wmic process where "ProcessId=\"$1\"" get "Name,ProcessId,ThreadCount" /format:csv 2>/dev/null | grep $1)
+    if [ ! -z $proc_info ]; then
+      proc_name=$(echo $proc_info | cut -f2 -d ',')
+      proc_threads=$(echo $proc_info | cut -f4 -d ',')
+    fi
+  }
+else
+  get_pids() { proc_pids=$(pgrep $1); }
+  get_process_info() {
+    proc_name=$(ps -p $1 -o comm=)
+    # /proc is available on Linux platforms
+    if [ -f /proc/$1/status ]; then
+      set_sudo
+      proc_threads=$($sudo grep Threads /proc/$1/status | sed "s/\s//g" | cut -f2 -d ":")
+    else
+      proc_threads=$(ps -AM $1 | grep -vc PID)
+    fi
+  }
+fi
+while [ 1 ]; do
+  get_pids $proc_list
+  if [ ! -z "$proc_pids" ]; then
+    printf "Running process/thread counter\n"
+    printf "PROCESS\tPID\tTHREADS\n"
+  fi
+  for pid in $proc_pids; do
+    get_process_info $pid
+    if [ ! -z "$proc_name" ]; then
+      printf "$proc_name\t$pid\t$proc_threads\n"
+    fi
+  done
+  sleep 60
+done
diff --git a/evergreen/functions/resmoke_jobs_determine.sh b/evergreen/functions/resmoke_jobs_determine.sh
new file mode 100755
index 00000000000..036e8b00154
--- /dev/null
+++ b/evergreen/functions/resmoke_jobs_determine.sh
@@ -0,0 +1,16 @@
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
+. "$DIR/../prelude.sh"
+
+cd src
+
+set -o verbose
+set -o errexit
+
+activate_venv
+$python buildscripts/evergreen_resmoke_job_count.py \
+  --taskName ${task_name} \
+  --buildVariant ${build_variant} \
+  --distro ${distro_id} \
+  --jobFactor ${resmoke_jobs_factor} \
+  --jobsMax ${resmoke_jobs_max} \
+  --outFile resmoke_jobs_expansion.yml
diff --git a/evergreen/functions/selected_tests_credentials_configure.sh b/evergreen/functions/selected_tests_credentials_configure.sh
new file mode 100755
index 00000000000..964cdc128a0
--- /dev/null
+++ b/evergreen/functions/selected_tests_credentials_configure.sh
@@ -0,0 +1,12 @@
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
+. "$DIR/../prelude.sh"
+
+cd src
+
+# Create the Evergreen API credentials
+cat >.selected_tests.yml <<END_OF_CREDS
+url: "https://selected-tests.server-tig.prod.corp.mongodb.com"
+project: "${project}"
+auth_user: "${selected_tests_auth_user}"
+auth_token: "${selected_tests_auth_token}"
+END_OF_CREDS
diff --git a/evergreen/functions/shared_scons_cache_pruning.sh b/evergreen/functions/shared_scons_cache_pruning.sh
new file mode 100755
index 00000000000..87219355a82
--- /dev/null
+++ b/evergreen/functions/shared_scons_cache_pruning.sh
@@ -0,0 +1,76 @@
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
+. "$DIR/../prelude.sh"
+
+cd src
+
+set -o errexit
+set -o verbose
+# removes files from the shared scons cache.
+
+# Only prune on master branch
+if [[ "${project}" == "mongodb-mongo-master" ]]; then
+
+  set +o errexit
+
+  if [ "Windows_NT" = "$OS" ]; then
+    ./win_mount.sh
+  else
+    mount | grep "\/efs" >/dev/null
+  fi
+  if [ $? -eq 0 ]; then
+    echo "Shared cache is already mounted"
+  else
+    echo "Shared cache - mounting file system"
+    if [ "Windows_NT" = "$OS" ]; then
+      ./win_mount.sh
+    else
+      sudo mount /efs
+    fi
+  fi
+  set -o errexit
+
+  if [ "Windows_NT" = "$OS" ]; then
+    cache_folder=/cygdrive/x
+  else
+    cache_folder=/efs
+  fi
+  dirs=$(ls -l $cache_folder | grep -v total | awk '{print $NF}')
+
+  echo "Pruning shared SCons directories"
+
+  for dir in $dirs; do
+    echo "Pruning $cache_folder/$dir/scons-cache"
+
+    if [ -e $cache_folder/$dir/info/distro_name ]; then
+      distro=$(cat $cache_folder/$dir/info/distro_name)
+    fi
+
+    # Set cache sizes by distro
+    case $distro in
+    ubuntu1604 | ubuntu1804 | rhel62 | rhel70)
+      cache_size=600
+      ;;
+    *)
+      # default
+      cache_size=400
+      ;;
+    esac
+
+    if [ "Windows_NT" = "$OS" ]; then
+      echo "dir="$dir
+      python buildscripts/scons_cache_prune.py --cache-dir x:/$dir/scons-cache --cache-size $cache_size --prune-ratio 1.0
+    else
+      sudo python buildscripts/scons_cache_prune.py --cache-dir /efs/$dir/scons-cache --cache-size $cache_size --prune-ratio 1.0
+    fi
+    echo ""
+  done
+
+  if [ "Windows_NT" = "$OS" ]; then
+    net use X: /delete || true
+  else
+    sudo umount /efs || true
+  fi
+
+else
+  echo "Not on master, shared SCons cache pruning skipped"
+fi
diff --git a/evergreen/functions/shared_scons_directory_umount.sh b/evergreen/functions/shared_scons_directory_umount.sh
new file mode 100755
index 00000000000..6d50da0fdae
--- /dev/null
+++ b/evergreen/functions/shared_scons_directory_umount.sh
@@ -0,0 +1,19 @@
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
+. "$DIR/../prelude.sh"
+
+cd src
+
+set -o errexit
+set -o verbose
+
+if [ "${disable_shared_scons_cache}" = true ]; then
+  exit
+fi
+if [ "${scons_cache_scope}" = "shared" ]; then
+  if [ "Windows_NT" = "$OS" ]; then
+    net use X: /delete || true
+  else
+    set_sudo
+    $sudo umount /efs || true
+  fi
+fi
diff --git a/evergreen/functions/system_resource_info_collect.sh b/evergreen/functions/system_resource_info_collect.sh
new file mode 100755
index 00000000000..5fb5c69ca29
--- /dev/null
+++ b/evergreen/functions/system_resource_info_collect.sh
@@ -0,0 +1,5 @@
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
+. "$DIR/../prelude.sh"
+
+activate_venv
+$python buildscripts/collect_resource_info.py -o system_resource_info.json -i 5
diff --git a/evergreen/functions/task_timeout_determine.sh b/evergreen/functions/task_timeout_determine.sh
new file mode 100644
index 00000000000..7a43e097939
--- /dev/null
+++ b/evergreen/functions/task_timeout_determine.sh
@@ -0,0 +1,15 @@
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
+. "$DIR/../prelude.sh"
+
+cd src
+
+set -o verbose
+set -o errexit
+activate_venv
+$python buildscripts/evergreen_task_timeout.py \
+  --task-name ${task_name} \
+  --build-variant ${build_variant} \
+  --evg-alias '${alias}' \
+  --timeout ${timeout_secs} \
+  --exec-timeout ${exec_timeout_secs} \
+  --out-file task_timeout_expansions.yml
diff --git a/evergreen/functions/venv_setup.sh b/evergreen/functions/venv_setup.sh
new file mode 100644
index 00000000000..68febe34d71
--- /dev/null
+++ b/evergreen/functions/venv_setup.sh
@@ -0,0 +1,43 @@
+# exit immediately if virtualenv is not found
+set -o errexit
+
+evergreen_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
+. "$evergreen_dir/../prelude_python.sh"
+
+python_loc=$(which ${python:-/opt/mongodbtoolchain/v3/bin/python3})
+venv_dir="${workdir}/venv"
+if [ -d "$venv_dir" ]; then
+  exit 0
+fi
+"$python_loc" -m venv --system-site-packages "$venv_dir"
+
+# venv creates its Scripts/activate file with CLRF endings, which
+# cygwin bash does not like. dos2unix it
+# (See https://bugs.python.org/issue32451)
+if [ "Windows_NT" = "$OS" ]; then
+  dos2unix "${workdir}/venv/Scripts/activate"
+fi
+
+export VIRTUAL_ENV_DISABLE_PROMPT=yes
+
+# Not all git get project calls clone into ${workdir}/src so we allow
+# callers to tell us where the pip requirements files are.
+pip_dir="${pip_dir}"
+if [[ -z $pip_dir ]]; then
+  # Default to most common location
+  pip_dir="${workdir}/src/etc/pip"
+fi
+
+# Same as above we have to use quotes to preserve the
+# Windows path separator
+toolchain_txt="$pip_dir/toolchain-requirements.txt"
+
+# the whole prelude cannot be imported because it requires pyyaml to be
+# installed, which happens just below.
+. "$evergreen_dir/../prelude_venv.sh"
+
+activate_venv
+echo "Upgrading pip to 21.0.1"
+python -m pip install "pip==21.0.1"
+python -m pip install -r "$toolchain_txt" -q
+python -m pip freeze >pip-requirements.txt
diff --git a/evergreen/functions/win_mount_script_setup.sh b/evergreen/functions/win_mount_script_setup.sh
new file mode 100755
index 00000000000..187777e213b
--- /dev/null
+++ b/evergreen/functions/win_mount_script_setup.sh
@@ -0,0 +1,9 @@
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
+. "$DIR/../prelude.sh"
+
+cd src
+
+cat <<EOF >win_mount.sh
+net use X: '\\\\${win_scons_endpoint}\\share' /USER:"wincache.build.com\${win_scons_user}" '${win_scons_pass}'
+EOF
+chmod +x win_mount.sh
diff --git a/evergreen/functions/wiredtiger_develop_use.sh b/evergreen/functions/wiredtiger_develop_use.sh
new file mode 100755
index 00000000000..db913aac939
--- /dev/null
+++ b/evergreen/functions/wiredtiger_develop_use.sh
@@ -0,0 +1,14 @@
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
+. "$DIR/../prelude.sh"
+
+cd src
+
+set -o errexit
+set -o verbose
+if [ "${use_wt_develop}" = "true" ]; then
+  cd src/third_party
+  for wtdir in dist examples ext lang src test tools; do
+    rm -rf wiredtiger/$wtdir
+    mv wtdevelop/$wtdir wiredtiger/
+  done
+fi
diff --git a/evergreen/prelude.sh b/evergreen/prelude.sh
index 823b29449af..86f41a53c44 100755
--- a/evergreen/prelude.sh
+++ b/evergreen/prelude.sh
@@ -16,33 +16,8 @@ if [ "$PWD" != "$calculated_workdir" ]; then
   exit 1
 fi
 
-function activate_venv {
-  set -e
-  # check if virtualenv is set up
-  if [ -d "${workdir}/venv" ]; then
-    if [ "Windows_NT" = "$OS" ]; then
-      # Need to quote the path on Windows to preserve the separator.
-      . "${workdir}/venv/Scripts/activate" 2>/tmp/activate_error.log
-    else
-      . ${workdir}/venv/bin/activate 2>/tmp/activate_error.log
-    fi
-    if [ $? -ne 0 ]; then
-      echo "Failed to activate virtualenv: $(cat /tmp/activate_error.log)"
-    fi
-    python=python
-  else
-    python=${python:-/opt/mongodbtoolchain/v3/bin/python3}
-  fi
-
-  if [ "Windows_NT" = "$OS" ]; then
-    export PYTHONPATH="$PYTHONPATH;$(cygpath -w ${workdir}/src)"
-  else
-    export PYTHONPATH="$PYTHONPATH:${workdir}/src"
-  fi
-
-  echo "python set to $(which $python)"
-  set +e
-}
+. "$evergreen_dir/prelude_python.sh"
+. "$evergreen_dir/prelude_venv.sh"
 
 expansions_yaml="$evergreen_dir/../../expansions.yml"
 expansions_default_yaml="$evergreen_dir/../etc/expansions.default.yml"
diff --git a/evergreen/prelude_python.sh b/evergreen/prelude_python.sh
new file mode 100644
index 00000000000..d6192c253bc
--- /dev/null
+++ b/evergreen/prelude_python.sh
@@ -0,0 +1,5 @@
+if [ "Windows_NT" = "$OS" ]; then
+  python='/cygdrive/c/python/python37/python.exe'
+else
+  python="/opt/mongodbtoolchain/v3/bin/python3"
+fi
diff --git a/evergreen/prelude_venv.sh b/evergreen/prelude_venv.sh
new file mode 100644
index 00000000000..953f4e63d02
--- /dev/null
+++ b/evergreen/prelude_venv.sh
@@ -0,0 +1,27 @@
+function activate_venv {
+  set -e
+  # check if virtualenv is set up
+  if [ -d "${workdir}/venv" ]; then
+    if [ "Windows_NT" = "$OS" ]; then
+      # Need to quote the path on Windows to preserve the separator.
+      . "${workdir}/venv/Scripts/activate" 2>/tmp/activate_error.log
+    else
+      . ${workdir}/venv/bin/activate 2>/tmp/activate_error.log
+    fi
+    if [ $? -ne 0 ]; then
+      echo "Failed to activate virtualenv: $(cat /tmp/activate_error.log)"
+    fi
+    python=python
+  else
+    python=${python:-/opt/mongodbtoolchain/v3/bin/python3}
+  fi
+
+  if [ "Windows_NT" = "$OS" ]; then
+    export PYTHONPATH="$PYTHONPATH;$(cygpath -w ${workdir}/src)"
+  else
+    export PYTHONPATH="$PYTHONPATH:${workdir}/src"
+  fi
+
+  echo "python set to $(which $python)"
+  set +e
+}