SERVER-55626 Extract normal shell scripts from evergreen.yml part 3

28 files changed, 636 insertions, 492 deletions

diff --git a/buildscripts/evergreen_expansions2bash.py b/buildscripts/evergreen_expansions2bash.py
index fe6966a0f1d..f89b245129f 100644
--- a/buildscripts/evergreen_expansions2bash.py
+++ b/buildscripts/evergreen_expansions2bash.py
@@ -4,15 +4,20 @@ import platform
 from shlex import quote
 from typing import Any
 
-import yaml
-import click
-
 
 def _error(msg: str) -> None:
     print(f"___expansions_error={quote(msg)}")
     sys.exit(1)
 
 
+try:
+    import yaml
+    import click
+except ModuleNotFoundError:
+    _error("ERROR: Failed to import a dependency. This is almost certainly because "
+           "the task did not initialize the venv immediately after cloning the repository.")
+
+
 def _load_defaults(defaults_file: str) -> dict:
     with open(defaults_file) as fh:
         defaults = yaml.safe_load(fh)
diff --git a/etc/evergreen.yml b/etc/evergreen.yml
index 1080644af5e..8bbc4de28f7 100644
--- a/etc/evergreen.yml
+++ b/etc/evergreen.yml
@@ -137,6 +137,7 @@ variables:
   setup_task:
   - func: "apply compile expansions"
   - func: "set task expansion macros"
+  - func: "f_expansions_write"
   teardown_task:
   - func: "attach scons logs"
   - func: "attach report"
@@ -174,18 +175,21 @@ variables:
   - func: "git get project"
   - func: "kill processes"
   - func: "cleanup environment"
-  - func: "get all modified patch files"
   - func: "set task expansion macros"
   # The python virtual environment is installed in ${workdir}, which is created in
   # "set up venv".
   - func: "set up venv"
   - func: "upload pip requirements"
+  - func: "get all modified patch files"
+  - func: "f_expansions_write"
   - func: "configure evergreen api credentials"
   - func: "get buildnumber"
+  - func: "f_expansions_write"
   - func: "set up credentials"
   - func: "use WiredTiger develop" # noop if ${use_wt_develop} is not "true"
   - func: "set up win mount script"
   - func: "generate compile expansions"
+  - func: "f_expansions_write"
   teardown_group:
   - func: "umount shared scons directory"
   - func: "cleanup environment"
@@ -255,6 +259,7 @@ variables:
     - func: "set up venv"
     - func: "upload pip requirements"
     - func: "get buildnumber"
+    - func: "f_expansions_write"
     - func: "set up win mount script"
     - func: "generate compile expansions"
   teardown_group:
@@ -362,52 +367,42 @@ variables:
 #######################################
 
 functions:
-  "remove files": &remove_files
-    command: shell.exec
+  "f_expansions_write": &f_expansions_write
+    command: expansions.write
     params:
-      shell: bash
-      script: |
-        if [ -z "${files}" ]; then
-          exit 0
-        fi
-        for file in ${files}
-        do
-          if [ -f "$file" ]; then
-            echo "Removing file $file"
-            rm -f $file
-          fi
-        done
+      file: expansions.yml
+      redacted: true
+
+  "remove files":
+    - *f_expansions_write
+    - command: subprocess.exec
+      params:
+        binary: bash
+        args:
+          - "src/evergreen/functions/files_remove.sh"
+        env:
+          workdir: ${workdir}
 
   "configure evergreen api credentials": &configure_evergreen_api_credentials
-    command: shell.exec
+    command: subprocess.exec
     type: test
     params:
-      shell: bash
-      working_dir: src
-      silent: true
-      script: |
-        # Create the Evergreen API credentials
-        cat > .evergreen.yml <<END_OF_CREDS
-        api_server_host: https://evergreen.mongodb.com/api
-        api_key: "${evergreen_api_key}"
-        user: "${evergreen_api_user}"
-        END_OF_CREDS
+      binary: bash
+      args:
+        - "src/evergreen/functions/evergreen_api_credentials_configure.sh"
+      env:
+        workdir: ${workdir}
 
   "configure selected tests credentials": &configure_selected_tests_credentials
-    command: shell.exec
+    command: subprocess.exec
     type: test
     params:
-      working_dir: src
+      binary: bash
       silent: true
-      shell: bash
-      script: |
-        # Create the Evergreen API credentials
-        cat > .selected_tests.yml <<END_OF_CREDS
-        url: "https://selected-tests.server-tig.prod.corp.mongodb.com"
-        project: "${project}"
-        auth_user: "${selected_tests_auth_user}"
-        auth_token: "${selected_tests_auth_token}"
-        END_OF_CREDS
+      args:
+        - "src/evergreen/functions/selected_tests_credentials_configure.sh"
+      env:
+        workdir: ${workdir}
 
   "git get project": &git_get_project
     - command: git.get_project
@@ -464,31 +459,22 @@ functions:
       local_file: src/mongo-binaries.tgz
 
   "extract binaries": &extract_binaries
-    command: shell.exec
+    command: subprocess.exec
     params:
-      working_dir: src
-      shell: bash
-      script: |
-        set -o errexit
-        ${decompress|tar xzvf} mongo-binaries.tgz
+      binary: bash
+      args:
+        - "src/evergreen/functions/binaries_extract.sh"
+      env:
+        workdir: ${workdir}
 
   "check binary version": &check_binary_version
-    command: shell.exec
+    command: subprocess.exec
     params:
-      working_dir: src
-      shell: bash
-      script: |
-        set -o errexit
-        mongo_binary=dist-test/bin/mongo${exe}
-        ${activate_virtualenv}
-        bin_ver=$($python -c "import yaml; print(yaml.safe_load(open('compile_expansions.yml'))['version']);" | tr -d '[ \r\n]')
-        # Due to SERVER-23810, we cannot use $mongo_binary --quiet --nodb --eval "version();"
-        mongo_ver=$($mongo_binary --version | perl -pe '/version v([^\"]*)/; $_ = $1;' | tr -d '[ \r\n]')
-        # The versions must match
-        if [ "$bin_ver" != "$mongo_ver" ]; then
-          echo "The mongo version is $mongo_ver, expected version is $bin_ver"
-          exit 1
-        fi
+      binary: bash
+      args:
+        - "src/evergreen/functions/binary_version_check.sh"
+      env:
+        workdir: ${workdir}
 
   "fetch benchmarks": &fetch_benchmarks
     command: s3.get
@@ -567,88 +553,41 @@ functions:
       system_log: true
       binary: bash
       args:
-        - "./src/evergreen/run_diskstats.sh"
+        - "./src/evergreen/functions/run_diskstats.sh"
 
   "collect system resource info": &collect_system_resource_info
-    command: shell.exec
+    command: subprocess.exec
     params:
-      working_dir: src
       background: true
       system_log: true
-      shell: bash
-      script: |
-        ${activate_virtualenv}
-        $python buildscripts/collect_resource_info.py -o system_resource_info.json -i 5
+      binary: bash
+      args:
+        - "./src/evergreen/functions/system_resource_info_collect.sh"
+      env:
+        workdir: ${workdir}
 
   # Run a monitor process as a background, system task to periodically
   # display how many threads interesting processes are using.
   "monitor process threads": &monitor_process_threads
-    command: shell.exec
+    command: subprocess.exec
     params:
       background: true
       system_log: true
-      shell: bash
-      script: |
-        proc_list="(java|lein|mongo|python|_test$|_test\.exe$)"
-        if [ "Windows_NT" = "$OS" ]; then
-          get_pids() {
-            proc_pids=$(tasklist /fo:csv |
-                        awk -F'","' '{x=$1; gsub("\"","",x); print $2, x}' |
-                        grep -iE $1 |
-                        cut -f1 -d ' ');
-          }
-          get_process_info() {
-            proc_name="";
-            proc_info=$(wmic process where "ProcessId=\"$1\"" get "Name,ProcessId,ThreadCount" /format:csv 2> /dev/null | grep $1);
-            if [ ! -z $proc_info ]; then
-              proc_name=$(echo $proc_info | cut -f2 -d ',');
-              proc_threads=$(echo $proc_info | cut -f4 -d ',');
-            fi;
-          }
-        else
-          get_pids() { proc_pids=$(pgrep $1); }
-          get_process_info() {
-            proc_name=$(ps -p $1 -o comm=);
-            # /proc is available on Linux platforms
-            if [ -f /proc/$1/status ]; then
-              ${set_sudo}
-              proc_threads=$($sudo grep Threads /proc/$1/status | sed "s/\s//g" | cut -f2 -d ":");
-            else
-              proc_threads=$(ps -AM $1 | grep -vc PID);
-            fi;
-          }
-        fi
-        while [ 1 ]
-        do
-          get_pids $proc_list
-          if [ ! -z "$proc_pids" ]; then
-            printf "Running process/thread counter\n"
-            printf "PROCESS\tPID\tTHREADS\n"
-          fi
-          for pid in $proc_pids
-          do
-            get_process_info $pid
-            if [ ! -z "$proc_name" ]; then
-              printf "$proc_name\t$pid\t$proc_threads\n"
-            fi
-          done
-          sleep 60
-        done
+      binary: bash
+      args:
+        - "./src/evergreen/functions/process_threads_monitor.sh"
+      env:
+        workdir: ${workdir}
 
   "set up credentials": &set_up_credentials
-    command: shell.exec
+    command: subprocess.exec
     params:
-      working_dir: src
+      binary: bash
       silent: true
-      shell: bash
-      script: |
-        cat > mci.buildlogger <<END_OF_CREDS
-        slavename='${slave}'
-        passwd='${passwd}'
-        builder='MCI_${build_variant}'
-        build_num=${builder_num}
-        build_phase='${task_name}_${execution}'
-        END_OF_CREDS
+      args:
+        - "./src/evergreen/functions/credentials_setup.sh"
+      env:
+        workdir: ${workdir}
 
   "set up win mount script": &set_up_win_mount_script
     command: shell.exec
@@ -662,22 +601,16 @@ functions:
         EOF
         chmod +x win_mount.sh
 
-  "set up notary client credentials": &set_up_notary_client_credentials
-    command: shell.exec
-    params:
-      working_dir: src
-      silent: true
-      shell: bash
-      script: |
-        set -o errexit
-
-        cat <<EOF > notary_env.sh
-        export NOTARY_TOKEN=${signing_auth_token_50}
-        export BARQUE_USERNAME=${barque_user}
-        export BARQUE_API_KEY=${barque_api_key}
-        EOF
-
-        echo "${signing_auth_token_50}" > signing_auth_token
+  "set up notary client credentials":
+    - *f_expansions_write
+    - command: subprocess.exec
+      params:
+        binary: bash
+        silent: true
+        args:
+          - "./src/evergreen/functions/notary_client_credentials_setup.sh"
+        env:
+          workdir: ${workdir}
 
   "set up remote credentials": &set_up_remote_credentials
     command: shell.exec
@@ -728,29 +661,15 @@ functions:
         EOF
 
   "call BF Suggestion service":
-    command: shell.exec
-    params:
-      working_dir: src
-      shell: bash
-      silent: true
-      script: |
-        report_file="report.json"
-        # Check if the report file exists and has failures.
-        if [ -f $report_file ] && grep -Eq "\"failures\": [1-9]" $report_file; then
-          # Calling the BF Suggestion server endpoint to start feature extraction.
-          payload="{\"task_id\": \"${task_id}\", \"execution\": ${execution}}"
-          echo "Sending task info to the BF suggestion service"
-          # The --user option is passed through stdin to avoid showing in process list.
-          user_option="--user ${bfsuggestion_user}:${bfsuggestion_password}"
-          curl --header "Content-Type: application/json" \
-               --data "$payload" \
-               --max-time 10 \
-               --silent \
-               --show-error \
-               --config - \
-               https://bfsuggestion.corp.mongodb.com/tasks <<< $user_option
-          echo "Request to BF Suggestion service status: $?"
-        fi
+    - *f_expansions_write
+    - command: subprocess.exec
+      params:
+        binary: bash
+        silent: true
+        args:
+          - "./src/evergreen/functions/bf_suggestion_service_call.sh"
+        env:
+          workdir: ${workdir}
 
   "upload debugsymbols": &upload_debugsymbols
     command: s3.put
@@ -773,177 +692,65 @@ functions:
       local_file: src/mongo-debugsymbols.tgz
 
   "use WiredTiger develop":
-    command: shell.exec
-    params:
-      working_dir: src
-      shell: bash
-      script: |
-        set -o errexit
-        set -o verbose
-        if [ "${use_wt_develop|}" = "true" ]; then
-        cd src/third_party
-        for wtdir in dist examples ext lang src test tools ; do
-          rm -rf wiredtiger/$wtdir
-          mv wtdevelop/$wtdir wiredtiger/
-        done
-        fi
+    - *f_expansions_write
+    - command: subprocess.exec
+      params:
+        binary: bash
+        args:
+          - "./src/evergreen/functions/wiredtiger_develop_use.sh"
+        env:
+          workdir: ${workdir}
 
   "shared scons cache pruning":
-    command: shell.exec
-    type: system
-    params:
-      shell: bash
-      working_dir: src
-      script: |
-        set -o errexit
-        set -o verbose
-        # removes files from the shared scons cache.
-
-        # Only prune on master branch
-        if [[ "${project}" == "mongodb-mongo-master" ]]; then
-
-
-          set +o errexit
-
-          if [ "Windows_NT" = "$OS" ]; then
-            ./win_mount.sh
-          else
-            mount | grep "\/efs" > /dev/null
-          fi
-          if [ $? -eq 0 ]; then
-            echo "Shared cache is already mounted"
-          else
-            echo "Shared cache - mounting file system"
-            if [ "Windows_NT" = "$OS" ]; then
-              ./win_mount.sh
-            else
-              sudo mount /efs
-            fi
-          fi
-          set -o errexit
-
-          if [ "Windows_NT" = "$OS" ]; then
-            cache_folder=/cygdrive/x
-          else
-            cache_folder=/efs
-          fi
-          dirs=$(ls -l $cache_folder | grep -v total | awk '{print $NF}')
-
-          echo "Pruning shared SCons directories"
-
-          for dir in $dirs; do
-            echo "Pruning $cache_folder/$dir/scons-cache"
-
-            if [ -e $cache_folder/$dir/info/distro_name ]; then
-              distro=$(cat $cache_folder/$dir/info/distro_name)
-            fi
-
-            # Set cache sizes by distro
-            case $distro in
-                 ubuntu1604|ubuntu1804|rhel62|rhel70)
-                      cache_size=600
-                      ;;
-                 *)
-                      # default
-                      cache_size=400
-                      ;;
-            esac
-
-            if [ "Windows_NT" = "$OS" ]; then
-              echo "dir="$dir
-              python buildscripts/scons_cache_prune.py --cache-dir x:/$dir/scons-cache --cache-size $cache_size --prune-ratio 1.0
-            else
-              sudo python buildscripts/scons_cache_prune.py --cache-dir /efs/$dir/scons-cache --cache-size $cache_size --prune-ratio 1.0
-            fi
-            echo ""
-          done
-
-          if [ "Windows_NT" = "$OS" ]; then
-            net use X: /delete || true
-          else
-            sudo umount /efs || true
-          fi
-
-        else
-          echo "Not on master, shared SCons cache pruning skipped"
-        fi
+    - *f_expansions_write
+    - command: subprocess.exec
+      type: system
+      params:
+        binary: bash
+        args:
+          - "./src/evergreen/functions/shared_scons_cache_pruning.sh"
+        env:
+          workdir: ${workdir}
 
   "umount shared scons directory":
-    command: shell.exec
-    params:
-      working_dir: src
-      shell: bash
-      script: |
-        set -o errexit
-        set -o verbose
-
-        if [ "${disable_shared_scons_cache}" = true ]; then
-            exit
-        fi
-        if [ "${scons_cache_scope}" = "shared" ]; then
-            if [ "Windows_NT" = "$OS" ]; then
-                net use X: /delete || true
-            else
-                ${set_sudo}
-                $sudo umount /efs || true
-            fi
-        fi
+    - *f_expansions_write
+    - command: subprocess.exec
+      params:
+        binary: bash
+        args:
+          - "./src/evergreen/functions/shared_scons_directory_umount.sh"
+        env:
+          workdir: ${workdir}
 
   "get all modified patch files":
-    command: shell.exec
-    params:
-      working_dir: src
-      shell: bash
-      script: |
-        set -o verbose
-        set -o errexit
-
-        # For patch builds gather the modified patch files.
-        if [ "${is_patch}" = "true" ]; then
-          # Get list of patched files
-          git diff HEAD --name-only >> patch_files.txt
-          if [ -d src/mongo/db/modules/enterprise ]; then
-            pushd src/mongo/db/modules/enterprise
-            # Update the patch_files.txt in the mongo repo.
-            git diff HEAD --name-only >> ~1/patch_files.txt
-            popd
-          fi
-        fi
+    - *f_expansions_write
+    - command: subprocess.exec
+      params:
+        binary: bash
+        args:
+          - "./src/evergreen/functions/modified_patch_files_get_all.sh"
+        env:
+          workdir: ${workdir}
 
   # This function should only be called from patch-build-only tasks.
   "get added and modified patch files":
-    command: shell.exec
-    params:
-      working_dir: src
-      shell: bash
-      script: |
-        set -o verbose
-        set -o errexit
-
-        git diff HEAD --name-only --line-prefix="${workdir}/src/" --diff-filter=d >> modified_and_created_patch_files.txt
-        if [ -d src/mongo/db/modules/enterprise ]; then
-          pushd src/mongo/db/modules/enterprise
-          git diff HEAD --name-only --line-prefix="${workdir}/src/src/mongo/db/modules/enterprise/" --diff-filter=d >> ~1/modified_and_created_patch_files.txt
-          popd
-        fi
+    - *f_expansions_write
+    - command: subprocess.exec
+      params:
+        binary: bash
+        args:
+          - "./src/evergreen/functions/added_and_modified_patch_files_get.sh"
+        env:
+          workdir: ${workdir}
 
   "determine resmoke jobs": &determine_resmoke_jobs
-    command: shell.exec
+    command: subprocess.exec
     params:
-      working_dir: src
-      shell: bash
-      script: |
-        set -o verbose
-        set -o errexit
-
-        ${activate_virtualenv}
-        $python buildscripts/evergreen_resmoke_job_count.py \
-          --taskName ${task_name}                           \
-          --buildVariant ${build_variant}                   \
-          --distro ${distro_id}                             \
-          --jobFactor ${resmoke_jobs_factor|1}              \
-          --jobsMax ${resmoke_jobs_max|0}                   \
-          --outFile resmoke_jobs_expansion.yml
+      binary: bash
+      args:
+        - "./src/evergreen/functions/resmoke_jobs_determine.sh"
+      env:
+        workdir: ${workdir}
 
   "update resmoke jobs expansions": &update_resmoke_jobs_expansions
     command: expansions.update
@@ -952,22 +759,13 @@ functions:
       file: src/resmoke_jobs_expansion.yml
 
   "determine task timeout": &determine_task_timeout
-    command: shell.exec
+    command: subprocess.exec
     params:
-      working_dir: src
-      shell: bash
-      script: |
-        set -o verbose
-        set -o errexit
-
-        ${activate_virtualenv}
-        $python buildscripts/evergreen_task_timeout.py \
-          --task-name ${task_name}                     \
-          --build-variant ${build_variant}             \
-          --evg-alias '${alias|}'                      \
-          --timeout ${timeout_secs|0}                  \
-          --exec-timeout ${exec_timeout_secs|0}        \
-          --out-file task_timeout_expansions.yml
+      binary: bash
+      args:
+        - "./src/evergreen/functions/task_timeout_determine.sh"
+      env:
+        workdir: ${workdir}
 
   "update task timeout expansions": &update_task_timeout_expansions
     command: expansions.update
@@ -1069,42 +867,14 @@ functions:
         value: skip_test-${build_id}
 
   "set up venv": &set_up_venv
-    command: shell.exec
+    command: subprocess.exec
     params:
-        shell: bash
-        script: |
-          # exit immediately if virtualenv is not found
-          set -o errexit
-
-          python_loc=$(which ${python|/opt/mongodbtoolchain/v3/bin/python3})
-          venv_dir="${workdir}/venv"
-          "$python_loc" -m venv "$venv_dir"
-
-          # venv creates its Scripts/activate file with CLRF endings, which
-          # cygwin bash does not like. dos2unix it
-          # (See https://bugs.python.org/issue32451)
-          if [ "Windows_NT" = "$OS" ]; then
-            dos2unix "${workdir}/venv/Scripts/activate"
-          fi
-
-          export VIRTUAL_ENV_DISABLE_PROMPT=yes
-
-          # Not all git get project calls clone into ${workdir}/src so we allow
-          # callers to tell us where the pip requirements files are.
-          pip_dir="${pip_dir}"
-          if [[ -z $pip_dir ]]; then
-            # Default to most common location
-            pip_dir="${workdir}/src/etc/pip"
-          fi
-
-          # Same as above we have to use quotes to preserve the
-          # Windows path separator
-          toolchain_txt="$pip_dir/toolchain-requirements.txt"
-          ${activate_virtualenv}
-          echo "Upgrading pip to 21.0.1"
-          python -m pip install "pip==21.0.1"
-          python -m pip install -r "$toolchain_txt" -q
-          python -m pip freeze > pip-requirements.txt
+      binary: bash
+      args:
+        - "src/evergreen/functions/venv_setup.sh"
+      env:
+        workdir: ${workdir}
+        pip_dir: ${pip_dir}
 
   "upload pip requirements": &upload_pip_requirements
     command: s3.put
@@ -1125,15 +895,17 @@ functions:
       file: src/perf.json
 
   "do setup":
+  - *set_up_venv
+  - *upload_pip_requirements
   - *set_task_expansion_macros
+  - *f_expansions_write
   - *fetch_artifacts
   - *fetch_binaries
   - *fetch_debugsymbols_archive
   - *extract_binaries
-  - *set_up_venv
-  - *upload_pip_requirements
   - *check_binary_version
   - *get_buildnumber
+  - *f_expansions_write
   - *set_up_credentials
   - *run_diskstats
   - *monitor_process_threads
@@ -1144,6 +916,7 @@ functions:
   - *set_up_venv
   - *upload_pip_requirements
   - *get_buildnumber
+  - *f_expansions_write
   - *set_up_credentials
 
   "do benchmark setup":
@@ -1151,6 +924,7 @@ functions:
   - *set_up_venv
   - *upload_pip_requirements
   - *get_buildnumber
+  - *f_expansions_write
   - *set_up_credentials
   - *fetch_benchmarks
 
@@ -1438,6 +1212,7 @@ functions:
         - "./src/evergreen/extract_generated_test_configuration.sh"
 
   "generate selected tests":
+    - *f_expansions_write
     - command: expansions.write
       params:
         file: src/expansions.yml
@@ -1484,6 +1259,7 @@ functions:
     - command: expansions.write
       params:
         file: src/expansions.yml
+    - *f_expansions_write
     - *configure_evergreen_api_credentials
     - command: shell.exec
       type: test
@@ -1524,6 +1300,7 @@ functions:
     - *set_task_expansion_macros
     - *set_up_venv
     - *upload_pip_requirements
+    - *f_expansions_write
     - *configure_evergreen_api_credentials
     - command: expansions.write
       params:
@@ -1585,10 +1362,8 @@ functions:
     - *set_task_expansion_macros
     - *set_up_venv
     - *upload_pip_requirements
+    - *f_expansions_write
     - *configure_evergreen_api_credentials
-    - command: expansions.write
-      params:
-        file: src/expansions.yml
 
     - command: shell.exec
       type: test
@@ -1599,7 +1374,7 @@ functions:
           set -o errexit
 
           ${activate_virtualenv}
-          $python buildscripts/evergreen_generate_resmoke_tasks.py --expansion-file expansions.yml --verbose
+          $python buildscripts/evergreen_generate_resmoke_tasks.py --expansion-file ../expansions.yml --verbose
 
     - command: archive.targz_pack
       params:
@@ -1645,8 +1420,10 @@ functions:
       - key: aws_secret_remote
         value: ${mongodatafiles_aws_secret}
   - *set_up_remote_credentials
+  - *f_expansions_write
   - *determine_resmoke_jobs
   - *update_resmoke_jobs_expansions
+  - *f_expansions_write
   - *execute_resmoke_tests
     # The existence of the "run_tests_infrastructure_failure" file indicates this failure isn't
     # directly actionable. We use type=setup rather than type=system or type=test for this command
@@ -1659,6 +1436,7 @@ functions:
         - "./src/evergreen/check_run_tests_infrastructure_failure.sh"
 
   "run tests":
+    - *f_expansions_write
     - *determine_task_timeout
     - *update_task_timeout_expansions
     - *update_task_timeout
@@ -1675,8 +1453,10 @@ functions:
         - key: aws_secret_remote
           value: ${mongodatafiles_aws_secret}
     - *set_up_remote_credentials
+    - *f_expansions_write
     - *determine_resmoke_jobs
     - *update_resmoke_jobs_expansions
+    - *f_expansions_write
     - *execute_resmoke_tests
       # The existence of the "run_tests_infrastructure_failure" file indicates this failure isn't
       # directly actionable. We use type=setup rather than type=system or type=test for this command
@@ -1689,9 +1469,7 @@ functions:
           - "./src/evergreen/check_run_tests_infrastructure_failure.sh"
 
   "scons lint":
-    - command: expansions.write
-      params:
-        file: expansions.yml
+    - *f_expansions_write
     - command: subprocess.exec
       type: test
       params:
@@ -1699,13 +1477,10 @@ functions:
         args:
           - "src/evergreen/scons_lint.sh"
         env:
-          python: ${python}
           workdir: ${workdir}
 
   "scons compile":
-    - command: expansions.write
-      params:
-        file: expansions.yml
+    - *f_expansions_write
     - command: subprocess.exec
       type: test
       params:
@@ -1713,7 +1488,6 @@ functions:
         args:
           - "src/evergreen/scons_compile.sh"
         env:
-          python: ${python}
           workdir: ${workdir}
           patch_compile_flags: ${patch_compile_flags}
 
@@ -1785,9 +1559,10 @@ functions:
         fi
 
   "apply compile expansions":
-    command: expansions.update
-    params:
-      file: src/compile_expansions.yml
+    - command: expansions.update
+      params:
+        file: src/compile_expansions.yml
+    - *f_expansions_write
 
   "do jepsen setup":
     - command: subprocess.exec
@@ -1913,6 +1688,7 @@ functions:
     - *set_task_expansion_macros
     - *set_up_venv
     - *upload_pip_requirements
+    - *f_expansions_write
     - *configure_evergreen_api_credentials
     - *do_multiversion_setup
     - command: expansions.write
@@ -1964,6 +1740,7 @@ functions:
   "generate implicit multiversion tasks":
     - *set_task_expansion_macros
     - *set_up_venv
+    - *f_expansions_write
     - *configure_evergreen_api_credentials
     - command: expansions.write
       params:
@@ -2426,6 +2203,12 @@ functions:
       optional: true
 
   "save ec2 task artifacts":
+    - *f_expansions_write
+    # deliberate double write. Powercyle expects the expansions.yml file to
+    # be in src
+    - command: expansions.write
+      params:
+        file: src/expansions.yml
     - *save_powercycle_artifacts
     - *archive_remote_ec2_artifacts
     - *archive_remote_ec2_monitor_files
@@ -2465,7 +2248,7 @@ functions:
     params:
       binary: bash
       args:
-        - "./src/evergreen/cleanup_fuse_watchdog.sh"
+        - "./src/evergreen/functions/fuse_watchdog_cleanup.sh"
 
   ### Process & archive Code Coverage artifacts ###
   "process code coverage data": &process_code_coverage_data
@@ -2473,7 +2256,7 @@ functions:
     params:
       binary: bash
       args:
-        - "./src/evergreen/process_code_coverage_data.sh"
+        - "./src/evergreen/functions/code_coverage_data_process.sh"
 
   "tar code coverage data": &tar_code_coverage_data
     command: archive.targz_pack
@@ -2922,7 +2705,9 @@ pre:
   - func: "git get project"
   - func: "kill processes"
   - func: "cleanup environment"
+  - func: "set up venv"
   - func: "set task expansion macros"
+  - func: "f_expansions_write"
 
 # Post task steps
 post:
@@ -2994,9 +2779,7 @@ tasks:
         content_type: application/gzip
         display_name: Binaries
 
-    - command: expansions.write
-      params:
-        file: expansions.yml
+    - *f_expansions_write
     - command: subprocess.exec
       type: test
       params:
@@ -3004,7 +2787,6 @@ tasks:
         args:
           - "src/evergreen/gen_feature_flags.sh"
         env:
-          python: ${python}
           workdir: ${workdir}
 
     - command: archive.targz_pack
@@ -3134,16 +2916,13 @@ tasks:
 
 - name: libdeps_graph_linting
   commands:
-    - command: expansions.write
-      params:
-        file: expansions.yml
+    - *f_expansions_write
     - command: subprocess.exec
       params:
         binary: bash
         args:
           - "src/evergreen/libdeps_setup.sh"
         env:
-          python: ${python}
           workdir: ${workdir}
 
     - func: "scons compile"
@@ -3162,7 +2941,6 @@ tasks:
         args:
           - "src/evergreen/libdeps_run.sh"
         env:
-          python: ${python}
           workdir: ${workdir}
 
     - command: s3.put
@@ -3233,6 +3011,7 @@ tasks:
     - name: compile_unittests
   commands:
     - func: "run diskstats"
+    - func: "f_expansions_write"
     - func: "monitor process threads"
     - func: "collect system resource info"
     - func: "run tests"
@@ -3306,6 +3085,7 @@ tasks:
     - name: compile_dbtest
   commands:
     - func: "run diskstats"
+    - func: "f_expansions_write"
     - func: "monitor process threads"
     - func: "collect system resource info"
     - func: "run tests"
@@ -3353,16 +3133,13 @@ tasks:
 
 - name: embedded_sdk_build_cdriver
   commands:
-    - command: expansions.write
-      params:
-        file: expansions.yml
+    - func: f_expansions_write
     - command: subprocess.exec
       params:
         binary: bash
         args:
           - "src/evergreen/embedded_sdk_build_cdriver.sh"
         env:
-          python: ${python}
           workdir: ${workdir}
 
 - name: embedded_sdk_install_dev
@@ -3391,16 +3168,13 @@ tasks:
   depends_on:
     - name: embedded_sdk_install_dev
   commands:
-    - command: expansions.write
-      params:
-        file: expansions.yml
+    - *f_expansions_write
     - command: subprocess.exec
       params:
         binary: bash
         args:
           - "src/evergreen/embedded_sdk_s3_tar.sh"
         env:
-          python: ${python}
           workdir: ${workdir}
 
     # Upload it so we can download from EVG.
@@ -3437,9 +3211,7 @@ tasks:
   depends_on:
     - name: embedded_sdk_install_tests
   commands:
-    - command: expansions.write
-      params:
-        file: expansions.yml
+    - *f_expansions_write
     - command: subprocess.exec
       type: test
       params:
@@ -3447,7 +3219,6 @@ tasks:
         args:
           - "src/evergreen/embedded_sdk_tests_s3_tar.sh"
         env:
-          python: ${python}
           workdir: ${workdir}
 
     # Upload it so we can download from EVG.
@@ -3467,9 +3238,7 @@ tasks:
   depends_on:
     - name: embedded_sdk_install_tests
   commands:
-    - command: expansions.write
-      params:
-        file: expansions.yml
+    - *f_expansions_write
     - command: subprocess.exec
       type: test
       params:
@@ -3477,7 +3246,6 @@ tasks:
         args:
           - "src/evergreen/embedded_sdk_run_tests.sh"
         env:
-          python: ${python}
           workdir: ${workdir}
     - command: subprocess.exec
       params:
@@ -3485,7 +3253,6 @@ tasks:
         args:
           - "src/evergreen/embedded_sdk_run_tests_post.sh"
         env:
-          python: ${python}
           workdir: ${workdir}
 
 - name: embedded_sdk_s3_put_latest
@@ -3538,16 +3305,13 @@ tasks:
           --modules=
           --separate-debug
           DESTDIR='$BUILD_ROOT/stitch-support-lib-$MONGO_VERSION'
-    - command: expansions.write
-      params:
-        file: expansions.yml
+    - *f_expansions_write
     - command: subprocess.exec
       params:
         binary: bash
         args:
           - "src/evergreen/stitch_support_create_lib_tar.sh"
         env:
-          python: ${python}
           workdir: ${workdir}
     - command: s3.put
       params:
@@ -3581,9 +3345,7 @@ tasks:
   depends_on:
     - name: stitch_support_install_tests
   commands:
-    - command: expansions.write
-      params:
-        file: expansions.yml
+    - *f_expansions_write
     - command: subprocess.exec
       type: test
       params:
@@ -3591,7 +3353,6 @@ tasks:
         args:
           - "src/evergreen/stitch_support_run_tests.sh"
         env:
-          python: ${python}
           workdir: ${workdir}
 
 - name: compile_benchmarks
@@ -3601,6 +3362,7 @@ tasks:
     - func: "set up venv"
     - func: "upload pip requirements"
     - func: "get buildnumber"
+    - func: "f_expansions_write"
     - func: "set up credentials"
     - func: "use WiredTiger develop" # noop if ${use_wt_develop} is not "true"
     - func: "set up win mount script"
@@ -3692,9 +3454,7 @@ tasks:
   - func: "set task expansion macros"
   - func: "set up venv"
   - func: "upload pip requirements"
-  - command: expansions.write
-    params:
-      file: expansions.yml
+  - func: "f_expansions_write"
   - command: subprocess.exec
     type: test
     params:
@@ -3702,7 +3462,6 @@ tasks:
       args:
         - "src/evergreen/lint_yaml.sh"
       env:
-        python: ${python}
         workdir: ${workdir}
 
 - name: lint_shellscripts
@@ -3731,9 +3490,7 @@ tasks:
     - func: "set up venv"
     - func: "upload pip requirements"
     - func: "do setup"
-    - command: expansions.write
-      params:
-        file: expansions.yml
+    - func: "f_expansions_write"
     - command: subprocess.exec
       type: test
       params:
@@ -3741,7 +3498,6 @@ tasks:
         args:
           - "src/evergreen/check_idl_compat.sh"
         env:
-          python: ${python}
           workdir: ${workdir}
 
 - name: burn_in_tests_gen
@@ -3749,11 +3505,9 @@ tasks:
   - func: "set task expansion macros"
   - func: "set up venv"
   - func: "upload pip requirements"
+  - *f_expansions_write
   - func: "configure evergreen api credentials"
   - *do_multiversion_setup
-  - command: expansions.write
-    params:
-      file: expansions.yml
   - command: subprocess.exec
     type: test
     params:
@@ -3761,7 +3515,6 @@ tasks:
       args:
         - "src/evergreen/burn_in_tests.sh"
       env:
-        python: ${python}
         workdir: ${workdir}
   - command: archive.targz_pack
     params:
@@ -3790,11 +3543,9 @@ tasks:
   - func: "set task expansion macros"
   - func: "set up venv"
   - func: "upload pip requirements"
+  - func: "f_expansions_write"
   - func: "configure evergreen api credentials"
   - func: "do multiversion setup"
-  - command: expansions.write
-    params:
-      file: expansions.yml
   - command: subprocess.exec
     type: test
     params:
@@ -3802,7 +3553,6 @@ tasks:
       args:
         - "src/evergreen/burn_in_tests_multiversion.sh"
       env:
-        python: ${python}
         workdir: ${workdir}
 
   - command: archive.targz_pack
@@ -4869,6 +4619,7 @@ tasks:
     vars:
       pip_dir: ${workdir}/src/etc/pip
   - func: "upload pip requirements"
+  - *f_expansions_write
   - func: "configure evergreen api credentials"
   - func: "do multiversion setup"
   - func: "generate burn in tags"
@@ -5263,13 +5014,13 @@ tasks:
   name: external_auth
   commands:
   - func: "do setup"
+  - func: "f_expansions_write"
   - command: subprocess.exec
     params:
       binary: bash
       args:
         - "src/evergreen/external_auth_pip.sh"
       env:
-        python: ${python}
         workdir: ${workdir}
   - func: "run tests"
     vars:
@@ -5279,9 +5030,7 @@ tasks:
   name: external_auth_aws
   commands:
   - func: "do setup"
-  - command: expansions.write
-    params:
-      file: expansions.yml
+  - func: "f_expansions_write"
   - command: subprocess.exec
     params:
       binary: bash
@@ -5289,7 +5038,6 @@ tasks:
       args:
         - "src/evergreen/external_auth_aws_setup.sh"
       env:
-        python: ${python}
         workdir: ${workdir}
   - command: subprocess.exec
     params:
@@ -5297,7 +5045,6 @@ tasks:
       args:
         - "src/evergreen/external_auth_aws_pip.sh"
       env:
-        python: ${python}
         workdir: ${workdir}
   - func: "run tests"
     vars:
@@ -6894,6 +6641,7 @@ tasks:
     - func: "set task expansion macros"
     - func: "set up venv"
     - func: "upload pip requirements"
+    - *f_expansions_write
     - func: "configure evergreen api credentials"
     - func: "do multiversion setup"
       vars:
@@ -7042,9 +6790,7 @@ tasks:
   patchable: false
   commands:
   - func: "do non-compile setup"
-  - command: expansions.write
-    params:
-      file: expansions.yml
+  - func: "f_expansions_write"
   - command: subprocess.exec
     type: setup
     params:
@@ -7053,7 +6799,6 @@ tasks:
       args:
         - "src/evergreen/blackduck_setup.sh"
       env:
-        python: ${python}
         workdir: ${workdir}
   - command: subprocess.exec
     type: test
@@ -7062,7 +6807,6 @@ tasks:
       args:
         - "src/evergreen/blackduck_hub.sh"
       env:
-        python: ${python}
         workdir: ${workdir}
 
 - name: tla_plus
@@ -7084,6 +6828,7 @@ tasks:
   commands:
   - func: "do non-compile setup"
   - func: "set up remote credentials"
+  - *f_expansions_write
   - func: "configure evergreen api credentials"
   - func: "do multiversion setup"
   - func: "run tests"
@@ -7128,16 +6873,13 @@ tasks:
         task_compile_flags: >-
           --separate-debug
           --legacy-tarball
-    - command: expansions.write
-      params:
-        file: expansions.yml
+    - func: "f_expansions_write"
     - command: subprocess.exec
       params:
         binary: bash
         args:
           - "src/evergreen/package.sh"
         env:
-          python: ${python}
           workdir: ${workdir}
     - command: s3.put
       params:
@@ -8096,10 +7838,13 @@ task_groups:
     - func: "git get project"
     - func: "kill processes"
     - func: "cleanup environment"
-    - func: "set task expansion macros"
     - func: "set up venv"
+    - func: "upload pip requirements"
+    - func: "set task expansion macros"
+    - func: "f_expansions_write"
     - func: "configure evergreen api credentials"
     - func: "get buildnumber"
+    - func: "f_expansions_write"
     - func: "set up credentials"
     - func: "set up win mount script"
     - func: "generate compile expansions"
@@ -8120,10 +7865,11 @@ task_groups:
     - command: manifest.load
     - func: "git get project"
     - func: "get buildnumber"
-    - func: "set up credentials"
-    - func: "set task expansion macros"
     - func: "set up venv"
     - func: "upload pip requirements"
+    - func: "set task expansion macros"
+    - func: "f_expansions_write"
+    - func: "set up credentials"
     - func: "set up win mount script"
     - func: "generate compile expansions"
   teardown_group:
@@ -8143,10 +7889,11 @@ task_groups:
     - command: manifest.load
     - func: "git get project"
     - func: "get buildnumber"
-    - func: "set up credentials"
-    - func: "set task expansion macros"
     - func: "set up venv"
     - func: "upload pip requirements"
+    - func: "set task expansion macros"
+    - func: "f_expansions_write"
+    - func: "set up credentials"
     - func: "set up win mount script"
     - func: "generate compile expansions"
   teardown_group:
@@ -12427,6 +12174,8 @@ buildvariants:
   run_on:
   - windows-vsCurrent-small
   stepback: false
+  expansions:
+    python: '/cygdrive/c/python/python37/python.exe'
   tasks:
   - name: win_shared_scons_cache_pruning
 
diff --git a/etc/expansions.default.yml b/etc/expansions.default.yml
index 022d9009ae5..1d59269e742 100644
--- a/etc/expansions.default.yml
+++ b/etc/expansions.default.yml
@@ -11,7 +11,6 @@ resmoke_jobs_factor: "1"
 resmoke_jobs_max: "0"
 timeout_secs: "0"
 exec_timeout_secs: "0"
-python: /opt/mongodbtoolchain/v3/bin/python3
 multiversion_edition: base
 multiversion_platform: linux_x86_64
 multiversion_architecture: x86_64
diff --git a/evergreen/functions/added_and_modified_patch_files_get.sh b/evergreen/functions/added_and_modified_patch_files_get.sh
new file mode 100755
index 00000000000..9b230f3b081
--- /dev/null
+++ b/evergreen/functions/added_and_modified_patch_files_get.sh
@@ -0,0 +1,14 @@
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
+. "$DIR/../prelude.sh"
+
+cd src
+
+set -o verbose
+set -o errexit
+
+git diff HEAD --name-only --line-prefix="${workdir}/src/" --diff-filter=d >>modified_and_created_patch_files.txt
+if [ -d src/mongo/db/modules/enterprise ]; then
+  pushd src/mongo/db/modules/enterprise
+  git diff HEAD --name-only --line-prefix="${workdir}/src/src/mongo/db/modules/enterprise/" --diff-filter=d >>~1/modified_and_created_patch_files.txt
+  popd
+fi
diff --git a/evergreen/functions/bf_suggestion_service_call.sh b/evergreen/functions/bf_suggestion_service_call.sh
new file mode 100755
index 00000000000..d022bf38cf7
--- /dev/null
+++ b/evergreen/functions/bf_suggestion_service_call.sh
@@ -0,0 +1,22 @@
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
+. "$DIR/../prelude.sh"
+
+cd src
+
+report_file="report.json"
+# Check if the report file exists and has failures.
+if [ -f $report_file ] && grep -Eq "\"failures\": [1-9]" $report_file; then
+  # Calling the BF Suggestion server endpoint to start feature extraction.
+  payload="{\"task_id\": \"${task_id}\", \"execution\": ${execution}}"
+  echo "Sending task info to the BF suggestion service"
+  # The --user option is passed through stdin to avoid showing in process list.
+  user_option="--user ${bfsuggestion_user}:${bfsuggestion_password}"
+  curl --header "Content-Type: application/json" \
+    --data "$payload" \
+    --max-time 10 \
+    --silent \
+    --show-error \
+    --config - \
+    https://bfsuggestion.corp.mongodb.com/tasks <<<$user_option
+  echo "Request to BF Suggestion service status: $?"
+fi
diff --git a/evergreen/functions/binaries_extract.sh b/evergreen/functions/binaries_extract.sh
new file mode 100755
index 00000000000..efa7089ec3f
--- /dev/null
+++ b/evergreen/functions/binaries_extract.sh
@@ -0,0 +1,7 @@
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
+. "$DIR/../prelude.sh"
+
+cd src
+
+set -o errexit
+${decompress} mongo-binaries.tgz
diff --git a/evergreen/functions/binary_version_check.sh b/evergreen/functions/binary_version_check.sh
new file mode 100755
index 00000000000..f5f1a74487c
--- /dev/null
+++ b/evergreen/functions/binary_version_check.sh
@@ -0,0 +1,16 @@
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
+. "$DIR/../prelude.sh"
+
+cd src
+
+set -o errexit
+mongo_binary=dist-test/bin/mongo${exe}
+activate_venv
+bin_ver=$($python -c "import yaml; print(yaml.safe_load(open('compile_expansions.yml'))['version']);" | tr -d '[ \r\n]')
+# Due to SERVER-23810, we cannot use $mongo_binary --quiet --nodb --eval "version();"
+mongo_ver=$($mongo_binary --version | perl -pe '/version v([^\"]*)/; $_ = $1;' | tr -d '[ \r\n]')
+# The versions must match
+if [ "$bin_ver" != "$mongo_ver" ]; then
+  echo "The mongo version is $mongo_ver, expected version is $bin_ver"
+  exit 1
+fi
diff --git a/evergreen/process_code_coverage_data.sh b/evergreen/functions/code_coverage_data_process.sh
index c23114884ff..c23114884ff 100755
--- a/evergreen/process_code_coverage_data.sh
+++ b/evergreen/functions/code_coverage_data_process.sh
diff --git a/evergreen/functions/credentials_setup.sh b/evergreen/functions/credentials_setup.sh
new file mode 100755
index 00000000000..4f0605a45bf
--- /dev/null
+++ b/evergreen/functions/credentials_setup.sh
@@ -0,0 +1,12 @@
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
+. "$DIR/../prelude.sh"
+
+cd src
+
+cat >mci.buildlogger <<END_OF_CREDS
+slavename='${slave}'
+passwd='${passwd}'
+builder='MCI_${build_variant}'
+build_num=${builder_num}
+build_phase='${task_name}_${execution}'
+END_OF_CREDS
diff --git a/evergreen/functions/evergreen_api_credentials_configure.sh b/evergreen/functions/evergreen_api_credentials_configure.sh
new file mode 100755
index 00000000000..c42f637a2c9
--- /dev/null
+++ b/evergreen/functions/evergreen_api_credentials_configure.sh
@@ -0,0 +1,11 @@
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
+. "$DIR/../prelude.sh"
+
+cd src
+
+# Create the Evergreen API credentials
+cat >.evergreen.yml <<END_OF_CREDS
+api_server_host: https://evergreen.mongodb.com/api
+api_key: "${evergreen_api_key}"
+user: "${evergreen_api_user}"
+END_OF_CREDS
diff --git a/evergreen/functions/files_remove.sh b/evergreen/functions/files_remove.sh
new file mode 100755
index 00000000000..a2c905cdedd
--- /dev/null
+++ b/evergreen/functions/files_remove.sh
@@ -0,0 +1,12 @@
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
+. "$DIR/../prelude.sh"
+
+if [ -z "${files}" ]; then
+  exit 0
+fi
+for file in ${files}; do
+  if [ -f "$file" ]; then
+    echo "Removing file $file"
+    rm -f $file
+  fi
+done
diff --git a/evergreen/cleanup_fuse_watchdog.sh b/evergreen/functions/fuse_watchdog_cleanup.sh
index dd6dd587515..dd6dd587515 100755
--- a/evergreen/cleanup_fuse_watchdog.sh
+++ b/evergreen/functions/fuse_watchdog_cleanup.sh
diff --git a/evergreen/functions/modified_patch_files_get_all.sh b/evergreen/functions/modified_patch_files_get_all.sh
new file mode 100755
index 00000000000..5e79cc6ce0d
--- /dev/null
+++ b/evergreen/functions/modified_patch_files_get_all.sh
@@ -0,0 +1,19 @@
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
+. "$DIR/../prelude.sh"
+
+cd src
+
+set -o verbose
+set -o errexit
+
+# For patch builds gather the modified patch files.
+if [ "${is_patch}" = "true" ]; then
+  # Get list of patched files
+  git diff HEAD --name-only >>patch_files.txt
+  if [ -d src/mongo/db/modules/enterprise ]; then
+    pushd src/mongo/db/modules/enterprise
+    # Update the patch_files.txt in the mongo repo.
+    git diff HEAD --name-only >>~1/patch_files.txt
+    popd
+  fi
+fi
diff --git a/evergreen/functions/notary_client_credentials_setup.sh b/evergreen/functions/notary_client_credentials_setup.sh
new file mode 100755
index 00000000000..a1dab18b8cc
--- /dev/null
+++ b/evergreen/functions/notary_client_credentials_setup.sh
@@ -0,0 +1,14 @@
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
+. "$DIR/../prelude.sh"
+
+cd src
+
+set -o errexit
+
+cat <<EOF >notary_env.sh
+export NOTARY_TOKEN=${signing_auth_token_50}
+export BARQUE_USERNAME=${barque_user}
+export BARQUE_API_KEY=${barque_api_key}
+EOF
+
+echo "${signing_auth_token_50}" >signing_auth_token
diff --git a/evergreen/functions/process_threads_monitor.sh b/evergreen/functions/process_threads_monitor.sh
new file mode 100755
index 00000000000..c28269a1552
--- /dev/null
+++ b/evergreen/functions/process_threads_monitor.sh
@@ -0,0 +1,46 @@
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
+. "$DIR/../prelude.sh"
+
+proc_list="(java|lein|mongo|python|_test$|_test\.exe$)"
+if [ "Windows_NT" = "$OS" ]; then
+  get_pids() {
+    proc_pids=$(tasklist /fo:csv \
+      | awk -F'","' '{x=$1; gsub("\"","",x); print $2, x}' \
+      | grep -iE $1 \
+      | cut -f1 -d ' ')
+  }
+  get_process_info() {
+    proc_name=""
+    proc_info=$(wmic process where "ProcessId=\"$1\"" get "Name,ProcessId,ThreadCount" /format:csv 2>/dev/null | grep $1)
+    if [ ! -z $proc_info ]; then
+      proc_name=$(echo $proc_info | cut -f2 -d ',')
+      proc_threads=$(echo $proc_info | cut -f4 -d ',')
+    fi
+  }
+else
+  get_pids() { proc_pids=$(pgrep $1); }
+  get_process_info() {
+    proc_name=$(ps -p $1 -o comm=)
+    # /proc is available on Linux platforms
+    if [ -f /proc/$1/status ]; then
+      set_sudo
+      proc_threads=$($sudo grep Threads /proc/$1/status | sed "s/\s//g" | cut -f2 -d ":")
+    else
+      proc_threads=$(ps -AM $1 | grep -vc PID)
+    fi
+  }
+fi
+while [ 1 ]; do
+  get_pids $proc_list
+  if [ ! -z "$proc_pids" ]; then
+    printf "Running process/thread counter\n"
+    printf "PROCESS\tPID\tTHREADS\n"
+  fi
+  for pid in $proc_pids; do
+    get_process_info $pid
+    if [ ! -z "$proc_name" ]; then
+      printf "$proc_name\t$pid\t$proc_threads\n"
+    fi
+  done
+  sleep 60
+done
diff --git a/evergreen/functions/resmoke_jobs_determine.sh b/evergreen/functions/resmoke_jobs_determine.sh
new file mode 100755
index 00000000000..036e8b00154
--- /dev/null
+++ b/evergreen/functions/resmoke_jobs_determine.sh
@@ -0,0 +1,16 @@
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
+. "$DIR/../prelude.sh"
+
+cd src
+
+set -o verbose
+set -o errexit
+
+activate_venv
+$python buildscripts/evergreen_resmoke_job_count.py \
+  --taskName ${task_name} \
+  --buildVariant ${build_variant} \
+  --distro ${distro_id} \
+  --jobFactor ${resmoke_jobs_factor} \
+  --jobsMax ${resmoke_jobs_max} \
+  --outFile resmoke_jobs_expansion.yml
diff --git a/evergreen/run_diskstats.sh b/evergreen/functions/run_diskstats.sh
index d79141cdb2c..d79141cdb2c 100755
--- a/evergreen/run_diskstats.sh
+++ b/evergreen/functions/run_diskstats.sh
diff --git a/evergreen/functions/selected_tests_credentials_configure.sh b/evergreen/functions/selected_tests_credentials_configure.sh
new file mode 100755
index 00000000000..964cdc128a0
--- /dev/null
+++ b/evergreen/functions/selected_tests_credentials_configure.sh
@@ -0,0 +1,12 @@
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
+. "$DIR/../prelude.sh"
+
+cd src
+
+# Create the Evergreen API credentials
+cat >.selected_tests.yml <<END_OF_CREDS
+url: "https://selected-tests.server-tig.prod.corp.mongodb.com"
+project: "${project}"
+auth_user: "${selected_tests_auth_user}"
+auth_token: "${selected_tests_auth_token}"
+END_OF_CREDS
diff --git a/evergreen/functions/shared_scons_cache_pruning.sh b/evergreen/functions/shared_scons_cache_pruning.sh
new file mode 100755
index 00000000000..87219355a82
--- /dev/null
+++ b/evergreen/functions/shared_scons_cache_pruning.sh
@@ -0,0 +1,76 @@
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
+. "$DIR/../prelude.sh"
+
+cd src
+
+set -o errexit
+set -o verbose
+# removes files from the shared scons cache.
+
+# Only prune on master branch
+if [[ "${project}" == "mongodb-mongo-master" ]]; then
+
+  set +o errexit
+
+  if [ "Windows_NT" = "$OS" ]; then
+    ./win_mount.sh
+  else
+    mount | grep "\/efs" >/dev/null
+  fi
+  if [ $? -eq 0 ]; then
+    echo "Shared cache is already mounted"
+  else
+    echo "Shared cache - mounting file system"
+    if [ "Windows_NT" = "$OS" ]; then
+      ./win_mount.sh
+    else
+      sudo mount /efs
+    fi
+  fi
+  set -o errexit
+
+  if [ "Windows_NT" = "$OS" ]; then
+    cache_folder=/cygdrive/x
+  else
+    cache_folder=/efs
+  fi
+  dirs=$(ls -l $cache_folder | grep -v total | awk '{print $NF}')
+
+  echo "Pruning shared SCons directories"
+
+  for dir in $dirs; do
+    echo "Pruning $cache_folder/$dir/scons-cache"
+
+    if [ -e $cache_folder/$dir/info/distro_name ]; then
+      distro=$(cat $cache_folder/$dir/info/distro_name)
+    fi
+
+    # Set cache sizes by distro
+    case $distro in
+    ubuntu1604 | ubuntu1804 | rhel62 | rhel70)
+      cache_size=600
+      ;;
+    *)
+      # default
+      cache_size=400
+      ;;
+    esac
+
+    if [ "Windows_NT" = "$OS" ]; then
+      echo "dir="$dir
+      python buildscripts/scons_cache_prune.py --cache-dir x:/$dir/scons-cache --cache-size $cache_size --prune-ratio 1.0
+    else
+      sudo python buildscripts/scons_cache_prune.py --cache-dir /efs/$dir/scons-cache --cache-size $cache_size --prune-ratio 1.0
+    fi
+    echo ""
+  done
+
+  if [ "Windows_NT" = "$OS" ]; then
+    net use X: /delete || true
+  else
+    sudo umount /efs || true
+  fi
+
+else
+  echo "Not on master, shared SCons cache pruning skipped"
+fi
diff --git a/evergreen/functions/shared_scons_directory_umount.sh b/evergreen/functions/shared_scons_directory_umount.sh
new file mode 100755
index 00000000000..6d50da0fdae
--- /dev/null
+++ b/evergreen/functions/shared_scons_directory_umount.sh
@@ -0,0 +1,19 @@
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
+. "$DIR/../prelude.sh"
+
+cd src
+
+set -o errexit
+set -o verbose
+
+if [ "${disable_shared_scons_cache}" = true ]; then
+  exit
+fi
+if [ "${scons_cache_scope}" = "shared" ]; then
+  if [ "Windows_NT" = "$OS" ]; then
+    net use X: /delete || true
+  else
+    set_sudo
+    $sudo umount /efs || true
+  fi
+fi
diff --git a/evergreen/functions/system_resource_info_collect.sh b/evergreen/functions/system_resource_info_collect.sh
new file mode 100755
index 00000000000..9b9129e6abf
--- /dev/null
+++ b/evergreen/functions/system_resource_info_collect.sh
@@ -0,0 +1,7 @@
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
+. "$DIR/../prelude.sh"
+
+cd src
+
+activate_venv
+$python buildscripts/collect_resource_info.py -o system_resource_info.json -i 5
diff --git a/evergreen/functions/task_timeout_determine.sh b/evergreen/functions/task_timeout_determine.sh
new file mode 100644
index 00000000000..7a43e097939
--- /dev/null
+++ b/evergreen/functions/task_timeout_determine.sh
@@ -0,0 +1,15 @@
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
+. "$DIR/../prelude.sh"
+
+cd src
+
+set -o verbose
+set -o errexit
+activate_venv
+$python buildscripts/evergreen_task_timeout.py \
+  --task-name ${task_name} \
+  --build-variant ${build_variant} \
+  --evg-alias '${alias}' \
+  --timeout ${timeout_secs} \
+  --exec-timeout ${exec_timeout_secs} \
+  --out-file task_timeout_expansions.yml
diff --git a/evergreen/functions/venv_setup.sh b/evergreen/functions/venv_setup.sh
new file mode 100644
index 00000000000..7eb6c3d14a2
--- /dev/null
+++ b/evergreen/functions/venv_setup.sh
@@ -0,0 +1,43 @@
+# exit immediately if virtualenv is not found
+set -o errexit
+
+evergreen_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
+. "$evergreen_dir/../prelude_python.sh"
+
+python_loc=$(which ${python:-/opt/mongodbtoolchain/v3/bin/python3})
+venv_dir="${workdir}/venv"
+if [ -d "$venv_dir" ]; then
+  exit 0
+fi
+"$python_loc" -m venv "$venv_dir"
+
+# venv creates its Scripts/activate file with CLRF endings, which
+# cygwin bash does not like. dos2unix it
+# (See https://bugs.python.org/issue32451)
+if [ "Windows_NT" = "$OS" ]; then
+  dos2unix "${workdir}/venv/Scripts/activate"
+fi
+
+export VIRTUAL_ENV_DISABLE_PROMPT=yes
+
+# Not all git get project calls clone into ${workdir}/src so we allow
+# callers to tell us where the pip requirements files are.
+pip_dir="${pip_dir}"
+if [[ -z $pip_dir ]]; then
+  # Default to most common location
+  pip_dir="${workdir}/src/etc/pip"
+fi
+
+# Same as above we have to use quotes to preserve the
+# Windows path separator
+toolchain_txt="$pip_dir/toolchain-requirements.txt"
+
+# the whole prelude cannot be imported because it requires pyyaml to be
+# installed, which happens just below.
+. "$evergreen_dir/../prelude_venv.sh"
+
+activate_venv
+echo "Upgrading pip to 21.0.1"
+python -m pip install "pip==21.0.1"
+python -m pip install -r "$toolchain_txt" -q
+python -m pip freeze >pip-requirements.txt
diff --git a/evergreen/functions/win_mount_script_setup.sh b/evergreen/functions/win_mount_script_setup.sh
new file mode 100755
index 00000000000..187777e213b
--- /dev/null
+++ b/evergreen/functions/win_mount_script_setup.sh
@@ -0,0 +1,9 @@
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
+. "$DIR/../prelude.sh"
+
+cd src
+
+cat <<EOF >win_mount.sh
+net use X: '\\\\${win_scons_endpoint}\\share' /USER:"wincache.build.com\${win_scons_user}" '${win_scons_pass}'
+EOF
+chmod +x win_mount.sh
diff --git a/evergreen/functions/wiredtiger_develop_use.sh b/evergreen/functions/wiredtiger_develop_use.sh
new file mode 100755
index 00000000000..db913aac939
--- /dev/null
+++ b/evergreen/functions/wiredtiger_develop_use.sh
@@ -0,0 +1,14 @@
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
+. "$DIR/../prelude.sh"
+
+cd src
+
+set -o errexit
+set -o verbose
+if [ "${use_wt_develop}" = "true" ]; then
+  cd src/third_party
+  for wtdir in dist examples ext lang src test tools; do
+    rm -rf wiredtiger/$wtdir
+    mv wtdevelop/$wtdir wiredtiger/
+  done
+fi
diff --git a/evergreen/prelude.sh b/evergreen/prelude.sh
index 823b29449af..86f41a53c44 100755
--- a/evergreen/prelude.sh
+++ b/evergreen/prelude.sh
@@ -16,33 +16,8 @@ if [ "$PWD" != "$calculated_workdir" ]; then
   exit 1
 fi
 
-function activate_venv {
-  set -e
-  # check if virtualenv is set up
-  if [ -d "${workdir}/venv" ]; then
-    if [ "Windows_NT" = "$OS" ]; then
-      # Need to quote the path on Windows to preserve the separator.
-      . "${workdir}/venv/Scripts/activate" 2>/tmp/activate_error.log
-    else
-      . ${workdir}/venv/bin/activate 2>/tmp/activate_error.log
-    fi
-    if [ $? -ne 0 ]; then
-      echo "Failed to activate virtualenv: $(cat /tmp/activate_error.log)"
-    fi
-    python=python
-  else
-    python=${python:-/opt/mongodbtoolchain/v3/bin/python3}
-  fi
-
-  if [ "Windows_NT" = "$OS" ]; then
-    export PYTHONPATH="$PYTHONPATH;$(cygpath -w ${workdir}/src)"
-  else
-    export PYTHONPATH="$PYTHONPATH:${workdir}/src"
-  fi
-
-  echo "python set to $(which $python)"
-  set +e
-}
+. "$evergreen_dir/prelude_python.sh"
+. "$evergreen_dir/prelude_venv.sh"
 
 expansions_yaml="$evergreen_dir/../../expansions.yml"
 expansions_default_yaml="$evergreen_dir/../etc/expansions.default.yml"
diff --git a/evergreen/prelude_python.sh b/evergreen/prelude_python.sh
new file mode 100644
index 00000000000..d6192c253bc
--- /dev/null
+++ b/evergreen/prelude_python.sh
@@ -0,0 +1,5 @@
+if [ "Windows_NT" = "$OS" ]; then
+  python='/cygdrive/c/python/python37/python.exe'
+else
+  python="/opt/mongodbtoolchain/v3/bin/python3"
+fi
diff --git a/evergreen/prelude_venv.sh b/evergreen/prelude_venv.sh
new file mode 100644
index 00000000000..953f4e63d02
--- /dev/null
+++ b/evergreen/prelude_venv.sh
@@ -0,0 +1,27 @@
+function activate_venv {
+  set -e
+  # check if virtualenv is set up
+  if [ -d "${workdir}/venv" ]; then
+    if [ "Windows_NT" = "$OS" ]; then
+      # Need to quote the path on Windows to preserve the separator.
+      . "${workdir}/venv/Scripts/activate" 2>/tmp/activate_error.log
+    else
+      . ${workdir}/venv/bin/activate 2>/tmp/activate_error.log
+    fi
+    if [ $? -ne 0 ]; then
+      echo "Failed to activate virtualenv: $(cat /tmp/activate_error.log)"
+    fi
+    python=python
+  else
+    python=${python:-/opt/mongodbtoolchain/v3/bin/python3}
+  fi
+
+  if [ "Windows_NT" = "$OS" ]; then
+    export PYTHONPATH="$PYTHONPATH;$(cygpath -w ${workdir}/src)"
+  else
+    export PYTHONPATH="$PYTHONPATH:${workdir}/src"
+  fi
+
+  echo "python set to $(which $python)"
+  set +e
+}