SERVER-55626 Extract shell scripts from server yaml part 3

2 files changed, 189 insertions, 468 deletions

diff --git a/etc/evergreen.yml b/etc/evergreen.yml
index e15d59d2d3b..a9b2b32d06c 100644
--- a/etc/evergreen.yml
+++ b/etc/evergreen.yml
@@ -137,6 +137,7 @@ variables:
   setup_task:
   - func: "apply compile expansions"
   - func: "set task expansion macros"
+  - func: "f_expansions_write"
   teardown_task:
   - func: "attach scons logs"
   - func: "attach report"
@@ -174,18 +175,20 @@ variables:
   - func: "git get project"
   - func: "kill processes"
   - func: "cleanup environment"
-  - func: "get all modified patch files"
   - func: "set task expansion macros"
   # The python virtual environment is installed in ${workdir}, which is created in
   # "set up venv".
   - func: "set up venv"
   - func: "upload pip requirements"
+  - func: "get all modified patch files"
   - func: "configure evergreen api credentials"
   - func: "get buildnumber"
+  - func: "f_expansions_write"
   - func: "set up credentials"
   - func: "use WiredTiger develop" # noop if ${use_wt_develop} is not "true"
   - func: "set up win mount script"
   - func: "generate compile expansions"
+  - func: "f_expansions_write"
   teardown_group:
   - func: "umount shared scons directory"
   - func: "cleanup environment"
@@ -255,6 +258,7 @@ variables:
     - func: "set up venv"
     - func: "upload pip requirements"
     - func: "get buildnumber"
+    - func: "f_expansions_write"
     - func: "set up win mount script"
     - func: "generate compile expansions"
   teardown_group:
@@ -362,52 +366,42 @@ variables:
 #######################################
 
 functions:
-  "remove files": &remove_files
-    command: shell.exec
+  "f_expansions_write": &f_expansions_write
+    command: expansions.write
     params:
-      shell: bash
-      script: |
-        if [ -z "${files}" ]; then
-          exit 0
-        fi
-        for file in ${files}
-        do
-          if [ -f "$file" ]; then
-            echo "Removing file $file"
-            rm -f $file
-          fi
-        done
+      file: expansions.yml
+      redacted: true
+
+  "remove files":
+    - *f_expansions_write
+    - command: subprocess.exec
+      params:
+        binary: bash
+        args:
+          - "src/evergreen/functions/files_remove.sh"
+        env:
+          workdir: ${workdir}
 
   "configure evergreen api credentials": &configure_evergreen_api_credentials
-    command: shell.exec
+    command: subprocess.exec
     type: test
     params:
-      shell: bash
-      working_dir: src
-      silent: true
-      script: |
-        # Create the Evergreen API credentials
-        cat > .evergreen.yml <<END_OF_CREDS
-        api_server_host: https://evergreen.mongodb.com/api
-        api_key: "${evergreen_api_key}"
-        user: "${evergreen_api_user}"
-        END_OF_CREDS
+      binary: bash
+      args:
+        - "src/evergreen/functions/evergreen_api_credentials_configure.sh"
+      env:
+        workdir: ${workdir}
 
   "configure selected tests credentials": &configure_selected_tests_credentials
-    command: shell.exec
+    command: subprocess.exec
     type: test
     params:
-      working_dir: src
+      binary: bash
       silent: true
-      shell: bash
-      script: |
-        # Create the Evergreen API credentials
-        cat > .selected_tests.yml <<END_OF_CREDS
-        url: "https://selected-tests.server-tig.prod.corp.mongodb.com"
-        project: "${project}"
-        auth_user: "${selected_tests_auth_user}"
-        auth_token: "${selected_tests_auth_token}"
-        END_OF_CREDS
+      args:
+        - "src/evergreen/functions/selected_tests_credentials_configure.sh"
+      env:
+        workdir: ${workdir}
 
   "git get project": &git_get_project
     - command: git.get_project
@@ -464,31 +458,22 @@ functions:
       local_file: src/mongo-binaries.tgz
 
   "extract binaries": &extract_binaries
-    command: shell.exec
+    command: subprocess.exec
     params:
-      working_dir: src
-      shell: bash
-      script: |
-        set -o errexit
-        ${decompress|tar xzvf} mongo-binaries.tgz
+      binary: bash
+      args:
+        - "src/evergreen/functions/binaries_extract.sh"
+      env:
+        workdir: ${workdir}
 
   "check binary version": &check_binary_version
-    command: shell.exec
+    command: subprocess.exec
     params:
-      working_dir: src
-      shell: bash
-      script: |
-        set -o errexit
-        mongo_binary=dist-test/bin/mongo${exe}
-        ${activate_virtualenv}
-        bin_ver=$($python -c "import yaml; print(yaml.safe_load(open('compile_expansions.yml'))['version']);" | tr -d '[ \r\n]')
-        # Due to SERVER-23810, we cannot use $mongo_binary --quiet --nodb --eval "version();"
-        mongo_ver=$($mongo_binary --version | perl -pe '/version v([^\"]*)/; $_ = $1;' | tr -d '[ \r\n]')
-        # The versions must match
-        if [ "$bin_ver" != "$mongo_ver" ]; then
-          echo "The mongo version is $mongo_ver, expected version is $bin_ver"
-          exit 1
-        fi
+      binary: bash
+      args:
+        - "src/evergreen/functions/binary_version_check.sh"
+      env:
+        workdir: ${workdir}
 
   "fetch benchmarks": &fetch_benchmarks
     command: s3.get
@@ -567,88 +552,41 @@ functions:
       system_log: true
       binary: bash
       args:
-        - "./src/evergreen/run_diskstats.sh"
+        - "./src/evergreen/functions/run_diskstats.sh"
 
   "collect system resource info": &collect_system_resource_info
-    command: shell.exec
+    command: subprocess.exec
     params:
-      working_dir: src
       background: true
       system_log: true
-      shell: bash
-      script: |
-        ${activate_virtualenv}
-        $python buildscripts/collect_resource_info.py -o system_resource_info.json -i 5
+      binary: bash
+      args:
+        - "./src/evergreen/functions/system_resource_info_collect.sh"
+      env:
+        workdir: ${workdir}
 
   # Run a monitor process as a background, system task to periodically
   # display how many threads interesting processes are using.
   "monitor process threads": &monitor_process_threads
-    command: shell.exec
+    command: subprocess.exec
     params:
       background: true
       system_log: true
-      shell: bash
-      script: |
-        proc_list="(java|lein|mongo|python|_test$|_test\.exe$)"
-        if [ "Windows_NT" = "$OS" ]; then
-          get_pids() {
-            proc_pids=$(tasklist /fo:csv |
-                        awk -F'","' '{x=$1; gsub("\"","",x); print $2, x}' |
-                        grep -iE $1 |
-                        cut -f1 -d ' ');
-          }
-          get_process_info() {
-            proc_name="";
-            proc_info=$(wmic process where "ProcessId=\"$1\"" get "Name,ProcessId,ThreadCount" /format:csv 2> /dev/null | grep $1);
-            if [ ! -z $proc_info ]; then
-              proc_name=$(echo $proc_info | cut -f2 -d ',');
-              proc_threads=$(echo $proc_info | cut -f4 -d ',');
-            fi;
-          }
-        else
-          get_pids() { proc_pids=$(pgrep $1); }
-          get_process_info() {
-            proc_name=$(ps -p $1 -o comm=);
-            # /proc is available on Linux platforms
-            if [ -f /proc/$1/status ]; then
-              ${set_sudo}
-              proc_threads=$($sudo grep Threads /proc/$1/status | sed "s/\s//g" | cut -f2 -d ":");
-            else
-              proc_threads=$(ps -AM $1 | grep -vc PID);
-            fi;
-          }
-        fi
-        while [ 1 ]
-        do
-          get_pids $proc_list
-          if [ ! -z "$proc_pids" ]; then
-            printf "Running process/thread counter\n"
-            printf "PROCESS\tPID\tTHREADS\n"
-          fi
-          for pid in $proc_pids
-          do
-            get_process_info $pid
-            if [ ! -z "$proc_name" ]; then
-              printf "$proc_name\t$pid\t$proc_threads\n"
-            fi
-          done
-          sleep 60
-        done
+      binary: bash
+      args:
+        - "./src/evergreen/functions/process_threads_monitor.sh"
+      env:
+        workdir: ${workdir}
 
   "set up credentials": &set_up_credentials
-    command: shell.exec
+    command: subprocess.exec
     params:
-      working_dir: src
+      binary: bash
       silent: true
-      shell: bash
-      script: |
-        cat > mci.buildlogger <<END_OF_CREDS
-        slavename='${slave}'
-        passwd='${passwd}'
-        builder='MCI_${build_variant}'
-        build_num=${builder_num}
-        build_phase='${task_name}_${execution}'
-        END_OF_CREDS
+      args:
+        - "./src/evergreen/functions/credentials_setup.sh"
+      env:
+        workdir: ${workdir}
 
   "set up win mount script": &set_up_win_mount_script
     command: shell.exec
@@ -663,21 +601,14 @@ functions:
         chmod +x win_mount.sh
 
   "set up notary client credentials": &set_up_notary_client_credentials
-    command: shell.exec
+    command: subprocess.exec
     params:
-      working_dir: src
+      binary: bash
       silent: true
-      shell: bash
-      script: |
-        set -o errexit
-
-        cat <<EOF > notary_env.sh
-        export NOTARY_TOKEN=${signing_auth_token_50}
-        export BARQUE_USERNAME=${barque_user}
-        export BARQUE_API_KEY=${barque_api_key}
-        EOF
-
-        echo "${signing_auth_token_50}" > signing_auth_token
+      args:
+        - "./src/evergreen/functions/notary_client_credentials_setup.sh"
+      env:
+        workdir: ${workdir}
 
   "set up remote credentials": &set_up_remote_credentials
     command: shell.exec
@@ -728,29 +659,14 @@ functions:
         EOF
 
   "call BF Suggestion service":
-    command: shell.exec
+    command: subprocess.exec
     params:
-      working_dir: src
-      shell: bash
+      binary: bash
       silent: true
-      script: |
-        report_file="report.json"
-        # Check if the report file exists and has failures.
-        if [ -f $report_file ] && grep -Eq "\"failures\": [1-9]" $report_file; then
-          # Calling the BF Suggestion server endpoint to start feature extraction.
-          payload="{\"task_id\": \"${task_id}\", \"execution\": ${execution}}"
-          echo "Sending task info to the BF suggestion service"
-          # The --user option is passed through stdin to avoid showing in process list.
-          user_option="--user ${bfsuggestion_user}:${bfsuggestion_password}"
-          curl --header "Content-Type: application/json" \
-               --data "$payload" \
-               --max-time 10 \
-               --silent \
-               --show-error \
-               --config - \
-               https://bfsuggestion.corp.mongodb.com/tasks <<< $user_option
-          echo "Request to BF Suggestion service status: $?"
-        fi
+      args:
+        - "./src/evergreen/functions/bf_suggestion_service_call.sh"
+      env:
+        workdir: ${workdir}
 
   "upload debugsymbols": &upload_debugsymbols
     command: s3.put
@@ -773,177 +689,65 @@ functions:
       local_file: src/mongo-debugsymbols.tgz
 
   "use WiredTiger develop":
-    command: shell.exec
-    params:
-      working_dir: src
-      shell: bash
-      script: |
-        set -o errexit
-        set -o verbose
-        if [ "${use_wt_develop|}" = "true" ]; then
-        cd src/third_party
-        for wtdir in dist examples ext lang src test tools ; do
-          rm -rf wiredtiger/$wtdir
-          mv wtdevelop/$wtdir wiredtiger/
-        done
-        fi
+    - *f_expansions_write
+    - command: subprocess.exec
+      params:
+        binary: bash
+        args:
+          - "./src/evergreen/functions/wiredtiger_develop_use.sh"
+        env:
+          workdir: ${workdir}
 
   "shared scons cache pruning":
-    command: shell.exec
-    type: system
-    params:
-      shell: bash
-      working_dir: src
-      script: |
-        set -o errexit
-        set -o verbose
-        # removes files from the shared scons cache.
-
-        # Only prune on master branch
-        if [[ "${project}" == "mongodb-mongo-master" ]]; then
-
-
-          set +o errexit
-
-          if [ "Windows_NT" = "$OS" ]; then
-            ./win_mount.sh
-          else
-            mount | grep "\/efs" > /dev/null
-          fi
-          if [ $? -eq 0 ]; then
-            echo "Shared cache is already mounted"
-          else
-            echo "Shared cache - mounting file system"
-            if [ "Windows_NT" = "$OS" ]; then
-              ./win_mount.sh
-            else
-              sudo mount /efs
-            fi
-          fi
-          set -o errexit
-
-          if [ "Windows_NT" = "$OS" ]; then
-            cache_folder=/cygdrive/x
-          else
-            cache_folder=/efs
-          fi
-          dirs=$(ls -l $cache_folder | grep -v total | awk '{print $NF}')
-
-          echo "Pruning shared SCons directories"
-
-          for dir in $dirs; do
-            echo "Pruning $cache_folder/$dir/scons-cache"
-
-            if [ -e $cache_folder/$dir/info/distro_name ]; then
-              distro=$(cat $cache_folder/$dir/info/distro_name)
-            fi
-
-            # Set cache sizes by distro
-            case $distro in
-                 ubuntu1604|ubuntu1804|rhel62|rhel70)
-                      cache_size=600
-                      ;;
-                 *)
-                      # default
-                      cache_size=400
-                      ;;
-            esac
-
-            if [ "Windows_NT" = "$OS" ]; then
-              echo "dir="$dir
-              python buildscripts/scons_cache_prune.py --cache-dir x:/$dir/scons-cache --cache-size $cache_size --prune-ratio 1.0
-            else
-              sudo python buildscripts/scons_cache_prune.py --cache-dir /efs/$dir/scons-cache --cache-size $cache_size --prune-ratio 1.0
-            fi
-            echo ""
-          done
-
-          if [ "Windows_NT" = "$OS" ]; then
-            net use X: /delete || true
-          else
-            sudo umount /efs || true
-          fi
-
-        else
-          echo "Not on master, shared SCons cache pruning skipped"
-        fi
+    - *f_expansions_write
+    - command: subprocess.exec
+      type: system
+      params:
+        binary: bash
+        args:
+          - "./src/evergreen/functions/shared_scons_cache_pruning.sh"
+        env:
+          workdir: ${workdir}
 
   "umount shared scons directory":
-    command: shell.exec
-    params:
-      working_dir: src
-      shell: bash
-      script: |
-        set -o errexit
-        set -o verbose
-
-        if [ "${disable_shared_scons_cache}" = true ]; then
-            exit
-        fi
-        if [ "${scons_cache_scope}" = "shared" ]; then
-            if [ "Windows_NT" = "$OS" ]; then
-                net use X: /delete || true
-            else
-                ${set_sudo}
-                $sudo umount /efs || true
-            fi
-        fi
+    - *f_expansions_write
+    - command: subprocess.exec
+      params:
+        binary: bash
+        args:
+          - "./src/evergreen/functions/shared_scons_directory_umount.sh"
+        env:
+          workdir: ${workdir}
 
   "get all modified patch files":
-    command: shell.exec
-    params:
-      working_dir: src
-      shell: bash
-      script: |
-        set -o verbose
-        set -o errexit
-
-        # For patch builds gather the modified patch files.
-        if [ "${is_patch}" = "true" ]; then
-          # Get list of patched files
-          git diff HEAD --name-only >> patch_files.txt
-          if [ -d src/mongo/db/modules/enterprise ]; then
-            pushd src/mongo/db/modules/enterprise
-            # Update the patch_files.txt in the mongo repo.
-            git diff HEAD --name-only >> ~1/patch_files.txt
-            popd
-          fi
-        fi
+    - *f_expansions_write
+    - command: subprocess.exec
+      params:
+        binary: bash
+        args:
+          - "./src/evergreen/functions/modified_patch_files_get_all.sh"
+        env:
+          workdir: ${workdir}
 
   # This function should only be called from patch-build-only tasks.
   "get added and modified patch files":
-    command: shell.exec
-    params:
-      working_dir: src
-      shell: bash
-      script: |
-        set -o verbose
-        set -o errexit
-
-        git diff HEAD --name-only --line-prefix="${workdir}/src/" --diff-filter=d >> modified_and_created_patch_files.txt
-        if [ -d src/mongo/db/modules/enterprise ]; then
-          pushd src/mongo/db/modules/enterprise
-          git diff HEAD --name-only --line-prefix="${workdir}/src/src/mongo/db/modules/enterprise/" --diff-filter=d >> ~1/modified_and_created_patch_files.txt
-          popd
-        fi
+    - *f_expansions_write
+    - command: subprocess.exec
+      params:
+        binary: bash
+        args:
+          - "./src/evergreen/functions/added_and_modified_patch_files_get.sh"
+        env:
+          workdir: ${workdir}
 
   "determine resmoke jobs": &determine_resmoke_jobs
-    command: shell.exec
+    command: subprocess.exec
     params:
-      working_dir: src
-      shell: bash
-      script: |
-        set -o verbose
-        set -o errexit
-
-        ${activate_virtualenv}
-        $python buildscripts/evergreen_resmoke_job_count.py \
-          --taskName ${task_name}                           \
-          --buildVariant ${build_variant}                   \
-          --distro ${distro_id}                             \
-          --jobFactor ${resmoke_jobs_factor|1}              \
-          --jobsMax ${resmoke_jobs_max|0}                   \
-          --outFile resmoke_jobs_expansion.yml
+      binary: bash
+      args:
+        - "./src/evergreen/functions/resmoke_jobs_determine.sh"
+      env:
+        workdir: ${workdir}
 
   "update resmoke jobs expansions": &update_resmoke_jobs_expansions
     command: expansions.update
@@ -952,22 +756,13 @@ functions:
       file: src/resmoke_jobs_expansion.yml
 
   "determine task timeout": &determine_task_timeout
-    command: shell.exec
+    command: subprocess.exec
     params:
-      working_dir: src
-      shell: bash
-      script: |
-        set -o verbose
-        set -o errexit
-
-        ${activate_virtualenv}
-        $python buildscripts/evergreen_task_timeout.py \
-          --task-name ${task_name}                     \
-          --build-variant ${build_variant}             \
-          --evg-alias '${alias|}'                      \
-          --timeout ${timeout_secs|0}                  \
-          --exec-timeout ${exec_timeout_secs|0}        \
-          --out-file task_timeout_expansions.yml
+      binary: bash
+      args:
+        - "./src/evergreen/functions/task_timeout_determine.sh"
+      env:
+        workdir: ${workdir}
 
   "update task timeout expansions": &update_task_timeout_expansions
     command: expansions.update
@@ -1069,42 +864,14 @@ functions:
         value: skip_test-${build_id}
 
   "set up venv": &set_up_venv
-    command: shell.exec
+    command: subprocess.exec
     params:
-        shell: bash
-        script: |
-          # exit immediately if virtualenv is not found
-          set -o errexit
-
-          python_loc=$(which ${python|/opt/mongodbtoolchain/v3/bin/python3})
-          venv_dir="${workdir}/venv"
-          "$python_loc" -m venv --system-site-packages "$venv_dir"
-
-          # venv creates its Scripts/activate file with CLRF endings, which
-          # cygwin bash does not like. dos2unix it
-          # (See https://bugs.python.org/issue32451)
-          if [ "Windows_NT" = "$OS" ]; then
-            dos2unix "${workdir}/venv/Scripts/activate"
-          fi
-
-          export VIRTUAL_ENV_DISABLE_PROMPT=yes
-
-          # Not all git get project calls clone into ${workdir}/src so we allow
-          # callers to tell us where the pip requirements files are.
-          pip_dir="${pip_dir}"
-          if [[ -z $pip_dir ]]; then
-            # Default to most common location
-            pip_dir="${workdir}/src/etc/pip"
-          fi
-
-          # Same as above we have to use quotes to preserve the
-          # Windows path separator
-          toolchain_txt="$pip_dir/toolchain-requirements.txt"
-          ${activate_virtualenv}
-          echo "Upgrading pip to 21.0.1"
-          python -m pip install "pip==21.0.1"
-          python -m pip install -r "$toolchain_txt" -q
-          python -m pip freeze > pip-requirements.txt
+        binary: bash
+        args:
+          - "src/evergreen/functions/venv_setup.sh"
+        env:
+          workdir: ${workdir}
+          pip_dir: ${pip_dir}
 
   "upload pip requirements": &upload_pip_requirements
     command: s3.put
@@ -1125,15 +892,17 @@ functions:
       file: src/perf.json
 
   "do setup":
+  - *set_up_venv
+  - *upload_pip_requirements
   - *set_task_expansion_macros
+  - *f_expansions_write
   - *fetch_artifacts
   - *fetch_binaries
   - *fetch_debugsymbols_archive
   - *extract_binaries
-  - *set_up_venv
-  - *upload_pip_requirements
   - *check_binary_version
   - *get_buildnumber
+  - *f_expansions_write
   - *set_up_credentials
   - *run_diskstats
   - *monitor_process_threads
@@ -1144,6 +913,7 @@ functions:
   - *set_up_venv
   - *upload_pip_requirements
   - *get_buildnumber
+  - *f_expansions_write
   - *set_up_credentials
 
   "do benchmark setup":
@@ -1151,6 +921,7 @@ functions:
   - *set_up_venv
   - *upload_pip_requirements
   - *get_buildnumber
+  - *f_expansions_write
   - *set_up_credentials
   - *fetch_benchmarks
 
@@ -1438,6 +1209,7 @@ functions:
         - "./src/evergreen/extract_generated_test_configuration.sh"
 
   "generate selected tests":
+    - *f_expansions_write
     - command: expansions.write
       params:
         file: src/expansions.yml
@@ -1586,9 +1358,7 @@ functions:
     - *set_up_venv
     - *upload_pip_requirements
     - *configure_evergreen_api_credentials
-    - command: expansions.write
-      params:
-        file: src/expansions.yml
+    - *f_expansions_write
 
     - command: shell.exec
       type: test
@@ -1599,7 +1369,7 @@ functions:
           set -o errexit
 
           ${activate_virtualenv}
-          $python buildscripts/evergreen_generate_resmoke_tasks.py --expansion-file expansions.yml --verbose
+          $python buildscripts/evergreen_generate_resmoke_tasks.py --expansion-file ../expansions.yml --verbose
 
     - command: archive.targz_pack
       params:
@@ -1647,6 +1417,7 @@ functions:
   - *set_up_remote_credentials
   - *determine_resmoke_jobs
   - *update_resmoke_jobs_expansions
+  - *f_expansions_write
   - *execute_resmoke_tests
     # The existence of the "run_tests_infrastructure_failure" file indicates this failure isn't
     # directly actionable. We use type=setup rather than type=system or type=test for this command
@@ -1677,6 +1448,7 @@ functions:
     - *set_up_remote_credentials
     - *determine_resmoke_jobs
     - *update_resmoke_jobs_expansions
+    - *f_expansions_write
     - *execute_resmoke_tests
       # The existence of the "run_tests_infrastructure_failure" file indicates this failure isn't
       # directly actionable. We use type=setup rather than type=system or type=test for this command
@@ -1689,9 +1461,7 @@ functions:
           - "./src/evergreen/check_run_tests_infrastructure_failure.sh"
 
   "scons lint":
-    - command: expansions.write
-      params:
-        file: expansions.yml
+    - *f_expansions_write
     - command: subprocess.exec
       type: test
       params:
@@ -1699,13 +1469,10 @@ functions:
         args:
           - "src/evergreen/scons_lint.sh"
         env:
-          python: ${python}
           workdir: ${workdir}
 
   "scons compile":
-    - command: expansions.write
-      params:
-        file: expansions.yml
+    - *f_expansions_write
     - command: subprocess.exec
       type: test
       params:
@@ -1713,7 +1480,6 @@ functions:
         args:
           - "src/evergreen/scons_compile.sh"
         env:
-          python: ${python}
           workdir: ${workdir}
           patch_compile_flags: ${patch_compile_flags}
 
@@ -1725,7 +1491,6 @@ functions:
       script: |
         set -o errexit
         set -o verbose
-
         # We get the raw version string (r1.2.3-45-gabcdef) from git
         MONGO_VERSION=$(git describe --abbrev=7)
         # If this is a patch build, we add the patch version id to the version string so we know
@@ -1733,28 +1498,20 @@ functions:
         if [ "${is_patch}" = "true" ]; then
           MONGO_VERSION="$MONGO_VERSION-patch-${version_id}"
         fi
-
         echo "MONGO_VERSION = ${MONGO_VERSION}"
-
         ${activate_virtualenv}
-
         # shared scons cache testing
         # if 'scons_cache_scope' enabled and project level 'disable_shared_scons_cache' is not true
         # 'scons_cache_scope' is set on a per variant basis
         # 'disable_shared_scons_cache' is set on a project level and applies to all variants
-
         # Shared - if scons_cache_scope is set, then use new shared scons cache settings
         if [ ! -z ${scons_cache_scope} ]; then
-
           if [ "${disable_shared_scons_cache}" = "true" ]; then
-
             echo "SCons Cache disabled. All shared scons settings will be ignored"
             scons_cache_scope=none
-
           else
             scons_cache_scope=${scons_cache_scope}
           fi
-
           if [ "$scons_cache_scope" = "shared" ]; then
             set +o errexit
             if [ "Windows_NT" = "$OS" ]; then
@@ -1771,10 +1528,8 @@ functions:
             fi
             set -o errexit
           fi
-
           echo "Shared Cache with setting: ${scons_cache_scope}"
           MONGO_VERSION=$MONGO_VERSION SCONS_CACHE_MODE=${scons_cache_mode|nolinked} SCONS_CACHE_SCOPE=$scons_cache_scope IS_PATCH=${is_patch} IS_COMMIT_QUEUE=${is_commit_queue|false} $python buildscripts/generate_compile_expansions_shared_cache.py --out compile_expansions.yml
-
         # Legacy Expansion generation
         else
           echo "Using legacy expansion generation"
@@ -1785,9 +1540,10 @@ functions:
         fi
 
   "apply compile expansions":
-    command: expansions.update
-    params:
-      file: src/compile_expansions.yml
+    - command: expansions.update
+      params:
+        file: src/compile_expansions.yml
+    - *f_expansions_write
 
   "do jepsen setup":
     - command: subprocess.exec
@@ -2426,6 +2182,12 @@ functions:
       optional: true
 
   "save ec2 task artifacts":
+    - *f_expansions_write
+    # deliberate double write. Powercyle expects the expansions.yml file to
+    # be in src
+    - command: expansions.write
+      params:
+        file: src/expansions.yml
     - *save_powercycle_artifacts
     - *archive_remote_ec2_artifacts
     - *archive_remote_ec2_monitor_files
@@ -2465,7 +2227,7 @@ functions:
     params:
       binary: bash
       args:
-        - "./src/evergreen/cleanup_fuse_watchdog.sh"
+        - "./src/evergreen/functions/fuse_watchdog_cleanup.sh"
 
   ### Process & archive Code Coverage artifacts ###
   "process code coverage data": &process_code_coverage_data
@@ -2473,7 +2235,7 @@ functions:
     params:
       binary: bash
       args:
-        - "./src/evergreen/process_code_coverage_data.sh"
+        - "./src/evergreen/functions/code_coverage_data_process.sh"
 
   "tar code coverage data": &tar_code_coverage_data
     command: archive.targz_pack
@@ -2922,7 +2684,9 @@ pre:
   - func: "git get project"
   - func: "kill processes"
   - func: "cleanup environment"
+  - func: "set up venv"
   - func: "set task expansion macros"
+  - func: "f_expansions_write"
 
 # Post task steps
 post:
@@ -2994,9 +2758,7 @@ tasks:
         content_type: application/gzip
         display_name: Binaries
 
-    - command: expansions.write
-      params:
-        file: expansions.yml
+    - *f_expansions_write
     - command: subprocess.exec
       type: test
       params:
@@ -3004,7 +2766,6 @@ tasks:
         args:
           - "src/evergreen/gen_feature_flags.sh"
         env:
-          python: ${python}
           workdir: ${workdir}
 
     - command: archive.targz_pack
@@ -3134,16 +2895,13 @@ tasks:
 
 - name: libdeps_graph_linting
   commands:
-    - command: expansions.write
-      params:
-        file: expansions.yml
+    - *f_expansions_write
     - command: subprocess.exec
       params:
         binary: bash
         args:
           - "src/evergreen/libdeps_setup.sh"
         env:
-          python: ${python}
           workdir: ${workdir}
 
     - func: "scons compile"
@@ -3162,7 +2920,6 @@ tasks:
         args:
           - "src/evergreen/libdeps_run.sh"
         env:
-          python: ${python}
           workdir: ${workdir}
 
     - command: s3.put
@@ -3353,16 +3110,13 @@ tasks:
 
 - name: embedded_sdk_build_cdriver
   commands:
-    - command: expansions.write
-      params:
-        file: expansions.yml
+    - func: f_expansions_write
     - command: subprocess.exec
       params:
         binary: bash
         args:
           - "src/evergreen/embedded_sdk_build_cdriver.sh"
         env:
-          python: ${python}
           workdir: ${workdir}
 
 - name: embedded_sdk_install_dev
@@ -3391,16 +3145,13 @@ tasks:
   depends_on:
     - name: embedded_sdk_install_dev
   commands:
-    - command: expansions.write
-      params:
-        file: expansions.yml
+    - *f_expansions_write
     - command: subprocess.exec
       params:
         binary: bash
         args:
           - "src/evergreen/embedded_sdk_s3_tar.sh"
         env:
-          python: ${python}
           workdir: ${workdir}
 
     # Upload it so we can download from EVG.
@@ -3437,9 +3188,7 @@ tasks:
   depends_on:
     - name: embedded_sdk_install_tests
   commands:
-    - command: expansions.write
-      params:
-        file: expansions.yml
+    - *f_expansions_write
     - command: subprocess.exec
       type: test
       params:
@@ -3447,7 +3196,6 @@ tasks:
         args:
           - "src/evergreen/embedded_sdk_tests_s3_tar.sh"
         env:
-          python: ${python}
           workdir: ${workdir}
 
     # Upload it so we can download from EVG.
@@ -3467,9 +3215,7 @@ tasks:
   depends_on:
     - name: embedded_sdk_install_tests
   commands:
-    - command: expansions.write
-      params:
-        file: expansions.yml
+    - *f_expansions_write
     - command: subprocess.exec
       type: test
       params:
@@ -3477,7 +3223,6 @@ tasks:
         args:
           - "src/evergreen/embedded_sdk_run_tests.sh"
         env:
-          python: ${python}
           workdir: ${workdir}
     - command: subprocess.exec
       params:
@@ -3485,7 +3230,6 @@ tasks:
         args:
           - "src/evergreen/embedded_sdk_run_tests_post.sh"
         env:
-          python: ${python}
           workdir: ${workdir}
 
 - name: embedded_sdk_s3_put_latest
@@ -3538,16 +3282,13 @@ tasks:
           --modules=
           --separate-debug
           DESTDIR='$BUILD_ROOT/stitch-support-lib-$MONGO_VERSION'
-    - command: expansions.write
-      params:
-        file: expansions.yml
+    - *f_expansions_write
     - command: subprocess.exec
       params:
         binary: bash
         args:
           - "src/evergreen/stitch_support_create_lib_tar.sh"
         env:
-          python: ${python}
           workdir: ${workdir}
     - command: s3.put
       params:
@@ -3581,9 +3322,7 @@ tasks:
   depends_on:
     - name: stitch_support_install_tests
   commands:
-    - command: expansions.write
-      params:
-        file: expansions.yml
+    - *f_expansions_write
     - command: subprocess.exec
       type: test
       params:
@@ -3591,7 +3330,6 @@ tasks:
         args:
           - "src/evergreen/stitch_support_run_tests.sh"
         env:
-          python: ${python}
           workdir: ${workdir}
 
 - name: compile_benchmarks
@@ -3601,6 +3339,7 @@ tasks:
     - func: "set up venv"
     - func: "upload pip requirements"
     - func: "get buildnumber"
+    - func: "f_expansions_write"
     - func: "set up credentials"
     - func: "use WiredTiger develop" # noop if ${use_wt_develop} is not "true"
     - func: "set up win mount script"
@@ -3692,9 +3431,7 @@ tasks:
   - func: "set task expansion macros"
   - func: "set up venv"
   - func: "upload pip requirements"
-  - command: expansions.write
-    params:
-      file: expansions.yml
+  - func: "f_expansions_write"
   - command: subprocess.exec
     type: test
     params:
@@ -3702,7 +3439,6 @@ tasks:
       args:
         - "src/evergreen/lint_yaml.sh"
       env:
-        python: ${python}
         workdir: ${workdir}
 
 - name: lint_shellscripts
@@ -3731,9 +3467,7 @@ tasks:
     - func: "set up venv"
     - func: "upload pip requirements"
     - func: "do setup"
-    - command: expansions.write
-      params:
-        file: expansions.yml
+    - func: "f_expansions_write"
     - command: subprocess.exec
       type: test
       params:
@@ -3741,7 +3475,6 @@ tasks:
         args:
           - "src/evergreen/check_idl_compat.sh"
         env:
-          python: ${python}
           workdir: ${workdir}
 
 - name: burn_in_tests_gen
@@ -3751,9 +3484,7 @@ tasks:
   - func: "upload pip requirements"
   - func: "configure evergreen api credentials"
   - *do_multiversion_setup
-  - command: expansions.write
-    params:
-      file: expansions.yml
+  - func: "f_expansions_write"
   - command: subprocess.exec
     type: test
     params:
@@ -3761,7 +3492,6 @@ tasks:
       args:
         - "src/evergreen/burn_in_tests.sh"
       env:
-        python: ${python}
         workdir: ${workdir}
   - command: archive.targz_pack
     params:
@@ -3792,9 +3522,7 @@ tasks:
   - func: "upload pip requirements"
   - func: "configure evergreen api credentials"
   - func: "do multiversion setup"
-  - command: expansions.write
-    params:
-      file: expansions.yml
+  - func: "f_expansions_write"
   - command: subprocess.exec
     type: test
     params:
@@ -3802,7 +3530,6 @@ tasks:
       args:
         - "src/evergreen/burn_in_tests_multiversion.sh"
       env:
-        python: ${python}
         workdir: ${workdir}
 
   - command: archive.targz_pack
@@ -5269,7 +4996,6 @@ tasks:
       args:
         - "src/evergreen/external_auth_pip.sh"
       env:
-        python: ${python}
         workdir: ${workdir}
   - func: "run tests"
     vars:
@@ -5279,9 +5005,7 @@ tasks:
   name: external_auth_aws
   commands:
   - func: "do setup"
-  - command: expansions.write
-    params:
-      file: expansions.yml
+  - func: "f_expansions_write"
   - command: subprocess.exec
     params:
       binary: bash
@@ -5289,7 +5013,6 @@ tasks:
       args:
         - "src/evergreen/external_auth_aws_setup.sh"
       env:
-        python: ${python}
         workdir: ${workdir}
   - command: subprocess.exec
     params:
@@ -5297,7 +5020,6 @@ tasks:
       args:
         - "src/evergreen/external_auth_aws_pip.sh"
       env:
-        python: ${python}
         workdir: ${workdir}
   - func: "run tests"
     vars:
@@ -7042,9 +6764,7 @@ tasks:
   patchable: false
   commands:
   - func: "do non-compile setup"
-  - command: expansions.write
-    params:
-      file: expansions.yml
+  - func: "f_expansions_write"
   - command: subprocess.exec
     type: setup
     params:
@@ -7053,7 +6773,6 @@ tasks:
       args:
         - "src/evergreen/blackduck_setup.sh"
       env:
-        python: ${python}
         workdir: ${workdir}
   - command: subprocess.exec
     type: test
@@ -7062,7 +6781,6 @@ tasks:
       args:
         - "src/evergreen/blackduck_hub.sh"
       env:
-        python: ${python}
         workdir: ${workdir}
 
 - name: tla_plus
@@ -7128,16 +6846,13 @@ tasks:
         task_compile_flags: >-
           --separate-debug
           --legacy-tarball
-    - command: expansions.write
-      params:
-        file: expansions.yml
+    - func: "f_expansions_write"
     - command: subprocess.exec
       params:
         binary: bash
         args:
           - "src/evergreen/package.sh"
         env:
-          python: ${python}
           workdir: ${workdir}
     - command: s3.put
       params:
@@ -8068,10 +7783,13 @@ task_groups:
     - func: "git get project"
     - func: "kill processes"
     - func: "cleanup environment"
-    - func: "set task expansion macros"
     - func: "set up venv"
+    - func: "upload pip requirements"
+    - func: "set task expansion macros"
+    - func: "f_expansions_write"
     - func: "configure evergreen api credentials"
     - func: "get buildnumber"
+    - func: "f_expansions_write"
     - func: "set up credentials"
     - func: "set up win mount script"
     - func: "generate compile expansions"
@@ -8092,10 +7810,11 @@ task_groups:
     - command: manifest.load
     - func: "git get project"
     - func: "get buildnumber"
-    - func: "set up credentials"
-    - func: "set task expansion macros"
     - func: "set up venv"
     - func: "upload pip requirements"
+    - func: "set task expansion macros"
+    - func: "f_expansions_write"
+    - func: "set up credentials"
     - func: "set up win mount script"
     - func: "generate compile expansions"
   teardown_group:
@@ -8115,10 +7834,11 @@ task_groups:
     - command: manifest.load
     - func: "git get project"
     - func: "get buildnumber"
-    - func: "set up credentials"
-    - func: "set task expansion macros"
     - func: "set up venv"
     - func: "upload pip requirements"
+    - func: "set task expansion macros"
+    - func: "f_expansions_write"
+    - func: "set up credentials"
     - func: "set up win mount script"
     - func: "generate compile expansions"
   teardown_group:
@@ -12370,6 +12090,8 @@ buildvariants:
   run_on:
   - windows-vsCurrent-small
   stepback: false
+  expansions:
+    python: '/cygdrive/c/python/python37/python.exe'
   tasks:
   - name: win_shared_scons_cache_pruning
 
diff --git a/etc/expansions.default.yml b/etc/expansions.default.yml
index 022d9009ae5..1d59269e742 100644
--- a/etc/expansions.default.yml
+++ b/etc/expansions.default.yml
@@ -11,7 +11,6 @@ resmoke_jobs_factor: "1"
 resmoke_jobs_max: "0"
 timeout_secs: "0"
 exec_timeout_secs: "0"
-python: /opt/mongodbtoolchain/v3/bin/python3
 multiversion_edition: base
 multiversion_platform: linux_x86_64
 multiversion_architecture: x86_64