SERVER-41795 Enable Windows Shared SCons Cache

2 files changed, 98 insertions, 24 deletions

diff --git a/buildscripts/generate_compile_expansions_shared_cache.py b/buildscripts/generate_compile_expansions_shared_cache.py
index ee6ce5ad04d..fcfa491009b 100755
--- a/buildscripts/generate_compile_expansions_shared_cache.py
+++ b/buildscripts/generate_compile_expansions_shared_cache.py
@@ -93,9 +93,12 @@ def generate_scons_cache_expansions():
 
     # Global shared cache using EFS
     if os.getenv("SCONS_CACHE_SCOPE") == "shared":
-        default_cache_path = os.path.join("/efs", system_uuid, "scons-cache")
+        if sys.platform.startswith("win"):
+            shared_mount_root = 'X:\\'
+        else:
+            shared_mount_root = '/efs'
+        default_cache_path = os.path.join(shared_mount_root, system_uuid, "scons-cache")
         expansions["scons_cache_path"] = default_cache_path
-
         # Patches are read only
         if os.getenv("IS_PATCH"):
             expansions[
diff --git a/etc/evergreen.yml b/etc/evergreen.yml
index 9f2eac86021..be10d88fcd1 100644
--- a/etc/evergreen.yml
+++ b/etc/evergreen.yml
@@ -261,6 +261,7 @@ variables:
   - func: "get buildnumber"
   - func: "set up credentials"
   - func: "fetch and build OpenSSL"
+  - func: "set up win mount script"
   - func: "generate compile expansions"
   teardown_group:
   - func: "scons cache pruning"
@@ -635,6 +636,18 @@ functions:
         build_phase='${task_name}_${execution}'
         END_OF_CREDS
 
+  "set up win mount script": &set_up_win_mount_script
+    command: shell.exec
+    params:
+      working_dir: src
+      shell: bash
+      silent: true
+      script: |
+        cat <<EOF > win_mount.sh
+        net use X: '\\\\${win_scons_endpoint}\\share' /USER:${win_scons_user} '${win_scons_pass}'
+        EOF
+        chmod +x win_mount.sh
+
   "set up notary client credentials": &set_up_notary_client_credentials
     command: shell.exec
     params:
@@ -888,24 +901,38 @@ functions:
 
 
           set +o errexit
-          mount | grep "\/efs" > /dev/null
+
+          if [ "Windows_NT" = "$OS" ]; then
+            ./win_mount.sh
+          else
+            mount | grep "\/efs" > /dev/null
+          fi
           if [ $? -eq 0 ]; then
             echo "Shared cache is already mounted"
           else
             echo "Shared cache - mounting file system"
-            sudo mount /efs
+            if [ "Windows_NT" = "$OS" ]; then
+              ./win_mount.sh
+            else
+              sudo mount /efs
+            fi
           fi
           set -o errexit
 
-          dirs=$(ls -l /efs | grep -v total | awk '{print $NF}')
+          if [ "Windows_NT" = "$OS" ]; then
+            cache_folder=/cygdrive/x
+          else
+            cache_folder=/efs
+          fi
+          dirs=$(ls -l $cache_folder | grep -v total | awk '{print $NF}')
 
           echo "Pruning shared SCons directories"
 
           for dir in $dirs; do
-            echo "Pruning /efs/$dir/scons-cache"
+            echo "Pruning $cache_folder/$dir/scons-cache"
 
-            if [ -e /efs/$dir/info/distro_name ]; then
-              distro=$(cat /efs/$dir/info/distro_name)
+            if [ -e $cache_folder/$dir/info/distro_name ]; then
+              distro=$(cat $cache_folder/$dir/info/distro_name)
             fi
 
             # Set cache sizes by distro
@@ -919,11 +946,20 @@ functions:
                       ;;
             esac
 
-            sudo python buildscripts/scons_cache_prune.py --cache-dir /efs/$dir/scons-cache --cache-size $cache_size --prune-ratio 1.0
+            if [ "Windows_NT" = "$OS" ]; then
+              echo "dir="$dir
+              python buildscripts/scons_cache_prune.py --cache-dir x:/$dir/scons-cache --cache-size $cache_size --prune-ratio 1.0
+            else
+              sudo python buildscripts/scons_cache_prune.py --cache-dir /efs/$dir/scons-cache --cache-size $cache_size --prune-ratio 1.0
+            fi
             echo ""
           done
 
-          sudo umount /efs || true
+          if [ "Windows_NT" = "$OS" ]; then
+            net use X: /delete || true
+          else
+            sudo umount /efs || true
+          fi
 
         else
           echo "Not on master, shared SCons cache pruning skipped"
@@ -953,8 +989,12 @@ functions:
         set -o verbose
 
         if [ "${scons_cache_scope}" = "shared" ]; then
-            ${set_sudo}
-            $sudo umount /efs || true
+            if [ "Windows_NT" = "$OS" ]; then
+                net use X: /delete || true
+            else
+                ${set_sudo}
+                $sudo umount /efs || true
+            fi
         fi
 
   "build new tools":
@@ -1607,13 +1647,17 @@ functions:
 
           if [ "$scons_cache_scope" = "shared" ]; then
             set +o errexit
-            mount | grep "\/efs" > /dev/null
-            if [ $? -eq 0 ]; then
-              echo "Shared cache is already mounted"
+            if [ "Windows_NT" = "$OS" ]; then
+              ./win_mount.sh
             else
-              echo "Shared cache - mounting file system"
-              ${set_sudo}
-              $sudo mount /efs
+              mount | grep "\/efs" > /dev/null
+              if [ $? -eq 0 ]; then
+                echo "Shared cache is already mounted"
+              else
+                echo "Shared cache - mounting file system"
+                ${set_sudo}
+                $sudo mount /efs
+              fi
             fi
             set -o errexit
           fi
@@ -3937,6 +3981,7 @@ tasks:
     - func: "setup android toolchain" # noop if ${setup_android_toolchain} is not "true"
     - func: "set task expansion macros"
     - func: "set up virtualenv"
+    - func: "set up win mount script"
     - func: "generate compile expansions"
     - func: "apply compile expansions"
     - command: s3.get
@@ -4029,6 +4074,7 @@ tasks:
     - func: "setup android toolchain" # noop if ${setup_android_toolchain} is not "true"
     - func: "set task expansion macros"
     - func: "set up virtualenv"
+    - func: "set up win mount script"
     - func: "generate compile expansions"
     - func: "apply compile expansions"
     - command: s3.get
@@ -4594,6 +4640,7 @@ tasks:
     - func: "set up virtualenv"
     - func: "get buildnumber"
     - func: "set up credentials"
+    - func: "set up win mount script"
     - func: "generate compile expansions"
     # Then we load the generated version data into the agent so we can use it in task definitions
     - func: "apply compile expansions"
@@ -5835,6 +5882,7 @@ tasks:
   - command: manifest.load
   - func: "git get project"
   - func: "do setup"
+  - func: "set up win mount script"
   - func: "generate compile expansions"  # Generate compile expansions needs to be run to mount the shared scons cache.
   - func: "apply compile expansions"
   - func: "scons compile"
@@ -5854,6 +5902,7 @@ tasks:
   - command: manifest.load
   - func: "git get project"
   - func: "do setup"
+  - func: "set up win mount script"
   - func: "generate compile expansions"  # Generate compile expansions needs to be run to mount the shared scons cache.
   - func: "apply compile expansions"
   - func: "scons compile"
@@ -5873,6 +5922,7 @@ tasks:
   - command: manifest.load
   - func: "git get project"
   - func: "do setup"
+  - func: "set up win mount script"
   - func: "generate compile expansions"  # Generate compile expansions needs to be run to mount the shared scons cache.
   - func: "apply compile expansions"
   - func: "scons compile"
@@ -5892,6 +5942,7 @@ tasks:
   - command: manifest.load
   - func: "git get project"
   - func: "do setup"
+  - func: "set up win mount script"
   - func: "generate compile expansions"  # Generate compile expansions needs to be run to mount the shared scons cache.
   - func: "apply compile expansions"
   - func: "scons compile"
@@ -8024,6 +8075,15 @@ tasks:
   - func: "git get project"
   - func: "shared scons cache pruning"
 
+- name: win_shared_scons_cache_pruning
+  exec_timeout_secs: 7200 # 2 hour timeout for the task overall
+  depends_on: []
+  commands:
+  - command: manifest.load
+  - func: "git get project"
+  - func: "set up win mount script"
+  - func: "shared scons cache pruning"
+
 #######################################
 #             Task Groups             #
 #######################################
@@ -8056,6 +8116,7 @@ task_groups:
     - func: "setup android toolchain" # noop if ${setup_android_toolchain} is not "true"
     - func: "set task expansion macros"
     - func: "set up virtualenv"
+    - func: "set up win mount script"
     - func: "generate compile expansions"
   teardown_group:
     - func: "scons cache pruning"
@@ -9060,7 +9121,7 @@ buildvariants:
     num_scons_link_jobs_available: $(( $(grep -c ^processor /proc/cpuinfo) / 4 ))
     python: '/cygdrive/c/python/python37/python.exe'
     ext: zip
-    use_scons_cache: true
+    scons_cache_scope: shared
     multiversion_platform: windows
     multiversion_edition: enterprise
     tooltags: "ssl sasl gssapi"
@@ -9097,7 +9158,7 @@ buildvariants:
     num_scons_link_jobs_available: $(( $(grep -c ^processor /proc/cpuinfo) / 4 ))
     python: '/cygdrive/c/python/python37/python.exe'
     ext: zip
-    use_scons_cache: true
+    scons_cache_scope: shared
     multiversion_platform: windows
     multiversion_edition: enterprise
     tooltags: ""
@@ -9250,7 +9311,7 @@ buildvariants:
     num_scons_link_jobs_available: $(( $(grep -c ^processor /proc/cpuinfo) / 4 ))
     python: '/cygdrive/c/python/python37/python.exe'
     ext: zip
-    use_scons_cache: true
+    scons_cache_scope: shared
     test_flags: |- # Use the ServiceExecutorAdaptive with a reasonable number of starting threads
         --serviceExecutor=adaptive \
         --mongodSetParameters="adaptiveServiceExecutorReservedThreads: 8" \
@@ -9284,7 +9345,7 @@ buildvariants:
     python: '/cygdrive/c/python/python37/python.exe'
     test_flags: --storageEngine=inMemory --excludeWithAnyTags=requires_persistence,requires_journaling,uses_transactions
     ext: zip
-    use_scons_cache: true
+    scons_cache_scope: shared
     multiversion_platform: windows
     multiversion_edition: enterprise
     tooltags: "ssl sasl gssapi"
@@ -9340,7 +9401,7 @@ buildvariants:
     num_scons_link_jobs_available: $(( $(grep -c ^processor /proc/cpuinfo) / 4 ))
     python: '/cygdrive/c/python/python37/python.exe'
     ext: zip
-    use_scons_cache: true
+    scons_cache_scope: shared
     tooltags: "ssl"
     build_mongoreplay: false
     large_distro_name: windows-64-vs2017-compile
@@ -9400,7 +9461,7 @@ buildvariants:
     num_scons_link_jobs_available: $(( $(grep -c ^processor /proc/cpuinfo) / 4 ))
     python: '/cygdrive/c/python/python37/python.exe'
     ext: zip
-    use_scons_cache: true
+    scons_cache_scope: shared
     tooltags: "ssl sasl gssapi"
     build_mongoreplay: false
   tasks:
@@ -12158,3 +12219,13 @@ buildvariants:
   stepback: false
   tasks:
   - name: shared_scons_cache_pruning
+
+
+- name: windows-shared-scons-cache-pruning
+  display_name: "Windows shared SCons Cache Pruning"
+  run_on:
+  - windows-64-vs2017-test
+  batchtime: 1440 # 1 day
+  stepback: false
+  tasks:
+  - name: win_shared_scons_cache_pruning