SERVER-33340 Turn on shared cache for non-shipping builders

3 files changed, 192 insertions, 45 deletions

diff --git a/buildscripts/generate_compile_expansions_shared_cache.py b/buildscripts/generate_compile_expansions_shared_cache.py
index 485be3806a3..4dc937cce12 100755
--- a/buildscripts/generate_compile_expansions_shared_cache.py
+++ b/buildscripts/generate_compile_expansions_shared_cache.py
@@ -90,7 +90,7 @@ def generate_scons_cache_expansions():
     # Set the scons shared cache setting
 
     # Global shared cache using EFS
-    if os.getenv("USE_SCONS_CACHE") == "shared":
+    if os.getenv("SCONS_CACHE_SCOPE") == "shared":
         default_cache_path = os.path.join("/efs", system_uuid, "scons-cache")
         expansions["scons_cache_path"] = default_cache_path
 
@@ -102,7 +102,7 @@ def generate_scons_cache_expansions():
             expansions["scons_cache_args"] = "--cache={0} --cache-dir='{1}'".format(scons_cache_mode, default_cache_path)
 
     # Local shared cache - host-based
-    elif os.getenv("USE_SCONS_CACHE") == "local":
+    elif os.getenv("SCONS_CACHE_SCOPE") == "local":
 
         if sys.platform.startswith("win"):
             default_cache_path_base = r"z:\data\scons-cache"
@@ -115,7 +115,7 @@ def generate_scons_cache_expansions():
     # No cache
     else:
         # Anything else is 'none'
-        print("No shared cache used")
+        print("No cache used")
 
     return expansions
 
diff --git a/buildscripts/prune_check.py b/buildscripts/prune_check.py
new file mode 100644
index 00000000000..da9dab59f72
--- /dev/null
+++ b/buildscripts/prune_check.py
@@ -0,0 +1,103 @@
+#!/usr/bin/env python2
+
+""" This program stamps the shared scons directory with a timestamp so we can
+    determine the last prune time and run the prune script on a schedule.
+    It is meant to be invoked from the shell:
+
+    if python prune.py; then
+      echo 'Pruning'
+    else
+      echo 'Less than 24 hours, waiting ...'
+    fi
+
+    If it is time to prune (ie. more than 24 hours since the last timestamp),
+    the script exits with a 0 return code.
+    Otherwise the script returns exit code 1 and pruning is skipped.
+"""
+
+import argparse
+from datetime import datetime
+import os
+import sys
+import time
+
+DATE_TIME_STR = "%Y-%m-%d %H:%M:%S"
+
+
+def get_prune_file_path(mount_point):
+    """ Get the shared scons directory for this AMI """
+    with open('/etc/mongodb-build-system-id', 'r') as f:
+        uuid = f.read().strip()
+    return os.path.join(mount_point, uuid, 'info', 'last_prune_time')
+
+
+def write_last_prune_time(last_prune_time, prune_file_path):
+    """ Write the last prune timestamp in a 'last_prune_time' file """
+    with open(prune_file_path, 'w') as f:
+        f.write(last_prune_time.strftime(DATE_TIME_STR) + '\n')
+
+
+def retrieve_last_prune_time(prune_file_path):
+    """ Get the last prune time from the 'last_prune_time' file """
+    if os.path.isfile(prune_file_path):
+        with open(prune_file_path, 'r') as f:
+            last_prune_time_str = f.read().strip()
+            last_prune_time = datetime.strptime(last_prune_time_str, DATE_TIME_STR)
+    else:
+        last_prune_time = datetime.utcnow()
+        write_last_prune_time(last_prune_time, prune_file_path)
+
+    return last_prune_time
+
+
+def check_last_prune_time(args):
+    """ Returns exit code 0 if time to run again, else returns exit code 1
+        This is meant to be called from the shell
+    """
+
+    seconds_since_last_prune = args.prune_seconds
+    prune_file_path = get_prune_file_path(args.mount_point)
+
+    now = datetime.utcnow()
+    last_prune_time = retrieve_last_prune_time(prune_file_path)
+
+    diff = now - last_prune_time
+
+    # if it's been longer than 'seconds_since_last_prune',
+    # update/overwrite the prune file and return 0 to the shell.
+    # A 0 return code signals our Evergreen task that we should run the prune script
+    # Otherwise, return 1 and skip pruning.
+    if diff.total_seconds() > seconds_since_last_prune:
+        print("It has been {0:.2f} seconds ({1:.2f} hours) since last prune.  Pruning directory"
+              .format(diff.total_seconds(), diff.total_seconds()/60/60))
+        last_prune_time = datetime.utcnow()
+        write_last_prune_time(last_prune_time, prune_file_path)
+        sys.exit(0)
+    else:
+        print("It has been {0:.2f} seconds ({1:.2f} hours) since last prune.  Skipping directory prune."
+              .format(diff.total_seconds(), diff.total_seconds()/60/60))
+        sys.exit(1)
+
+
+def get_command_line_args():
+    """ Get the command line arguments """
+    parser = argparse.ArgumentParser()
+    parser.add_argument('-m', '--mount_point', type=str, required=False,
+                        help="The base mount where efs is mounted",
+                        default='/efs')
+    parser.add_argument('-p', '--prune_seconds', type=int, required=False,
+                        help="Seconds to wait since last prune - default is 86400 (one day)",
+                        default=86400)
+    args = parser.parse_args()
+    return args
+
+
+def main():
+    args = get_command_line_args()
+    mount_point = args.mount_point
+
+    check_last_prune_time(args)
+
+
+if __name__ == '__main__':
+    main()
diff --git a/etc/evergreen.yml b/etc/evergreen.yml
index 5764659459e..e68e9d90860 100644
--- a/etc/evergreen.yml
+++ b/etc/evergreen.yml
@@ -87,18 +87,37 @@ variables:
 
       ${activate_virtualenv}
       # shared scons cache testing
-      # if 'shared_scons_cache' enabled and project level 'disable_shared_scons_cache' is not true
-      # 'shared_scons_cache is set on a per variant basis
+      # if 'scons_cache_scope' enabled and project level 'disable_shared_scons_cache' is not true
+      # 'scons_cache_scope' is set on a per variant basis
       # 'disable_shared_scons_cache' is set on a project level and applies to all variants
-      if [ "${shared_scons_cache}" = "true" ] && [ "${disable_shared_scons_cache}" != "true" ]; then
-        echo "Shared cache - mounting file system"
-        sudo mount /efs -v || mount /efs
-        MONGO_VERSION=$MONGO_VERSION SCONS_CACHE_MODE=${scons_cache_mode|nolinked} USE_SCONS_CACHE=shared IS_PATCH=${is_patch} ${python|/opt/mongodbtoolchain/v2/bin/python2} buildscripts/generate_compile_expansions_shared_cache.py --out compile_expansions.yml
+
+      # Shared - if scons_cache_scope is set, then use new shared scons cache settings
+      if [ ! -z ${scons_cache_scope} ]; then
+
+        if [ "${disable_shared_scons_cache}" = "true" ]; then
+
+          echo "SCons Cache disabled. All shared scons settings will be ignored"
+          scons_cache_scope=none
+
+        else
+          scons_cache_scope=${scons_cache_scope}
+        fi
+
+        if [ $scons_cache_scope = "shared" ]; then
+          echo "Shared cache - mounting file system"
+          sudo mount /efs || mount /efs
+        fi
+
+        echo "Shared Cache with setting: ${scons_cache_scope}"
+        MONGO_VERSION=$MONGO_VERSION SCONS_CACHE_MODE=${scons_cache_mode|nolinked} SCONS_CACHE_SCOPE=$scons_cache_scope IS_PATCH=${is_patch} ${python|/opt/mongodbtoolchain/v2/bin/python2} buildscripts/generate_compile_expansions_shared_cache.py --out compile_expansions.yml
+
+      # Legacy Expansion generation
       else
-      # Proceed with regular expansions generated
-      # This script converts the generated version string into a sanitized version string for
-      # use by scons and uploading artifacts as well as information about for the scons cache.
-      MONGO_VERSION=$MONGO_VERSION SCONS_CACHE_MODE=${scons_cache_mode|nolinked} USE_SCONS_CACHE=${use_scons_cache|false} ${python|/opt/mongodbtoolchain/v2/bin/python2} buildscripts/generate_compile_expansions.py --out compile_expansions.yml
+        echo "Using legacy expansion generation"
+        # Proceed with regular expansions generated
+        # This script converts the generated version string into a sanitized version string for
+        # use by scons and uploading artifacts as well as information about for the scons cache.
+        MONGO_VERSION=$MONGO_VERSION SCONS_CACHE_MODE=${scons_cache_mode|nolinked} USE_SCONS_CACHE=${use_scons_cache|false} ${python|/opt/mongodbtoolchain/v2/bin/python2} buildscripts/generate_compile_expansions.py --out compile_expansions.yml
       fi
 
 - &apply_compile_expansions
@@ -514,6 +533,51 @@ functions:
             PYTHON=$python ${compile_env|} src/buildscripts/setup-android-toolchain.sh
         fi
 
+  "scons cache pruning" :
+    command: shell.exec
+    params:
+      working_dir: src
+      script: |
+        set -o errexit
+        set -o verbose
+        # removes files from the (local) or shared scons cache when it's over a
+        # threshold, to the $prune_ratio percentage. Ideally override
+        # these default values in the distro config in evergreen.
+
+        ${activate_virtualenv}
+        prune_scons_cache="$python buildscripts/scons_cache_prune.py --cache-dir ${scons_cache_path} --cache-size ${scons_cache_size|200} --prune-ratio ${scons_prune_ratio|0.8}"
+        $prune_scons_cache
+        if [ "${scons_cache_scope}" = "shared" ]; then
+            if [ "${shared_scons_pruning}" = "true" ] && [ "${is_patch}" != "true" ]; then
+              echo "shared_scons_pruning is set to true, checking last prune time"
+              if $python buildscripts/prune_check.py; then
+                echo "Pruning shared cache"
+                $prune_scons_cache
+              else
+                echo "We are less than the 'shared_scons_pruning' num seconds threshold, pruning will be skipped"
+              fi
+            else
+              echo "Shared cache enabled but 'shared_scons_pruning' is not enabled for this variant or we are running a patch build, pruning will be skipped"
+            fi
+        else
+            echo "Legacy local shared cache prune"
+            if [ -d "${scons_cache_path}" ]; then
+              $prune_scons_cache
+            fi
+        fi
+
+  "umount shared scons directory" :
+    command: shell.exec
+    params:
+      working_dir: src
+      script: |
+        set -o errexit
+        set -o verbose
+
+        if [ "${scons_cache_scope}" = "shared" ]; then
+            sudo umount /efs || umount /efs || true
+        fi
+
   "build cdriver" :
     command: shell.exec
     params:
@@ -2196,31 +2260,6 @@ post:
       optional: true
 
   # Cleanup steps.
-  - command: shell.exec
-    params:
-      working_dir: src
-      script: |
-        # removes files from the (local) scons cache when it's over a
-        # threshold, to the $prune_ratio percentage. Ideally override
-        # these default values in the distro config in evergreen.
-
-        ${activate_virtualenv}
-        if [ -d "${scons_cache_path}" ]; then
-            $python buildscripts/scons_cache_prune.py --cache-dir '${scons_cache_path}' --cache-size ${scons_cache_size|200} --prune-ratio ${scons_prune_ratio|0.8}
-        fi
-
-  - command: shell.exec
-    type: test
-    params:
-      working_dir: src
-      shell: bash
-      script: |
-        set -o verbose
-        set -o errexit
-
-        # umount shared scons mounts
-        sudo umount /efs || umount /efs
-
   - func: "cleanup environment"
 
 timeout:
@@ -2500,6 +2539,8 @@ tasks:
         ignore_artifacts_for_spawn: false
         files:
           - src/artifacts.json
+    - func: "scons cache pruning"
+    - func: "umount shared scons directory"
 
 ## compile_all - build all scons targets including unittests ##
 - name: compile_all
@@ -2563,6 +2604,8 @@ tasks:
     # Run the android embedded CAPI tests in the android emulator
     # noop if ${enable_android_emu_tests} is not "true"
     - func: "run android emulator tests"
+    - func: "scons cache pruning"
+    - func: "umount shared scons directory"
 
 ## lint ##
 - name: lint
@@ -4767,7 +4810,7 @@ buildvariants:
     compile_flags: -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_gcc.vars
     num_jobs_available: $(grep -c ^processor /proc/cpuinfo)
     test_flags: --repeat=10 --shuffle
-    use_scons_cache: true
+    scons_cache_scope: shared
     build_mongoreplay: true
   tasks:
   - name: compile
@@ -4899,7 +4942,7 @@ buildvariants:
     # Running WiredTiger with --nojournal in a replica set is no longer supported,
     # so this variant does not include replica set tests.
     test_flags: --nojournal --excludeWithAnyTags=requires_journaling,requires_replication,requires_mmapv1
-    use_scons_cache: true
+    scons_cache_scope: shared
     build_mongoreplay: true
   tasks:
   - name: compile
@@ -4934,7 +4977,7 @@ buildvariants:
     compile_flags: -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_gcc.vars
     num_jobs_available: $(grep -c ^processor /proc/cpuinfo)
     test_flags: --wiredTigerCollectionConfig=type=lsm --wiredTigerIndexConfig=type=lsm --excludeWithAnyTags=requires_mmapv1
-    use_scons_cache: true
+    scons_cache_scope: shared
     build_mongoreplay: true
   tasks:
   - name: compile
@@ -6831,7 +6874,8 @@ buildvariants:
     test_flags: --excludeWithAnyTags=requires_mmapv1
     num_jobs_available: $(($(grep -c ^processor /proc/cpuinfo) / 2)) # Avoid starting too many mongod's
     compile_flags: MONGO_DISTMOD=ubuntu1604 --dbg=on --opt=on -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_gcc.vars
-    shared_scons_cache: true
+    scons_cache_scope: shared
+    shared_scons_pruning: true
     build_mongoreplay: true
   tasks:
   - name: compile
@@ -8233,7 +8277,7 @@ buildvariants:
     multiversion_platform: rhel62
     multiversion_edition: enterprise
     num_jobs_available: $(grep -c ^processor /proc/cpuinfo)
-    use_scons_cache: true
+    scons_cache_scope: shared
     build_mongoreplay: true
   tasks:
   - name: compile
@@ -8452,7 +8496,7 @@ buildvariants:
     test_flags: --storageEngine=ephemeralForTest --excludeWithAnyTags=requires_persistence,requires_fsync,SERVER-21420,SERVER-21658,requires_journaling,requires_mmapv1,requires_wiredtiger
     compile_flags: -j$(grep -c ^processor /proc/cpuinfo) --dbg=off --opt=on --variables-files=etc/scons/mongodbtoolchain_gcc.vars
     num_jobs_available: $(grep -c ^processor /proc/cpuinfo)
-    use_scons_cache: true
+    scons_cache_scope: shared
     build_mongoreplay: true
   tasks:
   - name: compile
@@ -9122,7 +9166,7 @@ buildvariants:
     lang_environment: LANG=C
     compile_flags: --ssl MONGO_DISTMOD=ubuntu1604 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_gcc.vars --link-model=dynamic
     num_jobs_available: $(grep -c ^processor /proc/cpuinfo)
-    shared_scons_cache: true
+    scons_cache_scope: shared
     scons_cache_mode: all
     build_mongoreplay: true
   tasks: