SERVER-62285 added processing of cache-dir push failures.

(cherry picked from commit 021c3209622eb7e273e31424bdca2f70b964e1dc)

3 files changed, 41 insertions, 35 deletions

diff --git a/buildscripts/scons_cache_prune.py b/buildscripts/scons_cache_prune.py
index 4306cc56a96..6391fbc0b9d 100644
--- a/buildscripts/scons_cache_prune.py
+++ b/buildscripts/scons_cache_prune.py
@@ -24,10 +24,9 @@ GIGBYTES = 1024 * 1024 * 1024
 CacheItem = collections.namedtuple("CacheContents", ["path", "time", "size"])
 
 
-def get_cachefile_size(file_path):
+def get_cachefile_size(file_path, is_cksum):
     """Get the size of the cachefile."""
-
-    if file_path.lower().endswith('.cksum') or file_path.lower().endswith('.cksum.del'):
+    if is_cksum:
         size = 0
         for cksum_path in os.listdir(file_path):
             cksum_path = os.path.join(file_path, cksum_path)
@@ -51,19 +50,26 @@ def collect_cache_contents(cache_path):
             for file_name in os.listdir(path):
                 file_path = os.path.join(path, file_name)
                 # Cache prune script is allowing only directories with this extension
-                # which comes from the validate_cache_dir.py tool in scons, it must match
+                # which comes from the validate_cache_dir.py tool in SCons, it must match
                 # the extension set in that file.
-                if os.path.isdir(file_path) and not file_path.lower().endswith(
-                        '.cksum') and not file_path.lower().endswith('.cksum.del'):
-                    LOGGER.warning(
-                        "cache item %s is a directory and not a file. "
-                        "The cache may be corrupt.", file_path)
-                    continue
+                cksum_type = False
+                if os.path.isdir(file_path):
+                    hash_length = -32
+                    tmp_length = -len('.cksum.tmp') + hash_length
+                    cksum_type = (file_path.lower().endswith('.cksum')
+                                  or file_path.lower().endswith('.del')
+                                  or file_path.lower()[tmp_length:hash_length] == '.cksum.tmp')
+
+                    if not cksum_type:
+                        LOGGER.warning(
+                            "cache item %s is a directory and not a file. "
+                            "The cache may be corrupt.", file_path)
+                        continue
 
                 try:
 
                     item = CacheItem(path=file_path, time=os.stat(file_path).st_atime,
-                                     size=get_cachefile_size(file_path))
+                                     size=get_cachefile_size(file_path, cksum_type))
 
                     total += item.size
 
diff --git a/site_scons/site_tools/validate_cache_dir.py b/site_scons/site_tools/validate_cache_dir.py
index b1df5326eab..b5faee9b3e0 100644
--- a/site_scons/site_tools/validate_cache_dir.py
+++ b/site_scons/site_tools/validate_cache_dir.py
@@ -23,6 +23,7 @@
 import datetime
 import json
 import os
+import sys
 import pathlib
 import shutil
 import tempfile
@@ -93,8 +94,8 @@ class CacheDirValidate(SCons.CacheDir.CacheDir):
             raise UnsupportedError(cls.__name__, "timestamp-newer")
 
         src_file = cls.get_file_contents_path(src)
-
-        if cls.get_bad_cachefile_path(src).exists():
+        # using os.path.exists here because: https://bugs.python.org/issue35306
+        if os.path.exists(str(cls.get_bad_cachefile_path(src))):
             raise InvalidChecksum(cls.get_hash_path(src_file), dst, f"cachefile marked as bad checksum")
 
         csig = None
@@ -129,10 +130,10 @@ class CacheDirValidate(SCons.CacheDir.CacheDir):
         # dst is bsig/file from cachepath method, so
         # we make sure to make the bsig dir first
         dst = pathlib.Path(dst)
-        os.makedirs(dst, exist_ok=True)
-
         dst_file = dst / dst.name.split('.')[0]
+
         try:
+            os.makedirs(dst, exist_ok=True)
             super().copy_to_cache(env, src, dst_file)
         except OSError as ex:
             raise CacheTransferFailed(src, dst_file, f"failed to copy to cache: {ex}") from ex
@@ -198,33 +199,31 @@ class CacheDirValidate(SCons.CacheDir.CacheDir):
                 'cache_dir': str(pathlib.Path(cachefile).parent.parent),
             })
 
+            # capture exception information
+            if sys.exc_info()[1]:
+                json_data.update({'error': self._format_exception_msg()})
+
             self.json_log.write(json.dumps(json_data) + '\n')
 
     def CacheDebug(self, fmt, target, cachefile):
-        # The target cachefile will live in a directory with the special
-        # extension for this cachedir class. Check if this cachefile is
-        # in a directory like that and customize the debug logs.
-        cksum_cachefile = str(pathlib.Path(cachefile).parent)
-        if cksum_cachefile.endswith(self.get_ext()):
-            super().CacheDebug(fmt, target, cksum_cachefile)
-        else:
-            super().CacheDebug(fmt, target, cachefile)
 
-    def _log(self, msg, log_msg, json_info, realnode, cachefile):
-        self.CacheDebug(log_msg + cache_debug_suffix, realnode, cachefile)
+        super().CacheDebug(fmt, target, cachefile)
+        # Capture exception information into the cache debug log
+        if sys.exc_info()[1] and self.debugFP:
+            self.debugFP.write(self._format_exception_msg())
+
+    def _format_exception_msg(self):
+        return ('An exception was detected while using the cache:\n' +
+                '    ' + "\n    ".join("".join(traceback.format_exc()).split("\n"))) + '\n'
 
-        # Write the exception and/or error info to the cache debug log file if in use.
-        if self.debugFP:
-            self.debugFP.write(msg + '\n')
+    def _log(self, log_msg, json_info, realnode, cachefile):
+        self.CacheDebug(log_msg + cache_debug_suffix, realnode, cachefile)
         self.CacheDebugJson(json_info, realnode, cachefile)
 
     def print_cache_issue(self, node, ex):
 
         cksum_dir = pathlib.Path(self.cachepath(node)[1])
-        msg = ('An issue was detected while validating the cache:\n' +
-              '    ' + "\n    ".join("".join(traceback.format_exc()).split("\n")))
-
-        self._log(msg, str(ex), {'type': 'error', 'error': msg}, node, cksum_dir)
+        self._log(str(ex), {'type': 'error'}, node, cksum_dir)
 
     def clean_bad_cachefile(self, node, cache_csig, computed_csig):
 
@@ -238,11 +237,11 @@ class CacheDirValidate(SCons.CacheDir.CacheDir):
             cksum_dir.replace(rm_path)
         except OSError as ex:
             msg = f"Failed to rename {cksum_dir} to {rm_path}: {ex}"
-            self._log(msg, msg, {'type': 'error', 'error': msg}, node, cksum_dir)
+            self._log(msg, {'type': 'error'}, node, cksum_dir)
             return
 
         msg = f"Removed bad cachefile {cksum_dir} found in cache."
-        self._log(msg, msg, {
+        self._log(msg, {
             'type': 'invalid_checksum',
             'cache_csig': cache_csig,
             'computed_csig': computed_csig
diff --git a/src/third_party/scons-3.1.2/scons-local-3.1.2/SCons/CacheDir.py b/src/third_party/scons-3.1.2/scons-local-3.1.2/SCons/CacheDir.py
index 804a315c717..26f04a1bc9d 100644
--- a/src/third_party/scons-3.1.2/scons-local-3.1.2/SCons/CacheDir.py
+++ b/src/third_party/scons-3.1.2/scons-local-3.1.2/SCons/CacheDir.py
@@ -131,7 +131,8 @@ def CachePushFunc(target, source, env):
         # the CacheDir being on a separate file system that's full.
         # In any case, inability to push a file to cache doesn't affect
         # the correctness of the build, so just print a warning.
-        msg = errfmt % (str(target), cachefile)
+        msg = errfmt % (str(t), cachefile)
+        cd.CacheDebug(errfmt + '\n', str(t), cachefile)
         SCons.Warnings.warn(SCons.Warnings.CacheWriteErrorWarning, msg)
 
 CachePush = SCons.Action.Action(CachePushFunc, None)