Update mtimes of objects for requested references.

This also remove references when some objects are missing. This is in
preparation for the move from reference to object garbage collection.

3 files changed, 37 insertions, 11 deletions

diff --git a/buildstream/_artifactcache/cascache.py b/buildstream/_artifactcache/cascache.py
index 0f7a55dc1..6b3bdb471 100644
--- a/buildstream/_artifactcache/cascache.py
+++ b/buildstream/_artifactcache/cascache.py
@@ -682,6 +682,10 @@ class CASCache():
 
         return pruned
 
+    def update_tree_mtime(self, tree):
+        reachable = set()
+        self._reachable_refs_dir(reachable, tree, update_mtime=True)
+
     ################################################
     #             Local Private Methods            #
     ################################################
@@ -828,10 +832,13 @@ class CASCache():
                 a += 1
                 b += 1
 
-    def _reachable_refs_dir(self, reachable, tree):
+    def _reachable_refs_dir(self, reachable, tree, update_mtime=False):
         if tree.hash in reachable:
             return
 
+        if update_mtime:
+            os.utime(self.objpath(tree))
+
         reachable.add(tree.hash)
 
         directory = remote_execution_pb2.Directory()
@@ -840,10 +847,12 @@ class CASCache():
             directory.ParseFromString(f.read())
 
         for filenode in directory.files:
+            if update_mtime:
+                os.utime(self.objpath(filenode.digest))
             reachable.add(filenode.digest.hash)
 
         for dirnode in directory.directories:
-            self._reachable_refs_dir(reachable, dirnode.digest)
+            self._reachable_refs_dir(reachable, dirnode.digest, update_mtime=update_mtime)
 
     def _required_blobs(self, directory_digest):
         # parse directory, and recursively add blobs
diff --git a/buildstream/_artifactcache/casserver.py b/buildstream/_artifactcache/casserver.py
index 15301cf5e..e2344f0f7 100644
--- a/buildstream/_artifactcache/casserver.py
+++ b/buildstream/_artifactcache/casserver.py
@@ -247,10 +247,17 @@ class _ContentAddressableStorageServicer(remote_execution_pb2_grpc.ContentAddres
     def FindMissingBlobs(self, request, context):
         response = remote_execution_pb2.FindMissingBlobsResponse()
         for digest in request.blob_digests:
-            if not _has_object(self.cas, digest):
-                d = response.missing_blob_digests.add()
-                d.hash = digest.hash
-                d.size_bytes = digest.size_bytes
+            objpath = self.cas.objpath(digest)
+            try:
+                os.utime(objpath)
+            except OSError as e:
+                if e.errno != errno.ENOENT:
+                    raise
+                else:
+                    d = response.missing_blob_digests.add()
+                    d.hash = digest.hash
+                    d.size_bytes = digest.size_bytes
+
         return response
 
     def BatchReadBlobs(self, request, context):
@@ -347,6 +354,12 @@ class _ReferenceStorageServicer(buildstream_pb2_grpc.ReferenceStorageServicer):
 
         try:
             tree = self.cas.resolve_ref(request.key, update_mtime=True)
+            try:
+                self.cas.update_tree_mtime(tree)
+            except FileNotFoundError:
+                self.cas.remove(request.key, defer_prune=True)
+                context.set_code(grpc.StatusCode.NOT_FOUND)
+                return response
 
             response.digest.hash = tree.hash
             response.digest.size_bytes = tree.size_bytes
@@ -419,11 +432,6 @@ def _digest_from_upload_resource_name(resource_name):
         return None
 
 
-def _has_object(cas, digest):
-    objpath = cas.objpath(digest)
-    return os.path.exists(objpath)
-
-
 # _clean_up_cache()
 #
 # Keep removing Least Recently Pushed (LRP) artifacts in a cache until there
diff --git a/tests/testutils/artifactshare.py b/tests/testutils/artifactshare.py
index 31b96be73..c1ddc2c46 100644
--- a/tests/testutils/artifactshare.py
+++ b/tests/testutils/artifactshare.py
@@ -136,6 +136,15 @@ class ArtifactShare():
 
         try:
             tree = self.cas.resolve_ref(artifact_key)
+            reachable = set()
+            try:
+                self.cas._reachable_refs_dir(reachable, tree, update_mtime=False)
+            except FileNotFoundError:
+                return None
+            for digest in reachable:
+                object_name = os.path.join(self.cas.casdir, 'objects', digest[:2], digest[2:])
+                if not os.path.exists(object_name):
+                    return None
             return tree
         except CASError:
             return None