diff options
author | Valentin David <valentin.david@codethink.co.uk> | 2018-11-15 14:39:40 +0100 |
---|---|---|
committer | Valentin David <valentin.david@codethink.co.uk> | 2018-11-29 17:17:27 +0100 |
commit | 2259dff16fdd4257edd3bc8ba3e4b0ca7b054d77 (patch) | |
tree | 79592d0d52a8e2aaa14351b7fa63837d380941aa | |
parent | 1c4b410f7e527cc7fa0efb5af909d9583511573e (diff) | |
download | buildstream-2259dff16fdd4257edd3bc8ba3e4b0ca7b054d77.tar.gz |
Update mtimes of objects for requested references.
This also remove references when some objects are missing. This is in
preparation for the move from reference to object garbage collection.
-rw-r--r-- | buildstream/_artifactcache/cascache.py | 13 | ||||
-rw-r--r-- | buildstream/_artifactcache/casserver.py | 26 | ||||
-rw-r--r-- | tests/testutils/artifactshare.py | 9 |
3 files changed, 37 insertions, 11 deletions
diff --git a/buildstream/_artifactcache/cascache.py b/buildstream/_artifactcache/cascache.py index 1f3f975b7..905a6cc7d 100644 --- a/buildstream/_artifactcache/cascache.py +++ b/buildstream/_artifactcache/cascache.py @@ -577,6 +577,10 @@ class CASCache(ArtifactCache): return pruned + def update_tree_mtime(self, tree): + reachable = set() + self._reachable_refs_dir(reachable, tree, update_mtime=True) + ################################################ # Local Private Methods # ################################################ @@ -718,10 +722,13 @@ class CASCache(ArtifactCache): a += 1 b += 1 - def _reachable_refs_dir(self, reachable, tree): + def _reachable_refs_dir(self, reachable, tree, update_mtime=False): if tree.hash in reachable: return + if update_mtime: + os.utime(self.objpath(tree)) + reachable.add(tree.hash) directory = remote_execution_pb2.Directory() @@ -730,10 +737,12 @@ class CASCache(ArtifactCache): directory.ParseFromString(f.read()) for filenode in directory.files: + if update_mtime: + os.utime(self.objpath(filenode.digest)) reachable.add(filenode.digest.hash) for dirnode in directory.directories: - self._reachable_refs_dir(reachable, dirnode.digest) + self._reachable_refs_dir(reachable, dirnode.digest, update_mtime=update_mtime) def _initialize_remote(self, remote_spec, q): try: diff --git a/buildstream/_artifactcache/casserver.py b/buildstream/_artifactcache/casserver.py index c99ef888e..df9c4740c 100644 --- a/buildstream/_artifactcache/casserver.py +++ b/buildstream/_artifactcache/casserver.py @@ -250,10 +250,17 @@ class _ContentAddressableStorageServicer(remote_execution_pb2_grpc.ContentAddres def FindMissingBlobs(self, request, context): response = remote_execution_pb2.FindMissingBlobsResponse() for digest in request.blob_digests: - if not _has_object(self.cas, digest): - d = response.missing_blob_digests.add() - d.hash = digest.hash - d.size_bytes = digest.size_bytes + objpath = self.cas.objpath(digest) + try: + os.utime(objpath) + except OSError as e: + if e.errno != errno.ENOENT: + raise + else: + d = response.missing_blob_digests.add() + d.hash = digest.hash + d.size_bytes = digest.size_bytes + return response def BatchReadBlobs(self, request, context): @@ -350,6 +357,12 @@ class _ReferenceStorageServicer(buildstream_pb2_grpc.ReferenceStorageServicer): try: tree = self.cas.resolve_ref(request.key, update_mtime=True) + try: + self.cas.update_tree_mtime(tree) + except FileNotFoundError: + self.cas.remove(request.key, defer_prune=True) + context.set_code(grpc.StatusCode.NOT_FOUND) + return response response.digest.hash = tree.hash response.digest.size_bytes = tree.size_bytes @@ -422,11 +435,6 @@ def _digest_from_upload_resource_name(resource_name): return None -def _has_object(cas, digest): - objpath = cas.objpath(digest) - return os.path.exists(objpath) - - # _clean_up_cache() # # Keep removing Least Recently Pushed (LRP) artifacts in a cache until there diff --git a/tests/testutils/artifactshare.py b/tests/testutils/artifactshare.py index 05e87a499..3319cb0a9 100644 --- a/tests/testutils/artifactshare.py +++ b/tests/testutils/artifactshare.py @@ -118,6 +118,15 @@ class ArtifactShare(): try: tree = self.cas.resolve_ref(artifact_key) + reachable = set() + try: + self.cas._reachable_refs_dir(reachable, tree, update_mtime=False) + except FileNotFoundError: + return False + for digest in reachable: + object_name = os.path.join(self.cas.casdir, 'objects', digest[:2], digest[2:]) + if not os.path.exists(object_name): + return False return True except ArtifactError: return False |