diff options
author | Jürg Billeter <j@bitron.ch> | 2020-03-30 12:14:01 +0200 |
---|---|---|
committer | Jürg Billeter <j@bitron.ch> | 2020-04-20 14:27:16 +0200 |
commit | 50dc14900bcad11878175068dd7361ee1b424f59 (patch) | |
tree | b310fefe3a525fc5adf70378d463c52d49a0e86e | |
parent | d946255d0b94d8e52c56ffbd19a74737946107a9 (diff) | |
download | buildstream-50dc14900bcad11878175068dd7361ee1b424f59.tar.gz |
cascache.py: Add allow_partial parameter to fetch_blobs()
This fixes handling of missing blobs in `ArtifactCache.pull()`.
-rw-r--r-- | src/buildstream/_artifactcache.py | 2 | ||||
-rw-r--r-- | src/buildstream/_cas/cascache.py | 9 | ||||
-rw-r--r-- | src/buildstream/_sourcecache.py | 6 | ||||
-rw-r--r-- | src/buildstream/sandbox/_sandboxremote.py | 7 |
4 files changed, 9 insertions, 15 deletions
diff --git a/src/buildstream/_artifactcache.py b/src/buildstream/_artifactcache.py index 9b800acaa..9cebeb1a3 100644 --- a/src/buildstream/_artifactcache.py +++ b/src/buildstream/_artifactcache.py @@ -405,7 +405,7 @@ class ArtifactCache(BaseCache): remote.init() # fetch_blobs() will return the blobs that are still missing - missing_blobs = self.cas.fetch_blobs(remote, missing_blobs) + missing_blobs = self.cas.fetch_blobs(remote, missing_blobs, allow_partial=True) if missing_blobs: raise ArtifactError("Blobs not found on configured artifact servers") diff --git a/src/buildstream/_cas/cascache.py b/src/buildstream/_cas/cascache.py index 03be75c72..61a1a8f50 100644 --- a/src/buildstream/_cas/cascache.py +++ b/src/buildstream/_cas/cascache.py @@ -647,16 +647,19 @@ class CASCache: # fetch_blobs(): # - # Fetch blobs from remote CAS. Returns missing blobs that could not be fetched. + # Fetch blobs from remote CAS. Optionally returns missing blobs that could + # not be fetched. # # Args: # remote (CASRemote): The remote repository to fetch from # digests (list): The Digests of blobs to fetch + # allow_partial (bool): True to return missing blobs, False to raise a + # BlobNotFound error if a blob is missing # # Returns: The Digests of the blobs that were not available on the remote CAS # - def fetch_blobs(self, remote, digests): - missing_blobs = [] + def fetch_blobs(self, remote, digests, *, allow_partial=False): + missing_blobs = [] if allow_partial else None remote.init() diff --git a/src/buildstream/_sourcecache.py b/src/buildstream/_sourcecache.py index e485fbd47..4533a2580 100644 --- a/src/buildstream/_sourcecache.py +++ b/src/buildstream/_sourcecache.py @@ -242,11 +242,7 @@ class SourceCache(BaseCache): self.cas._fetch_directory(remote, source_proto.files) required_blobs = self.cas.required_blobs_for_directory(source_proto.files) missing_blobs = self.cas.local_missing_blobs(required_blobs) - missing_blobs = self.cas.fetch_blobs(remote, missing_blobs) - - if missing_blobs: - source.info("Remote cas ({}) does not have source {} cached".format(remote, display_key)) - continue + self.cas.fetch_blobs(remote, missing_blobs) source.info("Pulled source {} <- {}".format(display_key, remote)) return True diff --git a/src/buildstream/sandbox/_sandboxremote.py b/src/buildstream/sandbox/_sandboxremote.py index 3dcbb2ccc..5b03852f6 100644 --- a/src/buildstream/sandbox/_sandboxremote.py +++ b/src/buildstream/sandbox/_sandboxremote.py @@ -291,12 +291,7 @@ class SandboxRemote(SandboxREAPI): blobs_to_fetch = artifactcache.find_missing_blobs(project, local_missing_blobs) with CASRemote(self.storage_remote_spec, cascache) as casremote: - remote_missing_blobs = cascache.fetch_blobs(casremote, blobs_to_fetch) - - if remote_missing_blobs: - raise SandboxError( - "{} output files are missing on the CAS server".format(len(remote_missing_blobs)) - ) + cascache.fetch_blobs(casremote, blobs_to_fetch) def _execute_action(self, action, flags): stdout, stderr = self._get_output() |