diff options
author | Jürg Billeter <j@bitron.ch> | 2020-09-15 11:07:10 +0200 |
---|---|---|
committer | Jürg Billeter <j@bitron.ch> | 2020-12-22 13:02:07 +0100 |
commit | fb36a74815ae858c7e7623bfee45b931aad6b9dc (patch) | |
tree | e890f427ae64b7bb278858fbd318fbf5e3325253 | |
parent | afd578ba876b608cb88b875b688c611bdde5eec4 (diff) | |
download | buildstream-fb36a74815ae858c7e7623bfee45b931aad6b9dc.tar.gz |
_artifact.py: Make cache query explicit
Cache query can be fairly expensive as it checks the presence of all
artifact blobs. Make this more explicit with a `query_cache()` method,
instead of implicitly querying the cache on the first call to
`cached()`.
-rw-r--r-- | src/buildstream/_artifact.py | 29 | ||||
-rw-r--r-- | src/buildstream/_frontend/widget.py | 2 | ||||
-rw-r--r-- | src/buildstream/element.py | 16 |
3 files changed, 25 insertions, 22 deletions
diff --git a/src/buildstream/_artifact.py b/src/buildstream/_artifact.py index d4a716ff1..b63cff6e5 100644 --- a/src/buildstream/_artifact.py +++ b/src/buildstream/_artifact.py @@ -540,7 +540,7 @@ class Artifact: return dependency_refs - # cached(): + # query_cache(): # # Check whether the artifact corresponding to the stored cache key is # available. This also checks whether all required parts of the artifact @@ -550,11 +550,7 @@ class Artifact: # Returns: # (bool): Whether artifact is in local cache # - def cached(self): - - if self._cached is not None: - return self._cached - + def query_cache(self): context = self._context artifact = self._load_proto() @@ -587,6 +583,18 @@ class Artifact: self._cached = True return True + # cached() + # + # Return whether the artifact is available in the local cache. This must + # be called after `query_cache()` or `set_cached()`. + # + # Returns: + # (bool): Whether artifact is in local cache + # + def cached(self): + assert self._cached is not None + return self._cached + # cached_logs() # # Check if the artifact is cached with log files. @@ -600,15 +608,6 @@ class Artifact: # If the artifact is cached, its log files are available as well. return self._element._cached() - # reset_cached() - # - # Allow the Artifact to query the filesystem to determine whether it - # is cached or not. - # - def reset_cached(self): - self._proto = None - self._cached = None - # set_cached() # # Mark the artifact as cached without querying the filesystem. diff --git a/src/buildstream/_frontend/widget.py b/src/buildstream/_frontend/widget.py index 0d5379f43..d289ef2f9 100644 --- a/src/buildstream/_frontend/widget.py +++ b/src/buildstream/_frontend/widget.py @@ -357,6 +357,8 @@ class LogLine(Widget): else: if element.get_kind() == "junction": line = p.fmt_subst(line, "state", "junction", fg="magenta") + elif not element._can_query_cache(): + line = p.fmt_subst(line, "state", "unknown", fg="bright_black") elif element._cached_failure(): line = p.fmt_subst(line, "state", "failed", fg="red") elif element._cached_success(): diff --git a/src/buildstream/element.py b/src/buildstream/element.py index 945ca5cf7..0f2a01ced 100644 --- a/src/buildstream/element.py +++ b/src/buildstream/element.py @@ -1179,9 +1179,6 @@ class Element(Plugin): # the artifact cache # def _cached(self): - if not self.__artifact: - return False - return self.__artifact.cached() # _cached_remotely(): @@ -1297,7 +1294,7 @@ class Element(Plugin): # def _can_query_cache(self): # cache cannot be queried until strict cache key is available - return self.__strict_cache_key is not None + return self.__artifact is not None # _initialize_state() # @@ -1585,7 +1582,9 @@ class Element(Plugin): def __should_schedule(self): # We're processing if we're already scheduled, we've # finished assembling or if we're waiting to pull. - processing = self.__assemble_scheduled or self.__assemble_done or self._pull_pending() + processing = ( + self.__assemble_scheduled or self.__assemble_done or (self._can_query_cache() and self._pull_pending()) + ) # We should schedule a build when return ( @@ -1651,7 +1650,7 @@ class Element(Plugin): self.__artifact.set_cached() self.__cached_successfully = True else: - self.__artifact.reset_cached() + self.__artifact.query_cache() # When we're building in non-strict mode, we may have # assembled everything to this point without a strong cache @@ -1898,7 +1897,7 @@ class Element(Plugin): # Artifact may become cached after pulling, so let it query the # filesystem again to check - self.__artifact.reset_cached() + self.__artifact.query_cache() # We may not have actually pulled an artifact - the pull may # have failed. We might therefore need to schedule assembly. @@ -2570,6 +2569,7 @@ class Element(Plugin): return None artifact = Artifact(self, self._get_context(), strong_key=workspace.last_build) + artifact.query_cache() if not artifact.cached(): return None @@ -3279,12 +3279,14 @@ class Element(Plugin): strict_key=self.__strict_cache_key, weak_key=self.__weak_cache_key, ) + strict_artifact.query_cache() if context.get_strict() or strict_artifact.cached(): self.__artifact = strict_artifact else: self.__artifact = Artifact( self, context, strict_key=self.__strict_cache_key, weak_key=self.__weak_cache_key ) + self.__artifact.query_cache() if not context.get_strict() and self.__artifact.cached(): # In non-strict mode, strong cache key becomes available when |