diff options
| -rw-r--r-- | buildstream/element.py | 89 | ||||
| -rw-r--r-- | buildstream/plugins/sources/patch.py | 2 | ||||
| -rw-r--r-- | buildstream/source.py | 14 | ||||
| -rw-r--r-- | tests/sources/previous_source_access/plugins/sources/foo_transform.py | 1 |
4 files changed, 82 insertions, 24 deletions
diff --git a/buildstream/element.py b/buildstream/element.py index 05884c008..e89799202 100644 --- a/buildstream/element.py +++ b/buildstream/element.py @@ -107,6 +107,7 @@ from ._artifact import Artifact from .storage.directory import Directory from .storage._filebaseddirectory import FileBasedDirectory +from .storage._casbaseddirectory import CasBasedDirectory from .storage.directory import VirtualDirectoryError @@ -1537,9 +1538,17 @@ class Element(Plugin): if self.__sources: - sourcecache = self._get_context().sourcecache + sourcecache = context.sourcecache + # find last required source + last_required_previous_ix = 0 + for ix, source in enumerate(self.__sources): + if source.BST_REQUIRES_PREVIOUS_SOURCES_CACHE: + last_required_previous_ix = ix + try: - import_dir = sourcecache.export(self.__sources[-1]) + import_dir = CasBasedDirectory(context.get_cascache()) + for source in self.__sources[last_required_previous_ix:]: + import_dir.import_files(sourcecache.export(source)) except SourceCacheError as e: raise ElementError("Error trying to export source for {}: {}" .format(self.name, e)) @@ -2192,24 +2201,32 @@ class Element(Plugin): def _fetch(self, fetch_original=False): previous_sources = [] sources = self.__sources + fetch_needed = False if sources and not fetch_original: - source = sources[-1] - if self.__sourcecache.contains(source): - return + last_requires_previous = 0 + for ix, source in enumerate(self.__sources): + if source.BST_REQUIRES_PREVIOUS_SOURCES_CACHE: + last_requires_previous = ix - # try and fetch from source cache - if source._get_consistency() < Consistency.CACHED and \ - self.__sourcecache.has_fetch_remotes() and \ - not self.__sourcecache.contains(source): - if self.__sourcecache.pull(source): - return + for source in self.__sources[last_requires_previous:]: + if self.__sourcecache.contains(source): + continue + + # try and fetch from source cache + if source._get_consistency() < Consistency.CACHED and \ + self.__sourcecache.has_fetch_remotes(): + if self.__sourcecache.pull(source): + continue + + fetch_needed = True # We need to fetch original sources - for source in self.sources(): - source_consistency = source._get_consistency() - if source_consistency != Consistency.CACHED: - source._fetch(previous_sources) - previous_sources.append(source) + if fetch_needed or fetch_original: + for source in self.sources(): + source_consistency = source._get_consistency() + if source_consistency != Consistency.CACHED: + source._fetch(previous_sources) + previous_sources.append(source) self.__cache_sources() @@ -2264,12 +2281,27 @@ class Element(Plugin): # Check if sources are cached, generating the source key if it hasn't been def _source_cached(self): if self.__sources: - last_source = self.__sources[-1] - if not last_source._key: - last_source._generate_key(self.__sources[:-1]) - return self._get_context().sourcecache.contains(last_source) - else: - return True + last_requires_previous = 0 + for ix, source in enumerate(self.__sources): + if source.BST_REQUIRES_PREVIOUS_SOURCES_CACHE: + last_requires_previous = ix + + sourcecache = self._get_context().sourcecache + if not self.__sources[last_requires_previous]._key: + self.__sources[last_requires_previous]._generate_key( + self.__sources[:last_requires_previous]) + + # Go through individual sources + for source in self.__sources[last_requires_previous+1:]: + if not source._key: + source._generate_key([]) + if not sourcecache.contains(source): + return False + + if not sourcecache.contains(self.__sources[last_requires_previous]): + return False + + return True def _should_fetch(self, fetch_original=False): """ return bool of if we need to run the fetch stage for this element @@ -2918,7 +2950,18 @@ class Element(Plugin): def __cache_sources(self): sources = self.__sources if sources and not self._source_cached(): - sources[-1]._cache(sources[:-1]) + last_requires_previous = 0 + for ix, source in enumerate(sources): + if source.BST_REQUIRES_PREVIOUS_SOURCES_CACHE: + last_requires_previous = ix + + # cache last source that requires previous sources + self.__sourcecache.commit(sources[last_requires_previous], + sources[:last_requires_previous]) + + # commit all other sources by themselves + for source in sources[last_requires_previous+1:]: + self.__sourcecache.commit(source, []) # __update_state_recursively() # diff --git a/buildstream/plugins/sources/patch.py b/buildstream/plugins/sources/patch.py index 8e833b411..6da1ea034 100644 --- a/buildstream/plugins/sources/patch.py +++ b/buildstream/plugins/sources/patch.py @@ -52,6 +52,8 @@ from buildstream import utils class PatchSource(Source): # pylint: disable=attribute-defined-outside-init + BST_REQUIRES_PREVIOUS_SOURCES_CACHE = True + def configure(self, node): self.path = self.node_get_project_path(node, 'path', check_is_file=True) diff --git a/buildstream/source.py b/buildstream/source.py index 6f4ff575b..94cb82e2f 100644 --- a/buildstream/source.py +++ b/buildstream/source.py @@ -286,6 +286,15 @@ class Source(Plugin): *Since: 1.4* """ + BST_REQUIRES_PREVIOUS_SOURCES_CACHE = False + """Whether access to previous sources is required during cache + + When set to True: + * all sources listed before current source in the given element will be + passed to the source when it's cached. + * This source can not be the first source for an element. + """ + def __init__(self, context, project, meta, *, alias_override=None): provenance = _yaml.node_get_provenance(meta.config) super().__init__("{}-{}".format(meta.element_name, meta.element_index), @@ -709,7 +718,10 @@ class Source(Plugin): def _cache(self, previous_sources): # stage the source into the source cache - self.__source_cache.commit(self, previous_sources) + if self.BST_REQUIRES_PREVIOUS_SOURCES_CACHE: + self.__source_cache.commit(self, previous_sources) + else: + self.__source_cache.commit(self, []) # Wrapper for stage() api which gives the source # plugin a fully constructed path considering the diff --git a/tests/sources/previous_source_access/plugins/sources/foo_transform.py b/tests/sources/previous_source_access/plugins/sources/foo_transform.py index 820946454..bec4f9913 100644 --- a/tests/sources/previous_source_access/plugins/sources/foo_transform.py +++ b/tests/sources/previous_source_access/plugins/sources/foo_transform.py @@ -18,6 +18,7 @@ class FooTransformSource(Source): # We need access to previous both at track time and fetch time BST_REQUIRES_PREVIOUS_SOURCES_TRACK = True BST_REQUIRES_PREVIOUS_SOURCES_FETCH = True + BST_REQUIRES_PREVIOUS_SOURCES_CACHE = True @property def mirror(self): |