diff options
author | Abderrahim Kitouni <akitouni@gnome.org> | 2020-06-28 09:20:39 +0100 |
---|---|---|
committer | Abderrahim Kitouni <akitouni@gnome.org> | 2020-06-30 15:58:28 +0100 |
commit | 829e30b1f50ff70bdd7ffa24fb73549e31c6a460 (patch) | |
tree | efcaa1f7e5f0ef5e455ae176f24a1ab5064ba5f5 | |
parent | 17825d790cffdbf236eae0801ffcc20fc04385e9 (diff) | |
download | buildstream-829e30b1f50ff70bdd7ffa24fb73549e31c6a460.tar.gz |
move caching sources from Element to SourceCache
-rw-r--r-- | src/buildstream/_sourcecache.py | 34 | ||||
-rw-r--r-- | src/buildstream/element.py | 45 |
2 files changed, 31 insertions, 48 deletions
diff --git a/src/buildstream/_sourcecache.py b/src/buildstream/_sourcecache.py index dcde0b426..41efa5d89 100644 --- a/src/buildstream/_sourcecache.py +++ b/src/buildstream/_sourcecache.py @@ -161,15 +161,24 @@ class SourceCache(BaseCache): # dependent on previous sources, such as the patch source. # # Args: - # source: last source - # previous_sources: rest of the sources. - def commit(self, source, previous_sources): + # sources: list of the sources of an element + def commit(self, sources): + last_requires_previous = 0 + # commit all other sources by themselves + for ix, source in enumerate(sources): + if source.BST_REQUIRES_PREVIOUS_SOURCES_STAGE: + self._commit_one(source, sources[last_requires_previous:ix]) + last_requires_previous = ix + else: + self._commit_one(source, []) + + def _commit_one(self, source, previous_sources): ref = source._get_source_name() # Use tmpdir for now vdir = CasBasedDirectory(self.cas) for previous_source in previous_sources: - vdir.import_files(self.export(previous_source)) + vdir.import_files(self._export_one(previous_source)) if not source.BST_STAGE_VIRTUAL_DIRECTORY: with utils._tempdir(dir=self.context.tmpdir, prefix="staging-temp") as tmpdir: @@ -191,7 +200,22 @@ class SourceCache(BaseCache): # # Returns: # CASBasedDirectory - def export(self, source): + def export(self, sources): + # find last required source + last_requires_previous_ix = 0 + for ix, source in enumerate(sources): + if source.BST_REQUIRES_PREVIOUS_SOURCES_STAGE: + last_requires_previous_ix = ix + + import_dir = CasBasedDirectory(self.cas) + + for source in sources[last_requires_previous_ix:]: + source_dir = self._export_one(source) + import_dir.import_files(source_dir) + + return import_dir + + def _export_one(self, source): ref = source._get_source_name() source = self._get_source(ref) return CasBasedDirectory(self.cas, digest=source.files) diff --git a/src/buildstream/element.py b/src/buildstream/element.py index 6a0fa5fab..0a3f41c24 100644 --- a/src/buildstream/element.py +++ b/src/buildstream/element.py @@ -106,7 +106,6 @@ from ._artifact import Artifact from .storage.directory import Directory from .storage._filebaseddirectory import FileBasedDirectory -from .storage._casbaseddirectory import CasBasedDirectory from .storage.directory import VirtualDirectoryError if TYPE_CHECKING: @@ -1352,16 +1351,8 @@ class Element(Plugin): if self.__sources: - sourcecache = context.sourcecache - # find last required source - last_required_previous_ix = self.__last_source_requires_previous() - import_dir = CasBasedDirectory(context.get_cascache()) - try: - for source in self.__sources[last_required_previous_ix:]: - source_dir = sourcecache.export(source) - import_dir.import_files(source_dir) - + import_dir = context.sourcecache.export(self.__sources) except SourceCacheError as e: raise ElementError("Error trying to export source for {}: {}".format(self.name, e)) except VirtualDirectoryError as e: @@ -2056,7 +2047,7 @@ class Element(Plugin): source._fetch(previous_sources) previous_sources.append(source) - self.__cache_sources() + self.__sourcecache.commit(self.__sources) # _calculate_cache_key(): # @@ -2889,38 +2880,6 @@ class Element(Plugin): return True - # __cache_sources(): - # - # Caches the sources into the local CAS - # - def __cache_sources(self): - if self.__sources and not self._has_all_sources_in_source_cache(): - last_requires_previous = 0 - # commit all other sources by themselves - for ix, source in enumerate(self.__sources): - if source.BST_REQUIRES_PREVIOUS_SOURCES_STAGE: - self.__sourcecache.commit(source, self.__sources[last_requires_previous:ix]) - last_requires_previous = ix - else: - self.__sourcecache.commit(source, []) - - # __last_source_requires_previous - # - # This is the last source that requires previous sources to be cached. - # Sources listed after this will be cached separately. - # - # Returns: - # (int): index of last source that requires previous sources - # - def __last_source_requires_previous(self): - if self.__last_source_requires_previous_ix is None: - last_requires_previous = 0 - for ix, source in enumerate(self.__sources): - if source.BST_REQUIRES_PREVIOUS_SOURCES_STAGE: - last_requires_previous = ix - self.__last_source_requires_previous_ix = last_requires_previous - return self.__last_source_requires_previous_ix - # __update_cache_keys() # # Updates weak and strict cache keys |