diff options
author | Benjamin Schubert <ben.c.schubert@gmail.com> | 2019-07-15 18:27:22 +0100 |
---|---|---|
committer | bst-marge-bot <marge-bot@buildstream.build> | 2019-07-17 09:04:08 +0000 |
commit | bbf65daff319f707c6010bfc8aeddd19dd1c6e16 (patch) | |
tree | 40aa391c47f9c8e2c979adc26200a0cfb5430015 /src | |
parent | f44f97cbc012f8b5e219a11b06701c6544c308e6 (diff) | |
download | buildstream-bbf65daff319f707c6010bfc8aeddd19dd1c6e16.tar.gz |
downloadablefilesource: Cache 'mirror_dir' and 'default_mirror_file'
This allows faster access to those two files and reduces the time spent
in their getters by more than 50%, which can be 2-3s on medium projects
Diffstat (limited to 'src')
-rw-r--r-- | src/buildstream/_utils.pyx | 3 | ||||
-rw-r--r-- | src/buildstream/plugins/sources/_downloadablefilesource.py | 23 |
2 files changed, 17 insertions, 9 deletions
diff --git a/src/buildstream/_utils.pyx b/src/buildstream/_utils.pyx index 3a3a97017..6605cc82a 100644 --- a/src/buildstream/_utils.pyx +++ b/src/buildstream/_utils.pyx @@ -50,3 +50,6 @@ cdef Py_UNICODE _transl(Py_UNICODE x): if ("a" <= x <= "z") or ("A" <= x <= "Z") or ("0" <= x <= "9") or x == "%": return x return "_" + + +# get_mirror_directory in configure of _downlaodablefilesource diff --git a/src/buildstream/plugins/sources/_downloadablefilesource.py b/src/buildstream/plugins/sources/_downloadablefilesource.py index 10418691e..2d53f8a56 100644 --- a/src/buildstream/plugins/sources/_downloadablefilesource.py +++ b/src/buildstream/plugins/sources/_downloadablefilesource.py @@ -75,11 +75,14 @@ class DownloadableFileSource(Source): COMMON_CONFIG_KEYS = Source.COMMON_CONFIG_KEYS + ['url', 'ref', 'etag'] __urlopener = None + __default_mirror_file = None def configure(self, node): self.original_url = node.get_str('url') self.ref = node.get_str('ref', None) self.url = self.translate_url(self.original_url) + self._mirror_dir = os.path.join(self.get_mirror_directory(), + utils.url_directory_name(self.original_url)) self._warn_deprecated_etag(node) def preflight(self): @@ -149,7 +152,7 @@ class DownloadableFileSource(Source): self.warn('{} "etag" is deprecated and ignored.'.format(provenance)) def _get_etag(self, ref): - etagfilename = os.path.join(self._get_mirror_dir(), '{}.etag'.format(ref)) + etagfilename = os.path.join(self._mirror_dir, '{}.etag'.format(ref)) if os.path.exists(etagfilename): with open(etagfilename, 'r') as etagfile: return etagfile.read() @@ -157,7 +160,7 @@ class DownloadableFileSource(Source): return None def _store_etag(self, ref, etag): - etagfilename = os.path.join(self._get_mirror_dir(), '{}.etag'.format(ref)) + etagfilename = os.path.join(self._mirror_dir, '{}.etag'.format(ref)) with utils.save_file_atomic(etagfilename) as etagfile: etagfile.write(etag) @@ -192,8 +195,8 @@ class DownloadableFileSource(Source): shutil.copyfileobj(response, dest) # Make sure url-specific mirror dir exists. - if not os.path.isdir(self._get_mirror_dir()): - os.makedirs(self._get_mirror_dir()) + if not os.path.isdir(self._mirror_dir): + os.makedirs(self._mirror_dir) # Store by sha256sum sha256 = utils.sha256sum(local_file) @@ -220,12 +223,14 @@ class DownloadableFileSource(Source): raise SourceError("{}: Error mirroring {}: {}" .format(self, self.url, e), temporary=True) from e - def _get_mirror_dir(self): - return os.path.join(self.get_mirror_directory(), - utils.url_directory_name(self.original_url)) - def _get_mirror_file(self, sha=None): - return os.path.join(self._get_mirror_dir(), sha or self.ref) + if sha is not None: + return os.path.join(self._mirror_dir, sha) + + if self.__default_mirror_file is None: + self.__default_mirror_file = os.path.join(self._mirror_dir, self.ref) + + return self.__default_mirror_file def __get_urlopener(self): if not DownloadableFileSource.__urlopener: |