Merge branch 'bschubert/optimize-downloadable-sources' into 'master'

Optimize downloadable sources

See merge request BuildStream/buildstream!1483

6 files changed, 85 insertions, 32 deletions

diff --git a/.pylintrc b/.pylintrc
index 63ff1b756..e07219a41 100644
--- a/.pylintrc
+++ b/.pylintrc
@@ -7,6 +7,7 @@ extension-pkg-whitelist=
     buildstream.node,
     buildstream._loader._loader,
     buildstream._loader.types,
+    buildstream._utils,
     buildstream._variables,
     buildstream._yaml
 
diff --git a/setup.py b/setup.py
index 1ea423e1a..284e74d7f 100755
--- a/setup.py
+++ b/setup.py
@@ -406,6 +406,7 @@ register_cython_module("buildstream.node")
 register_cython_module("buildstream._loader._loader")
 register_cython_module("buildstream._loader.types", dependencies=["buildstream.node"])
 register_cython_module("buildstream._yaml", dependencies=["buildstream.node"])
+register_cython_module("buildstream._utils")
 register_cython_module("buildstream._variables", dependencies=["buildstream.node"])
 
 #####################################################
diff --git a/src/buildstream/_utils.pyx b/src/buildstream/_utils.pyx
new file mode 100644
index 000000000..6605cc82a
--- /dev/null
+++ b/src/buildstream/_utils.pyx
@@ -0,0 +1,55 @@
+#
+#  Copyright (C) 2019 Bloomberg L.P.
+#
+#  This program is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU Lesser General Public
+#  License as published by the Free Software Foundation; either
+#  version 2 of the License, or (at your option) any later version.
+#
+#  This library is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	 See the GNU
+#  Lesser General Public License for more details.
+#
+#  You should have received a copy of the GNU Lesser General Public
+#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+#  Authors:
+#        Benjamin Schubert <bschubert15@bloomberg.net>
+#
+
+"""
+This module contains utilities that have been optimized in Cython
+"""
+
+
+def url_directory_name(str url):
+    """Normalizes a url into a directory name
+
+    Args:
+       url (str): A url string
+
+    Returns:
+       A string which can be used as a directory name
+    """
+    return ''.join([_transl(x) for x in url])
+
+
+#############################################################
+#                 Module local helper Methods               #
+#############################################################
+
+
+# _transl(x)
+#
+# Helper for `url_directory_name`
+#
+# This transforms the value to "_" if is it not a ascii letter, a digit or "%" or "_"
+#
+cdef Py_UNICODE _transl(Py_UNICODE x):
+    if ("a" <= x <= "z") or ("A" <= x <= "Z") or ("0" <= x <= "9") or x == "%":
+        return x
+    return "_"
+
+
+# get_mirror_directory in configure of _downlaodablefilesource
diff --git a/src/buildstream/plugins/sources/_downloadablefilesource.py b/src/buildstream/plugins/sources/_downloadablefilesource.py
index 10418691e..2d53f8a56 100644
--- a/src/buildstream/plugins/sources/_downloadablefilesource.py
+++ b/src/buildstream/plugins/sources/_downloadablefilesource.py
@@ -75,11 +75,14 @@ class DownloadableFileSource(Source):
     COMMON_CONFIG_KEYS = Source.COMMON_CONFIG_KEYS + ['url', 'ref', 'etag']
 
     __urlopener = None
+    __default_mirror_file = None
 
     def configure(self, node):
         self.original_url = node.get_str('url')
         self.ref = node.get_str('ref', None)
         self.url = self.translate_url(self.original_url)
+        self._mirror_dir = os.path.join(self.get_mirror_directory(),
+                                        utils.url_directory_name(self.original_url))
         self._warn_deprecated_etag(node)
 
     def preflight(self):
@@ -149,7 +152,7 @@ class DownloadableFileSource(Source):
             self.warn('{} "etag" is deprecated and ignored.'.format(provenance))
 
     def _get_etag(self, ref):
-        etagfilename = os.path.join(self._get_mirror_dir(), '{}.etag'.format(ref))
+        etagfilename = os.path.join(self._mirror_dir, '{}.etag'.format(ref))
         if os.path.exists(etagfilename):
             with open(etagfilename, 'r') as etagfile:
                 return etagfile.read()
@@ -157,7 +160,7 @@ class DownloadableFileSource(Source):
         return None
 
     def _store_etag(self, ref, etag):
-        etagfilename = os.path.join(self._get_mirror_dir(), '{}.etag'.format(ref))
+        etagfilename = os.path.join(self._mirror_dir, '{}.etag'.format(ref))
         with utils.save_file_atomic(etagfilename) as etagfile:
             etagfile.write(etag)
 
@@ -192,8 +195,8 @@ class DownloadableFileSource(Source):
                         shutil.copyfileobj(response, dest)
 
                 # Make sure url-specific mirror dir exists.
-                if not os.path.isdir(self._get_mirror_dir()):
-                    os.makedirs(self._get_mirror_dir())
+                if not os.path.isdir(self._mirror_dir):
+                    os.makedirs(self._mirror_dir)
 
                 # Store by sha256sum
                 sha256 = utils.sha256sum(local_file)
@@ -220,12 +223,14 @@ class DownloadableFileSource(Source):
             raise SourceError("{}: Error mirroring {}: {}"
                               .format(self, self.url, e), temporary=True) from e
 
-    def _get_mirror_dir(self):
-        return os.path.join(self.get_mirror_directory(),
-                            utils.url_directory_name(self.original_url))
-
     def _get_mirror_file(self, sha=None):
-        return os.path.join(self._get_mirror_dir(), sha or self.ref)
+        if sha is not None:
+            return os.path.join(self._mirror_dir, sha)
+
+        if self.__default_mirror_file is None:
+            self.__default_mirror_file = os.path.join(self._mirror_dir, self.ref)
+
+        return self.__default_mirror_file
 
     def __get_urlopener(self):
         if not DownloadableFileSource.__urlopener:
diff --git a/src/buildstream/source.py b/src/buildstream/source.py
index 59b6d3644..03c1301c5 100644
--- a/src/buildstream/source.py
+++ b/src/buildstream/source.py
@@ -330,6 +330,9 @@ class Source(Plugin):
         self.__config = self.__extract_config(meta)
         self.__first_pass = meta.first_pass
 
+        # cached values for commonly access values on the source
+        self.__mirror_directory = None
+
         self._configure(self.__config)
 
     COMMON_CONFIG_KEYS = ['kind', 'directory']
@@ -531,12 +534,14 @@ class Source(Plugin):
         Returns:
            (str): The directory belonging to this source
         """
+        if self.__mirror_directory is None:
+            # Create the directory if it doesnt exist
+            context = self._get_context()
+            directory = os.path.join(context.sourcedir, self.get_kind())
+            os.makedirs(directory, exist_ok=True)
+            self.__mirror_directory = directory
 
-        # Create the directory if it doesnt exist
-        context = self._get_context()
-        directory = os.path.join(context.sourcedir, self.get_kind())
-        os.makedirs(directory, exist_ok=True)
-        return directory
+        return self.__mirror_directory
 
     def translate_url(self, url, *, alias_override=None, primary=True):
         """Translates the given url which may be specified with an alias
diff --git a/src/buildstream/utils.py b/src/buildstream/utils.py
index 82cd4134b..2c57925d4 100644
--- a/src/buildstream/utils.py
+++ b/src/buildstream/utils.py
@@ -30,7 +30,6 @@ import shutil
 import signal
 import stat
 from stat import S_ISDIR
-import string
 import subprocess
 import tempfile
 import itertools
@@ -42,6 +41,10 @@ from . import _signals
 from ._exceptions import BstError, ErrorDomain
 from ._protos.build.bazel.remote.execution.v2 import remote_execution_pb2
 
+# Contains utils that have been rewritten in Cython for speed benefits
+# This makes them available when importing from utils
+from ._utils import url_directory_name  # pylint: disable=unused-import
+
 # The magic number for timestamps: 2011-11-11 11:11:11
 BST_ARBITRARY_TIMESTAMP = calendar.timegm([2011, 11, 11, 11, 11, 11])
 
@@ -456,23 +459,6 @@ def get_host_tool(name):
     return program_path
 
 
-def url_directory_name(url):
-    """Normalizes a url into a directory name
-
-    Args:
-       url (str): A url string
-
-    Returns:
-       A string which can be used as a directory name
-    """
-    valid_chars = string.digits + string.ascii_letters + '%_'
-
-    def transl(x):
-        return x if x in valid_chars else '_'
-
-    return ''.join([transl(x) for x in url])
-
-
 def get_bst_version():
     """Gets the major, minor release portion of the
     BuildStream version.