Merge branch 'juerg/fetch' into 'master'

Do not call fetch() for cached sources

See merge request BuildStream/buildstream!992

6 files changed, 113 insertions, 4 deletions

diff --git a/buildstream/_scheduler/queues/fetchqueue.py b/buildstream/_scheduler/queues/fetchqueue.py
index 446dbbd3b..c58bfdb57 100644
--- a/buildstream/_scheduler/queues/fetchqueue.py
+++ b/buildstream/_scheduler/queues/fetchqueue.py
@@ -40,10 +40,7 @@ class FetchQueue(Queue):
         self._skip_cached = skip_cached
 
     def process(self, element):
-        previous_sources = []
-        for source in element.sources():
-            source._fetch(previous_sources)
-            previous_sources.append(source)
+        element._fetch()
 
     def status(self, element):
         # state of dependencies may have changed, recalculate element state
diff --git a/buildstream/element.py b/buildstream/element.py
index e44a7cc32..2b6c68c5d 100644
--- a/buildstream/element.py
+++ b/buildstream/element.py
@@ -2022,6 +2022,20 @@ class Element(Plugin):
 
         return True
 
+    # _fetch()
+    #
+    # Fetch the element's sources.
+    #
+    # Raises:
+    #    SourceError: If one of the element sources has an error
+    #
+    def _fetch(self):
+        previous_sources = []
+        for source in self.sources():
+            if source._get_consistency() < Consistency.CACHED:
+                source._fetch(previous_sources)
+            previous_sources.append(source)
+
     #############################################################
     #                   Private Local Methods                   #
     #############################################################
diff --git a/tests/sources/no-fetch-cached/files/file b/tests/sources/no-fetch-cached/files/file
new file mode 100644
index 000000000..980a0d5f1
--- /dev/null
+++ b/tests/sources/no-fetch-cached/files/file
@@ -0,0 +1 @@
+Hello World!
diff --git a/tests/sources/no-fetch-cached/plugins/sources/always_cached.py b/tests/sources/no-fetch-cached/plugins/sources/always_cached.py
new file mode 100644
index 000000000..fa143a020
--- /dev/null
+++ b/tests/sources/no-fetch-cached/plugins/sources/always_cached.py
@@ -0,0 +1,45 @@
+"""
+always_cached
+=============
+
+This is a test source plugin that is always cached.
+Used to test that BuildStream core does not call fetch() for cached sources.
+
+"""
+
+from buildstream import Consistency, Source
+
+
+class AlwaysCachedSource(Source):
+
+    def configure(self, node):
+        pass
+
+    def preflight(self):
+        pass
+
+    def get_unique_key(self):
+        return None
+
+    def get_consistency(self):
+        return Consistency.CACHED
+
+    def load_ref(self, node):
+        pass
+
+    def get_ref(self):
+        return None
+
+    def set_ref(self, ref, node):
+        pass
+
+    def fetch(self):
+        # Source is always cached, so fetch() should never be called
+        assert False
+
+    def stage(self, directory):
+        pass
+
+
+def setup():
+    return AlwaysCachedSource
diff --git a/tests/sources/no-fetch-cached/project.conf b/tests/sources/no-fetch-cached/project.conf
new file mode 100644
index 000000000..af0cec15c
--- /dev/null
+++ b/tests/sources/no-fetch-cached/project.conf
@@ -0,0 +1,8 @@
+# Project with local source plugins
+name: no-fetch-cached
+
+plugins:
+- origin: local
+  path: plugins/sources
+  sources:
+    always_cached: 0
diff --git a/tests/sources/no_fetch_cached.py b/tests/sources/no_fetch_cached.py
new file mode 100644
index 000000000..d2880f628
--- /dev/null
+++ b/tests/sources/no_fetch_cached.py
@@ -0,0 +1,44 @@
+import os
+import pytest
+
+from buildstream import _yaml
+
+from tests.testutils import cli, create_repo
+from tests.testutils.site import HAVE_GIT
+
+DATA_DIR = os.path.join(
+    os.path.dirname(os.path.realpath(__file__)),
+    'no-fetch-cached'
+)
+
+
+##################################################################
+#                              Tests                             #
+##################################################################
+# Test that fetch() is not called for cached sources
+@pytest.mark.skipif(HAVE_GIT is False, reason="git is not available")
+@pytest.mark.datafiles(DATA_DIR)
+def test_no_fetch_cached(cli, tmpdir, datafiles):
+    project = os.path.join(datafiles.dirname, datafiles.basename)
+
+    # Create the repo from 'files' subdir
+    repo = create_repo('git', str(tmpdir))
+    ref = repo.create(os.path.join(project, 'files'))
+
+    # Write out test target with a cached and a non-cached source
+    element = {
+        'kind': 'import',
+        'sources': [
+            repo.source_config(ref=ref),
+            {
+                'kind': 'always_cached'
+            }
+        ]
+    }
+    _yaml.dump(element, os.path.join(project, 'target.bst'))
+
+    # Test fetch of target with a cached and a non-cached source
+    result = cli.run(project=project, args=[
+        'source', 'fetch', 'target.bst'
+    ])
+    result.assert_success()