Merge branch 'raoul/1043-disabling-blob-fetching' into 'master'

Configuration option for disabling blob fetching with RE

Closes #1043

See merge request BuildStream/buildstream!1402

5 files changed, 43 insertions, 12 deletions

diff --git a/doc/source/using_config.rst b/doc/source/using_config.rst
index 2582e711f..8a6b9ac76 100644
--- a/doc/source/using_config.rst
+++ b/doc/source/using_config.rst
@@ -153,8 +153,11 @@ the user configuration.
 Remote execution
 ~~~~~~~~~~~~~~~~
 
-The same configuration for :ref:`remote execution <project_remote_execution>`
-in ``project.conf`` can be provided in the user configuation.
+The configuration for :ref:`remote execution <project_remote_execution>`
+in ``project.conf`` can be provided in the user configuation. The global
+configuration also has a ``pull-artifact-files`` option, which specifies when
+remote execution is being performed whether to pull file blobs of artifacts, or
+just the directory trees required to perform remote builds.
 
 There is only one remote execution configuration used per project.
 
@@ -178,6 +181,7 @@ configuration will be used as fallback.
     action-cache-service:
       url: http://cache.flalback.example.com:50052
       instance-name: main
+    pull-artifact-files: True
 
 2. Project override:
 
diff --git a/src/buildstream/_context.py b/src/buildstream/_context.py
index 52e4c3db9..2bdf5b0b4 100644
--- a/src/buildstream/_context.py
+++ b/src/buildstream/_context.py
@@ -129,6 +129,9 @@ class Context():
         # Whether or not to attempt to pull build trees globally
         self.pull_buildtrees = None
 
+        # Whether to pull the files of an artifact when doing remote execution
+        self.pull_artifact_files = None
+
         # Whether or not to cache build trees on artifact creation
         self.cache_buildtrees = None
 
@@ -258,6 +261,19 @@ class Context():
         # Load source cache config
         self.source_cache_specs = SourceCache.specs_from_config_node(defaults)
 
+        # Load remote execution config getting pull-artifact-files from it
+        remote_execution = _yaml.node_get(defaults, dict, 'remote-execution', default_value=None)
+        if remote_execution:
+            self.pull_artifact_files = _yaml.node_get(
+                remote_execution, bool, 'pull-artifact-files', default_value=True)
+            # This stops it being used in the remote service set up
+            _yaml.node_del(remote_execution, 'pull-artifact-files', safe=True)
+            # Don't pass the remote execution settings if that was the only option
+            if _yaml.node_keys(remote_execution) == []:
+                _yaml.node_del(defaults, 'remote-execution')
+        else:
+            self.pull_artifact_files = True
+
         self.remote_execution_specs = SandboxRemote.specs_from_config_node(defaults)
 
         # Load pull build trees configuration
diff --git a/src/buildstream/_stream.py b/src/buildstream/_stream.py
index 7e68f1afd..705e0a5d3 100644
--- a/src/buildstream/_stream.py
+++ b/src/buildstream/_stream.py
@@ -272,8 +272,12 @@ class Stream():
             # Remote execution is configured for all projects.
             # Require artifact files only for target elements and their runtime dependencies.
             self._context.set_artifact_files_optional()
-            for element in self.targets:
-                element._set_artifact_files_required()
+
+            # fetch blobs of targets if options set
+            if self._context.pull_artifact_files:
+                scope = Scope.ALL if build_all else Scope.RUN
+                for element in self.targets:
+                    element._set_artifact_files_required(scope=scope)
 
         # Now construct the queues
         #
diff --git a/src/buildstream/element.py b/src/buildstream/element.py
index 758a0b97d..ccb373f1f 100644
--- a/src/buildstream/element.py
+++ b/src/buildstream/element.py
@@ -1529,7 +1529,7 @@ class Element(Plugin):
     # Mark artifact files for this element and its runtime dependencies as
     # required in the local cache.
     #
-    def _set_artifact_files_required(self):
+    def _set_artifact_files_required(self, scope=Scope.RUN):
         if self.__artifact_files_required:
             # Already done
             return
@@ -1537,8 +1537,8 @@ class Element(Plugin):
         self.__artifact_files_required = True
 
         # Request artifact files of runtime dependencies
-        for dep in self.dependencies(Scope.RUN, recurse=False):
-            dep._set_artifact_files_required()
+        for dep in self.dependencies(scope, recurse=False):
+            dep._set_artifact_files_required(scope=scope)
 
     # _artifact_files_required():
     #
diff --git a/tests/remoteexecution/partial.py b/tests/remoteexecution/partial.py
index a684bda18..bf5106f8d 100644
--- a/tests/remoteexecution/partial.py
+++ b/tests/remoteexecution/partial.py
@@ -23,7 +23,8 @@ DATA_DIR = os.path.join(
 # Test that `bst build` does not download file blobs of a build-only dependency
 # to the local cache.
 @pytest.mark.datafiles(DATA_DIR)
-def test_build_dependency_partial_local_cas(cli, datafiles):
+@pytest.mark.parametrize('pull_artifact_files', [True, False])
+def test_build_dependency_partial_local_cas(cli, datafiles, pull_artifact_files):
     project = str(datafiles)
     element_name = 'no-runtime-deps.bst'
     builddep_element_name = 'autotools/amhello.bst'
@@ -33,16 +34,22 @@ def test_build_dependency_partial_local_cas(cli, datafiles):
     services = cli.ensure_services()
     assert set(services) == set(['action-cache', 'execution', 'storage'])
 
+    # configure pull blobs
+    cli.config['remote-execution']['pull-artifact-files'] = pull_artifact_files
+
     result = cli.run(project=project, args=['build', element_name])
     result.assert_success()
 
-    # Verify that the target element is available in local cache
+    # Verify artifact is pulled bar files when ensure artifact files is set
     result = cli.run(project=project, args=['artifact', 'checkout', element_name,
                                             '--directory', checkout])
-    result.assert_success()
-    assert_contains(checkout, ['/test'])
+    if pull_artifact_files:
+        result.assert_success()
+        assert_contains(checkout, ['/test'])
+    else:
+        result.assert_main_error(ErrorDomain.STREAM, 'uncached-checkout-attempt')
 
-    # Verify that the build-only dependency is not (complete) in the local cache
+    # Verify build dependencies is pulled for ALL and BUILD
     result = cli.run(project=project, args=['artifact', 'checkout', builddep_element_name,
                                             '--directory', builddep_checkout])
     result.assert_main_error(ErrorDomain.STREAM, 'uncached-checkout-attempt')