diff options
author | Jürg Billeter <j@bitron.ch> | 2020-10-20 09:34:29 +0200 |
---|---|---|
committer | Jürg Billeter <j@bitron.ch> | 2020-10-20 09:34:29 +0200 |
commit | f0862243d4b63026fb808cbb4cd6da6ae23ba656 (patch) | |
tree | 828ae06aa5a730825f0e806b4dd68a99f2c7eedf | |
parent | 05a5d63357837076198200190e6498af3b987bf2 (diff) | |
download | buildstream-f0862243d4b63026fb808cbb4cd6da6ae23ba656.tar.gz |
wip: _sandboxremote.py: Make storage-service optional with remote cache
-rw-r--r-- | src/buildstream/_context.py | 2 | ||||
-rw-r--r-- | src/buildstream/_project.py | 7 | ||||
-rw-r--r-- | src/buildstream/sandbox/_sandboxremote.py | 126 |
3 files changed, 75 insertions, 60 deletions
diff --git a/src/buildstream/_context.py b/src/buildstream/_context.py index 2c40e855c..faca889c4 100644 --- a/src/buildstream/_context.py +++ b/src/buildstream/_context.py @@ -354,7 +354,7 @@ class Context: else: self.pull_artifact_files = True - self.remote_execution_specs = SandboxRemote.specs_from_config_node(defaults) + self.remote_execution_specs = SandboxRemote.specs_from_config_node(defaults, remote_cache=bool(remote_cache)) # Load pull build trees configuration self.pull_buildtrees = cache.get_bool("pull-buildtrees") diff --git a/src/buildstream/_project.py b/src/buildstream/_project.py index 6154c51c0..baadda051 100644 --- a/src/buildstream/_project.py +++ b/src/buildstream/_project.py @@ -887,8 +887,11 @@ class Project: self.source_cache_specs = SourceCache.specs_from_config_node(config, self.directory) # Load remote-execution configuration for this project - project_specs = SandboxRemote.specs_from_config_node(config, self.directory) - override_specs = SandboxRemote.specs_from_config_node(self._context.get_overrides(self.name), self.directory) + remote_cache = bool(self._context.remote_cache_spec) + project_specs = SandboxRemote.specs_from_config_node(config, self.directory, remote_cache=remote_cache) + override_specs = SandboxRemote.specs_from_config_node( + self._context.get_overrides(self.name), self.directory, remote_cache=remote_cache + ) if override_specs is not None: self.remote_execution_specs = override_specs diff --git a/src/buildstream/sandbox/_sandboxremote.py b/src/buildstream/sandbox/_sandboxremote.py index 2ac159337..4b16dfcce 100644 --- a/src/buildstream/sandbox/_sandboxremote.py +++ b/src/buildstream/sandbox/_sandboxremote.py @@ -58,7 +58,6 @@ class SandboxRemote(SandboxREAPI): if config is None: return - self.storage_url = config.storage_service["url"] self.exec_url = config.exec_service["url"] exec_certs = {} @@ -73,6 +72,23 @@ class SandboxRemote(SandboxREAPI): certificate_chain=exec_certs.get("client-cert"), ) + # Storage service is optional if a remote cache is configured + if config.storage_service: + self.storage_url = config.storage_service["url"] + self.storage_instance = config.storage_service.get("instance-name", None) + self.storage_remote_spec = RemoteSpec( + self.storage_url, + push=True, + server_cert=config.storage_service.get("server-cert"), + client_key=config.storage_service.get("client-key"), + client_cert=config.storage_service.get("client-cert"), + instance_name=self.storage_instance, + ) + else: + self.storage_url = None + self.storage_instance = None + self.storage_remote_spec = None + action_certs = {} for key in ["client-cert", "client-key", "server-cert"]: if key in config.action_service: @@ -93,23 +109,14 @@ class SandboxRemote(SandboxREAPI): self.action_credentials = None self.exec_instance = config.exec_service.get("instance-name", None) - self.storage_instance = config.storage_service.get("instance-name", None) - - self.storage_remote_spec = RemoteSpec( - self.storage_url, - push=True, - server_cert=config.storage_service.get("server-cert"), - client_key=config.storage_service.get("client-key"), - client_cert=config.storage_service.get("client-cert"), - instance_name=self.storage_instance, - ) + self.operation_name = None def info(self, msg): self._get_context().messenger.message(Message(MessageType.INFO, msg, element_name=self._get_element_name())) @staticmethod - def specs_from_config_node(config_node, basedir=None): + def specs_from_config_node(config_node, basedir=None, *, remote_cache=False): def require_node(config, keyname): val = config.get_mapping(keyname, default=None) if val is None: @@ -130,7 +137,10 @@ class SandboxRemote(SandboxREAPI): remote_config.validate_keys(["url", *service_keys]) exec_config = require_node(remote_config, "execution-service") - storage_config = require_node(remote_config, "storage-service") + if remote_cache: + storage_config = remote_config.get_mapping("storage-service", default={}) + else: + storage_config = require_node(remote_config, "storage-service") action_config = remote_config.get_mapping("action-cache-service", default={}) tls_keys = ["client-key", "client-cert", "server-cert"] @@ -305,42 +315,43 @@ class SandboxRemote(SandboxREAPI): action_result = self._check_action_cache(action_digest) if not action_result: - with CASRemote(self.storage_remote_spec, cascache) as casremote: - try: - casremote.init() - except grpc.RpcError as e: - raise SandboxError( - "Failed to contact remote execution CAS endpoint at {}: {}".format(self.storage_url, e) - ) from e - - with self._get_context().messenger.timed_activity( - "Uploading input root", element_name=self._get_element_name() - ): - # Determine blobs missing on remote - try: - input_root_digest = action.input_root_digest - missing_blobs = list(cascache.remote_missing_blobs_for_directory(casremote, input_root_digest)) - except grpc.RpcError as e: - raise SandboxError("Failed to determine missing blobs: {}".format(e)) from e - - # Check if any blobs are also missing locally (partial artifact) - # and pull them from the artifact cache. - try: - local_missing_blobs = cascache.local_missing_blobs(missing_blobs) - if local_missing_blobs: - artifactcache.fetch_missing_blobs(project, local_missing_blobs) - except (grpc.RpcError, BstError) as e: - raise SandboxError("Failed to pull missing blobs from artifact cache: {}".format(e)) from e - - # Add command and action messages to blob list to push - missing_blobs.append(action.command_digest) - missing_blobs.append(action_digest) - - # Now, push the missing blobs to the remote. + if self.storage_remote_spec: + with CASRemote(self.storage_remote_spec, cascache) as casremote: try: - cascache.send_blobs(casremote, missing_blobs) + casremote.init() except grpc.RpcError as e: - raise SandboxError("Failed to push source directory to remote: {}".format(e)) from e + raise SandboxError( + "Failed to contact remote execution CAS endpoint at {}: {}".format(self.storage_url, e) + ) from e + + with self._get_context().messenger.timed_activity( + "Uploading input root", element_name=self._get_element_name() + ): + # Determine blobs missing on remote + try: + input_root_digest = action.input_root_digest + missing_blobs = list(cascache.remote_missing_blobs_for_directory(casremote, input_root_digest)) + except grpc.RpcError as e: + raise SandboxError("Failed to determine missing blobs: {}".format(e)) from e + + # Check if any blobs are also missing locally (partial artifact) + # and pull them from the artifact cache. + try: + local_missing_blobs = cascache.local_missing_blobs(missing_blobs) + if local_missing_blobs: + artifactcache.fetch_missing_blobs(project, local_missing_blobs) + except (grpc.RpcError, BstError) as e: + raise SandboxError("Failed to pull missing blobs from artifact cache: {}".format(e)) from e + + # Add command and action messages to blob list to push + missing_blobs.append(action.command_digest) + missing_blobs.append(action_digest) + + # Now, push the missing blobs to the remote. + try: + cascache.send_blobs(casremote, missing_blobs) + except grpc.RpcError as e: + raise SandboxError("Failed to push source directory to remote: {}".format(e)) from e # Next, try to create a communication channel to the BuildGrid server. url = urlparse(self.exec_url) @@ -364,18 +375,19 @@ class SandboxRemote(SandboxREAPI): operation = self.run_remote_command(channel, action_digest) action_result = self._extract_action_result(operation) - # Fetch outputs - with CASRemote(self.storage_remote_spec, cascache) as casremote: - for output_directory in action_result.output_directories: - tree_digest = output_directory.tree_digest - if tree_digest is None or not tree_digest.hash: - raise SandboxError("Output directory structure had no digest attached.") + if self.storage_remote_spec: + # Fetch outputs + with CASRemote(self.storage_remote_spec, cascache) as casremote: + for output_directory in action_result.output_directories: + tree_digest = output_directory.tree_digest + if tree_digest is None or not tree_digest.hash: + raise SandboxError("Output directory structure had no digest attached.") - # Now do a pull to ensure we have the full directory structure. - cascache.pull_tree(casremote, tree_digest) + # Now do a pull to ensure we have the full directory structure. + cascache.pull_tree(casremote, tree_digest) - # Fetch stdout and stderr blobs - cascache.fetch_blobs(casremote, [action_result.stdout_digest, action_result.stderr_digest]) + # Fetch stdout and stderr blobs + cascache.fetch_blobs(casremote, [action_result.stdout_digest, action_result.stderr_digest]) # Forward remote stdout and stderr if stdout: |