diff options
author | bst-marge-bot <marge-bot@buildstream.build> | 2020-10-27 08:48:47 +0000 |
---|---|---|
committer | bst-marge-bot <marge-bot@buildstream.build> | 2020-10-27 08:48:47 +0000 |
commit | 7038a505cdfd85a9770329825bb50c184d9ac45f (patch) | |
tree | 2042589b678f46c3f0970b370c98a733d7611b21 | |
parent | dda4690b5a604d25ce85e5ff963313498771e57a (diff) | |
parent | 06a703d0a6ad47c8a3796fb1c51f96da03ff976b (diff) | |
download | buildstream-7038a505cdfd85a9770329825bb50c184d9ac45f.tar.gz |
Merge branch 'juerg/shell' into 'master'
Simplify `bst shell` implementation
See merge request BuildStream/buildstream!2084
34 files changed, 129 insertions, 277 deletions
@@ -23,6 +23,13 @@ Core * BREAKING CHANGE: Changed ScriptElement.layout_add() API to take Element instances in place of Element names +CLI +--- + + o BREAKING CHANGE: `bst shell --use-buildtree` is now a boolean flag. + `--use-buildtree=ask` and `--use-buildtree=try` are no longer supported. + + ================== buildstream 1.93.5 ================== diff --git a/man/bst-artifact-checkout.1 b/man/bst-artifact-checkout.1 index d9d62d875..d17bba65f 100644 --- a/man/bst-artifact-checkout.1 +++ b/man/bst-artifact-checkout.1 @@ -1,4 +1,4 @@ -.TH "BST ARTIFACT CHECKOUT" "1" "2020-08-13" "" "bst artifact checkout Manual" +.TH "BST ARTIFACT CHECKOUT" "1" "2020-10-14" "" "bst artifact checkout Manual" .SH NAME bst\-artifact\-checkout \- Checkout contents of an artifact .SH SYNOPSIS diff --git a/man/bst-artifact-delete.1 b/man/bst-artifact-delete.1 index 76c383d6b..c4df14ba2 100644 --- a/man/bst-artifact-delete.1 +++ b/man/bst-artifact-delete.1 @@ -1,4 +1,4 @@ -.TH "BST ARTIFACT DELETE" "1" "2020-08-13" "" "bst artifact delete Manual" +.TH "BST ARTIFACT DELETE" "1" "2020-10-14" "" "bst artifact delete Manual" .SH NAME bst\-artifact\-delete \- Remove artifacts from the local cache .SH SYNOPSIS diff --git a/man/bst-artifact-list-contents.1 b/man/bst-artifact-list-contents.1 index e3d5f1487..f42c049cf 100644 --- a/man/bst-artifact-list-contents.1 +++ b/man/bst-artifact-list-contents.1 @@ -1,4 +1,4 @@ -.TH "BST ARTIFACT LIST-CONTENTS" "1" "2020-08-13" "" "bst artifact list-contents Manual" +.TH "BST ARTIFACT LIST-CONTENTS" "1" "2020-10-14" "" "bst artifact list-contents Manual" .SH NAME bst\-artifact\-list-contents \- List the contents of an artifact .SH SYNOPSIS diff --git a/man/bst-artifact-log.1 b/man/bst-artifact-log.1 index 769907e21..093f26771 100644 --- a/man/bst-artifact-log.1 +++ b/man/bst-artifact-log.1 @@ -1,4 +1,4 @@ -.TH "BST ARTIFACT LOG" "1" "2020-08-13" "" "bst artifact log Manual" +.TH "BST ARTIFACT LOG" "1" "2020-10-14" "" "bst artifact log Manual" .SH NAME bst\-artifact\-log \- Show logs of artifacts .SH SYNOPSIS diff --git a/man/bst-artifact-pull.1 b/man/bst-artifact-pull.1 index c31b1b4c9..14bba67f1 100644 --- a/man/bst-artifact-pull.1 +++ b/man/bst-artifact-pull.1 @@ -1,4 +1,4 @@ -.TH "BST ARTIFACT PULL" "1" "2020-08-13" "" "bst artifact pull Manual" +.TH "BST ARTIFACT PULL" "1" "2020-10-14" "" "bst artifact pull Manual" .SH NAME bst\-artifact\-pull \- Pull a built artifact .SH SYNOPSIS diff --git a/man/bst-artifact-push.1 b/man/bst-artifact-push.1 index cb312c9e5..551ca3c22 100644 --- a/man/bst-artifact-push.1 +++ b/man/bst-artifact-push.1 @@ -1,4 +1,4 @@ -.TH "BST ARTIFACT PUSH" "1" "2020-08-13" "" "bst artifact push Manual" +.TH "BST ARTIFACT PUSH" "1" "2020-10-14" "" "bst artifact push Manual" .SH NAME bst\-artifact\-push \- Push a built artifact .SH SYNOPSIS diff --git a/man/bst-artifact-server.1 b/man/bst-artifact-server.1 index d7120a1a1..757f6b9e1 100644 --- a/man/bst-artifact-server.1 +++ b/man/bst-artifact-server.1 @@ -1,4 +1,4 @@ -.TH "BST-ARTIFACT-SERVER" "1" "2020-08-13" "" "bst-artifact-server Manual" +.TH "BST-ARTIFACT-SERVER" "1" "2020-10-14" "" "bst-artifact-server Manual" .SH NAME bst-artifact-server \- CAS Artifact Server .SH SYNOPSIS diff --git a/man/bst-artifact-show.1 b/man/bst-artifact-show.1 index fc59ba320..8899dc3ad 100644 --- a/man/bst-artifact-show.1 +++ b/man/bst-artifact-show.1 @@ -1,4 +1,4 @@ -.TH "BST ARTIFACT SHOW" "1" "2020-08-13" "" "bst artifact show Manual" +.TH "BST ARTIFACT SHOW" "1" "2020-10-14" "" "bst artifact show Manual" .SH NAME bst\-artifact\-show \- Show the cached state of artifacts .SH SYNOPSIS diff --git a/man/bst-artifact.1 b/man/bst-artifact.1 index 3cb170557..9506ff7f9 100644 --- a/man/bst-artifact.1 +++ b/man/bst-artifact.1 @@ -1,4 +1,4 @@ -.TH "BST ARTIFACT" "1" "2020-08-13" "" "bst artifact Manual" +.TH "BST ARTIFACT" "1" "2020-10-14" "" "bst artifact Manual" .SH NAME bst\-artifact \- Manipulate cached artifacts. .SH SYNOPSIS diff --git a/man/bst-build.1 b/man/bst-build.1 index a5a25536b..d0c271f05 100644 --- a/man/bst-build.1 +++ b/man/bst-build.1 @@ -1,4 +1,4 @@ -.TH "BST BUILD" "1" "2020-08-13" "" "bst build Manual" +.TH "BST BUILD" "1" "2020-10-14" "" "bst build Manual" .SH NAME bst\-build \- Build elements in a pipeline .SH SYNOPSIS diff --git a/man/bst-help.1 b/man/bst-help.1 index efd495ba1..58837e34b 100644 --- a/man/bst-help.1 +++ b/man/bst-help.1 @@ -1,4 +1,4 @@ -.TH "BST HELP" "1" "2020-08-13" "" "bst help Manual" +.TH "BST HELP" "1" "2020-10-14" "" "bst help Manual" .SH NAME bst\-help \- Print usage information .SH SYNOPSIS diff --git a/man/bst-init.1 b/man/bst-init.1 index 433b7de67..2bf2e836d 100644 --- a/man/bst-init.1 +++ b/man/bst-init.1 @@ -1,4 +1,4 @@ -.TH "BST INIT" "1" "2020-08-13" "" "bst init Manual" +.TH "BST INIT" "1" "2020-10-14" "" "bst init Manual" .SH NAME bst\-init \- Initialize a new BuildStream project .SH SYNOPSIS diff --git a/man/bst-shell.1 b/man/bst-shell.1 index 925ae1f56..40636e643 100644 --- a/man/bst-shell.1 +++ b/man/bst-shell.1 @@ -1,4 +1,4 @@ -.TH "BST SHELL" "1" "2020-08-13" "" "bst shell Manual" +.TH "BST SHELL" "1" "2020-10-14" "" "bst shell Manual" .SH NAME bst\-shell \- Shell into an element's sandbox environment .SH SYNOPSIS @@ -36,8 +36,8 @@ Mount a file or directory into the sandbox \fB\-\-isolate\fP Create an isolated build sandbox .TP -\fB\-t,\fP \-\-use\-buildtree [ask|try|always|never] -Stage a buildtree. If `always` is set, will always fail to build if a buildtree is not available. --pull and pull-buildtrees configuration is needed if trying to query for remotely cached buildtrees. [default: ask] +\fB\-t,\fP \-\-use\-buildtree +Stage a buildtree. Will fail if a buildtree is not available. --pull and pull-buildtrees configuration is needed if trying to query for remotely cached buildtrees. .TP \fB\-\-pull\fP Attempt to pull missing or incomplete artifacts diff --git a/man/bst-show.1 b/man/bst-show.1 index eaee58be6..1ce152f94 100644 --- a/man/bst-show.1 +++ b/man/bst-show.1 @@ -1,4 +1,4 @@ -.TH "BST SHOW" "1" "2020-08-13" "" "bst show Manual" +.TH "BST SHOW" "1" "2020-10-14" "" "bst show Manual" .SH NAME bst\-show \- Show elements in the pipeline .SH SYNOPSIS diff --git a/man/bst-source-checkout.1 b/man/bst-source-checkout.1 index f450d97f3..134593114 100644 --- a/man/bst-source-checkout.1 +++ b/man/bst-source-checkout.1 @@ -1,4 +1,4 @@ -.TH "BST SOURCE CHECKOUT" "1" "2020-08-13" "" "bst source checkout Manual" +.TH "BST SOURCE CHECKOUT" "1" "2020-10-14" "" "bst source checkout Manual" .SH NAME bst\-source\-checkout \- Checkout sources of an element .SH SYNOPSIS diff --git a/man/bst-source-fetch.1 b/man/bst-source-fetch.1 index 4fc2633a2..35a94c6e5 100644 --- a/man/bst-source-fetch.1 +++ b/man/bst-source-fetch.1 @@ -1,4 +1,4 @@ -.TH "BST SOURCE FETCH" "1" "2020-08-13" "" "bst source fetch Manual" +.TH "BST SOURCE FETCH" "1" "2020-10-14" "" "bst source fetch Manual" .SH NAME bst\-source\-fetch \- Fetch sources in a pipeline .SH SYNOPSIS diff --git a/man/bst-source-push.1 b/man/bst-source-push.1 index d517e74b5..531a1e461 100644 --- a/man/bst-source-push.1 +++ b/man/bst-source-push.1 @@ -1,4 +1,4 @@ -.TH "BST SOURCE PUSH" "1" "2020-08-13" "" "bst source push Manual" +.TH "BST SOURCE PUSH" "1" "2020-10-14" "" "bst source push Manual" .SH NAME bst\-source\-push \- Push sources in a pipeline .SH SYNOPSIS diff --git a/man/bst-source-track.1 b/man/bst-source-track.1 index 3cd3b17b5..db8631e79 100644 --- a/man/bst-source-track.1 +++ b/man/bst-source-track.1 @@ -1,4 +1,4 @@ -.TH "BST SOURCE TRACK" "1" "2020-08-13" "" "bst source track Manual" +.TH "BST SOURCE TRACK" "1" "2020-10-14" "" "bst source track Manual" .SH NAME bst\-source\-track \- Track new source references .SH SYNOPSIS diff --git a/man/bst-source.1 b/man/bst-source.1 index 94a268a57..0f308aa66 100644 --- a/man/bst-source.1 +++ b/man/bst-source.1 @@ -1,4 +1,4 @@ -.TH "BST SOURCE" "1" "2020-08-13" "" "bst source Manual" +.TH "BST SOURCE" "1" "2020-10-14" "" "bst source Manual" .SH NAME bst\-source \- Manipulate sources for an element .SH SYNOPSIS diff --git a/man/bst-workspace-close.1 b/man/bst-workspace-close.1 index 6abce3485..24fc7a4d9 100644 --- a/man/bst-workspace-close.1 +++ b/man/bst-workspace-close.1 @@ -1,4 +1,4 @@ -.TH "BST WORKSPACE CLOSE" "1" "2020-08-13" "" "bst workspace close Manual" +.TH "BST WORKSPACE CLOSE" "1" "2020-10-14" "" "bst workspace close Manual" .SH NAME bst\-workspace\-close \- Close workspaces .SH SYNOPSIS diff --git a/man/bst-workspace-list.1 b/man/bst-workspace-list.1 index 4e7aacd52..04ff5f323 100644 --- a/man/bst-workspace-list.1 +++ b/man/bst-workspace-list.1 @@ -1,4 +1,4 @@ -.TH "BST WORKSPACE LIST" "1" "2020-08-13" "" "bst workspace list Manual" +.TH "BST WORKSPACE LIST" "1" "2020-10-14" "" "bst workspace list Manual" .SH NAME bst\-workspace\-list \- List open workspaces .SH SYNOPSIS diff --git a/man/bst-workspace-open.1 b/man/bst-workspace-open.1 index 900205a7a..ab3d7f68e 100644 --- a/man/bst-workspace-open.1 +++ b/man/bst-workspace-open.1 @@ -1,4 +1,4 @@ -.TH "BST WORKSPACE OPEN" "1" "2020-08-13" "" "bst workspace open Manual" +.TH "BST WORKSPACE OPEN" "1" "2020-10-14" "" "bst workspace open Manual" .SH NAME bst\-workspace\-open \- Open a new workspace .SH SYNOPSIS diff --git a/man/bst-workspace-reset.1 b/man/bst-workspace-reset.1 index 3a73b473b..63e31cef1 100644 --- a/man/bst-workspace-reset.1 +++ b/man/bst-workspace-reset.1 @@ -1,4 +1,4 @@ -.TH "BST WORKSPACE RESET" "1" "2020-08-13" "" "bst workspace reset Manual" +.TH "BST WORKSPACE RESET" "1" "2020-10-14" "" "bst workspace reset Manual" .SH NAME bst\-workspace\-reset \- Reset a workspace to its original state .SH SYNOPSIS diff --git a/man/bst-workspace.1 b/man/bst-workspace.1 index 89af0b819..a4c931d5c 100644 --- a/man/bst-workspace.1 +++ b/man/bst-workspace.1 @@ -1,4 +1,4 @@ -.TH "BST WORKSPACE" "1" "2020-08-13" "" "bst workspace Manual" +.TH "BST WORKSPACE" "1" "2020-10-14" "" "bst workspace Manual" .SH NAME bst\-workspace \- Manipulate developer workspaces .SH SYNOPSIS @@ -1,4 +1,4 @@ -.TH "BST" "1" "2020-08-13" "" "bst Manual" +.TH "BST" "1" "2020-10-14" "" "bst Manual" .SH NAME bst \- Build and manipulate BuildStream projects... .SH SYNOPSIS diff --git a/src/buildstream/_frontend/app.py b/src/buildstream/_frontend/app.py index b25a421c3..59bc3513b 100644 --- a/src/buildstream/_frontend/app.py +++ b/src/buildstream/_frontend/app.py @@ -446,15 +446,16 @@ class App: # if they are available in the execution context. # # Args: - # element_name (str): The element's full name - # element_key (tuple): The element's display key + # element (Element): The element # # Returns: # (str): The formatted prompt to display in the shell # - def shell_prompt(self, element_name, element_key): + def shell_prompt(self, element): - _, key, dim = element_key + element_name = element._get_full_name() + + _, key, dim = element._get_display_key() if self.colors: prompt = ( @@ -703,10 +704,14 @@ class App: if choice == "shell": click.echo("\nDropping into an interactive shell in the failed build sandbox\n", err=True) try: - unique_id, element_key = element - prompt = self.shell_prompt(full_name, element_key) + unique_id, _ = element self.stream.shell( - None, _Scope.BUILD, prompt, isolate=True, usebuildtree="always", unique_id=unique_id + None, + _Scope.BUILD, + self.shell_prompt, + isolate=True, + usebuildtree="always", + unique_id=unique_id, ) except BstError as e: click.echo("Error while attempting to create interactive shell: {}".format(e), err=True) diff --git a/src/buildstream/_frontend/cli.py b/src/buildstream/_frontend/cli.py index 946024bb7..5e42bda68 100644 --- a/src/buildstream/_frontend/cli.py +++ b/src/buildstream/_frontend/cli.py @@ -578,12 +578,9 @@ def show(app, elements, deps, except_, order, format_): "--use-buildtree", "-t", "cli_buildtree", - type=click.Choice(["ask", "try", "always", "never"]), - default="ask", - show_default=True, + is_flag=True, help=( - "Stage a buildtree. If `always` is set, will always fail to " - "build if a buildtree is not available." + "Stage a buildtree. Will fail if a buildtree is not available." " --pull and pull-buildtrees configuration is needed " "if trying to query for remotely cached buildtrees." ), @@ -617,11 +614,11 @@ def shell(app, element, mount, isolate, build_, cli_buildtree, pull_, command): from ..element import _Scope from .._project import HostMount - scope = _Scope.BUILD if build_ else _Scope.RUN + # Buildtree can only be used with build shells + if cli_buildtree: + build_ = True - # We may need to fetch dependency artifacts if we're pulling the artifact - selection = _PipelineSelection.ALL if pull_ else _PipelineSelection.NONE - use_buildtree = None + scope = _Scope.BUILD if build_ else _Scope.RUN with app.initialized(): if not element: @@ -629,104 +626,18 @@ def shell(app, element, mount, isolate, build_, cli_buildtree, pull_, command): if not element: raise AppError('Missing argument "ELEMENT".') - elements = app.stream.load_selection((element,), selection=selection, use_artifact_config=True) - - # last one will be the element we want to stage, previous ones are - # elements to try and pull - element = elements[-1] - pull_dependencies = elements[:-1] if pull_ else None - - element_name = element._get_full_name() - element_key = element._get_display_key() - - prompt = app.shell_prompt(element_name, element_key) mounts = [HostMount(path, host_path) for host_path, path in mount] - artifact_is_cached = element._cached() - buildtree_is_cached = element._cached_buildtree() - buildtree_exists = element._buildtree_exists() - can_attempt_pull = app.context.pull_buildtrees and pull_ - - if cli_buildtree in ("always", "try"): - if buildtree_is_cached: - use_buildtree = cli_buildtree - # If element is already cached, we can check the proto to see if the buildtree existed - elif artifact_is_cached: - if not buildtree_exists: - if cli_buildtree == "always": - # Exit early if it won't be possible to even fetch a buildtree with always option - raise AppError("Artifact was created without buildtree, unable to launch shell with it") - click.echo( - "WARNING: Artifact created without buildtree, shell will be loaded without it", err=True - ) - elif can_attempt_pull: - use_buildtree = cli_buildtree - click.echo( - "WARNING: buildtree is not cached locally but did exist, will attempt to pull from available remotes", - err=True, - ) - else: - if cli_buildtree == "always": - # Exit early if it won't be possible to perform a fetch as pull semantics aren't present - raise AppError( - "Artifact has a buildtree but it isn't cached. Can be retried with --pull and pull-buildtrees configured" - ) - click.echo("WARNING: buildtree is not cached locally, shell will be loaded without it", err=True) - # If element isn't cached at all, we can't check the proto to see if it existed so can't exit early - elif can_attempt_pull: - use_buildtree = cli_buildtree - if use_buildtree == "always": - click.echo( - "WARNING: Element is not cached so buildtree status unknown, will attempt to pull from available remotes", - err=True, - ) - else: - if cli_buildtree == "always": - # Exit early as there is no buildtree locally & can_attempt_pull is False - raise AppError( - "Artifact not cached locally. Can be retried with --pull and pull-buildtrees configured" - ) - click.echo("WARNING: buildtree is not cached locally, shell will be loaded without it", err=True) - else: - # If the value has defaulted to ask and in non interactive mode, don't consider the buildtree, this - # being the default behaviour of the command - if app.interactive and cli_buildtree == "ask": - if buildtree_is_cached and bool(click.confirm("Do you want to use the cached buildtree?")): - use_buildtree = "always" - elif can_attempt_pull: - # If buildtree not cached, check if it's worth presenting the user a dialogue - message = None - if artifact_is_cached: - if buildtree_exists: - message = "Buildtree not cached but can be pulled if in available remotes, do you want to use it?" - else: - message = "Element is not cached so buildtree status unknown, do you want to pull and use it?" - if message: - try: - choice = click.prompt( - message, type=click.Choice(["try", "always", "never"]), err=True, show_choices=True, - ) - except click.Abort: - click.echo("Aborting", err=True) - sys.exit(-1) - - if choice != "never": - use_buildtree = choice - - # Raise warning if the element is cached in a failed state - if use_buildtree and element._cached_failure(): - click.echo("WARNING: using a buildtree from a failed build.", err=True) - try: exitcode = app.stream.shell( element, scope, - prompt, + app.shell_prompt, mounts=mounts, isolate=isolate, command=command, - usebuildtree=use_buildtree, - pull_dependencies=pull_dependencies, + usebuildtree=cli_buildtree, + pull_=pull_, ) except BstError as e: raise AppError("Error launching shell: {}".format(e), detail=e.detail) from e diff --git a/src/buildstream/_stream.py b/src/buildstream/_stream.py index 3b0a308e7..af0537fe1 100644 --- a/src/buildstream/_stream.py +++ b/src/buildstream/_stream.py @@ -167,14 +167,14 @@ class Stream: # Run a shell # # Args: - # element (Element): An Element object to run the shell for + # element (str): The name of the element to run the shell for # scope (_Scope): The scope for the shell (_Scope.BUILD or _Scope.RUN) - # prompt (str): The prompt to display in the shell + # prompt (function): A function to return the prompt to display in the shell # mounts (list of HostMount): Additional directories to mount into the sandbox # isolate (bool): Whether to isolate the environment like we do in builds # command (list): An argv to launch in the sandbox, or None - # usebuildtree (str): Whether to use a buildtree as the source, given cli option - # pull_dependencies ([Element]|None): Elements to attempt to pull + # usebuildtree (bool): Whether to use a buildtree as the source, given cli option + # pull_ (bool): Whether to attempt to pull missing or incomplete artifacts # unique_id: (str): Whether to use a unique_id to load an Element instance # # Returns: @@ -189,61 +189,62 @@ class Stream: mounts=None, isolate=False, command=None, - usebuildtree=None, - pull_dependencies=None, + usebuildtree=False, + pull_=False, unique_id=None ): # Load the Element via the unique_id if given if unique_id and element is None: element = Plugin._lookup(unique_id) + else: + selection = _PipelineSelection.BUILD if scope == _Scope.BUILD else _PipelineSelection.RUN + + elements = self.load_selection((element,), selection=selection, use_artifact_config=True) + + # Get element to stage from `targets` list. + # If scope is BUILD, it will not be in the `elements` list. + assert len(self.targets) == 1 + element = self.targets[0] + element._set_required(scope) + + if pull_: + self._scheduler.clear_queues() + self._add_queue(PullQueue(self._scheduler)) + plan = self._pipeline.add_elements([element], elements) + self._enqueue_plan(plan) + self._run() missing_deps = [dep for dep in self._pipeline.dependencies([element], scope) if not dep._cached()] if missing_deps: - if not pull_dependencies: - raise StreamError( - "Elements need to be built or downloaded before staging a shell environment", - detail="\n".join(list(map(lambda x: x._get_full_name(), missing_deps))), - ) - self._message(MessageType.INFO, "Attempting to fetch missing or incomplete artifacts") - self._scheduler.clear_queues() - self._add_queue(PullQueue(self._scheduler)) - plan = self._pipeline.add_elements([element], missing_deps) - self._enqueue_plan(plan) - self._run() + raise StreamError( + "Elements need to be built or downloaded before staging a shell environment", + detail="\n".join(list(map(lambda x: x._get_full_name(), missing_deps))), + ) - buildtree = False # Check if we require a pull queue attempt, with given artifact state and context if usebuildtree: if not element._cached_buildtree(): - require_buildtree = self._buildtree_pull_required([element]) - # Attempt a pull queue for the given element if remote and context allow it - if require_buildtree: - self._message(MessageType.INFO, "Attempting to fetch missing artifact buildtree") - self._scheduler.clear_queues() - self._add_queue(PullQueue(self._scheduler)) - self._enqueue_plan(require_buildtree) - self._run() - # Now check if the buildtree was successfully fetched - if element._cached_buildtree(): - buildtree = True - - if not buildtree: - message = "Buildtree is not cached locally or in available remotes" - if usebuildtree == "always": - raise StreamError(message) - - self._message(MessageType.INFO, message + ", shell will be loaded without it") - else: - buildtree = True + remotes_message = " or in available remotes" if pull_ else "" + if not element._cached(): + message = "Artifact not cached locally" + remotes_message + elif element._buildtree_exists(): + message = "Buildtree is not cached locally" + remotes_message + else: + message = "Artifact was created without buildtree" + raise StreamError(message) + + # Raise warning if the element is cached in a failed state + if element._cached_failure(): + self._message(MessageType.WARN, "using a buildtree from a failed build.") # Ensure we have our sources if we are launching a build shell - if scope == _Scope.BUILD and not buildtree: + if scope == _Scope.BUILD and not usebuildtree: self._fetch([element]) self._pipeline.assert_sources_cached([element]) return element._shell( - scope, mounts=mounts, isolate=isolate, prompt=prompt, command=command, usebuildtree=buildtree + scope, mounts=mounts, isolate=isolate, prompt=prompt(element), command=command, usebuildtree=usebuildtree ) # build() diff --git a/src/buildstream/element.py b/src/buildstream/element.py index 3df8894f0..c9480d4fc 100644 --- a/src/buildstream/element.py +++ b/src/buildstream/element.py @@ -1557,10 +1557,13 @@ class Element(Plugin): # _set_required(): # - # Mark this element and its runtime dependencies as required. + # Mark this element and its dependencies as required. # This unblocks pull/fetch/build. # - def _set_required(self): + # Args: + # scope (_Scope): The scope of dependencies to mark as required + # + def _set_required(self, scope=_Scope.RUN): assert utils._is_main_process(), "This has an impact on all elements and must be run in the main process" if self.__required: @@ -1569,9 +1572,9 @@ class Element(Plugin): self.__required = True - # Request artifacts of runtime dependencies - for dep in self._dependencies(_Scope.RUN, recurse=False): - dep._set_required() + # Request artifacts of dependencies + for dep in self._dependencies(scope, recurse=False): + dep._set_required(scope=_Scope.RUN) # When an element becomes required, it must be assembled for # the current pipeline. `__schedule_assembly_when_necessary()` diff --git a/tests/integration/shellbuildtrees.py b/tests/integration/shellbuildtrees.py index 0d80c1640..5164c0209 100644 --- a/tests/integration/shellbuildtrees.py +++ b/tests/integration/shellbuildtrees.py @@ -46,7 +46,7 @@ def test_buildtree_staged_forced_true(cli_integration, datafiles): res.assert_success() res = cli_integration.run( - project=project, args=["shell", "--build", "--use-buildtree", "always", element_name, "--", "cat", "test"] + project=project, args=["shell", "--build", "--use-buildtree", element_name, "--", "cat", "test"] ) res.assert_success() assert "Hi" in res.output @@ -67,53 +67,10 @@ def test_buildtree_staged_warn_empty_cached(cli_integration, tmpdir, datafiles): res.assert_success() res = cli_integration.run( - project=project, args=["shell", "--build", "--use-buildtree", "always", element_name, "--", "cat", "test"] + project=project, args=["shell", "--build", "--use-buildtree", element_name, "--", "cat", "test"] ) res.assert_main_error(ErrorDomain.APP, None) - assert "Artifact was created without buildtree, unable to launch shell with it" in res.stderr - - # Now attempt the same with the try option, this should not attempt to find a buildtree - # and just launch the shell, however the cat should still fail. - res = cli_integration.run( - project=project, args=["shell", "--build", "--use-buildtree", "try", element_name, "--", "cat", "test"] - ) - assert "Artifact created without buildtree, shell will be loaded without it" in res.stderr - assert "Hi" not in res.output - - -@pytest.mark.datafiles(DATA_DIR) -@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") -def test_buildtree_staged_if_available(cli_integration, datafiles): - # Test that a build tree can be correctly detected. - project = str(datafiles) - element_name = "build-shell/buildtree.bst" - - res = cli_integration.run(project=project, args=["--cache-buildtrees", "always", "build", element_name]) - res.assert_success() - - res = cli_integration.run( - project=project, args=["shell", "--build", "--use-buildtree", "try", element_name, "--", "cat", "test"] - ) - res.assert_success() - assert "Hi" in res.output - - -@pytest.mark.datafiles(DATA_DIR) -@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") -def test_buildtree_staged_forced_false(cli_integration, datafiles): - # Test that if we ask not to have a build tree it is not there - project = str(datafiles) - element_name = "build-shell/buildtree.bst" - - res = cli_integration.run(project=project, args=["--cache-buildtrees", "always", "build", element_name]) - res.assert_success() - - res = cli_integration.run( - project=project, args=["shell", "--build", "--use-buildtree", "never", element_name, "--", "cat", "test"] - ) - res.assert_shell_error() - - assert "Hi" not in res.output + assert "Error launching shell: Artifact was created without buildtree" in res.stderr @pytest.mark.datafiles(DATA_DIR) @@ -128,10 +85,10 @@ def test_buildtree_from_failure(cli_integration, datafiles): # Assert that file has expected contents res = cli_integration.run( - project=project, args=["shell", "--build", element_name, "--use-buildtree", "always", "--", "cat", "test"] + project=project, args=["shell", "--build", element_name, "--use-buildtree", "--", "cat", "test"] ) res.assert_success() - assert "WARNING: using a buildtree from a failed build" in res.stderr + assert "WARNING using a buildtree from a failed build" in res.stderr assert "Hi" in res.output @@ -150,10 +107,10 @@ def test_buildtree_from_failure_option_never(cli_integration, tmpdir, datafiles) res.assert_main_error(ErrorDomain.STREAM, None) res = cli_integration.run( - project=project, args=["shell", "--build", element_name, "--use-buildtree", "always", "--", "cat", "test"] + project=project, args=["shell", "--build", element_name, "--use-buildtree", "--", "cat", "test"] ) res.assert_main_error(ErrorDomain.APP, None) - assert "Artifact was created without buildtree, unable to launch shell with it" in res.stderr + assert "Error launching shell: Artifact was created without buildtree" in res.stderr @pytest.mark.datafiles(DATA_DIR) @@ -172,10 +129,10 @@ def test_buildtree_from_failure_option_always(cli_integration, tmpdir, datafiles res.assert_main_error(ErrorDomain.STREAM, None) res = cli_integration.run( - project=project, args=["shell", "--build", element_name, "--use-buildtree", "always", "--", "cat", "test"] + project=project, args=["shell", "--build", element_name, "--use-buildtree", "--", "cat", "test"] ) res.assert_success() - assert "WARNING: using a buildtree from a failed build" in res.stderr + assert "WARNING using a buildtree from a failed build" in res.stderr assert "Hi" in res.output @@ -206,9 +163,7 @@ def test_buildtree_pulled(cli, tmpdir, datafiles): result.assert_success() # Check it's using the cached build tree - res = cli.run( - project=project, args=["shell", "--build", element_name, "--use-buildtree", "always", "--", "cat", "test"] - ) + res = cli.run(project=project, args=["shell", "--build", element_name, "--use-buildtree", "--", "cat", "test"]) res.assert_success() @@ -237,9 +192,7 @@ def test_buildtree_options(cli, tmpdir, datafiles): result.assert_success() # Check it's not using the cached build tree - res = cli.run( - project=project, args=["shell", "--build", element_name, "--use-buildtree", "never", "--", "cat", "test"] - ) + res = cli.run(project=project, args=["shell", "--build", element_name, "--", "cat", "test"]) res.assert_shell_error() assert "Hi" not in res.output @@ -249,17 +202,7 @@ def test_buildtree_options(cli, tmpdir, datafiles): res.assert_shell_error() assert "Hi" not in res.output - # Check correctly handling the lack of buildtree, with 'try' not attempting to - # pull the buildtree as the user context is by default set to not pull them - # and --pull not given - res = cli.run( - project=project, args=["shell", "--build", element_name, "--use-buildtree", "try", "--", "cat", "test"] - ) - assert "Hi" not in res.output - assert "Attempting to fetch missing artifact buildtrees" not in res.stderr - assert "WARNING: buildtree is not cached locally, shell will be loaded without it" in res.stderr - - # Check correctly handling the lack of buildtree, with 'try' attempting and succeeding + # Check correctly handling the lack of buildtree, with '--use-buildtree' attempting and succeeding # to pull the buildtree as the user context allow the pulling of buildtrees and it is # available in the remote and --pull given res = cli.run( @@ -271,35 +214,27 @@ def test_buildtree_options(cli, tmpdir, datafiles): element_name, "--pull", "--use-buildtree", - "try", "--", "cat", "test", ], ) - assert "Attempting to fetch missing artifact buildtree" in res.stderr assert "Hi" in res.output shutil.rmtree(os.path.join(os.path.join(str(tmpdir), "cache", "cas"))) shutil.rmtree(os.path.join(os.path.join(str(tmpdir), "cache", "artifacts"))) assert cli.get_element_state(project, element_name) != "cached" - # Check it's not loading the shell at all with always set for the buildtree, when the + # Check it's not loading the shell at all with `--use-buildtree`, when the # user context does not allow for buildtree pulling and --pull is not given result = cli.run(project=project, args=["artifact", "pull", "--deps", "all", element_name]) result.assert_success() - res = cli.run( - project=project, args=["shell", "--build", element_name, "--use-buildtree", "always", "--", "cat", "test"] - ) + res = cli.run(project=project, args=["shell", "--build", element_name, "--use-buildtree", "--", "cat", "test"]) res.assert_main_error(ErrorDomain.APP, None) - assert ( - "Artifact has a buildtree but it isn't cached. Can be retried with --pull and pull-buildtrees configured" - in res.stderr - ) + assert "Buildtree is not cached locally" in res.stderr assert "Hi" not in res.output - assert "Attempting to fetch missing artifact buildtree" not in res.stderr # Check that when user context is set to pull buildtrees and a remote has the buildtree, - # 'always' will attempt and succeed at pulling the missing buildtree with --pull set. + # '--use-buildtree' will attempt and succeed at pulling the missing buildtree with --pull set. res = cli.run( project=project, args=[ @@ -309,17 +244,13 @@ def test_buildtree_options(cli, tmpdir, datafiles): element_name, "--pull", "--use-buildtree", - "always", "--", "cat", "test", ], ) assert "Hi" in res.output - assert ( - "buildtree is not cached locally but did exist, will attempt to pull from available remotes" in res.stderr - ) - assert "Attempting to fetch missing artifact buildtree" in res.stderr + assert res.get_pulled_elements() == [element_name] # Tests running pull and pull-buildtree options at the same time. @@ -345,10 +276,10 @@ def test_pull_buildtree_pulled(cli, tmpdir, datafiles): # and pull-buildtrees were not both set res = cli.run( project=project, - args=["shell", "--build", element_name, "--pull", "--use-buildtree", "always", "--", "cat", "test",], + args=["shell", "--build", element_name, "--pull", "--use-buildtree", "--", "cat", "test",], ) res.assert_main_error(ErrorDomain.APP, None) - assert "Artifact not cached locally. Can be retried with --pull and pull-buildtrees configured" in res.stderr + assert "Buildtree is not cached locally" in res.stderr # Check it's using the cached build tree, because --pull # and pull-buildtrees were both set @@ -361,7 +292,6 @@ def test_pull_buildtree_pulled(cli, tmpdir, datafiles): element_name, "--pull", "--use-buildtree", - "always", "--", "cat", "test", diff --git a/tests/integration/workspace.py b/tests/integration/workspace.py index a2ea4841a..fab45b123 100644 --- a/tests/integration/workspace.py +++ b/tests/integration/workspace.py @@ -80,7 +80,7 @@ def test_workspace_commanddir(cli, datafiles): # using the cached buildtree. res = cli.run( project=project, - args=["shell", "--build", element_name, "--use-buildtree", "always", "--", "find", "..", "-mindepth", "1",], + args=["shell", "--build", element_name, "--use-buildtree", "--", "find", "..", "-mindepth", "1",], ) res.assert_success() @@ -290,7 +290,7 @@ def test_incremental_configure_commands_run_only_once(cli, datafiles): # the configure should have been run in the sandbox, so check the buildtree res = cli.run( project=project, - args=["shell", "--build", element_name, "--use-buildtree", "always", "--", "find", ".", "-mindepth", "1",], + args=["shell", "--build", element_name, "--use-buildtree", "--", "find", ".", "-mindepth", "1",], ) res.assert_success() @@ -311,7 +311,7 @@ def test_incremental_configure_commands_run_only_once(cli, datafiles): assert not os.path.exists(os.path.join(workspace, "prepared-again")) res = cli.run( project=project, - args=["shell", "--build", element_name, "--use-buildtree", "always", "--", "find", ".", "-mindepth", "1",], + args=["shell", "--build", element_name, "--use-buildtree", "--", "find", ".", "-mindepth", "1",], ) res.assert_success() @@ -382,9 +382,7 @@ def test_workspace_failed_logs(cli, datafiles): def get_buildtree_file_contents(cli, project, element_name, filename): - res = cli.run( - project=project, args=["shell", "--build", element_name, "--use-buildtree", "always", "--", "cat", filename,], - ) + res = cli.run(project=project, args=["shell", "--build", element_name, "--use-buildtree", "--", "cat", filename,],) res.assert_success() return res.output diff --git a/tests/remoteexecution/buildtree.py b/tests/remoteexecution/buildtree.py index 57e25cd14..317747fe8 100644 --- a/tests/remoteexecution/buildtree.py +++ b/tests/remoteexecution/buildtree.py @@ -63,8 +63,6 @@ def test_buildtree_remote(cli, tmpdir, datafiles): res.assert_success() # check it works this time - res = cli.run( - project=project, args=["shell", "--build", element_name, "--use-buildtree", "always", "--", "cat", "test"] - ) + res = cli.run(project=project, args=["shell", "--build", element_name, "--use-buildtree", "--", "cat", "test"]) res.assert_success() assert "Hi" in res.output diff --git a/tests/remoteexecution/workspace.py b/tests/remoteexecution/workspace.py index b525cefcd..cf8587230 100644 --- a/tests/remoteexecution/workspace.py +++ b/tests/remoteexecution/workspace.py @@ -115,7 +115,6 @@ def check_buildtree( "--build", element_name, "--use-buildtree", - "always", "--", "find", ".", @@ -176,7 +175,7 @@ def check_buildtree( def get_timemark(cli, project, element_name, marker): result = cli.run( - project=project, args=["shell", "--build", element_name, "--use-buildtree", "always", "--", "cat", marker[1:]], + project=project, args=["shell", "--build", element_name, "--use-buildtree", "--", "cat", marker[1:]], ) result.assert_success() marker_time = int(result.output) |