diff options
author | Tristan Maat <tristan.maat@codethink.co.uk> | 2017-10-31 16:01:52 +0000 |
---|---|---|
committer | Tristan Van Berkom <tristan.vanberkom@codethink.co.uk> | 2017-11-04 17:35:30 +0900 |
commit | 54f2eb8502851086804a50d67e279cbb58d9a187 (patch) | |
tree | 2c997e39d95af57a6e5b5bd1234403ca69dbfc26 | |
parent | 6d6f44eaa38222ce7f339e1d9e0321c96614f656 (diff) | |
download | buildstream-54f2eb8502851086804a50d67e279cbb58d9a187.tar.gz |
Adjust commands to multiple targets
-rw-r--r-- | buildstream/_frontend/main.py | 60 | ||||
-rw-r--r-- | buildstream/_pipeline.py | 61 |
2 files changed, 67 insertions, 54 deletions
diff --git a/buildstream/_frontend/main.py b/buildstream/_frontend/main.py index 073bb8c3e..ee4ba8efa 100644 --- a/buildstream/_frontend/main.py +++ b/buildstream/_frontend/main.py @@ -197,13 +197,13 @@ def cli(context, **kwargs): help="Build elements that would not be needed for the current build plan") @click.option('--track', default=False, is_flag=True, help="Track new source references before building (implies --all)") -@click.argument('target', +@click.argument('targets', nargs=-1, type=click.Path(dir_okay=False, readable=True)) @click.pass_obj -def build(app, target, all, track): +def build(app, targets, all, track): """Build elements in a pipeline""" - app.initialize(target, rewritable=track, inconsistent=track, fetch_remote_refs=True) + app.initialize(targets, rewritable=track, inconsistent=track, fetch_remote_refs=True) app.print_heading() try: app.pipeline.build(app.scheduler, all, track) @@ -226,10 +226,10 @@ def build(app, target, all, track): help='The dependencies to fetch (default: plan)') @click.option('--track', default=False, is_flag=True, help="Track new source references before fetching") -@click.argument('target', +@click.argument('targets', nargs=-1, type=click.Path(dir_okay=False, readable=True)) @click.pass_obj -def fetch(app, target, deps, track, except_): +def fetch(app, targets, deps, track, except_): """Fetch sources required to build the pipeline By default this will only try to fetch sources which are @@ -244,7 +244,7 @@ def fetch(app, target, deps, track, except_): plan: Only dependencies required for the build plan all: All dependencies """ - app.initialize(target, rewritable=track, inconsistent=track) + app.initialize(targets, rewritable=track, inconsistent=track) try: dependencies = app.pipeline.deps_elements(deps, except_) app.print_heading(deps=dependencies) @@ -266,10 +266,10 @@ def fetch(app, target, deps, track, except_): @click.option('--deps', '-d', default='none', type=click.Choice(['none', 'all']), help='The dependencies to track (default: none)') -@click.argument('target', +@click.argument('targets', nargs=-1, type=click.Path(dir_okay=False, readable=True)) @click.pass_obj -def track(app, target, deps, except_): +def track(app, targets, deps, except_): """Consults the specified tracking branches for new versions available to build and updates the project with any newly available references. @@ -282,7 +282,7 @@ def track(app, target, deps, except_): none: No dependencies, just the element itself all: All dependencies """ - app.initialize(target, rewritable=True, inconsistent=True) + app.initialize(targets, rewritable=True, inconsistent=True) try: dependencies = app.pipeline.deps_elements(deps, except_) app.print_heading(deps=dependencies) @@ -302,10 +302,10 @@ def track(app, target, deps, except_): @click.option('--deps', '-d', default='none', type=click.Choice(['none', 'all']), help='The dependency artifacts to pull (default: none)') -@click.argument('target', +@click.argument('targets', nargs=-1, type=click.Path(dir_okay=False, readable=True)) @click.pass_obj -def pull(app, target, deps): +def pull(app, targets, deps): """Pull a built artifact from the configured remote artifact cache. Specify `--deps` to control which artifacts to pull: @@ -314,7 +314,7 @@ def pull(app, target, deps): none: No dependencies, just the element itself all: All dependencies """ - app.initialize(target, fetch_remote_refs=True) + app.initialize(targets, fetch_remote_refs=True) try: to_pull = app.pipeline.deps_elements(deps) app.pipeline.pull(app.scheduler, to_pull) @@ -332,10 +332,10 @@ def pull(app, target, deps): @click.option('--deps', '-d', default='none', type=click.Choice(['none', 'all']), help='The dependencies to push (default: none)') -@click.argument('target', +@click.argument('targets', nargs=-1, type=click.Path(dir_okay=False, readable=True)) @click.pass_obj -def push(app, target, deps): +def push(app, targets, deps): """Push a built artifact to the configured remote artifact cache. Specify `--deps` to control which artifacts to push: @@ -344,7 +344,7 @@ def push(app, target, deps): none: No dependencies, just the element itself all: All dependencies """ - app.initialize(target, fetch_remote_refs=True) + app.initialize(targets, fetch_remote_refs=True) try: to_push = app.pipeline.deps_elements(deps) app.pipeline.push(app.scheduler, to_push) @@ -372,10 +372,10 @@ def push(app, target, deps): help='Format string for each element') @click.option('--downloadable', default=False, is_flag=True, help="Refresh downloadable state") -@click.argument('target', +@click.argument('targets', nargs=-1, type=click.Path(dir_okay=False, readable=True)) @click.pass_obj -def show(app, target, deps, except_, order, format, downloadable): +def show(app, targets, deps, except_, order, format, downloadable): """Show elements in the pipeline By default this will show all of the dependencies of the @@ -422,7 +422,7 @@ def show(app, target, deps, except_, order, format, downloadable): bst show target.bst --format \\ $'---------- %{name} ----------\\n%{vars}' """ - app.initialize(target, fetch_remote_refs=downloadable) + app.initialize(targets, fetch_remote_refs=downloadable) try: dependencies = app.pipeline.deps_elements(deps, except_) except PipelineError as e: @@ -474,7 +474,7 @@ def shell(app, target, sysroot, build, command): else: scope = Scope.RUN - app.initialize(target) + app.initialize([target]) # Assert we have everything we need built. missing_deps = [] @@ -493,7 +493,7 @@ def shell(app, target, sysroot, build, command): sys.exit(-1) try: - exitcode = app.pipeline.target._shell(scope, sysroot, command=command) + exitcode = app.pipeline.targets[0]._shell(scope, sysroot, command=command) sys.exit(exitcode) except _BstError as e: click.echo("") @@ -514,7 +514,7 @@ def shell(app, target, sysroot, build, command): def checkout(app, target, directory, force): """Checkout a built artifact to the specified directory """ - app.initialize(target) + app.initialize([target]) try: app.pipeline.checkout(directory, force) click.echo("") @@ -545,7 +545,7 @@ def checkout(app, target, directory, force): def source_bundle(app, target, force, directory, track, compression, except_): """Produce a source bundle to be manually executed""" - app.initialize(target, rewritable=track, inconsistent=track) + app.initialize([target], rewritable=track, inconsistent=track) try: dependencies = app.pipeline.deps_elements('all', except_) app.print_heading(dependencies) @@ -586,7 +586,7 @@ def workspace(): def workspace_open(app, no_checkout, force, source, track, element, directory): """Open a workspace for manual source modification""" - app.initialize(element, rewritable=track, inconsistent=track) + app.initialize([element], rewritable=track, inconsistent=track) try: app.pipeline.open_workspace(app.scheduler, directory, source, no_checkout, track, force) click.echo("") @@ -610,7 +610,7 @@ def workspace_open(app, no_checkout, force, source, track, element, directory): def workspace_close(app, source, remove_dir, element): """Close a workspace""" - app.initialize(element) + app.initialize([element]) if app.interactive and remove_dir: if not click.confirm('This will remove all your changes, are you sure?'): click.echo('Aborting') @@ -640,7 +640,7 @@ def workspace_close(app, source, remove_dir, element): @click.pass_obj def workspace_reset(app, source, track, no_checkout, element): """Reset a workspace to its original state""" - app.initialize(element) + app.initialize([element]) if app.interactive: if not click.confirm('This will remove all your changes, are you sure?'): click.echo('Aborting') @@ -761,10 +761,9 @@ class App(): # # Initialize the main pipeline # - def initialize(self, target, rewritable=False, inconsistent=False, fetch_remote_refs=False): - self.target = target + def initialize(self, targets, rewritable=False, inconsistent=False, fetch_remote_refs=False): - profile_start(Topics.LOAD_PIPELINE, target.replace(os.sep, '-') + '-' + + profile_start(Topics.LOAD_PIPELINE, "_".join(t.replace(os.sep, '-') for t in targets) + '-' + self.host_arch + '-' + self.target_arch) directory = self.main_options['directory'] @@ -836,7 +835,7 @@ class App(): sys.exit(-1) try: - self.pipeline = Pipeline(self.context, self.project, target, + self.pipeline = Pipeline(self.context, self.project, targets, inconsistent=inconsistent, rewritable=rewritable, fetch_remote_refs=fetch_remote_refs, @@ -858,7 +857,8 @@ class App(): self.logger.size_request(self.pipeline) self.messaging_enabled = True - profile_end(Topics.LOAD_PIPELINE, target.replace(os.sep, '-') + '-' + self.host_arch + '-' + self.target_arch) + profile_end(Topics.LOAD_PIPELINE, "_".join(t.replace(os.sep, '-') for t in targets) + '-' + + self.host_arch + '-' + self.target_arch) # # Render the status area, conditional on some internal state diff --git a/buildstream/_pipeline.py b/buildstream/_pipeline.py index 05801895e..08f231723 100644 --- a/buildstream/_pipeline.py +++ b/buildstream/_pipeline.py @@ -95,6 +95,7 @@ class Planner(): depth_sorted = sorted(self.depth_map.items(), key=itemgetter(1), reverse=True) return [item[0] for item in depth_sorted if not item[0]._cached()] + # Pipeline() # # Args: @@ -477,6 +478,9 @@ class Pipeline(): # force (bool): Force overwrite files which exist in `directory` # def checkout(self, directory, force): + # We only have one target in a checkout command + target = self.targets[0] + try: os.makedirs(directory, exist_ok=True) except OSError as e: @@ -498,11 +502,11 @@ class Pipeline(): "Host-incompatible checkout -- no integration commands can be run") # Stage deps into a temporary sandbox first - with self.target._prepare_sandbox(Scope.RUN, None, integrate=can_integrate) as sandbox: + with target._prepare_sandbox(Scope.RUN, None, integrate=can_integrate) as sandbox: # Make copies from the sandbox into to the desired directory sandbox_root = sandbox.get_directory() - with self.target.timed_activity("Copying files to {}".format(directory)): + with target.timed_activity("Copying files to {}".format(directory)): try: utils.copy_files(sandbox_root, directory) except OSError as e: @@ -520,8 +524,10 @@ class Pipeline(): # force (bool): Whether to ignore contents in an existing directory # def open_workspace(self, scheduler, directory, source_index, no_checkout, track_first, force): + # When working on workspaces we only have one target + target = self.targets[0] workdir = os.path.abspath(directory) - sources = list(self.target.sources()) + sources = list(target.sources()) source_index = self.validate_workspace_index(source_index) # Check directory @@ -534,11 +540,11 @@ class Pipeline(): raise PipelineError("Checkout directory is not empty: {}".format(directory)) # Check for workspace config - if self.project._get_workspace(self.target.name, source_index): + if self.project._get_workspace(target.name, source_index): raise PipelineError("Workspace '{}' is already defined." - .format(self.target.name + " - " + str(source_index))) + .format(target.name + " - " + str(source_index))) - plan = [self.target] + plan = [target] # Track/fetch if required queues = [] @@ -571,7 +577,7 @@ class Pipeline(): if not no_checkout: source = sources[source_index] - with self.target.timed_activity("Staging source to {}".format(directory)): + with target.timed_activity("Staging source to {}".format(directory)): if source.get_consistency() != Consistency.CACHED: raise PipelineError("Could not stage uncached source. " + "Use `--track` to track and " + @@ -579,9 +585,9 @@ class Pipeline(): "source.") source._stage(directory) - self.project._set_workspace(self.target, source_index, workdir) + self.project._set_workspace(target, source_index, workdir) - with self.target.timed_activity("Saving workspace configuration"): + with target.timed_activity("Saving workspace configuration"): self.project._save_workspace_config() # close_workspace @@ -593,14 +599,16 @@ class Pipeline(): # remove_dir (bool) - Whether to remove the associated directory # def close_workspace(self, source_index, remove_dir): + # When working on workspaces we only have one target + target = self.targets[0] source_index = self.validate_workspace_index(source_index) # Remove workspace directory if prompted if remove_dir: - path = self.project._get_workspace(self.target.name, source_index) + path = self.project._get_workspace(target.name, source_index) if path is not None: - with self.target.timed_activity("Removing workspace directory {}" - .format(path)): + with target.timed_activity("Removing workspace directory {}" + .format(path)): try: shutil.rmtree(path) except OSError as e: @@ -608,18 +616,18 @@ class Pipeline(): .format(path, e)) from e # Delete the workspace config entry - with self.target.timed_activity("Removing workspace"): + with target.timed_activity("Removing workspace"): try: - self.project._delete_workspace(self.target.name, source_index) + self.project._delete_workspace(target.name, source_index) except KeyError: raise PipelineError("Workspace '{}' is currently not defined" - .format(self.target.name + " - " + str(source_index))) + .format(target.name + " - " + str(source_index))) # Update workspace config self.project._save_workspace_config() # Reset source to avoid checking out the (now empty) workspace - source = list(self.target.sources())[source_index] + source = list(target.sources())[source_index] source._del_workspace() # reset_workspace @@ -634,12 +642,14 @@ class Pipeline(): # no_checkout (bool): Whether to check out the source (at all) # def reset_workspace(self, scheduler, source_index, track, no_checkout): + # When working on workspaces we only have one target + target = self.targets[0] source_index = self.validate_workspace_index(source_index) - workspace_dir = self.project._get_workspace(self.target.name, source_index) + workspace_dir = self.project._get_workspace(target.name, source_index) if workspace_dir is None: raise PipelineError("Workspace '{}' is currently not defined" - .format(self.target.name + " - " + str(source_index))) + .format(target.name + " - " + str(source_index))) self.close_workspace(source_index, True) @@ -770,7 +780,7 @@ class Pipeline(): return [element for element in tree if element not in to_remove] def validate_workspace_index(self, source_index): - sources = list(self.target.sources()) + sources = list(self.targets[0].sources()) # Validate source_index if len(sources) < 1: @@ -815,8 +825,11 @@ class Pipeline(): def source_bundle(self, scheduler, dependencies, force, track_first, compression, directory): + # source-bundle only supports one target + target = self.targets[0] + # Find the correct filename for the compression algorithm - tar_location = os.path.join(directory, self.target.normal_name + ".tar") + tar_location = os.path.join(directory, target.normal_name + ".tar") if compression != "none": tar_location += "." + compression @@ -838,8 +851,8 @@ class Pipeline(): # bound. # Create a temporary directory to build the source tree in - builddir = self.target.get_context().builddir - prefix = "{}-".format(self.target.normal_name) + builddir = target.get_context().builddir + prefix = "{}-".format(target.normal_name) with TemporaryDirectory(prefix=prefix, dir=builddir) as tempdir: source_directory = os.path.join(tempdir, 'source') @@ -857,7 +870,7 @@ class Pipeline(): self._write_element_sources(tempdir, plan) self._write_build_script(tempdir, plan) self._collect_sources(tempdir, tar_location, - self.target.normal_name, compression) + target.normal_name, compression) # Write the element build script to the given directory def _write_element_script(self, directory, element): @@ -894,7 +907,7 @@ class Pipeline(): # Collect the sources in the given sandbox into a tarfile def _collect_sources(self, directory, tar_name, element_name, compression): - with self.target.timed_activity("Creating tarball {}".format(tar_name)): + with self.targets[0].timed_activity("Creating tarball {}".format(tar_name)): if compression == "none": permissions = "w:" else: |