diff options
-rw-r--r-- | src/buildstream/_frontend/cli.py | 84 | ||||
-rw-r--r-- | src/buildstream/_scheduler/__init__.py | 1 | ||||
-rw-r--r-- | src/buildstream/_scheduler/queues/sourcepullqueue.py | 43 | ||||
-rw-r--r-- | src/buildstream/_stream.py | 48 | ||||
-rw-r--r-- | src/buildstream/element.py | 16 | ||||
-rw-r--r-- | tests/frontend/completions.py | 2 |
6 files changed, 193 insertions, 1 deletions
diff --git a/src/buildstream/_frontend/cli.py b/src/buildstream/_frontend/cli.py index 1365cede4..8f0e1fbd0 100644 --- a/src/buildstream/_frontend/cli.py +++ b/src/buildstream/_frontend/cli.py @@ -664,6 +664,90 @@ def source(): """Manipulate sources for an element""" +@source.command(name="push", short_help="Push an element's sources") +@click.option('--deps', '-d', default='none', + type=click.Choice(['none', 'all']), + help='The dependencies to push (default: none)') +@click.option('--remote', '-r', default=None, + help="The URL of the remote cache (defaults to the first configured cache)") +@click.argument('elements', nargs=-1, + type=click.Path(readable=False)) +@click.pass_obj +def source_push(app, elements, deps, remote): + """Push an element's sources to a remote source cache. + + Specifying no elements will result in pushing the default targets + of the project. If no default targets are configured, all project + elements will be pushed. + + When this command is executed from a workspace directory, the default + is to push the workspace element. + + The default destination is the highest priority configured cache. You can + override this by passing a different cache URL with the `--remote` flag. + + Specify `--deps` to control which sources to push: + + \b + none: No dependencies, just the element itself + all: All dependencies + """ + with app.initialized(session_name="Push"): + ignore_junction_targets = False + + if not elements: + elements = app.project.get_default_targets() + # Junction elements cannot be pushed, exclude them from default targets + ignore_junction_targets = True + + app.stream.push_sources(elements, selection=deps, remote=remote, + ignore_junction_targets=ignore_junction_targets) + + +@source.command(name="pull", short_help="Pull an element's sources") +@click.option('--deps', '-d', default='none', + type=click.Choice(['none', 'all']), + help='The dependency sources to pull (default: none)') +@click.option('--remote', '-r', default=None, + help="The URL of the remote cache (defaults to the first configured cache)") +@click.argument('elements', nargs=-1, + type=click.Path(readable=False)) +@click.pass_obj +def source_pull(app, elements, deps, remote): + """Pull sources from the configured remote source cache. + + Specifying no elements will result in pulling the default targets + of the project. If no default targets are configured, all project + elements will be pulled. + + When this command is executed from a workspace directory, the default + is to pull the workspace element. + + By default the sources will be pulled from one of the configured + caches if possible, following the usual priority order. If the + `--remote` flag is given, only the specified cache will be + queried. + + Specify `--deps` to control which sources to pull: + + \b + none: No dependencies, just the element itself + all: All dependencies + + """ + + with app.initialized(session_name="Pull"): + ignore_junction_targets = False + + if not elements: + elements = app.project.get_default_targets() + # Junction elements cannot be pulled, exclude them from default targets + ignore_junction_targets = True + + app.stream.pull_sources(elements, selection=deps, remote=remote, + ignore_junction_targets=ignore_junction_targets) + + ################################################################## # Source Fetch Command # ################################################################## diff --git a/src/buildstream/_scheduler/__init__.py b/src/buildstream/_scheduler/__init__.py index d2f458fa5..123f3299b 100644 --- a/src/buildstream/_scheduler/__init__.py +++ b/src/buildstream/_scheduler/__init__.py @@ -21,6 +21,7 @@ from .queues import Queue, QueueStatus from .queues.fetchqueue import FetchQueue from .queues.sourcepushqueue import SourcePushQueue +from .queues.sourcepullqueue import SourcePullQueue from .queues.trackqueue import TrackQueue from .queues.buildqueue import BuildQueue from .queues.artifactpushqueue import ArtifactPushQueue diff --git a/src/buildstream/_scheduler/queues/sourcepullqueue.py b/src/buildstream/_scheduler/queues/sourcepullqueue.py new file mode 100644 index 000000000..255edf443 --- /dev/null +++ b/src/buildstream/_scheduler/queues/sourcepullqueue.py @@ -0,0 +1,43 @@ +# +# Copyright (C) 2019 Bloomberg Finance LP +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see <http://www.gnu.org/licenses/>. +# + +from . import Queue, QueueStatus +from ..resources import ResourceType +from ..._exceptions import SkipJob + + +# A queue which pushes staged sources +# +class SourcePullQueue(Queue): + + action_name = "Src-pull" + complete_name = "Sources pulled" + resources = [ResourceType.DOWNLOAD, ResourceType.CACHE] + + def get_process_func(self): + return SourcePullQueue._pull_or_skip + + def status(self, element): + if element._skip_source_pull(): + return QueueStatus.SKIP + + return QueueStatus.READY + + @staticmethod + def _pull_or_skip(element): + if not element._source_pull(): + raise SkipJob(SourcePullQueue.action_name) diff --git a/src/buildstream/_stream.py b/src/buildstream/_stream.py index 453670ad1..9f3792520 100644 --- a/src/buildstream/_stream.py +++ b/src/buildstream/_stream.py @@ -36,7 +36,7 @@ from ._artifactelement import verify_artifact_ref, ArtifactElement from ._exceptions import StreamError, ImplError, BstError, ArtifactElementError, ArtifactError from ._message import Message, MessageType from ._scheduler import Scheduler, SchedStatus, TrackQueue, FetchQueue, \ - SourcePushQueue, BuildQueue, PullQueue, ArtifactPushQueue + SourcePushQueue, SourcePullQueue, BuildQueue, PullQueue, ArtifactPushQueue from ._pipeline import Pipeline, PipelineSelection from ._profile import Topics, PROFILER from ._state import State @@ -311,6 +311,52 @@ class Stream(): self._enqueue_plan(elements) self._run() + def push_sources(self, targets, *, + selection=PipelineSelection.NONE, + ignore_junction_targets=False, + remote=None): + use_config = True + if remote: + use_config = False + + elements, _ = self._load(targets, (), + selection=selection, + ignore_junction_targets=ignore_junction_targets, + source_remote_url=remote, + use_source_config=use_config) + + if not self._sourcecache.has_push_remotes(): + raise StreamError("No source caches available for pushing sources") + + self._scheduler.clear_queues() + push_queue = SourcePushQueue(self._scheduler) + self._add_queue(push_queue) + self._enqueue_plan(elements, queue=push_queue) + self._run() + + def pull_sources(self, targets, *, + selection=PipelineSelection.NONE, + ignore_junction_targets=False, + remote=None): + use_config = True + if remote: + use_config = False + + elements, _ = self._load(targets, (), + selection=selection, + ignore_junction_targets=ignore_junction_targets, + source_remote_url=remote, + use_source_config=use_config) + + if not self._sourcecache.has_fetch_remotes(): + raise StreamError("No source caches available for pulling sources") + + self._scheduler.clear_queues() + pull_queue = SourcePullQueue(self._scheduler) + self._add_queue(push_queue) + self._enqueue_plan(elements, queue=push_queue) + self._run() + # fetch() # # Fetches sources on the pipeline. diff --git a/src/buildstream/element.py b/src/buildstream/element.py index ca573bee1..0bac84617 100644 --- a/src/buildstream/element.py +++ b/src/buildstream/element.py @@ -1836,6 +1836,22 @@ class Element(Plugin): # Notify successfull download return True + def _skip_source_pull(self): + if not self.__sources or self._get_workspace(): + return True + return self._source_cached() or not self.__sourcecache.has_fetch_remotes(plugin=self) + + def _source_pull(self): + if self.__sourcecache.has_fetch_remotes(plugin=self) and not self._source_cached(): + for source in self.sources(): + if self.__sourcecache.pull(source): + # Once we find a cache with the source and manage + # to pull from it, we're done + return True + + # If no caches gave us the source, this failed + return False + def _skip_source_push(self): if not self.__sources or self._get_workspace(): return True diff --git a/tests/frontend/completions.py b/tests/frontend/completions.py index e9fa25b73..c0ea47385 100644 --- a/tests/frontend/completions.py +++ b/tests/frontend/completions.py @@ -56,6 +56,8 @@ MAIN_OPTIONS = [ SOURCE_COMMANDS = [ 'checkout ', 'fetch ', + 'pull ', + 'push ', 'track ', ] |