From 270da0609978ce6d269865b6bd451ec560a983ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BCrg=20Billeter?= Date: Thu, 10 May 2018 11:23:01 +0200 Subject: Do not pull/fetch/build elements that are not required For `bst build --deps plan`, do not process elements in pull/fetch/build queues until they are requested by a reverse dependency. --- buildstream/_scheduler/buildqueue.py | 5 +++++ buildstream/_scheduler/fetchqueue.py | 5 +++++ buildstream/_scheduler/pullqueue.py | 5 +++++ buildstream/_stream.py | 16 ++++++++++++++-- buildstream/element.py | 35 +++++++++++++++++++++++++++++++++-- 5 files changed, 62 insertions(+), 4 deletions(-) diff --git a/buildstream/_scheduler/buildqueue.py b/buildstream/_scheduler/buildqueue.py index 46ce72ca7..24a124b32 100644 --- a/buildstream/_scheduler/buildqueue.py +++ b/buildstream/_scheduler/buildqueue.py @@ -38,6 +38,11 @@ class BuildQueue(Queue): # state of dependencies may have changed, recalculate element state element._update_state() + if not element._is_required(): + # Artifact is not currently required but it may be requested later. + # Keep it in the queue. + return QueueStatus.WAIT + if element._cached(): return QueueStatus.SKIP diff --git a/buildstream/_scheduler/fetchqueue.py b/buildstream/_scheduler/fetchqueue.py index c2bceb270..61055725d 100644 --- a/buildstream/_scheduler/fetchqueue.py +++ b/buildstream/_scheduler/fetchqueue.py @@ -47,6 +47,11 @@ class FetchQueue(Queue): # state of dependencies may have changed, recalculate element state element._update_state() + if not element._is_required(): + # Artifact is not currently required but it may be requested later. + # Keep it in the queue. + return QueueStatus.WAIT + # Optionally skip elements that are already in the artifact cache if self._skip_cached: if not element._can_query_cache(): diff --git a/buildstream/_scheduler/pullqueue.py b/buildstream/_scheduler/pullqueue.py index 0d6b5dbfb..f9928a342 100644 --- a/buildstream/_scheduler/pullqueue.py +++ b/buildstream/_scheduler/pullqueue.py @@ -39,6 +39,11 @@ class PullQueue(Queue): # state of dependencies may have changed, recalculate element state element._update_state() + if not element._is_required(): + # Artifact is not currently required but it may be requested later. + # Keep it in the queue. + return QueueStatus.WAIT + if not element._can_query_cache(): return QueueStatus.WAIT diff --git a/buildstream/_stream.py b/buildstream/_stream.py index 6bde4eb02..ca999ad6e 100644 --- a/buildstream/_stream.py +++ b/buildstream/_stream.py @@ -181,7 +181,8 @@ class Stream(): track_except_targets=track_except, track_cross_junctions=track_cross_junctions, use_artifact_config=True, - fetch_subprojects=True) + fetch_subprojects=True, + dynamic_plan=True) # Remove the tracking elements from the main targets elements = self._pipeline.subtract_elements(elements, track_elements) @@ -755,7 +756,8 @@ class Stream(): track_cross_junctions=False, use_artifact_config=False, artifact_remote_url=None, - fetch_subprojects=False): + fetch_subprojects=False, + dynamic_plan=False): # Load rewritable if we have any tracking selection to make rewritable = False @@ -806,6 +808,16 @@ class Stream(): selected, except_elements) + if selection == PipelineSelection.PLAN and dynamic_plan: + # We use a dynamic build plan, only request artifacts of top-level targets, + # others are requested dynamically as needed. + # This avoids pulling, fetching, or building unneeded build-only dependencies. + for element in elements: + element._set_required() + else: + for element in selected: + element._set_required() + return selected, track_selected # _message() diff --git a/buildstream/element.py b/buildstream/element.py index 29b3bd59a..70bee42e6 100644 --- a/buildstream/element.py +++ b/buildstream/element.py @@ -225,6 +225,7 @@ class Element(Plugin): self.__whitelist_regex = None # Resolved regex object to check if file is allowed to overlap self.__staged_sources_directory = None # Location where Element.stage_sources() was called self.__tainted = None # Whether the artifact is tainted and should not be shared + self.__required = False # Whether the artifact is required in the current session # hash tables of loaded artifact metadata, hashed by key self.__metadata_keys = {} # Strong and weak keys for this key @@ -1069,7 +1070,7 @@ class Element(Plugin): # until the full cache query below. cached = self.__artifacts.contains(self, self.__weak_cache_key) if (not self.__assemble_scheduled and not self.__assemble_done and - not cached and not self._pull_pending()): + not cached and not self._pull_pending() and self._is_required()): self._schedule_assemble() return @@ -1091,7 +1092,7 @@ class Element(Plugin): self.__strong_cached = self.__artifacts.contains(self, self.__strict_cache_key) if (not self.__assemble_scheduled and not self.__assemble_done and - not self.__cached and not self._pull_pending()): + not self.__cached and not self._pull_pending() and self._is_required()): # Workspaced sources are considered unstable if a build is pending # as the build will modify the contents of the workspace. # Determine as early as possible if a build is pending to discard @@ -1322,15 +1323,45 @@ class Element(Plugin): # Ensure deterministic owners of sources at build time utils._set_deterministic_user(directory) + # _set_required(): + # + # Mark this element and its runtime dependencies as required. + # This unblocks pull/fetch/build. + # + def _set_required(self): + if self.__required: + # Already done + return + + self.__required = True + + # Request artifacts of runtime dependencies + for dep in self.dependencies(Scope.RUN, recurse=False): + dep._set_required() + + self._update_state() + + # _is_required(): + # + # Returns whether this element has been marked as required. + # + def _is_required(self): + return self.__required + # _schedule_assemble(): # # This is called in the main process before the element is assembled # in a subprocess. # def _schedule_assemble(self): + assert self._is_required() assert not self.__assemble_scheduled self.__assemble_scheduled = True + # Requests artifacts of build dependencies + for dep in self.dependencies(Scope.BUILD, recurse=False): + dep._set_required() + # Invalidate workspace key as the build modifies the workspace directory workspace = self._get_workspace() if workspace: -- cgit v1.2.1