Stop updating state in queue.status()

Statuses of an element can be changed when:

1) It is pulled
2) It is fetched
3) It is workspaced and it finished building
4) One of its dependencies is tracked
5) One of its dependencies is workspaced and finished building

We can therefore update the statuses at those moments and we don't
need to check all the time.

This reduces considerably the calls to update_states that are done

4 files changed, 21 insertions, 12 deletions

diff --git a/buildstream/_scheduler/queues/buildqueue.py b/buildstream/_scheduler/queues/buildqueue.py
index 60ec19ff4..aa489f381 100644
--- a/buildstream/_scheduler/queues/buildqueue.py
+++ b/buildstream/_scheduler/queues/buildqueue.py
@@ -70,9 +70,6 @@ class BuildQueue(Queue):
         return element._assemble()
 
     def status(self, element):
-        # state of dependencies may have changed, recalculate element state
-        element._update_state()
-
         if not element._is_required():
             # Artifact is not currently required but it may be requested later.
             # Keep it in the queue.
diff --git a/buildstream/_scheduler/queues/fetchqueue.py b/buildstream/_scheduler/queues/fetchqueue.py
index db5e470f9..5295d4d21 100644
--- a/buildstream/_scheduler/queues/fetchqueue.py
+++ b/buildstream/_scheduler/queues/fetchqueue.py
@@ -45,9 +45,6 @@ class FetchQueue(Queue):
         element._fetch(fetch_original=self._fetch_original)
 
     def status(self, element):
-        # state of dependencies may have changed, recalculate element state
-        element._update_state()
-
         if not element._is_required():
             # Artifact is not currently required but it may be requested later.
             # Keep it in the queue.
diff --git a/buildstream/_scheduler/queues/pullqueue.py b/buildstream/_scheduler/queues/pullqueue.py
index dbeb806e5..013ee6489 100644
--- a/buildstream/_scheduler/queues/pullqueue.py
+++ b/buildstream/_scheduler/queues/pullqueue.py
@@ -39,9 +39,6 @@ class PullQueue(Queue):
             raise SkipJob(self.action_name)
 
     def status(self, element):
-        # state of dependencies may have changed, recalculate element state
-        element._update_state()
-
         if not element._is_required():
             # Artifact is not currently required but it may be requested later.
             # Keep it in the queue.
diff --git a/buildstream/element.py b/buildstream/element.py
index e5bc0792e..250e4ae87 100644
--- a/buildstream/element.py
+++ b/buildstream/element.py
@@ -204,6 +204,7 @@ class Element(Plugin):
 
         self.__runtime_dependencies = []        # Direct runtime dependency Elements
         self.__build_dependencies = []          # Direct build dependency Elements
+        self.__reverse_dependencies = []  # Direct reverse dependency Elements
         self.__sources = []                     # List of Sources
         self.__weak_cache_key = None            # Our cached weak cache key
         self.__strict_cache_key = None          # Our cached cache key for strict builds
@@ -979,6 +980,9 @@ class Element(Plugin):
             dependency = Element._new_from_meta(meta_dep)
             element.__build_dependencies.append(dependency)
 
+        for build_dep in element.dependencies(scope=Scope.BUILD):
+            build_dep.__reverse_dependencies.append(element)
+
         return element
 
     # _clear_meta_elements_cache()
@@ -1373,7 +1377,7 @@ class Element(Plugin):
             source = sources.pop()
             source._generate_key(sources)
 
-        self._update_state()
+        self.__update_state_recursively()
 
     # _track():
     #
@@ -1580,7 +1584,7 @@ class Element(Plugin):
         self.__assemble_scheduled = False
         self.__assemble_done = True
 
-        self._update_state()
+        self.__update_state_recursively()
 
         if self._get_workspace() and self._cached_success():
             assert utils._is_main_process(), \
@@ -1807,7 +1811,7 @@ class Element(Plugin):
     def _pull_done(self):
         self.__pull_done = True
 
-        self._update_state()
+        self.__update_state_recursively()
 
     # _pull():
     #
@@ -2915,6 +2919,20 @@ class Element(Plugin):
             if not sourcecache.contains(sources[-1]):
                 sources[-1]._cache(sources[:-1])
 
+    # __update_state_recursively()
+    #
+    # Update the state of all reverse dependencies, recursively.
+    #
+    def __update_state_recursively(self):
+        old_cache_key = self.__cache_key
+        old_weak_cache_key = self.__weak_cache_key
+
+        self._update_state()
+
+        if self.__cache_key != old_cache_key or old_weak_cache_key != self.__weak_cache_key:
+            for rdep in self.__reverse_dependencies:
+                rdep.__update_state_recursively()
+
 
 def _overlap_error_detail(f, forbidden_overlap_elements, elements):
     if forbidden_overlap_elements: