source.py: Add a new 'is_resolved' to get whether a source is resolved.

`get_consistency` is coarse grained and hard to optimize, in addition
to being un-userfriendly.

This adds a new `is_resolved` that has for default implementation
`get_ref() is not None`, which is true for most sources in BuildStream.

Sources for which this is not true can override the method to give a
more accurate description.

Checking for this before looking whether the source is cached can
reduce the amount of work necessary in some pipeline and opens the
door for more optimizations and the removal of the source state check.

12 files changed, 55 insertions, 16 deletions

diff --git a/src/buildstream/_gitsourcebase.py b/src/buildstream/_gitsourcebase.py
index 5b7f051b9..ce2ef9731 100644
--- a/src/buildstream/_gitsourcebase.py
+++ b/src/buildstream/_gitsourcebase.py
@@ -524,6 +524,9 @@ class _GitSourceBase(Source):
             return Consistency.RESOLVED
         return Consistency.INCONSISTENT
 
+    def is_resolved(self):
+        return self.mirror.ref is not None
+
     def load_ref(self, node):
         self.mirror.ref = node.get_str("ref", None)
         self.mirror.tags = self._load_tags(node)
diff --git a/src/buildstream/_pipeline.py b/src/buildstream/_pipeline.py
index 1a98fccf5..8de97bea6 100644
--- a/src/buildstream/_pipeline.py
+++ b/src/buildstream/_pipeline.py
@@ -351,7 +351,7 @@ class Pipeline:
             for element in inconsistent:
                 detail += "  Element: {} is inconsistent\n".format(element._get_full_name())
                 for source in element.sources():
-                    if not source._is_resolved():
+                    if not source.is_resolved():
                         detail += "    {} is missing ref\n".format(source)
                 detail += "\n"
             detail += "Try tracking these elements first with `bst source track`\n"
diff --git a/src/buildstream/element.py b/src/buildstream/element.py
index fa94a08a1..f28fdf789 100644
--- a/src/buildstream/element.py
+++ b/src/buildstream/element.py
@@ -2362,15 +2362,16 @@ class Element(Plugin):
 
         # Determine overall consistency of the element
         for source in self.__sources:
-            # FIXME: It'd be nice to remove this eventually
-            source._update_state()
-
-            if source._is_cached():
-                self.__consistency = min(self.__consistency, Consistency.CACHED)
-            elif source._is_resolved():
-                self.__consistency = min(self.__consistency, Consistency.RESOLVED)
-            else:
+            if not source.is_resolved():
                 self.__consistency = Consistency.INCONSISTENT
+            else:
+                # FIXME: It'd be nice to remove this eventually
+                source._update_state()
+
+                if source._is_cached():
+                    self.__consistency = min(self.__consistency, Consistency.CACHED)
+                else:
+                    self.__consistency = min(self.__consistency, Consistency.RESOLVED)
 
         # If the source state changes, our cache key must also change,
         # since it contains the source's key.
diff --git a/src/buildstream/plugins/sources/local.py b/src/buildstream/plugins/sources/local.py
index 90d8a8f6f..f4a7270e6 100644
--- a/src/buildstream/plugins/sources/local.py
+++ b/src/buildstream/plugins/sources/local.py
@@ -64,6 +64,9 @@ class LocalSource(Source):
     def get_consistency(self):
         return Consistency.CACHED
 
+    def is_resolved(self):
+        return True
+
     # We dont have a ref, we're a local file...
     def load_ref(self, node):
         pass
diff --git a/src/buildstream/plugins/sources/patch.py b/src/buildstream/plugins/sources/patch.py
index 082983023..cc3e6e2c4 100644
--- a/src/buildstream/plugins/sources/patch.py
+++ b/src/buildstream/plugins/sources/patch.py
@@ -67,6 +67,9 @@ class PatchSource(Source):
     def get_unique_key(self):
         return [self.path, utils.sha256sum(self.fullpath), self.strip_level]
 
+    def is_resolved(self):
+        return True
+
     def get_consistency(self):
         return Consistency.CACHED
 
diff --git a/src/buildstream/plugins/sources/workspace.py b/src/buildstream/plugins/sources/workspace.py
index 3d4c93b5c..88c32f754 100644
--- a/src/buildstream/plugins/sources/workspace.py
+++ b/src/buildstream/plugins/sources/workspace.py
@@ -69,6 +69,9 @@ class WorkspaceSource(Source):
     def preflight(self) -> None:
         pass  # pragma: nocover
 
+    def is_resolved(self):
+        return os.path.exists(self._get_local_path())
+
     def get_ref(self) -> None:
         return None
 
diff --git a/src/buildstream/source.py b/src/buildstream/source.py
index 4f1133de7..46922b685 100644
--- a/src/buildstream/source.py
+++ b/src/buildstream/source.py
@@ -705,6 +705,22 @@ class Source(Plugin):
         with utils._tempdir(dir=mirrordir) as tempdir:
             yield tempdir
 
+    def is_resolved(self) -> bool:
+        """Get whether the source is resolved.
+
+        This has a default implementation that checks whether the source
+        has a ref or not. If it has a ref, it is assumed to be resolved.
+
+        Sources that never have a ref or have uncommon requirements can
+        override this method to specify when they should be considered
+        resolved
+
+        Returns: whether the source is fully resolved or not
+
+        *Since: 1.91.4*
+        """
+        return self.get_ref() is not None
+
     #############################################################
     #       Private Abstract Methods used in BuildStream        #
     #############################################################
@@ -783,11 +799,6 @@ class Source(Plugin):
                 if self.__consistency == Consistency.CACHED:
                     self.validate_cache()
 
-    # Get whether the source is consistent
-    #
-    def _is_resolved(self):
-        return self.__consistency >= Consistency.RESOLVED
-
     # Get whether the source is cached by the source plugin
     #
     def _is_cached(self):
@@ -1416,7 +1427,7 @@ class Source(Plugin):
         for index, src in enumerate(previous_sources):
             # BuildStream should track sources in the order they appear so
             # previous sources should never be in an inconsistent state
-            assert src.get_consistency() != Consistency.INCONSISTENT
+            assert src.is_resolved()
 
             if src.get_consistency() == Consistency.RESOLVED:
                 src._fetch(previous_sources[0:index])
diff --git a/tests/frontend/consistencyerror/plugins/consistencybug.py b/tests/frontend/consistencyerror/plugins/consistencybug.py
index c442d883a..c4a3217ec 100644
--- a/tests/frontend/consistencyerror/plugins/consistencybug.py
+++ b/tests/frontend/consistencyerror/plugins/consistencybug.py
@@ -11,6 +11,9 @@ class ConsistencyBugSource(Source):
     def get_unique_key(self):
         return {}
 
+    def is_resolved(self):
+        return True
+
     def get_consistency(self):
 
         # Raise an unhandled exception (not a BstError)
diff --git a/tests/frontend/consistencyerror/plugins/consistencyerror.py b/tests/frontend/consistencyerror/plugins/consistencyerror.py
index 125baf39c..d7433e368 100644
--- a/tests/frontend/consistencyerror/plugins/consistencyerror.py
+++ b/tests/frontend/consistencyerror/plugins/consistencyerror.py
@@ -11,6 +11,9 @@ class ConsistencyErrorSource(Source):
     def get_unique_key(self):
         return {}
 
+    def is_resolved(self):
+        return True
+
     def get_consistency(self):
 
         # Raise an error unconditionally
diff --git a/tests/frontend/project/sources/fetch_source.py b/tests/frontend/project/sources/fetch_source.py
index 51bfe1049..d634adfa3 100644
--- a/tests/frontend/project/sources/fetch_source.py
+++ b/tests/frontend/project/sources/fetch_source.py
@@ -66,6 +66,9 @@ class FetchSource(Source):
     def get_unique_key(self):
         return {"urls": self.original_urls, "output_file": self.output_file}
 
+    def is_resolved(self):
+        return True
+
     def get_consistency(self):
         if not os.path.exists(self.output_file):
             return Consistency.RESOLVED
diff --git a/tests/sources/no-fetch-cached/plugins/sources/always_cached.py b/tests/sources/no-fetch-cached/plugins/sources/always_cached.py
index 623ab19ab..5f05b592b 100644
--- a/tests/sources/no-fetch-cached/plugins/sources/always_cached.py
+++ b/tests/sources/no-fetch-cached/plugins/sources/always_cached.py
@@ -20,6 +20,9 @@ class AlwaysCachedSource(Source):
     def get_unique_key(self):
         return None
 
+    def is_resolved(self):
+        return True
+
     def get_consistency(self):
         return Consistency.CACHED
 
diff --git a/tests/sources/project_key_test/plugins/sources/key-test.py b/tests/sources/project_key_test/plugins/sources/key-test.py
index 428846a3e..30256929c 100644
--- a/tests/sources/project_key_test/plugins/sources/key-test.py
+++ b/tests/sources/project_key_test/plugins/sources/key-test.py
@@ -11,7 +11,10 @@ class KeyTest(Source):
         pass
 
     def configure(self, node):
-        self.ref = node.get_bool("ref", False)
+        if node.get_scalar("ref", None).is_none():
+            self.ref = None
+        else:
+            self.ref = True
 
     def get_unique_key(self):
         assert self.ref