source.py: Remove the reliance on consistency to get whether a source is cached

This removes the need to use consistency in Sources, by asking
explicitely whether the source is cached or not.

This introduces a new public method on source: `is_cached` that needs
implementation and that should return whether the source has a local
copy or not.

- On fetch, also reset whether the source was cached or set if as
  cached when we know it was.

- Validate the cache's source after fetching it

  This doesn't need to be run in the scheduler's process and can be
  offloaded to the child, which will allow better multiprocessing

6 files changed, 19 insertions, 0 deletions

diff --git a/src/buildstream/plugins/sources/_downloadablefilesource.py b/src/buildstream/plugins/sources/_downloadablefilesource.py
index 4e43ee3e3..581d32e7d 100644
--- a/src/buildstream/plugins/sources/_downloadablefilesource.py
+++ b/src/buildstream/plugins/sources/_downloadablefilesource.py
@@ -88,6 +88,9 @@ class DownloadableFileSource(Source):
     def get_unique_key(self):
         return [self.original_url, self.ref]
 
+    def is_cached(self) -> bool:
+        return os.path.isfile(self._get_mirror_file())
+
     def get_consistency(self):
         if self.ref is None:
             return Consistency.INCONSISTENT
diff --git a/src/buildstream/plugins/sources/bzr.py b/src/buildstream/plugins/sources/bzr.py
index 30ce55585..f9d95fca2 100644
--- a/src/buildstream/plugins/sources/bzr.py
+++ b/src/buildstream/plugins/sources/bzr.py
@@ -81,6 +81,10 @@ class BzrSource(Source):
     def get_unique_key(self):
         return [self.original_url, self.tracking, self.ref]
 
+    def is_cached(self):
+        with self._locked():
+            return self._check_ref()
+
     def get_consistency(self):
         if self.ref is None or self.tracking is None:
             return Consistency.INCONSISTENT
diff --git a/src/buildstream/plugins/sources/local.py b/src/buildstream/plugins/sources/local.py
index f4a7270e6..206dcb8c1 100644
--- a/src/buildstream/plugins/sources/local.py
+++ b/src/buildstream/plugins/sources/local.py
@@ -67,6 +67,9 @@ class LocalSource(Source):
     def is_resolved(self):
         return True
 
+    def is_cached(self):
+        return True
+
     # We dont have a ref, we're a local file...
     def load_ref(self, node):
         pass
diff --git a/src/buildstream/plugins/sources/patch.py b/src/buildstream/plugins/sources/patch.py
index cc3e6e2c4..cf6ce99cc 100644
--- a/src/buildstream/plugins/sources/patch.py
+++ b/src/buildstream/plugins/sources/patch.py
@@ -70,6 +70,9 @@ class PatchSource(Source):
     def is_resolved(self):
         return True
 
+    def is_cached(self):
+        return True
+
     def get_consistency(self):
         return Consistency.CACHED
 
diff --git a/src/buildstream/plugins/sources/pip.py b/src/buildstream/plugins/sources/pip.py
index 2c9773787..d45ef70c7 100644
--- a/src/buildstream/plugins/sources/pip.py
+++ b/src/buildstream/plugins/sources/pip.py
@@ -136,6 +136,9 @@ class PipSource(Source):
     def get_unique_key(self):
         return [self.original_url, self.ref]
 
+    def is_cached(self):
+        return os.path.exists(self._mirror) and os.listdir(self._mirror)
+
     def get_consistency(self):
         if not self.ref:
             return Consistency.INCONSISTENT
diff --git a/src/buildstream/plugins/sources/workspace.py b/src/buildstream/plugins/sources/workspace.py
index 88c32f754..5cea85dc2 100644
--- a/src/buildstream/plugins/sources/workspace.py
+++ b/src/buildstream/plugins/sources/workspace.py
@@ -69,6 +69,9 @@ class WorkspaceSource(Source):
     def preflight(self) -> None:
         pass  # pragma: nocover
 
+    def is_cached(self):
+        return True
+
     def is_resolved(self):
         return os.path.exists(self._get_local_path())