Remove newly unused API surfaces in CASCache

This also involves a number of changes to tests and other parts of the
codebase since they were hacking about wit API that shouldn't have
existed.

7 files changed, 84 insertions, 179 deletions

diff --git a/src/buildstream/_artifactcache.py b/src/buildstream/_artifactcache.py
index d9112cd58..91225eab0 100644
--- a/src/buildstream/_artifactcache.py
+++ b/src/buildstream/_artifactcache.py
@@ -21,7 +21,7 @@ import os
 import grpc
 
 from ._basecache import BaseCache
-from ._exceptions import ArtifactError, CASError, CASCacheError, CASRemoteError
+from ._exceptions import ArtifactError, CASError, CacheError, CASRemoteError
 from ._protos.buildstream.v2 import buildstream_pb2, buildstream_pb2_grpc, \
     artifact_pb2, artifact_pb2_grpc
 
@@ -203,8 +203,8 @@ class ArtifactCache(BaseCache):
     #
     def remove(self, ref):
         try:
-            self.cas.remove(ref, basedir=self.artifactdir)
-        except CASCacheError as e:
+            self._remove_ref(ref, self.artifactdir)
+        except CacheError as e:
             raise ArtifactError("{}".format(e)) from e
 
     # diff():
diff --git a/src/buildstream/_basecache.py b/src/buildstream/_basecache.py
index fc2e92456..69da0f9df 100644
--- a/src/buildstream/_basecache.py
+++ b/src/buildstream/_basecache.py
@@ -17,6 +17,7 @@
 #        Raoul Hidalgo Charman <raoul.hidalgocharman@codethink.co.uk>
 #
 import os
+import errno
 from fnmatch import fnmatch
 from itertools import chain
 from typing import TYPE_CHECKING
@@ -25,7 +26,7 @@ from . import utils
 from . import _yaml
 from ._cas import CASRemote
 from ._message import Message, MessageType
-from ._exceptions import LoadError, RemoteError
+from ._exceptions import LoadError, RemoteError, CacheError
 from ._remote import RemoteSpec, RemoteType
 
 
@@ -425,3 +426,54 @@ class BaseCache():
                 if not glob_expr or fnmatch(relative_path, glob_expr):
                     # Obtain the mtime (the time a file was last modified)
                     yield (os.path.getmtime(ref_path), relative_path)
+
+    # _remove_ref()
+    #
+    # Removes a ref.
+    #
+    # This also takes care of pruning away directories which can
+    # be removed after having removed the given ref.
+    #
+    # Args:
+    #    ref (str): The ref to remove
+    #    basedir (str): Path of base directory the ref is in
+    #
+    # Raises:
+    #    (CASCacheError): If the ref didnt exist, or a system error
+    #                     occurred while removing it
+    #
+    def _remove_ref(self, ref, basedir):
+
+        # Remove the ref itself
+        refpath = os.path.join(basedir, ref)
+
+        try:
+            os.unlink(refpath)
+        except FileNotFoundError as e:
+            raise CacheError("Could not find ref '{}'".format(ref)) from e
+
+        # Now remove any leading directories
+
+        components = list(os.path.split(ref))
+        while components:
+            components.pop()
+            refdir = os.path.join(basedir, *components)
+
+            # Break out once we reach the base
+            if refdir == basedir:
+                break
+
+            try:
+                os.rmdir(refdir)
+            except FileNotFoundError:
+                # The parent directory did not exist, but it's
+                # parent directory might still be ready to prune
+                pass
+            except OSError as e:
+                if e.errno == errno.ENOTEMPTY:
+                    # The parent directory was not empty, so we
+                    # cannot prune directories beyond this point
+                    break
+
+                # Something went wrong here
+                raise CacheError("System error while removing ref '{}': {}".format(ref, e)) from e
diff --git a/src/buildstream/_cas/cascache.py b/src/buildstream/_cas/cascache.py
index 022730445..11f15bdd3 100644
--- a/src/buildstream/_cas/cascache.py
+++ b/src/buildstream/_cas/cascache.py
@@ -21,7 +21,6 @@
 import itertools
 import os
 import stat
-import errno
 import contextlib
 import ctypes
 import multiprocessing
@@ -75,7 +74,6 @@ class CASCache():
     ):
         self.casdir = os.path.join(path, 'cas')
         self.tmpdir = os.path.join(path, 'tmp')
-        os.makedirs(os.path.join(self.casdir, 'refs', 'heads'), exist_ok=True)
         os.makedirs(os.path.join(self.casdir, 'objects'), exist_ok=True)
         os.makedirs(self.tmpdir, exist_ok=True)
 
@@ -179,9 +177,7 @@ class CASCache():
     # Preflight check.
     #
     def preflight(self):
-        headdir = os.path.join(self.casdir, 'refs', 'heads')
-        objdir = os.path.join(self.casdir, 'objects')
-        if not (os.path.isdir(headdir) and os.path.isdir(objdir)):
+        if not os.path.join(self.casdir, 'objects'):
             raise CASCacheError("CAS repository check failed for '{}'".format(self.casdir))
 
     # has_open_grpc_channels():
@@ -216,21 +212,6 @@ class CASCache():
             self._terminate_casd_process(messenger)
             shutil.rmtree(self._casd_socket_tempdir)
 
-    # contains():
-    #
-    # Check whether the specified ref is already available in the local CAS cache.
-    #
-    # Args:
-    #     ref (str): The ref to check
-    #
-    # Returns: True if the ref is in the cache, False otherwise
-    #
-    def contains(self, ref):
-        refpath = self._refpath(ref)
-
-        # This assumes that the repository doesn't have any dangling pointers
-        return os.path.exists(refpath)
-
     # contains_file():
     #
     # Check whether a digest corresponds to a file which exists in CAS
@@ -309,28 +290,6 @@ class CASCache():
             fullpath = os.path.join(dest, symlinknode.name)
             os.symlink(symlinknode.target, fullpath)
 
-    # diff():
-    #
-    # Return a list of files that have been added or modified between
-    # the refs described by ref_a and ref_b.
-    #
-    # Args:
-    #     ref_a (str): The first ref
-    #     ref_b (str): The second ref
-    #     subdir (str): A subdirectory to limit the comparison to
-    #
-    def diff(self, ref_a, ref_b):
-        tree_a = self.resolve_ref(ref_a)
-        tree_b = self.resolve_ref(ref_b)
-
-        added = []
-        removed = []
-        modified = []
-
-        self.diff_trees(tree_a, tree_b, added=added, removed=removed, modified=modified)
-
-        return modified, removed, added
-
     # pull_tree():
     #
     # Pull a single Tree rather than a ref.
@@ -457,74 +416,6 @@ class CASCache():
 
         return utils._message_digest(root_directory)
 
-    # set_ref():
-    #
-    # Create or replace a ref.
-    #
-    # Args:
-    #     ref (str): The name of the ref
-    #
-    def set_ref(self, ref, tree):
-        refpath = self._refpath(ref)
-        os.makedirs(os.path.dirname(refpath), exist_ok=True)
-        with utils.save_file_atomic(refpath, 'wb', tempdir=self.tmpdir) as f:
-            f.write(tree.SerializeToString())
-
-    # resolve_ref():
-    #
-    # Resolve a ref to a digest.
-    #
-    # Args:
-    #     ref (str): The name of the ref
-    #     update_mtime (bool): Whether to update the mtime of the ref
-    #
-    # Returns:
-    #     (Digest): The digest stored in the ref
-    #
-    def resolve_ref(self, ref, *, update_mtime=False):
-        refpath = self._refpath(ref)
-
-        try:
-            with open(refpath, 'rb') as f:
-                if update_mtime:
-                    os.utime(refpath)
-
-                digest = remote_execution_pb2.Digest()
-                digest.ParseFromString(f.read())
-                return digest
-
-        except FileNotFoundError as e:
-            raise CASCacheError("Attempt to access unavailable ref: {}".format(e)) from e
-
-    # update_mtime()
-    #
-    # Update the mtime of a ref.
-    #
-    # Args:
-    #     ref (str): The ref to update
-    #
-    def update_mtime(self, ref):
-        try:
-            os.utime(self._refpath(ref))
-        except FileNotFoundError as e:
-            raise CASCacheError("Attempt to access unavailable ref: {}".format(e)) from e
-
-    # remove():
-    #
-    # Removes the given symbolic ref from the repo.
-    #
-    # Args:
-    #    ref (str): A symbolic ref
-    #    basedir (str): Path of base directory the ref is in, defaults to
-    #                   CAS refs heads
-    #
-    def remove(self, ref, *, basedir=None):
-
-        if basedir is None:
-            basedir = os.path.join(self.casdir, 'refs', 'heads')
-        # Remove cache ref
-        self._remove_ref(ref, basedir)
-
     def update_tree_mtime(self, tree):
         reachable = set()
         self._reachable_refs_dir(reachable, tree, update_mtime=True)
@@ -702,60 +593,6 @@ class CASCache():
 
         return os.path.join(log_dir, str(self._casd_start_time) + ".log")
 
-    def _refpath(self, ref):
-        return os.path.join(self.casdir, 'refs', 'heads', ref)
-
-    # _remove_ref()
-    #
-    # Removes a ref.
-    #
-    # This also takes care of pruning away directories which can
-    # be removed after having removed the given ref.
-    #
-    # Args:
-    #    ref (str): The ref to remove
-    #    basedir (str): Path of base directory the ref is in
-    #
-    # Raises:
-    #    (CASCacheError): If the ref didnt exist, or a system error
-    #                     occurred while removing it
-    #
-    def _remove_ref(self, ref, basedir):
-
-        # Remove the ref itself
-        refpath = os.path.join(basedir, ref)
-
-        try:
-            os.unlink(refpath)
-        except FileNotFoundError as e:
-            raise CASCacheError("Could not find ref '{}'".format(ref)) from e
-
-        # Now remove any leading directories
-
-        components = list(os.path.split(ref))
-        while components:
-            components.pop()
-            refdir = os.path.join(basedir, *components)
-
-            # Break out once we reach the base
-            if refdir == basedir:
-                break
-
-            try:
-                os.rmdir(refdir)
-            except FileNotFoundError:
-                # The parent directory did not exist, but it's
-                # parent directory might still be ready to prune
-                pass
-            except OSError as e:
-                if e.errno == errno.ENOTEMPTY:
-                    # The parent directory was not empty, so we
-                    # cannot prune directories beyond this point
-                    break
-
-                # Something went wrong here
-                raise CASCacheError("System error while removing ref '{}': {}".format(ref, e)) from e
-
     def _get_subdir(self, tree, subdir):
         head, name = os.path.split(subdir)
         if head:
diff --git a/src/buildstream/_exceptions.py b/src/buildstream/_exceptions.py
index 947b83149..319518f51 100644
--- a/src/buildstream/_exceptions.py
+++ b/src/buildstream/_exceptions.py
@@ -275,6 +275,15 @@ class SandboxError(BstError):
 
 # SourceCacheError
 #
+# Raised when errors are encountered in either type of cache
+#
+class CacheError(BstError):
+    def __init__(self, message, detail=None, reason=None):
+        super().__init__(message, detail=detail, domain=ErrorDomain.SANDBOX, reason=reason)
+
+
+# SourceCacheError
+#
 # Raised when errors are encountered in the source caches
 #
 class SourceCacheError(BstError):
diff --git a/tests/sourcecache/fetch.py b/tests/sourcecache/fetch.py
index a5863b867..48683d6b9 100644
--- a/tests/sourcecache/fetch.py
+++ b/tests/sourcecache/fetch.py
@@ -92,8 +92,7 @@ def test_source_fetch(cli, tmpdir, datafiles):
             assert not element._source_cached()
             source = list(element.sources())[0]
 
-            cas = context.get_cascache()
-            assert not cas.contains(source._get_source_name())
+            assert not share.get_source_proto(source._get_source_name())
 
             # Just check that we sensibly fetch and build the element
             res = cli.run(project=project_dir, args=['build', element_name])
@@ -139,8 +138,7 @@ def test_fetch_fallback(cli, tmpdir, datafiles):
             assert not element._source_cached()
             source = list(element.sources())[0]
 
-            cas = context.get_cascache()
-            assert not cas.contains(source._get_source_name())
+            assert not share.get_source_proto(source._get_source_name())
             assert not os.path.exists(os.path.join(cache_dir, 'sources'))
 
             # Now check if it falls back to the source fetch method.
@@ -198,8 +196,7 @@ def test_source_pull_partial_fallback_fetch(cli, tmpdir, datafiles):
             assert not element._source_cached()
             source = list(element.sources())[0]
 
-            cas = context.get_cascache()
-            assert not cas.contains(source._get_source_name())
+            assert not share.get_artifact_proto(source._get_source_name())
 
             # Just check that we sensibly fetch and build the element
             res = cli.run(project=project_dir, args=['build', element_name])
diff --git a/tests/sourcecache/push.py b/tests/sourcecache/push.py
index 406aeba9f..8c0ac0644 100644
--- a/tests/sourcecache/push.py
+++ b/tests/sourcecache/push.py
@@ -98,8 +98,7 @@ def test_source_push_split(cli, tmpdir, datafiles):
             source = list(element.sources())[0]
 
             # check we don't have it in the current cache
-            cas = context.get_cascache()
-            assert not cas.contains(source._get_source_name())
+            assert not index.get_source_proto(source._get_source_name())
 
             # build the element, this should fetch and then push the source to the
             # remote
@@ -156,8 +155,7 @@ def test_source_push(cli, tmpdir, datafiles):
             source = list(element.sources())[0]
 
             # check we don't have it in the current cache
-            cas = context.get_cascache()
-            assert not cas.contains(source._get_source_name())
+            assert not share.get_source_proto(source._get_source_name())
 
             # build the element, this should fetch and then push the source to the
             # remote
diff --git a/tests/testutils/artifactshare.py b/tests/testutils/artifactshare.py
index d86cafa3c..ba02d397d 100644
--- a/tests/testutils/artifactshare.py
+++ b/tests/testutils/artifactshare.py
@@ -11,7 +11,7 @@ from buildstream._cas import CASCache
 from buildstream._cas.casserver import create_server
 from buildstream._exceptions import CASError
 from buildstream._protos.build.bazel.remote.execution.v2 import remote_execution_pb2
-from buildstream._protos.buildstream.v2 import artifact_pb2
+from buildstream._protos.buildstream.v2 import artifact_pb2, source_pb2
 
 
 # ArtifactShare()
@@ -42,6 +42,7 @@ class ArtifactShare():
 
         self.cas = CASCache(self.repodir, casd=casd)
         self.artifactdir = os.path.join(self.repodir, 'artifacts', 'refs')
+        self.sourcedir = os.path.join(self.repodir, 'source_protos', 'refs')
 
         self.quota = quota
         self.index_only = index_only
@@ -127,6 +128,17 @@ class ArtifactShare():
 
         return artifact_proto
 
+    def get_source_proto(self, source_name):
+        source_proto = source_pb2.Source()
+        source_path = os.path.join(self.sourcedir, source_name)
+
+        try:
+            with open(source_path, 'rb') as f:
+                source_proto.ParseFromString(f.read())
+        except FileNotFoundError:
+            return None
+        return source_proto
+
     def get_cas_files(self, artifact_proto):
 
         reachable = set()