cascache.py: Start buildbox-casd and set up channel

6 files changed, 122 insertions, 15 deletions

diff --git a/src/buildstream/_cas/cascache.py b/src/buildstream/_cas/cascache.py
index d4c73d464..7bfe7f856 100644
--- a/src/buildstream/_cas/cascache.py
+++ b/src/buildstream/_cas/cascache.py
@@ -24,10 +24,15 @@ import stat
 import errno
 import uuid
 import contextlib
+import shutil
+import subprocess
+import tempfile
+import time
 
 import grpc
 
-from .._protos.build.bazel.remote.execution.v2 import remote_execution_pb2
+from .._protos.build.bazel.remote.execution.v2 import remote_execution_pb2, remote_execution_pb2_grpc
+from .._protos.build.buildgrid import local_cas_pb2_grpc
 from .._protos.buildstream.v2 import buildstream_pb2
 
 from .. import utils
@@ -45,17 +50,74 @@ CACHE_SIZE_FILE = "cache_size"
 #
 # Args:
 #     path (str): The root directory for the CAS repository
+#     casd (bool): True to spawn buildbox-casd (default) or False (testing only)
 #     cache_quota (int): User configured cache quota
 #
 class CASCache():
 
-    def __init__(self, path):
+    def __init__(self, path, *, casd=True):
         self.casdir = os.path.join(path, 'cas')
         self.tmpdir = os.path.join(path, 'tmp')
         os.makedirs(os.path.join(self.casdir, 'refs', 'heads'), exist_ok=True)
         os.makedirs(os.path.join(self.casdir, 'objects'), exist_ok=True)
         os.makedirs(self.tmpdir, exist_ok=True)
 
+        if casd:
+            # Place socket in global/user temporary directory to avoid hitting
+            # the socket path length limit.
+            self._casd_socket_tempdir = tempfile.mkdtemp(prefix='buildstream')
+            self._casd_socket_path = os.path.join(self._casd_socket_tempdir, 'casd.sock')
+
+            casd_args = [utils.get_host_tool('buildbox-casd')]
+            casd_args.append('--bind=unix:' + self._casd_socket_path)
+
+            casd_args.append(path)
+            self._casd_process = subprocess.Popen(casd_args, cwd=path)
+            self._casd_start_time = time.time()
+        else:
+            self._casd_process = None
+
+        self._casd_channel = None
+        self._local_cas = None
+
+    def __getstate__(self):
+        state = self.__dict__.copy()
+
+        # Popen objects are not pickle-able, however, child processes only
+        # need the information whether a casd subprocess was started or not.
+        assert '_casd_process' in state
+        state['_casd_process'] = bool(self._casd_process)
+
+        return state
+
+    def _get_local_cas(self):
+        assert self._casd_process, "CASCache was instantiated without buildbox-casd"
+
+        if not self._local_cas:
+            # gRPC doesn't support fork without exec, which is used in the main process.
+            assert not utils._is_main_process()
+
+            self._casd_channel = grpc.insecure_channel('unix:' + self._casd_socket_path)
+            self._local_cas = local_cas_pb2_grpc.LocalContentAddressableStorageStub(self._casd_channel)
+
+            # Call GetCapabilities() to establish connection to casd
+            capabilities = remote_execution_pb2_grpc.CapabilitiesStub(self._casd_channel)
+            while True:
+                try:
+                    capabilities.GetCapabilities(remote_execution_pb2.GetCapabilitiesRequest())
+                    break
+                except grpc.RpcError as e:
+                    if e.code() == grpc.StatusCode.UNAVAILABLE:
+                        # casd is not ready yet, try again after a 10ms delay,
+                        # but don't wait for more than 15s
+                        if time.time() < self._casd_start_time + 15:
+                            time.sleep(1 / 100)
+                            continue
+
+                    raise
+
+        return self._local_cas
+
     # preflight():
     #
     # Preflight check.
@@ -71,7 +133,25 @@ class CASCache():
     # Release resources used by CASCache.
     #
     def release_resources(self, messenger=None):
-        pass
+        if self._casd_process:
+            self._casd_process.terminate()
+            try:
+                # Don't print anything if buildbox-casd terminates quickly
+                self._casd_process.wait(timeout=0.5)
+            except subprocess.TimeoutExpired:
+                if messenger:
+                    cm = messenger.timed_activity("Terminating buildbox-casd")
+                else:
+                    cm = contextlib.suppress()
+                with cm:
+                    try:
+                        self._casd_process.wait(timeout=15)
+                    except subprocess.TimeoutExpired:
+                        self._casd_process.kill()
+                        self._casd_process.wait(timeout=15)
+            self._casd_process = None
+
+            shutil.rmtree(self._casd_socket_tempdir)
 
     # contains():
     #
diff --git a/src/buildstream/testing/runcli.py b/src/buildstream/testing/runcli.py
index 95bf83eff..6c3ab3496 100644
--- a/src/buildstream/testing/runcli.py
+++ b/src/buildstream/testing/runcli.py
@@ -746,7 +746,7 @@ class TestArtifact():
     def _extract_subdirectory(self, tmpdir, digest):
         with tempfile.TemporaryDirectory() as extractdir:
             try:
-                cas = CASCache(str(tmpdir))
+                cas = CASCache(str(tmpdir), casd=False)
                 cas.checkout(extractdir, digest)
                 yield extractdir
             except FileNotFoundError:
diff --git a/tests/integration/cachedfail.py b/tests/integration/cachedfail.py
index e3b5b2796..f8dd52aa6 100644
--- a/tests/integration/cachedfail.py
+++ b/tests/integration/cachedfail.py
@@ -20,7 +20,7 @@
 import os
 import pytest
 
-from buildstream import _yaml
+from buildstream import utils, _yaml
 from buildstream._exceptions import ErrorDomain
 from buildstream.testing import cli_integration as cli  # pylint: disable=unused-import
 from buildstream.testing._utils.site import HAVE_SANDBOX
@@ -185,7 +185,12 @@ def test_push_cached_fail(cli, tmpdir, datafiles, on_error):
 
 @pytest.mark.skipif(HAVE_SANDBOX != 'bwrap', reason='Only available with bubblewrap on Linux')
 @pytest.mark.datafiles(DATA_DIR)
-def test_host_tools_errors_are_not_cached(cli, datafiles):
+def test_host_tools_errors_are_not_cached(cli, datafiles, tmp_path):
+    # Create symlink to buildbox-casd to work with custom PATH
+    buildbox_casd = tmp_path.joinpath('bin/buildbox-casd')
+    buildbox_casd.parent.mkdir()
+    os.symlink(utils.get_host_tool('buildbox-casd'), str(buildbox_casd))
+
     project = str(datafiles)
     element_path = os.path.join(project, 'elements', 'element.bst')
 
@@ -207,7 +212,11 @@ def test_host_tools_errors_are_not_cached(cli, datafiles):
     _yaml.roundtrip_dump(element, element_path)
 
     # Build without access to host tools, this will fail
-    result1 = cli.run(project=project, args=['build', 'element.bst'], env={'PATH': '', 'BST_FORCE_SANDBOX': None})
+    result1 = cli.run(
+        project=project,
+        args=['build', 'element.bst'],
+        env={'PATH': str(tmp_path.joinpath('bin')),
+             'BST_FORCE_SANDBOX': None})
     result1.assert_task_error(ErrorDomain.SANDBOX, 'unavailable-local-sandbox')
     assert cli.get_element_state(project, 'element.bst') == 'buildable'
 
diff --git a/tests/sandboxes/missing_dependencies.py b/tests/sandboxes/missing_dependencies.py
index 33a169ca2..975c8eb00 100644
--- a/tests/sandboxes/missing_dependencies.py
+++ b/tests/sandboxes/missing_dependencies.py
@@ -5,7 +5,7 @@ import os
 
 import pytest
 
-from buildstream import _yaml
+from buildstream import utils, _yaml
 from buildstream._exceptions import ErrorDomain
 from buildstream.testing._utils.site import IS_LINUX
 from buildstream.testing import cli  # pylint: disable=unused-import
@@ -20,7 +20,12 @@ DATA_DIR = os.path.join(
 
 @pytest.mark.skipif(not IS_LINUX, reason='Only available on Linux')
 @pytest.mark.datafiles(DATA_DIR)
-def test_missing_brwap_has_nice_error_message(cli, datafiles):
+def test_missing_brwap_has_nice_error_message(cli, datafiles, tmp_path):
+    # Create symlink to buildbox-casd to work with custom PATH
+    buildbox_casd = tmp_path.joinpath('bin/buildbox-casd')
+    buildbox_casd.parent.mkdir()
+    os.symlink(utils.get_host_tool('buildbox-casd'), str(buildbox_casd))
+
     project = str(datafiles)
     element_path = os.path.join(project, 'elements', 'element.bst')
 
@@ -45,8 +50,8 @@ def test_missing_brwap_has_nice_error_message(cli, datafiles):
     result = cli.run(
         project=project,
         args=['build', 'element.bst'],
-        env={'PATH': '', 'BST_FORCE_SANDBOX': None}
-    )
+        env={'PATH': str(tmp_path.joinpath('bin')),
+             'BST_FORCE_SANDBOX': None})
     result.assert_task_error(ErrorDomain.SANDBOX, 'unavailable-local-sandbox')
     assert "not found" in result.stderr
 
@@ -64,6 +69,10 @@ def test_old_brwap_has_nice_error_message(cli, datafiles, tmp_path):
 
     bwrap.chmod(0o755)
 
+    # Create symlink to buildbox-casd to work with custom PATH
+    buildbox_casd = tmp_path.joinpath('bin/buildbox-casd')
+    os.symlink(utils.get_host_tool('buildbox-casd'), str(buildbox_casd))
+
     project = str(datafiles)
     element_path = os.path.join(project, 'elements', 'element3.bst')
 
diff --git a/tests/sandboxes/selection.py b/tests/sandboxes/selection.py
index eff247a95..b4bbb1b00 100644
--- a/tests/sandboxes/selection.py
+++ b/tests/sandboxes/selection.py
@@ -19,7 +19,7 @@
 import os
 import pytest
 
-from buildstream import _yaml
+from buildstream import utils, _yaml
 from buildstream._exceptions import ErrorDomain
 from buildstream.testing import cli  # pylint: disable=unused-import
 
@@ -64,7 +64,12 @@ def test_force_sandbox(cli, datafiles):
 
 
 @pytest.mark.datafiles(DATA_DIR)
-def test_dummy_sandbox_fallback(cli, datafiles):
+def test_dummy_sandbox_fallback(cli, datafiles, tmp_path):
+    # Create symlink to buildbox-casd to work with custom PATH
+    buildbox_casd = tmp_path.joinpath('bin/buildbox-casd')
+    buildbox_casd.parent.mkdir()
+    os.symlink(utils.get_host_tool('buildbox-casd'), str(buildbox_casd))
+
     project = str(datafiles)
     element_path = os.path.join(project, 'elements', 'element.bst')
 
@@ -86,7 +91,11 @@ def test_dummy_sandbox_fallback(cli, datafiles):
     _yaml.roundtrip_dump(element, element_path)
 
     # Build without access to host tools, this will fail
-    result = cli.run(project=project, args=['build', 'element.bst'], env={'PATH': '', 'BST_FORCE_SANDBOX': None})
+    result = cli.run(
+        project=project,
+        args=['build', 'element.bst'],
+        env={'PATH': str(tmp_path.joinpath('bin')),
+             'BST_FORCE_SANDBOX': None})
     # But if we dont spesify a sandbox then we fall back to dummy, we still
     # fail early but only once we know we need a facny sandbox and that
     # dumy is not enough, there for element gets fetched and so is buildable
diff --git a/tests/testutils/artifactshare.py b/tests/testutils/artifactshare.py
index 89fd6d3f8..231559a56 100644
--- a/tests/testutils/artifactshare.py
+++ b/tests/testutils/artifactshare.py
@@ -46,7 +46,7 @@ class ArtifactShare():
         self.artifactdir = os.path.join(self.repodir, 'artifacts', 'refs')
         os.makedirs(self.artifactdir)
 
-        self.cas = CASCache(self.repodir)
+        self.cas = CASCache(self.repodir, casd=False)
 
         self.total_space = total_space
         self.free_space = free_space