Merge branch 'juerg/reapi-log' into 'master'

_sandboxremote.py: Support stdout and stderr digests

See merge request BuildStream/buildstream!1821

4 files changed, 30 insertions, 10 deletions

diff --git a/src/buildstream/_cas/cascache.py b/src/buildstream/_cas/cascache.py
index c733bacac..cfdd4af09 100644
--- a/src/buildstream/_cas/cascache.py
+++ b/src/buildstream/_cas/cascache.py
@@ -649,7 +649,8 @@ class CASCache:
         batch = _CASBatchRead(remote)
 
         for digest in digests:
-            batch.add(digest)
+            if digest.hash:
+                batch.add(digest)
 
         batch.send(missing_blobs=missing_blobs)
 
diff --git a/src/buildstream/sandbox/_sandboxreapi.py b/src/buildstream/sandbox/_sandboxreapi.py
index ee7fc72ae..2f500cd51 100644
--- a/src/buildstream/sandbox/_sandboxreapi.py
+++ b/src/buildstream/sandbox/_sandboxreapi.py
@@ -39,8 +39,6 @@ class SandboxREAPI(Sandbox):
         return True
 
     def _run(self, command, flags, *, cwd, env):
-        stdout, stderr = self._get_output()
-
         context = self._get_context()
         cascache = context.get_cascache()
 
@@ -93,13 +91,6 @@ class SandboxREAPI(Sandbox):
             cwd, action_result.output_directories, action_result.output_files, failure=action_result.exit_code != 0
         )
 
-        if stdout:
-            if action_result.stdout_raw:
-                stdout.write(str(action_result.stdout_raw, "utf-8", errors="ignore"))
-        if stderr:
-            if action_result.stderr_raw:
-                stderr.write(str(action_result.stderr_raw, "utf-8", errors="ignore"))
-
         # Non-zero exit code means a normal error during the build:
         # the remote execution system has worked correctly but the command failed.
         return action_result.exit_code
diff --git a/src/buildstream/sandbox/_sandboxremote.py b/src/buildstream/sandbox/_sandboxremote.py
index 5ec1c974b..3dcbb2ccc 100644
--- a/src/buildstream/sandbox/_sandboxremote.py
+++ b/src/buildstream/sandbox/_sandboxremote.py
@@ -19,6 +19,7 @@
 #        Jim MacArthur <jim.macarthur@codethink.co.uk>
 
 import os
+import shutil
 from collections import namedtuple
 from urllib.parse import urlparse
 from functools import partial
@@ -298,6 +299,8 @@ class SandboxRemote(SandboxREAPI):
                     )
 
     def _execute_action(self, action, flags):
+        stdout, stderr = self._get_output()
+
         context = self._get_context()
         project = self._get_project()
         cascache = context.get_cascache()
@@ -375,6 +378,23 @@ class SandboxRemote(SandboxREAPI):
                 # Now do a pull to ensure we have the full directory structure.
                 cascache.pull_tree(casremote, tree_digest)
 
+            # Fetch stdout and stderr blobs
+            cascache.fetch_blobs(casremote, [action_result.stdout_digest, action_result.stderr_digest])
+
+        # Forward remote stdout and stderr
+        if stdout:
+            if action_result.stdout_digest.hash:
+                with open(cascache.objpath(action_result.stdout_digest), "r") as f:
+                    shutil.copyfileobj(f, stdout)
+            elif action_result.stdout_raw:
+                stdout.write(str(action_result.stdout_raw, "utf-8", errors="ignore"))
+        if stderr:
+            if action_result.stderr_digest.hash:
+                with open(cascache.objpath(action_result.stderr_digest), "r") as f:
+                    shutil.copyfileobj(f, stderr)
+            elif action_result.stderr_raw:
+                stderr.write(str(action_result.stderr_raw, "utf-8", errors="ignore"))
+
         return action_result
 
     def _check_action_cache(self, action_digest):
diff --git a/tests/integration/autotools.py b/tests/integration/autotools.py
index 86a06aa4f..d1ab82e53 100644
--- a/tests/integration/autotools.py
+++ b/tests/integration/autotools.py
@@ -44,6 +44,14 @@ def test_autotools_build(cli, datafiles):
         ],
     )
 
+    # Check the log
+    result = cli.run(project=project, args=["artifact", "log", element_name])
+    assert result.exit_code == 0
+    log = result.output
+
+    # Verify we get expected output exactly once
+    assert log.count("Making all in src") == 1
+
 
 # Test that an autotools build 'works' - we use the autotools sample
 # amhello project for this.