From 25b2577604a05df7314a7e7cc78f8f163cd6dbd4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BCrg=20Billeter?= Date: Thu, 20 Feb 2020 08:40:27 +0100 Subject: tests/integration/autotools.py: Check build log --- tests/integration/autotools.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/integration/autotools.py b/tests/integration/autotools.py index 86a06aa4f..d1ab82e53 100644 --- a/tests/integration/autotools.py +++ b/tests/integration/autotools.py @@ -44,6 +44,14 @@ def test_autotools_build(cli, datafiles): ], ) + # Check the log + result = cli.run(project=project, args=["artifact", "log", element_name]) + assert result.exit_code == 0 + log = result.output + + # Verify we get expected output exactly once + assert log.count("Making all in src") == 1 + # Test that an autotools build 'works' - we use the autotools sample # amhello project for this. -- cgit v1.2.1 From f65299c68c2eccf885aa440d5efe88d0ffbd4b14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BCrg=20Billeter?= Date: Thu, 20 Feb 2020 10:53:36 +0100 Subject: cascache.py: Don't trip up on empty digests in fetch_blobs() --- src/buildstream/_cas/cascache.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/buildstream/_cas/cascache.py b/src/buildstream/_cas/cascache.py index c733bacac..cfdd4af09 100644 --- a/src/buildstream/_cas/cascache.py +++ b/src/buildstream/_cas/cascache.py @@ -649,7 +649,8 @@ class CASCache: batch = _CASBatchRead(remote) for digest in digests: - batch.add(digest) + if digest.hash: + batch.add(digest) batch.send(missing_blobs=missing_blobs) -- cgit v1.2.1 From cccf7425aa0b952b771df037a512873aea53b02c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BCrg=20Billeter?= Date: Thu, 20 Feb 2020 09:24:18 +0100 Subject: sandbox: Move log forwarding from SandboxREAPI to SandboxRemote SandboxBuildBoxRun doesn't require log forwarding as we use stdout and stderr of the buildbox-run process. --- src/buildstream/sandbox/_sandboxreapi.py | 9 --------- src/buildstream/sandbox/_sandboxremote.py | 10 ++++++++++ 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/src/buildstream/sandbox/_sandboxreapi.py b/src/buildstream/sandbox/_sandboxreapi.py index ee7fc72ae..2f500cd51 100644 --- a/src/buildstream/sandbox/_sandboxreapi.py +++ b/src/buildstream/sandbox/_sandboxreapi.py @@ -39,8 +39,6 @@ class SandboxREAPI(Sandbox): return True def _run(self, command, flags, *, cwd, env): - stdout, stderr = self._get_output() - context = self._get_context() cascache = context.get_cascache() @@ -93,13 +91,6 @@ class SandboxREAPI(Sandbox): cwd, action_result.output_directories, action_result.output_files, failure=action_result.exit_code != 0 ) - if stdout: - if action_result.stdout_raw: - stdout.write(str(action_result.stdout_raw, "utf-8", errors="ignore")) - if stderr: - if action_result.stderr_raw: - stderr.write(str(action_result.stderr_raw, "utf-8", errors="ignore")) - # Non-zero exit code means a normal error during the build: # the remote execution system has worked correctly but the command failed. return action_result.exit_code diff --git a/src/buildstream/sandbox/_sandboxremote.py b/src/buildstream/sandbox/_sandboxremote.py index 5ec1c974b..fe7812ba5 100644 --- a/src/buildstream/sandbox/_sandboxremote.py +++ b/src/buildstream/sandbox/_sandboxremote.py @@ -298,6 +298,8 @@ class SandboxRemote(SandboxREAPI): ) def _execute_action(self, action, flags): + stdout, stderr = self._get_output() + context = self._get_context() project = self._get_project() cascache = context.get_cascache() @@ -375,6 +377,14 @@ class SandboxRemote(SandboxREAPI): # Now do a pull to ensure we have the full directory structure. cascache.pull_tree(casremote, tree_digest) + # Forward remote stdout and stderr + if stdout: + if action_result.stdout_raw: + stdout.write(str(action_result.stdout_raw, "utf-8", errors="ignore")) + if stderr: + if action_result.stderr_raw: + stderr.write(str(action_result.stderr_raw, "utf-8", errors="ignore")) + return action_result def _check_action_cache(self, action_digest): -- cgit v1.2.1 From 802165cc287089652c9800b28102d3af9532d5b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BCrg=20Billeter?= Date: Thu, 20 Feb 2020 08:51:56 +0100 Subject: _sandboxremote.py: Support stdout and stderr digests Fetch blobs from remote CAS and then forward them to the sandbox output. --- src/buildstream/sandbox/_sandboxremote.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/buildstream/sandbox/_sandboxremote.py b/src/buildstream/sandbox/_sandboxremote.py index fe7812ba5..3dcbb2ccc 100644 --- a/src/buildstream/sandbox/_sandboxremote.py +++ b/src/buildstream/sandbox/_sandboxremote.py @@ -19,6 +19,7 @@ # Jim MacArthur import os +import shutil from collections import namedtuple from urllib.parse import urlparse from functools import partial @@ -377,12 +378,21 @@ class SandboxRemote(SandboxREAPI): # Now do a pull to ensure we have the full directory structure. cascache.pull_tree(casremote, tree_digest) + # Fetch stdout and stderr blobs + cascache.fetch_blobs(casremote, [action_result.stdout_digest, action_result.stderr_digest]) + # Forward remote stdout and stderr if stdout: - if action_result.stdout_raw: + if action_result.stdout_digest.hash: + with open(cascache.objpath(action_result.stdout_digest), "r") as f: + shutil.copyfileobj(f, stdout) + elif action_result.stdout_raw: stdout.write(str(action_result.stdout_raw, "utf-8", errors="ignore")) if stderr: - if action_result.stderr_raw: + if action_result.stderr_digest.hash: + with open(cascache.objpath(action_result.stderr_digest), "r") as f: + shutil.copyfileobj(f, stderr) + elif action_result.stderr_raw: stderr.write(str(action_result.stderr_raw, "utf-8", errors="ignore")) return action_result -- cgit v1.2.1