1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
|
import os
import subprocess
from .. import utils
from .._exceptions import SandboxError
from .._protos.build.bazel.remote.execution.v2 import remote_execution_pb2
from .._protos.google.rpc import code_pb2
from . import Sandbox, SandboxCommandError
from .._cas import CASRemote
from ..storage._casbaseddirectory import CasBasedDirectory
# SandboxBuildBoxRun()
#
# BuildBox-based sandbox implementation.
#
class SandboxBuildBoxRun(Sandbox):
build_root = "/persistent-cache/userchroot/buildstream"
@classmethod
def check_available(cls):
try:
utils.get_host_tool("buildbox-run")
except utils.ProgramNotFoundError as Error:
cls._dummy_reasons += ["buildbox-run not found"]
raise SandboxError(" and ".join(cls._dummy_reasons),
reason="unavailable-local-sandbox") from Error
@classmethod
def check_sandbox_config(cls, platform, config):
# Report error for elements requiring non-0 UID/GID
if config.build_uid != 0 or config.build_gid != 0:
return False
# Check host os and architecture match
if config.build_os != platform.get_host_os():
raise SandboxError("Configured and host OS don't match.")
if config.build_arch != platform.get_host_arch():
raise SandboxError("Configured and host architecture don't match.")
# FIXME: check that buildbox-casd is configured correctly
# FIXME: check that userchroot is configured correctly
return True
def _run(self, command, flags, *, cwd, env):
stdout, stderr = self._get_output()
if not self._has_command(command[0], env):
raise SandboxCommandError("Staged artifacts do not provide command "
"'{}'".format(command[0]),
reason="missing-command")
cas_cache = self.get_virtual_directory().cas_cache
with utils._tempnamedfile() as action_file, utils._tempnamedfile() as result_file, utils._tempdir(dir=self.build_root) as bldroot:
command = self._create_command(command, cwd, env).SerializeToString()
cas_cache.add_object(buffer=command)
# FIXME: we need to have
self._create_action(action_file, command)
buildbox_command = [
utils.get_host_tool("buildbox-run"),
"--userchroot-bin={}".format(utils.get_host_tool("userchroot")),
"--use-localcas",
"--remote=unix://{}".format(cas_cache._casd_socket_path),
"--action={}".format(action_file.name),
"--action-result={}".format(result_file.name),
"--log-level=trace",
"--workspace-path={}".format(bldroot),
]
# FIXME: handle pausing, sigint, etc correctly
subprocess.check_call(buildbox_command, stdout=stdout, stderr=stderr)
action_result = remote_execution_pb2.ActionResult().FromString(result_file.read())
# Get output of build
self.process_job_output(action_result.output_directories, action_result.output_files,
failure=action_result.exit_code != 0)
if stdout:
if action_result.stdout_raw:
stdout.write(str(action_result.stdout_raw, 'utf-8', errors='ignore'))
if stderr:
if action_result.stderr_raw:
stderr.write(str(action_result.stderr_raw, 'utf-8', errors='ignore'))
if action_result.exit_code != 0:
# A normal error during the build: the remote execution system
# has worked correctly but the command failed.
return action_result.exit_code
return 0
def _create_command(self, command, working_directory, environment):
# Creates a command proto
environment_variables = [remote_execution_pb2.Command.
EnvironmentVariable(name=k, value=v)
for (k, v) in environment.items()]
# Request the whole directory tree as output
output_directory = os.path.relpath(os.path.sep, start=working_directory)
return remote_execution_pb2.Command(arguments=command,
working_directory=working_directory,
environment_variables=environment_variables,
output_files=[],
output_directories=[output_directory],
platform=None)
def _create_action(self, action_file, command):
command_digest = utils._message_digest(command)
input_root_digest = self.get_virtual_directory()._get_digest()
action = remote_execution_pb2.Action(command_digest=command_digest, input_root_digest=input_root_digest)
action_file.write(action.SerializeToString())
action_file.seek(0)
def _use_cas_based_directory(self):
# Always use CasBasedDirectory for BuildBoxRun
return True
def process_job_output(self, output_directories, output_files, *, failure):
# Reads the remote execution server response to an execution request.
#
# output_directories is an array of OutputDirectory objects.
# output_files is an array of OutputFile objects.
#
# We only specify one output_directory, so it's an error
# for there to be any output files or more than one directory at the moment.
#
if output_files:
raise SandboxError("Output files were returned when we didn't request any.")
if not output_directories:
error_text = "No output directory was returned from the build server."
raise SandboxError(error_text)
if len(output_directories) > 1:
error_text = "More than one output directory was returned from the build server: {}."
raise SandboxError(error_text.format(output_directories))
dir_digest = output_directories[0].tree_digest
if dir_digest is None or not dir_digest.hash:
raise SandboxError("Output directory structure had no digest attached.")
context = self._get_context()
cascache = context.get_cascache()
artifactcache = context.artifactcache
if dir_digest is None or not dir_digest.hash or not dir_digest.size_bytes:
raise SandboxError("Output directory structure pulling from remote failed.")
# At the moment, we will get the whole directory back in the first directory argument and we need
# to replace the sandbox's virtual directory with that. Creating a new virtual directory object
# from another hash will be interesting, though...
new_dir = CasBasedDirectory(artifactcache.cas, digest=dir_digest)
self._set_virtual_directory(new_dir)
|
