summaryrefslogtreecommitdiff
path: root/chromium/build/fuchsia
diff options
context:
space:
mode:
authorAllan Sandfeld Jensen <allan.jensen@qt.io>2020-10-12 14:27:29 +0200
committerAllan Sandfeld Jensen <allan.jensen@qt.io>2020-10-13 09:35:20 +0000
commitc30a6232df03e1efbd9f3b226777b07e087a1122 (patch)
treee992f45784689f373bcc38d1b79a239ebe17ee23 /chromium/build/fuchsia
parent7b5b123ac58f58ffde0f4f6e488bcd09aa4decd3 (diff)
downloadqtwebengine-chromium-85-based.tar.gz
BASELINE: Update Chromium to 85.0.4183.14085-based
Change-Id: Iaa42f4680837c57725b1344f108c0196741f6057 Reviewed-by: Allan Sandfeld Jensen <allan.jensen@qt.io>
Diffstat (limited to 'chromium/build/fuchsia')
-rw-r--r--chromium/build/fuchsia/aemu_target.py20
-rw-r--r--chromium/build/fuchsia/common_args.py2
-rw-r--r--chromium/build/fuchsia/device_target.py6
-rw-r--r--chromium/build/fuchsia/emu_target.py46
-rw-r--r--chromium/build/fuchsia/linux.sdk.sha12
-rw-r--r--chromium/build/fuchsia/mac.sdk.sha12
-rw-r--r--chromium/build/fuchsia/net_test_server.py1
-rw-r--r--chromium/build/fuchsia/qemu_image.py16
-rw-r--r--chromium/build/fuchsia/qemu_target.py8
-rw-r--r--chromium/build/fuchsia/remote_cmd.py23
-rw-r--r--chromium/build/fuchsia/run_package.py20
-rw-r--r--chromium/build/fuchsia/runner_exceptions.py35
-rw-r--r--chromium/build/fuchsia/runner_logs.py96
-rw-r--r--chromium/build/fuchsia/sdk-bucket.txt1
-rw-r--r--chromium/build/fuchsia/sdk-hash-files.list1
-rw-r--r--chromium/build/fuchsia/symbolizer.py9
-rw-r--r--chromium/build/fuchsia/target.py10
-rwxr-xr-xchromium/build/fuchsia/test_runner.py66
-rwxr-xr-xchromium/build/fuchsia/update_sdk.py86
19 files changed, 302 insertions, 148 deletions
diff --git a/chromium/build/fuchsia/aemu_target.py b/chromium/build/fuchsia/aemu_target.py
index 0e3bc8f4ead..e997541ab8c 100644
--- a/chromium/build/fuchsia/aemu_target.py
+++ b/chromium/build/fuchsia/aemu_target.py
@@ -11,7 +11,9 @@ import logging
from common import GetEmuRootForPlatform
+
class AemuTarget(qemu_target.QemuTarget):
+
def __init__(self, output_dir, target_cpu, system_log_file, emu_type,
cpu_cores, require_kvm, ram_size_mb, enable_graphics,
hardware_gpu):
@@ -60,7 +62,7 @@ class AemuTarget(qemu_target.QemuTarget):
# All args after -fuchsia flag gets passed to QEMU
aemu_command.extend([
'-feature', aemu_features, '-window-size', '1024x600', '-gpu',
- gpu_target, '-fuchsia'
+ gpu_target, '-verbose', '-fuchsia'
])
aemu_command.extend(self._BuildQemuConfig())
@@ -73,3 +75,19 @@ class AemuTarget(qemu_target.QemuTarget):
'-device', 'ich9-ahci,id=ahci'])
logging.info(' '.join(aemu_command))
return aemu_command
+
+ def _GetVulkanIcdFile(self):
+ return os.path.join(GetEmuRootForPlatform(self._emu_type), 'lib64',
+ 'vulkan', 'vk_swiftshader_icd.json')
+
+ def _SetEnv(self):
+ env = os.environ.copy()
+ aemu_logging_env = {
+ "ANDROID_EMU_VK_NO_CLEANUP": "1",
+ "ANDROID_EMUGL_LOG_PRINT": "1",
+ "ANDROID_EMUGL_VERBOSE": "1",
+ "VK_ICD_FILENAMES": self._GetVulkanIcdFile(),
+ "VK_LOADER_DEBUG": "info,error",
+ }
+ env.update(aemu_logging_env)
+ return env
diff --git a/chromium/build/fuchsia/common_args.py b/chromium/build/fuchsia/common_args.py
index 24544f2c305..855bc1149c3 100644
--- a/chromium/build/fuchsia/common_args.py
+++ b/chromium/build/fuchsia/common_args.py
@@ -66,6 +66,8 @@ def AddCommonArgs(arg_parser):
help='Path to a Fuchsia build output directory. '
'Equivalent to setting --ssh_config and '
'---os-check=ignore')
+ common_args.add_argument('--runner-logs-dir',
+ help='Directory to write test runner logs to.')
common_args.add_argument('--system-log-file',
help='File to write system logs to. Specify - to '
'log to stdout.')
diff --git a/chromium/build/fuchsia/device_target.py b/chromium/build/fuchsia/device_target.py
index 8a5f7d7c5b9..9ba173ba95a 100644
--- a/chromium/build/fuchsia/device_target.py
+++ b/chromium/build/fuchsia/device_target.py
@@ -141,7 +141,7 @@ class DeviceTarget(target.Target):
or waits up to |timeout| seconds and returns False if the device couldn't
be found."""
- dev_finder_path = GetHostToolPathFromPlatform('dev_finder')
+ dev_finder_path = GetHostToolPathFromPlatform('device-finder')
if self._node_name:
command = [dev_finder_path, 'resolve',
@@ -163,13 +163,13 @@ class DeviceTarget(target.Target):
return False
if self._node_name:
- # Handle the result of "dev_finder resolve".
+ # Handle the result of "device-finder resolve".
self._host = output.pop().strip()
else:
name_host_pairs = [x.strip().split(' ') for x in output]
- # Handle the output of "dev_finder list".
+ # Handle the output of "device-finder list".
if len(name_host_pairs) > 1:
print('More than one device was discovered on the network.')
print('Use --node-name <name> to specify the device to use.')
diff --git a/chromium/build/fuchsia/emu_target.py b/chromium/build/fuchsia/emu_target.py
index 83e10167914..6ccdd832fbd 100644
--- a/chromium/build/fuchsia/emu_target.py
+++ b/chromium/build/fuchsia/emu_target.py
@@ -8,6 +8,7 @@ import amber_repo
import boot_data
import logging
import os
+import runner_logs
import subprocess
import sys
import target
@@ -34,6 +35,9 @@ class EmuTarget(target.Target):
"""Build the command that will be run to start Fuchsia in the emulator."""
pass
+ def _SetEnv(self):
+ return os.environ.copy()
+
# Used by the context manager to ensure that the emulator is killed when
# the Python process exits.
def __exit__(self, exc_type, exc_val, exc_tb):
@@ -52,27 +56,28 @@ class EmuTarget(target.Target):
# Zircon sends debug logs to serial port (see kernel.serial=legacy flag
# above). Serial port is redirected to a file through emulator stdout.
- # Unless a |_system_log_file| is explicitly set, we output the kernel serial
- # log to a temporary file, and print that out if we are unable to connect to
+ # Unless runner_pogs are enabled, we output the kernel serial log
+ # to a temporary file, and print that out if we are unable to connect to
# the emulator guest, to make it easier to diagnose connectivity issues.
- temporary_system_log_file = None
- if self._system_log_file:
- stdout = self._system_log_file
- stderr = subprocess.STDOUT
+ temporary_log_file = None
+ if runner_logs.IsEnabled():
+ stdout = runner_logs.FileStreamFor('serial_log')
else:
- temporary_system_log_file = tempfile.NamedTemporaryFile('w')
- stdout = temporary_system_log_file
- stderr = sys.stderr
+ temporary_log_file = tempfile.NamedTemporaryFile('w')
+ stdout = temporary_log_file
- self._emu_process = subprocess.Popen(emu_command, stdin=open(os.devnull),
- stdout=stdout, stderr=stderr)
+ self._emu_process = subprocess.Popen(emu_command,
+ stdin=open(os.devnull),
+ stdout=stdout,
+ stderr=subprocess.STDOUT,
+ env=self._SetEnv())
try:
- self._WaitUntilReady();
+ self._WaitUntilReady()
except target.FuchsiaTargetException:
- if temporary_system_log_file:
+ if temporary_log_file:
logging.info('Kernel logs:\n' +
- open(temporary_system_log_file.name, 'r').read())
+ open(temporary_log_file.name, 'r').read())
raise
def GetAmberRepo(self):
@@ -86,15 +91,18 @@ class EmuTarget(target.Target):
logging.error('%s did not start' % (self._GetEmulatorName()))
return
returncode = self._emu_process.poll()
- if returncode:
- logging.error('%s quit unexpectedly with exit code %d' %
- (self._GetEmulatorName(), returncode))
+ if returncode == None:
+ logging.info('Shutting down %s' % (self._GetEmulatorName()))
+ self._emu_process.kill()
elif returncode == 0:
logging.info('%s quit unexpectedly without errors' %
self._GetEmulatorName())
+ elif returncode < 0:
+ logging.error('%s was terminated by signal %d' %
+ (self._GetEmulatorName(), -returncode))
else:
- logging.info('Shutting down %s' % (self._GetEmulatorName()))
- self._emu_process.kill()
+ logging.error('%s quit unexpectedly with exit code %d' %
+ (self._GetEmulatorName(), returncode))
def _IsEmuStillRunning(self):
if not self._emu_process:
diff --git a/chromium/build/fuchsia/linux.sdk.sha1 b/chromium/build/fuchsia/linux.sdk.sha1
index 7552ab1c396..a7cac626a07 100644
--- a/chromium/build/fuchsia/linux.sdk.sha1
+++ b/chromium/build/fuchsia/linux.sdk.sha1
@@ -1 +1 @@
-0.20200514.2.1
+0.20200625.3.1
diff --git a/chromium/build/fuchsia/mac.sdk.sha1 b/chromium/build/fuchsia/mac.sdk.sha1
index da07900e95d..a7cac626a07 100644
--- a/chromium/build/fuchsia/mac.sdk.sha1
+++ b/chromium/build/fuchsia/mac.sdk.sha1
@@ -1 +1 @@
-0.20200514.1.1
+0.20200625.3.1
diff --git a/chromium/build/fuchsia/net_test_server.py b/chromium/build/fuchsia/net_test_server.py
index 94f71635433..60ad78f60fa 100644
--- a/chromium/build/fuchsia/net_test_server.py
+++ b/chromium/build/fuchsia/net_test_server.py
@@ -43,6 +43,7 @@ class SSHPortForwarder(chrome_test_server_spawner.PortForwarder):
'-NT', '-O', 'cancel', '-R', '0:localhost:%d' % host_port]
task = self._target.RunCommandPiped([],
ssh_args=forwarding_args,
+ stdout=open(os.devnull, 'w'),
stderr=subprocess.PIPE)
task.wait()
if task.returncode != 0:
diff --git a/chromium/build/fuchsia/qemu_image.py b/chromium/build/fuchsia/qemu_image.py
index 5126074fe02..ab5e040acbd 100644
--- a/chromium/build/fuchsia/qemu_image.py
+++ b/chromium/build/fuchsia/qemu_image.py
@@ -18,6 +18,7 @@ TODO(crbug.com/1046861): Remove this workaround when the bug is fixed.
import logging
import subprocess
+import tempfile
import time
@@ -33,7 +34,9 @@ def _ExecQemuImgWithTimeout(command):
"""
logging.info('qemu-img starting')
- p = subprocess.Popen(command)
+ command_output_file = tempfile.NamedTemporaryFile('w')
+ p = subprocess.Popen(command, stdout=command_output_file,
+ stderr=subprocess.STDOUT)
start_sec = time.time()
while p.poll() is None and time.time() - start_sec < QEMU_IMG_TIMEOUT_SEC:
time.sleep(1)
@@ -41,10 +44,17 @@ def _ExecQemuImgWithTimeout(command):
logging.info('qemu-img duration: %f' % float(stop_sec - start_sec))
if p.poll() is None:
+ returncode = None
p.kill()
- return None
+ p.wait()
+ else:
+ returncode = p.returncode
- return p.returncode
+ log_level = logging.WARN if returncode else logging.DEBUG
+ for line in open(command_output_file.name, 'r'):
+ logging.log(log_level, 'qemu-img stdout: ' + line.strip())
+
+ return returncode
def ExecQemuImgWithRetry(command):
diff --git a/chromium/build/fuchsia/qemu_target.py b/chromium/build/fuchsia/qemu_target.py
index bdc397e797b..96d7465bf9e 100644
--- a/chromium/build/fuchsia/qemu_target.py
+++ b/chromium/build/fuchsia/qemu_target.py
@@ -55,7 +55,11 @@ class QemuTarget(emu_target.EmuTarget):
if kvm_supported and same_arch:
return True
elif self._require_kvm:
- raise FuchsiaTargetException('KVM required but unavailable.')
+ if same_arch:
+ raise FuchsiaTargetException('KVM required but unavailable.')
+ else:
+ raise FuchsiaTargetException('KVM unavailable when CPU architecture of'\
+ ' host is different from that of target.')
else:
return False
@@ -119,7 +123,7 @@ class QemuTarget(emu_target.EmuTarget):
if self._target_cpu == 'arm64':
kvm_command.append('host')
else:
- kvm_command.append('host,migratable=no')
+ kvm_command.append('host,migratable=no,+invtsc')
else:
logging.warning('Unable to launch %s with KVM acceleration.'
% (self._emu_type) +
diff --git a/chromium/build/fuchsia/remote_cmd.py b/chromium/build/fuchsia/remote_cmd.py
index 93ce32ce287..019c2dc9abd 100644
--- a/chromium/build/fuchsia/remote_cmd.py
+++ b/chromium/build/fuchsia/remote_cmd.py
@@ -5,7 +5,6 @@
import logging
import os
import subprocess
-import sys
import threading
_SSH = ['ssh']
@@ -57,31 +56,36 @@ class CommandRunner(object):
_SSH_LOGGER.debug('ssh exec: ' + ' '.join(ssh_command))
if silent:
devnull = open(os.devnull, 'w')
- process = subprocess.Popen(ssh_command, stderr=devnull, stdout=devnull)
+ process = subprocess.Popen(ssh_command, stdout=devnull, stderr=devnull)
else:
- process = subprocess.Popen(ssh_command)
+ process = subprocess.Popen(ssh_command, stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT)
timeout_timer = None
if timeout_secs:
timeout_timer = threading.Timer(timeout_secs, process.kill)
timeout_timer.start()
- process.wait()
+ if not silent:
+ for line in process.stdout:
+ print(line)
+ process.wait()
if timeout_timer:
timeout_timer.cancel()
-
if process.returncode == -9:
raise Exception('Timeout when executing \"%s\".' % ' '.join(command))
return process.returncode
- def RunCommandPiped(self, command = None, ssh_args = None, **kwargs):
+ def RunCommandPiped(self, command, stdout, stderr, ssh_args = None, **kwargs):
"""Executes an SSH command on the remote host and returns a process object
with access to the command's stdio streams. Does not block.
command: A list of strings containing the command and its arguments.
+ stdout: subprocess stdout. Must not be None.
+ stderr: subprocess stderr. Must not be None.
ssh_args: Arguments that will be passed to SSH.
kwargs: A dictionary of parameters to be passed to subprocess.Popen().
The parameters can be used to override stdin and stdout, for
@@ -89,14 +93,15 @@ class CommandRunner(object):
Returns a Popen object for the command."""
- if not command:
- command = []
+ if not stdout or not stderr:
+ raise Exception('Stdout/stderr must be specified explicitly')
+
if not ssh_args:
ssh_args = []
ssh_command = self._GetSshCommandLinePrefix() + ssh_args + ['--'] + command
_SSH_LOGGER.debug(' '.join(ssh_command))
- return subprocess.Popen(ssh_command, **kwargs)
+ return subprocess.Popen(ssh_command, stdout=stdout, stderr=stderr, **kwargs)
def RunScp(self, sources, dest, direction, recursive=False):
diff --git a/chromium/build/fuchsia/run_package.py b/chromium/build/fuchsia/run_package.py
index 62bbd3b8fd8..ed2cca3bf12 100644
--- a/chromium/build/fuchsia/run_package.py
+++ b/chromium/build/fuchsia/run_package.py
@@ -16,12 +16,10 @@ import re
import select
import subprocess
import sys
-import time
import threading
import uuid
-from symbolizer import RunSymbolizer
-from symbolizer import SymbolizerFilter
+from symbolizer import BuildIdsPaths, RunSymbolizer, SymbolizerFilter
FAR = common.GetHostToolPathFromPlatform('far')
@@ -34,16 +32,8 @@ def _AttachKernelLogReader(target):
logging.info('Attaching kernel logger.')
return target.RunCommandPiped(['dlog', '-f'], stdin=open(os.devnull, 'r'),
- stdout=subprocess.PIPE)
-
-
-def _BuildIdsPaths(package_paths):
- """Generate build ids paths for symbolizer processes."""
- build_ids_paths = map(
- lambda package_path: os.path.join(
- os.path.dirname(package_path), 'ids.txt'),
- package_paths)
- return build_ids_paths
+ stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT)
class SystemLogReader(object):
@@ -77,7 +67,7 @@ class SystemLogReader(object):
self._system_log = open(system_log_file,'w', buffering=1)
self._symbolizer_proc = RunSymbolizer(self._listener_proc.stdout,
self._system_log,
- _BuildIdsPaths(package_paths))
+ BuildIdsPaths(package_paths))
class MergedInputStream(object):
@@ -239,7 +229,7 @@ def RunPackage(output_dir, target, package_paths, package_name,
# Run the log data through the symbolizer process.
output_stream = SymbolizerFilter(output_stream,
- _BuildIdsPaths(package_paths))
+ BuildIdsPaths(package_paths))
for next_line in output_stream:
print(next_line.rstrip())
diff --git a/chromium/build/fuchsia/runner_exceptions.py b/chromium/build/fuchsia/runner_exceptions.py
index cedf99bbd7a..03f872e453f 100644
--- a/chromium/build/fuchsia/runner_exceptions.py
+++ b/chromium/build/fuchsia/runner_exceptions.py
@@ -7,6 +7,7 @@
This makes it easier to query build tables for particular error types as
exit codes are visible to queries while exception stack traces are not."""
+import errno
import fcntl
import logging
import os
@@ -23,17 +24,15 @@ def _PrintException(value, trace):
print(str(value))
-# TODO(crbug.com/1080858): Delete function when the stdout print bug is fixed.
-def _LogStdoutBlockingStatus():
- """Log whether sys.stdout is blocking or non-blocking.
+def IsStdoutBlocking():
+ """Returns True if sys.stdout is blocking or False if non-blocking.
- It should be blocking, but there are intermittent IO errors that suggest
- that it is set to non-blocking at times during test runs."""
+ sys.stdout should always be blocking. Non-blocking is associated with
+ intermittent IOErrors (crbug.com/1080858).
+ """
- if fcntl.fcntl(sys.stdout, fcntl.F_GETFD) & os.O_NONBLOCK:
- logging.error('sys.stdout is non-blocking')
- else:
- logging.info('sys.stdout is blocking')
+ nonblocking = fcntl.fcntl(sys.stdout, fcntl.F_GETFL) & os.O_NONBLOCK
+ return not nonblocking
def HandleExceptionAndReturnExitCode():
@@ -57,19 +56,23 @@ def HandleExceptionAndReturnExitCode():
if type is FuchsiaTargetException:
if 'ssh' in str(value).lower():
- print('Error: FuchsiaTargetException: SSH to Fuchsia target failed.')
- return 65
+ print('Error: FuchsiaTargetException: SSH to Fuchsia target failed.')
+ return 65
return 64
elif type is IOError:
- if value.errno == 11:
- print('Info: Python print to sys.stdout probably failed')
- _LogStdoutBlockingStatus()
- return 73
+ if value.errno == errno.EAGAIN:
+ logging.info('Python print to sys.stdout probably failed')
+ if not IsStdoutBlocking():
+ logging.warn('sys.stdout is non-blocking')
+ return 73
return 72
elif type is subprocess.CalledProcessError:
- if value.cmd[0] == 'scp':
+ if os.path.basename(value.cmd[0]) == 'scp':
print('Error: scp operation failed - %s' % str(value))
return 81
+ if os.path.basename(value.cmd[0]) == 'qemu-img':
+ print('Error: qemu-img fuchsia image generation failed.')
+ return 82
return 80
else:
return 1
diff --git a/chromium/build/fuchsia/runner_logs.py b/chromium/build/fuchsia/runner_logs.py
new file mode 100644
index 00000000000..19da4f5a045
--- /dev/null
+++ b/chromium/build/fuchsia/runner_logs.py
@@ -0,0 +1,96 @@
+# Copyright 2020 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Creates and manages test runner log file objects.
+
+Provides a context manager object for use in a with statement
+and a module level FileStreamFor function for use by clients.
+"""
+
+import collections
+import multiprocessing
+import os
+
+from symbolizer import RunSymbolizer
+
+SYMBOLIZED_SUFFIX = '.symbolized'
+
+_RunnerLogEntry = collections.namedtuple(
+ '_RunnerLogEntry', ['name', 'log_file', 'path', 'symbolize'])
+
+# Module singleton variable.
+_instance = None
+
+
+class RunnerLogManager(object):
+ """ Runner logs object for use in a with statement."""
+
+ def __init__(self, log_dir, build_ids_files):
+ global _instance
+ if _instance:
+ raise Exception('Only one RunnerLogManager can be instantiated')
+
+ self._log_dir = log_dir
+ self._build_ids_files = build_ids_files
+ self._runner_logs = []
+
+ if self._log_dir:
+ os.makedirs(self._log_dir)
+
+ _instance = self
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, exc_type, exc_value, traceback):
+ pool = multiprocessing.Pool(4)
+ for log_entry in self._runner_logs:
+ pool.apply_async(_FinalizeLog, (log_entry, self._build_ids_files))
+ pool.close()
+ pool.join()
+ _instance = None
+
+
+ def _FileStreamFor(self, name, symbolize):
+ if any(elem.name == name for elem in self._runner_logs):
+ raise Exception('RunnerLogManager can only open "%" once' % name)
+
+ path = os.path.join(self._log_dir, name) if self._log_dir else os.devnull
+ log_file = open(path, 'w')
+
+ self._runner_logs.append(_RunnerLogEntry(name, log_file, path, symbolize))
+
+ return log_file
+
+
+def _FinalizeLog(log_entry, build_ids_files):
+ log_entry.log_file.close()
+
+ if log_entry.symbolize:
+ input_file = open(log_entry.path, 'r')
+ output_file = open(log_entry.path + SYMBOLIZED_SUFFIX, 'w')
+ proc = RunSymbolizer(input_file, output_file, build_ids_files)
+ proc.wait()
+ output_file.close()
+ input_file.close()
+
+
+def IsEnabled():
+ """Returns True if the RunnerLogManager has been created, or False if not."""
+
+ return _instance is not None and _instance._log_dir is not None
+
+
+def FileStreamFor(name, symbolize=False):
+ """Opens a test runner file stream in the test runner log directory.
+
+ If no test runner log directory is specified, output is discarded.
+
+ name: log file name
+ symbolize: if True, make a symbolized copy of the log after closing it.
+
+ Returns an opened log file object."""
+
+ return _instance._FileStreamFor(name, symbolize) if IsEnabled() else open(
+ os.devnull, 'w')
diff --git a/chromium/build/fuchsia/sdk-bucket.txt b/chromium/build/fuchsia/sdk-bucket.txt
new file mode 100644
index 00000000000..9f860bce1d7
--- /dev/null
+++ b/chromium/build/fuchsia/sdk-bucket.txt
@@ -0,0 +1 @@
+fuchsia
diff --git a/chromium/build/fuchsia/sdk-hash-files.list b/chromium/build/fuchsia/sdk-hash-files.list
new file mode 100644
index 00000000000..6f37bcd9f77
--- /dev/null
+++ b/chromium/build/fuchsia/sdk-hash-files.list
@@ -0,0 +1 @@
+{platform}.sdk.sha1
diff --git a/chromium/build/fuchsia/symbolizer.py b/chromium/build/fuchsia/symbolizer.py
index 1432feea3f0..325cc23fc61 100644
--- a/chromium/build/fuchsia/symbolizer.py
+++ b/chromium/build/fuchsia/symbolizer.py
@@ -14,6 +14,15 @@ from common import GetHostToolPathFromPlatform
ARM64_DOCKER_LLVM_SYMBOLIZER_PATH = os.path.join('/', 'usr', 'lib', 'llvm-3.8',
'bin', 'llvm-symbolizer')
+def BuildIdsPaths(package_paths):
+ """Generate build ids paths for symbolizer processes."""
+ build_ids_paths = map(
+ lambda package_path: os.path.join(
+ os.path.dirname(package_path), 'ids.txt'),
+ package_paths)
+ return build_ids_paths
+
+
def RunSymbolizer(input_file, output_file, build_ids_files):
"""Starts a symbolizer process.
diff --git a/chromium/build/fuchsia/target.py b/chromium/build/fuchsia/target.py
index d8c1741f7d8..254c5fd5789 100644
--- a/chromium/build/fuchsia/target.py
+++ b/chromium/build/fuchsia/target.py
@@ -8,6 +8,7 @@ import json
import logging
import os
import remote_cmd
+import runner_logs
import shutil
import subprocess
import sys
@@ -21,7 +22,7 @@ _ATTACH_RETRY_SECONDS = 120
# Amount of time to wait for Amber to complete package installation, as a
# mitigation against hangs due to amber/network-related failures.
-_INSTALL_TIMEOUT_SECS = 5 * 60
+_INSTALL_TIMEOUT_SECS = 10 * 60
def _GetPackageUri(package_name):
@@ -216,9 +217,14 @@ class Target(object):
host, port = self._GetEndpoint()
end_time = time.time() + _ATTACH_RETRY_SECONDS
+ ssh_diagnostic_log = runner_logs.FileStreamFor('ssh_diagnostic_log')
while time.time() < end_time:
runner = remote_cmd.CommandRunner(self._GetSshConfigPath(), host, port)
- if runner.RunCommand(['true'], True) == 0:
+ ssh_proc = runner.RunCommandPiped(['true'],
+ ssh_args=['-v'],
+ stdout=ssh_diagnostic_log,
+ stderr=subprocess.STDOUT)
+ if ssh_proc.wait() == 0:
logging.info('Connected!')
self._started = True
return True
diff --git a/chromium/build/fuchsia/test_runner.py b/chromium/build/fuchsia/test_runner.py
index 37992fd2fe1..5033a245c49 100755
--- a/chromium/build/fuchsia/test_runner.py
+++ b/chromium/build/fuchsia/test_runner.py
@@ -10,6 +10,7 @@ import argparse
import json
import logging
import os
+import runner_logs
import socket
import subprocess
import sys
@@ -20,6 +21,8 @@ from common_args import AddCommonArgs, ConfigureLogging, GetDeploymentTargetForA
from net_test_server import SetupTestServer
from run_package import RunPackage, RunPackageArgs, SystemLogReader
from runner_exceptions import HandleExceptionAndReturnExitCode
+from runner_logs import RunnerLogManager
+from symbolizer import BuildIdsPaths
DEFAULT_TEST_SERVER_CONCURRENCY = 4
@@ -140,37 +143,38 @@ def main():
child_args.extend(args.child_args)
try:
- with GetDeploymentTargetForArgs(args) as target:
- with SystemLogReader() as system_logger:
- target.Start()
-
- if args.system_log_file and args.system_log_file != '-':
- system_logger.Start(target, args.package, args.system_log_file)
-
- if args.test_launcher_filter_file:
- target.PutFile(args.test_launcher_filter_file, TEST_FILTER_PATH,
- for_package=args.package_name)
- child_args.append('--test-launcher-filter-file=' + TEST_FILTER_PATH)
-
- test_server = None
- if args.enable_test_server:
- assert test_concurrency
- test_server = SetupTestServer(target, test_concurrency,
- args.package_name)
-
- run_package_args = RunPackageArgs.FromCommonArgs(args)
- returncode = RunPackage(
- args.output_directory, target, args.package, args.package_name,
- child_args, run_package_args)
-
- if test_server:
- test_server.Stop()
-
- if args.test_launcher_summary_output:
- target.GetFile(TEST_RESULT_PATH, args.test_launcher_summary_output,
- for_package=args.package_name)
-
- return returncode
+ with GetDeploymentTargetForArgs(args) as target, \
+ SystemLogReader() as system_logger, \
+ RunnerLogManager(args.runner_logs_dir, BuildIdsPaths(args.package)):
+ target.Start()
+
+ if args.system_log_file and args.system_log_file != '-':
+ system_logger.Start(target, args.package, args.system_log_file)
+
+ if args.test_launcher_filter_file:
+ target.PutFile(args.test_launcher_filter_file, TEST_FILTER_PATH,
+ for_package=args.package_name)
+ child_args.append('--test-launcher-filter-file=' + TEST_FILTER_PATH)
+
+ test_server = None
+ if args.enable_test_server:
+ assert test_concurrency
+ test_server = SetupTestServer(target, test_concurrency,
+ args.package_name)
+
+ run_package_args = RunPackageArgs.FromCommonArgs(args)
+ returncode = RunPackage(
+ args.output_directory, target, args.package, args.package_name,
+ child_args, run_package_args)
+
+ if test_server:
+ test_server.Stop()
+
+ if args.test_launcher_summary_output:
+ target.GetFile(TEST_RESULT_PATH, args.test_launcher_summary_output,
+ for_package=args.package_name)
+
+ return returncode
except:
return HandleExceptionAndReturnExitCode()
diff --git a/chromium/build/fuchsia/update_sdk.py b/chromium/build/fuchsia/update_sdk.py
index 0a7752e4146..6e36f8dcdd7 100755
--- a/chromium/build/fuchsia/update_sdk.py
+++ b/chromium/build/fuchsia/update_sdk.py
@@ -26,60 +26,56 @@ SDK_SIGNATURE_FILE = '.hash'
EXTRA_SDK_HASH_PREFIX = ''
SDK_TARBALL_PATH_TEMPLATE = (
- 'gs://fuchsia/development/{sdk_hash}/sdk/{platform}-amd64/gn.tar.gz')
+ 'gs://{bucket}/development/{sdk_hash}/sdk/{platform}-amd64/gn.tar.gz')
-def GetSdkGeneration(hash):
+def ReadFile(filename):
+ with open(os.path.join(os.path.dirname(__file__), filename), 'r') as f:
+ return f.read()
+
+
+def GetCloudStorageBucket():
+ return ReadFile('sdk-bucket.txt').strip()
+
+
+def GetSdkHash(bucket):
+ hashes = GetSdkHashList()
+ return max(hashes, key=lambda sdk:GetSdkGeneration(bucket, sdk)) if hashes else None
+
+
+def GetSdkHashList():
+ """Read filename entries from sdk-hash-files.list (one per line), substitute
+ {platform} in each entry if present, and read from each filename."""
+ platform = GetHostOsFromPlatform()
+ filenames = [
+ line.strip() for line in ReadFile('sdk-hash-files.list').replace(
+ '{platform}', platform).splitlines()
+ ]
+ sdk_hashes = [ReadFile(filename).strip() for filename in filenames]
+ return sdk_hashes
+
+
+def GetSdkGeneration(bucket, hash):
if not hash:
return None
+ sdk_path = GetSdkTarballPath(bucket, hash)
cmd = [
os.path.join(find_depot_tools.DEPOT_TOOLS_PATH, 'gsutil.py'), 'ls', '-L',
- GetSdkTarballForPlatformAndHash(hash)
+ sdk_path
]
logging.debug("Running '%s'", " ".join(cmd))
sdk_details = subprocess.check_output(cmd)
m = re.search('Generation:\s*(\d*)', sdk_details)
if not m:
- return None
+ raise RuntimeError('Could not find SDK generation for {sdk_path}'.format(
+ sdk_path=sdk_path))
return int(m.group(1))
-def GetSdkHashForPlatform():
- filename = '{platform}.sdk.sha1'.format(platform = GetHostOsFromPlatform())
-
- # Get the hash of the SDK in chromium.
- sdk_hash = None
- hash_file = os.path.join(os.path.dirname(__file__), filename)
- with open(hash_file, 'r') as f:
- sdk_hash = f.read().strip()
-
- # Get the hash of the SDK with the extra prefix.
- extra_sdk_hash = None
- if EXTRA_SDK_HASH_PREFIX:
- extra_hash_file = os.path.join(os.path.dirname(__file__),
- EXTRA_SDK_HASH_PREFIX + filename)
- with open(extra_hash_file, 'r') as f:
- extra_sdk_hash = f.read().strip()
-
- # If both files are empty, return an error.
- if not sdk_hash and not extra_sdk_hash:
- logging.error(
- 'No SHA1 found in {} or {}'.format(hash_file, extra_hash_file),
- file=sys.stderr)
- return 1
-
- # Return the newer SDK based on the generation number.
- sdk_generation = GetSdkGeneration(sdk_hash)
- extra_sdk_generation = GetSdkGeneration(extra_sdk_hash)
- if extra_sdk_generation > sdk_generation:
- return extra_sdk_hash
- return sdk_hash
-
-
-def GetSdkTarballForPlatformAndHash(sdk_hash):
+def GetSdkTarballPath(bucket, sdk_hash):
return SDK_TARBALL_PATH_TEMPLATE.format(
- sdk_hash=sdk_hash, platform=GetHostOsFromPlatform())
+ bucket=bucket, sdk_hash=sdk_hash, platform=GetHostOsFromPlatform())
def GetSdkSignature(sdk_hash, boot_images):
@@ -124,7 +120,7 @@ def DownloadAndUnpackFromCloudStorage(url, output_dir):
task.stderr.read())
-def DownloadSdkBootImages(sdk_hash, boot_image_names):
+def DownloadSdkBootImages(bucket, sdk_hash, boot_image_names):
if not boot_image_names:
return
@@ -151,10 +147,9 @@ def DownloadSdkBootImages(sdk_hash, boot_image_names):
logging.info(
'Downloading Fuchsia boot images for %s.%s...' % (device_type, arch))
- images_tarball_url = \
- 'gs://fuchsia/development/{sdk_hash}/images/'\
+ images_tarball_url = 'gs://{bucket}/development/{sdk_hash}/images/'\
'{device_type}-{arch}.tgz'.format(
- sdk_hash=sdk_hash, device_type=device_type, arch=arch)
+ bucket=bucket, sdk_hash=sdk_hash, device_type=device_type, arch=arch)
DownloadAndUnpackFromCloudStorage(images_tarball_url, image_output_dir)
@@ -178,7 +173,8 @@ def main():
except:
return 0
- sdk_hash = GetSdkHashForPlatform()
+ bucket = GetCloudStorageBucket()
+ sdk_hash = GetSdkHash(bucket)
if not sdk_hash:
return 1
@@ -193,7 +189,7 @@ def main():
EnsureDirExists(SDK_ROOT)
DownloadAndUnpackFromCloudStorage(
- GetSdkTarballForPlatformAndHash(sdk_hash), SDK_ROOT)
+ GetSdkTarballPath(bucket, sdk_hash), SDK_ROOT)
# Clean out the boot images directory.
if (os.path.exists(IMAGES_ROOT)):
@@ -204,7 +200,7 @@ def main():
# Ensure that the boot images are downloaded for this SDK.
# If the developer opted into downloading hardware boot images in their
# .gclient file, then only the hardware boot images will be downloaded.
- DownloadSdkBootImages(sdk_hash, args.boot_images)
+ DownloadSdkBootImages(bucket, sdk_hash, args.boot_images)
except subprocess.CalledProcessError as e:
logging.error(("command '%s' failed with status %d.%s"), " ".join(e.cmd),
e.returncode, " Details: " + e.output if e.output else "")