diff options
author | Allan Sandfeld Jensen <allan.jensen@qt.io> | 2020-10-12 14:27:29 +0200 |
---|---|---|
committer | Allan Sandfeld Jensen <allan.jensen@qt.io> | 2020-10-13 09:35:20 +0000 |
commit | c30a6232df03e1efbd9f3b226777b07e087a1122 (patch) | |
tree | e992f45784689f373bcc38d1b79a239ebe17ee23 /chromium/build/fuchsia | |
parent | 7b5b123ac58f58ffde0f4f6e488bcd09aa4decd3 (diff) | |
download | qtwebengine-chromium-85-based.tar.gz |
BASELINE: Update Chromium to 85.0.4183.14085-based
Change-Id: Iaa42f4680837c57725b1344f108c0196741f6057
Reviewed-by: Allan Sandfeld Jensen <allan.jensen@qt.io>
Diffstat (limited to 'chromium/build/fuchsia')
-rw-r--r-- | chromium/build/fuchsia/aemu_target.py | 20 | ||||
-rw-r--r-- | chromium/build/fuchsia/common_args.py | 2 | ||||
-rw-r--r-- | chromium/build/fuchsia/device_target.py | 6 | ||||
-rw-r--r-- | chromium/build/fuchsia/emu_target.py | 46 | ||||
-rw-r--r-- | chromium/build/fuchsia/linux.sdk.sha1 | 2 | ||||
-rw-r--r-- | chromium/build/fuchsia/mac.sdk.sha1 | 2 | ||||
-rw-r--r-- | chromium/build/fuchsia/net_test_server.py | 1 | ||||
-rw-r--r-- | chromium/build/fuchsia/qemu_image.py | 16 | ||||
-rw-r--r-- | chromium/build/fuchsia/qemu_target.py | 8 | ||||
-rw-r--r-- | chromium/build/fuchsia/remote_cmd.py | 23 | ||||
-rw-r--r-- | chromium/build/fuchsia/run_package.py | 20 | ||||
-rw-r--r-- | chromium/build/fuchsia/runner_exceptions.py | 35 | ||||
-rw-r--r-- | chromium/build/fuchsia/runner_logs.py | 96 | ||||
-rw-r--r-- | chromium/build/fuchsia/sdk-bucket.txt | 1 | ||||
-rw-r--r-- | chromium/build/fuchsia/sdk-hash-files.list | 1 | ||||
-rw-r--r-- | chromium/build/fuchsia/symbolizer.py | 9 | ||||
-rw-r--r-- | chromium/build/fuchsia/target.py | 10 | ||||
-rwxr-xr-x | chromium/build/fuchsia/test_runner.py | 66 | ||||
-rwxr-xr-x | chromium/build/fuchsia/update_sdk.py | 86 |
19 files changed, 302 insertions, 148 deletions
diff --git a/chromium/build/fuchsia/aemu_target.py b/chromium/build/fuchsia/aemu_target.py index 0e3bc8f4ead..e997541ab8c 100644 --- a/chromium/build/fuchsia/aemu_target.py +++ b/chromium/build/fuchsia/aemu_target.py @@ -11,7 +11,9 @@ import logging from common import GetEmuRootForPlatform + class AemuTarget(qemu_target.QemuTarget): + def __init__(self, output_dir, target_cpu, system_log_file, emu_type, cpu_cores, require_kvm, ram_size_mb, enable_graphics, hardware_gpu): @@ -60,7 +62,7 @@ class AemuTarget(qemu_target.QemuTarget): # All args after -fuchsia flag gets passed to QEMU aemu_command.extend([ '-feature', aemu_features, '-window-size', '1024x600', '-gpu', - gpu_target, '-fuchsia' + gpu_target, '-verbose', '-fuchsia' ]) aemu_command.extend(self._BuildQemuConfig()) @@ -73,3 +75,19 @@ class AemuTarget(qemu_target.QemuTarget): '-device', 'ich9-ahci,id=ahci']) logging.info(' '.join(aemu_command)) return aemu_command + + def _GetVulkanIcdFile(self): + return os.path.join(GetEmuRootForPlatform(self._emu_type), 'lib64', + 'vulkan', 'vk_swiftshader_icd.json') + + def _SetEnv(self): + env = os.environ.copy() + aemu_logging_env = { + "ANDROID_EMU_VK_NO_CLEANUP": "1", + "ANDROID_EMUGL_LOG_PRINT": "1", + "ANDROID_EMUGL_VERBOSE": "1", + "VK_ICD_FILENAMES": self._GetVulkanIcdFile(), + "VK_LOADER_DEBUG": "info,error", + } + env.update(aemu_logging_env) + return env diff --git a/chromium/build/fuchsia/common_args.py b/chromium/build/fuchsia/common_args.py index 24544f2c305..855bc1149c3 100644 --- a/chromium/build/fuchsia/common_args.py +++ b/chromium/build/fuchsia/common_args.py @@ -66,6 +66,8 @@ def AddCommonArgs(arg_parser): help='Path to a Fuchsia build output directory. ' 'Equivalent to setting --ssh_config and ' '---os-check=ignore') + common_args.add_argument('--runner-logs-dir', + help='Directory to write test runner logs to.') common_args.add_argument('--system-log-file', help='File to write system logs to. Specify - to ' 'log to stdout.') diff --git a/chromium/build/fuchsia/device_target.py b/chromium/build/fuchsia/device_target.py index 8a5f7d7c5b9..9ba173ba95a 100644 --- a/chromium/build/fuchsia/device_target.py +++ b/chromium/build/fuchsia/device_target.py @@ -141,7 +141,7 @@ class DeviceTarget(target.Target): or waits up to |timeout| seconds and returns False if the device couldn't be found.""" - dev_finder_path = GetHostToolPathFromPlatform('dev_finder') + dev_finder_path = GetHostToolPathFromPlatform('device-finder') if self._node_name: command = [dev_finder_path, 'resolve', @@ -163,13 +163,13 @@ class DeviceTarget(target.Target): return False if self._node_name: - # Handle the result of "dev_finder resolve". + # Handle the result of "device-finder resolve". self._host = output.pop().strip() else: name_host_pairs = [x.strip().split(' ') for x in output] - # Handle the output of "dev_finder list". + # Handle the output of "device-finder list". if len(name_host_pairs) > 1: print('More than one device was discovered on the network.') print('Use --node-name <name> to specify the device to use.') diff --git a/chromium/build/fuchsia/emu_target.py b/chromium/build/fuchsia/emu_target.py index 83e10167914..6ccdd832fbd 100644 --- a/chromium/build/fuchsia/emu_target.py +++ b/chromium/build/fuchsia/emu_target.py @@ -8,6 +8,7 @@ import amber_repo import boot_data import logging import os +import runner_logs import subprocess import sys import target @@ -34,6 +35,9 @@ class EmuTarget(target.Target): """Build the command that will be run to start Fuchsia in the emulator.""" pass + def _SetEnv(self): + return os.environ.copy() + # Used by the context manager to ensure that the emulator is killed when # the Python process exits. def __exit__(self, exc_type, exc_val, exc_tb): @@ -52,27 +56,28 @@ class EmuTarget(target.Target): # Zircon sends debug logs to serial port (see kernel.serial=legacy flag # above). Serial port is redirected to a file through emulator stdout. - # Unless a |_system_log_file| is explicitly set, we output the kernel serial - # log to a temporary file, and print that out if we are unable to connect to + # Unless runner_pogs are enabled, we output the kernel serial log + # to a temporary file, and print that out if we are unable to connect to # the emulator guest, to make it easier to diagnose connectivity issues. - temporary_system_log_file = None - if self._system_log_file: - stdout = self._system_log_file - stderr = subprocess.STDOUT + temporary_log_file = None + if runner_logs.IsEnabled(): + stdout = runner_logs.FileStreamFor('serial_log') else: - temporary_system_log_file = tempfile.NamedTemporaryFile('w') - stdout = temporary_system_log_file - stderr = sys.stderr + temporary_log_file = tempfile.NamedTemporaryFile('w') + stdout = temporary_log_file - self._emu_process = subprocess.Popen(emu_command, stdin=open(os.devnull), - stdout=stdout, stderr=stderr) + self._emu_process = subprocess.Popen(emu_command, + stdin=open(os.devnull), + stdout=stdout, + stderr=subprocess.STDOUT, + env=self._SetEnv()) try: - self._WaitUntilReady(); + self._WaitUntilReady() except target.FuchsiaTargetException: - if temporary_system_log_file: + if temporary_log_file: logging.info('Kernel logs:\n' + - open(temporary_system_log_file.name, 'r').read()) + open(temporary_log_file.name, 'r').read()) raise def GetAmberRepo(self): @@ -86,15 +91,18 @@ class EmuTarget(target.Target): logging.error('%s did not start' % (self._GetEmulatorName())) return returncode = self._emu_process.poll() - if returncode: - logging.error('%s quit unexpectedly with exit code %d' % - (self._GetEmulatorName(), returncode)) + if returncode == None: + logging.info('Shutting down %s' % (self._GetEmulatorName())) + self._emu_process.kill() elif returncode == 0: logging.info('%s quit unexpectedly without errors' % self._GetEmulatorName()) + elif returncode < 0: + logging.error('%s was terminated by signal %d' % + (self._GetEmulatorName(), -returncode)) else: - logging.info('Shutting down %s' % (self._GetEmulatorName())) - self._emu_process.kill() + logging.error('%s quit unexpectedly with exit code %d' % + (self._GetEmulatorName(), returncode)) def _IsEmuStillRunning(self): if not self._emu_process: diff --git a/chromium/build/fuchsia/linux.sdk.sha1 b/chromium/build/fuchsia/linux.sdk.sha1 index 7552ab1c396..a7cac626a07 100644 --- a/chromium/build/fuchsia/linux.sdk.sha1 +++ b/chromium/build/fuchsia/linux.sdk.sha1 @@ -1 +1 @@ -0.20200514.2.1 +0.20200625.3.1 diff --git a/chromium/build/fuchsia/mac.sdk.sha1 b/chromium/build/fuchsia/mac.sdk.sha1 index da07900e95d..a7cac626a07 100644 --- a/chromium/build/fuchsia/mac.sdk.sha1 +++ b/chromium/build/fuchsia/mac.sdk.sha1 @@ -1 +1 @@ -0.20200514.1.1 +0.20200625.3.1 diff --git a/chromium/build/fuchsia/net_test_server.py b/chromium/build/fuchsia/net_test_server.py index 94f71635433..60ad78f60fa 100644 --- a/chromium/build/fuchsia/net_test_server.py +++ b/chromium/build/fuchsia/net_test_server.py @@ -43,6 +43,7 @@ class SSHPortForwarder(chrome_test_server_spawner.PortForwarder): '-NT', '-O', 'cancel', '-R', '0:localhost:%d' % host_port] task = self._target.RunCommandPiped([], ssh_args=forwarding_args, + stdout=open(os.devnull, 'w'), stderr=subprocess.PIPE) task.wait() if task.returncode != 0: diff --git a/chromium/build/fuchsia/qemu_image.py b/chromium/build/fuchsia/qemu_image.py index 5126074fe02..ab5e040acbd 100644 --- a/chromium/build/fuchsia/qemu_image.py +++ b/chromium/build/fuchsia/qemu_image.py @@ -18,6 +18,7 @@ TODO(crbug.com/1046861): Remove this workaround when the bug is fixed. import logging import subprocess +import tempfile import time @@ -33,7 +34,9 @@ def _ExecQemuImgWithTimeout(command): """ logging.info('qemu-img starting') - p = subprocess.Popen(command) + command_output_file = tempfile.NamedTemporaryFile('w') + p = subprocess.Popen(command, stdout=command_output_file, + stderr=subprocess.STDOUT) start_sec = time.time() while p.poll() is None and time.time() - start_sec < QEMU_IMG_TIMEOUT_SEC: time.sleep(1) @@ -41,10 +44,17 @@ def _ExecQemuImgWithTimeout(command): logging.info('qemu-img duration: %f' % float(stop_sec - start_sec)) if p.poll() is None: + returncode = None p.kill() - return None + p.wait() + else: + returncode = p.returncode - return p.returncode + log_level = logging.WARN if returncode else logging.DEBUG + for line in open(command_output_file.name, 'r'): + logging.log(log_level, 'qemu-img stdout: ' + line.strip()) + + return returncode def ExecQemuImgWithRetry(command): diff --git a/chromium/build/fuchsia/qemu_target.py b/chromium/build/fuchsia/qemu_target.py index bdc397e797b..96d7465bf9e 100644 --- a/chromium/build/fuchsia/qemu_target.py +++ b/chromium/build/fuchsia/qemu_target.py @@ -55,7 +55,11 @@ class QemuTarget(emu_target.EmuTarget): if kvm_supported and same_arch: return True elif self._require_kvm: - raise FuchsiaTargetException('KVM required but unavailable.') + if same_arch: + raise FuchsiaTargetException('KVM required but unavailable.') + else: + raise FuchsiaTargetException('KVM unavailable when CPU architecture of'\ + ' host is different from that of target.') else: return False @@ -119,7 +123,7 @@ class QemuTarget(emu_target.EmuTarget): if self._target_cpu == 'arm64': kvm_command.append('host') else: - kvm_command.append('host,migratable=no') + kvm_command.append('host,migratable=no,+invtsc') else: logging.warning('Unable to launch %s with KVM acceleration.' % (self._emu_type) + diff --git a/chromium/build/fuchsia/remote_cmd.py b/chromium/build/fuchsia/remote_cmd.py index 93ce32ce287..019c2dc9abd 100644 --- a/chromium/build/fuchsia/remote_cmd.py +++ b/chromium/build/fuchsia/remote_cmd.py @@ -5,7 +5,6 @@ import logging import os import subprocess -import sys import threading _SSH = ['ssh'] @@ -57,31 +56,36 @@ class CommandRunner(object): _SSH_LOGGER.debug('ssh exec: ' + ' '.join(ssh_command)) if silent: devnull = open(os.devnull, 'w') - process = subprocess.Popen(ssh_command, stderr=devnull, stdout=devnull) + process = subprocess.Popen(ssh_command, stdout=devnull, stderr=devnull) else: - process = subprocess.Popen(ssh_command) + process = subprocess.Popen(ssh_command, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) timeout_timer = None if timeout_secs: timeout_timer = threading.Timer(timeout_secs, process.kill) timeout_timer.start() - process.wait() + if not silent: + for line in process.stdout: + print(line) + process.wait() if timeout_timer: timeout_timer.cancel() - if process.returncode == -9: raise Exception('Timeout when executing \"%s\".' % ' '.join(command)) return process.returncode - def RunCommandPiped(self, command = None, ssh_args = None, **kwargs): + def RunCommandPiped(self, command, stdout, stderr, ssh_args = None, **kwargs): """Executes an SSH command on the remote host and returns a process object with access to the command's stdio streams. Does not block. command: A list of strings containing the command and its arguments. + stdout: subprocess stdout. Must not be None. + stderr: subprocess stderr. Must not be None. ssh_args: Arguments that will be passed to SSH. kwargs: A dictionary of parameters to be passed to subprocess.Popen(). The parameters can be used to override stdin and stdout, for @@ -89,14 +93,15 @@ class CommandRunner(object): Returns a Popen object for the command.""" - if not command: - command = [] + if not stdout or not stderr: + raise Exception('Stdout/stderr must be specified explicitly') + if not ssh_args: ssh_args = [] ssh_command = self._GetSshCommandLinePrefix() + ssh_args + ['--'] + command _SSH_LOGGER.debug(' '.join(ssh_command)) - return subprocess.Popen(ssh_command, **kwargs) + return subprocess.Popen(ssh_command, stdout=stdout, stderr=stderr, **kwargs) def RunScp(self, sources, dest, direction, recursive=False): diff --git a/chromium/build/fuchsia/run_package.py b/chromium/build/fuchsia/run_package.py index 62bbd3b8fd8..ed2cca3bf12 100644 --- a/chromium/build/fuchsia/run_package.py +++ b/chromium/build/fuchsia/run_package.py @@ -16,12 +16,10 @@ import re import select import subprocess import sys -import time import threading import uuid -from symbolizer import RunSymbolizer -from symbolizer import SymbolizerFilter +from symbolizer import BuildIdsPaths, RunSymbolizer, SymbolizerFilter FAR = common.GetHostToolPathFromPlatform('far') @@ -34,16 +32,8 @@ def _AttachKernelLogReader(target): logging.info('Attaching kernel logger.') return target.RunCommandPiped(['dlog', '-f'], stdin=open(os.devnull, 'r'), - stdout=subprocess.PIPE) - - -def _BuildIdsPaths(package_paths): - """Generate build ids paths for symbolizer processes.""" - build_ids_paths = map( - lambda package_path: os.path.join( - os.path.dirname(package_path), 'ids.txt'), - package_paths) - return build_ids_paths + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) class SystemLogReader(object): @@ -77,7 +67,7 @@ class SystemLogReader(object): self._system_log = open(system_log_file,'w', buffering=1) self._symbolizer_proc = RunSymbolizer(self._listener_proc.stdout, self._system_log, - _BuildIdsPaths(package_paths)) + BuildIdsPaths(package_paths)) class MergedInputStream(object): @@ -239,7 +229,7 @@ def RunPackage(output_dir, target, package_paths, package_name, # Run the log data through the symbolizer process. output_stream = SymbolizerFilter(output_stream, - _BuildIdsPaths(package_paths)) + BuildIdsPaths(package_paths)) for next_line in output_stream: print(next_line.rstrip()) diff --git a/chromium/build/fuchsia/runner_exceptions.py b/chromium/build/fuchsia/runner_exceptions.py index cedf99bbd7a..03f872e453f 100644 --- a/chromium/build/fuchsia/runner_exceptions.py +++ b/chromium/build/fuchsia/runner_exceptions.py @@ -7,6 +7,7 @@ This makes it easier to query build tables for particular error types as exit codes are visible to queries while exception stack traces are not.""" +import errno import fcntl import logging import os @@ -23,17 +24,15 @@ def _PrintException(value, trace): print(str(value)) -# TODO(crbug.com/1080858): Delete function when the stdout print bug is fixed. -def _LogStdoutBlockingStatus(): - """Log whether sys.stdout is blocking or non-blocking. +def IsStdoutBlocking(): + """Returns True if sys.stdout is blocking or False if non-blocking. - It should be blocking, but there are intermittent IO errors that suggest - that it is set to non-blocking at times during test runs.""" + sys.stdout should always be blocking. Non-blocking is associated with + intermittent IOErrors (crbug.com/1080858). + """ - if fcntl.fcntl(sys.stdout, fcntl.F_GETFD) & os.O_NONBLOCK: - logging.error('sys.stdout is non-blocking') - else: - logging.info('sys.stdout is blocking') + nonblocking = fcntl.fcntl(sys.stdout, fcntl.F_GETFL) & os.O_NONBLOCK + return not nonblocking def HandleExceptionAndReturnExitCode(): @@ -57,19 +56,23 @@ def HandleExceptionAndReturnExitCode(): if type is FuchsiaTargetException: if 'ssh' in str(value).lower(): - print('Error: FuchsiaTargetException: SSH to Fuchsia target failed.') - return 65 + print('Error: FuchsiaTargetException: SSH to Fuchsia target failed.') + return 65 return 64 elif type is IOError: - if value.errno == 11: - print('Info: Python print to sys.stdout probably failed') - _LogStdoutBlockingStatus() - return 73 + if value.errno == errno.EAGAIN: + logging.info('Python print to sys.stdout probably failed') + if not IsStdoutBlocking(): + logging.warn('sys.stdout is non-blocking') + return 73 return 72 elif type is subprocess.CalledProcessError: - if value.cmd[0] == 'scp': + if os.path.basename(value.cmd[0]) == 'scp': print('Error: scp operation failed - %s' % str(value)) return 81 + if os.path.basename(value.cmd[0]) == 'qemu-img': + print('Error: qemu-img fuchsia image generation failed.') + return 82 return 80 else: return 1 diff --git a/chromium/build/fuchsia/runner_logs.py b/chromium/build/fuchsia/runner_logs.py new file mode 100644 index 00000000000..19da4f5a045 --- /dev/null +++ b/chromium/build/fuchsia/runner_logs.py @@ -0,0 +1,96 @@ +# Copyright 2020 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Creates and manages test runner log file objects. + +Provides a context manager object for use in a with statement +and a module level FileStreamFor function for use by clients. +""" + +import collections +import multiprocessing +import os + +from symbolizer import RunSymbolizer + +SYMBOLIZED_SUFFIX = '.symbolized' + +_RunnerLogEntry = collections.namedtuple( + '_RunnerLogEntry', ['name', 'log_file', 'path', 'symbolize']) + +# Module singleton variable. +_instance = None + + +class RunnerLogManager(object): + """ Runner logs object for use in a with statement.""" + + def __init__(self, log_dir, build_ids_files): + global _instance + if _instance: + raise Exception('Only one RunnerLogManager can be instantiated') + + self._log_dir = log_dir + self._build_ids_files = build_ids_files + self._runner_logs = [] + + if self._log_dir: + os.makedirs(self._log_dir) + + _instance = self + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + pool = multiprocessing.Pool(4) + for log_entry in self._runner_logs: + pool.apply_async(_FinalizeLog, (log_entry, self._build_ids_files)) + pool.close() + pool.join() + _instance = None + + + def _FileStreamFor(self, name, symbolize): + if any(elem.name == name for elem in self._runner_logs): + raise Exception('RunnerLogManager can only open "%" once' % name) + + path = os.path.join(self._log_dir, name) if self._log_dir else os.devnull + log_file = open(path, 'w') + + self._runner_logs.append(_RunnerLogEntry(name, log_file, path, symbolize)) + + return log_file + + +def _FinalizeLog(log_entry, build_ids_files): + log_entry.log_file.close() + + if log_entry.symbolize: + input_file = open(log_entry.path, 'r') + output_file = open(log_entry.path + SYMBOLIZED_SUFFIX, 'w') + proc = RunSymbolizer(input_file, output_file, build_ids_files) + proc.wait() + output_file.close() + input_file.close() + + +def IsEnabled(): + """Returns True if the RunnerLogManager has been created, or False if not.""" + + return _instance is not None and _instance._log_dir is not None + + +def FileStreamFor(name, symbolize=False): + """Opens a test runner file stream in the test runner log directory. + + If no test runner log directory is specified, output is discarded. + + name: log file name + symbolize: if True, make a symbolized copy of the log after closing it. + + Returns an opened log file object.""" + + return _instance._FileStreamFor(name, symbolize) if IsEnabled() else open( + os.devnull, 'w') diff --git a/chromium/build/fuchsia/sdk-bucket.txt b/chromium/build/fuchsia/sdk-bucket.txt new file mode 100644 index 00000000000..9f860bce1d7 --- /dev/null +++ b/chromium/build/fuchsia/sdk-bucket.txt @@ -0,0 +1 @@ +fuchsia diff --git a/chromium/build/fuchsia/sdk-hash-files.list b/chromium/build/fuchsia/sdk-hash-files.list new file mode 100644 index 00000000000..6f37bcd9f77 --- /dev/null +++ b/chromium/build/fuchsia/sdk-hash-files.list @@ -0,0 +1 @@ +{platform}.sdk.sha1 diff --git a/chromium/build/fuchsia/symbolizer.py b/chromium/build/fuchsia/symbolizer.py index 1432feea3f0..325cc23fc61 100644 --- a/chromium/build/fuchsia/symbolizer.py +++ b/chromium/build/fuchsia/symbolizer.py @@ -14,6 +14,15 @@ from common import GetHostToolPathFromPlatform ARM64_DOCKER_LLVM_SYMBOLIZER_PATH = os.path.join('/', 'usr', 'lib', 'llvm-3.8', 'bin', 'llvm-symbolizer') +def BuildIdsPaths(package_paths): + """Generate build ids paths for symbolizer processes.""" + build_ids_paths = map( + lambda package_path: os.path.join( + os.path.dirname(package_path), 'ids.txt'), + package_paths) + return build_ids_paths + + def RunSymbolizer(input_file, output_file, build_ids_files): """Starts a symbolizer process. diff --git a/chromium/build/fuchsia/target.py b/chromium/build/fuchsia/target.py index d8c1741f7d8..254c5fd5789 100644 --- a/chromium/build/fuchsia/target.py +++ b/chromium/build/fuchsia/target.py @@ -8,6 +8,7 @@ import json import logging import os import remote_cmd +import runner_logs import shutil import subprocess import sys @@ -21,7 +22,7 @@ _ATTACH_RETRY_SECONDS = 120 # Amount of time to wait for Amber to complete package installation, as a # mitigation against hangs due to amber/network-related failures. -_INSTALL_TIMEOUT_SECS = 5 * 60 +_INSTALL_TIMEOUT_SECS = 10 * 60 def _GetPackageUri(package_name): @@ -216,9 +217,14 @@ class Target(object): host, port = self._GetEndpoint() end_time = time.time() + _ATTACH_RETRY_SECONDS + ssh_diagnostic_log = runner_logs.FileStreamFor('ssh_diagnostic_log') while time.time() < end_time: runner = remote_cmd.CommandRunner(self._GetSshConfigPath(), host, port) - if runner.RunCommand(['true'], True) == 0: + ssh_proc = runner.RunCommandPiped(['true'], + ssh_args=['-v'], + stdout=ssh_diagnostic_log, + stderr=subprocess.STDOUT) + if ssh_proc.wait() == 0: logging.info('Connected!') self._started = True return True diff --git a/chromium/build/fuchsia/test_runner.py b/chromium/build/fuchsia/test_runner.py index 37992fd2fe1..5033a245c49 100755 --- a/chromium/build/fuchsia/test_runner.py +++ b/chromium/build/fuchsia/test_runner.py @@ -10,6 +10,7 @@ import argparse import json import logging import os +import runner_logs import socket import subprocess import sys @@ -20,6 +21,8 @@ from common_args import AddCommonArgs, ConfigureLogging, GetDeploymentTargetForA from net_test_server import SetupTestServer from run_package import RunPackage, RunPackageArgs, SystemLogReader from runner_exceptions import HandleExceptionAndReturnExitCode +from runner_logs import RunnerLogManager +from symbolizer import BuildIdsPaths DEFAULT_TEST_SERVER_CONCURRENCY = 4 @@ -140,37 +143,38 @@ def main(): child_args.extend(args.child_args) try: - with GetDeploymentTargetForArgs(args) as target: - with SystemLogReader() as system_logger: - target.Start() - - if args.system_log_file and args.system_log_file != '-': - system_logger.Start(target, args.package, args.system_log_file) - - if args.test_launcher_filter_file: - target.PutFile(args.test_launcher_filter_file, TEST_FILTER_PATH, - for_package=args.package_name) - child_args.append('--test-launcher-filter-file=' + TEST_FILTER_PATH) - - test_server = None - if args.enable_test_server: - assert test_concurrency - test_server = SetupTestServer(target, test_concurrency, - args.package_name) - - run_package_args = RunPackageArgs.FromCommonArgs(args) - returncode = RunPackage( - args.output_directory, target, args.package, args.package_name, - child_args, run_package_args) - - if test_server: - test_server.Stop() - - if args.test_launcher_summary_output: - target.GetFile(TEST_RESULT_PATH, args.test_launcher_summary_output, - for_package=args.package_name) - - return returncode + with GetDeploymentTargetForArgs(args) as target, \ + SystemLogReader() as system_logger, \ + RunnerLogManager(args.runner_logs_dir, BuildIdsPaths(args.package)): + target.Start() + + if args.system_log_file and args.system_log_file != '-': + system_logger.Start(target, args.package, args.system_log_file) + + if args.test_launcher_filter_file: + target.PutFile(args.test_launcher_filter_file, TEST_FILTER_PATH, + for_package=args.package_name) + child_args.append('--test-launcher-filter-file=' + TEST_FILTER_PATH) + + test_server = None + if args.enable_test_server: + assert test_concurrency + test_server = SetupTestServer(target, test_concurrency, + args.package_name) + + run_package_args = RunPackageArgs.FromCommonArgs(args) + returncode = RunPackage( + args.output_directory, target, args.package, args.package_name, + child_args, run_package_args) + + if test_server: + test_server.Stop() + + if args.test_launcher_summary_output: + target.GetFile(TEST_RESULT_PATH, args.test_launcher_summary_output, + for_package=args.package_name) + + return returncode except: return HandleExceptionAndReturnExitCode() diff --git a/chromium/build/fuchsia/update_sdk.py b/chromium/build/fuchsia/update_sdk.py index 0a7752e4146..6e36f8dcdd7 100755 --- a/chromium/build/fuchsia/update_sdk.py +++ b/chromium/build/fuchsia/update_sdk.py @@ -26,60 +26,56 @@ SDK_SIGNATURE_FILE = '.hash' EXTRA_SDK_HASH_PREFIX = '' SDK_TARBALL_PATH_TEMPLATE = ( - 'gs://fuchsia/development/{sdk_hash}/sdk/{platform}-amd64/gn.tar.gz') + 'gs://{bucket}/development/{sdk_hash}/sdk/{platform}-amd64/gn.tar.gz') -def GetSdkGeneration(hash): +def ReadFile(filename): + with open(os.path.join(os.path.dirname(__file__), filename), 'r') as f: + return f.read() + + +def GetCloudStorageBucket(): + return ReadFile('sdk-bucket.txt').strip() + + +def GetSdkHash(bucket): + hashes = GetSdkHashList() + return max(hashes, key=lambda sdk:GetSdkGeneration(bucket, sdk)) if hashes else None + + +def GetSdkHashList(): + """Read filename entries from sdk-hash-files.list (one per line), substitute + {platform} in each entry if present, and read from each filename.""" + platform = GetHostOsFromPlatform() + filenames = [ + line.strip() for line in ReadFile('sdk-hash-files.list').replace( + '{platform}', platform).splitlines() + ] + sdk_hashes = [ReadFile(filename).strip() for filename in filenames] + return sdk_hashes + + +def GetSdkGeneration(bucket, hash): if not hash: return None + sdk_path = GetSdkTarballPath(bucket, hash) cmd = [ os.path.join(find_depot_tools.DEPOT_TOOLS_PATH, 'gsutil.py'), 'ls', '-L', - GetSdkTarballForPlatformAndHash(hash) + sdk_path ] logging.debug("Running '%s'", " ".join(cmd)) sdk_details = subprocess.check_output(cmd) m = re.search('Generation:\s*(\d*)', sdk_details) if not m: - return None + raise RuntimeError('Could not find SDK generation for {sdk_path}'.format( + sdk_path=sdk_path)) return int(m.group(1)) -def GetSdkHashForPlatform(): - filename = '{platform}.sdk.sha1'.format(platform = GetHostOsFromPlatform()) - - # Get the hash of the SDK in chromium. - sdk_hash = None - hash_file = os.path.join(os.path.dirname(__file__), filename) - with open(hash_file, 'r') as f: - sdk_hash = f.read().strip() - - # Get the hash of the SDK with the extra prefix. - extra_sdk_hash = None - if EXTRA_SDK_HASH_PREFIX: - extra_hash_file = os.path.join(os.path.dirname(__file__), - EXTRA_SDK_HASH_PREFIX + filename) - with open(extra_hash_file, 'r') as f: - extra_sdk_hash = f.read().strip() - - # If both files are empty, return an error. - if not sdk_hash and not extra_sdk_hash: - logging.error( - 'No SHA1 found in {} or {}'.format(hash_file, extra_hash_file), - file=sys.stderr) - return 1 - - # Return the newer SDK based on the generation number. - sdk_generation = GetSdkGeneration(sdk_hash) - extra_sdk_generation = GetSdkGeneration(extra_sdk_hash) - if extra_sdk_generation > sdk_generation: - return extra_sdk_hash - return sdk_hash - - -def GetSdkTarballForPlatformAndHash(sdk_hash): +def GetSdkTarballPath(bucket, sdk_hash): return SDK_TARBALL_PATH_TEMPLATE.format( - sdk_hash=sdk_hash, platform=GetHostOsFromPlatform()) + bucket=bucket, sdk_hash=sdk_hash, platform=GetHostOsFromPlatform()) def GetSdkSignature(sdk_hash, boot_images): @@ -124,7 +120,7 @@ def DownloadAndUnpackFromCloudStorage(url, output_dir): task.stderr.read()) -def DownloadSdkBootImages(sdk_hash, boot_image_names): +def DownloadSdkBootImages(bucket, sdk_hash, boot_image_names): if not boot_image_names: return @@ -151,10 +147,9 @@ def DownloadSdkBootImages(sdk_hash, boot_image_names): logging.info( 'Downloading Fuchsia boot images for %s.%s...' % (device_type, arch)) - images_tarball_url = \ - 'gs://fuchsia/development/{sdk_hash}/images/'\ + images_tarball_url = 'gs://{bucket}/development/{sdk_hash}/images/'\ '{device_type}-{arch}.tgz'.format( - sdk_hash=sdk_hash, device_type=device_type, arch=arch) + bucket=bucket, sdk_hash=sdk_hash, device_type=device_type, arch=arch) DownloadAndUnpackFromCloudStorage(images_tarball_url, image_output_dir) @@ -178,7 +173,8 @@ def main(): except: return 0 - sdk_hash = GetSdkHashForPlatform() + bucket = GetCloudStorageBucket() + sdk_hash = GetSdkHash(bucket) if not sdk_hash: return 1 @@ -193,7 +189,7 @@ def main(): EnsureDirExists(SDK_ROOT) DownloadAndUnpackFromCloudStorage( - GetSdkTarballForPlatformAndHash(sdk_hash), SDK_ROOT) + GetSdkTarballPath(bucket, sdk_hash), SDK_ROOT) # Clean out the boot images directory. if (os.path.exists(IMAGES_ROOT)): @@ -204,7 +200,7 @@ def main(): # Ensure that the boot images are downloaded for this SDK. # If the developer opted into downloading hardware boot images in their # .gclient file, then only the hardware boot images will be downloaded. - DownloadSdkBootImages(sdk_hash, args.boot_images) + DownloadSdkBootImages(bucket, sdk_hash, args.boot_images) except subprocess.CalledProcessError as e: logging.error(("command '%s' failed with status %d.%s"), " ".join(e.cmd), e.returncode, " Details: " + e.output if e.output else "") |