summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorJack Rosenthal <jrosenth@chromium.org>2021-01-08 12:30:48 -0700
committerCommit Bot <commit-bot@chromium.org>2021-01-09 03:28:08 +0000
commit2fa27104aa0e97f3c750aa3b04acfc76db5e7123 (patch)
tree7271f546938af224721a621ebeee2a1369048e0e
parent407a3cfc7c7423b3f03289b094bc1dfd2082a3c4 (diff)
downloadchrome-ec-2fa27104aa0e97f3c750aa3b04acfc76db5e7123.tar.gz
zephyr: copy zmake to platform/ec
This copies zmake into platform/ec/zephyr/zmake, as explained in go/zephyr-fewer-repos. Follow-on CLs will be submitted to: - Update the chromeos-base/zephyr-build-tools ebuild to reference this directory instead of the one in zephyr-chrome. - Remove the copy of zmake in zephyr-chrome. Those interested in the git history of this code prior to it being moved to platform/ec can find it here: https://chromium.googlesource.com/chromiumos/platform/zephyr-chrome/+log/bacea2e3e62c41000e5bdb4ed6433f24386d14bf/util BUG=b:177003034 BRANCH=none TEST=emerge with new path (requires follow-on CL) Signed-off-by: Jack Rosenthal <jrosenth@chromium.org> Change-Id: Ia957b3e35ce3b732968ebf8df603ef13298cc6b3 Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/platform/ec/+/2618501 Reviewed-by: Yuval Peress <peress@chromium.org>
-rw-r--r--.gitignore6
-rw-r--r--zephyr/zmake/setup.py48
-rw-r--r--zephyr/zmake/zmake/__init__.py0
-rw-r--r--zephyr/zmake/zmake/__main__.py122
-rw-r--r--zephyr/zmake/zmake/build_config.py75
-rw-r--r--zephyr/zmake/zmake/jobserver.py142
-rw-r--r--zephyr/zmake/zmake/modules.py82
-rw-r--r--zephyr/zmake/zmake/multiproc.py235
-rw-r--r--zephyr/zmake/zmake/output_packers.py163
-rw-r--r--zephyr/zmake/zmake/project.py103
-rw-r--r--zephyr/zmake/zmake/toolchains.py72
-rw-r--r--zephyr/zmake/zmake/util.py147
-rw-r--r--zephyr/zmake/zmake/zmake.py271
13 files changed, 1465 insertions, 1 deletions
diff --git a/.gitignore b/.gitignore
index bb266eff45..167dec2ca1 100644
--- a/.gitignore
+++ b/.gitignore
@@ -4,7 +4,6 @@ private*/
local/
*~
*.swp
-*.pyc
tags
TAGS
cscope.*
@@ -17,3 +16,8 @@ cscope.*
!.vscode/README.md
!.vscode/settings.json.default
*.code-workspace
+
+# Python ignores
+*.pyc
+__pycache__
+*.egg-info
diff --git a/zephyr/zmake/setup.py b/zephyr/zmake/setup.py
new file mode 100644
index 0000000000..681a5b8d6a
--- /dev/null
+++ b/zephyr/zmake/setup.py
@@ -0,0 +1,48 @@
+# Copyright 2020 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import setuptools
+
+
+setuptools.setup(
+ name='zephyr-chrome-utils',
+ version='0.1',
+
+ description='CrOS Zephyr Utilities',
+ long_description='Utilities used for working on a Zephyr-based EC',
+
+ url='https://chromium.googlesource.com/chromiumos/platform/zephyr-chrome',
+
+ author='Chromium OS Authors',
+ author_email='chromiumos-dev@chromium.org',
+
+ license='BSD',
+
+ # What does your project relate to?
+ keywords='chromeos',
+
+ # You can just specify the packages manually here if your project is
+ # simple. Or you can use find_packages().
+ packages=['zmake'],
+
+ python_requires='>=3.6, <4',
+
+ # List run-time dependencies here. These will be installed by pip when
+ # your project is installed. For an analysis of "install_requires" vs pip's
+ # requirements files see:
+ # https://packaging.python.org/en/latest/requirements.html
+ install_requires=[
+ 'jsonschema>=3.2.0',
+ 'pyyaml>=3.13',
+ ],
+
+ # To provide executable scripts, use entry points in preference to the
+ # "scripts" keyword. Entry points provide cross-platform support and allow
+ # pip to create the appropriate form of executable for the target platform.
+ entry_points={
+ 'console_scripts': [
+ 'zmake=zmake.__main__:main',
+ ],
+ },
+)
diff --git a/zephyr/zmake/zmake/__init__.py b/zephyr/zmake/zmake/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/zephyr/zmake/zmake/__init__.py
diff --git a/zephyr/zmake/zmake/__main__.py b/zephyr/zmake/zmake/__main__.py
new file mode 100644
index 0000000000..1eab278cc3
--- /dev/null
+++ b/zephyr/zmake/zmake/__main__.py
@@ -0,0 +1,122 @@
+# Copyright 2020 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""The entry point into zmake."""
+import argparse
+import inspect
+import logging
+import pathlib
+import sys
+
+import zmake.multiproc as multiproc
+import zmake.util as util
+import zmake.zmake as zm
+
+
+def call_with_namespace(func, namespace):
+ """Call a function with arguments applied from a Namespace.
+
+ Args:
+ func: The callable to call.
+ namespace: The namespace to apply to the callable.
+
+ Returns:
+ The result of calling the callable.
+ """
+ kwds = {}
+ sig = inspect.signature(func)
+ names = [p.name for p in sig.parameters.values()]
+ for name, value in vars(namespace).items():
+ pyname = name.replace('-', '_')
+ if pyname in names:
+ kwds[pyname] = value
+ return func(**kwds)
+
+
+# Dictionary used to map log level strings to their corresponding int values.
+log_level_map = {
+ 'DEBUG': logging.DEBUG,
+ 'INFO': logging.INFO,
+ 'WARNING': logging.WARNING,
+ 'ERROR': logging.ERROR,
+ 'CRITICAL': logging.CRITICAL
+}
+
+
+def main(argv=None):
+ """The main function.
+
+ Args:
+ argv: Optionally, the command-line to parse, not including argv[0].
+
+ Returns:
+ Zero upon success, or non-zero upon failure.
+ """
+ if argv is None:
+ argv = sys.argv[1:]
+
+ parser = argparse.ArgumentParser()
+ parser.add_argument('--checkout', type=pathlib.Path,
+ help='Path to ChromiumOS checkout')
+ parser.add_argument('-j', '--jobs', type=int,
+ help='Degree of multiprogramming to use')
+ parser.add_argument('-l', '--log-level', choices=list(log_level_map.keys()),
+ default='WARNING',
+ dest='log_level',
+ help='Set the logging level (default=WARNING)')
+ parser.add_argument('-L', '--no-log-label', action='store_true',
+ default=False,
+ help='Turn off logging labels')
+ sub = parser.add_subparsers(dest='subcommand', help='Subcommand')
+ sub.required = True
+
+ configure = sub.add_parser('configure')
+ configure.add_argument(
+ '--ignore-unsupported-zephyr-version', action='store_true',
+ help="Don't warn about using an unsupported Zephyr version")
+ configure.add_argument('-v', '--version', type=util.parse_zephyr_version,
+ help='Zephyr RTOS version')
+ configure.add_argument('-t', '--toolchain', help='Name of toolchain to use')
+ configure.add_argument('--zephyr-base', type=pathlib.Path,
+ help='Path to Zephyr source')
+ configure.add_argument('-B', '--build-dir', type=pathlib.Path,
+ required=True, help='Build directory')
+ configure.add_argument('-b', '--build', action='store_true',
+ dest='build_after_configure',
+ help='Run the build after configuration')
+ configure.add_argument('--test', action='store_true',
+ dest='test_after_configure',
+ help='Test the .elf file after configuration')
+ configure.add_argument('project_dir', type=pathlib.Path,
+ help='Path to the project to build')
+
+ build = sub.add_parser('build')
+ build.add_argument('build_dir', type=pathlib.Path,
+ help='The build directory used during configuration')
+
+ test = sub.add_parser('test')
+ test.add_argument('build_dir', type=pathlib.Path,
+ help='The build directory used during configuration')
+
+ testall = sub.add_parser('testall')
+ testall.add_argument('--fail-fast', action='store_true',
+ help='stop testing after the first error')
+
+ opts = parser.parse_args(argv)
+
+ if opts.no_log_label:
+ log_format = '%(message)s'
+ else:
+ log_format = '%(asctime)s - %(name)s/%(levelname)s: %(message)s'
+ logging.basicConfig(format=log_format, level=log_level_map.get(opts.log_level))
+
+ zmake = call_with_namespace(zm.Zmake, opts)
+ subcommand_method = getattr(zmake, opts.subcommand.replace('-', '_'))
+ result = call_with_namespace(subcommand_method, opts)
+ multiproc.wait_for_log_end()
+ return result
+
+
+if __name__ == '__main__':
+ sys.exit(main())
diff --git a/zephyr/zmake/zmake/build_config.py b/zephyr/zmake/zmake/build_config.py
new file mode 100644
index 0000000000..eaa579a777
--- /dev/null
+++ b/zephyr/zmake/zmake/build_config.py
@@ -0,0 +1,75 @@
+# Copyright 2020 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+"""Encapsulation of a build configuration."""
+
+
+import zmake.util as util
+
+
+class BuildConfig:
+ """A container for build configurations.
+
+ A build config is a tuple of environment variables, cmake
+ variables, kconfig definitons, and kconfig files.
+ """
+ def __init__(self, environ_defs={}, cmake_defs={}, kconfig_defs={},
+ kconfig_files=[]):
+ self.environ_defs = dict(environ_defs)
+ self.cmake_defs = dict(cmake_defs)
+ self.kconfig_defs = dict(kconfig_defs)
+ self.kconfig_files = kconfig_files
+
+ def popen_cmake(self, jobclient, project_dir, build_dir, kconfig_path=None,
+ **kwargs):
+ """Run Cmake with this config using a jobclient.
+
+ Args:
+ jobclient: A JobClient instance.
+ project_dir: The project directory.
+ build_dir: Directory to use for Cmake build.
+ kconfig_path: The path to write out Kconfig definitions.
+ kwargs: forwarded to popen.
+ """
+ kconfig_files = list(self.kconfig_files)
+ if kconfig_path:
+ util.write_kconfig_file(kconfig_path, self.kconfig_defs)
+ kconfig_files.append(kconfig_path)
+ elif self.kconfig_defs:
+ raise ValueError(
+ 'Cannot start Cmake on a config with Kconfig items without a '
+ 'kconfig_path')
+
+ if kconfig_files:
+ base_config = BuildConfig(environ_defs=self.environ_defs,
+ cmake_defs=self.cmake_defs)
+ conf_file_config = BuildConfig(
+ cmake_defs={'CONF_FILE': ';'.join(
+ str(p.resolve()) for p in kconfig_files)})
+ return (base_config | conf_file_config).popen_cmake(
+ jobclient, project_dir, build_dir, **kwargs)
+
+ kwargs['env'] = dict(**kwargs.get('env', {}), **self.environ_defs)
+ return jobclient.popen(
+ ['/usr/bin/cmake', '-S', project_dir, '-B', build_dir, '-GNinja',
+ *('-D{}={}'.format(*pair) for pair in self.cmake_defs.items())],
+ **kwargs)
+
+ def __or__(self, other):
+ """Combine two BuildConfig instances."""
+ if not isinstance(other, BuildConfig):
+ raise TypeError("Unsupported operation | for {} and {}".format(
+ type(self), type(other)))
+
+ return BuildConfig(
+ environ_defs=dict(**self.environ_defs, **other.environ_defs),
+ cmake_defs=dict(**self.cmake_defs, **other.cmake_defs),
+ kconfig_defs=dict(**self.kconfig_defs, **other.kconfig_defs),
+ kconfig_files=list({*self.kconfig_files, *other.kconfig_files}))
+
+ def __repr__(self):
+ return 'BuildConfig({})'.format(', '.join(
+ '{}={!r}'.format(name, getattr(self, name))
+ for name in ['environ_defs', 'cmake_defs', 'kconfig_defs',
+ 'kconfig_files']
+ if getattr(self, name)))
diff --git a/zephyr/zmake/zmake/jobserver.py b/zephyr/zmake/zmake/jobserver.py
new file mode 100644
index 0000000000..d9788c6ac6
--- /dev/null
+++ b/zephyr/zmake/zmake/jobserver.py
@@ -0,0 +1,142 @@
+# Copyright 2020 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+"""Module for job counters, limiting the amount of concurrent executions."""
+
+import multiprocessing
+import os
+import re
+import select
+import subprocess
+
+
+class JobHandle:
+ """Small object to handle claim of a job."""
+ def __init__(self, release_func, *args, **kwargs):
+ self.release_func = release_func
+ self.args = args
+ self.kwargs = kwargs
+
+ def __enter__(self):
+ pass
+
+ def __exit__(self, exc_type, exc_value, traceback):
+ self.release_func(*self.args, **self.kwargs)
+
+
+class JobClient:
+ """Abstract base class for all job clients."""
+ def get_job(self):
+ """Claim a job."""
+ raise NotImplementedError('Abstract method not implemented')
+
+ def env(self):
+ """Get the environment variables necessary to share the job server."""
+ return {}
+
+ def popen(self, *args, claim_job=True, **kwargs):
+ """Start a process using subprocess.Popen, optionally claiming a job.
+
+ Args:
+ claim_job: True if a job should be claimed.
+
+ All other arguments are passed to subprocess.Popen.
+
+ Returns:
+ A Popen object.
+ """
+ if claim_job:
+ with self.get_job():
+ return self.popen(*args, claim_job=False, **kwargs)
+
+ kwargs.setdefault('env', os.environ)
+ kwargs['env'].update(self.env())
+
+ return subprocess.Popen(*args, **kwargs)
+
+ def run(self, *args, claim_job=True, **kwargs):
+ """Run a process using subprocess.run, optionally claiming a job.
+
+ Args:
+ claim_job: True if a job should be claimed.
+
+ All other arguments are passed to subprocess.run.
+
+ Returns:
+ A CompletedProcess object.
+ """
+ if claim_job:
+ with self.get_job():
+ return self.run(*args, claim_job=False, **kwargs)
+
+ kwargs.setdefault('env', os.environ)
+ kwargs['env'].update(self.env())
+
+ return subprocess.run(*args, **kwargs)
+
+
+class JobServer(JobClient):
+ """Abstract Job Server."""
+ def __init__(self, jobs=0):
+ raise NotImplementedError('Abstract method not implemented')
+
+
+class GNUMakeJobClient(JobClient):
+ def __init__(self, read_fd, write_fd):
+ self._pipe = [read_fd, write_fd]
+
+ @classmethod
+ def from_environ(cls, env=None):
+ """Create a job client from an environment with the MAKEFLAGS variable.
+
+ If we are started under a GNU Make Job Server, we can search
+ the environment for a string "--jobserver-auth=R,W", where R
+ and W will be the read and write file descriptors to the pipe
+ respectively. If we don't find this environment variable (or
+ the string inside of it), this will raise an OSError.
+
+ Args:
+ env: Optionally, the environment to search.
+
+ Returns:
+ A GNUMakeJobClient configured appropriately.
+ """
+ if env is None:
+ env = os.environ
+ makeflags = env.get('MAKEFLAGS')
+ if not makeflags:
+ raise OSError('MAKEFLAGS is not set in the environment')
+ match = re.search(r'--jobserver-auth=(\d+),(\d+)', makeflags)
+ if not match:
+ raise OSError('MAKEFLAGS did not contain jobserver flags')
+ read_fd, write_fd = map(int, match.groups())
+ return cls(read_fd, write_fd)
+
+ def get_job(self):
+ """Claim a job.
+
+ Returns:
+ A JobHandle object.
+ """
+ byte = os.read(self._pipe[0], 1)
+ return JobHandle(lambda: os.write(self._pipe[1], byte))
+
+ def env(self):
+ """Get the environment variables necessary to share the job server."""
+ return {'MAKEFLAGS': '--jobserver-auth={},{}'.format(*self._pipe)}
+
+
+class GNUMakeJobServer(JobServer, GNUMakeJobClient):
+ """Implements a GNU Make POSIX Job Server.
+
+ See https://www.gnu.org/software/make/manual/html_node/POSIX-Jobserver.html
+ for specification.
+ """
+ def __init__(self, jobs=0):
+ if not jobs:
+ jobs = multiprocessing.cpu_count()
+ elif jobs > select.PIPE_BUF:
+ jobs = select.PIPE_BUF
+
+ self._pipe = os.pipe()
+ os.write(self._pipe[1], b'+' * jobs)
diff --git a/zephyr/zmake/zmake/modules.py b/zephyr/zmake/zmake/modules.py
new file mode 100644
index 0000000000..ee05aa67cf
--- /dev/null
+++ b/zephyr/zmake/zmake/modules.py
@@ -0,0 +1,82 @@
+# Copyright 2020 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+"""Registry of known Zephyr modules."""
+
+import pathlib
+import os
+
+import zmake.build_config as build_config
+import zmake.util as util
+
+
+def third_party_module(name, checkout, version):
+ """Common callback in registry for all third_party/zephyr modules.
+
+ Args:
+ name: The name of the module.
+ checkout: The path to the chromiumos source.
+ version: The zephyr version.
+
+ Return:
+ The path to the module module.
+ """
+ if not version or len(version) < 2:
+ return None
+ return checkout / 'src' / 'third_party' / 'zephyr' / name / 'v{}.{}'.format(
+ version[0], version[1])
+
+
+known_modules = {
+ 'hal_stm32': third_party_module,
+ 'cmsis': third_party_module,
+ 'ec-shim': lambda name, checkout, version: (
+ checkout / 'src' / 'platform' / 'ec'),
+ 'zephyr-chrome': lambda name, checkout, version: (
+ checkout / 'src' / 'platform' / 'zephyr-chrome'),
+}
+
+
+def locate_modules(checkout_dir, version, modules=known_modules):
+ """Resolve module locations from a known_modules dictionary.
+
+ Args:
+ checkout_dir: The path to the chromiumos source.
+ version: The zephyr version, as a two or three tuple of ints.
+ modules: The known_modules dictionary to use for resolution.
+
+ Returns:
+ A dictionary mapping module names to paths.
+ """
+ result = {}
+ for name, locator in known_modules.items():
+ result[name] = locator(name, checkout_dir, version)
+ return result
+
+
+def setup_module_symlinks(output_dir, modules):
+ """Setup a directory with symlinks to modules.
+
+ Args:
+ output_dir: The directory to place the symlinks in.
+ modules: A dictionary of module names mapping to paths.
+
+ Returns:
+ The resultant BuildConfig that should be applied to use each
+ of these modules.
+ """
+ if not output_dir.exists():
+ output_dir.mkdir(parents=True)
+
+ module_links = []
+
+ for name, path in modules.items():
+ link_path = output_dir.resolve() / name
+ util.update_symlink(path, link_path)
+ module_links.append(link_path)
+
+ if module_links:
+ return build_config.BuildConfig(
+ cmake_defs={'ZEPHYR_MODULES': ';'.join(map(str, module_links))})
+ else:
+ return build_config.BuildConfig()
diff --git a/zephyr/zmake/zmake/multiproc.py b/zephyr/zmake/zmake/multiproc.py
new file mode 100644
index 0000000000..a9a8501206
--- /dev/null
+++ b/zephyr/zmake/zmake/multiproc.py
@@ -0,0 +1,235 @@
+# Copyright 2020 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+import logging
+import os
+import select
+import threading
+
+"""Zmake multiprocessing utility module.
+
+This module is used to aid in zmake's multiprocessing. It contains tools
+available to log output from multiple processes on the fly. This means that a
+process does not need to finish before the output is available to the developer
+on the screen.
+"""
+
+# A local pipe use to signal the look that a new file descriptor was added and
+# should be included in the select statement.
+_logging_interrupt_pipe = os.pipe()
+# A condition variable used to synchronize logging operations.
+_logging_cv = threading.Condition()
+# A map of file descriptors to their logger/logging level tuple.
+_logging_map = {}
+
+
+def _log_fd(fd):
+ """Log information from a single file descriptor.
+
+ This function is BLOCKING. It will read from the given file descriptor until
+ either the end of line is read or EOF. Once EOF is read it will remove the
+ file descriptor from _logging_map so it will no longer be used.
+ Additionally, in some cases, the file descriptor will be closed (caused by
+ a call to Popen.wait()). In these cases, the file descriptor will also be
+ removed from the map as it is no longer valid.
+ """
+ with _logging_cv:
+ logger, log_level = _logging_map[fd]
+ if fd.closed:
+ del _logging_map[fd]
+ _logging_cv.notify_all()
+ return
+ line = fd.readline()
+ if not line:
+ # EOF
+ del _logging_map[fd]
+ _logging_cv.notify_all()
+ return
+ line = line.strip()
+ if line:
+ logger.log(log_level, line)
+
+
+def _prune_logging_fds():
+ """Prune the current file descriptors under _logging_map.
+
+ This function will iterate over the logging map and check for closed file
+ descriptors. Every closed file descriptor will be removed.
+ """
+ with _logging_cv:
+ remove = [fd for fd in _logging_map.keys() if fd.closed]
+ for fd in remove:
+ del _logging_map[fd]
+ if remove:
+ _logging_cv.notify_all()
+
+
+def _logging_loop():
+ """The primary logging thread loop.
+
+ This is the entry point of the logging thread. It will listen for (1) any
+ new data on the output file descriptors that were added via log_output and
+ (2) any new file descriptors being added by log_output. Once a file
+ descriptor is ready to be read, this function will call _log_fd to perform
+ the actual read and logging.
+ """
+ while True:
+ with _logging_cv:
+ _logging_cv.wait_for(lambda: _logging_map)
+ keys = list(_logging_map.keys()) + [_logging_interrupt_pipe[0]]
+ try:
+ fds, _, _ = select.select(keys, [], [])
+ except ValueError:
+ # One of the file descriptors must be closed, prune them and try
+ # again.
+ _prune_logging_fds()
+ continue
+ if _logging_interrupt_pipe[0] in fds:
+ # We got a dummy byte sent by log_output, this is a signal used to
+ # break out of the blocking select.select call to tell us that the
+ # file descriptor set has changed. We just need to read the byte and
+ # remove this descriptor from the list. If we actually have data
+ # that should be read it will be read in the for loop below.
+ os.read(_logging_interrupt_pipe[0], 1)
+ fds.remove(_logging_interrupt_pipe[0])
+ for fd in fds:
+ _log_fd(fd)
+
+
+_logging_thread = threading.Thread(target=_logging_loop, daemon=True)
+
+
+def log_output(logger, log_level, file_descriptor):
+ """Log the output from the given file descriptor.
+
+ Args:
+ logger: The logger object to use.
+ log_level: The logging level to use.
+ file_descriptor: The file descriptor to read from.
+ """
+ with _logging_cv:
+ if not _logging_thread.is_alive():
+ _logging_thread.start()
+ _logging_map[file_descriptor] = (logger, log_level)
+ # Write a dummy byte to the pipe to break the select so we can add the
+ # new fd.
+ os.write(_logging_interrupt_pipe[1], b'x')
+ # Notify the condition so we can run the select on the current fds.
+ _logging_cv.notify_all()
+
+
+def wait_for_log_end():
+ """Wait for all the logs to be printed.
+
+ This method will block execution until all the logs have been flushed out.
+ """
+ with _logging_cv:
+ _logging_cv.wait_for(lambda: not _logging_map)
+
+
+class Executor:
+ """Parallel executor helper class.
+
+ This class is used to run multiple functions in parallel. The functions MUST
+ return an integer result code (or throw an exception). This class will start
+ a thread per operation and wait() for all the threads to resolve. If
+ fail_fast is set to True, then not all threads must return before wait()
+ returns. Instead either ALL threads must return 0 OR any thread must return
+ a non zero result (or throw an exception).
+
+ Attributes:
+ fail_fast: Whether or not the first function's error code should
+ terminate the executor.
+ lock: The condition variable used to synchronize across threads.
+ threads: A list of threading.Thread objects currently under this
+ Executor.
+ results: A list of result codes returned by each of the functions called
+ by this Executor.
+ """
+ def __init__(self, fail_fast):
+ self.fail_fast = fail_fast
+ self.lock = threading.Condition()
+ self.threads = []
+ self.results = []
+ self.logger = logging.getLogger(self.__class__.__name__)
+
+ def append(self, func):
+ """Append the given function to the wait list.
+
+ Once added, the function's return value will be used to determine the
+ Executor's final result value. The function must return an int result
+ code or throw an exception. For example: If two functions were added
+ to the Executor, they will both be run in parallel and their results
+ will determine whether or not the Executor succeeded. If both functions
+ returned 0, then the Executor's wait function will also return 0.
+
+ Args:
+ func: A function which returns an int result code or throws an
+ exception.
+ """
+ with self.lock:
+ thread = threading.Thread(target=lambda: self._run_fn(func),
+ daemon=True)
+ thread.start()
+ self.threads.append(thread)
+
+ def wait(self):
+ """Wait for a result to be available.
+
+ This function waits for the executor to resolve. Being resolved depends on
+ the initial fail_fast setting.
+ - If fail_fast is True then the executor is resolved as soon as any thread
+ throws an exception or returns a non-zero result. Or, all the threads
+ returned a zero result code.
+ - If fail_fast is False, then all the threads must have returned a result
+ code or have thrown.
+
+ Returns:
+ An integer result code of either the first failed function or 0 if
+ they all succeeded.
+ """
+ with self.lock:
+ self.lock.wait_for(predicate=lambda: self._is_finished)
+ return self._result
+
+ def _run_fn(self, func):
+ """Entry point to each running thread.
+
+ This function will run the function provided in the append() function.
+ The result value of the function will be used to determine the
+ Executor's result value. If the function throws any exception it will be
+ caught and -1 will be used as the assumed result value.
+
+ Args:
+ func: The function to run.
+ """
+ try:
+ result = func()
+ except Exception as ex:
+ self.logger.exception(ex)
+ result = -1
+ with self.lock:
+ self.results.append(result)
+ self.lock.notify_all()
+
+ @property
+ def _is_finished(self):
+ """Whether or not the Executor is considered to be done.
+
+ Returns:
+ True if the Executor is considered done.
+ """
+ if len(self.threads) == len(self.results):
+ return True
+ return self.fail_fast and any([result for result in self.results])
+
+ @property
+ def _result(self):
+ """The result code of the Executor.
+
+ Note that _is_finished must be True for this to have any meaning.
+
+ Returns:
+ An int representing the result value of the underlying functions.
+ """
+ return next((result for result in self.results if result), 0)
diff --git a/zephyr/zmake/zmake/output_packers.py b/zephyr/zmake/zmake/output_packers.py
new file mode 100644
index 0000000000..d8c1a8db43
--- /dev/null
+++ b/zephyr/zmake/zmake/output_packers.py
@@ -0,0 +1,163 @@
+# Copyright 2020 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+"""Types which provide many builds and composite them into a single binary."""
+import logging
+import pathlib
+import subprocess
+
+import zmake.build_config as build_config
+import zmake.multiproc
+import zmake.util as util
+
+
+def _write_dts_file(dts_file, config_header, output_bin, ro_filename, rw_filename):
+ """Generate the .dts file used for binman.
+
+ Args:
+ dts_file: The dts file to write to.
+ config_header: The full path to the generated autoconf.h header.
+ output_bin: The full path to the binary that binman should output.
+ ro_filename: The RO image file name.
+ rw_filename: The RW image file name.
+
+ Returns:
+ The path to the .dts file that was generated.
+ """
+ dts_file.write("""
+ /dts-v1/;
+ #include "{config_header}"
+ / {{
+ #address-cells = <1>;
+ #size-cells = <1>;
+ binman {{
+ filename = "{output_bin}";
+ pad-byte = <0x1d>;
+ section@0 {{
+ read-only;
+ offset = <CONFIG_CROS_EC_RO_MEM_OFF>;
+ size = <CONFIG_CROS_EC_RO_SIZE>;
+ blob {{
+ filename = "{ro_filename}";
+ }};
+ }};
+ section@1 {{
+ offset = <CONFIG_CROS_EC_RW_MEM_OFF>;
+ size = <CONFIG_CROS_EC_RW_SIZE>;
+ blob {{
+ filename = "{rw_filename}";
+ }};
+ }};
+ }};
+ }};""".format(
+ output_bin=output_bin,
+ config_header=config_header,
+ ro_filename=ro_filename,
+ rw_filename=rw_filename
+ ))
+
+
+class BasePacker:
+ """Abstract base for all packers."""
+ def __init__(self, project):
+ self.project = project
+
+ def configs(self):
+ """Get all of the build configurations necessary.
+
+ Yields:
+ 2-tuples of config name and a BuildConfig.
+ """
+ yield 'singleimage', build_config.BuildConfig()
+
+ def pack_firmware(self, work_dir, jobclient):
+ """Pack a firmware image.
+
+ Config names from the configs generator are passed as keyword
+ arguments, with each argument being set to the path of the
+ build directory.
+
+ Args:
+ work_dir: A directory to write outputs and temporary files
+ into.
+ jobclient: A JobClient object to use.
+
+ Yields:
+ 2-tuples of the path of each file in the work_dir (or any
+ other directory) which should be copied into the output
+ directory, and the output filename.
+ """
+ raise NotImplementedError('Abstract method not implemented')
+
+
+class ElfPacker(BasePacker):
+ """Raw proxy for ELF output of a single build."""
+ def pack_firmware(self, work_dir, jobclient, singleimage):
+ yield singleimage / 'zephyr' / 'zephyr.elf', 'zephyr.elf'
+
+
+class RawBinPacker(BasePacker):
+ """Packer for RO/RW image to generate a .bin build using FMAP."""
+ def __init__(self, project):
+ self.logger = logging.getLogger(self.__class__.__name__)
+ super().__init__(project)
+
+ def configs(self):
+ yield 'ro', build_config.BuildConfig(kconfig_defs={'CONFIG_CROS_EC_RO': 'y'})
+ yield 'rw', build_config.BuildConfig(kconfig_defs={'CONFIG_CROS_EC_RW': 'y'})
+
+ def pack_firmware(self, work_dir, jobclient, ro, rw):
+ """Pack the 'raw' binary.
+
+ This combines the RO and RW images as specified in the Kconfig file for
+ the project. For this function to work, the following config values must
+ be defined:
+ * CONFIG_CROS_EC_RO_MEM_OFF - The offset in bytes of the RO image from
+ the start of the resulting binary.
+ * CONFIG_CROS_EC_RO_SIZE - The maximum allowed size (in bytes) of the RO
+ image.
+ * CONFIG_CROS_EC_RW_MEM_OFF - The offset in bytes of the RW image from
+ the start of the resulting binary (must be >= RO_MEM_OFF + RO_SIZE).
+ * CONFIG_CROS_EC_RW_SIZE - The maximum allowed size (in bytes) of the RW
+ image.
+
+ Args:
+ work_dir: The directory used for packing.
+ jobclient: The client used to run subprocesses.
+ ro: Directory containing the RO image build.
+ rw: Directory containing the RW image build.
+
+ Returns:
+ Tuple mapping the resulting .bin file to the output filename.
+ """
+ work_dir = pathlib.Path(work_dir).resolve()
+ ro = pathlib.Path(ro).resolve()
+ rw = pathlib.Path(rw).resolve()
+ dts_file_path = work_dir / 'project.dts'
+ with open(dts_file_path, 'w+') as dts_file:
+ _write_dts_file(
+ dts_file=dts_file,
+ config_header=ro / 'zephyr' / 'include' / 'generated' / 'autoconf.h',
+ output_bin=work_dir / 'zephyr.bin',
+ ro_filename=ro / 'zephyr' / 'zephyr.bin',
+ rw_filename=rw / 'zephyr' / 'zephyr.bin')
+
+ proc = jobclient.popen(
+ ['binman', '-v', '5', 'build', '-d', dts_file_path, '-m'],
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ encoding='utf-8')
+
+ zmake.multiproc.log_output(self.logger, logging.DEBUG, proc.stdout)
+ zmake.multiproc.log_output(self.logger, logging.ERROR, proc.stderr)
+ if proc.wait(timeout=5):
+ raise OSError('Failed to run binman')
+
+ yield work_dir / 'zephyr.bin', 'zephyr.bin'
+
+
+# A dictionary mapping packer config names to classes.
+packer_registry = {
+ 'elf': ElfPacker,
+ 'raw': RawBinPacker,
+}
diff --git a/zephyr/zmake/zmake/project.py b/zephyr/zmake/zmake/project.py
new file mode 100644
index 0000000000..7bb68a7018
--- /dev/null
+++ b/zephyr/zmake/zmake/project.py
@@ -0,0 +1,103 @@
+# Copyright 2020 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+"""Module for project config wrapper object."""
+
+import jsonschema
+import yaml
+
+import zmake.build_config as build_config
+import zmake.output_packers as packers
+import zmake.util as util
+
+
+class ProjectConfig:
+ """An object wrapping zmake.yaml."""
+ validator = jsonschema.Draft7Validator
+ schema = {
+ 'type': 'object',
+ 'required': ['supported-zephyr-versions', 'board', 'output-type',
+ 'toolchain'],
+ 'properties': {
+ 'supported-zephyr-versions': {
+ 'type': 'array',
+ 'items': {
+ 'type': 'string',
+ 'enum': ['v2.4'],
+ },
+ 'minItems': 1,
+ 'uniqueItems': True,
+ },
+ 'board': {
+ 'type': 'string',
+ },
+ 'output-type': {
+ 'type': 'string',
+ 'enum': list(packers.packer_registry),
+ },
+ 'toolchain': {
+ 'type': 'string',
+ },
+ 'prefer-zephyr-sdk': {
+ 'type': 'boolean',
+ },
+ 'is-test': {
+ 'type': 'boolean',
+ },
+ },
+ }
+
+ def __init__(self, config_dict):
+ self.validator.check_schema(self.schema)
+ jsonschema.validate(config_dict, self.schema, cls=self.validator)
+ self.config_dict = config_dict
+
+ @property
+ def supported_zephyr_versions(self):
+ return [util.parse_zephyr_version(x)
+ for x in self.config_dict['supported-zephyr-versions']]
+
+ @property
+ def board(self):
+ return self.config_dict['board']
+
+ @property
+ def output_packer(self):
+ return packers.packer_registry[self.config_dict['output-type']]
+
+ @property
+ def toolchain(self):
+ return self.config_dict['toolchain']
+
+ @property
+ def zephyr_sdk_is_preferred(self):
+ return self.config_dict.get('prefer-zephyr-sdk', False)
+
+ @property
+ def is_test(self):
+ return self.config_dict.get('is-test', False)
+
+
+class Project:
+ """An object encapsulating a project directory."""
+ def __init__(self, project_dir):
+ self.project_dir = project_dir.resolve()
+ with open(self.project_dir / 'zmake.yaml') as f:
+ self.config = ProjectConfig(yaml.safe_load(f))
+ self.packer = self.config.output_packer(self)
+
+ def iter_builds(self):
+ """Iterate thru the build combinations provided by the project's packer.
+
+ Yields:
+ 2-tuples of a build configuration name and a BuildConfig.
+ """
+ conf = build_config.BuildConfig(cmake_defs={'BOARD': self.config.board})
+ if (self.project_dir / 'boards').is_dir():
+ conf |= build_config.BuildConfig(
+ cmake_defs={'BOARD_ROOT': str(self.project_dir)})
+ prj_conf = self.project_dir / 'prj.conf'
+ if prj_conf.is_file():
+ conf |= build_config.BuildConfig(kconfig_files=[prj_conf])
+ for build_name, packer_config in self.packer.configs():
+ yield build_name, conf | packer_config
diff --git a/zephyr/zmake/zmake/toolchains.py b/zephyr/zmake/zmake/toolchains.py
new file mode 100644
index 0000000000..1ef79f6849
--- /dev/null
+++ b/zephyr/zmake/zmake/toolchains.py
@@ -0,0 +1,72 @@
+# Copyright 2020 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+"""Definitions of toolchain variables."""
+
+import glob
+import os
+import pathlib
+
+import zmake.build_config as build_config
+
+
+def find_zephyr_sdk():
+ """Find the Zephyr SDK, if it's installed.
+
+ Returns:
+ The path to the Zephyr SDK, using the search rules defined by
+ https://docs.zephyrproject.org/latest/getting_started/installation_linux.html
+ """
+ def _gen_sdk_paths():
+ yield os.getenv('ZEPHYR_SDK_INSTALL_DIR')
+
+ for searchpath in ('~/zephyr-sdk', '~/.local/zephyr-sdk',
+ '~/.local/opt/zephyr-sdk', '~/bin/zephyr-sdk',
+ '/opt/zephyr-sdk', '/usr/zephyr-sdk',
+ '/usr/local/zephyr-sdk'):
+ for suffix in ('', '-*'):
+ yield from glob.glob(os.path.expanduser(searchpath + suffix))
+
+ for path in _gen_sdk_paths():
+ if not path:
+ continue
+ path = pathlib.Path(path)
+ if (path / 'sdk_version').is_file():
+ return path
+
+ raise OSError('Unable to find the Zephyr SDK')
+
+
+# Mapping of toolchain names -> (λ (module-paths) build-config)
+toolchains = {
+ 'coreboot-sdk': lambda modules: build_config.BuildConfig(
+ cmake_defs={'TOOLCHAIN_ROOT': str(modules['zephyr-chrome']),
+ 'ZEPHYR_TOOLCHAIN_VARIANT': 'coreboot-sdk'}),
+ 'llvm': lambda modules: build_config.BuildConfig(
+ cmake_defs={'TOOLCHAIN_ROOT': str(modules['zephyr-chrome']),
+ 'ZEPHYR_TOOLCHAIN_VARIANT': 'llvm'}),
+ 'zephyr': lambda _: build_config.BuildConfig(
+ cmake_defs={'ZEPHYR_TOOLCHAIN_VARIANT': 'zephyr',
+ 'ZEPHYR_SDK_INSTALL_DIR': str(find_zephyr_sdk())}),
+ 'arm-none-eabi': lambda _: build_config.BuildConfig(
+ cmake_defs={'ZEPHYR_TOOLCHAIN_VARIANT': 'cross-compile',
+ 'CROSS_COMPILE': '/usr/bin/arm-none-eabi-'}),
+}
+
+
+def get_toolchain(name, module_paths):
+ """Get a toolchain by name.
+
+ Args:
+ name: The name of the toolchain.
+ module_paths: Dictionary mapping module names to paths.
+
+ Returns:
+ The corresponding BuildConfig from the defined toolchains, if
+ one exists, otherwise a simple BuildConfig which sets
+ ZEPHYR_TOOLCHAIN_VARIANT to the corresponding name.
+ """
+ if name in toolchains:
+ return toolchains[name](module_paths)
+ return build_config.BuildConfig(
+ cmake_defs={'ZEPHYR_TOOLCHAIN_VARIANT': name})
diff --git a/zephyr/zmake/zmake/util.py b/zephyr/zmake/zmake/util.py
new file mode 100644
index 0000000000..fc81176b6c
--- /dev/null
+++ b/zephyr/zmake/zmake/util.py
@@ -0,0 +1,147 @@
+# Copyright 2020 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+"""Common miscellaneous utility functions for zmake."""
+
+import os
+import pathlib
+import re
+import shlex
+
+
+def locate_cros_checkout():
+ """Find the path to the ChromiumOS checkout.
+
+ Returns:
+ The first directory found with a .repo directory in it,
+ starting by checking the CROS_WORKON_SRCROOT environment
+ variable, then scanning upwards from the current directory,
+ and finally from a known set of common paths.
+ """
+ def propose_checkouts():
+ yield os.getenv('CROS_WORKON_SRCROOT')
+
+ path = pathlib.Path.cwd()
+ while path.resolve() != pathlib.Path('/'):
+ yield path
+ path = path / '..'
+
+ yield '/mnt/host/source'
+ yield pathlib.Path.home() / 'trunk'
+ yield pathlib.Path.home() / 'chromiumos'
+
+ for path in propose_checkouts():
+ if not path:
+ continue
+ path = pathlib.Path(path)
+ if (path / '.repo').is_dir():
+ return path.resolve()
+
+ raise FileNotFoundError('Unable to locate a ChromiumOS checkout')
+
+
+def locate_zephyr_base(checkout, version):
+ """Locate the path to the Zephyr RTOS in a ChromiumOS checkout.
+
+ Args:
+ checkout: The path to the ChromiumOS checkout.
+ version: The requested zephyr version, as a tuple of integers.
+
+ Returns:
+ The path to the Zephyr source.
+ """
+ return (checkout / 'src' / 'third_party' / 'zephyr' / 'main' /
+ 'v{}.{}'.format(*version[:2]))
+
+
+def read_kconfig_file(path):
+ """Parse a Kconfig file.
+
+ Args:
+ path: The path to open.
+
+ Returns:
+ A dictionary of kconfig items to their values.
+ """
+ result = {}
+ with open(path) as f:
+ for line in f:
+ line, _, _ = line.partition('#')
+ line = line.strip()
+ if line:
+ name, _, value = line.partition('=')
+ result[name.strip()] = value.strip()
+ return result
+
+
+def write_kconfig_file(path, config, only_if_changed=True):
+ """Write out a dictionary to Kconfig format.
+
+ Args:
+ path: The path to write to.
+ config: The dictionary to write.
+ only_if_changed: Set to True if the file should not be written
+ unless it has changed.
+ """
+ if only_if_changed:
+ if path.exists() and read_kconfig_file(path) == config:
+ return
+ with open(path, "w") as f:
+ for name, value in config.items():
+ f.write('{}={}\n'.format(name, value))
+
+
+def parse_zephyr_version(version_string):
+ """Parse a human-readable version string (e.g., "v2.4") as a tuple.
+
+ Args:
+ version_string: The human-readable version string.
+
+ Returns:
+ A 2-tuple or 3-tuple of integers representing the version.
+ """
+ match = re.fullmatch(r'v?(\d+)[._](\d+)(?:[._](\d+))?', version_string)
+ if not match:
+ raise ValueError(
+ "{} does not look like a Zephyr version.".format(version_string))
+ return tuple(int(x) for x in match.groups() if x is not None)
+
+
+def repr_command(argv):
+ """Represent an argument array as a string.
+
+ Args:
+ argv: The arguments of the command.
+
+ Returns:
+ A string which could be pasted into a shell for execution.
+ """
+ return ' '.join(shlex.quote(str(arg)) for arg in argv)
+
+
+def update_symlink(target_path, link_path):
+ """Create a symlink if it does not exist, or links to a different path.
+
+ Args:
+ target_path: A Path-like object of the desired symlink path.
+ link_path: A Path-like object of the symlink.
+ """
+ target = target_path.resolve()
+ if (not link_path.is_symlink()
+ or pathlib.Path(os.readlink(link_path)).resolve() != target):
+ if link_path.exists():
+ link_path.unlink()
+ link_path.symlink_to(target)
+
+
+def log_multi_line(logger, level, message):
+ """Log a potentially multi-line message to the logger.
+
+ Args:
+ logger: The Logger object to log to.
+ level: The logging level to use when logging.
+ message: The (potentially) multi-line message to log.
+ """
+ for line in message.splitlines():
+ if line:
+ logger.log(level, line)
diff --git a/zephyr/zmake/zmake/zmake.py b/zephyr/zmake/zmake/zmake.py
new file mode 100644
index 0000000000..3830b57ab3
--- /dev/null
+++ b/zephyr/zmake/zmake/zmake.py
@@ -0,0 +1,271 @@
+# Copyright 2020 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Module encapsulating Zmake wrapper object."""
+import logging
+import os
+import pathlib
+import shutil
+import subprocess
+import tempfile
+
+import zmake.build_config
+import zmake.modules
+import zmake.jobserver
+import zmake.multiproc
+import zmake.project
+import zmake.toolchains as toolchains
+import zmake.util as util
+
+
+class Zmake:
+ """Wrapper class encapsulating zmake's supported operations."""
+ def __init__(self, checkout=None, jobserver=None, jobs=0):
+ if checkout:
+ self.checkout = pathlib.Path(checkout)
+ else:
+ self.checkout = util.locate_cros_checkout()
+ assert self.checkout.exists()
+
+ if jobserver:
+ self.jobserver = jobserver
+ else:
+ try:
+ self.jobserver = zmake.jobserver.GNUMakeJobClient.from_environ()
+ except OSError:
+ self.jobserver = zmake.jobserver.GNUMakeJobServer(jobs=jobs)
+
+ self.logger = logging.getLogger(self.__class__.__name__)
+
+ def configure(self, project_dir, build_dir,
+ version=None, zephyr_base=None, module_paths=None,
+ toolchain=None, ignore_unsupported_zephyr_version=False,
+ build_after_configure=False, test_after_configure=False):
+ """Set up a build directory to later be built by "zmake build"."""
+ project = zmake.project.Project(project_dir)
+ if version:
+ # Ignore the patchset.
+ version = version[:2]
+ if (not ignore_unsupported_zephyr_version
+ and version not in project.config.supported_zephyr_versions):
+ raise ValueError(
+ 'Requested version (v{}.{}) is not supported by the '
+ 'project. You may wish to either configure zmake.yaml to '
+ 'support this version, or pass '
+ '--ignore-unsupported-zephyr-version.'.format(*version))
+ else:
+ # Assume the highest supported version by default.
+ version = max(project.config.supported_zephyr_versions)
+ if not zephyr_base:
+ zephyr_base = util.locate_zephyr_base(self.checkout, version)
+ zephyr_base = zephyr_base.resolve()
+
+ if not module_paths:
+ module_paths = zmake.modules.locate_modules(self.checkout, version)
+
+ if not module_paths['zephyr-chrome']:
+ raise OSError("Missing zephyr-chrome module")
+
+ base_config = zmake.build_config.BuildConfig(
+ environ_defs={'ZEPHYR_BASE': str(zephyr_base),
+ 'PATH': '/usr/bin'},
+ cmake_defs={'DTS_ROOT': module_paths['zephyr-chrome']})
+ module_config = zmake.modules.setup_module_symlinks(
+ build_dir / 'modules', module_paths)
+
+ if not toolchain:
+ toolchain = project.config.toolchain
+ if project.config.zephyr_sdk_is_preferred:
+ try:
+ toolchains.find_zephyr_sdk()
+ except OSError:
+ self.logger.warning(
+ 'Unable to find the Zephyr SDK, which is the preferred '
+ 'toolchain for this project (however, unavailable in '
+ 'the chroot by default). Using %r instead, which '
+ 'will probably compile but may not actually work at '
+ 'all. See go/zephyr-care for more info.', toolchain)
+ else:
+ self.logger.info(
+ 'Zephyr SDK is available. Using it instead of %r.',
+ toolchain)
+ toolchain = 'zephyr'
+
+ toolchain_config = toolchains.get_toolchain(toolchain, module_paths)
+ if not build_dir.exists():
+ build_dir = build_dir.mkdir()
+ processes = []
+ self.logger.info('Building %s in %s.', project_dir, build_dir)
+ for build_name, build_config in project.iter_builds():
+ self.logger.info('Configuring %s:%s.', project_dir, build_name)
+ config = (base_config
+ | toolchain_config
+ | module_config
+ | build_config)
+ output_dir = build_dir / 'build-{}'.format(build_name)
+ kconfig_file = build_dir / 'kconfig-{}.conf'.format(build_name)
+ proc = config.popen_cmake(self.jobserver, project_dir, output_dir,
+ kconfig_file, stdin=subprocess.DEVNULL,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ encoding='utf-8',
+ errors='replace')
+ zmake.multiproc.log_output(self.logger, logging.DEBUG, proc.stdout)
+ zmake.multiproc.log_output(self.logger, logging.ERROR, proc.stderr)
+ processes.append(proc)
+ for proc in processes:
+ if proc.wait():
+ raise OSError(
+ "Execution of {} failed (return code={})!\n".format(
+ util.repr_command(proc.args), proc.returncode))
+
+ # Create symlink to project
+ util.update_symlink(project_dir, build_dir / 'project')
+
+ if test_after_configure:
+ return self.test(build_dir=build_dir)
+ elif build_after_configure:
+ return self.build(build_dir=build_dir)
+
+ def build(self, build_dir, output_files_out=None):
+ """Build a pre-configured build directory."""
+ project = zmake.project.Project(build_dir / 'project')
+
+ procs = []
+ dirs = {}
+ for build_name, build_config in project.iter_builds():
+ self.logger.info('Building %s:%s.', build_dir, build_name)
+ dirs[build_name] = build_dir / 'build-{}'.format(build_name)
+ proc = self.jobserver.popen(
+ ['/usr/bin/ninja', '-C', dirs[build_name]],
+ # Ninja will connect as a job client instead and claim
+ # many jobs.
+ claim_job=False,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ encoding='utf-8',
+ errors='replace')
+ zmake.multiproc.log_output(self.logger, logging.DEBUG, proc.stdout)
+ zmake.multiproc.log_output(self.logger, logging.ERROR, proc.stderr)
+ procs.append(proc)
+
+ for proc in procs:
+ if proc.wait():
+ raise OSError(
+ "Execution of {} failed (return code={})!\n".format(
+ util.repr_command(proc.args), proc.returncode))
+
+ # Run the packer.
+ packer_work_dir = build_dir / 'packer'
+ output_dir = build_dir / 'output'
+ for d in output_dir, packer_work_dir:
+ if not d.exists():
+ d.mkdir()
+
+ if output_files_out is None:
+ output_files_out = []
+ for output_file, output_name in project.packer.pack_firmware(
+ packer_work_dir, self.jobserver, **dirs):
+ shutil.copy2(output_file, output_dir / output_name)
+ self.logger.info('Output file \'%r\' created.', output_file)
+ output_files_out.append(output_file)
+
+ return 0
+
+ def test(self, build_dir):
+ """Test a build directory."""
+ procs = []
+ output_files = []
+ self.build(build_dir, output_files_out=output_files)
+
+ # If the project built but isn't a test, just bail.
+ project = zmake.project.Project(build_dir / 'project')
+ if not project.config.is_test:
+ return 0
+
+ for output_file in output_files:
+ self.logger.info('Running tests in %s.', output_file)
+ proc = self.jobserver.popen(
+ [output_file],
+ claim_job=True,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ encoding='utf-8',
+ errors='replace')
+ zmake.multiproc.log_output(self.logger, logging.DEBUG, proc.stdout)
+ zmake.multiproc.log_output(self.logger, logging.ERROR, proc.stderr)
+ procs.append(proc)
+
+ for idx, proc in enumerate(procs):
+ if proc.wait():
+ raise OSError(
+ "Execution of {} failed (return code={})!\n".format(
+ util.repr_command(proc.args), proc.returncode))
+ return 0
+
+ def _run_pytest(self, directory):
+ """Run pytest on a given directory.
+
+ This is a utility function to help parallelize running pytest on
+ multiple directories.
+
+ Args:
+ directory: The directory that we should run pytest on.
+ Returns:
+ The status code of pytest.
+ """
+ proc = self.jobserver.popen(
+ ['pytest', directory],
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ encoding='utf-8',
+ errors='replace')
+ # Log stdout as DEBUG log messages.
+ zmake.multiproc.log_output(self.logger, logging.DEBUG, proc.stdout)
+ # Log stderr as ERROR log messages
+ zmake.multiproc.log_output(self.logger, logging.ERROR, proc.stderr)
+ return proc.wait()
+
+ def testall(self, fail_fast=False):
+ """Test all the valid test targets"""
+ modules = zmake.modules.locate_modules(self.checkout, version=None)
+ root_dirs = [modules['zephyr-chrome'] / 'projects',
+ modules['zephyr-chrome'] / 'tests',
+ modules['ec-shim'] / 'zephyr/test']
+ project_dirs = []
+ for root_dir in root_dirs:
+ self.logger.info('Finding zmake target under \'%s\'.', root_dir)
+ for path in pathlib.Path(root_dir).rglob('zmake.yaml'):
+ project_dirs.append(path.parent)
+
+ executor = zmake.multiproc.Executor(fail_fast=fail_fast)
+ tmp_dirs = []
+ for project_dir in project_dirs:
+ is_test = zmake.project.Project(project_dir).config.is_test
+ temp_build_dir = tempfile.mkdtemp(
+ suffix='-{}'.format(os.path.basename(project_dir.as_posix())),
+ prefix='zbuild-')
+ tmp_dirs.append(temp_build_dir)
+ # Configure and run the test.
+ executor.append(
+ func=lambda: self.configure(
+ project_dir=pathlib.Path(project_dir),
+ build_dir=pathlib.Path(temp_build_dir),
+ build_after_configure=True,
+ test_after_configure=is_test))
+
+ # Find all the directories under zephyr-chrome/tests containing a
+ # test_*.py file. We're using a set here to avoid running the same tests
+ # multiple times.
+ project_dirs = set()
+ for path in pathlib.Path(modules['zephyr-chrome'] / 'tests').rglob('test_*.py'):
+ project_dirs.add(path.parent)
+
+ for project_dir in project_dirs:
+ executor.append(func=lambda: self._run_pytest(directory=project_dir))
+
+ rv = executor.wait()
+ for tmpdir in tmp_dirs:
+ shutil.rmtree(tmpdir)
+ return rv