diff options
author | Jack Rosenthal <jrosenth@chromium.org> | 2021-11-04 12:11:58 -0600 |
---|---|---|
committer | Commit Bot <commit-bot@chromium.org> | 2021-11-05 04:22:34 +0000 |
commit | 252457d4b21f46889eebad61d4c0a65331919cec (patch) | |
tree | 01856c4d31d710b20e85a74c8d7b5836e35c3b98 /zephyr/zmake/zmake | |
parent | 08f5a1e6fc2c9467230444ac9b582dcf4d9f0068 (diff) | |
download | chrome-ec-stabilize-14496.B-ish.tar.gz |
ish: Trim down the release branchstabilize-wristpin-14469.59.B-ishstabilize-voshyr-14637.B-ishstabilize-quickfix-14695.187.B-ishstabilize-quickfix-14695.124.B-ishstabilize-quickfix-14526.91.B-ishstabilize-14695.85.B-ishstabilize-14695.107.B-ishstabilize-14682.B-ishstabilize-14633.B-ishstabilize-14616.B-ishstabilize-14589.B-ishstabilize-14588.98.B-ishstabilize-14588.14.B-ishstabilize-14588.123.B-ishstabilize-14536.B-ishstabilize-14532.B-ishstabilize-14528.B-ishstabilize-14526.89.B-ishstabilize-14526.84.B-ishstabilize-14526.73.B-ishstabilize-14526.67.B-ishstabilize-14526.57.B-ishstabilize-14498.B-ishstabilize-14496.B-ishstabilize-14477.B-ishstabilize-14469.9.B-ishstabilize-14469.8.B-ishstabilize-14469.58.B-ishstabilize-14469.41.B-ishstabilize-14442.B-ishstabilize-14438.B-ishstabilize-14411.B-ishstabilize-14396.B-ishstabilize-14395.B-ishstabilize-14388.62.B-ishstabilize-14388.61.B-ishstabilize-14388.52.B-ishstabilize-14385.B-ishstabilize-14345.B-ishstabilize-14336.B-ishstabilize-14333.B-ishrelease-R99-14469.B-ishrelease-R98-14388.B-ishrelease-R102-14695.B-ishrelease-R101-14588.B-ishrelease-R100-14526.B-ishfirmware-cherry-14454.B-ishfirmware-brya-14505.B-ishfirmware-brya-14505.71.B-ishfactory-kukui-14374.B-ishfactory-guybrush-14600.B-ishfactory-cherry-14455.B-ishfactory-brya-14517.B-ish
In the interest of making long-term branch maintenance incur as little
technical debt on us as possible, we should not maintain any files on
the branch we are not actually using.
This has the added effect of making it extremely clear when merging CLs
from the main branch when changes have the possibility to affect us.
The follow-on CL adds a convenience script to actually pull updates from
the main branch and generate a CL for the update.
BUG=b:204206272
BRANCH=ish
TEST=make BOARD=arcada_ish && make BOARD=drallion_ish
Signed-off-by: Jack Rosenthal <jrosenth@chromium.org>
Change-Id: I17e4694c38219b5a0823e0a3e55a28d1348f4b18
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/platform/ec/+/3262038
Reviewed-by: Jett Rink <jettrink@chromium.org>
Reviewed-by: Tom Hughes <tomhughes@chromium.org>
Diffstat (limited to 'zephyr/zmake/zmake')
-rw-r--r-- | zephyr/zmake/zmake/__init__.py | 0 | ||||
-rw-r--r-- | zephyr/zmake/zmake/__main__.py | 273 | ||||
-rw-r--r-- | zephyr/zmake/zmake/build_config.py | 100 | ||||
-rw-r--r-- | zephyr/zmake/zmake/jobserver.py | 144 | ||||
-rw-r--r-- | zephyr/zmake/zmake/modules.py | 99 | ||||
-rw-r--r-- | zephyr/zmake/zmake/multiproc.py | 322 | ||||
-rw-r--r-- | zephyr/zmake/zmake/output_packers.py | 230 | ||||
-rw-r--r-- | zephyr/zmake/zmake/project.py | 248 | ||||
-rw-r--r-- | zephyr/zmake/zmake/toolchains.py | 154 | ||||
-rw-r--r-- | zephyr/zmake/zmake/util.py | 255 | ||||
-rw-r--r-- | zephyr/zmake/zmake/version.py | 166 | ||||
-rw-r--r-- | zephyr/zmake/zmake/zmake.py | 757 |
12 files changed, 0 insertions, 2748 deletions
diff --git a/zephyr/zmake/zmake/__init__.py b/zephyr/zmake/zmake/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 --- a/zephyr/zmake/zmake/__init__.py +++ /dev/null diff --git a/zephyr/zmake/zmake/__main__.py b/zephyr/zmake/zmake/__main__.py deleted file mode 100644 index 31f0436b5a..0000000000 --- a/zephyr/zmake/zmake/__main__.py +++ /dev/null @@ -1,273 +0,0 @@ -# Copyright 2020 The Chromium OS Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -"""The entry point into zmake.""" -import argparse -import inspect -import logging -import os -import pathlib -import sys - -import zmake.multiproc as multiproc -import zmake.zmake as zm - - -def maybe_reexec(argv): - """Re-exec zmake from the EC source tree, if possible and desired. - - Zmake installs into the users' chroot, which makes it convenient - to execute, but can sometimes become tedious when zmake changes - land and users haven't upgraded their chroots yet. - - We can partially subvert this problem by re-execing zmake from the - source if it's available. This won't make it so developers never - need to upgrade their chroots (e.g., a toolchain upgrade could - require chroot upgrades), but at least makes it slightly more - convenient for an average repo sync. - - Args: - argv: The argument list passed to the main function, not - including the executable path. - - Returns: - None, if the re-exec did not happen, or never returns if the - re-exec did happen. - """ - # We only re-exec if we are inside of a chroot (since if installed - # standalone using pip, there's already an "editable install" - # feature for that in pip.) - env = dict(os.environ) - srcroot = env.get("CROS_WORKON_SRCROOT") - if not srcroot: - return - - # If for some reason we decide to move zmake in the future, then - # we don't want to use the re-exec logic. - zmake_path = ( - pathlib.Path(srcroot) / "src" / "platform" / "ec" / "zephyr" / "zmake" - ).resolve() - if not zmake_path.is_dir(): - return - - # If PYTHONPATH is set, it is either because we just did a - # re-exec, or because the user wants to run a specific copy of - # zmake. In either case, we don't want to re-exec. - if "PYTHONPATH" in env: - return - - # Set PYTHONPATH so that we run zmake from source. - env["PYTHONPATH"] = str(zmake_path) - - os.execve(sys.executable, [sys.executable, "-m", "zmake", *argv], env) - - -def call_with_namespace(func, namespace): - """Call a function with arguments applied from a Namespace. - - Args: - func: The callable to call. - namespace: The namespace to apply to the callable. - - Returns: - The result of calling the callable. - """ - kwds = {} - sig = inspect.signature(func) - names = [p.name for p in sig.parameters.values()] - for name, value in vars(namespace).items(): - pyname = name.replace("-", "_") - if pyname in names: - kwds[pyname] = value - return func(**kwds) - - -# Dictionary used to map log level strings to their corresponding int values. -log_level_map = { - "DEBUG": logging.DEBUG, - "INFO": logging.INFO, - "WARNING": logging.WARNING, - "ERROR": logging.ERROR, - "CRITICAL": logging.CRITICAL, -} - - -def main(argv=None): - """The main function. - - Args: - argv: Optionally, the command-line to parse, not including argv[0]. - - Returns: - Zero upon success, or non-zero upon failure. - """ - if argv is None: - argv = sys.argv[1:] - - maybe_reexec(argv) - - parser = argparse.ArgumentParser() - parser.add_argument( - "--checkout", type=pathlib.Path, help="Path to ChromiumOS checkout" - ) - parser.add_argument( - "-D", - "--debug", - action="store_true", - default=False, - help=("Turn on debug features (e.g., stack trace, " "verbose logging)"), - ) - parser.add_argument( - "-j", - "--jobs", - # TODO(b/178196029): ninja doesn't know how to talk to a - # jobserver properly and spams our CPU on all cores. Default - # to -j1 to execute sequentially until we switch to GNU Make. - default=1, - type=int, - help="Degree of multiprogramming to use", - ) - parser.add_argument( - "-l", - "--log-level", - choices=list(log_level_map.keys()), - dest="log_level", - help="Set the logging level (default=INFO)", - ) - parser.add_argument( - "-L", - "--no-log-label", - action="store_false", - help="Turn off logging labels", - dest="log_label", - default=None, - ) - parser.add_argument( - "--log-label", - action="store_true", - help="Turn on logging labels", - dest="log_label", - default=None, - ) - parser.add_argument( - "--modules-dir", - type=pathlib.Path, - help="The path to a directory containing all modules " - "needed. If unspecified, zmake will assume you have " - "a Chrome OS checkout and try locating them in the " - "checkout.", - ) - parser.add_argument( - "--zephyr-base", type=pathlib.Path, help="Path to Zephyr OS repository" - ) - - sub = parser.add_subparsers(dest="subcommand", help="Subcommand") - sub.required = True - - configure = sub.add_parser("configure") - configure.add_argument( - "--ignore-unsupported-zephyr-version", - action="store_true", - help="Don't warn about using an unsupported Zephyr version", - ) - configure.add_argument("-t", "--toolchain", help="Name of toolchain to use") - configure.add_argument( - "--bringup", - action="store_true", - dest="bringup", - help="Enable bringup debugging features", - ) - configure.add_argument( - "-B", "--build-dir", type=pathlib.Path, help="Build directory" - ) - configure.add_argument( - "-b", - "--build", - action="store_true", - dest="build_after_configure", - help="Run the build after configuration", - ) - configure.add_argument( - "--test", - action="store_true", - dest="test_after_configure", - help="Test the .elf file after configuration", - ) - configure.add_argument( - "project_dir", type=pathlib.Path, help="Path to the project to build" - ) - configure.add_argument( - "-c", - "--coverage", - action="store_true", - dest="coverage", - help="Enable CONFIG_COVERAGE Kconfig.", - ) - - build = sub.add_parser("build") - build.add_argument( - "build_dir", - type=pathlib.Path, - help="The build directory used during configuration", - ) - build.add_argument( - "-w", - "--fail-on-warnings", - action="store_true", - help="Exit with code 2 if warnings are detected", - ) - - test = sub.add_parser("test") - test.add_argument( - "build_dir", - type=pathlib.Path, - help="The build directory used during configuration", - ) - - sub.add_parser("testall") - - coverage = sub.add_parser("coverage") - coverage.add_argument( - "build_dir", - type=pathlib.Path, - help="The build directory used during configuration", - ) - - opts = parser.parse_args(argv) - - # Default logging - log_level = logging.INFO - log_label = False - - if opts.log_level: - log_level = log_level_map[opts.log_level] - log_label = True - elif opts.debug: - log_level = logging.DEBUG - log_label = True - - if opts.log_label is not None: - log_label = opts.log_label - if log_label: - log_format = "%(levelname)s: %(message)s" - else: - log_format = "%(message)s" - multiproc.log_job_names = False - - logging.basicConfig(format=log_format, level=log_level) - - if not opts.debug: - sys.tracebacklimit = 0 - - try: - zmake = call_with_namespace(zm.Zmake, opts) - subcommand_method = getattr(zmake, opts.subcommand.replace("-", "_")) - result = call_with_namespace(subcommand_method, opts) - return result - finally: - multiproc.wait_for_log_end() - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/zephyr/zmake/zmake/build_config.py b/zephyr/zmake/zmake/build_config.py deleted file mode 100644 index 9a9c7f36a2..0000000000 --- a/zephyr/zmake/zmake/build_config.py +++ /dev/null @@ -1,100 +0,0 @@ -# Copyright 2020 The Chromium OS Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. -"""Encapsulation of a build configuration.""" - - -import zmake.util as util - - -class BuildConfig: - """A container for build configurations. - - A build config is a tuple of environment variables, cmake - variables, kconfig definitons, and kconfig files. - """ - - def __init__( - self, environ_defs={}, cmake_defs={}, kconfig_defs={}, kconfig_files=[] - ): - self.environ_defs = dict(environ_defs) - self.cmake_defs = dict(cmake_defs) - self.kconfig_defs = dict(kconfig_defs) - self.kconfig_files = kconfig_files - - def popen_cmake( - self, jobclient, project_dir, build_dir, kconfig_path=None, **kwargs - ): - """Run Cmake with this config using a jobclient. - - Args: - jobclient: A JobClient instance. - project_dir: The project directory. - build_dir: Directory to use for Cmake build. - kconfig_path: The path to write out Kconfig definitions. - kwargs: forwarded to popen. - """ - kconfig_files = list(self.kconfig_files) - if kconfig_path: - util.write_kconfig_file(kconfig_path, self.kconfig_defs) - kconfig_files.append(kconfig_path) - elif self.kconfig_defs: - raise ValueError( - "Cannot start Cmake on a config with Kconfig items without a " - "kconfig_path" - ) - - if kconfig_files: - base_config = BuildConfig( - environ_defs=self.environ_defs, cmake_defs=self.cmake_defs - ) - conf_file_config = BuildConfig( - cmake_defs={ - "CONF_FILE": ";".join(str(p.resolve()) for p in kconfig_files) - } - ) - return (base_config | conf_file_config).popen_cmake( - jobclient, project_dir, build_dir, **kwargs - ) - - kwargs["env"] = dict(**kwargs.get("env", {}), **self.environ_defs) - return jobclient.popen( - [ - "/usr/bin/cmake", - "-S", - project_dir, - "-B", - build_dir, - "-GNinja", - *("-D{}={}".format(*pair) for pair in self.cmake_defs.items()), - ], - **kwargs - ) - - def __or__(self, other): - """Combine two BuildConfig instances.""" - if not isinstance(other, BuildConfig): - raise TypeError( - "Unsupported operation | for {} and {}".format(type(self), type(other)) - ) - - return BuildConfig( - environ_defs=dict(**self.environ_defs, **other.environ_defs), - cmake_defs=dict(**self.cmake_defs, **other.cmake_defs), - kconfig_defs=dict(**self.kconfig_defs, **other.kconfig_defs), - kconfig_files=list({*self.kconfig_files, *other.kconfig_files}), - ) - - def __repr__(self): - return "BuildConfig({})".format( - ", ".join( - "{}={!r}".format(name, getattr(self, name)) - for name in [ - "environ_defs", - "cmake_defs", - "kconfig_defs", - "kconfig_files", - ] - if getattr(self, name) - ) - ) diff --git a/zephyr/zmake/zmake/jobserver.py b/zephyr/zmake/zmake/jobserver.py deleted file mode 100644 index 69199a2dc8..0000000000 --- a/zephyr/zmake/zmake/jobserver.py +++ /dev/null @@ -1,144 +0,0 @@ -# Copyright 2020 The Chromium OS Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. -"""Module for job counters, limiting the amount of concurrent executions.""" - -import logging -import multiprocessing -import os -import re -import select -import subprocess - -import zmake - - -class JobHandle: - """Small object to handle claim of a job.""" - - def __init__(self, release_func, *args, **kwargs): - self.release_func = release_func - self.args = args - self.kwargs = kwargs - - def __enter__(self): - pass - - def __exit__(self, exc_type, exc_value, traceback): - self.release_func(*self.args, **self.kwargs) - - -class JobClient: - """Abstract base class for all job clients.""" - - def get_job(self): - """Claim a job.""" - raise NotImplementedError("Abstract method not implemented") - - def env(self): - """Get the environment variables necessary to share the job server.""" - return {} - - def popen(self, *args, **kwargs): - """Start a process using subprocess.Popen - - All other arguments are passed to subprocess.Popen. - - Returns: - A Popen object. - """ - kwargs.setdefault("env", os.environ) - kwargs["env"].update(self.env()) - - logger = logging.getLogger(self.__class__.__name__) - logger.debug("Running %s", zmake.util.repr_command(*args)) - return subprocess.Popen(*args, **kwargs) - - def run(self, *args, claim_job=True, **kwargs): - """Run a process using subprocess.run, optionally claiming a job. - - Args: - claim_job: True if a job should be claimed. - - All other arguments are passed to subprocess.run. - - Returns: - A CompletedProcess object. - """ - if claim_job: - with self.get_job(): - return self.run(*args, claim_job=False, **kwargs) - - kwargs.setdefault("env", os.environ) - kwargs["env"].update(self.env()) - - return subprocess.run(*args, **kwargs) - - -class JobServer(JobClient): - """Abstract Job Server.""" - - def __init__(self, jobs=0): - raise NotImplementedError("Abstract method not implemented") - - -class GNUMakeJobClient(JobClient): - def __init__(self, read_fd, write_fd): - self._pipe = [read_fd, write_fd] - - @classmethod - def from_environ(cls, env=None): - """Create a job client from an environment with the MAKEFLAGS variable. - - If we are started under a GNU Make Job Server, we can search - the environment for a string "--jobserver-auth=R,W", where R - and W will be the read and write file descriptors to the pipe - respectively. If we don't find this environment variable (or - the string inside of it), this will raise an OSError. - - Args: - env: Optionally, the environment to search. - - Returns: - A GNUMakeJobClient configured appropriately. - """ - if env is None: - env = os.environ - makeflags = env.get("MAKEFLAGS") - if not makeflags: - raise OSError("MAKEFLAGS is not set in the environment") - match = re.search(r"--jobserver-auth=(\d+),(\d+)", makeflags) - if not match: - raise OSError("MAKEFLAGS did not contain jobserver flags") - read_fd, write_fd = map(int, match.groups()) - return cls(read_fd, write_fd) - - def get_job(self): - """Claim a job. - - Returns: - A JobHandle object. - """ - byte = os.read(self._pipe[0], 1) - return JobHandle(lambda: os.write(self._pipe[1], byte)) - - def env(self): - """Get the environment variables necessary to share the job server.""" - return {"MAKEFLAGS": "--jobserver-auth={},{}".format(*self._pipe)} - - -class GNUMakeJobServer(JobServer, GNUMakeJobClient): - """Implements a GNU Make POSIX Job Server. - - See https://www.gnu.org/software/make/manual/html_node/POSIX-Jobserver.html - for specification. - """ - - def __init__(self, jobs=0): - if not jobs: - jobs = multiprocessing.cpu_count() - elif jobs > select.PIPE_BUF: - jobs = select.PIPE_BUF - - self._pipe = os.pipe() - os.write(self._pipe[1], b"+" * jobs) diff --git a/zephyr/zmake/zmake/modules.py b/zephyr/zmake/zmake/modules.py deleted file mode 100644 index 5ba0ef73f8..0000000000 --- a/zephyr/zmake/zmake/modules.py +++ /dev/null @@ -1,99 +0,0 @@ -# Copyright 2020 The Chromium OS Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. -"""Registry of known Zephyr modules.""" - -import zmake.build_config as build_config -import zmake.util as util - - -def third_party_module(name, checkout): - """Common callback in registry for all third_party/zephyr modules. - - Args: - name: The name of the module. - checkout: The path to the chromiumos source. - - Return: - The path to the module module. - """ - return checkout / "src" / "third_party" / "zephyr" / name - - -known_modules = { - "hal_stm32": third_party_module, - "cmsis": third_party_module, - "ec": lambda name, checkout: (checkout / "src" / "platform" / "ec"), - "nanopb": third_party_module, -} - - -def locate_from_checkout(checkout_dir): - """Find modules from a Chrome OS checkout. - - Important: this function should only conditionally be called if a - checkout exists. Zmake *can* be used without a Chrome OS source - tree. You should call locate_from_directory if outside of a - Chrome OS source tree. - - Args: - checkout_dir: The path to the chromiumos source. - - Returns: - A dictionary mapping module names to paths. - """ - result = {} - for name, locator in known_modules.items(): - result[name] = locator(name, checkout_dir) - return result - - -def locate_from_directory(directory): - """Create a modules dictionary from a directory. - - This takes a directory, and searches for the known module names - located in it. - - Args: - directory: the directory to search in. - - Returns: - A dictionary mapping module names to paths. - """ - result = {} - - for name in known_modules: - modpath = (directory / name).resolve() - if (modpath / "zephyr" / "module.yml").is_file(): - result[name] = modpath - - return result - - -def setup_module_symlinks(output_dir, modules): - """Setup a directory with symlinks to modules. - - Args: - output_dir: The directory to place the symlinks in. - modules: A dictionary of module names mapping to paths. - - Returns: - The resultant BuildConfig that should be applied to use each - of these modules. - """ - if not output_dir.exists(): - output_dir.mkdir(parents=True) - - module_links = [] - - for name, path in modules.items(): - link_path = output_dir.resolve() / name - util.update_symlink(path, link_path) - module_links.append(link_path) - - if module_links: - return build_config.BuildConfig( - cmake_defs={"ZEPHYR_MODULES": ";".join(map(str, module_links))} - ) - else: - return build_config.BuildConfig() diff --git a/zephyr/zmake/zmake/multiproc.py b/zephyr/zmake/zmake/multiproc.py deleted file mode 100644 index 5e98374c8c..0000000000 --- a/zephyr/zmake/zmake/multiproc.py +++ /dev/null @@ -1,322 +0,0 @@ -# Copyright 2020 The Chromium OS Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. -import collections -import logging -import os -import select -import threading - -"""Zmake multiprocessing utility module. - -This module is used to aid in zmake's multiprocessing. It contains tools -available to log output from multiple processes on the fly. This means that a -process does not need to finish before the output is available to the developer -on the screen. -""" - -# A local pipe use to signal the look that a new file descriptor was added and -# should be included in the select statement. -_logging_interrupt_pipe = os.pipe() -# A condition variable used to synchronize logging operations. -_logging_cv = threading.Condition() -# A map of file descriptors to their LogWriter -_logging_map = {} -# Should we log job names or not -log_job_names = True - - -def reset(): - """Reset this module to its starting state (useful for tests)""" - global _logging_map - - _logging_map = {} - - -class LogWriter: - """Contains information about a file descriptor that is producing output - - There is typically one of these for each file descriptor that a process is - writing to while running (stdout and stderr). - - Properties: - _logger: The logger object to use. - _log_level: The logging level to use. - _override_func: A function used to override the log level. The - function will be called once per line prior to logging and will be - passed the arguments of the line and the default log level. - _written_at_level: dict: - key: log_level - value: True if output was written at that level - _job_id: The name to prepend to logged lines - _file_descriptor: The file descriptor being logged. - """ - - def __init__( - self, logger, log_level, log_level_override_func, job_id, file_descriptor - ): - self._logger = logger - self._log_level = log_level - self._override_func = log_level_override_func - # A map whether output was printed at each logging level - self._written_at_level = collections.defaultdict(lambda: False) - self._job_id = job_id - self._file_descriptor = file_descriptor - - def log_line(self, line): - """Log a line of output - - If the log-level override function requests a change in log level, that - causes self._log_level to be updated accordingly. - - Args: - line: Text line to log - """ - if self._override_func: - # Get the new log level and update the default. The reason we - # want to update the default is that if we hit an error, all - # future logging should be moved to the new logging level. This - # greatly simplifies the logic that is needed to update the log - # level. - self._log_level = self._override_func(line, self._log_level) - if self._job_id and log_job_names: - self._logger.log(self._log_level, "[%s]%s", self._job_id, line) - else: - self._logger.log(self._log_level, line) - self._written_at_level[self._log_level] = True - - def has_written(self, log_level): - """Check if output was written at a certain log level - - Args: - log_level: log level to check - - Returns: - True if any output was written at that log level, False if not - """ - return self._written_at_level[log_level] - - def wait(self): - """Wait for this LogWriter to finish. - - This method will block execution until all the logs have been flushed out. - """ - with _logging_cv: - _logging_cv.wait_for(lambda: self._file_descriptor not in _logging_map) - - -def _log_fd(fd): - """Log information from a single file descriptor. - - This function is BLOCKING. It will read from the given file descriptor until - either the end of line is read or EOF. Once EOF is read it will remove the - file descriptor from _logging_map so it will no longer be used. - Additionally, in some cases, the file descriptor will be closed (caused by - a call to Popen.wait()). In these cases, the file descriptor will also be - removed from the map as it is no longer valid. - """ - with _logging_cv: - writer = _logging_map[fd] - if fd.closed: - del _logging_map[fd] - _logging_cv.notify_all() - return - line = fd.readline() - if not line: - # EOF - del _logging_map[fd] - _logging_cv.notify_all() - return - line = line.rstrip("\n") - if line: - writer.log_line(line) - - -def _prune_logging_fds(): - """Prune the current file descriptors under _logging_map. - - This function will iterate over the logging map and check for closed file - descriptors. Every closed file descriptor will be removed. - """ - with _logging_cv: - remove = [fd for fd in _logging_map.keys() if fd.closed] - for fd in remove: - del _logging_map[fd] - if remove: - _logging_cv.notify_all() - - -def _logging_loop(): - """The primary logging thread loop. - - This is the entry point of the logging thread. It will listen for (1) any - new data on the output file descriptors that were added via log_output() and - (2) any new file descriptors being added by log_output(). Once a file - descriptor is ready to be read, this function will call _log_fd to perform - the actual read and logging. - """ - while True: - with _logging_cv: - _logging_cv.wait_for(lambda: _logging_map) - keys = list(_logging_map.keys()) + [_logging_interrupt_pipe[0]] - try: - fds, _, _ = select.select(keys, [], []) - except ValueError: - # One of the file descriptors must be closed, prune them and try - # again. - _prune_logging_fds() - continue - if _logging_interrupt_pipe[0] in fds: - # We got a dummy byte sent by log_output(), this is a signal used to - # break out of the blocking select.select call to tell us that the - # file descriptor set has changed. We just need to read the byte and - # remove this descriptor from the list. If we actually have data - # that should be read it will be read in the for loop below. - os.read(_logging_interrupt_pipe[0], 1) - fds.remove(_logging_interrupt_pipe[0]) - for fd in fds: - _log_fd(fd) - - -_logging_thread = None - - -def log_output( - logger, log_level, file_descriptor, log_level_override_func=None, job_id=None -): - """Log the output from the given file descriptor. - - Args: - logger: The logger object to use. - log_level: The logging level to use. - file_descriptor: The file descriptor to read from. - log_level_override_func: A function used to override the log level. The - function will be called once per line prior to logging and will be - passed the arguments of the line and the default log level. - - Returns: - LogWriter object for the resulting output - """ - with _logging_cv: - global _logging_thread - if _logging_thread is None or not _logging_thread.is_alive(): - # First pass or thread must have died, create a new one. - _logging_thread = threading.Thread(target=_logging_loop, daemon=True) - _logging_thread.start() - - writer = LogWriter( - logger, log_level, log_level_override_func, job_id, file_descriptor - ) - _logging_map[file_descriptor] = writer - # Write a dummy byte to the pipe to break the select so we can add the - # new fd. - os.write(_logging_interrupt_pipe[1], b"x") - # Notify the condition so we can run the select on the current fds. - _logging_cv.notify_all() - return writer - - -def wait_for_log_end(): - """Wait for all the logs to be printed. - - This method will block execution until all the logs have been flushed out. - """ - with _logging_cv: - _logging_cv.wait_for(lambda: not _logging_map) - - -class Executor: - """Parallel executor helper class. - - This class is used to run multiple functions in parallel. The functions MUST - return an integer result code (or throw an exception). This class will start - a thread per operation and wait() for all the threads to resolve. - - Attributes: - lock: The condition variable used to synchronize across threads. - threads: A list of threading.Thread objects currently under this - Executor. - results: A list of result codes returned by each of the functions called - by this Executor. - """ - - def __init__(self): - self.lock = threading.Condition() - self.threads = [] - self.results = [] - self.logger = logging.getLogger(self.__class__.__name__) - - def append(self, func): - """Append the given function to the wait list. - - Once added, the function's return value will be used to determine the - Executor's final result value. The function must return an int result - code or throw an exception. For example: If two functions were added - to the Executor, they will both be run in parallel and their results - will determine whether or not the Executor succeeded. If both functions - returned 0, then the Executor's wait function will also return 0. - - Args: - func: A function which returns an int result code or throws an - exception. - """ - with self.lock: - thread = threading.Thread(target=lambda: self._run_fn(func), daemon=True) - thread.start() - self.threads.append(thread) - - def wait(self): - """Wait for a result to be available. - - This function waits for the executor to resolve (i.e., all - threads have finished). - - Returns: - An integer result code of either the first failed function or 0 if - they all succeeded. - """ - with self.lock: - self.lock.wait_for(predicate=lambda: self._is_finished) - return self._result - - def _run_fn(self, func): - """Entry point to each running thread. - - This function will run the function provided in the append() function. - The result value of the function will be used to determine the - Executor's result value. If the function throws any exception it will be - caught and -1 will be used as the assumed result value. - - Args: - func: The function to run. - """ - try: - result = func() - except Exception as ex: - self.logger.exception(ex) - result = -1 - with self.lock: - self.results.append(result) - self.lock.notify_all() - - @property - def _is_finished(self): - """Whether or not the Executor is considered to be done. - - Returns: - True if the Executor is considered done. - """ - if len(self.threads) == len(self.results): - return True - return False - - @property - def _result(self): - """The result code of the Executor. - - Note that _is_finished must be True for this to have any meaning. - - Returns: - An int representing the result value of the underlying functions. - """ - return next((result for result in self.results if result), 0) diff --git a/zephyr/zmake/zmake/output_packers.py b/zephyr/zmake/zmake/output_packers.py deleted file mode 100644 index 1ba38cf96c..0000000000 --- a/zephyr/zmake/zmake/output_packers.py +++ /dev/null @@ -1,230 +0,0 @@ -# Copyright 2020 The Chromium OS Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. -"""Types which provide many builds and composite them into a single binary.""" -import logging -import shutil -import subprocess - -import zmake.build_config as build_config -import zmake.multiproc -import zmake.util as util - - -class BasePacker: - """Abstract base for all packers.""" - - def __init__(self, project): - self.project = project - - def configs(self): - """Get all of the build configurations necessary. - - Yields: - 2-tuples of config name and a BuildConfig. - """ - yield "singleimage", build_config.BuildConfig() - - def pack_firmware(self, work_dir, jobclient, version_string=""): - """Pack a firmware image. - - Config names from the configs generator are passed as keyword - arguments, with each argument being set to the path of the - build directory. - - Args: - work_dir: A directory to write outputs and temporary files - into. - jobclient: A JobClient object to use. - version_string: The version string, which may end up in - certain parts of the outputs. - - Yields: - 2-tuples of the path of each file in the work_dir (or any - other directory) which should be copied into the output - directory, and the output filename. - """ - raise NotImplementedError("Abstract method not implemented") - - def _get_max_image_bytes(self): - """Get the maximum allowed image size (in bytes). - - This value will generally be found in CONFIG_FLASH_SIZE but may vary - depending on the specific way things are being packed. - - Returns: - The maximum allowed size of the image in bytes. - """ - raise NotImplementedError("Abstract method not implemented") - - def _is_size_bound(self, path): - """Check whether the given path should be constrained by size. - - Generally, .elf files will be unconstrained while .bin files will be - constrained. - - Args: - path: A file's path to test. - - Returns: - True if the file size should be checked. False otherwise. - """ - return path.suffix == ".bin" - - def _check_packed_file_size(self, file, dirs): - """Check that a packed file passes size constraints. - - Args: - file: A file to test. - dirs: A map of the arguments to pass to _get_max_image_bytes - - Returns: - The file if it passes the test. - """ - if not self._is_size_bound( - file - ) or file.stat().st_size <= self._get_max_image_bytes(**dirs): - return file - raise RuntimeError("Output file ({}) too large".format(file)) - - -class ElfPacker(BasePacker): - """Raw proxy for ELF output of a single build.""" - - def pack_firmware(self, work_dir, jobclient, singleimage, version_string=""): - yield singleimage / "zephyr" / "zephyr.elf", "zephyr.elf" - - -class RawBinPacker(BasePacker): - """Raw proxy for zephyr.bin output of a single build.""" - - def pack_firmware(self, work_dir, jobclient, singleimage, version_string=""): - yield singleimage / "zephyr" / "zephyr.bin", "zephyr.bin" - - -class BinmanPacker(BasePacker): - """Packer for RO/RW image to generate a .bin build using FMAP.""" - - ro_file = "zephyr.bin" - rw_file = "zephyr.bin" - - def __init__(self, project): - self.logger = logging.getLogger(self.__class__.__name__) - super().__init__(project) - - def configs(self): - yield "ro", build_config.BuildConfig(kconfig_defs={"CONFIG_CROS_EC_RO": "y"}) - yield "rw", build_config.BuildConfig(kconfig_defs={"CONFIG_CROS_EC_RW": "y"}) - - def pack_firmware(self, work_dir, jobclient, ro, rw, version_string=""): - """Pack RO and RW sections using Binman. - - Binman configuration is expected to be found in the RO build - device-tree configuration. - - Args: - work_dir: The directory used for packing. - jobclient: The client used to run subprocesses. - ro: Directory containing the RO image build. - rw: Directory containing the RW image build. - version_string: The version string to use in FRID/FWID. - - Yields: - 2-tuples of the path of each file in the work_dir that - should be copied into the output directory, and the output - filename. - """ - dts_file_path = ro / "zephyr" / "zephyr.dts" - - # Copy the inputs into the work directory so that Binman can - # find them under a hard-coded name. - shutil.copy2(ro / "zephyr" / self.ro_file, work_dir / "zephyr_ro.bin") - shutil.copy2(rw / "zephyr" / self.rw_file, work_dir / "zephyr_rw.bin") - - # Version in FRID/FWID can be at most 31 bytes long (32, minus - # one for null character). - if len(version_string) > 31: - version_string = version_string[:31] - - proc = jobclient.popen( - [ - "binman", - "-v", - "5", - "build", - "-a", - "version={}".format(version_string), - "-d", - dts_file_path, - "-m", - "-O", - work_dir, - ], - cwd=work_dir, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - encoding="utf-8", - ) - - zmake.multiproc.log_output(self.logger, logging.DEBUG, proc.stdout) - zmake.multiproc.log_output(self.logger, logging.ERROR, proc.stderr) - if proc.wait(timeout=60): - raise OSError("Failed to run binman") - - yield work_dir / "zephyr.bin", "zephyr.bin" - yield ro / "zephyr" / "zephyr.elf", "zephyr.ro.elf" - yield rw / "zephyr" / "zephyr.elf", "zephyr.rw.elf" - - -class NpcxPacker(BinmanPacker): - """Packer for RO/RW image to generate a .bin build using FMAP. - - This expects that the build is setup to generate a - zephyr.npcx.bin for the RO image, which should be packed using - Nuvoton's loader format. - """ - - ro_file = "zephyr.npcx.bin" - npcx_monitor = "npcx_monitor.bin" - - # TODO(b/192401039): CONFIG_FLASH_SIZE is nuvoton-only. Since - # binman already checks sizes, perhaps we can just remove this - # code? - def _get_max_image_bytes(self, ro, rw): - ro_size = util.read_kconfig_autoconf_value( - ro / "zephyr" / "include" / "generated", "CONFIG_FLASH_SIZE" - ) - rw_size = util.read_kconfig_autoconf_value( - ro / "zephyr" / "include" / "generated", "CONFIG_FLASH_SIZE" - ) - return max(int(ro_size, 0), int(rw_size, 0)) * 1024 - - # This can probably be removed too and just rely on binman to - # check the sizes... see the comment above. - def pack_firmware(self, work_dir, jobclient, ro, rw, version_string=""): - for path, output_file in super().pack_firmware( - work_dir, - jobclient, - ro, - rw, - version_string=version_string, - ): - if output_file == "zephyr.bin": - yield ( - self._check_packed_file_size(path, {"ro": ro, "rw": rw}), - "zephyr.bin", - ) - else: - yield path, output_file - - # Include the NPCX monitor file as an output artifact. - yield ro / self.npcx_monitor, self.npcx_monitor - - -# A dictionary mapping packer config names to classes. -packer_registry = { - "binman": BinmanPacker, - "elf": ElfPacker, - "npcx": NpcxPacker, - "raw": RawBinPacker, -} diff --git a/zephyr/zmake/zmake/project.py b/zephyr/zmake/zmake/project.py deleted file mode 100644 index 84151a90b3..0000000000 --- a/zephyr/zmake/zmake/project.py +++ /dev/null @@ -1,248 +0,0 @@ -# Copyright 2020 The Chromium OS Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. -"""Module for project config wrapper object.""" - -import logging -import pathlib -import warnings - -import yaml - -import zmake.build_config as build_config -import zmake.modules as modules -import zmake.output_packers as packers -import zmake.toolchains as toolchains -import zmake.util as util - -# The version of jsonschema in the chroot has a bunch of -# DeprecationWarnings that fire when we import it. Suppress these -# during the import to keep the noise down. -with warnings.catch_warnings(): - warnings.simplefilter("ignore") - import jsonschema - - -def module_dts_overlay_name(modpath, board_name): - """Given a board name, return the expected DTS overlay path. - - Args: - modpath: the module path as a pathlib.Path object - board_name: the name of the board - - Returns: - A pathlib.Path object to the expected overlay path. - """ - return modpath / "zephyr" / "dts" / "board-overlays" / "{}.dts".format(board_name) - - -def find_projects(root_dir): - """Finds all zmake projects in root_dir. - - Args: - root_dir: the root dir as a pathlib.Path object - - Yields: - Project: The next project found. - """ - logging.info("Finding zmake targets under '%s'.", root_dir) - for path in pathlib.Path(root_dir).rglob("zmake.yaml"): - yield Project(path.parent) - - -class ProjectConfig: - """An object wrapping zmake.yaml.""" - - validator = jsonschema.Draft7Validator - schema = { - "type": "object", - "required": [ - "board", - "output-type", - "supported-toolchains", - "supported-zephyr-versions", - ], - "properties": { - "supported-zephyr-versions": { - "type": "array", - "items": { - "type": "string", - "enum": ["v2.6"], - }, - "minItems": 1, - "uniqueItems": True, - }, - "board": { - "type": "string", - }, - "modules": { - "type": "array", - "items": { - "type": "string", - "enum": list(modules.known_modules), - }, - }, - "output-type": { - "type": "string", - "enum": list(packers.packer_registry), - }, - "supported-toolchains": { - "type": "array", - "items": { - "type": "string", - "enum": list(toolchains.support_classes), - }, - }, - "is-test": { - "type": "boolean", - }, - "dts-overlays": { - "type": "array", - "items": { - "type": "string", - }, - }, - }, - } - - def __init__(self, config_dict): - self.validator.check_schema(self.schema) - jsonschema.validate(config_dict, self.schema, cls=self.validator) - self.config_dict = config_dict - - @property - def supported_zephyr_versions(self): - return [ - util.parse_zephyr_version(x) - for x in self.config_dict["supported-zephyr-versions"] - ] - - @property - def board(self): - return self.config_dict["board"] - - @property - def modules(self): - return self.config_dict.get("modules", list(modules.known_modules)) - - @property - def output_packer(self): - return packers.packer_registry[self.config_dict["output-type"]] - - @property - def supported_toolchains(self): - return self.config_dict["supported-toolchains"] - - @property - def is_test(self): - return self.config_dict.get("is-test", False) - - @property - def dts_overlays(self): - return self.config_dict.get("dts-overlays", []) - - -class Project: - """An object encapsulating a project directory.""" - - def __init__(self, project_dir, config_dict=None): - self.project_dir = project_dir.resolve() - if not config_dict: - with open(self.project_dir / "zmake.yaml") as f: - config_dict = yaml.safe_load(f) - self.config = ProjectConfig(config_dict) - self.packer = self.config.output_packer(self) - - def iter_builds(self): - """Iterate thru the build combinations provided by the project's packer. - - Yields: - 2-tuples of a build configuration name and a BuildConfig. - """ - conf = build_config.BuildConfig(cmake_defs={"BOARD": self.config.board}) - prj_conf = self.project_dir / "prj.conf" - if prj_conf.is_file(): - conf |= build_config.BuildConfig(kconfig_files=[prj_conf]) - for build_name, packer_config in self.packer.configs(): - yield build_name, conf | packer_config - - def find_dts_overlays(self, modules): - """Find appropriate dts overlays from registered modules. - - Args: - modules: A dictionary of module names mapping to paths. - - Returns: - A BuildConfig with relevant configurations to enable the - found DTS overlay files. - """ - overlays = [] - for module_path in modules.values(): - dts_path = module_dts_overlay_name(module_path, self.config.board) - if dts_path.is_file(): - overlays.append(dts_path.resolve()) - - overlays.extend(self.project_dir / f for f in self.config.dts_overlays) - - if overlays: - return build_config.BuildConfig( - cmake_defs={"DTC_OVERLAY_FILE": ";".join(map(str, overlays))} - ) - else: - return build_config.BuildConfig() - - def prune_modules(self, module_paths): - """Reduce a modules dict to the ones required by this project. - - If this project does not define a modules list in the - configuration, it is assumed that all known modules to Zmake - are required. This is typically inconsequential as Zephyr - module design conventions require a Kconfig option to actually - enable most modules. - - Args: - module_paths: A dictionary mapping module names to their - paths. This dictionary is not modified. - - Returns: - A new module_paths dictionary with only the modules - required by this project. - - Raises: - A KeyError, if a required module is unavailable. - """ - result = {} - for module in self.config.modules: - try: - result[module] = module_paths[module] - except KeyError as e: - raise KeyError( - "The {!r} module is required by the {} project, but is not " - "available.".format(module, self.project_dir) - ) from e - return result - - def get_toolchain(self, module_paths, override=None): - if override: - if override not in self.config.supported_toolchains: - logging.warning( - "Toolchain %r isn't supported by this project. You're on your own.", - override, - ) - support_class = toolchains.support_classes.get( - override, toolchains.GenericToolchain - ) - return support_class(name=override, modules=module_paths) - else: - for name in self.config.supported_toolchains: - support_class = toolchains.support_classes[name] - toolchain = support_class(name=name, modules=module_paths) - if toolchain.probe(): - logging.info("Toolchain %r selected by probe function.", toolchain) - return toolchain - raise OSError( - "No supported toolchains could be found on your system. If you see " - "this message in the chroot, it indicates a bug. Otherwise, you'll " - "either want to setup your system with a supported toolchain, or " - "manually select an unsupported toolchain with the -t flag." - ) diff --git a/zephyr/zmake/zmake/toolchains.py b/zephyr/zmake/zmake/toolchains.py deleted file mode 100644 index 924448aec5..0000000000 --- a/zephyr/zmake/zmake/toolchains.py +++ /dev/null @@ -1,154 +0,0 @@ -# Copyright 2020 The Chromium OS Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. -"""Definitions of toolchain variables.""" - -import os -import pathlib - -import zmake.build_config as build_config - - -class GenericToolchain: - """Default toolchain if not known to zmake. - - Simply pass ZEPHYR_TOOLCHAIN_VARIANT=name to the build, with - nothing extra. - """ - - def __init__(self, name, modules=None): - self.name = name - self.modules = modules or {} - - def probe(self): - """Probe if the toolchain is available on the system.""" - # Since the toolchain is not known to zmake, we have no way to - # know if it's installed. Simply return False to indicate not - # installed. An unknown toolchain would only be used if -t - # was manually passed to zmake, and is not valid to put in a - # zmake.yaml file. - return False - - def get_build_config(self): - """Get the build configuration for the toolchain. - - Returns: - A build_config.BuildConfig to be applied to the build. - """ - return build_config.BuildConfig( - cmake_defs={ - "ZEPHYR_TOOLCHAIN_VARIANT": self.name, - }, - ) - - -class CorebootSdkToolchain(GenericToolchain): - def probe(self): - # For now, we always assume it's at /opt/coreboot-sdk, since - # that's where it's installed in the chroot. We may want to - # consider adding support for a coreboot-sdk built in the - # user's home directory, for example, which happens if a - # "make crossgcc" is done from the coreboot repository. - return pathlib.Path("/opt/coreboot-sdk").is_dir() - - def get_build_config(self): - return ( - build_config.BuildConfig( - cmake_defs={ - "TOOLCHAIN_ROOT": str(self.modules["ec"] / "zephyr"), - }, - ) - | super().get_build_config() - ) - - -class ZephyrToolchain(GenericToolchain): - def __init__(self, *args, **kwargs): - self.zephyr_sdk_install_dir = self._find_zephyr_sdk() - super().__init__(*args, **kwargs) - - @staticmethod - def _find_zephyr_sdk(): - """Find the Zephyr SDK, if it's installed. - - Returns: - The path to the Zephyr SDK, using the search rules defined by - https://docs.zephyrproject.org/latest/getting_started/installation_linux.html, - or None, if one cannot be found on the system. - """ - from_env = os.getenv("ZEPHYR_SDK_INSTALL_DIR") - if from_env: - return pathlib.Path(from_env) - - def _gen_sdk_paths(): - for prefix in ( - "~", - "~/.local", - "~/.local/opt", - "~/bin", - "/opt", - "/usr", - "/usr/local", - ): - prefix = pathlib.Path(os.path.expanduser(prefix)) - yield prefix / "zephyr-sdk" - yield from prefix.glob("zephyr-sdk-*") - - for path in _gen_sdk_paths(): - if (path / "sdk_version").is_file(): - return path - - return None - - def probe(self): - return bool(self.zephyr_sdk_install_dir) - - def get_build_config(self): - assert self.zephyr_sdk_install_dir - tc_vars = { - "ZEPHYR_SDK_INSTALL_DIR": str(self.zephyr_sdk_install_dir), - } - return ( - build_config.BuildConfig( - environ_defs=tc_vars, - cmake_defs=tc_vars, - ) - | super().get_build_config() - ) - - -class LlvmToolchain(GenericToolchain): - def probe(self): - # TODO: differentiate chroot llvm path vs. something more - # generic? - return pathlib.Path("/usr/bin/x86_64-pc-linux-gnu-clang").exists() - - def get_build_config(self): - # TODO: this contains custom settings for the chroot. Plumb a - # toolchain for "generic-llvm" for external uses? - return ( - build_config.BuildConfig( - cmake_defs={ - "TOOLCHAIN_ROOT": str(self.modules["ec"] / "zephyr"), - }, - ) - | super().get_build_config() - ) - - -class HostToolchain(GenericToolchain): - def probe(self): - # "host" toolchain for Zephyr means GCC. - for search_path in os.getenv("PATH", "/usr/bin").split(":"): - if (pathlib.Path(search_path) / "gcc").exists(): - return True - return False - - -# Mapping of toolchain names -> support class -support_classes = { - "coreboot-sdk": CorebootSdkToolchain, - "host": HostToolchain, - "llvm": LlvmToolchain, - "zephyr": ZephyrToolchain, -} diff --git a/zephyr/zmake/zmake/util.py b/zephyr/zmake/zmake/util.py deleted file mode 100644 index 455cb7c9d6..0000000000 --- a/zephyr/zmake/zmake/util.py +++ /dev/null @@ -1,255 +0,0 @@ -# Copyright 2020 The Chromium OS Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. -"""Common miscellaneous utility functions for zmake.""" - -import os -import pathlib -import re -import shlex - - -def c_str(input_str): - """Make a string that can be included as a literal in C source code. - - Args: - input_str: The string to process. - - Returns: - A string which can be included in C source code. - """ - - def c_chr(char): - # Convert a char in a string to the C representation. Per the - # C standard, we can use all characters but quote, newline, - # and backslash directly with no replacements. - return { - '"': r"\"", - "\n": r"\n", - "\\": "\\\\", - }.get(char, char) - - return '"{}"'.format("".join(map(c_chr, input_str))) - - -def locate_cros_checkout(): - """Find the path to the ChromiumOS checkout. - - Returns: - The first directory found with a .repo directory in it, - starting by checking the CROS_WORKON_SRCROOT environment - variable, then scanning upwards from the current directory, - and finally from a known set of common paths. - """ - - def propose_checkouts(): - yield os.getenv("CROS_WORKON_SRCROOT") - - path = pathlib.Path.cwd() - while path.resolve() != pathlib.Path("/"): - yield path - path = path / ".." - - yield "/mnt/host/source" - yield pathlib.Path.home() / "trunk" - yield pathlib.Path.home() / "chromiumos" - - for path in propose_checkouts(): - if not path: - continue - path = pathlib.Path(path) - if (path / ".repo").is_dir(): - return path.resolve() - - raise FileNotFoundError("Unable to locate a ChromiumOS checkout") - - -def locate_zephyr_base(checkout, version): - """Locate the path to the Zephyr RTOS in a ChromiumOS checkout. - - Args: - checkout: The path to the ChromiumOS checkout. - version: The requested zephyr version, as a tuple of integers. - - Returns: - The path to the Zephyr source. - """ - return ( - checkout - / "src" - / "third_party" - / "zephyr" - / "main" - / "v{}.{}".format(*version[:2]) - ) - - -def read_kconfig_file(path): - """Parse a Kconfig file. - - Args: - path: The path to open. - - Returns: - A dictionary of kconfig items to their values. - """ - result = {} - with open(path) as f: - for line in f: - line, _, _ = line.partition("#") - line = line.strip() - if line: - name, _, value = line.partition("=") - result[name.strip()] = value.strip() - return result - - -def read_kconfig_autoconf_value(path, key): - """Parse an autoconf.h file for a resolved kconfig value - - Args: - path: The path to the autoconf.h file. - key: The define key to lookup. - - Returns: - The value associated with the key or nothing if the key wasn't found. - """ - prog = re.compile(r"^#define\s{}\s(\S+)$".format(key)) - with open(path / "autoconf.h") as f: - for line in f: - m = prog.match(line) - if m: - return m.group(1) - - -def write_kconfig_file(path, config, only_if_changed=True): - """Write out a dictionary to Kconfig format. - - Args: - path: The path to write to. - config: The dictionary to write. - only_if_changed: Set to True if the file should not be written - unless it has changed. - """ - if only_if_changed: - if path.exists() and read_kconfig_file(path) == config: - return - with open(path, "w") as f: - for name, value in config.items(): - f.write("{}={}\n".format(name, value)) - - -def parse_zephyr_version(version_string): - """Parse a human-readable version string (e.g., "v2.4") as a tuple. - - Args: - version_string: The human-readable version string. - - Returns: - A 2-tuple or 3-tuple of integers representing the version. - """ - match = re.fullmatch(r"v?(\d+)[._](\d+)(?:[._](\d+))?", version_string) - if not match: - raise ValueError( - "{} does not look like a Zephyr version.".format(version_string) - ) - return tuple(int(x) for x in match.groups() if x is not None) - - -def read_zephyr_version(zephyr_base): - """Read the Zephyr version from a Zephyr OS checkout. - - Args: - zephyr_base: path to the Zephyr OS repository. - - Returns: - A 3-tuple of the version number (major, minor, patchset). - """ - version_file = pathlib.Path(zephyr_base) / "VERSION" - - file_vars = {} - with open(version_file) as f: - for line in f: - key, sep, value = line.partition("=") - file_vars[key.strip()] = value.strip() - - return ( - int(file_vars["VERSION_MAJOR"]), - int(file_vars["VERSION_MINOR"]), - int(file_vars["PATCHLEVEL"]), - ) - - -def repr_command(argv): - """Represent an argument array as a string. - - Args: - argv: The arguments of the command. - - Returns: - A string which could be pasted into a shell for execution. - """ - return " ".join(shlex.quote(str(arg)) for arg in argv) - - -def update_symlink(target_path, link_path): - """Create a symlink if it does not exist, or links to a different path. - - Args: - target_path: A Path-like object of the desired symlink path. - link_path: A Path-like object of the symlink. - """ - target = target_path.resolve() - if ( - not link_path.is_symlink() - or pathlib.Path(os.readlink(link_path)).resolve() != target - ): - if link_path.exists(): - link_path.unlink() - link_path.symlink_to(target) - - -def log_multi_line(logger, level, message): - """Log a potentially multi-line message to the logger. - - Args: - logger: The Logger object to log to. - level: The logging level to use when logging. - message: The (potentially) multi-line message to log. - """ - for line in message.splitlines(): - if line: - logger.log(level, line) - - -def resolve_build_dir(platform_ec_dir, project_dir, build_dir): - """Resolve the build directory using platform/ec/build/... as default. - - Args: - platform_ec_dir: The path to the chromiumos source's platform/ec - directory. - project_dir: The directory of the project. - build_dir: The directory to build in (may be None). - Returns: - The resolved build directory (using build_dir if not None). - """ - if build_dir: - return build_dir - - if not pathlib.Path.exists(project_dir / "zmake.yaml"): - raise OSError("Invalid configuration") - - # Resolve project_dir to absolute path. - project_dir = project_dir.resolve() - - # Compute the path of project_dir relative to platform_ec_dir. - project_relative_path = pathlib.Path.relative_to(project_dir, platform_ec_dir) - - # Make sure that the project_dir is a subdirectory of platform_ec_dir. - if platform_ec_dir / project_relative_path != project_dir: - raise OSError( - "Can't resolve project directory {} which is not a subdirectory" - " of the platform/ec directory {}".format(project_dir, platform_ec_dir) - ) - - return platform_ec_dir / "build" / project_relative_path diff --git a/zephyr/zmake/zmake/version.py b/zephyr/zmake/zmake/version.py deleted file mode 100644 index 47aba6d804..0000000000 --- a/zephyr/zmake/zmake/version.py +++ /dev/null @@ -1,166 +0,0 @@ -# Copyright 2021 The Chromium OS Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -import datetime -import getpass -import io -import os -import platform -import subprocess - -import zmake.util as util - - -def _get_num_commits(repo): - """Get the number of commits that have been made. - - If a Git repository is available, return the number of commits that have - been made. Otherwise return a fixed count. - - Args: - repo: The path to the git repo. - - Returns: - An integer, the number of commits that have been made. - """ - try: - result = subprocess.run( - ["git", "-C", repo, "rev-list", "HEAD", "--count"], - check=True, - stdout=subprocess.PIPE, - stderr=subprocess.DEVNULL, - encoding="utf-8", - ) - except subprocess.CalledProcessError: - commits = "9999" - else: - commits = result.stdout - - return int(commits) - - -def _get_revision(repo): - """Get the current revision hash. - - If a Git repository is available, return the hash of the current index. - Otherwise return the hash of the VCSID environment variable provided by - the packaging system. - - Args: - repo: The path to the git repo. - - Returns: - A string, of the current revision. - """ - try: - result = subprocess.run( - ["git", "-C", repo, "log", "-n1", "--format=%H"], - check=True, - stdout=subprocess.PIPE, - stderr=subprocess.DEVNULL, - encoding="utf-8", - ) - except subprocess.CalledProcessError: - # Fall back to the VCSID provided by the packaging system. - # Format is 0.0.1-r425-032666c418782c14fe912ba6d9f98ffdf0b941e9 for - # releases and 9999-032666c418782c14fe912ba6d9f98ffdf0b941e9 for - # 9999 ebuilds. - vcsid = os.environ.get("VCSID", "9999-unknown") - revision = vcsid.rsplit("-", 1)[1] - else: - revision = result.stdout - - return revision - - -def get_version_string(project, zephyr_base, modules, static=False): - """Get the version string associated with a build. - - Args: - project: a zmake.project.Project object - zephyr_base: the path to the zephyr directory - modules: a dictionary mapping module names to module paths - static: if set, create a version string not dependent on git - commits, thus allowing binaries to be compared between two - commits. - - Returns: - A version string which can be placed in FRID, FWID, or used in - the build for the OS. - """ - major_version, minor_version, *_ = util.read_zephyr_version(zephyr_base) - project_id = project.project_dir.parts[-1] - num_commits = 0 - - if static: - vcs_hashes = "STATIC" - else: - repos = { - "os": zephyr_base, - **modules, - } - - for repo in repos.values(): - num_commits += _get_num_commits(repo) - - vcs_hashes = ",".join( - "{}:{}".format(name, _get_revision(repo)[:6]) - for name, repo in sorted( - repos.items(), - # Put the EC module first, then Zephyr OS kernel, as - # these are probably the most important hashes to - # developers. - key=lambda p: (p[0] != "ec", p[0] != "os", p), - ) - ) - - return "{}_v{}.{}.{}-{}".format( - project_id, major_version, minor_version, num_commits, vcs_hashes - ) - - -def write_version_header(version_str, output_path, static=False): - """Generate a version header and write it to the specified path. - - Generate a version header in the format expected by the EC build - system, and write it out only if the version header does not exist - or changes. We don't write in the case that the version header - does exist and was unchanged, which allows "zmake build" commands - on an unchanged tree to be an effective no-op. - - Args: - version_str: The version string to be used in the header, such - as one generated by get_version_string. - output_path: The file path to write at (a pathlib.Path - object). - static: If true, generate a header which does not include - information like the username, hostname, or date, allowing - the build to be reproducible. - """ - output = io.StringIO() - output.write("/* This file is automatically generated by zmake */\n") - - def add_def(name, value): - output.write("#define {} {}\n".format(name, util.c_str(value))) - - def add_def_unquoted(name, value): - output.write("#define {} {}\n".format(name, value)) - - add_def("VERSION", version_str) - add_def("CROS_EC_VERSION32", version_str[:31]) - - if static: - add_def("BUILDER", "reproducible@build") - add_def("DATE", "STATIC_VERSION_DATE") - else: - add_def("BUILDER", "{}@{}".format(getpass.getuser(), platform.node())) - add_def("DATE", datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")) - - add_def("CROS_FWID_MISSING_STR", "CROS_FWID_MISSING") - # TODO(b/198475757): Add zmake support for getting CROS_FWID32 - add_def_unquoted("CROS_FWID32", "CROS_FWID_MISSING_STR") - - contents = output.getvalue() - if not output_path.exists() or output_path.read_text() != contents: - output_path.write_text(contents) diff --git a/zephyr/zmake/zmake/zmake.py b/zephyr/zmake/zmake/zmake.py deleted file mode 100644 index 1b60e66f66..0000000000 --- a/zephyr/zmake/zmake/zmake.py +++ /dev/null @@ -1,757 +0,0 @@ -# Copyright 2020 The Chromium OS Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -"""Module encapsulating Zmake wrapper object.""" -import logging -import os -import pathlib -import re -import shutil -import subprocess -import tempfile - -import zmake.build_config -import zmake.jobserver -import zmake.modules -import zmake.multiproc -import zmake.project -import zmake.util as util -import zmake.version - -ninja_warnings = re.compile(r"^(\S*: )?warning:.*") -ninja_errors = re.compile(r"error:.*") - - -def ninja_stdout_log_level_override(line, current_log_level): - """Update the log level for ninja builds if we hit an error. - - Ninja builds prints everything to stdout, but really we want to start - logging things to CRITICAL - - Args: - line: The line that is about to be logged. - current_log_level: The active logging level that would be used for the - line. - """ - # Output lines from Zephyr that are not normally useful - # Send any lines that start with these strings to INFO - cmake_suppress = [ - "-- ", # device tree messages - "Loaded configuration", - "Including boilerplate", - "Parsing ", - "No change to configuration", - "No change to Kconfig header", - ] - - # Herewith a long list of things which are really for debugging, not - # development. Return logging.DEBUG for each of these. - - # ninja puts progress information on stdout - if line.startswith("["): - return logging.DEBUG - # we don't care about entering directories since it happens every time - if line.startswith("ninja: Entering directory"): - return logging.DEBUG - # we know the build stops from the compiler messages and ninja return code - if line.startswith("ninja: build stopped"): - return logging.DEBUG - # someone prints a *** SUCCESS *** message which we don't need - if line.startswith("***"): - return logging.DEBUG - # dopey ninja puts errors on stdout, so fix that. It does not look - # likely that it will be fixed upstream: - # https://github.com/ninja-build/ninja/issues/1537 - # Try to drop output about the device tree - if any(line.startswith(x) for x in cmake_suppress): - return logging.INFO - # this message is a bit like make failing. We already got the error output. - if line.startswith("FAILED: CMakeFiles"): - return logging.INFO - # if a particular file fails it shows the build line used, but that is not - # useful except for debugging. - if line.startswith("ccache"): - return logging.DEBUG - if ninja_warnings.match(line): - return logging.WARNING - if ninja_errors.match(line): - return logging.ERROR - # When we see "Memory region" go into INFO, and stay there as long as the - # line starts with \S+: - if line.startswith("Memory region"): - return logging.INFO - if current_log_level == logging.INFO and line.split()[0].endswith(":"): - return current_log_level - if current_log_level == logging.WARNING: - return current_log_level - return logging.ERROR - - -def cmake_log_level_override(line, default_log_level): - """Update the log level for cmake output if we hit an error. - - Cmake prints some messages that are less than useful during - development. - - Args: - line: The line that is about to be logged. - default_log_level: The default logging level that will be used for the - line. - """ - # Strange output from Zephyr that we normally ignore - if line.startswith("Including boilerplate"): - return logging.DEBUG - elif line.startswith("devicetree error:"): - return logging.ERROR - if ninja_warnings.match(line): - return logging.WARNING - if ninja_errors.match(line): - return logging.ERROR - return default_log_level - - -def get_process_failure_msg(proc): - """Creates a suitable failure message if something exits badly - - Args: - proc: subprocess.Popen object containing the thing that failed - - Returns: - Failure message as a string: - """ - return "Execution failed (return code={}): {}\n".format( - proc.returncode, util.repr_command(proc.args) - ) - - -class Zmake: - """Wrapper class encapsulating zmake's supported operations. - - The invocations of the constructor and the methods actually comes - from the main function. The command line arguments are translated - such that dashes are replaced with underscores and applied as - keyword arguments to the constructor and the method, and the - subcommand invoked becomes the method run. - - As such, you won't find documentation for each method's parameters - here, as it would be duplicate of the help strings from the - command line. Run "zmake --help" for full documentation of each - parameter. - - Properties: - executor: a zmake.multiproc.Executor object for submitting - tasks to. - _sequential: True to check the results of each build job sequentially, - before launching more, False to just do this after all jobs complete - """ - - def __init__( - self, checkout=None, jobserver=None, jobs=0, modules_dir=None, zephyr_base=None - ): - zmake.multiproc.reset() - self._checkout = checkout - self._zephyr_base = zephyr_base - - if modules_dir: - self.module_paths = zmake.modules.locate_from_directory(modules_dir) - else: - self.module_paths = zmake.modules.locate_from_checkout(self.checkout) - - if jobserver: - self.jobserver = jobserver - else: - try: - self.jobserver = zmake.jobserver.GNUMakeJobClient.from_environ() - except OSError: - self.jobserver = zmake.jobserver.GNUMakeJobServer(jobs=jobs) - - self.logger = logging.getLogger(self.__class__.__name__) - self.executor = zmake.multiproc.Executor() - self._sequential = jobs == 1 - - @property - def checkout(self): - if not self._checkout: - self._checkout = util.locate_cros_checkout() - return self._checkout.resolve() - - def locate_zephyr_base(self, version): - """Locate the Zephyr OS repository. - - Args: - version: If a Zephyr OS base was not supplied to Zmake, - which version to search for as a tuple of integers. - This argument is ignored if a Zephyr base was supplied - to Zmake. - Returns: - A pathlib.Path to the found Zephyr OS repository. - """ - if self._zephyr_base: - return self._zephyr_base - - return util.locate_zephyr_base(self.checkout, version) - - def configure( - self, - project_dir, - build_dir=None, - toolchain=None, - ignore_unsupported_zephyr_version=False, - build_after_configure=False, - test_after_configure=False, - bringup=False, - coverage=False, - ): - """Set up a build directory to later be built by "zmake build".""" - project = zmake.project.Project(project_dir) - supported_versions = project.config.supported_zephyr_versions - - zephyr_base = self.locate_zephyr_base(max(supported_versions)).resolve() - - # Ignore the patchset from the Zephyr version. - zephyr_version = util.read_zephyr_version(zephyr_base)[:2] - - if ( - not ignore_unsupported_zephyr_version - and zephyr_version not in supported_versions - ): - raise ValueError( - "The Zephyr OS version (v{}.{}) is not supported by the " - "project. You may wish to either configure zmake.yaml to " - "support this version, or pass " - "--ignore-unsupported-zephyr-version.".format(*zephyr_version) - ) - - # Resolve build_dir if needed. - build_dir = util.resolve_build_dir( - platform_ec_dir=self.module_paths["ec"], - project_dir=project_dir, - build_dir=build_dir, - ) - # Make sure the build directory is clean. - if os.path.exists(build_dir): - self.logger.info("Clearing old build directory %s", build_dir) - shutil.rmtree(build_dir) - - generated_include_dir = (build_dir / "include").resolve() - base_config = zmake.build_config.BuildConfig( - environ_defs={"ZEPHYR_BASE": str(zephyr_base), "PATH": "/usr/bin"}, - cmake_defs={ - "DTS_ROOT": str(self.module_paths["ec"] / "zephyr"), - "SYSCALL_INCLUDE_DIRS": str( - self.module_paths["ec"] / "zephyr" / "include" / "drivers" - ), - "ZMAKE_INCLUDE_DIR": str(generated_include_dir), - }, - ) - - # Prune the module paths to just those required by the project. - module_paths = project.prune_modules(self.module_paths) - - module_config = zmake.modules.setup_module_symlinks( - build_dir / "modules", module_paths - ) - - # Symlink the Zephyr base into the build directory so it can - # be used in the build phase. - util.update_symlink(zephyr_base, build_dir / "zephyr_base") - - dts_overlay_config = project.find_dts_overlays(module_paths) - - toolchain_support = project.get_toolchain(module_paths, override=toolchain) - toolchain_config = toolchain_support.get_build_config() - - if bringup: - base_config |= zmake.build_config.BuildConfig( - kconfig_defs={"CONFIG_PLATFORM_EC_BRINGUP": "y"} - ) - if coverage: - base_config |= zmake.build_config.BuildConfig( - kconfig_defs={"CONFIG_COVERAGE": "y"} - ) - - if not build_dir.exists(): - build_dir = build_dir.mkdir() - if not generated_include_dir.exists(): - generated_include_dir.mkdir() - processes = [] - self.logger.info("Building %s in %s.", project_dir, build_dir) - for build_name, build_config in project.iter_builds(): - self.logger.info("Configuring %s:%s.", project_dir, build_name) - config = ( - base_config - | toolchain_config - | module_config - | dts_overlay_config - | build_config - ) - output_dir = build_dir / "build-{}".format(build_name) - kconfig_file = build_dir / "kconfig-{}.conf".format(build_name) - proc = config.popen_cmake( - self.jobserver, - project_dir, - output_dir, - kconfig_file, - stdin=subprocess.DEVNULL, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - encoding="utf-8", - errors="replace", - ) - job_id = "{}:{}".format(project_dir, build_name) - zmake.multiproc.log_output( - self.logger, - logging.DEBUG, - proc.stdout, - log_level_override_func=cmake_log_level_override, - job_id=job_id, - ) - zmake.multiproc.log_output( - self.logger, - logging.ERROR, - proc.stderr, - log_level_override_func=cmake_log_level_override, - job_id=job_id, - ) - if self._sequential: - if proc.wait(): - raise OSError(get_process_failure_msg(proc)) - else: - processes.append(proc) - for proc in processes: - if proc.wait(): - raise OSError(get_process_failure_msg(proc)) - - # Create symlink to project - util.update_symlink(project_dir, build_dir / "project") - - if test_after_configure: - return self.test(build_dir=build_dir) - elif build_after_configure: - return self.build(build_dir=build_dir) - - def build(self, build_dir, output_files_out=None, fail_on_warnings=False): - """Build a pre-configured build directory.""" - - def wait_and_check_success(procs, writers): - """Wait for processes to complete and check for errors - - Args: - procs: List of subprocess.Popen objects to check - writers: List of LogWriter objects to check - - Returns: - True if all if OK - False if an error was found (so that zmake should exit) - """ - bad = None - for proc in procs: - if proc.wait() and not bad: - bad = proc - if bad: - # Just show the first bad process for now. Both builds likely - # produce the same error anyway. If they don't, the user can - # still take action on the errors/warnings provided. Showing - # multiple 'Execution failed' messages is not very friendly - # since it exposes the fragmented nature of the build. - raise OSError(get_process_failure_msg(bad)) - - # Let all output be produced before exiting - for writer in writers: - writer.wait() - if fail_on_warnings and any( - w.has_written(logging.WARNING) or w.has_written(logging.ERROR) - for w in writers - ): - self.logger.warning("zmake: Warnings detected in build: aborting") - return False - return True - - procs = [] - log_writers = [] - dirs = {} - - build_dir = build_dir.resolve() - project = zmake.project.Project(build_dir / "project") - - # Compute the version string. - version_string = zmake.version.get_version_string( - project, - build_dir / "zephyr_base", - zmake.modules.locate_from_directory(build_dir / "modules"), - ) - - # The version header needs to generated during the build phase - # instead of configure, as the tree may have changed since - # configure was run. - zmake.version.write_version_header( - version_string, - build_dir / "include" / "ec_version.h", - ) - - for build_name, build_config in project.iter_builds(): - with self.jobserver.get_job(): - dirs[build_name] = build_dir / "build-{}".format(build_name) - cmd = ["/usr/bin/ninja", "-C", dirs[build_name].as_posix()] - self.logger.info( - "Building %s:%s: %s", - build_dir, - build_name, - zmake.util.repr_command(cmd), - ) - proc = self.jobserver.popen( - cmd, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - encoding="utf-8", - errors="replace", - ) - job_id = "{}:{}".format(build_dir, build_name) - out = zmake.multiproc.log_output( - logger=self.logger, - log_level=logging.INFO, - file_descriptor=proc.stdout, - log_level_override_func=ninja_stdout_log_level_override, - job_id=job_id, - ) - err = zmake.multiproc.log_output( - self.logger, - logging.ERROR, - proc.stderr, - job_id=job_id, - ) - - if self._sequential: - if not wait_and_check_success([proc], [out, err]): - return 2 - else: - procs.append(proc) - log_writers += [out, err] - - if not wait_and_check_success(procs, log_writers): - return 2 - - # Run the packer. - packer_work_dir = build_dir / "packer" - output_dir = build_dir / "output" - for d in output_dir, packer_work_dir: - if not d.exists(): - d.mkdir() - - if output_files_out is None: - output_files_out = [] - for output_file, output_name in project.packer.pack_firmware( - packer_work_dir, self.jobserver, version_string=version_string, **dirs - ): - shutil.copy2(output_file, output_dir / output_name) - self.logger.debug("Output file '%s' created.", output_file) - output_files_out.append(output_file) - - return 0 - - def test(self, build_dir): - """Test a build directory.""" - procs = [] - output_files = [] - self.build(build_dir, output_files_out=output_files) - - # If the project built but isn't a test, just bail. - project = zmake.project.Project(build_dir / "project") - if not project.config.is_test: - return 0 - - for output_file in output_files: - self.logger.info("Running tests in %s.", output_file) - with self.jobserver.get_job(): - proc = self.jobserver.popen( - [output_file], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - encoding="utf-8", - errors="replace", - ) - job_id = "test {}".format(output_file) - zmake.multiproc.log_output( - self.logger, - logging.DEBUG, - proc.stdout, - job_id=job_id, - ) - zmake.multiproc.log_output( - self.logger, - logging.ERROR, - proc.stderr, - job_id=job_id, - ) - procs.append(proc) - - for idx, proc in enumerate(procs): - if proc.wait(): - raise OSError(get_process_failure_msg(proc)) - return 0 - - def testall(self): - """Test all the valid test targets""" - tmp_dirs = [] - for project in zmake.project.find_projects(self.module_paths["ec"] / "zephyr"): - is_test = project.config.is_test - temp_build_dir = tempfile.mkdtemp( - suffix="-{}".format(os.path.basename(project.project_dir.as_posix())), - prefix="zbuild-", - ) - tmp_dirs.append(temp_build_dir) - # Configure and run the test. - self.executor.append( - func=lambda: self.configure( - project_dir=project.project_dir, - build_dir=pathlib.Path(temp_build_dir), - build_after_configure=True, - test_after_configure=is_test, - ) - ) - - rv = self.executor.wait() - for tmpdir in tmp_dirs: - shutil.rmtree(tmpdir) - return rv - - def _run_lcov(self, build_dir, lcov_file, initial=False, gcov=""): - gcov = os.path.abspath(gcov) - with self.jobserver.get_job(): - if initial: - self.logger.info("Running (initial) lcov on %s.", build_dir) - else: - self.logger.info("Running lcov on %s.", build_dir) - cmd = [ - "/usr/bin/lcov", - "--gcov-tool", - gcov, - "-q", - "-o", - "-", - "-c", - "-d", - build_dir, - "-t", - lcov_file.stem, - "--exclude", - "*/build-*/zephyr/*/generated/*", - "--exclude", - "*/ec/test/*", - "--exclude", - "*/ec/zephyr/shim/chip/npcx/npcx_monitor/*", - "--exclude", - "*/ec/zephyr/emul/*", - "--exclude", - "*/ec/zephyr/test/*", - "--exclude", - "*/testsuite/*", - "--exclude", - "*/subsys/emul/*", - ] - if initial: - cmd += ["-i"] - proc = self.jobserver.popen( - cmd, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - encoding="utf-8", - errors="replace", - ) - zmake.multiproc.log_output( - self.logger, - logging.WARNING, - proc.stderr, - job_id="{}-lcov".format(build_dir), - ) - - with open(lcov_file, "w") as outfile: - for line in proc.stdout: - if line.startswith("SF:"): - path = line[3:].rstrip() - outfile.write("SF:%s\n" % os.path.realpath(path)) - else: - outfile.write(line) - if proc.wait(): - raise OSError(get_process_failure_msg(proc)) - - return 0 - - def _coverage_compile_only(self, project, build_dir, lcov_file): - self.logger.info("Building %s in %s", project.project_dir, build_dir) - rv = self.configure( - project_dir=project.project_dir, - build_dir=build_dir, - build_after_configure=False, - test_after_configure=False, - coverage=True, - ) - if rv: - return rv - - # Compute the version string. - version_string = zmake.version.get_version_string( - project, - build_dir / "zephyr_base", - zmake.modules.locate_from_directory(build_dir / "modules"), - ) - - # The version header needs to generated during the build phase - # instead of configure, as the tree may have changed since - # configure was run. - zmake.version.write_version_header( - version_string, - build_dir / "include" / "ec_version.h", - ) - - # Use ninja to compile the all.libraries target. - build_project = zmake.project.Project(build_dir / "project") - - procs = [] - dirs = {} - gcov = "gcov.sh-not-found" - for build_name, build_config in build_project.iter_builds(): - self.logger.info("Building %s:%s all.libraries.", build_dir, build_name) - dirs[build_name] = build_dir / "build-{}".format(build_name) - gcov = dirs[build_name] / "gcov.sh" - proc = self.jobserver.popen( - ["/usr/bin/ninja", "-C", dirs[build_name], "all.libraries"], - # Ninja will connect as a job client instead and claim - # many jobs. - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - encoding="utf-8", - errors="replace", - ) - job_id = "{}:{}".format(build_dir, build_name) - zmake.multiproc.log_output( - logger=self.logger, - log_level=logging.DEBUG, - file_descriptor=proc.stdout, - log_level_override_func=ninja_stdout_log_level_override, - job_id=job_id, - ) - zmake.multiproc.log_output( - self.logger, - logging.ERROR, - proc.stderr, - job_id=job_id, - ) - if self._sequential: - if proc.wait(): - raise OSError(get_process_failure_msg(proc)) - else: - procs.append(proc) - - for proc in procs: - if proc.wait(): - raise OSError(get_process_failure_msg(proc)) - - return self._run_lcov(build_dir, lcov_file, initial=True, gcov=gcov) - - def _coverage_run_test(self, project, build_dir, lcov_file): - self.logger.info("Running test %s in %s", project.project_dir, build_dir) - rv = self.configure( - project_dir=project.project_dir, - build_dir=build_dir, - build_after_configure=True, - test_after_configure=True, - coverage=True, - ) - if rv: - return rv - gcov = "gcov.sh-not-found" - for build_name, build_config in project.iter_builds(): - gcov = build_dir / "build-{}".format(build_name) / "gcov.sh" - return self._run_lcov(build_dir, lcov_file, initial=False, gcov=gcov) - - def coverage(self, build_dir): - """Builds all targets with coverage enabled, and then runs the tests.""" - all_lcov_files = [] - root_dir = self.module_paths["ec"] / "zephyr" - for project in zmake.project.find_projects(root_dir): - is_test = project.config.is_test - rel_path = project.project_dir.relative_to(root_dir) - project_build_dir = pathlib.Path(build_dir).joinpath(rel_path) - lcov_file = pathlib.Path(build_dir).joinpath( - str(rel_path).replace("/", "_") + ".info" - ) - all_lcov_files.append(lcov_file) - if is_test: - # Configure and run the test. - self.executor.append( - func=lambda: self._coverage_run_test( - project, project_build_dir, lcov_file - ) - ) - else: - # Configure and compile the non-test project. - self.executor.append( - func=lambda: self._coverage_compile_only( - project, project_build_dir, lcov_file - ) - ) - if self._sequential: - rv = self.executor.wait() - if rv: - return rv - - rv = self.executor.wait() - if rv: - return rv - - with self.jobserver.get_job(): - # Merge info files into a single lcov.info - self.logger.info("Merging coverage data into %s.", build_dir / "lcov.info") - cmd = ["/usr/bin/lcov", "-o", build_dir / "lcov.info"] - for info in all_lcov_files: - cmd += ["-a", info] - proc = self.jobserver.popen( - cmd, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - encoding="utf-8", - errors="replace", - ) - zmake.multiproc.log_output( - self.logger, logging.ERROR, proc.stderr, job_id="lcov" - ) - zmake.multiproc.log_output( - self.logger, logging.DEBUG, proc.stdout, job_id="lcov" - ) - if proc.wait(): - raise OSError(get_process_failure_msg(proc)) - - # Find the common root dir - prefixdir = os.path.commonprefix(list(self.module_paths.values())) - - # Merge into a nice html report - self.logger.info("Creating coverage report %s.", build_dir / "coverage_rpt") - proc = self.jobserver.popen( - [ - "/usr/bin/genhtml", - "-q", - "-o", - build_dir / "coverage_rpt", - "-t", - "Zephyr EC Unittest", - "-p", - prefixdir, - "-s", - ] - + all_lcov_files, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - encoding="utf-8", - errors="replace", - ) - zmake.multiproc.log_output( - self.logger, logging.ERROR, proc.stderr, job_id="genhtml" - ) - zmake.multiproc.log_output( - self.logger, logging.DEBUG, proc.stdout, job_id="genhtml" - ) - if proc.wait(): - raise OSError(get_process_failure_msg(proc)) - return 0 |