diff options
| author | Jack Rosenthal <jrosenth@chromium.org> | 2021-07-02 11:41:24 -0600 |
|---|---|---|
| committer | Commit Bot <commit-bot@chromium.org> | 2021-07-12 19:15:42 +0000 |
| commit | 6580c97e48e7abca694f8f5d3e955bdf12a6ec55 (patch) | |
| tree | bcda10463444802c8004fb46b4cb86ef5bc3a017 /zephyr/zmake | |
| parent | 0a639566d7e7769b7f627614c3359128c0c42763 (diff) | |
| download | chrome-ec-6580c97e48e7abca694f8f5d3e955bdf12a6ec55.tar.gz | |
zephyr: zmake: run black on all files, enable check in run_tests.sh
Run black on all files. Enable check in run_tests.sh to enforce
future formatting requirements.
BUG=b:192389533
BRANCH=none
TEST=run_tests.sh
Signed-off-by: Jack Rosenthal <jrosenth@chromium.org>
Change-Id: I5d93ef61d32d0dab4fe4bf3a77faf3f6693be627
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/platform/ec/+/3002839
Commit-Queue: Yuval Peress <peress@chromium.org>
Reviewed-by: Yuval Peress <peress@chromium.org>
Diffstat (limited to 'zephyr/zmake')
25 files changed, 989 insertions, 724 deletions
diff --git a/zephyr/zmake/run_tests.sh b/zephyr/zmake/run_tests.sh index 30c1bf4159..7d5f7ac010 100755 --- a/zephyr/zmake/run_tests.sh +++ b/zephyr/zmake/run_tests.sh @@ -27,3 +27,6 @@ pytest --hypothesis-profile=cq . # Check import sorting. isort --check . + +# Check black formatting. +black --check . diff --git a/zephyr/zmake/setup.py b/zephyr/zmake/setup.py index 0aed990e09..4328dc48d7 100644 --- a/zephyr/zmake/setup.py +++ b/zephyr/zmake/setup.py @@ -5,43 +5,34 @@ import setuptools setuptools.setup( - name='zephyr-chrome-utils', - version='0.1', - - description='CrOS Zephyr Utilities', - long_description='Utilities used for working on a Zephyr-based EC', - - url='https://chromium.googlesource.com/chromiumos/platform/ec', - - author='Chromium OS Authors', - author_email='chromiumos-dev@chromium.org', - - license='BSD', - + name="zephyr-chrome-utils", + version="0.1", + description="CrOS Zephyr Utilities", + long_description="Utilities used for working on a Zephyr-based EC", + url="https://chromium.googlesource.com/chromiumos/platform/ec", + author="Chromium OS Authors", + author_email="chromiumos-dev@chromium.org", + license="BSD", # What does your project relate to? - keywords='chromeos', - + keywords="chromeos", # You can just specify the packages manually here if your project is # simple. Or you can use find_packages(). - packages=['zmake'], - - python_requires='>=3.6, <4', - + packages=["zmake"], + python_requires=">=3.6, <4", # List run-time dependencies here. These will be installed by pip when # your project is installed. For an analysis of "install_requires" vs pip's # requirements files see: # https://packaging.python.org/en/latest/requirements.html install_requires=[ - 'jsonschema>=3.2.0', - 'pyyaml>=3.13', + "jsonschema>=3.2.0", + "pyyaml>=3.13", ], - # To provide executable scripts, use entry points in preference to the # "scripts" keyword. Entry points provide cross-platform support and allow # pip to create the appropriate form of executable for the target platform. entry_points={ - 'console_scripts': [ - 'zmake=zmake.__main__:main', + "console_scripts": [ + "zmake=zmake.__main__:main", ], }, ) diff --git a/zephyr/zmake/tests/conftest.py b/zephyr/zmake/tests/conftest.py index ad1c6aedfc..4572e23ffe 100644 --- a/zephyr/zmake/tests/conftest.py +++ b/zephyr/zmake/tests/conftest.py @@ -4,5 +4,6 @@ import hypothesis -hypothesis.settings.register_profile('cq', - suppress_health_check=hypothesis.HealthCheck.all()) +hypothesis.settings.register_profile( + "cq", suppress_health_check=hypothesis.HealthCheck.all() +) diff --git a/zephyr/zmake/tests/test_build_config.py b/zephyr/zmake/tests/test_build_config.py index 5eb72eb058..bf69b8a1fa 100644 --- a/zephyr/zmake/tests/test_build_config.py +++ b/zephyr/zmake/tests/test_build_config.py @@ -18,31 +18,34 @@ from zmake.build_config import BuildConfig # Strategies for use with hypothesis filenames = st.text( - alphabet=set(string.printable) - {'/', ';'}, - min_size=1, - max_size=254).filter(lambda name: name not in ('.', '..')) -paths = st.builds(lambda parts: pathlib.Path('/', *parts), - st.iterables(filenames, min_size=1)) -config_keys = st.text(alphabet=set(string.ascii_uppercase) | {'_'}, min_size=1) -config_values = st.builds(str, st.just('y') | st.just('n') | st.integers()) + alphabet=set(string.printable) - {"/", ";"}, min_size=1, max_size=254 +).filter(lambda name: name not in (".", "..")) +paths = st.builds( + lambda parts: pathlib.Path("/", *parts), st.iterables(filenames, min_size=1) +) +config_keys = st.text(alphabet=set(string.ascii_uppercase) | {"_"}, min_size=1) +config_values = st.builds(str, st.just("y") | st.just("n") | st.integers()) config_dicts = st.dictionaries(keys=config_keys, values=config_values) -config_dicts_at_least_one_entry = st.dictionaries(keys=config_keys, - values=config_values, - min_size=1) - -build_configs = st.builds(BuildConfig, - environ_defs=config_dicts, - cmake_defs=config_dicts, - kconfig_defs=config_dicts, - kconfig_files=st.lists(paths)) -build_configs_no_kconfig = st.builds(BuildConfig, - environ_defs=config_dicts, - cmake_defs=config_dicts) +config_dicts_at_least_one_entry = st.dictionaries( + keys=config_keys, values=config_values, min_size=1 +) + +build_configs = st.builds( + BuildConfig, + environ_defs=config_dicts, + cmake_defs=config_dicts, + kconfig_defs=config_dicts, + kconfig_files=st.lists(paths), +) +build_configs_no_kconfig = st.builds( + BuildConfig, environ_defs=config_dicts, cmake_defs=config_dicts +) build_configs_with_at_least_one_kconfig = st.builds( BuildConfig, environ_defs=config_dicts, cmake_defs=config_dicts, - kconfig_defs=config_dicts_at_least_one_entry) + kconfig_defs=config_dicts_at_least_one_entry, +) @hypothesis.given(st.data(), build_configs) @@ -50,6 +53,7 @@ def test_merge(coins, combined): """Test that when splitting a config in half and merging the two halves, we get the original config back. """ + def split(iterable): left = [] right = [] @@ -67,10 +71,18 @@ def test_merge(coins, combined): kconf1, kconf2 = split(combined.kconfig_defs.items()) files1, files2 = split(combined.kconfig_files) - c1 = BuildConfig(environ_defs=dict(env1), cmake_defs=dict(cmake1), - kconfig_defs=dict(kconf1), kconfig_files=files1) - c2 = BuildConfig(environ_defs=dict(env2), cmake_defs=dict(cmake2), - kconfig_defs=dict(kconf2), kconfig_files=files2) + c1 = BuildConfig( + environ_defs=dict(env1), + cmake_defs=dict(cmake1), + kconfig_defs=dict(kconf1), + kconfig_files=files1, + ) + c2 = BuildConfig( + environ_defs=dict(env2), + cmake_defs=dict(cmake2), + kconfig_defs=dict(kconf2), + kconfig_files=files2, + ) # Merge the split configs merged = c1 | c2 @@ -84,6 +96,7 @@ def test_merge(coins, combined): class FakeJobClient(zmake.jobserver.JobClient): """Simple job client to capture argv/environ.""" + def __init__(self): self.captured_argv = [] self.captured_env = {} @@ -109,21 +122,21 @@ def parse_cmake_args(argv): A 2-tuple of a namespace from argparse and the corresponding parsed Cmake definitions. """ - assert argv[0] == '/usr/bin/cmake' + assert argv[0] == "/usr/bin/cmake" parser = argparse.ArgumentParser(add_help=False) - parser.add_argument('-S', dest='source_dir', type=pathlib.Path) - parser.add_argument('-B', dest='build_dir', type=pathlib.Path) - parser.add_argument('-G', dest='generator') - parser.add_argument('-D', dest='defs', action='append', default=[]) + parser.add_argument("-S", dest="source_dir", type=pathlib.Path) + parser.add_argument("-B", dest="build_dir", type=pathlib.Path) + parser.add_argument("-G", dest="generator") + parser.add_argument("-D", dest="defs", action="append", default=[]) args = parser.parse_args(argv[1:]) # Build the definition dictionary cmake_defs = {} for defn in args.defs: - key, sep, val = defn.partition('=') + key, sep, val = defn.partition("=") if not sep: - val = '1' + val = "1" assert key not in cmake_defs cmake_defs[key] = val @@ -160,12 +173,13 @@ def test_popen_cmake_kconfig_but_no_file(conf, project_dir, build_dir): def test_popen_cmake_kconfig(conf, project_dir, build_dir): job_client = FakeJobClient() - with tempfile.NamedTemporaryFile('w', delete=False) as f: + with tempfile.NamedTemporaryFile("w", delete=False) as f: temp_path = f.name try: - conf.popen_cmake(job_client, project_dir, build_dir, - kconfig_path=pathlib.Path(temp_path)) + conf.popen_cmake( + job_client, project_dir, build_dir, kconfig_path=pathlib.Path(temp_path) + ) args, cmake_defs = parse_cmake_args(job_client.captured_argv) @@ -173,9 +187,9 @@ def test_popen_cmake_kconfig(conf, project_dir, build_dir): expected_kconfig_files.add(temp_path) if expected_kconfig_files: - kconfig_files = set(cmake_defs.pop('CONF_FILE').split(';')) + kconfig_files = set(cmake_defs.pop("CONF_FILE").split(";")) else: - assert 'CONF_FILE' not in cmake_defs + assert "CONF_FILE" not in cmake_defs kconfig_files = set() assert cmake_defs == conf.cmake_defs diff --git a/zephyr/zmake/tests/test_modules.py b/zephyr/zmake/tests/test_modules.py index 2c934628f6..87e5d7bfc9 100644 --- a/zephyr/zmake/tests/test_modules.py +++ b/zephyr/zmake/tests/test_modules.py @@ -10,8 +10,9 @@ import hypothesis.strategies as st import zmake.modules -module_lists = st.lists(st.one_of(*map(st.just, zmake.modules.known_modules)), - unique=True) +module_lists = st.lists( + st.one_of(*map(st.just, zmake.modules.known_modules)), unique=True +) @hypothesis.given(module_lists) @@ -26,13 +27,12 @@ def test_locate_in_directory(modules): for module in modules: module_dir = modules_dir / module - zephyr_dir = module_dir / 'zephyr' + zephyr_dir = module_dir / "zephyr" zephyr_dir.mkdir(parents=True) - module_yml = zephyr_dir / 'module.yml' - module_yml.write_bytes(b'') + module_yml = zephyr_dir / "module.yml" + module_yml.write_bytes(b"") expected_modules[module] = module_dir - assert (zmake.modules.locate_from_directory(modules_dir) - == expected_modules) + assert zmake.modules.locate_from_directory(modules_dir) == expected_modules diff --git a/zephyr/zmake/tests/test_multiproc_executor.py b/zephyr/zmake/tests/test_multiproc_executor.py index 46aafbb823..ebc2be5e4f 100644 --- a/zephyr/zmake/tests/test_multiproc_executor.py +++ b/zephyr/zmake/tests/test_multiproc_executor.py @@ -20,7 +20,7 @@ def test_single_function_executor_fail(): def test_single_function_executor_raise(): executor = zmake.multiproc.Executor() - executor.append(lambda: 1/0) + executor.append(lambda: 1 / 0) assert executor.wait() != 0 diff --git a/zephyr/zmake/tests/test_multiproc_logging.py b/zephyr/zmake/tests/test_multiproc_logging.py index 11fef0be97..2eac9326d3 100644 --- a/zephyr/zmake/tests/test_multiproc_logging.py +++ b/zephyr/zmake/tests/test_multiproc_logging.py @@ -13,19 +13,19 @@ import zmake.multiproc def test_read_output_from_pipe(): semaphore = threading.Semaphore(0) pipe = os.pipe() - fd = io.TextIOWrapper(os.fdopen(pipe[0], 'rb'), encoding='utf-8') + fd = io.TextIOWrapper(os.fdopen(pipe[0], "rb"), encoding="utf-8") logger = mock.Mock(spec=logging.Logger) logger.log.side_effect = lambda log_lvl, line: semaphore.release() zmake.multiproc.log_output(logger, logging.DEBUG, fd, job_id="") - os.write(pipe[1], 'Hello\n'.encode('utf-8')) + os.write(pipe[1], "Hello\n".encode("utf-8")) semaphore.acquire() - logger.log.assert_called_with(logging.DEBUG, 'Hello') + logger.log.assert_called_with(logging.DEBUG, "Hello") def test_read_output_change_log_level(): semaphore = threading.Semaphore(0) pipe = os.pipe() - fd = io.TextIOWrapper(os.fdopen(pipe[0], 'rb'), encoding='utf-8') + fd = io.TextIOWrapper(os.fdopen(pipe[0], "rb"), encoding="utf-8") logger = mock.Mock(spec=logging.Logger) logger.log.side_effect = lambda log_lvl, line: semaphore.release() # This call will log output from fd (the file descriptor) to DEBUG, though @@ -35,18 +35,24 @@ def test_read_output_change_log_level(): logger=logger, log_level=logging.DEBUG, file_descriptor=fd, - log_level_override_func=lambda line, lvl: - logging.CRITICAL if line.startswith('World') else lvl, - job_id="",) - os.write(pipe[1], 'Hello\n'.encode('utf-8')) + log_level_override_func=lambda line, lvl: logging.CRITICAL + if line.startswith("World") + else lvl, + job_id="", + ) + os.write(pipe[1], "Hello\n".encode("utf-8")) semaphore.acquire() - os.write(pipe[1], 'World\n'.encode('utf-8')) + os.write(pipe[1], "World\n".encode("utf-8")) semaphore.acquire() - os.write(pipe[1], 'Bye\n'.encode('utf-8')) + os.write(pipe[1], "Bye\n".encode("utf-8")) semaphore.acquire() - logger.log.assert_has_calls([mock.call(logging.DEBUG, 'Hello'), - mock.call(logging.CRITICAL, 'World'), - mock.call(logging.CRITICAL, 'Bye')]) + logger.log.assert_has_calls( + [ + mock.call(logging.DEBUG, "Hello"), + mock.call(logging.CRITICAL, "World"), + mock.call(logging.CRITICAL, "Bye"), + ] + ) def test_read_output_from_second_pipe(): @@ -58,8 +64,10 @@ def test_read_output_from_second_pipe(): """ semaphore = threading.Semaphore(0) pipes = [os.pipe(), os.pipe()] - fds = [io.TextIOWrapper(os.fdopen(pipes[0][0], 'rb'), encoding='utf-8'), - io.TextIOWrapper(os.fdopen(pipes[1][0], 'rb'), encoding='utf-8')] + fds = [ + io.TextIOWrapper(os.fdopen(pipes[0][0], "rb"), encoding="utf-8"), + io.TextIOWrapper(os.fdopen(pipes[1][0], "rb"), encoding="utf-8"), + ] logger = mock.Mock(spec=logging.Logger) logger.log.side_effect = lambda log_lvl, fmt, id, line: semaphore.release() @@ -67,9 +75,9 @@ def test_read_output_from_second_pipe(): zmake.multiproc.log_output(logger, logging.DEBUG, fds[0], job_id="0") zmake.multiproc.log_output(logger, logging.ERROR, fds[1], job_id="1") - os.write(pipes[1][1], 'Hello\n'.encode('utf-8')) + os.write(pipes[1][1], "Hello\n".encode("utf-8")) semaphore.acquire() - logger.log.assert_called_with(logging.ERROR, '[%s]%s', '1', 'Hello') + logger.log.assert_called_with(logging.ERROR, "[%s]%s", "1", "Hello") def test_read_output_after_another_pipe_closed(): @@ -81,8 +89,10 @@ def test_read_output_after_another_pipe_closed(): """ semaphore = threading.Semaphore(0) pipes = [os.pipe(), os.pipe()] - fds = [io.TextIOWrapper(os.fdopen(pipes[0][0], 'rb'), encoding='utf-8'), - io.TextIOWrapper(os.fdopen(pipes[1][0], 'rb'), encoding='utf-8')] + fds = [ + io.TextIOWrapper(os.fdopen(pipes[0][0], "rb"), encoding="utf-8"), + io.TextIOWrapper(os.fdopen(pipes[1][0], "rb"), encoding="utf-8"), + ] logger = mock.Mock(spec=logging.Logger) logger.log.side_effect = lambda log_lvl, fmt, id, line: semaphore.release() @@ -91,6 +101,6 @@ def test_read_output_after_another_pipe_closed(): zmake.multiproc.log_output(logger, logging.ERROR, fds[1], job_id="1") fds[0].close() - os.write(pipes[1][1], 'Hello\n'.encode('utf-8')) + os.write(pipes[1][1], "Hello\n".encode("utf-8")) semaphore.acquire() - logger.log.assert_called_with(logging.ERROR, '[%s]%s', '1', 'Hello') + logger.log.assert_called_with(logging.ERROR, "[%s]%s", "1", "Hello") diff --git a/zephyr/zmake/tests/test_packers.py b/zephyr/zmake/tests/test_packers.py index 34b256bce6..1709c68098 100644 --- a/zephyr/zmake/tests/test_packers.py +++ b/zephyr/zmake/tests/test_packers.py @@ -15,38 +15,43 @@ import zmake.output_packers as packers # Strategies for use with hypothesis absolute_path = st.from_regex(regex=r"\A/[\w/]*\Z") + @hypothesis.given(absolute_path) @hypothesis.settings(deadline=60000) def test_file_size_unbounded(path): packer = packers.BasePacker(project=None) - packer._is_size_bound = mock.Mock(name='_is_size_bound', return_value=False) - file = pathlib.Path(path) / 'zephyr.bin' + packer._is_size_bound = mock.Mock(name="_is_size_bound", return_value=False) + file = pathlib.Path(path) / "zephyr.bin" assert packer._check_packed_file_size(file=file, dirs={}) == file packer._is_size_bound.assert_called_once_with(file) + @hypothesis.given(st.binary(min_size=5, max_size=100)) @hypothesis.settings(deadline=60000) def test_file_size_in_bounds(data): packer = packers.BasePacker(project=None) - packer._is_size_bound = mock.Mock(name='_is_size_bound', return_value=True) - packer._get_max_image_bytes = mock.Mock(name='_get_max_image_bytes', - return_value=100) + packer._is_size_bound = mock.Mock(name="_is_size_bound", return_value=True) + packer._get_max_image_bytes = mock.Mock( + name="_get_max_image_bytes", return_value=100 + ) with tempfile.TemporaryDirectory() as temp_dir_name: - file = pathlib.Path(temp_dir_name) / 'zephyr.bin' - with open(file, 'wb') as f: + file = pathlib.Path(temp_dir_name) / "zephyr.bin" + with open(file, "wb") as f: f.write(data) assert packer._check_packed_file_size(file=file, dirs={}) == file + @hypothesis.given(st.binary(min_size=101, max_size=200)) @hypothesis.settings(deadline=60000) def test_file_size_out_of_bounds(data): packer = packers.BasePacker(project=None) - packer._is_size_bound = mock.Mock(name='_is_size_bound', return_value=True) - packer._get_max_image_bytes = mock.Mock(name='_get_max_image_bytes', - return_value=100) + packer._is_size_bound = mock.Mock(name="_is_size_bound", return_value=True) + packer._get_max_image_bytes = mock.Mock( + name="_get_max_image_bytes", return_value=100 + ) with tempfile.TemporaryDirectory() as temp_dir_name: - file = pathlib.Path(temp_dir_name) / 'zephyr.bin' - with open(file, 'wb') as f: + file = pathlib.Path(temp_dir_name) / "zephyr.bin" + with open(file, "wb") as f: f.write(data) with pytest.raises(RuntimeError): - packer._check_packed_file_size(file=file, dirs={})
\ No newline at end of file + packer._check_packed_file_size(file=file, dirs={}) diff --git a/zephyr/zmake/tests/test_print_versions.py b/zephyr/zmake/tests/test_print_versions.py index 68abe79c43..f65fa569a3 100644 --- a/zephyr/zmake/tests/test_print_versions.py +++ b/zephyr/zmake/tests/test_print_versions.py @@ -9,30 +9,30 @@ import zmake.zmake as zm def generate_zmake_yaml(dest, versions): - with open(dest, 'w') as f: - f.write('board: test\n') - f.write('output-type: elf\n') - f.write('toolchain: llvm\n') - f.write('supported-zephyr-versions:\n') + with open(dest, "w") as f: + f.write("board: test\n") + f.write("output-type: elf\n") + f.write("toolchain: llvm\n") + f.write("supported-zephyr-versions:\n") for version in versions: - f.write(' - {}\n'.format(version)) + f.write(" - {}\n".format(version)) def test_single_version(capsys): with tempfile.TemporaryDirectory() as temp_dir_name: - generate_zmake_yaml(pathlib.Path(temp_dir_name) / 'zmake.yaml', - ['v2.5']) + generate_zmake_yaml(pathlib.Path(temp_dir_name) / "zmake.yaml", ["v2.5"]) zm.Zmake().print_versions(temp_dir_name) captured = capsys.readouterr().out.splitlines() - assert captured == ['v2.5'] + assert captured == ["v2.5"] def test_multiple_versions(capsys): with tempfile.TemporaryDirectory() as temp_dir_name: - generate_zmake_yaml(pathlib.Path(temp_dir_name) / 'zmake.yaml', - ['v2.5', 'v2.6']) + generate_zmake_yaml( + pathlib.Path(temp_dir_name) / "zmake.yaml", ["v2.5", "v2.6"] + ) zm.Zmake().print_versions(temp_dir_name) captured = capsys.readouterr().out.splitlines() # Compare the sorted lists since we don't guarantee anything about the # print order - assert sorted(captured) == sorted(['v2.5', 'v2.6']) + assert sorted(captured) == sorted(["v2.5", "v2.6"]) diff --git a/zephyr/zmake/tests/test_project.py b/zephyr/zmake/tests/test_project.py index 3279d6ce31..d729b79d71 100644 --- a/zephyr/zmake/tests/test_project.py +++ b/zephyr/zmake/tests/test_project.py @@ -13,8 +13,7 @@ import pytest import zmake.modules import zmake.project -board_names = st.text(alphabet=set(string.ascii_lowercase) | {'_'}, - min_size=1) +board_names = st.text(alphabet=set(string.ascii_lowercase) | {"_"}, min_size=1) sets_of_board_names = st.lists(st.lists(board_names, unique=True)) @@ -24,6 +23,7 @@ class TemporaryProject(tempfile.TemporaryDirectory): Args: config: The config dictionary to be used with the project. """ + def __init__(self, config): self.config = config super().__init__() @@ -47,12 +47,12 @@ def test_find_dts_overlays(modules): with tempfile.TemporaryDirectory() as modpath: modpath = pathlib.Path(modpath) for board in boards: - dts_path = zmake.project.module_dts_overlay_name( - modpath, board) + dts_path = zmake.project.module_dts_overlay_name(modpath, board) dts_path.parent.mkdir(parents=True, exist_ok=True) dts_path.touch() setup_modules_and_dispatch( - modules[1:], test_fn, module_list=module_list + (modpath,)) + modules[1:], test_fn, module_list=module_list + (modpath,) + ) else: test_fn(module_list) @@ -62,30 +62,33 @@ def test_find_dts_overlays(modules): board_file_mapping = {} for modpath, board_list in zip(module_paths, modules): for board in board_list: - file_name = zmake.project.module_dts_overlay_name( - modpath, board) + file_name = zmake.project.module_dts_overlay_name(modpath, board) files = board_file_mapping.get(board, set()) board_file_mapping[board] = files | {file_name} for board, expected_dts_files in board_file_mapping.items(): with TemporaryProject( - {'board': board, - 'toolchain': 'foo', - 'output-type': 'elf', - 'supported-zephyr-versions': ['v2.5']}) as project: - config = project.find_dts_overlays( - dict(enumerate(module_paths))) + { + "board": board, + "toolchain": "foo", + "output-type": "elf", + "supported-zephyr-versions": ["v2.5"], + } + ) as project: + config = project.find_dts_overlays(dict(enumerate(module_paths))) actual_dts_files = set( - config.cmake_defs.get('DTC_OVERLAY_FILE', '').split(';')) + config.cmake_defs.get("DTC_OVERLAY_FILE", "").split(";") + ) assert actual_dts_files == set(map(str, expected_dts_files)) setup_modules_and_dispatch(modules, testcase) -module_lists = st.lists(st.one_of(*map(st.just, zmake.modules.known_modules)), - unique=True) +module_lists = st.lists( + st.one_of(*map(st.just, zmake.modules.known_modules)), unique=True +) @hypothesis.given(module_lists) @@ -94,16 +97,19 @@ def test_prune_modules(modules): """Test the Project.prune_modules method in the usual case (all modules available).""" module_paths = { - name: pathlib.Path('/fake/module/path', name) + name: pathlib.Path("/fake/module/path", name) for name in zmake.modules.known_modules } with TemporaryProject( - {'board': 'native_posix', - 'toolchain': 'coreboot-sdk', - 'output-type': 'elf', - 'supported-zephyr-versions': ['v2.5'], - 'modules': modules}) as project: + { + "board": "native_posix", + "toolchain": "coreboot-sdk", + "output-type": "elf", + "supported-zephyr-versions": ["v2.5"], + "modules": modules, + } + ) as project: assert set(project.prune_modules(module_paths)) == set(modules) @@ -113,15 +119,18 @@ def test_prune_modules_unavailable(): # Missing 'cmsis' module_paths = { - 'hal_stm32': pathlib.Path('/mod/halstm'), + "hal_stm32": pathlib.Path("/mod/halstm"), } with TemporaryProject( - {'board': 'native_posix', - 'toolchain': 'coreboot-sdk', - 'output-type': 'elf', - 'supported-zephyr-versions': ['v2.5'], - 'modules': ['hal_stm32', 'cmsis']}) as project: + { + "board": "native_posix", + "toolchain": "coreboot-sdk", + "output-type": "elf", + "supported-zephyr-versions": ["v2.5"], + "modules": ["hal_stm32", "cmsis"], + } + ) as project: with pytest.raises(KeyError): project.prune_modules(module_paths) @@ -142,23 +151,22 @@ output-type: npcx def test_find_projects(tmp_path): """Test the find_projects method when there are projects.""" - dir = tmp_path.joinpath('one') + dir = tmp_path.joinpath("one") dir.mkdir() - dir.joinpath('zmake.yaml').write_text("board: one\n" + YAML_FILE) - tmp_path.joinpath('two').mkdir() - dir = tmp_path.joinpath('two/a') + dir.joinpath("zmake.yaml").write_text("board: one\n" + YAML_FILE) + tmp_path.joinpath("two").mkdir() + dir = tmp_path.joinpath("two/a") dir.mkdir() - dir.joinpath('zmake.yaml').write_text( - "board: twoa\nis-test: true\n" + YAML_FILE) - dir = tmp_path.joinpath('two/b') + dir.joinpath("zmake.yaml").write_text("board: twoa\nis-test: true\n" + YAML_FILE) + dir = tmp_path.joinpath("two/b") dir.mkdir() - dir.joinpath('zmake.yaml').write_text("board: twob\n" + YAML_FILE) + dir.joinpath("zmake.yaml").write_text("board: twob\n" + YAML_FILE) projects = list(zmake.project.find_projects(tmp_path)) projects.sort(key=lambda x: x.project_dir) assert len(projects) == 3 - assert projects[0].project_dir == tmp_path.joinpath('one') - assert projects[1].project_dir == tmp_path.joinpath('two/a') - assert projects[2].project_dir == tmp_path.joinpath('two/b') + assert projects[0].project_dir == tmp_path.joinpath("one") + assert projects[1].project_dir == tmp_path.joinpath("two/a") + assert projects[2].project_dir == tmp_path.joinpath("two/b") assert not projects[0].config.is_test assert projects[1].config.is_test assert not projects[2].config.is_test diff --git a/zephyr/zmake/tests/test_toolchains.py b/zephyr/zmake/tests/test_toolchains.py index d5549faebf..266013b0e0 100644 --- a/zephyr/zmake/tests/test_toolchains.py +++ b/zephyr/zmake/tests/test_toolchains.py @@ -10,26 +10,26 @@ import zmake.toolchains as toolchains def test_coreboot_sdk(): - config = toolchains.get_toolchain('coreboot-sdk', {'ec': pathlib.Path('/')}) - assert config.cmake_defs['ZEPHYR_TOOLCHAIN_VARIANT'] == 'coreboot-sdk' - assert config.cmake_defs['TOOLCHAIN_ROOT'] == '/zephyr' + config = toolchains.get_toolchain("coreboot-sdk", {"ec": pathlib.Path("/")}) + assert config.cmake_defs["ZEPHYR_TOOLCHAIN_VARIANT"] == "coreboot-sdk" + assert config.cmake_defs["TOOLCHAIN_ROOT"] == "/zephyr" def test_llvm(): - config = toolchains.get_toolchain('llvm', {'ec': pathlib.Path('/')}) - assert config.cmake_defs['ZEPHYR_TOOLCHAIN_VARIANT'] == 'llvm' - assert config.cmake_defs['TOOLCHAIN_ROOT'] == '/zephyr' + config = toolchains.get_toolchain("llvm", {"ec": pathlib.Path("/")}) + assert config.cmake_defs["ZEPHYR_TOOLCHAIN_VARIANT"] == "llvm" + assert config.cmake_defs["TOOLCHAIN_ROOT"] == "/zephyr" -@mock.patch('zmake.toolchains.find_zephyr_sdk', return_value='/opt/zephyr-sdk') +@mock.patch("zmake.toolchains.find_zephyr_sdk", return_value="/opt/zephyr-sdk") def test_zephyr(find_zephyr_sdk): - config = toolchains.get_toolchain('zephyr', {}) - assert config.cmake_defs['ZEPHYR_TOOLCHAIN_VARIANT'] == 'zephyr' - assert config.cmake_defs['ZEPHYR_SDK_INSTALL_DIR'] == '/opt/zephyr-sdk' - assert config.environ_defs['ZEPHYR_SDK_INSTALL_DIR'] == '/opt/zephyr-sdk' + config = toolchains.get_toolchain("zephyr", {}) + assert config.cmake_defs["ZEPHYR_TOOLCHAIN_VARIANT"] == "zephyr" + assert config.cmake_defs["ZEPHYR_SDK_INSTALL_DIR"] == "/opt/zephyr-sdk" + assert config.environ_defs["ZEPHYR_SDK_INSTALL_DIR"] == "/opt/zephyr-sdk" def test_arm_none_eabi(): - config = toolchains.get_toolchain('arm-none-eabi', {}) - assert config.cmake_defs['ZEPHYR_TOOLCHAIN_VARIANT'] == 'cross-compile' - assert config.cmake_defs['CROSS_COMPILE'] == '/usr/bin/arm-none-eabi-' + config = toolchains.get_toolchain("arm-none-eabi", {}) + assert config.cmake_defs["ZEPHYR_TOOLCHAIN_VARIANT"] == "cross-compile" + assert config.cmake_defs["CROSS_COMPILE"] == "/usr/bin/arm-none-eabi-" diff --git a/zephyr/zmake/tests/test_util.py b/zephyr/zmake/tests/test_util.py index 258694d890..15f92f58ab 100644 --- a/zephyr/zmake/tests/test_util.py +++ b/zephyr/zmake/tests/test_util.py @@ -17,13 +17,16 @@ relative_path = st.from_regex(regex=r"\A\w+[\w/]*\Z") @hypothesis.given(relative_path, relative_path, relative_path) @hypothesis.settings(deadline=60000) -def test_resolve_build_dir_with_build_dir(platform_ec_subdir, project_subdir, build_subdir): +def test_resolve_build_dir_with_build_dir( + platform_ec_subdir, project_subdir, build_subdir +): with tempfile.TemporaryDirectory() as temp_dir_name: platform_ec_dir = pathlib.Path(temp_dir_name) / platform_ec_subdir build_dir = util.resolve_build_dir( platform_ec_dir=platform_ec_dir, project_dir=platform_ec_dir / project_subdir, - build_dir=platform_ec_dir / build_subdir) + build_dir=platform_ec_dir / build_subdir, + ) assert build_dir == platform_ec_dir / build_subdir @@ -37,7 +40,8 @@ def test_resolve_build_dir_invalid_project(platform_ec_subdir, project_subdir): util.resolve_build_dir( platform_ec_dir=platform_ec_dir, project_dir=platform_ec_dir / project_subdir, - build_dir=None) + build_dir=None, + ) pytest.fail() except Exception: pass @@ -50,12 +54,11 @@ def test_resolve_build_dir_from_project(platform_ec_subdir, project_subdir): platform_ec_dir = pathlib.Path(temp_dir_name) / platform_ec_subdir project_dir = platform_ec_dir / project_subdir project_dir.mkdir(parents=True) - (project_dir / 'zmake.yaml').touch() + (project_dir / "zmake.yaml").touch() build_dir = util.resolve_build_dir( - platform_ec_dir=platform_ec_dir, - project_dir=project_dir, - build_dir=None) - assert build_dir == platform_ec_dir / 'build' / project_subdir + platform_ec_dir=platform_ec_dir, project_dir=project_dir, build_dir=None + ) + assert build_dir == platform_ec_dir / "build" / project_subdir version_integers = st.integers(min_value=0) @@ -66,20 +69,21 @@ version_tuples = st.tuples(version_integers, version_integers, version_integers) @hypothesis.settings(deadline=60000) def test_read_zephyr_version(version_tuple): with tempfile.TemporaryDirectory() as zephyr_base: - with open(pathlib.Path(zephyr_base) / 'VERSION', 'w') as f: - for name, value in zip(('VERSION_MAJOR', 'VERSION_MINOR', - 'PATCHLEVEL'), - version_tuple): - f.write('{} = {}\n'.format(name, value)) + with open(pathlib.Path(zephyr_base) / "VERSION", "w") as f: + for name, value in zip( + ("VERSION_MAJOR", "VERSION_MINOR", "PATCHLEVEL"), version_tuple + ): + f.write("{} = {}\n".format(name, value)) assert util.read_zephyr_version(zephyr_base) == version_tuple + @hypothesis.given(st.integers()) @hypothesis.settings(deadline=60000) def test_read_kconfig_autoconf_value(value): with tempfile.TemporaryDirectory() as dir: path = pathlib.Path(dir) - with open(path / 'autoconf.h', 'w') as f: - f.write('#define TEST {}'.format(value)) - read_value = util.read_kconfig_autoconf_value(path, 'TEST') + with open(path / "autoconf.h", "w") as f: + f.write("#define TEST {}".format(value)) + read_value = util.read_kconfig_autoconf_value(path, "TEST") assert int(read_value) == value diff --git a/zephyr/zmake/tests/test_version.py b/zephyr/zmake/tests/test_version.py index 8384a98308..7f1304066c 100644 --- a/zephyr/zmake/tests/test_version.py +++ b/zephyr/zmake/tests/test_version.py @@ -11,27 +11,27 @@ import zmake.version as version def _git_init(repo): """Create a new git repository.""" repo.mkdir() - subprocess.run(['git', '-c', 'init.defaultBranch=main', '-C', repo, 'init'], - check=True) + subprocess.run( + ["git", "-c", "init.defaultBranch=main", "-C", repo, "init"], check=True + ) -def _git_add(repo, path, contents='example!\n'): +def _git_add(repo, path, contents="example!\n"): """Write contents and stage a file.""" path.write_text(contents) - subprocess.run(['git', '-C', repo, 'add', path], check=True) + subprocess.run(["git", "-C", repo, "add", path], check=True) -def _git_commit(repo, message='message!'): +def _git_commit(repo, message="message!"): env = { - 'GIT_AUTHOR_NAME': 'Alyssa P. Hacker', - 'GIT_AUTHOR_EMAIL': 'aphacker@example.org', - 'GIT_AUTHOR_DATE': 'Thu, 07 Apr 2005 22:13:13 +0200', - 'GIT_COMMITTER_NAME': 'Ben Bitdiddle', - 'GIT_COMMITTER_EMAIL': 'bitdiddle@example.org', - 'GIT_COMMITTER_DATE': 'Tue, 30 Aug 2005 10:50:30 -0700', + "GIT_AUTHOR_NAME": "Alyssa P. Hacker", + "GIT_AUTHOR_EMAIL": "aphacker@example.org", + "GIT_AUTHOR_DATE": "Thu, 07 Apr 2005 22:13:13 +0200", + "GIT_COMMITTER_NAME": "Ben Bitdiddle", + "GIT_COMMITTER_EMAIL": "bitdiddle@example.org", + "GIT_COMMITTER_DATE": "Tue, 30 Aug 2005 10:50:30 -0700", } - subprocess.run(['git', '-C', repo, 'commit', '-m', message], check=True, - env=env) + subprocess.run(["git", "-C", repo, "commit", "-m", message], check=True, env=env) def _setup_example_repos(tmp_path): @@ -43,46 +43,56 @@ def _setup_example_repos(tmp_path): Returns: A 3-tuple of project, zephyr_base, modules_dict. """ - project_path = tmp_path / 'prj' + project_path = tmp_path / "prj" project_path.mkdir() project = zmake.project.Project( project_path, - config_dict={'board': 'foo', 'toolchain': 'bar', - 'output-type': 'raw', - 'supported-zephyr-versions': ['v2.5']}) + config_dict={ + "board": "foo", + "toolchain": "bar", + "output-type": "raw", + "supported-zephyr-versions": ["v2.5"], + }, + ) # Has one commit. - zephyr_base = tmp_path / 'zephyr_base' + zephyr_base = tmp_path / "zephyr_base" _git_init(zephyr_base) - _git_add(zephyr_base, zephyr_base / 'VERSION', - 'VERSION_MAJOR=2\nVERSION_MINOR=5\nPATCHLEVEL=99\n') - _git_commit(zephyr_base, 'Added version file') + _git_add( + zephyr_base, + zephyr_base / "VERSION", + "VERSION_MAJOR=2\nVERSION_MINOR=5\nPATCHLEVEL=99\n", + ) + _git_commit(zephyr_base, "Added version file") # Has one commit. - mod1 = tmp_path / 'mod1' + mod1 = tmp_path / "mod1" _git_init(mod1) - _git_add(mod1, mod1 / 'file1') + _git_add(mod1, mod1 / "file1") _git_commit(mod1) # Has two commits. - mod2 = tmp_path / 'mod2' + mod2 = tmp_path / "mod2" _git_init(mod2) - _git_add(mod2, mod2 / 'file2') + _git_add(mod2, mod2 / "file2") _git_commit(mod2) - _git_add(mod2, mod2 / 'file3') + _git_add(mod2, mod2 / "file3") _git_commit(mod2) - return project, zephyr_base, {'mod1': mod1, 'mod2': mod2} + return project, zephyr_base, {"mod1": mod1, "mod2": mod2} def test_version_string(tmp_path): project, zephyr_base, modules = _setup_example_repos(tmp_path) - assert (version.get_version_string(project, zephyr_base, modules) - == 'prj_v2.5.4-mod1:02fd7a,mod2:b5991f,os:2c65cb') + assert ( + version.get_version_string(project, zephyr_base, modules) + == "prj_v2.5.4-mod1:02fd7a,mod2:b5991f,os:2c65cb" + ) def test_version_string_static(tmp_path): project, zephyr_base, modules = _setup_example_repos(tmp_path) - assert (version.get_version_string(project, zephyr_base, modules, - static=True) - == 'prj_v2.5.0-STATIC') + assert ( + version.get_version_string(project, zephyr_base, modules, static=True) + == "prj_v2.5.0-STATIC" + ) diff --git a/zephyr/zmake/tests/test_zmake.py b/zephyr/zmake/tests/test_zmake.py index f037ab82f8..a5c132dbaa 100644 --- a/zephyr/zmake/tests/test_zmake.py +++ b/zephyr/zmake/tests/test_zmake.py @@ -26,12 +26,13 @@ OUR_PATH = os.path.dirname(os.path.realpath(__file__)) class FakeProject: """A fake project which requests two builds and does no packing""" + # pylint: disable=too-few-public-methods def __init__(self): self.packer = mock.Mock() self.packer.pack_firmware = mock.Mock(return_value=[]) - self.project_dir = pathlib.Path('FakeProjectDir') + self.project_dir = pathlib.Path("FakeProjectDir") self.config = mock.Mock() self.config.supported_zephyr_versions = [(2, 5)] @@ -39,11 +40,11 @@ class FakeProject: @staticmethod def iter_builds(): """Yield the two builds that zmake normally does""" - yield 'build-ro', zmake.build_config.BuildConfig() - yield 'build-rw', zmake.build_config.BuildConfig() + yield "build-ro", zmake.build_config.BuildConfig() + yield "build-rw", zmake.build_config.BuildConfig() def prune_modules(self, paths): - return {} #pathlib.Path('path')] + return {} # pathlib.Path('path')] def find_dts_overlays(self, module_paths): return zmake.build_config.BuildConfig() @@ -72,14 +73,13 @@ class FakeJobserver(zmake.jobserver.GNUMakeJobServer): """Ignores the provided command and just runs 'cat' instead""" for pattern, filename in self.fnames.items(): # Convert to a list of strings - cmd = [isinstance(c, pathlib.PosixPath) and c.as_posix() or c - for c in cmd] + cmd = [isinstance(c, pathlib.PosixPath) and c.as_posix() or c for c in cmd] if pattern.match(" ".join(cmd)): - new_cmd = ['cat', filename] + new_cmd = ["cat", filename] break else: raise Exception('No pattern matched "%s"' % " ".join(cmd)) - kwargs.pop('env', None) + kwargs.pop("env", None) return self.jobserver.popen(new_cmd, *args, **kwargs) @@ -92,7 +92,7 @@ def get_test_filepath(suffix): Returns: Full path to the test file """ - return os.path.join(OUR_PATH, 'files', 'sample_{}.txt'.format(suffix)) + return os.path.join(OUR_PATH, "files", "sample_{}.txt".format(suffix)) def do_test_with_log_level(log_level, use_configure=False, fnames=None): @@ -112,30 +112,34 @@ def do_test_with_log_level(log_level, use_configure=False, fnames=None): """ if fnames is None: fnames = { - re.compile(r".*build-ro"): get_test_filepath('ro'), - re.compile(r".*build-rw"): get_test_filepath('rw'), + re.compile(r".*build-ro"): get_test_filepath("ro"), + re.compile(r".*build-rw"): get_test_filepath("rw"), } zephyr_base = mock.Mock() - zmk = zm.Zmake(jobserver=FakeJobserver(fnames), - zephyr_base=zephyr_base,) + zmk = zm.Zmake( + jobserver=FakeJobserver(fnames), + zephyr_base=zephyr_base, + ) with LogCapture(level=log_level) as cap: with tempfile.TemporaryDirectory() as tmpname: - with open(os.path.join(tmpname, 'VERSION'), 'w') as fd: - fd.write('''VERSION_MAJOR = 2 + with open(os.path.join(tmpname, "VERSION"), "w") as fd: + fd.write( + """VERSION_MAJOR = 2 VERSION_MINOR = 5 PATCHLEVEL = 0 VERSION_TWEAK = 0 EXTRAVERSION = -''') +""" + ) zephyr_base.resolve = mock.Mock(return_value=pathlib.Path(tmpname)) - with patch('zmake.version.get_version_string', return_value='123'): - with patch.object(zmake.project, 'Project', - return_value=FakeProject()): + with patch("zmake.version.get_version_string", return_value="123"): + with patch.object(zmake.project, "Project", return_value=FakeProject()): if use_configure: - zmk.configure(pathlib.Path(tmpname), - build_dir=pathlib.Path('build')) + zmk.configure( + pathlib.Path(tmpname), build_dir=pathlib.Path("build") + ) else: zmk.build(pathlib.Path(tmpname)) multiproc.wait_for_log_end() @@ -152,57 +156,59 @@ class TestFilters(unittest.TestCase): recs, _ = do_test_with_log_level(logging.ERROR) self.assertFalse(recs) - def test_filter_info(self): """Test what appears on the INFO level""" recs, tmpname = do_test_with_log_level(logging.INFO) # TODO: Remove sets and figure out how to check the lines are in the # right order. expected = { - 'Building {}:build-ro: /usr/bin/ninja -C {}/build-build-ro'.format( - tmpname, tmpname), - 'Building {}:build-rw: /usr/bin/ninja -C {}/build-build-rw'.format( - tmpname, tmpname), + "Building {}:build-ro: /usr/bin/ninja -C {}/build-build-ro".format( + tmpname, tmpname + ), + "Building {}:build-rw: /usr/bin/ninja -C {}/build-build-rw".format( + tmpname, tmpname + ), } - for suffix in ['ro', 'rw']: - with open(get_test_filepath('%s_INFO' % suffix)) as f: + for suffix in ["ro", "rw"]: + with open(get_test_filepath("%s_INFO" % suffix)) as f: for line in f: expected.add( - "[{}:build-{}]{}".format(tmpname, suffix, line.strip())) + "[{}:build-{}]{}".format(tmpname, suffix, line.strip()) + ) # This produces an easy-to-read diff if there is a difference self.assertEqual(expected, set(recs)) - def test_filter_debug(self): """Test what appears on the DEBUG level""" recs, tmpname = do_test_with_log_level(logging.DEBUG) # TODO: Remove sets and figure out how to check the lines are in the # right order. expected = { - 'Building {}:build-ro: /usr/bin/ninja -C {}/build-build-ro'.format( - tmpname, tmpname), - 'Building {}:build-rw: /usr/bin/ninja -C {}/build-build-rw'.format( - tmpname, tmpname), - 'Running cat {}/files/sample_ro.txt'.format(OUR_PATH), - 'Running cat {}/files/sample_rw.txt'.format(OUR_PATH), + "Building {}:build-ro: /usr/bin/ninja -C {}/build-build-ro".format( + tmpname, tmpname + ), + "Building {}:build-rw: /usr/bin/ninja -C {}/build-build-rw".format( + tmpname, tmpname + ), + "Running cat {}/files/sample_ro.txt".format(OUR_PATH), + "Running cat {}/files/sample_rw.txt".format(OUR_PATH), } - for suffix in ['ro', 'rw']: + for suffix in ["ro", "rw"]: with open(get_test_filepath(suffix)) as f: for line in f: expected.add( - "[{}:build-{}]{}".format(tmpname, suffix, line.strip())) + "[{}:build-{}]{}".format(tmpname, suffix, line.strip()) + ) # This produces an easy-to-read diff if there is a difference self.assertEqual(expected, set(recs)) - def test_filter_devicetree_error(self): """Test that devicetree errors appear""" recs, tmpname = do_test_with_log_level( - logging.ERROR, - True, - {re.compile(r'.*'): get_test_filepath('err')}) + logging.ERROR, True, {re.compile(r".*"): get_test_filepath("err")} + ) - dt_errs = [rec for rec in recs if 'adc' in rec] + dt_errs = [rec for rec in recs if "adc" in rec] assert "devicetree error: 'adc' is marked as required" in list(dt_errs)[0] diff --git a/zephyr/zmake/zmake/__main__.py b/zephyr/zmake/zmake/__main__.py index 389d5f09b6..163404bf39 100644 --- a/zephyr/zmake/zmake/__main__.py +++ b/zephyr/zmake/zmake/__main__.py @@ -27,7 +27,7 @@ def call_with_namespace(func, namespace): sig = inspect.signature(func) names = [p.name for p in sig.parameters.values()] for name, value in vars(namespace).items(): - pyname = name.replace('-', '_') + pyname = name.replace("-", "_") if pyname in names: kwds[pyname] = value return func(**kwds) @@ -35,11 +35,11 @@ def call_with_namespace(func, namespace): # Dictionary used to map log level strings to their corresponding int values. log_level_map = { - 'DEBUG': logging.DEBUG, - 'INFO': logging.INFO, - 'WARNING': logging.WARNING, - 'ERROR': logging.ERROR, - 'CRITICAL': logging.CRITICAL + "DEBUG": logging.DEBUG, + "INFO": logging.INFO, + "WARNING": logging.WARNING, + "ERROR": logging.ERROR, + "CRITICAL": logging.CRITICAL, } @@ -56,79 +56,128 @@ def main(argv=None): argv = sys.argv[1:] parser = argparse.ArgumentParser() - parser.add_argument('--checkout', type=pathlib.Path, - help='Path to ChromiumOS checkout') - parser.add_argument('-D', '--debug', action='store_true', default=False, - help=('Turn on debug features (e.g., stack trace, ' - 'verbose logging)')) - parser.add_argument('-j', '--jobs', type=int, - help='Degree of multiprogramming to use') - parser.add_argument('-l', '--log-level', choices=list(log_level_map.keys()), - dest='log_level', - help='Set the logging level (default=WARNING)') - parser.add_argument('-L', '--no-log-label', action='store_true', - default=False, - help='Turn off logging labels') - parser.add_argument('--modules-dir', - type=pathlib.Path, - help='The path to a directory containing all modules ' - 'needed. If unspecified, zmake will assume you have ' - 'a Chrome OS checkout and try locating them in the ' - 'checkout.') - parser.add_argument('--zephyr-base', type=pathlib.Path, - help='Path to Zephyr OS repository') - - sub = parser.add_subparsers(dest='subcommand', help='Subcommand') + parser.add_argument( + "--checkout", type=pathlib.Path, help="Path to ChromiumOS checkout" + ) + parser.add_argument( + "-D", + "--debug", + action="store_true", + default=False, + help=("Turn on debug features (e.g., stack trace, " "verbose logging)"), + ) + parser.add_argument( + "-j", "--jobs", type=int, help="Degree of multiprogramming to use" + ) + parser.add_argument( + "-l", + "--log-level", + choices=list(log_level_map.keys()), + dest="log_level", + help="Set the logging level (default=WARNING)", + ) + parser.add_argument( + "-L", + "--no-log-label", + action="store_true", + default=False, + help="Turn off logging labels", + ) + parser.add_argument( + "--modules-dir", + type=pathlib.Path, + help="The path to a directory containing all modules " + "needed. If unspecified, zmake will assume you have " + "a Chrome OS checkout and try locating them in the " + "checkout.", + ) + parser.add_argument( + "--zephyr-base", type=pathlib.Path, help="Path to Zephyr OS repository" + ) + + sub = parser.add_subparsers(dest="subcommand", help="Subcommand") sub.required = True - print_versions = sub.add_parser('print-versions') - print_versions.add_argument('project_dir', type=pathlib.Path, - help='Path to the project to build') + print_versions = sub.add_parser("print-versions") + print_versions.add_argument( + "project_dir", type=pathlib.Path, help="Path to the project to build" + ) - configure = sub.add_parser('configure') + configure = sub.add_parser("configure") configure.add_argument( - '--ignore-unsupported-zephyr-version', action='store_true', - help="Don't warn about using an unsupported Zephyr version") - configure.add_argument('-t', '--toolchain', help='Name of toolchain to use') - configure.add_argument('--bringup', action='store_true', - dest='bringup', - help='Enable bringup debugging features') - configure.add_argument('-B', '--build-dir', type=pathlib.Path, - help='Build directory') - configure.add_argument('-b', '--build', action='store_true', - dest='build_after_configure', - help='Run the build after configuration') - configure.add_argument('--test', action='store_true', - dest='test_after_configure', - help='Test the .elf file after configuration') - configure.add_argument('project_dir', type=pathlib.Path, - help='Path to the project to build') - configure.add_argument('-c', '--coverage', action='store_true', - dest='coverage', - help='Enable CONFIG_COVERAGE Kconfig.') - - build = sub.add_parser('build') - build.add_argument('build_dir', type=pathlib.Path, - help='The build directory used during configuration') - build.add_argument('-w', '--fail-on-warnings', action='store_true', - help='Exit with code 2 if warnings are detected') - - test = sub.add_parser('test') - test.add_argument('build_dir', type=pathlib.Path, - help='The build directory used during configuration') - - testall = sub.add_parser('testall') - - coverage = sub.add_parser('coverage') - coverage.add_argument('build_dir', type=pathlib.Path, - help='The build directory used during configuration') + "--ignore-unsupported-zephyr-version", + action="store_true", + help="Don't warn about using an unsupported Zephyr version", + ) + configure.add_argument("-t", "--toolchain", help="Name of toolchain to use") + configure.add_argument( + "--bringup", + action="store_true", + dest="bringup", + help="Enable bringup debugging features", + ) + configure.add_argument( + "-B", "--build-dir", type=pathlib.Path, help="Build directory" + ) + configure.add_argument( + "-b", + "--build", + action="store_true", + dest="build_after_configure", + help="Run the build after configuration", + ) + configure.add_argument( + "--test", + action="store_true", + dest="test_after_configure", + help="Test the .elf file after configuration", + ) + configure.add_argument( + "project_dir", type=pathlib.Path, help="Path to the project to build" + ) + configure.add_argument( + "-c", + "--coverage", + action="store_true", + dest="coverage", + help="Enable CONFIG_COVERAGE Kconfig.", + ) + + build = sub.add_parser("build") + build.add_argument( + "build_dir", + type=pathlib.Path, + help="The build directory used during configuration", + ) + build.add_argument( + "-w", + "--fail-on-warnings", + action="store_true", + help="Exit with code 2 if warnings are detected", + ) + + test = sub.add_parser("test") + test.add_argument( + "build_dir", + type=pathlib.Path, + help="The build directory used during configuration", + ) + + testall = sub.add_parser("testall") + + coverage = sub.add_parser("coverage") + coverage.add_argument( + "build_dir", + type=pathlib.Path, + help="The build directory used during configuration", + ) opts = parser.parse_args(argv) if opts.no_log_label: - log_format = '%(message)s' + log_format = "%(message)s" else: - log_format = '%(levelname)s: %(message)s' + log_format = "%(levelname)s: %(message)s" log_level = logging.WARNING if opts.log_level: @@ -142,12 +191,12 @@ def main(argv=None): try: zmake = call_with_namespace(zm.Zmake, opts) - subcommand_method = getattr(zmake, opts.subcommand.replace('-', '_')) + subcommand_method = getattr(zmake, opts.subcommand.replace("-", "_")) result = call_with_namespace(subcommand_method, opts) return result finally: multiproc.wait_for_log_end() -if __name__ == '__main__': +if __name__ == "__main__": sys.exit(main()) diff --git a/zephyr/zmake/zmake/build_config.py b/zephyr/zmake/zmake/build_config.py index eaa579a777..9a9c7f36a2 100644 --- a/zephyr/zmake/zmake/build_config.py +++ b/zephyr/zmake/zmake/build_config.py @@ -13,15 +13,18 @@ class BuildConfig: A build config is a tuple of environment variables, cmake variables, kconfig definitons, and kconfig files. """ - def __init__(self, environ_defs={}, cmake_defs={}, kconfig_defs={}, - kconfig_files=[]): + + def __init__( + self, environ_defs={}, cmake_defs={}, kconfig_defs={}, kconfig_files=[] + ): self.environ_defs = dict(environ_defs) self.cmake_defs = dict(cmake_defs) self.kconfig_defs = dict(kconfig_defs) self.kconfig_files = kconfig_files - def popen_cmake(self, jobclient, project_dir, build_dir, kconfig_path=None, - **kwargs): + def popen_cmake( + self, jobclient, project_dir, build_dir, kconfig_path=None, **kwargs + ): """Run Cmake with this config using a jobclient. Args: @@ -37,39 +40,61 @@ class BuildConfig: kconfig_files.append(kconfig_path) elif self.kconfig_defs: raise ValueError( - 'Cannot start Cmake on a config with Kconfig items without a ' - 'kconfig_path') + "Cannot start Cmake on a config with Kconfig items without a " + "kconfig_path" + ) if kconfig_files: - base_config = BuildConfig(environ_defs=self.environ_defs, - cmake_defs=self.cmake_defs) + base_config = BuildConfig( + environ_defs=self.environ_defs, cmake_defs=self.cmake_defs + ) conf_file_config = BuildConfig( - cmake_defs={'CONF_FILE': ';'.join( - str(p.resolve()) for p in kconfig_files)}) + cmake_defs={ + "CONF_FILE": ";".join(str(p.resolve()) for p in kconfig_files) + } + ) return (base_config | conf_file_config).popen_cmake( - jobclient, project_dir, build_dir, **kwargs) + jobclient, project_dir, build_dir, **kwargs + ) - kwargs['env'] = dict(**kwargs.get('env', {}), **self.environ_defs) + kwargs["env"] = dict(**kwargs.get("env", {}), **self.environ_defs) return jobclient.popen( - ['/usr/bin/cmake', '-S', project_dir, '-B', build_dir, '-GNinja', - *('-D{}={}'.format(*pair) for pair in self.cmake_defs.items())], - **kwargs) + [ + "/usr/bin/cmake", + "-S", + project_dir, + "-B", + build_dir, + "-GNinja", + *("-D{}={}".format(*pair) for pair in self.cmake_defs.items()), + ], + **kwargs + ) def __or__(self, other): """Combine two BuildConfig instances.""" if not isinstance(other, BuildConfig): - raise TypeError("Unsupported operation | for {} and {}".format( - type(self), type(other))) + raise TypeError( + "Unsupported operation | for {} and {}".format(type(self), type(other)) + ) return BuildConfig( environ_defs=dict(**self.environ_defs, **other.environ_defs), cmake_defs=dict(**self.cmake_defs, **other.cmake_defs), kconfig_defs=dict(**self.kconfig_defs, **other.kconfig_defs), - kconfig_files=list({*self.kconfig_files, *other.kconfig_files})) + kconfig_files=list({*self.kconfig_files, *other.kconfig_files}), + ) def __repr__(self): - return 'BuildConfig({})'.format(', '.join( - '{}={!r}'.format(name, getattr(self, name)) - for name in ['environ_defs', 'cmake_defs', 'kconfig_defs', - 'kconfig_files'] - if getattr(self, name))) + return "BuildConfig({})".format( + ", ".join( + "{}={!r}".format(name, getattr(self, name)) + for name in [ + "environ_defs", + "cmake_defs", + "kconfig_defs", + "kconfig_files", + ] + if getattr(self, name) + ) + ) diff --git a/zephyr/zmake/zmake/jobserver.py b/zephyr/zmake/zmake/jobserver.py index a4a8512b82..69199a2dc8 100644 --- a/zephyr/zmake/zmake/jobserver.py +++ b/zephyr/zmake/zmake/jobserver.py @@ -15,6 +15,7 @@ import zmake class JobHandle: """Small object to handle claim of a job.""" + def __init__(self, release_func, *args, **kwargs): self.release_func = release_func self.args = args @@ -29,9 +30,10 @@ class JobHandle: class JobClient: """Abstract base class for all job clients.""" + def get_job(self): """Claim a job.""" - raise NotImplementedError('Abstract method not implemented') + raise NotImplementedError("Abstract method not implemented") def env(self): """Get the environment variables necessary to share the job server.""" @@ -45,8 +47,8 @@ class JobClient: Returns: A Popen object. """ - kwargs.setdefault('env', os.environ) - kwargs['env'].update(self.env()) + kwargs.setdefault("env", os.environ) + kwargs["env"].update(self.env()) logger = logging.getLogger(self.__class__.__name__) logger.debug("Running %s", zmake.util.repr_command(*args)) @@ -67,16 +69,17 @@ class JobClient: with self.get_job(): return self.run(*args, claim_job=False, **kwargs) - kwargs.setdefault('env', os.environ) - kwargs['env'].update(self.env()) + kwargs.setdefault("env", os.environ) + kwargs["env"].update(self.env()) return subprocess.run(*args, **kwargs) class JobServer(JobClient): """Abstract Job Server.""" + def __init__(self, jobs=0): - raise NotImplementedError('Abstract method not implemented') + raise NotImplementedError("Abstract method not implemented") class GNUMakeJobClient(JobClient): @@ -101,12 +104,12 @@ class GNUMakeJobClient(JobClient): """ if env is None: env = os.environ - makeflags = env.get('MAKEFLAGS') + makeflags = env.get("MAKEFLAGS") if not makeflags: - raise OSError('MAKEFLAGS is not set in the environment') - match = re.search(r'--jobserver-auth=(\d+),(\d+)', makeflags) + raise OSError("MAKEFLAGS is not set in the environment") + match = re.search(r"--jobserver-auth=(\d+),(\d+)", makeflags) if not match: - raise OSError('MAKEFLAGS did not contain jobserver flags') + raise OSError("MAKEFLAGS did not contain jobserver flags") read_fd, write_fd = map(int, match.groups()) return cls(read_fd, write_fd) @@ -121,7 +124,7 @@ class GNUMakeJobClient(JobClient): def env(self): """Get the environment variables necessary to share the job server.""" - return {'MAKEFLAGS': '--jobserver-auth={},{}'.format(*self._pipe)} + return {"MAKEFLAGS": "--jobserver-auth={},{}".format(*self._pipe)} class GNUMakeJobServer(JobServer, GNUMakeJobClient): @@ -130,6 +133,7 @@ class GNUMakeJobServer(JobServer, GNUMakeJobClient): See https://www.gnu.org/software/make/manual/html_node/POSIX-Jobserver.html for specification. """ + def __init__(self, jobs=0): if not jobs: jobs = multiprocessing.cpu_count() @@ -137,4 +141,4 @@ class GNUMakeJobServer(JobServer, GNUMakeJobClient): jobs = select.PIPE_BUF self._pipe = os.pipe() - os.write(self._pipe[1], b'+' * jobs) + os.write(self._pipe[1], b"+" * jobs) diff --git a/zephyr/zmake/zmake/modules.py b/zephyr/zmake/zmake/modules.py index e8503c88b2..2541783858 100644 --- a/zephyr/zmake/zmake/modules.py +++ b/zephyr/zmake/zmake/modules.py @@ -17,13 +17,13 @@ def third_party_module(name, checkout): Return: The path to the module module. """ - return checkout / 'src' / 'third_party' / 'zephyr' / name + return checkout / "src" / "third_party" / "zephyr" / name known_modules = { - 'hal_stm32': third_party_module, - 'cmsis': third_party_module, - 'ec': lambda name, checkout: (checkout / 'src' / 'platform' / 'ec'), + "hal_stm32": third_party_module, + "cmsis": third_party_module, + "ec": lambda name, checkout: (checkout / "src" / "platform" / "ec"), } @@ -63,7 +63,7 @@ def locate_from_directory(directory): for name in known_modules: modpath = (directory / name).resolve() - if (modpath / 'zephyr' / 'module.yml').is_file(): + if (modpath / "zephyr" / "module.yml").is_file(): result[name] = modpath return result @@ -92,6 +92,7 @@ def setup_module_symlinks(output_dir, modules): if module_links: return build_config.BuildConfig( - cmake_defs={'ZEPHYR_MODULES': ';'.join(map(str, module_links))}) + cmake_defs={"ZEPHYR_MODULES": ";".join(map(str, module_links))} + ) else: return build_config.BuildConfig() diff --git a/zephyr/zmake/zmake/multiproc.py b/zephyr/zmake/zmake/multiproc.py index 19e5aefef4..492f3b09e8 100644 --- a/zephyr/zmake/zmake/multiproc.py +++ b/zephyr/zmake/zmake/multiproc.py @@ -48,6 +48,7 @@ class LogWriter: value: True if output was written at that level _job_id: The name to prepend to logged lines """ + def __init__(self, logger, log_level, log_level_override_func, job_id): self._logger = logger self._log_level = log_level @@ -166,8 +167,9 @@ def _logging_loop(): _logging_thread = None -def log_output(logger, log_level, file_descriptor, - log_level_override_func=None, job_id=None): +def log_output( + logger, log_level, file_descriptor, log_level_override_func=None, job_id=None +): """Log the output from the given file descriptor. Args: @@ -185,19 +187,14 @@ def log_output(logger, log_level, file_descriptor, global _logging_thread if _logging_thread is None or not _logging_thread.is_alive(): # First pass or thread must have died, create a new one. - _logging_thread = threading.Thread(target=_logging_loop, - daemon=True) + _logging_thread = threading.Thread(target=_logging_loop, daemon=True) _logging_thread.start() - writer = LogWriter( - logger, - log_level, - log_level_override_func, - job_id) + writer = LogWriter(logger, log_level, log_level_override_func, job_id) _logging_map[file_descriptor] = writer # Write a dummy byte to the pipe to break the select so we can add the # new fd. - os.write(_logging_interrupt_pipe[1], b'x') + os.write(_logging_interrupt_pipe[1], b"x") # Notify the condition so we can run the select on the current fds. _logging_cv.notify_all() return writer @@ -226,6 +223,7 @@ class Executor: results: A list of result codes returned by each of the functions called by this Executor. """ + def __init__(self): self.lock = threading.Condition() self.threads = [] @@ -247,8 +245,7 @@ class Executor: exception. """ with self.lock: - thread = threading.Thread(target=lambda: self._run_fn(func), - daemon=True) + thread = threading.Thread(target=lambda: self._run_fn(func), daemon=True) thread.start() self.threads.append(thread) diff --git a/zephyr/zmake/zmake/output_packers.py b/zephyr/zmake/zmake/output_packers.py index a4c1dbd759..049f18cbf9 100644 --- a/zephyr/zmake/zmake/output_packers.py +++ b/zephyr/zmake/zmake/output_packers.py @@ -13,6 +13,7 @@ import zmake.util as util class BasePacker: """Abstract base for all packers.""" + def __init__(self, project): self.project = project @@ -22,7 +23,7 @@ class BasePacker: Yields: 2-tuples of config name and a BuildConfig. """ - yield 'singleimage', build_config.BuildConfig() + yield "singleimage", build_config.BuildConfig() def pack_firmware(self, work_dir, jobclient, version_string=""): """Pack a firmware image. @@ -43,7 +44,7 @@ class BasePacker: other directory) which should be copied into the output directory, and the output filename. """ - raise NotImplementedError('Abstract method not implemented') + raise NotImplementedError("Abstract method not implemented") def _get_max_image_bytes(self): """Get the maximum allowed image size (in bytes). @@ -54,7 +55,7 @@ class BasePacker: Returns: The maximum allowed size of the image in bytes. """ - raise NotImplementedError('Abstract method not implemented') + raise NotImplementedError("Abstract method not implemented") def _is_size_bound(self, path): """Check whether the given path should be constrained by size. @@ -68,7 +69,7 @@ class BasePacker: Returns: True if the file size should be checked. False otherwise. """ - return path.suffix == '.bin' + return path.suffix == ".bin" def _check_packed_file_size(self, file, dirs): """Check that a packed file passes size constraints. @@ -81,37 +82,39 @@ class BasePacker: The file if it passes the test. """ if not self._is_size_bound( - file) or file.stat().st_size <= self._get_max_image_bytes(**dirs): + file + ) or file.stat().st_size <= self._get_max_image_bytes(**dirs): return file - raise RuntimeError('Output file ({}) too large'.format(file)) + raise RuntimeError("Output file ({}) too large".format(file)) class ElfPacker(BasePacker): """Raw proxy for ELF output of a single build.""" - def pack_firmware(self, work_dir, jobclient, singleimage, - version_string=""): - yield singleimage / 'zephyr' / 'zephyr.elf', 'zephyr.elf' + + def pack_firmware(self, work_dir, jobclient, singleimage, version_string=""): + yield singleimage / "zephyr" / "zephyr.elf", "zephyr.elf" class RawBinPacker(BasePacker): """Raw proxy for zephyr.bin output of a single build.""" - def pack_firmware(self, work_dir, jobclient, singleimage, - version_string=""): - yield singleimage / 'zephyr' / 'zephyr.bin', 'zephyr.bin' + + def pack_firmware(self, work_dir, jobclient, singleimage, version_string=""): + yield singleimage / "zephyr" / "zephyr.bin", "zephyr.bin" class BinmanPacker(BasePacker): """Packer for RO/RW image to generate a .bin build using FMAP.""" - ro_file = 'zephyr.bin' - rw_file = 'zephyr.bin' + + ro_file = "zephyr.bin" + rw_file = "zephyr.bin" def __init__(self, project): self.logger = logging.getLogger(self.__class__.__name__) super().__init__(project) def configs(self): - yield 'ro', build_config.BuildConfig(kconfig_defs={'CONFIG_CROS_EC_RO': 'y'}) - yield 'rw', build_config.BuildConfig(kconfig_defs={'CONFIG_CROS_EC_RW': 'y'}) + yield "ro", build_config.BuildConfig(kconfig_defs={"CONFIG_CROS_EC_RO": "y"}) + yield "rw", build_config.BuildConfig(kconfig_defs={"CONFIG_CROS_EC_RW": "y"}) def pack_firmware(self, work_dir, jobclient, ro, rw, version_string=""): """Pack RO and RW sections using Binman. @@ -131,12 +134,12 @@ class BinmanPacker(BasePacker): should be copied into the output directory, and the output filename. """ - dts_file_path = ro / 'zephyr' / 'zephyr.dts' + dts_file_path = ro / "zephyr" / "zephyr.dts" # Copy the inputs into the work directory so that Binman can # find them under a hard-coded name. - shutil.copy2(ro / 'zephyr' / self.ro_file, work_dir / 'zephyr_ro.bin') - shutil.copy2(rw / 'zephyr' / self.rw_file, work_dir / 'zephyr_rw.bin') + shutil.copy2(ro / "zephyr" / self.ro_file, work_dir / "zephyr_ro.bin") + shutil.copy2(rw / "zephyr" / self.rw_file, work_dir / "zephyr_rw.bin") # Version in FRID/FWID can be at most 31 bytes long (32, minus # one for null character). @@ -144,22 +147,33 @@ class BinmanPacker(BasePacker): version_string = version_string[:31] proc = jobclient.popen( - ['binman', '-v', '5', 'build', - '-a', 'version={}'.format(version_string), - '-d', dts_file_path, '-m', '-O', work_dir], + [ + "binman", + "-v", + "5", + "build", + "-a", + "version={}".format(version_string), + "-d", + dts_file_path, + "-m", + "-O", + work_dir, + ], cwd=work_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - encoding='utf-8') + encoding="utf-8", + ) zmake.multiproc.log_output(self.logger, logging.DEBUG, proc.stdout) zmake.multiproc.log_output(self.logger, logging.ERROR, proc.stderr) if proc.wait(timeout=60): - raise OSError('Failed to run binman') + raise OSError("Failed to run binman") - yield work_dir / 'zephyr.bin', 'zephyr.bin' - yield ro / 'zephyr' / 'zephyr.elf', 'zephyr.ro.elf' - yield rw / 'zephyr' / 'zephyr.elf', 'zephyr.rw.elf' + yield work_dir / "zephyr.bin", "zephyr.bin" + yield ro / "zephyr" / "zephyr.elf", "zephyr.ro.elf" + yield rw / "zephyr" / "zephyr.elf", "zephyr.rw.elf" class NpcxPacker(BinmanPacker): @@ -169,30 +183,35 @@ class NpcxPacker(BinmanPacker): zephyr.npcx.bin for the RO image, which should be packed using Nuvoton's loader format. """ - ro_file = 'zephyr.npcx.bin' + + ro_file = "zephyr.npcx.bin" # TODO(b/192401039): CONFIG_FLASH_SIZE is nuvoton-only. Since # binman already checks sizes, perhaps we can just remove this # code? def _get_max_image_bytes(self, ro, rw): ro_size = util.read_kconfig_autoconf_value( - ro / 'zephyr' / 'include' / 'generated', - 'CONFIG_FLASH_SIZE') + ro / "zephyr" / "include" / "generated", "CONFIG_FLASH_SIZE" + ) rw_size = util.read_kconfig_autoconf_value( - ro / 'zephyr' / 'include' / 'generated', - 'CONFIG_FLASH_SIZE') + ro / "zephyr" / "include" / "generated", "CONFIG_FLASH_SIZE" + ) return max(int(ro_size, 0), int(rw_size, 0)) * 1024 # This can probably be removed too and just rely on binman to # check the sizes... see the comment above. def pack_firmware(self, work_dir, jobclient, ro, rw, version_string=""): for path, output_file in super().pack_firmware( - work_dir, jobclient, ro, rw, version_string=version_string, + work_dir, + jobclient, + ro, + rw, + version_string=version_string, ): - if output_file == 'zephyr.bin': + if output_file == "zephyr.bin": yield ( - self._check_packed_file_size(path, {'ro': ro, 'rw': rw}), - 'zephyr.bin', + self._check_packed_file_size(path, {"ro": ro, "rw": rw}), + "zephyr.bin", ) else: yield path, output_file @@ -200,8 +219,8 @@ class NpcxPacker(BinmanPacker): # A dictionary mapping packer config names to classes. packer_registry = { - 'binman': BinmanPacker, - 'elf': ElfPacker, - 'npcx': NpcxPacker, - 'raw': RawBinPacker, + "binman": BinmanPacker, + "elf": ElfPacker, + "npcx": NpcxPacker, + "raw": RawBinPacker, } diff --git a/zephyr/zmake/zmake/project.py b/zephyr/zmake/zmake/project.py index fcbf01c30e..8fb75b54de 100644 --- a/zephyr/zmake/zmake/project.py +++ b/zephyr/zmake/zmake/project.py @@ -13,7 +13,7 @@ import yaml # DeprecationWarnings that fire when we import it. Suppress these # during the import to keep the noise down. with warnings.catch_warnings(): - warnings.simplefilter('ignore') + warnings.simplefilter("ignore") import jsonschema import zmake.build_config as build_config @@ -32,8 +32,7 @@ def module_dts_overlay_name(modpath, board_name): Returns: A pathlib.Path object to the expected overlay path. """ - return modpath / 'zephyr' / 'dts' / 'board-overlays' / '{}.dts'.format( - board_name) + return modpath / "zephyr" / "dts" / "board-overlays" / "{}.dts".format(board_name) def find_projects(root_dir): @@ -45,52 +44,52 @@ def find_projects(root_dir): Yields: Project: The next project found. """ - logging.info('Finding zmake targets under \'%s\'.', root_dir) - for path in pathlib.Path(root_dir).rglob('zmake.yaml'): + logging.info("Finding zmake targets under '%s'.", root_dir) + for path in pathlib.Path(root_dir).rglob("zmake.yaml"): yield Project(path.parent) class ProjectConfig: """An object wrapping zmake.yaml.""" + validator = jsonschema.Draft7Validator schema = { - 'type': 'object', - 'required': ['supported-zephyr-versions', 'board', 'output-type', - 'toolchain'], - 'properties': { - 'supported-zephyr-versions': { - 'type': 'array', - 'items': { - 'type': 'string', - 'enum': ['v2.5', 'v2.6'], + "type": "object", + "required": ["supported-zephyr-versions", "board", "output-type", "toolchain"], + "properties": { + "supported-zephyr-versions": { + "type": "array", + "items": { + "type": "string", + "enum": ["v2.5", "v2.6"], }, - 'minItems': 1, - 'uniqueItems': True, + "minItems": 1, + "uniqueItems": True, }, - 'board': { - 'type': 'string', + "board": { + "type": "string", }, - 'modules': { - 'type': 'array', - 'items': { - 'type': 'string', - 'enum': list(modules.known_modules), + "modules": { + "type": "array", + "items": { + "type": "string", + "enum": list(modules.known_modules), }, }, - 'output-type': { - 'type': 'string', - 'enum': list(packers.packer_registry), + "output-type": { + "type": "string", + "enum": list(packers.packer_registry), }, - 'toolchain': { - 'type': 'string', + "toolchain": { + "type": "string", }, - 'is-test': { - 'type': 'boolean', + "is-test": { + "type": "boolean", }, - 'dts-overlays': { - 'type': 'array', - 'items': { - 'type': 'string', + "dts-overlays": { + "type": "array", + "items": { + "type": "string", }, }, }, @@ -103,40 +102,43 @@ class ProjectConfig: @property def supported_zephyr_versions(self): - return [util.parse_zephyr_version(x) - for x in self.config_dict['supported-zephyr-versions']] + return [ + util.parse_zephyr_version(x) + for x in self.config_dict["supported-zephyr-versions"] + ] @property def board(self): - return self.config_dict['board'] + return self.config_dict["board"] @property def modules(self): - return self.config_dict.get('modules', list(modules.known_modules)) + return self.config_dict.get("modules", list(modules.known_modules)) @property def output_packer(self): - return packers.packer_registry[self.config_dict['output-type']] + return packers.packer_registry[self.config_dict["output-type"]] @property def toolchain(self): - return self.config_dict['toolchain'] + return self.config_dict["toolchain"] @property def is_test(self): - return self.config_dict.get('is-test', False) + return self.config_dict.get("is-test", False) @property def dts_overlays(self): - return self.config_dict.get('dts-overlays', []) + return self.config_dict.get("dts-overlays", []) class Project: """An object encapsulating a project directory.""" + def __init__(self, project_dir, config_dict=None): self.project_dir = project_dir.resolve() if not config_dict: - with open(self.project_dir / 'zmake.yaml') as f: + with open(self.project_dir / "zmake.yaml") as f: config_dict = yaml.safe_load(f) self.config = ProjectConfig(config_dict) self.packer = self.config.output_packer(self) @@ -147,11 +149,12 @@ class Project: Yields: 2-tuples of a build configuration name and a BuildConfig. """ - conf = build_config.BuildConfig(cmake_defs={'BOARD': self.config.board}) - if (self.project_dir / 'boards').is_dir(): + conf = build_config.BuildConfig(cmake_defs={"BOARD": self.config.board}) + if (self.project_dir / "boards").is_dir(): conf |= build_config.BuildConfig( - cmake_defs={'BOARD_ROOT': str(self.project_dir)}) - prj_conf = self.project_dir / 'prj.conf' + cmake_defs={"BOARD_ROOT": str(self.project_dir)} + ) + prj_conf = self.project_dir / "prj.conf" if prj_conf.is_file(): conf |= build_config.BuildConfig(kconfig_files=[prj_conf]) for build_name, packer_config in self.packer.configs(): @@ -177,7 +180,8 @@ class Project: if overlays: return build_config.BuildConfig( - cmake_defs={'DTC_OVERLAY_FILE': ';'.join(map(str, overlays))}) + cmake_defs={"DTC_OVERLAY_FILE": ";".join(map(str, overlays))} + ) else: return build_config.BuildConfig() @@ -207,6 +211,7 @@ class Project: result[module] = module_paths[module] except KeyError as e: raise KeyError( - 'The {!r} module is required by the {} project, but is not ' - 'available.'.format(module, self.project_dir)) from e + "The {!r} module is required by the {} project, but is not " + "available.".format(module, self.project_dir) + ) from e return result diff --git a/zephyr/zmake/zmake/toolchains.py b/zephyr/zmake/zmake/toolchains.py index 0c0510e970..b07a2080d6 100644 --- a/zephyr/zmake/zmake/toolchains.py +++ b/zephyr/zmake/zmake/toolchains.py @@ -17,41 +17,59 @@ def find_zephyr_sdk(): The path to the Zephyr SDK, using the search rules defined by https://docs.zephyrproject.org/latest/getting_started/installation_linux.html """ + def _gen_sdk_paths(): - yield os.getenv('ZEPHYR_SDK_INSTALL_DIR') + yield os.getenv("ZEPHYR_SDK_INSTALL_DIR") - for searchpath in ('~/zephyr-sdk', '~/.local/zephyr-sdk', - '~/.local/opt/zephyr-sdk', '~/bin/zephyr-sdk', - '/opt/zephyr-sdk', '/usr/zephyr-sdk', - '/usr/local/zephyr-sdk'): - for suffix in ('', '-*'): + for searchpath in ( + "~/zephyr-sdk", + "~/.local/zephyr-sdk", + "~/.local/opt/zephyr-sdk", + "~/bin/zephyr-sdk", + "/opt/zephyr-sdk", + "/usr/zephyr-sdk", + "/usr/local/zephyr-sdk", + ): + for suffix in ("", "-*"): yield from glob.glob(os.path.expanduser(searchpath + suffix)) for path in _gen_sdk_paths(): if not path: continue path = pathlib.Path(path) - if (path / 'sdk_version').is_file(): + if (path / "sdk_version").is_file(): return path - raise OSError('Unable to find the Zephyr SDK') + raise OSError("Unable to find the Zephyr SDK") # Mapping of toolchain names -> (λ (module-paths) build-config) toolchains = { - 'coreboot-sdk': lambda modules: build_config.BuildConfig( - cmake_defs={'TOOLCHAIN_ROOT': str(modules['ec'] / 'zephyr'), - 'ZEPHYR_TOOLCHAIN_VARIANT': 'coreboot-sdk'}), - 'llvm': lambda modules: build_config.BuildConfig( - cmake_defs={'TOOLCHAIN_ROOT': str(modules['ec'] / 'zephyr'), - 'ZEPHYR_TOOLCHAIN_VARIANT': 'llvm'}), - 'zephyr': lambda _: build_config.BuildConfig( - cmake_defs={'ZEPHYR_TOOLCHAIN_VARIANT': 'zephyr', - 'ZEPHYR_SDK_INSTALL_DIR': str(find_zephyr_sdk())}, - environ_defs={'ZEPHYR_SDK_INSTALL_DIR': str(find_zephyr_sdk())}), - 'arm-none-eabi': lambda _: build_config.BuildConfig( - cmake_defs={'ZEPHYR_TOOLCHAIN_VARIANT': 'cross-compile', - 'CROSS_COMPILE': '/usr/bin/arm-none-eabi-'}), + "coreboot-sdk": lambda modules: build_config.BuildConfig( + cmake_defs={ + "TOOLCHAIN_ROOT": str(modules["ec"] / "zephyr"), + "ZEPHYR_TOOLCHAIN_VARIANT": "coreboot-sdk", + } + ), + "llvm": lambda modules: build_config.BuildConfig( + cmake_defs={ + "TOOLCHAIN_ROOT": str(modules["ec"] / "zephyr"), + "ZEPHYR_TOOLCHAIN_VARIANT": "llvm", + } + ), + "zephyr": lambda _: build_config.BuildConfig( + cmake_defs={ + "ZEPHYR_TOOLCHAIN_VARIANT": "zephyr", + "ZEPHYR_SDK_INSTALL_DIR": str(find_zephyr_sdk()), + }, + environ_defs={"ZEPHYR_SDK_INSTALL_DIR": str(find_zephyr_sdk())}, + ), + "arm-none-eabi": lambda _: build_config.BuildConfig( + cmake_defs={ + "ZEPHYR_TOOLCHAIN_VARIANT": "cross-compile", + "CROSS_COMPILE": "/usr/bin/arm-none-eabi-", + } + ), } @@ -69,5 +87,4 @@ def get_toolchain(name, module_paths): """ if name in toolchains: return toolchains[name](module_paths) - return build_config.BuildConfig( - cmake_defs={'ZEPHYR_TOOLCHAIN_VARIANT': name}) + return build_config.BuildConfig(cmake_defs={"ZEPHYR_TOOLCHAIN_VARIANT": name}) diff --git a/zephyr/zmake/zmake/util.py b/zephyr/zmake/zmake/util.py index ca38831ef6..18d03dd7f0 100644 --- a/zephyr/zmake/zmake/util.py +++ b/zephyr/zmake/zmake/util.py @@ -18,26 +18,27 @@ def locate_cros_checkout(): variable, then scanning upwards from the current directory, and finally from a known set of common paths. """ + def propose_checkouts(): - yield os.getenv('CROS_WORKON_SRCROOT') + yield os.getenv("CROS_WORKON_SRCROOT") path = pathlib.Path.cwd() - while path.resolve() != pathlib.Path('/'): + while path.resolve() != pathlib.Path("/"): yield path - path = path / '..' + path = path / ".." - yield '/mnt/host/source' - yield pathlib.Path.home() / 'trunk' - yield pathlib.Path.home() / 'chromiumos' + yield "/mnt/host/source" + yield pathlib.Path.home() / "trunk" + yield pathlib.Path.home() / "chromiumos" for path in propose_checkouts(): if not path: continue path = pathlib.Path(path) - if (path / '.repo').is_dir(): + if (path / ".repo").is_dir(): return path.resolve() - raise FileNotFoundError('Unable to locate a ChromiumOS checkout') + raise FileNotFoundError("Unable to locate a ChromiumOS checkout") def locate_zephyr_base(checkout, version): @@ -50,8 +51,14 @@ def locate_zephyr_base(checkout, version): Returns: The path to the Zephyr source. """ - return (checkout / 'src' / 'third_party' / 'zephyr' / 'main' / - 'v{}.{}'.format(*version[:2])) + return ( + checkout + / "src" + / "third_party" + / "zephyr" + / "main" + / "v{}.{}".format(*version[:2]) + ) def read_kconfig_file(path): @@ -66,13 +73,14 @@ def read_kconfig_file(path): result = {} with open(path) as f: for line in f: - line, _, _ = line.partition('#') + line, _, _ = line.partition("#") line = line.strip() if line: - name, _, value = line.partition('=') + name, _, value = line.partition("=") result[name.strip()] = value.strip() return result + def read_kconfig_autoconf_value(path, key): """Parse an autoconf.h file for a resolved kconfig value @@ -83,8 +91,8 @@ def read_kconfig_autoconf_value(path, key): Returns: The value associated with the key or nothing if the key wasn't found. """ - prog = re.compile(r'^#define\s{}\s(\S+)$'.format(key)) - with open(path / 'autoconf.h') as f: + prog = re.compile(r"^#define\s{}\s(\S+)$".format(key)) + with open(path / "autoconf.h") as f: for line in f: m = prog.match(line) if m: @@ -105,7 +113,7 @@ def write_kconfig_file(path, config, only_if_changed=True): return with open(path, "w") as f: for name, value in config.items(): - f.write('{}={}\n'.format(name, value)) + f.write("{}={}\n".format(name, value)) def parse_zephyr_version(version_string): @@ -117,10 +125,11 @@ def parse_zephyr_version(version_string): Returns: A 2-tuple or 3-tuple of integers representing the version. """ - match = re.fullmatch(r'v?(\d+)[._](\d+)(?:[._](\d+))?', version_string) + match = re.fullmatch(r"v?(\d+)[._](\d+)(?:[._](\d+))?", version_string) if not match: raise ValueError( - "{} does not look like a Zephyr version.".format(version_string)) + "{} does not look like a Zephyr version.".format(version_string) + ) return tuple(int(x) for x in match.groups() if x is not None) @@ -133,17 +142,19 @@ def read_zephyr_version(zephyr_base): Returns: A 3-tuple of the version number (major, minor, patchset). """ - version_file = pathlib.Path(zephyr_base) / 'VERSION' + version_file = pathlib.Path(zephyr_base) / "VERSION" file_vars = {} with open(version_file) as f: for line in f: - key, sep, value = line.partition('=') + key, sep, value = line.partition("=") file_vars[key.strip()] = value.strip() - return (int(file_vars['VERSION_MAJOR']), - int(file_vars['VERSION_MINOR']), - int(file_vars['PATCHLEVEL'])) + return ( + int(file_vars["VERSION_MAJOR"]), + int(file_vars["VERSION_MINOR"]), + int(file_vars["PATCHLEVEL"]), + ) def repr_command(argv): @@ -155,7 +166,7 @@ def repr_command(argv): Returns: A string which could be pasted into a shell for execution. """ - return ' '.join(shlex.quote(str(arg)) for arg in argv) + return " ".join(shlex.quote(str(arg)) for arg in argv) def update_symlink(target_path, link_path): @@ -166,11 +177,13 @@ def update_symlink(target_path, link_path): link_path: A Path-like object of the symlink. """ target = target_path.resolve() - if (not link_path.is_symlink() - or pathlib.Path(os.readlink(link_path)).resolve() != target): - if link_path.exists(): - link_path.unlink() - link_path.symlink_to(target) + if ( + not link_path.is_symlink() + or pathlib.Path(os.readlink(link_path)).resolve() != target + ): + if link_path.exists(): + link_path.unlink() + link_path.symlink_to(target) def log_multi_line(logger, level, message): @@ -200,21 +213,20 @@ def resolve_build_dir(platform_ec_dir, project_dir, build_dir): if build_dir: return build_dir - if not pathlib.Path.exists(project_dir / 'zmake.yaml'): + if not pathlib.Path.exists(project_dir / "zmake.yaml"): raise OSError("Invalid configuration") # Resolve project_dir to absolute path. project_dir = project_dir.resolve() # Compute the path of project_dir relative to platform_ec_dir. - project_relative_path = pathlib.Path.relative_to(project_dir, - platform_ec_dir) + project_relative_path = pathlib.Path.relative_to(project_dir, platform_ec_dir) # Make sure that the project_dir is a subdirectory of platform_ec_dir. if platform_ec_dir / project_relative_path != project_dir: raise OSError( - 'Can\'t resolve project directory {} which is not a subdirectory' - ' of the platform/ec directory {}'.format(project_dir, - platform_ec_dir)) + "Can't resolve project directory {} which is not a subdirectory" + " of the platform/ec directory {}".format(project_dir, platform_ec_dir) + ) - return platform_ec_dir / 'build' / project_relative_path + return platform_ec_dir / "build" / project_relative_path diff --git a/zephyr/zmake/zmake/version.py b/zephyr/zmake/zmake/version.py index cddce86ca5..d026c07c2f 100644 --- a/zephyr/zmake/zmake/version.py +++ b/zephyr/zmake/zmake/version.py @@ -21,12 +21,15 @@ def _get_num_commits(repo): An integer, the number of commits that have been made. """ try: - result = subprocess.run(['git', '-C', repo, 'rev-list', 'HEAD', - '--count'], - check=True, stdout=subprocess.PIPE, - stderr=subprocess.DEVNULL, encoding='utf-8') + result = subprocess.run( + ["git", "-C", repo, "rev-list", "HEAD", "--count"], + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + encoding="utf-8", + ) except subprocess.CalledProcessError: - commits = '9999' + commits = "9999" else: commits = result.stdout @@ -47,17 +50,20 @@ def _get_revision(repo): A string, of the current revision. """ try: - result = subprocess.run(['git', '-C', repo, 'log', '-n1', - '--format=%H'], - check=True, stdout=subprocess.PIPE, - stderr=subprocess.DEVNULL, encoding='utf-8') + result = subprocess.run( + ["git", "-C", repo, "log", "-n1", "--format=%H"], + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + encoding="utf-8", + ) except subprocess.CalledProcessError: # Fall back to the VCSID provided by the packaging system. # Format is 0.0.1-r425-032666c418782c14fe912ba6d9f98ffdf0b941e9 for # releases and 9999-032666c418782c14fe912ba6d9f98ffdf0b941e9 for # 9999 ebuilds. - vcsid = os.environ.get('VCSID', '9999-unknown') - revision = vcsid.rsplit('-', 1)[1] + vcsid = os.environ.get("VCSID", "9999-unknown") + revision = vcsid.rsplit("-", 1)[1] else: revision = result.stdout @@ -84,19 +90,21 @@ def get_version_string(project, zephyr_base, modules, static=False): num_commits = 0 if static: - vcs_hashes = 'STATIC' + vcs_hashes = "STATIC" else: repos = { - 'os': zephyr_base, + "os": zephyr_base, **modules, } for repo in repos.values(): num_commits += _get_num_commits(repo) - vcs_hashes = ','.join( - '{}:{}'.format(name, _get_revision(repo)[:6]) - for name, repo in sorted(repos.items())) + vcs_hashes = ",".join( + "{}:{}".format(name, _get_revision(repo)[:6]) + for name, repo in sorted(repos.items()) + ) - return '{}_v{}.{}.{}-{}'.format( - project_id, major_version, minor_version, num_commits, vcs_hashes) + return "{}_v{}.{}.{}-{}".format( + project_id, major_version, minor_version, num_commits, vcs_hashes + ) diff --git a/zephyr/zmake/zmake/zmake.py b/zephyr/zmake/zmake/zmake.py index c535ebc656..baaced8ca8 100644 --- a/zephyr/zmake/zmake/zmake.py +++ b/zephyr/zmake/zmake/zmake.py @@ -20,8 +20,8 @@ import zmake.toolchains as toolchains import zmake.util as util import zmake.version -ninja_warnings = re.compile(r'^(\S*: )?warning:.*') -ninja_errors = re.compile(r'error:.*') +ninja_warnings = re.compile(r"^(\S*: )?warning:.*") +ninja_errors = re.compile(r"error:.*") def ninja_stdout_log_level_override(line, current_log_level): @@ -38,12 +38,12 @@ def ninja_stdout_log_level_override(line, current_log_level): # Output lines from Zephyr that are not normally useful # Send any lines that start with these strings to INFO cmake_suppress = [ - '-- ', # device tree messages - 'Loaded configuration', - 'Including boilerplate', - 'Parsing ', - 'No change to configuration', - 'No change to Kconfig header', + "-- ", # device tree messages + "Loaded configuration", + "Including boilerplate", + "Parsing ", + "No change to configuration", + "No change to Kconfig header", ] # Herewith a long list of things which are really for debugging, not @@ -122,7 +122,8 @@ def get_process_failure_msg(proc): Failure message as a string: """ return "Execution failed (return code={}): {}\n".format( - proc.returncode, util.repr_command(proc.args)) + proc.returncode, util.repr_command(proc.args) + ) class Zmake: @@ -145,8 +146,10 @@ class Zmake: _sequential: True to check the results of each build job sequentially, before launching more, False to just do this after all jobs complete """ - def __init__(self, checkout=None, jobserver=None, jobs=0, modules_dir=None, - zephyr_base=None): + + def __init__( + self, checkout=None, jobserver=None, jobs=0, modules_dir=None, zephyr_base=None + ): zmake.multiproc.reset() self._checkout = checkout self._zephyr_base = zephyr_base @@ -154,8 +157,7 @@ class Zmake: if modules_dir: self.module_paths = zmake.modules.locate_from_directory(modules_dir) else: - self.module_paths = zmake.modules.locate_from_checkout( - self.checkout) + self.module_paths = zmake.modules.locate_from_checkout(self.checkout) if jobserver: self.jobserver = jobserver @@ -196,12 +198,19 @@ class Zmake: project = zmake.project.Project(pathlib.Path(project_dir)) supported_versions = project.config.supported_zephyr_versions for version in supported_versions: - print('v{}.{}'.format(*version[:2])) - - def configure(self, project_dir, build_dir=None, - toolchain=None, ignore_unsupported_zephyr_version=False, - build_after_configure=False, test_after_configure=False, - bringup=False, coverage=False): + print("v{}.{}".format(*version[:2])) + + def configure( + self, + project_dir, + build_dir=None, + toolchain=None, + ignore_unsupported_zephyr_version=False, + build_after_configure=False, + test_after_configure=False, + bringup=False, + coverage=False, + ): """Set up a build directory to later be built by "zmake build".""" project = zmake.project.Project(project_dir) supported_versions = project.config.supported_zephyr_versions @@ -211,42 +220,48 @@ class Zmake: # Ignore the patchset from the Zephyr version. zephyr_version = util.read_zephyr_version(zephyr_base)[:2] - if (not ignore_unsupported_zephyr_version - and zephyr_version not in supported_versions): + if ( + not ignore_unsupported_zephyr_version + and zephyr_version not in supported_versions + ): raise ValueError( - 'The Zephyr OS version (v{}.{}) is not supported by the ' - 'project. You may wish to either configure zmake.yaml to ' - 'support this version, or pass ' - '--ignore-unsupported-zephyr-version.'.format(*zephyr_version)) + "The Zephyr OS version (v{}.{}) is not supported by the " + "project. You may wish to either configure zmake.yaml to " + "support this version, or pass " + "--ignore-unsupported-zephyr-version.".format(*zephyr_version) + ) # Resolve build_dir if needed. build_dir = util.resolve_build_dir( - platform_ec_dir=self.module_paths['ec'], + platform_ec_dir=self.module_paths["ec"], project_dir=project_dir, - build_dir=build_dir) + build_dir=build_dir, + ) # Make sure the build directory is clean. if os.path.exists(build_dir): self.logger.info("Clearing old build directory %s", build_dir) shutil.rmtree(build_dir) base_config = zmake.build_config.BuildConfig( - environ_defs={'ZEPHYR_BASE': str(zephyr_base), - 'PATH': '/usr/bin'}, + environ_defs={"ZEPHYR_BASE": str(zephyr_base), "PATH": "/usr/bin"}, cmake_defs={ - 'DTS_ROOT': str(self.module_paths['ec'] / 'zephyr'), - 'SYSCALL_INCLUDE_DIRS': str( - self.module_paths['ec'] / 'zephyr' / 'include' / 'drivers'), - }) + "DTS_ROOT": str(self.module_paths["ec"] / "zephyr"), + "SYSCALL_INCLUDE_DIRS": str( + self.module_paths["ec"] / "zephyr" / "include" / "drivers" + ), + }, + ) # Prune the module paths to just those required by the project. module_paths = project.prune_modules(self.module_paths) module_config = zmake.modules.setup_module_symlinks( - build_dir / 'modules', module_paths) + build_dir / "modules", module_paths + ) # Symlink the Zephyr base into the build directory so it can # be used in the build phase. - util.update_symlink(zephyr_base, build_dir / 'zephyr_base') + util.update_symlink(zephyr_base, build_dir / "zephyr_base") dts_overlay_config = project.find_dts_overlays(module_paths) @@ -257,46 +272,61 @@ class Zmake: if bringup: base_config |= zmake.build_config.BuildConfig( - kconfig_defs={'CONFIG_PLATFORM_EC_BRINGUP': 'y'}) + kconfig_defs={"CONFIG_PLATFORM_EC_BRINGUP": "y"} + ) if coverage: base_config |= zmake.build_config.BuildConfig( - kconfig_defs={'CONFIG_COVERAGE': 'y'}) + kconfig_defs={"CONFIG_COVERAGE": "y"} + ) if not build_dir.exists(): build_dir = build_dir.mkdir() processes = [] - self.logger.info('Building %s in %s.', project_dir, build_dir) + self.logger.info("Building %s in %s.", project_dir, build_dir) for build_name, build_config in project.iter_builds(): - self.logger.info('Configuring %s:%s.', project_dir, build_name) - config = (base_config - | toolchain_config - | module_config - | dts_overlay_config - | build_config) - output_dir = build_dir / 'build-{}'.format(build_name) - kconfig_file = build_dir / 'kconfig-{}.conf'.format(build_name) - proc = config.popen_cmake(self.jobserver, project_dir, output_dir, - kconfig_file, stdin=subprocess.DEVNULL, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - encoding='utf-8', - errors='replace') + self.logger.info("Configuring %s:%s.", project_dir, build_name) + config = ( + base_config + | toolchain_config + | module_config + | dts_overlay_config + | build_config + ) + output_dir = build_dir / "build-{}".format(build_name) + kconfig_file = build_dir / "kconfig-{}.conf".format(build_name) + proc = config.popen_cmake( + self.jobserver, + project_dir, + output_dir, + kconfig_file, + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + encoding="utf-8", + errors="replace", + ) job_id = "{}:{}".format(project_dir, build_name) zmake.multiproc.log_output( - self.logger, logging.DEBUG, proc.stdout, + self.logger, + logging.DEBUG, + proc.stdout, log_level_override_func=cmake_log_level_override, - job_id=job_id,) + job_id=job_id, + ) zmake.multiproc.log_output( - self.logger, logging.ERROR, proc.stderr, + self.logger, + logging.ERROR, + proc.stderr, log_level_override_func=cmake_log_level_override, - job_id=job_id,) + job_id=job_id, + ) processes.append(proc) for proc in processes: if proc.wait(): raise OSError(get_process_failure_msg(proc)) # Create symlink to project - util.update_symlink(project_dir, build_dir / 'project') + util.update_symlink(project_dir, build_dir / "project") if test_after_configure: return self.test(build_dir=build_dir) @@ -305,6 +335,7 @@ class Zmake: def build(self, build_dir, output_files_out=None, fail_on_warnings=False): """Build a pre-configured build directory.""" + def wait_and_check_success(procs, writers): """Wait for processes to complete and check for errors @@ -329,11 +360,11 @@ class Zmake: # since it exposes the fragmented nature of the build. raise OSError(get_process_failure_msg(bad)) - if (fail_on_warnings and - any(w.has_written(logging.WARNING) or - w.has_written(logging.ERROR) for w in writers)): - self.logger.warning( - "zmake: Warnings detected in build: aborting") + if fail_on_warnings and any( + w.has_written(logging.WARNING) or w.has_written(logging.ERROR) + for w in writers + ): + self.logger.warning("zmake: Warnings detected in build: aborting") return False return True @@ -342,37 +373,46 @@ class Zmake: dirs = {} build_dir = build_dir.resolve() - project = zmake.project.Project(build_dir / 'project') + project = zmake.project.Project(build_dir / "project") # Compute the version string. version_string = zmake.version.get_version_string( - project, build_dir / 'zephyr_base', - zmake.modules.locate_from_directory(build_dir / 'modules')) + project, + build_dir / "zephyr_base", + zmake.modules.locate_from_directory(build_dir / "modules"), + ) for build_name, build_config in project.iter_builds(): with self.jobserver.get_job(): - dirs[build_name] = build_dir / 'build-{}'.format(build_name) - cmd = ['/usr/bin/ninja', '-C', dirs[build_name].as_posix()] - self.logger.info('Building %s:%s: %s', build_dir, build_name, - zmake.util.repr_command(cmd)) + dirs[build_name] = build_dir / "build-{}".format(build_name) + cmd = ["/usr/bin/ninja", "-C", dirs[build_name].as_posix()] + self.logger.info( + "Building %s:%s: %s", + build_dir, + build_name, + zmake.util.repr_command(cmd), + ) proc = self.jobserver.popen( cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - encoding='utf-8', - errors='replace') + encoding="utf-8", + errors="replace", + ) job_id = "{}:{}".format(build_dir, build_name) out = zmake.multiproc.log_output( logger=self.logger, log_level=logging.INFO, file_descriptor=proc.stdout, log_level_override_func=ninja_stdout_log_level_override, - job_id=job_id,) + job_id=job_id, + ) err = zmake.multiproc.log_output( self.logger, logging.ERROR, proc.stderr, - job_id=job_id,) + job_id=job_id, + ) if self._sequential: if not wait_and_check_success([proc], [out, err]): @@ -385,8 +425,8 @@ class Zmake: return 2 # Run the packer. - packer_work_dir = build_dir / 'packer' - output_dir = build_dir / 'output' + packer_work_dir = build_dir / "packer" + output_dir = build_dir / "output" for d in output_dir, packer_work_dir: if not d.exists(): d.mkdir() @@ -394,10 +434,10 @@ class Zmake: if output_files_out is None: output_files_out = [] for output_file, output_name in project.packer.pack_firmware( - packer_work_dir, self.jobserver, version_string=version_string, - **dirs): + packer_work_dir, self.jobserver, version_string=version_string, **dirs + ): shutil.copy2(output_file, output_dir / output_name) - self.logger.info('Output file \'%r\' created.', output_file) + self.logger.info("Output file '%r' created.", output_file) output_files_out.append(output_file) return 0 @@ -409,26 +449,33 @@ class Zmake: self.build(build_dir, output_files_out=output_files) # If the project built but isn't a test, just bail. - project = zmake.project.Project(build_dir / 'project') + project = zmake.project.Project(build_dir / "project") if not project.config.is_test: return 0 for output_file in output_files: - self.logger.info('Running tests in %s.', output_file) + self.logger.info("Running tests in %s.", output_file) with self.jobserver.get_job(): proc = self.jobserver.popen( [output_file], stdout=subprocess.PIPE, stderr=subprocess.PIPE, - encoding='utf-8', - errors='replace') + encoding="utf-8", + errors="replace", + ) job_id = "test {}".format(output_file) zmake.multiproc.log_output( - self.logger, logging.DEBUG, - proc.stdout, job_id=job_id,) + self.logger, + logging.DEBUG, + proc.stdout, + job_id=job_id, + ) zmake.multiproc.log_output( - self.logger, logging.ERROR, - proc.stderr, job_id=job_id,) + self.logger, + logging.ERROR, + proc.stderr, + job_id=job_id, + ) procs.append(proc) for idx, proc in enumerate(procs): @@ -439,13 +486,12 @@ class Zmake: def testall(self): """Test all the valid test targets""" tmp_dirs = [] - for project in zmake.project.find_projects( - self.module_paths['ec'] / 'zephyr'): + for project in zmake.project.find_projects(self.module_paths["ec"] / "zephyr"): is_test = project.config.is_test temp_build_dir = tempfile.mkdtemp( - suffix='-{}'.format(os.path.basename( - project.project_dir.as_posix())), - prefix='zbuild-') + suffix="-{}".format(os.path.basename(project.project_dir.as_posix())), + prefix="zbuild-", + ) tmp_dirs.append(temp_build_dir) # Configure and run the test. self.executor.append( @@ -453,7 +499,9 @@ class Zmake: project_dir=project.project_dir, build_dir=pathlib.Path(temp_build_dir), build_after_configure=True, - test_after_configure=is_test)) + test_after_configure=is_test, + ) + ) rv = self.executor.wait() for tmpdir in tmp_dirs: @@ -463,36 +511,55 @@ class Zmake: def _run_lcov(self, build_dir, lcov_file, initial=False): with self.jobserver.get_job(): if initial: - self.logger.info('Running (initial) lcov on %s.', build_dir) + self.logger.info("Running (initial) lcov on %s.", build_dir) else: - self.logger.info('Running lcov on %s.', build_dir) - cmd = ['/usr/bin/lcov', '--gcov-tool', - self.module_paths['ec'] / - 'util/llvm-gcov.sh', '-q', '-o', '-', - '-c', '-d', build_dir, '-t', lcov_file.stem, - '--exclude', '*/build-*/zephyr/*/generated/*', - '--exclude', '*/ec/test/*', - '--exclude', '*/ec/zephyr/emul/*', - '--exclude', '*/ec/zephyr/test/*', - '--exclude', '*/testsuite/*', - '--exclude', '*/subsys/emul/*', - ] + self.logger.info("Running lcov on %s.", build_dir) + cmd = [ + "/usr/bin/lcov", + "--gcov-tool", + self.module_paths["ec"] / "util/llvm-gcov.sh", + "-q", + "-o", + "-", + "-c", + "-d", + build_dir, + "-t", + lcov_file.stem, + "--exclude", + "*/build-*/zephyr/*/generated/*", + "--exclude", + "*/ec/test/*", + "--exclude", + "*/ec/zephyr/emul/*", + "--exclude", + "*/ec/zephyr/test/*", + "--exclude", + "*/testsuite/*", + "--exclude", + "*/subsys/emul/*", + ] if initial: - cmd += ['-i'] - proc = self.jobserver.popen(cmd, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - encoding='utf-8', - errors='replace') + cmd += ["-i"] + proc = self.jobserver.popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + encoding="utf-8", + errors="replace", + ) zmake.multiproc.log_output( - self.logger, logging.WARNING, proc.stderr, - job_id="{}-lcov".format(build_dir),) + self.logger, + logging.WARNING, + proc.stderr, + job_id="{}-lcov".format(build_dir), + ) - with open(lcov_file, 'w') as outfile: + with open(lcov_file, "w") as outfile: for line in proc.stdout: - if line.startswith('SF:'): + if line.startswith("SF:"): path = line[3:].rstrip() - outfile.write('SF:%s\n' % os.path.realpath(path)) + outfile.write("SF:%s\n" % os.path.realpath(path)) else: outfile.write(line) if proc.wait(): @@ -501,44 +568,48 @@ class Zmake: return 0 def _coverage_compile_only(self, project, build_dir, lcov_file): - self.logger.info("Building %s in %s", - project.project_dir, build_dir) + self.logger.info("Building %s in %s", project.project_dir, build_dir) rv = self.configure( project_dir=project.project_dir, build_dir=build_dir, build_after_configure=False, test_after_configure=False, - coverage=True) + coverage=True, + ) if rv: return rv # Use ninja to compile the all.libraries target. - build_project = zmake.project.Project(build_dir / 'project') + build_project = zmake.project.Project(build_dir / "project") procs = [] dirs = {} for build_name, build_config in build_project.iter_builds(): - self.logger.info('Building %s:%s all.libraries.', - build_dir, build_name) - dirs[build_name] = build_dir / 'build-{}'.format(build_name) + self.logger.info("Building %s:%s all.libraries.", build_dir, build_name) + dirs[build_name] = build_dir / "build-{}".format(build_name) proc = self.jobserver.popen( - ['/usr/bin/ninja', '-C', dirs[build_name], 'all.libraries'], + ["/usr/bin/ninja", "-C", dirs[build_name], "all.libraries"], # Ninja will connect as a job client instead and claim # many jobs. stdout=subprocess.PIPE, stderr=subprocess.PIPE, - encoding='utf-8', - errors='replace') + encoding="utf-8", + errors="replace", + ) job_id = "{}:{}".format(build_dir, build_name) zmake.multiproc.log_output( logger=self.logger, log_level=logging.DEBUG, file_descriptor=proc.stdout, log_level_override_func=ninja_stdout_log_level_override, - job_id=job_id,) + job_id=job_id, + ) zmake.multiproc.log_output( - self.logger, logging.ERROR, proc.stderr, - job_id=job_id,) + self.logger, + logging.ERROR, + proc.stderr, + job_id=job_id, + ) procs.append(proc) for proc in procs: @@ -548,14 +619,14 @@ class Zmake: return self._run_lcov(build_dir, lcov_file, initial=True) def _coverage_run_test(self, project, build_dir, lcov_file): - self.logger.info("Running test %s in %s", - project.project_dir, build_dir) + self.logger.info("Running test %s in %s", project.project_dir, build_dir) rv = self.configure( project_dir=project.project_dir, build_dir=build_dir, build_after_configure=True, test_after_configure=True, - coverage=True) + coverage=True, + ) if rv: return rv return self._run_lcov(build_dir, lcov_file, initial=False) @@ -563,28 +634,29 @@ class Zmake: def coverage(self, build_dir): """Builds all targets with coverage enabled, and then runs the tests.""" all_lcov_files = [] - root_dir = self.module_paths['ec'] / 'zephyr' + root_dir = self.module_paths["ec"] / "zephyr" for project in zmake.project.find_projects(root_dir): is_test = project.config.is_test rel_path = project.project_dir.relative_to(root_dir) project_build_dir = pathlib.Path(build_dir).joinpath(rel_path) lcov_file = pathlib.Path(build_dir).joinpath( - str(rel_path).replace('/', '_') + '.info') + str(rel_path).replace("/", "_") + ".info" + ) all_lcov_files.append(lcov_file) if is_test: # Configure and run the test. self.executor.append( func=lambda: self._coverage_run_test( - project, - project_build_dir, - lcov_file)) + project, project_build_dir, lcov_file + ) + ) else: # Configure and compile the non-test project. self.executor.append( func=lambda: self._coverage_compile_only( - project, - project_build_dir, - lcov_file)) + project, project_build_dir, lcov_file + ) + ) rv = self.executor.wait() if rv: @@ -593,18 +665,16 @@ class Zmake: with self.jobserver.get_job(): # Get the build version proc = self.jobserver.popen( - [self.module_paths['ec'] / - 'util/getversion.sh'], + [self.module_paths["ec"] / "util/getversion.sh"], stdout=subprocess.PIPE, stderr=subprocess.PIPE, - encoding='utf-8', - errors='replace') + encoding="utf-8", + errors="replace", + ) zmake.multiproc.log_output( - self.logger, - logging.ERROR, - proc.stderr, - job_id="getversion.sh") - version = '' + self.logger, logging.ERROR, proc.stderr, job_id="getversion.sh" + ) + version = "" for line in proc.stdout: match = re.search(r'#define VERSION "(.*)"', line) if match: @@ -613,46 +683,52 @@ class Zmake: raise OSError(get_process_failure_msg(proc)) # Merge info files into a single lcov.info - self.logger.info("Merging coverage data into %s.", - build_dir / 'lcov.info') - cmd = ['/usr/bin/lcov', '-o', build_dir / 'lcov.info'] + self.logger.info("Merging coverage data into %s.", build_dir / "lcov.info") + cmd = ["/usr/bin/lcov", "-o", build_dir / "lcov.info"] for info in all_lcov_files: - cmd += ['-a', info] + cmd += ["-a", info] proc = self.jobserver.popen( cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - encoding='utf-8', - errors='replace') + encoding="utf-8", + errors="replace", + ) zmake.multiproc.log_output( - self.logger, logging.ERROR, proc.stderr, job_id="lcov") + self.logger, logging.ERROR, proc.stderr, job_id="lcov" + ) zmake.multiproc.log_output( - self.logger, logging.DEBUG, proc.stdout, job_id="lcov") + self.logger, logging.DEBUG, proc.stdout, job_id="lcov" + ) if proc.wait(): raise OSError(get_process_failure_msg(proc)) # Merge into a nice html report - self.logger.info("Creating coverage report %s.", - build_dir / 'coverage_rpt') + self.logger.info("Creating coverage report %s.", build_dir / "coverage_rpt") proc = self.jobserver.popen( - ['/usr/bin/genhtml', '-q', '-o', - build_dir / 'coverage_rpt', '-t', - "Zephyr EC Unittest {}".format(version), '-p', - self.checkout / 'src', '-s'] + all_lcov_files, + [ + "/usr/bin/genhtml", + "-q", + "-o", + build_dir / "coverage_rpt", + "-t", + "Zephyr EC Unittest {}".format(version), + "-p", + self.checkout / "src", + "-s", + ] + + all_lcov_files, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - encoding='utf-8', - errors='replace') + encoding="utf-8", + errors="replace", + ) zmake.multiproc.log_output( - self.logger, - logging.ERROR, - proc.stderr, - job_id="genhtml") + self.logger, logging.ERROR, proc.stderr, job_id="genhtml" + ) zmake.multiproc.log_output( - self.logger, - logging.DEBUG, - proc.stdout, - job_id="genhtml") + self.logger, logging.DEBUG, proc.stdout, job_id="genhtml" + ) if proc.wait(): raise OSError(get_process_failure_msg(proc)) return 0 |