diff options
Diffstat (limited to 'zephyr/firmware_builder.py')
| -rwxr-xr-x | zephyr/firmware_builder.py | 209 |
1 files changed, 108 insertions, 101 deletions
diff --git a/zephyr/firmware_builder.py b/zephyr/firmware_builder.py index f77e51d6c4..c0963a84db 100755 --- a/zephyr/firmware_builder.py +++ b/zephyr/firmware_builder.py @@ -15,14 +15,12 @@ import re import subprocess import sys -from google.protobuf import json_format # pylint: disable=import-error import zmake.project - from chromite.api.gen_sdk.chromite.api import firmware_pb2 +from google.protobuf import json_format # pylint: disable=import-error - -DEFAULT_BUNDLE_DIRECTORY = '/tmp/artifact_bundles' -DEFAULT_BUNDLE_METADATA_FILE = '/tmp/artifact_bundle_metadata' +DEFAULT_BUNDLE_DIRECTORY = "/tmp/artifact_bundles" +DEFAULT_BUNDLE_METADATA_FILE = "/tmp/artifact_bundle_metadata" def build(opts): @@ -33,54 +31,52 @@ def build(opts): platform_ec = zephyr_dir.resolve().parent subprocess.run([platform_ec / "util" / "check_clang_format.py"], check=True) - cmd = ['zmake', '-D', 'build', '-a'] + cmd = ["zmake", "-D", "build", "-a"] if opts.code_coverage: - cmd.append('--coverage') + cmd.append("--coverage") subprocess.run(cmd, cwd=pathlib.Path(__file__).parent, check=True) if not opts.code_coverage: for project in zmake.project.find_projects(zephyr_dir).values(): if project.config.is_test: continue - build_dir = ( - platform_ec / 'build' / 'zephyr' / project.config.project_name - ) + build_dir = platform_ec / "build" / "zephyr" / project.config.project_name metric = metric_list.value.add() metric.target_name = project.config.project_name metric.platform_name = project.config.zephyr_board for (variant, _) in project.iter_builds(): - build_log = build_dir / f'build-{variant}' / 'build.log' + build_log = build_dir / f"build-{variant}" / "build.log" parse_buildlog(build_log, metric, variant.upper()) - with open(opts.metrics, 'w') as file: + with open(opts.metrics, "w") as file: file.write(json_format.MessageToJson(metric_list)) return 0 UNITS = { - 'B': 1, - 'KB': 1024, - 'MB': 1024 * 1024, - 'GB': 1024 * 1024 * 1024, + "B": 1, + "KB": 1024, + "MB": 1024 * 1024, + "GB": 1024 * 1024 * 1024, } def parse_buildlog(filename, metric, variant): """Parse the build.log generated by zmake to find the size of the image.""" - with open(filename, 'r') as infile: + with open(filename, "r") as infile: # Skip over all lines until the memory report is found while True: line = infile.readline() if not line: return - if line.startswith('Memory region'): + if line.startswith("Memory region"): break for line in infile.readlines(): # Skip any lines that are not part of the report - if not line.startswith(' '): + if not line.startswith(" "): continue parts = line.split() fw_section = metric.fw_section.add() - fw_section.region = variant + '_' + parts[0][:-1] + fw_section.region = variant + "_" + parts[0][:-1] fw_section.used = int(parts[1]) * UNITS[parts[2]] fw_section.total = int(parts[3]) * UNITS[parts[4]] fw_section.track_on_gerrit = False @@ -114,7 +110,7 @@ def write_metadata(opts, info): bundle_metadata_file = ( opts.metadata if opts.metadata else DEFAULT_BUNDLE_METADATA_FILE ) - with open(bundle_metadata_file, 'w') as file: + with open(bundle_metadata_file, "w") as file: file.write(json_format.MessageToJson(info)) @@ -125,10 +121,10 @@ def bundle_coverage(opts): bundle_dir = get_bundle_dir(opts) zephyr_dir = pathlib.Path(__file__).parent platform_ec = zephyr_dir.resolve().parent - build_dir = platform_ec / 'build' / 'zephyr' - tarball_name = 'coverage.tbz2' + build_dir = platform_ec / "build" / "zephyr" + tarball_name = "coverage.tbz2" tarball_path = bundle_dir / tarball_name - cmd = ['tar', 'cvfj', tarball_path, 'lcov.info'] + cmd = ["tar", "cvfj", tarball_path, "lcov.info"] subprocess.run(cmd, cwd=build_dir, check=True) meta = info.objects.add() meta.file_name = tarball_name @@ -149,13 +145,11 @@ def bundle_firmware(opts): for project in zmake.project.find_projects(zephyr_dir).values(): if project.config.is_test: continue - build_dir = ( - platform_ec / 'build' / 'zephyr' / project.config.project_name - ) - artifacts_dir = build_dir / 'output' - tarball_name = f'{project.config.project_name}.firmware.tbz2' + build_dir = platform_ec / "build" / "zephyr" / project.config.project_name + artifacts_dir = build_dir / "output" + tarball_name = f"{project.config.project_name}.firmware.tbz2" tarball_path = bundle_dir.joinpath(tarball_name) - cmd = ['tar', 'cvfj', tarball_path, '.'] + cmd = ["tar", "cvfj", tarball_path, "."] subprocess.run(cmd, cwd=artifacts_dir, check=True) meta = info.objects.add() meta.file_name = tarball_name @@ -176,76 +170,92 @@ def test(opts): # Run zmake tests to ensure we have a fully working zmake before # proceeding. - subprocess.run([zephyr_dir / 'zmake' / 'run_tests.sh'], check=True) + subprocess.run([zephyr_dir / "zmake" / "run_tests.sh"], check=True) # Run formatting checks on all BUILD.py files. - config_files = zephyr_dir.rglob('**/BUILD.py') - subprocess.run(['black', '--diff', '--check', *config_files], check=True) + config_files = zephyr_dir.rglob("**/BUILD.py") + subprocess.run(["black", "--diff", "--check", *config_files], check=True) - cmd = ['zmake', '-D', 'test', '-a', '--no-rebuild'] + cmd = ["zmake", "-D", "test", "-a", "--no-rebuild"] if opts.code_coverage: - cmd.append('--coverage') + cmd.append("--coverage") ret = subprocess.run(cmd, check=True).returncode if ret: return ret if opts.code_coverage: platform_ec = zephyr_dir.parent - build_dir = platform_ec / 'build' / 'zephyr' + build_dir = platform_ec / "build" / "zephyr" # Merge lcov files here because bundle failures are "infra" failures. cmd = [ - '/usr/bin/lcov', - '-o', - build_dir / 'zephyr_merged.info', - '--rc', - 'lcov_branch_coverage=1', - '-a', - build_dir / 'all_tests.info', - '-a', - build_dir / 'all_builds.info', + "/usr/bin/lcov", + "-o", + build_dir / "zephyr_merged.info", + "--rc", + "lcov_branch_coverage=1", + "-a", + build_dir / "all_tests.info", + "-a", + build_dir / "all_builds.info", ] output = subprocess.run( - cmd, cwd=pathlib.Path(__file__).parent, check=True, - stdout=subprocess.PIPE, universal_newlines=True).stdout - _extract_lcov_summary('EC_ZEPHYR_MERGED', metrics, output) + cmd, + cwd=pathlib.Path(__file__).parent, + check=True, + stdout=subprocess.PIPE, + universal_newlines=True, + ).stdout + _extract_lcov_summary("EC_ZEPHYR_MERGED", metrics, output) output = subprocess.run( - ['/usr/bin/lcov', '--summary', build_dir / 'all_tests.info'], - cwd=pathlib.Path(__file__).parent, check=True, - stdout=subprocess.PIPE, universal_newlines=True).stdout - _extract_lcov_summary('EC_ZEPHYR_TESTS', metrics, output) - - cmd = ['make', 'coverage', f'-j{opts.cpus}'] + ["/usr/bin/lcov", "--summary", build_dir / "all_tests.info"], + cwd=pathlib.Path(__file__).parent, + check=True, + stdout=subprocess.PIPE, + universal_newlines=True, + ).stdout + _extract_lcov_summary("EC_ZEPHYR_TESTS", metrics, output) + + cmd = ["make", "coverage", f"-j{opts.cpus}"] print(f"# Running {' '.join(cmd)}.") subprocess.run(cmd, cwd=platform_ec, check=True) output = subprocess.run( - ['/usr/bin/lcov', '--summary', platform_ec / 'build/coverage/lcov.info'], - cwd=pathlib.Path(__file__).parent, check=True, - stdout=subprocess.PIPE, universal_newlines=True).stdout - _extract_lcov_summary('EC_LEGACY_MERGED', metrics, output) + ["/usr/bin/lcov", "--summary", platform_ec / "build/coverage/lcov.info"], + cwd=pathlib.Path(__file__).parent, + check=True, + stdout=subprocess.PIPE, + universal_newlines=True, + ).stdout + _extract_lcov_summary("EC_LEGACY_MERGED", metrics, output) cmd = [ - '/usr/bin/lcov', - '-o', - build_dir / 'lcov.info', - '--rc', - 'lcov_branch_coverage=1', - '-a', - build_dir / 'zephyr_merged.info', - '-a', - platform_ec / 'build/coverage/lcov.info', + "/usr/bin/lcov", + "-o", + build_dir / "lcov.info", + "--rc", + "lcov_branch_coverage=1", + "-a", + build_dir / "zephyr_merged.info", + "-a", + platform_ec / "build/coverage/lcov.info", ] output = subprocess.run( - cmd, cwd=pathlib.Path(__file__).parent, check=True, - stdout=subprocess.PIPE, universal_newlines=True).stdout - _extract_lcov_summary('ALL_MERGED', metrics, output) - - with open(opts.metrics, 'w') as file: + cmd, + cwd=pathlib.Path(__file__).parent, + check=True, + stdout=subprocess.PIPE, + universal_newlines=True, + ).stdout + _extract_lcov_summary("ALL_MERGED", metrics, output) + + with open(opts.metrics, "w") as file: file.write(json_format.MessageToJson(metrics)) return 0 -COVERAGE_RE = re.compile(r'lines\.*: *([0-9\.]+)% \(([0-9]+) of ([0-9]+) lines\)') +COVERAGE_RE = re.compile(r"lines\.*: *([0-9\.]+)% \(([0-9]+) of ([0-9]+) lines\)") + + def _extract_lcov_summary(name, metrics, output): re_match = COVERAGE_RE.search(output) if re_match: @@ -255,12 +265,13 @@ def _extract_lcov_summary(name, metrics, output): metric.covered_lines = int(re_match.group(2)) metric.total_lines = int(re_match.group(3)) + def main(args): """Builds and tests all of the Zephyr targets and reports build metrics""" opts = parse_args(args) - if not hasattr(opts, 'func'): - print('Must select a valid sub command!') + if not hasattr(opts, "func"): + print("Must select a valid sub command!") return -1 # Run selected sub command function @@ -272,70 +283,66 @@ def parse_args(args): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( - '--cpus', + "--cpus", default=multiprocessing.cpu_count(), - help='The number of cores to use.', + help="The number of cores to use.", ) parser.add_argument( - '--metrics', - dest='metrics', + "--metrics", + dest="metrics", required=True, - help='File to write the json-encoded MetricsList proto message.', + help="File to write the json-encoded MetricsList proto message.", ) parser.add_argument( - '--metadata', + "--metadata", required=False, help=( - 'Full pathname for the file in which to write build artifact ' - 'metadata.' + "Full pathname for the file in which to write build artifact " "metadata." ), ) parser.add_argument( - '--output-dir', + "--output-dir", required=False, - help=( - 'Full pathname for the directory in which to bundle build ' - 'artifacts.' - ), + help=("Full pathname for the directory in which to bundle build " "artifacts."), ) parser.add_argument( - '--code-coverage', + "--code-coverage", required=False, - action='store_true', - help='Build host-based unit tests for code coverage.', + action="store_true", + help="Build host-based unit tests for code coverage.", ) parser.add_argument( - '--bcs-version', - dest='bcs_version', - default='', + "--bcs-version", + dest="bcs_version", + default="", required=False, # TODO(b/180008931): make this required=True. - help='BCS version to include in metadata.', + help="BCS version to include in metadata.", ) # Would make this required=True, but not available until 3.7 sub_cmds = parser.add_subparsers() - build_cmd = sub_cmds.add_parser('build', help='Builds all firmware targets') + build_cmd = sub_cmds.add_parser("build", help="Builds all firmware targets") build_cmd.set_defaults(func=build) build_cmd = sub_cmds.add_parser( - 'bundle', - help='Creates a tarball containing build ' - 'artifacts from all firmware targets', + "bundle", + help="Creates a tarball containing build " + "artifacts from all firmware targets", ) build_cmd.set_defaults(func=bundle) - test_cmd = sub_cmds.add_parser('test', help='Runs all firmware unit tests') + test_cmd = sub_cmds.add_parser("test", help="Runs all firmware unit tests") test_cmd.set_defaults(func=test) return parser.parse_args(args) -if __name__ == '__main__': +if __name__ == "__main__": sys.exit(main(sys.argv[1:])) |