From 5756bbb429dc258ff3f2334fd6161fa49512e820 Mon Sep 17 00:00:00 2001 From: Madhurima Paruchuri Date: Sun, 15 Jan 2023 09:24:47 +0000 Subject: zephyr: zmake: Add config and device tree check capability to compare-builds The invocation of command with -con arg now will compare config files, -dt will compare devicetree files along with binary files and outputs the failed projects lists separately along with combined list. With -bin arg now binaries comparisons can be avoided. BUG=b:262374758 BRANCH=none TEST=zmake compare-builds it8xxx2_evb -bin -con -dt TEST=Modified dt and config in Rex and saw comparison failing Change-Id: I29751c640b833792f6bfbacbdb80b9119bdb8efc Signed-off-by: Madhurima Paruchuri Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/platform/ec/+/4169805 Reviewed-by: Fabio Baltieri --- zephyr/zmake/README.md | 5 +- zephyr/zmake/zmake/__main__.py | 27 +++++- zephyr/zmake/zmake/compare_builds.py | 173 ++++++++++++++++++++++++++--------- zephyr/zmake/zmake/zmake.py | 21 ++++- 4 files changed, 178 insertions(+), 48 deletions(-) diff --git a/zephyr/zmake/README.md b/zephyr/zmake/README.md index f8adbdc81e..87af4242f3 100644 --- a/zephyr/zmake/README.md +++ b/zephyr/zmake/README.md @@ -91,7 +91,7 @@ Chromium OS's meta-build tool for Zephyr ### zmake compare-builds -**Usage:** `zmake compare-builds [-h] [--ref1 REF1] [--ref2 REF2] [-k] [-t TOOLCHAIN] [--extra-cflags EXTRA_CFLAGS] (-a | project_name [project_name ...])` +**Usage:** `zmake compare-builds [-h] [--ref1 REF1] [--ref2 REF2] [-k] [-n] [-b] [-d] [-t TOOLCHAIN] [--extra-cflags EXTRA_CFLAGS] (-a | project_name [project_name ...])` #### Positional Arguments @@ -107,6 +107,9 @@ Chromium OS's meta-build tool for Zephyr | `--ref1 REF1` | 1st git reference (commit, branch, etc), default=HEAD | | `--ref2 REF2` | 2nd git reference (commit, branch, etc), default=HEAD~ | | `-k`, `--keep-temps` | Keep temporary build directories on exit | +| `-n`, `--compare-configs` | Compare configs of build outputs | +| `-b`, `--compare-binaries-disable` | Don't compare binaries of build outputs | +| `-d`, `--compare-devicetrees` | Compare devicetrees of build outputs | | `-t TOOLCHAIN`, `--toolchain TOOLCHAIN` | Name of toolchain to use | | `--extra-cflags EXTRA_CFLAGS` | Additional CFLAGS to use for target builds | | `-a`, `--all` | Select all projects | diff --git a/zephyr/zmake/zmake/__main__.py b/zephyr/zmake/zmake/__main__.py index 99b69747ea..7b79fb23e6 100644 --- a/zephyr/zmake/zmake/__main__.py +++ b/zephyr/zmake/zmake/__main__.py @@ -205,6 +205,24 @@ def get_argparser(): action="store_true", help="Keep temporary build directories on exit", ) + compare_builds.add_argument( + "-n", + "--compare-configs", + action="store_true", + help="Compare configs of build outputs", + ) + compare_builds.add_argument( + "-b", + "--compare-binaries-disable", + action="store_true", + help="Don't compare binaries of build outputs", + ) + compare_builds.add_argument( + "-d", + "--compare-devicetrees", + action="store_true", + help="Compare devicetrees of build outputs", + ) add_common_build_args(compare_builds) list_projects = sub.add_parser( @@ -410,8 +428,15 @@ def main(argv=None): return result or wait_rv finally: multiproc.LogWriter.wait_for_log_end() + for file, failed_projects in zmake.cmp_failed_projects.items(): + if failed_projects: + logging.error( + "Failed projects by diff in %s: %s", + file, + ", ".join(failed_projects), + ) if zmake.failed_projects: - logging.error("Failed projects: %s", zmake.failed_projects) + logging.error("All failed projects: %s", zmake.failed_projects) if __name__ == "__main__": diff --git a/zephyr/zmake/zmake/compare_builds.py b/zephyr/zmake/zmake/compare_builds.py index 92e197de75..a1cc1ee953 100644 --- a/zephyr/zmake/zmake/compare_builds.py +++ b/zephyr/zmake/zmake/compare_builds.py @@ -105,6 +105,18 @@ def create_bin_from_elf(elf_input, bin_output): sys.exit(1) +def _compare_non_test_projects(projects, cmp_method, *args): + failed_projects = [] + for project in projects: + if project.config.is_test: + continue + + if not cmp_method(project, *args): + failed_projects.append(project.config.project_name) + + return failed_projects + + @dataclasses.dataclass class CheckoutConfig: """All the information needed to build the EC at a specific checkout.""" @@ -174,6 +186,51 @@ class CompareBuilds: git_ref="HEAD", ) + def _compare_binaries(self, project): + output_path = ( + pathlib.Path("ec") + / "build" + / "zephyr" + / pathlib.Path(project.config.project_name) + / "output" + ) + + output_dir1 = self.checkouts[0].modules_dir / output_path + output_dir2 = self.checkouts[1].modules_dir / output_path + + bin_output1 = output_dir1 / "ec.bin" + bin_output2 = output_dir2 / "ec.bin" + + # ELF executables don't compare due to meta data. Convert to a binary + # for the comparison + if project.config.output_packer == packer_registry["elf"]: + create_bin_from_elf( + elf_input=output_dir1 / "zephyr.elf", bin_output=bin_output1 + ) + create_bin_from_elf( + elf_input=output_dir2 / "zephyr.elf", bin_output=bin_output2 + ) + + bin1_path = pathlib.Path(bin_output1) + bin2_path = pathlib.Path(bin_output2) + if not os.path.isfile(bin1_path) or not os.path.isfile(bin2_path): + logging.error( + "Zephyr EC binary not found for project %s", + project.config.project_name, + ) + return False + + try: + subprocess.run( + ["cmp", bin_output1, bin_output2], + check=True, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + except subprocess.CalledProcessError: + return False + return True + def check_binaries(self, projects): """Compare Zephyr EC binaries for two different source trees @@ -185,53 +242,83 @@ class CompareBuilds: all projects compared successfully. """ - failed_projects = [] - for project in projects: - if project.config.is_test: - continue - - output_path = ( - pathlib.Path("ec") - / "build" - / "zephyr" - / pathlib.Path(project.config.project_name) - / "output" + failed_projects = _compare_non_test_projects( + projects, self._compare_binaries + ) + return failed_projects + + def _compare_build_files(self, project, build_mode, file): + build_path = ( + pathlib.Path("ec") + / "build" + / "zephyr" + / pathlib.Path(project.config.project_name) + / f"build-{build_mode}" + / "zephyr" + ) + + build_dir1 = self.checkouts[0].modules_dir / build_path + build_dir2 = self.checkouts[1].modules_dir / build_path + + file1 = build_dir1 / file + file2 = build_dir2 / file + + try: + data1 = "" + data2 = "" + with open(file1) as fp1, open(file2) as fp2: + data1 = fp1.read() + data2 = fp2.read() + data1 = data1.replace(self.checkouts[0].full_ref, "") + data2 = data2.replace(self.checkouts[1].full_ref, "") + return data1 == data2 + except FileNotFoundError as err: + logging.error( + "Zephyr build-%s %s file not found for project %s: %s", + build_mode, + file, + project.config.project_name, + err, ) + return False - output_dir1 = self.checkouts[0].modules_dir / output_path - output_dir2 = self.checkouts[1].modules_dir / output_path + def _check_build_files(self, project, file): + return self._compare_build_files( + project, "ro", file + ) and self._compare_build_files(project, "rw", file) - bin_output1 = output_dir1 / "ec.bin" - bin_output2 = output_dir2 / "ec.bin" + def check_configs(self, projects): + """Compare Zephyr EC Config files for two different source trees - # ELF executables don't compare due to meta data. Convert to a binary - # for the comparison - if project.config.output_packer == packer_registry["elf"]: - create_bin_from_elf( - elf_input=output_dir1 / "zephyr.elf", bin_output=bin_output1 - ) - create_bin_from_elf( - elf_input=output_dir2 / "zephyr.elf", bin_output=bin_output2 - ) + Args: + projects: List of projects to compare the .config files. - bin1_path = pathlib.Path(bin_output1) - bin2_path = pathlib.Path(bin_output2) - if not os.path.isfile(bin1_path) or not os.path.isfile(bin2_path): - failed_projects.append(project.config.project_name) - logging.error( - "Zephyr EC binary not found for project %s", - project.config.project_name, - ) - continue - - try: - subprocess.run( - ["cmp", bin_output1, bin_output2], - check=True, - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - ) - except subprocess.CalledProcessError: - failed_projects.append(project.config.project_name) + Returns: + A list of projects that failed to compare. An empty list indicates that + all projects compared successfully. + """ + + failed_projects = _compare_non_test_projects( + projects, + self._check_build_files, + ".config", + ) + return failed_projects + + def check_devicetrees(self, projects): + """Compare Zephyr EC devicetree files for two different source trees + + Args: + projects: List of projects to compare the zephyr.dts files. + + Returns: + A list of projects that failed to compare. An empty list indicates that + all projects compared successfully. + """ + failed_projects = _compare_non_test_projects( + projects, + self._check_build_files, + "zephyr.dts", + ) return failed_projects diff --git a/zephyr/zmake/zmake/zmake.py b/zephyr/zmake/zmake/zmake.py index 6454049d53..30d56cc08f 100644 --- a/zephyr/zmake/zmake/zmake.py +++ b/zephyr/zmake/zmake/zmake.py @@ -191,6 +191,7 @@ class Zmake: self.executor = zmake.multiproc.Executor() self._sequential = self.jobserver.is_sequential() and not goma + self.cmp_failed_projects = {} self.failed_projects = [] @property @@ -342,6 +343,9 @@ class Zmake: extra_cflags=None, keep_temps=False, cmake_defs=None, + compare_configs=False, + compare_binaries_disable=False, + compare_devicetrees=False, ): """Compare EC builds at two commits.""" temp_dir = tempfile.mkdtemp(prefix="zcompare-") @@ -403,9 +407,20 @@ class Zmake: checkout.ref, ) return result - - self.failed_projects = cmp_builds.check_binaries(projects) - + if not compare_binaries_disable: + failed_projects = cmp_builds.check_binaries(projects) + self.cmp_failed_projects["binary"] = failed_projects + self.failed_projects.extend(failed_projects) + if compare_configs: + failed_projects = cmp_builds.check_configs(projects) + self.cmp_failed_projects["config"] = failed_projects + self.failed_projects.extend(failed_projects) + if compare_devicetrees: + failed_projects = cmp_builds.check_devicetrees(projects) + self.cmp_failed_projects["devicetree"] = failed_projects + self.failed_projects.extend(failed_projects) + + self.failed_projects = list(set(self.failed_projects)) if len(self.failed_projects) == 0: self.logger.info("Zephyr compare builds successful:") for checkout in cmp_builds.checkouts: -- cgit v1.2.1