zephyr: Move ec_version.h generation logic from zmake to CMake

(This CL needs further testing to make sure it works in all conditions)

 * Add a new Python script that allows generating the ec_version.h
   header while being somewhat separated from zmake.
 * Call this new script from CMake so builds can be initiated through
   Twister without needing zmake to create the header as a pre-build
   step.

BRANCH=None
BUG=None
TEST=zmake test test-drivers (--clobber); zephyr/zmake/run_tests.sh

Signed-off-by: Tristan Honscheid <honscheid@google.com>
Change-Id: Iccb670fecd2c40ecc6360521d65259f411b3c7e7
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/platform/ec/+/3656258
Reviewed-by: Yuval Peress <peress@google.com>
Reviewed-by: Keith Short <keithshort@chromium.org>

6 files changed, 188 insertions, 19 deletions

diff --git a/zephyr/zmake/setup.py b/zephyr/zmake/setup.py
index 3786335f19..97c126d424 100644
--- a/zephyr/zmake/setup.py
+++ b/zephyr/zmake/setup.py
@@ -18,7 +18,7 @@ setuptools.setup(
     keywords="chromeos",
     # You can just specify the packages manually here if your project is
     # simple. Or you can use find_packages().
-    packages=["zmake"],
+    packages=["zmake", "zephyr_build_tools"],
     python_requires=">=3.6, <4",
     # List run-time dependencies here.  These will be installed by pip when
     # your project is installed. For an analysis of "install_requires" vs pip's
diff --git a/zephyr/zmake/tests/test_version.py b/zephyr/zmake/tests/test_version.py
index 9e00473752..a81d682b77 100644
--- a/zephyr/zmake/tests/test_version.py
+++ b/zephyr/zmake/tests/test_version.py
@@ -8,7 +8,7 @@ import datetime
 import subprocess
 import unittest.mock as mock
 
-import pytest
+import pytest  # pylint: disable=import-error
 
 import zmake.output_packers
 import zmake.project
@@ -95,7 +95,7 @@ def test_version_string(tmp_path):
     """Test a that version string is as expected."""
     project, zephyr_base, modules = _setup_example_repos(tmp_path)
     assert (
-        version.get_version_string(project, zephyr_base, modules)
+        version.get_version_string(project.config.project_name, zephyr_base, modules)
         == "prj_v2.6.4-ec:b5991f,os:377d26,mod1:02fd7a"
     )
 
@@ -104,7 +104,9 @@ def test_version_string_static(tmp_path):
     """Test a that version string with no git hashes."""
     project, zephyr_base, modules = _setup_example_repos(tmp_path)
     assert (
-        version.get_version_string(project, zephyr_base, modules, static=True)
+        version.get_version_string(
+            project.config.project_name, zephyr_base, modules, static=True
+        )
         == "prj_v2.6.0-STATIC"
     )
 
@@ -128,7 +130,7 @@ def fake_date():
 
 HEADER_VERSION_STR = "trogdor_v2.6.1004-cmsis:0dead0,hal_stm32:0beef0,os:ad00da"
 EXPECTED_HEADER = (
-    "/* This file is automatically generated by zmake */\n"
+    "/* This file is automatically generated by zmake_tests */\n"
     '#define VERSION "trogdor_v2.6.1004-cmsis:0dead0,hal_stm32:0beef0,os:ad00da"\n'
     '#define CROS_EC_VERSION32 "trogdor_v2.6.1004-cmsis:0dead0,"\n'
     '#define BUILDER "toukmond@pokey"\n'
@@ -137,7 +139,7 @@ EXPECTED_HEADER = (
 )
 HEADER_VERSION_STR_STATIC = "trogdor_v2.6.0-STATIC"
 EXPECTED_HEADER_STATIC = (
-    "/* This file is automatically generated by zmake */\n"
+    "/* This file is automatically generated by zmake_tests */\n"
     '#define VERSION "trogdor_v2.6.0-STATIC"\n'
     '#define CROS_EC_VERSION32 "trogdor_v2.6.0-STATIC"\n'
     '#define BUILDER "reproducible@build"\n'
@@ -150,7 +152,7 @@ def test_header_gen(fake_user_hostname, fake_date, tmp_path):
     """Test generating the version header."""
     # Test the simple case (static=False, no existing header).
     output_file = tmp_path / "ec_version.h"
-    version.write_version_header(HEADER_VERSION_STR, output_file)
+    version.write_version_header(HEADER_VERSION_STR, output_file, "zmake_tests")
     assert output_file.read_text() == EXPECTED_HEADER
 
 
@@ -158,7 +160,9 @@ def test_header_gen_reproducible_build(tmp_path):
     """Test that reproducible builds produce the right header."""
     # With static=True this time.
     output_file = tmp_path / "ec_version.h"
-    version.write_version_header(HEADER_VERSION_STR_STATIC, output_file, static=True)
+    version.write_version_header(
+        HEADER_VERSION_STR_STATIC, output_file, "zmake_tests", static=True
+    )
     assert output_file.read_text() == EXPECTED_HEADER_STATIC
 
 
@@ -168,11 +172,11 @@ def test_header_gen_exists_not_changed(fake_user_hostname, fake_date, tmp_path):
     output_file = tmp_path / "ec_version.h"
 
     # First time, write and record mtime.
-    version.write_version_header(HEADER_VERSION_STR, output_file)
+    version.write_version_header(HEADER_VERSION_STR, output_file, "zmake_tests")
     expected_mtime = output_file.stat().st_mtime
 
     # Do another write (contents should be unchanged).
-    version.write_version_header(HEADER_VERSION_STR, output_file)
+    version.write_version_header(HEADER_VERSION_STR, output_file, "zmake_tests")
 
     # Assert we didn't write again.
     assert output_file.stat().st_mtime == expected_mtime
@@ -184,11 +188,13 @@ def test_header_gen_exists_needs_changes(fake_user_hostname, fake_date, tmp_path
     output_file = tmp_path / "ec_version.h"
 
     # First time, write and save contents.
-    version.write_version_header(HEADER_VERSION_STR, output_file)
+    version.write_version_header(HEADER_VERSION_STR, output_file, "zmake_tests")
     original_contents = output_file.read_text()
 
     # Do another write (contents should be changed).
-    version.write_version_header(HEADER_VERSION_STR_STATIC, output_file, static=True)
+    version.write_version_header(
+        HEADER_VERSION_STR_STATIC, output_file, "zmake_tests", static=True
+    )
 
     # Assert we overwrote.
     assert output_file.read_text() != original_contents
diff --git a/zephyr/zmake/zephyr_build_tools/__init__.py b/zephyr/zmake/zephyr_build_tools/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/zephyr/zmake/zephyr_build_tools/__init__.py
diff --git a/zephyr/zmake/zephyr_build_tools/generate_ec_version.py b/zephyr/zmake/zephyr_build_tools/generate_ec_version.py
new file mode 100755
index 0000000000..96f9013ca4
--- /dev/null
+++ b/zephyr/zmake/zephyr_build_tools/generate_ec_version.py
@@ -0,0 +1,161 @@
+#!/usr/bin/env python3
+
+# Copyright 2022 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Code to generate the ec_version.h file."""
+
+import argparse
+import logging
+import os.path
+import pathlib
+import sys
+
+import zmake.version
+
+
+def convert_module_list_to_dict(modules: list) -> dict:
+    """Convert a list of string paths to modules in to a dict of module
+    names to paths."""
+
+    if not modules:
+        return {}
+
+    dict_out = {}
+    for mod in modules:
+        if not mod.is_dir():
+            raise FileNotFoundError(f"Module '{mod}' not found")
+
+        dict_out[mod.name] = mod
+
+    return dict_out
+
+
+def main():
+    """CLI entry point for generating the ec_version.h header"""
+    logging.basicConfig(level=logging.INFO, stream=sys.stderr)
+
+    parser = argparse.ArgumentParser()
+
+    parser.add_argument("header_path", help="Path to write ec_version.h to")
+    parser.add_argument(
+        "-s",
+        "--static",
+        action="store_true",
+        help="If set, generate a header which does not include information "
+        "like the username, hostname, or date, allowing the build to be"
+        "reproducible.",
+    )
+    parser.add_argument(
+        "--base",
+        default=os.environ.get("ZEPHYR_BASE"),
+        help="Path to Zephyr base directory. Uses ZEPHYR_BASE env var if unset.",
+    )
+    parser.add_argument(
+        "-m",
+        "--module",
+        action="append",
+        help="Specify modules paths to include in version hash. Uses "
+        "ZEPHYR_MODULES env var if unset",
+    )
+    parser.add_argument("-n", "--name", required=True, type=str, help="Project name")
+
+    args = parser.parse_args()
+
+    if args.base is None:
+        logging.error(
+            "No Zephyr base is defined. Pass --base or set env var ZEPHYR_BASE"
+        )
+        return 1
+
+    logging.info("Zephyr Base: %s", args.base)
+
+    if args.static:
+        logging.info("Using a static version string")
+
+    # Make a dict of modules from the list. Modules can be added one at a time
+    # by repeating the -m flag, or once as a semicolon-separated list. In the
+    # later case, we need to expand the modules list.
+
+    if args.module is None:
+        # No modules specified on command line. Default to environment variable.
+        env_modules = os.environ.get("ZEPHYR_MODULES")
+        args.module = env_modules.split(";") if env_modules else []
+        logging.info("No modules passed via CLI. Getting list from ZEPHYR_MODULES")
+
+    elif len(args.module) == 1:
+        # In case of a single -m flag, treat value as a semicolon-delimited
+        # list.
+        args.module = args.module[0].split(";")
+
+    try:
+        module_dict = convert_module_list_to_dict(map(pathlib.Path, args.module))
+    except FileNotFoundError as err:
+        logging.error("Cannot find module: %s", str(err))
+        return 1
+
+    logging.info("Including modules: [%s]", ", ".join(args.module))
+
+    # Generate the version string that gets inserted in to the header. Will get
+    # commit IDs from Git
+    ver = zmake.version.get_version_string(
+        args.name, args.base, module_dict, args.static
+    )
+    logging.info("Version string: %s", ver)
+
+    # Now write the actual header file or put version string in stdout
+    if args.header_path == "-":
+        print(ver)
+    else:
+        output_path = pathlib.Path(args.header_path)
+        output_path.parent.mkdir(parents=True, exist_ok=True)
+
+        logging.info("Writing header to %s", args.header_path)
+        zmake.version.write_version_header(ver, output_path, sys.argv[0], args.static)
+
+    return 0
+
+
+def maybe_reexec():
+    """Re-exec using the zmake package from the EC source tree, as
+    opposed to the system's copy of zmake. This is useful for
+    development when engineers need to make changes to zmake. This
+    script relies on the zmake package for version string generation
+    logic.
+
+    Returns:
+        None, if the re-exec did not happen, or never returns if the
+        re-exec did happen.
+    """
+    # We only re-exec if we are inside of a chroot (since if installed
+    # standalone using pip, there's already an "editable install"
+    # feature for that in pip.)
+    env = dict(os.environ)
+    srcroot = env.get("CROS_WORKON_SRCROOT")
+    if not srcroot:
+        return
+
+    # If for some reason we decide to move zmake in the future, then
+    # we don't want to use the re-exec logic.
+    zmake_path = (
+        pathlib.Path(srcroot) / "src" / "platform" / "ec" / "zephyr" / "zmake"
+    ).resolve()
+    if not zmake_path.is_dir():
+        return
+
+    # If PYTHONPATH is set, it is either because we just did a
+    # re-exec, or because the user wants to run a specific copy of
+    # zmake.  In either case, we don't want to re-exec.
+    if "PYTHONPATH" in env:
+        return
+
+    # Set PYTHONPATH so that we run zmake from source.
+    env["PYTHONPATH"] = str(zmake_path)
+
+    os.execve(sys.argv[0], sys.argv, env)
+
+
+if __name__ == "__main__":
+    maybe_reexec()
+    sys.exit(main())
diff --git a/zephyr/zmake/zmake/version.py b/zephyr/zmake/zmake/version.py
index 5ac060f23b..76d9777eda 100644
--- a/zephyr/zmake/zmake/version.py
+++ b/zephyr/zmake/zmake/version.py
@@ -80,7 +80,7 @@ def get_version_string(project, zephyr_base, modules, static=False):
     """Get the version string associated with a build.
 
     Args:
-        project: a zmake.project.Project object
+        project: a string project name
         zephyr_base: the path to the zephyr directory
         modules: a dictionary mapping module names to module paths
         static: if set, create a version string not dependent on git
@@ -117,7 +117,7 @@ def get_version_string(project, zephyr_base, modules, static=False):
         )
 
     return "{}_v{}.{}.{}-{}".format(
-        project.config.project_name,
+        project,
         major_version,
         minor_version,
         num_commits,
@@ -125,7 +125,7 @@ def get_version_string(project, zephyr_base, modules, static=False):
     )
 
 
-def write_version_header(version_str, output_path, static=False):
+def write_version_header(version_str, output_path, tool, static=False):
     """Generate a version header and write it to the specified path.
 
     Generate a version header in the format expected by the EC build
@@ -139,12 +139,15 @@ def write_version_header(version_str, output_path, static=False):
             as one generated by get_version_string.
         output_path: The file path to write at (a pathlib.Path
             object).
+        tool: Name of the tool that is invoking this function ("zmake",
+            "generate_ec_version.py", etc). Included in a comment in the
+            header.
         static: If true, generate a header which does not include
             information like the username, hostname, or date, allowing
             the build to be reproducible.
     """
     output = io.StringIO()
-    output.write("/* This file is automatically generated by zmake */\n")
+    output.write(f"/* This file is automatically generated by {tool} */\n")
 
     def add_def(name, value):
         output.write("#define {} {}\n".format(name, util.c_str(value)))
diff --git a/zephyr/zmake/zmake/zmake.py b/zephyr/zmake/zmake/zmake.py
index 698cf40b2b..71fef5f16b 100644
--- a/zephyr/zmake/zmake/zmake.py
+++ b/zephyr/zmake/zmake/zmake.py
@@ -691,7 +691,7 @@ class Zmake:
 
         # Compute the version string.
         version_string = zmake.version.get_version_string(
-            project,
+            project.config.project_name,
             build_dir / "zephyr_base",
             zmake.modules.locate_from_directory(build_dir / "modules"),
         )
@@ -700,8 +700,7 @@ class Zmake:
         # instead of configure, as the tree may have changed since
         # configure was run.
         zmake.version.write_version_header(
-            version_string,
-            build_dir / "include" / "ec_version.h",
+            version_string, build_dir / "include" / "ec_version.h", "zmake"
         )
 
         gcov = "gcov.sh-not-found"