cq: Run tests with both gcc and clang.

To ensure compatibility with gitlab, run all the zephyr tests with both
gcc and llvm/clang.  This adds 6 minutes to the build time on my machine
but adds 0.4% coverage overall, and 1.7% to herobrine, since it fixes
all the uncovered switch case labels.

BRANCH=None
BUG=None
TEST=Ran firmware_builder.py locally

Signed-off-by: Jeremy Bettis <jbettis@google.com>
Change-Id: I5a7c1f2e8f226e18d642eb886fd90c778c7e77d2
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/platform/ec/+/4004761
Reviewed-by: Al Semjonovs <asemjonovs@google.com>
Commit-Queue: Jeremy Bettis <jbettis@chromium.org>
Tested-by: Jeremy Bettis <jbettis@chromium.org>
Code-Coverage: Zoss <zoss-cl-coverage@prod.google.com>
Reviewed-by: Tristan Honscheid <honscheid@google.com>

1 files changed, 68 insertions, 11 deletions

diff --git a/zephyr/firmware_builder.py b/zephyr/firmware_builder.py
index fd46f200f9..7f0776e2be 100755
--- a/zephyr/firmware_builder.py
+++ b/zephyr/firmware_builder.py
@@ -10,6 +10,7 @@ This is the entry point for the custom firmware builder workflow recipe.
 
 import argparse
 import multiprocessing
+import os
 import pathlib
 import re
 import shlex
@@ -27,18 +28,45 @@ DEFAULT_BUNDLE_METADATA_FILE = "/tmp/artifact_bundle_metadata"
 SPECIAL_BOARDS = ["herobrine", "krabby", "nivviks", "skyrim", "kingler"]
 
 
-def log_cmd(cmd):
+def log_cmd(cmd, env=None):
     """Log subprocess command."""
+    if env is not None:
+        print("env", end=" ")
+        [  # pylint:disable=expression-not-assigned
+            print(key + "=" + shlex.quote(str(value)), end=" ")
+            for key, value in env.items()
+        ]
     print(" ".join(shlex.quote(str(x)) for x in cmd))
     sys.stdout.flush()
 
 
-def run_twister(platform_ec, code_coverage=False, extra_args=None):
-    """Build the tests using twister."""
+def run_twister(
+    platform_ec, code_coverage=False, extra_args=None, use_gcc=False
+):
+    """Build the tests using twister.
+
+    Returns the path to the twister-out dir.
+    """
+
+    third_party_zephyr = platform_ec.parent.parent / "third_party/zephyr/main"
+    if use_gcc:
+        c_compiler = "/usr/bin/x86_64-pc-linux-gnu-gcc"
+        cxx_compiler = "/usr/bin/x86_64-pc-linux-gnu-g++"
+        outdir = "twister-out-gcc"
+        env = {
+            "ZEPHYR_TOOLCHAIN_VARIANT": "host",
+            "TOOLCHAIN_ROOT": third_party_zephyr,
+        }
+    else:
+        c_compiler = "/usr/bin/x86_64-pc-linux-gnu-clang"
+        cxx_compiler = "/usr/bin/x86_64-pc-linux-gnu-clang++"
+        outdir = "twister-out-llvm"
+        env = {}
+
     cmd = [
         platform_ec / "twister",
         "--outdir",
-        platform_ec / "twister-out",
+        platform_ec / outdir,
         "-v",
         "-i",
         "-p",
@@ -46,8 +74,8 @@ def run_twister(platform_ec, code_coverage=False, extra_args=None):
         "-p",
         "unit_testing",
         "--no-upload-cros-rdb",
-        "-x=CMAKE_C_COMPILER=/usr/bin/x86_64-pc-linux-gnu-gcc",
-        "-x=CMAKE_CXX_COMPILER=/usr/bin/x86_64-pc-linux-gnu-g++",
+        "-x=CMAKE_C_COMPILER=" + c_compiler,
+        "-x=CMAKE_CXX_COMPILER=" + cxx_compiler,
     ]
 
     if extra_args:
@@ -61,8 +89,18 @@ def run_twister(platform_ec, code_coverage=False, extra_args=None):
                 "--coverage",
             ]
         )
-    log_cmd(cmd)
-    subprocess.run(cmd, check=True, cwd=platform_ec, stdin=subprocess.DEVNULL)
+    log_cmd(cmd, env=env)
+    my_env = os.environ.copy()
+    my_env.update(env)
+
+    subprocess.run(
+        cmd,
+        check=True,
+        cwd=platform_ec,
+        stdin=subprocess.DEVNULL,
+        env=my_env,
+    )
+    return platform_ec / outdir
 
 
 def build(opts):
@@ -250,7 +288,10 @@ def test(opts):
     # Twister-based tests
     platform_ec = zephyr_dir.parent
     third_party = platform_ec.parent.parent / "third_party"
-    run_twister(platform_ec, opts.code_coverage)
+    twister_out_dir = run_twister(platform_ec, opts.code_coverage)
+    twister_out_dir_gcc = run_twister(
+        platform_ec, opts.code_coverage, use_gcc=True
+    )
 
     if opts.code_coverage:
         build_dir = platform_ec / "build" / "zephyr"
@@ -259,7 +300,7 @@ def test(opts):
             [
                 "/usr/bin/lcov",
                 "--summary",
-                platform_ec / "twister-out" / "coverage.info",
+                twister_out_dir / "coverage.info",
             ],
             cwd=zephyr_dir,
             check=True,
@@ -269,6 +310,20 @@ def test(opts):
         ).stdout
         _extract_lcov_summary("EC_ZEPHYR_TESTS", metrics, output)
 
+        output = subprocess.run(
+            [
+                "/usr/bin/lcov",
+                "--summary",
+                twister_out_dir_gcc / "coverage.info",
+            ],
+            cwd=zephyr_dir,
+            check=True,
+            stdout=subprocess.PIPE,
+            universal_newlines=True,
+            stdin=subprocess.DEVNULL,
+        ).stdout
+        _extract_lcov_summary("EC_ZEPHYR_TESTS_GCC", metrics, output)
+
         cmd = ["make", "test-coverage", f"-j{opts.cpus}", "V=1"]
         log_cmd(cmd)
         subprocess.run(
@@ -298,7 +353,9 @@ def test(opts):
             "-a",
             platform_ec / "build/coverage/lcov.info",
             "-a",
-            platform_ec / "twister-out" / "coverage.info",
+            twister_out_dir / "coverage.info",
+            "-a",
+            twister_out_dir_gcc / "coverage.info",
         ]
         log_cmd(cmd)
         output = subprocess.run(