ec: New tool to merge lcov files

Added new tool to merge lcov files using one file as the template and
including only lines that are present in the template file.

The name is because you use the template file like a "stencil" and the
matching coverage shows through. Not great, but it's not really a
"set intersect" or a "merge" either.

Switch firmware_builder.py and gitlab to use the new tool in place of
the filename filtering.

See
https://jbettis.users.x20web.corp.google.com/www/herobrine_rpt/index.html
for updated coverage output.

BRANCH=None
BUG=None
TEST=Ran firmware_builder.py locally.

Signed-off-by: Jeremy Bettis <jbettis@google.com>
Change-Id: I0e9672c346971b0df4602b0adce27fea6367c6b4
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/platform/ec/+/3943261
Reviewed-by: Abe Levkoy <alevkoy@chromium.org>
Commit-Queue: Jeremy Bettis <jbettis@chromium.org>
Auto-Submit: Jeremy Bettis <jbettis@chromium.org>
Tested-by: Jeremy Bettis <jbettis@chromium.org>
Commit-Queue: Abe Levkoy <alevkoy@chromium.org>

3 files changed, 246 insertions, 27 deletions

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 434cd8387a..83f5df5eec 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -156,8 +156,12 @@ before_script:
       "${EC_DIR}/zephyr/shim/chip/npcx/npcx_monitor/**"
       "${EC_DIR}/zephyr/emul/**" "${EC_DIR}/zephyr/test/**"
       "**/testsuite/**" "**/subsys/emul/**"
-    - lcov --rc lcov_branch_coverage=1 -o "${BUILD_DIR}/${PROJECT}/output/merged_no_zephyr.info"
-      -r "${BUILD_DIR}/${PROJECT}/output/merged.info"
+    - util/lcov_stencil.py -o "${BUILD_DIR}/${PROJECT}/output/prefiltered.info"
+      "${BUILD_DIR}/${PROJECT}/output/no_zephyr.info"
+      "${BUILD_DIR}/${PROJECT}/output/merged.info"
+    - lcov --rc lcov_branch_coverage=1
+      -o "${BUILD_DIR}/${PROJECT}/output/filtered_no_zephyr.info"
+      -r "${BUILD_DIR}/${PROJECT}/output/prefiltered.info"
       "${ZEPHYR_BASE}/**" "${MODULES_DIR}/**"
       "${EC_DIR}/zephyr/drivers/**" "${EC_DIR}/zephyr/include/drivers/**"
       "${EC_DIR}/zephyr/shim/chip/**" "${EC_DIR}/zephyr/shim/core/**"
@@ -166,10 +170,6 @@ before_script:
       "${EC_DIR}/zephyr/shim/chip/npcx/npcx_monitor/**"
       "${EC_DIR}/zephyr/emul/**" "${EC_DIR}/zephyr/test/**"
       "**/testsuite/**" "**/subsys/emul/**"
-    - grep "SF:" "${BUILD_DIR}/${PROJECT}/output/no_zephyr.info" | sort -u |
-      sed -e 's|^SF:||' | xargs lcov --rc lcov_branch_coverage=1
-      -o "${BUILD_DIR}/${PROJECT}/output/filtered_no_zephyr.info"
-      -e "${BUILD_DIR}/${PROJECT}/output/merged_no_zephyr.info"
     - /usr/bin/genhtml --branch-coverage -q
       -o "${BUILD_DIR}/${PROJECT}/output/filtered_no_zephyr_rpt"
       -t "${PROJECT} coverage w/o zephyr"
diff --git a/util/lcov_stencil.py b/util/lcov_stencil.py
new file mode 100755
index 0000000000..88fdf6b608
--- /dev/null
+++ b/util/lcov_stencil.py
@@ -0,0 +1,218 @@
+#!/usr/bin/env python3
+# Copyright 2022 The ChromiumOS Authors
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Merge lcov files, discarding all lines that are not in the template file.
+
+Given 2 or more lcov files, merge the results only for the lines present in
+the template file.
+
+File format reverse engineered from
+https://github.com/linux-test-project/lcov/blob/master/bin/geninfo
+"""
+
+import logging
+import re
+import sys
+from collections import defaultdict
+from typing import Dict, Set
+
+from chromite.lib import commandline
+
+EXTRACT_LINE = re.compile(r"^(FN|DA|BRDA):(\d+),")
+EXTRACT_FN = re.compile(r"^(FN):(\d+),(\S+)")
+EXTRACT_FNDA = re.compile(r"^(FNDA):(\d+),(\S+)")
+EXTRACT_DA = re.compile(r"^(DA):(\d+),(\d+)")
+EXTRACT_BRDA = re.compile(r"^(BRDA):(\d+),(\d+),(\d+),([-\d]+)")
+EXTRACT_COUNT = re.compile(r"^([A-Z]+):(\d+)")
+
+
+def parse_args(argv=None):
+    """Parses command line args"""
+    parser = commandline.ArgumentParser()
+    parser.add_argument(
+        "--output-file",
+        "-o",
+        help="destination filename, defaults to stdout",
+    )
+    parser.add_argument(
+        "template_file",
+        help="lcov info file to use as template",
+    )
+    parser.add_argument(
+        "lcov_input",
+        nargs="+",
+        help="lcov info file to merge",
+    )
+    return parser.parse_args(argv)
+
+
+def parse_template_file(filename) -> Dict[str, Set[str]]:
+    """Reads the template file and returns covered lines.
+
+    Reads the lines that indicate covered line numbers (FN, DA, and BRDA)
+    and adds them to the returned data structure.
+
+    Returns
+    -------
+    Dict[str, Set[str]]
+        A dictionary of filename to set of covered line numbers (as strings)
+    """
+    logging.info("Reading template file %s", filename)
+    with open(filename, "r") as template_file:
+        data_by_path: Dict[str, Set[str]] = defaultdict(set)
+        file_name = None
+        for line in template_file.readlines():
+            line = line.strip()
+            if line == "end_of_record":
+                file_name = None
+            elif (
+                line.startswith(  # pylint:disable=too-many-boolean-expressions
+                    "TN:"
+                )
+                or line.startswith("FNDA:")
+                or line.startswith("FNF:")
+                or line.startswith("FNH:")
+                or line.startswith("BRF:")
+                or line.startswith("BRH:")
+                or line.startswith("LF:")
+                or line.startswith("LH:")
+            ):
+                pass
+            elif line.startswith("SF:"):
+                file_name = line
+            else:
+                match = EXTRACT_LINE.match(line)
+                if file_name and match:
+                    data_by_path[file_name].add(match.group(2))
+                else:
+                    raise NotImplementedError(line)
+        return data_by_path
+
+
+def filter_coverage_file(filename, output_file, data_by_path):
+    """Reads a coverage file from filename and writes filtered lines to
+    output_file.
+
+    For each line in filename, if it covers the same lines as the template
+    in data_by_path, then write the line to output_file.
+
+    Directives that act as totals (FNF, FNH, BRF, BRH, LF, LH) are recalculated
+    after filtering, and records that refer to unknown files are omitted.
+    """
+    logging.info("Merging file %s", filename)
+    with open(filename, "r") as input_file:
+
+        def empty_record():
+            return {
+                "text": "",
+                "function_names": set(),
+            }
+
+        record = empty_record()
+        for line in input_file.readlines():
+            line = line.strip()
+            if line == "end_of_record":
+                record["text"] += line + "\n"
+                if record.get("should_write_record", False):
+                    output_file.write(record["text"])
+                else:
+                    logging.debug("Omitting record %s", record["text"])
+                record = empty_record()
+            elif line.startswith("SF:"):
+                record["file_name"] = line
+                record["text"] += line + "\n"
+            elif line.startswith("TN:"):
+                record["text"] += line + "\n"
+            elif line.startswith("FN:"):
+                match = EXTRACT_FN.match(line)
+                if (
+                    match
+                    and match.group(2) in data_by_path[record["file_name"]]
+                ):
+                    record["text"] += line + "\n"
+                    record["functions_found"] = (
+                        record.get("functions_found", 0) + 1
+                    )
+                    record["should_write_record"] = True
+                    record["function_names"].add(match.group(3))
+                else:
+                    logging.debug("Omitting %s", line)
+            elif line.startswith("FNDA:"):
+                match = EXTRACT_FNDA.match(line)
+                if match and match.group(3) in record["function_names"]:
+                    record["text"] += line + "\n"
+                    record["should_write_record"] = True
+                    if match.group(2) != "0":
+                        record["functions_hit"] = (
+                            record.get("functions_hit", 0) + 1
+                        )
+                else:
+                    logging.debug("Omitting %s", line)
+            elif line.startswith("DA:"):
+                match = EXTRACT_DA.match(line)
+                if (
+                    match
+                    and match.group(2) in data_by_path[record["file_name"]]
+                ):
+                    record["text"] += line + "\n"
+                    record["lines_found"] = record.get("lines_found", 0) + 1
+                    record["should_write_record"] = True
+                    if match.group(3) != "0":
+                        record["lines_hit"] = record.get("lines_hit", 0) + 1
+                else:
+                    logging.debug("Omitting %s", line)
+            elif line.startswith("BRDA:"):
+                match = EXTRACT_BRDA.match(line)
+                if (
+                    match
+                    and match.group(2) in data_by_path[record["file_name"]]
+                ):
+                    record["text"] += line + "\n"
+                    record["branches_found"] = (
+                        record.get("branches_found", 0) + 1
+                    )
+                    record["should_write_record"] = True
+                    if match.group(4) != "-" and match.group(4) != "0":
+                        record["branches_hit"] = (
+                            record.get("branches_hit", 0) + 1
+                        )
+                else:
+                    logging.debug("Omitting %s", line)
+            elif line.startswith("FNF:"):
+                record["text"] += "FNF:%s\n" % record.get("functions_found", 0)
+            elif line.startswith("FNH:"):
+                record["text"] += "FNH:%s\n" % record.get("functions_hit", 0)
+            elif line.startswith("BRF:"):
+                record["text"] += "BRF:%s\n" % record.get("branches_found", 0)
+            elif line.startswith("BRH:"):
+                record["text"] += "BRH:%s\n" % record.get("branches_hit", 0)
+            elif line.startswith("LF:"):
+                record["text"] += "LF:%s\n" % record.get("lines_found", 0)
+            elif line.startswith("LH:"):
+                record["text"] += "LH:%s\n" % record.get("lines_hit", 0)
+            else:
+                logging.debug("record = %s", record)
+                raise NotImplementedError(line)
+
+
+def main(argv=None):
+    """Merges lcov files."""
+    opts = parse_args(argv)
+
+    output_file = sys.stdout
+    if opts.output_file:
+        logging.info("Writing output to %s", opts.output_file)
+        output_file = open(  # pylint:disable=consider-using-with
+            opts.output_file, "w"
+        )
+
+    data_by_path = parse_template_file(opts.template_file)
+    with output_file:
+        for lcov_input in [opts.template_file] + opts.lcov_input:
+            filter_coverage_file(lcov_input, output_file, data_by_path)
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv[1:]))
diff --git a/zephyr/firmware_builder.py b/zephyr/firmware_builder.py
index e6234f00d1..6081c774fa 100755
--- a/zephyr/firmware_builder.py
+++ b/zephyr/firmware_builder.py
@@ -440,16 +440,31 @@ def test(opts):
                 check=True,
                 stdin=subprocess.DEVNULL,
             )
+            # Filter to only code in the baseline board coverage
+            cmd = [
+                platform_ec / "util/lcov_stencil.py",
+                "-o",
+                build_dir / (board + "_stenciled.info"),
+                build_dir / board / "output/zephyr.info",
+                build_dir / (board + "_merged.info"),
+            ]
+            log_cmd(cmd)
+            subprocess.run(
+                cmd,
+                cwd=zephyr_dir,
+                check=True,
+                stdin=subprocess.DEVNULL,
+            )
             # Exclude file patterns we don't want
             cmd = (
                 [
                     "/usr/bin/lcov",
                     "-o",
-                    build_dir / (board + "_filtered.info"),
+                    build_dir / (board + "_final.info"),
                     "--rc",
                     "lcov_branch_coverage=1",
                     "-r",
-                    build_dir / (board + "_merged.info"),
+                    build_dir / (board + "_stenciled.info"),
                     # Exclude third_party code (specifically zephyr)
                     third_party / "**",
                     # These are questionable, but they are essentially untestable
@@ -470,26 +485,12 @@ def test(opts):
                 check=True,
                 stdin=subprocess.DEVNULL,
             )
-            # Then keep only files present in the board build
-            filenames = set()
-            with open(
-                build_dir / board / "output/zephyr.info", "r"
-            ) as board_cov:
-                for line in board_cov.readlines():
-                    if line.startswith("SF:"):
-                        filenames.add(line[3:-1])
-            cmd = [
-                "/usr/bin/lcov",
-                "-o",
-                build_dir / (board + "_final.info"),
-                "--rc",
-                "lcov_branch_coverage=1",
-                "-e",
-                build_dir / (board + "_filtered.info"),
-            ] + list(filenames)
-            log_cmd(cmd)
             output = subprocess.run(
-                cmd,
+                [
+                    "/usr/bin/lcov",
+                    "--summary",
+                    build_dir / (board + "_final.info"),
+                ],
                 cwd=zephyr_dir,
                 check=True,
                 stdout=subprocess.PIPE,