zmake: Add flags to build boards and delete tmps

The gitlab builder is running out of disk space.  Add flags to build
fewer things, and to delete the intermediate files and only leave the
output files.

Before this change zmake build of all boards took 2.1G, and 81M
afterwards.

BRANCH=None
BUG=None
TEST=make clobber
TEST=zmake build --boards-only -c --delete-intermediates
TEST=du -s -h build

Change-Id: I507abff512ca1348681a1fd7431d5267efaf5d65
Signed-off-by: Jeremy Bettis <jbettis@google.com>
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/platform/ec/+/3726959
Commit-Queue: Jack Rosenthal <jrosenth@chromium.org>
Auto-Submit: Jeremy Bettis <jbettis@chromium.org>
Tested-by: Jeremy Bettis <jbettis@chromium.org>
Reviewed-by: Jack Rosenthal <jrosenth@chromium.org>

3 files changed, 58 insertions, 16 deletions

diff --git a/zephyr/zmake/README.md b/zephyr/zmake/README.md
index 6e2690959b..a329396038 100644
--- a/zephyr/zmake/README.md
+++ b/zephyr/zmake/README.md
@@ -35,7 +35,7 @@ Chromium OS's meta-build tool for Zephyr
 
 ### zmake configure
 
-**Usage:** `zmake configure [-h] [-t TOOLCHAIN] [--bringup] [--clobber] [--allow-warnings] [-B BUILD_DIR] [-c] [--extra-cflags EXTRA_CFLAGS] (-a | --host-tests-only | project_name [project_name ...])`
+**Usage:** `zmake configure [-h] [-t TOOLCHAIN] [--bringup] [--clobber] [--allow-warnings] [-B BUILD_DIR] [-c] [--extra-cflags EXTRA_CFLAGS] [--delete-intermediates] (-a | --host-tests-only | --boards-only | project_name [project_name ...])`
 
 #### Positional Arguments
 
@@ -55,12 +55,14 @@ Chromium OS's meta-build tool for Zephyr
 | `-B BUILD_DIR`, `--build-dir BUILD_DIR` | Root build directory, project files will be in ${build_dir}/${project_name} |
 | `-c`, `--coverage` | Enable CONFIG_COVERAGE Kconfig. |
 | `--extra-cflags EXTRA_CFLAGS` | Additional CFLAGS to use for target builds |
+| `--delete-intermediates` | Delete intermediate files to save disk space |
 | `-a`, `--all` | Select all projects |
 | `--host-tests-only` | Select all test projects |
+| `--boards-only` | Select all board projects (not tests) |
 
 ### zmake build
 
-**Usage:** `zmake build [-h] [-t TOOLCHAIN] [--bringup] [--clobber] [--allow-warnings] [-B BUILD_DIR] [-c] [--extra-cflags EXTRA_CFLAGS] (-a | --host-tests-only | project_name [project_name ...])`
+**Usage:** `zmake build [-h] [-t TOOLCHAIN] [--bringup] [--clobber] [--allow-warnings] [-B BUILD_DIR] [-c] [--extra-cflags EXTRA_CFLAGS] [--delete-intermediates] (-a | --host-tests-only | --boards-only | project_name [project_name ...])`
 
 #### Positional Arguments
 
@@ -80,8 +82,10 @@ Chromium OS's meta-build tool for Zephyr
 | `-B BUILD_DIR`, `--build-dir BUILD_DIR` | Root build directory, project files will be in ${build_dir}/${project_name} |
 | `-c`, `--coverage` | Enable CONFIG_COVERAGE Kconfig. |
 | `--extra-cflags EXTRA_CFLAGS` | Additional CFLAGS to use for target builds |
+| `--delete-intermediates` | Delete intermediate files to save disk space |
 | `-a`, `--all` | Select all projects |
 | `--host-tests-only` | Select all test projects |
+| `--boards-only` | Select all board projects (not tests) |
 
 ### zmake list-projects
 
@@ -102,7 +106,7 @@ Chromium OS's meta-build tool for Zephyr
 
 ### zmake test
 
-**Usage:** `zmake test [-h] [--no-rebuild] [-t TOOLCHAIN] [--bringup] [--clobber] [--allow-warnings] [-B BUILD_DIR] [-c] [--extra-cflags EXTRA_CFLAGS] (-a | --host-tests-only | project_name [project_name ...])`
+**Usage:** `zmake test [-h] [--no-rebuild] [-t TOOLCHAIN] [--bringup] [--clobber] [--allow-warnings] [-B BUILD_DIR] [-c] [--extra-cflags EXTRA_CFLAGS] [--delete-intermediates] (-a | --host-tests-only | --boards-only | project_name [project_name ...])`
 
 #### Positional Arguments
 
@@ -123,8 +127,10 @@ Chromium OS's meta-build tool for Zephyr
 | `-B BUILD_DIR`, `--build-dir BUILD_DIR` | Root build directory, project files will be in ${build_dir}/${project_name} |
 | `-c`, `--coverage` | Enable CONFIG_COVERAGE Kconfig. |
 | `--extra-cflags EXTRA_CFLAGS` | Additional CFLAGS to use for target builds |
+| `--delete-intermediates` | Delete intermediate files to save disk space |
 | `-a`, `--all` | Select all projects |
 | `--host-tests-only` | Select all test projects |
+| `--boards-only` | Select all board projects (not tests) |
 
 ### zmake testall
 
diff --git a/zephyr/zmake/zmake/__main__.py b/zephyr/zmake/zmake/__main__.py
index 23be20d54f..7054601ac1 100644
--- a/zephyr/zmake/zmake/__main__.py
+++ b/zephyr/zmake/zmake/__main__.py
@@ -288,6 +288,12 @@ def add_common_configure_args(sub_parser: argparse.ArgumentParser):
         "--extra-cflags",
         help="Additional CFLAGS to use for target builds",
     )
+    sub_parser.add_argument(
+        "--delete-intermediates",
+        action="store_true",
+        dest="delete_intermediates",
+        help="Delete intermediate files to save disk space",
+    )
     group = sub_parser.add_mutually_exclusive_group(required=True)
     group.add_argument(
         "-a",
@@ -303,6 +309,12 @@ def add_common_configure_args(sub_parser: argparse.ArgumentParser):
         help="Select all test projects",
     )
     group.add_argument(
+        "--boards-only",
+        action="store_true",
+        dest="boards_only",
+        help="Select all board projects (not tests)",
+    )
+    group.add_argument(
         "project_names",
         nargs="*",
         metavar="project_name",
diff --git a/zephyr/zmake/zmake/zmake.py b/zephyr/zmake/zmake/zmake.py
index d4af57a738..69c4239f69 100644
--- a/zephyr/zmake/zmake/zmake.py
+++ b/zephyr/zmake/zmake/zmake.py
@@ -198,7 +198,11 @@ class Zmake:
         return self._checkout.resolve()
 
     def _resolve_projects(
-        self, project_names, all_projects=False, host_tests_only=False
+        self,
+        project_names,
+        all_projects=False,
+        host_tests_only=False,
+        boards_only=False,
     ) -> Set[zmake.project.Project]:
         """Finds all projects for the specified command line flags.
 
@@ -209,6 +213,8 @@ class Zmake:
             projects = set(found_projects.values())
         elif host_tests_only:
             projects = {p for p in found_projects.values() if p.config.is_test}
+        elif boards_only:
+            projects = {p for p in found_projects.values() if not p.config.is_test}
         else:
             projects = set()
             for project_name in project_names:
@@ -232,6 +238,8 @@ class Zmake:
         all_projects=False,
         host_tests_only=False,
         extra_cflags=None,
+        delete_intermediates=False,
+        boards_only=False,
     ):
         """Locate and configure the specified projects."""
         # Resolve build_dir if needed.
@@ -242,6 +250,7 @@ class Zmake:
             project_names,
             all_projects=all_projects,
             host_tests_only=host_tests_only,
+            boards_only=boards_only,
         )
         for project in projects:
             project_build_dir = pathlib.Path(build_dir) / project.config.project_name
@@ -259,6 +268,7 @@ class Zmake:
                     allow_warnings=allow_warnings,
                     extra_cflags=extra_cflags,
                     multiproject=len(projects) > 1,
+                    delete_intermediates=delete_intermediates,
                 )
             )
             if self._sequential:
@@ -302,6 +312,8 @@ class Zmake:
         all_projects=False,
         host_tests_only=False,
         extra_cflags=None,
+        delete_intermediates=False,
+        boards_only=False,
     ):
         """Locate and build the specified projects."""
         return self.configure(
@@ -316,6 +328,8 @@ class Zmake:
             host_tests_only=host_tests_only,
             extra_cflags=extra_cflags,
             build_after_configure=True,
+            delete_intermediates=delete_intermediates,
+            boards_only=boards_only,
         )
 
     def test(  # pylint: disable=too-many-arguments,too-many-locals
@@ -331,6 +345,8 @@ class Zmake:
         host_tests_only=False,
         extra_cflags=None,
         no_rebuild=False,
+        delete_intermediates=False,
+        boards_only=False,
     ):
         """Locate and build the specified projects."""
         if not no_rebuild:
@@ -346,6 +362,8 @@ class Zmake:
                 host_tests_only=host_tests_only,
                 extra_cflags=extra_cflags,
                 test_after_configure=True,
+                delete_intermediates=delete_intermediates,
+                boards_only=boards_only,
             )
         # Resolve build_dir if needed.
         if not build_dir:
@@ -355,6 +373,7 @@ class Zmake:
             project_names,
             all_projects=all_projects,
             host_tests_only=host_tests_only,
+            boards_only=boards_only,
         )
         test_projects = [p for p in projects if p.config.is_test]
         for project in test_projects:
@@ -416,7 +435,7 @@ class Zmake:
     def _configure(
         self,
         project,
-        build_dir=None,
+        build_dir: pathlib.Path,
         toolchain=None,
         build_after_configure=False,
         test_after_configure=False,
@@ -426,20 +445,12 @@ class Zmake:
         allow_warnings=False,
         extra_cflags=None,
         multiproject=False,
+        delete_intermediates=False,
     ):
         # pylint: disable=too-many-arguments,too-many-locals,too-many-branches
         # pylint: disable=too-many-statements
         """Set up a build directory to later be built by "zmake build"."""
         try:
-            # Resolve build_dir if needed.
-            if not build_dir:
-                build_dir = (
-                    self.module_paths["ec"]
-                    / "build"
-                    / "zephyr"
-                    / project.config.project_name
-                )
-
             # Clobber build directory if requested.
             if clobber and build_dir.exists():
                 self.logger.info(
@@ -501,7 +512,7 @@ class Zmake:
                 )
 
             if not build_dir.exists():
-                build_dir = build_dir.mkdir()
+                build_dir.mkdir()
             if not generated_include_dir.exists():
                 generated_include_dir.mkdir()
             processes = []
@@ -617,6 +628,16 @@ class Zmake:
                         timeout=project.config.test_timeout_secs,
                     )
                 )
+
+            if delete_intermediates:
+                outdir = build_dir / "output"
+                for child in build_dir.iterdir():
+                    if child != outdir:
+                        logging.debug("Deleting %s", child)
+                        if not child.is_symlink() and child.is_dir():
+                            shutil.rmtree(child)
+                        else:
+                            child.unlink()
             return 0
         except Exception:
             self.failed_projects.append(project.config.project_name)
@@ -711,7 +732,10 @@ class Zmake:
                 )
                 job_id = "{}:{}".format(project.config.project_name, build_name)
                 dirs[build_name].mkdir(parents=True, exist_ok=True)
-                build_log = open(dirs[build_name] / "build.log", "w")
+                build_log = open(  # pylint:disable=consider-using-with
+                    dirs[build_name] / "build.log",
+                    "w",
+                )
                 out = zmake.multiproc.LogWriter.log_output(
                     logger=self.logger,
                     log_level=logging.INFO,