diff options
| -rw-r--r-- | .gitlab-ci.yml | 11 | ||||
| -rw-r--r-- | docs/code_coverage.md | 30 | ||||
| -rw-r--r-- | docs/zephyr/zephyr_shim.md | 2 | ||||
| -rw-r--r-- | docs/zephyr/ztest.md | 2 | ||||
| -rwxr-xr-x | zephyr/firmware_builder.py | 62 | ||||
| -rw-r--r-- | zephyr/test/drivers/README.md | 6 | ||||
| -rw-r--r-- | zephyr/zmake/README.md | 14 | ||||
| -rw-r--r-- | zephyr/zmake/zmake/__main__.py | 16 | ||||
| -rw-r--r-- | zephyr/zmake/zmake/zmake.py | 162 |
9 files changed, 171 insertions, 134 deletions
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index dbfdd52c5f..0f59495070 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -226,12 +226,11 @@ zephyr_coverage: needs: ["seed_cache"] script: - zmake --zephyr-base "${ZEPHYR_BASE}" - --modules-dir "${MODULES_DIR}" -l DEBUG coverage - "${BUILD_DIR}/zcoverage" + --modules-dir "${MODULES_DIR}" -l DEBUG test + --coverage --host-tests-only artifacts: paths: - - build/zcoverage/coverage_rpt/* - - build/zcoverage/*.info + - build/zephyr/all_tests.info expire_in: 1 week coverage: '/lines\.*: \d+\.\d+%/' @@ -239,7 +238,7 @@ merged_coverage: stage: test needs: ["ec_coverage", "zephyr_coverage"] script: - - lcov --rc lcov_branch_coverage=1 -o build/merged.info -a build/coverage/lcov.info -a build/zcoverage/lcov.info + - lcov --rc lcov_branch_coverage=1 -o build/merged.info -a build/coverage/lcov.info -a build/zephyr/all_tests.info - lcov --rc lcov_branch_coverage=1 -o build/merged_no_zephyr.info -r build/merged.info "${ZEPHYR_BASE}/**" "${MODULES_DIR}/**" "${EC_DIR}/zephyr/drivers/**" "${EC_DIR}/zephyr/include/drivers/**" @@ -258,7 +257,7 @@ testall: needs: ["seed_cache"] script: - zmake --zephyr-base "${ZEPHYR_BASE}" - --modules-dir "${MODULES_DIR}" -l DEBUG testall + --modules-dir "${MODULES_DIR}" -l DEBUG test --all twister_coverage: stage: test diff --git a/docs/code_coverage.md b/docs/code_coverage.md index 93637ac019..a15c61b358 100644 --- a/docs/code_coverage.md +++ b/docs/code_coverage.md @@ -38,20 +38,28 @@ appear to be caused in part by using relative paths instead of absolute paths.) To build the Zephyr unit tests for code coverage run: -`zmake coverage build/ztest-coverage` +`zmake test --host-tests-only --coverage` +`genhtml -q -o build/zephyr/coverage_rpt/ build/zephyr/all_tests.info` -This target will compile, without linking, all zephyr projects with -`CONFIG_COVERAGE` Kconfig option enabled, run the tests, and then process the -profiling data into a code coverage report using the `lcov` and `genhtml` -tools. This requires the `HAS_COVERAGE_SUPPORT` option, which can only be -selected in `Kconfig.board`. +The coverage report top-level page is +`build/zephyr/coverage_rpt/index.html`. + +However you probably want to merge that with a single board's coverage report +also, so that you can include code that is not part of any test as well. + +``` +zmake build --coverage herobrine +zmake test --host-tests-only --coverage +genhtml -q -s --branch-coverage -o build/zephyr/coverage_rpt/ \ + build/zephyr/all_tests.info build/zephyr/herobrine/output/zephyr.info +``` The coverage report top-level page is -`build/ztest-coverage/coverage_rpt/index.html`. +`build/zephyr/coverage_rpt/index.html`. -For manual coverage report you can run: -`zmake configure --test --coverage <TESTNAME>` +For coverage report for a single test you can run: +`zmake test --coverage <TESTNAME>` Example: -`zmake configure --test --coverage test-drivers` -`genhtml -q -o build/ztest-coverage/coverage_rpt/ build/zephyr/test-drivers/output/zephyr.info` +`zmake test --coverage test-drivers` +`genhtml -q -o build/zephyr/test-drivers/output/coverage_rpt/ build/zephyr/test-drivers/output/zephyr.info` diff --git a/docs/zephyr/zephyr_shim.md b/docs/zephyr/zephyr_shim.md index fac0383aff..3c6b48f209 100644 --- a/docs/zephyr/zephyr_shim.md +++ b/docs/zephyr/zephyr_shim.md @@ -347,7 +347,7 @@ Follow these steps: Unit tests, implemented using the Ztest framework, can be found in `zephyr/test`. -To run all unit tests, you use `zmake testall`. +To build all projects and run all unit tests, you use `zmake test --all`. ## Alternatives Considered diff --git a/docs/zephyr/ztest.md b/docs/zephyr/ztest.md index 4a33d0a103..6bb121045d 100644 --- a/docs/zephyr/ztest.md +++ b/docs/zephyr/ztest.md @@ -178,7 +178,7 @@ uses. Use `zmake` to build and run the test: ``` -(cr) $ zmake -l DEBUG configure --test -B build/ztest/base32 test-base32 +(cr) $ zmake -l DEBUG test test-base32 ... UART_0 connected to pseudotty: /dev/pts/1 *** Booting Zephyr OS build zephyr-v2.4.0-1-g63b2330a85cd *** diff --git a/zephyr/firmware_builder.py b/zephyr/firmware_builder.py index ae19b82c81..2b138357f3 100755 --- a/zephyr/firmware_builder.py +++ b/zephyr/firmware_builder.py @@ -31,10 +31,10 @@ def build(opts): with open(opts.metrics, 'w') as f: f.write(json_format.MessageToJson(metrics)) - # Nothing to do, as the test phase actually does the builds. - # TODO(b/217788621): Do a build-only here once we can separate build - # and test phases on zmake CLI. - return 0 + cmd = ['zmake', '-D', 'build', '-a'] + if opts.code_coverage: + cmd.append('--coverage') + return subprocess.run(cmd, cwd=pathlib.Path(__file__).parent).returncode def bundle(opts): @@ -71,27 +71,7 @@ def bundle_coverage(opts): bundle_dir = get_bundle_dir(opts) zephyr_dir = pathlib.Path(__file__).parent platform_ec = zephyr_dir.resolve().parent - # Find the zephyr.info for every project and merge them - all_lcov_files = [platform_ec / 'build' / 'zephyr-coverage' / 'lcov.info'] - for project in zmake.project.find_projects(zephyr_dir).values(): - if not project.config.is_test: - build_dir = platform_ec / "build" / "zephyr" / project.config.project_name - artifacts_dir = build_dir / 'output' - all_lcov_files.append(artifacts_dir / 'zephyr.info') - build_dir = platform_ec / "build" - print("all_lcov_files = %s" % all_lcov_files) - cmd = [ - "/usr/bin/lcov", - "-o", - build_dir / "lcov.info", - "--rc", - "lcov_branch_coverage=1", - ] - for lcov_file in all_lcov_files: - cmd += ["-a", lcov_file] - rv = subprocess.run(cmd, cwd=pathlib.Path(__file__).parent).returncode - if rv != 0: - return rv + build_dir = platform_ec / "build" / "zephyr" tarball_name = 'coverage.tbz2' tarball_path = bundle_dir / tarball_name cmd = ['tar', 'cvfj', tarball_path, 'lcov.info'] @@ -112,12 +92,10 @@ def bundle_firmware(opts): zephyr_dir = pathlib.Path(__file__).parent platform_ec = zephyr_dir.resolve().parent for project in zmake.project.find_projects(zephyr_dir).values(): + if project.config.is_test: + continue build_dir = platform_ec / "build" / "zephyr" / project.config.project_name artifacts_dir = build_dir / 'output' - # TODO(kmshelton): Remove once the build command does not rely - # on a pre-defined list of targets. - if not artifacts_dir.is_dir(): - continue tarball_name = '{}.firmware.tbz2'.format(project.config.project_name) tarball_path = bundle_dir.joinpath(tarball_name) cmd = ['tar', 'cvfj', tarball_path, '.'] @@ -149,13 +127,29 @@ def test(opts): config_files = zephyr_dir.rglob("**/BUILD.py") subprocess.run(["black", "--diff", "--check", *config_files], check=True) + cmd = ['zmake', '-D', 'test', '-a', '--no-rebuild'] + if opts.code_coverage: + cmd.append('--coverage') + rv = subprocess.run(cmd, check=True).returncode + if rv: + return rv if opts.code_coverage: platform_ec = zephyr_dir.parent - build_dir = platform_ec / 'build/zephyr-coverage' - return subprocess.run( - ['zmake', '-D', 'coverage', build_dir], cwd=platform_ec).returncode - else: - return subprocess.run(['zmake', '-D', 'testall'], check=True).returncode + build_dir = platform_ec / "build" / "zephyr" + # Merge lcov files here because bundle failures are "infra" failures. + cmd = [ + "/usr/bin/lcov", + "-o", + build_dir / "lcov.info", + "--rc", + "lcov_branch_coverage=1", + "-a", build_dir / 'all_tests.info', + "-a", build_dir / 'all_builds.info', + ] + rv = subprocess.run(cmd, cwd=pathlib.Path(__file__).parent).returncode + if rv != 0: + return rv + return 0 def main(args): diff --git a/zephyr/test/drivers/README.md b/zephyr/test/drivers/README.md index cdde4d4f6c..11c913baa8 100644 --- a/zephyr/test/drivers/README.md +++ b/zephyr/test/drivers/README.md @@ -5,13 +5,13 @@ so we can test interactions between different subsystems easily. ## Run all the test suites ```bash -(chroot) zmake configure --test test-drivers +(chroot) zmake test test-drivers ``` To see all the output of zmake (for example if the build fails) ```bash -(chroot) zmake -l DEBUG -j 1 configure --test test-drivers +(chroot) zmake -l DEBUG -j 1 test test-drivers ``` ## Code coverage @@ -19,7 +19,7 @@ To see all the output of zmake (for example if the build fails) To calculate code coverage for this test only ```bash -(chroot) zmake configure --coverage --test test-drivers +(chroot) zmake test --coverage test-drivers (chroot) genhtml --branch-coverage -q \ -o build/zephyr/test-drivers/output/coverage_rpt \ build/zephyr/test-drivers/output/zephyr.info diff --git a/zephyr/zmake/README.md b/zephyr/zmake/README.md index 97ff4a434f..fb7d4b8414 100644 --- a/zephyr/zmake/README.md +++ b/zephyr/zmake/README.md @@ -102,20 +102,28 @@ Chromium OS's meta-build tool for Zephyr ### zmake test -**Usage:** `zmake test [-h] [-c] build_dir` +**Usage:** `zmake test [-h] [--no-rebuild] [-t TOOLCHAIN] [--bringup] [--clobber] [--allow-warnings] [-B BUILD_DIR] [-c] (-a | --host-tests-only | project_name [project_name ...])` #### Positional Arguments | | | |---|---| -| `build_dir` | The build directory used during configuration | +| `project_name` | Name(s) of the project(s) to build | #### Optional Arguments | | | |---|---| | `-h`, `--help` | show this help message and exit | -| `-c`, `--coverage` | Run lcov after running test to generate coverage info file. | +| `--no-rebuild` | Do not configure or build before running tests. | +| `-t TOOLCHAIN`, `--toolchain TOOLCHAIN` | Name of toolchain to use | +| `--bringup` | Enable bringup debugging features | +| `--clobber` | Delete existing build directories, even if configuration is unchanged | +| `--allow-warnings` | Do not treat warnings as errors | +| `-B BUILD_DIR`, `--build-dir BUILD_DIR` | Root build directory, project files will be in ${build_dir}/${project_name} | +| `-c`, `--coverage` | Enable CONFIG_COVERAGE Kconfig. | +| `-a`, `--all` | Select all projects | +| `--host-tests-only` | Select all test projects | ### zmake testall diff --git a/zephyr/zmake/zmake/__main__.py b/zephyr/zmake/zmake/__main__.py index 14f47bc462..c39a645a81 100644 --- a/zephyr/zmake/zmake/__main__.py +++ b/zephyr/zmake/zmake/__main__.py @@ -222,24 +222,18 @@ def get_argparser(): test = sub.add_parser( "test", - help="Execute tests from a build directory", + help="Configure, build and run tests on specified projects", ) test.add_argument( - "-c", - "--coverage", + "--no-rebuild", action="store_true", - dest="coverage", - help="Run lcov after running test to generate coverage info file.", - ) - test.add_argument( - "build_dir", - type=pathlib.Path, - help="The build directory used during configuration", + help="Do not configure or build before running tests.", ) + add_common_configure_args(test) testall = sub.add_parser( "testall", - help="Execute all known builds and tests", + help="Alias for test --all", ) testall.add_argument( "--clobber", diff --git a/zephyr/zmake/zmake/zmake.py b/zephyr/zmake/zmake/zmake.py index f697075c59..585726f5af 100644 --- a/zephyr/zmake/zmake/zmake.py +++ b/zephyr/zmake/zmake/zmake.py @@ -11,6 +11,7 @@ import pathlib import re import shutil import subprocess +from typing import List import zmake.build_config import zmake.generate_readme @@ -192,7 +193,7 @@ class Zmake: def _resolve_projects( self, project_names, all_projects=False, host_tests_only=False - ): + ) -> List[zmake.project.Project]: """Finds all projects for the specified command line flags. Returns a list of projects. @@ -256,17 +257,19 @@ class Zmake: rv = self.executor.wait() if rv: return rv - if len(projects) > 1 and coverage and test_after_configure: + test_projects = [p for p in projects if p.config.is_test] + if len(test_projects) > 1 and coverage and test_after_configure: rv = self._merge_lcov_files( - projects=[p for p in projects if p.config.is_test], + projects=test_projects, build_dir=build_dir, output_file=build_dir / "all_tests.info", ) if rv: return rv - if len(projects) > 1 and coverage and build_after_configure: + non_test_projects = [p for p in projects if not p.config.is_test] + if len(non_test_projects) > 1 and coverage and build_after_configure: rv = self._merge_lcov_files( - projects=[p for p in projects if not p.config.is_test], + projects=non_test_projects, build_dir=build_dir, output_file=build_dir / "all_builds.info", ) @@ -300,6 +303,95 @@ class Zmake: build_after_configure=True, ) + def test( + self, + project_names, + build_dir=None, + toolchain=None, + clobber=False, + bringup=False, + coverage=False, + allow_warnings=False, + all_projects=False, + host_tests_only=False, + no_rebuild=False, + ): + """Locate and build the specified projects.""" + if not no_rebuild: + return self.configure( + project_names, + build_dir=build_dir, + toolchain=toolchain, + clobber=clobber, + bringup=bringup, + coverage=coverage, + allow_warnings=allow_warnings, + all_projects=all_projects, + host_tests_only=host_tests_only, + test_after_configure=True, + ) + # Resolve build_dir if needed. + if not build_dir: + build_dir = self.module_paths["ec"] / "build" / "zephyr" + + projects = self._resolve_projects( + project_names, all_projects=all_projects, host_tests_only=host_tests_only + ) + test_projects = [p for p in projects if p.config.is_test] + for project in test_projects: + project_build_dir = pathlib.Path(build_dir) / project.config.project_name + gcov = "gcov.sh-not-found" + for build_name, _ in project.iter_builds(): + target_build_dir = project_build_dir / "build-{}".format(build_name) + gcov = target_build_dir / "gcov.sh" + self.executor.append( + func=functools.partial( + self._run_test, + elf_file=project_build_dir / "output" / "zephyr.elf", + coverage=coverage, + gcov=gcov, + build_dir=project_build_dir, + lcov_file=project_build_dir / "output" / "zephyr.info", + timeout=project.config.test_timeout_secs, + ) + ) + if self._sequential: + rv = self.executor.wait() + if rv: + return rv + rv = self.executor.wait() + if rv: + return rv + if len(test_projects) > 1 and coverage: + rv = self._merge_lcov_files( + projects=test_projects, + build_dir=build_dir, + output_file=build_dir / "all_tests.info", + ) + if rv: + return rv + return 0 + + def testall( + self, + build_dir=None, + toolchain=None, + clobber=False, + bringup=False, + coverage=False, + allow_warnings=False, + ): + return self.test( + [], + build_dir=build_dir, + toolchain=toolchain, + clobber=clobber, + bringup=bringup, + coverage=coverage, + allow_warnings=allow_warnings, + all_projects=True, + ) + def _configure( self, project, @@ -499,7 +591,7 @@ class Zmake: def _build( self, build_dir, - project, + project: zmake.project.Project, output_files_out=None, fail_on_warnings=False, coverage=False, @@ -693,64 +785,6 @@ class Zmake: with self.jobserver.get_job(): _run() - def test(self, build_dir, coverage=False): - """Test a build directory.""" - output_files = [] - build_dir = build_dir.resolve() - found_projects = zmake.project.find_projects(build_dir / "project") - project = found_projects[(build_dir / "project_name.txt").read_text()] - self._build( - build_dir, project=project, output_files_out=output_files, coverage=coverage - ) - - # If the project built but isn't a test, just bail. - if not project.config.is_test: - return 0 - - gcov = "gcov.sh-not-found" - if coverage: - for build_name, _ in project.iter_builds(): - gcov = build_dir / "build-{}".format(build_name) / "gcov.sh" - - for output_file in output_files: - self.executor.append( - func=functools.partial( - self._run_test, - elf_file=output_file, - coverage=coverage, - gcov=gcov, - build_dir=build_dir, - lcov_file=build_dir / "output" / "zephyr.info", - timeout=project.config.test_timeout_secs, - ) - ) - - return 0 - - def testall(self, build_dir=None, clobber=False): - """Test all the valid test targets""" - for project in zmake.project.find_projects( - self.module_paths["ec"] / "zephyr" - ).values(): - is_test = project.config.is_test - if build_dir: - project_build_dir = build_dir / project.config.project_name - else: - project_build_dir = None - # Configure and run the test. - self.executor.append( - func=functools.partial( - self._configure, - project=project, - build_dir=project_build_dir, - build_after_configure=True, - test_after_configure=is_test, - clobber=clobber, - ) - ) - - return self.executor.wait() - def _run_lcov(self, build_dir, lcov_file, initial=False, gcov=""): gcov = os.path.abspath(gcov) if initial: |