Improve unit test and fuzzer documentation story

* Update **test** directory description to README.md.
* Add ** fuzz** directory description to README.md.
* Add unit test and fuzzer make targets to `make help`
* Change showboards to print-boards to be more consistent
* Change showboards to use auto pretty print
* Add print-tests, print-host-tests, and print-host-fuzzers
  This is necessary to remove the ambiguity about what a special
  name is for a given unit test.

Documentation Story:
The idea is to give a brief overview of what the test and fuzz
directories are in README.md.
README.md also mentions you should see `make help` for more detail.
Running `make help` shows you more general test/fuzz commands,
including the print-* commands.
Running the print-* commands show you exact target names for all
possible unit/fuzz test (both the build-only and run target names).

BRANCH=none
BUG=none
TEST=make help
TEST=make print-tests
TEST=make print-host-tests
TEST=make print-host-fuzzers
TEST=make print-host-fuzzers | cat
TEST=make print-boards
TEST=make print-boards | cat
TEST=make buildall -j

Change-Id: I34b68196ac635ba71a1d45ceb5d35a3b36fd129f
Signed-off-by: Craig Hesling <hesling@chromium.org>
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/platform/ec/+/1684714
Reviewed-by: Aseda Aboagye <aaboagye@chromium.org>
Reviewed-by: Daisuke Nojiri <dnojiri@chromium.org>

3 files changed, 56 insertions, 8 deletions

diff --git a/Makefile.rules b/Makefile.rules
index d6280c098f..be651c9d1d 100644
--- a/Makefile.rules
+++ b/Makefile.rules
@@ -146,6 +146,14 @@ cmd_etags = etags -o $@ $(shell cat $<)
 cmd_ctags = ctags -o $@ $(shell cat $<)
 targ_if_prog = $(if $(shell which $(1) 2>/dev/null),$(2),)
 
+# If outputing to tty and column command exists, pipe into column.
+# Otherwise, print in newline format.
+cmd_pretty_print_list = @echo $(1) \
+				| tr ' ' '\n'                    \
+				| { [ -t 1 ]                     \
+				    && which column &>/dev/null  \
+				    && column || cat ; }         \
+
 # By default, the "build_boards" and "try_build_boards" targets will build all
 # of the boards listed in $(boards). However, the invoker can provide a
 # different list via the BOARDS variable. Providing an empty value for BOARDS
@@ -192,8 +200,10 @@ buildalltests: | $(FAILED_BOARDS_DIR)
 	$(MAKE) try_build_tests
 	@echo "$@ completed successfully!"
 
-showboards:
-	@echo $(sort $(boards))
+.PHONY: print-boards
+print-boards:
+	$(call cmd_pretty_print_list,\
+		$(sort $(boards)))
 
 # Print any important notices at the end of the build.
 .PHONY: notice
@@ -255,6 +265,11 @@ endif
 .PHONY: tests
 tests: $(test-targets)
 
+.PHONY: print-tests
+print-tests:
+	$(call cmd_pretty_print_list, \
+		$(sort $(test-targets)))
+
 # Emulator test executables
 host-test-targets=$(foreach t,$(test-list-host),host-$(t))
 run-test-targets=$(foreach t,$(test-list-host),run-$(t))
@@ -268,6 +283,11 @@ $(run-test-targets): run-%: host-%
 	$(call quiet,run_host_test,TEST   )
 	@rm -f $(FAILED_BOARDS_DIR)/test-$*
 
+.PHONY: print-host-tests
+print-host-tests:
+	$(call cmd_pretty_print_list, \
+		$(sort $(host-test-targets) $(run-test-targets)))
+
 # Fuzzing tests
 
 fuzz-test-targets=$(foreach t,$(fuzz-test-list-host),host-$(t))
@@ -289,6 +309,11 @@ $(run-fuzz-test-targets): run-%: host-%
 $(FAILED_BOARDS_DIR):
 	@mkdir $(FAILED_BOARDS_DIR)
 
+.PHONY: print-host-fuzzers
+print-host-fuzzers:
+	$(call cmd_pretty_print_list, \
+		$(sort $(fuzz-test-targets) $(run-fuzz-test-targets)))
+
 .PHONY: buildfuzztests
 buildfuzztests: $(fuzz-test-targets)
 
@@ -650,6 +675,19 @@ help:
 	@echo "  proj-<boardname>     - Build a single board (similar to 'all BOARD=boardname')"
 	@echo "  savesizes            - Save the filesizes of currently built boards for comparison"
 	@echo "  newsizes             - Compare previously saved filesizes against new sizes"
+	@echo ""
+	@echo "  tests [BOARD=]       - Build all unit tests for a specific board"
+	@echo "  hosttests            - Build all host unit tests"
+	@echo "  runhosttests         - Build and run all host unit tests"
+	@echo "  buildfuzztests       - Build all host fuzzers"
+	@echo "  runfuzztests         - Build and run all host fuzzers for one round"
+	@echo ""
+	@echo "  print-tests          - Prints all device specific test targets"
+	@echo "  print-host-tests     - Prints all host test targets"
+	@echo "  print-host-fuzzers   - Prints all host fuzz targets"
+	@echo "  print-boards         - Prints all boards"
+	@echo "  print-chip-variant [BOARD=] - Print chip variant used for the target board"
+	@echo "  print-configs [BOARD=]      - Print CONFIG_* options for the target board"
 	@echo "Common Variables:"
 	@echo "  V=1                  - Show make output"
 	@echo "  BOARD=               - Set the board name to build (Default is $(BOARD))"
diff --git a/README.md b/README.md
index 2d37f835f7..05d0a3e151 100644
--- a/README.md
+++ b/README.md
@@ -77,9 +77,21 @@ I2C/onewire LED controllers, and I2C temperature sensors.
 **util** - Host utilities and scripts for flashing the EC. Also includes
 “ectool” used to query and send commands to the EC from userspace.
 
-**test** - Unit tests for the EC. Use “make tests -j $jobs BOARD=$board” to run
-them against your build target. Set $jobs to the number of cores in your build
-machine. Please contribute new tests if writing new functionality.
+**test** - Unit tests for EC components. These can be run locally in
+           a mock "host" environment or compiled for a target board.
+           If building for a target board, the test must be flashed and
+           run manually on the device.
+           All unit tests and fuzzers are build/run using the local
+           host environment during a `buildall`.
+           To run all unit tests locally, run `make runhosttests -j`.
+           To build a specific unit test for a specific board, run
+           `make test-<test_name> BOARD=<board_name>`.
+           Please contribute new tests if writing new functionality.
+           Please run `make help` for more detail.
+
+**fuzz** - Fuzzers for EC components. These fuzzers are expected to
+           run in the mock host environment. They follow the same rules
+           as unit tests, as thus use the same commands to build and run.
 
 ## Firmware Branches
 
diff --git a/test/build.mk b/test/build.mk
index 1e164e798c..ba57220e29 100644
--- a/test/build.mk
+++ b/test/build.mk
@@ -2,10 +2,8 @@
 # Copyright 2013 The Chromium OS Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
-#
-# on-board test binaries build
-#
 
+# Device test binaries
 test-list-y ?= pingpong timer_calib timer_dos timer_jump mutex utils utils_str
 #disable: powerdemo