run_device_tests: Add option to test with production app image

Add the ability to run tests against the production application
image. It should be noted that the production image does not support the
"runtest" command so any tests using a production image must supply
appropriate commands using pre/post test callbacks (to be added in a
future CL) and/or supplying finish_regexes. Also add a test to verify
this functionality. Tests using the standard application image should
not have a target buildable via make, as they use the standard app)
but may be run from run_device_tests.py.

BUG=b:180445334
BRANCH=none
TEST=run_device_tests.py -b dartmonkey

Signed-off-by: Bobby Casey <bobbycasey@google.com>
Change-Id: I76cc248775a4cafdfcc4278799156a291277a19b
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/platform/ec/+/4193835
Reviewed-by: Tom Hughes <tomhughes@chromium.org>
Reviewed-by: Andrea Grandi <agrandi@google.com>

1 files changed, 53 insertions, 16 deletions

diff --git a/test/run_device_tests.py b/test/run_device_tests.py
index a85fa94da4..639ba11ff8 100755
--- a/test/run_device_tests.py
+++ b/test/run_device_tests.py
@@ -73,6 +73,8 @@ ALL_TESTS_FAILED_REGEX = re.compile(r"Fail! \(\d+ tests\)\r\n")
 SINGLE_CHECK_PASSED_REGEX = re.compile(r"Pass: .*")
 SINGLE_CHECK_FAILED_REGEX = re.compile(r".*failed:.*")
 
+RW_IMAGE_BOOTED_REGEX = re.compile(r"^\[Image: RW.*")
+
 ASSERTION_FAILURE_REGEX = re.compile(r"ASSERTION FAILURE.*")
 
 DATA_ACCESS_VIOLATION_8020000_REGEX = re.compile(
@@ -133,6 +135,15 @@ class ImageType(Enum):
     RW = 2
 
 
+class ApplicationType(Enum):
+    """
+    Select the application type to use (test or production)
+    """
+
+    TEST = 1
+    PRODUCTION = 2
+
+
 @dataclass
 class BoardConfig:
     """Board-specific configuration."""
@@ -152,7 +163,8 @@ class TestConfig:
 
     # pylint: disable=too-many-instance-attributes
     test_name: str
-    image_to_use: ImageType = ImageType.RW
+    imagetype_to_use: ImageType = ImageType.RW
+    apptype_to_use: ApplicationType = ApplicationType.TEST
     finish_regexes: List = None
     fail_regexes: List = None
     toggle_power: bool = False
@@ -206,6 +218,12 @@ class AllTests:
     def get_public_tests(board_config: BoardConfig) -> List[TestConfig]:
         """Return public test configs for the specified board."""
         tests = [
+            TestConfig(
+                test_name="production_app_test",
+                finish_regexes=[RW_IMAGE_BOOTED_REGEX],
+                imagetype_to_use=ImageType.RW,
+                apptype_to_use=ApplicationType.PRODUCTION,
+            ),
             TestConfig(test_name="abort"),
             TestConfig(test_name="aes"),
             TestConfig(test_name="always_memset"),
@@ -216,12 +234,12 @@ class AllTests:
             TestConfig(test_name="exception"),
             TestConfig(
                 test_name="flash_physical",
-                image_to_use=ImageType.RO,
+                imagetype_to_use=ImageType.RO,
                 toggle_power=True,
             ),
             TestConfig(
                 test_name="flash_write_protect",
-                image_to_use=ImageType.RO,
+                imagetype_to_use=ImageType.RO,
                 toggle_power=True,
                 enable_hw_write_protect=True,
             ),
@@ -229,7 +247,7 @@ class AllTests:
             TestConfig(
                 config_name="fpsensor_spi_ro",
                 test_name="fpsensor",
-                image_to_use=ImageType.RO,
+                imagetype_to_use=ImageType.RO,
                 test_args=["spi"],
             ),
             TestConfig(
@@ -240,7 +258,7 @@ class AllTests:
             TestConfig(
                 config_name="fpsensor_uart_ro",
                 test_name="fpsensor",
-                image_to_use=ImageType.RO,
+                imagetype_to_use=ImageType.RO,
                 test_args=["uart"],
             ),
             TestConfig(
@@ -258,7 +276,7 @@ class AllTests:
             TestConfig(
                 config_name="mpu_ro",
                 test_name="mpu",
-                image_to_use=ImageType.RO,
+                imagetype_to_use=ImageType.RO,
                 finish_regexes=[board_config.mpu_regex],
             ),
             TestConfig(
@@ -284,7 +302,9 @@ class AllTests:
                 finish_regexes=[board_config.rollback_region1_regex],
                 test_args=["region1"],
             ),
-            TestConfig(test_name="rollback_entropy", image_to_use=ImageType.RO),
+            TestConfig(
+                test_name="rollback_entropy", imagetype_to_use=ImageType.RO
+            ),
             TestConfig(test_name="rtc"),
             TestConfig(test_name="sha256"),
             TestConfig(test_name="sha256_unrolled"),
@@ -527,7 +547,9 @@ def hw_write_protect(enable: bool) -> None:
     subprocess.run(cmd, check=False).check_returncode()
 
 
-def build(test_name: str, board_name: str, compiler: str) -> None:
+def build(
+    test_name: str, board_name: str, compiler: str, app_type: ApplicationType
+) -> None:
     """Build specified test for specified board."""
     cmd = ["make"]
 
@@ -536,10 +558,15 @@ def build(test_name: str, board_name: str, compiler: str) -> None:
 
     cmd = cmd + [
         "BOARD=" + board_name,
-        "test-" + test_name,
         "-j",
     ]
 
+    # If the image type is a test image, then apply test- prefix to the target name
+    if app_type == ApplicationType.TEST:
+        cmd = cmd + [
+            "test-" + test_name,
+        ]
+
     logging.debug('Running command: "%s"', " ".join(cmd))
     subprocess.run(cmd, check=False).check_returncode()
 
@@ -637,12 +664,14 @@ def run_test(
     # Wait for boot to finish
     time.sleep(1)
     console.write("\n".encode())
-    if test.image_to_use == ImageType.RO:
+    if test.imagetype_to_use == ImageType.RO:
         console.write("reboot ro\n".encode())
         time.sleep(1)
 
-    test_cmd = "runtest " + " ".join(test.test_args) + "\n"
-    console.write(test_cmd.encode())
+    # Skip runtest if using standard app type
+    if test.apptype_to_use != ApplicationType.PRODUCTION:
+        test_cmd = "runtest " + " ".join(test.test_args) + "\n"
+        console.write(test_cmd.encode())
 
     while True:
         console.flush()
@@ -717,11 +746,19 @@ def flash_and_run_test(
         build_board = test.build_board
 
     # build test binary
-    build(test.test_name, build_board, args.compiler)
+    build(test.test_name, build_board, args.compiler, test.apptype_to_use)
 
-    image_path = os.path.join(
-        EC_DIR, "build", build_board, test.test_name, test.test_name + ".bin"
-    )
+    if test.apptype_to_use == ApplicationType.PRODUCTION:
+        image_path = os.path.join(EC_DIR, "build", build_board, "ec.bin")
+    else:
+        image_path = os.path.join(
+            EC_DIR,
+            "build",
+            build_board,
+            test.test_name,
+            test.test_name + ".bin",
+        )
+    logging.debug("image_path: %s", image_path)
 
     if test.ro_image is not None:
         try: