eCTS: Check order and expectation of test results

This patch makes the framework verify not only the result but also
the execution order of the tests. It also allows each test to
specify expected return code and strings printed by TH and DUT.
The final test results depends on the return code and the expectation.
Therefore, the output now includes 'RESULT' column showing PASS or FAIL:

   test name       TH_RETURN_CODE DUT_RETURN_CODE TH_STR DUT_STR RESULT
test_task_switch          SUCCESS         SUCCESS      1       1   PASS
test_task_priority        SUCCESS         FAILURE      1       1   FAIL
test_stack_overflow   DID_NOT_END     DID_NOT_END      1       1   PASS

Additionally, this patch:

 * Adds CTS_RC_DID_NOT_START and CTS_RC_DID_NOT_END to indicate whether
   the test did start or end, respectively.
 * Makes stack overflow test check whether stack overflow was detected
   and reboot occurred
 * Removes post_corruption_test and conflict test since now
   the test results are stricly compared against expected results.
 * Fixes gpylint errors.

BUG=none
BRANCH=none
TEST=Run gpio, meta, timer, interrupt, and cts/cts.py -m task

Change-Id: I3b7005236e705dcac0c8f4711b44c85ff9a4f676
Reviewed-on: https://chromium-review.googlesource.com/538878
Commit-Ready: Daisuke Nojiri <dnojiri@chromium.org>
Tested-by: Daisuke Nojiri <dnojiri@chromium.org>
Reviewed-by: Daisuke Nojiri <dnojiri@chromium.org>

19 files changed, 313 insertions, 199 deletions

diff --git a/cts/common/board.py b/cts/common/board.py
index d478b1fca8..d3db78b792 100644
--- a/cts/common/board.py
+++ b/cts/common/board.py
@@ -2,13 +2,13 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
-from abc import ABCMeta, abstractmethod
+from abc import ABCMeta
+from abc import abstractmethod
 import fcntl
 import os
 import select
 import shutil
 import subprocess as sp
-import time
 
 
 OCD_SCRIPT_DIR = '/usr/local/share/openocd/scripts'
@@ -26,7 +26,7 @@ REBOOT_MARKER = 'UART initialized after reboot'
 
 
 class Board(object):
-  """Class representing a single board connected to a host machine
+  """Class representing a single board connected to a host machine.
 
   Attributes:
     board: String containing actual type of board, i.e. nucleo-f072rb
@@ -40,19 +40,23 @@ class Board(object):
   """
 
   __metaclass__ = ABCMeta  # This is an Abstract Base Class (ABC)
+
   def __init__(self, board, module, hla_serial=None):
-    """Initializes a board object with given attributes
+    """Initializes a board object with given attributes.
 
     Args:
       board: String containing board name
       module: String of the test module you are building,
         i.e. gpio, timer, etc.
       hla_serial: Serial number if board's adaptor is an HLA
+
+    Raises:
+      RuntimeError: Board is not supported
     """
-    if not board in OPENOCD_CONFIGS:
+    if board not in OPENOCD_CONFIGS:
       msg = 'OpenOcd configuration not found for ' + board
       raise RuntimeError(msg)
-    if not board in FLASH_OFFSETS:
+    if board not in FLASH_OFFSETS:
       msg = 'Flash offset not found for ' + board
       raise RuntimeError(msg)
     self.board = board
@@ -64,14 +68,14 @@ class Board(object):
     self.tty = None
 
   def reset_log_dir(self):
-    """Reset log directory"""
+    """Reset log directory."""
     if os.path.isdir(self.log_dir):
       shutil.rmtree(self.log_dir)
     os.makedirs(self.log_dir)
 
   @staticmethod
   def get_stlink_serials():
-    """Gets serial numbers of all st-link v2.1 board attached to host
+    """Gets serial numbers of all st-link v2.1 board attached to host.
 
     Returns:
       List of serials
@@ -81,7 +85,7 @@ class Board(object):
     st_link_info = usb_process.communicate()[0]
     st_serials = []
     for line in st_link_info.split('\n'):
-      if not 'iSerial' in line:
+      if 'iSerial' not in line:
         continue
       words = line.split()
       if len(words) <= 2:
@@ -91,14 +95,17 @@ class Board(object):
 
   @abstractmethod
   def get_serial(self):
-    """Subclass should implement this"""
+    """Subclass should implement this."""
     pass
 
   def send_open_ocd_commands(self, commands):
-    """Send a command to the board via openocd
+    """Send a command to the board via openocd.
 
     Args:
       commands: A list of commands to send
+
+    Returns:
+      True if execution is successful or False otherwise.
     """
     args = ['openocd', '-s', OCD_SCRIPT_DIR,
             '-f', self.openocd_config, '-c', 'hla_serial ' + self.hla_serial]
@@ -120,11 +127,14 @@ class Board(object):
     with open(self.openocd_log) as log:
       print log.read()
 
-  def build(self, module, ec_dir):
-    """Builds test suite module for board
+  def build(self, ec_dir):
+    """Builds test suite module for board.
 
     Args:
       ec_dir: String of the ec directory path
+
+    Returns:
+      True if build is successful or False otherwise.
     """
     cmds = ['make',
             '--directory=' + ec_dir,
@@ -146,7 +156,7 @@ class Board(object):
       print log.read()
 
   def flash(self, image_path):
-    """Flashes board with most recent build ec.bin"""
+    """Flashes board with most recent build ec.bin."""
     cmd = ['reset_config connect_assert_srst',
            'init',
            'reset init',
@@ -163,11 +173,12 @@ class Board(object):
     return s
 
   def reset(self):
-    """Reset then halt board """
+    """Reset then halt board."""
     return self.send_open_ocd_commands(['init', 'reset halt'])
 
   def setup_tty(self):
     """Call this before calling read_tty for the first time.
+
     This is not in the initialization because caller only should call
     this function after serial numbers are setup
     """
@@ -185,12 +196,12 @@ class Board(object):
     self.tty = tty
 
   def read_tty(self, max_boot_count=1):
-    """Read info from a serial port described by a file descriptor
+    """Read info from a serial port described by a file descriptor.
 
     Args:
       max_boot_count: Stop reading if boot count exceeds this number
 
-    Return:
+    Returns:
       result: characters read from tty
       boot: boot counts
     """
@@ -219,19 +230,15 @@ class Board(object):
     return result, boot
 
   def identify_tty_port(self):
-    """Saves this board's serial port"""
+    """Saves this board's serial port."""
     dev_dir = '/dev'
     id_prefix = 'ID_SERIAL_SHORT='
     com_devices = [f for f in os.listdir(dev_dir) if f.startswith('ttyACM')]
 
     for device in com_devices:
       self.tty_port = os.path.join(dev_dir, device)
-      properties = sp.check_output(['udevadm',
-                      'info',
-                      '-a',
-                      '-n',
-                      self.tty_port,
-                      '--query=property'])
+      properties = sp.check_output(
+          ['udevadm', 'info', '-a', '-n', self.tty_port, '--query=property'])
       for line in [l.strip() for l in properties.split('\n')]:
         if line.startswith(id_prefix):
           if self.hla_serial == line[len(id_prefix):]:
@@ -241,7 +248,7 @@ class Board(object):
     raise RuntimeError('The device dev path could not be found')
 
   def open_tty(self):
-    """Read available bytes from device dev path"""
+    """Read available bytes from device dev path."""
     fd = os.open(self.tty_port, os.O_RDONLY)
     flag = fcntl.fcntl(fd, fcntl.F_GETFL)
     fcntl.fcntl(fd, fcntl.F_SETFL, flag | os.O_NONBLOCK)
@@ -249,18 +256,19 @@ class Board(object):
 
 
 class TestHarness(Board):
-  """Subclass of Board representing a Test Harness
+  """Subclass of Board representing a Test Harness.
 
   Attributes:
     serial_path: Path to file containing serial number
   """
 
   def __init__(self, board, module, log_dir, serial_path):
-    """Initializes a board object with given attributes
+    """Initializes a board object with given attributes.
 
     Args:
       board: board name
       module: module name
+      log_dir: Directory where log file is stored
       serial_path: Path to file containing serial number
     """
     Board.__init__(self, board, module)
@@ -271,9 +279,9 @@ class TestHarness(Board):
     self.reset_log_dir()
 
   def get_serial(self):
-    """Loads serial number from saved location"""
+    """Loads serial number from saved location."""
     if self.hla_serial:
-      return # serial was already loaded
+      return  # serial was already loaded
     try:
       with open(self.serial_path, mode='r') as ser_f:
         self.hla_serial = ser_f.read()
@@ -284,7 +292,7 @@ class TestHarness(Board):
       raise RuntimeError(msg)
 
   def save_serial(self):
-    """Saves the TH serial number to a file"""
+    """Saves the TH serial number to a file."""
     serials = Board.get_stlink_serials()
     if len(serials) > 1:
       msg = ('There are more than one test board connected to the host.'
@@ -296,9 +304,9 @@ class TestHarness(Board):
       raise RuntimeError(msg)
 
     serial = serials[0]
-    dir = os.path.dirname(self.serial_path)
-    if not os.path.exists(dir):
-      os.makedirs(dir)
+    serial_dir = os.path.dirname(self.serial_path)
+    if not os.path.exists(serial_dir):
+      os.makedirs(serial_dir)
     with open(self.serial_path, mode='w') as ser_f:
       ser_f.write(serial)
       self.hla_serial = serial
@@ -308,20 +316,21 @@ class TestHarness(Board):
 
 
 class DeviceUnderTest(Board):
-  """Subclass of Board representing a DUT board
+  """Subclass of Board representing a DUT board.
 
   Attributes:
     th: Reference to test harness board to which this DUT is attached
   """
 
   def __init__(self, board, th, module, log_dir, hla_ser=None):
-    """Initializes a Device Under Test object with given attributes
+    """Initializes a DUT object.
 
     Args:
       board: String containing board name
       th: Reference to test harness board to which this DUT is attached
       module: module name
-      hla_serial: Serial number if board uses an HLA adaptor
+      log_dir: Directory where log file is stored
+      hla_ser: Serial number if board uses an HLA adaptor
     """
     Board.__init__(self, board, module, hla_serial=hla_ser)
     self.th = th
@@ -335,8 +344,11 @@ class DeviceUnderTest(Board):
 
     Precondition: The DUT and TH must both be connected, and th.hla_serial
     must hold the correct value (the th's serial #)
+
+    Raises:
+      RuntimeError: DUT isn't found or multiple DUTs are found.
     """
-    if self.hla_serial != None:
+    if self.hla_serial is not None:
       # serial was already set ('' is a valid serial)
       return
 
diff --git a/cts/common/cts.rc b/cts/common/cts.rc
index 8d66a8af4e..264b982655 100644
--- a/cts/common/cts.rc
+++ b/cts/common/cts.rc
@@ -14,11 +14,21 @@
  * where <NAME> will be printed on the result screen.
  */
 
-/* Host only codes. Should not be needed by th.c or dut.c. */
-CTS_RC_DUPLICATE_RUN = -2,
-CTS_RC_NO_RESULT = -1,
+/*
+ * Host only return codes. Should not be needed by th.c or dut.c.
+ */
+/* Test didn't run */
+CTS_RC_DID_NOT_START = -1,
+/* Test didn't end */
+CTS_RC_DID_NOT_END = -2,
+/* Results were reported twice or more */
+CTS_RC_DUPLICATE_RUN = -3,
+/* Error in parsing return code. Probably it was null or not an integer. */
+CTS_RC_INVALID_RC = -4,
 
-/* Regular codes */
+/*
+ * Regular return codes. Used by DUT and TH.
+ */
 CTS_RC_SUCCESS = 0,
 CTS_RC_FAILURE,
 CTS_RC_BAD_SYNC,
diff --git a/cts/common/cts_testlist.h b/cts/common/cts_testlist.h
index a9a53c56f6..9e18c7f8eb 100644
--- a/cts/common/cts_testlist.h
+++ b/cts/common/cts_testlist.h
@@ -3,18 +3,25 @@
  * found in the LICENSE file.
  */
 
+/*
+ * CTS_TEST macro is used by dut.c, th.c, and cts.py. Currently, the 2nd
+ * and 3rd arguments are only used by cts.py. They specify the expected
+ * strings output by TH and DUT, respectively.
+ */
+
 struct cts_test {
 	enum cts_rc (*run)(void);
 	char *name;
 };
 
-#define CTS_TEST(test)	{test, STRINGIFY(test)},
+#define CTS_TEST(test, th_rc, th_string, dut_rc, dut_string) \
+	{test, STRINGIFY(test)},
 struct cts_test tests[] = {
 #include "cts.testlist"
 };
 
 #undef CTS_TEST
-#define CTS_TEST(test)	CTS_TEST_ID_##test,
+#define CTS_TEST(test, th_rc, th_string, dut_rc, dut_string) CTS_TEST_ID_##test,
 enum {
 #include "cts.testlist"
 	CTS_TEST_ID_COUNT,
diff --git a/cts/cts.py b/cts/cts.py
index b046bc4ca5..ed0889698d 100755
--- a/cts/cts.py
+++ b/cts/cts.py
@@ -1,4 +1,5 @@
-#!/usr/bin/python2
+#!/usr/bin/python
+#
 # Copyright 2016 The Chromium OS Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
@@ -17,32 +18,33 @@
 
 
 import argparse
-from collections import defaultdict
-import common.board as board
 import os
 import shutil
 import time
+import common.board as board
 
 
-# Host only return codes. Make sure they match values in cts.rc
-CTS_RC_DUPLICATE_RUN = -2  # The test was run multiple times.
-CTS_RC_NO_RESULT = -1  # The test did not run.
-
+CTS_RC_PREFIX = 'CTS_RC_'
 DEFAULT_TH = 'stm32l476g-eval'
 DEFAULT_DUT = 'nucleo-f072rb'
 MAX_SUITE_TIME_SEC = 5
 CTS_TEST_RESULT_DIR = '/tmp/ects'
 
+# Host only return codes. Make sure they match values in cts.rc
+CTS_RC_DID_NOT_START = -1          # test did not run.
+CTS_RC_DID_NOT_END = -2            # test did not run.
+CTS_RC_DUPLICATE_RUN = -3          # test was run multiple times.
+CTS_RC_INVALID_RETURN_CODE = -4    # failed to parse return code
+
 
 class Cts(object):
-  """Class that represents a CTS testing setup and provides
-  interface to boards (building, flashing, etc.)
+  """Class that represents a eCTS run.
 
   Attributes:
-    dut: DeviceUnderTest object representing dut
-    th: TestHarness object representing th
+    dut: DeviceUnderTest object representing DUT
+    th: TestHarness object representing a test harness
     module: Name of module to build/run tests for
-    test_names: List of strings of test names contained in given module
+    testlist: List of strings of test names contained in given module
     return_codes: Dict of strings of return codes, with a code's integer
       value being the index for the corresponding string representation
   """
@@ -68,22 +70,21 @@ class Cts(object):
     self.dut = board.DeviceUnderTest(dut, self.th, module, self.results_dir)
     cts_dir = os.path.join(self.ec_dir, 'cts')
     testlist_path = os.path.join(cts_dir, self.module, 'cts.testlist')
-    self.test_names = Cts.get_macro_args(testlist_path, 'CTS_TEST')
-
     return_codes_path = os.path.join(cts_dir, 'common', 'cts.rc')
-    self.get_return_codes(return_codes_path, 'CTS_RC_')
+    self.get_return_codes(return_codes_path)
+    self.testlist = self.get_macro_args(testlist_path, 'CTS_TEST')
 
   def build(self):
-    """Build images for DUT and TH"""
+    """Build images for DUT and TH."""
     print 'Building DUT image...'
-    if not self.dut.build(self.module, self.ec_dir):
+    if not self.dut.build(self.ec_dir):
       raise RuntimeError('Building module %s for DUT failed' % (self.module))
     print 'Building TH image...'
-    if not self.th.build(self.module, self.ec_dir):
+    if not self.th.build(self.ec_dir):
       raise RuntimeError('Building module %s for TH failed' % (self.module))
 
   def flash_boards(self):
-    """Flashes th and dut boards with their most recently build ec.bin"""
+    """Flashes TH and DUT with their most recently built ec.bin."""
     cts_module = 'cts_' + self.module
     image_path = os.path.join('build', self.th.board, cts_module, 'ec.bin')
     self.identify_boards()
@@ -96,102 +97,207 @@ class Cts(object):
       raise RuntimeError('Flashing DUT failed')
 
   def setup(self):
-    """Setup boards"""
+    """Setup boards."""
     self.th.save_serial()
 
   def identify_boards(self):
-    """Updates serials of both th and dut, in that order (order matters)"""
+    """Updates serials of TH and DUT in that order (order matters)."""
     self.th.get_serial()
     self.dut.get_serial()
 
-  @staticmethod
-  def get_macro_args(filepath, macro):
-    """Get list of args of a certain macro in a file when macro is used
-    by itself on a line
+  def get_macro_args(self, filepath, macro):
+    """Get list of args of a macro in a file when macro.
 
     Args:
       filepath: String containing absolute path to the file
       macro: String containing text of macro to get args of
+
+    Returns:
+      List of dictionaries where each entry is:
+        'name': Test name,
+        'th_string': Expected string from TH,
+        'dut_string': Expected string from DUT,
     """
-    args = []
+    tests = []
     with open(filepath, 'r') as f:
-      for l in f.readlines():
+      lines = f.readlines()
+      joined = ''.join(lines).replace('\\\n', '').splitlines()
+      for l in joined:
         if not l.strip().startswith(macro):
           continue
+        d = {}
         l = l.strip()[len(macro):]
-        args.append(l.strip('()').replace(',', ''))
-    return args
-
-  def get_return_codes(self, file, prefix):
-    """Extract return code names from the definition file (cts.rc)"""
+        l = l.strip('()').split(',')
+        d['name'] = l[0].strip()
+        d['th_rc'] = self.get_return_code_value(l[1].strip().strip('"'))
+        d['th_string'] = l[2].strip().strip('"')
+        d['dut_rc'] = self.get_return_code_value(l[3].strip().strip('"'))
+        d['dut_string'] = l[4].strip().strip('"')
+        tests.append(d)
+    return tests
+
+  def get_return_codes(self, filepath):
+    """Read return code names from the return code definition file."""
     self.return_codes = {}
     val = 0
-    with open(file, 'r') as f:
-      for line in f.readlines():
+    with open(filepath, 'r') as f:
+      for line in f:
         line = line.strip()
-        if not line.startswith(prefix):
+        if not line.startswith(CTS_RC_PREFIX):
           continue
-        line = line[len(prefix):]
         line = line.split(',')[0]
         if '=' in line:
           tokens = line.split('=')
           line = tokens[0].strip()
           val = int(tokens[1].strip())
-        self.return_codes[val] = line
+        self.return_codes[line] = val
         val += 1
 
   def parse_output(self, output):
-    results = defaultdict(lambda: CTS_RC_NO_RESULT)
+    """Parse console output from DUT or TH.
 
+    Args:
+      output: String containing consoule output
+
+    Returns:
+      List of dictionaries where each key and value are:
+        name = 'ects_test_x',
+        started = True/False,
+        ended = True/False,
+        rc = CTS_RC_*,
+        output = All text between 'ects_test_x start' and 'ects_test_x end'
+    """
+    results = []
+    i = 0
+    for test in self.testlist:
+      results.append({})
+      results[i]['name'] = test['name']
+      results[i]['started'] = False
+      results[i]['rc'] = CTS_RC_DID_NOT_START
+      results[i]['string'] = False
+      results[i]['output'] = []
+      i += 1
+
+    i = 0
     for ln in [ln.strip() for ln in output.split('\n')]:
+      if i + 1 > len(results):
+        break
       tokens = ln.split()
-      if len(tokens) != 2:
-        continue
-      test_name = tokens[0].strip()
-      if test_name not in self.test_names:
-        continue
-      try:
-        return_code = int(tokens[1])
-      except ValueError: # Second token is not an int
-        continue
-      if test_name in results:
-        results[test_name] = CTS_RC_DUPLICATE_RUN
-      else:
-        results[test_name] = return_code
+      if len(tokens) >= 2:
+        if tokens[0].strip() == results[i]['name']:
+          if tokens[1].strip() == 'start':
+            # start line found
+            if results[i]['started']:   # Already started
+              results[i]['rc'] = CTS_RC_DUPLICATE_RUN
+            else:
+              results[i]['rc'] = CTS_RC_DID_NOT_END
+              results[i]['started'] = True
+            continue
+          elif results[i]['started'] and tokens[1].strip() == 'end':
+            # end line found
+            results[i]['rc'] = CTS_RC_INVALID_RETURN_CODE
+            if len(tokens) == 3:
+              try:
+                results[i]['rc'] = int(tokens[2].strip())
+              except ValueError:
+                pass
+            # Since index is incremented when 'end' is encountered, we don't
+            # need to check duplicate 'end'.
+            i += 1
+            continue
+      if results[i]['started']:
+        results[i]['output'].append(ln)
 
     return results
 
-  def get_return_code_name(self, code):
-    return self.return_codes.get(code, '%d' % code)
+  def get_return_code_name(self, code, strip_prefix=False):
+    name = ''
+    for k, v in self.return_codes.iteritems():
+      if v == code:
+        if strip_prefix:
+          name = k[len(CTS_RC_PREFIX):]
+        else:
+          name = k
+    return name
+
+  def get_return_code_value(self, name):
+    if name:
+      return self.return_codes[name]
+    return  0
 
   def evaluate_run(self, dut_output, th_output):
-    """Parse outputs to derive test results
+    """Parse outputs to derive test results.
 
-    Args;
+    Args:
       dut_output: String output of DUT
       th_output: String output of TH
+
+    Returns:
+      th_results: list of test results for TH
+      dut_results: list of test results for DUT
     """
-    dut_results = self.parse_output(dut_output)
     th_results = self.parse_output(th_output)
+    dut_results = self.parse_output(dut_output)
+
+    # Search for expected string in each output
+    for i, v in enumerate(self.testlist):
+      if v['th_string'] in th_results[i]['output'] or not v['th_string']:
+        th_results[i]['string'] = True
+      if v['dut_string'] in dut_results[i]['output'] or not v['dut_string']:
+        dut_results[i]['string'] = True
 
-    len_test_name = max(len(s) for s in self.test_names)
-    len_code_name = max(len(s) for s in self.return_codes.values())
+    return th_results, dut_results
+
+  def print_result(self, th_results, dut_results):
+    """Print results to the screen.
+
+    Args:
+      th_results: list of test results for TH
+      dut_results: list of test results for DUT
+    """
+    len_test_name = max(len(s['name']) for s in self.testlist)
+    len_code_name = max(len(self.get_return_code_name(v, True))
+                        for v in self.return_codes.values())
 
     head = '{:^' + str(len_test_name) + '} '
     head += '{:^' + str(len_code_name) + '} '
-    head += '{:^' + str(len_code_name) + '}\n'
+    head += '{:^' + str(len_code_name) + '}'
+    head += '{:^' + str(len(' TH_STR')) + '}'
+    head += '{:^' + str(len(' DUT_STR')) + '}'
+    head += '{:^' + str(len(' RESULT')) + '}\n'
     fmt = '{:' + str(len_test_name) + '} '
     fmt += '{:>' + str(len_code_name) + '} '
-    fmt += '{:>' + str(len_code_name) + '}\n'
-
-    self.formatted_results = head.format('test name', 'TH', 'DUT')
-    for test_name in self.test_names:
-      th_cn = self.get_return_code_name(th_results[test_name])
-      dut_cn = self.get_return_code_name(dut_results[test_name])
-      self.formatted_results += fmt.format(test_name, th_cn, dut_cn)
+    fmt += '{:>' + str(len_code_name) + '}'
+    fmt += '{:>' + str(len(' TH_STR')) + '}'
+    fmt += '{:>' + str(len(' DUT_STR')) + '}'
+    fmt += '{:>' + str(len(' RESULT')) + '}\n'
+
+    self.formatted_results = head.format(
+        'test name', 'TH_RETURN_CODE', 'DUT_RETURN_CODE',
+        ' TH_STR', ' DUT_STR', ' RESULT')
+    for i, d in enumerate(dut_results):
+      th_cn = self.get_return_code_name(th_results[i]['rc'], True)
+      dut_cn = self.get_return_code_name(dut_results[i]['rc'], True)
+      th_res = self.evaluate_result(th_results[i],
+                                    self.testlist[i]['th_rc'],
+                                    self.testlist[i]['th_string'])
+      dut_res = self.evaluate_result(dut_results[i],
+                                     self.testlist[i]['dut_rc'],
+                                     self.testlist[i]['dut_string'])
+      self.formatted_results += fmt.format(
+          d['name'], th_cn, dut_cn,
+          th_results[i]['string'], dut_results[i]['string'],
+          'PASS' if th_res and dut_res else 'FAIL')
+
+  def evaluate_result(self, result, expected_rc, expected_string):
+    if result['rc'] != expected_rc:
+      return False
+    if expected_string and expected_string not in result['output']:
+      return False
+    return True
 
   def run(self):
-    """Resets boards, records test results in results dir"""
+    """Resets boards, records test results in results dir."""
     print 'Reading serials...'
     self.identify_boards()
     print 'Opening DUT tty...'
@@ -228,9 +334,9 @@ class Cts(object):
     time.sleep(MAX_SUITE_TIME_SEC)
 
     print 'Reading DUT tty...'
-    dut_output, dut_boot = self.dut.read_tty()
+    dut_output, _ = self.dut.read_tty()
     print 'Reading TH tty...'
-    th_output, th_boot = self.th.read_tty()
+    th_output, _ = self.th.read_tty()
 
     print 'Halting TH...'
     if not self.th.send_open_ocd_commands(['init', 'reset halt']):
@@ -245,7 +351,10 @@ class Cts(object):
                        'again.')
 
     print 'Pursing results...'
-    self.evaluate_run(dut_output, th_output)
+    th_results, dut_results = self.evaluate_run(dut_output, th_output)
+
+    # Print out results
+    self.print_result(th_results, dut_results)
 
     # Write results
     dest = os.path.join(self.results_dir, 'results.log')
@@ -262,11 +371,10 @@ class Cts(object):
 
     print self.formatted_results
 
-    # TODO: Should set exit code for the shell
+    # TODO(chromium:735652): Should set exit code for the shell
 
 
 def main():
-  """Main entry point for CTS script from command line"""
   ec_dir = os.path.realpath(os.path.join(
       os.path.dirname(os.path.abspath(__file__)), '..'))
   os.chdir(ec_dir)
@@ -321,5 +429,5 @@ def main():
     cts.flash_boards()
     cts.run()
 
-if __name__ == "__main__":
+if __name__ == '__main__':
   main()
diff --git a/cts/gpio/cts.testlist b/cts/gpio/cts.testlist
index e59be1c7ce..be303067a3 100644
--- a/cts/gpio/cts.testlist
+++ b/cts/gpio/cts.testlist
@@ -6,15 +6,20 @@
 /* Currently tests will execute in the order they are listed here */
 
 /* Test whether sync completes successfully */
-CTS_TEST(sync_test)
+CTS_TEST(sync_test,,,,)
+
 /* Check if the dut can set a line low */
-CTS_TEST(set_low_test)
+CTS_TEST(set_low_test,,,,)
+
 /* Check if the dut can set a line high */
-CTS_TEST(set_high_test)
+CTS_TEST(set_high_test,,,,)
+
 /* Check if the dut can read a line that is low */
-CTS_TEST(read_high_test)
+CTS_TEST(read_high_test,,,,)
+
 /* Check if the dut can read a line that is high */
-CTS_TEST(read_low_test)
+CTS_TEST(read_low_test,,,,)
+
 /* Check if the dut reads its true pin level (success)
    or its register level when configured as a low open drain output pin */
-CTS_TEST(od_read_high_test)
+CTS_TEST(od_read_high_test,,,,)
diff --git a/cts/gpio/dut.c b/cts/gpio/dut.c
index 22694b8c4f..c5b6bf813f 100644
--- a/cts/gpio/dut.c
+++ b/cts/gpio/dut.c
@@ -81,8 +81,9 @@ void cts_task(void)
 	uart_flush_output();
 	for (i = 0; i < CTS_TEST_ID_COUNT; i++) {
 		sync();
+		CPRINTF("\n%s start\n", tests[i].name);
 		result = tests[i].run();
-		CPRINTF("\n%s %d\n", tests[i].name, result);
+		CPRINTF("\n%s end %d\n", tests[i].name, result);
 		uart_flush_output();
 	}
 
diff --git a/cts/gpio/th.c b/cts/gpio/th.c
index 7350ff0e55..d18c1c367a 100644
--- a/cts/gpio/th.c
+++ b/cts/gpio/th.c
@@ -75,8 +75,9 @@ void cts_task(void)
 	uart_flush_output();
 	for (i = 0; i < CTS_TEST_ID_COUNT; i++) {
 		sync();
+		CPRINTF("\n%s start\n", tests[i].name);
 		result = tests[i].run();
-		CPRINTF("\n%s %d\n", tests[i].name, result);
+		CPRINTF("\n%s end %d\n", tests[i].name, result);
 		uart_flush_output();
 	}
 
diff --git a/cts/i2c/cts.testlist b/cts/i2c/cts.testlist
index 79732dc65b..809daaa322 100644
--- a/cts/i2c/cts.testlist
+++ b/cts/i2c/cts.testlist
@@ -6,9 +6,9 @@
 /* Currently tests will execute in the order they are listed here */
 
 /* Test whether sync completes successfully */
-CTS_TEST(write8_test)
-CTS_TEST(write16_test)
-CTS_TEST(write32_test)
-CTS_TEST(read8_test)
-CTS_TEST(read16_test)
-CTS_TEST(read32_test)
\ No newline at end of file
+CTS_TEST(write8_test,,,,)
+CTS_TEST(write16_test,,,,)
+CTS_TEST(write32_test,,,,)
+CTS_TEST(read8_test,,,,)
+CTS_TEST(read16_test,,,,)
+CTS_TEST(read32_test,,,,)
\ No newline at end of file
diff --git a/cts/interrupt/cts.testlist b/cts/interrupt/cts.testlist
index 6e0265ed60..0fdaf6fca2 100644
--- a/cts/interrupt/cts.testlist
+++ b/cts/interrupt/cts.testlist
@@ -4,14 +4,14 @@
  */
 
 /* Test interrupt_enable/disable */
-CTS_TEST(test_interrupt_enable)
-CTS_TEST(test_interrupt_disable)
+CTS_TEST(test_interrupt_enable,,,,)
+CTS_TEST(test_interrupt_disable,,,,)
 
 /* Test task_wait_for_event */
-CTS_TEST(test_task_wait_event)
+CTS_TEST(test_task_wait_event,,,,)
 
 /* Test task_disable_irq */
-CTS_TEST(test_task_disable_irq)
+CTS_TEST(test_task_disable_irq,,,,)
 
 /* Test nested interrupt. Lower priority IRQ is fired, followed by
  * higher priority IRQ. Handler executions should be nested.
@@ -23,7 +23,7 @@ CTS_TEST(test_task_disable_irq)
  * task_cts ----*                       *----
  *                A      B     C      D
  */
-CTS_TEST(test_nested_interrupt_low_high)
+CTS_TEST(test_nested_interrupt_low_high,,,,)
 
 /* Test nested interrupt. Higher priority IRQ is fired, followed by
  * lower priority IRQ. Handlers should be executed sequentially.
@@ -35,7 +35,7 @@ CTS_TEST(test_nested_interrupt_low_high)
  * task_cts ----*                   *----
  *                  B     C A     D
  */
-CTS_TEST(test_nested_interrupt_high_low)
+CTS_TEST(test_nested_interrupt_high_low,,,,)
 
 /*
  * Other ideas
diff --git a/cts/interrupt/dut.c b/cts/interrupt/dut.c
index a04d5659d5..35b47001ca 100644
--- a/cts/interrupt/dut.c
+++ b/cts/interrupt/dut.c
@@ -185,9 +185,10 @@ void cts_task(void)
 	for (i = 0; i < CTS_TEST_ID_COUNT; i++) {
 		clear_state();
 		sync();
+		CPRINTF("\n%s start\n", tests[i].name);
 		rc = tests[i].run();
 		interrupt_enable();
-		CPRINTF("\n%s %d\n", tests[i].name, rc);
+		CPRINTF("\n%s end %d\n", tests[i].name, rc);
 		cflush();
 	}
 
diff --git a/cts/interrupt/th.c b/cts/interrupt/th.c
index 0949faa8b2..b409fbd8ef 100644
--- a/cts/interrupt/th.c
+++ b/cts/interrupt/th.c
@@ -74,8 +74,9 @@ void cts_task(void)
 		gpio_set_level(GPIO_OUTPUT_TEST, 1);
 		gpio_set_level(GPIO_CTS_IRQ2, 1);
 		sync();
+		CPRINTF("\n%s start\n", tests[i].name);
 		rc = tests[i].run();
-		CPRINTF("\n%s %d\n", tests[i].name, rc);
+		CPRINTF("\n%s end %d\n", tests[i].name, rc);
 		cflush();
 	}
 
diff --git a/cts/meta/cts.testlist b/cts/meta/cts.testlist
index 778b0f62cc..d453d33e51 100644
--- a/cts/meta/cts.testlist
+++ b/cts/meta/cts.testlist
@@ -12,53 +12,41 @@
  * 3
  * 4
  */
-CTS_TEST(debug_test)
+CTS_TEST(debug_test,,,,)
 
 /* Test should succeed if both report success
  * (expected result: success)
  */
-CTS_TEST(success_test)
+CTS_TEST(success_test,,,,)
 
 /* Test should fail if one reports success and
  * (one reports failure (expected result: failure)
  */
-CTS_TEST(fail_dut_test)
+CTS_TEST(fail_dut_test,,, CTS_RC_FAILURE,)
 
 /* Test should fail if one reports success and
  * (one reports failure (expected result: failure)
  */
-CTS_TEST(fail_th_test)
+CTS_TEST(fail_th_test, CTS_RC_FAILURE,,,)
 
 /* Test should fail when both boards report failure
  * (expected result: failure)
  */
-CTS_TEST(fail_both_test)
+CTS_TEST(fail_both_test, CTS_RC_FAILURE,, CTS_RC_FAILURE,)
 
 /* Test should fail with bad sync if one reports bad
  * sync and the other reports success (expected result:
  * bad_sync)
  */
-CTS_TEST(bad_sync_and_success_test)
+CTS_TEST(bad_sync_and_success_test, CTS_RC_BAD_SYNC,,,)
 
 /* Test should fail with bad sync if both boards report
  * bad sync (expected result: bad_sync)
  */
-CTS_TEST(bad_sync_both_test)
-
-/* Test should report conflict if one reports bad sync
- * and the other reports failure
- * (expected result: conflict)
- */
-CTS_TEST(bad_sync_failure_test)
+CTS_TEST(bad_sync_both_test, CTS_RC_BAD_SYNC,, CTS_RC_BAD_SYNC,)
 
 /* Test should be listed as corrupted if one test hangs,
  * regardless of what the other test outputs
  * (expected result: corrupted)
  */
-CTS_TEST(hang_test)
-
-/* Test should be corrupted if it follows a corrupted
- * test, regardless of what the actual result was
- * reported as
- */
-CTS_TEST(post_corruption_success)
\ No newline at end of file
+CTS_TEST(hang_test, CTS_RC_SUCCESS,, CTS_RC_DID_NOT_END,)
diff --git a/cts/meta/dut.c b/cts/meta/dut.c
index ad772bc10b..b80628449b 100644
--- a/cts/meta/dut.c
+++ b/cts/meta/dut.c
@@ -37,7 +37,7 @@ enum cts_rc fail_both_test(void)
 
 enum cts_rc bad_sync_and_success_test(void)
 {
-	return CTS_RC_BAD_SYNC;
+	return CTS_RC_SUCCESS;
 }
 
 enum cts_rc bad_sync_both_test(void)
@@ -45,11 +45,6 @@ enum cts_rc bad_sync_both_test(void)
 	return CTS_RC_BAD_SYNC;
 }
 
-enum cts_rc bad_sync_failure_test(void)
-{
-	return CTS_RC_BAD_SYNC;
-}
-
 enum cts_rc hang_test(void)
 {
 	while (1) {
@@ -60,11 +55,6 @@ enum cts_rc hang_test(void)
 	return CTS_RC_SUCCESS;
 }
 
-enum cts_rc post_corruption_success(void)
-{
-	return CTS_RC_SUCCESS;
-}
-
 #include "cts_testlist.h"
 
 void cts_task(void)
@@ -75,12 +65,13 @@ void cts_task(void)
 	cflush();
 	for (i = 0; i < CTS_TEST_ID_COUNT; i++) {
 		sync();
+		CPRINTF("\n%s start\n", tests[i].name);
 		result = tests[i].run();
-		CPRINTF("\n%s %d\n", tests[i].name, result);
+		CPRINTF("\n%s end %d\n", tests[i].name, result);
 		cflush();
 	}
 
-	CPRINTS("GPIO test suite finished");
+	CPRINTS("Meta test finished");
 	cflush();
 	while (1) {
 		watchdog_reload();
diff --git a/cts/meta/th.c b/cts/meta/th.c
index bdab9ce6f1..c1331ff2a7 100644
--- a/cts/meta/th.c
+++ b/cts/meta/th.c
@@ -55,23 +55,12 @@ enum cts_rc bad_sync_both_test(void)
 	return CTS_RC_BAD_SYNC;
 }
 
-enum cts_rc bad_sync_failure_test(void)
-{
-	CTS_DEBUG_PRINTF("Expect: Conflict");
-	return CTS_RC_FAILURE;
-}
-
 enum cts_rc hang_test(void)
 {
 	CTS_DEBUG_PRINTF("This and next, expect: Corrupted");
 	return CTS_RC_SUCCESS;
 }
 
-enum cts_rc post_corruption_success(void)
-{
-	return CTS_RC_SUCCESS;
-}
-
 #include "cts_testlist.h"
 
 void cts_task(void)
@@ -82,12 +71,13 @@ void cts_task(void)
 	cflush();
 	for (i = 0; i < CTS_TEST_ID_COUNT; i++) {
 		sync();
+		CPRINTF("\n%s start\n", tests[i].name);
 		result = tests[i].run();
-		CPRINTF("\n%s %d\n", tests[i].name, result);
+		CPRINTF("\n%s end %d\n", tests[i].name, result);
 		cflush();
 	}
 
-	CPRINTS("GPIO test suite finished");
+	CPRINTS("Meta test finished");
 	cflush();
 	while (1) {
 		watchdog_reload();
diff --git a/cts/task/cts.testlist b/cts/task/cts.testlist
index 25dbcc313a..c4b7bc3231 100644
--- a/cts/task/cts.testlist
+++ b/cts/task/cts.testlist
@@ -9,18 +9,19 @@
  * repeated TEST_COUNT times. It's expected all tasks to run exactly
  * TEST_COUNT times. Tick task runs to inject some irregularity.
  */
-CTS_TEST(test_task_switch)
+CTS_TEST(test_task_switch,,,,)
 
 /*
  * Test task priority. CTS task wakes up A and C then goes to sleep. Since C
  * has a higher priority, C should run first. This should result in C running
  * one more time than A (or B).
  */
-CTS_TEST(test_task_priority)
+CTS_TEST(test_task_priority,,,,)
 
 /*
  * Test stack overflow. CTS task overflows the stack and it should be detected
  * when task switch happens. Reboot is expected.
- * TODO: Verify stack overflow detection and reboot.
  */
-CTS_TEST(test_stack_overflow)
\ No newline at end of file
+CTS_TEST(test_stack_overflow,\
+	 CTS_RC_DID_NOT_END, "Stack overflow in CTS task!",\
+	 CTS_RC_DID_NOT_END, "Stack overflow in CTS task!")
diff --git a/cts/task/dut.c b/cts/task/dut.c
index e34dd0babb..4cc6e3e4a4 100644
--- a/cts/task/dut.c
+++ b/cts/task/dut.c
@@ -125,19 +125,14 @@ static void recurse(int x)
 	CPRINTS("-%d", x);
 }
 
-enum cts_rc test_stack_overflow(void);
-
-#include "cts_testlist.h"
-
 enum cts_rc test_stack_overflow(void)
 {
-	/* recurse() is expected to overflow the stack, which leads to reboot.
-	 * So, we print output proactively. */
-	CPRINTF("\n%s %d\n", tests[CTS_TEST_ID_COUNT - 1].name, CTS_RC_SUCCESS);
 	recurse(0);
 	return CTS_RC_FAILURE;
 }
 
+#include "cts_testlist.h"
+
 void cts_task(void)
 {
 	enum cts_rc rc;
@@ -147,8 +142,9 @@ void cts_task(void)
 
 	for (i = 0; i < CTS_TEST_ID_COUNT; i++) {
 		clear_state();
+		CPRINTF("\n%s start\n", tests[i].name);
 		rc = tests[i].run();
-		CPRINTF("\n%s %d\n", tests[i].name, rc);
+		CPRINTF("\n%s end %d\n", tests[i].name, rc);
 		cflush();
 	}
 
diff --git a/cts/timer/cts.testlist b/cts/timer/cts.testlist
index d4cb3c30b3..9b5da0d6c9 100644
--- a/cts/timer/cts.testlist
+++ b/cts/timer/cts.testlist
@@ -16,4 +16,4 @@
  * - GPIO_OUTPUT connection for sending notification from DUT
  * - Calibrated TH timer
  */
-CTS_TEST(timer_calibration_test)
+CTS_TEST(timer_calibration_test,,,,)
diff --git a/cts/timer/dut.c b/cts/timer/dut.c
index 29003f36af..4089964791 100644
--- a/cts/timer/dut.c
+++ b/cts/timer/dut.c
@@ -29,8 +29,9 @@ void cts_task(void)
 
 	for (i = 0; i < CTS_TEST_ID_COUNT; i++) {
 		sync();
+		CPRINTF("\n%s start\n", tests[i].name);
 		rc = tests[i].run();
-		CPRINTF("\n%s %d\n", tests[i].name, rc);
+		CPRINTF("\n%s end %d\n", tests[i].name, rc);
 		cflush();
 	}
 
diff --git a/cts/timer/th.c b/cts/timer/th.c
index a6e9d575b1..4c9912fdda 100644
--- a/cts/timer/th.c
+++ b/cts/timer/th.c
@@ -62,8 +62,9 @@ void cts_task(void)
 
 	for (i = 0; i < CTS_TEST_ID_COUNT; i++) {
 		sync();
+		CPRINTF("\n%s start\n", tests[i].name);
 		rc = tests[i].run();
-		CPRINTF("\n%s %d\n", tests[i].name, rc);
+		CPRINTF("\n%s end %d\n", tests[i].name, rc);
 		cflush();
 	}