diff options
author | Tom Hughes <tomhughes@chromium.org> | 2022-09-21 14:08:36 -0700 |
---|---|---|
committer | Tom Hughes <tomhughes@chromium.org> | 2022-09-22 12:59:38 -0700 |
commit | c453fd704268ef72de871b0c5ac7a989de662334 (patch) | |
tree | fcf6ce5810f9ff9e3c8cce434812dd75492269ed /cts/cts.py | |
parent | 6c1587ca70f558b4f96b3f0b18ad8b027d3ba99d (diff) | |
parent | 28712dae9d7ed1e694f7622cc083afa71090d4d5 (diff) | |
download | chrome-ec-firmware-fpmcu-dartmonkey-release.tar.gz |
Merge remote-tracking branch cros/main into firmware-fpmcu-dartmonkey-releasefirmware-fpmcu-dartmonkey-release
Generated by: ./util/update_release_branch.py --board dartmonkey --relevant_paths_file
./util/fingerprint-relevant-paths.txt firmware-fpmcu-dartmonkey-release
Relevant changes:
git log --oneline 6c1587ca70..28712dae9d -- board/nocturne_fp
board/dartmonkey common/fpsensor docs/fingerprint driver/fingerprint
util/getversion.sh
ded9307b79 util/getversion.sh: Fix version when not in a git repo
956055e692 board: change Google USB vendor info
71b2ef709d Update license boilerplate text in source code files
33e11afda0 Revert "fpsensor: Build fpsensor source file with C++"
c8d0360723 fpsensor: Build fpsensor source file with C++
bc113abd53 fpsensor: Fix g++ compiler error
150a58a0dc fpsensor: Fix fp_set_sensor_mode return type
b33b5ce85b fpsensor: Remove nested designators for C++ compatibility
2e864b2539 tree-wide: const-ify argv for console commands
56d8b360f9 test: Add test for get ikm failure when seed not set
3a3d6c3690 test: Add test for fpsensor trivial key failure
233e6bbd08 fpsensor_crypto: Abstract calls to hmac_SHA256
0a041b285b docs/fingerprint: Typo correction
c03fab67e2 docs/fingerprint: Fix the path of fputils.py
0b5d4baf5a util/getversion.sh: Fix empty file list handling
6e128fe760 FPMCU dev board environment with Satlab
3eb29b6aa5 builtin: Move ssize_t to sys/types.h
345d62ebd1 docs/fingerprint: Update power numbers for latest dartmonkey release
c25ffdb316 common: Conditionally support printf %l and %i modifiers
9a3c514b45 test: Add a test to check if the debugger is connected
54e603413f Move standard library tests to their own file
43fa6b4bf8 docs/fingerprint: Update power numbers for latest bloonchipper release
25536f9a84 driver/fingerprint/fpc/bep/fpc_sensor_spi.c: Format with clang-format
4face99efd driver/fingerprint/fpc/libfp/fpc_sensor_pal.h: Format with clang-format
738de2b575 trng: Rename rand to trng_rand
14b8270edd docs/fingerprint: Update dragonclaw power numbers
0b268f93d1 driver/fingerprint/fpc/libfp/fpc_private.c: Format with clang-format
f80da163f2 driver/fingerprint/fpc/libfp/fpc_private.h: Format with clang-format
a0751778f4 board/nocturne_fp/ro_workarounds.c: Format with clang-format
5e9c85c9b1 driver/fingerprint/fpc/libfp/fpc_sensor_pal.c: Format with clang-format
c1f9dd3cf8 driver/fingerprint/fpc/libfp/fpc_bio_algorithm.h: Format with clang-format
eb1e1bed8d driver/fingerprint/fpc/libfp/fpc1145_private.h: Format with clang-format
6e7b611821 driver/fingerprint/fpc/bep/fpc_bio_algorithm.h: Format with clang-format
e0589cd5e2 driver/fingerprint/fpc/bep/fpc1035_private.h: Format with clang-format
58f0246dbe board/nocturne_fp/board_ro.c: Format with clang-format
7905e556a0 common/fpsensor/fpsensor_crypto.c: Format with clang-format
21289d170c driver/fingerprint/fpc/bep/fpc1025_private.h: Format with clang-format
98a20f937e common/fpsensor/fpsensor_state.c: Format with clang-format
a2d255d8af common/fpsensor/fpsensor.c: Format with clang-format
84e53a65da board/nocturne_fp/board.h: Format with clang-format
73055eeb3f driver/fingerprint/fpc/bep/fpc_private.c: Format with clang-format
0f7b5cb509 common/fpsensor/fpsensor_private.h: Format with clang-format
1ceade6e65 driver/fingerprint/fpc/bep/fpc_private.h: Format with clang-format
dca9d74321 Revert "trng: Rename rand to trng_rand"
a6b0b3554f trng: Rename rand to trng_rand
28d0b75b70 third_party/boringssl: Remove unused header
BRANCH=None
BUG=b:244387210 b:242720240 b:215613183 b:242720910 b:236386294
BUG=b:234181908 b:244781166 b:234781655 b:234143158 b:234181908
BUG=b:237344361 b:236025198 b:234181908 b:180945056 chromium:1098010
BUG=b:246424843 b:234181908 b:131913998
TEST=`make -j buildall`
TEST=./util/run_device_tests.py --board dartmonkey
Test "aes": PASSED
Test "cec": PASSED
Test "cortexm_fpu": PASSED
Test "crc": PASSED
Test "flash_physical": PASSED
Test "flash_write_protect": PASSED
Test "fpsensor_hw": PASSED
Test "fpsensor_spi_ro": PASSED
Test "fpsensor_spi_rw": PASSED
Test "fpsensor_uart_ro": PASSED
Test "fpsensor_uart_rw": PASSED
Test "mpu_ro": PASSED
Test "mpu_rw": PASSED
Test "mutex": PASSED
Test "pingpong": PASSED
Test "printf": PASSED
Test "queue": PASSED
Test "rollback_region0": PASSED
Test "rollback_region1": PASSED
Test "rollback_entropy": PASSED
Test "rtc": PASSED
Test "sha256": PASSED
Test "sha256_unrolled": PASSED
Test "static_if": PASSED
Test "stdlib": PASSED
Test "system_is_locked_wp_on": PASSED
Test "system_is_locked_wp_off": PASSED
Test "timer_dos": PASSED
Test "utils": PASSED
Test "utils_str": PASSED
Test "panic_data_dartmonkey_v2.0.2887": PASSED
Test "panic_data_nocturne_fp_v2.2.64": PASSED
Test "panic_data_nami_fp_v2.2.144": PASSED
Force-Relevant-Builds: all
Signed-off-by: Tom Hughes <tomhughes@chromium.org>
Change-Id: I2c312583a709fedae8fe11d92c22328c3b634bc7
Diffstat (limited to 'cts/cts.py')
-rwxr-xr-x | cts/cts.py | 829 |
1 files changed, 426 insertions, 403 deletions
diff --git a/cts/cts.py b/cts/cts.py index c3e0335cab..f61de47398 100755 --- a/cts/cts.py +++ b/cts/cts.py @@ -1,12 +1,8 @@ #!/usr/bin/env python # -# Copyright 2016 The Chromium OS Authors. All rights reserved. +# Copyright 2016 The ChromiumOS Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. -# -# Ignore indention messages, since legacy scripts use 2 spaces instead of 4. -# pylint: disable=bad-indentation,docstring-section-indent -# pylint: disable=docstring-trailing-quotes # A script which builds, flashes, and runs EC CTS # @@ -28,416 +24,443 @@ import argparse import os import shutil import time -import common.board as board +import common.board as board -CTS_RC_PREFIX = 'CTS_RC_' -DEFAULT_TH = 'stm32l476g-eval' -DEFAULT_DUT = 'nucleo-f072rb' +CTS_RC_PREFIX = "CTS_RC_" +DEFAULT_TH = "stm32l476g-eval" +DEFAULT_DUT = "nucleo-f072rb" MAX_SUITE_TIME_SEC = 5 -CTS_TEST_RESULT_DIR = '/tmp/ects' +CTS_TEST_RESULT_DIR = "/tmp/ects" # Host only return codes. Make sure they match values in cts.rc -CTS_RC_DID_NOT_START = -1 # test did not run. -CTS_RC_DID_NOT_END = -2 # test did not run. -CTS_RC_DUPLICATE_RUN = -3 # test was run multiple times. -CTS_RC_INVALID_RETURN_CODE = -4 # failed to parse return code +CTS_RC_DID_NOT_START = -1 # test did not run. +CTS_RC_DID_NOT_END = -2 # test did not run. +CTS_RC_DUPLICATE_RUN = -3 # test was run multiple times. +CTS_RC_INVALID_RETURN_CODE = -4 # failed to parse return code class Cts(object): - """Class that represents a eCTS run. - - Attributes: - dut: DeviceUnderTest object representing DUT - th: TestHarness object representing a test harness - module: Name of module to build/run tests for - testlist: List of strings of test names contained in given module - return_codes: Dict of strings of return codes, with a code's integer - value being the index for the corresponding string representation - """ - - def __init__(self, ec_dir, th, dut, module): - """Initializes cts class object with given arguments. - - Args: - ec_dir: Path to ec directory - th: Name of the test harness board - dut: Name of the device under test board - module: Name of module to build/run tests for (e.g. gpio, interrupt) - """ - self.results_dir = os.path.join(CTS_TEST_RESULT_DIR, dut, module) - if os.path.isdir(self.results_dir): - shutil.rmtree(self.results_dir) - else: - os.makedirs(self.results_dir) - self.ec_dir = ec_dir - self.module = module - serial_path = os.path.join(CTS_TEST_RESULT_DIR, 'th_serial') - self.th = board.TestHarness(th, module, self.results_dir, serial_path) - self.dut = board.DeviceUnderTest(dut, self.th, module, self.results_dir) - cts_dir = os.path.join(self.ec_dir, 'cts') - testlist_path = os.path.join(cts_dir, self.module, 'cts.testlist') - return_codes_path = os.path.join(cts_dir, 'common', 'cts.rc') - self.get_return_codes(return_codes_path) - self.testlist = self.get_macro_args(testlist_path, 'CTS_TEST') - - def build(self): - """Build images for DUT and TH.""" - print('Building DUT image...') - if not self.dut.build(self.ec_dir): - raise RuntimeError('Building module %s for DUT failed' % (self.module)) - print('Building TH image...') - if not self.th.build(self.ec_dir): - raise RuntimeError('Building module %s for TH failed' % (self.module)) - - def flash_boards(self): - """Flashes TH and DUT with their most recently built ec.bin.""" - cts_module = 'cts_' + self.module - image_path = os.path.join('build', self.th.board, cts_module, 'ec.bin') - self.identify_boards() - print('Flashing TH with', image_path) - if not self.th.flash(image_path): - raise RuntimeError('Flashing TH failed') - image_path = os.path.join('build', self.dut.board, cts_module, 'ec.bin') - print('Flashing DUT with', image_path) - if not self.dut.flash(image_path): - raise RuntimeError('Flashing DUT failed') - - def setup(self): - """Setup boards.""" - self.th.save_serial() - - def identify_boards(self): - """Updates serials of TH and DUT in that order (order matters).""" - self.th.get_serial() - self.dut.get_serial() - - def get_macro_args(self, filepath, macro): - """Get list of args of a macro in a file when macro. - - Args: - filepath: String containing absolute path to the file - macro: String containing text of macro to get args of - - Returns: - List of dictionaries where each entry is: - 'name': Test name, - 'th_string': Expected string from TH, - 'dut_string': Expected string from DUT, + """Class that represents a eCTS run. + + Attributes: + dut: DeviceUnderTest object representing DUT + th: TestHarness object representing a test harness + module: Name of module to build/run tests for + testlist: List of strings of test names contained in given module + return_codes: Dict of strings of return codes, with a code's integer + value being the index for the corresponding string representation """ - tests = [] - with open(filepath, 'r') as f: - lines = f.readlines() - joined = ''.join(lines).replace('\\\n', '').splitlines() - for l in joined: - if not l.strip().startswith(macro): - continue - d = {} - l = l.strip()[len(macro):] - l = l.strip('()').split(',') - d['name'] = l[0].strip() - d['th_rc'] = self.get_return_code_value(l[1].strip().strip('"')) - d['th_string'] = l[2].strip().strip('"') - d['dut_rc'] = self.get_return_code_value(l[3].strip().strip('"')) - d['dut_string'] = l[4].strip().strip('"') - tests.append(d) - return tests - - def get_return_codes(self, filepath): - """Read return code names from the return code definition file.""" - self.return_codes = {} - val = 0 - with open(filepath, 'r') as f: - for line in f: - line = line.strip() - if not line.startswith(CTS_RC_PREFIX): - continue - line = line.split(',')[0] - if '=' in line: - tokens = line.split('=') - line = tokens[0].strip() - val = int(tokens[1].strip()) - self.return_codes[line] = val - val += 1 - - def parse_output(self, output): - """Parse console output from DUT or TH. - - Args: - output: String containing consoule output - - Returns: - List of dictionaries where each key and value are: - name = 'ects_test_x', - started = True/False, - ended = True/False, - rc = CTS_RC_*, - output = All text between 'ects_test_x start' and 'ects_test_x end' - """ - results = [] - i = 0 - for test in self.testlist: - results.append({}) - results[i]['name'] = test['name'] - results[i]['started'] = False - results[i]['rc'] = CTS_RC_DID_NOT_START - results[i]['string'] = False - results[i]['output'] = [] - i += 1 - - i = 0 - for ln in [ln.strip() for ln in output.split('\n')]: - if i + 1 > len(results): - break - tokens = ln.split() - if len(tokens) >= 2: - if tokens[0].strip() == results[i]['name']: - if tokens[1].strip() == 'start': - # start line found - if results[i]['started']: # Already started - results[i]['rc'] = CTS_RC_DUPLICATE_RUN - else: - results[i]['rc'] = CTS_RC_DID_NOT_END - results[i]['started'] = True - continue - elif results[i]['started'] and tokens[1].strip() == 'end': - # end line found - results[i]['rc'] = CTS_RC_INVALID_RETURN_CODE - if len(tokens) == 3: - try: - results[i]['rc'] = int(tokens[2].strip()) - except ValueError: - pass - # Since index is incremented when 'end' is encountered, we don't - # need to check duplicate 'end'. - i += 1 - continue - if results[i]['started']: - results[i]['output'].append(ln) - - return results - - def get_return_code_name(self, code, strip_prefix=False): - name = '' - for k, v in self.return_codes.items(): - if v == code: - if strip_prefix: - name = k[len(CTS_RC_PREFIX):] - else: - name = k - return name - - def get_return_code_value(self, name): - if name: - return self.return_codes[name] - return 0 - - def evaluate_run(self, dut_output, th_output): - """Parse outputs to derive test results. - Args: - dut_output: String output of DUT - th_output: String output of TH - - Returns: - th_results: list of test results for TH - dut_results: list of test results for DUT - """ - th_results = self.parse_output(th_output) - dut_results = self.parse_output(dut_output) - - # Search for expected string in each output - for i, v in enumerate(self.testlist): - if v['th_string'] in th_results[i]['output'] or not v['th_string']: - th_results[i]['string'] = True - if v['dut_string'] in dut_results[i]['output'] or not v['dut_string']: - dut_results[i]['string'] = True + def __init__(self, ec_dir, th, dut, module): + """Initializes cts class object with given arguments. + + Args: + ec_dir: Path to ec directory + th: Name of the test harness board + dut: Name of the device under test board + module: Name of module to build/run tests for (e.g. gpio, interrupt) + """ + self.results_dir = os.path.join(CTS_TEST_RESULT_DIR, dut, module) + if os.path.isdir(self.results_dir): + shutil.rmtree(self.results_dir) + else: + os.makedirs(self.results_dir) + self.ec_dir = ec_dir + self.module = module + serial_path = os.path.join(CTS_TEST_RESULT_DIR, "th_serial") + self.th = board.TestHarness(th, module, self.results_dir, serial_path) + self.dut = board.DeviceUnderTest(dut, self.th, module, self.results_dir) + cts_dir = os.path.join(self.ec_dir, "cts") + testlist_path = os.path.join(cts_dir, self.module, "cts.testlist") + return_codes_path = os.path.join(cts_dir, "common", "cts.rc") + self.get_return_codes(return_codes_path) + self.testlist = self.get_macro_args(testlist_path, "CTS_TEST") + + def build(self): + """Build images for DUT and TH.""" + print("Building DUT image...") + if not self.dut.build(self.ec_dir): + raise RuntimeError( + "Building module %s for DUT failed" % (self.module) + ) + print("Building TH image...") + if not self.th.build(self.ec_dir): + raise RuntimeError( + "Building module %s for TH failed" % (self.module) + ) + + def flash_boards(self): + """Flashes TH and DUT with their most recently built ec.bin.""" + cts_module = "cts_" + self.module + image_path = os.path.join("build", self.th.board, cts_module, "ec.bin") + self.identify_boards() + print("Flashing TH with", image_path) + if not self.th.flash(image_path): + raise RuntimeError("Flashing TH failed") + image_path = os.path.join("build", self.dut.board, cts_module, "ec.bin") + print("Flashing DUT with", image_path) + if not self.dut.flash(image_path): + raise RuntimeError("Flashing DUT failed") + + def setup(self): + """Setup boards.""" + self.th.save_serial() + + def identify_boards(self): + """Updates serials of TH and DUT in that order (order matters).""" + self.th.get_serial() + self.dut.get_serial() + + def get_macro_args(self, filepath, macro): + """Get list of args of a macro in a file when macro. + + Args: + filepath: String containing absolute path to the file + macro: String containing text of macro to get args of + + Returns: + List of dictionaries where each entry is: + 'name': Test name, + 'th_string': Expected string from TH, + 'dut_string': Expected string from DUT, + """ + tests = [] + with open(filepath, "r") as f: + lines = f.readlines() + joined = "".join(lines).replace("\\\n", "").splitlines() + for l in joined: + if not l.strip().startswith(macro): + continue + d = {} + l = l.strip()[len(macro) :] + l = l.strip("()").split(",") + d["name"] = l[0].strip() + d["th_rc"] = self.get_return_code_value(l[1].strip().strip('"')) + d["th_string"] = l[2].strip().strip('"') + d["dut_rc"] = self.get_return_code_value( + l[3].strip().strip('"') + ) + d["dut_string"] = l[4].strip().strip('"') + tests.append(d) + return tests + + def get_return_codes(self, filepath): + """Read return code names from the return code definition file.""" + self.return_codes = {} + val = 0 + with open(filepath, "r") as f: + for line in f: + line = line.strip() + if not line.startswith(CTS_RC_PREFIX): + continue + line = line.split(",")[0] + if "=" in line: + tokens = line.split("=") + line = tokens[0].strip() + val = int(tokens[1].strip()) + self.return_codes[line] = val + val += 1 + + def parse_output(self, output): + """Parse console output from DUT or TH. + + Args: + output: String containing consoule output + + Returns: + List of dictionaries where each key and value are: + name = 'ects_test_x', + started = True/False, + ended = True/False, + rc = CTS_RC_*, + output = All text between 'ects_test_x start' and 'ects_test_x end' + """ + results = [] + i = 0 + for test in self.testlist: + results.append({}) + results[i]["name"] = test["name"] + results[i]["started"] = False + results[i]["rc"] = CTS_RC_DID_NOT_START + results[i]["string"] = False + results[i]["output"] = [] + i += 1 - return th_results, dut_results + i = 0 + for ln in [ln.strip() for ln in output.split("\n")]: + if i + 1 > len(results): + break + tokens = ln.split() + if len(tokens) >= 2: + if tokens[0].strip() == results[i]["name"]: + if tokens[1].strip() == "start": + # start line found + if results[i]["started"]: # Already started + results[i]["rc"] = CTS_RC_DUPLICATE_RUN + else: + results[i]["rc"] = CTS_RC_DID_NOT_END + results[i]["started"] = True + continue + elif results[i]["started"] and tokens[1].strip() == "end": + # end line found + results[i]["rc"] = CTS_RC_INVALID_RETURN_CODE + if len(tokens) == 3: + try: + results[i]["rc"] = int(tokens[2].strip()) + except ValueError: + pass + # Since index is incremented when 'end' is encountered, we don't + # need to check duplicate 'end'. + i += 1 + continue + if results[i]["started"]: + results[i]["output"].append(ln) + + return results + + def get_return_code_name(self, code, strip_prefix=False): + name = "" + for k, v in self.return_codes.items(): + if v == code: + if strip_prefix: + name = k[len(CTS_RC_PREFIX) :] + else: + name = k + return name + + def get_return_code_value(self, name): + if name: + return self.return_codes[name] + return 0 + + def evaluate_run(self, dut_output, th_output): + """Parse outputs to derive test results. + + Args: + dut_output: String output of DUT + th_output: String output of TH + + Returns: + th_results: list of test results for TH + dut_results: list of test results for DUT + """ + th_results = self.parse_output(th_output) + dut_results = self.parse_output(dut_output) + + # Search for expected string in each output + for i, v in enumerate(self.testlist): + if v["th_string"] in th_results[i]["output"] or not v["th_string"]: + th_results[i]["string"] = True + if ( + v["dut_string"] in dut_results[i]["output"] + or not v["dut_string"] + ): + dut_results[i]["string"] = True + + return th_results, dut_results + + def print_result(self, th_results, dut_results): + """Print results to the screen. + + Args: + th_results: list of test results for TH + dut_results: list of test results for DUT + """ + len_test_name = max(len(s["name"]) for s in self.testlist) + len_code_name = max( + len(self.get_return_code_name(v, True)) + for v in self.return_codes.values() + ) + + head = "{:^" + str(len_test_name) + "} " + head += "{:^" + str(len_code_name) + "} " + head += "{:^" + str(len_code_name) + "}" + head += "{:^" + str(len(" TH_STR")) + "}" + head += "{:^" + str(len(" DUT_STR")) + "}" + head += "{:^" + str(len(" RESULT")) + "}\n" + fmt = "{:" + str(len_test_name) + "} " + fmt += "{:>" + str(len_code_name) + "} " + fmt += "{:>" + str(len_code_name) + "}" + fmt += "{:>" + str(len(" TH_STR")) + "}" + fmt += "{:>" + str(len(" DUT_STR")) + "}" + fmt += "{:>" + str(len(" RESULT")) + "}\n" + + self.formatted_results = head.format( + "TEST NAME", "TH_RC", "DUT_RC", " TH_STR", " DUT_STR", " RESULT" + ) + for i, d in enumerate(dut_results): + th_cn = self.get_return_code_name(th_results[i]["rc"], True) + dut_cn = self.get_return_code_name(dut_results[i]["rc"], True) + th_res = self.evaluate_result( + th_results[i], + self.testlist[i]["th_rc"], + self.testlist[i]["th_string"], + ) + dut_res = self.evaluate_result( + dut_results[i], + self.testlist[i]["dut_rc"], + self.testlist[i]["dut_string"], + ) + self.formatted_results += fmt.format( + d["name"], + th_cn, + dut_cn, + "YES" if th_results[i]["string"] else "NO", + "YES" if dut_results[i]["string"] else "NO", + "PASS" if th_res and dut_res else "FAIL", + ) + + def evaluate_result(self, result, expected_rc, expected_string): + if result["rc"] != expected_rc: + return False + if expected_string and expected_string not in result["output"]: + return False + return True + + def run(self): + """Resets boards, records test results in results dir.""" + print("Reading serials...") + self.identify_boards() + print("Opening DUT tty...") + self.dut.setup_tty() + print("Opening TH tty...") + self.th.setup_tty() + + # Boards might be still writing to tty. Wait a few seconds before flashing. + time.sleep(3) + + # clear buffers + print("Clearing DUT tty...") + self.dut.read_tty() + print("Clearing TH tty...") + self.th.read_tty() + + # Resets the boards and allows them to run tests + # Due to current (7/27/16) version of sync function, + # both boards must be rest and halted, with the th + # resuming first, in order for the test suite to run in sync + print("Halting TH...") + if not self.th.reset_halt(): + raise RuntimeError("Failed to halt TH") + print("Halting DUT...") + if not self.dut.reset_halt(): + raise RuntimeError("Failed to halt DUT") + print("Resuming TH...") + if not self.th.resume(): + raise RuntimeError("Failed to resume TH") + print("Resuming DUT...") + if not self.dut.resume(): + raise RuntimeError("Failed to resume DUT") + + time.sleep(MAX_SUITE_TIME_SEC) + + print("Reading DUT tty...") + dut_output, _ = self.dut.read_tty() + self.dut.close_tty() + print("Reading TH tty...") + th_output, _ = self.th.read_tty() + self.th.close_tty() + + print("Halting TH...") + if not self.th.reset_halt(): + raise RuntimeError("Failed to halt TH") + print("Halting DUT...") + if not self.dut.reset_halt(): + raise RuntimeError("Failed to halt DUT") + + if not dut_output or not th_output: + raise ValueError( + "Output missing from boards. If you have a process " + "reading ttyACMx, please kill that process and try " + "again." + ) + + print("Pursing results...") + th_results, dut_results = self.evaluate_run(dut_output, th_output) + + # Print out results + self.print_result(th_results, dut_results) + + # Write results + dest = os.path.join(self.results_dir, "results.log") + with open(dest, "w") as fl: + fl.write(self.formatted_results) + + # Write UART outputs + dest = os.path.join(self.results_dir, "uart_th.log") + with open(dest, "w") as fl: + fl.write(th_output) + dest = os.path.join(self.results_dir, "uart_dut.log") + with open(dest, "w") as fl: + fl.write(dut_output) + + print(self.formatted_results) + + # TODO(chromium:735652): Should set exit code for the shell - def print_result(self, th_results, dut_results): - """Print results to the screen. - Args: - th_results: list of test results for TH - dut_results: list of test results for DUT - """ - len_test_name = max(len(s['name']) for s in self.testlist) - len_code_name = max(len(self.get_return_code_name(v, True)) - for v in self.return_codes.values()) - - head = '{:^' + str(len_test_name) + '} ' - head += '{:^' + str(len_code_name) + '} ' - head += '{:^' + str(len_code_name) + '}' - head += '{:^' + str(len(' TH_STR')) + '}' - head += '{:^' + str(len(' DUT_STR')) + '}' - head += '{:^' + str(len(' RESULT')) + '}\n' - fmt = '{:' + str(len_test_name) + '} ' - fmt += '{:>' + str(len_code_name) + '} ' - fmt += '{:>' + str(len_code_name) + '}' - fmt += '{:>' + str(len(' TH_STR')) + '}' - fmt += '{:>' + str(len(' DUT_STR')) + '}' - fmt += '{:>' + str(len(' RESULT')) + '}\n' - - self.formatted_results = head.format( - 'TEST NAME', 'TH_RC', 'DUT_RC', - ' TH_STR', ' DUT_STR', ' RESULT') - for i, d in enumerate(dut_results): - th_cn = self.get_return_code_name(th_results[i]['rc'], True) - dut_cn = self.get_return_code_name(dut_results[i]['rc'], True) - th_res = self.evaluate_result(th_results[i], - self.testlist[i]['th_rc'], - self.testlist[i]['th_string']) - dut_res = self.evaluate_result(dut_results[i], - self.testlist[i]['dut_rc'], - self.testlist[i]['dut_string']) - self.formatted_results += fmt.format( - d['name'], th_cn, dut_cn, - 'YES' if th_results[i]['string'] else 'NO', - 'YES' if dut_results[i]['string'] else 'NO', - 'PASS' if th_res and dut_res else 'FAIL') - - def evaluate_result(self, result, expected_rc, expected_string): - if result['rc'] != expected_rc: - return False - if expected_string and expected_string not in result['output']: - return False - return True - - def run(self): - """Resets boards, records test results in results dir.""" - print('Reading serials...') - self.identify_boards() - print('Opening DUT tty...') - self.dut.setup_tty() - print('Opening TH tty...') - self.th.setup_tty() - - # Boards might be still writing to tty. Wait a few seconds before flashing. - time.sleep(3) - - # clear buffers - print('Clearing DUT tty...') - self.dut.read_tty() - print('Clearing TH tty...') - self.th.read_tty() - - # Resets the boards and allows them to run tests - # Due to current (7/27/16) version of sync function, - # both boards must be rest and halted, with the th - # resuming first, in order for the test suite to run in sync - print('Halting TH...') - if not self.th.reset_halt(): - raise RuntimeError('Failed to halt TH') - print('Halting DUT...') - if not self.dut.reset_halt(): - raise RuntimeError('Failed to halt DUT') - print('Resuming TH...') - if not self.th.resume(): - raise RuntimeError('Failed to resume TH') - print('Resuming DUT...') - if not self.dut.resume(): - raise RuntimeError('Failed to resume DUT') - - time.sleep(MAX_SUITE_TIME_SEC) - - print('Reading DUT tty...') - dut_output, _ = self.dut.read_tty() - self.dut.close_tty() - print('Reading TH tty...') - th_output, _ = self.th.read_tty() - self.th.close_tty() - - print('Halting TH...') - if not self.th.reset_halt(): - raise RuntimeError('Failed to halt TH') - print('Halting DUT...') - if not self.dut.reset_halt(): - raise RuntimeError('Failed to halt DUT') - - if not dut_output or not th_output: - raise ValueError('Output missing from boards. If you have a process ' - 'reading ttyACMx, please kill that process and try ' - 'again.') - - print('Pursing results...') - th_results, dut_results = self.evaluate_run(dut_output, th_output) - - # Print out results - self.print_result(th_results, dut_results) - - # Write results - dest = os.path.join(self.results_dir, 'results.log') - with open(dest, 'w') as fl: - fl.write(self.formatted_results) - - # Write UART outputs - dest = os.path.join(self.results_dir, 'uart_th.log') - with open(dest, 'w') as fl: - fl.write(th_output) - dest = os.path.join(self.results_dir, 'uart_dut.log') - with open(dest, 'w') as fl: - fl.write(dut_output) - - print(self.formatted_results) - - # TODO(chromium:735652): Should set exit code for the shell +def main(): + ec_dir = os.path.realpath( + os.path.join(os.path.dirname(os.path.abspath(__file__)), "..") + ) + os.chdir(ec_dir) + + dut = DEFAULT_DUT + module = "meta" + + parser = argparse.ArgumentParser(description="Used to build/flash boards") + parser.add_argument( + "-d", "--dut", help="Specify DUT you want to build/flash" + ) + parser.add_argument( + "-m", "--module", help="Specify module you want to build/flash" + ) + parser.add_argument( + "-s", + "--setup", + action="store_true", + help="Connect only the TH to save its serial", + ) + parser.add_argument( + "-b", + "--build", + action="store_true", + help="Build test suite (no flashing)", + ) + parser.add_argument( + "-f", + "--flash", + action="store_true", + help="Flash boards with most recent images", + ) + parser.add_argument( + "-r", "--run", action="store_true", help="Run tests without flashing" + ) + + args = parser.parse_args() + + if args.module: + module = args.module + + if args.dut: + dut = args.dut + + cts = Cts(ec_dir, DEFAULT_TH, dut=dut, module=module) + + if args.setup: + cts.setup() + elif args.build: + cts.build() + elif args.flash: + cts.flash_boards() + elif args.run: + cts.run() + else: + cts.build() + cts.flash_boards() + cts.run() -def main(): - ec_dir = os.path.realpath(os.path.join( - os.path.dirname(os.path.abspath(__file__)), '..')) - os.chdir(ec_dir) - - dut = DEFAULT_DUT - module = 'meta' - - parser = argparse.ArgumentParser(description='Used to build/flash boards') - parser.add_argument('-d', - '--dut', - help='Specify DUT you want to build/flash') - parser.add_argument('-m', - '--module', - help='Specify module you want to build/flash') - parser.add_argument('-s', - '--setup', - action='store_true', - help='Connect only the TH to save its serial') - parser.add_argument('-b', - '--build', - action='store_true', - help='Build test suite (no flashing)') - parser.add_argument('-f', - '--flash', - action='store_true', - help='Flash boards with most recent images') - parser.add_argument('-r', - '--run', - action='store_true', - help='Run tests without flashing') - - args = parser.parse_args() - - if args.module: - module = args.module - - if args.dut: - dut = args.dut - - cts = Cts(ec_dir, DEFAULT_TH, dut=dut, module=module) - - if args.setup: - cts.setup() - elif args.build: - cts.build() - elif args.flash: - cts.flash_boards() - elif args.run: - cts.run() - else: - cts.build() - cts.flash_boards() - cts.run() - -if __name__ == '__main__': - main() +if __name__ == "__main__": + main() |