diff options
Diffstat (limited to 'chromium/build/android/pylib/local')
4 files changed, 250 insertions, 132 deletions
diff --git a/chromium/build/android/pylib/local/device/local_device_instrumentation_test_run.py b/chromium/build/android/pylib/local/device/local_device_instrumentation_test_run.py index 6a64e190969..5a46e6fcb1c 100644 --- a/chromium/build/android/pylib/local/device/local_device_instrumentation_test_run.py +++ b/chromium/build/android/pylib/local/device/local_device_instrumentation_test_run.py @@ -36,17 +36,16 @@ from pylib.instrumentation import instrumentation_test_instance from pylib.local.device import local_device_environment from pylib.local.device import local_device_test_run from pylib.output import remote_output_manager +from pylib.utils import chrome_proxy_utils from pylib.utils import gold_utils from pylib.utils import instrumentation_tracing from pylib.utils import shared_preference_utils - from py_trace_event import trace_event from py_trace_event import trace_time from py_utils import contextlib_ext from py_utils import tempfile_ext import tombstones - with host_paths.SysPath( os.path.join(host_paths.DIR_SOURCE_ROOT, 'third_party'), 0): import jinja2 # pylint: disable=import-error @@ -57,6 +56,10 @@ _JINJA_TEMPLATE_DIR = os.path.join( host_paths.DIR_SOURCE_ROOT, 'build', 'android', 'pylib', 'instrumentation') _JINJA_TEMPLATE_FILENAME = 'render_test.html.jinja' +_WPR_GO_LINUX_X86_64_PATH = os.path.join(host_paths.DIR_SOURCE_ROOT, + 'third_party', 'webpagereplay', 'bin', + 'linux', 'x86_64', 'wpr') + _TAG = 'test_runner_py' TIMEOUT_ANNOTATIONS = [ @@ -88,6 +91,8 @@ _EXTRA_PACKAGE_UNDER_TEST = ('org.chromium.chrome.test.pagecontroller.rules.' FEATURE_ANNOTATION = 'Feature' RENDER_TEST_FEATURE_ANNOTATION = 'RenderTest' +WPR_ARCHIVE_FILE_PATH_ANNOTATION = 'WPRArchiveDirectory' +WPR_RECORD_REPLAY_TEST_FEATURE_ANNOTATION = 'WPRRecordReplayTest' # This needs to be kept in sync with formatting in |RenderUtils.imageName| RE_RENDER_IMAGE_NAME = re.compile( @@ -101,6 +106,8 @@ RENDER_TEST_MODEL_SDK_CONFIGS = { 'Nexus 5X': [23], } +_TEST_BATCH_MAX_GROUP_SIZE = 256 + @contextlib.contextmanager def _LogTestEndpoints(device, test_name): @@ -136,16 +143,24 @@ _CURRENT_FOCUS_CRASH_RE = re.compile( r'\s*mCurrentFocus.*Application (Error|Not Responding): (\S+)}') +def _GetTargetPackageName(test_apk): + # apk_under_test does not work for smoke tests, where it is set to an + # apk that is not listed as the targetPackage in the test apk's manifest. + return test_apk.GetAllInstrumentations()[0]['android:targetPackage'] + + class LocalDeviceInstrumentationTestRun( local_device_test_run.LocalDeviceTestRun): def __init__(self, env, test_instance): super(LocalDeviceInstrumentationTestRun, self).__init__( env, test_instance) + self._chrome_proxy = None self._context_managers = collections.defaultdict(list) self._flag_changers = {} + self._render_tests_device_output_dir = None self._shared_prefs_to_restore = [] - self._skia_gold_work_dir = None self._skia_gold_session_manager = None + self._skia_gold_work_dir = None #override def TestPackage(self): @@ -153,6 +168,8 @@ class LocalDeviceInstrumentationTestRun( #override def SetUp(self): + target_package = _GetTargetPackageName(self._test_instance.test_apk) + @local_device_environment.handle_shard_failures_with( self._env.BlacklistDevice) @trace_event.traced @@ -267,18 +284,10 @@ class LocalDeviceInstrumentationTestRun( def set_debug_app(dev): # Set debug app in order to enable reading command line flags on user # builds - package_name = None - if self._test_instance.apk_under_test: - package_name = self._test_instance.apk_under_test.GetPackageName() - elif self._test_instance.test_apk: - package_name = self._test_instance.test_apk.GetPackageName() - else: - logging.error("Couldn't set debug app: no package name found") - return cmd = ['am', 'set-debug-app', '--persistent'] if self._test_instance.wait_for_java_debugger: cmd.append('-w') - cmd.append(package_name) + cmd.append(target_package) dev.RunShellCommand(cmd, check_return=True) @trace_event.traced @@ -379,13 +388,12 @@ class LocalDeviceInstrumentationTestRun( # expectations can be re-used between tests, saving a significant amount # of time. self._skia_gold_work_dir = tempfile.mkdtemp() - self._skia_gold_session_manager = gold_utils.SkiaGoldSessionManager( + self._skia_gold_session_manager = gold_utils.AndroidSkiaGoldSessionManager( self._skia_gold_work_dir, self._test_instance.skia_gold_properties) if self._test_instance.wait_for_java_debugger: - apk = self._test_instance.apk_under_test or self._test_instance.test_apk logging.warning('*' * 80) logging.warning('Waiting for debugger to attach to process: %s', - apk.GetPackageName()) + target_package) logging.warning('*' * 80) #override @@ -459,6 +467,31 @@ class LocalDeviceInstrumentationTestRun( return tests #override + def _GroupTests(self, tests): + batched_tests = dict() + other_tests = [] + for test in tests: + if 'Batch' in test['annotations']: + batch_name = test['annotations']['Batch']['value'] + if not batch_name: + batch_name = test['class'] + if not batch_name in batched_tests: + batched_tests[batch_name] = [] + batched_tests[batch_name].append(test) + else: + other_tests.append(test) + + all_tests = [] + for _, tests in batched_tests.items(): + tests.sort() # Ensure a consistent ordering across external shards. + all_tests.extend([ + tests[i:i + _TEST_BATCH_MAX_GROUP_SIZE] + for i in range(0, len(tests), _TEST_BATCH_MAX_GROUP_SIZE) + ]) + all_tests.extend(other_tests) + return all_tests + + #override def _GetUniqueTestName(self, test): return instrumentation_test_instance.GetUniqueTestName(test) @@ -506,12 +539,9 @@ class LocalDeviceInstrumentationTestRun( device.adb, suffix='.json', dir=device.GetExternalStoragePath()) extras[EXTRA_TRACE_FILE] = trace_device_file.name + target = '%s/%s' % (self._test_instance.test_package, + self._test_instance.junit4_runner_class) if isinstance(test, list): - if not self._test_instance.driver_apk: - raise Exception('driver_apk does not exist. ' - 'Please build it and try again.') - if any(t.get('is_junit4') for t in test): - raise Exception('driver apk does not support JUnit4 tests') def name_and_timeout(t): n = instrumentation_test_instance.GetTestName(t) @@ -520,26 +550,15 @@ class LocalDeviceInstrumentationTestRun( test_names, timeouts = zip(*(name_and_timeout(t) for t in test)) - test_name = ','.join(test_names) + test_name = instrumentation_test_instance.GetTestName(test[0]) + '_batch' + extras['class'] = ','.join(test_names) test_display_name = test_name - target = '%s/%s' % ( - self._test_instance.driver_package, - self._test_instance.driver_name) - extras.update( - self._test_instance.GetDriverEnvironmentVars( - test_list=test_names)) timeout = sum(timeouts) else: + assert test['is_junit4'] test_name = instrumentation_test_instance.GetTestName(test) test_display_name = self._GetUniqueTestName(test) - if test['is_junit4']: - target = '%s/%s' % ( - self._test_instance.test_package, - self._test_instance.junit4_runner_class) - else: - target = '%s/%s' % ( - self._test_instance.test_package, - self._test_instance.junit3_runner_class) + extras['class'] = test_name if 'flags' in test and test['flags']: flags_to_add.extend(test['flags']) @@ -556,14 +575,39 @@ class LocalDeviceInstrumentationTestRun( timeout = None logging.info('preparing to run %s: %s', test_display_name, test) - render_tests_device_output_dir = None if _IsRenderTest(test): # TODO(mikecase): Add DeviceTempDirectory class and use that instead. - render_tests_device_output_dir = posixpath.join( - device.GetExternalStoragePath(), - 'render_test_output_dir') + self._render_tests_device_output_dir = posixpath.join( + device.GetExternalStoragePath(), 'render_test_output_dir') flags_to_add.append('--render-test-output-dir=%s' % - render_tests_device_output_dir) + self._render_tests_device_output_dir) + + if _IsWPRRecordReplayTest(test): + wpr_archive_relative_path = _GetWPRArchivePath(test) + if not wpr_archive_relative_path: + raise RuntimeError('Could not find the WPR archive file path ' + 'from annotation.') + wpr_archive_path = os.path.join(host_paths.DIR_SOURCE_ROOT, + wpr_archive_relative_path) + if not os.path.isdir(wpr_archive_path): + raise RuntimeError('WPRArchiveDirectory annotation should point' + 'to a directory only.') + + archive_path = os.path.join(wpr_archive_path, + self._GetUniqueTestName(test) + '.wprgo') + + if not os.path.exists(_WPR_GO_LINUX_X86_64_PATH): + # If we got to this stage, then we should have + # checkout_android set. + raise RuntimeError( + 'WPR Go binary not found at {}'.format(_WPR_GO_LINUX_X86_64_PATH)) + # Tells the server to use the binaries retrieved from CIPD. + chrome_proxy_utils.ChromeProxySession.SetWPRServerBinary( + _WPR_GO_LINUX_X86_64_PATH) + self._chrome_proxy = chrome_proxy_utils.ChromeProxySession() + self._chrome_proxy.wpr_record_mode = self._test_instance.wpr_record_mode + self._chrome_proxy.Start(device, archive_path) + flags_to_add.extend(self._chrome_proxy.GetFlags()) if flags_to_add: self._CreateFlagChangerIfNeeded(device) @@ -588,7 +632,7 @@ class LocalDeviceInstrumentationTestRun( result_code, result_bundle, statuses = ( self._test_instance.ParseAmInstrumentRawOutput(output)) results = self._test_instance.GenerateTestResults( - result_code, result_bundle, statuses, start_ms, duration_ms, + result_code, result_bundle, statuses, duration_ms, device.product_cpu_abi, self._test_instance.symbolizer) if self._env.trace_output: @@ -620,11 +664,12 @@ class LocalDeviceInstrumentationTestRun( # check to see if any failure images were generated even if the test # does not fail. try: - self._ProcessRenderTestResults( - device, render_tests_device_output_dir, results) + self._ProcessRenderTestResults(device, results) finally: - device.RemovePath(render_tests_device_output_dir, - recursive=True, force=True) + device.RemovePath(self._render_tests_device_output_dir, + recursive=True, + force=True) + self._render_tests_device_output_dir = None def pull_ui_screen_captures(): screenshots = [] @@ -653,13 +698,23 @@ class LocalDeviceInstrumentationTestRun( json_data['image_link'] = image_archive.Link() return json_data + def stop_chrome_proxy(): + # Removes the port forwarding + if self._chrome_proxy: + self._chrome_proxy.Stop(device) + if not self._chrome_proxy.wpr_replay_mode: + logging.info('WPR Record test generated archive file %s', + self._chrome_proxy.wpr_archive_path) + self._chrome_proxy = None + + # While constructing the TestResult objects, we can parallelize several # steps that involve ADB. These steps should NOT depend on any info in # the results! Things such as whether the test CRASHED have not yet been # determined. post_test_steps = [ - restore_flags, restore_timeout_scale, handle_coverage_data, - handle_render_test_data, pull_ui_screen_captures + restore_flags, restore_timeout_scale, stop_chrome_proxy, + handle_coverage_data, handle_render_test_data, pull_ui_screen_captures ] if self._env.concurrent_adb: reraiser_thread.RunAsync(post_test_steps) @@ -920,16 +975,14 @@ class LocalDeviceInstrumentationTestRun( screenshot_device_file.close() _SetLinkOnResults(results, link_name, screenshot_host_file.Link()) - def _ProcessRenderTestResults( - self, device, render_tests_device_output_dir, results): - self._ProcessSkiaGoldRenderTestResults( - device, render_tests_device_output_dir, results) - self._ProcessLocalRenderTestResults(device, render_tests_device_output_dir, - results) + def _ProcessRenderTestResults(self, device, results): + if not self._render_tests_device_output_dir: + return + self._ProcessSkiaGoldRenderTestResults(device, results) - def _ProcessSkiaGoldRenderTestResults( - self, device, render_tests_device_output_dir, results): - gold_dir = posixpath.join(render_tests_device_output_dir, _DEVICE_GOLD_DIR) + def _ProcessSkiaGoldRenderTestResults(self, device, results): + gold_dir = posixpath.join(self._render_tests_device_output_dir, + _DEVICE_GOLD_DIR) if not device.FileExists(gold_dir): return @@ -958,8 +1011,27 @@ class LocalDeviceInstrumentationTestRun( 'when doing Skia Gold comparison.' % image_name) continue + # Add 'ignore': '1' if a comparison failure would not be surfaced, as + # that implies that we aren't actively maintaining baselines for the + # test. This helps prevent unrelated CLs from getting comments posted to + # them. + with open(json_path) as infile: + # All the key/value pairs in the JSON file are strings, so convert + # to a bool. + json_dict = json.load(infile) + fail_on_unsupported = json_dict.get('fail_on_unsupported_configs', + 'false') + fail_on_unsupported = fail_on_unsupported.lower() == 'true' + should_hide_failure = ( + device.build_version_sdk not in RENDER_TEST_MODEL_SDK_CONFIGS.get( + device.product_model, []) and not fail_on_unsupported) + if should_hide_failure: + json_dict['ignore'] = '1' + with open(json_path, 'w') as outfile: + json.dump(json_dict, outfile) + gold_session = self._skia_gold_session_manager.GetSkiaGoldSession( - keys_file=json_path) + keys_input=json_path) try: status, error = gold_session.RunComparison( @@ -978,14 +1050,7 @@ class LocalDeviceInstrumentationTestRun( # Don't fail the test if we ran on an unsupported configuration unless # the test has explicitly opted in, as it's likely that baselines # aren't maintained for that configuration. - with open(json_path) as infile: - # All the key/value pairs in the JSON file are strings, so convert - # to a bool. - fail_on_unsupported = json.load(infile).get( - 'fail_on_unsupported_configs', 'false') - fail_on_unsupported = fail_on_unsupported.lower() == 'true' - if device.build_version_sdk not in RENDER_TEST_MODEL_SDK_CONFIGS.get( - device.product_model, []) and not fail_on_unsupported: + if should_hide_failure: if self._test_instance.skia_gold_properties.local_pixel_tests: _AppendToLog( results, 'Gold comparison for %s failed, but model %s with SDK ' @@ -1004,7 +1069,7 @@ class LocalDeviceInstrumentationTestRun( failure_log = ( 'Skia Gold reported failure for RenderTest %s. See ' 'RENDER_TESTS.md for how to fix this failure.' % render_name) - status_codes = gold_utils.SkiaGoldSession.StatusCodes + status_codes = gold_utils.AndroidSkiaGoldSession.StatusCodes if status == status_codes.AUTH_FAILURE: _AppendToLog(results, 'Gold authentication failed with output %s' % error) @@ -1053,63 +1118,6 @@ class LocalDeviceInstrumentationTestRun( 'Given unhandled SkiaGoldSession StatusCode %s with error %s', status, error) - def _ProcessLocalRenderTestResults(self, device, - render_tests_device_output_dir, results): - failure_images_device_dir = posixpath.join( - render_tests_device_output_dir, 'failures') - if not device.FileExists(failure_images_device_dir): - return - - diff_images_device_dir = posixpath.join( - render_tests_device_output_dir, 'diffs') - - golden_images_device_dir = posixpath.join( - render_tests_device_output_dir, 'goldens') - - for failure_filename in device.ListDirectory(failure_images_device_dir): - - with self._env.output_manager.ArchivedTempfile( - 'fail_%s' % failure_filename, 'render_tests', - output_manager.Datatype.PNG) as failure_image_host_file: - device.PullFile( - posixpath.join(failure_images_device_dir, failure_filename), - failure_image_host_file.name) - failure_link = failure_image_host_file.Link() - - golden_image_device_file = posixpath.join( - golden_images_device_dir, failure_filename) - if device.PathExists(golden_image_device_file): - with self._env.output_manager.ArchivedTempfile( - 'golden_%s' % failure_filename, 'render_tests', - output_manager.Datatype.PNG) as golden_image_host_file: - device.PullFile( - golden_image_device_file, golden_image_host_file.name) - golden_link = golden_image_host_file.Link() - else: - golden_link = '' - - diff_image_device_file = posixpath.join( - diff_images_device_dir, failure_filename) - if device.PathExists(diff_image_device_file): - with self._env.output_manager.ArchivedTempfile( - 'diff_%s' % failure_filename, 'render_tests', - output_manager.Datatype.PNG) as diff_image_host_file: - device.PullFile( - diff_image_device_file, diff_image_host_file.name) - diff_link = diff_image_host_file.Link() - else: - diff_link = '' - - processed_template_output = _GenerateRenderTestHtml( - failure_filename, failure_link, golden_link, diff_link) - - with self._env.output_manager.ArchivedTempfile( - '%s.html' % failure_filename, 'render_tests', - output_manager.Datatype.HTML) as html_results: - html_results.write(processed_template_output) - html_results.flush() - _SetLinkOnResults(results, failure_filename, html_results.Link()) - #override def _ShouldRetry(self, test, result): # We've tried to disable retries in the past with mixed results. @@ -1145,6 +1153,22 @@ class LocalDeviceInstrumentationTestRun( return timeout +def _IsWPRRecordReplayTest(test): + """Determines whether a test or a list of tests is a WPR RecordReplay Test.""" + if not isinstance(test, list): + test = [test] + return any([ + WPR_RECORD_REPLAY_TEST_FEATURE_ANNOTATION in t['annotations'].get( + FEATURE_ANNOTATION, {}).get('value', ()) for t in test + ]) + + +def _GetWPRArchivePath(test): + """Retrieves the archive path from the WPRArchiveDirectory annotation.""" + return test['annotations'].get(WPR_ARCHIVE_FILE_PATH_ANNOTATION, + {}).get('value', ()) + + def _IsRenderTest(test): """Determines if a test or list of tests has a RenderTest amongst them.""" if not isinstance(test, list): diff --git a/chromium/build/android/pylib/local/device/local_device_instrumentation_test_run_test.py b/chromium/build/android/pylib/local/device/local_device_instrumentation_test_run_test.py index 3129c1121b0..dd57d92061e 100755 --- a/chromium/build/android/pylib/local/device/local_device_instrumentation_test_run_test.py +++ b/chromium/build/android/pylib/local/device/local_device_instrumentation_test_run_test.py @@ -61,6 +61,88 @@ class LocalDeviceInstrumentationTestRunTest(unittest.TestCase): 'SadTest.testNotRun', base_test_result.ResultType.NOTRUN) self.assertTrue(self._obj._ShouldRetry(test, result)) + def testIsWPRRecordReplayTest_matchedWithKey(self): + test = { + 'annotations': { + 'Feature': { + 'value': ['WPRRecordReplayTest', 'dummy'] + } + }, + 'class': 'WPRDummyTest', + 'method': 'testRun', + 'is_junit4': True, + } + self.assertTrue( + local_device_instrumentation_test_run._IsWPRRecordReplayTest(test)) + + def testIsWPRRecordReplayTest_noMatchedKey(self): + test = { + 'annotations': { + 'Feature': { + 'value': ['abc', 'dummy'] + } + }, + 'class': 'WPRDummyTest', + 'method': 'testRun', + 'is_junit4': True, + } + self.assertFalse( + local_device_instrumentation_test_run._IsWPRRecordReplayTest(test)) + + def testGetWPRArchivePath_matchedWithKey(self): + test = { + 'annotations': { + 'WPRArchiveDirectory': { + 'value': 'abc' + } + }, + 'class': 'WPRDummyTest', + 'method': 'testRun', + 'is_junit4': True, + } + self.assertEqual( + local_device_instrumentation_test_run._GetWPRArchivePath(test), 'abc') + + def testGetWPRArchivePath_noMatchedWithKey(self): + test = { + 'annotations': { + 'Feature': { + 'value': 'abc' + } + }, + 'class': 'WPRDummyTest', + 'method': 'testRun', + 'is_junit4': True, + } + self.assertFalse( + local_device_instrumentation_test_run._GetWPRArchivePath(test)) + + def testIsRenderTest_matchedWithKey(self): + test = { + 'annotations': { + 'Feature': { + 'value': ['RenderTest', 'dummy'] + } + }, + 'class': 'DummyTest', + 'method': 'testRun', + 'is_junit4': True, + } + self.assertTrue(local_device_instrumentation_test_run._IsRenderTest(test)) + + def testIsRenderTest_noMatchedKey(self): + test = { + 'annotations': { + 'Feature': { + 'value': ['abc', 'dummy'] + } + }, + 'class': 'DummyTest', + 'method': 'testRun', + 'is_junit4': True, + } + self.assertFalse(local_device_instrumentation_test_run._IsRenderTest(test)) + if __name__ == '__main__': unittest.main(verbosity=2) diff --git a/chromium/build/android/pylib/local/device/local_device_test_run.py b/chromium/build/android/pylib/local/device/local_device_test_run.py index 2018751fed5..69b27186507 100644 --- a/chromium/build/android/pylib/local/device/local_device_test_run.py +++ b/chromium/build/android/pylib/local/device/local_device_test_run.py @@ -137,6 +137,7 @@ class LocalDeviceTestRun(test_run.TestRun): with signal_handler.AddSignalHandler(signal.SIGTERM, stop_tests): tries = 0 while tries < self._env.max_tries and tests: + grouped_tests = self._GroupTests(tests) logging.info('STARTING TRY #%d/%d', tries + 1, self._env.max_tries) if tries > 0 and self._env.recover_devices: if any(d.build_version_sdk == version_codes.LOLLIPOP_MR1 @@ -171,12 +172,14 @@ class LocalDeviceTestRun(test_run.TestRun): try: if self._ShouldShard(): - tc = test_collection.TestCollection(self._CreateShards(tests)) + tc = test_collection.TestCollection( + self._CreateShards(grouped_tests)) self._env.parallel_devices.pMap( run_tests_on_device, tc, try_results).pGet(None) else: - self._env.parallel_devices.pMap( - run_tests_on_device, tests, try_results).pGet(None) + self._env.parallel_devices.pMap(run_tests_on_device, + grouped_tests, + try_results).pGet(None) except TestsTerminated: for unknown_result in try_results.GetUnknown(): try_results.AddResult( @@ -236,9 +239,16 @@ class LocalDeviceTestRun(test_run.TestRun): if total_shards < 0 or shard_index < 0 or total_shards <= shard_index: raise InvalidShardingSettings(shard_index, total_shards) - return [ - t for t in tests - if hash(self._GetUniqueTestName(t)) % total_shards == shard_index] + sharded_tests = [] + for t in self._GroupTests(tests): + if (hash(self._GetUniqueTestName(t[0] if isinstance(t, list) else t)) % + total_shards == shard_index): + if isinstance(t, list): + sharded_tests.extend(t) + else: + sharded_tests.append(t) + + return sharded_tests def GetTool(self, device): if str(device) not in self._tools: @@ -260,6 +270,10 @@ class LocalDeviceTestRun(test_run.TestRun): def _GetTests(self): raise NotImplementedError + def _GroupTests(self, tests): + # pylint: disable=no-self-use + return tests + def _RunTest(self, device, test): raise NotImplementedError diff --git a/chromium/build/android/pylib/local/device/local_device_test_run_test.py b/chromium/build/android/pylib/local/device/local_device_test_run_test.py index 525bf25200b..aeea5881c8c 100755 --- a/chromium/build/android/pylib/local/device/local_device_test_run_test.py +++ b/chromium/build/android/pylib/local/device/local_device_test_run_test.py @@ -8,11 +8,9 @@ import unittest from pylib.base import base_test_result -from pylib.constants import host_paths from pylib.local.device import local_device_test_run -with host_paths.SysPath(host_paths.PYMOCK_PATH): - import mock # pylint: disable=import-error +import mock # pylint: disable=import-error class SubstituteDeviceRootTest(unittest.TestCase): |