diff options
Diffstat (limited to 'chromium/build/android/pylib')
20 files changed, 898 insertions, 1634 deletions
diff --git a/chromium/build/android/pylib/base/mock_environment.py b/chromium/build/android/pylib/base/mock_environment.py index 9ebb083a086..5bdefd0a0d9 100644 --- a/chromium/build/android/pylib/base/mock_environment.py +++ b/chromium/build/android/pylib/base/mock_environment.py @@ -3,10 +3,8 @@ # found in the LICENSE file. from pylib.base import environment -from pylib.constants import host_paths -with host_paths.SysPath(host_paths.PYMOCK_PATH): - import mock # pylint: disable=import-error +import mock # pylint: disable=import-error MockEnvironment = mock.MagicMock(environment.Environment) diff --git a/chromium/build/android/pylib/base/mock_test_instance.py b/chromium/build/android/pylib/base/mock_test_instance.py index 18def019903..8ef723bf050 100644 --- a/chromium/build/android/pylib/base/mock_test_instance.py +++ b/chromium/build/android/pylib/base/mock_test_instance.py @@ -3,10 +3,8 @@ # found in the LICENSE file. from pylib.base import test_instance -from pylib.constants import host_paths -with host_paths.SysPath(host_paths.PYMOCK_PATH): - import mock # pylint: disable=import-error +import mock # pylint: disable=import-error MockTestInstance = mock.MagicMock(test_instance.TestInstance) diff --git a/chromium/build/android/pylib/constants/host_paths.py b/chromium/build/android/pylib/constants/host_paths.py index b249d3c2919..e00e0e79eb8 100644 --- a/chromium/build/android/pylib/constants/host_paths.py +++ b/chromium/build/android/pylib/constants/host_paths.py @@ -20,10 +20,9 @@ BUILD_COMMON_PATH = os.path.join( ANDROID_PLATFORM_DEVELOPMENT_SCRIPTS_PATH = os.path.join( DIR_SOURCE_ROOT, 'third_party', 'android_platform', 'development', 'scripts') +BUILD_PATH = os.path.join(DIR_SOURCE_ROOT, 'build') DEVIL_PATH = os.path.join( DIR_SOURCE_ROOT, 'third_party', 'catapult', 'devil') -PYMOCK_PATH = os.path.join( - DIR_SOURCE_ROOT, 'third_party', 'pymock') TRACING_PATH = os.path.join( DIR_SOURCE_ROOT, 'third_party', 'catapult', 'tracing') diff --git a/chromium/build/android/pylib/device/commands/BUILD.gn b/chromium/build/android/pylib/device/commands/BUILD.gn index a3ee6462706..13b69f618cf 100644 --- a/chromium/build/android/pylib/device/commands/BUILD.gn +++ b/chromium/build/android/pylib/device/commands/BUILD.gn @@ -8,10 +8,13 @@ group("commands") { data_deps = [ ":chromium_commands_java" ] } -android_library("chromium_commands_java") { +android_library("unzip_java") { jacoco_never_instrument = true sources = [ "java/src/org/chromium/android/commands/unzip/Unzip.java" ] - dex_path = "$root_build_dir/lib.java/chromium_commands.dex.jar" - deps = [ "//base:base_java" ] - data = [ dex_path ] +} + +dist_dex("chromium_commands_java") { + deps = [ ":unzip_java" ] + output = "$root_build_dir/lib.java/chromium_commands.dex.jar" + data = [ output ] } diff --git a/chromium/build/android/pylib/gtest/filter/unit_tests_disabled b/chromium/build/android/pylib/gtest/filter/unit_tests_disabled index 706e1abcf57..97811c83a4a 100644 --- a/chromium/build/android/pylib/gtest/filter/unit_tests_disabled +++ b/chromium/build/android/pylib/gtest/filter/unit_tests_disabled @@ -19,9 +19,6 @@ AutofillTableTest.UpdateAutofillProfile AutofillProfileTest.* CreditCardTest.SetInfoExpirationMonth -# crbug.com/139398 -DownloadItemModelTest.InterruptTooltip - # Tests crashing in the APK # l10n_util.cc(655)] Check failed: std::string::npos != pos DownloadItemModelTest.InterruptStatus diff --git a/chromium/build/android/pylib/instrumentation/instrumentation_parser.py b/chromium/build/android/pylib/instrumentation/instrumentation_parser.py index 8605178924f..d38f6a5551c 100644 --- a/chromium/build/android/pylib/instrumentation/instrumentation_parser.py +++ b/chromium/build/android/pylib/instrumentation/instrumentation_parser.py @@ -20,6 +20,8 @@ STATUS_CODE_SKIP = -3 # http://junit.org/junit4/javadoc/4.12/org/junit/AssumptionViolatedException.html STATUS_CODE_ASSUMPTION_FAILURE = -4 +STATUS_CODE_TEST_DURATION = 1337 + # http://developer.android.com/reference/android/app/Activity.html RESULT_CODE_OK = -1 RESULT_CODE_CANCELED = 0 diff --git a/chromium/build/android/pylib/instrumentation/instrumentation_test_instance.py b/chromium/build/android/pylib/instrumentation/instrumentation_test_instance.py index 3b61977278e..a30334c6d09 100644 --- a/chromium/build/android/pylib/instrumentation/instrumentation_test_instance.py +++ b/chromium/build/android/pylib/instrumentation/instrumentation_test_instance.py @@ -61,6 +61,23 @@ _PARAMETERIZED_COMMAND_LINE_FLAGS_SWITCHES = ( _NATIVE_CRASH_RE = re.compile('(process|native) crash', re.IGNORECASE) _PICKLE_FORMAT_VERSION = 12 +# The ID of the bundle value Instrumentation uses to report which test index the +# results are for in a collection of tests. Note that this index is 1-based. +_BUNDLE_CURRENT_ID = 'current' +# The ID of the bundle value Instrumentation uses to report the test class. +_BUNDLE_CLASS_ID = 'class' +# The ID of the bundle value Instrumentation uses to report the test name. +_BUNDLE_TEST_ID = 'test' +# The ID of the bundle value Instrumentation uses to report if a test was +# skipped. +_BUNDLE_SKIPPED_ID = 'test_skipped' +# The ID of the bundle value Instrumentation uses to report the crash stack, if +# the test crashed. +_BUNDLE_STACK_ID = 'stack' + +# The ID of the bundle value Chrome uses to report the test duration. +_BUNDLE_DURATION_ID = 'duration_ms' + class MissingSizeAnnotationError(test_exception.TestException): def __init__(self, class_name): @@ -103,9 +120,8 @@ def ParseAmInstrumentRawOutput(raw_output): return (code, bundle, statuses) -def GenerateTestResults( - result_code, result_bundle, statuses, start_ms, duration_ms, device_abi, - symbolizer): +def GenerateTestResults(result_code, result_bundle, statuses, duration_ms, + device_abi, symbolizer): """Generate test results from |statuses|. Args: @@ -116,7 +132,6 @@ def GenerateTestResults( - the bundle dump as a dict mapping string keys to string values Note that this is the same as the third item in the 3-tuple returned by |_ParseAmInstrumentRawOutput|. - start_ms: The start time of the test in milliseconds. duration_ms: The duration of the test in milliseconds. device_abi: The device_abi, which is needed for symbolization. symbolizer: The symbolizer used to symbolize stack. @@ -129,10 +144,21 @@ def GenerateTestResults( results = [] current_result = None + cumulative_duration = 0 for status_code, bundle in statuses: - test_class = bundle.get('class', '') - test_method = bundle.get('test', '') + if status_code == instrumentation_parser.STATUS_CODE_TEST_DURATION: + # For the first result, duration will be set below to the difference + # between the reported and actual durations to account for overhead like + # starting instrumentation. + if len(results) > 1: + current_duration = int(bundle.get(_BUNDLE_DURATION_ID, duration_ms)) + current_result.SetDuration(current_duration) + cumulative_duration += current_duration + continue + + test_class = bundle.get(_BUNDLE_CLASS_ID, '') + test_method = bundle.get(_BUNDLE_TEST_ID, '') if test_class and test_method: test_name = '%s#%s' % (test_class, test_method) else: @@ -142,10 +168,10 @@ def GenerateTestResults( if current_result: results.append(current_result) current_result = test_result.InstrumentationTestResult( - test_name, base_test_result.ResultType.UNKNOWN, start_ms, duration_ms) + test_name, base_test_result.ResultType.UNKNOWN, duration_ms) else: if status_code == instrumentation_parser.STATUS_CODE_OK: - if bundle.get('test_skipped', '').lower() in ('true', '1', 'yes'): + if bundle.get(_BUNDLE_SKIPPED_ID, '').lower() in ('true', '1', 'yes'): current_result.SetType(base_test_result.ResultType.SKIP) elif current_result.GetType() == base_test_result.ResultType.UNKNOWN: current_result.SetType(base_test_result.ResultType.PASS) @@ -159,15 +185,13 @@ def GenerateTestResults( logging.error('Unrecognized status code %d. Handling as an error.', status_code) current_result.SetType(base_test_result.ResultType.FAIL) - if 'stack' in bundle: + if _BUNDLE_STACK_ID in bundle: if symbolizer and device_abi: - current_result.SetLog( - '%s\n%s' % ( - bundle['stack'], - '\n'.join(symbolizer.ExtractAndResolveNativeStackTraces( - bundle['stack'], device_abi)))) + current_result.SetLog('%s\n%s' % (bundle[_BUNDLE_STACK_ID], '\n'.join( + symbolizer.ExtractAndResolveNativeStackTraces( + bundle[_BUNDLE_STACK_ID], device_abi)))) else: - current_result.SetLog(bundle['stack']) + current_result.SetLog(bundle[_BUNDLE_STACK_ID]) if current_result: if current_result.GetType() == base_test_result.ResultType.UNKNOWN: @@ -179,6 +203,9 @@ def GenerateTestResults( results.append(current_result) + if results: + results[0].SetDuration(duration_ms - cumulative_duration) + return results @@ -521,6 +548,8 @@ class InstrumentationTestInstance(test_instance.TestInstance): self._skia_gold_properties = None self._initializeSkiaGoldAttributes(args) + self._wpr_enable_record = args.wpr_enable_record + self._external_shard_index = args.test_launcher_shard_index self._total_external_shards = args.test_launcher_total_shards @@ -731,7 +760,7 @@ class InstrumentationTestInstance(test_instance.TestInstance): self._use_webview_provider = args.use_webview_provider def _initializeSkiaGoldAttributes(self, args): - self._skia_gold_properties = gold_utils.SkiaGoldProperties(args) + self._skia_gold_properties = gold_utils.AndroidSkiaGoldProperties(args) @property def additional_apks(self): @@ -865,6 +894,14 @@ class InstrumentationTestInstance(test_instance.TestInstance): def wait_for_java_debugger(self): return self._wait_for_java_debugger + @property + def wpr_record_mode(self): + return self._wpr_enable_record + + @property + def wpr_replay_mode(self): + return not self._wpr_enable_record + #override def TestType(self): return 'instrumentation' @@ -930,7 +967,8 @@ class InstrumentationTestInstance(test_instance.TestInstance): 'class': c['class'], 'method': m['method'], 'annotations': a, - 'is_junit4': c['superclass'] == 'java.lang.Object' + # TODO(https://crbug.com/1084729): Remove is_junit4. + 'is_junit4': True }) return inflated_tests @@ -1005,11 +1043,10 @@ class InstrumentationTestInstance(test_instance.TestInstance): return ParseAmInstrumentRawOutput(raw_output) @staticmethod - def GenerateTestResults( - result_code, result_bundle, statuses, start_ms, duration_ms, - device_abi, symbolizer): + def GenerateTestResults(result_code, result_bundle, statuses, duration_ms, + device_abi, symbolizer): return GenerateTestResults(result_code, result_bundle, statuses, - start_ms, duration_ms, device_abi, symbolizer) + duration_ms, device_abi, symbolizer) #override def TearDown(self): diff --git a/chromium/build/android/pylib/instrumentation/instrumentation_test_instance_test.py b/chromium/build/android/pylib/instrumentation/instrumentation_test_instance_test.py index d3003b8239e..fdb4114a63d 100755 --- a/chromium/build/android/pylib/instrumentation/instrumentation_test_instance_test.py +++ b/chromium/build/android/pylib/instrumentation/instrumentation_test_instance_test.py @@ -12,11 +12,9 @@ import tempfile import unittest from pylib.base import base_test_result -from pylib.constants import host_paths from pylib.instrumentation import instrumentation_test_instance -with host_paths.SysPath(host_paths.PYMOCK_PATH): - import mock # pylint: disable=import-error +import mock # pylint: disable=import-error _INSTRUMENTATION_TEST_INSTANCE_PATH = ( 'pylib.instrumentation.instrumentation_test_instance.%s') @@ -497,15 +495,17 @@ class InstrumentationTestInstanceTest(unittest.TestCase): ] expected_tests = [ - { - 'annotations': { - 'Feature': {'value': ['Foo']}, - 'MediumTest': None, + { + 'annotations': { + 'Feature': { + 'value': ['Foo'] + }, + 'MediumTest': None, + }, + 'class': 'org.chromium.test.SampleTest', + 'is_junit4': True, + 'method': 'testMethod2', }, - 'class': 'org.chromium.test.SampleTest', - 'is_junit4': False, - 'method': 'testMethod2', - }, ] o._excluded_annotations = [('SmallTest', None)] @@ -556,16 +556,18 @@ class InstrumentationTestInstanceTest(unittest.TestCase): ] expected_tests = [ - { - 'annotations': { - 'Feature': {'value': ['Foo']}, - 'SmallTest': None, - 'TestValue': '1', + { + 'annotations': { + 'Feature': { + 'value': ['Foo'] + }, + 'SmallTest': None, + 'TestValue': '1', + }, + 'class': 'org.chromium.test.SampleTest', + 'is_junit4': True, + 'method': 'testMethod1', }, - 'class': 'org.chromium.test.SampleTest', - 'is_junit4': False, - 'method': 'testMethod1', - }, ] o._annotations = [('TestValue', '1')] @@ -724,24 +726,28 @@ class InstrumentationTestInstanceTest(unittest.TestCase): ] expected_tests = [ - { - 'annotations': { - 'Feature': {'value': ['Baz']}, - 'MediumTest': None, + { + 'annotations': { + 'Feature': { + 'value': ['Baz'] + }, + 'MediumTest': None, + }, + 'class': 'org.chromium.test.SampleTest', + 'is_junit4': True, + 'method': 'testMethod2', }, - 'class': 'org.chromium.test.SampleTest', - 'is_junit4': False, - 'method': 'testMethod2', - }, - { - 'annotations': { - 'Feature': {'value': ['Bar']}, - 'SmallTest': None, + { + 'annotations': { + 'Feature': { + 'value': ['Bar'] + }, + 'SmallTest': None, + }, + 'class': 'org.chromium.test.SampleTest2', + 'is_junit4': True, + 'method': 'testMethod1', }, - 'class': 'org.chromium.test.SampleTest2', - 'is_junit4': False, - 'method': 'testMethod1', - }, ] o._annotations = [('Feature', 'Bar'), ('Feature', 'Baz')] @@ -753,7 +759,7 @@ class InstrumentationTestInstanceTest(unittest.TestCase): def testGenerateTestResults_noStatus(self): results = instrumentation_test_instance.GenerateTestResults( - None, None, [], 0, 1000, None, None) + None, None, [], 1000, None, None) self.assertEqual([], results) def testGenerateTestResults_testPassed(self): @@ -768,7 +774,7 @@ class InstrumentationTestInstanceTest(unittest.TestCase): }), ] results = instrumentation_test_instance.GenerateTestResults( - None, None, statuses, 0, 1000, None, None) + None, None, statuses, 1000, None, None) self.assertEqual(1, len(results)) self.assertEqual(base_test_result.ResultType.PASS, results[0].GetType()) @@ -789,7 +795,7 @@ class InstrumentationTestInstanceTest(unittest.TestCase): }), ] results = instrumentation_test_instance.GenerateTestResults( - None, None, statuses, 0, 1000, None, None) + None, None, statuses, 1000, None, None) self.assertEqual(1, len(results)) self.assertEqual(base_test_result.ResultType.SKIP, results[0].GetType()) @@ -808,7 +814,7 @@ class InstrumentationTestInstanceTest(unittest.TestCase): }), ] results = instrumentation_test_instance.GenerateTestResults( - None, None, statuses, 0, 1000, None, None) + None, None, statuses, 1000, None, None) self.assertEqual(1, len(results)) self.assertEqual(base_test_result.ResultType.PASS, results[0].GetType()) @@ -824,7 +830,7 @@ class InstrumentationTestInstanceTest(unittest.TestCase): }), ] results = instrumentation_test_instance.GenerateTestResults( - None, None, statuses, 0, 1000, None, None) + None, None, statuses, 1000, None, None) self.assertEqual(1, len(results)) self.assertEqual(base_test_result.ResultType.FAIL, results[0].GetType()) @@ -842,7 +848,7 @@ class InstrumentationTestInstanceTest(unittest.TestCase): }), ] results = instrumentation_test_instance.GenerateTestResults( - None, None, statuses, 0, 1000, None, None) + None, None, statuses, 1000, None, None) self.assertEqual(1, len(results)) self.assertEqual(base_test_result.ResultType.FAIL, results[0].GetType()) self.assertEqual(stacktrace, results[0].GetLog()) @@ -859,7 +865,7 @@ class InstrumentationTestInstanceTest(unittest.TestCase): }), ] results = instrumentation_test_instance.GenerateTestResults( - None, None, statuses, 0, 1000, None, None) + None, None, statuses, 1000, None, None) self.assertEqual(1, len(results)) self.assertEqual(base_test_result.ResultType.SKIP, results[0].GetType()) diff --git a/chromium/build/android/pylib/instrumentation/test_result.py b/chromium/build/android/pylib/instrumentation/test_result.py index 24e80a8e5fb..a1c7307fce9 100644 --- a/chromium/build/android/pylib/instrumentation/test_result.py +++ b/chromium/build/android/pylib/instrumentation/test_result.py @@ -8,13 +8,12 @@ from pylib.base import base_test_result class InstrumentationTestResult(base_test_result.BaseTestResult): """Result information for a single instrumentation test.""" - def __init__(self, full_name, test_type, start_date, dur, log=''): + def __init__(self, full_name, test_type, dur, log=''): """Construct an InstrumentationTestResult object. Args: full_name: Full name of the test. test_type: Type of the test result as defined in ResultType. - start_date: Date in milliseconds when the test began running. dur: Duration of the test run in milliseconds. log: A string listing any errors. """ @@ -27,4 +26,7 @@ class InstrumentationTestResult(base_test_result.BaseTestResult): else: self._class_name = full_name self._test_name = full_name - self._start_date = start_date + + def SetDuration(self, duration): + """Set the test duration.""" + self._duration = duration diff --git a/chromium/build/android/pylib/local/device/local_device_instrumentation_test_run.py b/chromium/build/android/pylib/local/device/local_device_instrumentation_test_run.py index 6a64e190969..5a46e6fcb1c 100644 --- a/chromium/build/android/pylib/local/device/local_device_instrumentation_test_run.py +++ b/chromium/build/android/pylib/local/device/local_device_instrumentation_test_run.py @@ -36,17 +36,16 @@ from pylib.instrumentation import instrumentation_test_instance from pylib.local.device import local_device_environment from pylib.local.device import local_device_test_run from pylib.output import remote_output_manager +from pylib.utils import chrome_proxy_utils from pylib.utils import gold_utils from pylib.utils import instrumentation_tracing from pylib.utils import shared_preference_utils - from py_trace_event import trace_event from py_trace_event import trace_time from py_utils import contextlib_ext from py_utils import tempfile_ext import tombstones - with host_paths.SysPath( os.path.join(host_paths.DIR_SOURCE_ROOT, 'third_party'), 0): import jinja2 # pylint: disable=import-error @@ -57,6 +56,10 @@ _JINJA_TEMPLATE_DIR = os.path.join( host_paths.DIR_SOURCE_ROOT, 'build', 'android', 'pylib', 'instrumentation') _JINJA_TEMPLATE_FILENAME = 'render_test.html.jinja' +_WPR_GO_LINUX_X86_64_PATH = os.path.join(host_paths.DIR_SOURCE_ROOT, + 'third_party', 'webpagereplay', 'bin', + 'linux', 'x86_64', 'wpr') + _TAG = 'test_runner_py' TIMEOUT_ANNOTATIONS = [ @@ -88,6 +91,8 @@ _EXTRA_PACKAGE_UNDER_TEST = ('org.chromium.chrome.test.pagecontroller.rules.' FEATURE_ANNOTATION = 'Feature' RENDER_TEST_FEATURE_ANNOTATION = 'RenderTest' +WPR_ARCHIVE_FILE_PATH_ANNOTATION = 'WPRArchiveDirectory' +WPR_RECORD_REPLAY_TEST_FEATURE_ANNOTATION = 'WPRRecordReplayTest' # This needs to be kept in sync with formatting in |RenderUtils.imageName| RE_RENDER_IMAGE_NAME = re.compile( @@ -101,6 +106,8 @@ RENDER_TEST_MODEL_SDK_CONFIGS = { 'Nexus 5X': [23], } +_TEST_BATCH_MAX_GROUP_SIZE = 256 + @contextlib.contextmanager def _LogTestEndpoints(device, test_name): @@ -136,16 +143,24 @@ _CURRENT_FOCUS_CRASH_RE = re.compile( r'\s*mCurrentFocus.*Application (Error|Not Responding): (\S+)}') +def _GetTargetPackageName(test_apk): + # apk_under_test does not work for smoke tests, where it is set to an + # apk that is not listed as the targetPackage in the test apk's manifest. + return test_apk.GetAllInstrumentations()[0]['android:targetPackage'] + + class LocalDeviceInstrumentationTestRun( local_device_test_run.LocalDeviceTestRun): def __init__(self, env, test_instance): super(LocalDeviceInstrumentationTestRun, self).__init__( env, test_instance) + self._chrome_proxy = None self._context_managers = collections.defaultdict(list) self._flag_changers = {} + self._render_tests_device_output_dir = None self._shared_prefs_to_restore = [] - self._skia_gold_work_dir = None self._skia_gold_session_manager = None + self._skia_gold_work_dir = None #override def TestPackage(self): @@ -153,6 +168,8 @@ class LocalDeviceInstrumentationTestRun( #override def SetUp(self): + target_package = _GetTargetPackageName(self._test_instance.test_apk) + @local_device_environment.handle_shard_failures_with( self._env.BlacklistDevice) @trace_event.traced @@ -267,18 +284,10 @@ class LocalDeviceInstrumentationTestRun( def set_debug_app(dev): # Set debug app in order to enable reading command line flags on user # builds - package_name = None - if self._test_instance.apk_under_test: - package_name = self._test_instance.apk_under_test.GetPackageName() - elif self._test_instance.test_apk: - package_name = self._test_instance.test_apk.GetPackageName() - else: - logging.error("Couldn't set debug app: no package name found") - return cmd = ['am', 'set-debug-app', '--persistent'] if self._test_instance.wait_for_java_debugger: cmd.append('-w') - cmd.append(package_name) + cmd.append(target_package) dev.RunShellCommand(cmd, check_return=True) @trace_event.traced @@ -379,13 +388,12 @@ class LocalDeviceInstrumentationTestRun( # expectations can be re-used between tests, saving a significant amount # of time. self._skia_gold_work_dir = tempfile.mkdtemp() - self._skia_gold_session_manager = gold_utils.SkiaGoldSessionManager( + self._skia_gold_session_manager = gold_utils.AndroidSkiaGoldSessionManager( self._skia_gold_work_dir, self._test_instance.skia_gold_properties) if self._test_instance.wait_for_java_debugger: - apk = self._test_instance.apk_under_test or self._test_instance.test_apk logging.warning('*' * 80) logging.warning('Waiting for debugger to attach to process: %s', - apk.GetPackageName()) + target_package) logging.warning('*' * 80) #override @@ -459,6 +467,31 @@ class LocalDeviceInstrumentationTestRun( return tests #override + def _GroupTests(self, tests): + batched_tests = dict() + other_tests = [] + for test in tests: + if 'Batch' in test['annotations']: + batch_name = test['annotations']['Batch']['value'] + if not batch_name: + batch_name = test['class'] + if not batch_name in batched_tests: + batched_tests[batch_name] = [] + batched_tests[batch_name].append(test) + else: + other_tests.append(test) + + all_tests = [] + for _, tests in batched_tests.items(): + tests.sort() # Ensure a consistent ordering across external shards. + all_tests.extend([ + tests[i:i + _TEST_BATCH_MAX_GROUP_SIZE] + for i in range(0, len(tests), _TEST_BATCH_MAX_GROUP_SIZE) + ]) + all_tests.extend(other_tests) + return all_tests + + #override def _GetUniqueTestName(self, test): return instrumentation_test_instance.GetUniqueTestName(test) @@ -506,12 +539,9 @@ class LocalDeviceInstrumentationTestRun( device.adb, suffix='.json', dir=device.GetExternalStoragePath()) extras[EXTRA_TRACE_FILE] = trace_device_file.name + target = '%s/%s' % (self._test_instance.test_package, + self._test_instance.junit4_runner_class) if isinstance(test, list): - if not self._test_instance.driver_apk: - raise Exception('driver_apk does not exist. ' - 'Please build it and try again.') - if any(t.get('is_junit4') for t in test): - raise Exception('driver apk does not support JUnit4 tests') def name_and_timeout(t): n = instrumentation_test_instance.GetTestName(t) @@ -520,26 +550,15 @@ class LocalDeviceInstrumentationTestRun( test_names, timeouts = zip(*(name_and_timeout(t) for t in test)) - test_name = ','.join(test_names) + test_name = instrumentation_test_instance.GetTestName(test[0]) + '_batch' + extras['class'] = ','.join(test_names) test_display_name = test_name - target = '%s/%s' % ( - self._test_instance.driver_package, - self._test_instance.driver_name) - extras.update( - self._test_instance.GetDriverEnvironmentVars( - test_list=test_names)) timeout = sum(timeouts) else: + assert test['is_junit4'] test_name = instrumentation_test_instance.GetTestName(test) test_display_name = self._GetUniqueTestName(test) - if test['is_junit4']: - target = '%s/%s' % ( - self._test_instance.test_package, - self._test_instance.junit4_runner_class) - else: - target = '%s/%s' % ( - self._test_instance.test_package, - self._test_instance.junit3_runner_class) + extras['class'] = test_name if 'flags' in test and test['flags']: flags_to_add.extend(test['flags']) @@ -556,14 +575,39 @@ class LocalDeviceInstrumentationTestRun( timeout = None logging.info('preparing to run %s: %s', test_display_name, test) - render_tests_device_output_dir = None if _IsRenderTest(test): # TODO(mikecase): Add DeviceTempDirectory class and use that instead. - render_tests_device_output_dir = posixpath.join( - device.GetExternalStoragePath(), - 'render_test_output_dir') + self._render_tests_device_output_dir = posixpath.join( + device.GetExternalStoragePath(), 'render_test_output_dir') flags_to_add.append('--render-test-output-dir=%s' % - render_tests_device_output_dir) + self._render_tests_device_output_dir) + + if _IsWPRRecordReplayTest(test): + wpr_archive_relative_path = _GetWPRArchivePath(test) + if not wpr_archive_relative_path: + raise RuntimeError('Could not find the WPR archive file path ' + 'from annotation.') + wpr_archive_path = os.path.join(host_paths.DIR_SOURCE_ROOT, + wpr_archive_relative_path) + if not os.path.isdir(wpr_archive_path): + raise RuntimeError('WPRArchiveDirectory annotation should point' + 'to a directory only.') + + archive_path = os.path.join(wpr_archive_path, + self._GetUniqueTestName(test) + '.wprgo') + + if not os.path.exists(_WPR_GO_LINUX_X86_64_PATH): + # If we got to this stage, then we should have + # checkout_android set. + raise RuntimeError( + 'WPR Go binary not found at {}'.format(_WPR_GO_LINUX_X86_64_PATH)) + # Tells the server to use the binaries retrieved from CIPD. + chrome_proxy_utils.ChromeProxySession.SetWPRServerBinary( + _WPR_GO_LINUX_X86_64_PATH) + self._chrome_proxy = chrome_proxy_utils.ChromeProxySession() + self._chrome_proxy.wpr_record_mode = self._test_instance.wpr_record_mode + self._chrome_proxy.Start(device, archive_path) + flags_to_add.extend(self._chrome_proxy.GetFlags()) if flags_to_add: self._CreateFlagChangerIfNeeded(device) @@ -588,7 +632,7 @@ class LocalDeviceInstrumentationTestRun( result_code, result_bundle, statuses = ( self._test_instance.ParseAmInstrumentRawOutput(output)) results = self._test_instance.GenerateTestResults( - result_code, result_bundle, statuses, start_ms, duration_ms, + result_code, result_bundle, statuses, duration_ms, device.product_cpu_abi, self._test_instance.symbolizer) if self._env.trace_output: @@ -620,11 +664,12 @@ class LocalDeviceInstrumentationTestRun( # check to see if any failure images were generated even if the test # does not fail. try: - self._ProcessRenderTestResults( - device, render_tests_device_output_dir, results) + self._ProcessRenderTestResults(device, results) finally: - device.RemovePath(render_tests_device_output_dir, - recursive=True, force=True) + device.RemovePath(self._render_tests_device_output_dir, + recursive=True, + force=True) + self._render_tests_device_output_dir = None def pull_ui_screen_captures(): screenshots = [] @@ -653,13 +698,23 @@ class LocalDeviceInstrumentationTestRun( json_data['image_link'] = image_archive.Link() return json_data + def stop_chrome_proxy(): + # Removes the port forwarding + if self._chrome_proxy: + self._chrome_proxy.Stop(device) + if not self._chrome_proxy.wpr_replay_mode: + logging.info('WPR Record test generated archive file %s', + self._chrome_proxy.wpr_archive_path) + self._chrome_proxy = None + + # While constructing the TestResult objects, we can parallelize several # steps that involve ADB. These steps should NOT depend on any info in # the results! Things such as whether the test CRASHED have not yet been # determined. post_test_steps = [ - restore_flags, restore_timeout_scale, handle_coverage_data, - handle_render_test_data, pull_ui_screen_captures + restore_flags, restore_timeout_scale, stop_chrome_proxy, + handle_coverage_data, handle_render_test_data, pull_ui_screen_captures ] if self._env.concurrent_adb: reraiser_thread.RunAsync(post_test_steps) @@ -920,16 +975,14 @@ class LocalDeviceInstrumentationTestRun( screenshot_device_file.close() _SetLinkOnResults(results, link_name, screenshot_host_file.Link()) - def _ProcessRenderTestResults( - self, device, render_tests_device_output_dir, results): - self._ProcessSkiaGoldRenderTestResults( - device, render_tests_device_output_dir, results) - self._ProcessLocalRenderTestResults(device, render_tests_device_output_dir, - results) + def _ProcessRenderTestResults(self, device, results): + if not self._render_tests_device_output_dir: + return + self._ProcessSkiaGoldRenderTestResults(device, results) - def _ProcessSkiaGoldRenderTestResults( - self, device, render_tests_device_output_dir, results): - gold_dir = posixpath.join(render_tests_device_output_dir, _DEVICE_GOLD_DIR) + def _ProcessSkiaGoldRenderTestResults(self, device, results): + gold_dir = posixpath.join(self._render_tests_device_output_dir, + _DEVICE_GOLD_DIR) if not device.FileExists(gold_dir): return @@ -958,8 +1011,27 @@ class LocalDeviceInstrumentationTestRun( 'when doing Skia Gold comparison.' % image_name) continue + # Add 'ignore': '1' if a comparison failure would not be surfaced, as + # that implies that we aren't actively maintaining baselines for the + # test. This helps prevent unrelated CLs from getting comments posted to + # them. + with open(json_path) as infile: + # All the key/value pairs in the JSON file are strings, so convert + # to a bool. + json_dict = json.load(infile) + fail_on_unsupported = json_dict.get('fail_on_unsupported_configs', + 'false') + fail_on_unsupported = fail_on_unsupported.lower() == 'true' + should_hide_failure = ( + device.build_version_sdk not in RENDER_TEST_MODEL_SDK_CONFIGS.get( + device.product_model, []) and not fail_on_unsupported) + if should_hide_failure: + json_dict['ignore'] = '1' + with open(json_path, 'w') as outfile: + json.dump(json_dict, outfile) + gold_session = self._skia_gold_session_manager.GetSkiaGoldSession( - keys_file=json_path) + keys_input=json_path) try: status, error = gold_session.RunComparison( @@ -978,14 +1050,7 @@ class LocalDeviceInstrumentationTestRun( # Don't fail the test if we ran on an unsupported configuration unless # the test has explicitly opted in, as it's likely that baselines # aren't maintained for that configuration. - with open(json_path) as infile: - # All the key/value pairs in the JSON file are strings, so convert - # to a bool. - fail_on_unsupported = json.load(infile).get( - 'fail_on_unsupported_configs', 'false') - fail_on_unsupported = fail_on_unsupported.lower() == 'true' - if device.build_version_sdk not in RENDER_TEST_MODEL_SDK_CONFIGS.get( - device.product_model, []) and not fail_on_unsupported: + if should_hide_failure: if self._test_instance.skia_gold_properties.local_pixel_tests: _AppendToLog( results, 'Gold comparison for %s failed, but model %s with SDK ' @@ -1004,7 +1069,7 @@ class LocalDeviceInstrumentationTestRun( failure_log = ( 'Skia Gold reported failure for RenderTest %s. See ' 'RENDER_TESTS.md for how to fix this failure.' % render_name) - status_codes = gold_utils.SkiaGoldSession.StatusCodes + status_codes = gold_utils.AndroidSkiaGoldSession.StatusCodes if status == status_codes.AUTH_FAILURE: _AppendToLog(results, 'Gold authentication failed with output %s' % error) @@ -1053,63 +1118,6 @@ class LocalDeviceInstrumentationTestRun( 'Given unhandled SkiaGoldSession StatusCode %s with error %s', status, error) - def _ProcessLocalRenderTestResults(self, device, - render_tests_device_output_dir, results): - failure_images_device_dir = posixpath.join( - render_tests_device_output_dir, 'failures') - if not device.FileExists(failure_images_device_dir): - return - - diff_images_device_dir = posixpath.join( - render_tests_device_output_dir, 'diffs') - - golden_images_device_dir = posixpath.join( - render_tests_device_output_dir, 'goldens') - - for failure_filename in device.ListDirectory(failure_images_device_dir): - - with self._env.output_manager.ArchivedTempfile( - 'fail_%s' % failure_filename, 'render_tests', - output_manager.Datatype.PNG) as failure_image_host_file: - device.PullFile( - posixpath.join(failure_images_device_dir, failure_filename), - failure_image_host_file.name) - failure_link = failure_image_host_file.Link() - - golden_image_device_file = posixpath.join( - golden_images_device_dir, failure_filename) - if device.PathExists(golden_image_device_file): - with self._env.output_manager.ArchivedTempfile( - 'golden_%s' % failure_filename, 'render_tests', - output_manager.Datatype.PNG) as golden_image_host_file: - device.PullFile( - golden_image_device_file, golden_image_host_file.name) - golden_link = golden_image_host_file.Link() - else: - golden_link = '' - - diff_image_device_file = posixpath.join( - diff_images_device_dir, failure_filename) - if device.PathExists(diff_image_device_file): - with self._env.output_manager.ArchivedTempfile( - 'diff_%s' % failure_filename, 'render_tests', - output_manager.Datatype.PNG) as diff_image_host_file: - device.PullFile( - diff_image_device_file, diff_image_host_file.name) - diff_link = diff_image_host_file.Link() - else: - diff_link = '' - - processed_template_output = _GenerateRenderTestHtml( - failure_filename, failure_link, golden_link, diff_link) - - with self._env.output_manager.ArchivedTempfile( - '%s.html' % failure_filename, 'render_tests', - output_manager.Datatype.HTML) as html_results: - html_results.write(processed_template_output) - html_results.flush() - _SetLinkOnResults(results, failure_filename, html_results.Link()) - #override def _ShouldRetry(self, test, result): # We've tried to disable retries in the past with mixed results. @@ -1145,6 +1153,22 @@ class LocalDeviceInstrumentationTestRun( return timeout +def _IsWPRRecordReplayTest(test): + """Determines whether a test or a list of tests is a WPR RecordReplay Test.""" + if not isinstance(test, list): + test = [test] + return any([ + WPR_RECORD_REPLAY_TEST_FEATURE_ANNOTATION in t['annotations'].get( + FEATURE_ANNOTATION, {}).get('value', ()) for t in test + ]) + + +def _GetWPRArchivePath(test): + """Retrieves the archive path from the WPRArchiveDirectory annotation.""" + return test['annotations'].get(WPR_ARCHIVE_FILE_PATH_ANNOTATION, + {}).get('value', ()) + + def _IsRenderTest(test): """Determines if a test or list of tests has a RenderTest amongst them.""" if not isinstance(test, list): diff --git a/chromium/build/android/pylib/local/device/local_device_instrumentation_test_run_test.py b/chromium/build/android/pylib/local/device/local_device_instrumentation_test_run_test.py index 3129c1121b0..dd57d92061e 100755 --- a/chromium/build/android/pylib/local/device/local_device_instrumentation_test_run_test.py +++ b/chromium/build/android/pylib/local/device/local_device_instrumentation_test_run_test.py @@ -61,6 +61,88 @@ class LocalDeviceInstrumentationTestRunTest(unittest.TestCase): 'SadTest.testNotRun', base_test_result.ResultType.NOTRUN) self.assertTrue(self._obj._ShouldRetry(test, result)) + def testIsWPRRecordReplayTest_matchedWithKey(self): + test = { + 'annotations': { + 'Feature': { + 'value': ['WPRRecordReplayTest', 'dummy'] + } + }, + 'class': 'WPRDummyTest', + 'method': 'testRun', + 'is_junit4': True, + } + self.assertTrue( + local_device_instrumentation_test_run._IsWPRRecordReplayTest(test)) + + def testIsWPRRecordReplayTest_noMatchedKey(self): + test = { + 'annotations': { + 'Feature': { + 'value': ['abc', 'dummy'] + } + }, + 'class': 'WPRDummyTest', + 'method': 'testRun', + 'is_junit4': True, + } + self.assertFalse( + local_device_instrumentation_test_run._IsWPRRecordReplayTest(test)) + + def testGetWPRArchivePath_matchedWithKey(self): + test = { + 'annotations': { + 'WPRArchiveDirectory': { + 'value': 'abc' + } + }, + 'class': 'WPRDummyTest', + 'method': 'testRun', + 'is_junit4': True, + } + self.assertEqual( + local_device_instrumentation_test_run._GetWPRArchivePath(test), 'abc') + + def testGetWPRArchivePath_noMatchedWithKey(self): + test = { + 'annotations': { + 'Feature': { + 'value': 'abc' + } + }, + 'class': 'WPRDummyTest', + 'method': 'testRun', + 'is_junit4': True, + } + self.assertFalse( + local_device_instrumentation_test_run._GetWPRArchivePath(test)) + + def testIsRenderTest_matchedWithKey(self): + test = { + 'annotations': { + 'Feature': { + 'value': ['RenderTest', 'dummy'] + } + }, + 'class': 'DummyTest', + 'method': 'testRun', + 'is_junit4': True, + } + self.assertTrue(local_device_instrumentation_test_run._IsRenderTest(test)) + + def testIsRenderTest_noMatchedKey(self): + test = { + 'annotations': { + 'Feature': { + 'value': ['abc', 'dummy'] + } + }, + 'class': 'DummyTest', + 'method': 'testRun', + 'is_junit4': True, + } + self.assertFalse(local_device_instrumentation_test_run._IsRenderTest(test)) + if __name__ == '__main__': unittest.main(verbosity=2) diff --git a/chromium/build/android/pylib/local/device/local_device_test_run.py b/chromium/build/android/pylib/local/device/local_device_test_run.py index 2018751fed5..69b27186507 100644 --- a/chromium/build/android/pylib/local/device/local_device_test_run.py +++ b/chromium/build/android/pylib/local/device/local_device_test_run.py @@ -137,6 +137,7 @@ class LocalDeviceTestRun(test_run.TestRun): with signal_handler.AddSignalHandler(signal.SIGTERM, stop_tests): tries = 0 while tries < self._env.max_tries and tests: + grouped_tests = self._GroupTests(tests) logging.info('STARTING TRY #%d/%d', tries + 1, self._env.max_tries) if tries > 0 and self._env.recover_devices: if any(d.build_version_sdk == version_codes.LOLLIPOP_MR1 @@ -171,12 +172,14 @@ class LocalDeviceTestRun(test_run.TestRun): try: if self._ShouldShard(): - tc = test_collection.TestCollection(self._CreateShards(tests)) + tc = test_collection.TestCollection( + self._CreateShards(grouped_tests)) self._env.parallel_devices.pMap( run_tests_on_device, tc, try_results).pGet(None) else: - self._env.parallel_devices.pMap( - run_tests_on_device, tests, try_results).pGet(None) + self._env.parallel_devices.pMap(run_tests_on_device, + grouped_tests, + try_results).pGet(None) except TestsTerminated: for unknown_result in try_results.GetUnknown(): try_results.AddResult( @@ -236,9 +239,16 @@ class LocalDeviceTestRun(test_run.TestRun): if total_shards < 0 or shard_index < 0 or total_shards <= shard_index: raise InvalidShardingSettings(shard_index, total_shards) - return [ - t for t in tests - if hash(self._GetUniqueTestName(t)) % total_shards == shard_index] + sharded_tests = [] + for t in self._GroupTests(tests): + if (hash(self._GetUniqueTestName(t[0] if isinstance(t, list) else t)) % + total_shards == shard_index): + if isinstance(t, list): + sharded_tests.extend(t) + else: + sharded_tests.append(t) + + return sharded_tests def GetTool(self, device): if str(device) not in self._tools: @@ -260,6 +270,10 @@ class LocalDeviceTestRun(test_run.TestRun): def _GetTests(self): raise NotImplementedError + def _GroupTests(self, tests): + # pylint: disable=no-self-use + return tests + def _RunTest(self, device, test): raise NotImplementedError diff --git a/chromium/build/android/pylib/local/device/local_device_test_run_test.py b/chromium/build/android/pylib/local/device/local_device_test_run_test.py index 525bf25200b..aeea5881c8c 100755 --- a/chromium/build/android/pylib/local/device/local_device_test_run_test.py +++ b/chromium/build/android/pylib/local/device/local_device_test_run_test.py @@ -8,11 +8,9 @@ import unittest from pylib.base import base_test_result -from pylib.constants import host_paths from pylib.local.device import local_device_test_run -with host_paths.SysPath(host_paths.PYMOCK_PATH): - import mock # pylint: disable=import-error +import mock # pylint: disable=import-error class SubstituteDeviceRootTest(unittest.TestCase): diff --git a/chromium/build/android/pylib/output/remote_output_manager_test.py b/chromium/build/android/pylib/output/remote_output_manager_test.py index 6917260dd7c..d87c6eb3a9c 100755 --- a/chromium/build/android/pylib/output/remote_output_manager_test.py +++ b/chromium/build/android/pylib/output/remote_output_manager_test.py @@ -9,11 +9,9 @@ import unittest from pylib.base import output_manager from pylib.base import output_manager_test_case -from pylib.constants import host_paths from pylib.output import remote_output_manager -with host_paths.SysPath(host_paths.PYMOCK_PATH): - import mock # pylint: disable=import-error +import mock # pylint: disable=import-error @mock.patch('pylib.utils.google_storage_helper') diff --git a/chromium/build/android/pylib/symbols/deobfuscator.py b/chromium/build/android/pylib/symbols/deobfuscator.py index 42084ddc789..ffc23b87048 100644 --- a/chromium/build/android/pylib/symbols/deobfuscator.py +++ b/chromium/build/android/pylib/symbols/deobfuscator.py @@ -150,7 +150,7 @@ class DeobfuscatorPool(object): # De-obfuscation is broken. if self._num_restarts == _MAX_RESTARTS: - return lines + raise Exception('Deobfuscation seems broken.') # Restart any closed Deobfuscators. for i, d in enumerate(self._pool): diff --git a/chromium/build/android/pylib/utils/app_bundle_utils.py b/chromium/build/android/pylib/utils/app_bundle_utils.py index f076ed39cd6..59efb775a66 100644 --- a/chromium/build/android/pylib/utils/app_bundle_utils.py +++ b/chromium/build/android/pylib/utils/app_bundle_utils.py @@ -18,6 +18,8 @@ import bundletool # List of valid modes for GenerateBundleApks() BUILD_APKS_MODES = ('default', 'universal', 'system', 'system_compressed') +OPTIMIZE_FOR_OPTIONS = ('ABI', 'SCREEN_DENSITY', 'LANGUAGE', + 'TEXTURE_COMPRESSION_FORMAT') _SYSTEM_MODES = ('system_compressed', 'system') _ALL_ABIS = ['armeabi-v7a', 'arm64-v8a', 'x86', 'x86_64'] @@ -50,7 +52,8 @@ def GenerateBundleApks(bundle_path, minimal=False, minimal_sdk_version=None, check_for_noop=True, - system_image_locales=None): + system_image_locales=None, + optimize_for=None): """Generate an .apks archive from a an app bundle if needed. Args: @@ -68,6 +71,8 @@ def GenerateBundleApks(bundle_path, check_for_noop: Use md5_check to short-circuit when inputs have not changed. system_image_locales: Locales to package in the APK when mode is "system" or "system_compressed". + optimize_for: Overrides split configuration, which must be None or + one of OPTIMIZE_FOR_OPTIONS. """ device_spec = None if minimal_sdk_version: @@ -110,6 +115,13 @@ def GenerateBundleApks(bundle_path, (mode, BUILD_APKS_MODES)) cmd_args += ['--mode=' + mode] + if optimize_for: + if optimize_for not in OPTIMIZE_FOR_OPTIONS: + raise Exception('Invalid optimize_for parameter %s ' + '(should be in %s)' % + (mode, OPTIMIZE_FOR_OPTIONS)) + cmd_args += ['--optimize-for=' + optimize_for] + with tempfile.NamedTemporaryFile(suffix='.json') as spec_file: if device_spec: json.dump(device_spec, spec_file) diff --git a/chromium/build/android/pylib/utils/chrome_proxy_utils.py b/chromium/build/android/pylib/utils/chrome_proxy_utils.py new file mode 100644 index 00000000000..149d0b9c8c5 --- /dev/null +++ b/chromium/build/android/pylib/utils/chrome_proxy_utils.py @@ -0,0 +1,171 @@ +# Copyright 2020 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. +"""Utilities for setting up and tear down WPR and TsProxy service.""" + +from py_utils import ts_proxy_server +from py_utils import webpagereplay_go_server + +from devil.android import forwarder + +PROXY_HOST_IP = '127.0.0.1' +# From Catapult/WebPageReplay document. +IGNORE_CERT_ERROR_SPKI_LIST = 'PhrPvGIaAMmd29hj8BCZOq096yj7uMpRNHpn5PDxI6I=' +PROXY_SERVER = 'socks5://localhost' +DEFAULT_DEVICE_PORT = 1080 +DEFAULT_ROUND_TRIP_LATENCY_MS = 100 +DEFAULT_DOWNLOAD_BANDWIDTH_KBPS = 72000 +DEFAULT_UPLOAD_BANDWIDTH_KBPS = 72000 + + +class WPRServer(object): + """Utils to set up a webpagereplay_go_server instance.""" + + def __init__(self): + self._archive_path = None + self._host_http_port = 0 + self._host_https_port = 0 + self._record_mode = False + self._server = None + + def StartServer(self, wpr_archive_path): + """Starts a webpagereplay_go_server instance.""" + if wpr_archive_path == self._archive_path and self._server: + # Reuse existing webpagereplay_go_server instance. + return + + if self._server: + self.StopServer() + + replay_options = [] + if self._record_mode: + replay_options.append('--record') + + ports = {} + if not self._server: + self._server = webpagereplay_go_server.ReplayServer( + wpr_archive_path, + PROXY_HOST_IP, + http_port=self._host_http_port, + https_port=self._host_https_port, + replay_options=replay_options) + self._archive_path = wpr_archive_path + ports = self._server.StartServer() + + self._host_http_port = ports['http'] + self._host_https_port = ports['https'] + + def StopServer(self): + """Stops the webpagereplay_go_server instance and resets archive.""" + self._server.StopServer() + self._server = None + self._host_http_port = 0 + self._host_https_port = 0 + + @staticmethod + def SetServerBinaryPath(go_binary_path): + """Sets the go_binary_path for webpagereplay_go_server.ReplayServer.""" + webpagereplay_go_server.ReplayServer.SetGoBinaryPath(go_binary_path) + + @property + def record_mode(self): + return self._record_mode + + @record_mode.setter + def record_mode(self, value): + self._record_mode = value + + @property + def http_port(self): + return self._host_http_port + + @property + def https_port(self): + return self._host_https_port + + @property + def archive_path(self): + return self._archive_path + + +class ChromeProxySession(object): + """Utils to help set up a Chrome Proxy.""" + + def __init__(self, device_proxy_port=DEFAULT_DEVICE_PORT): + self._device_proxy_port = device_proxy_port + self._ts_proxy_server = ts_proxy_server.TsProxyServer(PROXY_HOST_IP) + self._wpr_server = WPRServer() + + @property + def wpr_record_mode(self): + """Returns whether this proxy session was running in record mode.""" + return self._wpr_server.record_mode + + @wpr_record_mode.setter + def wpr_record_mode(self, value): + self._wpr_server.record_mode = value + + @property + def wpr_replay_mode(self): + """Returns whether this proxy session was running in replay mode.""" + return not self._wpr_server.record_mode + + @property + def wpr_archive_path(self): + """Returns the wpr archive file path used in this proxy session.""" + return self._wpr_server.archive_path + + @property + def device_proxy_port(self): + return self._device_proxy_port + + def GetFlags(self): + """Gets the chrome command line flags to be needed by ChromeProxySession.""" + extra_flags = [] + + extra_flags.append('--ignore-certificate-errors-spki-list=%s' % + IGNORE_CERT_ERROR_SPKI_LIST) + extra_flags.append('--proxy-server=%s:%s' % + (PROXY_SERVER, self._device_proxy_port)) + return extra_flags + + @staticmethod + def SetWPRServerBinary(go_binary_path): + """Sets the WPR server go_binary_path.""" + WPRServer.SetServerBinaryPath(go_binary_path) + + def Start(self, device, wpr_archive_path): + """Starts the wpr_server as well as the ts_proxy server and setups env. + + Args: + device: A DeviceUtils instance. + wpr_archive_path: A abs path to the wpr archive file. + + """ + self._wpr_server.StartServer(wpr_archive_path) + self._ts_proxy_server.StartServer() + + # Maps device port to host port + forwarder.Forwarder.Map( + [(self._device_proxy_port, self._ts_proxy_server.port)], device) + # Maps tsProxy port to wpr http/https ports + self._ts_proxy_server.UpdateOutboundPorts( + http_port=self._wpr_server.http_port, + https_port=self._wpr_server.https_port) + self._ts_proxy_server.UpdateTrafficSettings( + round_trip_latency_ms=DEFAULT_ROUND_TRIP_LATENCY_MS, + download_bandwidth_kbps=DEFAULT_DOWNLOAD_BANDWIDTH_KBPS, + upload_bandwidth_kbps=DEFAULT_UPLOAD_BANDWIDTH_KBPS) + + def Stop(self, device): + """Stops the wpr_server, and ts_proxy server and tears down env. + + Note that Stop does not reset wpr_record_mode, wpr_replay_mode, + wpr_archive_path property. + + Args: + device: A DeviceUtils instance. + """ + self._wpr_server.StopServer() + self._ts_proxy_server.StopServer() + forwarder.Forwarder.UnmapDevicePort(self._device_proxy_port, device) diff --git a/chromium/build/android/pylib/utils/chrome_proxy_utils_test.py b/chromium/build/android/pylib/utils/chrome_proxy_utils_test.py new file mode 100755 index 00000000000..b38b268fe8a --- /dev/null +++ b/chromium/build/android/pylib/utils/chrome_proxy_utils_test.py @@ -0,0 +1,235 @@ +#!/usr/bin/env vpython +# Copyright 2020 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. +"""Tests for chrome_proxy_utils.""" + +#pylint: disable=protected-access + +import os +import unittest + +from pylib.utils import chrome_proxy_utils + +from devil.android import forwarder +from devil.android import device_utils +from devil.android.sdk import adb_wrapper +from py_utils import ts_proxy_server +from py_utils import webpagereplay_go_server + +import mock # pylint: disable=import-error + + +def _DeviceUtilsMock(test_serial, is_ready=True): + """Returns a DeviceUtils instance based on given serial.""" + adb = mock.Mock(spec=adb_wrapper.AdbWrapper) + adb.__str__ = mock.Mock(return_value=test_serial) + adb.GetDeviceSerial.return_value = test_serial + adb.is_ready = is_ready + return device_utils.DeviceUtils(adb) + + +class ChromeProxySessionTest(unittest.TestCase): + """Unittest for ChromeProxySession.""" + + #pylint: disable=no-self-use + + @mock.patch.object(forwarder.Forwarder, 'Map') + @mock.patch.object(chrome_proxy_utils.WPRServer, 'StartServer') + @mock.patch.object(ts_proxy_server.TsProxyServer, 'StartServer') + @mock.patch.object(ts_proxy_server.TsProxyServer, 'UpdateOutboundPorts') + @mock.patch.object(ts_proxy_server.TsProxyServer, 'UpdateTrafficSettings') + @mock.patch('py_utils.ts_proxy_server.TsProxyServer.port', + new_callable=mock.PropertyMock) + def test_Start(self, port_mock, traffic_setting_mock, outboundport_mock, + start_server_mock, wpr_mock, forwarder_mock): + chrome_proxy = chrome_proxy_utils.ChromeProxySession(4) + chrome_proxy._wpr_server._host_http_port = 1 + chrome_proxy._wpr_server._host_https_port = 2 + port_mock.return_value = 3 + device = _DeviceUtilsMock('01234') + chrome_proxy.Start(device, 'abc') + + forwarder_mock.assert_called_once_with([(4, 3)], device) + wpr_mock.assert_called_once_with('abc') + start_server_mock.assert_called_once() + outboundport_mock.assert_called_once_with(http_port=1, https_port=2) + traffic_setting_mock.assert_called_once_with(download_bandwidth_kbps=72000, + round_trip_latency_ms=100, + upload_bandwidth_kbps=72000) + port_mock.assert_called_once() + + @mock.patch.object(forwarder.Forwarder, 'UnmapDevicePort') + @mock.patch.object(chrome_proxy_utils.WPRServer, 'StopServer') + @mock.patch.object(ts_proxy_server.TsProxyServer, 'StopServer') + def test_Stop(self, ts_proxy_mock, wpr_mock, forwarder_mock): + chrome_proxy = chrome_proxy_utils.ChromeProxySession(4) + device = _DeviceUtilsMock('01234') + chrome_proxy.wpr_record_mode = True + chrome_proxy._wpr_server._archive_path = 'abc' + chrome_proxy.Stop(device) + + forwarder_mock.assert_called_once_with(4, device) + wpr_mock.assert_called_once_with() + ts_proxy_mock.assert_called_once_with() + + #pylint: enable=no-self-use + + @mock.patch.object(forwarder.Forwarder, 'UnmapDevicePort') + @mock.patch.object(webpagereplay_go_server.ReplayServer, 'StopServer') + @mock.patch.object(ts_proxy_server.TsProxyServer, 'StopServer') + def test_Stop_WithProperties(self, ts_proxy_mock, wpr_mock, forwarder_mock): + chrome_proxy = chrome_proxy_utils.ChromeProxySession(4) + chrome_proxy._wpr_server._server = webpagereplay_go_server.ReplayServer( + os.path.abspath(__file__), chrome_proxy_utils.PROXY_HOST_IP, 0, 0, []) + chrome_proxy._wpr_server._archive_path = os.path.abspath(__file__) + device = _DeviceUtilsMock('01234') + chrome_proxy.wpr_record_mode = True + chrome_proxy.Stop(device) + + forwarder_mock.assert_called_once_with(4, device) + wpr_mock.assert_called_once_with() + ts_proxy_mock.assert_called_once_with() + self.assertFalse(chrome_proxy.wpr_replay_mode) + self.assertEquals(chrome_proxy.wpr_archive_path, os.path.abspath(__file__)) + + def test_SetWPRRecordMode(self): + chrome_proxy = chrome_proxy_utils.ChromeProxySession(4) + chrome_proxy.wpr_record_mode = True + self.assertTrue(chrome_proxy._wpr_server.record_mode) + self.assertTrue(chrome_proxy.wpr_record_mode) + self.assertFalse(chrome_proxy.wpr_replay_mode) + + chrome_proxy.wpr_record_mode = False + self.assertFalse(chrome_proxy._wpr_server.record_mode) + self.assertFalse(chrome_proxy.wpr_record_mode) + self.assertTrue(chrome_proxy.wpr_replay_mode) + + def test_SetWPRArchivePath(self): + chrome_proxy = chrome_proxy_utils.ChromeProxySession(4) + chrome_proxy._wpr_server._archive_path = 'abc' + self.assertEquals(chrome_proxy.wpr_archive_path, 'abc') + + def test_UseDefaultDeviceProxyPort(self): + chrome_proxy = chrome_proxy_utils.ChromeProxySession() + expected_flags = [ + '--ignore-certificate-errors-spki-list=' + 'PhrPvGIaAMmd29hj8BCZOq096yj7uMpRNHpn5PDxI6I=', + '--proxy-server=socks5://localhost:1080' + ] + self.assertEquals(chrome_proxy.device_proxy_port, 1080) + self.assertListEqual(chrome_proxy.GetFlags(), expected_flags) + + def test_UseNewDeviceProxyPort(self): + chrome_proxy = chrome_proxy_utils.ChromeProxySession(1) + expected_flags = [ + '--ignore-certificate-errors-spki-list=' + 'PhrPvGIaAMmd29hj8BCZOq096yj7uMpRNHpn5PDxI6I=', + '--proxy-server=socks5://localhost:1' + ] + self.assertEquals(chrome_proxy.device_proxy_port, 1) + self.assertListEqual(chrome_proxy.GetFlags(), expected_flags) + + +class WPRServerTest(unittest.TestCase): + @mock.patch('py_utils.webpagereplay_go_server.ReplayServer') + def test_StartSever_fresh_replaymode(self, wpr_mock): + wpr_server = chrome_proxy_utils.WPRServer() + wpr_archive_file = os.path.abspath(__file__) + wpr_server.StartServer(wpr_archive_file) + + wpr_mock.assert_called_once_with(wpr_archive_file, + '127.0.0.1', + http_port=0, + https_port=0, + replay_options=[]) + + self.assertEqual(wpr_server._archive_path, wpr_archive_file) + self.assertTrue(wpr_server._server) + + @mock.patch('py_utils.webpagereplay_go_server.ReplayServer') + def test_StartSever_fresh_recordmode(self, wpr_mock): + wpr_server = chrome_proxy_utils.WPRServer() + wpr_server.record_mode = True + wpr_server.StartServer(os.path.abspath(__file__)) + wpr_archive_file = os.path.abspath(__file__) + + wpr_mock.assert_called_once_with(wpr_archive_file, + '127.0.0.1', + http_port=0, + https_port=0, + replay_options=['--record']) + + self.assertEqual(wpr_server._archive_path, os.path.abspath(__file__)) + self.assertTrue(wpr_server._server) + + #pylint: disable=no-self-use + + @mock.patch.object(webpagereplay_go_server.ReplayServer, 'StartServer') + def test_StartSever_recordmode(self, start_server_mock): + wpr_server = chrome_proxy_utils.WPRServer() + start_server_mock.return_value = {'http': 1, 'https': 2} + wpr_server.StartServer(os.path.abspath(__file__)) + + start_server_mock.assert_called_once() + self.assertEqual(wpr_server._host_http_port, 1) + self.assertEqual(wpr_server._host_https_port, 2) + self.assertEqual(wpr_server._archive_path, os.path.abspath(__file__)) + self.assertTrue(wpr_server._server) + + @mock.patch.object(webpagereplay_go_server.ReplayServer, 'StartServer') + def test_StartSever_reuseServer(self, start_server_mock): + wpr_server = chrome_proxy_utils.WPRServer() + wpr_server._server = webpagereplay_go_server.ReplayServer( + os.path.abspath(__file__), + chrome_proxy_utils.PROXY_HOST_IP, + http_port=0, + https_port=0, + replay_options=[]) + wpr_server._archive_path = os.path.abspath(__file__) + wpr_server.StartServer(os.path.abspath(__file__)) + start_server_mock.assert_not_called() + + @mock.patch.object(webpagereplay_go_server.ReplayServer, 'StartServer') + @mock.patch.object(webpagereplay_go_server.ReplayServer, 'StopServer') + def test_StartSever_notReuseServer(self, stop_server_mock, start_server_mock): + wpr_server = chrome_proxy_utils.WPRServer() + wpr_server._server = webpagereplay_go_server.ReplayServer( + os.path.abspath(__file__), + chrome_proxy_utils.PROXY_HOST_IP, + http_port=0, + https_port=0, + replay_options=[]) + wpr_server._archive_path = '' + wpr_server.StartServer(os.path.abspath(__file__)) + start_server_mock.assert_called_once() + stop_server_mock.assert_called_once() + + #pylint: enable=no-self-use + + @mock.patch.object(webpagereplay_go_server.ReplayServer, 'StopServer') + def test_StopServer(self, stop_server_mock): + wpr_server = chrome_proxy_utils.WPRServer() + wpr_server._server = webpagereplay_go_server.ReplayServer( + os.path.abspath(__file__), + chrome_proxy_utils.PROXY_HOST_IP, + http_port=0, + https_port=0, + replay_options=[]) + wpr_server.StopServer() + stop_server_mock.assert_called_once() + self.assertFalse(wpr_server._server) + self.assertFalse(wpr_server._archive_path) + self.assertFalse(wpr_server.http_port) + self.assertFalse(wpr_server.https_port) + + def test_SetWPRRecordMode(self): + wpr_server = chrome_proxy_utils.WPRServer() + wpr_server.record_mode = True + self.assertTrue(wpr_server.record_mode) + wpr_server.record_mode = False + self.assertFalse(wpr_server.record_mode) + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff --git a/chromium/build/android/pylib/utils/gold_utils.py b/chromium/build/android/pylib/utils/gold_utils.py index 2b0aa60434f..f4f0840e429 100644 --- a/chromium/build/android/pylib/utils/gold_utils.py +++ b/chromium/build/android/pylib/utils/gold_utils.py @@ -1,332 +1,31 @@ # Copyright 2020 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. -"""Utilities for interacting with the Skia Gold image diffing service.""" +"""//build/android implementations of //testing/skia_gold_common. + +Used for interacting with the Skia Gold image diffing service. +""" -import json -import logging import os import shutil -import tempfile from devil.utils import cmd_helper from pylib.base.output_manager import Datatype from pylib.constants import host_paths -from pylib.utils import local_utils from pylib.utils import repo_utils -DEFAULT_INSTANCE = 'chrome' - -GOLDCTL_BINARY = os.path.join(host_paths.DIR_SOURCE_ROOT, 'tools', - 'skia_goldctl', 'linux', 'goldctl') - - -class SkiaGoldSession(object): - class StatusCodes(object): - """Status codes for RunComparison.""" - SUCCESS = 0 - AUTH_FAILURE = 1 - INIT_FAILURE = 2 - COMPARISON_FAILURE_REMOTE = 3 - COMPARISON_FAILURE_LOCAL = 4 - LOCAL_DIFF_FAILURE = 5 - NO_OUTPUT_MANAGER = 6 - - class ComparisonResults(object): - """Struct-like object for storing results of an image comparison.""" - - def __init__(self): - self.triage_link = None - self.triage_link_omission_reason = None - self.local_diff_given_image = None - self.local_diff_closest_image = None - self.local_diff_diff_image = None - - def __init__(self, - working_dir, - gold_properties, - keys_file, - corpus, - instance=DEFAULT_INSTANCE): - """A class to handle all aspects of an image comparison via Skia Gold. - - A single SkiaGoldSession is valid for a single instance/corpus/keys_file - combination. - - Args: - working_dir: The directory to store config files, etc. Sharing the same - working directory between multiple SkiaGoldSessions allows re-use of - authentication and downloaded baselines. - gold_properties: A SkiaGoldProperties instance for the current test run. - keys_file: A path to a JSON file containing various comparison config - data such as corpus and debug information like the hardware/software - configuration the images will be produced on. - corpus: The corpus that images that will be compared belong to. - instance: The name of the Skia Gold instance to interact with. - """ - self._working_dir = working_dir - self._gold_properties = gold_properties - self._keys_file = keys_file - self._corpus = corpus - self._instance = instance - self._triage_link_file = tempfile.NamedTemporaryFile( - suffix='.txt', dir=working_dir, delete=False).name - # A map of image name (string) to ComparisonResults for that image. - self._comparison_results = {} - self._authenticated = False - self._initialized = False - - # pylint: disable=too-many-return-statements - def RunComparison(self, - name, - png_file, - output_manager, - use_luci=True): - """Helper method to run all steps to compare a produced image. - - Handles authentication, initialization, comparison, and, if necessary, - local diffing. +with host_paths.SysPath(host_paths.BUILD_PATH): + from skia_gold_common import skia_gold_session + from skia_gold_common import skia_gold_session_manager + from skia_gold_common import skia_gold_properties - Args: - name: The name of the image being compared. - png_file: A path to a PNG file containing the image to be compared. - output_manager: The output manager used to save local diff images if - necessary. Can be None, but will fail if it ends up needing to be used - and is not set. - use_luci: If true, authentication will use the service account provided - by the LUCI context. If false, will attempt to use whatever is set up - in gsutil, which is only supported for local runs. - Returns: - A tuple (status, error). |status| is a value from - SkiaGoldSession.StatusCodes signifying the result of the comparison. - |error| is an error message describing the status if not successful. - """ - auth_rc, auth_stdout = self.Authenticate(use_luci=use_luci) - if auth_rc: - return self.StatusCodes.AUTH_FAILURE, auth_stdout - - init_rc, init_stdout = self.Initialize() - if init_rc: - return self.StatusCodes.INIT_FAILURE, init_stdout - - compare_rc, compare_stdout = self.Compare(name=name, png_file=png_file) - if not compare_rc: - return self.StatusCodes.SUCCESS, None - - logging.error('Gold comparison failed: %s', compare_stdout) - if not self._gold_properties.local_pixel_tests: - return self.StatusCodes.COMPARISON_FAILURE_REMOTE, compare_stdout - - if not output_manager: - return (self.StatusCodes.NO_OUTPUT_MANAGER, - 'No output manager for local diff images') - diff_rc, diff_stdout = self.Diff( - name=name, png_file=png_file, output_manager=output_manager) - if diff_rc: - return self.StatusCodes.LOCAL_DIFF_FAILURE, diff_stdout - return self.StatusCodes.COMPARISON_FAILURE_LOCAL, compare_stdout - - def Authenticate(self, use_luci=True): - """Authenticates with Skia Gold for this session. +class AndroidSkiaGoldSession(skia_gold_session.SkiaGoldSession): + def _StoreDiffLinks(self, image_name, output_manager, output_dir): + """See SkiaGoldSession._StoreDiffLinks for general documentation. - Args: - use_luci: If true, authentication will use the service account provided - by the LUCI context. If false, will attempt to use whatever is set up - in gsutil, which is only supported for local runs. - - Returns: - A tuple (return_code, output). |return_code| is the return code of the - authentication process. |output| is the stdout + stderr of the - authentication process. + |output_manager| must be a build.android.pylib.base.OutputManager instance. """ - if self._authenticated: - return 0, None - if self._gold_properties.bypass_skia_gold_functionality: - logging.warning('Not actually authenticating with Gold due to ' - '--bypass-skia-gold-functionality being present.') - return 0, None - - auth_cmd = [GOLDCTL_BINARY, 'auth', '--work-dir', self._working_dir] - if use_luci: - auth_cmd.append('--luci') - elif not self._gold_properties.local_pixel_tests: - raise RuntimeError( - 'Cannot authenticate to Skia Gold with use_luci=False unless running ' - 'local pixel tests') - - rc, stdout, _ = cmd_helper.GetCmdStatusOutputAndError( - auth_cmd, merge_stderr=True) - if rc == 0: - self._authenticated = True - return rc, stdout - - def Initialize(self): - """Initializes the working directory if necessary. - - This can technically be skipped if the same information is passed to the - command used for image comparison, but that is less efficient under the - hood. Doing it that way effectively requires an initialization for every - comparison (~250 ms) instead of once at the beginning. - - Returns: - A tuple (return_code, output). |return_code| is the return code of the - initialization process. |output| is the stdout + stderr of the - initialization process. - """ - if self._initialized: - return 0, None - if self._gold_properties.bypass_skia_gold_functionality: - logging.warning('Not actually initializing Gold due to ' - '--bypass-skia-gold-functionality being present.') - return 0, None - - init_cmd = [ - GOLDCTL_BINARY, - 'imgtest', - 'init', - '--passfail', - '--instance', - self._instance, - '--corpus', - self._corpus, - '--keys-file', - self._keys_file, - '--work-dir', - self._working_dir, - '--failure-file', - self._triage_link_file, - '--commit', - self._gold_properties.git_revision, - ] - if self._gold_properties.IsTryjobRun(): - init_cmd.extend([ - '--issue', - str(self._gold_properties.issue), - '--patchset', - str(self._gold_properties.patchset), - '--jobid', - str(self._gold_properties.job_id), - '--crs', - str(self._gold_properties.code_review_system), - '--cis', - str(self._gold_properties.continuous_integration_system), - ]) - rc, stdout, _ = cmd_helper.GetCmdStatusOutputAndError( - init_cmd, merge_stderr=True) - if rc == 0: - self._initialized = True - return rc, stdout - - def Compare(self, name, png_file): - """Compares the given image to images known to Gold. - - Triage links can later be retrieved using GetTriageLink(). - - Args: - name: The name of the image being compared. - png_file: A path to a PNG file containing the image to be compared. - - Returns: - A tuple (return_code, output). |return_code| is the return code of the - comparison process. |output| is the stdout + stderr of the comparison - process. - """ - if self._gold_properties.bypass_skia_gold_functionality: - logging.warning('Not actually comparing with Gold due to ' - '--bypass-skia-gold-functionality being present.') - return 0, None - - compare_cmd = [ - GOLDCTL_BINARY, - 'imgtest', - 'add', - '--test-name', - name, - '--png-file', - png_file, - '--work-dir', - self._working_dir, - ] - if self._gold_properties.local_pixel_tests: - compare_cmd.append('--dryrun') - - rc, stdout, _ = cmd_helper.GetCmdStatusOutputAndError( - compare_cmd, merge_stderr=True) - - self._comparison_results[name] = self.ComparisonResults() - if rc == 0: - self._comparison_results[name].triage_link_omission_reason = ( - 'Comparison succeeded, no triage link') - elif self._gold_properties.IsTryjobRun(): - # TODO(skbug.com/9879): Remove the explicit corpus when Gold's UI is - # updated to show results from all corpora for tryjobs. - cl_triage_link = ('https://{instance}-gold.skia.org/search?' - 'issue={issue}&' - 'new_clstore=true&' - 'query=source_type%3D{corpus}') - cl_triage_link = cl_triage_link.format( - instance=self._instance, - issue=self._gold_properties.issue, - corpus=self._corpus) - self._comparison_results[name].triage_link = cl_triage_link - else: - try: - with open(self._triage_link_file) as tlf: - triage_link = tlf.read().strip() - self._comparison_results[name].triage_link = triage_link - except IOError: - self._comparison_results[name].triage_link_omission_reason = ( - 'Failed to read triage link from file') - return rc, stdout - - def Diff(self, name, png_file, output_manager): - """Performs a local image diff against the closest known positive in Gold. - - This is used for running tests on a workstation, where uploading data to - Gold for ingestion is not allowed, and thus the web UI is not available. - - Image links can later be retrieved using Get*ImageLink(). - - Args: - name: The name of the image being compared. - png_file: The path to a PNG file containing the image to be diffed. - output_manager: The output manager used to save local diff images. - - Returns: - A tuple (return_code, output). |return_code| is the return code of the - diff process. |output| is the stdout + stderr of the diff process. - """ - # Instead of returning that everything is okay and putting in dummy links, - # just fail since this should only be called when running locally and - # --bypass-skia-gold-functionality is only meant for use on the bots. - if self._gold_properties.bypass_skia_gold_functionality: - raise RuntimeError( - '--bypass-skia-gold-functionality is not supported when running ' - 'tests locally.') - - # Output managers only support archived files, not directories, so we have - # to use a temporary directory and later move the data into the archived - # files. - output_dir = tempfile.mkdtemp(dir=self._working_dir) - diff_cmd = [ - GOLDCTL_BINARY, - 'diff', - '--corpus', - self._corpus, - '--instance', - self._instance, - '--input', - png_file, - '--test', - name, - '--work-dir', - self._working_dir, - '--out-dir', - output_dir, - ] - rc, stdout, _ = cmd_helper.GetCmdStatusOutputAndError( - diff_cmd, merge_stderr=True) given_path = closest_path = diff_path = None # The directory should contain "input-<hash>.png", "closest-<hash>.png", # and "diff.png". @@ -338,272 +37,46 @@ class SkiaGoldSession(object): closest_path = filepath elif f == 'diff.png': diff_path = filepath - results = self._comparison_results.setdefault(name, + results = self._comparison_results.setdefault(image_name, self.ComparisonResults()) if given_path: - with output_manager.ArchivedTempfile('given_%s.png' % name, + with output_manager.ArchivedTempfile('given_%s.png' % image_name, 'gold_local_diffs', Datatype.PNG) as given_file: shutil.move(given_path, given_file.name) results.local_diff_given_image = given_file.Link() if closest_path: - with output_manager.ArchivedTempfile('closest_%s.png' % name, + with output_manager.ArchivedTempfile('closest_%s.png' % image_name, 'gold_local_diffs', Datatype.PNG) as closest_file: shutil.move(closest_path, closest_file.name) results.local_diff_closest_image = closest_file.Link() if diff_path: - with output_manager.ArchivedTempfile( - 'diff_%s.png' % name, 'gold_local_diffs', Datatype.PNG) as diff_file: + with output_manager.ArchivedTempfile('diff_%s.png' % image_name, + 'gold_local_diffs', + Datatype.PNG) as diff_file: shutil.move(diff_path, diff_file.name) results.local_diff_diff_image = diff_file.Link() - return rc, stdout - - def GetTriageLink(self, name): - """Gets the triage link for the given image. - Args: - name: The name of the image to retrieve the triage link for. - - Returns: - A string containing the triage link if it is available, or None if it is - not available for some reason. The reason can be retrieved using - GetTriageLinkOmissionReason. - """ - return self._comparison_results.get(name, - self.ComparisonResults()).triage_link - - def GetTriageLinkOmissionReason(self, name): - """Gets the reason why a triage link is not available for an image. - - Args: - name: The name of the image whose triage link does not exist. - - Returns: - A string containing the reason why a triage link is not available. - """ - if name not in self._comparison_results: - return 'No image comparison performed for %s' % name - results = self._comparison_results[name] - # This method should not be called if there is a valid triage link. - assert results.triage_link is None - if results.triage_link_omission_reason: - return results.triage_link_omission_reason - if results.local_diff_given_image: - return 'Gold only used to do a local image diff' - raise RuntimeError( - 'Somehow have a ComparisonResults instance for %s that should not ' - 'exist' % name) - - def GetGivenImageLink(self, name): - """Gets the link to the given image used for local diffing. - - Args: - name: The name of the image that was diffed. - - Returns: - A string containing the link to where the image is saved, or None if it - does not exist. Since local diffing should only be done when running - locally, this *should* be a file:// URL, but there is no guarantee of - that. - """ - assert name in self._comparison_results - return self._comparison_results[name].local_diff_given_image - - def GetClosestImageLink(self, name): - """Gets the link to the closest known image used for local diffing. - - Args: - name: The name of the image that was diffed. - - Returns: - A string containing the link to where the image is saved, or None if it - does not exist. Since local diffing should only be done when running - locally, this *should* be a file:// URL, but there is no guarantee of - that. - """ - assert name in self._comparison_results - return self._comparison_results[name].local_diff_closest_image - - def GetDiffImageLink(self, name): - """Gets the link to the diff between the given and closest images. - - Args: - name: The name of the image that was diffed. - - Returns: - A string containing the link to where the image is saved, or None if it - does not exist. Since local diffing should only be done when running - locally, this *should* be a file:// URL, but there is no guarantee of - that. - """ - assert name in self._comparison_results - return self._comparison_results[name].local_diff_diff_image - - -class SkiaGoldSessionManager(object): - def __init__(self, working_dir, gold_properties): - """Class to manage one or more SkiaGoldSessions. - - A separate session is required for each instance/corpus/keys_file - combination, so this class will lazily create them as necessary. - - Args: - working_dir: The working directory under which each individual - SkiaGoldSessions' working directory will be created. - gold_properties: A SkiaGoldProperties instance that will be used to create - any SkiaGoldSessions. - """ - self._working_dir = working_dir - self._gold_properties = gold_properties - self._sessions = {} - - def GetSkiaGoldSession(self, - keys_file, - corpus=None, - instance=DEFAULT_INSTANCE): - """Gets a SkiaGoldSession for the given arguments. - - Lazily creates one if necessary. - - Args: - keys_file: A path to a JSON file containing various comparison config - data such as corpus and debug information like the hardware/software - configuration the image was produced on. - corpus: The corpus the session is for. If None, the corpus will be - determined using available information. - instance: The name of the Skia Gold instance to interact with. - """ - with open(keys_file) as f: - keys = json.load(f) - keys_string = json.dumps(keys, sort_keys=True) - if corpus is None: - corpus = keys.get('source_type', instance) - # Use the string representation of the keys JSON as a proxy for a hash since - # dicts themselves are not hashable. - session = self._sessions.setdefault(instance, - {}).setdefault(corpus, {}).setdefault( - keys_string, None) - if not session: - working_dir = tempfile.mkdtemp(dir=self._working_dir) - session = SkiaGoldSession(working_dir, self._gold_properties, keys_file, - corpus, instance) - self._sessions[instance][corpus][keys_string] = session - return session - - -class SkiaGoldProperties(object): - def __init__(self, args): - """Class to validate and store properties related to Skia Gold. - - Args: - args: The parsed arguments from an argparse.ArgumentParser. - """ - self._git_revision = None - self._issue = None - self._patchset = None - self._job_id = None - self._local_pixel_tests = None - self._no_luci_auth = None - self._bypass_skia_gold_functionality = None - - # Could in theory be configurable, but hard-coded for now since there's - # no plan to support anything else. - self._code_review_system = 'gerrit' - self._continuous_integration_system = 'buildbucket' - - self._InitializeProperties(args) - - def IsTryjobRun(self): - return self.issue is not None - - @property - def continuous_integration_system(self): - return self._continuous_integration_system - - @property - def code_review_system(self): - return self._code_review_system - - @property - def git_revision(self): - return self._GetGitRevision() - - @property - def issue(self): - return self._issue - - @property - def job_id(self): - return self._job_id - - @property - def local_pixel_tests(self): - return self._IsLocalRun() - - @property - def no_luci_auth(self): - return self._no_luci_auth - - @property - def patchset(self): - return self._patchset - - @property - def bypass_skia_gold_functionality(self): - return self._bypass_skia_gold_functionality - - def _GetGitRevision(self): - if not self._git_revision: - # Automated tests should always pass the revision, so assume we're on - # a workstation and try to get the local origin/master HEAD. - if not self._IsLocalRun(): - raise RuntimeError( - '--git-revision was not passed when running on a bot') - revision = repo_utils.GetGitOriginMasterHeadSHA1( - host_paths.DIR_SOURCE_ROOT) - if not revision or len(revision) != 40: - raise RuntimeError( - '--git-revision not passed and unable to determine from git') - self._git_revision = revision - return self._git_revision - - def _IsLocalRun(self): - if self._local_pixel_tests is None: - self._local_pixel_tests = not local_utils.IsOnSwarming() - if self._local_pixel_tests: - logging.warning( - 'Automatically determined that test is running on a workstation') - else: - logging.warning( - 'Automatically determined that test is running on a bot') - return self._local_pixel_tests + @staticmethod + def _RunCmdForRcAndOutput(cmd): + rc, stdout, _ = cmd_helper.GetCmdStatusOutputAndError(cmd, + merge_stderr=True) + return rc, stdout - def _InitializeProperties(self, args): - if hasattr(args, 'local_pixel_tests'): - # If not set, will be automatically determined later if needed. - self._local_pixel_tests = args.local_pixel_tests - if hasattr(args, 'no_luci_auth'): - self._no_luci_auth = args.no_luci_auth +class AndroidSkiaGoldSessionManager( + skia_gold_session_manager.SkiaGoldSessionManager): + @staticmethod + def _GetDefaultInstance(): + return 'chrome' - if hasattr(args, 'bypass_skia_gold_functionality'): - self._bypass_skia_gold_functionality = args.bypass_skia_gold_functionality + @staticmethod + def _GetSessionClass(): + return AndroidSkiaGoldSession - # Will be automatically determined later if needed. - if not hasattr(args, 'git_revision') or not args.git_revision: - return - self._git_revision = args.git_revision - # Only expected on tryjob runs. - if not hasattr(args, 'gerrit_issue') or not args.gerrit_issue: - return - self._issue = args.gerrit_issue - if not hasattr(args, 'gerrit_patchset') or not args.gerrit_patchset: - raise RuntimeError( - '--gerrit-issue passed, but --gerrit-patchset not passed.') - self._patchset = args.gerrit_patchset - if not hasattr(args, 'buildbucket_id') or not args.buildbucket_id: - raise RuntimeError( - '--gerrit-issue passed, but --buildbucket-id not passed.') - self._job_id = args.buildbucket_id +class AndroidSkiaGoldProperties(skia_gold_properties.SkiaGoldProperties): + @staticmethod + def _GetGitOriginMasterHeadSha1(): + return repo_utils.GetGitOriginMasterHeadSHA1(host_paths.DIR_SOURCE_ROOT) diff --git a/chromium/build/android/pylib/utils/gold_utils_test.py b/chromium/build/android/pylib/utils/gold_utils_test.py index ae3f7ecb3c3..3499484e612 100755 --- a/chromium/build/android/pylib/utils/gold_utils_test.py +++ b/chromium/build/android/pylib/utils/gold_utils_test.py @@ -6,39 +6,21 @@ #pylint: disable=protected-access -import collections -import json +import contextlib import os +import tempfile import unittest from pylib.constants import host_paths from pylib.utils import gold_utils -from py_utils import tempfile_ext -with host_paths.SysPath(host_paths.PYMOCK_PATH): - import mock # pylint: disable=import-error +with host_paths.SysPath(host_paths.BUILD_PATH): + from skia_gold_common import unittest_utils -_SkiaGoldArgs = collections.namedtuple('_SkiaGoldArgs', [ - 'local_pixel_tests', - 'no_luci_auth', - 'git_revision', - 'gerrit_issue', - 'gerrit_patchset', - 'buildbucket_id', - 'bypass_skia_gold_functionality', -]) +import mock # pylint: disable=import-error +from pyfakefs import fake_filesystem_unittest # pylint: disable=import-error - -def createSkiaGoldArgs(local_pixel_tests=None, - no_luci_auth=None, - git_revision=None, - gerrit_issue=None, - gerrit_patchset=None, - buildbucket_id=None, - bypass_skia_gold_functionality=None): - return _SkiaGoldArgs(local_pixel_tests, no_luci_auth, git_revision, - gerrit_issue, gerrit_patchset, buildbucket_id, - bypass_skia_gold_functionality) +createSkiaGoldArgs = unittest_utils.createSkiaGoldArgs def assertArgWith(test, arg_list, arg, value): @@ -46,852 +28,85 @@ def assertArgWith(test, arg_list, arg, value): test.assertEqual(arg_list[i + 1], value) -class SkiaGoldSessionRunComparisonTest(unittest.TestCase): - """Tests the functionality of SkiaGoldSession.RunComparison.""" - - @mock.patch.object(gold_utils.SkiaGoldSession, 'Diff') - @mock.patch.object(gold_utils.SkiaGoldSession, 'Compare') - @mock.patch.object(gold_utils.SkiaGoldSession, 'Initialize') - @mock.patch.object(gold_utils.SkiaGoldSession, 'Authenticate') - def test_comparisonSuccess(self, auth_mock, init_mock, compare_mock, - diff_mock): - auth_mock.return_value = (0, None) - init_mock.return_value = (0, None) - compare_mock.return_value = (0, None) - with tempfile_ext.NamedTemporaryDirectory() as working_dir: - keys_file = os.path.join(working_dir, 'keys.json') - with open(os.path.join(working_dir, 'keys.json'), 'w') as f: - json.dump({}, f) - session = gold_utils.SkiaGoldSession(working_dir, None, keys_file, None) - status, _ = session.RunComparison(None, None, None) - self.assertEqual(status, gold_utils.SkiaGoldSession.StatusCodes.SUCCESS) - self.assertEqual(auth_mock.call_count, 1) - self.assertEqual(init_mock.call_count, 1) - self.assertEqual(compare_mock.call_count, 1) - self.assertEqual(diff_mock.call_count, 0) - - @mock.patch.object(gold_utils.SkiaGoldSession, 'Diff') - @mock.patch.object(gold_utils.SkiaGoldSession, 'Compare') - @mock.patch.object(gold_utils.SkiaGoldSession, 'Initialize') - @mock.patch.object(gold_utils.SkiaGoldSession, 'Authenticate') - def test_authFailure(self, auth_mock, init_mock, compare_mock, diff_mock): - auth_mock.return_value = (1, 'Auth failed') - with tempfile_ext.NamedTemporaryDirectory() as working_dir: - session = gold_utils.SkiaGoldSession(working_dir, None, None, None) - status, error = session.RunComparison(None, None, None) - self.assertEqual(status, - gold_utils.SkiaGoldSession.StatusCodes.AUTH_FAILURE) - self.assertEqual(error, 'Auth failed') - self.assertEqual(auth_mock.call_count, 1) - self.assertEqual(init_mock.call_count, 0) - self.assertEqual(compare_mock.call_count, 0) - self.assertEqual(diff_mock.call_count, 0) - - @mock.patch.object(gold_utils.SkiaGoldSession, 'Diff') - @mock.patch.object(gold_utils.SkiaGoldSession, 'Compare') - @mock.patch.object(gold_utils.SkiaGoldSession, 'Initialize') - @mock.patch.object(gold_utils.SkiaGoldSession, 'Authenticate') - def test_initFailure(self, auth_mock, init_mock, compare_mock, diff_mock): - auth_mock.return_value = (0, None) - init_mock.return_value = (1, 'Init failed') - with tempfile_ext.NamedTemporaryDirectory() as working_dir: - session = gold_utils.SkiaGoldSession(working_dir, None, None, None) - status, error = session.RunComparison(None, None, None) - self.assertEqual(status, - gold_utils.SkiaGoldSession.StatusCodes.INIT_FAILURE) - self.assertEqual(error, 'Init failed') - self.assertEqual(auth_mock.call_count, 1) - self.assertEqual(init_mock.call_count, 1) - self.assertEqual(compare_mock.call_count, 0) - self.assertEqual(diff_mock.call_count, 0) - - @mock.patch.object(gold_utils.SkiaGoldSession, 'Diff') - @mock.patch.object(gold_utils.SkiaGoldSession, 'Compare') - @mock.patch.object(gold_utils.SkiaGoldSession, 'Initialize') - @mock.patch.object(gold_utils.SkiaGoldSession, 'Authenticate') - def test_compareFailureRemote(self, auth_mock, init_mock, compare_mock, - diff_mock): - auth_mock.return_value = (0, None) - init_mock.return_value = (0, None) - compare_mock.return_value = (1, 'Compare failed') - args = createSkiaGoldArgs(local_pixel_tests=False) - sgp = gold_utils.SkiaGoldProperties(args) - with tempfile_ext.NamedTemporaryDirectory() as working_dir: - keys_file = os.path.join(working_dir, 'keys.json') - with open(os.path.join(working_dir, 'keys.json'), 'w') as f: - json.dump({}, f) - session = gold_utils.SkiaGoldSession(working_dir, sgp, keys_file, None) - status, error = session.RunComparison(None, None, None) - self.assertEqual( - status, - gold_utils.SkiaGoldSession.StatusCodes.COMPARISON_FAILURE_REMOTE) - self.assertEqual(error, 'Compare failed') - self.assertEqual(auth_mock.call_count, 1) - self.assertEqual(init_mock.call_count, 1) - self.assertEqual(compare_mock.call_count, 1) - self.assertEqual(diff_mock.call_count, 0) - - @mock.patch.object(gold_utils.SkiaGoldSession, 'Diff') - @mock.patch.object(gold_utils.SkiaGoldSession, 'Compare') - @mock.patch.object(gold_utils.SkiaGoldSession, 'Initialize') - @mock.patch.object(gold_utils.SkiaGoldSession, 'Authenticate') - def test_compareFailureLocal(self, auth_mock, init_mock, compare_mock, - diff_mock): - auth_mock.return_value = (0, None) - init_mock.return_value = (0, None) - compare_mock.return_value = (1, 'Compare failed') - diff_mock.return_value = (0, None) - args = createSkiaGoldArgs(local_pixel_tests=True) - sgp = gold_utils.SkiaGoldProperties(args) - with tempfile_ext.NamedTemporaryDirectory() as working_dir: - keys_file = os.path.join(working_dir, 'keys.json') - with open(os.path.join(working_dir, 'keys.json'), 'w') as f: - json.dump({}, f) - session = gold_utils.SkiaGoldSession(working_dir, sgp, keys_file, None) - status, error = session.RunComparison(None, None, - 'Definitely an output manager') - self.assertEqual( - status, - gold_utils.SkiaGoldSession.StatusCodes.COMPARISON_FAILURE_LOCAL) - self.assertEqual(error, 'Compare failed') - self.assertEqual(auth_mock.call_count, 1) - self.assertEqual(init_mock.call_count, 1) - self.assertEqual(compare_mock.call_count, 1) - self.assertEqual(diff_mock.call_count, 1) - - @mock.patch.object(gold_utils.SkiaGoldSession, 'Diff') - @mock.patch.object(gold_utils.SkiaGoldSession, 'Compare') - @mock.patch.object(gold_utils.SkiaGoldSession, 'Initialize') - @mock.patch.object(gold_utils.SkiaGoldSession, 'Authenticate') - def test_diffFailure(self, auth_mock, init_mock, compare_mock, diff_mock): - auth_mock.return_value = (0, None) - init_mock.return_value = (0, None) - compare_mock.return_value = (1, 'Compare failed') - diff_mock.return_value = (1, 'Diff failed') - args = createSkiaGoldArgs(local_pixel_tests=True) - sgp = gold_utils.SkiaGoldProperties(args) - with tempfile_ext.NamedTemporaryDirectory() as working_dir: - keys_file = os.path.join(working_dir, 'keys.json') - with open(os.path.join(working_dir, 'keys.json'), 'w') as f: - json.dump({}, f) - session = gold_utils.SkiaGoldSession(working_dir, sgp, keys_file, None) - status, error = session.RunComparison(None, None, - 'Definitely an output manager') - self.assertEqual( - status, gold_utils.SkiaGoldSession.StatusCodes.LOCAL_DIFF_FAILURE) - self.assertEqual(error, 'Diff failed') - self.assertEqual(auth_mock.call_count, 1) - self.assertEqual(init_mock.call_count, 1) - self.assertEqual(compare_mock.call_count, 1) - self.assertEqual(diff_mock.call_count, 1) - - @mock.patch.object(gold_utils.SkiaGoldSession, 'Diff') - @mock.patch.object(gold_utils.SkiaGoldSession, 'Compare') - @mock.patch.object(gold_utils.SkiaGoldSession, 'Initialize') - @mock.patch.object(gold_utils.SkiaGoldSession, 'Authenticate') - def test_noOutputDirLocal(self, auth_mock, init_mock, compare_mock, - diff_mock): - auth_mock.return_value = (0, None) - init_mock.return_value = (0, None) - compare_mock.return_value = (1, 'Compare failed') - diff_mock.return_value = (0, None) - args = createSkiaGoldArgs(local_pixel_tests=True) - sgp = gold_utils.SkiaGoldProperties(args) - with tempfile_ext.NamedTemporaryDirectory() as working_dir: - keys_file = os.path.join(working_dir, 'keys.json') - with open(os.path.join(working_dir, 'keys.json'), 'w') as f: - json.dump({}, f) - session = gold_utils.SkiaGoldSession(working_dir, sgp, keys_file, None) - status, error = session.RunComparison(None, None, None) - self.assertEqual(status, - gold_utils.SkiaGoldSession.StatusCodes.NO_OUTPUT_MANAGER) - self.assertEqual(error, 'No output manager for local diff images') - self.assertEqual(auth_mock.call_count, 1) - self.assertEqual(compare_mock.call_count, 1) - self.assertEqual(diff_mock.call_count, 0) - - -class SkiaGoldSessionAuthenticateTest(unittest.TestCase): - """Tests the functionality of SkiaGoldSession.Authenticate.""" - - @mock.patch('devil.utils.cmd_helper.GetCmdStatusOutputAndError') - def test_commandOutputReturned(self, cmd_mock): - cmd_mock.return_value = (1, 'Something bad :(', None) - args = createSkiaGoldArgs(git_revision='a') - sgp = gold_utils.SkiaGoldProperties(args) - with tempfile_ext.NamedTemporaryDirectory() as working_dir: - session = gold_utils.SkiaGoldSession(working_dir, sgp, None, None) - rc, stdout = session.Authenticate() - self.assertEqual(cmd_mock.call_count, 1) - self.assertEqual(rc, 1) - self.assertEqual(stdout, 'Something bad :(') - - @mock.patch('devil.utils.cmd_helper.GetCmdStatusOutputAndError') - def test_bypassSkiaGoldFunctionality(self, cmd_mock): - cmd_mock.return_value = (None, None, None) - args = createSkiaGoldArgs( - git_revision='a', bypass_skia_gold_functionality=True) - sgp = gold_utils.SkiaGoldProperties(args) - with tempfile_ext.NamedTemporaryDirectory() as working_dir: - session = gold_utils.SkiaGoldSession(working_dir, sgp, None, None) - rc, _ = session.Authenticate() - self.assertEqual(rc, 0) - cmd_mock.assert_not_called() +class AndroidSkiaGoldSessionDiffTest(fake_filesystem_unittest.TestCase): + def setUp(self): + self.setUpPyfakefs() + self._working_dir = tempfile.mkdtemp() - @mock.patch('devil.utils.cmd_helper.GetCmdStatusOutputAndError') - def test_shortCircuitAlreadyAuthenticated(self, cmd_mock): - cmd_mock.return_value = (None, None, None) - args = createSkiaGoldArgs(git_revision='a') - sgp = gold_utils.SkiaGoldProperties(args) - with tempfile_ext.NamedTemporaryDirectory() as working_dir: - session = gold_utils.SkiaGoldSession(working_dir, sgp, None, None) - session._authenticated = True - rc, _ = session.Authenticate() - self.assertEqual(rc, 0) - cmd_mock.assert_not_called() - - @mock.patch('devil.utils.cmd_helper.GetCmdStatusOutputAndError') - def test_successSetsShortCircuit(self, cmd_mock): - cmd_mock.return_value = (0, None, None) - args = createSkiaGoldArgs(git_revision='a') - sgp = gold_utils.SkiaGoldProperties(args) - with tempfile_ext.NamedTemporaryDirectory() as working_dir: - session = gold_utils.SkiaGoldSession(working_dir, sgp, None, None) - self.assertFalse(session._authenticated) - rc, _ = session.Authenticate() - self.assertEqual(rc, 0) - self.assertTrue(session._authenticated) - cmd_mock.assert_not_called() - - @mock.patch('devil.utils.cmd_helper.GetCmdStatusOutputAndError') - def test_failureDoesNotSetShortCircuit(self, cmd_mock): - cmd_mock.return_value = (1, None, None) - args = createSkiaGoldArgs(git_revision='a') - sgp = gold_utils.SkiaGoldProperties(args) - with tempfile_ext.NamedTemporaryDirectory() as working_dir: - session = gold_utils.SkiaGoldSession(working_dir, sgp, None, None) - self.assertFalse(session._authenticated) - rc, _ = session.Authenticate() - self.assertEqual(rc, 1) - self.assertFalse(session._authenticated) - cmd_mock.assert_not_called() - - @mock.patch('devil.utils.cmd_helper.GetCmdStatusOutputAndError') - def test_commandWithUseLuciTrue(self, cmd_mock): - cmd_mock.return_value = (None, None, None) - args = createSkiaGoldArgs(git_revision='a') - sgp = gold_utils.SkiaGoldProperties(args) - with tempfile_ext.NamedTemporaryDirectory() as working_dir: - session = gold_utils.SkiaGoldSession(working_dir, sgp, None, None) - session.Authenticate(use_luci=True) - self.assertIn('--luci', cmd_mock.call_args[0][0]) - - @mock.patch('devil.utils.cmd_helper.GetCmdStatusOutputAndError') - def test_commandWithUseLuciFalse(self, cmd_mock): - cmd_mock.return_value = (None, None, None) - args = createSkiaGoldArgs(git_revision='a', local_pixel_tests=True) - sgp = gold_utils.SkiaGoldProperties(args) - with tempfile_ext.NamedTemporaryDirectory() as working_dir: - session = gold_utils.SkiaGoldSession(working_dir, sgp, None, None) - session.Authenticate(use_luci=False) - self.assertNotIn('--luci', cmd_mock.call_args[0][0]) - - @mock.patch('devil.utils.cmd_helper.GetCmdStatusOutputAndError') - def test_commandWithUseLuciFalseNotLocal(self, cmd_mock): - cmd_mock.return_value = (None, None, None) - args = createSkiaGoldArgs(git_revision='a', local_pixel_tests=False) - sgp = gold_utils.SkiaGoldProperties(args) - with tempfile_ext.NamedTemporaryDirectory() as working_dir: - session = gold_utils.SkiaGoldSession(working_dir, sgp, None, None) - with self.assertRaises(RuntimeError): - session.Authenticate(use_luci=False) - - @mock.patch('devil.utils.cmd_helper.GetCmdStatusOutputAndError') + @mock.patch.object(gold_utils.AndroidSkiaGoldSession, '_RunCmdForRcAndOutput') def test_commandCommonArgs(self, cmd_mock): - cmd_mock.return_value = (None, None, None) - args = createSkiaGoldArgs(git_revision='a') - sgp = gold_utils.SkiaGoldProperties(args) - with tempfile_ext.NamedTemporaryDirectory() as working_dir: - session = gold_utils.SkiaGoldSession(working_dir, sgp, None, None) - session.Authenticate() - call_args = cmd_mock.call_args[0][0] - self.assertIn('auth', call_args) - assertArgWith(self, call_args, '--work-dir', working_dir) - - -class SkiaGoldSessionInitializeTest(unittest.TestCase): - """Tests the functionality of SkiaGoldSession.Initialize.""" - - @mock.patch('devil.utils.cmd_helper.GetCmdStatusOutputAndError') - def test_bypassSkiaGoldFunctionality(self, cmd_mock): - cmd_mock.return_value = (None, None, None) - args = createSkiaGoldArgs( - git_revision='a', bypass_skia_gold_functionality=True) - sgp = gold_utils.SkiaGoldProperties(args) - with tempfile_ext.NamedTemporaryDirectory() as working_dir: - session = gold_utils.SkiaGoldSession(working_dir, sgp, None, None) - rc, _ = session.Initialize() - self.assertEqual(rc, 0) - cmd_mock.assert_not_called() - - @mock.patch('devil.utils.cmd_helper.GetCmdStatusOutputAndError') - def test_shortCircuitAlreadyInitialized(self, cmd_mock): - cmd_mock.return_value = (None, None, None) - args = createSkiaGoldArgs(git_revision='a') - sgp = gold_utils.SkiaGoldProperties(args) - with tempfile_ext.NamedTemporaryDirectory() as working_dir: - session = gold_utils.SkiaGoldSession(working_dir, sgp, None, None) - session._initialized = True - rc, _ = session.Initialize() - self.assertEqual(rc, 0) - cmd_mock.assert_not_called() - - @mock.patch('devil.utils.cmd_helper.GetCmdStatusOutputAndError') - def test_successSetsShortCircuit(self, cmd_mock): - cmd_mock.return_value = (0, None, None) - args = createSkiaGoldArgs(git_revision='a') - sgp = gold_utils.SkiaGoldProperties(args) - with tempfile_ext.NamedTemporaryDirectory() as working_dir: - session = gold_utils.SkiaGoldSession(working_dir, sgp, None, None) - self.assertFalse(session._initialized) - rc, _ = session.Initialize() - self.assertEqual(rc, 0) - self.assertTrue(session._initialized) - cmd_mock.assert_not_called() - - @mock.patch('devil.utils.cmd_helper.GetCmdStatusOutputAndError') - def test_failureDoesNotSetShortCircuit(self, cmd_mock): - cmd_mock.return_value = (1, None, None) - args = createSkiaGoldArgs(git_revision='a') - sgp = gold_utils.SkiaGoldProperties(args) - with tempfile_ext.NamedTemporaryDirectory() as working_dir: - session = gold_utils.SkiaGoldSession(working_dir, sgp, None, None) - self.assertFalse(session._initialized) - rc, _ = session.Initialize() - self.assertEqual(rc, 1) - self.assertFalse(session._initialized) - cmd_mock.assert_not_called() - - @mock.patch('devil.utils.cmd_helper.GetCmdStatusOutputAndError') - def test_commandCommonArgs(self, cmd_mock): - cmd_mock.return_value = (None, None, None) - args = createSkiaGoldArgs(git_revision='a') - sgp = gold_utils.SkiaGoldProperties(args) - with tempfile_ext.NamedTemporaryDirectory() as working_dir: - session = gold_utils.SkiaGoldSession( - working_dir, sgp, 'keys_file', 'corpus', instance='instance') - session.Initialize() - call_args = cmd_mock.call_args[0][0] - self.assertIn('imgtest', call_args) - self.assertIn('init', call_args) - self.assertIn('--passfail', call_args) - assertArgWith(self, call_args, '--instance', 'instance') - assertArgWith(self, call_args, '--corpus', 'corpus') - assertArgWith(self, call_args, '--keys-file', 'keys_file') - assertArgWith(self, call_args, '--work-dir', working_dir) - assertArgWith(self, call_args, '--failure-file', session._triage_link_file) - assertArgWith(self, call_args, '--commit', 'a') - - @mock.patch('devil.utils.cmd_helper.GetCmdStatusOutputAndError') - def test_commandTryjobArgs(self, cmd_mock): - cmd_mock.return_value = (None, None, None) - args = createSkiaGoldArgs( - git_revision='a', gerrit_issue=1, gerrit_patchset=2, buildbucket_id=3) - sgp = gold_utils.SkiaGoldProperties(args) - with tempfile_ext.NamedTemporaryDirectory() as working_dir: - session = gold_utils.SkiaGoldSession(working_dir, sgp, None, None) - session.Initialize() - call_args = cmd_mock.call_args[0][0] - assertArgWith(self, call_args, '--issue', '1') - assertArgWith(self, call_args, '--patchset', '2') - assertArgWith(self, call_args, '--jobid', '3') - assertArgWith(self, call_args, '--crs', 'gerrit') - assertArgWith(self, call_args, '--cis', 'buildbucket') - - @mock.patch('devil.utils.cmd_helper.GetCmdStatusOutputAndError') - def test_commandTryjobArgsMissing(self, cmd_mock): - cmd_mock.return_value = (None, None, None) - args = createSkiaGoldArgs(git_revision='a') - sgp = gold_utils.SkiaGoldProperties(args) - with tempfile_ext.NamedTemporaryDirectory() as working_dir: - session = gold_utils.SkiaGoldSession(working_dir, sgp, None, None) - session.Initialize() - call_args = cmd_mock.call_args[0][0] - self.assertNotIn('--issue', call_args) - self.assertNotIn('--patchset', call_args) - self.assertNotIn('--jobid', call_args) - self.assertNotIn('--crs', call_args) - self.assertNotIn('--cis', call_args) - - -class SkiaGoldSessionCompareTest(unittest.TestCase): - """Tests the functionality of SkiaGoldSession.Compare.""" - - @mock.patch('devil.utils.cmd_helper.GetCmdStatusOutputAndError') - def test_commandOutputReturned(self, cmd_mock): - cmd_mock.return_value = (1, 'Something bad :(', None) - args = createSkiaGoldArgs(git_revision='a') - sgp = gold_utils.SkiaGoldProperties(args) - with tempfile_ext.NamedTemporaryDirectory() as working_dir: - session = gold_utils.SkiaGoldSession(working_dir, sgp, None, None) - rc, stdout = session.Compare(None, None) - self.assertEqual(cmd_mock.call_count, 1) - self.assertEqual(rc, 1) - self.assertEqual(stdout, 'Something bad :(') - - @mock.patch('devil.utils.cmd_helper.GetCmdStatusOutputAndError') - def test_bypassSkiaGoldFunctionality(self, cmd_mock): - cmd_mock.return_value = (None, None, None) - args = createSkiaGoldArgs( - git_revision='a', bypass_skia_gold_functionality=True) - sgp = gold_utils.SkiaGoldProperties(args) - with tempfile_ext.NamedTemporaryDirectory() as working_dir: - session = gold_utils.SkiaGoldSession(working_dir, sgp, None, None) - rc, _ = session.Compare(None, None) - self.assertEqual(rc, 0) - cmd_mock.assert_not_called() - - @mock.patch('devil.utils.cmd_helper.GetCmdStatusOutputAndError') - def test_commandWithLocalPixelTestsTrue(self, cmd_mock): - cmd_mock.return_value = (None, None, None) - args = createSkiaGoldArgs(git_revision='a', local_pixel_tests=True) - sgp = gold_utils.SkiaGoldProperties(args) - with tempfile_ext.NamedTemporaryDirectory() as working_dir: - session = gold_utils.SkiaGoldSession(working_dir, sgp, None, None) - session.Compare(None, None) - self.assertIn('--dryrun', cmd_mock.call_args[0][0]) - - @mock.patch('devil.utils.cmd_helper.GetCmdStatusOutputAndError') - def test_commandWithLocalPixelTestsFalse(self, cmd_mock): - cmd_mock.return_value = (None, None, None) + cmd_mock.return_value = (None, None) args = createSkiaGoldArgs(git_revision='a', local_pixel_tests=False) - sgp = gold_utils.SkiaGoldProperties(args) - with tempfile_ext.NamedTemporaryDirectory() as working_dir: - session = gold_utils.SkiaGoldSession(working_dir, sgp, None, None) - session.Compare(None, None) - self.assertNotIn('--dryrun', cmd_mock.call_args[0][0]) - - @mock.patch('devil.utils.cmd_helper.GetCmdStatusOutputAndError') - def test_commandCommonArgs(self, cmd_mock): - cmd_mock.return_value = (None, None, None) - args = createSkiaGoldArgs(git_revision='a') - sgp = gold_utils.SkiaGoldProperties(args) - with tempfile_ext.NamedTemporaryDirectory() as working_dir: - session = gold_utils.SkiaGoldSession( - working_dir, sgp, 'keys_file', 'corpus', instance='instance') - session.Compare('name', 'png_file') - call_args = cmd_mock.call_args[0][0] - self.assertIn('imgtest', call_args) - self.assertIn('add', call_args) - assertArgWith(self, call_args, '--test-name', 'name') - assertArgWith(self, call_args, '--png-file', 'png_file') - assertArgWith(self, call_args, '--work-dir', working_dir) - - @mock.patch('devil.utils.cmd_helper.GetCmdStatusOutputAndError') - def test_noLinkOnSuccess(self, cmd_mock): - cmd_mock.return_value = (0, None, None) - args = createSkiaGoldArgs(git_revision='a') - sgp = gold_utils.SkiaGoldProperties(args) - with tempfile_ext.NamedTemporaryDirectory() as working_dir: - session = gold_utils.SkiaGoldSession(working_dir, sgp, 'keys_file', None) - rc, _ = session.Compare('name', 'png_file') - self.assertEqual(rc, 0) - self.assertEqual(session._comparison_results['name'].triage_link, None) - self.assertNotEqual( - session._comparison_results['name'].triage_link_omission_reason, None) - - @mock.patch('devil.utils.cmd_helper.GetCmdStatusOutputAndError') - def test_clLinkOnTrybot(self, cmd_mock): - cmd_mock.return_value = (1, None, None) - args = createSkiaGoldArgs( - git_revision='a', gerrit_issue=1, gerrit_patchset=2, buildbucket_id=3) - sgp = gold_utils.SkiaGoldProperties(args) - with tempfile_ext.NamedTemporaryDirectory() as working_dir: - session = gold_utils.SkiaGoldSession(working_dir, sgp, 'keys_file', None) - rc, _ = session.Compare('name', 'png_file') - self.assertEqual(rc, 1) - self.assertNotEqual(session._comparison_results['name'].triage_link, None) - self.assertIn('issue=1', session._comparison_results['name'].triage_link) - self.assertEqual( - session._comparison_results['name'].triage_link_omission_reason, None) - - @mock.patch('devil.utils.cmd_helper.GetCmdStatusOutputAndError') - def test_individualLinkOnCi(self, cmd_mock): - cmd_mock.return_value = (1, None, None) - args = createSkiaGoldArgs(git_revision='a') - sgp = gold_utils.SkiaGoldProperties(args) - with tempfile_ext.NamedTemporaryDirectory() as working_dir: - session = gold_utils.SkiaGoldSession(working_dir, sgp, 'keys_file', None) - m = mock.mock_open(read_data='foobar') - with mock.patch('__builtin__.open', m, create=True): - rc, _ = session.Compare('name', 'png_file') - self.assertEqual(rc, 1) - self.assertNotEqual(session._comparison_results['name'].triage_link, None) - self.assertEqual(session._comparison_results['name'].triage_link, 'foobar') - self.assertEqual( - session._comparison_results['name'].triage_link_omission_reason, None) - - @mock.patch('devil.utils.cmd_helper.GetCmdStatusOutputAndError') - def test_validOmissionOnIoError(self, cmd_mock): - cmd_mock.return_value = (1, None, None) - args = createSkiaGoldArgs(git_revision='a') - sgp = gold_utils.SkiaGoldProperties(args) - with tempfile_ext.NamedTemporaryDirectory() as working_dir: - session = gold_utils.SkiaGoldSession(working_dir, sgp, 'keys_file', None) - m = mock.mock_open() - m.side_effect = IOError('No read today') - with mock.patch('__builtin__.open', m, create=True): - rc, _ = session.Compare('name', 'png_file') - self.assertEqual(rc, 1) - self.assertEqual(session._comparison_results['name'].triage_link, None) - self.assertNotEqual( - session._comparison_results['name'].triage_link_omission_reason, None) - self.assertIn( - 'Failed to read', - session._comparison_results['name'].triage_link_omission_reason) - - -class SkiaGoldSessionDiffTest(unittest.TestCase): - """Tests the functionality of SkiaGoldSession.Diff.""" - - @mock.patch('devil.utils.cmd_helper.GetCmdStatusOutputAndError') - def test_commandOutputReturned(self, cmd_mock): - cmd_mock.return_value = (1, 'Something bad :(', None) - args = createSkiaGoldArgs(git_revision='a', local_pixel_tests=False) - sgp = gold_utils.SkiaGoldProperties(args) - with tempfile_ext.NamedTemporaryDirectory() as working_dir: - session = gold_utils.SkiaGoldSession(working_dir, sgp, None, None) - rc, stdout = session.Diff(None, None, None) - self.assertEqual(cmd_mock.call_count, 1) - self.assertEqual(rc, 1) - self.assertEqual(stdout, 'Something bad :(') - - @mock.patch('devil.utils.cmd_helper.GetCmdStatusOutputAndError') - def test_bypassSkiaGoldFunctionality(self, cmd_mock): - cmd_mock.return_value = (None, None, None) - args = createSkiaGoldArgs( - git_revision='a', bypass_skia_gold_functionality=True) - sgp = gold_utils.SkiaGoldProperties(args) - with tempfile_ext.NamedTemporaryDirectory() as working_dir: - session = gold_utils.SkiaGoldSession(working_dir, sgp, None, None) - with self.assertRaises(RuntimeError): - session.Diff(None, None, None) - - @mock.patch('devil.utils.cmd_helper.GetCmdStatusOutputAndError') - def test_commandCommonArgs(self, cmd_mock): - cmd_mock.return_value = (None, None, None) - args = createSkiaGoldArgs(git_revision='a', local_pixel_tests=False) - sgp = gold_utils.SkiaGoldProperties(args) - with tempfile_ext.NamedTemporaryDirectory() as working_dir: - session = gold_utils.SkiaGoldSession( - working_dir, sgp, None, 'corpus', instance='instance') - session.Diff('name', 'png_file', None) + sgp = gold_utils.AndroidSkiaGoldProperties(args) + session = gold_utils.AndroidSkiaGoldSession(self._working_dir, + sgp, + None, + 'corpus', + instance='instance') + session.Diff('name', 'png_file', None) call_args = cmd_mock.call_args[0][0] self.assertIn('diff', call_args) assertArgWith(self, call_args, '--corpus', 'corpus') assertArgWith(self, call_args, '--instance', 'instance') assertArgWith(self, call_args, '--input', 'png_file') assertArgWith(self, call_args, '--test', 'name') - assertArgWith(self, call_args, '--work-dir', working_dir) + assertArgWith(self, call_args, '--work-dir', self._working_dir) i = call_args.index('--out-dir') # The output directory should be a subdirectory of the working directory. - self.assertIn(working_dir, call_args[i + 1]) - - -class SkiaGoldSessionTriageLinkOmissionTest(unittest.TestCase): - """Tests the functionality of SkiaGoldSession.GetTriageLinkOmissionReason.""" - - # Avoid having to bother with the working directory. - class FakeGoldSession(gold_utils.SkiaGoldSession): - def __init__(self): # pylint: disable=super-init-not-called - self._comparison_results = { - 'foo': gold_utils.SkiaGoldSession.ComparisonResults(), - } - - def test_noComparison(self): - session = self.FakeGoldSession() - session._comparison_results = {} - reason = session.GetTriageLinkOmissionReason('foo') - self.assertEqual(reason, 'No image comparison performed for foo') - - def test_validReason(self): - session = self.FakeGoldSession() - session._comparison_results['foo'].triage_link_omission_reason = 'bar' - reason = session.GetTriageLinkOmissionReason('foo') - self.assertEqual(reason, 'bar') - - def test_onlyLocal(self): - session = self.FakeGoldSession() - session._comparison_results['foo'].local_diff_given_image = 'bar' - reason = session.GetTriageLinkOmissionReason('foo') - self.assertEqual(reason, 'Gold only used to do a local image diff') - - def test_onlyWithoutTriageLink(self): - session = self.FakeGoldSession() - session._comparison_results['foo'].triage_link = 'bar' - with self.assertRaises(AssertionError): - session.GetTriageLinkOmissionReason('foo') - - def test_resultsShouldNotExist(self): - session = self.FakeGoldSession() - with self.assertRaises(RuntimeError): - session.GetTriageLinkOmissionReason('foo') - - -class SkiaGoldSessionManagerGetSessionTest(unittest.TestCase): - """Tests the functionality of SkiaGoldSessionManager.GetSkiaGoldSession.""" - - @mock.patch('gold_utils.SkiaGoldSession') - def test_ArgsForwardedToSession(self, _): - args = createSkiaGoldArgs() - sgp = gold_utils.SkiaGoldProperties(args) - with tempfile_ext.NamedTemporaryDirectory() as working_dir: - sgsm = gold_utils.SkiaGoldSessionManager(working_dir, sgp) - keys_file = os.path.join(working_dir, 'keys.json') - with open(keys_file, 'w') as f: - json.dump({}, f) - session = sgsm.GetSkiaGoldSession(keys_file, 'corpus', 'instance') - self.assertEqual(session._keys_file, keys_file) - self.assertEqual(session._corpus, 'corpus') - self.assertEqual(session._instance, 'instance') - # Make sure the session's working directory is a subdirectory of the - # manager's working directory. - self.assertEqual(os.path.dirname(session._working_dir), working_dir) - - @mock.patch('gold_utils.SkiaGoldSession') - def test_corpusFromJson(self, _): - args = createSkiaGoldArgs() - sgp = gold_utils.SkiaGoldProperties(args) - with tempfile_ext.NamedTemporaryDirectory() as working_dir: - sgsm = gold_utils.SkiaGoldSessionManager(working_dir, sgp) - keys_file = os.path.join(working_dir, 'keys.json') - with open(keys_file, 'w') as f: - json.dump({'source_type': 'foobar'}, f) - session = sgsm.GetSkiaGoldSession(keys_file, None, 'instance') - self.assertEqual(session._keys_file, keys_file) - self.assertEqual(session._corpus, 'foobar') - self.assertEqual(session._instance, 'instance') + self.assertIn(self._working_dir, call_args[i + 1]) - @mock.patch('gold_utils.SkiaGoldSession') - def test_corpusDefaultsToInstance(self, _): - args = createSkiaGoldArgs() - sgp = gold_utils.SkiaGoldProperties(args) - with tempfile_ext.NamedTemporaryDirectory() as working_dir: - sgsm = gold_utils.SkiaGoldSessionManager(working_dir, sgp) - keys_file = os.path.join(working_dir, 'keys.json') - with open(keys_file, 'w') as f: - json.dump({}, f) - session = sgsm.GetSkiaGoldSession(keys_file, None, 'instance') - self.assertEqual(session._keys_file, keys_file) - self.assertEqual(session._corpus, 'instance') - self.assertEqual(session._instance, 'instance') - @mock.patch.object(gold_utils.SkiaGoldSession, '__init__') - def test_matchingSessionReused(self, session_mock): - session_mock.return_value = None - args = createSkiaGoldArgs() - sgp = gold_utils.SkiaGoldProperties(args) - with tempfile_ext.NamedTemporaryDirectory() as working_dir: - sgsm = gold_utils.SkiaGoldSessionManager(working_dir, sgp) - keys_file = os.path.join(working_dir, 'keys.json') - with open(keys_file, 'w') as f: - json.dump({}, f) - session1 = sgsm.GetSkiaGoldSession(keys_file, 'corpus', 'instance') - session2 = sgsm.GetSkiaGoldSession(keys_file, 'corpus', 'instance') - self.assertEqual(session1, session2) - # For some reason, session_mock.assert_called_once() always passes, - # so check the call count directly. - self.assertEqual(session_mock.call_count, 1) +class AndroidSkiaGoldSessionDiffLinksTest(fake_filesystem_unittest.TestCase): + class FakeArchivedFile(object): + def __init__(self, path): + self.name = path - @mock.patch.object(gold_utils.SkiaGoldSession, '__init__') - def test_separateSessionsFromKeys(self, session_mock): - session_mock.return_value = None - args = createSkiaGoldArgs() - sgp = gold_utils.SkiaGoldProperties(args) - with tempfile_ext.NamedTemporaryDirectory() as working_dir: - sgsm = gold_utils.SkiaGoldSessionManager(working_dir, sgp) - keys_file1 = os.path.join(working_dir, 'keys1.json') - with open(keys_file1, 'w') as f: - json.dump({}, f) - keys_file2 = os.path.join(working_dir, 'keys2.json') - with open(keys_file2, 'w') as f: - json.dump({'something different': 1}, f) - session1 = sgsm.GetSkiaGoldSession(keys_file1, 'corpus', 'instance') - session2 = sgsm.GetSkiaGoldSession(keys_file2, 'corpus', 'instance') - self.assertNotEqual(session1, session2) - self.assertEqual(session_mock.call_count, 2) + def Link(self): + return 'file://' + self.name - @mock.patch.object(gold_utils.SkiaGoldSession, '__init__') - def test_separateSessionsFromCorpus(self, session_mock): - session_mock.return_value = None - args = createSkiaGoldArgs() - sgp = gold_utils.SkiaGoldProperties(args) - with tempfile_ext.NamedTemporaryDirectory() as working_dir: - sgsm = gold_utils.SkiaGoldSessionManager(working_dir, sgp) - keys_file = os.path.join(working_dir, 'keys.json') - with open(keys_file, 'w') as f: - json.dump({}, f) - session1 = sgsm.GetSkiaGoldSession(keys_file, 'corpus1', 'instance') - session2 = sgsm.GetSkiaGoldSession(keys_file, 'corpus2', 'instance') - self.assertNotEqual(session1, session2) - self.assertEqual(session_mock.call_count, 2) + class FakeOutputManager(object): + def __init__(self): + self.output_dir = tempfile.mkdtemp() - @mock.patch.object(gold_utils.SkiaGoldSession, '__init__') - def test_separateSessionsFromInstance(self, session_mock): - session_mock.return_value = None - args = createSkiaGoldArgs() - sgp = gold_utils.SkiaGoldProperties(args) - with tempfile_ext.NamedTemporaryDirectory() as working_dir: - sgsm = gold_utils.SkiaGoldSessionManager(working_dir, sgp) - keys_file = os.path.join(working_dir, 'keys.json') - with open(keys_file, 'w') as f: - json.dump({}, f) - session1 = sgsm.GetSkiaGoldSession(keys_file, 'corpus', 'instance1') - session2 = sgsm.GetSkiaGoldSession(keys_file, 'corpus', 'instance2') - self.assertNotEqual(session1, session2) - self.assertEqual(session_mock.call_count, 2) + @contextlib.contextmanager + def ArchivedTempfile(self, image_name, _, __): + filepath = os.path.join(self.output_dir, image_name) + yield AndroidSkiaGoldSessionDiffLinksTest.FakeArchivedFile(filepath) + def setUp(self): + self.setUpPyfakefs() + self._working_dir = tempfile.mkdtemp() -class SkiaGoldPropertiesInitializationTest(unittest.TestCase): - """Tests that SkiaGoldProperties initializes (or doesn't) when expected.""" - - def verifySkiaGoldProperties(self, instance, expected): - self.assertEqual(instance._local_pixel_tests, - expected.get('local_pixel_tests')) - self.assertEqual(instance._no_luci_auth, expected.get('no_luci_auth')) - self.assertEqual(instance._git_revision, expected.get('git_revision')) - self.assertEqual(instance._issue, expected.get('gerrit_issue')) - self.assertEqual(instance._patchset, expected.get('gerrit_patchset')) - self.assertEqual(instance._job_id, expected.get('buildbucket_id')) - self.assertEqual(instance._bypass_skia_gold_functionality, - expected.get('bypass_skia_gold_functionality')) - - def test_initializeSkiaGoldAttributes_unsetLocal(self): - args = createSkiaGoldArgs() - sgp = gold_utils.SkiaGoldProperties(args) - self.verifySkiaGoldProperties(sgp, {}) - - def test_initializeSkiaGoldAttributes_explicitLocal(self): - args = createSkiaGoldArgs(local_pixel_tests=True) - sgp = gold_utils.SkiaGoldProperties(args) - self.verifySkiaGoldProperties(sgp, {'local_pixel_tests': True}) - - def test_initializeSkiaGoldAttributes_explicitNonLocal(self): - args = createSkiaGoldArgs(local_pixel_tests=False) - sgp = gold_utils.SkiaGoldProperties(args) - self.verifySkiaGoldProperties(sgp, {'local_pixel_tests': False}) - - def test_initializeSkiaGoldAttributes_explicitNoLuciAuth(self): - args = createSkiaGoldArgs(no_luci_auth=True) - sgp = gold_utils.SkiaGoldProperties(args) - self.verifySkiaGoldProperties(sgp, {'no_luci_auth': True}) - - def test_initializeSkiaGoldAttributes_bypassExplicitTrue(self): - args = createSkiaGoldArgs(bypass_skia_gold_functionality=True) - sgp = gold_utils.SkiaGoldProperties(args) - self.verifySkiaGoldProperties(sgp, {'bypass_skia_gold_functionality': True}) - - def test_initializeSkiaGoldAttributes_explicitGitRevision(self): - args = createSkiaGoldArgs(git_revision='a') - sgp = gold_utils.SkiaGoldProperties(args) - self.verifySkiaGoldProperties(sgp, {'git_revision': 'a'}) - - def test_initializeSkiaGoldAttributes_tryjobArgsIgnoredWithoutRevision(self): - args = createSkiaGoldArgs( - gerrit_issue=1, gerrit_patchset=2, buildbucket_id=3) - sgp = gold_utils.SkiaGoldProperties(args) - self.verifySkiaGoldProperties(sgp, {}) - - def test_initializeSkiaGoldAttributes_tryjobArgs(self): - args = createSkiaGoldArgs( - git_revision='a', gerrit_issue=1, gerrit_patchset=2, buildbucket_id=3) - sgp = gold_utils.SkiaGoldProperties(args) - self.verifySkiaGoldProperties( - sgp, { - 'git_revision': 'a', - 'gerrit_issue': 1, - 'gerrit_patchset': 2, - 'buildbucket_id': 3 - }) - - def test_initializeSkiaGoldAttributes_tryjobMissingPatchset(self): - args = createSkiaGoldArgs( - git_revision='a', gerrit_issue=1, buildbucket_id=3) - with self.assertRaises(RuntimeError): - gold_utils.SkiaGoldProperties(args) - - def test_initializeSkiaGoldAttributes_tryjobMissingBuildbucket(self): - args = createSkiaGoldArgs( - git_revision='a', gerrit_issue=1, gerrit_patchset=2) - with self.assertRaises(RuntimeError): - gold_utils.SkiaGoldProperties(args) - - -class SkiaGoldPropertiesCalculationTest(unittest.TestCase): - """Tests that SkiaGoldProperties properly calculates certain properties.""" - - def testLocalPixelTests_determineTrue(self): - args = createSkiaGoldArgs() - sgp = gold_utils.SkiaGoldProperties(args) - with mock.patch.dict(os.environ, {}, clear=True): - self.assertTrue(sgp.local_pixel_tests) - - def testLocalPixelTests_determineFalse(self): - args = createSkiaGoldArgs() - sgp = gold_utils.SkiaGoldProperties(args) - with mock.patch.dict(os.environ, {'SWARMING_SERVER': ''}, clear=True): - self.assertFalse(sgp.local_pixel_tests) - - def testIsTryjobRun_noIssue(self): - args = createSkiaGoldArgs() - sgp = gold_utils.SkiaGoldProperties(args) - self.assertFalse(sgp.IsTryjobRun()) - - def testIsTryjobRun_issue(self): - args = createSkiaGoldArgs( - git_revision='a', gerrit_issue=1, gerrit_patchset=2, buildbucket_id=3) - sgp = gold_utils.SkiaGoldProperties(args) - self.assertTrue(sgp.IsTryjobRun()) - - def testGetGitRevision_revisionSet(self): - args = createSkiaGoldArgs(git_revision='a') - sgp = gold_utils.SkiaGoldProperties(args) - self.assertEqual(sgp.git_revision, 'a') - - def testGetGitRevision_findValidRevision(self): - args = createSkiaGoldArgs(local_pixel_tests=True) - sgp = gold_utils.SkiaGoldProperties(args) - with mock.patch( - 'pylib.utils.repo_utils.GetGitOriginMasterHeadSHA1') as patched_head: - expected = 'a' * 40 - patched_head.return_value = expected - self.assertEqual(sgp.git_revision, expected) - # Should be cached. - self.assertEqual(sgp._git_revision, expected) - - def testGetGitRevision_noExplicitOnBot(self): - args = createSkiaGoldArgs(local_pixel_tests=False) - sgp = gold_utils.SkiaGoldProperties(args) - with self.assertRaises(RuntimeError): - _ = sgp.git_revision - - def testGetGitRevision_findEmptyRevision(self): - args = createSkiaGoldArgs(local_pixel_tests=True) - sgp = gold_utils.SkiaGoldProperties(args) - with mock.patch( - 'pylib.utils.repo_utils.GetGitOriginMasterHeadSHA1') as patched_head: - patched_head.return_value = '' - with self.assertRaises(RuntimeError): - _ = sgp.git_revision - - def testGetGitRevision_findMalformedRevision(self): - args = createSkiaGoldArgs(local_pixel_tests=True) - sgp = gold_utils.SkiaGoldProperties(args) - with mock.patch( - 'pylib.utils.repo_utils.GetGitOriginMasterHeadSHA1') as patched_head: - patched_head.return_value = 'a' * 39 - with self.assertRaises(RuntimeError): - _ = sgp.git_revision + def test_outputManagerUsed(self): + args = createSkiaGoldArgs(git_revision='a', local_pixel_tests=True) + sgp = gold_utils.AndroidSkiaGoldProperties(args) + session = gold_utils.AndroidSkiaGoldSession(self._working_dir, sgp, None, + None, None) + with open(os.path.join(self._working_dir, 'input-inputhash.png'), 'w') as f: + f.write('input') + with open(os.path.join(self._working_dir, 'closest-closesthash.png'), + 'w') as f: + f.write('closest') + with open(os.path.join(self._working_dir, 'diff.png'), 'w') as f: + f.write('diff') + + output_manager = AndroidSkiaGoldSessionDiffLinksTest.FakeOutputManager() + session._StoreDiffLinks('foo', output_manager, self._working_dir) + + copied_input = os.path.join(output_manager.output_dir, 'given_foo.png') + copied_closest = os.path.join(output_manager.output_dir, 'closest_foo.png') + copied_diff = os.path.join(output_manager.output_dir, 'diff_foo.png') + with open(copied_input) as f: + self.assertEqual(f.read(), 'input') + with open(copied_closest) as f: + self.assertEqual(f.read(), 'closest') + with open(copied_diff) as f: + self.assertEqual(f.read(), 'diff') + + self.assertEqual(session.GetGivenImageLink('foo'), 'file://' + copied_input) + self.assertEqual(session.GetClosestImageLink('foo'), + 'file://' + copied_closest) + self.assertEqual(session.GetDiffImageLink('foo'), 'file://' + copied_diff) if __name__ == '__main__': |