diff options
author | Allan Sandfeld Jensen <allan.jensen@qt.io> | 2020-10-12 14:27:29 +0200 |
---|---|---|
committer | Allan Sandfeld Jensen <allan.jensen@qt.io> | 2020-10-13 09:35:20 +0000 |
commit | c30a6232df03e1efbd9f3b226777b07e087a1122 (patch) | |
tree | e992f45784689f373bcc38d1b79a239ebe17ee23 /chromium/build/skia_gold_common | |
parent | 7b5b123ac58f58ffde0f4f6e488bcd09aa4decd3 (diff) | |
download | qtwebengine-chromium-85-based.tar.gz |
BASELINE: Update Chromium to 85.0.4183.14085-based
Change-Id: Iaa42f4680837c57725b1344f108c0196741f6057
Reviewed-by: Allan Sandfeld Jensen <allan.jensen@qt.io>
Diffstat (limited to 'chromium/build/skia_gold_common')
-rw-r--r-- | chromium/build/skia_gold_common/.style.yapf | 6 | ||||
-rw-r--r-- | chromium/build/skia_gold_common/OWNERS | 1 | ||||
-rw-r--r-- | chromium/build/skia_gold_common/PRESUBMIT.py | 34 | ||||
-rw-r--r-- | chromium/build/skia_gold_common/README.md | 6 | ||||
-rw-r--r-- | chromium/build/skia_gold_common/__init__.py | 3 | ||||
-rw-r--r-- | chromium/build/skia_gold_common/skia_gold_properties.py | 139 | ||||
-rwxr-xr-x | chromium/build/skia_gold_common/skia_gold_properties_unittest.py | 169 | ||||
-rw-r--r-- | chromium/build/skia_gold_common/skia_gold_session.py | 432 | ||||
-rw-r--r-- | chromium/build/skia_gold_common/skia_gold_session_manager.py | 115 | ||||
-rwxr-xr-x | chromium/build/skia_gold_common/skia_gold_session_manager_unittest.py | 176 | ||||
-rwxr-xr-x | chromium/build/skia_gold_common/skia_gold_session_unittest.py | 649 | ||||
-rw-r--r-- | chromium/build/skia_gold_common/unittest_utils.py | 28 |
12 files changed, 1758 insertions, 0 deletions
diff --git a/chromium/build/skia_gold_common/.style.yapf b/chromium/build/skia_gold_common/.style.yapf new file mode 100644 index 00000000000..239e0a247f3 --- /dev/null +++ b/chromium/build/skia_gold_common/.style.yapf @@ -0,0 +1,6 @@ +[style] +based_on_style = pep8 + +column_limit = 80 +indent_width = 2 + diff --git a/chromium/build/skia_gold_common/OWNERS b/chromium/build/skia_gold_common/OWNERS new file mode 100644 index 00000000000..71de6595104 --- /dev/null +++ b/chromium/build/skia_gold_common/OWNERS @@ -0,0 +1 @@ +bsheedy@google.com diff --git a/chromium/build/skia_gold_common/PRESUBMIT.py b/chromium/build/skia_gold_common/PRESUBMIT.py new file mode 100644 index 00000000000..41e1bb2f7de --- /dev/null +++ b/chromium/build/skia_gold_common/PRESUBMIT.py @@ -0,0 +1,34 @@ +# Copyright 2020 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. +"""Presubmit script for //build/skia_gold_common/. + +See http://dev.chromium.org/developers/how-tos/depottools/presubmit-scripts +for more details on the presubmit API built into depot_tools. +""" + + +def CommonChecks(input_api, output_api): + output = [] + build_path = input_api.os_path.join(input_api.PresubmitLocalPath(), '..') + skia_gold_env = dict(input_api.environ) + skia_gold_env.update({ + 'PYTHONPATH': build_path, + 'PYTHONDONTWRITEBYTECODE': '1', + }) + output.extend( + input_api.canned_checks.RunUnitTestsInDirectory( + input_api, + output_api, + input_api.PresubmitLocalPath(), [r'^.+_unittest\.py$'], + env=skia_gold_env)) + output.extend(input_api.canned_checks.RunPylint(input_api, output_api)) + return output + + +def CheckChangeOnUpload(input_api, output_api): + return CommonChecks(input_api, output_api) + + +def CheckChangeOnCommit(input_api, output_api): + return CommonChecks(input_api, output_api) diff --git a/chromium/build/skia_gold_common/README.md b/chromium/build/skia_gold_common/README.md new file mode 100644 index 00000000000..ec721117480 --- /dev/null +++ b/chromium/build/skia_gold_common/README.md @@ -0,0 +1,6 @@ +This directory contains Python code used for interacting with the Skia Gold +image diff service. It is used by multiple test harnesses, e.g. +`//build/android/test_runner.py` and +`//content/test/gpu/run_gpu_integration_test.py`. A place such as +`//testing/` would likely be a better location, but causes issues with +V8 since it imports `//build/` but not all of Chromium src. diff --git a/chromium/build/skia_gold_common/__init__.py b/chromium/build/skia_gold_common/__init__.py new file mode 100644 index 00000000000..ae1922e1cc4 --- /dev/null +++ b/chromium/build/skia_gold_common/__init__.py @@ -0,0 +1,3 @@ +# Copyright 2020 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. diff --git a/chromium/build/skia_gold_common/skia_gold_properties.py b/chromium/build/skia_gold_common/skia_gold_properties.py new file mode 100644 index 00000000000..6ee38eedb55 --- /dev/null +++ b/chromium/build/skia_gold_common/skia_gold_properties.py @@ -0,0 +1,139 @@ +# Copyright 2020 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. +"""Class for storing Skia Gold comparison properties. + +Examples: +* git revision being tested +* Whether the test is being run locally or on a bot +* What the continuous integration system is +""" + +import logging +import os +import subprocess +import sys + + +class SkiaGoldProperties(object): + def __init__(self, args): + """Abstract class to validate and store properties related to Skia Gold. + + Args: + args: The parsed arguments from an argparse.ArgumentParser. + """ + self._git_revision = None + self._issue = None + self._patchset = None + self._job_id = None + self._local_pixel_tests = None + self._no_luci_auth = None + self._bypass_skia_gold_functionality = None + + # Could in theory be configurable, but hard-coded for now since there's + # no plan to support anything else. + self._code_review_system = 'gerrit' + self._continuous_integration_system = 'buildbucket' + + self._InitializeProperties(args) + + def IsTryjobRun(self): + return self.issue is not None + + @property + def continuous_integration_system(self): + return self._continuous_integration_system + + @property + def code_review_system(self): + return self._code_review_system + + @property + def git_revision(self): + return self._GetGitRevision() + + @property + def issue(self): + return self._issue + + @property + def job_id(self): + return self._job_id + + @property + def local_pixel_tests(self): + return self._IsLocalRun() + + @property + def no_luci_auth(self): + return self._no_luci_auth + + @property + def patchset(self): + return self._patchset + + @property + def bypass_skia_gold_functionality(self): + return self._bypass_skia_gold_functionality + + @staticmethod + def _GetGitOriginMasterHeadSha1(): + raise NotImplementedError() + + def _GetGitRevision(self): + if not self._git_revision: + # Automated tests should always pass the revision, so assume we're on + # a workstation and try to get the local origin/master HEAD. + if not self._IsLocalRun(): + raise RuntimeError( + '--git-revision was not passed when running on a bot') + revision = self._GetGitOriginMasterHeadSha1() + if not revision or len(revision) != 40: + raise RuntimeError( + '--git-revision not passed and unable to determine from git') + self._git_revision = revision + return self._git_revision + + def _IsLocalRun(self): + if self._local_pixel_tests is None: + # Look for the presence of the SWARMING_SERVER environment variable as a + # heuristic to determine whether we're running on a workstation or a bot. + # This should always be set on swarming, but would be strange to be set on + # a workstation. + self._local_pixel_tests = 'SWARMING_SERVER' not in os.environ + if self._local_pixel_tests: + logging.warning( + 'Automatically determined that test is running on a workstation') + else: + logging.warning( + 'Automatically determined that test is running on a bot') + return self._local_pixel_tests + + def _InitializeProperties(self, args): + if hasattr(args, 'local_pixel_tests'): + # If not set, will be automatically determined later if needed. + self._local_pixel_tests = args.local_pixel_tests + + if hasattr(args, 'no_luci_auth'): + self._no_luci_auth = args.no_luci_auth + + if hasattr(args, 'bypass_skia_gold_functionality'): + self._bypass_skia_gold_functionality = args.bypass_skia_gold_functionality + + # Will be automatically determined later if needed. + if not hasattr(args, 'git_revision') or not args.git_revision: + return + self._git_revision = args.git_revision + + # Only expected on tryjob runs. + if not hasattr(args, 'gerrit_issue') or not args.gerrit_issue: + return + self._issue = args.gerrit_issue + if not hasattr(args, 'gerrit_patchset') or not args.gerrit_patchset: + raise RuntimeError( + '--gerrit-issue passed, but --gerrit-patchset not passed.') + self._patchset = args.gerrit_patchset + if not hasattr(args, 'buildbucket_id') or not args.buildbucket_id: + raise RuntimeError( + '--gerrit-issue passed, but --buildbucket-id not passed.') + self._job_id = args.buildbucket_id diff --git a/chromium/build/skia_gold_common/skia_gold_properties_unittest.py b/chromium/build/skia_gold_common/skia_gold_properties_unittest.py new file mode 100755 index 00000000000..348512c7f91 --- /dev/null +++ b/chromium/build/skia_gold_common/skia_gold_properties_unittest.py @@ -0,0 +1,169 @@ +#!/usr/bin/env vpython +# Copyright 2020 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +#pylint: disable=protected-access + +import os +import unittest + +import mock + +from skia_gold_common import skia_gold_properties +from skia_gold_common import unittest_utils + +createSkiaGoldArgs = unittest_utils.createSkiaGoldArgs + + +class SkiaGoldPropertiesInitializationTest(unittest.TestCase): + """Tests that SkiaGoldProperties initializes (or doesn't) when expected.""" + + def verifySkiaGoldProperties(self, instance, expected): + self.assertEqual(instance._local_pixel_tests, + expected.get('local_pixel_tests')) + self.assertEqual(instance._no_luci_auth, expected.get('no_luci_auth')) + self.assertEqual(instance._git_revision, expected.get('git_revision')) + self.assertEqual(instance._issue, expected.get('gerrit_issue')) + self.assertEqual(instance._patchset, expected.get('gerrit_patchset')) + self.assertEqual(instance._job_id, expected.get('buildbucket_id')) + self.assertEqual(instance._bypass_skia_gold_functionality, + expected.get('bypass_skia_gold_functionality')) + + def test_initializeSkiaGoldAttributes_unsetLocal(self): + args = createSkiaGoldArgs() + sgp = skia_gold_properties.SkiaGoldProperties(args) + self.verifySkiaGoldProperties(sgp, {}) + + def test_initializeSkiaGoldAttributes_explicitLocal(self): + args = createSkiaGoldArgs(local_pixel_tests=True) + sgp = skia_gold_properties.SkiaGoldProperties(args) + self.verifySkiaGoldProperties(sgp, {'local_pixel_tests': True}) + + def test_initializeSkiaGoldAttributes_explicitNonLocal(self): + args = createSkiaGoldArgs(local_pixel_tests=False) + sgp = skia_gold_properties.SkiaGoldProperties(args) + self.verifySkiaGoldProperties(sgp, {'local_pixel_tests': False}) + + def test_initializeSkiaGoldAttributes_explicitNoLuciAuth(self): + args = createSkiaGoldArgs(no_luci_auth=True) + sgp = skia_gold_properties.SkiaGoldProperties(args) + self.verifySkiaGoldProperties(sgp, {'no_luci_auth': True}) + + def test_initializeSkiaGoldAttributes_bypassExplicitTrue(self): + args = createSkiaGoldArgs(bypass_skia_gold_functionality=True) + sgp = skia_gold_properties.SkiaGoldProperties(args) + self.verifySkiaGoldProperties(sgp, {'bypass_skia_gold_functionality': True}) + + def test_initializeSkiaGoldAttributes_explicitGitRevision(self): + args = createSkiaGoldArgs(git_revision='a') + sgp = skia_gold_properties.SkiaGoldProperties(args) + self.verifySkiaGoldProperties(sgp, {'git_revision': 'a'}) + + def test_initializeSkiaGoldAttributes_tryjobArgsIgnoredWithoutRevision(self): + args = createSkiaGoldArgs(gerrit_issue=1, + gerrit_patchset=2, + buildbucket_id=3) + sgp = skia_gold_properties.SkiaGoldProperties(args) + self.verifySkiaGoldProperties(sgp, {}) + + def test_initializeSkiaGoldAttributes_tryjobArgs(self): + args = createSkiaGoldArgs(git_revision='a', + gerrit_issue=1, + gerrit_patchset=2, + buildbucket_id=3) + sgp = skia_gold_properties.SkiaGoldProperties(args) + self.verifySkiaGoldProperties( + sgp, { + 'git_revision': 'a', + 'gerrit_issue': 1, + 'gerrit_patchset': 2, + 'buildbucket_id': 3 + }) + + def test_initializeSkiaGoldAttributes_tryjobMissingPatchset(self): + args = createSkiaGoldArgs(git_revision='a', + gerrit_issue=1, + buildbucket_id=3) + with self.assertRaises(RuntimeError): + skia_gold_properties.SkiaGoldProperties(args) + + def test_initializeSkiaGoldAttributes_tryjobMissingBuildbucket(self): + args = createSkiaGoldArgs(git_revision='a', + gerrit_issue=1, + gerrit_patchset=2) + with self.assertRaises(RuntimeError): + skia_gold_properties.SkiaGoldProperties(args) + + +class SkiaGoldPropertiesCalculationTest(unittest.TestCase): + """Tests that SkiaGoldProperties properly calculates certain properties.""" + + def testLocalPixelTests_determineTrue(self): + args = createSkiaGoldArgs() + sgp = skia_gold_properties.SkiaGoldProperties(args) + with mock.patch.dict(os.environ, {}, clear=True): + self.assertTrue(sgp.local_pixel_tests) + + def testLocalPixelTests_determineFalse(self): + args = createSkiaGoldArgs() + sgp = skia_gold_properties.SkiaGoldProperties(args) + with mock.patch.dict(os.environ, {'SWARMING_SERVER': ''}, clear=True): + self.assertFalse(sgp.local_pixel_tests) + + def testIsTryjobRun_noIssue(self): + args = createSkiaGoldArgs() + sgp = skia_gold_properties.SkiaGoldProperties(args) + self.assertFalse(sgp.IsTryjobRun()) + + def testIsTryjobRun_issue(self): + args = createSkiaGoldArgs(git_revision='a', + gerrit_issue=1, + gerrit_patchset=2, + buildbucket_id=3) + sgp = skia_gold_properties.SkiaGoldProperties(args) + self.assertTrue(sgp.IsTryjobRun()) + + def testGetGitRevision_revisionSet(self): + args = createSkiaGoldArgs(git_revision='a') + sgp = skia_gold_properties.SkiaGoldProperties(args) + self.assertEqual(sgp.git_revision, 'a') + + def testGetGitRevision_findValidRevision(self): + args = createSkiaGoldArgs(local_pixel_tests=True) + sgp = skia_gold_properties.SkiaGoldProperties(args) + with mock.patch.object(skia_gold_properties.SkiaGoldProperties, + '_GetGitOriginMasterHeadSha1') as patched_head: + expected = 'a' * 40 + patched_head.return_value = expected + self.assertEqual(sgp.git_revision, expected) + # Should be cached. + self.assertEqual(sgp._git_revision, expected) + + def testGetGitRevision_noExplicitOnBot(self): + args = createSkiaGoldArgs(local_pixel_tests=False) + sgp = skia_gold_properties.SkiaGoldProperties(args) + with self.assertRaises(RuntimeError): + _ = sgp.git_revision + + def testGetGitRevision_findEmptyRevision(self): + args = createSkiaGoldArgs(local_pixel_tests=True) + sgp = skia_gold_properties.SkiaGoldProperties(args) + with mock.patch.object(skia_gold_properties.SkiaGoldProperties, + '_GetGitOriginMasterHeadSha1') as patched_head: + patched_head.return_value = '' + with self.assertRaises(RuntimeError): + _ = sgp.git_revision + + def testGetGitRevision_findMalformedRevision(self): + args = createSkiaGoldArgs(local_pixel_tests=True) + sgp = skia_gold_properties.SkiaGoldProperties(args) + with mock.patch.object(skia_gold_properties.SkiaGoldProperties, + '_GetGitOriginMasterHeadSha1') as patched_head: + patched_head.return_value = 'a' * 39 + with self.assertRaises(RuntimeError): + _ = sgp.git_revision + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff --git a/chromium/build/skia_gold_common/skia_gold_session.py b/chromium/build/skia_gold_common/skia_gold_session.py new file mode 100644 index 00000000000..fcdc06fbe58 --- /dev/null +++ b/chromium/build/skia_gold_common/skia_gold_session.py @@ -0,0 +1,432 @@ +# Copyright 2020 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. +"""Class for interacting with the Skia Gold image diffing service.""" + +import logging +import os +import subprocess +import sys +import tempfile + +CHROMIUM_SRC = os.path.realpath( + os.path.join(os.path.dirname(__file__), '..', '..')) + +GOLDCTL_BINARY = os.path.join(CHROMIUM_SRC, 'tools', 'skia_goldctl') +if sys.platform == 'win32': + GOLDCTL_BINARY = os.path.join(GOLDCTL_BINARY, 'win', 'goldctl') + '.exe' +elif sys.platform == 'darwin': + GOLDCTL_BINARY = os.path.join(GOLDCTL_BINARY, 'mac', 'goldctl') +else: + GOLDCTL_BINARY = os.path.join(GOLDCTL_BINARY, 'linux', 'goldctl') + + +class SkiaGoldSession(object): + class StatusCodes(object): + """Status codes for RunComparison.""" + SUCCESS = 0 + AUTH_FAILURE = 1 + INIT_FAILURE = 2 + COMPARISON_FAILURE_REMOTE = 3 + COMPARISON_FAILURE_LOCAL = 4 + LOCAL_DIFF_FAILURE = 5 + NO_OUTPUT_MANAGER = 6 + + class ComparisonResults(object): + """Struct-like object for storing results of an image comparison.""" + + def __init__(self): + self.triage_link = None + self.triage_link_omission_reason = None + self.local_diff_given_image = None + self.local_diff_closest_image = None + self.local_diff_diff_image = None + + def __init__(self, working_dir, gold_properties, keys_file, corpus, instance): + """Abstract class to handle all aspects of image comparison via Skia Gold. + + A single SkiaGoldSession is valid for a single instance/corpus/keys_file + combination. + + Args: + working_dir: The directory to store config files, etc. + gold_properties: A skia_gold_properties.SkiaGoldProperties instance for + the current test run. + keys_file: A path to a JSON file containing various comparison config data + such as corpus and debug information like the hardware/software + configuration the images will be produced on. + corpus: The corpus that images that will be compared belong to. + instance: The name of the Skia Gold instance to interact with. + """ + self._working_dir = working_dir + self._gold_properties = gold_properties + self._keys_file = keys_file + self._corpus = corpus + self._instance = instance + self._triage_link_file = tempfile.NamedTemporaryFile(suffix='.txt', + dir=working_dir, + delete=False).name + # A map of image name (string) to ComparisonResults for that image. + self._comparison_results = {} + self._authenticated = False + self._initialized = False + + def RunComparison(self, name, png_file, output_manager, use_luci=True): + """Helper method to run all steps to compare a produced image. + + Handles authentication, itnitialization, comparison, and, if necessary, + local diffing. + + Args: + name: The name of the image being compared. + png_file: A path to a PNG file containing the image to be compared. + output_manager: An output manager to use to store diff links. The + argument's type depends on what type a subclasses' _StoreDiffLinks + implementation expects. Can be None even if _StoreDiffLinks expects + a valid input, but will fail if it ever actually needs to be used. + use_luci: If true, authentication will use the service account provided by + the LUCI context. If false, will attempt to use whatever is set up in + gsutil, which is only supported for local runs. + + Returns: + A tuple (status, error). |status| is a value from + SkiaGoldSession.StatusCodes signifying the result of the comparison. + |error| is an error message describing the status if not successful. + """ + auth_rc, auth_stdout = self.Authenticate(use_luci=use_luci) + if auth_rc: + return self.StatusCodes.AUTH_FAILURE, auth_stdout + + init_rc, init_stdout = self.Initialize() + if init_rc: + return self.StatusCodes.INIT_FAILURE, init_stdout + + compare_rc, compare_stdout = self.Compare(name=name, png_file=png_file) + if not compare_rc: + return self.StatusCodes.SUCCESS, None + + logging.error('Gold comparison failed: %s', compare_stdout) + if not self._gold_properties.local_pixel_tests: + return self.StatusCodes.COMPARISON_FAILURE_REMOTE, compare_stdout + + if not output_manager: + return (self.StatusCodes.NO_OUTPUT_MANAGER, + 'No output manager for local diff images') + + diff_rc, diff_stdout = self.Diff(name=name, + png_file=png_file, + output_manager=output_manager) + if diff_rc: + return self.StatusCodes.LOCAL_DIFF_FAILURE, diff_stdout + return self.StatusCodes.COMPARISON_FAILURE_LOCAL, compare_stdout + + def Authenticate(self, use_luci=True): + """Authenticates with Skia Gold for this session. + + Args: + use_luci: If true, authentication will use the service account provided + by the LUCI context. If false, will attempt to use whatever is set up + in gsutil, which is only supported for local runs. + + Returns: + A tuple (return_code, output). |return_code| is the return code of the + authentication process. |output| is the stdout + stderr of the + authentication process. + """ + if self._authenticated: + return 0, None + if self._gold_properties.bypass_skia_gold_functionality: + logging.warning('Not actually authenticating with Gold due to ' + '--bypass-skia-gold-functionality being present.') + return 0, None + + auth_cmd = [GOLDCTL_BINARY, 'auth', '--work-dir', self._working_dir] + if use_luci: + auth_cmd.append('--luci') + elif not self._gold_properties.local_pixel_tests: + raise RuntimeError( + 'Cannot authenticate to Skia Gold with use_luci=False unless running ' + 'local pixel tests') + + rc, stdout = self._RunCmdForRcAndOutput(auth_cmd) + if rc == 0: + self._authenticated = True + return rc, stdout + + def Initialize(self): + """Initializes the working directory if necessary. + + This can technically be skipped if the same information is passed to the + command used for image comparison, but that is less efficient under the + hood. Doing it that way effectively requires an initialization for every + comparison (~250 ms) instead of once at the beginning. + + Returns: + A tuple (return_code, output). |return_code| is the return code of the + initialization process. |output| is the stdout + stderr of the + initialization process. + """ + if self._initialized: + return 0, None + if self._gold_properties.bypass_skia_gold_functionality: + logging.warning('Not actually initializing Gold due to ' + '--bypass-skia-gold-functionality being present.') + return 0, None + + init_cmd = [ + GOLDCTL_BINARY, + 'imgtest', + 'init', + '--passfail', + '--instance', + self._instance, + '--corpus', + self._corpus, + '--keys-file', + self._keys_file, + '--work-dir', + self._working_dir, + '--failure-file', + self._triage_link_file, + '--commit', + self._gold_properties.git_revision, + ] + if self._gold_properties.IsTryjobRun(): + init_cmd.extend([ + '--issue', + str(self._gold_properties.issue), + '--patchset', + str(self._gold_properties.patchset), + '--jobid', + str(self._gold_properties.job_id), + '--crs', + str(self._gold_properties.code_review_system), + '--cis', + str(self._gold_properties.continuous_integration_system), + ]) + + rc, stdout = self._RunCmdForRcAndOutput(init_cmd) + if rc == 0: + self._initialized = True + return rc, stdout + + def Compare(self, name, png_file): + """Compares the given image to images known to Gold. + + Triage links can later be retrieved using GetTriageLink(). + + Args: + name: The name of the image being compared. + png_file: A path to a PNG file containing the image to be compared. + + Returns: + A tuple (return_code, output). |return_code| is the return code of the + comparison process. |output| is the stdout + stderr of the comparison + process. + """ + if self._gold_properties.bypass_skia_gold_functionality: + logging.warning('Not actually comparing with Gold due to ' + '--bypass-skia-gold-functionality being present.') + return 0, None + + compare_cmd = [ + GOLDCTL_BINARY, + 'imgtest', + 'add', + '--test-name', + name, + '--png-file', + png_file, + '--work-dir', + self._working_dir, + ] + if self._gold_properties.local_pixel_tests: + compare_cmd.append('--dryrun') + + self._ClearTriageLinkFile() + rc, stdout = self._RunCmdForRcAndOutput(compare_cmd) + + self._comparison_results[name] = self.ComparisonResults() + if rc == 0: + self._comparison_results[name].triage_link_omission_reason = ( + 'Comparison succeeded, no triage link') + elif self._gold_properties.IsTryjobRun(): + cl_triage_link = ('https://{instance}-gold.skia.org/cl/{crs}/{issue}') + cl_triage_link = cl_triage_link.format( + instance=self._instance, + crs=self._gold_properties.code_review_system, + issue=self._gold_properties.issue) + self._comparison_results[name].triage_link = cl_triage_link + else: + try: + with open(self._triage_link_file) as tlf: + triage_link = tlf.read().strip() + self._comparison_results[name].triage_link = triage_link + except IOError: + self._comparison_results[name].triage_link_omission_reason = ( + 'Failed to read triage link from file') + return rc, stdout + + def Diff(self, name, png_file, output_manager): + """Performs a local image diff against the closest known positive in Gold. + + This is used for running tests on a workstation, where uploading data to + Gold for ingestion is not allowed, and thus the web UI is not available. + + Image links can later be retrieved using Get*ImageLink(). + + Args: + name: The name of the image being compared. + png_file: The path to a PNG file containing the image to be diffed. + output_manager: An output manager to use to store diff links. The + argument's type depends on what type a subclasses' _StoreDiffLinks + implementation expects. + + Returns: + A tuple (return_code, output). |return_code| is the return code of the + diff process. |output| is the stdout + stderr of the diff process. + """ + # Instead of returning that everything is okay and putting in dummy links, + # just fail since this should only be called when running locally and + # --bypass-skia-gold-functionality is only meant for use on the bots. + if self._gold_properties.bypass_skia_gold_functionality: + raise RuntimeError( + '--bypass-skia-gold-functionality is not supported when running ' + 'tests locally.') + + output_dir = self._CreateDiffOutputDir() + diff_cmd = [ + GOLDCTL_BINARY, + 'diff', + '--corpus', + self._corpus, + '--instance', + self._instance, + '--input', + png_file, + '--test', + name, + '--work-dir', + self._working_dir, + '--out-dir', + output_dir, + ] + rc, stdout = self._RunCmdForRcAndOutput(diff_cmd) + self._StoreDiffLinks(name, output_manager, output_dir) + return rc, stdout + + def GetTriageLink(self, name): + """Gets the triage link for the given image. + + Args: + name: The name of the image to retrieve the triage link for. + + Returns: + A string containing the triage link if it is available, or None if it is + not available for some reason. The reason can be retrieved using + GetTriageLinkOmissionReason. + """ + return self._comparison_results.get(name, + self.ComparisonResults()).triage_link + + def GetTriageLinkOmissionReason(self, name): + """Gets the reason why a triage link is not available for an image. + + Args: + name: The name of the image whose triage link does not exist. + + Returns: + A string containing the reason why a triage link is not available. + """ + if name not in self._comparison_results: + return 'No image comparison performed for %s' % name + results = self._comparison_results[name] + # This method should not be called if there is a valid triage link. + assert results.triage_link is None + if results.triage_link_omission_reason: + return results.triage_link_omission_reason + if results.local_diff_given_image: + return 'Gold only used to do a local image diff' + raise RuntimeError( + 'Somehow have a ComparisonResults instance for %s that should not ' + 'exist' % name) + + def GetGivenImageLink(self, name): + """Gets the link to the given image used for local diffing. + + Args: + name: The name of the image that was diffed. + + Returns: + A string containing the link to where the image is saved, or None if it + does not exist. + """ + assert name in self._comparison_results + return self._comparison_results[name].local_diff_given_image + + def GetClosestImageLink(self, name): + """Gets the link to the closest known image used for local diffing. + + Args: + name: The name of the image that was diffed. + + Returns: + A string containing the link to where the image is saved, or None if it + does not exist. + """ + assert name in self._comparison_results + return self._comparison_results[name].local_diff_closest_image + + def GetDiffImageLink(self, name): + """Gets the link to the diff between the given and closest images. + + Args: + name: The name of the image that was diffed. + + Returns: + A string containing the link to where the image is saved, or None if it + does not exist. + """ + assert name in self._comparison_results + return self._comparison_results[name].local_diff_diff_image + + def _ClearTriageLinkFile(self): + """Clears the contents of the triage link file. + + This should be done before every comparison since goldctl appends to the + file instead of overwriting its contents, which results in multiple triage + links getting concatenated together if there are multiple failures. + """ + open(self._triage_link_file, 'w').close() + + def _CreateDiffOutputDir(self): + return tempfile.mkdtemp(dir=self._working_dir) + + def _StoreDiffLinks(self, image_name, output_manager, output_dir): + """Stores the local diff files as links. + + The ComparisonResults entry for |image_name| should have its *_image fields + filled after this unless corresponding images were not found on disk. + + Args: + image_name: A string containing the name of the image that was diffed. + output_manager: An output manager used used to surface links to users, + if necessary. The expected argument type depends on each subclasses' + implementation of this method. + output_dir: A string containing the path to the directory where diff + output image files where saved. + """ + raise NotImplementedError() + + @staticmethod + def _RunCmdForRcAndOutput(cmd): + """Runs |cmd| and returns its returncode and output. + + Args: + cmd: A list containing the command line to run. + + Returns: + A tuple (rc, output), where, |rc| is the returncode of the command and + |output| is the stdout + stderr of the command. + """ + raise NotImplementedError() diff --git a/chromium/build/skia_gold_common/skia_gold_session_manager.py b/chromium/build/skia_gold_common/skia_gold_session_manager.py new file mode 100644 index 00000000000..f4f5f178817 --- /dev/null +++ b/chromium/build/skia_gold_common/skia_gold_session_manager.py @@ -0,0 +1,115 @@ +# Copyright 2020 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. +"""Class for managing multiple SkiaGoldSessions.""" + +import json +import tempfile + + +class SkiaGoldSessionManager(object): + def __init__(self, working_dir, gold_properties): + """Abstract class to manage one or more skia_gold_session.SkiaGoldSessions. + + A separate session is required for each instance/corpus/keys_file + combination, so this class will lazily create them as necessary. + + Args: + working_dir: The working directory under which each individual + SkiaGoldSessions' working directory will be created. + gold_properties: A SkiaGoldProperties instance that will be used to create + any SkiaGoldSessions. + """ + self._working_dir = working_dir + self._gold_properties = gold_properties + self._sessions = {} + + def GetSkiaGoldSession(self, keys_input, corpus=None, instance=None): + """Gets a SkiaGoldSession for the given arguments. + + Lazily creates one if necessary. + + Args: + keys_input: A way of retrieving various comparison config data such as + corpus and debug information like the hardware/software configuration + the image was produced on. Can be either a dict or a filepath to a + file containing JSON to read. + corpus: A string containing the corpus the session is for. If None, the + corpus will be determined using available information. + instance: The name of the Skia Gold instance to interact with. It None, + will use whatever default the subclass sets. + """ + instance = instance or self._GetDefaultInstance() + keys_dict = _GetKeysAsDict(keys_input) + keys_string = json.dumps(keys_dict, sort_keys=True) + if corpus is None: + corpus = keys_dict.get('source_type', instance) + # Use the string representation of the keys JSON as a proxy for a hash since + # dicts themselves are not hashable. + session = self._sessions.setdefault(instance, + {}).setdefault(corpus, {}).setdefault( + keys_string, None) + if not session: + working_dir = tempfile.mkdtemp(dir=self._working_dir) + keys_file = _GetKeysAsJson(keys_input, working_dir) + session = self._GetSessionClass()(working_dir, self._gold_properties, + keys_file, corpus, instance) + self._sessions[instance][corpus][keys_string] = session + return session + + @staticmethod + def _GetDefaultInstance(): + """Gets the default Skia Gold instance. + + Returns: + A string containing the default instance. + """ + raise NotImplementedError + + @staticmethod + def _GetSessionClass(): + """Gets the SkiaGoldSession class to use for session creation. + + Returns: + A reference to a SkiaGoldSession class. + """ + raise NotImplementedError + + +def _GetKeysAsDict(keys_input): + """Converts |keys_input| into a dictionary. + + Args: + keys_input: A dictionary or a string pointing to a JSON file. The contents + of either should be Skia Gold config data. + + Returns: + A dictionary containing the Skia Gold config data. + """ + if isinstance(keys_input, dict): + return keys_input + assert isinstance(keys_input, str) + with open(keys_input) as f: + return json.load(f) + + +def _GetKeysAsJson(keys_input, session_work_dir): + """Converts |keys_input| into a JSON file on disk. + + Args: + keys_input: A dictionary or a string pointing to a JSON file. The contents + of either should be Skia Gold config data. + + Returns: + A string containing a filepath to a JSON file with containing |keys_input|'s + data. + """ + if isinstance(keys_input, str): + return keys_input + assert isinstance(keys_input, dict) + keys_file = tempfile.NamedTemporaryFile(suffix='.json', + dir=session_work_dir, + delete=False).name + with open(keys_file, 'w') as f: + json.dump(keys_input, f) + return keys_file diff --git a/chromium/build/skia_gold_common/skia_gold_session_manager_unittest.py b/chromium/build/skia_gold_common/skia_gold_session_manager_unittest.py new file mode 100755 index 00000000000..453ec9a9ae2 --- /dev/null +++ b/chromium/build/skia_gold_common/skia_gold_session_manager_unittest.py @@ -0,0 +1,176 @@ +#!/usr/bin/env vpython +# Copyright 2020 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +#pylint: disable=protected-access + +import json +import os +import tempfile +import unittest + +import mock + +from pyfakefs import fake_filesystem_unittest + +from skia_gold_common import skia_gold_properties +from skia_gold_common import skia_gold_session +from skia_gold_common import skia_gold_session_manager +from skia_gold_common import unittest_utils + +createSkiaGoldArgs = unittest_utils.createSkiaGoldArgs + + +class SkiaGoldSessionManagerGetSessionTest(fake_filesystem_unittest.TestCase): + """Tests the functionality of SkiaGoldSessionManager.GetSkiaGoldSession.""" + + def setUp(self): + self.setUpPyfakefs() + self._working_dir = tempfile.mkdtemp() + self._patcher = mock.patch.object( + skia_gold_session_manager.SkiaGoldSessionManager, '_GetSessionClass') + self._session_class_mock = self._patcher.start() + self._session_class_mock.return_value = skia_gold_session.SkiaGoldSession + self.addCleanup(self._patcher.stop) + + def test_ArgsForwardedToSession(self): + args = createSkiaGoldArgs() + sgp = skia_gold_properties.SkiaGoldProperties(args) + sgsm = skia_gold_session_manager.SkiaGoldSessionManager( + self._working_dir, sgp) + session = sgsm.GetSkiaGoldSession({}, 'corpus', 'instance') + self.assertTrue(session._keys_file.startswith(self._working_dir)) + self.assertEqual(session._corpus, 'corpus') + self.assertEqual(session._instance, 'instance') + # Make sure the session's working directory is a subdirectory of the + # manager's working directory. + self.assertEqual(os.path.dirname(session._working_dir), self._working_dir) + + def test_corpusFromJson(self): + args = createSkiaGoldArgs() + sgp = skia_gold_properties.SkiaGoldProperties(args) + sgsm = skia_gold_session_manager.SkiaGoldSessionManager( + self._working_dir, sgp) + session = sgsm.GetSkiaGoldSession({'source_type': 'foobar'}, None, + 'instance') + self.assertTrue(session._keys_file.startswith(self._working_dir)) + self.assertEqual(session._corpus, 'foobar') + self.assertEqual(session._instance, 'instance') + + def test_corpusDefaultsToInstance(self): + args = createSkiaGoldArgs() + sgp = skia_gold_properties.SkiaGoldProperties(args) + sgsm = skia_gold_session_manager.SkiaGoldSessionManager( + self._working_dir, sgp) + session = sgsm.GetSkiaGoldSession({}, None, 'instance') + self.assertTrue(session._keys_file.startswith(self._working_dir)) + self.assertEqual(session._corpus, 'instance') + self.assertEqual(session._instance, 'instance') + + @mock.patch.object(skia_gold_session_manager.SkiaGoldSessionManager, + '_GetDefaultInstance') + def test_getDefaultInstance(self, default_instance_mock): + default_instance_mock.return_value = 'default' + args = createSkiaGoldArgs() + sgp = skia_gold_properties.SkiaGoldProperties(args) + sgsm = skia_gold_session_manager.SkiaGoldSessionManager( + self._working_dir, sgp) + session = sgsm.GetSkiaGoldSession({}, None, None) + self.assertTrue(session._keys_file.startswith(self._working_dir)) + self.assertEqual(session._corpus, 'default') + self.assertEqual(session._instance, 'default') + + @mock.patch.object(skia_gold_session.SkiaGoldSession, '__init__') + def test_matchingSessionReused(self, session_mock): + session_mock.return_value = None + args = createSkiaGoldArgs() + sgp = skia_gold_properties.SkiaGoldProperties(args) + sgsm = skia_gold_session_manager.SkiaGoldSessionManager( + self._working_dir, sgp) + session1 = sgsm.GetSkiaGoldSession({}, 'corpus', 'instance') + session2 = sgsm.GetSkiaGoldSession({}, 'corpus', 'instance') + self.assertEqual(session1, session2) + # For some reason, session_mock.assert_called_once() always passes, + # so check the call count directly. + self.assertEqual(session_mock.call_count, 1) + + @mock.patch.object(skia_gold_session.SkiaGoldSession, '__init__') + def test_separateSessionsFromKeys(self, session_mock): + session_mock.return_value = None + args = createSkiaGoldArgs() + sgp = skia_gold_properties.SkiaGoldProperties(args) + sgsm = skia_gold_session_manager.SkiaGoldSessionManager( + self._working_dir, sgp) + session1 = sgsm.GetSkiaGoldSession({}, 'corpus', 'instance') + session2 = sgsm.GetSkiaGoldSession({'something_different': 1}, 'corpus', + 'instance') + self.assertNotEqual(session1, session2) + self.assertEqual(session_mock.call_count, 2) + + @mock.patch.object(skia_gold_session.SkiaGoldSession, '__init__') + def test_separateSessionsFromCorpus(self, session_mock): + session_mock.return_value = None + args = createSkiaGoldArgs() + sgp = skia_gold_properties.SkiaGoldProperties(args) + sgsm = skia_gold_session_manager.SkiaGoldSessionManager( + self._working_dir, sgp) + session1 = sgsm.GetSkiaGoldSession({}, 'corpus1', 'instance') + session2 = sgsm.GetSkiaGoldSession({}, 'corpus2', 'instance') + self.assertNotEqual(session1, session2) + self.assertEqual(session_mock.call_count, 2) + + @mock.patch.object(skia_gold_session.SkiaGoldSession, '__init__') + def test_separateSessionsFromInstance(self, session_mock): + session_mock.return_value = None + args = createSkiaGoldArgs() + sgp = skia_gold_properties.SkiaGoldProperties(args) + self._working_dir = tempfile.mkdtemp() + sgsm = skia_gold_session_manager.SkiaGoldSessionManager( + self._working_dir, sgp) + session1 = sgsm.GetSkiaGoldSession({}, 'corpus', 'instance1') + session2 = sgsm.GetSkiaGoldSession({}, 'corpus', 'instance2') + self.assertNotEqual(session1, session2) + self.assertEqual(session_mock.call_count, 2) + + +class SkiaGoldSessionManagerKeyConversionTest(fake_filesystem_unittest.TestCase + ): + def setUp(self): + self.setUpPyfakefs() + self._working_dir = tempfile.mkdtemp() + + def test_getKeysAsDict(self): + keys_dict = {'foo': 'bar'} + keys_file_contents = {'bar': 'baz'} + keys_file = tempfile.NamedTemporaryFile(delete=False).name + with open(keys_file, 'w') as f: + json.dump(keys_file_contents, f) + + self.assertEqual(skia_gold_session_manager._GetKeysAsDict(keys_dict), + keys_dict) + self.assertEqual(skia_gold_session_manager._GetKeysAsDict(keys_file), + keys_file_contents) + with self.assertRaises(AssertionError): + skia_gold_session_manager._GetKeysAsDict(1) + + def test_getKeysAsJson(self): + keys_dict = {'foo': 'bar'} + keys_file_contents = {'bar': 'baz'} + keys_file = tempfile.NamedTemporaryFile(delete=False).name + with open(keys_file, 'w') as f: + json.dump(keys_file_contents, f) + + self.assertEqual(skia_gold_session_manager._GetKeysAsJson(keys_file, None), + keys_file) + keys_dict_as_json = skia_gold_session_manager._GetKeysAsJson( + keys_dict, self._working_dir) + self.assertTrue(keys_dict_as_json.startswith(self._working_dir)) + with open(keys_dict_as_json) as f: + self.assertEqual(json.load(f), keys_dict) + with self.assertRaises(AssertionError): + skia_gold_session_manager._GetKeysAsJson(1, None) + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff --git a/chromium/build/skia_gold_common/skia_gold_session_unittest.py b/chromium/build/skia_gold_common/skia_gold_session_unittest.py new file mode 100755 index 00000000000..65e353b72f1 --- /dev/null +++ b/chromium/build/skia_gold_common/skia_gold_session_unittest.py @@ -0,0 +1,649 @@ +#!/usr/bin/env vpython +# Copyright 2020 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +#pylint: disable=protected-access + +import json +import os +import tempfile +import unittest + +import mock + +from pyfakefs import fake_filesystem_unittest + +from skia_gold_common import skia_gold_properties +from skia_gold_common import skia_gold_session +from skia_gold_common import unittest_utils + +createSkiaGoldArgs = unittest_utils.createSkiaGoldArgs + + +def assertArgWith(test, arg_list, arg, value): + i = arg_list.index(arg) + test.assertEqual(arg_list[i + 1], value) + + +class SkiaGoldSessionRunComparisonTest(fake_filesystem_unittest.TestCase): + """Tests the functionality of SkiaGoldSession.RunComparison.""" + + def setUp(self): + self.setUpPyfakefs() + self._working_dir = tempfile.mkdtemp() + + @mock.patch.object(skia_gold_session.SkiaGoldSession, 'Diff') + @mock.patch.object(skia_gold_session.SkiaGoldSession, 'Compare') + @mock.patch.object(skia_gold_session.SkiaGoldSession, 'Initialize') + @mock.patch.object(skia_gold_session.SkiaGoldSession, 'Authenticate') + def test_comparisonSuccess(self, auth_mock, init_mock, compare_mock, + diff_mock): + auth_mock.return_value = (0, None) + init_mock.return_value = (0, None) + compare_mock.return_value = (0, None) + keys_file = os.path.join(self._working_dir, 'keys.json') + with open(os.path.join(self._working_dir, 'keys.json'), 'w') as f: + json.dump({}, f) + session = skia_gold_session.SkiaGoldSession(self._working_dir, None, + keys_file, None, None) + status, _ = session.RunComparison(None, None, None) + self.assertEqual(status, + skia_gold_session.SkiaGoldSession.StatusCodes.SUCCESS) + self.assertEqual(auth_mock.call_count, 1) + self.assertEqual(init_mock.call_count, 1) + self.assertEqual(compare_mock.call_count, 1) + self.assertEqual(diff_mock.call_count, 0) + + @mock.patch.object(skia_gold_session.SkiaGoldSession, 'Diff') + @mock.patch.object(skia_gold_session.SkiaGoldSession, 'Compare') + @mock.patch.object(skia_gold_session.SkiaGoldSession, 'Initialize') + @mock.patch.object(skia_gold_session.SkiaGoldSession, 'Authenticate') + def test_authFailure(self, auth_mock, init_mock, compare_mock, diff_mock): + auth_mock.return_value = (1, 'Auth failed') + session = skia_gold_session.SkiaGoldSession(self._working_dir, None, None, + None, None) + status, error = session.RunComparison(None, None, None) + self.assertEqual(status, + skia_gold_session.SkiaGoldSession.StatusCodes.AUTH_FAILURE) + self.assertEqual(error, 'Auth failed') + self.assertEqual(auth_mock.call_count, 1) + self.assertEqual(init_mock.call_count, 0) + self.assertEqual(compare_mock.call_count, 0) + self.assertEqual(diff_mock.call_count, 0) + + @mock.patch.object(skia_gold_session.SkiaGoldSession, 'Diff') + @mock.patch.object(skia_gold_session.SkiaGoldSession, 'Compare') + @mock.patch.object(skia_gold_session.SkiaGoldSession, 'Initialize') + @mock.patch.object(skia_gold_session.SkiaGoldSession, 'Authenticate') + def test_initFailure(self, auth_mock, init_mock, compare_mock, diff_mock): + auth_mock.return_value = (0, None) + init_mock.return_value = (1, 'Init failed') + session = skia_gold_session.SkiaGoldSession(self._working_dir, None, None, + None, None) + status, error = session.RunComparison(None, None, None) + self.assertEqual(status, + skia_gold_session.SkiaGoldSession.StatusCodes.INIT_FAILURE) + self.assertEqual(error, 'Init failed') + self.assertEqual(auth_mock.call_count, 1) + self.assertEqual(init_mock.call_count, 1) + self.assertEqual(compare_mock.call_count, 0) + self.assertEqual(diff_mock.call_count, 0) + + @mock.patch.object(skia_gold_session.SkiaGoldSession, 'Diff') + @mock.patch.object(skia_gold_session.SkiaGoldSession, 'Compare') + @mock.patch.object(skia_gold_session.SkiaGoldSession, 'Initialize') + @mock.patch.object(skia_gold_session.SkiaGoldSession, 'Authenticate') + def test_compareFailureRemote(self, auth_mock, init_mock, compare_mock, + diff_mock): + auth_mock.return_value = (0, None) + init_mock.return_value = (0, None) + compare_mock.return_value = (1, 'Compare failed') + args = createSkiaGoldArgs(local_pixel_tests=False) + sgp = skia_gold_properties.SkiaGoldProperties(args) + keys_file = os.path.join(self._working_dir, 'keys.json') + with open(os.path.join(self._working_dir, 'keys.json'), 'w') as f: + json.dump({}, f) + session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, + keys_file, None, None) + status, error = session.RunComparison(None, None, None) + self.assertEqual( + status, + skia_gold_session.SkiaGoldSession.StatusCodes.COMPARISON_FAILURE_REMOTE) + self.assertEqual(error, 'Compare failed') + self.assertEqual(auth_mock.call_count, 1) + self.assertEqual(init_mock.call_count, 1) + self.assertEqual(compare_mock.call_count, 1) + self.assertEqual(diff_mock.call_count, 0) + + @mock.patch.object(skia_gold_session.SkiaGoldSession, 'Diff') + @mock.patch.object(skia_gold_session.SkiaGoldSession, 'Compare') + @mock.patch.object(skia_gold_session.SkiaGoldSession, 'Initialize') + @mock.patch.object(skia_gold_session.SkiaGoldSession, 'Authenticate') + def test_compareFailureLocal(self, auth_mock, init_mock, compare_mock, + diff_mock): + auth_mock.return_value = (0, None) + init_mock.return_value = (0, None) + compare_mock.return_value = (1, 'Compare failed') + diff_mock.return_value = (0, None) + args = createSkiaGoldArgs(local_pixel_tests=True) + sgp = skia_gold_properties.SkiaGoldProperties(args) + keys_file = os.path.join(self._working_dir, 'keys.json') + with open(os.path.join(self._working_dir, 'keys.json'), 'w') as f: + json.dump({}, f) + session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, + keys_file, None, None) + status, error = session.RunComparison(None, None, + 'Definitely an output manager') + self.assertEqual( + status, + skia_gold_session.SkiaGoldSession.StatusCodes.COMPARISON_FAILURE_LOCAL) + self.assertEqual(error, 'Compare failed') + self.assertEqual(auth_mock.call_count, 1) + self.assertEqual(init_mock.call_count, 1) + self.assertEqual(compare_mock.call_count, 1) + self.assertEqual(diff_mock.call_count, 1) + + @mock.patch.object(skia_gold_session.SkiaGoldSession, 'Diff') + @mock.patch.object(skia_gold_session.SkiaGoldSession, 'Compare') + @mock.patch.object(skia_gold_session.SkiaGoldSession, 'Initialize') + @mock.patch.object(skia_gold_session.SkiaGoldSession, 'Authenticate') + def test_diffFailure(self, auth_mock, init_mock, compare_mock, diff_mock): + auth_mock.return_value = (0, None) + init_mock.return_value = (0, None) + compare_mock.return_value = (1, 'Compare failed') + diff_mock.return_value = (1, 'Diff failed') + args = createSkiaGoldArgs(local_pixel_tests=True) + sgp = skia_gold_properties.SkiaGoldProperties(args) + keys_file = os.path.join(self._working_dir, 'keys.json') + with open(os.path.join(self._working_dir, 'keys.json'), 'w') as f: + json.dump({}, f) + session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, + keys_file, None, None) + status, error = session.RunComparison(None, None, + 'Definitely an output manager') + self.assertEqual( + status, + skia_gold_session.SkiaGoldSession.StatusCodes.LOCAL_DIFF_FAILURE) + self.assertEqual(error, 'Diff failed') + self.assertEqual(auth_mock.call_count, 1) + self.assertEqual(init_mock.call_count, 1) + self.assertEqual(compare_mock.call_count, 1) + self.assertEqual(diff_mock.call_count, 1) + + @mock.patch.object(skia_gold_session.SkiaGoldSession, 'Diff') + @mock.patch.object(skia_gold_session.SkiaGoldSession, 'Compare') + @mock.patch.object(skia_gold_session.SkiaGoldSession, 'Initialize') + @mock.patch.object(skia_gold_session.SkiaGoldSession, 'Authenticate') + def test_noOutputManagerLocal(self, auth_mock, init_mock, compare_mock, + diff_mock): + auth_mock.return_value = (0, None) + init_mock.return_value = (0, None) + compare_mock.return_value = (1, 'Compare failed') + diff_mock.return_value = (0, None) + args = createSkiaGoldArgs(local_pixel_tests=True) + sgp = skia_gold_properties.SkiaGoldProperties(args) + keys_file = os.path.join(self._working_dir, 'keys.json') + with open(os.path.join(self._working_dir, 'keys.json'), 'w') as f: + json.dump({}, f) + session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, + keys_file, None, None) + status, error = session.RunComparison(None, None, None) + self.assertEqual( + status, skia_gold_session.SkiaGoldSession.StatusCodes.NO_OUTPUT_MANAGER) + self.assertEqual(error, 'No output manager for local diff images') + self.assertEqual(auth_mock.call_count, 1) + self.assertEqual(compare_mock.call_count, 1) + self.assertEqual(diff_mock.call_count, 0) + + +class SkiaGoldSessionAuthenticateTest(fake_filesystem_unittest.TestCase): + """Tests the functionality of SkiaGoldSession.Authenticate.""" + + def setUp(self): + self.setUpPyfakefs() + self._working_dir = tempfile.mkdtemp() + + @mock.patch.object(skia_gold_session.SkiaGoldSession, '_RunCmdForRcAndOutput') + def test_commandOutputReturned(self, cmd_mock): + cmd_mock.return_value = (1, 'Something bad :(') + args = createSkiaGoldArgs(git_revision='a') + sgp = skia_gold_properties.SkiaGoldProperties(args) + session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, None, + None, None) + rc, stdout = session.Authenticate() + self.assertEqual(cmd_mock.call_count, 1) + self.assertEqual(rc, 1) + self.assertEqual(stdout, 'Something bad :(') + + @mock.patch.object(skia_gold_session.SkiaGoldSession, '_RunCmdForRcAndOutput') + def test_bypassSkiaGoldFunctionality(self, cmd_mock): + cmd_mock.return_value = (None, None) + args = createSkiaGoldArgs(git_revision='a', + bypass_skia_gold_functionality=True) + sgp = skia_gold_properties.SkiaGoldProperties(args) + session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, None, + None, None) + rc, _ = session.Authenticate() + self.assertEqual(rc, 0) + cmd_mock.assert_not_called() + + @mock.patch.object(skia_gold_session.SkiaGoldSession, '_RunCmdForRcAndOutput') + def test_shortCircuitAlreadyAuthenticated(self, cmd_mock): + cmd_mock.return_value = (None, None) + args = createSkiaGoldArgs(git_revision='a') + sgp = skia_gold_properties.SkiaGoldProperties(args) + session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, None, + None, None) + session._authenticated = True + rc, _ = session.Authenticate() + self.assertEqual(rc, 0) + cmd_mock.assert_not_called() + + @mock.patch.object(skia_gold_session.SkiaGoldSession, '_RunCmdForRcAndOutput') + def test_successSetsShortCircuit(self, cmd_mock): + cmd_mock.return_value = (0, None) + args = createSkiaGoldArgs(git_revision='a') + sgp = skia_gold_properties.SkiaGoldProperties(args) + session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, None, + None, None) + self.assertFalse(session._authenticated) + rc, _ = session.Authenticate() + self.assertEqual(rc, 0) + self.assertTrue(session._authenticated) + cmd_mock.assert_called_once() + + @mock.patch.object(skia_gold_session.SkiaGoldSession, '_RunCmdForRcAndOutput') + def test_failureDoesNotSetShortCircuit(self, cmd_mock): + cmd_mock.return_value = (1, None) + args = createSkiaGoldArgs(git_revision='a') + sgp = skia_gold_properties.SkiaGoldProperties(args) + session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, None, + None, None) + self.assertFalse(session._authenticated) + rc, _ = session.Authenticate() + self.assertEqual(rc, 1) + self.assertFalse(session._authenticated) + cmd_mock.assert_called_once() + + @mock.patch.object(skia_gold_session.SkiaGoldSession, '_RunCmdForRcAndOutput') + def test_commandWithUseLuciTrue(self, cmd_mock): + cmd_mock.return_value = (None, None) + args = createSkiaGoldArgs(git_revision='a') + sgp = skia_gold_properties.SkiaGoldProperties(args) + session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, None, + None, None) + session.Authenticate(use_luci=True) + self.assertIn('--luci', cmd_mock.call_args[0][0]) + + @mock.patch.object(skia_gold_session.SkiaGoldSession, '_RunCmdForRcAndOutput') + def test_commandWithUseLuciFalse(self, cmd_mock): + cmd_mock.return_value = (None, None) + args = createSkiaGoldArgs(git_revision='a', local_pixel_tests=True) + sgp = skia_gold_properties.SkiaGoldProperties(args) + session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, None, + None, None) + session.Authenticate(use_luci=False) + self.assertNotIn('--luci', cmd_mock.call_args[0][0]) + + @mock.patch.object(skia_gold_session.SkiaGoldSession, '_RunCmdForRcAndOutput') + def test_commandWithUseLuciFalseNotLocal(self, cmd_mock): + cmd_mock.return_value = (None, None) + args = createSkiaGoldArgs(git_revision='a', local_pixel_tests=False) + sgp = skia_gold_properties.SkiaGoldProperties(args) + session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, None, + None, None) + with self.assertRaises(RuntimeError): + session.Authenticate(use_luci=False) + + @mock.patch.object(skia_gold_session.SkiaGoldSession, '_RunCmdForRcAndOutput') + def test_commandCommonArgs(self, cmd_mock): + cmd_mock.return_value = (None, None) + args = createSkiaGoldArgs(git_revision='a') + sgp = skia_gold_properties.SkiaGoldProperties(args) + session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, None, + None, None) + session.Authenticate() + call_args = cmd_mock.call_args[0][0] + self.assertIn('auth', call_args) + assertArgWith(self, call_args, '--work-dir', self._working_dir) + + +class SkiaGoldSessionInitializeTest(fake_filesystem_unittest.TestCase): + """Tests the functionality of SkiaGoldSession.Initialize.""" + + def setUp(self): + self.setUpPyfakefs() + self._working_dir = tempfile.mkdtemp() + + @mock.patch.object(skia_gold_session.SkiaGoldSession, '_RunCmdForRcAndOutput') + def test_bypassSkiaGoldFunctionality(self, cmd_mock): + cmd_mock.return_value = (None, None) + args = createSkiaGoldArgs(git_revision='a', + bypass_skia_gold_functionality=True) + sgp = skia_gold_properties.SkiaGoldProperties(args) + session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, None, + None, None) + rc, _ = session.Initialize() + self.assertEqual(rc, 0) + cmd_mock.assert_not_called() + + @mock.patch.object(skia_gold_session.SkiaGoldSession, '_RunCmdForRcAndOutput') + def test_shortCircuitAlreadyInitialized(self, cmd_mock): + cmd_mock.return_value = (None, None) + args = createSkiaGoldArgs(git_revision='a') + sgp = skia_gold_properties.SkiaGoldProperties(args) + session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, None, + None, None) + session._initialized = True + rc, _ = session.Initialize() + self.assertEqual(rc, 0) + cmd_mock.assert_not_called() + + @mock.patch.object(skia_gold_session.SkiaGoldSession, '_RunCmdForRcAndOutput') + def test_successSetsShortCircuit(self, cmd_mock): + cmd_mock.return_value = (0, None) + args = createSkiaGoldArgs(git_revision='a') + sgp = skia_gold_properties.SkiaGoldProperties(args) + session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, None, + None, None) + self.assertFalse(session._initialized) + rc, _ = session.Initialize() + self.assertEqual(rc, 0) + self.assertTrue(session._initialized) + cmd_mock.assert_called_once() + + @mock.patch.object(skia_gold_session.SkiaGoldSession, '_RunCmdForRcAndOutput') + def test_failureDoesNotSetShortCircuit(self, cmd_mock): + cmd_mock.return_value = (1, None) + args = createSkiaGoldArgs(git_revision='a') + sgp = skia_gold_properties.SkiaGoldProperties(args) + session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, None, + None, None) + self.assertFalse(session._initialized) + rc, _ = session.Initialize() + self.assertEqual(rc, 1) + self.assertFalse(session._initialized) + cmd_mock.assert_called_once() + + @mock.patch.object(skia_gold_session.SkiaGoldSession, '_RunCmdForRcAndOutput') + def test_commandCommonArgs(self, cmd_mock): + cmd_mock.return_value = (None, None) + args = createSkiaGoldArgs(git_revision='a') + sgp = skia_gold_properties.SkiaGoldProperties(args) + session = skia_gold_session.SkiaGoldSession(self._working_dir, + sgp, + 'keys_file', + 'corpus', + instance='instance') + session.Initialize() + call_args = cmd_mock.call_args[0][0] + self.assertIn('imgtest', call_args) + self.assertIn('init', call_args) + self.assertIn('--passfail', call_args) + assertArgWith(self, call_args, '--instance', 'instance') + assertArgWith(self, call_args, '--corpus', 'corpus') + assertArgWith(self, call_args, '--keys-file', 'keys_file') + assertArgWith(self, call_args, '--work-dir', self._working_dir) + assertArgWith(self, call_args, '--failure-file', session._triage_link_file) + assertArgWith(self, call_args, '--commit', 'a') + + @mock.patch.object(skia_gold_session.SkiaGoldSession, '_RunCmdForRcAndOutput') + def test_commandTryjobArgs(self, cmd_mock): + cmd_mock.return_value = (None, None) + args = createSkiaGoldArgs(git_revision='a', + gerrit_issue=1, + gerrit_patchset=2, + buildbucket_id=3) + sgp = skia_gold_properties.SkiaGoldProperties(args) + session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, None, + None, None) + session.Initialize() + call_args = cmd_mock.call_args[0][0] + assertArgWith(self, call_args, '--issue', '1') + assertArgWith(self, call_args, '--patchset', '2') + assertArgWith(self, call_args, '--jobid', '3') + assertArgWith(self, call_args, '--crs', 'gerrit') + assertArgWith(self, call_args, '--cis', 'buildbucket') + + @mock.patch.object(skia_gold_session.SkiaGoldSession, '_RunCmdForRcAndOutput') + def test_commandTryjobArgsMissing(self, cmd_mock): + cmd_mock.return_value = (None, None) + args = createSkiaGoldArgs(git_revision='a') + sgp = skia_gold_properties.SkiaGoldProperties(args) + session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, None, + None, None) + session.Initialize() + call_args = cmd_mock.call_args[0][0] + self.assertNotIn('--issue', call_args) + self.assertNotIn('--patchset', call_args) + self.assertNotIn('--jobid', call_args) + self.assertNotIn('--crs', call_args) + self.assertNotIn('--cis', call_args) + + +class SkiaGoldSessionCompareTest(fake_filesystem_unittest.TestCase): + """Tests the functionality of SkiaGoldSession.Compare.""" + + def setUp(self): + self.setUpPyfakefs() + self._working_dir = tempfile.mkdtemp() + + @mock.patch.object(skia_gold_session.SkiaGoldSession, '_RunCmdForRcAndOutput') + def test_commandOutputReturned(self, cmd_mock): + cmd_mock.return_value = (1, 'Something bad :(') + args = createSkiaGoldArgs(git_revision='a') + sgp = skia_gold_properties.SkiaGoldProperties(args) + session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, None, + None, None) + rc, stdout = session.Compare(None, None) + self.assertEqual(cmd_mock.call_count, 1) + self.assertEqual(rc, 1) + self.assertEqual(stdout, 'Something bad :(') + + @mock.patch.object(skia_gold_session.SkiaGoldSession, '_RunCmdForRcAndOutput') + def test_bypassSkiaGoldFunctionality(self, cmd_mock): + cmd_mock.return_value = (None, None) + args = createSkiaGoldArgs(git_revision='a', + bypass_skia_gold_functionality=True) + sgp = skia_gold_properties.SkiaGoldProperties(args) + session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, None, + None, None) + rc, _ = session.Compare(None, None) + self.assertEqual(rc, 0) + cmd_mock.assert_not_called() + + @mock.patch.object(skia_gold_session.SkiaGoldSession, '_RunCmdForRcAndOutput') + def test_commandWithLocalPixelTestsTrue(self, cmd_mock): + cmd_mock.return_value = (None, None) + args = createSkiaGoldArgs(git_revision='a', local_pixel_tests=True) + sgp = skia_gold_properties.SkiaGoldProperties(args) + session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, None, + None, None) + session.Compare(None, None) + self.assertIn('--dryrun', cmd_mock.call_args[0][0]) + + @mock.patch.object(skia_gold_session.SkiaGoldSession, '_RunCmdForRcAndOutput') + def test_commandWithLocalPixelTestsFalse(self, cmd_mock): + cmd_mock.return_value = (None, None) + args = createSkiaGoldArgs(git_revision='a', local_pixel_tests=False) + sgp = skia_gold_properties.SkiaGoldProperties(args) + session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, None, + None, None) + session.Compare(None, None) + self.assertNotIn('--dryrun', cmd_mock.call_args[0][0]) + + @mock.patch.object(skia_gold_session.SkiaGoldSession, '_RunCmdForRcAndOutput') + def test_commandCommonArgs(self, cmd_mock): + cmd_mock.return_value = (None, None) + args = createSkiaGoldArgs(git_revision='a') + sgp = skia_gold_properties.SkiaGoldProperties(args) + session = skia_gold_session.SkiaGoldSession(self._working_dir, + sgp, + 'keys_file', + 'corpus', + instance='instance') + session.Compare('name', 'png_file') + call_args = cmd_mock.call_args[0][0] + self.assertIn('imgtest', call_args) + self.assertIn('add', call_args) + assertArgWith(self, call_args, '--test-name', 'name') + assertArgWith(self, call_args, '--png-file', 'png_file') + assertArgWith(self, call_args, '--work-dir', self._working_dir) + + @mock.patch.object(skia_gold_session.SkiaGoldSession, '_RunCmdForRcAndOutput') + def test_noLinkOnSuccess(self, cmd_mock): + cmd_mock.return_value = (0, None) + args = createSkiaGoldArgs(git_revision='a') + sgp = skia_gold_properties.SkiaGoldProperties(args) + session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, + 'keys_file', None, None) + rc, _ = session.Compare('name', 'png_file') + self.assertEqual(rc, 0) + self.assertEqual(session._comparison_results['name'].triage_link, None) + self.assertNotEqual( + session._comparison_results['name'].triage_link_omission_reason, None) + + @mock.patch.object(skia_gold_session.SkiaGoldSession, '_RunCmdForRcAndOutput') + def test_clLinkOnTrybot(self, cmd_mock): + cmd_mock.return_value = (1, None) + args = createSkiaGoldArgs(git_revision='a', + gerrit_issue=1, + gerrit_patchset=2, + buildbucket_id=3) + sgp = skia_gold_properties.SkiaGoldProperties(args) + session = skia_gold_session.SkiaGoldSession(self._working_dir, + sgp, + 'keys_file', + None, + instance='instance') + rc, _ = session.Compare('name', 'png_file') + self.assertEqual(rc, 1) + self.assertNotEqual(session._comparison_results['name'].triage_link, None) + self.assertEqual(session._comparison_results['name'].triage_link, + 'https://instance-gold.skia.org/cl/gerrit/1') + self.assertEqual( + session._comparison_results['name'].triage_link_omission_reason, None) + + @mock.patch.object(skia_gold_session.SkiaGoldSession, '_RunCmdForRcAndOutput') + def test_individualLinkOnCi(self, cmd_mock): + args = createSkiaGoldArgs(git_revision='a') + sgp = skia_gold_properties.SkiaGoldProperties(args) + session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, + 'keys_file', None, None) + + def WriteTriageLinkFile(_): + with open(session._triage_link_file, 'w') as f: + f.write('foobar') + return (1, None) + + cmd_mock.side_effect = WriteTriageLinkFile + rc, _ = session.Compare('name', 'png_file') + self.assertEqual(rc, 1) + self.assertNotEqual(session._comparison_results['name'].triage_link, None) + self.assertEqual(session._comparison_results['name'].triage_link, 'foobar') + self.assertEqual( + session._comparison_results['name'].triage_link_omission_reason, None) + + @mock.patch.object(skia_gold_session.SkiaGoldSession, '_RunCmdForRcAndOutput') + def test_validOmissionOnIoError(self, cmd_mock): + cmd_mock.return_value = (1, None) + args = createSkiaGoldArgs(git_revision='a') + sgp = skia_gold_properties.SkiaGoldProperties(args) + session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, + 'keys_file', None, None) + + def DeleteTriageLinkFile(_): + os.remove(session._triage_link_file) + return (1, None) + + cmd_mock.side_effect = DeleteTriageLinkFile + rc, _ = session.Compare('name', 'png_file') + self.assertEqual(rc, 1) + self.assertEqual(session._comparison_results['name'].triage_link, None) + self.assertNotEqual( + session._comparison_results['name'].triage_link_omission_reason, None) + self.assertIn( + 'Failed to read', + session._comparison_results['name'].triage_link_omission_reason) + + +class SkiaGoldSessionDiffTest(fake_filesystem_unittest.TestCase): + """Tests the functionality of SkiaGoldSession.Diff.""" + + def setUp(self): + self.setUpPyfakefs() + self._working_dir = tempfile.mkdtemp() + + @mock.patch.object(skia_gold_session.SkiaGoldSession, '_StoreDiffLinks') + @mock.patch.object(skia_gold_session.SkiaGoldSession, '_RunCmdForRcAndOutput') + def test_commandOutputReturned(self, cmd_mock, _): + cmd_mock.return_value = (1, 'Something bad :(') + args = createSkiaGoldArgs(git_revision='a', local_pixel_tests=False) + sgp = skia_gold_properties.SkiaGoldProperties(args) + session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, None, + None, None) + rc, stdout = session.Diff(None, None, None) + self.assertEqual(cmd_mock.call_count, 1) + self.assertEqual(rc, 1) + self.assertEqual(stdout, 'Something bad :(') + + @mock.patch.object(skia_gold_session.SkiaGoldSession, '_RunCmdForRcAndOutput') + def test_bypassSkiaGoldFunctionality(self, cmd_mock): + cmd_mock.return_value = (None, None) + args = createSkiaGoldArgs(git_revision='a', + bypass_skia_gold_functionality=True) + sgp = skia_gold_properties.SkiaGoldProperties(args) + session = skia_gold_session.SkiaGoldSession(self._working_dir, sgp, None, + None, None) + with self.assertRaises(RuntimeError): + session.Diff(None, None, None) + + +class SkiaGoldSessionTriageLinkOmissionTest(fake_filesystem_unittest.TestCase): + """Tests the functionality of SkiaGoldSession.GetTriageLinkOmissionReason.""" + + def setUp(self): + self.setUpPyfakefs() + self._working_dir = tempfile.mkdtemp() + + def _CreateSession(self): + session = skia_gold_session.SkiaGoldSession(self._working_dir, None, None, + None, None) + session._comparison_results = { + 'foo': skia_gold_session.SkiaGoldSession.ComparisonResults(), + } + return session + + def test_noComparison(self): + session = self._CreateSession() + session._comparison_results = {} + reason = session.GetTriageLinkOmissionReason('foo') + self.assertEqual(reason, 'No image comparison performed for foo') + + def test_validReason(self): + session = self._CreateSession() + session._comparison_results['foo'].triage_link_omission_reason = 'bar' + reason = session.GetTriageLinkOmissionReason('foo') + self.assertEqual(reason, 'bar') + + def test_onlyLocal(self): + session = self._CreateSession() + session._comparison_results['foo'].local_diff_given_image = 'bar' + reason = session.GetTriageLinkOmissionReason('foo') + self.assertEqual(reason, 'Gold only used to do a local image diff') + + def test_onlyWithoutTriageLink(self): + session = self._CreateSession() + session._comparison_results['foo'].triage_link = 'bar' + with self.assertRaises(AssertionError): + session.GetTriageLinkOmissionReason('foo') + + def test_resultsShouldNotExist(self): + session = self._CreateSession() + with self.assertRaises(RuntimeError): + session.GetTriageLinkOmissionReason('foo') + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff --git a/chromium/build/skia_gold_common/unittest_utils.py b/chromium/build/skia_gold_common/unittest_utils.py new file mode 100644 index 00000000000..e57498442e5 --- /dev/null +++ b/chromium/build/skia_gold_common/unittest_utils.py @@ -0,0 +1,28 @@ +# Copyright 2020 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. +"""Utility methods for Skia Gold functionality unittests.""" + +import collections + +_SkiaGoldArgs = collections.namedtuple('_SkiaGoldArgs', [ + 'local_pixel_tests', + 'no_luci_auth', + 'git_revision', + 'gerrit_issue', + 'gerrit_patchset', + 'buildbucket_id', + 'bypass_skia_gold_functionality', +]) + + +def createSkiaGoldArgs(local_pixel_tests=None, + no_luci_auth=None, + git_revision=None, + gerrit_issue=None, + gerrit_patchset=None, + buildbucket_id=None, + bypass_skia_gold_functionality=None): + return _SkiaGoldArgs(local_pixel_tests, no_luci_auth, git_revision, + gerrit_issue, gerrit_patchset, buildbucket_id, + bypass_skia_gold_functionality) |