diff options
Diffstat (limited to 'Tools/Scripts/webkitpy')
21 files changed, 193 insertions, 235 deletions
diff --git a/Tools/Scripts/webkitpy/common/config/committers.py b/Tools/Scripts/webkitpy/common/config/committers.py index 4c2216289..693b47ed4 100644 --- a/Tools/Scripts/webkitpy/common/config/committers.py +++ b/Tools/Scripts/webkitpy/common/config/committers.py @@ -239,6 +239,7 @@ committers_unable_to_review = [ Committer("Dmitry Lomov", ["dslomov@google.com", "dslomov@chromium.org"], "dslomov"), Committer("Dominic Cooney", ["dominicc@chromium.org", "dominicc@google.com"], "dominicc"), Committer("Dominic Mazzoni", ["dmazzoni@google.com", "dmazzoni@chromium.org"], "dmazzoni"), + Committer(u"Dominik R\u00f6ttsches", ["dominik.rottsches@intel.com", "d-r@roettsches.de"], "drott"), Committer("Drew Wilson", "atwilson@chromium.org", "atwilson"), Committer("Eli Fidler", ["eli@staikos.net", "efidler@rim.com"], "efidler"), Committer("Elliot Poger", "epoger@chromium.org", "epoger"), diff --git a/Tools/Scripts/webkitpy/common/net/resultsjsonparser.py b/Tools/Scripts/webkitpy/common/net/resultsjsonparser.py index 99e8528de..d30ad55aa 100644 --- a/Tools/Scripts/webkitpy/common/net/resultsjsonparser.py +++ b/Tools/Scripts/webkitpy/common/net/resultsjsonparser.py @@ -34,6 +34,7 @@ from webkitpy.common.system.deprecated_logging import log # FIXME: common should never import from new-run-webkit-tests, one of these files needs to move. from webkitpy.layout_tests.layout_package import json_results_generator from webkitpy.layout_tests.models import test_expectations, test_results, test_failures +from webkitpy.layout_tests.models.test_expectations import TestExpectations # These are helper functions for navigating the results json structure. @@ -72,15 +73,15 @@ class JSONTestResult(object): expected_results = self._expected_as_tokens() # FIXME: We should only call remove_pixel_failures when this JSONResult # came from a test run without pixel tests! - if not test_expectations.has_pixel_failures(actual_results): - expected_results = test_expectations.remove_pixel_failures(expected_results) + if not TestExpectations.has_pixel_failures(actual_results): + expected_results = TestExpectations.remove_pixel_failures(expected_results) for actual_result in actual_results: - if not test_expectations.result_was_expected(actual_result, expected_results, False, False): + if not TestExpectations.result_was_expected(actual_result, expected_results, False, False): return False return True def _tokenize(self, results_string): - tokens = map(test_expectations.TestExpectations.expectation_from_string, results_string.split(' ')) + tokens = map(TestExpectations.expectation_from_string, results_string.split(' ')) if None in tokens: log("Unrecognized result in %s" % results_string) return set(tokens) diff --git a/Tools/Scripts/webkitpy/common/system/platforminfo.py b/Tools/Scripts/webkitpy/common/system/platforminfo.py index 74cff5412..a9717cc84 100644 --- a/Tools/Scripts/webkitpy/common/system/platforminfo.py +++ b/Tools/Scripts/webkitpy/common/system/platforminfo.py @@ -103,6 +103,7 @@ class PlatformInfo(object): '5': 'leopard', '6': 'snowleopard', '7': 'lion', + '8': 'mountainlion', } assert release_version >= min(version_strings.keys()) return version_strings.get(release_version, 'future') diff --git a/Tools/Scripts/webkitpy/common/system/platforminfo_unittest.py b/Tools/Scripts/webkitpy/common/system/platforminfo_unittest.py index 8fc961b08..445ef5f7d 100644 --- a/Tools/Scripts/webkitpy/common/system/platforminfo_unittest.py +++ b/Tools/Scripts/webkitpy/common/system/platforminfo_unittest.py @@ -133,7 +133,8 @@ class TestPlatformInfo(unittest.TestCase): self.assertEquals(self.make_info(fake_sys('darwin'), fake_platform('10.5.1')).os_version, 'leopard') self.assertEquals(self.make_info(fake_sys('darwin'), fake_platform('10.6.1')).os_version, 'snowleopard') self.assertEquals(self.make_info(fake_sys('darwin'), fake_platform('10.7.1')).os_version, 'lion') - self.assertEquals(self.make_info(fake_sys('darwin'), fake_platform('10.8.0')).os_version, 'future') + self.assertEquals(self.make_info(fake_sys('darwin'), fake_platform('10.8.1')).os_version, 'mountainlion') + self.assertEquals(self.make_info(fake_sys('darwin'), fake_platform('10.9.0')).os_version, 'future') self.assertEquals(self.make_info(fake_sys('linux2')).os_version, 'lucid') diff --git a/Tools/Scripts/webkitpy/layout_tests/controllers/single_test_runner.py b/Tools/Scripts/webkitpy/layout_tests/controllers/single_test_runner.py index 7379d97c3..ae3470a5d 100644 --- a/Tools/Scripts/webkitpy/layout_tests/controllers/single_test_runner.py +++ b/Tools/Scripts/webkitpy/layout_tests/controllers/single_test_runner.py @@ -110,7 +110,7 @@ class SingleTestRunner(object): expected_driver_output.strip_metrics() driver_output.strip_metrics() - test_result = self._compare_output(driver_output, expected_driver_output) + test_result = self._compare_output(expected_driver_output, driver_output) if self._options.new_test_results: self._add_missing_baselines(test_result, driver_output) test_result_writer.write_test_result(self._filesystem, self._port, self._test_name, driver_output, expected_driver_output, test_result.failures) @@ -209,7 +209,7 @@ class SingleTestRunner(object): _log.debug(" %s" % line) return failures - def _compare_output(self, driver_output, expected_driver_output): + def _compare_output(self, expected_driver_output, driver_output): failures = [] failures.extend(self._handle_error(driver_output)) @@ -218,26 +218,26 @@ class SingleTestRunner(object): # In case of timeouts, we continue since we still want to see the text and image output. return TestResult(self._test_name, failures, driver_output.test_time, driver_output.has_stderr()) - failures.extend(self._compare_text(driver_output.text, expected_driver_output.text)) - failures.extend(self._compare_audio(driver_output.audio, expected_driver_output.audio)) + failures.extend(self._compare_text(expected_driver_output.text, driver_output.text)) + failures.extend(self._compare_audio(expected_driver_output.audio, driver_output.audio)) if self._should_run_pixel_test: - failures.extend(self._compare_image(driver_output, expected_driver_output)) + failures.extend(self._compare_image(expected_driver_output, driver_output)) return TestResult(self._test_name, failures, driver_output.test_time, driver_output.has_stderr()) - def _compare_text(self, actual_text, expected_text): + def _compare_text(self, expected_text, actual_text): failures = [] if (expected_text and actual_text and # Assuming expected_text is already normalized. - self._port.do_text_results_differ(self._get_normalized_output_text(actual_text), expected_text)): + self._port.do_text_results_differ(expected_text, self._get_normalized_output_text(actual_text))): failures.append(test_failures.FailureTextMismatch()) elif actual_text and not expected_text: failures.append(test_failures.FailureMissingResult()) return failures - def _compare_audio(self, actual_audio, expected_audio): + def _compare_audio(self, expected_audio, actual_audio): failures = [] if (expected_audio and actual_audio and - self._port.do_audio_results_differ(actual_audio, expected_audio)): + self._port.do_audio_results_differ(expected_audio, actual_audio)): failures.append(test_failures.FailureAudioMismatch()) elif actual_audio and not expected_audio: failures.append(test_failures.FailureMissingAudio()) @@ -254,7 +254,7 @@ class SingleTestRunner(object): # FIXME: This function also creates the image diff. Maybe that work should # be handled elsewhere? - def _compare_image(self, driver_output, expected_driver_output): + def _compare_image(self, expected_driver_output, driver_output): failures = [] # If we didn't produce a hash file, this test must be text-only. if driver_output.image_hash is None: @@ -264,12 +264,9 @@ class SingleTestRunner(object): elif not expected_driver_output.image_hash: failures.append(test_failures.FailureMissingImageHash()) elif driver_output.image_hash != expected_driver_output.image_hash: - diff_result = self._port.diff_image(driver_output.image, expected_driver_output.image) + diff_result = self._port.diff_image(expected_driver_output.image, driver_output.image) err_str = diff_result[2] - # FIXME: see https://bugs.webkit.org/show_bug.cgi?id=94277 and - # https://bugs.webkit.org/show_bug.cgi?id=81962; ImageDiff doesn't - # seem to be working with WTR properly and tons of tests are failing. - if err_str and not self._options.webkit_test_runner: + if err_str: _log.warning(' %s : %s' % (self._test_name, err_str)) failures.append(test_failures.FailureImageHashMismatch()) driver_output.error = (driver_output.error or '') + err_str @@ -279,7 +276,7 @@ class SingleTestRunner(object): failures.append(test_failures.FailureImageHashMismatch(diff_result[1])) else: # See https://bugs.webkit.org/show_bug.cgi?id=69444 for why this isn't a full failure. - _log.warning(' %s -> pixel hash failed (but pixel test still passes)' % self._test_name) + _log.warning(' %s -> pixel hash failed (but diff passed)' % self._test_name) return failures def _run_reftest(self): @@ -298,7 +295,7 @@ class SingleTestRunner(object): for expectation, reference_filename in putAllMismatchBeforeMatch(self._reference_files): reference_test_name = self._port.relative_test_filename(reference_filename) reference_output = self._driver.run_test(DriverInput(reference_test_name, self._timeout, test_output.image_hash, should_run_pixel_test=True), self._stop_when_done) - test_result = self._compare_output_with_reference(test_output, reference_output, reference_filename, expectation == '!=') + test_result = self._compare_output_with_reference(reference_output, test_output, reference_filename, expectation == '!=') if (expectation == '!=' and test_result.failures) or (expectation == '==' and not test_result.failures): break @@ -308,31 +305,33 @@ class SingleTestRunner(object): test_result_writer.write_test_result(self._filesystem, self._port, self._test_name, test_output, reference_output, test_result.failures) return TestResult(self._test_name, test_result.failures, total_test_time + test_result.test_run_time, test_result.has_stderr) - def _compare_output_with_reference(self, driver_output1, driver_output2, reference_filename, mismatch): - total_test_time = driver_output1.test_time + driver_output2.test_time - has_stderr = driver_output1.has_stderr() or driver_output2.has_stderr() + def _compare_output_with_reference(self, reference_driver_output, actual_driver_output, reference_filename, mismatch): + total_test_time = reference_driver_output.test_time + actual_driver_output.test_time + has_stderr = reference_driver_output.has_stderr() or actual_driver_output.has_stderr() failures = [] - failures.extend(self._handle_error(driver_output1)) + failures.extend(self._handle_error(actual_driver_output)) if failures: # Don't continue any more if we already have crash or timeout. return TestResult(self._test_name, failures, total_test_time, has_stderr) - failures.extend(self._handle_error(driver_output2, reference_filename=reference_filename)) + failures.extend(self._handle_error(reference_driver_output, reference_filename=reference_filename)) if failures: return TestResult(self._test_name, failures, total_test_time, has_stderr) - if self._options.webkit_test_runner and not self._options.pixel_tests: - # don't check pixel results for WTR/WK2; they're broken. - return TestResult(self._test_name, failures, total_test_time, has_stderr) - - if not driver_output1.image_hash and not driver_output2.image_hash: + if not reference_driver_output.image_hash and not actual_driver_output.image_hash: failures.append(test_failures.FailureReftestNoImagesGenerated(reference_filename)) elif mismatch: - if driver_output1.image_hash == driver_output2.image_hash: - failures.append(test_failures.FailureReftestMismatchDidNotOccur(reference_filename)) - elif driver_output1.image_hash != driver_output2.image_hash: - failures.append(test_failures.FailureReftestMismatch(reference_filename)) + if reference_driver_output.image_hash == actual_driver_output.image_hash: + diff_result = self._port.diff_image(reference_driver_output.image, actual_driver_output.image) + if not diff_result[0]: + failures.append(test_failures.FailureReftestMismatchDidNotOccur(reference_filename)) + else: + _log.warning(" %s -> ref test hashes matched but diff failed" % self._test_name) - # recompute in case we added to stderr during diff_image - has_stderr = driver_output1.has_stderr() or driver_output2.has_stderr() + elif reference_driver_output.image_hash != actual_driver_output.image_hash: + diff_result = self._port.diff_image(reference_driver_output.image, actual_driver_output.image) + if diff_result[0]: + failures.append(test_failures.FailureReftestMismatch(reference_filename)) + else: + _log.warning(" %s -> ref test hashes didn't match but diff passed" % self._test_name) return TestResult(self._test_name, failures, total_test_time, has_stderr) diff --git a/Tools/Scripts/webkitpy/layout_tests/controllers/test_result_writer.py b/Tools/Scripts/webkitpy/layout_tests/controllers/test_result_writer.py index 67f42e3b6..be178ab32 100644 --- a/Tools/Scripts/webkitpy/layout_tests/controllers/test_result_writer.py +++ b/Tools/Scripts/webkitpy/layout_tests/controllers/test_result_writer.py @@ -69,12 +69,12 @@ def write_test_result(filesystem, port, test_name, driver_output, # FIXME: This work should be done earlier in the pipeline (e.g., when we compare images for non-ref tests). # FIXME: We should always have 2 images here. if driver_output.image and expected_driver_output.image: - diff_image, diff_percent, err_str = port.diff_image(driver_output.image, expected_driver_output.image, tolerance=0) + diff_image, diff_percent, err_str = port.diff_image(expected_driver_output.image, driver_output.image, tolerance=0) if diff_image: writer.write_image_diff_files(diff_image) failure.diff_percent = diff_percent else: - _log.warn('Can not get image diff. ImageDiff program might not work correctly.') + _log.warn('ref test mismatch did not produce an image diff.') writer.write_reftest(failure.reference_filename) elif isinstance(failure, test_failures.FailureReftestMismatchDidNotOccur): writer.write_image_files(driver_output.image, expected_image=None) diff --git a/Tools/Scripts/webkitpy/layout_tests/models/test_expectations.py b/Tools/Scripts/webkitpy/layout_tests/models/test_expectations.py index d3015dbdd..6b949397b 100644 --- a/Tools/Scripts/webkitpy/layout_tests/models/test_expectations.py +++ b/Tools/Scripts/webkitpy/layout_tests/models/test_expectations.py @@ -45,57 +45,10 @@ _log = logging.getLogger(__name__) (PASS, FAIL, TEXT, IMAGE, IMAGE_PLUS_TEXT, AUDIO, TIMEOUT, CRASH, SKIP, WONTFIX, SLOW, REBASELINE, MISSING, FLAKY, NOW, NONE) = range(16) - -def result_was_expected(result, expected_results, test_needs_rebaselining, test_is_skipped): - """Returns whether we got a result we were expecting. - Args: - result: actual result of a test execution - expected_results: set of results listed in test_expectations - test_needs_rebaselining: whether test was marked as REBASELINE - test_is_skipped: whether test was marked as SKIP""" - if result in expected_results: - return True - if result == MISSING and test_needs_rebaselining: - return True - if result == SKIP and test_is_skipped: - return True - return False - - -def remove_pixel_failures(expected_results): - """Returns a copy of the expected results for a test, except that we - drop any pixel failures and return the remaining expectations. For example, - if we're not running pixel tests, then tests expected to fail as IMAGE - will PASS.""" - expected_results = expected_results.copy() - if IMAGE in expected_results: - expected_results.remove(IMAGE) - expected_results.add(PASS) - if IMAGE_PLUS_TEXT in expected_results: - expected_results.remove(IMAGE_PLUS_TEXT) - expected_results.add(TEXT) - return expected_results - - -def has_pixel_failures(actual_results): - return IMAGE in actual_results or IMAGE_PLUS_TEXT in actual_results - - # FIXME: Perhas these two routines should be part of the Port instead? BASELINE_SUFFIX_LIST = ('png', 'wav', 'txt') -def suffixes_for_expectations(expectations): - suffixes = set() - if expectations.intersection(set([TEXT, IMAGE_PLUS_TEXT])): - suffixes.add('txt') - if expectations.intersection(set([IMAGE, IMAGE_PLUS_TEXT])): - suffixes.add('png') - if AUDIO in expectations: - suffixes.add('wav') - return set(suffixes) - - class ParseError(Exception): def __init__(self, warnings): super(ParseError, self).__init__() @@ -235,10 +188,10 @@ class TestExpectationParser(object): if not expectation_line.name: return - expectation_line.is_file = self._port.test_isfile(expectation_line.name) - if not expectation_line.is_file and self._check_path_does_not_exist(expectation_line): + if not self._check_test_exists(expectation_line): return + expectation_line.is_file = self._port.test_isfile(expectation_line.name) if expectation_line.is_file: expectation_line.path = expectation_line.name else: @@ -292,18 +245,17 @@ class TestExpectationParser(object): result.add(expectation) expectation_line.parsed_expectations = result - def _check_path_does_not_exist(self, expectation_line): + def _check_test_exists(self, expectation_line): # WebKit's way of skipping tests is to add a -disabled suffix. # So we should consider the path existing if the path or the # -disabled version exists. - if (not self._port.test_exists(expectation_line.name) - and not self._port.test_exists(expectation_line.name + '-disabled')): + if not self._port.test_exists(expectation_line.name) and not self._port.test_exists(expectation_line.name + '-disabled'): # Log a warning here since you hit this case any # time you update TestExpectations without syncing # the LayoutTests directory expectation_line.warnings.append('Path does not exist.') - return True - return False + return False + return True def _collect_matching_tests(self, expectation_line): """Convert the test specification to an absolute, normalized @@ -718,6 +670,52 @@ class TestExpectations(object): assert(' ' not in string) # This only handles one expectation at a time. return cls.EXPECTATIONS.get(string.lower()) + @staticmethod + def result_was_expected(result, expected_results, test_needs_rebaselining, test_is_skipped): + """Returns whether we got a result we were expecting. + Args: + result: actual result of a test execution + expected_results: set of results listed in test_expectations + test_needs_rebaselining: whether test was marked as REBASELINE + test_is_skipped: whether test was marked as SKIP""" + if result in expected_results: + return True + if result == MISSING and test_needs_rebaselining: + return True + if result == SKIP and test_is_skipped: + return True + return False + + @staticmethod + def remove_pixel_failures(expected_results): + """Returns a copy of the expected results for a test, except that we + drop any pixel failures and return the remaining expectations. For example, + if we're not running pixel tests, then tests expected to fail as IMAGE + will PASS.""" + expected_results = expected_results.copy() + if IMAGE in expected_results: + expected_results.remove(IMAGE) + expected_results.add(PASS) + if IMAGE_PLUS_TEXT in expected_results: + expected_results.remove(IMAGE_PLUS_TEXT) + expected_results.add(TEXT) + return expected_results + + @staticmethod + def has_pixel_failures(actual_results): + return IMAGE in actual_results or IMAGE_PLUS_TEXT in actual_results + + @staticmethod + def suffixes_for_expectations(expectations): + suffixes = set() + if expectations.intersection(set([TEXT, IMAGE_PLUS_TEXT])): + suffixes.add('txt') + if expectations.intersection(set([IMAGE, IMAGE_PLUS_TEXT])): + suffixes.add('png') + if AUDIO in expectations: + suffixes.add('wav') + return set(suffixes) + def __init__(self, port, tests=None, is_lint_mode=False, include_overrides=True): self._full_test_list = tests self._test_config = port.test_configuration() @@ -788,8 +786,8 @@ class TestExpectations(object): def matches_an_expected_result(self, test, result, pixel_tests_are_enabled): expected_results = self._model.get_expectations(test) if not pixel_tests_are_enabled: - expected_results = remove_pixel_failures(expected_results) - return result_was_expected(result, + expected_results = self.remove_pixel_failures(expected_results) + return self.result_was_expected(result, expected_results, self.is_rebaselining(test), self._model.has_modifier(test, SKIP)) diff --git a/Tools/Scripts/webkitpy/layout_tests/models/test_expectations_unittest.py b/Tools/Scripts/webkitpy/layout_tests/models/test_expectations_unittest.py index 9fc118360..5abb21302 100644 --- a/Tools/Scripts/webkitpy/layout_tests/models/test_expectations_unittest.py +++ b/Tools/Scripts/webkitpy/layout_tests/models/test_expectations_unittest.py @@ -50,47 +50,6 @@ class MockBugManager(object): return "BUG_NEWLY_CREATED" -class FunctionsTest(unittest.TestCase): - def test_result_was_expected(self): - # test basics - self.assertEquals(result_was_expected(PASS, set([PASS]), - False, False), True) - self.assertEquals(result_was_expected(TEXT, set([PASS]), - False, False), False) - - # test handling of SKIPped tests and results - self.assertEquals(result_was_expected(SKIP, set([CRASH]), - False, True), True) - self.assertEquals(result_was_expected(SKIP, set([CRASH]), - False, False), False) - - # test handling of MISSING results and the REBASELINE modifier - self.assertEquals(result_was_expected(MISSING, set([PASS]), - True, False), True) - self.assertEquals(result_was_expected(MISSING, set([PASS]), - False, False), False) - - def test_remove_pixel_failures(self): - self.assertEquals(remove_pixel_failures(set([TEXT])), - set([TEXT])) - self.assertEquals(remove_pixel_failures(set([PASS])), - set([PASS])) - self.assertEquals(remove_pixel_failures(set([IMAGE])), - set([PASS])) - self.assertEquals(remove_pixel_failures(set([IMAGE_PLUS_TEXT])), - set([TEXT])) - self.assertEquals(remove_pixel_failures(set([PASS, IMAGE, CRASH])), - set([PASS, CRASH])) - - def test_suffixes_for_expectations(self): - self.assertEquals(suffixes_for_expectations(set([TEXT])), set(['txt'])) - self.assertEquals(suffixes_for_expectations(set([IMAGE_PLUS_TEXT])), set(['txt', 'png'])) - self.assertEquals(suffixes_for_expectations(set([IMAGE])), set(['png'])) - self.assertEquals(suffixes_for_expectations(set([AUDIO])), set(['wav'])) - self.assertEquals(suffixes_for_expectations(set([TEXT, IMAGE, CRASH])), set(['txt', 'png'])) - self.assertEquals(suffixes_for_expectations(set()), set()) - - class Base(unittest.TestCase): # Note that all of these tests are written assuming the configuration # being tested is Windows XP, Release build. @@ -137,6 +96,34 @@ BUG_TEST WONTFIX MAC : failures/expected/image.html = IMAGE def assert_bad_expectations(self, expectations, overrides=None): self.assertRaises(ParseError, self.parse_exp, expectations, is_lint_mode=True, overrides=overrides) + def test_result_was_expected(self): + # test basics + self.assertEquals(TestExpectations.result_was_expected(PASS, set([PASS]), test_needs_rebaselining=False, test_is_skipped=False), True) + self.assertEquals(TestExpectations.result_was_expected(TEXT, set([PASS]), test_needs_rebaselining=False, test_is_skipped=False), False) + + # test handling of SKIPped tests and results + self.assertEquals(TestExpectations.result_was_expected(SKIP, set([CRASH]), test_needs_rebaselining=False, test_is_skipped=True), True) + self.assertEquals(TestExpectations.result_was_expected(SKIP, set([CRASH]), test_needs_rebaselining=False, test_is_skipped=False), False) + + # test handling of MISSING results and the REBASELINE modifier + self.assertEquals(TestExpectations.result_was_expected(MISSING, set([PASS]), test_needs_rebaselining=True, test_is_skipped=False), True) + self.assertEquals(TestExpectations.result_was_expected(MISSING, set([PASS]), test_needs_rebaselining=False, test_is_skipped=False), False) + + def test_remove_pixel_failures(self): + self.assertEquals(TestExpectations.remove_pixel_failures(set([TEXT])), set([TEXT])) + self.assertEquals(TestExpectations.remove_pixel_failures(set([PASS])), set([PASS])) + self.assertEquals(TestExpectations.remove_pixel_failures(set([IMAGE])), set([PASS])) + self.assertEquals(TestExpectations.remove_pixel_failures(set([IMAGE_PLUS_TEXT])), set([TEXT])) + self.assertEquals(TestExpectations.remove_pixel_failures(set([PASS, IMAGE, CRASH])), set([PASS, CRASH])) + + def test_suffixes_for_expectations(self): + self.assertEquals(TestExpectations.suffixes_for_expectations(set([TEXT])), set(['txt'])) + self.assertEquals(TestExpectations.suffixes_for_expectations(set([IMAGE_PLUS_TEXT])), set(['txt', 'png'])) + self.assertEquals(TestExpectations.suffixes_for_expectations(set([IMAGE])), set(['png'])) + self.assertEquals(TestExpectations.suffixes_for_expectations(set([AUDIO])), set(['wav'])) + self.assertEquals(TestExpectations.suffixes_for_expectations(set([TEXT, IMAGE, CRASH])), set(['txt', 'png'])) + self.assertEquals(TestExpectations.suffixes_for_expectations(set()), set()) + class BasicTests(Base): def test_basic(self): @@ -200,13 +187,19 @@ BUGX WONTFIX : failures/expected = IMAGE def test_parse_warning(self): try: - self.parse_exp("""FOO : failures/expected/text.html = TEXT -SKIP : failures/expected/image.html""", is_lint_mode=True) + filesystem = self._port.host.filesystem + filesystem.write_text_file(filesystem.join(self._port.layout_tests_dir(), 'disabled-test.html-disabled'), 'content') + self.get_test('disabled-test.html-disabled'), + self.parse_exp("FOO : failures/expected/text.html = TEXT\n" + "SKIP : failures/expected/image.html\n" + "BUGRNIWA : non-existent-test.html = TEXT\n" + "BUGRNIWA : disabled-test.html-disabled = IMAGE", is_lint_mode=True) self.assertFalse(True, "ParseError wasn't raised") except ParseError, e: warnings = ("expectations:1 Test lacks BUG modifier. failures/expected/text.html\n" "expectations:1 Unrecognized modifier 'foo' failures/expected/text.html\n" - "expectations:2 Missing expectations SKIP : failures/expected/image.html") + "expectations:2 Missing expectations SKIP : failures/expected/image.html\n" + "expectations:3 Path does not exist. non-existent-test.html") self.assertEqual(str(e), warnings) try: diff --git a/Tools/Scripts/webkitpy/layout_tests/port/apple.py b/Tools/Scripts/webkitpy/layout_tests/port/apple.py index 28ecc6380..58b5b64b6 100644 --- a/Tools/Scripts/webkitpy/layout_tests/port/apple.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/apple.py @@ -76,8 +76,14 @@ class ApplePort(Port): allowed_port_names = self.VERSION_FALLBACK_ORDER + [self.operating_system() + "-future"] port_name = port_name.replace('-wk2', '') - assert port_name in allowed_port_names, "%s is not in %s" % (port_name, allowed_port_names) self._version = self._strip_port_name_prefix(port_name) + assert port_name in allowed_port_names, "%s is not in %s" % (port_name, allowed_port_names) + + def _skipped_file_search_paths(self): + # We don't have a dedicated Skipped file for the most recent version of the port; + # we just use the one in platform/{mac,win} + most_recent_name = self.VERSION_FALLBACK_ORDER[-1] + return set(filter(lambda name: name != most_recent_name, super(ApplePort, self)._skipped_file_search_paths())) # FIXME: A more sophisticated version of this function should move to WebKitPort and replace all calls to name(). # This is also a misleading name, since 'mac-future' gets remapped to 'mac'. @@ -87,14 +93,7 @@ class ApplePort(Port): def _generate_all_test_configurations(self): configurations = [] for port_name in self.VERSION_FALLBACK_ORDER: - if '-' in port_name: - version = self._strip_port_name_prefix(port_name) - else: - # The version for the "base" port is currently defined as "future" - # since TestConfiguration doesn't allow None as a valid version. - version = self.FUTURE_VERSION - for build_type in self.ALL_BUILD_TYPES: for architecture in self.ARCHITECTURES: - configurations.append(TestConfiguration(version=version, architecture=architecture, build_type=build_type)) + configurations.append(TestConfiguration(version=self._strip_port_name_prefix(port_name), architecture=architecture, build_type=build_type)) return configurations diff --git a/Tools/Scripts/webkitpy/layout_tests/port/builders.py b/Tools/Scripts/webkitpy/layout_tests/port/builders.py index 21fcc9801..3d03fc7af 100644 --- a/Tools/Scripts/webkitpy/layout_tests/port/builders.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/builders.py @@ -49,9 +49,12 @@ _exact_matches = { "Webkit Mac10.7": {"port_name": "chromium-mac-lion", "specifiers": set(["lion"])}, # These builders are on build.webkit.org. - # FIXME: Remove rebaseline_override_dir once there is an Apple buildbot that corresponds to platform/mac (i.e. a Mountain Lion bot). - "Apple Lion Release WK1 (Tests)": {"port_name": "mac-lion", "specifiers": set(["lion"]), "rebaseline_override_dir": "mac"}, - "Apple Lion Debug WK1 (Tests)": {"port_name": "mac-lion", "specifiers": set(["lion", "debug"]), "rebaseline_override_dir": "mac"}, + "Apple MountainLion Release WK1 (Tests)": {"port_name": "mac-mountainlion", "specifiers": set(["mountainlion"]), "rebaseline_override_dir": "mac"}, + "Apple MountainLion Debug WK1 (Tests)": {"port_name": "mac-mountainlion", "specifiers": set(["mountainlion", "debug"]), "rebaseline_override_dir": "mac"}, + "Apple MountainLion Release WK2 (Tests)": {"port_name": "mac-mountainlion", "specifiers": set(["mountainlion", "wk2"]), "rebaseline_override_dir": "mac"}, + "Apple MountainLion Debug WK2 (Tests)": {"port_name": "mac-mountainlion", "specifiers": set(["mountainlion", "wk2", "debug"]), "rebaseline_override_dir": "mac"}, + "Apple Lion Release WK1 (Tests)": {"port_name": "mac-lion", "specifiers": set(["lion"])}, + "Apple Lion Debug WK1 (Tests)": {"port_name": "mac-lion", "specifiers": set(["lion", "debug"])}, "Apple Lion Release WK2 (Tests)": {"port_name": "mac-lion", "specifiers": set(["lion", "wk2"])}, "Apple Lion Debug WK2 (Tests)": {"port_name": "mac-lion", "specifiers": set(["lion", "wk2", "debug"])}, diff --git a/Tools/Scripts/webkitpy/layout_tests/port/chromium.py b/Tools/Scripts/webkitpy/layout_tests/port/chromium.py index 38ce4b198..2ff21cc5d 100755 --- a/Tools/Scripts/webkitpy/layout_tests/port/chromium.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/chromium.py @@ -216,6 +216,7 @@ class ChromiumPort(Port): native_diff_filename = self._convert_path(diff_filename) executable = self._path_to_image_diff() + # Note that although we are handed 'old', 'new', image_diff wants 'new', 'old'. comand = [executable, '--diff', native_actual_filename, native_expected_filename, native_diff_filename] result = None diff --git a/Tools/Scripts/webkitpy/layout_tests/port/chromium_mac.py b/Tools/Scripts/webkitpy/layout_tests/port/chromium_mac.py index baea0639b..554feca23 100644 --- a/Tools/Scripts/webkitpy/layout_tests/port/chromium_mac.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/chromium_mac.py @@ -39,7 +39,7 @@ _log = logging.getLogger(__name__) class ChromiumMacPort(chromium.ChromiumPort): - SUPPORTED_OS_VERSIONS = ('snowleopard', 'lion', 'future') + SUPPORTED_OS_VERSIONS = ('snowleopard', 'lion', 'mountainlion', 'future') port_name = 'chromium-mac' FALLBACK_PATHS = { @@ -54,6 +54,11 @@ class ChromiumMacPort(chromium.ChromiumPort): 'chromium', 'mac', ], + 'mountainlion': [ # FIXME: we don't treat ML different from Lion yet. + 'chromium-mac', + 'chromium', + 'mac', + ], 'future': [ 'chromium-mac', 'chromium', diff --git a/Tools/Scripts/webkitpy/layout_tests/port/factory_unittest.py b/Tools/Scripts/webkitpy/layout_tests/port/factory_unittest.py index 776433f80..cba5d2b39 100644 --- a/Tools/Scripts/webkitpy/layout_tests/port/factory_unittest.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/factory_unittest.py @@ -56,8 +56,8 @@ class FactoryTest(unittest.TestCase): self.assertTrue(isinstance(port, cls)) def test_mac(self): - self.assert_port(port_name='mac-leopard', cls=mac.MacPort) - self.assert_port(port_name='mac-leopard-wk2', cls=mac.MacPort) + self.assert_port(port_name='mac-lion', cls=mac.MacPort) + self.assert_port(port_name='mac-lion-wk2', cls=mac.MacPort) self.assert_port(port_name='mac', os_name='mac', os_version='lion', cls=mac.MacPort) self.assert_port(port_name=None, os_name='mac', os_version='lion', cls=mac.MacPort) diff --git a/Tools/Scripts/webkitpy/layout_tests/port/image_diff.py b/Tools/Scripts/webkitpy/layout_tests/port/image_diff.py index 08202ace5..72d061ffc 100644 --- a/Tools/Scripts/webkitpy/layout_tests/port/image_diff.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/image_diff.py @@ -57,6 +57,7 @@ class ImageDiffer(object): if not self._process: self._start(tolerance) + # Note that although we are handed 'old', 'new', ImageDiff wants 'new', 'old'. self._process.write('Content-Length: %d\n%sContent-Length: %d\n%s' % ( len(actual_contents), actual_contents, len(expected_contents), expected_contents)) diff --git a/Tools/Scripts/webkitpy/layout_tests/port/mac.py b/Tools/Scripts/webkitpy/layout_tests/port/mac.py index d0caa5b22..deee3d1be 100644 --- a/Tools/Scripts/webkitpy/layout_tests/port/mac.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/mac.py @@ -46,9 +46,7 @@ _log = logging.getLogger(__name__) class MacPort(ApplePort): port_name = "mac" - # This is a list of all supported OS-VERSION pairs for the AppleMac port - # and the order of fallback between them. Matches ORWT. - VERSION_FALLBACK_ORDER = ["mac-leopard", "mac-snowleopard", "mac-lion", "mac"] + VERSION_FALLBACK_ORDER = ['mac-snowleopard', 'mac-lion', 'mac-mountainlion'] ARCHITECTURES = ['x86_64', 'x86'] @@ -73,25 +71,15 @@ class MacPort(ApplePort): def _build_driver_flags(self): return ['ARCHS=i386'] if self.architecture() == 'x86' else [] - def _most_recent_version(self): - # This represents the most recently-shipping version of the operating system. - return self.VERSION_FALLBACK_ORDER[-2] - def should_retry_crashes(self): # On Apple Mac, we retry crashes due to https://bugs.webkit.org/show_bug.cgi?id=82233 return True - def baseline_path(self): - if self.name() == self._most_recent_version(): - # Baselines for the most recently shiping version should go into 'mac', not 'mac-foo'. - if self.get_option('webkit_test_runner'): - return self._webkit_baseline_path('mac-wk2') - return self._webkit_baseline_path('mac') - return ApplePort.baseline_path(self) - def default_baseline_search_path(self): - fallback_index = self.VERSION_FALLBACK_ORDER.index(self._port_name_with_version()) - fallback_names = list(self.VERSION_FALLBACK_ORDER[fallback_index:]) + if self._name.endswith(self.FUTURE_VERSION): + fallback_names = [self.port_name] + else: + fallback_names = self.VERSION_FALLBACK_ORDER[self.VERSION_FALLBACK_ORDER.index(self._name):-1] + [self.port_name] if self.get_option('webkit_test_runner'): fallback_names.insert(0, self._wk2_port_name()) # Note we do not add 'wk2' here, even though it's included in _skipped_search_paths(). @@ -110,22 +98,10 @@ class MacPort(ApplePort): def operating_system(self): return 'mac' - # Belongs on a Platform object. - def is_leopard(self): - return self._version == "leopard" - - # Belongs on a Platform object. - def is_snowleopard(self): - return self._version == "snowleopard" - - # Belongs on a Platform object. - def is_lion(self): - return self._version == "lion" - def default_child_processes(self): # FIXME: The Printer isn't initialized when this is called, so using _log would just show an unitialized logger error. - if self.is_snowleopard(): + if self._version == "snowleopard": print >> sys.stderr, "Cannot run tests in parallel on Snow Leopard due to rdar://problem/10621525." return 1 diff --git a/Tools/Scripts/webkitpy/layout_tests/port/mac_unittest.py b/Tools/Scripts/webkitpy/layout_tests/port/mac_unittest.py index bc7b0df44..e75aeeff7 100644 --- a/Tools/Scripts/webkitpy/layout_tests/port/mac_unittest.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/mac_unittest.py @@ -37,8 +37,8 @@ from webkitpy.common.system.systemhost_mock import MockSystemHost class MacTest(port_testcase.PortTestCase): os_name = 'mac' - os_version = 'leopard' - port_name = 'mac-leopard' + os_version = 'lion' + port_name = 'mac-lion' port_maker = MacPort def assert_skipped_file_search_paths(self, port_name, expected_paths, use_webkit2=False): @@ -50,16 +50,15 @@ class MacTest(port_testcase.PortTestCase): self.assertEquals(self.make_port(options=MockOptions(guard_malloc=True)).default_timeout_ms(), 350000) def test_skipped_file_search_paths(self): + # We should have two skipped files - platform+version and platform; however, we don't + # have platform+version for either the most recent version or mac-future. self.assert_skipped_file_search_paths('mac-snowleopard', set(['mac-snowleopard', 'mac'])) - self.assert_skipped_file_search_paths('mac-leopard', set(['mac-leopard', 'mac'])) self.assert_skipped_file_search_paths('mac-lion', set(['mac-lion', 'mac'])) - - # Note that there is no platform/mac-future/Skipped. + self.assert_skipped_file_search_paths('mac-mountainlion', set(['mac'])) self.assert_skipped_file_search_paths('mac-future', set(['mac'])) self.assert_skipped_file_search_paths('mac-snowleopard', set(['mac-snowleopard', 'mac', 'mac-wk2', 'wk2']), use_webkit2=True) - self.assert_skipped_file_search_paths('mac-leopard', set(['mac-leopard', 'mac', 'mac-wk2', 'wk2']), use_webkit2=True) - self.assert_skipped_file_search_paths('mac-lion', set(['mac-lion', 'mac', 'mac-wk2', 'wk2']), use_webkit2=True) + self.assert_skipped_file_search_paths('mac-lion', set(['mac', 'mac-lion', 'mac-wk2', 'wk2']), use_webkit2=True) self.assert_skipped_file_search_paths('mac-future', set(['mac', 'mac-wk2', 'wk2']), use_webkit2=True) @@ -110,41 +109,22 @@ java/ self.assertTrue(port.version()) def test_versions(self): - self.assert_name('mac', 'leopard', 'mac-leopard') - self.assert_name('mac-leopard', 'tiger', 'mac-leopard') - self.assert_name('mac-leopard', 'leopard', 'mac-leopard') - self.assert_name('mac-leopard', 'snowleopard', 'mac-leopard') - + # Note: these tests don't need to be exhaustive as long as we get path coverage. self.assert_name('mac', 'snowleopard', 'mac-snowleopard') - self.assert_name('mac-snowleopard', 'tiger', 'mac-snowleopard') self.assert_name('mac-snowleopard', 'leopard', 'mac-snowleopard') - self.assert_name('mac-snowleopard', 'snowleopard', 'mac-snowleopard') + self.assert_name('mac-snowleopard', 'lion', 'mac-snowleopard') self.assert_name('mac', 'lion', 'mac-lion') self.assert_name('mac-lion', 'lion', 'mac-lion') + self.assert_name('mac', 'mountainlion', 'mac-mountainlion') + self.assert_name('mac-mountainlion', 'lion', 'mac-mountainlion') + self.assert_name('mac', 'future', 'mac-future') self.assert_name('mac-future', 'future', 'mac-future') self.assertRaises(AssertionError, self.assert_name, 'mac-tiger', 'leopard', 'mac-leopard') - - def test_is_version_methods(self): - leopard_port = self.make_port(port_name='mac-leopard') - self.assertTrue(leopard_port.is_leopard()) - self.assertFalse(leopard_port.is_snowleopard()) - self.assertFalse(leopard_port.is_lion()) - - snowleopard_port = self.make_port(port_name='mac-snowleopard') - self.assertFalse(snowleopard_port.is_leopard()) - self.assertTrue(snowleopard_port.is_snowleopard()) - self.assertFalse(snowleopard_port.is_lion()) - - lion_port = self.make_port(port_name='mac-lion') - self.assertFalse(lion_port.is_leopard()) - self.assertFalse(lion_port.is_snowleopard()) - self.assertTrue(lion_port.is_lion()) - def test_setup_environ_for_server(self): port = self.make_port(options=MockOptions(leaks=True, guard_malloc=True)) env = port.setup_environ_for_server(port.driver_name()) @@ -158,18 +138,14 @@ java/ self.assertEquals(port.baseline_search_path(), absolute_search_paths) def test_baseline_search_path(self): - self._assert_search_path('mac-leopard', 'mac-leopard', ['mac-leopard', 'mac-snowleopard', 'mac-lion', 'mac']) + # Note that we don't need total coverage here, just path coverage, since this is all data driven. self._assert_search_path('mac-snowleopard', 'mac-snowleopard', ['mac-snowleopard', 'mac-lion', 'mac']) - - # Note that mac-lion writes baselines into mac, not mac-lion! (but it will read from mac-lion) - self._assert_search_path('mac-lion', 'mac', ['mac-lion', 'mac']) - - # Note that there is no 'mac-future'; it uses the 'mac' directory as well. + self._assert_search_path('mac-lion', 'mac-lion', ['mac-lion', 'mac']) + self._assert_search_path('mac-mountainlion', 'mac', ['mac']) self._assert_search_path('mac-future', 'mac', ['mac']) - - self._assert_search_path('mac-leopard', 'mac-wk2', ['mac-wk2', 'mac-leopard', 'mac-snowleopard', 'mac-lion', 'mac'], use_webkit2=True) self._assert_search_path('mac-snowleopard', 'mac-wk2', ['mac-wk2', 'mac-snowleopard', 'mac-lion', 'mac'], use_webkit2=True) self._assert_search_path('mac-lion', 'mac-wk2', ['mac-wk2', 'mac-lion', 'mac'], use_webkit2=True) + self._assert_search_path('mac-mountainlion', 'mac-wk2', ['mac-wk2', 'mac'], use_webkit2=True) self._assert_search_path('mac-future', 'mac-wk2', ['mac-wk2', 'mac'], use_webkit2=True) def test_show_results_html_file(self): diff --git a/Tools/Scripts/webkitpy/layout_tests/port/win.py b/Tools/Scripts/webkitpy/layout_tests/port/win.py index 28465bff3..ff473fec2 100644 --- a/Tools/Scripts/webkitpy/layout_tests/port/win.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/win.py @@ -42,9 +42,7 @@ _log = logging.getLogger(__name__) class WinPort(ApplePort): port_name = "win" - # This is a list of all supported OS-VERSION pairs for the AppleWin port - # and the order of fallback between them. Matches ORWT. - VERSION_FALLBACK_ORDER = ["win-xp", "win-vista", "win-7sp0", "win"] + VERSION_FALLBACK_ORDER = ["win-xp", "win-vista", "win-7sp0", "win-win7"] ARCHITECTURES = ['x86'] @@ -63,8 +61,10 @@ class WinPort(ApplePort): return expected_text != actual_text def default_baseline_search_path(self): - fallback_index = self.VERSION_FALLBACK_ORDER.index(self._port_name_with_version()) - fallback_names = list(self.VERSION_FALLBACK_ORDER[fallback_index:]) + if self._name.endswith(self.FUTURE_VERSION): + fallback_names = [self.port_name] + else: + fallback_names = self.VERSION_FALLBACK_ORDER[self.VERSION_FALLBACK_ORDER.index(self._name):-1] + [self.port_name] # FIXME: The AppleWin port falls back to AppleMac for some results. Eventually we'll have a shared 'apple' port. if self.get_option('webkit_test_runner'): fallback_names.insert(0, 'win-wk2') diff --git a/Tools/Scripts/webkitpy/tool/commands/queues.py b/Tools/Scripts/webkitpy/tool/commands/queues.py index e8db17c7b..475e505d0 100644 --- a/Tools/Scripts/webkitpy/tool/commands/queues.py +++ b/Tools/Scripts/webkitpy/tool/commands/queues.py @@ -332,7 +332,7 @@ class CommitQueue(AbstractPatchQueue, StepSequenceErrorHandler, CommitQueueTaskD return self._layout_test_results_reader.archive(patch) def build_style(self): - return "both" + return "release" def refetch_patch(self, patch): return self._tool.bugs.fetch_attachment(patch.id()) diff --git a/Tools/Scripts/webkitpy/tool/commands/queues_unittest.py b/Tools/Scripts/webkitpy/tool/commands/queues_unittest.py index 450a912e0..345f206fc 100644 --- a/Tools/Scripts/webkitpy/tool/commands/queues_unittest.py +++ b/Tools/Scripts/webkitpy/tool/commands/queues_unittest.py @@ -324,7 +324,7 @@ MOCK run_and_throw_if_fail: ['echo', '--status-host=example.com', 'apply-attachm MOCK: update_status: commit-queue Applied patch MOCK run_and_throw_if_fail: ['echo', '--status-host=example.com', 'validate-changelog', '--non-interactive', 10000, '--port=%(port_name)s'], cwd=/mock-checkout MOCK: update_status: commit-queue ChangeLog validated -MOCK run_and_throw_if_fail: ['echo', '--status-host=example.com', 'build', '--no-clean', '--no-update', '--build-style=both', '--port=%(port_name)s'], cwd=/mock-checkout +MOCK run_and_throw_if_fail: ['echo', '--status-host=example.com', 'build', '--no-clean', '--no-update', '--build-style=release', '--port=%(port_name)s'], cwd=/mock-checkout MOCK: update_status: commit-queue Built patch MOCK run_and_throw_if_fail: ['echo', '--status-host=example.com', 'build-and-test', '--no-clean', '--no-update', '--test', '--non-interactive', '--port=%(port_name)s'], cwd=/mock-checkout MOCK: update_status: commit-queue Passed tests diff --git a/Tools/Scripts/webkitpy/tool/commands/rebaseline.py b/Tools/Scripts/webkitpy/tool/commands/rebaseline.py index 7ccbf565a..63de323d1 100644 --- a/Tools/Scripts/webkitpy/tool/commands/rebaseline.py +++ b/Tools/Scripts/webkitpy/tool/commands/rebaseline.py @@ -44,7 +44,7 @@ from webkitpy.common.system.user import User from webkitpy.layout_tests.controllers.test_result_writer import TestResultWriter from webkitpy.layout_tests.models import test_failures from webkitpy.layout_tests.models.test_configuration import TestConfiguration -from webkitpy.layout_tests.models.test_expectations import TestExpectations, suffixes_for_expectations, BASELINE_SUFFIX_LIST +from webkitpy.layout_tests.models.test_expectations import TestExpectations, BASELINE_SUFFIX_LIST from webkitpy.layout_tests.port import builders from webkitpy.tool.grammar import pluralize from webkitpy.tool.multicommandtool import AbstractDeclarativeCommand @@ -333,7 +333,7 @@ class RebaselineExpectations(AbstractParallelRebaselineCommand): tests_to_rebaseline = {} expectations = TestExpectations(port, include_overrides=True) for test in expectations.get_rebaselining_failures(): - tests_to_rebaseline[test] = suffixes_for_expectations(expectations.get_expectations(test)) + tests_to_rebaseline[test] = TestExpectations.suffixes_for_expectations(expectations.get_expectations(test)) return tests_to_rebaseline def _add_tests_to_rebaseline_for_port(self, port_name): diff --git a/Tools/Scripts/webkitpy/tool/commands/rebaseline_unittest.py b/Tools/Scripts/webkitpy/tool/commands/rebaseline_unittest.py index 93bd5c500..cea6e2e99 100644 --- a/Tools/Scripts/webkitpy/tool/commands/rebaseline_unittest.py +++ b/Tools/Scripts/webkitpy/tool/commands/rebaseline_unittest.py @@ -49,7 +49,7 @@ class TestRebaseline(unittest.TestCase): command.bind_to_tool(tool) self.assertEqual(command._baseline_directory("Apple Win XP Debug (Tests)"), "/mock-checkout/LayoutTests/platform/win-xp") self.assertEqual(command._baseline_directory("Apple Win 7 Release (Tests)"), "/mock-checkout/LayoutTests/platform/win") - self.assertEqual(command._baseline_directory("Apple Lion Release WK1 (Tests)"), "/mock-checkout/LayoutTests/platform/mac") + self.assertEqual(command._baseline_directory("Apple Lion Release WK1 (Tests)"), "/mock-checkout/LayoutTests/platform/mac-lion") self.assertEqual(command._baseline_directory("Apple Lion Release WK2 (Tests)"), "/mock-checkout/LayoutTests/platform/mac-wk2") self.assertEqual(command._baseline_directory("GTK Linux 32-bit Release"), "/mock-checkout/LayoutTests/platform/gtk") self.assertEqual(command._baseline_directory("EFL Linux 64-bit Debug"), "/mock-checkout/LayoutTests/platform/efl") @@ -272,6 +272,9 @@ Retrieving results for gtk from GTK Linux 64-bit Release. Retrieving results for mac-lion from Apple Lion Release WK1 (Tests). userscripts/another-test.html (txt) userscripts/images.svg (png) +Retrieving results for mac-mountainlion from Apple MountainLion Release WK1 (Tests). + userscripts/another-test.html (txt) + userscripts/images.svg (png) Retrieving results for qt-linux from Qt Linux Release. userscripts/another-test.html (txt) userscripts/images.svg (png) @@ -280,7 +283,7 @@ Retrieving results for win-7sp0 from Apple Win 7 Release (Tests). userscripts/images.svg (png) """ - expected_stdout = """[(['echo', 'rebaseline-test-internal', '--suffixes', 'txt', '--builder', 'Webkit Linux 32', '--test', 'userscripts/another-test.html'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'txt', '--builder', 'Webkit Linux', '--test', 'userscripts/another-test.html'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'txt', '--builder', 'Webkit Mac10.6', '--test', 'userscripts/another-test.html'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'txt', '--builder', 'Webkit Mac10.7', '--test', 'userscripts/another-test.html'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'txt', '--builder', 'Webkit Win7', '--test', 'userscripts/another-test.html'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'txt', '--builder', 'Apple Win 7 Release (Tests)', '--test', 'userscripts/another-test.html'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'txt', '--builder', 'EFL Linux 64-bit Release', '--test', 'userscripts/another-test.html'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'txt', '--builder', 'Webkit Win', '--test', 'userscripts/another-test.html'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'txt', '--builder', 'GTK Linux 64-bit Release', '--test', 'userscripts/another-test.html'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'txt', '--builder', 'Qt Linux Release', '--test', 'userscripts/another-test.html'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'txt', '--builder', 'Apple Lion Release WK1 (Tests)', '--test', 'userscripts/another-test.html'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'png', '--builder', 'Webkit Linux 32', '--test', 'userscripts/images.svg'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'png', '--builder', 'Webkit Linux', '--test', 'userscripts/images.svg'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'png', '--builder', 'Webkit Mac10.6', '--test', 'userscripts/images.svg'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'png', '--builder', 'Webkit Mac10.7', '--test', 'userscripts/images.svg'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'png', '--builder', 'Webkit Win7', '--test', 'userscripts/images.svg'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'png', '--builder', 'Apple Win 7 Release (Tests)', '--test', 'userscripts/images.svg'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'png', '--builder', 'EFL Linux 64-bit Release', '--test', 'userscripts/images.svg'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'png', '--builder', 'Webkit Win', '--test', 'userscripts/images.svg'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'png', '--builder', 'GTK Linux 64-bit Release', '--test', 'userscripts/images.svg'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'png', '--builder', 'Qt Linux Release', '--test', 'userscripts/images.svg'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'png', '--builder', 'Apple Lion Release WK1 (Tests)', '--test', 'userscripts/images.svg'], '/mock-checkout')] + expected_stdout = """[(['echo', 'rebaseline-test-internal', '--suffixes', 'txt', '--builder', 'Webkit Linux 32', '--test', 'userscripts/another-test.html'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'txt', '--builder', 'Webkit Linux', '--test', 'userscripts/another-test.html'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'txt', '--builder', 'Webkit Mac10.6', '--test', 'userscripts/another-test.html'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'txt', '--builder', 'Webkit Mac10.7', '--test', 'userscripts/another-test.html'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'txt', '--builder', 'Webkit Win7', '--test', 'userscripts/another-test.html'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'txt', '--builder', 'Apple Win 7 Release (Tests)', '--test', 'userscripts/another-test.html'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'txt', '--builder', 'EFL Linux 64-bit Release', '--test', 'userscripts/another-test.html'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'txt', '--builder', 'Webkit Win', '--test', 'userscripts/another-test.html'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'txt', '--builder', 'GTK Linux 64-bit Release', '--test', 'userscripts/another-test.html'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'txt', '--builder', 'Qt Linux Release', '--test', 'userscripts/another-test.html'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'txt', '--builder', 'Apple Lion Release WK1 (Tests)', '--test', 'userscripts/another-test.html'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'txt', '--builder', 'Apple MountainLion Release WK1 (Tests)', '--test', 'userscripts/another-test.html'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'png', '--builder', 'Webkit Linux 32', '--test', 'userscripts/images.svg'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'png', '--builder', 'Webkit Linux', '--test', 'userscripts/images.svg'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'png', '--builder', 'Webkit Mac10.6', '--test', 'userscripts/images.svg'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'png', '--builder', 'Webkit Mac10.7', '--test', 'userscripts/images.svg'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'png', '--builder', 'Webkit Win7', '--test', 'userscripts/images.svg'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'png', '--builder', 'Apple Win 7 Release (Tests)', '--test', 'userscripts/images.svg'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'png', '--builder', 'EFL Linux 64-bit Release', '--test', 'userscripts/images.svg'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'png', '--builder', 'Webkit Win', '--test', 'userscripts/images.svg'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'png', '--builder', 'GTK Linux 64-bit Release', '--test', 'userscripts/images.svg'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'png', '--builder', 'Qt Linux Release', '--test', 'userscripts/images.svg'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'png', '--builder', 'Apple Lion Release WK1 (Tests)', '--test', 'userscripts/images.svg'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'png', '--builder', 'Apple MountainLion Release WK1 (Tests)', '--test', 'userscripts/images.svg'], '/mock-checkout')] """ expected_stderr = """MOCK run_command: ['qmake', '-v'], cwd=None |