diff options
| author | Simon Hausmann <simon.hausmann@digia.com> | 2012-10-17 16:21:14 +0200 |
|---|---|---|
| committer | Simon Hausmann <simon.hausmann@digia.com> | 2012-10-17 16:21:14 +0200 |
| commit | 8995b83bcbfbb68245f779b64e5517627c6cc6ea (patch) | |
| tree | 17985605dab9263cc2444bd4d45f189e142cca7c /Tools/Scripts/webkitpy | |
| parent | b9c9652036d5e9f1e29c574f40bc73a35c81ace6 (diff) | |
| download | qtwebkit-8995b83bcbfbb68245f779b64e5517627c6cc6ea.tar.gz | |
Imported WebKit commit cf4f8fc6f19b0629f51860cb2d4b25e139d07e00 (http://svn.webkit.org/repository/webkit/trunk@131592)
New snapshot that includes the build fixes for Mac OS X 10.6 and earlier as well
as the previously cherry-picked changes
Diffstat (limited to 'Tools/Scripts/webkitpy')
69 files changed, 3593 insertions, 1174 deletions
diff --git a/Tools/Scripts/webkitpy/common/config/committers.py b/Tools/Scripts/webkitpy/common/config/committers.py index e98b25936..1c231d37a 100644 --- a/Tools/Scripts/webkitpy/common/config/committers.py +++ b/Tools/Scripts/webkitpy/common/config/committers.py @@ -108,18 +108,17 @@ watchers_who_are_not_contributors = [ contributors_who_are_not_committers = [ Contributor("Aharon Lanin", "aharon@google.com"), - Contributor("Alan Stearns", "stearns@adobe.com", 'astearns'), + Contributor("Alan Stearns", "stearns@adobe.com", "astearns"), Contributor("Alexey Marinichev", ["amarinichev@chromium.org", "amarinichev@google.com"], "amarinichev"), Contributor("Andras Piroska", "pandras@inf.u-szeged.hu", "andris88"), Contributor("Andrei Bucur", "abucur@adobe.com", "abucur"), Contributor("Anne van Kesteren", "annevankesteren+webkit@gmail.com", "annevk"), Contributor("Annie Sullivan", "sullivan@chromium.org", "annie"), Contributor("Aryeh Gregor", "ayg@aryeh.name", "AryehGregor"), - Contributor("Balazs Ankes", "bank@inf.u-szeged.hu", 'abalazs'), + Contributor("Balazs Ankes", "bank@inf.u-szeged.hu", "abalazs"), Contributor("Brian Salomon", "bsalomon@google.com"), Contributor("Commit Queue", "commit-queue@webkit.org"), Contributor("Daniel Sievers", "sievers@chromium.org"), - Contributor("David Barr", "davidbarr@chromium.org", "barrbrain"), Contributor("David Dorwin", "ddorwin@chromium.org", "ddorwin"), Contributor("David Reveman", "reveman@chromium.org", "reveman"), Contributor("Dongsung Huang", "luxtella@company100.net", "Huang"), @@ -136,8 +135,8 @@ contributors_who_are_not_committers = [ Contributor("Gregg Tavares", ["gman@google.com", "gman@chromium.org"], "gman"), Contributor("Hao Zheng", "zhenghao@chromium.org"), Contributor("Ian Hickson", "ian@hixie.ch", "hixie"), - Contributor("Janos Badics", "jbadics@inf.u-szeged.hu", 'dicska'), - Contributor("Jonathan Backer", "backer@chromium.org", 'backer'), + Contributor("Janos Badics", "jbadics@inf.u-szeged.hu", "dicska"), + Contributor("Jonathan Backer", "backer@chromium.org", "backer"), Contributor("Jeff Timanus", ["twiz@chromium.org", "twiz@google.com"], "twiz"), Contributor("Jing Zhao", "jingzhao@chromium.org"), Contributor("John Bates", ["jbates@google.com", "jbates@chromium.org"], "jbates"), @@ -151,7 +150,7 @@ contributors_who_are_not_committers = [ Contributor("Oliver Varga", ["voliver@inf.u-szeged.hu", "Varga.Oliver@stud.u-szeged.hu"], "TwistO"), Contributor("Peter Gal", "galpeter@inf.u-szeged.hu", "elecro"), Contributor("Peter Linss", "peter.linss@hp.com", "plinss"), - Contributor("Pravin D", "pravind.2k4@gmail.com", 'pravind'), + Contributor("Pravin D", "pravind.2k4@gmail.com", "pravind"), Contributor("Radar WebKit Bug Importer", "webkit-bug-importer@group.apple.com"), Contributor("Raul Hudea", "rhudea@adobe.com", "rhudea"), Contributor("Roland Takacs", "rtakacs@inf.u-szeged.hu", "rtakacs"), @@ -179,7 +178,7 @@ contributors_who_are_not_committers = [ committers_unable_to_review = [ Committer("Aaron Boodman", "aa@chromium.org", "aboodman"), Committer("Adam Bergkvist", "adam.bergkvist@ericsson.com", "adambe"), - Committer("Adam Kallai", "kadam@inf.u-szeged.hu", 'kadam'), + Committer("Adam Kallai", "kadam@inf.u-szeged.hu", "kadam"), Committer("Adam Klein", "adamk@chromium.org", "aklein"), Committer("Adam Langley", "agl@chromium.org", "agl"), Committer("Ademar de Souza Reis Jr", ["ademar.reis@gmail.com", "ademar@webkit.org"], "ademar"), @@ -216,7 +215,6 @@ committers_unable_to_review = [ Committer("Benjamin Otte", ["otte@gnome.org", "otte@webkit.org"], "otte"), Committer("Bill Budge", ["bbudge@chromium.org", "bbudge@gmail.com"], "bbudge"), Committer("Brett Wilson", "brettw@chromium.org", "brettx"), - Committer("Caio Marcelo de Oliveira Filho", ["cmarcelo@webkit.org", "caio.oliveira@openbossa.org"], "cmarcelo"), Committer("Cameron McCormack", ["cam@mcc.id.au", "cam@webkit.org"], "heycam"), Committer("Carol Szabo", ["carol@webkit.org", "carol.szabo@nokia.com"], "cszabo1"), Committer("Cary Clark", ["caryclark@google.com", "caryclark@chromium.org"], "caryclark"), @@ -233,9 +231,11 @@ committers_unable_to_review = [ Committer("Daniel Cheng", "dcheng@chromium.org", "dcheng"), Committer("Dave Barton", "dbarton@mathscribe.com", "dbarton"), Committer("Dave Tharp", "dtharp@codeaurora.org", "dtharp"), + Committer("David Michael Barr", ["davidbarr@chromium.org", "davidbarr@google.com", "b@rr-dav.id.au"], "barrbrain"), Committer("David Grogan", ["dgrogan@chromium.org", "dgrogan@google.com"], "dgrogan"), Committer("David Smith", ["catfish.man@gmail.com", "dsmith@webkit.org"], "catfishman"), Committer("Diego Gonzalez", ["diegohcg@webkit.org", "diego.gonzalez@openbossa.org"], "diegohcg"), + Committer("Dinu Jacob", "dinu.s.jacob@intel.com", "dsjacob"), Committer("Dmitry Lomov", ["dslomov@google.com", "dslomov@chromium.org"], "dslomov"), Committer("Dominic Cooney", ["dominicc@chromium.org", "dominicc@google.com"], "dominicc"), Committer("Dominic Mazzoni", ["dmazzoni@google.com", "dmazzoni@chromium.org"], "dmazzoni"), @@ -243,7 +243,6 @@ committers_unable_to_review = [ Committer("Drew Wilson", "atwilson@chromium.org", "atwilson"), Committer("Eli Fidler", ["eli@staikos.net", "efidler@rim.com"], "efidler"), Committer("Elliot Poger", "epoger@chromium.org", "epoger"), - Committer("Emil A Eklund", "eae@chromium.org", "eae"), Committer("Erik Arvidsson", "arv@chromium.org", "arv"), Committer("Eric Roman", "eroman@chromium.org", "eroman"), Committer("Eric Uhrhane", "ericu@chromium.org", "ericu"), @@ -299,6 +298,7 @@ committers_unable_to_review = [ Committer("Joshua Bell", ["jsbell@chromium.org", "jsbell@google.com"], "jsbell"), Committer("Julie Parent", ["jparent@google.com", "jparent@chromium.org"], "jparent"), Committer("Jungshik Shin", "jshin@chromium.org"), + Committer("Justin Novosad", ["junov@google.com", "junov@chromium.org"], "junov"), Committer("Justin Schuh", "jschuh@chromium.org", "jschuh"), Committer("Kaustubh Atrawalkar", ["kaustubh@motorola.com"], "silverroots"), Committer("Keishi Hattori", "keishi@webkit.org", "keishi"), @@ -378,6 +378,7 @@ committers_unable_to_review = [ Committer("Siddharth Mathur", "siddharth.mathur@nokia.com", "simathur"), Committer("Stephen Chenney", "schenney@chromium.org", "schenney"), Committer("Steve Lacey", "sjl@chromium.org", "stevela"), + Committer("Taiju Tsuiki", "tzik@chromium.org", "tzik"), Committer("Takashi Toyoshima", "toyoshim@chromium.org", "toyoshim"), Committer("Thomas Sepez", "tsepez@chromium.org", "tsepez"), Committer("Tom Hudson", ["tomhudson@google.com", "tomhudson@chromium.org"], "tomhudson"), @@ -441,6 +442,7 @@ reviewers_list = [ Reviewer("Brady Eidson", "beidson@apple.com", "bradee-oh"), Reviewer("Brent Fulgham", "bfulgham@webkit.org", "bfulgham"), Reviewer("Brian Weinstein", "bweinstein@apple.com", "bweinstein"), + Reviewer("Caio Marcelo de Oliveira Filho", ["cmarcelo@webkit.org", "caio.oliveira@openbossa.org"], "cmarcelo"), Reviewer("Cameron Zwarich", ["zwarich@apple.com", "cwzwarich@apple.com", "cwzwarich@webkit.org"]), Reviewer("Carlos Garcia Campos", ["cgarcia@igalia.com", "carlosgc@gnome.org", "carlosgc@webkit.org"], "KaL"), Reviewer("Chang Shu", ["cshu@webkit.org", "c.shu@sisa.samsung.com"], "cshu"), @@ -465,6 +467,7 @@ reviewers_list = [ Reviewer("Dmitry Titov", "dimich@chromium.org", "dimich"), Reviewer("Don Melton", "gramps@apple.com", "gramps"), Reviewer("Dumitru Daniliuc", "dumi@chromium.org", "dumi"), + Reviewer("Emil A Eklund", "eae@chromium.org", "eae"), Reviewer("Enrica Casucci", "enrica@apple.com", "enrica"), Reviewer("Eric Carlson", "eric.carlson@apple.com", "eric_carlson"), Reviewer("Eric Seidel", "eric@webkit.org", "eseidel"), @@ -497,7 +500,7 @@ reviewers_list = [ Reviewer("Kevin McCullough", "kmccullough@apple.com", "maculloch"), Reviewer("Kevin Ollivier", ["kevino@theolliviers.com", "kevino@webkit.org"], "kollivier"), Reviewer("Lars Knoll", ["lars@trolltech.com", "lars@kde.org", "lars.knoll@nokia.com"], "lars"), - Reviewer("Laszlo Gombos", "laszlo.1.gombos@nokia.com", "lgombos"), + Reviewer("Laszlo Gombos", ["laszlo.gombos@webkit.org", "l.gombos@samsung.com", "laszlo.1.gombos@nokia.com"], "lgombos"), Reviewer("Levi Weintraub", ["leviw@chromium.org", "leviw@google.com", "lweintraub@apple.com"], "leviw"), Reviewer("Luiz Agostini", ["luiz@webkit.org", "luiz.agostini@openbossa.org"], "lca"), Reviewer("Maciej Stachowiak", "mjs@apple.com", "othermaciej"), diff --git a/Tools/Scripts/webkitpy/common/config/watchlist b/Tools/Scripts/webkitpy/common/config/watchlist index 2ab16b0c2..0f3dd65a1 100755 --- a/Tools/Scripts/webkitpy/common/config/watchlist +++ b/Tools/Scripts/webkitpy/common/config/watchlist @@ -56,6 +56,15 @@ "webkitpy": { "filename": r"Tools/Scripts/webkitpy/", }, + "webkitperl": { + "filename": r"Tools/Scripts/webkitperl/" + r"|Tools/Scripts/webkitdirs.pm" + r"|Tools/Scripts/VCSUtils.pm" + r"|Tools/Scripts/test-webkitperl", + }, + "SVNScripts": { + "filename": r"Tools/Scripts/svn-.*", + }, "TestFailures": { "filename": r"Tools/BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/", }, @@ -64,6 +73,9 @@ "less": r"[Ss]ecurityOrigin(?!\.(h|cpp))", "filename": r"XSS|[Ss]ecurity", }, + "XSS": { + "filename": r".*XSS", + }, "SkiaGraphics": { "filename": r"Source/WebCore/platform/graphics/skia/" r"|Source/WebCore/platform/graphics/filters/skia/", @@ -103,8 +115,8 @@ }, "QtGraphics": { "filename": r"Source/WebCore/platform/graphics/qt/" - r"|Source/WebKit2/WebProcess/WebPage/LayerTreeCoordinator/" - r"|Source/WebKit2/UIProcess/WebLayerTreeRenderer(?!\.(h|cpp))", + r"|Source/WebKit2/WebProcess/WebPage/CoordinatedGraphics/" + r"|Source/WebKit2/UIProcess/CoordinatedGraphics", }, "TextureMapper": { "filename": r"Source/WebCore/platform/graphics/texmap/", @@ -232,10 +244,27 @@ r"|Source/WebCore/svg/graphics/filters" r"|Source/WebCore/svg/graphics/.*Filter.*", }, + "TouchAdjustment": { + "filename": r"Source/WebCore/page/TouchAdjustment.*" + r"|LayoutTests/touchadjustment" + r"|Source/WebKit/blackberry/WebKitSupport/FatFingers.*", + }, "SVG": { "filename": r"Source/WebCore/svg" r"|Source/WebCore/rendering/svg", }, + "WebInspectorProtocol": { + "filename": r"Source/WebCore/inspector/Inspector.json", + }, + "WebSocket": { + "filename": r"Source/WebCore/Modules/websockets" + r"|Source/WebCore/platform/network/(|.+/)SocketStream.*", + }, + "MediaStream": { + "filename": r"Source/WebCore/Modules/mediastream" + r"|Source/WebCore/platform/mediastream" + r"|LayoutTests/fast/mediastream", + }, }, "CC_RULES": { # Note: All email addresses listed must be registered with bugzilla. @@ -243,9 +272,9 @@ # two different accounts as far as bugzilla is concerned. "AppleMacPublicApi": [ "timothy@apple.com" ], "Battery": [ "gyuyoung.kim@samsung.com" ], - "BlackBerry": [ "mifenton@rim.com", "rwlbuis@gmail.com" ], + "BlackBerry": [ "mifenton@rim.com", "rwlbuis@gmail.com", "tonikitoo@webkit.org" ], "CMake": [ "rakuco@webkit.org", "gyuyoung.kim@samsung.com" ], - "CSS": [ "alexis@webkit.org", "macpherson@chromium.org", "cmarcelo@webkit.org" ], + "CSS": [ "alexis@webkit.org", "macpherson@chromium.org", "cmarcelo@webkit.org", "allan.jensen@digia.com" ], "ChromiumGraphics": [ "jamesr@chromium.org", "cc-bugs@chromium.org" ], "ChromiumPublicApi": [ "abarth@webkit.org", "dglazkov@chromium.org", "fishd@chromium.org", "jamesr@chromium.org", "tkent+wkapi@chromium.org" ], "DOMAttributes": [ "cmarcelo@webkit.org", ], @@ -261,29 +290,36 @@ "Loader": [ "japhet@chromium.org" ], "MathML": [ "dbarton@mathscribe.com" ], "Media": [ "feature-media-reviews@chromium.org", "eric.carlson@apple.com" ], + "MediaStream": [ "tommyw@google.com" ], "NetworkInfo": [ "gyuyoung.kim@samsung.com" ], "OpenGL" : [ "noam.rosenthal@nokia.com", "dino@apple.com" ], - "SkiaGraphics": [ "senorblanco@chromium.org" ], "QtBuildSystem" : [ "vestbo@webkit.org", "abecsi@webkit.org" ], "QtGraphics" : [ "noam.rosenthal@nokia.com" ], - "QtWebKit2PlatformSpecific": [ "alexis@webkit.org", "zoltan@webkit.org", "cmarcelo@webkit.org", "abecsi@webkit.org" ], - "QtWebKit2PublicAPI": [ "alexis@webkit.org", "zoltan@webkit.org", "cmarcelo@webkit.org", "abecsi@webkit.org" ], + "QtWebKit2PlatformSpecific": [ "alexis@webkit.org", "cmarcelo@webkit.org", "abecsi@webkit.org" ], + "QtWebKit2PublicAPI": [ "alexis@webkit.org", "cmarcelo@webkit.org", "abecsi@webkit.org" ], "Rendering": [ "eric@webkit.org" ], + "SVG": ["schenney@chromium.org", "pdr@google.com", "fmalita@chromium.org" ], + "SVNScripts": [ "dbates@webkit.org" ], "ScrollingCoordinator": [ "andersca@apple.com", "jamesr@chromium.org", "tonikitoo@webkit.org" ], "SecurityCritical": [ "abarth@webkit.org" ], + "SkiaGraphics": [ "senorblanco@chromium.org" ], "SoupNetwork": [ "rakuco@webkit.org", "gns@gnome.org", "mrobinson@webkit.org", "danw@gnome.org" ], "StyleChecker": [ "levin@chromium.org", ], - "SVG": ["schenney@chromium.org", "pdr@google.com", "fmalita@chromium.org" ], "TestFailures": [ "abarth@webkit.org", "dglazkov@chromium.org" ], "TextureMapper" : [ "noam.rosenthal@nokia.com" ], "ThreadingFiles|ThreadingUsage": [ "levin+threading@chromium.org", ], + "TouchAdjustment" : [ "allan.jensen@digia.com" ], "V8Bindings|BindingsScripts": [ "abarth@webkit.org", "japhet@chromium.org", "haraken@chromium.org" ], + "WTF": [ "benjamin@webkit.org",], "WatchListScript": [ "levin+watchlist@chromium.org", ], "WebGL": [ "dino@apple.com" ], "WebIDL": [ "abarth@webkit.org", "ojan@chromium.org" ], + "WebInspectorProtocol": [ "timothy@apple.com", ], "WebKitGTKTranslations": [ "gns@gnome.org", "mrobinson@webkit.org" ], + "WebSocket": [ "yutak@chromium.org" ], + "XSS": [ "dbates@webkit.org" ], + "webkitperl": [ "dbates@webkit.org" ], "webkitpy": [ "abarth@webkit.org", "ojan@chromium.org", "dpranke@chromium.org" ], - "WTF": [ "benjamin@webkit.org",], }, "MESSAGE_RULES": { "ChromiumPublicApi": [ "Please wait for approval from abarth@webkit.org, dglazkov@chromium.org, " diff --git a/Tools/Scripts/webkitpy/common/host.py b/Tools/Scripts/webkitpy/common/host.py index 53889657b..7dd5ad024 100644 --- a/Tools/Scripts/webkitpy/common/host.py +++ b/Tools/Scripts/webkitpy/common/host.py @@ -125,7 +125,7 @@ class Host(SystemHost): except OSError, e: _log.debug('Failed to engage git.bat Windows hack.') - def _initialize_scm(self, patch_directories=None): + def initialize_scm(self, patch_directories=None): if sys.platform == "win32": self._engage_awesome_windows_hacks() detector = SCMDetector(self.filesystem, self.executive) diff --git a/Tools/Scripts/webkitpy/common/host_mock.py b/Tools/Scripts/webkitpy/common/host_mock.py index ca4f78eb3..8b508bf8f 100644 --- a/Tools/Scripts/webkitpy/common/host_mock.py +++ b/Tools/Scripts/webkitpy/common/host_mock.py @@ -50,7 +50,7 @@ class MockHost(MockSystemHost): # FIXME: we should never initialize the SCM by default, since the real # object doesn't either. This has caused at least one bug (see bug 89498). if initialize_scm_by_default: - self._initialize_scm() + self.initialize_scm() self.bugs = MockBugzilla() self.buildbot = MockBuildBot() self._chromium_buildbot = MockBuildBot() @@ -61,7 +61,7 @@ class MockHost(MockSystemHost): self._watch_list = MockWatchList() - def _initialize_scm(self, patch_directories=None): + def initialize_scm(self, patch_directories=None): self._scm = MockSCM(filesystem=self.filesystem, executive=self.executive) # Various pieces of code (wrongly) call filesystem.chdir(checkout_root). # Making the checkout_root exist in the mock filesystem makes that chdir not raise. diff --git a/Tools/Scripts/webkitpy/common/system/executive_unittest.py b/Tools/Scripts/webkitpy/common/system/executive_unittest.py index 8a322e480..57c557369 100644 --- a/Tools/Scripts/webkitpy/common/system/executive_unittest.py +++ b/Tools/Scripts/webkitpy/common/system/executive_unittest.py @@ -162,6 +162,10 @@ class ExecutiveTest(unittest.TestCase): # FIXME: https://bugs.webkit.org/show_bug.cgi?id=54790 # We seem to get either 0 or 1 here for some reason. self.assertTrue(process.wait() in (0, 1)) + elif sys.platform == "cygwin": + # FIXME: https://bugs.webkit.org/show_bug.cgi?id=98196 + # cygwin seems to give us either SIGABRT or SIGKILL + self.assertTrue(process.wait() in (-signal.SIGABRT, -signal.SIGKILL)) else: expected_exit_code = -signal.SIGKILL self.assertEqual(process.wait(), expected_exit_code) diff --git a/Tools/Scripts/webkitpy/layout_tests/controllers/layout_test_finder.py b/Tools/Scripts/webkitpy/layout_tests/controllers/layout_test_finder.py index a7b49831c..6447c8fb4 100644 --- a/Tools/Scripts/webkitpy/layout_tests/controllers/layout_test_finder.py +++ b/Tools/Scripts/webkitpy/layout_tests/controllers/layout_test_finder.py @@ -106,11 +106,9 @@ class LayoutTestFinder(object): tests_to_skip = all_tests - tests_to_skip elif self._options.skipped == 'ignore': tests_to_skip = set() - elif self._options.skipped == 'default': - pass # listed for completeness - - # make sure we're explicitly running any tests passed on the command line. - tests_to_skip -= paths + elif self._options.skipped != 'always': + # make sure we're explicitly running any tests passed on the command line; equivalent to 'default'. + tests_to_skip -= paths # unless of course we don't want to run the HTTP tests :) if not self._options.http: diff --git a/Tools/Scripts/webkitpy/layout_tests/controllers/manager.py b/Tools/Scripts/webkitpy/layout_tests/controllers/manager.py index d06ed7153..c0a70e615 100644 --- a/Tools/Scripts/webkitpy/layout_tests/controllers/manager.py +++ b/Tools/Scripts/webkitpy/layout_tests/controllers/manager.py @@ -239,8 +239,9 @@ def summarize_results(port_obj, expectations, result_summary, retry_summary, tes try: # We only use the svn revision for using trac links in the results.html file, # Don't do this by default since it takes >100ms. + # FIXME: Do we really need to populate this both here and in the json_results_generator? if use_trac_links_in_results_html(port_obj): - port_obj.host._initialize_scm() + port_obj.host.initialize_scm() results['revision'] = port_obj.host.scm().head_svn_revision() except Exception, e: _log.warn("Failed to determine svn revision for checkout (cwd: %s, webkit_base: %s), leaving 'revision' key blank in full_results.json.\n%s" % (port_obj._filesystem.getcwd(), port_obj.path_from_webkit_base(), e)) diff --git a/Tools/Scripts/webkitpy/layout_tests/layout_package/json_results_generator.py b/Tools/Scripts/webkitpy/layout_tests/layout_package/json_results_generator.py index c7d7c3eb7..73834f0ad 100644 --- a/Tools/Scripts/webkitpy/layout_tests/layout_package/json_results_generator.py +++ b/Tools/Scripts/webkitpy/layout_tests/layout_package/json_results_generator.py @@ -281,7 +281,8 @@ class JSONResultsGeneratorBase(object): results_for_builder = results_json[builder_name] - self._insert_generic_metadata(results_for_builder) + if builder_name: + self._insert_generic_metadata(results_for_builder) self._insert_failure_summaries(results_for_builder) @@ -377,15 +378,17 @@ class JSONResultsGeneratorBase(object): return self.__class__.PASS_RESULT - # FIXME: Callers should use scm.py instead. - # FIXME: Identify and fix the run-time errors that were observed on Windows - # chromium buildbot when we had updated this code to use scm.py once before. def _get_svn_revision(self, in_directory): """Returns the svn revision for the given directory. Args: in_directory: The directory where svn is to be run. """ + + # FIXME: We initialize this here in order to engage the stupid windows hacks :). + # We can't reuse an existing scm object because the specific directories may + # be part of other checkouts. + self._port.host.initialize_scm() scm = SCMDetector(self._filesystem, self._executive).detect_scm_system(in_directory) if scm: return scm.svn_revision(in_directory) diff --git a/Tools/Scripts/webkitpy/layout_tests/models/test_expectations.py b/Tools/Scripts/webkitpy/layout_tests/models/test_expectations.py index 42b518f7f..b48c5b933 100644 --- a/Tools/Scripts/webkitpy/layout_tests/models/test_expectations.py +++ b/Tools/Scripts/webkitpy/layout_tests/models/test_expectations.py @@ -155,7 +155,7 @@ class TestExpectationParser(object): else: parsed_specifiers.add(modifier) - if not expectation_line.parsed_bug_modifiers and not has_wontfix and not has_bugid: + if not expectation_line.parsed_bug_modifiers and not has_wontfix and not has_bugid and self._port.warn_if_bug_missing_in_test_expectations(): expectation_line.warnings.append(self.MISSING_BUG_WARNING) if self._allow_rebaseline_modifier and self.REBASELINE_MODIFIER in modifiers: @@ -869,7 +869,7 @@ class TestExpectations(object): return self._model def get_rebaselining_failures(self): - return self._model.get_test_set(REBASELINE, IMAGE) | self._model.get_test_set(REBASELINE, FAIL) + return self._model.get_test_set(REBASELINE) # FIXME: Change the callsites to use TestExpectationsModel and remove. def get_expectations(self, test): @@ -970,7 +970,7 @@ class TestExpectations(object): expectation.name in except_these_tests and 'rebaseline' in expectation.parsed_modifiers)) - return self.list_to_string(filter(without_rebaseline_modifier, self._expectations)) + return self.list_to_string(filter(without_rebaseline_modifier, self._expectations), reconstitute_only_these=[]) def _add_expectations(self, expectation_list): for expectation_line in expectation_list: diff --git a/Tools/Scripts/webkitpy/layout_tests/models/test_expectations_unittest.py b/Tools/Scripts/webkitpy/layout_tests/models/test_expectations_unittest.py index c5606071d..c3fc02658 100644 --- a/Tools/Scripts/webkitpy/layout_tests/models/test_expectations_unittest.py +++ b/Tools/Scripts/webkitpy/layout_tests/models/test_expectations_unittest.py @@ -179,8 +179,7 @@ class MiscTests(Base): "Bug(rniwa) disabled-test.html-disabled [ ImageOnlyFailure ]", is_lint_mode=True) self.assertFalse(True, "ParseError wasn't raised") except ParseError, e: - warnings = ("expectations:1 Test lacks BUG modifier. failures/expected/text.html\n" - "expectations:1 Unrecognized modifier 'foo' failures/expected/text.html\n" + warnings = ("expectations:1 Unrecognized modifier 'foo' failures/expected/text.html\n" "expectations:2 Path does not exist. non-existent-test.html") self.assertEqual(str(e), warnings) @@ -372,7 +371,9 @@ class SemanticTests(Base): def test_missing_bugid(self): self.parse_exp('failures/expected/text.html [ Failure ]') - self.assertTrue(self._exp.has_warnings()) + self.assertFalse(self._exp.has_warnings()) + + self._port.warn_if_bug_missing_in_test_expectations = lambda: True self.parse_exp('failures/expected/text.html [ Failure ]') line = self._exp._model.get_expectation_line('failures/expected/text.html') @@ -512,8 +513,22 @@ class RebaseliningTest(Base): 'Bug(z) failures/expected/crash.html [ Crash ]\n', 'Bug(x0) failures/expected/image.html [ Crash ]\n') + # Ensure that we don't modify unrelated lines, even if we could rewrite them. + # i.e., the second line doesn't get rewritten to "Bug(y) failures/expected/skip.html" + self.assertRemove('Bug(x) failures/expected/text.html [ Failure Rebaseline ]\n' + 'Bug(Y) failures/expected/image.html [ Skip ]\n' + 'Bug(z) failures/expected/crash.html\n', + '', + ['failures/expected/text.html'], + 'Bug(Y) failures/expected/image.html [ Skip ]\n' + 'Bug(z) failures/expected/crash.html\n', + '') + + def test_get_rebaselining_failures(self): + # Make sure we find a test as needing a rebaseline even if it is not marked as a failure. + self.parse_exp('Bug(x) failures/expected/text.html [ Rebaseline ]\n') + self.assertEqual(len(self._exp.get_rebaselining_failures()), 1) - def test_no_get_rebaselining_failures(self): self.parse_exp(self.get_basic_expectations()) self.assertEqual(len(self._exp.get_rebaselining_failures()), 0) diff --git a/Tools/Scripts/webkitpy/layout_tests/port/base.py b/Tools/Scripts/webkitpy/layout_tests/port/base.py index cd57032e9..ae55c684d 100755 --- a/Tools/Scripts/webkitpy/layout_tests/port/base.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/base.py @@ -182,6 +182,10 @@ class Port(object): """Return the number of DumpRenderTree instances to use for this port.""" return self._executive.cpu_count() + def default_max_locked_shards(self): + """Return the number of "locked" shards to run in parallel (like the http tests).""" + return 1 + def worker_startup_delay_secs(self): # FIXME: If we start workers up too quickly, DumpRenderTree appears # to thrash on something and time out its first few tests. Until @@ -870,6 +874,7 @@ class Port(object): # Most ports (?): 'WEBKIT_TESTFONTS', + 'WEBKITOUTPUTDIR', ] for variable in variables_to_copy: self._copy_value_from_environ_if_set(clean_env, variable) @@ -997,6 +1002,9 @@ class Port(object): # some ports have Skipped files which are returned as part of test_expectations(). return self._filesystem.exists(self.path_to_test_expectations_file()) + def warn_if_bug_missing_in_test_expectations(self): + return False + def expectations_dict(self): """Returns an OrderedDict of name -> expectations strings. The names are expected to be (but not required to be) paths in the filesystem. @@ -1162,9 +1170,18 @@ class Port(object): return "apache2-httpd.conf" def _path_to_apache_config_file(self): - """Returns the full path to the apache binary. + """Returns the full path to the apache configuration file. + + If the WEBKIT_HTTP_SERVER_CONF_PATH environment variable is set, its + contents will be used instead. This is needed only by ports that use the apache_http_server module.""" + config_file_from_env = os.environ.get('WEBKIT_HTTP_SERVER_CONF_PATH') + if config_file_from_env: + if not self._filesystem.exists(config_file_from_env): + raise IOError('%s was not found on the system' % config_file_from_env) + return config_file_from_env + config_file_name = self._apache_config_file_name_for_platform(sys.platform) return self._filesystem.join(self.layout_tests_dir(), 'http', 'conf', config_file_name) @@ -1223,11 +1240,15 @@ class Port(object): This is needed only by ports that use the http_server.py module.""" raise NotImplementedError('Port._path_to_lighttpd_php') + @memoized def _path_to_wdiff(self): """Returns the full path to the wdiff binary, or None if it is not available. This is likely used only by wdiff_text()""" - return 'wdiff' + for path in ("/usr/bin/wdiff", "/usr/bin/dwdiff"): + if self._filesystem.exists(path): + return path + return None def _webkit_baseline_path(self, platform): """Return the full path to the top of the baseline tree for a @@ -1275,7 +1296,7 @@ class Port(object): base_tests = self._real_tests([suite.base]) suite.tests = {} for test in base_tests: - suite.tests[test.replace(suite.base, suite.name)] = test + suite.tests[test.replace(suite.base, suite.name, 1)] = test return suites def _virtual_tests(self, paths, suites): @@ -1292,7 +1313,7 @@ class Port(object): def lookup_virtual_test_base(self, test_name): for suite in self.populated_virtual_test_suites(): if test_name.startswith(suite.name): - return test_name.replace(suite.name, suite.base) + return test_name.replace(suite.name, suite.base, 1) return None def lookup_virtual_test_args(self, test_name): diff --git a/Tools/Scripts/webkitpy/layout_tests/port/base_unittest.py b/Tools/Scripts/webkitpy/layout_tests/port/base_unittest.py index f6a6eb01f..e9b2f060d 100644 --- a/Tools/Scripts/webkitpy/layout_tests/port/base_unittest.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/base_unittest.py @@ -437,11 +437,15 @@ class PortTest(unittest.TestCase): tests = port.tests(['passes']) self.assertTrue('passes/text.html' in tests) + self.assertTrue('passes/passes/test-virtual-passes.html' in tests) self.assertFalse('virtual/passes/text.html' in tests) tests = port.tests(['virtual/passes']) self.assertFalse('passes/text.html' in tests) - self.assertTrue('virtual/passes/text.html' in tests) + self.assertTrue('virtual/passes/test-virtual-passes.html' in tests) + self.assertTrue('virtual/passes/passes/test-virtual-passes.html' in tests) + self.assertFalse('virtual/passes/test-virtual-virtual/passes.html' in tests) + self.assertFalse('virtual/passes/virtual/passes/test-virtual-passes.html' in tests) def test_build_path(self): port = self.make_port(options=optparse.Values({'build_directory': '/my-build-directory/'})) diff --git a/Tools/Scripts/webkitpy/layout_tests/port/builders.py b/Tools/Scripts/webkitpy/layout_tests/port/builders.py index 3d03fc7af..605b8cccc 100644 --- a/Tools/Scripts/webkitpy/layout_tests/port/builders.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/builders.py @@ -37,16 +37,16 @@ from webkitpy.common.memoized import memoized # * specifiers -- a set of specifiers, representing configurations covered by this builder. _exact_matches = { # These builders are on build.chromium.org. - "Webkit Win": {"port_name": "chromium-win-xp", "specifiers": set(["xp", "release"])}, - "Webkit Win7": {"port_name": "chromium-win-win7", "specifiers": set(["win7"])}, - "Webkit Win (dbg)(1)": {"port_name": "chromium-win-xp", "specifiers": set(["win", "debug"])}, - "Webkit Win (dbg)(2)": {"port_name": "chromium-win-xp", "specifiers": set(["win", "debug"])}, - "Webkit Linux": {"port_name": "chromium-linux-x86_64", "specifiers": set(["linux", "x86_64", "release"])}, - "Webkit Linux 32": {"port_name": "chromium-linux-x86", "specifiers": set(["linux", "x86"])}, - "Webkit Linux (dbg)": {"port_name": "chromium-linux-x86_64", "specifiers": set(["linux", "debug"])}, - "Webkit Mac10.6": {"port_name": "chromium-mac-snowleopard", "specifiers": set(["snowleopard"])}, - "Webkit Mac10.6 (dbg)": {"port_name": "chromium-mac-snowleopard", "specifiers": set(["snowleopard", "debug"])}, - "Webkit Mac10.7": {"port_name": "chromium-mac-lion", "specifiers": set(["lion"])}, + "WebKit XP": {"port_name": "chromium-win-xp", "specifiers": set(["xp", "release"])}, + "WebKit Win7": {"port_name": "chromium-win-win7", "specifiers": set(["win7"])}, + "WebKit Win7 (dbg)(1)": {"port_name": "chromium-win-xp", "specifiers": set(["win", "debug"])}, + "WebKit Win7 (dbg)(2)": {"port_name": "chromium-win-xp", "specifiers": set(["win", "debug"])}, + "WebKit Linux": {"port_name": "chromium-linux-x86_64", "specifiers": set(["linux", "x86_64", "release"])}, + "WebKit Linux 32": {"port_name": "chromium-linux-x86", "specifiers": set(["linux", "x86"])}, + "WebKit Linux (dbg)": {"port_name": "chromium-linux-x86_64", "specifiers": set(["linux", "debug"])}, + "WebKit Mac10.6": {"port_name": "chromium-mac-snowleopard", "specifiers": set(["snowleopard"])}, + "WebKit Mac10.6 (dbg)": {"port_name": "chromium-mac-snowleopard", "specifiers": set(["snowleopard", "debug"])}, + "WebKit Mac10.7": {"port_name": "chromium-mac-lion", "specifiers": set(["lion"])}, # These builders are on build.webkit.org. "Apple MountainLion Release WK1 (Tests)": {"port_name": "mac-mountainlion", "specifiers": set(["mountainlion"]), "rebaseline_override_dir": "mac"}, @@ -92,6 +92,8 @@ _ports_without_builders = [ "qt-mac", "qt-win", "qt-wk2", + # FIXME: Move to _extact_matches. + "chromium-android", ] diff --git a/Tools/Scripts/webkitpy/layout_tests/port/chromium.py b/Tools/Scripts/webkitpy/layout_tests/port/chromium.py index 44c98a383..c310eb15b 100755 --- a/Tools/Scripts/webkitpy/layout_tests/port/chromium.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/chromium.py @@ -53,6 +53,7 @@ class ChromiumPort(Port): ALL_SYSTEMS = ( ('snowleopard', 'x86'), ('lion', 'x86'), + ('mountainlion', 'x86'), ('xp', 'x86'), ('win7', 'x86'), ('lucid', 'x86'), @@ -62,13 +63,13 @@ class ChromiumPort(Port): ('icecreamsandwich', 'x86')) ALL_BASELINE_VARIANTS = [ - 'chromium-mac-lion', 'chromium-mac-snowleopard', 'chromium-mac-leopard', + 'chromium-mac-mountainlion', 'chromium-mac-lion', 'chromium-mac-snowleopard', 'chromium-win-win7', 'chromium-win-xp', 'chromium-linux-x86_64', 'chromium-linux-x86', ] CONFIGURATION_SPECIFIER_MACROS = { - 'mac': ['snowleopard', 'lion'], + 'mac': ['snowleopard', 'lion', 'mountainlion'], 'win': ['xp', 'win7'], 'linux': ['lucid'], 'android': ['icecreamsandwich'], @@ -111,6 +112,13 @@ class ChromiumPort(Port): def is_chromium(self): return True + def default_max_locked_shards(self): + """Return the number of "locked" shards to run in parallel (like the http tests).""" + max_locked_shards = int(self.default_child_processes()) / 4 + if not max_locked_shards: + return 1 + return max_locked_shards + def default_pixel_tests(self): return True @@ -332,13 +340,16 @@ class ChromiumPort(Port): 'win_layout_rel', ]) + def warn_if_bug_missing_in_test_expectations(self): + return True + def expectations_files(self): paths = [self.path_to_test_expectations_file()] skia_expectations_path = self.path_from_chromium_base('skia', 'skia_test_expectations.txt') + # FIXME: we should probably warn if this file is missing in some situations. + # See the discussion in webkit.org/b/97699. if self._filesystem.exists(skia_expectations_path): paths.append(skia_expectations_path) - else: - _log.warning("Using the chromium port without having the downstream skia_test_expectations.txt file checked out. Expectations related things might be wonky.") builder_name = self.get_option('builder_name', 'DUMMY_BUILDER_NAME') if builder_name == 'DUMMY_BUILDER_NAME' or '(deps)' in builder_name or builder_name in self.try_builder_names: @@ -376,6 +387,9 @@ class ChromiumPort(Port): VirtualTestSuite('platform/chromium/virtual/gpu/fast/hidpi', 'fast/hidpi', ['--force-compositing-mode']), + VirtualTestSuite('platform/chromium/virtual/softwarecompositing', + 'compositing', + ['--enable-software-compositing']), ] # diff --git a/Tools/Scripts/webkitpy/layout_tests/port/chromium_android.py b/Tools/Scripts/webkitpy/layout_tests/port/chromium_android.py index 3dfeab7a3..e246b8870 100644 --- a/Tools/Scripts/webkitpy/layout_tests/port/chromium_android.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/chromium_android.py @@ -77,45 +77,51 @@ TEST_PATH_PREFIX = '/all-tests' FORWARD_PORTS = '8000 8080 8443 8880 9323' MS_TRUETYPE_FONTS_DIR = '/usr/share/fonts/truetype/msttcorefonts/' +MS_TRUETYPE_FONTS_PACKAGE = 'ttf-mscorefonts-installer' # Timeout in seconds to wait for start/stop of DumpRenderTree. DRT_START_STOP_TIMEOUT_SECS = 10 -# List of fonts that layout tests expect, copied from DumpRenderTree/gtk/TestShellX11.cpp. +# List of fonts that layout tests expect, copied from DumpRenderTree/chromium/TestShellX11.cpp. HOST_FONT_FILES = [ - [MS_TRUETYPE_FONTS_DIR, 'Arial.ttf'], - [MS_TRUETYPE_FONTS_DIR, 'Arial_Bold.ttf'], - [MS_TRUETYPE_FONTS_DIR, 'Arial_Bold_Italic.ttf'], - [MS_TRUETYPE_FONTS_DIR, 'Arial_Italic.ttf'], - [MS_TRUETYPE_FONTS_DIR, 'Comic_Sans_MS.ttf'], - [MS_TRUETYPE_FONTS_DIR, 'Comic_Sans_MS_Bold.ttf'], - [MS_TRUETYPE_FONTS_DIR, 'Courier_New.ttf'], - [MS_TRUETYPE_FONTS_DIR, 'Courier_New_Bold.ttf'], - [MS_TRUETYPE_FONTS_DIR, 'Courier_New_Bold_Italic.ttf'], - [MS_TRUETYPE_FONTS_DIR, 'Courier_New_Italic.ttf'], - [MS_TRUETYPE_FONTS_DIR, 'Georgia.ttf'], - [MS_TRUETYPE_FONTS_DIR, 'Georgia_Bold.ttf'], - [MS_TRUETYPE_FONTS_DIR, 'Georgia_Bold_Italic.ttf'], - [MS_TRUETYPE_FONTS_DIR, 'Georgia_Italic.ttf'], - [MS_TRUETYPE_FONTS_DIR, 'Impact.ttf'], - [MS_TRUETYPE_FONTS_DIR, 'Trebuchet_MS.ttf'], - [MS_TRUETYPE_FONTS_DIR, 'Trebuchet_MS_Bold.ttf'], - [MS_TRUETYPE_FONTS_DIR, 'Trebuchet_MS_Bold_Italic.ttf'], - [MS_TRUETYPE_FONTS_DIR, 'Trebuchet_MS_Italic.ttf'], - [MS_TRUETYPE_FONTS_DIR, 'Times_New_Roman.ttf'], - [MS_TRUETYPE_FONTS_DIR, 'Times_New_Roman_Bold.ttf'], - [MS_TRUETYPE_FONTS_DIR, 'Times_New_Roman_Bold_Italic.ttf'], - [MS_TRUETYPE_FONTS_DIR, 'Times_New_Roman_Italic.ttf'], - [MS_TRUETYPE_FONTS_DIR, 'Verdana.ttf'], - [MS_TRUETYPE_FONTS_DIR, 'Verdana_Bold.ttf'], - [MS_TRUETYPE_FONTS_DIR, 'Verdana_Bold_Italic.ttf'], - [MS_TRUETYPE_FONTS_DIR, 'Verdana_Italic.ttf'], + [[MS_TRUETYPE_FONTS_DIR], 'Arial.ttf', MS_TRUETYPE_FONTS_PACKAGE], + [[MS_TRUETYPE_FONTS_DIR], 'Arial_Bold.ttf', MS_TRUETYPE_FONTS_PACKAGE], + [[MS_TRUETYPE_FONTS_DIR], 'Arial_Bold_Italic.ttf', MS_TRUETYPE_FONTS_PACKAGE], + [[MS_TRUETYPE_FONTS_DIR], 'Arial_Italic.ttf', MS_TRUETYPE_FONTS_PACKAGE], + [[MS_TRUETYPE_FONTS_DIR], 'Comic_Sans_MS.ttf', MS_TRUETYPE_FONTS_PACKAGE], + [[MS_TRUETYPE_FONTS_DIR], 'Comic_Sans_MS_Bold.ttf', MS_TRUETYPE_FONTS_PACKAGE], + [[MS_TRUETYPE_FONTS_DIR], 'Courier_New.ttf', MS_TRUETYPE_FONTS_PACKAGE], + [[MS_TRUETYPE_FONTS_DIR], 'Courier_New_Bold.ttf', MS_TRUETYPE_FONTS_PACKAGE], + [[MS_TRUETYPE_FONTS_DIR], 'Courier_New_Bold_Italic.ttf', MS_TRUETYPE_FONTS_PACKAGE], + [[MS_TRUETYPE_FONTS_DIR], 'Courier_New_Italic.ttf', MS_TRUETYPE_FONTS_PACKAGE], + [[MS_TRUETYPE_FONTS_DIR], 'Georgia.ttf', MS_TRUETYPE_FONTS_PACKAGE], + [[MS_TRUETYPE_FONTS_DIR], 'Georgia_Bold.ttf', MS_TRUETYPE_FONTS_PACKAGE], + [[MS_TRUETYPE_FONTS_DIR], 'Georgia_Bold_Italic.ttf', MS_TRUETYPE_FONTS_PACKAGE], + [[MS_TRUETYPE_FONTS_DIR], 'Georgia_Italic.ttf', MS_TRUETYPE_FONTS_PACKAGE], + [[MS_TRUETYPE_FONTS_DIR], 'Impact.ttf', MS_TRUETYPE_FONTS_PACKAGE], + [[MS_TRUETYPE_FONTS_DIR], 'Trebuchet_MS.ttf', MS_TRUETYPE_FONTS_PACKAGE], + [[MS_TRUETYPE_FONTS_DIR], 'Trebuchet_MS_Bold.ttf', MS_TRUETYPE_FONTS_PACKAGE], + [[MS_TRUETYPE_FONTS_DIR], 'Trebuchet_MS_Bold_Italic.ttf', MS_TRUETYPE_FONTS_PACKAGE], + [[MS_TRUETYPE_FONTS_DIR], 'Trebuchet_MS_Italic.ttf', MS_TRUETYPE_FONTS_PACKAGE], + [[MS_TRUETYPE_FONTS_DIR], 'Times_New_Roman.ttf', MS_TRUETYPE_FONTS_PACKAGE], + [[MS_TRUETYPE_FONTS_DIR], 'Times_New_Roman_Bold.ttf', MS_TRUETYPE_FONTS_PACKAGE], + [[MS_TRUETYPE_FONTS_DIR], 'Times_New_Roman_Bold_Italic.ttf', MS_TRUETYPE_FONTS_PACKAGE], + [[MS_TRUETYPE_FONTS_DIR], 'Times_New_Roman_Italic.ttf', MS_TRUETYPE_FONTS_PACKAGE], + [[MS_TRUETYPE_FONTS_DIR], 'Verdana.ttf', MS_TRUETYPE_FONTS_PACKAGE], + [[MS_TRUETYPE_FONTS_DIR], 'Verdana_Bold.ttf', MS_TRUETYPE_FONTS_PACKAGE], + [[MS_TRUETYPE_FONTS_DIR], 'Verdana_Bold_Italic.ttf', MS_TRUETYPE_FONTS_PACKAGE], + [[MS_TRUETYPE_FONTS_DIR], 'Verdana_Italic.ttf', MS_TRUETYPE_FONTS_PACKAGE], # The Microsoft font EULA - ['/usr/share/doc/ttf-mscorefonts-installer/', 'READ_ME!.gz'], - ['/usr/share/fonts/truetype/ttf-dejavu/', 'DejaVuSans.ttf'], + [['/usr/share/doc/ttf-mscorefonts-installer/'], 'READ_ME!.gz', MS_TRUETYPE_FONTS_PACKAGE], + # Other fonts: Arabic, CJK, Indic, Thai, etc. + [['/usr/share/fonts/truetype/ttf-dejavu/'], 'DejaVuSans.ttf', 'ttf-dejavu'], + [['/usr/share/fonts/truetype/kochi/'], 'kochi-mincho.ttf', 'ttf-kochi-mincho'], + [['/usr/share/fonts/truetype/ttf-indic-fonts-core/'], 'lohit_hi.ttf', 'ttf-indic-fonts-core'], + [['/usr/share/fonts/truetype/ttf-indic-fonts-core/'], 'lohit_ta.ttf', 'ttf-indic-fonts-core'], + [['/usr/share/fonts/truetype/ttf-indic-fonts-core/'], 'MuktiNarrow.ttf', 'ttf-indic-fonts-core'], + [['/usr/share/fonts/truetype/thai/', '/usr/share/fonts/truetype/tlwg/'], 'Garuda.ttf', 'fonts-tlwg-garuda'], + [['/usr/share/fonts/truetype/ttf-indic-fonts-core/', '/usr/share/fonts/truetype/ttf-punjabi-fonts/'], 'lohit_pa.ttf', 'ttf-indic-fonts-core'], ] -# Should increase this version after changing HOST_FONT_FILES. -FONT_FILES_VERSION = 2 DEVICE_FONTS_DIR = DEVICE_DRT_DIR + 'fonts/' @@ -198,7 +204,8 @@ class ChromiumAndroidPort(chromium.ChromiumPort): def check_build(self, needs_http): result = super(ChromiumAndroidPort, self).check_build(needs_http) - result = self.check_wdiff() and result + result = self._check_file_exists(self._path_to_md5sum(), 'md5sum utility') and result + result = self._check_file_exists(self._path_to_forwarder(), 'forwarder utility') and result if not result: _log.error('For complete Android build requirements, please see:') _log.error('') @@ -207,11 +214,15 @@ class ChromiumAndroidPort(chromium.ChromiumPort): return result def check_sys_deps(self, needs_http): - for (font_dir, font_file) in HOST_FONT_FILES: - font_path = font_dir + font_file - if not self._check_file_exists(font_path, 'font file'): - _log.error('You are missing %s. Try installing msttcorefonts. ' - 'See build instructions.' % font_path) + for (font_dirs, font_file, package) in HOST_FONT_FILES: + exists = False + for font_dir in font_dirs: + font_path = font_dir + font_file + if self._check_file_exists(font_path, '', logging=False): + exists = True + break + if not exists: + _log.error('You are missing %s under %s. Try installing %s. See build instructions.' % (font_file, font_dirs, package)) return False return True @@ -271,6 +282,9 @@ class ChromiumAndroidPort(chromium.ChromiumPort): def _path_to_forwarder(self): return self._build_path('forwarder') + def _path_to_md5sum(self): + return self._build_path(MD5SUM_DEVICE_FILE_NAME) + def _path_to_image_diff(self): return self._host_port._path_to_image_diff() @@ -330,12 +344,10 @@ class ChromiumAndroidDriver(driver.Driver): super(ChromiumAndroidDriver, self).__del__() def _setup_md5sum_and_push_data_if_needed(self): - self._md5sum_path = self._port._build_path_with_configuration(self._port.get_option('configuration'), MD5SUM_DEVICE_FILE_NAME) - assert os.path.exists(self._md5sum_path) - + self._md5sum_path = self._port._path_to_md5sum() if not self._file_exists_on_device(MD5SUM_DEVICE_PATH): if not self._push_to_device(self._md5sum_path, MD5SUM_DEVICE_PATH): - _log.error('Could not push md5sum to device') + raise AssertionError('Could not push md5sum to device') self._push_executable() self._push_fonts() @@ -401,31 +413,20 @@ class ChromiumAndroidDriver(driver.Driver): self._abort('Failed to install %s onto device: %s' % (drt_host_path, install_result)) def _push_fonts(self): - if not self._check_version(DEVICE_FONTS_DIR, FONT_FILES_VERSION): - self._log_debug('Pushing fonts') - path_to_ahem_font = self._port._build_path('AHEM____.TTF') - self._push_to_device(path_to_ahem_font, DEVICE_FONTS_DIR + 'AHEM____.TTF') - for (host_dir, font_file) in HOST_FONT_FILES: - self._push_to_device(host_dir + font_file, DEVICE_FONTS_DIR + font_file) - self._link_device_file('/system/fonts/DroidSansFallback.ttf', DEVICE_FONTS_DIR + 'DroidSansFallback.ttf') - self._update_version(DEVICE_FONTS_DIR, FONT_FILES_VERSION) + self._log_debug('Pushing fonts') + path_to_ahem_font = self._port._build_path('AHEM____.TTF') + self._push_file_if_needed(path_to_ahem_font, DEVICE_FONTS_DIR + 'AHEM____.TTF') + for (host_dirs, font_file, package) in HOST_FONT_FILES: + for host_dir in host_dirs: + host_font_path = host_dir + font_file + if self._port._check_file_exists(host_font_path, '', logging=False): + self._push_file_if_needed(host_font_path, DEVICE_FONTS_DIR + font_file) def _push_test_resources(self): self._log_debug('Pushing test resources') for resource in TEST_RESOURCES_TO_PUSH: self._push_file_if_needed(self._port.layout_tests_dir() + '/' + resource, DEVICE_LAYOUT_TESTS_DIR + resource) - def _check_version(self, dir, version): - assert(dir.endswith('/')) - try: - device_version = int(self._run_adb_command(['shell', 'cat %sVERSION || echo 0' % dir])) - return device_version == version - except: - return False - - def _update_version(self, dir, version): - self._run_adb_command(['shell', 'echo %d > %sVERSION' % (version, dir)]) - def _run_adb_command(self, cmd, ignore_error=False): self._log_debug('Run adb command: ' + str(cmd)) if ignore_error: diff --git a/Tools/Scripts/webkitpy/layout_tests/port/chromium_android_unittest.py b/Tools/Scripts/webkitpy/layout_tests/port/chromium_android_unittest.py index 2ffb77979..bb4229e65 100644 --- a/Tools/Scripts/webkitpy/layout_tests/port/chromium_android_unittest.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/chromium_android_unittest.py @@ -30,6 +30,7 @@ import optparse import StringIO import time import unittest +import sys from webkitpy.common.system import executive_mock from webkitpy.common.system.executive_mock import MockExecutive2 @@ -244,7 +245,8 @@ class ChromiumAndroidDriverTest(unittest.TestCase): def test_command_from_driver_input(self): driver_input = driver.DriverInput('foo/bar/test.html', 10, 'checksum', True) expected_command = "/data/local/tmp/third_party/WebKit/LayoutTests/foo/bar/test.html'--pixel-test'checksum\n" - self.assertEquals(self.driver._command_from_driver_input(driver_input), expected_command) + if (sys.platform != "cygwin"): + self.assertEquals(self.driver._command_from_driver_input(driver_input), expected_command) driver_input = driver.DriverInput('http/tests/foo/bar/test.html', 10, 'checksum', True) expected_command = "http://127.0.0.1:8000/foo/bar/test.html'--pixel-test'checksum\n" diff --git a/Tools/Scripts/webkitpy/layout_tests/port/chromium_linux.py b/Tools/Scripts/webkitpy/layout_tests/port/chromium_linux.py index fa8c274ea..a2252c1b3 100644 --- a/Tools/Scripts/webkitpy/layout_tests/port/chromium_linux.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/chromium_linux.py @@ -155,14 +155,6 @@ class ChromiumLinuxPort(chromium.ChromiumPort): else: return '/usr/sbin/apache2' - def _path_to_apache_config_file(self): - if self._is_redhat_based(): - config_name = 'fedora-httpd.conf' - else: - config_name = 'apache2-debian-httpd.conf' - - return self._filesystem.join(self.layout_tests_dir(), 'http', 'conf', config_name) - def _path_to_lighttpd(self): return "/usr/sbin/lighttpd" @@ -178,12 +170,3 @@ class ChromiumLinuxPort(chromium.ChromiumPort): def _path_to_helper(self): return None - - def _path_to_wdiff(self): - if self._is_redhat_based(): - return '/usr/bin/dwdiff' - else: - return '/usr/bin/wdiff' - - def _is_redhat_based(self): - return self._filesystem.exists(self._filesystem.join('/etc', 'redhat-release')) diff --git a/Tools/Scripts/webkitpy/layout_tests/port/chromium_mac.py b/Tools/Scripts/webkitpy/layout_tests/port/chromium_mac.py index db82d5c79..08c1ede0f 100644 --- a/Tools/Scripts/webkitpy/layout_tests/port/chromium_mac.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/chromium_mac.py @@ -94,6 +94,14 @@ class ChromiumMacPort(chromium.ChromiumPort): def operating_system(self): return 'mac' + def expectations_files(self): + # FIXME: This is a temporary hack while getting the 10.8 baselines up to date. + # See https://bugs.webkit.org/show_bug.cgi?id=99505 + files = super(ChromiumMacPort, self).expectations_files() + if self.name() == 'chromium-mac-mountainlion': + files.append(self._filesystem.join(self._webkit_baseline_path(self.name()), 'TestExpectations')) + return files + # # PROTECTED METHODS # diff --git a/Tools/Scripts/webkitpy/layout_tests/port/chromium_mac_unittest.py b/Tools/Scripts/webkitpy/layout_tests/port/chromium_mac_unittest.py index edf92ea20..14e5a2b84 100644 --- a/Tools/Scripts/webkitpy/layout_tests/port/chromium_mac_unittest.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/chromium_mac_unittest.py @@ -95,6 +95,9 @@ class ChromiumMacPortTest(chromium_port_testcase.ChromiumPortTestCase): def test_path_to_image_diff(self): self.assertEquals(self.make_port()._path_to_image_diff(), '/mock-checkout/out/Release/ImageDiff') + def test_ml_expectations(self): + self.assertTrue(self.make_port(port_name='chromium-mac-mountainlion').expectations_files()[-1].endswith('-mountainlion/TestExpectations')) + self.assertFalse(self.make_port(port_name='chromium-mac-lion').expectations_files()[-1].endswith('-mountainlion/TestExpectations')) if __name__ == '__main__': port_testcase.main() diff --git a/Tools/Scripts/webkitpy/layout_tests/port/chromium_port_testcase.py b/Tools/Scripts/webkitpy/layout_tests/port/chromium_port_testcase.py index fb90d1b9b..a082a131c 100644 --- a/Tools/Scripts/webkitpy/layout_tests/port/chromium_port_testcase.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/chromium_port_testcase.py @@ -48,6 +48,13 @@ class ChromiumPortTestCase(port_testcase.PortTestCase): port = self.make_port() port.check_build(needs_http=True) + def test_default_max_locked_shards(self): + port = self.make_port() + port.default_child_processes = lambda: 16 + self.assertEquals(port.default_max_locked_shards(), 4) + port.default_child_processes = lambda: 2 + self.assertEquals(port.default_max_locked_shards(), 1) + def test_default_timeout_ms(self): self.assertEquals(self.make_port(options=MockOptions(configuration='Release')).default_timeout_ms(), 6000) self.assertEquals(self.make_port(options=MockOptions(configuration='Debug')).default_timeout_ms(), 12000) @@ -72,6 +79,8 @@ class ChromiumPortTestCase(port_testcase.PortTestCase): TestConfiguration('snowleopard', 'x86', 'release'), TestConfiguration('lion', 'x86', 'debug'), TestConfiguration('lion', 'x86', 'release'), + TestConfiguration('mountainlion', 'x86', 'debug'), + TestConfiguration('mountainlion', 'x86', 'release'), TestConfiguration('xp', 'x86', 'debug'), TestConfiguration('xp', 'x86', 'release'), TestConfiguration('win7', 'x86', 'debug'), diff --git a/Tools/Scripts/webkitpy/layout_tests/port/driver.py b/Tools/Scripts/webkitpy/layout_tests/port/driver.py index 4bf53d46a..7993d0577 100644 --- a/Tools/Scripts/webkitpy/layout_tests/port/driver.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/driver.py @@ -76,7 +76,7 @@ class DriverOutput(object): strip_patterns.append((re.compile('scrolled to [0-9]+,[0-9]+'), 'scrolled')) def __init__(self, text, image, image_hash, audio, crash=False, - test_time=0, timeout=False, error='', crashed_process_name='??', + test_time=0, measurements=None, timeout=False, error='', crashed_process_name='??', crashed_pid=None, crash_log=None): # FIXME: Args could be renamed to better clarify what they do. self.text = text @@ -89,6 +89,7 @@ class DriverOutput(object): self.crashed_pid = crashed_pid self.crash_log = crash_log self.test_time = test_time + self.measurements = measurements self.timeout = timeout self.error = error # stderr output @@ -138,6 +139,8 @@ class Driver(object): self.err_seen_eof = False self._server_process = None + self._measurements = {} + def __del__(self): self.stop() @@ -193,7 +196,7 @@ class Driver(object): crash_log += '\nstdout:\n%s\nstderr:\n%s\n' % (text, self.error_from_test) return DriverOutput(text, image, actual_image_hash, audio, - crash=crashed, test_time=time.time() - test_begin_time, + crash=crashed, test_time=time.time() - test_begin_time, measurements=self._measurements, timeout=timed_out, error=self.error_from_test, crashed_process_name=self._crashed_process_name, crashed_pid=self._crashed_pid, crash_log=crash_log) @@ -360,6 +363,10 @@ class Driver(object): def _read_first_block(self, deadline): # returns (text_content, audio_content) block = self._read_block(deadline) + if block.malloc: + self._measurements['Malloc'] = float(block.malloc) + if block.js_heap: + self._measurements['JSHeap'] = float(block.js_heap) if block.content_type == 'audio/wav': return (None, block.decoded_content) return (block.decoded_content, None) @@ -384,7 +391,9 @@ class Driver(object): if (self._read_header(block, line, 'Content-Type: ', 'content_type') or self._read_header(block, line, 'Content-Transfer-Encoding: ', 'encoding') or self._read_header(block, line, 'Content-Length: ', '_content_length', int) - or self._read_header(block, line, 'ActualHash: ', 'content_hash')): + or self._read_header(block, line, 'ActualHash: ', 'content_hash') + or self._read_header(block, line, 'DumpMalloc: ', 'malloc') + or self._read_header(block, line, 'DumpJSHeap: ', 'js_heap')): return # Note, we're not reading ExpectedHash: here, but we could. # If the line wasn't a header, we just append it to the content. @@ -450,6 +459,8 @@ class ContentBlock(object): # Content is treated as binary data even though the text output is usually UTF-8. self.content = str() # FIXME: Should be bytearray() once we require Python 2.6. self.decoded_content = None + self.malloc = None + self.js_heap = None def decode_content(self): if self.encoding == 'base64' and self.content is not None: diff --git a/Tools/Scripts/webkitpy/layout_tests/port/efl.py b/Tools/Scripts/webkitpy/layout_tests/port/efl.py index 269146aed..1022cd7b7 100644 --- a/Tools/Scripts/webkitpy/layout_tests/port/efl.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/efl.py @@ -32,7 +32,7 @@ import os from webkitpy.layout_tests.models.test_configuration import TestConfiguration from webkitpy.layout_tests.port.base import Port from webkitpy.layout_tests.port.pulseaudio_sanitizer import PulseAudioSanitizer - +from webkitpy.layout_tests.port.xvfbdriver import XvfbDriver class EflPort(Port, PulseAudioSanitizer): port_name = 'efl' @@ -80,6 +80,9 @@ class EflPort(Port, PulseAudioSanitizer): def _generate_all_test_configurations(self): return [TestConfiguration(version=self._version, architecture='x86', build_type=build_type) for build_type in self.ALL_BUILD_TYPES] + def _driver_class(self): + return XvfbDriver + def _path_to_driver(self): return self._build_path('bin', self.driver_name()) @@ -98,18 +101,22 @@ class EflPort(Port, PulseAudioSanitizer): search_paths = [] if self.get_option('webkit_test_runner'): search_paths.append(self.port_name + '-wk2') + search_paths.append('wk2') else: search_paths.append(self.port_name + '-wk1') search_paths.append(self.port_name) return search_paths def expectations_files(self): + # FIXME: We should be able to use the default algorithm here. return list(reversed([self._filesystem.join(self._webkit_baseline_path(p), 'TestExpectations') for p in self._search_paths()])) def show_results_html_file(self, results_filename): # FIXME: We should find a way to share this implmentation with Gtk, # or teach run-launcher how to call run-safari and move this down to WebKitPort. run_launcher_args = ["file://%s" % results_filename] + if self.get_option('webkit_test_runner'): + run_launcher_args.append('-2') # FIXME: old-run-webkit-tests also added ["-graphicssystem", "raster", "-style", "windows"] # FIXME: old-run-webkit-tests converted results_filename path for cygwin. self._run_script("run-launcher", run_launcher_args) diff --git a/Tools/Scripts/webkitpy/layout_tests/port/factory_unittest.py b/Tools/Scripts/webkitpy/layout_tests/port/factory_unittest.py index bb077c40a..2980c2d23 100644 --- a/Tools/Scripts/webkitpy/layout_tests/port/factory_unittest.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/factory_unittest.py @@ -105,7 +105,7 @@ class FactoryTest(unittest.TestCase): self.assertRaises(NotImplementedError, factory.PortFactory(MockSystemHost(os_name='vms')).get) def test_get_from_builder_name(self): - self.assertEquals(factory.PortFactory(MockSystemHost()).get_from_builder_name('Webkit Mac10.7').name(), + self.assertEquals(factory.PortFactory(MockSystemHost()).get_from_builder_name('WebKit Mac10.7').name(), 'chromium-mac-lion') diff --git a/Tools/Scripts/webkitpy/layout_tests/port/gtk.py b/Tools/Scripts/webkitpy/layout_tests/port/gtk.py index f02d14819..6c57b5363 100644 --- a/Tools/Scripts/webkitpy/layout_tests/port/gtk.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/gtk.py @@ -41,6 +41,9 @@ class GtkPort(Port, PulseAudioSanitizer): def expectations_files(self): return [self._filesystem.join(self._webkit_baseline_path(d), 'TestExpectations') for d in self._skipped_file_search_paths()] + def warn_if_bug_missing_in_test_expectations(self): + return True + def _port_flag_for_scripts(self): return "--gtk" @@ -89,18 +92,6 @@ class GtkPort(Port, PulseAudioSanitizer): def _path_to_image_diff(self): return self._build_path('Programs', 'ImageDiff') - def _path_to_apache(self): - if self._is_redhat_based(): - return '/usr/sbin/httpd' - else: - return '/usr/sbin/apache2' - - def _path_to_wdiff(self): - if self._is_redhat_based(): - return '/usr/bin/dwdiff' - else: - return '/usr/bin/wdiff' - def _path_to_webcore_library(self): gtk_library_names = [ "libwebkitgtk-1.0.so", diff --git a/Tools/Scripts/webkitpy/layout_tests/port/port_testcase.py b/Tools/Scripts/webkitpy/layout_tests/port/port_testcase.py index 985241ede..f704a7a13 100755 --- a/Tools/Scripts/webkitpy/layout_tests/port/port_testcase.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/port_testcase.py @@ -30,6 +30,7 @@ import errno import logging +import os import socket import sys import time @@ -89,6 +90,13 @@ class PortTestCase(unittest.TestCase): port_name = self.port_maker.determine_full_port_name(host, options, port_name) return self.port_maker(host, port_name, options=options, config=config, **kwargs) + def test_default_max_locked_shards(self): + port = self.make_port() + port.default_child_processes = lambda: 16 + self.assertEquals(port.default_max_locked_shards(), 1) + port.default_child_processes = lambda: 2 + self.assertEquals(port.default_max_locked_shards(), 1) + def test_default_timeout_ms(self): self.assertEquals(self.make_port(options=MockOptions(configuration='Release')).default_timeout_ms(), 35000) self.assertEquals(self.make_port(options=MockOptions(configuration='Debug')).default_timeout_ms(), 35000) @@ -582,10 +590,29 @@ class PortTestCase(unittest.TestCase): def test_path_to_apache_config_file(self): port = TestWebKitPort() + + saved_environ = os.environ.copy() + try: + os.environ['WEBKIT_HTTP_SERVER_CONF_PATH'] = '/path/to/httpd.conf' + self.assertRaises(IOError, port._path_to_apache_config_file) + port._filesystem.write_text_file('/existing/httpd.conf', 'Hello, world!') + os.environ['WEBKIT_HTTP_SERVER_CONF_PATH'] = '/existing/httpd.conf' + self.assertEquals(port._path_to_apache_config_file(), '/existing/httpd.conf') + finally: + os.environ = saved_environ.copy() + # Mock out _apache_config_file_name_for_platform to ignore the passed sys.platform value. port._apache_config_file_name_for_platform = lambda platform: 'httpd.conf' self.assertEquals(port._path_to_apache_config_file(), '/mock-checkout/LayoutTests/http/conf/httpd.conf') + # Check that even if we mock out _apache_config_file_name, the environment variable takes precedence. + saved_environ = os.environ.copy() + try: + os.environ['WEBKIT_HTTP_SERVER_CONF_PATH'] = '/existing/httpd.conf' + self.assertEquals(port._path_to_apache_config_file(), '/existing/httpd.conf') + finally: + os.environ = saved_environ.copy() + def test_check_build(self): port = self.make_port(options=MockOptions(build=True)) self.build_called = False diff --git a/Tools/Scripts/webkitpy/layout_tests/port/qt.py b/Tools/Scripts/webkitpy/layout_tests/port/qt.py index 828a80d64..5b8342d9c 100644 --- a/Tools/Scripts/webkitpy/layout_tests/port/qt.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/qt.py @@ -88,9 +88,9 @@ class QtPort(Port): def _path_to_webcore_library(self): if self.operating_system() == 'mac': - return self._build_path('lib/QtWebKit.framework/QtWebKit') + return self._build_path('lib/QtWebKitWidgets.framework/QtWebKitWidgets') else: - return self._build_path('lib/libQtWebKit.so') + return self._build_path('lib/libQtWebKitWidgets.so') def _modules_to_search_for_symbols(self): # We search in every library to be reliable in the case of building with CONFIG+=force_static_libs_as_shared. @@ -135,23 +135,21 @@ class QtPort(Port): search_paths.append('qt-4.8') elif version: search_paths.append('qt-5.0') - search_paths.append(self.port_name + '-' + self.host.platform.os_name) + search_paths.append(self.port_name + '-' + self.operating_system()) search_paths.append(self.port_name) return search_paths def default_baseline_search_path(self): return map(self._webkit_baseline_path, self._search_paths()) - def _skipped_file_search_paths(self): - skipped_path = self._search_paths() - if self.get_option('webkit_test_runner') and '5.0' in self.qt_version(): - skipped_path.append('wk2') - return skipped_path - def expectations_files(self): + paths = self._search_paths() + if self.get_option('webkit_test_runner'): + paths.append('wk2') + # expectations_files() uses the directories listed in _search_paths reversed. # e.g. qt -> qt-linux -> qt-4.8 - return list(reversed([self._filesystem.join(self._webkit_baseline_path(p), 'TestExpectations') for p in self._search_paths()])) + return list(reversed([self._filesystem.join(self._webkit_baseline_path(p), 'TestExpectations') for p in paths])) def setup_environ_for_server(self, server_name=None): clean_env = super(QtPort, self).setup_environ_for_server(server_name) diff --git a/Tools/Scripts/webkitpy/layout_tests/port/qt_unittest.py b/Tools/Scripts/webkitpy/layout_tests/port/qt_unittest.py index 374b10270..cf09bd8e0 100644 --- a/Tools/Scripts/webkitpy/layout_tests/port/qt_unittest.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/qt_unittest.py @@ -71,14 +71,6 @@ class QtPortTest(port_testcase.PortTestCase): absolute_search_paths = map(port._webkit_baseline_path, search_paths) self.assertEquals(port.baseline_search_path(), absolute_search_paths) - def _assert_skipped_path(self, search_paths, os_name, use_webkit2=False, qt_version='4.8'): - host = MockSystemHost(os_name=os_name) - host.executive = MockExecutive2(self._qt_version(qt_version)) - port_name = 'qt-' + os_name - port = self.make_port(host=host, qt_version=qt_version, port_name=port_name, - options=MockOptions(webkit_test_runner=use_webkit2, platform='qt')) - self.assertEquals(port._skipped_file_search_paths(), search_paths) - def _assert_expectations_files(self, search_paths, os_name, use_webkit2=False, qt_version='4.8'): # FIXME: Port constructors should not "parse" the port name, but # rather be passed components (directly or via setters). Once @@ -100,19 +92,13 @@ class QtPortTest(port_testcase.PortTestCase): for case in self.search_paths_cases: self._assert_search_path(**case) - def test_skipped_file_search_path(self): - caselist = self.search_paths_cases[:] - for case in caselist: - if case['use_webkit2'] and case['qt_version'] == '5.0': - case['search_paths'].append("wk2") - self._assert_skipped_path(**case) - def test_expectations_files(self): for case in self.search_paths_cases: expectations_case = deepcopy(case) - expectations_case['search_paths'] = [] - for path in reversed(case['search_paths']): - expectations_case['search_paths'].append('/mock-checkout/LayoutTests/platform/%s/TestExpectations' % (path)) + if expectations_case['use_webkit2']: + expectations_case['search_paths'].append("wk2") + expectations_case['search_paths'].reverse() + expectations_case['search_paths'] = map(lambda path: '/mock-checkout/LayoutTests/platform/%s/TestExpectations' % (path), expectations_case['search_paths']) self._assert_expectations_files(**expectations_case) def test_show_results_html_file(self): diff --git a/Tools/Scripts/webkitpy/layout_tests/port/server_process.py b/Tools/Scripts/webkitpy/layout_tests/port/server_process.py index bfa6aab80..bfdf8301b 100644 --- a/Tools/Scripts/webkitpy/layout_tests/port/server_process.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/server_process.py @@ -84,6 +84,17 @@ class ServerProcess(object): return self._pid def _reset(self): + if getattr(self, '_proc', None): + if self._proc.stdin: + self._proc.stdin.close() + self._proc.stdin = None + if self._proc.stdout: + self._proc.stdout.close() + self._proc.stdout = None + if self._proc.stderr: + self._proc.stderr.close() + self._proc.stderr = None + self._proc = None self._output = str() # bytesarray() once we require Python 2.6 self._error = str() # bytesarray() once we require Python 2.6 @@ -321,8 +332,11 @@ class ServerProcess(object): now = time.time() self._proc.stdin.close() + self._proc.stdin = None + killed = False if not timeout_secs: self._kill() + killed = True elif not self._host.platform.is_win(): # FIXME: Why aren't we calling this on win? deadline = now + timeout_secs @@ -331,13 +345,15 @@ class ServerProcess(object): if self._proc.poll() is None: _log.warning('stopping %s timed out, killing it' % self._name) self._kill() + killed = True _log.warning('killed') # read any remaining data on the pipes and return it. - if self._use_win32_apis: - self._wait_for_data_and_update_buffers_using_win32_apis(now) - else: - self._wait_for_data_and_update_buffers_using_select(now, stopping=True) + if not killed: + if self._use_win32_apis: + self._wait_for_data_and_update_buffers_using_win32_apis(now) + else: + self._wait_for_data_and_update_buffers_using_select(now, stopping=True) out, err = self._output, self._error self._reset() return (out, err) diff --git a/Tools/Scripts/webkitpy/layout_tests/port/server_process_unittest.py b/Tools/Scripts/webkitpy/layout_tests/port/server_process_unittest.py index 48c41e6f2..7a5ac45d4 100644 --- a/Tools/Scripts/webkitpy/layout_tests/port/server_process_unittest.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/server_process_unittest.py @@ -57,6 +57,7 @@ class TrivialMockPort(object): class MockFile(object): def __init__(self, server_process): self._server_process = server_process + self.closed = False def fileno(self): return 1 @@ -66,7 +67,7 @@ class MockFile(object): raise IOError def close(self): - pass + self.closed = True class MockProc(object): @@ -87,6 +88,8 @@ class FakeServerProcess(server_process.ServerProcess): def _start(self): self._proc = MockProc(self) self.stdin = self._proc.stdin + self.stdout = self._proc.stdout + self.stderr = self._proc.stderr self._pid = self._proc.pid self.broken_pipes = [] @@ -121,6 +124,15 @@ class TestServerProcess(unittest.TestCase): proc.stop(0) + def test_cleanup(self): + port_obj = TrivialMockPort() + server_process = FakeServerProcess(port_obj=port_obj, name="test", cmd=["test"]) + server_process._start() + server_process.stop() + self.assertTrue(server_process.stdin.closed) + self.assertTrue(server_process.stdout.closed) + self.assertTrue(server_process.stderr.closed) + def test_broken_pipe(self): port_obj = TrivialMockPort() diff --git a/Tools/Scripts/webkitpy/layout_tests/port/test.py b/Tools/Scripts/webkitpy/layout_tests/port/test.py index d008d995d..726575614 100644 --- a/Tools/Scripts/webkitpy/layout_tests/port/test.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/test.py @@ -239,6 +239,11 @@ layer at (0,0) size 800x34 actual_image='image_not_in_pixeldir-pngtEXtchecksum\x00checksum_fail', expected_image='image_not_in_pixeldir-pngtEXtchecksum\x00checksum-png') + # For testing that virtual test suites don't expand names containing themselves + # See webkit.org/b/97925 and base_unittest.PortTest.test_tests(). + tests.add('passes/test-virtual-passes.html') + tests.add('passes/passes/test-virtual-passes.html') + return tests diff --git a/Tools/Scripts/webkitpy/layout_tests/port/xvfbdriver.py b/Tools/Scripts/webkitpy/layout_tests/port/xvfbdriver.py index 7e386a108..b927720db 100644 --- a/Tools/Scripts/webkitpy/layout_tests/port/xvfbdriver.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/xvfbdriver.py @@ -28,41 +28,60 @@ import logging import os +import re +import time from webkitpy.layout_tests.port.server_process import ServerProcess from webkitpy.layout_tests.port.driver import Driver +from webkitpy.common.system.file_lock import FileLock _log = logging.getLogger(__name__) class XvfbDriver(Driver): - def _start(self, pixel_tests, per_test_args): - - # Collect the number of X servers running already and make - # sure our Xvfb process doesn't clash with any of them. - def x_filter(process_name): - return process_name.find("Xorg") > -1 + def __init__(self, *args, **kwargs): + Driver.__init__(self, *args, **kwargs) + self._guard_lock = None + self._startup_delay_secs = 1.0 - running_displays = len(self._port.host.executive.running_pids(x_filter)) + def _next_free_display(self): + running_pids = self._port.host.executive.run_command(['ps', '-eo', 'comm,command']) + reserved_screens = set() + for pid in running_pids.split('\n'): + match = re.match('(X|Xvfb|Xorg)\s+.*\s:(?P<screen_number>\d+)', pid) + if match: + reserved_screens.add(int(match.group('screen_number'))) + for i in range(99): + if i not in reserved_screens: + _guard_lock_file = self._port.host.filesystem.join('/tmp', 'WebKitXvfb.lock.%i' % i) + self._guard_lock = FileLock(_guard_lock_file) + if self._guard_lock.acquire_lock(): + return i + def _start(self, pixel_tests, per_test_args): # Use even displays for pixel tests and odd ones otherwise. When pixel tests are disabled, # DriverProxy creates two drivers, one for normal and the other for ref tests. Both have # the same worker number, so this prevents them from using the same Xvfb instance. - display_id = self._worker_number * 2 + running_displays - if pixel_tests: - display_id += 1 + display_id = self._next_free_display() self._lock_file = "/tmp/.X%d-lock" % display_id run_xvfb = ["Xvfb", ":%d" % display_id, "-screen", "0", "800x600x24", "-nolisten", "tcp"] with open(os.devnull, 'w') as devnull: self._xvfb_process = self._port.host.executive.popen(run_xvfb, stderr=devnull) + # Crashes intend to occur occasionally in the first few tests that are run through each + # worker because the Xvfb display isn't ready yet. Halting execution a bit should avoid that. + time.sleep(self._startup_delay_secs) + server_name = self._port.driver_name() environment = self._port.setup_environ_for_server(server_name) # We must do this here because the DISPLAY number depends on _worker_number environment['DISPLAY'] = ":%d" % display_id # Drivers should use separate application cache locations environment['XDG_CACHE_HOME'] = self._port.host.filesystem.join(self._port.results_directory(), '%s-appcache-%d' % (server_name, self._worker_number)) + self._driver_tempdir = self._port._filesystem.mkdtemp(prefix='%s-' % self._port.driver_name()) + environment['DUMPRENDERTREE_TEMP'] = str(self._driver_tempdir) + environment['LOCAL_RESOURCE_ROOT'] = self._port.layout_tests_dir() self._crashed_process_name = None self._crashed_pid = None @@ -71,6 +90,9 @@ class XvfbDriver(Driver): def stop(self): super(XvfbDriver, self).stop() + if self._guard_lock: + self._guard_lock.release_lock() + self._guard_lock = None if getattr(self, '_xvfb_process', None): self._port.host.executive.kill_process(self._xvfb_process.pid) self._xvfb_process = None diff --git a/Tools/Scripts/webkitpy/layout_tests/port/xvfbdriver_unittest.py b/Tools/Scripts/webkitpy/layout_tests/port/xvfbdriver_unittest.py index 37a2fbd4d..220dd3517 100644 --- a/Tools/Scripts/webkitpy/layout_tests/port/xvfbdriver_unittest.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/xvfbdriver_unittest.py @@ -30,6 +30,7 @@ import unittest from webkitpy.common.system.deprecated_logging import log from webkitpy.common.system.filesystem_mock import MockFileSystem +from webkitpy.common.system.executive_mock import MockExecutive2 from webkitpy.common.system.outputcapture import OutputCapture from webkitpy.common.system.systemhost_mock import MockSystemHost from webkitpy.layout_tests.port import Port @@ -39,13 +40,14 @@ from webkitpy.layout_tests.port.xvfbdriver import XvfbDriver class XvfbDriverTest(unittest.TestCase): - def make_driver(self, worker_number=0, xorg_running=False): - port = Port(host=MockSystemHost(log_executive=True), config=MockConfig()) + def make_driver(self, worker_number=0, xorg_running=False, executive=None): + port = Port(host=MockSystemHost(log_executive=True, executive=executive), config=MockConfig()) port._server_process_constructor = MockServerProcess if xorg_running: port._executive._running_pids['Xorg'] = 108 driver = XvfbDriver(port, worker_number=worker_number, pixel_tests=True) + driver._startup_delay_secs = 0 return driver def cleanup_driver(self, driver): @@ -61,26 +63,54 @@ class XvfbDriverTest(unittest.TestCase): def test_start_no_pixel_tests(self): driver = self.make_driver() - expected_stderr = "MOCK running_pids: []\nMOCK popen: ['Xvfb', ':0', '-screen', '0', '800x600x24', '-nolisten', 'tcp']\n" + expected_stderr = "MOCK run_command: ['ps', '-eo', 'comm,command'], cwd=None\nMOCK popen: ['Xvfb', ':0', '-screen', '0', '800x600x24', '-nolisten', 'tcp']\n" self.assertDriverStartSuccessful(driver, expected_stderr=expected_stderr, expected_display=":0") self.cleanup_driver(driver) def test_start_pixel_tests(self): driver = self.make_driver() - expected_stderr = "MOCK running_pids: []\nMOCK popen: ['Xvfb', ':1', '-screen', '0', '800x600x24', '-nolisten', 'tcp']\n" - self.assertDriverStartSuccessful(driver, expected_stderr=expected_stderr, expected_display=":1", pixel_tests=True) + expected_stderr = "MOCK run_command: ['ps', '-eo', 'comm,command'], cwd=None\nMOCK popen: ['Xvfb', ':0', '-screen', '0', '800x600x24', '-nolisten', 'tcp']\n" + self.assertDriverStartSuccessful(driver, expected_stderr=expected_stderr, expected_display=":0", pixel_tests=True) self.cleanup_driver(driver) def test_start_arbitrary_worker_number(self): driver = self.make_driver(worker_number=17) - expected_stderr = "MOCK running_pids: []\nMOCK popen: ['Xvfb', ':35', '-screen', '0', '800x600x24', '-nolisten', 'tcp']\n" - self.assertDriverStartSuccessful(driver, expected_stderr=expected_stderr, expected_display=":35", pixel_tests=True) + expected_stderr = "MOCK run_command: ['ps', '-eo', 'comm,command'], cwd=None\nMOCK popen: ['Xvfb', ':0', '-screen', '0', '800x600x24', '-nolisten', 'tcp']\n" + self.assertDriverStartSuccessful(driver, expected_stderr=expected_stderr, expected_display=":0", pixel_tests=True) self.cleanup_driver(driver) - def test_start_existing_xorg_process(self): - driver = self.make_driver(xorg_running=True) - expected_stderr = "MOCK running_pids: [108]\nMOCK popen: ['Xvfb', ':1', '-screen', '0', '800x600x24', '-nolisten', 'tcp']\n" - self.assertDriverStartSuccessful(driver, expected_stderr=expected_stderr, expected_display=":1") + def disabled_test_next_free_display(self): + output = "Xorg /usr/bin/X :0 -auth /var/run/lightdm/root/:0 -nolisten tcp vt7 -novtswitch -background none\nXvfb Xvfb :1 -screen 0 800x600x24 -nolisten tcp" + executive = MockExecutive2(output) + driver = self.make_driver(executive=executive) + self.assertEqual(driver._next_free_display(), 2) + self.cleanup_driver(driver) + output = "X /usr/bin/X :0 vt7 -nolisten tcp -auth /var/run/xauth/A:0-8p7Ybb" + executive = MockExecutive2(output) + driver = self.make_driver(executive=executive) + self.assertEqual(driver._next_free_display(), 1) + self.cleanup_driver(driver) + output = "Xvfb Xvfb :0 -screen 0 800x600x24 -nolisten tcp" + executive = MockExecutive2(output) + driver = self.make_driver(executive=executive) + self.assertEqual(driver._next_free_display(), 1) + self.cleanup_driver(driver) + output = "Xvfb Xvfb :1 -screen 0 800x600x24 -nolisten tcp\nXvfb Xvfb :0 -screen 0 800x600x24 -nolisten tcp\nXvfb Xvfb :3 -screen 0 800x600x24 -nolisten tcp" + executive = MockExecutive2(output) + driver = self.make_driver(executive=executive) + self.assertEqual(driver._next_free_display(), 2) + self.cleanup_driver(driver) + + def test_start_next_worker(self): + driver = self.make_driver() + driver._next_free_display = lambda: 0 + expected_stderr = "MOCK popen: ['Xvfb', ':0', '-screen', '0', '800x600x24', '-nolisten', 'tcp']\n" + self.assertDriverStartSuccessful(driver, expected_stderr=expected_stderr, expected_display=":0", pixel_tests=True) + self.cleanup_driver(driver) + driver = self.make_driver() + driver._next_free_display = lambda: 3 + expected_stderr = "MOCK popen: ['Xvfb', ':3', '-screen', '0', '800x600x24', '-nolisten', 'tcp']\n" + self.assertDriverStartSuccessful(driver, expected_stderr=expected_stderr, expected_display=":3", pixel_tests=True) self.cleanup_driver(driver) def test_stop(self): diff --git a/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests.py b/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests.py index e784cb61d..89522079c 100755 --- a/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests.py +++ b/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests.py @@ -129,6 +129,9 @@ def _set_up_derived_options(port, options): if not options.child_processes: options.child_processes = os.environ.get("WEBKIT_TEST_CHILD_PROCESSES", str(port.default_child_processes())) + if not options.max_locked_shards: + options.max_locked_shards = int(os.environ.get("WEBKIT_TEST_MAX_LOCKED_SHARDS", + str(port.default_max_locked_shards()))) if not options.configuration: options.configuration = port.default_configuration() @@ -375,7 +378,11 @@ def parse_args(args=None): optparse.make_option("--test-list", action="append", help="read list of tests to run from file", metavar="FILE"), optparse.make_option("--skipped", action="store", default="default", - help="control how tests marked SKIP are run. 'default' == Skip, 'ignore' == Run them anyway, 'only' == only run the SKIP tests."), + help=("control how tests marked SKIP are run. " + "'default' == Skip tests unless explicitly listed on the command line, " + "'ignore' == Run them anyway, " + "'only' == only run the SKIP tests, " + "'always' == always skip, even if listed on the command line.")), optparse.make_option("--force", dest="skipped", action="store_const", const='ignore', help="Run all tests, even those marked SKIP in the test list (same as --skipped=ignore)"), optparse.make_option("--time-out-ms", @@ -412,7 +419,7 @@ def parse_args(args=None): optparse.make_option("--no-retry-failures", action="store_false", dest="retry_failures", help="Don't re-try any tests that produce unexpected results."), - optparse.make_option("--max-locked-shards", type="int", default=1, + optparse.make_option("--max-locked-shards", type="int", default=0, help="Set the maximum number of locked shards"), optparse.make_option("--additional-env-var", type="string", action="append", default=[], help="Passes that environment variable to the tests (--additional-env-var=NAME=VALUE)"), diff --git a/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py b/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py index 4afcc1466..85437449b 100755 --- a/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py +++ b/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py @@ -301,6 +301,12 @@ class MainTest(unittest.TestCase, StreamTestingMixin): for batch in batch_tests_run: self.assertTrue(len(batch) <= 2, '%s had too many tests' % ', '.join(batch)) + def test_max_locked_shards(self): + if not self.should_test_processes: + return + _, _, regular_output, _ = logging_run(['--debug-rwt-logging', '--child-processes', '2'], shared_port=False) + self.assertTrue(any(['(1 locked)' in line for line in regular_output.buflist])) + def test_child_processes_2(self): if self.should_test_processes: _, _, regular_output, _ = logging_run( @@ -310,7 +316,7 @@ class MainTest(unittest.TestCase, StreamTestingMixin): def test_child_processes_min(self): if self.should_test_processes: _, _, regular_output, _ = logging_run( - ['--debug-rwt-logging', '--child-processes', '2', 'passes'], + ['--debug-rwt-logging', '--child-processes', '2', '-i', 'passes/passes', 'passes'], tests_included=True, shared_port=False) self.assertTrue(any(['Running 1 ' in line for line in regular_output.buflist])) @@ -418,6 +424,10 @@ class MainTest(unittest.TestCase, StreamTestingMixin): self.assertEquals(get_tests_run(['--skipped=only', 'passes'], tests_included=True, flatten_batches=True), ['passes/skipped/skip.html']) + # Now check that we don't run anything. + self.assertEquals(get_tests_run(['--skipped=always', 'passes/skipped/skip.html'], tests_included=True, flatten_batches=True), + []) + def test_iterations(self): tests_to_run = ['passes/image.html', 'passes/text.html'] tests_run = get_tests_run(['--iterations', '2'] + tests_to_run, tests_included=True, flatten_batches=True) @@ -775,7 +785,7 @@ class MainTest(unittest.TestCase, StreamTestingMixin): # These next tests test that we run the tests in ascending alphabetical # order per directory. HTTP tests are sharded separately from other tests, # so we have to test both. - tests_run = get_tests_run(['passes'], tests_included=True, flatten_batches=True) + tests_run = get_tests_run(['-i', 'passes/passes', 'passes'], tests_included=True, flatten_batches=True) self.assertEquals(tests_run, sorted(tests_run)) tests_run = get_tests_run(['http/tests/passes'], tests_included=True, flatten_batches=True) @@ -922,6 +932,11 @@ class MainTest(unittest.TestCase, StreamTestingMixin): # child process (e.g., on win32) and we need to make sure that works and we still # see the verbose log output. However, we can't use logging_run() because using # outputcapture to capture stdout and stderr latter results in a nonpicklable host. + + # Test is flaky on Windows: https://bugs.webkit.org/show_bug.cgi?id=98559 + if not self.should_test_processes: + return + options, parsed_args = parse_args(['--verbose', '--fully-parallel', '--child-processes', '2', 'passes/text.html', 'passes/image.html'], tests_included=True, print_nothing=False) host = MockHost() port_obj = host.port_factory.get(port_name=options.platform, options=options) diff --git a/Tools/Scripts/webkitpy/layout_tests/servers/apache_http_server.py b/Tools/Scripts/webkitpy/layout_tests/servers/apache_http_server.py index 7cd4af538..a616fab5b 100644 --- a/Tools/Scripts/webkitpy/layout_tests/servers/apache_http_server.py +++ b/Tools/Scripts/webkitpy/layout_tests/servers/apache_http_server.py @@ -54,7 +54,6 @@ class LayoutTestApacheHttpd(http_server_base.HttpServerBase): self._name = 'httpd' self._mappings = [{'port': 8000}, {'port': 8080}, - {'port': 8081}, {'port': 8443, 'sslcert': True}] self._output_dir = output_dir self._filesystem.maybe_make_directory(output_dir) @@ -79,7 +78,6 @@ class LayoutTestApacheHttpd(http_server_base.HttpServerBase): '-c', "\'Alias /js-test-resources \"%s\"'" % js_test_resources_dir, '-c', "\'Alias /media-resources \"%s\"'" % media_resources_dir, '-C', "\'Listen %s\'" % "127.0.0.1:8000", - '-C', "\'Listen %s\'" % "127.0.0.1:8081", '-c', "\'TypesConfig \"%s\"\'" % mime_types_path, '-c', "\'CustomLog \"%s\" common\'" % access_log, '-c', "\'ErrorLog \"%s\"\'" % error_log, diff --git a/Tools/Scripts/webkitpy/performance_tests/perftest.py b/Tools/Scripts/webkitpy/performance_tests/perftest.py index fdac35b11..32b9d8bc6 100644 --- a/Tools/Scripts/webkitpy/performance_tests/perftest.py +++ b/Tools/Scripts/webkitpy/performance_tests/perftest.py @@ -1,5 +1,6 @@ #!/usr/bin/env python # Copyright (C) 2012 Google Inc. All rights reserved. +# Copyright (C) 2012 Zoltan Horvath, Adobe Systems Incorporated. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are @@ -202,11 +203,40 @@ class ChromiumStylePerfTest(PerfTest): class PageLoadingPerfTest(PerfTest): + _FORCE_GC_FILE = 'resources/force-gc.html' + def __init__(self, port, test_name, path_or_url): super(PageLoadingPerfTest, self).__init__(port, test_name, path_or_url) + self.force_gc_test = self._port.host.filesystem.join(self._port.perf_tests_dir(), self._FORCE_GC_FILE) + + def run_single(self, driver, path_or_url, time_out_ms, should_run_pixel_test=False): + # Force GC to prevent pageload noise. See https://bugs.webkit.org/show_bug.cgi?id=98203 + super(PageLoadingPerfTest, self).run_single(driver, self.force_gc_test, time_out_ms, False) + return super(PageLoadingPerfTest, self).run_single(driver, path_or_url, time_out_ms, should_run_pixel_test) + + def calculate_statistics(self, values): + sorted_values = sorted(values) + + # Compute the mean and variance using Knuth's online algorithm (has good numerical stability). + squareSum = 0 + mean = 0 + for i, time in enumerate(sorted_values): + delta = time - mean + sweep = i + 1.0 + mean += delta / sweep + squareSum += delta * (time - mean) + + middle = int(len(sorted_values) / 2) + result = {'avg': mean, + 'min': sorted_values[0], + 'max': sorted_values[-1], + 'median': sorted_values[middle] if len(sorted_values) % 2 else (sorted_values[middle - 1] + sorted_values[middle]) / 2, + 'stdev': math.sqrt(squareSum / (len(sorted_values) - 1))} + return result def run(self, driver, time_out_ms): - test_times = [] + results = {} + results.setdefault(self.test_name(), {'unit': 'ms', 'values': []}) for i in range(0, 20): output = self.run_single(driver, self.path_or_url(), time_out_ms) @@ -214,30 +244,25 @@ class PageLoadingPerfTest(PerfTest): return None if i == 0: continue - test_times.append(output.test_time * 1000) - sorted_test_times = sorted(test_times) + results[self.test_name()]['values'].append(output.test_time * 1000) - # Compute the mean and variance using a numerically stable algorithm. - squareSum = 0 - mean = 0 - valueSum = sum(sorted_test_times) - for i, time in enumerate(sorted_test_times): - delta = time - mean - sweep = i + 1.0 - mean += delta / sweep - squareSum += delta * delta * (i / sweep) - - middle = int(len(test_times) / 2) - results = {'values': test_times, - 'avg': mean, - 'min': sorted_test_times[0], - 'max': sorted_test_times[-1], - 'median': sorted_test_times[middle] if len(sorted_test_times) % 2 else (sorted_test_times[middle - 1] + sorted_test_times[middle]) / 2, - 'stdev': math.sqrt(squareSum), - 'unit': 'ms'} - self.output_statistics(self.test_name(), results, '') - return {self.test_name(): results} + if not output.measurements: + continue + + for result_class, result in output.measurements.items(): + name = self.test_name() + ':' + result_class + if not name in results: + results.setdefault(name, {'values': []}) + results[name]['values'].append(result) + if result_class == 'Malloc' or result_class == 'JSHeap': + results[name]['unit'] = 'bytes' + + for result_class in results.keys(): + results[result_class].update(self.calculate_statistics(results[result_class]['values'])) + self.output_statistics(result_class, results[result_class], '') + + return results class ReplayServer(object): diff --git a/Tools/Scripts/webkitpy/performance_tests/perftest_unittest.py b/Tools/Scripts/webkitpy/performance_tests/perftest_unittest.py index 27a4bb385..4410903e9 100755 --- a/Tools/Scripts/webkitpy/performance_tests/perftest_unittest.py +++ b/Tools/Scripts/webkitpy/performance_tests/perftest_unittest.py @@ -43,6 +43,10 @@ from webkitpy.performance_tests.perftest import PerfTestFactory from webkitpy.performance_tests.perftest import ReplayPerfTest +class MockPort(TestPort): + def __init__(self, custom_run_test=None): + super(MockPort, self).__init__(host=MockHost(), custom_run_test=custom_run_test) + class MainTest(unittest.TestCase): def test_parse_output(self): output = DriverOutput('\n'.join([ @@ -98,39 +102,69 @@ class MainTest(unittest.TestCase): class TestPageLoadingPerfTest(unittest.TestCase): class MockDriver(object): - def __init__(self, values): + def __init__(self, values, test, measurements=None): self._values = values self._index = 0 + self._test = test + self._measurements = measurements def run_test(self, input, stop_when_done): + if input.test_name == self._test.force_gc_test: + return value = self._values[self._index] self._index += 1 if isinstance(value, str): return DriverOutput('some output', image=None, image_hash=None, audio=None, error=value) else: - return DriverOutput('some output', image=None, image_hash=None, audio=None, test_time=self._values[self._index - 1]) + return DriverOutput('some output', image=None, image_hash=None, audio=None, test_time=self._values[self._index - 1], measurements=self._measurements) def test_run(self): - test = PageLoadingPerfTest(None, 'some-test', '/path/some-dir/some-test') - driver = TestPageLoadingPerfTest.MockDriver(range(1, 21)) + port = MockPort() + test = PageLoadingPerfTest(port, 'some-test', '/path/some-dir/some-test') + driver = TestPageLoadingPerfTest.MockDriver(range(1, 21), test) output_capture = OutputCapture() output_capture.capture_output() try: self.assertEqual(test.run(driver, None), - {'some-test': {'max': 20000, 'avg': 11000.0, 'median': 11000, 'stdev': math.sqrt(570 * 1000 * 1000), 'min': 2000, 'unit': 'ms', + {'some-test': {'max': 20000, 'avg': 11000.0, 'median': 11000, 'stdev': 5627.314338711378, 'min': 2000, 'unit': 'ms', 'values': [i * 1000 for i in range(2, 21)]}}) finally: actual_stdout, actual_stderr, actual_logs = output_capture.restore_output() self.assertEqual(actual_stdout, '') self.assertEqual(actual_stderr, '') - self.assertEqual(actual_logs, 'RESULT some-test= 11000.0 ms\nmedian= 11000 ms, stdev= 23874.6727726 ms, min= 2000 ms, max= 20000 ms\n') + self.assertEqual(actual_logs, 'RESULT some-test= 11000.0 ms\nmedian= 11000 ms, stdev= 5627.31433871 ms, min= 2000 ms, max= 20000 ms\n') + + def test_run_with_memory_output(self): + port = MockPort() + test = PageLoadingPerfTest(port, 'some-test', '/path/some-dir/some-test') + memory_results = {'Malloc': 10, 'JSHeap': 5} + self.maxDiff = None + driver = TestPageLoadingPerfTest.MockDriver(range(1, 21), test, memory_results) + output_capture = OutputCapture() + output_capture.capture_output() + try: + self.assertEqual(test.run(driver, None), + {'some-test': {'max': 20000, 'avg': 11000.0, 'median': 11000, 'stdev': 5627.314338711378, 'min': 2000, 'unit': 'ms', + 'values': [i * 1000 for i in range(2, 21)]}, + 'some-test:Malloc': {'max': 10, 'avg': 10.0, 'median': 10, 'min': 10, 'stdev': 0.0, 'unit': 'bytes', + 'values': [10] * 19}, + 'some-test:JSHeap': {'max': 5, 'avg': 5.0, 'median': 5, 'min': 5, 'stdev': 0.0, 'unit': 'bytes', + 'values': [5] * 19}}) + finally: + actual_stdout, actual_stderr, actual_logs = output_capture.restore_output() + self.assertEqual(actual_stdout, '') + self.assertEqual(actual_stderr, '') + self.assertEqual(actual_logs, 'RESULT some-test= 11000.0 ms\nmedian= 11000 ms, stdev= 5627.31433871 ms, min= 2000 ms, max= 20000 ms\n' + + 'RESULT some-test: Malloc= 10.0 bytes\nmedian= 10 bytes, stdev= 0.0 bytes, min= 10 bytes, max= 10 bytes\n' + + 'RESULT some-test: JSHeap= 5.0 bytes\nmedian= 5 bytes, stdev= 0.0 bytes, min= 5 bytes, max= 5 bytes\n') def test_run_with_bad_output(self): output_capture = OutputCapture() output_capture.capture_output() try: - test = PageLoadingPerfTest(None, 'some-test', '/path/some-dir/some-test') - driver = TestPageLoadingPerfTest.MockDriver([1, 2, 3, 4, 5, 6, 7, 'some error', 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20]) + port = MockPort() + test = PageLoadingPerfTest(port, 'some-test', '/path/some-dir/some-test') + driver = TestPageLoadingPerfTest.MockDriver([1, 2, 3, 4, 5, 6, 7, 'some error', 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20], test) self.assertEqual(test.run(driver, None), None) finally: actual_stdout, actual_stderr, actual_logs = output_capture.restore_output() @@ -141,7 +175,7 @@ class TestPageLoadingPerfTest(unittest.TestCase): class TestReplayPerfTest(unittest.TestCase): - class ReplayTestPort(TestPort): + class ReplayTestPort(MockPort): def __init__(self, custom_run_test=None): class ReplayTestDriver(TestDriver): @@ -149,7 +183,7 @@ class TestReplayPerfTest(unittest.TestCase): return custom_run_test(text_input, stop_when_done) if custom_run_test else None self._custom_driver_class = ReplayTestDriver - super(self.__class__, self).__init__(host=MockHost()) + super(self.__class__, self).__init__() def _driver_class(self): return self._custom_driver_class @@ -179,6 +213,9 @@ class TestReplayPerfTest(unittest.TestCase): loaded_pages = [] def run_test(test_input, stop_when_done): + if test_input.test_name == test.force_gc_test: + loaded_pages.append(test_input) + return if test_input.test_name != "about:blank": self.assertEqual(test_input.test_name, 'http://some-test/') loaded_pages.append(test_input) @@ -196,8 +233,9 @@ class TestReplayPerfTest(unittest.TestCase): finally: actual_stdout, actual_stderr, actual_logs = output_capture.restore_output() - self.assertEqual(len(loaded_pages), 1) - self.assertEqual(loaded_pages[0].test_name, 'http://some-test/') + self.assertEqual(len(loaded_pages), 2) + self.assertEqual(loaded_pages[0].test_name, test.force_gc_test) + self.assertEqual(loaded_pages[1].test_name, 'http://some-test/') self.assertEqual(actual_stdout, '') self.assertEqual(actual_stderr, '') self.assertEqual(actual_logs, '') @@ -262,8 +300,9 @@ class TestReplayPerfTest(unittest.TestCase): finally: actual_stdout, actual_stderr, actual_logs = output_capture.restore_output() - self.assertEqual(len(loaded_pages), 1) - self.assertEqual(loaded_pages[0].test_name, 'http://some-test/') + self.assertEqual(len(loaded_pages), 2) + self.assertEqual(loaded_pages[0].test_name, test.force_gc_test) + self.assertEqual(loaded_pages[1].test_name, 'http://some-test/') self.assertEqual(actual_stdout, '') self.assertEqual(actual_stderr, '') self.assertEqual(actual_logs, 'error: some-test.replay\nsome error\n') @@ -316,15 +355,15 @@ class TestReplayPerfTest(unittest.TestCase): class TestPerfTestFactory(unittest.TestCase): def test_regular_test(self): - test = PerfTestFactory.create_perf_test(None, 'some-dir/some-test', '/path/some-dir/some-test') + test = PerfTestFactory.create_perf_test(MockPort(), 'some-dir/some-test', '/path/some-dir/some-test') self.assertEqual(test.__class__, PerfTest) def test_inspector_test(self): - test = PerfTestFactory.create_perf_test(None, 'inspector/some-test', '/path/inspector/some-test') + test = PerfTestFactory.create_perf_test(MockPort(), 'inspector/some-test', '/path/inspector/some-test') self.assertEqual(test.__class__, ChromiumStylePerfTest) def test_page_loading_test(self): - test = PerfTestFactory.create_perf_test(None, 'PageLoad/some-test', '/path/PageLoad/some-test') + test = PerfTestFactory.create_perf_test(MockPort(), 'PageLoad/some-test', '/path/PageLoad/some-test') self.assertEqual(test.__class__, PageLoadingPerfTest) diff --git a/Tools/Scripts/webkitpy/performance_tests/perftestsrunner.py b/Tools/Scripts/webkitpy/performance_tests/perftestsrunner.py index c34d0b3e4..42e0d96e1 100755 --- a/Tools/Scripts/webkitpy/performance_tests/perftestsrunner.py +++ b/Tools/Scripts/webkitpy/performance_tests/perftestsrunner.py @@ -29,19 +29,17 @@ """Run Inspector's perf tests in perf mode.""" +import os import json import logging import optparse -import re -import sys import time from webkitpy.common import find_files +from webkitpy.common.checkout.scm.detection import SCMDetector from webkitpy.common.host import Host from webkitpy.common.net.file_uploader import FileUploader -from webkitpy.layout_tests.views import printing from webkitpy.performance_tests.perftest import PerfTestFactory -from webkitpy.performance_tests.perftest import ReplayPerfTest _log = logging.getLogger(__name__) @@ -65,7 +63,7 @@ class PerfTestsRunner(object): else: self._host = Host() self._port = self._host.port_factory.get(self._options.platform, self._options) - self._host._initialize_scm() + self._host.initialize_scm() self._webkit_base_dir_len = len(self._port.webkit_base()) self._base_path = self._port.perf_tests_dir() self._results = {} @@ -73,6 +71,9 @@ class PerfTestsRunner(object): @staticmethod def _parse_args(args=None): + def _expand_path(option, opt_str, value, parser): + path = os.path.expandvars(os.path.expanduser(value)) + setattr(parser.values, option.dest, path) perf_option_list = [ optparse.make_option('--debug', action='store_const', const='Debug', dest="configuration", help='Set the configuration to Debug'), @@ -98,15 +99,12 @@ class PerfTestsRunner(object): help="Pause before running the tests to let user attach a performance monitor."), optparse.make_option("--no-results", action="store_false", dest="generate_results", default=True, help="Do no generate results JSON and results page."), - optparse.make_option("--output-json-path", + optparse.make_option("--output-json-path", action='callback', callback=_expand_path, type="str", help="Path to generate a JSON file at; may contain previous results if it already exists."), optparse.make_option("--reset-results", action="store_true", help="Clears the content in the generated JSON file before adding the results."), - optparse.make_option("--slave-config-json-path", + optparse.make_option("--slave-config-json-path", action='callback', callback=_expand_path, type="str", help="Only used on bots. Path to a slave configuration file."), - optparse.make_option("--source-json-path", dest="slave_config_json_path", - # FIXME: Remove this option once build.webkit.org is updated to use --slave-config-json-path. - help="Deprecated. Overrides --slave-config-json-path."), optparse.make_option("--description", help="Add a description to the output JSON file if one is generated"), optparse.make_option("--no-show-results", action="store_false", default=True, dest="show_results", @@ -181,11 +179,6 @@ class PerfTestsRunner(object): def _generate_and_show_results(self): options = self._options - if options.test_results_server: - # Remove this code once build.webkit.org started using --no-show-results and --reset-results - options.reset_results = True - options.show_results = False - output_json_path = self._output_json_path() output = self._generate_results_dict(self._timestamp, options.description, options.platform, options.builder_name, options.build_number) @@ -213,7 +206,8 @@ class PerfTestsRunner(object): if description: contents['description'] = description for (name, path) in self._port.repository_paths(): - contents[name + '-revision'] = self._host.scm().svn_revision(path) + scm = SCMDetector(self._host.filesystem, self._host.executive).detect_scm_system(path) or self._host.scm() + contents[name + '-revision'] = scm.svn_revision(path) # FIXME: Add --branch or auto-detect the branch we're in for key, value in {'timestamp': int(timestamp), 'branch': self._default_branch, 'platform': platform, diff --git a/Tools/Scripts/webkitpy/performance_tests/perftestsrunner_unittest.py b/Tools/Scripts/webkitpy/performance_tests/perftestsrunner_unittest.py index d3de7b3df..6119c61d3 100755 --- a/Tools/Scripts/webkitpy/performance_tests/perftestsrunner_unittest.py +++ b/Tools/Scripts/webkitpy/performance_tests/perftestsrunner_unittest.py @@ -473,7 +473,7 @@ max 548000 bytes def test_run_with_slave_config_json(self): runner, port = self.create_runner_and_setup_results_template(args=['--output-json-path=/mock-checkout/output.json', - '--source-json-path=/mock-checkout/slave-config.json', '--test-results-server=some.host']) + '--slave-config-json-path=/mock-checkout/slave-config.json', '--test-results-server=some.host']) port.host.filesystem.write_text_file('/mock-checkout/slave-config.json', '{"key": "value"}') self._test_run_with_json_output(runner, port.host.filesystem, upload_suceeds=True) self.assertEqual(runner.load_output_json(), [{ @@ -625,8 +625,10 @@ max 548000 bytes '--builder-name', 'webkit-mac-1', '--build-number=56', '--time-out-ms=42', + '--no-show-results', + '--reset-results', '--output-json-path=a/output.json', - '--source-json-path=a/source.json', + '--slave-config-json-path=a/source.json', '--test-results-server=somehost', '--debug']) self.assertEqual(options.build, True) @@ -636,6 +638,8 @@ max 548000 bytes self.assertEqual(options.build_number, '56') self.assertEqual(options.time_out_ms, '42') self.assertEqual(options.configuration, 'Debug') + self.assertEqual(options.show_results, False) + self.assertEqual(options.reset_results, True) self.assertEqual(options.output_json_path, 'a/output.json') self.assertEqual(options.slave_config_json_path, 'a/source.json') self.assertEqual(options.test_results_server, 'somehost') diff --git a/Tools/Scripts/webkitpy/style/checker.py b/Tools/Scripts/webkitpy/style/checker.py index 5b11af483..9f27c36da 100644 --- a/Tools/Scripts/webkitpy/style/checker.py +++ b/Tools/Scripts/webkitpy/style/checker.py @@ -96,7 +96,6 @@ _BASE_FILTER_RULES = [ '-runtime/rtti', '-whitespace/blank_line', '-whitespace/end_of_line', - '-whitespace/labels', # List Python pep8 categories last. # # Because much of WebKit's Python code base does not abide by the diff --git a/Tools/Scripts/webkitpy/style/checkers/cpp.py b/Tools/Scripts/webkitpy/style/checkers/cpp.py index a0051c979..ebbd1ad2f 100644 --- a/Tools/Scripts/webkitpy/style/checkers/cpp.py +++ b/Tools/Scripts/webkitpy/style/checkers/cpp.py @@ -2060,6 +2060,31 @@ def check_directive_indentation(clean_lines, line_number, file_state, error): error(line_number, 'whitespace/indent', 4, 'preprocessor directives (e.g., #ifdef, #define, #import) should never be indented.') +def get_initial_spaces_for_line(clean_line): + initial_spaces = 0 + while initial_spaces < len(clean_line) and clean_line[initial_spaces] == ' ': + initial_spaces += 1 + return initial_spaces + + +def check_indentation_amount(clean_lines, line_number, error): + line = clean_lines.elided[line_number] + initial_spaces = get_initial_spaces_for_line(line) + + if initial_spaces % 4: + error(line_number, 'whitespace/indent', 3, + 'Weird number of spaces at line-start. Are you using a 4-space indent?') + return + + previous_line = get_previous_non_blank_line(clean_lines, line_number)[0] + if not previous_line.strip() or match(r'\s*\w+\s*:\s*$', previous_line) or previous_line[0] == '#': + return + + previous_line_initial_spaces = get_initial_spaces_for_line(previous_line) + if initial_spaces > previous_line_initial_spaces + 4: + error(line_number, 'whitespace/indent', 3, 'When wrapping a line, only indent 4 spaces.') + + def check_using_std(clean_lines, line_number, file_state, error): """Looks for 'using std::foo;' statements which should be replaced with 'using namespace std;'. @@ -2535,44 +2560,10 @@ def check_style(clean_lines, line_number, file_extension, class_state, file_stat error(line_number, 'whitespace/tab', 1, 'Tab found; better to use spaces') - # One or three blank spaces at the beginning of the line is weird; it's - # hard to reconcile that with 4-space indents. - # NOTE: here are the conditions rob pike used for his tests. Mine aren't - # as sophisticated, but it may be worth becoming so: RLENGTH==initial_spaces - # if(RLENGTH > 20) complain = 0; - # if(match($0, " +(error|private|public|protected):")) complain = 0; - # if(match(prev, "&& *$")) complain = 0; - # if(match(prev, "\\|\\| *$")) complain = 0; - # if(match(prev, "[\",=><] *$")) complain = 0; - # if(match($0, " <<")) complain = 0; - # if(match(prev, " +for \\(")) complain = 0; - # if(prevodd && match(prevprev, " +for \\(")) complain = 0; - initial_spaces = 0 cleansed_line = clean_lines.elided[line_number] - while initial_spaces < len(line) and line[initial_spaces] == ' ': - initial_spaces += 1 if line and line[-1].isspace(): error(line_number, 'whitespace/end_of_line', 4, 'Line ends in whitespace. Consider deleting these extra spaces.') - # There are certain situations we allow one space, notably for labels - elif ((initial_spaces >= 1 and initial_spaces <= 3) - and not match(r'\s*\w+\s*:\s*$', cleansed_line)): - error(line_number, 'whitespace/indent', 3, - 'Weird number of spaces at line-start. ' - 'Are you using a 4-space indent?') - # Labels should always be indented at least one space. - elif not initial_spaces and line[:2] != '//': - label_match = match(r'(?P<label>[^:]+):\s*$', line) - - if label_match: - label = label_match.group('label') - # Only throw errors for stuff that is definitely not a goto label, - # because goto labels can in fact occur at the start of the line. - if label in ['public', 'private', 'protected'] or label.find(' ') != -1: - error(line_number, 'whitespace/labels', 4, - 'Labels should always be indented at least one space. ' - 'If this is a member-initializer list in a constructor, ' - 'the colon should be on the line after the definition header.') if (cleansed_line.count(';') > 1 # for loops are allowed two ;'s (and may run over two lines). @@ -2612,6 +2603,7 @@ def check_style(clean_lines, line_number, file_extension, class_state, file_stat check_check(clean_lines, line_number, error) check_for_comparisons_to_zero(clean_lines, line_number, error) check_for_null(clean_lines, line_number, file_state, error) + check_indentation_amount(clean_lines, line_number, error) _RE_PATTERN_INCLUDE_NEW_STYLE = re.compile(r'#include +"[^/]+\.h"') @@ -3634,7 +3626,6 @@ class CppChecker(object): 'whitespace/end_of_line', 'whitespace/ending_newline', 'whitespace/indent', - 'whitespace/labels', 'whitespace/line_length', 'whitespace/newline', 'whitespace/operators', diff --git a/Tools/Scripts/webkitpy/style/checkers/cpp_unittest.py b/Tools/Scripts/webkitpy/style/checkers/cpp_unittest.py index a1b91f292..6f001e0cb 100644 --- a/Tools/Scripts/webkitpy/style/checkers/cpp_unittest.py +++ b/Tools/Scripts/webkitpy/style/checkers/cpp_unittest.py @@ -1086,7 +1086,8 @@ class CppStyleTest(CppStyleTestBase): */ ''', '') self.assert_multi_line_lint( - r'''/* int a = 0; multi-liner + '''\ + /* int a = 0; multi-liner static const int b = 0;''', ['Could not find end of multi-line comment' ' [readability/multiline_comment] [5]', @@ -1125,97 +1126,111 @@ class CppStyleTest(CppStyleTestBase): def test_explicit_single_argument_constructors(self): # missing explicit is bad self.assert_multi_line_lint( - '''class Foo { - Foo(int f); - };''', + '''\ + class Foo { + Foo(int f); + };''', 'Single-argument constructors should be marked explicit.' ' [runtime/explicit] [5]') # missing explicit is bad, even with whitespace self.assert_multi_line_lint( - '''class Foo { - Foo (int f); - };''', + '''\ + class Foo { + Foo (int f); + };''', ['Extra space before ( in function call [whitespace/parens] [4]', 'Single-argument constructors should be marked explicit.' ' [runtime/explicit] [5]']) # missing explicit, with distracting comment, is still bad self.assert_multi_line_lint( - '''class Foo { - Foo(int f); // simpler than Foo(blargh, blarg) - };''', + '''\ + class Foo { + Foo(int f); // simpler than Foo(blargh, blarg) + };''', 'Single-argument constructors should be marked explicit.' ' [runtime/explicit] [5]') # missing explicit, with qualified classname self.assert_multi_line_lint( - '''class Qualifier::AnotherOne::Foo { - Foo(int f); - };''', + '''\ + class Qualifier::AnotherOne::Foo { + Foo(int f); + };''', 'Single-argument constructors should be marked explicit.' ' [runtime/explicit] [5]') # structs are caught as well. self.assert_multi_line_lint( - '''struct Foo { - Foo(int f); - };''', + '''\ + struct Foo { + Foo(int f); + };''', 'Single-argument constructors should be marked explicit.' ' [runtime/explicit] [5]') # Templatized classes are caught as well. self.assert_multi_line_lint( - '''template<typename T> class Foo { - Foo(int f); - };''', + '''\ + template<typename T> class Foo { + Foo(int f); + };''', 'Single-argument constructors should be marked explicit.' ' [runtime/explicit] [5]') # proper style is okay self.assert_multi_line_lint( - '''class Foo { - explicit Foo(int f); - };''', + '''\ + class Foo { + explicit Foo(int f); + };''', '') # two argument constructor is okay self.assert_multi_line_lint( - '''class Foo { - Foo(int f, int b); - };''', + '''\ + class Foo { + Foo(int f, int b); + };''', '') # two argument constructor, across two lines, is okay self.assert_multi_line_lint( - '''class Foo { - Foo(int f, - int b); - };''', + '''\ + class Foo { + Foo(int f, + int b); + };''', '') # non-constructor (but similar name), is okay self.assert_multi_line_lint( - '''class Foo { - aFoo(int f); - };''', + '''\ + class Foo { + aFoo(int f); + };''', '') # constructor with void argument is okay self.assert_multi_line_lint( - '''class Foo { - Foo(void); - };''', + '''\ + class Foo { + Foo(void); + };''', '') # single argument method is okay self.assert_multi_line_lint( - '''class Foo { - Bar(int b); - };''', + '''\ + class Foo { + Bar(int b); + };''', '') # comments should be ignored self.assert_multi_line_lint( - '''class Foo { - // Foo(int f); - };''', + '''\ + class Foo { + // Foo(int f); + };''', '') # single argument function following class definition is okay # (okay, it's not actually valid, but we don't want a false positive) self.assert_multi_line_lint( - '''class Foo { - Foo(int f, int b); - }; - Foo(int f);''', + '''\ + class Foo { + Foo(int f, int b); + }; + Foo(int f);''', '') # single argument function is okay self.assert_multi_line_lint( @@ -1223,14 +1238,16 @@ class CppStyleTest(CppStyleTestBase): '') # single argument copy constructor is okay. self.assert_multi_line_lint( - '''class Foo { - Foo(const Foo&); - };''', + '''\ + class Foo { + Foo(const Foo&); + };''', '') self.assert_multi_line_lint( - '''class Foo { - Foo(Foo&); - };''', + '''\ + class Foo { + Foo(Foo&); + };''', '') def test_slash_star_comment_on_single_line(self): @@ -1248,9 +1265,6 @@ class CppStyleTest(CppStyleTestBase): ''' /*/ static Foo(int f);''', 'Could not find end of multi-line comment' ' [readability/multiline_comment] [5]') - self.assert_multi_line_lint( - ''' /**/ static Foo(int f);''', - '') # Test suspicious usage of "if" like this: # if (a == b) { @@ -1383,23 +1397,27 @@ class CppStyleTest(CppStyleTestBase): # or initializing an array self.assert_lint('int a[3] = { 1, 2, 3 };', '') self.assert_lint( - '''const int foo[] = - {1, 2, 3 };''', + '''\ + const int foo[] = + {1, 2, 3 };''', '') # For single line, unmatched '}' with a ';' is ignored (not enough context) self.assert_multi_line_lint( - '''int a[3] = { 1, - 2, - 3 };''', + '''\ + int a[3] = { 1, + 2, + 3 };''', '') self.assert_multi_line_lint( - '''int a[2][3] = { { 1, 2 }, - { 3, 4 } };''', + '''\ + int a[2][3] = { { 1, 2 }, + { 3, 4 } };''', '') self.assert_multi_line_lint( - '''int a[2][3] = - { { 1, 2 }, - { 3, 4 } };''', + '''\ + int a[2][3] = + { { 1, 2 }, + { 3, 4 } };''', '') # CHECK/EXPECT_TRUE/EXPECT_FALSE replacements @@ -1753,7 +1771,7 @@ class CppStyleTest(CppStyleTestBase): self.assert_lint('default:;', 'Semicolon defining empty statement. Use { } instead.' ' [whitespace/semicolon] [5]') - self.assert_lint(' ;', + self.assert_lint(' ;', 'Line contains only semicolon. If this should be an empty ' 'statement, use { } instead.' ' [whitespace/semicolon] [5]') @@ -1795,10 +1813,10 @@ class CppStyleTest(CppStyleTestBase): ' VeryLongNameType veryLongNameVariable) { }', '') self.assert_lint('template<>\n' 'string FunctionTemplateSpecialization<SomeType>(\n' - ' int x) { return ""; }', '') + ' int x) { return ""; }', '') self.assert_lint('template<>\n' 'string FunctionTemplateSpecialization<vector<A::B>* >(\n' - ' int x) { return ""; }', '') + ' int x) { return ""; }', '') # should not catch methods of template classes. self.assert_lint('string Class<Type>::Method() const\n' @@ -2028,21 +2046,27 @@ class CppStyleTest(CppStyleTestBase): self.assert_lint(' char* oneSpaceIndent = "public:";', 'Weird number of spaces at line-start. ' 'Are you using a 4-space indent? [whitespace/indent] [3]') - self.assert_lint(' public:', '') - self.assert_lint(' public:', '') - self.assert_lint(' public:', '') - - def test_label(self): - self.assert_lint('public:', - 'Labels should always be indented at least one space. ' - 'If this is a member-initializer list in a constructor, ' - 'the colon should be on the line after the definition ' - 'header. [whitespace/labels] [4]') - self.assert_lint(' public:', '') - self.assert_lint(' public:', '') - self.assert_lint(' public:', '') - self.assert_lint(' public:', '') - self.assert_lint(' public:', '') + self.assert_lint(' public:', + 'Weird number of spaces at line-start. ' + 'Are you using a 4-space indent? [whitespace/indent] [3]') + self.assert_lint(' public:', + 'Weird number of spaces at line-start. ' + 'Are you using a 4-space indent? [whitespace/indent] [3]') + self.assert_lint(' public:', + 'Weird number of spaces at line-start. ' + 'Are you using a 4-space indent? [whitespace/indent] [3]') + self.assert_multi_line_lint( + 'class Foo {\n' + 'public:\n' + ' enum Bar {\n' + ' Alpha,\n' + ' Beta,\n' + '#if ENABLED_BETZ\n' + ' Charlie,\n' + '#endif\n' + ' };\n' + '};', + '') def test_not_alabel(self): self.assert_lint('MyVeryLongNamespace::MyVeryLongClassName::', '') @@ -2080,8 +2104,9 @@ class CppStyleTest(CppStyleTestBase): 'class Foo;', '') self.assert_multi_line_lint( - '''struct Foo* - foo = NewFoo();''', + '''\ + struct Foo* + foo = NewFoo();''', '') # Here is an example where the linter gets confused, even though # the code doesn't violate the style guide. @@ -3171,31 +3196,35 @@ class NoNonVirtualDestructorsTest(CppStyleTestBase): def test_no_error(self): self.assert_multi_line_lint( - '''class Foo { - virtual ~Foo(); - virtual void foo(); - };''', + '''\ + class Foo { + virtual ~Foo(); + virtual void foo(); + };''', '') self.assert_multi_line_lint( - '''class Foo { - virtual inline ~Foo(); - virtual void foo(); - };''', + '''\ + class Foo { + virtual inline ~Foo(); + virtual void foo(); + };''', '') self.assert_multi_line_lint( - '''class Foo { - inline virtual ~Foo(); - virtual void foo(); - };''', + '''\ + class Foo { + inline virtual ~Foo(); + virtual void foo(); + };''', '') self.assert_multi_line_lint( - '''class Foo::Goo { - virtual ~Goo(); - virtual void goo(); - };''', + '''\ + class Foo::Goo { + virtual ~Goo(); + virtual void goo(); + };''', '') self.assert_multi_line_lint( 'class Foo { void foo(); };', @@ -3215,92 +3244,92 @@ class NoNonVirtualDestructorsTest(CppStyleTestBase): 'More than one command on the same line [whitespace/newline] [4]') self.assert_multi_line_lint( - '''class Qualified::Goo : public Foo { - virtual void goo(); - };''', - '') - - self.assert_multi_line_lint( - # Line-ending : - '''class Goo : - public Foo { + '''\ + class Qualified::Goo : public Foo { virtual void goo(); - };''', - 'Labels should always be indented at least one space. If this is a ' - 'member-initializer list in a constructor, the colon should be on the ' - 'line after the definition header. [whitespace/labels] [4]') + };''', + '') def test_no_destructor_when_virtual_needed(self): self.assert_multi_line_lint_re( - '''class Foo { - virtual void foo(); - };''', + '''\ + class Foo { + virtual void foo(); + };''', 'The class Foo probably needs a virtual destructor') def test_destructor_non_virtual_when_virtual_needed(self): self.assert_multi_line_lint_re( - '''class Foo { - ~Foo(); - virtual void foo(); - };''', + '''\ + class Foo { + ~Foo(); + virtual void foo(); + };''', 'The class Foo probably needs a virtual destructor') def test_no_warn_when_derived(self): self.assert_multi_line_lint( - '''class Foo : public Goo { - virtual void foo(); - };''', + '''\ + class Foo : public Goo { + virtual void foo(); + };''', '') def test_internal_braces(self): self.assert_multi_line_lint_re( - '''class Foo { - enum Goo { - GOO - }; - virtual void foo(); - };''', + '''\ + class Foo { + enum Goo { + GOO + }; + virtual void foo(); + };''', 'The class Foo probably needs a virtual destructor') def test_inner_class_needs_virtual_destructor(self): self.assert_multi_line_lint_re( - '''class Foo { - class Goo { - virtual void goo(); - }; - };''', + '''\ + class Foo { + class Goo { + virtual void goo(); + }; + };''', 'The class Goo probably needs a virtual destructor') def test_outer_class_needs_virtual_destructor(self): self.assert_multi_line_lint_re( - '''class Foo { - class Goo { - }; - virtual void foo(); - };''', + '''\ + class Foo { + class Goo { + }; + virtual void foo(); + };''', 'The class Foo probably needs a virtual destructor') def test_qualified_class_needs_virtual_destructor(self): self.assert_multi_line_lint_re( - '''class Qualified::Foo { - virtual void foo(); - };''', + '''\ + class Qualified::Foo { + virtual void foo(); + };''', 'The class Qualified::Foo probably needs a virtual destructor') def test_multi_line_declaration_no_error(self): self.assert_multi_line_lint_re( - '''class Foo - : public Goo { - virtual void foo(); - };''', + '''\ + class Foo + : public Goo { + virtual void foo(); + };''', '') def test_multi_line_declaration_with_error(self): self.assert_multi_line_lint( - '''class Foo - { - virtual void foo(); - };''', + '''\ + class Foo + { + virtual void foo(); + };''', ['This { should be at the end of the previous line ' '[whitespace/braces] [4]', 'The class Foo probably needs a virtual destructor due to having ' @@ -3495,7 +3524,6 @@ class WebKitStyleTest(CppStyleTestBase): ' int goo;\n' '};', 'Weird number of spaces at line-start. Are you using a 4-space indent? [whitespace/indent] [3]') - # FIXME: No tests for 8-spaces. # 3. In a header, code inside a namespace should not be indented. self.assert_multi_line_lint( @@ -4180,7 +4208,14 @@ class WebKitStyleTest(CppStyleTestBase): ' myFunction(reallyLongParam1, reallyLongParam2,\n' ' reallyLongParam3);\n' '}\n', - '') + 'Weird number of spaces at line-start. Are you using a 4-space indent? [whitespace/indent] [3]') + + self.assert_multi_line_lint( + 'if (true) {\n' + ' myFunction(reallyLongParam1, reallyLongParam2,\n' + ' reallyLongParam3);\n' + '}\n', + 'When wrapping a line, only indent 4 spaces. [whitespace/indent] [3]') # 4. Control clauses without a body should use empty braces. self.assert_multi_line_lint( @@ -4189,7 +4224,7 @@ class WebKitStyleTest(CppStyleTestBase): self.assert_multi_line_lint( 'for ( ; current;\n' ' current = current->next) { }\n', - '') + 'Weird number of spaces at line-start. Are you using a 4-space indent? [whitespace/indent] [3]') self.assert_multi_line_lint( 'for ( ; current; current = current->next);\n', 'Semicolon defining empty statement for this loop. Use { } instead. [whitespace/semicolon] [5]') diff --git a/Tools/Scripts/webkitpy/style/checkers/test_expectations.py b/Tools/Scripts/webkitpy/style/checkers/test_expectations.py index 46403b7db..1ce40cd39 100644 --- a/Tools/Scripts/webkitpy/style/checkers/test_expectations.py +++ b/Tools/Scripts/webkitpy/style/checkers/test_expectations.py @@ -67,7 +67,7 @@ class TestExpectationsChecker(object): # FIXME: host should be a required parameter, not an optional one. host = host or Host() - host._initialize_scm() + host.initialize_scm() self._port_obj = self._determine_port_from_expectations_path(host, file_path) @@ -94,8 +94,6 @@ class TestExpectationsChecker(object): expectations = '\n'.join(lines) if self._port_obj: self.check_test_expectations(expectations_str=expectations, tests=None) - else: - self._handle_style_error(1, 'test/expectations', 5, - 'No port uses path %s for test_expectations' % self._file_path) + # Warn tabs in lines as well self.check_tabs(lines) diff --git a/Tools/Scripts/webkitpy/style/checkers/test_expectations_unittest.py b/Tools/Scripts/webkitpy/style/checkers/test_expectations_unittest.py index f12397787..1516de797 100644 --- a/Tools/Scripts/webkitpy/style/checkers/test_expectations_unittest.py +++ b/Tools/Scripts/webkitpy/style/checkers/test_expectations_unittest.py @@ -82,6 +82,10 @@ class TestExpectationsTestCase(unittest.TestCase): self._expect_port_for_expectations_path('efl', 'LayoutTests/platform/efl/TestExpectations') self._expect_port_for_expectations_path('efl', 'LayoutTests/platform/efl-wk1/TestExpectations') self._expect_port_for_expectations_path('efl', 'LayoutTests/platform/efl-wk2/TestExpectations') + self._expect_port_for_expectations_path('qt', 'LayoutTests/platform/qt-win/TestExpectations') + # FIXME: check-webkit-style doesn't know how to create port objects for all Qt version (4.8, 5.0) and + # will only check files based on the installed version of Qt. + #self._expect_port_for_expectations_path('qt', 'LayoutTests/platform/qt-5.0-wk2/TestExpectations') def assert_lines_lint(self, lines, should_pass, expected_output=None): self._error_collector.reset_errors() diff --git a/Tools/Scripts/webkitpy/style/main.py b/Tools/Scripts/webkitpy/style/main.py index e90d98a42..574368a3e 100644 --- a/Tools/Scripts/webkitpy/style/main.py +++ b/Tools/Scripts/webkitpy/style/main.py @@ -124,7 +124,7 @@ class CheckWebKitStyle(object): args = sys.argv[1:] host = Host() - host._initialize_scm() + host.initialize_scm() stderr = self._engage_awesome_stderr_hacks() diff --git a/Tools/Scripts/webkitpy/test/main.py b/Tools/Scripts/webkitpy/test/main.py index 28de8a6e0..852413299 100644 --- a/Tools/Scripts/webkitpy/test/main.py +++ b/Tools/Scripts/webkitpy/test/main.py @@ -81,7 +81,7 @@ class Tester(object): def skip(self, names, reason, bugid): self.finder.skip(names, reason, bugid) - def _parse_args(self): + def _parse_args(self, argv=None): parser = optparse.OptionParser(usage='usage: %prog [options] [args...]') parser.add_option('-a', '--all', action='store_true', default=False, help='run all the tests') @@ -103,7 +103,7 @@ class Tester(object): parser.epilog = ('[args...] is an optional list of modules, test_classes, or individual tests. ' 'If no args are given, all the tests will be run.') - return parser.parse_args() + return parser.parse_args(argv) def run(self): self._options, args = self._parse_args() @@ -188,10 +188,15 @@ class Tester(object): loader.test_method_prefixes = [] serial_tests = [] - loader.test_method_prefixes.extend(['serial_test_', 'serial_integration_test_']) + loader.test_method_prefixes = ['serial_test_', 'serial_integration_test_'] for name in names: serial_tests.extend(self._all_test_names(loader.loadTestsFromName(name, None))) + # loader.loadTestsFromName() will not verify that names begin with one of the test_method_prefixes + # if the names were explicitly provided (e.g., MainTest.test_basic), so this means that any individual + # tests will be included in both parallel_tests and serial_tests, and we need to de-dup them. + serial_tests = list(set(serial_tests).difference(set(parallel_tests))) + return (parallel_tests, serial_tests) def _all_test_names(self, suite): diff --git a/Tools/Scripts/webkitpy/test/main_unittest.py b/Tools/Scripts/webkitpy/test/main_unittest.py index ca7ebba0e..2020f5b60 100644 --- a/Tools/Scripts/webkitpy/test/main_unittest.py +++ b/Tools/Scripts/webkitpy/test/main_unittest.py @@ -25,7 +25,7 @@ import unittest import StringIO from webkitpy.common.system.outputcapture import OutputCapture -from webkitpy.test.main import Tester +from webkitpy.test.main import Tester, _Loader class TesterTest(unittest.TestCase): @@ -52,3 +52,10 @@ class TesterTest(unittest.TestCase): self.assertTrue('No tests to run' in errors.getvalue()) self.assertTrue('No tests to run' in logs) + + def test_individual_names_are_not_run_twice(self): + tester = Tester() + tester._options, args = tester._parse_args(["webkitpy.test.main_unittest.TesterTest.test_no_tests_found"]) + parallel_tests, serial_tests = tester._test_names(_Loader(), args) + self.assertEquals(parallel_tests, args) + self.assertEquals(serial_tests, []) diff --git a/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/__init__.py b/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/__init__.py index c154da4a1..454ae0c45 100644 --- a/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/__init__.py +++ b/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/__init__.py @@ -34,7 +34,8 @@ mod_pywebsocket is a WebSocket extension for Apache HTTP Server intended for testing or experimental purposes. mod_python is required. -Installation: +Installation +============ 0. Prepare an Apache HTTP Server for which mod_python is enabled. @@ -60,11 +61,6 @@ Installation: <scan_dir> is useful in saving scan time when <websock_handlers> contains many non-WebSocket handler files. - If you want to support old handshake based on - draft-hixie-thewebsocketprotocol-75: - - PythonOption mod_pywebsocket.allow_draft75 On - If you want to allow handlers whose canonical path is not under the root directory (i.e. symbolic link is in root directory but its target is not), configure as follows: @@ -89,7 +85,8 @@ Installation: 3. Verify installation. You can use example/console.html to poke the server. -Writing WebSocket handlers: +Writing WebSocket handlers +========================== When a WebSocket request comes in, the resource name specified in the handshake is considered as if it is a file path under @@ -118,28 +115,36 @@ extra handshake (web_socket_do_extra_handshake): - ws_resource - ws_origin - ws_version -- ws_location (Hixie 75 and HyBi 00 only) -- ws_extensions (Hybi 06 and later) +- ws_location (HyBi 00 only) +- ws_extensions (HyBi 06 and later) - ws_deflate (HyBi 06 and later) - ws_protocol - ws_requested_protocols (HyBi 06 and later) -The last two are a bit tricky. +The last two are a bit tricky. See the next subsection. + + +Subprotocol Negotiation +----------------------- For HyBi 06 and later, ws_protocol is always set to None when web_socket_do_extra_handshake is called. If ws_requested_protocols is not None, you must choose one subprotocol from this list and set it to ws_protocol. -For Hixie 75 and HyBi 00, when web_socket_do_extra_handshake is called, +For HyBi 00, when web_socket_do_extra_handshake is called, ws_protocol is set to the value given by the client in -Sec-WebSocket-Protocol (WebSocket-Protocol for Hixie 75) header or None if +Sec-WebSocket-Protocol header or None if such header was not found in the opening handshake request. Finish extra handshake with ws_protocol untouched to accept the request subprotocol. -Then, Sec-WebSocket-Protocol (or WebSocket-Protocol) header will be sent to +Then, Sec-WebSocket-Protocol header will be sent to the client in response with the same value as requested. Raise an exception in web_socket_do_extra_handshake to reject the requested subprotocol. + +Data Transfer +------------- + web_socket_transfer_data is called after the handshake completed successfully. A handler can receive/send messages from/to the client using request. mod_pywebsocket.msgutil module provides utilities @@ -159,12 +164,16 @@ You can send a message by the following statement. request.ws_stream.send_message(message) + +Closing Connection +------------------ + Executing the following statement or just return-ing from web_socket_transfer_data cause connection close. request.ws_stream.close_connection() -When you're using IETF HyBi 00 or later protocol, close_connection will wait +close_connection will wait for closing handshake acknowledgement coming from the client. When it couldn't receive a valid acknowledgement, raises an exception. @@ -176,6 +185,10 @@ use in web_socket_passive_closing_handshake. - ws_close_code - ws_close_reason + +Threading +--------- + A WebSocket handler must be thread-safe if the server (Apache or standalone.py) is configured to use threads. """ diff --git a/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/_stream_hixie75.py b/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/_stream_hixie75.py index c84ca6e07..94cf5b31b 100644 --- a/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/_stream_hixie75.py +++ b/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/_stream_hixie75.py @@ -32,7 +32,8 @@ protocol version HyBi 00 and Hixie 75. Specification: -http://tools.ietf.org/html/draft-ietf-hybi-thewebsocketprotocol-00 +- HyBi 00 http://tools.ietf.org/html/draft-ietf-hybi-thewebsocketprotocol-00 +- Hixie 75 http://tools.ietf.org/html/draft-hixie-thewebsocketprotocol-75 """ diff --git a/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/_stream_hybi.py b/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/_stream_hybi.py index 34fa7a60e..bd158fa6b 100644 --- a/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/_stream_hybi.py +++ b/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/_stream_hybi.py @@ -37,6 +37,7 @@ http://tools.ietf.org/html/rfc6455 from collections import deque +import logging import os import struct import time @@ -162,14 +163,145 @@ def create_text_frame( frame_filters) +def parse_frame(receive_bytes, logger=None, + ws_version=common.VERSION_HYBI_LATEST, + unmask_receive=True): + """Parses a frame. Returns a tuple containing each header field and + payload. + + Args: + receive_bytes: a function that reads frame data from a stream or + something similar. The function takes length of the bytes to be + read. The function must raise ConnectionTerminatedException if + there is not enough data to be read. + logger: a logging object. + ws_version: the version of WebSocket protocol. + unmask_receive: unmask received frames. When received unmasked + frame, raises InvalidFrameException. + + Raises: + ConnectionTerminatedException: when receive_bytes raises it. + InvalidFrameException: when the frame contains invalid data. + """ + + if not logger: + logger = logging.getLogger() + + logger.log(common.LOGLEVEL_FINE, 'Receive the first 2 octets of a frame') + + received = receive_bytes(2) + + first_byte = ord(received[0]) + fin = (first_byte >> 7) & 1 + rsv1 = (first_byte >> 6) & 1 + rsv2 = (first_byte >> 5) & 1 + rsv3 = (first_byte >> 4) & 1 + opcode = first_byte & 0xf + + second_byte = ord(received[1]) + mask = (second_byte >> 7) & 1 + payload_length = second_byte & 0x7f + + logger.log(common.LOGLEVEL_FINE, + 'FIN=%s, RSV1=%s, RSV2=%s, RSV3=%s, opcode=%s, ' + 'Mask=%s, Payload_length=%s', + fin, rsv1, rsv2, rsv3, opcode, mask, payload_length) + + if (mask == 1) != unmask_receive: + raise InvalidFrameException( + 'Mask bit on the received frame did\'nt match masking ' + 'configuration for received frames') + + # The HyBi and later specs disallow putting a value in 0x0-0xFFFF + # into the 8-octet extended payload length field (or 0x0-0xFD in + # 2-octet field). + valid_length_encoding = True + length_encoding_bytes = 1 + if payload_length == 127: + logger.log(common.LOGLEVEL_FINE, + 'Receive 8-octet extended payload length') + + extended_payload_length = receive_bytes(8) + payload_length = struct.unpack( + '!Q', extended_payload_length)[0] + if payload_length > 0x7FFFFFFFFFFFFFFF: + raise InvalidFrameException( + 'Extended payload length >= 2^63') + if ws_version >= 13 and payload_length < 0x10000: + valid_length_encoding = False + length_encoding_bytes = 8 + + logger.log(common.LOGLEVEL_FINE, + 'Decoded_payload_length=%s', payload_length) + elif payload_length == 126: + logger.log(common.LOGLEVEL_FINE, + 'Receive 2-octet extended payload length') + + extended_payload_length = receive_bytes(2) + payload_length = struct.unpack( + '!H', extended_payload_length)[0] + if ws_version >= 13 and payload_length < 126: + valid_length_encoding = False + length_encoding_bytes = 2 + + logger.log(common.LOGLEVEL_FINE, + 'Decoded_payload_length=%s', payload_length) + + if not valid_length_encoding: + logger.warning( + 'Payload length is not encoded using the minimal number of ' + 'bytes (%d is encoded using %d bytes)', + payload_length, + length_encoding_bytes) + + if mask == 1: + logger.log(common.LOGLEVEL_FINE, 'Receive mask') + + masking_nonce = receive_bytes(4) + masker = util.RepeatedXorMasker(masking_nonce) + + logger.log(common.LOGLEVEL_FINE, 'Mask=%r', masking_nonce) + else: + masker = _NOOP_MASKER + + logger.log(common.LOGLEVEL_FINE, 'Receive payload data') + if logger.isEnabledFor(common.LOGLEVEL_FINE): + receive_start = time.time() + + raw_payload_bytes = receive_bytes(payload_length) + + if logger.isEnabledFor(common.LOGLEVEL_FINE): + logger.log( + common.LOGLEVEL_FINE, + 'Done receiving payload data at %s MB/s', + payload_length / (time.time() - receive_start) / 1000 / 1000) + logger.log(common.LOGLEVEL_FINE, 'Unmask payload data') + + if logger.isEnabledFor(common.LOGLEVEL_FINE): + unmask_start = time.time() + + bytes = masker.mask(raw_payload_bytes) + + if logger.isEnabledFor(common.LOGLEVEL_FINE): + logger.log( + common.LOGLEVEL_FINE, + 'Done unmasking payload data at %s MB/s', + payload_length / (time.time() - unmask_start) / 1000 / 1000) + + return opcode, bytes, fin, rsv1, rsv2, rsv3 + + class FragmentedFrameBuilder(object): """A stateful class to send a message as fragments.""" - def __init__(self, mask, frame_filters=[]): + def __init__(self, mask, frame_filters=[], encode_utf8=True): """Constructs an instance.""" self._mask = mask self._frame_filters = frame_filters + # This is for skipping UTF-8 encoding when building text type frames + # from compressed data. + self._encode_utf8 = encode_utf8 self._started = False @@ -177,7 +309,7 @@ class FragmentedFrameBuilder(object): # frames in the message are all the same. self._opcode = common.OPCODE_TEXT - def build(self, message, end, binary): + def build(self, payload_data, end, binary): if binary: frame_type = common.OPCODE_BINARY else: @@ -198,12 +330,12 @@ class FragmentedFrameBuilder(object): self._started = True fin = 0 - if binary: + if binary or not self._encode_utf8: return create_binary_frame( - message, opcode, fin, self._mask, self._frame_filters) + payload_data, opcode, fin, self._mask, self._frame_filters) else: return create_text_frame( - message, opcode, fin, self._mask, self._frame_filters) + payload_data, opcode, fin, self._mask, self._frame_filters) def _create_control_frame(opcode, body, mask, frame_filters): @@ -235,6 +367,22 @@ def create_close_frame(body, mask=False, frame_filters=[]): common.OPCODE_CLOSE, body, mask, frame_filters) +def create_closing_handshake_body(code, reason): + body = '' + if code is not None: + if (code > common.STATUS_USER_PRIVATE_MAX or + code < common.STATUS_NORMAL_CLOSURE): + raise BadOperationException('Status code is out of range') + if (code == common.STATUS_NO_STATUS_RECEIVED or + code == common.STATUS_ABNORMAL_CLOSURE or + code == common.STATUS_TLS_HANDSHAKE): + raise BadOperationException('Status code is reserved pseudo ' + 'code') + encoded_reason = reason.encode('utf-8') + body = struct.pack('!H', code) + encoded_reason + return body + + class StreamOptions(object): """Holds option values to configure Stream objects.""" @@ -248,8 +396,16 @@ class StreamOptions(object): self.outgoing_frame_filters = [] self.incoming_frame_filters = [] + # Filters applied to messages. Control frames are not affected by them. + self.outgoing_message_filters = [] + self.incoming_message_filters = [] + + self.encode_text_message_to_utf8 = True self.mask_send = False self.unmask_receive = True + # RFC6455 disallows fragmented control frames, but mux extension + # relaxes the restriction. + self.allow_fragmented_control_frame = False class Stream(StreamBase): @@ -283,7 +439,8 @@ class Stream(StreamBase): self._original_opcode = None self._writer = FragmentedFrameBuilder( - self._options.mask_send, self._options.outgoing_frame_filters) + self._options.mask_send, self._options.outgoing_frame_filters, + self._options.encode_text_message_to_utf8) self._ping_queue = deque() @@ -297,109 +454,13 @@ class Stream(StreamBase): InvalidFrameException: when the frame contains invalid data. """ - self._logger.log(common.LOGLEVEL_FINE, - 'Receive the first 2 octets of a frame') - - received = self.receive_bytes(2) - - first_byte = ord(received[0]) - fin = (first_byte >> 7) & 1 - rsv1 = (first_byte >> 6) & 1 - rsv2 = (first_byte >> 5) & 1 - rsv3 = (first_byte >> 4) & 1 - opcode = first_byte & 0xf - - second_byte = ord(received[1]) - mask = (second_byte >> 7) & 1 - payload_length = second_byte & 0x7f - - self._logger.log(common.LOGLEVEL_FINE, - 'FIN=%s, RSV1=%s, RSV2=%s, RSV3=%s, opcode=%s, ' - 'Mask=%s, Payload_length=%s', - fin, rsv1, rsv2, rsv3, opcode, mask, payload_length) - - if (mask == 1) != self._options.unmask_receive: - raise InvalidFrameException( - 'Mask bit on the received frame did\'nt match masking ' - 'configuration for received frames') - - # The Hybi-13 and later specs disallow putting a value in 0x0-0xFFFF - # into the 8-octet extended payload length field (or 0x0-0xFD in - # 2-octet field). - valid_length_encoding = True - length_encoding_bytes = 1 - if payload_length == 127: - self._logger.log(common.LOGLEVEL_FINE, - 'Receive 8-octet extended payload length') - - extended_payload_length = self.receive_bytes(8) - payload_length = struct.unpack( - '!Q', extended_payload_length)[0] - if payload_length > 0x7FFFFFFFFFFFFFFF: - raise InvalidFrameException( - 'Extended payload length >= 2^63') - if self._request.ws_version >= 13 and payload_length < 0x10000: - valid_length_encoding = False - length_encoding_bytes = 8 - - self._logger.log(common.LOGLEVEL_FINE, - 'Decoded_payload_length=%s', payload_length) - elif payload_length == 126: - self._logger.log(common.LOGLEVEL_FINE, - 'Receive 2-octet extended payload length') - - extended_payload_length = self.receive_bytes(2) - payload_length = struct.unpack( - '!H', extended_payload_length)[0] - if self._request.ws_version >= 13 and payload_length < 126: - valid_length_encoding = False - length_encoding_bytes = 2 - - self._logger.log(common.LOGLEVEL_FINE, - 'Decoded_payload_length=%s', payload_length) - - if not valid_length_encoding: - self._logger.warning( - 'Payload length is not encoded using the minimal number of ' - 'bytes (%d is encoded using %d bytes)', - payload_length, - length_encoding_bytes) - - if mask == 1: - self._logger.log(common.LOGLEVEL_FINE, 'Receive mask') - - masking_nonce = self.receive_bytes(4) - masker = util.RepeatedXorMasker(masking_nonce) - - self._logger.log(common.LOGLEVEL_FINE, 'Mask=%r', masking_nonce) - else: - masker = _NOOP_MASKER - - self._logger.log(common.LOGLEVEL_FINE, 'Receive payload data') - if self._logger.isEnabledFor(common.LOGLEVEL_FINE): - receive_start = time.time() - - raw_payload_bytes = self.receive_bytes(payload_length) - - if self._logger.isEnabledFor(common.LOGLEVEL_FINE): - self._logger.log( - common.LOGLEVEL_FINE, - 'Done receiving payload data at %s MB/s', - payload_length / (time.time() - receive_start) / 1000 / 1000) - self._logger.log(common.LOGLEVEL_FINE, 'Unmask payload data') - - if self._logger.isEnabledFor(common.LOGLEVEL_FINE): - unmask_start = time.time() - - bytes = masker.mask(raw_payload_bytes) + def _receive_bytes(length): + return self.receive_bytes(length) - if self._logger.isEnabledFor(common.LOGLEVEL_FINE): - self._logger.log( - common.LOGLEVEL_FINE, - 'Done unmasking payload data at %s MB/s', - payload_length / (time.time() - unmask_start) / 1000 / 1000) - - return opcode, bytes, fin, rsv1, rsv2, rsv3 + return parse_frame(receive_bytes=_receive_bytes, + logger=self._logger, + ws_version=self._request.ws_version, + unmask_receive=self._options.unmask_receive) def _receive_frame_as_frame_object(self): opcode, bytes, fin, rsv1, rsv2, rsv3 = self._receive_frame() @@ -407,6 +468,32 @@ class Stream(StreamBase): return Frame(fin=fin, rsv1=rsv1, rsv2=rsv2, rsv3=rsv3, opcode=opcode, payload=bytes) + def receive_filtered_frame(self): + """Receives a frame and applies frame filters and message filters. + The frame to be received must satisfy following conditions: + - The frame is not fragmented. + - The opcode of the frame is TEXT or BINARY. + + DO NOT USE this method except for testing purpose. + """ + + frame = self._receive_frame_as_frame_object() + if not frame.fin: + raise InvalidFrameException( + 'Segmented frames must not be received via ' + 'receive_filtered_frame()') + if (frame.opcode != common.OPCODE_TEXT and + frame.opcode != common.OPCODE_BINARY): + raise InvalidFrameException( + 'Control frames must not be received via ' + 'receive_filtered_frame()') + + for frame_filter in self._options.incoming_frame_filters: + frame_filter.filter(frame) + for message_filter in self._options.incoming_message_filters: + frame.payload = message_filter.filter(frame.payload) + return frame + def send_message(self, message, end=True, binary=False): """Send message. @@ -428,11 +515,219 @@ class Stream(StreamBase): raise BadOperationException( 'Message for binary frame must be instance of str') + for message_filter in self._options.outgoing_message_filters: + message = message_filter.filter(message, end, binary) + try: - self._write(self._writer.build(message, end, binary)) + # Set this to any positive integer to limit maximum size of data in + # payload data of each frame. + MAX_PAYLOAD_DATA_SIZE = -1 + + if MAX_PAYLOAD_DATA_SIZE <= 0: + self._write(self._writer.build(message, end, binary)) + return + + bytes_written = 0 + while True: + end_for_this_frame = end + bytes_to_write = len(message) - bytes_written + if (MAX_PAYLOAD_DATA_SIZE > 0 and + bytes_to_write > MAX_PAYLOAD_DATA_SIZE): + end_for_this_frame = False + bytes_to_write = MAX_PAYLOAD_DATA_SIZE + + frame = self._writer.build( + message[bytes_written:bytes_written + bytes_to_write], + end_for_this_frame, + binary) + self._write(frame) + + bytes_written += bytes_to_write + + # This if must be placed here (the end of while block) so that + # at least one frame is sent. + if len(message) <= bytes_written: + break except ValueError, e: raise BadOperationException(e) + def _get_message_from_frame(self, frame): + """Gets a message from frame. If the message is composed of fragmented + frames and the frame is not the last fragmented frame, this method + returns None. The whole message will be returned when the last + fragmented frame is passed to this method. + + Raises: + InvalidFrameException: when the frame doesn't match defragmentation + context, or the frame contains invalid data. + """ + + if frame.opcode == common.OPCODE_CONTINUATION: + if not self._received_fragments: + if frame.fin: + raise InvalidFrameException( + 'Received a termination frame but fragmentation ' + 'not started') + else: + raise InvalidFrameException( + 'Received an intermediate frame but ' + 'fragmentation not started') + + if frame.fin: + # End of fragmentation frame + self._received_fragments.append(frame.payload) + message = ''.join(self._received_fragments) + self._received_fragments = [] + return message + else: + # Intermediate frame + self._received_fragments.append(frame.payload) + return None + else: + if self._received_fragments: + if frame.fin: + raise InvalidFrameException( + 'Received an unfragmented frame without ' + 'terminating existing fragmentation') + else: + raise InvalidFrameException( + 'New fragmentation started without terminating ' + 'existing fragmentation') + + if frame.fin: + # Unfragmented frame + + self._original_opcode = frame.opcode + return frame.payload + else: + # Start of fragmentation frame + + if (not self._options.allow_fragmented_control_frame and + common.is_control_opcode(frame.opcode)): + raise InvalidFrameException( + 'Control frames must not be fragmented') + + self._original_opcode = frame.opcode + self._received_fragments.append(frame.payload) + return None + + def _process_close_message(self, message): + """Processes close message. + + Args: + message: close message. + + Raises: + InvalidFrameException: when the message is invalid. + """ + + self._request.client_terminated = True + + # Status code is optional. We can have status reason only if we + # have status code. Status reason can be empty string. So, + # allowed cases are + # - no application data: no code no reason + # - 2 octet of application data: has code but no reason + # - 3 or more octet of application data: both code and reason + if len(message) == 0: + self._logger.debug('Received close frame (empty body)') + self._request.ws_close_code = ( + common.STATUS_NO_STATUS_RECEIVED) + elif len(message) == 1: + raise InvalidFrameException( + 'If a close frame has status code, the length of ' + 'status code must be 2 octet') + elif len(message) >= 2: + self._request.ws_close_code = struct.unpack( + '!H', message[0:2])[0] + self._request.ws_close_reason = message[2:].decode( + 'utf-8', 'replace') + self._logger.debug( + 'Received close frame (code=%d, reason=%r)', + self._request.ws_close_code, + self._request.ws_close_reason) + + # Drain junk data after the close frame if necessary. + self._drain_received_data() + + if self._request.server_terminated: + self._logger.debug( + 'Received ack for server-initiated closing handshake') + return + + self._logger.debug( + 'Received client-initiated closing handshake') + + code = common.STATUS_NORMAL_CLOSURE + reason = '' + if hasattr(self._request, '_dispatcher'): + dispatcher = self._request._dispatcher + code, reason = dispatcher.passive_closing_handshake( + self._request) + if code is None and reason is not None and len(reason) > 0: + self._logger.warning( + 'Handler specified reason despite code being None') + reason = '' + if reason is None: + reason = '' + self._send_closing_handshake(code, reason) + self._logger.debug( + 'Sent ack for client-initiated closing handshake ' + '(code=%r, reason=%r)', code, reason) + + def _process_ping_message(self, message): + """Processes ping message. + + Args: + message: ping message. + """ + + try: + handler = self._request.on_ping_handler + if handler: + handler(self._request, message) + return + except AttributeError, e: + pass + self._send_pong(message) + + def _process_pong_message(self, message): + """Processes pong message. + + Args: + message: pong message. + """ + + # TODO(tyoshino): Add ping timeout handling. + + inflight_pings = deque() + + while True: + try: + expected_body = self._ping_queue.popleft() + if expected_body == message: + # inflight_pings contains pings ignored by the + # other peer. Just forget them. + self._logger.debug( + 'Ping %r is acked (%d pings were ignored)', + expected_body, len(inflight_pings)) + break + else: + inflight_pings.append(expected_body) + except IndexError, e: + # The received pong was unsolicited pong. Keep the + # ping queue as is. + self._ping_queue = inflight_pings + self._logger.debug('Received a unsolicited pong') + break + + try: + handler = self._request.on_pong_handler + if handler: + handler(self._request, message) + except AttributeError, e: + pass + def receive_message(self): """Receive a WebSocket frame and return its payload as a text in unicode or a binary in str. @@ -482,52 +777,12 @@ class Stream(StreamBase): 'Unsupported flag is set (rsv = %d%d%d)' % (frame.rsv1, frame.rsv2, frame.rsv3)) - if frame.opcode == common.OPCODE_CONTINUATION: - if not self._received_fragments: - if frame.fin: - raise InvalidFrameException( - 'Received a termination frame but fragmentation ' - 'not started') - else: - raise InvalidFrameException( - 'Received an intermediate frame but ' - 'fragmentation not started') - - if frame.fin: - # End of fragmentation frame - self._received_fragments.append(frame.payload) - message = ''.join(self._received_fragments) - self._received_fragments = [] - else: - # Intermediate frame - self._received_fragments.append(frame.payload) - continue - else: - if self._received_fragments: - if frame.fin: - raise InvalidFrameException( - 'Received an unfragmented frame without ' - 'terminating existing fragmentation') - else: - raise InvalidFrameException( - 'New fragmentation started without terminating ' - 'existing fragmentation') - - if frame.fin: - # Unfragmented frame - - self._original_opcode = frame.opcode - message = frame.payload - else: - # Start of fragmentation frame - - if common.is_control_opcode(frame.opcode): - raise InvalidFrameException( - 'Control frames must not be fragmented') + message = self._get_message_from_frame(frame) + if message is None: + continue - self._original_opcode = frame.opcode - self._received_fragments.append(frame.payload) - continue + for message_filter in self._options.incoming_message_filters: + message = message_filter.filter(message) if self._original_opcode == common.OPCODE_TEXT: # The WebSocket protocol section 4.4 specifies that invalid @@ -540,124 +795,21 @@ class Stream(StreamBase): elif self._original_opcode == common.OPCODE_BINARY: return message elif self._original_opcode == common.OPCODE_CLOSE: - self._request.client_terminated = True - - # Status code is optional. We can have status reason only if we - # have status code. Status reason can be empty string. So, - # allowed cases are - # - no application data: no code no reason - # - 2 octet of application data: has code but no reason - # - 3 or more octet of application data: both code and reason - if len(message) == 0: - self._logger.debug('Received close frame (empty body)') - self._request.ws_close_code = ( - common.STATUS_NO_STATUS_RECEIVED) - elif len(message) == 1: - raise InvalidFrameException( - 'If a close frame has status code, the length of ' - 'status code must be 2 octet') - elif len(message) >= 2: - self._request.ws_close_code = struct.unpack( - '!H', message[0:2])[0] - self._request.ws_close_reason = message[2:].decode( - 'utf-8', 'replace') - self._logger.debug( - 'Received close frame (code=%d, reason=%r)', - self._request.ws_close_code, - self._request.ws_close_reason) - - # Drain junk data after the close frame if necessary. - self._drain_received_data() - - if self._request.server_terminated: - self._logger.debug( - 'Received ack for server-initiated closing handshake') - return None - - self._logger.debug( - 'Received client-initiated closing handshake') - - code = common.STATUS_NORMAL_CLOSURE - reason = '' - if hasattr(self._request, '_dispatcher'): - dispatcher = self._request._dispatcher - code, reason = dispatcher.passive_closing_handshake( - self._request) - if code is None and reason is not None and len(reason) > 0: - self._logger.warning( - 'Handler specified reason despite code being None') - reason = '' - if reason is None: - reason = '' - self._send_closing_handshake(code, reason) - self._logger.debug( - 'Sent ack for client-initiated closing handshake ' - '(code=%r, reason=%r)', code, reason) + self._process_close_message(message) return None elif self._original_opcode == common.OPCODE_PING: - try: - handler = self._request.on_ping_handler - if handler: - handler(self._request, message) - continue - except AttributeError, e: - pass - self._send_pong(message) + self._process_ping_message(message) elif self._original_opcode == common.OPCODE_PONG: - # TODO(tyoshino): Add ping timeout handling. - - inflight_pings = deque() - - while True: - try: - expected_body = self._ping_queue.popleft() - if expected_body == message: - # inflight_pings contains pings ignored by the - # other peer. Just forget them. - self._logger.debug( - 'Ping %r is acked (%d pings were ignored)', - expected_body, len(inflight_pings)) - break - else: - inflight_pings.append(expected_body) - except IndexError, e: - # The received pong was unsolicited pong. Keep the - # ping queue as is. - self._ping_queue = inflight_pings - self._logger.debug('Received a unsolicited pong') - break - - try: - handler = self._request.on_pong_handler - if handler: - handler(self._request, message) - continue - except AttributeError, e: - pass - - continue + self._process_pong_message(message) else: raise UnsupportedFrameException( 'Opcode %d is not supported' % self._original_opcode) def _send_closing_handshake(self, code, reason): - body = '' - if code is not None: - if (code > common.STATUS_USER_PRIVATE_MAX or - code < common.STATUS_NORMAL_CLOSURE): - raise BadOperationException('Status code is out of range') - if (code == common.STATUS_NO_STATUS_RECEIVED or - code == common.STATUS_ABNORMAL_CLOSURE or - code == common.STATUS_TLS_HANDSHAKE): - raise BadOperationException('Status code is reserved pseudo ' - 'code') - encoded_reason = reason.encode('utf-8') - body = struct.pack('!H', code) + encoded_reason - + body = create_closing_handshake_body(code, reason) frame = create_close_frame( - body, - self._options.mask_send, - self._options.outgoing_frame_filters) + body, mask=self._options.mask_send, + frame_filters=self._options.outgoing_frame_filters) self._request.server_terminated = True @@ -731,6 +883,14 @@ class Stream(StreamBase): self._options.outgoing_frame_filters) self._write(frame) + def get_last_received_opcode(self): + """Returns the opcode of the WebSocket message which the last received + frame belongs to. The return value is valid iff immediately after + receive_message call. + """ + + return self._original_opcode + def _drain_received_data(self): """Drains unread data in the receive buffer to avoid sending out TCP RST packet. This is because when deflate-stream is enabled, some diff --git a/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/common.py b/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/common.py index 710967c80..2388379c0 100644 --- a/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/common.py +++ b/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/common.py @@ -104,7 +104,10 @@ SEC_WEBSOCKET_LOCATION_HEADER = 'Sec-WebSocket-Location' DEFLATE_STREAM_EXTENSION = 'deflate-stream' DEFLATE_FRAME_EXTENSION = 'deflate-frame' PERFRAME_COMPRESSION_EXTENSION = 'perframe-compress' +PERMESSAGE_COMPRESSION_EXTENSION = 'permessage-compress' X_WEBKIT_DEFLATE_FRAME_EXTENSION = 'x-webkit-deflate-frame' +X_WEBKIT_PERMESSAGE_COMPRESSION_EXTENSION = 'x-webkit-permessage-compress' +MUX_EXTENSION = 'mux_DO_NOT_USE' # Status codes # Code STATUS_NO_STATUS_RECEIVED, STATUS_ABNORMAL_CLOSURE, and @@ -125,7 +128,7 @@ STATUS_INVALID_FRAME_PAYLOAD_DATA = 1007 STATUS_POLICY_VIOLATION = 1008 STATUS_MESSAGE_TOO_BIG = 1009 STATUS_MANDATORY_EXTENSION = 1010 -STATUS_INTERNAL_SERVER_ERROR = 1011 +STATUS_INTERNAL_ENDPOINT_ERROR = 1011 STATUS_TLS_HANDSHAKE = 1015 STATUS_USER_REGISTERED_BASE = 3000 STATUS_USER_REGISTERED_MAX = 3999 diff --git a/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/dispatch.py b/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/dispatch.py index ab1eb4fb3..25905f180 100644 --- a/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/dispatch.py +++ b/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/dispatch.py @@ -39,6 +39,7 @@ import re from mod_pywebsocket import common from mod_pywebsocket import handshake from mod_pywebsocket import msgutil +from mod_pywebsocket import mux from mod_pywebsocket import stream from mod_pywebsocket import util @@ -277,13 +278,18 @@ class Dispatcher(object): AbortedByUserException: when user handler abort connection """ - handler_suite = self.get_handler_suite(request.ws_resource) - if handler_suite is None: - raise DispatchException('No handler for: %r' % request.ws_resource) - transfer_data_ = handler_suite.transfer_data # TODO(tyoshino): Terminate underlying TCP connection if possible. try: - transfer_data_(request) + if mux.use_mux(request): + mux.start(request, self) + else: + handler_suite = self.get_handler_suite(request.ws_resource) + if handler_suite is None: + raise DispatchException('No handler for: %r' % + request.ws_resource) + transfer_data_ = handler_suite.transfer_data + transfer_data_(request) + if not request.server_terminated: request.ws_stream.close_connection() # Catch non-critical exceptions the handler didn't handle. diff --git a/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/extensions.py b/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/extensions.py index 52b7a4a19..03dbf9ee1 100644 --- a/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/extensions.py +++ b/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/extensions.py @@ -38,6 +38,9 @@ _available_processors = {} class ExtensionProcessorInterface(object): + def name(self): + return None + def get_extension_response(self): return None @@ -46,13 +49,21 @@ class ExtensionProcessorInterface(object): class DeflateStreamExtensionProcessor(ExtensionProcessorInterface): - """WebSocket DEFLATE stream extension processor.""" + """WebSocket DEFLATE stream extension processor. + + Specification: + Section 9.2.1 in + http://tools.ietf.org/html/draft-ietf-hybi-thewebsocketprotocol-10 + """ def __init__(self, request): self._logger = util.get_class_logger(self) self._request = request + def name(self): + return common.DEFLATE_STREAM_EXTENSION + def get_extension_response(self): if len(self._request.get_parameter_names()) != 0: return None @@ -70,8 +81,40 @@ _available_processors[common.DEFLATE_STREAM_EXTENSION] = ( DeflateStreamExtensionProcessor) +def _log_compression_ratio(logger, original_bytes, total_original_bytes, + filtered_bytes, total_filtered_bytes): + # Print inf when ratio is not available. + ratio = float('inf') + average_ratio = float('inf') + if original_bytes != 0: + ratio = float(filtered_bytes) / original_bytes + if total_original_bytes != 0: + average_ratio = ( + float(total_filtered_bytes) / total_original_bytes) + logger.debug('Outgoing compress ratio: %f (average: %f)' % + (ratio, average_ratio)) + + +def _log_decompression_ratio(logger, received_bytes, total_received_bytes, + filtered_bytes, total_filtered_bytes): + # Print inf when ratio is not available. + ratio = float('inf') + average_ratio = float('inf') + if received_bytes != 0: + ratio = float(received_bytes) / filtered_bytes + if total_filtered_bytes != 0: + average_ratio = ( + float(total_received_bytes) / total_filtered_bytes) + logger.debug('Incoming compress ratio: %f (average: %f)' % + (ratio, average_ratio)) + + class DeflateFrameExtensionProcessor(ExtensionProcessorInterface): - """WebSocket Per-frame DEFLATE extension processor.""" + """WebSocket Per-frame DEFLATE extension processor. + + Specification: + http://tools.ietf.org/html/draft-tyoshino-hybi-websocket-perframe-deflate + """ _WINDOW_BITS_PARAM = 'max_window_bits' _NO_CONTEXT_TAKEOVER_PARAM = 'no_context_takeover' @@ -83,6 +126,7 @@ class DeflateFrameExtensionProcessor(ExtensionProcessorInterface): self._response_window_bits = None self._response_no_context_takeover = False + self._bfinal = False # Counters for statistics. @@ -96,6 +140,9 @@ class DeflateFrameExtensionProcessor(ExtensionProcessorInterface): # Total number of incoming bytes obtained after applying this filter. self._total_filtered_incoming_payload_bytes = 0 + def name(self): + return common.DEFLATE_FRAME_EXTENSION + def get_extension_response(self): # Any unknown parameter will be just ignored. @@ -173,6 +220,9 @@ class DeflateFrameExtensionProcessor(ExtensionProcessorInterface): def set_response_no_context_takeover(self, value): self._response_no_context_takeover = value + def set_bfinal(self, value): + self._bfinal = value + def enable_outgoing_compression(self): self._compress_outgoing = True @@ -193,24 +243,17 @@ class DeflateFrameExtensionProcessor(ExtensionProcessorInterface): original_payload_size) return - frame.payload = self._deflater.filter(frame.payload) + frame.payload = self._deflater.filter( + frame.payload, bfinal=self._bfinal) frame.rsv1 = 1 filtered_payload_size = len(frame.payload) self._total_filtered_outgoing_payload_bytes += filtered_payload_size - # Print inf when ratio is not available. - ratio = float('inf') - average_ratio = float('inf') - if original_payload_size != 0: - ratio = float(filtered_payload_size) / original_payload_size - if self._total_outgoing_payload_bytes != 0: - average_ratio = ( - float(self._total_filtered_outgoing_payload_bytes) / - self._total_outgoing_payload_bytes) - self._logger.debug( - 'Outgoing compress ratio: %f (average: %f)' % - (ratio, average_ratio)) + _log_compression_ratio(self._logger, original_payload_size, + self._total_outgoing_payload_bytes, + filtered_payload_size, + self._total_filtered_outgoing_payload_bytes) def _incoming_filter(self, frame): """Transform incoming frames. This method is called only by @@ -231,18 +274,10 @@ class DeflateFrameExtensionProcessor(ExtensionProcessorInterface): filtered_payload_size = len(frame.payload) self._total_filtered_incoming_payload_bytes += filtered_payload_size - # Print inf when ratio is not available. - ratio = float('inf') - average_ratio = float('inf') - if received_payload_size != 0: - ratio = float(received_payload_size) / filtered_payload_size - if self._total_filtered_incoming_payload_bytes != 0: - average_ratio = ( - float(self._total_incoming_payload_bytes) / - self._total_filtered_incoming_payload_bytes) - self._logger.debug( - 'Incoming compress ratio: %f (average: %f)' % - (ratio, average_ratio)) + _log_decompression_ratio(self._logger, received_payload_size, + self._total_incoming_payload_bytes, + filtered_payload_size, + self._total_filtered_incoming_payload_bytes) _available_processors[common.DEFLATE_FRAME_EXTENSION] = ( @@ -250,7 +285,7 @@ _available_processors[common.DEFLATE_FRAME_EXTENSION] = ( # Adding vendor-prefixed deflate-frame extension. -# TODO(bashi): Remove this after WebKit stops using vender prefix. +# TODO(bashi): Remove this after WebKit stops using vendor prefix. _available_processors[common.X_WEBKIT_DEFLATE_FRAME_EXTENSION] = ( DeflateFrameExtensionProcessor) @@ -270,21 +305,22 @@ def _create_accepted_method_desc(method_name, method_params): return common.format_extension(extension) -class PerFrameCompressionExtensionProcessor(ExtensionProcessorInterface): - """WebSocket Per-frame compression extension processor.""" +class CompressionExtensionProcessorBase(ExtensionProcessorInterface): + """Base class for Per-frame and Per-message compression extension.""" _METHOD_PARAM = 'method' - _DEFLATE_METHOD = 'deflate' def __init__(self, request): self._logger = util.get_class_logger(self) self._request = request self._compression_method_name = None self._compression_processor = None + self._compression_processor_hook = None + + def name(self): + return '' def _lookup_compression_processor(self, method_desc): - if method_desc.name() == self._DEFLATE_METHOD: - return DeflateFrameExtensionProcessor(method_desc) return None def _get_compression_processor_response(self): @@ -311,6 +347,10 @@ class PerFrameCompressionExtensionProcessor(ExtensionProcessorInterface): break if compression_processor is None: return None + + if self._compression_processor_hook: + self._compression_processor_hook(compression_processor) + processor_response = compression_processor.get_extension_response() if processor_response is None: return None @@ -337,14 +377,345 @@ class PerFrameCompressionExtensionProcessor(ExtensionProcessorInterface): return self._compression_processor.setup_stream_options(stream_options) + def set_compression_processor_hook(self, hook): + self._compression_processor_hook = hook + def get_compression_processor(self): return self._compression_processor +class PerFrameCompressionExtensionProcessor(CompressionExtensionProcessorBase): + """WebSocket Per-frame compression extension processor. + + Specification: + http://tools.ietf.org/html/draft-ietf-hybi-websocket-perframe-compression + """ + + _DEFLATE_METHOD = 'deflate' + + def __init__(self, request): + CompressionExtensionProcessorBase.__init__(self, request) + + def name(self): + return common.PERFRAME_COMPRESSION_EXTENSION + + def _lookup_compression_processor(self, method_desc): + if method_desc.name() == self._DEFLATE_METHOD: + return DeflateFrameExtensionProcessor(method_desc) + return None + + _available_processors[common.PERFRAME_COMPRESSION_EXTENSION] = ( PerFrameCompressionExtensionProcessor) +class DeflateMessageProcessor(ExtensionProcessorInterface): + """Per-message deflate processor.""" + + _S2C_MAX_WINDOW_BITS_PARAM = 's2c_max_window_bits' + _S2C_NO_CONTEXT_TAKEOVER_PARAM = 's2c_no_context_takeover' + _C2S_MAX_WINDOW_BITS_PARAM = 'c2s_max_window_bits' + _C2S_NO_CONTEXT_TAKEOVER_PARAM = 'c2s_no_context_takeover' + + def __init__(self, request): + self._request = request + self._logger = util.get_class_logger(self) + + self._c2s_max_window_bits = None + self._c2s_no_context_takeover = False + self._bfinal = False + + self._compress_outgoing_enabled = False + + # True if a message is fragmented and compression is ongoing. + self._compress_ongoing = False + + # Counters for statistics. + + # Total number of outgoing bytes supplied to this filter. + self._total_outgoing_payload_bytes = 0 + # Total number of bytes sent to the network after applying this filter. + self._total_filtered_outgoing_payload_bytes = 0 + + # Total number of bytes received from the network. + self._total_incoming_payload_bytes = 0 + # Total number of incoming bytes obtained after applying this filter. + self._total_filtered_incoming_payload_bytes = 0 + + def name(self): + return 'deflate' + + def get_extension_response(self): + # Any unknown parameter will be just ignored. + + s2c_max_window_bits = self._request.get_parameter_value( + self._S2C_MAX_WINDOW_BITS_PARAM) + if s2c_max_window_bits is not None: + try: + s2c_max_window_bits = int(s2c_max_window_bits) + except ValueError, e: + return None + if s2c_max_window_bits < 8 or s2c_max_window_bits > 15: + return None + + s2c_no_context_takeover = self._request.has_parameter( + self._S2C_NO_CONTEXT_TAKEOVER_PARAM) + if (s2c_no_context_takeover and + self._request.get_parameter_value( + self._S2C_NO_CONTEXT_TAKEOVER_PARAM) is not None): + return None + + self._deflater = util._RFC1979Deflater( + s2c_max_window_bits, s2c_no_context_takeover) + + self._inflater = util._RFC1979Inflater() + + self._compress_outgoing_enabled = True + + response = common.ExtensionParameter(self._request.name()) + + if s2c_max_window_bits is not None: + response.add_parameter( + self._S2C_MAX_WINDOW_BITS_PARAM, str(s2c_max_window_bits)) + + if s2c_no_context_takeover: + response.add_parameter( + self._S2C_NO_CONTEXT_TAKEOVER_PARAM, None) + + if self._c2s_max_window_bits is not None: + response.add_parameter( + self._C2S_MAX_WINDOW_BITS_PARAM, + str(self._c2s_max_window_bits)) + if self._c2s_no_context_takeover: + response.add_parameter( + self._C2S_NO_CONTEXT_TAKEOVER_PARAM, None) + + self._logger.debug( + 'Enable %s extension (' + 'request: s2c_max_window_bits=%s; s2c_no_context_takeover=%r, ' + 'response: c2s_max_window_bits=%s; c2s_no_context_takeover=%r)' % + (self._request.name(), + s2c_max_window_bits, + s2c_no_context_takeover, + self._c2s_max_window_bits, + self._c2s_no_context_takeover)) + + return response + + def setup_stream_options(self, stream_options): + class _OutgoingMessageFilter(object): + + def __init__(self, parent): + self._parent = parent + + def filter(self, message, end=True, binary=False): + return self._parent._process_outgoing_message( + message, end, binary) + + class _IncomingMessageFilter(object): + + def __init__(self, parent): + self._parent = parent + self._decompress_next_message = False + + def decompress_next_message(self): + self._decompress_next_message = True + + def filter(self, message): + message = self._parent._process_incoming_message( + message, self._decompress_next_message) + self._decompress_next_message = False + return message + + self._outgoing_message_filter = _OutgoingMessageFilter(self) + self._incoming_message_filter = _IncomingMessageFilter(self) + stream_options.outgoing_message_filters.append( + self._outgoing_message_filter) + stream_options.incoming_message_filters.append( + self._incoming_message_filter) + + class _OutgoingFrameFilter(object): + + def __init__(self, parent): + self._parent = parent + self._set_compression_bit = False + + def set_compression_bit(self): + self._set_compression_bit = True + + def filter(self, frame): + self._parent._process_outgoing_frame( + frame, self._set_compression_bit) + self._set_compression_bit = False + + class _IncomingFrameFilter(object): + + def __init__(self, parent): + self._parent = parent + + def filter(self, frame): + self._parent._process_incoming_frame(frame) + + self._outgoing_frame_filter = _OutgoingFrameFilter(self) + self._incoming_frame_filter = _IncomingFrameFilter(self) + stream_options.outgoing_frame_filters.append( + self._outgoing_frame_filter) + stream_options.incoming_frame_filters.append( + self._incoming_frame_filter) + + stream_options.encode_text_message_to_utf8 = False + + def set_c2s_max_window_bits(self, value): + self._c2s_max_window_bits = value + + def set_c2s_no_context_takeover(self, value): + self._c2s_no_context_takeover = value + + def set_bfinal(self, value): + self._bfinal = value + + def enable_outgoing_compression(self): + self._compress_outgoing_enabled = True + + def disable_outgoing_compression(self): + self._compress_outgoing_enabled = False + + def _process_incoming_message(self, message, decompress): + if not decompress: + return message + + received_payload_size = len(message) + self._total_incoming_payload_bytes += received_payload_size + + message = self._inflater.filter(message) + + filtered_payload_size = len(message) + self._total_filtered_incoming_payload_bytes += filtered_payload_size + + _log_decompression_ratio(self._logger, received_payload_size, + self._total_incoming_payload_bytes, + filtered_payload_size, + self._total_filtered_incoming_payload_bytes) + + return message + + def _process_outgoing_message(self, message, end, binary): + if not binary: + message = message.encode('utf-8') + + if not self._compress_outgoing_enabled: + return message + + original_payload_size = len(message) + self._total_outgoing_payload_bytes += original_payload_size + + message = self._deflater.filter( + message, flush=end, bfinal=self._bfinal) + + filtered_payload_size = len(message) + self._total_filtered_outgoing_payload_bytes += filtered_payload_size + + _log_compression_ratio(self._logger, original_payload_size, + self._total_outgoing_payload_bytes, + filtered_payload_size, + self._total_filtered_outgoing_payload_bytes) + + if not self._compress_ongoing: + self._outgoing_frame_filter.set_compression_bit() + self._compress_ongoing = not end + return message + + def _process_incoming_frame(self, frame): + if frame.rsv1 == 1 and not common.is_control_opcode(frame.opcode): + self._incoming_message_filter.decompress_next_message() + frame.rsv1 = 0 + + def _process_outgoing_frame(self, frame, compression_bit): + if (not compression_bit or + common.is_control_opcode(frame.opcode)): + return + + frame.rsv1 = 1 + + +class PerMessageCompressionExtensionProcessor( + CompressionExtensionProcessorBase): + """WebSocket Per-message compression extension processor. + + Specification: + http://tools.ietf.org/html/draft-ietf-hybi-permessage-compression + """ + + _DEFLATE_METHOD = 'deflate' + + def __init__(self, request): + CompressionExtensionProcessorBase.__init__(self, request) + + def name(self): + return common.PERMESSAGE_COMPRESSION_EXTENSION + + def _lookup_compression_processor(self, method_desc): + if method_desc.name() == self._DEFLATE_METHOD: + return DeflateMessageProcessor(method_desc) + return None + + +_available_processors[common.PERMESSAGE_COMPRESSION_EXTENSION] = ( + PerMessageCompressionExtensionProcessor) + + +# Adding vendor-prefixed permessage-compress extension. +# TODO(bashi): Remove this after WebKit stops using vendor prefix. +_available_processors[common.X_WEBKIT_PERMESSAGE_COMPRESSION_EXTENSION] = ( + PerMessageCompressionExtensionProcessor) + + +class MuxExtensionProcessor(ExtensionProcessorInterface): + """WebSocket multiplexing extension processor.""" + + _QUOTA_PARAM = 'quota' + + def __init__(self, request): + self._request = request + + def name(self): + return common.MUX_EXTENSION + + def get_extension_response(self, ws_request, + logical_channel_extensions): + # Mux extension cannot be used after extensions that depend on + # frame boundary, extension data field, or any reserved bits + # which are attributed to each frame. + for extension in logical_channel_extensions: + name = extension.name() + if (name == common.PERFRAME_COMPRESSION_EXTENSION or + name == common.DEFLATE_FRAME_EXTENSION or + name == common.X_WEBKIT_DEFLATE_FRAME_EXTENSION): + return None + + quota = self._request.get_parameter_value(self._QUOTA_PARAM) + if quota is None: + ws_request.mux_quota = 0 + else: + try: + quota = int(quota) + except ValueError, e: + return None + if quota < 0 or quota >= 2 ** 32: + return None + ws_request.mux_quota = quota + + ws_request.mux = True + ws_request.mux_extensions = logical_channel_extensions + return common.ExtensionParameter(common.MUX_EXTENSION) + + def setup_stream_options(self, stream_options): + pass + + +_available_processors[common.MUX_EXTENSION] = MuxExtensionProcessor + + def get_extension_processor(extension_request): global _available_processors processor_class = _available_processors.get(extension_request.name()) diff --git a/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/handshake/__init__.py b/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/handshake/__init__.py index 10a178314..194f6b395 100644 --- a/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/handshake/__init__.py +++ b/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/handshake/__init__.py @@ -37,7 +37,6 @@ successfully established. import logging from mod_pywebsocket import common -from mod_pywebsocket.handshake import draft75 from mod_pywebsocket.handshake import hybi00 from mod_pywebsocket.handshake import hybi # Export AbortedByUserException, HandshakeException, and VersionException @@ -56,10 +55,8 @@ def do_handshake(request, dispatcher, allowDraft75=False, strict=False): Args: request: mod_python request. dispatcher: Dispatcher (dispatch.Dispatcher). - allowDraft75: allow draft 75 handshake protocol. - strict: Strictly check handshake request in draft 75. - Default: False. If True, request.connection must provide - get_memorized_lines method. + allowDraft75: obsolete argument. ignored. + strict: obsolete argument. ignored. Handshaker will add attributes such as ws_resource in performing handshake. @@ -86,9 +83,6 @@ def do_handshake(request, dispatcher, allowDraft75=False, strict=False): ('RFC 6455', hybi.Handshaker(request, dispatcher))) handshakers.append( ('HyBi 00', hybi00.Handshaker(request, dispatcher))) - if allowDraft75: - handshakers.append( - ('Hixie 75', draft75.Handshaker(request, dispatcher, strict))) for name, handshaker in handshakers: _LOGGER.debug('Trying protocol version %s', name) diff --git a/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/handshake/_base.py b/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/handshake/_base.py index bc095b129..e5c94ca90 100644 --- a/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/handshake/_base.py +++ b/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/handshake/_base.py @@ -85,13 +85,16 @@ def get_default_port(is_secure): def validate_subprotocol(subprotocol, hixie): - """Validate a value in subprotocol fields such as WebSocket-Protocol, - Sec-WebSocket-Protocol. + """Validate a value in the Sec-WebSocket-Protocol field. See - RFC 6455: Section 4.1., 4.2.2., and 4.3. - HyBi 00: Section 4.1. Opening handshake - - Hixie 75: Section 4.1. Handshake + + Args: + hixie: if True, checks if characters in subprotocol are in range + between U+0020 and U+007E. It's required by HyBi 00 but not by + RFC 6455. """ if not subprotocol: @@ -170,7 +173,11 @@ def check_request_line(request): # 5.1 1. The three character UTF-8 string "GET". # 5.1 2. A UTF-8-encoded U+0020 SPACE character (0x20 byte). if request.method != 'GET': - raise HandshakeException('Method is not GET') + raise HandshakeException('Method is not GET: %r' % request.method) + + if request.protocol != 'HTTP/1.1': + raise HandshakeException('Version is not HTTP/1.1: %r' % + request.protocol) def check_header_lines(request, mandatory_headers): diff --git a/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/handshake/draft75.py b/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/handshake/draft75.py deleted file mode 100644 index 802a31c9a..000000000 --- a/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/handshake/draft75.py +++ /dev/null @@ -1,190 +0,0 @@ -# Copyright 2011, Google Inc. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - -"""WebSocket handshaking defined in draft-hixie-thewebsocketprotocol-75.""" - - -# Note: request.connection.write is used in this module, even though mod_python -# document says that it should be used only in connection handlers. -# Unfortunately, we have no other options. For example, request.write is not -# suitable because it doesn't allow direct raw bytes writing. - - -import logging -import re - -from mod_pywebsocket import common -from mod_pywebsocket.stream import StreamHixie75 -from mod_pywebsocket import util -from mod_pywebsocket.handshake._base import HandshakeException -from mod_pywebsocket.handshake._base import build_location -from mod_pywebsocket.handshake._base import validate_subprotocol - - -_MANDATORY_HEADERS = [ - # key, expected value or None - ['Upgrade', 'WebSocket'], - ['Connection', 'Upgrade'], - ['Host', None], - ['Origin', None], -] - -_FIRST_FIVE_LINES = map(re.compile, [ - r'^GET /[\S]* HTTP/1.1\r\n$', - r'^Upgrade: WebSocket\r\n$', - r'^Connection: Upgrade\r\n$', - r'^Host: [\S]+\r\n$', - r'^Origin: [\S]+\r\n$', -]) - -_SIXTH_AND_LATER = re.compile( - r'^' - r'(WebSocket-Protocol: [\x20-\x7e]+\r\n)?' - r'(Cookie: [^\r]*\r\n)*' - r'(Cookie2: [^\r]*\r\n)?' - r'(Cookie: [^\r]*\r\n)*' - r'\r\n') - - -class Handshaker(object): - """This class performs WebSocket handshake.""" - - def __init__(self, request, dispatcher, strict=False): - """Construct an instance. - - Args: - request: mod_python request. - dispatcher: Dispatcher (dispatch.Dispatcher). - strict: Strictly check handshake request. Default: False. - If True, request.connection must provide get_memorized_lines - method. - - Handshaker will add attributes such as ws_resource in performing - handshake. - """ - - self._logger = util.get_class_logger(self) - - self._request = request - self._dispatcher = dispatcher - self._strict = strict - - def do_handshake(self): - """Perform WebSocket Handshake. - - On _request, we set - ws_resource, ws_origin, ws_location, ws_protocol - ws_challenge_md5: WebSocket handshake information. - ws_stream: Frame generation/parsing class. - ws_version: Protocol version. - """ - - self._check_header_lines() - self._set_resource() - self._set_origin() - self._set_location() - self._set_subprotocol() - self._set_protocol_version() - - self._dispatcher.do_extra_handshake(self._request) - - self._send_handshake() - - self._logger.debug('Sent opening handshake response') - - def _set_resource(self): - self._request.ws_resource = self._request.uri - - def _set_origin(self): - self._request.ws_origin = self._request.headers_in['Origin'] - - def _set_location(self): - self._request.ws_location = build_location(self._request) - - def _set_subprotocol(self): - subprotocol = self._request.headers_in.get('WebSocket-Protocol') - if subprotocol is not None: - validate_subprotocol(subprotocol, hixie=True) - self._request.ws_protocol = subprotocol - - def _set_protocol_version(self): - self._logger.debug('IETF Hixie 75 protocol') - self._request.ws_version = common.VERSION_HIXIE75 - self._request.ws_stream = StreamHixie75(self._request) - - def _sendall(self, data): - self._request.connection.write(data) - - def _send_handshake(self): - self._sendall('HTTP/1.1 101 Web Socket Protocol Handshake\r\n') - self._sendall('Upgrade: WebSocket\r\n') - self._sendall('Connection: Upgrade\r\n') - self._sendall('WebSocket-Origin: %s\r\n' % self._request.ws_origin) - self._sendall('WebSocket-Location: %s\r\n' % self._request.ws_location) - if self._request.ws_protocol: - self._sendall( - 'WebSocket-Protocol: %s\r\n' % self._request.ws_protocol) - self._sendall('\r\n') - - def _check_header_lines(self): - for key, expected_value in _MANDATORY_HEADERS: - actual_value = self._request.headers_in.get(key) - if not actual_value: - raise HandshakeException('Header %s is not defined' % key) - if expected_value: - if actual_value != expected_value: - raise HandshakeException( - 'Expected %r for header %s but found %r' % - (expected_value, key, actual_value)) - if self._strict: - try: - lines = self._request.connection.get_memorized_lines() - except AttributeError, e: - raise AttributeError( - 'Strict handshake is specified but the connection ' - 'doesn\'t provide get_memorized_lines()') - self._check_first_lines(lines) - - def _check_first_lines(self, lines): - if len(lines) < len(_FIRST_FIVE_LINES): - raise HandshakeException('Too few header lines: %d' % len(lines)) - for line, regexp in zip(lines, _FIRST_FIVE_LINES): - if not regexp.search(line): - raise HandshakeException( - 'Unexpected header: %r doesn\'t match %r' - % (line, regexp.pattern)) - sixth_and_later = ''.join(lines[5:]) - if not _SIXTH_AND_LATER.search(sixth_and_later): - raise HandshakeException( - 'Unexpected header: %r doesn\'t match %r' - % (sixth_and_later, _SIXTH_AND_LATER.pattern)) - - -# vi:sts=4 sw=4 et diff --git a/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/handshake/hybi.py b/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/handshake/hybi.py index 2883acbf8..fc0e2a096 100644 --- a/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/handshake/hybi.py +++ b/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/handshake/hybi.py @@ -182,34 +182,60 @@ class Handshaker(object): # Extra handshake handler may modify/remove processors. self._dispatcher.do_extra_handshake(self._request) + processors = filter(lambda processor: processor is not None, + self._request.ws_extension_processors) + + accepted_extensions = [] + + # We need to take care of mux extension here. Extensions that + # are placed before mux should be applied to logical channels. + mux_index = -1 + for i, processor in enumerate(processors): + if processor.name() == common.MUX_EXTENSION: + mux_index = i + break + if mux_index >= 0: + mux_processor = processors[mux_index] + logical_channel_processors = processors[:mux_index] + processors = processors[mux_index+1:] + + for processor in logical_channel_processors: + extension_response = processor.get_extension_response() + if extension_response is None: + # Rejected. + continue + accepted_extensions.append(extension_response) + # Pass a shallow copy of accepted_extensions as extensions for + # logical channels. + mux_response = mux_processor.get_extension_response( + self._request, accepted_extensions[:]) + if mux_response is not None: + accepted_extensions.append(mux_response) stream_options = StreamOptions() - self._request.ws_extensions = None - for processor in self._request.ws_extension_processors: - if processor is None: - # Some processors may be removed by extra handshake - # handler. - continue + # When there is mux extension, here, |processors| contain only + # prosessors for extensions placed after mux. + for processor in processors: extension_response = processor.get_extension_response() if extension_response is None: # Rejected. continue - if self._request.ws_extensions is None: - self._request.ws_extensions = [] - self._request.ws_extensions.append(extension_response) + accepted_extensions.append(extension_response) processor.setup_stream_options(stream_options) - if self._request.ws_extensions is not None: + if len(accepted_extensions) > 0: + self._request.ws_extensions = accepted_extensions self._logger.debug( 'Extensions accepted: %r', - map(common.ExtensionParameter.name, - self._request.ws_extensions)) + map(common.ExtensionParameter.name, accepted_extensions)) + else: + self._request.ws_extensions = None - self._request.ws_stream = Stream(self._request, stream_options) + self._request.ws_stream = self._create_stream(stream_options) if self._request.ws_requested_protocols is not None: if self._request.ws_protocol is None: @@ -268,7 +294,7 @@ class Handshaker(object): protocol_header = self._request.headers_in.get( common.SEC_WEBSOCKET_PROTOCOL_HEADER) - if not protocol_header: + if protocol_header is None: self._request.ws_requested_protocols = None return @@ -341,7 +367,10 @@ class Handshaker(object): return key - def _send_handshake(self, accept): + def _create_stream(self, stream_options): + return Stream(self._request, stream_options) + + def _create_handshake_response(self, accept): response = [] response.append('HTTP/1.1 101 Switching Protocols\r\n') @@ -363,7 +392,10 @@ class Handshaker(object): common.format_extensions(self._request.ws_extensions))) response.append('\r\n') - raw_response = ''.join(response) + return ''.join(response) + + def _send_handshake(self, accept): + raw_response = self._create_handshake_response(accept) self._request.connection.write(raw_response) self._logger.debug('Sent server\'s opening handshake: %r', raw_response) diff --git a/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/headerparserhandler.py b/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/headerparserhandler.py index b68c240e1..2cc62de04 100644 --- a/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/headerparserhandler.py +++ b/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/headerparserhandler.py @@ -63,8 +63,9 @@ _PYOPT_ALLOW_HANDLERS_OUTSIDE_ROOT = ( _PYOPT_ALLOW_HANDLERS_OUTSIDE_ROOT_DEFINITION = { 'off': False, 'no': False, 'on': True, 'yes': True} -# PythonOption to specify to allow draft75 handshake. -# The default is None (Off) +# (Obsolete option. Ignored.) +# PythonOption to specify to allow handshake defined in Hixie 75 version +# protocol. The default is None (Off) _PYOPT_ALLOW_DRAFT75 = 'mod_pywebsocket.allow_draft75' # Map from values to their meanings. _PYOPT_ALLOW_DRAFT75_DEFINITION = {'off': False, 'on': True} diff --git a/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/msgutil.py b/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/msgutil.py index 21ffdacf6..4c1a0114b 100644 --- a/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/msgutil.py +++ b/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/msgutil.py @@ -59,20 +59,20 @@ def close_connection(request): request.ws_stream.close_connection() -def send_message(request, message, end=True, binary=False): - """Send message. +def send_message(request, payload_data, end=True, binary=False): + """Send a message (or part of a message). Args: request: mod_python request. - message: unicode text or str binary to send. - end: False to send message as a fragment. All messages until the - first call with end=True (inclusive) will be delivered to the - client in separate frames but as one WebSocket message. - binary: send message as binary frame. + payload_data: unicode text or str binary to send. + end: True to terminate a message. + False to send payload_data as part of a message that is to be + terminated by next or later send_message call with end=True. + binary: send payload_data as binary frame(s). Raises: BadOperationException: when server already terminated. """ - request.ws_stream.send_message(message, end, binary) + request.ws_stream.send_message(payload_data, end, binary) def receive_message(request): diff --git a/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/mux.py b/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/mux.py new file mode 100644 index 000000000..f0bdd2461 --- /dev/null +++ b/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/mux.py @@ -0,0 +1,1636 @@ +# Copyright 2012, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +"""This file provides classes and helper functions for multiplexing extension. + +Specification: +http://tools.ietf.org/html/draft-ietf-hybi-websocket-multiplexing-06 +""" + + +import collections +import copy +import email +import email.parser +import logging +import math +import struct +import threading +import traceback + +from mod_pywebsocket import common +from mod_pywebsocket import handshake +from mod_pywebsocket import util +from mod_pywebsocket._stream_base import BadOperationException +from mod_pywebsocket._stream_base import ConnectionTerminatedException +from mod_pywebsocket._stream_hybi import Frame +from mod_pywebsocket._stream_hybi import Stream +from mod_pywebsocket._stream_hybi import StreamOptions +from mod_pywebsocket._stream_hybi import create_binary_frame +from mod_pywebsocket._stream_hybi import create_closing_handshake_body +from mod_pywebsocket._stream_hybi import create_header +from mod_pywebsocket._stream_hybi import create_length_header +from mod_pywebsocket._stream_hybi import parse_frame +from mod_pywebsocket.handshake import hybi + + +_CONTROL_CHANNEL_ID = 0 +_DEFAULT_CHANNEL_ID = 1 + +_MUX_OPCODE_ADD_CHANNEL_REQUEST = 0 +_MUX_OPCODE_ADD_CHANNEL_RESPONSE = 1 +_MUX_OPCODE_FLOW_CONTROL = 2 +_MUX_OPCODE_DROP_CHANNEL = 3 +_MUX_OPCODE_NEW_CHANNEL_SLOT = 4 + +_MAX_CHANNEL_ID = 2 ** 29 - 1 + +_INITIAL_NUMBER_OF_CHANNEL_SLOTS = 64 +_INITIAL_QUOTA_FOR_CLIENT = 8 * 1024 + +_HANDSHAKE_ENCODING_IDENTITY = 0 +_HANDSHAKE_ENCODING_DELTA = 1 + +# We need only these status code for now. +_HTTP_BAD_RESPONSE_MESSAGES = { + common.HTTP_STATUS_BAD_REQUEST: 'Bad Request', +} + +# DropChannel reason code +# TODO(bashi): Define all reason code defined in -05 draft. +_DROP_CODE_NORMAL_CLOSURE = 1000 + +_DROP_CODE_INVALID_ENCAPSULATING_MESSAGE = 2001 +_DROP_CODE_CHANNEL_ID_TRUNCATED = 2002 +_DROP_CODE_ENCAPSULATED_FRAME_IS_TRUNCATED = 2003 +_DROP_CODE_UNKNOWN_MUX_OPCODE = 2004 +_DROP_CODE_INVALID_MUX_CONTROL_BLOCK = 2005 +_DROP_CODE_CHANNEL_ALREADY_EXISTS = 2006 +_DROP_CODE_NEW_CHANNEL_SLOT_VIOLATION = 2007 + +_DROP_CODE_UNKNOWN_REQUEST_ENCODING = 3002 +_DROP_CODE_SEND_QUOTA_VIOLATION = 3005 +_DROP_CODE_ACKNOWLEDGED = 3008 + + +class MuxUnexpectedException(Exception): + """Exception in handling multiplexing extension.""" + pass + + +# Temporary +class MuxNotImplementedException(Exception): + """Raised when a flow enters unimplemented code path.""" + pass + + +class LogicalConnectionClosedException(Exception): + """Raised when logical connection is gracefully closed.""" + pass + + +class PhysicalConnectionError(Exception): + """Raised when there is a physical connection error.""" + def __init__(self, drop_code, message=''): + super(PhysicalConnectionError, self).__init__( + 'code=%d, message=%r' % (drop_code, message)) + self.drop_code = drop_code + self.message = message + + +class LogicalChannelError(Exception): + """Raised when there is a logical channel error.""" + def __init__(self, channel_id, drop_code, message=''): + super(LogicalChannelError, self).__init__( + 'channel_id=%d, code=%d, message=%r' % ( + channel_id, drop_code, message)) + self.channel_id = channel_id + self.drop_code = drop_code + self.message = message + + +def _encode_channel_id(channel_id): + if channel_id < 0: + raise ValueError('Channel id %d must not be negative' % channel_id) + + if channel_id < 2 ** 7: + return chr(channel_id) + if channel_id < 2 ** 14: + return struct.pack('!H', 0x8000 + channel_id) + if channel_id < 2 ** 21: + first = chr(0xc0 + (channel_id >> 16)) + return first + struct.pack('!H', channel_id & 0xffff) + if channel_id < 2 ** 29: + return struct.pack('!L', 0xe0000000 + channel_id) + + raise ValueError('Channel id %d is too large' % channel_id) + + +def _encode_number(number): + return create_length_header(number, False) + + +def _create_add_channel_response(channel_id, encoded_handshake, + encoding=0, rejected=False, + outer_frame_mask=False): + if encoding != 0 and encoding != 1: + raise ValueError('Invalid encoding %d' % encoding) + + first_byte = ((_MUX_OPCODE_ADD_CHANNEL_RESPONSE << 5) | + (rejected << 4) | encoding) + block = (chr(first_byte) + + _encode_channel_id(channel_id) + + _encode_number(len(encoded_handshake)) + + encoded_handshake) + payload = _encode_channel_id(_CONTROL_CHANNEL_ID) + block + return create_binary_frame(payload, mask=outer_frame_mask) + + +def _create_drop_channel(channel_id, code=None, message='', + outer_frame_mask=False): + if len(message) > 0 and code is None: + raise ValueError('Code must be specified if message is specified') + + first_byte = _MUX_OPCODE_DROP_CHANNEL << 5 + block = chr(first_byte) + _encode_channel_id(channel_id) + if code is None: + block += _encode_number(0) # Reason size + else: + reason = struct.pack('!H', code) + message + reason_size = _encode_number(len(reason)) + block += reason_size + reason + + payload = _encode_channel_id(_CONTROL_CHANNEL_ID) + block + return create_binary_frame(payload, mask=outer_frame_mask) + + +def _create_flow_control(channel_id, replenished_quota, + outer_frame_mask=False): + first_byte = _MUX_OPCODE_FLOW_CONTROL << 5 + block = (chr(first_byte) + + _encode_channel_id(channel_id) + + _encode_number(replenished_quota)) + payload = _encode_channel_id(_CONTROL_CHANNEL_ID) + block + return create_binary_frame(payload, mask=outer_frame_mask) + + +def _create_new_channel_slot(slots, send_quota, outer_frame_mask=False): + if slots < 0 or send_quota < 0: + raise ValueError('slots and send_quota must be non-negative.') + first_byte = _MUX_OPCODE_NEW_CHANNEL_SLOT << 5 + block = (chr(first_byte) + + _encode_number(slots) + + _encode_number(send_quota)) + payload = _encode_channel_id(_CONTROL_CHANNEL_ID) + block + return create_binary_frame(payload, mask=outer_frame_mask) + + +def _create_fallback_new_channel_slot(outer_frame_mask=False): + first_byte = (_MUX_OPCODE_NEW_CHANNEL_SLOT << 5) | 1 # Set the F flag + block = (chr(first_byte) + _encode_number(0) + _encode_number(0)) + payload = _encode_channel_id(_CONTROL_CHANNEL_ID) + block + return create_binary_frame(payload, mask=outer_frame_mask) + + +def _parse_request_text(request_text): + request_line, header_lines = request_text.split('\r\n', 1) + + words = request_line.split(' ') + if len(words) != 3: + raise ValueError('Bad Request-Line syntax %r' % request_line) + [command, path, version] = words + if version != 'HTTP/1.1': + raise ValueError('Bad request version %r' % version) + + # email.parser.Parser() parses RFC 2822 (RFC 822) style headers. + # RFC 6455 refers RFC 2616 for handshake parsing, and RFC 2616 refers + # RFC 822. + headers = email.parser.Parser().parsestr(header_lines) + return command, path, version, headers + + +class _ControlBlock(object): + """A structure that holds parsing result of multiplexing control block. + Control block specific attributes will be added by _MuxFramePayloadParser. + (e.g. encoded_handshake will be added for AddChannelRequest and + AddChannelResponse) + """ + + def __init__(self, opcode): + self.opcode = opcode + + +class _MuxFramePayloadParser(object): + """A class that parses multiplexed frame payload.""" + + def __init__(self, payload): + self._data = payload + self._read_position = 0 + self._logger = util.get_class_logger(self) + + def read_channel_id(self): + """Reads channel id. + + Raises: + ValueError: when the payload doesn't contain + valid channel id. + """ + + remaining_length = len(self._data) - self._read_position + pos = self._read_position + if remaining_length == 0: + raise ValueError('Invalid channel id format') + + channel_id = ord(self._data[pos]) + channel_id_length = 1 + if channel_id & 0xe0 == 0xe0: + if remaining_length < 4: + raise ValueError('Invalid channel id format') + channel_id = struct.unpack('!L', + self._data[pos:pos+4])[0] & 0x1fffffff + channel_id_length = 4 + elif channel_id & 0xc0 == 0xc0: + if remaining_length < 3: + raise ValueError('Invalid channel id format') + channel_id = (((channel_id & 0x1f) << 16) + + struct.unpack('!H', self._data[pos+1:pos+3])[0]) + channel_id_length = 3 + elif channel_id & 0x80 == 0x80: + if remaining_length < 2: + raise ValueError('Invalid channel id format') + channel_id = struct.unpack('!H', + self._data[pos:pos+2])[0] & 0x3fff + channel_id_length = 2 + self._read_position += channel_id_length + + return channel_id + + def read_inner_frame(self): + """Reads an inner frame. + + Raises: + PhysicalConnectionError: when the inner frame is invalid. + """ + + if len(self._data) == self._read_position: + raise PhysicalConnectionError( + _DROP_CODE_ENCAPSULATED_FRAME_IS_TRUNCATED) + + bits = ord(self._data[self._read_position]) + self._read_position += 1 + fin = (bits & 0x80) == 0x80 + rsv1 = (bits & 0x40) == 0x40 + rsv2 = (bits & 0x20) == 0x20 + rsv3 = (bits & 0x10) == 0x10 + opcode = bits & 0xf + payload = self.remaining_data() + # Consume rest of the message which is payload data of the original + # frame. + self._read_position = len(self._data) + return fin, rsv1, rsv2, rsv3, opcode, payload + + def _read_number(self): + if self._read_position + 1 > len(self._data): + raise PhysicalConnectionError( + _DROP_CODE_INVALID_MUX_CONTROL_BLOCK, + 'Cannot read the first byte of number field') + + number = ord(self._data[self._read_position]) + if number & 0x80 == 0x80: + raise PhysicalConnectionError( + _DROP_CODE_INVALID_MUX_CONTROL_BLOCK, + 'The most significant bit of the first byte of number should ' + 'be unset') + self._read_position += 1 + pos = self._read_position + if number == 127: + if pos + 8 > len(self._data): + raise PhysicalConnectionError( + _DROP_CODE_INVALID_MUX_CONTROL_BLOCK, + 'Invalid number field') + self._read_position += 8 + number = struct.unpack('!Q', self._data[pos:pos+8])[0] + if number > 0x7FFFFFFFFFFFFFFF: + raise PhysicalConnectionError( + _DROP_CODE_INVALID_MUX_CONTROL_BLOCK, + 'Encoded number >= 2^63') + if number <= 0xFFFF: + raise PhysicalConnectionError( + _DROP_CODE_INVALID_MUX_CONTROL_BLOCK, + '%d should not be encoded by 9 bytes encoding' % number) + return number + if number == 126: + if pos + 2 > len(self._data): + raise PhysicalConnectionError( + _DROP_CODE_INVALID_MUX_CONTROL_BLOCK, + 'Invalid number field') + self._read_position += 2 + number = struct.unpack('!H', self._data[pos:pos+2])[0] + if number <= 125: + raise PhysicalConnectionError( + _DROP_CODE_INVALID_MUX_CONTROL_BLOCK, + '%d should not be encoded by 3 bytes encoding' % number) + return number + + def _read_size_and_contents(self): + """Reads data that consists of followings: + - the size of the contents encoded the same way as payload length + of the WebSocket Protocol with 1 bit padding at the head. + - the contents. + """ + + size = self._read_number() + pos = self._read_position + if pos + size > len(self._data): + raise PhysicalConnectionError( + _DROP_CODE_INVALID_MUX_CONTROL_BLOCK, + 'Cannot read %d bytes data' % size) + + self._read_position += size + return self._data[pos:pos+size] + + def _read_add_channel_request(self, first_byte, control_block): + reserved = (first_byte >> 2) & 0x7 + if reserved != 0: + raise PhysicalConnectionError( + _DROP_CODE_INVALID_MUX_CONTROL_BLOCK, + 'Reserved bits must be unset') + + # Invalid encoding will be handled by MuxHandler. + encoding = first_byte & 0x3 + try: + control_block.channel_id = self.read_channel_id() + except ValueError, e: + raise PhysicalConnectionError(_DROP_CODE_INVALID_MUX_CONTROL_BLOCK) + control_block.encoding = encoding + encoded_handshake = self._read_size_and_contents() + control_block.encoded_handshake = encoded_handshake + return control_block + + def _read_add_channel_response(self, first_byte, control_block): + reserved = (first_byte >> 2) & 0x3 + if reserved != 0: + raise PhysicalConnectionError( + _DROP_CODE_INVALID_MUX_CONTROL_BLOCK, + 'Reserved bits must be unset') + + control_block.accepted = (first_byte >> 4) & 1 + control_block.encoding = first_byte & 0x3 + try: + control_block.channel_id = self.read_channel_id() + except ValueError, e: + raise PhysicalConnectionError(_DROP_CODE_INVALID_MUX_CONTROL_BLOCK) + control_block.encoded_handshake = self._read_size_and_contents() + return control_block + + def _read_flow_control(self, first_byte, control_block): + reserved = first_byte & 0x1f + if reserved != 0: + raise PhysicalConnectionError( + _DROP_CODE_INVALID_MUX_CONTROL_BLOCK, + 'Reserved bits must be unset') + + try: + control_block.channel_id = self.read_channel_id() + except ValueError, e: + raise PhysicalConnectionError(_DROP_CODE_INVALID_MUX_CONTROL_BLOCK) + control_block.send_quota = self._read_number() + return control_block + + def _read_drop_channel(self, first_byte, control_block): + reserved = first_byte & 0x1f + if reserved != 0: + raise PhysicalConnectionError( + _DROP_CODE_INVALID_MUX_CONTROL_BLOCK, + 'Reserved bits must be unset') + + try: + control_block.channel_id = self.read_channel_id() + except ValueError, e: + raise PhysicalConnectionError(_DROP_CODE_INVALID_MUX_CONTROL_BLOCK) + reason = self._read_size_and_contents() + if len(reason) == 0: + control_block.drop_code = None + control_block.drop_message = '' + elif len(reason) >= 2: + control_block.drop_code = struct.unpack('!H', reason[:2])[0] + control_block.drop_message = reason[2:] + else: + raise PhysicalConnectionError( + _DROP_CODE_INVALID_MUX_CONTROL_BLOCK, + 'Received DropChannel that conains only 1-byte reason') + return control_block + + def _read_new_channel_slot(self, first_byte, control_block): + reserved = first_byte & 0x1e + if reserved != 0: + raise PhysicalConnectionError( + _DROP_CODE_INVALID_MUX_CONTROL_BLOCK, + 'Reserved bits must be unset') + control_block.fallback = first_byte & 1 + control_block.slots = self._read_number() + control_block.send_quota = self._read_number() + return control_block + + def read_control_blocks(self): + """Reads control block(s). + + Raises: + PhysicalConnectionError: when the payload contains invalid control + block(s). + StopIteration: when no control blocks left. + """ + + while self._read_position < len(self._data): + first_byte = ord(self._data[self._read_position]) + self._read_position += 1 + opcode = (first_byte >> 5) & 0x7 + control_block = _ControlBlock(opcode=opcode) + if opcode == _MUX_OPCODE_ADD_CHANNEL_REQUEST: + yield self._read_add_channel_request(first_byte, control_block) + elif opcode == _MUX_OPCODE_ADD_CHANNEL_RESPONSE: + yield self._read_add_channel_response( + first_byte, control_block) + elif opcode == _MUX_OPCODE_FLOW_CONTROL: + yield self._read_flow_control(first_byte, control_block) + elif opcode == _MUX_OPCODE_DROP_CHANNEL: + yield self._read_drop_channel(first_byte, control_block) + elif opcode == _MUX_OPCODE_NEW_CHANNEL_SLOT: + yield self._read_new_channel_slot(first_byte, control_block) + else: + raise PhysicalConnectionError( + _DROP_CODE_UNKNOWN_MUX_OPCODE, + 'Invalid opcode %d' % opcode) + + assert self._read_position == len(self._data) + raise StopIteration + + def remaining_data(self): + """Returns remaining data.""" + + return self._data[self._read_position:] + + +class _LogicalRequest(object): + """Mimics mod_python request.""" + + def __init__(self, channel_id, command, path, protocol, headers, + connection): + """Constructs an instance. + + Args: + channel_id: the channel id of the logical channel. + command: HTTP request command. + path: HTTP request path. + headers: HTTP headers. + connection: _LogicalConnection instance. + """ + + self.channel_id = channel_id + self.method = command + self.uri = path + self.protocol = protocol + self.headers_in = headers + self.connection = connection + self.server_terminated = False + self.client_terminated = False + + def is_https(self): + """Mimics request.is_https(). Returns False because this method is + used only by old protocols (hixie and hybi00). + """ + + return False + + +class _LogicalConnection(object): + """Mimics mod_python mp_conn.""" + + # For details, see the comment of set_read_state(). + STATE_ACTIVE = 1 + STATE_GRACEFULLY_CLOSED = 2 + STATE_TERMINATED = 3 + + def __init__(self, mux_handler, channel_id): + """Constructs an instance. + + Args: + mux_handler: _MuxHandler instance. + channel_id: channel id of this connection. + """ + + self._mux_handler = mux_handler + self._channel_id = channel_id + self._incoming_data = '' + self._write_condition = threading.Condition() + self._waiting_write_completion = False + self._read_condition = threading.Condition() + self._read_state = self.STATE_ACTIVE + + def get_local_addr(self): + """Getter to mimic mp_conn.local_addr.""" + + return self._mux_handler.physical_connection.get_local_addr() + local_addr = property(get_local_addr) + + def get_remote_addr(self): + """Getter to mimic mp_conn.remote_addr.""" + + return self._mux_handler.physical_connection.get_remote_addr() + remote_addr = property(get_remote_addr) + + def get_memorized_lines(self): + """Gets memorized lines. Not supported.""" + + raise MuxUnexpectedException('_LogicalConnection does not support ' + 'get_memorized_lines') + + def write(self, data): + """Writes data. mux_handler sends data asynchronously. The caller will + be suspended until write done. + + Args: + data: data to be written. + + Raises: + MuxUnexpectedException: when called before finishing the previous + write. + """ + + try: + self._write_condition.acquire() + if self._waiting_write_completion: + raise MuxUnexpectedException( + 'Logical connection %d is already waiting the completion ' + 'of write' % self._channel_id) + + self._waiting_write_completion = True + self._mux_handler.send_data(self._channel_id, data) + self._write_condition.wait() + finally: + self._write_condition.release() + + def write_control_data(self, data): + """Writes data via the control channel. Don't wait finishing write + because this method can be called by mux dispatcher. + + Args: + data: data to be written. + """ + + self._mux_handler.send_control_data(data) + + def notify_write_done(self): + """Called when sending data is completed.""" + + try: + self._write_condition.acquire() + if not self._waiting_write_completion: + raise MuxUnexpectedException( + 'Invalid call of notify_write_done for logical connection' + ' %d' % self._channel_id) + self._waiting_write_completion = False + self._write_condition.notify() + finally: + self._write_condition.release() + + def append_frame_data(self, frame_data): + """Appends incoming frame data. Called when mux_handler dispatches + frame data to the corresponding application. + + Args: + frame_data: incoming frame data. + """ + + self._read_condition.acquire() + self._incoming_data += frame_data + self._read_condition.notify() + self._read_condition.release() + + def read(self, length): + """Reads data. Blocks until enough data has arrived via physical + connection. + + Args: + length: length of data to be read. + Raises: + LogicalConnectionClosedException: when closing handshake for this + logical channel has been received. + ConnectionTerminatedException: when the physical connection has + closed, or an error is caused on the reader thread. + """ + + self._read_condition.acquire() + while (self._read_state == self.STATE_ACTIVE and + len(self._incoming_data) < length): + self._read_condition.wait() + + try: + if self._read_state == self.STATE_GRACEFULLY_CLOSED: + raise LogicalConnectionClosedException( + 'Logical channel %d has closed.' % self._channel_id) + elif self._read_state == self.STATE_TERMINATED: + raise ConnectionTerminatedException( + 'Receiving %d byte failed. Logical channel (%d) closed' % + (length, self._channel_id)) + + value = self._incoming_data[:length] + self._incoming_data = self._incoming_data[length:] + finally: + self._read_condition.release() + + return value + + def set_read_state(self, new_state): + """Sets the state of this connection. Called when an event for this + connection has occurred. + + Args: + new_state: state to be set. new_state must be one of followings: + - STATE_GRACEFULLY_CLOSED: when closing handshake for this + connection has been received. + - STATE_TERMINATED: when the physical connection has closed or + DropChannel of this connection has received. + """ + + self._read_condition.acquire() + self._read_state = new_state + self._read_condition.notify() + self._read_condition.release() + + +class _LogicalStream(Stream): + """Mimics the Stream class. This class interprets multiplexed WebSocket + frames. + """ + + def __init__(self, request, send_quota, receive_quota): + """Constructs an instance. + + Args: + request: _LogicalRequest instance. + send_quota: Initial send quota. + receive_quota: Initial receive quota. + """ + + # TODO(bashi): Support frame filters. + stream_options = StreamOptions() + # Physical stream is responsible for masking. + stream_options.unmask_receive = False + # Control frames can be fragmented on logical channel. + stream_options.allow_fragmented_control_frame = True + Stream.__init__(self, request, stream_options) + self._send_quota = send_quota + self._send_quota_condition = threading.Condition() + self._receive_quota = receive_quota + self._write_inner_frame_semaphore = threading.Semaphore() + + def _create_inner_frame(self, opcode, payload, end=True): + # TODO(bashi): Support extensions that use reserved bits. + first_byte = (end << 7) | opcode + return (_encode_channel_id(self._request.channel_id) + + chr(first_byte) + payload) + + def _write_inner_frame(self, opcode, payload, end=True): + payload_length = len(payload) + write_position = 0 + + try: + # An inner frame will be fragmented if there is no enough send + # quota. This semaphore ensures that fragmented inner frames are + # sent in order on the logical channel. + # Note that frames that come from other logical channels or + # multiplexing control blocks can be inserted between fragmented + # inner frames on the physical channel. + self._write_inner_frame_semaphore.acquire() + while write_position < payload_length: + try: + self._send_quota_condition.acquire() + while self._send_quota == 0: + self._logger.debug( + 'No quota. Waiting FlowControl message for %d.' % + self._request.channel_id) + self._send_quota_condition.wait() + + remaining = payload_length - write_position + write_length = min(self._send_quota, remaining) + inner_frame_end = ( + end and + (write_position + write_length == payload_length)) + + inner_frame = self._create_inner_frame( + opcode, + payload[write_position:write_position+write_length], + inner_frame_end) + frame_data = self._writer.build( + inner_frame, end=True, binary=True) + self._send_quota -= write_length + self._logger.debug('Consumed quota=%d, remaining=%d' % + (write_length, self._send_quota)) + finally: + self._send_quota_condition.release() + + # Writing data will block the worker so we need to release + # _send_quota_condition before writing. + self._logger.debug('Sending inner frame: %r' % frame_data) + self._request.connection.write(frame_data) + write_position += write_length + + opcode = common.OPCODE_CONTINUATION + + except ValueError, e: + raise BadOperationException(e) + finally: + self._write_inner_frame_semaphore.release() + + def replenish_send_quota(self, send_quota): + """Replenish send quota.""" + + self._send_quota_condition.acquire() + self._send_quota += send_quota + self._logger.debug('Replenished send quota for channel id %d: %d' % + (self._request.channel_id, self._send_quota)) + self._send_quota_condition.notify() + self._send_quota_condition.release() + + def consume_receive_quota(self, amount): + """Consumes receive quota. Returns False on failure.""" + + if self._receive_quota < amount: + self._logger.debug('Violate quota on channel id %d: %d < %d' % + (self._request.channel_id, + self._receive_quota, amount)) + return False + self._receive_quota -= amount + return True + + def send_message(self, message, end=True, binary=False): + """Override Stream.send_message.""" + + if self._request.server_terminated: + raise BadOperationException( + 'Requested send_message after sending out a closing handshake') + + if binary and isinstance(message, unicode): + raise BadOperationException( + 'Message for binary frame must be instance of str') + + if binary: + opcode = common.OPCODE_BINARY + else: + opcode = common.OPCODE_TEXT + message = message.encode('utf-8') + + self._write_inner_frame(opcode, message, end) + + def _receive_frame(self): + """Overrides Stream._receive_frame. + + In addition to call Stream._receive_frame, this method adds the amount + of payload to receiving quota and sends FlowControl to the client. + We need to do it here because Stream.receive_message() handles + control frames internally. + """ + + opcode, payload, fin, rsv1, rsv2, rsv3 = Stream._receive_frame(self) + amount = len(payload) + self._receive_quota += amount + frame_data = _create_flow_control(self._request.channel_id, + amount) + self._logger.debug('Sending flow control for %d, replenished=%d' % + (self._request.channel_id, amount)) + self._request.connection.write_control_data(frame_data) + return opcode, payload, fin, rsv1, rsv2, rsv3 + + def receive_message(self): + """Overrides Stream.receive_message.""" + + # Just call Stream.receive_message(), but catch + # LogicalConnectionClosedException, which is raised when the logical + # connection has closed gracefully. + try: + return Stream.receive_message(self) + except LogicalConnectionClosedException, e: + self._logger.debug('%s', e) + return None + + def _send_closing_handshake(self, code, reason): + """Overrides Stream._send_closing_handshake.""" + + body = create_closing_handshake_body(code, reason) + self._logger.debug('Sending closing handshake for %d: (%r, %r)' % + (self._request.channel_id, code, reason)) + self._write_inner_frame(common.OPCODE_CLOSE, body, end=True) + + self._request.server_terminated = True + + def send_ping(self, body=''): + """Overrides Stream.send_ping""" + + self._logger.debug('Sending ping on logical channel %d: %r' % + (self._request.channel_id, body)) + self._write_inner_frame(common.OPCODE_PING, body, end=True) + + self._ping_queue.append(body) + + def _send_pong(self, body): + """Overrides Stream._send_pong""" + + self._logger.debug('Sending pong on logical channel %d: %r' % + (self._request.channel_id, body)) + self._write_inner_frame(common.OPCODE_PONG, body, end=True) + + def close_connection(self, code=common.STATUS_NORMAL_CLOSURE, reason=''): + """Overrides Stream.close_connection.""" + + # TODO(bashi): Implement + self._logger.debug('Closing logical connection %d' % + self._request.channel_id) + self._request.server_terminated = True + + def _drain_received_data(self): + """Overrides Stream._drain_received_data. Nothing need to be done for + logical channel. + """ + + pass + + +class _OutgoingData(object): + """A structure that holds data to be sent via physical connection and + origin of the data. + """ + + def __init__(self, channel_id, data): + self.channel_id = channel_id + self.data = data + + +class _PhysicalConnectionWriter(threading.Thread): + """A thread that is responsible for writing data to physical connection. + + TODO(bashi): Make sure there is no thread-safety problem when the reader + thread reads data from the same socket at a time. + """ + + def __init__(self, mux_handler): + """Constructs an instance. + + Args: + mux_handler: _MuxHandler instance. + """ + + threading.Thread.__init__(self) + self._logger = util.get_class_logger(self) + self._mux_handler = mux_handler + self.setDaemon(True) + self._stop_requested = False + self._deque = collections.deque() + self._deque_condition = threading.Condition() + + def put_outgoing_data(self, data): + """Puts outgoing data. + + Args: + data: _OutgoingData instance. + + Raises: + BadOperationException: when the thread has been requested to + terminate. + """ + + try: + self._deque_condition.acquire() + if self._stop_requested: + raise BadOperationException('Cannot write data anymore') + + self._deque.append(data) + self._deque_condition.notify() + finally: + self._deque_condition.release() + + def _write_data(self, outgoing_data): + try: + self._mux_handler.physical_connection.write(outgoing_data.data) + except Exception, e: + util.prepend_message_to_exception( + 'Failed to send message to %r: ' % + (self._mux_handler.physical_connection.remote_addr,), e) + raise + + # TODO(bashi): It would be better to block the thread that sends + # control data as well. + if outgoing_data.channel_id != _CONTROL_CHANNEL_ID: + self._mux_handler.notify_write_done(outgoing_data.channel_id) + + def run(self): + self._deque_condition.acquire() + while not self._stop_requested: + if len(self._deque) == 0: + self._deque_condition.wait() + continue + + outgoing_data = self._deque.popleft() + self._deque_condition.release() + self._write_data(outgoing_data) + self._deque_condition.acquire() + + # Flush deque + try: + while len(self._deque) > 0: + outgoing_data = self._deque.popleft() + self._write_data(outgoing_data) + finally: + self._deque_condition.release() + + def stop(self): + """Stops the writer thread.""" + + self._deque_condition.acquire() + self._stop_requested = True + self._deque_condition.notify() + self._deque_condition.release() + + +class _PhysicalConnectionReader(threading.Thread): + """A thread that is responsible for reading data from physical connection. + """ + + def __init__(self, mux_handler): + """Constructs an instance. + + Args: + mux_handler: _MuxHandler instance. + """ + + threading.Thread.__init__(self) + self._logger = util.get_class_logger(self) + self._mux_handler = mux_handler + self.setDaemon(True) + + def run(self): + while True: + try: + physical_stream = self._mux_handler.physical_stream + message = physical_stream.receive_message() + if message is None: + break + # Below happens only when a data message is received. + opcode = physical_stream.get_last_received_opcode() + if opcode != common.OPCODE_BINARY: + self._mux_handler.fail_physical_connection( + _DROP_CODE_INVALID_ENCAPSULATING_MESSAGE, + 'Received a text message on physical connection') + break + + except ConnectionTerminatedException, e: + self._logger.debug('%s', e) + break + + try: + self._mux_handler.dispatch_message(message) + except PhysicalConnectionError, e: + self._mux_handler.fail_physical_connection( + e.drop_code, e.message) + break + except LogicalChannelError, e: + self._mux_handler.fail_logical_channel( + e.channel_id, e.drop_code, e.message) + except Exception, e: + self._logger.debug(traceback.format_exc()) + break + + self._mux_handler.notify_reader_done() + + +class _Worker(threading.Thread): + """A thread that is responsible for running the corresponding application + handler. + """ + + def __init__(self, mux_handler, request): + """Constructs an instance. + + Args: + mux_handler: _MuxHandler instance. + request: _LogicalRequest instance. + """ + + threading.Thread.__init__(self) + self._logger = util.get_class_logger(self) + self._mux_handler = mux_handler + self._request = request + self.setDaemon(True) + + def run(self): + self._logger.debug('Logical channel worker started. (id=%d)' % + self._request.channel_id) + try: + # Non-critical exceptions will be handled by dispatcher. + self._mux_handler.dispatcher.transfer_data(self._request) + finally: + self._mux_handler.notify_worker_done(self._request.channel_id) + + +class _MuxHandshaker(hybi.Handshaker): + """Opening handshake processor for multiplexing.""" + + _DUMMY_WEBSOCKET_KEY = 'dGhlIHNhbXBsZSBub25jZQ==' + + def __init__(self, request, dispatcher, send_quota, receive_quota): + """Constructs an instance. + Args: + request: _LogicalRequest instance. + dispatcher: Dispatcher instance (dispatch.Dispatcher). + send_quota: Initial send quota. + receive_quota: Initial receive quota. + """ + + hybi.Handshaker.__init__(self, request, dispatcher) + self._send_quota = send_quota + self._receive_quota = receive_quota + + # Append headers which should not be included in handshake field of + # AddChannelRequest. + # TODO(bashi): Make sure whether we should raise exception when + # these headers are included already. + request.headers_in[common.UPGRADE_HEADER] = ( + common.WEBSOCKET_UPGRADE_TYPE) + request.headers_in[common.CONNECTION_HEADER] = ( + common.UPGRADE_CONNECTION_TYPE) + request.headers_in[common.SEC_WEBSOCKET_VERSION_HEADER] = ( + str(common.VERSION_HYBI_LATEST)) + request.headers_in[common.SEC_WEBSOCKET_KEY_HEADER] = ( + self._DUMMY_WEBSOCKET_KEY) + + def _create_stream(self, stream_options): + """Override hybi.Handshaker._create_stream.""" + + self._logger.debug('Creating logical stream for %d' % + self._request.channel_id) + return _LogicalStream(self._request, self._send_quota, + self._receive_quota) + + def _create_handshake_response(self, accept): + """Override hybi._create_handshake_response.""" + + response = [] + + response.append('HTTP/1.1 101 Switching Protocols\r\n') + + # Upgrade, Connection and Sec-WebSocket-Accept should be excluded. + if self._request.ws_protocol is not None: + response.append('%s: %s\r\n' % ( + common.SEC_WEBSOCKET_PROTOCOL_HEADER, + self._request.ws_protocol)) + if (self._request.ws_extensions is not None and + len(self._request.ws_extensions) != 0): + response.append('%s: %s\r\n' % ( + common.SEC_WEBSOCKET_EXTENSIONS_HEADER, + common.format_extensions(self._request.ws_extensions))) + response.append('\r\n') + + return ''.join(response) + + def _send_handshake(self, accept): + """Override hybi.Handshaker._send_handshake.""" + + # Don't send handshake response for the default channel + if self._request.channel_id == _DEFAULT_CHANNEL_ID: + return + + handshake_response = self._create_handshake_response(accept) + frame_data = _create_add_channel_response( + self._request.channel_id, + handshake_response) + self._logger.debug('Sending handshake response for %d: %r' % + (self._request.channel_id, frame_data)) + self._request.connection.write_control_data(frame_data) + + +class _LogicalChannelData(object): + """A structure that holds information about logical channel. + """ + + def __init__(self, request, worker): + self.request = request + self.worker = worker + self.drop_code = _DROP_CODE_NORMAL_CLOSURE + self.drop_message = '' + + +class _HandshakeDeltaBase(object): + """A class that holds information for delta-encoded handshake.""" + + def __init__(self, headers): + self._headers = headers + + def create_headers(self, delta=None): + """Creates request headers for an AddChannelRequest that has + delta-encoded handshake. + + Args: + delta: headers should be overridden. + """ + + headers = copy.copy(self._headers) + if delta: + for key, value in delta.items(): + # The spec requires that a header with an empty value is + # removed from the delta base. + if len(value) == 0 and headers.has_key(key): + del headers[key] + else: + headers[key] = value + # TODO(bashi): Support extensions + headers['Sec-WebSocket-Extensions'] = '' + return headers + + +class _MuxHandler(object): + """Multiplexing handler. When a handler starts, it launches three + threads; the reader thread, the writer thread, and a worker thread. + + The reader thread reads data from the physical stream, i.e., the + ws_stream object of the underlying websocket connection. The reader + thread interprets multiplexed frames and dispatches them to logical + channels. Methods of this class are mostly called by the reader thread. + + The writer thread sends multiplexed frames which are created by + logical channels via the physical connection. + + The worker thread launched at the starting point handles the + "Implicitly Opened Connection". If multiplexing handler receives + an AddChannelRequest and accepts it, the handler will launch a new worker + thread and dispatch the request to it. + """ + + def __init__(self, request, dispatcher): + """Constructs an instance. + + Args: + request: mod_python request of the physical connection. + dispatcher: Dispatcher instance (dispatch.Dispatcher). + """ + + self.original_request = request + self.dispatcher = dispatcher + self.physical_connection = request.connection + self.physical_stream = request.ws_stream + self._logger = util.get_class_logger(self) + self._logical_channels = {} + self._logical_channels_condition = threading.Condition() + # Holds client's initial quota + self._channel_slots = collections.deque() + self._handshake_base = None + self._worker_done_notify_received = False + self._reader = None + self._writer = None + + def start(self): + """Starts the handler. + + Raises: + MuxUnexpectedException: when the handler already started, or when + opening handshake of the default channel fails. + """ + + if self._reader or self._writer: + raise MuxUnexpectedException('MuxHandler already started') + + self._reader = _PhysicalConnectionReader(self) + self._writer = _PhysicalConnectionWriter(self) + self._reader.start() + self._writer.start() + + # Create "Implicitly Opened Connection". + logical_connection = _LogicalConnection(self, _DEFAULT_CHANNEL_ID) + self._handshake_base = _HandshakeDeltaBase( + self.original_request.headers_in) + logical_request = _LogicalRequest( + _DEFAULT_CHANNEL_ID, + self.original_request.method, + self.original_request.uri, + self.original_request.protocol, + self._handshake_base.create_headers(), + logical_connection) + # Client's send quota for the implicitly opened connection is zero, + # but we will send FlowControl later so set the initial quota to + # _INITIAL_QUOTA_FOR_CLIENT. + self._channel_slots.append(_INITIAL_QUOTA_FOR_CLIENT) + if not self._do_handshake_for_logical_request( + logical_request, send_quota=self.original_request.mux_quota): + raise MuxUnexpectedException( + 'Failed handshake on the default channel id') + self._add_logical_channel(logical_request) + + # Send FlowControl for the implicitly opened connection. + frame_data = _create_flow_control(_DEFAULT_CHANNEL_ID, + _INITIAL_QUOTA_FOR_CLIENT) + logical_request.connection.write_control_data(frame_data) + + def add_channel_slots(self, slots, send_quota): + """Adds channel slots. + + Args: + slots: number of slots to be added. + send_quota: initial send quota for slots. + """ + + self._channel_slots.extend([send_quota] * slots) + # Send NewChannelSlot to client. + frame_data = _create_new_channel_slot(slots, send_quota) + self.send_control_data(frame_data) + + def wait_until_done(self, timeout=None): + """Waits until all workers are done. Returns False when timeout has + occurred. Returns True on success. + + Args: + timeout: timeout in sec. + """ + + self._logical_channels_condition.acquire() + try: + while len(self._logical_channels) > 0: + self._logger.debug('Waiting workers(%d)...' % + len(self._logical_channels)) + self._worker_done_notify_received = False + self._logical_channels_condition.wait(timeout) + if not self._worker_done_notify_received: + self._logger.debug('Waiting worker(s) timed out') + return False + + finally: + self._logical_channels_condition.release() + + # Flush pending outgoing data + self._writer.stop() + self._writer.join() + + return True + + def notify_write_done(self, channel_id): + """Called by the writer thread when a write operation has done. + + Args: + channel_id: objective channel id. + """ + + try: + self._logical_channels_condition.acquire() + if channel_id in self._logical_channels: + channel_data = self._logical_channels[channel_id] + channel_data.request.connection.notify_write_done() + else: + self._logger.debug('Seems that logical channel for %d has gone' + % channel_id) + finally: + self._logical_channels_condition.release() + + def send_control_data(self, data): + """Sends data via the control channel. + + Args: + data: data to be sent. + """ + + self._writer.put_outgoing_data(_OutgoingData( + channel_id=_CONTROL_CHANNEL_ID, data=data)) + + def send_data(self, channel_id, data): + """Sends data via given logical channel. This method is called by + worker threads. + + Args: + channel_id: objective channel id. + data: data to be sent. + """ + + self._writer.put_outgoing_data(_OutgoingData( + channel_id=channel_id, data=data)) + + def _send_drop_channel(self, channel_id, code=None, message=''): + frame_data = _create_drop_channel(channel_id, code, message) + self._logger.debug( + 'Sending drop channel for channel id %d' % channel_id) + self.send_control_data(frame_data) + + def _send_error_add_channel_response(self, channel_id, status=None): + if status is None: + status = common.HTTP_STATUS_BAD_REQUEST + + if status in _HTTP_BAD_RESPONSE_MESSAGES: + message = _HTTP_BAD_RESPONSE_MESSAGES[status] + else: + self._logger.debug('Response message for %d is not found' % status) + message = '???' + + response = 'HTTP/1.1 %d %s\r\n\r\n' % (status, message) + frame_data = _create_add_channel_response(channel_id, + encoded_handshake=response, + encoding=0, rejected=True) + self.send_control_data(frame_data) + + def _create_logical_request(self, block): + if block.channel_id == _CONTROL_CHANNEL_ID: + # TODO(bashi): Raise PhysicalConnectionError with code 2006 + # instead of MuxUnexpectedException. + raise MuxUnexpectedException( + 'Received the control channel id (0) as objective channel ' + 'id for AddChannel') + + if block.encoding > _HANDSHAKE_ENCODING_DELTA: + raise PhysicalConnectionError( + _DROP_CODE_UNKNOWN_REQUEST_ENCODING) + + method, path, version, headers = _parse_request_text( + block.encoded_handshake) + if block.encoding == _HANDSHAKE_ENCODING_DELTA: + headers = self._handshake_base.create_headers(headers) + + connection = _LogicalConnection(self, block.channel_id) + request = _LogicalRequest(block.channel_id, method, path, version, + headers, connection) + return request + + def _do_handshake_for_logical_request(self, request, send_quota=0): + try: + receive_quota = self._channel_slots.popleft() + except IndexError: + raise LogicalChannelError( + request.channel_id, _DROP_CODE_NEW_CHANNEL_SLOT_VIOLATION) + + handshaker = _MuxHandshaker(request, self.dispatcher, + send_quota, receive_quota) + try: + handshaker.do_handshake() + except handshake.VersionException, e: + self._logger.info('%s', e) + self._send_error_add_channel_response( + request.channel_id, status=common.HTTP_STATUS_BAD_REQUEST) + return False + except handshake.HandshakeException, e: + # TODO(bashi): Should we _Fail the Logical Channel_ with 3001 + # instead? + self._logger.info('%s', e) + self._send_error_add_channel_response(request.channel_id, + status=e.status) + return False + except handshake.AbortedByUserException, e: + self._logger.info('%s', e) + self._send_error_add_channel_response(request.channel_id) + return False + + return True + + def _add_logical_channel(self, logical_request): + try: + self._logical_channels_condition.acquire() + if logical_request.channel_id in self._logical_channels: + self._logger.debug('Channel id %d already exists' % + logical_request.channel_id) + raise PhysicalConnectionError( + _DROP_CODE_CHANNEL_ALREADY_EXISTS, + 'Channel id %d already exists' % + logical_request.channel_id) + worker = _Worker(self, logical_request) + channel_data = _LogicalChannelData(logical_request, worker) + self._logical_channels[logical_request.channel_id] = channel_data + worker.start() + finally: + self._logical_channels_condition.release() + + def _process_add_channel_request(self, block): + try: + logical_request = self._create_logical_request(block) + except ValueError, e: + self._logger.debug('Failed to create logical request: %r' % e) + self._send_error_add_channel_response( + block.channel_id, status=common.HTTP_STATUS_BAD_REQUEST) + return + if self._do_handshake_for_logical_request(logical_request): + if block.encoding == _HANDSHAKE_ENCODING_IDENTITY: + # Update handshake base. + # TODO(bashi): Make sure this is the right place to update + # handshake base. + self._handshake_base = _HandshakeDeltaBase( + logical_request.headers_in) + self._add_logical_channel(logical_request) + else: + self._send_error_add_channel_response( + block.channel_id, status=common.HTTP_STATUS_BAD_REQUEST) + + def _process_flow_control(self, block): + try: + self._logical_channels_condition.acquire() + if not block.channel_id in self._logical_channels: + return + channel_data = self._logical_channels[block.channel_id] + channel_data.request.ws_stream.replenish_send_quota( + block.send_quota) + finally: + self._logical_channels_condition.release() + + def _process_drop_channel(self, block): + self._logger.debug( + 'DropChannel received for %d: code=%r, reason=%r' % + (block.channel_id, block.drop_code, block.drop_message)) + try: + self._logical_channels_condition.acquire() + if not block.channel_id in self._logical_channels: + return + channel_data = self._logical_channels[block.channel_id] + channel_data.drop_code = _DROP_CODE_ACKNOWLEDGED + # Close the logical channel + channel_data.request.connection.set_read_state( + _LogicalConnection.STATE_TERMINATED) + finally: + self._logical_channels_condition.release() + + def _process_control_blocks(self, parser): + for control_block in parser.read_control_blocks(): + opcode = control_block.opcode + self._logger.debug('control block received, opcode: %d' % opcode) + if opcode == _MUX_OPCODE_ADD_CHANNEL_REQUEST: + self._process_add_channel_request(control_block) + elif opcode == _MUX_OPCODE_ADD_CHANNEL_RESPONSE: + raise PhysicalConnectionError( + _DROP_CODE_INVALID_MUX_CONTROL_BLOCK, + 'Received AddChannelResponse') + elif opcode == _MUX_OPCODE_FLOW_CONTROL: + self._process_flow_control(control_block) + elif opcode == _MUX_OPCODE_DROP_CHANNEL: + self._process_drop_channel(control_block) + elif opcode == _MUX_OPCODE_NEW_CHANNEL_SLOT: + raise PhysicalConnectionError( + _DROP_CODE_INVALID_MUX_CONTROL_BLOCK, + 'Received NewChannelSlot') + else: + raise MuxUnexpectedException( + 'Unexpected opcode %r' % opcode) + + def _process_logical_frame(self, channel_id, parser): + self._logger.debug('Received a frame. channel id=%d' % channel_id) + try: + self._logical_channels_condition.acquire() + if not channel_id in self._logical_channels: + # We must ignore the message for an inactive channel. + return + channel_data = self._logical_channels[channel_id] + fin, rsv1, rsv2, rsv3, opcode, payload = parser.read_inner_frame() + if not channel_data.request.ws_stream.consume_receive_quota( + len(payload)): + # The client violates quota. Close logical channel. + raise LogicalChannelError( + channel_id, _DROP_CODE_SEND_QUOTA_VIOLATION) + header = create_header(opcode, len(payload), fin, rsv1, rsv2, rsv3, + mask=False) + frame_data = header + payload + channel_data.request.connection.append_frame_data(frame_data) + finally: + self._logical_channels_condition.release() + + def dispatch_message(self, message): + """Dispatches message. The reader thread calls this method. + + Args: + message: a message that contains encapsulated frame. + Raises: + PhysicalConnectionError: if the message contains physical + connection level errors. + LogicalChannelError: if the message contains logical channel + level errors. + """ + + parser = _MuxFramePayloadParser(message) + try: + channel_id = parser.read_channel_id() + except ValueError, e: + raise PhysicalConnectionError(_DROP_CODE_CHANNEL_ID_TRUNCATED) + if channel_id == _CONTROL_CHANNEL_ID: + self._process_control_blocks(parser) + else: + self._process_logical_frame(channel_id, parser) + + def notify_worker_done(self, channel_id): + """Called when a worker has finished. + + Args: + channel_id: channel id corresponded with the worker. + """ + + self._logger.debug('Worker for channel id %d terminated' % channel_id) + try: + self._logical_channels_condition.acquire() + if not channel_id in self._logical_channels: + raise MuxUnexpectedException( + 'Channel id %d not found' % channel_id) + channel_data = self._logical_channels.pop(channel_id) + finally: + self._worker_done_notify_received = True + self._logical_channels_condition.notify() + self._logical_channels_condition.release() + + if not channel_data.request.server_terminated: + self._send_drop_channel( + channel_id, code=channel_data.drop_code, + message=channel_data.drop_message) + + def notify_reader_done(self): + """This method is called by the reader thread when the reader has + finished. + """ + + # Terminate all logical connections + self._logger.debug('termiating all logical connections...') + self._logical_channels_condition.acquire() + for channel_data in self._logical_channels.values(): + try: + channel_data.request.connection.set_read_state( + _LogicalConnection.STATE_TERMINATED) + except Exception: + pass + self._logical_channels_condition.release() + + def fail_physical_connection(self, code, message): + """Fail the physical connection. + + Args: + code: drop reason code. + message: drop message. + """ + + self._logger.debug('Failing the physical connection...') + self._send_drop_channel(_CONTROL_CHANNEL_ID, code, message) + self.physical_stream.close_connection( + common.STATUS_INTERNAL_ENDPOINT_ERROR) + + def fail_logical_channel(self, channel_id, code, message): + """Fail a logical channel. + + Args: + channel_id: channel id. + code: drop reason code. + message: drop message. + """ + + self._logger.debug('Failing logical channel %d...' % channel_id) + try: + self._logical_channels_condition.acquire() + if channel_id in self._logical_channels: + channel_data = self._logical_channels[channel_id] + # Close the logical channel. notify_worker_done() will be + # called later and it will send DropChannel. + channel_data.drop_code = code + channel_data.drop_message = message + channel_data.request.connection.set_read_state( + _LogicalConnection.STATE_TERMINATED) + else: + self._send_drop_channel(channel_id, code, message) + finally: + self._logical_channels_condition.release() + + +def use_mux(request): + return hasattr(request, 'mux') and request.mux + + +def start(request, dispatcher): + mux_handler = _MuxHandler(request, dispatcher) + mux_handler.start() + + mux_handler.add_channel_slots(_INITIAL_NUMBER_OF_CHANNEL_SLOTS, + _INITIAL_QUOTA_FOR_CLIENT) + + mux_handler.wait_until_done() + + +# vi:sts=4 sw=4 et diff --git a/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/standalone.py b/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/standalone.py index 850aa5cd4..07a33d9c9 100755 --- a/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/standalone.py +++ b/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/standalone.py @@ -32,27 +32,44 @@ """Standalone WebSocket server. +Use this file to launch pywebsocket without Apache HTTP Server. + + BASIC USAGE -Use this server to run mod_pywebsocket without Apache HTTP Server. +Go to the src directory and run -Usage: - python standalone.py [-p <ws_port>] [-w <websock_handlers>] - [-s <scan_dir>] - [-d <document_root>] - [-m <websock_handlers_map_file>] - ... for other options, see _main below ... + $ python mod_pywebsocket/standalone.py [-p <ws_port>] + [-w <websock_handlers>] + [-d <document_root>] <ws_port> is the port number to use for ws:// connection. <document_root> is the path to the root directory of HTML files. <websock_handlers> is the path to the root directory of WebSocket handlers. -See __init__.py for details of <websock_handlers> and how to write WebSocket -handlers. If this path is relative, <document_root> is used as the base. +If not specified, <document_root> will be used. See __init__.py (or +run $ pydoc mod_pywebsocket) for how to write WebSocket handlers. + +For more detail and other options, run + + $ python mod_pywebsocket/standalone.py --help + +or see _build_option_parser method below. + +For trouble shooting, adding "--log_level debug" might help you. + -<scan_dir> is a path under the root directory. If specified, only the -handlers under scan_dir are scanned. This is useful in saving scan time. +TRY DEMO + +Go to the src directory and run + + $ python standalone.py -d example + +to launch pywebsocket with the sample handler and html on port 80. Open +http://localhost/console.html, click the connect button, type something into +the text box next to the send button and click the send button. If everything +is working, you'll see the message you typed echoed by the server. SUPPORTING TLS @@ -63,10 +80,10 @@ To support TLS, run standalone.py with -t, -k, and -c options. SUPPORTING CLIENT AUTHENTICATION To support client authentication with TLS, run standalone.py with -t, -k, -c, -and --ca-certificate options. +and --tls-client-auth, and --tls-client-ca options. E.g., $./standalone.py -d ../example -p 10443 -t -c ../test/cert/cert.pem -k -../test/cert/key.pem --ca-certificate=../test/cert/cacert.pem +../test/cert/key.pem --tls-client-auth --tls-client-ca=../test/cert/cacert.pem CONFIGURATION FILE @@ -110,6 +127,7 @@ import CGIHTTPServer import SimpleHTTPServer import SocketServer import ConfigParser +import base64 import httplib import logging import logging.handlers @@ -224,6 +242,12 @@ class _StandaloneRequest(object): return self._request_handler.command method = property(get_method) + def get_protocol(self): + """Getter to mimic request.protocol.""" + + return self._request_handler.request_version + protocol = property(get_protocol) + def is_https(self): """Mimic request.is_https().""" @@ -264,6 +288,32 @@ class _StandaloneSSLConnection(object): return socket._fileobject(self._connection, mode, bufsize) +def _alias_handlers(dispatcher, websock_handlers_map_file): + """Set aliases specified in websock_handler_map_file in dispatcher. + + Args: + dispatcher: dispatch.Dispatcher instance + websock_handler_map_file: alias map file + """ + + fp = open(websock_handlers_map_file) + try: + for line in fp: + if line[0] == '#' or line.isspace(): + continue + m = re.match('(\S+)\s+(\S+)', line) + if not m: + logging.warning('Wrong format in map file:' + line) + continue + try: + dispatcher.add_resource_path_alias( + m.group(1), m.group(2)) + except dispatch.DispatchException, e: + logging.error(str(e)) + finally: + fp.close() + + class WebSocketServer(SocketServer.ThreadingMixIn, BaseHTTPServer.HTTPServer): """HTTPServer specialized for WebSocket.""" @@ -278,6 +328,20 @@ class WebSocketServer(SocketServer.ThreadingMixIn, BaseHTTPServer.HTTPServer): if necessary. """ + # Share a Dispatcher among request handlers to save time for + # instantiation. Dispatcher can be shared because it is thread-safe. + options.dispatcher = dispatch.Dispatcher( + options.websock_handlers, + options.scan_dir, + options.allow_handlers_outside_root_dir) + if options.websock_handlers_map_file: + _alias_handlers(options.dispatcher, + options.websock_handlers_map_file) + warnings = options.dispatcher.source_warnings() + if warnings: + for warning in warnings: + logging.warning('mod_pywebsocket: %s' % warning) + self._logger = util.get_class_logger(self) self.request_queue_size = options.request_queue_size @@ -325,7 +389,7 @@ class WebSocketServer(SocketServer.ThreadingMixIn, BaseHTTPServer.HTTPServer): continue if self.websocket_server_options.use_tls: if _HAS_SSL: - if self.websocket_server_options.ca_certificate: + if self.websocket_server_options.tls_client_auth: client_cert_ = ssl.CERT_REQUIRED else: client_cert_ = ssl.CERT_NONE @@ -333,7 +397,7 @@ class WebSocketServer(SocketServer.ThreadingMixIn, BaseHTTPServer.HTTPServer): keyfile=self.websocket_server_options.private_key, certfile=self.websocket_server_options.certificate, ssl_version=ssl.PROTOCOL_SSLv23, - ca_certs=self.websocket_server_options.ca_certificate, + ca_certs=self.websocket_server_options.tls_client_ca, cert_reqs=client_cert_) if _HAS_OPEN_SSL: ctx = OpenSSL.SSL.Context(OpenSSL.SSL.SSLv23_METHOD) @@ -362,6 +426,15 @@ class WebSocketServer(SocketServer.ThreadingMixIn, BaseHTTPServer.HTTPServer): self._logger.info('Skip by failure: %r', e) socket_.close() failed_sockets.append(socketinfo) + if self.server_address[1] == 0: + # The operating system assigns the actual port number for port + # number 0. This case, the second and later sockets should use + # the same port number. Also self.server_port is rewritten + # because it is exported, and will be used by external code. + self.server_address = ( + self.server_name, socket_.getsockname()[1]) + self.server_port = self.server_address[1] + self._logger.info('Port %r is assigned', self.server_port) for socketinfo in failed_sockets: self._sockets.remove(socketinfo) @@ -386,6 +459,10 @@ class WebSocketServer(SocketServer.ThreadingMixIn, BaseHTTPServer.HTTPServer): for socketinfo in failed_sockets: self._sockets.remove(socketinfo) + if len(self._sockets) == 0: + self._logger.critical( + 'No sockets activated. Use info log level to see the reason.') + def server_close(self): """Override SocketServer.TCPServer.server_close to enable multiple sockets close. @@ -513,6 +590,17 @@ class WebSocketRequestHandler(CGIHTTPServer.CGIHTTPRequestHandler): # attributes). if not CGIHTTPServer.CGIHTTPRequestHandler.parse_request(self): return False + + if self._options.use_basic_auth: + auth = self.headers.getheader('Authorization') + if auth != self._options.basic_auth_credential: + self.send_response(401) + self.send_header('WWW-Authenticate', + 'Basic realm="Pywebsocket"') + self.end_headers() + self._logger.info('Request basic authentication') + return True + host, port, resource = http_header_util.parse_uri(self.path) if resource is None: self._logger.info('Invalid URI: %r', self.path) @@ -648,32 +736,6 @@ def _configure_logging(options): deflate_log_level_name) -def _alias_handlers(dispatcher, websock_handlers_map_file): - """Set aliases specified in websock_handler_map_file in dispatcher. - - Args: - dispatcher: dispatch.Dispatcher instance - websock_handler_map_file: alias map file - """ - - fp = open(websock_handlers_map_file) - try: - for line in fp: - if line[0] == '#' or line.isspace(): - continue - m = re.match('(\S+)\s+(\S+)', line) - if not m: - logging.warning('Wrong format in map file:' + line) - continue - try: - dispatcher.add_resource_path_alias( - m.group(1), m.group(2)) - except dispatch.DispatchException, e: - logging.error(str(e)) - finally: - fp.close() - - def _build_option_parser(): parser = optparse.OptionParser() @@ -700,7 +762,9 @@ def _build_option_parser(): parser.add_option('-w', '--websock-handlers', '--websock_handlers', dest='websock_handlers', default='.', - help='WebSocket handlers root directory.') + help=('The root directory of WebSocket handler files. ' + 'If the path is relative, --document-root is used ' + 'as the base.')) parser.add_option('-m', '--websock-handlers-map-file', '--websock_handlers_map_file', dest='websock_handlers_map_file', @@ -710,15 +774,20 @@ def _build_option_parser(): 'existing_resource_path, separated by spaces.')) parser.add_option('-s', '--scan-dir', '--scan_dir', dest='scan_dir', default=None, - help=('WebSocket handlers scan directory. ' - 'Must be a directory under websock_handlers.')) + help=('Must be a directory under --websock-handlers. ' + 'Only handlers under this directory are scanned ' + 'and registered to the server. ' + 'Useful for saving scan time when the handler ' + 'root directory contains lots of files that are ' + 'not handler file or are handler files but you ' + 'don\'t want them to be registered. ')) parser.add_option('--allow-handlers-outside-root-dir', '--allow_handlers_outside_root_dir', dest='allow_handlers_outside_root_dir', action='store_true', default=False, help=('Scans WebSocket handlers even if their canonical ' - 'path is not under websock_handlers.')) + 'path is not under --websock-handlers.')) parser.add_option('-d', '--document-root', '--document_root', dest='document_root', default='.', help='Document root directory.') @@ -735,9 +804,20 @@ def _build_option_parser(): default='', help='TLS private key file.') parser.add_option('-c', '--certificate', dest='certificate', default='', help='TLS certificate file.') - parser.add_option('--ca-certificate', dest='ca_certificate', default='', - help=('TLS CA certificate file for client ' - 'authentication.')) + parser.add_option('--tls-client-auth', dest='tls_client_auth', + action='store_true', default=False, + help='Requires TLS client auth on every connection.') + parser.add_option('--tls-client-ca', dest='tls_client_ca', default='', + help=('Specifies a pem file which contains a set of ' + 'concatenated CA certificates which are used to ' + 'validate certificates passed from clients')) + parser.add_option('--basic-auth', dest='use_basic_auth', + action='store_true', default=False, + help='Requires Basic authentication.') + parser.add_option('--basic-auth-credential', + dest='basic_auth_credential', default='test:test', + help='Specifies the credential of basic authentication ' + 'by username:password pair (e.g. test:test).') parser.add_option('-l', '--log-file', '--log_file', dest='log_file', default='', help='Log file.') # Custom log level: @@ -771,9 +851,9 @@ def _build_option_parser(): help='Log backup count') parser.add_option('--allow-draft75', dest='allow_draft75', action='store_true', default=False, - help='Allow draft 75 handshake') + help='Obsolete option. Ignored.') parser.add_option('--strict', dest='strict', action='store_true', - default=False, help='Strictly check handshake request') + default=False, help='Obsolete option. Ignored.') parser.add_option('-q', '--queue', dest='request_queue_size', type='int', default=_DEFAULT_REQUEST_QUEUE_SIZE, help='request queue size') @@ -841,6 +921,12 @@ def _parse_args_and_config(args): def _main(args=None): + """You can call this function from your own program, but please note that + this function has some side-effects that might affect your program. For + example, util.wrap_popen3_for_win use in this method replaces implementation + of os.popen3. + """ + options, args = _parse_args_and_config(args=args) os.chdir(options.document_root) @@ -877,7 +963,7 @@ def _main(args=None): 'To use TLS, specify private_key and certificate.') sys.exit(1) - if options.ca_certificate: + if options.tls_client_auth: if not options.use_tls: logging.critical('TLS must be enabled for client authentication.') sys.exit(1) @@ -887,26 +973,16 @@ def _main(args=None): if not options.scan_dir: options.scan_dir = options.websock_handlers + if options.use_basic_auth: + options.basic_auth_credential = 'Basic ' + base64.b64encode( + options.basic_auth_credential) + try: if options.thread_monitor_interval_in_sec > 0: # Run a thread monitor to show the status of server threads for # debugging. ThreadMonitor(options.thread_monitor_interval_in_sec).start() - # Share a Dispatcher among request handlers to save time for - # instantiation. Dispatcher can be shared because it is thread-safe. - options.dispatcher = dispatch.Dispatcher( - options.websock_handlers, - options.scan_dir, - options.allow_handlers_outside_root_dir) - if options.websock_handlers_map_file: - _alias_handlers(options.dispatcher, - options.websock_handlers_map_file) - warnings = options.dispatcher.source_warnings() - if warnings: - for warning in warnings: - logging.warning('mod_pywebsocket: %s' % warning) - server = WebSocketServer(options) server.serve_forever() except Exception, e: diff --git a/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/stream.py b/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/stream.py index d051eee20..edc533279 100644 --- a/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/stream.py +++ b/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/stream.py @@ -51,6 +51,7 @@ from mod_pywebsocket._stream_hybi import create_ping_frame from mod_pywebsocket._stream_hybi import create_pong_frame from mod_pywebsocket._stream_hybi import create_binary_frame from mod_pywebsocket._stream_hybi import create_text_frame +from mod_pywebsocket._stream_hybi import create_closing_handshake_body # vi:sts=4 sw=4 et diff --git a/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/util.py b/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/util.py index 6146e052f..7bb0b5d9e 100644 --- a/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/util.py +++ b/Tools/Scripts/webkitpy/thirdparty/mod_pywebsocket/util.py @@ -232,6 +232,12 @@ class _Deflater(object): self._compress = zlib.compressobj( zlib.Z_DEFAULT_COMPRESSION, zlib.DEFLATED, -window_bits) + def compress(self, bytes): + compressed_bytes = self._compress.compress(bytes) + self._logger.debug('Compress input %r', bytes) + self._logger.debug('Compress result %r', compressed_bytes) + return compressed_bytes + def compress_and_flush(self, bytes): compressed_bytes = self._compress.compress(bytes) compressed_bytes += self._compress.flush(zlib.Z_SYNC_FLUSH) @@ -239,6 +245,12 @@ class _Deflater(object): self._logger.debug('Compress result %r', compressed_bytes) return compressed_bytes + def compress_and_finish(self, bytes): + compressed_bytes = self._compress.compress(bytes) + compressed_bytes += self._compress.flush(zlib.Z_FINISH) + self._logger.debug('Compress input %r', bytes) + self._logger.debug('Compress result %r', compressed_bytes) + return compressed_bytes class _Inflater(object): @@ -318,14 +330,21 @@ class _RFC1979Deflater(object): self._window_bits = window_bits self._no_context_takeover = no_context_takeover - def filter(self, bytes): - if self._deflater is None or self._no_context_takeover: + def filter(self, bytes, flush=True, bfinal=False): + if self._deflater is None or (self._no_context_takeover and flush): self._deflater = _Deflater(self._window_bits) - # Strip last 4 octets which is LEN and NLEN field of a non-compressed - # block added for Z_SYNC_FLUSH. - return self._deflater.compress_and_flush(bytes)[:-4] - + if bfinal: + result = self._deflater.compress_and_finish(bytes) + # Add a padding block with BFINAL = 0 and BTYPE = 0. + result = result + chr(0) + self._deflater = None + return result + if flush: + # Strip last 4 octets which is LEN and NLEN field of a + # non-compressed block added for Z_SYNC_FLUSH. + return self._deflater.compress_and_flush(bytes)[:-4] + return self._deflater.compress(bytes) class _RFC1979Inflater(object): """A decompressor class for byte sequence compressed and flushed following diff --git a/Tools/Scripts/webkitpy/tool/commands/download_unittest.py b/Tools/Scripts/webkitpy/tool/commands/download_unittest.py index 79b729aad..b71f3daaf 100644 --- a/Tools/Scripts/webkitpy/tool/commands/download_unittest.py +++ b/Tools/Scripts/webkitpy/tool/commands/download_unittest.py @@ -285,7 +285,7 @@ Reason component: MOCK component cc: MOCK cc blocked: 50004 -MOCK reopen_bug 50004 with comment 'Re-opened since this is blocked by 60001' +MOCK reopen_bug 50004 with comment 'Re-opened since this is blocked by bug 60001' MOCK add_patch_to_bug: bug_id=60001, description=ROLLOUT of r3001, mark_for_review=False, mark_for_commit_queue=True, mark_for_landing=False -- Begin comment -- Any committer can land this patch automatically by marking it commit-queue+. The commit-queue will build and test the patch before landing to ensure that the rollout will be successful. This process takes approximately 15 minutes. diff --git a/Tools/Scripts/webkitpy/tool/commands/rebaseline_unittest.py b/Tools/Scripts/webkitpy/tool/commands/rebaseline_unittest.py index 22fb1704b..4b63f2f67 100644 --- a/Tools/Scripts/webkitpy/tool/commands/rebaseline_unittest.py +++ b/Tools/Scripts/webkitpy/tool/commands/rebaseline_unittest.py @@ -54,17 +54,15 @@ class TestRebaseline(unittest.TestCase): self.assertEqual(command._baseline_directory("GTK Linux 32-bit Release"), "/mock-checkout/LayoutTests/platform/gtk") self.assertEqual(command._baseline_directory("EFL Linux 64-bit Debug"), "/mock-checkout/LayoutTests/platform/efl") self.assertEqual(command._baseline_directory("Qt Linux Release"), "/mock-checkout/LayoutTests/platform/qt") - self.assertEqual(command._baseline_directory("Webkit Mac10.7"), "/mock-checkout/LayoutTests/platform/chromium-mac") - self.assertEqual(command._baseline_directory("Webkit Mac10.6"), "/mock-checkout/LayoutTests/platform/chromium-mac-snowleopard") + self.assertEqual(command._baseline_directory("WebKit Mac10.7"), "/mock-checkout/LayoutTests/platform/chromium-mac") + self.assertEqual(command._baseline_directory("WebKit Mac10.6"), "/mock-checkout/LayoutTests/platform/chromium-mac-snowleopard") def test_rebaseline_updates_expectations_file_noop(self): command = RebaselineTest() tool = MockTool() command.bind_to_tool(tool) - lion_port = tool.port_factory.get_from_builder_name("Webkit Mac10.7") - # FIXME: work around the chromium skia expectations file to avoid getting a bunch of confusing warnings. - tool.filesystem.write_text_file(lion_port.path_from_chromium_base('skia', 'skia_test_expectations.txt'), '') + lion_port = tool.port_factory.get_from_builder_name("WebKit Mac10.7") for path in lion_port.expectations_files(): tool.filesystem.write_text_file(path, '') tool.filesystem.write_text_file(lion_port.path_to_test_expectations_file(), """Bug(B) [ Mac Linux XP Debug ] fast/dom/Window/window-postmessage-clone-really-deep-array.html [ Pass ] @@ -74,11 +72,11 @@ Bug(A) [ Debug ] : fast/css/large-list-of-rules-crash.html [ Failure ] tool.filesystem.write_text_file(os.path.join(lion_port.layout_tests_dir(), "fast/css/large-list-of-rules-crash.html"), "Dummy test contents") tool.filesystem.write_text_file(os.path.join(lion_port.layout_tests_dir(), "userscripts/another-test.html"), "Dummy test contents") - expected_logs = """Retrieving http://example.com/f/builders/Webkit Mac10.7/results/layout-test-results/userscripts/another-test-actual.png. -Retrieving http://example.com/f/builders/Webkit Mac10.7/results/layout-test-results/userscripts/another-test-actual.wav. -Retrieving http://example.com/f/builders/Webkit Mac10.7/results/layout-test-results/userscripts/another-test-actual.txt. + expected_logs = """Retrieving http://example.com/f/builders/WebKit Mac10.7/results/layout-test-results/userscripts/another-test-actual.png. +Retrieving http://example.com/f/builders/WebKit Mac10.7/results/layout-test-results/userscripts/another-test-actual.wav. +Retrieving http://example.com/f/builders/WebKit Mac10.7/results/layout-test-results/userscripts/another-test-actual.txt. """ - OutputCapture().assert_outputs(self, command._rebaseline_test_and_update_expectations, ["Webkit Mac10.7", "userscripts/another-test.html", None], expected_logs=expected_logs) + OutputCapture().assert_outputs(self, command._rebaseline_test_and_update_expectations, ["WebKit Mac10.7", "userscripts/another-test.html", None], expected_logs=expected_logs) new_expectations = tool.filesystem.read_text_file(lion_port.path_to_test_expectations_file()) self.assertEqual(new_expectations, """Bug(B) [ Mac Linux XP Debug ] fast/dom/Window/window-postmessage-clone-really-deep-array.html [ Pass ] @@ -90,54 +88,52 @@ Bug(A) [ Debug ] : fast/css/large-list-of-rules-crash.html [ Failure ] tool = MockTool() command.bind_to_tool(tool) - lion_port = tool.port_factory.get_from_builder_name("Webkit Mac10.7") - tool.filesystem.write_text_file(lion_port.path_from_chromium_base('skia', 'skia_test_expectations.txt'), '') + lion_port = tool.port_factory.get_from_builder_name("WebKit Mac10.7") tool.filesystem.write_text_file(lion_port.path_to_test_expectations_file(), "Bug(x) [ Mac ] userscripts/another-test.html [ ImageOnlyFailure ]\nbug(z) [ Linux ] userscripts/another-test.html [ ImageOnlyFailure ]\n") tool.filesystem.write_text_file(os.path.join(lion_port.layout_tests_dir(), "userscripts/another-test.html"), "Dummy test contents") - expected_logs = """Retrieving http://example.com/f/builders/Webkit Mac10.7/results/layout-test-results/userscripts/another-test-actual.png. -Retrieving http://example.com/f/builders/Webkit Mac10.7/results/layout-test-results/userscripts/another-test-actual.wav. -Retrieving http://example.com/f/builders/Webkit Mac10.7/results/layout-test-results/userscripts/another-test-actual.txt. + expected_logs = """Retrieving http://example.com/f/builders/WebKit Mac10.7/results/layout-test-results/userscripts/another-test-actual.png. +Retrieving http://example.com/f/builders/WebKit Mac10.7/results/layout-test-results/userscripts/another-test-actual.wav. +Retrieving http://example.com/f/builders/WebKit Mac10.7/results/layout-test-results/userscripts/another-test-actual.txt. """ - OutputCapture().assert_outputs(self, command._rebaseline_test_and_update_expectations, ["Webkit Mac10.7", "userscripts/another-test.html", None], expected_logs=expected_logs) + OutputCapture().assert_outputs(self, command._rebaseline_test_and_update_expectations, ["WebKit Mac10.7", "userscripts/another-test.html", None], expected_logs=expected_logs) new_expectations = tool.filesystem.read_text_file(lion_port.path_to_test_expectations_file()) - self.assertEqual(new_expectations, "Bug(x) [ SnowLeopard ] userscripts/another-test.html [ ImageOnlyFailure ]\nbug(z) [ Linux ] userscripts/another-test.html [ ImageOnlyFailure ]\n") + self.assertEqual(new_expectations, "Bug(x) [ MountainLion SnowLeopard ] userscripts/another-test.html [ ImageOnlyFailure ]\nbug(z) [ Linux ] userscripts/another-test.html [ ImageOnlyFailure ]\n") def test_rebaseline_does_not_include_overrides(self): command = RebaselineTest() tool = MockTool() command.bind_to_tool(tool) - lion_port = tool.port_factory.get_from_builder_name("Webkit Mac10.7") - tool.filesystem.write_text_file(lion_port.path_from_chromium_base('skia', 'skia_test_expectations.txt'), '') + lion_port = tool.port_factory.get_from_builder_name("WebKit Mac10.7") tool.filesystem.write_text_file(lion_port.path_to_test_expectations_file(), "Bug(x) [ Mac ] userscripts/another-test.html [ ImageOnlyFailure ]\nBug(z) [ Linux ] userscripts/another-test.html [ ImageOnlyFailure ]\n") tool.filesystem.write_text_file(lion_port.path_from_chromium_base('skia', 'skia_test_expectations.txt'), "Bug(y) [ Mac ] other-test.html [ Failure ]\n") tool.filesystem.write_text_file(os.path.join(lion_port.layout_tests_dir(), "userscripts/another-test.html"), "Dummy test contents") - expected_logs = """Retrieving http://example.com/f/builders/Webkit Mac10.7/results/layout-test-results/userscripts/another-test-actual.png. -Retrieving http://example.com/f/builders/Webkit Mac10.7/results/layout-test-results/userscripts/another-test-actual.wav. -Retrieving http://example.com/f/builders/Webkit Mac10.7/results/layout-test-results/userscripts/another-test-actual.txt. + expected_logs = """Retrieving http://example.com/f/builders/WebKit Mac10.7/results/layout-test-results/userscripts/another-test-actual.png. +Retrieving http://example.com/f/builders/WebKit Mac10.7/results/layout-test-results/userscripts/another-test-actual.wav. +Retrieving http://example.com/f/builders/WebKit Mac10.7/results/layout-test-results/userscripts/another-test-actual.txt. """ - OutputCapture().assert_outputs(self, command._rebaseline_test_and_update_expectations, ["Webkit Mac10.7", "userscripts/another-test.html", None], expected_logs=expected_logs) + OutputCapture().assert_outputs(self, command._rebaseline_test_and_update_expectations, ["WebKit Mac10.7", "userscripts/another-test.html", None], expected_logs=expected_logs) new_expectations = tool.filesystem.read_text_file(lion_port.path_to_test_expectations_file()) - self.assertEqual(new_expectations, "Bug(x) [ SnowLeopard ] userscripts/another-test.html [ ImageOnlyFailure ]\nBug(z) [ Linux ] userscripts/another-test.html [ ImageOnlyFailure ]\n") + self.assertEqual(new_expectations, "Bug(x) [ MountainLion SnowLeopard ] userscripts/another-test.html [ ImageOnlyFailure ]\nBug(z) [ Linux ] userscripts/another-test.html [ ImageOnlyFailure ]\n") def test_rebaseline_test(self): command = RebaselineTest() command.bind_to_tool(MockTool()) - expected_logs = "Retrieving http://example.com/f/builders/Webkit Linux/results/layout-test-results/userscripts/another-test-actual.txt.\n" - OutputCapture().assert_outputs(self, command._rebaseline_test, ["Webkit Linux", "userscripts/another-test.html", None, "txt"], expected_logs=expected_logs) + expected_logs = "Retrieving http://example.com/f/builders/WebKit Linux/results/layout-test-results/userscripts/another-test-actual.txt.\n" + OutputCapture().assert_outputs(self, command._rebaseline_test, ["WebKit Linux", "userscripts/another-test.html", None, "txt"], expected_logs=expected_logs) def test_rebaseline_test_and_print_scm_changes(self): command = RebaselineTest() command.bind_to_tool(MockTool()) - expected_logs = "Retrieving http://example.com/f/builders/Webkit Linux/results/layout-test-results/userscripts/another-test-actual.txt.\n" + expected_logs = "Retrieving http://example.com/f/builders/WebKit Linux/results/layout-test-results/userscripts/another-test-actual.txt.\n" command._print_scm_changes = True command._scm_changes = {'add': [], 'delete': []} command._tool._scm.exists = lambda x: False - OutputCapture().assert_outputs(self, command._rebaseline_test, ["Webkit Linux", "userscripts/another-test.html", None, "txt"], expected_logs=expected_logs) + OutputCapture().assert_outputs(self, command._rebaseline_test, ["WebKit Linux", "userscripts/another-test.html", None, "txt"], expected_logs=expected_logs) self.assertEquals(command._scm_changes, {'add': ['/mock-checkout/LayoutTests/platform/chromium-linux/userscripts/another-test-expected.txt'], 'delete': []}) def test_rebaseline_and_copy_test(self): @@ -145,13 +141,13 @@ Retrieving http://example.com/f/builders/Webkit Mac10.7/results/layout-test-resu tool = MockTool() command.bind_to_tool(tool) - lion_port = tool.port_factory.get_from_builder_name("Webkit Mac10.7") + lion_port = tool.port_factory.get_from_builder_name("WebKit Mac10.7") tool.filesystem.write_text_file(os.path.join(lion_port.layout_tests_dir(), "userscripts/another-test-expected.txt"), "Dummy expected result") expected_logs = """Copying baseline from /mock-checkout/LayoutTests/userscripts/another-test-expected.txt to /mock-checkout/LayoutTests/platform/chromium-mac-snowleopard/userscripts/another-test-expected.txt. -Retrieving http://example.com/f/builders/Webkit Mac10.7/results/layout-test-results/userscripts/another-test-actual.txt. +Retrieving http://example.com/f/builders/WebKit Mac10.7/results/layout-test-results/userscripts/another-test-actual.txt. """ - OutputCapture().assert_outputs(self, command._rebaseline_test, ["Webkit Mac10.7", "userscripts/another-test.html", ["chromium-mac-snowleopard"], "txt"], expected_logs=expected_logs) + OutputCapture().assert_outputs(self, command._rebaseline_test, ["WebKit Mac10.7", "userscripts/another-test.html", ["chromium-mac-snowleopard"], "txt"], expected_logs=expected_logs) def test_rebaseline_and_copy_test_no_existing_result(self): command = RebaselineTest() @@ -159,38 +155,38 @@ Retrieving http://example.com/f/builders/Webkit Mac10.7/results/layout-test-resu command.bind_to_tool(tool) expected_logs = """No existing baseline for userscripts/another-test.html. -Retrieving http://example.com/f/builders/Webkit Mac10.7/results/layout-test-results/userscripts/another-test-actual.txt. +Retrieving http://example.com/f/builders/WebKit Mac10.7/results/layout-test-results/userscripts/another-test-actual.txt. """ - OutputCapture().assert_outputs(self, command._rebaseline_test, ["Webkit Mac10.7", "userscripts/another-test.html", ["chromium-mac-snowleopard"], "txt"], expected_logs=expected_logs) + OutputCapture().assert_outputs(self, command._rebaseline_test, ["WebKit Mac10.7", "userscripts/another-test.html", ["chromium-mac-snowleopard"], "txt"], expected_logs=expected_logs) def test_rebaseline_and_copy_test_with_lion_result(self): command = RebaselineTest() tool = MockTool() command.bind_to_tool(tool) - lion_port = tool.port_factory.get_from_builder_name("Webkit Mac10.7") + lion_port = tool.port_factory.get_from_builder_name("WebKit Mac10.7") tool.filesystem.write_text_file(os.path.join(lion_port.baseline_path(), "userscripts/another-test-expected.txt"), "Dummy expected result") expected_logs = """Copying baseline from /mock-checkout/LayoutTests/platform/chromium-mac/userscripts/another-test-expected.txt to /mock-checkout/LayoutTests/platform/chromium-mac-snowleopard/userscripts/another-test-expected.txt. -Retrieving http://example.com/f/builders/Webkit Mac10.7/results/layout-test-results/userscripts/another-test-actual.txt. +Retrieving http://example.com/f/builders/WebKit Mac10.7/results/layout-test-results/userscripts/another-test-actual.txt. """ - OutputCapture().assert_outputs(self, command._rebaseline_test, ["Webkit Mac10.7", "userscripts/another-test.html", ["chromium-mac-snowleopard"], "txt"], expected_logs=expected_logs) + OutputCapture().assert_outputs(self, command._rebaseline_test, ["WebKit Mac10.7", "userscripts/another-test.html", ["chromium-mac-snowleopard"], "txt"], expected_logs=expected_logs) def test_rebaseline_and_copy_no_overwrite_test(self): command = RebaselineTest() tool = MockTool() command.bind_to_tool(tool) - lion_port = tool.port_factory.get_from_builder_name("Webkit Mac10.7") + lion_port = tool.port_factory.get_from_builder_name("WebKit Mac10.7") tool.filesystem.write_text_file(os.path.join(lion_port.baseline_path(), "userscripts/another-test-expected.txt"), "Dummy expected result") - snowleopard_port = tool.port_factory.get_from_builder_name("Webkit Mac10.6") + snowleopard_port = tool.port_factory.get_from_builder_name("WebKit Mac10.6") tool.filesystem.write_text_file(os.path.join(snowleopard_port.baseline_path(), "userscripts/another-test-expected.txt"), "Dummy expected result") expected_logs = """Existing baseline at /mock-checkout/LayoutTests/platform/chromium-mac-snowleopard/userscripts/another-test-expected.txt, not copying over it. -Retrieving http://example.com/f/builders/Webkit Mac10.7/results/layout-test-results/userscripts/another-test-actual.txt. +Retrieving http://example.com/f/builders/WebKit Mac10.7/results/layout-test-results/userscripts/another-test-actual.txt. """ - OutputCapture().assert_outputs(self, command._rebaseline_test, ["Webkit Mac10.7", "userscripts/another-test.html", ["chromium-mac-snowleopard"], "txt"], expected_logs=expected_logs) + OutputCapture().assert_outputs(self, command._rebaseline_test, ["WebKit Mac10.7", "userscripts/another-test.html", ["chromium-mac-snowleopard"], "txt"], expected_logs=expected_logs) def test_rebaseline_all(self): old_exact_matches = builders._exact_matches @@ -228,9 +224,7 @@ MOCK run_command: ['echo', 'optimize-baselines', '--suffixes', 'txt', 'user-scri tool = MockTool() command.bind_to_tool(tool) - # FIXME: work around the chromium skia expectations file to avoid getting a bunch of confusing warnings. - lion_port = tool.port_factory.get_from_builder_name("Webkit Mac10.7") - tool.filesystem.write_text_file(lion_port.path_from_chromium_base('skia', 'skia_test_expectations.txt'), '') + lion_port = tool.port_factory.get_from_builder_name("WebKit Mac10.7") for port_name in tool.port_factory.all_port_names(): port = tool.port_factory.get(port_name) for path in port.expectations_files(): @@ -245,46 +239,9 @@ MOCK run_command: ['echo', 'optimize-baselines', '--suffixes', 'txt', 'user-scri tool.executive.run_in_parallel = run_in_parallel - expected_logs = """Retrieving results for chromium-linux-x86 from Webkit Linux 32. - userscripts/another-test.html (txt) - userscripts/images.svg (png) -Retrieving results for chromium-linux-x86_64 from Webkit Linux. - userscripts/another-test.html (txt) - userscripts/images.svg (png) -Retrieving results for chromium-mac-lion from Webkit Mac10.7. - userscripts/another-test.html (txt) - userscripts/images.svg (png) -Retrieving results for chromium-mac-snowleopard from Webkit Mac10.6. - userscripts/another-test.html (txt) - userscripts/images.svg (png) -Retrieving results for chromium-win-win7 from Webkit Win7. - userscripts/another-test.html (txt) - userscripts/images.svg (png) -Retrieving results for chromium-win-xp from Webkit Win. - userscripts/another-test.html (txt) - userscripts/images.svg (png) -Retrieving results for efl from EFL Linux 64-bit Release. - userscripts/another-test.html (txt) - userscripts/images.svg (png) -Retrieving results for gtk from GTK Linux 64-bit Release. - userscripts/another-test.html (txt) - userscripts/images.svg (png) -Retrieving results for mac-lion from Apple Lion Release WK1 (Tests). - userscripts/another-test.html (txt) - userscripts/images.svg (png) -Retrieving results for mac-mountainlion from Apple MountainLion Release WK1 (Tests). - userscripts/another-test.html (txt) - userscripts/images.svg (png) -Retrieving results for qt-linux from Qt Linux Release. - userscripts/another-test.html (txt) - userscripts/images.svg (png) -Retrieving results for win-7sp0 from Apple Win 7 Release (Tests). - userscripts/another-test.html (txt) - userscripts/images.svg (png) -""" + expected_logs = "Retrieving results for chromium-linux-x86 from WebKit Linux 32.\n userscripts/another-test.html (txt)\n userscripts/images.svg (png)\nRetrieving results for chromium-linux-x86_64 from WebKit Linux.\n userscripts/another-test.html (txt)\n userscripts/images.svg (png)\nRetrieving results for chromium-mac-lion from WebKit Mac10.7.\n userscripts/another-test.html (txt)\n userscripts/images.svg (png)\nRetrieving results for chromium-mac-snowleopard from WebKit Mac10.6.\n userscripts/another-test.html (txt)\n userscripts/images.svg (png)\nRetrieving results for chromium-win-win7 from WebKit Win7.\n userscripts/another-test.html (txt)\n userscripts/images.svg (png)\nRetrieving results for chromium-win-xp from WebKit XP.\n userscripts/another-test.html (txt)\n userscripts/images.svg (png)\nRetrieving results for efl from EFL Linux 64-bit Release.\n userscripts/another-test.html (txt)\n userscripts/images.svg (png)\nRetrieving results for gtk from GTK Linux 64-bit Release.\n userscripts/another-test.html (txt)\n userscripts/images.svg (png)\nRetrieving results for mac-lion from Apple Lion Release WK1 (Tests).\n userscripts/another-test.html (txt)\n userscripts/images.svg (png)\nRetrieving results for mac-mountainlion from Apple MountainLion Release WK1 (Tests).\n userscripts/another-test.html (txt)\n userscripts/images.svg (png)\nRetrieving results for qt-linux from Qt Linux Release.\n userscripts/another-test.html (txt)\n userscripts/images.svg (png)\nRetrieving results for win-7sp0 from Apple Win 7 Release (Tests).\n userscripts/another-test.html (txt)\n userscripts/images.svg (png)\n" - expected_stdout = """[(['echo', 'rebaseline-test-internal', '--suffixes', 'txt', '--builder', 'Webkit Linux 32', '--test', 'userscripts/another-test.html'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'txt', '--builder', 'Webkit Linux', '--test', 'userscripts/another-test.html'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'txt', '--builder', 'Webkit Mac10.6', '--test', 'userscripts/another-test.html'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'txt', '--builder', 'Webkit Mac10.7', '--test', 'userscripts/another-test.html'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'txt', '--builder', 'Webkit Win7', '--test', 'userscripts/another-test.html'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'txt', '--builder', 'Apple Win 7 Release (Tests)', '--test', 'userscripts/another-test.html'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'txt', '--builder', 'EFL Linux 64-bit Release', '--test', 'userscripts/another-test.html'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'txt', '--builder', 'Webkit Win', '--test', 'userscripts/another-test.html'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'txt', '--builder', 'GTK Linux 64-bit Release', '--test', 'userscripts/another-test.html'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'txt', '--builder', 'Qt Linux Release', '--test', 'userscripts/another-test.html'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'txt', '--builder', 'Apple Lion Release WK1 (Tests)', '--test', 'userscripts/another-test.html'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'txt', '--builder', 'Apple MountainLion Release WK1 (Tests)', '--test', 'userscripts/another-test.html'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'png', '--builder', 'Webkit Linux 32', '--test', 'userscripts/images.svg'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'png', '--builder', 'Webkit Linux', '--test', 'userscripts/images.svg'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'png', '--builder', 'Webkit Mac10.6', '--test', 'userscripts/images.svg'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'png', '--builder', 'Webkit Mac10.7', '--test', 'userscripts/images.svg'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'png', '--builder', 'Webkit Win7', '--test', 'userscripts/images.svg'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'png', '--builder', 'Apple Win 7 Release (Tests)', '--test', 'userscripts/images.svg'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'png', '--builder', 'EFL Linux 64-bit Release', '--test', 'userscripts/images.svg'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'png', '--builder', 'Webkit Win', '--test', 'userscripts/images.svg'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'png', '--builder', 'GTK Linux 64-bit Release', '--test', 'userscripts/images.svg'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'png', '--builder', 'Qt Linux Release', '--test', 'userscripts/images.svg'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'png', '--builder', 'Apple Lion Release WK1 (Tests)', '--test', 'userscripts/images.svg'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'png', '--builder', 'Apple MountainLion Release WK1 (Tests)', '--test', 'userscripts/images.svg'], '/mock-checkout')] -""" + expected_stdout = "[(['echo', 'rebaseline-test-internal', '--suffixes', 'txt', '--builder', 'WebKit Linux 32', '--test', 'userscripts/another-test.html'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'txt', '--builder', 'WebKit Linux', '--test', 'userscripts/another-test.html'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'txt', '--builder', 'WebKit Mac10.6', '--test', 'userscripts/another-test.html'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'txt', '--builder', 'WebKit Mac10.7', '--test', 'userscripts/another-test.html'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'txt', '--builder', 'WebKit Win7', '--test', 'userscripts/another-test.html'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'txt', '--builder', 'Apple Win 7 Release (Tests)', '--test', 'userscripts/another-test.html'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'txt', '--builder', 'EFL Linux 64-bit Release', '--test', 'userscripts/another-test.html'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'txt', '--builder', 'GTK Linux 64-bit Release', '--test', 'userscripts/another-test.html'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'txt', '--builder', 'Qt Linux Release', '--test', 'userscripts/another-test.html'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'txt', '--builder', 'Apple Lion Release WK1 (Tests)', '--test', 'userscripts/another-test.html'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'txt', '--builder', 'WebKit XP', '--test', 'userscripts/another-test.html'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'txt', '--builder', 'Apple MountainLion Release WK1 (Tests)', '--test', 'userscripts/another-test.html'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'png', '--builder', 'WebKit Linux 32', '--test', 'userscripts/images.svg'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'png', '--builder', 'WebKit Linux', '--test', 'userscripts/images.svg'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'png', '--builder', 'WebKit Mac10.6', '--test', 'userscripts/images.svg'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'png', '--builder', 'WebKit Mac10.7', '--test', 'userscripts/images.svg'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'png', '--builder', 'WebKit Win7', '--test', 'userscripts/images.svg'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'png', '--builder', 'Apple Win 7 Release (Tests)', '--test', 'userscripts/images.svg'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'png', '--builder', 'EFL Linux 64-bit Release', '--test', 'userscripts/images.svg'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'png', '--builder', 'GTK Linux 64-bit Release', '--test', 'userscripts/images.svg'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'png', '--builder', 'Qt Linux Release', '--test', 'userscripts/images.svg'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'png', '--builder', 'Apple Lion Release WK1 (Tests)', '--test', 'userscripts/images.svg'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'png', '--builder', 'WebKit XP', '--test', 'userscripts/images.svg'], '/mock-checkout'), (['echo', 'rebaseline-test-internal', '--suffixes', 'png', '--builder', 'Apple MountainLion Release WK1 (Tests)', '--test', 'userscripts/images.svg'], '/mock-checkout')]\n" expected_stderr = """MOCK run_command: ['qmake', '-v'], cwd=None MOCK run_command: ['qmake', '-v'], cwd=None @@ -321,7 +278,6 @@ MOCK run_command: ['qmake', '-v'], cwd=None command.bind_to_tool(tool) port = tool.port_factory.get('chromium-mac-lion') - tool.filesystem.write_text_file(port.path_from_chromium_base('skia', 'skia_test_expectations.txt'), '') for port_name in tool.port_factory.all_port_names(): port = tool.port_factory.get(port_name) diff --git a/Tools/Scripts/webkitpy/tool/main.py b/Tools/Scripts/webkitpy/tool/main.py index d1fde74b8..68348a05a 100755 --- a/Tools/Scripts/webkitpy/tool/main.py +++ b/Tools/Scripts/webkitpy/tool/main.py @@ -91,7 +91,7 @@ class WebKitPatch(MultiCommandTool, Host): # FIXME: This may be unnecessary since we pass global options to all commands during execute() as well. def handle_global_options(self, options): - self._initialize_scm(options.patch_directories) + self.initialize_scm(options.patch_directories) if options.status_host: self.status_server.set_host(options.status_host) if options.bot_id: diff --git a/Tools/Scripts/webkitpy/tool/servers/gardeningserver_unittest.py b/Tools/Scripts/webkitpy/tool/servers/gardeningserver_unittest.py index 22e853491..6c64bdd7e 100644 --- a/Tools/Scripts/webkitpy/tool/servers/gardeningserver_unittest.py +++ b/Tools/Scripts/webkitpy/tool/servers/gardeningserver_unittest.py @@ -90,8 +90,8 @@ class BuildCoverageExtrapolatorTest(unittest.TestCase): port = host.port_factory.get('chromium-win-win7', None) converter = TestConfigurationConverter(port.all_test_configurations(), port.configuration_specifier_macros()) extrapolator = BuildCoverageExtrapolator(converter) - self.assertEquals(extrapolator.extrapolate_test_configurations("Webkit Win"), set([TestConfiguration(version='xp', architecture='x86', build_type='release')])) - self.assertEquals(extrapolator.extrapolate_test_configurations("Webkit Win7"), set([ + self.assertEquals(extrapolator.extrapolate_test_configurations("WebKit XP"), set([TestConfiguration(version='xp', architecture='x86', build_type='release')])) + self.assertEquals(extrapolator.extrapolate_test_configurations("WebKit Win7"), set([ TestConfiguration(version='win7', architecture='x86', build_type='debug'), TestConfiguration(version='win7', architecture='x86', build_type='release')])) self.assertRaises(KeyError, extrapolator.extrapolate_test_configurations, "Potato") diff --git a/Tools/Scripts/webkitpy/tool/servers/rebaselineserver.py b/Tools/Scripts/webkitpy/tool/servers/rebaselineserver.py index 0e7458727..9e9c379d6 100644 --- a/Tools/Scripts/webkitpy/tool/servers/rebaselineserver.py +++ b/Tools/Scripts/webkitpy/tool/servers/rebaselineserver.py @@ -178,7 +178,7 @@ def get_test_baselines(test_file, test_config): # FIXME: This should get the Host from the test_config to be mockable! host = Host() - host._initialize_scm() + host.initialize_scm() host.filesystem = test_config.filesystem all_platforms_port = AllPlatformsPort(host) diff --git a/Tools/Scripts/webkitpy/tool/steps/createbug.py b/Tools/Scripts/webkitpy/tool/steps/createbug.py index 4605892e7..7e4a83504 100644 --- a/Tools/Scripts/webkitpy/tool/steps/createbug.py +++ b/Tools/Scripts/webkitpy/tool/steps/createbug.py @@ -53,4 +53,4 @@ class CreateBug(AbstractStep): if blocks: status = self._tool.bugs.fetch_bug(blocks).status() if status == 'RESOLVED': - self._tool.bugs.reopen_bug(blocks, "Re-opened since this is blocked by %s" % state["bug_id"]) + self._tool.bugs.reopen_bug(blocks, "Re-opened since this is blocked by bug %s" % state["bug_id"]) |