diff options
Diffstat (limited to 'src/3rdparty/v8/tools/testrunner')
32 files changed, 3942 insertions, 0 deletions
diff --git a/src/3rdparty/v8/tools/testrunner/README b/src/3rdparty/v8/tools/testrunner/README new file mode 100644 index 0000000..8f0c01f --- /dev/null +++ b/src/3rdparty/v8/tools/testrunner/README @@ -0,0 +1,174 @@ +Test suite runner for V8, including support for distributed running. +==================================================================== + + +Local usage instructions: +========================= + +Run the main script with --help to get detailed usage instructions: + +$ tools/run-tests.py --help + +The interface is mostly the same as it was for the old test runner. +You'll likely want something like this: + +$ tools/run-tests.py --nonetwork --arch ia32 --mode release + +--nonetwork is the default on Mac and Windows. If you don't specify --arch +and/or --mode, all available values will be used and run in turn (e.g., +omitting --mode from the above example will run ia32 in both Release and Debug +modes). + + +Networked usage instructions: +============================= + +Networked running is only supported on Linux currently. Make sure that all +machines participating in the cluster are binary-compatible (e.g. mixing +Ubuntu Lucid and Precise doesn't work). + +Setup: +------ + +1.) Copy tools/test-server.py to a new empty directory anywhere on your hard + drive (preferably not inside your V8 checkout just to keep things clean). + Please do create a copy, not just a symlink. + +2.) Navigate to the new directory and let the server setup itself: + +$ ./test-server.py setup + + This will install PIP and UltraJSON, create a V8 working directory, and + generate a keypair. + +3.) Swap public keys with someone who's already part of the networked cluster. + +$ cp trusted/`cat data/mypubkey`.pem /where/peers/can/see/it/myname.pem +$ ./test-server.py approve /wherever/they/put/it/yourname.pem + + +Usage: +------ + +1.) Start your server: + +$ ./test-server.py start + +2.) (Optionally) inspect the server's status: + +$ ./test-server.py status + +3.) From your regular V8 working directory, run tests: + +$ tool/run-tests.py --arch ia32 --mode debug + +4.) (Optionally) enjoy the speeeeeeeeeeeeeeeed + + +Architecture overview: +====================== + +Code organization: +------------------ + +This section is written from the point of view of the tools/ directory. + +./run-tests.py: + Main script. Parses command-line options and drives the test execution + procedure from a high level. Imports the actual implementation of all + steps from the testrunner/ directory. + +./test-server.py: + Interface to interact with the server. Contains code to setup the server's + working environment and can start and stop server daemon processes. + Imports some stuff from the testrunner/server/ directory. + +./testrunner/local/*: + Implementation needed to run tests locally. Used by run-tests.py. Inspired by + (and partly copied verbatim from) the original test.py script. + +./testrunner/local/old_statusfile.py: + Provides functionality to read an old-style <testsuite>.status file and + convert it to new-style syntax. This can be removed once the new-style + syntax becomes authoritative (and old-style syntax is no longer supported). + ./status-file-converter.py provides a stand-alone interface to this. + +./testrunner/objects/*: + A bunch of data container classes, used by the scripts in the various other + directories; serializable for transmission over the network. + +./testrunner/network/*: + Equivalents and extensions of some of the functionality in ./testrunner/local/ + as required when dispatching tests to peers on the network. + +./testrunner/network/network_execution.py: + Drop-in replacement for ./testrunner/local/execution that distributes + test jobs to network peers instead of running them locally. + +./testrunner/network/endpoint.py: + Receiving end of a network distributed job, uses the implementation + in ./testrunner/local/execution.py for actually running the tests. + +./testrunner/server/*: + Implementation of the daemon that accepts and runs test execution jobs from + peers on the network. Should ideally have no dependencies on any of the other + directories, but that turned out to be impractical, so there are a few + exceptions. + +./testrunner/server/compression.py: + Defines a wrapper around Python TCP sockets that provides JSON based + serialization, gzip based compression, and ensures message completeness. + + +Networking architecture: +------------------------ + +The distribution stuff is designed to be a layer between deciding which tests +to run on the one side, and actually running them on the other. The frontend +that the user interacts with is the same for local and networked execution, +and the actual test execution and result gathering code is the same too. + +The server daemon starts four separate servers, each listening on another port: +- "Local": Communication with a run-tests.py script running on the same host. + The test driving script e.g. needs to ask for available peers. It then talks + to those peers directly (one of them will be the locally running server). +- "Work": Listens for test job requests from run-tests.py scripts on the network + (including localhost). Accepts an arbitrary number of connections at the + same time, but only works on them in a serialized fashion. +- "Status": Used for communication with other servers on the network, e.g. for + exchanging trusted public keys to create the transitive trust closure. +- "Discovery": Used to detect presence of other peers on the network. + In contrast to the other three, this uses UDP (as opposed to TCP). + + +Give us a diagram! We love diagrams! +------------------------------------ + . + Machine A . Machine B + . ++------------------------------+ . +| run-tests.py | . +| with flag: | . +|--nonetwork --network | . +| | / | | . +| | / | | . +| v / v | . +|BACKEND / distribution | . ++--------- / --------| \ ------+ . + / | \_____________________ + / | . \ + / | . \ ++----- v ----------- v --------+ . +---- v -----------------------+ +| LocalHandler | WorkHandler | . | WorkHandler | LocalHandler | +| | | | . | | | | +| | v | . | v | | +| | BACKEND | . | BACKEND | | +|------------- +---------------| . |---------------+--------------| +| Discovery | StatusHandler <----------> StatusHandler | Discovery | ++---- ^ -----------------------+ . +-------------------- ^ -------+ + | . | + +---------------------------------------------------------+ + +Note that the three occurrences of "BACKEND" are the same code +(testrunner/local/execution.py and its imports), but running from three +distinct directories (and on two different machines). diff --git a/src/3rdparty/v8/tools/testrunner/__init__.py b/src/3rdparty/v8/tools/testrunner/__init__.py new file mode 100644 index 0000000..202a262 --- /dev/null +++ b/src/3rdparty/v8/tools/testrunner/__init__.py @@ -0,0 +1,26 @@ +# Copyright 2012 the V8 project authors. All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/src/3rdparty/v8/tools/testrunner/local/__init__.py b/src/3rdparty/v8/tools/testrunner/local/__init__.py new file mode 100644 index 0000000..202a262 --- /dev/null +++ b/src/3rdparty/v8/tools/testrunner/local/__init__.py @@ -0,0 +1,26 @@ +# Copyright 2012 the V8 project authors. All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/src/3rdparty/v8/tools/testrunner/local/commands.py b/src/3rdparty/v8/tools/testrunner/local/commands.py new file mode 100644 index 0000000..01f170d --- /dev/null +++ b/src/3rdparty/v8/tools/testrunner/local/commands.py @@ -0,0 +1,153 @@ +# Copyright 2012 the V8 project authors. All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +import os +import signal +import subprocess +import sys +import tempfile +import time + +from ..local import utils +from ..objects import output + + +def KillProcessWithID(pid): + if utils.IsWindows(): + os.popen('taskkill /T /F /PID %d' % pid) + else: + os.kill(pid, signal.SIGTERM) + + +MAX_SLEEP_TIME = 0.1 +INITIAL_SLEEP_TIME = 0.0001 +SLEEP_TIME_FACTOR = 1.25 + +SEM_INVALID_VALUE = -1 +SEM_NOGPFAULTERRORBOX = 0x0002 # Microsoft Platform SDK WinBase.h + + +def Win32SetErrorMode(mode): + prev_error_mode = SEM_INVALID_VALUE + try: + import ctypes + prev_error_mode = \ + ctypes.windll.kernel32.SetErrorMode(mode) #@UndefinedVariable + except ImportError: + pass + return prev_error_mode + + +def RunProcess(verbose, timeout, args, **rest): + if verbose: print "#", " ".join(args) + popen_args = args + prev_error_mode = SEM_INVALID_VALUE + if utils.IsWindows(): + popen_args = subprocess.list2cmdline(args) + # Try to change the error mode to avoid dialogs on fatal errors. Don't + # touch any existing error mode flags by merging the existing error mode. + # See http://blogs.msdn.com/oldnewthing/archive/2004/07/27/198410.aspx. + error_mode = SEM_NOGPFAULTERRORBOX + prev_error_mode = Win32SetErrorMode(error_mode) + Win32SetErrorMode(error_mode | prev_error_mode) + process = subprocess.Popen( + shell=utils.IsWindows(), + args=popen_args, + **rest + ) + if (utils.IsWindows() and prev_error_mode != SEM_INVALID_VALUE): + Win32SetErrorMode(prev_error_mode) + # Compute the end time - if the process crosses this limit we + # consider it timed out. + if timeout is None: end_time = None + else: end_time = time.time() + timeout + timed_out = False + # Repeatedly check the exit code from the process in a + # loop and keep track of whether or not it times out. + exit_code = None + sleep_time = INITIAL_SLEEP_TIME + try: + while exit_code is None: + if (not end_time is None) and (time.time() >= end_time): + # Kill the process and wait for it to exit. + KillProcessWithID(process.pid) + exit_code = process.wait() + timed_out = True + else: + exit_code = process.poll() + time.sleep(sleep_time) + sleep_time = sleep_time * SLEEP_TIME_FACTOR + if sleep_time > MAX_SLEEP_TIME: + sleep_time = MAX_SLEEP_TIME + return (exit_code, timed_out) + except KeyboardInterrupt: + raise + + +def PrintError(string): + sys.stderr.write(string) + sys.stderr.write("\n") + + +def CheckedUnlink(name): + # On Windows, when run with -jN in parallel processes, + # OS often fails to unlink the temp file. Not sure why. + # Need to retry. + # Idea from https://bugs.webkit.org/attachment.cgi?id=75982&action=prettypatch + retry_count = 0 + while retry_count < 30: + try: + os.unlink(name) + return + except OSError, e: + retry_count += 1 + time.sleep(retry_count * 0.1) + PrintError("os.unlink() " + str(e)) + + +def Execute(args, verbose=False, timeout=None): + args = [ c for c in args if c != "" ] + (fd_out, outname) = tempfile.mkstemp() + (fd_err, errname) = tempfile.mkstemp() + try: + (exit_code, timed_out) = RunProcess( + verbose, + timeout, + args=args, + stdout=fd_out, + stderr=fd_err + ) + except: + raise + os.close(fd_out) + os.close(fd_err) + out = file(outname).read() + errors = file(errname).read() + CheckedUnlink(outname) + CheckedUnlink(errname) + return output.Output(exit_code, timed_out, out, errors) diff --git a/src/3rdparty/v8/tools/testrunner/local/execution.py b/src/3rdparty/v8/tools/testrunner/local/execution.py new file mode 100644 index 0000000..6004367 --- /dev/null +++ b/src/3rdparty/v8/tools/testrunner/local/execution.py @@ -0,0 +1,182 @@ +# Copyright 2012 the V8 project authors. All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +import multiprocessing +import os +import threading +import time + +from . import commands +from . import utils + + +BREAK_NOW = -1 +EXCEPTION = -2 + + +class Job(object): + def __init__(self, command, dep_command, test_id, timeout, verbose): + self.command = command + self.dep_command = dep_command + self.id = test_id + self.timeout = timeout + self.verbose = verbose + + +def RunTest(job): + try: + start_time = time.time() + if job.dep_command is not None: + dep_output = commands.Execute(job.dep_command, job.verbose, job.timeout) + # TODO(jkummerow): We approximate the test suite specific function + # IsFailureOutput() by just checking the exit code here. Currently + # only cctests define dependencies, for which this simplification is + # correct. + if dep_output.exit_code != 0: + return (job.id, dep_output, time.time() - start_time) + output = commands.Execute(job.command, job.verbose, job.timeout) + return (job.id, output, time.time() - start_time) + except KeyboardInterrupt: + return (-1, BREAK_NOW, 0) + except Exception, e: + print(">>> EXCEPTION: %s" % e) + return (-1, EXCEPTION, 0) + + +class Runner(object): + + def __init__(self, suites, progress_indicator, context): + self.tests = [ t for s in suites for t in s.tests ] + self._CommonInit(len(self.tests), progress_indicator, context) + + def _CommonInit(self, num_tests, progress_indicator, context): + self.indicator = progress_indicator + progress_indicator.runner = self + self.context = context + self.succeeded = 0 + self.total = num_tests + self.remaining = num_tests + self.failed = [] + self.crashed = 0 + self.terminate = False + self.lock = threading.Lock() + + def Run(self, jobs): + self.indicator.Starting() + self._RunInternal(jobs) + self.indicator.Done() + if self.failed: + return 1 + return 0 + + def _RunInternal(self, jobs): + pool = multiprocessing.Pool(processes=jobs) + test_map = {} + queue = [] + queued_exception = None + for test in self.tests: + assert test.id >= 0 + test_map[test.id] = test + try: + command = self.GetCommand(test) + except Exception, e: + # If this failed, save the exception and re-raise it later (after + # all other tests have had a chance to run). + queued_exception = e + continue + timeout = self.context.timeout + if ("--stress-opt" in test.flags or + "--stress-opt" in self.context.mode_flags or + "--stress-opt" in self.context.extra_flags): + timeout *= 4 + if test.dependency is not None: + dep_command = [ c.replace(test.path, test.dependency) for c in command ] + else: + dep_command = None + job = Job(command, dep_command, test.id, timeout, self.context.verbose) + queue.append(job) + try: + kChunkSize = 1 + it = pool.imap_unordered(RunTest, queue, kChunkSize) + for result in it: + test_id = result[0] + if test_id < 0: + if result[1] == BREAK_NOW: + self.terminate = True + else: + continue + if self.terminate: + pool.terminate() + pool.join() + raise BreakNowException("User pressed Ctrl+C or IO went wrong") + test = test_map[test_id] + self.indicator.AboutToRun(test) + test.output = result[1] + test.duration = result[2] + if test.suite.HasUnexpectedOutput(test): + self.failed.append(test) + if test.output.HasCrashed(): + self.crashed += 1 + else: + self.succeeded += 1 + self.remaining -= 1 + self.indicator.HasRun(test) + except KeyboardInterrupt: + pool.terminate() + pool.join() + raise + except Exception, e: + print("Exception: %s" % e) + pool.terminate() + pool.join() + raise + if queued_exception: + raise queued_exception + return + + + def GetCommand(self, test): + d8testflag = [] + shell = test.suite.shell() + if shell == "d8": + d8testflag = ["--test"] + if utils.IsWindows(): + shell += ".exe" + cmd = ([self.context.command_prefix] + + [os.path.abspath(os.path.join(self.context.shell_dir, shell))] + + d8testflag + + test.suite.GetFlagsForTestCase(test, self.context) + + [self.context.extra_flags]) + return cmd + + +class BreakNowException(Exception): + def __init__(self, value): + self.value = value + def __str__(self): + return repr(self.value) diff --git a/src/3rdparty/v8/tools/testrunner/local/old_statusfile.py b/src/3rdparty/v8/tools/testrunner/local/old_statusfile.py new file mode 100644 index 0000000..a16941b --- /dev/null +++ b/src/3rdparty/v8/tools/testrunner/local/old_statusfile.py @@ -0,0 +1,460 @@ +# Copyright 2012 the V8 project authors. All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +import cStringIO +import re + +# These outcomes can occur in a TestCase's outcomes list: +SKIP = 'SKIP' +FAIL = 'FAIL' +PASS = 'PASS' +OKAY = 'OKAY' +TIMEOUT = 'TIMEOUT' +CRASH = 'CRASH' +SLOW = 'SLOW' +# These are just for the status files and are mapped below in DEFS: +FAIL_OK = 'FAIL_OK' +PASS_OR_FAIL = 'PASS_OR_FAIL' + +KEYWORDS = {SKIP: SKIP, + FAIL: FAIL, + PASS: PASS, + OKAY: OKAY, + TIMEOUT: TIMEOUT, + CRASH: CRASH, + SLOW: SLOW, + FAIL_OK: FAIL_OK, + PASS_OR_FAIL: PASS_OR_FAIL} + +class Expression(object): + pass + + +class Constant(Expression): + + def __init__(self, value): + self.value = value + + def Evaluate(self, env, defs): + return self.value + + +class Variable(Expression): + + def __init__(self, name): + self.name = name + + def GetOutcomes(self, env, defs): + if self.name in env: return set([env[self.name]]) + else: return set([]) + + def Evaluate(self, env, defs): + return env[self.name] + + def __str__(self): + return self.name + + def string(self, logical): + return self.__str__() + + +class Outcome(Expression): + + def __init__(self, name): + self.name = name + + def GetOutcomes(self, env, defs): + if self.name in defs: + return defs[self.name].GetOutcomes(env, defs) + else: + return set([self.name]) + + def __str__(self): + if self.name in KEYWORDS: + return "%s" % KEYWORDS[self.name] + return "'%s'" % self.name + + def string(self, logical): + if logical: + return "%s" % self.name + return self.__str__() + + +class Operation(Expression): + + def __init__(self, left, op, right): + self.left = left + self.op = op + self.right = right + + def Evaluate(self, env, defs): + if self.op == '||' or self.op == ',': + return self.left.Evaluate(env, defs) or self.right.Evaluate(env, defs) + elif self.op == 'if': + return False + elif self.op == '==': + return not self.left.GetOutcomes(env, defs).isdisjoint(self.right.GetOutcomes(env, defs)) + elif self.op == '!=': + return self.left.GetOutcomes(env, defs).isdisjoint(self.right.GetOutcomes(env, defs)) + else: + assert self.op == '&&' + return self.left.Evaluate(env, defs) and self.right.Evaluate(env, defs) + + def GetOutcomes(self, env, defs): + if self.op == '||' or self.op == ',': + return self.left.GetOutcomes(env, defs) | self.right.GetOutcomes(env, defs) + elif self.op == 'if': + if self.right.Evaluate(env, defs): return self.left.GetOutcomes(env, defs) + else: return set([]) + else: + assert self.op == '&&' + return self.left.GetOutcomes(env, defs) & self.right.GetOutcomes(env, defs) + + def __str__(self): + return self.string(False) + + def string(self, logical=False): + if self.op == 'if': + return "['%s', %s]" % (self.right.string(True), self.left.string(logical)) + elif self.op == "||" or self.op == ",": + if logical: + return "%s or %s" % (self.left.string(True), self.right.string(True)) + else: + return "%s, %s" % (self.left, self.right) + elif self.op == "&&": + return "%s and %s" % (self.left.string(True), self.right.string(True)) + return "%s %s %s" % (self.left.string(logical), self.op, + self.right.string(logical)) + + +def IsAlpha(string): + for char in string: + if not (char.isalpha() or char.isdigit() or char == '_'): + return False + return True + + +class Tokenizer(object): + """A simple string tokenizer that chops expressions into variables, + parens and operators""" + + def __init__(self, expr): + self.index = 0 + self.expr = expr + self.length = len(expr) + self.tokens = None + + def Current(self, length=1): + if not self.HasMore(length): return "" + return self.expr[self.index:self.index + length] + + def HasMore(self, length=1): + return self.index < self.length + (length - 1) + + def Advance(self, count=1): + self.index = self.index + count + + def AddToken(self, token): + self.tokens.append(token) + + def SkipSpaces(self): + while self.HasMore() and self.Current().isspace(): + self.Advance() + + def Tokenize(self): + self.tokens = [ ] + while self.HasMore(): + self.SkipSpaces() + if not self.HasMore(): + return None + if self.Current() == '(': + self.AddToken('(') + self.Advance() + elif self.Current() == ')': + self.AddToken(')') + self.Advance() + elif self.Current() == '$': + self.AddToken('$') + self.Advance() + elif self.Current() == ',': + self.AddToken(',') + self.Advance() + elif IsAlpha(self.Current()): + buf = "" + while self.HasMore() and IsAlpha(self.Current()): + buf += self.Current() + self.Advance() + self.AddToken(buf) + elif self.Current(2) == '&&': + self.AddToken('&&') + self.Advance(2) + elif self.Current(2) == '||': + self.AddToken('||') + self.Advance(2) + elif self.Current(2) == '==': + self.AddToken('==') + self.Advance(2) + elif self.Current(2) == '!=': + self.AddToken('!=') + self.Advance(2) + else: + return None + return self.tokens + + +class Scanner(object): + """A simple scanner that can serve out tokens from a given list""" + + def __init__(self, tokens): + self.tokens = tokens + self.length = len(tokens) + self.index = 0 + + def HasMore(self): + return self.index < self.length + + def Current(self): + return self.tokens[self.index] + + def Advance(self): + self.index = self.index + 1 + + +def ParseAtomicExpression(scan): + if scan.Current() == "true": + scan.Advance() + return Constant(True) + elif scan.Current() == "false": + scan.Advance() + return Constant(False) + elif IsAlpha(scan.Current()): + name = scan.Current() + scan.Advance() + return Outcome(name) + elif scan.Current() == '$': + scan.Advance() + if not IsAlpha(scan.Current()): + return None + name = scan.Current() + scan.Advance() + return Variable(name.lower()) + elif scan.Current() == '(': + scan.Advance() + result = ParseLogicalExpression(scan) + if (not result) or (scan.Current() != ')'): + return None + scan.Advance() + return result + else: + return None + + +BINARIES = ['==', '!='] +def ParseOperatorExpression(scan): + left = ParseAtomicExpression(scan) + if not left: return None + while scan.HasMore() and (scan.Current() in BINARIES): + op = scan.Current() + scan.Advance() + right = ParseOperatorExpression(scan) + if not right: + return None + left = Operation(left, op, right) + return left + + +def ParseConditionalExpression(scan): + left = ParseOperatorExpression(scan) + if not left: return None + while scan.HasMore() and (scan.Current() == 'if'): + scan.Advance() + right = ParseOperatorExpression(scan) + if not right: + return None + left = Operation(left, 'if', right) + return left + + +LOGICALS = ["&&", "||", ","] +def ParseLogicalExpression(scan): + left = ParseConditionalExpression(scan) + if not left: return None + while scan.HasMore() and (scan.Current() in LOGICALS): + op = scan.Current() + scan.Advance() + right = ParseConditionalExpression(scan) + if not right: + return None + left = Operation(left, op, right) + return left + + +def ParseCondition(expr): + """Parses a logical expression into an Expression object""" + tokens = Tokenizer(expr).Tokenize() + if not tokens: + print "Malformed expression: '%s'" % expr + return None + scan = Scanner(tokens) + ast = ParseLogicalExpression(scan) + if not ast: + print "Malformed expression: '%s'" % expr + return None + if scan.HasMore(): + print "Malformed expression: '%s'" % expr + return None + return ast + + +class Section(object): + """A section of the configuration file. Sections are enabled or + disabled prior to running the tests, based on their conditions""" + + def __init__(self, condition): + self.condition = condition + self.rules = [ ] + + def AddRule(self, rule): + self.rules.append(rule) + + +class Rule(object): + """A single rule that specifies the expected outcome for a single + test.""" + + def __init__(self, raw_path, path, value): + self.raw_path = raw_path + self.path = path + self.value = value + + def GetOutcomes(self, env, defs): + return self.value.GetOutcomes(env, defs) + + def Contains(self, path): + if len(self.path) > len(path): + return False + for i in xrange(len(self.path)): + if not self.path[i].match(path[i]): + return False + return True + + +HEADER_PATTERN = re.compile(r'\[([^]]+)\]') +RULE_PATTERN = re.compile(r'\s*([^: ]*)\s*:(.*)') +DEF_PATTERN = re.compile(r'^def\s*(\w+)\s*=(.*)$') +PREFIX_PATTERN = re.compile(r'^\s*prefix\s+([\w\_\.\-\/]+)$') + + +class ConvertNotation(object): + def __init__(self, path): + self.path = path + self.indent = "" + self.comment = [] + self.init = False + self.section = False + self.out = cStringIO.StringIO() + + def OpenGlobal(self): + if self.init: return + self.WriteComment() + print >> self.out, "[" + self.init = True + + def CloseGlobal(self): + if not self.init: return + print >> self.out, "]" + self.init = False + + def OpenSection(self, condition="ALWAYS"): + if self.section: return + self.OpenGlobal() + if type(condition) != str: + condition = "'%s'" % condition.string(True) + print >> self.out, "%s[%s, {" % (self.indent, condition) + self.indent += " " * 2 + self.section = condition + + def CloseSection(self): + if not self.section: return + self.indent = self.indent[:-2] + print >> self.out, "%s}], # %s" % (self.indent, self.section) + self.section = False + + def WriteComment(self): + if not self.comment: return + for c in self.comment: + if len(c.strip()) == 0: + print >> self.out, "" + else: + print >> self.out, "%s%s" % (self.indent, c), + self.comment = [] + + def GetOutput(self): + with open(self.path) as f: + for line in f: + if line[0] == '#': + self.comment += [line] + continue + if len(line.strip()) == 0: + self.comment += [line] + continue + header_match = HEADER_PATTERN.match(line) + if header_match: + condition = ParseCondition(header_match.group(1).strip()) + self.CloseSection() + self.WriteComment() + self.OpenSection(condition) + continue + rule_match = RULE_PATTERN.match(line) + if rule_match: + self.OpenSection() + self.WriteComment() + path = rule_match.group(1).strip() + value_str = rule_match.group(2).strip() + comment = "" + if '#' in value_str: + pos = value_str.find('#') + comment = " %s" % value_str[pos:].strip() + value_str = value_str[:pos].strip() + value = ParseCondition(value_str) + print >> self.out, ("%s'%s': [%s],%s" % + (self.indent, path, value, comment)) + continue + def_match = DEF_PATTERN.match(line) + if def_match: + # Custom definitions are deprecated. + continue + prefix_match = PREFIX_PATTERN.match(line) + if prefix_match: + continue + print "Malformed line: '%s'." % line + self.CloseSection() + self.CloseGlobal() + result = self.out.getvalue() + self.out.close() + return result diff --git a/src/3rdparty/v8/tools/testrunner/local/progress.py b/src/3rdparty/v8/tools/testrunner/local/progress.py new file mode 100644 index 0000000..9075a95 --- /dev/null +++ b/src/3rdparty/v8/tools/testrunner/local/progress.py @@ -0,0 +1,238 @@ +# Copyright 2012 the V8 project authors. All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +import sys +import time + +def EscapeCommand(command): + parts = [] + for part in command: + if ' ' in part: + # Escape spaces. We may need to escape more characters for this + # to work properly. + parts.append('"%s"' % part) + else: + parts.append(part) + return " ".join(parts) + + +class ProgressIndicator(object): + + def __init__(self): + self.runner = None + + def Starting(self): + pass + + def Done(self): + pass + + def AboutToRun(self, test): + pass + + def HasRun(self, test): + pass + + def PrintFailureHeader(self, test): + if test.suite.IsNegativeTest(test): + negative_marker = '[negative] ' + else: + negative_marker = '' + print "=== %(label)s %(negative)s===" % { + 'label': test.GetLabel(), + 'negative': negative_marker + } + + +class SimpleProgressIndicator(ProgressIndicator): + """Abstract base class for {Verbose,Dots}ProgressIndicator""" + + def Starting(self): + print 'Running %i tests' % self.runner.total + + def Done(self): + print + for failed in self.runner.failed: + self.PrintFailureHeader(failed) + if failed.output.stderr: + print "--- stderr ---" + print failed.output.stderr.strip() + if failed.output.stdout: + print "--- stdout ---" + print failed.output.stdout.strip() + print "Command: %s" % EscapeCommand(self.runner.GetCommand(failed)) + if failed.output.HasCrashed(): + print "--- CRASHED ---" + if failed.output.HasTimedOut(): + print "--- TIMEOUT ---" + if len(self.runner.failed) == 0: + print "===" + print "=== All tests succeeded" + print "===" + else: + print + print "===" + print "=== %i tests failed" % len(self.runner.failed) + if self.runner.crashed > 0: + print "=== %i tests CRASHED" % self.runner.crashed + print "===" + + +class VerboseProgressIndicator(SimpleProgressIndicator): + + def AboutToRun(self, test): + print 'Starting %s...' % test.GetLabel() + sys.stdout.flush() + + def HasRun(self, test): + if test.suite.HasUnexpectedOutput(test): + if test.output.HasCrashed(): + outcome = 'CRASH' + else: + outcome = 'FAIL' + else: + outcome = 'pass' + print 'Done running %s: %s' % (test.GetLabel(), outcome) + + +class DotsProgressIndicator(SimpleProgressIndicator): + + def HasRun(self, test): + total = self.runner.succeeded + len(self.runner.failed) + if (total > 1) and (total % 50 == 1): + sys.stdout.write('\n') + if test.suite.HasUnexpectedOutput(test): + if test.output.HasCrashed(): + sys.stdout.write('C') + sys.stdout.flush() + elif test.output.HasTimedOut(): + sys.stdout.write('T') + sys.stdout.flush() + else: + sys.stdout.write('F') + sys.stdout.flush() + else: + sys.stdout.write('.') + sys.stdout.flush() + + +class CompactProgressIndicator(ProgressIndicator): + """Abstract base class for {Color,Monochrome}ProgressIndicator""" + + def __init__(self, templates): + super(CompactProgressIndicator, self).__init__() + self.templates = templates + self.last_status_length = 0 + self.start_time = time.time() + + def Done(self): + self.PrintProgress('Done') + print "" # Line break. + + def AboutToRun(self, test): + self.PrintProgress(test.GetLabel()) + + def HasRun(self, test): + if test.suite.HasUnexpectedOutput(test): + self.ClearLine(self.last_status_length) + self.PrintFailureHeader(test) + stdout = test.output.stdout.strip() + if len(stdout): + print self.templates['stdout'] % stdout + stderr = test.output.stderr.strip() + if len(stderr): + print self.templates['stderr'] % stderr + print "Command: %s" % EscapeCommand(self.runner.GetCommand(test)) + if test.output.HasCrashed(): + print "exit code: %d" % test.output.exit_code + print "--- CRASHED ---" + if test.output.HasTimedOut(): + print "--- TIMEOUT ---" + + def Truncate(self, string, length): + if length and (len(string) > (length - 3)): + return string[:(length - 3)] + "..." + else: + return string + + def PrintProgress(self, name): + self.ClearLine(self.last_status_length) + elapsed = time.time() - self.start_time + status = self.templates['status_line'] % { + 'passed': self.runner.succeeded, + 'remaining': (((self.runner.total - self.runner.remaining) * 100) // + self.runner.total), + 'failed': len(self.runner.failed), + 'test': name, + 'mins': int(elapsed) / 60, + 'secs': int(elapsed) % 60 + } + status = self.Truncate(status, 78) + self.last_status_length = len(status) + print status, + sys.stdout.flush() + + +class ColorProgressIndicator(CompactProgressIndicator): + + def __init__(self): + templates = { + 'status_line': ("[%(mins)02i:%(secs)02i|" + "\033[34m%%%(remaining) 4d\033[0m|" + "\033[32m+%(passed) 4d\033[0m|" + "\033[31m-%(failed) 4d\033[0m]: %(test)s"), + 'stdout': "\033[1m%s\033[0m", + 'stderr': "\033[31m%s\033[0m", + } + super(ColorProgressIndicator, self).__init__(templates) + + def ClearLine(self, last_line_length): + print "\033[1K\r", + + +class MonochromeProgressIndicator(CompactProgressIndicator): + + def __init__(self): + templates = { + 'status_line': ("[%(mins)02i:%(secs)02i|%%%(remaining) 4d|" + "+%(passed) 4d|-%(failed) 4d]: %(test)s"), + 'stdout': '%s', + 'stderr': '%s', + } + super(MonochromeProgressIndicator, self).__init__(templates) + + def ClearLine(self, last_line_length): + print ("\r" + (" " * last_line_length) + "\r"), + + +PROGRESS_INDICATORS = { + 'verbose': VerboseProgressIndicator, + 'dots': DotsProgressIndicator, + 'color': ColorProgressIndicator, + 'mono': MonochromeProgressIndicator +} diff --git a/src/3rdparty/v8/tools/testrunner/local/statusfile.py b/src/3rdparty/v8/tools/testrunner/local/statusfile.py new file mode 100644 index 0000000..bf1de45 --- /dev/null +++ b/src/3rdparty/v8/tools/testrunner/local/statusfile.py @@ -0,0 +1,145 @@ +# Copyright 2012 the V8 project authors. All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +# These imports are required for the on-demand conversion from +# old to new status file format. +from os.path import exists +from os.path import getmtime + +from . import old_statusfile + + +# These outcomes can occur in a TestCase's outcomes list: +SKIP = "SKIP" +FAIL = "FAIL" +PASS = "PASS" +OKAY = "OKAY" +TIMEOUT = "TIMEOUT" +CRASH = "CRASH" +SLOW = "SLOW" +# These are just for the status files and are mapped below in DEFS: +FAIL_OK = "FAIL_OK" +PASS_OR_FAIL = "PASS_OR_FAIL" + +ALWAYS = "ALWAYS" + +KEYWORDS = {} +for key in [SKIP, FAIL, PASS, OKAY, TIMEOUT, CRASH, SLOW, FAIL_OK, + PASS_OR_FAIL, ALWAYS]: + KEYWORDS[key] = key + +DEFS = {FAIL_OK: [FAIL, OKAY], + PASS_OR_FAIL: [PASS, FAIL]} + +# Support arches, modes to be written as keywords instead of strings. +VARIABLES = {ALWAYS: True} +for var in ["debug", "release", "android_arm", "android_ia32", "arm", "ia32", + "mipsel", "x64"]: + VARIABLES[var] = var + + +def DoSkip(outcomes): + return SKIP in outcomes or SLOW in outcomes + + +def IsFlaky(outcomes): + return ((PASS in outcomes) and (FAIL in outcomes) and + (not CRASH in outcomes) and (not OKAY in outcomes)) + + +def IsFailOk(outcomes): + return (FAIL in outcomes) and (OKAY in outcomes) + + +def _AddOutcome(result, new): + global DEFS + if new in DEFS: + mapped = DEFS[new] + if type(mapped) == list: + for m in mapped: + _AddOutcome(result, m) + elif type(mapped) == str: + _AddOutcome(result, mapped) + else: + result.add(new) + + +def _ParseOutcomeList(rule, outcomes, target_dict, variables): + result = set([]) + if type(outcomes) == str: + outcomes = [outcomes] + for item in outcomes: + if type(item) == str: + _AddOutcome(result, item) + elif type(item) == list: + if not eval(item[0], variables): continue + for outcome in item[1:]: + assert type(outcome) == str + _AddOutcome(result, outcome) + else: + assert False + if len(result) == 0: return + if rule in target_dict: + target_dict[rule] |= result + else: + target_dict[rule] = result + + +def ReadStatusFile(path, variables): + # As long as the old-format .status files are authoritative, just + # create the converted version on demand and cache it to speed up + # subsequent runs. + if path.endswith(".status"): + newpath = path + "2" + if not exists(newpath) or getmtime(newpath) < getmtime(path): + print "Converting status file." + converted = old_statusfile.ConvertNotation(path).GetOutput() + with open(newpath, 'w') as f: + f.write(converted) + path = newpath + + with open(path) as f: + global KEYWORDS + contents = eval(f.read(), KEYWORDS) + + rules = {} + wildcards = {} + variables.update(VARIABLES) + for section in contents: + assert type(section) == list + assert len(section) == 2 + if not eval(section[0], variables): continue + section = section[1] + assert type(section) == dict + for rule in section: + assert type(rule) == str + if rule[-1] == '*': + _ParseOutcomeList(rule, section[rule], wildcards, variables) + else: + _ParseOutcomeList(rule, section[rule], rules, variables) + return rules, wildcards diff --git a/src/3rdparty/v8/tools/testrunner/local/testsuite.py b/src/3rdparty/v8/tools/testrunner/local/testsuite.py new file mode 100644 index 0000000..de5cddd --- /dev/null +++ b/src/3rdparty/v8/tools/testrunner/local/testsuite.py @@ -0,0 +1,184 @@ +# Copyright 2012 the V8 project authors. All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +import imp +import os + +from . import statusfile + +class TestSuite(object): + + @staticmethod + def LoadTestSuite(root): + name = root.split(os.path.sep)[-1] + f = None + try: + (f, pathname, description) = imp.find_module("testcfg", [root]) + module = imp.load_module("testcfg", f, pathname, description) + suite = module.GetSuite(name, root) + finally: + if f: + f.close() + return suite + + def __init__(self, name, root): + self.name = name # string + self.root = root # string containing path + self.tests = None # list of TestCase objects + self.rules = None # dictionary mapping test path to list of outcomes + self.wildcards = None # dictionary mapping test paths to list of outcomes + self.total_duration = None # float, assigned on demand + + def shell(self): + return "d8" + + def suffix(self): + return ".js" + + def status_file(self): + return "%s/%s.status" % (self.root, self.name) + + # Used in the status file and for stdout printing. + def CommonTestName(self, testcase): + return testcase.path + + def ListTests(self, context): + raise NotImplementedError + + def VariantFlags(self): + return None + + def DownloadData(self): + pass + + def ReadStatusFile(self, variables): + (self.rules, self.wildcards) = \ + statusfile.ReadStatusFile(self.status_file(), variables) + + def ReadTestCases(self, context): + self.tests = self.ListTests(context) + + def FilterTestCasesByStatus(self, warn_unused_rules): + filtered = [] + used_rules = set() + for t in self.tests: + testname = self.CommonTestName(t) + if testname in self.rules: + used_rules.add(testname) + outcomes = self.rules[testname] + t.outcomes = outcomes # Even for skipped tests, as the TestCase + # object stays around and PrintReport() uses it. + if statusfile.DoSkip(outcomes): + continue # Don't add skipped tests to |filtered|. + if len(self.wildcards) != 0: + skip = False + for rule in self.wildcards: + assert rule[-1] == '*' + if testname.startswith(rule[:-1]): + used_rules.add(rule) + outcomes = self.wildcards[rule] + t.outcomes = outcomes + if statusfile.DoSkip(outcomes): + skip = True + break # "for rule in self.wildcards" + if skip: continue # "for t in self.tests" + filtered.append(t) + self.tests = filtered + + if not warn_unused_rules: + return + + for rule in self.rules: + if rule not in used_rules: + print("Unused rule: %s -> %s" % (rule, self.rules[rule])) + for rule in self.wildcards: + if rule not in used_rules: + print("Unused rule: %s -> %s" % (rule, self.wildcards[rule])) + + def FilterTestCasesByArgs(self, args): + filtered = [] + filtered_args = [] + for a in args: + argpath = a.split(os.path.sep) + if argpath[0] != self.name: + continue + if len(argpath) == 1 or (len(argpath) == 2 and argpath[1] == '*'): + return # Don't filter, run all tests in this suite. + path = os.path.sep.join(argpath[1:]) + if path[-1] == '*': + path = path[:-1] + filtered_args.append(path) + for t in self.tests: + for a in filtered_args: + if t.path.startswith(a): + filtered.append(t) + break + self.tests = filtered + + def GetFlagsForTestCase(self, testcase, context): + raise NotImplementedError + + def GetSourceForTest(self, testcase): + return "(no source available)" + + def IsFailureOutput(self, output, testpath): + return output.exit_code != 0 + + def IsNegativeTest(self, testcase): + return False + + def HasFailed(self, testcase): + execution_failed = self.IsFailureOutput(testcase.output, testcase.path) + if self.IsNegativeTest(testcase): + return not execution_failed + else: + return execution_failed + + def HasUnexpectedOutput(self, testcase): + if testcase.output.HasCrashed(): + outcome = statusfile.CRASH + elif testcase.output.HasTimedOut(): + outcome = statusfile.TIMEOUT + elif self.HasFailed(testcase): + outcome = statusfile.FAIL + else: + outcome = statusfile.PASS + if not testcase.outcomes: + return outcome != statusfile.PASS + return not outcome in testcase.outcomes + + def StripOutputForTransmit(self, testcase): + if not self.HasUnexpectedOutput(testcase): + testcase.output.stdout = "" + testcase.output.stderr = "" + + def CalculateTotalDuration(self): + self.total_duration = 0.0 + for t in self.tests: + self.total_duration += t.duration + return self.total_duration diff --git a/src/3rdparty/v8/tools/testrunner/local/utils.py b/src/3rdparty/v8/tools/testrunner/local/utils.py new file mode 100644 index 0000000..b7caa12 --- /dev/null +++ b/src/3rdparty/v8/tools/testrunner/local/utils.py @@ -0,0 +1,108 @@ +# Copyright 2012 the V8 project authors. All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +import os +from os.path import exists +from os.path import isdir +from os.path import join +import platform +import re + + +def GetSuitePaths(test_root): + def IsSuite(path): + return isdir(path) and exists(join(path, 'testcfg.py')) + return [ f for f in os.listdir(test_root) if IsSuite(join(test_root, f)) ] + + +# Reads a file into an array of strings +def ReadLinesFrom(name): + lines = [] + with open(name) as f: + for line in f: + if line.startswith('#'): continue + if '#' in line: + line = line[:line.find('#')] + line = line.strip() + if not line: continue + lines.append(line) + return lines + + +def GuessOS(): + system = platform.system() + if system == 'Linux': + return 'linux' + elif system == 'Darwin': + return 'macos' + elif system.find('CYGWIN') >= 0: + return 'cygwin' + elif system == 'Windows' or system == 'Microsoft': + # On Windows Vista platform.system() can return 'Microsoft' with some + # versions of Python, see http://bugs.python.org/issue1082 + return 'win32' + elif system == 'FreeBSD': + return 'freebsd' + elif system == 'OpenBSD': + return 'openbsd' + elif system == 'SunOS': + return 'solaris' + elif system == 'NetBSD': + return 'netbsd' + else: + return None + + +# This will default to building the 32 bit VM even on machines that are +# capable of running the 64 bit VM. +def DefaultArch(): + machine = platform.machine() + machine = machine.lower() # Windows 7 capitalizes 'AMD64'. + if machine.startswith('arm'): + return 'arm' + elif (not machine) or (not re.match('(x|i[3-6])86$', machine) is None): + return 'ia32' + elif machine == 'i86pc': + return 'ia32' + elif machine == 'x86_64': + return 'ia32' + elif machine == 'amd64': + return 'ia32' + else: + return None + + +def GuessWordsize(): + if '64' in platform.machine(): + return '64' + else: + return '32' + + +def IsWindows(): + return GuessOS() == 'win32' diff --git a/src/3rdparty/v8/tools/testrunner/local/verbose.py b/src/3rdparty/v8/tools/testrunner/local/verbose.py new file mode 100644 index 0000000..f693467 --- /dev/null +++ b/src/3rdparty/v8/tools/testrunner/local/verbose.py @@ -0,0 +1,99 @@ +# Copyright 2012 the V8 project authors. All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +import sys +import time + +from . import statusfile + + +REPORT_TEMPLATE = ( +"""Total: %(total)i tests + * %(skipped)4d tests will be skipped + * %(timeout)4d tests are expected to timeout sometimes + * %(nocrash)4d tests are expected to be flaky but not crash + * %(pass)4d tests are expected to pass + * %(fail_ok)4d tests are expected to fail that we won't fix + * %(fail)4d tests are expected to fail that we should fix""") + + +def PrintReport(tests): + total = len(tests) + skipped = timeout = nocrash = passes = fail_ok = fail = 0 + for t in tests: + if "outcomes" not in dir(t) or not t.outcomes: + passes += 1 + continue + o = t.outcomes + if statusfile.DoSkip(o): + skipped += 1 + continue + if statusfile.TIMEOUT in o: timeout += 1 + if statusfile.IsFlaky(o): nocrash += 1 + if list(o) == [statusfile.PASS]: passes += 1 + if statusfile.IsFailOk(o): fail_ok += 1 + if list(o) == [statusfile.FAIL]: fail += 1 + print REPORT_TEMPLATE % { + "total": total, + "skipped": skipped, + "timeout": timeout, + "nocrash": nocrash, + "pass": passes, + "fail_ok": fail_ok, + "fail": fail + } + + +def PrintTestSource(tests): + for test in tests: + suite = test.suite + source = suite.GetSourceForTest(test).strip() + if len(source) > 0: + print "--- begin source: %s/%s ---" % (suite.name, test.path) + print source + print "--- end source: %s/%s ---" % (suite.name, test.path) + + +def FormatTime(d): + millis = round(d * 1000) % 1000 + return time.strftime("%M:%S.", time.gmtime(d)) + ("%03i" % millis) + + +def PrintTestDurations(suites, overall_time): + # Write the times to stderr to make it easy to separate from the + # test output. + print + sys.stderr.write("--- Total time: %s ---\n" % FormatTime(overall_time)) + timed_tests = [ t for s in suites for t in s.tests + if t.duration is not None ] + timed_tests.sort(lambda a, b: cmp(b.duration, a.duration)) + index = 1 + for entry in timed_tests[:20]: + t = FormatTime(entry.duration) + sys.stderr.write("%4i (%s) %s\n" % (index, t, entry.GetLabel())) + index += 1 diff --git a/src/3rdparty/v8/tools/testrunner/network/__init__.py b/src/3rdparty/v8/tools/testrunner/network/__init__.py new file mode 100644 index 0000000..202a262 --- /dev/null +++ b/src/3rdparty/v8/tools/testrunner/network/__init__.py @@ -0,0 +1,26 @@ +# Copyright 2012 the V8 project authors. All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/src/3rdparty/v8/tools/testrunner/network/distro.py b/src/3rdparty/v8/tools/testrunner/network/distro.py new file mode 100644 index 0000000..9d5a471 --- /dev/null +++ b/src/3rdparty/v8/tools/testrunner/network/distro.py @@ -0,0 +1,90 @@ +# Copyright 2012 the V8 project authors. All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +class Shell(object): + def __init__(self, shell): + self.shell = shell + self.tests = [] + self.total_duration = 0.0 + + def AddSuite(self, suite): + self.tests += suite.tests + self.total_duration += suite.total_duration + + def SortTests(self): + self.tests.sort(cmp=lambda x, y: cmp(x.duration, y.duration)) + + +def Assign(suites, peers): + total_work = 0.0 + for s in suites: + total_work += s.CalculateTotalDuration() + + total_power = 0.0 + for p in peers: + p.assigned_work = 0.0 + total_power += p.jobs * p.relative_performance + for p in peers: + p.needed_work = total_work * p.jobs * p.relative_performance / total_power + + shells = {} + for s in suites: + shell = s.shell() + if not shell in shells: + shells[shell] = Shell(shell) + shells[shell].AddSuite(s) + # Convert |shells| to list and sort it, shortest total_duration first. + shells = [ shells[s] for s in shells ] + shells.sort(cmp=lambda x, y: cmp(x.total_duration, y.total_duration)) + # Sort tests within each shell, longest duration last (so it's + # pop()'ed first). + for s in shells: s.SortTests() + # Sort peers, least needed_work first. + peers.sort(cmp=lambda x, y: cmp(x.needed_work, y.needed_work)) + index = 0 + for shell in shells: + while len(shell.tests) > 0: + while peers[index].needed_work <= 0: + index += 1 + if index == len(peers): + print("BIG FAT WARNING: Assigning tests to peers failed. " + "Remaining tests: %d. Going to slow mode." % len(shell.tests)) + # Pick the least-busy peer. Sorting the list for each test + # is terribly slow, but this is just an emergency fallback anyway. + peers.sort(cmp=lambda x, y: cmp(x.needed_work, y.needed_work)) + peers[0].ForceAddOneTest(shell.tests.pop(), shell) + # If the peer already has a shell assigned and would need this one + # and then yet another, try to avoid it. + peer = peers[index] + if (shell.total_duration < peer.needed_work and + len(peer.shells) > 0 and + index < len(peers) - 1 and + shell.total_duration <= peers[index + 1].needed_work): + peers[index + 1].AddTests(shell) + else: + peer.AddTests(shell) diff --git a/src/3rdparty/v8/tools/testrunner/network/endpoint.py b/src/3rdparty/v8/tools/testrunner/network/endpoint.py new file mode 100644 index 0000000..5dc2b9f --- /dev/null +++ b/src/3rdparty/v8/tools/testrunner/network/endpoint.py @@ -0,0 +1,124 @@ +# Copyright 2012 the V8 project authors. All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +import multiprocessing +import os +import Queue +import threading +import time + +from ..local import execution +from ..local import progress +from ..local import testsuite +from ..local import utils +from ..server import compression + + +class EndpointProgress(progress.ProgressIndicator): + def __init__(self, sock, server, ctx): + super(EndpointProgress, self).__init__() + self.sock = sock + self.server = server + self.context = ctx + self.results_queue = [] # Accessors must synchronize themselves. + self.sender_lock = threading.Lock() + self.senderthread = threading.Thread(target=self._SenderThread) + self.senderthread.start() + + def HasRun(self, test): + # The runners that call this have a lock anyway, so this is safe. + self.results_queue.append(test) + + def _SenderThread(self): + keep_running = True + tests = [] + self.sender_lock.acquire() + while keep_running: + time.sleep(0.1) + # This should be "atomic enough" without locking :-) + # (We don't care which list any new elements get appended to, as long + # as we don't lose any and the last one comes last.) + current = self.results_queue + self.results_queue = [] + for c in current: + if c is None: + keep_running = False + else: + tests.append(c) + if keep_running and len(tests) < 1: + continue # Wait for more results. + if len(tests) < 1: break # We're done here. + result = [] + for t in tests: + result.append(t.PackResult()) + try: + compression.Send(result, self.sock) + except: + self.runner.terminate = True + for t in tests: + self.server.CompareOwnPerf(t, self.context.arch, self.context.mode) + tests = [] + self.sender_lock.release() + + +def Execute(workspace, ctx, tests, sock, server): + suite_paths = utils.GetSuitePaths(os.path.join(workspace, "test")) + suites = [] + for root in suite_paths: + suite = testsuite.TestSuite.LoadTestSuite( + os.path.join(workspace, "test", root)) + if suite: + suites.append(suite) + + suites_dict = {} + for s in suites: + suites_dict[s.name] = s + s.tests = [] + for t in tests: + suite = suites_dict[t.suite] + t.suite = suite + suite.tests.append(t) + + suites = [ s for s in suites if len(s.tests) > 0 ] + for s in suites: + s.DownloadData() + + progress_indicator = EndpointProgress(sock, server, ctx) + runner = execution.Runner(suites, progress_indicator, ctx) + try: + runner.Run(server.jobs) + except IOError, e: + if e.errno == 2: + message = ("File not found: %s, maybe you forgot to 'git add' it?" % + e.filename) + else: + message = "%s" % e + compression.Send([[-1, message]], sock) + progress_indicator.HasRun(None) # Sentinel to signal the end. + progress_indicator.sender_lock.acquire() # Released when sending is done. + progress_indicator.sender_lock.release() diff --git a/src/3rdparty/v8/tools/testrunner/network/network_execution.py b/src/3rdparty/v8/tools/testrunner/network/network_execution.py new file mode 100644 index 0000000..ddb59e6 --- /dev/null +++ b/src/3rdparty/v8/tools/testrunner/network/network_execution.py @@ -0,0 +1,253 @@ +# Copyright 2012 the V8 project authors. All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +import os +import socket +import subprocess +import threading +import time + +from . import distro +from . import perfdata +from ..local import execution +from ..objects import peer +from ..objects import workpacket +from ..server import compression +from ..server import constants +from ..server import local_handler +from ..server import signatures + + +def GetPeers(): + data = local_handler.LocalQuery([constants.REQUEST_PEERS]) + if not data: return [] + return [ peer.Peer.Unpack(p) for p in data ] + + +class NetworkedRunner(execution.Runner): + def __init__(self, suites, progress_indicator, context, peers, workspace): + self.suites = suites + num_tests = 0 + datapath = os.path.join("out", "testrunner_data") + self.perf_data_manager = perfdata.PerfDataManager(datapath) + self.perfdata = self.perf_data_manager.GetStore(context.arch, context.mode) + for s in suites: + for t in s.tests: + t.duration = self.perfdata.FetchPerfData(t) or 1.0 + num_tests += len(s.tests) + self._CommonInit(num_tests, progress_indicator, context) + self.tests = [] # Only used if we need to fall back to local execution. + self.tests_lock = threading.Lock() + self.peers = peers + self.pubkey_fingerprint = None # Fetched later. + self.base_rev = subprocess.check_output( + "cd %s; git log -1 --format=%%H --grep=git-svn-id" % workspace, + shell=True).strip() + self.base_svn_rev = subprocess.check_output( + "cd %s; git log -1 %s" # Get commit description. + " | grep -e '^\s*git-svn-id:'" # Extract "git-svn-id" line. + " | awk '{print $2}'" # Extract "repository@revision" part. + " | sed -e 's/.*@//'" % # Strip away "repository@". + (workspace, self.base_rev), shell=True).strip() + self.patch = subprocess.check_output( + "cd %s; git diff %s" % (workspace, self.base_rev), shell=True) + self.binaries = {} + self.initialization_lock = threading.Lock() + self.initialization_lock.acquire() # Released when init is done. + self._OpenLocalConnection() + self.local_receiver_thread = threading.Thread( + target=self._ListenLocalConnection) + self.local_receiver_thread.daemon = True + self.local_receiver_thread.start() + self.initialization_lock.acquire() + self.initialization_lock.release() + + def _OpenLocalConnection(self): + self.local_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + code = self.local_socket.connect_ex(("localhost", constants.CLIENT_PORT)) + if code != 0: + raise RuntimeError("Failed to connect to local server") + compression.Send([constants.REQUEST_PUBKEY_FINGERPRINT], self.local_socket) + + def _ListenLocalConnection(self): + release_lock_countdown = 1 # Pubkey. + self.local_receiver = compression.Receiver(self.local_socket) + while not self.local_receiver.IsDone(): + data = self.local_receiver.Current() + if data[0] == constants.REQUEST_PUBKEY_FINGERPRINT: + pubkey = data[1] + if not pubkey: raise RuntimeError("Received empty public key") + self.pubkey_fingerprint = pubkey + release_lock_countdown -= 1 + if release_lock_countdown == 0: + self.initialization_lock.release() + release_lock_countdown -= 1 # Prevent repeated triggering. + self.local_receiver.Advance() + + def Run(self, jobs): + self.indicator.Starting() + need_libv8 = False + for s in self.suites: + shell = s.shell() + if shell not in self.binaries: + path = os.path.join(self.context.shell_dir, shell) + # Check if this is a shared library build. + try: + ldd = subprocess.check_output("ldd %s | grep libv8\\.so" % (path), + shell=True) + ldd = ldd.strip().split(" ") + assert ldd[0] == "libv8.so" + assert ldd[1] == "=>" + need_libv8 = True + binary_needs_libv8 = True + libv8 = signatures.ReadFileAndSignature(ldd[2]) + except: + binary_needs_libv8 = False + binary = signatures.ReadFileAndSignature(path) + if binary[0] is None: + print("Error: Failed to create signature.") + assert binary[1] != 0 + return binary[1] + binary.append(binary_needs_libv8) + self.binaries[shell] = binary + if need_libv8: + self.binaries["libv8.so"] = libv8 + distro.Assign(self.suites, self.peers) + # Spawn one thread for each peer. + threads = [] + for p in self.peers: + thread = threading.Thread(target=self._TalkToPeer, args=[p]) + threads.append(thread) + thread.start() + try: + for thread in threads: + # Use a timeout so that signals (Ctrl+C) will be processed. + thread.join(timeout=10000000) + self._AnalyzePeerRuntimes() + except KeyboardInterrupt: + self.terminate = True + raise + except Exception, _e: + # If there's an exception we schedule an interruption for any + # remaining threads... + self.terminate = True + # ...and then reraise the exception to bail out. + raise + compression.Send(constants.END_OF_STREAM, self.local_socket) + self.local_socket.close() + if self.tests: + self._RunInternal(jobs) + self.indicator.Done() + return not self.failed + + def _TalkToPeer(self, peer): + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.settimeout(self.context.timeout + 10) + code = sock.connect_ex((peer.address, constants.PEER_PORT)) + if code == 0: + try: + peer.runtime = None + start_time = time.time() + packet = workpacket.WorkPacket(peer=peer, context=self.context, + base_revision=self.base_svn_rev, + patch=self.patch, + pubkey=self.pubkey_fingerprint) + data, test_map = packet.Pack(self.binaries) + compression.Send(data, sock) + compression.Send(constants.END_OF_STREAM, sock) + rec = compression.Receiver(sock) + while not rec.IsDone() and not self.terminate: + data_list = rec.Current() + for data in data_list: + test_id = data[0] + if test_id < 0: + # The peer is reporting an error. + with self.lock: + print("\nPeer %s reports error: %s" % (peer.address, data[1])) + continue + test = test_map.pop(test_id) + test.MergeResult(data) + try: + self.perfdata.UpdatePerfData(test) + except Exception, e: + print("UpdatePerfData exception: %s" % e) + pass # Just keep working. + with self.lock: + perf_key = self.perfdata.GetKey(test) + compression.Send( + [constants.INFORM_DURATION, perf_key, test.duration, + self.context.arch, self.context.mode], + self.local_socket) + self.indicator.AboutToRun(test) + if test.suite.HasUnexpectedOutput(test): + self.failed.append(test) + if test.output.HasCrashed(): + self.crashed += 1 + else: + self.succeeded += 1 + self.remaining -= 1 + self.indicator.HasRun(test) + rec.Advance() + peer.runtime = time.time() - start_time + except KeyboardInterrupt: + sock.close() + raise + except Exception, e: + print("Got exception: %s" % e) + pass # Fall back to local execution. + else: + compression.Send([constants.UNRESPONSIVE_PEER, peer.address], + self.local_socket) + sock.close() + if len(test_map) > 0: + # Some tests have not received any results. Run them locally. + print("\nNo results for %d tests, running them locally." % len(test_map)) + self._EnqueueLocally(test_map) + + def _EnqueueLocally(self, test_map): + with self.tests_lock: + for test in test_map: + self.tests.append(test_map[test]) + + def _AnalyzePeerRuntimes(self): + total_runtime = 0.0 + total_work = 0.0 + for p in self.peers: + if p.runtime is None: + return + total_runtime += p.runtime + total_work += p.assigned_work + for p in self.peers: + p.assigned_work /= total_work + p.runtime /= total_runtime + perf_correction = p.assigned_work / p.runtime + old_perf = p.relative_performance + p.relative_performance = (old_perf + perf_correction) / 2.0 + compression.Send([constants.UPDATE_PERF, p.address, + p.relative_performance], + self.local_socket) diff --git a/src/3rdparty/v8/tools/testrunner/network/perfdata.py b/src/3rdparty/v8/tools/testrunner/network/perfdata.py new file mode 100644 index 0000000..2979dc4 --- /dev/null +++ b/src/3rdparty/v8/tools/testrunner/network/perfdata.py @@ -0,0 +1,120 @@ +# Copyright 2012 the V8 project authors. All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +import os +import shelve +import threading + + +class PerfDataEntry(object): + def __init__(self): + self.avg = 0.0 + self.count = 0 + + def AddResult(self, result): + kLearnRateLimiter = 99 # Greater value means slower learning. + # We use an approximation of the average of the last 100 results here: + # The existing average is weighted with kLearnRateLimiter (or less + # if there are fewer data points). + effective_count = min(self.count, kLearnRateLimiter) + self.avg = self.avg * effective_count + result + self.count = effective_count + 1 + self.avg /= self.count + + +class PerfDataStore(object): + def __init__(self, datadir, arch, mode): + filename = os.path.join(datadir, "%s.%s.perfdata" % (arch, mode)) + self.database = shelve.open(filename, protocol=2) + self.closed = False + self.lock = threading.Lock() + + def __del__(self): + self.close() + + def close(self): + if self.closed: return + self.database.close() + self.closed = True + + def GetKey(self, test): + """Computes the key used to access data for the given testcase.""" + flags = "".join(test.flags) + return str("%s.%s.%s" % (test.suitename(), test.path, flags)) + + def FetchPerfData(self, test): + """Returns the observed duration for |test| as read from the store.""" + key = self.GetKey(test) + if key in self.database: + return self.database[key].avg + return None + + def UpdatePerfData(self, test): + """Updates the persisted value in the store with test.duration.""" + testkey = self.GetKey(test) + self.RawUpdatePerfData(testkey, test.duration) + + def RawUpdatePerfData(self, testkey, duration): + with self.lock: + if testkey in self.database: + entry = self.database[testkey] + else: + entry = PerfDataEntry() + entry.AddResult(duration) + self.database[testkey] = entry + + +class PerfDataManager(object): + def __init__(self, datadir): + self.datadir = os.path.abspath(datadir) + if not os.path.exists(self.datadir): + os.makedirs(self.datadir) + self.stores = {} # Keyed by arch, then mode. + self.closed = False + self.lock = threading.Lock() + + def __del__(self): + self.close() + + def close(self): + if self.closed: return + for arch in self.stores: + modes = self.stores[arch] + for mode in modes: + store = modes[mode] + store.close() + self.closed = True + + def GetStore(self, arch, mode): + with self.lock: + if not arch in self.stores: + self.stores[arch] = {} + modes = self.stores[arch] + if not mode in modes: + modes[mode] = PerfDataStore(self.datadir, arch, mode) + return modes[mode] diff --git a/src/3rdparty/v8/tools/testrunner/objects/__init__.py b/src/3rdparty/v8/tools/testrunner/objects/__init__.py new file mode 100644 index 0000000..202a262 --- /dev/null +++ b/src/3rdparty/v8/tools/testrunner/objects/__init__.py @@ -0,0 +1,26 @@ +# Copyright 2012 the V8 project authors. All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/src/3rdparty/v8/tools/testrunner/objects/context.py b/src/3rdparty/v8/tools/testrunner/objects/context.py new file mode 100644 index 0000000..b72284b --- /dev/null +++ b/src/3rdparty/v8/tools/testrunner/objects/context.py @@ -0,0 +1,50 @@ +# Copyright 2012 the V8 project authors. All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +class Context(): + def __init__(self, arch, mode, shell_dir, mode_flags, verbose, timeout, + isolates, command_prefix, extra_flags): + self.arch = arch + self.mode = mode + self.shell_dir = shell_dir + self.mode_flags = mode_flags + self.verbose = verbose + self.timeout = timeout + self.isolates = isolates + self.command_prefix = command_prefix + self.extra_flags = extra_flags + + def Pack(self): + return [self.arch, self.mode, self.mode_flags, self.timeout, self.isolates, + self.extra_flags] + + @staticmethod + def Unpack(packed): + # For the order of the fields, refer to Pack() above. + return Context(packed[0], packed[1], None, packed[2], False, + packed[3], packed[4], "", packed[5]) diff --git a/src/3rdparty/v8/tools/testrunner/objects/output.py b/src/3rdparty/v8/tools/testrunner/objects/output.py new file mode 100644 index 0000000..87b4c84 --- /dev/null +++ b/src/3rdparty/v8/tools/testrunner/objects/output.py @@ -0,0 +1,60 @@ +# Copyright 2012 the V8 project authors. All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +import signal + +from ..local import utils + +class Output(object): + + def __init__(self, exit_code, timed_out, stdout, stderr): + self.exit_code = exit_code + self.timed_out = timed_out + self.stdout = stdout + self.stderr = stderr + + def HasCrashed(self): + if utils.IsWindows(): + return 0x80000000 & self.exit_code and not (0x3FFFFF00 & self.exit_code) + else: + # Timed out tests will have exit_code -signal.SIGTERM. + if self.timed_out: + return False + return (self.exit_code < 0 and + self.exit_code != -signal.SIGABRT) + + def HasTimedOut(self): + return self.timed_out + + def Pack(self): + return [self.exit_code, self.timed_out, self.stdout, self.stderr] + + @staticmethod + def Unpack(packed): + # For the order of the fields, refer to Pack() above. + return Output(packed[0], packed[1], packed[2], packed[3]) diff --git a/src/3rdparty/v8/tools/testrunner/objects/peer.py b/src/3rdparty/v8/tools/testrunner/objects/peer.py new file mode 100644 index 0000000..18a6bec --- /dev/null +++ b/src/3rdparty/v8/tools/testrunner/objects/peer.py @@ -0,0 +1,80 @@ +# Copyright 2012 the V8 project authors. All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +class Peer(object): + def __init__(self, address, jobs, rel_perf, pubkey): + self.address = address # string: IP address + self.jobs = jobs # integer: number of CPUs + self.relative_performance = rel_perf + self.pubkey = pubkey # string: pubkey's fingerprint + self.shells = set() # set of strings + self.needed_work = 0 + self.assigned_work = 0 + self.tests = [] # list of TestCase objects + self.trusting_me = False # This peer trusts my public key. + self.trusted = False # I trust this peer's public key. + + def __str__(self): + return ("Peer at %s, jobs: %d, performance: %.2f, trust I/O: %s/%s" % + (self.address, self.jobs, self.relative_performance, + self.trusting_me, self.trusted)) + + def AddTests(self, shell): + """Adds tests from |shell| to this peer. + + Stops when self.needed_work reaches zero, or when all of shell's tests + are assigned.""" + assert self.needed_work > 0 + if shell.shell not in self.shells: + self.shells.add(shell.shell) + while len(shell.tests) > 0 and self.needed_work > 0: + t = shell.tests.pop() + self.needed_work -= t.duration + self.assigned_work += t.duration + shell.total_duration -= t.duration + self.tests.append(t) + + def ForceAddOneTest(self, test, shell): + """Forcibly adds another test to this peer, disregarding needed_work.""" + if shell.shell not in self.shells: + self.shells.add(shell.shell) + self.needed_work -= test.duration + self.assigned_work += test.duration + shell.total_duration -= test.duration + self.tests.append(test) + + + def Pack(self): + """Creates a JSON serializable representation of this Peer.""" + return [self.address, self.jobs, self.relative_performance] + + @staticmethod + def Unpack(packed): + """Creates a Peer object built from a packed representation.""" + pubkey_dummy = "" # Callers of this don't care (only the server does). + return Peer(packed[0], packed[1], packed[2], pubkey_dummy) diff --git a/src/3rdparty/v8/tools/testrunner/objects/testcase.py b/src/3rdparty/v8/tools/testrunner/objects/testcase.py new file mode 100644 index 0000000..cfc522e --- /dev/null +++ b/src/3rdparty/v8/tools/testrunner/objects/testcase.py @@ -0,0 +1,83 @@ +# Copyright 2012 the V8 project authors. All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +from . import output + +class TestCase(object): + def __init__(self, suite, path, flags=[], dependency=None): + self.suite = suite # TestSuite object + self.path = path # string, e.g. 'div-mod', 'test-api/foo' + self.flags = flags # list of strings, flags specific to this test case + self.dependency = dependency # |path| for testcase that must be run first + self.outcomes = None + self.output = None + self.id = None # int, used to map result back to TestCase instance + self.duration = None # assigned during execution + + def CopyAddingFlags(self, flags): + copy = TestCase(self.suite, self.path, self.flags + flags, self.dependency) + copy.outcomes = self.outcomes + return copy + + def PackTask(self): + """ + Extracts those parts of this object that are required to run the test + and returns them as a JSON serializable object. + """ + assert self.id is not None + return [self.suitename(), self.path, self.flags, + self.dependency, list(self.outcomes or []), self.id] + + @staticmethod + def UnpackTask(task): + """Creates a new TestCase object based on packed task data.""" + # For the order of the fields, refer to PackTask() above. + test = TestCase(str(task[0]), task[1], task[2], task[3]) + test.outcomes = set(task[4]) + test.id = task[5] + return test + + def SetSuiteObject(self, suites): + self.suite = suites[self.suite] + + def PackResult(self): + """Serializes the output of the TestCase after it has run.""" + self.suite.StripOutputForTransmit(self) + return [self.id, self.output.Pack(), self.duration] + + def MergeResult(self, result): + """Applies the contents of a Result to this object.""" + assert result[0] == self.id + self.output = output.Output.Unpack(result[1]) + self.duration = result[2] + + def suitename(self): + return self.suite.name + + def GetLabel(self): + return self.suitename() + "/" + self.suite.CommonTestName(self) diff --git a/src/3rdparty/v8/tools/testrunner/objects/workpacket.py b/src/3rdparty/v8/tools/testrunner/objects/workpacket.py new file mode 100644 index 0000000..d07efe7 --- /dev/null +++ b/src/3rdparty/v8/tools/testrunner/objects/workpacket.py @@ -0,0 +1,90 @@ +# Copyright 2012 the V8 project authors. All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +from . import context +from . import testcase + +class WorkPacket(object): + def __init__(self, peer=None, context=None, tests=None, binaries=None, + base_revision=None, patch=None, pubkey=None): + self.peer = peer + self.context = context + self.tests = tests + self.binaries = binaries + self.base_revision = base_revision + self.patch = patch + self.pubkey_fingerprint = pubkey + + def Pack(self, binaries_dict): + """ + Creates a JSON serializable object containing the data of this + work packet. + """ + need_libv8 = False + binaries = [] + for shell in self.peer.shells: + prefetched_binary = binaries_dict[shell] + binaries.append({"name": shell, + "blob": prefetched_binary[0], + "sign": prefetched_binary[1]}) + if prefetched_binary[2]: + need_libv8 = True + if need_libv8: + libv8 = binaries_dict["libv8.so"] + binaries.append({"name": "libv8.so", + "blob": libv8[0], + "sign": libv8[1]}) + tests = [] + test_map = {} + for t in self.peer.tests: + test_map[t.id] = t + tests.append(t.PackTask()) + result = { + "binaries": binaries, + "pubkey": self.pubkey_fingerprint, + "context": self.context.Pack(), + "base_revision": self.base_revision, + "patch": self.patch, + "tests": tests + } + return result, test_map + + @staticmethod + def Unpack(packed): + """ + Creates a WorkPacket object from the given packed representation. + """ + binaries = packed["binaries"] + pubkey_fingerprint = packed["pubkey"] + ctx = context.Context.Unpack(packed["context"]) + base_revision = packed["base_revision"] + patch = packed["patch"] + tests = [ testcase.TestCase.UnpackTask(t) for t in packed["tests"] ] + return WorkPacket(context=ctx, tests=tests, binaries=binaries, + base_revision=base_revision, patch=patch, + pubkey=pubkey_fingerprint) diff --git a/src/3rdparty/v8/tools/testrunner/server/__init__.py b/src/3rdparty/v8/tools/testrunner/server/__init__.py new file mode 100644 index 0000000..202a262 --- /dev/null +++ b/src/3rdparty/v8/tools/testrunner/server/__init__.py @@ -0,0 +1,26 @@ +# Copyright 2012 the V8 project authors. All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/src/3rdparty/v8/tools/testrunner/server/compression.py b/src/3rdparty/v8/tools/testrunner/server/compression.py new file mode 100644 index 0000000..ce90c4f --- /dev/null +++ b/src/3rdparty/v8/tools/testrunner/server/compression.py @@ -0,0 +1,112 @@ +# Copyright 2012 the V8 project authors. All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +import cStringIO as StringIO +try: + import ujson as json +except ImportError: + print("You should install UltraJSON, it is much faster!") + import json +import os +import struct +import zlib + +from . import constants + +def Send(obj, sock): + """ + Sends a JSON encodable object over the specified socket (zlib-compressed). + """ + obj = json.dumps(obj) + compression_level = 2 # 1 = fastest, 9 = best compression + compressed = zlib.compress(obj, compression_level) + payload = struct.pack('>i', len(compressed)) + compressed + sock.sendall(payload) + + +class Receiver(object): + def __init__(self, sock): + self.sock = sock + self.data = StringIO.StringIO() + self.datalength = 0 + self._next = self._GetNext() + + def IsDone(self): + return self._next == None + + def Current(self): + return self._next + + def Advance(self): + try: + self._next = self._GetNext() + except: + raise + + def _GetNext(self): + try: + while self.datalength < constants.SIZE_T: + try: + chunk = self.sock.recv(8192) + except: + raise + if not chunk: return None + self._AppendData(chunk) + size = self._PopData(constants.SIZE_T) + size = struct.unpack(">i", size)[0] + while self.datalength < size: + try: + chunk = self.sock.recv(8192) + except: + raise + if not chunk: return None + self._AppendData(chunk) + result = self._PopData(size) + result = zlib.decompress(result) + result = json.loads(result) + if result == constants.END_OF_STREAM: + return None + return result + except: + raise + + def _AppendData(self, new): + self.data.seek(0, os.SEEK_END) + self.data.write(new) + self.datalength += len(new) + + def _PopData(self, length): + self.data.seek(0) + chunk = self.data.read(length) + remaining = self.data.read() + self.data.close() + self.data = StringIO.StringIO() + self.data.write(remaining) + assert self.datalength - length == len(remaining) + self.datalength = len(remaining) + return chunk diff --git a/src/3rdparty/v8/tools/testrunner/server/constants.py b/src/3rdparty/v8/tools/testrunner/server/constants.py new file mode 100644 index 0000000..5aefcba --- /dev/null +++ b/src/3rdparty/v8/tools/testrunner/server/constants.py @@ -0,0 +1,51 @@ +# Copyright 2012 the V8 project authors. All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +CLIENT_PORT = 9991 # Port for the local client to connect to. +PEER_PORT = 9992 # Port for peers on the network to connect to. +PRESENCE_PORT = 9993 # Port for presence daemon. +STATUS_PORT = 9994 # Port for network requests not related to workpackets. + +END_OF_STREAM = "end of dtest stream" # Marker for end of network requests. +SIZE_T = 4 # Number of bytes used for network request size header. + +# Messages understood by the local request handler. +ADD_TRUSTED = "add trusted" +INFORM_DURATION = "inform about duration" +REQUEST_PEERS = "get peers" +UNRESPONSIVE_PEER = "unresponsive peer" +REQUEST_PUBKEY_FINGERPRINT = "get pubkey fingerprint" +REQUEST_STATUS = "get status" +UPDATE_PERF = "update performance" + +# Messages understood by the status request handler. +LIST_TRUSTED_PUBKEYS = "list trusted pubkeys" +GET_SIGNED_PUBKEY = "pass on signed pubkey" +NOTIFY_NEW_TRUSTED = "new trusted peer" +TRUST_YOU_NOW = "trust you now" +DO_YOU_TRUST = "do you trust" diff --git a/src/3rdparty/v8/tools/testrunner/server/daemon.py b/src/3rdparty/v8/tools/testrunner/server/daemon.py new file mode 100644 index 0000000..baa66fb --- /dev/null +++ b/src/3rdparty/v8/tools/testrunner/server/daemon.py @@ -0,0 +1,147 @@ +#!/usr/bin/env python + +# This code has been written by Sander Marechal and published at: +# http://www.jejik.com/articles/2007/02/a_simple_unix_linux_daemon_in_python/ +# where the author has placed it in the public domain (see comment #6 at +# http://www.jejik.com/articles/2007/02/a_simple_unix_linux_daemon_in_python/#c6 +# ). +# Some minor modifications have been made by the V8 authors. The work remains +# in the public domain. + +import atexit +import os +from signal import SIGTERM +from signal import SIGINT +import sys +import time + + +class Daemon(object): + """ + A generic daemon class. + + Usage: subclass the Daemon class and override the run() method + """ + def __init__(self, pidfile, stdin='/dev/null', + stdout='/dev/null', stderr='/dev/null'): + self.stdin = stdin + self.stdout = stdout + self.stderr = stderr + self.pidfile = pidfile + + def daemonize(self): + """ + do the UNIX double-fork magic, see Stevens' "Advanced + Programming in the UNIX Environment" for details (ISBN 0201563177) + http://www.erlenstar.demon.co.uk/unix/faq_2.html#SEC16 + """ + try: + pid = os.fork() + if pid > 0: + # exit first parent + sys.exit(0) + except OSError, e: + sys.stderr.write("fork #1 failed: %d (%s)\n" % (e.errno, e.strerror)) + sys.exit(1) + + # decouple from parent environment + os.chdir("/") + os.setsid() + os.umask(0) + + # do second fork + try: + pid = os.fork() + if pid > 0: + # exit from second parent + sys.exit(0) + except OSError, e: + sys.stderr.write("fork #2 failed: %d (%s)\n" % (e.errno, e.strerror)) + sys.exit(1) + + # redirect standard file descriptors + sys.stdout.flush() + sys.stderr.flush() + si = file(self.stdin, 'r') + so = file(self.stdout, 'a+') + se = file(self.stderr, 'a+', 0) + # TODO: (debug) re-enable this! + #os.dup2(si.fileno(), sys.stdin.fileno()) + #os.dup2(so.fileno(), sys.stdout.fileno()) + #os.dup2(se.fileno(), sys.stderr.fileno()) + + # write pidfile + atexit.register(self.delpid) + pid = str(os.getpid()) + file(self.pidfile, 'w+').write("%s\n" % pid) + + def delpid(self): + os.remove(self.pidfile) + + def start(self): + """ + Start the daemon + """ + # Check for a pidfile to see if the daemon already runs + try: + pf = file(self.pidfile, 'r') + pid = int(pf.read().strip()) + pf.close() + except IOError: + pid = None + + if pid: + message = "pidfile %s already exist. Daemon already running?\n" + sys.stderr.write(message % self.pidfile) + sys.exit(1) + + # Start the daemon + self.daemonize() + self.run() + + def stop(self): + """ + Stop the daemon + """ + # Get the pid from the pidfile + try: + pf = file(self.pidfile, 'r') + pid = int(pf.read().strip()) + pf.close() + except IOError: + pid = None + + if not pid: + message = "pidfile %s does not exist. Daemon not running?\n" + sys.stderr.write(message % self.pidfile) + return # not an error in a restart + + # Try killing the daemon process + try: + # Give the process a one-second chance to exit gracefully. + os.kill(pid, SIGINT) + time.sleep(1) + while 1: + os.kill(pid, SIGTERM) + time.sleep(0.1) + except OSError, err: + err = str(err) + if err.find("No such process") > 0: + if os.path.exists(self.pidfile): + os.remove(self.pidfile) + else: + print str(err) + sys.exit(1) + + def restart(self): + """ + Restart the daemon + """ + self.stop() + self.start() + + def run(self): + """ + You should override this method when you subclass Daemon. It will be + called after the process has been daemonized by start() or restart(). + """ diff --git a/src/3rdparty/v8/tools/testrunner/server/local_handler.py b/src/3rdparty/v8/tools/testrunner/server/local_handler.py new file mode 100644 index 0000000..3b3ac49 --- /dev/null +++ b/src/3rdparty/v8/tools/testrunner/server/local_handler.py @@ -0,0 +1,119 @@ +# Copyright 2012 the V8 project authors. All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +import socket +import SocketServer +import StringIO + +from . import compression +from . import constants + + +def LocalQuery(query): + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + code = sock.connect_ex(("localhost", constants.CLIENT_PORT)) + if code != 0: return None + compression.Send(query, sock) + compression.Send(constants.END_OF_STREAM, sock) + rec = compression.Receiver(sock) + data = None + while not rec.IsDone(): + data = rec.Current() + assert data[0] == query[0] + data = data[1] + rec.Advance() + sock.close() + return data + + +class LocalHandler(SocketServer.BaseRequestHandler): + def handle(self): + rec = compression.Receiver(self.request) + while not rec.IsDone(): + data = rec.Current() + action = data[0] + + if action == constants.REQUEST_PEERS: + with self.server.daemon.peer_list_lock: + response = [ p.Pack() for p in self.server.daemon.peers + if p.trusting_me ] + compression.Send([action, response], self.request) + + elif action == constants.UNRESPONSIVE_PEER: + self.server.daemon.DeletePeer(data[1]) + + elif action == constants.REQUEST_PUBKEY_FINGERPRINT: + compression.Send([action, self.server.daemon.pubkey_fingerprint], + self.request) + + elif action == constants.REQUEST_STATUS: + compression.Send([action, self._GetStatusMessage()], self.request) + + elif action == constants.ADD_TRUSTED: + fingerprint = self.server.daemon.CopyToTrusted(data[1]) + compression.Send([action, fingerprint], self.request) + + elif action == constants.INFORM_DURATION: + test_key = data[1] + test_duration = data[2] + arch = data[3] + mode = data[4] + self.server.daemon.AddPerfData(test_key, test_duration, arch, mode) + + elif action == constants.UPDATE_PERF: + address = data[1] + perf = data[2] + self.server.daemon.UpdatePeerPerformance(data[1], data[2]) + + rec.Advance() + compression.Send(constants.END_OF_STREAM, self.request) + + def _GetStatusMessage(self): + sio = StringIO.StringIO() + sio.write("Peers:\n") + with self.server.daemon.peer_list_lock: + for p in self.server.daemon.peers: + sio.write("%s\n" % p) + sio.write("My own jobs: %d, relative performance: %.2f\n" % + (self.server.daemon.jobs, self.server.daemon.relative_perf)) + # Low-priority TODO: Return more information. Ideas: + # - currently running anything, + # - time since last job, + # - time since last repository fetch + # - number of workpackets/testcases handled since startup + # - slowest test(s) + result = sio.getvalue() + sio.close() + return result + + +class LocalSocketServer(SocketServer.ThreadingMixIn, SocketServer.TCPServer): + def __init__(self, daemon): + SocketServer.TCPServer.__init__(self, ("localhost", constants.CLIENT_PORT), + LocalHandler) + self.daemon = daemon diff --git a/src/3rdparty/v8/tools/testrunner/server/main.py b/src/3rdparty/v8/tools/testrunner/server/main.py new file mode 100644 index 0000000..1000713 --- /dev/null +++ b/src/3rdparty/v8/tools/testrunner/server/main.py @@ -0,0 +1,245 @@ +# Copyright 2012 the V8 project authors. All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +import multiprocessing +import os +import shutil +import subprocess +import threading +import time + +from . import daemon +from . import local_handler +from . import presence_handler +from . import signatures +from . import status_handler +from . import work_handler +from ..network import perfdata + + +class Server(daemon.Daemon): + + def __init__(self, pidfile, root, stdin="/dev/null", + stdout="/dev/null", stderr="/dev/null"): + super(Server, self).__init__(pidfile, stdin, stdout, stderr) + self.root = root + self.local_handler = None + self.local_handler_thread = None + self.work_handler = None + self.work_handler_thread = None + self.status_handler = None + self.status_handler_thread = None + self.presence_daemon = None + self.presence_daemon_thread = None + self.peers = [] + self.jobs = multiprocessing.cpu_count() + self.peer_list_lock = threading.Lock() + self.perf_data_lock = None + self.presence_daemon_lock = None + self.datadir = os.path.join(self.root, "data") + pubkey_fingerprint_filename = os.path.join(self.datadir, "mypubkey") + with open(pubkey_fingerprint_filename) as f: + self.pubkey_fingerprint = f.read().strip() + self.relative_perf_filename = os.path.join(self.datadir, "myperf") + if os.path.exists(self.relative_perf_filename): + with open(self.relative_perf_filename) as f: + try: + self.relative_perf = float(f.read()) + except: + self.relative_perf = 1.0 + else: + self.relative_perf = 1.0 + + def run(self): + os.nice(20) + self.ip = presence_handler.GetOwnIP() + self.perf_data_manager = perfdata.PerfDataManager(self.datadir) + self.perf_data_lock = threading.Lock() + + self.local_handler = local_handler.LocalSocketServer(self) + self.local_handler_thread = threading.Thread( + target=self.local_handler.serve_forever) + self.local_handler_thread.start() + + self.work_handler = work_handler.WorkSocketServer(self) + self.work_handler_thread = threading.Thread( + target=self.work_handler.serve_forever) + self.work_handler_thread.start() + + self.status_handler = status_handler.StatusSocketServer(self) + self.status_handler_thread = threading.Thread( + target=self.status_handler.serve_forever) + self.status_handler_thread.start() + + self.presence_daemon = presence_handler.PresenceDaemon(self) + self.presence_daemon_thread = threading.Thread( + target=self.presence_daemon.serve_forever) + self.presence_daemon_thread.start() + + self.presence_daemon.FindPeers() + time.sleep(0.5) # Give those peers some time to reply. + + with self.peer_list_lock: + for p in self.peers: + if p.address == self.ip: continue + status_handler.RequestTrustedPubkeys(p, self) + + while True: + try: + self.PeriodicTasks() + time.sleep(60) + except Exception, e: + print("MAIN LOOP EXCEPTION: %s" % e) + self.Shutdown() + break + except KeyboardInterrupt: + self.Shutdown() + break + + def Shutdown(self): + with open(self.relative_perf_filename, "w") as f: + f.write("%s" % self.relative_perf) + self.presence_daemon.shutdown() + self.presence_daemon.server_close() + self.local_handler.shutdown() + self.local_handler.server_close() + self.work_handler.shutdown() + self.work_handler.server_close() + self.status_handler.shutdown() + self.status_handler.server_close() + + def PeriodicTasks(self): + # If we know peers we don't trust, see if someone else trusts them. + with self.peer_list_lock: + for p in self.peers: + if p.trusted: continue + if self.IsTrusted(p.pubkey): + p.trusted = True + status_handler.ITrustYouNow(p) + continue + for p2 in self.peers: + if not p2.trusted: continue + status_handler.TryTransitiveTrust(p2, p.pubkey, self) + # TODO: Ping for more peers waiting to be discovered. + # TODO: Update the checkout (if currently idle). + + def AddPeer(self, peer): + with self.peer_list_lock: + for p in self.peers: + if p.address == peer.address: + return + self.peers.append(peer) + if peer.trusted: + status_handler.ITrustYouNow(peer) + + def DeletePeer(self, peer_address): + with self.peer_list_lock: + for i in xrange(len(self.peers)): + if self.peers[i].address == peer_address: + del self.peers[i] + return + + def MarkPeerAsTrusting(self, peer_address): + with self.peer_list_lock: + for p in self.peers: + if p.address == peer_address: + p.trusting_me = True + break + + def UpdatePeerPerformance(self, peer_address, performance): + with self.peer_list_lock: + for p in self.peers: + if p.address == peer_address: + p.relative_performance = performance + + def CopyToTrusted(self, pubkey_filename): + with open(pubkey_filename, "r") as f: + lines = f.readlines() + fingerprint = lines[-1].strip() + target_filename = self._PubkeyFilename(fingerprint) + shutil.copy(pubkey_filename, target_filename) + with self.peer_list_lock: + for peer in self.peers: + if peer.address == self.ip: continue + if peer.pubkey == fingerprint: + status_handler.ITrustYouNow(peer) + else: + result = self.SignTrusted(fingerprint) + status_handler.NotifyNewTrusted(peer, result) + return fingerprint + + def _PubkeyFilename(self, pubkey_fingerprint): + return os.path.join(self.root, "trusted", "%s.pem" % pubkey_fingerprint) + + def IsTrusted(self, pubkey_fingerprint): + return os.path.exists(self._PubkeyFilename(pubkey_fingerprint)) + + def ListTrusted(self): + path = os.path.join(self.root, "trusted") + if not os.path.exists(path): return [] + return [ f[:-4] for f in os.listdir(path) if f.endswith(".pem") ] + + def SignTrusted(self, pubkey_fingerprint): + if not self.IsTrusted(pubkey_fingerprint): + return [] + filename = self._PubkeyFilename(pubkey_fingerprint) + result = signatures.ReadFileAndSignature(filename) # Format: [key, sig]. + return [pubkey_fingerprint, result[0], result[1], self.pubkey_fingerprint] + + def AcceptNewTrusted(self, data): + # The format of |data| matches the return value of |SignTrusted()|. + if not data: return + fingerprint = data[0] + pubkey = data[1] + signature = data[2] + signer = data[3] + if not self.IsTrusted(signer): + return + if self.IsTrusted(fingerprint): + return # Already trust this guy. + filename = self._PubkeyFilename(fingerprint) + signer_pubkeyfile = self._PubkeyFilename(signer) + if not signatures.VerifySignature(filename, pubkey, signature, + signer_pubkeyfile): + return + return # Nothing more to do. + + def AddPerfData(self, test_key, duration, arch, mode): + data_store = self.perf_data_manager.GetStore(arch, mode) + data_store.RawUpdatePerfData(str(test_key), duration) + + def CompareOwnPerf(self, test, arch, mode): + data_store = self.perf_data_manager.GetStore(arch, mode) + observed = data_store.FetchPerfData(test) + if not observed: return + own_perf_estimate = observed / test.duration + with self.perf_data_lock: + kLearnRateLimiter = 9999 + self.relative_perf *= kLearnRateLimiter + self.relative_perf += own_perf_estimate + self.relative_perf /= (kLearnRateLimiter + 1) diff --git a/src/3rdparty/v8/tools/testrunner/server/presence_handler.py b/src/3rdparty/v8/tools/testrunner/server/presence_handler.py new file mode 100644 index 0000000..1dc2ef1 --- /dev/null +++ b/src/3rdparty/v8/tools/testrunner/server/presence_handler.py @@ -0,0 +1,120 @@ +# Copyright 2012 the V8 project authors. All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +import socket +import SocketServer +import threading +try: + import ujson as json +except: + import json + +from . import constants +from ..objects import peer + + +STARTUP_REQUEST = "V8 test peer starting up" +STARTUP_RESPONSE = "Let's rock some tests!" +EXIT_REQUEST = "V8 testing peer going down" + + +def GetOwnIP(): + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + s.connect(("8.8.8.8", 80)) + ip = s.getsockname()[0] + s.close() + return ip + + +class PresenceHandler(SocketServer.BaseRequestHandler): + + def handle(self): + data = json.loads(self.request[0].strip()) + + if data[0] == STARTUP_REQUEST: + jobs = data[1] + relative_perf = data[2] + pubkey_fingerprint = data[3] + trusted = self.server.daemon.IsTrusted(pubkey_fingerprint) + response = [STARTUP_RESPONSE, self.server.daemon.jobs, + self.server.daemon.relative_perf, + self.server.daemon.pubkey_fingerprint, trusted] + response = json.dumps(response) + self.server.SendTo(self.client_address[0], response) + p = peer.Peer(self.client_address[0], jobs, relative_perf, + pubkey_fingerprint) + p.trusted = trusted + self.server.daemon.AddPeer(p) + + elif data[0] == STARTUP_RESPONSE: + jobs = data[1] + perf = data[2] + pubkey_fingerprint = data[3] + p = peer.Peer(self.client_address[0], jobs, perf, pubkey_fingerprint) + p.trusted = self.server.daemon.IsTrusted(pubkey_fingerprint) + p.trusting_me = data[4] + self.server.daemon.AddPeer(p) + + elif data[0] == EXIT_REQUEST: + self.server.daemon.DeletePeer(self.client_address[0]) + if self.client_address[0] == self.server.daemon.ip: + self.server.shutdown_lock.release() + + +class PresenceDaemon(SocketServer.ThreadingMixIn, SocketServer.UDPServer): + def __init__(self, daemon): + self.daemon = daemon + address = (daemon.ip, constants.PRESENCE_PORT) + SocketServer.UDPServer.__init__(self, address, PresenceHandler) + self.shutdown_lock = threading.Lock() + + def shutdown(self): + self.shutdown_lock.acquire() + self.SendToAll(json.dumps([EXIT_REQUEST])) + self.shutdown_lock.acquire() + self.shutdown_lock.release() + SocketServer.UDPServer.shutdown(self) + + def SendTo(self, target, message): + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + sock.sendto(message, (target, constants.PRESENCE_PORT)) + sock.close() + + def SendToAll(self, message): + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + ip = self.daemon.ip.split(".") + for i in range(1, 254): + ip[-1] = str(i) + sock.sendto(message, (".".join(ip), constants.PRESENCE_PORT)) + sock.close() + + def FindPeers(self): + request = [STARTUP_REQUEST, self.daemon.jobs, self.daemon.relative_perf, + self.daemon.pubkey_fingerprint] + request = json.dumps(request) + self.SendToAll(request) diff --git a/src/3rdparty/v8/tools/testrunner/server/signatures.py b/src/3rdparty/v8/tools/testrunner/server/signatures.py new file mode 100644 index 0000000..9957a18 --- /dev/null +++ b/src/3rdparty/v8/tools/testrunner/server/signatures.py @@ -0,0 +1,63 @@ +# Copyright 2012 the V8 project authors. All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +import base64 +import os +import subprocess + + +def ReadFileAndSignature(filename): + with open(filename, "rb") as f: + file_contents = base64.b64encode(f.read()) + signature_file = filename + ".signature" + if (not os.path.exists(signature_file) or + os.path.getmtime(signature_file) < os.path.getmtime(filename)): + private_key = "~/.ssh/v8_dtest" + code = subprocess.call("openssl dgst -out %s -sign %s %s" % + (signature_file, private_key, filename), + shell=True) + if code != 0: return [None, code] + with open(signature_file) as f: + signature = base64.b64encode(f.read()) + return [file_contents, signature] + + +def VerifySignature(filename, file_contents, signature, pubkeyfile): + with open(filename, "wb") as f: + f.write(base64.b64decode(file_contents)) + signature_file = filename + ".foreign_signature" + with open(signature_file, "wb") as f: + f.write(base64.b64decode(signature)) + code = subprocess.call("openssl dgst -verify %s -signature %s %s" % + (pubkeyfile, signature_file, filename), + shell=True) + matched = (code == 0) + if not matched: + os.remove(signature_file) + os.remove(filename) + return matched diff --git a/src/3rdparty/v8/tools/testrunner/server/status_handler.py b/src/3rdparty/v8/tools/testrunner/server/status_handler.py new file mode 100644 index 0000000..3f2271d --- /dev/null +++ b/src/3rdparty/v8/tools/testrunner/server/status_handler.py @@ -0,0 +1,112 @@ +# Copyright 2012 the V8 project authors. All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +import socket +import SocketServer + +from . import compression +from . import constants + + +def _StatusQuery(peer, query): + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + code = sock.connect_ex((peer.address, constants.STATUS_PORT)) + if code != 0: + # TODO(jkummerow): disconnect (after 3 failures?) + return + compression.Send(query, sock) + compression.Send(constants.END_OF_STREAM, sock) + rec = compression.Receiver(sock) + data = None + while not rec.IsDone(): + data = rec.Current() + assert data[0] == query[0] + data = data[1] + rec.Advance() + sock.close() + return data + + +def RequestTrustedPubkeys(peer, server): + pubkey_list = _StatusQuery(peer, [constants.LIST_TRUSTED_PUBKEYS]) + for pubkey in pubkey_list: + if server.IsTrusted(pubkey): continue + result = _StatusQuery(peer, [constants.GET_SIGNED_PUBKEY, pubkey]) + server.AcceptNewTrusted(result) + + +def NotifyNewTrusted(peer, data): + _StatusQuery(peer, [constants.NOTIFY_NEW_TRUSTED] + data) + + +def ITrustYouNow(peer): + _StatusQuery(peer, [constants.TRUST_YOU_NOW]) + + +def TryTransitiveTrust(peer, pubkey, server): + if _StatusQuery(peer, [constants.DO_YOU_TRUST, pubkey]): + result = _StatusQuery(peer, [constants.GET_SIGNED_PUBKEY, pubkey]) + server.AcceptNewTrusted(result) + + +class StatusHandler(SocketServer.BaseRequestHandler): + def handle(self): + rec = compression.Receiver(self.request) + while not rec.IsDone(): + data = rec.Current() + action = data[0] + + if action == constants.LIST_TRUSTED_PUBKEYS: + response = self.server.daemon.ListTrusted() + compression.Send([action, response], self.request) + + elif action == constants.GET_SIGNED_PUBKEY: + response = self.server.daemon.SignTrusted(data[1]) + compression.Send([action, response], self.request) + + elif action == constants.NOTIFY_NEW_TRUSTED: + self.server.daemon.AcceptNewTrusted(data[1:]) + pass # No response. + + elif action == constants.TRUST_YOU_NOW: + self.server.daemon.MarkPeerAsTrusting(self.client_address[0]) + pass # No response. + + elif action == constants.DO_YOU_TRUST: + response = self.server.daemon.IsTrusted(data[1]) + compression.Send([action, response], self.request) + + rec.Advance() + compression.Send(constants.END_OF_STREAM, self.request) + + +class StatusSocketServer(SocketServer.ThreadingMixIn, SocketServer.TCPServer): + def __init__(self, daemon): + address = (daemon.ip, constants.STATUS_PORT) + SocketServer.TCPServer.__init__(self, address, StatusHandler) + self.daemon = daemon diff --git a/src/3rdparty/v8/tools/testrunner/server/work_handler.py b/src/3rdparty/v8/tools/testrunner/server/work_handler.py new file mode 100644 index 0000000..6bf7d43 --- /dev/null +++ b/src/3rdparty/v8/tools/testrunner/server/work_handler.py @@ -0,0 +1,150 @@ +# Copyright 2012 the V8 project authors. All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +import os +import SocketServer +import stat +import subprocess +import threading + +from . import compression +from . import constants +from . import signatures +from ..network import endpoint +from ..objects import workpacket + + +class WorkHandler(SocketServer.BaseRequestHandler): + + def handle(self): + rec = compression.Receiver(self.request) + while not rec.IsDone(): + data = rec.Current() + with self.server.job_lock: + self._WorkOnWorkPacket(data) + rec.Advance() + + def _WorkOnWorkPacket(self, data): + server_root = self.server.daemon.root + v8_root = os.path.join(server_root, "v8") + os.chdir(v8_root) + packet = workpacket.WorkPacket.Unpack(data) + self.ctx = packet.context + self.ctx.shell_dir = os.path.join("out", + "%s.%s" % (self.ctx.arch, self.ctx.mode)) + if not os.path.isdir(self.ctx.shell_dir): + os.makedirs(self.ctx.shell_dir) + for binary in packet.binaries: + if not self._UnpackBinary(binary, packet.pubkey_fingerprint): + return + + if not self._CheckoutRevision(packet.base_revision): + return + + if not self._ApplyPatch(packet.patch): + return + + tests = packet.tests + endpoint.Execute(v8_root, self.ctx, tests, self.request, self.server.daemon) + self._SendResponse() + + def _SendResponse(self, error_message=None): + try: + if error_message: + compression.Send([[-1, error_message]], self.request) + compression.Send(constants.END_OF_STREAM, self.request) + return + except Exception, e: + pass # Peer is gone. There's nothing we can do. + # Clean up. + self._Call("git checkout -f") + self._Call("git clean -f -d") + self._Call("rm -rf %s" % self.ctx.shell_dir) + + def _UnpackBinary(self, binary, pubkey_fingerprint): + binary_name = binary["name"] + if binary_name == "libv8.so": + libdir = os.path.join(self.ctx.shell_dir, "lib.target") + if not os.path.exists(libdir): os.makedirs(libdir) + target = os.path.join(libdir, binary_name) + else: + target = os.path.join(self.ctx.shell_dir, binary_name) + pubkeyfile = "../trusted/%s.pem" % pubkey_fingerprint + if not signatures.VerifySignature(target, binary["blob"], + binary["sign"], pubkeyfile): + self._SendResponse("Signature verification failed") + return False + os.chmod(target, stat.S_IRWXU) + return True + + def _CheckoutRevision(self, base_svn_revision): + get_hash_cmd = ( + "git log -1 --format=%%H --remotes --grep='^git-svn-id:.*@%s'" % + base_svn_revision) + try: + base_revision = subprocess.check_output(get_hash_cmd, shell=True) + if not base_revision: raise ValueError + except: + self._Call("git fetch") + try: + base_revision = subprocess.check_output(get_hash_cmd, shell=True) + if not base_revision: raise ValueError + except: + self._SendResponse("Base revision not found.") + return False + code = self._Call("git checkout -f %s" % base_revision) + if code != 0: + self._SendResponse("Error trying to check out base revision.") + return False + code = self._Call("git clean -f -d") + if code != 0: + self._SendResponse("Failed to reset checkout") + return False + return True + + def _ApplyPatch(self, patch): + if not patch: return True # Just skip if the patch is empty. + patchfilename = "_dtest_incoming_patch.patch" + with open(patchfilename, "w") as f: + f.write(patch) + code = self._Call("git apply %s" % patchfilename) + if code != 0: + self._SendResponse("Error applying patch.") + return False + return True + + def _Call(self, cmd): + return subprocess.call(cmd, shell=True) + + +class WorkSocketServer(SocketServer.ThreadingMixIn, SocketServer.TCPServer): + def __init__(self, daemon): + address = (daemon.ip, constants.PEER_PORT) + SocketServer.TCPServer.__init__(self, address, WorkHandler) + self.job_lock = threading.Lock() + self.daemon = daemon |