diff options
Diffstat (limited to 'src/3rdparty/v8/tools')
55 files changed, 5904 insertions, 232 deletions
diff --git a/src/3rdparty/v8/tools/android-build.sh b/src/3rdparty/v8/tools/android-build.sh new file mode 100755 index 0000000..e69de29 --- /dev/null +++ b/src/3rdparty/v8/tools/android-build.sh diff --git a/src/3rdparty/v8/tools/android-ll-prof.sh b/src/3rdparty/v8/tools/android-ll-prof.sh new file mode 100755 index 0000000..436f262 --- /dev/null +++ b/src/3rdparty/v8/tools/android-ll-prof.sh @@ -0,0 +1,69 @@ +#!/bin/bash +# Copyright 2012 the V8 project authors. All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# Runs d8 with the given arguments on the device under 'perf' and +# processes the profiler trace and v8 logs using ll_prof.py. +# +# Usage: +# > ./tools/android-ll-prof.sh (debug|release) "args to d8" "args to ll_prof.py" +# +# The script creates deploy directory deploy/data/local/tmp/v8, copies there +# the d8 binary either from out/android_arm.release or out/android_arm.debug, +# and then sync the deploy directory with /data/local/tmp/v8 on the device. +# You can put JS files in the deploy directory before running the script. +# Note: $ANDROID_NDK_ROOT must be set. + +MODE=$1 +RUN_ARGS=$2 +LL_PROF_ARGS=$3 + +BASE=`cd $(dirname "$0")/..; pwd` +DEPLOY="$BASE/deploy" + +set +e +mkdir -p "$DEPLOY/data/local/tmp/v8" + +cp "$BASE/out/android_arm.$MODE/d8" "$DEPLOY/data/local/tmp/v8/d8" + +adb -p "$DEPLOY" sync data + +adb shell "cd /data/local/tmp/v8;\ + perf record -R -e cycles -c 10000 -f -i \ + ./d8 --ll_prof --gc-fake-mmap=/data/local/tmp/__v8_gc__ $RUN_ARGS" + +adb pull /data/local/tmp/v8/v8.log . +adb pull /data/local/tmp/v8/v8.log.ll . +adb pull /data/perf.data . + +ARCH=arm-linux-androideabi-4.6 +TOOLCHAIN="${ANDROID_NDK_ROOT}/toolchains/$ARCH/prebuilt/linux-x86/bin" + +$BASE/tools/ll_prof.py --host-root="$BASE/deploy" \ + --gc-fake-mmap=/data/local/tmp/__v8_gc__ \ + --objdump="$TOOLCHAIN/arm-linux-androideabi-objdump" \ + $LL_PROF_ARGS diff --git a/src/3rdparty/v8/tools/android-run.py b/src/3rdparty/v8/tools/android-run.py new file mode 100755 index 0000000..1693c5b --- /dev/null +++ b/src/3rdparty/v8/tools/android-run.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python +# +# Copyright 2012 the V8 project authors. All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# This script executes the passed command line on Android device +# using 'adb shell' command. Unfortunately, 'adb shell' always +# returns exit code 0, ignoring the exit code of executed command. +# Since we need to return non-zero exit code if the command failed, +# we augment the passed command line with exit code checking statement +# and output special error string in case of non-zero exit code. +# Then we parse the output of 'adb shell' and look for that error string. + +import os +from os.path import join, dirname, abspath +import subprocess +import sys +import tempfile + +def Check(output, errors): + failed = any([s.startswith('/system/bin/sh:') or s.startswith('ANDROID') + for s in output.split('\n')]) + return 1 if failed else 0 + +def Execute(cmdline): + (fd_out, outname) = tempfile.mkstemp() + (fd_err, errname) = tempfile.mkstemp() + process = subprocess.Popen( + args=cmdline, + shell=True, + stdout=fd_out, + stderr=fd_err, + ) + exit_code = process.wait() + os.close(fd_out) + os.close(fd_err) + output = file(outname).read() + errors = file(errname).read() + os.unlink(outname) + os.unlink(errname) + sys.stdout.write(output) + sys.stderr.write(errors) + return exit_code or Check(output, errors) + +def Escape(arg): + def ShouldEscape(): + for x in arg: + if not x.isalnum() and x != '-' and x != '_': + return True + return False + + return arg if not ShouldEscape() else '"%s"' % (arg.replace('"', '\\"')) + +def WriteToTemporaryFile(data): + (fd, fname) = tempfile.mkstemp() + os.close(fd) + tmp_file = open(fname, "w") + tmp_file.write(data) + tmp_file.close() + return fname + +def Main(): + if (len(sys.argv) == 1): + print("Usage: %s <command-to-run-on-device>" % sys.argv[0]) + return 1 + workspace = abspath(join(dirname(sys.argv[0]), '..')) + android_workspace = os.getenv("ANDROID_V8", "/data/local/v8") + args = [Escape(arg) for arg in sys.argv[1:]] + script = (" ".join(args) + "\n" + "case $? in\n" + " 0) ;;\n" + " *) echo \"ANDROID: Error returned by test\";;\n" + "esac\n") + script = script.replace(workspace, android_workspace) + script_file = WriteToTemporaryFile(script) + android_script_file = android_workspace + "/" + script_file + command = ("adb push '%s' %s;" % (script_file, android_script_file) + + "adb shell 'sh %s';" % android_script_file + + "adb shell 'rm %s'" % android_script_file) + error_code = Execute(command) + os.unlink(script_file) + return error_code + +if __name__ == '__main__': + sys.exit(Main()) diff --git a/src/3rdparty/v8/tools/android-sync.sh b/src/3rdparty/v8/tools/android-sync.sh new file mode 100755 index 0000000..5d4ef2e --- /dev/null +++ b/src/3rdparty/v8/tools/android-sync.sh @@ -0,0 +1,105 @@ +#!/bin/bash +# Copyright 2012 the V8 project authors. All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# This script pushes android binaries and test data to the device. +# The first argument can be either "android.release" or "android.debug". +# The second argument is a relative path to the output directory with binaries. +# The third argument is the absolute path to the V8 directory on the host. +# The fourth argument is the absolute path to the V8 directory on the device. + +if [ ${#@} -lt 4 ] ; then + echo "$0: Error: need 4 arguments" + exit 1 +fi + +ARCH_MODE=$1 +OUTDIR=$2 +HOST_V8=$3 +ANDROID_V8=$4 + +function LINUX_MD5 { + local HASH=$(md5sum $1) + echo ${HASH%% *} +} + +function DARWIN_MD5 { + local HASH=$(md5 $1) + echo ${HASH} | cut -f2 -d "=" | cut -f2 -d " " +} + +host_os=$(uname -s) +case "${host_os}" in + "Linux") + MD5=LINUX_MD5 + ;; + "Darwin") + MD5=DARWIN_MD5 + ;; + *) + echo "$0: Host platform ${host_os} is not supported" >& 2 + exit 1 +esac + +function sync_file { + local FILE=$1 + local ANDROID_HASH=$(adb shell "md5 \"$ANDROID_V8/$FILE\"") + local HOST_HASH=$($MD5 "$HOST_V8/$FILE") + if [ "${ANDROID_HASH%% *}" != "${HOST_HASH}" ]; then + adb push "$HOST_V8/$FILE" "$ANDROID_V8/$FILE" &> /dev/null + fi + echo -n "." +} + +function sync_dir { + local DIR=$1 + echo -n "sync to $ANDROID_V8/$DIR" + for FILE in $(find "$HOST_V8/$DIR" -not -path "*.svn*" -type f); do + local RELATIVE_FILE=${FILE:${#HOST_V8}} + sync_file "$RELATIVE_FILE" + done + echo "" +} + +echo -n "sync to $ANDROID_V8/$OUTDIR/$ARCH_MODE" +sync_file "$OUTDIR/$ARCH_MODE/cctest" +sync_file "$OUTDIR/$ARCH_MODE/d8" +sync_file "$OUTDIR/$ARCH_MODE/preparser" +echo "" +echo -n "sync to $ANDROID_V8/tools" +sync_file tools/consarray.js +sync_file tools/codemap.js +sync_file tools/csvparser.js +sync_file tools/profile.js +sync_file tools/splaytree.js +sync_file tools/profile_view.js +sync_file tools/logreader.js +sync_file tools/tickprocessor.js +echo "" +sync_dir test/message +sync_dir test/mjsunit +sync_dir test/preparser diff --git a/src/3rdparty/v8/tools/bash-completion.sh b/src/3rdparty/v8/tools/bash-completion.sh index 9f65c67..9f65c67 100644..100755 --- a/src/3rdparty/v8/tools/bash-completion.sh +++ b/src/3rdparty/v8/tools/bash-completion.sh diff --git a/src/3rdparty/v8/tools/check-static-initializers.sh b/src/3rdparty/v8/tools/check-static-initializers.sh index 1103a97..1103a97 100644..100755 --- a/src/3rdparty/v8/tools/check-static-initializers.sh +++ b/src/3rdparty/v8/tools/check-static-initializers.sh diff --git a/src/3rdparty/v8/tools/common-includes.sh b/src/3rdparty/v8/tools/common-includes.sh index 2b806ca..7785e9f 100644 --- a/src/3rdparty/v8/tools/common-includes.sh +++ b/src/3rdparty/v8/tools/common-includes.sh @@ -36,9 +36,7 @@ TEMP_BRANCH=$BRANCHNAME-temporary-branch-created-by-script VERSION_FILE="src/version.cc" CHANGELOG_ENTRY_FILE="$PERSISTFILE_BASENAME-changelog-entry" PATCH_FILE="$PERSISTFILE_BASENAME-patch" -PATCH_OUTPUT_FILE="$PERSISTFILE_BASENAME-patch-output" COMMITMSG_FILE="$PERSISTFILE_BASENAME-commitmsg" -TOUCHED_FILES_FILE="$PERSISTFILE_BASENAME-touched-files" TRUNK_REVISION_FILE="$PERSISTFILE_BASENAME-trunkrevision" START_STEP=0 CURRENT_STEP=0 @@ -180,26 +178,21 @@ the uploaded CL." done } -# Takes a file containing the patch to apply as first argument. -apply_patch() { - patch $REVERSE_PATCH -p1 < "$1" > "$PATCH_OUTPUT_FILE" || \ - { cat "$PATCH_OUTPUT_FILE" && die "Applying the patch failed."; } - tee < "$PATCH_OUTPUT_FILE" >(grep "patching file" \ - | awk '{print $NF}' >> "$TOUCHED_FILES_FILE") - rm "$PATCH_OUTPUT_FILE" +wait_for_resolving_conflicts() { + echo "Applying the patch \"$1\" failed. Either type \"ABORT<Return>\", or \ +resolve the conflicts, stage *all* touched files with 'git add', and \ +type \"RESOLVED<Return>\"" + unset ANSWER + while [ "$ANSWER" != "RESOLVED" ] ; do + [[ "$ANSWER" == "ABORT" ]] && die "Applying the patch failed." + [[ -n "$ANSWER" ]] && echo "That was not 'RESOLVED' or 'ABORT'." + echo -n "> " + read ANSWER + done } -stage_files() { - # Stage added and modified files. - TOUCHED_FILES=$(cat "$TOUCHED_FILES_FILE") - for FILE in $TOUCHED_FILES ; do - git add "$FILE" - done - # Stage deleted files. - DELETED_FILES=$(git status -s -uno --porcelain | grep "^ D" \ - | awk '{print $NF}') - for FILE in $DELETED_FILES ; do - git rm "$FILE" - done - rm -f "$TOUCHED_FILES_FILE" +# Takes a file containing the patch to apply as first argument. +apply_patch() { + git apply --index --reject $REVERSE_PATCH "$1" || \ + wait_for_resolving_conflicts "$1"; } diff --git a/src/3rdparty/v8/tools/fuzz-harness.sh b/src/3rdparty/v8/tools/fuzz-harness.sh new file mode 100755 index 0000000..efbf864 --- /dev/null +++ b/src/3rdparty/v8/tools/fuzz-harness.sh @@ -0,0 +1,92 @@ +#!/bin/bash +# Copyright 2012 the V8 project authors. All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# A simple harness that downloads and runs 'jsfunfuzz' against d8. This +# takes a long time because it runs many iterations and is intended for +# automated usage. The package containing 'jsfunfuzz' can be found as an +# attachment to this bug: +# https://bugzilla.mozilla.org/show_bug.cgi?id=jsfunfuzz + +JSFUNFUZZ_URL="https://bugzilla.mozilla.org/attachment.cgi?id=310631" +JSFUNFUZZ_MD5="d0e497201c5cd7bffbb1cdc1574f4e32" + +v8_root=$(readlink -f $(dirname $BASH_SOURCE)/../) + +if [ -n "$1" ]; then + d8="${v8_root}/$1" +else + d8="${v8_root}/d8" +fi + +if [ ! -f "$d8" ]; then + echo "Failed to find d8 binary: $d8" + exit 1 +fi + +jsfunfuzz_file="$v8_root/tools/jsfunfuzz.zip" +if [ ! -f "$jsfunfuzz_file" ]; then + echo "Downloading $jsfunfuzz_file ..." + wget -q -O "$jsfunfuzz_file" $JSFUNFUZZ_URL || exit 1 +fi + +jsfunfuzz_sum=$(md5sum "$jsfunfuzz_file" | awk '{ print $1 }') +if [ $jsfunfuzz_sum != $JSFUNFUZZ_MD5 ]; then + echo "Failed to verify checksum!" + exit 1 +fi + +jsfunfuzz_dir="$v8_root/tools/jsfunfuzz" +if [ ! -d "$jsfunfuzz_dir" ]; then + echo "Unpacking into $jsfunfuzz_dir ..." + unzip "$jsfunfuzz_file" -d "$jsfunfuzz_dir" || exit 1 + echo "Patching runner ..." + cat << EOF | patch -s -p0 -d "$v8_root" +--- tools/jsfunfuzz/jsfunfuzz/multi_timed_run.py~ ++++ tools/jsfunfuzz/jsfunfuzz/multi_timed_run.py +@@ -125,7 +125,7 @@ + + def many_timed_runs(): + iteration = 0 +- while True: ++ while iteration < 100: + iteration += 1 + logfilename = "w%d" % iteration + one_timed_run(logfilename) +EOF +fi + +flags='--debug-code --expose-gc --verify-gc' +python -u "$jsfunfuzz_dir/jsfunfuzz/multi_timed_run.py" 300 \ + "$d8" $flags "$jsfunfuzz_dir/jsfunfuzz/jsfunfuzz.js" +exit_code=$(cat w* | grep " looking good" -c) +exit_code=$((100-exit_code)) +tar -cjf fuzz-results-$(date +%y%m%d).tar.bz2 err-* w* +rm -f err-* w* + +echo "Total failures: $exit_code" +exit $exit_code diff --git a/src/3rdparty/v8/tools/gen-postmortem-metadata.py b/src/3rdparty/v8/tools/gen-postmortem-metadata.py index b9b1625..71f58bf 100644 --- a/src/3rdparty/v8/tools/gen-postmortem-metadata.py +++ b/src/3rdparty/v8/tools/gen-postmortem-metadata.py @@ -61,7 +61,7 @@ consts_misc = [ { 'name': 'StringEncodingMask', 'value': 'kStringEncodingMask' }, { 'name': 'TwoByteStringTag', 'value': 'kTwoByteStringTag' }, - { 'name': 'AsciiStringTag', 'value': 'kAsciiStringTag' }, + { 'name': 'AsciiStringTag', 'value': 'kOneByteStringTag' }, { 'name': 'StringRepresentationMask', 'value': 'kStringRepresentationMask' }, @@ -78,8 +78,8 @@ consts_misc = [ { 'name': 'SmiValueShift', 'value': 'kSmiTagSize' }, { 'name': 'PointerSizeLog2', 'value': 'kPointerSizeLog2' }, - { 'name': 'prop_idx_content', - 'value': 'DescriptorArray::kContentArrayIndex' }, + { 'name': 'prop_idx_transitions', + 'value': 'DescriptorArray::kTransitionsIndex' }, { 'name': 'prop_idx_first', 'value': 'DescriptorArray::kFirstIndex' }, { 'name': 'prop_type_field', diff --git a/src/3rdparty/v8/tools/grokdump.py b/src/3rdparty/v8/tools/grokdump.py index 29d4755..46ead5e 100755 --- a/src/3rdparty/v8/tools/grokdump.py +++ b/src/3rdparty/v8/tools/grokdump.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Copyright 2011 the V8 project authors. All rights reserved. +# Copyright 2012 the V8 project authors. All rights reserved. # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: @@ -27,32 +27,31 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +import bisect import cmd +import codecs import ctypes +import disasm import mmap import optparse import os -import disasm -import sys -import types -import codecs import re import struct +import sys +import types -USAGE="""usage: %prog [OPTION]... +USAGE="""usage: %prog [OPTIONS] [DUMP-FILE] Minidump analyzer. Shows the processor state at the point of exception including the stack of the active thread and the referenced objects in the V8 heap. Code objects are disassembled and the addresses linked from the -stack (pushed return addresses) are marked with "=>". - +stack (e.g. pushed return addresses) are marked with "=>". Examples: - $ %prog 12345678-1234-1234-1234-123456789abcd-full.dmp -""" + $ %prog 12345678-1234-1234-1234-123456789abcd-full.dmp""" DEBUG=False @@ -108,21 +107,59 @@ class Descriptor(object): return Raw -def do_dump(reader, heap): +def FullDump(reader, heap): """Dump all available memory regions.""" def dump_region(reader, start, size, location): - print "%s - %s" % (reader.FormatIntPtr(start), - reader.FormatIntPtr(start + size)) - for slot in xrange(start, - start + size, - reader.PointerSize()): - maybe_address = reader.ReadUIntPtr(slot) - heap_object = heap.FindObject(maybe_address) - print "%s: %s" % (reader.FormatIntPtr(slot), - reader.FormatIntPtr(maybe_address)) - if heap_object: - heap_object.Print(Printer()) - print + print + while start & 3 != 0: + start += 1 + size -= 1 + location += 1 + is_executable = reader.IsProbableExecutableRegion(location, size) + is_ascii = reader.IsProbableASCIIRegion(location, size) + + if is_executable is not False: + lines = reader.GetDisasmLines(start, size) + for line in lines: + print FormatDisasmLine(start, heap, line) + print + + if is_ascii is not False: + # Output in the same format as the Unix hd command + addr = start + for slot in xrange(location, location + size, 16): + hex_line = "" + asc_line = "" + for i in xrange(0, 16): + if slot + i < location + size: + byte = ctypes.c_uint8.from_buffer(reader.minidump, slot + i).value + if byte >= 0x20 and byte < 0x7f: + asc_line += chr(byte) + else: + asc_line += "." + hex_line += " %02x" % (byte) + else: + hex_line += " " + if i == 7: + hex_line += " " + print "%s %s |%s|" % (reader.FormatIntPtr(addr), + hex_line, + asc_line) + addr += 16 + + if is_executable is not True and is_ascii is not True: + print "%s - %s" % (reader.FormatIntPtr(start), + reader.FormatIntPtr(start + size)) + for slot in xrange(start, + start + size, + reader.PointerSize()): + maybe_address = reader.ReadUIntPtr(slot) + heap_object = heap.FindObject(maybe_address) + print "%s: %s" % (reader.FormatIntPtr(slot), + reader.FormatIntPtr(maybe_address)) + if heap_object: + heap_object.Print(Printer()) + print reader.ForEachMemoryRegion(dump_region) @@ -144,6 +181,11 @@ MINIDUMP_LOCATION_DESCRIPTOR = Descriptor([ ("rva", ctypes.c_uint32) ]) +MINIDUMP_STRING = Descriptor([ + ("length", ctypes.c_uint32), + ("buffer", lambda t: ctypes.c_uint8 * (t.length + 2)) +]) + MINIDUMP_DIRECTORY = Descriptor([ ("stream_type", ctypes.c_uint32), ("location", MINIDUMP_LOCATION_DESCRIPTOR.ctype) @@ -364,6 +406,24 @@ MINIDUMP_THREAD_LIST = Descriptor([ ("threads", lambda t: MINIDUMP_THREAD.ctype * t.thread_count) ]) +MINIDUMP_RAW_MODULE = Descriptor([ + ("base_of_image", ctypes.c_uint64), + ("size_of_image", ctypes.c_uint32), + ("checksum", ctypes.c_uint32), + ("time_date_stamp", ctypes.c_uint32), + ("module_name_rva", ctypes.c_uint32), + ("version_info", ctypes.c_uint32 * 13), + ("cv_record", MINIDUMP_LOCATION_DESCRIPTOR.ctype), + ("misc_record", MINIDUMP_LOCATION_DESCRIPTOR.ctype), + ("reserved0", ctypes.c_uint32 * 2), + ("reserved1", ctypes.c_uint32 * 2) +]) + +MINIDUMP_MODULE_LIST = Descriptor([ + ("number_of_modules", ctypes.c_uint32), + ("modules", lambda t: MINIDUMP_RAW_MODULE.ctype * t.number_of_modules) +]) + MINIDUMP_RAW_SYSTEM_INFO = Descriptor([ ("processor_architecture", ctypes.c_uint16) ]) @@ -371,6 +431,20 @@ MINIDUMP_RAW_SYSTEM_INFO = Descriptor([ MD_CPU_ARCHITECTURE_X86 = 0 MD_CPU_ARCHITECTURE_AMD64 = 9 +class FuncSymbol: + def __init__(self, start, size, name): + self.start = start + self.end = self.start + size + self.name = name + + def __cmp__(self, other): + if isinstance(other, FuncSymbol): + return self.start - other.start + return self.start - other + + def Covers(self, addr): + return (self.start <= addr) and (addr < self.end) + class MinidumpReader(object): """Minidump (.dmp) reader.""" @@ -382,7 +456,7 @@ class MinidumpReader(object): self.minidump = mmap.mmap(self.minidump_file.fileno(), 0, mmap.MAP_PRIVATE) self.header = MINIDUMP_HEADER.Read(self.minidump, 0) if self.header.signature != MinidumpReader._HEADER_MAGIC: - print >>sys.stderr, "Warning: unsupported minidump header magic" + print >>sys.stderr, "Warning: Unsupported minidump header magic!" DebugPrint(self.header) directories = [] offset = self.header.stream_directories_rva @@ -394,8 +468,13 @@ class MinidumpReader(object): self.exception_context = None self.memory_list = None self.memory_list64 = None + self.module_list = None self.thread_map = {} + self.symdir = options.symdir + self.modules_with_symbols = [] + self.symbols = [] + # Find MDRawSystemInfo stream and determine arch. for d in directories: if d.stream_type == MD_SYSTEM_INFO_STREAM: @@ -425,8 +504,13 @@ class MinidumpReader(object): for thread in thread_list.threads: DebugPrint(thread) self.thread_map[thread.id] = thread + elif d.stream_type == MD_MODULE_LIST_STREAM: + assert self.module_list is None + self.module_list = MINIDUMP_MODULE_LIST.Read( + self.minidump, d.location.rva) + assert ctypes.sizeof(self.module_list) == d.location.data_size elif d.stream_type == MD_MEMORY_LIST_STREAM: - print >>sys.stderr, "Warning: not a full minidump" + print >>sys.stderr, "Warning: This is not a full minidump!" assert self.memory_list is None self.memory_list = MINIDUMP_MEMORY_LIST.Read( self.minidump, d.location.rva) @@ -470,6 +554,64 @@ class MinidumpReader(object): elif self.arch == MD_CPU_ARCHITECTURE_X86: return ctypes.c_uint32.from_buffer(self.minidump, location).value + def IsProbableASCIIRegion(self, location, length): + ascii_bytes = 0 + non_ascii_bytes = 0 + for loc in xrange(location, location + length): + byte = ctypes.c_uint8.from_buffer(self.minidump, loc).value + if byte >= 0x7f: + non_ascii_bytes += 1 + if byte < 0x20 and byte != 0: + non_ascii_bytes += 1 + if byte < 0x7f and byte >= 0x20: + ascii_bytes += 1 + if byte == 0xa: # newline + ascii_bytes += 1 + if ascii_bytes * 10 <= length: + return False + if length > 0 and ascii_bytes > non_ascii_bytes * 7: + return True + if ascii_bytes > non_ascii_bytes * 3: + return None # Maybe + return False + + def IsProbableExecutableRegion(self, location, length): + opcode_bytes = 0 + sixty_four = self.arch == MD_CPU_ARCHITECTURE_AMD64 + for loc in xrange(location, location + length): + byte = ctypes.c_uint8.from_buffer(self.minidump, loc).value + if (byte == 0x8b or # mov + byte == 0x89 or # mov reg-reg + (byte & 0xf0) == 0x50 or # push/pop + (sixty_four and (byte & 0xf0) == 0x40) or # rex prefix + byte == 0xc3 or # return + byte == 0x74 or # jeq + byte == 0x84 or # jeq far + byte == 0x75 or # jne + byte == 0x85 or # jne far + byte == 0xe8 or # call + byte == 0xe9 or # jmp far + byte == 0xeb): # jmp near + opcode_bytes += 1 + opcode_percent = (opcode_bytes * 100) / length + threshold = 20 + if opcode_percent > threshold + 2: + return True + if opcode_percent > threshold - 2: + return None # Maybe + return False + + def FindRegion(self, addr): + answer = [-1, -1] + def is_in(reader, start, size, location): + if addr >= start and addr < start + size: + answer[0] = start + answer[1] = size + self.ForEachMemoryRegion(is_in) + if answer[0] == -1: + return None + return answer + def ForEachMemoryRegion(self, cb): if self.memory_list64 is not None: for r in self.memory_list64.ranges: @@ -481,14 +623,14 @@ class MinidumpReader(object): for r in self.memory_list.ranges: cb(self, r.start, r.memory.data_size, r.memory.rva) - def FindWord(self, word): + def FindWord(self, word, alignment=0): def search_inside_region(reader, start, size, location): - for loc in xrange(location, location + size): + location = (location + alignment) & ~alignment + for loc in xrange(location, location + size - self.PointerSize()): if reader._ReadWord(loc) == word: slot = start + (loc - location) print "%s: %s" % (reader.FormatIntPtr(slot), reader.FormatIntPtr(word)) - self.ForEachMemoryRegion(search_inside_region) def FindLocation(self, address): @@ -550,6 +692,66 @@ class MinidumpReader(object): def Register(self, name): return self.exception_context.__getattribute__(name) + def ReadMinidumpString(self, rva): + string = bytearray(MINIDUMP_STRING.Read(self.minidump, rva).buffer) + string = string.decode("utf16") + return string[0:len(string) - 1] + + # Load FUNC records from a BreakPad symbol file + # + # http://code.google.com/p/google-breakpad/wiki/SymbolFiles + # + def _LoadSymbolsFrom(self, symfile, baseaddr): + print "Loading symbols from %s" % (symfile) + funcs = [] + with open(symfile) as f: + for line in f: + result = re.match( + r"^FUNC ([a-f0-9]+) ([a-f0-9]+) ([a-f0-9]+) (.*)$", line) + if result is not None: + start = int(result.group(1), 16) + size = int(result.group(2), 16) + name = result.group(4).rstrip() + bisect.insort_left(self.symbols, + FuncSymbol(baseaddr + start, size, name)) + print " ... done" + + def TryLoadSymbolsFor(self, modulename, module): + try: + symfile = os.path.join(self.symdir, + modulename.replace('.', '_') + ".pdb.sym") + self._LoadSymbolsFrom(symfile, module.base_of_image) + self.modules_with_symbols.append(module) + except Exception as e: + print " ... failure (%s)" % (e) + + # Returns true if address is covered by some module that has loaded symbols. + def _IsInModuleWithSymbols(self, addr): + for module in self.modules_with_symbols: + start = module.base_of_image + end = start + module.size_of_image + if (start <= addr) and (addr < end): + return True + return False + + # Find symbol covering the given address and return its name in format + # <symbol name>+<offset from the start> + def FindSymbol(self, addr): + if not self._IsInModuleWithSymbols(addr): + return None + + i = bisect.bisect_left(self.symbols, addr) + symbol = None + if (0 < i) and self.symbols[i - 1].Covers(addr): + symbol = self.symbols[i - 1] + elif (i < len(self.symbols)) and self.symbols[i].Covers(addr): + symbol = self.symbols[i] + else: + return None + diff = addr - symbol.start + return "%s+0x%x" % (symbol.name, diff) + + # List of V8 instance types. Obtained by adding the code below to any .cc file. # @@ -614,24 +816,182 @@ INSTANCE_TYPES = { 156: "SCRIPT_TYPE", 157: "CODE_CACHE_TYPE", 158: "POLYMORPHIC_CODE_CACHE_TYPE", - 161: "FIXED_ARRAY_TYPE", + 159: "TYPE_FEEDBACK_INFO_TYPE", + 160: "ALIASED_ARGUMENTS_ENTRY_TYPE", + 163: "FIXED_ARRAY_TYPE", 145: "FIXED_DOUBLE_ARRAY_TYPE", - 162: "SHARED_FUNCTION_INFO_TYPE", - 163: "JS_MESSAGE_OBJECT_TYPE", - 166: "JS_VALUE_TYPE", - 167: "JS_OBJECT_TYPE", - 168: "JS_CONTEXT_EXTENSION_OBJECT_TYPE", - 169: "JS_GLOBAL_OBJECT_TYPE", - 170: "JS_BUILTINS_OBJECT_TYPE", - 171: "JS_GLOBAL_PROXY_TYPE", - 172: "JS_ARRAY_TYPE", - 165: "JS_PROXY_TYPE", - 175: "JS_WEAK_MAP_TYPE", - 176: "JS_REGEXP_TYPE", - 177: "JS_FUNCTION_TYPE", - 164: "JS_FUNCTION_PROXY_TYPE", - 159: "DEBUG_INFO_TYPE", - 160: "BREAK_POINT_INFO_TYPE", + 164: "SHARED_FUNCTION_INFO_TYPE", + 165: "JS_MESSAGE_OBJECT_TYPE", + 168: "JS_VALUE_TYPE", + 169: "JS_DATE_TYPE", + 170: "JS_OBJECT_TYPE", + 171: "JS_CONTEXT_EXTENSION_OBJECT_TYPE", + 172: "JS_MODULE_TYPE", + 173: "JS_GLOBAL_OBJECT_TYPE", + 174: "JS_BUILTINS_OBJECT_TYPE", + 175: "JS_GLOBAL_PROXY_TYPE", + 176: "JS_ARRAY_TYPE", + 167: "JS_PROXY_TYPE", + 179: "JS_WEAK_MAP_TYPE", + 180: "JS_REGEXP_TYPE", + 181: "JS_FUNCTION_TYPE", + 166: "JS_FUNCTION_PROXY_TYPE", + 161: "DEBUG_INFO_TYPE", + 162: "BREAK_POINT_INFO_TYPE", +} + + +# List of known V8 maps. Used to determine the instance type and name +# for maps that are part of the root-set and hence on the first page of +# the map-space. Obtained by adding the code below to an IA32 release +# build with enabled snapshots to the end of the Isolate::Init method. +# +# #define ROOT_LIST_CASE(type, name, camel_name) \ +# if (o == heap_.name()) n = #camel_name; +# #define STRUCT_LIST_CASE(upper_name, camel_name, name) \ +# if (o == heap_.name##_map()) n = #camel_name "Map"; +# HeapObjectIterator it(heap_.map_space()); +# printf("KNOWN_MAPS = {\n"); +# for (Object* o = it.Next(); o != NULL; o = it.Next()) { +# Map* m = Map::cast(o); +# const char* n = ""; +# intptr_t p = reinterpret_cast<intptr_t>(m) & 0xfffff; +# int t = m->instance_type(); +# ROOT_LIST(ROOT_LIST_CASE) +# STRUCT_LIST(STRUCT_LIST_CASE) +# printf(" 0x%05x: (%d, \"%s\"),\n", p, t, n); +# } +# printf("}\n"); +KNOWN_MAPS = { + 0x08081: (134, "ByteArrayMap"), + 0x080a1: (128, "MetaMap"), + 0x080c1: (130, "OddballMap"), + 0x080e1: (163, "FixedArrayMap"), + 0x08101: (68, "AsciiSymbolMap"), + 0x08121: (132, "HeapNumberMap"), + 0x08141: (135, "FreeSpaceMap"), + 0x08161: (146, "OnePointerFillerMap"), + 0x08181: (146, "TwoPointerFillerMap"), + 0x081a1: (131, "GlobalPropertyCellMap"), + 0x081c1: (164, "SharedFunctionInfoMap"), + 0x081e1: (4, "AsciiStringMap"), + 0x08201: (163, "GlobalContextMap"), + 0x08221: (129, "CodeMap"), + 0x08241: (163, "ScopeInfoMap"), + 0x08261: (163, "FixedCOWArrayMap"), + 0x08281: (145, "FixedDoubleArrayMap"), + 0x082a1: (163, "HashTableMap"), + 0x082c1: (0, "StringMap"), + 0x082e1: (64, "SymbolMap"), + 0x08301: (1, "ConsStringMap"), + 0x08321: (5, "ConsAsciiStringMap"), + 0x08341: (3, "SlicedStringMap"), + 0x08361: (7, "SlicedAsciiStringMap"), + 0x08381: (65, "ConsSymbolMap"), + 0x083a1: (69, "ConsAsciiSymbolMap"), + 0x083c1: (66, "ExternalSymbolMap"), + 0x083e1: (74, "ExternalSymbolWithAsciiDataMap"), + 0x08401: (70, "ExternalAsciiSymbolMap"), + 0x08421: (2, "ExternalStringMap"), + 0x08441: (10, "ExternalStringWithAsciiDataMap"), + 0x08461: (6, "ExternalAsciiStringMap"), + 0x08481: (82, "ShortExternalSymbolMap"), + 0x084a1: (90, "ShortExternalSymbolWithAsciiDataMap"), + 0x084c1: (86, "ShortExternalAsciiSymbolMap"), + 0x084e1: (18, "ShortExternalStringMap"), + 0x08501: (26, "ShortExternalStringWithAsciiDataMap"), + 0x08521: (22, "ShortExternalAsciiStringMap"), + 0x08541: (0, "UndetectableStringMap"), + 0x08561: (4, "UndetectableAsciiStringMap"), + 0x08581: (144, "ExternalPixelArrayMap"), + 0x085a1: (136, "ExternalByteArrayMap"), + 0x085c1: (137, "ExternalUnsignedByteArrayMap"), + 0x085e1: (138, "ExternalShortArrayMap"), + 0x08601: (139, "ExternalUnsignedShortArrayMap"), + 0x08621: (140, "ExternalIntArrayMap"), + 0x08641: (141, "ExternalUnsignedIntArrayMap"), + 0x08661: (142, "ExternalFloatArrayMap"), + 0x08681: (143, "ExternalDoubleArrayMap"), + 0x086a1: (163, "NonStrictArgumentsElementsMap"), + 0x086c1: (163, "FunctionContextMap"), + 0x086e1: (163, "CatchContextMap"), + 0x08701: (163, "WithContextMap"), + 0x08721: (163, "BlockContextMap"), + 0x08741: (163, "ModuleContextMap"), + 0x08761: (165, "JSMessageObjectMap"), + 0x08781: (133, "ForeignMap"), + 0x087a1: (170, "NeanderMap"), + 0x087c1: (158, "PolymorphicCodeCacheMap"), + 0x087e1: (156, "ScriptMap"), + 0x08801: (147, "AccessorInfoMap"), + 0x08821: (148, "AccessorPairMap"), + 0x08841: (149, "AccessCheckInfoMap"), + 0x08861: (150, "InterceptorInfoMap"), + 0x08881: (151, "CallHandlerInfoMap"), + 0x088a1: (152, "FunctionTemplateInfoMap"), + 0x088c1: (153, "ObjectTemplateInfoMap"), + 0x088e1: (154, "SignatureInfoMap"), + 0x08901: (155, "TypeSwitchInfoMap"), + 0x08921: (157, "CodeCacheMap"), + 0x08941: (159, "TypeFeedbackInfoMap"), + 0x08961: (160, "AliasedArgumentsEntryMap"), + 0x08981: (161, "DebugInfoMap"), + 0x089a1: (162, "BreakPointInfoMap"), +} + + +# List of known V8 objects. Used to determine name for objects that are +# part of the root-set and hence on the first page of various old-space +# paged. Obtained by adding the code below to an IA32 release build with +# enabled snapshots to the end of the Isolate::Init method. +# +# #define ROOT_LIST_CASE(type, name, camel_name) \ +# if (o == heap_.name()) n = #camel_name; +# OldSpaces spit; +# printf("KNOWN_OBJECTS = {\n"); +# for (PagedSpace* s = spit.next(); s != NULL; s = spit.next()) { +# HeapObjectIterator it(s); +# const char* sname = AllocationSpaceName(s->identity()); +# for (Object* o = it.Next(); o != NULL; o = it.Next()) { +# const char* n = NULL; +# intptr_t p = reinterpret_cast<intptr_t>(o) & 0xfffff; +# ROOT_LIST(ROOT_LIST_CASE) +# if (n != NULL) { +# printf(" (\"%s\", 0x%05x): \"%s\",\n", sname, p, n); +# } +# } +# } +# printf("}\n"); +KNOWN_OBJECTS = { + ("OLD_POINTER_SPACE", 0x08081): "NullValue", + ("OLD_POINTER_SPACE", 0x08091): "UndefinedValue", + ("OLD_POINTER_SPACE", 0x080a1): "InstanceofCacheMap", + ("OLD_POINTER_SPACE", 0x080b1): "TrueValue", + ("OLD_POINTER_SPACE", 0x080c1): "FalseValue", + ("OLD_POINTER_SPACE", 0x080d1): "NoInterceptorResultSentinel", + ("OLD_POINTER_SPACE", 0x080e1): "ArgumentsMarker", + ("OLD_POINTER_SPACE", 0x080f1): "NumberStringCache", + ("OLD_POINTER_SPACE", 0x088f9): "SingleCharacterStringCache", + ("OLD_POINTER_SPACE", 0x08b01): "StringSplitCache", + ("OLD_POINTER_SPACE", 0x08f09): "TerminationException", + ("OLD_POINTER_SPACE", 0x08f19): "MessageListeners", + ("OLD_POINTER_SPACE", 0x08f35): "CodeStubs", + ("OLD_POINTER_SPACE", 0x09b61): "NonMonomorphicCache", + ("OLD_POINTER_SPACE", 0x0a175): "PolymorphicCodeCache", + ("OLD_POINTER_SPACE", 0x0a17d): "NativesSourceCache", + ("OLD_POINTER_SPACE", 0x0a1bd): "EmptyScript", + ("OLD_POINTER_SPACE", 0x0a1f9): "IntrinsicFunctionNames", + ("OLD_POINTER_SPACE", 0x24a49): "SymbolTable", + ("OLD_DATA_SPACE", 0x08081): "EmptyFixedArray", + ("OLD_DATA_SPACE", 0x080a1): "NanValue", + ("OLD_DATA_SPACE", 0x0811d): "EmptyByteArray", + ("OLD_DATA_SPACE", 0x08125): "EmptyString", + ("OLD_DATA_SPACE", 0x08131): "EmptyDescriptorArray", + ("OLD_DATA_SPACE", 0x08259): "InfinityValue", + ("OLD_DATA_SPACE", 0x08265): "MinusZeroValue", + ("OLD_DATA_SPACE", 0x08271): "PrototypeAccessors", + ("CODE_SPACE", 0x12b81): "JsEntryCode", + ("CODE_SPACE", 0x12c61): "JsConstructEntryCode", } @@ -799,18 +1159,42 @@ class ConsString(String): class Oddball(HeapObject): + # Should match declarations in objects.h + KINDS = [ + "False", + "True", + "TheHole", + "Null", + "ArgumentMarker", + "Undefined", + "Other" + ] + def ToStringOffset(self): return self.heap.PointerSize() + def ToNumberOffset(self): + return self.ToStringOffset() + self.heap.PointerSize() + + def KindOffset(self): + return self.ToNumberOffset() + self.heap.PointerSize() + def __init__(self, heap, map, address): HeapObject.__init__(self, heap, map, address) self.to_string = self.ObjectField(self.ToStringOffset()) + self.kind = self.SmiField(self.KindOffset()) def Print(self, p): p.Print(str(self)) def __str__(self): - return "<%s>" % self.to_string.GetChars() + if self.to_string: + return "Oddball(%08x, <%s>)" % (self.address, self.to_string.GetChars()) + else: + kind = "???" + if 0 <= self.kind < len(Oddball.KINDS): + kind = Oddball.KINDS[self.kind] + return "Oddball(%08x, kind=%s)" % (self.address, kind) class FixedArray(HeapObject): @@ -831,7 +1215,13 @@ class FixedArray(HeapObject): base_offset = self.ElementsOffset() for i in xrange(self.length): offset = base_offset + 4 * i - p.Print("[%08d] = %s" % (i, self.ObjectField(offset))) + try: + p.Print("[%08d] = %s" % (i, self.ObjectField(offset))) + except TypeError: + p.Dedent() + p.Print("...") + p.Print("}") + return p.Dedent() p.Print("}") @@ -936,6 +1326,27 @@ class Script(HeapObject): self.name = self.ObjectField(self.NameOffset()) +class CodeCache(HeapObject): + def DefaultCacheOffset(self): + return self.heap.PointerSize() + + def NormalTypeCacheOffset(self): + return self.DefaultCacheOffset() + self.heap.PointerSize() + + def __init__(self, heap, map, address): + HeapObject.__init__(self, heap, map, address) + self.default_cache = self.ObjectField(self.DefaultCacheOffset()) + self.normal_type_cache = self.ObjectField(self.NormalTypeCacheOffset()) + + def Print(self, p): + p.Print("CodeCache(%s) {" % self.heap.reader.FormatIntPtr(self.address)) + p.Indent() + p.Print("default cache: %s" % self.default_cache) + p.Print("normal type cache: %s" % self.normal_type_cache) + p.Dedent() + p.Print("}") + + class Code(HeapObject): CODE_ALIGNMENT_MASK = (1 << 5) - 1 @@ -986,14 +1397,14 @@ class V8Heap(object): "EXTERNAL_STRING_TYPE": ExternalString, "EXTERNAL_STRING_WITH_ASCII_DATA_TYPE": ExternalString, "EXTERNAL_ASCII_STRING_TYPE": ExternalString, - "MAP_TYPE": Map, "ODDBALL_TYPE": Oddball, "FIXED_ARRAY_TYPE": FixedArray, "JS_FUNCTION_TYPE": JSFunction, "SHARED_FUNCTION_INFO_TYPE": SharedFunctionInfo, "SCRIPT_TYPE": Script, - "CODE_TYPE": Code + "CODE_CACHE_TYPE": CodeCache, + "CODE_TYPE": Code, } def __init__(self, reader, stack_map): @@ -1051,109 +1462,383 @@ class V8Heap(object): elif self.reader.arch == MD_CPU_ARCHITECTURE_X86: return (1 << 5) - 1 + def PageAlignmentMask(self): + return (1 << 20) - 1 -EIP_PROXIMITY = 64 -CONTEXT_FOR_ARCH = { - MD_CPU_ARCHITECTURE_AMD64: - ['rax', 'rbx', 'rcx', 'rdx', 'rdi', 'rsi', 'rbp', 'rsp', 'rip'], - MD_CPU_ARCHITECTURE_X86: - ['eax', 'ebx', 'ecx', 'edx', 'edi', 'esi', 'ebp', 'esp', 'eip'] -} +class KnownObject(HeapObject): + def __init__(self, heap, known_name): + HeapObject.__init__(self, heap, None, None) + self.known_name = known_name + + def __str__(self): + return "<%s>" % self.known_name + + +class KnownMap(HeapObject): + def __init__(self, heap, known_name, instance_type): + HeapObject.__init__(self, heap, None, None) + self.instance_type = instance_type + self.known_name = known_name + + def __str__(self): + return "<%s>" % self.known_name + + +class InspectionPadawan(object): + """The padawan can improve annotations by sensing well-known objects.""" + def __init__(self, reader, heap): + self.reader = reader + self.heap = heap + self.known_first_map_page = 0 + self.known_first_data_page = 0 + self.known_first_pointer_page = 0 + + def __getattr__(self, name): + """An InspectionPadawan can be used instead of V8Heap, even though + it does not inherit from V8Heap (aka. mixin).""" + return getattr(self.heap, name) + + def GetPageOffset(self, tagged_address): + return tagged_address & self.heap.PageAlignmentMask() + + def IsInKnownMapSpace(self, tagged_address): + page_address = tagged_address & ~self.heap.PageAlignmentMask() + return page_address == self.known_first_map_page + + def IsInKnownOldSpace(self, tagged_address): + page_address = tagged_address & ~self.heap.PageAlignmentMask() + return page_address in [self.known_first_data_page, + self.known_first_pointer_page] + + def ContainingKnownOldSpaceName(self, tagged_address): + page_address = tagged_address & ~self.heap.PageAlignmentMask() + if page_address == self.known_first_data_page: return "OLD_DATA_SPACE" + if page_address == self.known_first_pointer_page: return "OLD_POINTER_SPACE" + return None + + def SenseObject(self, tagged_address): + if self.IsInKnownOldSpace(tagged_address): + offset = self.GetPageOffset(tagged_address) + lookup_key = (self.ContainingKnownOldSpaceName(tagged_address), offset) + known_obj_name = KNOWN_OBJECTS.get(lookup_key) + if known_obj_name: + return KnownObject(self, known_obj_name) + if self.IsInKnownMapSpace(tagged_address): + known_map = self.SenseMap(tagged_address) + if known_map: + return known_map + found_obj = self.heap.FindObject(tagged_address) + if found_obj: return found_obj + address = tagged_address - 1 + if self.reader.IsValidAddress(address): + map_tagged_address = self.reader.ReadUIntPtr(address) + map = self.SenseMap(map_tagged_address) + if map is None: return None + instance_type_name = INSTANCE_TYPES.get(map.instance_type) + if instance_type_name is None: return None + cls = V8Heap.CLASS_MAP.get(instance_type_name, HeapObject) + return cls(self, map, address) + return None + + def SenseMap(self, tagged_address): + if self.IsInKnownMapSpace(tagged_address): + offset = self.GetPageOffset(tagged_address) + known_map_info = KNOWN_MAPS.get(offset) + if known_map_info: + known_map_type, known_map_name = known_map_info + return KnownMap(self, known_map_name, known_map_type) + found_map = self.heap.FindMap(tagged_address) + if found_map: return found_map + return None + + def FindObjectOrSmi(self, tagged_address): + """When used as a mixin in place of V8Heap.""" + found_obj = self.SenseObject(tagged_address) + if found_obj: return found_obj + if (tagged_address & 1) == 0: + return "Smi(%d)" % (tagged_address / 2) + else: + return "Unknown(%s)" % self.reader.FormatIntPtr(tagged_address) + + def FindObject(self, tagged_address): + """When used as a mixin in place of V8Heap.""" + raise NotImplementedError + + def FindMap(self, tagged_address): + """When used as a mixin in place of V8Heap.""" + raise NotImplementedError + + def PrintKnowledge(self): + print " known_first_map_page = %s\n"\ + " known_first_data_page = %s\n"\ + " known_first_pointer_page = %s" % ( + self.reader.FormatIntPtr(self.known_first_map_page), + self.reader.FormatIntPtr(self.known_first_data_page), + self.reader.FormatIntPtr(self.known_first_pointer_page)) + class InspectionShell(cmd.Cmd): def __init__(self, reader, heap): cmd.Cmd.__init__(self) self.reader = reader self.heap = heap + self.padawan = InspectionPadawan(reader, heap) self.prompt = "(grok) " + def do_da(self, address): + """ + Print ASCII string starting at specified address. + """ + address = int(address, 16) + string = "" + while self.reader.IsValidAddress(address): + code = self.reader.ReadU8(address) + if code < 128: + string += chr(code) + else: + break + address += 1 + if string == "": + print "Not an ASCII string at %s" % self.reader.FormatIntPtr(address) + else: + print "%s\n" % string + def do_dd(self, address): - "Interpret memory at the given address (if available)"\ - " as a sequence of words." + """ + Interpret memory at the given address (if available) as a sequence + of words. Automatic alignment is not performed. + """ start = int(address, 16) + if (start & self.heap.ObjectAlignmentMask()) != 0: + print "Warning: Dumping un-aligned memory, is this what you had in mind?" for slot in xrange(start, start + self.reader.PointerSize() * 10, self.reader.PointerSize()): + if not self.reader.IsValidAddress(slot): + print "Address is not contained within the minidump!" + return maybe_address = self.reader.ReadUIntPtr(slot) - heap_object = self.heap.FindObject(maybe_address) - print "%s: %s" % (self.reader.FormatIntPtr(slot), - self.reader.FormatIntPtr(maybe_address)) - if heap_object: - heap_object.Print(Printer()) - print + heap_object = self.padawan.SenseObject(maybe_address) + print "%s: %s %s" % (self.reader.FormatIntPtr(slot), + self.reader.FormatIntPtr(maybe_address), + heap_object or '') + + def do_do(self, address): + """ + Interpret memory at the given address as a V8 object. Automatic + alignment makes sure that you can pass tagged as well as un-tagged + addresses. + """ + address = int(address, 16) + if (address & self.heap.ObjectAlignmentMask()) == 0: + address = address + 1 + elif (address & self.heap.ObjectAlignmentMask()) != 1: + print "Address doesn't look like a valid pointer!" + return + heap_object = self.padawan.SenseObject(address) + if heap_object: + heap_object.Print(Printer()) + else: + print "Address cannot be interpreted as object!" + + def do_dp(self, address): + """ + Interpret memory at the given address as being on a V8 heap page + and print information about the page header (if available). + """ + address = int(address, 16) + page_address = address & ~self.heap.PageAlignmentMask() + if self.reader.IsValidAddress(page_address): + raise NotImplementedError + else: + print "Page header is not available!" + + def do_k(self, arguments): + """ + Teach V8 heap layout information to the inspector. This increases + the amount of annotations the inspector can produce while dumping + data. The first page of each heap space is of particular interest + because it contains known objects that do not move. + """ + self.padawan.PrintKnowledge() + + def do_kd(self, address): + """ + Teach V8 heap layout information to the inspector. Set the first + data-space page by passing any pointer into that page. + """ + address = int(address, 16) + page_address = address & ~self.heap.PageAlignmentMask() + self.padawan.known_first_data_page = page_address + + def do_km(self, address): + """ + Teach V8 heap layout information to the inspector. Set the first + map-space page by passing any pointer into that page. + """ + address = int(address, 16) + page_address = address & ~self.heap.PageAlignmentMask() + self.padawan.known_first_map_page = page_address + + def do_kp(self, address): + """ + Teach V8 heap layout information to the inspector. Set the first + pointer-space page by passing any pointer into that page. + """ + address = int(address, 16) + page_address = address & ~self.heap.PageAlignmentMask() + self.padawan.known_first_pointer_page = page_address + + def do_list(self, smth): + """ + List all available memory regions. + """ + def print_region(reader, start, size, location): + print " %s - %s (%d bytes)" % (reader.FormatIntPtr(start), + reader.FormatIntPtr(start + size), + size) + print "Available memory regions:" + self.reader.ForEachMemoryRegion(print_region) def do_s(self, word): - "Search for a given word in available memory regions" - word = int(word, 0) - print "searching for word", word + """ + Search for a given word in available memory regions. The given word + is expanded to full pointer size and searched at aligned as well as + un-aligned memory locations. Use 'sa' to search aligned locations + only. + """ + try: + word = int(word, 0) + except ValueError: + print "Malformed word, prefix with '0x' to use hexadecimal format." + return + print "Searching for word %d/0x%s:" % (word, self.reader.FormatIntPtr(word)) self.reader.FindWord(word) - def do_list(self, smth): - """List all available memory regions.""" - def print_region(reader, start, size, location): - print "%s - %s" % (reader.FormatIntPtr(start), - reader.FormatIntPtr(start + size)) + def do_sh(self, none): + """ + Search for the V8 Heap object in all available memory regions. You + might get lucky and find this rare treasure full of invaluable + information. + """ + raise NotImplementedError + + def do_u(self, args): + """ + u 0x<address> 0x<size> + Unassemble memory in the region [address, address + size) + """ + args = args.split(' ') + start = int(args[0], 16) + size = int(args[1], 16) + lines = self.reader.GetDisasmLines(start, size) + for line in lines: + print FormatDisasmLine(start, self.heap, line) + print - self.reader.ForEachMemoryRegion(print_region) +EIP_PROXIMITY = 64 + +CONTEXT_FOR_ARCH = { + MD_CPU_ARCHITECTURE_AMD64: + ['rax', 'rbx', 'rcx', 'rdx', 'rdi', 'rsi', 'rbp', 'rsp', 'rip', + 'r8', 'r9', 'r10', 'r11', 'r12', 'r13', 'r14', 'r15'], + MD_CPU_ARCHITECTURE_X86: + ['eax', 'ebx', 'ecx', 'edx', 'edi', 'esi', 'ebp', 'esp', 'eip'] +} + +KNOWN_MODULES = {'chrome.exe', 'chrome.dll'} + +def GetModuleName(reader, module): + name = reader.ReadMinidumpString(module.module_name_rva) + return str(os.path.basename(str(name).replace("\\", "/"))) def AnalyzeMinidump(options, minidump_name): reader = MinidumpReader(options, minidump_name) + heap = None DebugPrint("========================================") if reader.exception is None: print "Minidump has no exception info" - return - print "Exception info:" - exception_thread = reader.thread_map[reader.exception.thread_id] - print " thread id: %d" % exception_thread.id - print " code: %08X" % reader.exception.exception.code - print " context:" - for r in CONTEXT_FOR_ARCH[reader.arch]: - print " %s: %s" % (r, reader.FormatIntPtr(reader.Register(r))) - # TODO(vitalyr): decode eflags. - print " eflags: %s" % bin(reader.exception_context.eflags)[2:] - print - - stack_top = reader.ExceptionSP() - stack_bottom = exception_thread.stack.start + \ - exception_thread.stack.memory.data_size - stack_map = {reader.ExceptionIP(): -1} - for slot in xrange(stack_top, stack_bottom, reader.PointerSize()): - maybe_address = reader.ReadUIntPtr(slot) - if not maybe_address in stack_map: - stack_map[maybe_address] = slot - heap = V8Heap(reader, stack_map) - - print "Disassembly around exception.eip:" - start = reader.ExceptionIP() - EIP_PROXIMITY - lines = reader.GetDisasmLines(start, 2 * EIP_PROXIMITY) - for line in lines: - print FormatDisasmLine(start, heap, line) - print + else: + print "Exception info:" + exception_thread = reader.thread_map[reader.exception.thread_id] + print " thread id: %d" % exception_thread.id + print " code: %08X" % reader.exception.exception.code + print " context:" + for r in CONTEXT_FOR_ARCH[reader.arch]: + print " %s: %s" % (r, reader.FormatIntPtr(reader.Register(r))) + # TODO(vitalyr): decode eflags. + print " eflags: %s" % bin(reader.exception_context.eflags)[2:] + print + print " modules:" + for module in reader.module_list.modules: + name = GetModuleName(reader, module) + if name in KNOWN_MODULES: + print " %s at %08X" % (name, module.base_of_image) + reader.TryLoadSymbolsFor(name, module) + print + + stack_top = reader.ExceptionSP() + stack_bottom = exception_thread.stack.start + \ + exception_thread.stack.memory.data_size + stack_map = {reader.ExceptionIP(): -1} + for slot in xrange(stack_top, stack_bottom, reader.PointerSize()): + maybe_address = reader.ReadUIntPtr(slot) + if not maybe_address in stack_map: + stack_map[maybe_address] = slot + heap = V8Heap(reader, stack_map) + + print "Disassembly around exception.eip:" + eip_symbol = reader.FindSymbol(reader.ExceptionIP()) + if eip_symbol is not None: + print eip_symbol + disasm_start = reader.ExceptionIP() - EIP_PROXIMITY + disasm_bytes = 2 * EIP_PROXIMITY + if (options.full): + full_range = reader.FindRegion(reader.ExceptionIP()) + if full_range is not None: + disasm_start = full_range[0] + disasm_bytes = full_range[1] + + lines = reader.GetDisasmLines(disasm_start, disasm_bytes) + + for line in lines: + print FormatDisasmLine(disasm_start, heap, line) + print + + if heap is None: + heap = V8Heap(reader, None) if options.full: - do_dump(reader, heap) + FullDump(reader, heap) if options.shell: InspectionShell(reader, heap).cmdloop("type help to get help") else: - print "Annotated stack (from exception.esp to bottom):" - for slot in xrange(stack_top, stack_bottom, reader.PointerSize()): - maybe_address = reader.ReadUIntPtr(slot) - heap_object = heap.FindObject(maybe_address) - print "%s: %s" % (reader.FormatIntPtr(slot), - reader.FormatIntPtr(maybe_address)) - if heap_object: - heap_object.Print(Printer()) - print + if reader.exception is not None: + print "Annotated stack (from exception.esp to bottom):" + for slot in xrange(stack_top, stack_bottom, reader.PointerSize()): + maybe_address = reader.ReadUIntPtr(slot) + heap_object = heap.FindObject(maybe_address) + maybe_symbol = reader.FindSymbol(maybe_address) + print "%s: %s %s" % (reader.FormatIntPtr(slot), + reader.FormatIntPtr(maybe_address), + maybe_symbol or "") + if heap_object: + heap_object.Print(Printer()) + print reader.Dispose() if __name__ == "__main__": parser = optparse.OptionParser(USAGE) - parser.add_option("-s", "--shell", dest="shell", action="store_true") - parser.add_option("-f", "--full", dest="full", action="store_true") + parser.add_option("-s", "--shell", dest="shell", action="store_true", + help="start an interactive inspector shell") + parser.add_option("-f", "--full", dest="full", action="store_true", + help="dump all information contained in the minidump") + parser.add_option("--symdir", dest="symdir", default=".", + help="directory containing *.pdb.sym file with symbols") options, args = parser.parse_args() if len(args) != 1: parser.print_help() diff --git a/src/3rdparty/v8/tools/gyp/v8.gyp b/src/3rdparty/v8/tools/gyp/v8.gyp index aa91139..aad07c7 100644 --- a/src/3rdparty/v8/tools/gyp/v8.gyp +++ b/src/3rdparty/v8/tools/gyp/v8.gyp @@ -69,6 +69,14 @@ ], }, 'conditions': [ + ['OS=="android"', { + 'libraries': [ + '-llog', + ], + 'include_dirs': [ + 'src/common/android/include', + ], + }], ['OS=="mac"', { 'xcode_settings': { 'OTHER_LDFLAGS': ['-dynamiclib', '-all_load'] @@ -234,10 +242,10 @@ '../../src/ast.h', '../../src/atomicops.h', '../../src/atomicops_internals_x86_gcc.cc', - '../../src/bignum.cc', - '../../src/bignum.h', '../../src/bignum-dtoa.cc', '../../src/bignum-dtoa.h', + '../../src/bignum.cc', + '../../src/bignum.h', '../../src/bootstrapper.cc', '../../src/bootstrapper.h', '../../src/builtins.cc', @@ -268,21 +276,21 @@ '../../src/conversions.h', '../../src/counters.cc', '../../src/counters.h', - '../../src/cpu.h', '../../src/cpu-profiler-inl.h', '../../src/cpu-profiler.cc', '../../src/cpu-profiler.h', + '../../src/cpu.h', '../../src/data-flow.cc', '../../src/data-flow.h', '../../src/date.cc', '../../src/date.h', + '../../src/dateparser-inl.h', '../../src/dateparser.cc', '../../src/dateparser.h', - '../../src/dateparser-inl.h', - '../../src/debug.cc', - '../../src/debug.h', '../../src/debug-agent.cc', '../../src/debug-agent.h', + '../../src/debug.cc', + '../../src/debug.h', '../../src/deoptimizer.cc', '../../src/deoptimizer.h', '../../src/disasm.h', @@ -293,17 +301,25 @@ '../../src/double.h', '../../src/dtoa.cc', '../../src/dtoa.h', + '../../src/elements-kind.cc', + '../../src/elements-kind.h', '../../src/elements.cc', '../../src/elements.h', '../../src/execution.cc', '../../src/execution.h', + '../../src/extensions/externalize-string-extension.cc', + '../../src/extensions/externalize-string-extension.h', + '../../src/extensions/gc-extension.cc', + '../../src/extensions/gc-extension.h', + '../../src/extensions/statistics-extension.cc', + '../../src/extensions/statistics-extension.h', '../../src/factory.cc', '../../src/factory.h', '../../src/fast-dtoa.cc', '../../src/fast-dtoa.h', - '../../src/flag-definitions.h', '../../src/fixed-dtoa.cc', '../../src/fixed-dtoa.h', + '../../src/flag-definitions.h', '../../src/flags.cc', '../../src/flags.h', '../../src/frames-inl.h', @@ -313,6 +329,8 @@ '../../src/full-codegen.h', '../../src/func-name-inferrer.cc', '../../src/func-name-inferrer.h', + '../../src/gdb-jit.cc', + '../../src/gdb-jit.h', '../../src/global-handles.cc', '../../src/global-handles.h', '../../src/globals.h', @@ -321,14 +339,14 @@ '../../src/handles.h', '../../src/hashmap.h', '../../src/heap-inl.h', - '../../src/heap.cc', - '../../src/heap.h', '../../src/heap-profiler.cc', '../../src/heap-profiler.h', - '../../src/hydrogen.cc', - '../../src/hydrogen.h', + '../../src/heap.cc', + '../../src/heap.h', '../../src/hydrogen-instructions.cc', '../../src/hydrogen-instructions.h', + '../../src/hydrogen.cc', + '../../src/hydrogen.h', '../../src/ic-inl.h', '../../src/ic.cc', '../../src/ic.h', @@ -340,19 +358,20 @@ '../../src/interface.h', '../../src/interpreter-irregexp.cc', '../../src/interpreter-irregexp.h', + '../../src/isolate.cc', + '../../src/isolate.h', '../../src/json-parser.h', + '../../src/json-stringifier.h', '../../src/jsregexp.cc', '../../src/jsregexp.h', - '../../src/isolate.cc', - '../../src/isolate.h', '../../src/lazy-instance.h', '../../src/list-inl.h', '../../src/list.h', - '../../src/lithium.cc', - '../../src/lithium.h', + '../../src/lithium-allocator-inl.h', '../../src/lithium-allocator.cc', '../../src/lithium-allocator.h', - '../../src/lithium-allocator-inl.h', + '../../src/lithium.cc', + '../../src/lithium.h', '../../src/liveedit.cc', '../../src/liveedit.h', '../../src/liveobjectlist-inl.h', @@ -370,14 +389,16 @@ '../../src/messages.h', '../../src/natives.h', '../../src/objects-debug.cc', - '../../src/objects-printer.cc', '../../src/objects-inl.h', + '../../src/objects-printer.cc', '../../src/objects-visiting.cc', '../../src/objects-visiting.h', '../../src/objects.cc', '../../src/objects.h', '../../src/once.cc', '../../src/once.h', + '../../src/optimizing-compiler-thread.h', + '../../src/optimizing-compiler-thread.cc', '../../src/parser.cc', '../../src/parser.h', '../../src/platform-posix.h', @@ -392,12 +413,12 @@ '../../src/preparser.h', '../../src/prettyprinter.cc', '../../src/prettyprinter.h', - '../../src/property.cc', - '../../src/property.h', - '../../src/property-details.h', '../../src/profile-generator-inl.h', '../../src/profile-generator.cc', '../../src/profile-generator.h', + '../../src/property-details.h', + '../../src/property.cc', + '../../src/property.h', '../../src/regexp-macro-assembler-irregexp-inl.h', '../../src/regexp-macro-assembler-irregexp.cc', '../../src/regexp-macro-assembler-irregexp.h', @@ -409,16 +430,16 @@ '../../src/regexp-stack.h', '../../src/rewriter.cc', '../../src/rewriter.h', - '../../src/runtime.cc', - '../../src/runtime.h', '../../src/runtime-profiler.cc', '../../src/runtime-profiler.h', + '../../src/runtime.cc', + '../../src/runtime.h', '../../src/safepoint-table.cc', '../../src/safepoint-table.h', - '../../src/scanner.cc', - '../../src/scanner.h', '../../src/scanner-character-streams.cc', '../../src/scanner-character-streams.h', + '../../src/scanner.cc', + '../../src/scanner.h', '../../src/scopeinfo.cc', '../../src/scopeinfo.h', '../../src/scopes.cc', @@ -426,7 +447,7 @@ '../../src/serialize.cc', '../../src/serialize.h', '../../src/small-pointer-list.h', - '../../src/smart-array-pointer.h', + '../../src/smart-pointers.h', '../../src/snapshot-common.cc', '../../src/snapshot.h', '../../src/spaces-inl.h', @@ -445,6 +466,9 @@ '../../src/stub-cache.h', '../../src/token.cc', '../../src/token.h', + '../../src/transitions-inl.h', + '../../src/transitions.cc', + '../../src/transitions.h', '../../src/type-info.cc', '../../src/type-info.h', '../../src/unbound-queue-inl.h', @@ -477,10 +501,6 @@ '../../src/zone-inl.h', '../../src/zone.cc', '../../src/zone.h', - '../../src/extensions/externalize-string-extension.cc', - '../../src/extensions/externalize-string-extension.h', - '../../src/extensions/gc-extension.cc', - '../../src/extensions/gc-extension.h', ], 'conditions': [ ['want_separate_host_toolset==1', { @@ -553,7 +573,7 @@ '../../src/ia32/stub-cache-ia32.cc', ], }], - ['v8_target_arch=="mips"', { + ['v8_target_arch=="mipsel"', { 'sources': [ '../../src/mips/assembler-mips.cc', '../../src/mips/assembler-mips.h', @@ -768,6 +788,7 @@ '../../src/macros.py', '../../src/proxy.js', '../../src/collection.js', + '../../src/object-observe.js' ], }, 'actions': [ diff --git a/src/3rdparty/v8/tools/linux-tick-processor b/src/3rdparty/v8/tools/linux-tick-processor index 7070ce6..93f143f 100755 --- a/src/3rdparty/v8/tools/linux-tick-processor +++ b/src/3rdparty/v8/tools/linux-tick-processor @@ -12,21 +12,21 @@ done tools_path=`cd $(dirname "$0");pwd` if [ ! "$D8_PATH" ]; then d8_public=`which d8` - if [ -x $d8_public ]; then D8_PATH=$(dirname "$d8_public"); fi + if [ -x "$d8_public" ]; then D8_PATH=$(dirname "$d8_public"); fi fi -[ "$D8_PATH" ] || D8_PATH=$tools_path/.. +[ -n "$D8_PATH" ] || D8_PATH=$tools_path/.. d8_exec=$D8_PATH/d8 -if [ ! -x $d8_exec ]; then +if [ ! -x "$d8_exec" ]; then D8_PATH=`pwd`/out/native d8_exec=$D8_PATH/d8 fi -if [ ! -x $d8_exec ]; then +if [ ! -x "$d8_exec" ]; then d8_exec=`grep -m 1 -o '".*/d8"' $log_file | sed 's/"//g'` fi -if [ ! -x $d8_exec ]; then +if [ ! -x "$d8_exec" ]; then echo "d8 shell not found in $D8_PATH" echo "To build, execute 'make native' from the V8 directory" exit 1 diff --git a/src/3rdparty/v8/tools/ll_prof.py b/src/3rdparty/v8/tools/ll_prof.py index 51ba672..3afe179 100755 --- a/src/3rdparty/v8/tools/ll_prof.py +++ b/src/3rdparty/v8/tools/ll_prof.py @@ -68,15 +68,9 @@ Examples: """ -# Must match kGcFakeMmap. -V8_GC_FAKE_MMAP = "/tmp/__v8_gc__" - JS_ORIGIN = "js" JS_SNAPSHOT_ORIGIN = "js-snapshot" -OBJDUMP_BIN = disasm.OBJDUMP_BIN - - class Code(object): """Code object.""" @@ -639,7 +633,7 @@ class TraceReader(object): # Read null-terminated filename. filename = self.trace[offset + self.header_size + ctypes.sizeof(mmap_info): offset + header.size] - mmap_info.filename = filename[:filename.find(chr(0))] + mmap_info.filename = HOST_ROOT + filename[:filename.find(chr(0))] return mmap_info def ReadSample(self, header, offset): @@ -858,6 +852,15 @@ if __name__ == "__main__": default=False, action="store_true", help="no auxiliary messages [default: %default]") + parser.add_option("--gc-fake-mmap", + default="/tmp/__v8_gc__", + help="gc fake mmap file [default: %default]") + parser.add_option("--objdump", + default="/usr/bin/objdump", + help="objdump tool to use [default: %default]") + parser.add_option("--host-root", + default="", + help="Path to the host root [default: %default]") options, args = parser.parse_args() if not options.quiet: @@ -869,6 +872,14 @@ if __name__ == "__main__": print "V8 log: %s, %s.ll (no snapshot)" % (options.log, options.log) print "Perf trace file: %s" % options.trace + V8_GC_FAKE_MMAP = options.gc_fake_mmap + HOST_ROOT = options.host_root + if os.path.exists(options.objdump): + disasm.OBJDUMP_BIN = options.objdump + OBJDUMP_BIN = options.objdump + else: + print "Cannot find %s, falling back to default objdump" % options.objdump + # Stats. events = 0 ticks = 0 @@ -905,7 +916,7 @@ if __name__ == "__main__": if header.type == PERF_RECORD_MMAP: start = time.time() mmap_info = trace_reader.ReadMmap(header, offset) - if mmap_info.filename == V8_GC_FAKE_MMAP: + if mmap_info.filename == HOST_ROOT + V8_GC_FAKE_MMAP: log_reader.ReadUpToGC() else: library_repo.Load(mmap_info, code_map, options) diff --git a/src/3rdparty/v8/tools/merge-to-branch.sh b/src/3rdparty/v8/tools/merge-to-branch.sh index aa590a3..e0011ed 100644..100755 --- a/src/3rdparty/v8/tools/merge-to-branch.sh +++ b/src/3rdparty/v8/tools/merge-to-branch.sh @@ -186,7 +186,6 @@ if [ $START_STEP -le $CURRENT_STEP ] ; then if [ -n "$EXTRA_PATCH" ] ; then apply_patch "$EXTRA_PATCH" fi - stage_files fi let CURRENT_STEP+=1 @@ -205,8 +204,9 @@ if [ $START_STEP -le $CURRENT_STEP ] ; then your EDITOR on $VERSION_FILE so you can make arbitrary changes. When \ you're done, save the file and exit your EDITOR.)" if [ $? -eq 0 ] ; then + echo $NEWPATCH $VERSION_FILE sed -e "/#define PATCH_LEVEL/s/[0-9]*$/$NEWPATCH/" \ - -i "$VERSION_FILE" + -i.bak "$VERSION_FILE" || die "Could not increment patch level" else $EDITOR "$VERSION_FILE" fi diff --git a/src/3rdparty/v8/tools/presubmit.py b/src/3rdparty/v8/tools/presubmit.py index a0b81e8..efa8724 100755 --- a/src/3rdparty/v8/tools/presubmit.py +++ b/src/3rdparty/v8/tools/presubmit.py @@ -307,6 +307,7 @@ class SourceProcessor(SourceFileProcessor): or (name == 'DerivedSources')) IGNORE_COPYRIGHTS = ['cpplint.py', + 'daemon.py', 'earley-boyer.js', 'raytrace.js', 'crypto.js', diff --git a/src/3rdparty/v8/tools/push-to-trunk.sh b/src/3rdparty/v8/tools/push-to-trunk.sh index ff6dd1d..8512d12 100755 --- a/src/3rdparty/v8/tools/push-to-trunk.sh +++ b/src/3rdparty/v8/tools/push-to-trunk.sh @@ -268,7 +268,6 @@ if [ $START_STEP -le $CURRENT_STEP ] ; then echo ">>> Step $CURRENT_STEP: Apply squashed changes." rm -f "$TOUCHED_FILES_FILE" apply_patch "$PATCH_FILE" - stage_files rm -f "$PATCH_FILE" fi @@ -304,11 +303,22 @@ fi let CURRENT_STEP+=1 if [ $START_STEP -le $CURRENT_STEP ] ; then echo ">>> Step $CURRENT_STEP: Commit to SVN." - git svn dcommit | tee >(grep -E "^Committed r[0-9]+" \ - | sed -e 's/^Committed r\([0-9]\+\)/\1/' \ - > "$TRUNK_REVISION_FILE") \ + git svn dcommit 2>&1 | tee >(grep -E "^Committed r[0-9]+" \ + | sed -e 's/^Committed r\([0-9]\+\)/\1/' \ + > "$TRUNK_REVISION_FILE") \ || die "'git svn dcommit' failed." TRUNK_REVISION=$(cat "$TRUNK_REVISION_FILE") + # Sometimes grepping for the revision fails. No idea why. If you figure + # out why it is flaky, please do fix it properly. + if [ -z "$TRUNK_REVISION" ] ; then + echo "Sorry, grepping for the SVN revision failed. Please look for it in \ +the last command's output above and provide it manually (just the number, \ +without the leading \"r\")." + while [ -z "$TRUNK_REVISION" ] ; do + echo -n "> " + read TRUNK_REVISION + done + fi persist "TRUNK_REVISION" rm -f "$TRUNK_REVISION_FILE" fi diff --git a/src/3rdparty/v8/tools/run-tests.py b/src/3rdparty/v8/tools/run-tests.py new file mode 100755 index 0000000..c09ea06 --- /dev/null +++ b/src/3rdparty/v8/tools/run-tests.py @@ -0,0 +1,369 @@ +#!/usr/bin/env python +# +# Copyright 2012 the V8 project authors. All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +import multiprocessing +import optparse +import os +from os.path import join +import subprocess +import sys +import time + +from testrunner.local import execution +from testrunner.local import progress +from testrunner.local import testsuite +from testrunner.local import utils +from testrunner.local import verbose +from testrunner.network import network_execution +from testrunner.objects import context + + +ARCH_GUESS = utils.DefaultArch() +DEFAULT_TESTS = ["mjsunit", "cctest", "message", "preparser"] +TIMEOUT_DEFAULT = 60 +TIMEOUT_SCALEFACTOR = {"debug" : 4, + "release" : 1 } + +# Use this to run several variants of the tests. +VARIANT_FLAGS = [[], + ["--stress-opt", "--always-opt"], + ["--nocrankshaft"]] +MODE_FLAGS = { + "debug" : ["--nobreak-on-abort", "--nodead-code-elimination", + "--enable-slow-asserts", "--debug-code", "--verify-heap"], + "release" : ["--nobreak-on-abort", "--nodead-code-elimination"]} + +SUPPORTED_ARCHS = ["android_arm", + "android_ia32", + "arm", + "ia32", + "mipsel", + "x64"] +# Double the timeout for these: +SLOW_ARCHS = ["android_arm", + "android_ia32", + "arm", + "mipsel"] + + +def BuildOptions(): + result = optparse.OptionParser() + result.add_option("--arch", + help=("The architecture to run tests for, " + "'auto' or 'native' for auto-detect"), + default="ia32,x64,arm") + result.add_option("--arch-and-mode", + help="Architecture and mode in the format 'arch.mode'", + default=None) + result.add_option("--buildbot", + help="Adapt to path structure used on buildbots", + default=False, action="store_true") + result.add_option("--cat", help="Print the source of the tests", + default=False, action="store_true") + result.add_option("--command-prefix", + help="Prepended to each shell command used to run a test", + default="") + result.add_option("--download-data", help="Download missing test suite data", + default=False, action="store_true") + result.add_option("--extra-flags", + help="Additional flags to pass to each test command", + default="") + result.add_option("--isolates", help="Whether to test isolates", + default=False, action="store_true") + result.add_option("-j", help="The number of parallel tasks to run", + default=0, type="int") + result.add_option("-m", "--mode", + help="The test modes in which to run (comma-separated)", + default="release,debug") + result.add_option("--no-network", "--nonetwork", + help="Don't distribute tests on the network", + default=(utils.GuessOS() != "linux"), + dest="no_network", action="store_true") + result.add_option("--no-presubmit", "--nopresubmit", + help='Skip presubmit checks', + default=False, dest="no_presubmit", action="store_true") + result.add_option("--no-stress", "--nostress", + help="Don't run crankshaft --always-opt --stress-op test", + default=False, dest="no_stress", action="store_true") + result.add_option("--outdir", help="Base directory with compile output", + default="out") + result.add_option("-p", "--progress", + help=("The style of progress indicator" + " (verbose, dots, color, mono)"), + choices=progress.PROGRESS_INDICATORS.keys(), default="mono") + result.add_option("--report", help="Print a summary of the tests to be run", + default=False, action="store_true") + result.add_option("--shard-count", + help="Split testsuites into this number of shards", + default=1, type="int") + result.add_option("--shard-run", + help="Run this shard from the split up tests.", + default=1, type="int") + result.add_option("--shell", help="DEPRECATED! use --shell-dir", default="") + result.add_option("--shell-dir", help="Directory containing executables", + default="") + result.add_option("--stress-only", + help="Only run tests with --always-opt --stress-opt", + default=False, action="store_true") + result.add_option("--time", help="Print timing information after running", + default=False, action="store_true") + result.add_option("-t", "--timeout", help="Timeout in seconds", + default= -1, type="int") + result.add_option("-v", "--verbose", help="Verbose output", + default=False, action="store_true") + result.add_option("--valgrind", help="Run tests through valgrind", + default=False, action="store_true") + result.add_option("--warn-unused", help="Report unused rules", + default=False, action="store_true") + return result + + +def ProcessOptions(options): + global VARIANT_FLAGS + + # Architecture and mode related stuff. + if options.arch_and_mode: + tokens = options.arch_and_mode.split(".") + options.arch = tokens[0] + options.mode = tokens[1] + options.mode = options.mode.split(",") + for mode in options.mode: + if not mode in ["debug", "release"]: + print "Unknown mode %s" % mode + return False + if options.arch in ["auto", "native"]: + options.arch = ARCH_GUESS + options.arch = options.arch.split(",") + for arch in options.arch: + if not arch in SUPPORTED_ARCHS: + print "Unknown architecture %s" % arch + return False + + # Special processing of other options, sorted alphabetically. + + if options.buildbot: + # Buildbots run presubmit tests as a separate step. + options.no_presubmit = True + options.no_network = True + if options.command_prefix: + print("Specifying --command-prefix disables network distribution, " + "running tests locally.") + options.no_network = True + if options.j == 0: + options.j = multiprocessing.cpu_count() + if options.no_stress: + VARIANT_FLAGS = [[], ["--nocrankshaft"]] + if not options.shell_dir: + if options.shell: + print "Warning: --shell is deprecated, use --shell-dir instead." + options.shell_dir = os.path.dirname(options.shell) + if options.stress_only: + VARIANT_FLAGS = [["--stress-opt", "--always-opt"]] + if options.valgrind: + run_valgrind = os.path.join("tools", "run-valgrind.py") + # This is OK for distributed running, so we don't need to set no_network. + options.command_prefix = ("python -u " + run_valgrind + + options.command_prefix) + return True + + +def ShardTests(tests, shard_count, shard_run): + if shard_count < 2: + return tests + if shard_run < 1 or shard_run > shard_count: + print "shard-run not a valid number, should be in [1:shard-count]" + print "defaulting back to running all tests" + return tests + count = 0 + shard = [] + for test in tests: + if count % shard_count == shard_run - 1: + shard.append(test) + count += 1 + return shard + + +def Main(): + parser = BuildOptions() + (options, args) = parser.parse_args() + if not ProcessOptions(options): + parser.print_help() + return 1 + + exit_code = 0 + workspace = os.path.abspath(join(os.path.dirname(sys.argv[0]), "..")) + if not options.no_presubmit: + print ">>> running presubmit tests" + code = subprocess.call( + [sys.executable, join(workspace, "tools", "presubmit.py")]) + exit_code = code + + suite_paths = utils.GetSuitePaths(join(workspace, "test")) + + if len(args) == 0: + suite_paths = [ s for s in suite_paths if s in DEFAULT_TESTS ] + else: + args_suites = set() + for arg in args: + suite = arg.split(os.path.sep)[0] + if not suite in args_suites: + args_suites.add(suite) + suite_paths = [ s for s in suite_paths if s in args_suites ] + + suites = [] + for root in suite_paths: + suite = testsuite.TestSuite.LoadTestSuite( + os.path.join(workspace, "test", root)) + if suite: + suites.append(suite) + + if options.download_data: + for s in suites: + s.DownloadData() + + for mode in options.mode: + for arch in options.arch: + code = Execute(arch, mode, args, options, suites, workspace) + exit_code = exit_code or code + return exit_code + + +def Execute(arch, mode, args, options, suites, workspace): + print(">>> Running tests for %s.%s" % (arch, mode)) + + shell_dir = options.shell_dir + if not shell_dir: + if options.buildbot: + shell_dir = os.path.join(workspace, options.outdir, mode) + mode = mode.lower() + else: + shell_dir = os.path.join(workspace, options.outdir, + "%s.%s" % (arch, mode)) + shell_dir = os.path.relpath(shell_dir) + + # Populate context object. + mode_flags = MODE_FLAGS[mode] + timeout = options.timeout + if timeout == -1: + # Simulators are slow, therefore allow a longer default timeout. + if arch in SLOW_ARCHS: + timeout = 2 * TIMEOUT_DEFAULT; + else: + timeout = TIMEOUT_DEFAULT; + + timeout *= TIMEOUT_SCALEFACTOR[mode] + ctx = context.Context(arch, mode, shell_dir, + mode_flags, options.verbose, + timeout, options.isolates, + options.command_prefix, + options.extra_flags) + + # Find available test suites and read test cases from them. + variables = { + "mode": mode, + "arch": arch, + "system": utils.GuessOS(), + "isolates": options.isolates + } + all_tests = [] + num_tests = 0 + test_id = 0 + for s in suites: + s.ReadStatusFile(variables) + s.ReadTestCases(ctx) + if len(args) > 0: + s.FilterTestCasesByArgs(args) + all_tests += s.tests + s.FilterTestCasesByStatus(options.warn_unused) + if options.cat: + verbose.PrintTestSource(s.tests) + continue + variant_flags = s.VariantFlags() or VARIANT_FLAGS + s.tests = [ t.CopyAddingFlags(v) for t in s.tests for v in variant_flags ] + s.tests = ShardTests(s.tests, options.shard_count, options.shard_run) + num_tests += len(s.tests) + for t in s.tests: + t.id = test_id + test_id += 1 + + if options.cat: + return 0 # We're done here. + + if options.report: + verbose.PrintReport(all_tests) + + if num_tests == 0: + print "No tests to run." + return 0 + + # Run the tests, either locally or distributed on the network. + try: + start_time = time.time() + progress_indicator = progress.PROGRESS_INDICATORS[options.progress]() + + run_networked = not options.no_network + if not run_networked: + print("Network distribution disabled, running tests locally.") + elif utils.GuessOS() != "linux": + print("Network distribution is only supported on Linux, sorry!") + run_networked = False + peers = [] + if run_networked: + peers = network_execution.GetPeers() + if not peers: + print("No connection to distribution server; running tests locally.") + run_networked = False + elif len(peers) == 1: + print("No other peers on the network; running tests locally.") + run_networked = False + elif num_tests <= 100: + print("Less than 100 tests, running them locally.") + run_networked = False + + if run_networked: + runner = network_execution.NetworkedRunner(suites, progress_indicator, + ctx, peers, workspace) + else: + runner = execution.Runner(suites, progress_indicator, ctx) + + exit_code = runner.Run(options.j) + if runner.terminate: + return exit_code + overall_duration = time.time() - start_time + except KeyboardInterrupt: + return 1 + + if options.time: + verbose.PrintTestDurations(suites, overall_duration) + return exit_code + + +if __name__ == "__main__": + sys.exit(Main()) diff --git a/src/3rdparty/v8/tools/status-file-converter.py b/src/3rdparty/v8/tools/status-file-converter.py new file mode 100755 index 0000000..ba063ee --- /dev/null +++ b/src/3rdparty/v8/tools/status-file-converter.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python +# +# Copyright 2012 the V8 project authors. All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +import sys +from testrunner.local import old_statusfile + +if len(sys.argv) != 2: + print "Usage: %s foo.status" % sys.argv[0] + print "Will read foo.status and print the converted version to stdout." + sys.exit(1) + +print old_statusfile.ConvertNotation(sys.argv[1]).GetOutput() diff --git a/src/3rdparty/v8/tools/test-server.py b/src/3rdparty/v8/tools/test-server.py new file mode 100755 index 0000000..df547ed --- /dev/null +++ b/src/3rdparty/v8/tools/test-server.py @@ -0,0 +1,224 @@ +#!/usr/bin/env python +# +# Copyright 2012 the V8 project authors. All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +import os +import subprocess +import sys + + +PIDFILE = "/tmp/v8-distributed-testing-server.pid" +ROOT = os.path.abspath(os.path.dirname(sys.argv[0])) + + +def _PrintUsage(): + print("""Usage: python %s COMMAND + +Where COMMAND can be any of: + start Starts the server. Forks to the background. + stop Stops the server. + restart Stops, then restarts the server. + setup Creates or updates the environment for the server to run. + update Alias for "setup". + trust <keyfile> Adds the given public key to the list of trusted keys. + help Displays this help text. + """ % sys.argv[0]) + + +def _IsDaemonRunning(): + return os.path.exists(PIDFILE) + + +def _Cmd(cmd): + code = subprocess.call(cmd, shell=True) + if code != 0: + print("Command '%s' returned error code %d" % (cmd, code)) + sys.exit(code) + + +def Update(): + # Create directory for private data storage. + data_dir = os.path.join(ROOT, "data") + if not os.path.exists(data_dir): + os.makedirs(data_dir) + + # Create directory for trusted public keys of peers (and self). + trusted_dir = os.path.join(ROOT, "trusted") + if not os.path.exists(trusted_dir): + os.makedirs(trusted_dir) + + # Install UltraJSON. It is much faster than Python's builtin json. + try: + import ujson #@UnusedImport + except ImportError: + # Install pip if it doesn't exist. + code = subprocess.call("which pip > /dev/null", shell=True) + if code != 0: + apt_get_code = subprocess.call("which apt-get > /dev/null", shell=True) + if apt_get_code == 0: + print("Installing pip...") + _Cmd("sudo apt-get install python-pip") + else: + print("Please install pip on your machine. You can get it at: " + "http://www.pip-installer.org/en/latest/installing.html " + "or via your distro's package manager.") + sys.exit(1) + print("Using pip to install UltraJSON...") + _Cmd("sudo pip install ujson") + + # Make sure we have a key pair for signing binaries. + privkeyfile = os.path.expanduser("~/.ssh/v8_dtest") + if not os.path.exists(privkeyfile): + _Cmd("ssh-keygen -t rsa -f %s -N '' -q" % privkeyfile) + fingerprint = subprocess.check_output("ssh-keygen -lf %s" % privkeyfile, + shell=True) + fingerprint = fingerprint.split(" ")[1].replace(":", "")[:16] + pubkeyfile = os.path.join(trusted_dir, "%s.pem" % fingerprint) + if (not os.path.exists(pubkeyfile) or + os.path.getmtime(pubkeyfile) < os.path.getmtime(privkeyfile)): + _Cmd("openssl rsa -in %s -out %s -pubout" % (privkeyfile, pubkeyfile)) + with open(pubkeyfile, "a") as f: + f.write(fingerprint + "\n") + datafile = os.path.join(data_dir, "mypubkey") + with open(datafile, "w") as f: + f.write(fingerprint + "\n") + + # Check out or update the server implementation in the current directory. + testrunner_dir = os.path.join(ROOT, "testrunner") + if os.path.exists(os.path.join(testrunner_dir, "server/daemon.py")): + _Cmd("cd %s; svn up" % testrunner_dir) + else: + path = ("http://v8.googlecode.com/svn/branches/bleeding_edge/" + "tools/testrunner") + _Cmd("svn checkout --force %s %s" % (path, testrunner_dir)) + + # Update this very script. + path = ("http://v8.googlecode.com/svn/branches/bleeding_edge/" + "tools/test-server.py") + scriptname = os.path.abspath(sys.argv[0]) + _Cmd("svn cat %s > %s" % (path, scriptname)) + + # The testcfg.py files currently need to be able to import the old test.py + # script, so we temporarily need to make that available. + # TODO(jkummerow): Remove this when removing test.py. + for filename in ("test.py", "utils.py"): + url = ("http://v8.googlecode.com/svn/branches/bleeding_edge/" + "tools/%s" % filename) + filepath = os.path.join(os.path.dirname(scriptname), filename) + _Cmd("svn cat %s > %s" % (url, filepath)) + + # Check out or update V8. + v8_dir = os.path.join(ROOT, "v8") + if os.path.exists(v8_dir): + _Cmd("cd %s; git fetch" % v8_dir) + else: + _Cmd("git clone git://github.com/v8/v8.git %s" % v8_dir) + + print("Finished.") + + +# Handle "setup" here, because when executing that we can't import anything +# else yet. +if __name__ == "__main__" and len(sys.argv) == 2: + if sys.argv[1] in ("setup", "update"): + if _IsDaemonRunning(): + print("Please stop the server before updating. Exiting.") + sys.exit(1) + Update() + sys.exit(0) + # Other parameters are handled below. + + +#========================================================== +# At this point we can assume that the implementation is available, +# so we can import it. +try: + from testrunner.server import constants + from testrunner.server import local_handler + from testrunner.server import main +except Exception, e: + print(e) + print("Failed to import implementation. Have you run 'setup'?") + sys.exit(1) + + +def _StartDaemon(daemon): + if not os.path.isdir(os.path.join(ROOT, "v8")): + print("No 'v8' working directory found. Have you run 'setup'?") + sys.exit(1) + daemon.start() + + +if __name__ == "__main__": + if len(sys.argv) == 2: + arg = sys.argv[1] + if arg == "start": + daemon = main.Server(PIDFILE, ROOT) + _StartDaemon(daemon) + elif arg == "stop": + daemon = main.Server(PIDFILE, ROOT) + daemon.stop() + elif arg == "restart": + daemon = main.Server(PIDFILE, ROOT) + daemon.stop() + _StartDaemon(daemon) + elif arg in ("help", "-h", "--help"): + _PrintUsage() + elif arg == "status": + if not _IsDaemonRunning(): + print("Server not running.") + else: + print(local_handler.LocalQuery([constants.REQUEST_STATUS])) + else: + print("Unknown command") + _PrintUsage() + sys.exit(2) + elif len(sys.argv) == 3: + arg = sys.argv[1] + if arg == "approve": + filename = sys.argv[2] + if not os.path.exists(filename): + print("%s does not exist.") + sys.exit(1) + filename = os.path.abspath(filename) + if _IsDaemonRunning(): + response = local_handler.LocalQuery([constants.ADD_TRUSTED, filename]) + else: + daemon = main.Server(PIDFILE, ROOT) + response = daemon.CopyToTrusted(filename) + print("Added certificate %s to trusted certificates." % response) + else: + print("Unknown command") + _PrintUsage() + sys.exit(2) + else: + print("Unknown command") + _PrintUsage() + sys.exit(2) + sys.exit(0) diff --git a/src/3rdparty/v8/tools/test-wrapper-gypbuild.py b/src/3rdparty/v8/tools/test-wrapper-gypbuild.py index eda2459..4dd6338 100755 --- a/src/3rdparty/v8/tools/test-wrapper-gypbuild.py +++ b/src/3rdparty/v8/tools/test-wrapper-gypbuild.py @@ -95,11 +95,14 @@ def BuildOptions(): default=1, type="int") result.add_option("--time", help="Print timing information after running", default=False, action="store_true") - result.add_option("--suppress-dialogs", help="Suppress Windows dialogs for crashing tests", - dest="suppress_dialogs", default=True, action="store_true") - result.add_option("--no-suppress-dialogs", help="Display Windows dialogs for crashing tests", - dest="suppress_dialogs", action="store_false") - result.add_option("--isolates", help="Whether to test isolates", default=False, action="store_true") + result.add_option("--suppress-dialogs", + help="Suppress Windows dialogs for crashing tests", + dest="suppress_dialogs", default=True, action="store_true") + result.add_option("--no-suppress-dialogs", + help="Display Windows dialogs for crashing tests", + dest="suppress_dialogs", action="store_false") + result.add_option("--isolates", help="Whether to test isolates", + default=False, action="store_true") result.add_option("--store-unexpected-output", help="Store the temporary JS files from tests that fails", dest="store_unexpected_output", default=True, action="store_true") @@ -112,9 +115,6 @@ def BuildOptions(): result.add_option("--nostress", help="Don't run crankshaft --always-opt --stress-op test", default=False, action="store_true") - result.add_option("--crankshaft", - help="Run with the --crankshaft flag", - default=False, action="store_true") result.add_option("--shard-count", help="Split testsuites into this number of shards", default=1, type="int") @@ -151,7 +151,8 @@ def ProcessOptions(options): print "Unknown mode %s" % mode return False for arch in options.arch: - if not arch in ['ia32', 'x64', 'arm', 'mips']: + if not arch in ['ia32', 'x64', 'arm', 'mipsel', 'android_arm', + 'android_ia32']: print "Unknown architecture %s" % arch return False if options.buildbot: @@ -199,8 +200,6 @@ def PassOnOptions(options): result += ['--stress-only'] if options.nostress: result += ['--nostress'] - if options.crankshaft: - result += ['--crankshaft'] if options.shard_count != 1: result += ['--shard-count=%s' % options.shard_count] if options.shard_run != 1: @@ -222,9 +221,11 @@ def Main(): if not options.no_presubmit: print ">>> running presubmit tests" - returncodes += subprocess.call([workspace + '/tools/presubmit.py']) + returncodes += subprocess.call([sys.executable, + workspace + '/tools/presubmit.py']) - args_for_children = [workspace + '/tools/test.py'] + PassOnOptions(options) + args_for_children = [sys.executable] + args_for_children += [workspace + '/tools/test.py'] + PassOnOptions(options) args_for_children += ['--no-build', '--build-system=gyp'] for arg in args: args_for_children += [arg] @@ -240,10 +241,11 @@ def Main(): shellpath = workspace + '/' + options.outdir + '/' + arch + '.' + mode env['LD_LIBRARY_PATH'] = shellpath + '/lib.target' shell = shellpath + "/d8" - child = subprocess.Popen(' '.join(args_for_children + - ['--arch=' + arch] + - ['--mode=' + mode] + - ['--shell=' + shell]), + cmdline = ' '.join(args_for_children + + ['--arch=' + arch] + + ['--mode=' + mode] + + ['--shell=' + shell]) + child = subprocess.Popen(cmdline, shell=True, cwd=workspace, env=env) diff --git a/src/3rdparty/v8/tools/test.py b/src/3rdparty/v8/tools/test.py index 0aacd99..b3b62b3 100755 --- a/src/3rdparty/v8/tools/test.py +++ b/src/3rdparty/v8/tools/test.py @@ -140,9 +140,9 @@ def EscapeCommand(command): parts = [] for part in command: if ' ' in part: - # Escape spaces. We may need to escape more characters for this - # to work properly. - parts.append('"%s"' % part) + # Escape spaces and double quotes. We may need to escape more characters + # for this to work properly. + parts.append('"%s"' % part.replace('"', '\\"')) else: parts.append(part) return " ".join(parts) @@ -299,8 +299,6 @@ class MonochromeProgressIndicator(CompactProgressIndicator): 'status_line': "[%(mins)02i:%(secs)02i|%%%(remaining) 4d|+%(passed) 4d|-%(failed) 4d]: %(test)s", 'stdout': '%s', 'stderr': '%s', - 'clear': lambda last_line_length: ("\r" + (" " * last_line_length) + "\r"), - 'max_length': 78 } super(MonochromeProgressIndicator, self).__init__(cases, templates) @@ -686,8 +684,9 @@ SUFFIX = { 'debug' : '_g', 'release' : '' } FLAGS = { - 'debug' : ['--nobreak-on-abort', '--enable-slow-asserts', '--debug-code', '--verify-heap'], - 'release' : ['--nobreak-on-abort']} + 'debug' : ['--nobreak-on-abort', '--nodead-code-elimination', + '--enable-slow-asserts', '--debug-code', '--verify-heap'], + 'release' : ['--nobreak-on-abort', '--nodead-code-elimination']} TIMEOUT_SCALEFACTOR = { 'debug' : 4, 'release' : 1 } @@ -1246,9 +1245,6 @@ def BuildOptions(): result.add_option("--nostress", help="Don't run crankshaft --always-opt --stress-op test", default=False, action="store_true") - result.add_option("--crankshaft", - help="Run with the --crankshaft flag", - default=False, action="store_true") result.add_option("--shard-count", help="Split testsuites into this number of shards", default=1, type="int") @@ -1286,7 +1282,7 @@ def ProcessOptions(options): options.scons_flags.append("arch=" + options.arch) # Simulators are slow, therefore allow a longer default timeout. if options.timeout == -1: - if options.arch == 'arm' or options.arch == 'mips': + if options.arch in ['android', 'arm', 'mipsel']: options.timeout = 2 * TIMEOUT_DEFAULT; else: options.timeout = TIMEOUT_DEFAULT; @@ -1300,11 +1296,6 @@ def ProcessOptions(options): VARIANT_FLAGS = [['--stress-opt', '--always-opt']] if options.nostress: VARIANT_FLAGS = [[],['--nocrankshaft']] - if options.crankshaft: - if options.special_command: - options.special_command += " --crankshaft" - else: - options.special_command = "@ --crankshaft" if options.shell.endswith("d8"): if options.special_command: options.special_command += " --test" @@ -1380,8 +1371,9 @@ def GetSpecialCommandProcessor(value): else: pos = value.find('@') import urllib - prefix = urllib.unquote(value[:pos]).split() - suffix = urllib.unquote(value[pos+1:]).split() + import shlex + prefix = shlex.split(urllib.unquote(value[:pos])) + suffix = shlex.split(urllib.unquote(value[pos+1:])) def ExpandCommand(args): return prefix + args + suffix return ExpandCommand @@ -1493,7 +1485,6 @@ def Main(): 'system': utils.GuessOS(), 'arch': options.arch, 'simulator': options.simulator, - 'crankshaft': options.crankshaft, 'isolates': options.isolates } test_list = root.ListTests([], path, context, mode, []) diff --git a/src/3rdparty/v8/tools/testrunner/README b/src/3rdparty/v8/tools/testrunner/README new file mode 100644 index 0000000..8f0c01f --- /dev/null +++ b/src/3rdparty/v8/tools/testrunner/README @@ -0,0 +1,174 @@ +Test suite runner for V8, including support for distributed running. +==================================================================== + + +Local usage instructions: +========================= + +Run the main script with --help to get detailed usage instructions: + +$ tools/run-tests.py --help + +The interface is mostly the same as it was for the old test runner. +You'll likely want something like this: + +$ tools/run-tests.py --nonetwork --arch ia32 --mode release + +--nonetwork is the default on Mac and Windows. If you don't specify --arch +and/or --mode, all available values will be used and run in turn (e.g., +omitting --mode from the above example will run ia32 in both Release and Debug +modes). + + +Networked usage instructions: +============================= + +Networked running is only supported on Linux currently. Make sure that all +machines participating in the cluster are binary-compatible (e.g. mixing +Ubuntu Lucid and Precise doesn't work). + +Setup: +------ + +1.) Copy tools/test-server.py to a new empty directory anywhere on your hard + drive (preferably not inside your V8 checkout just to keep things clean). + Please do create a copy, not just a symlink. + +2.) Navigate to the new directory and let the server setup itself: + +$ ./test-server.py setup + + This will install PIP and UltraJSON, create a V8 working directory, and + generate a keypair. + +3.) Swap public keys with someone who's already part of the networked cluster. + +$ cp trusted/`cat data/mypubkey`.pem /where/peers/can/see/it/myname.pem +$ ./test-server.py approve /wherever/they/put/it/yourname.pem + + +Usage: +------ + +1.) Start your server: + +$ ./test-server.py start + +2.) (Optionally) inspect the server's status: + +$ ./test-server.py status + +3.) From your regular V8 working directory, run tests: + +$ tool/run-tests.py --arch ia32 --mode debug + +4.) (Optionally) enjoy the speeeeeeeeeeeeeeeed + + +Architecture overview: +====================== + +Code organization: +------------------ + +This section is written from the point of view of the tools/ directory. + +./run-tests.py: + Main script. Parses command-line options and drives the test execution + procedure from a high level. Imports the actual implementation of all + steps from the testrunner/ directory. + +./test-server.py: + Interface to interact with the server. Contains code to setup the server's + working environment and can start and stop server daemon processes. + Imports some stuff from the testrunner/server/ directory. + +./testrunner/local/*: + Implementation needed to run tests locally. Used by run-tests.py. Inspired by + (and partly copied verbatim from) the original test.py script. + +./testrunner/local/old_statusfile.py: + Provides functionality to read an old-style <testsuite>.status file and + convert it to new-style syntax. This can be removed once the new-style + syntax becomes authoritative (and old-style syntax is no longer supported). + ./status-file-converter.py provides a stand-alone interface to this. + +./testrunner/objects/*: + A bunch of data container classes, used by the scripts in the various other + directories; serializable for transmission over the network. + +./testrunner/network/*: + Equivalents and extensions of some of the functionality in ./testrunner/local/ + as required when dispatching tests to peers on the network. + +./testrunner/network/network_execution.py: + Drop-in replacement for ./testrunner/local/execution that distributes + test jobs to network peers instead of running them locally. + +./testrunner/network/endpoint.py: + Receiving end of a network distributed job, uses the implementation + in ./testrunner/local/execution.py for actually running the tests. + +./testrunner/server/*: + Implementation of the daemon that accepts and runs test execution jobs from + peers on the network. Should ideally have no dependencies on any of the other + directories, but that turned out to be impractical, so there are a few + exceptions. + +./testrunner/server/compression.py: + Defines a wrapper around Python TCP sockets that provides JSON based + serialization, gzip based compression, and ensures message completeness. + + +Networking architecture: +------------------------ + +The distribution stuff is designed to be a layer between deciding which tests +to run on the one side, and actually running them on the other. The frontend +that the user interacts with is the same for local and networked execution, +and the actual test execution and result gathering code is the same too. + +The server daemon starts four separate servers, each listening on another port: +- "Local": Communication with a run-tests.py script running on the same host. + The test driving script e.g. needs to ask for available peers. It then talks + to those peers directly (one of them will be the locally running server). +- "Work": Listens for test job requests from run-tests.py scripts on the network + (including localhost). Accepts an arbitrary number of connections at the + same time, but only works on them in a serialized fashion. +- "Status": Used for communication with other servers on the network, e.g. for + exchanging trusted public keys to create the transitive trust closure. +- "Discovery": Used to detect presence of other peers on the network. + In contrast to the other three, this uses UDP (as opposed to TCP). + + +Give us a diagram! We love diagrams! +------------------------------------ + . + Machine A . Machine B + . ++------------------------------+ . +| run-tests.py | . +| with flag: | . +|--nonetwork --network | . +| | / | | . +| | / | | . +| v / v | . +|BACKEND / distribution | . ++--------- / --------| \ ------+ . + / | \_____________________ + / | . \ + / | . \ ++----- v ----------- v --------+ . +---- v -----------------------+ +| LocalHandler | WorkHandler | . | WorkHandler | LocalHandler | +| | | | . | | | | +| | v | . | v | | +| | BACKEND | . | BACKEND | | +|------------- +---------------| . |---------------+--------------| +| Discovery | StatusHandler <----------> StatusHandler | Discovery | ++---- ^ -----------------------+ . +-------------------- ^ -------+ + | . | + +---------------------------------------------------------+ + +Note that the three occurrences of "BACKEND" are the same code +(testrunner/local/execution.py and its imports), but running from three +distinct directories (and on two different machines). diff --git a/src/3rdparty/v8/tools/testrunner/__init__.py b/src/3rdparty/v8/tools/testrunner/__init__.py new file mode 100644 index 0000000..202a262 --- /dev/null +++ b/src/3rdparty/v8/tools/testrunner/__init__.py @@ -0,0 +1,26 @@ +# Copyright 2012 the V8 project authors. All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/src/3rdparty/v8/tools/testrunner/local/__init__.py b/src/3rdparty/v8/tools/testrunner/local/__init__.py new file mode 100644 index 0000000..202a262 --- /dev/null +++ b/src/3rdparty/v8/tools/testrunner/local/__init__.py @@ -0,0 +1,26 @@ +# Copyright 2012 the V8 project authors. All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/src/3rdparty/v8/tools/testrunner/local/commands.py b/src/3rdparty/v8/tools/testrunner/local/commands.py new file mode 100644 index 0000000..01f170d --- /dev/null +++ b/src/3rdparty/v8/tools/testrunner/local/commands.py @@ -0,0 +1,153 @@ +# Copyright 2012 the V8 project authors. All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +import os +import signal +import subprocess +import sys +import tempfile +import time + +from ..local import utils +from ..objects import output + + +def KillProcessWithID(pid): + if utils.IsWindows(): + os.popen('taskkill /T /F /PID %d' % pid) + else: + os.kill(pid, signal.SIGTERM) + + +MAX_SLEEP_TIME = 0.1 +INITIAL_SLEEP_TIME = 0.0001 +SLEEP_TIME_FACTOR = 1.25 + +SEM_INVALID_VALUE = -1 +SEM_NOGPFAULTERRORBOX = 0x0002 # Microsoft Platform SDK WinBase.h + + +def Win32SetErrorMode(mode): + prev_error_mode = SEM_INVALID_VALUE + try: + import ctypes + prev_error_mode = \ + ctypes.windll.kernel32.SetErrorMode(mode) #@UndefinedVariable + except ImportError: + pass + return prev_error_mode + + +def RunProcess(verbose, timeout, args, **rest): + if verbose: print "#", " ".join(args) + popen_args = args + prev_error_mode = SEM_INVALID_VALUE + if utils.IsWindows(): + popen_args = subprocess.list2cmdline(args) + # Try to change the error mode to avoid dialogs on fatal errors. Don't + # touch any existing error mode flags by merging the existing error mode. + # See http://blogs.msdn.com/oldnewthing/archive/2004/07/27/198410.aspx. + error_mode = SEM_NOGPFAULTERRORBOX + prev_error_mode = Win32SetErrorMode(error_mode) + Win32SetErrorMode(error_mode | prev_error_mode) + process = subprocess.Popen( + shell=utils.IsWindows(), + args=popen_args, + **rest + ) + if (utils.IsWindows() and prev_error_mode != SEM_INVALID_VALUE): + Win32SetErrorMode(prev_error_mode) + # Compute the end time - if the process crosses this limit we + # consider it timed out. + if timeout is None: end_time = None + else: end_time = time.time() + timeout + timed_out = False + # Repeatedly check the exit code from the process in a + # loop and keep track of whether or not it times out. + exit_code = None + sleep_time = INITIAL_SLEEP_TIME + try: + while exit_code is None: + if (not end_time is None) and (time.time() >= end_time): + # Kill the process and wait for it to exit. + KillProcessWithID(process.pid) + exit_code = process.wait() + timed_out = True + else: + exit_code = process.poll() + time.sleep(sleep_time) + sleep_time = sleep_time * SLEEP_TIME_FACTOR + if sleep_time > MAX_SLEEP_TIME: + sleep_time = MAX_SLEEP_TIME + return (exit_code, timed_out) + except KeyboardInterrupt: + raise + + +def PrintError(string): + sys.stderr.write(string) + sys.stderr.write("\n") + + +def CheckedUnlink(name): + # On Windows, when run with -jN in parallel processes, + # OS often fails to unlink the temp file. Not sure why. + # Need to retry. + # Idea from https://bugs.webkit.org/attachment.cgi?id=75982&action=prettypatch + retry_count = 0 + while retry_count < 30: + try: + os.unlink(name) + return + except OSError, e: + retry_count += 1 + time.sleep(retry_count * 0.1) + PrintError("os.unlink() " + str(e)) + + +def Execute(args, verbose=False, timeout=None): + args = [ c for c in args if c != "" ] + (fd_out, outname) = tempfile.mkstemp() + (fd_err, errname) = tempfile.mkstemp() + try: + (exit_code, timed_out) = RunProcess( + verbose, + timeout, + args=args, + stdout=fd_out, + stderr=fd_err + ) + except: + raise + os.close(fd_out) + os.close(fd_err) + out = file(outname).read() + errors = file(errname).read() + CheckedUnlink(outname) + CheckedUnlink(errname) + return output.Output(exit_code, timed_out, out, errors) diff --git a/src/3rdparty/v8/tools/testrunner/local/execution.py b/src/3rdparty/v8/tools/testrunner/local/execution.py new file mode 100644 index 0000000..6004367 --- /dev/null +++ b/src/3rdparty/v8/tools/testrunner/local/execution.py @@ -0,0 +1,182 @@ +# Copyright 2012 the V8 project authors. All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +import multiprocessing +import os +import threading +import time + +from . import commands +from . import utils + + +BREAK_NOW = -1 +EXCEPTION = -2 + + +class Job(object): + def __init__(self, command, dep_command, test_id, timeout, verbose): + self.command = command + self.dep_command = dep_command + self.id = test_id + self.timeout = timeout + self.verbose = verbose + + +def RunTest(job): + try: + start_time = time.time() + if job.dep_command is not None: + dep_output = commands.Execute(job.dep_command, job.verbose, job.timeout) + # TODO(jkummerow): We approximate the test suite specific function + # IsFailureOutput() by just checking the exit code here. Currently + # only cctests define dependencies, for which this simplification is + # correct. + if dep_output.exit_code != 0: + return (job.id, dep_output, time.time() - start_time) + output = commands.Execute(job.command, job.verbose, job.timeout) + return (job.id, output, time.time() - start_time) + except KeyboardInterrupt: + return (-1, BREAK_NOW, 0) + except Exception, e: + print(">>> EXCEPTION: %s" % e) + return (-1, EXCEPTION, 0) + + +class Runner(object): + + def __init__(self, suites, progress_indicator, context): + self.tests = [ t for s in suites for t in s.tests ] + self._CommonInit(len(self.tests), progress_indicator, context) + + def _CommonInit(self, num_tests, progress_indicator, context): + self.indicator = progress_indicator + progress_indicator.runner = self + self.context = context + self.succeeded = 0 + self.total = num_tests + self.remaining = num_tests + self.failed = [] + self.crashed = 0 + self.terminate = False + self.lock = threading.Lock() + + def Run(self, jobs): + self.indicator.Starting() + self._RunInternal(jobs) + self.indicator.Done() + if self.failed: + return 1 + return 0 + + def _RunInternal(self, jobs): + pool = multiprocessing.Pool(processes=jobs) + test_map = {} + queue = [] + queued_exception = None + for test in self.tests: + assert test.id >= 0 + test_map[test.id] = test + try: + command = self.GetCommand(test) + except Exception, e: + # If this failed, save the exception and re-raise it later (after + # all other tests have had a chance to run). + queued_exception = e + continue + timeout = self.context.timeout + if ("--stress-opt" in test.flags or + "--stress-opt" in self.context.mode_flags or + "--stress-opt" in self.context.extra_flags): + timeout *= 4 + if test.dependency is not None: + dep_command = [ c.replace(test.path, test.dependency) for c in command ] + else: + dep_command = None + job = Job(command, dep_command, test.id, timeout, self.context.verbose) + queue.append(job) + try: + kChunkSize = 1 + it = pool.imap_unordered(RunTest, queue, kChunkSize) + for result in it: + test_id = result[0] + if test_id < 0: + if result[1] == BREAK_NOW: + self.terminate = True + else: + continue + if self.terminate: + pool.terminate() + pool.join() + raise BreakNowException("User pressed Ctrl+C or IO went wrong") + test = test_map[test_id] + self.indicator.AboutToRun(test) + test.output = result[1] + test.duration = result[2] + if test.suite.HasUnexpectedOutput(test): + self.failed.append(test) + if test.output.HasCrashed(): + self.crashed += 1 + else: + self.succeeded += 1 + self.remaining -= 1 + self.indicator.HasRun(test) + except KeyboardInterrupt: + pool.terminate() + pool.join() + raise + except Exception, e: + print("Exception: %s" % e) + pool.terminate() + pool.join() + raise + if queued_exception: + raise queued_exception + return + + + def GetCommand(self, test): + d8testflag = [] + shell = test.suite.shell() + if shell == "d8": + d8testflag = ["--test"] + if utils.IsWindows(): + shell += ".exe" + cmd = ([self.context.command_prefix] + + [os.path.abspath(os.path.join(self.context.shell_dir, shell))] + + d8testflag + + test.suite.GetFlagsForTestCase(test, self.context) + + [self.context.extra_flags]) + return cmd + + +class BreakNowException(Exception): + def __init__(self, value): + self.value = value + def __str__(self): + return repr(self.value) diff --git a/src/3rdparty/v8/tools/testrunner/local/old_statusfile.py b/src/3rdparty/v8/tools/testrunner/local/old_statusfile.py new file mode 100644 index 0000000..a16941b --- /dev/null +++ b/src/3rdparty/v8/tools/testrunner/local/old_statusfile.py @@ -0,0 +1,460 @@ +# Copyright 2012 the V8 project authors. All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +import cStringIO +import re + +# These outcomes can occur in a TestCase's outcomes list: +SKIP = 'SKIP' +FAIL = 'FAIL' +PASS = 'PASS' +OKAY = 'OKAY' +TIMEOUT = 'TIMEOUT' +CRASH = 'CRASH' +SLOW = 'SLOW' +# These are just for the status files and are mapped below in DEFS: +FAIL_OK = 'FAIL_OK' +PASS_OR_FAIL = 'PASS_OR_FAIL' + +KEYWORDS = {SKIP: SKIP, + FAIL: FAIL, + PASS: PASS, + OKAY: OKAY, + TIMEOUT: TIMEOUT, + CRASH: CRASH, + SLOW: SLOW, + FAIL_OK: FAIL_OK, + PASS_OR_FAIL: PASS_OR_FAIL} + +class Expression(object): + pass + + +class Constant(Expression): + + def __init__(self, value): + self.value = value + + def Evaluate(self, env, defs): + return self.value + + +class Variable(Expression): + + def __init__(self, name): + self.name = name + + def GetOutcomes(self, env, defs): + if self.name in env: return set([env[self.name]]) + else: return set([]) + + def Evaluate(self, env, defs): + return env[self.name] + + def __str__(self): + return self.name + + def string(self, logical): + return self.__str__() + + +class Outcome(Expression): + + def __init__(self, name): + self.name = name + + def GetOutcomes(self, env, defs): + if self.name in defs: + return defs[self.name].GetOutcomes(env, defs) + else: + return set([self.name]) + + def __str__(self): + if self.name in KEYWORDS: + return "%s" % KEYWORDS[self.name] + return "'%s'" % self.name + + def string(self, logical): + if logical: + return "%s" % self.name + return self.__str__() + + +class Operation(Expression): + + def __init__(self, left, op, right): + self.left = left + self.op = op + self.right = right + + def Evaluate(self, env, defs): + if self.op == '||' or self.op == ',': + return self.left.Evaluate(env, defs) or self.right.Evaluate(env, defs) + elif self.op == 'if': + return False + elif self.op == '==': + return not self.left.GetOutcomes(env, defs).isdisjoint(self.right.GetOutcomes(env, defs)) + elif self.op == '!=': + return self.left.GetOutcomes(env, defs).isdisjoint(self.right.GetOutcomes(env, defs)) + else: + assert self.op == '&&' + return self.left.Evaluate(env, defs) and self.right.Evaluate(env, defs) + + def GetOutcomes(self, env, defs): + if self.op == '||' or self.op == ',': + return self.left.GetOutcomes(env, defs) | self.right.GetOutcomes(env, defs) + elif self.op == 'if': + if self.right.Evaluate(env, defs): return self.left.GetOutcomes(env, defs) + else: return set([]) + else: + assert self.op == '&&' + return self.left.GetOutcomes(env, defs) & self.right.GetOutcomes(env, defs) + + def __str__(self): + return self.string(False) + + def string(self, logical=False): + if self.op == 'if': + return "['%s', %s]" % (self.right.string(True), self.left.string(logical)) + elif self.op == "||" or self.op == ",": + if logical: + return "%s or %s" % (self.left.string(True), self.right.string(True)) + else: + return "%s, %s" % (self.left, self.right) + elif self.op == "&&": + return "%s and %s" % (self.left.string(True), self.right.string(True)) + return "%s %s %s" % (self.left.string(logical), self.op, + self.right.string(logical)) + + +def IsAlpha(string): + for char in string: + if not (char.isalpha() or char.isdigit() or char == '_'): + return False + return True + + +class Tokenizer(object): + """A simple string tokenizer that chops expressions into variables, + parens and operators""" + + def __init__(self, expr): + self.index = 0 + self.expr = expr + self.length = len(expr) + self.tokens = None + + def Current(self, length=1): + if not self.HasMore(length): return "" + return self.expr[self.index:self.index + length] + + def HasMore(self, length=1): + return self.index < self.length + (length - 1) + + def Advance(self, count=1): + self.index = self.index + count + + def AddToken(self, token): + self.tokens.append(token) + + def SkipSpaces(self): + while self.HasMore() and self.Current().isspace(): + self.Advance() + + def Tokenize(self): + self.tokens = [ ] + while self.HasMore(): + self.SkipSpaces() + if not self.HasMore(): + return None + if self.Current() == '(': + self.AddToken('(') + self.Advance() + elif self.Current() == ')': + self.AddToken(')') + self.Advance() + elif self.Current() == '$': + self.AddToken('$') + self.Advance() + elif self.Current() == ',': + self.AddToken(',') + self.Advance() + elif IsAlpha(self.Current()): + buf = "" + while self.HasMore() and IsAlpha(self.Current()): + buf += self.Current() + self.Advance() + self.AddToken(buf) + elif self.Current(2) == '&&': + self.AddToken('&&') + self.Advance(2) + elif self.Current(2) == '||': + self.AddToken('||') + self.Advance(2) + elif self.Current(2) == '==': + self.AddToken('==') + self.Advance(2) + elif self.Current(2) == '!=': + self.AddToken('!=') + self.Advance(2) + else: + return None + return self.tokens + + +class Scanner(object): + """A simple scanner that can serve out tokens from a given list""" + + def __init__(self, tokens): + self.tokens = tokens + self.length = len(tokens) + self.index = 0 + + def HasMore(self): + return self.index < self.length + + def Current(self): + return self.tokens[self.index] + + def Advance(self): + self.index = self.index + 1 + + +def ParseAtomicExpression(scan): + if scan.Current() == "true": + scan.Advance() + return Constant(True) + elif scan.Current() == "false": + scan.Advance() + return Constant(False) + elif IsAlpha(scan.Current()): + name = scan.Current() + scan.Advance() + return Outcome(name) + elif scan.Current() == '$': + scan.Advance() + if not IsAlpha(scan.Current()): + return None + name = scan.Current() + scan.Advance() + return Variable(name.lower()) + elif scan.Current() == '(': + scan.Advance() + result = ParseLogicalExpression(scan) + if (not result) or (scan.Current() != ')'): + return None + scan.Advance() + return result + else: + return None + + +BINARIES = ['==', '!='] +def ParseOperatorExpression(scan): + left = ParseAtomicExpression(scan) + if not left: return None + while scan.HasMore() and (scan.Current() in BINARIES): + op = scan.Current() + scan.Advance() + right = ParseOperatorExpression(scan) + if not right: + return None + left = Operation(left, op, right) + return left + + +def ParseConditionalExpression(scan): + left = ParseOperatorExpression(scan) + if not left: return None + while scan.HasMore() and (scan.Current() == 'if'): + scan.Advance() + right = ParseOperatorExpression(scan) + if not right: + return None + left = Operation(left, 'if', right) + return left + + +LOGICALS = ["&&", "||", ","] +def ParseLogicalExpression(scan): + left = ParseConditionalExpression(scan) + if not left: return None + while scan.HasMore() and (scan.Current() in LOGICALS): + op = scan.Current() + scan.Advance() + right = ParseConditionalExpression(scan) + if not right: + return None + left = Operation(left, op, right) + return left + + +def ParseCondition(expr): + """Parses a logical expression into an Expression object""" + tokens = Tokenizer(expr).Tokenize() + if not tokens: + print "Malformed expression: '%s'" % expr + return None + scan = Scanner(tokens) + ast = ParseLogicalExpression(scan) + if not ast: + print "Malformed expression: '%s'" % expr + return None + if scan.HasMore(): + print "Malformed expression: '%s'" % expr + return None + return ast + + +class Section(object): + """A section of the configuration file. Sections are enabled or + disabled prior to running the tests, based on their conditions""" + + def __init__(self, condition): + self.condition = condition + self.rules = [ ] + + def AddRule(self, rule): + self.rules.append(rule) + + +class Rule(object): + """A single rule that specifies the expected outcome for a single + test.""" + + def __init__(self, raw_path, path, value): + self.raw_path = raw_path + self.path = path + self.value = value + + def GetOutcomes(self, env, defs): + return self.value.GetOutcomes(env, defs) + + def Contains(self, path): + if len(self.path) > len(path): + return False + for i in xrange(len(self.path)): + if not self.path[i].match(path[i]): + return False + return True + + +HEADER_PATTERN = re.compile(r'\[([^]]+)\]') +RULE_PATTERN = re.compile(r'\s*([^: ]*)\s*:(.*)') +DEF_PATTERN = re.compile(r'^def\s*(\w+)\s*=(.*)$') +PREFIX_PATTERN = re.compile(r'^\s*prefix\s+([\w\_\.\-\/]+)$') + + +class ConvertNotation(object): + def __init__(self, path): + self.path = path + self.indent = "" + self.comment = [] + self.init = False + self.section = False + self.out = cStringIO.StringIO() + + def OpenGlobal(self): + if self.init: return + self.WriteComment() + print >> self.out, "[" + self.init = True + + def CloseGlobal(self): + if not self.init: return + print >> self.out, "]" + self.init = False + + def OpenSection(self, condition="ALWAYS"): + if self.section: return + self.OpenGlobal() + if type(condition) != str: + condition = "'%s'" % condition.string(True) + print >> self.out, "%s[%s, {" % (self.indent, condition) + self.indent += " " * 2 + self.section = condition + + def CloseSection(self): + if not self.section: return + self.indent = self.indent[:-2] + print >> self.out, "%s}], # %s" % (self.indent, self.section) + self.section = False + + def WriteComment(self): + if not self.comment: return + for c in self.comment: + if len(c.strip()) == 0: + print >> self.out, "" + else: + print >> self.out, "%s%s" % (self.indent, c), + self.comment = [] + + def GetOutput(self): + with open(self.path) as f: + for line in f: + if line[0] == '#': + self.comment += [line] + continue + if len(line.strip()) == 0: + self.comment += [line] + continue + header_match = HEADER_PATTERN.match(line) + if header_match: + condition = ParseCondition(header_match.group(1).strip()) + self.CloseSection() + self.WriteComment() + self.OpenSection(condition) + continue + rule_match = RULE_PATTERN.match(line) + if rule_match: + self.OpenSection() + self.WriteComment() + path = rule_match.group(1).strip() + value_str = rule_match.group(2).strip() + comment = "" + if '#' in value_str: + pos = value_str.find('#') + comment = " %s" % value_str[pos:].strip() + value_str = value_str[:pos].strip() + value = ParseCondition(value_str) + print >> self.out, ("%s'%s': [%s],%s" % + (self.indent, path, value, comment)) + continue + def_match = DEF_PATTERN.match(line) + if def_match: + # Custom definitions are deprecated. + continue + prefix_match = PREFIX_PATTERN.match(line) + if prefix_match: + continue + print "Malformed line: '%s'." % line + self.CloseSection() + self.CloseGlobal() + result = self.out.getvalue() + self.out.close() + return result diff --git a/src/3rdparty/v8/tools/testrunner/local/progress.py b/src/3rdparty/v8/tools/testrunner/local/progress.py new file mode 100644 index 0000000..9075a95 --- /dev/null +++ b/src/3rdparty/v8/tools/testrunner/local/progress.py @@ -0,0 +1,238 @@ +# Copyright 2012 the V8 project authors. All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +import sys +import time + +def EscapeCommand(command): + parts = [] + for part in command: + if ' ' in part: + # Escape spaces. We may need to escape more characters for this + # to work properly. + parts.append('"%s"' % part) + else: + parts.append(part) + return " ".join(parts) + + +class ProgressIndicator(object): + + def __init__(self): + self.runner = None + + def Starting(self): + pass + + def Done(self): + pass + + def AboutToRun(self, test): + pass + + def HasRun(self, test): + pass + + def PrintFailureHeader(self, test): + if test.suite.IsNegativeTest(test): + negative_marker = '[negative] ' + else: + negative_marker = '' + print "=== %(label)s %(negative)s===" % { + 'label': test.GetLabel(), + 'negative': negative_marker + } + + +class SimpleProgressIndicator(ProgressIndicator): + """Abstract base class for {Verbose,Dots}ProgressIndicator""" + + def Starting(self): + print 'Running %i tests' % self.runner.total + + def Done(self): + print + for failed in self.runner.failed: + self.PrintFailureHeader(failed) + if failed.output.stderr: + print "--- stderr ---" + print failed.output.stderr.strip() + if failed.output.stdout: + print "--- stdout ---" + print failed.output.stdout.strip() + print "Command: %s" % EscapeCommand(self.runner.GetCommand(failed)) + if failed.output.HasCrashed(): + print "--- CRASHED ---" + if failed.output.HasTimedOut(): + print "--- TIMEOUT ---" + if len(self.runner.failed) == 0: + print "===" + print "=== All tests succeeded" + print "===" + else: + print + print "===" + print "=== %i tests failed" % len(self.runner.failed) + if self.runner.crashed > 0: + print "=== %i tests CRASHED" % self.runner.crashed + print "===" + + +class VerboseProgressIndicator(SimpleProgressIndicator): + + def AboutToRun(self, test): + print 'Starting %s...' % test.GetLabel() + sys.stdout.flush() + + def HasRun(self, test): + if test.suite.HasUnexpectedOutput(test): + if test.output.HasCrashed(): + outcome = 'CRASH' + else: + outcome = 'FAIL' + else: + outcome = 'pass' + print 'Done running %s: %s' % (test.GetLabel(), outcome) + + +class DotsProgressIndicator(SimpleProgressIndicator): + + def HasRun(self, test): + total = self.runner.succeeded + len(self.runner.failed) + if (total > 1) and (total % 50 == 1): + sys.stdout.write('\n') + if test.suite.HasUnexpectedOutput(test): + if test.output.HasCrashed(): + sys.stdout.write('C') + sys.stdout.flush() + elif test.output.HasTimedOut(): + sys.stdout.write('T') + sys.stdout.flush() + else: + sys.stdout.write('F') + sys.stdout.flush() + else: + sys.stdout.write('.') + sys.stdout.flush() + + +class CompactProgressIndicator(ProgressIndicator): + """Abstract base class for {Color,Monochrome}ProgressIndicator""" + + def __init__(self, templates): + super(CompactProgressIndicator, self).__init__() + self.templates = templates + self.last_status_length = 0 + self.start_time = time.time() + + def Done(self): + self.PrintProgress('Done') + print "" # Line break. + + def AboutToRun(self, test): + self.PrintProgress(test.GetLabel()) + + def HasRun(self, test): + if test.suite.HasUnexpectedOutput(test): + self.ClearLine(self.last_status_length) + self.PrintFailureHeader(test) + stdout = test.output.stdout.strip() + if len(stdout): + print self.templates['stdout'] % stdout + stderr = test.output.stderr.strip() + if len(stderr): + print self.templates['stderr'] % stderr + print "Command: %s" % EscapeCommand(self.runner.GetCommand(test)) + if test.output.HasCrashed(): + print "exit code: %d" % test.output.exit_code + print "--- CRASHED ---" + if test.output.HasTimedOut(): + print "--- TIMEOUT ---" + + def Truncate(self, string, length): + if length and (len(string) > (length - 3)): + return string[:(length - 3)] + "..." + else: + return string + + def PrintProgress(self, name): + self.ClearLine(self.last_status_length) + elapsed = time.time() - self.start_time + status = self.templates['status_line'] % { + 'passed': self.runner.succeeded, + 'remaining': (((self.runner.total - self.runner.remaining) * 100) // + self.runner.total), + 'failed': len(self.runner.failed), + 'test': name, + 'mins': int(elapsed) / 60, + 'secs': int(elapsed) % 60 + } + status = self.Truncate(status, 78) + self.last_status_length = len(status) + print status, + sys.stdout.flush() + + +class ColorProgressIndicator(CompactProgressIndicator): + + def __init__(self): + templates = { + 'status_line': ("[%(mins)02i:%(secs)02i|" + "\033[34m%%%(remaining) 4d\033[0m|" + "\033[32m+%(passed) 4d\033[0m|" + "\033[31m-%(failed) 4d\033[0m]: %(test)s"), + 'stdout': "\033[1m%s\033[0m", + 'stderr': "\033[31m%s\033[0m", + } + super(ColorProgressIndicator, self).__init__(templates) + + def ClearLine(self, last_line_length): + print "\033[1K\r", + + +class MonochromeProgressIndicator(CompactProgressIndicator): + + def __init__(self): + templates = { + 'status_line': ("[%(mins)02i:%(secs)02i|%%%(remaining) 4d|" + "+%(passed) 4d|-%(failed) 4d]: %(test)s"), + 'stdout': '%s', + 'stderr': '%s', + } + super(MonochromeProgressIndicator, self).__init__(templates) + + def ClearLine(self, last_line_length): + print ("\r" + (" " * last_line_length) + "\r"), + + +PROGRESS_INDICATORS = { + 'verbose': VerboseProgressIndicator, + 'dots': DotsProgressIndicator, + 'color': ColorProgressIndicator, + 'mono': MonochromeProgressIndicator +} diff --git a/src/3rdparty/v8/tools/testrunner/local/statusfile.py b/src/3rdparty/v8/tools/testrunner/local/statusfile.py new file mode 100644 index 0000000..bf1de45 --- /dev/null +++ b/src/3rdparty/v8/tools/testrunner/local/statusfile.py @@ -0,0 +1,145 @@ +# Copyright 2012 the V8 project authors. All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +# These imports are required for the on-demand conversion from +# old to new status file format. +from os.path import exists +from os.path import getmtime + +from . import old_statusfile + + +# These outcomes can occur in a TestCase's outcomes list: +SKIP = "SKIP" +FAIL = "FAIL" +PASS = "PASS" +OKAY = "OKAY" +TIMEOUT = "TIMEOUT" +CRASH = "CRASH" +SLOW = "SLOW" +# These are just for the status files and are mapped below in DEFS: +FAIL_OK = "FAIL_OK" +PASS_OR_FAIL = "PASS_OR_FAIL" + +ALWAYS = "ALWAYS" + +KEYWORDS = {} +for key in [SKIP, FAIL, PASS, OKAY, TIMEOUT, CRASH, SLOW, FAIL_OK, + PASS_OR_FAIL, ALWAYS]: + KEYWORDS[key] = key + +DEFS = {FAIL_OK: [FAIL, OKAY], + PASS_OR_FAIL: [PASS, FAIL]} + +# Support arches, modes to be written as keywords instead of strings. +VARIABLES = {ALWAYS: True} +for var in ["debug", "release", "android_arm", "android_ia32", "arm", "ia32", + "mipsel", "x64"]: + VARIABLES[var] = var + + +def DoSkip(outcomes): + return SKIP in outcomes or SLOW in outcomes + + +def IsFlaky(outcomes): + return ((PASS in outcomes) and (FAIL in outcomes) and + (not CRASH in outcomes) and (not OKAY in outcomes)) + + +def IsFailOk(outcomes): + return (FAIL in outcomes) and (OKAY in outcomes) + + +def _AddOutcome(result, new): + global DEFS + if new in DEFS: + mapped = DEFS[new] + if type(mapped) == list: + for m in mapped: + _AddOutcome(result, m) + elif type(mapped) == str: + _AddOutcome(result, mapped) + else: + result.add(new) + + +def _ParseOutcomeList(rule, outcomes, target_dict, variables): + result = set([]) + if type(outcomes) == str: + outcomes = [outcomes] + for item in outcomes: + if type(item) == str: + _AddOutcome(result, item) + elif type(item) == list: + if not eval(item[0], variables): continue + for outcome in item[1:]: + assert type(outcome) == str + _AddOutcome(result, outcome) + else: + assert False + if len(result) == 0: return + if rule in target_dict: + target_dict[rule] |= result + else: + target_dict[rule] = result + + +def ReadStatusFile(path, variables): + # As long as the old-format .status files are authoritative, just + # create the converted version on demand and cache it to speed up + # subsequent runs. + if path.endswith(".status"): + newpath = path + "2" + if not exists(newpath) or getmtime(newpath) < getmtime(path): + print "Converting status file." + converted = old_statusfile.ConvertNotation(path).GetOutput() + with open(newpath, 'w') as f: + f.write(converted) + path = newpath + + with open(path) as f: + global KEYWORDS + contents = eval(f.read(), KEYWORDS) + + rules = {} + wildcards = {} + variables.update(VARIABLES) + for section in contents: + assert type(section) == list + assert len(section) == 2 + if not eval(section[0], variables): continue + section = section[1] + assert type(section) == dict + for rule in section: + assert type(rule) == str + if rule[-1] == '*': + _ParseOutcomeList(rule, section[rule], wildcards, variables) + else: + _ParseOutcomeList(rule, section[rule], rules, variables) + return rules, wildcards diff --git a/src/3rdparty/v8/tools/testrunner/local/testsuite.py b/src/3rdparty/v8/tools/testrunner/local/testsuite.py new file mode 100644 index 0000000..de5cddd --- /dev/null +++ b/src/3rdparty/v8/tools/testrunner/local/testsuite.py @@ -0,0 +1,184 @@ +# Copyright 2012 the V8 project authors. All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +import imp +import os + +from . import statusfile + +class TestSuite(object): + + @staticmethod + def LoadTestSuite(root): + name = root.split(os.path.sep)[-1] + f = None + try: + (f, pathname, description) = imp.find_module("testcfg", [root]) + module = imp.load_module("testcfg", f, pathname, description) + suite = module.GetSuite(name, root) + finally: + if f: + f.close() + return suite + + def __init__(self, name, root): + self.name = name # string + self.root = root # string containing path + self.tests = None # list of TestCase objects + self.rules = None # dictionary mapping test path to list of outcomes + self.wildcards = None # dictionary mapping test paths to list of outcomes + self.total_duration = None # float, assigned on demand + + def shell(self): + return "d8" + + def suffix(self): + return ".js" + + def status_file(self): + return "%s/%s.status" % (self.root, self.name) + + # Used in the status file and for stdout printing. + def CommonTestName(self, testcase): + return testcase.path + + def ListTests(self, context): + raise NotImplementedError + + def VariantFlags(self): + return None + + def DownloadData(self): + pass + + def ReadStatusFile(self, variables): + (self.rules, self.wildcards) = \ + statusfile.ReadStatusFile(self.status_file(), variables) + + def ReadTestCases(self, context): + self.tests = self.ListTests(context) + + def FilterTestCasesByStatus(self, warn_unused_rules): + filtered = [] + used_rules = set() + for t in self.tests: + testname = self.CommonTestName(t) + if testname in self.rules: + used_rules.add(testname) + outcomes = self.rules[testname] + t.outcomes = outcomes # Even for skipped tests, as the TestCase + # object stays around and PrintReport() uses it. + if statusfile.DoSkip(outcomes): + continue # Don't add skipped tests to |filtered|. + if len(self.wildcards) != 0: + skip = False + for rule in self.wildcards: + assert rule[-1] == '*' + if testname.startswith(rule[:-1]): + used_rules.add(rule) + outcomes = self.wildcards[rule] + t.outcomes = outcomes + if statusfile.DoSkip(outcomes): + skip = True + break # "for rule in self.wildcards" + if skip: continue # "for t in self.tests" + filtered.append(t) + self.tests = filtered + + if not warn_unused_rules: + return + + for rule in self.rules: + if rule not in used_rules: + print("Unused rule: %s -> %s" % (rule, self.rules[rule])) + for rule in self.wildcards: + if rule not in used_rules: + print("Unused rule: %s -> %s" % (rule, self.wildcards[rule])) + + def FilterTestCasesByArgs(self, args): + filtered = [] + filtered_args = [] + for a in args: + argpath = a.split(os.path.sep) + if argpath[0] != self.name: + continue + if len(argpath) == 1 or (len(argpath) == 2 and argpath[1] == '*'): + return # Don't filter, run all tests in this suite. + path = os.path.sep.join(argpath[1:]) + if path[-1] == '*': + path = path[:-1] + filtered_args.append(path) + for t in self.tests: + for a in filtered_args: + if t.path.startswith(a): + filtered.append(t) + break + self.tests = filtered + + def GetFlagsForTestCase(self, testcase, context): + raise NotImplementedError + + def GetSourceForTest(self, testcase): + return "(no source available)" + + def IsFailureOutput(self, output, testpath): + return output.exit_code != 0 + + def IsNegativeTest(self, testcase): + return False + + def HasFailed(self, testcase): + execution_failed = self.IsFailureOutput(testcase.output, testcase.path) + if self.IsNegativeTest(testcase): + return not execution_failed + else: + return execution_failed + + def HasUnexpectedOutput(self, testcase): + if testcase.output.HasCrashed(): + outcome = statusfile.CRASH + elif testcase.output.HasTimedOut(): + outcome = statusfile.TIMEOUT + elif self.HasFailed(testcase): + outcome = statusfile.FAIL + else: + outcome = statusfile.PASS + if not testcase.outcomes: + return outcome != statusfile.PASS + return not outcome in testcase.outcomes + + def StripOutputForTransmit(self, testcase): + if not self.HasUnexpectedOutput(testcase): + testcase.output.stdout = "" + testcase.output.stderr = "" + + def CalculateTotalDuration(self): + self.total_duration = 0.0 + for t in self.tests: + self.total_duration += t.duration + return self.total_duration diff --git a/src/3rdparty/v8/tools/testrunner/local/utils.py b/src/3rdparty/v8/tools/testrunner/local/utils.py new file mode 100644 index 0000000..b7caa12 --- /dev/null +++ b/src/3rdparty/v8/tools/testrunner/local/utils.py @@ -0,0 +1,108 @@ +# Copyright 2012 the V8 project authors. All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +import os +from os.path import exists +from os.path import isdir +from os.path import join +import platform +import re + + +def GetSuitePaths(test_root): + def IsSuite(path): + return isdir(path) and exists(join(path, 'testcfg.py')) + return [ f for f in os.listdir(test_root) if IsSuite(join(test_root, f)) ] + + +# Reads a file into an array of strings +def ReadLinesFrom(name): + lines = [] + with open(name) as f: + for line in f: + if line.startswith('#'): continue + if '#' in line: + line = line[:line.find('#')] + line = line.strip() + if not line: continue + lines.append(line) + return lines + + +def GuessOS(): + system = platform.system() + if system == 'Linux': + return 'linux' + elif system == 'Darwin': + return 'macos' + elif system.find('CYGWIN') >= 0: + return 'cygwin' + elif system == 'Windows' or system == 'Microsoft': + # On Windows Vista platform.system() can return 'Microsoft' with some + # versions of Python, see http://bugs.python.org/issue1082 + return 'win32' + elif system == 'FreeBSD': + return 'freebsd' + elif system == 'OpenBSD': + return 'openbsd' + elif system == 'SunOS': + return 'solaris' + elif system == 'NetBSD': + return 'netbsd' + else: + return None + + +# This will default to building the 32 bit VM even on machines that are +# capable of running the 64 bit VM. +def DefaultArch(): + machine = platform.machine() + machine = machine.lower() # Windows 7 capitalizes 'AMD64'. + if machine.startswith('arm'): + return 'arm' + elif (not machine) or (not re.match('(x|i[3-6])86$', machine) is None): + return 'ia32' + elif machine == 'i86pc': + return 'ia32' + elif machine == 'x86_64': + return 'ia32' + elif machine == 'amd64': + return 'ia32' + else: + return None + + +def GuessWordsize(): + if '64' in platform.machine(): + return '64' + else: + return '32' + + +def IsWindows(): + return GuessOS() == 'win32' diff --git a/src/3rdparty/v8/tools/testrunner/local/verbose.py b/src/3rdparty/v8/tools/testrunner/local/verbose.py new file mode 100644 index 0000000..f693467 --- /dev/null +++ b/src/3rdparty/v8/tools/testrunner/local/verbose.py @@ -0,0 +1,99 @@ +# Copyright 2012 the V8 project authors. All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +import sys +import time + +from . import statusfile + + +REPORT_TEMPLATE = ( +"""Total: %(total)i tests + * %(skipped)4d tests will be skipped + * %(timeout)4d tests are expected to timeout sometimes + * %(nocrash)4d tests are expected to be flaky but not crash + * %(pass)4d tests are expected to pass + * %(fail_ok)4d tests are expected to fail that we won't fix + * %(fail)4d tests are expected to fail that we should fix""") + + +def PrintReport(tests): + total = len(tests) + skipped = timeout = nocrash = passes = fail_ok = fail = 0 + for t in tests: + if "outcomes" not in dir(t) or not t.outcomes: + passes += 1 + continue + o = t.outcomes + if statusfile.DoSkip(o): + skipped += 1 + continue + if statusfile.TIMEOUT in o: timeout += 1 + if statusfile.IsFlaky(o): nocrash += 1 + if list(o) == [statusfile.PASS]: passes += 1 + if statusfile.IsFailOk(o): fail_ok += 1 + if list(o) == [statusfile.FAIL]: fail += 1 + print REPORT_TEMPLATE % { + "total": total, + "skipped": skipped, + "timeout": timeout, + "nocrash": nocrash, + "pass": passes, + "fail_ok": fail_ok, + "fail": fail + } + + +def PrintTestSource(tests): + for test in tests: + suite = test.suite + source = suite.GetSourceForTest(test).strip() + if len(source) > 0: + print "--- begin source: %s/%s ---" % (suite.name, test.path) + print source + print "--- end source: %s/%s ---" % (suite.name, test.path) + + +def FormatTime(d): + millis = round(d * 1000) % 1000 + return time.strftime("%M:%S.", time.gmtime(d)) + ("%03i" % millis) + + +def PrintTestDurations(suites, overall_time): + # Write the times to stderr to make it easy to separate from the + # test output. + print + sys.stderr.write("--- Total time: %s ---\n" % FormatTime(overall_time)) + timed_tests = [ t for s in suites for t in s.tests + if t.duration is not None ] + timed_tests.sort(lambda a, b: cmp(b.duration, a.duration)) + index = 1 + for entry in timed_tests[:20]: + t = FormatTime(entry.duration) + sys.stderr.write("%4i (%s) %s\n" % (index, t, entry.GetLabel())) + index += 1 diff --git a/src/3rdparty/v8/tools/testrunner/network/__init__.py b/src/3rdparty/v8/tools/testrunner/network/__init__.py new file mode 100644 index 0000000..202a262 --- /dev/null +++ b/src/3rdparty/v8/tools/testrunner/network/__init__.py @@ -0,0 +1,26 @@ +# Copyright 2012 the V8 project authors. All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/src/3rdparty/v8/tools/testrunner/network/distro.py b/src/3rdparty/v8/tools/testrunner/network/distro.py new file mode 100644 index 0000000..9d5a471 --- /dev/null +++ b/src/3rdparty/v8/tools/testrunner/network/distro.py @@ -0,0 +1,90 @@ +# Copyright 2012 the V8 project authors. All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +class Shell(object): + def __init__(self, shell): + self.shell = shell + self.tests = [] + self.total_duration = 0.0 + + def AddSuite(self, suite): + self.tests += suite.tests + self.total_duration += suite.total_duration + + def SortTests(self): + self.tests.sort(cmp=lambda x, y: cmp(x.duration, y.duration)) + + +def Assign(suites, peers): + total_work = 0.0 + for s in suites: + total_work += s.CalculateTotalDuration() + + total_power = 0.0 + for p in peers: + p.assigned_work = 0.0 + total_power += p.jobs * p.relative_performance + for p in peers: + p.needed_work = total_work * p.jobs * p.relative_performance / total_power + + shells = {} + for s in suites: + shell = s.shell() + if not shell in shells: + shells[shell] = Shell(shell) + shells[shell].AddSuite(s) + # Convert |shells| to list and sort it, shortest total_duration first. + shells = [ shells[s] for s in shells ] + shells.sort(cmp=lambda x, y: cmp(x.total_duration, y.total_duration)) + # Sort tests within each shell, longest duration last (so it's + # pop()'ed first). + for s in shells: s.SortTests() + # Sort peers, least needed_work first. + peers.sort(cmp=lambda x, y: cmp(x.needed_work, y.needed_work)) + index = 0 + for shell in shells: + while len(shell.tests) > 0: + while peers[index].needed_work <= 0: + index += 1 + if index == len(peers): + print("BIG FAT WARNING: Assigning tests to peers failed. " + "Remaining tests: %d. Going to slow mode." % len(shell.tests)) + # Pick the least-busy peer. Sorting the list for each test + # is terribly slow, but this is just an emergency fallback anyway. + peers.sort(cmp=lambda x, y: cmp(x.needed_work, y.needed_work)) + peers[0].ForceAddOneTest(shell.tests.pop(), shell) + # If the peer already has a shell assigned and would need this one + # and then yet another, try to avoid it. + peer = peers[index] + if (shell.total_duration < peer.needed_work and + len(peer.shells) > 0 and + index < len(peers) - 1 and + shell.total_duration <= peers[index + 1].needed_work): + peers[index + 1].AddTests(shell) + else: + peer.AddTests(shell) diff --git a/src/3rdparty/v8/tools/testrunner/network/endpoint.py b/src/3rdparty/v8/tools/testrunner/network/endpoint.py new file mode 100644 index 0000000..5dc2b9f --- /dev/null +++ b/src/3rdparty/v8/tools/testrunner/network/endpoint.py @@ -0,0 +1,124 @@ +# Copyright 2012 the V8 project authors. All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +import multiprocessing +import os +import Queue +import threading +import time + +from ..local import execution +from ..local import progress +from ..local import testsuite +from ..local import utils +from ..server import compression + + +class EndpointProgress(progress.ProgressIndicator): + def __init__(self, sock, server, ctx): + super(EndpointProgress, self).__init__() + self.sock = sock + self.server = server + self.context = ctx + self.results_queue = [] # Accessors must synchronize themselves. + self.sender_lock = threading.Lock() + self.senderthread = threading.Thread(target=self._SenderThread) + self.senderthread.start() + + def HasRun(self, test): + # The runners that call this have a lock anyway, so this is safe. + self.results_queue.append(test) + + def _SenderThread(self): + keep_running = True + tests = [] + self.sender_lock.acquire() + while keep_running: + time.sleep(0.1) + # This should be "atomic enough" without locking :-) + # (We don't care which list any new elements get appended to, as long + # as we don't lose any and the last one comes last.) + current = self.results_queue + self.results_queue = [] + for c in current: + if c is None: + keep_running = False + else: + tests.append(c) + if keep_running and len(tests) < 1: + continue # Wait for more results. + if len(tests) < 1: break # We're done here. + result = [] + for t in tests: + result.append(t.PackResult()) + try: + compression.Send(result, self.sock) + except: + self.runner.terminate = True + for t in tests: + self.server.CompareOwnPerf(t, self.context.arch, self.context.mode) + tests = [] + self.sender_lock.release() + + +def Execute(workspace, ctx, tests, sock, server): + suite_paths = utils.GetSuitePaths(os.path.join(workspace, "test")) + suites = [] + for root in suite_paths: + suite = testsuite.TestSuite.LoadTestSuite( + os.path.join(workspace, "test", root)) + if suite: + suites.append(suite) + + suites_dict = {} + for s in suites: + suites_dict[s.name] = s + s.tests = [] + for t in tests: + suite = suites_dict[t.suite] + t.suite = suite + suite.tests.append(t) + + suites = [ s for s in suites if len(s.tests) > 0 ] + for s in suites: + s.DownloadData() + + progress_indicator = EndpointProgress(sock, server, ctx) + runner = execution.Runner(suites, progress_indicator, ctx) + try: + runner.Run(server.jobs) + except IOError, e: + if e.errno == 2: + message = ("File not found: %s, maybe you forgot to 'git add' it?" % + e.filename) + else: + message = "%s" % e + compression.Send([[-1, message]], sock) + progress_indicator.HasRun(None) # Sentinel to signal the end. + progress_indicator.sender_lock.acquire() # Released when sending is done. + progress_indicator.sender_lock.release() diff --git a/src/3rdparty/v8/tools/testrunner/network/network_execution.py b/src/3rdparty/v8/tools/testrunner/network/network_execution.py new file mode 100644 index 0000000..ddb59e6 --- /dev/null +++ b/src/3rdparty/v8/tools/testrunner/network/network_execution.py @@ -0,0 +1,253 @@ +# Copyright 2012 the V8 project authors. All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +import os +import socket +import subprocess +import threading +import time + +from . import distro +from . import perfdata +from ..local import execution +from ..objects import peer +from ..objects import workpacket +from ..server import compression +from ..server import constants +from ..server import local_handler +from ..server import signatures + + +def GetPeers(): + data = local_handler.LocalQuery([constants.REQUEST_PEERS]) + if not data: return [] + return [ peer.Peer.Unpack(p) for p in data ] + + +class NetworkedRunner(execution.Runner): + def __init__(self, suites, progress_indicator, context, peers, workspace): + self.suites = suites + num_tests = 0 + datapath = os.path.join("out", "testrunner_data") + self.perf_data_manager = perfdata.PerfDataManager(datapath) + self.perfdata = self.perf_data_manager.GetStore(context.arch, context.mode) + for s in suites: + for t in s.tests: + t.duration = self.perfdata.FetchPerfData(t) or 1.0 + num_tests += len(s.tests) + self._CommonInit(num_tests, progress_indicator, context) + self.tests = [] # Only used if we need to fall back to local execution. + self.tests_lock = threading.Lock() + self.peers = peers + self.pubkey_fingerprint = None # Fetched later. + self.base_rev = subprocess.check_output( + "cd %s; git log -1 --format=%%H --grep=git-svn-id" % workspace, + shell=True).strip() + self.base_svn_rev = subprocess.check_output( + "cd %s; git log -1 %s" # Get commit description. + " | grep -e '^\s*git-svn-id:'" # Extract "git-svn-id" line. + " | awk '{print $2}'" # Extract "repository@revision" part. + " | sed -e 's/.*@//'" % # Strip away "repository@". + (workspace, self.base_rev), shell=True).strip() + self.patch = subprocess.check_output( + "cd %s; git diff %s" % (workspace, self.base_rev), shell=True) + self.binaries = {} + self.initialization_lock = threading.Lock() + self.initialization_lock.acquire() # Released when init is done. + self._OpenLocalConnection() + self.local_receiver_thread = threading.Thread( + target=self._ListenLocalConnection) + self.local_receiver_thread.daemon = True + self.local_receiver_thread.start() + self.initialization_lock.acquire() + self.initialization_lock.release() + + def _OpenLocalConnection(self): + self.local_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + code = self.local_socket.connect_ex(("localhost", constants.CLIENT_PORT)) + if code != 0: + raise RuntimeError("Failed to connect to local server") + compression.Send([constants.REQUEST_PUBKEY_FINGERPRINT], self.local_socket) + + def _ListenLocalConnection(self): + release_lock_countdown = 1 # Pubkey. + self.local_receiver = compression.Receiver(self.local_socket) + while not self.local_receiver.IsDone(): + data = self.local_receiver.Current() + if data[0] == constants.REQUEST_PUBKEY_FINGERPRINT: + pubkey = data[1] + if not pubkey: raise RuntimeError("Received empty public key") + self.pubkey_fingerprint = pubkey + release_lock_countdown -= 1 + if release_lock_countdown == 0: + self.initialization_lock.release() + release_lock_countdown -= 1 # Prevent repeated triggering. + self.local_receiver.Advance() + + def Run(self, jobs): + self.indicator.Starting() + need_libv8 = False + for s in self.suites: + shell = s.shell() + if shell not in self.binaries: + path = os.path.join(self.context.shell_dir, shell) + # Check if this is a shared library build. + try: + ldd = subprocess.check_output("ldd %s | grep libv8\\.so" % (path), + shell=True) + ldd = ldd.strip().split(" ") + assert ldd[0] == "libv8.so" + assert ldd[1] == "=>" + need_libv8 = True + binary_needs_libv8 = True + libv8 = signatures.ReadFileAndSignature(ldd[2]) + except: + binary_needs_libv8 = False + binary = signatures.ReadFileAndSignature(path) + if binary[0] is None: + print("Error: Failed to create signature.") + assert binary[1] != 0 + return binary[1] + binary.append(binary_needs_libv8) + self.binaries[shell] = binary + if need_libv8: + self.binaries["libv8.so"] = libv8 + distro.Assign(self.suites, self.peers) + # Spawn one thread for each peer. + threads = [] + for p in self.peers: + thread = threading.Thread(target=self._TalkToPeer, args=[p]) + threads.append(thread) + thread.start() + try: + for thread in threads: + # Use a timeout so that signals (Ctrl+C) will be processed. + thread.join(timeout=10000000) + self._AnalyzePeerRuntimes() + except KeyboardInterrupt: + self.terminate = True + raise + except Exception, _e: + # If there's an exception we schedule an interruption for any + # remaining threads... + self.terminate = True + # ...and then reraise the exception to bail out. + raise + compression.Send(constants.END_OF_STREAM, self.local_socket) + self.local_socket.close() + if self.tests: + self._RunInternal(jobs) + self.indicator.Done() + return not self.failed + + def _TalkToPeer(self, peer): + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.settimeout(self.context.timeout + 10) + code = sock.connect_ex((peer.address, constants.PEER_PORT)) + if code == 0: + try: + peer.runtime = None + start_time = time.time() + packet = workpacket.WorkPacket(peer=peer, context=self.context, + base_revision=self.base_svn_rev, + patch=self.patch, + pubkey=self.pubkey_fingerprint) + data, test_map = packet.Pack(self.binaries) + compression.Send(data, sock) + compression.Send(constants.END_OF_STREAM, sock) + rec = compression.Receiver(sock) + while not rec.IsDone() and not self.terminate: + data_list = rec.Current() + for data in data_list: + test_id = data[0] + if test_id < 0: + # The peer is reporting an error. + with self.lock: + print("\nPeer %s reports error: %s" % (peer.address, data[1])) + continue + test = test_map.pop(test_id) + test.MergeResult(data) + try: + self.perfdata.UpdatePerfData(test) + except Exception, e: + print("UpdatePerfData exception: %s" % e) + pass # Just keep working. + with self.lock: + perf_key = self.perfdata.GetKey(test) + compression.Send( + [constants.INFORM_DURATION, perf_key, test.duration, + self.context.arch, self.context.mode], + self.local_socket) + self.indicator.AboutToRun(test) + if test.suite.HasUnexpectedOutput(test): + self.failed.append(test) + if test.output.HasCrashed(): + self.crashed += 1 + else: + self.succeeded += 1 + self.remaining -= 1 + self.indicator.HasRun(test) + rec.Advance() + peer.runtime = time.time() - start_time + except KeyboardInterrupt: + sock.close() + raise + except Exception, e: + print("Got exception: %s" % e) + pass # Fall back to local execution. + else: + compression.Send([constants.UNRESPONSIVE_PEER, peer.address], + self.local_socket) + sock.close() + if len(test_map) > 0: + # Some tests have not received any results. Run them locally. + print("\nNo results for %d tests, running them locally." % len(test_map)) + self._EnqueueLocally(test_map) + + def _EnqueueLocally(self, test_map): + with self.tests_lock: + for test in test_map: + self.tests.append(test_map[test]) + + def _AnalyzePeerRuntimes(self): + total_runtime = 0.0 + total_work = 0.0 + for p in self.peers: + if p.runtime is None: + return + total_runtime += p.runtime + total_work += p.assigned_work + for p in self.peers: + p.assigned_work /= total_work + p.runtime /= total_runtime + perf_correction = p.assigned_work / p.runtime + old_perf = p.relative_performance + p.relative_performance = (old_perf + perf_correction) / 2.0 + compression.Send([constants.UPDATE_PERF, p.address, + p.relative_performance], + self.local_socket) diff --git a/src/3rdparty/v8/tools/testrunner/network/perfdata.py b/src/3rdparty/v8/tools/testrunner/network/perfdata.py new file mode 100644 index 0000000..2979dc4 --- /dev/null +++ b/src/3rdparty/v8/tools/testrunner/network/perfdata.py @@ -0,0 +1,120 @@ +# Copyright 2012 the V8 project authors. All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +import os +import shelve +import threading + + +class PerfDataEntry(object): + def __init__(self): + self.avg = 0.0 + self.count = 0 + + def AddResult(self, result): + kLearnRateLimiter = 99 # Greater value means slower learning. + # We use an approximation of the average of the last 100 results here: + # The existing average is weighted with kLearnRateLimiter (or less + # if there are fewer data points). + effective_count = min(self.count, kLearnRateLimiter) + self.avg = self.avg * effective_count + result + self.count = effective_count + 1 + self.avg /= self.count + + +class PerfDataStore(object): + def __init__(self, datadir, arch, mode): + filename = os.path.join(datadir, "%s.%s.perfdata" % (arch, mode)) + self.database = shelve.open(filename, protocol=2) + self.closed = False + self.lock = threading.Lock() + + def __del__(self): + self.close() + + def close(self): + if self.closed: return + self.database.close() + self.closed = True + + def GetKey(self, test): + """Computes the key used to access data for the given testcase.""" + flags = "".join(test.flags) + return str("%s.%s.%s" % (test.suitename(), test.path, flags)) + + def FetchPerfData(self, test): + """Returns the observed duration for |test| as read from the store.""" + key = self.GetKey(test) + if key in self.database: + return self.database[key].avg + return None + + def UpdatePerfData(self, test): + """Updates the persisted value in the store with test.duration.""" + testkey = self.GetKey(test) + self.RawUpdatePerfData(testkey, test.duration) + + def RawUpdatePerfData(self, testkey, duration): + with self.lock: + if testkey in self.database: + entry = self.database[testkey] + else: + entry = PerfDataEntry() + entry.AddResult(duration) + self.database[testkey] = entry + + +class PerfDataManager(object): + def __init__(self, datadir): + self.datadir = os.path.abspath(datadir) + if not os.path.exists(self.datadir): + os.makedirs(self.datadir) + self.stores = {} # Keyed by arch, then mode. + self.closed = False + self.lock = threading.Lock() + + def __del__(self): + self.close() + + def close(self): + if self.closed: return + for arch in self.stores: + modes = self.stores[arch] + for mode in modes: + store = modes[mode] + store.close() + self.closed = True + + def GetStore(self, arch, mode): + with self.lock: + if not arch in self.stores: + self.stores[arch] = {} + modes = self.stores[arch] + if not mode in modes: + modes[mode] = PerfDataStore(self.datadir, arch, mode) + return modes[mode] diff --git a/src/3rdparty/v8/tools/testrunner/objects/__init__.py b/src/3rdparty/v8/tools/testrunner/objects/__init__.py new file mode 100644 index 0000000..202a262 --- /dev/null +++ b/src/3rdparty/v8/tools/testrunner/objects/__init__.py @@ -0,0 +1,26 @@ +# Copyright 2012 the V8 project authors. All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/src/3rdparty/v8/tools/testrunner/objects/context.py b/src/3rdparty/v8/tools/testrunner/objects/context.py new file mode 100644 index 0000000..b72284b --- /dev/null +++ b/src/3rdparty/v8/tools/testrunner/objects/context.py @@ -0,0 +1,50 @@ +# Copyright 2012 the V8 project authors. All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +class Context(): + def __init__(self, arch, mode, shell_dir, mode_flags, verbose, timeout, + isolates, command_prefix, extra_flags): + self.arch = arch + self.mode = mode + self.shell_dir = shell_dir + self.mode_flags = mode_flags + self.verbose = verbose + self.timeout = timeout + self.isolates = isolates + self.command_prefix = command_prefix + self.extra_flags = extra_flags + + def Pack(self): + return [self.arch, self.mode, self.mode_flags, self.timeout, self.isolates, + self.extra_flags] + + @staticmethod + def Unpack(packed): + # For the order of the fields, refer to Pack() above. + return Context(packed[0], packed[1], None, packed[2], False, + packed[3], packed[4], "", packed[5]) diff --git a/src/3rdparty/v8/tools/testrunner/objects/output.py b/src/3rdparty/v8/tools/testrunner/objects/output.py new file mode 100644 index 0000000..87b4c84 --- /dev/null +++ b/src/3rdparty/v8/tools/testrunner/objects/output.py @@ -0,0 +1,60 @@ +# Copyright 2012 the V8 project authors. All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +import signal + +from ..local import utils + +class Output(object): + + def __init__(self, exit_code, timed_out, stdout, stderr): + self.exit_code = exit_code + self.timed_out = timed_out + self.stdout = stdout + self.stderr = stderr + + def HasCrashed(self): + if utils.IsWindows(): + return 0x80000000 & self.exit_code and not (0x3FFFFF00 & self.exit_code) + else: + # Timed out tests will have exit_code -signal.SIGTERM. + if self.timed_out: + return False + return (self.exit_code < 0 and + self.exit_code != -signal.SIGABRT) + + def HasTimedOut(self): + return self.timed_out + + def Pack(self): + return [self.exit_code, self.timed_out, self.stdout, self.stderr] + + @staticmethod + def Unpack(packed): + # For the order of the fields, refer to Pack() above. + return Output(packed[0], packed[1], packed[2], packed[3]) diff --git a/src/3rdparty/v8/tools/testrunner/objects/peer.py b/src/3rdparty/v8/tools/testrunner/objects/peer.py new file mode 100644 index 0000000..18a6bec --- /dev/null +++ b/src/3rdparty/v8/tools/testrunner/objects/peer.py @@ -0,0 +1,80 @@ +# Copyright 2012 the V8 project authors. All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +class Peer(object): + def __init__(self, address, jobs, rel_perf, pubkey): + self.address = address # string: IP address + self.jobs = jobs # integer: number of CPUs + self.relative_performance = rel_perf + self.pubkey = pubkey # string: pubkey's fingerprint + self.shells = set() # set of strings + self.needed_work = 0 + self.assigned_work = 0 + self.tests = [] # list of TestCase objects + self.trusting_me = False # This peer trusts my public key. + self.trusted = False # I trust this peer's public key. + + def __str__(self): + return ("Peer at %s, jobs: %d, performance: %.2f, trust I/O: %s/%s" % + (self.address, self.jobs, self.relative_performance, + self.trusting_me, self.trusted)) + + def AddTests(self, shell): + """Adds tests from |shell| to this peer. + + Stops when self.needed_work reaches zero, or when all of shell's tests + are assigned.""" + assert self.needed_work > 0 + if shell.shell not in self.shells: + self.shells.add(shell.shell) + while len(shell.tests) > 0 and self.needed_work > 0: + t = shell.tests.pop() + self.needed_work -= t.duration + self.assigned_work += t.duration + shell.total_duration -= t.duration + self.tests.append(t) + + def ForceAddOneTest(self, test, shell): + """Forcibly adds another test to this peer, disregarding needed_work.""" + if shell.shell not in self.shells: + self.shells.add(shell.shell) + self.needed_work -= test.duration + self.assigned_work += test.duration + shell.total_duration -= test.duration + self.tests.append(test) + + + def Pack(self): + """Creates a JSON serializable representation of this Peer.""" + return [self.address, self.jobs, self.relative_performance] + + @staticmethod + def Unpack(packed): + """Creates a Peer object built from a packed representation.""" + pubkey_dummy = "" # Callers of this don't care (only the server does). + return Peer(packed[0], packed[1], packed[2], pubkey_dummy) diff --git a/src/3rdparty/v8/tools/testrunner/objects/testcase.py b/src/3rdparty/v8/tools/testrunner/objects/testcase.py new file mode 100644 index 0000000..cfc522e --- /dev/null +++ b/src/3rdparty/v8/tools/testrunner/objects/testcase.py @@ -0,0 +1,83 @@ +# Copyright 2012 the V8 project authors. All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +from . import output + +class TestCase(object): + def __init__(self, suite, path, flags=[], dependency=None): + self.suite = suite # TestSuite object + self.path = path # string, e.g. 'div-mod', 'test-api/foo' + self.flags = flags # list of strings, flags specific to this test case + self.dependency = dependency # |path| for testcase that must be run first + self.outcomes = None + self.output = None + self.id = None # int, used to map result back to TestCase instance + self.duration = None # assigned during execution + + def CopyAddingFlags(self, flags): + copy = TestCase(self.suite, self.path, self.flags + flags, self.dependency) + copy.outcomes = self.outcomes + return copy + + def PackTask(self): + """ + Extracts those parts of this object that are required to run the test + and returns them as a JSON serializable object. + """ + assert self.id is not None + return [self.suitename(), self.path, self.flags, + self.dependency, list(self.outcomes or []), self.id] + + @staticmethod + def UnpackTask(task): + """Creates a new TestCase object based on packed task data.""" + # For the order of the fields, refer to PackTask() above. + test = TestCase(str(task[0]), task[1], task[2], task[3]) + test.outcomes = set(task[4]) + test.id = task[5] + return test + + def SetSuiteObject(self, suites): + self.suite = suites[self.suite] + + def PackResult(self): + """Serializes the output of the TestCase after it has run.""" + self.suite.StripOutputForTransmit(self) + return [self.id, self.output.Pack(), self.duration] + + def MergeResult(self, result): + """Applies the contents of a Result to this object.""" + assert result[0] == self.id + self.output = output.Output.Unpack(result[1]) + self.duration = result[2] + + def suitename(self): + return self.suite.name + + def GetLabel(self): + return self.suitename() + "/" + self.suite.CommonTestName(self) diff --git a/src/3rdparty/v8/tools/testrunner/objects/workpacket.py b/src/3rdparty/v8/tools/testrunner/objects/workpacket.py new file mode 100644 index 0000000..d07efe7 --- /dev/null +++ b/src/3rdparty/v8/tools/testrunner/objects/workpacket.py @@ -0,0 +1,90 @@ +# Copyright 2012 the V8 project authors. All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +from . import context +from . import testcase + +class WorkPacket(object): + def __init__(self, peer=None, context=None, tests=None, binaries=None, + base_revision=None, patch=None, pubkey=None): + self.peer = peer + self.context = context + self.tests = tests + self.binaries = binaries + self.base_revision = base_revision + self.patch = patch + self.pubkey_fingerprint = pubkey + + def Pack(self, binaries_dict): + """ + Creates a JSON serializable object containing the data of this + work packet. + """ + need_libv8 = False + binaries = [] + for shell in self.peer.shells: + prefetched_binary = binaries_dict[shell] + binaries.append({"name": shell, + "blob": prefetched_binary[0], + "sign": prefetched_binary[1]}) + if prefetched_binary[2]: + need_libv8 = True + if need_libv8: + libv8 = binaries_dict["libv8.so"] + binaries.append({"name": "libv8.so", + "blob": libv8[0], + "sign": libv8[1]}) + tests = [] + test_map = {} + for t in self.peer.tests: + test_map[t.id] = t + tests.append(t.PackTask()) + result = { + "binaries": binaries, + "pubkey": self.pubkey_fingerprint, + "context": self.context.Pack(), + "base_revision": self.base_revision, + "patch": self.patch, + "tests": tests + } + return result, test_map + + @staticmethod + def Unpack(packed): + """ + Creates a WorkPacket object from the given packed representation. + """ + binaries = packed["binaries"] + pubkey_fingerprint = packed["pubkey"] + ctx = context.Context.Unpack(packed["context"]) + base_revision = packed["base_revision"] + patch = packed["patch"] + tests = [ testcase.TestCase.UnpackTask(t) for t in packed["tests"] ] + return WorkPacket(context=ctx, tests=tests, binaries=binaries, + base_revision=base_revision, patch=patch, + pubkey=pubkey_fingerprint) diff --git a/src/3rdparty/v8/tools/testrunner/server/__init__.py b/src/3rdparty/v8/tools/testrunner/server/__init__.py new file mode 100644 index 0000000..202a262 --- /dev/null +++ b/src/3rdparty/v8/tools/testrunner/server/__init__.py @@ -0,0 +1,26 @@ +# Copyright 2012 the V8 project authors. All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/src/3rdparty/v8/tools/testrunner/server/compression.py b/src/3rdparty/v8/tools/testrunner/server/compression.py new file mode 100644 index 0000000..ce90c4f --- /dev/null +++ b/src/3rdparty/v8/tools/testrunner/server/compression.py @@ -0,0 +1,112 @@ +# Copyright 2012 the V8 project authors. All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +import cStringIO as StringIO +try: + import ujson as json +except ImportError: + print("You should install UltraJSON, it is much faster!") + import json +import os +import struct +import zlib + +from . import constants + +def Send(obj, sock): + """ + Sends a JSON encodable object over the specified socket (zlib-compressed). + """ + obj = json.dumps(obj) + compression_level = 2 # 1 = fastest, 9 = best compression + compressed = zlib.compress(obj, compression_level) + payload = struct.pack('>i', len(compressed)) + compressed + sock.sendall(payload) + + +class Receiver(object): + def __init__(self, sock): + self.sock = sock + self.data = StringIO.StringIO() + self.datalength = 0 + self._next = self._GetNext() + + def IsDone(self): + return self._next == None + + def Current(self): + return self._next + + def Advance(self): + try: + self._next = self._GetNext() + except: + raise + + def _GetNext(self): + try: + while self.datalength < constants.SIZE_T: + try: + chunk = self.sock.recv(8192) + except: + raise + if not chunk: return None + self._AppendData(chunk) + size = self._PopData(constants.SIZE_T) + size = struct.unpack(">i", size)[0] + while self.datalength < size: + try: + chunk = self.sock.recv(8192) + except: + raise + if not chunk: return None + self._AppendData(chunk) + result = self._PopData(size) + result = zlib.decompress(result) + result = json.loads(result) + if result == constants.END_OF_STREAM: + return None + return result + except: + raise + + def _AppendData(self, new): + self.data.seek(0, os.SEEK_END) + self.data.write(new) + self.datalength += len(new) + + def _PopData(self, length): + self.data.seek(0) + chunk = self.data.read(length) + remaining = self.data.read() + self.data.close() + self.data = StringIO.StringIO() + self.data.write(remaining) + assert self.datalength - length == len(remaining) + self.datalength = len(remaining) + return chunk diff --git a/src/3rdparty/v8/tools/testrunner/server/constants.py b/src/3rdparty/v8/tools/testrunner/server/constants.py new file mode 100644 index 0000000..5aefcba --- /dev/null +++ b/src/3rdparty/v8/tools/testrunner/server/constants.py @@ -0,0 +1,51 @@ +# Copyright 2012 the V8 project authors. All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +CLIENT_PORT = 9991 # Port for the local client to connect to. +PEER_PORT = 9992 # Port for peers on the network to connect to. +PRESENCE_PORT = 9993 # Port for presence daemon. +STATUS_PORT = 9994 # Port for network requests not related to workpackets. + +END_OF_STREAM = "end of dtest stream" # Marker for end of network requests. +SIZE_T = 4 # Number of bytes used for network request size header. + +# Messages understood by the local request handler. +ADD_TRUSTED = "add trusted" +INFORM_DURATION = "inform about duration" +REQUEST_PEERS = "get peers" +UNRESPONSIVE_PEER = "unresponsive peer" +REQUEST_PUBKEY_FINGERPRINT = "get pubkey fingerprint" +REQUEST_STATUS = "get status" +UPDATE_PERF = "update performance" + +# Messages understood by the status request handler. +LIST_TRUSTED_PUBKEYS = "list trusted pubkeys" +GET_SIGNED_PUBKEY = "pass on signed pubkey" +NOTIFY_NEW_TRUSTED = "new trusted peer" +TRUST_YOU_NOW = "trust you now" +DO_YOU_TRUST = "do you trust" diff --git a/src/3rdparty/v8/tools/testrunner/server/daemon.py b/src/3rdparty/v8/tools/testrunner/server/daemon.py new file mode 100644 index 0000000..baa66fb --- /dev/null +++ b/src/3rdparty/v8/tools/testrunner/server/daemon.py @@ -0,0 +1,147 @@ +#!/usr/bin/env python + +# This code has been written by Sander Marechal and published at: +# http://www.jejik.com/articles/2007/02/a_simple_unix_linux_daemon_in_python/ +# where the author has placed it in the public domain (see comment #6 at +# http://www.jejik.com/articles/2007/02/a_simple_unix_linux_daemon_in_python/#c6 +# ). +# Some minor modifications have been made by the V8 authors. The work remains +# in the public domain. + +import atexit +import os +from signal import SIGTERM +from signal import SIGINT +import sys +import time + + +class Daemon(object): + """ + A generic daemon class. + + Usage: subclass the Daemon class and override the run() method + """ + def __init__(self, pidfile, stdin='/dev/null', + stdout='/dev/null', stderr='/dev/null'): + self.stdin = stdin + self.stdout = stdout + self.stderr = stderr + self.pidfile = pidfile + + def daemonize(self): + """ + do the UNIX double-fork magic, see Stevens' "Advanced + Programming in the UNIX Environment" for details (ISBN 0201563177) + http://www.erlenstar.demon.co.uk/unix/faq_2.html#SEC16 + """ + try: + pid = os.fork() + if pid > 0: + # exit first parent + sys.exit(0) + except OSError, e: + sys.stderr.write("fork #1 failed: %d (%s)\n" % (e.errno, e.strerror)) + sys.exit(1) + + # decouple from parent environment + os.chdir("/") + os.setsid() + os.umask(0) + + # do second fork + try: + pid = os.fork() + if pid > 0: + # exit from second parent + sys.exit(0) + except OSError, e: + sys.stderr.write("fork #2 failed: %d (%s)\n" % (e.errno, e.strerror)) + sys.exit(1) + + # redirect standard file descriptors + sys.stdout.flush() + sys.stderr.flush() + si = file(self.stdin, 'r') + so = file(self.stdout, 'a+') + se = file(self.stderr, 'a+', 0) + # TODO: (debug) re-enable this! + #os.dup2(si.fileno(), sys.stdin.fileno()) + #os.dup2(so.fileno(), sys.stdout.fileno()) + #os.dup2(se.fileno(), sys.stderr.fileno()) + + # write pidfile + atexit.register(self.delpid) + pid = str(os.getpid()) + file(self.pidfile, 'w+').write("%s\n" % pid) + + def delpid(self): + os.remove(self.pidfile) + + def start(self): + """ + Start the daemon + """ + # Check for a pidfile to see if the daemon already runs + try: + pf = file(self.pidfile, 'r') + pid = int(pf.read().strip()) + pf.close() + except IOError: + pid = None + + if pid: + message = "pidfile %s already exist. Daemon already running?\n" + sys.stderr.write(message % self.pidfile) + sys.exit(1) + + # Start the daemon + self.daemonize() + self.run() + + def stop(self): + """ + Stop the daemon + """ + # Get the pid from the pidfile + try: + pf = file(self.pidfile, 'r') + pid = int(pf.read().strip()) + pf.close() + except IOError: + pid = None + + if not pid: + message = "pidfile %s does not exist. Daemon not running?\n" + sys.stderr.write(message % self.pidfile) + return # not an error in a restart + + # Try killing the daemon process + try: + # Give the process a one-second chance to exit gracefully. + os.kill(pid, SIGINT) + time.sleep(1) + while 1: + os.kill(pid, SIGTERM) + time.sleep(0.1) + except OSError, err: + err = str(err) + if err.find("No such process") > 0: + if os.path.exists(self.pidfile): + os.remove(self.pidfile) + else: + print str(err) + sys.exit(1) + + def restart(self): + """ + Restart the daemon + """ + self.stop() + self.start() + + def run(self): + """ + You should override this method when you subclass Daemon. It will be + called after the process has been daemonized by start() or restart(). + """ diff --git a/src/3rdparty/v8/tools/testrunner/server/local_handler.py b/src/3rdparty/v8/tools/testrunner/server/local_handler.py new file mode 100644 index 0000000..3b3ac49 --- /dev/null +++ b/src/3rdparty/v8/tools/testrunner/server/local_handler.py @@ -0,0 +1,119 @@ +# Copyright 2012 the V8 project authors. All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +import socket +import SocketServer +import StringIO + +from . import compression +from . import constants + + +def LocalQuery(query): + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + code = sock.connect_ex(("localhost", constants.CLIENT_PORT)) + if code != 0: return None + compression.Send(query, sock) + compression.Send(constants.END_OF_STREAM, sock) + rec = compression.Receiver(sock) + data = None + while not rec.IsDone(): + data = rec.Current() + assert data[0] == query[0] + data = data[1] + rec.Advance() + sock.close() + return data + + +class LocalHandler(SocketServer.BaseRequestHandler): + def handle(self): + rec = compression.Receiver(self.request) + while not rec.IsDone(): + data = rec.Current() + action = data[0] + + if action == constants.REQUEST_PEERS: + with self.server.daemon.peer_list_lock: + response = [ p.Pack() for p in self.server.daemon.peers + if p.trusting_me ] + compression.Send([action, response], self.request) + + elif action == constants.UNRESPONSIVE_PEER: + self.server.daemon.DeletePeer(data[1]) + + elif action == constants.REQUEST_PUBKEY_FINGERPRINT: + compression.Send([action, self.server.daemon.pubkey_fingerprint], + self.request) + + elif action == constants.REQUEST_STATUS: + compression.Send([action, self._GetStatusMessage()], self.request) + + elif action == constants.ADD_TRUSTED: + fingerprint = self.server.daemon.CopyToTrusted(data[1]) + compression.Send([action, fingerprint], self.request) + + elif action == constants.INFORM_DURATION: + test_key = data[1] + test_duration = data[2] + arch = data[3] + mode = data[4] + self.server.daemon.AddPerfData(test_key, test_duration, arch, mode) + + elif action == constants.UPDATE_PERF: + address = data[1] + perf = data[2] + self.server.daemon.UpdatePeerPerformance(data[1], data[2]) + + rec.Advance() + compression.Send(constants.END_OF_STREAM, self.request) + + def _GetStatusMessage(self): + sio = StringIO.StringIO() + sio.write("Peers:\n") + with self.server.daemon.peer_list_lock: + for p in self.server.daemon.peers: + sio.write("%s\n" % p) + sio.write("My own jobs: %d, relative performance: %.2f\n" % + (self.server.daemon.jobs, self.server.daemon.relative_perf)) + # Low-priority TODO: Return more information. Ideas: + # - currently running anything, + # - time since last job, + # - time since last repository fetch + # - number of workpackets/testcases handled since startup + # - slowest test(s) + result = sio.getvalue() + sio.close() + return result + + +class LocalSocketServer(SocketServer.ThreadingMixIn, SocketServer.TCPServer): + def __init__(self, daemon): + SocketServer.TCPServer.__init__(self, ("localhost", constants.CLIENT_PORT), + LocalHandler) + self.daemon = daemon diff --git a/src/3rdparty/v8/tools/testrunner/server/main.py b/src/3rdparty/v8/tools/testrunner/server/main.py new file mode 100644 index 0000000..1000713 --- /dev/null +++ b/src/3rdparty/v8/tools/testrunner/server/main.py @@ -0,0 +1,245 @@ +# Copyright 2012 the V8 project authors. All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +import multiprocessing +import os +import shutil +import subprocess +import threading +import time + +from . import daemon +from . import local_handler +from . import presence_handler +from . import signatures +from . import status_handler +from . import work_handler +from ..network import perfdata + + +class Server(daemon.Daemon): + + def __init__(self, pidfile, root, stdin="/dev/null", + stdout="/dev/null", stderr="/dev/null"): + super(Server, self).__init__(pidfile, stdin, stdout, stderr) + self.root = root + self.local_handler = None + self.local_handler_thread = None + self.work_handler = None + self.work_handler_thread = None + self.status_handler = None + self.status_handler_thread = None + self.presence_daemon = None + self.presence_daemon_thread = None + self.peers = [] + self.jobs = multiprocessing.cpu_count() + self.peer_list_lock = threading.Lock() + self.perf_data_lock = None + self.presence_daemon_lock = None + self.datadir = os.path.join(self.root, "data") + pubkey_fingerprint_filename = os.path.join(self.datadir, "mypubkey") + with open(pubkey_fingerprint_filename) as f: + self.pubkey_fingerprint = f.read().strip() + self.relative_perf_filename = os.path.join(self.datadir, "myperf") + if os.path.exists(self.relative_perf_filename): + with open(self.relative_perf_filename) as f: + try: + self.relative_perf = float(f.read()) + except: + self.relative_perf = 1.0 + else: + self.relative_perf = 1.0 + + def run(self): + os.nice(20) + self.ip = presence_handler.GetOwnIP() + self.perf_data_manager = perfdata.PerfDataManager(self.datadir) + self.perf_data_lock = threading.Lock() + + self.local_handler = local_handler.LocalSocketServer(self) + self.local_handler_thread = threading.Thread( + target=self.local_handler.serve_forever) + self.local_handler_thread.start() + + self.work_handler = work_handler.WorkSocketServer(self) + self.work_handler_thread = threading.Thread( + target=self.work_handler.serve_forever) + self.work_handler_thread.start() + + self.status_handler = status_handler.StatusSocketServer(self) + self.status_handler_thread = threading.Thread( + target=self.status_handler.serve_forever) + self.status_handler_thread.start() + + self.presence_daemon = presence_handler.PresenceDaemon(self) + self.presence_daemon_thread = threading.Thread( + target=self.presence_daemon.serve_forever) + self.presence_daemon_thread.start() + + self.presence_daemon.FindPeers() + time.sleep(0.5) # Give those peers some time to reply. + + with self.peer_list_lock: + for p in self.peers: + if p.address == self.ip: continue + status_handler.RequestTrustedPubkeys(p, self) + + while True: + try: + self.PeriodicTasks() + time.sleep(60) + except Exception, e: + print("MAIN LOOP EXCEPTION: %s" % e) + self.Shutdown() + break + except KeyboardInterrupt: + self.Shutdown() + break + + def Shutdown(self): + with open(self.relative_perf_filename, "w") as f: + f.write("%s" % self.relative_perf) + self.presence_daemon.shutdown() + self.presence_daemon.server_close() + self.local_handler.shutdown() + self.local_handler.server_close() + self.work_handler.shutdown() + self.work_handler.server_close() + self.status_handler.shutdown() + self.status_handler.server_close() + + def PeriodicTasks(self): + # If we know peers we don't trust, see if someone else trusts them. + with self.peer_list_lock: + for p in self.peers: + if p.trusted: continue + if self.IsTrusted(p.pubkey): + p.trusted = True + status_handler.ITrustYouNow(p) + continue + for p2 in self.peers: + if not p2.trusted: continue + status_handler.TryTransitiveTrust(p2, p.pubkey, self) + # TODO: Ping for more peers waiting to be discovered. + # TODO: Update the checkout (if currently idle). + + def AddPeer(self, peer): + with self.peer_list_lock: + for p in self.peers: + if p.address == peer.address: + return + self.peers.append(peer) + if peer.trusted: + status_handler.ITrustYouNow(peer) + + def DeletePeer(self, peer_address): + with self.peer_list_lock: + for i in xrange(len(self.peers)): + if self.peers[i].address == peer_address: + del self.peers[i] + return + + def MarkPeerAsTrusting(self, peer_address): + with self.peer_list_lock: + for p in self.peers: + if p.address == peer_address: + p.trusting_me = True + break + + def UpdatePeerPerformance(self, peer_address, performance): + with self.peer_list_lock: + for p in self.peers: + if p.address == peer_address: + p.relative_performance = performance + + def CopyToTrusted(self, pubkey_filename): + with open(pubkey_filename, "r") as f: + lines = f.readlines() + fingerprint = lines[-1].strip() + target_filename = self._PubkeyFilename(fingerprint) + shutil.copy(pubkey_filename, target_filename) + with self.peer_list_lock: + for peer in self.peers: + if peer.address == self.ip: continue + if peer.pubkey == fingerprint: + status_handler.ITrustYouNow(peer) + else: + result = self.SignTrusted(fingerprint) + status_handler.NotifyNewTrusted(peer, result) + return fingerprint + + def _PubkeyFilename(self, pubkey_fingerprint): + return os.path.join(self.root, "trusted", "%s.pem" % pubkey_fingerprint) + + def IsTrusted(self, pubkey_fingerprint): + return os.path.exists(self._PubkeyFilename(pubkey_fingerprint)) + + def ListTrusted(self): + path = os.path.join(self.root, "trusted") + if not os.path.exists(path): return [] + return [ f[:-4] for f in os.listdir(path) if f.endswith(".pem") ] + + def SignTrusted(self, pubkey_fingerprint): + if not self.IsTrusted(pubkey_fingerprint): + return [] + filename = self._PubkeyFilename(pubkey_fingerprint) + result = signatures.ReadFileAndSignature(filename) # Format: [key, sig]. + return [pubkey_fingerprint, result[0], result[1], self.pubkey_fingerprint] + + def AcceptNewTrusted(self, data): + # The format of |data| matches the return value of |SignTrusted()|. + if not data: return + fingerprint = data[0] + pubkey = data[1] + signature = data[2] + signer = data[3] + if not self.IsTrusted(signer): + return + if self.IsTrusted(fingerprint): + return # Already trust this guy. + filename = self._PubkeyFilename(fingerprint) + signer_pubkeyfile = self._PubkeyFilename(signer) + if not signatures.VerifySignature(filename, pubkey, signature, + signer_pubkeyfile): + return + return # Nothing more to do. + + def AddPerfData(self, test_key, duration, arch, mode): + data_store = self.perf_data_manager.GetStore(arch, mode) + data_store.RawUpdatePerfData(str(test_key), duration) + + def CompareOwnPerf(self, test, arch, mode): + data_store = self.perf_data_manager.GetStore(arch, mode) + observed = data_store.FetchPerfData(test) + if not observed: return + own_perf_estimate = observed / test.duration + with self.perf_data_lock: + kLearnRateLimiter = 9999 + self.relative_perf *= kLearnRateLimiter + self.relative_perf += own_perf_estimate + self.relative_perf /= (kLearnRateLimiter + 1) diff --git a/src/3rdparty/v8/tools/testrunner/server/presence_handler.py b/src/3rdparty/v8/tools/testrunner/server/presence_handler.py new file mode 100644 index 0000000..1dc2ef1 --- /dev/null +++ b/src/3rdparty/v8/tools/testrunner/server/presence_handler.py @@ -0,0 +1,120 @@ +# Copyright 2012 the V8 project authors. All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +import socket +import SocketServer +import threading +try: + import ujson as json +except: + import json + +from . import constants +from ..objects import peer + + +STARTUP_REQUEST = "V8 test peer starting up" +STARTUP_RESPONSE = "Let's rock some tests!" +EXIT_REQUEST = "V8 testing peer going down" + + +def GetOwnIP(): + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + s.connect(("8.8.8.8", 80)) + ip = s.getsockname()[0] + s.close() + return ip + + +class PresenceHandler(SocketServer.BaseRequestHandler): + + def handle(self): + data = json.loads(self.request[0].strip()) + + if data[0] == STARTUP_REQUEST: + jobs = data[1] + relative_perf = data[2] + pubkey_fingerprint = data[3] + trusted = self.server.daemon.IsTrusted(pubkey_fingerprint) + response = [STARTUP_RESPONSE, self.server.daemon.jobs, + self.server.daemon.relative_perf, + self.server.daemon.pubkey_fingerprint, trusted] + response = json.dumps(response) + self.server.SendTo(self.client_address[0], response) + p = peer.Peer(self.client_address[0], jobs, relative_perf, + pubkey_fingerprint) + p.trusted = trusted + self.server.daemon.AddPeer(p) + + elif data[0] == STARTUP_RESPONSE: + jobs = data[1] + perf = data[2] + pubkey_fingerprint = data[3] + p = peer.Peer(self.client_address[0], jobs, perf, pubkey_fingerprint) + p.trusted = self.server.daemon.IsTrusted(pubkey_fingerprint) + p.trusting_me = data[4] + self.server.daemon.AddPeer(p) + + elif data[0] == EXIT_REQUEST: + self.server.daemon.DeletePeer(self.client_address[0]) + if self.client_address[0] == self.server.daemon.ip: + self.server.shutdown_lock.release() + + +class PresenceDaemon(SocketServer.ThreadingMixIn, SocketServer.UDPServer): + def __init__(self, daemon): + self.daemon = daemon + address = (daemon.ip, constants.PRESENCE_PORT) + SocketServer.UDPServer.__init__(self, address, PresenceHandler) + self.shutdown_lock = threading.Lock() + + def shutdown(self): + self.shutdown_lock.acquire() + self.SendToAll(json.dumps([EXIT_REQUEST])) + self.shutdown_lock.acquire() + self.shutdown_lock.release() + SocketServer.UDPServer.shutdown(self) + + def SendTo(self, target, message): + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + sock.sendto(message, (target, constants.PRESENCE_PORT)) + sock.close() + + def SendToAll(self, message): + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + ip = self.daemon.ip.split(".") + for i in range(1, 254): + ip[-1] = str(i) + sock.sendto(message, (".".join(ip), constants.PRESENCE_PORT)) + sock.close() + + def FindPeers(self): + request = [STARTUP_REQUEST, self.daemon.jobs, self.daemon.relative_perf, + self.daemon.pubkey_fingerprint] + request = json.dumps(request) + self.SendToAll(request) diff --git a/src/3rdparty/v8/tools/testrunner/server/signatures.py b/src/3rdparty/v8/tools/testrunner/server/signatures.py new file mode 100644 index 0000000..9957a18 --- /dev/null +++ b/src/3rdparty/v8/tools/testrunner/server/signatures.py @@ -0,0 +1,63 @@ +# Copyright 2012 the V8 project authors. All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +import base64 +import os +import subprocess + + +def ReadFileAndSignature(filename): + with open(filename, "rb") as f: + file_contents = base64.b64encode(f.read()) + signature_file = filename + ".signature" + if (not os.path.exists(signature_file) or + os.path.getmtime(signature_file) < os.path.getmtime(filename)): + private_key = "~/.ssh/v8_dtest" + code = subprocess.call("openssl dgst -out %s -sign %s %s" % + (signature_file, private_key, filename), + shell=True) + if code != 0: return [None, code] + with open(signature_file) as f: + signature = base64.b64encode(f.read()) + return [file_contents, signature] + + +def VerifySignature(filename, file_contents, signature, pubkeyfile): + with open(filename, "wb") as f: + f.write(base64.b64decode(file_contents)) + signature_file = filename + ".foreign_signature" + with open(signature_file, "wb") as f: + f.write(base64.b64decode(signature)) + code = subprocess.call("openssl dgst -verify %s -signature %s %s" % + (pubkeyfile, signature_file, filename), + shell=True) + matched = (code == 0) + if not matched: + os.remove(signature_file) + os.remove(filename) + return matched diff --git a/src/3rdparty/v8/tools/testrunner/server/status_handler.py b/src/3rdparty/v8/tools/testrunner/server/status_handler.py new file mode 100644 index 0000000..3f2271d --- /dev/null +++ b/src/3rdparty/v8/tools/testrunner/server/status_handler.py @@ -0,0 +1,112 @@ +# Copyright 2012 the V8 project authors. All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +import socket +import SocketServer + +from . import compression +from . import constants + + +def _StatusQuery(peer, query): + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + code = sock.connect_ex((peer.address, constants.STATUS_PORT)) + if code != 0: + # TODO(jkummerow): disconnect (after 3 failures?) + return + compression.Send(query, sock) + compression.Send(constants.END_OF_STREAM, sock) + rec = compression.Receiver(sock) + data = None + while not rec.IsDone(): + data = rec.Current() + assert data[0] == query[0] + data = data[1] + rec.Advance() + sock.close() + return data + + +def RequestTrustedPubkeys(peer, server): + pubkey_list = _StatusQuery(peer, [constants.LIST_TRUSTED_PUBKEYS]) + for pubkey in pubkey_list: + if server.IsTrusted(pubkey): continue + result = _StatusQuery(peer, [constants.GET_SIGNED_PUBKEY, pubkey]) + server.AcceptNewTrusted(result) + + +def NotifyNewTrusted(peer, data): + _StatusQuery(peer, [constants.NOTIFY_NEW_TRUSTED] + data) + + +def ITrustYouNow(peer): + _StatusQuery(peer, [constants.TRUST_YOU_NOW]) + + +def TryTransitiveTrust(peer, pubkey, server): + if _StatusQuery(peer, [constants.DO_YOU_TRUST, pubkey]): + result = _StatusQuery(peer, [constants.GET_SIGNED_PUBKEY, pubkey]) + server.AcceptNewTrusted(result) + + +class StatusHandler(SocketServer.BaseRequestHandler): + def handle(self): + rec = compression.Receiver(self.request) + while not rec.IsDone(): + data = rec.Current() + action = data[0] + + if action == constants.LIST_TRUSTED_PUBKEYS: + response = self.server.daemon.ListTrusted() + compression.Send([action, response], self.request) + + elif action == constants.GET_SIGNED_PUBKEY: + response = self.server.daemon.SignTrusted(data[1]) + compression.Send([action, response], self.request) + + elif action == constants.NOTIFY_NEW_TRUSTED: + self.server.daemon.AcceptNewTrusted(data[1:]) + pass # No response. + + elif action == constants.TRUST_YOU_NOW: + self.server.daemon.MarkPeerAsTrusting(self.client_address[0]) + pass # No response. + + elif action == constants.DO_YOU_TRUST: + response = self.server.daemon.IsTrusted(data[1]) + compression.Send([action, response], self.request) + + rec.Advance() + compression.Send(constants.END_OF_STREAM, self.request) + + +class StatusSocketServer(SocketServer.ThreadingMixIn, SocketServer.TCPServer): + def __init__(self, daemon): + address = (daemon.ip, constants.STATUS_PORT) + SocketServer.TCPServer.__init__(self, address, StatusHandler) + self.daemon = daemon diff --git a/src/3rdparty/v8/tools/testrunner/server/work_handler.py b/src/3rdparty/v8/tools/testrunner/server/work_handler.py new file mode 100644 index 0000000..6bf7d43 --- /dev/null +++ b/src/3rdparty/v8/tools/testrunner/server/work_handler.py @@ -0,0 +1,150 @@ +# Copyright 2012 the V8 project authors. All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +import os +import SocketServer +import stat +import subprocess +import threading + +from . import compression +from . import constants +from . import signatures +from ..network import endpoint +from ..objects import workpacket + + +class WorkHandler(SocketServer.BaseRequestHandler): + + def handle(self): + rec = compression.Receiver(self.request) + while not rec.IsDone(): + data = rec.Current() + with self.server.job_lock: + self._WorkOnWorkPacket(data) + rec.Advance() + + def _WorkOnWorkPacket(self, data): + server_root = self.server.daemon.root + v8_root = os.path.join(server_root, "v8") + os.chdir(v8_root) + packet = workpacket.WorkPacket.Unpack(data) + self.ctx = packet.context + self.ctx.shell_dir = os.path.join("out", + "%s.%s" % (self.ctx.arch, self.ctx.mode)) + if not os.path.isdir(self.ctx.shell_dir): + os.makedirs(self.ctx.shell_dir) + for binary in packet.binaries: + if not self._UnpackBinary(binary, packet.pubkey_fingerprint): + return + + if not self._CheckoutRevision(packet.base_revision): + return + + if not self._ApplyPatch(packet.patch): + return + + tests = packet.tests + endpoint.Execute(v8_root, self.ctx, tests, self.request, self.server.daemon) + self._SendResponse() + + def _SendResponse(self, error_message=None): + try: + if error_message: + compression.Send([[-1, error_message]], self.request) + compression.Send(constants.END_OF_STREAM, self.request) + return + except Exception, e: + pass # Peer is gone. There's nothing we can do. + # Clean up. + self._Call("git checkout -f") + self._Call("git clean -f -d") + self._Call("rm -rf %s" % self.ctx.shell_dir) + + def _UnpackBinary(self, binary, pubkey_fingerprint): + binary_name = binary["name"] + if binary_name == "libv8.so": + libdir = os.path.join(self.ctx.shell_dir, "lib.target") + if not os.path.exists(libdir): os.makedirs(libdir) + target = os.path.join(libdir, binary_name) + else: + target = os.path.join(self.ctx.shell_dir, binary_name) + pubkeyfile = "../trusted/%s.pem" % pubkey_fingerprint + if not signatures.VerifySignature(target, binary["blob"], + binary["sign"], pubkeyfile): + self._SendResponse("Signature verification failed") + return False + os.chmod(target, stat.S_IRWXU) + return True + + def _CheckoutRevision(self, base_svn_revision): + get_hash_cmd = ( + "git log -1 --format=%%H --remotes --grep='^git-svn-id:.*@%s'" % + base_svn_revision) + try: + base_revision = subprocess.check_output(get_hash_cmd, shell=True) + if not base_revision: raise ValueError + except: + self._Call("git fetch") + try: + base_revision = subprocess.check_output(get_hash_cmd, shell=True) + if not base_revision: raise ValueError + except: + self._SendResponse("Base revision not found.") + return False + code = self._Call("git checkout -f %s" % base_revision) + if code != 0: + self._SendResponse("Error trying to check out base revision.") + return False + code = self._Call("git clean -f -d") + if code != 0: + self._SendResponse("Failed to reset checkout") + return False + return True + + def _ApplyPatch(self, patch): + if not patch: return True # Just skip if the patch is empty. + patchfilename = "_dtest_incoming_patch.patch" + with open(patchfilename, "w") as f: + f.write(patch) + code = self._Call("git apply %s" % patchfilename) + if code != 0: + self._SendResponse("Error applying patch.") + return False + return True + + def _Call(self, cmd): + return subprocess.call(cmd, shell=True) + + +class WorkSocketServer(SocketServer.ThreadingMixIn, SocketServer.TCPServer): + def __init__(self, daemon): + address = (daemon.ip, constants.PEER_PORT) + SocketServer.TCPServer.__init__(self, address, WorkHandler) + self.job_lock = threading.Lock() + self.daemon = daemon diff --git a/src/3rdparty/v8/tools/tickprocessor-driver.js b/src/3rdparty/v8/tools/tickprocessor-driver.js index 9af5ab6..313c6d4 100644 --- a/src/3rdparty/v8/tools/tickprocessor-driver.js +++ b/src/3rdparty/v8/tools/tickprocessor-driver.js @@ -1,4 +1,4 @@ -// Copyright 2009 the V8 project authors. All rights reserved. +// Copyright 2012 the V8 project authors. All rights reserved. // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are // met: @@ -50,7 +50,7 @@ if (params.snapshotLogFileName) { snapshotLogProcessor.processLogFile(params.snapshotLogFileName); } var tickProcessor = new TickProcessor( - new (entriesProviders[params.platform])(params.nm), + new (entriesProviders[params.platform])(params.nm, params.targetRootFS), params.separateIc, params.callGraphSize, params.ignoreUnknown, diff --git a/src/3rdparty/v8/tools/tickprocessor.js b/src/3rdparty/v8/tools/tickprocessor.js index 05a3369..f6e315d 100644 --- a/src/3rdparty/v8/tools/tickprocessor.js +++ b/src/3rdparty/v8/tools/tickprocessor.js @@ -1,4 +1,4 @@ -// Copyright 2009 the V8 project authors. All rights reserved. +// Copyright 2012 the V8 project authors. All rights reserved. // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are // met: @@ -231,8 +231,9 @@ TickProcessor.VmStates = { JS: 0, GC: 1, COMPILER: 2, - OTHER: 3, - EXTERNAL: 4 + PARALLEL_COMPILER_PROLOGUE: 3, + OTHER: 4, + EXTERNAL: 5 }; @@ -608,10 +609,11 @@ CppEntriesProvider.prototype.parseNextLine = function() { }; -function UnixCppEntriesProvider(nmExec) { +function UnixCppEntriesProvider(nmExec, targetRootFS) { this.symbols = []; this.parsePos = 0; this.nmExec = nmExec; + this.targetRootFS = targetRootFS; this.FUNC_RE = /^([0-9a-fA-F]{8,16}) ([0-9a-fA-F]{8,16} )?[tTwW] (.*)$/; }; inherits(UnixCppEntriesProvider, CppEntriesProvider); @@ -619,6 +621,7 @@ inherits(UnixCppEntriesProvider, CppEntriesProvider); UnixCppEntriesProvider.prototype.loadSymbols = function(libName) { this.parsePos = 0; + libName = this.targetRootFS + libName; try { this.symbols = [ os.system(this.nmExec, ['-C', '-n', '-S', libName], -1, -1), @@ -656,8 +659,8 @@ UnixCppEntriesProvider.prototype.parseNextLine = function() { }; -function MacCppEntriesProvider(nmExec) { - UnixCppEntriesProvider.call(this, nmExec); +function MacCppEntriesProvider(nmExec, targetRootFS) { + UnixCppEntriesProvider.call(this, nmExec, targetRootFS); // Note an empty group. It is required, as UnixCppEntriesProvider expects 3 groups. this.FUNC_RE = /^([0-9a-fA-F]{8,16}) ()[iItT] (.*)$/; }; @@ -666,6 +669,7 @@ inherits(MacCppEntriesProvider, UnixCppEntriesProvider); MacCppEntriesProvider.prototype.loadSymbols = function(libName) { this.parsePos = 0; + libName = this.targetRootFS + libName; try { this.symbols = [os.system(this.nmExec, ['-n', '-f', libName], -1, -1), '']; } catch (e) { @@ -675,7 +679,8 @@ MacCppEntriesProvider.prototype.loadSymbols = function(libName) { }; -function WindowsCppEntriesProvider() { +function WindowsCppEntriesProvider(_ignored_nmExec, targetRootFS) { + this.targetRootFS = targetRootFS; this.symbols = ''; this.parsePos = 0; }; @@ -698,6 +703,7 @@ WindowsCppEntriesProvider.EXE_IMAGE_BASE = 0x00400000; WindowsCppEntriesProvider.prototype.loadSymbols = function(libName) { + libName = this.targetRootFS + libName; var fileNameFields = libName.match(WindowsCppEntriesProvider.FILENAME_RE); if (!fileNameFields) return; var mapFileName = fileNameFields[1] + '.map'; @@ -785,6 +791,8 @@ function ArgumentsProcessor(args) { 'Specify that we are running on Mac OS X platform'], '--nm': ['nm', 'nm', 'Specify the \'nm\' executable to use (e.g. --nm=/my_dir/nm)'], + '--target': ['targetRootFS', '', + 'Specify the target root directory for cross environment'], '--snapshot-log': ['snapshotLogFileName', 'snapshot.log', 'Specify snapshot log file to use (e.g. --snapshot-log=snapshot.log)'] }; @@ -804,6 +812,7 @@ ArgumentsProcessor.DEFAULTS = { callGraphSize: 5, ignoreUnknown: false, separateIc: false, + targetRootFS: '', nm: 'nm' }; |