diff options
author | Luke Chen <luke.chen@mongodb.com> | 2017-12-14 11:34:35 -0500 |
---|---|---|
committer | Luke Chen <luke.chen@mongodb.com> | 2017-12-14 11:34:35 -0500 |
commit | ec894eecfba4009e2ccd685e15351e4dae5848ad (patch) | |
tree | b347f774ed408c0ff275d6889cc022ab8afe727b /src/third_party/wiredtiger/tools | |
parent | fe40a36217a2b4e4064165340d44cc1442d84e13 (diff) | |
download | mongo-ec894eecfba4009e2ccd685e15351e4dae5848ad.tar.gz |
Import wiredtiger: 1a29eac4dc8cf82de437292da546e3f4039268a4 from branch mongodb-3.8
ref: 596a3c7c01..1a29eac4dc
for: 3.7.1
WT-3079 Make sure eviction visits all trees
WT-3133 Detect or track long latency operations
WT-3295 Allow LSM to merge into custom data sources
WT-3587 Remove HAVE_VERBOSE conditional compilation
WT-3654 Fix warning in Windows build on evergreen
WT-3716 Restore the WT_VERB_TEMPORARY verbose flag.
WT-3720 flags macros cast flags to unsigned values, hiding warnings.
WT-3732 Handle adding WT indices while cursors on the table are open
WT-3734 Fix undefined behavior in verbose output
WT-3738 Review internal session allocation accounting
WT-3753 Building on Windows --enable-java
WT-3772 Hot backup causes uncontrolled growth of WiredTigerPreplog files
WT-3774 Enhance Python lookaside testing to cover cursor modify
WT-3776 Cursor remove operation unpins page too early
WT-3780 Improve error messages on invalid WT_CURSOR::modify usage
WT-3783 Fix transaction isolation to use the correct enum
WT-3786 Transactions with timestamps should read their writes
WT-3787 test_compact02 failed as compaction halted due to eviction pressure
WT-3790 Switch statistics to rdtsc from epoch calls
WT-3793 WiredTiger page debug dump functions should unpack integer keys
WT-3794 Coverity 1383547 and lint
WT-3795 lint cleanups for the op-tracking software, reduce record write size.
Diffstat (limited to 'src/third_party/wiredtiger/tools')
-rwxr-xr-x | src/third_party/wiredtiger/tools/wt_optrack_decode.py | 319 |
1 files changed, 319 insertions, 0 deletions
diff --git a/src/third_party/wiredtiger/tools/wt_optrack_decode.py b/src/third_party/wiredtiger/tools/wt_optrack_decode.py new file mode 100755 index 00000000000..1e063d1887c --- /dev/null +++ b/src/third_party/wiredtiger/tools/wt_optrack_decode.py @@ -0,0 +1,319 @@ +#!/usr/bin/env python +# +# Public Domain 2014-2017 MongoDB, Inc. +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +import argparse +import colorsys +from multiprocessing import Process +import multiprocessing +import os +import os.path +import struct +import sys +import subprocess +import time +import traceback + +# +# This log version must be the same as that defined in ../src/include/optrack.h +# +currentLogVersion = 1; + +class color: + PURPLE = '\033[95m' + CYAN = '\033[96m' + DARKCYAN = '\033[36m' + BLUE = '\033[94m' + GREEN = '\033[92m' + YELLOW = '\033[93m' + RED = '\033[91m' + BOLD = '\033[1m' + UNDERLINE = '\033[4m' + END = '\033[0m' + +functionMap = {}; + +def buildTranslationMap(mapFileName): + + mapFile = None; + + if not os.path.exists(mapFileName): + return False; + + try: + mapFile = open(mapFileName, "r"); + except: + print(color.BOLD + color.RED); + print("Could not open " + mapFileName + " for reading"); + print(color.END); + return; + + # Read lines from the map file and build an in-memory map + # of translations. Each line has a function ID followed by space and + # followed by the function name. + # + lines = mapFile.readlines(); # a map file is usually small + + for line in lines: + + words = line.split(" "); + if (len(words) < 2): + continue; + + try: + funcID = int(words[0]); + except: + continue; + + funcName = words[1].strip(); + + functionMap[funcID] = funcName; + + return True; + +def funcIDtoName(funcID): + + if (functionMap.has_key(funcID)): + return functionMap[funcID]; + else: + return "NULL"; + +# +# The format of the record is written down in src/include/optrack.h +# file in the WiredTiger source tree. The current implementation assumes +# a record of three fields. The first field is the 8-byte timestamp. +# The second field is the 2-byte function ID. The third field is the +# 2-byte operation type: '0' for function entry, '1' for function exit. +# The record size would be padded to 16 bytes in the C implementation by +# the compiler, because we keep an array of records, and each new record +# has to be 8-byte aligned, since the first field has the size 8 bytes. +# So we explicitly pad the track record structure in the implementation +# to make it clear what the record size is. +# +def parseOneRecord(file): + + bytesRead = ""; + record = (); + RECORD_SIZE = 16; + + try: + bytesRead = file.read(RECORD_SIZE); + except: + return None; + + if (len(bytesRead) < RECORD_SIZE): + return None; + + record = struct.unpack('Qhhxxxx', bytesRead); + + return record; + +# +# HEADER_SIZE must be the same as the size of WT_OPTRACK_HEADER +# structure defined in ../src/include/optrack.h +# +def validateHeader(file): + + global currentLogVersion; + + bytesRead = ""; + HEADER_SIZE = 8; + + try: + bytesRead = file.read(HEADER_SIZE); + except: + return False, -1; + + if (len(bytesRead) < HEADER_SIZE): + return False, -1; + + version, threadType = struct.unpack('II', bytesRead); + + if (version == currentLogVersion): + return True, threadType; + else: + return False, -1; + +def getStringFromThreadType(threadType): + + if (threadType == 0): + return "external"; + elif (threadType == 1): + return "internal"; + else: + return unknown; + + +def parseFile(fileName): + + done = False; + file = None; + threadType = 0; + threadTypeString = None; + outputFile = None; + outputFileName = ""; + totalRecords = 0; + validVersion = False; + + print(color.BOLD + "Processing file " + fileName + color.END); + + # Open the log file for reading + try: + file = open(fileName, "r"); + except: + print(color.BOLD + color.RED + + "Could not open " + fileName + " for reading" + color.END); + return; + + # Read and validate log header + validVersion, threadType = validateHeader(file); + if (not validVersion): + return; + + threadTypeString = getStringFromThreadType(threadType); + + # Open the text file for writing + try: + outputFileName = fileName + "-" + threadTypeString + ".txt"; + outputFile = open(outputFileName, "w"); + except: + print(color.BOLD + color.RED + + "Could not open file " + outputfileName + ".txt for writing." + + color.END); + return; + + print(color.BOLD + color.PURPLE + + "Writing to output file " + outputFileName + "." + color.END); + + while (not done): + record = parseOneRecord(file); + + if ((record is None) or len(record) < 3): + done = True; + else: + try: + time = record[0]; + funcName = funcIDtoName(record[1]); + opType = record[2]; + + outputFile.write(str(opType) + " " + funcName + " " + str(time) + + "\n"); + totalRecords += 1; + except: + exc_type, exc_value, exc_traceback = sys.exc_info() + traceback.print_exception(exc_type, exc_value, exc_traceback); + print(color.BOLD + color.RED); + print("Could not write record " + str(record) + + " to file " + fileName + ".txt."); + print(color.END); + done = True; + + print("Wrote " + str(totalRecords) + " records to " + outputFileName + "."); + file.close(); + outputFile.close(); + +def waitOnOneProcess(runningProcesses): + + success = False; + for fname, p in runningProcesses.items(): + if (not p.is_alive()): + del runningProcesses[fname]; + success = True; + + # If we have not found a terminated process, sleep for a while + if (not success): + time.sleep(5); + +def main(): + + runnableProcesses = {}; + returnValues = {}; + spawnedProcesses = {}; + successfullyProcessedFiles = []; + targetParallelism = multiprocessing.cpu_count(); + terminatedProcesses = {}; + + parser = argparse.ArgumentParser(description= + 'Convert WiredTiger operation \ + tracking logs from binary to \ + text format.'); + + parser.add_argument('files', type=str, nargs='*', + help='optrack log files to process'); + + parser.add_argument('-j', dest='jobParallelism', type=int, + default='0'); + + parser.add_argument('-m', '--mapfile', dest='mapFileName', type=str, + default='optrack-map'); + + args = parser.parse_args(); + + print("Running with the following parameters:"); + for key, value in vars(args).items(): + print ("\t" + key + ": " + str(value)); + + # Parse the map of function ID to name translations. + if (buildTranslationMap(args.mapFileName) is False): + print("Failed to locate or parse the map file " + + args.mapFileName); + print("Cannot proceed."); + return; + + # Determine the target job parallelism + if (args.jobParallelism > 0): + targetParallelism = args.jobParallelism; + if (targetParallelism == 0): + targetParallelism = len(args.files); + print(color.BLUE + color.BOLD + + "Will process " + str(targetParallelism) + " files in parallel." + + color.END); + + # Prepare the processes that will parse files, one per file + if (len(args.files) > 0): + for fname in args.files: + p = Process(target=parseFile, args=(fname,)); + runnableProcesses[fname] = p; + + # Spawn these processes, not exceeding the desired parallelism + while (len(runnableProcesses) > 0): + while (len(spawnedProcesses) < targetParallelism + and len(runnableProcesses) > 0): + + fname, p = runnableProcesses.popitem(); + p.start(); + spawnedProcesses[fname] = p; + + # Find at least one terminated process + waitOnOneProcess(spawnedProcesses); + + # Wait for all processes to terminate + while (len(spawnedProcesses) > 0): + waitOnOneProcess(spawnedProcesses); + +if __name__ == '__main__': + main() |