diff options
Diffstat (limited to 'tools')
| -rw-r--r-- | tools/wt_nvd3_util.py | 46 | ||||
| -rw-r--r-- | tools/wtperf_graph.py | 35 | ||||
| -rw-r--r-- | tools/wtperf_stats.py | 161 | ||||
| -rw-r--r-- | tools/wtstats.py | 42 |
4 files changed, 250 insertions, 34 deletions
diff --git a/tools/wt_nvd3_util.py b/tools/wt_nvd3_util.py new file mode 100644 index 00000000000..6bf1396b0ff --- /dev/null +++ b/tools/wt_nvd3_util.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python +# +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +from datetime import datetime +from nvd3 import lineChart + +# Add a multiChart type so we can overlay line graphs +class multiChart(lineChart): + def __init__(self, **kwargs): + lineChart.__init__(self, **kwargs) + + # Fix the axes + del self.axislist['yAxis'] + self.create_y_axis('yAxis1', format=kwargs.get('y_axis_format', '.02f')) + self.create_y_axis('yAxis2', format=kwargs.get('y_axis_format', '.02f')) + +TIMEFMT = "%b %d %H:%M:%S" + +thisyear = datetime.today().year +def parsetime(s): + return datetime.strptime(s, TIMEFMT).replace(year=thisyear) + diff --git a/tools/wtperf_graph.py b/tools/wtperf_graph.py index fb925ef28b9..1f50b6bafa1 100644 --- a/tools/wtperf_graph.py +++ b/tools/wtperf_graph.py @@ -33,11 +33,13 @@ from subprocess import call TIMEFMT = "%b %d %H:%M:%S" -def process_monitor(fname, ckptlist, opdict): +def process_monitor(fname, sfx, ckptlist, opdict): # Read the monitor file and figure out when a checkpoint was running. in_ckpt = 'N' ckptlist=[] + + ofname = 'monitor%s.png' % (sfx) # Monitor output format currently is: # time,totalsec,read,insert,update,ckpt,...latencies... ops = ('read', 'insert', 'update') @@ -45,6 +47,8 @@ def process_monitor(fname, ckptlist, opdict): with open(fname, 'r') as csvfile: reader = csv.reader(csvfile) for row in reader: + if row[0].lstrip().startswith('#'): + continue # Look for checkpoints and operations. if row[5] != in_ckpt: ckptlist.append(row[0]) @@ -77,13 +81,14 @@ set yrange [0:]\n''' % { }) it = iter(ckptlist) for start, stop in zip(it, it): - of.write('set object rectangle from first \'' + start +\ - '\', graph 0 ' + ' to first \'' + stop +\ - '\', graph 1 fc rgb "gray" back\n') - of.write('set output "' + fname + '.png"\n') - of.write('plot "' + fname + '" using 1:($3/1000) title "Reads", "' +\ - fname + '" using 1:($4/1000) title "Inserts", "' +\ - fname + '" using 1:($5/1000) title "Updates"\n') + of.write("set object rectangle from first '%i',\ + graph 0 to first '%i',\ + graph 1 fc rgb \"gray\" back\n" % (start, stop)) + of.write('set output "%s"\n' % (ofname)) + of.write("""plot "{name}" every ::1 using 1:($3/1000) title "Reads", \\ + "{name}" every ::1 using 1:($4/1000) title "Inserts",\\ + "{name}" every ::1 using 1:($5/1000) title "Updates" + """.format(name=fname)) of.close() call(["gnuplot", gcmd]) os.remove(gcmd) @@ -117,10 +122,12 @@ set yrange [1:]\n''' % { '\', graph 1 fc rgb "gray" back\n') ofname = name + sfx + '.latency1.png' of.write('set output "' + ofname + '"\n') - of.write('plot "' + fname + '" using 1:($' + repr(col_avg) +\ - ') title "Average Latency", "' + fname +'" using 1:($' +\ + of.write('plot "' +\ + fname + '" every ::1 using 1:($' + repr(col_avg) +\ + ') title "Average Latency", "' + fname +'" every ::1 using 1:($' +\ repr(col_min) + ') title "Minimum Latency", "' +\ - fname + '" using 1:($' + repr(col_max) + ') title "Maximum Latency"\n') + fname + '" every ::1 using 1:($' + repr(col_max) +\ + ') title "Maximum Latency"\n') of.close() call(["gnuplot", gcmd]) os.remove(gcmd) @@ -150,7 +157,7 @@ set yrange [0:]\n''') ofname = name + sfx + '.latency2.png' of.write('set output "' + ofname + '"\n') of.write('plot "' + lfile + sfx +\ - '" using (($2 * 100)/$4) title "' + name + '"\n') + '" every ::1 using (($2 * 100)/$4) title "' + name + '"\n') of.close() call(["gnuplot", gcmd]) os.remove(gcmd) @@ -182,7 +189,7 @@ set yrange [0:]\n''' % { ofname = name + sfx + '.latency3.png' of.write('set output "' + ofname + '"\n') of.write('plot "' + lfile + sfx +\ - '" using 1:(($3 * 100)/$4) title "' + name + '"\n') + '" every ::1 using 1:(($3 * 100)/$4) title "' + name + '"\n') of.close() call(["gnuplot", gcmd]) os.remove(gcmd) @@ -192,13 +199,13 @@ def process_file(fname): # NOTE: The operations below must be in this exact order to match # the operation latency output in the monitor file. opdict={'read':0, 'insert':0, 'update':0} - process_monitor(fname, ckptlist, opdict) # This assumes the monitor file has the string "monitor" # and any other (optional) characters in the filename are a suffix. sfx = os.path.basename(fname).replace('monitor','') dirname = os.path.dirname(fname) + process_monitor(fname, sfx, ckptlist, opdict) column = 7 # average, minimum, maximum start in column 7 for k, v in opdict.items(): if v != 0: diff --git a/tools/wtperf_stats.py b/tools/wtperf_stats.py new file mode 100644 index 00000000000..e18632c27ff --- /dev/null +++ b/tools/wtperf_stats.py @@ -0,0 +1,161 @@ +#!/usr/bin/env python +# +# Public Domain 2008-2014 WiredTiger, Inc. +# +# This is free and unencumbered software released into the public domain. +# +# Anyone is free to copy, modify, publish, use, compile, sell, or +# distribute this software, either in source code form or as a compiled +# binary, for any purpose, commercial or non-commercial, and by any +# means. +# +# In jurisdictions that recognize copyright laws, the author or authors +# of this software dedicate any and all copyright interest in the +# software to the public domain. We make this dedication for the benefit +# of the public at large and to the detriment of our heirs and +# successors. We intend this dedication to be an overt act of +# relinquishment in perpetuity of all present and future rights to this +# software under copyright law. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# + +import os, csv, operator +from time import mktime + +try: + from wt_nvd3_util import multiChart, parsetime +except ImportError: + print >>sys.stderr, "Could not import wt_nvd3_util.py, it should be\ + in the same directory as %s" % sys.argv[0] + sys.exit(-1) + +try: + from stat_data import no_scale_per_second_list +except ImportError: + print >>sys.stderr, "Could not import stat_data.py, it should be in\ + the same directory as %s" % sys.argv[0] + sys.exit(-1) + +# Fixup the names and values in a dictionary read in from a csv file. One +# field must be "#time" - which is used to calculate the interval. +# Input is a dictionary, output is a list of dictionaries with a single entry. +def munge_dict(values_dict, abstime): + sorted_values = sorted(values_dict, key=operator.itemgetter('#time')) + start_time = parsetime(sorted_values[0]['#time']) + seconds = (parsetime(sorted_values[1]['#time']) - start_time).seconds + if seconds == 0: + seconds = 1 + + ret = [] + for v in sorted_values: + if abstime: + # Build the time series, milliseconds since the epoch + v['#time'] = int(mktime(parsetime(v['#time']).timetuple())) * 1000 + else: + # Build the time series as seconds since the start of the data + v['#time'] = (parsetime(v['#time']) - start_time).seconds + next_val = {} + for title, value in v.items(): + if title.find('NS') != -1: + title = title.replace('NS', u'\u03BCs') + value = float(value) / 1000 + if title == 'checkpoints' and value == 'N': + value = 0 + elif title.find('time') != -1: + title = 'time' + elif title.find('latency') == -1 and \ + title.find('checkpoints') == -1: + title = title + ' per second' + value = float(value) / seconds + next_val[title] = value + ret.append(next_val) + + return ret + +def addPlotsToChart(chart, graph_data, wtstat_chart = False): + # Extract the times - they are the same for all lines. + times = [] + for v in graph_data: + times.append(v['time']) + + # Add a line to the graph for each field in the CSV file in alphabetical + # order, so the key is sorted. + for field in sorted(graph_data[0].keys()): + if field == 'time': + continue + # Split the latency and non-latency measurements onto different scales + axis = "1" + if not wtstat_chart and field.find('latency') == -1: + axis="2" + ydata = [] + for v in graph_data: + ydata.append(v[field]) + chart.add_serie(x=times, y=ydata, name=field, type="line", yaxis=axis) + +# Input parameters are a chart populated with WiredTiger statistics and +# the directory where the wtperf monitor file can be found. +def addPlotsToStatsChart(chart, dirname, abstime): + fname = os.path.join(dirname, 'monitor') + try: + with open(fname, 'rb') as csvfile: + reader = csv.DictReader(csvfile) + # Transform the data into something NVD3 can digest + graph_data = munge_dict(reader, abstime) + except IOError: + print >>sys.stderr, "Could not open wtperf monitor file." + sys.exit(-1) + addPlotsToChart(chart, graph_data, 1) + +def main(): + # Parse the command line + import argparse + + parser = argparse.ArgumentParser(description='Create graphs from WiredTiger statistics.') + parser.add_argument('--abstime', action='store_true', + help='use absolute time on the x axis') + parser.add_argument('--output', '-o', metavar='file', + default='wtperf_stats.html', help='HTML output file') + parser.add_argument('files', metavar='file', nargs='+', + help='input monitor file generated by WiredTiger wtperf application') + args = parser.parse_args() + + output_file = open(args.output, 'w') + + if len(args.files) != 1: + print 'Script currently only supports a single monitor file' + exit (1) + + chart_extra = {} + # Add in the x axis if the user wants time. + if args.abstime: + chart_extra['x_axis_format'] = '%H:%M:%S' + + for f in args.files: + with open(f, 'rb') as csvfile: + reader = csv.DictReader(csvfile) + # Transform the data into something NVD3 can digest + graph_data = munge_dict(reader, args.abstime) + + chart = multiChart(name='wtperf', + height=450 + 10*len(graph_data[0].keys()), + resize=True, + x_is_date=args.abstime, + assets_directory='http://source.wiredtiger.com/graphs/', + **chart_extra) + + addPlotsToChart(chart, graph_data) + + chart.buildhtml() + output_file.write(chart.htmlcontent) + output_file.close() + +if __name__ == '__main__': + main() + diff --git a/tools/wtstats.py b/tools/wtstats.py index 766c49989b6..37b28e6c13a 100644 --- a/tools/wtstats.py +++ b/tools/wtstats.py @@ -28,38 +28,35 @@ import fileinput, os, re, shutil, sys, textwrap from collections import defaultdict -from datetime import datetime from time import mktime from subprocess import call try: from stat_data import no_scale_per_second_list except ImportError: - print >>sys.stderr, "Could not import stat_data.py, it should be in the same directory as %s" % sys.argv[0] + print >>sys.stderr, "Could not import stat_data.py, it should be\ + in the same directory as %s" % sys.argv[0] sys.exit(-1) try: - from nvd3 import lineChart, lineWithFocusChart + from wtperf_stats import addPlotsToStatsChart except ImportError: - print >>sys.stderr, "Could not import nvd3. Please install it *from source* (other versions may be missing features that we rely on). Run these commands: git clone https://github.com/areski/python-nvd3.git ; cd python-nvd3 ; sudo python setup.py install" + print >>sys.stderr, "Could not import wtperf_stats.py, it should be\ + in the same directory as %s" % sys.argv[0] sys.exit(-1) +try: + from wt_nvd3_util import multiChart, parsetime +except ImportError: + print >>sys.stderr, "Could not import wt_nvd3_util.py, it should be\ + in the same directory as %s" % sys.argv[0] + sys.exit(-1) -# Add a multiChart type so we can overlay line graphs -class multiChart(lineChart): - def __init__(self, **kwargs): - lineChart.__init__(self, **kwargs) - - # Fix the axes - del self.axislist['yAxis'] - self.create_y_axis('yAxis1', format=kwargs.get('y_axis_format', '.02f')) - self.create_y_axis('yAxis2', format=kwargs.get('y_axis_format', '.02f')) - -TIMEFMT = "%b %d %H:%M:%S" - -thisyear = datetime.today().year -def parsetime(s): - return datetime.strptime(s, TIMEFMT).replace(year=thisyear) +try: + from nvd3 import lineChart, lineWithFocusChart +except ImportError: + print >>sys.stderr, "Could not import nvd3. Please install it *from source* (other versions may be missing features that we rely on). Run these commands: git clone https://github.com/areski/python-nvd3.git ; cd python-nvd3 ; sudo python setup.py install" + sys.exit(-1) # Plot a set of entries for a title. def munge(title, values): @@ -93,7 +90,7 @@ def munge(title, values): # Parse the command line import argparse -parser = argparse.ArgumentParser(description='Create graphs from WiredTIger statistics.') +parser = argparse.ArgumentParser(description='Create graphs from WiredTiger statistics.') parser.add_argument('--abstime', action='store_true', help='use absolute time on the x axis') parser.add_argument('--focus', action='store_true', @@ -108,6 +105,8 @@ parser.add_argument('--output', '-o', metavar='file', default='wtstats.html', parser.add_argument('--right', '-R', metavar='regexp', type=re.compile, action='append', help='use the right axis for series with titles matching the specifed regexp') +parser.add_argument('--wtperf', '-w', action='store_true', + help='Plot wtperf statistics on the same graph') parser.add_argument('files', metavar='file', nargs='+', help='input files generated by WiredTiger statistics logging') args = parser.parse_args() @@ -211,6 +210,9 @@ for title, yaxis, ydata in results: chart.add_serie(x=xdata, y=(ydata.get(x, 0) for x in xdata), name=title, type="line", yaxis="2" if yaxis else "1") +if args.wtperf: + addPlotsToStatsChart(chart, os.path.dirname(args.files[0]), args.abstime) + chart.buildhtml() output_file.write(chart.htmlcontent) |