diff options
author | Susan LoVerso <sue@wiredtiger.com> | 2014-12-01 10:35:51 -0500 |
---|---|---|
committer | Susan LoVerso <sue@wiredtiger.com> | 2014-12-01 10:35:51 -0500 |
commit | 3028504b548bbed07ec9e2e605c6954a3ce34836 (patch) | |
tree | 8897e0c7b1a15cb7ccf86ada4dc14830ced6bf9d /tools | |
parent | 2a9f969c38d58d6d97843216857bee8b1fb12f60 (diff) | |
parent | f57d610ea917e880a181d94b7ae83401983fcaa0 (diff) | |
download | mongo-3028504b548bbed07ec9e2e605c6954a3ce34836.tar.gz |
Merge branch 'develop' into log-recycle
Diffstat (limited to 'tools')
-rw-r--r-- | tools/stat_data.py | 16 | ||||
-rw-r--r-- | tools/wtstats.py | 123 |
2 files changed, 104 insertions, 35 deletions
diff --git a/tools/stat_data.py b/tools/stat_data.py index eb61729b536..dd73dbf5c6e 100644 --- a/tools/stat_data.py +++ b/tools/stat_data.py @@ -73,3 +73,19 @@ no_clear_list = [ 'transaction: transaction range of IDs currently pinned', 'session: open cursor count', ] +prefix_list = [ + 'data-handle', + 'reconciliation', + 'LSM', + 'log', + 'cache', + 'transaction', + 'cursor', + 'connection', + 'session', + 'block-manager', + 'async', + 'btree', + 'compression', +] +groups = {'cursor': ['cursor', 'session'], 'lsm': ['LSM', 'transaction'], 'system': ['connection', 'data-handle', 'session'], 'evict': ['cache', 'connection', 'block-manager'], 'memory': ['cache', 'connection', 'reconciliation']} diff --git a/tools/wtstats.py b/tools/wtstats.py index 882a6fb5be2..cc2ebd80877 100644 --- a/tools/wtstats.py +++ b/tools/wtstats.py @@ -28,6 +28,7 @@ import fileinput, os, re, shutil, sys, textwrap from collections import defaultdict +from glob import glob from time import mktime from subprocess import call @@ -39,7 +40,8 @@ tool_dir = os.path.split(sys.argv[0])[0] sys.path = [ os.path.join(tool_dir, "3rdparty") ] + sys.path try: - from stat_data import no_scale_per_second_list, no_clear_list + from stat_data \ + import groups, no_scale_per_second_list, no_clear_list, prefix_list except ImportError: print >>sys.stderr, "Could not import stat_data.py, it should be\ in the same directory as %s" % sys.argv[0] @@ -118,6 +120,8 @@ import argparse parser = argparse.ArgumentParser(description='Create graphs from WiredTiger statistics.') parser.add_argument('--abstime', action='store_true', help='use absolute time on the x axis') +parser.add_argument('--all', '-A', action='store_true', + help='generate all series as separate HTML output files by category') parser.add_argument('--clear', action='store_true', help='WiredTiger stats gathered with clear set') parser.add_argument('--focus', action='store_true', @@ -127,15 +131,15 @@ parser.add_argument('--include', '-I', metavar='regexp', help='include series with titles matching the specifed regexp') parser.add_argument('--list', action='store_true', help='list the series that would be displayed') -parser.add_argument('--output', '-o', metavar='file', default='wtstats.html', - help='HTML output file') +parser.add_argument('--output', '-o', metavar='file', default='wtstats', + help='HTML output file prefix') parser.add_argument('--right', '-R', metavar='regexp', type=re.compile, action='append', help='use the right axis for series with titles matching the specifed regexp') parser.add_argument('--wtperf', '-w', action='store_true', help='Plot wtperf statistics on the same graph') parser.add_argument('files', metavar='file', nargs='+', - help='input files generated by WiredTiger statistics logging') + help='input files or directories generated by WiredTiger statistics logging') args = parser.parse_args() # Don't require users to specify regexps twice for right axis @@ -148,8 +152,17 @@ if args.include and args.right: args.include += args.right # Read the input file(s) into a dictionary of lists. +def getfiles(l): + for f in l: + if os.path.isfile(f): + yield f + elif os.path.isdir(f): + for s in glob(os.path.join(f, 'WiredTigerStat*')): + print 'Processing ' + s + yield s + d = defaultdict(list) -for f in args.files: +for f in getfiles(args.files): for line in open(f, 'rU'): month, day, time, v, title = line.strip('\n').split(" ", 4) d[title].append((month + " " + day + " " + time, v)) @@ -179,13 +192,75 @@ def common_suffix(a, b): b = b[1:] return b +def output_series(results, prefix=None, grouplist=[]): + # open the output file based on prefix + if prefix == None: + outputname = args.output + '.html' + elif len(grouplist) == 0: + outputname = args.output +'.' + prefix + '.html' + else: + outputname = args.output +'.group.' + prefix + '.html' + + if prefix != None and len(grouplist) == 0: + this_series = [] + for title, yaxis, ydata in results: + if not prefix in title: + continue + #print 'Appending to dataset: ' + title + this_series.append((title, yaxis, ydata)) + elif prefix != None and len(grouplist) > 0: + this_series = [] + for title, yaxis, ydata in results: + for subgroup in grouplist: + if not subgroup in title: + continue + # print 'Appending to dataset: ' + title + this_series.append((title, yaxis, ydata)) + else: + this_series = results + + if len(this_series) == 0: + print 'Output: ' + outputname + ' has no data. Do not create.' + return + + #--------------------------------------- + if args.right: + charttype = multiChart + elif args.focus: + charttype = lineWithFocusChart + else: + charttype = lineChart + + chart_extra = {} + # Add in the x axis if the user wants time. + if args.abstime: + chart_extra['x_axis_format'] = '%H:%M:%S' + + # Create the chart, add the series + chart = charttype(name='statlog', height=450+10*len(this_series), resize=True, x_is_date=args.abstime, y_axis_format='g', assets_directory='http://source.wiredtiger.com/graphs/', **chart_extra) + + for title, yaxis, ydata in this_series: + chart.add_serie(x=xdata, y=(ydata.get(x, 0) for x in xdata), name=title, + type="line", yaxis="2" if yaxis else "1") + + if args.wtperf: + addPlotsToStatsChart(chart, os.path.dirname(args.files[0]), args.abstime) + + chart.buildhtml() + output_file = open(outputname, 'w') + output_file.write(chart.htmlcontent) + + #close Html file + output_file.close() + + # Split out the data, convert timestamps results = [] for title, values in sorted(d.iteritems()): title, ydata = munge(title, values) # Ignore entries if a list of regular expressions was given if args.include and not [r for r in args.include if r.search(title)]: - continue + continue yaxis = args.right and [r for r in args.right if r.search(title)] prefix = title if prefix is None else common_prefix(prefix, title) suffix = title if suffix is None else common_suffix(title, suffix) @@ -215,33 +290,11 @@ if args.list: # Figure out the full set of x axis values xdata = sorted(set(k for k in ydata.iterkeys() for ydata in results)) -# open the output file -output_file = open(args.output, 'w') -#--------------------------------------- -if args.right: - charttype = multiChart -elif args.focus: - charttype = lineWithFocusChart -else: - charttype = lineChart - -chart_extra = {} -# Add in the x axis if the user wants time. -if args.abstime: - chart_extra['x_axis_format'] = '%H:%M:%S' - -# Create the chart, add the series -chart = charttype(name='statlog', height=450+10*len(results), resize=True, x_is_date=args.abstime, y_axis_format='g', assets_directory='http://source.wiredtiger.com/graphs/', **chart_extra) - -for title, yaxis, ydata in results: - chart.add_serie(x=xdata, y=(ydata.get(x, 0) for x in xdata), name=title, - type="line", yaxis="2" if yaxis else "1") - -if args.wtperf: - addPlotsToStatsChart(chart, os.path.dirname(args.files[0]), args.abstime) - -chart.buildhtml() -output_file.write(chart.htmlcontent) +output_series(results) -#close Html file -output_file.close() +# If the user wants the stats split up by prefix type do so. +if args.all: + for prefix in prefix_list: + output_series(results, prefix) + for group in groups.keys(): + output_series(results, group, groups[group]) |