From f6d045c32716615b16fdde893b62b8ea5cbfa949 Mon Sep 17 00:00:00 2001 From: Susan LoVerso Date: Fri, 21 Nov 2014 16:12:51 -0500 Subject: Add --all option to create an HTML file for each prefix. #1365 --- tools/stat_data.py | 15 +++++++++ tools/wtstats.py | 96 +++++++++++++++++++++++++++++++++++------------------- 2 files changed, 78 insertions(+), 33 deletions(-) (limited to 'tools') diff --git a/tools/stat_data.py b/tools/stat_data.py index 56218f497b7..afa707c05b6 100644 --- a/tools/stat_data.py +++ b/tools/stat_data.py @@ -72,3 +72,18 @@ no_clear_list = [ 'transaction: transaction range of IDs currently pinned', 'session: open cursor count', ] +prefix_list = [ + 'data-handle', + 'reconciliation', + 'LSM', + 'log', + 'cache', + 'transaction', + 'cursor', + 'connection', + 'session', + 'block-manager', + 'async', + 'btree', + 'compression', +] diff --git a/tools/wtstats.py b/tools/wtstats.py index 882a6fb5be2..ff1dc9577c9 100644 --- a/tools/wtstats.py +++ b/tools/wtstats.py @@ -39,7 +39,7 @@ tool_dir = os.path.split(sys.argv[0])[0] sys.path = [ os.path.join(tool_dir, "3rdparty") ] + sys.path try: - from stat_data import no_scale_per_second_list, no_clear_list + from stat_data import no_scale_per_second_list, no_clear_list, prefix_list except ImportError: print >>sys.stderr, "Could not import stat_data.py, it should be\ in the same directory as %s" % sys.argv[0] @@ -118,6 +118,8 @@ import argparse parser = argparse.ArgumentParser(description='Create graphs from WiredTiger statistics.') parser.add_argument('--abstime', action='store_true', help='use absolute time on the x axis') +parser.add_argument('--all', '-A', action='store_true', + help='generate all series as separate HTML output files by category') parser.add_argument('--clear', action='store_true', help='WiredTiger stats gathered with clear set') parser.add_argument('--focus', action='store_true', @@ -127,8 +129,8 @@ parser.add_argument('--include', '-I', metavar='regexp', help='include series with titles matching the specifed regexp') parser.add_argument('--list', action='store_true', help='list the series that would be displayed') -parser.add_argument('--output', '-o', metavar='file', default='wtstats.html', - help='HTML output file') +parser.add_argument('--output', '-o', metavar='file', default='wtstats', + help='HTML output file prefix') parser.add_argument('--right', '-R', metavar='regexp', type=re.compile, action='append', help='use the right axis for series with titles matching the specifed regexp') @@ -179,13 +181,65 @@ def common_suffix(a, b): b = b[1:] return b +def output_series(results, prefix=None): + # open the output file based on prefix + if prefix == None: + outputname = args.output + '.html' + else: + outputname = args.output +'.' + prefix + '.html' + + if prefix != None: + this_series = [] + for title, yaxis, ydata in results: + if not prefix in title: + continue + # print 'Appending to dataset: ' + title + this_series.append((title, yaxis, ydata)) + else: + this_series = results + + if len(this_series) == 0: + print 'Prefix: ' + prefix + ' has no data' + return + + #--------------------------------------- + if args.right: + charttype = multiChart + elif args.focus: + charttype = lineWithFocusChart + else: + charttype = lineChart + + chart_extra = {} + # Add in the x axis if the user wants time. + if args.abstime: + chart_extra['x_axis_format'] = '%H:%M:%S' + + # Create the chart, add the series + chart = charttype(name='statlog', height=450+10*len(this_series), resize=True, x_is_date=args.abstime, y_axis_format='g', assets_directory='http://source.wiredtiger.com/graphs/', **chart_extra) + + for title, yaxis, ydata in this_series: + chart.add_serie(x=xdata, y=(ydata.get(x, 0) for x in xdata), name=title, + type="line", yaxis="2" if yaxis else "1") + + if args.wtperf: + addPlotsToStatsChart(chart, os.path.dirname(args.files[0]), args.abstime) + + chart.buildhtml() + output_file = open(outputname, 'w') + output_file.write(chart.htmlcontent) + + #close Html file + output_file.close() + + # Split out the data, convert timestamps results = [] for title, values in sorted(d.iteritems()): title, ydata = munge(title, values) # Ignore entries if a list of regular expressions was given if args.include and not [r for r in args.include if r.search(title)]: - continue + continue yaxis = args.right and [r for r in args.right if r.search(title)] prefix = title if prefix is None else common_prefix(prefix, title) suffix = title if suffix is None else common_suffix(title, suffix) @@ -215,33 +269,9 @@ if args.list: # Figure out the full set of x axis values xdata = sorted(set(k for k in ydata.iterkeys() for ydata in results)) -# open the output file -output_file = open(args.output, 'w') -#--------------------------------------- -if args.right: - charttype = multiChart -elif args.focus: - charttype = lineWithFocusChart -else: - charttype = lineChart - -chart_extra = {} -# Add in the x axis if the user wants time. -if args.abstime: - chart_extra['x_axis_format'] = '%H:%M:%S' - -# Create the chart, add the series -chart = charttype(name='statlog', height=450+10*len(results), resize=True, x_is_date=args.abstime, y_axis_format='g', assets_directory='http://source.wiredtiger.com/graphs/', **chart_extra) - -for title, yaxis, ydata in results: - chart.add_serie(x=xdata, y=(ydata.get(x, 0) for x in xdata), name=title, - type="line", yaxis="2" if yaxis else "1") - -if args.wtperf: - addPlotsToStatsChart(chart, os.path.dirname(args.files[0]), args.abstime) - -chart.buildhtml() -output_file.write(chart.htmlcontent) +output_series(results) -#close Html file -output_file.close() +# If the user wants the stats split up by prefix type do so. +if args.all: + for prefix in prefix_list: + output_series(results, prefix) -- cgit v1.2.1 From b675e61ba8a3e261caef77cb6e30715bac0beff1 Mon Sep 17 00:00:00 2001 From: Susan LoVerso Date: Mon, 24 Nov 2014 15:48:03 -0500 Subject: Accept either files or a directory for the list of stat files. #1365 --- tools/wtstats.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) (limited to 'tools') diff --git a/tools/wtstats.py b/tools/wtstats.py index ff1dc9577c9..6f38e1d7c0d 100644 --- a/tools/wtstats.py +++ b/tools/wtstats.py @@ -26,7 +26,7 @@ # OTHER DEALINGS IN THE SOFTWARE. # -import fileinput, os, re, shutil, sys, textwrap +import fileinput, glob, os, re, shutil, sys, textwrap from collections import defaultdict from time import mktime from subprocess import call @@ -137,7 +137,7 @@ parser.add_argument('--right', '-R', metavar='regexp', parser.add_argument('--wtperf', '-w', action='store_true', help='Plot wtperf statistics on the same graph') parser.add_argument('files', metavar='file', nargs='+', - help='input files generated by WiredTiger statistics logging') + help='input files or directories generated by WiredTiger statistics logging') args = parser.parse_args() # Don't require users to specify regexps twice for right axis @@ -152,9 +152,18 @@ if args.include and args.right: # Read the input file(s) into a dictionary of lists. d = defaultdict(list) for f in args.files: - for line in open(f, 'rU'): - month, day, time, v, title = line.strip('\n').split(" ", 4) - d[title].append((month + " " + day + " " + time, v)) + if os.path.isfile(f): + for line in open(f, 'rU'): + month, day, time, v, title = line.strip('\n').split(" ", 4) + d[title].append((month + " " + day + " " + time, v)) + elif os.path.isdir(f): + stat_regex = f + '/WiredTigerStat*' + statfiles = glob.glob(stat_regex) + for s in statfiles: + print 'Processing ' + s + for line in open(s, 'rU'): + month, day, time, v, title = line.strip('\n').split(" ", 4) + d[title].append((month + " " + day + " " + time, v)) # Process the series, eliminate constants for title, values in sorted(d.iteritems()): -- cgit v1.2.1 From 19f7180a6a0ef2ad96d83086bf36c6acc315dd45 Mon Sep 17 00:00:00 2001 From: Susan LoVerso Date: Wed, 26 Nov 2014 15:11:22 -0500 Subject: First pass at adding groupings for stats. #1365 --- tools/stat_data.py | 1 + tools/wtstats.py | 25 +++++++++++++++++++------ 2 files changed, 20 insertions(+), 6 deletions(-) (limited to 'tools') diff --git a/tools/stat_data.py b/tools/stat_data.py index afa707c05b6..e596fc014e7 100644 --- a/tools/stat_data.py +++ b/tools/stat_data.py @@ -87,3 +87,4 @@ prefix_list = [ 'btree', 'compression', ] +groups = {'cursor': ['cursor', 'session'], 'lsm': ['LSM', 'transaction'], 'system': ['connection', 'data-handle', 'session'], 'evict': ['cache', 'connection', 'block-manager'], 'memory': ['cache', 'connection', 'reconciliation']} \ No newline at end of file diff --git a/tools/wtstats.py b/tools/wtstats.py index 6f38e1d7c0d..249da3c4324 100644 --- a/tools/wtstats.py +++ b/tools/wtstats.py @@ -39,7 +39,8 @@ tool_dir = os.path.split(sys.argv[0])[0] sys.path = [ os.path.join(tool_dir, "3rdparty") ] + sys.path try: - from stat_data import no_scale_per_second_list, no_clear_list, prefix_list + from stat_data \ + import groups, no_scale_per_second_list, no_clear_list, prefix_list except ImportError: print >>sys.stderr, "Could not import stat_data.py, it should be\ in the same directory as %s" % sys.argv[0] @@ -190,25 +191,35 @@ def common_suffix(a, b): b = b[1:] return b -def output_series(results, prefix=None): +def output_series(results, prefix=None, grouplist=[]): # open the output file based on prefix if prefix == None: outputname = args.output + '.html' - else: + elif len(grouplist) == 0: outputname = args.output +'.' + prefix + '.html' + else: + outputname = args.output +'.group.' + prefix + '.html' - if prefix != None: + if prefix != None and len(grouplist) == 0: this_series = [] for title, yaxis, ydata in results: if not prefix in title: continue - # print 'Appending to dataset: ' + title + #print 'Appending to dataset: ' + title this_series.append((title, yaxis, ydata)) + elif prefix != None and len(grouplist) > 0: + this_series = [] + for title, yaxis, ydata in results: + for subgroup in grouplist: + if not subgroup in title: + continue + # print 'Appending to dataset: ' + title + this_series.append((title, yaxis, ydata)) else: this_series = results if len(this_series) == 0: - print 'Prefix: ' + prefix + ' has no data' + print 'Output: ' + outputname + ' has no data. Do not create.' return #--------------------------------------- @@ -284,3 +295,5 @@ output_series(results) if args.all: for prefix in prefix_list: output_series(results, prefix) + for group in groups.keys(): + output_series(results, group, groups[group]) -- cgit v1.2.1 From e55dc7cd6fa446cf6e04640fc0b3e8b646a52074 Mon Sep 17 00:00:00 2001 From: Michael Cahill Date: Fri, 28 Nov 2014 11:10:16 +1100 Subject: Make file list processing a little more Python idiomatic. --- tools/wtstats.py | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) (limited to 'tools') diff --git a/tools/wtstats.py b/tools/wtstats.py index 249da3c4324..cc2ebd80877 100644 --- a/tools/wtstats.py +++ b/tools/wtstats.py @@ -26,8 +26,9 @@ # OTHER DEALINGS IN THE SOFTWARE. # -import fileinput, glob, os, re, shutil, sys, textwrap +import fileinput, os, re, shutil, sys, textwrap from collections import defaultdict +from glob import glob from time import mktime from subprocess import call @@ -151,20 +152,20 @@ if args.include and args.right: args.include += args.right # Read the input file(s) into a dictionary of lists. +def getfiles(l): + for f in l: + if os.path.isfile(f): + yield f + elif os.path.isdir(f): + for s in glob(os.path.join(f, 'WiredTigerStat*')): + print 'Processing ' + s + yield s + d = defaultdict(list) -for f in args.files: - if os.path.isfile(f): - for line in open(f, 'rU'): - month, day, time, v, title = line.strip('\n').split(" ", 4) - d[title].append((month + " " + day + " " + time, v)) - elif os.path.isdir(f): - stat_regex = f + '/WiredTigerStat*' - statfiles = glob.glob(stat_regex) - for s in statfiles: - print 'Processing ' + s - for line in open(s, 'rU'): - month, day, time, v, title = line.strip('\n').split(" ", 4) - d[title].append((month + " " + day + " " + time, v)) +for f in getfiles(args.files): + for line in open(f, 'rU'): + month, day, time, v, title = line.strip('\n').split(" ", 4) + d[title].append((month + " " + day + " " + time, v)) # Process the series, eliminate constants for title, values in sorted(d.iteritems()): -- cgit v1.2.1