summaryrefslogtreecommitdiff
path: root/third-party/benchmark/tools/gbench/report.py
diff options
context:
space:
mode:
Diffstat (limited to 'third-party/benchmark/tools/gbench/report.py')
-rw-r--r--third-party/benchmark/tools/gbench/report.py991
1 files changed, 991 insertions, 0 deletions
diff --git a/third-party/benchmark/tools/gbench/report.py b/third-party/benchmark/tools/gbench/report.py
new file mode 100644
index 000000000000..6bea82f6bf7b
--- /dev/null
+++ b/third-party/benchmark/tools/gbench/report.py
@@ -0,0 +1,991 @@
+"""report.py - Utilities for reporting statistics about benchmark results
+"""
+
+import unittest
+import os
+import re
+import copy
+import random
+
+from scipy.stats import mannwhitneyu
+
+
+class BenchmarkColor(object):
+ def __init__(self, name, code):
+ self.name = name
+ self.code = code
+
+ def __repr__(self):
+ return '%s%r' % (self.__class__.__name__,
+ (self.name, self.code))
+
+ def __format__(self, format):
+ return self.code
+
+
+# Benchmark Colors Enumeration
+BC_NONE = BenchmarkColor('NONE', '')
+BC_MAGENTA = BenchmarkColor('MAGENTA', '\033[95m')
+BC_CYAN = BenchmarkColor('CYAN', '\033[96m')
+BC_OKBLUE = BenchmarkColor('OKBLUE', '\033[94m')
+BC_OKGREEN = BenchmarkColor('OKGREEN', '\033[32m')
+BC_HEADER = BenchmarkColor('HEADER', '\033[92m')
+BC_WARNING = BenchmarkColor('WARNING', '\033[93m')
+BC_WHITE = BenchmarkColor('WHITE', '\033[97m')
+BC_FAIL = BenchmarkColor('FAIL', '\033[91m')
+BC_ENDC = BenchmarkColor('ENDC', '\033[0m')
+BC_BOLD = BenchmarkColor('BOLD', '\033[1m')
+BC_UNDERLINE = BenchmarkColor('UNDERLINE', '\033[4m')
+
+UTEST_MIN_REPETITIONS = 2
+UTEST_OPTIMAL_REPETITIONS = 9 # Lowest reasonable number, More is better.
+UTEST_COL_NAME = "_pvalue"
+
+
+def color_format(use_color, fmt_str, *args, **kwargs):
+ """
+ Return the result of 'fmt_str.format(*args, **kwargs)' after transforming
+ 'args' and 'kwargs' according to the value of 'use_color'. If 'use_color'
+ is False then all color codes in 'args' and 'kwargs' are replaced with
+ the empty string.
+ """
+ assert use_color is True or use_color is False
+ if not use_color:
+ args = [arg if not isinstance(arg, BenchmarkColor) else BC_NONE
+ for arg in args]
+ kwargs = {key: arg if not isinstance(arg, BenchmarkColor) else BC_NONE
+ for key, arg in kwargs.items()}
+ return fmt_str.format(*args, **kwargs)
+
+
+def find_longest_name(benchmark_list):
+ """
+ Return the length of the longest benchmark name in a given list of
+ benchmark JSON objects
+ """
+ longest_name = 1
+ for bc in benchmark_list:
+ if len(bc['name']) > longest_name:
+ longest_name = len(bc['name'])
+ return longest_name
+
+
+def calculate_change(old_val, new_val):
+ """
+ Return a float representing the decimal change between old_val and new_val.
+ """
+ if old_val == 0 and new_val == 0:
+ return 0.0
+ if old_val == 0:
+ return float(new_val - old_val) / (float(old_val + new_val) / 2)
+ return float(new_val - old_val) / abs(old_val)
+
+
+def filter_benchmark(json_orig, family, replacement=""):
+ """
+ Apply a filter to the json, and only leave the 'family' of benchmarks.
+ """
+ regex = re.compile(family)
+ filtered = {}
+ filtered['benchmarks'] = []
+ for be in json_orig['benchmarks']:
+ if not regex.search(be['name']):
+ continue
+ filteredbench = copy.deepcopy(be) # Do NOT modify the old name!
+ filteredbench['name'] = regex.sub(replacement, filteredbench['name'])
+ filtered['benchmarks'].append(filteredbench)
+ return filtered
+
+
+def get_unique_benchmark_names(json):
+ """
+ While *keeping* the order, give all the unique 'names' used for benchmarks.
+ """
+ seen = set()
+ uniqued = [x['name'] for x in json['benchmarks']
+ if x['name'] not in seen and
+ (seen.add(x['name']) or True)]
+ return uniqued
+
+
+def intersect(list1, list2):
+ """
+ Given two lists, get a new list consisting of the elements only contained
+ in *both of the input lists*, while preserving the ordering.
+ """
+ return [x for x in list1 if x in list2]
+
+
+def is_potentially_comparable_benchmark(x):
+ return ('time_unit' in x and 'real_time' in x and 'cpu_time' in x)
+
+
+def partition_benchmarks(json1, json2):
+ """
+ While preserving the ordering, find benchmarks with the same names in
+ both of the inputs, and group them.
+ (i.e. partition/filter into groups with common name)
+ """
+ json1_unique_names = get_unique_benchmark_names(json1)
+ json2_unique_names = get_unique_benchmark_names(json2)
+ names = intersect(json1_unique_names, json2_unique_names)
+ partitions = []
+ for name in names:
+ time_unit = None
+ # Pick the time unit from the first entry of the lhs benchmark.
+ # We should be careful not to crash with unexpected input.
+ for x in json1['benchmarks']:
+ if (x['name'] == name and is_potentially_comparable_benchmark(x)):
+ time_unit = x['time_unit']
+ break
+ if time_unit is None:
+ continue
+ # Filter by name and time unit.
+ # All the repetitions are assumed to be comparable.
+ lhs = [x for x in json1['benchmarks'] if x['name'] == name and
+ x['time_unit'] == time_unit]
+ rhs = [x for x in json2['benchmarks'] if x['name'] == name and
+ x['time_unit'] == time_unit]
+ partitions.append([lhs, rhs])
+ return partitions
+
+
+def extract_field(partition, field_name):
+ # The count of elements may be different. We want *all* of them.
+ lhs = [x[field_name] for x in partition[0]]
+ rhs = [x[field_name] for x in partition[1]]
+ return [lhs, rhs]
+
+
+def calc_utest(timings_cpu, timings_time):
+ min_rep_cnt = min(len(timings_time[0]),
+ len(timings_time[1]),
+ len(timings_cpu[0]),
+ len(timings_cpu[1]))
+
+ # Does *everything* has at least UTEST_MIN_REPETITIONS repetitions?
+ if min_rep_cnt < UTEST_MIN_REPETITIONS:
+ return False, None, None
+
+ time_pvalue = mannwhitneyu(
+ timings_time[0], timings_time[1], alternative='two-sided').pvalue
+ cpu_pvalue = mannwhitneyu(
+ timings_cpu[0], timings_cpu[1], alternative='two-sided').pvalue
+
+ return (min_rep_cnt >= UTEST_OPTIMAL_REPETITIONS), cpu_pvalue, time_pvalue
+
+def print_utest(bc_name, utest, utest_alpha, first_col_width, use_color=True):
+ def get_utest_color(pval):
+ return BC_FAIL if pval >= utest_alpha else BC_OKGREEN
+
+ # Check if we failed miserably with minimum required repetitions for utest
+ if not utest['have_optimal_repetitions'] and utest['cpu_pvalue'] is None and utest['time_pvalue'] is None:
+ return []
+
+ dsc = "U Test, Repetitions: {} vs {}".format(
+ utest['nr_of_repetitions'], utest['nr_of_repetitions_other'])
+ dsc_color = BC_OKGREEN
+
+ # We still got some results to show but issue a warning about it.
+ if not utest['have_optimal_repetitions']:
+ dsc_color = BC_WARNING
+ dsc += ". WARNING: Results unreliable! {}+ repetitions recommended.".format(
+ UTEST_OPTIMAL_REPETITIONS)
+
+ special_str = "{}{:<{}s}{endc}{}{:16.4f}{endc}{}{:16.4f}{endc}{} {}"
+
+ return [color_format(use_color,
+ special_str,
+ BC_HEADER,
+ "{}{}".format(bc_name, UTEST_COL_NAME),
+ first_col_width,
+ get_utest_color(
+ utest['time_pvalue']), utest['time_pvalue'],
+ get_utest_color(
+ utest['cpu_pvalue']), utest['cpu_pvalue'],
+ dsc_color, dsc,
+ endc=BC_ENDC)]
+
+
+def get_difference_report(
+ json1,
+ json2,
+ utest=False):
+ """
+ Calculate and report the difference between each test of two benchmarks
+ runs specified as 'json1' and 'json2'. Output is another json containing
+ relevant details for each test run.
+ """
+ assert utest is True or utest is False
+
+ diff_report = []
+ partitions = partition_benchmarks(json1, json2)
+ for partition in partitions:
+ benchmark_name = partition[0][0]['name']
+ time_unit = partition[0][0]['time_unit']
+ measurements = []
+ utest_results = {}
+ # Careful, we may have different repetition count.
+ for i in range(min(len(partition[0]), len(partition[1]))):
+ bn = partition[0][i]
+ other_bench = partition[1][i]
+ measurements.append({
+ 'real_time': bn['real_time'],
+ 'cpu_time': bn['cpu_time'],
+ 'real_time_other': other_bench['real_time'],
+ 'cpu_time_other': other_bench['cpu_time'],
+ 'time': calculate_change(bn['real_time'], other_bench['real_time']),
+ 'cpu': calculate_change(bn['cpu_time'], other_bench['cpu_time'])
+ })
+
+ # After processing the whole partition, if requested, do the U test.
+ if utest:
+ timings_cpu = extract_field(partition, 'cpu_time')
+ timings_time = extract_field(partition, 'real_time')
+ have_optimal_repetitions, cpu_pvalue, time_pvalue = calc_utest(timings_cpu, timings_time)
+ if cpu_pvalue and time_pvalue:
+ utest_results = {
+ 'have_optimal_repetitions': have_optimal_repetitions,
+ 'cpu_pvalue': cpu_pvalue,
+ 'time_pvalue': time_pvalue,
+ 'nr_of_repetitions': len(timings_cpu[0]),
+ 'nr_of_repetitions_other': len(timings_cpu[1])
+ }
+
+ # Store only if we had any measurements for given benchmark.
+ # E.g. partition_benchmarks will filter out the benchmarks having
+ # time units which are not compatible with other time units in the
+ # benchmark suite.
+ if measurements:
+ run_type = partition[0][0]['run_type'] if 'run_type' in partition[0][0] else ''
+ aggregate_name = partition[0][0]['aggregate_name'] if run_type == 'aggregate' and 'aggregate_name' in partition[0][0] else ''
+ diff_report.append({
+ 'name': benchmark_name,
+ 'measurements': measurements,
+ 'time_unit': time_unit,
+ 'run_type': run_type,
+ 'aggregate_name': aggregate_name,
+ 'utest': utest_results
+ })
+
+ return diff_report
+
+
+def print_difference_report(
+ json_diff_report,
+ include_aggregates_only=False,
+ utest=False,
+ utest_alpha=0.05,
+ use_color=True):
+ """
+ Calculate and report the difference between each test of two benchmarks
+ runs specified as 'json1' and 'json2'.
+ """
+ assert utest is True or utest is False
+
+ def get_color(res):
+ if res > 0.05:
+ return BC_FAIL
+ elif res > -0.07:
+ return BC_WHITE
+ else:
+ return BC_CYAN
+
+ first_col_width = find_longest_name(json_diff_report)
+ first_col_width = max(
+ first_col_width,
+ len('Benchmark'))
+ first_col_width += len(UTEST_COL_NAME)
+ first_line = "{:<{}s}Time CPU Time Old Time New CPU Old CPU New".format(
+ 'Benchmark', 12 + first_col_width)
+ output_strs = [first_line, '-' * len(first_line)]
+
+ fmt_str = "{}{:<{}s}{endc}{}{:+16.4f}{endc}{}{:+16.4f}{endc}{:14.0f}{:14.0f}{endc}{:14.0f}{:14.0f}"
+ for benchmark in json_diff_report:
+ # *If* we were asked to only include aggregates,
+ # and if it is non-aggregate, then don't print it.
+ if not include_aggregates_only or not 'run_type' in benchmark or benchmark['run_type'] == 'aggregate':
+ for measurement in benchmark['measurements']:
+ output_strs += [color_format(use_color,
+ fmt_str,
+ BC_HEADER,
+ benchmark['name'],
+ first_col_width,
+ get_color(measurement['time']),
+ measurement['time'],
+ get_color(measurement['cpu']),
+ measurement['cpu'],
+ measurement['real_time'],
+ measurement['real_time_other'],
+ measurement['cpu_time'],
+ measurement['cpu_time_other'],
+ endc=BC_ENDC)]
+
+ # After processing the measurements, if requested and
+ # if applicable (e.g. u-test exists for given benchmark),
+ # print the U test.
+ if utest and benchmark['utest']:
+ output_strs += print_utest(benchmark['name'],
+ benchmark['utest'],
+ utest_alpha=utest_alpha,
+ first_col_width=first_col_width,
+ use_color=use_color)
+
+ return output_strs
+
+
+###############################################################################
+# Unit tests
+
+
+class TestGetUniqueBenchmarkNames(unittest.TestCase):
+ def load_results(self):
+ import json
+ testInputs = os.path.join(
+ os.path.dirname(
+ os.path.realpath(__file__)),
+ 'Inputs')
+ testOutput = os.path.join(testInputs, 'test3_run0.json')
+ with open(testOutput, 'r') as f:
+ json = json.load(f)
+ return json
+
+ def test_basic(self):
+ expect_lines = [
+ 'BM_One',
+ 'BM_Two',
+ 'short', # These two are not sorted
+ 'medium', # These two are not sorted
+ ]
+ json = self.load_results()
+ output_lines = get_unique_benchmark_names(json)
+ print("\n")
+ print("\n".join(output_lines))
+ self.assertEqual(len(output_lines), len(expect_lines))
+ for i in range(0, len(output_lines)):
+ self.assertEqual(expect_lines[i], output_lines[i])
+
+
+class TestReportDifference(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls):
+ def load_results():
+ import json
+ testInputs = os.path.join(
+ os.path.dirname(
+ os.path.realpath(__file__)),
+ 'Inputs')
+ testOutput1 = os.path.join(testInputs, 'test1_run1.json')
+ testOutput2 = os.path.join(testInputs, 'test1_run2.json')
+ with open(testOutput1, 'r') as f:
+ json1 = json.load(f)
+ with open(testOutput2, 'r') as f:
+ json2 = json.load(f)
+ return json1, json2
+
+ json1, json2 = load_results()
+ cls.json_diff_report = get_difference_report(json1, json2)
+
+ def test_json_diff_report_pretty_printing(self):
+ expect_lines = [
+ ['BM_SameTimes', '+0.0000', '+0.0000', '10', '10', '10', '10'],
+ ['BM_2xFaster', '-0.5000', '-0.5000', '50', '25', '50', '25'],
+ ['BM_2xSlower', '+1.0000', '+1.0000', '50', '100', '50', '100'],
+ ['BM_1PercentFaster', '-0.0100', '-0.0100', '100', '99', '100', '99'],
+ ['BM_1PercentSlower', '+0.0100', '+0.0100', '100', '101', '100', '101'],
+ ['BM_10PercentFaster', '-0.1000', '-0.1000', '100', '90', '100', '90'],
+ ['BM_10PercentSlower', '+0.1000', '+0.1000', '100', '110', '100', '110'],
+ ['BM_100xSlower', '+99.0000', '+99.0000',
+ '100', '10000', '100', '10000'],
+ ['BM_100xFaster', '-0.9900', '-0.9900',
+ '10000', '100', '10000', '100'],
+ ['BM_10PercentCPUToTime', '+0.1000',
+ '-0.1000', '100', '110', '100', '90'],
+ ['BM_ThirdFaster', '-0.3333', '-0.3334', '100', '67', '100', '67'],
+ ['BM_NotBadTimeUnit', '-0.9000', '+0.2000', '0', '0', '0', '1'],
+ ]
+ output_lines_with_header = print_difference_report(
+ self.json_diff_report, use_color=False)
+ output_lines = output_lines_with_header[2:]
+ print("\n")
+ print("\n".join(output_lines_with_header))
+ self.assertEqual(len(output_lines), len(expect_lines))
+ for i in range(0, len(output_lines)):
+ parts = [x for x in output_lines[i].split(' ') if x]
+ self.assertEqual(len(parts), 7)
+ self.assertEqual(expect_lines[i], parts)
+
+ def test_json_diff_report_output(self):
+ expected_output = [
+ {
+ 'name': 'BM_SameTimes',
+ 'measurements': [{'time': 0.0000, 'cpu': 0.0000, 'real_time': 10, 'real_time_other': 10, 'cpu_time': 10, 'cpu_time_other': 10}],
+ 'time_unit': 'ns',
+ 'utest': {}
+ },
+ {
+ 'name': 'BM_2xFaster',
+ 'measurements': [{'time': -0.5000, 'cpu': -0.5000, 'real_time': 50, 'real_time_other': 25, 'cpu_time': 50, 'cpu_time_other': 25}],
+ 'time_unit': 'ns',
+ 'utest': {}
+ },
+ {
+ 'name': 'BM_2xSlower',
+ 'measurements': [{'time': 1.0000, 'cpu': 1.0000, 'real_time': 50, 'real_time_other': 100, 'cpu_time': 50, 'cpu_time_other': 100}],
+ 'time_unit': 'ns',
+ 'utest': {}
+ },
+ {
+ 'name': 'BM_1PercentFaster',
+ 'measurements': [{'time': -0.0100, 'cpu': -0.0100, 'real_time': 100, 'real_time_other': 98.9999999, 'cpu_time': 100, 'cpu_time_other': 98.9999999}],
+ 'time_unit': 'ns',
+ 'utest': {}
+ },
+ {
+ 'name': 'BM_1PercentSlower',
+ 'measurements': [{'time': 0.0100, 'cpu': 0.0100, 'real_time': 100, 'real_time_other': 101, 'cpu_time': 100, 'cpu_time_other': 101}],
+ 'time_unit': 'ns',
+ 'utest': {}
+ },
+ {
+ 'name': 'BM_10PercentFaster',
+ 'measurements': [{'time': -0.1000, 'cpu': -0.1000, 'real_time': 100, 'real_time_other': 90, 'cpu_time': 100, 'cpu_time_other': 90}],
+ 'time_unit': 'ns',
+ 'utest': {}
+ },
+ {
+ 'name': 'BM_10PercentSlower',
+ 'measurements': [{'time': 0.1000, 'cpu': 0.1000, 'real_time': 100, 'real_time_other': 110, 'cpu_time': 100, 'cpu_time_other': 110}],
+ 'time_unit': 'ns',
+ 'utest': {}
+ },
+ {
+ 'name': 'BM_100xSlower',
+ 'measurements': [{'time': 99.0000, 'cpu': 99.0000, 'real_time': 100, 'real_time_other': 10000, 'cpu_time': 100, 'cpu_time_other': 10000}],
+ 'time_unit': 'ns',
+ 'utest': {}
+ },
+ {
+ 'name': 'BM_100xFaster',
+ 'measurements': [{'time': -0.9900, 'cpu': -0.9900, 'real_time': 10000, 'real_time_other': 100, 'cpu_time': 10000, 'cpu_time_other': 100}],
+ 'time_unit': 'ns',
+ 'utest': {}
+ },
+ {
+ 'name': 'BM_10PercentCPUToTime',
+ 'measurements': [{'time': 0.1000, 'cpu': -0.1000, 'real_time': 100, 'real_time_other': 110, 'cpu_time': 100, 'cpu_time_other': 90}],
+ 'time_unit': 'ns',
+ 'utest': {}
+ },
+ {
+ 'name': 'BM_ThirdFaster',
+ 'measurements': [{'time': -0.3333, 'cpu': -0.3334, 'real_time': 100, 'real_time_other': 67, 'cpu_time': 100, 'cpu_time_other': 67}],
+ 'time_unit': 'ns',
+ 'utest': {}
+ },
+ {
+ 'name': 'BM_NotBadTimeUnit',
+ 'measurements': [{'time': -0.9000, 'cpu': 0.2000, 'real_time': 0.4, 'real_time_other': 0.04, 'cpu_time': 0.5, 'cpu_time_other': 0.6}],
+ 'time_unit': 's',
+ 'utest': {}
+ },
+ ]
+ self.assertEqual(len(self.json_diff_report), len(expected_output))
+ for out, expected in zip(
+ self.json_diff_report, expected_output):
+ self.assertEqual(out['name'], expected['name'])
+ self.assertEqual(out['time_unit'], expected['time_unit'])
+ assert_utest(self, out, expected)
+ assert_measurements(self, out, expected)
+
+
+class TestReportDifferenceBetweenFamilies(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls):
+ def load_result():
+ import json
+ testInputs = os.path.join(
+ os.path.dirname(
+ os.path.realpath(__file__)),
+ 'Inputs')
+ testOutput = os.path.join(testInputs, 'test2_run.json')
+ with open(testOutput, 'r') as f:
+ json = json.load(f)
+ return json
+
+ json = load_result()
+ json1 = filter_benchmark(json, "BM_Z.ro", ".")
+ json2 = filter_benchmark(json, "BM_O.e", ".")
+ cls.json_diff_report = get_difference_report(json1, json2)
+
+ def test_json_diff_report_pretty_printing(self):
+ expect_lines = [
+ ['.', '-0.5000', '-0.5000', '10', '5', '10', '5'],
+ ['./4', '-0.5000', '-0.5000', '40', '20', '40', '20'],
+ ['Prefix/.', '-0.5000', '-0.5000', '20', '10', '20', '10'],
+ ['Prefix/./3', '-0.5000', '-0.5000', '30', '15', '30', '15'],
+ ]
+ output_lines_with_header = print_difference_report(
+ self.json_diff_report, use_color=False)
+ output_lines = output_lines_with_header[2:]
+ print("\n")
+ print("\n".join(output_lines_with_header))
+ self.assertEqual(len(output_lines), len(expect_lines))
+ for i in range(0, len(output_lines)):
+ parts = [x for x in output_lines[i].split(' ') if x]
+ self.assertEqual(len(parts), 7)
+ self.assertEqual(expect_lines[i], parts)
+
+ def test_json_diff_report(self):
+ expected_output = [
+ {
+ 'name': u'.',
+ 'measurements': [{'time': -0.5, 'cpu': -0.5, 'real_time': 10, 'real_time_other': 5, 'cpu_time': 10, 'cpu_time_other': 5}],
+ 'time_unit': 'ns',
+ 'utest': {}
+ },
+ {
+ 'name': u'./4',
+ 'measurements': [{'time': -0.5, 'cpu': -0.5, 'real_time': 40, 'real_time_other': 20, 'cpu_time': 40, 'cpu_time_other': 20}],
+ 'time_unit': 'ns',
+ 'utest': {},
+ },
+ {
+ 'name': u'Prefix/.',
+ 'measurements': [{'time': -0.5, 'cpu': -0.5, 'real_time': 20, 'real_time_other': 10, 'cpu_time': 20, 'cpu_time_other': 10}],
+ 'time_unit': 'ns',
+ 'utest': {}
+ },
+ {
+ 'name': u'Prefix/./3',
+ 'measurements': [{'time': -0.5, 'cpu': -0.5, 'real_time': 30, 'real_time_other': 15, 'cpu_time': 30, 'cpu_time_other': 15}],
+ 'time_unit': 'ns',
+ 'utest': {}
+ }
+ ]
+ self.assertEqual(len(self.json_diff_report), len(expected_output))
+ for out, expected in zip(
+ self.json_diff_report, expected_output):
+ self.assertEqual(out['name'], expected['name'])
+ self.assertEqual(out['time_unit'], expected['time_unit'])
+ assert_utest(self, out, expected)
+ assert_measurements(self, out, expected)
+
+
+class TestReportDifferenceWithUTest(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls):
+ def load_results():
+ import json
+ testInputs = os.path.join(
+ os.path.dirname(
+ os.path.realpath(__file__)),
+ 'Inputs')
+ testOutput1 = os.path.join(testInputs, 'test3_run0.json')
+ testOutput2 = os.path.join(testInputs, 'test3_run1.json')
+ with open(testOutput1, 'r') as f:
+ json1 = json.load(f)
+ with open(testOutput2, 'r') as f:
+ json2 = json.load(f)
+ return json1, json2
+
+ json1, json2 = load_results()
+ cls.json_diff_report = get_difference_report(
+ json1, json2, utest=True)
+
+ def test_json_diff_report_pretty_printing(self):
+ expect_lines = [
+ ['BM_One', '-0.1000', '+0.1000', '10', '9', '100', '110'],
+ ['BM_Two', '+0.1111', '-0.0111', '9', '10', '90', '89'],
+ ['BM_Two', '-0.1250', '-0.1628', '8', '7', '86', '72'],
+ ['BM_Two_pvalue',
+ '0.6985',
+ '0.6985',
+ 'U',
+ 'Test,',
+ 'Repetitions:',
+ '2',
+ 'vs',
+ '2.',
+ 'WARNING:',
+ 'Results',
+ 'unreliable!',
+ '9+',
+ 'repetitions',
+ 'recommended.'],
+ ['short', '-0.1250', '-0.0625', '8', '7', '80', '75'],
+ ['short', '-0.4325', '-0.1351', '8', '5', '77', '67'],
+ ['short_pvalue',
+ '0.7671',
+ '0.1489',
+ 'U',
+ 'Test,',
+ 'Repetitions:',
+ '2',
+ 'vs',
+ '3.',
+ 'WARNING:',
+ 'Results',
+ 'unreliable!',
+ '9+',
+ 'repetitions',
+ 'recommended.'],
+ ['medium', '-0.3750', '-0.3375', '8', '5', '80', '53'],
+ ]
+ output_lines_with_header = print_difference_report(
+ self.json_diff_report, utest=True, utest_alpha=0.05, use_color=False)
+ output_lines = output_lines_with_header[2:]
+ print("\n")
+ print("\n".join(output_lines_with_header))
+ self.assertEqual(len(output_lines), len(expect_lines))
+ for i in range(0, len(output_lines)):
+ parts = [x for x in output_lines[i].split(' ') if x]
+ self.assertEqual(expect_lines[i], parts)
+
+ def test_json_diff_report_pretty_printing_aggregates_only(self):
+ expect_lines = [
+ ['BM_One', '-0.1000', '+0.1000', '10', '9', '100', '110'],
+ ['BM_Two_pvalue',
+ '0.6985',
+ '0.6985',
+ 'U',
+ 'Test,',
+ 'Repetitions:',
+ '2',
+ 'vs',
+ '2.',
+ 'WARNING:',
+ 'Results',
+ 'unreliable!',
+ '9+',
+ 'repetitions',
+ 'recommended.'],
+ ['short', '-0.1250', '-0.0625', '8', '7', '80', '75'],
+ ['short', '-0.4325', '-0.1351', '8', '5', '77', '67'],
+ ['short_pvalue',
+ '0.7671',
+ '0.1489',
+ 'U',
+ 'Test,',
+ 'Repetitions:',
+ '2',
+ 'vs',
+ '3.',
+ 'WARNING:',
+ 'Results',
+ 'unreliable!',
+ '9+',
+ 'repetitions',
+ 'recommended.'],
+ ]
+ output_lines_with_header = print_difference_report(
+ self.json_diff_report, include_aggregates_only=True, utest=True, utest_alpha=0.05, use_color=False)
+ output_lines = output_lines_with_header[2:]
+ print("\n")
+ print("\n".join(output_lines_with_header))
+ self.assertEqual(len(output_lines), len(expect_lines))
+ for i in range(0, len(output_lines)):
+ parts = [x for x in output_lines[i].split(' ') if x]
+ self.assertEqual(expect_lines[i], parts)
+
+ def test_json_diff_report(self):
+ expected_output = [
+ {
+ 'name': u'BM_One',
+ 'measurements': [
+ {'time': -0.1,
+ 'cpu': 0.1,
+ 'real_time': 10,
+ 'real_time_other': 9,
+ 'cpu_time': 100,
+ 'cpu_time_other': 110}
+ ],
+ 'time_unit': 'ns',
+ 'utest': {}
+ },
+ {
+ 'name': u'BM_Two',
+ 'measurements': [
+ {'time': 0.1111111111111111,
+ 'cpu': -0.011111111111111112,
+ 'real_time': 9,
+ 'real_time_other': 10,
+ 'cpu_time': 90,
+ 'cpu_time_other': 89},
+ {'time': -0.125, 'cpu': -0.16279069767441862, 'real_time': 8,
+ 'real_time_other': 7, 'cpu_time': 86, 'cpu_time_other': 72}
+ ],
+ 'time_unit': 'ns',
+ 'utest': {
+ 'have_optimal_repetitions': False, 'cpu_pvalue': 0.6985353583033387, 'time_pvalue': 0.6985353583033387
+ }
+ },
+ {
+ 'name': u'short',
+ 'measurements': [
+ {'time': -0.125,
+ 'cpu': -0.0625,
+ 'real_time': 8,
+ 'real_time_other': 7,
+ 'cpu_time': 80,
+ 'cpu_time_other': 75},
+ {'time': -0.4325,
+ 'cpu': -0.13506493506493514,
+ 'real_time': 8,
+ 'real_time_other': 4.54,
+ 'cpu_time': 77,
+ 'cpu_time_other': 66.6}
+ ],
+ 'time_unit': 'ns',
+ 'utest': {
+ 'have_optimal_repetitions': False, 'cpu_pvalue': 0.14891467317876572, 'time_pvalue': 0.7670968684102772
+ }
+ },
+ {
+ 'name': u'medium',
+ 'measurements': [
+ {'time': -0.375,
+ 'cpu': -0.3375,
+ 'real_time': 8,
+ 'real_time_other': 5,
+ 'cpu_time': 80,
+ 'cpu_time_other': 53}
+ ],
+ 'time_unit': 'ns',
+ 'utest': {}
+ }
+ ]
+ self.assertEqual(len(self.json_diff_report), len(expected_output))
+ for out, expected in zip(
+ self.json_diff_report, expected_output):
+ self.assertEqual(out['name'], expected['name'])
+ self.assertEqual(out['time_unit'], expected['time_unit'])
+ assert_utest(self, out, expected)
+ assert_measurements(self, out, expected)
+
+
+class TestReportDifferenceWithUTestWhileDisplayingAggregatesOnly(
+ unittest.TestCase):
+ @classmethod
+ def setUpClass(cls):
+ def load_results():
+ import json
+ testInputs = os.path.join(
+ os.path.dirname(
+ os.path.realpath(__file__)),
+ 'Inputs')
+ testOutput1 = os.path.join(testInputs, 'test3_run0.json')
+ testOutput2 = os.path.join(testInputs, 'test3_run1.json')
+ with open(testOutput1, 'r') as f:
+ json1 = json.load(f)
+ with open(testOutput2, 'r') as f:
+ json2 = json.load(f)
+ return json1, json2
+
+ json1, json2 = load_results()
+ cls.json_diff_report = get_difference_report(
+ json1, json2, utest=True)
+
+ def test_json_diff_report_pretty_printing(self):
+ expect_lines = [
+ ['BM_One', '-0.1000', '+0.1000', '10', '9', '100', '110'],
+ ['BM_Two', '+0.1111', '-0.0111', '9', '10', '90', '89'],
+ ['BM_Two', '-0.1250', '-0.1628', '8', '7', '86', '72'],
+ ['BM_Two_pvalue',
+ '0.6985',
+ '0.6985',
+ 'U',
+ 'Test,',
+ 'Repetitions:',
+ '2',
+ 'vs',
+ '2.',
+ 'WARNING:',
+ 'Results',
+ 'unreliable!',
+ '9+',
+ 'repetitions',
+ 'recommended.'],
+ ['short', '-0.1250', '-0.0625', '8', '7', '80', '75'],
+ ['short', '-0.4325', '-0.1351', '8', '5', '77', '67'],
+ ['short_pvalue',
+ '0.7671',
+ '0.1489',
+ 'U',
+ 'Test,',
+ 'Repetitions:',
+ '2',
+ 'vs',
+ '3.',
+ 'WARNING:',
+ 'Results',
+ 'unreliable!',
+ '9+',
+ 'repetitions',
+ 'recommended.'],
+ ['medium', '-0.3750', '-0.3375', '8', '5', '80', '53']
+ ]
+ output_lines_with_header = print_difference_report(
+ self.json_diff_report,
+ utest=True, utest_alpha=0.05, use_color=False)
+ output_lines = output_lines_with_header[2:]
+ print("\n")
+ print("\n".join(output_lines_with_header))
+ self.assertEqual(len(output_lines), len(expect_lines))
+ for i in range(0, len(output_lines)):
+ parts = [x for x in output_lines[i].split(' ') if x]
+ self.assertEqual(expect_lines[i], parts)
+
+ def test_json_diff_report(self):
+ expected_output = [
+ {
+ 'name': u'BM_One',
+ 'measurements': [
+ {'time': -0.1,
+ 'cpu': 0.1,
+ 'real_time': 10,
+ 'real_time_other': 9,
+ 'cpu_time': 100,
+ 'cpu_time_other': 110}
+ ],
+ 'time_unit': 'ns',
+ 'utest': {}
+ },
+ {
+ 'name': u'BM_Two',
+ 'measurements': [
+ {'time': 0.1111111111111111,
+ 'cpu': -0.011111111111111112,
+ 'real_time': 9,
+ 'real_time_other': 10,
+ 'cpu_time': 90,
+ 'cpu_time_other': 89},
+ {'time': -0.125, 'cpu': -0.16279069767441862, 'real_time': 8,
+ 'real_time_other': 7, 'cpu_time': 86, 'cpu_time_other': 72}
+ ],
+ 'time_unit': 'ns',
+ 'utest': {
+ 'have_optimal_repetitions': False, 'cpu_pvalue': 0.6985353583033387, 'time_pvalue': 0.6985353583033387
+ }
+ },
+ {
+ 'name': u'short',
+ 'measurements': [
+ {'time': -0.125,
+ 'cpu': -0.0625,
+ 'real_time': 8,
+ 'real_time_other': 7,
+ 'cpu_time': 80,
+ 'cpu_time_other': 75},
+ {'time': -0.4325,
+ 'cpu': -0.13506493506493514,
+ 'real_time': 8,
+ 'real_time_other': 4.54,
+ 'cpu_time': 77,
+ 'cpu_time_other': 66.6}
+ ],
+ 'time_unit': 'ns',
+ 'utest': {
+ 'have_optimal_repetitions': False, 'cpu_pvalue': 0.14891467317876572, 'time_pvalue': 0.7670968684102772
+ }
+ },
+ {
+ 'name': u'medium',
+ 'measurements': [
+ {'real_time_other': 5,
+ 'cpu_time': 80,
+ 'time': -0.375,
+ 'real_time': 8,
+ 'cpu_time_other': 53,
+ 'cpu': -0.3375
+ }
+ ],
+ 'utest': {},
+ 'time_unit': u'ns',
+ 'aggregate_name': ''
+ }
+ ]
+ self.assertEqual(len(self.json_diff_report), len(expected_output))
+ for out, expected in zip(
+ self.json_diff_report, expected_output):
+ self.assertEqual(out['name'], expected['name'])
+ self.assertEqual(out['time_unit'], expected['time_unit'])
+ assert_utest(self, out, expected)
+ assert_measurements(self, out, expected)
+
+
+class TestReportSorting(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls):
+ def load_result():
+ import json
+ testInputs = os.path.join(
+ os.path.dirname(
+ os.path.realpath(__file__)),
+ 'Inputs')
+ testOutput = os.path.join(testInputs, 'test4_run.json')
+ with open(testOutput, 'r') as f:
+ json = json.load(f)
+ return json
+
+ cls.json = load_result()
+
+ def test_json_diff_report_pretty_printing(self):
+ import util
+
+ expected_names = [
+ "99 family 0 instance 0 repetition 0",
+ "98 family 0 instance 0 repetition 1",
+ "97 family 0 instance 0 aggregate",
+ "96 family 0 instance 1 repetition 0",
+ "95 family 0 instance 1 repetition 1",
+ "94 family 0 instance 1 aggregate",
+ "93 family 1 instance 0 repetition 0",
+ "92 family 1 instance 0 repetition 1",
+ "91 family 1 instance 0 aggregate",
+ "90 family 1 instance 1 repetition 0",
+ "89 family 1 instance 1 repetition 1",
+ "88 family 1 instance 1 aggregate"
+ ]
+
+ for n in range(len(self.json['benchmarks']) ** 2):
+ random.shuffle(self.json['benchmarks'])
+ sorted_benchmarks = util.sort_benchmark_results(self.json)[
+ 'benchmarks']
+ self.assertEqual(len(expected_names), len(sorted_benchmarks))
+ for out, expected in zip(sorted_benchmarks, expected_names):
+ self.assertEqual(out['name'], expected)
+
+
+def assert_utest(unittest_instance, lhs, rhs):
+ if lhs['utest']:
+ unittest_instance.assertAlmostEqual(
+ lhs['utest']['cpu_pvalue'],
+ rhs['utest']['cpu_pvalue'])
+ unittest_instance.assertAlmostEqual(
+ lhs['utest']['time_pvalue'],
+ rhs['utest']['time_pvalue'])
+ unittest_instance.assertEqual(
+ lhs['utest']['have_optimal_repetitions'],
+ rhs['utest']['have_optimal_repetitions'])
+ else:
+ # lhs is empty. assert if rhs is not.
+ unittest_instance.assertEqual(lhs['utest'], rhs['utest'])
+
+
+def assert_measurements(unittest_instance, lhs, rhs):
+ for m1, m2 in zip(lhs['measurements'], rhs['measurements']):
+ unittest_instance.assertEqual(m1['real_time'], m2['real_time'])
+ unittest_instance.assertEqual(m1['cpu_time'], m2['cpu_time'])
+ # m1['time'] and m1['cpu'] hold values which are being calculated,
+ # and therefore we must use almost-equal pattern.
+ unittest_instance.assertAlmostEqual(m1['time'], m2['time'], places=4)
+ unittest_instance.assertAlmostEqual(m1['cpu'], m2['cpu'], places=4)
+
+
+if __name__ == '__main__':
+ unittest.main()
+
+# vim: tabstop=4 expandtab shiftwidth=4 softtabstop=4
+# kate: tab-width: 4; replace-tabs on; indent-width 4; tab-indents: off;
+# kate: indent-mode python; remove-trailing-spaces modified;
View Lorry Controller Status