1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
|
"""Module for generating the test results file fed into the perf plugin."""
from __future__ import absolute_import
from __future__ import division
import collections
import datetime
import glob
import json
import os
import re
from buildscripts.resmokelib import config as _config
from buildscripts.resmokelib.testing.hooks import combine_benchmark_results as cbr
class CombineBenchrunEmbeddedResults(cbr.CombineBenchmarkResults):
"""CombineBenchrunEmbeddedResults class.
The CombineBenchrunEmbeddedResults hook combines test results from
individual benchmark embedded files to a single file. This is useful for
generating the json file to feed into the Evergreen performance
visualization plugin.
"""
DESCRIPTION = "Combine JSON results from embedded benchrun"
def __init__(self, hook_logger, fixture):
"""Initialize CombineBenchrunEmbeddedResults."""
cbr.CombineBenchmarkResults.__init__(self, hook_logger, fixture)
self.report_root = _config.BENCHRUN_REPORT_ROOT
def before_test(self, test, test_report):
"""Remove any existing mongoebench reports for this test."""
for bm_report in self._test_result_files(test):
os.remove(bm_report)
def after_test(self, test, test_report):
"""Update test report."""
for bm_report in self._test_result_files(test):
test_name = test.short_name()
thread_count = self._parse_report_name(bm_report)
with open(bm_report, "r") as report_file:
report_dict = json.load(report_file)
if test_name not in self.benchmark_reports:
self.benchmark_reports[test_name] = _BenchrunEmbeddedThreadsReport()
self.benchmark_reports[test_name].add_report(thread_count, report_dict)
def before_suite(self, test_report):
"""Set suite start time."""
self.create_time = datetime.datetime.now()
# Remove any existing perf reports.
if self.report_file and os.path.isfile(self.report_file):
os.remove(self.report_file)
def _generate_perf_plugin_report(self):
"""Format the data to look like a perf plugin report."""
perf_report = {
"start": self._strftime(self.create_time),
"end": self._strftime(self.end_time),
"errors": [], # There are no errors if we have gotten this far.
"results": []
}
for name, report in self.benchmark_reports.items():
test_report = {"name": name, "results": report.generate_perf_plugin_dict()}
perf_report["results"].append(test_report)
return perf_report
def _test_result_files(self, test):
"""Return a list of existing test result files based on the test.short_name()."""
return glob.glob(
os.path.join(self.report_root, test.short_name(), "**", "mongoebench[.]*[.]json"))
def _parse_report_name(self, report_path):
"""Parse mongoebench report path and return thread_count.
The format of the mongoebench report file name is defined in
../testing/testcases/benchrun_embedded_test.py
as self.report_root/<test_name>/thread<num threads>/mongoebench.<iteration num>.json
"""
_, report_subpath = report_path.split(self.report_root + os.sep)
_, thread_name, _ = report_subpath.split(os.sep)
return re.findall(r"\d+", thread_name)[0]
class _BenchrunEmbeddedThreadsReport(object):
"""_BenchrunEmbeddedThreadsReport class.
Class representation of a report for all thread levels of a single
benchmark test. Each report is designed to correspond to one graph
in the Evergreen perf plugin.
A raw mongoebench report looks like the following:
{
"note" : "values per second",
"errCount" : { "$numberLong" : "0" },
"trapped" : "error: not implemented",
"insertLatencyAverageMicros" : 389.4926654182272,
"totalOps" : { "$numberLong" : "12816" },
"totalOps/s" : 2563.095938304905,
"findOne" : 0,
"insert" : 2563.095938304905,
"delete" : 0,
"update" : 0,
"query" : 0,
"command" : 0,
"findOnes" : { "$numberLong" : "0" },
"inserts" : { "$numberLong" : "12816" },
"deletes" : { "$numberLong" : "0" },
"updates" : { "$numberLong" : "0" },
"queries" : { "$numberLong" : "0" },
"commands" : { "$numberLong" : "0" }
}
"""
def __init__(self):
# list of benchmark runs for each thread.
self.thread_benchmark_map = collections.defaultdict(list)
def add_report(self, thread_count, report):
"""Add to report."""
self.thread_benchmark_map[thread_count].append(report)
def generate_perf_plugin_dict(self):
"""Generate perf plugin data points of the following format.
"1": {
"error_values": [
0,
0,
0
],
"ops_per_sec": 9552.108279243452,
"ops_per_sec_values": [
9574.812658450564,
9522.642340821469,
9536.252775275878
]
},
"""
res = {}
for thread_count, reports in self.thread_benchmark_map.items():
thread_report = {"error_values": [], "ops_per_sec_values": []}
for report in reports:
thread_report["error_values"].append(report["errCount"]["$numberLong"])
thread_report["ops_per_sec_values"].append(report["totalOps/s"])
thread_report["ops_per_sec"] = sum(thread_report["ops_per_sec_values"]) / len(reports)
res[thread_count] = thread_report
return res
|
