diff options
author | Jonathan Abrahams <jonathan@mongodb.com> | 2018-07-27 10:52:11 -0400 |
---|---|---|
committer | Jonathan Abrahams <jonathan@mongodb.com> | 2018-08-13 12:23:29 -0400 |
commit | a444720202d97795b71cf2daaaa2bea94b430ef9 (patch) | |
tree | 6fcfe945019d2f3cf47b6b92969f270ffe771bf0 | |
parent | 78ca2bc9721cd3a3a0404e7e915ed6a3b5bc1bf6 (diff) | |
download | mongo-a444720202d97795b71cf2daaaa2bea94b430ef9.tar.gz |
SERVER-36076 Create new resmoke.py test suite for running mongoebench on a desktop
15 files changed, 651 insertions, 12 deletions
diff --git a/buildscripts/resmokeconfig/suites/benchrun_embedded_aggregation.yml b/buildscripts/resmokeconfig/suites/benchrun_embedded_aggregation.yml new file mode 100644 index 00000000000..65ed34dafcb --- /dev/null +++ b/buildscripts/resmokeconfig/suites/benchrun_embedded_aggregation.yml @@ -0,0 +1,9 @@ +test_kind: benchrun_embedded_test + +selector: + roots: + - benchrun_embedded/**/aggregation*.json + +executor: + hooks: + - class: CombineBenchrunEmbeddedResults diff --git a/buildscripts/resmokeconfig/suites/benchrun_embedded_commands.yml b/buildscripts/resmokeconfig/suites/benchrun_embedded_commands.yml new file mode 100644 index 00000000000..c1812defd37 --- /dev/null +++ b/buildscripts/resmokeconfig/suites/benchrun_embedded_commands.yml @@ -0,0 +1,9 @@ +test_kind: benchrun_embedded_test + +selector: + roots: + - benchrun_embedded/**/commands*.json + +executor: + hooks: + - class: CombineBenchrunEmbeddedResults diff --git a/buildscripts/resmokeconfig/suites/benchrun_embedded_insert.yml b/buildscripts/resmokeconfig/suites/benchrun_embedded_insert.yml new file mode 100644 index 00000000000..ec25d3147c3 --- /dev/null +++ b/buildscripts/resmokeconfig/suites/benchrun_embedded_insert.yml @@ -0,0 +1,9 @@ +test_kind: benchrun_embedded_test + +selector: + roots: + - benchrun_embedded/**/insert*.json + +executor: + hooks: + - class: CombineBenchrunEmbeddedResults diff --git a/buildscripts/resmokeconfig/suites/benchrun_embedded_misc.yml b/buildscripts/resmokeconfig/suites/benchrun_embedded_misc.yml new file mode 100644 index 00000000000..5e0877699fb --- /dev/null +++ b/buildscripts/resmokeconfig/suites/benchrun_embedded_misc.yml @@ -0,0 +1,18 @@ +test_kind: benchrun_embedded_test + +selector: + roots: + - benchrun_embedded/**/*.json + exclude_files: + - benchrun_embedded/**/aggregation*.json + - benchrun_embedded/**/commands*.json + - benchrun_embedded/**/insert*.json + - benchrun_embedded/**/mixed*.json + - benchrun_embedded/**/multi*.json + - benchrun_embedded/**/queries*.json + - benchrun_embedded/**/remove*.json + - benchrun_embedded/**/update*.json + +executor: + hooks: + - class: CombineBenchrunEmbeddedResults diff --git a/buildscripts/resmokeconfig/suites/benchrun_embedded_mixed_and_multi.yml b/buildscripts/resmokeconfig/suites/benchrun_embedded_mixed_and_multi.yml new file mode 100644 index 00000000000..d6909a10b53 --- /dev/null +++ b/buildscripts/resmokeconfig/suites/benchrun_embedded_mixed_and_multi.yml @@ -0,0 +1,10 @@ +test_kind: benchrun_embedded_test + +selector: + roots: + - benchrun_embedded/**/mixed*.json + - benchrun_embedded/**/multi*.json + +executor: + hooks: + - class: CombineBenchrunEmbeddedResults diff --git a/buildscripts/resmokeconfig/suites/benchrun_embedded_queries.yml b/buildscripts/resmokeconfig/suites/benchrun_embedded_queries.yml new file mode 100644 index 00000000000..5c4e1ed1fe6 --- /dev/null +++ b/buildscripts/resmokeconfig/suites/benchrun_embedded_queries.yml @@ -0,0 +1,9 @@ +test_kind: benchrun_embedded_test + +selector: + roots: + - benchrun_embedded/**/queries*.json + +executor: + hooks: + - class: CombineBenchrunEmbeddedResults diff --git a/buildscripts/resmokeconfig/suites/benchrun_embedded_remove.yml b/buildscripts/resmokeconfig/suites/benchrun_embedded_remove.yml new file mode 100644 index 00000000000..eecc24ff2b0 --- /dev/null +++ b/buildscripts/resmokeconfig/suites/benchrun_embedded_remove.yml @@ -0,0 +1,9 @@ +test_kind: benchrun_embedded_test + +selector: + roots: + - benchrun_embedded/**/remove*.json + +executor: + hooks: + - class: CombineBenchrunEmbeddedResults diff --git a/buildscripts/resmokeconfig/suites/benchrun_embedded_update.yml b/buildscripts/resmokeconfig/suites/benchrun_embedded_update.yml new file mode 100644 index 00000000000..3ea83564dd7 --- /dev/null +++ b/buildscripts/resmokeconfig/suites/benchrun_embedded_update.yml @@ -0,0 +1,9 @@ +test_kind: benchrun_embedded_test + +selector: + roots: + - benchrun_embedded/**/update*.json + +executor: + hooks: + - class: CombineBenchrunEmbeddedResults diff --git a/buildscripts/resmokelib/config.py b/buildscripts/resmokelib/config.py index 7206ff2b292..344fdd9c473 100644 --- a/buildscripts/resmokelib/config.py +++ b/buildscripts/resmokelib/config.py @@ -26,6 +26,7 @@ MONGO_RUNNER_SUBDIR = "mongorunner" # Default path for where to look for executables. DEFAULT_DBTEST_EXECUTABLE = os.path.join(os.curdir, "dbtest") DEFAULT_MONGO_EXECUTABLE = os.path.join(os.curdir, "mongo") +DEFAULT_MONGOEBENCH_EXECUTABLE = os.path.join(os.curdir, "mongoebench") DEFAULT_MONGOD_EXECUTABLE = os.path.join(os.curdir, "mongod") DEFAULT_MONGOS_EXECUTABLE = os.path.join(os.curdir, "mongos") @@ -53,6 +54,7 @@ DEFAULTS = { "mongo_executable": None, "mongod_executable": None, "mongod_set_parameters": None, + "mongoebench_executable": None, "mongos_executable": None, "mongos_set_parameters": None, "no_journal": False, @@ -270,6 +272,9 @@ MONGOD_EXECUTABLE = None # The --setParameter options passed to mongod. MONGOD_SET_PARAMETERS = None +# The path to the mongoebench executable used by resmoke.py. +MONGOEBENCH_EXECUTABLE = None + # The path to the mongos executable used by resmoke.py. MONGOS_EXECUTABLE = None @@ -372,4 +377,5 @@ DEFAULT_INTEGRATION_TEST_LIST = "build/integration_tests.txt" # External files or executables, used as suite selectors, that are created during the build and # therefore might not be available when creating a test membership map. EXTERNAL_SUITE_SELECTORS = (DEFAULT_BENCHMARK_TEST_LIST, DEFAULT_UNIT_TEST_LIST, - DEFAULT_INTEGRATION_TEST_LIST, DEFAULT_DBTEST_EXECUTABLE) + DEFAULT_INTEGRATION_TEST_LIST, DEFAULT_DBTEST_EXECUTABLE, + DEFAULT_MONGOEBENCH_EXECUTABLE) diff --git a/buildscripts/resmokelib/parser.py b/buildscripts/resmokelib/parser.py index bcdd0b917b4..a07a4bc464b 100644 --- a/buildscripts/resmokelib/parser.py +++ b/buildscripts/resmokelib/parser.py @@ -112,6 +112,10 @@ def _make_parser(): # pylint: disable=too-many-statements " started by resmoke.py. The argument is specified as bracketed YAML -" " i.e. JSON with support for single quoted and unquoted keys.")) + parser.add_option("--mongoebench", dest="mongoebench_executable", metavar="PATH", + help=("The path to the mongoebench (benchrun embedded) executable for" + " resmoke.py to use.")) + parser.add_option("--mongos", dest="mongos_executable", metavar="PATH", help="The path to the mongos executable for resmoke.py to use.") @@ -256,31 +260,33 @@ def _make_parser(): # pylint: disable=too-many-statements evergreen_options.add_option("--versionId", dest="version_id", metavar="VERSION_ID", help="Sets the version ID of the task.") - benchmark_options = optparse.OptionGroup(parser, title="Benchmark test options", - description="Options for running Benchmark tests") + benchmark_options = optparse.OptionGroup( + parser, title="Benchmark/Benchrun test options", + description="Options for running Benchmark/Benchrun tests") parser.add_option_group(benchmark_options) benchmark_options.add_option("--benchmarkFilter", type="string", dest="benchmark_filter", metavar="BENCHMARK_FILTER", - help="Regex to filter benchmark tests to run.") + help="Regex to filter Google benchmark tests to run.") benchmark_options.add_option("--benchmarkListTests", dest="benchmark_list_tests", action="store_true", metavar="BENCHMARK_LIST_TESTS", - help="Lists all benchmark test configurations in each test file.") + help=("Lists all Google benchmark test configurations in each" + " test file.")) benchmark_min_time_help = ( - "Minimum time to run each benchmark test for. Use this option instead of " + "Minimum time to run each benchmark/benchrun test for. Use this option instead of " "--benchmarkRepetitions to make a test run for a longer or shorter duration.") benchmark_options.add_option("--benchmarkMinTimeSecs", type="int", dest="benchmark_min_time_secs", metavar="BENCHMARK_MIN_TIME", help=benchmark_min_time_help) benchmark_repetitions_help = ( - "Set --benchmarkRepetitions=1 if you'd like to run the benchmark tests only once. By " - "default, each test is run multiple times to provide statistics on the variance between " - "runs; use --benchmarkMinTimeSecs if you'd like to run a test for a longer or shorter " - "duration.") + "Set --benchmarkRepetitions=1 if you'd like to run the benchmark/benchrun tests only once." + " By default, each test is run multiple times to provide statistics on the variance" + " between runs; use --benchmarkMinTimeSecs if you'd like to run a test for a longer or" + " shorter duration.") benchmark_options.add_option("--benchmarkRepetitions", type="int", dest="benchmark_repetitions", metavar="BENCHMARK_REPETITIONS", help=benchmark_repetitions_help) @@ -361,6 +367,7 @@ def _update_config_vars(values): # pylint: disable=too-many-statements _config.MONGO_EXECUTABLE = _expand_user(config.pop("mongo_executable")) _config.MONGOD_EXECUTABLE = _expand_user(config.pop("mongod_executable")) _config.MONGOD_SET_PARAMETERS = config.pop("mongod_set_parameters") + _config.MONGOEBENCH_EXECUTABLE = _expand_user(config.pop("mongoebench_executable")) _config.MONGOS_EXECUTABLE = _expand_user(config.pop("mongos_executable")) _config.MONGOS_SET_PARAMETERS = config.pop("mongos_set_parameters") _config.NO_JOURNAL = config.pop("no_journal") diff --git a/buildscripts/resmokelib/selector.py b/buildscripts/resmokelib/selector.py index 07653a38624..dd3649b74fc 100644 --- a/buildscripts/resmokelib/selector.py +++ b/buildscripts/resmokelib/selector.py @@ -676,6 +676,7 @@ _SELECTOR_REGISTRY = { "cpp_integration_test": (_CppTestSelectorConfig, _CppTestSelector), "cpp_unit_test": (_CppTestSelectorConfig, _CppTestSelector), "benchmark_test": (_CppTestSelectorConfig, _CppTestSelector), + "benchrun_embedded_test": (_JsonTestSelectorConfig, _Selector), "db_test": (_DbTestSelectorConfig, _DbTestSelector), "fsm_workload_test": (_JSTestSelectorConfig, _JSTestSelector), "parallel_fsm_workload_test": (_MultiJSTestSelectorConfig, _MultiJSTestSelector), @@ -693,8 +694,7 @@ def filter_tests(test_kind, selector_config, test_file_explorer=_DEFAULT_TEST_FI """Filter the tests according to a specified configuration. Args: - test_kind: the test kind, one of 'cpp_integration_test', 'cpp_unit_test', 'db_test', - 'json_schema_test', 'js_test'. + test_kind: the test kind, from _SELECTOR_REGISTRY. selector_config: a dict containing the selector configuration. test_file_explorer: the TestFileExplorer to use. Using a TestFileExplorer other than the default one should not be needed except for mocking purposes. diff --git a/buildscripts/resmokelib/testing/hooks/combine_benchrun_embedded_results.py b/buildscripts/resmokelib/testing/hooks/combine_benchrun_embedded_results.py new file mode 100644 index 00000000000..bf7af69244b --- /dev/null +++ b/buildscripts/resmokelib/testing/hooks/combine_benchrun_embedded_results.py @@ -0,0 +1,150 @@ +"""Module for generating the test results file fed into the perf plugin.""" + +from __future__ import absolute_import +from __future__ import division + +import collections +import datetime +import glob +import json +import os + +from buildscripts.resmokelib import config as _config +from buildscripts.resmokelib.testing.hooks import combine_benchmark_results as cbr + + +class CombineBenchrunEmbeddedResults(cbr.CombineBenchmarkResults): + """CombineBenchrunEmbeddedResults class. + + The CombineBenchrunEmbeddedResults hook combines test results from + individual benchmark embedded files to a single file. This is useful for + generating the json file to feed into the Evergreen performance + visualization plugin. + """ + + DESCRIPTION = "Combine JSON results from embedded benchrun" + + def before_test(self, test, test_report): + """Remove any existing mongoebench reports for this test.""" + for bm_report in self._test_result_files(test): + os.remove(bm_report) + + def after_test(self, test, test_report): + """Update test report.""" + for bm_report in self._test_result_files(test): + test_name, thread_count = self._parse_report_name(bm_report) + with open(bm_report, "r") as report_file: + report_dict = json.load(report_file) + if test_name not in self.benchmark_reports: + self.benchmark_reports[test_name] = _BenchrunEmbeddedThreadsReport() + self.benchmark_reports[test_name].add_report(thread_count, report_dict) + + def before_suite(self, test_report): + """Set suite start time.""" + self.create_time = datetime.datetime.now() + # Remove any existing perf reports. + if self.report_file and os.path.isfile(self.report_file): + os.remove(self.report_file) + + def _generate_perf_plugin_report(self): + """Format the data to look like a perf plugin report.""" + perf_report = { + "start": self._strftime(self.create_time), + "end": self._strftime(self.end_time), + "errors": [], # There are no errors if we have gotten this far. + "results": [] + } + + for name, report in self.benchmark_reports.items(): + test_report = {"name": name, "results": report.generate_perf_plugin_dict()} + + perf_report["results"].append(test_report) + + return perf_report + + @staticmethod + def _test_result_files(test): + """Return a list of existing test result files based on the test.short_name().""" + return glob.glob("mongoebench[.]{}[.]*[.]json".format(test.short_name())) + + @staticmethod + def _parse_report_name(report_path): + """Parse mongoebench report path and return test_name and thread_count. + + The format of the mongoebench report file name is defined in + ../testing/testcases/benchrun_embedded_test.py + as mongoebench.<test_name>.<num threads>.<iteration num>.json + """ + report_base = os.path.basename(report_path) + _, test_name, thread_count, _, _ = report_base.split(".") + return test_name, thread_count + + +class _BenchrunEmbeddedThreadsReport(object): + """_BenchrunEmbeddedThreadsReport class. + + Class representation of a report for all thread levels of a single + benchmark test. Each report is designed to correspond to one graph + in the Evergreen perf plugin. + + A raw mongoebench report looks like the following: + { + "note" : "values per second", + "errCount" : { "$numberLong" : "0" }, + "trapped" : "error: not implemented", + "insertLatencyAverageMicros" : 389.4926654182272, + "totalOps" : { "$numberLong" : "12816" }, + "totalOps/s" : 2563.095938304905, + "findOne" : 0, + "insert" : 2563.095938304905, + "delete" : 0, + "update" : 0, + "query" : 0, + "command" : 0, + "findOnes" : { "$numberLong" : "0" }, + "inserts" : { "$numberLong" : "12816" }, + "deletes" : { "$numberLong" : "0" }, + "updates" : { "$numberLong" : "0" }, + "queries" : { "$numberLong" : "0" }, + "commands" : { "$numberLong" : "0" } + } + """ + + def __init__(self): + # list of benchmark runs for each thread. + self.thread_benchmark_map = collections.defaultdict(list) + + def add_report(self, thread_count, report): + """Add to report.""" + self.thread_benchmark_map[str(thread_count)].append(report) + + def generate_perf_plugin_dict(self): + """Generate perf plugin data points of the following format. + + "1": { + "error_values": [ + 0, + 0, + 0 + ], + "ops_per_sec": 9552.108279243452, + "ops_per_sec_values": [ + 9574.812658450564, + 9522.642340821469, + 9536.252775275878 + ] + }, + """ + + res = {} + for thread_count, reports in self.thread_benchmark_map.items(): + thread_report = {"error_values": [], "ops_per_sec_values": []} + + for report in reports: + thread_report["error_values"].append(report["errCount"]["$numberLong"]) + thread_report["ops_per_sec_values"].append(report["totalOps/s"]) + thread_report["ops_per_sec"] = sum(thread_report["ops_per_sec_values"]) / len(reports) + + res[thread_count] = thread_report + + return res diff --git a/buildscripts/resmokelib/testing/testcases/benchrun_embedded_test.py b/buildscripts/resmokelib/testing/testcases/benchrun_embedded_test.py new file mode 100644 index 00000000000..5e9e0a09c08 --- /dev/null +++ b/buildscripts/resmokelib/testing/testcases/benchrun_embedded_test.py @@ -0,0 +1,96 @@ +"""The unittest.TestCase for tests using benchrun embedded (mongoebench).""" + +from __future__ import absolute_import + +import os + +from buildscripts.resmokelib import config as _config +from buildscripts.resmokelib import core +from buildscripts.resmokelib import parser +from buildscripts.resmokelib import utils +from buildscripts.resmokelib.testing.testcases import interface + + +class BenchrunEmbeddedTestCase(interface.ProcessTestCase): + """A Benchrun embedded test to execute.""" + + REGISTERED_NAME = "benchrun_embedded_test" + + def __init__(self, logger, mongoebench_config_file, program_options=None): + """Initialize the BenchrunEmbeddedTestCase with the executable to run.""" + + interface.ProcessTestCase.__init__(self, logger, "Benchmark embedded test", + mongoebench_config_file) + parser.validate_benchmark_options() + + self.benchrun_config_file = mongoebench_config_file + + # Command line options override the YAML configuration. + self.benchrun_executable = utils.default_if_none(_config.MONGOEBENCH_EXECUTABLE, + _config.DEFAULT_MONGOEBENCH_EXECUTABLE) + self.benchrun_repetitions = utils.default_if_none(_config.BENCHMARK_REPETITIONS, + _config.DEFAULT_BENCHMARK_REPETITIONS) + self.suite_benchrun_options = program_options + self.benchrun_threads = 1 + if program_options and "threads" in program_options: + self.benchrun_threads = program_options["threads"] + self.benchrun_options = {} + + # Set the dbpath. + dbpath = utils.default_if_none(_config.DBPATH_PREFIX, _config.DEFAULT_DBPATH_PREFIX) + self.dbpath = os.path.join(dbpath, "mongoebench") + + def configure(self, fixture, *args, **kwargs): + """Configure BenchrunEmbeddedTestCase.""" + interface.ProcessTestCase.configure(self, fixture, *args, **kwargs) + + # 1. Set the default benchmark options. + benchrun_options = {"time": _config.DEFAULT_BENCHMARK_MIN_TIME.total_seconds()} + + # 2. Override Benchmark options with options set through `program_options` in the suite + # configuration. + suite_benchrun_options = utils.default_if_none(self.suite_benchrun_options, {}) + benchrun_options.update(suite_benchrun_options) + + # 3. Override Benchmark options with options set through resmoke's command line. + resmoke_benchrun_options = {"dbpath": self.dbpath, "time": _config.BENCHMARK_MIN_TIME} + + for key, value in resmoke_benchrun_options.items(): + if value is not None: + # 4. sanitize options before passing them to Benchmark's command line. + if key == "time": + value = value.total_seconds() + benchrun_options[key] = value + + self.benchrun_options = benchrun_options + + # Create the dbpath. + self._clear_dbpath() + try: + os.makedirs(self.dbpath) + except os.error: + # Directory already exists. + pass + + def run_test(self): + """Run the test for specified number of iterations.""" + for it_num in xrange(self.benchrun_repetitions): + # Set the output file for each iteration. + self.benchrun_options["output"] = self._report_name(it_num) + interface.ProcessTestCase.run_test(self) + + def _clear_dbpath(self): + utils.rmtree(self.dbpath, ignore_errors=True) + + def _report_name(self, iter_num): + """Return the constructed report name. + + The report name is of the form mongoebench.<test_name>.<num threads>.<iteration num>.json. + """ + return "mongoebench.{}.{}.{}.json".format(self.short_name(), self.benchrun_threads, + iter_num) + + def _make_process(self): + return core.programs.generic_program(self.logger, + [self.benchrun_executable, self.benchrun_config_file], + **self.benchrun_options) diff --git a/buildscripts/tests/resmokelib/testing/hooks/test_combine_benchrun_embedded_results.py b/buildscripts/tests/resmokelib/testing/hooks/test_combine_benchrun_embedded_results.py new file mode 100755 index 00000000000..06744381706 --- /dev/null +++ b/buildscripts/tests/resmokelib/testing/hooks/test_combine_benchrun_embedded_results.py @@ -0,0 +1,163 @@ +#!/usr/bin/env python +"""Unit tests for the resmokelib.testing.hooks.combine_benchrun_embedded_results module.""" + +from __future__ import absolute_import + +import datetime +import unittest + +import mock + +import buildscripts.resmokelib.testing.hooks.combine_benchrun_embedded_results as cber + +# pylint: disable=missing-docstring,protected-access,attribute-defined-outside-init + +_BM_REPORT_INSERT_1 = { + "note": "values per second", "errCount": {"$numberLong": "0"}, + "trapped": "error: not implemented", "insertLatencyAverageMicros": 389.4926654182272, + "totalOps": {"$numberLong": "12816"}, "totalOps/s": 2563.095938304905, "findOne": 0, + "insert": 2563.095938304905, "delete": 0, "update": 0, "query": 0, "command": 0, "findOnes": { + "$numberLong": "0" + }, "inserts": {"$numberLong": "12816"}, "deletes": {"$numberLong": "0"}, "updates": { + "$numberLong": "0" + }, "queries": {"$numberLong": "0"}, "commands": {"$numberLong": "0"} +} + +_BM_REPORT_INSERT_2 = { + "note": "values per second", "errCount": {"$numberLong": "0"}, + "trapped": "error: not implemented", "insertLatencyAverageMicros": 2563.095938304905, + "totalOps": {"$numberLong": "7404"}, "totalOps/s": 2409.05, "findOne": 0, "insert": 2409.05, + "delete": 0, "update": 0, "query": 0, "command": 0, "findOnes": {"$numberLong": "0"}, + "inserts": {"$numberLong": "7404"}, "deletes": {"$numberLong": "0"}, "updates": { + "$numberLong": "0" + }, "queries": {"$numberLong": "0"}, "commands": {"$numberLong": "0"} +} + +_BM_REPORT_DELETE = { + "note": "values per second", "errCount": {"$numberLong": "0"}, + "trapped": "error: not implemented", "insertLatencyAverageMicros": "1234.56", "totalOps": { + "$numberLong": "2345" + }, "totalOps/s": 1234.56, "findOne": 0, "insert": 0, "delete": 1234.56, "update": 0, "query": 0, + "command": 0, "findOnes": {"$numberLong": "0"}, "inserts": {"$numberLong": "0"}, "deletes": { + "$numberLong": "2345" + }, "updates": {"$numberLong": "0"}, "queries": {"$numberLong": "0"}, + "commands": {"$numberLong": "0"} +} + +_BM_REPORT_UPDATE = { + "note": "values per second", "errCount": {"$numberLong": "0"}, + "trapped": "error: not implemented", "insertLatencyAverageMicros": 654.321, "totalOps": { + "$numberLong": "4521" + }, "totalOps/s": 4521.00, "findOne": 0, "insert": 0, "delete": 0, "update": 4521.00, "query": 0, + "command": 0, "findOnes": {"$numberLong": "0"}, "inserts": {"$numberLong": "0"}, "deletes": { + "$numberLong": "0" + }, "updates": {"$numberLong": "4521"}, "queries": {"$numberLong": "0"}, + "commands": {"$numberLong": "0"} +} + +_BM_REPORT_MULTI = { + "note": "values per second", "errCount": {"$numberLong": "0"}, + "trapped": "error: not implemented", "insertLatencyAverageMicros": 111.111, "totalOps": { + "$numberLong": "11532" + }, "totalOps/s": 5766.00, "findOne": 0, "insert": 2490.00, "delete": 0, "update": 9042.00, + "query": 0, "command": 0, "findOnes": {"$numberLong": "0"}, "inserts": { + "$numberLong": "2490.00" + }, "deletes": {"$numberLong": "0"}, "updates": {"$numberLong": "9042"}, + "queries": {"$numberLong": "0"}, "commands": {"$numberLong": "0"} +} + +_BM_ALL_REPORTS = [ + _BM_REPORT_INSERT_1, _BM_REPORT_INSERT_2, _BM_REPORT_DELETE, _BM_REPORT_UPDATE, _BM_REPORT_MULTI +] + +# 12/31/2999 @ 11:59pm (UTC) +_START_TIME = 32503679999 + +# 01/01/3000 @ 12:00am (UTC) +_END_TIME = 32503680000 + + +class CombineBenchrunEmbeddedResultsFixture(unittest.TestCase): + + # Mock the hook's parent class because we're testing only functionality of this hook and + # not anything related to or inherit from the parent class. + @mock.patch("buildscripts.resmokelib.testing.hooks.interface.Hook", autospec=True) + def setUp(self, MockHook): # pylint: disable=arguments-differ,unused-argument + self.cber_hook = cber.CombineBenchrunEmbeddedResults(None, None) + self.cber_hook.create_time = datetime.datetime.utcfromtimestamp(_START_TIME) + self.cber_hook.end_time = datetime.datetime.utcfromtimestamp(_END_TIME) + + +class TestCombineBenchmarkResults(CombineBenchrunEmbeddedResultsFixture): + def _setup_reports(self, reports, test_name, num_threads): + self.total_ops_per_sec = 0 + self.num_tests = len(reports) + self.cber_hook.benchmark_reports[test_name] = cber._BenchrunEmbeddedThreadsReport() + for rep in reports: + self.cber_hook.benchmark_reports[test_name].add_report(num_threads, rep) + self.total_ops_per_sec += rep["totalOps/s"] + self.ops_per_sec = self.total_ops_per_sec / self.num_tests + self.report = self.cber_hook._generate_perf_plugin_report() + + def test_generate_one_report(self): + test_name = "test_cber1" + num_threads = "2" + self._setup_reports([_BM_REPORT_MULTI], test_name, num_threads) + report_0 = self.report["results"][0] + self.assertEqual(report_0["name"], test_name) + self.assertEqual(report_0["results"][str(num_threads)]["ops_per_sec"], self.ops_per_sec) + + def test_generate_all_reports(self): + test_name = "test_cber2" + thread_num = "1" + self._setup_reports(_BM_ALL_REPORTS, test_name, thread_num) + self.assertEqual(len(self.report.keys()), 4) + report_0 = self.report["results"][0] + self.assertEqual(report_0["name"], test_name) + self.assertEqual(report_0["results"][thread_num]["ops_per_sec"], self.ops_per_sec) + self.assertEqual(self.report["start"], "2999-12-31T23:59:59Z") + self.assertEqual(self.report["end"], "3000-01-01T00:00:00Z") + + def test_parse_report_name(self): + test_name = "test1" + thread_num = "4" + file_name = "mongoebench.{}.{}.iter0.json".format(test_name, thread_num) + report_name, report_threads = self.cber_hook._parse_report_name(file_name) + self.assertEqual(report_name, test_name) + self.assertEqual(report_threads, thread_num) + + +class TestBenchrunEmbeddedThreadsReport(CombineBenchrunEmbeddedResultsFixture): + def test_generate_single_thread_perf_plugin_dict(self): + thread_report = cber._BenchrunEmbeddedThreadsReport() + thread_num = "1" + thread_report.add_report(thread_num, _BM_REPORT_INSERT_1) + perf_report = thread_report.generate_perf_plugin_dict() + self.assertEqual(len(perf_report.keys()), 1) + self.assertEqual(perf_report[thread_num]["ops_per_sec"], _BM_REPORT_INSERT_1["totalOps/s"]) + self.assertEqual(len(perf_report[thread_num]["ops_per_sec_values"]), 1) + + thread_report.add_report(thread_num, _BM_REPORT_INSERT_2) + perf_report = thread_report.generate_perf_plugin_dict() + self.assertEqual(len(perf_report.keys()), 1) + ops_per_sec = (_BM_REPORT_INSERT_1["totalOps/s"] + _BM_REPORT_INSERT_2["totalOps/s"]) / 2 + self.assertEqual(perf_report[thread_num]["ops_per_sec"], ops_per_sec) + self.assertEqual(len(perf_report[thread_num]["ops_per_sec_values"]), 2) + + def test_generate_multi_thread_perf_plugin_dict(self): + thread_report = cber._BenchrunEmbeddedThreadsReport() + thread_num = "1" + thread_report.add_report(thread_num, _BM_REPORT_INSERT_1) + perf_report = thread_report.generate_perf_plugin_dict() + self.assertEqual(len(perf_report.keys()), 1) + self.assertEqual(perf_report[thread_num]["ops_per_sec"], _BM_REPORT_INSERT_1["totalOps/s"]) + self.assertEqual(len(perf_report[thread_num]["ops_per_sec_values"]), 1) + + thread_num = "2" + thread_report.add_report(thread_num, _BM_REPORT_INSERT_2) + perf_report = thread_report.generate_perf_plugin_dict() + self.assertEqual(len(perf_report.keys()), 2) + self.assertEqual(perf_report["1"]["ops_per_sec"], _BM_REPORT_INSERT_1["totalOps/s"]) + self.assertEqual(len(perf_report["1"]["ops_per_sec_values"]), 1) + self.assertEqual(perf_report[thread_num]["ops_per_sec"], _BM_REPORT_INSERT_2["totalOps/s"]) + self.assertEqual(len(perf_report[thread_num]["ops_per_sec_values"]), 1) diff --git a/etc/evergreen.yml b/etc/evergreen.yml index b7e022690b4..f0a4c4859fb 100644 --- a/etc/evergreen.yml +++ b/etc/evergreen.yml @@ -175,6 +175,18 @@ variables: mongod_options: --mongodUsablePorts ${standard_port} ${secret_port} --dbPath=${db_path} --logPath=${log_path} mongod_extra_options: --mongodOptions=\"--setParameter enableTestCommands=1 --setParameter logComponentVerbosity='{storage:{recovery:2}}'\" +- &benchrun_embedded + name: benchrun_embedded + execution_tasks: + - benchrun_embedded_aggregation + - benchrun_embedded_commands + - benchrun_embedded_insert + - benchrun_embedded_misc + - benchrun_embedded_mixed_and_multi + - benchrun_embedded_queries + - benchrun_embedded_remove + - benchrun_embedded_update + - &replica_sets_auth name: replica_sets_auth execution_tasks: @@ -774,6 +786,15 @@ functions: bucket: mciuploads extract_to: src + "fetch benchmark embedded files" : &fetch_benchrun_embedded_files + command: s3.get + params: + aws_key: ${aws_key} + aws_secret: ${aws_secret} + remote_file: ${project}/benchrun_embedded/benchrun_json_files.tgz + bucket: mciuploads + extract_to: src/benchrun_embedded + "get buildnumber" : &get_buildnumber command: keyval.inc params: @@ -1220,6 +1241,13 @@ functions: - *set_up_credentials - *fetch_benchmarks + "do benchmark embedded setup" : + - *git_get_project + - *fetch_artifacts + - *get_buildnumber + - *set_up_credentials + - *fetch_benchrun_embedded_files + "set up virtualenv" : command: shell.exec type: test @@ -3201,6 +3229,7 @@ test_lifecycle_excluded_tasks: - aggregation_fuzzer* - burn_in_tests - compile* +- benchmarks* - dbtest* - idl_tests - integration* @@ -3864,6 +3893,86 @@ tasks: run_multiple_jobs: false - func: "send benchmark results" +- <<: *task_template + name: benchrun_embedded_aggregation + commands: + - func: "do benchmark embedded setup" + - func: "run tests" + vars: + resmoke_args: --suites=benchrun_embedded_aggregation + run_multiple_jobs: false + - func: "send benchmark results" + +- <<: *task_template + name: benchrun_embedded_commands + commands: + - func: "do benchmark embedded setup" + - func: "run tests" + vars: + resmoke_args: --suites=benchrun_embedded_commands + run_multiple_jobs: false + - func: "send benchmark results" + +- <<: *task_template + name: benchrun_embedded_insert + commands: + - func: "do benchmark embedded setup" + - func: "run tests" + vars: + resmoke_args: --suites=benchrun_embedded_insert + run_multiple_jobs: false + - func: "send benchmark results" + +- <<: *task_template + name: benchrun_embedded_misc + commands: + - func: "do benchmark embedded setup" + - func: "run tests" + vars: + resmoke_args: --suites=benchrun_embedded_misc + run_multiple_jobs: false + - func: "send benchmark results" + +- <<: *task_template + name: benchrun_embedded_mixed_and_multi + commands: + - func: "do benchmark embedded setup" + - func: "run tests" + vars: + resmoke_args: --suites=benchrun_embedded_mixed_and_multi + run_multiple_jobs: false + - func: "send benchmark results" + +- <<: *task_template + name: benchrun_embedded_queries + commands: + - func: "do benchmark embedded setup" + - func: "run tests" + vars: + resmoke_args: --suites=benchrun_embedded_queries + run_multiple_jobs: false + - func: "send benchmark results" + +- <<: *task_template + name: benchrun_embedded_remove + commands: + - func: "do benchmark embedded setup" + - func: "run tests" + vars: + resmoke_args: --suites=benchrun_embedded_remove + run_multiple_jobs: false + - func: "send benchmark results" + +- <<: *task_template + name: benchrun_embedded_update + commands: + - func: "do benchmark embedded setup" + - func: "run tests" + vars: + resmoke_args: --suites=benchrun_embedded_update + run_multiple_jobs: false + - func: "send benchmark results" + - <<: *run_jepsen_template name: jepsen_register_findAndModify commands: @@ -12532,6 +12641,7 @@ buildvariants: build_mongoreplay: true additional_targets: mongoebench mongoed display_tasks: + - *benchrun_embedded - *dbtest - *unittests tasks: @@ -12543,6 +12653,31 @@ buildvariants: - name: aggregation_facet_unwind_passthrough - name: auth - name: concurrency + - name: benchrun_embedded_aggregation + distros: + - centos6-perf + - name: benchrun_embedded_commands + distros: + - centos6-perf + - name: benchrun_embedded_insert + distros: + - centos6-perf + - name: benchrun_embedded_misc + distros: + - centos6-perf + - name: benchrun_embedded_mixed_and_multi + distros: + - centos6-perf + - name: benchrun_embedded_queries + distros: + - centos6-perf + - name: benchrun_embedded_remove + distros: + - centos6-perf + - name: benchrun_embedded_update + distros: + - centos6-perf + - name: dbtest - name: disk_mobile - name: failpoints - name: failpoints_auth |