SERVER-33746 pipe additional evergreen options into resmoke.py

1 files changed, 58 insertions, 25 deletions

diff --git a/buildscripts/tests/resmokelib/testing/hooks/test_combine_benchmark_results.py b/buildscripts/tests/resmokelib/testing/hooks/test_combine_benchmark_results.py
index 1b07980f1fb..a24c9d900c2 100755
--- a/buildscripts/tests/resmokelib/testing/hooks/test_combine_benchmark_results.py
+++ b/buildscripts/tests/resmokelib/testing/hooks/test_combine_benchmark_results.py
@@ -19,7 +19,27 @@ _BM_CONTEXT = {
 }
 
 _BM_REPORT = {
-    "name": "BM_Name/threads:10",
+    "name": "BM_Name1",
+    "iterations": 1000,
+    "real_time": 1202,
+    "cpu_time": 1303,
+    "bytes_per_second": 1404,
+    "items_per_second": 1505,
+    "custom_counter_1": 1606
+}
+
+_BM_MEAN_REPORT = {
+    "name": "BM_Name1_mean",
+    "iterations": 1000,
+    "real_time": 1200,
+    "cpu_time": 1300,
+    "bytes_per_second": 1400,
+    "items_per_second": 1500,
+    "custom_counter_1": 1600
+}
+
+_BM_MULTITHREAD_REPORT = {
+    "name": "BM_Name2/threads:10",
     "iterations": 100,
     "real_time": 202,
     "cpu_time": 303,
@@ -28,8 +48,8 @@ _BM_REPORT = {
     "custom_counter_1": 606
 }
 
-_BM_MEDIAN_REPORT = {
-    "name": "BM_Name/threads:10_median",
+_BM_MULTITHREAD_MEDIAN_REPORT = {
+    "name": "BM_Name2/threads:10_median",
     "iterations": 100,
     "real_time": 200,
     "cpu_time": 300,
@@ -42,7 +62,9 @@ _BM_FULL_REPORT = {
     "context": _BM_CONTEXT,
     "benchmarks": [
         _BM_REPORT,
-        _BM_MEDIAN_REPORT
+        _BM_MEAN_REPORT,
+        _BM_MULTITHREAD_REPORT,
+        _BM_MULTITHREAD_MEDIAN_REPORT
     ]
 }
 
@@ -55,15 +77,6 @@ _END_TIME = 32503680000
 
 class CombineBenchmarkResultsFixture(unittest.TestCase):
 
-    @classmethod
-    def setUpClass(cls):
-        _config.EVERGREEN_REVISION = "evergreen_revision"
-        _config.EVERGREEN_PATCH_BUILD = False
-        _config.EVERGREEN_PROJECT_NAME = "evergreen_project_name"
-        _config.EVERGREEN_TASK_ID = "evergreen_task_id"
-        _config.EVERGREEN_TASK_NAME = "evergreen_task_name"
-        _config.EVERGREEN_VARIANT_NAME = "evergreen_variant_name"
-
     # Mock the hook's parent class because we're testing only functionality of this hook and
     # not anything related to or inherit from the parent class.
     @mock.patch("buildscripts.resmokelib.testing.hooks.interface.Hook", autospec=True)
@@ -82,13 +95,13 @@ class TestCombineBenchmarkResults(CombineBenchmarkResultsFixture):
     def test_generate_reports(self):
         report = self.cbr_hook._generate_perf_plugin_report()
 
-        self.assertEqual(len(report.keys()), 12)
-        self.assertEqual(len(report["data"]["results"]), 1)
+        self.assertEqual(len(report.keys()), 4)
+        self.assertEqual(len(report["results"]), 2)
 
-        self.assertDictEqual(report["data"]["results"][0]["context"], _BM_CONTEXT)
+        self.assertDictEqual(report["results"][0]["context"], _BM_CONTEXT)
 
-        self.assertEqual(report["create_time"], "2999-12-31T23:59:59Z")
-        self.assertEqual(report["data"]["end"], "3000-01-01T00:00:00Z")
+        self.assertEqual(report["start"], "2999-12-31T23:59:59Z")
+        self.assertEqual(report["end"], "3000-01-01T00:00:00Z")
 
 
 class TestBenchmarkThreadsReport(CombineBenchmarkResultsFixture):
@@ -100,24 +113,44 @@ class TestBenchmarkThreadsReport(CombineBenchmarkResultsFixture):
         thread = self.bm_threads_report._thread_from_name("BM_Name/arg name:100/threads:10_mean")
         self.assertEqual(thread, "10_mean")
 
-    def test_generate_perf_plugin_dict(self):
+        thread = self.bm_threads_report._thread_from_name("BM_Name/threads:abcd")
+        self.assertEqual(thread, "abcd")
+
+        thread = self.bm_threads_report._thread_from_name("BM_Name")
+        self.assertEqual(thread, "1")
+
+        thread = self.bm_threads_report._thread_from_name("BM_Name_mean")
+        self.assertEqual(thread, "1_mean")
+
+        thread = self.bm_threads_report._thread_from_name("BM_Name/arg name:100")
+        self.assertEqual(thread, "1")
+
+    def test_generate_multithread_perf_plugin_dict(self):
         # Also test add_report() in the process.
-        self.bm_threads_report.add_report(_BM_REPORT)
-        self.bm_threads_report.add_report(_BM_MEDIAN_REPORT)
+        self.bm_threads_report.add_report(_BM_MULTITHREAD_REPORT)
+        self.bm_threads_report.add_report(_BM_MULTITHREAD_MEDIAN_REPORT)
         self.assertEqual(len(self.bm_threads_report.thread_benchmark_map.keys()), 2)
 
         report = self.bm_threads_report.generate_perf_plugin_dict()
 
+        self.assertEqual(len(report.keys()), 1)
         self.assertIn("10", report.keys())
-        self.assertIn("10_median", report.keys())
+        self.assertNotIn("10_median", report.keys())
 
         self.assertEqual(len(report["10"]["error_values"]), 1)
         self.assertEqual(len(report["10"]["ops_per_sec_values"]), 1)
         self.assertEqual(report["10"]["ops_per_sec"], -303.0)
 
-        self.assertEqual(len(report["10_median"]["error_values"]), 1)
-        self.assertEqual(len(report["10_median"]["ops_per_sec_values"]), 1)
-        self.assertEqual(report["10_median"]["ops_per_sec"], -300.0)
+    def test_generate_single_thread_perf_plugin_dict(self):
+        self.bm_threads_report.add_report(_BM_REPORT)
+        self.bm_threads_report.add_report(_BM_MEAN_REPORT)
+        self.assertEqual(len(self.bm_threads_report.thread_benchmark_map.keys()), 2)
+
+        report = self.bm_threads_report.generate_perf_plugin_dict()
+
+        self.assertEqual(len(report.keys()), 1)
+        self.assertIn("1", report.keys())
+        self.assertNotIn("1_mean", report.keys())
 
 
 if __name__ == "__main__":