SERVER-38109 Use new Evergreen test stats API

1 files changed, 84 insertions, 444 deletions

diff --git a/buildscripts/tests/test_generate_resmoke_suites.py b/buildscripts/tests/test_generate_resmoke_suites.py
index e5501f19548..a03b979800a 100644
--- a/buildscripts/tests/test_generate_resmoke_suites.py
+++ b/buildscripts/tests/test_generate_resmoke_suites.py
@@ -11,470 +11,127 @@ import yaml
 from mock import patch, mock_open, call, Mock
 
 from buildscripts import generate_resmoke_suites as grs
-from generate_resmoke_suites import render_suite, render_misc_suite, \
+from buildscripts.generate_resmoke_suites import render_suite, render_misc_suite, \
     prepare_directory_for_suite
 
 # pylint: disable=missing-docstring,invalid-name,unused-argument,no-self-use
 
+_DATE = datetime.datetime(2018, 7, 15)
 
-class GetStartAndEndCommitSinceDateTest(unittest.TestCase):
-    @patch('buildscripts.client.github.GithubApi')
-    def test_that_first_and_last_commits_returned(self, GithubApi):
-        GithubApi.get_commits.return_value = [
-            {"sha": "first"},
-            {"sha": "second"},
-            {"sha": "third"},
-        ]
-
-        target = grs.ProjectTarget("owner", "project", "branch")
-
-        today = datetime.datetime.utcnow().replace(microsecond=0, tzinfo=None)
-
-        commitRange = grs.get_start_and_end_commit_since_date(GithubApi, target, today)
-
-        self.assertEqual(commitRange.start, "third")
-        self.assertEqual(commitRange.end, "first")
-
-
-class GetHistoryByRevisionTest(unittest.TestCase):
-    @patch('buildscripts.client.evergreen.EvergreenApi')
-    def test_get_history_by_revision_call_evergreen(self, EvergreenApi):
-        grs.get_history_by_revision(EvergreenApi, '', grs.CommitRange('start', 'end'), '', None)
-
-        self.assertTrue(EvergreenApi.get_history.called)
-
-
-class GetTestHistoryTest(unittest.TestCase):
-    @patch('buildscripts.client.evergreen.EvergreenApi')
-    def test_get_test_history_returns_when_the_end_revision_is_given(self, EvergreenApi):
-        def get_history_mock(project, params):
-            return [{"revision": "end"}]
-
-        EvergreenApi.get_history = get_history_mock
-        grs.get_test_history(EvergreenApi, grs.ProjectTarget('', '', ''), '',
-                             grs.CommitRange('start', 'end'), None)
-
-    @patch('buildscripts.client.evergreen.EvergreenApi')
-    def test_get_test_history_can_be_called_multiple_times(self, EvergreenApi):
-        call_data = {"count": 0}
 
-        def get_history_mock(project, params):
-            returnValue = [
-                "nottheend",
-                "end",
-            ]
-
-            call_data["count"] += 1
-            return [{"revision": returnValue[call_data["count"] - 1]}]
-
-        EvergreenApi.get_history = get_history_mock
-
-        grs.get_test_history(EvergreenApi, grs.ProjectTarget('', '', ''), '',
-                             grs.CommitRange('start', 'end'), None)
-
-        self.assertEqual(call_data["count"], 2)
-
-
-class SplitHookRunsOutTest(unittest.TestCase):
-    def test_a_list_with_no_hooks_returns_no_hooks(self):
-        test_list = [
-            {"test_file": "file1.js"},
-            {"test_file": "file2.js"},
-            {"test_file": "file3.js"},
-            {"test_file": "file4.js"},
-            {"test_file": "file5.js"},
+class TestTestStats(unittest.TestCase):
+    def test_no_hooks(self):
+        evg_results = [
+            self._make_evg_result("dir/test1.js", 1, 10),
+            self._make_evg_result("dir/test2.js", 1, 30),
+            self._make_evg_result("dir/test1.js", 2, 25),
         ]
-
-        (tests, hooks) = grs.split_hook_runs_out(test_list)
-
-        self.assertEqual(len(tests), 5)
-        self.assertEqual(len(hooks), 0)
-
-    def test_a_list_with_only_hooks_returns_all_hooks(self):
-        test_list = [
-            {"test_file": "file1:js"},
-            {"test_file": "file2:js"},
-            {"test_file": "file3:js"},
-            {"test_file": "file4:js"},
-            {"test_file": "file5:js"},
+        test_stats = grs.TestStats(evg_results)
+        expected_runtimes = [
+            ("dir/test2.js", 30),
+            ("dir/test1.js", 20),
         ]
-
-        (tests, hooks) = grs.split_hook_runs_out(test_list)
-
-        self.assertEqual(len(tests), 0)
-        self.assertEqual(len(hooks), 5)
-
-    def test_a_list_with_a_mix_of_test_and_hooks_returns_both(self):
-        test_list = [
-            {"test_file": "file1:js"},
-            {"test_file": "file2.js"},
-            {"test_file": "file3:js"},
-            {"test_file": "file4.js"},
-            {"test_file": "file5:js"},
+        self.assertEqual(expected_runtimes, test_stats.get_tests_runtimes())
+
+    def test_hooks(self):
+        evg_results = [
+            self._make_evg_result("dir/test1.js", 1, 10),
+            self._make_evg_result("dir/test2.js", 1, 30),
+            self._make_evg_result("dir/test1.js", 2, 25),
+            self._make_evg_result("dir/test3.js", 5, 10),
+            self._make_evg_result("test3:CleanEveryN", 10, 30),
+            self._make_evg_result("test3:CheckReplDBHash", 10, 35),
         ]
-
-        (tests, hooks) = grs.split_hook_runs_out(test_list)
-
-        self.assertEqual(len(tests), 2)
-        self.assertEqual(len(hooks), 3)
-
-
-class OrganizeHooksTest(unittest.TestCase):
-    def test_calling_with_no_executions(self):
-        hooks = grs.organize_hooks([])
-
-        self.assertEqual(len(hooks), 0)
-
-    def test_one_hooks(self):
-        executions = [{
-            "revision": "revision1",
-            "variant": "variant1",
-            "test_file": "test1:hookName",
-            "duration": 1000000000,
-        }]
-
-        hooks = grs.organize_hooks(executions)
-
-        self.assertEqual(len(hooks), 1)
-        self.assertEqual(hooks["revision1"]["variant1"]["test1"], 1000000000)
-
-    def test_multiple_hooks_on_the_same_test(self):
-        executions = [{
-            "revision": "revision1",
-            "variant": "variant1",
-            "test_file": "test1:hookName",
-            "duration": 1000000000,
-        }, {
-            "revision": "revision1",
-            "variant": "variant1",
-            "test_file": "test1:hookName1",
-            "duration": 1000000000,
-        }, {
-            "revision": "revision1",
-            "variant": "variant1",
-            "test_file": "test1:hookName2",
-            "duration": 1000000000,
-        }]
-
-        hooks = grs.organize_hooks(executions)
-
-        self.assertEqual(len(hooks), 1)
-        self.assertEqual(hooks["revision1"]["variant1"]["test1"], 3000000000)
-
-    def test_multiple_hooks_on_different_variants(self):
-        executions = [{
-            "revision": "revision1",
-            "variant": "variant1",
-            "test_file": "test1:hookName",
-            "duration": 1000000000,
-        }, {
-            "revision": "revision1",
-            "variant": "variant2",
-            "test_file": "test1:hookName1",
-            "duration": 1000000000,
-        }, {
-            "revision": "revision1",
-            "variant": "variant3",
-            "test_file": "test1:hookName2",
-            "duration": 1000000000,
-        }]
-
-        hooks = grs.organize_hooks(executions)
-
-        self.assertEqual(len(hooks), 1)
-        self.assertEqual(hooks["revision1"]["variant1"]["test1"], 1000000000)
-        self.assertEqual(hooks["revision1"]["variant2"]["test1"], 1000000000)
-        self.assertEqual(hooks["revision1"]["variant3"]["test1"], 1000000000)
-
-    def test_multiple_hooks_on_different_revisions(self):
-        executions = [{
-            "revision": "revision1",
-            "variant": "variant1",
-            "test_file": "test1:hookName",
-            "duration": 1000000000,
-        }, {
-            "revision": "revision2",
-            "variant": "variant1",
-            "test_file": "test1:hookName1",
-            "duration": 1000000000,
-        }, {
-            "revision": "revision3",
-            "variant": "variant1",
-            "test_file": "test1:hookName2",
-            "duration": 1000000000,
-        }]
-
-        hooks = grs.organize_hooks(executions)
-
-        self.assertEqual(len(hooks), 3)
-        self.assertEqual(hooks["revision1"]["variant1"]["test1"], 1000000000)
-        self.assertEqual(hooks["revision2"]["variant1"]["test1"], 1000000000)
-        self.assertEqual(hooks["revision3"]["variant1"]["test1"], 1000000000)
-
-
-class ExecutionRuntimeTest(unittest.TestCase):
-    def test_execution_runtime_is_calculated_with_no_hooks(self):
-        execution = {
-            "revision": "revision1",
-            "variant": "variant1",
-            "duration": 1000000000,
-        }
-
-        runtime = grs.execution_runtime("test.js", execution, {})
-
-        self.assertEquals(runtime, 1)
-
-    def test_execution_runtime_is_calculated_with_no_applicable_hooks(self):
-        execution = {
-            "revision": "revision1",
-            "variant": "variant1",
-            "duration": 1000000000,
-        }
-        hooks = {"revision1": {"variant2": {"test": 1000000000, }}}
-
-        runtime = grs.execution_runtime("test.js", execution, hooks)
-
-        self.assertEquals(runtime, 1)
-
-    def test_execution_runtime_is_calculated_with_hooks(self):
-        execution = {
-            "revision": "revision1",
-            "variant": "variant1",
-            "duration": 1000000000,
-        }
-        hooks = {"revision1": {"variant1": {"test": 1000000000, }}}
-
-        runtime = grs.execution_runtime("test.js", execution, hooks)
-
-        self.assertEquals(runtime, 2)
-
-
-class OrganizeExecutionsByTestTest(unittest.TestCase):
-    def test_no_executions(self):
-        tests = grs.organize_executions_by_test([])
-
-        self.assertEquals(len(tests), 0)
-
-    @patch("buildscripts.generate_resmoke_suites.os")
-    def test_only_test_executions(self, mock_os):
-        mock_os.path.isfile.return_value = True
-
-        executions = [{
-            "revision": "revision1",
-            "variant": "variant1",
-            "test_file": "test1.js",
-            "duration": 1000000000,
-        }, {
-            "revision": "revision1",
-            "variant": "variant1",
-            "test_file": "test2.js",
-            "duration": 2000000000,
-        }, {
-            "revision": "revision1",
-            "variant": "variant1",
-            "test_file": "test3.js",
-            "duration": 3000000000,
-        }]
-
-        tests = grs.organize_executions_by_test(executions)
-
-        self.assertEquals(len(tests), 3)
-        self.assertEquals(tests["test1.js"]["variant1"], 1)
-        self.assertEquals(tests["test2.js"]["variant1"], 2)
-        self.assertEquals(tests["test3.js"]["variant1"], 3)
-
-    @patch("buildscripts.generate_resmoke_suites.os")
-    def test_mix_of_test_and_hook_executions(self, mock_os):
-        mock_os.path.isfile.return_value = True
-
-        executions = [
-            {
-                "revision": "revision1",
-                "variant": "variant1",
-                "test_file": "test1.js",
-                "duration": 1000000000,
-            },
-            {
-                "revision": "revision1",
-                "variant": "variant1",
-                "test_file": "test1:js",
-                "duration": 1000000000,
-            },
-            {
-                "revision": "revision1",
-                "variant": "variant1",
-                "test_file": "test2.js",
-                "duration": 2000000000,
-            },
-            {
-                "revision": "revision1",
-                "variant": "variant1",
-                "test_file": "test3:js",
-                "duration": 3000000000,
-            },
-            {
-                "revision": "revision1",
-                "variant": "variant1",
-                "test_file": "test3.js",
-                "duration": 3000000000,
-            },
+        test_stats = grs.TestStats(evg_results)
+        expected_runtimes = [
+            ("dir/test3.js", 42.5),
+            ("dir/test2.js", 30),
+            ("dir/test1.js", 20),
         ]
+        self.assertEqual(expected_runtimes, test_stats.get_tests_runtimes())
 
-        tests = grs.organize_executions_by_test(executions)
-
-        self.assertEquals(len(tests), 3)
-        self.assertEquals(tests["test1.js"]["variant1"], 2)
-        self.assertEquals(tests["test2.js"]["variant1"], 2)
-        self.assertEquals(tests["test3.js"]["variant1"], 6)
-
-    @patch("buildscripts.generate_resmoke_suites.os")
-    def test_multiple_revisions_for_same_test(self, mock_os):
-        mock_os.path.isfile.return_value = True
-
-        executions = [
-            {
-                "revision": "revision1",
-                "variant": "variant1",
-                "test_file": "test1.js",
-                "duration": 1000000000,
-            },
-            {
-                "revision": "revision2",
-                "variant": "variant1",
-                "test_file": "test1.js",
-                "duration": 1000000000,
-            },
-            {
-                "revision": "revision1",
-                "variant": "variant1",
-                "test_file": "test1:js",
-                "duration": 1000000000,
-            },
-            {
-                "revision": "revision1",
-                "variant": "variant1",
-                "test_file": "test2.js",
-                "duration": 2000000000,
-            },
-            {
-                "revision": "revision1",
-                "variant": "variant1",
-                "test_file": "test3:js",
-                "duration": 3000000000,
-            },
-            {
-                "revision": "revision2",
-                "variant": "variant1",
-                "test_file": "test3.js",
-                "duration": 3000000000,
-            },
-        ]
-
-        tests = grs.organize_executions_by_test(executions)
-
-        self.assertEquals(len(tests), 3)
-        self.assertEquals(tests["test1.js"]["variant1"], 1)
-        self.assertEquals(tests["test2.js"]["variant1"], 2)
-        self.assertEquals(tests["test3.js"]["variant1"], 3)
-
-    @patch("buildscripts.generate_resmoke_suites.os")
-    def test_non_files_are_not_included(self, mock_os):
-        mock_os.path.isfile.return_value = False
-
-        executions = [{
-            "revision": "revision1",
-            "variant": "variant1",
-            "test_file": "test1.js",
-            "duration": 1000000000,
-        }, {
-            "revision": "revision1",
-            "variant": "variant1",
-            "test_file": "test2.js",
-            "duration": 2000000000,
-        }, {
-            "revision": "revision1",
+    @staticmethod
+    def _make_evg_result(test_file="dir/test1.js", num_pass=0, duration=0):
+        return {
+            "test_file": test_file,
+            "task_name": "task1",
             "variant": "variant1",
-            "test_file": "test3.js",
-            "duration": 3000000000,
-        }]
-
-        tests = grs.organize_executions_by_test(executions)
-
-        self.assertEquals(len(tests), 0)
+            "distro": "distro1",
+            "date": _DATE,
+            "num_pass": num_pass,
+            "num_fail": 0,
+            "avg_duration_pass": duration,
+        }
 
 
 class DivideRemainingTestsAmongSuitesTest(unittest.TestCase):
     @staticmethod
-    def generate_tests(n_tests):
-        tests = {}
-        test_names = []
-        for idx in range(n_tests):
+    def generate_tests_runtimes(n_tests):
+        tests_runtimes = []
+        # Iterating backwards so the list is sorted by descending runtimes
+        for idx in range(n_tests - 1, -1, -1):
             name = "test_{0}".format(idx)
-            test_names.append(name)
-            tests[name] = {"max_runtime": 2 * idx}
+            tests_runtimes.append((name, 2 * idx))
 
-        return test_names, tests
+        return tests_runtimes
 
     def test_each_suite_gets_one_test(self):
         suites = [grs.Suite(), grs.Suite(), grs.Suite()]
-        test_names, tests = self.generate_tests(3)
+        tests_runtimes = self.generate_tests_runtimes(3)
 
-        grs.divide_remaining_tests_among_suites(test_names, tests, suites)
+        grs.divide_remaining_tests_among_suites(tests_runtimes, suites)
 
         for suite in suites:
             self.assertEqual(suite.get_test_count(), 1)
 
     def test_each_suite_gets_at_least_one_test(self):
         suites = [grs.Suite(), grs.Suite(), grs.Suite()]
-        test_names, tests = self.generate_tests(5)
+        tests_runtimes = self.generate_tests_runtimes(5)
 
-        grs.divide_remaining_tests_among_suites(test_names, tests, suites)
+        grs.divide_remaining_tests_among_suites(tests_runtimes, suites)
 
         total_tests = 0
         for suite in suites:
             total_tests += suite.get_test_count()
             self.assertGreaterEqual(suite.get_test_count(), 1)
 
-        self.assertEqual(total_tests, len(tests))
+        self.assertEqual(total_tests, len(tests_runtimes))
 
 
 class DivideTestsIntoSuitesByMaxtimeTest(unittest.TestCase):
     def test_if_less_total_than_max_only_one_suite_created(self):
         max_time = 20
-        test_names = ["test1", "test2", "test3"]
-        tests = {
-            test_names[0]: {"max_runtime": 5},
-            test_names[1]: {"max_runtime": 4},
-            test_names[2]: {"max_runtime": 3},
-        }
+        tests_runtimes = [
+            ("test1", 5),
+            ("test2", 4),
+            ("test3", 3),
+        ]
 
-        suites = grs.divide_tests_into_suites_by_maxtime(tests, test_names, max_time)
+        suites = grs.divide_tests_into_suites(tests_runtimes, max_time)
         self.assertEqual(len(suites), 1)
         self.assertEqual(suites[0].get_test_count(), 3)
         self.assertEqual(suites[0].get_runtime(), 12)
 
     def test_if_each_test_should_be_own_suite(self):
         max_time = 5
-        test_names = ["test1", "test2", "test3"]
-        tests = {
-            test_names[0]: {"max_runtime": 5},
-            test_names[1]: {"max_runtime": 4},
-            test_names[2]: {"max_runtime": 3},
-        }
+        tests_runtimes = [
+            ("test1", 5),
+            ("test2", 4),
+            ("test3", 3),
+        ]
 
-        suites = grs.divide_tests_into_suites_by_maxtime(tests, test_names, max_time)
+        suites = grs.divide_tests_into_suites(tests_runtimes, max_time)
         self.assertEqual(len(suites), 3)
 
     def test_if_test_is_greater_than_max_it_goes_alone(self):
         max_time = 7
-        test_names = ["test1", "test2", "test3"]
-        tests = {
-            test_names[0]: {"max_runtime": 15},
-            test_names[1]: {"max_runtime": 4},
-            test_names[2]: {"max_runtime": 3},
-        }
+        tests_runtimes = [
+            ("test1", 15),
+            ("test2", 4),
+            ("test3", 3),
+        ]
 
-        suites = grs.divide_tests_into_suites_by_maxtime(tests, test_names, max_time)
+        suites = grs.divide_tests_into_suites(tests_runtimes, max_time)
         self.assertEqual(len(suites), 2)
         self.assertEqual(suites[0].get_test_count(), 1)
         self.assertEqual(suites[0].get_runtime(), 15)
@@ -482,45 +139,28 @@ class DivideTestsIntoSuitesByMaxtimeTest(unittest.TestCase):
     def test_max_sub_suites_options(self):
         max_time = 5
         max_suites = 2
-        test_names = ["test1", "test2", "test3", "test4", "test5"]
-        tests = {
-            test_names[0]: {"max_runtime": 5},
-            test_names[1]: {"max_runtime": 4},
-            test_names[2]: {"max_runtime": 3},
-            test_names[3]: {"max_runtime": 4},
-            test_names[4]: {"max_runtime": 3},
-        }
+        tests_runtimes = [
+            ("test1", 5),
+            ("test2", 4),
+            ("test3", 3),
+            ("test4", 4),
+            ("test5", 3),
+        ]
 
-        suites = grs.divide_tests_into_suites_by_maxtime(tests, test_names, max_time,
-                                                         max_suites=max_suites)
+        suites = grs.divide_tests_into_suites(tests_runtimes, max_time, max_suites=max_suites)
         self.assertEqual(len(suites), max_suites)
         total_tests = 0
         for suite in suites:
             total_tests += suite.get_test_count()
-        self.assertEqual(total_tests, len(test_names))
+        self.assertEqual(total_tests, len(tests_runtimes))
 
 
 class SuiteTest(unittest.TestCase):
     def test_adding_tests_increases_count_and_runtime(self):
         suite = grs.Suite()
-        suite.add_test('test1', {
-            "max_runtime": 10,
-            "variant1": 5,
-            "variant2": 10,
-            "variant3": 7,
-        })
-        suite.add_test('test2', {
-            "max_runtime": 12,
-            "variant1": 12,
-            "variant2": 8,
-            "variant3": 6,
-        })
-        suite.add_test('test3', {
-            "max_runtime": 7,
-            "variant1": 6,
-            "variant2": 6,
-            "variant3": 7,
-        })
+        suite.add_test('test1', 10)
+        suite.add_test('test2', 12)
+        suite.add_test('test3', 7)
 
         self.assertEqual(suite.get_test_count(), 3)
         self.assertEqual(suite.get_runtime(), 29)
@@ -530,7 +170,7 @@ def create_suite(count=3, start=0):
     """ Create a suite with count tests."""
     suite = grs.Suite()
     for i in range(start, start + count):
-        suite.add_test('test{}'.format(i), {})
+        suite.add_test('test{}'.format(i), 1)
     return suite
 
 
@@ -552,7 +192,7 @@ class RenderSuites(unittest.TestCase):
         ]
 
         m = mock_open(read_data=yaml.dump({'selector': {'roots': [], 'excludes': ['fixed']}}))
-        with patch('generate_resmoke_suites.open', m, create=True):
+        with patch('buildscripts.generate_resmoke_suites.open', m, create=True):
             render_suite(suites, 'suite_name')
         handle = m()
 
@@ -584,7 +224,7 @@ class RenderMiscSuites(unittest.TestCase):
 
         test_list = ['test{}'.format(i) for i in range(10)]
         m = mock_open(read_data=yaml.dump({'selector': {'roots': []}}))
-        with patch('generate_resmoke_suites.open', m, create=True):
+        with patch('buildscripts.generate_resmoke_suites.open', m, create=True):
             render_misc_suite(test_list, 'suite_name')
         handle = m()
 
@@ -613,7 +253,7 @@ class RenderMiscSuites(unittest.TestCase):
 
 class PrepareDirectoryForSuite(unittest.TestCase):
     def test_no_directory(self):
-        with patch('generate_resmoke_suites.os') as mock_os:
+        with patch('buildscripts.generate_resmoke_suites.os') as mock_os:
             mock_os.path.exists.return_value = False
             prepare_directory_for_suite('tmp')