SERVER-45680: Burn_in_tests should pick up changed files in mongo-enterprise-modules files

3 files changed, 42 insertions, 26 deletions

diff --git a/buildscripts/burn_in_tests.py b/buildscripts/burn_in_tests.py
index 2cc09309329..62b9c13c6c5 100644
--- a/buildscripts/burn_in_tests.py
+++ b/buildscripts/burn_in_tests.py
@@ -12,7 +12,7 @@ import sys
 
 from math import ceil
 from collections import defaultdict
-from typing import Optional, Set, Tuple, List, Dict
+from typing import Optional, Set, Tuple, List, Dict, Iterable
 
 import click
 import requests
@@ -58,6 +58,7 @@ AVG_TEST_SETUP_SEC = 4 * 60
 AVG_TEST_TIME_MULTIPLIER = 3
 CONFIG_FILE = ".evergreen.yml"
 DEFAULT_PROJECT = "mongodb-mongo-master"
+DEFAULT_REPO_LOCATIONS = [".", "./src/mongo/db/modules/enterprise"]
 REPEAT_SUITES = 2
 EVERGREEN_FILE = "etc/evergreen.yml"
 MAX_TASKS_TO_CREATE = 1000
@@ -234,21 +235,26 @@ def is_file_a_test_file(file_path: str) -> bool:
     return True
 
 
-def find_changed_tests(repo: Repo) -> Set[str]:
+def find_changed_tests(repos: Iterable[Repo]) -> Set[str]:
     """
     Find the changed tests.
 
     Use git to find which files have changed in this patch.
-    TODO: This should be expanded to search for enterprise modules.
     The returned file paths are in normalized form (see os.path.normpath(path)).
 
     :returns: Set of changed tests.
     """
-    changed_files = find_changed_files(repo)
-    LOGGER.debug("Found changed files", files=changed_files)
-    changed_tests = {os.path.normpath(path) for path in changed_files if is_file_a_test_file(path)}
-    LOGGER.debug("Found changed tests", files=changed_tests)
-    return changed_tests
+    all_changed_tests = set()
+    for repo in repos:
+        changed_files = find_changed_files(repo)
+        LOGGER.debug("Found changed files", files=changed_files)
+        changed_tests = {
+            os.path.normpath(path)
+            for path in changed_files if is_file_a_test_file(path)
+        }
+        LOGGER.debug("Found changed tests", files=changed_tests)
+        all_changed_tests.update(changed_tests)
+    return all_changed_tests
 
 
 def find_excludes(selector_file: str) -> Tuple[List, List, List]:
@@ -704,16 +710,17 @@ def create_task_list_for_tests(
     return create_task_list(evg_conf, build_variant, tests_by_executor, exclude_tasks)
 
 
-def create_tests_by_task(build_variant: str, repo: Repo, evg_conf: EvergreenProjectConfig) -> Dict:
+def create_tests_by_task(build_variant: str, repos: Iterable[Repo],
+                         evg_conf: EvergreenProjectConfig) -> Dict:
     """
     Create a list of tests by task.
 
     :param build_variant: Build variant to collect tasks from.
-    :param repo: Git repo being tracked.
+    :param repos: Git repositories being tracked.
     :param evg_conf: Evergreen configuration.
     :return: Tests by task.
     """
-    changed_tests = find_changed_tests(repo)
+    changed_tests = find_changed_tests(repos)
     exclude_suites, exclude_tasks, exclude_tests = find_excludes(SELECTOR_FILE)
     changed_tests = filter_tests(changed_tests, exclude_tests)
 
@@ -812,8 +819,8 @@ def _get_evg_api(evg_api_config: str, local_mode: bool) -> Optional[EvergreenApi
 
 
 def burn_in(repeat_config: RepeatConfig, generate_config: GenerateConfig, resmoke_args: str,
-            generate_tasks_file: str, no_exec: bool, evg_conf: EvergreenProjectConfig, repo: Repo,
-            evg_api: EvergreenApi):
+            generate_tasks_file: str, no_exec: bool, evg_conf: EvergreenProjectConfig,
+            repos: Iterable[Repo], evg_api: EvergreenApi):
     """
     Run burn_in_tests with the given configuration.
 
@@ -823,13 +830,13 @@ def burn_in(repeat_config: RepeatConfig, generate_config: GenerateConfig, resmok
     :param generate_tasks_file: File to write generated config to.
     :param no_exec: Do not execute tests, just discover tests to run.
     :param evg_conf: Evergreen configuration.
-    :param repo: Git repository.
+    :param repos: Git repositories to check.
     :param evg_api: Evergreen API client.
     """
     # Populate the config values in order to use the helpers from resmokelib.suitesconfig.
     resmoke_cmd = _set_resmoke_cmd(repeat_config, list(resmoke_args))
 
-    tests_by_task = create_tests_by_task(generate_config.build_variant, repo, evg_conf)
+    tests_by_task = create_tests_by_task(generate_config.build_variant, repos, evg_conf)
     LOGGER.debug("tests and tasks found", tests_by_task=tests_by_task)
 
     if generate_tasks_file:
@@ -948,13 +955,15 @@ def main(build_variant, run_build_variant, distro, project, generate_tasks_file,
                                      project=project,
                                      task_id=task_id,
                                      use_multiversion=use_multiversion)  # yapf: disable
-    generate_config.validate(evg_conf, local_mode)
+    if generate_tasks_file:
+        generate_config.validate(evg_conf, local_mode)
 
     evg_api = _get_evg_api(evg_api_config, local_mode)
-    repo = Repo(".")
+
+    repos = [Repo(x) for x in DEFAULT_REPO_LOCATIONS if os.path.isdir(x)]
 
     burn_in(repeat_config, generate_config, resmoke_args, generate_tasks_file, no_exec, evg_conf,
-            repo, evg_api)
+            repos, evg_api)
 
 
 if __name__ == "__main__":
diff --git a/buildscripts/patch_builds/change_data.py b/buildscripts/patch_builds/change_data.py
index d5ddbb9d849..0ecab81cb39 100644
--- a/buildscripts/patch_builds/change_data.py
+++ b/buildscripts/patch_builds/change_data.py
@@ -1,5 +1,6 @@
 """Tools for detecting changes in a commit."""
 from typing import Any, Set
+import os
 
 from git import Repo, DiffIndex
 import structlog
@@ -63,4 +64,9 @@ def find_changed_files(repo: Repo) -> Set[str]:
     untracked_files = set(repo.untracked_files)
     LOGGER.info("untracked files", files=untracked_files, diff="untracked diff")
 
-    return work_tree_files.union(index_files).union(untracked_files)
+    paths = work_tree_files.union(index_files).union(untracked_files)
+
+    return [
+        os.path.relpath(f"{repo.working_dir}/{os.path.normpath(path)}", os.getcwd())
+        for path in paths
+    ]
diff --git a/buildscripts/tests/test_burn_in_tests.py b/buildscripts/tests/test_burn_in_tests.py
index 220db7b0cb8..bf207ddb37a 100644
--- a/buildscripts/tests/test_burn_in_tests.py
+++ b/buildscripts/tests/test_burn_in_tests.py
@@ -78,6 +78,7 @@ def mock_git_diff(change_list):
 def mock_changed_git_files(add_files):
     repo = MagicMock()
     repo.index.diff.return_value = mock_git_diff([mock_a_file(f) for f in add_files])
+    repo.working_dir = "."
     return repo
 
 
@@ -120,9 +121,9 @@ class TestAcceptance(unittest.TestCase):
     @patch(ns("_write_json_file"))
     def test_tests_generated_if_a_file_changed(self, write_json_mock):
         """
-        Given a git repository with no changes,
+        Given a git repository with changes,
         When burn_in_tests is run,
-        Then no tests are discovered to run.
+        Then tests are discovered to run.
         """
         # Note: this test is using actual tests and suites. So changes to those suites could
         # introduce failures and require this test to be updated.
@@ -130,7 +131,7 @@ class TestAcceptance(unittest.TestCase):
         # 'auth_audit' test suites. It needs to be in at least one of those for the test to pass.
         _config.NAMED_SUITES = None
         variant = "enterprise-rhel-62-64-bit"
-        repo = mock_changed_git_files(["jstests/auth/auth1.js"])
+        repos = [mock_changed_git_files(["jstests/auth/auth1.js"])]
         repeat_config = under_test.RepeatConfig()
         gen_config = under_test.GenerateConfig(
             variant,
@@ -139,7 +140,7 @@ class TestAcceptance(unittest.TestCase):
         )  # yapf: disable
         evg_config = get_evergreen_config("etc/evergreen.yml")
 
-        under_test.burn_in(repeat_config, gen_config, "", "testfile.json", False, evg_config, repo,
+        under_test.burn_in(repeat_config, gen_config, "", "testfile.json", False, evg_config, repos,
                            None)
 
         write_json_mock.assert_called_once()
@@ -1114,7 +1115,7 @@ class TestFindChangedTests(unittest.TestCase):
         changed_files_mock.return_value = set(file_list)
         is_file_mock.return_value = True
 
-        found_tests = under_test.find_changed_tests(repo_mock)
+        found_tests = under_test.find_changed_tests([repo_mock])
 
         self.assertIn(file_list[0], found_tests)
         self.assertIn(file_list[2], found_tests)
@@ -1132,7 +1133,7 @@ class TestFindChangedTests(unittest.TestCase):
         changed_files_mock.return_value = set(file_list)
         is_file_mock.return_value = False
 
-        found_tests = under_test.find_changed_tests(repo_mock)
+        found_tests = under_test.find_changed_tests([repo_mock])
 
         self.assertEqual(0, len(found_tests))
 
@@ -1148,7 +1149,7 @@ class TestFindChangedTests(unittest.TestCase):
         changed_files_mock.return_value = set(file_list)
         is_file_mock.return_value = True
 
-        found_tests = under_test.find_changed_tests(repo_mock)
+        found_tests = under_test.find_changed_tests([repo_mock])
 
         self.assertIn(file_list[0], found_tests)
         self.assertIn(file_list[2], found_tests)