diff options
author | Ian Boros <ian.boros@mongodb.com> | 2020-02-18 13:21:50 -0500 |
---|---|---|
committer | Evergreen Agent <no-reply@evergreen.mongodb.com> | 2020-02-18 18:43:37 +0000 |
commit | 46ee9603414d47d18c850e5eccd18834cd7f9669 (patch) | |
tree | adcd377cee6bdb3585fa9b7f8da56bbcc4f3333c | |
parent | 34dc015fcb40b8e4c2c99aadf1a78d7b64de6146 (diff) | |
download | mongo-46ee9603414d47d18c850e5eccd18834cd7f9669.tar.gz |
Revert "SERVER-45680: Burn_in_tests should pick up changed files in mongo-enterprise-modules files"
This reverts commit 77911ece0c066a68eb22f35d63b9ffd6c109b442.
-rw-r--r-- | buildscripts/burn_in_tests.py | 45 | ||||
-rw-r--r-- | buildscripts/patch_builds/change_data.py | 8 | ||||
-rw-r--r-- | buildscripts/tests/test_burn_in_tests.py | 15 |
3 files changed, 26 insertions, 42 deletions
diff --git a/buildscripts/burn_in_tests.py b/buildscripts/burn_in_tests.py index 62b9c13c6c5..2cc09309329 100644 --- a/buildscripts/burn_in_tests.py +++ b/buildscripts/burn_in_tests.py @@ -12,7 +12,7 @@ import sys from math import ceil from collections import defaultdict -from typing import Optional, Set, Tuple, List, Dict, Iterable +from typing import Optional, Set, Tuple, List, Dict import click import requests @@ -58,7 +58,6 @@ AVG_TEST_SETUP_SEC = 4 * 60 AVG_TEST_TIME_MULTIPLIER = 3 CONFIG_FILE = ".evergreen.yml" DEFAULT_PROJECT = "mongodb-mongo-master" -DEFAULT_REPO_LOCATIONS = [".", "./src/mongo/db/modules/enterprise"] REPEAT_SUITES = 2 EVERGREEN_FILE = "etc/evergreen.yml" MAX_TASKS_TO_CREATE = 1000 @@ -235,26 +234,21 @@ def is_file_a_test_file(file_path: str) -> bool: return True -def find_changed_tests(repos: Iterable[Repo]) -> Set[str]: +def find_changed_tests(repo: Repo) -> Set[str]: """ Find the changed tests. Use git to find which files have changed in this patch. + TODO: This should be expanded to search for enterprise modules. The returned file paths are in normalized form (see os.path.normpath(path)). :returns: Set of changed tests. """ - all_changed_tests = set() - for repo in repos: - changed_files = find_changed_files(repo) - LOGGER.debug("Found changed files", files=changed_files) - changed_tests = { - os.path.normpath(path) - for path in changed_files if is_file_a_test_file(path) - } - LOGGER.debug("Found changed tests", files=changed_tests) - all_changed_tests.update(changed_tests) - return all_changed_tests + changed_files = find_changed_files(repo) + LOGGER.debug("Found changed files", files=changed_files) + changed_tests = {os.path.normpath(path) for path in changed_files if is_file_a_test_file(path)} + LOGGER.debug("Found changed tests", files=changed_tests) + return changed_tests def find_excludes(selector_file: str) -> Tuple[List, List, List]: @@ -710,17 +704,16 @@ def create_task_list_for_tests( return create_task_list(evg_conf, build_variant, tests_by_executor, exclude_tasks) -def create_tests_by_task(build_variant: str, repos: Iterable[Repo], - evg_conf: EvergreenProjectConfig) -> Dict: +def create_tests_by_task(build_variant: str, repo: Repo, evg_conf: EvergreenProjectConfig) -> Dict: """ Create a list of tests by task. :param build_variant: Build variant to collect tasks from. - :param repos: Git repositories being tracked. + :param repo: Git repo being tracked. :param evg_conf: Evergreen configuration. :return: Tests by task. """ - changed_tests = find_changed_tests(repos) + changed_tests = find_changed_tests(repo) exclude_suites, exclude_tasks, exclude_tests = find_excludes(SELECTOR_FILE) changed_tests = filter_tests(changed_tests, exclude_tests) @@ -819,8 +812,8 @@ def _get_evg_api(evg_api_config: str, local_mode: bool) -> Optional[EvergreenApi def burn_in(repeat_config: RepeatConfig, generate_config: GenerateConfig, resmoke_args: str, - generate_tasks_file: str, no_exec: bool, evg_conf: EvergreenProjectConfig, - repos: Iterable[Repo], evg_api: EvergreenApi): + generate_tasks_file: str, no_exec: bool, evg_conf: EvergreenProjectConfig, repo: Repo, + evg_api: EvergreenApi): """ Run burn_in_tests with the given configuration. @@ -830,13 +823,13 @@ def burn_in(repeat_config: RepeatConfig, generate_config: GenerateConfig, resmok :param generate_tasks_file: File to write generated config to. :param no_exec: Do not execute tests, just discover tests to run. :param evg_conf: Evergreen configuration. - :param repos: Git repositories to check. + :param repo: Git repository. :param evg_api: Evergreen API client. """ # Populate the config values in order to use the helpers from resmokelib.suitesconfig. resmoke_cmd = _set_resmoke_cmd(repeat_config, list(resmoke_args)) - tests_by_task = create_tests_by_task(generate_config.build_variant, repos, evg_conf) + tests_by_task = create_tests_by_task(generate_config.build_variant, repo, evg_conf) LOGGER.debug("tests and tasks found", tests_by_task=tests_by_task) if generate_tasks_file: @@ -955,15 +948,13 @@ def main(build_variant, run_build_variant, distro, project, generate_tasks_file, project=project, task_id=task_id, use_multiversion=use_multiversion) # yapf: disable - if generate_tasks_file: - generate_config.validate(evg_conf, local_mode) + generate_config.validate(evg_conf, local_mode) evg_api = _get_evg_api(evg_api_config, local_mode) - - repos = [Repo(x) for x in DEFAULT_REPO_LOCATIONS if os.path.isdir(x)] + repo = Repo(".") burn_in(repeat_config, generate_config, resmoke_args, generate_tasks_file, no_exec, evg_conf, - repos, evg_api) + repo, evg_api) if __name__ == "__main__": diff --git a/buildscripts/patch_builds/change_data.py b/buildscripts/patch_builds/change_data.py index 0ecab81cb39..d5ddbb9d849 100644 --- a/buildscripts/patch_builds/change_data.py +++ b/buildscripts/patch_builds/change_data.py @@ -1,6 +1,5 @@ """Tools for detecting changes in a commit.""" from typing import Any, Set -import os from git import Repo, DiffIndex import structlog @@ -64,9 +63,4 @@ def find_changed_files(repo: Repo) -> Set[str]: untracked_files = set(repo.untracked_files) LOGGER.info("untracked files", files=untracked_files, diff="untracked diff") - paths = work_tree_files.union(index_files).union(untracked_files) - - return [ - os.path.relpath(f"{repo.working_dir}/{os.path.normpath(path)}", os.getcwd()) - for path in paths - ] + return work_tree_files.union(index_files).union(untracked_files) diff --git a/buildscripts/tests/test_burn_in_tests.py b/buildscripts/tests/test_burn_in_tests.py index bf207ddb37a..220db7b0cb8 100644 --- a/buildscripts/tests/test_burn_in_tests.py +++ b/buildscripts/tests/test_burn_in_tests.py @@ -78,7 +78,6 @@ def mock_git_diff(change_list): def mock_changed_git_files(add_files): repo = MagicMock() repo.index.diff.return_value = mock_git_diff([mock_a_file(f) for f in add_files]) - repo.working_dir = "." return repo @@ -121,9 +120,9 @@ class TestAcceptance(unittest.TestCase): @patch(ns("_write_json_file")) def test_tests_generated_if_a_file_changed(self, write_json_mock): """ - Given a git repository with changes, + Given a git repository with no changes, When burn_in_tests is run, - Then tests are discovered to run. + Then no tests are discovered to run. """ # Note: this test is using actual tests and suites. So changes to those suites could # introduce failures and require this test to be updated. @@ -131,7 +130,7 @@ class TestAcceptance(unittest.TestCase): # 'auth_audit' test suites. It needs to be in at least one of those for the test to pass. _config.NAMED_SUITES = None variant = "enterprise-rhel-62-64-bit" - repos = [mock_changed_git_files(["jstests/auth/auth1.js"])] + repo = mock_changed_git_files(["jstests/auth/auth1.js"]) repeat_config = under_test.RepeatConfig() gen_config = under_test.GenerateConfig( variant, @@ -140,7 +139,7 @@ class TestAcceptance(unittest.TestCase): ) # yapf: disable evg_config = get_evergreen_config("etc/evergreen.yml") - under_test.burn_in(repeat_config, gen_config, "", "testfile.json", False, evg_config, repos, + under_test.burn_in(repeat_config, gen_config, "", "testfile.json", False, evg_config, repo, None) write_json_mock.assert_called_once() @@ -1115,7 +1114,7 @@ class TestFindChangedTests(unittest.TestCase): changed_files_mock.return_value = set(file_list) is_file_mock.return_value = True - found_tests = under_test.find_changed_tests([repo_mock]) + found_tests = under_test.find_changed_tests(repo_mock) self.assertIn(file_list[0], found_tests) self.assertIn(file_list[2], found_tests) @@ -1133,7 +1132,7 @@ class TestFindChangedTests(unittest.TestCase): changed_files_mock.return_value = set(file_list) is_file_mock.return_value = False - found_tests = under_test.find_changed_tests([repo_mock]) + found_tests = under_test.find_changed_tests(repo_mock) self.assertEqual(0, len(found_tests)) @@ -1149,7 +1148,7 @@ class TestFindChangedTests(unittest.TestCase): changed_files_mock.return_value = set(file_list) is_file_mock.return_value = True - found_tests = under_test.find_changed_tests([repo_mock]) + found_tests = under_test.find_changed_tests(repo_mock) self.assertIn(file_list[0], found_tests) self.assertIn(file_list[2], found_tests) |