summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorIan Boros <ian.boros@mongodb.com>2020-02-18 13:21:50 -0500
committerEvergreen Agent <no-reply@evergreen.mongodb.com>2020-02-18 18:43:37 +0000
commit46ee9603414d47d18c850e5eccd18834cd7f9669 (patch)
treeadcd377cee6bdb3585fa9b7f8da56bbcc4f3333c
parent34dc015fcb40b8e4c2c99aadf1a78d7b64de6146 (diff)
downloadmongo-46ee9603414d47d18c850e5eccd18834cd7f9669.tar.gz
Revert "SERVER-45680: Burn_in_tests should pick up changed files in mongo-enterprise-modules files"
This reverts commit 77911ece0c066a68eb22f35d63b9ffd6c109b442.
-rw-r--r--buildscripts/burn_in_tests.py45
-rw-r--r--buildscripts/patch_builds/change_data.py8
-rw-r--r--buildscripts/tests/test_burn_in_tests.py15
3 files changed, 26 insertions, 42 deletions
diff --git a/buildscripts/burn_in_tests.py b/buildscripts/burn_in_tests.py
index 62b9c13c6c5..2cc09309329 100644
--- a/buildscripts/burn_in_tests.py
+++ b/buildscripts/burn_in_tests.py
@@ -12,7 +12,7 @@ import sys
from math import ceil
from collections import defaultdict
-from typing import Optional, Set, Tuple, List, Dict, Iterable
+from typing import Optional, Set, Tuple, List, Dict
import click
import requests
@@ -58,7 +58,6 @@ AVG_TEST_SETUP_SEC = 4 * 60
AVG_TEST_TIME_MULTIPLIER = 3
CONFIG_FILE = ".evergreen.yml"
DEFAULT_PROJECT = "mongodb-mongo-master"
-DEFAULT_REPO_LOCATIONS = [".", "./src/mongo/db/modules/enterprise"]
REPEAT_SUITES = 2
EVERGREEN_FILE = "etc/evergreen.yml"
MAX_TASKS_TO_CREATE = 1000
@@ -235,26 +234,21 @@ def is_file_a_test_file(file_path: str) -> bool:
return True
-def find_changed_tests(repos: Iterable[Repo]) -> Set[str]:
+def find_changed_tests(repo: Repo) -> Set[str]:
"""
Find the changed tests.
Use git to find which files have changed in this patch.
+ TODO: This should be expanded to search for enterprise modules.
The returned file paths are in normalized form (see os.path.normpath(path)).
:returns: Set of changed tests.
"""
- all_changed_tests = set()
- for repo in repos:
- changed_files = find_changed_files(repo)
- LOGGER.debug("Found changed files", files=changed_files)
- changed_tests = {
- os.path.normpath(path)
- for path in changed_files if is_file_a_test_file(path)
- }
- LOGGER.debug("Found changed tests", files=changed_tests)
- all_changed_tests.update(changed_tests)
- return all_changed_tests
+ changed_files = find_changed_files(repo)
+ LOGGER.debug("Found changed files", files=changed_files)
+ changed_tests = {os.path.normpath(path) for path in changed_files if is_file_a_test_file(path)}
+ LOGGER.debug("Found changed tests", files=changed_tests)
+ return changed_tests
def find_excludes(selector_file: str) -> Tuple[List, List, List]:
@@ -710,17 +704,16 @@ def create_task_list_for_tests(
return create_task_list(evg_conf, build_variant, tests_by_executor, exclude_tasks)
-def create_tests_by_task(build_variant: str, repos: Iterable[Repo],
- evg_conf: EvergreenProjectConfig) -> Dict:
+def create_tests_by_task(build_variant: str, repo: Repo, evg_conf: EvergreenProjectConfig) -> Dict:
"""
Create a list of tests by task.
:param build_variant: Build variant to collect tasks from.
- :param repos: Git repositories being tracked.
+ :param repo: Git repo being tracked.
:param evg_conf: Evergreen configuration.
:return: Tests by task.
"""
- changed_tests = find_changed_tests(repos)
+ changed_tests = find_changed_tests(repo)
exclude_suites, exclude_tasks, exclude_tests = find_excludes(SELECTOR_FILE)
changed_tests = filter_tests(changed_tests, exclude_tests)
@@ -819,8 +812,8 @@ def _get_evg_api(evg_api_config: str, local_mode: bool) -> Optional[EvergreenApi
def burn_in(repeat_config: RepeatConfig, generate_config: GenerateConfig, resmoke_args: str,
- generate_tasks_file: str, no_exec: bool, evg_conf: EvergreenProjectConfig,
- repos: Iterable[Repo], evg_api: EvergreenApi):
+ generate_tasks_file: str, no_exec: bool, evg_conf: EvergreenProjectConfig, repo: Repo,
+ evg_api: EvergreenApi):
"""
Run burn_in_tests with the given configuration.
@@ -830,13 +823,13 @@ def burn_in(repeat_config: RepeatConfig, generate_config: GenerateConfig, resmok
:param generate_tasks_file: File to write generated config to.
:param no_exec: Do not execute tests, just discover tests to run.
:param evg_conf: Evergreen configuration.
- :param repos: Git repositories to check.
+ :param repo: Git repository.
:param evg_api: Evergreen API client.
"""
# Populate the config values in order to use the helpers from resmokelib.suitesconfig.
resmoke_cmd = _set_resmoke_cmd(repeat_config, list(resmoke_args))
- tests_by_task = create_tests_by_task(generate_config.build_variant, repos, evg_conf)
+ tests_by_task = create_tests_by_task(generate_config.build_variant, repo, evg_conf)
LOGGER.debug("tests and tasks found", tests_by_task=tests_by_task)
if generate_tasks_file:
@@ -955,15 +948,13 @@ def main(build_variant, run_build_variant, distro, project, generate_tasks_file,
project=project,
task_id=task_id,
use_multiversion=use_multiversion) # yapf: disable
- if generate_tasks_file:
- generate_config.validate(evg_conf, local_mode)
+ generate_config.validate(evg_conf, local_mode)
evg_api = _get_evg_api(evg_api_config, local_mode)
-
- repos = [Repo(x) for x in DEFAULT_REPO_LOCATIONS if os.path.isdir(x)]
+ repo = Repo(".")
burn_in(repeat_config, generate_config, resmoke_args, generate_tasks_file, no_exec, evg_conf,
- repos, evg_api)
+ repo, evg_api)
if __name__ == "__main__":
diff --git a/buildscripts/patch_builds/change_data.py b/buildscripts/patch_builds/change_data.py
index 0ecab81cb39..d5ddbb9d849 100644
--- a/buildscripts/patch_builds/change_data.py
+++ b/buildscripts/patch_builds/change_data.py
@@ -1,6 +1,5 @@
"""Tools for detecting changes in a commit."""
from typing import Any, Set
-import os
from git import Repo, DiffIndex
import structlog
@@ -64,9 +63,4 @@ def find_changed_files(repo: Repo) -> Set[str]:
untracked_files = set(repo.untracked_files)
LOGGER.info("untracked files", files=untracked_files, diff="untracked diff")
- paths = work_tree_files.union(index_files).union(untracked_files)
-
- return [
- os.path.relpath(f"{repo.working_dir}/{os.path.normpath(path)}", os.getcwd())
- for path in paths
- ]
+ return work_tree_files.union(index_files).union(untracked_files)
diff --git a/buildscripts/tests/test_burn_in_tests.py b/buildscripts/tests/test_burn_in_tests.py
index bf207ddb37a..220db7b0cb8 100644
--- a/buildscripts/tests/test_burn_in_tests.py
+++ b/buildscripts/tests/test_burn_in_tests.py
@@ -78,7 +78,6 @@ def mock_git_diff(change_list):
def mock_changed_git_files(add_files):
repo = MagicMock()
repo.index.diff.return_value = mock_git_diff([mock_a_file(f) for f in add_files])
- repo.working_dir = "."
return repo
@@ -121,9 +120,9 @@ class TestAcceptance(unittest.TestCase):
@patch(ns("_write_json_file"))
def test_tests_generated_if_a_file_changed(self, write_json_mock):
"""
- Given a git repository with changes,
+ Given a git repository with no changes,
When burn_in_tests is run,
- Then tests are discovered to run.
+ Then no tests are discovered to run.
"""
# Note: this test is using actual tests and suites. So changes to those suites could
# introduce failures and require this test to be updated.
@@ -131,7 +130,7 @@ class TestAcceptance(unittest.TestCase):
# 'auth_audit' test suites. It needs to be in at least one of those for the test to pass.
_config.NAMED_SUITES = None
variant = "enterprise-rhel-62-64-bit"
- repos = [mock_changed_git_files(["jstests/auth/auth1.js"])]
+ repo = mock_changed_git_files(["jstests/auth/auth1.js"])
repeat_config = under_test.RepeatConfig()
gen_config = under_test.GenerateConfig(
variant,
@@ -140,7 +139,7 @@ class TestAcceptance(unittest.TestCase):
) # yapf: disable
evg_config = get_evergreen_config("etc/evergreen.yml")
- under_test.burn_in(repeat_config, gen_config, "", "testfile.json", False, evg_config, repos,
+ under_test.burn_in(repeat_config, gen_config, "", "testfile.json", False, evg_config, repo,
None)
write_json_mock.assert_called_once()
@@ -1115,7 +1114,7 @@ class TestFindChangedTests(unittest.TestCase):
changed_files_mock.return_value = set(file_list)
is_file_mock.return_value = True
- found_tests = under_test.find_changed_tests([repo_mock])
+ found_tests = under_test.find_changed_tests(repo_mock)
self.assertIn(file_list[0], found_tests)
self.assertIn(file_list[2], found_tests)
@@ -1133,7 +1132,7 @@ class TestFindChangedTests(unittest.TestCase):
changed_files_mock.return_value = set(file_list)
is_file_mock.return_value = False
- found_tests = under_test.find_changed_tests([repo_mock])
+ found_tests = under_test.find_changed_tests(repo_mock)
self.assertEqual(0, len(found_tests))
@@ -1149,7 +1148,7 @@ class TestFindChangedTests(unittest.TestCase):
changed_files_mock.return_value = set(file_list)
is_file_mock.return_value = True
- found_tests = under_test.find_changed_tests([repo_mock])
+ found_tests = under_test.find_changed_tests(repo_mock)
self.assertIn(file_list[0], found_tests)
self.assertIn(file_list[2], found_tests)