From 480579097465faffe5cbc4ce0ff002590be9bacf Mon Sep 17 00:00:00 2001 From: Lydia Stepanek Date: Wed, 24 Jun 2020 11:03:26 -0400 Subject: SERVER-47827 Request for a local burn_in_tests.py that locally runs all tests changed since master, including those committed in the local branch (cherry picked from commit c848b16c777e78fa3b21d7b0ad538ee2072346d4) --- buildscripts/burn_in_tags.py | 14 ++++++-- buildscripts/burn_in_tests.py | 57 ++++++++++++++++++++++++------ buildscripts/burn_in_tests_multiversion.py | 5 +-- buildscripts/tests/test_burn_in_tags.py | 4 +-- buildscripts/tests/test_burn_in_tests.py | 22 +++++++++--- 5 files changed, 80 insertions(+), 22 deletions(-) (limited to 'buildscripts') diff --git a/buildscripts/burn_in_tags.py b/buildscripts/burn_in_tags.py index 6ed3d1d5bec..a5d5efd1fb8 100644 --- a/buildscripts/burn_in_tags.py +++ b/buildscripts/burn_in_tags.py @@ -22,7 +22,7 @@ import buildscripts.util.read_config as read_config from buildscripts.ciconfig import evergreen from buildscripts.ciconfig.evergreen import EvergreenProjectConfig, Variant from buildscripts.burn_in_tests import create_generate_tasks_config, create_tests_by_task, \ - GenerateConfig, RepeatConfig, DEFAULT_REPO_LOCATIONS + find_changed_tests, GenerateConfig, RepeatConfig, DEFAULT_REPO_LOCATIONS # pylint: enable=wrong-import-position CONFIG_DIRECTORY = "generated_burn_in_tags_config" @@ -135,7 +135,8 @@ def _generate_evg_tasks(evergreen_api: EvergreenApi, shrub_project: ShrubProject """ for build_variant, run_build_variant in build_variant_map.items(): config_options = _get_config_options(task_expansions, build_variant, run_build_variant) - tests_by_task = create_tests_by_task(build_variant, repos, evg_conf) + changed_tests = find_changed_tests(repos) + tests_by_task = create_tests_by_task(build_variant, evg_conf, changed_tests) if tests_by_task: shrub_build_variant = _generate_evg_build_variant( evg_conf.get_variant(build_variant), run_build_variant, @@ -153,7 +154,14 @@ def _generate_evg_tasks(evergreen_api: EvergreenApi, shrub_project: ShrubProject def burn_in(task_expansions: Dict[str, Any], evg_conf: EvergreenProjectConfig, evergreen_api: RetryingEvergreenApi, repos: Iterable[Repo]): - """Execute Main program.""" + """ + Execute main program. + + :param task_expansions: Dictionary of expansions for the running task. + :param evergreen_conf: Evergreen configuration. + :param evergreen_api: Evergreen.py object. + :param repos: Git repositories. + """ shrub_project = ShrubProject.empty() build_variant_map = _create_evg_build_variant_map(task_expansions, evg_conf) _generate_evg_tasks(evergreen_api, shrub_project, task_expansions, build_variant_map, repos, diff --git a/buildscripts/burn_in_tests.py b/buildscripts/burn_in_tests.py index 2d8d61f5958..b327456098d 100644 --- a/buildscripts/burn_in_tests.py +++ b/buildscripts/burn_in_tests.py @@ -37,6 +37,7 @@ from buildscripts.util.teststats import TestStats from buildscripts.util.taskname import name_generated_task from buildscripts.patch_builds.task_generation import (resmoke_commands, TimeoutInfo, validate_task_generation_limit) + # pylint: enable=wrong-import-position structlog.configure(logger_factory=LoggerFactory()) @@ -200,7 +201,25 @@ def is_file_a_test_file(file_path: str) -> bool: return True -def find_changed_tests(repos: Iterable[Repo]) -> Set[str]: +def get_revision_map(repos: Iterable[Repo], origin_rev: Optional[str] = None) -> Set[str]: + """ + If origin_rev is specified, compare changes against this revision in the mongo repo. + + :param repos: List of repos containing changed files. + :param origin_rev: The revision that local changes will be compared against. + :returns: Set of changed tests. + """ + if origin_rev: + mongo_repo_dir = [ + repo.git_dir for repo in repos if os.path.basename(repo.working_dir) == "mongo" + ] + if len(mongo_repo_dir) != 1: + raise Exception("Mongo repo not found in repos list") + return {mongo_repo_dir[0]: origin_rev} + return None + + +def find_changed_tests(repos: Iterable[Repo], origin_rev: Optional[str] = None) -> Set[str]: """ Find the changed tests. @@ -208,9 +227,12 @@ def find_changed_tests(repos: Iterable[Repo]) -> Set[str]: The returned file paths are in normalized form (see os.path.normpath(path)). :param repos: List of repos containing changed files. + :param origin_rev: The revision that local changes will be compared against. :returns: Set of changed tests. """ - changed_files = find_changed_files_in_repos(repos) + revision_map = get_revision_map(repos, origin_rev) + LOGGER.info("Calculated revision map", revision_map=revision_map) + changed_files = find_changed_files_in_repos(repos, revision_map) return {os.path.normpath(path) for path in changed_files if is_file_a_test_file(path)} @@ -629,18 +651,16 @@ def create_task_list_for_tests( return create_task_list(evg_conf, build_variant, tests_by_executor, exclude_tasks) -def create_tests_by_task(build_variant: str, repos: Iterable[Repo], - evg_conf: EvergreenProjectConfig) -> Dict: +def create_tests_by_task(build_variant: str, evg_conf: EvergreenProjectConfig, + changed_tests: Set[str]) -> Dict: """ Create a list of tests by task. :param build_variant: Build variant to collect tasks from. - :param repos: Git repositories being tracked. :param evg_conf: Evergreen configuration. + :param changed_tests: Set of changed test files. :return: Tests by task. """ - changed_tests = find_changed_tests(repos) - LOGGER.debug("Found changed tests", files=changed_tests) exclude_suites, exclude_tasks, exclude_tests = find_excludes(SELECTOR_FILE) changed_tests = filter_tests(changed_tests, exclude_tests) @@ -735,7 +755,7 @@ def _get_evg_api(evg_api_config: str, local_mode: bool) -> Optional[EvergreenApi def burn_in(repeat_config: RepeatConfig, generate_config: GenerateConfig, resmoke_args: str, generate_tasks_file: str, no_exec: bool, evg_conf: EvergreenProjectConfig, - repos: Iterable[Repo], evg_api: EvergreenApi) -> None: + repos: Iterable[Repo], evg_api: EvergreenApi, origin_rev: Optional[str]) -> None: """ Run burn_in_tests with the given configuration. @@ -747,11 +767,16 @@ def burn_in(repeat_config: RepeatConfig, generate_config: GenerateConfig, resmok :param evg_conf: Evergreen configuration. :param repos: Git repositories to check. :param evg_api: Evergreen API client. + :param project: Evergreen project to query. + :param origin_rev: The revision that local changes will be compared against. """ + changed_tests = find_changed_tests(repos, origin_rev) + LOGGER.info("Found changed tests", files=changed_tests) + # Populate the config values in order to use the helpers from resmokelib.suitesconfig. resmoke_cmd = _set_resmoke_cmd(repeat_config, list(resmoke_args)) - tests_by_task = create_tests_by_task(generate_config.build_variant, repos, evg_conf) + tests_by_task = create_tests_by_task(generate_config.build_variant, evg_conf, changed_tests) LOGGER.debug("tests and tasks found", tests_by_task=tests_by_task) if generate_tasks_file: @@ -792,17 +817,26 @@ def burn_in(repeat_config: RepeatConfig, generate_config: GenerateConfig, resmok @click.option("--verbose", "verbose", default=False, is_flag=True, help="Enable extra logging.") @click.option("--task_id", "task_id", default=None, metavar='TASK_ID', help="The evergreen task id.") +@click.option( + "--origin-rev", "origin_rev", default=None, + help="The revision in the mongo repo that changes will be compared against if specified.") @click.argument("resmoke_args", nargs=-1, type=click.UNPROCESSED) # pylint: disable=too-many-arguments,too-many-locals def main(build_variant, run_build_variant, distro, project, generate_tasks_file, no_exec, repeat_tests_num, repeat_tests_min, repeat_tests_max, repeat_tests_secs, resmoke_args, - local_mode, evg_api_config, verbose, task_id): + local_mode, evg_api_config, verbose, task_id, origin_rev): """ Run new or changed tests in repeated mode to validate their stability. burn_in_tests detects jstests that are new or changed since the last git command and then runs those tests in a loop to validate their reliability. + The `--origin-rev` argument allows users to specify which revision should be used as the last + git command to compare against to find changed files. If the `--origin-rev` argument is provided, + we find changed files by comparing your latest changes to this revision. If not provided, we + find changed test files by comparing your latest changes to HEAD. The revision provided must + be a revision that exists in the mongodb repository. + The `--repeat-*` arguments allow configuration of how burn_in_tests repeats tests. Tests can either be repeated a specified number of times with the `--repeat-tests` option, or they can be repeated for a certain time period with the `--repeat-tests-secs` option. @@ -837,6 +871,7 @@ def main(build_variant, run_build_variant, distro, project, generate_tasks_file, :param local_mode: Don't call out to the evergreen API (used for testing). :param evg_api_config: Location of configuration file to connect to evergreen. :param verbose: Log extra debug information. + :param origin_rev: The revision that local changes will be compared against. """ _configure_logging(verbose) @@ -858,7 +893,7 @@ def main(build_variant, run_build_variant, distro, project, generate_tasks_file, repos = [Repo(x) for x in DEFAULT_REPO_LOCATIONS if os.path.isdir(x)] burn_in(repeat_config, generate_config, resmoke_args, generate_tasks_file, no_exec, evg_conf, - repos, evg_api) + repos, evg_api, origin_rev) if __name__ == "__main__": diff --git a/buildscripts/burn_in_tests_multiversion.py b/buildscripts/burn_in_tests_multiversion.py index ad0f008d75d..ad59089d46b 100644 --- a/buildscripts/burn_in_tests_multiversion.py +++ b/buildscripts/burn_in_tests_multiversion.py @@ -16,7 +16,7 @@ import buildscripts.evergreen_gen_multiversion_tests as gen_multiversion import buildscripts.evergreen_generate_resmoke_tasks as gen_resmoke from buildscripts.burn_in_tests import GenerateConfig, DEFAULT_PROJECT, CONFIG_FILE, _configure_logging, RepeatConfig, \ _get_evg_api, EVERGREEN_FILE, DEFAULT_REPO_LOCATIONS, _set_resmoke_cmd, create_tests_by_task, \ - run_tests + find_changed_tests, run_tests from buildscripts.ciconfig.evergreen import parse_evergreen_file from buildscripts.patch_builds.task_generation import validate_task_generation_limit from buildscripts.resmokelib.suitesconfig import get_named_suites_with_root_level_key @@ -160,7 +160,8 @@ def main(build_variant, run_build_variant, distro, project, generate_tasks_file, resmoke_cmd = _set_resmoke_cmd(repeat_config, list(resmoke_args)) - tests_by_task = create_tests_by_task(generate_config.build_variant, repos, evg_conf) + changed_tests = find_changed_tests(repos) + tests_by_task = create_tests_by_task(generate_config.build_variant, evg_conf, changed_tests) LOGGER.debug("tests and tasks found", tests_by_task=tests_by_task) if generate_tasks_file: diff --git a/buildscripts/tests/test_burn_in_tags.py b/buildscripts/tests/test_burn_in_tags.py index c9e9d46843b..e6c99d66662 100644 --- a/buildscripts/tests/test_burn_in_tags.py +++ b/buildscripts/tests/test_burn_in_tags.py @@ -202,7 +202,7 @@ CREATE_TEST_MEMBERSHIP_MAP = { class TestAcceptance(unittest.TestCase): @patch(ns("write_file_to_dir")) @patch(ns("_create_evg_build_variant_map")) - @patch(burn_in_tests_ns("find_changed_tests")) + @patch(ns("find_changed_tests")) def test_no_tests_run_if_none_changed(self, find_changed_tests_mock, create_evg_build_variant_map_mock, write_to_file_mock): """ @@ -225,7 +225,7 @@ class TestAcceptance(unittest.TestCase): @unittest.skipIf(sys.platform.startswith("win"), "not supported on windows") @patch(ns("write_file_to_dir")) @patch(ns("_create_evg_build_variant_map")) - @patch(burn_in_tests_ns("find_changed_tests")) + @patch(ns("find_changed_tests")) @patch(burn_in_tests_ns("create_test_membership_map")) def test_tests_generated_if_a_file_changed( self, create_test_membership_map_mock, find_changed_tests_mock, diff --git a/buildscripts/tests/test_burn_in_tests.py b/buildscripts/tests/test_burn_in_tests.py index 89f23e54795..523a3c20242 100644 --- a/buildscripts/tests/test_burn_in_tests.py +++ b/buildscripts/tests/test_burn_in_tests.py @@ -93,10 +93,9 @@ class TestAcceptance(unittest.TestCase): variant, "project", ) # yapf: disable - evg_conf_mock = MagicMock() - evg_conf_mock.get_task_names_by_tag.return_value = set() - under_test.burn_in(repeat_config, gen_config, "", "testfile.json", False, None, repos, None) + under_test.burn_in(repeat_config, gen_config, "", "testfile.json", False, None, repos, None, + None) write_json_mock.assert_called_once() written_config = json.loads(write_json_mock.call_args[0][1]) @@ -127,7 +126,7 @@ class TestAcceptance(unittest.TestCase): evg_config = get_evergreen_config("etc/evergreen.yml") under_test.burn_in(repeat_config, gen_config, "", "testfile.json", False, evg_config, repos, - None) + None, None) write_json_mock.assert_called_once() written_config = json.loads(write_json_mock.call_args[0][1]) @@ -886,3 +885,18 @@ class TestFindChangedTests(unittest.TestCase): self.assertIn(file_list[2], found_tests) self.assertNotIn(file_list[1], found_tests) self.assertEqual(2, len(found_tests)) + + +class TestGetRevisionMap(unittest.TestCase): + def test_get_revision_map(self): + repos = [ + MagicMock(git_dir="/mongo/.git", working_dir="/mongo"), + MagicMock(git_dir="/enterprise/.git", working_dir="/enterprise") + ] + revision_map = under_test.get_revision_map(repos, "sha") + assert revision_map == {"/mongo/.git": "sha"} + + def test_no_mongo_repo(self): + repos = [MagicMock(git_dir="/enterprise/.git", working_dir="/enterprise")] + with self.assertRaises(Exception): + under_test.get_revision_map(repos, "sha") -- cgit v1.2.1