diff options
author | Robert Guo <robert.guo@mongodb.com> | 2021-10-02 04:26:30 +0000 |
---|---|---|
committer | Evergreen Agent <no-reply@evergreen.mongodb.com> | 2021-10-02 04:55:02 +0000 |
commit | ba5cf359d87e3fb3b32d24056e0da1b8452ac304 (patch) | |
tree | 0820ab18a471e081c8c2b523b8a2807b9533d23a | |
parent | b8b8e0f26453fbcc239a6fef92ab32f6545870cd (diff) | |
download | mongo-ba5cf359d87e3fb3b32d24056e0da1b8452ac304.tar.gz |
SERVER-55857 remove NAMED_SUITES from resmoke config
26 files changed, 210 insertions, 161 deletions
diff --git a/.gitignore b/.gitignore index acb29394d1c..7c2303af13a 100644 --- a/.gitignore +++ b/.gitignore @@ -212,6 +212,9 @@ default.profraw /corpora /corpora-merged +# Linter and formatter cache +.mypy_cache/ + # RPM build temps /distsrc.tar /selinux/tmp diff --git a/buildscripts/burn_in_tests.py b/buildscripts/burn_in_tests.py index e7ca49ce25a..67a77fc7151 100755 --- a/buildscripts/burn_in_tests.py +++ b/buildscripts/burn_in_tests.py @@ -351,7 +351,7 @@ def create_task_list_for_tests(changed_tests: Set[str], build_variant: str, if not exclude_tasks: exclude_tasks = [] - suites = get_suites(suite_files=SUITE_FILES, test_files=changed_tests) + suites = get_suites(suite_names_or_paths=SUITE_FILES, test_files=changed_tests) LOGGER.debug("Found suites to run", suites=suites) tests_by_executor = create_executor_list(suites, exclude_suites) diff --git a/buildscripts/burn_in_tests_multiversion.py b/buildscripts/burn_in_tests_multiversion.py index 0092b5c4a73..779f060835b 100644 --- a/buildscripts/burn_in_tests_multiversion.py +++ b/buildscripts/burn_in_tests_multiversion.py @@ -18,12 +18,11 @@ from buildscripts.burn_in_tests import EVERGREEN_FILE, \ from buildscripts.ciconfig.evergreen import parse_evergreen_file, EvergreenProjectConfig from buildscripts.evergreen_burn_in_tests import GenerateConfig, DEFAULT_PROJECT, EvergreenFileChangeDetector from buildscripts.task_generation.constants import CONFIG_FILE -from buildscripts.resmokelib.suitesconfig import get_named_suites_with_root_level_key +from buildscripts.resmokelib.suitesconfig import burn_in_multiversion_suites from buildscripts.task_generation.evg_config_builder import EvgConfigBuilder from buildscripts.task_generation.gen_config import GenerationConfiguration from buildscripts.task_generation.generated_config import GeneratedConfiguration from buildscripts.task_generation.multiversion_util import MultiversionUtilService -from buildscripts.task_generation.resmoke_proxy import ResmokeProxyConfig from buildscripts.task_generation.suite_split import SuiteSplitConfig, SuiteSplitParameters from buildscripts.task_generation.suite_split_strategies import SplitStrategy, greedy_division, \ FallbackStrategy, round_robin_fallback @@ -34,7 +33,6 @@ from buildscripts.util.cmdutils import enable_logging structlog.configure(logger_factory=LoggerFactory()) LOGGER = structlog.getLogger(__name__) -MULTIVERSION_CONFIG_KEY = "use_in_multiversion" MULTIVERSION_PASSTHROUGH_TAG = "multiversion_passthrough" BURN_IN_MULTIVERSION_TASK = "burn_in_tests_multiversion" DEFAULT_CONFIG_DIR = "generated_resmoke_config" @@ -130,8 +128,7 @@ class MultiversionBurnInOrchestrator: tasks = set() if tests_by_task: # Get the multiversion suites that will run in as part of burn_in_multiversion. - multiversion_suites = get_named_suites_with_root_level_key(MULTIVERSION_CONFIG_KEY) - for suite in multiversion_suites: + for suite in burn_in_multiversion_suites(): task_name = suite["origin"] if task_name not in tests_by_task.keys(): # Only generate burn in multiversion tasks for suites that would run the @@ -264,8 +261,6 @@ def main(build_variant, run_build_variant, distro, project, generate_tasks_file, binder.bind(GenTaskOptions, gen_task_options) binder.bind(EvergreenApi, evg_api) binder.bind(GenerationConfiguration, GenerationConfiguration.from_yaml_file()) - binder.bind(ResmokeProxyConfig, - ResmokeProxyConfig(resmoke_suite_dir=DEFAULT_TEST_SUITE_DIR)) binder.bind(EvergreenFileChangeDetector, EvergreenFileChangeDetector(task_id, evg_api)) binder.bind(BurnInConfig, burn_in_config) diff --git a/buildscripts/evergreen_gen_build_variant.py b/buildscripts/evergreen_gen_build_variant.py index cd383b62e87..5625e420ccc 100644 --- a/buildscripts/evergreen_gen_build_variant.py +++ b/buildscripts/evergreen_gen_build_variant.py @@ -22,7 +22,6 @@ from buildscripts.task_generation.gen_config import GenerationConfiguration from buildscripts.task_generation.gen_task_validation import GenTaskValidationService from buildscripts.task_generation.multiversion_util import MultiversionUtilService, \ SHARDED_MIXED_VERSION_CONFIGS, REPL_MIXED_VERSION_CONFIGS -from buildscripts.task_generation.resmoke_proxy import ResmokeProxyConfig from buildscripts.task_generation.suite_split import SuiteSplitConfig, SuiteSplitParameters from buildscripts.task_generation.suite_split_strategies import SplitStrategy, FallbackStrategy, \ greedy_division, round_robin_fallback @@ -453,8 +452,6 @@ def main(expansion_file: str, evg_api_config: str, evg_project_config: str, outp binder.bind(EvergreenApi, RetryingEvergreenApi.get_api(config_file=evg_api_config)) binder.bind(EvergreenProjectConfig, parse_evergreen_file(evg_project_config)) binder.bind(GenerationConfiguration, GenerationConfiguration.from_yaml_file()) - binder.bind(ResmokeProxyConfig, - ResmokeProxyConfig(resmoke_suite_dir=DEFAULT_TEST_SUITE_DIR)) inject.configure(dependencies) diff --git a/buildscripts/resmokeconfig/matrix_suites/mappings/multiversion.yml b/buildscripts/resmokeconfig/matrix_suites/mappings/multiversion.yml index cd070f88195..ee8d51b7c23 100644 --- a/buildscripts/resmokeconfig/matrix_suites/mappings/multiversion.yml +++ b/buildscripts/resmokeconfig/matrix_suites/mappings/multiversion.yml @@ -1,33 +1,41 @@ # TODO SERVER-55857: check for history conflicts in overrides outside of explicitly set keys. # TODO SERVER-55857: use the generated suite if --originSuite is used (to indicate that a generated suite exists). -# TODO SERVER-55857: ensure listSuites work with matrix suites. -- suite_name: change_streams_multiversion_passthrough +# TODO: move sharding and replica sets multiversion +# TODO: Remove use_in_multiversion used by burn in multiversion and run all tasks from here instead. +- suite_name: change_streams_multiversion base_suite: change_streams overrides: - "multiversion.replica_fixture" -- suite_name: change_streams_sharded_collections_multiversion_passthrough + - "multiversion.test_kind" +- suite_name: change_streams_sharded_collections_multiversion base_suite: change_streams_sharded_collections_passthrough overrides: - "multiversion.sharded_fixture" -- suite_name: concurrency_replication_multiversion_passthrough + - "multiversion.test_kind" +- suite_name: concurrency_replication_multiversion base_suite: concurrency_replication overrides: - "multiversion.replica_fixture" -- suite_name: multiversion_sanity_check_passthrough + - "multiversion.test_kind" +- suite_name: multiversion_sanity_check base_suite: replica_sets_jscore_passthrough overrides: - "multiversion.replica_fixture" - "multiversion.multiversion_sanity_check_selector" -- suite_name: concurrency_sharded_replication_multiversion_passthrough + - "multiversion.test_kind" +- suite_name: concurrency_sharded_replication_multiversion base_suite: concurrency_sharded_replication overrides: - "multiversion.sharded_fixture" - "multiversion.concurrency_sharded_replication_multiversion_mongod_options" -- suite_name: sharding_jscore_multiversion_passthrough + - "multiversion.test_kind" +- suite_name: sharding_jscore_multiversion base_suite: sharding_jscore_passthrough overrides: - "multiversion.sharded_fixture" -- suite_name: sharded_collections_jscore_multiversion_passthrough + - "multiversion.test_kind" +- suite_name: sharded_collections_jscore_multiversion base_suite: sharded_collections_jscore_passthrough overrides: - "multiversion.sharded_fixture" + - "multiversion.test_kind" diff --git a/buildscripts/resmokeconfig/matrix_suites/overrides/multiversion.yml b/buildscripts/resmokeconfig/matrix_suites/overrides/multiversion.yml index ab316474592..0fedfaa3c82 100644 --- a/buildscripts/resmokeconfig/matrix_suites/overrides/multiversion.yml +++ b/buildscripts/resmokeconfig/matrix_suites/overrides/multiversion.yml @@ -12,6 +12,9 @@ num_shards: 2 num_rs_nodes_per_shard: 2 +- name: test_kind + value: + test_kind: all_versions_js_test ### Suite-specific overrides ### - name: multiversion_sanity_check_selector diff --git a/buildscripts/resmokeconfig/suites/replica_sets_multiversion.yml b/buildscripts/resmokeconfig/suites/replica_sets_multiversion.yml index a83ebb1e5e4..68903a8ee0f 100644 --- a/buildscripts/resmokeconfig/suites/replica_sets_multiversion.yml +++ b/buildscripts/resmokeconfig/suites/replica_sets_multiversion.yml @@ -1,4 +1,4 @@ -test_kind: js_test +test_kind: all_versions_js_test selector: roots: diff --git a/buildscripts/resmokeconfig/suites/sharding_multiversion.yml b/buildscripts/resmokeconfig/suites/sharding_multiversion.yml index 3ae8023351f..faf96dca13b 100644 --- a/buildscripts/resmokeconfig/suites/sharding_multiversion.yml +++ b/buildscripts/resmokeconfig/suites/sharding_multiversion.yml @@ -1,4 +1,4 @@ -test_kind: js_test +test_kind: all_versions_js_test selector: roots: diff --git a/buildscripts/resmokelib/config.py b/buildscripts/resmokelib/config.py index ad50a565a00..cb14c081ee7 100644 --- a/buildscripts/resmokelib/config.py +++ b/buildscripts/resmokelib/config.py @@ -567,7 +567,6 @@ EXTERNAL_SUITE_SELECTORS = (DEFAULT_BENCHMARK_TEST_LIST, DEFAULT_UNIT_TEST_LIST, # Where to look for logging and suite configuration files CONFIG_DIR = None -NAMED_SUITES = None LOGGER_DIR = None # Generated logging config for the current invocation. diff --git a/buildscripts/resmokelib/configure_resmoke.py b/buildscripts/resmokelib/configure_resmoke.py index fc5d352a922..06fe4b479d3 100644 --- a/buildscripts/resmokelib/configure_resmoke.py +++ b/buildscripts/resmokelib/configure_resmoke.py @@ -375,8 +375,6 @@ def _update_config_vars(values): # pylint: disable=too-many-statements,too-many configure_tests(config.pop("test_files"), config.pop("replay_file")) - _config.NAMED_SUITES = SuiteFinder.get_named_suites(_config.CONFIG_DIR) - _config.LOGGER_DIR = os.path.join(_config.CONFIG_DIR, "loggers") shuffle = config.pop("shuffle") diff --git a/buildscripts/resmokelib/run/__init__.py b/buildscripts/resmokelib/run/__init__.py index 4b5882432e8..c7edacd52cb 100644 --- a/buildscripts/resmokelib/run/__init__.py +++ b/buildscripts/resmokelib/run/__init__.py @@ -10,9 +10,6 @@ import shlex import sys import time import shutil -import tempfile -import requests -import dateutil.parser import curatorbin import pkg_resources @@ -40,6 +37,7 @@ from buildscripts.resmokelib.run import generate_multiversion_exclude_tags from buildscripts.resmokelib.run import runtime_recorder from buildscripts.resmokelib.run import list_tags from buildscripts.resmokelib.run.runtime_recorder import compare_start_time +from buildscripts.resmokelib.suitesconfig import get_suite_files _INTERNAL_OPTIONS_TITLE = "Internal Options" _MONGODB_SERVER_OPTIONS_TITLE = "MongoDB Server Options" @@ -153,11 +151,16 @@ class TestRunner(Subcommand): # pylint: disable=too-many-instance-attributes suite_names) def list_tags(self): - """List the tags and its documentation available in the suites.""" + """ + List the tags and its documentation available in the suites. + + Note: this currently ignores composed/matrix suites as it's not obvious the suite + a particular tag applies to. + """ tag_docs = {} out_tag_names = [] - for suite_name in suitesconfig.get_named_suites(): - suite_file = config.NAMED_SUITES.get(suite_name, "") + for suite_name, suite_file in get_suite_files(): + # Matrix suites are ignored. tags_blocks = list_tags.get_tags_blocks(suite_file) for tags_block in tags_blocks: diff --git a/buildscripts/resmokelib/selector.py b/buildscripts/resmokelib/selector.py index 8ce2f3a8761..238d305262f 100644 --- a/buildscripts/resmokelib/selector.py +++ b/buildscripts/resmokelib/selector.py @@ -721,6 +721,7 @@ _SELECTOR_REGISTRY = { "parallel_fsm_workload_test": (_MultiJSTestSelectorConfig, _MultiJSTestSelector), "json_schema_test": (_FileBasedSelectorConfig, _Selector), "js_test": (_JSTestSelectorConfig, _JSTestSelector), + "all_versions_js_test": (_JSTestSelectorConfig, _JSTestSelector), "mql_model_haskell_test": (_FileBasedSelectorConfig, _Selector), "mql_model_mongod_test": (_FileBasedSelectorConfig, _Selector), "multi_stmt_txn_passthrough": (_JSTestSelectorConfig, _JSTestSelector), diff --git a/buildscripts/resmokelib/suitesconfig.py b/buildscripts/resmokelib/suitesconfig.py index d3b3bd47af3..b98b111928b 100644 --- a/buildscripts/resmokelib/suitesconfig.py +++ b/buildscripts/resmokelib/suitesconfig.py @@ -1,9 +1,9 @@ """Module for retrieving the configuration of resmoke.py test suites.""" - import collections -import optparse import os +from typing import List, Dict + import buildscripts.resmokelib.utils.filesystem as fs from buildscripts.resmokelib import config as _config from buildscripts.resmokelib import errors @@ -11,27 +11,41 @@ from buildscripts.resmokelib import utils from buildscripts.resmokelib.testing import suite as _suite from buildscripts.resmokelib.utils import load_yaml_file +SuiteName = str + +_NAMED_SUITES = None + + +def get_named_suites() -> List[SuiteName]: + """Return a list of the suites names.""" + global _NAMED_SUITES # pylint: disable=global-statement + + if _NAMED_SUITES is None: + # Skip "with_*server" and "no_server" because they do not define any test files to run. + executor_only = {"with_server", "with_external_server", "no_server"} + + explicit_suite_names = [ + name for name in ExplicitSuiteConfig.get_named_suites() if name not in executor_only + ] + composed_suite_names = MatrixSuiteConfig.get_named_suites() + _NAMED_SUITES = explicit_suite_names + composed_suite_names + _NAMED_SUITES.sort() + return _NAMED_SUITES -def get_named_suites(): - """Return a sorted list of the suites names.""" - # Skip "with_*server" and "no_server" because they do not define any test files to run. - executor_only = {"with_server", "with_external_server", "no_server"} - names = [name for name in _config.NAMED_SUITES.keys() if name not in executor_only] - names += MatrixSuiteConfig.get_all_suite_names() - names.sort() - return names +def get_suite_files() -> List[str]: + """Get the physical files defining these suites for parsing comments.""" + return ExplicitSuiteConfig.get_suite_files() + MatrixSuiteConfig.get_suite_files() -def get_named_suites_with_root_level_key(root_level_key): - """Return the suites that contain the given root_level_key and their values.""" - all_suite_names = get_named_suites() + +def burn_in_multiversion_suites(): + """Return the suites that should run in burn_in_multiversion.""" suites_to_return = [] - for suite in all_suite_names: - suite_config = _get_suite_config(suite) - if root_level_key in suite_config.keys() and suite_config[root_level_key]: - suites_to_return.append( - {"origin": suite, "multiversion_name": suite_config[root_level_key]}) + for suite_name in get_named_suites(): + multiversion_name = get_suite(suite_name).burn_in_multiversion_task_name() + if multiversion_name is not None: + suites_to_return.append({"origin": suite_name, "multiversion_name": multiversion_name}) return suites_to_return @@ -50,13 +64,11 @@ def create_test_membership_map(fail_on_missing_selector=False, test_kind=None): test_kind = frozenset(test_kind) test_membership = collections.defaultdict(list) - suite_names = get_named_suites() - for suite_name in suite_names: + for suite_name in get_named_suites(): try: - suite_config = _get_suite_config(suite_name) - if test_kind and suite_config.get("test_kind") not in test_kind: + suite = get_suite(suite_name) + if test_kind and suite.get_test_kind_config() not in test_kind: continue - suite = _suite.Suite(suite_name, suite_config) except IOError as err: # We ignore errors from missing files referenced in the test suite's "selector" # section. Certain test suites (e.g. unittests.yml) have a dedicated text file to @@ -74,11 +86,11 @@ def create_test_membership_map(fail_on_missing_selector=False, test_kind=None): return test_membership -def get_suites(suite_files, test_files): +def get_suites(suite_names_or_paths, test_files): """Retrieve the Suite instances based on suite configuration files and override parameters. Args: - suite_files: A list of file paths pointing to suite YAML configuration files. For the suites + suite_names_or_paths: A list of file paths pointing to suite YAML configuration files. For the suites defined in 'buildscripts/resmokeconfig/suites/' and matrix suites, a shorthand name consisting of the filename without the extension can be used. test_files: A list of file paths pointing to test files overriding the roots for the suites. @@ -94,7 +106,7 @@ def get_suites(suite_files, test_files): suite_roots = _make_suite_roots(test_files) suites = [] - for suite_filename in suite_files: + for suite_filename in suite_names_or_paths: suite_config = _get_suite_config(suite_filename) if suite_roots: # Override the suite's default test files with those passed in from the command line. @@ -104,51 +116,79 @@ def get_suites(suite_files, test_files): return suites -def get_suite(suite_file): - """Retrieve the Suite instance corresponding to a suite configuration file.""" - suite_config = _get_suite_config(suite_file) - return _suite.Suite(suite_file, suite_config) - - def _make_suite_roots(files): return {"selector": {"roots": files}} -def _get_suite_config(suite_path): +def _get_suite_config(suite_name_or_path): """Attempt to read YAML configuration from 'suite_path' for the suite.""" - return SuiteFinder.get_config_obj(suite_path) + return SuiteFinder.get_config_obj(suite_name_or_path) -class SuiteConfigInterface(object): +class SuiteConfigInterface: """Interface for suite configs.""" - def __init__(self, yaml_path=None): - """Initialize the suite config interface.""" - self.yaml_path = yaml_path + @classmethod + def get_config_obj(cls, suite_name): + """Get the config object given the suite name, which can be a path.""" + pass + + @classmethod + def get_named_suites(cls): + """Populate the named suites by scanning `config_dir`.""" + pass + + @classmethod + def get_suite_files(cls): + """Get the physical files defining these suites for parsing comments.""" + pass class ExplicitSuiteConfig(SuiteConfigInterface): """Class for storing the resmoke.py suite YAML configuration.""" - @staticmethod - def get_config_obj(pathname): + @classmethod + def get_config_obj(cls, suite_name): """Get the suite config object in the given file.""" # Named executors or suites are specified as the basename of the file, without the .yml # extension. - if not fs.is_yaml_file(pathname) and not os.path.dirname(pathname): - if pathname not in _config.NAMED_SUITES: # pylint: disable=unsupported-membership-test - # Expand 'pathname' to full path. + if not fs.is_yaml_file(suite_name) and not os.path.dirname(suite_name): + named_suites = cls.get_named_suites() + if suite_name not in named_suites: # pylint: disable=unsupported-membership-test return None - pathname = _config.NAMED_SUITES[pathname] # pylint: disable=unsubscriptable-object + suite_name = named_suites[suite_name] # pylint: disable=unsubscriptable-object - if not fs.is_yaml_file(pathname) or not os.path.isfile(pathname): - raise optparse.OptionValueError("Expected a suite YAML config, but got '%s'" % pathname) - return utils.load_yaml_file(pathname) + if not fs.is_yaml_file(suite_name) or not os.path.isfile(suite_name): + raise ValueError("Expected a suite YAML config, but got '%s'" % suite_name) + return utils.load_yaml_file(suite_name) + + @classmethod + def get_named_suites(cls) -> Dict[str, str]: + """Populate the named suites by scanning config_dir/suites.""" + named_suites = {} + + suites_dir = os.path.join(_config.CONFIG_DIR, "suites") + root = os.path.abspath(suites_dir) + files = os.listdir(root) + for filename in files: + (short_name, ext) = os.path.splitext(filename) + if ext in (".yml", ".yaml"): + pathname = os.path.join(root, filename) + named_suites[short_name] = pathname + + return named_suites + + @classmethod + def get_suite_files(cls): + """Get the suite files.""" + return cls.get_named_suites() class MatrixSuiteConfig(SuiteConfigInterface): """Class for storing the resmoke.py suite YAML configuration.""" + _all_mappings = None + @staticmethod def get_all_yamls(target_dir): """Get all YAML files in the given directory.""" @@ -162,8 +202,7 @@ class MatrixSuiteConfig(SuiteConfigInterface): pathname = os.path.join(root, filename) if not fs.is_yaml_file(pathname) or not os.path.isfile(pathname): - raise optparse.OptionValueError( - "Expected a suite YAML config, but got '%s'" % pathname) + raise ValueError("Expected a suite YAML config, but got '%s'" % pathname) all_files[short_name] = load_yaml_file(pathname) return all_files @@ -172,10 +211,10 @@ class MatrixSuiteConfig(SuiteConfigInterface): return os.path.join(_config.CONFIG_DIR, "matrix_suites") @classmethod - def get_config_obj(cls, suite_path): + def get_config_obj(cls, suite_name): """Get the suite config object in the given file.""" suites_dir = cls._get_suites_dir() - matrix_suite = cls.parse_mappings_file(suites_dir, suite_path) + matrix_suite = cls.parse_mappings_file(suites_dir, suite_name) if not matrix_suite: return None @@ -228,32 +267,40 @@ class MatrixSuiteConfig(SuiteConfigInterface): """Get the mapping object for a given suite name and directory to search for suite mappings.""" all_matrix_suites = cls.get_all_mappings(suites_dir) - if suite_name in all_matrix_suites: - return all_matrix_suites[suite_name] + if suite_name in all_matrix_suites.keys(): + return all_matrix_suites[suite_name] # pylint: disable=unsubscriptable-object return None @classmethod - def get_all_suite_names(cls): + def get_named_suites(cls): """Get a list of all suite names.""" suites_dir = cls._get_suites_dir() all_mappings = cls.get_all_mappings(suites_dir) - return all_mappings.keys() + return list(all_mappings.keys()) + + @classmethod + def get_suite_files(cls): + """Get the physical files defining these suites for parsing comments.""" + mappings_dir = os.path.join(cls._get_suites_dir(), "mappings") + return cls.get_all_yamls(mappings_dir) @classmethod - def get_all_mappings(cls, suites_dir): + def get_all_mappings(cls, suites_dir) -> Dict[str, str]: """Get a dictionary of all suite mapping files keyed by the suite name.""" - mappings_dir = os.path.join(suites_dir, "mappings") - mappings_files = cls.get_all_yamls(mappings_dir) - - all_matrix_suites = {} - for _, suite_config_file in mappings_files.items(): - for suite_config in suite_config_file: - if "suite_name" in suite_config and "base_suite" in suite_config: - all_matrix_suites[suite_config["suite_name"]] = suite_config - else: - raise ValueError("Invalid suite configuration, missing required keys. ", - suite_config) - return all_matrix_suites + if cls._all_mappings is None: + mappings_dir = os.path.join(suites_dir, "mappings") + mappings_files = cls.get_all_yamls(mappings_dir) + + all_mappings = {} + for _, suite_config_file in mappings_files.items(): + for suite_config in suite_config_file: + if "suite_name" in suite_config and "base_suite" in suite_config: + all_mappings[suite_config["suite_name"]] = suite_config + else: + raise ValueError("Invalid suite configuration, missing required keys. ", + suite_config) + cls._all_mappings = all_mappings + return cls._all_mappings @classmethod def merge_dicts(cls, dict1, dict2): @@ -288,19 +335,8 @@ class SuiteFinder(object): return matrix_suite or explicit_suite - @staticmethod - def get_named_suites(config_dir): - """Populate the named suites by scanning config_dir/suites.""" - named_suites = {} - - suites_dir = os.path.join(config_dir, "suites") - root = os.path.abspath(suites_dir) - files = os.listdir(root) - for filename in files: - (short_name, ext) = os.path.splitext(filename) - if ext in (".yml", ".yaml"): - pathname = os.path.join(root, filename) - # TODO: store named suite in an object - named_suites[short_name] = pathname - return named_suites +def get_suite(suite_name_or_path) -> _suite.Suite: + """Retrieve the Suite instance corresponding to a suite configuration file.""" + suite_config = _get_suite_config(suite_name_or_path) + return _suite.Suite(suite_name_or_path, suite_config) diff --git a/buildscripts/resmokelib/testing/suite.py b/buildscripts/resmokelib/testing/suite.py index e02dcde1ae3..dc53d3785cb 100644 --- a/buildscripts/resmokelib/testing/suite.py +++ b/buildscripts/resmokelib/testing/suite.py @@ -36,6 +36,8 @@ EXIT_CODE_MAP = { 3221225725: "Stack Overflow", } +MULTIVERSION_CONFIG_KEY = "use_in_multiversion" + def translate_exit_code(exit_code): """ @@ -71,7 +73,8 @@ class Suite(object): # pylint: disable=too-many-instance-attributes self._suite_options = suite_options self.test_kind = self.get_test_kind_config() - self.tests, self.excluded = self._get_tests_for_kind(self.test_kind) + self._tests = None + self._excluded = None self.return_code = None # Set by the executor. @@ -90,6 +93,20 @@ class Suite(object): # pylint: disable=too-many-instance-attributes """Create a string representation of object for debugging.""" return f"{self.test_kind}:{self._suite_name}" + @property + def tests(self): + """Get the tests.""" + if self._tests is None: + self._tests, self._excluded = self._get_tests_for_kind(self.test_kind) + return self._tests + + @property + def excluded(self): + """Get the excluded.""" + if self._excluded is None: + self._tests, self._excluded = self._get_tests_for_kind(self.test_kind) + return self._excluded + def _get_tests_for_kind(self, test_kind): """Return the tests to run based on the 'test_kind'-specific filtering policy.""" selector_config = self.get_selector_config() @@ -151,6 +168,10 @@ class Suite(object): # pylint: disable=too-many-instance-attributes """Return the "test_kind" section of the YAML configuration.""" return self._suite_config["test_kind"] + def burn_in_multiversion_task_name(self): + """Check if the suite should run in burn_in_multiversion by using a special root-level key.""" + return self._suite_config.get(MULTIVERSION_CONFIG_KEY) + @property def options(self): """Get the options.""" diff --git a/buildscripts/resmokelib/testing/testcases/jstest.py b/buildscripts/resmokelib/testing/testcases/jstest.py index 2edd7c133dd..ccd5b4db5cd 100644 --- a/buildscripts/resmokelib/testing/testcases/jstest.py +++ b/buildscripts/resmokelib/testing/testcases/jstest.py @@ -275,3 +275,14 @@ class JSTestCase(interface.TestCase, interface.UndoDBUtilsMixin): f"Mongo shell exited with code {return_code} while running jstest {self.basename()}." " Further test execution may be unsafe.") raise self.propagate_error # pylint: disable=raising-bad-type + + +class AllVersionsJSTestCase(JSTestCase): + """ + Alias for JSTestCase for multiversion passthrough suites. + + It run with all combinations of versions of replica sets and sharded clusters. + The distinct name is picked up by task generation. + """ + + REGISTERED_NAME = "all_versions_js_test" diff --git a/buildscripts/selected_tests.py b/buildscripts/selected_tests.py index a535705cb98..a49ba2494c7 100644 --- a/buildscripts/selected_tests.py +++ b/buildscripts/selected_tests.py @@ -26,7 +26,6 @@ from buildscripts.patch_builds.selected_tests.selected_tests_client import Selec from buildscripts.task_generation.evg_config_builder import EvgConfigBuilder from buildscripts.task_generation.gen_config import GenerationConfiguration from buildscripts.task_generation.generated_config import GeneratedConfiguration -from buildscripts.task_generation.resmoke_proxy import ResmokeProxyConfig from buildscripts.task_generation.suite_split import SuiteSplitParameters, SuiteSplitConfig from buildscripts.task_generation.suite_split_strategies import SplitStrategy, FallbackStrategy, \ greedy_division, round_robin_fallback @@ -503,8 +502,6 @@ def main( binder.bind(FallbackStrategy, round_robin_fallback) binder.bind(GenTaskOptions, evg_expansions.build_gen_task_options()) binder.bind(GenerationConfiguration, GenerationConfiguration.from_yaml_file()) - binder.bind(ResmokeProxyConfig, - ResmokeProxyConfig(resmoke_suite_dir=DEFAULT_TEST_SUITE_DIR)) inject.configure(dependencies) diff --git a/buildscripts/task_generation/resmoke_proxy.py b/buildscripts/task_generation/resmoke_proxy.py index 487dd2a05cd..820e63fb284 100644 --- a/buildscripts/task_generation/resmoke_proxy.py +++ b/buildscripts/task_generation/resmoke_proxy.py @@ -8,9 +8,8 @@ import structlog import yaml import buildscripts.resmokelib.parser as _parser -import buildscripts.resmokelib.suitesconfig as suitesconfig +import buildscripts.resmokelib.suitesconfig as _suiteconfig from buildscripts.task_generation.generated_config import GeneratedFile -from buildscripts.util.fileops import read_yaml_file if TYPE_CHECKING: from buildscripts.task_generation.suite_split import GeneratedSuite, SubSuite @@ -23,29 +22,14 @@ HEADER_TEMPLATE = """# DO NOT EDIT THIS FILE. All manual edits will be lost. """ -class ResmokeProxyConfig(NamedTuple): - """ - Configuration for resmoke proxy. - - resmoke_suite_dir: Directory that contains resmoke suite configurations. - """ - - resmoke_suite_dir: str - - class ResmokeProxyService: """A service to proxy requests to resmoke.""" @inject.autoparams() - def __init__(self, proxy_config: ResmokeProxyConfig) -> None: - """ - Initialize the service. - - :param proxy_config: Configuration for the proxy. - """ + def __init__(self) -> None: + """Initialize the service.""" _parser.set_run_options() - self.suitesconfig = suitesconfig - self.resmoke_suite_dir = proxy_config.resmoke_suite_dir + self._suite_config = _suiteconfig def list_tests(self, suite_name: str) -> List[str]: """ @@ -54,9 +38,9 @@ class ResmokeProxyService: :param suite_name: Name of suite to query. :return: List of test names that belong to the suite. """ - suite_config = self.suitesconfig.get_suite(suite_name) + suite = self._suite_config.get_suite(suite_name) test_list = [] - for tests in suite_config.tests: + for tests in suite.tests: # `tests` could return individual tests or lists of tests, we need to handle both. if isinstance(tests, list): test_list.extend(tests) @@ -72,7 +56,7 @@ class ResmokeProxyService: :param suite_name: Name of suite to read. :return: Configuration of specified suite. """ - return read_yaml_file(os.path.join(self.resmoke_suite_dir, f"{suite_name}.yml")) + return self._suite_config.SuiteFinder.get_config_obj(suite_name) def render_suite_files(self, suites: List["SubSuite"], suite_name: str, generated_suite_filename: str, test_list: List[str], @@ -92,7 +76,7 @@ class ResmokeProxyService: :return: Dictionary of rendered resmoke config files. """ # pylint: disable=too-many-arguments - source_config = self.read_suite_config(suite_name) + source_config = self._suite_config.SuiteFinder.get_config_obj(suite_name) suite_configs = [ GeneratedFile(file_name=f"{suite.sub_suite_config_file(i)}.yml", content=sub_suite.generate_resmoke_config(source_config)) diff --git a/buildscripts/task_generation/task_types/multiversion_tasks.py b/buildscripts/task_generation/task_types/multiversion_tasks.py index a783cb743ea..0cb0fd17742 100644 --- a/buildscripts/task_generation/task_types/multiversion_tasks.py +++ b/buildscripts/task_generation/task_types/multiversion_tasks.py @@ -129,6 +129,7 @@ class MultiversionGenTaskService: :param params: Parameters for how tasks should be generated. :return: Arguments to pass to resmoke to run the generated task. """ + tag_file_location = self.gen_task_options.generated_file_location(EXCLUDE_TAGS_FILE) return ( diff --git a/buildscripts/tests/resmokelib/test_suitesconfig.py b/buildscripts/tests/resmokelib/test_suitesconfig.py index 31f65e40612..44d01c31ec1 100644 --- a/buildscripts/tests/resmokelib/test_suitesconfig.py +++ b/buildscripts/tests/resmokelib/test_suitesconfig.py @@ -23,31 +23,29 @@ class TestSuitesConfig(unittest.TestCase): membership_map = suitesconfig.create_test_membership_map(test_kind="nonexistent_test") self.assertEqual(membership_map, {}) - self.assertEqual(mock_suite_class.call_count, 0) + self.assertEqual(mock_suite_class.call_count, 2) - @mock.patch(RESMOKELIB + ".testing.suite.Suite") + @mock.patch(RESMOKELIB + ".testing.suite.Suite.tests") @mock.patch(RESMOKELIB + ".suitesconfig.get_named_suites") def test_multiple_suites_matching_single_test_kind(self, mock_get_named_suites, - mock_suite_class): + mock_suite_get_tests): all_suites = ["core", "replica_sets_jscore_passthrough"] mock_get_named_suites.return_value = all_suites - mock_suite_class.return_value.tests = ["test1", "test2"] + mock_suite_get_tests.__get__ = mock.Mock(return_value=["test1", "test2"]) membership_map = suitesconfig.create_test_membership_map(test_kind="js_test") self.assertEqual(membership_map, dict(test1=all_suites, test2=all_suites)) - self.assertEqual(mock_suite_class.call_count, 2) - @mock.patch(RESMOKELIB + ".testing.suite.Suite") + @mock.patch(RESMOKELIB + ".testing.suite.Suite.tests") @mock.patch(RESMOKELIB + ".suitesconfig.get_named_suites") def test_multiple_suites_matching_multiple_test_kinds(self, mock_get_named_suites, - mock_suite_class): + mock_suite_get_tests): all_suites = ["core", "concurrency"] mock_get_named_suites.return_value = all_suites - mock_suite_class.return_value.tests = ["test1", "test2"] + mock_suite_get_tests.__get__ = mock.Mock(return_value=["test1", "test2"]) membership_map = suitesconfig.create_test_membership_map( test_kind=("fsm_workload_test", "js_test")) self.assertEqual(membership_map, dict(test1=all_suites, test2=all_suites)) - self.assertEqual(mock_suite_class.call_count, 2) diff --git a/buildscripts/tests/task_generation/test_resmoke_proxy.py b/buildscripts/tests/task_generation/test_resmoke_proxy.py index 4752cf4659e..d2f6ba61572 100644 --- a/buildscripts/tests/task_generation/test_resmoke_proxy.py +++ b/buildscripts/tests/task_generation/test_resmoke_proxy.py @@ -12,9 +12,9 @@ class TestResmokeProxy(unittest.TestCase): mock_suite = MagicMock( tests=["test0", "test1", ["test2a", "tests2b", "test2c"], "test3", ["test4a"]]) - resmoke_proxy = under_test.ResmokeProxyService(under_test.ResmokeProxyConfig("suites_dir")) - resmoke_proxy.suitesconfig = MagicMock() - resmoke_proxy.suitesconfig.get_suite.return_value = mock_suite + resmoke_proxy = under_test.ResmokeProxyService() + resmoke_proxy._suite_config = MagicMock() + resmoke_proxy._suite_config.get_suite.return_value = mock_suite test_list = resmoke_proxy.list_tests("some suite") diff --git a/buildscripts/tests/test_burn_in_tests.py b/buildscripts/tests/test_burn_in_tests.py index 009749f5fc3..b8ea84a7d43 100644 --- a/buildscripts/tests/test_burn_in_tests.py +++ b/buildscripts/tests/test_burn_in_tests.py @@ -311,7 +311,7 @@ class CreateExecutorList(unittest.TestCase): mock_get_named_suites.return_value = ["dbtest"] under_test.create_executor_list([], []) - self.assertEqual(mock_suite_class.call_count, 0) + self.assertEqual(mock_suite_class.call_count, 1) def create_variant_task_mock(task_name, suite_name, distro="distro"): diff --git a/buildscripts/tests/test_burn_in_tests_multiversion.py b/buildscripts/tests/test_burn_in_tests_multiversion.py index 9e1ac0d2e0b..eb7ec81aa89 100644 --- a/buildscripts/tests/test_burn_in_tests_multiversion.py +++ b/buildscripts/tests/test_burn_in_tests_multiversion.py @@ -20,7 +20,6 @@ from buildscripts.evergreen_burn_in_tests import EvergreenFileChangeDetector from buildscripts.task_generation.gen_config import GenerationConfiguration from buildscripts.task_generation.multiversion_util import REPL_MIXED_VERSION_CONFIGS, \ SHARDED_MIXED_VERSION_CONFIGS -from buildscripts.task_generation.resmoke_proxy import ResmokeProxyConfig from buildscripts.task_generation.suite_split import SuiteSplitConfig from buildscripts.task_generation.suite_split_strategies import greedy_division, SplitStrategy, \ FallbackStrategy, round_robin_fallback @@ -165,8 +164,6 @@ def configure_dependencies(evg_api, split_config): binder.bind(GenTaskOptions, gen_task_options) binder.bind(EvergreenApi, evg_api) binder.bind(GenerationConfiguration, GenerationConfiguration.from_yaml_file()) - binder.bind(ResmokeProxyConfig, - ResmokeProxyConfig(resmoke_suite_dir=under_test.DEFAULT_TEST_SUITE_DIR)) binder.bind(EvergreenFileChangeDetector, None) binder.bind(EvergreenProjectConfig, MagicMock()) binder.bind( diff --git a/buildscripts/tests/test_evergreen_burn_in_tests.py b/buildscripts/tests/test_evergreen_burn_in_tests.py index 87624813cab..f2dabd0b295 100644 --- a/buildscripts/tests/test_evergreen_burn_in_tests.py +++ b/buildscripts/tests/test_evergreen_burn_in_tests.py @@ -98,7 +98,6 @@ class TestAcceptance(unittest.TestCase): # introduce failures and require this test to be updated. # You can see the test file it is using below. This test is used in the 'auth' and # 'auth_audit' test suites. It needs to be in at least one of those for the test to pass. - _config.NAMED_SUITES = None variant = "enterprise-rhel-80-64-bit" repos = [mock_changed_git_files(["jstests/auth/auth1.js"])] repeat_config = under_test.RepeatConfig() diff --git a/buildscripts/tests/test_selected_tests.py b/buildscripts/tests/test_selected_tests.py index 377ee59bc6b..8bd5f17ebb4 100644 --- a/buildscripts/tests/test_selected_tests.py +++ b/buildscripts/tests/test_selected_tests.py @@ -17,7 +17,6 @@ from buildscripts.patch_builds.selected_tests.selected_tests_client import Selec TaskMapping from buildscripts.selected_tests import EvgExpansions from buildscripts.task_generation.gen_config import GenerationConfiguration -from buildscripts.task_generation.resmoke_proxy import ResmokeProxyConfig from buildscripts.task_generation.suite_split import SuiteSplitConfig from buildscripts.task_generation.suite_split_strategies import SplitStrategy, greedy_division, \ FallbackStrategy, round_robin_fallback @@ -59,7 +58,6 @@ def configure_dependencies(evg_api, evg_expansions, evg_project_config, selected binder.bind(GenTaskOptions, evg_expansions.build_gen_task_options()) binder.bind(EvergreenApi, evg_api) binder.bind(GenerationConfiguration, GenerationConfiguration.from_yaml_file()) - binder.bind(ResmokeProxyConfig, ResmokeProxyConfig(resmoke_suite_dir=test_suites_dir)) binder.bind(SelectedTestsClient, selected_test_client) inject.clear_and_configure(dependencies) diff --git a/etc/evergreen.yml b/etc/evergreen.yml index d706fbd9de4..0f0b22597f6 100644 --- a/etc/evergreen.yml +++ b/etc/evergreen.yml @@ -5800,7 +5800,6 @@ tasks: suite: retryable_writes_downgrade_passthrough resmoke_args: --storageEngine=wiredTiger require_multiversion: true - implicit_multiversion: true - <<: *gen_task_template name: sharded_retryable_writes_downgrade_passthrough_gen diff --git a/evergreen/implicit_multiversions_tasks_generate.sh b/evergreen/implicit_multiversions_tasks_generate.sh index da452d69ab0..f39e003604c 100644 --- a/evergreen/implicit_multiversions_tasks_generate.sh +++ b/evergreen/implicit_multiversions_tasks_generate.sh @@ -8,6 +8,7 @@ set -o errexit activate_venv PATH="$PATH:/data/multiversion" +# TODO: remove this file and generate tags at execution time for suites defined in multiversion/ if [ -n "${require_multiversion}" ]; then $python buildscripts/resmoke.py generate-multiversion-exclude-tags --oldBinVersion=last_continuous fi |