diff options
Diffstat (limited to 'buildscripts')
| -rw-r--r-- | buildscripts/burn_in_tests_multiversion.py | 18 | ||||
| -rw-r--r-- | buildscripts/resmokeconfig/matrix_suites/overrides/multiversion.yml | 3 | ||||
| -rw-r--r-- | buildscripts/resmokeconfig/setup_multiversion/setup_multiversion_config.yml | 1 | ||||
| -rw-r--r-- | buildscripts/resmokelib/config.py | 2 | ||||
| -rw-r--r-- | buildscripts/resmokelib/configure_resmoke.py | 15 | ||||
| -rw-r--r-- | buildscripts/resmokelib/errors.py | 5 | ||||
| -rw-r--r-- | buildscripts/resmokelib/run/__init__.py | 2 | ||||
| -rw-r--r-- | buildscripts/resmokelib/suitesconfig.py | 206 |
8 files changed, 201 insertions, 51 deletions
diff --git a/buildscripts/burn_in_tests_multiversion.py b/buildscripts/burn_in_tests_multiversion.py index 5713c0309df..59282912aed 100644 --- a/buildscripts/burn_in_tests_multiversion.py +++ b/buildscripts/burn_in_tests_multiversion.py @@ -84,23 +84,6 @@ class MultiversionBurnInOrchestrator: self.multiversion_util = multiversion_util self.burn_in_config = burn_in_config - def validate_multiversion_tasks_and_suites(self) -> None: - """ - Validate that the multiversion suites and tasks match up. - - We expect the number of suites with MULTIVERSION_PASSTHROUGH_TAG to be the same as in - multiversion_suites. Multiversion passthrough suites must include - MULTIVERSION_CONFIG_KEY as a root level key and must be set to true. - - Throws an exception if there are inconsistencies. - """ - multiversion_tasks = self.evg_config.get_task_names_by_tag(MULTIVERSION_PASSTHROUGH_TAG) - LOGGER.debug("Multiversion tasks by tag", tasks=multiversion_tasks, - tag=MULTIVERSION_PASSTHROUGH_TAG) - - multiversion_suites = get_named_suites_with_root_level_key(MULTIVERSION_CONFIG_KEY) - assert len(multiversion_tasks) == len(multiversion_suites) - def generate_tests(self, repos: List[Repo], generate_config: GenerateConfig, target_file: str) -> None: """ @@ -289,7 +272,6 @@ def main(build_variant, run_build_variant, distro, project, generate_tasks_file, inject.configure(dependencies) burn_in_orchestrator = MultiversionBurnInOrchestrator() # pylint: disable=no-value-for-parameter - burn_in_orchestrator.validate_multiversion_tasks_and_suites() burn_in_orchestrator.generate_tests(repos, generate_config, generate_tasks_file) diff --git a/buildscripts/resmokeconfig/matrix_suites/overrides/multiversion.yml b/buildscripts/resmokeconfig/matrix_suites/overrides/multiversion.yml index c4b7d3af084..e24d35650a6 100644 --- a/buildscripts/resmokeconfig/matrix_suites/overrides/multiversion.yml +++ b/buildscripts/resmokeconfig/matrix_suites/overrides/multiversion.yml @@ -3,7 +3,8 @@ value: executor: fixture: - num_nodes: 2 + num_nodes: 3 + - name: sharded_fixture value: executor: diff --git a/buildscripts/resmokeconfig/setup_multiversion/setup_multiversion_config.yml b/buildscripts/resmokeconfig/setup_multiversion/setup_multiversion_config.yml index 8738ffef191..bbb3d30710b 100644 --- a/buildscripts/resmokeconfig/setup_multiversion/setup_multiversion_config.yml +++ b/buildscripts/resmokeconfig/setup_multiversion/setup_multiversion_config.yml @@ -7,6 +7,7 @@ evergreen_projects: - mongodb-mongo-master + - mongodb-mongo-v5.0 - mongodb-mongo-v4.9 - mongodb-mongo-v4.8 - mongodb-mongo-v4.7 diff --git a/buildscripts/resmokelib/config.py b/buildscripts/resmokelib/config.py index 2acc069ec25..4dcaca7061a 100644 --- a/buildscripts/resmokelib/config.py +++ b/buildscripts/resmokelib/config.py @@ -244,7 +244,7 @@ class MultiversionOptions(object): """Represent the multiversion version choices.""" LAST_LTS = "last_lts" - LAST_CONTINOUS = "last_continous" + LAST_CONTINOUS = "last_continuous" @classmethod def all_options(cls): diff --git a/buildscripts/resmokelib/configure_resmoke.py b/buildscripts/resmokelib/configure_resmoke.py index cec902f79c0..a338157676b 100644 --- a/buildscripts/resmokelib/configure_resmoke.py +++ b/buildscripts/resmokelib/configure_resmoke.py @@ -17,6 +17,7 @@ from buildscripts.idl.lib import ALL_FEATURE_FLAG_FILE from buildscripts.resmokelib import config as _config from buildscripts.resmokelib import utils from buildscripts.resmokelib import mongod_fuzzer_configs +from buildscripts.resmokelib.suitesconfig import SuiteFinder def validate_and_update_config(parser, args): @@ -371,9 +372,6 @@ def _update_config_vars(values): # pylint: disable=too-many-statements,too-many _config.UNDO_RECORDER_PATH = config.pop("undo_recorder_path") - # Populate the named suites by scanning config_dir/suites - named_suites = {} - def configure_tests(test_files, replay_file): # `_validate_options` has asserted that at most one of `test_files` and `replay_file` contains input. @@ -393,16 +391,7 @@ def _update_config_vars(values): # pylint: disable=too-many-statements,too-many configure_tests(config.pop("test_files"), config.pop("replay_file")) - suites_dir = os.path.join(_config.CONFIG_DIR, "suites") - root = os.path.abspath(suites_dir) - files = os.listdir(root) - for filename in files: - (short_name, ext) = os.path.splitext(filename) - if ext in (".yml", ".yaml"): - pathname = os.path.join(root, filename) - named_suites[short_name] = pathname - - _config.NAMED_SUITES = named_suites + _config.NAMED_SUITES = SuiteFinder.get_named_suites(_config.CONFIG_DIR) _config.LOGGER_DIR = os.path.join(_config.CONFIG_DIR, "loggers") diff --git a/buildscripts/resmokelib/errors.py b/buildscripts/resmokelib/errors.py index 0011b2b43c9..6f8f0172de9 100644 --- a/buildscripts/resmokelib/errors.py +++ b/buildscripts/resmokelib/errors.py @@ -11,6 +11,11 @@ class SuiteNotFound(ResmokeError): # noqa: D204 pass +class DuplicateSuiteDefinition(ResmokeError): # noqa: D204 + """A suite name with multiple definitions.""" + pass + + class StopExecution(ResmokeError): # noqa: D204 """Exception raised when resmoke.py should stop executing tests if failing fast is enabled.""" pass diff --git a/buildscripts/resmokelib/run/__init__.py b/buildscripts/resmokelib/run/__init__.py index 9e63fee9ea3..ab624e9027b 100644 --- a/buildscripts/resmokelib/run/__init__.py +++ b/buildscripts/resmokelib/run/__init__.py @@ -765,7 +765,7 @@ class RunPlugin(PluginInterface): parser.add_argument( "--multiversionBinVersion", type=str, dest="multiversion_bin_version", choices=config.MultiversionOptions.all_options(), - help="Chose the multiverion binary version as last-lts or last-continous.") + help="Chose the multiverion binary version as last-lts or last-continuous.") parser.add_argument( "--linearChain", action="store", dest="linear_chain", choices=("on", "off"), diff --git a/buildscripts/resmokelib/suitesconfig.py b/buildscripts/resmokelib/suitesconfig.py index 1e5c5e52adf..b79cb4bafca 100644 --- a/buildscripts/resmokelib/suitesconfig.py +++ b/buildscripts/resmokelib/suitesconfig.py @@ -8,6 +8,7 @@ from buildscripts.resmokelib import config as _config from buildscripts.resmokelib import errors from buildscripts.resmokelib import utils from buildscripts.resmokelib.testing import suite as _suite +from buildscripts.resmokelib.utils import load_yaml_file def get_named_suites(): @@ -15,6 +16,7 @@ def get_named_suites(): # Skip "with_*server" and "no_server" because they do not define any test files to run. executor_only = {"with_server", "with_external_server", "no_server"} names = [name for name in _config.NAMED_SUITES.keys() if name not in executor_only] + names += MatrixSuiteConfig.get_all_suite_names() names.sort() return names @@ -76,8 +78,8 @@ def get_suites(suite_files, test_files): Args: suite_files: A list of file paths pointing to suite YAML configuration files. For the suites - defined in 'buildscripts/resmokeconfig/suites/', a shorthand name consisting of the - filename without the extension can be used. + defined in 'buildscripts/resmokeconfig/suites/' and matrix suites, a shorthand name consisting + of the filename without the extension can be used. test_files: A list of file paths pointing to test files overriding the roots for the suites. """ suite_roots = None @@ -111,21 +113,191 @@ def _make_suite_roots(files): return {"selector": {"roots": files}} -def _get_suite_config(pathname): - """Attempt to read YAML configuration from 'pathname' for the suite.""" - return _get_yaml_config("suite", pathname) +def _get_suite_config(suite_path): + """Attempt to read YAML configuration from 'suite_path' for the suite.""" + return SuiteFinder.get_config_obj(suite_path) -def _get_yaml_config(kind, pathname): - # Named executors or suites are specified as the basename of the file, without the .yml - # extension. - if not utils.is_yaml_file(pathname) and not os.path.dirname(pathname): - if pathname not in _config.NAMED_SUITES: # pylint: disable=unsupported-membership-test - raise errors.SuiteNotFound("Unknown %s '%s'" % (kind, pathname)) - # Expand 'pathname' to full path. - pathname = _config.NAMED_SUITES[pathname] # pylint: disable=unsubscriptable-object +class SuiteConfigInterface(object): + """Interface for suite configs.""" - if not utils.is_yaml_file(pathname) or not os.path.isfile(pathname): - raise optparse.OptionValueError( - "Expected a %s YAML config, but got '%s'" % (kind, pathname)) - return utils.load_yaml_file(pathname) + def __init__(self, yaml_path=None): + """Initialize the suite config interface.""" + self.yaml_path = yaml_path + + +class ExplicitSuiteConfig(SuiteConfigInterface): + """Class for storing the resmoke.py suite YAML configuration.""" + + @staticmethod + def get_config_obj(pathname): + """Get the suite config object in the given file.""" + # Named executors or suites are specified as the basename of the file, without the .yml + # extension. + if not utils.is_yaml_file(pathname) and not os.path.dirname(pathname): + if pathname not in _config.NAMED_SUITES: # pylint: disable=unsupported-membership-test + # Expand 'pathname' to full path. + return None + pathname = _config.NAMED_SUITES[pathname] # pylint: disable=unsubscriptable-object + + if not utils.is_yaml_file(pathname) or not os.path.isfile(pathname): + raise optparse.OptionValueError("Expected a suite YAML config, but got '%s'" % pathname) + return utils.load_yaml_file(pathname) + + +class MatrixSuiteConfig(SuiteConfigInterface): + """Class for storing the resmoke.py suite YAML configuration.""" + + @staticmethod + def get_all_yamls(target_dir): + """Get all YAML files in the given directory.""" + all_files = {} + root = os.path.abspath(target_dir) + files = os.listdir(root) + + for filename in files: + (short_name, ext) = os.path.splitext(filename) + if ext in (".yml", ".yaml"): + pathname = os.path.join(root, filename) + + if not utils.is_yaml_file(pathname) or not os.path.isfile(pathname): + raise optparse.OptionValueError( + "Expected a suite YAML config, but got '%s'" % pathname) + all_files[short_name] = load_yaml_file(pathname) + return all_files + + @staticmethod + def _get_suites_dir(): + return os.path.join(_config.CONFIG_DIR, "matrix_suites") + + @classmethod + def get_config_obj(cls, suite_path): + """Get the suite config object in the given file.""" + suites_dir = cls._get_suites_dir() + matrix_suite = cls.parse_mappings_file(suites_dir, suite_path) + if not matrix_suite: + return None + + all_overrides = cls.parse_override_file(suites_dir) + + return cls.process_overrides(matrix_suite, all_overrides) + + @classmethod + def process_overrides(cls, suite, overrides): + """Provide override key-value pairs for a given matrix suite.""" + base_suite_name = suite["base_suite"] + suite_name = suite["suite_name"] + override_names = suite.get("overrides", None) + + base_suite = ExplicitSuiteConfig.get_config_obj(base_suite_name) + + if base_suite is None: + # pylint: disable=too-many-format-args + raise ValueError("Unknown base suite %s for matrix suite %s".format( + base_suite_name, suite_name)) + + res = base_suite.copy() + + if override_names: + for override_name in override_names: + cls.merge_dicts(res, overrides[override_name]) + + return res + + @classmethod + def parse_override_file(cls, suites_dir): + """Get a dictionary of all overrides in a given directory keyed by the suite name.""" + overrides_dir = os.path.join(suites_dir, "overrides") + overrides_files = cls.get_all_yamls(overrides_dir) + + all_overrides = {} + for filename, override_config_file in overrides_files.items(): + for override_config in override_config_file: + if "name" in override_config and "value" in override_config: + all_overrides[f"{filename}.{override_config['name']}"] = override_config[ + "value"] + else: + raise ValueError("Invalid override configuration, missing required keys. ", + override_config) + + return all_overrides + + @classmethod + def parse_mappings_file(cls, suites_dir, suite_name): + """Get the mapping object for a given suite name and directory to search for suite mappings.""" + all_matrix_suites = cls.get_all_mappings(suites_dir) + + if suite_name in all_matrix_suites: + return all_matrix_suites[suite_name] + return None + + @classmethod + def get_all_suite_names(cls): + """Get a list of all suite names.""" + suites_dir = cls._get_suites_dir() + all_mappings = cls.get_all_mappings(suites_dir) + return all_mappings.keys() + + @classmethod + def get_all_mappings(cls, suites_dir): + """Get a dictionary of all suite mapping files keyed by the suite name.""" + mappings_dir = os.path.join(suites_dir, "mappings") + mappings_files = cls.get_all_yamls(mappings_dir) + + all_matrix_suites = {} + for _, suite_config_file in mappings_files.items(): + for suite_config in suite_config_file: + if "suite_name" in suite_config and "base_suite" in suite_config: + all_matrix_suites[suite_config["suite_name"]] = suite_config + else: + raise ValueError("Invalid suite configuration, missing required keys. ", + suite_config) + return all_matrix_suites + + @classmethod + def merge_dicts(cls, dict1, dict2): + """Recursively merges dict2 into dict1.""" + if not isinstance(dict1, dict) or not isinstance(dict2, dict): + return dict2 + for k in dict2: + if k in dict1: + dict1[k] = cls.merge_dicts(dict1[k], dict2[k]) + else: + dict1[k] = dict2[k] + return dict1 + + +class SuiteFinder(object): + """Utility/Factory class for getting polymorphic suite classes given a directory.""" + + @staticmethod + def get_config_obj(suite_path): + """Get the suite config object in the given file.""" + explicit_suite = ExplicitSuiteConfig.get_config_obj(suite_path) + matrix_suite = MatrixSuiteConfig.get_config_obj(suite_path) + + if not (explicit_suite or matrix_suite): + raise errors.SuiteNotFound("Unknown suite 's'" % suite_path) + + if explicit_suite and matrix_suite: + raise errors.DuplicateSuiteDefinition( + "Multiple definitions for suite '%s'" % suite_path) + + return matrix_suite or explicit_suite + + @staticmethod + def get_named_suites(config_dir): + """Populate the named suites by scanning config_dir/suites.""" + named_suites = {} + + suites_dir = os.path.join(config_dir, "suites") + root = os.path.abspath(suites_dir) + files = os.listdir(root) + for filename in files: + (short_name, ext) = os.path.splitext(filename) + if ext in (".yml", ".yaml"): + pathname = os.path.join(root, filename) + # TODO: store named suite in an object + named_suites[short_name] = pathname + + return named_suites |