diff options
10 files changed, 535 insertions, 196 deletions
diff --git a/.gitignore b/.gitignore index a1564ef2888..efb56426b11 100644 --- a/.gitignore +++ b/.gitignore @@ -169,4 +169,3 @@ src/mongo/embedded/java/jar/build/ local.properties compile_commands.json -generated_resmoke_config diff --git a/buildscripts/generate_resmoke_suites.py b/buildscripts/generate_resmoke_suites.py index 14ae68b04a5..6268de9f258 100755 --- a/buildscripts/generate_resmoke_suites.py +++ b/buildscripts/generate_resmoke_suites.py @@ -10,35 +10,27 @@ from __future__ import absolute_import import argparse import datetime -import glob import itertools import logging -import math import os import sys -import yaml from collections import defaultdict from collections import namedtuple from operator import itemgetter +from jinja2 import Template + from client.github import GithubApi import client.evergreen as evergreen import util.testname as testname import util.time as timeutil -from evergreen_gen_fuzzer_tests import CONFIG_DIRECTORY LOGGER = logging.getLogger(__name__) +TEMPLATES_DIR = "buildscripts/templates/generate_resmoke_suites" TEST_SUITE_DIR = "buildscripts/resmokeconfig/suites" -GENERATED_SUITES_DIR = "buildscripts/resmokeconfig/generated" - -HEADER_TEMPLATE = """# DO NOT EDIT THIS FILE. All manual edits will be lost. -# This file was generated by {file} from -# {suite_file}. -""" - MAX_RUNTIME_KEY = "max_runtime" @@ -255,66 +247,32 @@ def divide_tests_into_suites_by_maxtime(tests, sorted_tests, max_time_seconds, m return suites -def get_suite_filename(suite_name): - """ Get the source suite filename""" - return "{dir}/{suite_name}.yml".format(dir=TEST_SUITE_DIR, suite_name=suite_name) - - -def get_subsuite_filename(directory, suite_name, index, index_width=None): - """Generate filename for sub-suite from directory, suite_name and index. zero fill index if - index_width is not None,""" - filled_index = index - if index_width is not None: - filled_index = str(index).zfill(index_width) - return "{dir}/{suite_name}_{index}.yml".format(dir=directory, - suite_name=suite_name, - index=filled_index) - - -def generate_subsuite_file(suite_file, target_file, roots=None, excludes=None): - """ Read and evaluate the yaml suite file. Over ride selector.roots and selector.excludes - with the provided values. Write the result to target_file.""" - with open(suite_file, "r") as fstream: - suite_config = yaml.load(fstream) - - with open(target_file, 'w') as out: - out.write(HEADER_TEMPLATE.format(file=__file__, suite_file=suite_file)) - if roots: - suite_config['selector']['roots'] = roots - if excludes: - suite_config['selector']['exclude_files'] = excludes - out.write(yaml.dump(suite_config, default_flow_style=False, Dumper=yaml.SafeDumper)) - - -def render_suite(suites, suite_name, dir): - """Render the given suites into yml files that can be used by resmoke.py.""" +def get_misc_model(test_list, extra_model_data=None): + """Build a model that will run any missing tests.""" + model = { + "is_misc": True, + "excluded_tests": test_list, + } - suite_file = get_suite_filename(suite_name) + if extra_model_data: + model.update(extra_model_data) - index_width = int(math.ceil(math.log10(len(suites)))) - for idx, suite in enumerate(suites): - subsuite_file = get_subsuite_filename(dir, suite_name, idx, index_width=index_width) - generate_subsuite_file(suite_file, subsuite_file, roots=suite.tests) + return model -def render_misc_suite(test_list, suite_name, dir): - """Render a misc suite to run any tests that might be added to the directory.""" - suite_file = get_suite_filename(suite_name) +def render_template(model, task, index): + """Render the specified model as a yml file in the test suites directory.""" + template_file = "{dir}/{task}.yml.j2".format(dir=TEMPLATES_DIR, task=task) + target_file = "{dir}/{task}_{index}.yml".format(dir=TEST_SUITE_DIR, task=task, index=index) - subsuite_file = get_subsuite_filename(dir, suite_name, 'misc') - generate_subsuite_file(suite_file, subsuite_file, excludes=test_list) + render(model, template_file, target_file) -def prepare_directory_for_suite(suite_name, dir): - """Ensure that dir exists and that it does not contain any suite related files.""" - if os.path.exists(dir): - suite_files_glob = get_subsuite_filename(dir, suite_name, '[0-9]*') - misc_file_glob = get_subsuite_filename(dir, suite_name, 'misc') - for suite_file in glob.glob(suite_files_glob) + glob.glob(misc_file_glob): - os.remove(suite_file) - LOGGER.debug("Removing %s", dir) - else: - os.makedirs(dir) +def render(model, source, destination): + """Render the specified model with the template at `source` to the file `destination`.""" + with open(source, "r") as inp, open(destination, "w") as out: + template = Template(inp.read(), trim_blocks=True) + out.write(template.render(model)) class Suite(object): @@ -346,6 +304,19 @@ class Suite(object): return len(self.tests) + def get_model(self, extra_model_data=None): + """Get a model of this suite that can be used to render a yml file.""" + + model = {"test_names": self.tests, "variants": []} + for variant in self.variant_runtime: + model["variants"].append( + {"name": variant, "runtime": self.variant_runtime[variant] / 60}) + + if extra_model_data: + model.update(extra_model_data) + + return model + class Main(object): """Orchestrate the execution of generate_resmoke_suites.""" @@ -362,7 +333,7 @@ class Main(object): parser = argparse.ArgumentParser(description=self.main.__doc__) parser.add_argument("--analysis-duration", dest="duration_days", default=14, - help="Number of days to analyze.", type=int) + help="Number of days to analyze.") parser.add_argument("--branch", dest="branch", default="master", help="Branch of project to analyze.") parser.add_argument("--end-commit", dest="end_commit", help="End analysis at this commit.") @@ -381,7 +352,7 @@ class Main(object): help="Max number of suites to divide into.") parser.add_argument("--verbose", dest="verbose", action="store_true", default=False, help="Enable verbose logging.") - parser.add_argument("suites", nargs='+', help="A list of suites to analyze.") + parser.add_argument("task", nargs=1, help="task to analyze.") options = parser.parse_args() @@ -405,6 +376,27 @@ class Main(object): return divide_tests_into_suites_by_maxtime(tests, self.test_list, execution_time_secs, self.options.max_sub_suites) + def render_suites(self, suites, task): + """Render the given suites into yml files that can be used by resmoke.py.""" + for idx, suite in enumerate(suites): + render_template(suite.get_model(self.extra_model_data()), task, idx) + + def render_misc_suite(self, task): + """Render a misc suite to run any tests that might be added to the task directory.""" + model = get_misc_model(self.test_list, self.extra_model_data()) + source = "{dir}/{task}.yml.j2".format(dir=TEMPLATES_DIR, task=task) + target = "{dir}/{task}_misc.yml".format(dir=TEST_SUITE_DIR, task=task) + + render(model, source, target) + + def extra_model_data(self): + """Build extra data to include in the model.""" + return { + "options": self.options, + "start_commit": self.commit_range.start, + "end_commit": self.commit_range.end, + } + def main(self): """Generate resmoke suites that run within a specified target execution time.""" @@ -419,20 +411,20 @@ class Main(object): self.commit_range = CommitRange(options.start_commit, options.end_commit) LOGGER.debug("Starting execution for options %s", options) + task = options.task[0] + today = datetime.datetime.utcnow().replace(microsecond=0) start_date = today - datetime.timedelta(days=options.duration_days) - target = ProjectTarget(options.owner, options.project, options.branch) - for suite in options.suites: - prepare_directory_for_suite(suite, CONFIG_DIRECTORY) + target = ProjectTarget(options.owner, options.project, options.branch) - data = self.get_data(target, start_date, suite, options.variants) - suites = self.calculate_suites(data, options.execution_time_minutes * 60) + data = self.get_data(target, start_date, task, options.variants) + suites = self.calculate_suites(data, options.execution_time_minutes * 60) - LOGGER.debug("Creating %d suites for %s", len(suites), suite) + LOGGER.debug("Creating %d suites", len(suites)) - render_suite(suites, suite, CONFIG_DIRECTORY) - render_misc_suite(self.test_list, suite, CONFIG_DIRECTORY) + self.render_suites(suites, task) + self.render_misc_suite(task) if __name__ == "__main__": diff --git a/buildscripts/templates/generate_resmoke_suites/replica_sets_auth.yml.j2 b/buildscripts/templates/generate_resmoke_suites/replica_sets_auth.yml.j2 new file mode 100644 index 00000000000..e2f89521be6 --- /dev/null +++ b/buildscripts/templates/generate_resmoke_suites/replica_sets_auth.yml.j2 @@ -0,0 +1,50 @@ +# This file was generated by buildscripts/generate_resmoke_suites.py and manual edits should also be +# made to replica_sets_auth.yml in order to ensure running the full test suite locally matches the +# behavior of running parts of the test suite in Evergreen. It was generated against commit range: +# {{ start_commit }} - {{ end_commit }} +# with the following options: +# {{ options }} + +# Section that is ignored by resmoke.py. +config_variables: +- &keyFile jstests/libs/authTestsKey +- &keyFileData Thiskeyisonlyforrunningthesuitewithauthenticationdontuseitinanytestsdirectly + +test_kind: js_test + +selector: +{% if variants is defined %} + # The following tests take approximately +{% for variant in variants %} + # {{ variant.runtime }} minutes to run on {{ variant.name }} +{% endfor %} +{% endif %} + roots: +{% for test in test_names %} + - {{ test }} +{% endfor %} +{% if is_misc is defined %} + - jstests/replsets/*.js + exclude_files: + # Skip any tests that run with auth explicitly. + - jstests/replsets/*[aA]uth*.js + # Also skip tests that require a ScopedThread, because ScopedThreads don't inherit credentials. + - jstests/replsets/interrupted_batch_insert.js +{% if excluded_tests is defined %} +{% for test in excluded_tests %} + - {{ test }} +{% endfor %} +{% endif %} +{% endif %} + +executor: + config: + shell_options: + global_vars: + TestData: + auth: true + authMechanism: SCRAM-SHA-1 + keyFile: *keyFile + keyFileData: *keyFileData + nodb: '' + readMode: commands diff --git a/buildscripts/templates/generate_resmoke_suites/replica_sets_ese.yml.j2 b/buildscripts/templates/generate_resmoke_suites/replica_sets_ese.yml.j2 new file mode 100644 index 00000000000..041cbfe22f8 --- /dev/null +++ b/buildscripts/templates/generate_resmoke_suites/replica_sets_ese.yml.j2 @@ -0,0 +1,42 @@ +# This file was generated by buildscripts/generate_resmoke_suites.py and manual edits should also be +# made to replica_sets_ese.yml in order to ensure running the full test suite locally matches the +# behavior of running parts of the test suite in Evergreen. It was generated against commit range: +# {{ start_commit }} - {{ end_commit }} +# with the following options: +# {{ options }} + +# Section that is ignored by resmoke.py. +config_variables: +- &keyFile src/mongo/db/modules/enterprise/jstests/encryptdb/libs/ekf2 + +test_kind: js_test + +selector: +{% if variants is defined %} + # The following tests take approximately +{% for variant in variants %} + # {{ variant.runtime }} minutes to run on {{ variant.name }} +{% endfor %} +{% endif %} + roots: +{% for test in test_names %} + - {{ test }} +{% endfor %} +{% if is_misc is defined %} + - jstests/replsets/*.js +{% if excluded_tests is defined %} +{% for test in excluded_tests %} + - {{ test }} +{% endfor %} +{% endif %} +{% endif %} + +executor: + config: + shell_options: + nodb: '' + global_vars: + TestData: + enableEncryption: '' + encryptionKeyFile: *keyFile + readMode: commands diff --git a/buildscripts/templates/generate_resmoke_suites/sharding.yml.j2 b/buildscripts/templates/generate_resmoke_suites/sharding.yml.j2 new file mode 100644 index 00000000000..f680444cbbc --- /dev/null +++ b/buildscripts/templates/generate_resmoke_suites/sharding.yml.j2 @@ -0,0 +1,34 @@ +# This file was generated by buildscripts/generate_resmoke_suites.py and manual edits should also be +# made to sharding.yml in order to ensure running the full test suite locally matches the behavior +# of running parts of the test suite in Evergreen. It was generated against commit range: +# {{ start_commit }} - {{ end_commit }} +# with the following options: +# {{ options }} +test_kind: js_test + +selector: +{% if variants is defined %} + # The following tests take approximately +{% for variant in variants %} + # {{ variant.runtime }} minutes to run on {{ variant.name }} +{% endfor %} +{% endif %} + roots: +{% for test in test_names %} + - {{ test }} +{% endfor %} +{% if is_misc is defined %} + - jstests/sharding/*.js +{% endif %} +{% if excluded_tests is defined %} + exclude_files: +{% for test in excluded_tests %} + - {{ test }} +{% endfor %} +{% endif %} + +executor: + config: + shell_options: + nodb: '' + readMode: commands diff --git a/buildscripts/templates/generate_resmoke_suites/sharding_auth.yml.j2 b/buildscripts/templates/generate_resmoke_suites/sharding_auth.yml.j2 new file mode 100644 index 00000000000..f7beb4884f1 --- /dev/null +++ b/buildscripts/templates/generate_resmoke_suites/sharding_auth.yml.j2 @@ -0,0 +1,71 @@ +# This file was generated by buildscripts/generate_resmoke_suites.py and manual edits should also be +# made to sharding_auth.yml in order to ensure running the full test suite locally matches the +# behavior of running parts of the test suite in Evergreen. It was generated against commit range: +# {{ start_commit }} - {{ end_commit }} +# with the following options: +# {{ options }} + +# Section that is ignored by resmoke.py. +config_variables: +- &keyFile jstests/libs/authTestsKey +- &keyFileData Thiskeyisonlyforrunningthesuitewithauthenticationdontuseitinanytestsdirectly + +test_kind: js_test + +selector: +{% if variants is defined %} + # The following tests take approximately +{% for variant in variants %} + # {{ variant.runtime }} minutes to run on {{ variant.name }} +{% endfor %} +{% endif %} + roots: +{% for test in test_names %} + - {{ test }} +{% endfor %} +{% if is_misc is defined %} + - jstests/sharding/*.js + exclude_files: + # Skip any tests that run with auth explicitly. + - jstests/sharding/*[aA]uth*.js + - jstests/sharding/advance_cluster_time_action_type.js + - jstests/sharding/aggregation_currentop.js # SERVER-19318 + - jstests/sharding/kill_sessions.js + # Skip these additional tests when running with auth enabled. + - jstests/sharding/parallel.js + # Skip these tests that run with enableTestCommands off. + - jstests/sharding/shard_config_db_collections.js + # Skip the testcases that do not have auth bypass when running ops in parallel. + - jstests/sharding/cleanup_orphaned_cmd_during_movechunk.js # SERVER-21713 + - jstests/sharding/cleanup_orphaned_cmd_during_movechunk_hashed.js # SERVER-21713 + - jstests/sharding/migration_ignore_interrupts_1.js # SERVER-21713 + - jstests/sharding/migration_ignore_interrupts_2.js # SERVER-21713 + - jstests/sharding/migration_ignore_interrupts_3.js # SERVER-21713 + - jstests/sharding/migration_ignore_interrupts_4.js # SERVER-21713 + - jstests/sharding/migration_move_chunk_after_receive.js # SERVER-21713 + - jstests/sharding/migration_server_status.js # SERVER-21713 + - jstests/sharding/migration_sets_fromMigrate_flag.js # SERVER-21713 + - jstests/sharding/migration_with_source_ops.js # SERVER-21713 + - jstests/sharding/movechunk_interrupt_at_primary_stepdown.js # SERVER-21713 + - jstests/sharding/movechunk_parallel.js # SERVER-21713 + - jstests/sharding/migration_critical_section_concurrency.js # SERVER-21713 + # Runs with auth enabled. + - jstests/sharding/mongod_returns_no_cluster_time_without_keys.js +{% if excluded_tests is defined %} +{% for test in excluded_tests %} + - {{ test }} +{% endfor %} +{% endif %} +{% endif %} + +executor: + config: + shell_options: + global_vars: + TestData: + auth: true + authMechanism: SCRAM-SHA-1 + keyFile: *keyFile + keyFileData: *keyFileData + nodb: '' + readMode: commands diff --git a/buildscripts/templates/generate_resmoke_suites/sharding_auth_audit.yml.j2 b/buildscripts/templates/generate_resmoke_suites/sharding_auth_audit.yml.j2 new file mode 100644 index 00000000000..8c7a770fd5e --- /dev/null +++ b/buildscripts/templates/generate_resmoke_suites/sharding_auth_audit.yml.j2 @@ -0,0 +1,73 @@ +# This file was generated by buildscripts/generate_resmoke_suites.py and manual edits should also be +# made to sharding_auth_audit.yml in order to ensure running the full test +# suite locally matches the behavior of running parts of the test suite in Evergreen. It was +# generated against commit range: +# {{ start_commit }} - {{ end_commit }} +# with the following options: +# {{ options }} + +# Section that is ignored by resmoke.py. +config_variables: +- &keyFile jstests/libs/authTestsKey +- &keyFileData Thiskeyisonlyforrunningthesuitewithauthenticationdontuseitinanytestsdirectly + +test_kind: js_test + +selector: +{% if variants is defined %} + # The following tests take approximately +{% for variant in variants %} + # {{ variant.runtime }} minutes to run on {{ variant.name }} +{% endfor %} +{% endif %} + roots: +{% for test in test_names %} + - {{ test }} +{% endfor %} +{% if is_misc is defined %} + - jstests/sharding/*.js + exclude_files: + # Skip any tests that run with auth explicitly. + - jstests/sharding/*[aA]uth*.js + - jstests/sharding/advance_cluster_time_action_type.js + - jstests/sharding/aggregation_currentop.js # SERVER-19318 + - jstests/sharding/kill_sessions.js + # Skip these additional tests when running with auth enabled. + - jstests/sharding/parallel.js + # Skip these tests that run with enableTestCommands off. + - jstests/sharding/shard_config_db_collections.js + # Skip the testcases that do not have auth bypass when running ops in parallel. + - jstests/sharding/cleanup_orphaned_cmd_during_movechunk.js # SERVER-21713 + - jstests/sharding/cleanup_orphaned_cmd_during_movechunk_hashed.js # SERVER-21713 + - jstests/sharding/migration_with_source_ops.js # SERVER-21713 + - jstests/sharding/migration_sets_fromMigrate_flag.js # SERVER-21713 + - jstests/sharding/migration_ignore_interrupts_1.js # SERVER-21713 + - jstests/sharding/migration_ignore_interrupts_2.js # SERVER-21713 + - jstests/sharding/migration_ignore_interrupts_3.js # SERVER-21713 + - jstests/sharding/migration_ignore_interrupts_4.js # SERVER-21713 + - jstests/sharding/movechunk_interrupt_at_primary_stepdown.js # SERVER-21713 + - jstests/sharding/movechunk_parallel.js # SERVER-21713 + - jstests/sharding/migration_server_status.js # SERVER-21713 + - jstests/sharding/migration_move_chunk_after_receive.js # SERVER-21713 + - jstests/sharding/migration_critical_section_concurrency.js # SERVER-21713 + # Runs with auth enabled. + - jstests/sharding/mongod_returns_no_cluster_time_without_keys.js +{% if excluded_tests is defined %} +{% for test in excluded_tests %} + - {{ test }} +{% endfor %} +{% endif %} +{% endif %} + +executor: + config: + shell_options: + global_vars: + TestData: + auditDestination: 'console' + auth: true + authMechanism: SCRAM-SHA-1 + keyFile: *keyFile + keyFileData: *keyFileData + nodb: '' + readMode: commands diff --git a/buildscripts/templates/generate_resmoke_suites/sharding_ese.yml.j2 b/buildscripts/templates/generate_resmoke_suites/sharding_ese.yml.j2 new file mode 100644 index 00000000000..16cd54389e4 --- /dev/null +++ b/buildscripts/templates/generate_resmoke_suites/sharding_ese.yml.j2 @@ -0,0 +1,43 @@ +# This file was generated by buildscripts/generate_resmoke_suites.py and manual edits should also be +# made to sharding_ese.yml in order to ensure running the full test suite locally matches the +# behavior of running parts of the test suite in Evergreen. It was generated against commit range: +# {{ start_commit }} - {{ end_commit }} +# with the following options: +# {{ options }} + +# Section that is ignored by resmoke.py. +config_variables: +- &keyFile src/mongo/db/modules/enterprise/jstests/encryptdb/libs/ekf2 + +test_kind: js_test + +selector: +{% if variants is defined %} + # The following tests take approximately +{% for variant in variants %} + # {{ variant.runtime }} minutes to run on {{ variant.name }} +{% endfor %} +{% endif %} + roots: +{% for test in test_names %} + - {{ test }} +{% endfor %} +{% if is_misc is defined %} + - jstests/sharding/*.js + exclude_files: +{% if excluded_tests is defined %} +{% for test in excluded_tests %} + - {{ test }} +{% endfor %} +{% endif %} +{% endif %} + +executor: + config: + shell_options: + nodb: '' + global_vars: + TestData: + enableEncryption: '' + encryptionKeyFile: *keyFile + readMode: commands diff --git a/buildscripts/templates/generate_resmoke_suites/sharding_last_stable_mongos_and_mixed_shards.yml.j2 b/buildscripts/templates/generate_resmoke_suites/sharding_last_stable_mongos_and_mixed_shards.yml.j2 new file mode 100644 index 00000000000..76176065e79 --- /dev/null +++ b/buildscripts/templates/generate_resmoke_suites/sharding_last_stable_mongos_and_mixed_shards.yml.j2 @@ -0,0 +1,97 @@ +# This file was generated by buildscripts/generate_resmoke_suites.py and manual edits should also be +# made to sharding_last_stable_mongos_and_mixed_shards.yml in order to ensure running the full test +# suite locally matches the behavior of running parts of the test suite in Evergreen. It was +# generated against commit range: +# {{ start_commit }} - {{ end_commit }} +# with the following options: +# {{ options }} + +test_kind: js_test + +selector: +{% if variants is defined %} + # The following tests take approximately +{% for variant in variants %} + # {{ variant.runtime }} minutes to run on {{ variant.name }} +{% endfor %} +{% endif %} + roots: +{% for test in test_names %} + - {{ test }} +{% endfor %} +{% if is_misc is defined %} + - jstests/sharding/*.js + exclude_files: + # Will always fail on last-stable. In order for the test to succeed, the setFCV + # command has to reach the shards. Since the cluster will already be + # running in fully downgraded version, the config server won't forward the + # command to the shards - it'll just return success immediately. + - jstests/sharding/max_time_ms_sharded_new_commands.js + # Requires fix to SERVER-31689 + - jstests/sharding/aggregation_currentop.js + # SERVER-33683: We added a restriction on using an aggregation within a transaction against + # mongos. This should be removed and the test can be adjusted and re-added to this passthrough. + - jstests/sharding/aggregations_in_session.js + # New waitForClusterTime + - jstests/sharding/auth_slaveok_routing.js + # This test should not be run with a mixed cluster environment. + - jstests/sharding/nonreplicated_uuids_on_shardservers.js + # Enable when SERVER-33538 is backported. + - jstests/sharding/mapReduce_outSharded_checkUUID.js + # Will always fail because we can't downgrade FCV before the last-stable binary mongos connects, + # meaning that either the test will stall, or mongos will crash due to connecting to an upgraded + # FCV cluster. + - jstests/sharding/mongos_wait_csrs_initiate.js + # Enable if SERVER-34971 is backported or 4.2 becomes last-stable + - jstests/sharding/update_replace_id.js + - jstests/sharding/stale_mongos_updates_and_removes.js + - jstests/sharding/geo_near_sharded.js + # Enable when 4.2 becomes last-stable. + - jstests/sharding/collation_targeting.js + - jstests/sharding/collation_targeting_inherited.js + - jstests/sharding/geo_near_random1.js + - jstests/sharding/geo_near_random2.js + - jstests/sharding/restart_transactions.js + - jstests/sharding/shard7.js + - jstests/sharding/shard_collection_existing_zones.js + - jstests/sharding/snapshot_cursor_commands_mongos.js + - jstests/sharding/transactions_error_labels.js + - jstests/sharding/transactions_implicit_abort.js + - jstests/sharding/transactions_multi_writes.js + - jstests/sharding/transactions_read_concerns.js + - jstests/sharding/transactions_recover_decision_from_local_participant.js + - jstests/sharding/transactions_reject_writes_for_moved_chunks.js + - jstests/sharding/transactions_snapshot_errors_first_statement.js + - jstests/sharding/transactions_snapshot_errors_subsequent_statements.js + - jstests/sharding/transactions_stale_database_version_errors.js + - jstests/sharding/transactions_stale_shard_version_errors.js + - jstests/sharding/transactions_target_at_point_in_time.js + - jstests/sharding/transactions_view_resolution.js + - jstests/sharding/transactions_writes_not_retryable.js + - jstests/sharding/txn_agg.js + - jstests/sharding/txn_basic_two_phase_commit.js + - jstests/sharding/txn_coordinator_commands_basic_requirements.js + - jstests/sharding/txn_writes_during_movechunk.js + - jstests/sharding/update_sharded.js + - jstests/sharding/shard_existing_coll_chunk_count.js + - jstests/sharding/failcommand_failpoint_not_parallel.js + - jstests/sharding/transactions_expiration.js + # Enable if SERVER-20865 is backported or 4.2 becomes last-stable + - jstests/sharding/sharding_statistics_server_status.js + +{% if excluded_tests is defined %} +{% for test in excluded_tests %} + - {{ test }} +{% endfor %} +{% endif %} +{% endif %} + +executor: + config: + shell_options: + global_vars: + TestData: + mongosBinVersion: 'last-stable' + shardMixedBinVersions: true + skipCheckingUUIDsConsistentAcrossCluster: true + nodb: '' diff --git a/buildscripts/tests/test_generate_resmoke_suites.py b/buildscripts/tests/test_generate_resmoke_suites.py index 76f3bc00de9..efb69f061ed 100644 --- a/buildscripts/tests/test_generate_resmoke_suites.py +++ b/buildscripts/tests/test_generate_resmoke_suites.py @@ -3,16 +3,11 @@ from __future__ import absolute_import import datetime -import math import unittest -import yaml -from mock import patch, mock_open, call +from mock import patch, Mock from buildscripts import generate_resmoke_suites as grs -from generate_resmoke_suites import render_suite, render_misc_suite, \ - prepare_directory_for_suite - # pylint: disable=missing-docstring,invalid-name,unused-argument,no-self-use @@ -525,119 +520,62 @@ class SuiteTest(unittest.TestCase): self.assertEqual(suite.get_test_count(), 3) self.assertEqual(suite.get_runtime(), 29) + def test_model_generation(self): + suite = grs.Suite() + suite.add_test('test1', { + "max_runtime": 10 * 60, + "variant1": 5 * 60, + "variant2": 10 * 60, + "variant3": 7 * 60, + }) + suite.add_test('test2', { + "max_runtime": 12 * 60, + "variant1": 12 * 60, + "variant2": 8 * 60, + "variant3": 6 * 60, + }) + suite.add_test('test3', { + "max_runtime": 7 * 60, + "variant1": 6 * 60, + "variant2": 6 * 60, + "variant3": 7 * 60, + }) + + model = suite.get_model() + + self.assertEqual(model["test_names"], ["test1", "test2", "test3"]) + self.assertIn({"runtime": 23, "name": "variant1"}, model["variants"]) + self.assertIn({"runtime": 24, "name": "variant2"}, model["variants"]) + self.assertIn({"runtime": 20, "name": "variant3"}, model["variants"]) + + +class GetMiscModelTest(unittest.TestCase): + def test_model_with_test_in_same_dir(self): + test_list = [ + "dir0/subdir0/test0", + "dir0/subdir0/test1", + "dir0/subdir0/test2", + "dir0/subdir0/test3", + ] + + model = grs.get_misc_model(test_list) + + self.assertIn("is_misc", model) + + self.assertIn("excluded_tests", model) + self.assertEqual(len(model["excluded_tests"]), 4) + self.assertIn("dir0/subdir0/test0", model["excluded_tests"]) + self.assertIn("dir0/subdir0/test1", model["excluded_tests"]) + self.assertIn("dir0/subdir0/test2", model["excluded_tests"]) + self.assertIn("dir0/subdir0/test3", model["excluded_tests"]) + + def test_model_includes_extra_data(self): + test_list = ["dir0/subdir0/test0"] + extra_data = { + "extra": "data", + } + + model = grs.get_misc_model(test_list, extra_data) -def create_suite(count=3, start=0): - """ Create a suite with count tests.""" - suite = grs.Suite() - for i in range(start, start + count): - suite.add_test('test{}'.format(i), {}) - return suite - - -class RenderSuites(unittest.TestCase): - EXPECTED_FORMAT = """selector: - excludes: - - fixed - roots: - - test{} - - test{} - - test{} -""" - - def _test(self, size): - - suites = [create_suite(start=3*i) for i in range(size)] - expected = [self.EXPECTED_FORMAT .format(*range(3 * i, 3 * (i+1))) - for i in range(len(suites))] - - m = mock_open(read_data=yaml.dump({'selector': {'roots': [], 'excludes': ['fixed']}})) - with patch('generate_resmoke_suites.open', m, create=True): - render_suite(suites, 'suite_name', 'tmp') - handle = m() - - # The other writes are for the headers. - self.assertEquals(len(suites) * 2, handle.write.call_count) - handle.write.assert_has_calls([call(e) for e in expected], any_order=True) - calls = [call('buildscripts/resmokeconfig/suites/suite_name.yml', 'r') - for _ in range(len(suites))] - m.assert_has_calls(calls, any_order=True) - filename = 'tmp/suite_name_{{:0{}}}.yml'.format(int(math.ceil(math.log10(size)))) - calls = [call(filename.format(i), 'w') for i in range(size)] - m.assert_has_calls(calls, any_order=True) - - def test_1_suite(self): - self._test(1) - - def test_11_suites(self): - self._test(11) - - def test_101_suites(self): - self._test(101) - - -class RenderMiscSuites(unittest.TestCase): - - def test_single_suite(self): - - test_list = ['test{}'.format(i) for i in range(10)] - m = mock_open(read_data=yaml.dump({'selector': {'roots': []}})) - with patch('generate_resmoke_suites.open', m, create=True): - render_misc_suite(test_list, 'suite_name', 'tmp') - handle = m() - - # The other writes are for the headers. - self.assertEquals(2, handle.write.call_count) - handle.write.assert_any_call("""selector: - exclude_files: - - test0 - - test1 - - test2 - - test3 - - test4 - - test5 - - test6 - - test7 - - test8 - - test9 - roots: [] -""") - calls = [call('buildscripts/resmokeconfig/suites/suite_name.yml', 'r')] - m.assert_has_calls(calls, any_order=True) - filename = 'tmp/suite_name_misc.yml' - calls = [call(filename, 'w')] - m.assert_has_calls(calls, any_order=True) - - -class PrepareDirectoryForSuite(unittest.TestCase): - - def test_no_directory(self): - with patch('generate_resmoke_suites.os') as mock_os,\ - patch('generate_resmoke_suites.glob.glob') as mock_glob: - mock_os.path.exists.return_value = False - prepare_directory_for_suite('suite_name', 'tmp') - - mock_os.makedirs.assert_called_once_with('tmp') - mock_glob.assert_not_called() - - def _test(self, matched=None): - if matched is None: - matched = [] - with patch('generate_resmoke_suites.os') as mock_os, \ - patch('generate_resmoke_suites.glob.glob') as mock_glob: - mock_os.path.exists.return_value = True - mock_glob.side_effect = (matched, []) - prepare_directory_for_suite('suite_name', 'tmp') - - mock_glob.assert_has_calls([call('tmp/suite_name_[0-9]*.yml'), - call('tmp/suite_name_misc.yml')]) - if matched: - mock_os.remove.assert_has_calls([call(filename) for filename in matched]) - else: - mock_os.remove.assert_not_called() - mock_os.makedirs.assert_not_called() - - def test_empty_directory(self): - self._test() - - def test_old_suite_files(self): - self._test(matched=['tmp/suite_name_{}.yml'.format(i) for i in range(3)]) + self.assertIn("extra", model) + self.assertEqual(model["extra"], "data") |