diff options
author | David Bradford <david.bradford@mongodb.com> | 2019-01-11 16:46:37 -0500 |
---|---|---|
committer | David Bradford <david.bradford@mongodb.com> | 2019-01-11 16:58:49 -0500 |
commit | 65c808b58b182e42fbc6a6d4c0adc3f57ad4c09a (patch) | |
tree | 1aaf4f72d64fc2cd75e81225dfd08c3312e8636b /buildscripts | |
parent | f7d9727c865b3603a9754e18f0bbfad92131f666 (diff) | |
download | mongo-65c808b58b182e42fbc6a6d4c0adc3f57ad4c09a.tar.gz |
SERVER-38946: Exclude newly blacklisted files when generating tasks
Diffstat (limited to 'buildscripts')
-rwxr-xr-x | buildscripts/evergreen_generate_resmoke_tasks.py | 43 | ||||
-rw-r--r-- | buildscripts/tests/test_evergreen_generate_resmoke_tasks.py | 195 |
2 files changed, 158 insertions, 80 deletions
diff --git a/buildscripts/evergreen_generate_resmoke_tasks.py b/buildscripts/evergreen_generate_resmoke_tasks.py index 4f76aeafaa3..19fd7c4662e 100755 --- a/buildscripts/evergreen_generate_resmoke_tasks.py +++ b/buildscripts/evergreen_generate_resmoke_tasks.py @@ -74,16 +74,16 @@ def enable_logging(): """Enable verbose logging for execution.""" logging.basicConfig( - format='[%(asctime)s - %(name)s - %(levelname)s] %(message)s', + format="[%(asctime)s - %(name)s - %(levelname)s] %(message)s", level=logging.DEBUG, stream=sys.stdout, ) def split_if_exists(str_to_split): - """Split the given string on ',' if it is not None.""" + """Split the given string on "," if it is not None.""" if str_to_split: - return str_to_split.split(',') + return str_to_split.split(",") return None @@ -192,19 +192,19 @@ def update_suite_config(suite_config, roots=None, excludes=None): :return: updated suite_config """ if roots: - suite_config['selector']['roots'] = roots + suite_config["selector"]["roots"] = roots if excludes: # This must be a misc file, if the exclude_files section exists, extend it, otherwise, # create it. - if 'exclude_files' in suite_config['selector']: - suite_config['selector']['exclude_files'] += excludes + if "exclude_files" in suite_config["selector"]: + suite_config["selector"]["exclude_files"] += excludes else: - suite_config['selector']['exclude_files'] = excludes + suite_config["selector"]["exclude_files"] = excludes else: - # if excludes was not specified this must not a misc file, so don't exclude anything. - if 'exclude_files' in suite_config['selector']: - del suite_config['selector']['exclude_files'] + # if excludes was not specified this must not a misc file, so don"t exclude anything. + if "exclude_files" in suite_config["selector"]: + del suite_config["selector"]["exclude_files"] return suite_config @@ -220,7 +220,7 @@ def generate_subsuite_file(source_suite_name, target_suite_name, roots=None, exc with open(source_file, "r") as fstream: suite_config = yaml.load(fstream) - with open(os.path.join(CONFIG_DIR, target_suite_name + ".yml"), 'w') as out: + with open(os.path.join(CONFIG_DIR, target_suite_name + ".yml"), "w") as out: out.write(HEADER_TEMPLATE.format(file=__file__, suite_file=source_file)) suite_config = update_suite_config(suite_config, roots, excludes) out.write(yaml.dump(suite_config, default_flow_style=False, Dumper=yaml.SafeDumper)) @@ -306,18 +306,18 @@ class EvergreenConfigGenerator(object): @staticmethod def _is_task_dependency(task, possible_dependency): - return re.match('{0}_(\\d|misc)'.format(task), possible_dependency) + return re.match("{0}_(\\d|misc)".format(task), possible_dependency) def _get_tasks_for_depends_on(self, dependent_task): return [ - str(task['display_name']) for task in self.build_tasks - if self._is_task_dependency(dependent_task, str(task['display_name'])) + str(task["display_name"]) for task in self.build_tasks + if self._is_task_dependency(dependent_task, str(task["display_name"])) ] def _add_dependencies(self, task): task.dependency(TaskDependency("compile")) if not self.options.is_patch: - # Don't worry about task dependencies in patch builds, only mainline. + # Don"t worry about task dependencies in patch builds, only mainline. if self.options.depends_on: for dep in self.options.depends_on: depends_on_tasks = self._get_tasks_for_depends_on(dep) @@ -379,6 +379,11 @@ class EvergreenConfigGenerator(object): return self.evg_config +def normalize_test_name(test_name): + """Normalize test names that may have been run on windows or unix.""" + return test_name.replace("\\", "/") + + class TestStats(object): """Represent the test statistics for the task that is being analyzed.""" @@ -437,7 +442,7 @@ class TestStats(object): hook_runtime_info = self._hook_runtime_by_test[test_name] if hook_runtime_info: duration += hook_runtime_info["duration"] - tests.append((test_file, duration)) + tests.append((normalize_test_name(test_file), duration)) return sorted(tests, key=lambda x: x[1], reverse=True) @@ -559,10 +564,10 @@ class Main(object): return divide_tests_into_suites(tests_runtimes, execution_time_secs, self.options.max_sub_suites) - @staticmethod - def filter_existing_tests(tests_runtimes): + def filter_existing_tests(self, tests_runtimes): """Filter out tests that do not exist in the filesystem.""" - return [info for info in tests_runtimes if os.path.exists(info[0])] + all_tests = [normalize_test_name(test) for test in self.list_tests()] + return [info for info in tests_runtimes if os.path.exists(info[0]) and info[0] in all_tests] def calculate_fallback_suites(self): """Divide tests into a fixed number of suites.""" diff --git a/buildscripts/tests/test_evergreen_generate_resmoke_tasks.py b/buildscripts/tests/test_evergreen_generate_resmoke_tasks.py index c15d6a06bab..767b8891922 100644 --- a/buildscripts/tests/test_evergreen_generate_resmoke_tasks.py +++ b/buildscripts/tests/test_evergreen_generate_resmoke_tasks.py @@ -20,6 +20,13 @@ from buildscripts.evergreen_generate_resmoke_tasks import render_suite, render_m _DATE = datetime.datetime(2018, 7, 15) +NS = "buildscripts.evergreen_generate_resmoke_tasks" + + +def ns(relative_name): # pylint: disable-invalid-name + """Return a full name from a name relative to the test module"s name space.""" + return NS + "." + relative_name + class TestTestStats(unittest.TestCase): def test_no_hooks(self): @@ -160,9 +167,9 @@ class DivideTestsIntoSuitesByMaxtimeTest(unittest.TestCase): class SuiteTest(unittest.TestCase): def test_adding_tests_increases_count_and_runtime(self): suite = grt.Suite() - suite.add_test('test1', 10) - suite.add_test('test2', 12) - suite.add_test('test3', 7) + suite.add_test("test1", 10) + suite.add_test("test2", 12) + suite.add_test("test3", 7) self.assertEqual(suite.get_test_count(), 3) self.assertEqual(suite.get_runtime(), 29) @@ -172,36 +179,36 @@ def create_suite(count=3, start=0): """ Create a suite with count tests.""" suite = grt.Suite() for i in range(start, start + count): - suite.add_test('test{}'.format(i), 1) + suite.add_test("test{}".format(i), 1) return suite class UpdateSuiteConfigTest(unittest.TestCase): def test_roots_are_updated(self): - config = {'selector': {}} + config = {"selector": {}} - updated_config = grt.update_suite_config(config, 'root value') - self.assertEqual('root value', updated_config['selector']['roots']) + updated_config = grt.update_suite_config(config, "root value") + self.assertEqual("root value", updated_config["selector"]["roots"]) def test_excluded_files_not_included_if_not_specified(self): - config = {'selector': {'excluded_files': 'files to exclude'}} + config = {"selector": {"excluded_files": "files to exclude"}} updated_config = grt.update_suite_config(config, excludes=None) - self.assertNotIn('exclude_files', updated_config['selector']) + self.assertNotIn("exclude_files", updated_config["selector"]) def test_excluded_files_added_to_misc(self): - config = {'selector': {}} + config = {"selector": {}} - updated_config = grt.update_suite_config(config, excludes='files to exclude') - self.assertEqual('files to exclude', updated_config['selector']['exclude_files']) + updated_config = grt.update_suite_config(config, excludes="files to exclude") + self.assertEqual("files to exclude", updated_config["selector"]["exclude_files"]) def test_excluded_files_extended_in_misc(self): - config = {'selector': {'exclude_files': ['file 0', 'file 1']}} + config = {"selector": {"exclude_files": ["file 0", "file 1"]}} - updated_config = grt.update_suite_config(config, excludes=['file 2', 'file 3']) - self.assertEqual(4, len(updated_config['selector']['exclude_files'])) - for exclude in ['file 0', 'file 1', 'file 2', 'file 3']: - self.assertIn(exclude, updated_config['selector']['exclude_files']) + updated_config = grt.update_suite_config(config, excludes=["file 2", "file 3"]) + self.assertEqual(4, len(updated_config["selector"]["exclude_files"])) + for exclude in ["file 0", "file 1", "file 2", "file 3"]: + self.assertIn(exclude, updated_config["selector"]["exclude_files"]) class RenderSuites(unittest.TestCase): @@ -221,22 +228,22 @@ class RenderSuites(unittest.TestCase): self.EXPECTED_FORMAT.format(*range(3 * i, 3 * (i + 1))) for i in range(len(suites)) ] - m = mock_open(read_data=yaml.dump({'selector': {'roots': [], 'excludes': ['fixed']}})) - with patch('buildscripts.evergreen_generate_resmoke_tasks.open', m, create=True): - render_suite(suites, 'suite_name') + m = mock_open(read_data=yaml.dump({"selector": {"roots": [], "excludes": ["fixed"]}})) + with patch(ns("open"), m, create=True): + render_suite(suites, "suite_name") handle = m() # The other writes are for the headers. self.assertEquals(len(suites) * 2, handle.write.call_count) handle.write.assert_has_calls([call(e) for e in expected], any_order=True) calls = [ - call(os.path.join(grt.TEST_SUITE_DIR, 'suite_name.yml'), 'r') + call(os.path.join(grt.TEST_SUITE_DIR, "suite_name.yml"), "r") for _ in range(len(suites)) ] m.assert_has_calls(calls, any_order=True) - filename = os.path.join(grt.CONFIG_DIR, 'suite_name_{{:0{}}}.yml'.format( + filename = os.path.join(grt.CONFIG_DIR, "suite_name_{{:0{}}}.yml".format( int(math.ceil(math.log10(size))))) - calls = [call(filename.format(i), 'w') for i in range(size)] + calls = [call(filename.format(i), "w") for i in range(size)] m.assert_has_calls(calls, any_order=True) def test_1_suite(self): @@ -252,10 +259,10 @@ class RenderSuites(unittest.TestCase): class RenderMiscSuites(unittest.TestCase): def test_single_suite(self): - test_list = ['test{}'.format(i) for i in range(10)] - m = mock_open(read_data=yaml.dump({'selector': {'roots': []}})) - with patch('buildscripts.evergreen_generate_resmoke_tasks.open', m, create=True): - render_misc_suite(test_list, 'suite_name') + test_list = ["test{}".format(i) for i in range(10)] + m = mock_open(read_data=yaml.dump({"selector": {"roots": []}})) + with patch(ns("open"), m, create=True): + render_misc_suite(test_list, "suite_name") handle = m() # The other writes are for the headers. @@ -274,20 +281,20 @@ class RenderMiscSuites(unittest.TestCase): - test9 roots: [] """) - calls = [call(os.path.join(grt.TEST_SUITE_DIR, 'suite_name.yml'), 'r')] + calls = [call(os.path.join(grt.TEST_SUITE_DIR, "suite_name.yml"), "r")] m.assert_has_calls(calls, any_order=True) - filename = os.path.join(grt.CONFIG_DIR, 'suite_name_misc.yml') - calls = [call(filename, 'w')] + filename = os.path.join(grt.CONFIG_DIR, "suite_name_misc.yml") + calls = [call(filename, "w")] m.assert_has_calls(calls, any_order=True) class PrepareDirectoryForSuite(unittest.TestCase): def test_no_directory(self): - with patch('buildscripts.evergreen_generate_resmoke_tasks.os') as mock_os: + with patch(ns("os")) as mock_os: mock_os.path.exists.return_value = False - prepare_directory_for_suite('tmp') + prepare_directory_for_suite("tmp") - mock_os.makedirs.assert_called_once_with('tmp') + mock_os.makedirs.assert_called_once_with("tmp") class CalculateTimeoutTest(unittest.TestCase): @@ -377,13 +384,13 @@ class EvergreenConfigGeneratorTest(unittest.TestCase): def test_selecting_tasks(self): is_task_dependency = grt.EvergreenConfigGenerator._is_task_dependency - self.assertFalse(is_task_dependency('sharding', 'sharding')) - self.assertFalse(is_task_dependency('sharding', 'other_task')) - self.assertFalse(is_task_dependency('sharding', 'sharding_gen')) + self.assertFalse(is_task_dependency("sharding", "sharding")) + self.assertFalse(is_task_dependency("sharding", "other_task")) + self.assertFalse(is_task_dependency("sharding", "sharding_gen")) - self.assertTrue(is_task_dependency('sharding', 'sharding_0')) - self.assertTrue(is_task_dependency('sharding', 'sharding_314')) - self.assertTrue(is_task_dependency('sharding', 'sharding_misc')) + self.assertTrue(is_task_dependency("sharding", "sharding_0")) + self.assertTrue(is_task_dependency("sharding", "sharding_314")) + self.assertTrue(is_task_dependency("sharding", "sharding_misc")) def test_get_tasks_depends_on(self): options = self.generate_mock_options() @@ -391,36 +398,36 @@ class EvergreenConfigGeneratorTest(unittest.TestCase): cfg_generator = grt.EvergreenConfigGenerator(suites, options, Mock()) cfg_generator.build_tasks = [ - {'display_name': 'sharding_gen'}, - {'display_name': 'sharding_0'}, - {'display_name': 'other_task'}, - {'display_name': 'other_task_2'}, - {'display_name': 'sharding_1'}, - {'display_name': 'compile'}, - {'display_name': 'sharding_misc'}, + {"display_name": "sharding_gen"}, + {"display_name": "sharding_0"}, + {"display_name": "other_task"}, + {"display_name": "other_task_2"}, + {"display_name": "sharding_1"}, + {"display_name": "compile"}, + {"display_name": "sharding_misc"}, ] - dependent_tasks = cfg_generator._get_tasks_for_depends_on('sharding') + dependent_tasks = cfg_generator._get_tasks_for_depends_on("sharding") self.assertEqual(3, len(dependent_tasks)) - self.assertIn('sharding_0', dependent_tasks) - self.assertIn('sharding_1', dependent_tasks) - self.assertIn('sharding_misc', dependent_tasks) + self.assertIn("sharding_0", dependent_tasks) + self.assertIn("sharding_1", dependent_tasks) + self.assertIn("sharding_misc", dependent_tasks) def test_specified_dependencies_are_added(self): options = self.generate_mock_options() - options.depends_on = ['sharding'] + options.depends_on = ["sharding"] options.is_patch = False suites = self.generate_mock_suites(3) cfg_generator = grt.EvergreenConfigGenerator(suites, options, Mock()) cfg_generator.build_tasks = [ - {'display_name': 'sharding_gen'}, - {'display_name': 'sharding_0'}, - {'display_name': 'other_task'}, - {'display_name': 'other_task_2'}, - {'display_name': 'sharding_1'}, - {'display_name': 'compile'}, - {'display_name': 'sharding_misc'}, + {"display_name": "sharding_gen"}, + {"display_name": "sharding_0"}, + {"display_name": "other_task"}, + {"display_name": "other_task_2"}, + {"display_name": "sharding_1"}, + {"display_name": "compile"}, + {"display_name": "sharding_misc"}, ] cfg_mock = Mock() @@ -428,6 +435,14 @@ class EvergreenConfigGeneratorTest(unittest.TestCase): self.assertEqual(4, cfg_mock.dependency.call_count) +class NormalizeTestNameTest(unittest.TestCase): + def test_unix_names(self): + self.assertEqual("/home/user/test.js", grt.normalize_test_name("/home/user/test.js")) + + def test_windows_names(self): + self.assertEqual("/home/user/test.js", grt.normalize_test_name("\\home\\user\\test.js")) + + class MainTest(unittest.TestCase): @staticmethod def get_mock_options(): @@ -446,8 +461,10 @@ class MainTest(unittest.TestCase): main.options = Mock() main.config_options = self.get_mock_options() - with patch('os.path.exists') as exists_mock: + with patch("os.path.exists") as exists_mock, patch(ns("suitesconfig")) as suitesconfig_mock: exists_mock.return_value = True + suitesconfig_mock.get_suite.return_value.tests = \ + [stat["test_file"] for stat in evg.test_stats.return_value] suites = main.calculate_suites(_DATE, _DATE) # There are 100 tests taking 1 minute, with a target of 10 min we expect 10 suites. @@ -495,11 +512,67 @@ class MainTest(unittest.TestCase): ("dir1/file3.js", 36.32), ] - with patch("os.path.exists") as exists_mock: + with patch("os.path.exists") as exists_mock, patch(ns("suitesconfig")) as suitesconfig_mock: exists_mock.side_effect = [False, True, True] - filtered_list = grt.Main.filter_existing_tests(tests_runtimes) + evg = Mock() + suitesconfig_mock.get_suite.return_value.tests = \ + [runtime[0] for runtime in tests_runtimes] + main = grt.Main(evg) + main.config_options = Mock() + main.config_options.suite = "suite" + filtered_list = main.filter_existing_tests(tests_runtimes) self.assertEqual(2, len(filtered_list)) self.assertNotIn(tests_runtimes[0], filtered_list) self.assertIn(tests_runtimes[2], filtered_list) self.assertIn(tests_runtimes[1], filtered_list) + + def test_filter_blacklist_files(self): + tests_runtimes = [ + ("dir1/file1.js", 20.32), + ("dir2/file2.js", 24.32), + ("dir1/file3.js", 36.32), + ] + + blacklisted_test = tests_runtimes[1][0] + + with patch("os.path.exists") as exists_mock, patch(ns("suitesconfig")) as suitesconfig_mock: + exists_mock.return_value = True + evg = Mock() + suitesconfig_mock.get_suite.return_value.tests = \ + [runtime[0] for runtime in tests_runtimes if runtime[0] != blacklisted_test] + main = grt.Main(evg) + main.config_options = Mock() + main.config_options.suite = "suite" + filtered_list = main.filter_existing_tests(tests_runtimes) + + self.assertEqual(2, len(filtered_list)) + self.assertNotIn(blacklisted_test, filtered_list) + self.assertIn(tests_runtimes[2], filtered_list) + self.assertIn(tests_runtimes[0], filtered_list) + + def test_filter_blacklist_files_for_windows(self): + tests_runtimes = [ + ("dir1/file1.js", 20.32), + ("dir2/file2.js", 24.32), + ("dir1/dir3/file3.js", 36.32), + ] + + blacklisted_test = tests_runtimes[1][0] + + with patch("os.path.exists") as exists_mock, patch(ns("suitesconfig")) as suitesconfig_mock: + exists_mock.return_value = True + evg = Mock() + suitesconfig_mock.get_suite.return_value.tests = [ + runtime[0].replace("/", "\\") for runtime in tests_runtimes + if runtime[0] != blacklisted_test + ] + main = grt.Main(evg) + main.config_options = Mock() + main.config_options.suite = "suite" + filtered_list = main.filter_existing_tests(tests_runtimes) + + self.assertNotIn(blacklisted_test, filtered_list) + self.assertIn(tests_runtimes[2], filtered_list) + self.assertIn(tests_runtimes[0], filtered_list) + self.assertEqual(2, len(filtered_list)) |