diff options
| -rwxr-xr-x | buildscripts/resmoke.py | 11 | ||||
| -rw-r--r-- | buildscripts/resmokelib/selector.py | 32 | ||||
| -rw-r--r-- | buildscripts/resmokelib/testing/suite.py | 10 | ||||
| -rw-r--r-- | buildscripts/tests/resmokelib/test_selector.py | 155 |
4 files changed, 152 insertions, 56 deletions
diff --git a/buildscripts/resmoke.py b/buildscripts/resmoke.py index 1d5bc7be694..54572a95af6 100755 --- a/buildscripts/resmoke.py +++ b/buildscripts/resmoke.py @@ -36,14 +36,21 @@ def _execute_suite(suite): if resmokelib.config.DRY_RUN == "tests": sb = [] - sb.append("Tests that would be run for %ss in suite %s:" - % (suite.test_kind, suite.get_display_name())) + sb.append("Tests that would be run in suite %s:" % suite.get_display_name()) if len(suite.tests) > 0: for test in suite.tests: sb.append(test) else: sb.append("(no tests)") logger.info("\n".join(sb)) + sb = [] + sb.append("Tests that would be excluded from suite %s:" % suite.get_display_name()) + if len(suite.excluded) > 0: + for test in suite.excluded: + sb.append(test) + else: + sb.append("(no tests)") + logger.info("\n".join(sb)) # Set a successful return code on the test suite because we want to output the tests # that would get run by any other suites the user specified. diff --git a/buildscripts/resmokelib/selector.py b/buildscripts/resmokelib/selector.py index 3fff181c188..6d80d06be87 100644 --- a/buildscripts/resmokelib/selector.py +++ b/buildscripts/resmokelib/selector.py @@ -230,10 +230,14 @@ class _TestList(object): The tests are returned in the same order as they are found in the root tests. """ tests = [] + excluded = [] for test in self._roots: - if test in self._filtered and test not in tests: - tests.append(test) - return tests + if test in self._filtered: + if test not in tests: + tests.append(test) + elif test not in excluded: + excluded.append(test) + return tests, excluded ############################## @@ -406,6 +410,8 @@ class _Selector(object): Args: selector_config: a _SelectorConfig instance. + Returns: + A tuple with the list of selected tests and the list of excluded tests. """ # 1. Find the root files. @@ -508,25 +514,27 @@ class _DbTestSelector(_Selector): _Selector.__init__(self, test_file_explorer, tests_are_files=False) def select(self, selector_config): + if selector_config.roots: + roots = selector_config.roots + else: + if not self._test_file_explorer.isfile(selector_config.binary): + raise IOError(errno.ENOENT, "File not found", selector_config.binary) + roots = self._test_file_explorer.list_dbtests(selector_config.binary) + if config.INCLUDE_WITH_ANY_TAGS: # The db_tests do not currently support tags so we always return an empty array when the # --includeWithAnyTags option is used. - return [] + return [], roots if selector_config.roots: # Tests have been specified on the command line. We use them without additional # filtering. - return selector_config.roots - - if not self._test_file_explorer.isfile(selector_config.binary): - raise IOError(errno.ENOENT, "File not found", selector_config.binary) - - dbtests = self._test_file_explorer.list_dbtests(selector_config.binary) + return selector_config.roots, [] if not selector_config.include_suites: - return dbtests + return roots, [] - test_files = _TestList(self._test_file_explorer, dbtests, tests_are_files=False) + test_files = _TestList(self._test_file_explorer, roots, tests_are_files=False) test_files.include_any_pattern(selector_config.include_suites) return test_files.get_tests() diff --git a/buildscripts/resmokelib/testing/suite.py b/buildscripts/resmokelib/testing/suite.py index 132a2d70d9d..31036f20578 100644 --- a/buildscripts/resmokelib/testing/suite.py +++ b/buildscripts/resmokelib/testing/suite.py @@ -42,7 +42,7 @@ class Suite(object): self._suite_options = suite_options self.test_kind = self.get_test_kind_config() - self.tests = self._get_tests_for_kind(self.test_kind) + self.tests, self.excluded = self._get_tests_for_kind(self.test_kind) self.return_code = None # Set by the executor. @@ -70,13 +70,13 @@ class Suite(object): mongos_options = test_info # Just for easier reading. if not isinstance(mongos_options, dict): raise TypeError("Expected dictionary of arguments to mongos") - return [mongos_options] + return [mongos_options], [] - tests = _selector.filter_tests(test_kind, test_info) + tests, excluded = _selector.filter_tests(test_kind, test_info) if _config.ORDER_TESTS_BY_NAME: - return sorted(tests, key=str.lower) + return sorted(tests, key=str.lower), sorted(excluded, key=str.lower) - return tests + return tests, excluded def get_name(self): """ diff --git a/buildscripts/tests/resmokelib/test_selector.py b/buildscripts/tests/resmokelib/test_selector.py index ae8b0a96c8d..83cc48c3a8d 100644 --- a/buildscripts/tests/resmokelib/test_selector.py +++ b/buildscripts/tests/resmokelib/test_selector.py @@ -168,18 +168,24 @@ class TestTestList(unittest.TestCase): def test_roots(self): roots = ["a", "b"] test_list = selector._TestList(self.test_file_explorer, roots, tests_are_files=False) - self.assertEqual(roots, test_list.get_tests()) + selected, excluded = test_list.get_tests() + self.assertEqual(roots, selected) + self.assertEqual([], excluded) def test_roots_with_glob(self): glob_roots = ["dir/subdir1/*.js"] expected_roots = ["dir/subdir1/test11.js", "dir/subdir1/test12.js"] test_list = selector._TestList(self.test_file_explorer, glob_roots) - self.assertEqual(expected_roots, test_list.get_tests()) + selected, excluded = test_list.get_tests() + self.assertEqual(expected_roots, selected) + self.assertEqual([], excluded) def test_roots_with_unmatching_glob(self): glob_roots = ["unknown/subdir1/*.js"] - with self.assertRaisesRegexp(ValueError, "Pattern does not match any files: unknown/subdir1/\*.js"): - selector._TestList(self.test_file_explorer, glob_roots) + test_list = selector._TestList(self.test_file_explorer, glob_roots) + selected, excluded = test_list.get_tests() + self.assertEqual([], selected) + self.assertEqual([], excluded) def test_roots_unknown_file(self): roots = ["dir/subdir1/unknown"] @@ -190,20 +196,29 @@ class TestTestList(unittest.TestCase): roots = ["dir/subdir1/*.js", "dir/subdir2/test21.*"] test_list = selector._TestList(self.test_file_explorer, roots) test_list.include_files(["dir/subdir2/test21.js"]) - self.assertEqual(["dir/subdir2/test21.js"], test_list.get_tests()) + selected, excluded = test_list.get_tests() + self.assertEqual(["dir/subdir2/test21.js"], selected) + self.assertEqual(["dir/subdir1/test11.js", + "dir/subdir1/test12.js"], excluded) def test_include_files_no_match(self): roots = ["dir/subdir1/*.js", "dir/subdir2/test21.*"] test_list = selector._TestList(self.test_file_explorer, roots) test_list.include_files(["dir/subdir2/test26.js"]) - self.assertEqual([], test_list.get_tests()) + selected, excluded = test_list.get_tests() + self.assertEqual([], selected) + self.assertEqual(["dir/subdir1/test11.js", + "dir/subdir1/test12.js", + "dir/subdir2/test21.js"], excluded) def test_exclude_files(self): roots = ["dir/subdir1/*.js", "dir/subdir2/test21.*"] test_list = selector._TestList(self.test_file_explorer, roots) test_list.exclude_files(["dir/subdir2/test21.js"]) + selected, excluded = test_list.get_tests() self.assertEqual(["dir/subdir1/test11.js", - "dir/subdir1/test12.js"], test_list.get_tests()) + "dir/subdir1/test12.js"], selected) + self.assertEqual(["dir/subdir2/test21.js"], excluded) def test_exclude_files_no_match(self): roots = ["dir/subdir1/*.js", "dir/subdir2/test21.*"] @@ -215,8 +230,10 @@ class TestTestList(unittest.TestCase): roots = ["dir/subdir1/*.js", "dir/subdir2/test21.*"] test_list = selector._TestList(self.test_file_explorer, roots) test_list.exclude_files(["dir/subdir2/*.js"]) + selected, excluded = test_list.get_tests() self.assertEqual(["dir/subdir1/test11.js", - "dir/subdir1/test12.js"], test_list.get_tests()) + "dir/subdir1/test12.js"], selected) + self.assertEqual(["dir/subdir2/test21.js"], excluded) def test_match_tag_expression(self): roots = ["dir/subdir1/*.js", "dir/subdir2/test21.*"] @@ -230,47 +247,76 @@ class TestTestList(unittest.TestCase): return self.test_file_explorer.jstest_tags(test_file) test_list.match_tag_expression(expression, get_tags) + selected, excluded = test_list.get_tests() self.assertEqual(["dir/subdir1/test11.js", - "dir/subdir1/test12.js"], test_list.get_tests()) + "dir/subdir1/test12.js"], selected) + self.assertEqual(["dir/subdir2/test21.js"], excluded) def test_include_any_pattern(self): roots = ["dir/subdir1/*.js", "dir/subdir2/test21.*", "dir/subdir3/a/test3a1.js"] # 1 pattern and 1 matching test_list = selector._TestList(self.test_file_explorer, roots) test_list.include_any_pattern(["dir/*3/a/*"]) - self.assertEqual(["dir/subdir3/a/test3a1.js"], test_list.get_tests()) + selected, excluded = test_list.get_tests() + self.assertEqual(["dir/subdir3/a/test3a1.js"], selected) + self.assertEqual(["dir/subdir1/test11.js", + "dir/subdir1/test12.js", + "dir/subdir2/test21.js"], excluded) # 1 pattern and 0 matching test_list = selector._TestList(self.test_file_explorer, roots) test_list.include_any_pattern(["dir/*4/a/*"]) - self.assertEqual([], test_list.get_tests()) + selected, excluded = test_list.get_tests() + self.assertEqual([], selected) + self.assertEqual(["dir/subdir1/test11.js", + "dir/subdir1/test12.js", + "dir/subdir2/test21.js", + "dir/subdir3/a/test3a1.js"], excluded) # 3 patterns and 1 matching test_list = selector._TestList(self.test_file_explorer, roots) test_list.include_any_pattern(["dir/*3/a/*", "notmaching/*", "notmatching2/*.js"]) - self.assertEqual(["dir/subdir3/a/test3a1.js"], test_list.get_tests()) + selected, excluded = test_list.get_tests() + self.assertEqual(["dir/subdir3/a/test3a1.js"], selected) + self.assertEqual(["dir/subdir1/test11.js", + "dir/subdir1/test12.js", + "dir/subdir2/test21.js"], excluded) # 3 patterns and 0 matching test_list = selector._TestList(self.test_file_explorer, roots) test_list.include_any_pattern(["dir2/*3/a/*", "notmaching/*", "notmatching2/*.js"]) - self.assertEqual([], test_list.get_tests()) + selected, excluded = test_list.get_tests() + self.assertEqual([], selected) + self.assertEqual(["dir/subdir1/test11.js", + "dir/subdir1/test12.js", + "dir/subdir2/test21.js", + "dir/subdir3/a/test3a1.js"], excluded) # 3 patterns and 3 matching test_list = selector._TestList(self.test_file_explorer, roots) test_list.include_any_pattern(["dir/*1/*11*", "dir/subdir3/**", "dir/subdir2/*.js"]) + selected, excluded = test_list.get_tests() self.assertEqual(["dir/subdir1/test11.js", "dir/subdir2/test21.js", "dir/subdir3/a/test3a1.js"], - test_list.get_tests()) + selected) + self.assertEqual(["dir/subdir1/test12.js"], excluded) def test_include_tests_no_force(self): roots = ["dir/subdir1/*.js", "dir/subdir2/test21.*"] test_list = selector._TestList(self.test_file_explorer, roots) test_list.exclude_files(["dir/subdir1/test11.js"]) test_list.include_files(["dir/subdir1/test11.js"], force=False) - self.assertEqual([], test_list.get_tests()) + selected, excluded = test_list.get_tests() + self.assertEqual([], selected) + self.assertEqual(["dir/subdir1/test11.js", + "dir/subdir1/test12.js", + "dir/subdir2/test21.js"], excluded) def test_include_tests_force(self): roots = ["dir/subdir1/*.js", "dir/subdir2/test21.*"] test_list = selector._TestList(self.test_file_explorer, roots) test_list.exclude_files(["dir/subdir1/test11.js"]) test_list.include_files(["dir/subdir1/test11.js"], force=True) - self.assertEqual(["dir/subdir1/test11.js"], test_list.get_tests()) + selected, excluded = test_list.get_tests() + self.assertEqual(["dir/subdir1/test11.js"], selected) + self.assertEqual(["dir/subdir1/test12.js", + "dir/subdir2/test21.js"], excluded) def test_tests_are_not_files(self): roots = ["a", "b"] @@ -299,41 +345,54 @@ class TestSelector(unittest.TestCase): def test_select_all(self): config = selector._SelectorConfig(roots=["dir/subdir1/*.js", "dir/subdir2/*.js", "dir/subdir3/a/*.js"]) - selected = self.selector.select(config) + selected, excluded = self.selector.select(config) self.assertEqual(["dir/subdir1/test11.js", "dir/subdir1/test12.js", "dir/subdir2/test21.js", "dir/subdir3/a/test3a1.js"], selected) + self.assertEqual([], excluded) def test_select_exclude_files(self): config = selector._SelectorConfig(roots=["dir/subdir1/*.js", "dir/subdir2/*.js", "dir/subdir3/a/*.js"], exclude_files=["dir/subdir2/test21.js"]) - selected = self.selector.select(config) + selected, excluded = self.selector.select(config) self.assertEqual(["dir/subdir1/test11.js", "dir/subdir1/test12.js", "dir/subdir3/a/test3a1.js"], selected) + self.assertEqual(["dir/subdir2/test21.js"], excluded) def test_select_include_files(self): config = selector._SelectorConfig(roots=["dir/subdir1/*.js", "dir/subdir2/*.js", "dir/subdir3/a/*.js"], include_files=["dir/subdir2/test21.js"]) - selected = self.selector.select(config) + selected, excluded = self.selector.select(config) self.assertEqual(["dir/subdir2/test21.js"], selected) + self.assertEqual(["dir/subdir1/test11.js", + "dir/subdir1/test12.js", + "dir/subdir3/a/test3a1.js"], excluded) def test_select_include_tags(self): config = selector._SelectorConfig(roots=["dir/subdir1/*.js", "dir/subdir2/*.js", "dir/subdir3/a/*.js"], include_tags="tag1") - selected = self.selector.select(config) + selected, excluded = self.selector.select(config) self.assertEqual([], selected) + self.assertEqual(["dir/subdir1/test11.js", + "dir/subdir1/test12.js", + "dir/subdir2/test21.js", + "dir/subdir3/a/test3a1.js"], excluded) def test_select_include_any_tags(self): config = selector._SelectorConfig(roots=["dir/subdir1/*.js", "dir/subdir2/*.js", "dir/subdir3/a/*.js"], include_with_any_tags=["tag1"]) - selected = self.selector.select(config) + selected, excluded = self.selector.select(config) self.assertEqual([], selected) + self.assertEqual(["dir/subdir1/test11.js", + "dir/subdir1/test12.js", + "dir/subdir2/test21.js", + "dir/subdir3/a/test3a1.js"], excluded) class TestFilterTests(unittest.TestCase): @@ -348,75 +407,90 @@ class TestFilterTests(unittest.TestCase): def test_cpp_all(self): config = {"root": "integrationtest.txt"} - selected = selector.filter_tests("cpp_integration_test", config, self.test_file_explorer) + selected, excluded = selector.filter_tests("cpp_integration_test", config, self.test_file_explorer) self.assertEqual(["build/testA", "build/testB"], selected) + self.assertEqual([], excluded) def test_cpp_roots_override(self): # When roots are specified for cpp tests they override all filtering since # 'roots' are populated with the command line arguments. config = {"include_files": "unknown_file", "roots": ["build/testC"]} - selected = selector.filter_tests("cpp_unit_test", config, self.test_file_explorer) + selected, excluded = selector.filter_tests("cpp_unit_test", config, self.test_file_explorer) self.assertEqual(["build/testC"], selected) - selected = selector.filter_tests("cpp_integration_test", config, self.test_file_explorer) + self.assertEqual([], excluded) + selected, excluded = selector.filter_tests("cpp_integration_test", config, self.test_file_explorer) self.assertEqual(["build/testC"], selected) + self.assertEqual([], excluded) def test_cpp_expand_roots(self): config = {"root": "integrationtest.txt", "roots": ["build/test*"]} - selected = selector.filter_tests("cpp_integration_test", config, self.test_file_explorer) + selected, excluded = selector.filter_tests("cpp_integration_test", config, self.test_file_explorer) self.assertEqual(["build/testA", "build/testB", "build/testC"], selected) + self.assertEqual([], excluded) - selected = selector.filter_tests("cpp_unit_test", config, self.test_file_explorer) + selected, excluded = selector.filter_tests("cpp_unit_test", config, self.test_file_explorer) self.assertEqual(["build/testA", "build/testB", "build/testC"], selected) + self.assertEqual([], excluded) def test_cpp_with_any_tags(self): buildscripts.resmokelib.config.INCLUDE_WITH_ANY_TAGS = ["tag1"] try: selector_config = {"root": "unittest.txt"} - selected = selector.filter_tests( + selected, excluded = selector.filter_tests( "cpp_unit_test", selector_config, test_file_explorer=self.test_file_explorer) self.assertEqual([], selected) + self.assertEqual(["build/testA", "build/testB"], excluded) finally: buildscripts.resmokelib.config.INCLUDE_WITH_ANY_TAGS = None def test_jstest_include_tags(self): config = {"roots": ["dir/subdir1/*.js", "dir/subdir2/*.js", "dir/subdir3/a/*.js"], "include_tags": "tag1"} - selected = selector.filter_tests("js_test", config, self.test_file_explorer) + selected, excluded = selector.filter_tests("js_test", config, self.test_file_explorer) self.assertEqual(["dir/subdir1/test11.js"], selected) + self.assertEqual(["dir/subdir1/test12.js", + "dir/subdir2/test21.js", + "dir/subdir3/a/test3a1.js"], excluded) def test_jstest_exclude_tags(self): config = {"roots": ["dir/subdir1/*.js", "dir/subdir2/*.js", "dir/subdir3/a/*.js"], "exclude_tags": "tag1"} - selected = selector.filter_tests("js_test", config, self.test_file_explorer) + selected, excluded = selector.filter_tests("js_test", config, self.test_file_explorer) self.assertEqual(["dir/subdir1/test12.js", "dir/subdir2/test21.js", "dir/subdir3/a/test3a1.js"], selected) + self.assertEqual(["dir/subdir1/test11.js"], excluded) def test_jstest_force_include(self): config = {"roots": ["dir/subdir1/*.js", "dir/subdir2/*.js", "dir/subdir3/a/*.js"], "include_files": ["dir/subdir1/*.js"], "exclude_tags": "tag1"} - selected = selector.filter_tests("js_test", config, self.test_file_explorer) + selected, excluded = selector.filter_tests("js_test", config, self.test_file_explorer) self.assertEqual(["dir/subdir1/test11.js", "dir/subdir1/test12.js"], selected) + self.assertEqual(["dir/subdir2/test21.js", + "dir/subdir3/a/test3a1.js"], excluded) def test_jstest_all(self): config = {"roots": ["dir/subdir1/*.js", "dir/subdir2/*.js", "dir/subdir3/a/*.js"]} - selected = selector.filter_tests("js_test", config, self.test_file_explorer) + selected, excluded = selector.filter_tests("js_test", config, self.test_file_explorer) self.assertEqual(["dir/subdir1/test11.js", "dir/subdir1/test12.js", "dir/subdir2/test21.js", "dir/subdir3/a/test3a1.js"], selected) + self.assertEqual([], excluded) def test_jstest_include_with_any_tags(self): config = {"roots": ["dir/subdir1/*.js", "dir/subdir2/*.js", "dir/subdir3/a/*.js"], "include_with_any_tags": ["tag2"]} - selected = selector.filter_tests("js_test", config, self.test_file_explorer) + selected, excluded = selector.filter_tests("js_test", config, self.test_file_explorer) self.assertEqual(["dir/subdir1/test11.js", "dir/subdir2/test21.js"], selected) + self.assertEqual(["dir/subdir1/test12.js", + "dir/subdir3/a/test3a1.js"], excluded) def test_jstest_unknown_file(self): config = {"roots": ["dir/subdir1/*.js", "dir/subdir1/unknown"]} @@ -426,21 +500,26 @@ class TestFilterTests(unittest.TestCase): def test_json_schema_exclude_files(self): config = {"roots": ["dir/subdir1/*.js", "dir/subdir2/*.js", "dir/subdir3/a/*.js"], "exclude_files": ["dir/subdir2/test21.js"]} - selected = selector.filter_tests("json_schema_test", config, self.test_file_explorer) + selected, excluded = selector.filter_tests("json_schema_test", config, self.test_file_explorer) self.assertEqual(["dir/subdir1/test11.js", "dir/subdir1/test12.js", "dir/subdir3/a/test3a1.js"], selected) + self.assertEqual(["dir/subdir2/test21.js"], excluded) def test_json_shcema_include_files(self): config = {"roots": ["dir/subdir1/*.js", "dir/subdir2/*.js", "dir/subdir3/a/*.js"], "include_files": ["dir/subdir2/test21.js"]} - selected = selector.filter_tests("json_schema_test", config, self.test_file_explorer) + selected, excluded = selector.filter_tests("json_schema_test", config, self.test_file_explorer) self.assertEqual(["dir/subdir2/test21.js"], selected) + self.assertEqual(["dir/subdir1/test11.js", + "dir/subdir1/test12.js", + "dir/subdir3/a/test3a1.js"], excluded) def test_db_tests_all(self): config = {"binary": self.test_file_explorer.binary} - selected = selector.filter_tests("db_test", config, self.test_file_explorer) + selected, excluded = selector.filter_tests("db_test", config, self.test_file_explorer) self.assertEqual(["dbtestA", "dbtestB", "dbtestC"], selected) + self.assertEqual([], excluded) def test_db_tests_roots_override(self): # When roots are specified for db_tests they override all filtering since @@ -448,14 +527,16 @@ class TestFilterTests(unittest.TestCase): config = {"binary": self.test_file_explorer.binary, "include_suites": ["dbtestB"], "roots": ["dbtestOverride"]} - selected = selector.filter_tests("db_test", config, self.test_file_explorer) + selected, excluded = selector.filter_tests("db_test", config, self.test_file_explorer) self.assertEqual(["dbtestOverride"], selected) + self.assertEqual([], excluded) def test_db_tests_include_suites(self): config = {"binary": self.test_file_explorer.binary, "include_suites": ["dbtestB"]} - selected = selector.filter_tests("db_test", config, self.test_file_explorer) + selected, excluded = selector.filter_tests("db_test", config, self.test_file_explorer) self.assertEqual(["dbtestB"], selected) + self.assertEqual(["dbtestA", "dbtestC"], excluded) if __name__ == "__main__": |