SERVER-29924 Display excluded tests in resmoke dry run

4 files changed, 152 insertions, 56 deletions

diff --git a/buildscripts/resmoke.py b/buildscripts/resmoke.py
index 1d5bc7be694..54572a95af6 100755
--- a/buildscripts/resmoke.py
+++ b/buildscripts/resmoke.py
@@ -36,14 +36,21 @@ def _execute_suite(suite):
 
     if resmokelib.config.DRY_RUN == "tests":
         sb = []
-        sb.append("Tests that would be run for %ss in suite %s:"
-                  % (suite.test_kind, suite.get_display_name()))
+        sb.append("Tests that would be run in suite %s:" % suite.get_display_name())
         if len(suite.tests) > 0:
             for test in suite.tests:
                 sb.append(test)
         else:
             sb.append("(no tests)")
         logger.info("\n".join(sb))
+        sb = []
+        sb.append("Tests that would be excluded from suite %s:" % suite.get_display_name())
+        if len(suite.excluded) > 0:
+            for test in suite.excluded:
+                sb.append(test)
+        else:
+            sb.append("(no tests)")
+        logger.info("\n".join(sb))
 
         # Set a successful return code on the test suite because we want to output the tests
         # that would get run by any other suites the user specified.
diff --git a/buildscripts/resmokelib/selector.py b/buildscripts/resmokelib/selector.py
index 3fff181c188..6d80d06be87 100644
--- a/buildscripts/resmokelib/selector.py
+++ b/buildscripts/resmokelib/selector.py
@@ -230,10 +230,14 @@ class _TestList(object):
         The tests are returned in the same order as they are found in the root tests.
         """
         tests = []
+        excluded = []
         for test in self._roots:
-            if test in self._filtered and test not in tests:
-                tests.append(test)
-        return tests
+            if test in self._filtered:
+                if test not in tests:
+                    tests.append(test)
+            elif test not in excluded:
+                excluded.append(test)
+        return tests, excluded
 
 
 ##############################
@@ -406,6 +410,8 @@ class _Selector(object):
 
         Args:
             selector_config: a _SelectorConfig instance.
+        Returns:
+            A tuple with the list of selected tests and the list of excluded tests.
         """
 
         # 1. Find the root files.
@@ -508,25 +514,27 @@ class _DbTestSelector(_Selector):
         _Selector.__init__(self, test_file_explorer, tests_are_files=False)
 
     def select(self, selector_config):
+        if selector_config.roots:
+            roots = selector_config.roots
+        else:
+            if not self._test_file_explorer.isfile(selector_config.binary):
+                raise IOError(errno.ENOENT, "File not found", selector_config.binary)
+            roots = self._test_file_explorer.list_dbtests(selector_config.binary)
+
         if config.INCLUDE_WITH_ANY_TAGS:
             # The db_tests do not currently support tags so we always return an empty array when the
             # --includeWithAnyTags option is used.
-            return []
+            return [], roots
 
         if selector_config.roots:
             # Tests have been specified on the command line. We use them without additional
             # filtering.
-            return selector_config.roots
-
-        if not self._test_file_explorer.isfile(selector_config.binary):
-            raise IOError(errno.ENOENT, "File not found", selector_config.binary)
-
-        dbtests = self._test_file_explorer.list_dbtests(selector_config.binary)
+            return selector_config.roots, []
 
         if not selector_config.include_suites:
-            return dbtests
+            return roots, []
 
-        test_files = _TestList(self._test_file_explorer, dbtests, tests_are_files=False)
+        test_files = _TestList(self._test_file_explorer, roots, tests_are_files=False)
         test_files.include_any_pattern(selector_config.include_suites)
 
         return test_files.get_tests()
diff --git a/buildscripts/resmokelib/testing/suite.py b/buildscripts/resmokelib/testing/suite.py
index 132a2d70d9d..31036f20578 100644
--- a/buildscripts/resmokelib/testing/suite.py
+++ b/buildscripts/resmokelib/testing/suite.py
@@ -42,7 +42,7 @@ class Suite(object):
         self._suite_options = suite_options
 
         self.test_kind = self.get_test_kind_config()
-        self.tests = self._get_tests_for_kind(self.test_kind)
+        self.tests, self.excluded = self._get_tests_for_kind(self.test_kind)
 
         self.return_code = None  # Set by the executor.
 
@@ -70,13 +70,13 @@ class Suite(object):
             mongos_options = test_info  # Just for easier reading.
             if not isinstance(mongos_options, dict):
                 raise TypeError("Expected dictionary of arguments to mongos")
-            return [mongos_options]
+            return [mongos_options], []
 
-        tests = _selector.filter_tests(test_kind, test_info)
+        tests, excluded = _selector.filter_tests(test_kind, test_info)
         if _config.ORDER_TESTS_BY_NAME:
-            return sorted(tests, key=str.lower)
+            return sorted(tests, key=str.lower), sorted(excluded, key=str.lower)
 
-        return tests
+        return tests, excluded
 
     def get_name(self):
         """
diff --git a/buildscripts/tests/resmokelib/test_selector.py b/buildscripts/tests/resmokelib/test_selector.py
index ae8b0a96c8d..83cc48c3a8d 100644
--- a/buildscripts/tests/resmokelib/test_selector.py
+++ b/buildscripts/tests/resmokelib/test_selector.py
@@ -168,18 +168,24 @@ class TestTestList(unittest.TestCase):
     def test_roots(self):
         roots = ["a", "b"]
         test_list = selector._TestList(self.test_file_explorer, roots, tests_are_files=False)
-        self.assertEqual(roots, test_list.get_tests())
+        selected, excluded = test_list.get_tests()
+        self.assertEqual(roots, selected)
+        self.assertEqual([], excluded)
 
     def test_roots_with_glob(self):
         glob_roots = ["dir/subdir1/*.js"]
         expected_roots = ["dir/subdir1/test11.js", "dir/subdir1/test12.js"]
         test_list = selector._TestList(self.test_file_explorer, glob_roots)
-        self.assertEqual(expected_roots, test_list.get_tests())
+        selected, excluded = test_list.get_tests()
+        self.assertEqual(expected_roots, selected)
+        self.assertEqual([], excluded)
 
     def test_roots_with_unmatching_glob(self):
         glob_roots = ["unknown/subdir1/*.js"]
-        with self.assertRaisesRegexp(ValueError, "Pattern does not match any files: unknown/subdir1/\*.js"):
-            selector._TestList(self.test_file_explorer, glob_roots)
+        test_list = selector._TestList(self.test_file_explorer, glob_roots)
+        selected, excluded = test_list.get_tests()
+        self.assertEqual([], selected)
+        self.assertEqual([], excluded)
 
     def test_roots_unknown_file(self):
         roots = ["dir/subdir1/unknown"]
@@ -190,20 +196,29 @@ class TestTestList(unittest.TestCase):
         roots = ["dir/subdir1/*.js", "dir/subdir2/test21.*"]
         test_list = selector._TestList(self.test_file_explorer, roots)
         test_list.include_files(["dir/subdir2/test21.js"])
-        self.assertEqual(["dir/subdir2/test21.js"], test_list.get_tests())
+        selected, excluded = test_list.get_tests()
+        self.assertEqual(["dir/subdir2/test21.js"], selected)
+        self.assertEqual(["dir/subdir1/test11.js",
+                          "dir/subdir1/test12.js"], excluded)
 
     def test_include_files_no_match(self):
         roots = ["dir/subdir1/*.js", "dir/subdir2/test21.*"]
         test_list = selector._TestList(self.test_file_explorer, roots)
         test_list.include_files(["dir/subdir2/test26.js"])
-        self.assertEqual([], test_list.get_tests())
+        selected, excluded = test_list.get_tests()
+        self.assertEqual([], selected)
+        self.assertEqual(["dir/subdir1/test11.js",
+                          "dir/subdir1/test12.js",
+                          "dir/subdir2/test21.js"], excluded)
 
     def test_exclude_files(self):
         roots = ["dir/subdir1/*.js", "dir/subdir2/test21.*"]
         test_list = selector._TestList(self.test_file_explorer, roots)
         test_list.exclude_files(["dir/subdir2/test21.js"])
+        selected, excluded = test_list.get_tests()
         self.assertEqual(["dir/subdir1/test11.js",
-                          "dir/subdir1/test12.js"], test_list.get_tests())
+                          "dir/subdir1/test12.js"], selected)
+        self.assertEqual(["dir/subdir2/test21.js"], excluded)
 
     def test_exclude_files_no_match(self):
         roots = ["dir/subdir1/*.js", "dir/subdir2/test21.*"]
@@ -215,8 +230,10 @@ class TestTestList(unittest.TestCase):
         roots = ["dir/subdir1/*.js", "dir/subdir2/test21.*"]
         test_list = selector._TestList(self.test_file_explorer, roots)
         test_list.exclude_files(["dir/subdir2/*.js"])
+        selected, excluded = test_list.get_tests()
         self.assertEqual(["dir/subdir1/test11.js",
-                          "dir/subdir1/test12.js"], test_list.get_tests())
+                          "dir/subdir1/test12.js"], selected)
+        self.assertEqual(["dir/subdir2/test21.js"], excluded)
 
     def test_match_tag_expression(self):
         roots = ["dir/subdir1/*.js", "dir/subdir2/test21.*"]
@@ -230,47 +247,76 @@ class TestTestList(unittest.TestCase):
             return self.test_file_explorer.jstest_tags(test_file)
 
         test_list.match_tag_expression(expression, get_tags)
+        selected, excluded = test_list.get_tests()
         self.assertEqual(["dir/subdir1/test11.js",
-                          "dir/subdir1/test12.js"], test_list.get_tests())
+                          "dir/subdir1/test12.js"], selected)
+        self.assertEqual(["dir/subdir2/test21.js"], excluded)
 
     def test_include_any_pattern(self):
         roots = ["dir/subdir1/*.js", "dir/subdir2/test21.*", "dir/subdir3/a/test3a1.js"]
         # 1 pattern and 1 matching
         test_list = selector._TestList(self.test_file_explorer, roots)
         test_list.include_any_pattern(["dir/*3/a/*"])
-        self.assertEqual(["dir/subdir3/a/test3a1.js"], test_list.get_tests())
+        selected, excluded = test_list.get_tests()
+        self.assertEqual(["dir/subdir3/a/test3a1.js"], selected)
+        self.assertEqual(["dir/subdir1/test11.js",
+                          "dir/subdir1/test12.js",
+                          "dir/subdir2/test21.js"], excluded)
         # 1 pattern and 0 matching
         test_list = selector._TestList(self.test_file_explorer, roots)
         test_list.include_any_pattern(["dir/*4/a/*"])
-        self.assertEqual([], test_list.get_tests())
+        selected, excluded = test_list.get_tests()
+        self.assertEqual([], selected)
+        self.assertEqual(["dir/subdir1/test11.js",
+                          "dir/subdir1/test12.js",
+                          "dir/subdir2/test21.js",
+                          "dir/subdir3/a/test3a1.js"], excluded)
         # 3 patterns and 1 matching
         test_list = selector._TestList(self.test_file_explorer, roots)
         test_list.include_any_pattern(["dir/*3/a/*", "notmaching/*", "notmatching2/*.js"])
-        self.assertEqual(["dir/subdir3/a/test3a1.js"], test_list.get_tests())
+        selected, excluded = test_list.get_tests()
+        self.assertEqual(["dir/subdir3/a/test3a1.js"], selected)
+        self.assertEqual(["dir/subdir1/test11.js",
+                          "dir/subdir1/test12.js",
+                          "dir/subdir2/test21.js"], excluded)
         # 3 patterns and 0 matching
         test_list = selector._TestList(self.test_file_explorer, roots)
         test_list.include_any_pattern(["dir2/*3/a/*", "notmaching/*", "notmatching2/*.js"])
-        self.assertEqual([], test_list.get_tests())
+        selected, excluded = test_list.get_tests()
+        self.assertEqual([], selected)
+        self.assertEqual(["dir/subdir1/test11.js",
+                          "dir/subdir1/test12.js",
+                          "dir/subdir2/test21.js",
+                          "dir/subdir3/a/test3a1.js"], excluded)
         # 3 patterns and 3 matching
         test_list = selector._TestList(self.test_file_explorer, roots)
         test_list.include_any_pattern(["dir/*1/*11*", "dir/subdir3/**", "dir/subdir2/*.js"])
+        selected, excluded = test_list.get_tests()
         self.assertEqual(["dir/subdir1/test11.js", "dir/subdir2/test21.js",
                           "dir/subdir3/a/test3a1.js"],
-                         test_list.get_tests())
+                         selected)
+        self.assertEqual(["dir/subdir1/test12.js"], excluded)
 
     def test_include_tests_no_force(self):
         roots = ["dir/subdir1/*.js", "dir/subdir2/test21.*"]
         test_list = selector._TestList(self.test_file_explorer, roots)
         test_list.exclude_files(["dir/subdir1/test11.js"])
         test_list.include_files(["dir/subdir1/test11.js"], force=False)
-        self.assertEqual([], test_list.get_tests())
+        selected, excluded = test_list.get_tests()
+        self.assertEqual([], selected)
+        self.assertEqual(["dir/subdir1/test11.js",
+                          "dir/subdir1/test12.js",
+                          "dir/subdir2/test21.js"], excluded)
 
     def test_include_tests_force(self):
         roots = ["dir/subdir1/*.js", "dir/subdir2/test21.*"]
         test_list = selector._TestList(self.test_file_explorer, roots)
         test_list.exclude_files(["dir/subdir1/test11.js"])
         test_list.include_files(["dir/subdir1/test11.js"], force=True)
-        self.assertEqual(["dir/subdir1/test11.js"], test_list.get_tests())
+        selected, excluded = test_list.get_tests()
+        self.assertEqual(["dir/subdir1/test11.js"], selected)
+        self.assertEqual(["dir/subdir1/test12.js",
+                          "dir/subdir2/test21.js"], excluded)
 
     def test_tests_are_not_files(self):
         roots = ["a", "b"]
@@ -299,41 +345,54 @@ class TestSelector(unittest.TestCase):
     def test_select_all(self):
         config = selector._SelectorConfig(roots=["dir/subdir1/*.js", "dir/subdir2/*.js",
                                                  "dir/subdir3/a/*.js"])
-        selected = self.selector.select(config)
+        selected, excluded = self.selector.select(config)
         self.assertEqual(["dir/subdir1/test11.js",
                           "dir/subdir1/test12.js",
                           "dir/subdir2/test21.js",
                           "dir/subdir3/a/test3a1.js"], selected)
+        self.assertEqual([], excluded)
 
     def test_select_exclude_files(self):
         config = selector._SelectorConfig(roots=["dir/subdir1/*.js", "dir/subdir2/*.js",
                                                  "dir/subdir3/a/*.js"],
                                           exclude_files=["dir/subdir2/test21.js"])
-        selected = self.selector.select(config)
+        selected, excluded = self.selector.select(config)
         self.assertEqual(["dir/subdir1/test11.js",
                           "dir/subdir1/test12.js",
                           "dir/subdir3/a/test3a1.js"], selected)
+        self.assertEqual(["dir/subdir2/test21.js"], excluded)
 
     def test_select_include_files(self):
         config = selector._SelectorConfig(roots=["dir/subdir1/*.js", "dir/subdir2/*.js",
                                                  "dir/subdir3/a/*.js"],
                                           include_files=["dir/subdir2/test21.js"])
-        selected = self.selector.select(config)
+        selected, excluded = self.selector.select(config)
         self.assertEqual(["dir/subdir2/test21.js"], selected)
+        self.assertEqual(["dir/subdir1/test11.js",
+                          "dir/subdir1/test12.js",
+                          "dir/subdir3/a/test3a1.js"], excluded)
 
     def test_select_include_tags(self):
         config = selector._SelectorConfig(roots=["dir/subdir1/*.js", "dir/subdir2/*.js",
                                                  "dir/subdir3/a/*.js"],
                                           include_tags="tag1")
-        selected = self.selector.select(config)
+        selected, excluded = self.selector.select(config)
         self.assertEqual([], selected)
+        self.assertEqual(["dir/subdir1/test11.js",
+                          "dir/subdir1/test12.js",
+                          "dir/subdir2/test21.js",
+                          "dir/subdir3/a/test3a1.js"], excluded)
 
     def test_select_include_any_tags(self):
         config = selector._SelectorConfig(roots=["dir/subdir1/*.js", "dir/subdir2/*.js",
                                                  "dir/subdir3/a/*.js"],
                                           include_with_any_tags=["tag1"])
-        selected = self.selector.select(config)
+        selected, excluded = self.selector.select(config)
         self.assertEqual([], selected)
+        self.assertEqual(["dir/subdir1/test11.js",
+                          "dir/subdir1/test12.js",
+                          "dir/subdir2/test21.js",
+                          "dir/subdir3/a/test3a1.js"], excluded)
 
 
 class TestFilterTests(unittest.TestCase):
@@ -348,75 +407,90 @@ class TestFilterTests(unittest.TestCase):
 
     def test_cpp_all(self):
         config = {"root": "integrationtest.txt"}
-        selected = selector.filter_tests("cpp_integration_test", config, self.test_file_explorer)
+        selected, excluded = selector.filter_tests("cpp_integration_test", config, self.test_file_explorer)
         self.assertEqual(["build/testA", "build/testB"], selected)
+        self.assertEqual([], excluded)
 
     def test_cpp_roots_override(self):
         # When roots are specified for cpp tests they override all filtering since
         # 'roots' are populated with the command line arguments.
         config = {"include_files": "unknown_file",
                   "roots": ["build/testC"]}
-        selected = selector.filter_tests("cpp_unit_test", config, self.test_file_explorer)
+        selected, excluded = selector.filter_tests("cpp_unit_test", config, self.test_file_explorer)
         self.assertEqual(["build/testC"], selected)
-        selected = selector.filter_tests("cpp_integration_test", config, self.test_file_explorer)
+        self.assertEqual([], excluded)
+        selected, excluded = selector.filter_tests("cpp_integration_test", config, self.test_file_explorer)
         self.assertEqual(["build/testC"], selected)
+        self.assertEqual([], excluded)
 
     def test_cpp_expand_roots(self):
         config = {"root": "integrationtest.txt", "roots": ["build/test*"]}
-        selected = selector.filter_tests("cpp_integration_test", config, self.test_file_explorer)
+        selected, excluded = selector.filter_tests("cpp_integration_test", config, self.test_file_explorer)
         self.assertEqual(["build/testA", "build/testB", "build/testC"], selected)
+        self.assertEqual([], excluded)
 
-        selected = selector.filter_tests("cpp_unit_test", config, self.test_file_explorer)
+        selected, excluded = selector.filter_tests("cpp_unit_test", config, self.test_file_explorer)
         self.assertEqual(["build/testA", "build/testB", "build/testC"], selected)
+        self.assertEqual([], excluded)
 
     def test_cpp_with_any_tags(self):
         buildscripts.resmokelib.config.INCLUDE_WITH_ANY_TAGS = ["tag1"]
         try:
             selector_config = {"root": "unittest.txt"}
-            selected = selector.filter_tests(
+            selected, excluded = selector.filter_tests(
                 "cpp_unit_test",
                 selector_config,
                 test_file_explorer=self.test_file_explorer)
             self.assertEqual([], selected)
+            self.assertEqual(["build/testA", "build/testB"], excluded)
         finally:
             buildscripts.resmokelib.config.INCLUDE_WITH_ANY_TAGS = None
 
     def test_jstest_include_tags(self):
         config = {"roots": ["dir/subdir1/*.js", "dir/subdir2/*.js", "dir/subdir3/a/*.js"],
                   "include_tags": "tag1"}
-        selected = selector.filter_tests("js_test", config, self.test_file_explorer)
+        selected, excluded = selector.filter_tests("js_test", config, self.test_file_explorer)
         self.assertEqual(["dir/subdir1/test11.js"], selected)
+        self.assertEqual(["dir/subdir1/test12.js",
+                          "dir/subdir2/test21.js",
+                          "dir/subdir3/a/test3a1.js"], excluded)
 
     def test_jstest_exclude_tags(self):
         config = {"roots": ["dir/subdir1/*.js", "dir/subdir2/*.js", "dir/subdir3/a/*.js"],
                   "exclude_tags": "tag1"}
-        selected = selector.filter_tests("js_test", config, self.test_file_explorer)
+        selected, excluded = selector.filter_tests("js_test", config, self.test_file_explorer)
         self.assertEqual(["dir/subdir1/test12.js",
                           "dir/subdir2/test21.js",
                           "dir/subdir3/a/test3a1.js"], selected)
+        self.assertEqual(["dir/subdir1/test11.js"], excluded)
 
     def test_jstest_force_include(self):
         config = {"roots": ["dir/subdir1/*.js", "dir/subdir2/*.js", "dir/subdir3/a/*.js"],
                   "include_files": ["dir/subdir1/*.js"],
                   "exclude_tags": "tag1"}
-        selected = selector.filter_tests("js_test", config, self.test_file_explorer)
+        selected, excluded = selector.filter_tests("js_test", config, self.test_file_explorer)
         self.assertEqual(["dir/subdir1/test11.js",
                           "dir/subdir1/test12.js"], selected)
+        self.assertEqual(["dir/subdir2/test21.js",
+                          "dir/subdir3/a/test3a1.js"], excluded)
 
     def test_jstest_all(self):
         config = {"roots": ["dir/subdir1/*.js", "dir/subdir2/*.js", "dir/subdir3/a/*.js"]}
-        selected = selector.filter_tests("js_test", config, self.test_file_explorer)
+        selected, excluded = selector.filter_tests("js_test", config, self.test_file_explorer)
         self.assertEqual(["dir/subdir1/test11.js",
                           "dir/subdir1/test12.js",
                           "dir/subdir2/test21.js",
                           "dir/subdir3/a/test3a1.js"], selected)
+        self.assertEqual([], excluded)
 
     def test_jstest_include_with_any_tags(self):
         config = {"roots": ["dir/subdir1/*.js", "dir/subdir2/*.js", "dir/subdir3/a/*.js"],
                   "include_with_any_tags": ["tag2"]}
-        selected = selector.filter_tests("js_test", config, self.test_file_explorer)
+        selected, excluded = selector.filter_tests("js_test", config, self.test_file_explorer)
         self.assertEqual(["dir/subdir1/test11.js",
                           "dir/subdir2/test21.js"], selected)
+        self.assertEqual(["dir/subdir1/test12.js",
+                          "dir/subdir3/a/test3a1.js"], excluded)
 
     def test_jstest_unknown_file(self):
         config = {"roots": ["dir/subdir1/*.js", "dir/subdir1/unknown"]}
@@ -426,21 +500,26 @@ class TestFilterTests(unittest.TestCase):
     def test_json_schema_exclude_files(self):
         config = {"roots": ["dir/subdir1/*.js", "dir/subdir2/*.js", "dir/subdir3/a/*.js"],
                   "exclude_files": ["dir/subdir2/test21.js"]}
-        selected = selector.filter_tests("json_schema_test", config, self.test_file_explorer)
+        selected, excluded = selector.filter_tests("json_schema_test", config, self.test_file_explorer)
         self.assertEqual(["dir/subdir1/test11.js",
                           "dir/subdir1/test12.js",
                           "dir/subdir3/a/test3a1.js"], selected)
+        self.assertEqual(["dir/subdir2/test21.js"], excluded)
 
     def test_json_shcema_include_files(self):
         config = {"roots": ["dir/subdir1/*.js", "dir/subdir2/*.js", "dir/subdir3/a/*.js"],
                   "include_files": ["dir/subdir2/test21.js"]}
-        selected = selector.filter_tests("json_schema_test", config, self.test_file_explorer)
+        selected, excluded = selector.filter_tests("json_schema_test", config, self.test_file_explorer)
         self.assertEqual(["dir/subdir2/test21.js"], selected)
+        self.assertEqual(["dir/subdir1/test11.js",
+                          "dir/subdir1/test12.js",
+                          "dir/subdir3/a/test3a1.js"], excluded)
 
     def test_db_tests_all(self):
         config = {"binary": self.test_file_explorer.binary}
-        selected = selector.filter_tests("db_test", config, self.test_file_explorer)
+        selected, excluded = selector.filter_tests("db_test", config, self.test_file_explorer)
         self.assertEqual(["dbtestA", "dbtestB", "dbtestC"], selected)
+        self.assertEqual([], excluded)
 
     def test_db_tests_roots_override(self):
         # When roots are specified for db_tests they override all filtering since
@@ -448,14 +527,16 @@ class TestFilterTests(unittest.TestCase):
         config = {"binary": self.test_file_explorer.binary,
                   "include_suites": ["dbtestB"],
                   "roots": ["dbtestOverride"]}
-        selected = selector.filter_tests("db_test", config, self.test_file_explorer)
+        selected, excluded = selector.filter_tests("db_test", config, self.test_file_explorer)
         self.assertEqual(["dbtestOverride"], selected)
+        self.assertEqual([], excluded)
 
     def test_db_tests_include_suites(self):
         config = {"binary": self.test_file_explorer.binary,
                   "include_suites": ["dbtestB"]}
-        selected = selector.filter_tests("db_test", config, self.test_file_explorer)
+        selected, excluded = selector.filter_tests("db_test", config, self.test_file_explorer)
         self.assertEqual(["dbtestB"], selected)
+        self.assertEqual(["dbtestA", "dbtestC"], excluded)
 
 
 if __name__ == "__main__":