path: root/buildscripts/tests/resmokelib/test_selector.py

"""Unit tests for the buildscripts.resmokelib.selector module."""

from __future__ import absolute_import

import fnmatch
import unittest

import buildscripts.resmokelib.selector as selector
import buildscripts.resmokelib.utils.globstar as globstar
import buildscripts.resmokelib.config
import buildscripts.resmokeconfig


class TestExpressions(unittest.TestCase):
    """Unit tests for the tag matching expressions."""

    def test_match_expression(self):
        tag = "test_tag"
        tags_with = ["other_tag", tag]
        tags_without = ["other_tag", "some_tag"]
        expression = selector.make_expression(tag)
        self.assertIsInstance(expression, selector._MatchExpression)
        self.assertTrue(expression(tags_with))
        self.assertFalse(expression(tags_without))
        self.assertFalse(expression([]))

    def test_allof_expression(self):
        tag1 = "test_tag"
        tag2 = "other_tag"
        tags_match = [tag2, tag1, "third_tag"]
        tags_nomatch = [tag2, "some_tag"]
        expression = selector.make_expression({"$allOf": [tag1, tag2]})
        self.assertIsInstance(expression, selector._AllOfExpression)
        self.assertTrue(expression(tags_match))
        self.assertFalse(expression(tags_nomatch))
        self.assertFalse(expression([]))

    def test_anyof_expression(self):
        tag1 = "test_tag"
        tag2 = "other_tag"
        tags_match = [tag1, "third_tag"]
        tags_nomatch = ["third_tag", "some_tag"]
        expression = selector.make_expression({"$anyOf": [tag1, tag2]})
        self.assertIsInstance(expression, selector._AnyOfExpression)
        self.assertTrue(expression(tags_match))
        self.assertFalse(expression(tags_nomatch))
        self.assertFalse(expression([]))

    def test_not_expression(self):
        tag = "test_tag"
        tags_match = ["other_tag_1"]
        tags_nomatch = ["other_tag_1", tag]
        expression = selector.make_expression({"$not": tag})
        self.assertIsInstance(expression, selector._NotExpression)
        self.assertTrue(expression(tags_match))
        self.assertTrue(expression([]))
        self.assertFalse(expression(tags_nomatch))

    def test_allof_anyof_expression(self):
        tag1 = "test_tag_1"
        tag2 = "test_tag_2"
        tag3 = "test_tag_3"
        tags_match_1 = [tag1, tag3]
        tags_match_2 = [tag2, tag3]
        tags_nomatch_1 = ["other_tag_1", tag3]
        tags_nomatch_2 = [tag1, "other_tag_2"]
        tags_nomatch_3 = [tag2, "other_tag_2"]
        tags_nomatch_4 = [tag2]
        tags_nomatch_5 = ["other_tag_2"]
        expression = selector.make_expression({"$allOf": [
            {"$anyOf": [tag1, tag2]},
            tag3,
        ]})
        self.assertIsInstance(expression, selector._AllOfExpression)
        self.assertTrue(expression(tags_match_1))
        self.assertTrue(expression(tags_match_2))
        self.assertFalse(expression(tags_nomatch_1))
        self.assertFalse(expression(tags_nomatch_2))
        self.assertFalse(expression(tags_nomatch_3))
        self.assertFalse(expression(tags_nomatch_4))
        self.assertFalse(expression(tags_nomatch_5))
        self.assertFalse(expression([]))

    def test_invalid_expression(self):
        with self.assertRaises(ValueError):
            selector.make_expression({"invalid": ["tag1", "tag2"]})
        with self.assertRaises(ValueError):
            selector.make_expression({"$anyOf": ["tag1", "tag2"], "invalid": "tag3"})


class TestTestFileExplorer(unittest.TestCase):
    @classmethod
    def setUpClass(cls):
        cls.test_file_explorer = selector.TestFileExplorer()

    def test_is_glob_pattern(self):
        self.assertTrue(self.test_file_explorer.is_glob_pattern("directory/*file.js"))
        self.assertFalse(self.test_file_explorer.is_glob_pattern("directory/file.js"))

    def test_fnmatchcase(self):
        pattern = "dir*/file.js"
        self.assertTrue(self.test_file_explorer.fnmatchcase("directory/file.js", pattern))
        self.assertFalse(self.test_file_explorer.fnmatchcase("other/file.js", pattern))


class MockTestFileExplorer(object):
    """Component giving access to mock test files data."""

    def __init__(self):
        self.files = [
            "dir/subdir1/test11.js", "dir/subdir1/test12.js", "dir/subdir2/test21.js",
            "dir/subdir3/a/test3a1.js", "build/testA", "build/testB", "build/testC", "dbtest"
        ]
        self.tags = {
            "dir/subdir1/test11.js": ["tag1", "tag2"], "dir/subdir1/test12.js": ["tag3"],
            "dir/subdir2/test21.js": ["tag2", "tag4"], "dir/subdir3/a/test3a1.js": ["tag4", "tag5"]
        }
        self.binary = "dbtest"
        self.jstest_tag_file = {"dir/subdir1/test11.js": "tagA", "dir/subdir3/a/test3a1.js": "tagB"}

    def is_glob_pattern(self, pattern):
        return globstar.is_glob_pattern(pattern)

    def iglob(self, pattern):
        globbed = []
        for test_file in self.files:
            if fnmatch.fnmatchcase(test_file, pattern):
                globbed.append(test_file)
        return globbed

    def jstest_tags(self, file_path):
        return self.tags.get(file_path, [])

    def read_root_file(self, root_file_path):
        return ["build/testA", "build/testB"]

    def fnmatchcase(self, name, pattern):
        return fnmatch.fnmatchcase(name, pattern)

    def isfile(self, path):
        return path in self.files

    def list_dbtests(self, binary):
        return ["dbtestA", "dbtestB", "dbtestC"]

    def parse_tag_file(self, test_kind):
        if test_kind == "js_test":
            return self.jstest_tag_file


class TestTestList(unittest.TestCase):
    @classmethod
    def setUpClass(cls):
        cls.test_file_explorer = MockTestFileExplorer()
        cls.tags_from_file = cls.test_file_explorer.jstest_tag_file

    def test_roots(self):
        roots = ["a", "b"]
        test_list = selector._TestList(self.test_file_explorer, roots, tests_are_files=False)
        selected, excluded = test_list.get_tests()
        self.assertEqual(roots, selected)
        self.assertEqual([], excluded)

    def test_roots_with_glob(self):
        glob_roots = ["dir/subdir1/*.js"]
        expected_roots = ["dir/subdir1/test11.js", "dir/subdir1/test12.js"]
        test_list = selector._TestList(self.test_file_explorer, glob_roots)
        selected, excluded = test_list.get_tests()
        self.assertEqual(expected_roots, selected)
        self.assertEqual([], excluded)

    def test_roots_with_unmatching_glob(self):
        glob_roots = ["unknown/subdir1/*.js"]
        test_list = selector._TestList(self.test_file_explorer, glob_roots)
        selected, excluded = test_list.get_tests()
        self.assertEqual([], selected)
        self.assertEqual([], excluded)

    def test_roots_unknown_file(self):
        roots = ["dir/subdir1/unknown"]
        with self.assertRaisesRegexp(ValueError, "Unrecognized test file: dir/subdir1/unknown"):
            selector._TestList(self.test_file_explorer, roots, tests_are_files=True)

    def test_include_files(self):
        roots = ["dir/subdir1/*.js", "dir/subdir2/test21.*"]
        test_list = selector._TestList(self.test_file_explorer, roots)
        test_list.include_files(["dir/subdir2/test21.js"])
        selected, excluded = test_list.get_tests()
        self.assertEqual(["dir/subdir2/test21.js"], selected)
        self.assertEqual(["dir/subdir1/test11.js", "dir/subdir1/test12.js"], excluded)

    def test_include_files_no_match(self):
        roots = ["dir/subdir1/*.js", "dir/subdir2/test21.*"]
        test_list = selector._TestList(self.test_file_explorer, roots)
        test_list.include_files(["dir/subdir2/test26.js"])
        selected, excluded = test_list.get_tests()
        self.assertEqual([], selected)
        self.assertEqual(
            ["dir/subdir1/test11.js", "dir/subdir1/test12.js", "dir/subdir2/test21.js"], excluded)

    def test_exclude_files(self):
        roots = ["dir/subdir1/*.js", "dir/subdir2/test21.*"]
        test_list = selector._TestList(self.test_file_explorer, roots)
        test_list.exclude_files(["dir/subdir2/test21.js"])
        selected, excluded = test_list.get_tests()
        self.assertEqual(["dir/subdir1/test11.js", "dir/subdir1/test12.js"], selected)
        self.assertEqual(["dir/subdir2/test21.js"], excluded)

    def test_exclude_files_no_match(self):
        roots = ["dir/subdir1/*.js", "dir/subdir2/test21.*"]
        test_list = selector._TestList(self.test_file_explorer, roots)
        with self.assertRaisesRegexp(ValueError, "Unrecognized test file: .*$"):
            test_list.exclude_files(["dir/subdir2/test26.js"])

    def test_exclude_files_glob(self):
        roots = ["dir/subdir1/*.js", "dir/subdir2/test21.*"]
        test_list = selector._TestList(self.test_file_explorer, roots)
        test_list.exclude_files(["dir/subdir2/*.js"])
        selected, excluded = test_list.get_tests()
        self.assertEqual(["dir/subdir1/test11.js", "dir/subdir1/test12.js"], selected)
        self.assertEqual(["dir/subdir2/test21.js"], excluded)

    def test_match_tag_expression(self):
        roots = ["dir/subdir1/*.js", "dir/subdir2/test21.*"]
        test_list = selector._TestList(self.test_file_explorer, roots)
        expression = selector.make_expression(
            {"$anyOf": [{"$allOf": ["tag1", "tag2"]}, "tag3", {"$allOf": ["tag5", "tag6"]}]})

        def get_tags(test_file):
            return self.test_file_explorer.jstest_tags(test_file)

        test_list.match_tag_expression(expression, get_tags)
        selected, excluded = test_list.get_tests()
        self.assertEqual(["dir/subdir1/test11.js", "dir/subdir1/test12.js"], selected)
        self.assertEqual(["dir/subdir2/test21.js"], excluded)

    def test_include_any_pattern(self):
        roots = ["dir/subdir1/*.js", "dir/subdir2/test21.*", "dir/subdir3/a/test3a1.js"]
        # 1 pattern and 1 matching
        test_list = selector._TestList(self.test_file_explorer, roots)
        test_list.include_any_pattern(["dir/*3/a/*"])
        selected, excluded = test_list.get_tests()
        self.assertEqual(["dir/subdir3/a/test3a1.js"], selected)
        self.assertEqual(
            ["dir/subdir1/test11.js", "dir/subdir1/test12.js", "dir/subdir2/test21.js"], excluded)
        # 1 pattern and 0 matching
        test_list = selector._TestList(self.test_file_explorer, roots)
        test_list.include_any_pattern(["dir/*4/a/*"])
        selected, excluded = test_list.get_tests()
        self.assertEqual([], selected)
        self.assertEqual([
            "dir/subdir1/test11.js", "dir/subdir1/test12.js", "dir/subdir2/test21.js",
            "dir/subdir3/a/test3a1.js"
        ], excluded)
        # 3 patterns and 1 matching
        test_list = selector._TestList(self.test_file_explorer, roots)
        test_list.include_any_pattern(["dir/*3/a/*", "notmaching/*", "notmatching2/*.js"])
        selected, excluded = test_list.get_tests()
        self.assertEqual(["dir/subdir3/a/test3a1.js"], selected)
        self.assertEqual(
            ["dir/subdir1/test11.js", "dir/subdir1/test12.js", "dir/subdir2/test21.js"], excluded)
        # 3 patterns and 0 matching
        test_list = selector._TestList(self.test_file_explorer, roots)
        test_list.include_any_pattern(["dir2/*3/a/*", "notmaching/*", "notmatching2/*.js"])
        selected, excluded = test_list.get_tests()
        self.assertEqual([], selected)
        self.assertEqual([
            "dir/subdir1/test11.js", "dir/subdir1/test12.js", "dir/subdir2/test21.js",
            "dir/subdir3/a/test3a1.js"
        ], excluded)
        # 3 patterns and 3 matching
        test_list = selector._TestList(self.test_file_explorer, roots)
        test_list.include_any_pattern(["dir/*1/*11*", "dir/subdir3/**", "dir/subdir2/*.js"])
        selected, excluded = test_list.get_tests()
        self.assertEqual(
            ["dir/subdir1/test11.js", "dir/subdir2/test21.js", "dir/subdir3/a/test3a1.js"],
            selected)
        self.assertEqual(["dir/subdir1/test12.js"], excluded)

    def test_include_tests_no_force(self):
        roots = ["dir/subdir1/*.js", "dir/subdir2/test21.*"]
        test_list = selector._TestList(self.test_file_explorer, roots)
        test_list.exclude_files(["dir/subdir1/test11.js"])
        test_list.include_files(["dir/subdir1/test11.js"], force=False)
        selected, excluded = test_list.get_tests()
        self.assertEqual([], selected)
        self.assertEqual(
            ["dir/subdir1/test11.js", "dir/subdir1/test12.js", "dir/subdir2/test21.js"], excluded)

    def test_include_tests_force(self):
        roots = ["dir/subdir1/*.js", "dir/subdir2/test21.*"]
        test_list = selector._TestList(self.test_file_explorer, roots)
        test_list.exclude_files(["dir/subdir1/test11.js"])
        test_list.include_files(["dir/subdir1/test11.js"], force=True)
        selected, excluded = test_list.get_tests()
        self.assertEqual(["dir/subdir1/test11.js"], selected)
        self.assertEqual(["dir/subdir1/test12.js", "dir/subdir2/test21.js"], excluded)

    def test_tests_are_not_files(self):
        roots = ["a", "b"]
        test_list = selector._TestList(self.test_file_explorer, roots, tests_are_files=False)
        with self.assertRaises(TypeError):
            test_list.include_files([])
        with self.assertRaises(TypeError):
            test_list.exclude_files([])


class TestSelectorConfig(unittest.TestCase):
    def test_root_roots(self):
        with self.assertRaises(ValueError):
            selector._SelectorConfig(root="path_to_root", roots=["test1", "test2"])

    def test_include_exclude_tags(self):
        with self.assertRaises(ValueError):
            selector._SelectorConfig(include_tags="tag1", exclude_tags="tag2")


class TestSelector(unittest.TestCase):
    @classmethod
    def setUpClass(cls):
        cls.selector = selector._Selector(MockTestFileExplorer())

    def test_select_all(self):
        config = selector._SelectorConfig(
            roots=["dir/subdir1/*.js", "dir/subdir2/*.js", "dir/subdir3/a/*.js"])
        selected, excluded = self.selector.select(config)
        self.assertEqual([
            "dir/subdir1/test11.js", "dir/subdir1/test12.js", "dir/subdir2/test21.js",
            "dir/subdir3/a/test3a1.js"
        ], selected)
        self.assertEqual([], excluded)

    def test_select_exclude_files(self):
        config = selector._SelectorConfig(
            roots=["dir/subdir1/*.js", "dir/subdir2/*.js",
                   "dir/subdir3/a/*.js"], exclude_files=["dir/subdir2/test21.js"])
        selected, excluded = self.selector.select(config)
        self.assertEqual(
            ["dir/subdir1/test11.js", "dir/subdir1/test12.js", "dir/subdir3/a/test3a1.js"],
            selected)
        self.assertEqual(["dir/subdir2/test21.js"], excluded)

    def test_select_include_files(self):
        config = selector._SelectorConfig(
            roots=["dir/subdir1/*.js", "dir/subdir2/*.js",
                   "dir/subdir3/a/*.js"], include_files=["dir/subdir2/test21.js"])
        selected, excluded = self.selector.select(config)
        self.assertEqual(["dir/subdir2/test21.js"], selected)
        self.assertEqual(
            ["dir/subdir1/test11.js", "dir/subdir1/test12.js", "dir/subdir3/a/test3a1.js"],
            excluded)

    def test_select_include_tags(self):
        config = selector._SelectorConfig(
            roots=["dir/subdir1/*.js", "dir/subdir2/*.js",
                   "dir/subdir3/a/*.js"], include_tags="tag1")
        selected, excluded = self.selector.select(config)
        self.assertEqual([], selected)
        self.assertEqual([
            "dir/subdir1/test11.js", "dir/subdir1/test12.js", "dir/subdir2/test21.js",
            "dir/subdir3/a/test3a1.js"
        ], excluded)

    def test_select_include_any_tags(self):
        config = selector._SelectorConfig(
            roots=["dir/subdir1/*.js", "dir/subdir2/*.js",
                   "dir/subdir3/a/*.js"], include_with_any_tags=["tag1"])
        selected, excluded = self.selector.select(config)
        self.assertEqual([], selected)
        self.assertEqual([
            "dir/subdir1/test11.js", "dir/subdir1/test12.js", "dir/subdir2/test21.js",
            "dir/subdir3/a/test3a1.js"
        ], excluded)


class TestFilterTests(unittest.TestCase):
    @classmethod
    def setUpClass(cls):
        cls.test_file_explorer = MockTestFileExplorer()

    def test_unknown_test_kind(self):
        with self.assertRaises(ValueError):
            selector.filter_tests("unknown_test", {})

    def test_cpp_all(self):
        config = {"root": "integrationtest.txt"}
        selected, excluded = selector.filter_tests("cpp_integration_test", config,
                                                   self.test_file_explorer)
        self.assertEqual(["build/testA", "build/testB"], selected)
        self.assertEqual([], excluded)

    def test_cpp_roots_override(self):
        # When roots are specified for cpp tests they override all filtering since
        # 'roots' are populated with the command line arguments.
        config = {"include_files": "unknown_file", "roots": ["build/testC"]}
        selected, excluded = selector.filter_tests("cpp_unit_test", config, self.test_file_explorer)
        self.assertEqual(["build/testC"], selected)
        self.assertEqual([], excluded)
        selected, excluded = selector.filter_tests("cpp_integration_test", config,
                                                   self.test_file_explorer)
        self.assertEqual(["build/testC"], selected)
        self.assertEqual([], excluded)

    def test_cpp_expand_roots(self):
        config = {"root": "integrationtest.txt", "roots": ["build/test*"]}
        selected, excluded = selector.filter_tests("cpp_integration_test", config,
                                                   self.test_file_explorer)
        self.assertEqual(["build/testA", "build/testB", "build/testC"], selected)
        self.assertEqual([], excluded)

        selected, excluded = selector.filter_tests("cpp_unit_test", config, self.test_file_explorer)
        self.assertEqual(["build/testA", "build/testB", "build/testC"], selected)
        self.assertEqual([], excluded)

    def test_cpp_with_any_tags(self):
        buildscripts.resmokelib.config.INCLUDE_WITH_ANY_TAGS = ["tag1"]
        try:
            selector_config = {"root": "unittest.txt"}
            selected, excluded = selector.filter_tests("cpp_unit_test", selector_config,
                                                       test_file_explorer=self.test_file_explorer)
            self.assertEqual([], selected)
            self.assertEqual(["build/testA", "build/testB"], excluded)
        finally:
            buildscripts.resmokelib.config.INCLUDE_WITH_ANY_TAGS = None

    def test_jstest_include_tags(self):
        config = {
            "roots": ["dir/subdir1/*.js", "dir/subdir2/*.js", "dir/subdir3/a/*.js"],
            "include_tags": "tag1"
        }
        selected, excluded = selector.filter_tests("js_test", config, self.test_file_explorer)
        self.assertEqual(["dir/subdir1/test11.js"], selected)
        self.assertEqual(
            ["dir/subdir1/test12.js", "dir/subdir2/test21.js", "dir/subdir3/a/test3a1.js"],
            excluded)

    def test_jstest_exclude_tags(self):
        config = {
            "roots": ["dir/subdir1/*.js", "dir/subdir2/*.js", "dir/subdir3/a/*.js"],
            "exclude_tags": "tag1"
        }
        selected, excluded = selector.filter_tests("js_test", config, self.test_file_explorer)
        self.assertEqual(
            ["dir/subdir1/test12.js", "dir/subdir2/test21.js", "dir/subdir3/a/test3a1.js"],
            selected)
        self.assertEqual(["dir/subdir1/test11.js"], excluded)

    def test_jstest_force_include(self):
        config = {
            "roots": ["dir/subdir1/*.js", "dir/subdir2/*.js", "dir/subdir3/a/*.js"],
            "include_files": ["dir/subdir1/*.js"], "exclude_tags": "tag1"
        }
        selected, excluded = selector.filter_tests("js_test", config, self.test_file_explorer)
        self.assertEqual(["dir/subdir1/test11.js", "dir/subdir1/test12.js"], selected)
        self.assertEqual(["dir/subdir2/test21.js", "dir/subdir3/a/test3a1.js"], excluded)

    def test_jstest_all(self):
        config = {"roots": ["dir/subdir1/*.js", "dir/subdir2/*.js", "dir/subdir3/a/*.js"]}
        selected, excluded = selector.filter_tests("js_test", config, self.test_file_explorer)
        self.assertEqual([
            "dir/subdir1/test11.js", "dir/subdir1/test12.js", "dir/subdir2/test21.js",
            "dir/subdir3/a/test3a1.js"
        ], selected)
        self.assertEqual([], excluded)

    def test_jstest_include_with_any_tags(self):
        config = {
            "roots": ["dir/subdir1/*.js", "dir/subdir2/*.js", "dir/subdir3/a/*.js"],
            "include_with_any_tags": ["tag2"]
        }
        selected, excluded = selector.filter_tests("js_test", config, self.test_file_explorer)
        self.assertEqual(["dir/subdir1/test11.js", "dir/subdir2/test21.js"], selected)
        self.assertEqual(["dir/subdir1/test12.js", "dir/subdir3/a/test3a1.js"], excluded)

    def test_jstest_unknown_file(self):
        config = {"roots": ["dir/subdir1/*.js", "dir/subdir1/unknown"]}
        with self.assertRaisesRegexp(ValueError, "Unrecognized test file: dir/subdir1/unknown"):
            selector.filter_tests("js_test", config, self.test_file_explorer)

    def test_json_schema_exclude_files(self):
        config = {
            "roots": ["dir/subdir1/*.js", "dir/subdir2/*.js", "dir/subdir3/a/*.js"],
            "exclude_files": ["dir/subdir2/test21.js"]
        }
        selected, excluded = selector.filter_tests("json_schema_test", config,
                                                   self.test_file_explorer)
        self.assertEqual(
            ["dir/subdir1/test11.js", "dir/subdir1/test12.js", "dir/subdir3/a/test3a1.js"],
            selected)
        self.assertEqual(["dir/subdir2/test21.js"], excluded)

    def test_json_shcema_include_files(self):
        config = {
            "roots": ["dir/subdir1/*.js", "dir/subdir2/*.js", "dir/subdir3/a/*.js"],
            "include_files": ["dir/subdir2/test21.js"]
        }
        selected, excluded = selector.filter_tests("json_schema_test", config,
                                                   self.test_file_explorer)
        self.assertEqual(["dir/subdir2/test21.js"], selected)
        self.assertEqual(
            ["dir/subdir1/test11.js", "dir/subdir1/test12.js", "dir/subdir3/a/test3a1.js"],
            excluded)

    def test_db_tests_all(self):
        config = {"binary": self.test_file_explorer.binary}
        selected, excluded = selector.filter_tests("db_test", config, self.test_file_explorer)
        self.assertEqual(["dbtestA", "dbtestB", "dbtestC"], selected)
        self.assertEqual([], excluded)

    def test_db_tests_roots_override(self):
        # When roots are specified for db_tests they override all filtering since
        # 'roots' are populated with the command line arguments.
        config = {
            "binary": self.test_file_explorer.binary, "include_suites": ["dbtestB"],
            "roots": ["dbtestOverride"]
        }
        selected, excluded = selector.filter_tests("db_test", config, self.test_file_explorer)
        self.assertEqual(["dbtestOverride"], selected)
        self.assertEqual([], excluded)

    def test_db_tests_include_suites(self):
        config = {"binary": self.test_file_explorer.binary, "include_suites": ["dbtestB"]}
        selected, excluded = selector.filter_tests("db_test", config, self.test_file_explorer)
        self.assertEqual(["dbtestB"], selected)
        self.assertEqual(["dbtestA", "dbtestC"], excluded)


if __name__ == "__main__":
    unittest.main()