SERVER-48048 Use resmoke tag files for multiversion blacklisting

7 files changed, 222 insertions, 108 deletions

diff --git a/buildscripts/tests/resmokelib/test_selector.py b/buildscripts/tests/resmokelib/test_selector.py
index f5d77acee77..09f3722643a 100644
--- a/buildscripts/tests/resmokelib/test_selector.py
+++ b/buildscripts/tests/resmokelib/test_selector.py
@@ -4,6 +4,7 @@ import fnmatch
 import os.path
 import sys
 import unittest
+import collections
 
 import buildscripts.resmokelib.config
 import buildscripts.resmokelib.parser as parser
@@ -12,6 +13,8 @@ import buildscripts.resmokelib.utils.globstar as globstar
 
 # pylint: disable=missing-docstring,protected-access
 
+FIXTURE_PREFIX = "buildscripts/tests/selftest_fixtures"
+
 
 class TestExpressions(unittest.TestCase):
     """Unit tests for the tag matching expressions."""
@@ -104,6 +107,26 @@ class TestTestFileExplorer(unittest.TestCase):
         self.assertTrue(self.test_file_explorer.fnmatchcase("directory/file.js", pattern))
         self.assertFalse(self.test_file_explorer.fnmatchcase("other/file.js", pattern))
 
+    def test_parse_tag_file(self):
+        tests = (os.path.join(FIXTURE_PREFIX, "one.js"), os.path.join(FIXTURE_PREFIX, "two.js"),
+                 os.path.join(FIXTURE_PREFIX, "three.js"))
+        expected = collections.defaultdict(list)
+        expected[tests[0]] = ["tag1", "tag2", "tag3"]
+        expected[tests[1]] = ["tag1", "tag2"]
+
+        tags = self.test_file_explorer.parse_tag_file("js_test",
+                                                      os.path.join(FIXTURE_PREFIX, "tag_file1.yml"))
+        # defaultdict isn't == comparable
+        for test in tests:
+            self.assertEqual(tags[test], expected[test])
+
+        expected[tests[1]] = ["tag1", "tag2", "tag4"]
+        tags = self.test_file_explorer.parse_tag_file("js_test",
+                                                      os.path.join(FIXTURE_PREFIX, "tag_file2.yml"),
+                                                      tags)
+        for test in tests:
+            self.assertEqual(tags[test], expected[test])
+
 
 class MockTestFileExplorer(object):
     """Component giving access to mock test files data."""
@@ -150,7 +173,7 @@ class MockTestFileExplorer(object):
     def list_dbtests(self, binary):  # pylint: disable=no-self-use,unused-argument
         return ["dbtestA", "dbtestB", "dbtestC"]
 
-    def parse_tag_file(self, test_kind):
+    def parse_tag_file(self, test_kind, tag_file=None, tagged_tests=None):  # pylint: disable=unused-argument
         if test_kind == "js_test":
             return self.jstest_tag_file
         return None
diff --git a/buildscripts/tests/selftest_fixtures/one.js b/buildscripts/tests/selftest_fixtures/one.js
new file mode 100644
index 00000000000..e69de29bb2d
--- /dev/null
+++ b/buildscripts/tests/selftest_fixtures/one.js
diff --git a/buildscripts/tests/selftest_fixtures/tag_file1.yml b/buildscripts/tests/selftest_fixtures/tag_file1.yml
new file mode 100644
index 00000000000..043ce804256
--- /dev/null
+++ b/buildscripts/tests/selftest_fixtures/tag_file1.yml
@@ -0,0 +1,9 @@
+selector:
+  js_test:
+    buildscripts/tests/selftest_fixtures/one.js:
+    - tag1
+    - tag2
+    - tag3
+    buildscripts/tests/selftest_fixtures/two.js:
+    - tag1
+    - tag2
diff --git a/buildscripts/tests/selftest_fixtures/tag_file2.yml b/buildscripts/tests/selftest_fixtures/tag_file2.yml
new file mode 100644
index 00000000000..381f64390a7
--- /dev/null
+++ b/buildscripts/tests/selftest_fixtures/tag_file2.yml
@@ -0,0 +1,4 @@
+selector:
+  js_test:
+    buildscripts/tests/selftest_fixtures/two.js:
+    - tag4
diff --git a/buildscripts/tests/selftest_fixtures/three.js b/buildscripts/tests/selftest_fixtures/three.js
new file mode 100644
index 00000000000..e69de29bb2d
--- /dev/null
+++ b/buildscripts/tests/selftest_fixtures/three.js
diff --git a/buildscripts/tests/selftest_fixtures/two.js b/buildscripts/tests/selftest_fixtures/two.js
new file mode 100644
index 00000000000..e69de29bb2d
--- /dev/null
+++ b/buildscripts/tests/selftest_fixtures/two.js
diff --git a/buildscripts/tests/test_evergreen_gen_multiversion_tests.py b/buildscripts/tests/test_evergreen_gen_multiversion_tests.py
index 4fdba6e44f9..662ba311d4d 100644
--- a/buildscripts/tests/test_evergreen_gen_multiversion_tests.py
+++ b/buildscripts/tests/test_evergreen_gen_multiversion_tests.py
@@ -14,7 +14,7 @@ from buildscripts import evergreen_gen_multiversion_tests as under_test
 from buildscripts.evergreen_generate_resmoke_tasks import read_yaml
 import buildscripts.evergreen_generate_resmoke_tasks as generate_resmoke
 
-# pylint: disable=missing-docstring
+# pylint: disable=missing-docstring, no-self-use
 
 
 class TestRun(unittest.TestCase):
@@ -70,119 +70,197 @@ class TestRun(unittest.TestCase):
                 os.remove(evg_conf.name)
 
 
-class TestGenerateExcludeFiles(unittest.TestCase):
+class TestGenerateExcludeYaml(unittest.TestCase):
     def setUp(self):
         self._tmpdir = TemporaryDirectory()
-        under_test.CONFIG_DIR = self._tmpdir.name
 
     def tearDown(self):
         if self._tmpdir is not None:
             self._tmpdir.cleanup()
-        under_test.CONFIG_DIR = generate_resmoke.DEFAULT_CONFIG_VALUES["generated_config_dir"]
 
-    def test_missing_dir_okay(self):
-        self._tmpdir.cleanup()
-        self._tmpdir = None
-        self.assertFalse(os.path.exists(under_test.CONFIG_DIR))
-
-        runner = CliRunner()
-        result = runner.invoke(
-            under_test.generate_exclude_yaml,
-            ['--suite=test', '--task-path-suffix=test', '--is-generated-suite=true'])
-        self.assertEqual(result.exit_code, 0, result)
-
-    def test_empty_dir_okay(self):
-        runner = CliRunner()
-        result = runner.invoke(
-            under_test.generate_exclude_yaml,
-            ['--suite=test', '--task-path-suffix=test', '--is-generated-suite=true'])
-        self.assertEqual(result.exit_code, 0, result)
-        self.assertEqual(len(os.listdir(under_test.CONFIG_DIR)), 0)
-
-    @patch('buildscripts.evergreen_gen_multiversion_tests.get_exclude_files')
-    def test_adds_exclude_file(self, get_exclude_files):
-        get_exclude_files.return_value = set()
-        get_exclude_files.return_value.add('jstests/core/count_plan_summary.js')
-        with open(self._tmpdir.name + "/sharding_jscore_passthrough_00.yml", mode='w') as fh:
-            fh.write(CONF)
-
-        runner = CliRunner()
-        result = runner.invoke(
-            under_test.generate_exclude_yaml,
-            ['--suite=test', '--task-path-suffix=/data/multiversion', '--is-generated-suite=true'])
-        self.assertEqual(result.exit_code, 0, result)
-        self.assertEqual(get_exclude_files.call_count, 1)
-        new_conf = read_yaml(self._tmpdir.name, "sharding_jscore_passthrough_00.yml")
-        self.assertEqual(new_conf["selector"]["exclude_files"],
-                         ["jstests/core/count_plan_summary.js"])
-
-
-CONF = """# DO NOT EDIT THIS FILE. All manual edits will be lost.
-# This file was generated by /data/mci/4d9a5adb3744dad3d44d93e5ff2a441f/src/buildscripts/evergreen_generate_resmoke_tasks.py from
-# sharded_collections_jscore_passthrough.
-executor:
-  archive:
-    hooks:
-    - CheckReplDBHash
-    - ValidateCollections
-  config:
-    shell_options:
-      eval: load("jstests/libs/override_methods/implicitly_shard_accessed_collections.js")
-      readMode: commands
-  fixture:
-    class: ShardedClusterFixture
-    enable_balancer: false
-    mongod_options:
-      set_parameters:
-        enableTestCommands: 1
-    mongos_options:
-      set_parameters:
-        enableTestCommands: 1
-    num_shards: 2
-  hooks:
-  - class: CheckReplDBHash
-  - class: ValidateCollections
-  - class: CleanEveryN
-    n: 20
-selector:
-  exclude_with_any_tags:
-  - assumes_against_mongod_not_mongos
-  - assumes_no_implicit_collection_creation_after_drop
-  - assumes_no_implicit_index_creation
-  - assumes_unsharded_collection
-  - cannot_create_unique_index_when_using_hashed_shard_key
-  - requires_profiling
-  roots:
-  - jstests/core/count_plan_summary.js
-  - jstests/core/json_schema/unique_items.js
-  - jstests/core/arrayfind9.js
-  - jstests/core/sort_numeric.js
-  - jstests/core/remove4.js
-  - jstests/core/aggregation_accepts_write_concern.js
-  - jstests/core/geo3.js
-  - jstests/core/minmax_edge.js
-  - jstests/core/json_schema/additional_properties.js
-  - jstests/core/update_min_max_examples.js
-  - jstests/core/where2.js
-  - jstests/core/geo_s2meridian.js
-  - jstests/core/bittest.js
-  - jstests/core/orj.js
-  - jstests/core/find_dedup.js
-  - jstests/core/hashed_index_queries.js
-  - jstests/core/geo_circle2a.js
-  - jstests/core/js4.js
-  - jstests/core/index_create_with_nul_in_name.js
-  - jstests/core/count_hint.js
-  - jstests/core/sorta.js
-  - jstests/core/orf.js
-  - jstests/core/geo_s2within.js
-  - jstests/core/json_schema/required.js
-  - jstests/core/pop_server_13516.js
-  - jstests/core/updatel.js
-  - jstests/core/geo_s2descindex.js
-test_kind: js_test
-use_in_multiversion: sharded_collections_jscore_multiversion_passthrough
-"""
+    def assert_contents(self, expected):
+        actual = read_yaml(self._tmpdir.name, under_test.EXCLUDE_TAGS_FILE)
+        self.assertEqual(actual, expected)
+
+    def patch_and_run(self, latest, last_lts):
+        """
+        Helper to patch and run the test.
+        """
+        mock_multiversion_methods = {
+            'get_backports_required_last_lts_hash': MagicMock(),
+            'get_last_lts_yaml': MagicMock(return_value=last_lts)
+        }
+
+        with patch.multiple('buildscripts.evergreen_gen_multiversion_tests',
+                            **mock_multiversion_methods):
+            with patch('buildscripts.evergreen_generate_resmoke_tasks.read_yaml',
+                       return_value=latest) as mock_read_yaml:
+
+                output = os.path.join(self._tmpdir.name, under_test.EXCLUDE_TAGS_FILE)
+                runner = CliRunner()
+                result = runner.invoke(
+                    under_test.generate_exclude_yaml,
+                    [f"--output={output}", '--task-path-suffix=/data/multiversion'])
+
+                self.assertEqual(result.exit_code, 0, result)
+                mock_read_yaml.assert_called_once()
+                mock_multiversion_methods[
+                    'get_backports_required_last_lts_hash'].assert_called_once()
+                mock_multiversion_methods['get_last_lts_yaml'].assert_called_once()
+
+    def test_create_yaml_suite1(self):
+        latest_yaml = {
+            'all': [{'ticket': 'fake_ticket0', 'test_file': 'jstests/fake_file0.js'}], 'suites': {
+                'suite1': [{'ticket': 'fake_ticket1', 'test_file': 'jstests/fake_file1.js'},
+                           {'ticket': 'fake_ticket2', 'test_file': 'jstests/fake_file2.js'}]
+            }
+        }
+
+        last_lts_yaml = {
+            'all': [{'ticket': 'fake_ticket0', 'test_file': 'jstests/fake_file0.js'}], 'suites': {
+                'suite1': [{'ticket': 'fake_ticket2', 'test_file': 'jstests/fake_file2.js'}]
+            }
+        }
+
+        expected = {
+            'selector': {
+                'js_test': {'jstests/fake_file1.js': ['suite1_backport_required_multiversion']}
+            }
+        }
+
+        self.patch_and_run(latest_yaml, last_lts_yaml)
+        self.assert_contents(expected)
+
+    def test_create_yaml_suite1_and_suite2(self):
+        latest_yaml = {
+            'all': [{'ticket': 'fake_ticket0', 'test_file': 'jstests/fake_file0.js'}], 'suites': {
+                'suite1': [{'ticket': 'fake_ticket1', 'test_file': 'jstests/fake_file1.js'},
+                           {'ticket': 'fake_ticket2', 'test_file': 'jstests/fake_file2.js'}],
+                'suite2': [{'ticket': 'fake_ticket1', 'test_file': 'jstests/fake_file1.js'}]
+            }
+        }
+
+        last_lts_yaml = {
+            'all': [{'ticket': 'fake_ticket0', 'test_file': 'jstests/fake_file0.js'}], 'suites': {
+                'suite1': [{'ticket': 'fake_ticket2', 'test_file': 'jstests/fake_file2.js'}]
+            }
+        }
+
+        expected = {
+            'selector': {
+                'js_test': {
+                    'jstests/fake_file1.js': [
+                        'suite1_backport_required_multiversion',
+                        'suite2_backport_required_multiversion'
+                    ]
+                }
+            }
+        }
+
+        self.patch_and_run(latest_yaml, last_lts_yaml)
+        self.assert_contents(expected)
+
+    def test_both_all_are_none(self):
+        latest_yaml = {
+            'all': None, 'suites': {
+                'suite1': [{'ticket': 'fake_ticket1', 'test_file': 'jstests/fake_file1.js'},
+                           {'ticket': 'fake_ticket2', 'test_file': 'jstests/fake_file2.js'}]
+            }
+        }
+
+        last_lts_yaml = {
+            'all': None, 'suites': {
+                'suite1': [{'ticket': 'fake_ticket2', 'test_file': 'jstests/fake_file2.js'}]
+            }
+        }
+
+        expected = {
+            'selector': {
+                'js_test': {'jstests/fake_file1.js': ['suite1_backport_required_multiversion']}
+            }
+        }
+
+        self.patch_and_run(latest_yaml, last_lts_yaml)
+        self.assert_contents(expected)
+
+    def test_old_all_is_none(self):
+        latest_yaml = {
+            'all': [{'ticket': 'fake_ticket0', 'test_file': 'jstests/fake_file0.js'}], 'suites': {
+                'suite1': [{'ticket': 'fake_ticket1', 'test_file': 'jstests/fake_file1.js'},
+                           {'ticket': 'fake_ticket2', 'test_file': 'jstests/fake_file2.js'}]
+            }
+        }
+
+        last_lts_yaml = {
+            'all': None, 'suites': {
+                'suite1': [{'ticket': 'fake_ticket2', 'test_file': 'jstests/fake_file2.js'}]
+            }
+        }
+
+        expected = {
+            'selector': {
+                'js_test': {
+                    'jstests/fake_file1.js': ['suite1_backport_required_multiversion'],
+                    'jstests/fake_file0.js': ['backport_required_multiversion']
+                }
+            }
+        }
+
+        self.patch_and_run(latest_yaml, last_lts_yaml)
+        self.assert_contents(expected)
+
+    def test_create_yaml_suite1_and_all(self):
+        latest_yaml = {
+            'all': [{'ticket': 'fake_ticket0', 'test_file': 'jstests/fake_file0.js'},
+                    {'ticket': 'fake_ticket4', 'test_file': 'jstests/fake_file4.js'}], 'suites': {
+                        'suite1': [{'ticket': 'fake_ticket1', 'test_file': 'jstests/fake_file1.js'},
+                                   {'ticket': 'fake_ticket2', 'test_file': 'jstests/fake_file2.js'}]
+                    }
+        }
+
+        last_lts_yaml = {
+            'all': [{'ticket': 'fake_ticket0', 'test_file': 'jstests/fake_file0.js'}], 'suites': {
+                'suite1': [{'ticket': 'fake_ticket2', 'test_file': 'jstests/fake_file2.js'}]
+            }
+        }
+
+        expected = {
+            'selector': {
+                'js_test': {
+                    'jstests/fake_file1.js': ['suite1_backport_required_multiversion'],
+                    'jstests/fake_file4.js': ['backport_required_multiversion']
+                }
+            }
+        }
+
+        self.patch_and_run(latest_yaml, last_lts_yaml)
+        self.assert_contents(expected)
+
+    # Can delete after backporting the changed yml syntax.
+    def test_not_backported(self):
+        latest_yaml = {
+            'all': [{'ticket': 'fake_ticket0', 'test_file': 'jstests/fake_file0.js'}], 'suites': {
+                'suite1': [{'ticket': 'fake_ticket2', 'test_file': 'jstests/fake_file2.js'},
+                           {'ticket': 'fake_ticket3', 'test_file': 'jstests/fake_file3.js'}]
+            }
+        }
+
+        last_lts_yaml = {
+            'suite1': [{'ticket': 'fake_ticket2', 'test_file': 'jstests/fake_file2.js'}]
+        }
+
+        expected = {
+            'selector': {
+                'js_test': {
+                    'jstests/fake_file0.js': ['backport_required_multiversion'],
+                    'jstests/fake_file3.js': ['suite1_backport_required_multiversion']
+                }
+            }
+        }
+
+        self.patch_and_run(latest_yaml, last_lts_yaml)
+        self.assert_contents(expected)
+
 
 EXPANSIONS = """task: t
 build_variant: bv