SERVER-29640 Cleanup test lifecycle invalid tags

6 files changed, 80 insertions, 4 deletions

diff --git a/buildscripts/ciconfig/evergreen.py b/buildscripts/ciconfig/evergreen.py
index 806590e062b..325e8b236bf 100644
--- a/buildscripts/ciconfig/evergreen.py
+++ b/buildscripts/ciconfig/evergreen.py
@@ -25,9 +25,7 @@ class EvergreenProjectConfig(object):
         self._variants_by_name = {variant.name: variant for variant in self.variants}
         self.distro_names = set()
         for variant in self.variants:
-            self.distro_names.update(variant.run_on)
-            for vtask in variant.tasks:
-                self.distro_names.update(vtask.run_on)
+            self.distro_names.update(variant.distro_names)
 
     @property
     def task_names(self):
@@ -110,6 +108,9 @@ class Variant(object):
         run_on = self.run_on
         self.tasks = [VariantTask(task_map.get(t["name"]), t.get("distros", run_on), self)
                       for t in conf_dict["tasks"]]
+        self.distro_names = set(run_on)
+        for task in self.tasks:
+            self.distro_names.update(task.run_on)
 
     @property
     def name(self):
diff --git a/buildscripts/ciconfig/tags.py b/buildscripts/ciconfig/tags.py
index dfab58832fa..aa522e433fb 100644
--- a/buildscripts/ciconfig/tags.py
+++ b/buildscripts/ciconfig/tags.py
@@ -86,6 +86,13 @@ class TagsConfig(object):
         if not tags:
             del patterns[test_pattern]
 
+    def remove_test_pattern(self, test_kind, test_pattern):
+        """Remove a test pattern."""
+        patterns = self._conf.get(test_kind)
+        if not patterns or test_pattern not in patterns:
+            return
+        del patterns[test_pattern]
+
     def is_modified(self):
         """Return True if the tags have been modified, False otherwise."""
         return self._conf != self._conf_copy
diff --git a/buildscripts/tests/ciconfig/evergreen.yml b/buildscripts/tests/ciconfig/evergreen.yml
index e07d236a554..9777fb5ef46 100644
--- a/buildscripts/tests/ciconfig/evergreen.yml
+++ b/buildscripts/tests/ciconfig/evergreen.yml
@@ -146,3 +146,5 @@ buildvariants:
   - name: failing_test
   - name: timeout_test
   - name: resmoke_task
+    distros:
+      - pdp-11
diff --git a/buildscripts/tests/ciconfig/test_evergreen.py b/buildscripts/tests/ciconfig/test_evergreen.py
index 0911bfa8e9f..56de9bca3d3 100644
--- a/buildscripts/tests/ciconfig/test_evergreen.py
+++ b/buildscripts/tests/ciconfig/test_evergreen.py
@@ -55,9 +55,10 @@ class TestEvergreenProjectConfig(unittest.TestCase):
         self.assertEqual("osx-108", variant.name)
 
     def test_list_distro_names(self):
-        self.assertEqual(2, len(self.conf.distro_names))
+        self.assertEqual(3, len(self.conf.distro_names))
         self.assertIn("localtestdistro", self.conf.distro_names)
         self.assertIn("ubuntu1404-test", self.conf.distro_names)
+        self.assertIn("pdp-11", self.conf.distro_names)
 
 
 class TestTask(unittest.TestCase):
@@ -145,6 +146,13 @@ class TestVariant(unittest.TestCase):
         variant_osx = self.conf.get_variant("osx-108")
         self.assertEqual(["localtestdistro"], variant_osx.run_on)
 
+    def test_distro_names(self):
+        variant_ubuntu = self.conf.get_variant("ubuntu")
+        self.assertEqual(set(["ubuntu1404-test", "pdp-11"]), variant_ubuntu.distro_names)
+
+        variant_osx = self.conf.get_variant("osx-108")
+        self.assertEqual(set(["localtestdistro"]), variant_osx.distro_names)
+
     def test_test_flags(self):
         variant_ubuntu = self.conf.get_variant("ubuntu")
         self.assertEqual("--param=value --ubuntu", variant_ubuntu.test_flags)
diff --git a/buildscripts/tests/ciconfig/test_tags.py b/buildscripts/tests/ciconfig/test_tags.py
index 8f9b3932f4a..cf1f9bef265 100644
--- a/buildscripts/tests/ciconfig/test_tags.py
+++ b/buildscripts/tests/ciconfig/test_tags.py
@@ -128,6 +128,13 @@ class TestTagsConfig(unittest.TestCase):
         patterns = self.conf.get_test_patterns(test_kind)
         self.assertNotIn(test_pattern, patterns)
 
+    def test_remove_pattern(self):
+        test_kind = "js_test"
+        test_pattern = "jstests/core/example.js"
+        self.assertIn(test_pattern, self.conf.get_test_patterns(test_kind))
+        self.conf.remove_test_pattern(test_kind, test_pattern)
+        self.assertNotIn(test_pattern, self.conf.get_test_patterns(test_kind))
+
     def test_tag_order(self):
         test_kind = "js_test"
         test_pattern = "jstests/core/example.js"
diff --git a/buildscripts/update_test_lifecycle.py b/buildscripts/update_test_lifecycle.py
index 9699a5418d8..1d533b1803d 100755
--- a/buildscripts/update_test_lifecycle.py
+++ b/buildscripts/update_test_lifecycle.py
@@ -23,6 +23,7 @@ if __name__ == "__main__" and __package__ is None:
     sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
 from buildscripts import resmokelib
+from buildscripts.resmokelib.utils import globstar
 from buildscripts import test_failures as tf
 from buildscripts.ciconfig import evergreen as ci_evergreen
 from buildscripts.ciconfig import tags as ci_tags
@@ -219,6 +220,9 @@ def update_lifecycle(lifecycle, report, method_test, add_tags, fail_rate, min_ru
 
 
 def compare_tags(tag_a, tag_b):
+    """Compare two tags and return 1, -1 or 0 if 'tag_a' is superior, inferior or
+    equal to 'tag_b'.
+    """
     return cmp(tag_a.split("|"), tag_b.split("|"))
 
 
@@ -324,6 +328,49 @@ def update_tags(lifecycle, config, report):
                          config.reliable_min_runs)
 
 
+def _split_tag(tag):
+    """Split a tag into its components.
+
+    Return a tuple containing task, variant, distro. The values are None if absent from the tag.
+    If the tag is invalid, the return value is (None, None, None).
+    """
+    elements = tag.split("|")
+    length = len(elements)
+    if elements[0] != "unreliable" or length < 2 or length > 4:
+        return None, None, None
+    # fillout the array
+    elements.extend([None] * (4 - length))
+    # return as a tuple
+    return tuple(elements[1:])
+
+
+def _is_tag_still_relevant(evg_conf, tag):
+    """Indicate if a tag still corresponds to a valid task/variant/distro combination."""
+    task, variant, distro = _split_tag(tag)
+    if not task or task not in evg_conf.task_names:
+        return False
+    if variant:
+        variant_conf = evg_conf.get_variant(variant)
+        if not variant_conf or task not in variant_conf.task_names:
+            return False
+        if distro and distro not in variant_conf.distros:
+            return False
+    return True
+
+
+def cleanup_tags(lifecycle, evg_conf):
+    """Remove the tags that do not correspond to a valid test/task/variant/distro combination."""
+    for test_kind in lifecycle.get_test_kinds():
+        for test_pattern in lifecycle.get_test_patterns(test_kind):
+            if not globstar.glob(test_pattern):
+                # The pattern does not match any file in the repository.
+                lifecycle.remove_test_pattern(test_kind, test_pattern)
+                continue
+            for tag in lifecycle.get_tags(test_kind, test_pattern):
+                if not _is_tag_still_relevant(evg_conf, tag):
+                    lifecycle.remove_tag(test_kind, test_pattern, tag)
+
+
 def main():
     """
     Utility for updating a resmoke.py tag file based on computing test failure rates from the
@@ -543,11 +590,15 @@ def main():
         report = tf.Report(history_data)
         update_tags(lifecycle, config, report)
 
+    # Remove tags that are no longer relevant
+    cleanup_tags(lifecycle, evg_conf)
+
     # We write the 'lifecycle' tag configuration to the 'options.lifecycle_file' file only if there
     # have been changes to the tags. In particular, we avoid modifying the file when only the header
     # comment for the YAML file would change.
     if lifecycle.is_modified():
         write_yaml_file(options.tag_file, lifecycle)
 
+
 if __name__ == "__main__":
     main()