diff options
| -rw-r--r-- | buildscripts/ciconfig/evergreen.py | 7 | ||||
| -rw-r--r-- | buildscripts/ciconfig/tags.py | 7 | ||||
| -rw-r--r-- | buildscripts/tests/ciconfig/evergreen.yml | 2 | ||||
| -rw-r--r-- | buildscripts/tests/ciconfig/test_evergreen.py | 10 | ||||
| -rw-r--r-- | buildscripts/tests/ciconfig/test_tags.py | 7 | ||||
| -rwxr-xr-x | buildscripts/update_test_lifecycle.py | 51 |
6 files changed, 80 insertions, 4 deletions
diff --git a/buildscripts/ciconfig/evergreen.py b/buildscripts/ciconfig/evergreen.py index 806590e062b..325e8b236bf 100644 --- a/buildscripts/ciconfig/evergreen.py +++ b/buildscripts/ciconfig/evergreen.py @@ -25,9 +25,7 @@ class EvergreenProjectConfig(object): self._variants_by_name = {variant.name: variant for variant in self.variants} self.distro_names = set() for variant in self.variants: - self.distro_names.update(variant.run_on) - for vtask in variant.tasks: - self.distro_names.update(vtask.run_on) + self.distro_names.update(variant.distro_names) @property def task_names(self): @@ -110,6 +108,9 @@ class Variant(object): run_on = self.run_on self.tasks = [VariantTask(task_map.get(t["name"]), t.get("distros", run_on), self) for t in conf_dict["tasks"]] + self.distro_names = set(run_on) + for task in self.tasks: + self.distro_names.update(task.run_on) @property def name(self): diff --git a/buildscripts/ciconfig/tags.py b/buildscripts/ciconfig/tags.py index dfab58832fa..aa522e433fb 100644 --- a/buildscripts/ciconfig/tags.py +++ b/buildscripts/ciconfig/tags.py @@ -86,6 +86,13 @@ class TagsConfig(object): if not tags: del patterns[test_pattern] + def remove_test_pattern(self, test_kind, test_pattern): + """Remove a test pattern.""" + patterns = self._conf.get(test_kind) + if not patterns or test_pattern not in patterns: + return + del patterns[test_pattern] + def is_modified(self): """Return True if the tags have been modified, False otherwise.""" return self._conf != self._conf_copy diff --git a/buildscripts/tests/ciconfig/evergreen.yml b/buildscripts/tests/ciconfig/evergreen.yml index e07d236a554..9777fb5ef46 100644 --- a/buildscripts/tests/ciconfig/evergreen.yml +++ b/buildscripts/tests/ciconfig/evergreen.yml @@ -146,3 +146,5 @@ buildvariants: - name: failing_test - name: timeout_test - name: resmoke_task + distros: + - pdp-11 diff --git a/buildscripts/tests/ciconfig/test_evergreen.py b/buildscripts/tests/ciconfig/test_evergreen.py index 0911bfa8e9f..56de9bca3d3 100644 --- a/buildscripts/tests/ciconfig/test_evergreen.py +++ b/buildscripts/tests/ciconfig/test_evergreen.py @@ -55,9 +55,10 @@ class TestEvergreenProjectConfig(unittest.TestCase): self.assertEqual("osx-108", variant.name) def test_list_distro_names(self): - self.assertEqual(2, len(self.conf.distro_names)) + self.assertEqual(3, len(self.conf.distro_names)) self.assertIn("localtestdistro", self.conf.distro_names) self.assertIn("ubuntu1404-test", self.conf.distro_names) + self.assertIn("pdp-11", self.conf.distro_names) class TestTask(unittest.TestCase): @@ -145,6 +146,13 @@ class TestVariant(unittest.TestCase): variant_osx = self.conf.get_variant("osx-108") self.assertEqual(["localtestdistro"], variant_osx.run_on) + def test_distro_names(self): + variant_ubuntu = self.conf.get_variant("ubuntu") + self.assertEqual(set(["ubuntu1404-test", "pdp-11"]), variant_ubuntu.distro_names) + + variant_osx = self.conf.get_variant("osx-108") + self.assertEqual(set(["localtestdistro"]), variant_osx.distro_names) + def test_test_flags(self): variant_ubuntu = self.conf.get_variant("ubuntu") self.assertEqual("--param=value --ubuntu", variant_ubuntu.test_flags) diff --git a/buildscripts/tests/ciconfig/test_tags.py b/buildscripts/tests/ciconfig/test_tags.py index 8f9b3932f4a..cf1f9bef265 100644 --- a/buildscripts/tests/ciconfig/test_tags.py +++ b/buildscripts/tests/ciconfig/test_tags.py @@ -128,6 +128,13 @@ class TestTagsConfig(unittest.TestCase): patterns = self.conf.get_test_patterns(test_kind) self.assertNotIn(test_pattern, patterns) + def test_remove_pattern(self): + test_kind = "js_test" + test_pattern = "jstests/core/example.js" + self.assertIn(test_pattern, self.conf.get_test_patterns(test_kind)) + self.conf.remove_test_pattern(test_kind, test_pattern) + self.assertNotIn(test_pattern, self.conf.get_test_patterns(test_kind)) + def test_tag_order(self): test_kind = "js_test" test_pattern = "jstests/core/example.js" diff --git a/buildscripts/update_test_lifecycle.py b/buildscripts/update_test_lifecycle.py index 9699a5418d8..1d533b1803d 100755 --- a/buildscripts/update_test_lifecycle.py +++ b/buildscripts/update_test_lifecycle.py @@ -23,6 +23,7 @@ if __name__ == "__main__" and __package__ is None: sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from buildscripts import resmokelib +from buildscripts.resmokelib.utils import globstar from buildscripts import test_failures as tf from buildscripts.ciconfig import evergreen as ci_evergreen from buildscripts.ciconfig import tags as ci_tags @@ -219,6 +220,9 @@ def update_lifecycle(lifecycle, report, method_test, add_tags, fail_rate, min_ru def compare_tags(tag_a, tag_b): + """Compare two tags and return 1, -1 or 0 if 'tag_a' is superior, inferior or + equal to 'tag_b'. + """ return cmp(tag_a.split("|"), tag_b.split("|")) @@ -324,6 +328,49 @@ def update_tags(lifecycle, config, report): config.reliable_min_runs) +def _split_tag(tag): + """Split a tag into its components. + + Return a tuple containing task, variant, distro. The values are None if absent from the tag. + If the tag is invalid, the return value is (None, None, None). + """ + elements = tag.split("|") + length = len(elements) + if elements[0] != "unreliable" or length < 2 or length > 4: + return None, None, None + # fillout the array + elements.extend([None] * (4 - length)) + # return as a tuple + return tuple(elements[1:]) + + +def _is_tag_still_relevant(evg_conf, tag): + """Indicate if a tag still corresponds to a valid task/variant/distro combination.""" + task, variant, distro = _split_tag(tag) + if not task or task not in evg_conf.task_names: + return False + if variant: + variant_conf = evg_conf.get_variant(variant) + if not variant_conf or task not in variant_conf.task_names: + return False + if distro and distro not in variant_conf.distros: + return False + return True + + +def cleanup_tags(lifecycle, evg_conf): + """Remove the tags that do not correspond to a valid test/task/variant/distro combination.""" + for test_kind in lifecycle.get_test_kinds(): + for test_pattern in lifecycle.get_test_patterns(test_kind): + if not globstar.glob(test_pattern): + # The pattern does not match any file in the repository. + lifecycle.remove_test_pattern(test_kind, test_pattern) + continue + for tag in lifecycle.get_tags(test_kind, test_pattern): + if not _is_tag_still_relevant(evg_conf, tag): + lifecycle.remove_tag(test_kind, test_pattern, tag) + + def main(): """ Utility for updating a resmoke.py tag file based on computing test failure rates from the @@ -543,11 +590,15 @@ def main(): report = tf.Report(history_data) update_tags(lifecycle, config, report) + # Remove tags that are no longer relevant + cleanup_tags(lifecycle, evg_conf) + # We write the 'lifecycle' tag configuration to the 'options.lifecycle_file' file only if there # have been changes to the tags. In particular, we avoid modifying the file when only the header # comment for the YAML file would change. if lifecycle.is_modified(): write_yaml_file(options.tag_file, lifecycle) + if __name__ == "__main__": main() |