summaryrefslogtreecommitdiff
path: root/buildscripts
diff options
context:
space:
mode:
authorDavid Bradford <david.bradford@mongodb.com>2019-04-24 14:07:36 -0400
committerDavid Bradford <david.bradford@mongodb.com>2019-04-24 14:07:36 -0400
commit8ffbd4892537721fca0a640a3bced2cfa21e4c59 (patch)
treeeb4c496176bb0528717168b527f40564dc64c44b /buildscripts
parent98eebc9356f85fa7611d6bc876eaa631cfe75009 (diff)
downloadmongo-8ffbd4892537721fca0a640a3bced2cfa21e4c59.tar.gz
SERVER-40486: Remove Test Lifecycle code
Diffstat (limited to 'buildscripts')
-rw-r--r--buildscripts/ciconfig/evergreen.py13
-rwxr-xr-xbuildscripts/fetch_test_lifecycle.py202
-rw-r--r--buildscripts/resmokeconfig/suites/buildscripts_test.yml1
-rw-r--r--buildscripts/tests/ciconfig/evergreen.yml4
-rw-r--r--buildscripts/tests/ciconfig/test_evergreen.py8
-rw-r--r--buildscripts/tests/test_fetch_test_lifecycle.py70
-rw-r--r--buildscripts/tests/test_update_test_lifecycle.py1121
-rwxr-xr-xbuildscripts/update_test_lifecycle.py1186
8 files changed, 0 insertions, 2605 deletions
diff --git a/buildscripts/ciconfig/evergreen.py b/buildscripts/ciconfig/evergreen.py
index 857f5a1779e..1e0f7065c74 100644
--- a/buildscripts/ciconfig/evergreen.py
+++ b/buildscripts/ciconfig/evergreen.py
@@ -77,19 +77,6 @@ class EvergreenProjectConfig(object): # pylint: disable=too-many-instance-attri
return self._task_groups_by_name.get(task_group_name)
@property
- def lifecycle_task_names(self):
- """Get the list of names of the tasks that have not been excluded from test lifecycle."""
- excluded = self._get_test_lifecycle_excluded_task_names()
- return [name for name in self.task_names if name not in excluded]
-
- def _get_test_lifecycle_excluded_task_names(self):
- excluded_patterns = self._conf.get("test_lifecycle_excluded_tasks", [])
- excluded = []
- for pattern in excluded_patterns:
- excluded.extend(fnmatch.filter(self.task_names, pattern))
- return excluded
-
- @property
def variant_names(self):
"""Get the list of build variant names."""
return list(self._variants_by_name.keys())
diff --git a/buildscripts/fetch_test_lifecycle.py b/buildscripts/fetch_test_lifecycle.py
deleted file mode 100755
index 1b07201e944..00000000000
--- a/buildscripts/fetch_test_lifecycle.py
+++ /dev/null
@@ -1,202 +0,0 @@
-#!/usr/bin/env python3
-"""Retrieve the etc/test_lifecycle.yml tag file from the metadata repository.
-
-This is performed for the current repository.
-
-Usage:
- python buildscsripts/fetch_test_lifecycle.py evergreen-project revision
-"""
-
-import logging
-import optparse
-import os
-import posixpath
-import sys
-import textwrap
-
-import yaml
-
-# Get relative imports to work when the package is not installed on the PYTHONPATH.
-if __name__ == "__main__" and __package__ is None:
- sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-
-from buildscripts import git # pylint: disable=wrong-import-position
-
-LOGGER = logging.getLogger(__name__)
-
-
-class MetadataRepository(object):
- """Represent the metadata repository containing the test lifecycle tags file."""
-
- def __init__(self, repository, references_file,
- lifecycle_file): # noqa: D214,D405,D406,D407,D411,D413
- """Initialize the MetadataRepository.
-
- Args:
- repository: the git.Repository object for the repository.
- references_file: the relative path from the root of the repository to the references
- yaml file.
- lifecycle_file: the relative path from the root of the repository to the test lifecycle
- tags yaml file.
- """
- self._repository = repository
- self._references_file = references_file
- self._lifecycle_file = lifecycle_file
- # The path to the lifecycle file, absolute or relative to the current working directory.
- self.lifecycle_path = os.path.join(repository.directory, lifecycle_file)
-
- def list_revisions(self): # noqa: D406,D407,D413
- """List the revisions from the HEAD of this repository.
-
- Returns:
- A list of str containing the git hashes for all the revisions from the newest (HEAD)
- to the oldest.
- """
- return self._repository.git_rev_list(["HEAD", "--", self._lifecycle_file]).splitlines()
-
- def _get_references_content(self, revision):
- references_content = self._repository.git_cat_file(
- ["blob", "%s:%s" % (revision, self._references_file)])
- return references_content
-
- def get_reference(self, metadata_revision, project):
- """Retrieve the reference revision (a revision of the project 'project').
-
- The revision is associated with the test lifecycle file present in the metadata repository
- at revision 'metadata_revision'.
-
- Args:
- metadata_revision: a revision (git hash) of this repository.
- project: an Evergreen project name (e.g. mongodb-mongo-master).
- """
- references_content = self._get_references_content(metadata_revision)
- references = yaml.safe_load(references_content)
- return references.get("test-lifecycle", {}).get(project)
-
- def get_lifecycle_file_content(self, metadata_revision):
- """Return the content of the test lifecycle file as it was at the given revision."""
- return self._repository.git_cat_file(
- ["blob", "%s:%s" % (metadata_revision, self._lifecycle_file)])
-
-
-def _clone_repository(url, branch):
- """Clone the repository present at the URL 'url' and use the branch 'branch'."""
- target_directory = posixpath.splitext(posixpath.basename(url))[0]
- LOGGER.info("Cloning the repository %s into the directory %s", url, target_directory)
- return git.Repository.clone(url, target_directory, branch)
-
-
-def _get_metadata_revision(metadata_repo, mongo_repo, project, revision):
- """Get the metadata revision that corresponds to a given repository, project, revision."""
- for metadata_revision in metadata_repo.list_revisions():
- reference = metadata_repo.get_reference(metadata_revision, project)
- if not reference:
- # No reference for this revision. This should not happen but we keep trying in
- # case we can find an older revision with a reference.
- continue
- if mongo_repo.is_ancestor(reference, revision):
- # We found a reference that is a parent of the current revision.
- return metadata_revision
- return None
-
-
-def fetch_test_lifecycle(metadata_repo_url, references_file, lifecycle_file, project,
- revision): # noqa: D214,D405,D406,D407,D411,D413
- """Fetch the test lifecycle file for the revision in the repository this script is invoked.
-
- Args:
- metadata_repo_url: the git repository URL for the metadata repository containing the test
- lifecycle file.
- references_file: the relative path from the root of the metadata repository to the
- references file.
- lifecycle_file: the relative path from the root of the metadata repository to the test
- lifecycle file.
- project: the Evergreen project name.
- revision: the current repository revision.
- """
- metadata_repo = MetadataRepository(
- _clone_repository(metadata_repo_url, project), references_file, lifecycle_file)
- mongo_repo = git.Repository(os.getcwd())
- metadata_revision = _get_metadata_revision(metadata_repo, mongo_repo, project, revision)
- if metadata_revision:
- LOGGER.info("Using metadata repository revision '%s'", metadata_revision)
- result = metadata_repo.get_lifecycle_file_content(metadata_revision)
- else:
- result = None
- return result
-
-
-def main():
- """Execute Main program.
-
- Utility to fetch the etc/test_lifecycle.yml file corresponding to a given revision from
- the mongo-test-metadata repository.
- """
- parser = optparse.OptionParser(
- description=textwrap.dedent(main.__doc__), usage="Usage: %prog [options] evergreen-project")
-
- parser.add_option(
- "--revision", dest="revision", metavar="<revision>", default="HEAD",
- help=("The project revision for which to retrieve the test lifecycle tags"
- " file."))
-
- parser.add_option(
- "--metadataRepo", dest="metadata_repo_url", metavar="<metadata-repo-url>",
- default="git@github.com:mongodb/mongo-test-metadata.git",
- help=("The URL to the metadata repository that contains the test lifecycle"
- " tags file."))
-
- parser.add_option(
- "--lifecycleFile", dest="lifecycle_file", metavar="<lifecycle-file>",
- default="etc/test_lifecycle.yml",
- help=("The path to the test lifecycle tags file, relative to the root of the"
- " metadata repository. Defaults to '%default'."))
-
- parser.add_option(
- "--referencesFile", dest="references_file", metavar="<references-file>",
- default="references.yml",
- help=("The path to the metadata references file, relative to the root of the"
- " metadata repository. Defaults to '%default'."))
-
- parser.add_option(
- "--destinationFile", dest="destination_file", metavar="<destination-file>",
- default="etc/test_lifecycle.yml",
- help=("The path where the lifecycle file should be available when this script"
- " completes successfully. This path is absolute or relative to the"
- " current working directory. Defaults to '%default'."))
-
- parser.add_option("--logLevel", dest="log_level", metavar="<log-level>",
- choices=["DEBUG", "INFO", "WARNING", "ERROR"], default="INFO",
- help="The log level: DEBUG, INFO, WARNING or ERROR. Defaults to '%default'.")
-
- parser.add_option(
- "--logFile", dest="log_file", metavar="<log-file>", default=None,
- help=("The destination file for the logs. If not set the script will log to"
- " the standard output"))
-
- options, args = parser.parse_args()
-
- if len(args) != 1:
- parser.print_help(file=sys.stderr)
- print(file=sys.stderr)
- parser.error("Must specify an Evergreen project")
- evergreen_project = args[0]
-
- logging.basicConfig(format="%(asctime)s %(levelname)s %(message)s", level=options.log_level,
- filename=options.log_file)
-
- lifecycle_file_content = fetch_test_lifecycle(options.metadata_repo_url,
- options.references_file, options.lifecycle_file,
- evergreen_project, options.revision)
- if not lifecycle_file_content:
- LOGGER.error("Failed to fetch the test lifecycle tag file.")
- sys.exit(1)
- else:
- LOGGER.info("Writing the test lifecycle file to '%s'.", options.destination_file)
- with open(options.destination_file, "wb") as destf:
- destf.write(lifecycle_file_content)
- LOGGER.info("Done.")
-
-
-if __name__ == "__main__":
- main()
diff --git a/buildscripts/resmokeconfig/suites/buildscripts_test.yml b/buildscripts/resmokeconfig/suites/buildscripts_test.yml
index 3e29cd214a3..cb449733e11 100644
--- a/buildscripts/resmokeconfig/suites/buildscripts_test.yml
+++ b/buildscripts/resmokeconfig/suites/buildscripts_test.yml
@@ -9,7 +9,6 @@ selector:
- buildscripts/tests/resmokelib/utils/test_archival.py # Requires boto3.
- buildscripts/tests/test_aws_ec2.py # Requires boto3.
- buildscripts/tests/test_remote_operations.py # Requires ssh to be enabled locally.
- - buildscripts/tests/test_update_test_lifecycle.py # Test assumes POSIX path.
executor: {}
diff --git a/buildscripts/tests/ciconfig/evergreen.yml b/buildscripts/tests/ciconfig/evergreen.yml
index 92433f5e10f..3deb0f1ea3c 100644
--- a/buildscripts/tests/ciconfig/evergreen.yml
+++ b/buildscripts/tests/ciconfig/evergreen.yml
@@ -68,10 +68,6 @@ post:
params:
file_location: src/results.json
-test_lifecycle_excluded_tasks:
-- burn_in_tests
-- no_lifecycle*
-
task_groups:
- name: tg_1
max_hosts: 1
diff --git a/buildscripts/tests/ciconfig/test_evergreen.py b/buildscripts/tests/ciconfig/test_evergreen.py
index f277bb6564a..6d18266d0d1 100644
--- a/buildscripts/tests/ciconfig/test_evergreen.py
+++ b/buildscripts/tests/ciconfig/test_evergreen.py
@@ -38,14 +38,6 @@ class TestEvergreenProjectConfig(unittest.TestCase):
self.assertEqual(1, len(self.conf.task_group_names))
self.assertIn("tg_1", self.conf.task_group_names)
- def test_list_lifecycle_task_names(self):
- self.assertEqual(5, len(self.conf.lifecycle_task_names))
- self.assertIn("compile", self.conf.task_names)
- self.assertIn("passing_test", self.conf.task_names)
- self.assertIn("failing_test", self.conf.task_names)
- self.assertIn("timeout_test", self.conf.task_names)
- self.assertIn("resmoke_task", self.conf.task_names)
-
def test_list_variants(self):
self.assertEqual(4, len(self.conf.variants))
self.assertEqual(4, len(self.conf.variant_names))
diff --git a/buildscripts/tests/test_fetch_test_lifecycle.py b/buildscripts/tests/test_fetch_test_lifecycle.py
deleted file mode 100644
index c2929c1a7d0..00000000000
--- a/buildscripts/tests/test_fetch_test_lifecycle.py
+++ /dev/null
@@ -1,70 +0,0 @@
-"""Unit tests for the fetch_test_lifecycle.py script."""
-
-import unittest
-
-import buildscripts.fetch_test_lifecycle as fetch
-
-# pylint: disable=missing-docstring,protected-access
-
-
-class TestFetchTestLifecycle(unittest.TestCase):
- def test_get_metadata_revision(self):
- metadata_repo = MockMetadataRepository([("metadata_revision_05", "mongo_revision_06"),
- ("metadata_revision_04", "mongo_revision_06"),
- ("metadata_revision_03", "mongo_revision_02"),
- ("metadata_revision_02", "mongo_revision_02"),
- ("metadata_revision_01", None)])
-
- mongo_repo = MockMongoRepository([
- "mongo_revision_07", "mongo_revision_06", "mongo_revision_05", "mongo_revision_04",
- "mongo_revision_03", "mongo_revision_02", "mongo_revision_01"
- ])
-
- self._check_metadata_revision(metadata_repo, mongo_repo, "mongo_revision_07",
- "metadata_revision_05")
-
- self._check_metadata_revision(metadata_repo, mongo_repo, "mongo_revision_06",
- "metadata_revision_05")
-
- self._check_metadata_revision(metadata_repo, mongo_repo, "mongo_revision_05",
- "metadata_revision_03")
-
- self._check_metadata_revision(metadata_repo, mongo_repo, "mongo_revision_04",
- "metadata_revision_03")
-
- self._check_metadata_revision(metadata_repo, mongo_repo, "mongo_revision_03",
- "metadata_revision_03")
-
- self._check_metadata_revision(metadata_repo, mongo_repo, "mongo_revision_02",
- "metadata_revision_03")
-
- self._check_metadata_revision(metadata_repo, mongo_repo, "mongo_revision_01", None)
-
- def _check_metadata_revision(self, metadata_repo, mongo_repo, mongo_revision,
- expected_metadata_revision):
- metadata_revision = fetch._get_metadata_revision(metadata_repo, mongo_repo, "project",
- mongo_revision)
- self.assertEqual(expected_metadata_revision, metadata_revision)
-
-
-class MockMongoRepository(object):
- def __init__(self, revisions):
- self.revisions = revisions
-
- def is_ancestor(self, parent, child):
- return (parent in self.revisions and child in self.revisions
- and self.revisions.index(parent) >= self.revisions.index(child))
-
-
-class MockMetadataRepository(object):
- def __init__(self, references_revisions):
- self.references_revisions = references_revisions
-
- def list_revisions(self):
- return [r[0] for r in self.references_revisions]
-
- def get_reference(self, revision, project): # pylint: disable=unused-argument
- for (metadata_revision, mongo_revision) in self.references_revisions:
- if metadata_revision == revision:
- return mongo_revision
- return None
diff --git a/buildscripts/tests/test_update_test_lifecycle.py b/buildscripts/tests/test_update_test_lifecycle.py
deleted file mode 100644
index 11fe754ac04..00000000000
--- a/buildscripts/tests/test_update_test_lifecycle.py
+++ /dev/null
@@ -1,1121 +0,0 @@
-"""
-Tests for buildscripts/update_test_lifecycle.py.
-"""
-
-import collections
-import copy
-import datetime
-import unittest
-
-from buildscripts import update_test_lifecycle as utl
-from buildscripts.ciconfig import tags as ci_tags
-
-# pylint: disable=invalid-name,missing-docstring,protected-access,too-many-lines
-
-
-class TestTestCombination(unittest.TestCase):
- def test_equal(self):
- args = ["test1", "task1", "variant1", "distro1"]
- while args:
- t1 = utl.TestCombination(*args)
- t2 = utl.TestCombination(*args)
- self.assertTrue(t1 == t2)
- self.assertTrue(t2 == t1)
- args.pop()
-
- def test_properties(self):
- t = utl.TestCombination("test1", "task1", "variant1", "distro1")
- self.assertEqual("test1", t.test)
- self.assertEqual("task1", t.task)
- self.assertEqual("variant1", t.variant)
- self.assertEqual("distro1", t.distro)
-
- def test_equal_group_by(self):
- args = ["test1", "task1", "variant1", "distro1"]
- # yapf: disable
- group_by = [
- utl.TestCombination.GROUP_BY_TEST,
- utl.TestCombination.GROUP_BY_TASK,
- utl.TestCombination.GROUP_BY_VARIANT,
- utl.TestCombination.GROUP_BY_DISTRO
- ]
- # yapf: enable
- while args:
- t1 = utl.TestCombination(*args)
- t2 = utl.TestCombination(*args)
- self.assertTrue(t1 == t2)
- self.assertTrue(t2 == t1)
- self.assertEqual(group_by.pop(), t1.group_by)
- args.pop()
-
- def test_tag(self):
- args = ["test1", "task1", "variant1", "distro1"]
- tags = [
- "unreliable",
- "unreliable|task1",
- "unreliable|task1|variant1",
- "unreliable|task1|variant1|distro1",
- ]
- while args:
- t = utl.TestCombination(*args)
- self.assertEqual(tags.pop(), t.tag)
- args.pop()
-
- def test_from_tag(self):
- test = "testA"
-
- combination = utl.TestCombination.from_tag(test, "unreliable")
- self.assertEqual(combination, utl.TestCombination(test, None, None, None))
-
- combination = utl.TestCombination.from_tag(test, "unreliable|taskA")
- self.assertEqual(combination, utl.TestCombination(test, "taskA", None, None))
-
- combination = utl.TestCombination.from_tag(test, "unreliable|taskA|variantA")
- self.assertEqual(combination, utl.TestCombination(test, "taskA", "variantA", None))
-
- combination = utl.TestCombination.from_tag(test, "unreliable|taskA|variantA|distroA")
- self.assertEqual(combination, utl.TestCombination(test, "taskA", "variantA", "distroA"))
-
-
-class TestTestHistory(unittest.TestCase):
- def test_get_rates_by_distro(self):
- th = utl.TestHistory("test1.js")
- th.add_reliable_period_stats([
- {
- "test_file": "test1.js", "task_name": "task1", "variant": "v", "distro": "d1",
- "num_pass": 1, "num_fail": 0
- },
- {
- "test_file": "test1.js", "task_name": "task1", "variant": "v", "distro": "d1",
- "num_pass": 0, "num_fail": 1
- },
- {
- "test_file": "test1.js", "task_name": "task1", "variant": "v", "distro": "d2",
- "num_pass": 0, "num_fail": 1
- },
- {
- "test_file": "test1.js", "task_name": "task2", "variant": "v", "distro": "d2",
- "num_pass": 1, "num_fail": 0
- },
- ])
- expected = [
- (utl.TestCombination("test1.js", "task1", "v", "d1"), 0.5, 2),
- (utl.TestCombination("test1.js", "task1", "v", "d2"), 1.0, 1),
- (utl.TestCombination("test1.js", "task2", "v", "d2"), 0.0, 1),
- ]
-
- rates = th.get_reliable_period_rates(group_by=utl.TestCombination.GROUP_BY_DISTRO)
-
- self.assertListEqual(expected, rates)
-
- def test_get_rates_by_variant(self):
- th = utl.TestHistory("test1.js")
- th.add_reliable_period_stats([
- {
- "test_file": "test1.js", "task_name": "task1", "variant": "v1", "distro": "d1",
- "num_pass": 1, "num_fail": 0
- },
- {
- "test_file": "test1.js", "task_name": "task1", "variant": "v1", "distro": "d2",
- "num_pass": 0, "num_fail": 1
- },
- {
- "test_file": "test1.js", "task_name": "task1", "variant": "v2", "distro": "d2",
- "num_pass": 0, "num_fail": 1
- },
- {
- "test_file": "test1.js", "task_name": "task2", "variant": "v2", "distro": "d2",
- "num_pass": 1, "num_fail": 0
- },
- ])
- expected = [
- (utl.TestCombination("test1.js", "task1", "v1", None), 0.5, 2),
- (utl.TestCombination("test1.js", "task1", "v2", None), 1.0, 1),
- (utl.TestCombination("test1.js", "task2", "v2", None), 0.0, 1),
- ]
-
- rates = th.get_reliable_period_rates(group_by=utl.TestCombination.GROUP_BY_VARIANT)
-
- self.assertListEqual(expected, rates)
-
- def test_get_rates_by_task(self):
- th = utl.TestHistory("test1.js")
- th.add_reliable_period_stats([
- {
- "test_file": "test1.js", "task_name": "task1", "variant": "v1", "distro": "d1",
- "num_pass": 1, "num_fail": 0
- },
- {
- "test_file": "test1.js", "task_name": "task1", "variant": "v1", "distro": "d2",
- "num_pass": 0, "num_fail": 1
- },
- {
- "test_file": "test1.js", "task_name": "task2", "variant": "v2", "distro": "d3",
- "num_pass": 2, "num_fail": 1
- },
- {
- "test_file": "test1.js", "task_name": "task2", "variant": "v2", "distro": "d4",
- "num_pass": 1, "num_fail": 0
- },
- ])
- expected = [
- (utl.TestCombination("test1.js", "task1", None, None), 0.5, 2),
- (utl.TestCombination("test1.js", "task2", None, None), 0.25, 4),
- ]
-
- rates = th.get_reliable_period_rates(group_by=utl.TestCombination.GROUP_BY_TASK)
-
- self.assertListEqual(expected, rates)
-
- def test_get_rates_by_test(self):
- th = utl.TestHistory("test1.js")
- th.add_reliable_period_stats([
- {
- "test_file": "test1.js", "task_name": "task1", "variant": "v1", "distro": "d1",
- "num_pass": 1, "num_fail": 0
- },
- {
- "test_file": "test1.js", "task_name": "task1", "variant": "v1", "distro": "d2",
- "num_pass": 0, "num_fail": 1
- },
- {
- "test_file": "test1.js", "task_name": "task2", "variant": "v2", "distro": "d3",
- "num_pass": 2, "num_fail": 1
- },
- {
- "test_file": "test1.js", "task_name": "task3", "variant": "v2", "distro": "d4",
- "num_pass": 1, "num_fail": 0
- },
- ])
- expected = [
- (utl.TestCombination("test1.js", None, None, None), float(1) / 3, 6),
- ]
-
- rates = th.get_reliable_period_rates(group_by=utl.TestCombination.GROUP_BY_TEST)
-
- self.assertListEqual(expected, rates)
-
-
-class TestValidateConfig(unittest.TestCase):
- """
- Tests for the validate_config() function.
- """
-
- # yapf: disable
- CONFIG = utl.Config(
- test_fail_rates=utl.Rates(acceptable=0, unacceptable=1),
- task_fail_rates=utl.Rates(acceptable=0, unacceptable=1),
- variant_fail_rates=utl.Rates(acceptable=0, unacceptable=1),
- distro_fail_rates=utl.Rates(acceptable=0, unacceptable=1),
- reliable_min_runs=2,
- reliable_time_period=datetime.timedelta(days=1),
- unreliable_min_runs=2,
- unreliable_time_period=datetime.timedelta(days=1))
- # yapf: enable
-
- def test_acceptable_test_fail_rate(self):
- """
- Tests the validation of the 'test_fail_rates.acceptable' attribute.
- """
-
- with self.assertRaises(TypeError):
- config = self.CONFIG._replace(
- test_fail_rates=self.CONFIG.test_fail_rates._replace(acceptable="not a number"))
- utl.validate_config(config)
-
- with self.assertRaises(ValueError):
- config = self.CONFIG._replace(
- test_fail_rates=self.CONFIG.test_fail_rates._replace(acceptable=-1))
- utl.validate_config(config)
-
- with self.assertRaises(ValueError):
- config = self.CONFIG._replace(
- test_fail_rates=self.CONFIG.test_fail_rates._replace(acceptable=2))
- utl.validate_config(config)
-
- def test_unacceptable_test_fail_rate(self):
- """
- Tests the validation of the 'test_fail_rates.unacceptable' attribute.
- """
-
- with self.assertRaises(TypeError):
- config = self.CONFIG._replace(
- test_fail_rates=self.CONFIG.test_fail_rates._replace(unacceptable="not a number"))
- utl.validate_config(config)
-
- with self.assertRaises(ValueError):
- config = self.CONFIG._replace(
- test_fail_rates=self.CONFIG.test_fail_rates._replace(unacceptable=-1))
- utl.validate_config(config)
-
- with self.assertRaises(ValueError):
- config = self.CONFIG._replace(
- test_fail_rates=self.CONFIG.test_fail_rates._replace(unacceptable=2))
- utl.validate_config(config)
-
- def test_test_fail_rates(self):
- """
- Tests the validation of the 'test_fail_rates' attribute.
- """
-
- with self.assertRaises(ValueError):
- config = self.CONFIG._replace(
- test_fail_rates=self.CONFIG.test_fail_rates._replace(acceptable=0.9,
- unacceptable=0.1))
- utl.validate_config(config)
-
- def test_acceptable_task_fail_rate(self):
- """
- Tests the validation of the 'test_fail_rates.acceptable' attribute.
- """
-
- with self.assertRaises(TypeError):
- config = self.CONFIG._replace(
- task_fail_rates=self.CONFIG.task_fail_rates._replace(acceptable="not a number"))
- utl.validate_config(config)
-
- with self.assertRaises(ValueError):
- config = self.CONFIG._replace(
- task_fail_rates=self.CONFIG.task_fail_rates._replace(acceptable=-1))
- utl.validate_config(config)
-
- with self.assertRaises(ValueError):
- config = self.CONFIG._replace(
- task_fail_rates=self.CONFIG.task_fail_rates._replace(acceptable=2))
- utl.validate_config(config)
-
- def test_unacceptable_task_fail_rate(self):
- """
- Tests the validation of the 'task_fail_rates.unacceptable' attribute.
- """
-
- with self.assertRaises(TypeError):
- config = self.CONFIG._replace(
- task_fail_rates=self.CONFIG.task_fail_rates._replace(unacceptable="not a number"))
- utl.validate_config(config)
-
- with self.assertRaises(ValueError):
- config = self.CONFIG._replace(
- task_fail_rates=self.CONFIG.task_fail_rates._replace(unacceptable=-1))
- utl.validate_config(config)
-
- with self.assertRaises(ValueError):
- config = self.CONFIG._replace(
- task_fail_rates=self.CONFIG.task_fail_rates._replace(unacceptable=2))
- utl.validate_config(config)
-
- def test_task_fail_rates(self):
- """
- Tests the validation of the 'task_fail_rates' attribute.
- """
-
- with self.assertRaises(ValueError):
- config = self.CONFIG._replace(
- task_fail_rates=self.CONFIG.task_fail_rates._replace(acceptable=0.9,
- unacceptable=0.1))
- utl.validate_config(config)
-
- def test_acceptable_variant_fail_rate(self):
- """
- Tests the validation of the 'variant_fail_rates.acceptable' attribute.
- """
-
- with self.assertRaises(TypeError):
- config = self.CONFIG._replace(
- variant_fail_rates=self.CONFIG.variant_fail_rates._replace(
- acceptable="not a number"))
- utl.validate_config(config)
-
- with self.assertRaises(ValueError):
- config = self.CONFIG._replace(
- variant_fail_rates=self.CONFIG.variant_fail_rates._replace(acceptable=-1))
- utl.validate_config(config)
-
- with self.assertRaises(ValueError):
- config = self.CONFIG._replace(
- variant_fail_rates=self.CONFIG.variant_fail_rates._replace(acceptable=2))
- utl.validate_config(config)
-
- def test_unacceptable_variant_fail_rate(self):
- """
- Tests the validation of the 'variant_fail_rates.unacceptable' attribute.
- """
-
- with self.assertRaises(TypeError):
- config = self.CONFIG._replace(
- variant_fail_rates=self.CONFIG.variant_fail_rates._replace(
- unacceptable="not a number"))
- utl.validate_config(config)
-
- with self.assertRaises(ValueError):
- config = self.CONFIG._replace(
- variant_fail_rates=self.CONFIG.variant_fail_rates._replace(unacceptable=-1))
- utl.validate_config(config)
-
- with self.assertRaises(ValueError):
- config = self.CONFIG._replace(
- variant_fail_rates=self.CONFIG.variant_fail_rates._replace(unacceptable=2))
- utl.validate_config(config)
-
- def test_variant_fail_rates(self):
- """
- Tests the validation of the 'variant_fail_rates' attribute.
- """
-
- with self.assertRaises(ValueError):
- config = self.CONFIG._replace(
- variant_fail_rates=self.CONFIG.variant_fail_rates._replace(
- acceptable=0.9, unacceptable=0.1))
- utl.validate_config(config)
-
- def test_acceptable_distro_fail_rate(self):
- """
- Tests the validation of the 'distro_fail_rates.acceptable' attribute.
- """
-
- with self.assertRaises(TypeError):
- config = self.CONFIG._replace(
- distro_fail_rates=self.CONFIG.distro_fail_rates._replace(acceptable="not a number"))
- utl.validate_config(config)
-
- with self.assertRaises(ValueError):
- config = self.CONFIG._replace(
- distro_fail_rates=self.CONFIG.distro_fail_rates._replace(acceptable=-1))
- utl.validate_config(config)
-
- with self.assertRaises(ValueError):
- config = self.CONFIG._replace(
- distro_fail_rates=self.CONFIG.distro_fail_rates._replace(acceptable=2))
- utl.validate_config(config)
-
- def test_unacceptable_distro_fail_rate(self):
- """
- Tests the validation of the 'distro_fail_rates.unacceptable' attribute.
- """
-
- with self.assertRaises(TypeError):
- config = self.CONFIG._replace(
- distro_fail_rates=self.CONFIG.distro_fail_rates._replace(
- unacceptable="not a number"))
- utl.validate_config(config)
-
- with self.assertRaises(ValueError):
- config = self.CONFIG._replace(
- distro_fail_rates=self.CONFIG.distro_fail_rates._replace(unacceptable=-1))
- utl.validate_config(config)
-
- with self.assertRaises(ValueError):
- config = self.CONFIG._replace(
- distro_fail_rates=self.CONFIG.distro_fail_rates._replace(unacceptable=2))
- utl.validate_config(config)
-
- def test_distro_fail_rates(self):
- """
- Tests the validation of the 'distro_fail_rates' attribute.
- """
-
- with self.assertRaises(ValueError):
- config = self.CONFIG._replace(
- distro_fail_rates=self.CONFIG.distro_fail_rates._replace(
- acceptable=0.9, unacceptable=0.1))
- utl.validate_config(config)
-
- def test_reliable_min_runs(self):
- """
- Tests the validation of the 'reliable_min_runs' attribute.
- """
-
- with self.assertRaises(TypeError):
- config = self.CONFIG._replace(reliable_min_runs="not a number")
- utl.validate_config(config)
-
- with self.assertRaises(ValueError):
- config = self.CONFIG._replace(reliable_min_runs=-1)
- utl.validate_config(config)
-
- with self.assertRaises(ValueError):
- config = self.CONFIG._replace(reliable_min_runs=0)
- utl.validate_config(config)
-
- with self.assertRaises(ValueError):
- config = self.CONFIG._replace(reliable_min_runs=1.5)
- utl.validate_config(config)
-
- def test_reliable_time_period(self):
- """
- Tests the validation of the 'reliable_time_period' attribute.
- """
-
- with self.assertRaises(TypeError):
- config = self.CONFIG._replace(reliable_time_period="not a datetime.timedelta")
- utl.validate_config(config)
-
- with self.assertRaises(ValueError):
- config = self.CONFIG._replace(reliable_time_period=datetime.timedelta(days=-1))
- utl.validate_config(config)
-
- with self.assertRaises(ValueError):
- config = self.CONFIG._replace(reliable_time_period=datetime.timedelta(days=0))
- utl.validate_config(config)
-
- with self.assertRaises(ValueError):
- config = self.CONFIG._replace(reliable_time_period=datetime.timedelta(days=1, hours=1))
- utl.validate_config(config)
-
- def test_unreliable_min_runs(self):
- """
- Tests the validation of the 'unreliable_min_runs' attribute.
- """
-
- with self.assertRaises(TypeError):
- config = self.CONFIG._replace(unreliable_min_runs="not a number")
- utl.validate_config(config)
-
- with self.assertRaises(ValueError):
- config = self.CONFIG._replace(unreliable_min_runs=-1)
- utl.validate_config(config)
-
- with self.assertRaises(ValueError):
- config = self.CONFIG._replace(unreliable_min_runs=0)
- utl.validate_config(config)
-
- with self.assertRaises(ValueError):
- config = self.CONFIG._replace(unreliable_min_runs=1.5)
- utl.validate_config(config)
-
- def test_unreliable_time_period(self):
- """
- Tests the validation of the 'unreliable_time_period' attribute.
- """
-
- with self.assertRaises(TypeError):
- config = self.CONFIG._replace(unreliable_time_period="not a datetime.timedelta")
- utl.validate_config(config)
-
- with self.assertRaises(ValueError):
- config = self.CONFIG._replace(unreliable_time_period=datetime.timedelta(days=-1))
- utl.validate_config(config)
-
- with self.assertRaises(ValueError):
- config = self.CONFIG._replace(unreliable_time_period=datetime.timedelta(days=0))
- utl.validate_config(config)
-
- with self.assertRaises(ValueError):
- config = self.CONFIG._replace(
- unreliable_time_period=datetime.timedelta(days=1, hours=1))
- utl.validate_config(config)
-
-
-class TestUpdateTags(unittest.TestCase): # pylint: disable=too-many-public-methods
- """
- Tests for the update_tags() function.
- """
- # yapf: disable
- CONFIG = utl.Config(
- test_fail_rates=utl.Rates(acceptable=0, unacceptable=1),
- task_fail_rates=utl.Rates(acceptable=0, unacceptable=1),
- variant_fail_rates=utl.Rates(acceptable=0, unacceptable=1),
- distro_fail_rates=utl.Rates(acceptable=0, unacceptable=1),
- reliable_min_runs=2,
- reliable_time_period=datetime.timedelta(days=1),
- unreliable_min_runs=2,
- unreliable_time_period=datetime.timedelta(days=1))
- # yapf: enable
-
- STATS = {
- "test": "jstests/core/all.js", "task": "jsCore_WT", "variant": "linux-64",
- "distro": "rhel62", "num_pass": 0, "num_fail": 0
- }
-
- @staticmethod
- def _stats( # pylint: disable=too-many-arguments
- test="jstests/core/all.js", task="jsCore_WT", variant="linux-64", distro="rhel62",
- num_pass=0, num_fail=0):
- return {
- "test_file": test,
- "task_name": task,
- "variant": variant,
- "distro": distro,
- "num_pass": num_pass,
- "num_fail": num_fail,
- }
-
- def assert_has_only_js_tests(self, lifecycle):
- """
- Raises an AssertionError exception if 'lifecycle' is not of the following form:
-
- selector:
- js_test:
- ...
- """
-
- self.assertIn("selector", lifecycle.raw)
- self.assertEqual(1, len(lifecycle.raw), msg=str(lifecycle.raw))
- self.assertIn("js_test", lifecycle.raw["selector"])
- self.assertEqual(1, len(lifecycle.raw["selector"]), msg=str(lifecycle.raw))
-
- return lifecycle.raw["selector"]["js_test"]
-
- def transition_from_reliable_to_unreliable(self, config, expected_tags):
- """
- Tests that update_tags() tags a formerly reliable combination as being unreliable.
- """
-
- initial_tags = collections.OrderedDict()
- lifecycle = ci_tags.TagsConfig.from_dict(
- dict(selector=dict(js_test=copy.deepcopy(initial_tags))))
- summary_lifecycle = utl.TagsConfigWithChangelog(lifecycle)
- self.assertEqual(collections.OrderedDict(), self.assert_has_only_js_tests(lifecycle))
-
- test_history = utl.TestHistory("jstests/core/all.js")
- stats = [
- self._stats(num_pass=0, num_fail=1),
- self._stats(num_pass=0, num_fail=1, task="jsCore"),
- self._stats(num_pass=0, num_fail=1, variant="linux-64-debug"),
- self._stats(num_pass=1, num_fail=0),
- self._stats(num_pass=0, num_fail=1, distro="rhel55"),
- ]
- test_history.add_reliable_period_stats(stats)
- test_history.add_unreliable_period_stats(stats)
-
- utl.validate_config(config)
- utl.update_tags(summary_lifecycle, config, test_history)
- updated_tags = self.assert_has_only_js_tests(lifecycle)
- self.assertEqual(updated_tags, expected_tags)
-
- def test_transition_test_from_reliable_to_unreliable(self):
- """
- Tests that update_tags() tags a formerly reliable (test,) combination as being unreliable.
- """
-
- config = self.CONFIG._replace(
- test_fail_rates=self.CONFIG.test_fail_rates._replace(unacceptable=0.1))
-
- self.transition_from_reliable_to_unreliable(
- config, collections.OrderedDict([
- ("jstests/core/all.js", ["unreliable"]),
- ]))
-
- def test_transition_task_from_reliable_to_unreliable(self):
- """
- Tests that update_tags() tags a formerly reliable (test, task) combination as being
- unreliable.
- """
-
- config = self.CONFIG._replace(
- task_fail_rates=self.CONFIG.task_fail_rates._replace(unacceptable=0.1))
-
- self.transition_from_reliable_to_unreliable(
- config, collections.OrderedDict([
- ("jstests/core/all.js", ["unreliable|jsCore_WT"]),
- ]))
-
- def test_transition_variant_from_reliable_to_unreliable(self):
- """
- Tests that update_tags() tags a formerly reliable (test, task, variant) combination as being
- unreliable.
- """
-
- config = self.CONFIG._replace(
- variant_fail_rates=self.CONFIG.variant_fail_rates._replace(unacceptable=0.1))
-
- self.transition_from_reliable_to_unreliable(
- config,
- collections.OrderedDict([
- ("jstests/core/all.js", ["unreliable|jsCore_WT|linux-64"]),
- ]))
-
- def test_transition_distro_from_reliable_to_unreliable(self):
- """
- Tests that update_tags() tags a formerly reliable (test, task, variant, distro) combination
- as being unreliable.
- """
-
- config = self.CONFIG._replace(
- distro_fail_rates=self.CONFIG.distro_fail_rates._replace(unacceptable=0.1))
-
- self.transition_from_reliable_to_unreliable(
- config,
- collections.OrderedDict([
- ("jstests/core/all.js", ["unreliable|jsCore_WT|linux-64|rhel62"]),
- ]))
-
- def test_transition_from_reliable_to_unreliable(self):
- """
- Tests that update_tags() tags multiple formerly reliable combination as being unreliable.
- """
-
- config = self.CONFIG._replace(
- test_fail_rates=self.CONFIG.test_fail_rates._replace(unacceptable=0.1),
- task_fail_rates=self.CONFIG.task_fail_rates._replace(unacceptable=0.1),
- variant_fail_rates=self.CONFIG.variant_fail_rates._replace(unacceptable=0.1),
- distro_fail_rates=self.CONFIG.distro_fail_rates._replace(unacceptable=0.1))
-
- self.transition_from_reliable_to_unreliable(
- config,
- collections.OrderedDict([
- ("jstests/core/all.js", [
- "unreliable",
- "unreliable|jsCore_WT",
- "unreliable|jsCore_WT|linux-64",
- "unreliable|jsCore_WT|linux-64|rhel62",
- ]),
- ]))
-
- def transition_from_unreliable_to_reliable(self, config, initial_tags):
- """
- Tests that update_tags() untags a formerly unreliable combination after it has become
- reliable again.
- """
-
- lifecycle = ci_tags.TagsConfig.from_dict(
- dict(selector=dict(js_test=copy.deepcopy(initial_tags))))
- summary_lifecycle = utl.TagsConfigWithChangelog(lifecycle)
- self.assertEqual(initial_tags, self.assert_has_only_js_tests(lifecycle))
-
- test_history = utl.TestHistory("jstests/core/all.js")
- stats = [
- self._stats(num_pass=1, num_fail=0),
- self._stats(num_pass=1, num_fail=0, task="jsCore"),
- self._stats(num_pass=1, num_fail=0, variant="linux-64-debug"),
- self._stats(num_pass=0, num_fail=1),
- self._stats(num_pass=1, num_fail=0, distro="rhel55"),
- ]
- test_history.add_reliable_period_stats(stats)
- test_history.add_unreliable_period_stats(stats)
-
- utl.validate_config(config)
- utl.update_tags(summary_lifecycle, config, test_history)
- updated_tags = self.assert_has_only_js_tests(lifecycle)
- self.assertEqual(updated_tags, collections.OrderedDict())
-
- def test_non_running_in_reliable_period_is_reliable(self):
- """
- Tests that tests that have a failure rate above the unacceptable rate during the unreliable
- period but haven't run during the reliable period are marked as reliable.
- """
- config = self.CONFIG._replace(
- test_fail_rates=self.CONFIG.test_fail_rates._replace(unacceptable=0.1),
- task_fail_rates=self.CONFIG.task_fail_rates._replace(unacceptable=0.1),
- variant_fail_rates=self.CONFIG.variant_fail_rates._replace(unacceptable=0.1),
- distro_fail_rates=self.CONFIG.distro_fail_rates._replace(unacceptable=0.1),
- unreliable_time_period=datetime.timedelta(days=2))
-
- initial_tags = collections.OrderedDict([
- ("jstests/core/all.js", [
- "unreliable",
- "unreliable|jsCore_WT",
- "unreliable|jsCore_WT|linux-64",
- "unreliable|jsCore_WT|linux-64|rhel62",
- ]),
- ])
-
- lifecycle = ci_tags.TagsConfig.from_dict(
- dict(selector=dict(js_test=copy.deepcopy(initial_tags))))
- summary_lifecycle = utl.TagsConfigWithChangelog(lifecycle)
- self.assertEqual(initial_tags, self.assert_has_only_js_tests(lifecycle))
-
- test_history = utl.TestHistory("jstests/core/all.js")
- # The test did not run on the reliable period on linux-64.
- reliable_period_stats = [
- self._stats(num_pass=3, num_fail=0, variant="linux-alt", distro="debian7"),
- ]
- unreliable_period_stats = [
- # Failing.
- self._stats(num_pass=0, num_fail=2),
- self._stats(num_pass=3, num_fail=0, variant="linux-alt", distro="debian7"),
- ]
- test_history.add_reliable_period_stats(reliable_period_stats)
- test_history.add_unreliable_period_stats(unreliable_period_stats)
-
- utl.validate_config(config)
- utl.update_tags(summary_lifecycle, config, test_history)
- updated_tags = self.assert_has_only_js_tests(lifecycle)
- # The tags for variant and distro have been removed.
- self.assertEqual(
- updated_tags,
- collections.OrderedDict([("jstests/core/all.js", ["unreliable",
- "unreliable|jsCore_WT"])]))
-
- def test_non_running_at_all_is_reliable(self):
- """
- Tests that tests that are tagged as unreliable but no longer running (either during the
- reliable or the unreliable period) have their tags removed.
- """
- config = self.CONFIG
-
- test_history = utl.TestHistory("jstests/core/all.js")
- test_history2 = utl.TestHistory("jstests/core/all2.js")
- initial_tags = collections.OrderedDict([
- ("jstests/core/all2.js", [
- "unreliable",
- "unreliable|jsCore_WT",
- "unreliable|jsCore_WT|linux-64",
- "unreliable|jsCore_WT|linux-64|rhel62",
- ]),
- ])
-
- lifecycle = ci_tags.TagsConfig.from_dict(
- dict(selector=dict(js_test=copy.deepcopy(initial_tags))))
- summary_lifecycle = utl.TagsConfigWithChangelog(lifecycle)
- self.assertEqual(initial_tags, self.assert_has_only_js_tests(lifecycle))
-
- # all2.js did not run at all
- test_history.add_reliable_period_stats([self._stats()])
-
- utl.validate_config(config)
- utl.update_tags(summary_lifecycle, config, test_history)
- utl.update_tags(summary_lifecycle, config, test_history2)
- updated_tags = self.assert_has_only_js_tests(lifecycle)
- # The tags for variant and distro have been removed.
- self.assertEqual(updated_tags, collections.OrderedDict([]))
-
- def test_transition_test_from_unreliable_to_reliable(self):
- """
- Tests that update_tags() untags a formerly unreliable (test,) combination after it has
- become reliable again.
- """
-
- config = self.CONFIG._replace(
- test_fail_rates=self.CONFIG.test_fail_rates._replace(acceptable=0.9))
-
- self.transition_from_unreliable_to_reliable(
- config, collections.OrderedDict([
- ("jstests/core/all.js", ["unreliable"]),
- ]))
-
- def test_transition_task_from_unreliable_to_reliable(self):
- """
- Tests that update_tags() untags a formerly unreliable (test, task) combination after it has
- become reliable again.
- """
-
- config = self.CONFIG._replace(
- task_fail_rates=self.CONFIG.task_fail_rates._replace(acceptable=0.9))
-
- self.transition_from_unreliable_to_reliable(
- config, collections.OrderedDict([
- ("jstests/core/all.js", ["unreliable|jsCore_WT"]),
- ]))
-
- def test_transition_variant_from_unreliable_to_reliable(self):
- """
- Tests that update_tags() untags a formerly unreliable (test, task, variant) combination
- after it has become reliable again.
- """
-
- config = self.CONFIG._replace(
- variant_fail_rates=self.CONFIG.variant_fail_rates._replace(acceptable=0.9))
-
- self.transition_from_unreliable_to_reliable(
- config,
- collections.OrderedDict([
- ("jstests/core/all.js", ["unreliable|jsCore_WT|linux-64"]),
- ]))
-
- def test_transition_distro_from_unreliable_to_reliable(self):
- """
- Tests that update_tags() untags a formerly unreliable (test, task, variant, distro)
- combination after it has become reliable again.
- """
-
- config = self.CONFIG._replace(
- distro_fail_rates=self.CONFIG.distro_fail_rates._replace(acceptable=0.9))
-
- self.transition_from_unreliable_to_reliable(
- config,
- collections.OrderedDict([
- ("jstests/core/all.js", ["unreliable|jsCore_WT|linux-64|rhel62"]),
- ]))
-
- def test_transition_from_unreliable_to_reliable(self):
- """
- Tests that update_tags() untags multiple formerly unreliable combination after it has become
- reliable again.
- """
-
- config = self.CONFIG._replace(
- test_fail_rates=self.CONFIG.test_fail_rates._replace(acceptable=0.9),
- task_fail_rates=self.CONFIG.task_fail_rates._replace(acceptable=0.9),
- variant_fail_rates=self.CONFIG.variant_fail_rates._replace(acceptable=0.9),
- distro_fail_rates=self.CONFIG.distro_fail_rates._replace(acceptable=0.9))
-
- self.transition_from_unreliable_to_reliable(
- config,
- collections.OrderedDict([
- ("jstests/core/all.js", [
- "unreliable",
- "unreliable|jsCore_WT",
- "unreliable|jsCore_WT|linux-64",
- "unreliable|jsCore_WT|linux-64|rhel62",
- ]),
- ]))
-
- def test_remain_reliable(self):
- """
- Tests that update_tags() preserves the absence of tags for reliable combinations.
- """
-
- config = self.CONFIG._replace(
- test_fail_rates=self.CONFIG.test_fail_rates._replace(acceptable=0.9),
- task_fail_rates=self.CONFIG.task_fail_rates._replace(acceptable=0.9),
- variant_fail_rates=self.CONFIG.variant_fail_rates._replace(acceptable=0.9),
- distro_fail_rates=self.CONFIG.distro_fail_rates._replace(acceptable=0.9))
-
- initial_tags = collections.OrderedDict()
- lifecycle = ci_tags.TagsConfig.from_dict(
- dict(selector=dict(js_test=copy.deepcopy(initial_tags))))
- summary_lifecycle = utl.TagsConfigWithChangelog(lifecycle)
- self.assertEqual(initial_tags, self.assert_has_only_js_tests(lifecycle))
-
- test_history = utl.TestHistory("jstests/core/all.js")
- stats = [
- self._stats(num_pass=1, num_fail=0),
- self._stats(num_pass=1, num_fail=0, task="jsCore"),
- self._stats(num_pass=1, num_fail=0, variant="linux-64-debug"),
- self._stats(num_pass=0, num_fail=1),
- self._stats(num_pass=1, num_fail=0, distro="rhel55"),
- ]
- test_history.add_reliable_period_stats(stats)
-
- utl.validate_config(config)
- utl.update_tags(summary_lifecycle, config, test_history)
- updated_tags = self.assert_has_only_js_tests(lifecycle)
- self.assertEqual(updated_tags, initial_tags)
-
- def test_remain_unreliable(self):
- """
- Tests that update_tags() preserves the tags for unreliable combinations.
- """
-
- config = self.CONFIG._replace(
- test_fail_rates=self.CONFIG.test_fail_rates._replace(unacceptable=0.1),
- task_fail_rates=self.CONFIG.task_fail_rates._replace(unacceptable=0.1),
- variant_fail_rates=self.CONFIG.variant_fail_rates._replace(unacceptable=0.1),
- distro_fail_rates=self.CONFIG.distro_fail_rates._replace(unacceptable=0.1))
-
- initial_tags = collections.OrderedDict([
- ("jstests/core/all.js", [
- "unreliable",
- "unreliable|jsCore_WT",
- "unreliable|jsCore_WT|linux-64",
- "unreliable|jsCore_WT|linux-64|rhel62",
- ]),
- ])
-
- lifecycle = ci_tags.TagsConfig.from_dict(
- dict(selector=dict(js_test=copy.deepcopy(initial_tags))))
- summary_lifecycle = utl.TagsConfigWithChangelog(lifecycle)
- self.assertEqual(initial_tags, self.assert_has_only_js_tests(lifecycle))
-
- test_history = utl.TestHistory("jstests/core/all.js")
- stats = [
- self._stats(num_pass=0, num_fail=1),
- self._stats(num_pass=0, num_fail=1, task="jsCore"),
- self._stats(num_pass=0, num_fail=1, variant="linux-64-debug"),
- self._stats(num_pass=1, num_fail=0),
- self._stats(num_pass=0, num_fail=1, distro="rhel55"),
- ]
- test_history.add_reliable_period_stats(stats)
-
- utl.validate_config(config)
- utl.update_tags(summary_lifecycle, config, test_history)
- updated_tags = self.assert_has_only_js_tests(lifecycle)
- self.assertEqual(updated_tags, initial_tags)
-
- def test_obeys_reliable_min_runs(self):
- """
- Tests that update_tags() considers a test reliable if it has fewer than 'reliable_min_runs'.
- """
-
- config = self.CONFIG._replace(
- test_fail_rates=self.CONFIG.test_fail_rates._replace(acceptable=0.9),
- task_fail_rates=self.CONFIG.task_fail_rates._replace(acceptable=0.9),
- variant_fail_rates=self.CONFIG.variant_fail_rates._replace(acceptable=0.9),
- distro_fail_rates=self.CONFIG.distro_fail_rates._replace(acceptable=0.9),
- reliable_min_runs=100)
-
- self.transition_from_unreliable_to_reliable(
- config,
- collections.OrderedDict([
- ("jstests/core/all.js", [
- "unreliable",
- "unreliable|jsCore_WT",
- "unreliable|jsCore_WT|linux-64",
- "unreliable|jsCore_WT|linux-64|rhel62",
- ]),
- ]))
-
- def test_obeys_unreliable_min_runs(self):
- """
- Tests that update_tags() only considers a test unreliable if it has more than
- 'unreliable_min_runs'.
- """
-
- config = self.CONFIG._replace(
- test_fail_rates=self.CONFIG.test_fail_rates._replace(unacceptable=0.1),
- task_fail_rates=self.CONFIG.task_fail_rates._replace(unacceptable=0.1),
- variant_fail_rates=self.CONFIG.variant_fail_rates._replace(unacceptable=0.1),
- distro_fail_rates=self.CONFIG.distro_fail_rates._replace(unacceptable=0.1),
- unreliable_min_runs=100)
-
- initial_tags = collections.OrderedDict()
- lifecycle = ci_tags.TagsConfig.from_dict(
- dict(selector=dict(js_test=copy.deepcopy(initial_tags))))
- summary_lifecycle = utl.TagsConfigWithChangelog(lifecycle)
- self.assertEqual(initial_tags, self.assert_has_only_js_tests(lifecycle))
-
- test_history = utl.TestHistory("jstests/core/all.js")
- stats = [
- self._stats(num_pass=0, num_fail=1),
- self._stats(num_pass=0, num_fail=1, task="jsCore"),
- self._stats(num_pass=0, num_fail=1, variant="linux-64-debug"),
- self._stats(num_pass=1, num_fail=0),
- self._stats(num_pass=0, num_fail=1, distro="rhel55"),
- ]
- test_history.add_reliable_period_stats(stats)
- test_history.add_unreliable_period_stats(stats)
-
- utl.validate_config(config)
- utl.update_tags(summary_lifecycle, config, test_history)
- updated_tags = self.assert_has_only_js_tests(lifecycle)
- self.assertEqual(updated_tags, initial_tags)
-
-
-class TestCleanUpTags(unittest.TestCase):
- @classmethod
- def setUpClass(cls):
- cls.evg = MockEvergreenConfig(
- ["task1", "task2", "task3"], {
- "variant1": {"tasks": ["task1", "task2"], "distros": ["distro1"]},
- "variant2": {"tasks": ["task3"], "distros": ["distro2"]}
- })
-
- def test_is_unreliable_tag_relevant(self):
- self.assertTrue(utl._is_tag_still_relevant(self.evg, "unreliable"))
-
- def test_is_unknown_task_relevant(self):
- self.assertFalse(utl._is_tag_still_relevant(self.evg, "unreliable|task_unknown"))
-
- def test_is_known_task_relevant(self):
- self.assertTrue(utl._is_tag_still_relevant(self.evg, "unreliable|task1"))
- self.assertTrue(utl._is_tag_still_relevant(self.evg, "unreliable|task2"))
- self.assertTrue(utl._is_tag_still_relevant(self.evg, "unreliable|task3"))
-
- def test_is_unknown_variant_relevant(self):
- self.assertFalse(utl._is_tag_still_relevant(self.evg, "unreliable|task1|variant3"))
-
- def test_is_unknown_task_variant_relevant(self):
- self.assertFalse(utl._is_tag_still_relevant(self.evg, "unreliable|task3|variant1"))
- self.assertFalse(utl._is_tag_still_relevant(self.evg, "unreliable|task1|variant2"))
-
- def test_is_known_task_variant_relevant(self):
- self.assertTrue(utl._is_tag_still_relevant(self.evg, "unreliable|task1|variant1"))
- self.assertTrue(utl._is_tag_still_relevant(self.evg, "unreliable|task2|variant1"))
- self.assertTrue(utl._is_tag_still_relevant(self.evg, "unreliable|task3|variant2"))
-
- def test_is_unknown_task_variant_distro_relevant(self):
- self.assertFalse(utl._is_tag_still_relevant(self.evg, "unreliable|task1|variant1|distro2"))
- self.assertFalse(utl._is_tag_still_relevant(self.evg, "unreliable|task3|variant2|distro1"))
-
- def test_is_known_task_variant_distro_relevant(self):
- self.assertTrue(utl._is_tag_still_relevant(self.evg, "unreliable|task1|variant1|distro1"))
- self.assertTrue(utl._is_tag_still_relevant(self.evg, "unreliable|task3|variant2|distro2"))
-
-
-class MockEvergreenConfig(object):
- def __init__(self, tasks, variants):
- self.task_names = tasks
- self.variants = {}
- for name, fields in list(variants.items()):
- self.variants[name] = MockVariant(fields["tasks"], fields["distros"])
-
- def get_variant(self, variant_name):
- return self.variants.get(variant_name)
-
-
-class MockVariant(object):
- def __init__(self, task_names, distros):
- self.task_names = task_names
- self.distro_names = distros
-
-
-class TestJiraIssueCreator(unittest.TestCase):
- def test_description(self):
- data = {
- "js_test": {
- "testfile1": {"tag1": 0.1, "tag2": 0.2}, "testfile2": {"tag1": 0.1, "tag3": 0.3}
- }
- }
- desc = utl.JiraIssueCreator._make_updated_tags_description(data)
- expected = ("- *js_test*\n"
- "-- {{testfile1}}\n"
- "--- {{tag1}} (0.10)\n"
- "--- {{tag2}} (0.20)\n"
- "-- {{testfile2}}\n"
- "--- {{tag1}} (0.10)\n"
- "--- {{tag3}} (0.30)")
- self.assertEqual(expected, desc)
-
- def test_description_empty(self):
- data = {}
- desc = utl.JiraIssueCreator._make_updated_tags_description(data)
- expected = "_None_"
- self.assertEqual(expected, desc)
-
- def test_clean_up_description(self):
- data = {"js_test": {"testfile1": ["tag1", "tag2"], "testfile2": []}}
- desc = utl.JiraIssueCreator._make_tags_cleaned_up_description(data)
- expected = ("- *js_test*\n"
- "-- {{testfile1}}\n"
- "--- {{tag1}}\n"
- "--- {{tag2}}\n"
- "-- {{testfile2}}\n"
- "--- ALL (test file removed or renamed as part of an earlier commit)")
- self.assertEqual(expected, desc)
-
- def test_clean_up_description_empty(self):
- data = {}
- desc = utl.JiraIssueCreator._make_tags_cleaned_up_description(data)
- expected = "_None_"
- self.assertEqual(expected, desc)
-
- def test_truncate_description(self):
- desc = "a" * (utl.JiraIssueCreator._MAX_DESCRIPTION_SIZE - 1)
- self.assertTrue(desc == utl.JiraIssueCreator._truncate_description(desc))
-
- desc += "a"
- self.assertTrue(desc == utl.JiraIssueCreator._truncate_description(desc))
-
- desc += "a"
- self.assertTrue(
- len(utl.JiraIssueCreator._truncate_description(desc)) <=
- utl.JiraIssueCreator._MAX_DESCRIPTION_SIZE)
-
-
-class TestTagsConfigWithChangelog(unittest.TestCase):
- def setUp(self):
- lifecycle = ci_tags.TagsConfig({"selector": {}})
- self.summary_lifecycle = utl.TagsConfigWithChangelog(lifecycle)
-
- def test_add_tag(self):
- self.summary_lifecycle.add_tag("js_test", "testfile1", "tag1", 0.1)
- self.assertEqual({"js_test": {"testfile1": {"tag1": 0.1}}}, self.summary_lifecycle.added)
-
- def test_remove_tag(self):
- self.summary_lifecycle.lifecycle.add_tag("js_test", "testfile1", "tag1")
- self.summary_lifecycle.remove_tag("js_test", "testfile1", "tag1", 0.1)
- self.assertEqual({"js_test": {"testfile1": {"tag1": 0.1}}}, self.summary_lifecycle.removed)
-
- def test_add_remove_tag(self):
- self.summary_lifecycle.add_tag("js_test", "testfile1", "tag1", 0.1)
- self.summary_lifecycle.remove_tag("js_test", "testfile1", "tag1", 0.4)
- self.assertEqual({}, self.summary_lifecycle.added)
- self.assertEqual({}, self.summary_lifecycle.removed)
-
- def test_remove_add_tag(self):
- self.summary_lifecycle.lifecycle.add_tag("js_test", "testfile1", "tag1")
- self.summary_lifecycle.remove_tag("js_test", "testfile1", "tag1", 0.1)
- self.summary_lifecycle.add_tag("js_test", "testfile1", "tag1", 0.1)
- self.assertEqual({}, self.summary_lifecycle.added)
- self.assertEqual({}, self.summary_lifecycle.removed)
diff --git a/buildscripts/update_test_lifecycle.py b/buildscripts/update_test_lifecycle.py
deleted file mode 100755
index ea9b4d05bf2..00000000000
--- a/buildscripts/update_test_lifecycle.py
+++ /dev/null
@@ -1,1186 +0,0 @@
-#!/usr/bin/env python3
-"""Test Failures module.
-
-Update etc/test_lifecycle.yml to tag unreliable tests based on historic failure rates.
-"""
-
-import collections
-import datetime
-import itertools
-import logging
-import multiprocessing.dummy
-import optparse
-import os.path
-import posixpath
-import sys
-import textwrap
-
-import yaml
-
-# Get relative imports to work when the package is not installed on the PYTHONPATH.
-if __name__ == "__main__" and __package__ is None:
- sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-
-# pylint: disable=wrong-import-position
-from buildscripts import git
-from buildscripts import jiraclient
-from buildscripts import resmokelib
-from buildscripts.resmokelib import utils
-from buildscripts.resmokelib.utils import globstar
-from buildscripts.ciconfig import evergreen as ci_evergreen
-from buildscripts.ciconfig import tags as ci_tags
-from buildscripts.client import evergreen
-from buildscripts.util import testname
-# pylint: enable=wrong-import-position
-
-# pylint: disable=too-many-lines
-
-LOGGER = logging.getLogger(__name__)
-_NUMBER_TYPES = (int, float)
-
-Rates = collections.namedtuple("Rates", ["acceptable", "unacceptable"])
-
-Config = collections.namedtuple("Config", [
- "test_fail_rates",
- "task_fail_rates",
- "variant_fail_rates",
- "distro_fail_rates",
- "reliable_min_runs",
- "reliable_time_period",
- "unreliable_min_runs",
- "unreliable_time_period",
-])
-
-DEFAULT_CONFIG = Config(
- test_fail_rates=Rates(acceptable=0.1, unacceptable=0.3),
- task_fail_rates=Rates(acceptable=0.1, unacceptable=0.3),
- variant_fail_rates=Rates(acceptable=0.2, unacceptable=0.4),
- distro_fail_rates=Rates(acceptable=0.2, unacceptable=0.4),
- reliable_min_runs=5,
- reliable_time_period=datetime.timedelta(weeks=1),
- unreliable_min_runs=20,
- unreliable_time_period=datetime.timedelta(weeks=4)) # yapf: disable
-
-DEFAULT_PROJECT = "mongodb-mongo-master"
-
-DEFAULT_NUM_THREADS = 10
-
-MAX_BATCH_SIZE = 50
-
-
-def get_suite_tasks_membership(evg_conf):
- """Return a dictionary with keys of all suites and list of associated tasks."""
- suite_membership = collections.defaultdict(list)
- for task in evg_conf.tasks:
- suite = task.resmoke_suite
- if suite:
- suite_membership[suite].append(task.name)
- return suite_membership
-
-
-def get_test_tasks_membership(evg_conf):
- """Return a dictionary with keys of all tests and list of associated tasks."""
- test_suites_membership = resmokelib.suitesconfig.create_test_membership_map(test_kind="js_test")
- suite_tasks_membership = get_suite_tasks_membership(evg_conf)
- test_tasks_membership = collections.defaultdict(list)
- for test in list(test_suites_membership.keys()):
- for suite in test_suites_membership[test]:
- test_tasks_membership[test].extend(suite_tasks_membership[suite])
- return test_tasks_membership
-
-
-def get_tests_from_tasks(tasks, test_tasks_membership):
- """Return a list of tests from list of specified tasks."""
- tests = []
- tasks_set = set(tasks)
- for test in list(test_tasks_membership.keys()):
- if not tasks_set.isdisjoint(test_tasks_membership[test]):
- tests.append(test)
- return tests
-
-
-class TestCombination(object):
- """
- Represent a combination of test, task, variant, and distro.
-
- task, variant and distro may be None. If so the following fields must also be None.
- """
-
- GROUP_BY_TEST = "test"
- GROUP_BY_TASK = "task"
- GROUP_BY_VARIANT = "variant"
- GROUP_BY_DISTRO = "distro"
-
- def __init__(self, test, task=None, variant=None, distro=None):
- """Initialize the TestCombination with the field values."""
- self._test = test
- self._task = task
- self._variant = variant
- self._distro = distro
- self._group_by = self._validate()
-
- def _validate(self):
- if self._test is None:
- raise ValueError("Test cannot be None in a combination")
- group_by = self.GROUP_BY_DISTRO
- if self._distro is None:
- group_by = self.GROUP_BY_VARIANT
- if self._variant is None:
- group_by = self.GROUP_BY_TASK
- if self._distro is not None:
- raise ValueError("Distro cannot be set if variant is None")
- if self._task is None:
- group_by = self.GROUP_BY_TEST
- if self._variant is not None:
- raise ValueError("Variant cannot be set if task is None")
- return group_by
-
- @property
- def test(self):
- """Return the test field value."""
- return self._test
-
- @property
- def task(self):
- """Return the task field value. May be None."""
- return self._task
-
- @property
- def variant(self):
- """Return the variant field value. May be None."""
- return self._variant
-
- @property
- def distro(self):
- """Return the distro field value. May be None."""
- return self._distro
-
- @property
- def group_by(self):
- """
- Return how the test combination is grouped.
-
- The possible values are `GROUP_BY_TEST`, `GROUP_BY_TASK`, `GROUP_BY_VARIANT`,
- and `GROUP_BY_DISTRO`.
- """
- return self._group_by
-
- @property
- def tag(self):
- """Return the unreliable tag matching this test combination."""
- if self._group_by == self.GROUP_BY_TEST:
- return "unreliable"
- elif self._group_by == self.GROUP_BY_TASK:
- return "unreliable|{}".format(self.task)
- elif self._group_by == self.GROUP_BY_VARIANT:
- return "unreliable|{}|{}".format(self.task, self.variant)
- else:
- return "unreliable|{}|{}|{}".format(self.task, self.variant, self.distro)
-
- @staticmethod
- def from_tag(test, tag):
- """Create a TestCombination from a test name and an unreliable tag."""
- elements = _split_tag(tag)
- return TestCombination(test=test, task=elements[0], variant=elements[1], distro=elements[2])
-
- def as_tuple(self):
- """Return this combination as a (test, task, variant, distro) tuple."""
- return (self.test, self.task, self.variant, self.distro)
-
- def __eq__(self, other):
- if type(other) is type(self):
- return self.__dict__ == other.__dict__
- return False
-
- def __ne__(self, other):
- return not self.__eq__(other)
-
- def __hash__(self):
- return hash(self.as_tuple())
-
- def __repr__(self):
- return "%s|%s|%s|%s" % (self.test, self.task, self.variant, self.distro)
-
-
-class TestHistory(object):
- """Represent a test execution history over a reliable and unreliable periods."""
-
- def __init__(self, test):
- """Initliaze the TestHistory with the test file name."""
- self.test = test
- self.reliable_stats = []
- self.unreliable_stats = []
-
- def get_reliable_period_rates(self, group_by=None):
- """
- Return (TestCombination, failure rate, number of runs) tuples for the reliable period.
-
- The test combinations are grouped according to the group_by parameter.
- """
- return self._get_rates(self.reliable_stats, group_by)
-
- def get_unreliable_period_rates(self, group_by=None):
- """
- Return (TestCombination, failure rate, number of runs) tuples for the unreliable period.
-
- The test combinations are grouped according to the group_by parameter.
- """
- return self._get_rates(self.unreliable_stats, group_by)
-
- def add_reliable_period_stats(self, test_stats_docs):
- """Add Evergreen test execution statistics documents that cover the reliable period."""
- self.reliable_stats.extend(test_stats_docs)
-
- def add_unreliable_period_stats(self, test_stats_docs):
- """Add Evergreen test execution statistics documents that cover the unreliable period."""
- self.unreliable_stats.extend(test_stats_docs)
-
- def _get_rates(self, stats, group_by=None):
- results = []
- if group_by == TestCombination.GROUP_BY_TEST:
- keyfunc = self._group_by_test_key
- elif group_by == TestCombination.GROUP_BY_TASK:
- keyfunc = self._group_by_task_key
- elif group_by == TestCombination.GROUP_BY_VARIANT:
- keyfunc = self._group_by_variant_key
- elif group_by == TestCombination.GROUP_BY_DISTRO or group_by is None:
- keyfunc = self._group_by_distro_key
- else:
- raise ValueError("Invalid group_by value: {}".format(group_by))
- sorted_stats = sorted(stats, key=keyfunc)
- groups = itertools.groupby(sorted_stats, keyfunc)
- for key, group in groups:
- group = list(group)
- results.append((key, self._get_failure_rate(group), self._get_num_run(group)))
- return results
-
- @staticmethod
- def _get_failure_rate(test_stats):
- num_pass = sum([d["num_pass"] for d in test_stats])
- num_fail = sum([d["num_fail"] for d in test_stats])
- if (num_pass + num_fail) == 0:
- return 0
- return float(num_fail) / (num_pass + num_fail)
-
- @staticmethod
- def _get_num_run(test_stats):
- num_pass = sum([d["num_pass"] for d in test_stats])
- num_fail = sum([d["num_fail"] for d in test_stats])
- return num_pass + num_fail
-
- @staticmethod
- def _group_by_distro_key(test_stats_doc):
- return TestCombination(
- test=testname.normalize_test_file(test_stats_doc["test_file"]),
- task=test_stats_doc["task_name"],
- variant=test_stats_doc["variant"],
- distro=test_stats_doc["distro"]) # yapf: disable
-
- @staticmethod
- def _group_by_variant_key(test_stats_doc):
- return TestCombination(
- test=testname.normalize_test_file(test_stats_doc["test_file"]),
- task=test_stats_doc["task_name"],
- variant=test_stats_doc["variant"],
- distro=None) # yapf: disable
-
- @staticmethod
- def _group_by_task_key(test_stats_doc):
- return TestCombination(
- test=testname.normalize_test_file(test_stats_doc["test_file"]),
- task=test_stats_doc["task_name"],
- variant=None,
- distro=None) # yapf: disable
-
- @staticmethod
- def _group_by_test_key(test_stats_doc):
- return TestCombination(
- test=testname.normalize_test_file(test_stats_doc["test_file"]),
- task=None,
- variant=None,
- distro=None) # yapf: disable
-
-
-class TestHistorySource(object): # pylint: disable=too-many-instance-attributes
- """A class used to parallelize requests to buildscripts.test_failures.TestHistory."""
-
- def __init__( # pylint: disable=too-many-arguments
- self, project, variants, distros, reliable_period, unreliable_period,
- thread_pool_size=DEFAULT_NUM_THREADS):
- """Initialize the TestHistorySource.
-
- Args:
- project: the Evergreen project name.
- variants: a list of variant names.
- distros: a list of distro names.
- reliable_period: the time delta for the reliable period.
- unreliable_period: the time delta for the unreliable period.
- thread_pool_size: the size of the thread pool used to make parallel requests.
- """
- self._project = project
- self._variants = variants
- self._distros = distros
- not_after = datetime.datetime.utcnow()
-
- self._not_after = not_after.strftime(format="%Y-%m-%d")
- self._reliable_not_before = (not_after - reliable_period).strftime(format="%Y-%m-%d")
-
- self._reliable_group_num_days = reliable_period.days
- self._unreliable_not_before = (not_after - unreliable_period).strftime(format="%Y-%m-%d")
- self._unreliable_group_num_days = unreliable_period.days
-
- self._thread_pool = multiprocessing.dummy.Pool(thread_pool_size)
- self._evg_api = evergreen.get_evergreen_apiv2(num_retries=3)
-
- def get_history(self, test_tasks_list):
- """Return an iterator with the TestHistory for each of the tests provided.
-
- Args:
- test_tasks_list: a list of (test, task list) tuples.
- """
- return self._thread_pool.imap_unordered(self._get_history, test_tasks_list)
-
- def _get_history(self, test_tasks):
- try:
- test, tasks = test_tasks
- history = TestHistory(test)
- tests = testname.denormalize_test_file(test)
- # Get stats for the reliable period.
- history.add_reliable_period_stats(
- self._get_test_stats(self._reliable_not_before, self._reliable_group_num_days,
- tests, tasks))
- # Get stats for the unreliable period.
- history.add_unreliable_period_stats(
- self._get_test_stats(self._unreliable_not_before, self._unreliable_group_num_days,
- tests, tasks))
- return history
- except Exception:
- LOGGER.exception("An error occurred while getting test history")
- raise
-
- def _get_test_stats(self, not_before, group_num_days, tests, tasks):
- return self._evg_api.test_stats(self._project, not_before, self._not_after, group_num_days,
- tests=tests, tasks=tasks, variants=self._variants,
- distros=self._distros)
-
-
-def unreliable_test(test_fr, unacceptable_fr, test_runs, min_run):
- """Check for an unreliable test.
-
- A test should be added to the set of tests believed not to run reliably when it has more
- than min_run executions with a failure percentage greater than unacceptable_fr.
- """
- return test_runs >= min_run and test_fr >= unacceptable_fr
-
-
-def reliable_test(test_fr, acceptable_fr, test_runs, min_run):
- """Check for a reliable test.
-
- A test should then removed from the set of tests believed not to run reliably when it has
- less than min_run executions or has a failure percentage less than acceptable_fr.
- """
- return test_runs < min_run or test_fr <= acceptable_fr
-
-
-def update_lifecycle( # pylint: disable=too-many-arguments
- lifecycle_tags_file, rates, method_test, add_tags, threshold, min_run):
- """Update the lifecycle object based on the test_method.
-
- The test_method checks unreliable or reliable fail_rates.
- """
- for combination, fail_rate, num_run in rates:
- if method_test(fail_rate, threshold, num_run, min_run):
- update_tag = combination.tag
- if add_tags:
- lifecycle_tags_file.add_tag("js_test", combination.test, update_tag, fail_rate)
- else:
- lifecycle_tags_file.remove_tag("js_test", combination.test, update_tag, fail_rate)
-
-
-def compare_tags(tag_a, tag_b):
- """Return 1, -1 or 0 if 'tag_a' is superior, inferior or equal to 'tag_b'."""
- a_split = tag_a.split("|")
- b_split = tag_b.split("|")
- return (a_split > b_split) - (a_split < b_split)
-
-
-def validate_config(config): # pylint: disable=too-many-branches
- """Raise a TypeError or ValueError exception if 'config' isn't a valid model."""
-
- for (name, fail_rates) in (("test", config.test_fail_rates), ("task", config.task_fail_rates),
- ("variant", config.variant_fail_rates), ("distro",
- config.distro_fail_rates)):
- if not isinstance(fail_rates.acceptable, _NUMBER_TYPES):
- raise TypeError("The acceptable {} failure rate must be a number, but got {}".format(
- name, fail_rates.acceptable))
- elif fail_rates.acceptable < 0 or fail_rates.acceptable > 1:
- raise ValueError(("The acceptable {} failure rate must be between 0 and 1 (inclusive),"
- " but got {}").format(name, fail_rates.acceptable))
- elif not isinstance(fail_rates.unacceptable, _NUMBER_TYPES):
- raise TypeError("The unacceptable {} failure rate must be a number, but got {}".format(
- name, fail_rates.unacceptable))
- elif fail_rates.unacceptable < 0 or fail_rates.unacceptable > 1:
- raise ValueError(("The unacceptable {} failure rate must be between 0 and 1"
- " (inclusive), but got {}").format(name, fail_rates.unacceptable))
- elif fail_rates.acceptable > fail_rates.unacceptable:
- raise ValueError(
- ("The acceptable {0} failure rate ({1}) must be no larger than unacceptable {0}"
- " failure rate ({2})").format(name, fail_rates.acceptable,
- fail_rates.unacceptable))
-
- for (name, min_runs) in (("reliable", config.reliable_min_runs), ("unreliable",
- config.unreliable_min_runs)):
- if not isinstance(min_runs, _NUMBER_TYPES):
- raise TypeError(("The minimum number of runs for considering a test {} must be a"
- " number, but got {}").format(name, min_runs))
- elif min_runs <= 0:
- raise ValueError(("The minimum number of runs for considering a test {} must be a"
- " positive integer, but got {}").format(name, min_runs))
- elif isinstance(min_runs, float) and not min_runs.is_integer():
- raise ValueError(("The minimum number of runs for considering a test {} must be an"
- " integer, but got {}").format(name, min_runs))
-
- for (name, time_period) in (("reliable", config.reliable_time_period),
- ("unreliable", config.unreliable_time_period)):
- if not isinstance(time_period, datetime.timedelta):
- raise TypeError(
- "The {} time period must be a datetime.timedelta instance, but got {}".format(
- name, time_period))
- elif time_period.days <= 0:
- raise ValueError(
- "The {} time period must be a positive number of days, but got {}".format(
- name, time_period))
- elif time_period - datetime.timedelta(days=time_period.days) > datetime.timedelta():
- raise ValueError(
- "The {} time period must be an integral number of days, but got {}".format(
- name, time_period))
-
-
-def update_tags(lifecycle_tags, config, test_history): # pylint: disable=too-many-locals
- """Update the tags in 'lifecycle_tags'.
-
- This is based on the historical test failures of tests 'tests'
- mentioned in 'report' according to the model described by 'config'.
- """
-
- # yapf: disable
- for (group_by, rates) in [(TestCombination.GROUP_BY_TEST, config.test_fail_rates),
- (TestCombination.GROUP_BY_TASK, config.task_fail_rates),
- (TestCombination.GROUP_BY_VARIANT, config.variant_fail_rates),
- (TestCombination.GROUP_BY_DISTRO, config.distro_fail_rates)]:
-
- reliable_rates = test_history.get_reliable_period_rates(group_by)
- update_lifecycle(lifecycle_tags, reliable_rates, reliable_test, False,
- rates.acceptable, config.unreliable_min_runs)
-
- unreliable_rates = test_history.get_unreliable_period_rates(group_by)
- update_lifecycle(lifecycle_tags, unreliable_rates, unreliable_test, True,
- rates.unacceptable, config.unreliable_min_runs)
-
- # Remove the tags that correspond to combinations that have not run during the reliable
- # period.
- test = test_history.test
- reliable_combinations = {r[0] for r in reliable_rates}
- tags = lifecycle_tags.lifecycle.get_tags("js_test", test)
- for tag in tags[:]:
- tag_combination = TestCombination.from_tag(test, tag)
- if tag_combination.group_by != group_by:
- continue
- if tag_combination not in reliable_combinations:
- LOGGER.info("Removing tag '%s' of test '%s' because the combination did not run"
- " during the reliable period", tag, test)
- lifecycle_tags.remove_tag("js_test", test, tag, failure_rate=0)
-
-
-def _split_tag(tag):
- """Split a tag into its components.
-
- Return a tuple containing task, variant, distro. The values are None if absent from the tag.
- If the tag is invalid, the return value is (None, None, None).
- """
- elements = tag.split("|")
- length = len(elements)
- if elements[0] != "unreliable" or length < 2 or length > 4:
- return None, None, None
- # Fill out the array.
- elements.extend([None] * (4 - length))
- # Return as a tuple.
- return tuple(elements[1:])
-
-
-def _is_tag_still_relevant(evg_conf, tag):
- """Indicate if a tag still corresponds to a valid task/variant/distro combination."""
- if tag == "unreliable":
- return True
- task, variant, distro = _split_tag(tag)
- if not task or task not in evg_conf.task_names:
- return False
- if variant:
- variant_conf = evg_conf.get_variant(variant)
- if not variant_conf or task not in variant_conf.task_names:
- return False
- if distro and distro not in variant_conf.distro_names:
- return False
- return True
-
-
-def clean_up_tags(lifecycle_tags, evg_conf):
- """Remove the tags that do not correspond to a valid test/task/variant/distro combination."""
- lifecycle = lifecycle_tags.lifecycle
- for test_kind in lifecycle.get_test_kinds():
- for test_pattern in lifecycle.get_test_patterns(test_kind):
- if not globstar.glob(test_pattern):
- # The pattern does not match any file in the repository.
- lifecycle_tags.clean_up_test(test_kind, test_pattern)
- continue
- for tag in lifecycle.get_tags(test_kind, test_pattern):
- if not _is_tag_still_relevant(evg_conf, tag):
- lifecycle_tags.clean_up_tag(test_kind, test_pattern, tag)
-
-
-def _config_as_options(config):
- return ("--reliableTestMinRuns {} "
- "--reliableDays {} "
- "--unreliableTestMinRuns {} "
- "--unreliableDays {} "
- "--testFailRates {} {} "
- "--taskFailRates {} {} "
- "--variantFailRates {} {} "
- "--distroFailRates {} {}").format(
- config.reliable_min_runs, config.reliable_time_period.days,
- config.unreliable_min_runs, config.unreliable_time_period.days,
- config.test_fail_rates.acceptable, config.test_fail_rates.unacceptable,
- config.task_fail_rates.acceptable, config.task_fail_rates.unacceptable,
- config.variant_fail_rates.acceptable, config.variant_fail_rates.unacceptable,
- config.distro_fail_rates.acceptable, config.distro_fail_rates.unacceptable)
-
-
-class TagsConfigWithChangelog(object):
- """A wrapper around TagsConfig to update a tags file and record the modifications made."""
-
- def __init__(self, lifecycle):
- """Initialize the TagsConfigWithChangelog with the lifecycle TagsConfig."""
- self.lifecycle = lifecycle
- self.added = {}
- self.removed = {}
- self.cleaned_up = {}
-
- @staticmethod
- def _cancel_tag_log(log_dict, test_kind, test, tag):
- """Remove a tag from a changelog dictionary.
-
- Used to remove a tag from the 'added' or 'removed' attribute.
- """
- kind_dict = log_dict[test_kind]
- test_dict = kind_dict[test]
- del test_dict[tag]
- if not test_dict:
- del kind_dict[test]
- if not kind_dict:
- del log_dict[test_kind]
-
- def add_tag(self, test_kind, test, tag, failure_rate):
- """Add a tag."""
- if self.lifecycle.add_tag(test_kind, test, tag):
- if tag in self.removed.get(test_kind, {}).get(test, {}):
- # The tag has just been removed.
- self._cancel_tag_log(self.removed, test_kind, test, tag)
- else:
- self.added.setdefault(test_kind, {}).setdefault(test, {})[tag] = failure_rate
-
- def remove_tag(self, test_kind, test, tag, failure_rate):
- """Remove a tag."""
- if self.lifecycle.remove_tag(test_kind, test, tag):
- if tag in self.added.get(test_kind, {}).get(test, {}):
- # The tag has just been added.
- self._cancel_tag_log(self.added, test_kind, test, tag)
- else:
- self.removed.setdefault(test_kind, {}).setdefault(test, {})[tag] = failure_rate
-
- def clean_up_tag(self, test_kind, test, tag):
- """Clean up an invalid tag."""
- self.lifecycle.remove_tag(test_kind, test, tag)
- self.cleaned_up.setdefault(test_kind, {}).setdefault(test, []).append(tag)
-
- def clean_up_test(self, test_kind, test):
- """Clean up an invalid test."""
- self.lifecycle.remove_test_pattern(test_kind, test)
- self.cleaned_up.setdefault(test_kind, {})[test] = []
-
-
-class JiraIssueCreator(object):
- """JiraIssueCreator class."""
-
- _LABEL = "test-lifecycle"
- _PROJECT = "TIGBOT"
- _MAX_DESCRIPTION_SIZE = 32767
-
- def __init__( # pylint: disable=too-many-arguments
- self, server=None, username=None, password=None, access_token=None,
- access_token_secret=None, consumer_key=None, key_cert=None):
- """Initialize JiraIssueCreator."""
- self._client = jiraclient.JiraClient(
- server=server, username=username, password=password, access_token=access_token,
- access_token_secret=access_token_secret, consumer_key=consumer_key, key_cert=key_cert)
-
- def create_issue( # pylint: disable=too-many-arguments
- self, evg_project, mongo_revision, model_config, added, removed, cleaned_up):
- """Create a JIRA issue for the test lifecycle tag update."""
- summary = self._get_jira_summary(evg_project)
- description = self._get_jira_description(evg_project, mongo_revision, model_config, added,
- removed, cleaned_up)
- issue_key = self._client.create_issue(self._PROJECT, summary, description, [self._LABEL])
- return issue_key
-
- def close_fix_issue(self, issue_key):
- """Close the issue with the "Fixed" resolution."""
- LOGGER.info("Closing issue '%s' as FIXED.", issue_key)
- self._client.close_issue(issue_key, self._client.FIXED_RESOLUTION_NAME)
-
- def close_wontfix_issue(self, issue_key):
- """Close the issue the with "Won't Fix" resolution."""
- LOGGER.info("Closing issue '%s' as WON'T FIX.", issue_key)
- self._client.close_issue(issue_key, self._client.WONT_FIX_RESOLUTION_NAME)
-
- @staticmethod
- def _get_jira_summary(project):
- return "Update of test lifecycle tags for {}".format(project)
-
- @staticmethod
- def _monospace(text):
- """Transform a text into a monospace JIRA text."""
- return "{{" + text + "}}"
-
- @staticmethod
- def _truncate_description(desc):
- max_size = JiraIssueCreator._MAX_DESCRIPTION_SIZE
- if len(desc) > max_size:
- warning = ("\nDescription truncated: "
- "exceeded max size of {} characters.").format(max_size)
- truncated_length = max_size - len(warning)
- desc = desc[:truncated_length] + warning
- return desc
-
- @staticmethod
- def _get_jira_description( # pylint: disable=too-many-arguments
- project, mongo_revision, model_config, added, removed, cleaned_up):
- mono = JiraIssueCreator._monospace
- config_desc = _config_as_options(model_config)
- added_desc = JiraIssueCreator._make_updated_tags_description(added)
- removed_desc = JiraIssueCreator._make_updated_tags_description(removed)
- cleaned_up_desc = JiraIssueCreator._make_tags_cleaned_up_description(cleaned_up)
- project_link = "[{0}|https://evergreen.mongodb.com/waterfall/{1}]".format(
- mono(project), project)
- revision_link = "[{0}|https://github.com/mongodb/mongo/commit/{1}]".format(
- mono(mongo_revision), mongo_revision)
- full_desc = ("h3. Automatic update of the test lifecycle tags\n"
- "Evergreen Project: {0}\n"
- "Revision: {1}\n\n"
- "{{{{update_test_lifecycle.py}}}} options:\n{2}\n\n"
- "h5. Tags added\n{3}\n\n"
- "h5. Tags removed\n{4}\n\n"
- "h5. Tags cleaned up (no longer relevant)\n{5}\n").format(
- project_link, revision_link, mono(config_desc), added_desc, removed_desc,
- cleaned_up_desc)
-
- return JiraIssueCreator._truncate_description(full_desc)
-
- @staticmethod
- def _make_updated_tags_description(data):
- mono = JiraIssueCreator._monospace
- tags_lines = []
- for test_kind in sorted(data.keys()):
- tests = data[test_kind]
- tags_lines.append("- *{0}*".format(test_kind))
- for test in sorted(tests.keys()):
- tags = tests[test]
- tags_lines.append("-- {0}".format(mono(test)))
- for tag in sorted(tags.keys()):
- coefficient = tags[tag]
- tags_lines.append("--- {0} ({1:.2f})".format(mono(tag), coefficient))
- if tags_lines:
- return "\n".join(tags_lines)
- return "_None_"
-
- @staticmethod
- def _make_tags_cleaned_up_description(cleaned_up):
- mono = JiraIssueCreator._monospace
- tags_cleaned_up_lines = []
- for test_kind in sorted(cleaned_up.keys()):
- test_tags = cleaned_up[test_kind]
- tags_cleaned_up_lines.append("- *{0}*".format(test_kind))
- for test in sorted(test_tags.keys()):
- tags = test_tags[test]
- tags_cleaned_up_lines.append("-- {0}".format(mono(test)))
- if not tags:
- tags_cleaned_up_lines.append("--- ALL (test file removed or renamed as part of"
- " an earlier commit)")
- else:
- for tag in sorted(tags):
- tags_cleaned_up_lines.append("--- {0}".format(mono(tag)))
- if tags_cleaned_up_lines:
- return "\n".join(tags_cleaned_up_lines)
- return "_None_"
-
-
-class LifecycleTagsFile(object): # pylint: disable=too-many-instance-attributes
- """Represent a test lifecycle tags file that can be written and committed."""
-
- def __init__( # pylint: disable=too-many-arguments
- self, project, lifecycle_file, metadata_repo_url=None, references_file=None,
- jira_issue_creator=None, git_info=None,
- model_config=None): # noqa: D214,D401,D405,D406,D407,D411,D413
- """Initalize the LifecycleTagsFile.
-
- Arguments:
- project: The Evergreen project name, e.g. "mongodb-mongo-master".
- lifecycle_file: The path to the lifecycle tags file. If 'metadata_repo_url' is
- specified, this path must be relative to the root of the metadata repository.
- metadata_repo_url: The URL of the metadat repository that contains the test lifecycle
- tags file.
- references_file: The path to the references file in the metadata repository.
- jira_issue_creator: A JiraIssueCreator instance.
- git_info: A tuple containing the git user's name and email to set before committing.
- model_config: The model configuration as a Config instance.
- """
- self.project = project
- self.mongo_repo = git.Repository(os.getcwd())
- self.mongo_revision = self.mongo_repo.get_current_revision()
- # The branch name is the same on both repositories.
- self.mongo_branch = self.mongo_repo.get_branch_name()
- self.metadata_branch = project
-
- if metadata_repo_url:
- # The file can be found in another repository. We clone it.
- self.metadata_repo = self._clone_repository(metadata_repo_url, self.project)
- self.relative_lifecycle_file = lifecycle_file
- self.lifecycle_file = os.path.join(self.metadata_repo.directory, lifecycle_file)
- self.relative_references_file = references_file
- self.references_file = os.path.join(self.metadata_repo.directory, references_file)
- if git_info:
- self.metadata_repo.configure("user.name", git_info[0])
- self.metadata_repo.configure("user.email", git_info[1])
- else:
- self.metadata_repo = None
- self.relative_lifecycle_file = lifecycle_file
- self.lifecycle_file = lifecycle_file
- self.relative_references_file = None
- self.references_file = None
- self.metadata_repo_url = metadata_repo_url
- self.lifecycle = ci_tags.TagsConfig.from_file(self.lifecycle_file, cmp_func=compare_tags)
- self.jira_issue_creator = jira_issue_creator
- self.model_config = model_config
- self.changelog_lifecycle = TagsConfigWithChangelog(self.lifecycle)
-
- @staticmethod
- def _clone_repository(metadata_repo_url, branch):
- directory_name = posixpath.splitext(posixpath.basename(metadata_repo_url))[0]
- LOGGER.info("Cloning the repository %s into the directory %s", metadata_repo_url,
- directory_name)
- return git.Repository.clone(metadata_repo_url, directory_name, branch)
-
- def is_modified(self):
- """Indicate if the tags have been modified."""
- return self.lifecycle.is_modified()
-
- def _create_issue(self):
- LOGGER.info("Creating a JIRA issue")
- issue_key = self.jira_issue_creator.create_issue(
- self.project, self.mongo_revision, self.model_config, self.changelog_lifecycle.added,
- self.changelog_lifecycle.removed, self.changelog_lifecycle.cleaned_up)
- LOGGER.info("JIRA issue created: %s", issue_key)
- return issue_key
-
- def write(self):
- """Write the test lifecycle tag file."""
- LOGGER.info("Writing the tag file to '%s'", self.lifecycle_file)
- comment = ("This file was generated by {} and shouldn't be edited by hand. It was"
- " generated against commit {} with the following options: {}.").format(
- sys.argv[0], self.mongo_repo.get_current_revision(),
- _config_as_options(self.model_config))
- self.lifecycle.write_file(self.lifecycle_file, comment)
-
- def _ready_for_commit(self, ref_branch, references):
- # Check that the test lifecycle tags file has changed.
- diff = self.metadata_repo.git_diff(
- ["--name-only", ref_branch, self.relative_lifecycle_file])
- if not diff:
- LOGGER.info("The local lifecycle file is identical to the the one on branch '%s'",
- ref_branch)
- return False
- # Check that the lifecycle file has not been updated after the current mongo revision.
- update_revision = references.get("test-lifecycle", {}).get(self.project)
- if update_revision and not self.mongo_repo.is_ancestor(update_revision,
- self.mongo_revision):
- LOGGER.warning(("The existing lifecycle file is based on revision '%s' which is not a"
- " parent revision of the current revision '%s'"), update_revision,
- self.mongo_revision)
- return False
- return True
-
- def _read_references(self, metadata_branch=None):
- branch = metadata_branch if metadata_branch is not None else ""
- references_content = self.metadata_repo.git_cat_file(
- ["blob", "{0}:{1}".format(branch, self.relative_references_file)])
- return yaml.safe_load(references_content)
-
- def _update_and_write_references(self, references):
- LOGGER.info("Writing the references file to '%s'", self.references_file)
- references.setdefault("test-lifecycle", {})[self.project] = self.mongo_revision
- with open(self.references_file, "w") as fstream:
- yaml.safe_dump(references, fstream, default_flow_style=False)
-
- def _commit_locally(self, issue_key):
- self.metadata_repo.git_add([self.relative_lifecycle_file])
- self.metadata_repo.git_add([self.relative_references_file])
- commit_message = "{} Update {}".format(issue_key, self.relative_lifecycle_file)
- self.metadata_repo.commit_with_message(commit_message)
- LOGGER.info("Change committed with message: %s", commit_message)
-
- def commit(self, nb_retries=10):
- """Commit the test lifecycle tag file.
-
- Args:
- nb_retries: the number of times the script will reset, fetch, recommit and retry when
- the push fails.
- """
- references = self._read_references()
- # Verify we are ready to commit.
- if not self._ready_for_commit(self.metadata_branch, references):
- return True
-
- # Write the references file.
- self._update_and_write_references(references)
-
- # Create the issue.
- issue_key = self._create_issue()
-
- # Commit the change.
- self._commit_locally(issue_key)
-
- # Push the change.
- tries = 0
- pushed = False
- upstream = "origin/{0}".format(self.metadata_branch)
- while tries < nb_retries:
- try:
- self.metadata_repo.push_to_remote_branch("origin", self.metadata_branch)
- pushed = True
- break
- except git.GitException:
- LOGGER.warning("git push command failed, fetching and retrying.")
- # Fetch upstream branch.
- LOGGER.info("Fetching branch %s of %s", self.metadata_branch,
- self.metadata_repo_url)
- self.metadata_repo.fetch_remote_branch("origin", self.metadata_branch)
- # Resetting the current branch to the origin branch
- LOGGER.info("Resetting branch %s to %s", self.metadata_branch, upstream)
- self.metadata_repo.git_reset(["--hard", upstream])
- # Rewrite the test lifecycle tags file
- self.write()
- # Rewrite the references file
- references = self._read_references()
- self._update_and_write_references(references)
- # Checking if we can still commit
- if not self._ready_for_commit(upstream, references):
- LOGGER.warning("Aborting.")
- break
- # Committing
- self._commit_locally(issue_key)
- tries += 1
- if pushed:
- self.jira_issue_creator.close_fix_issue(issue_key)
- return True
- self.jira_issue_creator.close_wontfix_issue(issue_key)
- return False
-
-
-def make_lifecycle_tags_file(options, model_config):
- """Create a LifecycleTagsFile based on the script options."""
- if options.commit:
- if not options.jira_config:
- LOGGER.error("JIRA configuration file is required when specifying --commit.")
- return None
- if not (options.git_user_name or options.git_user_email):
- LOGGER.error("Git configuration parameters are required when specifying --commit.")
- return None
- jira_issue_creator = JiraIssueCreator(**utils.load_yaml_file(options.jira_config))
- git_config = (options.git_user_name, options.git_user_email)
- else:
- jira_issue_creator = None
- git_config = None
-
- lifecycle_tags_file = LifecycleTagsFile(options.project, options.tag_file,
- options.metadata_repo_url, options.references_file,
- jira_issue_creator, git_config, model_config)
-
- return lifecycle_tags_file
-
-
-def main(): # pylint: disable=too-many-branches,too-many-locals,too-many-statements
- """Exexcute utility to update a resmoke.py tag file.
-
- This is based on computing test failure rates from the Evergreen API.
- """
-
- parser = optparse.OptionParser(
- description=textwrap.dedent(main.__doc__), usage="Usage: %prog [options] [test1 test2 ...]")
-
- data_options = optparse.OptionGroup(
- parser, title="Data options",
- description=("Options used to configure what historical test failure data to retrieve from"
- " Evergreen."))
- parser.add_option_group(data_options)
-
- data_options.add_option("--project", dest="project", metavar="<project-name>",
- default=DEFAULT_PROJECT,
- help="The Evergreen project to analyze. Defaults to '%default'.")
-
- data_options.add_option(
- "--tasks", dest="tasks", metavar="<task1,task2,...>",
- help=("The Evergreen tasks to analyze for tagging unreliable tests. If specified in"
- " additional to having test positional arguments, then only tests that run under the"
- " specified Evergreen tasks will be analyzed. If omitted, then the list of tasks"
- " defaults to the non-excluded list of tasks from the specified"
- " --evergreenProjectConfig file."))
-
- data_options.add_option(
- "--variants", dest="variants", metavar="<variant1,variant2,...>", default="",
- help="The Evergreen build variants to analyze for tagging unreliable tests.")
-
- data_options.add_option("--distros", dest="distros", metavar="<distro1,distro2,...>",
- default="",
- help="The Evergreen distros to analyze for tagging unreliable tests.")
-
- data_options.add_option(
- "--evergreenProjectConfig", dest="evergreen_project_config",
- metavar="<project-config-file>", default="etc/evergreen.yml",
- help=("The Evergreen project configuration file used to get the list of tasks if --tasks is"
- " omitted. Defaults to '%default'."))
-
- model_options = optparse.OptionGroup(
- parser, title="Model options",
- description=("Options used to configure whether (test,), (test, task),"
- " (test, task, variant), and (test, task, variant, distro) combinations are"
- " considered unreliable."))
- parser.add_option_group(model_options)
-
- model_options.add_option(
- "--reliableTestMinRuns", type="int", dest="reliable_test_min_runs",
- metavar="<reliable-min-runs>", default=DEFAULT_CONFIG.reliable_min_runs,
- help=("The minimum number of test executions required for a test's failure rate to"
- " determine whether the test is considered reliable. If a test has fewer than"
- " <reliable-min-runs> executions, then it cannot be considered unreliable."))
-
- model_options.add_option(
- "--unreliableTestMinRuns", type="int", dest="unreliable_test_min_runs",
- metavar="<unreliable-min-runs>", default=DEFAULT_CONFIG.unreliable_min_runs,
- help=("The minimum number of test executions required for a test's failure rate to"
- " determine whether the test is considered unreliable. If a test has fewer than"
- " <unreliable-min-runs> executions, then it cannot be considered unreliable."))
-
- model_options.add_option(
- "--testFailRates", type="float", nargs=2, dest="test_fail_rates",
- metavar="<test-acceptable-fail-rate> <test-unacceptable-fail-rate>",
- default=DEFAULT_CONFIG.test_fail_rates,
- help=("Controls how readily a test is considered unreliable. Each failure rate must be a"
- " number between 0 and 1 (inclusive) with"
- " <test-unacceptable-fail-rate> >= <test-acceptable-fail-rate>. If a test fails no"
- " more than <test-acceptable-fail-rate> in <reliable-days> time, then it is"
- " considered reliable. Otherwise, if a test fails at least as much as"
- " <test-unacceptable-fail-rate> in <test-unreliable-days> time, then it is considered"
- " unreliable. Defaults to %default."))
-
- model_options.add_option(
- "--taskFailRates", type="float", nargs=2, dest="task_fail_rates",
- metavar="<task-acceptable-fail-rate> <task-unacceptable-fail-rate>",
- default=DEFAULT_CONFIG.task_fail_rates,
- help=("Controls how readily a (test, task) combination is considered unreliable. Each"
- " failure rate must be a number between 0 and 1 (inclusive) with"
- " <task-unacceptable-fail-rate> >= <task-acceptable-fail-rate>. If a (test, task)"
- " combination fails no more than <task-acceptable-fail-rate> in <reliable-days> time,"
- " then it is considered reliable. Otherwise, if a test fails at least as much as"
- " <task-unacceptable-fail-rate> in <unreliable-days> time, then it is considered"
- " unreliable. Defaults to %default."))
-
- model_options.add_option(
- "--variantFailRates", type="float", nargs=2, dest="variant_fail_rates",
- metavar="<variant-acceptable-fail-rate> <variant-unacceptable-fail-rate>",
- default=DEFAULT_CONFIG.variant_fail_rates,
- help=("Controls how readily a (test, task, variant) combination is considered unreliable."
- " Each failure rate must be a number between 0 and 1 (inclusive) with"
- " <variant-unacceptable-fail-rate> >= <variant-acceptable-fail-rate>. If a"
- " (test, task, variant) combination fails no more than <variant-acceptable-fail-rate>"
- " in <reliable-days> time, then it is considered reliable. Otherwise, if a test fails"
- " at least as much as <variant-unacceptable-fail-rate> in <unreliable-days> time,"
- " then it is considered unreliable. Defaults to %default."))
-
- model_options.add_option(
- "--distroFailRates", type="float", nargs=2, dest="distro_fail_rates",
- metavar="<distro-acceptable-fail-rate> <distro-unacceptable-fail-rate>",
- default=DEFAULT_CONFIG.distro_fail_rates,
- help=("Controls how readily a (test, task, variant, distro) combination is considered"
- " unreliable. Each failure rate must be a number between 0 and 1 (inclusive) with"
- " <distro-unacceptable-fail-rate> >= <distro-acceptable-fail-rate>. If a"
- " (test, task, variant, distro) combination fails no more than"
- " <distro-acceptable-fail-rate> in <reliable-days> time, then it is considered"
- " reliable. Otherwise, if a test fails at least as much as"
- " <distro-unacceptable-fail-rate> in <unreliable-days> time, then it is considered"
- " unreliable. Defaults to %default."))
-
- model_options.add_option(
- "--reliableDays", type="int", dest="reliable_days", metavar="<ndays>",
- default=DEFAULT_CONFIG.reliable_time_period.days,
- help=("The time period to analyze when determining if a test has become reliable. Defaults"
- " to %default day(s)."))
-
- model_options.add_option(
- "--unreliableDays", type="int", dest="unreliable_days", metavar="<ndays>",
- default=DEFAULT_CONFIG.unreliable_time_period.days,
- help=("The time period to analyze when determining if a test has become unreliable."
- " Defaults to %default day(s)."))
-
- parser.add_option("--resmokeTagFile", dest="tag_file", metavar="<tagfile>",
- default="etc/test_lifecycle.yml",
- help=("The resmoke.py tag file to update. If --metadataRepo is specified, it"
- " is the relative path in the metadata repository, otherwise it can be"
- " an absolute path or a relative path from the current directory."
- " Defaults to '%default'."))
-
- parser.add_option("--metadataRepo", dest="metadata_repo_url", metavar="<metadata-repo-url>",
- default="git@github.com:mongodb/mongo-test-metadata.git",
- help=("The repository that contains the lifecycle file. "
- "It will be cloned in the current working directory. "
- "Defaults to '%default'."))
-
- parser.add_option("--referencesFile", dest="references_file", metavar="<references-file>",
- default="references.yml",
- help=("The YAML file in the metadata repository that contains the revision "
- "mappings. Defaults to '%default'."))
-
- parser.add_option("--requestThreads", type="int", dest="num_request_threads",
- metavar="<num-request-threads>", default=DEFAULT_NUM_THREADS,
- help=("The maximum number of threads to use when querying the Evergreen API."
- " Batches are processed sequentially but the test history is queried in"
- " parallel for each task. Defaults to %default."))
-
- commit_options = optparse.OptionGroup(
- parser, title="Commit options",
- description=("Options used to configure whether and how to commit the updated test"
- " lifecycle tags."))
- parser.add_option_group(commit_options)
-
- commit_options.add_option("--commit", action="store_true", dest="commit", default=False,
- help="Indicates that the updated tag file should be committed.")
-
- commit_options.add_option(
- "--jiraConfig", dest="jira_config", metavar="<jira-config>", default=None,
- help=("The YAML file containing the JIRA access configuration ('user', 'password',"
- "'server')."))
-
- commit_options.add_option(
- "--gitUserName", dest="git_user_name", metavar="<git-user-name>", default="Test Lifecycle",
- help=("The git user name that will be set before committing to the metadata repository."
- " Defaults to '%default'."))
-
- commit_options.add_option(
- "--gitUserEmail", dest="git_user_email", metavar="<git-user-email>",
- default="buil+testlifecycle@mongodb.com",
- help=("The git user email address that will be set before committing to the metadata"
- " repository. Defaults to '%default'."))
-
- logging_options = optparse.OptionGroup(
- parser, title="Logging options",
- description="Options used to configure the logging output of the script.")
- parser.add_option_group(logging_options)
-
- logging_options.add_option("--logLevel", dest="log_level", metavar="<log-level>", choices=[
- "DEBUG", "INFO", "WARNING", "ERROR"
- ], default="INFO", help=("The log level. Accepted values are: DEBUG, INFO, WARNING and ERROR."
- " Defaults to '%default'."))
-
- logging_options.add_option(
- "--logFile", dest="log_file", metavar="<log-file>", default=None,
- help="The destination file for the logs output. Defaults to the standard output.")
-
- (options, tests) = parser.parse_args()
-
- logging.basicConfig(format="%(asctime)s %(levelname)s %(message)s", level=options.log_level,
- filename=options.log_file)
- evg_conf = ci_evergreen.parse_evergreen_file(options.evergreen_project_config)
- use_test_tasks_membership = False
-
- tasks = options.tasks.split(",") if options.tasks else []
- if not tasks:
- # If no tasks are specified, then the list of tasks is all.
- tasks = evg_conf.lifecycle_task_names
- use_test_tasks_membership = True
-
- variants = options.variants.split(",") if options.variants else []
-
- distros = options.distros.split(",") if options.distros else []
-
- config = Config(
- test_fail_rates=Rates(*options.test_fail_rates),
- task_fail_rates=Rates(*options.task_fail_rates),
- variant_fail_rates=Rates(*options.variant_fail_rates),
- distro_fail_rates=Rates(*options.distro_fail_rates),
- reliable_min_runs=options.reliable_test_min_runs,
- reliable_time_period=datetime.timedelta(days=options.reliable_days),
- unreliable_min_runs=options.unreliable_test_min_runs,
- unreliable_time_period=datetime.timedelta(days=options.unreliable_days))
- validate_config(config)
-
- lifecycle_tags_file = make_lifecycle_tags_file(options, config)
- if not lifecycle_tags_file:
- sys.exit(1)
-
- test_tasks_membership = get_test_tasks_membership(evg_conf)
- # If no tests are specified then the list of tests is generated from the list of tasks.
- if not tests:
- tests = get_tests_from_tasks(tasks, test_tasks_membership)
- if not options.tasks:
- use_test_tasks_membership = True
-
- test_history_source = TestHistorySource(
- options.project, variants, distros, config.reliable_time_period,
- config.unreliable_time_period, options.num_request_threads)
- test_tasks_list = []
- for test in tests:
- # Find all associated tasks for the test if tasks or tests were not specified.
- if use_test_tasks_membership:
- tasks = test_tasks_membership[test]
- if not tasks:
- LOGGER.warning("No tasks found for tests %s, skipping this group.", tests)
- continue
- test_tasks_list.append((test, tasks))
- results = test_history_source.get_history(test_tasks_list)
-
- LOGGER.info("Updating the tags")
- nb_tests = len(test_tasks_list)
- count = 0
- for test_history in results:
- progress = round(100 * count / nb_tests, 2)
- LOGGER.info("Progress: %s %%", progress)
- update_tags(lifecycle_tags_file.changelog_lifecycle, config, test_history)
- count += 1
-
- # Remove tags that are no longer relevant
- clean_up_tags(lifecycle_tags_file.changelog_lifecycle, evg_conf)
-
- # We write the 'lifecycle' tag configuration to the 'options.lifecycle_file' file only if there
- # have been changes to the tags. In particular, we avoid modifying the file when only the header
- # comment for the YAML file would change.
- if lifecycle_tags_file.is_modified():
- lifecycle_tags_file.write()
-
- if options.commit:
- commit_ok = lifecycle_tags_file.commit()
- if not commit_ok:
- sys.exit(1)
- else:
- LOGGER.info("The tags have not been modified.")
-
-
-if __name__ == "__main__":
- main()