diff options
author | David Bradford <david.bradford@mongodb.com> | 2019-04-24 14:07:36 -0400 |
---|---|---|
committer | Evergreen Agent <no-reply@evergreen.mongodb.com> | 2021-10-25 18:04:04 +0000 |
commit | ff1db73de8c6285590e6e18a6a1eb5ca6d46b821 (patch) | |
tree | fde660ca82b53e1f8f331ef388601702b6b48ed6 | |
parent | 7bebe91ec9aecb5db917728ba40ce3881457f10a (diff) | |
download | mongo-ff1db73de8c6285590e6e18a6a1eb5ca6d46b821.tar.gz |
SERVER-40486: Remove Test Lifecycle code
(cherry picked from commit 8ffbd4892537721fca0a640a3bced2cfa21e4c59)
-rw-r--r-- | buildscripts/ciconfig/evergreen.py | 13 | ||||
-rwxr-xr-x | buildscripts/fetch_test_lifecycle.py | 199 | ||||
-rw-r--r-- | buildscripts/resmokeconfig/suites/buildscripts_test.yml | 1 | ||||
-rw-r--r-- | buildscripts/tests/ciconfig/evergreen.yml | 4 | ||||
-rw-r--r-- | buildscripts/tests/ciconfig/test_evergreen.py | 8 | ||||
-rw-r--r-- | buildscripts/tests/test_fetch_test_lifecycle.py | 71 | ||||
-rw-r--r-- | buildscripts/tests/test_update_test_lifecycle.py | 1058 | ||||
-rwxr-xr-x | buildscripts/update_test_lifecycle.py | 1110 | ||||
-rw-r--r-- | etc/evergreen.yml | 122 |
9 files changed, 0 insertions, 2586 deletions
diff --git a/buildscripts/ciconfig/evergreen.py b/buildscripts/ciconfig/evergreen.py index 1cbe11fd320..b102d03a181 100644 --- a/buildscripts/ciconfig/evergreen.py +++ b/buildscripts/ciconfig/evergreen.py @@ -79,19 +79,6 @@ class EvergreenProjectConfig(object): # pylint: disable=too-many-instance-attri return self._task_groups_by_name.get(task_group_name) @property - def lifecycle_task_names(self): - """Get the list of names of the tasks that have not been excluded from test lifecycle.""" - excluded = self._get_test_lifecycle_excluded_task_names() - return [name for name in self.task_names if name not in excluded] - - def _get_test_lifecycle_excluded_task_names(self): - excluded_patterns = self._conf.get("test_lifecycle_excluded_tasks", []) - excluded = [] - for pattern in excluded_patterns: - excluded.extend(fnmatch.filter(self.task_names, pattern)) - return excluded - - @property def variant_names(self): """Get the list of build variant names.""" return self._variants_by_name.keys() diff --git a/buildscripts/fetch_test_lifecycle.py b/buildscripts/fetch_test_lifecycle.py index 6d3cfc8d5a7..e69de29bb2d 100755 --- a/buildscripts/fetch_test_lifecycle.py +++ b/buildscripts/fetch_test_lifecycle.py @@ -1,199 +0,0 @@ -#!/usr/bin/env python -"""Retrieve the etc/test_lifecycle.yml tag file from the metadata repository. - -This is performed for the current repository. - -Usage: - python buildscsripts/fetch_test_lifecycle.py evergreen-project revision -""" - -from __future__ import absolute_import -from __future__ import print_function - -import logging -import optparse -import os -import posixpath -import sys -import textwrap - -import yaml - -# Get relative imports to work when the package is not installed on the PYTHONPATH. -if __name__ == "__main__" and __package__ is None: - sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - -from buildscripts import git # pylint: disable=wrong-import-position - -LOGGER = logging.getLogger(__name__) - - -class MetadataRepository(object): - """Represent the metadata repository containing the test lifecycle tags file.""" - - def __init__(self, repository, references_file, - lifecycle_file): # noqa: D214,D405,D406,D407,D411,D413 - """Initialize the MetadataRepository. - - Args: - repository: the git.Repository object for the repository. - references_file: the relative path from the root of the repository to the references - yaml file. - lifecycle_file: the relative path from the root of the repository to the test lifecycle - tags yaml file. - """ - self._repository = repository - self._references_file = references_file - self._lifecycle_file = lifecycle_file - # The path to the lifecycle file, absolute or relative to the current working directory. - self.lifecycle_path = os.path.join(repository.directory, lifecycle_file) - - def list_revisions(self): # noqa: D406,D407,D413 - """List the revisions from the HEAD of this repository. - - Returns: - A list of str containing the git hashes for all the revisions from the newest (HEAD) - to the oldest. - """ - return self._repository.git_rev_list(["HEAD", "--", self._lifecycle_file]).splitlines() - - def _get_references_content(self, revision): - references_content = self._repository.git_cat_file( - ["blob", "%s:%s" % (revision, self._references_file)]) - return references_content - - def get_reference(self, metadata_revision, project): - """Retrieve the reference revision (a revision of the project 'project'). - - The revision is associated with the test lifecycle file present in the metadata repository - at revision 'metadata_revision'. - - Args: - metadata_revision: a revision (git hash) of this repository. - project: an Evergreen project name (e.g. mongodb-mongo-master). - """ - references_content = self._get_references_content(metadata_revision) - references = yaml.safe_load(references_content) - return references.get("test-lifecycle", {}).get(project) - - def get_lifecycle_file_content(self, metadata_revision): - """Return the content of the test lifecycle file as it was at the given revision.""" - return self._repository.git_cat_file( - ["blob", "%s:%s" % (metadata_revision, self._lifecycle_file)]) - - -def _clone_repository(url, branch): - """Clone the repository present at the URL 'url' and use the branch 'branch'.""" - target_directory = posixpath.splitext(posixpath.basename(url))[0] - LOGGER.info("Cloning the repository %s into the directory %s", url, target_directory) - return git.Repository.clone(url, target_directory, branch) - - -def _get_metadata_revision(metadata_repo, mongo_repo, project, revision): - """Get the metadata revision that corresponds to a given repository, project, revision.""" - for metadata_revision in metadata_repo.list_revisions(): - reference = metadata_repo.get_reference(metadata_revision, project) - if not reference: - # No reference for this revision. This should not happen but we keep trying in - # case we can find an older revision with a reference. - continue - if mongo_repo.is_ancestor(reference, revision): - # We found a reference that is a parent of the current revision. - return metadata_revision - return None - - -def fetch_test_lifecycle(metadata_repo_url, references_file, lifecycle_file, project, - revision): # noqa: D214,D405,D406,D407,D411,D413 - """Fetch the test lifecycle file for the revision in the repository this script is invoked. - - Args: - metadata_repo_url: the git repository URL for the metadata repository containing the test - lifecycle file. - references_file: the relative path from the root of the metadata repository to the - references file. - lifecycle_file: the relative path from the root of the metadata repository to the test - lifecycle file. - project: the Evergreen project name. - revision: the current repository revision. - """ - metadata_repo = MetadataRepository( - _clone_repository(metadata_repo_url, project), references_file, lifecycle_file) - mongo_repo = git.Repository(os.getcwd()) - metadata_revision = _get_metadata_revision(metadata_repo, mongo_repo, project, revision) - if metadata_revision: - LOGGER.info("Using metadata repository revision '%s'", metadata_revision) - result = metadata_repo.get_lifecycle_file_content(metadata_revision) - else: - result = None - return result - - -def main(): - """Execute Main program. - - Utility to fetch the etc/test_lifecycle.yml file corresponding to a given revision from - the mongo-test-metadata repository. - """ - parser = optparse.OptionParser( - description=textwrap.dedent(main.__doc__), usage="Usage: %prog [options] evergreen-project") - - parser.add_option("--revision", dest="revision", metavar="<revision>", default="HEAD", - help=("The project revision for which to retrieve the test lifecycle tags" - " file.")) - - parser.add_option("--metadataRepo", dest="metadata_repo_url", metavar="<metadata-repo-url>", - default="git@github.com:mongodb/mongo-test-metadata.git", - help=("The URL to the metadata repository that contains the test lifecycle" - " tags file.")) - - parser.add_option("--lifecycleFile", dest="lifecycle_file", metavar="<lifecycle-file>", - default="etc/test_lifecycle.yml", - help=("The path to the test lifecycle tags file, relative to the root of the" - " metadata repository. Defaults to '%default'.")) - - parser.add_option("--referencesFile", dest="references_file", metavar="<references-file>", - default="references.yml", - help=("The path to the metadata references file, relative to the root of the" - " metadata repository. Defaults to '%default'.")) - - parser.add_option("--destinationFile", dest="destination_file", metavar="<destination-file>", - default="etc/test_lifecycle.yml", - help=("The path where the lifecycle file should be available when this script" - " completes successfully. This path is absolute or relative to the" - " current working directory. Defaults to '%default'.")) - - parser.add_option("--logLevel", dest="log_level", metavar="<log-level>", - choices=["DEBUG", "INFO", "WARNING", "ERROR"], default="INFO", - help="The log level: DEBUG, INFO, WARNING or ERROR. Defaults to '%default'.") - - parser.add_option("--logFile", dest="log_file", metavar="<log-file>", default=None, - help=("The destination file for the logs. If not set the script will log to" - " the standard output")) - - options, args = parser.parse_args() - - if len(args) != 1: - parser.print_help(file=sys.stderr) - print(file=sys.stderr) - parser.error("Must specify an Evergreen project") - evergreen_project = args[0] - - logging.basicConfig(format="%(asctime)s %(levelname)s %(message)s", level=options.log_level, - filename=options.log_file) - - lifecycle_file_content = fetch_test_lifecycle(options.metadata_repo_url, - options.references_file, options.lifecycle_file, - evergreen_project, options.revision) - if not lifecycle_file_content: - LOGGER.error("Failed to fetch the test lifecycle tag file.") - sys.exit(1) - else: - LOGGER.info("Writing the test lifecycle file to '%s'.", options.destination_file) - with open(options.destination_file, "wb") as destf: - destf.write(lifecycle_file_content) - LOGGER.info("Done.") - - -if __name__ == "__main__": - main() diff --git a/buildscripts/resmokeconfig/suites/buildscripts_test.yml b/buildscripts/resmokeconfig/suites/buildscripts_test.yml index 3184bb1fed9..7f438ee8d78 100644 --- a/buildscripts/resmokeconfig/suites/buildscripts_test.yml +++ b/buildscripts/resmokeconfig/suites/buildscripts_test.yml @@ -9,7 +9,6 @@ selector: - buildscripts/tests/resmokelib/test_selector.py # Test assumes POSIX path. - buildscripts/tests/test_aws_ec2.py # Requires boto3. - buildscripts/tests/test_remote_operations.py # Requires ssh to be enabled locally. - - buildscripts/tests/test_update_test_lifecycle.py # Test assumes POSIX path. executor: {} diff --git a/buildscripts/tests/ciconfig/evergreen.yml b/buildscripts/tests/ciconfig/evergreen.yml index 65d1fd860e6..3e1980a3ce5 100644 --- a/buildscripts/tests/ciconfig/evergreen.yml +++ b/buildscripts/tests/ciconfig/evergreen.yml @@ -68,10 +68,6 @@ post: params: file_location: src/results.json -test_lifecycle_excluded_tasks: -- burn_in_tests -- no_lifecycle* - task_groups: - name: tg_1 max_hosts: 1 diff --git a/buildscripts/tests/ciconfig/test_evergreen.py b/buildscripts/tests/ciconfig/test_evergreen.py index 8fd3d25ed9a..397f45c42c7 100644 --- a/buildscripts/tests/ciconfig/test_evergreen.py +++ b/buildscripts/tests/ciconfig/test_evergreen.py @@ -40,14 +40,6 @@ class TestEvergreenProjectConfig(unittest.TestCase): self.assertEqual(1, len(self.conf.task_group_names)) self.assertIn("tg_1", self.conf.task_group_names) - def test_list_lifecycle_task_names(self): - self.assertEqual(5, len(self.conf.lifecycle_task_names)) - self.assertIn("compile", self.conf.task_names) - self.assertIn("passing_test", self.conf.task_names) - self.assertIn("failing_test", self.conf.task_names) - self.assertIn("timeout_test", self.conf.task_names) - self.assertIn("resmoke_task", self.conf.task_names) - def test_list_variants(self): self.assertEqual(4, len(self.conf.variants)) self.assertEqual(4, len(self.conf.variant_names)) diff --git a/buildscripts/tests/test_fetch_test_lifecycle.py b/buildscripts/tests/test_fetch_test_lifecycle.py deleted file mode 100644 index 14c57435dab..00000000000 --- a/buildscripts/tests/test_fetch_test_lifecycle.py +++ /dev/null @@ -1,71 +0,0 @@ -"""Unit tests for the fetch_test_lifecycle.py script.""" -from __future__ import absolute_import - -import unittest - -import buildscripts.fetch_test_lifecycle as fetch - -# pylint: disable=missing-docstring,protected-access - - -class TestFetchTestLifecycle(unittest.TestCase): - def test_get_metadata_revision(self): - metadata_repo = MockMetadataRepository( - [("metadata_revision_05", "mongo_revision_06"), - ("metadata_revision_04", "mongo_revision_06"), ("metadata_revision_03", - "mongo_revision_02"), - ("metadata_revision_02", "mongo_revision_02"), ("metadata_revision_01", None)]) - - mongo_repo = MockMongoRepository([ - "mongo_revision_07", "mongo_revision_06", "mongo_revision_05", "mongo_revision_04", - "mongo_revision_03", "mongo_revision_02", "mongo_revision_01" - ]) - - self._check_metadata_revision(metadata_repo, mongo_repo, "mongo_revision_07", - "metadata_revision_05") - - self._check_metadata_revision(metadata_repo, mongo_repo, "mongo_revision_06", - "metadata_revision_05") - - self._check_metadata_revision(metadata_repo, mongo_repo, "mongo_revision_05", - "metadata_revision_03") - - self._check_metadata_revision(metadata_repo, mongo_repo, "mongo_revision_04", - "metadata_revision_03") - - self._check_metadata_revision(metadata_repo, mongo_repo, "mongo_revision_03", - "metadata_revision_03") - - self._check_metadata_revision(metadata_repo, mongo_repo, "mongo_revision_02", - "metadata_revision_03") - - self._check_metadata_revision(metadata_repo, mongo_repo, "mongo_revision_01", None) - - def _check_metadata_revision(self, metadata_repo, mongo_repo, mongo_revision, - expected_metadata_revision): - metadata_revision = fetch._get_metadata_revision(metadata_repo, mongo_repo, "project", - mongo_revision) - self.assertEqual(expected_metadata_revision, metadata_revision) - - -class MockMongoRepository(object): - def __init__(self, revisions): - self.revisions = revisions - - def is_ancestor(self, parent, child): - return (parent in self.revisions and child in self.revisions - and self.revisions.index(parent) >= self.revisions.index(child)) - - -class MockMetadataRepository(object): - def __init__(self, references_revisions): - self.references_revisions = references_revisions - - def list_revisions(self): - return [r[0] for r in self.references_revisions] - - def get_reference(self, revision, project): # pylint: disable=unused-argument - for (metadata_revision, mongo_revision) in self.references_revisions: - if metadata_revision == revision: - return mongo_revision - return None diff --git a/buildscripts/tests/test_update_test_lifecycle.py b/buildscripts/tests/test_update_test_lifecycle.py deleted file mode 100644 index f9447ba5902..00000000000 --- a/buildscripts/tests/test_update_test_lifecycle.py +++ /dev/null @@ -1,1058 +0,0 @@ -""" -Tests for buildscripts/update_test_lifecycle.py. -""" - -from __future__ import absolute_import - -import collections -import copy -import datetime -import unittest - -from buildscripts import lifecycle_test_failures as test_failures -from buildscripts import update_test_lifecycle -from buildscripts.ciconfig import tags as ci_tags - -# pylint: disable=invalid-name,missing-docstring,protected-access,too-many-lines - - -class TestValidateConfig(unittest.TestCase): - """ - Tests for the validate_config() function. - """ - - CONFIG = update_test_lifecycle.Config( - test_fail_rates=update_test_lifecycle.Rates( - acceptable=0, unacceptable=1), task_fail_rates=update_test_lifecycle.Rates( - acceptable=0, unacceptable=1), variant_fail_rates=update_test_lifecycle.Rates( - acceptable=0, unacceptable=1), distro_fail_rates=update_test_lifecycle.Rates( - acceptable=0, unacceptable=1), reliable_min_runs=2, - reliable_time_period=datetime.timedelta(days=1), unreliable_min_runs=2, - unreliable_time_period=datetime.timedelta(days=1)) - - def test_acceptable_test_fail_rate(self): - """ - Tests the validation of the 'test_fail_rates.acceptable' attribute. - """ - - with self.assertRaises(TypeError): - config = self.CONFIG._replace( - test_fail_rates=self.CONFIG.test_fail_rates._replace(acceptable="not a number")) - update_test_lifecycle.validate_config(config) - - with self.assertRaises(ValueError): - config = self.CONFIG._replace( - test_fail_rates=self.CONFIG.test_fail_rates._replace(acceptable=-1)) - update_test_lifecycle.validate_config(config) - - with self.assertRaises(ValueError): - config = self.CONFIG._replace( - test_fail_rates=self.CONFIG.test_fail_rates._replace(acceptable=2)) - update_test_lifecycle.validate_config(config) - - def test_unacceptable_test_fail_rate(self): - """ - Tests the validation of the 'test_fail_rates.unacceptable' attribute. - """ - - with self.assertRaises(TypeError): - config = self.CONFIG._replace( - test_fail_rates=self.CONFIG.test_fail_rates._replace(unacceptable="not a number")) - update_test_lifecycle.validate_config(config) - - with self.assertRaises(ValueError): - config = self.CONFIG._replace( - test_fail_rates=self.CONFIG.test_fail_rates._replace(unacceptable=-1)) - update_test_lifecycle.validate_config(config) - - with self.assertRaises(ValueError): - config = self.CONFIG._replace( - test_fail_rates=self.CONFIG.test_fail_rates._replace(unacceptable=2)) - update_test_lifecycle.validate_config(config) - - def test_test_fail_rates(self): - """ - Tests the validation of the 'test_fail_rates' attribute. - """ - - with self.assertRaises(ValueError): - config = self.CONFIG._replace( - test_fail_rates=self.CONFIG.test_fail_rates._replace(acceptable=0.9, - unacceptable=0.1)) - update_test_lifecycle.validate_config(config) - - def test_acceptable_task_fail_rate(self): - """ - Tests the validation of the 'test_fail_rates.acceptable' attribute. - """ - - with self.assertRaises(TypeError): - config = self.CONFIG._replace( - task_fail_rates=self.CONFIG.task_fail_rates._replace(acceptable="not a number")) - update_test_lifecycle.validate_config(config) - - with self.assertRaises(ValueError): - config = self.CONFIG._replace( - task_fail_rates=self.CONFIG.task_fail_rates._replace(acceptable=-1)) - update_test_lifecycle.validate_config(config) - - with self.assertRaises(ValueError): - config = self.CONFIG._replace( - task_fail_rates=self.CONFIG.task_fail_rates._replace(acceptable=2)) - update_test_lifecycle.validate_config(config) - - def test_unacceptable_task_fail_rate(self): - """ - Tests the validation of the 'task_fail_rates.unacceptable' attribute. - """ - - with self.assertRaises(TypeError): - config = self.CONFIG._replace( - task_fail_rates=self.CONFIG.task_fail_rates._replace(unacceptable="not a number")) - update_test_lifecycle.validate_config(config) - - with self.assertRaises(ValueError): - config = self.CONFIG._replace( - task_fail_rates=self.CONFIG.task_fail_rates._replace(unacceptable=-1)) - update_test_lifecycle.validate_config(config) - - with self.assertRaises(ValueError): - config = self.CONFIG._replace( - task_fail_rates=self.CONFIG.task_fail_rates._replace(unacceptable=2)) - update_test_lifecycle.validate_config(config) - - def test_task_fail_rates(self): - """ - Tests the validation of the 'task_fail_rates' attribute. - """ - - with self.assertRaises(ValueError): - config = self.CONFIG._replace( - task_fail_rates=self.CONFIG.task_fail_rates._replace(acceptable=0.9, - unacceptable=0.1)) - update_test_lifecycle.validate_config(config) - - def test_acceptable_variant_fail_rate(self): - """ - Tests the validation of the 'variant_fail_rates.acceptable' attribute. - """ - - with self.assertRaises(TypeError): - config = self.CONFIG._replace( - variant_fail_rates=self.CONFIG.variant_fail_rates._replace( - acceptable="not a number")) - update_test_lifecycle.validate_config(config) - - with self.assertRaises(ValueError): - config = self.CONFIG._replace( - variant_fail_rates=self.CONFIG.variant_fail_rates._replace(acceptable=-1)) - update_test_lifecycle.validate_config(config) - - with self.assertRaises(ValueError): - config = self.CONFIG._replace( - variant_fail_rates=self.CONFIG.variant_fail_rates._replace(acceptable=2)) - update_test_lifecycle.validate_config(config) - - def test_unacceptable_variant_fail_rate(self): - """ - Tests the validation of the 'variant_fail_rates.unacceptable' attribute. - """ - - with self.assertRaises(TypeError): - config = self.CONFIG._replace( - variant_fail_rates=self.CONFIG.variant_fail_rates._replace( - unacceptable="not a number")) - update_test_lifecycle.validate_config(config) - - with self.assertRaises(ValueError): - config = self.CONFIG._replace( - variant_fail_rates=self.CONFIG.variant_fail_rates._replace(unacceptable=-1)) - update_test_lifecycle.validate_config(config) - - with self.assertRaises(ValueError): - config = self.CONFIG._replace( - variant_fail_rates=self.CONFIG.variant_fail_rates._replace(unacceptable=2)) - update_test_lifecycle.validate_config(config) - - def test_variant_fail_rates(self): - """ - Tests the validation of the 'variant_fail_rates' attribute. - """ - - with self.assertRaises(ValueError): - config = self.CONFIG._replace( - variant_fail_rates=self.CONFIG.variant_fail_rates._replace( - acceptable=0.9, unacceptable=0.1)) - update_test_lifecycle.validate_config(config) - - def test_acceptable_distro_fail_rate(self): - """ - Tests the validation of the 'distro_fail_rates.acceptable' attribute. - """ - - with self.assertRaises(TypeError): - config = self.CONFIG._replace( - distro_fail_rates=self.CONFIG.distro_fail_rates._replace(acceptable="not a number")) - update_test_lifecycle.validate_config(config) - - with self.assertRaises(ValueError): - config = self.CONFIG._replace( - distro_fail_rates=self.CONFIG.distro_fail_rates._replace(acceptable=-1)) - update_test_lifecycle.validate_config(config) - - with self.assertRaises(ValueError): - config = self.CONFIG._replace( - distro_fail_rates=self.CONFIG.distro_fail_rates._replace(acceptable=2)) - update_test_lifecycle.validate_config(config) - - def test_unacceptable_distro_fail_rate(self): - """ - Tests the validation of the 'distro_fail_rates.unacceptable' attribute. - """ - - with self.assertRaises(TypeError): - config = self.CONFIG._replace( - distro_fail_rates=self.CONFIG.distro_fail_rates._replace( - unacceptable="not a number")) - update_test_lifecycle.validate_config(config) - - with self.assertRaises(ValueError): - config = self.CONFIG._replace( - distro_fail_rates=self.CONFIG.distro_fail_rates._replace(unacceptable=-1)) - update_test_lifecycle.validate_config(config) - - with self.assertRaises(ValueError): - config = self.CONFIG._replace( - distro_fail_rates=self.CONFIG.distro_fail_rates._replace(unacceptable=2)) - update_test_lifecycle.validate_config(config) - - def test_distro_fail_rates(self): - """ - Tests the validation of the 'distro_fail_rates' attribute. - """ - - with self.assertRaises(ValueError): - config = self.CONFIG._replace( - distro_fail_rates=self.CONFIG.distro_fail_rates._replace( - acceptable=0.9, unacceptable=0.1)) - update_test_lifecycle.validate_config(config) - - def test_reliable_min_runs(self): - """ - Tests the validation of the 'reliable_min_runs' attribute. - """ - - with self.assertRaises(TypeError): - config = self.CONFIG._replace(reliable_min_runs="not a number") - update_test_lifecycle.validate_config(config) - - with self.assertRaises(ValueError): - config = self.CONFIG._replace(reliable_min_runs=-1) - update_test_lifecycle.validate_config(config) - - with self.assertRaises(ValueError): - config = self.CONFIG._replace(reliable_min_runs=0) - update_test_lifecycle.validate_config(config) - - with self.assertRaises(ValueError): - config = self.CONFIG._replace(reliable_min_runs=1.5) - update_test_lifecycle.validate_config(config) - - def test_reliable_time_period(self): - """ - Tests the validation of the 'reliable_time_period' attribute. - """ - - with self.assertRaises(TypeError): - config = self.CONFIG._replace(reliable_time_period="not a datetime.timedelta") - update_test_lifecycle.validate_config(config) - - with self.assertRaises(ValueError): - config = self.CONFIG._replace(reliable_time_period=datetime.timedelta(days=-1)) - update_test_lifecycle.validate_config(config) - - with self.assertRaises(ValueError): - config = self.CONFIG._replace(reliable_time_period=datetime.timedelta(days=0)) - update_test_lifecycle.validate_config(config) - - with self.assertRaises(ValueError): - config = self.CONFIG._replace(reliable_time_period=datetime.timedelta(days=1, hours=1)) - update_test_lifecycle.validate_config(config) - - def test_unreliable_min_runs(self): - """ - Tests the validation of the 'unreliable_min_runs' attribute. - """ - - with self.assertRaises(TypeError): - config = self.CONFIG._replace(unreliable_min_runs="not a number") - update_test_lifecycle.validate_config(config) - - with self.assertRaises(ValueError): - config = self.CONFIG._replace(unreliable_min_runs=-1) - update_test_lifecycle.validate_config(config) - - with self.assertRaises(ValueError): - config = self.CONFIG._replace(unreliable_min_runs=0) - update_test_lifecycle.validate_config(config) - - with self.assertRaises(ValueError): - config = self.CONFIG._replace(unreliable_min_runs=1.5) - update_test_lifecycle.validate_config(config) - - def test_unreliable_time_period(self): - """ - Tests the validation of the 'unreliable_time_period' attribute. - """ - - with self.assertRaises(TypeError): - config = self.CONFIG._replace(unreliable_time_period="not a datetime.timedelta") - update_test_lifecycle.validate_config(config) - - with self.assertRaises(ValueError): - config = self.CONFIG._replace(unreliable_time_period=datetime.timedelta(days=-1)) - update_test_lifecycle.validate_config(config) - - with self.assertRaises(ValueError): - config = self.CONFIG._replace(unreliable_time_period=datetime.timedelta(days=0)) - update_test_lifecycle.validate_config(config) - - with self.assertRaises(ValueError): - config = self.CONFIG._replace( - unreliable_time_period=datetime.timedelta(days=1, hours=1)) - update_test_lifecycle.validate_config(config) - - -class TestUpdateTags(unittest.TestCase): # pylint: disable=too-many-public-methods - """ - Tests for the update_tags() function. - """ - - CONFIG = update_test_lifecycle.Config( - test_fail_rates=update_test_lifecycle.Rates( - acceptable=0, unacceptable=1), task_fail_rates=update_test_lifecycle.Rates( - acceptable=0, unacceptable=1), variant_fail_rates=update_test_lifecycle.Rates( - acceptable=0, unacceptable=1), distro_fail_rates=update_test_lifecycle.Rates( - acceptable=0, unacceptable=1), reliable_min_runs=2, - reliable_time_period=datetime.timedelta(days=1), unreliable_min_runs=2, - unreliable_time_period=datetime.timedelta(days=1)) - - ENTRY = test_failures.ReportEntry(test="jstests/core/all.js", task="jsCore_WT", - variant="linux-64", distro="rhel62", start_date=datetime.date( - 2017, 6, 3), end_date=datetime.date(2017, 6, 3), - num_pass=0, num_fail=0) - - def assert_has_only_js_tests(self, lifecycle): - """ - Raises an AssertionError exception if 'lifecycle' is not of the following form: - - selector: - js_test: - ... - """ - - self.assertIn("selector", lifecycle.raw) - self.assertEqual(1, len(lifecycle.raw), msg=str(lifecycle.raw)) - self.assertIn("js_test", lifecycle.raw["selector"]) - self.assertEqual(1, len(lifecycle.raw["selector"]), msg=str(lifecycle.raw)) - - return lifecycle.raw["selector"]["js_test"] - - def transition_from_reliable_to_unreliable(self, config, expected_tags): - """ - Tests that update_tags() tags a formerly reliable combination as being unreliable. - """ - - initial_tags = collections.OrderedDict() - lifecycle = ci_tags.TagsConfig.from_dict( - dict(selector=dict(js_test=copy.deepcopy(initial_tags)))) - summary_lifecycle = update_test_lifecycle.TagsConfigWithChangelog(lifecycle) - self.assertEqual(collections.OrderedDict(), self.assert_has_only_js_tests(lifecycle)) - - tests = ["jstests/core/all.js"] - report = test_failures.Report([ - self.ENTRY._replace(num_pass=0, num_fail=1), - self.ENTRY._replace(num_pass=0, num_fail=1, task="jsCore"), - self.ENTRY._replace(num_pass=0, num_fail=1, variant="linux-64-debug"), - self.ENTRY._replace(num_pass=1, num_fail=0), - self.ENTRY._replace(num_pass=0, num_fail=1, distro="rhel55"), - ]) - - update_test_lifecycle.validate_config(config) - update_test_lifecycle.update_tags(summary_lifecycle, config, report, tests) - updated_tags = self.assert_has_only_js_tests(lifecycle) - self.assertEqual(updated_tags, expected_tags) - - def test_transition_test_from_reliable_to_unreliable(self): - """ - Tests that update_tags() tags a formerly reliable (test,) combination as being unreliable. - """ - - config = self.CONFIG._replace( - test_fail_rates=self.CONFIG.test_fail_rates._replace(unacceptable=0.1)) - - self.transition_from_reliable_to_unreliable(config, - collections.OrderedDict([ - ("jstests/core/all.js", ["unreliable"]), - ])) - - def test_transition_task_from_reliable_to_unreliable(self): - """ - Tests that update_tags() tags a formerly reliable (test, task) combination as being - unreliable. - """ - - config = self.CONFIG._replace( - task_fail_rates=self.CONFIG.task_fail_rates._replace(unacceptable=0.1)) - - self.transition_from_reliable_to_unreliable(config, - collections.OrderedDict([ - ("jstests/core/all.js", - ["unreliable|jsCore_WT"]), - ])) - - def test_transition_variant_from_reliable_to_unreliable(self): - """ - Tests that update_tags() tags a formerly reliable (test, task, variant) combination as being - unreliable. - """ - - config = self.CONFIG._replace( - variant_fail_rates=self.CONFIG.variant_fail_rates._replace(unacceptable=0.1)) - - self.transition_from_reliable_to_unreliable(config, - collections.OrderedDict([ - ("jstests/core/all.js", - ["unreliable|jsCore_WT|linux-64"]), - ])) - - def test_transition_distro_from_reliable_to_unreliable(self): - """ - Tests that update_tags() tags a formerly reliable (test, task, variant, distro) combination - as being unreliable. - """ - - config = self.CONFIG._replace( - distro_fail_rates=self.CONFIG.distro_fail_rates._replace(unacceptable=0.1)) - - self.transition_from_reliable_to_unreliable(config, - collections.OrderedDict([ - ("jstests/core/all.js", - ["unreliable|jsCore_WT|linux-64|rhel62"]), - ])) - - def test_transition_from_reliable_to_unreliable(self): - """ - Tests that update_tags() tags multiple formerly reliable combination as being unreliable. - """ - - config = self.CONFIG._replace( - test_fail_rates=self.CONFIG.test_fail_rates._replace(unacceptable=0.1), - task_fail_rates=self.CONFIG.task_fail_rates._replace(unacceptable=0.1), - variant_fail_rates=self.CONFIG.variant_fail_rates._replace(unacceptable=0.1), - distro_fail_rates=self.CONFIG.distro_fail_rates._replace(unacceptable=0.1)) - - self.transition_from_reliable_to_unreliable(config, - collections.OrderedDict([ - ("jstests/core/all.js", [ - "unreliable", - "unreliable|jsCore_WT", - "unreliable|jsCore_WT|linux-64", - "unreliable|jsCore_WT|linux-64|rhel62", - ]), - ])) - - def transition_from_unreliable_to_reliable(self, config, initial_tags): - """ - Tests that update_tags() untags a formerly unreliable combination after it has become - reliable again. - """ - - lifecycle = ci_tags.TagsConfig.from_dict( - dict(selector=dict(js_test=copy.deepcopy(initial_tags)))) - summary_lifecycle = update_test_lifecycle.TagsConfigWithChangelog(lifecycle) - self.assertEqual(initial_tags, self.assert_has_only_js_tests(lifecycle)) - - tests = ["jstests/core/all.js"] - report = test_failures.Report([ - self.ENTRY._replace(num_pass=1, num_fail=0), - self.ENTRY._replace(num_pass=1, num_fail=0, task="jsCore"), - self.ENTRY._replace(num_pass=1, num_fail=0, variant="linux-64-debug"), - self.ENTRY._replace(num_pass=0, num_fail=1), - self.ENTRY._replace(num_pass=1, num_fail=0, distro="rhel55"), - ]) - - update_test_lifecycle.validate_config(config) - update_test_lifecycle.update_tags(summary_lifecycle, config, report, tests) - updated_tags = self.assert_has_only_js_tests(lifecycle) - self.assertEqual(updated_tags, collections.OrderedDict()) - - def test_non_running_in_reliable_period_is_reliable(self): - """ - Tests that tests that have a failure rate above the unacceptable rate during the unreliable - period but haven't run during the reliable period are marked as reliable. - """ - # Unreliable period is 2 days: 2017-06-03 to 2017-06-04. - # Reliable period is 1 day: 2016-06-04. - reliable_period_date = datetime.date(2017, 6, 4) - config = self.CONFIG._replace( - test_fail_rates=self.CONFIG.test_fail_rates._replace(unacceptable=0.1), - task_fail_rates=self.CONFIG.task_fail_rates._replace(unacceptable=0.1), - variant_fail_rates=self.CONFIG.variant_fail_rates._replace(unacceptable=0.1), - distro_fail_rates=self.CONFIG.distro_fail_rates._replace(unacceptable=0.1), - unreliable_time_period=datetime.timedelta(days=2)) - - tests = ["jstests/core/all.js"] - initial_tags = collections.OrderedDict([ - ("jstests/core/all.js", [ - "unreliable", - "unreliable|jsCore_WT", - "unreliable|jsCore_WT|linux-64", - "unreliable|jsCore_WT|linux-64|rhel62", - ]), - ]) - - lifecycle = ci_tags.TagsConfig.from_dict( - dict(selector=dict(js_test=copy.deepcopy(initial_tags)))) - summary_lifecycle = update_test_lifecycle.TagsConfigWithChangelog(lifecycle) - self.assertEqual(initial_tags, self.assert_has_only_js_tests(lifecycle)) - - # The test did not run on the reliable period on linux-64. - report = test_failures.Report([ - # Failing. - self.ENTRY._replace(num_pass=0, num_fail=2), - # Passing on a different variant. - self.ENTRY._replace(start_date=reliable_period_date, end_date=reliable_period_date, - num_pass=3, num_fail=0, variant="linux-alt", distro="debian7"), - ]) - - update_test_lifecycle.validate_config(config) - update_test_lifecycle.update_tags(summary_lifecycle, config, report, tests) - updated_tags = self.assert_has_only_js_tests(lifecycle) - # The tags for variant and distro have been removed. - self.assertEqual(updated_tags, - collections.OrderedDict([("jstests/core/all.js", - ["unreliable", "unreliable|jsCore_WT"])])) - - def test_non_running_at_all_is_reliable(self): - """ - Tests that tests that are tagged as unreliable but no longer running (either during the - reliable or the unreliable period) have their tags removed. - """ - config = self.CONFIG - - tests = ["jstests/core/all.js", "jstests/core/all2.js"] - initial_tags = collections.OrderedDict([ - ("jstests/core/all2.js", [ - "unreliable", - "unreliable|jsCore_WT", - "unreliable|jsCore_WT|linux-64", - "unreliable|jsCore_WT|linux-64|rhel62", - ]), - ]) - - lifecycle = ci_tags.TagsConfig.from_dict( - dict(selector=dict(js_test=copy.deepcopy(initial_tags)))) - summary_lifecycle = update_test_lifecycle.TagsConfigWithChangelog(lifecycle) - self.assertEqual(initial_tags, self.assert_has_only_js_tests(lifecycle)) - - # all2.js did not run at all - report = test_failures.Report([self.ENTRY]) - - update_test_lifecycle.validate_config(config) - update_test_lifecycle.update_tags(summary_lifecycle, config, report, tests) - updated_tags = self.assert_has_only_js_tests(lifecycle) - # The tags for variant and distro have been removed. - self.assertEqual(updated_tags, collections.OrderedDict([])) - - def test_transition_test_from_unreliable_to_reliable(self): - """ - Tests that update_tags() untags a formerly unreliable (test,) combination after it has - become reliable again. - """ - - config = self.CONFIG._replace( - test_fail_rates=self.CONFIG.test_fail_rates._replace(acceptable=0.9)) - - self.transition_from_unreliable_to_reliable(config, - collections.OrderedDict([ - ("jstests/core/all.js", ["unreliable"]), - ])) - - def test_transition_task_from_unreliable_to_reliable(self): - """ - Tests that update_tags() untags a formerly unreliable (test, task) combination after it has - become reliable again. - """ - - config = self.CONFIG._replace( - task_fail_rates=self.CONFIG.task_fail_rates._replace(acceptable=0.9)) - - self.transition_from_unreliable_to_reliable(config, - collections.OrderedDict([ - ("jstests/core/all.js", - ["unreliable|jsCore_WT"]), - ])) - - def test_transition_variant_from_unreliable_to_reliable(self): - """ - Tests that update_tags() untags a formerly unreliable (test, task, variant) combination - after it has become reliable again. - """ - - config = self.CONFIG._replace( - variant_fail_rates=self.CONFIG.variant_fail_rates._replace(acceptable=0.9)) - - self.transition_from_unreliable_to_reliable(config, - collections.OrderedDict([ - ("jstests/core/all.js", - ["unreliable|jsCore_WT|linux-64"]), - ])) - - def test_transition_distro_from_unreliable_to_reliable(self): - """ - Tests that update_tags() untags a formerly unreliable (test, task, variant, distro) - combination after it has become reliable again. - """ - - config = self.CONFIG._replace( - distro_fail_rates=self.CONFIG.distro_fail_rates._replace(acceptable=0.9)) - - self.transition_from_unreliable_to_reliable(config, - collections.OrderedDict([ - ("jstests/core/all.js", - ["unreliable|jsCore_WT|linux-64|rhel62"]), - ])) - - def test_transition_from_unreliable_to_reliable(self): - """ - Tests that update_tags() untags multiple formerly unreliable combination after it has become - reliable again. - """ - - config = self.CONFIG._replace( - test_fail_rates=self.CONFIG.test_fail_rates._replace(acceptable=0.9), - task_fail_rates=self.CONFIG.task_fail_rates._replace(acceptable=0.9), - variant_fail_rates=self.CONFIG.variant_fail_rates._replace(acceptable=0.9), - distro_fail_rates=self.CONFIG.distro_fail_rates._replace(acceptable=0.9)) - - self.transition_from_unreliable_to_reliable(config, - collections.OrderedDict([ - ("jstests/core/all.js", [ - "unreliable", - "unreliable|jsCore_WT", - "unreliable|jsCore_WT|linux-64", - "unreliable|jsCore_WT|linux-64|rhel62", - ]), - ])) - - def test_remain_reliable(self): - """ - Tests that update_tags() preserves the absence of tags for reliable combinations. - """ - - config = self.CONFIG._replace( - test_fail_rates=self.CONFIG.test_fail_rates._replace(acceptable=0.9), - task_fail_rates=self.CONFIG.task_fail_rates._replace(acceptable=0.9), - variant_fail_rates=self.CONFIG.variant_fail_rates._replace(acceptable=0.9), - distro_fail_rates=self.CONFIG.distro_fail_rates._replace(acceptable=0.9)) - - initial_tags = collections.OrderedDict() - lifecycle = ci_tags.TagsConfig.from_dict( - dict(selector=dict(js_test=copy.deepcopy(initial_tags)))) - summary_lifecycle = update_test_lifecycle.TagsConfigWithChangelog(lifecycle) - self.assertEqual(initial_tags, self.assert_has_only_js_tests(lifecycle)) - - tests = ["jstests/core/all.js"] - report = test_failures.Report([ - self.ENTRY._replace(num_pass=1, num_fail=0), - self.ENTRY._replace(num_pass=1, num_fail=0, task="jsCore"), - self.ENTRY._replace(num_pass=1, num_fail=0, variant="linux-64-debug"), - self.ENTRY._replace(num_pass=0, num_fail=1), - self.ENTRY._replace(num_pass=1, num_fail=0, distro="rhel55"), - ]) - - update_test_lifecycle.validate_config(config) - update_test_lifecycle.update_tags(summary_lifecycle, config, report, tests) - updated_tags = self.assert_has_only_js_tests(lifecycle) - self.assertEqual(updated_tags, initial_tags) - - def test_remain_unreliable(self): - """ - Tests that update_tags() preserves the tags for unreliable combinations. - """ - - config = self.CONFIG._replace( - test_fail_rates=self.CONFIG.test_fail_rates._replace(unacceptable=0.1), - task_fail_rates=self.CONFIG.task_fail_rates._replace(unacceptable=0.1), - variant_fail_rates=self.CONFIG.variant_fail_rates._replace(unacceptable=0.1), - distro_fail_rates=self.CONFIG.distro_fail_rates._replace(unacceptable=0.1)) - - initial_tags = collections.OrderedDict([ - ("jstests/core/all.js", [ - "unreliable", - "unreliable|jsCore_WT", - "unreliable|jsCore_WT|linux-64", - "unreliable|jsCore_WT|linux-64|rhel62", - ]), - ]) - - lifecycle = ci_tags.TagsConfig.from_dict( - dict(selector=dict(js_test=copy.deepcopy(initial_tags)))) - summary_lifecycle = update_test_lifecycle.TagsConfigWithChangelog(lifecycle) - self.assertEqual(initial_tags, self.assert_has_only_js_tests(lifecycle)) - - tests = ["jstests/core/all.js"] - report = test_failures.Report([ - self.ENTRY._replace(num_pass=0, num_fail=1), - self.ENTRY._replace(num_pass=0, num_fail=1, task="jsCore"), - self.ENTRY._replace(num_pass=0, num_fail=1, variant="linux-64-debug"), - self.ENTRY._replace(num_pass=1, num_fail=0), - self.ENTRY._replace(num_pass=0, num_fail=1, distro="rhel55"), - ]) - - update_test_lifecycle.validate_config(config) - update_test_lifecycle.update_tags(summary_lifecycle, config, report, tests) - updated_tags = self.assert_has_only_js_tests(lifecycle) - self.assertEqual(updated_tags, initial_tags) - - def test_obeys_reliable_min_runs(self): - """ - Tests that update_tags() considers a test reliable if it has fewer than 'reliable_min_runs'. - """ - - config = self.CONFIG._replace( - test_fail_rates=self.CONFIG.test_fail_rates._replace(acceptable=0.9), - task_fail_rates=self.CONFIG.task_fail_rates._replace(acceptable=0.9), - variant_fail_rates=self.CONFIG.variant_fail_rates._replace(acceptable=0.9), - distro_fail_rates=self.CONFIG.distro_fail_rates._replace(acceptable=0.9), - reliable_min_runs=100) - - self.transition_from_unreliable_to_reliable(config, - collections.OrderedDict([ - ("jstests/core/all.js", [ - "unreliable", - "unreliable|jsCore_WT", - "unreliable|jsCore_WT|linux-64", - "unreliable|jsCore_WT|linux-64|rhel62", - ]), - ])) - - def test_obeys_reliable_time_period(self): - """ - Tests that update_tags() ignores passes from before 'reliable_time_period'. - """ - - config = self.CONFIG._replace( - test_fail_rates=self.CONFIG.test_fail_rates._replace(acceptable=0.9), - task_fail_rates=self.CONFIG.task_fail_rates._replace(acceptable=0.9), - variant_fail_rates=self.CONFIG.variant_fail_rates._replace(acceptable=0.9), - distro_fail_rates=self.CONFIG.distro_fail_rates._replace(acceptable=0.9)) - - initial_tags = collections.OrderedDict() - lifecycle = ci_tags.TagsConfig.from_dict( - dict(selector=dict(js_test=copy.deepcopy(initial_tags)))) - summary_lifecycle = update_test_lifecycle.TagsConfigWithChangelog(lifecycle) - self.assertEqual(initial_tags, self.assert_has_only_js_tests(lifecycle)) - - tests = ["jstests/core/all.js"] - report = test_failures.Report([ - self.ENTRY._replace( - start_date=(self.ENTRY.start_date - datetime.timedelta(days=1)), - end_date=(self.ENTRY.end_date - datetime.timedelta(days=1)), num_pass=1, - num_fail=0), - self.ENTRY._replace( - start_date=(self.ENTRY.start_date - datetime.timedelta(days=2)), - end_date=(self.ENTRY.end_date - datetime.timedelta(days=2)), num_pass=1, - num_fail=0), - self.ENTRY._replace(num_pass=0, num_fail=1), - self.ENTRY._replace(num_pass=0, num_fail=1), - self.ENTRY._replace(num_pass=0, num_fail=1, task="jsCore"), - self.ENTRY._replace(num_pass=0, num_fail=1, variant="linux-64-debug"), - self.ENTRY._replace(num_pass=0, num_fail=1, distro="rhel55"), - ]) - - update_test_lifecycle.validate_config(config) - update_test_lifecycle.update_tags(summary_lifecycle, config, report, tests) - updated_tags = self.assert_has_only_js_tests(lifecycle) - self.assertEqual(updated_tags, - collections.OrderedDict([ - ("jstests/core/all.js", [ - "unreliable", - "unreliable|jsCore_WT", - "unreliable|jsCore_WT|linux-64", - "unreliable|jsCore_WT|linux-64|rhel62", - ]), - ])) - - def test_obeys_unreliable_min_runs(self): - """ - Tests that update_tags() only considers a test unreliable if it has more than - 'unreliable_min_runs'. - """ - - config = self.CONFIG._replace( - test_fail_rates=self.CONFIG.test_fail_rates._replace(unacceptable=0.1), - task_fail_rates=self.CONFIG.task_fail_rates._replace(unacceptable=0.1), - variant_fail_rates=self.CONFIG.variant_fail_rates._replace(unacceptable=0.1), - distro_fail_rates=self.CONFIG.distro_fail_rates._replace(unacceptable=0.1), - unreliable_min_runs=100) - - initial_tags = collections.OrderedDict() - lifecycle = ci_tags.TagsConfig.from_dict( - dict(selector=dict(js_test=copy.deepcopy(initial_tags)))) - summary_lifecycle = update_test_lifecycle.TagsConfigWithChangelog(lifecycle) - self.assertEqual(initial_tags, self.assert_has_only_js_tests(lifecycle)) - - tests = ["jstests/core/all.js"] - report = test_failures.Report([ - self.ENTRY._replace(num_pass=0, num_fail=1), - self.ENTRY._replace(num_pass=0, num_fail=1, task="jsCore"), - self.ENTRY._replace(num_pass=0, num_fail=1, variant="linux-64-debug"), - self.ENTRY._replace(num_pass=1, num_fail=0), - self.ENTRY._replace(num_pass=0, num_fail=1, distro="rhel55"), - ]) - - update_test_lifecycle.validate_config(config) - update_test_lifecycle.update_tags(summary_lifecycle, config, report, tests) - updated_tags = self.assert_has_only_js_tests(lifecycle) - self.assertEqual(updated_tags, initial_tags) - - def test_obeys_unreliable_time_period(self): - """ - Tests that update_tags() ignores failures from before 'unreliable_time_period'. - """ - - config = self.CONFIG._replace( - test_fail_rates=self.CONFIG.test_fail_rates._replace(unacceptable=0.1), - task_fail_rates=self.CONFIG.task_fail_rates._replace(unacceptable=0.1), - variant_fail_rates=self.CONFIG.variant_fail_rates._replace(unacceptable=0.1), - distro_fail_rates=self.CONFIG.distro_fail_rates._replace(unacceptable=0.1)) - - initial_tags = collections.OrderedDict([ - ("jstests/core/all.js", [ - "unreliable", - "unreliable|jsCore_WT", - "unreliable|jsCore_WT|linux-64", - "unreliable|jsCore_WT|linux-64|rhel62", - ]), - ]) - - lifecycle = ci_tags.TagsConfig.from_dict( - dict(selector=dict(js_test=copy.deepcopy(initial_tags)))) - summary_lifecycle = update_test_lifecycle.TagsConfigWithChangelog(lifecycle) - self.assertEqual(initial_tags, self.assert_has_only_js_tests(lifecycle)) - - tests = ["jstests/core/all.js"] - report = test_failures.Report([ - self.ENTRY._replace( - start_date=(self.ENTRY.start_date - datetime.timedelta(days=1)), - end_date=(self.ENTRY.end_date - datetime.timedelta(days=1)), num_pass=0, - num_fail=1), - self.ENTRY._replace( - start_date=(self.ENTRY.start_date - datetime.timedelta(days=2)), - end_date=(self.ENTRY.end_date - datetime.timedelta(days=2)), num_pass=0, - num_fail=1), - self.ENTRY._replace(num_pass=1, num_fail=0), - self.ENTRY._replace(num_pass=1, num_fail=0), - self.ENTRY._replace(num_pass=1, num_fail=0, task="jsCore"), - self.ENTRY._replace(num_pass=1, num_fail=0, variant="linux-64-debug"), - self.ENTRY._replace(num_pass=1, num_fail=0, distro="rhel55"), - ]) - - update_test_lifecycle.validate_config(config) - update_test_lifecycle.update_tags(summary_lifecycle, config, report, tests) - updated_tags = self.assert_has_only_js_tests(lifecycle) - self.assertEqual(updated_tags, collections.OrderedDict()) - - -class TestCombinationHelpers(unittest.TestCase): - def test_from_entry(self): - entry = test_failures._ReportEntry("testA", "taskA", "variantA", "distroA", - datetime.date.today(), datetime.date.today(), 0, 0) - combination = update_test_lifecycle._test_combination_from_entry( - entry, test_failures.Report.TEST) - self.assertEqual(combination, ("testA", )) - - combination = update_test_lifecycle._test_combination_from_entry( - entry, test_failures.Report.TEST_TASK) - self.assertEqual(combination, ("testA", "taskA")) - - combination = update_test_lifecycle._test_combination_from_entry( - entry, test_failures.Report.TEST_TASK_VARIANT) - self.assertEqual(combination, ("testA", "taskA", "variantA")) - - combination = update_test_lifecycle._test_combination_from_entry( - entry, test_failures.Report.TEST_TASK_VARIANT_DISTRO) - self.assertEqual(combination, ("testA", "taskA", "variantA", "distroA")) - - def test_make_from_tag(self): - test = "testA" - - combination = update_test_lifecycle._test_combination_from_tag(test, "unreliable") - self.assertEqual(combination, ("testA", )) - - combination = update_test_lifecycle._test_combination_from_tag(test, "unreliable|taskA") - self.assertEqual(combination, ("testA", "taskA")) - - combination = update_test_lifecycle._test_combination_from_tag( - test, "unreliable|taskA|variantA") - self.assertEqual(combination, ("testA", "taskA", "variantA")) - - combination = update_test_lifecycle._test_combination_from_tag( - test, "unreliable|taskA|variantA|distroA") - self.assertEqual(combination, ("testA", "taskA", "variantA", "distroA")) - - -class TestCleanUpTags(unittest.TestCase): - @classmethod - def setUpClass(cls): - cls.evg = MockEvergreenConfig( - ["task1", "task2", "task3"], { - "variant1": {"tasks": ["task1", "task2"], "distros": ["distro1"]}, - "variant2": {"tasks": ["task3"], "distros": ["distro2"]} - }) - - def test_is_unreliable_tag_relevant(self): - self.assertTrue(update_test_lifecycle._is_tag_still_relevant(self.evg, "unreliable")) - - def test_is_unknown_task_relevant(self): - self.assertFalse( - update_test_lifecycle._is_tag_still_relevant(self.evg, "unreliable|task_unknown")) - - def test_is_known_task_relevant(self): - self.assertTrue(update_test_lifecycle._is_tag_still_relevant(self.evg, "unreliable|task1")) - self.assertTrue(update_test_lifecycle._is_tag_still_relevant(self.evg, "unreliable|task2")) - self.assertTrue(update_test_lifecycle._is_tag_still_relevant(self.evg, "unreliable|task3")) - - def test_is_unknown_variant_relevant(self): - self.assertFalse( - update_test_lifecycle._is_tag_still_relevant(self.evg, "unreliable|task1|variant3")) - - def test_is_unknown_task_variant_relevant(self): - self.assertFalse( - update_test_lifecycle._is_tag_still_relevant(self.evg, "unreliable|task3|variant1")) - self.assertFalse( - update_test_lifecycle._is_tag_still_relevant(self.evg, "unreliable|task1|variant2")) - - def test_is_known_task_variant_relevant(self): - self.assertTrue( - update_test_lifecycle._is_tag_still_relevant(self.evg, "unreliable|task1|variant1")) - self.assertTrue( - update_test_lifecycle._is_tag_still_relevant(self.evg, "unreliable|task2|variant1")) - self.assertTrue( - update_test_lifecycle._is_tag_still_relevant(self.evg, "unreliable|task3|variant2")) - - def test_is_unknown_task_variant_distro_relevant(self): - self.assertFalse( - update_test_lifecycle._is_tag_still_relevant(self.evg, - "unreliable|task1|variant1|distro2")) - self.assertFalse( - update_test_lifecycle._is_tag_still_relevant(self.evg, - "unreliable|task3|variant2|distro1")) - - def test_is_known_task_variant_distro_relevant(self): - self.assertTrue( - update_test_lifecycle._is_tag_still_relevant(self.evg, - "unreliable|task1|variant1|distro1")) - self.assertTrue( - update_test_lifecycle._is_tag_still_relevant(self.evg, - "unreliable|task3|variant2|distro2")) - - -class MockEvergreenConfig(object): - def __init__(self, tasks, variants): - self.task_names = tasks - self.variants = {} - for name, fields in variants.items(): - self.variants[name] = MockVariant(fields["tasks"], fields["distros"]) - - def get_variant(self, variant_name): - return self.variants.get(variant_name) - - -class MockVariant(object): - def __init__(self, task_names, distros): - self.task_names = task_names - self.distros = distros - - -class TestJiraIssueCreator(unittest.TestCase): - def test_description(self): - data = { - "js_test": { - "testfile1": {"tag1": 0.1, "tag2": 0.2}, "testfile2": {"tag1": 0.1, "tag3": 0.3} - } - } - desc = update_test_lifecycle.JiraIssueCreator._make_updated_tags_description(data) - expected = ("- *js_test*\n" - "-- {{testfile1}}\n" - "--- {{tag1}} (0.10)\n" - "--- {{tag2}} (0.20)\n" - "-- {{testfile2}}\n" - "--- {{tag1}} (0.10)\n" - "--- {{tag3}} (0.30)") - self.assertEqual(expected, desc) - - def test_description_empty(self): - data = {} - desc = update_test_lifecycle.JiraIssueCreator._make_updated_tags_description(data) - expected = "_None_" - self.assertEqual(expected, desc) - - def test_clean_up_description(self): - data = {"js_test": {"testfile1": ["tag1", "tag2"], "testfile2": []}} - desc = update_test_lifecycle.JiraIssueCreator._make_tags_cleaned_up_description(data) - expected = ("- *js_test*\n" - "-- {{testfile1}}\n" - "--- {{tag1}}\n" - "--- {{tag2}}\n" - "-- {{testfile2}}\n" - "--- ALL (test file removed or renamed as part of an earlier commit)") - self.assertEqual(expected, desc) - - def test_clean_up_description_empty(self): - data = {} - desc = update_test_lifecycle.JiraIssueCreator._make_tags_cleaned_up_description(data) - expected = "_None_" - self.assertEqual(expected, desc) - - def test_truncate_description(self): - desc = "a" * (update_test_lifecycle.JiraIssueCreator._MAX_DESCRIPTION_SIZE - 1) - self.assertTrue(desc == update_test_lifecycle.JiraIssueCreator._truncate_description(desc)) - - desc += "a" - self.assertTrue(desc == update_test_lifecycle.JiraIssueCreator._truncate_description(desc)) - - desc += "a" - self.assertTrue( - len(update_test_lifecycle.JiraIssueCreator._truncate_description(desc)) <= - update_test_lifecycle.JiraIssueCreator._MAX_DESCRIPTION_SIZE) - - -class TestTagsConfigWithChangelog(unittest.TestCase): - def setUp(self): - lifecycle = ci_tags.TagsConfig({"selector": {}}) - self.summary_lifecycle = update_test_lifecycle.TagsConfigWithChangelog(lifecycle) - - def test_add_tag(self): - self.summary_lifecycle.add_tag("js_test", "testfile1", "tag1", 0.1) - self.assertEqual({"js_test": {"testfile1": {"tag1": 0.1}}}, self.summary_lifecycle.added) - - def test_remove_tag(self): - self.summary_lifecycle.lifecycle.add_tag("js_test", "testfile1", "tag1") - self.summary_lifecycle.remove_tag("js_test", "testfile1", "tag1", 0.1) - self.assertEqual({"js_test": {"testfile1": {"tag1": 0.1}}}, self.summary_lifecycle.removed) - - def test_add_remove_tag(self): - self.summary_lifecycle.add_tag("js_test", "testfile1", "tag1", 0.1) - self.summary_lifecycle.remove_tag("js_test", "testfile1", "tag1", 0.4) - self.assertEqual({}, self.summary_lifecycle.added) - self.assertEqual({}, self.summary_lifecycle.removed) - - def test_remove_add_tag(self): - self.summary_lifecycle.lifecycle.add_tag("js_test", "testfile1", "tag1") - self.summary_lifecycle.remove_tag("js_test", "testfile1", "tag1", 0.1) - self.summary_lifecycle.add_tag("js_test", "testfile1", "tag1", 0.1) - self.assertEqual({}, self.summary_lifecycle.added) - self.assertEqual({}, self.summary_lifecycle.removed) diff --git a/buildscripts/update_test_lifecycle.py b/buildscripts/update_test_lifecycle.py deleted file mode 100755 index bab5bcc91be..00000000000 --- a/buildscripts/update_test_lifecycle.py +++ /dev/null @@ -1,1110 +0,0 @@ -#!/usr/bin/env python -"""Test Failures module. - -Update etc/test_lifecycle.yml to tag unreliable tests based on historic failure rates. -""" - -from __future__ import absolute_import -from __future__ import division - -import collections -import datetime -import logging -import multiprocessing.dummy -import operator -import optparse -import os.path -import posixpath -import subprocess -import sys -import textwrap - -import yaml - -# Get relative imports to work when the package is not installed on the PYTHONPATH. -if __name__ == "__main__" and __package__ is None: - sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - -# pylint: disable=wrong-import-position -from buildscripts import git -from buildscripts import jiraclient -from buildscripts import resmokelib -from buildscripts.resmokelib import utils -from buildscripts.resmokelib.utils import globstar -from buildscripts import lifecycle_test_failures as tf -from buildscripts.ciconfig import evergreen as ci_evergreen -from buildscripts.ciconfig import tags as ci_tags -# pylint: enable=wrong-import-position - -# pylint: disable=too-many-lines - -LOGGER = logging.getLogger(__name__) - -if sys.version_info[0] == 2: - _NUMBER_TYPES = (int, long, float) -else: - _NUMBER_TYPES = (int, float) - -Rates = collections.namedtuple("Rates", ["acceptable", "unacceptable"]) - -Config = collections.namedtuple("Config", [ - "test_fail_rates", - "task_fail_rates", - "variant_fail_rates", - "distro_fail_rates", - "reliable_min_runs", - "reliable_time_period", - "unreliable_min_runs", - "unreliable_time_period", -]) - -DEFAULT_CONFIG = Config( - test_fail_rates=Rates(acceptable=0.1, unacceptable=0.3), task_fail_rates=Rates( - acceptable=0.1, unacceptable=0.3), - variant_fail_rates=Rates(acceptable=0.2, unacceptable=0.4), distro_fail_rates=Rates( - acceptable=0.2, - unacceptable=0.4), reliable_min_runs=5, reliable_time_period=datetime.timedelta(weeks=1), - unreliable_min_runs=20, unreliable_time_period=datetime.timedelta(weeks=4)) - -DEFAULT_PROJECT = "mongodb-mongo-master" - -DEFAULT_NUM_THREADS = 12 - -MAX_BATCH_SIZE = 50 - - -def get_suite_tasks_membership(evg_conf): - """Return a dictionary with keys of all suites and list of associated tasks.""" - suite_membership = collections.defaultdict(list) - for task in evg_conf.tasks: - suite = task.resmoke_suite - if suite: - suite_membership[suite].append(task.name) - return suite_membership - - -def get_test_tasks_membership(evg_conf): - """Return a dictionary with keys of all tests and list of associated tasks.""" - test_suites_membership = resmokelib.suitesconfig.create_test_membership_map(test_kind="js_test") - suite_tasks_membership = get_suite_tasks_membership(evg_conf) - test_tasks_membership = collections.defaultdict(list) - for test in test_suites_membership.keys(): - for suite in test_suites_membership[test]: - test_tasks_membership[test].extend(suite_tasks_membership[suite]) - return test_tasks_membership - - -def get_tests_from_tasks(tasks, test_tasks_membership): - """Return a list of tests from list of specified tasks.""" - tests = [] - tasks_set = set(tasks) - for test in test_tasks_membership.keys(): - if not tasks_set.isdisjoint(test_tasks_membership[test]): - tests.append(test) - return tests - - -def create_test_groups(tests): - """Return groups of tests by their directory, i.e., jstests/core.""" - test_groups = collections.defaultdict(list) - for test in tests: - test_split = test.split("/") - # If the test does not have a directory, then ignore it. - if len(test_split) <= 1: - continue - test_dir = test_split[1] - test_groups[test_dir].append(test) - return test_groups - - -def create_batch_groups(test_groups, batch_size): - """Return batch groups list of test_groups.""" - batch_groups = [] - for test_group_name in test_groups: - test_group = test_groups[test_group_name] - while test_group: - batch_groups.append(test_group[:batch_size]) - test_group = test_group[batch_size:] - return batch_groups - - -def get_date_range(num_days): - """Return a tuple of start_date, end_date, formatted as YYYY-MM-DD. - - The end_date is tomorrow as the test_history API excludes the before_date in the results. - The start_date is 'num_days' from tomorrow's date. - """ - tomorrow_dt = datetime.date.today() + datetime.timedelta(days=1) - start_dt = tomorrow_dt - datetime.timedelta(days=num_days) - return start_dt.strftime("%Y-%m-%d"), tomorrow_dt.strftime("%Y-%m-%d") - - -class TestHistorySource(object): - """A class used to parallelize requests to buildscripts.test_failures.TestHistory.""" - - def __init__( # pylint: disable=too-many-arguments - self, project, variants, distros, start_date, end_date, - thread_pool_size=DEFAULT_NUM_THREADS): - """Initialize the TestHistorySource. - - Args: - project: the Evergreen project name. - variants: a list of variant names. - distros: a list of distro names. - start_date: the start date of the history we want to retrieve. - end_revision: the end date of the history we want to retrieve. - thread_pool_size: the size of the thread pool used to make parallel requests. - """ - self._project = project - self._variants = variants - self._distros = distros - self._start_date = start_date - self._end_date = end_date - self._thread_pool = multiprocessing.dummy.Pool(thread_pool_size) - - def get_history_data(self, tests, tasks): - """Retrieve the history data for the given tests and tasks. - - The requests for each task will be parallelized using the internal thread pool. - """ - history_data = [] - jobs = [ - self._thread_pool.apply_async(self._get_task_history_data, (tests, task)) - for task in tasks - ] - for job in jobs: - history_data.extend(job.get()) - return history_data - - def _get_task_history_data(self, tests, task): - test_history = tf.TestHistory(project=self._project, tests=tests, tasks=[task], - variants=self._variants, distros=self._distros) - return test_history.get_history_by_date(start_date=self._start_date, - end_date=self._end_date) - - -def unreliable_test(test_fr, unacceptable_fr, test_runs, min_run): - """Check for an unreliable test. - - A test should be added to the set of tests believed not to run reliably when it has more - than min_run executions with a failure percentage greater than unacceptable_fr. - """ - return test_runs >= min_run and test_fr >= unacceptable_fr - - -def reliable_test(test_fr, acceptable_fr, test_runs, min_run): - """Check for a reliable test. - - A test should then removed from the set of tests believed not to run reliably when it has - less than min_run executions or has a failure percentage less than acceptable_fr. - """ - return test_runs < min_run or test_fr <= acceptable_fr - - -def unreliable_tag(task, variant, distro): - """Return the unreliable tag.""" - - for (component_name, component_value) in (("task", task), ("variant", variant), ("distro", - distro)): - if isinstance(component_value, (tf.Wildcard, tf.Missing)): - if component_name == "task": - return "unreliable" - elif component_name == "variant": - return "unreliable|{}".format(task) - elif component_name == "distro": - return "unreliable|{}|{}".format(task, variant) - - return "unreliable|{}|{}|{}".format(task, variant, distro) - - -def update_lifecycle( # pylint: disable=too-many-arguments - lifecycle_tags_file, report, method_test, add_tags, fail_rate, min_run): - """Update the lifecycle object based on the test_method. - - The test_method checks unreliable or reliable fail_rates. - """ - for summary in report: - if method_test(summary.fail_rate, fail_rate, summary.num_pass + summary.num_fail, min_run): - update_tag = unreliable_tag(summary.task, summary.variant, summary.distro) - if add_tags: - lifecycle_tags_file.add_tag("js_test", summary.test, update_tag, summary.fail_rate) - else: - lifecycle_tags_file.remove_tag("js_test", summary.test, update_tag, - summary.fail_rate) - - -def compare_tags(tag_a, tag_b): - """Return 1, -1 or 0 if 'tag_a' is superior, inferior or equal to 'tag_b'.""" - return cmp(tag_a.split("|"), tag_b.split("|")) - - -def validate_config(config): # pylint: disable=too-many-branches - """Raise a TypeError or ValueError exception if 'config' isn't a valid model.""" - - for (name, fail_rates) in (("test", config.test_fail_rates), ("task", config.task_fail_rates), - ("variant", config.variant_fail_rates), ("distro", - config.distro_fail_rates)): - if not isinstance(fail_rates.acceptable, _NUMBER_TYPES): - raise TypeError("The acceptable {} failure rate must be a number, but got {}".format( - name, fail_rates.acceptable)) - elif fail_rates.acceptable < 0 or fail_rates.acceptable > 1: - raise ValueError(("The acceptable {} failure rate must be between 0 and 1 (inclusive)," - " but got {}").format(name, fail_rates.acceptable)) - elif not isinstance(fail_rates.unacceptable, _NUMBER_TYPES): - raise TypeError("The unacceptable {} failure rate must be a number, but got {}".format( - name, fail_rates.unacceptable)) - elif fail_rates.unacceptable < 0 or fail_rates.unacceptable > 1: - raise ValueError(("The unacceptable {} failure rate must be between 0 and 1" - " (inclusive), but got {}").format(name, fail_rates.unacceptable)) - elif fail_rates.acceptable > fail_rates.unacceptable: - raise ValueError( - ("The acceptable {0} failure rate ({1}) must be no larger than unacceptable {0}" - " failure rate ({2})").format(name, fail_rates.acceptable, - fail_rates.unacceptable)) - - for (name, min_runs) in (("reliable", config.reliable_min_runs), ("unreliable", - config.unreliable_min_runs)): - if not isinstance(min_runs, _NUMBER_TYPES): - raise TypeError(("The minimum number of runs for considering a test {} must be a" - " number, but got {}").format(name, min_runs)) - elif min_runs <= 0: - raise ValueError(("The minimum number of runs for considering a test {} must be a" - " positive integer, but got {}").format(name, min_runs)) - elif isinstance(min_runs, float) and not min_runs.is_integer(): - raise ValueError(("The minimum number of runs for considering a test {} must be an" - " integer, but got {}").format(name, min_runs)) - - for (name, time_period) in (("reliable", config.reliable_time_period), - ("unreliable", config.unreliable_time_period)): - if not isinstance(time_period, datetime.timedelta): - raise TypeError( - "The {} time period must be a datetime.timedelta instance, but got {}".format( - name, time_period)) - elif time_period.days <= 0: - raise ValueError( - "The {} time period must be a positive number of days, but got {}".format( - name, time_period)) - elif time_period - datetime.timedelta(days=time_period.days) > datetime.timedelta(): - raise ValueError( - "The {} time period must be an integral number of days, but got {}".format( - name, time_period)) - - -def _test_combination_from_entry(entry, components): - """Create a test combination tuple from a tf._ReportEntry and target components. - - Return a tuple containing the entry fields specified in components. - """ - combination = [] - for component in components: - combination.append(operator.attrgetter(component)(entry)) - return tuple(combination) - - -def _test_combination_from_tag(test, tag): - """Create a test combination tuple from a test name and a tag. - - Return a tuple containing the test name and the components found in the tag. - """ - combination = [test] - for element in _split_tag(tag): - if element: - combination.append(element) - return tuple(combination) - - -def update_tags(lifecycle_tags, config, report, tests): # pylint: disable=too-many-locals - """Update the tags in 'lifecycle_tags'. - - This is based on the historical test failures of tests 'tests' - mentioned in 'report' according to the model described by 'config'. - """ - - # We initialize 'grouped_entries' to make PyLint not complain about 'grouped_entries' being used - # before assignment. - grouped_entries = None - # yapf: disable - for (idx, (components, rates)) in enumerate( - ((tf.Report.TEST_TASK_VARIANT_DISTRO, config.distro_fail_rates), - (tf.Report.TEST_TASK_VARIANT, config.variant_fail_rates), - (tf.Report.TEST_TASK, config.task_fail_rates), - (tf.Report.TEST, config.test_fail_rates))): - # yapf: enable - if idx > 0: - report = tf.Report(grouped_entries) - - # We reassign the value of 'grouped_entries' to take advantage of how data that is on - # (test, task, variant, distro) preserves enough information to be grouped on any subset of - # those components, etc. - grouped_entries = report.summarize_by(components, time_period=tf.Report.DAILY) - - # Create the reliable report. - # Filter out any test executions from prior to 'config.reliable_time_period'. - reliable_start_date = ( - report.end_date - config.reliable_time_period + datetime.timedelta(days=1)) - reliable_entries = [ - entry for entry in grouped_entries if entry.start_date >= reliable_start_date - ] - if reliable_entries: - reliable_report = tf.Report(reliable_entries) - reliable_combinations = { - _test_combination_from_entry(entry, components) - for entry in reliable_entries - } - reliable_summaries = reliable_report.summarize_by(components) - else: - reliable_combinations = set() - reliable_summaries = [] - - # Create the unreliable report. - # Filter out any test executions from prior to 'config.unreliable_time_period'. - # Also filter out any test that is not present in the reliable_report in order - # to avoid tagging as unreliable tests that are no longer running. - unreliable_start_date = ( - report.end_date - config.unreliable_time_period + datetime.timedelta(days=1)) - unreliable_entries = [ - entry for entry in grouped_entries - if (entry.start_date >= unreliable_start_date - and _test_combination_from_entry(entry, components) in reliable_combinations) - ] - if unreliable_entries: - unreliable_report = tf.Report(unreliable_entries) - unreliable_summaries = unreliable_report.summarize_by(components) - else: - unreliable_summaries = [] - - # Update the tags using the unreliable report. - update_lifecycle(lifecycle_tags, unreliable_summaries, unreliable_test, True, - rates.unacceptable, config.unreliable_min_runs) - - # Update the tags using the reliable report. - update_lifecycle(lifecycle_tags, reliable_summaries, reliable_test, False, rates.acceptable, - config.reliable_min_runs) - - def should_be_removed(test, tag, components, reliable_combinations): - """Return True if 'combination' shoud be removed.""" - combination = _test_combination_from_tag(test, tag) - if len(combination) != len(components): - # The tag is not for these components. - return False - return combination not in reliable_combinations - - # Remove the tags that correspond to tests that have not run during the reliable period. - for test in tests: - tags = lifecycle_tags.lifecycle.get_tags("js_test", test) - for tag in tags[:]: - if should_be_removed(test, tag, components, reliable_combinations): - LOGGER.info("Removing tag '%s' of test '%s' because the combination did not run" - " during the reliable period", tag, test) - lifecycle_tags.remove_tag("js_test", test, tag, failure_rate=0) - - -def _split_tag(tag): - """Split a tag into its components. - - Return a tuple containing task, variant, distro. The values are None if absent from the tag. - If the tag is invalid, the return value is (None, None, None). - """ - elements = tag.split("|") - length = len(elements) - if elements[0] != "unreliable" or length < 2 or length > 4: - return None, None, None - # fillout the array - elements.extend([None] * (4 - length)) - # return as a tuple - return tuple(elements[1:]) - - -def _is_tag_still_relevant(evg_conf, tag): - """Indicate if a tag still corresponds to a valid task/variant/distro combination.""" - if tag == "unreliable": - return True - task, variant, distro = _split_tag(tag) - if not task or task not in evg_conf.task_names: - return False - if variant: - variant_conf = evg_conf.get_variant(variant) - if not variant_conf or task not in variant_conf.task_names: - return False - if distro and distro not in variant_conf.distro_names: - return False - return True - - -def clean_up_tags(lifecycle_tags, evg_conf): - """Remove the tags that do not correspond to a valid test/task/variant/distro combination.""" - lifecycle = lifecycle_tags.lifecycle - for test_kind in lifecycle.get_test_kinds(): - for test_pattern in lifecycle.get_test_patterns(test_kind): - if not globstar.glob(test_pattern): - # The pattern does not match any file in the repository. - lifecycle_tags.clean_up_test(test_kind, test_pattern) - continue - for tag in lifecycle.get_tags(test_kind, test_pattern): - if not _is_tag_still_relevant(evg_conf, tag): - lifecycle_tags.clean_up_tag(test_kind, test_pattern, tag) - - -def _config_as_options(config): - return ("--reliableTestMinRuns {} " - "--reliableDays {} " - "--unreliableTestMinRuns {} " - "--unreliableDays {} " - "--testFailRates {} {} " - "--taskFailRates {} {} " - "--variantFailRates {} {} " - "--distroFailRates {} {}").format( - config.reliable_min_runs, config.reliable_time_period.days, - config.unreliable_min_runs, config.unreliable_time_period.days, - config.test_fail_rates.acceptable, config.test_fail_rates.unacceptable, - config.task_fail_rates.acceptable, config.task_fail_rates.unacceptable, - config.variant_fail_rates.acceptable, config.variant_fail_rates.unacceptable, - config.distro_fail_rates.acceptable, config.distro_fail_rates.unacceptable) - - -class TagsConfigWithChangelog(object): - """A wrapper around TagsConfig to update a tags file and record the modifications made.""" - - def __init__(self, lifecycle): - """Initialize the TagsConfigWithChangelog with the lifecycle TagsConfig.""" - self.lifecycle = lifecycle - self.added = {} - self.removed = {} - self.cleaned_up = {} - - @staticmethod - def _cancel_tag_log(log_dict, test_kind, test, tag): - """Remove a tag from a changelog dictionary. - - Used to remove a tag from the 'added' or 'removed' attribute. - """ - kind_dict = log_dict[test_kind] - test_dict = kind_dict[test] - del test_dict[tag] - if not test_dict: - del kind_dict[test] - if not kind_dict: - del log_dict[test_kind] - - def add_tag(self, test_kind, test, tag, failure_rate): - """Add a tag.""" - if self.lifecycle.add_tag(test_kind, test, tag): - if tag in self.removed.get(test_kind, {}).get(test, {}): - # The tag has just been removed. - self._cancel_tag_log(self.removed, test_kind, test, tag) - else: - self.added.setdefault(test_kind, {}).setdefault(test, {})[tag] = failure_rate - - def remove_tag(self, test_kind, test, tag, failure_rate): - """Remove a tag.""" - if self.lifecycle.remove_tag(test_kind, test, tag): - if tag in self.added.get(test_kind, {}).get(test, {}): - # The tag has just been added. - self._cancel_tag_log(self.added, test_kind, test, tag) - else: - self.removed.setdefault(test_kind, {}).setdefault(test, {})[tag] = failure_rate - - def clean_up_tag(self, test_kind, test, tag): - """Clean up an invalid tag.""" - self.lifecycle.remove_tag(test_kind, test, tag) - self.cleaned_up.setdefault(test_kind, {}).setdefault(test, []).append(tag) - - def clean_up_test(self, test_kind, test): - """Clean up an invalid test.""" - self.lifecycle.remove_test_pattern(test_kind, test) - self.cleaned_up.setdefault(test_kind, {})[test] = [] - - -class JiraIssueCreator(object): - """JiraIssueCreator class.""" - - _LABEL = "test-lifecycle" - _PROJECT = "TIGBOT" - _MAX_DESCRIPTION_SIZE = 32767 - - def __init__( # pylint: disable=too-many-arguments - self, server=None, username=None, password=None, access_token=None, - access_token_secret=None, consumer_key=None, key_cert=None): - """Initialize JiraIssueCreator.""" - self._client = jiraclient.JiraClient( - server=server, username=username, password=password, access_token=access_token, - access_token_secret=access_token_secret, consumer_key=consumer_key, key_cert=key_cert) - - def create_issue( # pylint: disable=too-many-arguments - self, evg_project, mongo_revision, model_config, added, removed, cleaned_up): - """Create a JIRA issue for the test lifecycle tag update.""" - summary = self._get_jira_summary(evg_project) - description = self._get_jira_description(evg_project, mongo_revision, model_config, added, - removed, cleaned_up) - issue_key = self._client.create_issue(self._PROJECT, summary, description, [self._LABEL]) - return issue_key - - def close_fix_issue(self, issue_key): - """Close the issue with the "Fixed" resolution.""" - LOGGER.info("Closing issue '%s' as FIXED.", issue_key) - self._client.close_issue(issue_key, self._client.FIXED_RESOLUTION_NAME) - - def close_wontfix_issue(self, issue_key): - """Close the issue the with "Won't Fix" resolution.""" - LOGGER.info("Closing issue '%s' as WON'T FIX.", issue_key) - self._client.close_issue(issue_key, self._client.WONT_FIX_RESOLUTION_NAME) - - @staticmethod - def _get_jira_summary(project): - return "Update of test lifecycle tags for {}".format(project) - - @staticmethod - def _monospace(text): - """Transform a text into a monospace JIRA text.""" - return "{{" + text + "}}" - - @staticmethod - def _truncate_description(desc): - max_size = JiraIssueCreator._MAX_DESCRIPTION_SIZE - if len(desc) > max_size: - warning = ("\nDescription truncated: " - "exceeded max size of {} characters.").format(max_size) - truncated_length = max_size - len(warning) - desc = desc[:truncated_length] + warning - return desc - - @staticmethod - def _get_jira_description( # pylint: disable=too-many-arguments - project, mongo_revision, model_config, added, removed, cleaned_up): - mono = JiraIssueCreator._monospace - config_desc = _config_as_options(model_config) - added_desc = JiraIssueCreator._make_updated_tags_description(added) - removed_desc = JiraIssueCreator._make_updated_tags_description(removed) - cleaned_up_desc = JiraIssueCreator._make_tags_cleaned_up_description(cleaned_up) - project_link = "[{0}|https://evergreen.mongodb.com/waterfall/{1}]".format( - mono(project), project) - revision_link = "[{0}|https://github.com/mongodb/mongo/commit/{1}]".format( - mono(mongo_revision), mongo_revision) - full_desc = ("h3. Automatic update of the test lifecycle tags\n" - "Evergreen Project: {0}\n" - "Revision: {1}\n\n" - "{{{{update_test_lifecycle.py}}}} options:\n{2}\n\n" - "h5. Tags added\n{3}\n\n" - "h5. Tags removed\n{4}\n\n" - "h5. Tags cleaned up (no longer relevant)\n{5}\n").format( - project_link, revision_link, mono(config_desc), added_desc, removed_desc, - cleaned_up_desc) - - return JiraIssueCreator._truncate_description(full_desc) - - @staticmethod - def _make_updated_tags_description(data): - mono = JiraIssueCreator._monospace - tags_lines = [] - for test_kind in sorted(data.keys()): - tests = data[test_kind] - tags_lines.append("- *{0}*".format(test_kind)) - for test in sorted(tests.keys()): - tags = tests[test] - tags_lines.append("-- {0}".format(mono(test))) - for tag in sorted(tags.keys()): - coefficient = tags[tag] - tags_lines.append("--- {0} ({1:.2f})".format(mono(tag), coefficient)) - if tags_lines: - return "\n".join(tags_lines) - return "_None_" - - @staticmethod - def _make_tags_cleaned_up_description(cleaned_up): - mono = JiraIssueCreator._monospace - tags_cleaned_up_lines = [] - for test_kind in sorted(cleaned_up.keys()): - test_tags = cleaned_up[test_kind] - tags_cleaned_up_lines.append("- *{0}*".format(test_kind)) - for test in sorted(test_tags.keys()): - tags = test_tags[test] - tags_cleaned_up_lines.append("-- {0}".format(mono(test))) - if not tags: - tags_cleaned_up_lines.append("--- ALL (test file removed or renamed as part of" - " an earlier commit)") - else: - for tag in sorted(tags): - tags_cleaned_up_lines.append("--- {0}".format(mono(tag))) - if tags_cleaned_up_lines: - return "\n".join(tags_cleaned_up_lines) - return "_None_" - - -class LifecycleTagsFile(object): # pylint: disable=too-many-instance-attributes - """Represent a test lifecycle tags file that can be written and committed.""" - - def __init__( # pylint: disable=too-many-arguments - self, project, lifecycle_file, metadata_repo_url=None, references_file=None, - jira_issue_creator=None, git_info=None, - model_config=None): # noqa: D214,D401,D405,D406,D407,D411,D413 - """Initalize the LifecycleTagsFile. - - Arguments: - project: The Evergreen project name, e.g. "mongodb-mongo-master". - lifecycle_file: The path to the lifecycle tags file. If 'metadata_repo_url' is - specified, this path must be relative to the root of the metadata repository. - metadata_repo_url: The URL of the metadat repository that contains the test lifecycle - tags file. - references_file: The path to the references file in the metadata repository. - jira_issue_creator: A JiraIssueCreator instance. - git_info: A tuple containing the git user's name and email to set before committing. - model_config: The model configuration as a Config instance. - """ - self.project = project - self.mongo_repo = git.Repository(os.getcwd()) - self.mongo_revision = self.mongo_repo.get_current_revision() - # The branch name is the same on both repositories. - self.mongo_branch = self.mongo_repo.get_branch_name() - self.metadata_branch = project - - if metadata_repo_url: - # The file can be found in another repository. We clone it. - self.metadata_repo = self._clone_repository(metadata_repo_url, self.project) - self.relative_lifecycle_file = lifecycle_file - self.lifecycle_file = os.path.join(self.metadata_repo.directory, lifecycle_file) - self.relative_references_file = references_file - self.references_file = os.path.join(self.metadata_repo.directory, references_file) - if git_info: - self.metadata_repo.configure("user.name", git_info[0]) - self.metadata_repo.configure("user.email", git_info[1]) - else: - self.metadata_repo = None - self.relative_lifecycle_file = lifecycle_file - self.lifecycle_file = lifecycle_file - self.relative_references_file = None - self.references_file = None - self.metadata_repo_url = metadata_repo_url - self.lifecycle = ci_tags.TagsConfig.from_file(self.lifecycle_file, cmp_func=compare_tags) - self.jira_issue_creator = jira_issue_creator - self.model_config = model_config - self.changelog_lifecycle = TagsConfigWithChangelog(self.lifecycle) - - @staticmethod - def _clone_repository(metadata_repo_url, branch): - directory_name = posixpath.splitext(posixpath.basename(metadata_repo_url))[0] - LOGGER.info("Cloning the repository %s into the directory %s", metadata_repo_url, - directory_name) - return git.Repository.clone(metadata_repo_url, directory_name, branch) - - def is_modified(self): - """Indicate if the tags have been modified.""" - return self.lifecycle.is_modified() - - def _create_issue(self): - LOGGER.info("Creating a JIRA issue") - issue_key = self.jira_issue_creator.create_issue( - self.project, self.mongo_revision, self.model_config, self.changelog_lifecycle.added, - self.changelog_lifecycle.removed, self.changelog_lifecycle.cleaned_up) - LOGGER.info("JIRA issue created: %s", issue_key) - return issue_key - - def write(self): - """Write the test lifecycle tag file.""" - LOGGER.info("Writing the tag file to '%s'", self.lifecycle_file) - comment = ("This file was generated by {} and shouldn't be edited by hand. It was" - " generated against commit {} with the following options: {}.").format( - sys.argv[0], self.mongo_repo.get_current_revision(), - _config_as_options(self.model_config)) - self.lifecycle.write_file(self.lifecycle_file, comment) - - def _ready_for_commit(self, ref_branch, references): - # Check that the test lifecycle tags file has changed. - diff = self.metadata_repo.git_diff( - ["--name-only", ref_branch, self.relative_lifecycle_file]) - if not diff: - LOGGER.info("The local lifecycle file is identical to the the one on branch '%s'", - ref_branch) - return False - # Check that the lifecycle file has not been updated after the current mongo revision. - update_revision = references.get("test-lifecycle", {}).get(self.project) - if update_revision and not self.mongo_repo.is_ancestor(update_revision, - self.mongo_revision): - LOGGER.warning(("The existing lifecycle file is based on revision '%s' which is not a" - " parent revision of the current revision '%s'"), update_revision, - self.mongo_revision) - return False - return True - - def _read_references(self, metadata_branch=None): - branch = metadata_branch if metadata_branch is not None else "" - references_content = self.metadata_repo.git_cat_file( - ["blob", "{0}:{1}".format(branch, self.relative_references_file)]) - return yaml.safe_load(references_content) - - def _update_and_write_references(self, references): - LOGGER.info("Writing the references file to '%s'", self.references_file) - references.setdefault("test-lifecycle", {})[self.project] = self.mongo_revision - with open(self.references_file, "w") as fstream: - yaml.safe_dump(references, fstream, default_flow_style=False) - - def _commit_locally(self, issue_key): - self.metadata_repo.git_add([self.relative_lifecycle_file]) - self.metadata_repo.git_add([self.relative_references_file]) - commit_message = "{} Update {}".format(issue_key, self.relative_lifecycle_file) - self.metadata_repo.commit_with_message(commit_message) - LOGGER.info("Change committed with message: %s", commit_message) - - def commit(self, nb_retries=10): - """Commit the test lifecycle tag file. - - Args: - nb_retries: the number of times the script will reset, fetch, recommit and retry when - the push fails. - """ - references = self._read_references() - # Verify we are ready to commit. - if not self._ready_for_commit(self.metadata_branch, references): - return True - - # Write the references file. - self._update_and_write_references(references) - - # Create the issue. - issue_key = self._create_issue() - - # Commit the change. - self._commit_locally(issue_key) - - # Push the change. - tries = 0 - pushed = False - upstream = "origin/{0}".format(self.metadata_branch) - while tries < nb_retries: - try: - self.metadata_repo.push_to_remote_branch("origin", self.metadata_branch) - pushed = True - break - except git.GitException: - LOGGER.warning("git push command failed, fetching and retrying.") - # Fetch upstream branch. - LOGGER.info("Fetching branch %s of %s", self.metadata_branch, - self.metadata_repo_url) - self.metadata_repo.fetch_remote_branch("origin", self.metadata_branch) - # Resetting the current branch to the origin branch - LOGGER.info("Resetting branch %s to %s", self.metadata_branch, upstream) - self.metadata_repo.git_reset(["--hard", upstream]) - # Rewrite the test lifecycle tags file - self.write() - # Rewrite the references file - references = self._read_references() - self._update_and_write_references(references) - # Checking if we can still commit - if not self._ready_for_commit(upstream, references): - LOGGER.warning("Aborting.") - break - # Committing - self._commit_locally(issue_key) - tries += 1 - if pushed: - self.jira_issue_creator.close_fix_issue(issue_key) - return True - self.jira_issue_creator.close_wontfix_issue(issue_key) - return False - - -def make_lifecycle_tags_file(options, model_config): - """Create a LifecycleTagsFile based on the script options.""" - if options.commit: - if not options.jira_config: - LOGGER.error("JIRA configuration file is required when specifying --commit.") - return None - if not (options.git_user_name or options.git_user_email): - LOGGER.error("Git configuration parameters are required when specifying --commit.") - return None - jira_issue_creator = JiraIssueCreator(**utils.load_yaml_file(options.jira_config)) - git_config = (options.git_user_name, options.git_user_email) - else: - jira_issue_creator = None - git_config = None - - lifecycle_tags_file = LifecycleTagsFile(options.project, options.tag_file, - options.metadata_repo_url, options.references_file, - jira_issue_creator, git_config, model_config) - - return lifecycle_tags_file - - -def main(): # pylint: disable=too-many-branches,too-many-locals,too-many-statements - """Exexcute utility to update a resmoke.py tag file. - - This is based on computing test failure rates from the Evergreen API. - """ - - parser = optparse.OptionParser( - description=textwrap.dedent(main.__doc__), usage="Usage: %prog [options] [test1 test2 ...]") - - data_options = optparse.OptionGroup( - parser, title="Data options", - description=("Options used to configure what historical test failure data to retrieve from" - " Evergreen.")) - parser.add_option_group(data_options) - - data_options.add_option("--project", dest="project", metavar="<project-name>", - default=tf.TestHistory.DEFAULT_PROJECT, - help="The Evergreen project to analyze. Defaults to '%default'.") - - data_options.add_option( - "--tasks", dest="tasks", metavar="<task1,task2,...>", - help=("The Evergreen tasks to analyze for tagging unreliable tests. If specified in" - " additional to having test positional arguments, then only tests that run under the" - " specified Evergreen tasks will be analyzed. If omitted, then the list of tasks" - " defaults to the non-excluded list of tasks from the specified" - " --evergreenProjectConfig file.")) - - data_options.add_option( - "--variants", dest="variants", metavar="<variant1,variant2,...>", default="", - help="The Evergreen build variants to analyze for tagging unreliable tests.") - - data_options.add_option("--distros", dest="distros", metavar="<distro1,distro2,...>", - default="", - help="The Evergreen distros to analyze for tagging unreliable tests.") - - data_options.add_option( - "--evergreenProjectConfig", dest="evergreen_project_config", - metavar="<project-config-file>", default="etc/evergreen.yml", - help=("The Evergreen project configuration file used to get the list of tasks if --tasks is" - " omitted. Defaults to '%default'.")) - - model_options = optparse.OptionGroup( - parser, title="Model options", - description=("Options used to configure whether (test,), (test, task)," - " (test, task, variant), and (test, task, variant, distro) combinations are" - " considered unreliable.")) - parser.add_option_group(model_options) - - model_options.add_option( - "--reliableTestMinRuns", type="int", dest="reliable_test_min_runs", - metavar="<reliable-min-runs>", default=DEFAULT_CONFIG.reliable_min_runs, - help=("The minimum number of test executions required for a test's failure rate to" - " determine whether the test is considered reliable. If a test has fewer than" - " <reliable-min-runs> executions, then it cannot be considered unreliable.")) - - model_options.add_option( - "--unreliableTestMinRuns", type="int", dest="unreliable_test_min_runs", - metavar="<unreliable-min-runs>", default=DEFAULT_CONFIG.unreliable_min_runs, - help=("The minimum number of test executions required for a test's failure rate to" - " determine whether the test is considered unreliable. If a test has fewer than" - " <unreliable-min-runs> executions, then it cannot be considered unreliable.")) - - model_options.add_option( - "--testFailRates", type="float", nargs=2, dest="test_fail_rates", - metavar="<test-acceptable-fail-rate> <test-unacceptable-fail-rate>", - default=DEFAULT_CONFIG.test_fail_rates, - help=("Controls how readily a test is considered unreliable. Each failure rate must be a" - " number between 0 and 1 (inclusive) with" - " <test-unacceptable-fail-rate> >= <test-acceptable-fail-rate>. If a test fails no" - " more than <test-acceptable-fail-rate> in <reliable-days> time, then it is" - " considered reliable. Otherwise, if a test fails at least as much as" - " <test-unacceptable-fail-rate> in <test-unreliable-days> time, then it is considered" - " unreliable. Defaults to %default.")) - - model_options.add_option( - "--taskFailRates", type="float", nargs=2, dest="task_fail_rates", - metavar="<task-acceptable-fail-rate> <task-unacceptable-fail-rate>", - default=DEFAULT_CONFIG.task_fail_rates, - help=("Controls how readily a (test, task) combination is considered unreliable. Each" - " failure rate must be a number between 0 and 1 (inclusive) with" - " <task-unacceptable-fail-rate> >= <task-acceptable-fail-rate>. If a (test, task)" - " combination fails no more than <task-acceptable-fail-rate> in <reliable-days> time," - " then it is considered reliable. Otherwise, if a test fails at least as much as" - " <task-unacceptable-fail-rate> in <unreliable-days> time, then it is considered" - " unreliable. Defaults to %default.")) - - model_options.add_option( - "--variantFailRates", type="float", nargs=2, dest="variant_fail_rates", - metavar="<variant-acceptable-fail-rate> <variant-unacceptable-fail-rate>", - default=DEFAULT_CONFIG.variant_fail_rates, - help=("Controls how readily a (test, task, variant) combination is considered unreliable." - " Each failure rate must be a number between 0 and 1 (inclusive) with" - " <variant-unacceptable-fail-rate> >= <variant-acceptable-fail-rate>. If a" - " (test, task, variant) combination fails no more than <variant-acceptable-fail-rate>" - " in <reliable-days> time, then it is considered reliable. Otherwise, if a test fails" - " at least as much as <variant-unacceptable-fail-rate> in <unreliable-days> time," - " then it is considered unreliable. Defaults to %default.")) - - model_options.add_option( - "--distroFailRates", type="float", nargs=2, dest="distro_fail_rates", - metavar="<distro-acceptable-fail-rate> <distro-unacceptable-fail-rate>", - default=DEFAULT_CONFIG.distro_fail_rates, - help=("Controls how readily a (test, task, variant, distro) combination is considered" - " unreliable. Each failure rate must be a number between 0 and 1 (inclusive) with" - " <distro-unacceptable-fail-rate> >= <distro-acceptable-fail-rate>. If a" - " (test, task, variant, distro) combination fails no more than" - " <distro-acceptable-fail-rate> in <reliable-days> time, then it is considered" - " reliable. Otherwise, if a test fails at least as much as" - " <distro-unacceptable-fail-rate> in <unreliable-days> time, then it is considered" - " unreliable. Defaults to %default.")) - - model_options.add_option( - "--reliableDays", type="int", dest="reliable_days", metavar="<ndays>", - default=DEFAULT_CONFIG.reliable_time_period.days, - help=("The time period to analyze when determining if a test has become reliable. Defaults" - " to %default day(s).")) - - model_options.add_option( - "--unreliableDays", type="int", dest="unreliable_days", metavar="<ndays>", - default=DEFAULT_CONFIG.unreliable_time_period.days, - help=("The time period to analyze when determining if a test has become unreliable." - " Defaults to %default day(s).")) - - parser.add_option("--resmokeTagFile", dest="tag_file", metavar="<tagfile>", - default="etc/test_lifecycle.yml", - help=("The resmoke.py tag file to update. If --metadataRepo is specified, it" - " is the relative path in the metadata repository, otherwise it can be" - " an absolute path or a relative path from the current directory." - " Defaults to '%default'.")) - - parser.add_option("--metadataRepo", dest="metadata_repo_url", metavar="<metadata-repo-url>", - default="git@github.com:mongodb/mongo-test-metadata.git", - help=("The repository that contains the lifecycle file. " - "It will be cloned in the current working directory. " - "Defaults to '%default'.")) - - parser.add_option("--referencesFile", dest="references_file", metavar="<references-file>", - default="references.yml", - help=("The YAML file in the metadata repository that contains the revision " - "mappings. Defaults to '%default'.")) - - parser.add_option("--requestBatchSize", type="int", dest="batch_size", metavar="<batch-size>", - default=10, - help=("The maximum number of tests to query the Evergreen API for in a single" - " request. A higher value for this option will reduce the number of" - " roundtrips between this client and Evergreen. The maximium value is" - " {}. Defaults to %default.".format(MAX_BATCH_SIZE))) - - parser.add_option("--requestThreads", type="int", dest="num_request_threads", - metavar="<num-request-threads>", default=DEFAULT_NUM_THREADS, - help=("The maximum number of threads to use when querying the Evergreen API." - " Batches are processed sequentially but the test history is queried in" - " parallel for each task. Defaults to %default.")) - - commit_options = optparse.OptionGroup( - parser, title="Commit options", - description=("Options used to configure whether and how to commit the updated test" - " lifecycle tags.")) - parser.add_option_group(commit_options) - - commit_options.add_option("--commit", action="store_true", dest="commit", default=False, - help="Indicates that the updated tag file should be committed.") - - commit_options.add_option( - "--jiraConfig", dest="jira_config", metavar="<jira-config>", default=None, - help=("The YAML file containing the JIRA access configuration ('user', 'password'," - "'server').")) - - commit_options.add_option( - "--gitUserName", dest="git_user_name", metavar="<git-user-name>", default="Test Lifecycle", - help=("The git user name that will be set before committing to the metadata repository." - " Defaults to '%default'.")) - - commit_options.add_option( - "--gitUserEmail", dest="git_user_email", metavar="<git-user-email>", - default="buil+testlifecycle@mongodb.com", - help=("The git user email address that will be set before committing to the metadata" - " repository. Defaults to '%default'.")) - - logging_options = optparse.OptionGroup( - parser, title="Logging options", - description="Options used to configure the logging output of the script.") - parser.add_option_group(logging_options) - - logging_options.add_option("--logLevel", dest="log_level", metavar="<log-level>", choices=[ - "DEBUG", "INFO", "WARNING", "ERROR" - ], default="INFO", help=("The log level. Accepted values are: DEBUG, INFO, WARNING and ERROR." - " Defaults to '%default'.")) - - logging_options.add_option( - "--logFile", dest="log_file", metavar="<log-file>", default=None, - help="The destination file for the logs output. Defaults to the standard output.") - - (options, tests) = parser.parse_args() - - if options.batch_size < 1 or options.batch_size > MAX_BATCH_SIZE: - parser.print_help() - parser.error("Invalid --requestBatchSize specified: {}.".format(options.batch_size)) - - logging.basicConfig(format="%(asctime)s %(levelname)s %(message)s", level=options.log_level, - filename=options.log_file) - evg_conf = ci_evergreen.parse_evergreen_file(options.evergreen_project_config) - use_test_tasks_membership = False - - tasks = options.tasks.split(",") if options.tasks else [] - if not tasks: - # If no tasks are specified, then the list of tasks is all. - tasks = evg_conf.lifecycle_task_names - use_test_tasks_membership = True - - variants = options.variants.split(",") if options.variants else [] - - distros = options.distros.split(",") if options.distros else [] - - config = Config( - test_fail_rates=Rates(*options.test_fail_rates), - task_fail_rates=Rates(*options.task_fail_rates), - variant_fail_rates=Rates(*options.variant_fail_rates), - distro_fail_rates=Rates(*options.distro_fail_rates), - reliable_min_runs=options.reliable_test_min_runs, - reliable_time_period=datetime.timedelta(days=options.reliable_days), - unreliable_min_runs=options.unreliable_test_min_runs, - unreliable_time_period=datetime.timedelta(days=options.unreliable_days)) - validate_config(config) - - lifecycle_tags_file = make_lifecycle_tags_file(options, config) - if not lifecycle_tags_file: - sys.exit(1) - - test_tasks_membership = get_test_tasks_membership(evg_conf) - # If no tests are specified then the list of tests is generated from the list of tasks. - if not tests: - tests = get_tests_from_tasks(tasks, test_tasks_membership) - if not options.tasks: - use_test_tasks_membership = True - - start_date, end_date = get_date_range(max(options.reliable_days, options.unreliable_days)) - - # For efficiency purposes, group the tests and process in batches of batch_size. - test_groups = create_batch_groups(create_test_groups(tests), options.batch_size) - - test_history_source = TestHistorySource(options.project, variants, distros, start_date, - end_date, options.num_request_threads) - - LOGGER.info("Updating the tags") - nb_groups = len(test_groups) - count = 0 - for tests in test_groups: - LOGGER.info("Progress: %s %%", 100 * count / nb_groups) - count += 1 - # Find all associated tasks for the test_group if tasks or tests were not specified. - if use_test_tasks_membership: - tasks_set = set() - for test in tests: - tasks_set = tasks_set.union(test_tasks_membership[test]) - tasks = list(tasks_set) - if not tasks: - LOGGER.warning("No tasks found for tests %s, skipping this group.", tests) - continue - history_data = test_history_source.get_history_data(tests, tasks) - if not history_data: - continue - report = tf.Report(history_data) - update_tags(lifecycle_tags_file.changelog_lifecycle, config, report, tests) - - # Remove tags that are no longer relevant - clean_up_tags(lifecycle_tags_file.changelog_lifecycle, evg_conf) - - # We write the 'lifecycle' tag configuration to the 'options.lifecycle_file' file only if there - # have been changes to the tags. In particular, we avoid modifying the file when only the header - # comment for the YAML file would change. - if lifecycle_tags_file.is_modified(): - lifecycle_tags_file.write() - - if options.commit: - commit_ok = lifecycle_tags_file.commit() - if not commit_ok: - sys.exit(1) - else: - LOGGER.info("The tags have not been modified.") - - -if __name__ == "__main__": - main() diff --git a/etc/evergreen.yml b/etc/evergreen.yml index 146bbc84699..1c9321245bf 100644 --- a/etc/evergreen.yml +++ b/etc/evergreen.yml @@ -945,12 +945,6 @@ functions: path_value="$path_value:${task_path_suffix}" fi - if [ "${is_patch}" = "true" ]; then - extra_args="$extra_args --tagFile=etc/test_lifecycle.yml --patchBuild" - else - extra_args="$extra_args --tagFile=etc/test_retrial.yml" - fi - resmoke_cmd_args="${resmoke_args}" # The "resmoke_wrapper" expansion is used by the 'burn_in_tests' task to wrap the resmoke.py @@ -1746,29 +1740,6 @@ functions: kitchen destroy "${packager_distro}"\$ || true test "$verified" = "true" - "fetch test_lifecycle.yml": - - command: shell.exec - type: test - params: - working_dir: src - script: | - set -o verbose - - ${activate_virtualenv} - $python buildscripts/fetch_test_lifecycle.py \ - --metadataRepo git@github.com:mongodb/mongo-test-metadata.git \ - --lifecycleFile etc/test_lifecycle.yml \ - --referencesFile references.yml \ - --destinationFile etc/test_lifecycle.yml \ - --revision ${revision} \ - ${project} - exit_code=$? - if [ ${fail_task_on_error|false} = true ]; then - exit $exit_code - else - exit 0 - fi - "copy ec2 monitor files": ©_ec2_monitor_files command: shell.exec params: @@ -3043,27 +3014,6 @@ timeout: # Tasks # ####################################### -## The test_lifecycle_excluded_tasks are a list of tasks names which include: -## - Non jstests, i.e., compile, unittests -## - Non standard jstests, i.e., jstestfuzz -## Note that the task name supports glob patterns. -test_lifecycle_excluded_tasks: -- burn_in_tests -- compile* -- benchmarks* -- dbtest* -- idl_tests -- integration* -- jepsen* -- jstestfuzz* -- lint -- mongos* -- package -- push -- rollback_fuzzer* -- unittests* -- update_fuzzer* - tasks: ## compile - build all scons targets except unittests ## @@ -3146,12 +3096,6 @@ tasks: $python ../buildscripts/make_archive.py -o mongodb-shell.${ext|tgz} $(find mongodb-* -type f) cd .. - # Test lifecycle is temporarily disabled to reduce load on Evergreen API. - # - func: "fetch test_lifecycle.yml" - # vars: - # # Do not fail the task if we fail to fetch the test_lifecycle.yml file - # fail_task_on_error: false - - command: archive.targz_pack params: target: "artifacts.tgz" @@ -3175,8 +3119,6 @@ tasks: - "./etc/pip/**" - "./etc/*san.suppressions" - "./etc/repo_config.yaml" - - "./etc/test_lifecycle.yml" - - "./etc/test_retrial.yml" - "./build/integration_tests/**" - "./build/integration_tests.txt" - "./build/**.gcno" @@ -6109,67 +6051,6 @@ tasks: - {'source': {'path': '${push_path}-STAGE/${push_name}/mongodb-${push_name}-${push_arch}-debugsymbols-${suffix}-${task_id}.${ext|tgz}.md5', 'bucket': 'build-push-testing'}, 'destination': {'path': '${push_path}/mongodb-${push_name}-${push_arch}-debugsymbols-${suffix}.${ext|tgz}.md5', 'bucket': '${push_bucket}'}} -- name: fetch_test_lifecycle - depends_on: [] - commands: - - func: "git get project" - - func: "fetch test_lifecycle.yml" - vars: - # This task is meant to fail if there is an error while fetching test_lifecycle.yml since - # the compile task won't fail. - fail_task_on_error: true - -- name: update_test_lifecycle - exec_timeout_secs: 21600 # 6 hour timeout for the task overall - depends_on: [] - commands: - - func: "git get project" - - func: "configure evergreen api credentials" - - command: shell.exec - timeout_secs: 14400 # Timeout if there is no output for 4 hours - type: test - params: - working_dir: src - silent: true - script: | - set -o errexit - - # Set up the git ssh private key - mkdir -p ~/.ssh - echo -n "${testlifecycle_ssh_key}" > ~/.ssh/test_lifecycle.pem - chmod 0600 ~/.ssh/test_lifecycle.pem - export GIT_SSH_COMMAND="ssh -i ~/.ssh/test_lifecycle.pem" - - # Create the jira credentials configuration file - cat > .jira.yml <<END_OF_CREDS - server: "https://jira.mongodb.org" - access_token: "${testlifecycle_jira_access_token}" - access_token_secret: "${testlifecycle_jira_access_token_secret}" - consumer_key: "${testlifecycle_jira_consumer_key}" - key_cert: | - $(echo "${testlifecycle_jira_key_certificate}" | sed 's/^/ /') - END_OF_CREDS - - set -o verbose - - ${activate_virtualenv} - # Install Python modules to support OAuth with pip until it is available in the toolchain. - pip install cryptography==1.7.2 pyjwt==1.5.3 - - # We use a small batch size to avoid hitting the load balancer timeout if the Evergreen - # API query is not fast enough. - # Evergreen executable is in $HOME. - PATH=$PATH:$HOME $python buildscripts/update_test_lifecycle.py \ - --project ${project} \ - --requestBatchSize 20 \ - --commit \ - --resmokeTagFile "etc/test_lifecycle.yml" \ - --metadataRepo "git@github.com:mongodb/mongo-test-metadata" \ - --referencesFile "references.yml" \ - --gitUserName "Test Lifecycle" \ - --gitUserEmail "build+testlifecycle@mongodb.com" \ - --jiraConfig .jira.yml - ####################################### # Task Groups # @@ -11153,9 +11034,6 @@ buildvariants: batchtime: 1440 # 1 day stepback: false tasks: -# Test lifecycle is temporarily disabled to reduce load on Evergreen API. -# - name: update_test_lifecycle - - name: fetch_test_lifecycle - name: lint_fuzzer_sanity_all - name: enterprise-rhel-62-64-bit-mmapv1 |