SERVER-29645 Task to update and commit test lifecycle tags

1 files changed, 505 insertions, 30 deletions

diff --git a/buildscripts/update_test_lifecycle.py b/buildscripts/update_test_lifecycle.py
index 1d533b1803d..a71cac9321e 100755
--- a/buildscripts/update_test_lifecycle.py
+++ b/buildscripts/update_test_lifecycle.py
@@ -7,21 +7,26 @@ Update etc/test_lifecycle.yml to tag unreliable tests based on historic failure
 
 from __future__ import absolute_import
 from __future__ import division
-from __future__ import print_function
 
 import collections
 import datetime
+import logging
 import optparse
 import os.path
+import posixpath
 import subprocess
 import sys
 import textwrap
 import warnings
 
+import yaml
+
 # Get relative imports to work when the package is not installed on the PYTHONPATH.
 if __name__ == "__main__" and __package__ is None:
     sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
+from buildscripts import git
+from buildscripts import jiraclient
 from buildscripts import resmokelib
 from buildscripts.resmokelib.utils import globstar
 from buildscripts import test_failures as tf
@@ -29,6 +34,8 @@ from buildscripts.ciconfig import evergreen as ci_evergreen
 from buildscripts.ciconfig import tags as ci_tags
 
 
+LOGGER = logging.getLogger(__name__)
+
 if sys.version_info[0] == 2:
     _NUMBER_TYPES = (int, long, float)
 else:
@@ -64,17 +71,6 @@ DEFAULT_CONFIG = Config(
 DEFAULT_PROJECT = "mongodb-mongo-master"
 
 
-def write_yaml_file(yaml_file, lifecycle):
-    """Writes the lifecycle object to yaml_file."""
-
-    comment = (
-        "This file was generated by {} and shouldn't be edited by hand. It was generated against"
-        " commit {} with the following invocation: {}."
-    ).format(sys.argv[0], callo(["git", "rev-parse", "HEAD"]).rstrip(), " ".join(sys.argv))
-
-    lifecycle.write_file(yaml_file, comment)
-
-
 def get_suite_tasks_membership(evg_conf):
     """Return a dictionary with keys of all suites and list of associated tasks."""
     suite_membership = collections.defaultdict(list)
@@ -202,7 +198,7 @@ def unreliable_tag(task, variant, distro):
     return "unreliable|{}|{}|{}".format(task, variant, distro)
 
 
-def update_lifecycle(lifecycle, report, method_test, add_tags, fail_rate, min_run):
+def update_lifecycle(lifecycle_tags_file, report, method_test, add_tags, fail_rate, min_run):
     """Updates the lifecycle object based on the test_method.
 
     The test_method checks unreliable or reliable fail_rates.
@@ -214,9 +210,11 @@ def update_lifecycle(lifecycle, report, method_test, add_tags, fail_rate, min_ru
                        min_run):
             update_tag = unreliable_tag(summary.task, summary.variant, summary.distro)
             if add_tags:
-                lifecycle.add_tag("js_test", summary.test, update_tag)
+                lifecycle_tags_file.add_tag("js_test", summary.test,
+                                            update_tag, summary.fail_rate)
             else:
-                lifecycle.remove_tag("js_test", summary.test, update_tag)
+                lifecycle_tags_file.remove_tag("js_test", summary.test,
+                                               update_tag, summary.fail_rate)
 
 
 def compare_tags(tag_a, tag_b):
@@ -281,10 +279,10 @@ def validate_config(config):
                     name, time_period))
 
 
-def update_tags(lifecycle, config, report):
+def update_tags(lifecycle_tags, config, report):
     """
-    Updates the tags in 'lifecycle' based on the historical test failures mentioned in 'report'
-    according to the model described by 'config'.
+    Updates the tags in 'lifecycle_tags' based on the historical test failures mentioned in
+    'report' according to the model described by 'config'.
     """
 
     # We initialize 'grouped_entries' to make PyLint not complain about 'grouped_entries' being used
@@ -308,7 +306,7 @@ def update_tags(lifecycle, config, report):
                                  + datetime.timedelta(days=1))
         unreliable_report = tf.Report(entry for entry in grouped_entries
                                       if entry.start_date >= unreliable_start_date)
-        update_lifecycle(lifecycle,
+        update_lifecycle(lifecycle_tags,
                          unreliable_report.summarize_by(components),
                          unreliable_test,
                          True,
@@ -320,7 +318,7 @@ def update_tags(lifecycle, config, report):
                                + datetime.timedelta(days=1))
         reliable_report = tf.Report(entry for entry in grouped_entries
                                     if entry.start_date >= reliable_start_date)
-        update_lifecycle(lifecycle,
+        update_lifecycle(lifecycle_tags,
                          reliable_report.summarize_by(components),
                          reliable_test,
                          False,
@@ -346,6 +344,8 @@ def _split_tag(tag):
 
 def _is_tag_still_relevant(evg_conf, tag):
     """Indicate if a tag still corresponds to a valid task/variant/distro combination."""
+    if tag == "unreliable":
+        return True
     task, variant, distro = _split_tag(tag)
     if not task or task not in evg_conf.task_names:
         return False
@@ -358,17 +358,411 @@ def _is_tag_still_relevant(evg_conf, tag):
     return True
 
 
-def cleanup_tags(lifecycle, evg_conf):
+def clean_up_tags(lifecycle_tags, evg_conf):
     """Remove the tags that do not correspond to a valid test/task/variant/distro combination."""
+    lifecycle = lifecycle_tags.lifecycle
     for test_kind in lifecycle.get_test_kinds():
         for test_pattern in lifecycle.get_test_patterns(test_kind):
             if not globstar.glob(test_pattern):
                 # The pattern does not match any file in the repository.
-                lifecycle.remove_test_pattern(test_kind, test_pattern)
+                lifecycle_tags.clean_up_test(test_kind, test_pattern)
                 continue
             for tag in lifecycle.get_tags(test_kind, test_pattern):
                 if not _is_tag_still_relevant(evg_conf, tag):
-                    lifecycle.remove_tag(test_kind, test_pattern, tag)
+                    lifecycle_tags.clean_up_tag(test_kind, test_pattern, tag)
+
+
+def _config_as_options(config):
+    return ("--reliableTestMinRuns {} "
+            "--reliableDays {} "
+            "--unreliableTestMinRuns {} "
+            "--unreliableDays {} "
+            "--testFailRates {} {} "
+            "--taskFailRates {} {} "
+            "--variantFailRates {} {} "
+            "--distroFailRates {} {}").format(
+                config.reliable_min_runs,
+                config.reliable_time_period.days,
+                config.unreliable_min_runs,
+                config.unreliable_time_period.days,
+                config.test_fail_rates.acceptable,
+                config.test_fail_rates.unacceptable,
+                config.task_fail_rates.acceptable,
+                config.task_fail_rates.unacceptable,
+                config.variant_fail_rates.acceptable,
+                config.variant_fail_rates.unacceptable,
+                config.distro_fail_rates.acceptable,
+                config.distro_fail_rates.unacceptable)
+
+
+class TagsConfigWithChangelog(object):
+    """A wrapper around TagsConfig that can perform updates on a tags file and record the
+    modifications made.
+    """
+
+    def __init__(self, lifecycle):
+        """Initialize the TagsConfigWithChangelog with the lifecycle TagsConfig."""
+        self.lifecycle = lifecycle
+        self.added = {}
+        self.removed = {}
+        self.cleaned_up = {}
+
+    @staticmethod
+    def _cancel_tag_log(log_dict, test_kind, test, tag):
+        """Remove a tag from a changelog dictionary.
+
+        Used to remove a tag from the 'added' or 'removed' attribute.
+        """
+        kind_dict = log_dict[test_kind]
+        test_dict = kind_dict[test]
+        del test_dict[tag]
+        if not test_dict:
+            del kind_dict[test]
+            if not kind_dict:
+                del log_dict[test_kind]
+
+    def add_tag(self, test_kind, test, tag, failure_rate):
+        """Add a tag."""
+        if self.lifecycle.add_tag(test_kind, test, tag):
+            if tag in self.removed.get(test_kind, {}).get(test, {}):
+                # The tag has just been removed.
+                self._cancel_tag_log(self.removed, test_kind, test, tag)
+            else:
+                self.added.setdefault(test_kind, {}).setdefault(test, {})[tag] = failure_rate
+
+    def remove_tag(self, test_kind, test, tag, failure_rate):
+        """Remove a tag."""
+        if self.lifecycle.remove_tag(test_kind, test, tag):
+            if tag in self.added.get(test_kind, {}).get(test, {}):
+                # The tag has just been added.
+                self._cancel_tag_log(self.added, test_kind, test, tag)
+            else:
+                self.removed.setdefault(test_kind, {}).setdefault(test, {})[tag] = failure_rate
+
+    def clean_up_tag(self, test_kind, test, tag):
+        """Clean up an invalid tag."""
+        self.lifecycle.remove_tag(test_kind, test, tag)
+        self.cleaned_up.setdefault(test_kind, {}).setdefault(test, []).append(tag)
+
+    def clean_up_test(self, test_kind, test):
+        """Clean up an invalid test."""
+        self.lifecycle.remove_test_pattern(test_kind, test)
+        self.cleaned_up.setdefault(test_kind, {})[test] = []
+
+
+class JiraIssueCreator(object):
+    _LABEL = "test-lifecycle"
+    _PROJECT = "TIGBOT"
+
+    def __init__(self, jira_server, jira_user, jira_password):
+        self._client = jiraclient.JiraClient(jira_server, jira_user, jira_password)
+
+    def create_issue(self, evg_project, mongo_revision, model_config, added, removed, cleaned_up):
+        """Create a JIRA issue for the test lifecycle tag update."""
+        summary = self._get_jira_summary(evg_project)
+        description = self._get_jira_description(evg_project, mongo_revision, model_config,
+                                                 added, removed, cleaned_up)
+        issue_key = self._client.create_issue(self._PROJECT, summary, description, [self._LABEL])
+        return issue_key
+
+    def close_fix_issue(self, issue_key):
+        """Close the issue with the "Fixed" resolution."""
+        LOGGER.info("Closing issue '%s' as FIXED.", issue_key)
+        self._client.close_issue(issue_key, self._client.FIXED_RESOLUTION_NAME)
+
+    def close_wontfix_issue(self, issue_key):
+        """Close the issue the with "Won't Fix" resolution."""
+        LOGGER.info("Closing issue '%s' as WON'T FIX.", issue_key)
+        self._client.close_issue(issue_key, self._client.WONT_FIX_RESOLUTION_NAME)
+
+    @staticmethod
+    def _get_jira_summary(project):
+        return "Update of test lifecycle tags for {}".format(project)
+
+    @staticmethod
+    def _monospace(text):
+        """Transform a text into a monospace JIRA text."""
+        return "{{" + text + "}}"
+
+    @staticmethod
+    def _get_jira_description(project, mongo_revision, model_config, added, removed, cleaned_up):
+        mono = JiraIssueCreator._monospace
+        config_desc = _config_as_options(model_config)
+        added_desc = JiraIssueCreator._make_updated_tags_description(added)
+        removed_desc = JiraIssueCreator._make_updated_tags_description(removed)
+        cleaned_up_desc = JiraIssueCreator._make_tags_cleaned_up_description(cleaned_up)
+        project_link = "[{0}|https://evergreen.mongodb.com/waterfall/{1}]".format(
+            mono(project), project)
+        revision_link = "[{0}|https://github.com/mongodb/mongo/commit/{1}]".format(
+            mono(mongo_revision), mongo_revision)
+        return ("h3. Automatic update of the test lifecycle tags\n"
+                "Evergreen Project: {0}\n"
+                "Revision: {1}\n\n"
+                "{{{{update_test_lifecycle.py}}}} options:\n{2}\n\n"
+                "h5. Tags added\n{3}\n\n"
+                "h5. Tags removed\n{4}\n\n"
+                "h5. Tags cleaned up (no longer relevant)\n{5}\n").format(
+                    project_link, revision_link, mono(config_desc),
+                    added_desc, removed_desc, cleaned_up_desc)
+
+    @staticmethod
+    def _make_updated_tags_description(data):
+        mono = JiraIssueCreator._monospace
+        tags_lines = []
+        for test_kind in sorted(data.keys()):
+            tests = data[test_kind]
+            tags_lines.append("- *{0}*".format(test_kind))
+            for test in sorted(tests.keys()):
+                tags = tests[test]
+                tags_lines.append("-- {0}".format(mono(test)))
+                for tag in sorted(tags.keys()):
+                    coefficient = tags[tag]
+                    tags_lines.append("--- {0} ({1:.2} %)".format(mono(tag), coefficient))
+        if tags_lines:
+            return "\n".join(tags_lines)
+        else:
+            return "_None_"
+
+    @staticmethod
+    def _make_tags_cleaned_up_description(cleaned_up):
+        mono = JiraIssueCreator._monospace
+        tags_cleaned_up_lines = []
+        for test_kind in sorted(cleaned_up.keys()):
+            test_tags = cleaned_up[test_kind]
+            tags_cleaned_up_lines.append("- *{0}*".format(test_kind))
+            for test in sorted(test_tags.keys()):
+                tags = test_tags[test]
+                tags_cleaned_up_lines.append("-- {0}".format(mono(test)))
+                if not tags:
+                    tags_cleaned_up_lines.append("--- ALL (test file removed or renamed as part of"
+                                                 " an earlier commit)")
+                else:
+                    for tag in sorted(tags):
+                        tags_cleaned_up_lines.append("--- {0}".format(mono(tag)))
+        if tags_cleaned_up_lines:
+            return "\n".join(tags_cleaned_up_lines)
+        else:
+            return "_None_"
+
+
+class LifecycleTagsFile(object):
+    """Represent a test lifecycle tags file that can be written and committed."""
+
+    def __init__(self, project, lifecycle_file, metadata_repo_url=None, references_file=None,
+                 jira_issue_creator=None, git_info=None, model_config=None):
+        """Initalize the LifecycleTagsFile.
+
+        Arguments:
+            project: The Evergreen project name, e.g. "mongodb-mongo-master".
+            lifecycle_file: The path to the lifecycle tags file. If 'metadata_repo_url' is
+                specified, this path must be relative to the root of the metadata repository.
+            metadata_repo_url: The URL of the metadat repository that contains the test lifecycle
+                tags file.
+            references_file: The path to the references file in the metadata repository.
+            jira_issue_creator: A JiraIssueCreator instance.
+            git_info: A tuple containing the git user's name and email to set before committing.
+            model_config: The model configuration as a Config instance.
+        """
+        self.project = project
+        self.mongo_repo = git.Repository(os.getcwd())
+        self.mongo_revision = self.mongo_repo.get_current_revision()
+        # The branch name is the same on both repositories.
+        self.mongo_branch = self.mongo_repo.get_branch_name()
+        self.metadata_branch = project
+
+        if metadata_repo_url:
+            # The file can be found in another repository. We clone it.
+            self.metadata_repo = self._clone_repository(metadata_repo_url, self.project)
+            self.relative_lifecycle_file = lifecycle_file
+            self.lifecycle_file = os.path.join(self.metadata_repo.directory, lifecycle_file)
+            self.relative_references_file = references_file
+            self.references_file = os.path.join(self.metadata_repo.directory, references_file)
+            if git_info:
+                self.metadata_repo.configure("user.name", git_info[0])
+                self.metadata_repo.configure("user.email", git_info[1])
+        else:
+            self.metadata_repo = None
+            self.relative_lifecycle_file = lifecycle_file
+            self.lifecycle_file = lifecycle_file
+            self.relative_references_file = None
+            self.references_file = None
+        self.metadata_repo_url = metadata_repo_url
+        self.lifecycle = ci_tags.TagsConfig.from_file(self.lifecycle_file, cmp_func=compare_tags)
+        self.jira_issue_creator = jira_issue_creator
+        self.model_config = model_config
+        self.changelog_lifecycle = TagsConfigWithChangelog(self.lifecycle)
+
+    @staticmethod
+    def _clone_repository(metadata_repo_url, branch):
+        directory_name = posixpath.splitext(posixpath.basename(metadata_repo_url))[0]
+        LOGGER.info("Cloning the repository %s into the directory %s",
+                    metadata_repo_url, directory_name)
+        return git.Repository.clone(metadata_repo_url, directory_name, branch)
+
+    def is_modified(self):
+        """Indicate if the tags have been modified."""
+        return self.lifecycle.is_modified()
+
+    def _create_issue(self):
+        LOGGER.info("Creating a JIRA issue")
+        issue_key = self.jira_issue_creator.create_issue(
+            self.project, self.mongo_revision, self.model_config,
+            self.changelog_lifecycle.added, self.changelog_lifecycle.removed,
+            self.changelog_lifecycle.cleaned_up)
+        LOGGER.info("JIRA issue created: %s", issue_key)
+        return issue_key
+
+    def write(self):
+        """Write the test lifecycle tag file."""
+        LOGGER.info("Writing the tag file to '%s'", self.lifecycle_file)
+        comment = ("This file was generated by {} and shouldn't be edited by hand. It was"
+                   " generated against commit {} with the following options: {}.").format(
+                       sys.argv[0], self.mongo_repo.get_current_revision(),
+                       _config_as_options(self.model_config))
+        self.lifecycle.write_file(self.lifecycle_file, comment)
+
+    def _ready_for_commit(self, ref_branch, references):
+        # Check that the test lifecycle tags file has changed.
+        diff = self.metadata_repo.git_diff(["--name-only", ref_branch,
+                                            self.relative_lifecycle_file])
+        if not diff:
+            LOGGER.info("The local lifecycle file is identical to the the one on branch '%s'",
+                        ref_branch)
+            return False
+        # Check that the lifecycle file has not been updated after the current mongo revision.
+        update_revision = references.get("test-lifecycle", {}).get(self.project)
+        if update_revision and not self.mongo_repo.is_ancestor(update_revision,
+                                                               self.mongo_revision):
+            LOGGER.warning(("The existing lifecycle file is based on revision '%s' which is not a"
+                            " parent revision of the current revision '%s'"),
+                           update_revision, self.mongo_revision)
+            return False
+        return True
+
+    def _read_references(self, metadata_branch=None):
+        branch = metadata_branch if metadata_branch is not None else ""
+        references_content = self.metadata_repo.git_cat_file(
+            ["blob", "{0}:{1}".format(branch, self.relative_references_file)])
+        return yaml.safe_load(references_content)
+
+    def _update_and_write_references(self, references):
+        LOGGER.info("Writing the references file to '%s'", self.references_file)
+        references.setdefault("test-lifecycle", {})[self.project] = self.mongo_revision
+        with open(self.references_file, "w") as fstream:
+            yaml.safe_dump(references, fstream, default_flow_style=False)
+
+    def _commit_locally(self, issue_key):
+        self.metadata_repo.git_add([self.relative_lifecycle_file])
+        self.metadata_repo.git_add([self.relative_references_file])
+        commit_message = "{} Update {}".format(issue_key, self.relative_lifecycle_file)
+        self.metadata_repo.commit_with_message(commit_message)
+        LOGGER.info("Change committed with message: %s", commit_message)
+
+    def commit(self, nb_retries=10):
+        """Commit the test lifecycle tag file.
+
+        Args:
+            nb_retries: the number of times the script will reset, fetch, recommit and retry when
+                the push fails.
+        """
+        references = self._read_references()
+        # Verify we are ready to commit.
+        if not self._ready_for_commit(self.metadata_branch, references):
+            return True
+
+        # Write the references file.
+        self._update_and_write_references(references)
+
+        # Create the issue.
+        issue_key = self._create_issue()
+
+        # Commit the change.
+        self._commit_locally(issue_key)
+
+        # Push the change.
+        tries = 0
+        pushed = False
+        upstream = "origin/{0}".format(self.metadata_branch)
+        while tries < nb_retries:
+            try:
+                self.metadata_repo.push_to_remote_branch("origin", self.metadata_branch)
+                pushed = True
+                break
+            except git.GitException:
+                LOGGER.warning("git push command failed, fetching and retrying.")
+                # Fetch upstream branch.
+                LOGGER.info("Fetching branch %s of %s", self.metadata_branch,
+                            self.metadata_repo_url)
+                self.metadata_repo.fetch_remote_branch("origin", self.metadata_branch)
+                # Resetting the current branch to the origin branch
+                LOGGER.info("Resetting branch %s to %s", self.metadata_branch, upstream)
+                self.metadata_repo.git_reset(["--hard", upstream])
+                # Rewrite the test lifecycle tags file
+                self.write()
+                # Rewrite the references file
+                references = self._read_references()
+                self._update_and_write_references(references)
+                # Checking if we can still commit
+                if not self._ready_for_commit(upstream, references):
+                    LOGGER.warning("Aborting.")
+                    break
+                # Committing
+                self._commit_locally(issue_key)
+            tries += 1
+        if pushed:
+            self.jira_issue_creator.close_fix_issue(issue_key)
+            return True
+        else:
+            self.jira_issue_creator.close_wontfix_issue(issue_key)
+            return False
+
+
+def _read_jira_configuration(jira_config_file):
+    with open(jira_config_file, "r") as fstream:
+        jira_config = yaml.safe_load(fstream)
+    return (_get_jira_parameter(jira_config, "server"),
+            _get_jira_parameter(jira_config, "user"),
+            _get_jira_parameter(jira_config, "password"))
+
+
+def _get_jira_parameter(jira_config, parameter_name):
+    value = jira_config.get(parameter_name)
+    if not value:
+        LOGGER.error("Missing parameter '%s' in JIRA configuration file", parameter_name)
+    return value
+
+
+def make_lifecycle_tags_file(options, model_config):
+    """Create a LifecycleTagsFile based on the script options."""
+    if options.commit:
+        if not options.jira_config:
+            LOGGER.error("JIRA configuration file is required when specifying --commit.")
+            return None
+        if not (options.git_user_name or options.git_user_email):
+            LOGGER.error("Git configuration parameters are required when specifying --commit.")
+            return None
+        jira_server, jira_user, jira_password = _read_jira_configuration(options.jira_config)
+        if not (jira_server and jira_user and jira_password):
+            return None
+
+        jira_issue_creator = JiraIssueCreator(jira_server,
+                                              jira_user,
+                                              jira_password)
+        git_config = (options.git_user_name, options.git_user_email)
+    else:
+        jira_issue_creator = None
+        git_config = None
+
+    lifecycle_tags_file = LifecycleTagsFile(
+        options.project,
+        options.tag_file,
+        options.metadata_repo_url,
+        options.references_file,
+        jira_issue_creator,
+        git_config,
+        model_config)
+
+    return lifecycle_tags_file
 
 
 def main():
@@ -511,7 +905,23 @@ def main():
     parser.add_option("--resmokeTagFile", dest="tag_file",
                       metavar="<tagfile>",
                       default="etc/test_lifecycle.yml",
-                      help="The resmoke.py tag file to update. Defaults to '%default'.")
+                      help=("The resmoke.py tag file to update. If --metadataRepo is specified, it"
+                            " is the relative path in the metadata repository, otherwise it can be"
+                            " an absolute path or a relative path from the current directory."
+                            " Defaults to '%default'."))
+
+    parser.add_option("--metadataRepo", dest="metadata_repo_url",
+                      metavar="<metadata-repo-url>",
+                      default="git@github.com:mongodb/mongo-test-metadata.git",
+                      help=("The repository that contains the lifecycle file. "
+                            "It will be cloned in the current working directory. "
+                            "Defaults to '%default'."))
+
+    parser.add_option("--referencesFile", dest="references_file",
+                      metavar="<references-file>",
+                      default="references.yml",
+                      help=("The YAML file in the metadata repository that contains the revision "
+                            "mappings. Defaults to '%default'."))
 
     parser.add_option("--requestBatchSize", type="int", dest="batch_size",
                       metavar="<batch-size>",
@@ -520,6 +930,59 @@ def main():
                             " request. A higher value for this option will reduce the number of"
                             " roundtrips between this client and Evergreen. Defaults to %default."))
 
+    commit_options = optparse.OptionGroup(
+        parser,
+        title="Commit options",
+        description=("Options used to configure whether and how to commit the updated test"
+                     " lifecycle tags."))
+    parser.add_option_group(commit_options)
+
+    commit_options.add_option(
+        "--commit", action="store_true", dest="commit",
+        default=False,
+        help="Indicates that the updated tag file should be committed.")
+
+    commit_options.add_option(
+        "--jiraConfig", dest="jira_config",
+        metavar="<jira-config>",
+        default=None,
+        help=("The YAML file containing the JIRA access configuration ('user', 'password',"
+              "'server')."))
+
+    commit_options.add_option(
+        "--gitUserName", dest="git_user_name",
+        metavar="<git-user-name>",
+        default="Test Lifecycle",
+        help=("The git user name that will be set before committing to the metadata repository."
+              " Defaults to '%default'."))
+
+    commit_options.add_option(
+        "--gitUserEmail", dest="git_user_email",
+        metavar="<git-user-email>",
+        default="buil+testlifecycle@mongodb.com",
+        help=("The git user email address that will be set before committing to the metadata"
+              " repository. Defaults to '%default'."))
+
+    logging_options = optparse.OptionGroup(
+        parser,
+        title="Logging options",
+        description="Options used to configure the logging output of the script.")
+    parser.add_option_group(logging_options)
+
+    logging_options.add_option(
+        "--logLevel", dest="log_level",
+        metavar="<log-level>",
+        choices=["DEBUG", "INFO", "WARNING", "ERROR"],
+        default="INFO",
+        help=("The log level. Accepted values are: DEBUG, INFO, WARNING and ERROR."
+              " Defaults to '%default'."))
+
+    logging_options.add_option(
+        "--logFile", dest="log_file",
+        metavar="<log-file>",
+        default=None,
+        help="The destination file for the logs output. Defaults to the standard output.")
+
     (options, tests) = parser.parse_args()
 
     if options.distros:
@@ -528,6 +991,8 @@ def main():
              " isn't returned by the Evergreen API. This option will therefore be ignored."),
             RuntimeWarning)
 
+    logging.basicConfig(format="%(asctime)s %(levelname)s %(message)s",
+                        level=options.log_level, filename=options.log_file)
     evg_conf = ci_evergreen.EvergreenProjectConfig(options.evergreen_project_config)
     use_test_tasks_membership = False
 
@@ -552,7 +1017,9 @@ def main():
         unreliable_time_period=datetime.timedelta(days=options.unreliable_days))
     validate_config(config)
 
-    lifecycle = ci_tags.TagsConfig.from_file(options.tag_file, cmp_func=compare_tags)
+    lifecycle_tags_file = make_lifecycle_tags_file(options, config)
+    if not lifecycle_tags_file:
+        sys.exit(1)
 
     test_tasks_membership = get_test_tasks_membership(evg_conf)
     # If no tests are specified then the list of tests is generated from the list of tasks.
@@ -567,6 +1034,7 @@ def main():
     # For efficiency purposes, group the tests and process in batches of batch_size.
     test_groups = create_batch_groups(create_test_groups(tests), options.batch_size)
 
+    LOGGER.info("Updating the tags")
     for tests in test_groups:
         # Find all associated tasks for the test_group if tasks or tests were not specified.
         if use_test_tasks_membership:
@@ -575,7 +1043,7 @@ def main():
                 tasks_set = tasks_set.union(test_tasks_membership[test])
             tasks = list(tasks_set)
         if not tasks:
-            print("Warning - No tasks found for tests {}, skipping this group.".format(tests))
+            LOGGER.warning("No tasks found for tests %s, skipping this group.", tests)
             continue
 
         test_history = tf.TestHistory(project=options.project,
@@ -588,16 +1056,23 @@ def main():
                                                             end_revision=commit_last)
 
         report = tf.Report(history_data)
-        update_tags(lifecycle, config, report)
+        update_tags(lifecycle_tags_file.changelog_lifecycle, config, report)
 
     # Remove tags that are no longer relevant
-    cleanup_tags(lifecycle, evg_conf)
+    clean_up_tags(lifecycle_tags_file.changelog_lifecycle, evg_conf)
 
     # We write the 'lifecycle' tag configuration to the 'options.lifecycle_file' file only if there
     # have been changes to the tags. In particular, we avoid modifying the file when only the header
     # comment for the YAML file would change.
-    if lifecycle.is_modified():
-        write_yaml_file(options.tag_file, lifecycle)
+    if lifecycle_tags_file.is_modified():
+        lifecycle_tags_file.write()
+
+        if options.commit:
+            commit_ok = lifecycle_tags_file.commit()
+            if not commit_ok:
+                sys.exit(1)
+    else:
+        LOGGER.info("The tags have not been modified.")
 
 
 if __name__ == "__main__":