diff options
Diffstat (limited to 'buildscripts/lifecycle_test_failures.py')
| -rwxr-xr-x | buildscripts/lifecycle_test_failures.py | 732 |
1 files changed, 732 insertions, 0 deletions
diff --git a/buildscripts/lifecycle_test_failures.py b/buildscripts/lifecycle_test_failures.py new file mode 100755 index 00000000000..a0803a78ed5 --- /dev/null +++ b/buildscripts/lifecycle_test_failures.py @@ -0,0 +1,732 @@ +#!/usr/bin/env python +"""Utility for computing test failure rates from the Evergreen API.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import collections +import datetime +import itertools +import logging +import operator +import optparse +import os +import sys +import time +import warnings + +try: + from urlparse import urlparse +except ImportError: + from urllib.parse import urlparse # type: ignore + +import requests +import requests.exceptions +import yaml + +LOGGER = logging.getLogger(__name__) + +if sys.version_info[0] == 2: + _STRING_TYPES = (basestring, ) +else: + _STRING_TYPES = (str, ) + +_ReportEntry = collections.namedtuple("_ReportEntry", [ + "test", + "task", + "variant", + "distro", + "start_date", + "end_date", + "num_pass", + "num_fail", +]) + + +class Wildcard(object): + """Class for representing there are multiple values associated with a particular component.""" + + def __init__(self, kind): + """Initialize Wildcard.""" + self._kind = kind + + def __eq__(self, other): + if not isinstance(other, Wildcard): + return NotImplemented + + return self._kind == other._kind # pylint: disable=protected-access + + def __ne__(self, other): + return not self == other + + def __hash__(self): + return hash(self._kind) + + def __str__(self): + return "<multiple {}>".format(self._kind) + + +class ReportEntry(_ReportEntry): + """Information about Evergreen test executions.""" + + _MULTIPLE_TESTS = Wildcard("tests") + _MULTIPLE_TASKS = Wildcard("tasks") + _MULTIPLE_VARIANTS = Wildcard("variants") + _MULTIPLE_DISTROS = Wildcard("distros") + + _MIN_DATE = datetime.date(datetime.MINYEAR, 1, 1) + _MAX_DATE = datetime.date(datetime.MAXYEAR, 12, 31) + + @property + def fail_rate(self): + """Get the fraction of test failures to total number of test executions. + + If a test hasn't been run at all, then we still say it has a failure rate of 0% for + convenience when applying thresholds. + """ + + if self.num_pass == self.num_fail == 0: + return 0.0 + return self.num_fail / (self.num_pass + self.num_fail) + + def period_start_date(self, start_date, period_size): + """Return a datetime.date() instance for the period start date. + + The result corresponds to the beginning of the time period containing 'self.start_date'. + """ + + if not isinstance(start_date, datetime.date): + raise TypeError("'start_date' argument must be a date") + + if not isinstance(period_size, datetime.timedelta): + raise TypeError("'period_size' argument must a datetime.timedelta instance") + elif period_size.days <= 0: + raise ValueError("'period_size' argument must be a positive number of days") + elif period_size - datetime.timedelta(days=period_size.days) > datetime.timedelta(): + raise ValueError("'period_size' argument must be an integral number of days") + + # 'start_day_offset' is the number of days 'self.start_date' is from the start of the time + # period. + start_day_offset = (self.start_date - start_date).days % period_size.days + return self.start_date - datetime.timedelta(days=start_day_offset) + + def week_start_date(self, start_day_of_week): + """Return a datetime.date() instance of the week's start date. + + The result corresponds to the beginning of the week containing 'self.start_date'. + The first day of the week can be specified as the strings "Sunday" or + "Monday", as well as an arbitrary datetime.date() instance. + """ + + if isinstance(start_day_of_week, _STRING_TYPES): + start_day_of_week = start_day_of_week.lower() + if start_day_of_week == "sunday": + start_weekday = 6 + elif start_day_of_week == "monday": + start_weekday = 0 + else: + raise ValueError( + "'start_day_of_week' can only be the string \"sunday\" or \"monday\"") + elif isinstance(start_day_of_week, datetime.date): + start_weekday = start_day_of_week.weekday() + else: + raise TypeError("'start_day_of_week' argument must be a string or a date") + + # 'start_day_offset' is the number of days 'self.start_date' is from the start of the week. + start_day_offset = (self.start_date.weekday() - start_weekday) % 7 + return self.start_date - datetime.timedelta(days=start_day_offset) + + @classmethod + def sum(cls, entries): + """Return a single ReportEntry() instance. + + The result corresponds to all test executions represented by 'entries'. + """ + + test = set() + task = set() + variant = set() + distro = set() + start_date = cls._MAX_DATE + end_date = cls._MIN_DATE + num_pass = 0 + num_fail = 0 + + for entry in entries: + test.add(entry.test) + task.add(entry.task) + variant.add(entry.variant) + distro.add(entry.distro) + start_date = min(entry.start_date, start_date) + end_date = max(entry.end_date, end_date) + num_pass += entry.num_pass + num_fail += entry.num_fail + + test = next(iter(test)) if len(test) == 1 else ReportEntry._MULTIPLE_TESTS + task = next(iter(task)) if len(task) == 1 else ReportEntry._MULTIPLE_TASKS + variant = next(iter(variant)) if len(variant) == 1 else ReportEntry._MULTIPLE_VARIANTS + distro = next(iter(distro)) if len(distro) == 1 else ReportEntry._MULTIPLE_DISTROS + + return ReportEntry(test=test, task=task, variant=variant, distro=distro, + start_date=start_date, end_date=end_date, num_pass=num_pass, + num_fail=num_fail) + + +class Report(object): + """Class for generating summarizations about Evergreen test executions.""" + + TEST = ("test", ) + TEST_TASK = ("test", "task") + TEST_TASK_VARIANT = ("test", "task", "variant") + TEST_TASK_VARIANT_DISTRO = ("test", "task", "variant", "distro") + + DAILY = "daily" + WEEKLY = "weekly" + + SUNDAY = "sunday" + MONDAY = "monday" + FIRST_DAY = "first-day" + + def __init__(self, entries): + """Initialize the Report instance.""" + + if not isinstance(entries, list): + # It is possible that 'entries' is a generator function, so we convert it to a list in + # order to be able to iterate it multiple times. + entries = list(entries) + + if not entries: + raise ValueError("Report cannot be created with no entries") + + self.start_date = min(entry.start_date for entry in entries) + self.end_date = max(entry.end_date for entry in entries) + + self._entries = entries + + @property + def raw_data(self): + """Get a copy of the list of ReportEntry instances underlying the report.""" + + return self._entries[:] + + def summarize_by( # pylint: disable=too-many-branches,too-many-locals + self, components, time_period=None, start_day_of_week=FIRST_DAY): + """Return a list of ReportEntry instances grouped by the following. + + Grouping: + 'components' if 'time_period' is None, + + 'components' followed by Entry.start_date if 'time_period' is "daily", + + 'components' followed by Entry.week_start_date(start_day_of_week) if 'time_period' is + "weekly". See Entry.week_start_date() for more details on the possible values for + 'start_day_of_week'. + + 'components' followed by Entry.period_start_date(self.start_date, time_period) if + 'time_period' is a datetime.timedelta instance. + """ + + if not isinstance(components, (list, tuple)): + raise TypeError("'components' argument must be a list or tuple") + + for component in components: + if not isinstance(component, _STRING_TYPES): + raise TypeError("Each element of 'components' argument must be a string") + elif component not in ReportEntry._fields: + raise ValueError("Each element of 'components' argument must be one of {}".format( + ReportEntry._fields)) + + group_by = [operator.attrgetter(component) for component in components] + + if start_day_of_week == self.FIRST_DAY: + start_day_of_week = self.start_date + + period_size = None + if isinstance(time_period, _STRING_TYPES): + if time_period == self.DAILY: + group_by.append(operator.attrgetter("start_date")) + period_size = datetime.timedelta(days=1) + elif time_period == self.WEEKLY: + group_by.append(lambda entry: entry.week_start_date(start_day_of_week)) + period_size = datetime.timedelta(days=7) + else: + raise ValueError( + "'time_period' argument can only be the string \"{}\" or \"{}\"".format( + self.DAILY, self.WEEKLY)) + elif isinstance(time_period, datetime.timedelta): + group_by.append(lambda entry: entry.period_start_date(self.start_date, time_period)) + period_size = time_period + elif time_period is not None: + raise TypeError(("'time_period' argument must be a string or a datetime.timedelta" + " instance")) + + def key_func(entry): + """Assign a key for sorting and grouping ReportEntry instances. + + The result is based on the combination of options summarize_by() was called with. + """ + + return [func(entry) for func in group_by] + + sorted_entries = sorted(self._entries, key=key_func) + grouped_entries = itertools.groupby(sorted_entries, key=key_func) + summed_entries = [ReportEntry.sum(group) for (_key, group) in grouped_entries] + + if period_size is not None and period_size.days > 1: + # Overwrite the 'start_date' and 'end_date' attributes so that they correspond to the + # beginning and end of the period, respectively. If the beginning or end of the week + # falls outside the range [self.start_date, self.end_date], then the new 'start_date' + # and 'end_date' attributes are clamped to that range. + for (i, summed_entry) in enumerate(summed_entries): + if time_period == self.WEEKLY: + period_start_date = summed_entry.week_start_date(start_day_of_week) + else: + period_start_date = summed_entry.period_start_date(self.start_date, period_size) + + period_end_date = period_start_date + period_size - datetime.timedelta(days=1) + start_date = max(period_start_date, self.start_date) + end_date = min(period_end_date, self.end_date) + summed_entries[i] = summed_entry._replace(start_date=start_date, end_date=end_date) + + return summed_entries + + +class Missing(object): + """Class for representing the value associated with a particular component is unknown.""" + + def __init__(self, kind): + """Initialize Missing.""" + self._kind = kind + + def __eq__(self, other): + if not isinstance(other, Missing): + return NotImplemented + + return self._kind == other._kind # pylint: disable=protected-access + + def __ne__(self, other): + return not self == other + + def __hash__(self): + return hash(self._kind) + + def __str__(self): + return "<unknown {}>".format(self._kind) + + +class TestHistory(object): + """Class for interacting with the /test_history Evergreen API endpoint.""" + + DEFAULT_API_SERVER = "https://evergreen.mongodb.com" + DEFAULT_PROJECT = "mongodb-mongo-master" + + DEFAULT_TEST_STATUSES = ("pass", "fail", "silentfail") + DEFAULT_TASK_STATUSES = ("success", "failed", "timeout", "sysfail") + + # The Evergreen API requires specifying the "limit" parameter when not specifying a range of + # revisions. + DEFAULT_LIMIT = 20 + + DEFAULT_NUM_RETRIES = 5 + + _MISSING_DISTRO = Missing("distro") + + def __init__( # pylint: disable=too-many-arguments + self, api_server=DEFAULT_API_SERVER, project=DEFAULT_PROJECT, tests=None, tasks=None, + variants=None, distros=None): + """Initialize the TestHistory instance with the list of tests, tasks, variants, and distros. + + The list of tests specified are augmented to ensure that failures on both POSIX and Windows + platforms are returned by the Evergreen API. + """ + + tests = tests if tests is not None else [] + tests = [test for test_file in tests for test in self._denormalize_test_file(test_file)] + + self._tests = tests + self._tasks = tasks if tasks is not None else [] + self._variants = variants if variants is not None else [] + self._distros = distros if distros is not None else [] + + # The number of API call retries on error. It can be set to 0 to disable the feature. + self.num_retries = TestHistory.DEFAULT_NUM_RETRIES + + self._test_history_url = "{api_server}/rest/v1/projects/{project}/test_history".format( + api_server=api_server, + project=project, + ) + + def get_history_by_revision(self, start_revision, end_revision, + test_statuses=DEFAULT_TEST_STATUSES, + task_statuses=DEFAULT_TASK_STATUSES): + """Return a list of ReportEntry instances. + + The result corresponds to each individual test execution between 'start_revision' and + 'end_revision'. + + Only tests with status 'test_statuses' are included in the result. Similarly, only tests + with status 'task_statuses' are included in the result. By default, both passing and failing + test executions are returned. + """ + + params = self._history_request_params(test_statuses, task_statuses) + params["beforeRevision"] = end_revision + + history_data = [] + + # Since the API limits the results, with each invocation being distinct, we can simulate + # pagination by making subsequent requests using "afterRevision". + while start_revision != end_revision: + params["afterRevision"] = start_revision + + test_results = self._get_history(params) + if not test_results: + break + + for test_result in test_results: + history_data.append(self._process_test_result(test_result)) + + # The first test will have the latest revision for this result set because + # TestHistory._history_request_params() sorts by "latest". + start_revision = test_results[0]["revision"] + + return history_data + + def get_history_by_date(self, start_date, end_date, test_statuses=DEFAULT_TEST_STATUSES, + task_statuses=DEFAULT_TASK_STATUSES): + """Return a list of ReportEntry instances. + + The result corresponds to each individual test execution between 'start_date' and + 'end_date'. + + Only tests with status 'test_statuses' are included in the result. Similarly, only tests + with status 'task_statuses' are included in the result. By default, both passing and + failing test executions are returned. + """ + + warnings.warn( + "Until https://jira.mongodb.org/browse/EVG-1653 is implemented, pagination using dates" + " isn't guaranteed to returned a complete result set. It is possible for the results" + " from an Evergreen task that started between the supplied start date and the" + " response's latest test start time to be omitted.", RuntimeWarning) + + params = self._history_request_params(test_statuses, task_statuses) + params["beforeDate"] = "{:%Y-%m-%d}T23:59:59Z".format(end_date) + params["limit"] = self.DEFAULT_LIMIT + + start_time = "{:%Y-%m-%d}T00:00:00Z".format(start_date) + history_data = set() + + # Since the API limits the results, with each invocation being distinct, we can simulate + # pagination by making subsequent requests using "afterDate" and being careful to filter + # out duplicate test results. + while True: + params["afterDate"] = start_time + + test_results = self._get_history(params) + if not test_results: + break + + original_size = len(history_data) + for test_result in test_results: + start_time = max(test_result["start_time"], start_time) + history_data.add(self._process_test_result(test_result)) + + # To prevent an infinite loop, we need to bail out if test results returned by the + # request were identical to the ones we got back in an earlier request. + if original_size == len(history_data): + break + + return list(history_data) + + def _get_history(self, params): + """Call the test_history API endpoint with the given parameters and return the JSON result. + + The API calls will be retried on HTTP and connection errors. + """ + retries = 0 + while True: + try: + LOGGER.debug("Request to the test_history endpoint") + start = time.time() + response = requests.get(url=self._test_history_url, params=params) + LOGGER.debug("Request took %fs", round(time.time() - start, 2)) + response.raise_for_status() + return self._get_json(response) + except (requests.exceptions.HTTPError, requests.exceptions.ConnectionError, + JSONResponseError) as err: + if isinstance(err, JSONResponseError): + err = err.cause + retries += 1 + LOGGER.error("Error while querying the test_history API: %s", str(err)) + if retries > self.num_retries: + raise err + # We use 'retries' as the value for the backoff duration to space out the + # requests when doing multiple retries. + backoff_secs = retries + LOGGER.info("Retrying after %ds", backoff_secs) + time.sleep(backoff_secs) + except: + LOGGER.exception("Unexpected error while querying the test_history API" + " with params: %s", params) + raise + + @staticmethod + def _get_json(response): + try: + return response.json() + except ValueError as err: + # ValueError can be raised if the connection is interrupted and we receive a truncated + # json response. We raise a JSONResponseError instead to distinguish this error from + # other ValueErrors. + raise JSONResponseError(err) + + def _process_test_result(self, test_result): + """Return a ReportEntry() tuple representing the 'test_result' dictionary.""" + + # For individual test executions, we intentionally use the "start_time" of the test as both + # its 'start_date' and 'end_date' to avoid complicating how the test history is potentially + # summarized by time. By the time the test has started, the Evergreen task has already been + # assigned to a particular machine and is using a specific set of binaries, so there's + # unlikely to be a significance to when the test actually finishes. + start_date = end_date = _parse_date(test_result["start_time"]) + + return ReportEntry( + test=self._normalize_test_file(test_result["test_file"]), + task=test_result["task_name"], variant=test_result["variant"], distro=test_result.get( + "distro", self._MISSING_DISTRO), start_date=start_date, end_date=end_date, + num_pass=(1 if test_result["test_status"] == "pass" else + 0), num_fail=(1 if test_result["test_status"] not in ("pass", "skip") else 0)) + + @staticmethod + def _normalize_test_file(test_file): + """Return normalized test_file name. + + If 'test_file' represents a Windows-style path, then it is converted to a POSIX-style path + with + + - backslashes (\\) as the path separator replaced with forward slashes (/) and + - the ".exe" extension, if present, removed. + + If 'test_file' already represents a POSIX-style path, then it is returned unmodified. + """ + + if "\\" in test_file: + posix_test_file = test_file.replace("\\", "/") + (test_file_root, test_file_ext) = os.path.splitext(posix_test_file) + if test_file_ext == ".exe": + return test_file_root + return posix_test_file + + return test_file + + def _denormalize_test_file(self, test_file): + """Return a list containing 'test_file' as both a POSIX-style and a Windows-style path. + + The conversion process may involving replacing forward slashes (/) as the path separator + with backslashes (\\), as well as adding a ".exe" extension if 'test_file' has no file + extension. + """ + + test_file = self._normalize_test_file(test_file) + + if "/" in test_file: + windows_test_file = test_file.replace("/", "\\") + if not os.path.splitext(test_file)[1]: + windows_test_file += ".exe" + return [test_file, windows_test_file] + + return [test_file] + + def _history_request_params(self, test_statuses, task_statuses): + """Return the query parameters for /test_history GET request as a dictionary.""" + + return { + "distros": ",".join(self._distros), + "sort": "latest", + "tasks": ",".join(self._tasks), + "tests": ",".join(self._tests), + "taskStatuses": ",".join(task_statuses), + "testStatuses": ",".join(test_statuses), + "variants": ",".join(self._variants), + } + + +def _parse_date(date_str): + """Return a datetime.date instance representing the specified yyyy-mm-dd date string. + + Note that any time component of 'date_str', including the timezone, is ignored. + """ + # We do not use strptime() because it is not thread safe (https://bugs.python.org/issue7980). + year, month, day = date_str.split("T")[0].split("-") + return datetime.date(int(year), int(month), int(day)) + + +class JSONResponseError(Exception): + """An exception raised when failing to decode the JSON from an Evergreen response.""" + + def __init__(self, cause): # pylint: disable=super-init-not-called + """Initialize the JSONResponseError. + + It it set with the exception raised by the requests library when decoding the response. + """ + self.cause = cause + + +def main(): + """Execute computing test failure rates from the Evergreen API.""" + + parser = optparse.OptionParser(description=main.__doc__, + usage="Usage: %prog [options] [test1 test2 ...]") + + parser.add_option("--project", dest="project", metavar="<project-name>", + default=TestHistory.DEFAULT_PROJECT, + help="The Evergreen project to analyze. Defaults to '%default'.") + + today = datetime.datetime.utcnow().replace(microsecond=0, tzinfo=None) + parser.add_option("--sinceDate", dest="since_date", metavar="<yyyy-mm-dd>", + default="{:%Y-%m-%d}".format(today - datetime.timedelta(days=6)), + help=("The starting period as a date in UTC to analyze the test history for," + " including the specified date. Defaults to 1 week ago (%default).")) + + parser.add_option("--untilDate", dest="until_date", metavar="<yyyy-mm-dd>", + default="{:%Y-%m-%d}".format(today), + help=("The ending period as a date in UTC to analyze the test history for," + " including the specified date. Defaults to today (%default).")) + + parser.add_option("--sinceRevision", dest="since_revision", metavar="<gitrevision>", + default=None, + help=("The starting period as a git revision to analyze the test history for," + " excluding the specified commit. This option must be specified in" + " conjuction with --untilRevision and takes precedence over --sinceDate" + " and --untilDate.")) + + parser.add_option("--untilRevision", dest="until_revision", metavar="<gitrevision>", + default=None, + help=("The ending period as a git revision to analyze the test history for," + " including the specified commit. This option must be specified in" + " conjuction with --sinceRevision and takes precedence over --sinceDate" + " and --untilDate.")) + + parser.add_option("--groupPeriod", dest="group_period", metavar="[{}]".format( + "|".join([Report.DAILY, Report.WEEKLY, "<ndays>"])), default=Report.WEEKLY, + help=("The time period over which to group test executions. Defaults to" + " '%default'.")) + + parser.add_option("--weekStartDay", dest="start_day_of_week", + choices=(Report.SUNDAY, Report.MONDAY, + Report.FIRST_DAY), metavar="[{}]".format( + "|".join([Report.SUNDAY, Report.MONDAY, + Report.FIRST_DAY])), default=Report.FIRST_DAY, + help=("The day to use as the beginning of the week when grouping over time." + " This option is only relevant in conjuction with --groupPeriod={}. If" + " '{}' is specified, then the day of week of the earliest date is used" + " as the beginning of the week. Defaults to '%default'.".format( + Report.WEEKLY, Report.FIRST_DAY))) + + parser.add_option("--tasks", dest="tasks", metavar="<task1,task2,...>", default="", + help="Comma-separated list of Evergreen task names to analyze.") + + parser.add_option("--variants", dest="variants", metavar="<variant1,variant2,...>", default="", + help="Comma-separated list of Evergreen build variants to analyze.") + + parser.add_option("--distros", dest="distros", metavar="<distro1,distro2,...>", default="", + help="Comma-separated list of Evergreen build distros to analyze.") + + parser.add_option("--numRequestRetries", dest="num_request_retries", + metavar="<num-request-retries>", default=TestHistory.DEFAULT_NUM_RETRIES, + help=("The number of times a request to the Evergreen API will be retried on" + " failure. Defaults to '%default'.")) + + (options, tests) = parser.parse_args() + + for (option_name, option_dest) in (("--sinceDate", "since_date"), ("--untilDate", + "until_date")): + option_value = getattr(options, option_dest) + try: + setattr(options, option_dest, _parse_date(option_value)) + except ValueError: + parser.print_help(file=sys.stderr) + print(file=sys.stderr) + parser.error("{} must be specified in yyyy-mm-dd format, but got {}".format( + option_name, option_value)) + + if options.since_revision and not options.until_revision: + parser.print_help(file=sys.stderr) + print(file=sys.stderr) + parser.error("Must specify --untilRevision in conjuction with --sinceRevision") + elif options.until_revision and not options.since_revision: + parser.print_help(file=sys.stderr) + print(file=sys.stderr) + parser.error("Must specify --sinceRevision in conjuction with --untilRevision") + + if options.group_period not in (Report.DAILY, Report.WEEKLY): + try: + options.group_period = datetime.timedelta(days=int(options.group_period)) + except ValueError: + parser.print_help(file=sys.stderr) + print(file=sys.stderr) + parser.error("--groupPeriod must be an integral number, but got {}".format( + options.group_period)) + + if not options.tasks and not tests: + parser.print_help(file=sys.stderr) + print(file=sys.stderr) + parser.error("Must specify either --tasks or at least one test") + + def read_evg_config(): + """ + Attempt to parse the user's or system's Evergreen configuration from its known locations. + + Returns None if the configuration file wasn't found anywhere. + """ + + known_locations = [ + "./.evergreen.yml", + os.path.expanduser("~/.evergreen.yml"), + os.path.expanduser("~/cli_bin/.evergreen.yml"), + ] + + for filename in known_locations: + if os.path.isfile(filename): + with open(filename, "r") as fstream: + return yaml.safe_load(fstream) + + return None + + evg_config = read_evg_config() + evg_config = evg_config if evg_config is not None else {} + api_server = "{url.scheme}://{url.netloc}".format( + url=urlparse(evg_config.get("api_server_host", TestHistory.DEFAULT_API_SERVER))) + + test_history = TestHistory(api_server=api_server, project=options.project, tests=tests, + tasks=options.tasks.split(","), variants=options.variants.split(","), + distros=options.distros.split(",")) + test_history.num_retries = options.num_request_retries + + if options.since_revision: + history_data = test_history.get_history_by_revision(start_revision=options.since_revision, + end_revision=options.until_revision) + elif options.since_date: + history_data = test_history.get_history_by_date(start_date=options.since_date, + end_date=options.until_date) + + report = Report(history_data) + summary = report.summarize_by(Report.TEST_TASK_VARIANT_DISTRO, time_period=options.group_period, + start_day_of_week=options.start_day_of_week) + + for entry in summary: + print("(test={e.test}," + " task={e.task}," + " variant={e.variant}," + " distro={e.distro}," + " start_date={e.start_date:%Y-%m-%d}," + " end_date={e.end_date:%Y-%m-%d}," + " num_pass={e.num_pass}," + " num_fail={e.num_fail}," + " fail_rate={e.fail_rate:0.2%})".format(e=entry)) + + +if __name__ == "__main__": + main() |