diff options
author | Mathew Robinson <chasinglogic@gmail.com> | 2019-02-19 10:50:57 -0500 |
---|---|---|
committer | Mathew Robinson <chasinglogic@gmail.com> | 2019-04-08 14:08:49 -0400 |
commit | 8dd6d4755734ed37c1b98dfdefce3ca6bc65f1f6 (patch) | |
tree | 69e936c4953cbead2e3bae2690157c5fe75e709d /buildscripts/resmokelib | |
parent | c600aa9d7423eca8151daf626e2799d9a6c7b31c (diff) | |
download | mongo-8dd6d4755734ed37c1b98dfdefce3ca6bc65f1f6.tar.gz |
SERVER-32295 Support Python 3
Diffstat (limited to 'buildscripts/resmokelib')
76 files changed, 371 insertions, 499 deletions
diff --git a/buildscripts/resmokelib/__init__.py b/buildscripts/resmokelib/__init__.py index c6a0bc9d079..e24f7425280 100644 --- a/buildscripts/resmokelib/__init__.py +++ b/buildscripts/resmokelib/__init__.py @@ -1,7 +1,5 @@ """Empty.""" -from __future__ import absolute_import - from . import config from . import errors from . import logging diff --git a/buildscripts/resmokelib/config.py b/buildscripts/resmokelib/config.py index 50af5c5ccdd..7705682cce1 100644 --- a/buildscripts/resmokelib/config.py +++ b/buildscripts/resmokelib/config.py @@ -1,7 +1,5 @@ """Configuration options for resmoke.py.""" -from __future__ import absolute_import - import collections import datetime import itertools @@ -75,7 +73,7 @@ DEFAULTS = { "repeat_tests_secs": None, "report_failure_status": "fail", "report_file": None, - "seed": long(time.time() * 256), # Taken from random.py code in Python 2.7. + "seed": int(time.time() * 256), # Taken from random.py code in Python 2.7. "service_executor": None, "shell_conn_string": None, "shell_port": None, @@ -181,18 +179,19 @@ class SuiteOptions(_SuiteOptions): description = None include_tags = None parent = dict( - zip(SuiteOptions._fields, [ - description, - FAIL_FAST, - include_tags, - JOBS, - REPEAT_SUITES, - REPEAT_TESTS, - REPEAT_TESTS_MAX, - REPEAT_TESTS_MIN, - REPEAT_TESTS_SECS, - REPORT_FAILURE_STATUS, - ])) + list( + zip(SuiteOptions._fields, [ + description, + FAIL_FAST, + include_tags, + JOBS, + REPEAT_SUITES, + REPEAT_TESTS, + REPEAT_TESTS_MAX, + REPEAT_TESTS_MIN, + REPEAT_TESTS_SECS, + REPORT_FAILURE_STATUS, + ]))) options = self._asdict() for field in SuiteOptions._fields: @@ -203,7 +202,7 @@ class SuiteOptions(_SuiteOptions): SuiteOptions.ALL_INHERITED = SuiteOptions( # type: ignore - **dict(zip(SuiteOptions._fields, itertools.repeat(SuiteOptions.INHERIT)))) + **dict(list(zip(SuiteOptions._fields, itertools.repeat(SuiteOptions.INHERIT))))) ## # Variables that are set by the user at the command line or with --options. diff --git a/buildscripts/resmokelib/core/__init__.py b/buildscripts/resmokelib/core/__init__.py index 78a8b4924b8..5f780a34e06 100644 --- a/buildscripts/resmokelib/core/__init__.py +++ b/buildscripts/resmokelib/core/__init__.py @@ -1,5 +1,4 @@ """Resmokelib core module.""" -from __future__ import absolute_import from . import process from . import programs diff --git a/buildscripts/resmokelib/core/jasper_process.py b/buildscripts/resmokelib/core/jasper_process.py index 60c349743ff..fc4ef79fa9c 100644 --- a/buildscripts/resmokelib/core/jasper_process.py +++ b/buildscripts/resmokelib/core/jasper_process.py @@ -3,8 +3,6 @@ Serves as an alternative to process.py. """ -from __future__ import absolute_import - import sys try: diff --git a/buildscripts/resmokelib/core/network.py b/buildscripts/resmokelib/core/network.py index b48a2221f01..e76e73bbb08 100644 --- a/buildscripts/resmokelib/core/network.py +++ b/buildscripts/resmokelib/core/network.py @@ -1,7 +1,5 @@ """Class used to allocate ports for mongod and mongos processes involved in running the tests.""" -from __future__ import absolute_import - import collections import functools import threading diff --git a/buildscripts/resmokelib/core/pipe.py b/buildscripts/resmokelib/core/pipe.py index 578b934bb7d..c52dbc1235a 100644 --- a/buildscripts/resmokelib/core/pipe.py +++ b/buildscripts/resmokelib/core/pipe.py @@ -5,8 +5,6 @@ Used to avoid deadlocks from the pipe buffer filling up and blocking the subproc being waited on. """ -from __future__ import absolute_import - import threading diff --git a/buildscripts/resmokelib/core/process.py b/buildscripts/resmokelib/core/process.py index 84c067d8e3b..4b8e0f25d0a 100644 --- a/buildscripts/resmokelib/core/process.py +++ b/buildscripts/resmokelib/core/process.py @@ -4,34 +4,13 @@ Uses job objects when running on Windows to ensure that all created processes are terminated. """ -from __future__ import absolute_import - import atexit import logging import os import os.path import sys import threading - -# The subprocess32 module resolves the thread-safety issues of the subprocess module in Python 2.x -# when the _posixsubprocess C extension module is also available. Additionally, the _posixsubprocess -# C extension module avoids triggering invalid free() calls on Python's internal data structure for -# thread-local storage by skipping the PyOS_AfterFork() call when the 'preexec_fn' parameter isn't -# specified to subprocess.Popen(). See SERVER-22219 for more details. -# -# The subprocess32 module is untested on Windows and thus isn't recommended for use, even when it's -# installed. See https://github.com/google/python-subprocess32/blob/3.2.7/README.md#usage. -if os.name == "posix" and sys.version_info[0] == 2: - try: - import subprocess32 as subprocess - except ImportError: - import warnings - warnings.warn(("Falling back to using the subprocess module because subprocess32 isn't" - " available. When using the subprocess module, a child process may trigger" - " an invalid free(). See SERVER-22219 for more details."), RuntimeWarning) - import subprocess # type: ignore -else: - import subprocess +import subprocess from . import pipe # pylint: disable=wrong-import-position from .. import utils # pylint: disable=wrong-import-position @@ -182,8 +161,8 @@ class Process(object): finally: win32api.CloseHandle(mongo_signal_handle) - print "Failed to cleanly exit the program, calling TerminateProcess() on PID: " +\ - str(self._process.pid) + print("Failed to cleanly exit the program, calling TerminateProcess() on PID: " +\ + str(self._process.pid)) # Adapted from implementation of Popen.terminate() in subprocess.py of Python 2.7 # because earlier versions do not catch exceptions. diff --git a/buildscripts/resmokelib/core/programs.py b/buildscripts/resmokelib/core/programs.py index d1c0b4ca03e..15db85721eb 100644 --- a/buildscripts/resmokelib/core/programs.py +++ b/buildscripts/resmokelib/core/programs.py @@ -3,8 +3,6 @@ Handles all the nitty-gritty parameter conversion. """ -from __future__ import absolute_import - import json import os import os.path diff --git a/buildscripts/resmokelib/logging/__init__.py b/buildscripts/resmokelib/logging/__init__.py index d0b4a48ac57..2d527b91434 100644 --- a/buildscripts/resmokelib/logging/__init__.py +++ b/buildscripts/resmokelib/logging/__init__.py @@ -1,7 +1,5 @@ """Extension to the logging package to support buildlogger.""" -from __future__ import absolute_import - # Alias the built-in logging.Logger class for type checking arguments. Those interested in # constructing a new Logger instance should use the loggers.new_logger() function instead. from logging import Logger diff --git a/buildscripts/resmokelib/logging/buildlogger.py b/buildscripts/resmokelib/logging/buildlogger.py index a6aec6af20e..3c7901dc6fe 100644 --- a/buildscripts/resmokelib/logging/buildlogger.py +++ b/buildscripts/resmokelib/logging/buildlogger.py @@ -1,7 +1,5 @@ """Define handlers for communicating with a buildlogger server.""" -from __future__ import absolute_import - import functools import json import os @@ -85,7 +83,7 @@ class _LogsSplitter(object): 2 is added to each string size to account for the array representation of the logs, as each line is preceded by a '[' or a space and followed by a ',' or a ']'. """ - return len(json.dumps(line, encoding="utf-8")) + 2 + return len(json.dumps(line)) + 2 curr_logs = [] curr_logs_size = 0 @@ -200,8 +198,8 @@ class _BaseBuildloggerHandler(handlers.BufferedHandler): # writing the messages to the fallback logkeeper to avoid putting additional pressure on # the Evergreen database. BUILDLOGGER_FALLBACK.warning( - "Failed to flush all log output (%d messages) to logkeeper.", len( - self.retry_buffer)) + "Failed to flush all log output (%d messages) to logkeeper.", + len(self.retry_buffer)) # We set a flag to indicate that we failed to flush all log output to logkeeper so # resmoke.py can exit with a special return code. @@ -226,10 +224,11 @@ class BuildloggerTestHandler(_BaseBuildloggerHandler): @_log_on_error def _finish_test(self, failed=False): """Send a POST request to the APPEND_TEST_LOGS_ENDPOINT with the test status.""" - self.post(self.endpoint, headers={ - "X-Sendlogs-Test-Done": "true", - "X-Sendlogs-Test-Failed": "true" if failed else "false", - }) + self.post( + self.endpoint, headers={ + "X-Sendlogs-Test-Done": "true", + "X-Sendlogs-Test-Failed": "true" if failed else "false", + }) def close(self): """Close the buildlogger handler.""" @@ -262,7 +261,9 @@ class BuildloggerServer(object): """Initialize BuildloggerServer.""" tmp_globals = {} self.config = {} - execfile(_BUILDLOGGER_CONFIG, tmp_globals, self.config) + exec( + compile(open(_BUILDLOGGER_CONFIG, "rb").read(), _BUILDLOGGER_CONFIG, 'exec'), + tmp_globals, self.config) # Rename "slavename" to "username" if present. if "slavename" in self.config and "username" not in self.config: @@ -285,11 +286,12 @@ class BuildloggerServer(object): handler = handlers.HTTPHandler(url_root=_config.BUILDLOGGER_URL, username=username, password=password, should_retry=True) - response = handler.post(CREATE_BUILD_ENDPOINT, data={ - "builder": builder, - "buildnum": build_num, - "task_id": _config.EVERGREEN_TASK_ID, - }) + response = handler.post( + CREATE_BUILD_ENDPOINT, data={ + "builder": builder, + "buildnum": build_num, + "task_id": _config.EVERGREEN_TASK_ID, + }) return response["id"] diff --git a/buildscripts/resmokelib/logging/flush.py b/buildscripts/resmokelib/logging/flush.py index f5c2b356468..b812fc2b606 100644 --- a/buildscripts/resmokelib/logging/flush.py +++ b/buildscripts/resmokelib/logging/flush.py @@ -3,14 +3,11 @@ These instances are used to send logs to buildlogger. """ -from __future__ import absolute_import - import logging +import sched import threading import time -from ..utils import scheduler - _FLUSH_THREAD_LOCK = threading.Lock() _FLUSH_THREAD = None @@ -96,7 +93,7 @@ class _FlushThread(threading.Thread): self.__schedule_updated.wait(secs) self.__schedule_updated.clear() - self.__scheduler = scheduler.Scheduler(time.time, interruptible_sleep) + self.__scheduler = sched.scheduler(time.time, interruptible_sleep) self.__schedule_updated = threading.Event() self.__should_stop = threading.Event() self.__terminated = threading.Event() diff --git a/buildscripts/resmokelib/logging/formatters.py b/buildscripts/resmokelib/logging/formatters.py index 450d5d29cd8..b08e736ddcc 100644 --- a/buildscripts/resmokelib/logging/formatters.py +++ b/buildscripts/resmokelib/logging/formatters.py @@ -1,7 +1,5 @@ """Custom formatters for the logging handlers.""" -from __future__ import absolute_import - import logging import time diff --git a/buildscripts/resmokelib/logging/handlers.py b/buildscripts/resmokelib/logging/handlers.py index d67bf13f724..fecbde906fa 100644 --- a/buildscripts/resmokelib/logging/handlers.py +++ b/buildscripts/resmokelib/logging/handlers.py @@ -1,7 +1,5 @@ """Additional handlers that are used as the base classes of the buildlogger handler.""" -from __future__ import absolute_import - import json import logging import sys @@ -193,18 +191,15 @@ class HTTPHandler(object): """ data = utils.default_if_none(data, []) - data = json.dumps(data, encoding="utf-8") + data = json.dumps(data) headers = utils.default_if_none(headers, {}) headers["Content-Type"] = "application/json; charset=utf-8" url = self._make_url(endpoint) - # Versions of Python earlier than 2.7.9 do not support certificate validation. So we - # disable certificate validation for older Python versions. - should_validate_certificates = sys.version_info >= (2, 7, 9) with warnings.catch_warnings(): - if urllib3_exceptions is not None and not should_validate_certificates: + if urllib3_exceptions is not None: try: warnings.simplefilter("ignore", urllib3_exceptions.InsecurePlatformWarning) except AttributeError: @@ -222,8 +217,7 @@ class HTTPHandler(object): pass response = self.session.post(url, data=data, headers=headers, timeout=timeout_secs, - auth=self.auth_handler, - verify=should_validate_certificates) + auth=self.auth_handler, verify=True) response.raise_for_status() diff --git a/buildscripts/resmokelib/logging/loggers.py b/buildscripts/resmokelib/logging/loggers.py index deb3f6c38a2..4028925af5a 100644 --- a/buildscripts/resmokelib/logging/loggers.py +++ b/buildscripts/resmokelib/logging/loggers.py @@ -1,7 +1,5 @@ """Module to hold the logger instances themselves.""" -from __future__ import absolute_import - import logging import sys @@ -304,8 +302,8 @@ class FixtureNodeLogger(BaseLogger): def new_fixture_node_logger(self, node_name): """Create a new child FixtureNodeLogger.""" - return FixtureNodeLogger(self.fixture_class, self.job_num, "%s:%s" % (self.node_name, - node_name), self) + return FixtureNodeLogger(self.fixture_class, self.job_num, + "%s:%s" % (self.node_name, node_name), self) class TestsRootLogger(RootLogger): diff --git a/buildscripts/resmokelib/parser.py b/buildscripts/resmokelib/parser.py index db30a407b8c..80e3b36e516 100644 --- a/buildscripts/resmokelib/parser.py +++ b/buildscripts/resmokelib/parser.py @@ -1,7 +1,5 @@ """Parser for command line arguments.""" -from __future__ import absolute_import - import collections import os import os.path @@ -26,40 +24,45 @@ def _make_parser(): # pylint: disable=too-many-statements """Create and return the command line arguments parser.""" parser = optparse.OptionParser() - parser.add_option("--suites", dest="suite_files", metavar="SUITE1,SUITE2", - help=("Comma separated list of YAML files that each specify the configuration" - " of a suite. If the file is located in the resmokeconfig/suites/" - " directory, then the basename without the .yml extension can be" - " specified, e.g. 'core'. If a list of files is passed in as" - " positional arguments, they will be run using the suites'" - " configurations")) - - parser.add_option("--log", dest="logger_file", metavar="LOGGER", - help=("A YAML file that specifies the logging configuration. If the file is" - " located in the resmokeconfig/suites/ directory, then the basename" - " without the .yml extension can be specified, e.g. 'console'.")) - - parser.add_option("--archiveFile", dest="archive_file", metavar="ARCHIVE_FILE", - help=("Sets the archive file name for the Evergreen task running the tests." - " The archive file is JSON format containing a list of tests that were" - " successfully archived to S3. If unspecified, no data files from tests" - " will be archived in S3. Tests can be designated for archival in the" - " task suite configuration file.")) - - parser.add_option("--archiveLimitMb", type="int", dest="archive_limit_mb", - metavar="ARCHIVE_LIMIT_MB", - help=("Sets the limit (in MB) for archived files to S3. A value of 0" - " indicates there is no limit.")) - - parser.add_option("--archiveLimitTests", type="int", dest="archive_limit_tests", - metavar="ARCHIVE_LIMIT_TESTS", - help=("Sets the maximum number of tests to archive to S3. A value" - " of 0 indicates there is no limit.")) - - parser.add_option("--basePort", dest="base_port", metavar="PORT", - help=("The starting port number to use for mongod and mongos processes" - " spawned by resmoke.py or the tests themselves. Each fixture and Job" - " allocates a contiguous range of ports.")) + parser.add_option( + "--suites", dest="suite_files", metavar="SUITE1,SUITE2", + help=("Comma separated list of YAML files that each specify the configuration" + " of a suite. If the file is located in the resmokeconfig/suites/" + " directory, then the basename without the .yml extension can be" + " specified, e.g. 'core'. If a list of files is passed in as" + " positional arguments, they will be run using the suites'" + " configurations")) + + parser.add_option( + "--log", dest="logger_file", metavar="LOGGER", + help=("A YAML file that specifies the logging configuration. If the file is" + " located in the resmokeconfig/suites/ directory, then the basename" + " without the .yml extension can be specified, e.g. 'console'.")) + + parser.add_option( + "--archiveFile", dest="archive_file", metavar="ARCHIVE_FILE", + help=("Sets the archive file name for the Evergreen task running the tests." + " The archive file is JSON format containing a list of tests that were" + " successfully archived to S3. If unspecified, no data files from tests" + " will be archived in S3. Tests can be designated for archival in the" + " task suite configuration file.")) + + parser.add_option( + "--archiveLimitMb", type="int", dest="archive_limit_mb", metavar="ARCHIVE_LIMIT_MB", + help=("Sets the limit (in MB) for archived files to S3. A value of 0" + " indicates there is no limit.")) + + parser.add_option( + "--archiveLimitTests", type="int", dest="archive_limit_tests", + metavar="ARCHIVE_LIMIT_TESTS", + help=("Sets the maximum number of tests to archive to S3. A value" + " of 0 indicates there is no limit.")) + + parser.add_option( + "--basePort", dest="base_port", metavar="PORT", + help=("The starting port number to use for mongod and mongos processes" + " spawned by resmoke.py or the tests themselves. Each fixture and Job" + " allocates a contiguous range of ports.")) parser.add_option("--buildloggerUrl", action="store", dest="buildlogger_url", metavar="URL", help="The root url of the buildlogger server.") @@ -67,19 +70,19 @@ def _make_parser(): # pylint: disable=too-many-statements parser.add_option("--continueOnFailure", action="store_true", dest="continue_on_failure", help="Executes all tests in all suites, even if some of them fail.") - parser.add_option("--dbpathPrefix", dest="dbpath_prefix", metavar="PATH", - help=("The directory which will contain the dbpaths of any mongod's started" - " by resmoke.py or the tests themselves.")) + parser.add_option( + "--dbpathPrefix", dest="dbpath_prefix", metavar="PATH", + help=("The directory which will contain the dbpaths of any mongod's started" + " by resmoke.py or the tests themselves.")) parser.add_option("--dbtest", dest="dbtest_executable", metavar="PATH", help="The path to the dbtest executable for resmoke to use.") - parser.add_option("--excludeWithAnyTags", action="append", dest="exclude_with_any_tags", - metavar="TAG1,TAG2", - help=("Comma separated list of tags. Any jstest that contains any of the" - " specified tags will be excluded from any suites that are run." - " The tag '{}' is implicitly part of this list.".format( - _config.EXCLUDED_TAG))) + parser.add_option( + "--excludeWithAnyTags", action="append", dest="exclude_with_any_tags", metavar="TAG1,TAG2", + help=("Comma separated list of tags. Any jstest that contains any of the" + " specified tags will be excluded from any suites that are run." + " The tag '{}' is implicitly part of this list.".format(_config.EXCLUDED_TAG))) parser.add_option("-f", "--findSuites", action="store_true", dest="find_suites", help="Lists the names of the suites that will execute the specified tests.") @@ -87,29 +90,30 @@ def _make_parser(): # pylint: disable=too-many-statements parser.add_option("--genny", dest="genny_executable", metavar="PATH", help="The path to the genny executable for resmoke to use.") - parser.add_option("--spawnUsing", type="choice", dest="spawn_using", choices=("python", - "jasper"), - help=("Allows you to spawn resmoke processes using python or Jasper." - "Defaults to python. Options are 'python' or 'jasper'.")) + parser.add_option( + "--spawnUsing", type="choice", dest="spawn_using", choices=("python", "jasper"), + help=("Allows you to spawn resmoke processes using python or Jasper." + "Defaults to python. Options are 'python' or 'jasper'.")) - parser.add_option("--includeWithAnyTags", action="append", dest="include_with_any_tags", - metavar="TAG1,TAG2", - help=("Comma separated list of tags. For the jstest portion of the suite(s)," - " only tests which have at least one of the specified tags will be" - " run.")) + parser.add_option( + "--includeWithAnyTags", action="append", dest="include_with_any_tags", metavar="TAG1,TAG2", + help=("Comma separated list of tags. For the jstest portion of the suite(s)," + " only tests which have at least one of the specified tags will be" + " run.")) parser.add_option("-n", action="store_const", const="tests", dest="dry_run", help="Outputs the tests that would be run.") # TODO: add support for --dryRun=commands - parser.add_option("--dryRun", type="choice", action="store", dest="dry_run", - choices=("off", "tests"), metavar="MODE", - help=("Instead of running the tests, outputs the tests that would be run" - " (if MODE=tests). Defaults to MODE=%default.")) + parser.add_option( + "--dryRun", type="choice", action="store", dest="dry_run", choices=("off", "tests"), + metavar="MODE", help=("Instead of running the tests, outputs the tests that would be run" + " (if MODE=tests). Defaults to MODE=%default.")) - parser.add_option("-j", "--jobs", type="int", dest="jobs", metavar="JOBS", - help=("The number of Job instances to use. Each instance will receive its" - " own MongoDB deployment to dispatch tests to.")) + parser.add_option( + "-j", "--jobs", type="int", dest="jobs", metavar="JOBS", + help=("The number of Job instances to use. Each instance will receive its" + " own MongoDB deployment to dispatch tests to.")) parser.add_option("-l", "--listSuites", action="store_true", dest="list_suites", help="Lists the names of the suites available to execute.") @@ -120,24 +124,27 @@ def _make_parser(): # pylint: disable=too-many-statements parser.add_option("--mongod", dest="mongod_executable", metavar="PATH", help="The path to the mongod executable for resmoke.py to use.") - parser.add_option("--mongodSetParameters", dest="mongod_set_parameters", - metavar="{key1: value1, key2: value2, ..., keyN: valueN}", - help=("Passes one or more --setParameter options to all mongod processes" - " started by resmoke.py. The argument is specified as bracketed YAML -" - " i.e. JSON with support for single quoted and unquoted keys.")) + parser.add_option( + "--mongodSetParameters", dest="mongod_set_parameters", + metavar="{key1: value1, key2: value2, ..., keyN: valueN}", + help=("Passes one or more --setParameter options to all mongod processes" + " started by resmoke.py. The argument is specified as bracketed YAML -" + " i.e. JSON with support for single quoted and unquoted keys.")) - parser.add_option("--mongoebench", dest="mongoebench_executable", metavar="PATH", - help=("The path to the mongoebench (benchrun embedded) executable for" - " resmoke.py to use.")) + parser.add_option( + "--mongoebench", dest="mongoebench_executable", metavar="PATH", + help=("The path to the mongoebench (benchrun embedded) executable for" + " resmoke.py to use.")) parser.add_option("--mongos", dest="mongos_executable", metavar="PATH", help="The path to the mongos executable for resmoke.py to use.") - parser.add_option("--mongosSetParameters", dest="mongos_set_parameters", - metavar="{key1: value1, key2: value2, ..., keyN: valueN}", - help=("Passes one or more --setParameter options to all mongos processes" - " started by resmoke.py. The argument is specified as bracketed YAML -" - " i.e. JSON with support for single quoted and unquoted keys.")) + parser.add_option( + "--mongosSetParameters", dest="mongos_set_parameters", + metavar="{key1: value1, key2: value2, ..., keyN: valueN}", + help=("Passes one or more --setParameter options to all mongos processes" + " started by resmoke.py. The argument is specified as bracketed YAML -" + " i.e. JSON with support for single quoted and unquoted keys.")) parser.add_option("--nojournal", action="store_true", dest="no_journal", help="Disables journaling for all mongod's.") @@ -148,55 +155,61 @@ def _make_parser(): # pylint: disable=too-many-statements parser.add_option("--perfReportFile", dest="perf_report_file", metavar="PERF_REPORT", help="Writes a JSON file with performance test results.") - parser.add_option("--shellConnString", dest="shell_conn_string", metavar="CONN_STRING", - help="Overrides the default fixture and connects with a mongodb:// connection" - " string to an existing MongoDB cluster instead. This is useful for" - " connecting to a MongoDB deployment started outside of resmoke.py including" - " one running in a debugger.") + parser.add_option( + "--shellConnString", dest="shell_conn_string", metavar="CONN_STRING", + help="Overrides the default fixture and connects with a mongodb:// connection" + " string to an existing MongoDB cluster instead. This is useful for" + " connecting to a MongoDB deployment started outside of resmoke.py including" + " one running in a debugger.") - parser.add_option("--shellPort", dest="shell_port", metavar="PORT", - help="Convenience form of --shellConnString for connecting to an" - " existing MongoDB cluster with the URL mongodb://localhost:[PORT]." - " This is useful for connecting to a server running in a debugger.") + parser.add_option( + "--shellPort", dest="shell_port", metavar="PORT", + help="Convenience form of --shellConnString for connecting to an" + " existing MongoDB cluster with the URL mongodb://localhost:[PORT]." + " This is useful for connecting to a server running in a debugger.") parser.add_option("--repeat", "--repeatSuites", type="int", dest="repeat_suites", metavar="N", help="Repeats the given suite(s) N times, or until one fails.") - parser.add_option("--repeatTests", type="int", dest="repeat_tests", metavar="N", - help="Repeats the tests inside each suite N times. This applies to tests" - " defined in the suite configuration as well as tests defined on the command" - " line.") - - parser.add_option("--repeatTestsMax", type="int", dest="repeat_tests_max", metavar="N", - help="Repeats the tests inside each suite no more than N time when" - " --repeatTestsSecs is specified. This applies to tests defined in the suite" - " configuration as well as tests defined on the command line.") - - parser.add_option("--repeatTestsMin", type="int", dest="repeat_tests_min", metavar="N", - help="Repeats the tests inside each suite at least N times when" - " --repeatTestsSecs is specified. This applies to tests defined in the suite" - " configuration as well as tests defined on the command line.") - - parser.add_option("--repeatTestsSecs", type="float", dest="repeat_tests_secs", - metavar="SECONDS", - help="Repeats the tests inside each suite this amount of time. Note that" - " this option is mutually exclusive with --repeatTests. This applies to" - " tests defined in the suite configuration as well as tests defined on the" - " command line.") - - parser.add_option("--reportFailureStatus", type="choice", action="store", - dest="report_failure_status", choices=("fail", - "silentfail"), metavar="STATUS", - help="Controls if the test failure status should be reported as failed" - " or be silently ignored (STATUS=silentfail). Dynamic test failures will" - " never be silently ignored. Defaults to STATUS=%default.") + parser.add_option( + "--repeatTests", type="int", dest="repeat_tests", metavar="N", + help="Repeats the tests inside each suite N times. This applies to tests" + " defined in the suite configuration as well as tests defined on the command" + " line.") + + parser.add_option( + "--repeatTestsMax", type="int", dest="repeat_tests_max", metavar="N", + help="Repeats the tests inside each suite no more than N time when" + " --repeatTestsSecs is specified. This applies to tests defined in the suite" + " configuration as well as tests defined on the command line.") + + parser.add_option( + "--repeatTestsMin", type="int", dest="repeat_tests_min", metavar="N", + help="Repeats the tests inside each suite at least N times when" + " --repeatTestsSecs is specified. This applies to tests defined in the suite" + " configuration as well as tests defined on the command line.") + + parser.add_option( + "--repeatTestsSecs", type="float", dest="repeat_tests_secs", metavar="SECONDS", + help="Repeats the tests inside each suite this amount of time. Note that" + " this option is mutually exclusive with --repeatTests. This applies to" + " tests defined in the suite configuration as well as tests defined on the" + " command line.") + + parser.add_option( + "--reportFailureStatus", type="choice", action="store", dest="report_failure_status", + choices=("fail", "silentfail"), metavar="STATUS", + help="Controls if the test failure status should be reported as failed" + " or be silently ignored (STATUS=silentfail). Dynamic test failures will" + " never be silently ignored. Defaults to STATUS=%default.") parser.add_option("--reportFile", dest="report_file", metavar="REPORT", help="Writes a JSON file with test status and timing information.") - parser.add_option("--seed", type="int", dest="seed", metavar="SEED", - help=("Seed for the random number generator. Useful in combination with the" - " --shuffle option for producing a consistent test execution order.")) + parser.add_option( + "--seed", type="int", dest="seed", metavar="SEED", + help=("Seed for the random number generator. Useful in combination with the" + " --shuffle option for producing a consistent test execution order.")) parser.add_option("--serviceExecutor", dest="service_executor", metavar="EXECUTOR", help="The service executor used by jstests") @@ -212,32 +225,36 @@ def _make_parser(): # pylint: disable=too-many-statements choices=("commands", "compatibility", "legacy"), metavar="WRITE_MODE", help="The write mode used by the mongo shell.") - parser.add_option("--shuffle", action="store_const", const="on", dest="shuffle", - help=("Randomizes the order in which tests are executed. This is equivalent" - " to specifying --shuffleMode=on.")) - - parser.add_option("--shuffleMode", type="choice", action="store", dest="shuffle", - choices=("on", "off", "auto"), metavar="ON|OFF|AUTO", - help=("Controls whether to randomize the order in which tests are executed." - " Defaults to auto when not supplied. auto enables randomization in" - " all cases except when the number of jobs requested is 1.")) - - parser.add_option("--staggerJobs", type="choice", action="store", dest="stagger_jobs", - choices=("on", "off"), metavar="ON|OFF", - help=("Enables or disables the stagger of launching resmoke jobs." - " Defaults to %default.")) - - parser.add_option("--majorityReadConcern", type="choice", action="store", - dest="majority_read_concern", choices=("on", "off"), metavar="ON|OFF", - help=("Enable or disable majority read concern support." - " Defaults to %default.")) + parser.add_option( + "--shuffle", action="store_const", const="on", dest="shuffle", + help=("Randomizes the order in which tests are executed. This is equivalent" + " to specifying --shuffleMode=on.")) + + parser.add_option( + "--shuffleMode", type="choice", action="store", dest="shuffle", + choices=("on", "off", "auto"), metavar="ON|OFF|AUTO", + help=("Controls whether to randomize the order in which tests are executed." + " Defaults to auto when not supplied. auto enables randomization in" + " all cases except when the number of jobs requested is 1.")) + + parser.add_option( + "--staggerJobs", type="choice", action="store", dest="stagger_jobs", choices=("on", "off"), + metavar="ON|OFF", help=("Enables or disables the stagger of launching resmoke jobs." + " Defaults to %default.")) + + parser.add_option( + "--majorityReadConcern", type="choice", action="store", dest="majority_read_concern", + choices=("on", + "off"), metavar="ON|OFF", help=("Enable or disable majority read concern support." + " Defaults to %default.")) parser.add_option("--storageEngine", dest="storage_engine", metavar="ENGINE", help="The storage engine used by dbtests and jstests.") - parser.add_option("--storageEngineCacheSizeGB", dest="storage_engine_cache_size_gb", - metavar="CONFIG", help="Sets the storage engine cache size configuration" - " setting for all mongod's.") + parser.add_option( + "--storageEngineCacheSizeGB", dest="storage_engine_cache_size_gb", metavar="CONFIG", + help="Sets the storage engine cache size configuration" + " setting for all mongod's.") parser.add_option("--tagFile", dest="tag_file", metavar="OPTIONS", help="A YAML file that associates tests and tags.") @@ -251,9 +268,10 @@ def _make_parser(): # pylint: disable=too-many-statements parser.add_option("--wiredTigerIndexConfigString", dest="wt_index_config", metavar="CONFIG", help="Sets the WiredTiger index configuration setting for all mongod's.") - parser.add_option("--executor", dest="executor_file", - help="OBSOLETE: Superceded by --suites; specify --suites=SUITE path/to/test" - " to run a particular test under a particular suite configuration.") + parser.add_option( + "--executor", dest="executor_file", + help="OBSOLETE: Superceded by --suites; specify --suites=SUITE path/to/test" + " to run a particular test under a particular suite configuration.") evergreen_options = optparse.OptionGroup( parser, title=_EVERGREEN_OPTIONS_TITLE, @@ -264,29 +282,33 @@ def _make_parser(): # pylint: disable=too-many-statements evergreen_options.add_option("--buildId", dest="build_id", metavar="BUILD_ID", help="Sets the build ID of the task.") - evergreen_options.add_option("--distroId", dest="distro_id", metavar="DISTRO_ID", - help=("Sets the identifier for the Evergreen distro running the" - " tests.")) + evergreen_options.add_option( + "--distroId", dest="distro_id", metavar="DISTRO_ID", + help=("Sets the identifier for the Evergreen distro running the" + " tests.")) - evergreen_options.add_option("--executionNumber", type="int", dest="execution_number", - metavar="EXECUTION_NUMBER", - help=("Sets the number for the Evergreen execution running the" - " tests.")) + evergreen_options.add_option( + "--executionNumber", type="int", dest="execution_number", metavar="EXECUTION_NUMBER", + help=("Sets the number for the Evergreen execution running the" + " tests.")) - evergreen_options.add_option("--gitRevision", dest="git_revision", metavar="GIT_REVISION", - help=("Sets the git revision for the Evergreen task running the" - " tests.")) + evergreen_options.add_option( + "--gitRevision", dest="git_revision", metavar="GIT_REVISION", + help=("Sets the git revision for the Evergreen task running the" + " tests.")) # We intentionally avoid adding a new command line option that starts with --suite so it doesn't # become ambiguous with the --suites option and break how engineers run resmoke.py locally. - evergreen_options.add_option("--originSuite", dest="origin_suite", metavar="SUITE", - help=("Indicates the name of the test suite prior to the" - " evergreen_generate_resmoke_tasks.py script splitting it" - " up.")) + evergreen_options.add_option( + "--originSuite", dest="origin_suite", metavar="SUITE", + help=("Indicates the name of the test suite prior to the" + " evergreen_generate_resmoke_tasks.py script splitting it" + " up.")) - evergreen_options.add_option("--patchBuild", action="store_true", dest="patch_build", - help=("Indicates that the Evergreen task running the tests is a" - " patch build.")) + evergreen_options.add_option( + "--patchBuild", action="store_true", dest="patch_build", + help=("Indicates that the Evergreen task running the tests is a" + " patch build.")) evergreen_options.add_option("--projectName", dest="project_name", metavar="PROJECT_NAME", help=("Sets the name of the Evergreen project running the tests.")) @@ -301,9 +323,10 @@ def _make_parser(): # pylint: disable=too-many-statements evergreen_options.add_option("--taskId", dest="task_id", metavar="TASK_ID", help="Sets the Id of the Evergreen task running the tests.") - evergreen_options.add_option("--variantName", dest="variant_name", metavar="VARIANT_NAME", - help=("Sets the name of the Evergreen build variant running the" - " tests.")) + evergreen_options.add_option( + "--variantName", dest="variant_name", metavar="VARIANT_NAME", + help=("Sets the name of the Evergreen build variant running the" + " tests.")) evergreen_options.add_option("--versionId", dest="version_id", metavar="VERSION_ID", help="Sets the version ID of the task.") @@ -318,10 +341,11 @@ def _make_parser(): # pylint: disable=too-many-statements metavar="BENCHMARK_FILTER", help="Regex to filter Google benchmark tests to run.") - benchmark_options.add_option("--benchmarkListTests", dest="benchmark_list_tests", - action="store_true", metavar="BENCHMARK_LIST_TESTS", - help=("Lists all Google benchmark test configurations in each" - " test file.")) + benchmark_options.add_option( + "--benchmarkListTests", dest="benchmark_list_tests", action="store_true", + metavar="BENCHMARK_LIST_TESTS", + help=("Lists all Google benchmark test configurations in each" + " test file.")) benchmark_min_time_help = ( "Minimum time to run each benchmark/benchrun test for. Use this option instead of " @@ -339,10 +363,10 @@ def _make_parser(): # pylint: disable=too-many-statements metavar="BENCHMARK_REPETITIONS", help=benchmark_repetitions_help) benchrun_devices = ["Android", "Desktop"] - benchmark_options.add_option("--benchrunDevice", dest="benchrun_device", metavar="DEVICE", - type="choice", action="store", choices=benchrun_devices, - help=("The device to run the benchrun test on, choose from {}." - " Defaults to DEVICE='%default'.".format(benchrun_devices))) + benchmark_options.add_option( + "--benchrunDevice", dest="benchrun_device", metavar="DEVICE", type="choice", action="store", + choices=benchrun_devices, help=("The device to run the benchrun test on, choose from {}." + " Defaults to DEVICE='%default'.".format(benchrun_devices))) benchmark_options.add_option("--benchrunReportRoot", dest="benchrun_report_root", metavar="PATH", help="The root path for benchrun test report.") @@ -436,8 +460,8 @@ def to_local_args(args=None): # pylint: disable=too-many-locals else: other_local_args.append(option_name) - return [arg for arg in (suites_arg, storage_engine_arg) - if arg is not None] + other_local_args + extra_args + return [arg for arg in (suites_arg, storage_engine_arg) if arg is not None + ] + other_local_args + extra_args def parse_command_line(): @@ -613,7 +637,7 @@ def _update_config_vars(values): # pylint: disable=too-many-statements _config.SHELL_CONN_STRING = conn_string if config: - raise optparse.OptionValueError("Unknown option(s): %s" % (config.keys())) + raise optparse.OptionValueError("Unknown option(s): %s" % (list(config.keys()))) def _get_logging_config(pathname): diff --git a/buildscripts/resmokelib/reportfile.py b/buildscripts/resmokelib/reportfile.py index 00841de2bc9..d4346b349f1 100644 --- a/buildscripts/resmokelib/reportfile.py +++ b/buildscripts/resmokelib/reportfile.py @@ -1,7 +1,5 @@ """Manage interactions with the report.json file.""" -from __future__ import absolute_import - import json from . import config diff --git a/buildscripts/resmokelib/selector.py b/buildscripts/resmokelib/selector.py index b9f5686df8b..36f2e608889 100644 --- a/buildscripts/resmokelib/selector.py +++ b/buildscripts/resmokelib/selector.py @@ -4,8 +4,6 @@ Defines filtering rules for what tests to include in a suite depending on whether they apply to C++ unit tests, dbtests, or JS tests. """ -from __future__ import absolute_import - import collections import errno import fnmatch @@ -71,7 +69,7 @@ class TestFileExplorer(object): A list of paths as a list(str). """ tests = [] - with open(root_file_path, "rb") as filep: + with open(root_file_path, "r") as filep: for test_path in filep: test_path = test_path.strip() tests.append(test_path) @@ -114,7 +112,7 @@ class TestFileExplorer(object): program = subprocess.Popen(command, stdout=subprocess.PIPE) stdout = program.communicate()[0] - return program.returncode, stdout + return program.returncode, stdout.decode("utf-8") @staticmethod def parse_tag_file(test_kind): @@ -313,7 +311,7 @@ def make_expression(conf): elif isinstance(conf, dict): if len(conf) != 1: raise ValueError("Tag matching expressions should only contain one key") - key = conf.keys()[0] + key = list(conf.keys())[0] value = conf[key] if key == "$allOf": return _AllOfExpression(_make_expression_list(value)) diff --git a/buildscripts/resmokelib/sighandler.py b/buildscripts/resmokelib/sighandler.py index c67d44eb759..5591e325a11 100644 --- a/buildscripts/resmokelib/sighandler.py +++ b/buildscripts/resmokelib/sighandler.py @@ -1,7 +1,5 @@ """Utility to support asynchronously signaling the current process.""" -from __future__ import absolute_import - import atexit import os import signal diff --git a/buildscripts/resmokelib/suitesconfig.py b/buildscripts/resmokelib/suitesconfig.py index 156f15e1413..d55ce7782dd 100644 --- a/buildscripts/resmokelib/suitesconfig.py +++ b/buildscripts/resmokelib/suitesconfig.py @@ -1,7 +1,5 @@ """Module for retrieving the configuration of resmoke.py test suites.""" -from __future__ import absolute_import - import collections import optparse import os @@ -33,7 +31,7 @@ def create_test_membership_map(fail_on_missing_selector=False, test_kind=None): """ if test_kind is not None: - if isinstance(test_kind, basestring): + if isinstance(test_kind, str): test_kind = [test_kind] test_kind = frozenset(test_kind) @@ -117,6 +115,6 @@ def _get_yaml_config(kind, pathname): pathname = resmokeconfig.NAMED_SUITES[pathname] # Expand 'pathname' to full path. if not utils.is_yaml_file(pathname) or not os.path.isfile(pathname): - raise optparse.OptionValueError("Expected a %s YAML config, but got '%s'" % (kind, - pathname)) + raise optparse.OptionValueError( + "Expected a %s YAML config, but got '%s'" % (kind, pathname)) return utils.load_yaml_file(pathname) diff --git a/buildscripts/resmokelib/testing/__init__.py b/buildscripts/resmokelib/testing/__init__.py index eb58f41f7fe..2a276f6bf85 100644 --- a/buildscripts/resmokelib/testing/__init__.py +++ b/buildscripts/resmokelib/testing/__init__.py @@ -1,6 +1,4 @@ """Extension to the unittest package to support buildlogger and parallel test execution.""" -from __future__ import absolute_import - from . import executor from . import suite diff --git a/buildscripts/resmokelib/testing/executor.py b/buildscripts/resmokelib/testing/executor.py index 024cce166a4..71fcd75f6d1 100644 --- a/buildscripts/resmokelib/testing/executor.py +++ b/buildscripts/resmokelib/testing/executor.py @@ -1,7 +1,5 @@ """Driver of the test execution framework.""" -from __future__ import absolute_import - import threading import time @@ -68,7 +66,7 @@ class TestSuiteExecutor(object): # pylint: disable=too-many-instance-attributes jobs_to_start = self.num_tests # Must be done after getting buildlogger configuration. - self._jobs = [self._make_job(job_num) for job_num in xrange(jobs_to_start)] + self._jobs = [self._make_job(job_num) for job_num in range(jobs_to_start)] def run(self): """Execute the test suite. @@ -130,8 +128,9 @@ class TestSuiteExecutor(object): # pylint: disable=too-many-instance-attributes test_results_num = len(test_report["results"]) # There should be at least as many tests results as expected number of tests. if test_results_num < self.num_tests: - raise errors.ResmokeError("{} reported tests is less than {} expected tests" - .format(test_results_num, self.num_tests)) + raise errors.ResmokeError( + "{} reported tests is less than {} expected tests".format( + test_results_num, self.num_tests)) # Clear the report so it can be reused for the next execution. for job in self._jobs: @@ -157,8 +156,9 @@ class TestSuiteExecutor(object): # pylint: disable=too-many-instance-attributes try: # Run each Job instance in its own thread. for job in self._jobs: - thr = threading.Thread(target=job, args=(test_queue, interrupt_flag), kwargs=dict( - setup_flag=setup_flag, teardown_flag=teardown_flag)) + thr = threading.Thread( + target=job, args=(test_queue, interrupt_flag), kwargs=dict( + setup_flag=setup_flag, teardown_flag=teardown_flag)) # Do not wait for tests to finish executing if interrupted by the user. thr.daemon = True thr.start() diff --git a/buildscripts/resmokelib/testing/fixtures/__init__.py b/buildscripts/resmokelib/testing/fixtures/__init__.py index 1c592c8f6c7..e70ad8259ee 100644 --- a/buildscripts/resmokelib/testing/fixtures/__init__.py +++ b/buildscripts/resmokelib/testing/fixtures/__init__.py @@ -1,7 +1,5 @@ """Fixture for executing JSTests against.""" -from __future__ import absolute_import - from .external import ExternalFixture as _ExternalFixture from .interface import NoOpFixture as _NoOpFixture from .interface import make_fixture diff --git a/buildscripts/resmokelib/testing/fixtures/external.py b/buildscripts/resmokelib/testing/fixtures/external.py index ff5eba3d652..091059f68bd 100644 --- a/buildscripts/resmokelib/testing/fixtures/external.py +++ b/buildscripts/resmokelib/testing/fixtures/external.py @@ -1,7 +1,5 @@ """External fixture for executing JSTests against.""" -from __future__ import absolute_import - from . import interface diff --git a/buildscripts/resmokelib/testing/fixtures/interface.py b/buildscripts/resmokelib/testing/fixtures/interface.py index 4a38051fffd..b6b028ceb45 100644 --- a/buildscripts/resmokelib/testing/fixtures/interface.py +++ b/buildscripts/resmokelib/testing/fixtures/interface.py @@ -1,7 +1,5 @@ """Interface of the different fixtures for executing JSTests against.""" -from __future__ import absolute_import - import os.path import time @@ -25,11 +23,9 @@ def make_fixture(class_name, *args, **kwargs): return _FIXTURES[class_name](*args, **kwargs) -class Fixture(object): +class Fixture(object, metaclass=registry.make_registry_metaclass(_FIXTURES)): """Base class for all fixtures.""" - __metaclass__ = registry.make_registry_metaclass(_FIXTURES) # type: ignore - # We explicitly set the 'REGISTERED_NAME' attribute so that PyLint realizes that the attribute # is defined for all subclasses of Fixture. REGISTERED_NAME = "Fixture" diff --git a/buildscripts/resmokelib/testing/fixtures/replicaset.py b/buildscripts/resmokelib/testing/fixtures/replicaset.py index 88d57cde26a..9c449dbc432 100644 --- a/buildscripts/resmokelib/testing/fixtures/replicaset.py +++ b/buildscripts/resmokelib/testing/fixtures/replicaset.py @@ -1,10 +1,9 @@ """Replica set fixture for executing JSTests against.""" -from __future__ import absolute_import - import os.path import time +import bson.errors import pymongo import pymongo.errors import pymongo.write_concern @@ -77,11 +76,11 @@ class ReplicaSetFixture(interface.ReplFixture): # pylint: disable=too-many-inst self.replset_name = self.mongod_options.get("replSet", "rs") if not self.nodes: - for i in xrange(self.num_nodes): + for i in range(self.num_nodes): node = self._new_mongod(i, self.replset_name) self.nodes.append(node) - for i in xrange(self.num_nodes): + for i in range(self.num_nodes): if self.linear_chain and i > 0: self.nodes[i].mongod_options["set_parameters"][ "failpoint.forceSyncSourceCandidate"] = { @@ -207,10 +206,17 @@ class ReplicaSetFixture(interface.ReplFixture): # pylint: disable=too-many-inst def check_rcmaj_optime(client, node): """Return True if all nodes have caught up with the primary.""" - res = client.admin.command({"replSetGetStatus": 1}) + # TODO SERVER-40078: The server is reporting invalid + # dates in its response to the replSetGetStatus + # command + try: + res = client.admin.command({"replSetGetStatus": 1}) + except bson.errors.InvalidBSON: + return False read_concern_majority_optime = res["optimes"]["readConcernMajorityOpTime"] - if read_concern_majority_optime >= primary_optime: + if (read_concern_majority_optime["t"] == primary_optime["t"] + and read_concern_majority_optime["ts"] >= primary_optime["ts"]): up_to_date_nodes.add(node.port) return len(up_to_date_nodes) == len(self.nodes) @@ -303,7 +309,15 @@ class ReplicaSetFixture(interface.ReplFixture): # pylint: disable=too-many-inst client_admin = client["admin"] while True: - status = client_admin.command("replSetGetStatus") + # TODO SERVER-40078: The server is reporting invalid + # dates in its response to the replSetGetStatus + # command + try: + status = client_admin.command("replSetGetStatus") + except bson.errors.InvalidBSON: + time.sleep(0.1) + continue + # The `lastStableRecoveryTimestamp` field contains a stable timestamp guaranteed to # exist on storage engine recovery to a stable timestamp. last_stable_recovery_timestamp = status.get("lastStableRecoveryTimestamp", None) diff --git a/buildscripts/resmokelib/testing/fixtures/shardedcluster.py b/buildscripts/resmokelib/testing/fixtures/shardedcluster.py index f646910a3bb..b3b744095ef 100644 --- a/buildscripts/resmokelib/testing/fixtures/shardedcluster.py +++ b/buildscripts/resmokelib/testing/fixtures/shardedcluster.py @@ -1,7 +1,5 @@ """Sharded cluster fixture for executing JSTests against.""" -from __future__ import absolute_import - import os.path import time @@ -69,7 +67,7 @@ class ShardedClusterFixture(interface.Fixture): # pylint: disable=too-many-inst self.configsvr.setup() if not self.shards: - for i in xrange(self.num_shards): + for i in range(self.num_shards): if self.num_rs_nodes_per_shard is None: shard = self._new_standalone_shard(i) elif isinstance(self.num_rs_nodes_per_shard, int): diff --git a/buildscripts/resmokelib/testing/fixtures/standalone.py b/buildscripts/resmokelib/testing/fixtures/standalone.py index 4174462c07e..0265832dbcb 100644 --- a/buildscripts/resmokelib/testing/fixtures/standalone.py +++ b/buildscripts/resmokelib/testing/fixtures/standalone.py @@ -1,7 +1,5 @@ """Standalone mongod fixture for executing JSTests against.""" -from __future__ import absolute_import - import os import os.path import time diff --git a/buildscripts/resmokelib/testing/fixtures/yesfixture.py b/buildscripts/resmokelib/testing/fixtures/yesfixture.py index 618ba8a48cc..eaaf0a1be56 100644 --- a/buildscripts/resmokelib/testing/fixtures/yesfixture.py +++ b/buildscripts/resmokelib/testing/fixtures/yesfixture.py @@ -1,7 +1,5 @@ """Fixture for generating lots of log messages.""" -from __future__ import absolute_import - import signal from . import interface diff --git a/buildscripts/resmokelib/testing/hook_test_archival.py b/buildscripts/resmokelib/testing/hook_test_archival.py index b0ef3725eb2..dd3ec6fbf9b 100644 --- a/buildscripts/resmokelib/testing/hook_test_archival.py +++ b/buildscripts/resmokelib/testing/hook_test_archival.py @@ -1,7 +1,5 @@ """Enable support for archiving tests or hooks.""" -from __future__ import absolute_import - import os import threading diff --git a/buildscripts/resmokelib/testing/hooks/__init__.py b/buildscripts/resmokelib/testing/hooks/__init__.py index 82772aa25da..e24cf9eae77 100644 --- a/buildscripts/resmokelib/testing/hooks/__init__.py +++ b/buildscripts/resmokelib/testing/hooks/__init__.py @@ -5,8 +5,6 @@ by allowing special code to be executed before or after each test, and before or after each suite. """ -from __future__ import absolute_import - from .interface import make_hook from ...utils import autoloader as _autoloader diff --git a/buildscripts/resmokelib/testing/hooks/cleanup.py b/buildscripts/resmokelib/testing/hooks/cleanup.py index ebbda2f1edb..0abf025f588 100644 --- a/buildscripts/resmokelib/testing/hooks/cleanup.py +++ b/buildscripts/resmokelib/testing/hooks/cleanup.py @@ -1,7 +1,5 @@ """Test hook for cleaning up data files created by the fixture.""" -from __future__ import absolute_import - import os from . import interface @@ -22,8 +20,9 @@ class CleanEveryN(interface.Hook): # Try to isolate what test triggers the leak by restarting the fixture each time. if "detect_leaks=1" in os.getenv("ASAN_OPTIONS", ""): - self.logger.info("ASAN_OPTIONS environment variable set to detect leaks, so restarting" - " the fixture after each test instead of after every %d.", n) + self.logger.info( + "ASAN_OPTIONS environment variable set to detect leaks, so restarting" + " the fixture after each test instead of after every %d.", n) n = 1 self.n = n # pylint: disable=invalid-name diff --git a/buildscripts/resmokelib/testing/hooks/cleanup_concurrency_workloads.py b/buildscripts/resmokelib/testing/hooks/cleanup_concurrency_workloads.py index 4719b806cf2..9f5e0957514 100644 --- a/buildscripts/resmokelib/testing/hooks/cleanup_concurrency_workloads.py +++ b/buildscripts/resmokelib/testing/hooks/cleanup_concurrency_workloads.py @@ -1,7 +1,5 @@ """Test hook for dropping databases created by the fixture.""" -from __future__ import absolute_import - import copy from buildscripts.resmokelib import utils diff --git a/buildscripts/resmokelib/testing/hooks/collect_embedded_resources.py b/buildscripts/resmokelib/testing/hooks/collect_embedded_resources.py index f6ddffc3963..f39399d6dc8 100644 --- a/buildscripts/resmokelib/testing/hooks/collect_embedded_resources.py +++ b/buildscripts/resmokelib/testing/hooks/collect_embedded_resources.py @@ -1,7 +1,5 @@ """Module for generating and collecting embedded resource results.""" -from __future__ import absolute_import - import os from buildscripts.mobile import adb_monitor diff --git a/buildscripts/resmokelib/testing/hooks/combine_benchmark_results.py b/buildscripts/resmokelib/testing/hooks/combine_benchmark_results.py index 77f0263b2cc..ec4b6d53235 100644 --- a/buildscripts/resmokelib/testing/hooks/combine_benchmark_results.py +++ b/buildscripts/resmokelib/testing/hooks/combine_benchmark_results.py @@ -1,8 +1,5 @@ """Module for generating the test results file fed into the perf plugin.""" -from __future__ import absolute_import -from __future__ import division - import collections import datetime import json @@ -71,7 +68,7 @@ class CombineBenchmarkResults(interface.Hook): "results": [] } - for name, report in self.benchmark_reports.items(): + for name, report in list(self.benchmark_reports.items()): test_report = { "name": name, "context": report.context._asdict(), "results": report.generate_perf_plugin_dict() @@ -168,7 +165,7 @@ class _BenchmarkThreadsReport(object): """ res = {} - for thread_count, reports in self.thread_benchmark_map.items(): + for thread_count, reports in list(self.thread_benchmark_map.items()): thread_report = { "error_values": [0 for _ in range(len(reports))], "ops_per_sec_values": [] # This is actually storing latency per op, not ops/s diff --git a/buildscripts/resmokelib/testing/hooks/combine_benchrun_embedded_results.py b/buildscripts/resmokelib/testing/hooks/combine_benchrun_embedded_results.py index b70e57d8741..62f442dba32 100644 --- a/buildscripts/resmokelib/testing/hooks/combine_benchrun_embedded_results.py +++ b/buildscripts/resmokelib/testing/hooks/combine_benchrun_embedded_results.py @@ -1,8 +1,5 @@ """Module for generating the test results file fed into the perf plugin.""" -from __future__ import absolute_import -from __future__ import division - import collections import datetime import glob @@ -62,7 +59,7 @@ class CombineBenchrunEmbeddedResults(cbr.CombineBenchmarkResults): "results": [] } - for name, report in self.benchmark_reports.items(): + for name, report in list(self.benchmark_reports.items()): test_report = {"name": name, "results": report.generate_perf_plugin_dict()} perf_report["results"].append(test_report) @@ -143,7 +140,7 @@ class _BenchrunEmbeddedThreadsReport(object): """ res = {} - for thread_count, reports in self.thread_benchmark_map.items(): + for thread_count, reports in list(self.thread_benchmark_map.items()): thread_report = {"error_values": [], "ops_per_sec_values": []} for report in reports: diff --git a/buildscripts/resmokelib/testing/hooks/dbhash.py b/buildscripts/resmokelib/testing/hooks/dbhash.py index 40caa5149c6..557c32bb29d 100644 --- a/buildscripts/resmokelib/testing/hooks/dbhash.py +++ b/buildscripts/resmokelib/testing/hooks/dbhash.py @@ -1,7 +1,5 @@ """Test hook for verifying data consistency across a replica set.""" -from __future__ import absolute_import - import os.path from . import jsfile diff --git a/buildscripts/resmokelib/testing/hooks/dbhash_background.py b/buildscripts/resmokelib/testing/hooks/dbhash_background.py index b8122f458b8..9cea6d6957e 100644 --- a/buildscripts/resmokelib/testing/hooks/dbhash_background.py +++ b/buildscripts/resmokelib/testing/hooks/dbhash_background.py @@ -4,8 +4,6 @@ Unlike dbhash.py, this version of the hook runs continously in a background thre running. """ -from __future__ import absolute_import - import os.path import sys import threading @@ -32,13 +30,14 @@ class CheckReplDBHashInBackground(jsfile.JSHook): client = self.fixture.mongo_client() server_status = client.admin.command("serverStatus") if not server_status["storageEngine"].get("supportsSnapshotReadConcern", False): - self.logger.info("Not enabling the background thread because '%s' storage engine" - " doesn't support snapshot reads.", - server_status["storageEngine"]["name"]) + self.logger.info( + "Not enabling the background thread because '%s' storage engine" + " doesn't support snapshot reads.", server_status["storageEngine"]["name"]) return if not server_status["storageEngine"].get("persistent", False): - self.logger.info("Not enabling the background thread because '%s' storage engine" - " is not persistent.", server_status["storageEngine"]["name"]) + self.logger.info( + "Not enabling the background thread because '%s' storage engine" + " is not persistent.", server_status["storageEngine"]["name"]) return self._background_job = _BackgroundJob() diff --git a/buildscripts/resmokelib/testing/hooks/drop_sharded_collections.py b/buildscripts/resmokelib/testing/hooks/drop_sharded_collections.py index 90752571909..9a5f15715b1 100644 --- a/buildscripts/resmokelib/testing/hooks/drop_sharded_collections.py +++ b/buildscripts/resmokelib/testing/hooks/drop_sharded_collections.py @@ -1,7 +1,5 @@ """Hook for cleaning up sharded collections created during tests.""" -from __future__ import absolute_import - import os.path from . import jsfile diff --git a/buildscripts/resmokelib/testing/hooks/initialsync.py b/buildscripts/resmokelib/testing/hooks/initialsync.py index 36ada61ab00..02156068859 100644 --- a/buildscripts/resmokelib/testing/hooks/initialsync.py +++ b/buildscripts/resmokelib/testing/hooks/initialsync.py @@ -1,11 +1,10 @@ """Test hook for verifying correctness of initial sync.""" -from __future__ import absolute_import - import os.path import random import bson +import bson.errors import pymongo.errors from . import cleanup @@ -73,28 +72,38 @@ class BackgroundInitialSyncTestCase(jsfile.DynamicJSTestCase): # If it's been 'n' tests so far, wait for the initial sync node to finish syncing. if self._hook.tests_run >= self._hook.n: - self.logger.info("%d tests have been run against the fixture, waiting for initial sync" - " node to go into SECONDARY state", self._hook.tests_run) + self.logger.info( + "%d tests have been run against the fixture, waiting for initial sync" + " node to go into SECONDARY state", self._hook.tests_run) self._hook.tests_run = 0 - cmd = bson.SON([("replSetTest", 1), ("waitForMemberState", 2), ("timeoutMillis", - 20 * 60 * 1000)]) + cmd = bson.SON([("replSetTest", 1), ("waitForMemberState", 2), + ("timeoutMillis", 20 * 60 * 1000)]) sync_node_conn.admin.command(cmd) # Check if the initial sync node is in SECONDARY state. If it's been 'n' tests, then it # should have waited to be in SECONDARY state and the test should be marked as a failure. # Otherwise, we just skip the hook and will check again after the next test. try: - state = sync_node_conn.admin.command("replSetGetStatus").get("myState") + while True: + # TODO SERVER-40078: The server is reporting invalid + # dates in its response to the replSetGetStatus + # command + try: + state = sync_node_conn.admin.command("replSetGetStatus").get("myState") + break + except bson.errors.InvalidBSON: + continue + if state != 2: if self._hook.tests_run == 0: msg = "Initial sync node did not catch up after waiting 20 minutes" self.logger.exception("{0} failed: {1}".format(self._hook.description, msg)) raise errors.TestFailure(msg) - self.logger.info("Initial sync node is in state %d, not state SECONDARY (2)." - " Skipping BackgroundInitialSync hook for %s", state, - self._base_test_name) + self.logger.info( + "Initial sync node is in state %d, not state SECONDARY (2)." + " Skipping BackgroundInitialSync hook for %s", state, self._base_test_name) # If we have not restarted initial sync since the last time we ran the data # validation, restart initial sync with a 20% probability. @@ -197,8 +206,8 @@ class IntermediateInitialSyncTestCase(jsfile.DynamicJSTestCase): # Do initial sync round. self.logger.info("Waiting for initial sync node to go into SECONDARY state") - cmd = bson.SON([("replSetTest", 1), ("waitForMemberState", 2), ("timeoutMillis", - 20 * 60 * 1000)]) + cmd = bson.SON([("replSetTest", 1), ("waitForMemberState", 2), + ("timeoutMillis", 20 * 60 * 1000)]) sync_node_conn.admin.command(cmd) # Run data validation and dbhash checking. diff --git a/buildscripts/resmokelib/testing/hooks/interface.py b/buildscripts/resmokelib/testing/hooks/interface.py index fe53f820fc5..d8ac37e159d 100644 --- a/buildscripts/resmokelib/testing/hooks/interface.py +++ b/buildscripts/resmokelib/testing/hooks/interface.py @@ -1,7 +1,5 @@ """Interface for customizing the behavior of a test fixture.""" -from __future__ import absolute_import - import sys from ..testcases import interface as testcase @@ -21,11 +19,9 @@ def make_hook(class_name, *args, **kwargs): return _HOOKS[class_name](*args, **kwargs) -class Hook(object): +class Hook(object, metaclass=registry.make_registry_metaclass(_HOOKS)): """Common interface all Hooks will inherit from.""" - __metaclass__ = registry.make_registry_metaclass(_HOOKS) # type: ignore - REGISTERED_NAME = registry.LEAVE_UNREGISTERED def __init__(self, hook_logger, fixture, description): diff --git a/buildscripts/resmokelib/testing/hooks/jsfile.py b/buildscripts/resmokelib/testing/hooks/jsfile.py index e95d3d6d780..6eedb3f87c4 100644 --- a/buildscripts/resmokelib/testing/hooks/jsfile.py +++ b/buildscripts/resmokelib/testing/hooks/jsfile.py @@ -1,7 +1,5 @@ """Interface for customizing the behavior of a test fixture by executing a JavaScript file.""" -from __future__ import absolute_import - from . import interface from ..testcases import jstest from ... import errors diff --git a/buildscripts/resmokelib/testing/hooks/oplog.py b/buildscripts/resmokelib/testing/hooks/oplog.py index ceb81bb8fd6..6822eecabf0 100644 --- a/buildscripts/resmokelib/testing/hooks/oplog.py +++ b/buildscripts/resmokelib/testing/hooks/oplog.py @@ -1,7 +1,5 @@ """Test hook for verifying members of a replica set have matching oplogs.""" -from __future__ import absolute_import - import os.path from . import jsfile diff --git a/buildscripts/resmokelib/testing/hooks/periodic_kill_secondaries.py b/buildscripts/resmokelib/testing/hooks/periodic_kill_secondaries.py index a6f44278140..daf18ad7671 100644 --- a/buildscripts/resmokelib/testing/hooks/periodic_kill_secondaries.py +++ b/buildscripts/resmokelib/testing/hooks/periodic_kill_secondaries.py @@ -1,7 +1,5 @@ """Test hook for verifying correctness of secondary's behavior during an unclean shutdown.""" -from __future__ import absolute_import - import time import bson @@ -189,8 +187,9 @@ class PeriodicKillSecondariesTestCase(interface.DynamicTestCase): for secondary in self.fixture.get_secondaries(): self._check_invariants_as_standalone(secondary) - self.logger.info("Restarting the secondary on port %d as a replica set node with" - " its data files intact...", secondary.port) + self.logger.info( + "Restarting the secondary on port %d as a replica set node with" + " its data files intact...", secondary.port) # Start the 'secondary' mongod back up as part of the replica set and wait for it to # reach state SECONDARY. secondary.setup() @@ -253,12 +252,13 @@ class PeriodicKillSecondariesTestCase(interface.DynamicTestCase): self.fixture.setup() self.fixture.await_ready() - def _check_invariants_as_standalone(self, secondary): + def _check_invariants_as_standalone(self, secondary): # pylint: disable=too-many-locals # pylint: disable=too-many-branches,too-many-statements # We remove the --replSet option in order to start the node as a standalone. replset_name = secondary.mongod_options.pop("replSet") - self.logger.info("Restarting the secondary on port %d as a standalone node with" - " its data files intact...", secondary.port) + self.logger.info( + "Restarting the secondary on port %d as a standalone node with" + " its data files intact...", secondary.port) try: secondary.setup() diff --git a/buildscripts/resmokelib/testing/hooks/stepdown.py b/buildscripts/resmokelib/testing/hooks/stepdown.py index fbda653a324..f218fffe49e 100644 --- a/buildscripts/resmokelib/testing/hooks/stepdown.py +++ b/buildscripts/resmokelib/testing/hooks/stepdown.py @@ -1,5 +1,4 @@ """Test hook that periodically makes the primary of a replica set step down.""" -from __future__ import absolute_import import collections import os.path @@ -179,7 +178,7 @@ class _StepdownThread(threading.Thread): # pylint: disable=too-many-instance-at try: while True: - if self._is_stopped(): + if self.__is_stopped(): break self._wait_for_permission_or_resume() now = time.time() @@ -210,7 +209,7 @@ class _StepdownThread(threading.Thread): # pylint: disable=too-many-instance-at self.resume() self.join() - def _is_stopped(self): + def __is_stopped(self): return self._is_stopped_evt.is_set() def pause(self): @@ -234,7 +233,7 @@ class _StepdownThread(threading.Thread): # pylint: disable=too-many-instance-at def _wait_for_permission_or_resume(self): # Wait until stop, _stepdown_permitted_file or resume. if self._stepdown_permitted_file: - while not os.path.isfile(self._stepdown_permitted_file) and not self._is_stopped(): + while not os.path.isfile(self._stepdown_permitted_file) and not self.__is_stopped(): # Set a short sleep during busy wait time for self._stepdown_permitted_file. self._wait(0.1) else: diff --git a/buildscripts/resmokelib/testing/hooks/validate.py b/buildscripts/resmokelib/testing/hooks/validate.py index 3239ddbdc06..81bd42ba6d9 100644 --- a/buildscripts/resmokelib/testing/hooks/validate.py +++ b/buildscripts/resmokelib/testing/hooks/validate.py @@ -1,7 +1,5 @@ """Test hook for verifying the consistency and integrity of collection and index data.""" -from __future__ import absolute_import - import os.path from . import jsfile diff --git a/buildscripts/resmokelib/testing/hooks/wait_for_replication.py b/buildscripts/resmokelib/testing/hooks/wait_for_replication.py index fbd786a170c..8720f9456e6 100644 --- a/buildscripts/resmokelib/testing/hooks/wait_for_replication.py +++ b/buildscripts/resmokelib/testing/hooks/wait_for_replication.py @@ -1,7 +1,5 @@ """Test hook to wait for replication to complete on a replica set.""" -from __future__ import absolute_import - import time from buildscripts.resmokelib import core diff --git a/buildscripts/resmokelib/testing/job.py b/buildscripts/resmokelib/testing/job.py index 908ae85832a..9b5023d0656 100644 --- a/buildscripts/resmokelib/testing/job.py +++ b/buildscripts/resmokelib/testing/job.py @@ -1,7 +1,5 @@ """Enable running tests simultaneously by processing them from a multi-consumer queue.""" -from __future__ import absolute_import - import sys import time @@ -175,8 +173,8 @@ class Job(object): # pylint: disable=too-many-instance-attributes test.short_description()) self.report.setFailure(test, return_code=2) # Always fail fast if the fixture fails. - raise errors.StopExecution("%s not running after %s" % (self.fixture, - test.short_description())) + raise errors.StopExecution( + "%s not running after %s" % (self.fixture, test.short_description())) finally: success = self.report.find_test_info(test).status == "pass" if self.archival: diff --git a/buildscripts/resmokelib/testing/report.py b/buildscripts/resmokelib/testing/report.py index fad0d20a0a2..3365a17075a 100644 --- a/buildscripts/resmokelib/testing/report.py +++ b/buildscripts/resmokelib/testing/report.py @@ -3,8 +3,6 @@ This is used to support additional test status and timing information for the report.json file. """ -from __future__ import absolute_import - import copy import threading import time diff --git a/buildscripts/resmokelib/testing/suite.py b/buildscripts/resmokelib/testing/suite.py index e0f2dda5151..b369261e147 100644 --- a/buildscripts/resmokelib/testing/suite.py +++ b/buildscripts/resmokelib/testing/suite.py @@ -1,7 +1,5 @@ """Holder for the (test kind, list of tests) pair with additional metadata their execution.""" -from __future__ import absolute_import - import itertools import threading import time @@ -269,7 +267,7 @@ class Suite(object): # pylint: disable=too-many-instance-attributes sb.append("Executed %d times in %0.2f seconds:" % (num_iterations, total_time_taken)) combined_summary = _summary.Summary(0, 0.0, 0, 0, 0, 0) - for iteration in xrange(num_iterations): + for iteration in range(num_iterations): # Summarize each execution as a bulleted list of results. bulleter_sb = [] summary = self._summarize_report(reports[iteration], start_times[iteration], @@ -341,8 +339,8 @@ class Suite(object): # pylint: disable=too-many-instance-attributes def log_summaries(logger, suites, time_taken): """Log summary of all suites.""" sb = [] - sb.append("Summary of all suites: %d suites ran in %0.2f seconds" % (len(suites), - time_taken)) + sb.append( + "Summary of all suites: %d suites ran in %0.2f seconds" % (len(suites), time_taken)) for suite in suites: suite_sb = [] suite.summarize(suite_sb) diff --git a/buildscripts/resmokelib/testing/summary.py b/buildscripts/resmokelib/testing/summary.py index dc92e0b5b34..a5d439b64b0 100644 --- a/buildscripts/resmokelib/testing/summary.py +++ b/buildscripts/resmokelib/testing/summary.py @@ -1,7 +1,5 @@ """Holder for summary information about a test suite.""" -from __future__ import absolute_import - import collections Summary = collections.namedtuple( @@ -12,6 +10,6 @@ Summary = collections.namedtuple( def combine(summary1, summary2): """Return a summary representing the sum of 'summary1' and 'summary2'.""" args = [] - for i in xrange(len(Summary._fields)): + for i in range(len(Summary._fields)): args.append(summary1[i] + summary2[i]) return Summary._make(args) diff --git a/buildscripts/resmokelib/testing/testcases/__init__.py b/buildscripts/resmokelib/testing/testcases/__init__.py index 52869d99de8..d828d97a0b3 100644 --- a/buildscripts/resmokelib/testing/testcases/__init__.py +++ b/buildscripts/resmokelib/testing/testcases/__init__.py @@ -1,7 +1,5 @@ """Package containing subclasses of unittest.TestCase.""" -from __future__ import absolute_import - from .interface import make_test_case from ...utils import autoloader as _autoloader diff --git a/buildscripts/resmokelib/testing/testcases/benchmark_test.py b/buildscripts/resmokelib/testing/testcases/benchmark_test.py index ea506c4f7e5..e7760799e42 100644 --- a/buildscripts/resmokelib/testing/testcases/benchmark_test.py +++ b/buildscripts/resmokelib/testing/testcases/benchmark_test.py @@ -1,7 +1,5 @@ """The unittest.TestCase for tests using a MongoDB vendored version of Google Benchmark.""" -from __future__ import absolute_import - from buildscripts.resmokelib import config as _config from buildscripts.resmokelib import core from buildscripts.resmokelib import parser @@ -52,7 +50,7 @@ class BenchmarkTestCase(interface.ProcessTestCase): "benchmark_repetitions": _config.BENCHMARK_REPETITIONS } - for key, value in resmoke_bm_options.items(): + for key, value in list(resmoke_bm_options.items()): if value is not None: # 4. sanitize options before passing them to Benchmark's command line. if key == "benchmark_min_time": diff --git a/buildscripts/resmokelib/testing/testcases/benchrun_embedded_test.py b/buildscripts/resmokelib/testing/testcases/benchrun_embedded_test.py index 89b95424cdb..d4666c94074 100644 --- a/buildscripts/resmokelib/testing/testcases/benchrun_embedded_test.py +++ b/buildscripts/resmokelib/testing/testcases/benchrun_embedded_test.py @@ -1,7 +1,5 @@ """The unittest.TestCase for tests using benchrun embedded (mongoebench).""" -from __future__ import absolute_import - import os import posixpath @@ -73,7 +71,7 @@ class BenchrunEmbeddedTestCase( # pylint: disable=too-many-instance-attributes # 3. Override Benchmark options with options set through resmoke's command line. resmoke_benchrun_options = {"dbpath": self.dbpath, "time": _config.BENCHMARK_MIN_TIME} - for key, value in resmoke_benchrun_options.items(): + for key, value in list(resmoke_benchrun_options.items()): if value is not None: # 4. sanitize options before passing them to Benchmark's command line. if key == "time": @@ -104,7 +102,7 @@ class BenchrunEmbeddedTestCase( # pylint: disable=too-many-instance-attributes def run_test(self): """Run the test for specified number of iterations.""" - for iter_num in xrange(self.benchrun_repetitions): + for iter_num in range(self.benchrun_repetitions): # Set the output file for each iteration. local_report_path = self._report_path(iter_num) device_report_path = self._device_report_path(iter_num) @@ -134,8 +132,8 @@ class BenchrunEmbeddedTestCase( # pylint: disable=too-many-instance-attributes def _report_dir(self): """Return the report directory. Reports are stored in <report_root>/<testname>/<thread>.""" - return os.path.join(self.report_root, self.short_name(), "thread{}".format( - self.benchrun_threads)) + return os.path.join(self.report_root, self.short_name(), + "thread{}".format(self.benchrun_threads)) @staticmethod def _report_name(iter_num): diff --git a/buildscripts/resmokelib/testing/testcases/cpp_integration_test.py b/buildscripts/resmokelib/testing/testcases/cpp_integration_test.py index df6d7c9fa41..c0313046a20 100644 --- a/buildscripts/resmokelib/testing/testcases/cpp_integration_test.py +++ b/buildscripts/resmokelib/testing/testcases/cpp_integration_test.py @@ -1,7 +1,5 @@ """The unittest.TestCase for C++ integration tests.""" -from __future__ import absolute_import - from . import interface from ... import core from ... import utils diff --git a/buildscripts/resmokelib/testing/testcases/cpp_unittest.py b/buildscripts/resmokelib/testing/testcases/cpp_unittest.py index b9fb427d0da..c54b45a7792 100644 --- a/buildscripts/resmokelib/testing/testcases/cpp_unittest.py +++ b/buildscripts/resmokelib/testing/testcases/cpp_unittest.py @@ -1,7 +1,5 @@ """The unittest.TestCase for C++ unit tests.""" -from __future__ import absolute_import - from . import interface from ... import core from ... import utils diff --git a/buildscripts/resmokelib/testing/testcases/dbtest.py b/buildscripts/resmokelib/testing/testcases/dbtest.py index 4cfbb8c6385..44ea8410c58 100644 --- a/buildscripts/resmokelib/testing/testcases/dbtest.py +++ b/buildscripts/resmokelib/testing/testcases/dbtest.py @@ -1,7 +1,5 @@ """The unittest.TestCase for dbtests.""" -from __future__ import absolute_import - import os import os.path diff --git a/buildscripts/resmokelib/testing/testcases/fsm_workload_test.py b/buildscripts/resmokelib/testing/testcases/fsm_workload_test.py index 6cbda33abe9..d184fe84fc5 100644 --- a/buildscripts/resmokelib/testing/testcases/fsm_workload_test.py +++ b/buildscripts/resmokelib/testing/testcases/fsm_workload_test.py @@ -1,7 +1,5 @@ """The unittest.TestCase for FSM workloads.""" -from __future__ import absolute_import - import hashlib import threading @@ -97,5 +95,5 @@ class ParallelFSMWorkloadTestCase(FSMWorkloadTestCase): """Get an unique identifier for a workload group.""" uid = hashlib.md5() for workload_name in sorted(selected_tests): - uid.update(workload_name) + uid.update(workload_name.encode("utf-8")) return uid.hexdigest() diff --git a/buildscripts/resmokelib/testing/testcases/gennylib_test.py b/buildscripts/resmokelib/testing/testcases/gennylib_test.py index 91200b6c6c1..96eb08c6cb5 100644 --- a/buildscripts/resmokelib/testing/testcases/gennylib_test.py +++ b/buildscripts/resmokelib/testing/testcases/gennylib_test.py @@ -1,7 +1,5 @@ """The unittest.TestCase for gennylib integration tests.""" -from __future__ import absolute_import - from . import interface from ... import core from ... import utils diff --git a/buildscripts/resmokelib/testing/testcases/gennytest.py b/buildscripts/resmokelib/testing/testcases/gennytest.py index 1db9ca48e94..11e51014803 100644 --- a/buildscripts/resmokelib/testing/testcases/gennytest.py +++ b/buildscripts/resmokelib/testing/testcases/gennytest.py @@ -1,7 +1,5 @@ """The unittest.TestCase for genny.""" -from __future__ import absolute_import - import os import os.path diff --git a/buildscripts/resmokelib/testing/testcases/interface.py b/buildscripts/resmokelib/testing/testcases/interface.py index 1f078828b1c..e1c1e2faa0c 100644 --- a/buildscripts/resmokelib/testing/testcases/interface.py +++ b/buildscripts/resmokelib/testing/testcases/interface.py @@ -3,8 +3,6 @@ This is used to perform the actual test case. """ -from __future__ import absolute_import - import os import os.path import unittest @@ -23,11 +21,9 @@ def make_test_case(test_kind, *args, **kwargs): return _TEST_CASES[test_kind](*args, **kwargs) -class TestCase(unittest.TestCase): # pylint: disable=too-many-instance-attributes +class TestCase(unittest.TestCase, metaclass=registry.make_registry_metaclass(_TEST_CASES)): # pylint: disable=too-many-instance-attributes """A test case to execute.""" - __metaclass__ = registry.make_registry_metaclass(_TEST_CASES) # type: ignore - REGISTERED_NAME = registry.LEAVE_UNREGISTERED def __init__(self, logger, test_kind, test_name, dynamic=False): @@ -37,10 +33,10 @@ class TestCase(unittest.TestCase): # pylint: disable=too-many-instance-attribut if not isinstance(logger, logging.Logger): raise TypeError("logger must be a Logger instance") - if not isinstance(test_kind, basestring): + if not isinstance(test_kind, str): raise TypeError("test_kind must be a string") - if not isinstance(test_name, basestring): + if not isinstance(test_name, str): raise TypeError("test_name must be a string") self._id = uuid.uuid4() diff --git a/buildscripts/resmokelib/testing/testcases/json_schema_test.py b/buildscripts/resmokelib/testing/testcases/json_schema_test.py index 08e5a2d71a8..6f40b8af714 100644 --- a/buildscripts/resmokelib/testing/testcases/json_schema_test.py +++ b/buildscripts/resmokelib/testing/testcases/json_schema_test.py @@ -1,7 +1,5 @@ """The unittest.TestCase for JSON Schema tests.""" -from __future__ import absolute_import - from buildscripts.resmokelib import config from buildscripts.resmokelib import core from buildscripts.resmokelib import utils diff --git a/buildscripts/resmokelib/testing/testcases/jsrunnerfile.py b/buildscripts/resmokelib/testing/testcases/jsrunnerfile.py index c2da41faf37..ece5e2ceb15 100644 --- a/buildscripts/resmokelib/testing/testcases/jsrunnerfile.py +++ b/buildscripts/resmokelib/testing/testcases/jsrunnerfile.py @@ -1,7 +1,5 @@ """The unittest.TestCase for tests with a static JavaScript runner file.""" -from __future__ import absolute_import - from buildscripts.resmokelib import config from buildscripts.resmokelib import core from buildscripts.resmokelib import utils diff --git a/buildscripts/resmokelib/testing/testcases/jstest.py b/buildscripts/resmokelib/testing/testcases/jstest.py index 4d36ac965b0..6b6a5ee45e2 100644 --- a/buildscripts/resmokelib/testing/testcases/jstest.py +++ b/buildscripts/resmokelib/testing/testcases/jstest.py @@ -1,7 +1,5 @@ """The unittest.TestCase for JavaScript tests.""" -from __future__ import absolute_import - import os import os.path import sys @@ -201,7 +199,7 @@ class JSTestCase(interface.ProcessTestCase): test_cases = [] try: # If there are multiple clients, make a new thread for each client. - for thread_id in xrange(self.num_clients): + for thread_id in range(self.num_clients): logger = self.logger.new_test_thread_logger(self.test_kind, str(thread_id)) test_case = self._create_test_case_for_thread(logger, thread_id) test_cases.append(test_case) diff --git a/buildscripts/resmokelib/testing/testcases/mongos_test.py b/buildscripts/resmokelib/testing/testcases/mongos_test.py index 9914ba8677f..f00a93a9bdb 100644 --- a/buildscripts/resmokelib/testing/testcases/mongos_test.py +++ b/buildscripts/resmokelib/testing/testcases/mongos_test.py @@ -1,7 +1,5 @@ """The unittest.TestCase for mongos --test.""" -from __future__ import absolute_import - from . import interface from ... import config from ... import core diff --git a/buildscripts/resmokelib/testing/testcases/mql_model_haskell_test.py b/buildscripts/resmokelib/testing/testcases/mql_model_haskell_test.py index 7911aa1b7d6..fff521ddae5 100644 --- a/buildscripts/resmokelib/testing/testcases/mql_model_haskell_test.py +++ b/buildscripts/resmokelib/testing/testcases/mql_model_haskell_test.py @@ -1,7 +1,5 @@ """The unittest.TestCase for MQL Haskell tests.""" -from __future__ import absolute_import - import os import os.path diff --git a/buildscripts/resmokelib/testing/testcases/mql_model_mongod_test.py b/buildscripts/resmokelib/testing/testcases/mql_model_mongod_test.py index eddfef2b7b3..249b0f18fb2 100644 --- a/buildscripts/resmokelib/testing/testcases/mql_model_mongod_test.py +++ b/buildscripts/resmokelib/testing/testcases/mql_model_mongod_test.py @@ -1,7 +1,5 @@ """The unittest.TestCase for MQL MongoD Model tests.""" -from __future__ import absolute_import - import os import os.path diff --git a/buildscripts/resmokelib/testing/testcases/multi_stmt_txn_test.py b/buildscripts/resmokelib/testing/testcases/multi_stmt_txn_test.py index 1e790612153..81ddfa289ae 100644 --- a/buildscripts/resmokelib/testing/testcases/multi_stmt_txn_test.py +++ b/buildscripts/resmokelib/testing/testcases/multi_stmt_txn_test.py @@ -1,7 +1,5 @@ """unittest.TestCase for multi-statement transaction passthrough tests.""" -from __future__ import absolute_import - from buildscripts.resmokelib import config from buildscripts.resmokelib import core from buildscripts.resmokelib import utils diff --git a/buildscripts/resmokelib/testing/testcases/sleeptest.py b/buildscripts/resmokelib/testing/testcases/sleeptest.py index 163100095d3..4f073ee6f25 100644 --- a/buildscripts/resmokelib/testing/testcases/sleeptest.py +++ b/buildscripts/resmokelib/testing/testcases/sleeptest.py @@ -1,7 +1,5 @@ """The unittest.TestCase for sleeping a given amount of time.""" -from __future__ import absolute_import - import time from . import interface diff --git a/buildscripts/resmokelib/utils/__init__.py b/buildscripts/resmokelib/utils/__init__.py index 5f60f162696..02681bca0b5 100644 --- a/buildscripts/resmokelib/utils/__init__.py +++ b/buildscripts/resmokelib/utils/__init__.py @@ -1,8 +1,5 @@ """Helper functions.""" -from __future__ import absolute_import -from __future__ import print_function - import contextlib import os.path import shutil @@ -40,19 +37,7 @@ def default_if_none(value, default): def rmtree(path, **kwargs): - """Wrap shutil.rmtreee. - - Use a UTF-8 unicode path if Windows. - See https://bugs.python.org/issue24672, where shutil.rmtree can fail with UTF-8. - Use a bytes path to rmtree, otherwise. - See https://github.com/pypa/setuptools/issues/706. - """ - if is_windows(): - if not isinstance(path, unicode): - path = unicode(path, "utf-8") - else: - if isinstance(path, unicode): - path = path.encode("utf-8") + """Wrap shutil.rmtree.""" shutil.rmtree(path, **kwargs) @@ -72,12 +57,12 @@ def remove_if_exists(path): def is_string_list(lst): """Return true if 'lst' is a list of strings, and false otherwise.""" - return isinstance(lst, list) and all(isinstance(x, basestring) for x in lst) + return isinstance(lst, list) and all(isinstance(x, str) for x in lst) def is_string_set(value): """Return true if 'value' is a set of strings, and false otherwise.""" - return isinstance(value, set) and all(isinstance(x, basestring) for x in value) + return isinstance(value, set) and all(isinstance(x, str) for x in value) def is_js_file(filename): diff --git a/buildscripts/resmokelib/utils/archival.py b/buildscripts/resmokelib/utils/archival.py index 8ccb3127f47..a31f8fd2d31 100644 --- a/buildscripts/resmokelib/utils/archival.py +++ b/buildscripts/resmokelib/utils/archival.py @@ -1,8 +1,6 @@ """Archival utility.""" -from __future__ import absolute_import - -import Queue +import queue import collections import json import math @@ -45,7 +43,7 @@ def file_list_size(files): def directory_size(directory): """Return size (in bytes) of files in 'directory' tree.""" dir_bytes = 0 - for root_dir, _, files in os.walk(unicode(directory)): + for root_dir, _, files in os.walk(str(directory)): for name in files: full_name = os.path.join(root_dir, name) try: @@ -103,7 +101,7 @@ class Archival(object): # pylint: disable=too-many-instance-attributes self._lock = threading.Lock() # Start the worker thread to update the 'archival_json_file'. - self._archive_file_queue = Queue.Queue() + self._archive_file_queue = queue.Queue() self._archive_file_worker = threading.Thread(target=self._update_archive_file_wkr, args=(self._archive_file_queue, logger), name="archive_file_worker") @@ -115,10 +113,10 @@ class Archival(object): # pylint: disable=too-many-instance-attributes self.s3_client = s3_client # Start the worker thread which uploads the archive. - self._upload_queue = Queue.Queue() - self._upload_worker = threading.Thread(target=self._upload_to_s3_wkr, - args=(self._upload_queue, self._archive_file_queue, - logger, self.s3_client), name="upload_worker") + self._upload_queue = queue.Queue() + self._upload_worker = threading.Thread( + target=self._upload_to_s3_wkr, args=(self._upload_queue, self._archive_file_queue, + logger, self.s3_client), name="upload_worker") self._upload_worker.setDaemon(True) self._upload_worker.start() @@ -161,14 +159,14 @@ class Archival(object): # pylint: disable=too-many-instance-attributes return status, message @staticmethod - def _update_archive_file_wkr(queue, logger): - """Worker thread: Update the archival JSON file from 'queue'.""" + def _update_archive_file_wkr(work_queue, logger): + """Worker thread: Update the archival JSON file from 'work_queue'.""" archival_json = [] while True: - archive_args = queue.get() + archive_args = work_queue.get() # Exit worker thread when sentinel is received. if archive_args is None: - queue.task_done() + work_queue.task_done() break archival_record = { "name": archive_args.display_name, "link": archive_args.remote_file, @@ -179,17 +177,17 @@ class Archival(object): # pylint: disable=too-many-instance-attributes archival_json.append(archival_record) with open(archive_args.archival_file, "w") as archival_fh: json.dump(archival_json, archival_fh) - queue.task_done() + work_queue.task_done() @staticmethod - def _upload_to_s3_wkr(queue, archive_file_queue, logger, s3_client): - """Worker thread: Upload to S3 from 'queue', dispatch to 'archive_file_queue'.""" + def _upload_to_s3_wkr(work_queue, archive_file_work_queue, logger, s3_client): + """Worker thread: Upload to S3 from 'work_queue', dispatch to 'archive_file_work_queue'.""" while True: - upload_args = queue.get() + upload_args = work_queue.get() # Exit worker thread when sentinel is received. if upload_args is None: - queue.task_done() - archive_file_queue.put(None) + work_queue.task_done() + archive_file_work_queue.put(None) break extra_args = {"ContentType": upload_args.content_type, "ACL": "public-read"} logger.debug("Uploading to S3 %s to bucket %s path %s", upload_args.local_file, @@ -212,10 +210,10 @@ class Archival(object): # pylint: disable=too-many-instance-attributes remote_file = "https://s3.amazonaws.com/{}/{}".format(upload_args.s3_bucket, upload_args.s3_path) if upload_completed: - archive_file_queue.put( + archive_file_work_queue.put( ArchiveArgs(upload_args.archival_file, upload_args.display_name, remote_file)) - queue.task_done() + work_queue.task_done() def _archive_files(self, display_name, input_files, s3_bucket, s3_path): """ diff --git a/buildscripts/resmokelib/utils/autoloader.py b/buildscripts/resmokelib/utils/autoloader.py index 73b58563451..ab1ace84798 100644 --- a/buildscripts/resmokelib/utils/autoloader.py +++ b/buildscripts/resmokelib/utils/autoloader.py @@ -1,7 +1,5 @@ """Utility for loading all modules within a package.""" -from __future__ import absolute_import - import importlib import pkgutil diff --git a/buildscripts/resmokelib/utils/globstar.py b/buildscripts/resmokelib/utils/globstar.py index 1e016875f94..5857870e627 100644 --- a/buildscripts/resmokelib/utils/globstar.py +++ b/buildscripts/resmokelib/utils/globstar.py @@ -1,7 +1,5 @@ """Filename globbing utility.""" -from __future__ import absolute_import - import glob as _glob import os import os.path @@ -134,7 +132,7 @@ def _list_dir(pathname): """ try: - (_root, dirs, files) = os.walk(pathname).next() + (_root, dirs, files) = next(os.walk(pathname)) return (dirs, files) except StopIteration: return None # 'pathname' directory does not exist diff --git a/buildscripts/resmokelib/utils/jscomment.py b/buildscripts/resmokelib/utils/jscomment.py index 67758197c5c..21d1cfa783c 100644 --- a/buildscripts/resmokelib/utils/jscomment.py +++ b/buildscripts/resmokelib/utils/jscomment.py @@ -1,13 +1,11 @@ """Utility for parsing JS comments.""" -from __future__ import absolute_import - import re import yaml # TODO: use a more robust regular expression for matching tags -_JSTEST_TAGS_RE = re.compile(r".*@tags\s*:\s*(\[[^\]]*\])", re.DOTALL) +_JSTEST_TAGS_RE = re.compile(b".*@tags\s*:\s*(\[[^\]]*\])", re.DOTALL) def get_tags(pathname): @@ -29,19 +27,19 @@ def get_tags(pathname): */ """ - with open(pathname) as fp: + with open(pathname, 'rb') as fp: match = _JSTEST_TAGS_RE.match(fp.read()) if match: try: # TODO: it might be worth supporting the block (indented) style of YAML lists in # addition to the flow (bracketed) style tags = yaml.safe_load(_strip_jscomments(match.group(1))) - if not isinstance(tags, list) and all(isinstance(tag, basestring) for tag in tags): + if not isinstance(tags, list) and all(isinstance(tag, str) for tag in tags): raise TypeError("Expected a list of string tags, but got '%s'" % (tags)) return tags except yaml.YAMLError as err: - raise ValueError("File '%s' contained invalid tags (expected YAML): %s" % (pathname, - err)) + raise ValueError( + "File '%s' contained invalid tags (expected YAML): %s" % (pathname, err)) return [] @@ -68,6 +66,9 @@ def _strip_jscomments(string): yaml_lines = [] + if isinstance(string, bytes): + string = string.decode("utf-8") + for line in string.splitlines(): # Remove leading whitespace and symbols that commonly appear in JS comments. line = line.lstrip("\t ").lstrip("*/") diff --git a/buildscripts/resmokelib/utils/queue.py b/buildscripts/resmokelib/utils/queue.py index c77692138b1..90c57408621 100644 --- a/buildscripts/resmokelib/utils/queue.py +++ b/buildscripts/resmokelib/utils/queue.py @@ -6,16 +6,14 @@ in order for KeyboardInterrupt exceptions to get propagated. See https://bugs.python.org/issue1167930 for more details. """ -from __future__ import absolute_import - -import Queue as _Queue +import queue as _queue import time # Exception that is raised when get_nowait() is called on an empty Queue. -Empty = _Queue.Empty +Empty = _queue.Empty -class Queue(_Queue.Queue): +class Queue(_queue.Queue): """A multi-producer, multi-consumer queue.""" def join(self, timeout=None): # pylint: disable=arguments-differ diff --git a/buildscripts/resmokelib/utils/registry.py b/buildscripts/resmokelib/utils/registry.py index 0aa02f4b2b5..4248b8c38b3 100644 --- a/buildscripts/resmokelib/utils/registry.py +++ b/buildscripts/resmokelib/utils/registry.py @@ -6,8 +6,6 @@ This pattern enables the associated class to be looked up later by using its name. """ -from __future__ import absolute_import - # Specifying 'LEAVE_UNREGISTERED' as the "REGISTERED_NAME" attribute will cause the class to be # omitted from the registry. This is particularly useful for base classes that define an interface # or common functionality, and aren't intended to be constructed explicitly. @@ -23,7 +21,7 @@ def make_registry_metaclass(registry_store): class Registry(type): """A metaclass that stores a reference to all registered classes.""" - def __new__(mcs, class_name, base_classes, class_dict): + def __new__(mcs, class_name, base_classes, class_dict): # pylint: disable=bad-mcs-classmethod-argument """Create and returns a new instance of Registry. The registry is a class named 'class_name' derived from 'base_classes' @@ -46,9 +44,9 @@ def make_registry_metaclass(registry_store): if registered_name is not LEAVE_UNREGISTERED: if registered_name in registry_store: - raise ValueError("The name %s is already registered; a different value for the" - " 'REGISTERED_NAME' attribute must be chosen" % - (registered_name)) + raise ValueError( + "The name %s is already registered; a different value for the" + " 'REGISTERED_NAME' attribute must be chosen" % (registered_name)) registry_store[registered_name] = cls return cls diff --git a/buildscripts/resmokelib/utils/scheduler.py b/buildscripts/resmokelib/utils/scheduler.py index 04abafcd330..9f57d0a110e 100644 --- a/buildscripts/resmokelib/utils/scheduler.py +++ b/buildscripts/resmokelib/utils/scheduler.py @@ -1,7 +1,5 @@ """Thread-safe version of sched.scheduler; the class wasn't made thread-safe until Python 3.3.""" -from __future__ import absolute_import - import heapq import sched import threading |