summaryrefslogtreecommitdiff
path: root/buildscripts/resmokelib
diff options
context:
space:
mode:
authorMathew Robinson <chasinglogic@gmail.com>2019-02-19 10:50:57 -0500
committerMathew Robinson <chasinglogic@gmail.com>2019-04-08 14:08:49 -0400
commit8dd6d4755734ed37c1b98dfdefce3ca6bc65f1f6 (patch)
tree69e936c4953cbead2e3bae2690157c5fe75e709d /buildscripts/resmokelib
parentc600aa9d7423eca8151daf626e2799d9a6c7b31c (diff)
downloadmongo-8dd6d4755734ed37c1b98dfdefce3ca6bc65f1f6.tar.gz
SERVER-32295 Support Python 3
Diffstat (limited to 'buildscripts/resmokelib')
-rw-r--r--buildscripts/resmokelib/__init__.py2
-rw-r--r--buildscripts/resmokelib/config.py31
-rw-r--r--buildscripts/resmokelib/core/__init__.py1
-rw-r--r--buildscripts/resmokelib/core/jasper_process.py2
-rw-r--r--buildscripts/resmokelib/core/network.py2
-rw-r--r--buildscripts/resmokelib/core/pipe.py2
-rw-r--r--buildscripts/resmokelib/core/process.py27
-rw-r--r--buildscripts/resmokelib/core/programs.py2
-rw-r--r--buildscripts/resmokelib/logging/__init__.py2
-rw-r--r--buildscripts/resmokelib/logging/buildlogger.py32
-rw-r--r--buildscripts/resmokelib/logging/flush.py7
-rw-r--r--buildscripts/resmokelib/logging/formatters.py2
-rw-r--r--buildscripts/resmokelib/logging/handlers.py12
-rw-r--r--buildscripts/resmokelib/logging/loggers.py6
-rw-r--r--buildscripts/resmokelib/parser.py364
-rw-r--r--buildscripts/resmokelib/reportfile.py2
-rw-r--r--buildscripts/resmokelib/selector.py8
-rw-r--r--buildscripts/resmokelib/sighandler.py2
-rw-r--r--buildscripts/resmokelib/suitesconfig.py8
-rw-r--r--buildscripts/resmokelib/testing/__init__.py2
-rw-r--r--buildscripts/resmokelib/testing/executor.py14
-rw-r--r--buildscripts/resmokelib/testing/fixtures/__init__.py2
-rw-r--r--buildscripts/resmokelib/testing/fixtures/external.py2
-rw-r--r--buildscripts/resmokelib/testing/fixtures/interface.py6
-rw-r--r--buildscripts/resmokelib/testing/fixtures/replicaset.py28
-rw-r--r--buildscripts/resmokelib/testing/fixtures/shardedcluster.py4
-rw-r--r--buildscripts/resmokelib/testing/fixtures/standalone.py2
-rw-r--r--buildscripts/resmokelib/testing/fixtures/yesfixture.py2
-rw-r--r--buildscripts/resmokelib/testing/hook_test_archival.py2
-rw-r--r--buildscripts/resmokelib/testing/hooks/__init__.py2
-rw-r--r--buildscripts/resmokelib/testing/hooks/cleanup.py7
-rw-r--r--buildscripts/resmokelib/testing/hooks/cleanup_concurrency_workloads.py2
-rw-r--r--buildscripts/resmokelib/testing/hooks/collect_embedded_resources.py2
-rw-r--r--buildscripts/resmokelib/testing/hooks/combine_benchmark_results.py7
-rw-r--r--buildscripts/resmokelib/testing/hooks/combine_benchrun_embedded_results.py7
-rw-r--r--buildscripts/resmokelib/testing/hooks/dbhash.py2
-rw-r--r--buildscripts/resmokelib/testing/hooks/dbhash_background.py13
-rw-r--r--buildscripts/resmokelib/testing/hooks/drop_sharded_collections.py2
-rw-r--r--buildscripts/resmokelib/testing/hooks/initialsync.py33
-rw-r--r--buildscripts/resmokelib/testing/hooks/interface.py6
-rw-r--r--buildscripts/resmokelib/testing/hooks/jsfile.py2
-rw-r--r--buildscripts/resmokelib/testing/hooks/oplog.py2
-rw-r--r--buildscripts/resmokelib/testing/hooks/periodic_kill_secondaries.py14
-rw-r--r--buildscripts/resmokelib/testing/hooks/stepdown.py7
-rw-r--r--buildscripts/resmokelib/testing/hooks/validate.py2
-rw-r--r--buildscripts/resmokelib/testing/hooks/wait_for_replication.py2
-rw-r--r--buildscripts/resmokelib/testing/job.py6
-rw-r--r--buildscripts/resmokelib/testing/report.py2
-rw-r--r--buildscripts/resmokelib/testing/suite.py8
-rw-r--r--buildscripts/resmokelib/testing/summary.py4
-rw-r--r--buildscripts/resmokelib/testing/testcases/__init__.py2
-rw-r--r--buildscripts/resmokelib/testing/testcases/benchmark_test.py4
-rw-r--r--buildscripts/resmokelib/testing/testcases/benchrun_embedded_test.py10
-rw-r--r--buildscripts/resmokelib/testing/testcases/cpp_integration_test.py2
-rw-r--r--buildscripts/resmokelib/testing/testcases/cpp_unittest.py2
-rw-r--r--buildscripts/resmokelib/testing/testcases/dbtest.py2
-rw-r--r--buildscripts/resmokelib/testing/testcases/fsm_workload_test.py4
-rw-r--r--buildscripts/resmokelib/testing/testcases/gennylib_test.py2
-rw-r--r--buildscripts/resmokelib/testing/testcases/gennytest.py2
-rw-r--r--buildscripts/resmokelib/testing/testcases/interface.py10
-rw-r--r--buildscripts/resmokelib/testing/testcases/json_schema_test.py2
-rw-r--r--buildscripts/resmokelib/testing/testcases/jsrunnerfile.py2
-rw-r--r--buildscripts/resmokelib/testing/testcases/jstest.py4
-rw-r--r--buildscripts/resmokelib/testing/testcases/mongos_test.py2
-rw-r--r--buildscripts/resmokelib/testing/testcases/mql_model_haskell_test.py2
-rw-r--r--buildscripts/resmokelib/testing/testcases/mql_model_mongod_test.py2
-rw-r--r--buildscripts/resmokelib/testing/testcases/multi_stmt_txn_test.py2
-rw-r--r--buildscripts/resmokelib/testing/testcases/sleeptest.py2
-rw-r--r--buildscripts/resmokelib/utils/__init__.py21
-rw-r--r--buildscripts/resmokelib/utils/archival.py40
-rw-r--r--buildscripts/resmokelib/utils/autoloader.py2
-rw-r--r--buildscripts/resmokelib/utils/globstar.py4
-rw-r--r--buildscripts/resmokelib/utils/jscomment.py15
-rw-r--r--buildscripts/resmokelib/utils/queue.py8
-rw-r--r--buildscripts/resmokelib/utils/registry.py10
-rw-r--r--buildscripts/resmokelib/utils/scheduler.py2
76 files changed, 371 insertions, 499 deletions
diff --git a/buildscripts/resmokelib/__init__.py b/buildscripts/resmokelib/__init__.py
index c6a0bc9d079..e24f7425280 100644
--- a/buildscripts/resmokelib/__init__.py
+++ b/buildscripts/resmokelib/__init__.py
@@ -1,7 +1,5 @@
"""Empty."""
-from __future__ import absolute_import
-
from . import config
from . import errors
from . import logging
diff --git a/buildscripts/resmokelib/config.py b/buildscripts/resmokelib/config.py
index 50af5c5ccdd..7705682cce1 100644
--- a/buildscripts/resmokelib/config.py
+++ b/buildscripts/resmokelib/config.py
@@ -1,7 +1,5 @@
"""Configuration options for resmoke.py."""
-from __future__ import absolute_import
-
import collections
import datetime
import itertools
@@ -75,7 +73,7 @@ DEFAULTS = {
"repeat_tests_secs": None,
"report_failure_status": "fail",
"report_file": None,
- "seed": long(time.time() * 256), # Taken from random.py code in Python 2.7.
+ "seed": int(time.time() * 256), # Taken from random.py code in Python 2.7.
"service_executor": None,
"shell_conn_string": None,
"shell_port": None,
@@ -181,18 +179,19 @@ class SuiteOptions(_SuiteOptions):
description = None
include_tags = None
parent = dict(
- zip(SuiteOptions._fields, [
- description,
- FAIL_FAST,
- include_tags,
- JOBS,
- REPEAT_SUITES,
- REPEAT_TESTS,
- REPEAT_TESTS_MAX,
- REPEAT_TESTS_MIN,
- REPEAT_TESTS_SECS,
- REPORT_FAILURE_STATUS,
- ]))
+ list(
+ zip(SuiteOptions._fields, [
+ description,
+ FAIL_FAST,
+ include_tags,
+ JOBS,
+ REPEAT_SUITES,
+ REPEAT_TESTS,
+ REPEAT_TESTS_MAX,
+ REPEAT_TESTS_MIN,
+ REPEAT_TESTS_SECS,
+ REPORT_FAILURE_STATUS,
+ ])))
options = self._asdict()
for field in SuiteOptions._fields:
@@ -203,7 +202,7 @@ class SuiteOptions(_SuiteOptions):
SuiteOptions.ALL_INHERITED = SuiteOptions( # type: ignore
- **dict(zip(SuiteOptions._fields, itertools.repeat(SuiteOptions.INHERIT))))
+ **dict(list(zip(SuiteOptions._fields, itertools.repeat(SuiteOptions.INHERIT)))))
##
# Variables that are set by the user at the command line or with --options.
diff --git a/buildscripts/resmokelib/core/__init__.py b/buildscripts/resmokelib/core/__init__.py
index 78a8b4924b8..5f780a34e06 100644
--- a/buildscripts/resmokelib/core/__init__.py
+++ b/buildscripts/resmokelib/core/__init__.py
@@ -1,5 +1,4 @@
"""Resmokelib core module."""
-from __future__ import absolute_import
from . import process
from . import programs
diff --git a/buildscripts/resmokelib/core/jasper_process.py b/buildscripts/resmokelib/core/jasper_process.py
index 60c349743ff..fc4ef79fa9c 100644
--- a/buildscripts/resmokelib/core/jasper_process.py
+++ b/buildscripts/resmokelib/core/jasper_process.py
@@ -3,8 +3,6 @@
Serves as an alternative to process.py.
"""
-from __future__ import absolute_import
-
import sys
try:
diff --git a/buildscripts/resmokelib/core/network.py b/buildscripts/resmokelib/core/network.py
index b48a2221f01..e76e73bbb08 100644
--- a/buildscripts/resmokelib/core/network.py
+++ b/buildscripts/resmokelib/core/network.py
@@ -1,7 +1,5 @@
"""Class used to allocate ports for mongod and mongos processes involved in running the tests."""
-from __future__ import absolute_import
-
import collections
import functools
import threading
diff --git a/buildscripts/resmokelib/core/pipe.py b/buildscripts/resmokelib/core/pipe.py
index 578b934bb7d..c52dbc1235a 100644
--- a/buildscripts/resmokelib/core/pipe.py
+++ b/buildscripts/resmokelib/core/pipe.py
@@ -5,8 +5,6 @@ Used to avoid deadlocks from the pipe buffer filling up and blocking the subproc
being waited on.
"""
-from __future__ import absolute_import
-
import threading
diff --git a/buildscripts/resmokelib/core/process.py b/buildscripts/resmokelib/core/process.py
index 84c067d8e3b..4b8e0f25d0a 100644
--- a/buildscripts/resmokelib/core/process.py
+++ b/buildscripts/resmokelib/core/process.py
@@ -4,34 +4,13 @@ Uses job objects when running on Windows to ensure that all created
processes are terminated.
"""
-from __future__ import absolute_import
-
import atexit
import logging
import os
import os.path
import sys
import threading
-
-# The subprocess32 module resolves the thread-safety issues of the subprocess module in Python 2.x
-# when the _posixsubprocess C extension module is also available. Additionally, the _posixsubprocess
-# C extension module avoids triggering invalid free() calls on Python's internal data structure for
-# thread-local storage by skipping the PyOS_AfterFork() call when the 'preexec_fn' parameter isn't
-# specified to subprocess.Popen(). See SERVER-22219 for more details.
-#
-# The subprocess32 module is untested on Windows and thus isn't recommended for use, even when it's
-# installed. See https://github.com/google/python-subprocess32/blob/3.2.7/README.md#usage.
-if os.name == "posix" and sys.version_info[0] == 2:
- try:
- import subprocess32 as subprocess
- except ImportError:
- import warnings
- warnings.warn(("Falling back to using the subprocess module because subprocess32 isn't"
- " available. When using the subprocess module, a child process may trigger"
- " an invalid free(). See SERVER-22219 for more details."), RuntimeWarning)
- import subprocess # type: ignore
-else:
- import subprocess
+import subprocess
from . import pipe # pylint: disable=wrong-import-position
from .. import utils # pylint: disable=wrong-import-position
@@ -182,8 +161,8 @@ class Process(object):
finally:
win32api.CloseHandle(mongo_signal_handle)
- print "Failed to cleanly exit the program, calling TerminateProcess() on PID: " +\
- str(self._process.pid)
+ print("Failed to cleanly exit the program, calling TerminateProcess() on PID: " +\
+ str(self._process.pid))
# Adapted from implementation of Popen.terminate() in subprocess.py of Python 2.7
# because earlier versions do not catch exceptions.
diff --git a/buildscripts/resmokelib/core/programs.py b/buildscripts/resmokelib/core/programs.py
index d1c0b4ca03e..15db85721eb 100644
--- a/buildscripts/resmokelib/core/programs.py
+++ b/buildscripts/resmokelib/core/programs.py
@@ -3,8 +3,6 @@
Handles all the nitty-gritty parameter conversion.
"""
-from __future__ import absolute_import
-
import json
import os
import os.path
diff --git a/buildscripts/resmokelib/logging/__init__.py b/buildscripts/resmokelib/logging/__init__.py
index d0b4a48ac57..2d527b91434 100644
--- a/buildscripts/resmokelib/logging/__init__.py
+++ b/buildscripts/resmokelib/logging/__init__.py
@@ -1,7 +1,5 @@
"""Extension to the logging package to support buildlogger."""
-from __future__ import absolute_import
-
# Alias the built-in logging.Logger class for type checking arguments. Those interested in
# constructing a new Logger instance should use the loggers.new_logger() function instead.
from logging import Logger
diff --git a/buildscripts/resmokelib/logging/buildlogger.py b/buildscripts/resmokelib/logging/buildlogger.py
index a6aec6af20e..3c7901dc6fe 100644
--- a/buildscripts/resmokelib/logging/buildlogger.py
+++ b/buildscripts/resmokelib/logging/buildlogger.py
@@ -1,7 +1,5 @@
"""Define handlers for communicating with a buildlogger server."""
-from __future__ import absolute_import
-
import functools
import json
import os
@@ -85,7 +83,7 @@ class _LogsSplitter(object):
2 is added to each string size to account for the array representation of the logs,
as each line is preceded by a '[' or a space and followed by a ',' or a ']'.
"""
- return len(json.dumps(line, encoding="utf-8")) + 2
+ return len(json.dumps(line)) + 2
curr_logs = []
curr_logs_size = 0
@@ -200,8 +198,8 @@ class _BaseBuildloggerHandler(handlers.BufferedHandler):
# writing the messages to the fallback logkeeper to avoid putting additional pressure on
# the Evergreen database.
BUILDLOGGER_FALLBACK.warning(
- "Failed to flush all log output (%d messages) to logkeeper.", len(
- self.retry_buffer))
+ "Failed to flush all log output (%d messages) to logkeeper.",
+ len(self.retry_buffer))
# We set a flag to indicate that we failed to flush all log output to logkeeper so
# resmoke.py can exit with a special return code.
@@ -226,10 +224,11 @@ class BuildloggerTestHandler(_BaseBuildloggerHandler):
@_log_on_error
def _finish_test(self, failed=False):
"""Send a POST request to the APPEND_TEST_LOGS_ENDPOINT with the test status."""
- self.post(self.endpoint, headers={
- "X-Sendlogs-Test-Done": "true",
- "X-Sendlogs-Test-Failed": "true" if failed else "false",
- })
+ self.post(
+ self.endpoint, headers={
+ "X-Sendlogs-Test-Done": "true",
+ "X-Sendlogs-Test-Failed": "true" if failed else "false",
+ })
def close(self):
"""Close the buildlogger handler."""
@@ -262,7 +261,9 @@ class BuildloggerServer(object):
"""Initialize BuildloggerServer."""
tmp_globals = {}
self.config = {}
- execfile(_BUILDLOGGER_CONFIG, tmp_globals, self.config)
+ exec(
+ compile(open(_BUILDLOGGER_CONFIG, "rb").read(), _BUILDLOGGER_CONFIG, 'exec'),
+ tmp_globals, self.config)
# Rename "slavename" to "username" if present.
if "slavename" in self.config and "username" not in self.config:
@@ -285,11 +286,12 @@ class BuildloggerServer(object):
handler = handlers.HTTPHandler(url_root=_config.BUILDLOGGER_URL, username=username,
password=password, should_retry=True)
- response = handler.post(CREATE_BUILD_ENDPOINT, data={
- "builder": builder,
- "buildnum": build_num,
- "task_id": _config.EVERGREEN_TASK_ID,
- })
+ response = handler.post(
+ CREATE_BUILD_ENDPOINT, data={
+ "builder": builder,
+ "buildnum": build_num,
+ "task_id": _config.EVERGREEN_TASK_ID,
+ })
return response["id"]
diff --git a/buildscripts/resmokelib/logging/flush.py b/buildscripts/resmokelib/logging/flush.py
index f5c2b356468..b812fc2b606 100644
--- a/buildscripts/resmokelib/logging/flush.py
+++ b/buildscripts/resmokelib/logging/flush.py
@@ -3,14 +3,11 @@
These instances are used to send logs to buildlogger.
"""
-from __future__ import absolute_import
-
import logging
+import sched
import threading
import time
-from ..utils import scheduler
-
_FLUSH_THREAD_LOCK = threading.Lock()
_FLUSH_THREAD = None
@@ -96,7 +93,7 @@ class _FlushThread(threading.Thread):
self.__schedule_updated.wait(secs)
self.__schedule_updated.clear()
- self.__scheduler = scheduler.Scheduler(time.time, interruptible_sleep)
+ self.__scheduler = sched.scheduler(time.time, interruptible_sleep)
self.__schedule_updated = threading.Event()
self.__should_stop = threading.Event()
self.__terminated = threading.Event()
diff --git a/buildscripts/resmokelib/logging/formatters.py b/buildscripts/resmokelib/logging/formatters.py
index 450d5d29cd8..b08e736ddcc 100644
--- a/buildscripts/resmokelib/logging/formatters.py
+++ b/buildscripts/resmokelib/logging/formatters.py
@@ -1,7 +1,5 @@
"""Custom formatters for the logging handlers."""
-from __future__ import absolute_import
-
import logging
import time
diff --git a/buildscripts/resmokelib/logging/handlers.py b/buildscripts/resmokelib/logging/handlers.py
index d67bf13f724..fecbde906fa 100644
--- a/buildscripts/resmokelib/logging/handlers.py
+++ b/buildscripts/resmokelib/logging/handlers.py
@@ -1,7 +1,5 @@
"""Additional handlers that are used as the base classes of the buildlogger handler."""
-from __future__ import absolute_import
-
import json
import logging
import sys
@@ -193,18 +191,15 @@ class HTTPHandler(object):
"""
data = utils.default_if_none(data, [])
- data = json.dumps(data, encoding="utf-8")
+ data = json.dumps(data)
headers = utils.default_if_none(headers, {})
headers["Content-Type"] = "application/json; charset=utf-8"
url = self._make_url(endpoint)
- # Versions of Python earlier than 2.7.9 do not support certificate validation. So we
- # disable certificate validation for older Python versions.
- should_validate_certificates = sys.version_info >= (2, 7, 9)
with warnings.catch_warnings():
- if urllib3_exceptions is not None and not should_validate_certificates:
+ if urllib3_exceptions is not None:
try:
warnings.simplefilter("ignore", urllib3_exceptions.InsecurePlatformWarning)
except AttributeError:
@@ -222,8 +217,7 @@ class HTTPHandler(object):
pass
response = self.session.post(url, data=data, headers=headers, timeout=timeout_secs,
- auth=self.auth_handler,
- verify=should_validate_certificates)
+ auth=self.auth_handler, verify=True)
response.raise_for_status()
diff --git a/buildscripts/resmokelib/logging/loggers.py b/buildscripts/resmokelib/logging/loggers.py
index deb3f6c38a2..4028925af5a 100644
--- a/buildscripts/resmokelib/logging/loggers.py
+++ b/buildscripts/resmokelib/logging/loggers.py
@@ -1,7 +1,5 @@
"""Module to hold the logger instances themselves."""
-from __future__ import absolute_import
-
import logging
import sys
@@ -304,8 +302,8 @@ class FixtureNodeLogger(BaseLogger):
def new_fixture_node_logger(self, node_name):
"""Create a new child FixtureNodeLogger."""
- return FixtureNodeLogger(self.fixture_class, self.job_num, "%s:%s" % (self.node_name,
- node_name), self)
+ return FixtureNodeLogger(self.fixture_class, self.job_num,
+ "%s:%s" % (self.node_name, node_name), self)
class TestsRootLogger(RootLogger):
diff --git a/buildscripts/resmokelib/parser.py b/buildscripts/resmokelib/parser.py
index db30a407b8c..80e3b36e516 100644
--- a/buildscripts/resmokelib/parser.py
+++ b/buildscripts/resmokelib/parser.py
@@ -1,7 +1,5 @@
"""Parser for command line arguments."""
-from __future__ import absolute_import
-
import collections
import os
import os.path
@@ -26,40 +24,45 @@ def _make_parser(): # pylint: disable=too-many-statements
"""Create and return the command line arguments parser."""
parser = optparse.OptionParser()
- parser.add_option("--suites", dest="suite_files", metavar="SUITE1,SUITE2",
- help=("Comma separated list of YAML files that each specify the configuration"
- " of a suite. If the file is located in the resmokeconfig/suites/"
- " directory, then the basename without the .yml extension can be"
- " specified, e.g. 'core'. If a list of files is passed in as"
- " positional arguments, they will be run using the suites'"
- " configurations"))
-
- parser.add_option("--log", dest="logger_file", metavar="LOGGER",
- help=("A YAML file that specifies the logging configuration. If the file is"
- " located in the resmokeconfig/suites/ directory, then the basename"
- " without the .yml extension can be specified, e.g. 'console'."))
-
- parser.add_option("--archiveFile", dest="archive_file", metavar="ARCHIVE_FILE",
- help=("Sets the archive file name for the Evergreen task running the tests."
- " The archive file is JSON format containing a list of tests that were"
- " successfully archived to S3. If unspecified, no data files from tests"
- " will be archived in S3. Tests can be designated for archival in the"
- " task suite configuration file."))
-
- parser.add_option("--archiveLimitMb", type="int", dest="archive_limit_mb",
- metavar="ARCHIVE_LIMIT_MB",
- help=("Sets the limit (in MB) for archived files to S3. A value of 0"
- " indicates there is no limit."))
-
- parser.add_option("--archiveLimitTests", type="int", dest="archive_limit_tests",
- metavar="ARCHIVE_LIMIT_TESTS",
- help=("Sets the maximum number of tests to archive to S3. A value"
- " of 0 indicates there is no limit."))
-
- parser.add_option("--basePort", dest="base_port", metavar="PORT",
- help=("The starting port number to use for mongod and mongos processes"
- " spawned by resmoke.py or the tests themselves. Each fixture and Job"
- " allocates a contiguous range of ports."))
+ parser.add_option(
+ "--suites", dest="suite_files", metavar="SUITE1,SUITE2",
+ help=("Comma separated list of YAML files that each specify the configuration"
+ " of a suite. If the file is located in the resmokeconfig/suites/"
+ " directory, then the basename without the .yml extension can be"
+ " specified, e.g. 'core'. If a list of files is passed in as"
+ " positional arguments, they will be run using the suites'"
+ " configurations"))
+
+ parser.add_option(
+ "--log", dest="logger_file", metavar="LOGGER",
+ help=("A YAML file that specifies the logging configuration. If the file is"
+ " located in the resmokeconfig/suites/ directory, then the basename"
+ " without the .yml extension can be specified, e.g. 'console'."))
+
+ parser.add_option(
+ "--archiveFile", dest="archive_file", metavar="ARCHIVE_FILE",
+ help=("Sets the archive file name for the Evergreen task running the tests."
+ " The archive file is JSON format containing a list of tests that were"
+ " successfully archived to S3. If unspecified, no data files from tests"
+ " will be archived in S3. Tests can be designated for archival in the"
+ " task suite configuration file."))
+
+ parser.add_option(
+ "--archiveLimitMb", type="int", dest="archive_limit_mb", metavar="ARCHIVE_LIMIT_MB",
+ help=("Sets the limit (in MB) for archived files to S3. A value of 0"
+ " indicates there is no limit."))
+
+ parser.add_option(
+ "--archiveLimitTests", type="int", dest="archive_limit_tests",
+ metavar="ARCHIVE_LIMIT_TESTS",
+ help=("Sets the maximum number of tests to archive to S3. A value"
+ " of 0 indicates there is no limit."))
+
+ parser.add_option(
+ "--basePort", dest="base_port", metavar="PORT",
+ help=("The starting port number to use for mongod and mongos processes"
+ " spawned by resmoke.py or the tests themselves. Each fixture and Job"
+ " allocates a contiguous range of ports."))
parser.add_option("--buildloggerUrl", action="store", dest="buildlogger_url", metavar="URL",
help="The root url of the buildlogger server.")
@@ -67,19 +70,19 @@ def _make_parser(): # pylint: disable=too-many-statements
parser.add_option("--continueOnFailure", action="store_true", dest="continue_on_failure",
help="Executes all tests in all suites, even if some of them fail.")
- parser.add_option("--dbpathPrefix", dest="dbpath_prefix", metavar="PATH",
- help=("The directory which will contain the dbpaths of any mongod's started"
- " by resmoke.py or the tests themselves."))
+ parser.add_option(
+ "--dbpathPrefix", dest="dbpath_prefix", metavar="PATH",
+ help=("The directory which will contain the dbpaths of any mongod's started"
+ " by resmoke.py or the tests themselves."))
parser.add_option("--dbtest", dest="dbtest_executable", metavar="PATH",
help="The path to the dbtest executable for resmoke to use.")
- parser.add_option("--excludeWithAnyTags", action="append", dest="exclude_with_any_tags",
- metavar="TAG1,TAG2",
- help=("Comma separated list of tags. Any jstest that contains any of the"
- " specified tags will be excluded from any suites that are run."
- " The tag '{}' is implicitly part of this list.".format(
- _config.EXCLUDED_TAG)))
+ parser.add_option(
+ "--excludeWithAnyTags", action="append", dest="exclude_with_any_tags", metavar="TAG1,TAG2",
+ help=("Comma separated list of tags. Any jstest that contains any of the"
+ " specified tags will be excluded from any suites that are run."
+ " The tag '{}' is implicitly part of this list.".format(_config.EXCLUDED_TAG)))
parser.add_option("-f", "--findSuites", action="store_true", dest="find_suites",
help="Lists the names of the suites that will execute the specified tests.")
@@ -87,29 +90,30 @@ def _make_parser(): # pylint: disable=too-many-statements
parser.add_option("--genny", dest="genny_executable", metavar="PATH",
help="The path to the genny executable for resmoke to use.")
- parser.add_option("--spawnUsing", type="choice", dest="spawn_using", choices=("python",
- "jasper"),
- help=("Allows you to spawn resmoke processes using python or Jasper."
- "Defaults to python. Options are 'python' or 'jasper'."))
+ parser.add_option(
+ "--spawnUsing", type="choice", dest="spawn_using", choices=("python", "jasper"),
+ help=("Allows you to spawn resmoke processes using python or Jasper."
+ "Defaults to python. Options are 'python' or 'jasper'."))
- parser.add_option("--includeWithAnyTags", action="append", dest="include_with_any_tags",
- metavar="TAG1,TAG2",
- help=("Comma separated list of tags. For the jstest portion of the suite(s),"
- " only tests which have at least one of the specified tags will be"
- " run."))
+ parser.add_option(
+ "--includeWithAnyTags", action="append", dest="include_with_any_tags", metavar="TAG1,TAG2",
+ help=("Comma separated list of tags. For the jstest portion of the suite(s),"
+ " only tests which have at least one of the specified tags will be"
+ " run."))
parser.add_option("-n", action="store_const", const="tests", dest="dry_run",
help="Outputs the tests that would be run.")
# TODO: add support for --dryRun=commands
- parser.add_option("--dryRun", type="choice", action="store", dest="dry_run",
- choices=("off", "tests"), metavar="MODE",
- help=("Instead of running the tests, outputs the tests that would be run"
- " (if MODE=tests). Defaults to MODE=%default."))
+ parser.add_option(
+ "--dryRun", type="choice", action="store", dest="dry_run", choices=("off", "tests"),
+ metavar="MODE", help=("Instead of running the tests, outputs the tests that would be run"
+ " (if MODE=tests). Defaults to MODE=%default."))
- parser.add_option("-j", "--jobs", type="int", dest="jobs", metavar="JOBS",
- help=("The number of Job instances to use. Each instance will receive its"
- " own MongoDB deployment to dispatch tests to."))
+ parser.add_option(
+ "-j", "--jobs", type="int", dest="jobs", metavar="JOBS",
+ help=("The number of Job instances to use. Each instance will receive its"
+ " own MongoDB deployment to dispatch tests to."))
parser.add_option("-l", "--listSuites", action="store_true", dest="list_suites",
help="Lists the names of the suites available to execute.")
@@ -120,24 +124,27 @@ def _make_parser(): # pylint: disable=too-many-statements
parser.add_option("--mongod", dest="mongod_executable", metavar="PATH",
help="The path to the mongod executable for resmoke.py to use.")
- parser.add_option("--mongodSetParameters", dest="mongod_set_parameters",
- metavar="{key1: value1, key2: value2, ..., keyN: valueN}",
- help=("Passes one or more --setParameter options to all mongod processes"
- " started by resmoke.py. The argument is specified as bracketed YAML -"
- " i.e. JSON with support for single quoted and unquoted keys."))
+ parser.add_option(
+ "--mongodSetParameters", dest="mongod_set_parameters",
+ metavar="{key1: value1, key2: value2, ..., keyN: valueN}",
+ help=("Passes one or more --setParameter options to all mongod processes"
+ " started by resmoke.py. The argument is specified as bracketed YAML -"
+ " i.e. JSON with support for single quoted and unquoted keys."))
- parser.add_option("--mongoebench", dest="mongoebench_executable", metavar="PATH",
- help=("The path to the mongoebench (benchrun embedded) executable for"
- " resmoke.py to use."))
+ parser.add_option(
+ "--mongoebench", dest="mongoebench_executable", metavar="PATH",
+ help=("The path to the mongoebench (benchrun embedded) executable for"
+ " resmoke.py to use."))
parser.add_option("--mongos", dest="mongos_executable", metavar="PATH",
help="The path to the mongos executable for resmoke.py to use.")
- parser.add_option("--mongosSetParameters", dest="mongos_set_parameters",
- metavar="{key1: value1, key2: value2, ..., keyN: valueN}",
- help=("Passes one or more --setParameter options to all mongos processes"
- " started by resmoke.py. The argument is specified as bracketed YAML -"
- " i.e. JSON with support for single quoted and unquoted keys."))
+ parser.add_option(
+ "--mongosSetParameters", dest="mongos_set_parameters",
+ metavar="{key1: value1, key2: value2, ..., keyN: valueN}",
+ help=("Passes one or more --setParameter options to all mongos processes"
+ " started by resmoke.py. The argument is specified as bracketed YAML -"
+ " i.e. JSON with support for single quoted and unquoted keys."))
parser.add_option("--nojournal", action="store_true", dest="no_journal",
help="Disables journaling for all mongod's.")
@@ -148,55 +155,61 @@ def _make_parser(): # pylint: disable=too-many-statements
parser.add_option("--perfReportFile", dest="perf_report_file", metavar="PERF_REPORT",
help="Writes a JSON file with performance test results.")
- parser.add_option("--shellConnString", dest="shell_conn_string", metavar="CONN_STRING",
- help="Overrides the default fixture and connects with a mongodb:// connection"
- " string to an existing MongoDB cluster instead. This is useful for"
- " connecting to a MongoDB deployment started outside of resmoke.py including"
- " one running in a debugger.")
+ parser.add_option(
+ "--shellConnString", dest="shell_conn_string", metavar="CONN_STRING",
+ help="Overrides the default fixture and connects with a mongodb:// connection"
+ " string to an existing MongoDB cluster instead. This is useful for"
+ " connecting to a MongoDB deployment started outside of resmoke.py including"
+ " one running in a debugger.")
- parser.add_option("--shellPort", dest="shell_port", metavar="PORT",
- help="Convenience form of --shellConnString for connecting to an"
- " existing MongoDB cluster with the URL mongodb://localhost:[PORT]."
- " This is useful for connecting to a server running in a debugger.")
+ parser.add_option(
+ "--shellPort", dest="shell_port", metavar="PORT",
+ help="Convenience form of --shellConnString for connecting to an"
+ " existing MongoDB cluster with the URL mongodb://localhost:[PORT]."
+ " This is useful for connecting to a server running in a debugger.")
parser.add_option("--repeat", "--repeatSuites", type="int", dest="repeat_suites", metavar="N",
help="Repeats the given suite(s) N times, or until one fails.")
- parser.add_option("--repeatTests", type="int", dest="repeat_tests", metavar="N",
- help="Repeats the tests inside each suite N times. This applies to tests"
- " defined in the suite configuration as well as tests defined on the command"
- " line.")
-
- parser.add_option("--repeatTestsMax", type="int", dest="repeat_tests_max", metavar="N",
- help="Repeats the tests inside each suite no more than N time when"
- " --repeatTestsSecs is specified. This applies to tests defined in the suite"
- " configuration as well as tests defined on the command line.")
-
- parser.add_option("--repeatTestsMin", type="int", dest="repeat_tests_min", metavar="N",
- help="Repeats the tests inside each suite at least N times when"
- " --repeatTestsSecs is specified. This applies to tests defined in the suite"
- " configuration as well as tests defined on the command line.")
-
- parser.add_option("--repeatTestsSecs", type="float", dest="repeat_tests_secs",
- metavar="SECONDS",
- help="Repeats the tests inside each suite this amount of time. Note that"
- " this option is mutually exclusive with --repeatTests. This applies to"
- " tests defined in the suite configuration as well as tests defined on the"
- " command line.")
-
- parser.add_option("--reportFailureStatus", type="choice", action="store",
- dest="report_failure_status", choices=("fail",
- "silentfail"), metavar="STATUS",
- help="Controls if the test failure status should be reported as failed"
- " or be silently ignored (STATUS=silentfail). Dynamic test failures will"
- " never be silently ignored. Defaults to STATUS=%default.")
+ parser.add_option(
+ "--repeatTests", type="int", dest="repeat_tests", metavar="N",
+ help="Repeats the tests inside each suite N times. This applies to tests"
+ " defined in the suite configuration as well as tests defined on the command"
+ " line.")
+
+ parser.add_option(
+ "--repeatTestsMax", type="int", dest="repeat_tests_max", metavar="N",
+ help="Repeats the tests inside each suite no more than N time when"
+ " --repeatTestsSecs is specified. This applies to tests defined in the suite"
+ " configuration as well as tests defined on the command line.")
+
+ parser.add_option(
+ "--repeatTestsMin", type="int", dest="repeat_tests_min", metavar="N",
+ help="Repeats the tests inside each suite at least N times when"
+ " --repeatTestsSecs is specified. This applies to tests defined in the suite"
+ " configuration as well as tests defined on the command line.")
+
+ parser.add_option(
+ "--repeatTestsSecs", type="float", dest="repeat_tests_secs", metavar="SECONDS",
+ help="Repeats the tests inside each suite this amount of time. Note that"
+ " this option is mutually exclusive with --repeatTests. This applies to"
+ " tests defined in the suite configuration as well as tests defined on the"
+ " command line.")
+
+ parser.add_option(
+ "--reportFailureStatus", type="choice", action="store", dest="report_failure_status",
+ choices=("fail", "silentfail"), metavar="STATUS",
+ help="Controls if the test failure status should be reported as failed"
+ " or be silently ignored (STATUS=silentfail). Dynamic test failures will"
+ " never be silently ignored. Defaults to STATUS=%default.")
parser.add_option("--reportFile", dest="report_file", metavar="REPORT",
help="Writes a JSON file with test status and timing information.")
- parser.add_option("--seed", type="int", dest="seed", metavar="SEED",
- help=("Seed for the random number generator. Useful in combination with the"
- " --shuffle option for producing a consistent test execution order."))
+ parser.add_option(
+ "--seed", type="int", dest="seed", metavar="SEED",
+ help=("Seed for the random number generator. Useful in combination with the"
+ " --shuffle option for producing a consistent test execution order."))
parser.add_option("--serviceExecutor", dest="service_executor", metavar="EXECUTOR",
help="The service executor used by jstests")
@@ -212,32 +225,36 @@ def _make_parser(): # pylint: disable=too-many-statements
choices=("commands", "compatibility", "legacy"), metavar="WRITE_MODE",
help="The write mode used by the mongo shell.")
- parser.add_option("--shuffle", action="store_const", const="on", dest="shuffle",
- help=("Randomizes the order in which tests are executed. This is equivalent"
- " to specifying --shuffleMode=on."))
-
- parser.add_option("--shuffleMode", type="choice", action="store", dest="shuffle",
- choices=("on", "off", "auto"), metavar="ON|OFF|AUTO",
- help=("Controls whether to randomize the order in which tests are executed."
- " Defaults to auto when not supplied. auto enables randomization in"
- " all cases except when the number of jobs requested is 1."))
-
- parser.add_option("--staggerJobs", type="choice", action="store", dest="stagger_jobs",
- choices=("on", "off"), metavar="ON|OFF",
- help=("Enables or disables the stagger of launching resmoke jobs."
- " Defaults to %default."))
-
- parser.add_option("--majorityReadConcern", type="choice", action="store",
- dest="majority_read_concern", choices=("on", "off"), metavar="ON|OFF",
- help=("Enable or disable majority read concern support."
- " Defaults to %default."))
+ parser.add_option(
+ "--shuffle", action="store_const", const="on", dest="shuffle",
+ help=("Randomizes the order in which tests are executed. This is equivalent"
+ " to specifying --shuffleMode=on."))
+
+ parser.add_option(
+ "--shuffleMode", type="choice", action="store", dest="shuffle",
+ choices=("on", "off", "auto"), metavar="ON|OFF|AUTO",
+ help=("Controls whether to randomize the order in which tests are executed."
+ " Defaults to auto when not supplied. auto enables randomization in"
+ " all cases except when the number of jobs requested is 1."))
+
+ parser.add_option(
+ "--staggerJobs", type="choice", action="store", dest="stagger_jobs", choices=("on", "off"),
+ metavar="ON|OFF", help=("Enables or disables the stagger of launching resmoke jobs."
+ " Defaults to %default."))
+
+ parser.add_option(
+ "--majorityReadConcern", type="choice", action="store", dest="majority_read_concern",
+ choices=("on",
+ "off"), metavar="ON|OFF", help=("Enable or disable majority read concern support."
+ " Defaults to %default."))
parser.add_option("--storageEngine", dest="storage_engine", metavar="ENGINE",
help="The storage engine used by dbtests and jstests.")
- parser.add_option("--storageEngineCacheSizeGB", dest="storage_engine_cache_size_gb",
- metavar="CONFIG", help="Sets the storage engine cache size configuration"
- " setting for all mongod's.")
+ parser.add_option(
+ "--storageEngineCacheSizeGB", dest="storage_engine_cache_size_gb", metavar="CONFIG",
+ help="Sets the storage engine cache size configuration"
+ " setting for all mongod's.")
parser.add_option("--tagFile", dest="tag_file", metavar="OPTIONS",
help="A YAML file that associates tests and tags.")
@@ -251,9 +268,10 @@ def _make_parser(): # pylint: disable=too-many-statements
parser.add_option("--wiredTigerIndexConfigString", dest="wt_index_config", metavar="CONFIG",
help="Sets the WiredTiger index configuration setting for all mongod's.")
- parser.add_option("--executor", dest="executor_file",
- help="OBSOLETE: Superceded by --suites; specify --suites=SUITE path/to/test"
- " to run a particular test under a particular suite configuration.")
+ parser.add_option(
+ "--executor", dest="executor_file",
+ help="OBSOLETE: Superceded by --suites; specify --suites=SUITE path/to/test"
+ " to run a particular test under a particular suite configuration.")
evergreen_options = optparse.OptionGroup(
parser, title=_EVERGREEN_OPTIONS_TITLE,
@@ -264,29 +282,33 @@ def _make_parser(): # pylint: disable=too-many-statements
evergreen_options.add_option("--buildId", dest="build_id", metavar="BUILD_ID",
help="Sets the build ID of the task.")
- evergreen_options.add_option("--distroId", dest="distro_id", metavar="DISTRO_ID",
- help=("Sets the identifier for the Evergreen distro running the"
- " tests."))
+ evergreen_options.add_option(
+ "--distroId", dest="distro_id", metavar="DISTRO_ID",
+ help=("Sets the identifier for the Evergreen distro running the"
+ " tests."))
- evergreen_options.add_option("--executionNumber", type="int", dest="execution_number",
- metavar="EXECUTION_NUMBER",
- help=("Sets the number for the Evergreen execution running the"
- " tests."))
+ evergreen_options.add_option(
+ "--executionNumber", type="int", dest="execution_number", metavar="EXECUTION_NUMBER",
+ help=("Sets the number for the Evergreen execution running the"
+ " tests."))
- evergreen_options.add_option("--gitRevision", dest="git_revision", metavar="GIT_REVISION",
- help=("Sets the git revision for the Evergreen task running the"
- " tests."))
+ evergreen_options.add_option(
+ "--gitRevision", dest="git_revision", metavar="GIT_REVISION",
+ help=("Sets the git revision for the Evergreen task running the"
+ " tests."))
# We intentionally avoid adding a new command line option that starts with --suite so it doesn't
# become ambiguous with the --suites option and break how engineers run resmoke.py locally.
- evergreen_options.add_option("--originSuite", dest="origin_suite", metavar="SUITE",
- help=("Indicates the name of the test suite prior to the"
- " evergreen_generate_resmoke_tasks.py script splitting it"
- " up."))
+ evergreen_options.add_option(
+ "--originSuite", dest="origin_suite", metavar="SUITE",
+ help=("Indicates the name of the test suite prior to the"
+ " evergreen_generate_resmoke_tasks.py script splitting it"
+ " up."))
- evergreen_options.add_option("--patchBuild", action="store_true", dest="patch_build",
- help=("Indicates that the Evergreen task running the tests is a"
- " patch build."))
+ evergreen_options.add_option(
+ "--patchBuild", action="store_true", dest="patch_build",
+ help=("Indicates that the Evergreen task running the tests is a"
+ " patch build."))
evergreen_options.add_option("--projectName", dest="project_name", metavar="PROJECT_NAME",
help=("Sets the name of the Evergreen project running the tests."))
@@ -301,9 +323,10 @@ def _make_parser(): # pylint: disable=too-many-statements
evergreen_options.add_option("--taskId", dest="task_id", metavar="TASK_ID",
help="Sets the Id of the Evergreen task running the tests.")
- evergreen_options.add_option("--variantName", dest="variant_name", metavar="VARIANT_NAME",
- help=("Sets the name of the Evergreen build variant running the"
- " tests."))
+ evergreen_options.add_option(
+ "--variantName", dest="variant_name", metavar="VARIANT_NAME",
+ help=("Sets the name of the Evergreen build variant running the"
+ " tests."))
evergreen_options.add_option("--versionId", dest="version_id", metavar="VERSION_ID",
help="Sets the version ID of the task.")
@@ -318,10 +341,11 @@ def _make_parser(): # pylint: disable=too-many-statements
metavar="BENCHMARK_FILTER",
help="Regex to filter Google benchmark tests to run.")
- benchmark_options.add_option("--benchmarkListTests", dest="benchmark_list_tests",
- action="store_true", metavar="BENCHMARK_LIST_TESTS",
- help=("Lists all Google benchmark test configurations in each"
- " test file."))
+ benchmark_options.add_option(
+ "--benchmarkListTests", dest="benchmark_list_tests", action="store_true",
+ metavar="BENCHMARK_LIST_TESTS",
+ help=("Lists all Google benchmark test configurations in each"
+ " test file."))
benchmark_min_time_help = (
"Minimum time to run each benchmark/benchrun test for. Use this option instead of "
@@ -339,10 +363,10 @@ def _make_parser(): # pylint: disable=too-many-statements
metavar="BENCHMARK_REPETITIONS", help=benchmark_repetitions_help)
benchrun_devices = ["Android", "Desktop"]
- benchmark_options.add_option("--benchrunDevice", dest="benchrun_device", metavar="DEVICE",
- type="choice", action="store", choices=benchrun_devices,
- help=("The device to run the benchrun test on, choose from {}."
- " Defaults to DEVICE='%default'.".format(benchrun_devices)))
+ benchmark_options.add_option(
+ "--benchrunDevice", dest="benchrun_device", metavar="DEVICE", type="choice", action="store",
+ choices=benchrun_devices, help=("The device to run the benchrun test on, choose from {}."
+ " Defaults to DEVICE='%default'.".format(benchrun_devices)))
benchmark_options.add_option("--benchrunReportRoot", dest="benchrun_report_root",
metavar="PATH", help="The root path for benchrun test report.")
@@ -436,8 +460,8 @@ def to_local_args(args=None): # pylint: disable=too-many-locals
else:
other_local_args.append(option_name)
- return [arg for arg in (suites_arg, storage_engine_arg)
- if arg is not None] + other_local_args + extra_args
+ return [arg for arg in (suites_arg, storage_engine_arg) if arg is not None
+ ] + other_local_args + extra_args
def parse_command_line():
@@ -613,7 +637,7 @@ def _update_config_vars(values): # pylint: disable=too-many-statements
_config.SHELL_CONN_STRING = conn_string
if config:
- raise optparse.OptionValueError("Unknown option(s): %s" % (config.keys()))
+ raise optparse.OptionValueError("Unknown option(s): %s" % (list(config.keys())))
def _get_logging_config(pathname):
diff --git a/buildscripts/resmokelib/reportfile.py b/buildscripts/resmokelib/reportfile.py
index 00841de2bc9..d4346b349f1 100644
--- a/buildscripts/resmokelib/reportfile.py
+++ b/buildscripts/resmokelib/reportfile.py
@@ -1,7 +1,5 @@
"""Manage interactions with the report.json file."""
-from __future__ import absolute_import
-
import json
from . import config
diff --git a/buildscripts/resmokelib/selector.py b/buildscripts/resmokelib/selector.py
index b9f5686df8b..36f2e608889 100644
--- a/buildscripts/resmokelib/selector.py
+++ b/buildscripts/resmokelib/selector.py
@@ -4,8 +4,6 @@ Defines filtering rules for what tests to include in a suite depending
on whether they apply to C++ unit tests, dbtests, or JS tests.
"""
-from __future__ import absolute_import
-
import collections
import errno
import fnmatch
@@ -71,7 +69,7 @@ class TestFileExplorer(object):
A list of paths as a list(str).
"""
tests = []
- with open(root_file_path, "rb") as filep:
+ with open(root_file_path, "r") as filep:
for test_path in filep:
test_path = test_path.strip()
tests.append(test_path)
@@ -114,7 +112,7 @@ class TestFileExplorer(object):
program = subprocess.Popen(command, stdout=subprocess.PIPE)
stdout = program.communicate()[0]
- return program.returncode, stdout
+ return program.returncode, stdout.decode("utf-8")
@staticmethod
def parse_tag_file(test_kind):
@@ -313,7 +311,7 @@ def make_expression(conf):
elif isinstance(conf, dict):
if len(conf) != 1:
raise ValueError("Tag matching expressions should only contain one key")
- key = conf.keys()[0]
+ key = list(conf.keys())[0]
value = conf[key]
if key == "$allOf":
return _AllOfExpression(_make_expression_list(value))
diff --git a/buildscripts/resmokelib/sighandler.py b/buildscripts/resmokelib/sighandler.py
index c67d44eb759..5591e325a11 100644
--- a/buildscripts/resmokelib/sighandler.py
+++ b/buildscripts/resmokelib/sighandler.py
@@ -1,7 +1,5 @@
"""Utility to support asynchronously signaling the current process."""
-from __future__ import absolute_import
-
import atexit
import os
import signal
diff --git a/buildscripts/resmokelib/suitesconfig.py b/buildscripts/resmokelib/suitesconfig.py
index 156f15e1413..d55ce7782dd 100644
--- a/buildscripts/resmokelib/suitesconfig.py
+++ b/buildscripts/resmokelib/suitesconfig.py
@@ -1,7 +1,5 @@
"""Module for retrieving the configuration of resmoke.py test suites."""
-from __future__ import absolute_import
-
import collections
import optparse
import os
@@ -33,7 +31,7 @@ def create_test_membership_map(fail_on_missing_selector=False, test_kind=None):
"""
if test_kind is not None:
- if isinstance(test_kind, basestring):
+ if isinstance(test_kind, str):
test_kind = [test_kind]
test_kind = frozenset(test_kind)
@@ -117,6 +115,6 @@ def _get_yaml_config(kind, pathname):
pathname = resmokeconfig.NAMED_SUITES[pathname] # Expand 'pathname' to full path.
if not utils.is_yaml_file(pathname) or not os.path.isfile(pathname):
- raise optparse.OptionValueError("Expected a %s YAML config, but got '%s'" % (kind,
- pathname))
+ raise optparse.OptionValueError(
+ "Expected a %s YAML config, but got '%s'" % (kind, pathname))
return utils.load_yaml_file(pathname)
diff --git a/buildscripts/resmokelib/testing/__init__.py b/buildscripts/resmokelib/testing/__init__.py
index eb58f41f7fe..2a276f6bf85 100644
--- a/buildscripts/resmokelib/testing/__init__.py
+++ b/buildscripts/resmokelib/testing/__init__.py
@@ -1,6 +1,4 @@
"""Extension to the unittest package to support buildlogger and parallel test execution."""
-from __future__ import absolute_import
-
from . import executor
from . import suite
diff --git a/buildscripts/resmokelib/testing/executor.py b/buildscripts/resmokelib/testing/executor.py
index 024cce166a4..71fcd75f6d1 100644
--- a/buildscripts/resmokelib/testing/executor.py
+++ b/buildscripts/resmokelib/testing/executor.py
@@ -1,7 +1,5 @@
"""Driver of the test execution framework."""
-from __future__ import absolute_import
-
import threading
import time
@@ -68,7 +66,7 @@ class TestSuiteExecutor(object): # pylint: disable=too-many-instance-attributes
jobs_to_start = self.num_tests
# Must be done after getting buildlogger configuration.
- self._jobs = [self._make_job(job_num) for job_num in xrange(jobs_to_start)]
+ self._jobs = [self._make_job(job_num) for job_num in range(jobs_to_start)]
def run(self):
"""Execute the test suite.
@@ -130,8 +128,9 @@ class TestSuiteExecutor(object): # pylint: disable=too-many-instance-attributes
test_results_num = len(test_report["results"])
# There should be at least as many tests results as expected number of tests.
if test_results_num < self.num_tests:
- raise errors.ResmokeError("{} reported tests is less than {} expected tests"
- .format(test_results_num, self.num_tests))
+ raise errors.ResmokeError(
+ "{} reported tests is less than {} expected tests".format(
+ test_results_num, self.num_tests))
# Clear the report so it can be reused for the next execution.
for job in self._jobs:
@@ -157,8 +156,9 @@ class TestSuiteExecutor(object): # pylint: disable=too-many-instance-attributes
try:
# Run each Job instance in its own thread.
for job in self._jobs:
- thr = threading.Thread(target=job, args=(test_queue, interrupt_flag), kwargs=dict(
- setup_flag=setup_flag, teardown_flag=teardown_flag))
+ thr = threading.Thread(
+ target=job, args=(test_queue, interrupt_flag), kwargs=dict(
+ setup_flag=setup_flag, teardown_flag=teardown_flag))
# Do not wait for tests to finish executing if interrupted by the user.
thr.daemon = True
thr.start()
diff --git a/buildscripts/resmokelib/testing/fixtures/__init__.py b/buildscripts/resmokelib/testing/fixtures/__init__.py
index 1c592c8f6c7..e70ad8259ee 100644
--- a/buildscripts/resmokelib/testing/fixtures/__init__.py
+++ b/buildscripts/resmokelib/testing/fixtures/__init__.py
@@ -1,7 +1,5 @@
"""Fixture for executing JSTests against."""
-from __future__ import absolute_import
-
from .external import ExternalFixture as _ExternalFixture
from .interface import NoOpFixture as _NoOpFixture
from .interface import make_fixture
diff --git a/buildscripts/resmokelib/testing/fixtures/external.py b/buildscripts/resmokelib/testing/fixtures/external.py
index ff5eba3d652..091059f68bd 100644
--- a/buildscripts/resmokelib/testing/fixtures/external.py
+++ b/buildscripts/resmokelib/testing/fixtures/external.py
@@ -1,7 +1,5 @@
"""External fixture for executing JSTests against."""
-from __future__ import absolute_import
-
from . import interface
diff --git a/buildscripts/resmokelib/testing/fixtures/interface.py b/buildscripts/resmokelib/testing/fixtures/interface.py
index 4a38051fffd..b6b028ceb45 100644
--- a/buildscripts/resmokelib/testing/fixtures/interface.py
+++ b/buildscripts/resmokelib/testing/fixtures/interface.py
@@ -1,7 +1,5 @@
"""Interface of the different fixtures for executing JSTests against."""
-from __future__ import absolute_import
-
import os.path
import time
@@ -25,11 +23,9 @@ def make_fixture(class_name, *args, **kwargs):
return _FIXTURES[class_name](*args, **kwargs)
-class Fixture(object):
+class Fixture(object, metaclass=registry.make_registry_metaclass(_FIXTURES)):
"""Base class for all fixtures."""
- __metaclass__ = registry.make_registry_metaclass(_FIXTURES) # type: ignore
-
# We explicitly set the 'REGISTERED_NAME' attribute so that PyLint realizes that the attribute
# is defined for all subclasses of Fixture.
REGISTERED_NAME = "Fixture"
diff --git a/buildscripts/resmokelib/testing/fixtures/replicaset.py b/buildscripts/resmokelib/testing/fixtures/replicaset.py
index 88d57cde26a..9c449dbc432 100644
--- a/buildscripts/resmokelib/testing/fixtures/replicaset.py
+++ b/buildscripts/resmokelib/testing/fixtures/replicaset.py
@@ -1,10 +1,9 @@
"""Replica set fixture for executing JSTests against."""
-from __future__ import absolute_import
-
import os.path
import time
+import bson.errors
import pymongo
import pymongo.errors
import pymongo.write_concern
@@ -77,11 +76,11 @@ class ReplicaSetFixture(interface.ReplFixture): # pylint: disable=too-many-inst
self.replset_name = self.mongod_options.get("replSet", "rs")
if not self.nodes:
- for i in xrange(self.num_nodes):
+ for i in range(self.num_nodes):
node = self._new_mongod(i, self.replset_name)
self.nodes.append(node)
- for i in xrange(self.num_nodes):
+ for i in range(self.num_nodes):
if self.linear_chain and i > 0:
self.nodes[i].mongod_options["set_parameters"][
"failpoint.forceSyncSourceCandidate"] = {
@@ -207,10 +206,17 @@ class ReplicaSetFixture(interface.ReplFixture): # pylint: disable=too-many-inst
def check_rcmaj_optime(client, node):
"""Return True if all nodes have caught up with the primary."""
- res = client.admin.command({"replSetGetStatus": 1})
+ # TODO SERVER-40078: The server is reporting invalid
+ # dates in its response to the replSetGetStatus
+ # command
+ try:
+ res = client.admin.command({"replSetGetStatus": 1})
+ except bson.errors.InvalidBSON:
+ return False
read_concern_majority_optime = res["optimes"]["readConcernMajorityOpTime"]
- if read_concern_majority_optime >= primary_optime:
+ if (read_concern_majority_optime["t"] == primary_optime["t"]
+ and read_concern_majority_optime["ts"] >= primary_optime["ts"]):
up_to_date_nodes.add(node.port)
return len(up_to_date_nodes) == len(self.nodes)
@@ -303,7 +309,15 @@ class ReplicaSetFixture(interface.ReplFixture): # pylint: disable=too-many-inst
client_admin = client["admin"]
while True:
- status = client_admin.command("replSetGetStatus")
+ # TODO SERVER-40078: The server is reporting invalid
+ # dates in its response to the replSetGetStatus
+ # command
+ try:
+ status = client_admin.command("replSetGetStatus")
+ except bson.errors.InvalidBSON:
+ time.sleep(0.1)
+ continue
+
# The `lastStableRecoveryTimestamp` field contains a stable timestamp guaranteed to
# exist on storage engine recovery to a stable timestamp.
last_stable_recovery_timestamp = status.get("lastStableRecoveryTimestamp", None)
diff --git a/buildscripts/resmokelib/testing/fixtures/shardedcluster.py b/buildscripts/resmokelib/testing/fixtures/shardedcluster.py
index f646910a3bb..b3b744095ef 100644
--- a/buildscripts/resmokelib/testing/fixtures/shardedcluster.py
+++ b/buildscripts/resmokelib/testing/fixtures/shardedcluster.py
@@ -1,7 +1,5 @@
"""Sharded cluster fixture for executing JSTests against."""
-from __future__ import absolute_import
-
import os.path
import time
@@ -69,7 +67,7 @@ class ShardedClusterFixture(interface.Fixture): # pylint: disable=too-many-inst
self.configsvr.setup()
if not self.shards:
- for i in xrange(self.num_shards):
+ for i in range(self.num_shards):
if self.num_rs_nodes_per_shard is None:
shard = self._new_standalone_shard(i)
elif isinstance(self.num_rs_nodes_per_shard, int):
diff --git a/buildscripts/resmokelib/testing/fixtures/standalone.py b/buildscripts/resmokelib/testing/fixtures/standalone.py
index 4174462c07e..0265832dbcb 100644
--- a/buildscripts/resmokelib/testing/fixtures/standalone.py
+++ b/buildscripts/resmokelib/testing/fixtures/standalone.py
@@ -1,7 +1,5 @@
"""Standalone mongod fixture for executing JSTests against."""
-from __future__ import absolute_import
-
import os
import os.path
import time
diff --git a/buildscripts/resmokelib/testing/fixtures/yesfixture.py b/buildscripts/resmokelib/testing/fixtures/yesfixture.py
index 618ba8a48cc..eaaf0a1be56 100644
--- a/buildscripts/resmokelib/testing/fixtures/yesfixture.py
+++ b/buildscripts/resmokelib/testing/fixtures/yesfixture.py
@@ -1,7 +1,5 @@
"""Fixture for generating lots of log messages."""
-from __future__ import absolute_import
-
import signal
from . import interface
diff --git a/buildscripts/resmokelib/testing/hook_test_archival.py b/buildscripts/resmokelib/testing/hook_test_archival.py
index b0ef3725eb2..dd3ec6fbf9b 100644
--- a/buildscripts/resmokelib/testing/hook_test_archival.py
+++ b/buildscripts/resmokelib/testing/hook_test_archival.py
@@ -1,7 +1,5 @@
"""Enable support for archiving tests or hooks."""
-from __future__ import absolute_import
-
import os
import threading
diff --git a/buildscripts/resmokelib/testing/hooks/__init__.py b/buildscripts/resmokelib/testing/hooks/__init__.py
index 82772aa25da..e24cf9eae77 100644
--- a/buildscripts/resmokelib/testing/hooks/__init__.py
+++ b/buildscripts/resmokelib/testing/hooks/__init__.py
@@ -5,8 +5,6 @@ by allowing special code to be executed before or after each test, and
before or after each suite.
"""
-from __future__ import absolute_import
-
from .interface import make_hook
from ...utils import autoloader as _autoloader
diff --git a/buildscripts/resmokelib/testing/hooks/cleanup.py b/buildscripts/resmokelib/testing/hooks/cleanup.py
index ebbda2f1edb..0abf025f588 100644
--- a/buildscripts/resmokelib/testing/hooks/cleanup.py
+++ b/buildscripts/resmokelib/testing/hooks/cleanup.py
@@ -1,7 +1,5 @@
"""Test hook for cleaning up data files created by the fixture."""
-from __future__ import absolute_import
-
import os
from . import interface
@@ -22,8 +20,9 @@ class CleanEveryN(interface.Hook):
# Try to isolate what test triggers the leak by restarting the fixture each time.
if "detect_leaks=1" in os.getenv("ASAN_OPTIONS", ""):
- self.logger.info("ASAN_OPTIONS environment variable set to detect leaks, so restarting"
- " the fixture after each test instead of after every %d.", n)
+ self.logger.info(
+ "ASAN_OPTIONS environment variable set to detect leaks, so restarting"
+ " the fixture after each test instead of after every %d.", n)
n = 1
self.n = n # pylint: disable=invalid-name
diff --git a/buildscripts/resmokelib/testing/hooks/cleanup_concurrency_workloads.py b/buildscripts/resmokelib/testing/hooks/cleanup_concurrency_workloads.py
index 4719b806cf2..9f5e0957514 100644
--- a/buildscripts/resmokelib/testing/hooks/cleanup_concurrency_workloads.py
+++ b/buildscripts/resmokelib/testing/hooks/cleanup_concurrency_workloads.py
@@ -1,7 +1,5 @@
"""Test hook for dropping databases created by the fixture."""
-from __future__ import absolute_import
-
import copy
from buildscripts.resmokelib import utils
diff --git a/buildscripts/resmokelib/testing/hooks/collect_embedded_resources.py b/buildscripts/resmokelib/testing/hooks/collect_embedded_resources.py
index f6ddffc3963..f39399d6dc8 100644
--- a/buildscripts/resmokelib/testing/hooks/collect_embedded_resources.py
+++ b/buildscripts/resmokelib/testing/hooks/collect_embedded_resources.py
@@ -1,7 +1,5 @@
"""Module for generating and collecting embedded resource results."""
-from __future__ import absolute_import
-
import os
from buildscripts.mobile import adb_monitor
diff --git a/buildscripts/resmokelib/testing/hooks/combine_benchmark_results.py b/buildscripts/resmokelib/testing/hooks/combine_benchmark_results.py
index 77f0263b2cc..ec4b6d53235 100644
--- a/buildscripts/resmokelib/testing/hooks/combine_benchmark_results.py
+++ b/buildscripts/resmokelib/testing/hooks/combine_benchmark_results.py
@@ -1,8 +1,5 @@
"""Module for generating the test results file fed into the perf plugin."""
-from __future__ import absolute_import
-from __future__ import division
-
import collections
import datetime
import json
@@ -71,7 +68,7 @@ class CombineBenchmarkResults(interface.Hook):
"results": []
}
- for name, report in self.benchmark_reports.items():
+ for name, report in list(self.benchmark_reports.items()):
test_report = {
"name": name, "context": report.context._asdict(),
"results": report.generate_perf_plugin_dict()
@@ -168,7 +165,7 @@ class _BenchmarkThreadsReport(object):
"""
res = {}
- for thread_count, reports in self.thread_benchmark_map.items():
+ for thread_count, reports in list(self.thread_benchmark_map.items()):
thread_report = {
"error_values": [0 for _ in range(len(reports))],
"ops_per_sec_values": [] # This is actually storing latency per op, not ops/s
diff --git a/buildscripts/resmokelib/testing/hooks/combine_benchrun_embedded_results.py b/buildscripts/resmokelib/testing/hooks/combine_benchrun_embedded_results.py
index b70e57d8741..62f442dba32 100644
--- a/buildscripts/resmokelib/testing/hooks/combine_benchrun_embedded_results.py
+++ b/buildscripts/resmokelib/testing/hooks/combine_benchrun_embedded_results.py
@@ -1,8 +1,5 @@
"""Module for generating the test results file fed into the perf plugin."""
-from __future__ import absolute_import
-from __future__ import division
-
import collections
import datetime
import glob
@@ -62,7 +59,7 @@ class CombineBenchrunEmbeddedResults(cbr.CombineBenchmarkResults):
"results": []
}
- for name, report in self.benchmark_reports.items():
+ for name, report in list(self.benchmark_reports.items()):
test_report = {"name": name, "results": report.generate_perf_plugin_dict()}
perf_report["results"].append(test_report)
@@ -143,7 +140,7 @@ class _BenchrunEmbeddedThreadsReport(object):
"""
res = {}
- for thread_count, reports in self.thread_benchmark_map.items():
+ for thread_count, reports in list(self.thread_benchmark_map.items()):
thread_report = {"error_values": [], "ops_per_sec_values": []}
for report in reports:
diff --git a/buildscripts/resmokelib/testing/hooks/dbhash.py b/buildscripts/resmokelib/testing/hooks/dbhash.py
index 40caa5149c6..557c32bb29d 100644
--- a/buildscripts/resmokelib/testing/hooks/dbhash.py
+++ b/buildscripts/resmokelib/testing/hooks/dbhash.py
@@ -1,7 +1,5 @@
"""Test hook for verifying data consistency across a replica set."""
-from __future__ import absolute_import
-
import os.path
from . import jsfile
diff --git a/buildscripts/resmokelib/testing/hooks/dbhash_background.py b/buildscripts/resmokelib/testing/hooks/dbhash_background.py
index b8122f458b8..9cea6d6957e 100644
--- a/buildscripts/resmokelib/testing/hooks/dbhash_background.py
+++ b/buildscripts/resmokelib/testing/hooks/dbhash_background.py
@@ -4,8 +4,6 @@ Unlike dbhash.py, this version of the hook runs continously in a background thre
running.
"""
-from __future__ import absolute_import
-
import os.path
import sys
import threading
@@ -32,13 +30,14 @@ class CheckReplDBHashInBackground(jsfile.JSHook):
client = self.fixture.mongo_client()
server_status = client.admin.command("serverStatus")
if not server_status["storageEngine"].get("supportsSnapshotReadConcern", False):
- self.logger.info("Not enabling the background thread because '%s' storage engine"
- " doesn't support snapshot reads.",
- server_status["storageEngine"]["name"])
+ self.logger.info(
+ "Not enabling the background thread because '%s' storage engine"
+ " doesn't support snapshot reads.", server_status["storageEngine"]["name"])
return
if not server_status["storageEngine"].get("persistent", False):
- self.logger.info("Not enabling the background thread because '%s' storage engine"
- " is not persistent.", server_status["storageEngine"]["name"])
+ self.logger.info(
+ "Not enabling the background thread because '%s' storage engine"
+ " is not persistent.", server_status["storageEngine"]["name"])
return
self._background_job = _BackgroundJob()
diff --git a/buildscripts/resmokelib/testing/hooks/drop_sharded_collections.py b/buildscripts/resmokelib/testing/hooks/drop_sharded_collections.py
index 90752571909..9a5f15715b1 100644
--- a/buildscripts/resmokelib/testing/hooks/drop_sharded_collections.py
+++ b/buildscripts/resmokelib/testing/hooks/drop_sharded_collections.py
@@ -1,7 +1,5 @@
"""Hook for cleaning up sharded collections created during tests."""
-from __future__ import absolute_import
-
import os.path
from . import jsfile
diff --git a/buildscripts/resmokelib/testing/hooks/initialsync.py b/buildscripts/resmokelib/testing/hooks/initialsync.py
index 36ada61ab00..02156068859 100644
--- a/buildscripts/resmokelib/testing/hooks/initialsync.py
+++ b/buildscripts/resmokelib/testing/hooks/initialsync.py
@@ -1,11 +1,10 @@
"""Test hook for verifying correctness of initial sync."""
-from __future__ import absolute_import
-
import os.path
import random
import bson
+import bson.errors
import pymongo.errors
from . import cleanup
@@ -73,28 +72,38 @@ class BackgroundInitialSyncTestCase(jsfile.DynamicJSTestCase):
# If it's been 'n' tests so far, wait for the initial sync node to finish syncing.
if self._hook.tests_run >= self._hook.n:
- self.logger.info("%d tests have been run against the fixture, waiting for initial sync"
- " node to go into SECONDARY state", self._hook.tests_run)
+ self.logger.info(
+ "%d tests have been run against the fixture, waiting for initial sync"
+ " node to go into SECONDARY state", self._hook.tests_run)
self._hook.tests_run = 0
- cmd = bson.SON([("replSetTest", 1), ("waitForMemberState", 2), ("timeoutMillis",
- 20 * 60 * 1000)])
+ cmd = bson.SON([("replSetTest", 1), ("waitForMemberState", 2),
+ ("timeoutMillis", 20 * 60 * 1000)])
sync_node_conn.admin.command(cmd)
# Check if the initial sync node is in SECONDARY state. If it's been 'n' tests, then it
# should have waited to be in SECONDARY state and the test should be marked as a failure.
# Otherwise, we just skip the hook and will check again after the next test.
try:
- state = sync_node_conn.admin.command("replSetGetStatus").get("myState")
+ while True:
+ # TODO SERVER-40078: The server is reporting invalid
+ # dates in its response to the replSetGetStatus
+ # command
+ try:
+ state = sync_node_conn.admin.command("replSetGetStatus").get("myState")
+ break
+ except bson.errors.InvalidBSON:
+ continue
+
if state != 2:
if self._hook.tests_run == 0:
msg = "Initial sync node did not catch up after waiting 20 minutes"
self.logger.exception("{0} failed: {1}".format(self._hook.description, msg))
raise errors.TestFailure(msg)
- self.logger.info("Initial sync node is in state %d, not state SECONDARY (2)."
- " Skipping BackgroundInitialSync hook for %s", state,
- self._base_test_name)
+ self.logger.info(
+ "Initial sync node is in state %d, not state SECONDARY (2)."
+ " Skipping BackgroundInitialSync hook for %s", state, self._base_test_name)
# If we have not restarted initial sync since the last time we ran the data
# validation, restart initial sync with a 20% probability.
@@ -197,8 +206,8 @@ class IntermediateInitialSyncTestCase(jsfile.DynamicJSTestCase):
# Do initial sync round.
self.logger.info("Waiting for initial sync node to go into SECONDARY state")
- cmd = bson.SON([("replSetTest", 1), ("waitForMemberState", 2), ("timeoutMillis",
- 20 * 60 * 1000)])
+ cmd = bson.SON([("replSetTest", 1), ("waitForMemberState", 2),
+ ("timeoutMillis", 20 * 60 * 1000)])
sync_node_conn.admin.command(cmd)
# Run data validation and dbhash checking.
diff --git a/buildscripts/resmokelib/testing/hooks/interface.py b/buildscripts/resmokelib/testing/hooks/interface.py
index fe53f820fc5..d8ac37e159d 100644
--- a/buildscripts/resmokelib/testing/hooks/interface.py
+++ b/buildscripts/resmokelib/testing/hooks/interface.py
@@ -1,7 +1,5 @@
"""Interface for customizing the behavior of a test fixture."""
-from __future__ import absolute_import
-
import sys
from ..testcases import interface as testcase
@@ -21,11 +19,9 @@ def make_hook(class_name, *args, **kwargs):
return _HOOKS[class_name](*args, **kwargs)
-class Hook(object):
+class Hook(object, metaclass=registry.make_registry_metaclass(_HOOKS)):
"""Common interface all Hooks will inherit from."""
- __metaclass__ = registry.make_registry_metaclass(_HOOKS) # type: ignore
-
REGISTERED_NAME = registry.LEAVE_UNREGISTERED
def __init__(self, hook_logger, fixture, description):
diff --git a/buildscripts/resmokelib/testing/hooks/jsfile.py b/buildscripts/resmokelib/testing/hooks/jsfile.py
index e95d3d6d780..6eedb3f87c4 100644
--- a/buildscripts/resmokelib/testing/hooks/jsfile.py
+++ b/buildscripts/resmokelib/testing/hooks/jsfile.py
@@ -1,7 +1,5 @@
"""Interface for customizing the behavior of a test fixture by executing a JavaScript file."""
-from __future__ import absolute_import
-
from . import interface
from ..testcases import jstest
from ... import errors
diff --git a/buildscripts/resmokelib/testing/hooks/oplog.py b/buildscripts/resmokelib/testing/hooks/oplog.py
index ceb81bb8fd6..6822eecabf0 100644
--- a/buildscripts/resmokelib/testing/hooks/oplog.py
+++ b/buildscripts/resmokelib/testing/hooks/oplog.py
@@ -1,7 +1,5 @@
"""Test hook for verifying members of a replica set have matching oplogs."""
-from __future__ import absolute_import
-
import os.path
from . import jsfile
diff --git a/buildscripts/resmokelib/testing/hooks/periodic_kill_secondaries.py b/buildscripts/resmokelib/testing/hooks/periodic_kill_secondaries.py
index a6f44278140..daf18ad7671 100644
--- a/buildscripts/resmokelib/testing/hooks/periodic_kill_secondaries.py
+++ b/buildscripts/resmokelib/testing/hooks/periodic_kill_secondaries.py
@@ -1,7 +1,5 @@
"""Test hook for verifying correctness of secondary's behavior during an unclean shutdown."""
-from __future__ import absolute_import
-
import time
import bson
@@ -189,8 +187,9 @@ class PeriodicKillSecondariesTestCase(interface.DynamicTestCase):
for secondary in self.fixture.get_secondaries():
self._check_invariants_as_standalone(secondary)
- self.logger.info("Restarting the secondary on port %d as a replica set node with"
- " its data files intact...", secondary.port)
+ self.logger.info(
+ "Restarting the secondary on port %d as a replica set node with"
+ " its data files intact...", secondary.port)
# Start the 'secondary' mongod back up as part of the replica set and wait for it to
# reach state SECONDARY.
secondary.setup()
@@ -253,12 +252,13 @@ class PeriodicKillSecondariesTestCase(interface.DynamicTestCase):
self.fixture.setup()
self.fixture.await_ready()
- def _check_invariants_as_standalone(self, secondary):
+ def _check_invariants_as_standalone(self, secondary): # pylint: disable=too-many-locals
# pylint: disable=too-many-branches,too-many-statements
# We remove the --replSet option in order to start the node as a standalone.
replset_name = secondary.mongod_options.pop("replSet")
- self.logger.info("Restarting the secondary on port %d as a standalone node with"
- " its data files intact...", secondary.port)
+ self.logger.info(
+ "Restarting the secondary on port %d as a standalone node with"
+ " its data files intact...", secondary.port)
try:
secondary.setup()
diff --git a/buildscripts/resmokelib/testing/hooks/stepdown.py b/buildscripts/resmokelib/testing/hooks/stepdown.py
index fbda653a324..f218fffe49e 100644
--- a/buildscripts/resmokelib/testing/hooks/stepdown.py
+++ b/buildscripts/resmokelib/testing/hooks/stepdown.py
@@ -1,5 +1,4 @@
"""Test hook that periodically makes the primary of a replica set step down."""
-from __future__ import absolute_import
import collections
import os.path
@@ -179,7 +178,7 @@ class _StepdownThread(threading.Thread): # pylint: disable=too-many-instance-at
try:
while True:
- if self._is_stopped():
+ if self.__is_stopped():
break
self._wait_for_permission_or_resume()
now = time.time()
@@ -210,7 +209,7 @@ class _StepdownThread(threading.Thread): # pylint: disable=too-many-instance-at
self.resume()
self.join()
- def _is_stopped(self):
+ def __is_stopped(self):
return self._is_stopped_evt.is_set()
def pause(self):
@@ -234,7 +233,7 @@ class _StepdownThread(threading.Thread): # pylint: disable=too-many-instance-at
def _wait_for_permission_or_resume(self):
# Wait until stop, _stepdown_permitted_file or resume.
if self._stepdown_permitted_file:
- while not os.path.isfile(self._stepdown_permitted_file) and not self._is_stopped():
+ while not os.path.isfile(self._stepdown_permitted_file) and not self.__is_stopped():
# Set a short sleep during busy wait time for self._stepdown_permitted_file.
self._wait(0.1)
else:
diff --git a/buildscripts/resmokelib/testing/hooks/validate.py b/buildscripts/resmokelib/testing/hooks/validate.py
index 3239ddbdc06..81bd42ba6d9 100644
--- a/buildscripts/resmokelib/testing/hooks/validate.py
+++ b/buildscripts/resmokelib/testing/hooks/validate.py
@@ -1,7 +1,5 @@
"""Test hook for verifying the consistency and integrity of collection and index data."""
-from __future__ import absolute_import
-
import os.path
from . import jsfile
diff --git a/buildscripts/resmokelib/testing/hooks/wait_for_replication.py b/buildscripts/resmokelib/testing/hooks/wait_for_replication.py
index fbd786a170c..8720f9456e6 100644
--- a/buildscripts/resmokelib/testing/hooks/wait_for_replication.py
+++ b/buildscripts/resmokelib/testing/hooks/wait_for_replication.py
@@ -1,7 +1,5 @@
"""Test hook to wait for replication to complete on a replica set."""
-from __future__ import absolute_import
-
import time
from buildscripts.resmokelib import core
diff --git a/buildscripts/resmokelib/testing/job.py b/buildscripts/resmokelib/testing/job.py
index 908ae85832a..9b5023d0656 100644
--- a/buildscripts/resmokelib/testing/job.py
+++ b/buildscripts/resmokelib/testing/job.py
@@ -1,7 +1,5 @@
"""Enable running tests simultaneously by processing them from a multi-consumer queue."""
-from __future__ import absolute_import
-
import sys
import time
@@ -175,8 +173,8 @@ class Job(object): # pylint: disable=too-many-instance-attributes
test.short_description())
self.report.setFailure(test, return_code=2)
# Always fail fast if the fixture fails.
- raise errors.StopExecution("%s not running after %s" % (self.fixture,
- test.short_description()))
+ raise errors.StopExecution(
+ "%s not running after %s" % (self.fixture, test.short_description()))
finally:
success = self.report.find_test_info(test).status == "pass"
if self.archival:
diff --git a/buildscripts/resmokelib/testing/report.py b/buildscripts/resmokelib/testing/report.py
index fad0d20a0a2..3365a17075a 100644
--- a/buildscripts/resmokelib/testing/report.py
+++ b/buildscripts/resmokelib/testing/report.py
@@ -3,8 +3,6 @@
This is used to support additional test status and timing information for the report.json file.
"""
-from __future__ import absolute_import
-
import copy
import threading
import time
diff --git a/buildscripts/resmokelib/testing/suite.py b/buildscripts/resmokelib/testing/suite.py
index e0f2dda5151..b369261e147 100644
--- a/buildscripts/resmokelib/testing/suite.py
+++ b/buildscripts/resmokelib/testing/suite.py
@@ -1,7 +1,5 @@
"""Holder for the (test kind, list of tests) pair with additional metadata their execution."""
-from __future__ import absolute_import
-
import itertools
import threading
import time
@@ -269,7 +267,7 @@ class Suite(object): # pylint: disable=too-many-instance-attributes
sb.append("Executed %d times in %0.2f seconds:" % (num_iterations, total_time_taken))
combined_summary = _summary.Summary(0, 0.0, 0, 0, 0, 0)
- for iteration in xrange(num_iterations):
+ for iteration in range(num_iterations):
# Summarize each execution as a bulleted list of results.
bulleter_sb = []
summary = self._summarize_report(reports[iteration], start_times[iteration],
@@ -341,8 +339,8 @@ class Suite(object): # pylint: disable=too-many-instance-attributes
def log_summaries(logger, suites, time_taken):
"""Log summary of all suites."""
sb = []
- sb.append("Summary of all suites: %d suites ran in %0.2f seconds" % (len(suites),
- time_taken))
+ sb.append(
+ "Summary of all suites: %d suites ran in %0.2f seconds" % (len(suites), time_taken))
for suite in suites:
suite_sb = []
suite.summarize(suite_sb)
diff --git a/buildscripts/resmokelib/testing/summary.py b/buildscripts/resmokelib/testing/summary.py
index dc92e0b5b34..a5d439b64b0 100644
--- a/buildscripts/resmokelib/testing/summary.py
+++ b/buildscripts/resmokelib/testing/summary.py
@@ -1,7 +1,5 @@
"""Holder for summary information about a test suite."""
-from __future__ import absolute_import
-
import collections
Summary = collections.namedtuple(
@@ -12,6 +10,6 @@ Summary = collections.namedtuple(
def combine(summary1, summary2):
"""Return a summary representing the sum of 'summary1' and 'summary2'."""
args = []
- for i in xrange(len(Summary._fields)):
+ for i in range(len(Summary._fields)):
args.append(summary1[i] + summary2[i])
return Summary._make(args)
diff --git a/buildscripts/resmokelib/testing/testcases/__init__.py b/buildscripts/resmokelib/testing/testcases/__init__.py
index 52869d99de8..d828d97a0b3 100644
--- a/buildscripts/resmokelib/testing/testcases/__init__.py
+++ b/buildscripts/resmokelib/testing/testcases/__init__.py
@@ -1,7 +1,5 @@
"""Package containing subclasses of unittest.TestCase."""
-from __future__ import absolute_import
-
from .interface import make_test_case
from ...utils import autoloader as _autoloader
diff --git a/buildscripts/resmokelib/testing/testcases/benchmark_test.py b/buildscripts/resmokelib/testing/testcases/benchmark_test.py
index ea506c4f7e5..e7760799e42 100644
--- a/buildscripts/resmokelib/testing/testcases/benchmark_test.py
+++ b/buildscripts/resmokelib/testing/testcases/benchmark_test.py
@@ -1,7 +1,5 @@
"""The unittest.TestCase for tests using a MongoDB vendored version of Google Benchmark."""
-from __future__ import absolute_import
-
from buildscripts.resmokelib import config as _config
from buildscripts.resmokelib import core
from buildscripts.resmokelib import parser
@@ -52,7 +50,7 @@ class BenchmarkTestCase(interface.ProcessTestCase):
"benchmark_repetitions": _config.BENCHMARK_REPETITIONS
}
- for key, value in resmoke_bm_options.items():
+ for key, value in list(resmoke_bm_options.items()):
if value is not None:
# 4. sanitize options before passing them to Benchmark's command line.
if key == "benchmark_min_time":
diff --git a/buildscripts/resmokelib/testing/testcases/benchrun_embedded_test.py b/buildscripts/resmokelib/testing/testcases/benchrun_embedded_test.py
index 89b95424cdb..d4666c94074 100644
--- a/buildscripts/resmokelib/testing/testcases/benchrun_embedded_test.py
+++ b/buildscripts/resmokelib/testing/testcases/benchrun_embedded_test.py
@@ -1,7 +1,5 @@
"""The unittest.TestCase for tests using benchrun embedded (mongoebench)."""
-from __future__ import absolute_import
-
import os
import posixpath
@@ -73,7 +71,7 @@ class BenchrunEmbeddedTestCase( # pylint: disable=too-many-instance-attributes
# 3. Override Benchmark options with options set through resmoke's command line.
resmoke_benchrun_options = {"dbpath": self.dbpath, "time": _config.BENCHMARK_MIN_TIME}
- for key, value in resmoke_benchrun_options.items():
+ for key, value in list(resmoke_benchrun_options.items()):
if value is not None:
# 4. sanitize options before passing them to Benchmark's command line.
if key == "time":
@@ -104,7 +102,7 @@ class BenchrunEmbeddedTestCase( # pylint: disable=too-many-instance-attributes
def run_test(self):
"""Run the test for specified number of iterations."""
- for iter_num in xrange(self.benchrun_repetitions):
+ for iter_num in range(self.benchrun_repetitions):
# Set the output file for each iteration.
local_report_path = self._report_path(iter_num)
device_report_path = self._device_report_path(iter_num)
@@ -134,8 +132,8 @@ class BenchrunEmbeddedTestCase( # pylint: disable=too-many-instance-attributes
def _report_dir(self):
"""Return the report directory. Reports are stored in <report_root>/<testname>/<thread>."""
- return os.path.join(self.report_root, self.short_name(), "thread{}".format(
- self.benchrun_threads))
+ return os.path.join(self.report_root, self.short_name(),
+ "thread{}".format(self.benchrun_threads))
@staticmethod
def _report_name(iter_num):
diff --git a/buildscripts/resmokelib/testing/testcases/cpp_integration_test.py b/buildscripts/resmokelib/testing/testcases/cpp_integration_test.py
index df6d7c9fa41..c0313046a20 100644
--- a/buildscripts/resmokelib/testing/testcases/cpp_integration_test.py
+++ b/buildscripts/resmokelib/testing/testcases/cpp_integration_test.py
@@ -1,7 +1,5 @@
"""The unittest.TestCase for C++ integration tests."""
-from __future__ import absolute_import
-
from . import interface
from ... import core
from ... import utils
diff --git a/buildscripts/resmokelib/testing/testcases/cpp_unittest.py b/buildscripts/resmokelib/testing/testcases/cpp_unittest.py
index b9fb427d0da..c54b45a7792 100644
--- a/buildscripts/resmokelib/testing/testcases/cpp_unittest.py
+++ b/buildscripts/resmokelib/testing/testcases/cpp_unittest.py
@@ -1,7 +1,5 @@
"""The unittest.TestCase for C++ unit tests."""
-from __future__ import absolute_import
-
from . import interface
from ... import core
from ... import utils
diff --git a/buildscripts/resmokelib/testing/testcases/dbtest.py b/buildscripts/resmokelib/testing/testcases/dbtest.py
index 4cfbb8c6385..44ea8410c58 100644
--- a/buildscripts/resmokelib/testing/testcases/dbtest.py
+++ b/buildscripts/resmokelib/testing/testcases/dbtest.py
@@ -1,7 +1,5 @@
"""The unittest.TestCase for dbtests."""
-from __future__ import absolute_import
-
import os
import os.path
diff --git a/buildscripts/resmokelib/testing/testcases/fsm_workload_test.py b/buildscripts/resmokelib/testing/testcases/fsm_workload_test.py
index 6cbda33abe9..d184fe84fc5 100644
--- a/buildscripts/resmokelib/testing/testcases/fsm_workload_test.py
+++ b/buildscripts/resmokelib/testing/testcases/fsm_workload_test.py
@@ -1,7 +1,5 @@
"""The unittest.TestCase for FSM workloads."""
-from __future__ import absolute_import
-
import hashlib
import threading
@@ -97,5 +95,5 @@ class ParallelFSMWorkloadTestCase(FSMWorkloadTestCase):
"""Get an unique identifier for a workload group."""
uid = hashlib.md5()
for workload_name in sorted(selected_tests):
- uid.update(workload_name)
+ uid.update(workload_name.encode("utf-8"))
return uid.hexdigest()
diff --git a/buildscripts/resmokelib/testing/testcases/gennylib_test.py b/buildscripts/resmokelib/testing/testcases/gennylib_test.py
index 91200b6c6c1..96eb08c6cb5 100644
--- a/buildscripts/resmokelib/testing/testcases/gennylib_test.py
+++ b/buildscripts/resmokelib/testing/testcases/gennylib_test.py
@@ -1,7 +1,5 @@
"""The unittest.TestCase for gennylib integration tests."""
-from __future__ import absolute_import
-
from . import interface
from ... import core
from ... import utils
diff --git a/buildscripts/resmokelib/testing/testcases/gennytest.py b/buildscripts/resmokelib/testing/testcases/gennytest.py
index 1db9ca48e94..11e51014803 100644
--- a/buildscripts/resmokelib/testing/testcases/gennytest.py
+++ b/buildscripts/resmokelib/testing/testcases/gennytest.py
@@ -1,7 +1,5 @@
"""The unittest.TestCase for genny."""
-from __future__ import absolute_import
-
import os
import os.path
diff --git a/buildscripts/resmokelib/testing/testcases/interface.py b/buildscripts/resmokelib/testing/testcases/interface.py
index 1f078828b1c..e1c1e2faa0c 100644
--- a/buildscripts/resmokelib/testing/testcases/interface.py
+++ b/buildscripts/resmokelib/testing/testcases/interface.py
@@ -3,8 +3,6 @@
This is used to perform the actual test case.
"""
-from __future__ import absolute_import
-
import os
import os.path
import unittest
@@ -23,11 +21,9 @@ def make_test_case(test_kind, *args, **kwargs):
return _TEST_CASES[test_kind](*args, **kwargs)
-class TestCase(unittest.TestCase): # pylint: disable=too-many-instance-attributes
+class TestCase(unittest.TestCase, metaclass=registry.make_registry_metaclass(_TEST_CASES)): # pylint: disable=too-many-instance-attributes
"""A test case to execute."""
- __metaclass__ = registry.make_registry_metaclass(_TEST_CASES) # type: ignore
-
REGISTERED_NAME = registry.LEAVE_UNREGISTERED
def __init__(self, logger, test_kind, test_name, dynamic=False):
@@ -37,10 +33,10 @@ class TestCase(unittest.TestCase): # pylint: disable=too-many-instance-attribut
if not isinstance(logger, logging.Logger):
raise TypeError("logger must be a Logger instance")
- if not isinstance(test_kind, basestring):
+ if not isinstance(test_kind, str):
raise TypeError("test_kind must be a string")
- if not isinstance(test_name, basestring):
+ if not isinstance(test_name, str):
raise TypeError("test_name must be a string")
self._id = uuid.uuid4()
diff --git a/buildscripts/resmokelib/testing/testcases/json_schema_test.py b/buildscripts/resmokelib/testing/testcases/json_schema_test.py
index 08e5a2d71a8..6f40b8af714 100644
--- a/buildscripts/resmokelib/testing/testcases/json_schema_test.py
+++ b/buildscripts/resmokelib/testing/testcases/json_schema_test.py
@@ -1,7 +1,5 @@
"""The unittest.TestCase for JSON Schema tests."""
-from __future__ import absolute_import
-
from buildscripts.resmokelib import config
from buildscripts.resmokelib import core
from buildscripts.resmokelib import utils
diff --git a/buildscripts/resmokelib/testing/testcases/jsrunnerfile.py b/buildscripts/resmokelib/testing/testcases/jsrunnerfile.py
index c2da41faf37..ece5e2ceb15 100644
--- a/buildscripts/resmokelib/testing/testcases/jsrunnerfile.py
+++ b/buildscripts/resmokelib/testing/testcases/jsrunnerfile.py
@@ -1,7 +1,5 @@
"""The unittest.TestCase for tests with a static JavaScript runner file."""
-from __future__ import absolute_import
-
from buildscripts.resmokelib import config
from buildscripts.resmokelib import core
from buildscripts.resmokelib import utils
diff --git a/buildscripts/resmokelib/testing/testcases/jstest.py b/buildscripts/resmokelib/testing/testcases/jstest.py
index 4d36ac965b0..6b6a5ee45e2 100644
--- a/buildscripts/resmokelib/testing/testcases/jstest.py
+++ b/buildscripts/resmokelib/testing/testcases/jstest.py
@@ -1,7 +1,5 @@
"""The unittest.TestCase for JavaScript tests."""
-from __future__ import absolute_import
-
import os
import os.path
import sys
@@ -201,7 +199,7 @@ class JSTestCase(interface.ProcessTestCase):
test_cases = []
try:
# If there are multiple clients, make a new thread for each client.
- for thread_id in xrange(self.num_clients):
+ for thread_id in range(self.num_clients):
logger = self.logger.new_test_thread_logger(self.test_kind, str(thread_id))
test_case = self._create_test_case_for_thread(logger, thread_id)
test_cases.append(test_case)
diff --git a/buildscripts/resmokelib/testing/testcases/mongos_test.py b/buildscripts/resmokelib/testing/testcases/mongos_test.py
index 9914ba8677f..f00a93a9bdb 100644
--- a/buildscripts/resmokelib/testing/testcases/mongos_test.py
+++ b/buildscripts/resmokelib/testing/testcases/mongos_test.py
@@ -1,7 +1,5 @@
"""The unittest.TestCase for mongos --test."""
-from __future__ import absolute_import
-
from . import interface
from ... import config
from ... import core
diff --git a/buildscripts/resmokelib/testing/testcases/mql_model_haskell_test.py b/buildscripts/resmokelib/testing/testcases/mql_model_haskell_test.py
index 7911aa1b7d6..fff521ddae5 100644
--- a/buildscripts/resmokelib/testing/testcases/mql_model_haskell_test.py
+++ b/buildscripts/resmokelib/testing/testcases/mql_model_haskell_test.py
@@ -1,7 +1,5 @@
"""The unittest.TestCase for MQL Haskell tests."""
-from __future__ import absolute_import
-
import os
import os.path
diff --git a/buildscripts/resmokelib/testing/testcases/mql_model_mongod_test.py b/buildscripts/resmokelib/testing/testcases/mql_model_mongod_test.py
index eddfef2b7b3..249b0f18fb2 100644
--- a/buildscripts/resmokelib/testing/testcases/mql_model_mongod_test.py
+++ b/buildscripts/resmokelib/testing/testcases/mql_model_mongod_test.py
@@ -1,7 +1,5 @@
"""The unittest.TestCase for MQL MongoD Model tests."""
-from __future__ import absolute_import
-
import os
import os.path
diff --git a/buildscripts/resmokelib/testing/testcases/multi_stmt_txn_test.py b/buildscripts/resmokelib/testing/testcases/multi_stmt_txn_test.py
index 1e790612153..81ddfa289ae 100644
--- a/buildscripts/resmokelib/testing/testcases/multi_stmt_txn_test.py
+++ b/buildscripts/resmokelib/testing/testcases/multi_stmt_txn_test.py
@@ -1,7 +1,5 @@
"""unittest.TestCase for multi-statement transaction passthrough tests."""
-from __future__ import absolute_import
-
from buildscripts.resmokelib import config
from buildscripts.resmokelib import core
from buildscripts.resmokelib import utils
diff --git a/buildscripts/resmokelib/testing/testcases/sleeptest.py b/buildscripts/resmokelib/testing/testcases/sleeptest.py
index 163100095d3..4f073ee6f25 100644
--- a/buildscripts/resmokelib/testing/testcases/sleeptest.py
+++ b/buildscripts/resmokelib/testing/testcases/sleeptest.py
@@ -1,7 +1,5 @@
"""The unittest.TestCase for sleeping a given amount of time."""
-from __future__ import absolute_import
-
import time
from . import interface
diff --git a/buildscripts/resmokelib/utils/__init__.py b/buildscripts/resmokelib/utils/__init__.py
index 5f60f162696..02681bca0b5 100644
--- a/buildscripts/resmokelib/utils/__init__.py
+++ b/buildscripts/resmokelib/utils/__init__.py
@@ -1,8 +1,5 @@
"""Helper functions."""
-from __future__ import absolute_import
-from __future__ import print_function
-
import contextlib
import os.path
import shutil
@@ -40,19 +37,7 @@ def default_if_none(value, default):
def rmtree(path, **kwargs):
- """Wrap shutil.rmtreee.
-
- Use a UTF-8 unicode path if Windows.
- See https://bugs.python.org/issue24672, where shutil.rmtree can fail with UTF-8.
- Use a bytes path to rmtree, otherwise.
- See https://github.com/pypa/setuptools/issues/706.
- """
- if is_windows():
- if not isinstance(path, unicode):
- path = unicode(path, "utf-8")
- else:
- if isinstance(path, unicode):
- path = path.encode("utf-8")
+ """Wrap shutil.rmtree."""
shutil.rmtree(path, **kwargs)
@@ -72,12 +57,12 @@ def remove_if_exists(path):
def is_string_list(lst):
"""Return true if 'lst' is a list of strings, and false otherwise."""
- return isinstance(lst, list) and all(isinstance(x, basestring) for x in lst)
+ return isinstance(lst, list) and all(isinstance(x, str) for x in lst)
def is_string_set(value):
"""Return true if 'value' is a set of strings, and false otherwise."""
- return isinstance(value, set) and all(isinstance(x, basestring) for x in value)
+ return isinstance(value, set) and all(isinstance(x, str) for x in value)
def is_js_file(filename):
diff --git a/buildscripts/resmokelib/utils/archival.py b/buildscripts/resmokelib/utils/archival.py
index 8ccb3127f47..a31f8fd2d31 100644
--- a/buildscripts/resmokelib/utils/archival.py
+++ b/buildscripts/resmokelib/utils/archival.py
@@ -1,8 +1,6 @@
"""Archival utility."""
-from __future__ import absolute_import
-
-import Queue
+import queue
import collections
import json
import math
@@ -45,7 +43,7 @@ def file_list_size(files):
def directory_size(directory):
"""Return size (in bytes) of files in 'directory' tree."""
dir_bytes = 0
- for root_dir, _, files in os.walk(unicode(directory)):
+ for root_dir, _, files in os.walk(str(directory)):
for name in files:
full_name = os.path.join(root_dir, name)
try:
@@ -103,7 +101,7 @@ class Archival(object): # pylint: disable=too-many-instance-attributes
self._lock = threading.Lock()
# Start the worker thread to update the 'archival_json_file'.
- self._archive_file_queue = Queue.Queue()
+ self._archive_file_queue = queue.Queue()
self._archive_file_worker = threading.Thread(target=self._update_archive_file_wkr,
args=(self._archive_file_queue,
logger), name="archive_file_worker")
@@ -115,10 +113,10 @@ class Archival(object): # pylint: disable=too-many-instance-attributes
self.s3_client = s3_client
# Start the worker thread which uploads the archive.
- self._upload_queue = Queue.Queue()
- self._upload_worker = threading.Thread(target=self._upload_to_s3_wkr,
- args=(self._upload_queue, self._archive_file_queue,
- logger, self.s3_client), name="upload_worker")
+ self._upload_queue = queue.Queue()
+ self._upload_worker = threading.Thread(
+ target=self._upload_to_s3_wkr, args=(self._upload_queue, self._archive_file_queue,
+ logger, self.s3_client), name="upload_worker")
self._upload_worker.setDaemon(True)
self._upload_worker.start()
@@ -161,14 +159,14 @@ class Archival(object): # pylint: disable=too-many-instance-attributes
return status, message
@staticmethod
- def _update_archive_file_wkr(queue, logger):
- """Worker thread: Update the archival JSON file from 'queue'."""
+ def _update_archive_file_wkr(work_queue, logger):
+ """Worker thread: Update the archival JSON file from 'work_queue'."""
archival_json = []
while True:
- archive_args = queue.get()
+ archive_args = work_queue.get()
# Exit worker thread when sentinel is received.
if archive_args is None:
- queue.task_done()
+ work_queue.task_done()
break
archival_record = {
"name": archive_args.display_name, "link": archive_args.remote_file,
@@ -179,17 +177,17 @@ class Archival(object): # pylint: disable=too-many-instance-attributes
archival_json.append(archival_record)
with open(archive_args.archival_file, "w") as archival_fh:
json.dump(archival_json, archival_fh)
- queue.task_done()
+ work_queue.task_done()
@staticmethod
- def _upload_to_s3_wkr(queue, archive_file_queue, logger, s3_client):
- """Worker thread: Upload to S3 from 'queue', dispatch to 'archive_file_queue'."""
+ def _upload_to_s3_wkr(work_queue, archive_file_work_queue, logger, s3_client):
+ """Worker thread: Upload to S3 from 'work_queue', dispatch to 'archive_file_work_queue'."""
while True:
- upload_args = queue.get()
+ upload_args = work_queue.get()
# Exit worker thread when sentinel is received.
if upload_args is None:
- queue.task_done()
- archive_file_queue.put(None)
+ work_queue.task_done()
+ archive_file_work_queue.put(None)
break
extra_args = {"ContentType": upload_args.content_type, "ACL": "public-read"}
logger.debug("Uploading to S3 %s to bucket %s path %s", upload_args.local_file,
@@ -212,10 +210,10 @@ class Archival(object): # pylint: disable=too-many-instance-attributes
remote_file = "https://s3.amazonaws.com/{}/{}".format(upload_args.s3_bucket,
upload_args.s3_path)
if upload_completed:
- archive_file_queue.put(
+ archive_file_work_queue.put(
ArchiveArgs(upload_args.archival_file, upload_args.display_name, remote_file))
- queue.task_done()
+ work_queue.task_done()
def _archive_files(self, display_name, input_files, s3_bucket, s3_path):
"""
diff --git a/buildscripts/resmokelib/utils/autoloader.py b/buildscripts/resmokelib/utils/autoloader.py
index 73b58563451..ab1ace84798 100644
--- a/buildscripts/resmokelib/utils/autoloader.py
+++ b/buildscripts/resmokelib/utils/autoloader.py
@@ -1,7 +1,5 @@
"""Utility for loading all modules within a package."""
-from __future__ import absolute_import
-
import importlib
import pkgutil
diff --git a/buildscripts/resmokelib/utils/globstar.py b/buildscripts/resmokelib/utils/globstar.py
index 1e016875f94..5857870e627 100644
--- a/buildscripts/resmokelib/utils/globstar.py
+++ b/buildscripts/resmokelib/utils/globstar.py
@@ -1,7 +1,5 @@
"""Filename globbing utility."""
-from __future__ import absolute_import
-
import glob as _glob
import os
import os.path
@@ -134,7 +132,7 @@ def _list_dir(pathname):
"""
try:
- (_root, dirs, files) = os.walk(pathname).next()
+ (_root, dirs, files) = next(os.walk(pathname))
return (dirs, files)
except StopIteration:
return None # 'pathname' directory does not exist
diff --git a/buildscripts/resmokelib/utils/jscomment.py b/buildscripts/resmokelib/utils/jscomment.py
index 67758197c5c..21d1cfa783c 100644
--- a/buildscripts/resmokelib/utils/jscomment.py
+++ b/buildscripts/resmokelib/utils/jscomment.py
@@ -1,13 +1,11 @@
"""Utility for parsing JS comments."""
-from __future__ import absolute_import
-
import re
import yaml
# TODO: use a more robust regular expression for matching tags
-_JSTEST_TAGS_RE = re.compile(r".*@tags\s*:\s*(\[[^\]]*\])", re.DOTALL)
+_JSTEST_TAGS_RE = re.compile(b".*@tags\s*:\s*(\[[^\]]*\])", re.DOTALL)
def get_tags(pathname):
@@ -29,19 +27,19 @@ def get_tags(pathname):
*/
"""
- with open(pathname) as fp:
+ with open(pathname, 'rb') as fp:
match = _JSTEST_TAGS_RE.match(fp.read())
if match:
try:
# TODO: it might be worth supporting the block (indented) style of YAML lists in
# addition to the flow (bracketed) style
tags = yaml.safe_load(_strip_jscomments(match.group(1)))
- if not isinstance(tags, list) and all(isinstance(tag, basestring) for tag in tags):
+ if not isinstance(tags, list) and all(isinstance(tag, str) for tag in tags):
raise TypeError("Expected a list of string tags, but got '%s'" % (tags))
return tags
except yaml.YAMLError as err:
- raise ValueError("File '%s' contained invalid tags (expected YAML): %s" % (pathname,
- err))
+ raise ValueError(
+ "File '%s' contained invalid tags (expected YAML): %s" % (pathname, err))
return []
@@ -68,6 +66,9 @@ def _strip_jscomments(string):
yaml_lines = []
+ if isinstance(string, bytes):
+ string = string.decode("utf-8")
+
for line in string.splitlines():
# Remove leading whitespace and symbols that commonly appear in JS comments.
line = line.lstrip("\t ").lstrip("*/")
diff --git a/buildscripts/resmokelib/utils/queue.py b/buildscripts/resmokelib/utils/queue.py
index c77692138b1..90c57408621 100644
--- a/buildscripts/resmokelib/utils/queue.py
+++ b/buildscripts/resmokelib/utils/queue.py
@@ -6,16 +6,14 @@ in order for KeyboardInterrupt exceptions to get propagated.
See https://bugs.python.org/issue1167930 for more details.
"""
-from __future__ import absolute_import
-
-import Queue as _Queue
+import queue as _queue
import time
# Exception that is raised when get_nowait() is called on an empty Queue.
-Empty = _Queue.Empty
+Empty = _queue.Empty
-class Queue(_Queue.Queue):
+class Queue(_queue.Queue):
"""A multi-producer, multi-consumer queue."""
def join(self, timeout=None): # pylint: disable=arguments-differ
diff --git a/buildscripts/resmokelib/utils/registry.py b/buildscripts/resmokelib/utils/registry.py
index 0aa02f4b2b5..4248b8c38b3 100644
--- a/buildscripts/resmokelib/utils/registry.py
+++ b/buildscripts/resmokelib/utils/registry.py
@@ -6,8 +6,6 @@ This pattern enables the associated class to be looked up later by using
its name.
"""
-from __future__ import absolute_import
-
# Specifying 'LEAVE_UNREGISTERED' as the "REGISTERED_NAME" attribute will cause the class to be
# omitted from the registry. This is particularly useful for base classes that define an interface
# or common functionality, and aren't intended to be constructed explicitly.
@@ -23,7 +21,7 @@ def make_registry_metaclass(registry_store):
class Registry(type):
"""A metaclass that stores a reference to all registered classes."""
- def __new__(mcs, class_name, base_classes, class_dict):
+ def __new__(mcs, class_name, base_classes, class_dict): # pylint: disable=bad-mcs-classmethod-argument
"""Create and returns a new instance of Registry.
The registry is a class named 'class_name' derived from 'base_classes'
@@ -46,9 +44,9 @@ def make_registry_metaclass(registry_store):
if registered_name is not LEAVE_UNREGISTERED:
if registered_name in registry_store:
- raise ValueError("The name %s is already registered; a different value for the"
- " 'REGISTERED_NAME' attribute must be chosen" %
- (registered_name))
+ raise ValueError(
+ "The name %s is already registered; a different value for the"
+ " 'REGISTERED_NAME' attribute must be chosen" % (registered_name))
registry_store[registered_name] = cls
return cls
diff --git a/buildscripts/resmokelib/utils/scheduler.py b/buildscripts/resmokelib/utils/scheduler.py
index 04abafcd330..9f57d0a110e 100644
--- a/buildscripts/resmokelib/utils/scheduler.py
+++ b/buildscripts/resmokelib/utils/scheduler.py
@@ -1,7 +1,5 @@
"""Thread-safe version of sched.scheduler; the class wasn't made thread-safe until Python 3.3."""
-from __future__ import absolute_import
-
import heapq
import sched
import threading