diff options
Diffstat (limited to 'buildscripts/resmokelib/testing')
49 files changed, 91 insertions, 163 deletions
diff --git a/buildscripts/resmokelib/testing/__init__.py b/buildscripts/resmokelib/testing/__init__.py index eb58f41f7fe..2a276f6bf85 100644 --- a/buildscripts/resmokelib/testing/__init__.py +++ b/buildscripts/resmokelib/testing/__init__.py @@ -1,6 +1,4 @@ """Extension to the unittest package to support buildlogger and parallel test execution.""" -from __future__ import absolute_import - from . import executor from . import suite diff --git a/buildscripts/resmokelib/testing/executor.py b/buildscripts/resmokelib/testing/executor.py index 024cce166a4..71fcd75f6d1 100644 --- a/buildscripts/resmokelib/testing/executor.py +++ b/buildscripts/resmokelib/testing/executor.py @@ -1,7 +1,5 @@ """Driver of the test execution framework.""" -from __future__ import absolute_import - import threading import time @@ -68,7 +66,7 @@ class TestSuiteExecutor(object): # pylint: disable=too-many-instance-attributes jobs_to_start = self.num_tests # Must be done after getting buildlogger configuration. - self._jobs = [self._make_job(job_num) for job_num in xrange(jobs_to_start)] + self._jobs = [self._make_job(job_num) for job_num in range(jobs_to_start)] def run(self): """Execute the test suite. @@ -130,8 +128,9 @@ class TestSuiteExecutor(object): # pylint: disable=too-many-instance-attributes test_results_num = len(test_report["results"]) # There should be at least as many tests results as expected number of tests. if test_results_num < self.num_tests: - raise errors.ResmokeError("{} reported tests is less than {} expected tests" - .format(test_results_num, self.num_tests)) + raise errors.ResmokeError( + "{} reported tests is less than {} expected tests".format( + test_results_num, self.num_tests)) # Clear the report so it can be reused for the next execution. for job in self._jobs: @@ -157,8 +156,9 @@ class TestSuiteExecutor(object): # pylint: disable=too-many-instance-attributes try: # Run each Job instance in its own thread. for job in self._jobs: - thr = threading.Thread(target=job, args=(test_queue, interrupt_flag), kwargs=dict( - setup_flag=setup_flag, teardown_flag=teardown_flag)) + thr = threading.Thread( + target=job, args=(test_queue, interrupt_flag), kwargs=dict( + setup_flag=setup_flag, teardown_flag=teardown_flag)) # Do not wait for tests to finish executing if interrupted by the user. thr.daemon = True thr.start() diff --git a/buildscripts/resmokelib/testing/fixtures/__init__.py b/buildscripts/resmokelib/testing/fixtures/__init__.py index 1c592c8f6c7..e70ad8259ee 100644 --- a/buildscripts/resmokelib/testing/fixtures/__init__.py +++ b/buildscripts/resmokelib/testing/fixtures/__init__.py @@ -1,7 +1,5 @@ """Fixture for executing JSTests against.""" -from __future__ import absolute_import - from .external import ExternalFixture as _ExternalFixture from .interface import NoOpFixture as _NoOpFixture from .interface import make_fixture diff --git a/buildscripts/resmokelib/testing/fixtures/external.py b/buildscripts/resmokelib/testing/fixtures/external.py index ff5eba3d652..091059f68bd 100644 --- a/buildscripts/resmokelib/testing/fixtures/external.py +++ b/buildscripts/resmokelib/testing/fixtures/external.py @@ -1,7 +1,5 @@ """External fixture for executing JSTests against.""" -from __future__ import absolute_import - from . import interface diff --git a/buildscripts/resmokelib/testing/fixtures/interface.py b/buildscripts/resmokelib/testing/fixtures/interface.py index 4a38051fffd..b6b028ceb45 100644 --- a/buildscripts/resmokelib/testing/fixtures/interface.py +++ b/buildscripts/resmokelib/testing/fixtures/interface.py @@ -1,7 +1,5 @@ """Interface of the different fixtures for executing JSTests against.""" -from __future__ import absolute_import - import os.path import time @@ -25,11 +23,9 @@ def make_fixture(class_name, *args, **kwargs): return _FIXTURES[class_name](*args, **kwargs) -class Fixture(object): +class Fixture(object, metaclass=registry.make_registry_metaclass(_FIXTURES)): """Base class for all fixtures.""" - __metaclass__ = registry.make_registry_metaclass(_FIXTURES) # type: ignore - # We explicitly set the 'REGISTERED_NAME' attribute so that PyLint realizes that the attribute # is defined for all subclasses of Fixture. REGISTERED_NAME = "Fixture" diff --git a/buildscripts/resmokelib/testing/fixtures/replicaset.py b/buildscripts/resmokelib/testing/fixtures/replicaset.py index 88d57cde26a..9c449dbc432 100644 --- a/buildscripts/resmokelib/testing/fixtures/replicaset.py +++ b/buildscripts/resmokelib/testing/fixtures/replicaset.py @@ -1,10 +1,9 @@ """Replica set fixture for executing JSTests against.""" -from __future__ import absolute_import - import os.path import time +import bson.errors import pymongo import pymongo.errors import pymongo.write_concern @@ -77,11 +76,11 @@ class ReplicaSetFixture(interface.ReplFixture): # pylint: disable=too-many-inst self.replset_name = self.mongod_options.get("replSet", "rs") if not self.nodes: - for i in xrange(self.num_nodes): + for i in range(self.num_nodes): node = self._new_mongod(i, self.replset_name) self.nodes.append(node) - for i in xrange(self.num_nodes): + for i in range(self.num_nodes): if self.linear_chain and i > 0: self.nodes[i].mongod_options["set_parameters"][ "failpoint.forceSyncSourceCandidate"] = { @@ -207,10 +206,17 @@ class ReplicaSetFixture(interface.ReplFixture): # pylint: disable=too-many-inst def check_rcmaj_optime(client, node): """Return True if all nodes have caught up with the primary.""" - res = client.admin.command({"replSetGetStatus": 1}) + # TODO SERVER-40078: The server is reporting invalid + # dates in its response to the replSetGetStatus + # command + try: + res = client.admin.command({"replSetGetStatus": 1}) + except bson.errors.InvalidBSON: + return False read_concern_majority_optime = res["optimes"]["readConcernMajorityOpTime"] - if read_concern_majority_optime >= primary_optime: + if (read_concern_majority_optime["t"] == primary_optime["t"] + and read_concern_majority_optime["ts"] >= primary_optime["ts"]): up_to_date_nodes.add(node.port) return len(up_to_date_nodes) == len(self.nodes) @@ -303,7 +309,15 @@ class ReplicaSetFixture(interface.ReplFixture): # pylint: disable=too-many-inst client_admin = client["admin"] while True: - status = client_admin.command("replSetGetStatus") + # TODO SERVER-40078: The server is reporting invalid + # dates in its response to the replSetGetStatus + # command + try: + status = client_admin.command("replSetGetStatus") + except bson.errors.InvalidBSON: + time.sleep(0.1) + continue + # The `lastStableRecoveryTimestamp` field contains a stable timestamp guaranteed to # exist on storage engine recovery to a stable timestamp. last_stable_recovery_timestamp = status.get("lastStableRecoveryTimestamp", None) diff --git a/buildscripts/resmokelib/testing/fixtures/shardedcluster.py b/buildscripts/resmokelib/testing/fixtures/shardedcluster.py index f646910a3bb..b3b744095ef 100644 --- a/buildscripts/resmokelib/testing/fixtures/shardedcluster.py +++ b/buildscripts/resmokelib/testing/fixtures/shardedcluster.py @@ -1,7 +1,5 @@ """Sharded cluster fixture for executing JSTests against.""" -from __future__ import absolute_import - import os.path import time @@ -69,7 +67,7 @@ class ShardedClusterFixture(interface.Fixture): # pylint: disable=too-many-inst self.configsvr.setup() if not self.shards: - for i in xrange(self.num_shards): + for i in range(self.num_shards): if self.num_rs_nodes_per_shard is None: shard = self._new_standalone_shard(i) elif isinstance(self.num_rs_nodes_per_shard, int): diff --git a/buildscripts/resmokelib/testing/fixtures/standalone.py b/buildscripts/resmokelib/testing/fixtures/standalone.py index 4174462c07e..0265832dbcb 100644 --- a/buildscripts/resmokelib/testing/fixtures/standalone.py +++ b/buildscripts/resmokelib/testing/fixtures/standalone.py @@ -1,7 +1,5 @@ """Standalone mongod fixture for executing JSTests against.""" -from __future__ import absolute_import - import os import os.path import time diff --git a/buildscripts/resmokelib/testing/fixtures/yesfixture.py b/buildscripts/resmokelib/testing/fixtures/yesfixture.py index 618ba8a48cc..eaaf0a1be56 100644 --- a/buildscripts/resmokelib/testing/fixtures/yesfixture.py +++ b/buildscripts/resmokelib/testing/fixtures/yesfixture.py @@ -1,7 +1,5 @@ """Fixture for generating lots of log messages.""" -from __future__ import absolute_import - import signal from . import interface diff --git a/buildscripts/resmokelib/testing/hook_test_archival.py b/buildscripts/resmokelib/testing/hook_test_archival.py index b0ef3725eb2..dd3ec6fbf9b 100644 --- a/buildscripts/resmokelib/testing/hook_test_archival.py +++ b/buildscripts/resmokelib/testing/hook_test_archival.py @@ -1,7 +1,5 @@ """Enable support for archiving tests or hooks.""" -from __future__ import absolute_import - import os import threading diff --git a/buildscripts/resmokelib/testing/hooks/__init__.py b/buildscripts/resmokelib/testing/hooks/__init__.py index 82772aa25da..e24cf9eae77 100644 --- a/buildscripts/resmokelib/testing/hooks/__init__.py +++ b/buildscripts/resmokelib/testing/hooks/__init__.py @@ -5,8 +5,6 @@ by allowing special code to be executed before or after each test, and before or after each suite. """ -from __future__ import absolute_import - from .interface import make_hook from ...utils import autoloader as _autoloader diff --git a/buildscripts/resmokelib/testing/hooks/cleanup.py b/buildscripts/resmokelib/testing/hooks/cleanup.py index ebbda2f1edb..0abf025f588 100644 --- a/buildscripts/resmokelib/testing/hooks/cleanup.py +++ b/buildscripts/resmokelib/testing/hooks/cleanup.py @@ -1,7 +1,5 @@ """Test hook for cleaning up data files created by the fixture.""" -from __future__ import absolute_import - import os from . import interface @@ -22,8 +20,9 @@ class CleanEveryN(interface.Hook): # Try to isolate what test triggers the leak by restarting the fixture each time. if "detect_leaks=1" in os.getenv("ASAN_OPTIONS", ""): - self.logger.info("ASAN_OPTIONS environment variable set to detect leaks, so restarting" - " the fixture after each test instead of after every %d.", n) + self.logger.info( + "ASAN_OPTIONS environment variable set to detect leaks, so restarting" + " the fixture after each test instead of after every %d.", n) n = 1 self.n = n # pylint: disable=invalid-name diff --git a/buildscripts/resmokelib/testing/hooks/cleanup_concurrency_workloads.py b/buildscripts/resmokelib/testing/hooks/cleanup_concurrency_workloads.py index 4719b806cf2..9f5e0957514 100644 --- a/buildscripts/resmokelib/testing/hooks/cleanup_concurrency_workloads.py +++ b/buildscripts/resmokelib/testing/hooks/cleanup_concurrency_workloads.py @@ -1,7 +1,5 @@ """Test hook for dropping databases created by the fixture.""" -from __future__ import absolute_import - import copy from buildscripts.resmokelib import utils diff --git a/buildscripts/resmokelib/testing/hooks/collect_embedded_resources.py b/buildscripts/resmokelib/testing/hooks/collect_embedded_resources.py index f6ddffc3963..f39399d6dc8 100644 --- a/buildscripts/resmokelib/testing/hooks/collect_embedded_resources.py +++ b/buildscripts/resmokelib/testing/hooks/collect_embedded_resources.py @@ -1,7 +1,5 @@ """Module for generating and collecting embedded resource results.""" -from __future__ import absolute_import - import os from buildscripts.mobile import adb_monitor diff --git a/buildscripts/resmokelib/testing/hooks/combine_benchmark_results.py b/buildscripts/resmokelib/testing/hooks/combine_benchmark_results.py index 77f0263b2cc..ec4b6d53235 100644 --- a/buildscripts/resmokelib/testing/hooks/combine_benchmark_results.py +++ b/buildscripts/resmokelib/testing/hooks/combine_benchmark_results.py @@ -1,8 +1,5 @@ """Module for generating the test results file fed into the perf plugin.""" -from __future__ import absolute_import -from __future__ import division - import collections import datetime import json @@ -71,7 +68,7 @@ class CombineBenchmarkResults(interface.Hook): "results": [] } - for name, report in self.benchmark_reports.items(): + for name, report in list(self.benchmark_reports.items()): test_report = { "name": name, "context": report.context._asdict(), "results": report.generate_perf_plugin_dict() @@ -168,7 +165,7 @@ class _BenchmarkThreadsReport(object): """ res = {} - for thread_count, reports in self.thread_benchmark_map.items(): + for thread_count, reports in list(self.thread_benchmark_map.items()): thread_report = { "error_values": [0 for _ in range(len(reports))], "ops_per_sec_values": [] # This is actually storing latency per op, not ops/s diff --git a/buildscripts/resmokelib/testing/hooks/combine_benchrun_embedded_results.py b/buildscripts/resmokelib/testing/hooks/combine_benchrun_embedded_results.py index b70e57d8741..62f442dba32 100644 --- a/buildscripts/resmokelib/testing/hooks/combine_benchrun_embedded_results.py +++ b/buildscripts/resmokelib/testing/hooks/combine_benchrun_embedded_results.py @@ -1,8 +1,5 @@ """Module for generating the test results file fed into the perf plugin.""" -from __future__ import absolute_import -from __future__ import division - import collections import datetime import glob @@ -62,7 +59,7 @@ class CombineBenchrunEmbeddedResults(cbr.CombineBenchmarkResults): "results": [] } - for name, report in self.benchmark_reports.items(): + for name, report in list(self.benchmark_reports.items()): test_report = {"name": name, "results": report.generate_perf_plugin_dict()} perf_report["results"].append(test_report) @@ -143,7 +140,7 @@ class _BenchrunEmbeddedThreadsReport(object): """ res = {} - for thread_count, reports in self.thread_benchmark_map.items(): + for thread_count, reports in list(self.thread_benchmark_map.items()): thread_report = {"error_values": [], "ops_per_sec_values": []} for report in reports: diff --git a/buildscripts/resmokelib/testing/hooks/dbhash.py b/buildscripts/resmokelib/testing/hooks/dbhash.py index 40caa5149c6..557c32bb29d 100644 --- a/buildscripts/resmokelib/testing/hooks/dbhash.py +++ b/buildscripts/resmokelib/testing/hooks/dbhash.py @@ -1,7 +1,5 @@ """Test hook for verifying data consistency across a replica set.""" -from __future__ import absolute_import - import os.path from . import jsfile diff --git a/buildscripts/resmokelib/testing/hooks/dbhash_background.py b/buildscripts/resmokelib/testing/hooks/dbhash_background.py index b8122f458b8..9cea6d6957e 100644 --- a/buildscripts/resmokelib/testing/hooks/dbhash_background.py +++ b/buildscripts/resmokelib/testing/hooks/dbhash_background.py @@ -4,8 +4,6 @@ Unlike dbhash.py, this version of the hook runs continously in a background thre running. """ -from __future__ import absolute_import - import os.path import sys import threading @@ -32,13 +30,14 @@ class CheckReplDBHashInBackground(jsfile.JSHook): client = self.fixture.mongo_client() server_status = client.admin.command("serverStatus") if not server_status["storageEngine"].get("supportsSnapshotReadConcern", False): - self.logger.info("Not enabling the background thread because '%s' storage engine" - " doesn't support snapshot reads.", - server_status["storageEngine"]["name"]) + self.logger.info( + "Not enabling the background thread because '%s' storage engine" + " doesn't support snapshot reads.", server_status["storageEngine"]["name"]) return if not server_status["storageEngine"].get("persistent", False): - self.logger.info("Not enabling the background thread because '%s' storage engine" - " is not persistent.", server_status["storageEngine"]["name"]) + self.logger.info( + "Not enabling the background thread because '%s' storage engine" + " is not persistent.", server_status["storageEngine"]["name"]) return self._background_job = _BackgroundJob() diff --git a/buildscripts/resmokelib/testing/hooks/drop_sharded_collections.py b/buildscripts/resmokelib/testing/hooks/drop_sharded_collections.py index 90752571909..9a5f15715b1 100644 --- a/buildscripts/resmokelib/testing/hooks/drop_sharded_collections.py +++ b/buildscripts/resmokelib/testing/hooks/drop_sharded_collections.py @@ -1,7 +1,5 @@ """Hook for cleaning up sharded collections created during tests.""" -from __future__ import absolute_import - import os.path from . import jsfile diff --git a/buildscripts/resmokelib/testing/hooks/initialsync.py b/buildscripts/resmokelib/testing/hooks/initialsync.py index 36ada61ab00..02156068859 100644 --- a/buildscripts/resmokelib/testing/hooks/initialsync.py +++ b/buildscripts/resmokelib/testing/hooks/initialsync.py @@ -1,11 +1,10 @@ """Test hook for verifying correctness of initial sync.""" -from __future__ import absolute_import - import os.path import random import bson +import bson.errors import pymongo.errors from . import cleanup @@ -73,28 +72,38 @@ class BackgroundInitialSyncTestCase(jsfile.DynamicJSTestCase): # If it's been 'n' tests so far, wait for the initial sync node to finish syncing. if self._hook.tests_run >= self._hook.n: - self.logger.info("%d tests have been run against the fixture, waiting for initial sync" - " node to go into SECONDARY state", self._hook.tests_run) + self.logger.info( + "%d tests have been run against the fixture, waiting for initial sync" + " node to go into SECONDARY state", self._hook.tests_run) self._hook.tests_run = 0 - cmd = bson.SON([("replSetTest", 1), ("waitForMemberState", 2), ("timeoutMillis", - 20 * 60 * 1000)]) + cmd = bson.SON([("replSetTest", 1), ("waitForMemberState", 2), + ("timeoutMillis", 20 * 60 * 1000)]) sync_node_conn.admin.command(cmd) # Check if the initial sync node is in SECONDARY state. If it's been 'n' tests, then it # should have waited to be in SECONDARY state and the test should be marked as a failure. # Otherwise, we just skip the hook and will check again after the next test. try: - state = sync_node_conn.admin.command("replSetGetStatus").get("myState") + while True: + # TODO SERVER-40078: The server is reporting invalid + # dates in its response to the replSetGetStatus + # command + try: + state = sync_node_conn.admin.command("replSetGetStatus").get("myState") + break + except bson.errors.InvalidBSON: + continue + if state != 2: if self._hook.tests_run == 0: msg = "Initial sync node did not catch up after waiting 20 minutes" self.logger.exception("{0} failed: {1}".format(self._hook.description, msg)) raise errors.TestFailure(msg) - self.logger.info("Initial sync node is in state %d, not state SECONDARY (2)." - " Skipping BackgroundInitialSync hook for %s", state, - self._base_test_name) + self.logger.info( + "Initial sync node is in state %d, not state SECONDARY (2)." + " Skipping BackgroundInitialSync hook for %s", state, self._base_test_name) # If we have not restarted initial sync since the last time we ran the data # validation, restart initial sync with a 20% probability. @@ -197,8 +206,8 @@ class IntermediateInitialSyncTestCase(jsfile.DynamicJSTestCase): # Do initial sync round. self.logger.info("Waiting for initial sync node to go into SECONDARY state") - cmd = bson.SON([("replSetTest", 1), ("waitForMemberState", 2), ("timeoutMillis", - 20 * 60 * 1000)]) + cmd = bson.SON([("replSetTest", 1), ("waitForMemberState", 2), + ("timeoutMillis", 20 * 60 * 1000)]) sync_node_conn.admin.command(cmd) # Run data validation and dbhash checking. diff --git a/buildscripts/resmokelib/testing/hooks/interface.py b/buildscripts/resmokelib/testing/hooks/interface.py index fe53f820fc5..d8ac37e159d 100644 --- a/buildscripts/resmokelib/testing/hooks/interface.py +++ b/buildscripts/resmokelib/testing/hooks/interface.py @@ -1,7 +1,5 @@ """Interface for customizing the behavior of a test fixture.""" -from __future__ import absolute_import - import sys from ..testcases import interface as testcase @@ -21,11 +19,9 @@ def make_hook(class_name, *args, **kwargs): return _HOOKS[class_name](*args, **kwargs) -class Hook(object): +class Hook(object, metaclass=registry.make_registry_metaclass(_HOOKS)): """Common interface all Hooks will inherit from.""" - __metaclass__ = registry.make_registry_metaclass(_HOOKS) # type: ignore - REGISTERED_NAME = registry.LEAVE_UNREGISTERED def __init__(self, hook_logger, fixture, description): diff --git a/buildscripts/resmokelib/testing/hooks/jsfile.py b/buildscripts/resmokelib/testing/hooks/jsfile.py index e95d3d6d780..6eedb3f87c4 100644 --- a/buildscripts/resmokelib/testing/hooks/jsfile.py +++ b/buildscripts/resmokelib/testing/hooks/jsfile.py @@ -1,7 +1,5 @@ """Interface for customizing the behavior of a test fixture by executing a JavaScript file.""" -from __future__ import absolute_import - from . import interface from ..testcases import jstest from ... import errors diff --git a/buildscripts/resmokelib/testing/hooks/oplog.py b/buildscripts/resmokelib/testing/hooks/oplog.py index ceb81bb8fd6..6822eecabf0 100644 --- a/buildscripts/resmokelib/testing/hooks/oplog.py +++ b/buildscripts/resmokelib/testing/hooks/oplog.py @@ -1,7 +1,5 @@ """Test hook for verifying members of a replica set have matching oplogs.""" -from __future__ import absolute_import - import os.path from . import jsfile diff --git a/buildscripts/resmokelib/testing/hooks/periodic_kill_secondaries.py b/buildscripts/resmokelib/testing/hooks/periodic_kill_secondaries.py index a6f44278140..daf18ad7671 100644 --- a/buildscripts/resmokelib/testing/hooks/periodic_kill_secondaries.py +++ b/buildscripts/resmokelib/testing/hooks/periodic_kill_secondaries.py @@ -1,7 +1,5 @@ """Test hook for verifying correctness of secondary's behavior during an unclean shutdown.""" -from __future__ import absolute_import - import time import bson @@ -189,8 +187,9 @@ class PeriodicKillSecondariesTestCase(interface.DynamicTestCase): for secondary in self.fixture.get_secondaries(): self._check_invariants_as_standalone(secondary) - self.logger.info("Restarting the secondary on port %d as a replica set node with" - " its data files intact...", secondary.port) + self.logger.info( + "Restarting the secondary on port %d as a replica set node with" + " its data files intact...", secondary.port) # Start the 'secondary' mongod back up as part of the replica set and wait for it to # reach state SECONDARY. secondary.setup() @@ -253,12 +252,13 @@ class PeriodicKillSecondariesTestCase(interface.DynamicTestCase): self.fixture.setup() self.fixture.await_ready() - def _check_invariants_as_standalone(self, secondary): + def _check_invariants_as_standalone(self, secondary): # pylint: disable=too-many-locals # pylint: disable=too-many-branches,too-many-statements # We remove the --replSet option in order to start the node as a standalone. replset_name = secondary.mongod_options.pop("replSet") - self.logger.info("Restarting the secondary on port %d as a standalone node with" - " its data files intact...", secondary.port) + self.logger.info( + "Restarting the secondary on port %d as a standalone node with" + " its data files intact...", secondary.port) try: secondary.setup() diff --git a/buildscripts/resmokelib/testing/hooks/stepdown.py b/buildscripts/resmokelib/testing/hooks/stepdown.py index fbda653a324..f218fffe49e 100644 --- a/buildscripts/resmokelib/testing/hooks/stepdown.py +++ b/buildscripts/resmokelib/testing/hooks/stepdown.py @@ -1,5 +1,4 @@ """Test hook that periodically makes the primary of a replica set step down.""" -from __future__ import absolute_import import collections import os.path @@ -179,7 +178,7 @@ class _StepdownThread(threading.Thread): # pylint: disable=too-many-instance-at try: while True: - if self._is_stopped(): + if self.__is_stopped(): break self._wait_for_permission_or_resume() now = time.time() @@ -210,7 +209,7 @@ class _StepdownThread(threading.Thread): # pylint: disable=too-many-instance-at self.resume() self.join() - def _is_stopped(self): + def __is_stopped(self): return self._is_stopped_evt.is_set() def pause(self): @@ -234,7 +233,7 @@ class _StepdownThread(threading.Thread): # pylint: disable=too-many-instance-at def _wait_for_permission_or_resume(self): # Wait until stop, _stepdown_permitted_file or resume. if self._stepdown_permitted_file: - while not os.path.isfile(self._stepdown_permitted_file) and not self._is_stopped(): + while not os.path.isfile(self._stepdown_permitted_file) and not self.__is_stopped(): # Set a short sleep during busy wait time for self._stepdown_permitted_file. self._wait(0.1) else: diff --git a/buildscripts/resmokelib/testing/hooks/validate.py b/buildscripts/resmokelib/testing/hooks/validate.py index 3239ddbdc06..81bd42ba6d9 100644 --- a/buildscripts/resmokelib/testing/hooks/validate.py +++ b/buildscripts/resmokelib/testing/hooks/validate.py @@ -1,7 +1,5 @@ """Test hook for verifying the consistency and integrity of collection and index data.""" -from __future__ import absolute_import - import os.path from . import jsfile diff --git a/buildscripts/resmokelib/testing/hooks/wait_for_replication.py b/buildscripts/resmokelib/testing/hooks/wait_for_replication.py index fbd786a170c..8720f9456e6 100644 --- a/buildscripts/resmokelib/testing/hooks/wait_for_replication.py +++ b/buildscripts/resmokelib/testing/hooks/wait_for_replication.py @@ -1,7 +1,5 @@ """Test hook to wait for replication to complete on a replica set.""" -from __future__ import absolute_import - import time from buildscripts.resmokelib import core diff --git a/buildscripts/resmokelib/testing/job.py b/buildscripts/resmokelib/testing/job.py index 908ae85832a..9b5023d0656 100644 --- a/buildscripts/resmokelib/testing/job.py +++ b/buildscripts/resmokelib/testing/job.py @@ -1,7 +1,5 @@ """Enable running tests simultaneously by processing them from a multi-consumer queue.""" -from __future__ import absolute_import - import sys import time @@ -175,8 +173,8 @@ class Job(object): # pylint: disable=too-many-instance-attributes test.short_description()) self.report.setFailure(test, return_code=2) # Always fail fast if the fixture fails. - raise errors.StopExecution("%s not running after %s" % (self.fixture, - test.short_description())) + raise errors.StopExecution( + "%s not running after %s" % (self.fixture, test.short_description())) finally: success = self.report.find_test_info(test).status == "pass" if self.archival: diff --git a/buildscripts/resmokelib/testing/report.py b/buildscripts/resmokelib/testing/report.py index fad0d20a0a2..3365a17075a 100644 --- a/buildscripts/resmokelib/testing/report.py +++ b/buildscripts/resmokelib/testing/report.py @@ -3,8 +3,6 @@ This is used to support additional test status and timing information for the report.json file. """ -from __future__ import absolute_import - import copy import threading import time diff --git a/buildscripts/resmokelib/testing/suite.py b/buildscripts/resmokelib/testing/suite.py index e0f2dda5151..b369261e147 100644 --- a/buildscripts/resmokelib/testing/suite.py +++ b/buildscripts/resmokelib/testing/suite.py @@ -1,7 +1,5 @@ """Holder for the (test kind, list of tests) pair with additional metadata their execution.""" -from __future__ import absolute_import - import itertools import threading import time @@ -269,7 +267,7 @@ class Suite(object): # pylint: disable=too-many-instance-attributes sb.append("Executed %d times in %0.2f seconds:" % (num_iterations, total_time_taken)) combined_summary = _summary.Summary(0, 0.0, 0, 0, 0, 0) - for iteration in xrange(num_iterations): + for iteration in range(num_iterations): # Summarize each execution as a bulleted list of results. bulleter_sb = [] summary = self._summarize_report(reports[iteration], start_times[iteration], @@ -341,8 +339,8 @@ class Suite(object): # pylint: disable=too-many-instance-attributes def log_summaries(logger, suites, time_taken): """Log summary of all suites.""" sb = [] - sb.append("Summary of all suites: %d suites ran in %0.2f seconds" % (len(suites), - time_taken)) + sb.append( + "Summary of all suites: %d suites ran in %0.2f seconds" % (len(suites), time_taken)) for suite in suites: suite_sb = [] suite.summarize(suite_sb) diff --git a/buildscripts/resmokelib/testing/summary.py b/buildscripts/resmokelib/testing/summary.py index dc92e0b5b34..a5d439b64b0 100644 --- a/buildscripts/resmokelib/testing/summary.py +++ b/buildscripts/resmokelib/testing/summary.py @@ -1,7 +1,5 @@ """Holder for summary information about a test suite.""" -from __future__ import absolute_import - import collections Summary = collections.namedtuple( @@ -12,6 +10,6 @@ Summary = collections.namedtuple( def combine(summary1, summary2): """Return a summary representing the sum of 'summary1' and 'summary2'.""" args = [] - for i in xrange(len(Summary._fields)): + for i in range(len(Summary._fields)): args.append(summary1[i] + summary2[i]) return Summary._make(args) diff --git a/buildscripts/resmokelib/testing/testcases/__init__.py b/buildscripts/resmokelib/testing/testcases/__init__.py index 52869d99de8..d828d97a0b3 100644 --- a/buildscripts/resmokelib/testing/testcases/__init__.py +++ b/buildscripts/resmokelib/testing/testcases/__init__.py @@ -1,7 +1,5 @@ """Package containing subclasses of unittest.TestCase.""" -from __future__ import absolute_import - from .interface import make_test_case from ...utils import autoloader as _autoloader diff --git a/buildscripts/resmokelib/testing/testcases/benchmark_test.py b/buildscripts/resmokelib/testing/testcases/benchmark_test.py index ea506c4f7e5..e7760799e42 100644 --- a/buildscripts/resmokelib/testing/testcases/benchmark_test.py +++ b/buildscripts/resmokelib/testing/testcases/benchmark_test.py @@ -1,7 +1,5 @@ """The unittest.TestCase for tests using a MongoDB vendored version of Google Benchmark.""" -from __future__ import absolute_import - from buildscripts.resmokelib import config as _config from buildscripts.resmokelib import core from buildscripts.resmokelib import parser @@ -52,7 +50,7 @@ class BenchmarkTestCase(interface.ProcessTestCase): "benchmark_repetitions": _config.BENCHMARK_REPETITIONS } - for key, value in resmoke_bm_options.items(): + for key, value in list(resmoke_bm_options.items()): if value is not None: # 4. sanitize options before passing them to Benchmark's command line. if key == "benchmark_min_time": diff --git a/buildscripts/resmokelib/testing/testcases/benchrun_embedded_test.py b/buildscripts/resmokelib/testing/testcases/benchrun_embedded_test.py index 89b95424cdb..d4666c94074 100644 --- a/buildscripts/resmokelib/testing/testcases/benchrun_embedded_test.py +++ b/buildscripts/resmokelib/testing/testcases/benchrun_embedded_test.py @@ -1,7 +1,5 @@ """The unittest.TestCase for tests using benchrun embedded (mongoebench).""" -from __future__ import absolute_import - import os import posixpath @@ -73,7 +71,7 @@ class BenchrunEmbeddedTestCase( # pylint: disable=too-many-instance-attributes # 3. Override Benchmark options with options set through resmoke's command line. resmoke_benchrun_options = {"dbpath": self.dbpath, "time": _config.BENCHMARK_MIN_TIME} - for key, value in resmoke_benchrun_options.items(): + for key, value in list(resmoke_benchrun_options.items()): if value is not None: # 4. sanitize options before passing them to Benchmark's command line. if key == "time": @@ -104,7 +102,7 @@ class BenchrunEmbeddedTestCase( # pylint: disable=too-many-instance-attributes def run_test(self): """Run the test for specified number of iterations.""" - for iter_num in xrange(self.benchrun_repetitions): + for iter_num in range(self.benchrun_repetitions): # Set the output file for each iteration. local_report_path = self._report_path(iter_num) device_report_path = self._device_report_path(iter_num) @@ -134,8 +132,8 @@ class BenchrunEmbeddedTestCase( # pylint: disable=too-many-instance-attributes def _report_dir(self): """Return the report directory. Reports are stored in <report_root>/<testname>/<thread>.""" - return os.path.join(self.report_root, self.short_name(), "thread{}".format( - self.benchrun_threads)) + return os.path.join(self.report_root, self.short_name(), + "thread{}".format(self.benchrun_threads)) @staticmethod def _report_name(iter_num): diff --git a/buildscripts/resmokelib/testing/testcases/cpp_integration_test.py b/buildscripts/resmokelib/testing/testcases/cpp_integration_test.py index df6d7c9fa41..c0313046a20 100644 --- a/buildscripts/resmokelib/testing/testcases/cpp_integration_test.py +++ b/buildscripts/resmokelib/testing/testcases/cpp_integration_test.py @@ -1,7 +1,5 @@ """The unittest.TestCase for C++ integration tests.""" -from __future__ import absolute_import - from . import interface from ... import core from ... import utils diff --git a/buildscripts/resmokelib/testing/testcases/cpp_unittest.py b/buildscripts/resmokelib/testing/testcases/cpp_unittest.py index b9fb427d0da..c54b45a7792 100644 --- a/buildscripts/resmokelib/testing/testcases/cpp_unittest.py +++ b/buildscripts/resmokelib/testing/testcases/cpp_unittest.py @@ -1,7 +1,5 @@ """The unittest.TestCase for C++ unit tests.""" -from __future__ import absolute_import - from . import interface from ... import core from ... import utils diff --git a/buildscripts/resmokelib/testing/testcases/dbtest.py b/buildscripts/resmokelib/testing/testcases/dbtest.py index 4cfbb8c6385..44ea8410c58 100644 --- a/buildscripts/resmokelib/testing/testcases/dbtest.py +++ b/buildscripts/resmokelib/testing/testcases/dbtest.py @@ -1,7 +1,5 @@ """The unittest.TestCase for dbtests.""" -from __future__ import absolute_import - import os import os.path diff --git a/buildscripts/resmokelib/testing/testcases/fsm_workload_test.py b/buildscripts/resmokelib/testing/testcases/fsm_workload_test.py index 6cbda33abe9..d184fe84fc5 100644 --- a/buildscripts/resmokelib/testing/testcases/fsm_workload_test.py +++ b/buildscripts/resmokelib/testing/testcases/fsm_workload_test.py @@ -1,7 +1,5 @@ """The unittest.TestCase for FSM workloads.""" -from __future__ import absolute_import - import hashlib import threading @@ -97,5 +95,5 @@ class ParallelFSMWorkloadTestCase(FSMWorkloadTestCase): """Get an unique identifier for a workload group.""" uid = hashlib.md5() for workload_name in sorted(selected_tests): - uid.update(workload_name) + uid.update(workload_name.encode("utf-8")) return uid.hexdigest() diff --git a/buildscripts/resmokelib/testing/testcases/gennylib_test.py b/buildscripts/resmokelib/testing/testcases/gennylib_test.py index 91200b6c6c1..96eb08c6cb5 100644 --- a/buildscripts/resmokelib/testing/testcases/gennylib_test.py +++ b/buildscripts/resmokelib/testing/testcases/gennylib_test.py @@ -1,7 +1,5 @@ """The unittest.TestCase for gennylib integration tests.""" -from __future__ import absolute_import - from . import interface from ... import core from ... import utils diff --git a/buildscripts/resmokelib/testing/testcases/gennytest.py b/buildscripts/resmokelib/testing/testcases/gennytest.py index 1db9ca48e94..11e51014803 100644 --- a/buildscripts/resmokelib/testing/testcases/gennytest.py +++ b/buildscripts/resmokelib/testing/testcases/gennytest.py @@ -1,7 +1,5 @@ """The unittest.TestCase for genny.""" -from __future__ import absolute_import - import os import os.path diff --git a/buildscripts/resmokelib/testing/testcases/interface.py b/buildscripts/resmokelib/testing/testcases/interface.py index 1f078828b1c..e1c1e2faa0c 100644 --- a/buildscripts/resmokelib/testing/testcases/interface.py +++ b/buildscripts/resmokelib/testing/testcases/interface.py @@ -3,8 +3,6 @@ This is used to perform the actual test case. """ -from __future__ import absolute_import - import os import os.path import unittest @@ -23,11 +21,9 @@ def make_test_case(test_kind, *args, **kwargs): return _TEST_CASES[test_kind](*args, **kwargs) -class TestCase(unittest.TestCase): # pylint: disable=too-many-instance-attributes +class TestCase(unittest.TestCase, metaclass=registry.make_registry_metaclass(_TEST_CASES)): # pylint: disable=too-many-instance-attributes """A test case to execute.""" - __metaclass__ = registry.make_registry_metaclass(_TEST_CASES) # type: ignore - REGISTERED_NAME = registry.LEAVE_UNREGISTERED def __init__(self, logger, test_kind, test_name, dynamic=False): @@ -37,10 +33,10 @@ class TestCase(unittest.TestCase): # pylint: disable=too-many-instance-attribut if not isinstance(logger, logging.Logger): raise TypeError("logger must be a Logger instance") - if not isinstance(test_kind, basestring): + if not isinstance(test_kind, str): raise TypeError("test_kind must be a string") - if not isinstance(test_name, basestring): + if not isinstance(test_name, str): raise TypeError("test_name must be a string") self._id = uuid.uuid4() diff --git a/buildscripts/resmokelib/testing/testcases/json_schema_test.py b/buildscripts/resmokelib/testing/testcases/json_schema_test.py index 08e5a2d71a8..6f40b8af714 100644 --- a/buildscripts/resmokelib/testing/testcases/json_schema_test.py +++ b/buildscripts/resmokelib/testing/testcases/json_schema_test.py @@ -1,7 +1,5 @@ """The unittest.TestCase for JSON Schema tests.""" -from __future__ import absolute_import - from buildscripts.resmokelib import config from buildscripts.resmokelib import core from buildscripts.resmokelib import utils diff --git a/buildscripts/resmokelib/testing/testcases/jsrunnerfile.py b/buildscripts/resmokelib/testing/testcases/jsrunnerfile.py index c2da41faf37..ece5e2ceb15 100644 --- a/buildscripts/resmokelib/testing/testcases/jsrunnerfile.py +++ b/buildscripts/resmokelib/testing/testcases/jsrunnerfile.py @@ -1,7 +1,5 @@ """The unittest.TestCase for tests with a static JavaScript runner file.""" -from __future__ import absolute_import - from buildscripts.resmokelib import config from buildscripts.resmokelib import core from buildscripts.resmokelib import utils diff --git a/buildscripts/resmokelib/testing/testcases/jstest.py b/buildscripts/resmokelib/testing/testcases/jstest.py index 4d36ac965b0..6b6a5ee45e2 100644 --- a/buildscripts/resmokelib/testing/testcases/jstest.py +++ b/buildscripts/resmokelib/testing/testcases/jstest.py @@ -1,7 +1,5 @@ """The unittest.TestCase for JavaScript tests.""" -from __future__ import absolute_import - import os import os.path import sys @@ -201,7 +199,7 @@ class JSTestCase(interface.ProcessTestCase): test_cases = [] try: # If there are multiple clients, make a new thread for each client. - for thread_id in xrange(self.num_clients): + for thread_id in range(self.num_clients): logger = self.logger.new_test_thread_logger(self.test_kind, str(thread_id)) test_case = self._create_test_case_for_thread(logger, thread_id) test_cases.append(test_case) diff --git a/buildscripts/resmokelib/testing/testcases/mongos_test.py b/buildscripts/resmokelib/testing/testcases/mongos_test.py index 9914ba8677f..f00a93a9bdb 100644 --- a/buildscripts/resmokelib/testing/testcases/mongos_test.py +++ b/buildscripts/resmokelib/testing/testcases/mongos_test.py @@ -1,7 +1,5 @@ """The unittest.TestCase for mongos --test.""" -from __future__ import absolute_import - from . import interface from ... import config from ... import core diff --git a/buildscripts/resmokelib/testing/testcases/mql_model_haskell_test.py b/buildscripts/resmokelib/testing/testcases/mql_model_haskell_test.py index 7911aa1b7d6..fff521ddae5 100644 --- a/buildscripts/resmokelib/testing/testcases/mql_model_haskell_test.py +++ b/buildscripts/resmokelib/testing/testcases/mql_model_haskell_test.py @@ -1,7 +1,5 @@ """The unittest.TestCase for MQL Haskell tests.""" -from __future__ import absolute_import - import os import os.path diff --git a/buildscripts/resmokelib/testing/testcases/mql_model_mongod_test.py b/buildscripts/resmokelib/testing/testcases/mql_model_mongod_test.py index eddfef2b7b3..249b0f18fb2 100644 --- a/buildscripts/resmokelib/testing/testcases/mql_model_mongod_test.py +++ b/buildscripts/resmokelib/testing/testcases/mql_model_mongod_test.py @@ -1,7 +1,5 @@ """The unittest.TestCase for MQL MongoD Model tests.""" -from __future__ import absolute_import - import os import os.path diff --git a/buildscripts/resmokelib/testing/testcases/multi_stmt_txn_test.py b/buildscripts/resmokelib/testing/testcases/multi_stmt_txn_test.py index 1e790612153..81ddfa289ae 100644 --- a/buildscripts/resmokelib/testing/testcases/multi_stmt_txn_test.py +++ b/buildscripts/resmokelib/testing/testcases/multi_stmt_txn_test.py @@ -1,7 +1,5 @@ """unittest.TestCase for multi-statement transaction passthrough tests.""" -from __future__ import absolute_import - from buildscripts.resmokelib import config from buildscripts.resmokelib import core from buildscripts.resmokelib import utils diff --git a/buildscripts/resmokelib/testing/testcases/sleeptest.py b/buildscripts/resmokelib/testing/testcases/sleeptest.py index 163100095d3..4f073ee6f25 100644 --- a/buildscripts/resmokelib/testing/testcases/sleeptest.py +++ b/buildscripts/resmokelib/testing/testcases/sleeptest.py @@ -1,7 +1,5 @@ """The unittest.TestCase for sleeping a given amount of time.""" -from __future__ import absolute_import - import time from . import interface |