diff options
Diffstat (limited to 'buildscripts/resmokelib/testing/hooks')
17 files changed, 45 insertions, 67 deletions
diff --git a/buildscripts/resmokelib/testing/hooks/__init__.py b/buildscripts/resmokelib/testing/hooks/__init__.py index 82772aa25da..e24cf9eae77 100644 --- a/buildscripts/resmokelib/testing/hooks/__init__.py +++ b/buildscripts/resmokelib/testing/hooks/__init__.py @@ -5,8 +5,6 @@ by allowing special code to be executed before or after each test, and before or after each suite. """ -from __future__ import absolute_import - from .interface import make_hook from ...utils import autoloader as _autoloader diff --git a/buildscripts/resmokelib/testing/hooks/cleanup.py b/buildscripts/resmokelib/testing/hooks/cleanup.py index ebbda2f1edb..0abf025f588 100644 --- a/buildscripts/resmokelib/testing/hooks/cleanup.py +++ b/buildscripts/resmokelib/testing/hooks/cleanup.py @@ -1,7 +1,5 @@ """Test hook for cleaning up data files created by the fixture.""" -from __future__ import absolute_import - import os from . import interface @@ -22,8 +20,9 @@ class CleanEveryN(interface.Hook): # Try to isolate what test triggers the leak by restarting the fixture each time. if "detect_leaks=1" in os.getenv("ASAN_OPTIONS", ""): - self.logger.info("ASAN_OPTIONS environment variable set to detect leaks, so restarting" - " the fixture after each test instead of after every %d.", n) + self.logger.info( + "ASAN_OPTIONS environment variable set to detect leaks, so restarting" + " the fixture after each test instead of after every %d.", n) n = 1 self.n = n # pylint: disable=invalid-name diff --git a/buildscripts/resmokelib/testing/hooks/cleanup_concurrency_workloads.py b/buildscripts/resmokelib/testing/hooks/cleanup_concurrency_workloads.py index 4719b806cf2..9f5e0957514 100644 --- a/buildscripts/resmokelib/testing/hooks/cleanup_concurrency_workloads.py +++ b/buildscripts/resmokelib/testing/hooks/cleanup_concurrency_workloads.py @@ -1,7 +1,5 @@ """Test hook for dropping databases created by the fixture.""" -from __future__ import absolute_import - import copy from buildscripts.resmokelib import utils diff --git a/buildscripts/resmokelib/testing/hooks/collect_embedded_resources.py b/buildscripts/resmokelib/testing/hooks/collect_embedded_resources.py index f6ddffc3963..f39399d6dc8 100644 --- a/buildscripts/resmokelib/testing/hooks/collect_embedded_resources.py +++ b/buildscripts/resmokelib/testing/hooks/collect_embedded_resources.py @@ -1,7 +1,5 @@ """Module for generating and collecting embedded resource results.""" -from __future__ import absolute_import - import os from buildscripts.mobile import adb_monitor diff --git a/buildscripts/resmokelib/testing/hooks/combine_benchmark_results.py b/buildscripts/resmokelib/testing/hooks/combine_benchmark_results.py index 77f0263b2cc..ec4b6d53235 100644 --- a/buildscripts/resmokelib/testing/hooks/combine_benchmark_results.py +++ b/buildscripts/resmokelib/testing/hooks/combine_benchmark_results.py @@ -1,8 +1,5 @@ """Module for generating the test results file fed into the perf plugin.""" -from __future__ import absolute_import -from __future__ import division - import collections import datetime import json @@ -71,7 +68,7 @@ class CombineBenchmarkResults(interface.Hook): "results": [] } - for name, report in self.benchmark_reports.items(): + for name, report in list(self.benchmark_reports.items()): test_report = { "name": name, "context": report.context._asdict(), "results": report.generate_perf_plugin_dict() @@ -168,7 +165,7 @@ class _BenchmarkThreadsReport(object): """ res = {} - for thread_count, reports in self.thread_benchmark_map.items(): + for thread_count, reports in list(self.thread_benchmark_map.items()): thread_report = { "error_values": [0 for _ in range(len(reports))], "ops_per_sec_values": [] # This is actually storing latency per op, not ops/s diff --git a/buildscripts/resmokelib/testing/hooks/combine_benchrun_embedded_results.py b/buildscripts/resmokelib/testing/hooks/combine_benchrun_embedded_results.py index b70e57d8741..62f442dba32 100644 --- a/buildscripts/resmokelib/testing/hooks/combine_benchrun_embedded_results.py +++ b/buildscripts/resmokelib/testing/hooks/combine_benchrun_embedded_results.py @@ -1,8 +1,5 @@ """Module for generating the test results file fed into the perf plugin.""" -from __future__ import absolute_import -from __future__ import division - import collections import datetime import glob @@ -62,7 +59,7 @@ class CombineBenchrunEmbeddedResults(cbr.CombineBenchmarkResults): "results": [] } - for name, report in self.benchmark_reports.items(): + for name, report in list(self.benchmark_reports.items()): test_report = {"name": name, "results": report.generate_perf_plugin_dict()} perf_report["results"].append(test_report) @@ -143,7 +140,7 @@ class _BenchrunEmbeddedThreadsReport(object): """ res = {} - for thread_count, reports in self.thread_benchmark_map.items(): + for thread_count, reports in list(self.thread_benchmark_map.items()): thread_report = {"error_values": [], "ops_per_sec_values": []} for report in reports: diff --git a/buildscripts/resmokelib/testing/hooks/dbhash.py b/buildscripts/resmokelib/testing/hooks/dbhash.py index 40caa5149c6..557c32bb29d 100644 --- a/buildscripts/resmokelib/testing/hooks/dbhash.py +++ b/buildscripts/resmokelib/testing/hooks/dbhash.py @@ -1,7 +1,5 @@ """Test hook for verifying data consistency across a replica set.""" -from __future__ import absolute_import - import os.path from . import jsfile diff --git a/buildscripts/resmokelib/testing/hooks/dbhash_background.py b/buildscripts/resmokelib/testing/hooks/dbhash_background.py index b8122f458b8..9cea6d6957e 100644 --- a/buildscripts/resmokelib/testing/hooks/dbhash_background.py +++ b/buildscripts/resmokelib/testing/hooks/dbhash_background.py @@ -4,8 +4,6 @@ Unlike dbhash.py, this version of the hook runs continously in a background thre running. """ -from __future__ import absolute_import - import os.path import sys import threading @@ -32,13 +30,14 @@ class CheckReplDBHashInBackground(jsfile.JSHook): client = self.fixture.mongo_client() server_status = client.admin.command("serverStatus") if not server_status["storageEngine"].get("supportsSnapshotReadConcern", False): - self.logger.info("Not enabling the background thread because '%s' storage engine" - " doesn't support snapshot reads.", - server_status["storageEngine"]["name"]) + self.logger.info( + "Not enabling the background thread because '%s' storage engine" + " doesn't support snapshot reads.", server_status["storageEngine"]["name"]) return if not server_status["storageEngine"].get("persistent", False): - self.logger.info("Not enabling the background thread because '%s' storage engine" - " is not persistent.", server_status["storageEngine"]["name"]) + self.logger.info( + "Not enabling the background thread because '%s' storage engine" + " is not persistent.", server_status["storageEngine"]["name"]) return self._background_job = _BackgroundJob() diff --git a/buildscripts/resmokelib/testing/hooks/drop_sharded_collections.py b/buildscripts/resmokelib/testing/hooks/drop_sharded_collections.py index 90752571909..9a5f15715b1 100644 --- a/buildscripts/resmokelib/testing/hooks/drop_sharded_collections.py +++ b/buildscripts/resmokelib/testing/hooks/drop_sharded_collections.py @@ -1,7 +1,5 @@ """Hook for cleaning up sharded collections created during tests.""" -from __future__ import absolute_import - import os.path from . import jsfile diff --git a/buildscripts/resmokelib/testing/hooks/initialsync.py b/buildscripts/resmokelib/testing/hooks/initialsync.py index 36ada61ab00..02156068859 100644 --- a/buildscripts/resmokelib/testing/hooks/initialsync.py +++ b/buildscripts/resmokelib/testing/hooks/initialsync.py @@ -1,11 +1,10 @@ """Test hook for verifying correctness of initial sync.""" -from __future__ import absolute_import - import os.path import random import bson +import bson.errors import pymongo.errors from . import cleanup @@ -73,28 +72,38 @@ class BackgroundInitialSyncTestCase(jsfile.DynamicJSTestCase): # If it's been 'n' tests so far, wait for the initial sync node to finish syncing. if self._hook.tests_run >= self._hook.n: - self.logger.info("%d tests have been run against the fixture, waiting for initial sync" - " node to go into SECONDARY state", self._hook.tests_run) + self.logger.info( + "%d tests have been run against the fixture, waiting for initial sync" + " node to go into SECONDARY state", self._hook.tests_run) self._hook.tests_run = 0 - cmd = bson.SON([("replSetTest", 1), ("waitForMemberState", 2), ("timeoutMillis", - 20 * 60 * 1000)]) + cmd = bson.SON([("replSetTest", 1), ("waitForMemberState", 2), + ("timeoutMillis", 20 * 60 * 1000)]) sync_node_conn.admin.command(cmd) # Check if the initial sync node is in SECONDARY state. If it's been 'n' tests, then it # should have waited to be in SECONDARY state and the test should be marked as a failure. # Otherwise, we just skip the hook and will check again after the next test. try: - state = sync_node_conn.admin.command("replSetGetStatus").get("myState") + while True: + # TODO SERVER-40078: The server is reporting invalid + # dates in its response to the replSetGetStatus + # command + try: + state = sync_node_conn.admin.command("replSetGetStatus").get("myState") + break + except bson.errors.InvalidBSON: + continue + if state != 2: if self._hook.tests_run == 0: msg = "Initial sync node did not catch up after waiting 20 minutes" self.logger.exception("{0} failed: {1}".format(self._hook.description, msg)) raise errors.TestFailure(msg) - self.logger.info("Initial sync node is in state %d, not state SECONDARY (2)." - " Skipping BackgroundInitialSync hook for %s", state, - self._base_test_name) + self.logger.info( + "Initial sync node is in state %d, not state SECONDARY (2)." + " Skipping BackgroundInitialSync hook for %s", state, self._base_test_name) # If we have not restarted initial sync since the last time we ran the data # validation, restart initial sync with a 20% probability. @@ -197,8 +206,8 @@ class IntermediateInitialSyncTestCase(jsfile.DynamicJSTestCase): # Do initial sync round. self.logger.info("Waiting for initial sync node to go into SECONDARY state") - cmd = bson.SON([("replSetTest", 1), ("waitForMemberState", 2), ("timeoutMillis", - 20 * 60 * 1000)]) + cmd = bson.SON([("replSetTest", 1), ("waitForMemberState", 2), + ("timeoutMillis", 20 * 60 * 1000)]) sync_node_conn.admin.command(cmd) # Run data validation and dbhash checking. diff --git a/buildscripts/resmokelib/testing/hooks/interface.py b/buildscripts/resmokelib/testing/hooks/interface.py index fe53f820fc5..d8ac37e159d 100644 --- a/buildscripts/resmokelib/testing/hooks/interface.py +++ b/buildscripts/resmokelib/testing/hooks/interface.py @@ -1,7 +1,5 @@ """Interface for customizing the behavior of a test fixture.""" -from __future__ import absolute_import - import sys from ..testcases import interface as testcase @@ -21,11 +19,9 @@ def make_hook(class_name, *args, **kwargs): return _HOOKS[class_name](*args, **kwargs) -class Hook(object): +class Hook(object, metaclass=registry.make_registry_metaclass(_HOOKS)): """Common interface all Hooks will inherit from.""" - __metaclass__ = registry.make_registry_metaclass(_HOOKS) # type: ignore - REGISTERED_NAME = registry.LEAVE_UNREGISTERED def __init__(self, hook_logger, fixture, description): diff --git a/buildscripts/resmokelib/testing/hooks/jsfile.py b/buildscripts/resmokelib/testing/hooks/jsfile.py index e95d3d6d780..6eedb3f87c4 100644 --- a/buildscripts/resmokelib/testing/hooks/jsfile.py +++ b/buildscripts/resmokelib/testing/hooks/jsfile.py @@ -1,7 +1,5 @@ """Interface for customizing the behavior of a test fixture by executing a JavaScript file.""" -from __future__ import absolute_import - from . import interface from ..testcases import jstest from ... import errors diff --git a/buildscripts/resmokelib/testing/hooks/oplog.py b/buildscripts/resmokelib/testing/hooks/oplog.py index ceb81bb8fd6..6822eecabf0 100644 --- a/buildscripts/resmokelib/testing/hooks/oplog.py +++ b/buildscripts/resmokelib/testing/hooks/oplog.py @@ -1,7 +1,5 @@ """Test hook for verifying members of a replica set have matching oplogs.""" -from __future__ import absolute_import - import os.path from . import jsfile diff --git a/buildscripts/resmokelib/testing/hooks/periodic_kill_secondaries.py b/buildscripts/resmokelib/testing/hooks/periodic_kill_secondaries.py index a6f44278140..daf18ad7671 100644 --- a/buildscripts/resmokelib/testing/hooks/periodic_kill_secondaries.py +++ b/buildscripts/resmokelib/testing/hooks/periodic_kill_secondaries.py @@ -1,7 +1,5 @@ """Test hook for verifying correctness of secondary's behavior during an unclean shutdown.""" -from __future__ import absolute_import - import time import bson @@ -189,8 +187,9 @@ class PeriodicKillSecondariesTestCase(interface.DynamicTestCase): for secondary in self.fixture.get_secondaries(): self._check_invariants_as_standalone(secondary) - self.logger.info("Restarting the secondary on port %d as a replica set node with" - " its data files intact...", secondary.port) + self.logger.info( + "Restarting the secondary on port %d as a replica set node with" + " its data files intact...", secondary.port) # Start the 'secondary' mongod back up as part of the replica set and wait for it to # reach state SECONDARY. secondary.setup() @@ -253,12 +252,13 @@ class PeriodicKillSecondariesTestCase(interface.DynamicTestCase): self.fixture.setup() self.fixture.await_ready() - def _check_invariants_as_standalone(self, secondary): + def _check_invariants_as_standalone(self, secondary): # pylint: disable=too-many-locals # pylint: disable=too-many-branches,too-many-statements # We remove the --replSet option in order to start the node as a standalone. replset_name = secondary.mongod_options.pop("replSet") - self.logger.info("Restarting the secondary on port %d as a standalone node with" - " its data files intact...", secondary.port) + self.logger.info( + "Restarting the secondary on port %d as a standalone node with" + " its data files intact...", secondary.port) try: secondary.setup() diff --git a/buildscripts/resmokelib/testing/hooks/stepdown.py b/buildscripts/resmokelib/testing/hooks/stepdown.py index fbda653a324..f218fffe49e 100644 --- a/buildscripts/resmokelib/testing/hooks/stepdown.py +++ b/buildscripts/resmokelib/testing/hooks/stepdown.py @@ -1,5 +1,4 @@ """Test hook that periodically makes the primary of a replica set step down.""" -from __future__ import absolute_import import collections import os.path @@ -179,7 +178,7 @@ class _StepdownThread(threading.Thread): # pylint: disable=too-many-instance-at try: while True: - if self._is_stopped(): + if self.__is_stopped(): break self._wait_for_permission_or_resume() now = time.time() @@ -210,7 +209,7 @@ class _StepdownThread(threading.Thread): # pylint: disable=too-many-instance-at self.resume() self.join() - def _is_stopped(self): + def __is_stopped(self): return self._is_stopped_evt.is_set() def pause(self): @@ -234,7 +233,7 @@ class _StepdownThread(threading.Thread): # pylint: disable=too-many-instance-at def _wait_for_permission_or_resume(self): # Wait until stop, _stepdown_permitted_file or resume. if self._stepdown_permitted_file: - while not os.path.isfile(self._stepdown_permitted_file) and not self._is_stopped(): + while not os.path.isfile(self._stepdown_permitted_file) and not self.__is_stopped(): # Set a short sleep during busy wait time for self._stepdown_permitted_file. self._wait(0.1) else: diff --git a/buildscripts/resmokelib/testing/hooks/validate.py b/buildscripts/resmokelib/testing/hooks/validate.py index 3239ddbdc06..81bd42ba6d9 100644 --- a/buildscripts/resmokelib/testing/hooks/validate.py +++ b/buildscripts/resmokelib/testing/hooks/validate.py @@ -1,7 +1,5 @@ """Test hook for verifying the consistency and integrity of collection and index data.""" -from __future__ import absolute_import - import os.path from . import jsfile diff --git a/buildscripts/resmokelib/testing/hooks/wait_for_replication.py b/buildscripts/resmokelib/testing/hooks/wait_for_replication.py index fbd786a170c..8720f9456e6 100644 --- a/buildscripts/resmokelib/testing/hooks/wait_for_replication.py +++ b/buildscripts/resmokelib/testing/hooks/wait_for_replication.py @@ -1,7 +1,5 @@ """Test hook to wait for replication to complete on a replica set.""" -from __future__ import absolute_import - import time from buildscripts.resmokelib import core |