SERVER-18390 Check dbhash of all databases during small_oplog suites

5 files changed, 101 insertions, 19 deletions

diff --git a/buildscripts/resmokelib/testing/fixtures/interface.py b/buildscripts/resmokelib/testing/fixtures/interface.py
index a6d12890b66..7367bb10ef7 100644
--- a/buildscripts/resmokelib/testing/fixtures/interface.py
+++ b/buildscripts/resmokelib/testing/fixtures/interface.py
@@ -4,6 +4,11 @@ Interface of the different fixtures for executing JSTests against.
 
 from __future__ import absolute_import
 
+import time
+
+import pymongo
+
+from ... import errors
 from ... import logging
 
 
@@ -67,6 +72,8 @@ class ReplFixture(Fixture):
     Base class for all fixtures that support replication.
     """
 
+    AWAIT_REPL_TIMEOUT_MINS = 5
+
     def get_primary(self):
         """
         Returns the primary of a replica set, or the master of a
@@ -87,3 +94,27 @@ class ReplFixture(Fixture):
         replicated to all other nodes.
         """
         raise NotImplementedError("await_repl must be implemented by ReplFixture subclasses")
+
+    def retry_until_wtimeout(self, insert_fn):
+        """
+        Given a callback function representing an insert operation on
+        the primary, handle any connection failures, and keep retrying
+        the operation for up to 'AWAIT_REPL_TIMEOUT_MINS' minutes.
+
+        The insert operation callback should take an argument for the
+        number of remaining seconds to provide as the timeout for the
+        operation.
+        """
+
+        deadline = time.time() + ReplFixture.AWAIT_REPL_TIMEOUT_MINS * 60
+
+        while True:
+            try:
+                remaining = deadline - time.time()
+                insert_fn(remaining)
+                break
+            except pymongo.errors.ConnectionFailure:
+                remaining = deadline - time.time()
+                if remaining <= 0.0:
+                    raise errors.ServerFailure("Failed to connect to the primary on port %d" %
+                                               self.port)
diff --git a/buildscripts/resmokelib/testing/fixtures/masterslave.py b/buildscripts/resmokelib/testing/fixtures/masterslave.py
index 79a9bce3e99..cc3607bb218 100644
--- a/buildscripts/resmokelib/testing/fixtures/masterslave.py
+++ b/buildscripts/resmokelib/testing/fixtures/masterslave.py
@@ -6,6 +6,8 @@ from __future__ import absolute_import
 
 import os.path
 
+import pymongo
+
 from . import interface
 from . import standalone
 from ... import config
@@ -19,8 +21,6 @@ class MasterSlaveFixture(interface.ReplFixture):
     run against.
     """
 
-    AWAIT_REPL_TIMEOUT_MINS = 5
-
     def __init__(self,
                  logger,
                  job_num,
@@ -103,15 +103,49 @@ class MasterSlaveFixture(interface.ReplFixture):
         return [self.slave]
 
     def await_repl(self):
-        self.logger.info("Awaiting replication of insert (w=2, wtimeout=%d min) to master on port"
-                         " %d", MasterSlaveFixture.AWAIT_REPL_TIMEOUT_MINS, self.port)
-        repl_timeout = MasterSlaveFixture.AWAIT_REPL_TIMEOUT_MINS * 60 * 1000
+        """
+        Inserts a document into each database on the master and waits
+        for all write operations to be acknowledged by the master-slave
+        deployment.
+        """
+
         client = utils.new_mongo_client(self.port)
 
-        # Use the same database as the jstests to ensure that the slave doesn't acknowledge the
-        # write as having completed before it has synced the test database.
-        client.test.resmoke_await_repl.insert({}, w=2, wtimeout=repl_timeout)
-        self.logger.info("Replication of write operation completed.")
+        # We verify that each database has replicated to the slave because in the case of an initial
+        # sync, the slave may acknowledge writes to one database before it has finished syncing
+        # others.
+        db_names = client.database_names()
+        self.logger.info("Awaiting replication of inserts to each of the following databases on"
+                         " master on port %d: %s",
+                         self.port,
+                         db_names)
+
+        for db_name in db_names:
+            if db_name == "local":
+                continue  # The local database is expected to differ, ignore.
+
+            self.logger.info("Awaiting replication of insert to database %s (w=2, wtimeout=%d min)"
+                             " to master on port %d",
+                             db_name,
+                             interface.ReplFixture.AWAIT_REPL_TIMEOUT_MINS,
+                             self.port)
+
+            # Keep retrying this until it times out waiting for replication.
+            def insert_fn(remaining_secs):
+                remaining_millis = int(round(remaining_secs * 1000))
+                client[db_name].resmoke_await_repl.insert({"awaiting": "repl"},
+                                                          w=2,
+                                                          wtimeout=remaining_millis)
+
+            try:
+                self.retry_until_wtimeout(insert_fn)
+            except pymongo.errors.WTimeoutError:
+                self.logger.info("Replication of write operation timed out.")
+                raise
+
+            self.logger.info("Replication of write operation completed for database %s.", db_name)
+
+        self.logger.info("Finished awaiting replication.")
 
     def _new_mongod(self, mongod_logger, mongod_options):
         """
diff --git a/buildscripts/resmokelib/testing/fixtures/replicaset.py b/buildscripts/resmokelib/testing/fixtures/replicaset.py
index 73ec510279c..44aadb0a76a 100644
--- a/buildscripts/resmokelib/testing/fixtures/replicaset.py
+++ b/buildscripts/resmokelib/testing/fixtures/replicaset.py
@@ -21,8 +21,6 @@ class ReplicaSetFixture(interface.ReplFixture):
     Fixture which provides JSTests with a replica set to run against.
     """
 
-    AWAIT_REPL_TIMEOUT_MINS = 5
-
     def __init__(self,
                  logger,
                  job_num,
@@ -144,11 +142,22 @@ class ReplicaSetFixture(interface.ReplFixture):
 
     def await_repl(self):
         self.logger.info("Awaiting replication of insert (w=%d, wtimeout=%d min) to primary on port"
-                         " %d", self.num_nodes, ReplicaSetFixture.AWAIT_REPL_TIMEOUT_MINS,
+                         " %d", self.num_nodes, interface.ReplFixture.AWAIT_REPL_TIMEOUT_MINS,
                          self.port)
-        repl_timeout = ReplicaSetFixture.AWAIT_REPL_TIMEOUT_MINS * 60 * 1000
-        client = utils.new_mongo_client(port=self.port)
-        client.resmoke.await_repl.insert({}, w=self.num_nodes, wtimeout=repl_timeout)
+
+        # Keep retrying this until it times out waiting for replication.
+        def insert_fn(remaining_secs):
+            remaining_millis = int(round(remaining_secs * 1000))
+            client = utils.new_mongo_client(port=self.port)
+            client.resmoke.await_repl.insert({"awaiting": "repl"},
+                                             w=self.num_nodes,
+                                             wtimeout=remaining_millis)
+        try:
+            self.retry_until_wtimeout(insert_fn)
+        except pymongo.errors.WTimeoutError:
+            self.logger.info("Replication of write operation timed out.")
+            raise
+
         self.logger.info("Replication of write operation completed.")
 
     def _new_mongod(self, index, replset_name):
diff --git a/buildscripts/resmokelib/testing/fixtures/shardedcluster.py b/buildscripts/resmokelib/testing/fixtures/shardedcluster.py
index a89b7d75855..467d0b308c5 100644
--- a/buildscripts/resmokelib/testing/fixtures/shardedcluster.py
+++ b/buildscripts/resmokelib/testing/fixtures/shardedcluster.py
@@ -319,10 +319,14 @@ class _MongoSFixture(interface.Fixture):
                                  self.mongos.pid)
                 self.mongos.stop()
 
-            success = self.mongos.wait() == 0
+            exit_code = self.mongos.wait()
+            success = exit_code == 0
 
             if running_at_start:
-                self.logger.info("Successfully terminated the mongos on port %d.", self.port)
+                self.logger.info("Successfully terminated the mongos on port %d, exited with code"
+                                 " %d",
+                                 self.port,
+                                 exit_code)
 
         return success
 
diff --git a/buildscripts/resmokelib/testing/fixtures/standalone.py b/buildscripts/resmokelib/testing/fixtures/standalone.py
index e4e5ea35a80..2a2158ef6f2 100644
--- a/buildscripts/resmokelib/testing/fixtures/standalone.py
+++ b/buildscripts/resmokelib/testing/fixtures/standalone.py
@@ -126,10 +126,14 @@ class MongoDFixture(interface.Fixture):
                                  self.mongod.pid)
                 self.mongod.stop()
 
-            success = self.mongod.wait() == 0
+            exit_code = self.mongod.wait()
+            success = exit_code == 0
 
             if running_at_start:
-                self.logger.info("Successfully terminated the mongod on port %d.", self.port)
+                self.logger.info("Successfully terminated the mongod on port %d, exited with code"
+                                 " %d.",
+                                 self.port,
+                                 exit_code)
 
         return success