SERVER-48379 Make ReplSetTest wait for secondaries before checking db hashes

(cherry picked from commit 9a6b5b9d099eb25aafc26fbf17c27fd32ad4b5ef)

2 files changed, 58 insertions, 0 deletions

diff --git a/jstests/replsets/replsettest_checks_wait_for_secondaries.js b/jstests/replsets/replsettest_checks_wait_for_secondaries.js
new file mode 100644
index 00000000000..861f038b8f1
--- /dev/null
+++ b/jstests/replsets/replsettest_checks_wait_for_secondaries.js
@@ -0,0 +1,51 @@
+/**
+ * Tests that ReplSetTest consistency checks, namely checkDBHashesForReplSet and
+ * checkCollectionCounts, wait for secondaries to have fully transitioned to SECONDARY state before
+ * attempting data reads.
+ */
+(function() {
+"use strict";
+load("jstests/libs/fail_point_util.js");
+
+const testName = jsTestName();
+const dbName = "testdb";
+const collName = "testcoll";
+
+const rst = new ReplSetTest({name: testName, nodes: 1});
+rst.startSet();
+rst.initiateWithHighElectionTimeout();
+
+const primary = rst.getPrimary();
+const primaryDb = primary.getDB(dbName);
+const primaryColl = primaryDb.getCollection(collName);
+
+assert.commandWorked(primaryColl.insert({"starting": "doc"}));
+
+jsTestLog("Adding a new node to the replica set");
+const secondaryParams = {
+    'failpoint.initialSyncHangBeforeCopyingDatabases': tojson({mode: 'alwaysOn'}),
+    'numInitialSyncAttempts': 1,
+};
+const secondary = rst.add({rsConfig: {priority: 0}, setParameter: secondaryParams});
+rst.reInitiate();
+
+jsTestLog("Waiting for node to reach initial sync");
+assert.commandWorked(secondary.adminCommand({
+    waitForFailPoint: "initialSyncHangBeforeCopyingDatabases",
+    timesEntered: 1,
+    maxTimeMS: kDefaultWaitForFailPointTimeout
+}));
+
+// Turn off the failpoint and immediately proceeed with collection counts checks.
+jsTestLog("Trying checkCollectionCounts");
+assert.commandWorked(secondary.adminCommand(
+    {configureFailPoint: "initialSyncHangBeforeCopyingDatabases", mode: "off"}));
+rst.checkCollectionCounts();
+
+// Restart the node so we can try checkReplicatedDBHashes.
+rst.stop(1);
+rst.start(1, {startClean: true});
+
+// stopSet() will call checkReplicatedDBHashes
+rst.stopSet();
+})();
\ No newline at end of file
diff --git a/src/mongo/shell/replsettest.js b/src/mongo/shell/replsettest.js
index c3d8fd7fff6..ecc14b01547 100644
--- a/src/mongo/shell/replsettest.js
+++ b/src/mongo/shell/replsettest.js
@@ -2219,6 +2219,9 @@ var ReplSetTest = function(opts) {
             var combinedDBs = new Set(primary.getDBNames());
             const replSetConfig = rst.getReplSetConfigFromNode();
 
+            print("checkDBHashesForReplSet waiting for secondaries to be ready: " + tojson(slaves));
+            this.awaitSecondaryNodes(self.kDefaultTimeoutMS, slaves);
+
             print("checkDBHashesForReplSet checking data hashes against primary: " + primary.host);
 
             slaves.forEach(node => {
@@ -2665,6 +2668,10 @@ var ReplSetTest = function(opts) {
         }
 
         function checkCollectionCountsForReplSet(rst) {
+            print("checkCollectionCountsForReplSet waiting for secondaries to be ready: " +
+                  tojson(rst.nodes));
+            this.awaitSecondaryNodes();
+
             rst.nodes.forEach(node => {
                 // Arbiters have no replicated collections.
                 if (isNodeArbiter(node)) {