SERVER-52641 Wait for all three index builds to start before restarting the secondary in index_build_restart_secondary.js

(cherry picked from commit 6d0b5b3ea630ba6b69597e7f0c670cf2e80e4265)
(cherry picked from commit f8af526ef229defb64664ca3455524554b52ece4)

2 files changed, 93 insertions, 148 deletions

diff --git a/jstests/noPassthrough/index_build_restart_secondary.js b/jstests/noPassthrough/index_build_restart_secondary.js
new file mode 100644
index 00000000000..01c64ff36c7
--- /dev/null
+++ b/jstests/noPassthrough/index_build_restart_secondary.js
@@ -0,0 +1,93 @@
+/**
+ * Tests restarting a secondary once an index build starts. The index build should complete when the
+ * node starts back up.
+ *
+ * @tags: [
+ *   requires_journaling,
+ *   requires_persistence,
+ *   requires_replication,
+ * ]
+ */
+(function() {
+'use strict';
+
+load('jstests/noPassthrough/libs/index_build.js');
+
+const replTest = new ReplSetTest({
+    nodes: [
+        {},
+        {
+            // Disallow elections on the secondary.
+            rsConfig: {
+                priority: 0,
+                votes: 0,
+            },
+            slowms: 30000,  // Don't log slow operations on the secondary. See SERVER-44821.
+        },
+    ]
+});
+
+replTest.startSet();
+replTest.initiate();
+
+const primary = replTest.getPrimary();
+const secondary = replTest.getSecondary();
+
+const primaryDB = primary.getDB('test');
+const secondaryDB = secondary.getDB('test');
+
+const collectionName = jsTestName();
+const coll = primaryDB.getCollection(collectionName);
+
+const bulk = coll.initializeUnorderedBulkOp();
+for (let i = 0; i < 100; ++i) {
+    bulk.insert({i: i, x: i, y: i});
+}
+assert.commandWorked(bulk.execute({j: true}));
+
+// Make sure the documents make it to the secondary.
+replTest.awaitReplication();
+
+// Pause the index build on the primary after replicating the startIndexBuild oplog entry.
+IndexBuildTest.pauseIndexBuilds(primaryDB);
+IndexBuildTest.startIndexBuild(primary, coll.getFullName(), {i: 1});
+IndexBuildTest.startIndexBuild(primary, coll.getFullName(), {x: 1});
+IndexBuildTest.startIndexBuild(primary, coll.getFullName(), {y: 1});
+
+// Wait for build to start on the secondary.
+jsTestLog("Waiting for all index builds to start on the secondary");
+IndexBuildTest.waitForIndexBuildToStart(secondaryDB, coll.getName(), "i_1");
+IndexBuildTest.waitForIndexBuildToStart(secondaryDB, coll.getName(), "x_1");
+IndexBuildTest.waitForIndexBuildToStart(secondaryDB, coll.getName(), "y_1");
+
+replTest.stop(secondary);
+replTest.start(secondary,
+               {
+                   setParameter: {
+                       "failpoint.hangAfterSettingUpIndexBuildUnlocked": tojson({mode: "alwaysOn"}),
+                   }
+               },
+               true /* restart */);
+
+// Verify that we do not wait for the index build to complete on startup.
+assert.soonNoExcept(() => {
+    IndexBuildTest.assertIndexes(secondaryDB.getCollection(collectionName),
+                                 4,
+                                 ["_id_"],
+                                 ["i_1", "x_1", "y_1"],
+                                 {includeBuildUUIDS: true});
+    return true;
+});
+
+assert.commandWorked(secondary.adminCommand(
+    {configureFailPoint: 'hangAfterSettingUpIndexBuildUnlocked', mode: 'off'}));
+
+// Let index build complete on primary, which replicates a commitIndexBuild to the secondary.
+IndexBuildTest.resumeIndexBuilds(primaryDB);
+
+assert.soonNoExcept(function() {
+    return 4 === secondaryDB.getCollection(collectionName).getIndexes().length;
+}, "Index build did not complete after restart");
+
+replTest.stopSet();
+}());
\ No newline at end of file
diff --git a/jstests/noPassthrough/indexbg_restart_secondary.js b/jstests/noPassthrough/indexbg_restart_secondary.js
deleted file mode 100644
index 9f5de7dc7f2..00000000000
--- a/jstests/noPassthrough/indexbg_restart_secondary.js
+++ /dev/null
@@ -1,148 +0,0 @@
-/**
- * Starts a replica set with arbiter, builds an index in background.
- * Kills the secondary once the index build starts with a failpoint.
- * The index build should resume when the secondary is restarted.
- *
- * @tags: [
- *   requires_persistence,
- *   requires_journaling,
- *   requires_replication,
- * ]
- */
-
-(function() {
-'use strict';
-
-load('jstests/noPassthrough/libs/index_build.js');
-
-// Set up replica set
-const replTest = new ReplSetTest({
-    nodes: [
-        {},
-        {
-            // Disallow elections on secondary.
-            rsConfig: {
-                priority: 0,
-                votes: 0,
-            },
-            slowms: 30000,  // Don't log slow operations on secondary. See SERVER-44821.
-        },
-    ]
-});
-
-// We need an arbiter to ensure that the primary doesn't step down
-// when we restart the secondary.
-const nodes = replTest.startSet();
-replTest.initiate();
-
-var primary = replTest.getPrimary();
-var second = replTest.getSecondary();
-
-var primaryDB = primary.getDB('bgIndexSec');
-var secondDB = second.getDB('bgIndexSec');
-
-var collectionName = 'jstests_bgsec';
-
-var coll = primaryDB.getCollection(collectionName);
-
-var size = 100;
-
-var bulk = primaryDB.jstests_bgsec.initializeUnorderedBulkOp();
-for (var i = 0; i < size; ++i) {
-    bulk.insert({i: i, x: i, y: i});
-}
-assert.commandWorked(bulk.execute({j: true}));
-assert.eq(size, coll.count(), 'unexpected number of documents after bulk insert.');
-
-// Make sure the documents make it to the secondary.
-replTest.awaitReplication();
-
-if (IndexBuildTest.supportsTwoPhaseIndexBuild(primary)) {
-    // Pause the index build on the primary after replicating the startIndexBuild oplog entry.
-    IndexBuildTest.pauseIndexBuilds(primaryDB);
-    IndexBuildTest.startIndexBuild(primary, coll.getFullName(), {i: 1});
-    IndexBuildTest.startIndexBuild(primary, coll.getFullName(), {x: 1});
-    IndexBuildTest.startIndexBuild(primary, coll.getFullName(), {y: 1});
-
-    // Wait for build to start on the secondary.
-    jsTestLog("waiting for all index builds to start on secondary");
-    IndexBuildTest.waitForIndexBuildToStart(secondDB, coll.getName(), "y_1");
-} else {
-    assert.commandWorked(secondDB.adminCommand(
-        {configureFailPoint: 'leaveIndexBuildUnfinishedForShutdown', mode: 'alwaysOn'}));
-
-    try {
-        coll.createIndex({i: 1});
-        coll.createIndex({x: 1});
-        coll.createIndex({y: 1});
-        primaryDB.getLastError(2);
-        assert.eq(4, coll.getIndexes().length);
-
-        // Make sure all writes are durable on the secondary so that we can restart it knowing that
-        // the index build will be found on startup.
-        // Waiting for durable is important for both (A) the record that we started the index build
-        // so it is rebuild on restart, and (B) the update to minvalid to show that we've already
-        // applied the oplog entry so it isn't replayed. If (A) is present without (B), then there
-        // are two ways that the index can be rebuilt on startup and this test is only for the one
-        // triggered by (A).
-        secondDB.adminCommand({fsync: 1});
-    } finally {
-        assert.commandWorked(secondDB.adminCommand(
-            {configureFailPoint: 'leaveIndexBuildUnfinishedForShutdown', mode: 'off'}));
-    }
-}
-
-MongoRunner.stopMongod(second);
-
-if (IndexBuildTest.supportsTwoPhaseIndexBuild(primary)) {
-    replTest.start(
-        second,
-        {
-            setParameter:
-                {"failpoint.hangAfterSettingUpIndexBuildUnlocked": tojson({mode: "alwaysOn"})}
-        },
-        /*restart=*/true,
-        /*wait=*/true);
-} else {
-    replTest.start(second,
-                   {},
-                   /*restart=*/true,
-                   /*wait=*/true);
-}
-
-// Make sure secondary comes back.
-try {
-    assert.soon(function() {
-        try {
-            secondDB.isMaster();  // trigger a reconnect if needed
-            return true;
-        } catch (e) {
-            return false;
-        }
-    }, "secondary didn't restart", 30000, 1000);
-
-    if (IndexBuildTest.supportsTwoPhaseIndexBuild(primary)) {
-        // Verify that we do not wait for the index build to complete on startup.
-        assert.eq(size, secondDB.getCollection(collectionName).find({}).itcount());
-
-        // Verify that only the _id index is ready.
-        checkLog.containsJson(second, 4585201);
-        IndexBuildTest.assertIndexes(secondDB.getCollection(collectionName),
-                                     4,
-                                     ["_id_"],
-                                     ["i_1", "x_1", "y_1"],
-                                     {includeBuildUUIDS: true});
-    }
-} finally {
-    assert.commandWorked(second.adminCommand(
-        {configureFailPoint: 'hangAfterSettingUpIndexBuildUnlocked', mode: 'off'}));
-
-    // Let index build complete on primary, which replicates a commitIndexBuild to the secondary.
-    IndexBuildTest.resumeIndexBuilds(primaryDB);
-}
-
-assert.soonNoExcept(function() {
-    return 4 == secondDB.getCollection(collectionName).getIndexes().length;
-}, "Index build not resumed after restart", 30000, 50);
-replTest.stopSet();
-}());