SERVER-47883 Override stepdown suites to ensure background index builds are complete after stepdown

(cherry picked from commit 045cbbe721087ab7d36c5b0c99096103eb7a7d45)

Backports causally_consistent_index_builds.js

5 files changed, 55 insertions, 1 deletions

diff --git a/buildscripts/resmokeconfig/suites/replica_sets_kill_primary_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/replica_sets_kill_primary_jscore_passthrough.yml
index e7b0bcf858c..c0978bf7f0a 100644
--- a/buildscripts/resmokeconfig/suites/replica_sets_kill_primary_jscore_passthrough.yml
+++ b/buildscripts/resmokeconfig/suites/replica_sets_kill_primary_jscore_passthrough.yml
@@ -144,6 +144,7 @@ executor:
         testingReplication = true;
         load("jstests/libs/override_methods/auto_retry_on_network_error.js");
         db = connect(TestData.connectionString);
+        load('jstests/libs/override_methods/causally_consistent_index_builds.js');
         load("jstests/libs/override_methods/enable_sessions.js");
         load("jstests/libs/override_methods/set_read_and_write_concerns.js");
         load("jstests/libs/override_methods/fail_unclean_shutdown_incompatible_commands.js");
diff --git a/buildscripts/resmokeconfig/suites/retryable_writes_jscore_stepdown_passthrough.yml b/buildscripts/resmokeconfig/suites/retryable_writes_jscore_stepdown_passthrough.yml
index f39c4c2b396..b20c7a3d7ab 100644
--- a/buildscripts/resmokeconfig/suites/retryable_writes_jscore_stepdown_passthrough.yml
+++ b/buildscripts/resmokeconfig/suites/retryable_writes_jscore_stepdown_passthrough.yml
@@ -130,6 +130,7 @@ executor:
         testingReplication = true;
         load("jstests/libs/override_methods/auto_retry_on_network_error.js");
         db = connect(TestData.connectionString);
+        load('jstests/libs/override_methods/causally_consistent_index_builds.js');
         load("jstests/libs/override_methods/enable_sessions.js");
         load("jstests/libs/override_methods/set_read_and_write_concerns.js");
       global_vars:
diff --git a/jstests/concurrency/fsm_libs/worker_thread.js b/jstests/concurrency/fsm_libs/worker_thread.js
index 8b1ea3be492..0d70add109d 100644
--- a/jstests/concurrency/fsm_libs/worker_thread.js
+++ b/jstests/concurrency/fsm_libs/worker_thread.js
@@ -141,6 +141,10 @@ var workerThread = (function() {
                     Object.assign(TestData, newOptions);
 
                     load('jstests/libs/override_methods/auto_retry_on_network_error.js');
+
+                    // After a step-up to primary, background index builds started on a secondary
+                    // may not be complete. Use this override to ensure causality.
+                    load('jstests/libs/override_methods/causally_consistent_index_builds.js');
                 }
 
                 // Operations that run after a "dropDatabase" command has been issued may fail with
diff --git a/jstests/libs/override_methods/auto_retry_on_network_error.js b/jstests/libs/override_methods/auto_retry_on_network_error.js
index 10952ebaf0a..40c4552391e 100644
--- a/jstests/libs/override_methods/auto_retry_on_network_error.js
+++ b/jstests/libs/override_methods/auto_retry_on_network_error.js
@@ -65,7 +65,6 @@
         "clone",
         "cloneCollection",
         "cloneCollectionAsCapped",
-        "collMod",
         "convertToCapped",
         "copydb",
         "create",
diff --git a/jstests/libs/override_methods/causally_consistent_index_builds.js b/jstests/libs/override_methods/causally_consistent_index_builds.js
new file mode 100644
index 00000000000..e5853527aaf
--- /dev/null
+++ b/jstests/libs/override_methods/causally_consistent_index_builds.js
@@ -0,0 +1,49 @@
+/**
+ * Overrides runCommand so that background index builds are causally consistent.
+ */
+(function() {
+    "use strict";
+
+    load("jstests/libs/override_methods/override_helpers.js");
+
+    // This override runs a collMod after a createIndexes command. After collMod completes
+    // we can guarantee the background index build started earlier has also completed. We update the
+    // command response operationTime and $clusterTime so causally consistent reads only read from
+    // that point onwards.
+    function runCommandWithCollMod(conn, dbName, commandName, commandObj, func, makeFuncArgs) {
+        if (typeof commandObj !== "object" || commandObj === null) {
+            return func.apply(conn, makeFuncArgs(commandObj));
+        }
+
+        let res = func.apply(conn, makeFuncArgs(commandObj));
+        if (commandName !== "createIndexes") {
+            return res;
+        }
+        if (!res.ok) {
+            return res;
+        }
+
+        let collModCmd = {collMod: commandObj[commandName]};
+        let collModRes = func.apply(conn, makeFuncArgs(collModCmd));
+
+        // If a follow-up collMod fails, another command was likely able to execute after the
+        // createIndexes command. That means it is safe to use the latest operationTime for
+        // causal consistency purposes.
+        if (!collModRes.ok) {
+            print('note: ignoring collMod failure after sending createIndex command: ' +
+                  tojson(collModRes));
+        }
+
+        // Overwrite the createIndex command's operation and cluster times, so that the owning
+        // session can perform causal reads.
+        if (collModRes.hasOwnProperty("operationTime")) {
+            res.operationTime = collModRes["operationTime"];
+        }
+        if (collModRes.hasOwnProperty("$clusterTime")) {
+            res.$clusterTime = collModRes["$clusterTime"];
+        }
+        return res;
+    }
+
+    OverrideHelpers.overrideRunCommand(runCommandWithCollMod);
+})();