SERVER-39624 Put internal router retries for stale version and snapshot errors behind a fail point

15 files changed, 95 insertions, 0 deletions

diff --git a/jstests/concurrency/fsm_workloads/snapshot_read_catalog_operations.js b/jstests/concurrency/fsm_workloads/snapshot_read_catalog_operations.js
index f54c48c6540..1ce1239a4a0 100644
--- a/jstests/concurrency/fsm_workloads/snapshot_read_catalog_operations.js
+++ b/jstests/concurrency/fsm_workloads/snapshot_read_catalog_operations.js
@@ -33,6 +33,7 @@ var $config = (function() {
             const readErrorCodes = [
                 ErrorCodes.NoSuchTransaction,
                 ErrorCodes.SnapshotUnavailable,
+                ErrorCodes.SnapshotTooOld,
                 ErrorCodes.LockTimeout,
                 ErrorCodes.ConflictingOperationInProgress
             ];
diff --git a/jstests/multiVersion/libs/sharded_txn_upgrade_downgrade_cluster_shared.js b/jstests/multiVersion/libs/sharded_txn_upgrade_downgrade_cluster_shared.js
index 4180b52fa01..0f77983adbf 100644
--- a/jstests/multiVersion/libs/sharded_txn_upgrade_downgrade_cluster_shared.js
+++ b/jstests/multiVersion/libs/sharded_txn_upgrade_downgrade_cluster_shared.js
@@ -3,6 +3,8 @@
  * multiversion/sharded_txn_downgrade_cluster.js.
  */
 
+load("jstests/sharding/libs/sharded_transactions_helpers.js");
+
 // Define autocommit as a variable so it can be used in object literals w/o an explicit value.
 const autocommit = false;
 
@@ -30,6 +32,8 @@ function setUpTwoShardClusterWithBinVersion(dbName, collName, binVersion) {
     assert.commandWorked(
         st.s.adminCommand({moveChunk: ns, find: {skey: 1}, to: st.shard1.shardName}));
 
+    flushRoutersAndRefreshShardMetadata(st, {ns, dbNames: [dbName]});
+
     return st;
 }
 
diff --git a/jstests/noPassthrough/readConcern_snapshot_mongos.js b/jstests/noPassthrough/readConcern_snapshot_mongos.js
index e318cf75932..a30f3ac9aae 100644
--- a/jstests/noPassthrough/readConcern_snapshot_mongos.js
+++ b/jstests/noPassthrough/readConcern_snapshot_mongos.js
@@ -3,6 +3,8 @@
 (function() {
     "use strict";
 
+    load("jstests/sharding/libs/sharded_transactions_helpers.js");
+
     // Runs the command as the first in a multi statement txn that is aborted right after, expecting
     // success.
     function expectSuccessInTxnThenAbort(session, sessionConn, cmdObj) {
@@ -29,6 +31,8 @@
     // Insert data to create the collection.
     assert.writeOK(testDB[collName].insert({x: 1}));
 
+    flushRoutersAndRefreshShardMetadata(st, {ns: dbName + "." + collName, dbNames: [dbName]});
+
     // noPassthrough tests
 
     // readConcern 'snapshot' is not allowed outside session context.
diff --git a/jstests/sharding/libs/sharded_transactions_helpers.js b/jstests/sharding/libs/sharded_transactions_helpers.js
index 05263868d30..efac12c1a4d 100644
--- a/jstests/sharding/libs/sharded_transactions_helpers.js
+++ b/jstests/sharding/libs/sharded_transactions_helpers.js
@@ -88,3 +88,44 @@ function waitForFailpoint(hitFailpointStr, numTimes) {
         return (rawMongoProgramOutput().match(re) || []).length == numTimes;
     }, 'Failed to find "' + hitFailpointStr + '" logged ' + numTimes + ' times');
 }
+
+// Enables the transaction router to retry on stale version (db or shard version) and snapshot
+// errors within a transaction.
+//
+// TODO SERVER-39704: Remove this function.
+function enableStaleVersionAndSnapshotRetriesWithinTransactions(st) {
+    assert.commandWorked(st.s.adminCommand({
+        configureFailPoint: "enableStaleVersionAndSnapshotRetriesWithinTransactions",
+        mode: "alwaysOn"
+    }));
+}
+
+// TODO SERVER-39704: Remove this function.
+function disableStaleVersionAndSnapshotRetriesWithinTransactions(st) {
+    assert.commandWorked(st.s.adminCommand({
+        configureFailPoint: "enableStaleVersionAndSnapshotRetriesWithinTransactions",
+        mode: "off"
+    }));
+}
+
+// Flush each router's metadata and force refreshes on each shard for the given namespace and/or
+// database names.
+//
+// TODO SERVER-39704: Remove this function.
+function flushRoutersAndRefreshShardMetadata(st, {ns, dbNames = []} = {}) {
+    st._mongos.forEach((s) => {
+        assert.commandWorked(s.adminCommand({flushRouterConfig: 1}));
+    });
+
+    st._rs.forEach((rs) => {
+        if (ns) {
+            assert.commandWorked(
+                rs.test.getPrimary().adminCommand({_flushRoutingTableCacheUpdates: ns}));
+        }
+
+        dbNames.forEach((dbName) => {
+            assert.commandWorked(
+                rs.test.getPrimary().adminCommand({_flushDatabaseCacheUpdates: dbName}));
+        });
+    });
+}
diff --git a/jstests/sharding/snapshot_cursor_commands_mongos.js b/jstests/sharding/snapshot_cursor_commands_mongos.js
index 37b60727a6c..bd776bdef9f 100644
--- a/jstests/sharding/snapshot_cursor_commands_mongos.js
+++ b/jstests/sharding/snapshot_cursor_commands_mongos.js
@@ -8,6 +8,7 @@
     TestData.disableImplicitSessions = true;
 
     load("jstests/libs/global_snapshot_reads_util.js");
+    load("jstests/sharding/libs/sharded_transactions_helpers.js");
 
     const dbName = "test";
     const shardedCollName = "shardedColl";
@@ -100,6 +101,8 @@
                 assert.eq(
                     1, mongos.getDB('config').chunks.count({ns: ns, shard: st.shard2.shardName}));
 
+                flushRoutersAndRefreshShardMetadata(st, {ns});
+
                 return st;
             }
         },
@@ -135,6 +138,8 @@
                 assert.eq(
                     1, mongos.getDB('config').chunks.count({ns: ns, shard: st.shard2.shardName}));
 
+                flushRoutersAndRefreshShardMetadata(st, {ns});
+
                 return st;
             }
         },
diff --git a/jstests/sharding/transactions_causal_consistency.js b/jstests/sharding/transactions_causal_consistency.js
index 88a7762d501..aef05abfa59 100644
--- a/jstests/sharding/transactions_causal_consistency.js
+++ b/jstests/sharding/transactions_causal_consistency.js
@@ -9,12 +9,16 @@
 (function() {
     "use strict";
 
+    load("jstests/sharding/libs/sharded_transactions_helpers.js");
+
     const dbName = "test";
     const collName = "foo";
     const ns = dbName + "." + collName;
 
     const st = new ShardingTest({shards: 2, mongos: 2});
 
+    enableStaleVersionAndSnapshotRetriesWithinTransactions(st);
+
     // Set up a sharded collection with 2 chunks, [min, 0) and [0, max), one on each shard, with one
     // document in each.
 
@@ -74,5 +78,7 @@
         runTest(st, {level: readConcernLevel});
     }
 
+    disableStaleVersionAndSnapshotRetriesWithinTransactions(st);
+
     st.stop();
 })();
diff --git a/jstests/sharding/transactions_error_labels.js b/jstests/sharding/transactions_error_labels.js
index 5341c5843a8..50864cde1c6 100644
--- a/jstests/sharding/transactions_error_labels.js
+++ b/jstests/sharding/transactions_error_labels.js
@@ -3,6 +3,8 @@
 (function() {
     "use strict";
 
+    load("jstests/sharding/libs/sharded_transactions_helpers.js");
+
     const dbName = "test";
     const collName = "foo";
     const ns = dbName + "." + collName;
@@ -163,6 +165,7 @@
     // commitTransaction for multi-shard transaction (mongos sends coordinateCommitTransaction)
     assert.commandWorked(
         st.s.adminCommand({moveChunk: ns, find: {_id: 0}, to: st.shard1.shardName}));
+    flushRoutersAndRefreshShardMetadata(st, {ns});
     runCommitTests("coordinateCommitTransaction");
 
     st.stop();
diff --git a/jstests/sharding/transactions_multi_writes.js b/jstests/sharding/transactions_multi_writes.js
index cf795aa7d7b..ca5163a62b2 100644
--- a/jstests/sharding/transactions_multi_writes.js
+++ b/jstests/sharding/transactions_multi_writes.js
@@ -8,12 +8,16 @@
 (function() {
     "use strict";
 
+    load("jstests/sharding/libs/sharded_transactions_helpers.js");
+
     const dbName = "test";
     const collName = "foo";
     const ns = dbName + "." + collName;
 
     const st = new ShardingTest({shards: 3, config: 1, mongos: 2});
 
+    enableStaleVersionAndSnapshotRetriesWithinTransactions(st);
+
     // Set up a sharded collection with 3 chunks, [min, 0), [0, 10), [10, max), one on each shard,
     // with one document in each.
 
@@ -136,5 +140,7 @@
     runTest(st, session, multiDelete, false /*staleRouter*/);
     runTest(st, session, multiDelete, true /*staleRouter*/);
 
+    disableStaleVersionAndSnapshotRetriesWithinTransactions(st);
+
     st.stop();
 })();
diff --git a/jstests/sharding/transactions_snapshot_errors_first_statement.js b/jstests/sharding/transactions_snapshot_errors_first_statement.js
index a839fc67b35..80d95f863c9 100644
--- a/jstests/sharding/transactions_snapshot_errors_first_statement.js
+++ b/jstests/sharding/transactions_snapshot_errors_first_statement.js
@@ -109,6 +109,8 @@
 
     const st = new ShardingTest({shards: 2, mongos: 1, config: 1});
 
+    enableStaleVersionAndSnapshotRetriesWithinTransactions(st);
+
     jsTestLog("Unsharded transaction");
 
     assert.writeOK(st.s.getDB(dbName)[collName].insert({_id: 5}, {writeConcern: {w: "majority"}}));
@@ -152,5 +154,7 @@
         runTest(st, collName, 1, errorCode, true);
     }
 
+    disableStaleVersionAndSnapshotRetriesWithinTransactions(st);
+
     st.stop();
 })();
diff --git a/jstests/sharding/transactions_snapshot_errors_subsequent_statements.js b/jstests/sharding/transactions_snapshot_errors_subsequent_statements.js
index 5a488731516..b2556517750 100644
--- a/jstests/sharding/transactions_snapshot_errors_subsequent_statements.js
+++ b/jstests/sharding/transactions_snapshot_errors_subsequent_statements.js
@@ -67,6 +67,8 @@
 
     const st = new ShardingTest({shards: 2, mongos: 1, config: 1});
 
+    enableStaleVersionAndSnapshotRetriesWithinTransactions(st);
+
     jsTestLog("Unsharded transaction");
 
     assert.writeOK(st.s.getDB(dbName)[collName].insert({_id: 5}, {writeConcern: {w: "majority"}}));
@@ -108,5 +110,7 @@
         runTest(st, collName, errorCode);
     }
 
+    disableStaleVersionAndSnapshotRetriesWithinTransactions(st);
+
     st.stop();
 })();
diff --git a/jstests/sharding/transactions_stale_database_version_errors.js b/jstests/sharding/transactions_stale_database_version_errors.js
index aacf9bb541c..6eb95f28c1f 100644
--- a/jstests/sharding/transactions_stale_database_version_errors.js
+++ b/jstests/sharding/transactions_stale_database_version_errors.js
@@ -11,6 +11,8 @@
 
     const st = new ShardingTest({shards: 2, mongos: 1, config: 1});
 
+    enableStaleVersionAndSnapshotRetriesWithinTransactions(st);
+
     // Set up two unsharded collections in different databases with shard0 as their primary.
 
     assert.writeOK(st.s.getDB(dbName)[collName].insert({_id: 0}, {writeConcern: {w: "majority"}}));
@@ -121,5 +123,7 @@
     assert.commandWorked(st.rs0.getPrimary().adminCommand(
         {configureFailPoint: "skipDatabaseVersionMetadataRefresh", mode: "off"}));
 
+    disableStaleVersionAndSnapshotRetriesWithinTransactions(st);
+
     st.stop();
 })();
diff --git a/jstests/sharding/transactions_stale_shard_version_errors.js b/jstests/sharding/transactions_stale_shard_version_errors.js
index 910a1f883a0..10c10958fb5 100644
--- a/jstests/sharding/transactions_stale_shard_version_errors.js
+++ b/jstests/sharding/transactions_stale_shard_version_errors.js
@@ -20,6 +20,8 @@
 
     const st = new ShardingTest({shards: 3, mongos: 2, config: 1});
 
+    enableStaleVersionAndSnapshotRetriesWithinTransactions(st);
+
     // Disable the best-effort recipient metadata refresh after migrations to simplify simulating
     // stale shard version errors.
     assert.commandWorked(st.rs0.getPrimary().adminCommand(
@@ -256,5 +258,7 @@
     assert.commandWorked(st.rs0.getPrimary().adminCommand(
         {configureFailPoint: "skipShardFilteringMetadataRefresh", mode: "off"}));
 
+    disableStaleVersionAndSnapshotRetriesWithinTransactions(st);
+
     st.stop();
 })();
diff --git a/jstests/sharding/transactions_view_resolution.js b/jstests/sharding/transactions_view_resolution.js
index 262fbcf081f..993c1832213 100644
--- a/jstests/sharding/transactions_view_resolution.js
+++ b/jstests/sharding/transactions_view_resolution.js
@@ -10,6 +10,7 @@
     "use strict";
 
     load("jstests/aggregation/extras/utils.js");  // For arrayEq.
+    load("jstests/sharding/libs/sharded_transactions_helpers.js");
 
     const shardedDbName = "shardedDB";
     const shardedCollName = "sharded";
@@ -52,6 +53,8 @@
         assert.commandWorked(shardedView.runCommand(
             "create", {viewOn: shardedCollName, pipeline: [], writeConcern: {w: "majority"}}));
 
+        flushRoutersAndRefreshShardMetadata(st, {ns, dbNames: [shardedDbName, unshardedDbName]});
+
         return shardedView;
     }
 
diff --git a/jstests/sharding/txn_two_phase_commit_failover.js b/jstests/sharding/txn_two_phase_commit_failover.js
index b74e5cb54e6..9e7f7fb191a 100644
--- a/jstests/sharding/txn_two_phase_commit_failover.js
+++ b/jstests/sharding/txn_two_phase_commit_failover.js
@@ -110,6 +110,8 @@ TestData.skipCheckingUUIDsConsistentAcrossCluster = true;
             assert.commandWorked(
                 st.s.adminCommand({moveChunk: ns, find: {_id: 10}, to: participant2.shardName}));
 
+            flushRoutersAndRefreshShardMetadata(st, {ns});
+
             // Start a new transaction by inserting a document onto each shard.
             assert.commandWorked(st.s.getDB(dbName).runCommand({
                 insert: collName,
diff --git a/jstests/sharding/unsharded_lookup_in_txn.js b/jstests/sharding/unsharded_lookup_in_txn.js
index 74ac92114b7..620052a634f 100644
--- a/jstests/sharding/unsharded_lookup_in_txn.js
+++ b/jstests/sharding/unsharded_lookup_in_txn.js
@@ -6,6 +6,8 @@
 (function() {
     "use strict";
 
+    load("jstests/sharding/libs/sharded_transactions_helpers.js");
+
     const st = new ShardingTest({shards: 2, mongos: 1});
     const kDBName = "unsharded_lookup_in_txn";
 
@@ -24,6 +26,7 @@
     // Move all of the data to shard 1.
     assert.commandWorked(st.s.adminCommand(
         {moveChunk: shardedColl.getFullName(), find: {_id: 0}, to: st.shard1.shardName}));
+    flushRoutersAndRefreshShardMetadata(st, {ns: shardedColl.getFullName()});
 
     // Insert a bunch of documents, all of which reside on the same chunk (on shard 1).
     for (let i = -10; i < 10; i++) {
@@ -85,6 +88,7 @@
     assert.commandWorked(st.s.adminCommand({split: shardedColl.getFullName(), middle: {_id: 0}}));
     assert.commandWorked(st.s.adminCommand(
         {moveChunk: shardedColl.getFullName(), find: {_id: -1}, to: st.shard0.shardName}));
+    flushRoutersAndRefreshShardMetadata(st, {ns: shardedColl.getFullName()});
 
     // Run the test again.
     testLookupDoesNotSeeDocumentsOutsideSnapshot();