SERVER-35262 Add concurrency_simultaneous_replication.yml test suite

5 files changed, 216 insertions, 29 deletions

diff --git a/buildscripts/resmokeconfig/suites/concurrency_simultaneous_replication.yml b/buildscripts/resmokeconfig/suites/concurrency_simultaneous_replication.yml
new file mode 100644
index 00000000000..f16db546d66
--- /dev/null
+++ b/buildscripts/resmokeconfig/suites/concurrency_simultaneous_replication.yml
@@ -0,0 +1,54 @@
+test_kind: parallel_fsm_workload_test
+
+selector:
+  roots:
+  - jstests/concurrency/fsm_workloads/**/*.js
+  exclude_files:
+  # These workloads implicitly assume that their tid ranges are [0, $config.threadCount). This
+  # isn't guaranteed to be true when they are run in parallel with other workloads.
+  - jstests/concurrency/fsm_workloads/list_indexes.js
+  - jstests/concurrency/fsm_workloads/update_inc_capped.js
+
+  # These workloads uses >100MB of data, which can overwhelm test hosts.
+  - jstests/concurrency/fsm_workloads/agg_group_external.js
+  - jstests/concurrency/fsm_workloads/agg_sort_external.js
+  # The findAndModify_update_grow.js workload can cause OOM kills on test hosts.
+  - jstests/concurrency/fsm_workloads/findAndModify_update_grow.js
+
+  # This workload kills random cursors which takes a collection lock.
+  # TODO: SERVER-35567.
+  - jstests/concurrency/fsm_workloads/snapshot_read_kill_operations.js
+  exclude_with_any_tags:
+  - requires_sharding
+
+  group_size: 10
+  group_count_multiplier: 1.0
+
+executor:
+  archive:
+    hooks:
+      - CheckReplDBHashInBackground
+      - CheckReplDBHash
+      - ValidateCollections
+    tests: true
+  config:
+    shell_options:
+      readMode: commands
+  hooks:
+  # The CheckReplDBHash hook waits until all operations have replicated to and have been applied
+  # on the secondaries, so we run the ValidateCollections hook after it to ensure we're
+  # validating the entire contents of the collection.
+  #
+  # TODO SERVER-26466: Add CheckReplOplogs hook to the concurrency suite.
+  - class: CheckReplDBHashInBackground
+  - class: CheckReplDBHash
+  - class: ValidateCollections
+  - class: CleanupConcurrencyWorkloads
+  fixture:
+    class: ReplicaSetFixture
+    mongod_options:
+      oplogSize: 1024
+      set_parameters:
+        enableTestCommands: 1
+        numInitialSyncAttempts: 1
+    num_nodes: 3
diff --git a/etc/evergreen.yml b/etc/evergreen.yml
index 8d11e7298fd..d51baa9b6c8 100644
--- a/etc/evergreen.yml
+++ b/etc/evergreen.yml
@@ -4728,6 +4728,14 @@ tasks:
       resmoke_args: --suites=concurrency_simultaneous --storageEngine=wiredTiger
 
 - <<: *task_template
+  name: concurrency_simultaneous_replication
+  commands:
+  - func: "do setup"
+  - func: "run tests"
+    vars:
+      resmoke_args: --suites=concurrency_simultaneous_replication --storageEngine=wiredTiger
+
+- <<: *task_template
   name: rlp
   commands:
   - func: "do setup"
@@ -6882,6 +6890,7 @@ buildvariants:
   - name: concurrency_sharded_replication
   - name: concurrency_sharded_replication_with_balancer
   - name: concurrency_simultaneous
+  - name: concurrency_simultaneous_replication
   - name: read_concern_linearizable_passthrough
   - name: read_concern_majority_passthrough
   - name: read_only
@@ -6942,6 +6951,9 @@ buildvariants:
     distros:
     - rhel62-large
   - name: concurrency_simultaneous
+  - name: concurrency_simultaneous_replication
+    distros:
+    - rhel62-large
 
 - &linux-64-debug-template
   name: linux-64-debug
@@ -6998,6 +7010,7 @@ buildvariants:
   - name: concurrency_sharded_with_stepdowns
   - name: concurrency_sharded_with_stepdowns_and_balancer
   - name: concurrency_simultaneous
+  - name: concurrency_simultaneous_replication
   - name: dbtest
   - name: disk_wiredtiger
   - name: failpoints
@@ -7344,6 +7357,7 @@ buildvariants:
   - name: concurrency_sharded_replication
   - name: concurrency_sharded_replication_with_balancer
   - name: concurrency_simultaneous
+  - name: concurrency_simultaneous_replication
   - name: replica_sets
   - name: replica_sets_auth
   - name: replica_sets_jscore_passthrough
@@ -7611,6 +7625,7 @@ buildvariants:
   - name: concurrency_sharded_replication
   - name: concurrency_sharded_replication_with_balancer
   - name: concurrency_simultaneous
+  - name: concurrency_simultaneous_replication
   - name: replica_sets
   - name: replica_sets_auth
   - name: replica_sets_jscore_passthrough
@@ -7693,6 +7708,7 @@ buildvariants:
   - name: concurrency_sharded_replication
   - name: concurrency_sharded_replication_with_balancer
   - name: concurrency_simultaneous
+  - name: concurrency_simultaneous_replication
   - name: replica_sets
   - name: replica_sets_auth
   - name: replica_sets_jscore_passthrough
@@ -7810,6 +7826,7 @@ buildvariants:
   - name: concurrency_sharded_replication
   - name: concurrency_sharded_replication_with_balancer
   - name: concurrency_simultaneous
+  - name: concurrency_simultaneous_replication
   - name: replica_sets
   - name: replica_sets_auth
   - name: replica_sets_jscore_passthrough
@@ -7872,6 +7889,7 @@ buildvariants:
   - name: concurrency_sharded_replication
   - name: concurrency_sharded_replication_with_balancer
   - name: concurrency_simultaneous
+  - name: concurrency_simultaneous_replication
   - name: dbtest
   - name: ese
   - name: failpoints
@@ -8080,6 +8098,7 @@ buildvariants:
   - name: concurrency_sharded_replication
   - name: concurrency_sharded_replication_with_balancer
   - name: concurrency_simultaneous
+  - name: concurrency_simultaneous_replication
   - name: replica_sets
   - name: replica_sets_auth
   - name: replica_sets_jscore_passthrough
@@ -8255,6 +8274,7 @@ buildvariants:
   - name: concurrency_sharded_replication
   - name: concurrency_sharded_replication_with_balancer
   - name: concurrency_simultaneous
+  - name: concurrency_simultaneous_replication
   - name: replica_sets
   - name: replica_sets_auth
   - name: replica_sets_jscore_passthrough
@@ -8377,6 +8397,9 @@ buildvariants:
   - name: concurrency_simultaneous
     distros:
     - windows-64-vs2015-large
+  - name: concurrency_simultaneous_replication
+    distros:
+    - windows-64-vs2015-large
   - name: read_concern_linearizable_passthrough
     distros:
     - windows-64-vs2015-large
@@ -8812,6 +8835,7 @@ buildvariants:
   - name: concurrency_sharded_replication
   - name: concurrency_sharded_replication_with_balancer
   - name: concurrency_simultaneous
+  - name: concurrency_simultaneous_replication
   - name: jsCore
   - name: jsCore_auth
   - name: jsCore_txns
@@ -8884,6 +8908,7 @@ buildvariants:
   - name: concurrency_sharded_replication
   - name: concurrency_sharded_replication_with_balancer
   - name: concurrency_simultaneous
+  - name: concurrency_simultaneous_replication
   - name: dbtest
   - name: disk_wiredtiger
   - name: failpoints
@@ -9042,6 +9067,7 @@ buildvariants:
   - name: concurrency_sharded_with_stepdowns
   - name: concurrency_sharded_with_stepdowns_and_balancer
   - name: concurrency_simultaneous
+  - name: concurrency_simultaneous_replication
   - name: dbtest
   - name: disk_wiredtiger
   - name: failpoints
@@ -9702,6 +9728,7 @@ buildvariants:
   - name: concurrency_sharded_with_stepdowns
   - name: concurrency_sharded_with_stepdowns_and_balancer
   - name: concurrency_simultaneous
+  - name: concurrency_simultaneous_replication
   - name: dbtest
   - name: disk_wiredtiger
   - name: ese
@@ -10208,6 +10235,7 @@ buildvariants:
   - name: concurrency_sharded_with_stepdowns
   - name: concurrency_sharded_with_stepdowns_and_balancer
   - name: concurrency_simultaneous
+  - name: concurrency_simultaneous_replication
   - name: dbtest
   - name: disk_wiredtiger
   - name: ese
@@ -10450,6 +10478,7 @@ buildvariants:
   - name: concurrency_sharded_replication
   - name: concurrency_sharded_replication_with_balancer
   - name: concurrency_simultaneous
+  - name: concurrency_simultaneous_replication
   - name: replica_sets
   - name: replica_sets_auth
   - name: replica_sets_jscore_passthrough
@@ -10548,6 +10577,9 @@ buildvariants:
   - name: concurrency_simultaneous
     distros:
     - rhel70
+  - name: concurrency_simultaneous_replication
+    distros:
+    - rhel70
   - name: replica_sets
   - name: replica_sets_auth
   - name: replica_sets_jscore_passthrough
@@ -10612,6 +10644,9 @@ buildvariants:
   - name: concurrency_sharded_replication
   - name: concurrency_sharded_replication_with_balancer
   - name: concurrency_simultaneous
+  - name: concurrency_simultaneous_replication
+    distros:
+    - rhel72-zseries-build
   - name: dbtest
   - name: ese
   - name: failpoints
@@ -10720,6 +10755,7 @@ buildvariants:
   - name: concurrency_sharded_replication
   - name: concurrency_sharded_replication_with_balancer
   - name: concurrency_simultaneous
+  - name: concurrency_simultaneous_replication
   - name: dbtest
   - name: ese
   - name: failpoints
@@ -10826,6 +10862,7 @@ buildvariants:
   - name: concurrency_sharded_replication
   - name: concurrency_sharded_replication_with_balancer
   - name: concurrency_simultaneous
+  - name: concurrency_simultaneous_replication
   - name: dbtest
   - name: ese
   - name: failpoints
@@ -11191,6 +11228,7 @@ buildvariants:
   - name: concurrency_sharded_replication
   - name: concurrency_sharded_replication_with_balancer
   - name: concurrency_simultaneous
+  - name: concurrency_simultaneous_replication
   - name: dbtest
   - name: ese
   - name: failpoints
@@ -11324,6 +11362,7 @@ buildvariants:
   - name: concurrency_sharded_replication
   - name: concurrency_sharded_replication_with_balancer
   - name: concurrency_simultaneous
+  - name: concurrency_simultaneous_replication
   - name: replica_sets
   - name: replica_sets_auth
   - name: replica_sets_jscore_passthrough
@@ -11469,6 +11508,7 @@ buildvariants:
   - name: concurrency_sharded_replication
   - name: concurrency_sharded_replication_with_balancer
   - name: concurrency_simultaneous
+  - name: concurrency_simultaneous_replication
   - name: replica_sets
   - name: replica_sets_auth
   - name: replica_sets_jscore_passthrough
@@ -11619,6 +11659,7 @@ buildvariants:
   - name: concurrency_sharded_replication
   - name: concurrency_sharded_replication_with_balancer
   - name: concurrency_simultaneous
+  - name: concurrency_simultaneous_replication
   - name: replica_sets
   - name: replica_sets_auth
   - name: replica_sets_jscore_passthrough
@@ -11703,6 +11744,9 @@ buildvariants:
   - name: concurrency_sharded_with_stepdowns
   - name: concurrency_sharded_with_stepdowns_and_balancer
   - name: concurrency_simultaneous
+  - name: concurrency_simultaneous_replication
+    distros:
+    - rhel62-large
   - name: dbtest
   - name: failpoints
   - name: failpoints_auth
@@ -12006,6 +12050,9 @@ buildvariants:
   - name: concurrency_sharded_replication
   - name: concurrency_sharded_replication_with_balancer
   - name: concurrency_simultaneous
+  - name: concurrency_simultaneous_replication
+    distros:
+    - rhel62-large
   - name: dbtest
   - name: failpoints
   - name: failpoints_auth
@@ -12091,6 +12138,7 @@ buildvariants:
   - name: concurrency_sharded_replication
   - name: concurrency_sharded_replication_with_balancer
   - name: concurrency_simultaneous
+  - name: concurrency_simultaneous_replication
   - name: dbtest
   - name: failpoints
   - name: failpoints_auth
@@ -12184,6 +12232,9 @@ buildvariants:
   - name: concurrency_simultaneous
     distros:
     - rhel72-zseries-build
+  - name: concurrency_simultaneous_replication
+    distros:
+    - rhel72-zseries-build
   - name: dbtest
   - name: failpoints
   - name: failpoints_auth
@@ -12326,6 +12377,7 @@ buildvariants:
   - name: concurrency_sharded_with_stepdowns
   - name: concurrency_sharded_with_stepdowns_and_balancer
   - name: concurrency_simultaneous
+  - name: concurrency_simultaneous_replication
   - name: dbtest
   - name: disk_wiredtiger
   - name: ese
@@ -12502,6 +12554,7 @@ buildvariants:
   - name: concurrency_sharded_with_stepdowns
   - name: concurrency_sharded_with_stepdowns_and_balancer
   - name: concurrency_simultaneous
+  - name: concurrency_simultaneous_replication
   - name: dbtest
   - name: disk_wiredtiger
   - name: ese
@@ -12645,6 +12698,7 @@ buildvariants:
   - name: concurrency_sharded_replication
   - name: concurrency_sharded_replication_with_balancer
   - name: concurrency_simultaneous
+  - name: concurrency_simultaneous_replication
   - name: dbtest
   - name: disk_wiredtiger
   - name: ese
@@ -12737,4 +12791,3 @@ buildvariants:
   - name: fetch_test_lifecycle
     distros:
     - rhel62-small
-
diff --git a/jstests/concurrency/fsm_workload_helpers/snapshot_read_utils.js b/jstests/concurrency/fsm_workload_helpers/snapshot_read_utils.js
index f78f7d17b8e..7c3ea64b9cd 100644
--- a/jstests/concurrency/fsm_workload_helpers/snapshot_read_utils.js
+++ b/jstests/concurrency/fsm_workload_helpers/snapshot_read_utils.js
@@ -55,13 +55,7 @@ function doSnapshotFind(sortByAscending, collName, data, findErrorCodes) {
     const cursor = parseCursor(res);
 
     if (!cursor) {
-        const abortCmd = {
-            abortTransaction: 1,
-            txnNumber: NumberLong(data.txnNumber),
-            autocommit: false
-        };
-        res = data.sessionDb.adminCommand(abortCmd);
-        assertWorkedOrFailed(abortCmd, res, [ErrorCodes.NoSuchTransaction]);
+        abortTransaction(data.sessionDb, data.txnNumber, [ErrorCodes.NoSuchTransaction]);
         data.cursorId = 0;
     } else {
         assert(cursor.hasOwnProperty("firstBatch"), tojson(res));
@@ -105,8 +99,8 @@ function doSnapshotGetMore(collName, data, getMoreErrorCodes, commitTransactionE
 /**
  * This function can be used to share session data across threads.
  */
-function insertSessionDoc(db, collName, data, session) {
-    const sessionDoc = {"_id": "sessionDoc" + data.tid, "id": session.getSessionId().id};
+function insertSessionDoc(db, collName, tid, sessionId) {
+    const sessionDoc = {"_id": "sessionDoc" + tid, "id": sessionId};
     const res = db[collName].insert(sessionDoc);
     assert.writeOK(res);
     assert.eq(1, res.nInserted);
@@ -114,13 +108,57 @@ function insertSessionDoc(db, collName, data, session) {
 
 /**
  * This function can be used in conjunction with insertSessionDoc to kill any active sessions on
- * teardown.
+ * teardown or iteration completion.
  */
-function killSessionsFromDocs(db, collName) {
-    const sessionDocCursor = db[collName].find({"_id": {$regex: "sessionDoc*"}});
-    assert(sessionDocCursor.hasNext());
-    while (sessionDocCursor.hasNext()) {
-        const sessionDoc = sessionDocCursor.next();
-        assert.commandWorked(db.runCommand({killSessions: [{id: sessionDoc.id}]}));
+function killSessionsFromDocs(db, collName, tid) {
+    // Cleanup up all sessions, unless 'tid' is supplied.
+    let docs = {$regex: /^sessionDoc/};
+    if (tid !== undefined) {
+        docs = "sessionDoc" + tid;
     }
+    let sessionIds = db[collName].find({"_id": docs}, {_id: 0, id: 1}).toArray();
+    assert.commandWorked(db.runCommand({killSessions: sessionIds}));
 }
+
+/**
+ * Abort the transaction on the session and return result.
+ */
+function abortTransaction(db, txnNumber, errorCodes) {
+    abortCmd = {abortTransaction: 1, txnNumber: NumberLong(txnNumber), autocommit: false};
+    res = db.adminCommand(abortCmd);
+    assertWorkedOrFailed(abortCmd, res, errorCodes);
+    return res;
+}
+
+/**
+ * This function operates on the last iteration of each thread to abort any active transactions.
+ */
+var {cleanupOnLastIteration} = (function() {
+    function cleanupOnLastIteration(data, func) {
+        const abortErrorCodes = [
+            ErrorCodes.NoSuchTransaction,
+            ErrorCodes.TransactionCommitted,
+            ErrorCodes.TransactionTooOld
+        ];
+        let lastIteration = ++data.iteration >= data.iterations;
+        try {
+            func();
+        } catch (e) {
+            lastIteration = true;
+            throw e;
+        } finally {
+            if (lastIteration) {
+                // Abort the latest transactions for this session as some may have been skipped due
+                // to incrementing data.txnNumber.
+                for (let i = data.txnNumber; i >= 0; i--) {
+                    let res = abortTransaction(data.sessionDb, i, abortErrorCodes);
+                    if (res.ok === 1) {
+                        break;
+                    }
+                }
+            }
+        }
+    }
+
+    return {cleanupOnLastIteration};
+})();
diff --git a/jstests/concurrency/fsm_workloads/snapshot_read_kill_op_only.js b/jstests/concurrency/fsm_workloads/snapshot_read_kill_op_only.js
new file mode 100644
index 00000000000..c6ed3d55b46
--- /dev/null
+++ b/jstests/concurrency/fsm_workloads/snapshot_read_kill_op_only.js
@@ -0,0 +1,26 @@
+'use strict';
+
+/**
+ * Test a snapshot read spanning a find and getmore that runs concurrently with
+ * killOp and txnNumber change.
+
+ * TODO: SERVER-35567 - Delete this workload.
+
+ * @tags: [uses_transactions]
+ */
+
+load('jstests/concurrency/fsm_libs/extend_workload.js');                     // for extendWorkload
+load('jstests/concurrency/fsm_workloads/snapshot_read_kill_operations.js');  // for $config
+
+var $config = extendWorkload($config, function($config, $super) {
+
+    $config.transitions = {
+        init: {snapshotFind: 1.0},
+        snapshotFind: {incrementTxnNumber: 0.33, killOp: 0.34, snapshotGetMore: 0.33},
+        incrementTxnNumber: {snapshotGetMore: 1.0},
+        killOp: {snapshotGetMore: 1.0},
+        snapshotGetMore: {snapshotFind: 1.0}
+    };
+
+    return $config;
+});
diff --git a/jstests/concurrency/fsm_workloads/snapshot_read_kill_operations.js b/jstests/concurrency/fsm_workloads/snapshot_read_kill_operations.js
index af7a79f7852..13d0d75325c 100644
--- a/jstests/concurrency/fsm_workloads/snapshot_read_kill_operations.js
+++ b/jstests/concurrency/fsm_workloads/snapshot_read_kill_operations.js
@@ -9,30 +9,37 @@
 load('jstests/concurrency/fsm_workload_helpers/snapshot_read_utils.js');
 var $config = (function() {
     const data = {numIds: 100, batchSize: 50};
-    const threadCount = 5;
 
     const states = {
         init: function init(db, collName) {
             let session = db.getMongo().startSession({causalConsistency: false});
             // Store the session ID in the database so any unterminated transactions can be aborted
             // at teardown.
-            insertSessionDoc(db, collName, this, session);
+            insertSessionDoc(db, collName, this.tid, session.getSessionId().id);
             this.sessionDb = session.getDatabase(db.getName());
             this.txnNumber = 0;
             this.stmtId = 0;
+            this.iteration = 1;
         },
 
         snapshotFind: function snapshotFind(db, collName) {
             const sortByAscending = false;
-            doSnapshotFind(sortByAscending, collName, this, [ErrorCodes.NoSuchTransaction]);
+            doSnapshotFind(sortByAscending,
+                           collName,
+                           this,
+                           [ErrorCodes.NoSuchTransaction, ErrorCodes.LockTimeout]);
         },
 
         snapshotGetMore: function snapshotGetMore(db, collName) {
-            doSnapshotGetMore(
-                collName,
-                this,
-                [ErrorCodes.NoSuchTransaction, ErrorCodes.CursorNotFound, ErrorCodes.Interrupted],
-                [ErrorCodes.NoSuchTransaction]);
+            doSnapshotGetMore(collName,
+                              this,
+                              [
+                                ErrorCodes.NoSuchTransaction,
+                                ErrorCodes.CursorNotFound,
+                                ErrorCodes.Interrupted,
+                                ErrorCodes.LockTimeout
+                              ],
+                              [ErrorCodes.NoSuchTransaction]);
         },
 
         incrementTxnNumber: function incrementTxnNumber(db, collName) {
@@ -41,7 +48,7 @@ var $config = (function() {
 
         killSessions: function killSessions(db, collName) {
             // Kill a random active session.
-            const idToKill = "sessionDoc" + Math.floor(Math.random() * threadCount);
+            const idToKill = "sessionDoc" + Math.floor(Math.random() * this.threadCount);
             const sessionDocToKill = db[collName].find({"_id": idToKill});
             assert.commandWorked(
                 this.sessionDb.runCommand({killSessions: [{id: sessionDocToKill.id}]}));
@@ -63,9 +70,18 @@ var $config = (function() {
             const killCursorCmd = {killCursors: collName, cursors: [this.cursorId]};
             const res = this.sessionDb.runCommand(killCursorCmd);
             assertWorkedOrFailed(killCursorCmd, res, [ErrorCodes.CursorNotFound]);
-        }
+        },
+
     };
 
+    // Wrap each state in a cleanupOnLastIteration() invocation.
+    for (let stateName of Object.keys(states)) {
+        const stateFn = states[stateName];
+        states[stateName] = function(db, collName) {
+            cleanupOnLastIteration(this, () => stateFn.apply(this, arguments));
+        };
+    }
+
     const transitions = {
         init: {snapshotFind: 1.0},
         snapshotFind: {
@@ -85,7 +101,7 @@ var $config = (function() {
     function setup(db, collName, cluster) {
         assertWhenOwnColl.commandWorked(db.runCommand({create: collName}));
         for (let i = 0; i < this.numIds; ++i) {
-            const res = db[collName].insert({_id: i, value: this.valueToBeIncremented});
+            const res = db[collName].insert({_id: i, value: i});
             assert.writeOK(res);
             assert.eq(1, res.nInserted);
         }
@@ -97,7 +113,7 @@ var $config = (function() {
     }
 
     return {
-        threadCount: threadCount,
+        threadCount: 5,
         iterations: 10,
         startState: 'init',
         states: states,