SERVER-22178 Fix tests on ephemeralForTest storage engine.

Also update comments on other concurrency tests.

12 files changed, 135 insertions, 77 deletions

diff --git a/jstests/concurrency/fsm_workload_helpers/server_types.js b/jstests/concurrency/fsm_workload_helpers/server_types.js
index 3a5a6930bce..12d4933a052 100644
--- a/jstests/concurrency/fsm_workload_helpers/server_types.js
+++ b/jstests/concurrency/fsm_workload_helpers/server_types.js
@@ -43,6 +43,15 @@ function isMMAPv1(db) {
 }
 
 /**
+ * Returns true if an update can cause the RecordId of a document to change.
+ */
+function recordIdCanChangeOnUpdate(db) {
+    // A RecordId on MMAPv1 is just its location on disk, which can change if the document grows and
+    // needs to be moved.
+    return isMMAPv1(db);
+}
+
+/**
  * Returns true if the current storage engine is wiredTiger, and false otherwise.
  */
 function isWiredTiger(db) {
@@ -56,3 +65,12 @@ function isEphemeral(db) {
     var engine = getStorageEngineName(db);
     return (engine === 'inMemory') || (engine === 'ephemeralForTest');
 }
+
+/**
+ * Returns true if the current storage engine supports document-level concurrency, and false
+ * otherwise.
+ */
+function supportsDocumentLevelConcurrency(db) {
+    var engine = getStorageEngineName(db);
+    return ['wiredTiger', 'rocksdb', 'inMemory'].indexOf(engine) !== -1;
+}
diff --git a/jstests/concurrency/fsm_workloads/findAndModify_inc.js b/jstests/concurrency/fsm_workloads/findAndModify_inc.js
index cf0a50284ff..5c5d55b55b8 100644
--- a/jstests/concurrency/fsm_workloads/findAndModify_inc.js
+++ b/jstests/concurrency/fsm_workloads/findAndModify_inc.js
@@ -10,7 +10,9 @@
  *
  * This workload was designed to reproduce SERVER-15892.
  */
-load('jstests/concurrency/fsm_workload_helpers/server_types.js');  // for isMongod and isMMAPv1
+
+// For isMongod and supportsDocumentLevelConcurrency.
+load('jstests/concurrency/fsm_workload_helpers/server_types.js');
 
 var $config = (function() {
 
@@ -34,11 +36,11 @@ var $config = (function() {
             // If the document was invalidated during a yield, then we wouldn't have modified it.
             // The "findAndModify" command returns a null value in this case. See SERVER-22002 for
             // more details.
-            if (isMongod(db) && !isMMAPv1(db)) {
-                // For storage engines other than MMAPv1, if the document is modified by another
-                // thread during a yield, then the operation is retried internally. We never expect
-                // to see a null value returned by the "findAndModify" command when it is known that
-                // a matching document exists in the collection.
+            if (isMongod(db) && supportsDocumentLevelConcurrency(db)) {
+                // For storage engines that support document-level concurrency, if the document is
+                // modified by another thread during a yield, then the operation is retried
+                // internally. We never expect to see a null value returned by the "findAndModify"
+                // command when it is known that a matching document exists in the collection.
                 assertWhenOwnColl(res.value !== null, 'query spec should have matched a document');
             }
 
diff --git a/jstests/concurrency/fsm_workloads/findAndModify_mixed_queue_unindexed.js b/jstests/concurrency/fsm_workloads/findAndModify_mixed_queue_unindexed.js
index 793f531c9b9..11ac81d63fd 100644
--- a/jstests/concurrency/fsm_workloads/findAndModify_mixed_queue_unindexed.js
+++ b/jstests/concurrency/fsm_workloads/findAndModify_mixed_queue_unindexed.js
@@ -17,7 +17,9 @@
  */
 load('jstests/concurrency/fsm_libs/extend_workload.js');                  // for extendWorkload
 load('jstests/concurrency/fsm_workloads/findAndModify_remove_queue.js');  // for $config
-load('jstests/concurrency/fsm_workload_helpers/server_types.js');         // for isMMAPv1
+
+// For isMongod and supportsDocumentLevelConcurrency.
+load('jstests/concurrency/fsm_workload_helpers/server_types.js');
 
 var $config = extendWorkload(
     $config,
@@ -45,10 +47,11 @@ var $config = extendWorkload(
             assertAlways.commandWorked(res);
 
             var doc = res.value;
-            if (isMongod(db) && !isMMAPv1(db)) {
-                // MMAPv1 does not automatically retry if there was a conflict, so it is expected
-                // that it may return null in the case of a conflict. All other storage engines
-                // should automatically retry the operation, and thus should never return null.
+            if (isMongod(db) && supportsDocumentLevelConcurrency(db)) {
+                // Storage engines which do not support document-level concurrency will not
+                // automatically retry if there was a conflict, so it is expected that it may return
+                // null in the case of a conflict. All other storage engines should automatically
+                // retry the operation, and thus should never return null.
                 assertWhenOwnColl.neq(
                     doc, null, 'findAndModify should have found a matching document');
             }
diff --git a/jstests/concurrency/fsm_workloads/findAndModify_remove_queue.js b/jstests/concurrency/fsm_workloads/findAndModify_remove_queue.js
index 037eaf47250..3a330529e0c 100644
--- a/jstests/concurrency/fsm_workloads/findAndModify_remove_queue.js
+++ b/jstests/concurrency/fsm_workloads/findAndModify_remove_queue.js
@@ -10,7 +10,9 @@
  *
  * This workload was designed to reproduce SERVER-18304.
  */
-load('jstests/concurrency/fsm_workload_helpers/server_types.js');  // for isMongod and isMMAPv1
+
+// For isMongod and supportsDocumentLevelConcurrency.
+load('jstests/concurrency/fsm_workload_helpers/server_types.js');
 
 var $config = (function() {
 
@@ -71,10 +73,11 @@ var $config = (function() {
             assertAlways.commandWorked(res);
 
             var doc = res.value;
-            if (isMongod(db) && !isMMAPv1(db)) {
-                // MMAPv1 does not automatically retry if there was a conflict, so it is expected
-                // that it may return null in the case of a conflict. All other storage engines
-                // should automatically retry the operation, and thus should never return null.
+            if (isMongod(db) && supportsDocumentLevelConcurrency(db)) {
+                // Storage engines which do not support document-level concurrency will not
+                // automatically retry if there was a conflict, so it is expected that it may return
+                // null in the case of a conflict. All other storage engines should automatically
+                // retry the operation, and thus should never return null.
                 assertWhenOwnColl.neq(
                     doc, null, 'findAndModify should have found and removed a matching document');
             }
@@ -121,11 +124,10 @@ var $config = (function() {
         var ownedDB = db.getSiblingDB(db.getName() + this.uniqueDBName);
 
         if (this.opName === 'removed') {
-            if (isMongod(db) && !isMMAPv1(db)) {
-                // On storage engines other than MMAPv1, each findAndModify should remove exactly
-                // one document. This is not true on MMAPv1 since it will not automatically retry a
-                // findAndModify when there is a conflict, indicating there were no matches instead.
-                // Since this.numDocs == this.iterations * this.threadCount, there should not be any
+            if (isMongod(db) && supportsDocumentLevelConcurrency(db)) {
+                // On storage engines which support document-level concurrency, each findAndModify
+                // should be internally retried until it removes exactly one document. Since
+                // this.numDocs == this.iterations * this.threadCount, there should not be any
                 // documents remaining.
                 assertWhenOwnColl.eq(db[collName].find().itcount(),
                                      0,
diff --git a/jstests/concurrency/fsm_workloads/findAndModify_update_queue.js b/jstests/concurrency/fsm_workloads/findAndModify_update_queue.js
index 9637de739c5..104b299b317 100644
--- a/jstests/concurrency/fsm_workloads/findAndModify_update_queue.js
+++ b/jstests/concurrency/fsm_workloads/findAndModify_update_queue.js
@@ -13,7 +13,9 @@
  */
 load('jstests/concurrency/fsm_libs/extend_workload.js');                  // for extendWorkload
 load('jstests/concurrency/fsm_workloads/findAndModify_remove_queue.js');  // for $config
-load('jstests/concurrency/fsm_workload_helpers/server_types.js');  // for isMongod and isMMAPv1
+
+// For isMongod and supportsDocumentLevelConcurrency.
+load('jstests/concurrency/fsm_workload_helpers/server_types.js');
 
 var $config = extendWorkload(
     $config,
@@ -50,11 +52,11 @@ var $config = extendWorkload(
                 assertAlways.commandWorked(res);
 
                 var doc = res.value;
-                if (isMongod(db) && !isMMAPv1(db)) {
-                    // MMAPv1 does not automatically retry if there was a conflict, so it is
-                    // expected
-                    // that it may return null in the case of a conflict. All other storage engines
-                    // should automatically retry the operation, and thus should never return null.
+                if (isMongod(db) && supportsDocumentLevelConcurrency(db)) {
+                    // Storage engines which do not support document-level concurrency will not
+                    // automatically retry if there was a conflict, so it is expected that it may
+                    // return null in the case of a conflict. All other storage engines should
+                    // automatically retry the operation, and thus should never return null.
                     assertWhenOwnColl.neq(
                         doc,
                         null,
diff --git a/jstests/concurrency/fsm_workloads/update_array.js b/jstests/concurrency/fsm_workloads/update_array.js
index e275d290911..192626c2430 100644
--- a/jstests/concurrency/fsm_workloads/update_array.js
+++ b/jstests/concurrency/fsm_workloads/update_array.js
@@ -9,7 +9,9 @@
  * though other threads in the workload may be modifying the array between the
  * update and the find, because thread ids are unique.
  */
-load('jstests/concurrency/fsm_workload_helpers/server_types.js');  // for isMongod and isMMAPv1
+
+// For isMongod and supportsDocumentLevelConcurrency.
+load('jstests/concurrency/fsm_workload_helpers/server_types.js');
 
 var $config = (function() {
 
@@ -21,14 +23,19 @@ var $config = (function() {
         function assertUpdateSuccess(db, res, nModifiedPossibilities) {
             assertAlways.eq(0, res.nUpserted, tojson(res));
 
-            if (isMongod(db) && !isMMAPv1(db)) {
+            if (isMongod(db) && supportsDocumentLevelConcurrency(db)) {
+                // Storage engines which support document-level concurrency will automatically retry
+                // any operations when there are conflicts, so the update should have succeeded if
+                // a matching document existed.
                 assertWhenOwnColl.contains(1, nModifiedPossibilities, tojson(res));
                 if (db.getMongo().writeMode() === 'commands') {
                     assertWhenOwnColl.contains(res.nModified, nModifiedPossibilities, tojson(res));
                 }
             } else {
-                // Zero matches are possible for MMAP v1 because the update will skip a document
-                // that was invalidated during a yield.
+                // On storage engines that do not support document-level concurrency, it is possible
+                // that the update will not update all matching documents. This can happen if
+                // another thread updated a target document during a yield, triggering an
+                // invalidation.
                 assertWhenOwnColl.contains(res.nMatched, [0, 1], tojson(res));
                 if (db.getMongo().writeMode() === 'commands') {
                     assertWhenOwnColl.contains(res.nModified, [0, 1], tojson(res));
diff --git a/jstests/concurrency/fsm_workloads/update_inc.js b/jstests/concurrency/fsm_workloads/update_inc.js
index bd4c832e96f..f16d841d3bb 100644
--- a/jstests/concurrency/fsm_workloads/update_inc.js
+++ b/jstests/concurrency/fsm_workloads/update_inc.js
@@ -8,7 +8,9 @@
  * field. Asserts that the field has the correct value based on the number
  * of increments performed.
  */
-load('jstests/concurrency/fsm_workload_helpers/server_types.js');  // for isMongod and isMMAPv1
+
+// For isMongod and supportsDocumentLevelConcurrency.
+load('jstests/concurrency/fsm_workload_helpers/server_types.js');
 
 var $config = (function() {
 
@@ -33,16 +35,18 @@ var $config = (function() {
             var res = db[collName].update({_id: this.id}, updateDoc);
             assertAlways.eq(0, res.nUpserted, tojson(res));
 
-            if (isMongod(db) && !isMMAPv1(db)) {
-                // For non-mmap storage engines we can have a strong assertion that exactly one doc
-                // will be modified.
+            if (isMongod(db) && supportsDocumentLevelConcurrency(db)) {
+                // Storage engines which support document-level concurrency will automatically retry
+                // any operations when there are conflicts, so we should always see a matching
+                // document.
                 assertWhenOwnColl.eq(res.nMatched, 1, tojson(res));
                 if (db.getMongo().writeMode() === 'commands') {
                     assertWhenOwnColl.eq(res.nModified, 1, tojson(res));
                 }
             } else {
-                // Zero matches are possible for MMAP v1 because the update will skip a document
-                // that was invalidated during a yield.
+                // On storage engines that do not support document-level concurrency, it is possible
+                // that the query will not find the document. This can happen if another thread
+                // updated the target document during a yield, triggering an invalidation.
                 assertWhenOwnColl.contains(res.nMatched, [0, 1], tojson(res));
                 if (db.getMongo().writeMode() === 'commands') {
                     assertWhenOwnColl.contains(res.nModified, [0, 1], tojson(res));
diff --git a/jstests/concurrency/fsm_workloads/update_multifield.js b/jstests/concurrency/fsm_workloads/update_multifield.js
index af520797ac8..1f62472f754 100644
--- a/jstests/concurrency/fsm_workloads/update_multifield.js
+++ b/jstests/concurrency/fsm_workloads/update_multifield.js
@@ -7,7 +7,8 @@
  * The collection has an index for each field, and a compound index for all fields.
  */
 
-load('jstests/concurrency/fsm_workload_helpers/server_types.js');  // for isMongod and isMMAPv1
+// For isMongod and supportsDocumentLevelConcurrency.
+load('jstests/concurrency/fsm_workload_helpers/server_types.js');
 
 var $config = (function() {
 
@@ -91,16 +92,19 @@ var $config = (function() {
             assertResult: function(res, db, collName, query) {
                 assertAlways.eq(0, res.nUpserted, tojson(res));
 
-                if (isMongod(db) && !isMMAPv1(db)) {
-                    // For non-mmap storage engines we can have a strong assertion that exactly one
-                    // doc will be modified.
+                if (isMongod(db) && supportsDocumentLevelConcurrency(db)) {
+                    // Storage engines which support document-level concurrency will automatically
+                    // retry any operations when there are conflicts, so we should always see a
+                    // matching document.
                     assertWhenOwnColl.eq(res.nMatched, 1, tojson(res));
                     if (db.getMongo().writeMode() === 'commands') {
                         assertWhenOwnColl.eq(res.nModified, 1, tojson(res));
                     }
                 } else {
-                    // Zero matches are possible for MMAP v1 because the update will skip a document
-                    // that was invalidated during a yield.
+                    // On storage engines that do not support document-level concurrency, it is
+                    // possible that the query will not find the document. This can happen if
+                    // another thread updated the target document during a yield, triggering an
+                    // invalidation.
                     assertWhenOwnColl.contains(res.nMatched, [0, 1], tojson(res));
                     if (db.getMongo().writeMode() === 'commands') {
                         assertWhenOwnColl.contains(res.nModified, [0, 1], tojson(res));
diff --git a/jstests/concurrency/fsm_workloads/update_multifield_multiupdate.js b/jstests/concurrency/fsm_workloads/update_multifield_multiupdate.js
index 799d104a323..2cc975085ca 100644
--- a/jstests/concurrency/fsm_workloads/update_multifield_multiupdate.js
+++ b/jstests/concurrency/fsm_workloads/update_multifield_multiupdate.js
@@ -6,9 +6,11 @@
  * Does updates that affect multiple fields on multiple documents.
  * The collection has an index for each field, and a multikey index for all fields.
  */
-load('jstests/concurrency/fsm_libs/extend_workload.js');           // for extendWorkload
-load('jstests/concurrency/fsm_workloads/update_multifield.js');    // for $config
-load('jstests/concurrency/fsm_workload_helpers/server_types.js');  // for isMongod and isMMAPv1
+load('jstests/concurrency/fsm_libs/extend_workload.js');         // for extendWorkload
+load('jstests/concurrency/fsm_workloads/update_multifield.js');  // for $config
+
+// For isMongod and recordIdCanChangeOnUpdate.
+load('jstests/concurrency/fsm_workload_helpers/server_types.js');
 
 var $config =
     extendWorkload($config,
@@ -20,21 +22,17 @@ var $config =
                            assertAlways.eq(0, res.nUpserted, tojson(res));
 
                            if (isMongod(db)) {
-                               if (isMMAPv1(db)) {
-                                   // If an update triggers a document to move forward, then
-                                   // that document can be matched multiple times. If an update
-                                   // triggers a document to move backwards, then that document
-                                   // can be missed by other threads.
-                                   assertAlways.gte(res.nMatched, 0, tojson(res));
-                               } else {  // non-mmapv1 storage engine
-                                   // TODO: Can we assert exact equality with WiredTiger?
-                                   //       What about for other storage engines?
+                               if (!recordIdCanChangeOnUpdate(db)) {
+                                   // If a document's RecordId cannot change, then we should not
+                                   // have updated any document more than once, since the update
+                                   // stage internally de-duplicates based on RecordId.
                                    assertWhenOwnColl.lte(this.numDocs, res.nMatched, tojson(res));
+                               } else {
+                                   // If RecordIds can change, then there are no guarantees on how
+                                   // many documents were updated.
+                                   assertAlways.gte(res.nMatched, 0, tojson(res));
                                }
                            } else {  // mongos
-                               // In a mixed cluster, it is unknown what underlying storage engine
-                               // the update operations will be executed against. Thus, we can only
-                               // make the weakest of all assertions above.
                                assertAlways.gte(res.nMatched, 0, tojson(res));
                            }
 
diff --git a/jstests/concurrency/fsm_workloads/update_ordered_bulk_inc.js b/jstests/concurrency/fsm_workloads/update_ordered_bulk_inc.js
index a799d5dfe43..0d6b4651a5d 100644
--- a/jstests/concurrency/fsm_workloads/update_ordered_bulk_inc.js
+++ b/jstests/concurrency/fsm_workloads/update_ordered_bulk_inc.js
@@ -10,7 +10,9 @@
  *
  * Uses an ordered, bulk operation to perform the updates.
  */
-load('jstests/concurrency/fsm_workload_helpers/server_types.js');  // for isMMAPv1 and isMongod
+
+// For isMongod, recordIdCanChangeOnUpdate, and supportsDocumentLevelConcurrency.
+load('jstests/concurrency/fsm_workload_helpers/server_types.js');
 
 var $config = (function() {
 
@@ -41,15 +43,22 @@ var $config = (function() {
         find: function find(db, collName) {
             var docs = db[collName].find().toArray();
 
-            // In MMAP v1, some documents may appear twice due to moves
-            if (isMongod(db) && !isMMAPv1(db)) {
+            if (isMongod(db) && !recordIdCanChangeOnUpdate(db)) {
+                // If the RecordId cannot change and we aren't updating any fields in any indexes,
+                // we should always see all matching documents, since they would not be able to move
+                // ahead or behind our collection scan or index scan.
                 assertWhenOwnColl.eq(this.docCount, docs.length);
+            } else {
+                // On MMAPv1, we may see more than 'this.docCount' documents during our find. This
+                // can happen if an update causes the document to grow such that it is moved in
+                // front of an index or collection scan which has already returned it.
+                assertWhenOwnColl.gte(docs.length, this.docCount);
             }
-            assertWhenOwnColl.gte(docs.length, this.docCount);
 
-            // It is possible that a document was not incremented in MMAP v1 because
-            // updates skip documents that were invalidated during yielding
-            if (isMongod(db) && !isMMAPv1(db)) {
+            if (isMongod(db) && supportsDocumentLevelConcurrency(db)) {
+                // Storage engines which support document-level concurrency will automatically retry
+                // any operations when there are conflicts, so we should have updated all matching
+                // documents.
                 docs.forEach(function(doc) {
                     assertWhenOwnColl.eq(this.count, doc[this.fieldName]);
                 }, this);
diff --git a/jstests/concurrency/fsm_workloads/update_replace.js b/jstests/concurrency/fsm_workloads/update_replace.js
index b9d4cf75380..f7edb05126d 100644
--- a/jstests/concurrency/fsm_workloads/update_replace.js
+++ b/jstests/concurrency/fsm_workloads/update_replace.js
@@ -6,7 +6,9 @@
  * Does updates that replace an entire document.
  * The collection has indexes on some but not all fields.
  */
-load('jstests/concurrency/fsm_workload_helpers/server_types.js');  // for isMongod and isMMAPv1
+
+// For isMongod and supportsDocumentLevelConcurrency.
+load('jstests/concurrency/fsm_workload_helpers/server_types.js');
 
 var $config = (function() {
 
@@ -14,13 +16,15 @@ var $config = (function() {
     function assertResult(db, res) {
         assertAlways.eq(0, res.nUpserted, tojson(res));
 
-        if (isMongod(db) && !isMMAPv1(db)) {
-            // For non-MMAPv1 storage engines we can make a stong assertion that exactly one
-            // document was matched.
+        if (isMongod(db) && supportsDocumentLevelConcurrency(db)) {
+            // Storage engines which support document-level concurrency will automatically retry
+            // any operations when there are conflicts, so we should always see a matching
+            // document.
             assertWhenOwnColl.eq(res.nMatched, 1, tojson(res));
         } else {
-            // It's possible to match zero documents with MMAPv1 because the update can skip a
-            // document that was invalidated during a yield.
+            // On storage engines that do not support document-level concurrency, it is possible
+            // that the query will not find the document. This can happen if another thread
+            // updated the target document during a yield, triggering an invalidation.
             assertWhenOwnColl.contains(res.nMatched, [0, 1], tojson(res));
         }
 
diff --git a/jstests/concurrency/fsm_workloads/update_simple.js b/jstests/concurrency/fsm_workloads/update_simple.js
index ae694ace309..85f877e6090 100644
--- a/jstests/concurrency/fsm_workloads/update_simple.js
+++ b/jstests/concurrency/fsm_workloads/update_simple.js
@@ -8,7 +8,9 @@
  *  - whether to $set or $unset its field
  *  - what value to $set the field to
  */
-load('jstests/concurrency/fsm_workload_helpers/server_types.js');  // for isMongod and isMMAPv1
+
+// For isMongod and supportsDocumentLevelConcurrency.
+load('jstests/concurrency/fsm_workload_helpers/server_types.js');
 
 var $config = (function() {
 
@@ -55,13 +57,16 @@ var $config = (function() {
             assertResult: function assertResult(db, res) {
                 assertAlways.eq(0, res.nUpserted, tojson(res));
 
-                if (isMongod(db) && !isMMAPv1(db)) {
-                    // For non-mmap storage engines we can have a strong assertion that exactly one
-                    // doc will be modified.
+                if (isMongod(db) && supportsDocumentLevelConcurrency(db)) {
+                    // Storage engines which support document-level concurrency will automatically
+                    // retry any operations when there are conflicts, so we should always see a
+                    // matching document.
                     assertWhenOwnColl.eq(res.nMatched, 1, tojson(res));
                 } else {
-                    // Zero matches are possible for MMAP v1 because the update will skip a document
-                    // that was invalidated during a yield.
+                    // On storage engines that do not support document-level concurrency, it is
+                    // possible that the query will not find the document. This can happen if
+                    // another thread updated the target document during a yield, triggering an
+                    // invalidation.
                     assertWhenOwnColl.contains(res.nMatched, [0, 1], tojson(res));
                 }