SERVER-56377: Add fsm test for `storeFindAndModifyImagesInSideCollection`.

(cherry picked from commit c9658dab44272cdc6e8cb949b81f8fae1288b4e8)

2 files changed, 183 insertions, 0 deletions

diff --git a/etc/backports_required_for_multiversion_tests.yml b/etc/backports_required_for_multiversion_tests.yml
index 6cce49b41b2..99afd407b25 100644
--- a/etc/backports_required_for_multiversion_tests.yml
+++ b/etc/backports_required_for_multiversion_tests.yml
@@ -137,6 +137,8 @@ all:
     test_file: jstests/replsets/invalidate_images_when_minvalid.js
   - ticket: SERVER-56468
     test_file: jstests/core/neq_null_correctly_cache.js
+  - ticket: SERVER-56377
+    test_file: jstests/concurrency/fsm_workloads/findAndModify_flip_location.js
 
 suites:
 
diff --git a/jstests/concurrency/fsm_workloads/findAndModify_flip_location.js b/jstests/concurrency/fsm_workloads/findAndModify_flip_location.js
new file mode 100644
index 00000000000..08e76d81a56
--- /dev/null
+++ b/jstests/concurrency/fsm_workloads/findAndModify_flip_location.js
@@ -0,0 +1,181 @@
+'use strict';
+
+/**
+ * Each thread uses its own LSID and performs `findAndModify`s with retries on documents while the
+ * `storeFindAndModifyImagesInSideCollection` server parameter gets flipped.
+ *
+ * @tags: [requires_replication, requires_non_retryable_commands, uses_transactions];
+ */
+var $config = (function() {
+    var data = {
+        numDocs: 100,
+    };
+
+    var states = (function() {
+        function init(db, collName) {
+            this._lastTxnId = 0;
+            this._lsid = UUID();
+        }
+
+        function findAndModifyUpsert(db, collName) {
+            // `auto_retry_transactions` is not compatible with explicitly testing retryable writes.
+            // This avoids issues regarding the multi_stmt tasks.
+            fsm.forceRunningOutsideTransaction(this);
+
+            this._lastTxnId += 1;
+            this._lastCmd = {
+                findandmodify: collName,
+                lsid: {id: this._lsid},
+                txnNumber: NumberLong(this._lastTxnId),
+                stmtId: NumberInt(1),
+                query: {_id: Math.round(Math.random() * this.numDocs)},
+                new: Math.random() > 0.5,
+                upsert: true,
+                update: {$inc: {counter: 1}},
+            };
+            // The lambda passed into 'assert.soon' does not have access to 'this'.
+            let data = {"lastCmd": this._lastCmd};
+            assert.soon(function() {
+                try {
+                    data.lastResponse = assert.commandWorked(db.runCommand(data.lastCmd));
+                    return true;
+                } catch (e) {
+                    if (e.code === ErrorCodes.DuplicateKey) {
+                        // It is possible that two threads race to upsert the same '_id' into the
+                        // same collection, a scenario described in SERVER-47212. In this case, we
+                        // retry the upsert.
+                        print('Encountered DuplicateKey error. Retrying upsert:' +
+                              tojson(data.lastCmd));
+                        return false;
+                    }
+                    throw e;
+                }
+            });
+            this._lastResponse = data.lastResponse;
+        }
+
+        function findAndModifyUpdate(db, collName) {
+            // `auto_retry_transactions` is not compatible with explicitly testing retryable writes.
+            // This avoids issues regarding the multi_stmt tasks.
+            fsm.forceRunningOutsideTransaction(this);
+
+            this._lastTxnId += 1;
+            this._lastCmd = {
+                findandmodify: collName,
+                lsid: {id: this._lsid},
+                txnNumber: NumberLong(this._lastTxnId),
+                stmtId: NumberInt(1),
+                query: {_id: Math.round(Math.random() * this.numDocs)},
+                new: Math.random() > 0.5,
+                upsert: false,
+                update: {$inc: {counter: 1}},
+            };
+            this._lastResponse = assert.commandWorked(db.runCommand(this._lastCmd));
+        }
+
+        function findAndModifyDelete(db, collName) {
+            // `auto_retry_transactions` is not compatible with explicitly testing retryable writes.
+            // This avoids issues regarding the multi_stmt tasks.
+            fsm.forceRunningOutsideTransaction(this);
+
+            this._lastTxnId += 1;
+            this._lastCmd = {
+                findandmodify: collName,
+                lsid: {id: this._lsid},
+                txnNumber: NumberLong(this._lastTxnId),
+                stmtId: NumberInt(1),
+                query: {_id: Math.round(Math.random() * this.numDocs)},
+                // Deletes may not ask for the postImage
+                new: false,
+                remove: true,
+            };
+            this._lastResponse = assert.commandWorked(db.runCommand(this._lastCmd));
+        }
+
+        function findAndModifyRetry(db, collName) {
+            // `auto_retry_transactions` is not compatible with explicitly testing retryable writes.
+            // This avoids issues regarding the multi_stmt tasks.
+            fsm.forceRunningOutsideTransaction(this);
+
+            assert(this._lastCmd);
+            assert(this._lastResponse);
+
+            let response = assert.commandWorked(db.runCommand(this._lastCmd));
+            let debugMsg = {
+                "TID": this.tid,
+                "LastCmd": this._lastCmd,
+                "LastResponse": this._lastResponse,
+                "Response": response
+            };
+            assert.eq(this._lastResponse.hasOwnProperty("lastErrorObject"),
+                      response.hasOwnProperty("lastErrorObject"),
+                      debugMsg);
+            if (response.hasOwnProperty("lastErrorObject") &&
+                // If the original command affected `n=1` document, all retries must return
+                // identical results. If an original command receives `n=0`, then a retry may find a
+                // match and return `n=1`. Only compare `lastErrorObject` and `value` when retries
+                // must be identical.
+                this._lastResponse["lastErrorObject"].n === 1) {
+                assert.eq(
+                    this._lastResponse["lastErrorObject"], response["lastErrorObject"], debugMsg);
+            }
+            assert.eq(this._lastResponse.hasOwnProperty("value"),
+                      response.hasOwnProperty("value"),
+                      debugMsg);
+            if (response.hasOwnProperty("value") && this._lastResponse["lastErrorObject"].n === 1) {
+                assert.eq(this._lastResponse["value"], response["value"], debugMsg);
+            }
+
+            // Have all workers participate in creating some chaos.
+            assert.commandWorked(db.adminCommand({
+                setParameter: 1,
+                storeFindAndModifyImagesInSideCollection: Math.random() > 0.5,
+            }));
+        }
+
+        return {
+            init: init,
+            findAndModifyUpsert: findAndModifyUpsert,
+            findAndModifyUpdate: findAndModifyUpdate,
+            findAndModifyDelete: findAndModifyDelete,
+            findAndModifyRetry: findAndModifyRetry
+        };
+    })();
+
+    var transitions = {
+        init: {findAndModifyUpsert: 1.0},
+        findAndModifyUpsert: {
+            findAndModifyRetry: 3.0,
+            findAndModifyUpsert: 1.0,
+            findAndModifyUpdate: 1.0,
+            findAndModifyDelete: 1.0
+        },
+        findAndModifyUpdate: {
+            findAndModifyRetry: 3.0,
+            findAndModifyUpsert: 1.0,
+            findAndModifyUpdate: 1.0,
+            findAndModifyDelete: 1.0
+        },
+        findAndModifyDelete: {
+            findAndModifyRetry: 3.0,
+            findAndModifyUpsert: 1.0,
+            findAndModifyUpdate: 1.0,
+            findAndModifyDelete: 1.0
+        },
+        findAndModifyRetry: {
+            findAndModifyRetry: 1.0,
+            findAndModifyUpsert: 1.0,
+            findAndModifyUpdate: 1.0,
+            findAndModifyDelete: 1.0
+        },
+    };
+
+    return {
+        threadCount: 10,
+        iterations: 100,
+        data: data,
+        states: states,
+        transitions: transitions,
+        setup: function() {},
+    };
+})();