SERVER-71203 Test the resilience of the new movePrimary

2 files changed, 241 insertions, 6 deletions

diff --git a/jstests/concurrency/fsm_workloads/move_primary_with_crud.js b/jstests/concurrency/fsm_workloads/move_primary_with_crud.js
new file mode 100644
index 00000000000..36fceaba3fa
--- /dev/null
+++ b/jstests/concurrency/fsm_workloads/move_primary_with_crud.js
@@ -0,0 +1,212 @@
+'use strict';
+
+/**
+ * Randomly performs a series of CRUD and movePrimary operations on unsharded collections, checking
+ * for data consistency as a consequence of these operations.
+ *
+ * @tags: [
+ *   requires_sharding,
+ *   # TODO (SERVER-71308): As soon as the feature flag is enabled, replace it with requires_fcv_70.
+ *   featureFlagResilientMovePrimary
+ *  ]
+ */
+
+const $config = (function() {
+    const kCollNamePrefix = 'unsharded_coll_';
+    const kInitialCollSize = 100;
+    const kBatchSizeForDocsLookup = kInitialCollSize * 2;
+
+    /**
+     * Utility function that asserts that the specified command is executed successfully. However,
+     * if the error is in `retryableErrorCodes`, then the command is retried.
+     */
+    const assertCommandWorked = function(cmd, retryableErrorCodes) {
+        if (!Array.isArray(retryableErrorCodes)) {
+            retryableErrorCodes = [retryableErrorCodes];
+        }
+
+        let res = undefined;
+        assertAlways.soon(() => {
+            try {
+                res = cmd();
+                return true;
+            } catch (err) {
+                if (err instanceof BulkWriteError && err.hasWriteErrors()) {
+                    for (let writeErr of err.getWriteErrors()) {
+                        if (retryableErrorCodes.includes(writeErr.code)) {
+                            return false;
+                        } else {
+                            throw err;
+                        }
+                    }
+                    return true;
+                } else if (retryableErrorCodes.includes(err.code)) {
+                    return false;
+                }
+                throw err;
+            }
+        });
+        return res;
+    };
+
+    const data = {
+        // In-memory copy of the collection data. Every CRUD operation on the persisted collection
+        // is reflected on this object. The collection consistency check is performed by comparing
+        // its data with those managed by this copy.
+        collMirror: {},
+
+        // ID of the last document inserted into the collection. It's used as a generator of unique
+        // IDs for new documents to insert.
+        lastId: undefined,
+
+        getRandomDoc: function() {
+            const keys = Object.keys(this.collMirror);
+            return this.collMirror[keys[Random.randInt(keys.length)]];
+        }
+    };
+
+    const states = {
+        init: function(db, collName, connCache) {
+            // Insert an initial amount of documents into the collection, with a progressive _id and
+            // the update counter set to zero.
+
+            this.collName = `${kCollNamePrefix}${this.tid}`;
+            let coll = db[this.collName];
+            jsTestLog(`Initializing data: coll=${coll}`);
+
+            for (let i = 0; i < kInitialCollSize; ++i) {
+                this.collMirror[i] = {_id: i, updateCount: 0};
+            }
+            this.lastId = kInitialCollSize - 1;
+
+            // Session with retryable writes is required to recover from a primary node step-down
+            // event during bulk insert processing.
+            this.session = db.getMongo().startSession({retryWrites: true});
+            let sessionColl = this.session.getDatabase(db.getName()).getCollection(this.collName);
+
+            assertCommandWorked(() => {
+                let bulkOp = sessionColl.initializeUnorderedBulkOp();
+                for (let i = 0; i < kInitialCollSize; ++i) {
+                    bulkOp.insert(
+                        {_id: i, updateCount: 0},
+                    );
+                }
+                bulkOp.execute();
+            }, ErrorCodes.MovePrimaryInProgress);
+        },
+        insert: function(db, collName, connCache) {
+            // Insert a document into the collection, with an _id greater than all those already
+            // present (last + 1) and the update counter set to zero.
+
+            let coll = db[this.collName];
+
+            const newId = this.lastId += 1;
+            jsTestLog(`Inserting document: coll=${coll} _id=${newId}`);
+
+            this.collMirror[newId] = {_id: newId, updateCount: 0};
+
+            assertCommandWorked(() => {
+                coll.insertOne({_id: newId, updateCount: 0});
+            }, ErrorCodes.MovePrimaryInProgress);
+        },
+        update: function(db, collName, connCache) {
+            // Increment the update counter of a random document of the collection.
+
+            let coll = db[this.collName];
+
+            const randomId = this.getRandomDoc()._id;
+            jsTestLog(`Updating document: coll=${coll} _id=${randomId}`);
+
+            const newUpdateCount = this.collMirror[randomId].updateCount += 1;
+
+            assertCommandWorked(() => {
+                coll.updateOne({_id: randomId}, {$set: {updateCount: newUpdateCount}});
+            }, ErrorCodes.MovePrimaryInProgress);
+        },
+        delete: function(db, collName, connCache) {
+            // Remove a random document from the collection.
+
+            let coll = db[this.collName];
+
+            const randomId = this.getRandomDoc()._id;
+            jsTestLog(`Deleting document: coll=${coll} _id=${randomId}`);
+
+            delete this.collMirror[randomId];
+
+            assertCommandWorked(() => {
+                coll.deleteOne({_id: randomId});
+            }, ErrorCodes.MovePrimaryInProgress);
+        },
+        movePrimary: function(db, collName, connCache) {
+            // Move the primary shard of the database to a random shard (which could coincide with
+            // the starting one).
+
+            const shards = Object.keys(connCache.shards);
+            const toShard = shards[Random.randInt(shards.length)];
+            jsTestLog(`Running movePrimary: db=${db} to=${toShard}`);
+
+            assertAlways.commandWorkedOrFailedWithCode(
+                db.adminCommand({movePrimary: db.getName(), to: toShard}), [
+                    // Caused by a concurrent movePrimary operation on the same database but a
+                    // different destination shard.
+                    ErrorCodes.ConflictingOperationInProgress,
+                    // Due to a stepdown of the donor during the cloning phase, the movePrimary
+                    // operation failed. It is not automatically recovered, but any orphaned data on
+                    // the recipient has been deleted.
+                    7120202,
+                    // Due to a stepdown of the recipient during the cloning phase, the
+                    // _shardsvrCloneCatalogData command is retried by the donor, finding orphaned
+                    // documents. The movePrimary operation fails and is not automatically
+                    // recovered, but orphaned data on the recipient has been deleted.
+                    ErrorCodes.NamespaceExists
+                ]);
+        },
+        verifyDocuments: function(db, collName, connCache) {
+            // Verify the correctness of the collection data by checking that each document matches
+            // its copy in memory.
+
+            const coll = db[this.collName];
+            jsTestLog(`Verifying data: coll=${coll}`);
+
+            let docs = assertCommandWorked(
+                () => {
+                    return coll.find().batchSize(kBatchSizeForDocsLookup).toArray();
+                },
+                // Caused by a concurrent movePrimary operation.
+                ErrorCodes.QueryPlanKilled);
+
+            assertAlways.eq(Object.keys(this.collMirror).length,
+                            docs.length,
+                            `expectedData=${JSON.stringify(this.collMirror)}} actualData=${
+                                JSON.stringify(docs)}`);
+
+            docs.forEach(doc => {
+                assertAlways.eq(this.collMirror[doc._id],
+                                doc,
+                                `expectedData=${JSON.stringify(this.collMirror)}} actualData=${
+                                    JSON.stringify(docs)}`);
+            });
+        }
+    };
+
+    const standardTransition =
+        {insert: 0.22, update: 0.22, delete: 0.22, movePrimary: 0.12, verifyDocuments: 0.22};
+
+    const transitions = {
+        init: standardTransition,
+        insert: standardTransition,
+        update: standardTransition,
+        delete: standardTransition,
+        movePrimary: standardTransition,
+        verifyDocuments: standardTransition
+    };
+
+    return {
+        threadCount: 8,
+        iterations: 32,
+        states: states,
+        transitions: transitions,
+        data: data,
+        passConnectionCache: true
+    };
+})();
diff --git a/jstests/concurrency/fsm_workloads/random_DDL_operations.js b/jstests/concurrency/fsm_workloads/random_DDL_operations.js
index 106858f49e2..bb45b8e93eb 100644
--- a/jstests/concurrency/fsm_workloads/random_DDL_operations.js
+++ b/jstests/concurrency/fsm_workloads/random_DDL_operations.js
@@ -23,12 +23,18 @@ function getRandomCollection(db) {
     return db[collPrefix + Random.randInt(collCount)];
 }
 
+function getRandomShard(connCache) {
+    const shards = Object.keys(connCache.shards);
+    return shards[Random.randInt(shards.length)];
+}
+
 var $config = (function() {
     let states = {
         create: function(db, collName, connCache) {
             db = getRandomDb(db);
             const coll = getRandomCollection(db);
             const fullNs = coll.getFullName();
+
             jsTestLog('Executing create state: ' + fullNs);
             assertAlways.commandWorked(
                 db.adminCommand({shardCollection: fullNs, key: {_id: 1}, unique: false}));
@@ -44,12 +50,10 @@ var $config = (function() {
             db = getRandomDb(db);
             const srcColl = getRandomCollection(db);
             const srcCollName = srcColl.getFullName();
-
-            // Rename collection
             const destCollNS = getRandomCollection(db).getFullName();
             const destCollName = destCollNS.split('.')[1];
-            jsTestLog('Executing rename state:' + srcCollName + ' to ' + destCollNS);
 
+            jsTestLog('Executing rename state:' + srcCollName + ' to ' + destCollNS);
             assertAlways.commandWorkedOrFailedWithCode(
                 srcColl.renameCollection(destCollName, true /* dropTarget */), [
                     ErrorCodes.NamespaceNotFound,
@@ -57,6 +61,24 @@ var $config = (function() {
                     ErrorCodes.IllegalOperation
                 ]);
         },
+        movePrimary: function(db, collName, connCache) {
+            db = getRandomDb(db);
+            const shardId = getRandomShard(connCache);
+
+            jsTestLog('Executing movePrimary state: ' + db.getName() + ' to ' + shardId);
+            assertAlways.commandWorkedOrFailedWithCode(
+                db.adminCommand({movePrimary: db.getName(), to: shardId}), [
+                    ErrorCodes.ConflictingOperationInProgress,
+                    // The cloning phase has failed (e.g. as a result of a stepdown). When a failure
+                    // occurs at this phase, the movePrimary operation does not recover.
+                    7120202
+                ]);
+
+            // TODO (SERVER-71308): Remove explicit updating of database metadata on recipient. The
+            // recipient of a movePrimary operation is an agnostic participant of the protocol and
+            // doesn't update its cached metadata as a consequence of the operation.
+            assert.commandWorked(db.runCommand({listCollections: 1, nameOnly: true}));
+        },
         collMod: function(db, collName, connCache) {
             db = getRandomDb(db);
             const coll = getRandomCollection(db);
@@ -82,9 +104,10 @@ var $config = (function() {
     };
 
     let transitions = {
-        create: {create: 0.33, drop: 0.33, rename: 0.34},
-        drop: {create: 0.34, drop: 0.33, rename: 0.33},
-        rename: {create: 0.33, drop: 0.34, rename: 0.33}
+        create: {create: 0.25, drop: 0.25, rename: 0.25, movePrimary: 0.25},
+        drop: {create: 0.25, drop: 0.25, rename: 0.25, movePrimary: 0.25},
+        rename: {create: 0.25, drop: 0.25, rename: 0.25, movePrimary: 0.25},
+        movePrimary: {create: 0.25, drop: 0.25, rename: 0.25, movePrimary: 0.25}
     };
 
     return {