diff options
author | Antonio Fuschetto <antonio.fuschetto@mongodb.com> | 2023-02-07 07:52:06 +0000 |
---|---|---|
committer | Evergreen Agent <no-reply@evergreen.mongodb.com> | 2023-02-07 08:36:44 +0000 |
commit | 39d3dd4e28c188cb112a10039a783d6bacd7588e (patch) | |
tree | 2f59867469cdb0b6fc3e05bc0ea0ff3e7e9bee63 | |
parent | ef3ad37a33f38df9b20510c1c33510b0ceee8438 (diff) | |
download | mongo-39d3dd4e28c188cb112a10039a783d6bacd7588e.tar.gz |
SERVER-71203 Test the resilience of the new movePrimary
-rw-r--r-- | jstests/concurrency/fsm_workloads/move_primary_with_crud.js | 212 | ||||
-rw-r--r-- | jstests/concurrency/fsm_workloads/random_DDL_operations.js | 35 |
2 files changed, 241 insertions, 6 deletions
diff --git a/jstests/concurrency/fsm_workloads/move_primary_with_crud.js b/jstests/concurrency/fsm_workloads/move_primary_with_crud.js new file mode 100644 index 00000000000..36fceaba3fa --- /dev/null +++ b/jstests/concurrency/fsm_workloads/move_primary_with_crud.js @@ -0,0 +1,212 @@ +'use strict'; + +/** + * Randomly performs a series of CRUD and movePrimary operations on unsharded collections, checking + * for data consistency as a consequence of these operations. + * + * @tags: [ + * requires_sharding, + * # TODO (SERVER-71308): As soon as the feature flag is enabled, replace it with requires_fcv_70. + * featureFlagResilientMovePrimary + * ] + */ + +const $config = (function() { + const kCollNamePrefix = 'unsharded_coll_'; + const kInitialCollSize = 100; + const kBatchSizeForDocsLookup = kInitialCollSize * 2; + + /** + * Utility function that asserts that the specified command is executed successfully. However, + * if the error is in `retryableErrorCodes`, then the command is retried. + */ + const assertCommandWorked = function(cmd, retryableErrorCodes) { + if (!Array.isArray(retryableErrorCodes)) { + retryableErrorCodes = [retryableErrorCodes]; + } + + let res = undefined; + assertAlways.soon(() => { + try { + res = cmd(); + return true; + } catch (err) { + if (err instanceof BulkWriteError && err.hasWriteErrors()) { + for (let writeErr of err.getWriteErrors()) { + if (retryableErrorCodes.includes(writeErr.code)) { + return false; + } else { + throw err; + } + } + return true; + } else if (retryableErrorCodes.includes(err.code)) { + return false; + } + throw err; + } + }); + return res; + }; + + const data = { + // In-memory copy of the collection data. Every CRUD operation on the persisted collection + // is reflected on this object. The collection consistency check is performed by comparing + // its data with those managed by this copy. + collMirror: {}, + + // ID of the last document inserted into the collection. It's used as a generator of unique + // IDs for new documents to insert. + lastId: undefined, + + getRandomDoc: function() { + const keys = Object.keys(this.collMirror); + return this.collMirror[keys[Random.randInt(keys.length)]]; + } + }; + + const states = { + init: function(db, collName, connCache) { + // Insert an initial amount of documents into the collection, with a progressive _id and + // the update counter set to zero. + + this.collName = `${kCollNamePrefix}${this.tid}`; + let coll = db[this.collName]; + jsTestLog(`Initializing data: coll=${coll}`); + + for (let i = 0; i < kInitialCollSize; ++i) { + this.collMirror[i] = {_id: i, updateCount: 0}; + } + this.lastId = kInitialCollSize - 1; + + // Session with retryable writes is required to recover from a primary node step-down + // event during bulk insert processing. + this.session = db.getMongo().startSession({retryWrites: true}); + let sessionColl = this.session.getDatabase(db.getName()).getCollection(this.collName); + + assertCommandWorked(() => { + let bulkOp = sessionColl.initializeUnorderedBulkOp(); + for (let i = 0; i < kInitialCollSize; ++i) { + bulkOp.insert( + {_id: i, updateCount: 0}, + ); + } + bulkOp.execute(); + }, ErrorCodes.MovePrimaryInProgress); + }, + insert: function(db, collName, connCache) { + // Insert a document into the collection, with an _id greater than all those already + // present (last + 1) and the update counter set to zero. + + let coll = db[this.collName]; + + const newId = this.lastId += 1; + jsTestLog(`Inserting document: coll=${coll} _id=${newId}`); + + this.collMirror[newId] = {_id: newId, updateCount: 0}; + + assertCommandWorked(() => { + coll.insertOne({_id: newId, updateCount: 0}); + }, ErrorCodes.MovePrimaryInProgress); + }, + update: function(db, collName, connCache) { + // Increment the update counter of a random document of the collection. + + let coll = db[this.collName]; + + const randomId = this.getRandomDoc()._id; + jsTestLog(`Updating document: coll=${coll} _id=${randomId}`); + + const newUpdateCount = this.collMirror[randomId].updateCount += 1; + + assertCommandWorked(() => { + coll.updateOne({_id: randomId}, {$set: {updateCount: newUpdateCount}}); + }, ErrorCodes.MovePrimaryInProgress); + }, + delete: function(db, collName, connCache) { + // Remove a random document from the collection. + + let coll = db[this.collName]; + + const randomId = this.getRandomDoc()._id; + jsTestLog(`Deleting document: coll=${coll} _id=${randomId}`); + + delete this.collMirror[randomId]; + + assertCommandWorked(() => { + coll.deleteOne({_id: randomId}); + }, ErrorCodes.MovePrimaryInProgress); + }, + movePrimary: function(db, collName, connCache) { + // Move the primary shard of the database to a random shard (which could coincide with + // the starting one). + + const shards = Object.keys(connCache.shards); + const toShard = shards[Random.randInt(shards.length)]; + jsTestLog(`Running movePrimary: db=${db} to=${toShard}`); + + assertAlways.commandWorkedOrFailedWithCode( + db.adminCommand({movePrimary: db.getName(), to: toShard}), [ + // Caused by a concurrent movePrimary operation on the same database but a + // different destination shard. + ErrorCodes.ConflictingOperationInProgress, + // Due to a stepdown of the donor during the cloning phase, the movePrimary + // operation failed. It is not automatically recovered, but any orphaned data on + // the recipient has been deleted. + 7120202, + // Due to a stepdown of the recipient during the cloning phase, the + // _shardsvrCloneCatalogData command is retried by the donor, finding orphaned + // documents. The movePrimary operation fails and is not automatically + // recovered, but orphaned data on the recipient has been deleted. + ErrorCodes.NamespaceExists + ]); + }, + verifyDocuments: function(db, collName, connCache) { + // Verify the correctness of the collection data by checking that each document matches + // its copy in memory. + + const coll = db[this.collName]; + jsTestLog(`Verifying data: coll=${coll}`); + + let docs = assertCommandWorked( + () => { + return coll.find().batchSize(kBatchSizeForDocsLookup).toArray(); + }, + // Caused by a concurrent movePrimary operation. + ErrorCodes.QueryPlanKilled); + + assertAlways.eq(Object.keys(this.collMirror).length, + docs.length, + `expectedData=${JSON.stringify(this.collMirror)}} actualData=${ + JSON.stringify(docs)}`); + + docs.forEach(doc => { + assertAlways.eq(this.collMirror[doc._id], + doc, + `expectedData=${JSON.stringify(this.collMirror)}} actualData=${ + JSON.stringify(docs)}`); + }); + } + }; + + const standardTransition = + {insert: 0.22, update: 0.22, delete: 0.22, movePrimary: 0.12, verifyDocuments: 0.22}; + + const transitions = { + init: standardTransition, + insert: standardTransition, + update: standardTransition, + delete: standardTransition, + movePrimary: standardTransition, + verifyDocuments: standardTransition + }; + + return { + threadCount: 8, + iterations: 32, + states: states, + transitions: transitions, + data: data, + passConnectionCache: true + }; +})(); diff --git a/jstests/concurrency/fsm_workloads/random_DDL_operations.js b/jstests/concurrency/fsm_workloads/random_DDL_operations.js index 106858f49e2..bb45b8e93eb 100644 --- a/jstests/concurrency/fsm_workloads/random_DDL_operations.js +++ b/jstests/concurrency/fsm_workloads/random_DDL_operations.js @@ -23,12 +23,18 @@ function getRandomCollection(db) { return db[collPrefix + Random.randInt(collCount)]; } +function getRandomShard(connCache) { + const shards = Object.keys(connCache.shards); + return shards[Random.randInt(shards.length)]; +} + var $config = (function() { let states = { create: function(db, collName, connCache) { db = getRandomDb(db); const coll = getRandomCollection(db); const fullNs = coll.getFullName(); + jsTestLog('Executing create state: ' + fullNs); assertAlways.commandWorked( db.adminCommand({shardCollection: fullNs, key: {_id: 1}, unique: false})); @@ -44,12 +50,10 @@ var $config = (function() { db = getRandomDb(db); const srcColl = getRandomCollection(db); const srcCollName = srcColl.getFullName(); - - // Rename collection const destCollNS = getRandomCollection(db).getFullName(); const destCollName = destCollNS.split('.')[1]; - jsTestLog('Executing rename state:' + srcCollName + ' to ' + destCollNS); + jsTestLog('Executing rename state:' + srcCollName + ' to ' + destCollNS); assertAlways.commandWorkedOrFailedWithCode( srcColl.renameCollection(destCollName, true /* dropTarget */), [ ErrorCodes.NamespaceNotFound, @@ -57,6 +61,24 @@ var $config = (function() { ErrorCodes.IllegalOperation ]); }, + movePrimary: function(db, collName, connCache) { + db = getRandomDb(db); + const shardId = getRandomShard(connCache); + + jsTestLog('Executing movePrimary state: ' + db.getName() + ' to ' + shardId); + assertAlways.commandWorkedOrFailedWithCode( + db.adminCommand({movePrimary: db.getName(), to: shardId}), [ + ErrorCodes.ConflictingOperationInProgress, + // The cloning phase has failed (e.g. as a result of a stepdown). When a failure + // occurs at this phase, the movePrimary operation does not recover. + 7120202 + ]); + + // TODO (SERVER-71308): Remove explicit updating of database metadata on recipient. The + // recipient of a movePrimary operation is an agnostic participant of the protocol and + // doesn't update its cached metadata as a consequence of the operation. + assert.commandWorked(db.runCommand({listCollections: 1, nameOnly: true})); + }, collMod: function(db, collName, connCache) { db = getRandomDb(db); const coll = getRandomCollection(db); @@ -82,9 +104,10 @@ var $config = (function() { }; let transitions = { - create: {create: 0.33, drop: 0.33, rename: 0.34}, - drop: {create: 0.34, drop: 0.33, rename: 0.33}, - rename: {create: 0.33, drop: 0.34, rename: 0.33} + create: {create: 0.25, drop: 0.25, rename: 0.25, movePrimary: 0.25}, + drop: {create: 0.25, drop: 0.25, rename: 0.25, movePrimary: 0.25}, + rename: {create: 0.25, drop: 0.25, rename: 0.25, movePrimary: 0.25}, + movePrimary: {create: 0.25, drop: 0.25, rename: 0.25, movePrimary: 0.25} }; return { |