SERVER-52620 Add withReshardingInBackground() to ReshardingTest fixture.

Switches over most of the tests either running reshardCollection
explicitly or calling startReshardingInBackground() to use it.

SERVER-53460 Use assert.soon() when verifying post-resharding state.

The config.localReshardingOperations.{donor,recipient} collections are
not guaranteed to be empty immediately after the reshardCollection
command has returned.

8 files changed, 642 insertions, 429 deletions

diff --git a/jstests/sharding/libs/resharding_test_fixture.js b/jstests/sharding/libs/resharding_test_fixture.js
index dc16064b8ca..feda4500e5f 100644
--- a/jstests/sharding/libs/resharding_test_fixture.js
+++ b/jstests/sharding/libs/resharding_test_fixture.js
@@ -16,9 +16,10 @@ load("jstests/sharding/libs/create_sharded_collection_util.js");
  *      const sourceCollection = reshardingTest.createShardedCollection(...);
  *      // ... Do some operations before resharding starts ...
  *      assert.commandWorked(sourceCollection.insert({_id: 0}));
- *      reshardingTest.startReshardingInBackground(...);
- *      // ... Do some operations during the resharding operation ...
- *      assert.commandWorked(sourceCollection.update({_id: 0}, {$inc: {a: 1}}));
+ *      reshardingTest.withReshardingInBackground({...}, () => {
+ *          // ... Do some operations during the resharding operation ...
+ *          assert.commandWorked(sourceCollection.update({_id: 0}, {$inc: {a: 1}}));
+ *      });
  *      reshardingTest.teardown();
  */
 var ReshardingTest = class {
@@ -27,24 +28,43 @@ var ReshardingTest = class {
         numRecipients: numRecipients = 1,
         reshardInPlace: reshardInPlace = false,
     } = {}) {
+        // The @private JSDoc comments cause VS Code to not display the corresponding properties and
+        // methods in its autocomplete list. This makes it simpler for test authors to know what the
+        // public interface of the ReshardingTest class is.
+
+        /** @private */
         this._numDonors = numDonors;
+        /** @private */
         this._numRecipients = numRecipients;
+        /** @private */
         this._reshardInPlace = reshardInPlace;
+        /** @private */
         this._numShards = this._reshardInPlace ? Math.max(this._numDonors, this._numRecipients)
                                                : this._numDonors + this._numRecipients;
 
-        this._dbName = undefined;
-        this._collName = undefined;
-        this._ns = undefined;
-        this._sourceCollectionUUIDString = undefined;
+        // Properties set by setup().
+        /** @private */
+        this._st = undefined;
 
-        this._tempCollName = undefined;
+        // Properties set by createShardedCollection().
+        /** @private */
+        this._ns = undefined;
+        /** @private */
+        this._currentShardKey = undefined;
+        /** @private */
+        this._sourceCollectionUUID = undefined;
+        /** @private */
         this._tempNs = undefined;
 
-        this._st = undefined;
-        this._reshardingThread = undefined;
-        this._pauseCoordinatorInSteadyStateFailpoint = undefined;
+        // Properties set by startReshardingInBackground() and withReshardingInBackground().
+        /** @private */
         this._newShardKey = undefined;
+        /** @private */
+        this._pauseCoordinatorInSteadyStateFailpoint = undefined;
+        /** @private */
+        this._reshardingThread = undefined;
+        /** @private */
+        this._isReshardingActive = false;
     }
 
     setup() {
@@ -87,6 +107,7 @@ var ReshardingTest = class {
         }
     }
 
+    /** @private */
     _donorShards() {
         return Array.from({length: this._numDonors}, (_, i) => this._st[`shard${i}`]);
     }
@@ -95,6 +116,7 @@ var ReshardingTest = class {
         return this._donorShards().map(shard => shard.shardName);
     }
 
+    /** @private */
     _recipientShards() {
         return Array
             .from({length: this._numRecipients},
@@ -106,10 +128,6 @@ var ReshardingTest = class {
         return this._recipientShards().map(shard => shard.shardName);
     }
 
-    get temporaryReshardingCollectionName() {
-        return this._tempCollName;
-    }
-
     /**
      * Shards a non-existing collection using the specified shard key and chunk ranges.
      *
@@ -119,22 +137,25 @@ var ReshardingTest = class {
      */
     createShardedCollection({ns, shardKeyPattern, chunks}) {
         this._ns = ns;
+        this._currentShardKey = Object.assign({}, shardKeyPattern);
 
         const sourceCollection = this._st.s.getCollection(ns);
         const sourceDB = sourceCollection.getDB();
 
-        this._dbName = sourceDB.getName();
-        this._collName = sourceCollection.getName();
-
         CreateShardedCollectionUtil.shardCollectionWithChunks(
             sourceCollection, shardKeyPattern, chunks);
 
-        const sourceCollectionUUID =
+        this._sourceCollectionUUID =
             getUUIDFromListCollections(sourceDB, sourceCollection.getName());
-        this._sourceCollectionUUIDString = extractUUIDFromObject(sourceCollectionUUID);
+        const sourceCollectionUUIDString = extractUUIDFromObject(this._sourceCollectionUUID);
+
+        this._tempNs = `${sourceDB.getName()}.system.resharding.${sourceCollectionUUIDString}`;
 
-        this._tempCollName = `system.resharding.${this._sourceCollectionUUIDString}`;
-        this._tempNs = `${this._dbName}.${this._tempCollName}`;
+        // mongos won't know about the temporary resharding collection and will therefore assume the
+        // collection is unsharded. We configure one of the recipient shards to be the primary shard
+        // for the database so mongos still ends up routing operations to a shard which owns the
+        // temporary resharding collection.
+        this._st.ensurePrimaryShard(sourceDB.getName(), this.recipientShardNames[0]);
 
         return sourceCollection;
     }
@@ -145,28 +166,189 @@ var ReshardingTest = class {
      * @param newChunks - an array of
      * {min: <shardKeyValue0>, max: <shardKeyValue1>, shard: <shardName>} objects. The chunks must
      * form a partition of the {shardKey: MinKey} --> {shardKey: MaxKey} space.
+     *
+     * @deprecated prefer using the withReshardingInBackground() method instead.
      */
     startReshardingInBackground({newShardKeyPattern, newChunks}) {
+        this._startReshardingInBackgroundAndAllowCommandFailure({newShardKeyPattern, newChunks},
+                                                                ErrorCodes.OK);
+    }
+
+    /** @private */
+    _startReshardingInBackgroundAndAllowCommandFailure({newShardKeyPattern, newChunks},
+                                                       expectedCode) {
         newChunks = newChunks.map(
             chunk => ({min: chunk.min, max: chunk.max, recipientShardId: chunk.shard}));
 
-        this._newShardKey = newShardKeyPattern;
+        this._newShardKey = Object.assign({}, newShardKeyPattern);
 
         this._pauseCoordinatorInSteadyStateFailpoint = configureFailPoint(
             this._st.configRS.getPrimary(), "reshardingPauseCoordinatorInSteadyState");
 
-        this._reshardingThread = new Thread(function(host, ns, newShardKeyPattern, newChunks) {
-            const conn = new Mongo(host);
-            assert.commandWorked(conn.adminCommand({
-                reshardCollection: ns,
-                key: newShardKeyPattern,
-                _presetReshardedChunks: newChunks,
-            }));
-        }, this._st.s.host, this._ns, newShardKeyPattern, newChunks);
+        const commandDoneSignal = new CountDownLatch(1);
+
+        this._reshardingThread = new Thread(
+            function(host, ns, newShardKeyPattern, newChunks, commandDoneSignal, expectedCode) {
+                const conn = new Mongo(host);
+                const res = conn.adminCommand({
+                    reshardCollection: ns,
+                    key: newShardKeyPattern,
+                    _presetReshardedChunks: newChunks,
+                });
+                commandDoneSignal.countDown();
+
+                if (expectedCode === ErrorCodes.OK) {
+                    assert.commandWorked(res);
+                } else {
+                    assert.commandFailedWithCode(res, expectedCode);
+                }
+            },
+            this._st.s.host,
+            this._ns,
+            newShardKeyPattern,
+            newChunks,
+            commandDoneSignal,
+            expectedCode);
 
         this._reshardingThread.start();
+        this._isReshardingActive = true;
+
+        return commandDoneSignal;
     }
 
+    /**
+     * Reshards an existing collection using the specified new shard key and new chunk ranges.
+     *
+     * @param newChunks - an array of
+     * {min: <shardKeyValue0>, max: <shardKeyValue1>, shard: <shardName>} objects. The chunks must
+     * form a partition of the {shardKey: MinKey} --> {shardKey: MaxKey} space.
+     *
+     * @param duringReshardingFn - a function which optionally accepts the temporary resharding
+     * namespace string. It is only guaranteed to be called after mongos has started running the
+     * reshardCollection command. Callers should use DiscoverTopology.findConnectedNodes() to
+     * introspect the state of the donor or recipient shards if they need more specific
+     * synchronization.
+     *
+     * @param expectedCode - the expected response code for the reshardCollection command. Callers
+     * of interruptReshardingThread() will want to set this to ErrorCodes.Interrupted, for example.
+     */
+    withReshardingInBackground({newShardKeyPattern, newChunks},
+                               duringReshardingFn = (tempNs) => {},
+                               expectedCode = ErrorCodes.OK) {
+        const commandDoneSignal = this._startReshardingInBackgroundAndAllowCommandFailure(
+            {newShardKeyPattern, newChunks}, expectedCode);
+
+        assert.soon(() => {
+            const op = this._findReshardingCommandOp();
+            return op !== undefined ||
+                (expectedCode !== ErrorCodes.OK && commandDoneSignal.getCount() === 0);
+        }, "failed to find reshardCollection in $currentOp output");
+
+        this._callFunctionSafely(() => duringReshardingFn(this._tempNs));
+        this._checkConsistencyAndPostState(expectedCode);
+    }
+
+    /** @private */
+    _findReshardingCommandOp() {
+        return this._st.admin
+            .aggregate([
+                {$currentOp: {allUsers: true, localOps: true}},
+                {$match: {"command.reshardCollection": this._ns}},
+            ])
+            .toArray()[0];
+    }
+
+    /**
+     * Wrapper around invoking a 0-argument function to make test failures less confusing.
+     *
+     * This helper attempts to disable the reshardingPauseCoordinatorInSteadyState failpoint when an
+     * exception is thrown to prevent the mongo shell from hanging (really the config server) on top
+     * of having a JavaScript error.
+     *
+     * This helper attempts to interrupt and join the resharding thread when an exception is thrown
+     * to prevent the mongo shell from aborting on top of having a JavaScript error.
+     *
+     * @private
+     */
+    _callFunctionSafely(fn) {
+        try {
+            fn();
+        } catch (duringReshardingError) {
+            try {
+                this._pauseCoordinatorInSteadyStateFailpoint.off();
+            } catch (disableFailpointError) {
+                print(`Ignoring error from disabling the resharding coordinator failpoint: ${
+                    tojson(disableFailpointError)}`);
+
+                print("The config server primary and the mongo shell along with it are expected" +
+                      " to hang due to the resharding coordinator being left uninterrupted");
+            }
+
+            try {
+                this.interruptReshardingThread();
+
+                try {
+                    this._reshardingThread.join();
+                } catch (joinError) {
+                    print(`Ignoring error from the resharding thread: ${tojson(joinError)}`);
+                }
+            } catch (killOpError) {
+                print(`Ignoring error from sending killOp to the reshardCollection command: ${
+                    tojson(killOpError)}`);
+
+                print("The mongo shell is expected to abort due to the resharding thread being" +
+                      " left unjoined");
+            }
+
+            throw duringReshardingError;
+        }
+    }
+
+    interruptReshardingThread() {
+        const op = this._findReshardingCommandOp();
+        assert.neq(undefined, op, "failed to find reshardCollection in $currentOp output");
+        assert.commandWorked(this._st.admin.killOp(op.opid));
+    }
+
+    /** @private */
+    _checkConsistencyAndPostState(expectedCode) {
+        if (expectedCode === ErrorCodes.OK) {
+            this._callFunctionSafely(() => {
+                // We use the reshardingPauseCoordinatorInSteadyState failpoint so that any
+                // intervening writes performed on the sharded collection (from when the resharding
+                // operation had started until now) are eventually applied by the recipient shards.
+                // We then use the reshardingPauseCoordinatorBeforeCommit to wait for all of the
+                // recipient shards to have applied through all of the oplog entries from all of the
+                // donor shards.
+                this._pauseCoordinatorInSteadyStateFailpoint.wait();
+                const pauseCoordinatorBeforeCommitFailpoint =
+                    configureFailPoint(this._pauseCoordinatorInSteadyStateFailpoint.conn,
+                                       "reshardingPauseCoordinatorBeforeCommit");
+
+                this._pauseCoordinatorInSteadyStateFailpoint.off();
+                pauseCoordinatorBeforeCommitFailpoint.wait();
+
+                this._checkConsistency();
+
+                pauseCoordinatorBeforeCommitFailpoint.off();
+            });
+        } else {
+            this._callFunctionSafely(() => {
+                this._pauseCoordinatorInSteadyStateFailpoint.off();
+            });
+        }
+
+        this._reshardingThread.join();
+        this._isReshardingActive = false;
+
+        // TODO SERVER-52838: Call _checkPostState() when donor and recipient shards clean up their
+        // local metadata on error.
+        if (expectedCode === ErrorCodes.OK) {
+            this._checkPostState(expectedCode);
+        }
+    }
+
+    /** @private */
     _checkConsistency() {
         const nsCursor = this._st.s.getCollection(this._ns).find().sort({_id: 1});
         const tempNsCursor = this._st.s.getCollection(this._tempNs).find().sort({_id: 1});
@@ -187,64 +369,94 @@ var ReshardingTest = class {
         });
     }
 
-    _checkConsistencyPostReshardingComplete() {
-        ///
-        // Check that resharding content on the configsvr is cleaned up.
-        ///
-        assert.eq(0, this._st.config.reshardingOperations.find({nss: this._ns}).itcount());
+    /** @private */
+    _checkPostState(expectedCode) {
+        this._checkCoordinatorPostState(expectedCode);
 
-        assert.eq(
-            0,
-            this._st.config.collections.find({reshardingFields: {$exists: true}, _id: this._ns})
-                .itcount());
-
-        assert.eq(0, this._st.config.collections.find({_id: this._tempNs}).itcount());
-
-        ///
-        // Check that resharding content local to each participant is cleaned up.
-        ///
-        this._donorShards().forEach((donor) => {
-            assert.eq(0,
-                      donor.getDB("config")
-                          .localReshardingOperations.donor.find({nss: this._ns})
-                          .itcount());
-        });
-
-        this._recipientShards().forEach((recipient) => {
-            assert(!recipient.getCollection(this._tempNs).exists());
-            assert.eq(0,
-                      recipient.getDB("config")
-                          .localReshardingOperations.recipient.find({nss: this._ns})
-                          .itcount());
-        });
-
-        ///
-        // Check that the collection is updated from the resharding operation.
-        ///
-        const finalReshardedCollectionUUID =
-            getUUIDFromListCollections(this._st.s.getDB(this._dbName), this._collName);
-        assert.neq(this._sourceCollectionUUIDString,
-                   extractUUIDFromObject(finalReshardedCollectionUUID));
+        for (let recipient of this._recipientShards()) {
+            this._checkRecipientPostState(recipient);
+        }
 
-        const actualShardKey = this._st.config.collections.findOne({_id: this._ns}).key;
-        assert.eq(this._newShardKey, actualShardKey);
+        for (let donor of this._donorShards()) {
+            this._checkDonorPostState(donor);
+        }
     }
 
-    teardown() {
-        this._pauseCoordinatorInSteadyStateFailpoint.wait();
-        const pauseCoordinatorBeforeCommitFailpoint =
-            configureFailPoint(this._pauseCoordinatorInSteadyStateFailpoint.conn,
-                               "reshardingPauseCoordinatorBeforeCommit");
-
-        this._pauseCoordinatorInSteadyStateFailpoint.off();
-        pauseCoordinatorBeforeCommitFailpoint.wait();
+    /** @private */
+    _checkCoordinatorPostState(expectedCode) {
+        assert.eq([],
+                  this._st.config.reshardingOperations.find({nss: this._ns}).toArray(),
+                  "expected config.reshardingOperations to be empty, but found it wasn't");
+
+        assert.eq([],
+                  this._st.config.collections.find({reshardingFields: {$exists: true}}).toArray(),
+                  "expected there to be no config.collections entries with 'reshardingFields' set");
+
+        assert.eq([],
+                  this._st.config.collections.find({allowMigrations: {$exists: true}}).toArray(),
+                  "expected there to be no config.collections entries with 'allowMigrations' set");
+
+        assert.eq([],
+                  this._st.config.collections.find({_id: this._tempNs}).toArray(),
+                  "expected there to not be a config.collections entry for the temporary" +
+                      " resharding collection");
+
+        const collEntry = this._st.config.collections.findOne({_id: this._ns});
+        assert.neq(null, collEntry, `didn't find config.collections entry for ${this._ns}`);
+
+        if (expectedCode === ErrorCodes.OK) {
+            assert.eq(this._newShardKey,
+                      collEntry.key,
+                      "shard key pattern didn't change despite resharding having succeeded");
+            assert.neq(this._sourceCollectionUUID,
+                       collEntry.uuid,
+                       "collection UUID didn't change despite resharding having succeeded");
+        } else {
+            assert.eq(this._currentShardKey,
+                      collEntry.key,
+                      "shard key pattern changed despite resharding having failed");
+            assert.eq(this._sourceCollectionUUID,
+                      collEntry.uuid,
+                      "collection UUID changed despite resharding having failed");
+        }
+    }
 
-        this._checkConsistency();
+    /** @private */
+    _checkRecipientPostState(recipient) {
+        assert.eq(
+            null,
+            recipient.getCollection(this._tempNs).exists(),
+            `expected the temporary resharding collection to not exist, but found it does on ${
+                recipient.shardName}`);
+
+        const localRecipientOpsNs = "config.localReshardingOperations.recipient";
+        let res;
+        assert.soon(
+            () => {
+                res = recipient.getCollection(localRecipientOpsNs).find().toArray();
+                return res.length === 0;
+            },
+            () => `${localRecipientOpsNs} document wasn't cleaned up on ${recipient.shardName}: ${
+                tojson(res)}`);
+    }
 
-        pauseCoordinatorBeforeCommitFailpoint.off();
-        this._reshardingThread.join();
+    /** @private */
+    _checkDonorPostState(donor) {
+        const localDonorOpsNs = "config.localReshardingOperations.donor";
+        let res;
+        assert.soon(
+            () => {
+                res = donor.getCollection(localDonorOpsNs).find().toArray();
+                return res.length === 0;
+            },
+            () => `${localDonorOpsNs} document wasn't cleaned up on ${donor.shardName}: ${
+                tojson(res)}`);
+    }
 
-        this._checkConsistencyPostReshardingComplete();
+    teardown() {
+        if (this._isReshardingActive) {
+            this._checkConsistencyAndPostState(ErrorCodes.OK);
+        }
 
         this._st.stop();
     }
diff --git a/jstests/sharding/reshard_collection_existing_sk_index_not_duplicated.js b/jstests/sharding/reshard_collection_existing_sk_index_not_duplicated.js
index 2042eb2fe92..f3a1260d2d5 100644
--- a/jstests/sharding/reshard_collection_existing_sk_index_not_duplicated.js
+++ b/jstests/sharding/reshard_collection_existing_sk_index_not_duplicated.js
@@ -9,82 +9,62 @@
 //
 
 (function() {
-'use strict';
+"use strict";
 
 load("jstests/sharding/libs/resharding_test_fixture.js");
 
-const dbName = "reshardingDb";
-const collName = "coll";
-const ns = dbName + "." + collName;
+const reshardingTest = new ReshardingTest();
+reshardingTest.setup();
 
-let getIndexes = (dbName, collName, conn) => {
-    const indexRes = conn.getDB(dbName).runCommand({listIndexes: collName});
-    assert.commandWorked(indexRes);
-    return indexRes.cursor.firstBatch;
-};
+const testCases = [
+    {ns: "reshardingDb.no_compatible_index"},
+    {ns: "reshardingDb.has_compatible_index", indexToCreateBeforeResharding: {newKey: 1}},
+    {
+        ns: "reshardingDb.compatible_index_with_extra",
+        indexToCreateBeforeResharding: {newKey: 1, extra: 1}
+    },
+];
 
-let createShardedCollection = (reshardingTest) => {
+for (let {ns, indexToCreateBeforeResharding} of testCases) {
     const donorShardNames = reshardingTest.donorShardNames;
-    return reshardingTest.createShardedCollection({
+    const sourceCollection = reshardingTest.createShardedCollection({
         ns,
         shardKeyPattern: {oldKey: 1},
-        chunks: [
-            {min: {oldKey: MinKey}, max: {oldKey: 0}, shard: donorShardNames[0]},
-            {min: {oldKey: 0}, max: {oldKey: MaxKey}, shard: donorShardNames[1]},
-        ],
+        chunks: [{min: {oldKey: MinKey}, max: {oldKey: MaxKey}, shard: donorShardNames[0]}],
     });
-};
 
-let startReshardingInBackground = (reshardingTest) => {
+    if (indexToCreateBeforeResharding !== undefined) {
+        assert.commandWorked(sourceCollection.createIndex(indexToCreateBeforeResharding));
+    }
+
+    // Create an index which won't be compatible with the {newKey: 1} shard key pattern but should
+    // still exist post-resharding.
+    assert.commandWorked(sourceCollection.createIndex({extra: 1}));
+    const indexesBeforeResharding = sourceCollection.getIndexes();
+
     const recipientShardNames = reshardingTest.recipientShardNames;
-    reshardingTest.startReshardingInBackground({
+    reshardingTest.withReshardingInBackground({
         newShardKeyPattern: {newKey: 1},
-        newChunks: [
-            {min: {newKey: MinKey}, max: {newKey: 0}, shard: recipientShardNames[0]},
-            {min: {newKey: 0}, max: {newKey: MaxKey}, shard: recipientShardNames[1]},
-        ],
+        newChunks: [{min: {newKey: MinKey}, max: {newKey: MaxKey}, shard: recipientShardNames[0]}],
     });
-};
-
-let awaitReshardingInState = (sourceCollection, state) => {
-    const mongos = sourceCollection.getMongo();
-    assert.soon(() => {
-        const coordinatorDoc = mongos.getCollection("config.reshardingOperations").findOne();
-        return coordinatorDoc !== null && coordinatorDoc.state === state;
-    });
-};
-
-let runReshardingCollectionVerifyIndexConsistency = (indexToCreateBeforeResharding) => {
-    // reshardInPlace is required for this test so that the primary shard is guaranteed to know
-    // about the indexes on the temporary resharding collection.
-    const reshardingTest =
-        new ReshardingTest({numDonors: 2, numRecipients: 2, reshardInPlace: true});
-    reshardingTest.setup();
-
-    let sourceCollection = createShardedCollection(reshardingTest);
-    let mongos = sourceCollection.getMongo();
 
-    let indexesBeforeResharding;
-    if (indexToCreateBeforeResharding) {
-        sourceCollection.createIndex(indexToCreateBeforeResharding);
-        indexesBeforeResharding = getIndexes(dbName, collName, mongos);
-    }
-
-    startReshardingInBackground(reshardingTest);
-    awaitReshardingInState(sourceCollection, "applying");
-
-    if (indexToCreateBeforeResharding) {
-        let indexesAfterResharding = getIndexes(dbName, collName, mongos);
+    const indexesAfterResharding = sourceCollection.getIndexes();
+    if (indexToCreateBeforeResharding !== undefined) {
         assert.sameMembers(indexesBeforeResharding, indexesAfterResharding);
     } else {
-        ShardedIndexUtil.assertIndexExistsOnShard(
-            mongos, dbName, reshardingTest.temporaryReshardingCollectionName, {newKey: 1});
-    }
+        const shardKeyIndexPos = indexesAfterResharding.findIndex(
+            indexInfo => bsonBinaryEqual(indexInfo.key, {newKey: 1}));
+
+        assert.lte(0,
+                   shardKeyIndexPos,
+                   `resharding didn't create index on new shard key pattern: ${
+                       tojson(indexesAfterResharding)}`);
 
-    reshardingTest.teardown();
-};
+        const indexesAfterReshardingToCompare = indexesAfterResharding.slice();
+        indexesAfterReshardingToCompare.splice(shardKeyIndexPos, 1);
+        assert.sameMembers(indexesBeforeResharding, indexesAfterReshardingToCompare);
+    }
+}
 
-runReshardingCollectionVerifyIndexConsistency();
-runReshardingCollectionVerifyIndexConsistency({newKey: 1});
-runReshardingCollectionVerifyIndexConsistency({newKey: 1, extra: 1});
+reshardingTest.teardown();
 })();
diff --git a/jstests/sharding/resharding_allowMigrations.js b/jstests/sharding/resharding_allowMigrations.js
index 9f1fa19756a..c61a946cc40 100644
--- a/jstests/sharding/resharding_allowMigrations.js
+++ b/jstests/sharding/resharding_allowMigrations.js
@@ -12,53 +12,39 @@
 
 load("jstests/sharding/libs/resharding_test_fixture.js");
 
-const reshardingTest = new ReshardingTest({numDonors: 2, numRecipients: 2, reshardInPlace: true});
+const reshardingTest = new ReshardingTest({numDonors: 2});
 reshardingTest.setup();
 
-const db = 'reshardingDb';
-const col = 'coll';
-const ns = `${db}.${col}`;
 const donorShardNames = reshardingTest.donorShardNames;
 const sourceCollection = reshardingTest.createShardedCollection({
-    ns,
+    ns: "reshardingDb.coll",
     shardKeyPattern: {oldKey: 1},
-    chunks: [
-        {min: {oldKey: MinKey}, max: {oldKey: 0}, shard: donorShardNames[0]},
-        {min: {oldKey: 0}, max: {oldKey: MaxKey}, shard: donorShardNames[1]},
-    ],
+    chunks: [{min: {oldKey: MinKey}, max: {oldKey: MaxKey}, shard: donorShardNames[0]}],
 });
 
-// Perform some inserts before resharding starts so there's data to clone.
-assert.commandWorked(sourceCollection.insert(
-    [
-        {_id: "stays on shard0", oldKey: -10, newKey: -10},
-        {_id: "moves to shard0", oldKey: 10, newKey: -10},
-        {_id: "moves to shard1", oldKey: -10, newKey: 10},
-        {_id: "stays on shard1", oldKey: 10, newKey: 10},
-    ],
-    {writeConcern: {w: "majority"}}));
-
 const recipientShardNames = reshardingTest.recipientShardNames;
-reshardingTest.startReshardingInBackground({
-    newShardKeyPattern: {newKey: 1},
-    newChunks: [
-        {min: {newKey: MinKey}, max: {newKey: 0}, shard: recipientShardNames[0]},
-        {min: {newKey: 0}, max: {newKey: MaxKey}, shard: recipientShardNames[1]},
-    ],
-});
-
-const mongos = sourceCollection.getMongo();
-const tempns = `${db}.${reshardingTest.temporaryReshardingCollectionName}`;
-assert.soon(() => {
-    return mongos.getDB("config").collections.findOne({_id: ns}).allowMigrations === false;
-});
-assert.soon(() => {
-    return mongos.getDB("config").collections.findOne({_id: tempns}).allowMigrations === false;
-});
-
-assert.commandFailedWithCode(
-    mongos.adminCommand({moveChunk: ns, find: {oldKey: -10}, to: donorShardNames[1]}),
-    ErrorCodes.ConflictingOperationInProgress);
+reshardingTest.withReshardingInBackground(
+    {
+        newShardKeyPattern: {newKey: 1},
+        newChunks: [{min: {newKey: MinKey}, max: {newKey: MaxKey}, shard: recipientShardNames[0]}],
+    },
+    (tempNs) => {
+        const mongos = sourceCollection.getMongo();
+        const ns = sourceCollection.getFullName();
+
+        let res;
+        assert.soon(() => {
+            res = mongos.getCollection("config.collections")
+                      .find({_id: {$in: [ns, tempNs]}})
+                      .toArray();
+
+            return res.length === 2 && res.every(collEntry => collEntry.allowMigrations === false);
+        }, () => `timed out waiting for collections to have allowMigrations=false: ${tojson(res)}`);
+
+        assert.commandFailedWithCode(
+            mongos.adminCommand({moveChunk: ns, find: {oldKey: -10}, to: donorShardNames[1]}),
+            ErrorCodes.ConflictingOperationInProgress);
+    });
 
 reshardingTest.teardown();
 })();
diff --git a/jstests/sharding/resharding_clones_duplicate_key.js b/jstests/sharding/resharding_clones_duplicate_key.js
index ff44282265b..4ccb50d541c 100644
--- a/jstests/sharding/resharding_clones_duplicate_key.js
+++ b/jstests/sharding/resharding_clones_duplicate_key.js
@@ -10,99 +10,73 @@
 (function() {
 "use strict";
 
-load("jstests/libs/parallelTester.js");
-load("jstests/sharding/libs/create_sharded_collection_util.js");
 load("jstests/libs/fail_point_util.js");
+load("jstests/libs/discover_topology.js");
+load("jstests/sharding/libs/resharding_test_fixture.js");
 
-const st = new ShardingTest({
-    mongos: 1,
-    config: 1,
-    shards: 2,
-    rs: {nodes: 1},
-    rsOptions: {
-        setParameter: {
-            "failpoint.WTPreserveSnapshotHistoryIndefinitely": tojson({mode: "alwaysOn"}),
-        }
-    }
-});
-
-const inputCollection = st.s.getCollection("reshardingDb.coll");
+const reshardingTest = new ReshardingTest({numDonors: 2, numRecipients: 1});
+reshardingTest.setup();
 
-CreateShardedCollectionUtil.shardCollectionWithChunks(inputCollection, {oldKey: 1}, [
-    {min: {oldKey: MinKey}, max: {oldKey: 0}, shard: st.shard0.shardName},
-    {min: {oldKey: 0}, max: {oldKey: MaxKey}, shard: st.shard1.shardName},
-]);
+const donorShardNames = reshardingTest.donorShardNames;
+const inputCollection = reshardingTest.createShardedCollection({
+    ns: "reshardingDb.coll",
+    shardKeyPattern: {oldKey: 1},
+    chunks: [
+        {min: {oldKey: MinKey}, max: {oldKey: 0}, shard: donorShardNames[0]},
+        {min: {oldKey: 0}, max: {oldKey: MaxKey}, shard: donorShardNames[1]},
+    ],
+});
 
 // The following documents violate the global _id uniqueness assumption of sharded collections. It
 // is possible to construct such a sharded collection due to how each shard independently enforces
 // the uniqueness of _id values for only the documents it owns. The resharding operation is expected
 // to abort upon discovering this violation.
-assert.commandWorked(inputCollection.insert(
-    [
-        {_id: 0, info: "stays on shard0", oldKey: -10, newKey: -10},
-        {_id: 0, info: "moves to shard0", oldKey: 10, newKey: -10},
-        {_id: 1, info: "moves to shard1", oldKey: -10, newKey: 10},
-        {_id: 1, info: "stays on shard1", oldKey: 10, newKey: 10},
-    ],
-    {writeConcern: {w: "majority"}}));
+assert.commandWorked(inputCollection.insert([
+    {_id: 0, info: `moves from ${donorShardNames[0]}`, oldKey: -10, newKey: -10},
+    {_id: 0, info: `moves from ${donorShardNames[1]}`, oldKey: 10, newKey: 10},
+]));
 
-// In the current implementation, the _configsvrReshardCollection command won't ever complete if one
-// of the donor or recipient shards encounters an unrecoverable error. To work around this
-// limitation, we verify the recipient shard transitioned itself into the "error" state as a result
-// of the duplicate key error during resharding's collection cloning.
-//
-// TODO SERVER-50584: Remove the separate thread from this test and instead directly assert that the
-// reshardCollection command fails with an error.
-//
-// We use a special appName to identify the _configsvrReshardCollection command because the command
-// is too large and would otherwise be truncated by the currentOp() output.
-const kAppName = "testReshardCollectionThread";
-const thread = new Thread(function(host, appName, commandObj) {
-    const conn = new Mongo(`mongodb://${host}/?appName=${appName}`);
-    assert.commandFailedWithCode(conn.adminCommand(commandObj), ErrorCodes.Interrupted);
-}, st.s.host, kAppName, {
-    reshardCollection: inputCollection.getFullName(),
-    key: {newKey: 1},
-    _presetReshardedChunks: [
-        {min: {newKey: MinKey}, max: {newKey: 0}, recipientShardId: st.shard0.shardName},
-        {min: {newKey: 0}, max: {newKey: MaxKey}, recipientShardId: st.shard1.shardName},
-    ],
-});
-
-let failPoint1 = configureFailPoint(st.shard0, "removeRecipientDocFailpoint");
-let failPoint2 = configureFailPoint(st.shard1, "removeRecipientDocFailpoint");
-
-thread.start();
-
-function assertEventuallyErrorsLocally(shard) {
-    const recipientCollection =
-        shard.rs.getPrimary().getCollection("config.localReshardingOperations.recipient");
+function assertEventuallyErrorsLocally(shardConn, shardName) {
+    const localRecipientOpsCollection =
+        shardConn.getCollection("config.localReshardingOperations.recipient");
 
     assert.soon(
         () => {
-            return recipientCollection.findOne({state: "error"}) !== null;
+            return localRecipientOpsCollection.findOne({state: "error"}) !== null;
         },
         () => {
-            return "recipient shard " + shard.shardName +
-                " never transitioned to the error state: " + tojson(recipientCollection.findOne());
+            return "recipient shard " + shardName + " never transitioned to the error state: " +
+                tojson(localRecipientOpsCollection.find().toArray());
         });
 }
 
-assertEventuallyErrorsLocally(st.shard0);
-assertEventuallyErrorsLocally(st.shard1);
+const mongos = inputCollection.getMongo();
+const recipientShardNames = reshardingTest.recipientShardNames;
 
-const configPrimary = st.configRS.getPrimary().getDB("config");
-const ops = assert.commandWorked(configPrimary.currentOp({appName: kAppName}));
-assert.eq(1, ops.inprog.length, () => {
-    return "failed to find _configsvrReshardCollection command in: " +
-        tojson(configPrimary.currentOp());
-});
+const topology = DiscoverTopology.findConnectedNodes(mongos);
+const recipient0 = new Mongo(topology.shards[recipientShardNames[0]].primary);
+
+const fp = configureFailPoint(recipient0, "removeRecipientDocFailpoint");
 
-assert.commandWorked(configPrimary.killOp(ops.inprog[0].opid));
-thread.join();
+// In the current implementation, the reshardCollection command won't ever complete if one of the
+// donor or recipient shards encounters an unrecoverable error. To work around this limitation, we
+// verify the recipient shard transitioned itself into the "error" state as a result of the
+// duplicate key error during resharding's collection cloning.
+//
+// TODO SERVER-50584: Remove the call to interruptReshardingThread() from this test and instead
+// directly assert that the reshardCollection command fails with an error.
+reshardingTest.withReshardingInBackground(  //
+    {
+        newShardKeyPattern: {newKey: 1},
+        newChunks: [{min: {newKey: MinKey}, max: {newKey: MaxKey}, shard: recipientShardNames[0]}],
+    },
+    () => {
+        assertEventuallyErrorsLocally(recipient0, recipientShardNames[0]);
+        reshardingTest.interruptReshardingThread();
+    },
+    ErrorCodes.Interrupted);
 
-failPoint1.off();
-failPoint2.off();
+fp.off();
 
-st.stop();
+reshardingTest.teardown();
 })();
diff --git a/jstests/sharding/resharding_clones_initial_data.js b/jstests/sharding/resharding_clones_initial_data.js
index 7e1f9651300..9e87ef644c6 100644
--- a/jstests/sharding/resharding_clones_initial_data.js
+++ b/jstests/sharding/resharding_clones_initial_data.js
@@ -9,65 +9,60 @@
 (function() {
 "use strict";
 
-load("jstests/libs/uuid_util.js");
-load("jstests/sharding/libs/create_sharded_collection_util.js");
+load("jstests/libs/discover_topology.js");
+load("jstests/sharding/libs/resharding_test_fixture.js");
 
-const st = new ShardingTest({
-    mongos: 1,
-    config: 1,
-    shards: 2,
-    rs: {nodes: 1},
-    rsOptions: {
-        setParameter: {
-            "failpoint.WTPreserveSnapshotHistoryIndefinitely": tojson({mode: "alwaysOn"}),
-        }
-    }
-});
-
-const inputCollection = st.s.getCollection("reshardingDb.coll");
+const reshardingTest = new ReshardingTest({numDonors: 2, numRecipients: 2, reshardInPlace: true});
 
-CreateShardedCollectionUtil.shardCollectionWithChunks(inputCollection, {oldKey: 1}, [
-    {min: {oldKey: MinKey}, max: {oldKey: 0}, shard: st.shard0.shardName},
-    {min: {oldKey: 0}, max: {oldKey: MaxKey}, shard: st.shard1.shardName},
-]);
+reshardingTest.setup();
 
-assert.commandWorked(inputCollection.insert(
-    [
-        {_id: "stays on shard0", oldKey: -10, newKey: -10},
-        {_id: "moves to shard0", oldKey: 10, newKey: -10},
-        {_id: "moves to shard1", oldKey: -10, newKey: 10},
-        {_id: "stays on shard1", oldKey: 10, newKey: 10},
+const donorShardNames = reshardingTest.donorShardNames;
+const inputCollection = reshardingTest.createShardedCollection({
+    ns: "reshardingDb.coll",
+    shardKeyPattern: {oldKey: 1},
+    chunks: [
+        {min: {oldKey: MinKey}, max: {oldKey: 0}, shard: donorShardNames[0]},
+        {min: {oldKey: 0}, max: {oldKey: MaxKey}, shard: donorShardNames[1]},
     ],
-    {writeConcern: {w: "majority"}}));
+});
 
-assert.commandWorked(st.s.adminCommand({
-    reshardCollection: inputCollection.getFullName(),
-    key: {newKey: 1},
-    _presetReshardedChunks: [
-        {min: {newKey: MinKey}, max: {newKey: 0}, recipientShardId: st.shard0.shardName},
-        {min: {newKey: 0}, max: {newKey: MaxKey}, recipientShardId: st.shard1.shardName},
+assert.commandWorked(inputCollection.insert([
+    {_id: "stays on shard0", oldKey: -10, newKey: -10},
+    {_id: "moves to shard0", oldKey: 10, newKey: -10},
+    {_id: "moves to shard1", oldKey: -10, newKey: 10},
+    {_id: "stays on shard1", oldKey: 10, newKey: 10},
+]));
+
+const recipientShardNames = reshardingTest.recipientShardNames;
+reshardingTest.withReshardingInBackground({
+    newShardKeyPattern: {newKey: 1},
+    newChunks: [
+        {min: {newKey: MinKey}, max: {newKey: 0}, shard: recipientShardNames[0]},
+        {min: {newKey: 0}, max: {newKey: MaxKey}, shard: recipientShardNames[1]},
     ],
-}));
+});
 
-function assertClonedContents(shard, expectedDocs) {
+function assertClonedContents(shardConn, expectedDocs) {
     // We sort by oldKey so the order of `expectedDocs` can be deterministic.
-    assert.eq(expectedDocs,
-              shard.rs.getPrimary()
-                  .getCollection(inputCollection.getFullName())
-                  .find()
-                  .sort({oldKey: 1})
-                  .toArray());
+    assert.eq(
+        expectedDocs,
+        shardConn.getCollection(inputCollection.getFullName()).find().sort({oldKey: 1}).toArray());
 }
 
-assertClonedContents(st.shard0, [
+const mongos = inputCollection.getMongo();
+const topology = DiscoverTopology.findConnectedNodes(mongos);
+const recipient0 = new Mongo(topology.shards[recipientShardNames[0]].primary);
+const recipient1 = new Mongo(topology.shards[recipientShardNames[1]].primary);
+
+assertClonedContents(recipient0, [
     {_id: "stays on shard0", oldKey: -10, newKey: -10},
     {_id: "moves to shard0", oldKey: 10, newKey: -10},
 ]);
 
-assertClonedContents(st.shard1, [
+assertClonedContents(recipient1, [
     {_id: "moves to shard1", oldKey: -10, newKey: 10},
     {_id: "stays on shard1", oldKey: 10, newKey: 10},
 ]);
 
-st.stop();
+reshardingTest.teardown();
 })();
diff --git a/jstests/sharding/resharding_replicate_updates_as_insert_delete.js b/jstests/sharding/resharding_replicate_updates_as_insert_delete.js
index e333f32a790..0afa0409b93 100644
--- a/jstests/sharding/resharding_replicate_updates_as_insert_delete.js
+++ b/jstests/sharding/resharding_replicate_updates_as_insert_delete.js
@@ -1,138 +1,148 @@
 //
 // Test to verify that updates that would change the resharding key value are replicated as an
 // insert, delete pair.
-// @tags: [requires_fcv_47]
+// @tags: [
+//   requires_fcv_47,
+//   uses_atclustertime,
+// ]
 //
 
 (function() {
 'use strict';
 
-load("jstests/libs/fail_point_util.js");
-load('jstests/libs/parallel_shell_helpers.js');
-load('jstests/libs/uuid_util.js');
-
-const st = new ShardingTest({mongos: 1, shards: 2});
-const dbName = 'test';
-const collName = 'foo';
-const ns = dbName + '.' + collName;
-const mongos = st.s0;
-
-let testDB = mongos.getDB(dbName);
-let testColl = testDB.foo;
-
-assert.commandWorked(mongos.adminCommand({enableSharding: dbName}));
-st.ensurePrimaryShard(dbName, st.shard0.shardName);
-assert.commandWorked(mongos.adminCommand({shardCollection: ns, key: {x: 1}}));
-assert.commandWorked(mongos.adminCommand({split: ns, middle: {x: 5}}));
-// assert.commandWorked(mongos.adminCommand({ moveChunk: ns, find: { x: 5 }, to: st.shard1.shardName
-// }));
-
-assert.commandWorked(testColl.insert({_id: 0, x: 2, y: 2}));
-let shard0Coll = st.shard0.getCollection(ns);
-assert.eq(shard0Coll.find().itcount(), 1);
-let shard1Coll = st.shard1.getCollection(ns);
-assert.eq(shard1Coll.find().itcount(), 0);
-
-// TODO(SERVER-52620): Remove this simulation section once the reshardCollection command provides
-// the needed setup for this test. Simulate resharding operation conditions on donor.
-let uuid = getUUIDFromListCollections(testDB, collName);
-
-const tempReshardingColl = "system.resharding." + extractUUIDFromObject(uuid);
-const tempReshardingNss = dbName + "." + tempReshardingColl;
-assert.commandWorked(testDB.createCollection(tempReshardingColl));
-assert.commandWorked(mongos.adminCommand({shardCollection: tempReshardingNss, key: {y: 1}}));
-assert.commandWorked(mongos.adminCommand({split: tempReshardingNss, middle: {y: 5}}));
-assert.commandWorked(
-    mongos.adminCommand({moveChunk: tempReshardingNss, find: {y: 5}, to: st.shard1.shardName}));
-
-jsTestLog("Updating resharding fields");
-let donorReshardingFields = {
-    "uuid": uuid,
-    "state": "preparing-to-donate",
-    "donorFields": {"reshardingKey": {y: 1}}
-};
-assert.commandWorked(st.configRS.getPrimary().getDB("config").collections.update(
-    {_id: ns}, {"$set": {"reshardingFields": donorReshardingFields}}));
-
-assert.commandWorked(mongos.adminCommand({moveChunk: ns, find: {x: 5}, to: st.shard1.shardName}));
-
-jsTestLog("Flushing routing table updates");
-assert.commandWorked(st.shard0.adminCommand({_flushDatabaseCacheUpdates: dbName}));
-assert.commandWorked(
-    st.shard0.adminCommand({_flushRoutingTableCacheUpdates: ns, syncFromConfig: true}));
-assert.commandWorked(st.shard1.adminCommand({_flushDatabaseCacheUpdates: dbName}));
-assert.commandWorked(
-    st.shard1.adminCommand({_flushRoutingTableCacheUpdates: ns, syncFromConfig: true}));
-
-assert.commandWorked(st.shard0.adminCommand(
-    {_flushRoutingTableCacheUpdates: tempReshardingNss, syncFromConfig: true}));
-assert.commandWorked(st.shard1.adminCommand(
-    {_flushRoutingTableCacheUpdates: tempReshardingNss, syncFromConfig: true}));
-st.refreshCatalogCacheForNs(mongos, ns);
-
-// TODO(SERVER-52620): Use the actual reshardColleciton command to set up the environment for the
-// test. const donor = st.rs0.getPrimary(); const config = st.configRS.getPrimary();
-
-// const failpoint = configureFailPoint(config, "reshardingFieldsInitialized");
-
-// let reshardCollection = (ns, shard) => {
-//    jsTestLog("Starting reshardCollection: " + ns + " " + shard);
-//    let adminDB = db.getSiblingDB("admin");
-
-//    assert.commandWorked(adminDB.runCommand({
-//        reshardCollection: ns,
-//        key: { y: 1 }
-//    }));
-
-//    jsTestLog("Returned from reshardCollection");
-//};
-
-// const awaitShell = startParallelShell(funWithArgs(reshardCollection, ns, st.shard1.shardName),
-// st.s.port);
-
-// failpoint.wait();
-
-(() => {
-    jsTestLog("Updating doc without a transaction");
-
-    assert.commandFailedWithCode(testColl.update({_id: 0, x: 2}, {$set: {y: 10}}),
-                                 ErrorCodes.IllegalOperation);
-
-    jsTestLog("Updated doc");
-})();
-
-(() => {
-    jsTestLog("Updating doc in a transaction");
-
-    let session = testDB.getMongo().startSession();
-    let sessionDB = session.getDatabase(dbName);
-
-    session.startTransaction();
-    assert.commandWorked(sessionDB.foo.update({_id: 0, x: 2}, {$set: {y: 10}}));
-    session.commitTransaction();
-
-    jsTestLog("Updated doc");
-
-    let donor = st.shard0;
-    let donorLocal = donor.getDB('local');
-    const ts = session.getOperationTime();
-
-    let donorOplog = donorLocal.oplog.rs.find({ts: {$eq: ts}});
-    let oplogEntries = donorOplog.toArray();
-    assert.eq(oplogEntries.length, 1);
-
-    // Verify that the applyOps entry contains a delete followed by an insert for the updated
-    // collection.
-    let applyOps = oplogEntries[0].o.applyOps;
-    assert.eq(applyOps.length, 2);
-    assert.eq(applyOps[0].op, "d");
-    assert.eq(applyOps[0].ns, ns);
-    assert.eq(applyOps[1].op, "i");
-    assert.eq(applyOps[1].ns, ns);
-})();
-
-// TODO(SERVER-52620): Use the actual reshardCollection command to set up the environment for the
-// test. failpoint.off(); awaitShell();
-
-st.stop();
+load('jstests/libs/discover_topology.js');
+load('jstests/sharding/libs/resharding_test_fixture.js');
+
+const reshardingTest = new ReshardingTest({numDonors: 2, numRecipients: 2, reshardInPlace: true});
+reshardingTest.setup();
+
+const donorShardNames = reshardingTest.donorShardNames;
+const testColl = reshardingTest.createShardedCollection({
+    ns: 'test.foo',
+    shardKeyPattern: {x: 1},
+    chunks: [
+        {min: {x: MinKey}, max: {x: 5}, shard: donorShardNames[0]},
+        {min: {x: 5}, max: {x: MaxKey}, shard: donorShardNames[1]},
+    ],
+});
+
+const docToUpdate = ({_id: 0, x: 2, y: 2});
+assert.commandWorked(testColl.insert(docToUpdate));
+
+let retryableWriteTs;
+let txnWriteTs;
+
+const mongos = testColl.getMongo();
+const recipientShardNames = reshardingTest.recipientShardNames;
+reshardingTest.withReshardingInBackground(  //
+    {
+        newShardKeyPattern: {y: 1},
+        newChunks: [
+            {min: {y: MinKey}, max: {y: 5}, shard: recipientShardNames[0]},
+            {min: {y: 5}, max: {y: MaxKey}, shard: recipientShardNames[1]},
+        ],
+    },
+    (tempNs) => {
+        // Wait for cloning to have finished on both recipient shards to know that the donor shards
+        // have begun including the "destinedRecipient" field in their oplog entries. It would
+        // technically be sufficient to only wait for cloning to have *started*, but querying the
+        // temporary resharding collection through mongos may cause the RecipientStateMachine to
+        // never be constructed on recipientShardNames[0].
+        //
+        // TODO SERVER-53539: Replace the assert.soon() with the following code.
+        //
+        // const tempColl = mongos.getCollection(tempNs);
+        // assert.soon(() => tempColl.findOne(docToUpdate) !== null);
+        assert.soon(() => {
+            const coordinatorDoc = mongos.getCollection("config.reshardingOperations").findOne({
+                nss: testColl.getFullName()
+            });
+
+            return coordinatorDoc !== null && coordinatorDoc.state === "applying";
+        });
+
+        // TODO SERVER-52683: Change assertion to say the update succeeds. Also capture the
+        // operationTime associated with the write and assert the generated oplog entry is a
+        // delete+insert in an applyOps.
+        assert.commandFailedWithCode(
+            testColl.update({_id: 0, x: 2}, {$set: {y: 10}}),
+            ErrorCodes.IllegalOperation,
+            'was able to update value under new shard key as ordinary write');
+
+        const session = testColl.getMongo().startSession({retryWrites: true});
+        const sessionColl =
+            session.getDatabase(testColl.getDB().getName()).getCollection(testColl.getName());
+
+        // TODO SERVER-52683: Remove the manual retries for the update once mongos is no longer
+        // responsible for converting retryable writes into transactions.
+        let res;
+        assert.soon(
+            () => {
+                res = sessionColl.update({_id: 0, x: 2}, {$set: {y: 20}});
+
+                if (res.nModified === 1) {
+                    assert.commandWorked(res);
+                    retryableWriteTs = session.getOperationTime();
+                    return true;
+                }
+
+                assert.commandFailedWithCode(
+                    res, [ErrorCodes.StaleConfig, ErrorCodes.NoSuchTransaction]);
+                return false;
+            },
+            () => `was unable to update value under new shard key as retryable write: ${
+                tojson(res)}`);
+
+        assert.soon(() => {
+            session.startTransaction();
+            res = sessionColl.update({_id: 0, x: 2}, {$set: {y: -30}});
+
+            if (res.nModified === 1) {
+                session.commitTransaction();
+                txnWriteTs = session.getOperationTime();
+                return true;
+            }
+
+            // mongos will automatically retry the update as a pair of delete and insert commands in
+            // a multi-document transaction. We permit NoSuchTransaction errors because it is
+            // possible for the resharding operation running in the background to cause the shard
+            // version to be bumped. The StaleConfig error won't be automatically retried by mongos
+            // for the second statement in the transaction (the insert) and would lead to a
+            // NoSuchTransaction error.
+            //
+            // TODO SERVER-52683: Remove the manual retries for the update once mongos is no longer
+            // responsible for converting the update into a delete + insert.
+            assert.commandFailedWithCode(res, ErrorCodes.NoSuchTransaction);
+            session.abortTransaction();
+            return false;
+        }, () => `was unable to update value under new shard key in transaction: ${tojson(res)}`);
+
+        // TODO SERVER-49907: Remove the call to interruptReshardingThread() from this test once
+        // recipient shards no longer error on applyOps oplog entries.
+        reshardingTest.interruptReshardingThread();
+    },
+    ErrorCodes.Interrupted);
+
+const topology = DiscoverTopology.findConnectedNodes(mongos);
+const donor0 = new Mongo(topology.shards[donorShardNames[0]].primary);
+const donorOplogColl0 = donor0.getCollection('local.oplog.rs');
+
+function assertOplogEntryIsDeleteInsertApplyOps(entry) {
+    assert(entry.o.hasOwnProperty('applyOps'), entry);
+    assert.eq(entry.o.applyOps.length, 2, entry);
+    assert.eq(entry.o.applyOps[0].op, 'd', entry);
+    assert.eq(entry.o.applyOps[0].ns, testColl.getFullName(), entry);
+    assert.eq(entry.o.applyOps[1].op, 'i', entry);
+    assert.eq(entry.o.applyOps[1].ns, testColl.getFullName(), entry);
+}
+
+const retryableWriteEntry = donorOplogColl0.findOne({ts: retryableWriteTs});
+assert.neq(null, retryableWriteEntry, 'failed to find oplog entry for retryable write');
+assertOplogEntryIsDeleteInsertApplyOps(retryableWriteEntry);
+
+const txnWriteEntry = donorOplogColl0.findOne({ts: txnWriteTs});
+assert.neq(null, txnWriteEntry, 'failed to find oplog entry for transaction');
+assertOplogEntryIsDeleteInsertApplyOps(txnWriteEntry);
+
+reshardingTest.teardown();
 })();
diff --git a/jstests/sharding/test_resharding_test_fixture_using_with_syntax.js b/jstests/sharding/test_resharding_test_fixture_using_with_syntax.js
new file mode 100644
index 00000000000..e42fe9565da
--- /dev/null
+++ b/jstests/sharding/test_resharding_test_fixture_using_with_syntax.js
@@ -0,0 +1,51 @@
+/**
+ * Test for the ReshardingTest fixture itself.
+ *
+ * Verifies that an uncaught exception in withReshardingInBackground() won't cause the mongo shell
+ * to abort.
+ *
+ * @tags: [
+ *   requires_fcv_49,
+ *   uses_atclustertime,
+ * ]
+ */
+(function() {
+"use strict";
+
+if (_isWindows()) {
+    jsTest.log("Skipping test on Windows because it makes assumptions about exit codes for" +
+               " std::terminate()");
+    return;
+}
+
+const awaitShell = startParallelShell(function() {
+    load("jstests/sharding/libs/resharding_test_fixture.js");
+
+    const reshardingTest = new ReshardingTest();
+    reshardingTest.setup();
+
+    const ns = "reshardingDb.coll";
+    const donorShardNames = reshardingTest.donorShardNames;
+    reshardingTest.createShardedCollection({
+        ns,
+        shardKeyPattern: {oldKey: 1},
+        chunks: [{min: {oldKey: MinKey}, max: {oldKey: MaxKey}, shard: donorShardNames[0]}],
+    });
+
+    const recipientShardNames = reshardingTest.recipientShardNames;
+    reshardingTest.withReshardingInBackground(
+        {
+            newShardKeyPattern: {newKey: 1},
+            newChunks: [
+                {min: {newKey: MinKey}, max: {newKey: MaxKey}, shard: recipientShardNames[0]},
+            ],
+        },
+        () => {
+            throw new Error("Intentionally throwing exception to simulate assertion failure");
+        });
+}, undefined, true);
+
+const exitCode = awaitShell({checkExitSuccess: false});
+assert.neq(exitCode, 0);
+assert.neq(exitCode, MongoRunner.EXIT_ABORT);
+})();
diff --git a/src/mongo/s/client/shard_remote.cpp b/src/mongo/s/client/shard_remote.cpp
index 30ceec26868..8a99723e7f6 100644
--- a/src/mongo/s/client/shard_remote.cpp
+++ b/src/mongo/s/client/shard_remote.cpp
@@ -455,8 +455,13 @@ Status ShardRemote::runAggregation(
             }
         }
 
-        if (!callback(data.documents)) {
-            *nextAction = Fetcher::NextAction::kNoAction;
+        try {
+            if (!callback(data.documents)) {
+                *nextAction = Fetcher::NextAction::kNoAction;
+            }
+        } catch (...) {
+            status = exceptionToStatus();
+            return;
         }
 
         status = Status::OK();