SERVER-54681 Delay start of txnCloners to prevent early write of kIncompleteHistoryStmtId during resharding operation

8 files changed, 32 insertions, 19 deletions

diff --git a/src/mongo/db/s/collection_metadata_test.cpp b/src/mongo/db/s/collection_metadata_test.cpp
index 5bbf258a5ab..b5f9e77045d 100644
--- a/src/mongo/db/s/collection_metadata_test.cpp
+++ b/src/mongo/db/s/collection_metadata_test.cpp
@@ -118,7 +118,7 @@ protected:
 
         if (state == CoordinatorStateEnum::kDecisionPersisted) {
             TypeCollectionRecipientFields recipientFields{
-                {kThisShard, kOtherShard}, existingUuid, kNss};
+                {kThisShard, kOtherShard}, existingUuid, kNss, 5000};
             reshardingFields.setRecipientFields(std::move(recipientFields));
         } else if (state == CoordinatorStateEnum::kBlockingWrites) {
             TypeCollectionDonorFields donorFields{
diff --git a/src/mongo/db/s/resharding/recipient_document.idl b/src/mongo/db/s/resharding/recipient_document.idl
index 4c56423307e..5efe3d36deb 100644
--- a/src/mongo/db/s/resharding/recipient_document.idl
+++ b/src/mongo/db/s/resharding/recipient_document.idl
@@ -54,3 +54,5 @@ structs:
             donorShards:
                 type: array<shard_id>
                 description: "The list of donor shards that report to this recipient."
+            minimumOperationDurationMillis:
+                type: long
diff --git a/src/mongo/db/s/resharding/resharding_coordinator_service.cpp b/src/mongo/db/s/resharding/resharding_coordinator_service.cpp
index 88aa8f87930..7811e6db54a 100644
--- a/src/mongo/db/s/resharding/resharding_coordinator_service.cpp
+++ b/src/mongo/db/s/resharding/resharding_coordinator_service.cpp
@@ -197,10 +197,14 @@ void writeToCoordinatorStateNss(OperationContext* opCtx,
  */
 TypeCollectionRecipientFields constructRecipientFields(
     const ReshardingCoordinatorDocument& coordinatorDoc) {
-    auto donorShardIds = resharding::extractShardIds(coordinatorDoc.getDonorShards());
     TypeCollectionRecipientFields recipientFields(
-        std::move(donorShardIds), coordinatorDoc.getSourceUUID(), coordinatorDoc.getSourceNss());
+        resharding::extractShardIds(coordinatorDoc.getDonorShards()),
+        coordinatorDoc.getSourceUUID(),
+        coordinatorDoc.getSourceNss(),
+        resharding::gReshardingMinimumOperationDurationMillis.load());
+
     emplaceFetchTimestampIfExists(recipientFields, coordinatorDoc.getFetchTimestamp());
+
     return recipientFields;
 }
 
diff --git a/src/mongo/db/s/resharding/resharding_donor_recipient_common.cpp b/src/mongo/db/s/resharding/resharding_donor_recipient_common.cpp
index 18d10d50547..4ac8df3cd22 100644
--- a/src/mongo/db/s/resharding/resharding_donor_recipient_common.cpp
+++ b/src/mongo/db/s/resharding/resharding_donor_recipient_common.cpp
@@ -333,7 +333,9 @@ ReshardingRecipientDocument constructRecipientDocumentFromReshardingFields(
     recipientCtx.setState(RecipientStateEnum::kAwaitingFetchTimestamp);
 
     auto recipientDoc = ReshardingRecipientDocument{
-        std::move(recipientCtx), reshardingFields.getRecipientFields()->getDonorShardIds()};
+        std::move(recipientCtx),
+        reshardingFields.getRecipientFields()->getDonorShardIds(),
+        reshardingFields.getRecipientFields()->getMinimumOperationDurationMillis()};
 
     auto sourceNss = reshardingFields.getRecipientFields()->getSourceNss();
     auto sourceUUID = reshardingFields.getRecipientFields()->getSourceUUID();
diff --git a/src/mongo/db/s/resharding/resharding_donor_recipient_common_test.h b/src/mongo/db/s/resharding/resharding_donor_recipient_common_test.h
index d3ecac2041b..b707a975444 100644
--- a/src/mongo/db/s/resharding/resharding_donor_recipient_common_test.h
+++ b/src/mongo/db/s/resharding/resharding_donor_recipient_common_test.h
@@ -146,7 +146,7 @@ protected:
         const NamespaceString& originalNss,
         const boost::optional<Timestamp>& fetchTimestamp = boost::none) {
         auto recipientFields =
-            TypeCollectionRecipientFields(donorShardIds, existingUUID, originalNss);
+            TypeCollectionRecipientFields(donorShardIds, existingUUID, originalNss, 5000);
         emplaceFetchTimestampIfExists(recipientFields, fetchTimestamp);
         fields.setRecipientFields(std::move(recipientFields));
     }
diff --git a/src/mongo/db/s/resharding/resharding_recipient_service.cpp b/src/mongo/db/s/resharding/resharding_recipient_service.cpp
index d27bca9afec..dd1761a6742 100644
--- a/src/mongo/db/s/resharding/resharding_recipient_service.cpp
+++ b/src/mongo/db/s/resharding/resharding_recipient_service.cpp
@@ -226,6 +226,7 @@ ReshardingRecipientService::RecipientStateMachine::RecipientStateMachine(
     : repl::PrimaryOnlyService::TypedInstance<RecipientStateMachine>(),
       _metadata{recipientDoc.getCommonReshardingMetadata()},
       _donorShardIds{recipientDoc.getDonorShards()},
+      _minimumOperationDuration{Milliseconds{recipientDoc.getMinimumOperationDurationMillis()}},
       _recipientCtx{recipientDoc.getMutableState()},
       _fetchTimestamp{recipientDoc.getFetchTimestamp()} {}
 
@@ -433,9 +434,7 @@ void ReshardingRecipientService::RecipientStateMachine::_initTxnCloner(
         catalogCache->getShardedCollectionRoutingInfo(opCtx, _metadata.getSourceNss());
     std::set<ShardId> shardList;
 
-    const auto myShardId = ShardingState::get(opCtx)->shardId();
     routingInfo.getAllShardIds(&shardList);
-    shardList.erase(myShardId);
 
     for (const auto& shard : shardList) {
         _txnCloners.push_back(std::make_unique<ReshardingTxnCloner>(
@@ -509,21 +508,25 @@ ReshardingRecipientService::RecipientStateMachine::_cloneThenTransitionToApplyin
                 }));
     }
 
-    return _collectionCloner->run(**executor, abortToken)
-        .then([this, executor, abortToken] {
-            if (_txnCloners.empty()) {
-                return SemiFuture<void>::makeReady();
-            }
+    return whenAllSucceed(_collectionCloner->run(**executor, abortToken),
+                          (*executor)
+                              ->sleepFor(_minimumOperationDuration, abortToken)
+                              .then([this, executor, abortToken] {
+                                  if (_txnCloners.empty()) {
+                                      return SemiFuture<void>::makeReady();
+                                  }
 
-            auto serviceContext = Client::getCurrent()->getServiceContext();
+                                  auto serviceContext = Client::getCurrent()->getServiceContext();
 
-            std::vector<ExecutorFuture<void>> txnClonerFutures;
-            for (auto&& txnCloner : _txnCloners) {
-                txnClonerFutures.push_back(txnCloner->run(serviceContext, **executor, abortToken));
-            }
+                                  std::vector<ExecutorFuture<void>> txnClonerFutures;
+                                  for (auto&& txnCloner : _txnCloners) {
+                                      txnClonerFutures.push_back(
+                                          txnCloner->run(serviceContext, **executor, abortToken));
+                                  }
 
-            return whenAllSucceed(std::move(txnClonerFutures));
-        })
+                                  return whenAllSucceed(std::move(txnClonerFutures));
+                              }))
+        .thenRunOn(**executor)
         .then([this] {
             // ReshardingTxnCloners must complete before the recipient transitions to kApplying to
             // avoid errors caused by donor shards unpinning the fetchTimestamp.
diff --git a/src/mongo/db/s/resharding/resharding_recipient_service.h b/src/mongo/db/s/resharding/resharding_recipient_service.h
index 1e667da7195..8460178d087 100644
--- a/src/mongo/db/s/resharding/resharding_recipient_service.h
+++ b/src/mongo/db/s/resharding/resharding_recipient_service.h
@@ -205,6 +205,7 @@ private:
     // config.localReshardingOperations.recipient.
     const CommonReshardingMetadata _metadata;
     const std::vector<ShardId> _donorShardIds;
+    const Milliseconds _minimumOperationDuration;
 
     // The in-memory representation of the mutable portion of the document in
     // config.localReshardingOperations.recipient.
diff --git a/src/mongo/db/s/resharding/resharding_recipient_service_test.cpp b/src/mongo/db/s/resharding/resharding_recipient_service_test.cpp
index 5d73cd21ddd..cc725f3b3d6 100644
--- a/src/mongo/db/s/resharding/resharding_recipient_service_test.cpp
+++ b/src/mongo/db/s/resharding/resharding_recipient_service_test.cpp
@@ -154,6 +154,7 @@ public:
             // Populating the set of donor shard ids isn't necessary to test the functionality of
             // creating the temporary resharding collection.
             recipientFields.setDonorShardIds({});
+            recipientFields.setMinimumOperationDurationMillis(5000);
 
             reshardingFields.setRecipientFields(recipientFields);
             coll.setReshardingFields(reshardingFields);