SERVER-59292 completion future can be set more than once because of WithAutomaticRetry

(cherry picked from commit 2132263cf6983b42f8494715a7e915cb66106be0)

2 files changed, 96 insertions, 7 deletions

diff --git a/src/mongo/db/s/resharding/resharding_coordinator_service.cpp b/src/mongo/db/s/resharding/resharding_coordinator_service.cpp
index 3ddaae7ff63..e945219b93a 100644
--- a/src/mongo/db/s/resharding/resharding_coordinator_service.cpp
+++ b/src/mongo/db/s/resharding/resharding_coordinator_service.cpp
@@ -1458,9 +1458,12 @@ void ReshardingCoordinatorService::ReshardingCoordinator::onOkayToEnterCritical(
 
 void ReshardingCoordinatorService::ReshardingCoordinator::_insertCoordDocAndChangeOrigCollEntry() {
     if (_coordinatorDoc.getState() > CoordinatorStateEnum::kUnused) {
-        _coordinatorDocWrittenPromise.emplaceValue();
-        ReshardingMetrics::get(cc().getServiceContext())
-            ->onStepUp(ReshardingMetrics::Role::kCoordinator);
+        if (!_coordinatorDocWrittenPromise.getFuture().isReady()) {
+            _coordinatorDocWrittenPromise.emplaceValue();
+            ReshardingMetrics::get(cc().getServiceContext())
+                ->onStepUp(ReshardingMetrics::Role::kCoordinator);
+        }
+
         return;
     }
 
@@ -1470,11 +1473,15 @@ void ReshardingCoordinatorService::ReshardingCoordinator::_insertCoordDocAndChan
     resharding::insertCoordDocAndChangeOrigCollEntry(opCtx.get(), updatedCoordinatorDoc);
     installCoordinatorDoc(opCtx.get(), updatedCoordinatorDoc);
 
-    _coordinatorDocWrittenPromise.emplaceValue();
+    {
+        // Note: don't put blocking or interruptible code in this block.
+        _coordinatorDocWrittenPromise.emplaceValue();
+
+        // TODO SERVER-53914 to accommodate loading metrics for the coordinator.
+        ReshardingMetrics::get(cc().getServiceContext())
+            ->onStart(ReshardingMetrics::Role::kCoordinator, getCurrentTime());
+    }
 
-    // TODO SERVER-53914 to accommodate loading metrics for the coordinator.
-    ReshardingMetrics::get(cc().getServiceContext())
-        ->onStart(ReshardingMetrics::Role::kCoordinator, getCurrentTime());
     pauseBeforeInsertCoordinatorDoc.pauseWhileSet();
 }
 
diff --git a/src/mongo/db/s/resharding/resharding_coordinator_service_test.cpp b/src/mongo/db/s/resharding/resharding_coordinator_service_test.cpp
index 806639af7dc..c04bfce2fbd 100644
--- a/src/mongo/db/s/resharding/resharding_coordinator_service_test.cpp
+++ b/src/mongo/db/s/resharding/resharding_coordinator_service_test.cpp
@@ -183,6 +183,8 @@ public:
     }
 
     void tearDown() override {
+        globalFailPointRegistry().disableAllFailpoints();
+
         TransactionCoordinatorService::get(operationContext())->onStepDown();
         WaitForMajorityService::get(getServiceContext()).shutDown();
         ConfigServerTestFixture::tearDown();
@@ -484,6 +486,24 @@ public:
         }
     }
 
+    void killAllReshardingCoordinatorOps() {
+        for (ServiceContext::LockedClientsCursor cursor(getServiceContext());
+             Client* client = cursor.next();) {
+            invariant(client);
+
+            stdx::lock_guard<Client> lk(*client);
+            if (auto opCtx = client->getOperationContext()) {
+                StringData desc(client->desc());
+
+                // Resharding coordinator doc changes are run under WithTransaction, which uses
+                // AlternativeSessionRegion.
+                if (desc.find("alternative-session-region") != std::string::npos) {
+                    getServiceContext()->killOperation(lk, opCtx);
+                }
+            }
+        }
+    }
+
     repl::PrimaryOnlyService* _service = nullptr;
 
     std::shared_ptr<CoordinatorStateTransitionController> _controller;
@@ -579,6 +599,68 @@ TEST_F(ReshardingCoordinatorServiceTest, ReshardingCoordinatorSuccessfullyTransi
     coordinator->getCompletionFuture().get(opCtx);
 }
 
+TEST_F(ReshardingCoordinatorServiceTest, ReshardingCoordinatorTransitionsTokDoneWithInterrupt) {
+    const std::vector<CoordinatorStateEnum> coordinatorStates{
+        CoordinatorStateEnum::kPreparingToDonate,
+        CoordinatorStateEnum::kCloning,
+        CoordinatorStateEnum::kApplying,
+        CoordinatorStateEnum::kBlockingWrites,
+        CoordinatorStateEnum::kCommitting};
+    PauseDuringStateTransitions stateTransitionsGuard{controller(), coordinatorStates};
+
+    auto doc = insertStateAndCatalogEntries(CoordinatorStateEnum::kUnused, _originalEpoch);
+    auto opCtx = operationContext();
+    auto donorChunk = makeAndInsertChunksForDonorShard(
+        _originalUUID, _originalEpoch, _oldShardKey, std::vector{OID::gen(), OID::gen()});
+
+    auto initialChunks =
+        makeChunks(_reshardingUUID, _tempEpoch, _newShardKey, std::vector{OID::gen(), OID::gen()});
+
+    std::vector<ReshardedChunk> presetReshardedChunks;
+    for (const auto& chunk : initialChunks) {
+        presetReshardedChunks.emplace_back(chunk.getShard(), chunk.getMin(), chunk.getMax());
+    }
+
+    doc.setPresetReshardedChunks(presetReshardedChunks);
+
+    auto coordinator = ReshardingCoordinator::getOrCreate(opCtx, _service, doc.toBSON());
+
+    stateTransitionsGuard.wait(CoordinatorStateEnum::kPreparingToDonate);
+    killAllReshardingCoordinatorOps();
+    stateTransitionsGuard.unset(CoordinatorStateEnum::kPreparingToDonate);
+    waitUntilCommittedCoordinatorDocReach(opCtx, CoordinatorStateEnum::kPreparingToDonate);
+    makeDonorsReadyToDonate(opCtx);
+
+    stateTransitionsGuard.wait(CoordinatorStateEnum::kCloning);
+    killAllReshardingCoordinatorOps();
+    stateTransitionsGuard.unset(CoordinatorStateEnum::kCloning);
+    waitUntilCommittedCoordinatorDocReach(opCtx, CoordinatorStateEnum::kCloning);
+
+    makeRecipientsFinishedCloning(opCtx);
+    stateTransitionsGuard.wait(CoordinatorStateEnum::kApplying);
+    killAllReshardingCoordinatorOps();
+    stateTransitionsGuard.unset(CoordinatorStateEnum::kApplying);
+    waitUntilCommittedCoordinatorDocReach(opCtx, CoordinatorStateEnum::kApplying);
+
+    coordinator->onOkayToEnterCritical();
+    stateTransitionsGuard.wait(CoordinatorStateEnum::kBlockingWrites);
+    killAllReshardingCoordinatorOps();
+    stateTransitionsGuard.unset(CoordinatorStateEnum::kBlockingWrites);
+    waitUntilCommittedCoordinatorDocReach(opCtx, CoordinatorStateEnum::kBlockingWrites);
+
+    makeRecipientsBeInStrictConsistency(opCtx);
+
+    stateTransitionsGuard.wait(CoordinatorStateEnum::kCommitting);
+    stateTransitionsGuard.unset(CoordinatorStateEnum::kCommitting);
+
+    waitUntilCommittedCoordinatorDocReach(opCtx, CoordinatorStateEnum::kCommitting);
+
+    makeDonorsProceedToDone(opCtx);
+    makeRecipientsProceedToDone(opCtx);
+
+    coordinator->getCompletionFuture().get(opCtx);
+}
+
 TEST_F(ReshardingCoordinatorServiceTest, StepDownStepUpDuringInitializing) {
     PauseDuringStateTransitions stateTransitionsGuard{controller(),
                                                       CoordinatorStateEnum::kPreparingToDonate};