SERVER-62296 MoveChunk should recover any unfinished migration before starting a new one

3 files changed, 36 insertions, 0 deletions

diff --git a/src/mongo/db/s/migration_source_manager.cpp b/src/mongo/db/s/migration_source_manager.cpp
index 0a0f0ab82e4..4d2be1c2081 100644
--- a/src/mongo/db/s/migration_source_manager.cpp
+++ b/src/mongo/db/s/migration_source_manager.cpp
@@ -176,6 +176,18 @@ MigrationSourceManager::MigrationSourceManager(OperationContext* opCtx,
     // command.
     onShardVersionMismatch(_opCtx, _args.getNss(), boost::none);
 
+    // Complete any unfinished migration pending recovery
+    {
+        migrationutil::drainMigrationsPendingRecovery(opCtx);
+
+        // Since the moveChunk command is holding the ActiveMigrationRegistry and we just drained
+        // all migrations pending recovery, now there cannot be any document in
+        // config.migrationCoordinators.
+        PersistentTaskStore<MigrationCoordinatorDocument> store(
+            NamespaceString::kMigrationCoordinatorsNamespace);
+        invariant(store.count(opCtx) == 0);
+    }
+
     // Snapshot the committed metadata from the time the migration starts
     const auto [collectionMetadata, collectionUUID] = [&] {
         UninterruptibleLockGuard noInterrupt(_opCtx->lockState());
diff --git a/src/mongo/db/s/migration_util.cpp b/src/mongo/db/s/migration_util.cpp
index 05896b00291..61fe9f728eb 100644
--- a/src/mongo/db/s/migration_util.cpp
+++ b/src/mongo/db/s/migration_util.cpp
@@ -1177,5 +1177,23 @@ void resumeMigrationRecipientsOnStepUp(OperationContext* opCtx) {
                 "ongoingRecipientCritSecCount"_attr = ongoingMigrationRecipientsCount);
 }
 
+void drainMigrationsPendingRecovery(OperationContext* opCtx) {
+    PersistentTaskStore<MigrationCoordinatorDocument> store(
+        NamespaceString::kMigrationCoordinatorsNamespace);
+
+    while (store.count(opCtx)) {
+        store.forEach(opCtx, BSONObj(), [opCtx](const MigrationCoordinatorDocument& doc) {
+            try {
+                onShardVersionMismatch(opCtx, doc.getNss(), boost::none);
+            } catch (DBException& ex) {
+                ex.addContext(str::stream() << "Failed to recover pending migration for document "
+                                            << doc.toBSON());
+                throw;
+            }
+            return true;
+        });
+    }
+}
+
 }  // namespace migrationutil
 }  // namespace mongo
diff --git a/src/mongo/db/s/migration_util.h b/src/mongo/db/s/migration_util.h
index 2b16a601bb2..4b956c8a6ab 100644
--- a/src/mongo/db/s/migration_util.h
+++ b/src/mongo/db/s/migration_util.h
@@ -254,5 +254,11 @@ void deleteMigrationRecipientRecoveryDocument(OperationContext* opCtx, const UUI
  */
 void resumeMigrationRecipientsOnStepUp(OperationContext* opCtx);
 
+/**
+ * Recovers all unfinished migrations pending recovery.
+ * Note: This method assumes its caller is preventing new migrations from starting.
+ */
+void drainMigrationsPendingRecovery(OperationContext* opCtx);
+
 }  // namespace migrationutil
 }  // namespace mongo