SERVER-69890 Concurrent movePrimary and removeShard can move database to a no-longer existent shard

2 files changed, 20 insertions, 6 deletions

diff --git a/src/mongo/db/s/config/sharding_catalog_manager.h b/src/mongo/db/s/config/sharding_catalog_manager.h
index bb0bc6e6b40..c267dd0cbaf 100644
--- a/src/mongo/db/s/config/sharding_catalog_manager.h
+++ b/src/mongo/db/s/config/sharding_catalog_manager.h
@@ -412,7 +412,7 @@ public:
     void commitMovePrimary(OperationContext* opCtx,
                            const DatabaseName& dbName,
                            const DatabaseVersion& expectedDbVersion,
-                           const ShardId& toShard);
+                           const ShardId& toShardId);
 
     //
     // Collection Operations
diff --git a/src/mongo/db/s/config/sharding_catalog_manager_database_operations.cpp b/src/mongo/db/s/config/sharding_catalog_manager_database_operations.cpp
index 789aa1a03ce..262b6d69edb 100644
--- a/src/mongo/db/s/config/sharding_catalog_manager_database_operations.cpp
+++ b/src/mongo/db/s/config/sharding_catalog_manager_database_operations.cpp
@@ -289,11 +289,25 @@ DatabaseType ShardingCatalogManager::createDatabase(
 void ShardingCatalogManager::commitMovePrimary(OperationContext* opCtx,
                                                const DatabaseName& dbName,
                                                const DatabaseVersion& expectedDbVersion,
-                                               const ShardId& toShard) {
+                                               const ShardId& toShardId) {
     // Hold the shard lock until the entire commit finishes to serialize with removeShard.
     Lock::SharedLock shardLock(opCtx, _kShardMembershipLock);
 
-    const auto transactionChain = [opCtx, dbName, expectedDbVersion, toShard](
+    const auto toShardDoc = [&] {
+        DBDirectClient dbClient(opCtx);
+        return dbClient.findOne(NamespaceString::kConfigsvrShardsNamespace,
+                                BSON(ShardType::name << toShardId));
+    }();
+    uassert(ErrorCodes::ShardNotFound,
+            "Requested primary shard {} does not exist"_format(toShardId.toString()),
+            !toShardDoc.isEmpty());
+
+    const auto toShardEntry = uassertStatusOK(ShardType::fromBSON(toShardDoc));
+    uassert(ErrorCodes::ShardNotFound,
+            "Requested primary shard {} is draining"_format(toShardId.toString()),
+            !toShardEntry.getDraining());
+
+    const auto transactionChain = [opCtx, dbName, expectedDbVersion, toShardId](
                                       const txn_api::TransactionClient& txnClient,
                                       ExecutorPtr txnExec) {
         const auto updateDatabaseEntryOp = [&] {
@@ -313,7 +327,7 @@ void ShardingCatalogManager::commitMovePrimary(OperationContext* opCtx,
                 const auto newDbVersion = expectedDbVersion.makeUpdated();
 
                 BSONObjBuilder bsonBuilder;
-                bsonBuilder.append(DatabaseType::kPrimaryFieldName, toShard);
+                bsonBuilder.append(DatabaseType::kPrimaryFieldName, toShardId);
                 bsonBuilder.append(DatabaseType::kVersionFieldName, newDbVersion.toBSON());
                 return BSON("$set" << bsonBuilder.obj());
             }();
@@ -331,7 +345,7 @@ void ShardingCatalogManager::commitMovePrimary(OperationContext* opCtx,
 
         return txnClient.runCRUDOp(updateDatabaseEntryOp, {})
             .thenRunOn(txnExec)
-            .then([&txnClient, &txnExec, opCtx, &dbName, toShard](
+            .then([&txnClient, &txnExec, opCtx, &dbName, toShardId](
                       const BatchedCommandResponse& updateCatalogDatabaseEntryResponse) {
                 uassertStatusOK(updateCatalogDatabaseEntryResponse.toStatus());
 
@@ -352,7 +366,7 @@ void ShardingCatalogManager::commitMovePrimary(OperationContext* opCtx,
                 const auto clusterTime = now.clusterTime().asTimestamp();
 
                 NamespacePlacementType placementInfo(
-                    NamespaceString(dbName), clusterTime, std::vector<mongo::ShardId>{toShard});
+                    NamespaceString(dbName), clusterTime, std::vector<mongo::ShardId>{toShardId});
 
                 write_ops::InsertCommandRequest insertPlacementHistoryOp(
                     NamespaceString::kConfigsvrPlacementHistoryNamespace);