SERVER-73217 Use idl for config.shard.collections entries

6 files changed, 182 insertions, 90 deletions

diff --git a/src/mongo/db/commands/set_feature_compatibility_version_command.cpp b/src/mongo/db/commands/set_feature_compatibility_version_command.cpp
index d97a1bead71..fed4f6db247 100644
--- a/src/mongo/db/commands/set_feature_compatibility_version_command.cpp
+++ b/src/mongo/db/commands/set_feature_compatibility_version_command.cpp
@@ -73,6 +73,7 @@
 #include "mongo/db/s/resharding/coordinator_document_gen.h"
 #include "mongo/db/s/resharding/resharding_coordinator_service.h"
 #include "mongo/db/s/resharding/resharding_donor_recipient_common.h"
+#include "mongo/db/s/shard_authoritative_catalog_gen.h"
 #include "mongo/db/s/sharding_ddl_coordinator_service.h"
 #include "mongo/db/s/sharding_util.h"
 #include "mongo/db/s/transaction_coordinator_service.h"
@@ -988,16 +989,14 @@ private:
                 LOGV2(7013200, "Clearing global indexes for all collections");
                 DBDirectClient client(opCtx);
                 FindCommandRequest findCmd{NamespaceString::kShardCollectionCatalogNamespace};
-                findCmd.setFilter(
-                    BSON(CollectionType::kIndexVersionFieldName << BSON("$exists" << true)));
+                findCmd.setFilter(BSON(ShardAuthoritativeCollectionType::kIndexVersionFieldName
+                                       << BSON("$exists" << true)));
                 auto cursor = client.find(std::move(findCmd));
                 while (cursor->more()) {
                     const auto collectionDoc = cursor->next();
-                    auto collUUID =
-                        uassertStatusOK(UUID::parse(collectionDoc[CollectionType::kUuidFieldName]));
-                    auto collNss =
-                        NamespaceString(collectionDoc[CollectionType::kNssFieldName].String());
-                    clearCollectionGlobalIndexes(opCtx, collNss, collUUID);
+                    auto collection = ShardAuthoritativeCollectionType::parse(
+                        IDLParserContext("FCVDropIndexCatalog"), collectionDoc);
+                    clearCollectionGlobalIndexes(opCtx, collection.getNss(), collection.getUuid());
                 }
 
                 LOGV2(6711905,
@@ -1022,10 +1021,11 @@ private:
                 write_ops::UpdateCommandRequest update(CollectionType::ConfigNS);
                 update.setUpdates({[&]() {
                     write_ops::UpdateOpEntry entry;
-                    entry.setQ(
-                        BSON(CollectionType::kIndexVersionFieldName << BSON("$exists" << true)));
-                    entry.setU(write_ops::UpdateModification::parseFromClassicUpdate(
-                        BSON("$unset" << BSON(CollectionType::kIndexVersionFieldName << true))));
+                    entry.setQ(BSON(ShardAuthoritativeCollectionType::kIndexVersionFieldName
+                                    << BSON("$exists" << true)));
+                    entry.setU(write_ops::UpdateModification::parseFromClassicUpdate(BSON(
+                        "$unset" << BSON(ShardAuthoritativeCollectionType::kIndexVersionFieldName
+                                         << true))));
                     entry.setMulti(true);
                     return entry;
                 }()});
diff --git a/src/mongo/db/s/SConscript b/src/mongo/db/s/SConscript
index 080165690e3..7068f738bc3 100644
--- a/src/mongo/db/s/SConscript
+++ b/src/mongo/db/s/SConscript
@@ -43,6 +43,7 @@ env.Library(
     target='sharding_catalog',
     source=[
         'global_index_ddl_util.cpp',
+        'shard_authoritative_catalog.idl',
     ],
     LIBDEPS_PRIVATE=[
         '$BUILD_DIR/mongo/base',
diff --git a/src/mongo/db/s/global_index_ddl_util.cpp b/src/mongo/db/s/global_index_ddl_util.cpp
index f2ac257377b..cfb1dc4592c 100644
--- a/src/mongo/db/s/global_index_ddl_util.cpp
+++ b/src/mongo/db/s/global_index_ddl_util.cpp
@@ -35,6 +35,7 @@
 #include "mongo/db/op_observer/op_observer.h"
 #include "mongo/db/ops/delete.h"
 #include "mongo/db/ops/update.h"
+#include "mongo/db/s/shard_authoritative_catalog_gen.h"
 #include "mongo/logv2/log.h"
 #include "mongo/s/catalog/type_collection.h"
 #include "mongo/s/catalog/type_index_catalog.h"
@@ -78,29 +79,31 @@ void renameGlobalIndexesMetadata(OperationContext* opCtx,
                                             {NamespaceString::kShardIndexCatalogNamespace}));
             {
                 // First get the document to check the index version if the document already exists
-                const auto queryTo = BSON(CollectionType::kNssFieldName << toNss.ns());
-                BSONObj collectionDoc;
+                const auto queryTo =
+                    BSON(ShardAuthoritativeCollectionType::kNssFieldName << toNss.ns());
+                BSONObj collectionToDoc;
                 bool docExists =
-                    Helpers::findOne(opCtx, collsColl.getCollection(), queryTo, collectionDoc);
-                if (docExists &&
-                    indexVersion <=
-                        collectionDoc[CollectionType::kIndexVersionFieldName].timestamp()) {
-                    LOGV2_DEBUG(
-                        7079500,
-                        1,
-                        "renameGlobalIndexesMetadata has index version older than current "
-                        "collection index version",
-                        "collectionIndexVersion"_attr =
-                            collectionDoc[CollectionType::kIndexVersionFieldName].timestamp(),
-                        "expectedIndexVersion"_attr = indexVersion);
-                    return;
+                    Helpers::findOne(opCtx, collsColl.getCollection(), queryTo, collectionToDoc);
+                if (docExists) {
+                    auto collectionTo = ShardAuthoritativeCollectionType::parse(
+                        IDLParserContext("renameGlobalIndexesMetadata"), collectionToDoc);
+                    auto toIndexVersion =
+                        collectionTo.getIndexVersion().get_value_or(Timestamp(0, 0));
+                    if (indexVersion <= toIndexVersion) {
+                        LOGV2_DEBUG(7079500,
+                                    1,
+                                    "renameGlobalIndexesMetadata has index version older than "
+                                    "current collection index version",
+                                    "collectionIndexVersion"_attr = toIndexVersion,
+                                    "expectedIndexVersion"_attr = indexVersion);
+                        return;
+                    }
+                    toUuid.emplace(collectionTo.getUuid());
                 }
                 // Update the document (or create it) with the new index version
                 repl::UnreplicatedWritesBlock unreplicatedWritesBlock(opCtx);
                 // Save uuid to remove the 'to' indexes later on.
                 if (docExists) {
-                    toUuid =
-                        uassertStatusOK(UUID::parse(collectionDoc[CollectionType::kUuidFieldName]));
                     // Remove the 'to' entry.
                     mongo::deleteObjects(opCtx,
                                          collsColl.getCollection(),
@@ -114,16 +117,20 @@ void renameGlobalIndexesMetadata(OperationContext* opCtx,
                 fassert(7082801,
                         Helpers::findOne(
                             opCtx, collsColl.getCollection(), queryFrom, collectionFromDoc));
-                auto finalDoc = collectionFromDoc.addField(
-                    BSON(CollectionType::kNssFieldName << toNss.ns()).firstElement());
+                auto collectionFrom = ShardAuthoritativeCollectionType::parse(
+                    IDLParserContext("renameGlobalIndexesMetadata"), collectionFromDoc);
+                collectionFrom.setNss(toNss);
 
                 mongo::deleteObjects(opCtx,
                                      collsColl.getCollection(),
                                      NamespaceString::kShardCollectionCatalogNamespace,
                                      queryFrom,
                                      true);
-                uassertStatusOK(collection_internal::insertDocument(
-                    opCtx, collsColl.getCollection(), InsertStatement(finalDoc), nullptr));
+                uassertStatusOK(
+                    collection_internal::insertDocument(opCtx,
+                                                        collsColl.getCollection(),
+                                                        InsertStatement(collectionFrom.toBSON()),
+                                                        nullptr));
             }
             AutoGetCollection idxColl(opCtx, NamespaceString::kShardIndexCatalogNamespace, MODE_IX);
 
@@ -172,24 +179,25 @@ void addGlobalIndexCatalogEntryToCollection(OperationContext* opCtx,
 
             {
                 // First get the document to check the index version if the document already exists
-                const auto query = BSON(CollectionType::kNssFieldName
-                                        << userCollectionNss.ns() << CollectionType::kUuidFieldName
-                                        << collectionUUID);
+                const auto query =
+                    BSON(ShardAuthoritativeCollectionType::kNssFieldName
+                         << userCollectionNss.ns()
+                         << ShardAuthoritativeCollectionType::kUuidFieldName << collectionUUID);
                 BSONObj collectionDoc;
                 bool docExists =
                     Helpers::findOne(opCtx, collsColl.getCollection(), query, collectionDoc);
-                if (docExists && !collectionDoc[CollectionType::kIndexVersionFieldName].eoo() &&
-                    lastmod <= collectionDoc[CollectionType::kIndexVersionFieldName].timestamp()) {
-                    LOGV2_DEBUG(
-                        6712300,
-                        1,
-                        "addGlobalIndexCatalogEntryToCollection has index version older than "
-                        "current "
-                        "collection index version",
-                        "collectionIndexVersion"_attr =
-                            collectionDoc[CollectionType::kIndexVersionFieldName].timestamp(),
-                        "expectedIndexVersion"_attr = lastmod);
-                    return;
+                if (docExists) {
+                    auto collection = ShardAuthoritativeCollectionType::parse(
+                        IDLParserContext("AddIndexCatalogEntry"), collectionDoc);
+                    if (collection.getIndexVersion() && lastmod <= *collection.getIndexVersion()) {
+                        LOGV2_DEBUG(6712300,
+                                    1,
+                                    "addGlobalIndexCatalogEntryToCollection has index version "
+                                    "older than current collection index version",
+                                    "collectionIndexVersion"_attr = *collection.getIndexVersion(),
+                                    "expectedIndexVersion"_attr = lastmod);
+                        return;
+                    }
                 }
                 // Update the document (or create it) with the new index version
                 repl::UnreplicatedWritesBlock unreplicatedWritesBlock(opCtx);
@@ -197,9 +205,10 @@ void addGlobalIndexCatalogEntryToCollection(OperationContext* opCtx,
                 request.setNamespaceString(NamespaceString::kShardCollectionCatalogNamespace);
                 request.setQuery(query);
                 request.setUpdateModification(
-                    BSON(CollectionType::kNssFieldName
-                         << userCollectionNss.ns() << CollectionType::kUuidFieldName
-                         << collectionUUID << CollectionType::kIndexVersionFieldName << lastmod));
+                    BSON(ShardAuthoritativeCollectionType::kNssFieldName
+                         << userCollectionNss.ns()
+                         << ShardAuthoritativeCollectionType::kUuidFieldName << collectionUUID
+                         << ShardAuthoritativeCollectionType::kIndexVersionFieldName << lastmod));
                 request.setUpsert(true);
                 request.setFromOplogApplication(true);
                 mongo::update(opCtx, collsColl.getDb(), request);
@@ -246,33 +255,34 @@ void removeGlobalIndexCatalogEntryFromCollection(OperationContext* opCtx,
                                             {NamespaceString::kShardIndexCatalogNamespace}));
             {
                 // First get the document to check the index version if the document already exists
-                const auto query = BSON(CollectionType::kNssFieldName
-                                        << nss.ns() << CollectionType::kUuidFieldName << uuid);
+                const auto query =
+                    BSON(ShardAuthoritativeCollectionType::kNssFieldName
+                         << nss.ns() << ShardAuthoritativeCollectionType::kUuidFieldName << uuid);
                 BSONObj collectionDoc;
                 bool docExists =
                     Helpers::findOne(opCtx, collsColl.getCollection(), query, collectionDoc);
-                if (docExists && !collectionDoc[CollectionType::kIndexVersionFieldName].eoo() &&
-                    lastmod <= collectionDoc[CollectionType::kIndexVersionFieldName].timestamp()) {
-                    LOGV2_DEBUG(
-                        6712301,
-                        1,
-                        "removeGlobalIndexCatalogEntryFromCollection has index version older than "
-                        "current "
-                        "collection index version",
-                        "collectionIndexVersion"_attr =
-                            collectionDoc[CollectionType::kIndexVersionFieldName].timestamp(),
-                        "expectedIndexVersion"_attr = lastmod);
-                    return;
+                if (docExists) {
+                    auto collection = ShardAuthoritativeCollectionType::parse(
+                        IDLParserContext("RemoveIndexCatalogEntry"), collectionDoc);
+                    if (collection.getIndexVersion() && lastmod <= *collection.getIndexVersion()) {
+                        LOGV2_DEBUG(6712301,
+                                    1,
+                                    "removeGlobalIndexCatalogEntryFromCollection has index version "
+                                    "older than current collection index version",
+                                    "collectionIndexVersion"_attr = *collection.getIndexVersion(),
+                                    "expectedIndexVersion"_attr = lastmod);
+                        return;
+                    }
                 }
                 // Update the document (or create it) with the new index version
                 repl::UnreplicatedWritesBlock unreplicatedWritesBlock(opCtx);
                 auto request = UpdateRequest();
                 request.setNamespaceString(NamespaceString::kShardCollectionCatalogNamespace);
                 request.setQuery(query);
-                request.setUpdateModification(BSON(CollectionType::kNssFieldName
-                                                   << nss.ns() << CollectionType::kUuidFieldName
-                                                   << uuid << CollectionType::kIndexVersionFieldName
-                                                   << lastmod));
+                request.setUpdateModification(
+                    BSON(ShardAuthoritativeCollectionType::kNssFieldName
+                         << nss.ns() << ShardAuthoritativeCollectionType::kUuidFieldName << uuid
+                         << ShardAuthoritativeCollectionType::kIndexVersionFieldName << lastmod));
                 request.setUpsert(true);
                 request.setFromOplogApplication(true);
                 mongo::update(opCtx, collsColl.getDb(), request);
@@ -317,18 +327,19 @@ void replaceCollectionGlobalIndexes(OperationContext* opCtx,
                                             {NamespaceString::kShardIndexCatalogNamespace}));
             {
                 // Set final indexVersion
-                const auto query = BSON(CollectionType::kNssFieldName
-                                        << nss.ns() << CollectionType::kUuidFieldName << uuid);
+                const auto query =
+                    BSON(ShardAuthoritativeCollectionType::kNssFieldName
+                         << nss.ns() << ShardAuthoritativeCollectionType::kUuidFieldName << uuid);
 
                 // Update the document (or create it) with the new index version
                 repl::UnreplicatedWritesBlock unreplicatedWritesBlock(opCtx);
                 auto request = UpdateRequest();
                 request.setNamespaceString(NamespaceString::kShardCollectionCatalogNamespace);
                 request.setQuery(query);
-                request.setUpdateModification(BSON(CollectionType::kNssFieldName
-                                                   << nss.ns() << CollectionType::kUuidFieldName
-                                                   << uuid << CollectionType::kIndexVersionFieldName
-                                                   << indexVersion));
+                request.setUpdateModification(BSON(
+                    ShardAuthoritativeCollectionType::kNssFieldName
+                    << nss.ns() << ShardAuthoritativeCollectionType::kUuidFieldName << uuid
+                    << ShardAuthoritativeCollectionType::kIndexVersionFieldName << indexVersion));
                 request.setUpsert(true);
                 request.setFromOplogApplication(true);
                 mongo::update(opCtx, collsColl.getDb(), request);
@@ -381,14 +392,16 @@ void dropCollectionGlobalIndexesMetadata(OperationContext* opCtx, const Namespac
                                         AutoGetCollection::Options{}.secondaryNssOrUUIDs(
                                             {NamespaceString::kShardIndexCatalogNamespace}));
             {
-                const auto query = BSON(CollectionType::kNssFieldName << nss.ns());
+                const auto query =
+                    BSON(ShardAuthoritativeCollectionType::kNssFieldName << nss.ns());
                 BSONObj collectionDoc;
                 // Return if there is nothing to clear.
                 if (!Helpers::findOne(opCtx, collsColl.getCollection(), query, collectionDoc)) {
                     return;
                 }
-                collectionUUID.emplace(
-                    uassertStatusOK(UUID::parse(collectionDoc[CollectionType::kUuidFieldName])));
+                auto collection = ShardAuthoritativeCollectionType::parse(
+                    IDLParserContext("DropIndexCatalogEntry"), collectionDoc);
+                collectionUUID.emplace(collection.getUuid());
                 repl::UnreplicatedWritesBlock unreplicatedWritesBlock(opCtx);
                 mongo::deleteObjects(opCtx,
                                      collsColl.getCollection(),
@@ -429,25 +442,37 @@ void clearCollectionGlobalIndexes(OperationContext* opCtx,
                                             {NamespaceString::kShardIndexCatalogNamespace}));
             {
                 // First unset the index version.
-                const auto query = BSON(CollectionType::kNssFieldName
-                                        << nss.ns() << CollectionType::kUuidFieldName << uuid);
+                const auto query =
+                    BSON(ShardAuthoritativeCollectionType::kNssFieldName
+                         << nss.ns() << ShardAuthoritativeCollectionType::kUuidFieldName << uuid);
                 BSONObj collectionDoc;
                 bool docExists =
                     Helpers::findOne(opCtx, collsColl.getCollection(), query, collectionDoc);
+
                 // Return if there is nothing to clear.
-                if (!docExists || collectionDoc[CollectionType::kIndexVersionFieldName].eoo()) {
+                if (!docExists) {
                     return;
                 }
+
+                auto collection = ShardAuthoritativeCollectionType::parse(
+                    IDLParserContext("ClearIndexCatalogEntry"), collectionDoc);
+
+                if (!collection.getIndexVersion()) {
+                    return;
+                }
+
                 repl::UnreplicatedWritesBlock unreplicatedWritesBlock(opCtx);
                 mongo::deleteObjects(opCtx,
                                      collsColl.getCollection(),
                                      NamespaceString::kShardCollectionCatalogNamespace,
                                      query,
                                      true);
-                auto finalDoc = collectionDoc.filterFieldsUndotted(
-                    BSON(CollectionType::kIndexVersionFieldName << 1), false);
-                uassertStatusOK(collection_internal::insertDocument(
-                    opCtx, collsColl.getCollection(), InsertStatement(finalDoc), nullptr));
+                collection.setIndexVersion(boost::none);
+                uassertStatusOK(
+                    collection_internal::insertDocument(opCtx,
+                                                        collsColl.getCollection(),
+                                                        InsertStatement(collection.toBSON()),
+                                                        nullptr));
             }
 
             AutoGetCollection idxColl(opCtx, NamespaceString::kShardIndexCatalogNamespace, MODE_IX);
diff --git a/src/mongo/db/s/shard_authoritative_catalog.idl b/src/mongo/db/s/shard_authoritative_catalog.idl
new file mode 100644
index 00000000000..b512027ef9c
--- /dev/null
+++ b/src/mongo/db/s/shard_authoritative_catalog.idl
@@ -0,0 +1,61 @@
+# Copyright (C) 2023-present MongoDB, Inc.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the Server Side Public License, version 1,
+# as published by MongoDB, Inc.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# Server Side Public License for more details.
+#
+# You should have received a copy of the Server Side Public License
+# along with this program. If not, see
+# <http://www.mongodb.com/licensing/server-side-public-license>.
+#
+# As a special exception, the copyright holders give permission to link the
+# code of portions of this program with the OpenSSL library under certain
+# conditions as described in each individual source file and distribute
+# linked combinations including the program with the OpenSSL library. You
+# must comply with the Server Side Public License in all respects for
+# all of the code used other than as permitted herein. If you modify file(s)
+# with this exception, you may extend this exception to your version of the
+# file(s), but you are not obligated to do so. If you do not wish to do so,
+# delete this exception statement from your version. If you delete this
+# exception statement from all source files in the program, then also delete
+# it in the license file.
+#
+
+# This file contains all the types that are authoritatively written in the shard server's
+# config.shard.xxx collections.
+
+global:
+    cpp_namespace: "mongo"
+
+imports:
+    - "mongo/db/basic_types.idl"
+
+structs:
+    ShardAuthoritativeCollectionType:
+        description: "Represents the layout and contents of documents contained in the shard
+                      server's config.shard.collections collection. This type must be 
+                      authoritatively written in shards. All manipulation of documents coming
+                      from that collection should be done with this class."
+        strict: false
+        fields:
+            _id:
+                cpp_name: nss
+                type: namespacestring
+                description: "The full namespace (with the database prefix)."
+                optional: false
+            uuid:
+                cpp_name: uuid
+                type: uuid
+                description: "The UUID that will be used to create the local collection on each of
+                              the shards which have chunks."
+                optional: false
+            indexVersion:
+                type: timestamp
+                description: "Current collection index version. It will tick everytime a global
+                              index is created or dropped."
+                optional: true
diff --git a/src/mongo/db/s/sharding_recovery_service.cpp b/src/mongo/db/s/sharding_recovery_service.cpp
index 1e30d725c9c..0805904d423 100644
--- a/src/mongo/db/s/sharding_recovery_service.cpp
+++ b/src/mongo/db/s/sharding_recovery_service.cpp
@@ -42,6 +42,7 @@
 #include "mongo/db/s/collection_critical_section_document_gen.h"
 #include "mongo/db/s/collection_sharding_runtime.h"
 #include "mongo/db/s/database_sharding_state.h"
+#include "mongo/db/s/shard_authoritative_catalog_gen.h"
 #include "mongo/db/s/sharding_migration_critical_section.h"
 #include "mongo/logv2/log.h"
 #include "mongo/s/catalog/type_collection.h"
@@ -94,7 +95,9 @@ AggregateCommandRequest makeCollectionsAndIndexesAggregation(OperationContext* o
     //      }
     // }
     stages.emplace_back(DocumentSourceMatch::create(
-        Doc{{CollectionType::kIndexVersionFieldName, Doc{{"$exists", true}}}}.toBson(), expCtx));
+        Doc{{ShardAuthoritativeCollectionType::kIndexVersionFieldName, Doc{{"$exists", true}}}}
+            .toBson(),
+        expCtx));
 
     // 2. Retrieve config.shard.indexes entries with the same uuid as the one from the
     // config.shard.collections document.
@@ -112,7 +115,7 @@ AggregateCommandRequest makeCollectionsAndIndexesAggregation(OperationContext* o
     // }
     const Doc lookupPipeline{{"from", NamespaceString::kShardIndexCatalogNamespace.coll()},
                              {"as", kGlobalIndexesFieldName},
-                             {"localField", CollectionType::kUuidFieldName},
+                             {"localField", ShardAuthoritativeCollectionType::kUuidFieldName},
                              {"foreignField", IndexCatalogType::kCollectionUUIDFieldName}};
 
     stages.emplace_back(DocumentSourceLookUp::createFromBson(
diff --git a/src/mongo/db/s/sharding_util.cpp b/src/mongo/db/s/sharding_util.cpp
index 000236fdb98..5133c99936b 100644
--- a/src/mongo/db/s/sharding_util.cpp
+++ b/src/mongo/db/s/sharding_util.cpp
@@ -37,6 +37,7 @@
 #include "mongo/db/commands.h"
 #include "mongo/db/concurrency/exception_util.h"
 #include "mongo/db/index_builds_coordinator.h"
+#include "mongo/db/s/shard_authoritative_catalog_gen.h"
 #include "mongo/logv2/log.h"
 #include "mongo/s/catalog/type_collection.h"
 #include "mongo/s/catalog/type_index_catalog.h"
@@ -244,10 +245,11 @@ Status createGlobalIndexesIndexes(OperationContext* opCtx) {
 
 Status createShardCollectionCatalogIndexes(OperationContext* opCtx) {
     bool unique = true;
-    auto result = createIndexOnCollection(opCtx,
-                                          NamespaceString::kShardCollectionCatalogNamespace,
-                                          BSON(CollectionType::kUuidFieldName << 1),
-                                          !unique);
+    auto result =
+        createIndexOnCollection(opCtx,
+                                NamespaceString::kShardCollectionCatalogNamespace,
+                                BSON(ShardAuthoritativeCollectionType::kUuidFieldName << 1),
+                                !unique);
     if (!result.isOK()) {
         return result.withContext(str::stream()
                                   << "couldn't create uuid_1 index on "