From a0345214d33cbc02ff9f1587d4b74757217cd8d7 Mon Sep 17 00:00:00 2001 From: Blake Oler Date: Mon, 3 Aug 2020 19:32:00 +0000 Subject: SERVER-49561 Add resharding fields to config.collections --- src/mongo/db/s/SConscript | 1 - src/mongo/db/s/resharding/common_types.idl | 122 --------------------- src/mongo/db/s/resharding/coordinator_document.idl | 2 +- src/mongo/db/s/resharding/donor_document.idl | 2 +- src/mongo/db/s/resharding/recipient_document.idl | 2 +- src/mongo/s/SConscript | 2 + src/mongo/s/catalog/type_collection.cpp | 18 +++ src/mongo/s/catalog/type_collection.h | 30 +++++ src/mongo/s/catalog/type_collection_test.cpp | 10 +- src/mongo/s/resharding/common_types.idl | 122 +++++++++++++++++++++ src/mongo/s/resharding/type_collection_fields.idl | 68 ++++++++++++ src/mongo/util/uuid.h | 1 + 12 files changed, 253 insertions(+), 127 deletions(-) delete mode 100644 src/mongo/db/s/resharding/common_types.idl create mode 100644 src/mongo/s/resharding/common_types.idl create mode 100644 src/mongo/s/resharding/type_collection_fields.idl diff --git a/src/mongo/db/s/SConscript b/src/mongo/db/s/SConscript index 1299619916c..12904d41747 100644 --- a/src/mongo/db/s/SConscript +++ b/src/mongo/db/s/SConscript @@ -87,7 +87,6 @@ env.Library( 'start_chunk_clone_request.cpp', env.Idlc('migration_coordinator_document.idl')[0], env.Idlc('range_deletion_task.idl')[0], - env.Idlc('resharding/common_types.idl')[0], env.Idlc('resharding/coordinator_document.idl')[0], env.Idlc('resharding/donor_document.idl')[0], env.Idlc('resharding/recipient_document.idl')[0], diff --git a/src/mongo/db/s/resharding/common_types.idl b/src/mongo/db/s/resharding/common_types.idl deleted file mode 100644 index 6eed708138e..00000000000 --- a/src/mongo/db/s/resharding/common_types.idl +++ /dev/null @@ -1,122 +0,0 @@ -# Copyright (C) 2020-present MongoDB, Inc. -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the Server Side Public License, version 1, -# as published by MongoDB, Inc. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# Server Side Public License for more details. -# -# You should have received a copy of the Server Side Public License -# along with this program. If not, see -# . -# -# As a special exception, the copyright holders give permission to link the -# code of portions of this program with the OpenSSL library under certain -# conditions as described in each individual source file and distribute -# linked combinations including the program with the OpenSSL library. You -# must comply with the Server Side Public License in all respects for -# all of the code used other than as permitted herein. If you modify file(s) -# with this exception, you may extend this exception to your version of the -# file(s), but you are not obligated to do so. If you do not wish to do so, -# delete this exception statement from your version. If you delete this -# exception statement from all source files in the program, then also delete -# it in the license file. -# - -# This file defines types used among various resharding-related IDL structs. - -global: - cpp_namespace: "mongo" - -imports: - - "mongo/idl/basic_types.idl" - - "mongo/s/sharding_types.idl" - -enums: - CoordinatorState: - description: "The current state of a resharding operation on the coordinator." - type: string - values: - kUnused: "unused" - kInitializing: "initializing" - kInitialized: "initialized" - kPreparingToDonate: "preparing-to-donate" - kCloning: "cloning" - kMirroring: "mirroring" - kCommitted: "committed" - kDropping: "dropping" - kRenaming: "renaming" - kDone: "done" - kError: "error" - - DonorState: - description: "The current state of a donor shard for a resharding operation." - type: string - values: - kInitializing: "initialized" - kDonatingInitialData: "donating-initial-data" - kDonatingOplogEntries: "donating-oplog-entries" - kMirroring: "mirroring" - kDropping: "dropping" - kError: "error" - - RecipientState: - description: "The current state of a recipient shard for a resharding operation." - type: string - values: - kCloning: "cloning" - kApplying: "applying" - kSteadyState: "steady-state" - kStrictConsistency: "strict-consistency" - kRenaming: "renaming" - kError: "error" - -structs: - CommonReshardingMetadata: - description: "Metadata shared across all node types for a resharding operation." - generate_comparison_operators: false - strict: true - fields: - _id: - type: uuid - description: "A unique identifier for the resharding operation." - nss: - type: namespacestring - description: "The namespace of the collection being resharded." - reshardingKey: - type: object - description: "The index specification document to use as the new shard key." - - MinFetchTimestamp: - description: "Not meant to be used directly. Only use internal fields." - strict: true - fields: - minFetchTimestamp: - type: timestamp - description: "Timestamp after which a donor shard has guaranteed that oplog - entries contain recipient shard information." - optional: true - - FetchTimestamp: - description: "Not meant to be used directly. Only use internal fields." - strict: true - fields: - fetchTimestamp: - type: timestamp - description: "Timestamp after which it is acceptable for recipient shards to - retrieve documents from donor shards." - optional: true - - StrictConsistencyTimestamp: - description: "Not meant to be used directly. Only use internal fields." - strict: true - fields: - strictConsistencyTimestamp: - type: timestamp - description: "Timestamp after which all writes to a particular recipient shard for - the original resharding collection will be executed as distributed - transactions." - optional: true diff --git a/src/mongo/db/s/resharding/coordinator_document.idl b/src/mongo/db/s/resharding/coordinator_document.idl index e1c97b1760f..761e55fb6b2 100644 --- a/src/mongo/db/s/resharding/coordinator_document.idl +++ b/src/mongo/db/s/resharding/coordinator_document.idl @@ -33,7 +33,7 @@ global: cpp_namespace: "mongo" imports: - - "mongo/db/s/resharding/common_types.idl" + - "mongo/s/resharding/common_types.idl" structs: DonorShardEntry: diff --git a/src/mongo/db/s/resharding/donor_document.idl b/src/mongo/db/s/resharding/donor_document.idl index 7716c7a77f8..7febc85192b 100644 --- a/src/mongo/db/s/resharding/donor_document.idl +++ b/src/mongo/db/s/resharding/donor_document.idl @@ -33,7 +33,7 @@ global: cpp_namespace: "mongo" imports: - - "mongo/db/s/resharding/common_types.idl" + - "mongo/s/resharding/common_types.idl" structs: ReshardingDonorDocument: diff --git a/src/mongo/db/s/resharding/recipient_document.idl b/src/mongo/db/s/resharding/recipient_document.idl index f3946c5cf1b..3040f2ad681 100644 --- a/src/mongo/db/s/resharding/recipient_document.idl +++ b/src/mongo/db/s/resharding/recipient_document.idl @@ -33,7 +33,7 @@ global: cpp_namespace: "mongo" imports: - - "mongo/db/s/resharding/common_types.idl" + - "mongo/s/resharding/common_types.idl" structs: DonorShardMirroringEntry: diff --git a/src/mongo/s/SConscript b/src/mongo/s/SConscript index 70400e57ed0..89b1572754d 100644 --- a/src/mongo/s/SConscript +++ b/src/mongo/s/SConscript @@ -181,6 +181,8 @@ env.Library( env.Idlc('request_types/reshard_collection.idl')[0], env.Idlc('resharded_chunk.idl')[0], env.Idlc('request_types/wait_for_fail_point.idl')[0], + env.Idlc('resharding/common_types.idl')[0], + env.Idlc('resharding/type_collection_fields.idl')[0], ], LIBDEPS=[ '$BUILD_DIR/mongo/client/connection_string', diff --git a/src/mongo/s/catalog/type_collection.cpp b/src/mongo/s/catalog/type_collection.cpp index 3ff95150dfc..0141ecdc4a7 100644 --- a/src/mongo/s/catalog/type_collection.cpp +++ b/src/mongo/s/catalog/type_collection.cpp @@ -56,6 +56,7 @@ const BSONField CollectionType::defaultCollation("defaultCollation"); const BSONField CollectionType::unique("unique"); const BSONField CollectionType::uuid("uuid"); const BSONField CollectionType::distributionMode("distributionMode"); +const BSONField CollectionType::reshardingFields("reshardingFields"); StatusWith CollectionType::fromBSON(const BSONObj& source) { CollectionType coll; @@ -200,6 +201,15 @@ StatusWith CollectionType::fromBSON(const BSONObj& source) { } } + { + const auto reshardingFieldsElem = source.getField(reshardingFields.name()); + if (reshardingFieldsElem) { + coll._reshardingFields = + ReshardingFields::parse(IDLParserErrorContext("TypeCollectionReshardingFields"), + reshardingFieldsElem.Obj()); + } + } + return StatusWith(coll); } @@ -283,6 +293,10 @@ BSONObj CollectionType::toBSON() const { } } + if (_reshardingFields) { + builder.append(reshardingFields.name(), _reshardingFields->toBSON()); + } + return builder.obj(); } @@ -308,6 +322,10 @@ void CollectionType::setKeyPattern(const KeyPattern& keyPattern) { _keyPattern = keyPattern; } +void CollectionType::setReshardingFields(const ReshardingFields& reshardingFields) { + _reshardingFields = reshardingFields; +} + bool CollectionType::hasSameOptions(const CollectionType& other) const { // The relevant options must have been set on this CollectionType. invariant(_fullNs && _keyPattern && _unique); diff --git a/src/mongo/s/catalog/type_collection.h b/src/mongo/s/catalog/type_collection.h index dd40380453f..47190ee216a 100644 --- a/src/mongo/s/catalog/type_collection.h +++ b/src/mongo/s/catalog/type_collection.h @@ -35,6 +35,7 @@ #include "mongo/db/jsobj.h" #include "mongo/db/keypattern.h" #include "mongo/db/namespace_string.h" +#include "mongo/s/resharding/type_collection_fields_gen.h" #include "mongo/util/uuid.h" namespace mongo { @@ -43,6 +44,7 @@ class Status; template class StatusWith; +using ReshardingFields = TypeCollectionReshardingFields; /** * This class represents the layout and contents of documents contained in the config server's @@ -65,6 +67,24 @@ class StatusWith; * "uuid" : UUID, * "noBalance" : false, * "distributionMode" : "unsharded|sharded", + * // Only populated if the collection is currently undergoing a resharding operation. + * "reshardingFields" : { + * "uuid" : UUID, + * "state" : CoordinatorState, + * // Only populated if the collection is currently undergoing a resharding operation, + * // and this collection is the original sharded collection. + * "donorFields" : { + * "reshardingKey" : { + * "_notTheID" : 1 + * } + * }, + * // Only populated if this collection is the temporary resharding collection in a + * // resharding operation. + * "recipientFields" : { + * "fetchTimestamp" : Timestamp(3, 4), + * "originalNamespace" : "foo.bar", + * } + * } * } * */ @@ -81,6 +101,7 @@ public: static const BSONField unique; static const BSONField uuid; static const BSONField distributionMode; + static const BSONField reshardingFields; /** * Constructs a new CollectionType object from BSON. Also does validation of the contents. @@ -173,6 +194,12 @@ public: return _distributionMode.get_value_or(DistributionMode::kSharded); } + void setReshardingFields(const ReshardingFields& reshardingFields); + + boost::optional getReshardingFields() const { + return _reshardingFields; + } + bool hasSameOptions(const CollectionType& other) const; private: @@ -206,6 +233,9 @@ private: // Optional whether balancing is allowed for this collection. If missing, implies true. boost::optional _allowBalance; + + // Fields on the collection entry specific to resharding. + boost::optional _reshardingFields; }; } // namespace mongo diff --git a/src/mongo/s/catalog/type_collection_test.cpp b/src/mongo/s/catalog/type_collection_test.cpp index 9b4c9367cb3..92b70442e9c 100644 --- a/src/mongo/s/catalog/type_collection_test.cpp +++ b/src/mongo/s/catalog/type_collection_test.cpp @@ -74,13 +74,19 @@ TEST(CollectionType, Basic) { TEST(CollectionType, AllFieldsPresent) { const OID oid = OID::gen(); const auto uuid = UUID::gen(); + const auto reshardingUuid = UUID::gen(); + + ReshardingFields reshardingFields; + reshardingFields.setUuid(reshardingUuid); + StatusWith status = CollectionType::fromBSON(BSON( CollectionType::fullNs("db.coll") << CollectionType::epoch(oid) << CollectionType::updatedAt(Date_t::fromMillisSinceEpoch(1)) << CollectionType::keyPattern(BSON("a" << 1)) << CollectionType::defaultCollation(BSON("locale" << "fr_CA")) - << CollectionType::unique(true) << CollectionType::uuid() << uuid)); + << CollectionType::unique(true) << CollectionType::uuid() << uuid + << CollectionType::reshardingFields() << reshardingFields.toBSON())); ASSERT_TRUE(status.isOK()); CollectionType coll = status.getValue(); @@ -97,6 +103,8 @@ TEST(CollectionType, AllFieldsPresent) { ASSERT_EQUALS(coll.getDropped(), false); ASSERT_TRUE(coll.getUUID()); ASSERT_EQUALS(*coll.getUUID(), uuid); + ASSERT(coll.getReshardingFields()->getState() == CoordinatorStateEnum::kUnused); + ASSERT(coll.getReshardingFields()->getUuid() == reshardingUuid); } TEST(CollectionType, EmptyDefaultCollationFailsToParse) { diff --git a/src/mongo/s/resharding/common_types.idl b/src/mongo/s/resharding/common_types.idl new file mode 100644 index 00000000000..6eed708138e --- /dev/null +++ b/src/mongo/s/resharding/common_types.idl @@ -0,0 +1,122 @@ +# Copyright (C) 2020-present MongoDB, Inc. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the Server Side Public License, version 1, +# as published by MongoDB, Inc. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# Server Side Public License for more details. +# +# You should have received a copy of the Server Side Public License +# along with this program. If not, see +# . +# +# As a special exception, the copyright holders give permission to link the +# code of portions of this program with the OpenSSL library under certain +# conditions as described in each individual source file and distribute +# linked combinations including the program with the OpenSSL library. You +# must comply with the Server Side Public License in all respects for +# all of the code used other than as permitted herein. If you modify file(s) +# with this exception, you may extend this exception to your version of the +# file(s), but you are not obligated to do so. If you do not wish to do so, +# delete this exception statement from your version. If you delete this +# exception statement from all source files in the program, then also delete +# it in the license file. +# + +# This file defines types used among various resharding-related IDL structs. + +global: + cpp_namespace: "mongo" + +imports: + - "mongo/idl/basic_types.idl" + - "mongo/s/sharding_types.idl" + +enums: + CoordinatorState: + description: "The current state of a resharding operation on the coordinator." + type: string + values: + kUnused: "unused" + kInitializing: "initializing" + kInitialized: "initialized" + kPreparingToDonate: "preparing-to-donate" + kCloning: "cloning" + kMirroring: "mirroring" + kCommitted: "committed" + kDropping: "dropping" + kRenaming: "renaming" + kDone: "done" + kError: "error" + + DonorState: + description: "The current state of a donor shard for a resharding operation." + type: string + values: + kInitializing: "initialized" + kDonatingInitialData: "donating-initial-data" + kDonatingOplogEntries: "donating-oplog-entries" + kMirroring: "mirroring" + kDropping: "dropping" + kError: "error" + + RecipientState: + description: "The current state of a recipient shard for a resharding operation." + type: string + values: + kCloning: "cloning" + kApplying: "applying" + kSteadyState: "steady-state" + kStrictConsistency: "strict-consistency" + kRenaming: "renaming" + kError: "error" + +structs: + CommonReshardingMetadata: + description: "Metadata shared across all node types for a resharding operation." + generate_comparison_operators: false + strict: true + fields: + _id: + type: uuid + description: "A unique identifier for the resharding operation." + nss: + type: namespacestring + description: "The namespace of the collection being resharded." + reshardingKey: + type: object + description: "The index specification document to use as the new shard key." + + MinFetchTimestamp: + description: "Not meant to be used directly. Only use internal fields." + strict: true + fields: + minFetchTimestamp: + type: timestamp + description: "Timestamp after which a donor shard has guaranteed that oplog + entries contain recipient shard information." + optional: true + + FetchTimestamp: + description: "Not meant to be used directly. Only use internal fields." + strict: true + fields: + fetchTimestamp: + type: timestamp + description: "Timestamp after which it is acceptable for recipient shards to + retrieve documents from donor shards." + optional: true + + StrictConsistencyTimestamp: + description: "Not meant to be used directly. Only use internal fields." + strict: true + fields: + strictConsistencyTimestamp: + type: timestamp + description: "Timestamp after which all writes to a particular recipient shard for + the original resharding collection will be executed as distributed + transactions." + optional: true diff --git a/src/mongo/s/resharding/type_collection_fields.idl b/src/mongo/s/resharding/type_collection_fields.idl new file mode 100644 index 00000000000..460a2111d35 --- /dev/null +++ b/src/mongo/s/resharding/type_collection_fields.idl @@ -0,0 +1,68 @@ +# Copyright (C) 2020-present MongoDB, Inc. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the Server Side Public License, version 1, +# as published by MongoDB, Inc. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# Server Side Public License for more details. +# +# You should have received a copy of the Server Side Public License +# along with this program. If not, see +# . +# +# As a special exception, the copyright holders give permission to link the +# code of portions of this program with the OpenSSL library under certain +# conditions as described in each individual source file and distribute +# linked combinations including the program with the OpenSSL library. You +# must comply with the Server Side Public License in all respects for +# all of the code used other than as permitted herein. If you modify file(s) +# with this exception, you may extend this exception to your version of the +# file(s), but you are not obligated to do so. If you do not wish to do so, +# delete this exception statement from your version. If you delete this +# exception statement from all source files in the program, then also delete +# it in the license file. +# + +# This file defines the format of resharding-related fields stored in a config.collections entry. + +global: + cpp_namespace: "mongo" + +imports: + - "mongo/db/keypattern.idl" + - "mongo/s/resharding/common_types.idl" + +structs: + TypeCollectionDonorFields: + description: "Resharding-related fields specific to donor shards." + fields: + reshardingKey: + type: KeyPattern + + TypeCollectionRecipientFields: + description: "Resharding-related fields specific to recipient shards." + inline_chained_structs: true + chained_structs: + FetchTimestamp: FetchTimestampStruct + fields: + originalNamespace: + type: namespacestring + + TypeCollectionReshardingFields: + description: "Resharding-related fields meant to be stored in a config.collections + document." + fields: + uuid: + type: uuid + state: + type: CoordinatorState + default: kUnused + donorFields: + type: TypeCollectionDonorFields + optional: true + recipientFields: + type: TypeCollectionRecipientFields + optional: true diff --git a/src/mongo/util/uuid.h b/src/mongo/util/uuid.h index 4330eee758e..c5abb1cad7f 100644 --- a/src/mongo/util/uuid.h +++ b/src/mongo/util/uuid.h @@ -95,6 +95,7 @@ class UUID { friend class ShardsvrRenameCollection; friend class TenantMigrationDonorDocument; friend class TenantMigrationRecipientDocument; + friend class TypeCollectionReshardingFields; friend class VoteCommitIndexBuild; -- cgit v1.2.1