diff options
author | Kaloian Manassiev <kaloian.manassiev@mongodb.com> | 2021-03-31 04:55:06 -0400 |
---|---|---|
committer | Evergreen Agent <no-reply@evergreen.mongodb.com> | 2021-04-07 13:13:31 +0000 |
commit | ae4a8cfaef648518ea539dee841be2243d23eba3 (patch) | |
tree | 5448f837822b48f487be66393e463084c2fd7655 /src | |
parent | 659a367887558c4ed71bead6e1da9b5a5fe3d84b (diff) | |
download | mongo-ae4a8cfaef648518ea539dee841be2243d23eba3.tar.gz |
SERVER-54918 Pull the metadata format decisions out of InitialSplitPolicy
Diffstat (limited to 'src')
11 files changed, 92 insertions, 125 deletions
diff --git a/src/mongo/db/s/config/configsvr_drop_collection_command.cpp b/src/mongo/db/s/config/configsvr_drop_collection_command.cpp index 2bc36e522e1..89a9cb351ff 100644 --- a/src/mongo/db/s/config/configsvr_drop_collection_command.cpp +++ b/src/mongo/db/s/config/configsvr_drop_collection_command.cpp @@ -29,6 +29,8 @@ #define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kSharding +#include "mongo/platform/basic.h" + #include "mongo/db/auth/authorization_session.h" #include "mongo/db/client.h" #include "mongo/db/commands.h" diff --git a/src/mongo/db/s/config/initial_split_policy.cpp b/src/mongo/db/s/config/initial_split_policy.cpp index 43ad57a3f4a..4cded527e8a 100644 --- a/src/mongo/db/s/config/initial_split_policy.cpp +++ b/src/mongo/db/s/config/initial_split_policy.cpp @@ -237,13 +237,8 @@ InitialSplitPolicy::ShardCollectionConfig InitialSplitPolicy::generateShardColle finalSplitPoints.push_back(splitPoint); } - boost::optional<Timestamp> timestamp; - if (feature_flags::gShardingFullDDLSupportTimestampedVersion.isEnabled( - serverGlobalParams.featureCompatibility)) { - timestamp = validAfter; - } - - ChunkVersion version(1, 0, OID::gen(), timestamp); + ChunkVersion version( + 1, 0, OID::gen(), params.collectionUUID ? validAfter : boost::optional<Timestamp>()); const auto& keyPattern(shardKeyPattern.getKeyPattern()); std::vector<ChunkType> chunks; @@ -263,7 +258,7 @@ InitialSplitPolicy::ShardCollectionConfig InitialSplitPolicy::generateShardColle params.nss, params.collectionUUID, min, max, &version, validAfter, shardId, &chunks); } - return {std::move(chunks), validAfter}; + return {std::move(chunks)}; } std::unique_ptr<InitialSplitPolicy> InitialSplitPolicy::calculateOptimizationStrategy( @@ -323,29 +318,23 @@ InitialSplitPolicy::ShardCollectionConfig SingleChunkOnPrimarySplitPolicy::creat OperationContext* opCtx, const ShardKeyPattern& shardKeyPattern, const SplitPolicyParams& params) { - ShardCollectionConfig initialChunks; - const auto currentTime = VectorClock::get(opCtx)->getTime(); - const auto clusterTime = currentTime.clusterTime().asTimestamp(); - - boost::optional<Timestamp> timestamp; - if (feature_flags::gShardingFullDDLSupportTimestampedVersion.isEnabled( - serverGlobalParams.featureCompatibility)) { - timestamp = clusterTime; - } + const auto validAfter = currentTime.clusterTime().asTimestamp(); - ChunkVersion version(1, 0, OID::gen(), timestamp); + ChunkVersion version( + 1, 0, OID::gen(), params.collectionUUID ? validAfter : boost::optional<Timestamp>()); const auto& keyPattern = shardKeyPattern.getKeyPattern(); + std::vector<ChunkType> chunks; appendChunk(params.nss, params.collectionUUID, keyPattern.globalMin(), keyPattern.globalMax(), &version, - currentTime.clusterTime().asTimestamp(), + validAfter, params.primaryShardId, - &initialChunks.chunks); - initialChunks.creationTime = clusterTime; - return initialChunks; + &chunks); + + return {std::move(chunks)}; } InitialSplitPolicy::ShardCollectionConfig UnoptimizedSplitPolicy::createFirstChunks( @@ -430,13 +419,8 @@ InitialSplitPolicy::ShardCollectionConfig AbstractTagsBasedSplitPolicy::createFi return shardIds[indx++ % shardIds.size()]; }; - boost::optional<Timestamp> timestamp; - if (feature_flags::gShardingFullDDLSupportTimestampedVersion.isEnabled( - serverGlobalParams.featureCompatibility)) { - timestamp = validAfter; - } - - ChunkVersion version(1, 0, OID::gen(), timestamp); + ChunkVersion version( + 1, 0, OID::gen(), params.collectionUUID ? validAfter : boost::optional<Timestamp>()); auto lastChunkMax = keyPattern.globalMin(); std::vector<ChunkType> chunks; for (const auto& tag : _tags) { @@ -512,7 +496,7 @@ InitialSplitPolicy::ShardCollectionConfig AbstractTagsBasedSplitPolicy::createFi &chunks); } - return {std::move(chunks), validAfter}; + return {std::move(chunks)}; } AbstractTagsBasedSplitPolicy::SplitInfo PresplitHashedZonesSplitPolicy::buildSplitInfoForTag( @@ -790,22 +774,15 @@ InitialSplitPolicy::ShardCollectionConfig ReshardingSplitPolicy::createFirstChun const auto currentTime = VectorClock::get(opCtx)->getTime(); const auto validAfter = currentTime.clusterTime().asTimestamp(); - boost::optional<Timestamp> timestamp; - boost::optional<CollectionUUID> collectionUUID; - if (feature_flags::gShardingFullDDLSupportTimestampedVersion.isEnabled( - serverGlobalParams.featureCompatibility)) { - timestamp = validAfter; - collectionUUID = params.collectionUUID; - } - - ChunkVersion version(1, 0, OID::gen(), timestamp); + ChunkVersion version( + 1, 0, OID::gen(), params.collectionUUID ? validAfter : boost::optional<Timestamp>()); splitPoints.insert(keyPattern.globalMax()); for (const auto& splitPoint : splitPoints) { auto bestShard = selectBestShard( chunkDistribution, zoneInfo, zoneToShardMap, {lastChunkMax, splitPoint}); appendChunk(params.nss, - collectionUUID, + params.collectionUUID, lastChunkMax, splitPoint, &version, @@ -817,7 +794,7 @@ InitialSplitPolicy::ShardCollectionConfig ReshardingSplitPolicy::createFirstChun chunkDistribution[bestShard]++; } - return {std::move(chunks), validAfter}; + return {std::move(chunks)}; } BSONObjSet ReshardingSplitPolicy::_extractSplitPointsFromZones(const ShardKeyPattern& shardKey) { diff --git a/src/mongo/db/s/config/initial_split_policy.h b/src/mongo/db/s/config/initial_split_policy.h index ed2acc1675e..17f55434cc5 100644 --- a/src/mongo/db/s/config/initial_split_policy.h +++ b/src/mongo/db/s/config/initial_split_policy.h @@ -76,7 +76,6 @@ public: */ struct ShardCollectionConfig { std::vector<ChunkType> chunks; - Timestamp creationTime; const auto& collVersion() const { return chunks.back().getVersion(); diff --git a/src/mongo/db/s/create_collection_coordinator.cpp b/src/mongo/db/s/create_collection_coordinator.cpp index 8afca94d1c4..0aa339016fb 100644 --- a/src/mongo/db/s/create_collection_coordinator.cpp +++ b/src/mongo/db/s/create_collection_coordinator.cpp @@ -739,7 +739,7 @@ void CreateCollectionCoordinator::_commit(OperationContext* opCtx) { CollectionType coll(nss(), _initialChunks.collVersion().epoch(), - _initialChunks.creationTime, + _initialChunks.collVersion().getTimestamp(), Date_t::now(), *_collectionUUID); diff --git a/src/mongo/db/s/resharding/resharding_coordinator_service.cpp b/src/mongo/db/s/resharding/resharding_coordinator_service.cpp index 143577ffb7e..4fb82841a33 100644 --- a/src/mongo/db/s/resharding/resharding_coordinator_service.cpp +++ b/src/mongo/db/s/resharding/resharding_coordinator_service.cpp @@ -691,7 +691,12 @@ ParticipantShardsAndChunks calculateParticipantShardsAndChunks( // Note: The resharding initial split policy doesn't care about what is the real primary // shard, so just pass in a random shard. const SplitPolicyParams splitParams{ - tempNs, coordinatorDoc.getReshardingUUID(), *donorShardIds.begin()}; + tempNs, + feature_flags::gShardingFullDDLSupportTimestampedVersion.isEnabled( + serverGlobalParams.featureCompatibility) + ? boost::optional<UUID>(coordinatorDoc.getReshardingUUID()) + : boost::none, + *donorShardIds.begin()}; auto splitResult = initialSplitter.createFirstChunks(opCtx, shardKey, splitParams); initialChunks = std::move(splitResult.chunks); diff --git a/src/mongo/db/s/shard_collection_legacy.cpp b/src/mongo/db/s/shard_collection_legacy.cpp index 0714d9ac63f..42e36c0f8e2 100644 --- a/src/mongo/db/s/shard_collection_legacy.cpp +++ b/src/mongo/db/s/shard_collection_legacy.cpp @@ -38,7 +38,6 @@ #include "mongo/db/auth/authorization_session.h" #include "mongo/db/catalog_raii.h" #include "mongo/db/commands.h" -#include "mongo/db/commands/feature_compatibility_version.h" #include "mongo/db/dbdirectclient.h" #include "mongo/db/hasher.h" #include "mongo/db/index/index_descriptor.h" @@ -78,6 +77,9 @@ namespace mongo { namespace { +using FeatureCompatibility = ServerGlobalParams::FeatureCompatibility; +using FCVersion = FeatureCompatibility::Version; + MONGO_FAIL_POINT_DEFINE(pauseShardCollectionBeforeCriticalSection); MONGO_FAIL_POINT_DEFINE(pauseShardCollectionReadOnlyCriticalSection); MONGO_FAIL_POINT_DEFINE(pauseShardCollectionCommitPhase); @@ -113,23 +115,17 @@ boost::optional<CreateCollectionResponse> checkIfCollectionAlreadyShardedWithSam opCtx, nss, request.getKey(), *request.getCollation(), request.getUnique()); } -// TODO SERVER-54587: Remove the code bellow after the new shard collection path is resilient. boost::optional<UUID> getUUID(OperationContext* opCtx, const NamespaceString& nss) { AutoGetCollection autoColl(opCtx, nss, MODE_IS, AutoGetCollectionViewMode::kViewsForbidden); const auto& coll = autoColl.getCollection(); return coll ? boost::make_optional(coll->uuid()) : boost::none; } -void checkForExistingChunks(OperationContext* opCtx, const NamespaceString& nss) { +void checkForExistingChunks(OperationContext* opCtx, + const NamespaceString& nss, + const boost::optional<UUID>& optUUID) { BSONObjBuilder countBuilder; - boost::optional<UUID> optUUID; - // TODO SERVER-54587: Remove the code bellow the new shard collection path is resilient. - if (feature_flags::gShardingFullDDLSupportTimestampedVersion.isEnabled( - serverGlobalParams.featureCompatibility)) { - optUUID = getUUID(opCtx, nss); - if (!optUUID) { - return; - } + if (optUUID) { countBuilder.append("count", ChunkType::ConfigNS.coll()); countBuilder.append("query", BSON(ChunkType::collectionUUID << *optUUID)); } else { @@ -254,44 +250,6 @@ std::vector<TagsType> getTagsAndValidate(OperationContext* opCtx, return tags; } -boost::optional<UUID> getUUIDFromPrimaryShard(OperationContext* opCtx, const NamespaceString& nss) { - // Obtain the collection's UUID from the primary shard's listCollections response. - DBDirectClient localClient(opCtx); - BSONObj res; - { - std::list<BSONObj> all = - localClient.getCollectionInfos(nss.db().toString(), BSON("name" << nss.coll())); - if (!all.empty()) { - res = all.front().getOwned(); - } - } - - uassert(ErrorCodes::InternalError, - str::stream() << "expected to have an entry for " << nss.toString() - << " in listCollections response, but did not", - !res.isEmpty()); - - BSONObj collectionInfo; - if (res["info"].type() == BSONType::Object) { - collectionInfo = res["info"].Obj(); - } - - uassert(ErrorCodes::InternalError, - str::stream() << "expected to return 'info' field as part of " - "listCollections for " - << nss.ns() - << " because the cluster is in featureCompatibilityVersion=3.6, but got " - << res, - !collectionInfo.isEmpty()); - - uassert(ErrorCodes::InternalError, - str::stream() << "expected to return a UUID for collection " << nss.ns() - << " as part of 'info' field but got " << res, - collectionInfo.hasField("uuid")); - - return uassertStatusOK(UUID::parse(collectionInfo["uuid"])); -} - bool checkIfCollectionIsEmpty(OperationContext* opCtx, const NamespaceString& nss) { // Use find with predicate instead of count in order to ensure that the count // command doesn't just consult the cached metadata, which may not always be @@ -310,8 +268,7 @@ ShardCollectionTargetState calculateTargetState(OperationContext* opCtx, const NamespaceString& nss, const ShardsvrShardCollectionRequest& request) { auto tags = getTagsAndValidate(opCtx, nss, request.getKey()); - auto uuid = - request.getGetUUIDfromPrimaryShard() ? *getUUIDFromPrimaryShard(opCtx, nss) : UUID::gen(); + auto uuid = request.getGetUUIDfromPrimaryShard() ? *getUUID(opCtx, nss) : UUID::gen(); const bool isEmpty = checkIfCollectionIsEmpty(opCtx, nss); return {uuid, ShardKeyPattern(request.getKey()), tags, isEmpty}; @@ -433,14 +390,11 @@ void updateShardingCatalogEntryForCollection( ->makeFromBSON(defaultCollation)); } - boost::optional<Timestamp> creationTime; - if (feature_flags::gShardingFullDDLSupportTimestampedVersion.isEnabled( - serverGlobalParams.featureCompatibility)) { - creationTime = initialChunks.creationTime; - } - - CollectionType coll( - nss, initialChunks.collVersion().epoch(), creationTime, Date_t::now(), prerequisites.uuid); + CollectionType coll(nss, + initialChunks.collVersion().epoch(), + initialChunks.collVersion().getTimestamp(), + Date_t::now(), + prerequisites.uuid); coll.setKeyPattern(prerequisites.shardKeyPattern.toBSON()); if (defaultCollator) { coll.setDefaultCollation(defaultCollator->getSpec().toBSON()); @@ -494,7 +448,8 @@ CreateCollectionResponse shardCollection(OperationContext* opCtx, const BSONObj& cmdObj, const ShardsvrShardCollectionRequest& request, const ShardId& dbPrimaryShardId, - bool mustTakeDistLock) { + bool mustTakeDistLock, + bool use50MetadataFormat) { // Fast check for whether the collection is already sharded without taking any locks if (auto createCollectionResponseOpt = checkIfCollectionAlreadyShardedWithSameOptions(opCtx, request)) { @@ -531,14 +486,6 @@ CreateCollectionResponse shardCollection(OperationContext* opCtx, std::unique_ptr<InitialSplitPolicy> splitPolicy; InitialSplitPolicy::ShardCollectionConfig initialChunks; - bool shouldUseUUIDForChunkIndexing; - { - FixedFCVRegion fcvRegion(opCtx); - - shouldUseUUIDForChunkIndexing = - feature_flags::gShardingFullDDLSupportTimestampedVersion.isEnabled(*fcvRegion); - } - CreateCollectionResponse shardCollectionResponse; { @@ -608,7 +555,25 @@ CreateCollectionResponse shardCollection(OperationContext* opCtx, } // Fail if there are partially written chunks from a previous failed shardCollection. - checkForExistingChunks(opCtx, nss); + + if (feature_flags::gShardingFullDDLSupportTimestampedVersion.isEnabledAndIgnoreFCV()) { + if (serverGlobalParams.featureCompatibility.getVersion() == + FCVersion::kFullyDowngradedTo44) { + checkForExistingChunks(opCtx, nss, boost::none); + } else if (serverGlobalParams.featureCompatibility.getVersion() == + FCVersion::kVersion49) { + if (auto optUUID = getUUID(opCtx, nss)) + checkForExistingChunks(opCtx, nss, optUUID); + } else { + // In the intermediate state must check for leftovers from both formats + checkForExistingChunks(opCtx, nss, boost::none); + if (auto optUUID = getUUID(opCtx, nss)) + checkForExistingChunks(opCtx, nss, optUUID); + } + } else { + checkForExistingChunks(opCtx, nss, boost::none); + } + checkCollation(opCtx, request); // Create the collection locally @@ -639,7 +604,7 @@ CreateCollectionResponse shardCollection(OperationContext* opCtx, getNumShards(opCtx), targetState->collectionIsEmpty); boost::optional<CollectionUUID> optCollectionUUID; - if (shouldUseUUIDForChunkIndexing) { + if (use50MetadataFormat) { optCollectionUUID = targetState->uuid; } @@ -698,7 +663,8 @@ CreateCollectionResponse shardCollection(OperationContext* opCtx, CreateCollectionResponse shardCollectionLegacy(OperationContext* opCtx, const NamespaceString& nss, const BSONObj& cmdObj, - bool requestFromCSRS) { + bool requestFromCSRS, + bool use50MetadataFormat) { auto request = ShardsvrShardCollectionRequest::parse( IDLParserErrorContext("shardCollectionLegacy"), cmdObj); if (!request.getCollation()) @@ -726,7 +692,8 @@ CreateCollectionResponse shardCollectionLegacy(OperationContext* opCtx, cmdObj, request, ShardingState::get(opCtx)->shardId(), - !requestFromCSRS); + !requestFromCSRS, + use50MetadataFormat); } catch (const DBException& e) { scopedShardCollection.emplaceResponse(e.toStatus()); throw; diff --git a/src/mongo/db/s/shard_collection_legacy.h b/src/mongo/db/s/shard_collection_legacy.h index f245c9265f1..16f424c46b1 100644 --- a/src/mongo/db/s/shard_collection_legacy.h +++ b/src/mongo/db/s/shard_collection_legacy.h @@ -42,6 +42,7 @@ namespace mongo { CreateCollectionResponse shardCollectionLegacy(OperationContext* opCtx, const NamespaceString& nss, const BSONObj& cmdObj, - bool requestFromCSRS); + bool requestFromCSRS, + bool use50MetadataFormat); } // namespace mongo diff --git a/src/mongo/db/s/shardsvr_create_collection_command.cpp b/src/mongo/db/s/shardsvr_create_collection_command.cpp index 8ee7a4c88e7..5300e89713b 100644 --- a/src/mongo/db/s/shardsvr_create_collection_command.cpp +++ b/src/mongo/db/s/shardsvr_create_collection_command.cpp @@ -35,6 +35,7 @@ #include "mongo/db/audit.h" #include "mongo/db/auth/authorization_session.h" #include "mongo/db/commands.h" +#include "mongo/db/commands/feature_compatibility_version.h" #include "mongo/db/dbdirectclient.h" #include "mongo/db/query/collation/collator_factory_interface.h" #include "mongo/db/s/create_collection_coordinator.h" @@ -161,7 +162,13 @@ CreateCollectionResponse createCollectionLegacy(OperationContext* opCtx, inferCollationFromLocalCollection(opCtx, nss, request, &shardsvrShardCollectionRequest); } - return shardCollectionLegacy(opCtx, nss, shardsvrShardCollectionRequest.toBSON(), false); + return shardCollectionLegacy( + opCtx, + nss, + shardsvrShardCollectionRequest.toBSON(), + false /* requestIsFromCSRS */, + feature_flags::gShardingFullDDLSupportTimestampedVersion.isEnabled( + serverGlobalParams.featureCompatibility) /* use50MetadataFormat */); } CreateCollectionResponse createCollection(OperationContext* opCtx, diff --git a/src/mongo/db/s/shardsvr_drop_collection_command.cpp b/src/mongo/db/s/shardsvr_drop_collection_command.cpp index b1031faaf97..05a457d3b50 100644 --- a/src/mongo/db/s/shardsvr_drop_collection_command.cpp +++ b/src/mongo/db/s/shardsvr_drop_collection_command.cpp @@ -76,7 +76,7 @@ public: << opCtx->getWriteConcern().wMode, opCtx->getWriteConcern().wMode == WriteConcernOptions::kMajority); - const auto useNewPath = feature_flags::gShardingFullDDLSupport.isEnabled( + bool useNewPath = feature_flags::gShardingFullDDLSupport.isEnabled( serverGlobalParams.featureCompatibility); if (!useNewPath) { diff --git a/src/mongo/db/s/shardsvr_shard_collection_command.cpp b/src/mongo/db/s/shardsvr_shard_collection_command.cpp index b6c6e659135..c47dafa1232 100644 --- a/src/mongo/db/s/shardsvr_shard_collection_command.cpp +++ b/src/mongo/db/s/shardsvr_shard_collection_command.cpp @@ -33,6 +33,7 @@ #include "mongo/db/auth/authorization_session.h" #include "mongo/db/commands.h" +#include "mongo/db/commands/feature_compatibility_version.h" #include "mongo/db/namespace_string.h" #include "mongo/db/s/shard_collection_legacy.h" #include "mongo/db/s/sharding_state.h" @@ -40,7 +41,9 @@ #include "mongo/s/grid.h" #include "mongo/s/request_types/shard_collection_gen.h" #include "mongo/s/request_types/sharded_ddl_commands_gen.h" +#include "mongo/s/sharded_collections_ddl_parameters_gen.h" +// TODO (SERVER-54879): Remove this command entirely after 5.0 branches namespace mongo { namespace { @@ -90,7 +93,13 @@ public: const NamespaceString nss(parseNs(dbname, cmdObj)); - auto createCollectionResponse = shardCollectionLegacy(opCtx, nss, cmdObj, true); + auto createCollectionResponse = shardCollectionLegacy( + opCtx, + nss, + cmdObj, + true /* requestIsFromCSRS */, + feature_flags::gShardingFullDDLSupportTimestampedVersion.isEnabled( + serverGlobalParams.featureCompatibility) /* use50MetadataFormat */); createCollectionResponse.serialize(&result); result.append("collectionsharded", nss.toString()); diff --git a/src/mongo/s/sharded_collections_ddl_parameters.idl b/src/mongo/s/sharded_collections_ddl_parameters.idl index cf35479884b..203e4eb47bd 100644 --- a/src/mongo/s/sharded_collections_ddl_parameters.idl +++ b/src/mongo/s/sharded_collections_ddl_parameters.idl @@ -31,6 +31,11 @@ global: cpp_namespace: "mongo::feature_flags" feature_flags: + featureFlagShardingFullDDLSupportTimestampedVersion: + description: "Enables the usage of timestamps in Database/Chunk versions." + cpp_varname: gShardingFullDDLSupportTimestampedVersion + default: false + featureFlagShardingFullDDLSupport: description: "Ensures extra guarantees on DDL operations under a sharded cluster." cpp_varname: gShardingFullDDLSupport @@ -40,8 +45,3 @@ feature_flags: description: "Disable incomplete sharding DDL features." cpp_varname: gDisableIncompleteShardingDDLSupport default: false - - featureFlagShardingFullDDLSupportTimestampedVersion: - description: "Enables the usage of timestamps in Database/Chunk versions." - cpp_varname: gShardingFullDDLSupportTimestampedVersion - default: false |