diff options
author | Kaloian Manassiev <kaloian.manassiev@mongodb.com> | 2021-04-12 06:04:41 -0400 |
---|---|---|
committer | Evergreen Agent <no-reply@evergreen.mongodb.com> | 2021-04-15 11:11:44 +0000 |
commit | 0502194a53d5a4ed6ac907a8b6481432103ea608 (patch) | |
tree | d37464bd5dd1e55808adc02be2354b17d0490d7b /src/mongo | |
parent | 5e620d700d7c7c2b68fe1774bc8f070d47088bf7 (diff) | |
download | mongo-0502194a53d5a4ed6ac907a8b6481432103ea608.tar.gz |
SERVER-56063 Properly serialise the database entries format on transitions between 5.0 and 4.4
Diffstat (limited to 'src/mongo')
-rw-r--r-- | src/mongo/db/SConscript | 2 | ||||
-rw-r--r-- | src/mongo/db/s/SConscript | 19 | ||||
-rw-r--r-- | src/mongo/db/s/config/sharding_catalog_manager_database_operations.cpp | 5 | ||||
-rw-r--r-- | src/mongo/db/s/config/sharding_catalog_manager_shard_operations.cpp | 59 | ||||
-rw-r--r-- | src/mongo/db/s/resharding/resharding_coordinator_service.cpp | 11 | ||||
-rw-r--r-- | src/mongo/db/s/resharding/resharding_coordinator_service.h | 5 | ||||
-rw-r--r-- | src/mongo/db/s/sharding_ddl_50_upgrade_downgrade.cpp | 20 | ||||
-rw-r--r-- | src/mongo/db/s/sharding_ddl_50_upgrade_downgrade.h | 11 | ||||
-rw-r--r-- | src/mongo/s/SConscript | 2 | ||||
-rw-r--r-- | src/mongo/s/client/SConscript | 5 |
10 files changed, 83 insertions, 56 deletions
diff --git a/src/mongo/db/SConscript b/src/mongo/db/SConscript index fb02b1a78ce..f001f4ffac5 100644 --- a/src/mongo/db/SConscript +++ b/src/mongo/db/SConscript @@ -705,6 +705,7 @@ env.Library( 'curop_failpoint_helpers', 'dbdirectclient', 'index/index_access_method', + 'lasterror', 'query_exec', 'stats/fill_locker_info', 'stats/top', @@ -1298,6 +1299,7 @@ env.Library( 'catalog/database_holder', 'commands/server_status_core', 'kill_sessions', + 'lasterror', 'record_id_helpers', 'stats/resource_consumption_metrics', ], diff --git a/src/mongo/db/s/SConscript b/src/mongo/db/s/SConscript index 0dfe8d132e3..447ec87dff8 100644 --- a/src/mongo/db/s/SConscript +++ b/src/mongo/db/s/SConscript @@ -207,6 +207,7 @@ env.Library( '$BUILD_DIR/mongo/db/commands/server_status', '$BUILD_DIR/mongo/db/commands/txn_cmd_request', '$BUILD_DIR/mongo/db/dbdirectclient', + '$BUILD_DIR/mongo/db/lasterror', '$BUILD_DIR/mongo/db/repl/wait_for_majority_service', '$BUILD_DIR/mongo/db/rw_concern_d', '$BUILD_DIR/mongo/db/transaction', @@ -275,15 +276,6 @@ env.Library( 'type_locks.cpp', ], LIBDEPS=[ - '$BUILD_DIR/mongo/db/catalog_raii', - '$BUILD_DIR/mongo/db/repl/read_concern_args', - '$BUILD_DIR/mongo/db/rw_concern_d', - '$BUILD_DIR/mongo/db/transaction', - '$BUILD_DIR/mongo/executor/network_interface', - '$BUILD_DIR/mongo/s/catalog/sharding_catalog_client', - '$BUILD_DIR/mongo/s/client/sharding_client', - '$BUILD_DIR/mongo/s/coreshard', - '$BUILD_DIR/mongo/s/query/cluster_aggregate', 'type_shard_identity', ], LIBDEPS_PRIVATE=[ @@ -291,19 +283,26 @@ env.Library( '$BUILD_DIR/mongo/bson/util/bson_extract', '$BUILD_DIR/mongo/db/audit', '$BUILD_DIR/mongo/db/catalog/collection_options', + '$BUILD_DIR/mongo/db/catalog_raii', '$BUILD_DIR/mongo/db/commands/mongod_fcv', '$BUILD_DIR/mongo/db/commands/set_feature_compatibility_version_idl', '$BUILD_DIR/mongo/db/common', '$BUILD_DIR/mongo/db/dbdirectclient', '$BUILD_DIR/mongo/db/pipeline/process_interface/shardsvr_process_interface', '$BUILD_DIR/mongo/db/pipeline/sharded_agg_helpers', + '$BUILD_DIR/mongo/db/repl/read_concern_args', '$BUILD_DIR/mongo/db/repl/repl_coordinator_interface', '$BUILD_DIR/mongo/db/repl/replica_set_aware_service', + '$BUILD_DIR/mongo/db/rw_concern_d', '$BUILD_DIR/mongo/db/snapshot_window_options', + '$BUILD_DIR/mongo/db/transaction', '$BUILD_DIR/mongo/db/vector_clock_mongod', + '$BUILD_DIR/mongo/executor/network_interface', '$BUILD_DIR/mongo/s/catalog/sharding_catalog_client_impl', + '$BUILD_DIR/mongo/s/client/sharding_client', + '$BUILD_DIR/mongo/s/coreshard', + '$BUILD_DIR/mongo/s/query/cluster_aggregate', '$BUILD_DIR/mongo/util/log_and_backoff', - '$BUILD_DIR/mongo/util/options_parser/options_parser', 'sharding_logging', ], ) diff --git a/src/mongo/db/s/config/sharding_catalog_manager_database_operations.cpp b/src/mongo/db/s/config/sharding_catalog_manager_database_operations.cpp index 048c093f648..00d36d441d7 100644 --- a/src/mongo/db/s/config/sharding_catalog_manager_database_operations.cpp +++ b/src/mongo/db/s/config/sharding_catalog_manager_database_operations.cpp @@ -199,9 +199,10 @@ DatabaseType ShardingCatalogManager::createDatabase(OperationContext* opCtx, optPrimaryShard ? *optPrimaryShard : selectShardForNewDatabase(opCtx, shardRegistry))); + FixedFCVRegion fcvRegion(opCtx); + boost::optional<Timestamp> clusterTime; - if (feature_flags::gShardingFullDDLSupportTimestampedVersion.isEnabled( - serverGlobalParams.featureCompatibility)) { + if (DatabaseEntryFormat::get(fcvRegion) == DatabaseEntryFormat::kUUIDandTimestamp) { const auto now = VectorClock::get(opCtx)->getTime(); clusterTime = now.clusterTime().asTimestamp(); } diff --git a/src/mongo/db/s/config/sharding_catalog_manager_shard_operations.cpp b/src/mongo/db/s/config/sharding_catalog_manager_shard_operations.cpp index a12d2a0290b..8ea497ba801 100644 --- a/src/mongo/db/s/config/sharding_catalog_manager_shard_operations.cpp +++ b/src/mongo/db/s/config/sharding_catalog_manager_shard_operations.cpp @@ -524,6 +524,8 @@ StatusWith<std::string> ShardingCatalogManager::addShard( return {ErrorCodes::BadValue, "shard name cannot be empty"}; } + const auto shardRegistry = Grid::get(opCtx)->shardRegistry(); + // Only one addShard operation can be in progress at a time. Lock::ExclusiveLock lk(opCtx->lockState(), _kShardMembershipLock); @@ -543,8 +545,7 @@ StatusWith<std::string> ShardingCatalogManager::addShard( return existingShard.getValue()->getName(); } - const std::shared_ptr<Shard> shard{ - Grid::get(opCtx)->shardRegistry()->createConnection(shardConnectionString)}; + const std::shared_ptr<Shard> shard{shardRegistry->createConnection(shardConnectionString)}; auto targeter = shard->getTargeter(); auto stopMonitoringGuard = makeGuard([&] { @@ -586,7 +587,6 @@ StatusWith<std::string> ShardingCatalogManager::addShard( // Check that the shard candidate does not have a local config.system.sessions collection auto res = _dropSessionsCollection(opCtx, targeter); - if (!res.isOK()) { return res.withContext( "can't add shard with a local copy of config.system.sessions, please drop this " @@ -626,7 +626,6 @@ StatusWith<std::string> ShardingCatalogManager::addShard( // Use the _addShard command to add the shard, which in turn inserts a shardIdentity document // into the shard and triggers sharding state initialization. auto addShardStatus = runCmdOnNewShard(addShardCmd.toBSON({})); - if (!addShardStatus.isOK()) { return addShardStatus; } @@ -695,34 +694,33 @@ StatusWith<std::string> ShardingCatalogManager::addShard( "error"_attr = result.reason()); return result; } - } - // Add all databases which were discovered on the new shard - for (const auto& dbName : dbNamesStatus.getValue()) { - boost::optional<Timestamp> clusterTime; - if (feature_flags::gShardingFullDDLSupportTimestampedVersion.isEnabled( - serverGlobalParams.featureCompatibility)) { - const auto now = VectorClock::get(opCtx)->getTime(); - clusterTime = now.clusterTime().asTimestamp(); - } + // Add all databases which were discovered on the new shard + for (const auto& dbName : dbNamesStatus.getValue()) { + boost::optional<Timestamp> clusterTime; + if (DatabaseEntryFormat::get(fcvRegion) == DatabaseEntryFormat::kUUIDandTimestamp) { + const auto now = VectorClock::get(opCtx)->getTime(); + clusterTime = now.clusterTime().asTimestamp(); + } - DatabaseType dbt( - dbName, shardType.getName(), false, DatabaseVersion(UUID::gen(), clusterTime)); - - { - const auto status = Grid::get(opCtx)->catalogClient()->updateConfigDocument( - opCtx, - DatabaseType::ConfigNS, - BSON(DatabaseType::name(dbName)), - dbt.toBSON(), - true, - ShardingCatalogClient::kLocalWriteConcern); - if (!status.isOK()) { - LOGV2(21944, - "Adding shard {connectionString} even though could not add database {db}", - "Adding shard even though we could not add database", - "connectionString"_attr = shardConnectionString.toString(), - "db"_attr = dbName); + DatabaseType dbt( + dbName, shardType.getName(), false, DatabaseVersion(UUID::gen(), clusterTime)); + + { + const auto status = Grid::get(opCtx)->catalogClient()->updateConfigDocument( + opCtx, + DatabaseType::ConfigNS, + BSON(DatabaseType::name(dbName)), + dbt.toBSON(), + true, + ShardingCatalogClient::kLocalWriteConcern); + if (!status.isOK()) { + LOGV2(21944, + "Adding shard {connectionString} even though could not add database {db}", + "Adding shard even though we could not add database", + "connectionString"_attr = shardConnectionString.toString(), + "db"_attr = dbName); + } } } } @@ -736,7 +734,6 @@ StatusWith<std::string> ShardingCatalogManager::addShard( opCtx, "addShard", "", shardDetails.obj(), ShardingCatalogClient::kMajorityWriteConcern); // Ensure the added shard is visible to this process. - auto shardRegistry = Grid::get(opCtx)->shardRegistry(); shardRegistry->reload(opCtx); if (!shardRegistry->getShard(opCtx, shardType.getName()).isOK()) { return {ErrorCodes::OperationFailed, diff --git a/src/mongo/db/s/resharding/resharding_coordinator_service.cpp b/src/mongo/db/s/resharding/resharding_coordinator_service.cpp index 2e16e375d66..24c370eadbd 100644 --- a/src/mongo/db/s/resharding/resharding_coordinator_service.cpp +++ b/src/mongo/db/s/resharding/resharding_coordinator_service.cpp @@ -29,6 +29,8 @@ #define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kResharding +#include "mongo/platform/basic.h" + #include "mongo/db/s/resharding/resharding_coordinator_service.h" #include "mongo/db/auth/authorization_session_impl.h" @@ -429,7 +431,7 @@ void writeToConfigCollectionsForTempNss(OperationContext* opCtx, if (expectedNumModified) { assertNumDocsModifiedMatchesExpected(request, res, *expectedNumModified); } -} // namespace +} void insertChunkAndTagDocsForTempNss(OperationContext* opCtx, std::vector<ChunkType> initialChunks, @@ -545,6 +547,7 @@ void executeMetadataChangesInTxn( changeMetadataFunc(opCtx, txnNumber); }); } + } // namespace namespace resharding { @@ -834,6 +837,7 @@ void removeCoordinatorDocAndReshardingFields(OperationContext* opCtx, opCtx, updatedCoordinatorDoc, boost::none, boost::none, txnNumber); }); } + } // namespace resharding std::shared_ptr<repl::PrimaryOnlyService::Instance> ReshardingCoordinatorService::constructInstance( @@ -1188,8 +1192,7 @@ void ReshardingCoordinatorService::ReshardingCoordinator:: std::move(shardsAndChunks.initialChunks), std::move(zones)); installCoordinatorDoc(opCtx.get(), updatedCoordinatorDoc); -}; - +} void emplaceApproxBytesToCopyIfExists(ReshardingCoordinatorDocument& coordinatorDoc, boost::optional<ReshardingApproxCopySize> approxCopySize) { @@ -1385,7 +1388,7 @@ Future<void> ReshardingCoordinatorService::ReshardingCoordinator::_persistDecisi installCoordinatorDoc(opCtx.get(), updatedCoordinatorDoc); return Status::OK(); -}; +} ExecutorFuture<void> ReshardingCoordinatorService::ReshardingCoordinator::_awaitAllParticipantShardsDone( diff --git a/src/mongo/db/s/resharding/resharding_coordinator_service.h b/src/mongo/db/s/resharding/resharding_coordinator_service.h index 98448181c3b..60cd944803c 100644 --- a/src/mongo/db/s/resharding/resharding_coordinator_service.h +++ b/src/mongo/db/s/resharding/resharding_coordinator_service.h @@ -42,7 +42,6 @@ #include "mongo/util/future.h" namespace mongo { - namespace resharding { struct ParticipantShardsAndChunks { @@ -78,10 +77,8 @@ void writeStateTransitionAndCatalogUpdatesThenBumpShardVersions( void removeCoordinatorDocAndReshardingFields(OperationContext* opCtx, const ReshardingCoordinatorDocument& coordinatorDoc); -} // namespace resharding -class ServiceContext; -class OperationContext; +} // namespace resharding /** * Construct to encapsulate cancellation tokens and related semantics on the ReshardingCoordinator. diff --git a/src/mongo/db/s/sharding_ddl_50_upgrade_downgrade.cpp b/src/mongo/db/s/sharding_ddl_50_upgrade_downgrade.cpp index 6c691187976..4ca504a2b51 100644 --- a/src/mongo/db/s/sharding_ddl_50_upgrade_downgrade.cpp +++ b/src/mongo/db/s/sharding_ddl_50_upgrade_downgrade.cpp @@ -31,4 +31,22 @@ #include "mongo/db/s/sharding_ddl_50_upgrade_downgrade.h" -namespace mongo {} // namespace mongo +namespace mongo { + +using FeatureCompatibility = ServerGlobalParams::FeatureCompatibility; +using FCVersion = FeatureCompatibility::Version; + +DatabaseEntryFormat::Format DatabaseEntryFormat::get(const FixedFCVRegion& fcvRegion) { + switch (fcvRegion->getVersion()) { + case FCVersion::kUpgradingFrom44To50: + case FCVersion::kUpgradingFrom49To50: + case FCVersion::kVersion50: + return feature_flags::gShardingFullDDLSupportTimestampedVersion.isEnabledAndIgnoreFCV() + ? Format::kUUIDandTimestamp + : Format::kUUIDOnly; + default: + return Format::kUUIDOnly; + } +} + +} // namespace mongo diff --git a/src/mongo/db/s/sharding_ddl_50_upgrade_downgrade.h b/src/mongo/db/s/sharding_ddl_50_upgrade_downgrade.h index 8f94a935f6f..0244ecb4f28 100644 --- a/src/mongo/db/s/sharding_ddl_50_upgrade_downgrade.h +++ b/src/mongo/db/s/sharding_ddl_50_upgrade_downgrade.h @@ -29,11 +29,20 @@ #pragma once +#include "mongo/db/commands/feature_compatibility_version.h" #include "mongo/s/sharding_ddl_50_upgrade_downgrade_gen.h" namespace mongo { -// TODO SERVER-56063: Add the DatabaseEntryFormat selector +struct DatabaseEntryFormat { + enum Format { + kUUIDOnly, + kUUIDandTimestamp, + }; + + static Format get(const FixedFCVRegion& fcvRegion); +}; + // TODO SERVER-56065: Add the CollectionEntryFormat selector } // namespace mongo diff --git a/src/mongo/s/SConscript b/src/mongo/s/SConscript index b20f5ddcf52..4ba4fc5b372 100644 --- a/src/mongo/s/SConscript +++ b/src/mongo/s/SConscript @@ -34,6 +34,7 @@ env.Library( 'cluster_write.cpp', ], LIBDEPS=[ + '$BUILD_DIR/mongo/db/lasterror', 'query/cluster_query', 'write_ops/cluster_write_ops', ], @@ -501,6 +502,7 @@ env.Library( '$BUILD_DIR/mongo/db/commands/servers', '$BUILD_DIR/mongo/db/ftdc/ftdc_mongos', '$BUILD_DIR/mongo/db/initialize_server_security_state', + '$BUILD_DIR/mongo/db/lasterror', '$BUILD_DIR/mongo/db/log_process_details', '$BUILD_DIR/mongo/db/read_write_concern_defaults', '$BUILD_DIR/mongo/db/serverinit', diff --git a/src/mongo/s/client/SConscript b/src/mongo/s/client/SConscript index b8cffeb357b..2af79aff8a2 100644 --- a/src/mongo/s/client/SConscript +++ b/src/mongo/s/client/SConscript @@ -7,14 +7,13 @@ env = env.Clone() env.Library( target='sharding_client', source=[ - 'shard_remote.cpp', 'num_hosts_targeted_metrics.cpp', - 'shard_remote.idl', + 'shard_remote.cpp', + 'shard_remote.idl', ], LIBDEPS=[ '$BUILD_DIR/mongo/client/fetcher', '$BUILD_DIR/mongo/db/commands', - '$BUILD_DIR/mongo/db/lasterror', '$BUILD_DIR/mongo/executor/hedging_metrics', '$BUILD_DIR/mongo/executor/task_executor_pool', '$BUILD_DIR/mongo/s/grid', |