diff options
22 files changed, 525 insertions, 37 deletions
diff --git a/jstests/core/views/views_all_commands.js b/jstests/core/views/views_all_commands.js index 439fc9353ae..a92902e4676 100644 --- a/jstests/core/views/views_all_commands.js +++ b/jstests/core/views/views_all_commands.js @@ -106,6 +106,7 @@ let viewsCommandTests = { _configsvrRemoveShard: {skip: isAnInternalCommand}, _configsvrRemoveShardFromZone: {skip: isAnInternalCommand}, _configsvrRemoveTags: {skip: isAnInternalCommand}, + _configsvrRepairShardedCollectionChunksHistory: {skip: isAnInternalCommand}, _configsvrReshardCollection: {skip: isAnInternalCommand}, _configsvrSetAllowMigrations: {skip: isAnInternalCommand}, _configsvrShardCollection: {skip: isAnInternalCommand}, @@ -495,6 +496,13 @@ let viewsCommandTests = { } ], repairDatabase: {skip: isUnrelated}, + repairShardedCollectionChunksHistory: { + command: {repairShardedCollectionChunksHistory: "test.view"}, + skipStandalone: true, + isAdminCommand: true, + expectFailure: true, + expectedErrorCode: ErrorCodes.NamespaceNotFound, + }, replSetAbortPrimaryCatchUp: {skip: isUnrelated}, replSetFreeze: {skip: isUnrelated}, replSetGetConfig: {skip: isUnrelated}, diff --git a/jstests/replsets/db_reads_while_recovering_all_commands.js b/jstests/replsets/db_reads_while_recovering_all_commands.js index 0003ffa5eb8..4ef25c51e5b 100644 --- a/jstests/replsets/db_reads_while_recovering_all_commands.js +++ b/jstests/replsets/db_reads_while_recovering_all_commands.js @@ -52,7 +52,8 @@ const allCommands = { _configsvrRemoveShard: {skip: isPrimaryOnly}, _configsvrRemoveShardFromZone: {skip: isPrimaryOnly}, _configsvrRemoveTags: {skip: isPrimaryOnly}, - _configsvrRenameCollectionMetadata: {skip: isAnInternalCommand}, + _configsvrRepairShardedCollectionChunksHistory: {skip: isPrimaryOnly}, + _configsvrRenameCollectionMetadata: {skip: isPrimaryOnly}, _configsvrReshardCollection: {skip: isPrimaryOnly}, _configsvrSetAllowMigrations: {skip: isPrimaryOnly}, _configsvrShardCollection: {skip: isPrimaryOnly}, @@ -279,6 +280,7 @@ const allCommands = { reIndex: {skip: isNotAUserDataRead}, renameCollection: {skip: isPrimaryOnly}, repairDatabase: {skip: isNotAUserDataRead}, + repairShardedCollectionChunksHistory: {skip: isPrimaryOnly}, replSetAbortPrimaryCatchUp: {skip: isNotAUserDataRead}, replSetFreeze: {skip: isNotAUserDataRead}, replSetGetConfig: {skip: isNotAUserDataRead}, diff --git a/jstests/sharding/database_versioning_all_commands.js b/jstests/sharding/database_versioning_all_commands.js index cbeb0b01961..88f59592cd5 100644 --- a/jstests/sharding/database_versioning_all_commands.js +++ b/jstests/sharding/database_versioning_all_commands.js @@ -602,6 +602,7 @@ let testCases = { } } }, + repairShardedCollectionChunksHistory: {skip: "always targets the config server"}, replSetGetStatus: {skip: "not supported in mongos"}, reshardCollection: {skip: "does not forward command to primary shard"}, revokePrivilegesFromRole: {skip: "always targets the config server"}, diff --git a/jstests/sharding/read_write_concern_defaults_application.js b/jstests/sharding/read_write_concern_defaults_application.js index c89e9e45589..54f853697e2 100644 --- a/jstests/sharding/read_write_concern_defaults_application.js +++ b/jstests/sharding/read_write_concern_defaults_application.js @@ -103,6 +103,7 @@ let testCases = { _configsvrRemoveTags: {skip: "internal command"}, _configsvrRenameCollection: {skip: "internal command"}, _configsvrRenameCollectionMetadata: {skip: "internal command"}, + _configsvrRepairShardedCollectionChunksHistory: {skip: "internal command"}, _configsvrReshardCollection: {skip: "internal command"}, _configsvrSetAllowMigrations: {skip: "internal command"}, _configsvrShardCollection: {skip: "internal command"}, @@ -550,6 +551,7 @@ let testCases = { checkWriteConcern: true, }, repairDatabase: {skip: "does not accept read or write concern"}, + repairShardedCollectionChunksHistory: {skip: "does not accept read or write concern"}, replSetAbortPrimaryCatchUp: {skip: "does not accept read or write concern"}, replSetFreeze: {skip: "does not accept read or write concern"}, replSetGetConfig: {skip: "does not accept read or write concern"}, diff --git a/jstests/sharding/repair_sharded_collection_history.js b/jstests/sharding/repair_sharded_collection_history.js new file mode 100644 index 00000000000..cf7365cafb9 --- /dev/null +++ b/jstests/sharding/repair_sharded_collection_history.js @@ -0,0 +1,71 @@ +/** + * This test can be enabled in the multiversion suites when SERVER-62065 is released in 4.2 + * + * @tags: [backport_required_multiversion] + */ + +(function() { +"use strict"; + +var st = new ShardingTest({ + shards: 1, +}); + +let configPrimary = st.configRS.getPrimary(); +let configPrimaryAdminDB = configPrimary.getDB('admin'); +let shardPrimary = st.rs0.getPrimary(); +let shardPrimaryAdminDB = shardPrimary.getDB('admin'); +let shardPrimaryConfigDB = shardPrimary.getDB('config'); + +let testDB = st.s.getDB('test1'); + +// Create a sharded collection with primary shard 0. +assert.commandWorked(st.s.adminCommand({enableSharding: testDB.getName()})); +st.ensurePrimaryShard(testDB.getName(), st.shard0.shardName); +assert.commandWorked(st.s.adminCommand({shardCollection: testDB.foo.getFullName(), key: {a: 1}})); +assert.commandWorked(st.s.adminCommand({split: testDB.foo.getFullName(), middle: {a: 0}})); +assert.commandWorked(st.s.adminCommand({split: testDB.foo.getFullName(), middle: {a: -1000}})); +assert.commandWorked(st.s.adminCommand({split: testDB.foo.getFullName(), middle: {a: +1000}})); + +assert.writeOK(st.s.getDB('test1').foo.insert({_id: "id1", a: 1})); +assert.neq(null, st.s.getDB('test1').foo.findOne({_id: "id1", a: 1})); + +assert.writeOK(st.s.getDB('test1').foo.insert({_id: "id2", a: -1})); +assert.neq(null, st.s.getDB('test1').foo.findOne({_id: "id2", a: -1})); + +['5.0', '4.4'].forEach(function(fcv) { + jsTest.log('Testing with fcv: ' + fcv); + assert.commandWorked(st.s.adminCommand({setFeatureCompatibilityVersion: fcv})); + + const collection = st.s.getDB('config').collections.findOne({_id: 'test1.foo'}); + const allChunksQuery = collection.timestamp ? {uuid: collection.uuid} : {ns: collection._id}; + + // Manually clear the 'historyIsAt40' field from the config server and the history entries from + // the shards' cache collections in order to simulate a wrong upgrade due to SERVER-62065 + assert.writeOK(st.s.getDB('config').chunks.update( + allChunksQuery, {'$unset': {historyIsAt40: ''}}, {multi: true})); + assert.writeOK(shardPrimaryConfigDB.getCollection('cache.chunks.test1.foo') + .update({}, {'$unset': {history: ''}}, {multi: true})); + + assert.commandWorked(st.s.adminCommand({repairShardedCollectionChunksHistory: 'test1.foo'})); + + // Make sure chunks for test1.foo were given history after repair + var chunks = st.s.getDB('config').getCollection('chunks').find(allChunksQuery).toArray(); + assert.eq(chunks.length, 4); + chunks.forEach((chunk) => { + assert.neq(null, chunk); + assert(chunk.hasOwnProperty("history"), "test1.foo does not have a history after repair"); + assert(chunk.hasOwnProperty("historyIsAt40"), + "test1.foo does not have a historyIsAt40 after repair"); + }); + chunks = shardPrimaryConfigDB.getCollection('cache.chunks.test1.foo').find().toArray(); + assert.eq(chunks.length, 4); + chunks.forEach((chunk) => { + assert.neq(null, chunk); + assert(chunk.hasOwnProperty("history"), + "test1.foo does not have a history on the shard after repair"); + }); +}); + +st.stop(); +})(); diff --git a/jstests/sharding/safe_secondary_reads_drop_recreate.js b/jstests/sharding/safe_secondary_reads_drop_recreate.js index cba4d7330c3..3bf17db79c8 100644 --- a/jstests/sharding/safe_secondary_reads_drop_recreate.js +++ b/jstests/sharding/safe_secondary_reads_drop_recreate.js @@ -264,6 +264,7 @@ let testCases = { removeShard: {skip: "primary only"}, removeShardFromZone: {skip: "primary only"}, renameCollection: {skip: "primary only"}, + repairShardedCollectionChunksHistory: {skip: "does not return user data"}, replSetAbortPrimaryCatchUp: {skip: "does not return user data"}, replSetFreeze: {skip: "does not return user data"}, replSetGetConfig: {skip: "does not return user data"}, diff --git a/jstests/sharding/safe_secondary_reads_single_migration_suspend_range_deletion.js b/jstests/sharding/safe_secondary_reads_single_migration_suspend_range_deletion.js index 1f2ce2a8e5e..3d7e5feb7bc 100644 --- a/jstests/sharding/safe_secondary_reads_single_migration_suspend_range_deletion.js +++ b/jstests/sharding/safe_secondary_reads_single_migration_suspend_range_deletion.js @@ -335,6 +335,7 @@ let testCases = { removeShard: {skip: "primary only"}, removeShardFromZone: {skip: "primary only"}, renameCollection: {skip: "primary only"}, + repairShardedCollectionChunksHistory: {skip: "does not return user data"}, replSetAbortPrimaryCatchUp: {skip: "does not return user data"}, replSetFreeze: {skip: "does not return user data"}, replSetGetConfig: {skip: "does not return user data"}, diff --git a/jstests/sharding/safe_secondary_reads_single_migration_waitForDelete.js b/jstests/sharding/safe_secondary_reads_single_migration_waitForDelete.js index d9f87bc7fc4..1593700e738 100644 --- a/jstests/sharding/safe_secondary_reads_single_migration_waitForDelete.js +++ b/jstests/sharding/safe_secondary_reads_single_migration_waitForDelete.js @@ -270,6 +270,7 @@ let testCases = { removeShard: {skip: "primary only"}, removeShardFromZone: {skip: "primary only"}, renameCollection: {skip: "primary only"}, + repairShardedCollectionChunksHistory: {skip: "does not return user data"}, replSetAbortPrimaryCatchUp: {skip: "does not return user data"}, replSetFreeze: {skip: "does not return user data"}, replSetGetConfig: {skip: "does not return user data"}, diff --git a/src/mongo/db/s/SConscript b/src/mongo/db/s/SConscript index 7ac451860cd..ed9c7bdfdce 100644 --- a/src/mongo/db/s/SConscript +++ b/src/mongo/db/s/SConscript @@ -304,6 +304,7 @@ env.Library( 'config/configsvr_remove_tags_command.cpp', 'config/configsvr_rename_collection_metadata_command.cpp', 'config/configsvr_reshard_collection_cmd.cpp', + 'config/configsvr_repair_sharded_collection_chunks_history_command.cpp', 'config/configsvr_set_allow_migrations_command.cpp', 'config/configsvr_shard_collection_command.cpp', 'config/configsvr_split_chunk_command.cpp', diff --git a/src/mongo/db/s/config/configsvr_repair_sharded_collection_chunks_history_command.cpp b/src/mongo/db/s/config/configsvr_repair_sharded_collection_chunks_history_command.cpp new file mode 100644 index 00000000000..4586c03cc68 --- /dev/null +++ b/src/mongo/db/s/config/configsvr_repair_sharded_collection_chunks_history_command.cpp @@ -0,0 +1,117 @@ +/** + * Copyright (C) 2021-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kSharding + +#include "mongo/platform/basic.h" + +#include "mongo/db/auth/action_type.h" +#include "mongo/db/auth/authorization_session.h" +#include "mongo/db/auth/privilege.h" +#include "mongo/db/commands.h" +#include "mongo/db/namespace_string.h" +#include "mongo/db/s/config/sharding_catalog_manager.h" +#include "mongo/db/vector_clock.h" +#include "mongo/logv2/log.h" +#include "mongo/s/grid.h" + +namespace mongo { +namespace { + +class ConfigSvrRepairShardedCollectionChunksHistoryCommand : public BasicCommand { +public: + ConfigSvrRepairShardedCollectionChunksHistoryCommand() + : BasicCommand("_configsvrRepairShardedCollectionChunksHistory") {} + + std::string help() const override { + return "Internal command, which is exported by the sharding config server. Do not call " + "directly."; + } + + AllowedOnSecondary secondaryAllowed(ServiceContext*) const override { + return AllowedOnSecondary::kNever; + } + + bool adminOnly() const override { + return true; + } + + bool supportsWriteConcern(const BSONObj& cmd) const override { + return true; + } + + std::string parseNs(const std::string& unusedDbName, const BSONObj& cmdObj) const override { + return CommandHelpers::parseNsFullyQualified(cmdObj); + } + + Status checkAuthForCommand(Client* client, + const std::string& dbname, + const BSONObj& cmdObj) const override { + if (!AuthorizationSession::get(client)->isAuthorizedForActionsOnResource( + ResourcePattern::forClusterResource(), ActionType::internal)) { + return Status(ErrorCodes::Unauthorized, "Unauthorized"); + } + return Status::OK(); + } + + bool run(OperationContext* opCtx, + const std::string& unusedDbName, + const BSONObj& cmdObj, + BSONObjBuilder& result) override { + uassert(ErrorCodes::IllegalOperation, + "_configsvrRepairShardedCollectionChunksHistory can only be run on config servers", + serverGlobalParams.clusterRole == ClusterRole::ConfigServer); + + // Set the operation context read concern level to local for reads into the config database. + repl::ReadConcernArgs::get(opCtx) = + repl::ReadConcernArgs(repl::ReadConcernLevel::kLocalReadConcern); + + uassert(ErrorCodes::InvalidOptions, + str::stream() << "_configsvrRepairShardedCollectionChunksHistory must be called " + "with majority writeConcern, got " + << cmdObj, + opCtx->getWriteConcern().wMode == WriteConcernOptions::kMajority); + + const NamespaceString nss{parseNs(unusedDbName, cmdObj)}; + + auto currentTime = VectorClock::get(opCtx)->getTime(); + auto validAfter = currentTime.configTime().asTimestamp(); + + ShardingCatalogManager::get(opCtx)->upgradeChunksHistory( + opCtx, nss, cmdObj["force"].booleanSafe(), validAfter); + + Grid::get(opCtx)->catalogCache()->invalidateCollectionEntry_LINEARIZABLE(nss); + + return true; + } + +} configSvrRepairShardedCollectionChunksHistoryCommand; + +} // namespace +} // namespace mongo diff --git a/src/mongo/db/s/config/sharding_catalog_manager.h b/src/mongo/db/s/config/sharding_catalog_manager.h index 9c39d4fda98..aa2970c3e68 100644 --- a/src/mongo/db/s/config/sharding_catalog_manager.h +++ b/src/mongo/db/s/config/sharding_catalog_manager.h @@ -503,6 +503,18 @@ public: */ static void clearForTests(ServiceContext* serviceContext); + // + // Upgrade/downgrade + // + + /** + * Upgrade the chunk metadata to include the history field. + */ + void upgradeChunksHistory(OperationContext* opCtx, + const NamespaceString& nss, + bool force, + const Timestamp validAfter); + private: /** * Performs the necessary checks for version compatibility and creates a new config.version diff --git a/src/mongo/db/s/config/sharding_catalog_manager_chunk_operations.cpp b/src/mongo/db/s/config/sharding_catalog_manager_chunk_operations.cpp index d3d63098cae..cfcf9dd3f07 100644 --- a/src/mongo/db/s/config/sharding_catalog_manager_chunk_operations.cpp +++ b/src/mongo/db/s/config/sharding_catalog_manager_chunk_operations.cpp @@ -164,7 +164,7 @@ BSONArray buildMergeChunksTransactionPrecond(const std::vector<ChunkType>& chunk return preCond.arr(); } -/* +/** * Check that the chunk still exists and return its metadata. */ StatusWith<ChunkType> getCurrentChunk(OperationContext* opCtx, @@ -297,7 +297,9 @@ boost::optional<ChunkType> getControlChunkForMigrate(OperationContext* opCtx, return uassertStatusOK(ChunkType::fromConfigBSON(response.docs.front(), epoch, timestamp)); } -// Helper function to find collection version and shard version. +/** + * Helper function to find collection version and shard version. + */ StatusWith<ChunkVersion> getMaxChunkVersionFromQueryResponse( const CollectionType& coll, const StatusWith<Shard::QueryResponse>& queryResponse) { @@ -318,7 +320,9 @@ StatusWith<ChunkVersion> getMaxChunkVersionFromQueryResponse( return chunk.getVersion(); } -// Helper function to get the collection version for nss. Always uses kLocalReadConcern. +/** + * Helper function to get the collection version for nss. Always uses kLocalReadConcern. + */ StatusWith<ChunkVersion> getCollectionVersion(OperationContext* opCtx, const NamespaceString& nss) { auto const configShard = Grid::get(opCtx)->shardRegistry()->getConfigShard(); auto findCollResponse = @@ -353,7 +357,9 @@ StatusWith<ChunkVersion> getCollectionVersion(OperationContext* opCtx, const Nam 1)); // Limit 1. } -// Helper function to get collection version and donor shard version following a merge/move/split +/** + * Helper function to get collection version and donor shard version following a move/split/merge + */ BSONObj getShardAndCollectionVersion(OperationContext* opCtx, const CollectionType& coll, const ShardId& fromShard) { @@ -540,7 +546,6 @@ StatusWith<BSONObj> ShardingCatalogManager::commitChunkSplit( // Get the max chunk version for this namespace. auto swCollVersion = getCollectionVersion(opCtx, nss); - if (!swCollVersion.isOK()) { return swCollVersion.getStatus().withContext( str::stream() << "splitChunk cannot split chunk " << range.toString() << "."); @@ -758,16 +763,16 @@ StatusWith<BSONObj> ShardingCatalogManager::commitChunkMerge( // This method must never be called with empty chunks to merge invariant(!chunkBoundaries.empty()); + if (!validAfter) { + return {ErrorCodes::IllegalOperation, "chunk operation requires validAfter timestamp"}; + } + // Take _kChunkOpLock in exclusive mode to prevent concurrent chunk splits, merges, and // migrations // TODO(SERVER-25359): Replace with a collection-specific lock map to allow splits/merges/ // move chunks on different collections to proceed in parallel Lock::ExclusiveLock lk(opCtx->lockState(), _kChunkOpLock); - if (!validAfter) { - return {ErrorCodes::IllegalOperation, "chunk operation requires validAfter timestamp"}; - } - // Get the max chunk version for this namespace. auto swCollVersion = getCollectionVersion(opCtx, nss); if (!swCollVersion.isOK()) { @@ -882,8 +887,6 @@ StatusWith<BSONObj> ShardingCatalogManager::commitChunksMerge( return {ErrorCodes::IllegalOperation, "chunk operation requires validAfter timestamp"}; } - const auto configShard = Grid::get(opCtx)->shardRegistry()->getConfigShard(); - // Take _kChunkOpLock in exclusive mode to prevent concurrent chunk splits, merges, and // migrations // TODO(SERVER-25359): Replace with a collection-specific lock map to allow splits/merges/ @@ -898,6 +901,7 @@ StatusWith<BSONObj> ShardingCatalogManager::commitChunksMerge( } // 2. Retrieve the list of chunks belonging to the requested shard + key range. + const auto configShard = Grid::get(opCtx)->shardRegistry()->getConfigShard(); auto findCollResponse = uassertStatusOK( configShard->exhaustiveFindOnConfig(opCtx, ReadPreferenceSetting{ReadPreference::PrimaryOnly}, @@ -906,7 +910,6 @@ StatusWith<BSONObj> ShardingCatalogManager::commitChunksMerge( BSON(CollectionType::kNssFieldName << nss.ns()), {}, 1)); - if (findCollResponse.docs.empty()) { return {ErrorCodes::Error(5678601), str::stream() << "Collection '" << nss.ns() << "' no longer either exists"}; @@ -1034,6 +1037,9 @@ StatusWith<BSONObj> ShardingCatalogManager::commitChunkMigration( const ShardId& fromShard, const ShardId& toShard, const boost::optional<Timestamp>& validAfter) { + if (!validAfter) { + return {ErrorCodes::IllegalOperation, "chunk operation requires validAfter timestamp"}; + } // TODO(SERVER-53283): Remove the logic around fcvRegion to re-enable // the concurrent execution of moveChunk() and setFCV(). @@ -1043,11 +1049,10 @@ StatusWith<BSONObj> ShardingCatalogManager::commitChunkMigration( "while the cluster is being upgraded or downgraded", !fcvRegion->isUpgradingOrDowngrading()); - - auto const configShard = Grid::get(opCtx)->shardRegistry()->getConfigShard(); - // Must hold the shard lock until the entire commit finishes to serialize with removeShard. Lock::SharedLock shardLock(opCtx->lockState(), _kShardMembershipLock); + + auto const configShard = Grid::get(opCtx)->shardRegistry()->getConfigShard(); auto shardResult = uassertStatusOK( configShard->exhaustiveFindOnConfig(opCtx, ReadPreferenceSetting(ReadPreference::PrimaryOnly), @@ -1056,7 +1061,6 @@ StatusWith<BSONObj> ShardingCatalogManager::commitChunkMigration( BSON(ShardType::name(toShard.toString())), {}, boost::none)); - uassert(ErrorCodes::ShardNotFound, str::stream() << "Shard " << toShard << " does not exist", !shardResult.docs.empty()); @@ -1078,10 +1082,6 @@ StatusWith<BSONObj> ShardingCatalogManager::commitChunkMigration( // (Note: This is not needed while we have a global lock, taken here only for consistency.) Lock::ExclusiveLock lk(opCtx, opCtx->lockState(), _kChunkOpLock); - if (!validAfter) { - return {ErrorCodes::IllegalOperation, "chunk operation requires validAfter timestamp"}; - } - auto findCollResponse = uassertStatusOK( configShard->exhaustiveFindOnConfig(opCtx, ReadPreferenceSetting{ReadPreference::PrimaryOnly}, @@ -1093,6 +1093,7 @@ StatusWith<BSONObj> ShardingCatalogManager::commitChunkMigration( uassert(ErrorCodes::ConflictingOperationInProgress, "Collection does not exist", !findCollResponse.docs.empty()); + const CollectionType coll(findCollResponse.docs[0]); uassert(ErrorCodes::ConflictingOperationInProgress, "Collection is undergoing changes and chunks cannot be moved", @@ -1144,7 +1145,6 @@ StatusWith<BSONObj> ShardingCatalogManager::commitChunkMigration( const auto collNsOrUUID = getNsOrUUIDForChunkTargeting(coll); auto swCurrentChunk = getCurrentChunk(opCtx, collNsOrUUID, coll.getEpoch(), coll.getTimestamp(), migratedChunk); - if (!swCurrentChunk.isOK()) { return swCurrentChunk.getStatus(); } @@ -1309,6 +1309,151 @@ StatusWith<ChunkType> ShardingCatalogManager::_findChunkOnConfig( return ChunkType::fromConfigBSON(origChunks.front(), epoch, timestamp); } +void ShardingCatalogManager::upgradeChunksHistory(OperationContext* opCtx, + const NamespaceString& nss, + bool force, + const Timestamp validAfter) { + auto const catalogClient = Grid::get(opCtx)->catalogClient(); + const auto shardRegistry = Grid::get(opCtx)->shardRegistry(); + + FixedFCVRegion fcvRegion(opCtx); + uassert(ErrorCodes::ConflictingOperationInProgress, + "Cannot upgrade the chunks history while the cluster is being upgraded or downgraded", + !fcvRegion->isUpgradingOrDowngrading()); + + // Take _kChunkOpLock in exclusive mode to prevent concurrent chunk splits, merges, and + // migrations. + Lock::ExclusiveLock lk(opCtx->lockState(), _kChunkOpLock); + + auto const configShard = Grid::get(opCtx)->shardRegistry()->getConfigShard(); + const auto coll = [&] { + auto collDocs = uassertStatusOK(configShard->exhaustiveFindOnConfig( + opCtx, + ReadPreferenceSetting{ReadPreference::PrimaryOnly}, + repl::ReadConcernLevel::kLocalReadConcern, + CollectionType::ConfigNS, + BSON(CollectionType::kNssFieldName << nss.ns()), + {}, + 1)) + .docs; + uassert(ErrorCodes::NamespaceNotFound, "Collection does not exist", !collDocs.empty()); + + return CollectionType(collDocs[0].getOwned()); + }(); + + const auto nsOrUUID = getNsOrUUIDForChunkTargeting(coll); + const auto allChunksQuery = nsOrUUID.uuid() + ? BSON(ChunkType::collectionUUID << *nsOrUUID.uuid()) + : BSON(ChunkType::ns << nsOrUUID.nss()->ns()); + + if (force) { + LOGV2(620650, + "Resetting the 'historyIsAt40' field for all chunks in collection {namespace} in " + "order to force all chunks' history to get recreated", + "namespace"_attr = nss.ns()); + + BatchedCommandRequest request([&] { + write_ops::UpdateCommandRequest updateOp(ChunkType::ConfigNS); + updateOp.setUpdates({[&] { + write_ops::UpdateOpEntry entry; + entry.setQ(allChunksQuery); + entry.setU(write_ops::UpdateModification::parseFromClassicUpdate( + BSON("$unset" << BSON(ChunkType::historyIsAt40() << "")))); + entry.setUpsert(false); + entry.setMulti(true); + return entry; + }()}); + return updateOp; + }()); + request.setWriteConcern(ShardingCatalogClient::kLocalWriteConcern.toBSON()); + + auto response = configShard->runBatchWriteCommand( + opCtx, Shard::kDefaultConfigCommandTimeout, request, Shard::RetryPolicy::kIdempotent); + uassertStatusOK(response.toStatus()); + + uassert(ErrorCodes::Error(5760502), + str::stream() << "No chunks found for collection " << nss.ns(), + response.getN() > 0); + } + + // Find the collection version + const auto collVersion = uassertStatusOK(getCollectionVersion(opCtx, nss)); + + // Find the chunk history + const auto allChunksVector = [&] { + auto findChunksResponse = uassertStatusOK( + configShard->exhaustiveFindOnConfig(opCtx, + ReadPreferenceSetting{ReadPreference::PrimaryOnly}, + repl::ReadConcernLevel::kLocalReadConcern, + ChunkType::ConfigNS, + allChunksQuery, + BSONObj(), + boost::none)); + uassert(ErrorCodes::Error(5760503), + str::stream() << "No chunks found for collection " << nss.ns(), + !findChunksResponse.docs.empty()); + return std::move(findChunksResponse.docs); + }(); + + // Bump the major version in order to be guaranteed to trigger refresh on every shard + ChunkVersion newCollectionVersion( + collVersion.majorVersion() + 1, 0, collVersion.epoch(), collVersion.getTimestamp()); + std::set<ShardId> changedShardIds; + for (const auto& chunk : allChunksVector) { + auto upgradeChunk = uassertStatusOK( + ChunkType::fromConfigBSON(chunk, collVersion.epoch(), collVersion.getTimestamp())); + bool historyIsAt40 = chunk[ChunkType::historyIsAt40()].booleanSafe(); + if (historyIsAt40) { + uassert( + ErrorCodes::Error(5760504), + str::stream() << "Chunk " << upgradeChunk.getName() << " in collection " << nss.ns() + << " indicates that it has been upgraded to version 4.0, but is " + "missing the history field. This indicates a corrupted routing " + "table and requires a manual intervention to be fixed.", + !upgradeChunk.getHistory().empty()); + continue; + } + + upgradeChunk.setVersion(newCollectionVersion); + newCollectionVersion.incMinor(); + changedShardIds.emplace(upgradeChunk.getShard()); + + // Construct the fresh history. + upgradeChunk.setHistory({ChunkHistory{validAfter, upgradeChunk.getShard()}}); + + // Set the 'historyIsAt40' field so that it gets skipped if the command is re-run + BSONObjBuilder chunkObjBuilder(upgradeChunk.toConfigBSON()); + chunkObjBuilder.appendBool(ChunkType::historyIsAt40(), true); + + // Run the update + uassertStatusOK( + catalogClient->updateConfigDocument(opCtx, + ChunkType::ConfigNS, + BSON(ChunkType::name(upgradeChunk.getName())), + chunkObjBuilder.obj(), + false, + ShardingCatalogClient::kLocalWriteConcern)); + } + + // Wait for the writes to become majority committed so that the subsequent shard refreshes can + // see them + const auto clientOpTime = repl::ReplClientInfo::forClient(opCtx->getClient()).getLastOp(); + WriteConcernResult unusedWCResult; + uassertStatusOK(waitForWriteConcern( + opCtx, clientOpTime, ShardingCatalogClient::kMajorityWriteConcern, &unusedWCResult)); + + for (const auto& shardId : changedShardIds) { + auto shard = uassertStatusOK(shardRegistry->getShard(opCtx, shardId)); + uassertStatusOK( + Shard::CommandResponse::getEffectiveStatus(shard->runCommandWithFixedRetryAttempts( + opCtx, + ReadPreferenceSetting{ReadPreference::PrimaryOnly}, + "admin", + BSON("_flushRoutingTableCacheUpdates" << nss.ns()), + Shard::RetryPolicy::kIdempotent))); + } +} + void ShardingCatalogManager::clearJumboFlag(OperationContext* opCtx, const NamespaceString& nss, const OID& collectionEpoch, diff --git a/src/mongo/db/s/config/sharding_catalog_manager_collection_operations.cpp b/src/mongo/db/s/config/sharding_catalog_manager_collection_operations.cpp index da8bc080cf6..95cda739aec 100644 --- a/src/mongo/db/s/config/sharding_catalog_manager_collection_operations.cpp +++ b/src/mongo/db/s/config/sharding_catalog_manager_collection_operations.cpp @@ -171,7 +171,7 @@ void triggerFireAndForgetShardRefreshes(OperationContext* opCtx, const Collectio opCtx, ReadPreferenceSetting{ReadPreference::PrimaryOnly}, NamespaceString::kAdminDb.toString(), - BSON(_flushRoutingTableCacheUpdates::kCommandName << coll.getNss().ns())); + BSON("_flushRoutingTableCacheUpdates" << coll.getNss().ns())); } } } diff --git a/src/mongo/db/s/flush_routing_table_cache_updates_command.cpp b/src/mongo/db/s/flush_routing_table_cache_updates_command.cpp index d75b881e46c..cc695eecfa6 100644 --- a/src/mongo/db/s/flush_routing_table_cache_updates_command.cpp +++ b/src/mongo/db/s/flush_routing_table_cache_updates_command.cpp @@ -99,11 +99,13 @@ public: uassertStatusOK(shardingState->canAcceptShardedCommands()); uassert(ErrorCodes::IllegalOperation, - "Can't issue _flushRoutingTableCacheUpdates from 'eval'", + str::stream() << "Can't issue " << Derived::Request::kCommandName + << " from 'eval'", !opCtx->getClient()->isInDirectClient()); uassert(ErrorCodes::IllegalOperation, - "Can't call _flushRoutingTableCacheUpdates if in read-only mode", + str::stream() << "Can't call " << Derived::Request::kCommandName + << " if in read-only mode", !storageGlobalParams.readOnly); auto& oss = OperationShardingState::get(opCtx); @@ -145,7 +147,7 @@ public: class FlushRoutingTableCacheUpdatesCmd : public FlushRoutingTableCacheUpdatesCmdBase<FlushRoutingTableCacheUpdatesCmd> { public: - using Request = _flushRoutingTableCacheUpdates; + using Request = FlushRoutingTableCacheUpdates; static bool supportsWriteConcern() { return false; @@ -157,7 +159,7 @@ class FlushRoutingTableCacheUpdatesCmdWithWriteConcern : public FlushRoutingTableCacheUpdatesCmdBase< FlushRoutingTableCacheUpdatesCmdWithWriteConcern> { public: - using Request = _flushRoutingTableCacheUpdatesWithWriteConcern; + using Request = FlushRoutingTableCacheUpdatesWithWriteConcern; static bool supportsWriteConcern() { return true; diff --git a/src/mongo/db/s/resharding/resharding_coordinator_service.cpp b/src/mongo/db/s/resharding/resharding_coordinator_service.cpp index a87d299e8ca..9ded7e84b76 100644 --- a/src/mongo/db/s/resharding/resharding_coordinator_service.cpp +++ b/src/mongo/db/s/resharding/resharding_coordinator_service.cpp @@ -27,15 +27,13 @@ * it in the license file. */ -#include "mongo/base/string_data.h" -#include "mongo/s/write_ops/batched_command_request.h" - #define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kResharding #include "mongo/platform/basic.h" #include "mongo/db/s/resharding/resharding_coordinator_service.h" +#include "mongo/base/string_data.h" #include "mongo/bson/bsonobj.h" #include "mongo/bson/json.h" #include "mongo/db/auth/authorization_session_impl.h" @@ -69,6 +67,7 @@ #include "mongo/s/request_types/flush_resharding_state_change_gen.h" #include "mongo/s/request_types/flush_routing_table_cache_updates_gen.h" #include "mongo/s/shard_id.h" +#include "mongo/s/write_ops/batched_command_request.h" #include "mongo/s/write_ops/batched_command_response.h" #include "mongo/util/assert_util.h" #include "mongo/util/fail_point.h" @@ -548,7 +547,7 @@ void executeMetadataChangesInTxn( } BSONObj makeFlushRoutingTableCacheUpdatesCmd(const NamespaceString& nss) { - auto cmd = _flushRoutingTableCacheUpdatesWithWriteConcern(nss); + auto cmd = FlushRoutingTableCacheUpdatesWithWriteConcern(nss); cmd.setSyncFromConfig(true); cmd.setDbName(nss.db()); return cmd.toBSON( diff --git a/src/mongo/db/s/sharding_util.cpp b/src/mongo/db/s/sharding_util.cpp index 351da27f49c..ad529a7985e 100644 --- a/src/mongo/db/s/sharding_util.cpp +++ b/src/mongo/db/s/sharding_util.cpp @@ -30,10 +30,11 @@ #define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kSharding #include "mongo/platform/basic.h" -#include <fmt/format.h> #include "mongo/db/s/sharding_util.h" +#include <fmt/format.h> + #include "mongo/db/commands.h" #include "mongo/logv2/log.h" #include "mongo/s/async_requests_sender.h" @@ -49,7 +50,7 @@ void tellShardsToRefreshCollection(OperationContext* opCtx, const std::vector<ShardId>& shardIds, const NamespaceString& nss, const std::shared_ptr<executor::TaskExecutor>& executor) { - auto cmd = _flushRoutingTableCacheUpdatesWithWriteConcern(nss); + auto cmd = FlushRoutingTableCacheUpdatesWithWriteConcern(nss); cmd.setSyncFromConfig(true); cmd.setDbName(nss.db()); auto cmdObj = CommandHelpers::appendMajorityWriteConcern(cmd.toBSON({})); diff --git a/src/mongo/s/catalog/type_chunk.cpp b/src/mongo/s/catalog/type_chunk.cpp index 30e814256c9..ad1430685a8 100644 --- a/src/mongo/s/catalog/type_chunk.cpp +++ b/src/mongo/s/catalog/type_chunk.cpp @@ -59,6 +59,7 @@ const BSONField<Date_t> ChunkType::lastmod("lastmod"); const BSONField<OID> ChunkType::epoch("lastmodEpoch"); const BSONField<Timestamp> ChunkType::timestamp("lastmodTimestamp"); const BSONField<BSONObj> ChunkType::history("history"); +const BSONField<bool> ChunkType::historyIsAt40("historyIsAt40"); namespace { diff --git a/src/mongo/s/catalog/type_chunk.h b/src/mongo/s/catalog/type_chunk.h index 778d3af77aa..4f3d9177074 100644 --- a/src/mongo/s/catalog/type_chunk.h +++ b/src/mongo/s/catalog/type_chunk.h @@ -212,6 +212,7 @@ public: static const BSONField<OID> epoch; static const BSONField<Timestamp> timestamp; static const BSONField<BSONObj> history; + static const BSONField<bool> historyIsAt40; ChunkType(); ChunkType(NamespaceString nss, ChunkRange range, ChunkVersion version, ShardId shardId); diff --git a/src/mongo/s/client/shard.h b/src/mongo/s/client/shard.h index 9c28d5139c8..285aaf48994 100644 --- a/src/mongo/s/client/shard.h +++ b/src/mongo/s/client/shard.h @@ -41,11 +41,11 @@ #include "mongo/db/repl/read_concern_args.h" #include "mongo/executor/remote_command_response.h" #include "mongo/s/shard_id.h" +#include "mongo/s/write_ops/batched_command_request.h" +#include "mongo/s/write_ops/batched_command_response.h" namespace mongo { -class BatchedCommandRequest; -class BatchedCommandResponse; class OperationContext; class RemoteCommandTargeter; diff --git a/src/mongo/s/commands/SConscript b/src/mongo/s/commands/SConscript index dd39115b35b..0fb51bc21c6 100644 --- a/src/mongo/s/commands/SConscript +++ b/src/mongo/s/commands/SConscript @@ -84,6 +84,7 @@ env.Library( 'cluster_remove_shard_cmd.cpp', 'cluster_remove_shard_from_zone_cmd.cpp', 'cluster_rename_collection_cmd.cpp', + 'cluster_repair_sharded_collection_chunks_history_cmd.cpp', 'cluster_repl_set_get_status_cmd.cpp', 'cluster_reshard_collection_cmd.cpp', 'cluster_rwc_defaults_commands.cpp', diff --git a/src/mongo/s/commands/cluster_repair_sharded_collection_chunks_history_cmd.cpp b/src/mongo/s/commands/cluster_repair_sharded_collection_chunks_history_cmd.cpp new file mode 100644 index 00000000000..dd08b1574d4 --- /dev/null +++ b/src/mongo/s/commands/cluster_repair_sharded_collection_chunks_history_cmd.cpp @@ -0,0 +1,121 @@ +/** + * Copyright (C) 2021-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#define MONGO_LOG_DEFAULT_COMPONENT ::mongo::logger::LogComponent::kSharding + +#include "mongo/platform/basic.h" + +#include "mongo/db/audit.h" +#include "mongo/db/auth/action_set.h" +#include "mongo/db/auth/action_type.h" +#include "mongo/db/auth/authorization_manager.h" +#include "mongo/db/auth/authorization_session.h" +#include "mongo/db/client.h" +#include "mongo/db/commands.h" +#include "mongo/s/grid.h" + +namespace mongo { +namespace { + +class RepairShardedCollectionChunksHistoryCommand : public BasicCommand { +public: + RepairShardedCollectionChunksHistoryCommand() + : BasicCommand("repairShardedCollectionChunksHistory") {} + + AllowedOnSecondary secondaryAllowed(ServiceContext*) const override { + return AllowedOnSecondary::kAlways; + } + + bool adminOnly() const override { + return true; + } + + bool supportsWriteConcern(const BSONObj& cmd) const override { + return false; + } + + std::string help() const override { + return "Administrative command to repair the effects of SERVER-62065. If the collection " + "has been upgraded through a cluster comprised of binaries which do not contain " + "this command, the chunks cache collections on the shards will miss history " + "entries. This command will correct that and will mark such collections as " + "correctly repaired, so that a subsequent invocation will not cause any changes to " + "the routing information. In rare cases where the history entries are missing due " + "to corrupted restore, the 'force:true' parameter can be passed which will force " + "all history entries to be re-added."; + } + + // The command intentionally uses the permission control of split/mergeChunks since it only + // modifies the contents of chunk entries and increments the collection/shard versions without + // causing any data placement changes + Status checkAuthForCommand(Client* client, + const std::string& dbname, + const BSONObj& cmdObj) const override { + if (!AuthorizationSession::get(client)->isAuthorizedForActionsOnResource( + ResourcePattern::forExactNamespace(NamespaceString(parseNs(dbname, cmdObj))), + ActionType::splitChunk)) { + return Status(ErrorCodes::Unauthorized, "Unauthorized"); + } + return Status::OK(); + } + + std::string parseNs(const std::string& unusedDbName, const BSONObj& cmdObj) const override { + return CommandHelpers::parseNsFullyQualified(cmdObj); + } + + bool run(OperationContext* opCtx, + const std::string& unusedDbName, + const BSONObj& cmdObj, + BSONObjBuilder& result) override { + const NamespaceString nss{parseNs(unusedDbName, cmdObj)}; + + BSONObjBuilder cmdBuilder( + BSON("_configsvrRepairShardedCollectionChunksHistory" << nss.ns())); + if (cmdObj["force"].booleanSafe()) + cmdBuilder.appendBool("force", true); + + auto configShard = Grid::get(opCtx)->shardRegistry()->getConfigShard(); + auto cmdResponse = uassertStatusOK(configShard->runCommandWithFixedRetryAttempts( + opCtx, + ReadPreferenceSetting{ReadPreference::PrimaryOnly}, + "admin", + CommandHelpers::appendMajorityWriteConcern(cmdBuilder.obj(), opCtx->getWriteConcern()), + Shard::RetryPolicy::kIdempotent)); + uassertStatusOK(cmdResponse.commandStatus); + + // Append any return value from the response, which the config server returned + CommandHelpers::filterCommandReplyForPassthrough(cmdResponse.response, &result); + + return true; + } + +} repairShardedCollectionChunksHistoryCommand; + +} // namespace +} // namespace mongo diff --git a/src/mongo/s/request_types/flush_routing_table_cache_updates.idl b/src/mongo/s/request_types/flush_routing_table_cache_updates.idl index 3548b429c3a..62a8e3a94a5 100644 --- a/src/mongo/s/request_types/flush_routing_table_cache_updates.idl +++ b/src/mongo/s/request_types/flush_routing_table_cache_updates.idl @@ -26,8 +26,6 @@ # it in the license file. # -# _flushRoutingTableCacheUpdates IDL File - global: cpp_namespace: "mongo" @@ -40,6 +38,7 @@ commands: An internal command to wait for the last routing table cache refresh for a particular namespace to be persisted to disk. command_name: _flushRoutingTableCacheUpdates + cpp_name: FlushRoutingTableCacheUpdates strict: true namespace: type api_version: "" @@ -55,6 +54,7 @@ commands: description: >- The same behavior as _flushRoutingTableCacheUpdates but accepts writeConcern. command_name: _flushRoutingTableCacheUpdatesWithWriteConcern + cpp_name: FlushRoutingTableCacheUpdatesWithWriteConcern strict: true namespace: type api_version: "" |