diff options
author | Cheahuychou Mao <mao.cheahuychou@gmail.com> | 2022-08-30 19:17:30 +0000 |
---|---|---|
committer | Evergreen Agent <no-reply@evergreen.mongodb.com> | 2022-08-30 20:22:44 +0000 |
commit | 27f7896edc0828d0c4bae3d70bbec7e8b0a66d98 (patch) | |
tree | 6e4423599d581609da95a97848e1ac1fa3aec0ae | |
parent | 8b4b922cf267fc1d72e33c9cb588f2aadd685c14 (diff) | |
download | mongo-27f7896edc0828d0c4bae3d70bbec7e8b0a66d98.tar.gz |
SERVER-68751 Add analyzeShardKey and configureQueryAnalyzer commands on mongos
10 files changed, 298 insertions, 25 deletions
diff --git a/jstests/sharding/analyze_shard_key_basic.js b/jstests/sharding/analyze_shard_key_basic.js index 5f134efacc0..aa584bda0e9 100644 --- a/jstests/sharding/analyze_shard_key_basic.js +++ b/jstests/sharding/analyze_shard_key_basic.js @@ -35,11 +35,13 @@ function runTestNonExistingNs(conn, ns, key) { const st = new ShardingTest({shards: 2, rs: {nodes: 2}}); const dbName = "testDb"; + const shardedNs = dbName + ".shardedColl"; + const unshardedNs = dbName + ".unshardedColl"; const nonExistingNs = dbName + ".nonExistingColl"; - const candidateKeyWithIndex = {candidateKey: 1}; - const candidateKeyWithoutIndex = {candidateKey: "hashed"}; // does not have a supporting index. + const candidateKey0 = {candidateKey0: 1}; + const candidateKey1 = {candidateKey1: 1}; // does not have a supporting index. - const shardedNs = dbName + ".shardedColl"; + // Set up the sharded collection. const currentKey = {currentKey: 1}; const currentKeySplitPoint = {currentKey: 0}; assert.commandWorked(st.s.adminCommand({enableSharding: dbName})); @@ -48,44 +50,46 @@ function runTestNonExistingNs(conn, ns, key) { assert.commandWorked(st.s.adminCommand({split: shardedNs, middle: currentKeySplitPoint})); assert.commandWorked(st.s.adminCommand( {moveChunk: shardedNs, find: currentKeySplitPoint, to: st.shard1.shardName})); - assert.commandWorked(st.s.getCollection(shardedNs).createIndex(candidateKeyWithIndex)); + assert.commandWorked(st.s.getCollection(shardedNs).createIndex(candidateKey0)); - const unshardedNs = dbName + ".unshardedColl"; - assert.commandWorked(st.s.getCollection(unshardedNs).createIndex(candidateKeyWithIndex)); + // Set up the unsharded collection. + assert.commandWorked(st.s.getCollection(unshardedNs).createIndex(candidateKey0)); - // Verify that the command is supported on all shardsvr mongods (both primary and secondary). - function runTestSupported(conn, isPrimaryShard) { - // Can evaluate candidate shard keys for an existing unsharded collection. - if (isPrimaryShard) { - runTestExistingNs(conn, unshardedNs, candidateKeyWithIndex, candidateKeyWithoutIndex); + // Verify that the command is supported on mongos and all shardsvr mongods (both primary and + // secondary). + function runTestSupported(conn, isPrimaryShardOrMongos) { + // Can evaluate a candidate shard key for an unsharded collection. + if (isPrimaryShardOrMongos) { + runTestExistingNs(conn, unshardedNs, candidateKey0, candidateKey1); } else { - runTestNonExistingNs(conn, unshardedNs, candidateKeyWithIndex); + runTestNonExistingNs(conn, unshardedNs, candidateKey0); } // Can evaluate the current shard key for an existing sharded collection. runTestExistingNs(conn, shardedNs, currentKey); // Can evaluate candidate shard keys for an existing sharded collection. - runTestExistingNs(conn, shardedNs, candidateKeyWithIndex, candidateKeyWithoutIndex); + runTestExistingNs(conn, shardedNs, candidateKey0, candidateKey1); // Cannot evaluate a candidate shard key for a non-existing collection. - runTestNonExistingNs(conn, nonExistingNs, candidateKeyWithIndex); + runTestNonExistingNs(conn, nonExistingNs, candidateKey0); } + runTestSupported(st.s, true /* isPrimaryShardOrMongos */); st.rs0.nodes.forEach(node => { - runTestSupported(node, true /* isPrimaryShard */); + runTestSupported(node, true /* isPrimaryShardOrMongos */); }); st.rs1.nodes.forEach(node => { - runTestSupported(node, false /* isPrimaryShard */); + runTestSupported(node, false /* isPrimaryShardOrMongos */); }); // Verify that the command is not supported on configsvr mongods. function runTestNotSupported(conn) { assert.commandFailedWithCode( - conn.adminCommand({analyzeShardKey: unshardedNs, key: candidateKeyWithIndex}), + conn.adminCommand({analyzeShardKey: unshardedNs, key: candidateKey0}), ErrorCodes.IllegalOperation); assert.commandFailedWithCode( conn.adminCommand({analyzeShardKey: shardedNs, key: currentKey}), ErrorCodes.IllegalOperation); assert.commandFailedWithCode( - conn.adminCommand({analyzeShardKey: shardedNs, key: candidateKeyWithIndex}), + conn.adminCommand({analyzeShardKey: shardedNs, key: candidateKey0}), ErrorCodes.IllegalOperation); } @@ -103,19 +107,20 @@ function runTestNonExistingNs(conn, ns, key) { const primary = rst.getPrimary(); const dbName = "testDb"; + const unshardedNs = dbName + ".unshardedColl"; const nonExistingNs = dbName + ".nonExistingColl"; - const candidateKey = {candidateKey: 1}; - const invalidCandidateKey = {candidateKey: "hashed"}; // does not have a supporting index. + const candidateKey0 = {candidateKey0: 1}; + const candidateKey1 = {candidateKey1: 1}; // does not have a supporting index. - const unshardedNs = dbName + ".unshardedColl"; - assert.commandWorked(primary.getCollection(unshardedNs).createIndex(candidateKey)); + // Set up the unsharded collection. + assert.commandWorked(primary.getCollection(unshardedNs).createIndex(candidateKey0)); // Verify that the command is supported on all mongods (both primary and secondary). function runTestSupported(conn) { // Can evaluate a candidate shard key for an existing unsharded collection. - runTestExistingNs(conn, unshardedNs, candidateKey, invalidCandidateKey); + runTestExistingNs(conn, unshardedNs, candidateKey0, candidateKey1); // Cannot evaluate a candidate shard key for a non-existing collection. - runTestNonExistingNs(conn, nonExistingNs, candidateKey); + runTestNonExistingNs(conn, nonExistingNs, candidateKey0); } rst.nodes.forEach(node => { runTestSupported(node); diff --git a/jstests/sharding/configure_query_analyzer_basic.js b/jstests/sharding/configure_query_analyzer_basic.js index 9630b04b79f..d363007edf0 100644 --- a/jstests/sharding/configure_query_analyzer_basic.js +++ b/jstests/sharding/configure_query_analyzer_basic.js @@ -74,13 +74,14 @@ function runTestNonExistingNs(conn, ns) { const unshardedNs = dbName + "." + unshardedCollName; assert.commandWorked(st.s.getDB(dbName).createCollection(unshardedCollName)); - // Verify that the command is supported on configsvr primary mongod. + // Verify that the command is supported on mongos and configsvr primary mongod. function runTestSupported(conn) { runTestExistingNs(conn, unshardedNs); runTestExistingNs(conn, shardedNs); runTestNonExistingNs(conn, nonExistingNs); } + runTestSupported(st.s); runTestSupported(configPrimary); // Verify that the command is not supported on configsvr secondary mongods or any shardvr diff --git a/jstests/sharding/database_versioning_all_commands.js b/jstests/sharding/database_versioning_all_commands.js index 5751c630d21..e0ca698bc7a 100644 --- a/jstests/sharding/database_versioning_all_commands.js +++ b/jstests/sharding/database_versioning_all_commands.js @@ -271,6 +271,16 @@ let testCases = { analyze: { skip: "unimplemented. Serves only as a stub." }, // TODO SERVER-68055: Extend test to work with analyze + analyzeShardKey: { + run: { + runsAgainstAdminDb: true, + sendsDbVersion: true, + explicitlyCreateCollection: true, + command: function(dbName, collName) { + return {analyzeShardKey: dbName + "." + collName, key: {_id: 1}}; + }, + } + }, appendOplogNote: {skip: "unversioned and executes on all shards"}, authenticate: {skip: "does not forward command to primary shard"}, balancerCollectionStatus: {skip: "does not forward command to primary shard"}, @@ -305,6 +315,7 @@ let testCases = { compactStructuredEncryptionData: {skip: "requires encrypted collections"}, configureCollectionBalancing: {skip: "always targets the config server"}, configureFailPoint: {skip: "executes locally on mongos (not sent to any remote node)"}, + configureQueryAnalyzer: {skip: "always targets the config server"}, connPoolStats: {skip: "executes locally on mongos (not sent to any remote node)"}, connPoolSync: {skip: "executes locally on mongos (not sent to any remote node)"}, connectionStatus: {skip: "executes locally on mongos (not sent to any remote node)"}, diff --git a/jstests/sharding/libs/last_lts_mongos_commands.js b/jstests/sharding/libs/last_lts_mongos_commands.js index d1734daab6d..e9f91bd2f85 100644 --- a/jstests/sharding/libs/last_lts_mongos_commands.js +++ b/jstests/sharding/libs/last_lts_mongos_commands.js @@ -16,11 +16,13 @@ const commandsRemovedFromMongosSinceLastLTS = [ const commandsAddedToMongosSinceLastLTS = [ "abortReshardCollection", "analyze", + "analyzeShardKey", // TODO (SERVER-68977): Remove upgrade/downgrade for PM-1858. "appendOplogNote", "cleanupReshardCollection", "commitReshardCollection", "compactStructuredEncryptionData", "configureCollectionBalancing", + "configureQueryAnalyzer", // TODO (SERVER-68977): Remove upgrade/downgrade for PM-1858. "coordinateCommitTransaction", "getClusterParameter", "moveRange", diff --git a/jstests/sharding/safe_secondary_reads_drop_recreate.js b/jstests/sharding/safe_secondary_reads_drop_recreate.js index 3bb44e3ad7f..a9e4704161a 100644 --- a/jstests/sharding/safe_secondary_reads_drop_recreate.js +++ b/jstests/sharding/safe_secondary_reads_drop_recreate.js @@ -109,6 +109,7 @@ let testCases = { behavior: "versioned" }, analyze: {skip: "primary only"}, + analyzeShardKey: {skip: "does not return user data"}, appendOplogNote: {skip: "primary only"}, applyOps: {skip: "primary only"}, authSchemaUpgrade: {skip: "primary only"}, @@ -145,6 +146,7 @@ let testCases = { }, // TODO SERVER-62374: remove this once 5.3 becomes last continuos release configureCollectionBalancing: {skip: "does not return user data"}, configureFailPoint: {skip: "does not return user data"}, + configureQueryAnalyzer: {skip: "does not return user data"}, connPoolStats: {skip: "does not return user data"}, connPoolSync: {skip: "does not return user data"}, connectionStatus: {skip: "does not return user data"}, diff --git a/jstests/sharding/safe_secondary_reads_single_migration_suspend_range_deletion.js b/jstests/sharding/safe_secondary_reads_single_migration_suspend_range_deletion.js index 0c632f28c1d..86aac019113 100644 --- a/jstests/sharding/safe_secondary_reads_single_migration_suspend_range_deletion.js +++ b/jstests/sharding/safe_secondary_reads_single_migration_suspend_range_deletion.js @@ -125,6 +125,7 @@ let testCases = { behavior: "versioned" }, analyze: {skip: "primary only"}, + analyzeShardKey: {skip: "does not return user data"}, appendOplogNote: {skip: "primary only"}, applyOps: {skip: "primary only"}, authSchemaUpgrade: {skip: "primary only"}, @@ -161,6 +162,7 @@ let testCases = { compact: {skip: "does not return user data"}, compactStructuredEncryptionData: {skip: "does not return user data"}, configureFailPoint: {skip: "does not return user data"}, + configureQueryAnalyzer: {skip: "does not return user data"}, connPoolStats: {skip: "does not return user data"}, connPoolSync: {skip: "does not return user data"}, connectionStatus: {skip: "does not return user data"}, diff --git a/jstests/sharding/safe_secondary_reads_single_migration_waitForDelete.js b/jstests/sharding/safe_secondary_reads_single_migration_waitForDelete.js index 6081569042a..c5811a09c06 100644 --- a/jstests/sharding/safe_secondary_reads_single_migration_waitForDelete.js +++ b/jstests/sharding/safe_secondary_reads_single_migration_waitForDelete.js @@ -112,6 +112,7 @@ let testCases = { behavior: "versioned" }, analyze: {skip: "primary only"}, + analyzeShardKey: {skip: "does not return user data"}, appendOplogNote: {skip: "primary only"}, applyOps: {skip: "primary only"}, authenticate: {skip: "does not return user data"}, @@ -148,6 +149,7 @@ let testCases = { }, // TODO SERVER-62374: remove this once 5.3 becomes last continuos release configureCollectionBalancing: {skip: "does not return user data"}, configureFailPoint: {skip: "does not return user data"}, + configureQueryAnalyzer: {skip: "does not return user data"}, connPoolStats: {skip: "does not return user data"}, connPoolSync: {skip: "does not return user data"}, connectionStatus: {skip: "does not return user data"}, diff --git a/src/mongo/s/commands/SConscript b/src/mongo/s/commands/SConscript index 0d60bad7378..05cd828b268 100644 --- a/src/mongo/s/commands/SConscript +++ b/src/mongo/s/commands/SConscript @@ -199,9 +199,11 @@ env.Library( source=[ 'cluster_add_shard_cmd.cpp', 'cluster_add_shard_to_zone_cmd.cpp', + 'cluster_analyze_shard_key_cmd.cpp', 'cluster_balancer_collection_status_cmd.cpp', 'cluster_clear_jumbo_flag_cmd.cpp', 'cluster_configure_collection_balancing.cpp', + 'cluster_configure_query_analyzer_cmd.cpp', 'cluster_control_balancer_cmd.cpp', 'cluster_list_shards_cmd.cpp', 'cluster_merge_chunks_cmd.cpp', @@ -224,6 +226,7 @@ env.Library( '$BUILD_DIR/mongo/db/auth/saslauth', '$BUILD_DIR/mongo/db/commands', '$BUILD_DIR/mongo/db/commands/servers', + '$BUILD_DIR/mongo/db/s/analyze_shard_key', '$BUILD_DIR/mongo/db/server_base', '$BUILD_DIR/mongo/rpc/client_metadata', '$BUILD_DIR/mongo/rpc/rewrite_state_change_errors', diff --git a/src/mongo/s/commands/cluster_analyze_shard_key_cmd.cpp b/src/mongo/s/commands/cluster_analyze_shard_key_cmd.cpp new file mode 100644 index 00000000000..073dcdbea7e --- /dev/null +++ b/src/mongo/s/commands/cluster_analyze_shard_key_cmd.cpp @@ -0,0 +1,137 @@ +/** + * Copyright (C) 2022-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/platform/basic.h" + +#include "mongo/db/auth/authorization_session.h" +#include "mongo/db/commands.h" +#include "mongo/db/s/analyze_shard_key_cmd_gen.h" +#include "mongo/db/s/analyze_shard_key_feature_flag_gen.h" +#include "mongo/logv2/log.h" +#include "mongo/s/cluster_commands_helpers.h" +#include "mongo/s/grid.h" + +#define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kCommand + + +namespace mongo { + +namespace { + +ShardId getRandomOwningShardId(const ChunkManager& cm, const CachedDatabaseInfo& dbInfo) { + if (cm.isSharded()) { + std::set<ShardId> shardIds; + cm.getAllShardIds(&shardIds); + auto it = shardIds.begin(); + std::advance(it, std::rand() % shardIds.size()); + return *it; + } + return dbInfo->getPrimary(); +} + +class AnalyzeShardKeyCmd : public TypedCommand<AnalyzeShardKeyCmd> { +public: + using Request = AnalyzeShardKey; + using Response = AnalyzeShardKeyResponse; + + class Invocation final : public InvocationBase { + public: + using InvocationBase::InvocationBase; + + Response typedRun(OperationContext* opCtx) { + const auto& nss = ns(); + const auto& catalogCache = Grid::get(opCtx)->catalogCache(); + const auto cm = uassertStatusOK(catalogCache->getCollectionRoutingInfo(opCtx, nss)); + const auto dbInfo = uassertStatusOK(catalogCache->getDatabase(opCtx, nss.db())); + + auto shardId = getRandomOwningShardId(cm, dbInfo); + uassert(ErrorCodes::IllegalOperation, + "Cannot analyze a shard key for a collection on the config server", + shardId != ShardId::kConfigServerId); + const auto shard = + uassertStatusOK(Grid::get(opCtx)->shardRegistry()->getShard(opCtx, shardId)); + + auto cmdObj = CommandHelpers::filterCommandRequestForPassthrough(request().toBSON({})); + if (cm.isSharded()) { + cmdObj = appendShardVersion(cmdObj, cm.getVersion(shardId)); + } else { + cmdObj = appendShardVersion(cmdObj, ChunkVersion::UNSHARDED()); + cmdObj = appendDbVersionIfPresent(cmdObj, dbInfo->getVersion()); + } + + auto swResponse = shard->runCommandWithFixedRetryAttempts( + opCtx, + ReadPreferenceSetting{ReadPreference::Nearest}, + NamespaceString::kAdminDb.toString(), + cmdObj, + Shard::RetryPolicy::kIdempotent); + uassertStatusOK(Shard::CommandResponse::getEffectiveStatus(swResponse)); + + auto response = AnalyzeShardKeyResponse::parse( + IDLParserContext("clusterAnalyzeShardKey"), swResponse.getValue().response); + return response; + } + + private: + NamespaceString ns() const override { + return request().getCommandParameter(); + } + + bool supportsWriteConcern() const override { + return false; + } + + void doCheckAuthorization(OperationContext* opCtx) const override { + uassert(ErrorCodes::Unauthorized, + "Unauthorized", + AuthorizationSession::get(opCtx->getClient()) + ->isAuthorizedForActionsOnResource(ResourcePattern::forExactNamespace(ns()), + ActionType::shardCollection)); + } + }; + + AllowedOnSecondary secondaryAllowed(ServiceContext*) const override { + return AllowedOnSecondary::kNever; + } + + bool adminOnly() const override { + return true; + } + + std::string help() const override { + return "Returns metrics for evaluating a shard key for a collection."; + } +}; + +MONGO_REGISTER_FEATURE_FLAGGED_COMMAND(AnalyzeShardKeyCmd, + analyze_shard_key::gFeatureFlagAnalyzeShardKey); + +} // namespace + +} // namespace mongo diff --git a/src/mongo/s/commands/cluster_configure_query_analyzer_cmd.cpp b/src/mongo/s/commands/cluster_configure_query_analyzer_cmd.cpp new file mode 100644 index 00000000000..eaacc2ff522 --- /dev/null +++ b/src/mongo/s/commands/cluster_configure_query_analyzer_cmd.cpp @@ -0,0 +1,108 @@ +/** + * Copyright (C) 2022-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * <http://www.mongodb.com/licensing/server-side-public-license>. + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/platform/basic.h" + +#include "mongo/db/auth/authorization_session.h" +#include "mongo/db/commands.h" +#include "mongo/db/s/analyze_shard_key_feature_flag_gen.h" +#include "mongo/db/s/configure_query_analyzer_cmd_gen.h" +#include "mongo/logv2/log.h" +#include "mongo/s/grid.h" + +#define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kCommand + + +namespace mongo { + +namespace { + +class ConfigureQueryAnalyzerCmd : public TypedCommand<ConfigureQueryAnalyzerCmd> { +public: + using Request = ConfigureQueryAnalyzer; + using Response = ConfigureQueryAnalyzerResponse; + + class Invocation final : public InvocationBase { + public: + using InvocationBase::InvocationBase; + + Response typedRun(OperationContext* opCtx) { + const auto configShard = Grid::get(opCtx)->shardRegistry()->getConfigShard(); + + auto swResponse = configShard->runCommandWithFixedRetryAttempts( + opCtx, + ReadPreferenceSetting{ReadPreference::PrimaryOnly}, + NamespaceString::kAdminDb.toString(), + request().toBSON({}), + Shard::RetryPolicy::kIdempotent); + uassertStatusOK(Shard::CommandResponse::getEffectiveStatus(swResponse)); + + auto response = ConfigureQueryAnalyzerResponse::parse( + IDLParserContext("clusterConfigureQueryAnalyzer"), swResponse.getValue().response); + return response; + } + + private: + NamespaceString ns() const override { + return request().getCommandParameter(); + } + + bool supportsWriteConcern() const override { + return false; + } + + void doCheckAuthorization(OperationContext* opCtx) const override { + uassert(ErrorCodes::Unauthorized, + "Unauthorized", + AuthorizationSession::get(opCtx->getClient()) + ->isAuthorizedForActionsOnResource(ResourcePattern::forExactNamespace(ns()), + ActionType::createCollection)); + } + }; + + AllowedOnSecondary secondaryAllowed(ServiceContext*) const override { + return AllowedOnSecondary::kNever; + } + + bool adminOnly() const override { + return true; + } + + std::string help() const override { + return "Starts or stops collecting metrics about read and write queries against " + "collection."; + } +}; + +MONGO_REGISTER_FEATURE_FLAGGED_COMMAND(ConfigureQueryAnalyzerCmd, + analyze_shard_key::gFeatureFlagAnalyzeShardKey); + +} // namespace + +} // namespace mongo |