diff options
Diffstat (limited to 'src')
-rw-r--r-- | src/mongo/db/mongod_main.cpp | 5 | ||||
-rw-r--r-- | src/mongo/db/s/query_analysis_coordinator.cpp | 6 | ||||
-rw-r--r-- | src/mongo/db/s/query_analysis_coordinator.h | 3 | ||||
-rw-r--r-- | src/mongo/db/s/query_analysis_coordinator_test.cpp | 12 | ||||
-rw-r--r-- | src/mongo/db/stats/counters.cpp | 1 | ||||
-rw-r--r-- | src/mongo/db/stats/counters.h | 8 | ||||
-rw-r--r-- | src/mongo/s/analyze_shard_key_util.cpp | 8 | ||||
-rw-r--r-- | src/mongo/s/analyze_shard_key_util.h | 2 | ||||
-rw-r--r-- | src/mongo/s/mongos_main.cpp | 5 | ||||
-rw-r--r-- | src/mongo/s/query_analysis_sampler.cpp | 21 | ||||
-rw-r--r-- | src/mongo/s/query_analysis_sampler.h | 5 | ||||
-rw-r--r-- | src/mongo/s/query_analysis_sampler_test.cpp | 132 | ||||
-rw-r--r-- | src/mongo/s/refresh_query_analyzer_configuration_cmd.idl | 4 | ||||
-rw-r--r-- | src/mongo/s/sharding_initialization.cpp | 5 |
14 files changed, 182 insertions, 35 deletions
diff --git a/src/mongo/db/mongod_main.cpp b/src/mongo/db/mongod_main.cpp index 49387bcaad8..39dd91a5d39 100644 --- a/src/mongo/db/mongod_main.cpp +++ b/src/mongo/db/mongod_main.cpp @@ -198,6 +198,7 @@ #include "mongo/rpc/metadata/egress_metadata_hook_list.h" #include "mongo/s/client/shard_registry.h" #include "mongo/s/grid.h" +#include "mongo/s/query_analysis_sampler.h" #include "mongo/scripting/dbdirectclient_factory.h" #include "mongo/scripting/engine.h" #include "mongo/stdx/future.h" @@ -1321,6 +1322,10 @@ void shutdownTask(const ShutdownTaskArgs& shutdownArgs) { lsc->joinOnShutDown(); } + if (analyze_shard_key::supportsSamplingQueriesIgnoreFCV()) { + LOGV2(7114100, "Shutting down the QueryAnalysisSampler"); + analyze_shard_key::QueryAnalysisSampler::get(serviceContext).onShutdown(); + } if (analyze_shard_key::supportsPersistingSampledQueriesIgnoreFCV()) { LOGV2(7047303, "Shutting down the QueryAnalysisWriter"); analyze_shard_key::QueryAnalysisWriter::get(serviceContext).onShutdown(); diff --git a/src/mongo/db/s/query_analysis_coordinator.cpp b/src/mongo/db/s/query_analysis_coordinator.cpp index 4d71cb00aae..be26d8c645b 100644 --- a/src/mongo/db/s/query_analysis_coordinator.cpp +++ b/src/mongo/db/s/query_analysis_coordinator.cpp @@ -246,15 +246,17 @@ QueryAnalysisCoordinator::getNewConfigurationsForSampler(OperationContext* opCtx ? (1.0 / numActiveSamplers) : (weight / totalWeight); + if (sampleRateRatio == 0) { + return {}; + } + // Populate the query analyzer configurations for all collections. std::vector<CollectionQueryAnalyzerConfiguration> configurations; - for (const auto& [_, configuration] : _configurations) { configurations.emplace_back(configuration.getNs(), configuration.getCollectionUuid(), sampleRateRatio * configuration.getSampleRate()); } - return configurations; } diff --git a/src/mongo/db/s/query_analysis_coordinator.h b/src/mongo/db/s/query_analysis_coordinator.h index e7b62593fcb..0135ac8f0e1 100644 --- a/src/mongo/db/s/query_analysis_coordinator.h +++ b/src/mongo/db/s/query_analysis_coordinator.h @@ -44,7 +44,8 @@ namespace analyze_shard_key { * view of the query distribution across the samplers. * * Currently, query sampling is only supported on a sharded cluster. So a sampler must be a mongos - * and the coordinator must be the config server's primary mongod. + * or shardsvr mongod (acting as a router), and the coordinator must be the config server's primary + * mongod. */ class QueryAnalysisCoordinator : public ReplicaSetAwareService<QueryAnalysisCoordinator> { public: diff --git a/src/mongo/db/s/query_analysis_coordinator_test.cpp b/src/mongo/db/s/query_analysis_coordinator_test.cpp index 86c2ad89470..b15f0e81f8b 100644 --- a/src/mongo/db/s/query_analysis_coordinator_test.cpp +++ b/src/mongo/db/s/query_analysis_coordinator_test.cpp @@ -721,16 +721,8 @@ TEST_F(QueryAnalysisCoordinatorTest, GetNewConfigurationsMultipleSamplersBasic) // Query distribution after: [1.5, 0]. configurations1 = coordinator->getNewConfigurationsForSampler(operationContext(), mongosName1, 0); - expectedRatio1 = 0; - ASSERT_EQ(configurations1.size(), 2U); - assertContainsConfiguration(configurations1, - analyzerDoc0.getNs(), - analyzerDoc0.getCollectionUuid(), - expectedRatio1 * analyzerDoc0.getSampleRate().get()); - assertContainsConfiguration(configurations1, - analyzerDoc1.getNs(), - analyzerDoc1.getCollectionUuid(), - expectedRatio1 * analyzerDoc1.getSampleRate().get()); + // The weight for this mongos is 0 so no configurations should be returned. + ASSERT(configurations1.empty()); // Query distribution after: [0, 0]. configurations0 = diff --git a/src/mongo/db/stats/counters.cpp b/src/mongo/db/stats/counters.cpp index 4300e292d02..23ea8c46212 100644 --- a/src/mongo/db/stats/counters.cpp +++ b/src/mongo/db/stats/counters.cpp @@ -49,6 +49,7 @@ void OpCounters::_reset() { _delete->store(0); _getmore->store(0); _command->store(0); + _nestedAggregate->store(0); _queryDeprecated->store(0); diff --git a/src/mongo/db/stats/counters.h b/src/mongo/db/stats/counters.h index d066202d375..c1f9c0ae8bc 100644 --- a/src/mongo/db/stats/counters.h +++ b/src/mongo/db/stats/counters.h @@ -77,6 +77,10 @@ public: _checkWrap(&OpCounters::_queryDeprecated, 1); } + void gotNestedAggregate() { + _checkWrap(&OpCounters::_nestedAggregate, 1); + } + BSONObj getObj() const; // These opcounters record operations that would fail if we were fully enforcing our consistency @@ -116,6 +120,9 @@ public: const AtomicWord<long long>* getCommand() const { return &*_command; } + const AtomicWord<long long>* getNestedAggregate() const { + return &*_nestedAggregate; + } const AtomicWord<long long>* getInsertOnExistingDoc() const { return &*_insertOnExistingDoc; } @@ -150,6 +157,7 @@ private: CacheExclusive<AtomicWord<long long>> _delete; CacheExclusive<AtomicWord<long long>> _getmore; CacheExclusive<AtomicWord<long long>> _command; + CacheExclusive<AtomicWord<long long>> _nestedAggregate; CacheExclusive<AtomicWord<long long>> _insertOnExistingDoc; CacheExclusive<AtomicWord<long long>> _updateOnMissingDoc; diff --git a/src/mongo/s/analyze_shard_key_util.cpp b/src/mongo/s/analyze_shard_key_util.cpp index 88e78ee7abb..dad7139f9a0 100644 --- a/src/mongo/s/analyze_shard_key_util.cpp +++ b/src/mongo/s/analyze_shard_key_util.cpp @@ -59,7 +59,13 @@ bool supportsPersistingSampledQueriesIgnoreFCV() { } bool supportsSamplingQueries() { - return isFeatureFlagEnabled() && isMongos(); + return isFeatureFlagEnabled() && + (isMongos() || serverGlobalParams.clusterRole == ClusterRole::ShardServer); +} + +bool supportsSamplingQueriesIgnoreFCV() { + return isFeatureFlagEnabledIgnoreFCV() && + (isMongos() || serverGlobalParams.clusterRole == ClusterRole::ShardServer); } } // namespace analyze_shard_key diff --git a/src/mongo/s/analyze_shard_key_util.h b/src/mongo/s/analyze_shard_key_util.h index aeafed30c82..5e74cd7e5e6 100644 --- a/src/mongo/s/analyze_shard_key_util.h +++ b/src/mongo/s/analyze_shard_key_util.h @@ -46,5 +46,7 @@ bool supportsPersistingSampledQueriesIgnoreFCV(); bool supportsSamplingQueries(); +bool supportsSamplingQueriesIgnoreFCV(); + } // namespace analyze_shard_key } // namespace mongo diff --git a/src/mongo/s/mongos_main.cpp b/src/mongo/s/mongos_main.cpp index d365a2661fe..a359ea597fb 100644 --- a/src/mongo/s/mongos_main.cpp +++ b/src/mongo/s/mongos_main.cpp @@ -795,11 +795,6 @@ ExitCode runMongosServer(ServiceContext* serviceContext) { srand((unsigned)(curTimeMicros64()) ^ (unsigned(uintptr_t(&opCtx)))); - if (analyze_shard_key::isFeatureFlagEnabled()) { - LOGV2_OPTIONS(6973900, {LogComponent::kDefault}, "Starting the QueryAnalysisSampler"); - analyze_shard_key::QueryAnalysisSampler::get(serviceContext).onStartup(); - } - SessionKiller::set(serviceContext, std::make_shared<SessionKiller>(serviceContext, killSessionsRemote)); diff --git a/src/mongo/s/query_analysis_sampler.cpp b/src/mongo/s/query_analysis_sampler.cpp index ba0770f229b..8c3fbe27e31 100644 --- a/src/mongo/s/query_analysis_sampler.cpp +++ b/src/mongo/s/query_analysis_sampler.cpp @@ -62,9 +62,10 @@ QueryAnalysisSampler& QueryAnalysisSampler::get(OperationContext* opCtx) { } QueryAnalysisSampler& QueryAnalysisSampler::get(ServiceContext* serviceContext) { - invariant(analyze_shard_key::isFeatureFlagEnabled(), + invariant(analyze_shard_key::isFeatureFlagEnabledIgnoreFCV(), "Only support analyzing queries when the feature flag is enabled"); - invariant(isMongos(), "Only support analyzing queries on a sharded cluster"); + invariant(isMongos() || serverGlobalParams.clusterRole == ClusterRole::ShardServer, + "Only support analyzing queries on a sharded cluster"); return getQueryAnalysisSampler(serviceContext); } @@ -122,15 +123,23 @@ void QueryAnalysisSampler::QueryStats::refreshTotalCount(long long newTotalCount _lastTotalCount = newTotalCount; } +long long QueryAnalysisSampler::_getTotalQueriesCount() const { + if (isMongos()) { + return globalOpCounters.getQuery()->load() + globalOpCounters.getInsert()->load() + + globalOpCounters.getUpdate()->load() + globalOpCounters.getDelete()->load() + + globalOpCounters.getCommand()->load(); + } else if (serverGlobalParams.clusterRole == ClusterRole::ShardServer) { + return globalOpCounters.getNestedAggregate()->load(); + } + MONGO_UNREACHABLE; +} + void QueryAnalysisSampler::_refreshQueryStats() { if (MONGO_unlikely(disableQueryAnalysisSampler.shouldFail())) { return; } - long long newTotalCount = globalOpCounters.getQuery()->load() + - globalOpCounters.getInsert()->load() + globalOpCounters.getUpdate()->load() + - globalOpCounters.getDelete()->load() + globalOpCounters.getCommand()->load(); - + long long newTotalCount = _getTotalQueriesCount(); stdx::lock_guard<Latch> lk(_mutex); _queryStats.refreshTotalCount(newTotalCount); } diff --git a/src/mongo/s/query_analysis_sampler.h b/src/mongo/s/query_analysis_sampler.h index c03d36ffd25..b8532e91397 100644 --- a/src/mongo/s/query_analysis_sampler.h +++ b/src/mongo/s/query_analysis_sampler.h @@ -49,7 +49,8 @@ namespace analyze_shard_key { * on this sampler. * * Currently, query sampling is only supported on a sharded cluster. So a sampler must be a mongos - * and the coordinator must be the config server's primary mongod. + * or a shardsvr mongod (acting as a router), and the coordinator must be the config server's + * primary mongod. */ class QueryAnalysisSampler final { QueryAnalysisSampler(const QueryAnalysisSampler&) = delete; @@ -203,6 +204,8 @@ public: } private: + long long _getTotalQueriesCount() const; + void _refreshQueryStats(); void _refreshConfigurations(OperationContext* opCtx); diff --git a/src/mongo/s/query_analysis_sampler_test.cpp b/src/mongo/s/query_analysis_sampler_test.cpp index fda0a8d4688..91db4a91314 100644 --- a/src/mongo/s/query_analysis_sampler_test.cpp +++ b/src/mongo/s/query_analysis_sampler_test.cpp @@ -470,6 +470,7 @@ public: void tearDown() override { ShardingTestFixture::tearDown(); setMongos(_originalIsMongos); + serverGlobalParams.clusterRole = ClusterRole::None; } protected: @@ -549,12 +550,12 @@ DEATH_TEST_F(QueryAnalysisSamplerTest, CannotGetIfFeatureFlagNotEnabled, "invari QueryAnalysisSampler::get(operationContext()); } -DEATH_TEST_F(QueryAnalysisSamplerTest, CannotGetIfNotMongos, "invariant") { +DEATH_TEST_F(QueryAnalysisSamplerTest, CannotGetIfNotMongosOrShardServer, "invariant") { setMongos(false); QueryAnalysisSampler::get(operationContext()); } -TEST_F(QueryAnalysisSamplerTest, RefreshQueryStats_InitialCount) { +TEST_F(QueryAnalysisSamplerTest, RefreshQueryStatsMongos_InitialCount) { auto& sampler = QueryAnalysisSampler::get(operationContext()); sampler.refreshQueryStatsForTest(); @@ -565,7 +566,7 @@ TEST_F(QueryAnalysisSamplerTest, RefreshQueryStats_InitialCount) { ASSERT_EQ(*lastAvgCount, 0); } -TEST_F(QueryAnalysisSamplerTest, RefreshQueryStats_CountQueries) { +TEST_F(QueryAnalysisSamplerTest, RefreshQueryStatsMongos_CountQueries) { auto& sampler = QueryAnalysisSampler::get(operationContext()); globalOpCounters.gotQuery(); sampler.refreshQueryStatsForTest(); @@ -577,7 +578,7 @@ TEST_F(QueryAnalysisSamplerTest, RefreshQueryStats_CountQueries) { ASSERT_EQ(*lastAvgCount, 1); } -TEST_F(QueryAnalysisSamplerTest, RefreshQueryStats_CountInserts) { +TEST_F(QueryAnalysisSamplerTest, RefreshQueryStatsMongos_CountInserts) { auto& sampler = QueryAnalysisSampler::get(operationContext()); auto numInserts = 3; globalOpCounters.gotInserts(numInserts); @@ -590,7 +591,7 @@ TEST_F(QueryAnalysisSamplerTest, RefreshQueryStats_CountInserts) { ASSERT_EQ(*lastAvgCount, numInserts); } -TEST_F(QueryAnalysisSamplerTest, RefreshQueryStats_CountUpdates) { +TEST_F(QueryAnalysisSamplerTest, RefreshQueryStatsMongos_CountUpdates) { auto& sampler = QueryAnalysisSampler::get(operationContext()); globalOpCounters.gotUpdate(); sampler.refreshQueryStatsForTest(); @@ -602,7 +603,7 @@ TEST_F(QueryAnalysisSamplerTest, RefreshQueryStats_CountUpdates) { ASSERT_EQ(*lastAvgCount, 1); } -TEST_F(QueryAnalysisSamplerTest, RefreshQueryStats_CountDeletes) { +TEST_F(QueryAnalysisSamplerTest, RefreshQueryStatsMongos_CountDeletes) { auto& sampler = QueryAnalysisSampler::get(operationContext()); globalOpCounters.gotDelete(); sampler.refreshQueryStatsForTest(); @@ -614,7 +615,7 @@ TEST_F(QueryAnalysisSamplerTest, RefreshQueryStats_CountDeletes) { ASSERT_EQ(*lastAvgCount, 1); } -TEST_F(QueryAnalysisSamplerTest, RefreshQueryStats_CountCommands) { +TEST_F(QueryAnalysisSamplerTest, RefreshQueryStatsMongos_CountCommands) { auto& sampler = QueryAnalysisSampler::get(operationContext()); globalOpCounters.gotCommand(); sampler.refreshQueryStatsForTest(); @@ -626,6 +627,123 @@ TEST_F(QueryAnalysisSamplerTest, RefreshQueryStats_CountCommands) { ASSERT_EQ(*lastAvgCount, 1); } +TEST_F(QueryAnalysisSamplerTest, RefreshQueryStatsMongos_NotCountNestedAggregates) { + auto& sampler = QueryAnalysisSampler::get(operationContext()); + globalOpCounters.gotNestedAggregate(); + sampler.refreshQueryStatsForTest(); + + auto queryStats = sampler.getQueryStatsForTest(); + ASSERT_EQ(queryStats.getLastTotalCount(), 0); + auto lastAvgCount = queryStats.getLastAvgCount(); + ASSERT(lastAvgCount); + ASSERT_EQ(*lastAvgCount, 0); +} + +TEST_F(QueryAnalysisSamplerTest, RefreshQueryStatsShardSvr_InitialCount) { + setMongos(false); + serverGlobalParams.clusterRole = ClusterRole::ShardServer; + + auto& sampler = QueryAnalysisSampler::get(operationContext()); + sampler.refreshQueryStatsForTest(); + + auto queryStats = sampler.getQueryStatsForTest(); + ASSERT_EQ(queryStats.getLastTotalCount(), 0); + auto lastAvgCount = queryStats.getLastAvgCount(); + ASSERT(lastAvgCount); + ASSERT_EQ(*lastAvgCount, 0); +} + +TEST_F(QueryAnalysisSamplerTest, RefreshQueryStatsShardSvr_NotCountQueries) { + setMongos(false); + serverGlobalParams.clusterRole = ClusterRole::ShardServer; + + auto& sampler = QueryAnalysisSampler::get(operationContext()); + globalOpCounters.gotQuery(); + sampler.refreshQueryStatsForTest(); + + auto queryStats = sampler.getQueryStatsForTest(); + ASSERT_EQ(queryStats.getLastTotalCount(), 0); + auto lastAvgCount = queryStats.getLastAvgCount(); + ASSERT(lastAvgCount); + ASSERT_EQ(*lastAvgCount, 0); +} + +TEST_F(QueryAnalysisSamplerTest, RefreshQueryStatsShardSvr_NotCountInserts) { + setMongos(false); + serverGlobalParams.clusterRole = ClusterRole::ShardServer; + + auto& sampler = QueryAnalysisSampler::get(operationContext()); + auto numInserts = 3; + globalOpCounters.gotInserts(numInserts); + sampler.refreshQueryStatsForTest(); + + auto queryStats = sampler.getQueryStatsForTest(); + ASSERT_EQ(queryStats.getLastTotalCount(), 0); + auto lastAvgCount = queryStats.getLastAvgCount(); + ASSERT(lastAvgCount); + ASSERT_EQ(*lastAvgCount, 0); +} + +TEST_F(QueryAnalysisSamplerTest, RefreshQueryStatsShardSvr_NotCountUpdates) { + setMongos(false); + serverGlobalParams.clusterRole = ClusterRole::ShardServer; + + auto& sampler = QueryAnalysisSampler::get(operationContext()); + globalOpCounters.gotUpdate(); + sampler.refreshQueryStatsForTest(); + + auto queryStats = sampler.getQueryStatsForTest(); + ASSERT_EQ(queryStats.getLastTotalCount(), 0); + auto lastAvgCount = queryStats.getLastAvgCount(); + ASSERT(lastAvgCount); + ASSERT_EQ(*lastAvgCount, 0); +} + +TEST_F(QueryAnalysisSamplerTest, RefreshQueryStatsShardSvr_NotCountDeletes) { + setMongos(false); + serverGlobalParams.clusterRole = ClusterRole::ShardServer; + + auto& sampler = QueryAnalysisSampler::get(operationContext()); + globalOpCounters.gotDelete(); + sampler.refreshQueryStatsForTest(); + + auto queryStats = sampler.getQueryStatsForTest(); + ASSERT_EQ(queryStats.getLastTotalCount(), 0); + auto lastAvgCount = queryStats.getLastAvgCount(); + ASSERT(lastAvgCount); + ASSERT_EQ(*lastAvgCount, 0); +} + +TEST_F(QueryAnalysisSamplerTest, RefreshQueryStatsShardSvr_NotCountCommands) { + setMongos(false); + serverGlobalParams.clusterRole = ClusterRole::ShardServer; + + auto& sampler = QueryAnalysisSampler::get(operationContext()); + globalOpCounters.gotCommand(); + sampler.refreshQueryStatsForTest(); + + auto queryStats = sampler.getQueryStatsForTest(); + ASSERT_EQ(queryStats.getLastTotalCount(), 0); + auto lastAvgCount = queryStats.getLastAvgCount(); + ASSERT(lastAvgCount); + ASSERT_EQ(*lastAvgCount, 0); +} + +TEST_F(QueryAnalysisSamplerTest, RefreshQueryStatsShardSvr_CountNestedAggregates) { + setMongos(false); + serverGlobalParams.clusterRole = ClusterRole::ShardServer; + + auto& sampler = QueryAnalysisSampler::get(operationContext()); + globalOpCounters.gotNestedAggregate(); + sampler.refreshQueryStatsForTest(); + + auto queryStats = sampler.getQueryStatsForTest(); + ASSERT_EQ(queryStats.getLastTotalCount(), 1); + auto lastAvgCount = queryStats.getLastAvgCount(); + ASSERT(lastAvgCount); + ASSERT_EQ(*lastAvgCount, 1); +} + TEST_F(QueryAnalysisSamplerTest, RefreshQueryStatsAndConfigurations) { auto& sampler = QueryAnalysisSampler::get(operationContext()); diff --git a/src/mongo/s/refresh_query_analyzer_configuration_cmd.idl b/src/mongo/s/refresh_query_analyzer_configuration_cmd.idl index ff62fa4b45c..73d9a353a8a 100644 --- a/src/mongo/s/refresh_query_analyzer_configuration_cmd.idl +++ b/src/mongo/s/refresh_query_analyzer_configuration_cmd.idl @@ -51,8 +51,8 @@ commands: api_version: "" fields: name: - description: "The unique identifier for the sampler (currently, must be a mongos connection - string)." + description: "The unique identifier for the sampler (currently, must be the connection + string of a mongos or shardsvr mongod)." type: string numQueriesExecutedPerSecond: description: "The exponential moving average of the number of queries that the sampler diff --git a/src/mongo/s/sharding_initialization.cpp b/src/mongo/s/sharding_initialization.cpp index b526895db1e..8c171a6a8f8 100644 --- a/src/mongo/s/sharding_initialization.cpp +++ b/src/mongo/s/sharding_initialization.cpp @@ -67,6 +67,7 @@ #include "mongo/s/initialize_tenant_to_shard_cache.h" #include "mongo/s/mongod_and_mongos_server_parameters_gen.h" #include "mongo/s/query/cluster_cursor_manager.h" +#include "mongo/s/query_analysis_sampler.h" #include "mongo/s/sharding_task_executor.h" #include "mongo/s/sharding_task_executor_pool_controller.h" #include "mongo/s/sharding_task_executor_pool_gen.h" @@ -209,6 +210,10 @@ Status initializeGlobalShardingState(OperationContext* opCtx, LogicalTimeValidator::set(service, std::make_unique<LogicalTimeValidator>(keyManager)); initializeTenantToShardCache(service); + if (analyze_shard_key::supportsSamplingQueriesIgnoreFCV()) { + analyze_shard_key::QueryAnalysisSampler::get(service).onStartup(); + } + return Status::OK(); } |