diff options
| author | Kaloian Manassiev <kaloian.manassiev@mongodb.com> | 2023-02-03 16:50:47 +0000 |
|---|---|---|
| committer | Evergreen Agent <no-reply@evergreen.mongodb.com> | 2023-02-03 18:09:18 +0000 |
| commit | 5014eef0db04353eb8fc937c502f28de04ab7a85 (patch) | |
| tree | 5480ab9518fd2e746b9b8d084da45a25463bc6f5 | |
| parent | c4712f56b3c1cd32b514a6f68c414bab3aa226c8 (diff) | |
| download | mongo-5014eef0db04353eb8fc937c502f28de04ab7a85.tar.gz | |
SERVER-73469 Remove dependency of the ChunkManager on Query
47 files changed, 661 insertions, 663 deletions
diff --git a/src/mongo/client/SConscript b/src/mongo/client/SConscript index 372518ece45..ead270f65a5 100644 --- a/src/mongo/client/SConscript +++ b/src/mongo/client/SConscript @@ -304,7 +304,6 @@ env.Library( 'remote_command_targeter_factory_mock.cpp', ], LIBDEPS=[ - '$BUILD_DIR/mongo/s/coreshard', '$BUILD_DIR/mongo/util/net/network', ], ) diff --git a/src/mongo/db/SConscript b/src/mongo/db/SConscript index b12c878c3e5..a6da531de31 100644 --- a/src/mongo/db/SConscript +++ b/src/mongo/db/SConscript @@ -829,6 +829,7 @@ env.Library( 's/sharding_api_d_DO_NOT_ADD_MORE_USAGES', ], LIBDEPS_PRIVATE=[ + 'catalog/clustered_collection_options', 'server_base', ], ) @@ -1312,6 +1313,7 @@ env.Library( 'exec/shard_filterer_impl.cpp', ], LIBDEPS_PRIVATE=[ + 'exec/working_set', 'shard_role_api', ], ) @@ -1675,7 +1677,7 @@ env.Library( ], LIBDEPS=[ '$BUILD_DIR/mongo/db/session/logical_session_id', - '$BUILD_DIR/mongo/s/coreshard', + '$BUILD_DIR/mongo/s/grid', '$BUILD_DIR/mongo/s/query/cluster_cursor_manager', '$BUILD_DIR/mongo/util/clock_sources', '$BUILD_DIR/mongo/util/periodic_runner', diff --git a/src/mongo/db/auth/SConscript b/src/mongo/db/auth/SConscript index eb1f7560aa3..4a9dc2ab929 100644 --- a/src/mongo/db/auth/SConscript +++ b/src/mongo/db/auth/SConscript @@ -476,7 +476,7 @@ env.Library( LIBDEPS_PRIVATE=[ '$BUILD_DIR/mongo/db/multitenancy', '$BUILD_DIR/mongo/db/server_base', - '$BUILD_DIR/mongo/s/coreshard', + '$BUILD_DIR/mongo/s/grid', 'authservercommon', ], ) diff --git a/src/mongo/db/catalog/SConscript b/src/mongo/db/catalog/SConscript index 3cdfb6adcea..0d66453c412 100644 --- a/src/mongo/db/catalog/SConscript +++ b/src/mongo/db/catalog/SConscript @@ -128,6 +128,7 @@ env.Library( '$BUILD_DIR/mongo/db/index/index_access_method', '$BUILD_DIR/mongo/db/query/op_metrics', '$BUILD_DIR/mongo/db/shard_role', + '$BUILD_DIR/mongo/db/storage/duplicate_key_error_info', '$BUILD_DIR/mongo/db/storage/key_string', 'collection_crud', 'validate_state', @@ -236,6 +237,7 @@ env.Library( '$BUILD_DIR/mongo/db/repl/repl_coordinator_interface', '$BUILD_DIR/mongo/db/shard_role_api', '$BUILD_DIR/mongo/db/storage/capped_snapshots', + '$BUILD_DIR/mongo/db/storage/index_entry_comparison', '$BUILD_DIR/mongo/db/storage/record_store_base', '$BUILD_DIR/mongo/db/storage/storage_options', '$BUILD_DIR/mongo/db/storage/write_unit_of_work', @@ -477,6 +479,7 @@ env.Library( '$BUILD_DIR/mongo/db/timeseries/bucket_catalog/bucket_catalog', '$BUILD_DIR/mongo/db/timeseries/timeseries_options', 'catalog_impl', + 'clustered_collection_options', 'collection_options', 'index_catalog', 'index_key_validate', diff --git a/src/mongo/db/catalog/index_catalog_entry.h b/src/mongo/db/catalog/index_catalog_entry.h index b4f6d6d582e..71bad9953b3 100644 --- a/src/mongo/db/catalog/index_catalog_entry.h +++ b/src/mongo/db/catalog/index_catalog_entry.h @@ -44,6 +44,7 @@ #include "mongo/util/debug_util.h" namespace mongo { + class CollatorInterface; class Collection; class CollectionPtr; diff --git a/src/mongo/db/exec/sbe/SConscript b/src/mongo/db/exec/sbe/SConscript index 947fb945d5e..46163e0175a 100644 --- a/src/mongo/db/exec/sbe/SConscript +++ b/src/mongo/db/exec/sbe/SConscript @@ -126,9 +126,11 @@ env.Library( 'stages/scan.cpp', ], LIBDEPS=[ + '$BUILD_DIR/mongo/db/exec/scoped_timer', '$BUILD_DIR/mongo/db/index/index_access_method', '$BUILD_DIR/mongo/db/shard_role', '$BUILD_DIR/mongo/db/storage/execution_context', + '$BUILD_DIR/mongo/db/storage/index_entry_comparison', 'query_sbe', ], ) diff --git a/src/mongo/db/exec/sbe/values/value.h b/src/mongo/db/exec/sbe/values/value.h index fc5c8b1cc50..0459118ddf1 100644 --- a/src/mongo/db/exec/sbe/values/value.h +++ b/src/mongo/db/exec/sbe/values/value.h @@ -46,8 +46,10 @@ #include "mongo/config.h" #include "mongo/db/exec/shard_filterer.h" #include "mongo/db/fts/fts_matcher.h" +#include "mongo/db/matcher/expression.h" #include "mongo/db/query/bson_typemask.h" #include "mongo/db/query/collation/collator_interface.h" +#include "mongo/db/query/index_bounds.h" #include "mongo/platform/bits.h" #include "mongo/platform/decimal128.h" #include "mongo/platform/endian.h" diff --git a/src/mongo/db/pipeline/document_source_change_stream_handle_topology_change.cpp b/src/mongo/db/pipeline/document_source_change_stream_handle_topology_change.cpp index 7aa410dfe35..cfd2a66e854 100644 --- a/src/mongo/db/pipeline/document_source_change_stream_handle_topology_change.cpp +++ b/src/mongo/db/pipeline/document_source_change_stream_handle_topology_change.cpp @@ -37,7 +37,6 @@ #include "mongo/db/query/query_feature_flags_gen.h" #include "mongo/s/catalog/type_shard.h" #include "mongo/s/client/shard_registry.h" -#include "mongo/s/cluster_commands_helpers.h" #include "mongo/s/grid.h" #include "mongo/s/query/establish_cursors.h" #include "mongo/util/fail_point.h" diff --git a/src/mongo/db/query/get_executor.cpp b/src/mongo/db/query/get_executor.cpp index f4ece019da5..b3a28aef907 100644 --- a/src/mongo/db/query/get_executor.cpp +++ b/src/mongo/db/query/get_executor.cpp @@ -103,6 +103,7 @@ #include "mongo/db/service_context.h" #include "mongo/db/storage/storage_options.h" #include "mongo/logv2/log.h" +#include "mongo/s/shard_key_pattern_query_util.h" #include "mongo/scripting/engine.h" #include "mongo/util/duration.h" #include "mongo/util/processinfo.h" @@ -390,7 +391,7 @@ void fillOutPlannerParams(OperationContext* opCtx, // to include a shard filtering stage. By omitting the shard filter, it may be possible // to get a more efficient plan (for example, a COUNT_SCAN may be used if the query is // eligible). - const BSONObj extractedKey = shardKeyPattern.extractShardKeyFromQuery(*canonicalQuery); + const BSONObj extractedKey = extractShardKeyFromQuery(shardKeyPattern, *canonicalQuery); if (extractedKey.isEmpty()) { plannerParams->shardKey = shardKeyPattern.toBSON(); diff --git a/src/mongo/db/repl/SConscript b/src/mongo/db/repl/SConscript index d17bc24744e..9fa145065b1 100644 --- a/src/mongo/db/repl/SConscript +++ b/src/mongo/db/repl/SConscript @@ -393,6 +393,7 @@ env.Library( 'storage_interface', ], LIBDEPS_PRIVATE=[ + '$BUILD_DIR/mongo/db/catalog/clustered_collection_options', '$BUILD_DIR/mongo/db/catalog/document_validation', '$BUILD_DIR/mongo/db/ops/write_ops_exec', ], diff --git a/src/mongo/db/s/analyze_shard_key_read_write_distribution.cpp b/src/mongo/db/s/analyze_shard_key_read_write_distribution.cpp index 60153d6d8ac..1e8d200874b 100644 --- a/src/mongo/db/s/analyze_shard_key_read_write_distribution.cpp +++ b/src/mongo/db/s/analyze_shard_key_read_write_distribution.cpp @@ -38,6 +38,7 @@ #include "mongo/logv2/log.h" #include "mongo/s/analyze_shard_key_util.h" #include "mongo/s/cluster_commands_helpers.h" +#include "mongo/s/collection_routing_info_targeter.h" #include "mongo/s/grid.h" #include "mongo/s/shard_key_pattern_query_util.h" @@ -134,8 +135,13 @@ DistributionMetricsCalculator<DistributionMetricsType, SampleSizeType>::_increme std::set<ShardId> shardIds; // This is not used. std::set<ChunkRange> chunkRanges; bool targetMinkeyToMaxKey = false; - _getChunkManager().getShardIdsForQuery( - expCtx, filter, collation, &shardIds, &chunkRanges, &targetMinkeyToMaxKey); + getShardIdsForQuery(expCtx, + filter, + collation, + _getChunkManager(), + &shardIds, + &chunkRanges, + &targetMinkeyToMaxKey); _incrementNumDispatchedByRanges(chunkRanges); // Increment metrics about sharding targeting. diff --git a/src/mongo/db/s/migration_coordinator.h b/src/mongo/db/s/migration_coordinator.h index da0afc0a309..13cd7eba36b 100644 --- a/src/mongo/db/s/migration_coordinator.h +++ b/src/mongo/db/s/migration_coordinator.h @@ -32,6 +32,7 @@ #include "mongo/db/s/migration_coordinator_document_gen.h" #include "mongo/db/session/logical_session_id.h" #include "mongo/s/catalog/type_chunk.h" +#include "mongo/util/future.h" namespace mongo { namespace migrationutil { diff --git a/src/mongo/db/stats/SConscript b/src/mongo/db/stats/SConscript index e0b028c8bab..f3eafa07a07 100644 --- a/src/mongo/db/stats/SConscript +++ b/src/mongo/db/stats/SConscript @@ -125,6 +125,7 @@ env.Library( 'storage_stats.cpp', ], LIBDEPS_PRIVATE=[ + '$BUILD_DIR/mongo/db/catalog/clustered_collection_options', '$BUILD_DIR/mongo/db/catalog/index_catalog', '$BUILD_DIR/mongo/db/commands/server_status_core', '$BUILD_DIR/mongo/db/index/index_access_method', diff --git a/src/mongo/db/storage/SConscript b/src/mongo/db/storage/SConscript index ac181834b96..ba4af679c22 100644 --- a/src/mongo/db/storage/SConscript +++ b/src/mongo/db/storage/SConscript @@ -648,6 +648,7 @@ env.Library( LIBDEPS_PRIVATE=[ '$BUILD_DIR/mongo/db/audit', '$BUILD_DIR/mongo/db/catalog/catalog_helpers', + '$BUILD_DIR/mongo/db/catalog/clustered_collection_options', '$BUILD_DIR/mongo/db/catalog/index_catalog', '$BUILD_DIR/mongo/db/multitenancy', '$BUILD_DIR/mongo/db/resumable_index_builds_idl', diff --git a/src/mongo/db/storage/wiredtiger/SConscript b/src/mongo/db/storage/wiredtiger/SConscript index c7406576127..1c7534ab0d4 100644 --- a/src/mongo/db/storage/wiredtiger/SConscript +++ b/src/mongo/db/storage/wiredtiger/SConscript @@ -85,6 +85,7 @@ wtEnv.Library( '$BUILD_DIR/mongo/db/server_base', '$BUILD_DIR/mongo/db/shard_role', '$BUILD_DIR/mongo/db/snapshot_window_options', + '$BUILD_DIR/mongo/db/stats/resource_consumption_metrics', '$BUILD_DIR/mongo/db/storage/backup_block', '$BUILD_DIR/mongo/db/storage/capped_snapshots', '$BUILD_DIR/mongo/db/storage/storage_engine_parameters', diff --git a/src/mongo/idl/SConscript b/src/mongo/idl/SConscript index 846c0f55c20..3181216eba6 100644 --- a/src/mongo/idl/SConscript +++ b/src/mongo/idl/SConscript @@ -117,16 +117,14 @@ env.Library( 'generic_args_with_types.idl', ], LIBDEPS=[ - '$BUILD_DIR/mongo/base', '$BUILD_DIR/mongo/client/read_preference', + '$BUILD_DIR/mongo/db/repl/read_concern_args', '$BUILD_DIR/mongo/db/s/forwardable_operation_metadata', + '$BUILD_DIR/mongo/db/server_base', '$BUILD_DIR/mongo/db/session/logical_session_id', '$BUILD_DIR/mongo/rpc/client_metadata', '$BUILD_DIR/mongo/s/common_s', ], - LIBDEPS_PRIVATE=[ - '$BUILD_DIR/mongo/db/server_base', - ], ) env.CppUnitTest( diff --git a/src/mongo/idl/generic_args_with_types.idl b/src/mongo/idl/generic_args_with_types.idl index f34ae3acecd..90b270650c3 100644 --- a/src/mongo/idl/generic_args_with_types.idl +++ b/src/mongo/idl/generic_args_with_types.idl @@ -25,13 +25,13 @@ # exception statement from all source files in the program, then also delete # it in the license file. # + global: cpp_namespace: "mongo" cpp_includes: - "mongo/db/basic_types.h" - "mongo/s/shard_version.h" - imports: - "mongo/client/read_preference_setting.idl" - "mongo/db/basic_types.idl" diff --git a/src/mongo/rpc/SConscript b/src/mongo/rpc/SConscript index 4eaef7014d0..72e3bce9fd9 100644 --- a/src/mongo/rpc/SConscript +++ b/src/mongo/rpc/SConscript @@ -71,6 +71,7 @@ env.Library( 'metadata', ], LIBDEPS_PRIVATE=[ + '$BUILD_DIR/mongo/db/commands/test_commands_enabled', '$BUILD_DIR/mongo/db/server_base', 'rewrite_state_change_errors', ], diff --git a/src/mongo/s/SConscript b/src/mongo/s/SConscript index 1fe2c750479..4c2f2f8c593 100644 --- a/src/mongo/s/SConscript +++ b/src/mongo/s/SConscript @@ -32,7 +32,6 @@ env.Library( 'cluster_ddl.cpp', 'cluster_write.cpp', 'collection_routing_info_targeter.cpp', - 'shard_key_pattern_query_util.cpp', 'write_ops/batch_write_exec.cpp', 'write_ops/batch_write_op.cpp', 'write_ops/bulk_write_exec.cpp', @@ -80,6 +79,7 @@ env.Library( 'router_transactions_stats.idl', 'router.cpp', 'session_catalog_router.cpp', + 'shard_key_pattern_query_util.cpp', 'stale_shard_version_helpers.cpp', 'transaction_router_resource_yielder.cpp', 'transaction_router.cpp', @@ -99,6 +99,8 @@ env.Library( LIBDEPS_PRIVATE=[ '$BUILD_DIR/mongo/db/catalog/collection_uuid_mismatch_info', '$BUILD_DIR/mongo/db/internal_transactions_feature_flag', + '$BUILD_DIR/mongo/db/mongohasher', + '$BUILD_DIR/mongo/db/query/query_planner', '$BUILD_DIR/mongo/db/session/sessions_collection', ], ) @@ -209,8 +211,11 @@ env.Library( 'catalog/type_shard.cpp', 'catalog/type_tags.cpp', 'check_metadata_consistency.idl', + 'chunk.cpp', + 'chunk_manager.cpp', 'chunk_version.cpp', 'chunk_version.idl', + 'chunk_writes_tracker.cpp', 'configure_query_analyzer_cmd.idl', 'database_version.cpp', 'database_version.idl', @@ -256,6 +261,7 @@ env.Library( 'resharding/type_collection_fields.idl', 'shard_cannot_refresh_due_to_locks_held_exception.cpp', 'shard_invalidated_for_targeting_exception.cpp', + 'shard_key_pattern.cpp', 'shard_version.cpp', 'shard_version.idl', 'sharding_feature_flags.idl', @@ -270,14 +276,15 @@ env.Library( '$BUILD_DIR/mongo/db/commands/set_user_write_block_mode_idl', '$BUILD_DIR/mongo/db/common', '$BUILD_DIR/mongo/db/index_commands_idl', - '$BUILD_DIR/mongo/db/query/query_request', - '$BUILD_DIR/mongo/db/repl/optime', - '$BUILD_DIR/mongo/db/server_options', '$BUILD_DIR/mongo/rpc/message', + '$BUILD_DIR/mongo/util/caching', 'analyze_shard_key_idl', ], LIBDEPS_PRIVATE=[ + '$BUILD_DIR/mongo/db/matcher/path', + '$BUILD_DIR/mongo/db/mongohasher', '$BUILD_DIR/mongo/db/server_base', + '$BUILD_DIR/mongo/db/storage/key_string', '$BUILD_DIR/mongo/db/timeseries/timeseries_options', '$BUILD_DIR/mongo/util/namespace_string_database_name_util', ], @@ -303,28 +310,18 @@ env.Library( source=[ 'balancer_configuration.cpp', 'catalog_cache.cpp', - 'chunk_manager.cpp', - 'chunk_writes_tracker.cpp', - 'chunk.cpp', 'client/shard_factory.cpp', 'client/shard_registry.cpp', 'global_index_cache.cpp', 'grid.cpp', - 'shard_key_pattern.cpp', ], LIBDEPS=[ '$BUILD_DIR/mongo/client/clientdriver_network', '$BUILD_DIR/mongo/db/logical_time_metadata_hook', - '$BUILD_DIR/mongo/db/mongohasher', - '$BUILD_DIR/mongo/db/query/query_planner', - '$BUILD_DIR/mongo/db/query_expressions', '$BUILD_DIR/mongo/db/server_base', - '$BUILD_DIR/mongo/db/storage/key_string', '$BUILD_DIR/mongo/db/update/update_common', '$BUILD_DIR/mongo/executor/task_executor_pool', - '$BUILD_DIR/mongo/util/caching', '$BUILD_DIR/mongo/util/concurrency/thread_pool', - '$BUILD_DIR/mongo/util/concurrency/ticketholder', 'client/shard_interface', 'common_s', 'query/cluster_cursor_manager', @@ -611,6 +608,7 @@ env.Library( '$BUILD_DIR/mongo/executor/network_interface_mock', '$BUILD_DIR/mongo/executor/network_test_env', '$BUILD_DIR/mongo/executor/thread_pool_task_executor_test_fixture', + 'coreshard', 'write_ops/batch_write_types', ], ) @@ -640,8 +638,6 @@ env.CppUnitTest( 'async_requests_sender_test.cpp', 'async_rpc_shard_targeter_test.cpp', 'balancer_configuration_test.cpp', - 'catalog_cache_refresh_test.cpp', - 'catalog_cache_test.cpp', 'catalog/sharding_catalog_client_test.cpp', 'catalog/sharding_catalog_write_retry_test.cpp', 'catalog/type_changelog_test.cpp', @@ -652,7 +648,8 @@ env.CppUnitTest( 'catalog/type_mongos_test.cpp', 'catalog/type_shard_test.cpp', 'catalog/type_tags_test.cpp', - 'chunk_manager_index_bounds_test.cpp', + 'catalog_cache_refresh_test.cpp', + 'catalog_cache_test.cpp', 'chunk_manager_query_test.cpp', 'chunk_map_test.cpp', 'chunk_test.cpp', @@ -666,8 +663,8 @@ env.CppUnitTest( 'comparable_index_version_test.cpp', 'global_index_cache_test.cpp', 'load_balancer_support_test.cpp', - 'mongos_core_options_stub.cpp', 'mock_ns_targeter.cpp', + 'mongos_core_options_stub.cpp', 'mongos_topology_coordinator_test.cpp', 'query_analysis_sampler_test.cpp', 'request_types/add_shard_request_test.cpp', @@ -679,6 +676,7 @@ env.CppUnitTest( 'request_types/update_zone_key_range_request_test.cpp', 'routing_table_history_test.cpp', 'sessions_collection_sharded_test.cpp', + 'shard_key_pattern_query_util_index_bounds_test.cpp', 'shard_key_pattern_test.cpp', 'shard_version_test.cpp', 'sharding_task_executor_test.cpp', @@ -694,6 +692,7 @@ env.CppUnitTest( ], LIBDEPS=[ '$BUILD_DIR/mongo/db/auth/authmocks', + '$BUILD_DIR/mongo/db/mongohasher', '$BUILD_DIR/mongo/db/ops/write_ops_parsers_test_helpers', '$BUILD_DIR/mongo/db/pipeline/process_interface/mongos_process_interface_factory', '$BUILD_DIR/mongo/db/query/query_test_service_context', diff --git a/src/mongo/s/balancer_configuration_test.cpp b/src/mongo/s/balancer_configuration_test.cpp index c18b58de3b6..ed4020762b8 100644 --- a/src/mongo/s/balancer_configuration_test.cpp +++ b/src/mongo/s/balancer_configuration_test.cpp @@ -27,8 +27,6 @@ * it in the license file. */ -#include "mongo/platform/basic.h" - #include <boost/date_time/gregorian/gregorian_types.hpp> #include <boost/date_time/posix_time/posix_time.hpp> #include <boost/date_time/posix_time/posix_time_types.hpp> @@ -44,7 +42,6 @@ #include "mongo/rpc/metadata/repl_set_metadata.h" #include "mongo/rpc/metadata/tracking_metadata.h" #include "mongo/s/balancer_configuration.h" -#include "mongo/s/catalog/sharding_catalog_client.h" #include "mongo/s/sharding_router_test_fixture.h" #include "mongo/unittest/unittest.h" #include "mongo/util/net/hostandport.h" diff --git a/src/mongo/s/catalog/SConscript b/src/mongo/s/catalog/SConscript index 608ab31cef9..7215ee0c449 100644 --- a/src/mongo/s/catalog/SConscript +++ b/src/mongo/s/catalog/SConscript @@ -22,16 +22,18 @@ env.Library( 'sharding_catalog_client_impl.cpp', ], LIBDEPS=[ - '$BUILD_DIR/mongo/db/repl/read_concern_args', - '$BUILD_DIR/mongo/db/storage/duplicate_key_error_info', '$BUILD_DIR/mongo/executor/network_interface', '$BUILD_DIR/mongo/s/client/sharding_client', - '$BUILD_DIR/mongo/s/coreshard', 'sharding_catalog_client', ], LIBDEPS_PRIVATE=[ + '$BUILD_DIR/mongo/db/pipeline/pipeline', + '$BUILD_DIR/mongo/db/query/projection_ast', + '$BUILD_DIR/mongo/db/repl/read_concern_args', '$BUILD_DIR/mongo/db/session/logical_session_id_helpers', + '$BUILD_DIR/mongo/db/storage/duplicate_key_error_info', '$BUILD_DIR/mongo/s/common_s', + '$BUILD_DIR/mongo/s/grid', '$BUILD_DIR/mongo/util/pcre_wrapper', ], ) diff --git a/src/mongo/s/chunk.cpp b/src/mongo/s/chunk.cpp index 2acbd4ab2be..14bdc408551 100644 --- a/src/mongo/s/chunk.cpp +++ b/src/mongo/s/chunk.cpp @@ -27,18 +27,13 @@ * it in the license file. */ - -#include "mongo/platform/basic.h" - #include "mongo/s/chunk.h" -#include "mongo/platform/random.h" #include "mongo/s/chunk_writes_tracker.h" #include "mongo/util/str.h" #define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kSharding - namespace mongo { ChunkInfo::ChunkInfo(const ChunkType& from) diff --git a/src/mongo/s/chunk.h b/src/mongo/s/chunk.h index b2e96e2fe6e..1aa783ca7e9 100644 --- a/src/mongo/s/chunk.h +++ b/src/mongo/s/chunk.h @@ -141,6 +141,8 @@ public: Chunk(ChunkInfo& chunkInfo, const boost::optional<Timestamp>& atClusterTime) : _chunkInfo(chunkInfo), _atClusterTime(atClusterTime) {} + Chunk(const Chunk& other) = default; + const BSONObj& getMin() const { return _chunkInfo.getMin(); } diff --git a/src/mongo/s/chunk_manager.cpp b/src/mongo/s/chunk_manager.cpp index f56950c2930..0c213a7c5d3 100644 --- a/src/mongo/s/chunk_manager.cpp +++ b/src/mongo/s/chunk_manager.cpp @@ -27,17 +27,10 @@ * it in the license file. */ - -#include "mongo/platform/basic.h" - #include "mongo/s/chunk_manager.h" #include "mongo/bson/simple_bsonobj_comparator.h" -#include "mongo/db/matcher/extensions_callback_noop.h" #include "mongo/db/query/collation/collation_index_key.h" -#include "mongo/db/query/index_bounds_builder.h" -#include "mongo/db/query/query_planner.h" -#include "mongo/db/query/query_planner_common.h" #include "mongo/db/storage/key_string.h" #include "mongo/logv2/log.h" #include "mongo/s/chunk_writes_tracker.h" @@ -46,23 +39,13 @@ #define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kSharding - namespace mongo { namespace { -bool allElementsAreOfType(BSONType type, const BSONObj& obj) { - for (auto&& elem : obj) { - if (elem.type() != type) { - return false; - } - } - return true; -} - void checkAllElementsAreOfType(BSONType type, const BSONObj& o) { uassert(ErrorCodes::ConflictingOperationInProgress, str::stream() << "Not all elements of " << o << " are of type " << typeName(type), - allElementsAreOfType(type, o)); + ChunkMap::allElementsAreOfType(type, o)); } void appendChunkTo(std::vector<std::shared_ptr<ChunkInfo>>& chunks, @@ -286,6 +269,15 @@ BSONObj ChunkMap::toBSON() const { return builder.obj(); } +bool ChunkMap::allElementsAreOfType(BSONType type, const BSONObj& obj) { + for (auto&& elem : obj) { + if (elem.type() != type) { + return false; + } + } + return true; +} + ChunkMap::ChunkVector::const_iterator ChunkMap::_findIntersectingChunk(const BSONObj& shardKey, bool isMaxInclusive) const { auto shardKeyString = ShardKeyPattern::toKeyString(shardKey); @@ -410,107 +402,6 @@ bool ChunkManager::keyBelongsToShard(const BSONObj& shardKey, const ShardId& sha return chunkInfo->getShardIdAt(_clusterTime) == shardId; } -void ChunkManager::getShardIdsForQuery(boost::intrusive_ptr<ExpressionContext> expCtx, - const BSONObj& query, - const BSONObj& collation, - std::set<ShardId>* shardIds, - std::set<ChunkRange>* chunkRanges, - bool* targetMinKeyToMaxKey) const { - if (chunkRanges) { - invariant(chunkRanges->empty()); - } - - auto findCommand = std::make_unique<FindCommandRequest>(_rt->optRt->nss()); - findCommand->setFilter(query.getOwned()); - - expCtx->uuid = getUUID(); - - if (!collation.isEmpty()) { - findCommand->setCollation(collation.getOwned()); - } else if (_rt->optRt->getDefaultCollator()) { - auto defaultCollator = _rt->optRt->getDefaultCollator(); - findCommand->setCollation(defaultCollator->getSpec().toBSON()); - expCtx->setCollator(defaultCollator->clone()); - } - - auto cq = uassertStatusOK( - CanonicalQuery::canonicalize(expCtx->opCtx, - std::move(findCommand), - false, /* isExplain */ - expCtx, - ExtensionsCallbackNoop(), - MatchExpressionParser::kAllowAllSpecialFeatures)); - - // Fast path for targeting equalities on the shard key. - auto shardKeyToFind = _rt->optRt->getShardKeyPattern().extractShardKeyFromQuery(*cq); - if (!shardKeyToFind.isEmpty()) { - try { - auto chunk = findIntersectingChunk(shardKeyToFind, collation); - shardIds->insert(chunk.getShardId()); - if (chunkRanges) { - chunkRanges->insert(chunk.getRange()); - } - if (targetMinKeyToMaxKey) { - *targetMinKeyToMaxKey = false; - } - return; - } catch (const DBException&) { - // The query uses multiple shards - } - } - - // Transforms query into bounds for each field in the shard key - // for example : - // Key { a: 1, b: 1 }, - // Query { a : { $gte : 1, $lt : 2 }, - // b : { $gte : 3, $lt : 4 } } - // => Bounds { a : [1, 2), b : [3, 4) } - IndexBounds bounds = getIndexBoundsForQuery(_rt->optRt->getShardKeyPattern().toBSON(), *cq); - - // Transforms bounds for each shard key field into full shard key ranges - // for example : - // Key { a : 1, b : 1 } - // Bounds { a : [1, 2), b : [3, 4) } - // => Ranges { a : 1, b : 3 } => { a : 2, b : 4 } - BoundList ranges = _rt->optRt->getShardKeyPattern().flattenBounds(bounds); - - for (BoundList::const_iterator it = ranges.begin(); it != ranges.end(); ++it) { - const auto& min = it->first; - const auto& max = it->second; - - getShardIdsForRange(min, max, shardIds, chunkRanges); - if (targetMinKeyToMaxKey && allElementsAreOfType(MinKey, min) && - allElementsAreOfType(MaxKey, max)) { - *targetMinKeyToMaxKey = true; - } - - // Once we know we need to visit all shards no need to keep looping. - // However, this optimization does not apply when we are reading from a snapshot - // because _shardVersions contains shards with chunks and is built based on the last - // refresh. Therefore, it is possible for _shardVersions to have fewer entries if a shard - // no longer owns chunks when it used to at _clusterTime. - if (!_clusterTime && shardIds->size() == _rt->optRt->_shardVersions.size()) { - break; - } - } - - // SERVER-4914 Some clients of getShardIdsForQuery() assume at least one shard will be returned. - // For now, we satisfy that assumption by adding a shard with no matches rather than returning - // an empty set of shards. - if (shardIds->empty()) { - _rt->optRt->forEachChunk([&](const std::shared_ptr<ChunkInfo>& chunkInfo) { - shardIds->insert(chunkInfo->getShardIdAt(_clusterTime)); - if (chunkRanges) { - chunkRanges->insert(chunkInfo->getRange()); - } - if (targetMinKeyToMaxKey) { - *targetMinKeyToMaxKey = false; - } - return false; - }); - } -} - void ChunkManager::getShardIdsForRange(const BSONObj& min, const BSONObj& max, std::set<ShardId>* shardIds, @@ -520,7 +411,8 @@ void ChunkManager::getShardIdsForRange(const BSONObj& min, // contains shards with chunks and is built based on the last refresh. Therefore, it is // possible for _shardVersions to have fewer entries if a shard no longer owns chunks when it // used to at _clusterTime. - if (!_clusterTime && allElementsAreOfType(MinKey, min) && allElementsAreOfType(MaxKey, max)) { + if (!_clusterTime && ChunkMap::allElementsAreOfType(MinKey, min) && + ChunkMap::allElementsAreOfType(MaxKey, max)) { getAllShardIds(shardIds); if (chunkRanges) { getAllChunkRanges(chunkRanges); @@ -564,19 +456,18 @@ bool ChunkManager::rangeOverlapsShard(const ChunkRange& range, const ShardId& sh boost::optional<Chunk> ChunkManager::getNextChunkOnShard(const BSONObj& shardKey, const ShardId& shardId) const { - boost::optional<Chunk> chunk; - - _rt->optRt->forEachChunk( - [&](auto& chunkInfo) { - if (chunkInfo->getShardIdAt(_clusterTime) == shardId) { - chunk.emplace(*chunkInfo, _clusterTime); + boost::optional<Chunk> optChunk; + forEachChunk( + [&](const Chunk& chunk) { + if (chunk.getShardId() == shardId) { + optChunk.emplace(chunk); return false; } return true; }, shardKey); - return chunk; + return optChunk; } ShardId ChunkManager::getMinKeyShardIdWithSimpleCollation() const { @@ -600,154 +491,6 @@ void RoutingTableHistory::getAllChunkRanges(std::set<ChunkRange>* all) const { }); } -int RoutingTableHistory::getNShardsOwningChunks() const { - return _shardVersions.size(); -} - -IndexBounds ChunkManager::getIndexBoundsForQuery(const BSONObj& key, - const CanonicalQuery& canonicalQuery) { - // $text is not allowed in planning since we don't have text index on mongos. - // TODO: Treat $text query as a no-op in planning on mongos. So with shard key {a: 1}, - // the query { a: 2, $text: { ... } } will only target to {a: 2}. - if (QueryPlannerCommon::hasNode(canonicalQuery.root(), MatchExpression::TEXT)) { - IndexBounds bounds; - IndexBoundsBuilder::allValuesBounds(key, &bounds, false); // [minKey, maxKey] - return bounds; - } - - // Similarly, ignore GEO_NEAR queries in planning, since we do not have geo indexes on mongos. - if (QueryPlannerCommon::hasNode(canonicalQuery.root(), MatchExpression::GEO_NEAR)) { - // If the GEO_NEAR predicate is a child of AND, remove the GEO_NEAR and continue building - // bounds. Currently a CanonicalQuery can have at most one GEO_NEAR expression, and only at - // the top-level, so this check is sufficient. - auto geoIdx = [](auto root) -> boost::optional<size_t> { - if (root->matchType() == MatchExpression::AND) { - for (size_t i = 0; i < root->numChildren(); ++i) { - if (MatchExpression::GEO_NEAR == root->getChild(i)->matchType()) { - return boost::make_optional(i); - } - } - } - return boost::none; - }(canonicalQuery.root()); - - if (!geoIdx) { - IndexBounds bounds; - IndexBoundsBuilder::allValuesBounds(key, &bounds, false); - return bounds; - } - - canonicalQuery.root()->getChildVector()->erase( - canonicalQuery.root()->getChildVector()->begin() + geoIdx.value()); - } - - // Consider shard key as an index - std::string accessMethod = IndexNames::findPluginName(key); - dassert(accessMethod == IndexNames::BTREE || accessMethod == IndexNames::HASHED); - const auto indexType = IndexNames::nameToType(accessMethod); - - // Use query framework to generate index bounds - QueryPlannerParams plannerParams; - // Must use "shard key" index - plannerParams.options = QueryPlannerParams::NO_TABLE_SCAN; - IndexEntry indexEntry(key, - indexType, - IndexDescriptor::kLatestIndexVersion, - // The shard key index cannot be multikey. - false, - // Empty multikey paths, since the shard key index cannot be multikey. - MultikeyPaths{}, - // Empty multikey path set, since the shard key index cannot be multikey. - {}, - false /* sparse */, - false /* unique */, - IndexEntry::Identifier{"shardkey"}, - nullptr /* filterExpr */, - BSONObj(), - nullptr, /* collator */ - nullptr /* projExec */); - plannerParams.indices.push_back(std::move(indexEntry)); - - auto statusWithMultiPlanSolns = QueryPlanner::plan(canonicalQuery, plannerParams); - if (statusWithMultiPlanSolns.getStatus().code() != ErrorCodes::NoQueryExecutionPlans) { - auto solutions = uassertStatusOK(std::move(statusWithMultiPlanSolns)); - - // Pick any solution that has non-trivial IndexBounds. bounds.size() == 0 represents a - // trivial IndexBounds where none of the fields' values are bounded. - for (auto&& soln : solutions) { - IndexBounds bounds = collapseQuerySolution(soln->root()); - if (bounds.size() > 0) { - return bounds; - } - } - } - - // We cannot plan the query without collection scan, so target to all shards. - IndexBounds bounds; - IndexBoundsBuilder::allValuesBounds(key, &bounds, false); // [minKey, maxKey] - return bounds; -} - -IndexBounds ChunkManager::collapseQuerySolution(const QuerySolutionNode* node) { - if (node->children.empty()) { - invariant(node->getType() == STAGE_IXSCAN); - - const IndexScanNode* ixNode = static_cast<const IndexScanNode*>(node); - return ixNode->bounds; - } - - if (node->children.size() == 1) { - // e.g. FETCH -> IXSCAN - return collapseQuerySolution(node->children.front().get()); - } - - // children.size() > 1, assert it's OR / SORT_MERGE. - if (node->getType() != STAGE_OR && node->getType() != STAGE_SORT_MERGE) { - // Unexpected node. We should never reach here. - LOGV2_ERROR(23833, - "could not generate index bounds on query solution tree: {node}", - "node"_attr = redact(node->toString())); - dassert(false); // We'd like to know this error in testing. - - // Bail out with all shards in production, since this isn't a fatal error. - return IndexBounds(); - } - - IndexBounds bounds; - - for (auto it = node->children.begin(); it != node->children.end(); it++) { - // The first branch under OR - if (it == node->children.begin()) { - invariant(bounds.size() == 0); - bounds = collapseQuerySolution(it->get()); - if (bounds.size() == 0) { // Got unexpected node in query solution tree - return IndexBounds(); - } - continue; - } - - IndexBounds childBounds = collapseQuerySolution(it->get()); - if (childBounds.size() == 0) { - // Got unexpected node in query solution tree - return IndexBounds(); - } - - invariant(childBounds.size() == bounds.size()); - - for (size_t i = 0; i < bounds.size(); i++) { - bounds.fields[i].intervals.insert(bounds.fields[i].intervals.end(), - childBounds.fields[i].intervals.begin(), - childBounds.fields[i].intervals.end()); - } - } - - for (size_t i = 0; i < bounds.size(); i++) { - IndexBoundsBuilder::unionize(&bounds.fields[i]); - } - - return bounds; -} - ChunkManager ChunkManager::makeAtTime(const ChunkManager& cm, Timestamp clusterTime) { return ChunkManager(cm.dbPrimary(), cm.dbVersion(), cm._rt, clusterTime); } diff --git a/src/mongo/s/chunk_manager.h b/src/mongo/s/chunk_manager.h index 6e48a036a1e..e6d1d3fe8a7 100644 --- a/src/mongo/s/chunk_manager.h +++ b/src/mongo/s/chunk_manager.h @@ -46,8 +46,6 @@ namespace mongo { -class CanonicalQuery; -struct QuerySolutionNode; class ChunkManager; struct ShardVersionTargetingInfo { @@ -116,6 +114,8 @@ public: BSONObj toBSON() const; + static bool allElementsAreOfType(BSONType type, const BSONObj& obj); + private: ChunkVector::const_iterator _findIntersectingChunk(const BSONObj& shardKey, bool isMaxInclusive = true) const; @@ -268,7 +268,9 @@ public: /** * Returns the number of shards on which the collection has any chunks */ - int getNShardsOwningChunks() const; + size_t getNShardsOwningChunks() const { + return _shardVersions.size(); + } /** * Returns true if, for this shard, the chunks are identical in both chunk managers @@ -504,6 +506,10 @@ public: return bool(_rt->optRt); } + bool isAtPointInTime() const { + return bool(_clusterTime); + } + /** * Indicates that this collection must not honour any moveChunk requests, because it is required * to provide a stable view of its constituent shards. @@ -555,14 +561,15 @@ public: } template <typename Callable> - void forEachChunk(Callable&& handler) const { + void forEachChunk(Callable&& handler, const BSONObj& shardKey = BSONObj()) const { _rt->optRt->forEachChunk( [this, handler = std::forward<Callable>(handler)](const auto& chunkInfo) mutable { if (!handler(Chunk{*chunkInfo, _clusterTime})) return false; return true; - }); + }, + shardKey); } /** @@ -614,20 +621,6 @@ public: ShardId getMinKeyShardIdWithSimpleCollation() const; /** - * Finds the shard IDs for a given filter and collation. If collation is empty, we use the - * collection default collation for targeting. - * If 'chunkRanges' is not null, populates it with ChunkRanges that would be targeted by the - * query. If 'targetMinKeyToMaxKey' is not null, sets it to true if the query targets the entire - * shard key space. - */ - void getShardIdsForQuery(boost::intrusive_ptr<ExpressionContext> expCtx, - const BSONObj& query, - const BSONObj& collation, - std::set<ShardId>* shardIds, - std::set<ChunkRange>* chunkRanges = nullptr, - bool* targetMinKeyToMaxKey = nullptr) const; - - /** * Returns all shard ids which contain chunks overlapping the range [min, max]. Please note the * inclusive bounds on both sides (SERVER-20768). * If 'chunkRanges' is not null, populates it with ChunkRanges that would be targeted by the @@ -655,29 +648,10 @@ public: /** * Returns the number of shards on which the collection has any chunks */ - int getNShardsOwningChunks() const { + size_t getNShardsOwningChunks() const { return _rt->optRt->getNShardsOwningChunks(); } - // Transforms query into bounds for each field in the shard key - // for example : - // Key { a: 1, b: 1 }, - // Query { a : { $gte : 1, $lt : 2 }, - // b : { $gte : 3, $lt : 4 } } - // => Bounds { a : [1, 2), b : [3, 4) } - static IndexBounds getIndexBoundsForQuery(const BSONObj& key, - const CanonicalQuery& canonicalQuery); - - // Collapse query solution tree. - // - // If it has OR node, the result could be a superset of the index bounds generated. - // Since to give a single IndexBounds, this gives the union of bounds on each field. - // for example: - // OR: { a: (0, 1), b: (0, 1) }, - // { a: (2, 3), b: (2, 3) } - // => { a: (0, 1), (2, 3), b: (0, 1), (2, 3) } - static IndexBounds collapseQuerySolution(const QuerySolutionNode* node); - /** * Constructs a new ChunkManager, which is a view of the underlying routing table at a different * `clusterTime`. diff --git a/src/mongo/s/chunk_manager_query_test.cpp b/src/mongo/s/chunk_manager_query_test.cpp index 59633779a9f..a474e37691d 100644 --- a/src/mongo/s/chunk_manager_query_test.cpp +++ b/src/mongo/s/chunk_manager_query_test.cpp @@ -35,6 +35,7 @@ #include "mongo/db/query/collation/collator_interface_mock.h" #include "mongo/s/catalog_cache_test_fixture.h" #include "mongo/s/chunk_manager.h" +#include "mongo/s/shard_key_pattern_query_util.h" #include "mongo/util/assert_util.h" #define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kDefault @@ -87,8 +88,13 @@ protected: }(); auto expCtx = make_intrusive<ExpressionContextForTest>(operationContext(), kNss, std::move(cif)); - chunkManager.getShardIdsForQuery( - expCtx, query, queryCollation, &shardIds, &chunkRanges, &targetMinKeyToMaxKey); + getShardIdsForQuery(expCtx, + query, + queryCollation, + chunkManager, + &shardIds, + &chunkRanges, + &targetMinKeyToMaxKey); _assertShardIdsMatch(expectedShardIds, shardIds); ASSERT_EQ(expectTargetMinKeyToMaxKey, targetMinKeyToMaxKey); } @@ -606,7 +612,13 @@ TEST_F(ChunkManagerQueryTest, SnapshotQueryWithMoreShardsThanLatestMetadata) { const auto expCtx = make_intrusive<ExpressionContextForTest>(); shardIds.clear(); - chunkManager.getShardIdsForQuery(expCtx, BSON("x" << BSON("$gt" << -20)), {}, &shardIds); + getShardIdsForQuery(expCtx, + BSON("x" << BSON("$gt" << -20)), + {}, + chunkManager, + &shardIds, + nullptr /* chunkRanges */, + nullptr /* targetMinKeyToMaxKey */); ASSERT_EQ(2, shardIds.size()); } diff --git a/src/mongo/s/client/shard_registry.cpp b/src/mongo/s/client/shard_registry.cpp index 63a24e4b49c..c9b58cf8997 100644 --- a/src/mongo/s/client/shard_registry.cpp +++ b/src/mongo/s/client/shard_registry.cpp @@ -27,9 +27,6 @@ * it in the license file. */ - -#include "mongo/platform/basic.h" - #include "mongo/s/client/shard_registry.h" #include "mongo/client/replica_set_monitor.h" @@ -51,7 +48,6 @@ #define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kSharding - namespace mongo { namespace { diff --git a/src/mongo/s/cluster_commands_helpers.cpp b/src/mongo/s/cluster_commands_helpers.cpp index 25b0a1feeab..a2d021d50f7 100644 --- a/src/mongo/s/cluster_commands_helpers.cpp +++ b/src/mongo/s/cluster_commands_helpers.cpp @@ -27,13 +27,10 @@ * it in the license file. */ - -#include "mongo/platform/basic.h" +#include "mongo/s/cluster_commands_helpers.h" #include <boost/optional.hpp> -#include "mongo/s/cluster_commands_helpers.h" - #include "mongo/bson/util/bson_extract.h" #include "mongo/db/catalog/collection_uuid_mismatch_info.h" #include "mongo/db/commands.h" @@ -45,25 +42,23 @@ #include "mongo/db/query/collation/collator_factory_interface.h" #include "mongo/db/query/cursor_response.h" #include "mongo/db/repl/read_concern_args.h" -#include "mongo/db/shard_id.h" #include "mongo/executor/task_executor_pool.h" #include "mongo/rpc/get_status_from_command_result.h" #include "mongo/rpc/write_concern_error_detail.h" -#include "mongo/s/catalog/type_collection.h" #include "mongo/s/catalog_cache.h" #include "mongo/s/client/shard_registry.h" -#include "mongo/s/database_version.h" +#include "mongo/s/collection_routing_info_targeter.h" #include "mongo/s/grid.h" #include "mongo/s/multi_statement_transaction_requests_sender.h" #include "mongo/s/query_analysis_sampler_util.h" #include "mongo/s/request_types/sharded_ddl_commands_gen.h" +#include "mongo/s/shard_key_pattern_query_util.h" #include "mongo/s/stale_exception.h" #include "mongo/s/transaction_router.h" #include "mongo/util/scopeguard.h" #define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kCommand - namespace mongo { void appendWriteConcernErrorDetailToCmdResponse(const ShardId& shardId, @@ -185,7 +180,13 @@ std::vector<AsyncRequestsSender::Request> buildVersionedRequestsForTargetedShard } auto expCtx = make_intrusive<ExpressionContext>(opCtx, std::move(collator), nss); - cm.getShardIdsForQuery(expCtx, query, collation, &shardIds); + getShardIdsForQuery(expCtx, + query, + collation, + cm, + &shardIds, + nullptr /* chunkRanges */, + nullptr /* targetMinKeyToMaxKey */); const auto targetedSampleId = eligibleForSampling ? analyze_shard_key::tryGenerateTargetedSampleId(opCtx, nss, shardIds) @@ -659,7 +660,13 @@ std::set<ShardId> getTargetedShardsForQuery(boost::intrusive_ptr<ExpressionConte // The collection is sharded. Use the routing table to decide which shards to target based // on the query and collation. std::set<ShardId> shardIds; - cm.getShardIdsForQuery(expCtx, query, collation, &shardIds); + getShardIdsForQuery(expCtx, + query, + collation, + cm, + &shardIds, + nullptr /* chunkRanges */, + nullptr /* targetMinKeyToMaxKey */); return shardIds; } diff --git a/src/mongo/s/cluster_identity_loader.cpp b/src/mongo/s/cluster_identity_loader.cpp index 67537e4cb2c..25c6a665dce 100644 --- a/src/mongo/s/cluster_identity_loader.cpp +++ b/src/mongo/s/cluster_identity_loader.cpp @@ -27,20 +27,15 @@ * it in the license file. */ - -#include "mongo/platform/basic.h" - #include "mongo/s/cluster_identity_loader.h" #include "mongo/base/status_with.h" #include "mongo/db/operation_context.h" #include "mongo/db/service_context.h" -#include "mongo/s/catalog/sharding_catalog_client.h" #include "mongo/s/catalog/type_config_version.h" #define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kSharding - namespace mongo { namespace { diff --git a/src/mongo/s/collection_routing_info_targeter.cpp b/src/mongo/s/collection_routing_info_targeter.cpp index 467f5afac6c..444e7666c95 100644 --- a/src/mongo/s/collection_routing_info_targeter.cpp +++ b/src/mongo/s/collection_routing_info_targeter.cpp @@ -34,16 +34,16 @@ #include "mongo/db/internal_transactions_feature_flag_gen.h" #include "mongo/db/matcher/extensions_callback_noop.h" #include "mongo/db/pipeline/pipeline.h" -#include "mongo/db/pipeline/pipeline_d.h" #include "mongo/db/pipeline/process_interface/mongos_process_interface.h" #include "mongo/db/query/canonical_query.h" #include "mongo/db/query/collation/collation_index_key.h" #include "mongo/db/query/collation/collator_factory_interface.h" +#include "mongo/db/query/query_planner.h" +#include "mongo/db/query/query_planner_common.h" #include "mongo/db/stats/counters.h" #include "mongo/db/timeseries/timeseries_constants.h" #include "mongo/db/timeseries/timeseries_options.h" #include "mongo/db/timeseries/timeseries_update_delete_util.h" -#include "mongo/executor/task_executor_pool.h" #include "mongo/logv2/log.h" #include "mongo/s/client/shard_registry.h" #include "mongo/s/cluster_commands_helpers.h" @@ -56,7 +56,6 @@ #define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kSharding - namespace mongo { namespace { @@ -166,7 +165,7 @@ BSONObj getUpdateExprForTargeting(const boost::intrusive_ptr<ExpressionContext> * { foo : <anything> } => false */ bool isExactIdQuery(OperationContext* opCtx, const CanonicalQuery& query, const ChunkManager& cm) { - auto shardKey = kVirtualIdShardKey.extractShardKeyFromQuery(query); + auto shardKey = extractShardKeyFromQuery(kVirtualIdShardKey, query); BSONElement idElt = shardKey["_id"]; if (!idElt) { @@ -620,7 +619,7 @@ StatusWith<std::vector<ShardEndpoint>> CollectionRoutingInfoTargeter::_targetQue std::set<ShardId> shardIds; try { - _cri.cm.getShardIdsForQuery(expCtx, query, collation, &shardIds, chunkRanges); + getShardIdsForQuery(expCtx, query, collation, _cri.cm, &shardIds, chunkRanges); } catch (const DBException& ex) { return ex.toStatus(); } diff --git a/src/mongo/s/collection_routing_info_targeter.h b/src/mongo/s/collection_routing_info_targeter.h index f0fe7f610e7..992a813dd93 100644 --- a/src/mongo/s/collection_routing_info_targeter.h +++ b/src/mongo/s/collection_routing_info_targeter.h @@ -36,6 +36,7 @@ #include "mongo/bson/bsonobj_comparator_interface.h" #include "mongo/bson/simple_bsonobj_comparator.h" #include "mongo/db/namespace_string.h" +#include "mongo/db/pipeline/expression_context.h" #include "mongo/s/catalog_cache.h" #include "mongo/s/chunk_manager.h" #include "mongo/s/ns_targeter.h" diff --git a/src/mongo/s/collection_routing_info_targeter_test.cpp b/src/mongo/s/collection_routing_info_targeter_test.cpp index 7cc78b5376d..e81df416820 100644 --- a/src/mongo/s/collection_routing_info_targeter_test.cpp +++ b/src/mongo/s/collection_routing_info_targeter_test.cpp @@ -27,8 +27,6 @@ * it in the license file. */ -#include "mongo/platform/basic.h" - #include "mongo/db/hasher.h" #include "mongo/db/pipeline/expression_context_for_test.h" #include "mongo/db/service_context_test_fixture.h" diff --git a/src/mongo/s/commands/SConscript b/src/mongo/s/commands/SConscript index 9cdf12c20ac..a74fe2a814e 100644 --- a/src/mongo/s/commands/SConscript +++ b/src/mongo/s/commands/SConscript @@ -17,6 +17,8 @@ env.Library( LIBDEPS_PRIVATE=[ '$BUILD_DIR/mongo/db/catalog/index_key_validate', '$BUILD_DIR/mongo/db/commands', + '$BUILD_DIR/mongo/db/fts/base_fts', + '$BUILD_DIR/mongo/db/index/expression_params', '$BUILD_DIR/mongo/db/shard_role_api', '$BUILD_DIR/mongo/s/grid', '$BUILD_DIR/mongo/s/startup_initialization', diff --git a/src/mongo/s/commands/cluster_merge_chunks_cmd.cpp b/src/mongo/s/commands/cluster_merge_chunks_cmd.cpp index df268cd3901..420f4bc8e41 100644 --- a/src/mongo/s/commands/cluster_merge_chunks_cmd.cpp +++ b/src/mongo/s/commands/cluster_merge_chunks_cmd.cpp @@ -27,25 +27,17 @@ * it in the license file. */ -#include "mongo/platform/basic.h" - #include "mongo/client/connpool.h" #include "mongo/db/auth/action_type.h" #include "mongo/db/auth/authorization_session.h" #include "mongo/db/auth/privilege.h" #include "mongo/db/commands.h" #include "mongo/db/field_parser.h" -#include "mongo/db/namespace_string.h" -#include "mongo/s/catalog/sharding_catalog_client.h" #include "mongo/s/catalog_cache.h" #include "mongo/s/client/shard_registry.h" #include "mongo/s/grid.h" namespace mongo { - -using std::string; -using std::vector; - namespace { /** @@ -89,22 +81,22 @@ public: } // Required - static BSONField<string> nsField; - static BSONField<vector<BSONObj>> boundsField; + static BSONField<std::string> nsField; + static BSONField<std::vector<BSONObj>> boundsField; // Used to send sharding state - static BSONField<string> shardNameField; - static BSONField<string> configField; + static BSONField<std::string> shardNameField; + static BSONField<std::string> configField; bool errmsgRun(OperationContext* opCtx, - const string& dbname, + const std::string& dbname, const BSONObj& cmdObj, - string& errmsg, + std::string& errmsg, BSONObjBuilder& result) override { const NamespaceString nss(parseNs({boost::none, dbname}, cmdObj)); - vector<BSONObj> bounds; + std::vector<BSONObj> bounds; if (!FieldParser::extract(cmdObj, boundsField, &bounds, &errmsg)) { return false; } @@ -187,11 +179,11 @@ public: } clusterMergeChunksCommand; -BSONField<string> ClusterMergeChunksCommand::nsField("mergeChunks"); -BSONField<vector<BSONObj>> ClusterMergeChunksCommand::boundsField("bounds"); +BSONField<std::string> ClusterMergeChunksCommand::nsField("mergeChunks"); +BSONField<std::vector<BSONObj>> ClusterMergeChunksCommand::boundsField("bounds"); -BSONField<string> ClusterMergeChunksCommand::configField("config"); -BSONField<string> ClusterMergeChunksCommand::shardNameField("shardName"); +BSONField<std::string> ClusterMergeChunksCommand::configField("config"); +BSONField<std::string> ClusterMergeChunksCommand::shardNameField("shardName"); } // namespace } // namespace mongo diff --git a/src/mongo/s/commands/cluster_netstat_cmd.cpp b/src/mongo/s/commands/cluster_netstat_cmd.cpp index 1003b9b0e44..ca740074eea 100644 --- a/src/mongo/s/commands/cluster_netstat_cmd.cpp +++ b/src/mongo/s/commands/cluster_netstat_cmd.cpp @@ -27,11 +27,8 @@ * it in the license file. */ -#include "mongo/platform/basic.h" - #include "mongo/db/auth/authorization_session.h" #include "mongo/db/commands.h" -#include "mongo/s/catalog/sharding_catalog_client.h" #include "mongo/s/client/shard_registry.h" #include "mongo/s/grid.h" diff --git a/src/mongo/s/commands/cluster_query_without_shard_key_cmd.cpp b/src/mongo/s/commands/cluster_query_without_shard_key_cmd.cpp index 4d1f8ed17ec..fa68c48fd0b 100644 --- a/src/mongo/s/commands/cluster_query_without_shard_key_cmd.cpp +++ b/src/mongo/s/commands/cluster_query_without_shard_key_cmd.cpp @@ -37,6 +37,7 @@ #include "mongo/s/is_mongos.h" #include "mongo/s/multi_statement_transaction_requests_sender.h" #include "mongo/s/request_types/cluster_commands_without_shard_key_gen.h" +#include "mongo/s/shard_key_pattern_query_util.h" #include "mongo/s/write_ops/batch_write_op.h" #define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kCommand @@ -78,7 +79,13 @@ std::set<ShardId> getShardsToTarget(OperationContext* opCtx, CollatorFactoryInterface::get(opCtx->getServiceContext())->makeFromBSON(collation)); } auto expCtx = make_intrusive<ExpressionContext>(opCtx, std::move(collator), nss); - cm.getShardIdsForQuery(expCtx, query, collation, &allShardsContainingChunksForNs); + getShardIdsForQuery(expCtx, + query, + collation, + cm, + &allShardsContainingChunksForNs, + nullptr /* chunkRanges */, + nullptr /* targetMinKeyToMaxKey */); // We must either get a subset of shards to target in the case of a partial shard key or we must // target all shards. diff --git a/src/mongo/s/commands/cluster_remove_shard_from_zone_cmd.cpp b/src/mongo/s/commands/cluster_remove_shard_from_zone_cmd.cpp index 9fa3038eaf7..38b9fd4f22e 100644 --- a/src/mongo/s/commands/cluster_remove_shard_from_zone_cmd.cpp +++ b/src/mongo/s/commands/cluster_remove_shard_from_zone_cmd.cpp @@ -27,15 +27,11 @@ * it in the license file. */ - -#include "mongo/platform/basic.h" - #include <vector> #include "mongo/db/auth/authorization_session.h" #include "mongo/db/commands.h" #include "mongo/db/write_concern_options.h" -#include "mongo/s/catalog/sharding_catalog_client.h" #include "mongo/s/catalog/type_shard.h" #include "mongo/s/catalog/type_tags.h" #include "mongo/s/client/shard_registry.h" @@ -44,11 +40,7 @@ #define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kCommand - namespace mongo { - -using std::string; - namespace { const ReadPreferenceSetting kPrimaryOnlyReadPreference{ReadPreference::PrimaryOnly}; diff --git a/src/mongo/s/commands/cluster_update_zone_key_range_cmd.cpp b/src/mongo/s/commands/cluster_update_zone_key_range_cmd.cpp index eaf3fc21739..f3570edf82d 100644 --- a/src/mongo/s/commands/cluster_update_zone_key_range_cmd.cpp +++ b/src/mongo/s/commands/cluster_update_zone_key_range_cmd.cpp @@ -27,16 +27,12 @@ * it in the license file. */ - -#include "mongo/platform/basic.h" - #include <vector> #include "mongo/bson/util/bson_extract.h" #include "mongo/db/auth/authorization_session.h" #include "mongo/db/commands.h" #include "mongo/db/write_concern_options.h" -#include "mongo/s/catalog/sharding_catalog_client.h" #include "mongo/s/catalog/type_shard.h" #include "mongo/s/catalog/type_tags.h" #include "mongo/s/client/shard_registry.h" @@ -45,11 +41,7 @@ #define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kCommand - namespace mongo { - -using std::string; - namespace { const ReadPreferenceSetting kPrimaryOnlyReadPreference{ReadPreference::PrimaryOnly}; diff --git a/src/mongo/s/config_server_catalog_cache_loader.cpp b/src/mongo/s/config_server_catalog_cache_loader.cpp index 2a695671ac9..8b3736e79c5 100644 --- a/src/mongo/s/config_server_catalog_cache_loader.cpp +++ b/src/mongo/s/config_server_catalog_cache_loader.cpp @@ -27,31 +27,23 @@ * it in the license file. */ - -#include "mongo/platform/basic.h" - #include "mongo/s/config_server_catalog_cache_loader.h" -#include <memory> - #include "mongo/db/catalog_shard_feature_flag_gen.h" #include "mongo/db/client.h" #include "mongo/db/operation_context.h" #include "mongo/db/repl/replication_coordinator.h" #include "mongo/db/vector_clock.h" #include "mongo/logv2/log.h" -#include "mongo/s/catalog/sharding_catalog_client.h" #include "mongo/s/grid.h" #define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kSharding - namespace mongo { +namespace { using CollectionAndChangedChunks = CatalogCacheLoader::CollectionAndChangedChunks; -namespace { - /** * Blocking method, which returns the chunks which changed since the specified version. */ diff --git a/src/mongo/s/query/SConscript b/src/mongo/s/query/SConscript index 905c1cbe0f0..9f50a327ecf 100644 --- a/src/mongo/s/query/SConscript +++ b/src/mongo/s/query/SConscript @@ -127,7 +127,7 @@ env.Library( ], LIBDEPS=[ "$BUILD_DIR/mongo/db/cursor_server_params", - "$BUILD_DIR/mongo/s/coreshard", + "$BUILD_DIR/mongo/s/grid", "$BUILD_DIR/mongo/util/background_job", ], ) diff --git a/src/mongo/s/shard_key_pattern.cpp b/src/mongo/s/shard_key_pattern.cpp index d04632a8aa8..db4dce94066 100644 --- a/src/mongo/s/shard_key_pattern.cpp +++ b/src/mongo/s/shard_key_pattern.cpp @@ -33,21 +33,13 @@ #include "mongo/db/field_ref_set.h" #include "mongo/db/hasher.h" #include "mongo/db/index_names.h" -#include "mongo/db/matcher/extensions_callback_noop.h" #include "mongo/db/matcher/path_internal.h" -#include "mongo/db/update/path_support.h" +#include "mongo/db/storage/key_string.h" #include "mongo/util/str.h" -#include "mongo/util/transitional_tools_do_not_use/vector_spooling.h" namespace mongo { - -using pathsupport::EqualityMatches; - namespace { -// Maximum number of intervals produced by $in queries -constexpr size_t kMaxFlattenedInCombinations = 4000000; - constexpr auto kIdField = "_id"_sd; const BSONObj kNullObj = BSON("" << BSONNULL); @@ -110,19 +102,6 @@ bool isValidShardKeyElement(const BSONElement& element) { return !element.eoo() && element.type() != Array; } -bool isValidShardKeyElementForStorage(const BSONElement& element) { - if (!isValidShardKeyElement(element)) - return false; - - if (element.type() == RegEx) - return false; - - if (element.type() == Object && !element.embeddedObject().storageValidEmbedded().isOK()) - return false; - - return true; -} - BSONElement extractKeyElementFromDoc(const BSONObj& obj, StringData pathStr) { // Any arrays found get immediately returned. We are equipped up the call stack to specifically // deal with array values. @@ -130,21 +109,6 @@ BSONElement extractKeyElementFromDoc(const BSONObj& obj, StringData pathStr) { return getFieldDottedOrArray(obj, FieldRef(pathStr), &idxPath); } -BSONElement findEqualityElement(const EqualityMatches& equalities, const FieldRef& path) { - int parentPathPart; - const BSONElement parentEl = - pathsupport::findParentEqualityElement(equalities, path, &parentPathPart); - - if (parentPathPart == static_cast<int>(path.numParts())) - return parentEl; - - if (parentEl.type() != Object) - return BSONElement(); - - StringData suffixStr = path.dottedSubstring(parentPathPart, path.numParts()); - return extractKeyElementFromDoc(parentEl.Obj(), suffixStr); -} - /** * Extracts the BSONElement matching 'fieldName' from the 'indexKeyDataVector'. Returns a pair with * first field representing the matching BSONElement and the second field representing whether the @@ -183,6 +147,7 @@ BSONElement extractFieldFromDocumentKey(const BSONObj& documentKey, StringData f } return output; } + } // namespace Status ShardKeyPattern::checkShardKeyIsValidForMetadataStorage(const BSONObj& shardKey) { @@ -238,18 +203,6 @@ bool ShardKeyPattern::hasHashedPrefix() const { return isHashedPatternEl(_keyPattern.toBSON().firstElement()); } -BSONElement ShardKeyPattern::getHashedField() const { - return _hashedField; -} - -const KeyPattern& ShardKeyPattern::getKeyPattern() const { - return _keyPattern; -} - -const std::vector<std::unique_ptr<FieldRef>>& ShardKeyPattern::getKeyPatternFields() const { - return _keyPatternPaths; -} - const BSONObj& ShardKeyPattern::toBSON() const { return _keyPattern.toBSON(); } @@ -440,48 +393,6 @@ BSONObj ShardKeyPattern::emplaceMissingShardKeyValuesForDocument(const BSONObj d return fullDocBuilder.obj(); } -BSONObj ShardKeyPattern::extractShardKeyFromQuery(const CanonicalQuery& query) const { - // Extract equalities from query. - EqualityMatches equalities; - // TODO: Build the path set initially? - FieldRefSet keyPatternPathSet(transitional_tools_do_not_use::unspool_vector(_keyPatternPaths)); - // We only care about extracting the full key pattern paths - if they don't exist (or are - // conflicting), we don't contain the shard key. - Status eqStatus = - pathsupport::extractFullEqualityMatches(*query.root(), keyPatternPathSet, &equalities); - // NOTE: Failure to extract equality matches just means we return no shard key - it's not - // an error we propagate - if (!eqStatus.isOK()) - return BSONObj(); - - // Extract key from equalities - // NOTE: The method below is equivalent to constructing a BSONObj and running - // extractShardKeyFromDoc, but doesn't require creating the doc. - - BSONObjBuilder keyBuilder; - // Iterate the parsed paths to avoid re-parsing - for (auto it = _keyPatternPaths.begin(); it != _keyPatternPaths.end(); ++it) { - const FieldRef& patternPath = **it; - BSONElement equalEl = findEqualityElement(equalities, patternPath); - - if (!isValidShardKeyElementForStorage(equalEl)) - return BSONObj(); - - if (_hashedField && _hashedField.fieldNameStringData() == patternPath.dottedField()) { - keyBuilder.append( - patternPath.dottedField(), - BSONElementHasher::hash64(equalEl, BSONElementHasher::DEFAULT_HASH_SEED)); - } else { - // NOTE: The equal element may *not* have the same field name as the path - nested $and, - // $eq, for example - keyBuilder.appendAs(equalEl, patternPath.dottedField()); - } - } - - dassert(isShardKey(keyBuilder.asTempObj())); - return keyBuilder.obj(); -} - bool ShardKeyPattern::isIndexUniquenessCompatible(const BSONObj& indexPattern) const { if (!indexPattern.isEmpty() && indexPattern.firstElementFieldName() == kIdField) { return true; @@ -490,97 +401,6 @@ bool ShardKeyPattern::isIndexUniquenessCompatible(const BSONObj& indexPattern) c return _keyPattern.toBSON().isFieldNamePrefixOf(indexPattern); } -BoundList ShardKeyPattern::flattenBounds(const IndexBounds& indexBounds) const { - invariant(indexBounds.fields.size() == (size_t)_keyPattern.toBSON().nFields()); - - // If any field is unsatisfied, return empty bound list. - for (const auto& field : indexBounds.fields) { - if (field.intervals.empty()) { - return BoundList(); - } - } - - // To construct our bounds we will generate intervals based on bounds for the first field, then - // compound intervals based on constraints for the first 2 fields, then compound intervals for - // the first 3 fields, etc. - // - // As we loop through the fields, we start generating new intervals that will later get extended - // in another iteration of the loop. We define these partially constructed intervals using pairs - // of BSONObjBuilders (shared_ptrs, since after one iteration of the loop they still must exist - // outside their scope). - using BoundBuilders = std::vector<std::pair<BSONObjBuilder, BSONObjBuilder>>; - - BoundBuilders builders; - builders.emplace_back(); - - BSONObjIterator keyIter(_keyPattern.toBSON()); - // Until equalityOnly is false, we are just dealing with equality (no range or $in queries). - bool equalityOnly = true; - - for (size_t i = 0; i < indexBounds.fields.size(); ++i) { - BSONElement e = keyIter.next(); - - StringData fieldName = e.fieldNameStringData(); - - // Get the relevant intervals for this field, but we may have to transform the list of - // what's relevant according to the expression for this field - const OrderedIntervalList& oil = indexBounds.fields[i]; - const auto& intervals = oil.intervals; - - if (equalityOnly) { - if (intervals.size() == 1 && intervals.front().isPoint()) { - // This field is only a single point-interval - for (auto& builder : builders) { - builder.first.appendAs(intervals.front().start, fieldName); - builder.second.appendAs(intervals.front().end, fieldName); - } - } else { - // This clause is the first to generate more than a single point. We only execute - // this clause once. After that, we simplify the bound extensions to prevent - // combinatorial explosion. - equalityOnly = false; - - BoundBuilders newBuilders; - - for (auto& builder : builders) { - BSONObj first = builder.first.obj(); - BSONObj second = builder.second.obj(); - - for (const auto& interval : intervals) { - uassert(17439, - "combinatorial limit of $in partitioning of results exceeded", - newBuilders.size() < kMaxFlattenedInCombinations); - - newBuilders.emplace_back(); - - newBuilders.back().first.appendElements(first); - newBuilders.back().first.appendAs(interval.start, fieldName); - - newBuilders.back().second.appendElements(second); - newBuilders.back().second.appendAs(interval.end, fieldName); - } - } - - builders = std::move(newBuilders); - } - } else { - // If we've already generated a range or multiple point-intervals just extend what we've - // generated with min/max bounds for this field - for (auto& builder : builders) { - builder.first.appendAs(intervals.front().start, fieldName); - builder.second.appendAs(intervals.back().end, fieldName); - } - } - } - - BoundList ret; - for (auto& builder : builders) { - ret.emplace_back(builder.first.obj(), builder.second.obj()); - } - - return ret; -} - size_t ShardKeyPattern::getApproximateSize() const { auto computeVectorSize = [](const std::vector<std::unique_ptr<FieldRef>>& v) { size_t size = 0; @@ -596,4 +416,17 @@ size_t ShardKeyPattern::getApproximateSize() const { return size; } +bool ShardKeyPattern::isValidShardKeyElementForStorage(const BSONElement& element) { + if (!isValidShardKeyElement(element)) + return false; + + if (element.type() == RegEx) + return false; + + if (element.type() == Object && !element.embeddedObject().storageValidEmbedded().isOK()) + return false; + + return true; +} + } // namespace mongo diff --git a/src/mongo/s/shard_key_pattern.h b/src/mongo/s/shard_key_pattern.h index cd19df027e8..93dad3c5f26 100644 --- a/src/mongo/s/shard_key_pattern.h +++ b/src/mongo/s/shard_key_pattern.h @@ -32,12 +32,9 @@ #include <vector> #include "mongo/base/status_with.h" -#include "mongo/db/exec/filter.h" #include "mongo/db/field_ref.h" #include "mongo/db/jsobj.h" #include "mongo/db/keypattern.h" -#include "mongo/db/query/canonical_query.h" -#include "mongo/db/query/index_bounds.h" namespace mongo { @@ -109,11 +106,17 @@ public: bool hasHashedPrefix() const; - BSONElement getHashedField() const; + BSONElement getHashedField() const { + return _hashedField; + } - const KeyPattern& getKeyPattern() const; + const KeyPattern& getKeyPattern() const { + return _keyPattern; + } - const std::vector<std::unique_ptr<FieldRef>>& getKeyPatternFields() const; + const std::vector<std::unique_ptr<FieldRef>>& getKeyPatternFields() const { + return _keyPatternPaths; + } const BSONObj& toBSON() const; @@ -213,8 +216,6 @@ public: */ BSONObj emplaceMissingShardKeyValuesForDocument(BSONObj doc) const; - BSONObj extractShardKeyFromQuery(const CanonicalQuery& query) const; - /** * Returns true if the shard key pattern can ensure that the index uniqueness is respected * across all shards. @@ -249,27 +250,6 @@ public: bool isIndexUniquenessCompatible(const BSONObj& indexPattern) const; /** - * Return an ordered list of bounds generated using this KeyPattern and the - * bounds from the IndexBounds. This function is used in sharding to - * determine where to route queries according to the shard key pattern. - * - * Examples: - * - * Key { a: 1 }, Bounds a: [0] => { a: 0 } -> { a: 0 } - * Key { a: 1 }, Bounds a: [2, 3) => { a: 2 } -> { a: 3 } // bound inclusion ignored. - * - * The bounds returned by this function may be a superset of those defined - * by the constraints. For instance, if this KeyPattern is {a : 1, b: 1} - * Bounds: { a : {$in : [1,2]} , b : {$in : [3,4,5]} } - * => {a : 1 , b : 3} -> {a : 1 , b : 5}, {a : 2 , b : 3} -> {a : 2 , b : 5} - * - * If the IndexBounds are not defined for all the fields in this keypattern, which - * means some fields are unsatisfied, an empty BoundList could return. - * - */ - BoundList flattenBounds(const IndexBounds& indexBounds) const; - - /** * Returns true if the key pattern has an "_id" field of any flavor. */ bool hasId() const { @@ -278,6 +258,8 @@ public: size_t getApproximateSize() const; + static bool isValidShardKeyElementForStorage(const BSONElement& element); + private: KeyPattern _keyPattern; diff --git a/src/mongo/s/shard_key_pattern_query_util.cpp b/src/mongo/s/shard_key_pattern_query_util.cpp index 8b8d40c06ad..33063b6643f 100644 --- a/src/mongo/s/shard_key_pattern_query_util.cpp +++ b/src/mongo/s/shard_key_pattern_query_util.cpp @@ -29,7 +29,108 @@ #include "mongo/s/shard_key_pattern_query_util.h" +#include "mongo/db/matcher/extensions_callback_noop.h" +#include "mongo/db/matcher/path_internal.h" +#include "mongo/db/query/index_bounds_builder.h" +#include "mongo/db/query/query_planner.h" +#include "mongo/db/query/query_planner_common.h" +#include "mongo/db/update/path_support.h" +#include "mongo/logv2/log.h" +#include "mongo/util/transitional_tools_do_not_use/vector_spooling.h" + +#define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kSharding + namespace mongo { +namespace { + +using pathsupport::EqualityMatches; + +// Maximum number of intervals produced by $in queries +constexpr size_t kMaxFlattenedInCombinations = 4000000; + +IndexBounds collapseQuerySolution(const QuerySolutionNode* node) { + if (node->children.empty()) { + invariant(node->getType() == STAGE_IXSCAN); + + const IndexScanNode* ixNode = static_cast<const IndexScanNode*>(node); + return ixNode->bounds; + } + + if (node->children.size() == 1) { + // e.g. FETCH -> IXSCAN + return collapseQuerySolution(node->children.front().get()); + } + + // children.size() > 1, assert it's OR / SORT_MERGE. + if (node->getType() != STAGE_OR && node->getType() != STAGE_SORT_MERGE) { + // Unexpected node. We should never reach here. + LOGV2_ERROR(23833, + "could not generate index bounds on query solution tree: {node}", + "node"_attr = redact(node->toString())); + dassert(false); // We'd like to know this error in testing. + + // Bail out with all shards in production, since this isn't a fatal error. + return IndexBounds(); + } + + IndexBounds bounds; + + for (auto it = node->children.begin(); it != node->children.end(); it++) { + // The first branch under OR + if (it == node->children.begin()) { + invariant(bounds.size() == 0); + bounds = collapseQuerySolution(it->get()); + if (bounds.size() == 0) { // Got unexpected node in query solution tree + return IndexBounds(); + } + continue; + } + + IndexBounds childBounds = collapseQuerySolution(it->get()); + if (childBounds.size() == 0) { + // Got unexpected node in query solution tree + return IndexBounds(); + } + + invariant(childBounds.size() == bounds.size()); + + for (size_t i = 0; i < bounds.size(); i++) { + bounds.fields[i].intervals.insert(bounds.fields[i].intervals.end(), + childBounds.fields[i].intervals.begin(), + childBounds.fields[i].intervals.end()); + } + } + + for (size_t i = 0; i < bounds.size(); i++) { + IndexBoundsBuilder::unionize(&bounds.fields[i]); + } + + return bounds; +} + +BSONElement extractKeyElementFromDoc(const BSONObj& obj, StringData pathStr) { + // Any arrays found get immediately returned. We are equipped up the call stack to specifically + // deal with array values. + size_t idxPath; + return getFieldDottedOrArray(obj, FieldRef(pathStr), &idxPath); +} + +BSONElement findEqualityElement(const EqualityMatches& equalities, const FieldRef& path) { + int parentPathPart; + const BSONElement parentEl = + pathsupport::findParentEqualityElement(equalities, path, &parentPathPart); + + if (parentPathPart == static_cast<int>(path.numParts())) + return parentEl; + + if (parentEl.type() != Object) + return BSONElement(); + + StringData suffixStr = path.dottedSubstring(parentPathPart, path.numParts()); + return extractKeyElementFromDoc(parentEl.Obj(), suffixStr); +} + +} // namespace StatusWith<BSONObj> extractShardKeyFromBasicQuery(OperationContext* opCtx, const NamespaceString& nss, @@ -50,7 +151,7 @@ StatusWith<BSONObj> extractShardKeyFromBasicQuery(OperationContext* opCtx, return statusWithCQ.getStatus(); } - return shardKeyPattern.extractShardKeyFromQuery(*statusWithCQ.getValue()); + return extractShardKeyFromQuery(shardKeyPattern, *statusWithCQ.getValue()); } StatusWith<BSONObj> extractShardKeyFromBasicQueryWithContext( @@ -74,7 +175,331 @@ StatusWith<BSONObj> extractShardKeyFromBasicQueryWithContext( return statusWithCQ.getStatus(); } - return shardKeyPattern.extractShardKeyFromQuery(*statusWithCQ.getValue()); + return extractShardKeyFromQuery(shardKeyPattern, *statusWithCQ.getValue()); +} + +BSONObj extractShardKeyFromQuery(const ShardKeyPattern& shardKeyPattern, + const CanonicalQuery& query) { + // Extract equalities from query. + EqualityMatches equalities; + // TODO: Build the path set initially? + FieldRefSet keyPatternPathSet( + transitional_tools_do_not_use::unspool_vector(shardKeyPattern.getKeyPatternFields())); + // We only care about extracting the full key pattern paths - if they don't exist (or are + // conflicting), we don't contain the shard key. + Status eqStatus = + pathsupport::extractFullEqualityMatches(*query.root(), keyPatternPathSet, &equalities); + // NOTE: Failure to extract equality matches just means we return no shard key - it's not + // an error we propagate + if (!eqStatus.isOK()) + return BSONObj(); + + // Extract key from equalities + // NOTE: The method below is equivalent to constructing a BSONObj and running + // extractShardKeyFromDoc, but doesn't require creating the doc. + + BSONObjBuilder keyBuilder; + // Iterate the parsed paths to avoid re-parsing + for (auto it = shardKeyPattern.getKeyPatternFields().begin(); + it != shardKeyPattern.getKeyPatternFields().end(); + ++it) { + const FieldRef& patternPath = **it; + BSONElement equalEl = findEqualityElement(equalities, patternPath); + + if (!ShardKeyPattern::isValidShardKeyElementForStorage(equalEl)) + return BSONObj(); + + if (shardKeyPattern.getHashedField() && + shardKeyPattern.getHashedField().fieldNameStringData() == patternPath.dottedField()) { + keyBuilder.append( + patternPath.dottedField(), + BSONElementHasher::hash64(equalEl, BSONElementHasher::DEFAULT_HASH_SEED)); + } else { + // NOTE: The equal element may *not* have the same field name as the path - nested $and, + // $eq, for example + keyBuilder.appendAs(equalEl, patternPath.dottedField()); + } + } + + dassert(shardKeyPattern.isShardKey(keyBuilder.asTempObj())); + return keyBuilder.obj(); +} + +BoundList flattenBounds(const ShardKeyPattern& shardKeyPattern, const IndexBounds& indexBounds) { + invariant(indexBounds.fields.size() == (size_t)shardKeyPattern.toBSON().nFields()); + + // If any field is unsatisfied, return empty bound list. + for (const auto& field : indexBounds.fields) { + if (field.intervals.empty()) { + return BoundList(); + } + } + + // To construct our bounds we will generate intervals based on bounds for the first field, then + // compound intervals based on constraints for the first 2 fields, then compound intervals for + // the first 3 fields, etc. + // + // As we loop through the fields, we start generating new intervals that will later get extended + // in another iteration of the loop. We define these partially constructed intervals using pairs + // of BSONObjBuilders (shared_ptrs, since after one iteration of the loop they still must exist + // outside their scope). + using BoundBuilders = std::vector<std::pair<BSONObjBuilder, BSONObjBuilder>>; + + BoundBuilders builders; + builders.emplace_back(); + + BSONObjIterator keyIter(shardKeyPattern.toBSON()); + // Until equalityOnly is false, we are just dealing with equality (no range or $in queries). + bool equalityOnly = true; + + for (size_t i = 0; i < indexBounds.fields.size(); ++i) { + BSONElement e = keyIter.next(); + + StringData fieldName = e.fieldNameStringData(); + + // Get the relevant intervals for this field, but we may have to transform the list of + // what's relevant according to the expression for this field + const OrderedIntervalList& oil = indexBounds.fields[i]; + const auto& intervals = oil.intervals; + + if (equalityOnly) { + if (intervals.size() == 1 && intervals.front().isPoint()) { + // This field is only a single point-interval + for (auto& builder : builders) { + builder.first.appendAs(intervals.front().start, fieldName); + builder.second.appendAs(intervals.front().end, fieldName); + } + } else { + // This clause is the first to generate more than a single point. We only execute + // this clause once. After that, we simplify the bound extensions to prevent + // combinatorial explosion. + equalityOnly = false; + + BoundBuilders newBuilders; + + for (auto& builder : builders) { + BSONObj first = builder.first.obj(); + BSONObj second = builder.second.obj(); + + for (const auto& interval : intervals) { + uassert(17439, + "combinatorial limit of $in partitioning of results exceeded", + newBuilders.size() < kMaxFlattenedInCombinations); + + newBuilders.emplace_back(); + + newBuilders.back().first.appendElements(first); + newBuilders.back().first.appendAs(interval.start, fieldName); + + newBuilders.back().second.appendElements(second); + newBuilders.back().second.appendAs(interval.end, fieldName); + } + } + + builders = std::move(newBuilders); + } + } else { + // If we've already generated a range or multiple point-intervals just extend what we've + // generated with min/max bounds for this field + for (auto& builder : builders) { + builder.first.appendAs(intervals.front().start, fieldName); + builder.second.appendAs(intervals.back().end, fieldName); + } + } + } + + BoundList ret; + for (auto& builder : builders) { + ret.emplace_back(builder.first.obj(), builder.second.obj()); + } + + return ret; +} + +IndexBounds getIndexBoundsForQuery(const BSONObj& key, const CanonicalQuery& canonicalQuery) { + // $text is not allowed in planning since we don't have text index on mongos. + // TODO: Treat $text query as a no-op in planning on mongos. So with shard key {a: 1}, + // the query { a: 2, $text: { ... } } will only target to {a: 2}. + if (QueryPlannerCommon::hasNode(canonicalQuery.root(), MatchExpression::TEXT)) { + IndexBounds bounds; + IndexBoundsBuilder::allValuesBounds(key, &bounds, false); // [minKey, maxKey] + return bounds; + } + + // Similarly, ignore GEO_NEAR queries in planning, since we do not have geo indexes on mongos. + if (QueryPlannerCommon::hasNode(canonicalQuery.root(), MatchExpression::GEO_NEAR)) { + // If the GEO_NEAR predicate is a child of AND, remove the GEO_NEAR and continue building + // bounds. Currently a CanonicalQuery can have at most one GEO_NEAR expression, and only at + // the top-level, so this check is sufficient. + auto geoIdx = [](auto root) -> boost::optional<size_t> { + if (root->matchType() == MatchExpression::AND) { + for (size_t i = 0; i < root->numChildren(); ++i) { + if (MatchExpression::GEO_NEAR == root->getChild(i)->matchType()) { + return boost::make_optional(i); + } + } + } + return boost::none; + }(canonicalQuery.root()); + + if (!geoIdx) { + IndexBounds bounds; + IndexBoundsBuilder::allValuesBounds(key, &bounds, false); + return bounds; + } + + canonicalQuery.root()->getChildVector()->erase( + canonicalQuery.root()->getChildVector()->begin() + geoIdx.value()); + } + + // Consider shard key as an index + std::string accessMethod = IndexNames::findPluginName(key); + dassert(accessMethod == IndexNames::BTREE || accessMethod == IndexNames::HASHED); + const auto indexType = IndexNames::nameToType(accessMethod); + + // Use query framework to generate index bounds + QueryPlannerParams plannerParams; + // Must use "shard key" index + plannerParams.options = QueryPlannerParams::NO_TABLE_SCAN; + IndexEntry indexEntry(key, + indexType, + IndexDescriptor::kLatestIndexVersion, + // The shard key index cannot be multikey. + false, + // Empty multikey paths, since the shard key index cannot be multikey. + MultikeyPaths{}, + // Empty multikey path set, since the shard key index cannot be multikey. + {}, + false /* sparse */, + false /* unique */, + IndexEntry::Identifier{"shardkey"}, + nullptr /* filterExpr */, + BSONObj(), + nullptr, /* collator */ + nullptr /* projExec */); + plannerParams.indices.push_back(std::move(indexEntry)); + + auto statusWithMultiPlanSolns = QueryPlanner::plan(canonicalQuery, plannerParams); + if (statusWithMultiPlanSolns.getStatus().code() != ErrorCodes::NoQueryExecutionPlans) { + auto solutions = uassertStatusOK(std::move(statusWithMultiPlanSolns)); + + // Pick any solution that has non-trivial IndexBounds. bounds.size() == 0 represents a + // trivial IndexBounds where none of the fields' values are bounded. + for (auto&& soln : solutions) { + IndexBounds bounds = collapseQuerySolution(soln->root()); + if (bounds.size() > 0) { + return bounds; + } + } + } + + // We cannot plan the query without collection scan, so target to all shards. + IndexBounds bounds; + IndexBoundsBuilder::allValuesBounds(key, &bounds, false); // [minKey, maxKey] + return bounds; +} + +void getShardIdsForQuery(boost::intrusive_ptr<ExpressionContext> expCtx, + const BSONObj& query, + const BSONObj& collation, + const ChunkManager& cm, + std::set<ShardId>* shardIds, + std::set<ChunkRange>* chunkRanges, + bool* targetMinKeyToMaxKey) { + if (chunkRanges) { + invariant(chunkRanges->empty()); + } + + auto findCommand = std::make_unique<FindCommandRequest>(cm.getNss()); + findCommand->setFilter(query.getOwned()); + + expCtx->uuid = cm.getUUID(); + + if (!collation.isEmpty()) { + findCommand->setCollation(collation.getOwned()); + } else if (cm.getDefaultCollator()) { + auto defaultCollator = cm.getDefaultCollator(); + findCommand->setCollation(defaultCollator->getSpec().toBSON()); + expCtx->setCollator(defaultCollator->clone()); + } + + auto cq = uassertStatusOK( + CanonicalQuery::canonicalize(expCtx->opCtx, + std::move(findCommand), + false, /* isExplain */ + expCtx, + ExtensionsCallbackNoop(), + MatchExpressionParser::kAllowAllSpecialFeatures)); + + // Fast path for targeting equalities on the shard key. + auto shardKeyToFind = extractShardKeyFromQuery(cm.getShardKeyPattern(), *cq); + if (!shardKeyToFind.isEmpty()) { + try { + auto chunk = cm.findIntersectingChunk(shardKeyToFind, collation); + shardIds->insert(chunk.getShardId()); + if (chunkRanges) { + chunkRanges->insert(chunk.getRange()); + } + if (targetMinKeyToMaxKey) { + *targetMinKeyToMaxKey = false; + } + return; + } catch (const DBException&) { + // The query uses multiple shards + } + } + + // Transforms query into bounds for each field in the shard key + // for example : + // Key { a: 1, b: 1 }, + // Query { a : { $gte : 1, $lt : 2 }, + // b : { $gte : 3, $lt : 4 } } + // => Bounds { a : [1, 2), b : [3, 4) } + IndexBounds bounds = getIndexBoundsForQuery(cm.getShardKeyPattern().toBSON(), *cq); + + // Transforms bounds for each shard key field into full shard key ranges + // for example : + // Key { a : 1, b : 1 } + // Bounds { a : [1, 2), b : [3, 4) } + // => Ranges { a : 1, b : 3 } => { a : 2, b : 4 } + BoundList ranges = flattenBounds(cm.getShardKeyPattern(), bounds); + + for (BoundList::const_iterator it = ranges.begin(); it != ranges.end(); ++it) { + const auto& min = it->first; + const auto& max = it->second; + + cm.getShardIdsForRange(min, max, shardIds, chunkRanges); + + if (targetMinKeyToMaxKey && ChunkMap::allElementsAreOfType(MinKey, min) && + ChunkMap::allElementsAreOfType(MaxKey, max)) { + *targetMinKeyToMaxKey = true; + } + + // Once we know we need to visit all shards no need to keep looping. + // However, this optimization does not apply when we are reading from a snapshot + // because _shardVersions contains shards with chunks and is built based on the last + // refresh. Therefore, it is possible for _shardVersions to have fewer entries if a shard + // no longer owns chunks when it used to at _clusterTime. + if (!cm.isAtPointInTime() && shardIds->size() == cm.getNShardsOwningChunks()) { + break; + } + } + + // SERVER-4914 Some clients of getShardIdsForQuery() assume at least one shard will be returned. + // For now, we satisfy that assumption by adding a shard with no matches rather than returning + // an empty set of shards. + if (shardIds->empty()) { + cm.forEachChunk([&](const Chunk& chunk) { + shardIds->insert(chunk.getShardId()); + if (chunkRanges) { + chunkRanges->insert(chunk.getRange()); + } + if (targetMinKeyToMaxKey) { + *targetMinKeyToMaxKey = false; + } + return false; + }); + } } } // namespace mongo diff --git a/src/mongo/s/shard_key_pattern_query_util.h b/src/mongo/s/shard_key_pattern_query_util.h index 1872887f96b..07c3757370c 100644 --- a/src/mongo/s/shard_key_pattern_query_util.h +++ b/src/mongo/s/shard_key_pattern_query_util.h @@ -29,6 +29,9 @@ #pragma once +#include "mongo/db/query/canonical_query.h" +#include "mongo/db/query/index_bounds.h" +#include "mongo/s/chunk_manager.h" #include "mongo/s/shard_key_pattern.h" namespace mongo { @@ -72,4 +75,53 @@ StatusWith<BSONObj> extractShardKeyFromBasicQueryWithContext( const ShardKeyPattern& shardKeyPattern, const BSONObj& basicQuery); +BSONObj extractShardKeyFromQuery(const ShardKeyPattern& shardKeyPattern, + const CanonicalQuery& query); + +/** + * Return an ordered list of bounds generated using a ShardKeyPattern and the bounds from the + * IndexBounds. This function is used in sharding to determine where to route queries according to + * the shard key pattern. + * + * Examples: + * + * Key { a: 1 }, Bounds a: [0] => { a: 0 } -> { a: 0 } + * Key { a: 1 }, Bounds a: [2, 3) => { a: 2 } -> { a: 3 } // bound inclusion ignored. + * + * The bounds returned by this function may be a superset of those defined by the constraints. For + * instance, if this KeyPattern is {a : 1, b: 1} Bounds: { a : {$in : [1,2]} , b : {$in : [3,4,5]} } + * => {a : 1 , b : 3} -> {a : 1 , b : 5}, {a : 2 , b : 3} -> {a : 2 , b : 5} + * + * If the IndexBounds are not defined for all the fields in this keypattern, which means some fields + * are unsatisfied, an empty BoundList could return. + * + */ +BoundList flattenBounds(const ShardKeyPattern& shardKeyPattern, const IndexBounds& indexBounds); + +/** + * Transforms query into bounds for each field in the shard key. + * For example : + * Key { a: 1, b: 1 }, + * Query { a : { $gte : 1, $lt : 2 }, + * b : { $gte : 3, $lt : 4 } } + * => Bounds { a : [1, 2), b : [3, 4) } + */ +IndexBounds getIndexBoundsForQuery(const BSONObj& key, const CanonicalQuery& canonicalQuery); + +/** + * Finds the shard IDs for a given filter and collation. If collation is empty, we use the + * collection default collation for targeting. + * + * If 'chunkRanges' is not null, populates it with ChunkRanges that would be targeted by the query. + * If 'targetMinKeyToMaxKey' is not null, sets it to true if the query targets the entire shard key + * space. + */ +void getShardIdsForQuery(boost::intrusive_ptr<ExpressionContext> expCtx, + const BSONObj& query, + const BSONObj& collation, + const ChunkManager& cm, + std::set<ShardId>* shardIds, + std::set<ChunkRange>* chunkRanges = nullptr, + bool* targetMinKeyToMaxKey = nullptr); + } // namespace mongo diff --git a/src/mongo/s/chunk_manager_index_bounds_test.cpp b/src/mongo/s/shard_key_pattern_query_util_index_bounds_test.cpp index 11701f0525c..672067b9ad9 100644 --- a/src/mongo/s/chunk_manager_index_bounds_test.cpp +++ b/src/mongo/s/shard_key_pattern_query_util_index_bounds_test.cpp @@ -27,23 +27,17 @@ * it in the license file. */ - -#include "mongo/platform/basic.h" - #include "mongo/db/hasher.h" #include "mongo/db/json.h" #include "mongo/db/matcher/extensions_callback_noop.h" -#include "mongo/db/namespace_string.h" #include "mongo/db/pipeline/expression_context_for_test.h" #include "mongo/db/query/canonical_query.h" #include "mongo/logv2/log.h" -#include "mongo/s/chunk_manager.h" -#include "mongo/s/shard_key_pattern.h" +#include "mongo/s/shard_key_pattern_query_util.h" #include "mongo/s/sharding_router_test_fixture.h" #define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kTest - namespace mongo { namespace { @@ -82,7 +76,7 @@ protected: BSONObj key = fromjson(keyStr); - IndexBounds indexBounds = ChunkManager::getIndexBoundsForQuery(key, *query.get()); + IndexBounds indexBounds = getIndexBoundsForQuery(key, *query.get()); ASSERT_EQUALS(indexBounds.size(), expectedBounds.size()); for (size_t i = 0; i < indexBounds.size(); i++) { const OrderedIntervalList& oil = indexBounds.fields[i]; @@ -109,7 +103,7 @@ protected: BSONObj key = fromjson("{a: 1}"); - IndexBounds indexBounds = ChunkManager::getIndexBoundsForQuery(key, *query.get()); + IndexBounds indexBounds = getIndexBoundsForQuery(key, *query.get()); ASSERT_EQUALS(indexBounds.size(), 1U); const OrderedIntervalList& oil = indexBounds.fields.front(); @@ -296,7 +290,7 @@ TEST_F(CMCollapseTreeTest, BasicAllElemMatch) { BSONObj key = fromjson("{'foo.a': 1}"); - IndexBounds indexBounds = ChunkManager::getIndexBoundsForQuery(key, *query.get()); + IndexBounds indexBounds = getIndexBoundsForQuery(key, *query.get()); ASSERT_EQUALS(indexBounds.size(), 1U); const OrderedIntervalList& oil = indexBounds.fields.front(); ASSERT_EQUALS(oil.intervals.size(), 1U); @@ -475,7 +469,7 @@ TEST_F(CMKeyBoundsTest, Basic) { expectedList.emplace_back(fromjson("{a: 0}"), fromjson("{a: 0}")); ShardKeyPattern skeyPattern(fromjson("{a: 1}")); - BoundList list = skeyPattern.flattenBounds(indexBounds); + BoundList list = flattenBounds(skeyPattern, indexBounds); checkBoundList(list, expectedList); } @@ -490,7 +484,7 @@ TEST_F(CMKeyBoundsTest, SingleInterval) { expectedList.emplace_back(fromjson("{a: 2}"), fromjson("{a: 3}")); ShardKeyPattern skeyPattern(fromjson("{a: 1}")); - BoundList list = skeyPattern.flattenBounds(indexBounds); + BoundList list = flattenBounds(skeyPattern, indexBounds); checkBoundList(list, expectedList); } @@ -509,7 +503,7 @@ TEST_F(CMKeyBoundsTest, MultiIntervals) { expectedList.emplace_back(fromjson("{ a: 2, b: 2, c: 2 }"), fromjson("{ a: 3, b: 3, c: 3 }")); ShardKeyPattern skeyPattern(fromjson("{a: 1, b: 1, c: 1}")); - BoundList list = skeyPattern.flattenBounds(indexBounds); + BoundList list = flattenBounds(skeyPattern, indexBounds); checkBoundList(list, expectedList); } @@ -537,7 +531,7 @@ TEST_F(CMKeyBoundsTest, IntervalExpansion) { expectedList.emplace_back(fromjson("{ a: 0, b: 6, c: 2 }"), fromjson("{ a: 0, b: 6, c: 3 }")); ShardKeyPattern skeyPattern(fromjson("{a: 1, b: 1, c: 1}")); - BoundList list = skeyPattern.flattenBounds(indexBounds); + BoundList list = flattenBounds(skeyPattern, indexBounds); checkBoundList(list, expectedList); } @@ -562,7 +556,7 @@ TEST_F(CMKeyBoundsTest, NonPointIntervalExpasion) { expectedList.emplace_back(fromjson("{ a: 0, b: 4, c: 2 }"), fromjson("{ a: 1, b: 6, c: 3 }")); ShardKeyPattern skeyPattern(fromjson("{a: 1, b: 1, c: 1}")); - BoundList list = skeyPattern.flattenBounds(indexBounds); + BoundList list = flattenBounds(skeyPattern, indexBounds); checkBoundList(list, expectedList); } diff --git a/src/mongo/s/write_ops/batch_write_op.cpp b/src/mongo/s/write_ops/batch_write_op.cpp index d5e4dc79ba0..5b2df548795 100644 --- a/src/mongo/s/write_ops/batch_write_op.cpp +++ b/src/mongo/s/write_ops/batch_write_op.cpp @@ -39,7 +39,6 @@ #include "mongo/db/operation_context.h" #include "mongo/db/ops/write_ops.h" #include "mongo/s/client/num_hosts_targeted_metrics.h" -#include "mongo/s/cluster_commands_helpers.h" #include "mongo/s/collection_uuid_mismatch.h" #include "mongo/s/transaction_router.h" #include "mongo/s/write_ops/write_without_shard_key_util.h" diff --git a/src/mongo/s/write_ops/write_without_shard_key_util.cpp b/src/mongo/s/write_ops/write_without_shard_key_util.cpp index cb2d2d12884..3a623cc02e1 100644 --- a/src/mongo/s/write_ops/write_without_shard_key_util.cpp +++ b/src/mongo/s/write_ops/write_without_shard_key_util.cpp @@ -75,7 +75,7 @@ bool isExactIdQuery(OperationContext* opCtx, if (cq.isOK()) { // Only returns a shard key iff a query has a full shard key with direct/equality matches on // all shard key fields. - auto shardKey = kVirtualIdShardKey.extractShardKeyFromQuery(*cq.getValue()); + auto shardKey = extractShardKeyFromQuery(kVirtualIdShardKey, *cq.getValue()); BSONElement idElt = shardKey["_id"]; if (!idElt) { |