From 33201c068f9ec37196190be083d61f46dc83d470 Mon Sep 17 00:00:00 2001 From: Kaloian Manassiev Date: Tue, 16 Jun 2020 07:53:22 -0400 Subject: SERVER-48775 Move all Sharding unit-tests that assume a MongoD to the db/s directory With this change there are no more references from mongo/s (which is code common between MongoS and MongoD) and mongo/db/s (which is code specific for MongoD only). --- src/mongo/db/SConscript | 28 +- src/mongo/db/keys_collection_client_direct.h | 3 +- src/mongo/db/logical_clock_test_fixture.h | 3 +- src/mongo/db/pipeline/SConscript | 1 - src/mongo/db/pipeline/process_interface/SConscript | 2 +- src/mongo/db/rs_local_client.cpp | 155 ++++++++++ src/mongo/db/rs_local_client.h | 99 ++++++ src/mongo/db/s/SConscript | 92 +++--- .../db/s/collection_metadata_filtering_test.cpp | 2 +- .../db/s/collection_sharding_runtime_test.cpp | 2 +- .../db/s/config/config_server_test_fixture.cpp | 5 - src/mongo/db/s/config/config_server_test_fixture.h | 17 +- .../sharding_catalog_manager_remove_shard_test.cpp | 2 - src/mongo/db/s/metadata_manager_test.cpp | 2 +- .../migration_chunk_cloner_source_legacy_test.cpp | 2 +- .../db/s/migration_destination_manager_test.cpp | 3 +- src/mongo/db/s/migration_util_test.cpp | 3 +- src/mongo/db/s/op_observer_sharding_test.cpp | 2 +- src/mongo/db/s/persistent_task_queue_test.cpp | 3 +- src/mongo/db/s/persistent_task_store_test.cpp | 5 +- src/mongo/db/s/range_deletion_util_test.cpp | 3 +- .../session_catalog_migration_destination_test.cpp | 3 +- src/mongo/db/s/shard_local.cpp | 190 ++++++++++++ src/mongo/db/s/shard_local.h | 100 ++++++ src/mongo/db/s/shard_local_test.cpp | 273 +++++++++++++++++ src/mongo/db/s/shard_metadata_util_test.cpp | 3 +- .../s/shard_server_catalog_cache_loader_test.cpp | 6 +- src/mongo/db/s/shard_server_test_fixture.cpp | 102 +++++++ src/mongo/db/s/shard_server_test_fixture.h | 83 +++++ src/mongo/db/s/sharding_initialization_mongod.cpp | 2 +- .../db/s/sharding_initialization_mongod_test.cpp | 2 +- .../s/sharding_initialization_op_observer_test.cpp | 2 +- src/mongo/db/s/sharding_logging_test.cpp | 37 +-- src/mongo/db/s/sharding_mongod_test_fixture.cpp | 337 +++++++++++++++++++++ src/mongo/db/s/sharding_mongod_test_fixture.h | 152 ++++++++++ src/mongo/db/s/split_chunk_test.cpp | 5 +- src/mongo/db/s/split_vector_test.cpp | 3 +- .../transaction_coordinator_futures_util_test.cpp | 3 +- .../db/s/transaction_coordinator_test_fixture.h | 2 +- src/mongo/db/s/vector_clock_shard_server_test.cpp | 3 +- src/mongo/db/service_context_d_test_fixture.cpp | 11 +- src/mongo/db/service_context_d_test_fixture.h | 7 +- src/mongo/db/service_context_test_fixture.h | 1 - src/mongo/db/vector_clock_mongod_test.cpp | 3 +- 44 files changed, 1608 insertions(+), 156 deletions(-) create mode 100644 src/mongo/db/rs_local_client.cpp create mode 100644 src/mongo/db/rs_local_client.h create mode 100644 src/mongo/db/s/shard_local.cpp create mode 100644 src/mongo/db/s/shard_local.h create mode 100644 src/mongo/db/s/shard_local_test.cpp create mode 100644 src/mongo/db/s/shard_server_test_fixture.cpp create mode 100644 src/mongo/db/s/shard_server_test_fixture.h create mode 100644 src/mongo/db/s/sharding_mongod_test_fixture.cpp create mode 100644 src/mongo/db/s/sharding_mongod_test_fixture.h (limited to 'src/mongo/db') diff --git a/src/mongo/db/SConscript b/src/mongo/db/SConscript index 642e54fbb7a..adab7bc70e4 100644 --- a/src/mongo/db/SConscript +++ b/src/mongo/db/SConscript @@ -1506,6 +1506,17 @@ env.Library( ], ) +env.Library( + target='rs_local_client', + source=[ + 'rs_local_client.cpp', + ], + LIBDEPS_PRIVATE=[ + '$BUILD_DIR/mongo/db/repl/repl_coordinator_interface', + 'dbdirectclient', + ], +) + env.Library( target='keys_collection_document', source=[ @@ -1523,11 +1534,13 @@ env.Library( 'keys_collection_client_direct.cpp', ], LIBDEPS=[ - 'keys_collection_document', - 'logical_time', '$BUILD_DIR/mongo/s/catalog/sharding_catalog_client', - '$BUILD_DIR/mongo/s/client/rs_local_client', '$BUILD_DIR/mongo/s/client/shard_interface', + 'keys_collection_document', + 'logical_time', + ], + LIBDEPS_PRIVATE=[ + 'rs_local_client', ], ) @@ -1635,11 +1648,11 @@ env.Library( 'logical_clock_test_fixture.cpp', ], LIBDEPS= [ + '$BUILD_DIR/mongo/db/auth/authmocks', + '$BUILD_DIR/mongo/util/clock_source_mock', + 's/sharding_mongod_test_fixture', 'signed_logical_time', 'vector_clock', - '$BUILD_DIR/mongo/db/auth/authmocks', - '$BUILD_DIR/mongo/s/sharding_mongod_test_fixture', - '$BUILD_DIR/mongo/util/clock_source_mock' ], ) @@ -1663,7 +1676,6 @@ env.Library( 'service_context_d_test_fixture.cpp', ], LIBDEPS=[ - '$BUILD_DIR/mongo/unittest/unittest', 'service_context_test_fixture', ], LIBDEPS_PRIVATE=[ @@ -1989,7 +2001,7 @@ env.CppLibfuzzerTest( '$BUILD_DIR/mongo/transport/service_entry_point', '$BUILD_DIR/mongo/transport/transport_layer_mock', '$BUILD_DIR/mongo/db/repl/replmocks', - '$BUILD_DIR/mongo/unittest/unittest/', + '$BUILD_DIR/mongo/unittest/unittest', ], ) diff --git a/src/mongo/db/keys_collection_client_direct.h b/src/mongo/db/keys_collection_client_direct.h index 6e96d8e94ed..3b101d99167 100644 --- a/src/mongo/db/keys_collection_client_direct.h +++ b/src/mongo/db/keys_collection_client_direct.h @@ -32,9 +32,8 @@ #include #include -#include "mongo/base/status.h" #include "mongo/db/keys_collection_client.h" -#include "mongo/s/client/rs_local_client.h" +#include "mongo/db/rs_local_client.h" namespace mongo { diff --git a/src/mongo/db/logical_clock_test_fixture.h b/src/mongo/db/logical_clock_test_fixture.h index 4b3322d406b..4602ae629cd 100644 --- a/src/mongo/db/logical_clock_test_fixture.h +++ b/src/mongo/db/logical_clock_test_fixture.h @@ -29,8 +29,7 @@ #pragma once -#include "mongo/s/sharding_mongod_test_fixture.h" -#include "mongo/unittest/unittest.h" +#include "mongo/db/s/sharding_mongod_test_fixture.h" namespace mongo { diff --git a/src/mongo/db/pipeline/SConscript b/src/mongo/db/pipeline/SConscript index d7841f55b3c..a45860c9c15 100644 --- a/src/mongo/db/pipeline/SConscript +++ b/src/mongo/db/pipeline/SConscript @@ -402,7 +402,6 @@ env.CppUnitTest( '$BUILD_DIR/mongo/db/service_context_d_test_fixture', '$BUILD_DIR/mongo/db/service_context_test_fixture', '$BUILD_DIR/mongo/executor/thread_pool_task_executor_test_fixture', - '$BUILD_DIR/mongo/s/catalog_cache_test_fixture', '$BUILD_DIR/mongo/s/is_mongos', '$BUILD_DIR/mongo/s/query/router_exec_stage', '$BUILD_DIR/mongo/s/sharding_router_test_fixture', diff --git a/src/mongo/db/pipeline/process_interface/SConscript b/src/mongo/db/pipeline/process_interface/SConscript index aa783988fd3..5c74b6f7999 100644 --- a/src/mongo/db/pipeline/process_interface/SConscript +++ b/src/mongo/db/pipeline/process_interface/SConscript @@ -120,7 +120,7 @@ env.CppUnitTest( '$BUILD_DIR/mongo/db/query/query_test_service_context', '$BUILD_DIR/mongo/db/service_context_test_fixture', '$BUILD_DIR/mongo/db/vector_clock_mongod', - '$BUILD_DIR/mongo/s/catalog_cache_test_fixture', + '$BUILD_DIR/mongo/s/sharding_router_test_fixture', 'mongos_process_interface', 'shardsvr_process_interface', ] diff --git a/src/mongo/db/rs_local_client.cpp b/src/mongo/db/rs_local_client.cpp new file mode 100644 index 00000000000..253efa13eca --- /dev/null +++ b/src/mongo/db/rs_local_client.cpp @@ -0,0 +1,155 @@ +/** + * Copyright (C) 2018-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/platform/basic.h" + +#include "mongo/db/rs_local_client.h" + +#include "mongo/db/curop.h" +#include "mongo/db/dbdirectclient.h" +#include "mongo/db/repl/repl_client_info.h" +#include "mongo/db/repl/repl_set_config.h" +#include "mongo/db/repl/replication_coordinator.h" +#include "mongo/rpc/get_status_from_command_result.h" +#include "mongo/rpc/unique_message.h" +#include "mongo/util/scopeguard.h" + +namespace mongo { + +void RSLocalClient::_updateLastOpTimeFromClient(OperationContext* opCtx, + const repl::OpTime& previousOpTimeOnClient) { + const auto lastOpTimeFromClient = + repl::ReplClientInfo::forClient(opCtx->getClient()).getLastOp(); + + if (lastOpTimeFromClient.isNull() || lastOpTimeFromClient == previousOpTimeOnClient) { + return; + } + + stdx::lock_guard lk(_mutex); + if (lastOpTimeFromClient >= _lastOpTime) { + // It's always possible for lastOpTimeFromClient to be less than _lastOpTime if another + // thread started and completed a write through this ShardLocal (updating _lastOpTime) + // after this operation had completed its write but before it got here. + _lastOpTime = lastOpTimeFromClient; + } +} + +repl::OpTime RSLocalClient::_getLastOpTime() { + stdx::lock_guard lk(_mutex); + return _lastOpTime; +} + +StatusWith RSLocalClient::runCommandOnce(OperationContext* opCtx, + StringData dbName, + const BSONObj& cmdObj) { + const auto currentOpTimeFromClient = + repl::ReplClientInfo::forClient(opCtx->getClient()).getLastOp(); + ON_BLOCK_EXIT([this, &opCtx, ¤tOpTimeFromClient] { + _updateLastOpTimeFromClient(opCtx, currentOpTimeFromClient); + }); + + try { + DBDirectClient client(opCtx); + + rpc::UniqueReply commandResponse = + client.runCommand(OpMsgRequest::fromDBAndBody(dbName, cmdObj)); + + auto result = commandResponse->getCommandReply().getOwned(); + return Shard::CommandResponse(boost::none, + result, + getStatusFromCommandResult(result), + getWriteConcernStatusFromCommandResult(result)); + } catch (const DBException& ex) { + return ex.toStatus(); + } +} + +StatusWith RSLocalClient::queryOnce( + OperationContext* opCtx, + const ReadPreferenceSetting& readPref, + const repl::ReadConcernLevel& readConcernLevel, + const NamespaceString& nss, + const BSONObj& query, + const BSONObj& sort, + boost::optional limit) { + auto replCoord = repl::ReplicationCoordinator::get(opCtx); + + if (readConcernLevel == repl::ReadConcernLevel::kMajorityReadConcern) { + // Set up operation context with majority read snapshot so correct optime can be retrieved. + opCtx->recoveryUnit()->setTimestampReadSource(RecoveryUnit::ReadSource::kMajorityCommitted); + Status status = opCtx->recoveryUnit()->obtainMajorityCommittedSnapshot(); + + // Wait for any writes performed by this ShardLocal instance to be committed and visible. + Status readConcernStatus = replCoord->waitUntilOpTimeForRead( + opCtx, repl::ReadConcernArgs{_getLastOpTime(), readConcernLevel}); + if (!readConcernStatus.isOK()) { + return readConcernStatus; + } + + // Inform the storage engine to read from the committed snapshot for the rest of this + // operation. + status = opCtx->recoveryUnit()->obtainMajorityCommittedSnapshot(); + if (!status.isOK()) { + return status; + } + } else { + invariant(readConcernLevel == repl::ReadConcernLevel::kLocalReadConcern); + } + + DBDirectClient client(opCtx); + Query fullQuery(query); + if (!sort.isEmpty()) { + fullQuery.sort(sort); + } + fullQuery.readPref(readPref.pref, BSONArray()); + + try { + std::unique_ptr cursor = + client.query(nss, fullQuery, limit.get_value_or(0)); + + if (!cursor) { + return {ErrorCodes::OperationFailed, + str::stream() << "Failed to establish a cursor for reading " << nss.ns() + << " from local storage"}; + } + + std::vector documentVector; + while (cursor->more()) { + BSONObj document = cursor->nextSafe().getOwned(); + documentVector.push_back(std::move(document)); + } + + return Shard::QueryResponse{std::move(documentVector), + replCoord->getCurrentCommittedSnapshotOpTime()}; + } catch (const DBException& ex) { + return ex.toStatus(); + } +} + +} // namespace mongo diff --git a/src/mongo/db/rs_local_client.h b/src/mongo/db/rs_local_client.h new file mode 100644 index 00000000000..233732b5c8e --- /dev/null +++ b/src/mongo/db/rs_local_client.h @@ -0,0 +1,99 @@ +/** + * Copyright (C) 2018-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include "mongo/db/repl/optime.h" +#include "mongo/platform/mutex.h" +#include "mongo/s/client/shard.h" +#include "mongo/util/hierarchical_acquisition.h" + +namespace mongo { + +/** + * Implements the support for "read your own write" when run on a node of a replica set. Can be used + * in the scenarios where causal consistency is not available yet. + */ +class RSLocalClient { + RSLocalClient(const RSLocalClient&) = delete; + RSLocalClient& operator=(const RSLocalClient&) = delete; + +public: + explicit RSLocalClient() = default; + + ~RSLocalClient() = default; + + /** + * Runs the specified command returns the BSON command response plus parsed out Status of this + * response and write concern error (if present). + */ + StatusWith runCommandOnce(OperationContext* opCtx, + StringData dbName, + const BSONObj& cmdObj); + + /** + * Warning: This method exhausts the cursor and pulls all data into memory. + * Do not use other than for very small (i.e., admin or metadata) collections. + */ + StatusWith queryOnce(OperationContext* opCtx, + const ReadPreferenceSetting& readPref, + const repl::ReadConcernLevel& readConcernLevel, + const NamespaceString& nss, + const BSONObj& query, + const BSONObj& sort, + boost::optional limit); + +private: + /** + * Checks if an OpTime was set on the current Client (ie if the current operation performed a + * write) and if so updates _lastOpTime to the OpTime from the write that was just performed. + * The 'previousOpTimeOnClient' parameter is the optime that *was* the optime on this client + * before we ran this command through this RSLocalClient. By the time this method is called, + * if the optime set on the Client is different than 'previousOpTimeOnClient' then that means + * the command just run did a write and we should update _lastOpTime to capture the optime of + * the write we just did. If the current optime on the client is the same as + * 'previousOpTimeOnClient' then the command we just ran didn't do a write, and we should leave + * _lastOpTime alone. + */ + void _updateLastOpTimeFromClient(OperationContext* opCtx, + const repl::OpTime& previousOpTimeOnClient); + + repl::OpTime _getLastOpTime(); + + // Guards _lastOpTime below. + Mutex _mutex = MONGO_MAKE_LATCH(HierarchicalAcquisitionLevel(0), "RSLocalClient::_mutex"); + + // Stores the optime that was generated by the last operation to perform a write that was run + // through _runCommand. Used in _exhaustiveFindOnConfig for waiting for that optime to be + // committed so that readConcern majority reads will read the writes that were performed without + // a w:majority write concern. + repl::OpTime _lastOpTime{}; +}; + +} // namespace mongo diff --git a/src/mongo/db/s/SConscript b/src/mongo/db/s/SConscript index 841cddda124..d072b62a778 100644 --- a/src/mongo/db/s/SConscript +++ b/src/mongo/db/s/SConscript @@ -81,6 +81,7 @@ env.Library( 'sharding_state_recovery.cpp', 'sharding_statistics.cpp', 'split_chunk.cpp', + 'shard_local.cpp', 'split_vector.cpp', 'start_chunk_clone_request.cpp', env.Idlc('migration_coordinator_document.idl')[0], @@ -95,13 +96,11 @@ env.Library( '$BUILD_DIR/mongo/db/op_observer_impl', '$BUILD_DIR/mongo/db/ops/write_ops_exec', '$BUILD_DIR/mongo/db/repl/oplog', - '$BUILD_DIR/mongo/db/repl/repl_coordinator_interface', '$BUILD_DIR/mongo/db/rw_concern_d', '$BUILD_DIR/mongo/db/server_options_core', '$BUILD_DIR/mongo/db/storage/remove_saver', '$BUILD_DIR/mongo/db/transaction', '$BUILD_DIR/mongo/db/vector_clock_mongod', - '$BUILD_DIR/mongo/s/client/shard_local', '$BUILD_DIR/mongo/s/query/cluster_aggregate', '$BUILD_DIR/mongo/s/sharding_initialization', 'chunk_splitter', @@ -113,6 +112,7 @@ env.Library( ], LIBDEPS_PRIVATE=[ '$BUILD_DIR/mongo/db/index_builds_coordinator_interface', + '$BUILD_DIR/mongo/db/rs_local_client', '$BUILD_DIR/mongo/db/session_catalog', '$BUILD_DIR/mongo/idl/server_parameter', ], @@ -342,60 +342,40 @@ env.Library( ) env.Library( - target='config_server_test_fixture', + target='sharding_mongod_test_fixture', source=[ - 'config/config_server_test_fixture.cpp', + 'sharding_mongod_test_fixture.cpp', ], LIBDEPS=[ - '$BUILD_DIR/mongo/s/sharding_mongod_test_fixture', - 'sharding_catalog_manager', + '$BUILD_DIR/mongo/db/repl/drop_pending_collection_reaper', + '$BUILD_DIR/mongo/db/repl/replmocks', + '$BUILD_DIR/mongo/db/service_context_d_test_fixture', + '$BUILD_DIR/mongo/s/sharding_test_fixture_common', + 'sharding_runtime_d', ], ) -env.CppUnitTest( - target='db_s_test', +env.Library( + target='shard_server_test_fixture', source=[ - 'chunk_split_state_driver_test.cpp', - 'config_server_op_observer_test.cpp', - 'migration_session_id_test.cpp', - 'sharding_logging_test.cpp', - 'start_chunk_clone_request_test.cpp', - 'type_shard_identity_test.cpp', - 'vector_clock_config_server_test.cpp', - 'wait_for_majority_service_test.cpp', + 'shard_server_test_fixture.cpp', ], LIBDEPS=[ - '$BUILD_DIR/mongo/db/auth/authmocks', + '$BUILD_DIR/mongo/s/catalog/dist_lock_catalog_mock', '$BUILD_DIR/mongo/s/catalog/dist_lock_manager_mock', - '$BUILD_DIR/mongo/s/sharding_router_test_fixture', - 'chunk_splitter', - 'config_server_test_fixture', - 'sharding_logging', - 'sharding_runtime_d', - 'type_shard_identity', + 'sharding_mongod_test_fixture', ], ) -env.CppUnitTest( - target='db_s_balancer_test', +env.Library( + target='config_server_test_fixture', source=[ - 'balancer/balancer_chunk_selection_policy_test.cpp', - 'balancer/balancer_policy_test.cpp', - 'balancer/cluster_statistics_test.cpp', - 'balancer/core_options_stub.cpp', - 'balancer/migration_manager_test.cpp', - 'balancer/migration_test_fixture.cpp', - 'balancer/scoped_migration_request_test.cpp', - 'balancer/type_migration_test.cpp', + 'config/config_server_test_fixture.cpp', ], LIBDEPS=[ - '$BUILD_DIR/mongo/db/auth/authmocks', - '$BUILD_DIR/mongo/db/read_write_concern_defaults_mock', - '$BUILD_DIR/mongo/db/repl/replication_info', - '$BUILD_DIR/mongo/util/version_impl', - 'balancer', - 'config_server_test_fixture', - ] + 'sharding_catalog_manager', + 'sharding_mongod_test_fixture', + ], ) env.CppUnitTest( @@ -405,18 +385,25 @@ env.CppUnitTest( 'active_move_primaries_registry_test.cpp', 'active_shard_collection_registry_test.cpp', 'catalog_cache_loader_mock.cpp', + 'chunk_split_state_driver_test.cpp', 'migration_chunk_cloner_source_legacy_test.cpp', 'migration_destination_manager_test.cpp', + 'migration_session_id_test.cpp', 'migration_util_test.cpp', 'namespace_metadata_change_notifications_test.cpp', 'session_catalog_migration_destination_test.cpp', 'session_catalog_migration_source_test.cpp', + 'shard_local_test.cpp', 'shard_metadata_util_test.cpp', 'shard_server_catalog_cache_loader_test.cpp', 'sharding_initialization_mongod_test.cpp', 'sharding_initialization_op_observer_test.cpp', + 'sharding_logging_test.cpp', 'split_vector_test.cpp', + 'start_chunk_clone_request_test.cpp', + 'type_shard_identity_test.cpp', 'vector_clock_shard_server_test.cpp', + 'wait_for_majority_service_test.cpp', ], LIBDEPS_PRIVATE=[ '$BUILD_DIR/mongo/db/auth/authmocks', @@ -427,9 +414,9 @@ env.CppUnitTest( '$BUILD_DIR/mongo/db/repl/mock_repl_coord_server_fixture', '$BUILD_DIR/mongo/db/repl/storage_interface_impl', '$BUILD_DIR/mongo/s/catalog/dist_lock_manager_mock', - '$BUILD_DIR/mongo/s/catalog/sharding_catalog_client_impl', '$BUILD_DIR/mongo/s/catalog/sharding_catalog_client_mock', - '$BUILD_DIR/mongo/s/shard_server_test_fixture', + 'shard_server_test_fixture', + 'sharding_logging', 'sharding_runtime_d', ], ) @@ -450,9 +437,8 @@ env.CppUnitTest( '$BUILD_DIR/mongo/client/remote_command_targeter_mock', '$BUILD_DIR/mongo/db/auth/authmocks', '$BUILD_DIR/mongo/db/repl/replmocks', - '$BUILD_DIR/mongo/executor/network_test_env', '$BUILD_DIR/mongo/executor/thread_pool_task_executor_test_fixture', - '$BUILD_DIR/mongo/s/shard_server_test_fixture', + 'shard_server_test_fixture', ], ) @@ -470,13 +456,13 @@ env.CppUnitTest( '$BUILD_DIR/mongo/db/auth/authmocks', '$BUILD_DIR/mongo/db/commands/server_status', '$BUILD_DIR/mongo/s/catalog/sharding_catalog_client_mock', - '$BUILD_DIR/mongo/s/shard_server_test_fixture', + 'shard_server_test_fixture', 'transaction_coordinator', ], ) env.CppUnitTest( - target='db_s_sharding_catalog_manager_test', + target='db_s_config_server_test', source=[ 'config/initial_split_policy_test.cpp', 'config/sharding_catalog_manager_add_shard_test.cpp', @@ -494,10 +480,24 @@ env.CppUnitTest( 'config/sharding_catalog_manager_remove_shard_test.cpp', 'config/sharding_catalog_manager_shard_collection_test.cpp', 'config/sharding_catalog_manager_split_chunk_test.cpp', + 'balancer/balancer_chunk_selection_policy_test.cpp', + 'balancer/balancer_policy_test.cpp', + 'balancer/cluster_statistics_test.cpp', + 'balancer/core_options_stub.cpp', + 'balancer/migration_manager_test.cpp', + 'balancer/migration_test_fixture.cpp', + 'balancer/scoped_migration_request_test.cpp', + 'balancer/type_migration_test.cpp', + 'config_server_op_observer_test.cpp', + 'vector_clock_config_server_test.cpp', ], LIBDEPS=[ '$BUILD_DIR/mongo/db/auth/authmocks', + '$BUILD_DIR/mongo/db/read_write_concern_defaults_mock', + '$BUILD_DIR/mongo/db/repl/replication_info', + '$BUILD_DIR/mongo/s/catalog/dist_lock_manager_mock', '$BUILD_DIR/mongo/util/version_impl', + 'balancer', 'config_server_test_fixture', ] ) diff --git a/src/mongo/db/s/collection_metadata_filtering_test.cpp b/src/mongo/db/s/collection_metadata_filtering_test.cpp index 348cd18bfb4..ebb0d564bea 100644 --- a/src/mongo/db/s/collection_metadata_filtering_test.cpp +++ b/src/mongo/db/s/collection_metadata_filtering_test.cpp @@ -32,8 +32,8 @@ #include "mongo/db/catalog_raii.h" #include "mongo/db/s/collection_sharding_runtime.h" #include "mongo/db/s/operation_sharding_state.h" +#include "mongo/db/s/shard_server_test_fixture.h" #include "mongo/s/catalog/type_chunk.h" -#include "mongo/s/shard_server_test_fixture.h" namespace mongo { namespace { diff --git a/src/mongo/db/s/collection_sharding_runtime_test.cpp b/src/mongo/db/s/collection_sharding_runtime_test.cpp index c20f65db9d0..08d6ad3edad 100644 --- a/src/mongo/db/s/collection_sharding_runtime_test.cpp +++ b/src/mongo/db/s/collection_sharding_runtime_test.cpp @@ -34,8 +34,8 @@ #include "mongo/db/dbdirectclient.h" #include "mongo/db/s/collection_sharding_runtime.h" #include "mongo/db/s/operation_sharding_state.h" +#include "mongo/db/s/shard_server_test_fixture.h" #include "mongo/db/s/wait_for_majority_service.h" -#include "mongo/s/shard_server_test_fixture.h" #include "mongo/util/fail_point.h" namespace mongo { diff --git a/src/mongo/db/s/config/config_server_test_fixture.cpp b/src/mongo/db/s/config/config_server_test_fixture.cpp index 0cbc51e25b4..de55abb2be4 100644 --- a/src/mongo/db/s/config/config_server_test_fixture.cpp +++ b/src/mongo/db/s/config/config_server_test_fixture.cpp @@ -51,7 +51,6 @@ #include "mongo/db/repl/repl_settings.h" #include "mongo/db/repl/replication_coordinator_mock.h" #include "mongo/db/s/config/sharding_catalog_manager.h" -#include "mongo/executor/network_interface_mock.h" #include "mongo/executor/task_executor_pool.h" #include "mongo/executor/thread_pool_task_executor_test_fixture.h" #include "mongo/rpc/metadata/repl_set_metadata.h" @@ -60,17 +59,13 @@ #include "mongo/s/catalog/dist_lock_catalog_impl.h" #include "mongo/s/catalog/replset_dist_lock_manager.h" #include "mongo/s/catalog/sharding_catalog_client_impl.h" -#include "mongo/s/catalog/type_changelog.h" #include "mongo/s/catalog/type_chunk.h" #include "mongo/s/catalog/type_collection.h" #include "mongo/s/catalog/type_database.h" #include "mongo/s/catalog/type_shard.h" #include "mongo/s/catalog_cache.h" #include "mongo/s/chunk_version.h" -#include "mongo/s/client/shard_factory.h" -#include "mongo/s/client/shard_local.h" #include "mongo/s/client/shard_registry.h" -#include "mongo/s/client/shard_remote.h" #include "mongo/s/config_server_catalog_cache_loader.h" #include "mongo/s/database_version_helpers.h" #include "mongo/s/grid.h" diff --git a/src/mongo/db/s/config/config_server_test_fixture.h b/src/mongo/db/s/config/config_server_test_fixture.h index 0c64cfca33b..ba008a827f6 100644 --- a/src/mongo/db/s/config/config_server_test_fixture.h +++ b/src/mongo/db/s/config/config_server_test_fixture.h @@ -30,7 +30,7 @@ #pragma once #include "mongo/db/catalog_raii.h" -#include "mongo/s/sharding_mongod_test_fixture.h" +#include "mongo/db/s/sharding_mongod_test_fixture.h" namespace mongo { @@ -43,18 +43,19 @@ class Shard; class ShardId; class ShardRegistry; class ShardType; -template -class StatusWith; /** * Provides config-specific functionality in addition to the mock storage engine and mock network * provided by ShardingMongodTestFixture. */ class ConfigServerTestFixture : public ShardingMongodTestFixture { -public: +protected: ConfigServerTestFixture(); ~ConfigServerTestFixture(); + void setUp() override; + void tearDown() override; + std::shared_ptr getConfigShard() const; /** @@ -150,12 +151,6 @@ public: */ std::vector getKeys(OperationContext* opCtx); -protected: - /** - * Sets this node up as a mongod with sharding components for ClusterRole::ConfigServer. - */ - void setUp() override; - /** * Sets this node up and locks the config db in _setUp() before calling * initializeGlobalShardingStateForMongodForTest(). The RAII object for the database lock is @@ -170,8 +165,6 @@ protected: */ void setUpAndInitializeConfigDb(); - void tearDown() override; - std::unique_ptr makeDistLockCatalog() override; std::unique_ptr makeDistLockManager( diff --git a/src/mongo/db/s/config/sharding_catalog_manager_remove_shard_test.cpp b/src/mongo/db/s/config/sharding_catalog_manager_remove_shard_test.cpp index acf08255915..254afab7c12 100644 --- a/src/mongo/db/s/config/sharding_catalog_manager_remove_shard_test.cpp +++ b/src/mongo/db/s/config/sharding_catalog_manager_remove_shard_test.cpp @@ -39,11 +39,9 @@ #include "mongo/db/ops/write_ops.h" #include "mongo/db/s/config/config_server_test_fixture.h" #include "mongo/db/s/config/sharding_catalog_manager.h" -#include "mongo/executor/network_interface_mock.h" #include "mongo/executor/task_executor.h" #include "mongo/rpc/metadata/repl_set_metadata.h" #include "mongo/rpc/metadata/tracking_metadata.h" -#include "mongo/s/catalog/type_changelog.h" #include "mongo/s/catalog/type_chunk.h" #include "mongo/s/catalog/type_database.h" #include "mongo/s/catalog/type_shard.h" diff --git a/src/mongo/db/s/metadata_manager_test.cpp b/src/mongo/db/s/metadata_manager_test.cpp index dc053e3c202..760f04ea78e 100644 --- a/src/mongo/db/s/metadata_manager_test.cpp +++ b/src/mongo/db/s/metadata_manager_test.cpp @@ -37,6 +37,7 @@ #include "mongo/db/jsobj.h" #include "mongo/db/namespace_string.h" #include "mongo/db/s/metadata_manager.h" +#include "mongo/db/s/shard_server_test_fixture.h" #include "mongo/db/s/sharding_runtime_d_params_gen.h" #include "mongo/db/s/sharding_state.h" #include "mongo/db/server_options.h" @@ -44,7 +45,6 @@ #include "mongo/executor/task_executor.h" #include "mongo/s/catalog/type_chunk.h" #include "mongo/s/client/shard_registry.h" -#include "mongo/s/shard_server_test_fixture.h" #include "mongo/stdx/condition_variable.h" #include "mongo/unittest/unittest.h" #include "mongo/util/assert_util.h" diff --git a/src/mongo/db/s/migration_chunk_cloner_source_legacy_test.cpp b/src/mongo/db/s/migration_chunk_cloner_source_legacy_test.cpp index 730a2d4af3b..446b5984c68 100644 --- a/src/mongo/db/s/migration_chunk_cloner_source_legacy_test.cpp +++ b/src/mongo/db/s/migration_chunk_cloner_source_legacy_test.cpp @@ -36,10 +36,10 @@ #include "mongo/db/namespace_string.h" #include "mongo/db/s/collection_sharding_runtime.h" #include "mongo/db/s/migration_chunk_cloner_source_legacy.h" +#include "mongo/db/s/shard_server_test_fixture.h" #include "mongo/s/catalog/sharding_catalog_client_mock.h" #include "mongo/s/catalog/type_shard.h" #include "mongo/s/client/shard_registry.h" -#include "mongo/s/shard_server_test_fixture.h" #include "mongo/unittest/unittest.h" #include "mongo/util/clock_source_mock.h" diff --git a/src/mongo/db/s/migration_destination_manager_test.cpp b/src/mongo/db/s/migration_destination_manager_test.cpp index 9c5fc74de87..51aa7174b35 100644 --- a/src/mongo/db/s/migration_destination_manager_test.cpp +++ b/src/mongo/db/s/migration_destination_manager_test.cpp @@ -30,8 +30,7 @@ #include "mongo/platform/basic.h" #include "mongo/db/s/migration_destination_manager.h" -#include "mongo/s/shard_server_test_fixture.h" -#include "mongo/unittest/unittest.h" +#include "mongo/db/s/shard_server_test_fixture.h" namespace mongo { namespace { diff --git a/src/mongo/db/s/migration_util_test.cpp b/src/mongo/db/s/migration_util_test.cpp index 9ae0dfa8bed..f643224ccee 100644 --- a/src/mongo/db/s/migration_util_test.cpp +++ b/src/mongo/db/s/migration_util_test.cpp @@ -37,13 +37,12 @@ #include "mongo/db/s/persistent_task_store.h" #include "mongo/db/s/shard_filtering_metadata_refresh.h" #include "mongo/db/s/shard_server_catalog_cache_loader.h" +#include "mongo/db/s/shard_server_test_fixture.h" #include "mongo/db/s/sharding_state.h" #include "mongo/db/s/wait_for_majority_service.h" #include "mongo/s/catalog/sharding_catalog_client_mock.h" #include "mongo/s/catalog/type_shard.h" #include "mongo/s/database_version_helpers.h" -#include "mongo/s/shard_server_test_fixture.h" -#include "mongo/unittest/unittest.h" #include "mongo/util/future.h" namespace mongo { diff --git a/src/mongo/db/s/op_observer_sharding_test.cpp b/src/mongo/db/s/op_observer_sharding_test.cpp index 54b15903ce8..eaf8e55f78b 100644 --- a/src/mongo/db/s/op_observer_sharding_test.cpp +++ b/src/mongo/db/s/op_observer_sharding_test.cpp @@ -33,8 +33,8 @@ #include "mongo/db/s/collection_sharding_runtime.h" #include "mongo/db/s/op_observer_sharding_impl.h" #include "mongo/db/s/operation_sharding_state.h" +#include "mongo/db/s/shard_server_test_fixture.h" #include "mongo/db/s/type_shard_identity.h" -#include "mongo/s/shard_server_test_fixture.h" namespace mongo { namespace { diff --git a/src/mongo/db/s/persistent_task_queue_test.cpp b/src/mongo/db/s/persistent_task_queue_test.cpp index 96324d13761..6009b99860c 100644 --- a/src/mongo/db/s/persistent_task_queue_test.cpp +++ b/src/mongo/db/s/persistent_task_queue_test.cpp @@ -31,10 +31,9 @@ #include "mongo/db/db_raii.h" #include "mongo/db/s/collection_sharding_runtime.h" #include "mongo/db/s/persistent_task_queue.h" -#include "mongo/s/shard_server_test_fixture.h" +#include "mongo/db/s/shard_server_test_fixture.h" #include "mongo/stdx/thread.h" #include "mongo/unittest/barrier.h" -#include "mongo/unittest/unittest.h" namespace mongo { namespace { diff --git a/src/mongo/db/s/persistent_task_store_test.cpp b/src/mongo/db/s/persistent_task_store_test.cpp index c2abcf6a7be..177fe264d1d 100644 --- a/src/mongo/db/s/persistent_task_store_test.cpp +++ b/src/mongo/db/s/persistent_task_store_test.cpp @@ -27,11 +27,12 @@ * it in the license file. */ +#include "mongo/platform/basic.h" + #include "mongo/db/db_raii.h" #include "mongo/db/s/collection_sharding_runtime.h" #include "mongo/db/s/persistent_task_store.h" -#include "mongo/s/shard_server_test_fixture.h" -#include "mongo/unittest/unittest.h" +#include "mongo/db/s/shard_server_test_fixture.h" namespace mongo { namespace { diff --git a/src/mongo/db/s/range_deletion_util_test.cpp b/src/mongo/db/s/range_deletion_util_test.cpp index 0121d4b9e7c..919d723028f 100644 --- a/src/mongo/db/s/range_deletion_util_test.cpp +++ b/src/mongo/db/s/range_deletion_util_test.cpp @@ -37,11 +37,10 @@ #include "mongo/db/s/persistent_task_store.h" #include "mongo/db/s/range_deletion_task_gen.h" #include "mongo/db/s/range_deletion_util.h" +#include "mongo/db/s/shard_server_test_fixture.h" #include "mongo/db/s/sharding_runtime_d_params_gen.h" #include "mongo/db/s/wait_for_majority_service.h" -#include "mongo/s/shard_server_test_fixture.h" #include "mongo/unittest/death_test.h" -#include "mongo/unittest/unittest.h" #include "mongo/util/fail_point.h" namespace mongo { diff --git a/src/mongo/db/s/session_catalog_migration_destination_test.cpp b/src/mongo/db/s/session_catalog_migration_destination_test.cpp index 1f8e00bd3b2..68119925dfb 100644 --- a/src/mongo/db/s/session_catalog_migration_destination_test.cpp +++ b/src/mongo/db/s/session_catalog_migration_destination_test.cpp @@ -46,6 +46,7 @@ #include "mongo/db/s/collection_sharding_runtime.h" #include "mongo/db/s/migration_session_id.h" #include "mongo/db/s/session_catalog_migration_destination.h" +#include "mongo/db/s/shard_server_test_fixture.h" #include "mongo/db/server_options.h" #include "mongo/db/session_catalog_mongod.h" #include "mongo/db/session_txn_record_gen.h" @@ -55,9 +56,7 @@ #include "mongo/s/catalog/sharding_catalog_client_mock.h" #include "mongo/s/catalog/type_shard.h" #include "mongo/s/client/shard_registry.h" -#include "mongo/s/shard_server_test_fixture.h" #include "mongo/stdx/thread.h" -#include "mongo/unittest/unittest.h" namespace mongo { namespace { diff --git a/src/mongo/db/s/shard_local.cpp b/src/mongo/db/s/shard_local.cpp new file mode 100644 index 00000000000..6b18e6fa849 --- /dev/null +++ b/src/mongo/db/s/shard_local.cpp @@ -0,0 +1,190 @@ +/** + * Copyright (C) 2018-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/platform/basic.h" + +#include "mongo/db/s/shard_local.h" + +#include "mongo/client/remote_command_targeter.h" +#include "mongo/db/catalog/index_catalog.h" +#include "mongo/db/catalog_raii.h" +#include "mongo/db/concurrency/write_conflict_exception.h" +#include "mongo/db/index/index_descriptor.h" +#include "mongo/db/index_builds_coordinator.h" +#include "mongo/db/repl/repl_client_info.h" +#include "mongo/db/repl/repl_set_config.h" +#include "mongo/db/repl/replication_coordinator.h" +#include "mongo/db/server_options.h" +#include "mongo/util/assert_util.h" +#include "mongo/util/scopeguard.h" + +namespace mongo { + +ShardLocal::ShardLocal(const ShardId& id) : Shard(id), _rsLocalClient() { + // Currently ShardLocal only works for config servers. If we ever start using ShardLocal on + // shards we'll need to consider how to handle shards. + invariant(serverGlobalParams.clusterRole == ClusterRole::ConfigServer); +} + +const ConnectionString ShardLocal::getConnString() const { + return repl::ReplicationCoordinator::get(getGlobalServiceContext()) + ->getConfig() + .getConnectionString(); +} + +std::shared_ptr ShardLocal::getTargeter() const { + MONGO_UNREACHABLE; +}; + +const ConnectionString ShardLocal::originalConnString() const { + // Return the local connection string here as this method is only used for updating the + // ShardRegistry and we don't need a mapping from hosts in the replica set config to the shard + // for local shards. + return ConnectionString::forLocal(); +} + +void ShardLocal::updateReplSetMonitor(const HostAndPort& remoteHost, + const Status& remoteCommandStatus) { + MONGO_UNREACHABLE; +} + +void ShardLocal::updateLastCommittedOpTime(LogicalTime lastCommittedOpTime) { + MONGO_UNREACHABLE; +} + +LogicalTime ShardLocal::getLastCommittedOpTime() const { + MONGO_UNREACHABLE; +} + +std::string ShardLocal::toString() const { + return getId().toString() + ":"; +} + +bool ShardLocal::isRetriableError(ErrorCodes::Error code, RetryPolicy options) { + if (options == RetryPolicy::kNoRetry) { + return false; + } + + if (options == RetryPolicy::kIdempotent) { + return code == ErrorCodes::WriteConcernFailed; + } else { + invariant(options == RetryPolicy::kNotIdempotent); + return false; + } +} + +StatusWith ShardLocal::_runCommand(OperationContext* opCtx, + const ReadPreferenceSetting& unused, + StringData dbName, + Milliseconds maxTimeMSOverrideUnused, + const BSONObj& cmdObj) { + return _rsLocalClient.runCommandOnce(opCtx, dbName, cmdObj); +} + +StatusWith ShardLocal::_runExhaustiveCursorCommand( + OperationContext* opCtx, + const ReadPreferenceSetting& readPref, + StringData dbName, + Milliseconds maxTimeMSOverride, + const BSONObj& cmdObj) { + MONGO_UNREACHABLE; +} + +StatusWith ShardLocal::_exhaustiveFindOnConfig( + OperationContext* opCtx, + const ReadPreferenceSetting& readPref, + const repl::ReadConcernLevel& readConcernLevel, + const NamespaceString& nss, + const BSONObj& query, + const BSONObj& sort, + boost::optional limit) { + return _rsLocalClient.queryOnce(opCtx, readPref, readConcernLevel, nss, query, sort, limit); +} + +Status ShardLocal::createIndexOnConfig(OperationContext* opCtx, + const NamespaceString& ns, + const BSONObj& keys, + bool unique) { + invariant(ns.db() == "config" || ns.db() == "admin"); + + try { + AutoGetOrCreateDb autoDb(opCtx, ns.db(), MODE_IX); + AutoGetCollection autoColl(opCtx, ns, MODE_X); + auto collection = autoColl.getCollection(); + if (!collection) { + CollectionOptions options; + options.uuid = UUID::gen(); + writeConflictRetry(opCtx, "ShardLocal::createIndexOnConfig", ns.ns(), [&] { + WriteUnitOfWork wunit(opCtx); + auto db = autoDb.getDb(); + collection = db->createCollection(opCtx, ns, options); + invariant(collection, + str::stream() << "Failed to create collection " << ns.ns() + << " in config database for indexes: " << keys); + wunit.commit(); + }); + } + auto indexCatalog = collection->getIndexCatalog(); + IndexSpec index; + index.addKeys(keys); + index.unique(unique); + index.version(int(IndexDescriptor::kLatestIndexVersion)); + auto removeIndexBuildsToo = false; + auto indexSpecs = indexCatalog->removeExistingIndexes( + opCtx, + uassertStatusOK( + collection->addCollationDefaultsToIndexSpecsForCreate(opCtx, {index.toBSON()})), + removeIndexBuildsToo); + + if (indexSpecs.empty()) { + return Status::OK(); + } + + writeConflictRetry(opCtx, "ShardLocal::createIndexOnConfig", ns.ns(), [&] { + WriteUnitOfWork wunit(opCtx); + auto fromMigrate = true; + IndexBuildsCoordinator::get(opCtx)->createIndexesOnEmptyCollection( + opCtx, collection->uuid(), indexSpecs, fromMigrate); + wunit.commit(); + }); + } catch (const DBException& e) { + return e.toStatus(); + } + + return Status::OK(); +} + +void ShardLocal::runFireAndForgetCommand(OperationContext* opCtx, + const ReadPreferenceSetting& readPref, + const std::string& dbName, + const BSONObj& cmdObj) { + MONGO_UNREACHABLE; +} + +} // namespace mongo diff --git a/src/mongo/db/s/shard_local.h b/src/mongo/db/s/shard_local.h new file mode 100644 index 00000000000..2bce2dad081 --- /dev/null +++ b/src/mongo/db/s/shard_local.h @@ -0,0 +1,100 @@ +/** + * Copyright (C) 2018-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include "mongo/db/rs_local_client.h" +#include "mongo/s/client/shard.h" + +namespace mongo { + +class ShardLocal : public Shard { + ShardLocal(const ShardLocal&) = delete; + ShardLocal& operator=(const ShardLocal&) = delete; + +public: + explicit ShardLocal(const ShardId& id); + + ~ShardLocal() = default; + + /** + * These functions are implemented for the Shard interface's sake. They should not be called on + * ShardLocal because doing so triggers invariants. + */ + const ConnectionString getConnString() const override; + const ConnectionString originalConnString() const override; + std::shared_ptr getTargeter() const override; + void updateReplSetMonitor(const HostAndPort& remoteHost, + const Status& remoteCommandStatus) override; + + std::string toString() const override; + + bool isRetriableError(ErrorCodes::Error code, RetryPolicy options) final; + + Status createIndexOnConfig(OperationContext* opCtx, + const NamespaceString& ns, + const BSONObj& keys, + bool unique) override; + + void updateLastCommittedOpTime(LogicalTime lastCommittedOpTime) final; + + LogicalTime getLastCommittedOpTime() const final; + + void runFireAndForgetCommand(OperationContext* opCtx, + const ReadPreferenceSetting& readPref, + const std::string& dbName, + const BSONObj& cmdObj) override; + +private: + StatusWith _runCommand(OperationContext* opCtx, + const ReadPreferenceSetting& unused, + StringData dbName, + Milliseconds maxTimeMSOverrideUnused, + const BSONObj& cmdObj) final; + + StatusWith _runExhaustiveCursorCommand( + OperationContext* opCtx, + const ReadPreferenceSetting& readPref, + StringData dbName, + Milliseconds maxTimeMSOverride, + const BSONObj& cmdObj) final; + + StatusWith _exhaustiveFindOnConfig( + OperationContext* opCtx, + const ReadPreferenceSetting& readPref, + const repl::ReadConcernLevel& readConcernLevel, + const NamespaceString& nss, + const BSONObj& query, + const BSONObj& sort, + boost::optional limit) final; + + RSLocalClient _rsLocalClient; +}; + +} // namespace mongo diff --git a/src/mongo/db/s/shard_local_test.cpp b/src/mongo/db/s/shard_local_test.cpp new file mode 100644 index 00000000000..d7314c965cf --- /dev/null +++ b/src/mongo/db/s/shard_local_test.cpp @@ -0,0 +1,273 @@ +/** + * Copyright (C) 2018-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/platform/basic.h" + +#include "mongo/client/read_preference.h" +#include "mongo/db/catalog_raii.h" +#include "mongo/db/client.h" +#include "mongo/db/query/cursor_response.h" +#include "mongo/db/query/find_and_modify_request.h" +#include "mongo/db/repl/replication_coordinator.h" +#include "mongo/db/repl/replication_coordinator_mock.h" +#include "mongo/db/s/shard_local.h" +#include "mongo/db/service_context_d_test_fixture.h" +#include "mongo/db/write_concern_options.h" +#include "mongo/s/client/shard_registry.h" + +namespace mongo { +namespace { + +class ShardLocalTest : public ServiceContextMongoDTest { +protected: + ServiceContext::UniqueOperationContext _opCtx; + std::unique_ptr _shardLocal; + + /** + * Sets up and runs a FindAndModify command with ShardLocal's runCommand. Finds a document in + * namespace "nss" that matches "find" and updates the document with "set". Upsert and new are + * set to true in the FindAndModify request. + */ + StatusWith runFindAndModifyRunCommand(NamespaceString nss, + BSONObj find, + BSONObj set); + /** + * Facilitates running a find query by supplying the redundant parameters. Finds documents in + * namespace "nss" that match "query" and returns "limit" (if there are that many) number of + * documents in "sort" order. + */ + StatusWith runFindQuery(NamespaceString nss, + BSONObj query, + BSONObj sort, + boost::optional limit); + + /** + * Returns the index definitions that exist for the given collection. + */ + StatusWith> getIndexes(NamespaceString nss); + +private: + void setUp() override; + void tearDown() override; +}; + +void ShardLocalTest::setUp() { + ServiceContextMongoDTest::setUp(); + _opCtx = getGlobalServiceContext()->makeOperationContext(&cc()); + serverGlobalParams.clusterRole = ClusterRole::ConfigServer; + _shardLocal = std::make_unique(ShardRegistry::kConfigServerShardId); + const repl::ReplSettings replSettings = {}; + repl::ReplicationCoordinator::set( + getGlobalServiceContext(), + std::unique_ptr( + new repl::ReplicationCoordinatorMock(_opCtx->getServiceContext(), replSettings))); + ASSERT_OK(repl::ReplicationCoordinator::get(getGlobalServiceContext()) + ->setFollowerMode(repl::MemberState::RS_PRIMARY)); +} + +void ShardLocalTest::tearDown() { + _opCtx.reset(); + ServiceContextMongoDTest::tearDown(); + repl::ReplicationCoordinator::set(getGlobalServiceContext(), nullptr); +} + +StatusWith ShardLocalTest::runFindAndModifyRunCommand(NamespaceString nss, + BSONObj find, + BSONObj set) { + FindAndModifyRequest findAndModifyRequest = FindAndModifyRequest::makeUpdate(nss, find, set); + findAndModifyRequest.setUpsert(true); + findAndModifyRequest.setShouldReturnNew(true); + findAndModifyRequest.setWriteConcern(WriteConcernOptions( + WriteConcernOptions::kMajority, WriteConcernOptions::SyncMode::UNSET, Seconds(15))); + + return _shardLocal->runCommandWithFixedRetryAttempts( + _opCtx.get(), + ReadPreferenceSetting{ReadPreference::PrimaryOnly}, + nss.db().toString(), + findAndModifyRequest.toBSON({}), + Shard::RetryPolicy::kNoRetry); +} + +StatusWith> ShardLocalTest::getIndexes(NamespaceString nss) { + auto response = _shardLocal->runCommandWithFixedRetryAttempts( + _opCtx.get(), + ReadPreferenceSetting{ReadPreference::PrimaryOnly}, + nss.db().toString(), + BSON("listIndexes" << nss.coll().toString()), + Shard::RetryPolicy::kIdempotent); + if (!response.isOK()) { + return response.getStatus(); + } + if (!response.getValue().commandStatus.isOK()) { + return response.getValue().commandStatus; + } + + auto cursorResponse = CursorResponse::parseFromBSON(response.getValue().response); + if (!cursorResponse.isOK()) { + return cursorResponse.getStatus(); + } + return cursorResponse.getValue().getBatch(); +} + +/** + * Takes a FindAndModify command's BSON response and parses it for the returned "value" field. + */ +BSONObj extractFindAndModifyNewObj(const BSONObj& responseObj) { + const auto& newDocElem = responseObj["value"]; + ASSERT(!newDocElem.eoo()); + ASSERT(newDocElem.isABSONObj()); + return newDocElem.Obj(); +} + +StatusWith ShardLocalTest::runFindQuery(NamespaceString nss, + BSONObj query, + BSONObj sort, + boost::optional limit) { + return _shardLocal->exhaustiveFindOnConfig(_opCtx.get(), + ReadPreferenceSetting{ReadPreference::PrimaryOnly}, + repl::ReadConcernLevel::kMajorityReadConcern, + nss, + query, + sort, + limit); +} + +TEST_F(ShardLocalTest, RunCommand) { + NamespaceString nss("admin.bar"); + StatusWith findAndModifyResponse = runFindAndModifyRunCommand( + nss, BSON("fooItem" << 1), BSON("$set" << BSON("fooRandom" << 254))); + + Shard::CommandResponse commandResponse = unittest::assertGet(findAndModifyResponse); + BSONObj newDocument = extractFindAndModifyNewObj(commandResponse.response); + + ASSERT_EQUALS(1, newDocument["fooItem"].numberInt()); + ASSERT_EQUALS(254, newDocument["fooRandom"].numberInt()); +} + +TEST_F(ShardLocalTest, FindOneWithoutLimit) { + NamespaceString nss("admin.bar"); + + // Set up documents to be queried. + StatusWith findAndModifyResponse = runFindAndModifyRunCommand( + nss, BSON("fooItem" << 1), BSON("$set" << BSON("fooRandom" << 254))); + ASSERT_OK(findAndModifyResponse.getStatus()); + findAndModifyResponse = runFindAndModifyRunCommand( + nss, BSON("fooItem" << 3), BSON("$set" << BSON("fooRandom" << 452))); + ASSERT_OK(findAndModifyResponse.getStatus()); + + // Find a single document. + StatusWith response = + runFindQuery(nss, BSON("fooItem" << 3), BSONObj(), boost::none); + Shard::QueryResponse queryResponse = unittest::assertGet(response); + + std::vector docs = queryResponse.docs; + const unsigned long size = 1; + ASSERT_EQUALS(size, docs.size()); + BSONObj foundDoc = docs[0]; + ASSERT_EQUALS(3, foundDoc["fooItem"].numberInt()); + ASSERT_EQUALS(452, foundDoc["fooRandom"].numberInt()); +} + +TEST_F(ShardLocalTest, FindManyWithLimit) { + NamespaceString nss("admin.bar"); + + // Set up documents to be queried. + StatusWith findAndModifyResponse = runFindAndModifyRunCommand( + nss, BSON("fooItem" << 1), BSON("$set" << BSON("fooRandom" << 254))); + ASSERT_OK(findAndModifyResponse.getStatus()); + findAndModifyResponse = runFindAndModifyRunCommand( + nss, BSON("fooItem" << 2), BSON("$set" << BSON("fooRandom" << 444))); + ASSERT_OK(findAndModifyResponse.getStatus()); + findAndModifyResponse = runFindAndModifyRunCommand( + nss, BSON("fooItem" << 3), BSON("$set" << BSON("fooRandom" << 452))); + ASSERT_OK(findAndModifyResponse.getStatus()); + + // Find 2 of 3 documents. + StatusWith response = + runFindQuery(nss, BSONObj(), BSON("fooItem" << 1), 2LL); + Shard::QueryResponse queryResponse = unittest::assertGet(response); + + std::vector docs = queryResponse.docs; + const unsigned long size = 2; + ASSERT_EQUALS(size, docs.size()); + BSONObj firstDoc = docs[0]; + ASSERT_EQUALS(1, firstDoc["fooItem"].numberInt()); + ASSERT_EQUALS(254, firstDoc["fooRandom"].numberInt()); + BSONObj secondDoc = docs[1]; + ASSERT_EQUALS(2, secondDoc["fooItem"].numberInt()); + ASSERT_EQUALS(444, secondDoc["fooRandom"].numberInt()); +} + +TEST_F(ShardLocalTest, FindNoMatchingDocumentsEmpty) { + NamespaceString nss("admin.bar"); + + // Set up a document. + StatusWith findAndModifyResponse = runFindAndModifyRunCommand( + nss, BSON("fooItem" << 1), BSON("$set" << BSON("fooRandom" << 254))); + ASSERT_OK(findAndModifyResponse.getStatus()); + + // Run a query that won't find any results. + StatusWith response = + runFindQuery(nss, BSON("fooItem" << 3), BSONObj(), boost::none); + Shard::QueryResponse queryResponse = unittest::assertGet(response); + + std::vector docs = queryResponse.docs; + const unsigned long size = 0; + ASSERT_EQUALS(size, docs.size()); +} + +TEST_F(ShardLocalTest, CreateIndex) { + NamespaceString nss("config.foo"); + + ASSERT_EQUALS(ErrorCodes::NamespaceNotFound, getIndexes(nss).getStatus()); + + Status status = + _shardLocal->createIndexOnConfig(_opCtx.get(), nss, BSON("a" << 1 << "b" << 1), true); + // Creating the index should implicitly create the collection + ASSERT_OK(status); + + auto indexes = unittest::assertGet(getIndexes(nss)); + // There should be the index we just added as well as the _id index + ASSERT_EQ(2U, indexes.size()); + + // Making an identical index should be a no-op. + status = _shardLocal->createIndexOnConfig(_opCtx.get(), nss, BSON("a" << 1 << "b" << 1), true); + ASSERT_OK(status); + indexes = unittest::assertGet(getIndexes(nss)); + ASSERT_EQ(2U, indexes.size()); + + // Trying to make the same index as non-unique should fail. + status = _shardLocal->createIndexOnConfig(_opCtx.get(), nss, BSON("a" << 1 << "b" << 1), false); + ASSERT_EQUALS(ErrorCodes::IndexOptionsConflict, status); + indexes = unittest::assertGet(getIndexes(nss)); + ASSERT_EQ(2U, indexes.size()); +} + +} // namespace +} // namespace mongo diff --git a/src/mongo/db/s/shard_metadata_util_test.cpp b/src/mongo/db/s/shard_metadata_util_test.cpp index 771dbab13ec..0b963169f8d 100644 --- a/src/mongo/db/s/shard_metadata_util_test.cpp +++ b/src/mongo/db/s/shard_metadata_util_test.cpp @@ -35,11 +35,10 @@ #include "mongo/client/remote_command_targeter_mock.h" #include "mongo/db/commands.h" #include "mongo/db/dbdirectclient.h" +#include "mongo/db/s/shard_server_test_fixture.h" #include "mongo/rpc/get_status_from_command_result.h" #include "mongo/s/catalog/type_chunk.h" #include "mongo/s/catalog/type_shard_collection.h" -#include "mongo/s/shard_server_test_fixture.h" -#include "mongo/unittest/unittest.h" namespace mongo { namespace { diff --git a/src/mongo/db/s/shard_server_catalog_cache_loader_test.cpp b/src/mongo/db/s/shard_server_catalog_cache_loader_test.cpp index 902b09c3a64..5cfc7ce4124 100644 --- a/src/mongo/db/s/shard_server_catalog_cache_loader_test.cpp +++ b/src/mongo/db/s/shard_server_catalog_cache_loader_test.cpp @@ -29,13 +29,11 @@ #include "mongo/platform/basic.h" -#include "mongo/db/s/shard_server_catalog_cache_loader.h" - #include "mongo/db/s/catalog_cache_loader_mock.h" +#include "mongo/db/s/shard_server_catalog_cache_loader.h" +#include "mongo/db/s/shard_server_test_fixture.h" #include "mongo/s/catalog/type_chunk.h" #include "mongo/s/catalog/type_collection.h" -#include "mongo/s/shard_server_test_fixture.h" -#include "mongo/unittest/unittest.h" namespace mongo { namespace { diff --git a/src/mongo/db/s/shard_server_test_fixture.cpp b/src/mongo/db/s/shard_server_test_fixture.cpp new file mode 100644 index 00000000000..b916c674792 --- /dev/null +++ b/src/mongo/db/s/shard_server_test_fixture.cpp @@ -0,0 +1,102 @@ +/** + * Copyright (C) 2018-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/platform/basic.h" + +#include "mongo/db/s/shard_server_test_fixture.h" + +#include "mongo/client/remote_command_targeter_mock.h" +#include "mongo/db/commands.h" +#include "mongo/db/repl/replication_coordinator_mock.h" +#include "mongo/db/s/shard_server_catalog_cache_loader.h" +#include "mongo/db/s/sharding_state.h" +#include "mongo/s/catalog/dist_lock_catalog_mock.h" +#include "mongo/s/catalog/dist_lock_manager_mock.h" +#include "mongo/s/catalog/sharding_catalog_client_impl.h" +#include "mongo/s/catalog_cache.h" +#include "mongo/s/config_server_catalog_cache_loader.h" + +namespace mongo { + +const HostAndPort ShardServerTestFixture::kConfigHostAndPort("dummy", 123); + +ShardServerTestFixture::ShardServerTestFixture() = default; + +ShardServerTestFixture::~ShardServerTestFixture() = default; + +std::shared_ptr ShardServerTestFixture::configTargeterMock() { + return RemoteCommandTargeterMock::get(shardRegistry()->getConfigShard()->getTargeter()); +} + +void ShardServerTestFixture::setUp() { + ShardingMongodTestFixture::setUp(); + + replicationCoordinator()->alwaysAllowWrites(true); + + // Initialize sharding components as a shard server. + serverGlobalParams.clusterRole = ClusterRole::ShardServer; + + _clusterId = OID::gen(); + ShardingState::get(getServiceContext())->setInitialized(_myShardName, _clusterId); + + CatalogCacheLoader::set(getServiceContext(), + std::make_unique( + std::make_unique())); + + uassertStatusOK( + initializeGlobalShardingStateForMongodForTest(ConnectionString(kConfigHostAndPort))); + + // Set the findHost() return value on the mock targeter so that later calls to the + // config targeter's findHost() return the appropriate value. + configTargeterMock()->setFindHostReturnValue(kConfigHostAndPort); +} + +void ShardServerTestFixture::tearDown() { + CatalogCacheLoader::clearForTests(getServiceContext()); + + ShardingMongodTestFixture::tearDown(); +} + +std::unique_ptr ShardServerTestFixture::makeDistLockCatalog() { + return std::make_unique(); +} + +std::unique_ptr ShardServerTestFixture::makeDistLockManager( + std::unique_ptr distLockCatalog) { + invariant(distLockCatalog); + return std::make_unique(std::move(distLockCatalog)); +} + +std::unique_ptr ShardServerTestFixture::makeShardingCatalogClient( + std::unique_ptr distLockManager) { + invariant(distLockManager); + return std::make_unique(std::move(distLockManager)); +} + +} // namespace mongo diff --git a/src/mongo/db/s/shard_server_test_fixture.h b/src/mongo/db/s/shard_server_test_fixture.h new file mode 100644 index 00000000000..deee682e174 --- /dev/null +++ b/src/mongo/db/s/shard_server_test_fixture.h @@ -0,0 +1,83 @@ +/** + * Copyright (C) 2018-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include "mongo/db/s/sharding_mongod_test_fixture.h" + +namespace mongo { + +/** + * Test fixture for shard components, as opposed to config or mongos components. Provides a mock + * network and ephemeral storage engine via ShardingMongodTestFixture. Additionally sets up mock + * dist lock catalog and manager with a real catalog client. + */ +class ShardServerTestFixture : public ShardingMongodTestFixture { +protected: + ShardServerTestFixture(); + ~ShardServerTestFixture(); + + void setUp() override; + void tearDown() override; + + /** + * Returns the mock targeter for the config server. Useful to use like so, + * + * configTargeterMock()->setFindHostReturnValue(HostAndPort); + * configTargeterMock()->setFindHostReturnValue({ErrorCodes::InternalError, "can't target"}) + * + * Remote calls always need to resolve a host with RemoteCommandTargeterMock::findHost, so it + * must be set. + */ + std::shared_ptr configTargeterMock(); + + /** + * Creates a DistLockCatalogMock. + */ + std::unique_ptr makeDistLockCatalog() override; + + /** + * Creates a DistLockManagerMock. + */ + std::unique_ptr makeDistLockManager( + std::unique_ptr distLockCatalog) override; + + /** + * Creates a real ShardingCatalogClient. + */ + std::unique_ptr makeShardingCatalogClient( + std::unique_ptr distLockManager) override; + + static const HostAndPort kConfigHostAndPort; + + const ShardId _myShardName{"myShardName"}; + OID _clusterId; +}; + +} // namespace mongo diff --git a/src/mongo/db/s/sharding_initialization_mongod.cpp b/src/mongo/db/s/sharding_initialization_mongod.cpp index accb9c65406..0ab2d96fab7 100644 --- a/src/mongo/db/s/sharding_initialization_mongod.cpp +++ b/src/mongo/db/s/sharding_initialization_mongod.cpp @@ -50,6 +50,7 @@ #include "mongo/db/s/chunk_splitter.h" #include "mongo/db/s/periodic_balancer_config_refresher.h" #include "mongo/db/s/read_only_catalog_cache_loader.h" +#include "mongo/db/s/shard_local.h" #include "mongo/db/s/shard_server_catalog_cache_loader.h" #include "mongo/db/s/sharding_config_optime_gossip.h" #include "mongo/db/s/transaction_coordinator_service.h" @@ -59,7 +60,6 @@ #include "mongo/rpc/metadata/egress_metadata_hook_list.h" #include "mongo/s/catalog_cache.h" #include "mongo/s/client/shard_factory.h" -#include "mongo/s/client/shard_local.h" #include "mongo/s/client/shard_remote.h" #include "mongo/s/client/sharding_connection_hook.h" #include "mongo/s/config_server_catalog_cache_loader.h" diff --git a/src/mongo/db/s/sharding_initialization_mongod_test.cpp b/src/mongo/db/s/sharding_initialization_mongod_test.cpp index 6af6ff00399..a724a245f84 100644 --- a/src/mongo/db/s/sharding_initialization_mongod_test.cpp +++ b/src/mongo/db/s/sharding_initialization_mongod_test.cpp @@ -40,6 +40,7 @@ #include "mongo/db/s/op_observer_sharding_impl.h" #include "mongo/db/s/shard_server_catalog_cache_loader.h" #include "mongo/db/s/shard_server_op_observer.h" +#include "mongo/db/s/shard_server_test_fixture.h" #include "mongo/db/s/sharding_initialization_mongod.h" #include "mongo/db/s/type_shard_identity.h" #include "mongo/db/server_options.h" @@ -47,7 +48,6 @@ #include "mongo/s/catalog/sharding_catalog_client_impl.h" #include "mongo/s/client/shard_registry.h" #include "mongo/s/config_server_catalog_cache_loader.h" -#include "mongo/s/shard_server_test_fixture.h" namespace mongo { namespace { diff --git a/src/mongo/db/s/sharding_initialization_op_observer_test.cpp b/src/mongo/db/s/sharding_initialization_op_observer_test.cpp index 6cd2ccd2605..563a6325454 100644 --- a/src/mongo/db/s/sharding_initialization_op_observer_test.cpp +++ b/src/mongo/db/s/sharding_initialization_op_observer_test.cpp @@ -40,12 +40,12 @@ #include "mongo/db/s/shard_server_catalog_cache_loader.h" #include "mongo/db/s/shard_server_op_observer.h" #include "mongo/db/s/sharding_initialization_mongod.h" +#include "mongo/db/s/sharding_mongod_test_fixture.h" #include "mongo/db/s/type_shard_identity.h" #include "mongo/db/server_options.h" #include "mongo/s/catalog/dist_lock_manager_mock.h" #include "mongo/s/client/shard_registry.h" #include "mongo/s/config_server_catalog_cache_loader.h" -#include "mongo/s/sharding_mongod_test_fixture.h" namespace mongo { namespace { diff --git a/src/mongo/db/s/sharding_logging_test.cpp b/src/mongo/db/s/sharding_logging_test.cpp index a014091b49b..a5622798baa 100644 --- a/src/mongo/db/s/sharding_logging_test.cpp +++ b/src/mongo/db/s/sharding_logging_test.cpp @@ -33,14 +33,12 @@ #include -#include "mongo/client/remote_command_targeter_mock.h" #include "mongo/db/commands.h" +#include "mongo/db/s/shard_server_test_fixture.h" #include "mongo/db/s/sharding_logging.h" -#include "mongo/executor/network_interface_mock.h" #include "mongo/executor/task_executor.h" #include "mongo/s/catalog/sharding_catalog_client.h" #include "mongo/s/client/shard_registry.h" -#include "mongo/s/sharding_router_test_fixture.h" #include "mongo/stdx/chrono.h" #include "mongo/stdx/future.h" #include "mongo/util/str.h" @@ -49,34 +47,24 @@ namespace mongo { namespace { -using executor::NetworkInterfaceMock; -using executor::TaskExecutor; -using stdx::async; using unittest::assertGet; -const HostAndPort configHost{"TestHost1"}; - -class InfoLoggingTest : public ShardingTestFixture { +class InfoLoggingTest : public ShardServerTestFixture { public: enum CollType { ActionLog, ChangeLog }; InfoLoggingTest(CollType configCollType, int cappedSize) : _configCollType(configCollType), _cappedSize(cappedSize) {} - void setUp() override { - ShardingTestFixture::setUp(); - - configTargeter()->setFindHostReturnValue(configHost); - } - protected: void noRetryAfterSuccessfulCreate() { auto future = launchAsync([this] { log("moved a chunk", "foo.bar", BSON("min" << 3 << "max" << 4)).transitional_ignore(); }); - expectConfigCollectionCreate(configHost, getConfigCollName(), _cappedSize, BSON("ok" << 1)); - expectConfigCollectionInsert(configHost, + expectConfigCollectionCreate( + kConfigHostAndPort, getConfigCollName(), _cappedSize, BSON("ok" << 1)); + expectConfigCollectionInsert(kConfigHostAndPort, getConfigCollName(), network()->now(), "moved a chunk", @@ -92,7 +80,7 @@ protected: .transitional_ignore(); }); - expectConfigCollectionInsert(configHost, + expectConfigCollectionInsert(kConfigHostAndPort, getConfigCollName(), network()->now(), "moved a second chunk", @@ -112,8 +100,8 @@ protected: CommandHelpers::appendCommandStatusNoThrow( createResponseBuilder, Status(ErrorCodes::NamespaceExists, "coll already exists")); expectConfigCollectionCreate( - configHost, getConfigCollName(), _cappedSize, createResponseBuilder.obj()); - expectConfigCollectionInsert(configHost, + kConfigHostAndPort, getConfigCollName(), _cappedSize, createResponseBuilder.obj()); + expectConfigCollectionInsert(kConfigHostAndPort, getConfigCollName(), network()->now(), "moved a chunk", @@ -129,7 +117,7 @@ protected: .transitional_ignore(); }); - expectConfigCollectionInsert(configHost, + expectConfigCollectionInsert(kConfigHostAndPort, getConfigCollName(), network()->now(), "moved a second chunk", @@ -149,7 +137,7 @@ protected: CommandHelpers::appendCommandStatusNoThrow( createResponseBuilder, Status(ErrorCodes::Interrupted, "operation interrupted")); expectConfigCollectionCreate( - configHost, getConfigCollName(), _cappedSize, createResponseBuilder.obj()); + kConfigHostAndPort, getConfigCollName(), _cappedSize, createResponseBuilder.obj()); // Now wait for the logAction call to return future.default_timed_get(); @@ -160,8 +148,9 @@ protected: .transitional_ignore(); }); - expectConfigCollectionCreate(configHost, getConfigCollName(), _cappedSize, BSON("ok" << 1)); - expectConfigCollectionInsert(configHost, + expectConfigCollectionCreate( + kConfigHostAndPort, getConfigCollName(), _cappedSize, BSON("ok" << 1)); + expectConfigCollectionInsert(kConfigHostAndPort, getConfigCollName(), network()->now(), "moved a second chunk", diff --git a/src/mongo/db/s/sharding_mongod_test_fixture.cpp b/src/mongo/db/s/sharding_mongod_test_fixture.cpp new file mode 100644 index 00000000000..932b2ff5424 --- /dev/null +++ b/src/mongo/db/s/sharding_mongod_test_fixture.cpp @@ -0,0 +1,337 @@ +/** + * Copyright (C) 2018-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/platform/basic.h" + +#include "mongo/db/s/sharding_mongod_test_fixture.h" + +#include +#include +#include + +#include "mongo/base/checked_cast.h" +#include "mongo/client/replica_set_monitor.h" +#include "mongo/db/catalog_raii.h" +#include "mongo/db/client.h" +#include "mongo/db/commands.h" +#include "mongo/db/namespace_string.h" +#include "mongo/db/op_observer_registry.h" +#include "mongo/db/query/cursor_response.h" +#include "mongo/db/query/query_request.h" +#include "mongo/db/repl/drop_pending_collection_reaper.h" +#include "mongo/db/repl/oplog.h" +#include "mongo/db/repl/read_concern_args.h" +#include "mongo/db/repl/repl_settings.h" +#include "mongo/db/repl/replication_consistency_markers_mock.h" +#include "mongo/db/repl/replication_process.h" +#include "mongo/db/repl/replication_recovery_mock.h" +#include "mongo/db/repl/storage_interface_mock.h" +#include "mongo/db/s/collection_sharding_state_factory_shard.h" +#include "mongo/db/s/config_server_op_observer.h" +#include "mongo/db/s/op_observer_sharding_impl.h" +#include "mongo/db/s/shard_local.h" +#include "mongo/db/s/shard_server_op_observer.h" +#include "mongo/executor/task_executor_pool.h" +#include "mongo/executor/thread_pool_task_executor_test_fixture.h" +#include "mongo/rpc/metadata/repl_set_metadata.h" +#include "mongo/s/balancer_configuration.h" +#include "mongo/s/catalog/dist_lock_catalog.h" +#include "mongo/s/catalog/dist_lock_manager.h" +#include "mongo/s/catalog/sharding_catalog_client.h" +#include "mongo/s/catalog/type_collection.h" +#include "mongo/s/catalog/type_shard.h" +#include "mongo/s/catalog_cache.h" +#include "mongo/s/catalog_cache_loader.h" +#include "mongo/s/client/shard_factory.h" +#include "mongo/s/client/shard_registry.h" +#include "mongo/s/client/shard_remote.h" +#include "mongo/s/grid.h" +#include "mongo/s/query/cluster_cursor_manager.h" +#include "mongo/s/request_types/set_shard_version_request.h" +#include "mongo/util/clock_source_mock.h" +#include "mongo/util/tick_source_mock.h" + +namespace mongo { + +using executor::NetworkInterfaceMock; +using executor::NetworkTestEnv; +using executor::RemoteCommandRequest; +using executor::RemoteCommandResponse; +using repl::ReplicationCoordinator; +using repl::ReplicationCoordinatorMock; +using repl::ReplSettings; +using unittest::assertGet; + +ShardingMongodTestFixture::ShardingMongodTestFixture() { + const auto service = getServiceContext(); + + // Set up this node as shard node, which is part of a replica set + + repl::ReplSettings replSettings; + replSettings.setOplogSizeBytes(512'000); + replSettings.setReplSetString(ConnectionString::forReplicaSet(_setName, _servers).toString()); + auto replCoordPtr = makeReplicationCoordinator(replSettings); + _replCoord = replCoordPtr.get(); + + BSONArrayBuilder serversBob; + for (size_t i = 0; i < _servers.size(); ++i) { + serversBob.append(BSON("host" << _servers[i].toString() << "_id" << static_cast(i))); + } + + auto replSetConfig = + repl::ReplSetConfig::parse(BSON("_id" << _setName << "protocolVersion" << 1 << "version" + << 3 << "members" << serversBob.arr())); + replCoordPtr->setGetConfigReturnValue(replSetConfig); + + repl::ReplicationCoordinator::set(service, std::move(replCoordPtr)); + + auto storagePtr = std::make_unique(); + + repl::DropPendingCollectionReaper::set( + service, std::make_unique(storagePtr.get())); + + repl::ReplicationProcess::set(service, + std::make_unique( + storagePtr.get(), + std::make_unique(), + std::make_unique())); + + ASSERT_OK(repl::ReplicationProcess::get(operationContext()) + ->initializeRollbackID(operationContext())); + + repl::StorageInterface::set(service, std::move(storagePtr)); + + auto opObserver = checked_cast(service->getOpObserver()); + opObserver->addObserver(std::make_unique()); + opObserver->addObserver(std::make_unique()); + opObserver->addObserver(std::make_unique()); + + repl::setOplogCollectionName(service); + repl::createOplog(operationContext()); + + // Set the highest FCV because otherwise it defaults to the lower FCV. This way we default to + // testing this release's code, not backwards compatibility code. + serverGlobalParams.featureCompatibility.setVersion( + ServerGlobalParams::FeatureCompatibility::Version::kFullyUpgradedTo46); +} + +ShardingMongodTestFixture::~ShardingMongodTestFixture() = default; + +std::unique_ptr ShardingMongodTestFixture::makeReplicationCoordinator( + ReplSettings replSettings) { + auto coordinator = + std::make_unique(getServiceContext(), replSettings); + ASSERT_OK(coordinator->setFollowerMode(repl::MemberState::RS_PRIMARY)); + return coordinator; +} + +std::unique_ptr ShardingMongodTestFixture::_makeTaskExecutorPool() { + // Set up a NetworkInterfaceMock. Note, unlike NetworkInterfaceASIO, which has its own pool of + // threads, tasks in the NetworkInterfaceMock must be carried out synchronously by the (single) + // thread the unit test is running on. + auto netForFixedTaskExecutor = std::make_unique(); + _mockNetwork = netForFixedTaskExecutor.get(); + + // Set up a ThreadPoolTaskExecutor. Note, for local tasks this TaskExecutor uses a + // ThreadPoolMock, and for remote tasks it uses the NetworkInterfaceMock created above. However, + // note that the ThreadPoolMock uses the NetworkInterfaceMock's threads to run tasks, which is + // again just the (single) thread the unit test is running on. Therefore, all tasks, local and + // remote, must be carried out synchronously by the test thread. + auto fixedTaskExecutor = makeSharedThreadPoolTestExecutor(std::move(netForFixedTaskExecutor)); + _networkTestEnv = std::make_unique(fixedTaskExecutor.get(), _mockNetwork); + + // Set up (one) TaskExecutor for the set of arbitrary TaskExecutors. + std::vector> arbitraryExecutorsForExecutorPool; + arbitraryExecutorsForExecutorPool.emplace_back( + makeSharedThreadPoolTestExecutor(std::make_unique())); + + // Set up the TaskExecutorPool with the fixed TaskExecutor and set of arbitrary TaskExecutors. + auto executorPool = std::make_unique(); + executorPool->addExecutors(std::move(arbitraryExecutorsForExecutorPool), + std::move(fixedTaskExecutor)); + + return executorPool; +} + +std::unique_ptr ShardingMongodTestFixture::makeShardRegistry( + ConnectionString configConnStr) { + auto targeterFactory(std::make_unique()); + auto targeterFactoryPtr = targeterFactory.get(); + _targeterFactory = targeterFactoryPtr; + + ShardFactory::BuilderCallable setBuilder = [targeterFactoryPtr]( + const ShardId& shardId, + const ConnectionString& connStr) { + return std::make_unique(shardId, connStr, targeterFactoryPtr->create(connStr)); + }; + + ShardFactory::BuilderCallable masterBuilder = [targeterFactoryPtr]( + const ShardId& shardId, + const ConnectionString& connStr) { + return std::make_unique(shardId, connStr, targeterFactoryPtr->create(connStr)); + }; + + ShardFactory::BuildersMap buildersMap{{ConnectionString::SET, std::move(setBuilder)}, + {ConnectionString::MASTER, std::move(masterBuilder)}}; + + // Only config servers use ShardLocal for now. + if (serverGlobalParams.clusterRole == ClusterRole::ConfigServer) { + ShardFactory::BuilderCallable localBuilder = [](const ShardId& shardId, + const ConnectionString& connStr) { + return std::make_unique(shardId); + }; + buildersMap.insert( + std::pair( + ConnectionString::LOCAL, std::move(localBuilder))); + } + + auto shardFactory = + std::make_unique(std::move(buildersMap), std::move(targeterFactory)); + + return std::make_unique(std::move(shardFactory), configConnStr); +} + +std::unique_ptr ShardingMongodTestFixture::makeDistLockCatalog() { + return nullptr; +} + +std::unique_ptr ShardingMongodTestFixture::makeDistLockManager( + std::unique_ptr distLockCatalog) { + return nullptr; +} + +std::unique_ptr ShardingMongodTestFixture::makeClusterCursorManager() { + return nullptr; +} + +std::unique_ptr ShardingMongodTestFixture::makeBalancerConfiguration() { + return std::make_unique(); +} + +Status ShardingMongodTestFixture::initializeGlobalShardingStateForMongodForTest( + const ConnectionString& configConnStr) { + invariant(serverGlobalParams.clusterRole == ClusterRole::ShardServer || + serverGlobalParams.clusterRole == ClusterRole::ConfigServer); + + // Create and initialize each sharding component individually before moving them to the Grid + // in order to control the order of initialization, since some components depend on others. + + auto executorPoolPtr = _makeTaskExecutorPool(); + if (executorPoolPtr) { + executorPoolPtr->startup(); + } + + auto distLockCatalogPtr = makeDistLockCatalog(); + _distLockCatalog = distLockCatalogPtr.get(); + + auto distLockManagerPtr = makeDistLockManager(std::move(distLockCatalogPtr)); + _distLockManager = distLockManagerPtr.get(); + + auto const grid = Grid::get(operationContext()); + grid->init(makeShardingCatalogClient(std::move(distLockManagerPtr)), + std::make_unique(CatalogCacheLoader::get(getServiceContext())), + makeShardRegistry(configConnStr), + makeClusterCursorManager(), + makeBalancerConfiguration(), + std::move(executorPoolPtr), + _mockNetwork); + + // NOTE: ShardRegistry::startup() is not called because it starts a task executor with a + // self-rescheduling task to reload the ShardRegistry over the network. + // grid->shardRegistry()->startup(); + + if (grid->catalogClient()) { + grid->catalogClient()->startup(); + } + + return Status::OK(); +} + +void ShardingMongodTestFixture::tearDown() { + ReplicaSetMonitor::cleanup(); + + if (Grid::get(operationContext())->getExecutorPool() && !_executorPoolShutDown) { + Grid::get(operationContext())->getExecutorPool()->shutdownAndJoin(); + } + + if (Grid::get(operationContext())->catalogClient()) { + Grid::get(operationContext())->catalogClient()->shutDown(operationContext()); + } + + if (Grid::get(operationContext())->shardRegistry()) { + Grid::get(operationContext())->shardRegistry()->shutdown(); + } + + Grid::get(operationContext())->clearForUnitTests(); + + ServiceContextMongoDTest::tearDown(); +} + +ShardingCatalogClient* ShardingMongodTestFixture::catalogClient() const { + invariant(Grid::get(operationContext())->catalogClient()); + return Grid::get(operationContext())->catalogClient(); +} + +CatalogCache* ShardingMongodTestFixture::catalogCache() const { + invariant(Grid::get(operationContext())->catalogCache()); + return Grid::get(operationContext())->catalogCache(); +} + +ShardRegistry* ShardingMongodTestFixture::shardRegistry() const { + invariant(Grid::get(operationContext())->shardRegistry()); + return Grid::get(operationContext())->shardRegistry(); +} + +ClusterCursorManager* ShardingMongodTestFixture::clusterCursorManager() const { + invariant(Grid::get(operationContext())->getCursorManager()); + return Grid::get(operationContext())->getCursorManager(); +} + +executor::TaskExecutorPool* ShardingMongodTestFixture::executorPool() const { + invariant(Grid::get(operationContext())->getExecutorPool()); + return Grid::get(operationContext())->getExecutorPool(); +} + +void ShardingMongodTestFixture::shutdownExecutorPool() { + invariant(!_executorPoolShutDown); + executorPool()->shutdownAndJoin(); + _executorPoolShutDown = true; +} + +std::shared_ptr ShardingMongodTestFixture::executor() const { + invariant(Grid::get(operationContext())->getExecutorPool()); + return Grid::get(operationContext())->getExecutorPool()->getFixedExecutor(); +} + +repl::ReplicationCoordinatorMock* ShardingMongodTestFixture::replicationCoordinator() const { + invariant(_replCoord); + return _replCoord; +} + +} // namespace mongo diff --git a/src/mongo/db/s/sharding_mongod_test_fixture.h b/src/mongo/db/s/sharding_mongod_test_fixture.h new file mode 100644 index 00000000000..6495ee3377a --- /dev/null +++ b/src/mongo/db/s/sharding_mongod_test_fixture.h @@ -0,0 +1,152 @@ +/** + * Copyright (C) 2018-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +#include "mongo/db/repl/replication_coordinator_mock.h" +#include "mongo/db/service_context_d_test_fixture.h" +#include "mongo/s/sharding_test_fixture_common.h" + +namespace mongo { + +class CatalogCacheLoader; + +namespace repl { +class ReplSettings; +} // namespace repl + +/** + * Sets up this fixture as a mongod with a storage engine, OpObserver, and as a member of a replica + * set. + * + * Additionally, provides an interface for initializing sharding components, mimicking the process + * by which a real config or shard server does sharding initialization. Provides a set of default + * components (including a NetworkInterface/TaskExecutor subsystem backed by the NetworkTestEnv), + * but allows subclasses to replace any component with its real implementation, a mock, or nullptr. + */ +class ShardingMongodTestFixture : public ServiceContextMongoDTest, + public ShardingTestFixtureCommon { +protected: + ShardingMongodTestFixture(); + ~ShardingMongodTestFixture(); + + void tearDown() override; + + /** + * Initializes sharding components according to the cluster role in + * serverGlobalParams.clusterRole and puts the components on the Grid, mimicking the + * initialization done by an actual config or shard mongod server. + * + * It is illegal to call this if serverGlobalParams.clusterRole is not ClusterRole::ShardServer + * or ClusterRole::ConfigServer. + */ + Status initializeGlobalShardingStateForMongodForTest(const ConnectionString& configConnStr); + + // Syntactic sugar for getting sharding components off the Grid, if they have been initialized. + + ShardingCatalogClient* catalogClient() const; + CatalogCache* catalogCache() const; + ShardRegistry* shardRegistry() const; + std::shared_ptr executor() const; + ClusterCursorManager* clusterCursorManager() const; + executor::TaskExecutorPool* executorPool() const; + + /** + * Shuts down the TaskExecutorPool and remembers that it has been shut down, so that it is not + * shut down again on tearDown. + * + * Not safe to call from multiple threads. + */ + void shutdownExecutorPool(); + + repl::ReplicationCoordinatorMock* replicationCoordinator() const; + + // Methods for creating and returning sharding components. Some of these methods have been + // implemented to return the real implementation of the component as the default, while others + // return a mock or nullptr. Subclasses can override any of these methods to create and + // initialize a real implementation, a mock, or nullptr, as needed. + + // Warning: If a component takes ownership of another component for which a real or mock is + // being used, the component must also be real or mock implementation, so that it can actually + // take the ownership. + + /** + * Base class returns ReplicationCoordinatorMock. + */ + virtual std::unique_ptr makeReplicationCoordinator( + repl::ReplSettings replSettings); + + /** + * Base class returns a real implementation of ShardRegistry. + */ + virtual std::unique_ptr makeShardRegistry(ConnectionString configConnStr); + + /** + * Base class returns nullptr. + */ + virtual std::unique_ptr makeDistLockCatalog(); + + /** + * Base class returns nullptr. + * + * Note: DistLockManager takes ownership of the DistLockCatalog, so if DistLockCatalog is not + * nullptr, a real or mock DistLockManager must be supplied. + */ + virtual std::unique_ptr makeDistLockManager( + std::unique_ptr distLockCatalog); + + /** + * Base class returns nullptr. + */ + virtual std::unique_ptr makeClusterCursorManager(); + + /** + * Base class returns nullptr. + */ + virtual std::unique_ptr makeBalancerConfiguration(); + +private: + /** + * Base class returns a TaskExecutorPool with a fixed TaskExecutor and a set of arbitrary + * executors containing one TaskExecutor, each backed by a NetworkInterfaceMock/ThreadPoolMock + * subsytem. + */ + std::unique_ptr _makeTaskExecutorPool(); + + const std::string _setName = "mySet"; + const std::vector _servers{ + HostAndPort("node1:12345"), HostAndPort("node2:12345"), HostAndPort("node3:12345")}; + + repl::ReplicationCoordinatorMock* _replCoord = nullptr; + + // Records if a component has been shut down, so that it is only shut down once. + bool _executorPoolShutDown = false; +}; + +} // namespace mongo diff --git a/src/mongo/db/s/split_chunk_test.cpp b/src/mongo/db/s/split_chunk_test.cpp index cfdc61f2b61..532ff9dab55 100644 --- a/src/mongo/db/s/split_chunk_test.cpp +++ b/src/mongo/db/s/split_chunk_test.cpp @@ -33,12 +33,11 @@ #include -#include "mongo/client/remote_command_targeter_mock.h" #include "mongo/db/json.h" +#include "mongo/db/s/shard_server_test_fixture.h" #include "mongo/db/s/sharding_initialization_mongod.h" #include "mongo/db/s/split_chunk.h" #include "mongo/db/server_options.h" -#include "mongo/executor/network_interface_mock.h" #include "mongo/executor/remote_command_request.h" #include "mongo/executor/remote_command_response.h" #include "mongo/executor/task_executor.h" @@ -51,10 +50,8 @@ #include "mongo/s/catalog_cache_loader.h" #include "mongo/s/client/shard_registry.h" #include "mongo/s/grid.h" -#include "mongo/s/shard_server_test_fixture.h" #include "mongo/s/write_ops/batched_command_request.h" #include "mongo/s/write_ops/batched_command_response.h" -#include "mongo/unittest/unittest.h" namespace mongo { namespace { diff --git a/src/mongo/db/s/split_vector_test.cpp b/src/mongo/db/s/split_vector_test.cpp index 7350527d8b3..807bb35d4aa 100644 --- a/src/mongo/db/s/split_vector_test.cpp +++ b/src/mongo/db/s/split_vector_test.cpp @@ -34,9 +34,8 @@ #include "mongo/db/db_raii.h" #include "mongo/db/dbdirectclient.h" #include "mongo/db/s/collection_sharding_runtime.h" +#include "mongo/db/s/shard_server_test_fixture.h" #include "mongo/db/s/split_vector.h" -#include "mongo/s/shard_server_test_fixture.h" -#include "mongo/unittest/unittest.h" namespace mongo { namespace { diff --git a/src/mongo/db/s/transaction_coordinator_futures_util_test.cpp b/src/mongo/db/s/transaction_coordinator_futures_util_test.cpp index fb145b325aa..e46ab94599f 100644 --- a/src/mongo/db/s/transaction_coordinator_futures_util_test.cpp +++ b/src/mongo/db/s/transaction_coordinator_futures_util_test.cpp @@ -30,12 +30,11 @@ #include "mongo/platform/basic.h" #include "mongo/client/remote_command_targeter_mock.h" +#include "mongo/db/s/shard_server_test_fixture.h" #include "mongo/db/s/transaction_coordinator_futures_util.h" #include "mongo/s/catalog/sharding_catalog_client_mock.h" #include "mongo/s/catalog/type_shard.h" -#include "mongo/s/shard_server_test_fixture.h" #include "mongo/unittest/barrier.h" -#include "mongo/unittest/unittest.h" namespace mongo { namespace txn { diff --git a/src/mongo/db/s/transaction_coordinator_test_fixture.h b/src/mongo/db/s/transaction_coordinator_test_fixture.h index 03763d3e2a1..37232f0d340 100644 --- a/src/mongo/db/s/transaction_coordinator_test_fixture.h +++ b/src/mongo/db/s/transaction_coordinator_test_fixture.h @@ -33,9 +33,9 @@ #include #include "mongo/base/status.h" +#include "mongo/db/s/shard_server_test_fixture.h" #include "mongo/db/s/transaction_coordinator.h" #include "mongo/s/shard_id.h" -#include "mongo/s/shard_server_test_fixture.h" namespace mongo { diff --git a/src/mongo/db/s/vector_clock_shard_server_test.cpp b/src/mongo/db/s/vector_clock_shard_server_test.cpp index a2c7577769c..0f6af735ac6 100644 --- a/src/mongo/db/s/vector_clock_shard_server_test.cpp +++ b/src/mongo/db/s/vector_clock_shard_server_test.cpp @@ -32,10 +32,9 @@ #include "mongo/db/keys_collection_client_direct.h" #include "mongo/db/keys_collection_manager.h" #include "mongo/db/logical_time_validator.h" +#include "mongo/db/s/shard_server_test_fixture.h" #include "mongo/db/vector_clock_mutable.h" -#include "mongo/s/shard_server_test_fixture.h" #include "mongo/unittest/death_test.h" -#include "mongo/unittest/unittest.h" #include "mongo/util/clock_source_mock.h" namespace mongo { diff --git a/src/mongo/db/service_context_d_test_fixture.cpp b/src/mongo/db/service_context_d_test_fixture.cpp index 8c0b6b8abf5..a0b8caab275 100644 --- a/src/mongo/db/service_context_d_test_fixture.cpp +++ b/src/mongo/db/service_context_d_test_fixture.cpp @@ -90,6 +90,9 @@ ServiceContextMongoDTest::ServiceContextMongoDTest(std::string engine, RepairAct IndexAccessMethodFactory::set(serviceContext, std::make_unique()); Collection::Factory::set(serviceContext, std::make_unique()); IndexBuildsCoordinator::set(serviceContext, std::make_unique()); + CollectionShardingStateFactory::set( + getServiceContext(), + std::make_unique(getServiceContext())); } ServiceContextMongoDTest::~ServiceContextMongoDTest() { @@ -107,14 +110,6 @@ ServiceContextMongoDTest::~ServiceContextMongoDTest() { std::swap(storageGlobalParams.repair, _stashedStorageParams.repair); } -void ServiceContextMongoDTest::setUp() { - ServiceContextTest::setUp(); - - CollectionShardingStateFactory::set( - getServiceContext(), - std::make_unique(getServiceContext())); -} - void ServiceContextMongoDTest::tearDown() { { // Some tests set the current OperationContext but do not release it until destruction. diff --git a/src/mongo/db/service_context_d_test_fixture.h b/src/mongo/db/service_context_d_test_fixture.h index 0b25498a1a9..00f5f37883f 100644 --- a/src/mongo/db/service_context_d_test_fixture.h +++ b/src/mongo/db/service_context_d_test_fixture.h @@ -29,17 +29,15 @@ #pragma once -#include "mongo/db/operation_context.h" #include "mongo/db/service_context_test_fixture.h" #include "mongo/unittest/temp_dir.h" -#include "mongo/unittest/unittest.h" namespace mongo { /** * Test fixture class for tests that use the "ephemeralForTest" storage engine. */ -class ServiceContextMongoDTest : public ServiceContextTest { +class ServiceContextMongoDTest : public virtual ServiceContextTest { protected: enum class RepairAction { kNoRepair, kRepair }; @@ -52,8 +50,6 @@ protected: ServiceContextMongoDTest(std::string engine, RepairAction repair); virtual ~ServiceContextMongoDTest(); - void setUp() override; - void tearDown() override; private: @@ -62,6 +58,7 @@ private: bool engineSetByUser; bool repair; } _stashedStorageParams; + unittest::TempDir _tempDir; }; diff --git a/src/mongo/db/service_context_test_fixture.h b/src/mongo/db/service_context_test_fixture.h index e7508898c0f..a1097c6a943 100644 --- a/src/mongo/db/service_context_test_fixture.h +++ b/src/mongo/db/service_context_test_fixture.h @@ -35,7 +35,6 @@ namespace mongo { - class ScopedGlobalServiceContextForTest { public: /** diff --git a/src/mongo/db/vector_clock_mongod_test.cpp b/src/mongo/db/vector_clock_mongod_test.cpp index a54b966b8f6..758db55118e 100644 --- a/src/mongo/db/vector_clock_mongod_test.cpp +++ b/src/mongo/db/vector_clock_mongod_test.cpp @@ -32,10 +32,9 @@ #include "mongo/db/keys_collection_client_direct.h" #include "mongo/db/keys_collection_manager.h" #include "mongo/db/logical_time_validator.h" +#include "mongo/db/s/sharding_mongod_test_fixture.h" #include "mongo/db/vector_clock_mutable.h" -#include "mongo/s/sharding_mongod_test_fixture.h" #include "mongo/unittest/death_test.h" -#include "mongo/unittest/unittest.h" #include "mongo/util/clock_source_mock.h" namespace mongo { -- cgit v1.2.1