diff options
author | Jonathan Reams <jbreams@mongodb.com> | 2019-02-21 12:38:38 -0500 |
---|---|---|
committer | Jonathan Reams <jbreams@mongodb.com> | 2019-02-26 10:09:20 -0500 |
commit | 185c3482ceb8676f59a82e0b7471c31fee9d4a1d (patch) | |
tree | eeeefc59c17fe6b3dcb4447311a8179be2f14f87 | |
parent | 72425ec865efeeea9c1cb61c4cde9393fd4dc9d5 (diff) | |
download | mongo-185c3482ceb8676f59a82e0b7471c31fee9d4a1d.tar.gz |
SERVER-39538 Convert replication setParameters to IDL
-rw-r--r-- | jstests/noPassthrough/repl_write_threads_start_param.js | 8 | ||||
-rw-r--r-- | src/mongo/db/mongod_options.h | 1 | ||||
-rw-r--r-- | src/mongo/db/repl/SConscript | 35 | ||||
-rw-r--r-- | src/mongo/db/repl/abstract_oplog_fetcher.cpp | 13 | ||||
-rw-r--r-- | src/mongo/db/repl/bgsync.cpp | 28 | ||||
-rw-r--r-- | src/mongo/db/repl/collection_cloner.cpp | 10 | ||||
-rw-r--r-- | src/mongo/db/repl/data_replicator_external_state_impl.cpp | 11 | ||||
-rw-r--r-- | src/mongo/db/repl/database_cloner.cpp | 18 | ||||
-rw-r--r-- | src/mongo/db/repl/initial_syncer.cpp | 14 | ||||
-rw-r--r-- | src/mongo/db/repl/noop_writer.cpp | 4 | ||||
-rw-r--r-- | src/mongo/db/repl/oplog_applier.cpp | 28 | ||||
-rw-r--r-- | src/mongo/db/repl/repl_server_parameters.idl | 239 | ||||
-rw-r--r-- | src/mongo/db/repl/repl_set_config.cpp | 7 | ||||
-rw-r--r-- | src/mongo/db/repl/repl_set_config.idl | 41 | ||||
-rw-r--r-- | src/mongo/db/repl/repl_settings.cpp | 12 | ||||
-rw-r--r-- | src/mongo/db/repl/repl_settings.h | 5 | ||||
-rw-r--r-- | src/mongo/db/repl/repl_settings.idl | 61 | ||||
-rw-r--r-- | src/mongo/db/repl/repl_settings_init.cpp | 52 | ||||
-rw-r--r-- | src/mongo/db/repl/replication_coordinator_external_state_impl.cpp | 25 |
19 files changed, 390 insertions, 222 deletions
diff --git a/jstests/noPassthrough/repl_write_threads_start_param.js b/jstests/noPassthrough/repl_write_threads_start_param.js index 232fc75fe2b..e18d8de1259 100644 --- a/jstests/noPassthrough/repl_write_threads_start_param.js +++ b/jstests/noPassthrough/repl_write_threads_start_param.js @@ -11,21 +11,23 @@ clearRawMongoProgramOutput(); var mongo = MongoRunner.runMongod({setParameter: 'replWriterThreadCount=0'}); assert.soon(function() { - return rawMongoProgramOutput().match("replWriterThreadCount must be between 1 and 256"); + return rawMongoProgramOutput().match( + "Invalid value for parameter replWriterThreadCount: 0 is not greater than or equal to 1"); }, "mongod started with too low a value for replWriterThreadCount"); // too high a count clearRawMongoProgramOutput(); mongo = MongoRunner.runMongod({setParameter: 'replWriterThreadCount=257'}); assert.soon(function() { - return rawMongoProgramOutput().match("replWriterThreadCount must be between 1 and 256"); + return rawMongoProgramOutput().match( + "Invalid value for parameter replWriterThreadCount: 257 is not less than or equal to 256"); }, "mongod started with too high a value for replWriterThreadCount"); // proper count clearRawMongoProgramOutput(); mongo = MongoRunner.runMongod({setParameter: 'replWriterThreadCount=24'}); assert.neq(null, mongo, "mongod failed to start with a suitable replWriterThreadCount value"); - assert(!rawMongoProgramOutput().match("replWriterThreadCount must be between 1 and 256"), + assert(!rawMongoProgramOutput().match("Invalid value for parameter replWriterThreadCount"), "despite accepting the replWriterThreadCount value, mongod logged an error"); // getParameter to confirm the value was set diff --git a/src/mongo/db/mongod_options.h b/src/mongo/db/mongod_options.h index f1e56500a30..62f86b51611 100644 --- a/src/mongo/db/mongod_options.h +++ b/src/mongo/db/mongod_options.h @@ -47,7 +47,6 @@ class Environment; namespace moe = mongo::optionenvironment; -extern bool skipShardingConfigurationChecks; Status addMongodOptions(moe::OptionSection* options); diff --git a/src/mongo/db/repl/SConscript b/src/mongo/db/repl/SConscript index fd3f16f81b7..ae04d8d9ceb 100644 --- a/src/mongo/db/repl/SConscript +++ b/src/mongo/db/repl/SConscript @@ -5,6 +5,16 @@ Import("env") env = env.Clone() env.Library( + target='repl_server_parameters', + source=[ + env.Idlc('repl_server_parameters.idl')[0] + ], + LIBDEPS_PRIVATE=[ + '$BUILD_DIR/mongo/idl/server_parameter', + ] +) + +env.Library( target='oplog', source=[ 'apply_ops.cpp', @@ -149,11 +159,11 @@ env.Library( '$BUILD_DIR/mongo/client/connection_pool', '$BUILD_DIR/mongo/client/fetcher', '$BUILD_DIR/mongo/db/concurrency/write_conflict_exception', - '$BUILD_DIR/mongo/db/server_parameters', '$BUILD_DIR/mongo/db/service_context', '$BUILD_DIR/mongo/util/concurrency/thread_pool', ], LIBDEPS_PRIVATE=[ + 'repl_server_parameters', '$BUILD_DIR/mongo/executor/thread_pool_task_executor', '$BUILD_DIR/mongo/db/commands/test_commands_enabled', ], @@ -163,11 +173,11 @@ env.Library( target='repl_settings', source=[ 'repl_settings.cpp', - 'repl_settings_init.cpp', + env.Idlc('repl_settings.idl')[0], ], LIBDEPS=[ '$BUILD_DIR/mongo/base', - '$BUILD_DIR/mongo/db/server_parameters', + '$BUILD_DIR/mongo/idl/server_parameter', ], ) @@ -650,10 +660,12 @@ env.Library( '$BUILD_DIR/mongo/base', '$BUILD_DIR/mongo/db/auth/authorization_manager_global', '$BUILD_DIR/mongo/db/namespace_string', - '$BUILD_DIR/mongo/db/server_parameters', '$BUILD_DIR/mongo/util/concurrency/thread_pool', 'oplog_entry', ], + LIBDEPS_PRIVATE=[ + 'repl_server_parameters', + ], ) env.Library( @@ -1023,6 +1035,7 @@ env.Library('replica_set_messages', 'repl_set_tag.cpp', 'update_position_args.cpp', 'last_vote.cpp', + env.Idlc('repl_set_config.idl')[0], ], LIBDEPS=[ '$BUILD_DIR/mongo/base', @@ -1035,6 +1048,9 @@ env.Library('replica_set_messages', '$BUILD_DIR/mongo/rpc/command_status', 'optime', 'read_concern_args', + ], + LIBDEPS_PRIVATE=[ + '$BUILD_DIR/mongo/idl/server_parameter', ]) env.CppUnitTest('repl_set_config_test', @@ -1128,6 +1144,7 @@ env.Library( '$BUILD_DIR/mongo/executor/task_executor_interface', ], LIBDEPS_PRIVATE=[ + 'repl_server_parameters', '$BUILD_DIR/mongo/db/commands/server_status_core', ], ) @@ -1271,6 +1288,9 @@ env.Library( '$BUILD_DIR/mongo/s/query/async_results_merger', '$BUILD_DIR/mongo/util/progress_meter', ], + LIBDEPS_PRIVATE=[ + 'repl_server_parameters', + ], ) env.CppUnitTest( @@ -1296,6 +1316,7 @@ env.Library( '$BUILD_DIR/mongo/executor/task_executor_interface', ], LIBDEPS_PRIVATE=[ + 'repl_server_parameters', '$BUILD_DIR/mongo/db/commands/list_collections_filter', ], ) @@ -1450,7 +1471,9 @@ env.Library( 'repl_coordinator_interface', 'storage_interface', '$BUILD_DIR/mongo/base', - '$BUILD_DIR/mongo/db/server_parameters', + ], + LIBDEPS_PRIVATE=[ + 'repl_server_parameters', ], ) @@ -1500,6 +1523,7 @@ env.Library( 'storage_interface', ], LIBDEPS_PRIVATE=[ + 'repl_server_parameters', '$BUILD_DIR/mongo/db/commands/feature_compatibility_parsers', ] ) @@ -1641,6 +1665,7 @@ env.Library( 'rollback_source_impl', ], LIBDEPS_PRIVATE=[ + 'repl_server_parameters', '$BUILD_DIR/mongo/db/commands/mongod_fcv', '$BUILD_DIR/mongo/db/commands/test_commands_enabled', ], diff --git a/src/mongo/db/repl/abstract_oplog_fetcher.cpp b/src/mongo/db/repl/abstract_oplog_fetcher.cpp index e47909a3419..a3f27e65e94 100644 --- a/src/mongo/db/repl/abstract_oplog_fetcher.cpp +++ b/src/mongo/db/repl/abstract_oplog_fetcher.cpp @@ -37,7 +37,7 @@ #include "mongo/bson/util/bson_extract.h" #include "mongo/db/commands/server_status_metric.h" #include "mongo/db/jsobj.h" -#include "mongo/db/server_parameters.h" +#include "mongo/db/repl/repl_server_parameters_gen.h" #include "mongo/stdx/memory.h" #include "mongo/stdx/mutex.h" #include "mongo/util/assert_util.h" @@ -52,16 +52,6 @@ Counter64 readersCreatedStats; ServerStatusMetricField<Counter64> displayReadersCreated("repl.network.readersCreated", &readersCreatedStats); -// Number of seconds for the `maxTimeMS` on the initial `find` command. -// -// For the initial 'find' request, we provide a generous timeout, to account for the potentially -// slow process of a sync source finding the lastApplied optime provided in a node's query in its -// oplog. -MONGO_EXPORT_SERVER_PARAMETER(oplogInitialFindMaxSeconds, int, 60); - -// Number of seconds for the `maxTimeMS` on any retried `find` commands. -MONGO_EXPORT_SERVER_PARAMETER(oplogRetriedFindMaxSeconds, int, 2); - // Number of milliseconds to add to the `find` and `getMore` timeouts to calculate the network // timeout for the requests. const Milliseconds kNetworkTimeoutBufferMS{5000}; @@ -69,7 +59,6 @@ const Milliseconds kNetworkTimeoutBufferMS{5000}; // Default `maxTimeMS` timeout for `getMore`s. const Milliseconds kDefaultOplogGetMoreMaxMS{5000}; - } // namespace AbstractOplogFetcher::AbstractOplogFetcher(executor::TaskExecutor* executor, diff --git a/src/mongo/db/repl/bgsync.cpp b/src/mongo/db/repl/bgsync.cpp index 698edce2fab..3181bc132ee 100644 --- a/src/mongo/db/repl/bgsync.cpp +++ b/src/mongo/db/repl/bgsync.cpp @@ -47,6 +47,7 @@ #include "mongo/db/repl/oplog.h" #include "mongo/db/repl/oplog_interface_local.h" #include "mongo/db/repl/oplog_interface_remote.h" +#include "mongo/db/repl/repl_server_parameters_gen.h" #include "mongo/db/repl/replication_coordinator.h" #include "mongo/db/repl/replication_coordinator_impl.h" #include "mongo/db/repl/replication_process.h" @@ -54,7 +55,6 @@ #include "mongo/db/repl/rs_rollback.h" #include "mongo/db/repl/storage_interface.h" #include "mongo/db/s/shard_identity_rollback_notifier.h" -#include "mongo/db/server_parameters.h" #include "mongo/rpc/get_status_from_command_result.h" #include "mongo/rpc/metadata/repl_set_metadata.h" #include "mongo/stdx/memory.h" @@ -72,32 +72,6 @@ namespace { const int kSleepToAllowBatchingMillis = 2; const int kSmallBatchLimitBytes = 40000; const Milliseconds kRollbackOplogSocketTimeout(10 * 60 * 1000); -// 16MB max batch size / 12 byte min doc size * 10 (for good measure) = defaultBatchSize to use. -const auto defaultBatchSize = (16 * 1024 * 1024) / 12 * 10; - -// The batchSize to use for the find/getMore queries called by the OplogFetcher -MONGO_EXPORT_STARTUP_SERVER_PARAMETER(bgSyncOplogFetcherBatchSize, int, defaultBatchSize); - -// The batchSize to use for the find/getMore queries called by the rollback common point resolver. -// A batchSize of 0 means that the 'find' and 'getMore' commands will be given no batchSize. -// We set the default to 2000 to prevent the sync source from having to read too much data at once, -// and reduce the chance of a socket timeout. -// We choose 2000 for (10 minute timeout) * (60 sec / min) * (50 MB / second) / (16 MB / document). -constexpr int defaultRollbackBatchSize = 2000; -MONGO_EXPORT_SERVER_PARAMETER(rollbackRemoteOplogQueryBatchSize, int, defaultRollbackBatchSize) - ->withValidator([](const auto& potentialNewValue) { - if (potentialNewValue < 0) { - return Status(ErrorCodes::BadValue, - "rollbackRemoteOplogQueryBatchSize cannot be negative."); - } - - return Status::OK(); - }); - -// If 'forceRollbackViaRefetch' is true, always perform rollbacks via the refetch algorithm, even if -// the storage engine supports rollback via recover to timestamp. -constexpr bool forceRollbackViaRefetchByDefault = false; -MONGO_EXPORT_SERVER_PARAMETER(forceRollbackViaRefetch, bool, forceRollbackViaRefetchByDefault); /** * Extends DataReplicatorExternalStateImpl to be member state aware. diff --git a/src/mongo/db/repl/collection_cloner.cpp b/src/mongo/db/repl/collection_cloner.cpp index a30cf1496c8..ac0cfb22647 100644 --- a/src/mongo/db/repl/collection_cloner.cpp +++ b/src/mongo/db/repl/collection_cloner.cpp @@ -43,9 +43,9 @@ #include "mongo/db/namespace_string.h" #include "mongo/db/query/cursor_response.h" #include "mongo/db/repl/oplogreader.h" +#include "mongo/db/repl/repl_server_parameters_gen.h" #include "mongo/db/repl/storage_interface.h" #include "mongo/db/repl/storage_interface_mock.h" -#include "mongo/db/server_parameters.h" #include "mongo/rpc/get_status_from_command_result.h" #include "mongo/util/assert_util.h" #include "mongo/util/destructor_guard.h" @@ -66,14 +66,6 @@ constexpr auto kCountResponseDocumentCountFieldName = "n"_sd; const int kProgressMeterSecondsBetween = 60; const int kProgressMeterCheckInterval = 128; -// The number of attempts for the count command, which gets the document count. -MONGO_EXPORT_SERVER_PARAMETER(numInitialSyncCollectionCountAttempts, int, 3); -// The number of attempts for the listIndexes commands. -MONGO_EXPORT_SERVER_PARAMETER(numInitialSyncListIndexesAttempts, int, 3); -// The number of attempts for the find command, which gets the data. -MONGO_EXPORT_SERVER_PARAMETER(numInitialSyncCollectionFindAttempts, int, 3); -// Whether to use the "exhaust cursor" feature when retrieving collection data. -MONGO_EXPORT_STARTUP_SERVER_PARAMETER(collectionClonerUsesExhaust, bool, true); } // namespace // Failpoint which causes initial sync to hang before establishing its cursor to clone the diff --git a/src/mongo/db/repl/data_replicator_external_state_impl.cpp b/src/mongo/db/repl/data_replicator_external_state_impl.cpp index bcd91dd6c23..f2e2c1386d5 100644 --- a/src/mongo/db/repl/data_replicator_external_state_impl.cpp +++ b/src/mongo/db/repl/data_replicator_external_state_impl.cpp @@ -38,11 +38,11 @@ #include "mongo/db/repl/oplog_buffer_blocking_queue.h" #include "mongo/db/repl/oplog_buffer_collection.h" #include "mongo/db/repl/oplog_buffer_proxy.h" +#include "mongo/db/repl/repl_server_parameters_gen.h" #include "mongo/db/repl/replication_coordinator.h" #include "mongo/db/repl/replication_coordinator_external_state.h" #include "mongo/db/repl/replication_process.h" #include "mongo/db/repl/storage_interface.h" -#include "mongo/db/server_parameters.h" #include "mongo/util/log.h" namespace mongo { @@ -52,15 +52,6 @@ namespace { const char kCollectionOplogBufferName[] = "collection"; const char kBlockingQueueOplogBufferName[] = "inMemoryBlockingQueue"; -// Set this to specify whether to use a collection to buffer the oplog on the destination server -// during initial sync to prevent rolling over the oplog. -MONGO_EXPORT_STARTUP_SERVER_PARAMETER(initialSyncOplogBuffer, - std::string, - kCollectionOplogBufferName); - -// Set this to specify size of read ahead buffer in the OplogBufferCollection. -MONGO_EXPORT_STARTUP_SERVER_PARAMETER(initialSyncOplogBufferPeekCacheSize, int, 10000); - MONGO_INITIALIZER(initialSyncOplogBuffer)(InitializerContext*) { if ((initialSyncOplogBuffer != kCollectionOplogBufferName) && (initialSyncOplogBuffer != kBlockingQueueOplogBufferName)) { diff --git a/src/mongo/db/repl/database_cloner.cpp b/src/mongo/db/repl/database_cloner.cpp index 073cfced04b..3b29095c8fd 100644 --- a/src/mongo/db/repl/database_cloner.cpp +++ b/src/mongo/db/repl/database_cloner.cpp @@ -40,8 +40,8 @@ #include "mongo/client/remote_command_retry_scheduler.h" #include "mongo/db/catalog/collection_options.h" #include "mongo/db/commands/list_collections_filter.h" +#include "mongo/db/repl/repl_server_parameters_gen.h" #include "mongo/db/repl/storage_interface.h" -#include "mongo/db/server_parameters.h" #include "mongo/stdx/functional.h" #include "mongo/util/assert_util.h" #include "mongo/util/destructor_guard.h" @@ -66,22 +66,6 @@ const char* kOptionsFieldName = "options"; const char* kInfoFieldName = "info"; const char* kUUIDFieldName = "uuid"; -// The batch size (number of documents) to use for the queries in the CollectionCloner. Default of -// 0 means the limit is the number of documents which fit in a single BSON object. -MONGO_EXPORT_STARTUP_SERVER_PARAMETER(collectionClonerBatchSize, int, 0) - ->withValidator([](const int& batchSize) { - return (batchSize >= 0) - ? Status::OK() - : Status(ErrorCodes::Error(50952), - str::stream() - << "collectionClonerBatchSize must be greater than or equal to 0. '" - << batchSize - << "' is an invalid setting."); - }); - -// The number of attempts for the listCollections commands. -MONGO_EXPORT_SERVER_PARAMETER(numInitialSyncListCollectionsAttempts, int, 3); - // Failpoint which causes initial sync to hang right after listCollections, but before cloning // any colelctions in the 'database' database. MONGO_FAIL_POINT_DEFINE(initialSyncHangAfterListCollections); diff --git a/src/mongo/db/repl/initial_syncer.cpp b/src/mongo/db/repl/initial_syncer.cpp index 237985c6e8a..616b8961a94 100644 --- a/src/mongo/db/repl/initial_syncer.cpp +++ b/src/mongo/db/repl/initial_syncer.cpp @@ -53,11 +53,11 @@ #include "mongo/db/repl/oplog_buffer.h" #include "mongo/db/repl/oplog_fetcher.h" #include "mongo/db/repl/optime.h" +#include "mongo/db/repl/repl_server_parameters_gen.h" #include "mongo/db/repl/replication_consistency_markers.h" #include "mongo/db/repl/replication_process.h" #include "mongo/db/repl/storage_interface.h" #include "mongo/db/repl/sync_source_selector.h" -#include "mongo/db/server_parameters.h" #include "mongo/executor/task_executor.h" #include "mongo/executor/thread_pool_task_executor.h" #include "mongo/rpc/metadata/repl_set_metadata.h" @@ -109,18 +109,6 @@ using QueryResponseStatus = StatusWith<Fetcher::QueryResponse>; using UniqueLock = stdx::unique_lock<stdx::mutex>; using LockGuard = stdx::lock_guard<stdx::mutex>; -// 16MB max batch size / 12 byte min doc size * 10 (for good measure) = defaultBatchSize to use. -const auto defaultBatchSize = (16 * 1024 * 1024) / 12 * 10; - -// The number of attempts to connect to a sync source. -MONGO_EXPORT_SERVER_PARAMETER(numInitialSyncConnectAttempts, int, 10); - -// The number of attempts to call find on the remote oplog. -MONGO_EXPORT_SERVER_PARAMETER(numInitialSyncOplogFindAttempts, int, 3); - -// The batchSize to use for the find/getMore queries called by the OplogFetcher -MONGO_EXPORT_STARTUP_SERVER_PARAMETER(initialSyncOplogFetcherBatchSize, int, defaultBatchSize); - // The number of initial sync attempts that have failed since server startup. Each instance of // InitialSyncer may run multiple attempts to fulfill an initial sync request that is triggered // when InitialSyncer::startup() is called. diff --git a/src/mongo/db/repl/noop_writer.cpp b/src/mongo/db/repl/noop_writer.cpp index eb698db85ea..90b15f0f4ac 100644 --- a/src/mongo/db/repl/noop_writer.cpp +++ b/src/mongo/db/repl/noop_writer.cpp @@ -40,7 +40,7 @@ #include "mongo/db/op_observer.h" #include "mongo/db/repl/noop_writer.h" #include "mongo/db/repl/oplog.h" -#include "mongo/db/server_parameters.h" +#include "mongo/db/repl/repl_server_parameters_gen.h" #include "mongo/stdx/functional.h" #include "mongo/util/concurrency/idle_thread_block.h" #include "mongo/util/log.h" @@ -50,8 +50,6 @@ namespace repl { namespace { -MONGO_EXPORT_SERVER_PARAMETER(writePeriodicNoops, bool, true); - const auto kMsgObj = BSON("msg" << "periodic noop"); diff --git a/src/mongo/db/repl/oplog_applier.cpp b/src/mongo/db/repl/oplog_applier.cpp index b975cf9a3d1..eb06a50d681 100644 --- a/src/mongo/db/repl/oplog_applier.cpp +++ b/src/mongo/db/repl/oplog_applier.cpp @@ -35,39 +35,13 @@ #include "mongo/db/auth/authorization_session.h" #include "mongo/db/namespace_string.h" +#include "mongo/db/repl/repl_server_parameters_gen.h" #include "mongo/db/repl/sync_tail.h" -#include "mongo/db/server_parameters.h" #include "mongo/util/log.h" namespace mongo { namespace repl { -namespace { - -/** - * This server parameter determines the number of writer threads OplogApplier will have. - */ -MONGO_EXPORT_STARTUP_SERVER_PARAMETER(replWriterThreadCount, int, 16) - ->withValidator([](const int& newVal) { - if (newVal < 1 || newVal > 256) { - return Status(ErrorCodes::BadValue, "replWriterThreadCount must be between 1 and 256"); - } - - return Status::OK(); - }); - -MONGO_EXPORT_SERVER_PARAMETER(replBatchLimitOperations, int, 5 * 1000) - ->withValidator([](const int& newVal) { - if (newVal < 1 || newVal > (1000 * 1000)) { - return Status(ErrorCodes::BadValue, - "replBatchLimitOperations must be between 1 and 1 million, inclusive"); - } - - return Status::OK(); - }); - -} // namespace - using CallbackArgs = executor::TaskExecutor::CallbackArgs; // static diff --git a/src/mongo/db/repl/repl_server_parameters.idl b/src/mongo/db/repl/repl_server_parameters.idl new file mode 100644 index 00000000000..b7b2447649a --- /dev/null +++ b/src/mongo/db/repl/repl_server_parameters.idl @@ -0,0 +1,239 @@ +# Copyright (C) 2018-present MongoDB, Inc. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the Server Side Public License, version 1, +# as published by MongoDB, Inc. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# Server Side Public License for more details. +# +# You should have received a copy of the Server Side Public License +# along with this program. If not, see +# <http://www.mongodb.com/licensing/server-side-public-license>. +# +# As a special exception, the copyright holders give permission to link the +# code of portions of this program with the OpenSSL library under certain +# conditions as described in each individual source file and distribute +# linked combinations including the program with the OpenSSL library. You +# must comply with the Server Side Public License in all respects for +# all of the code used other than as permitted herein. If you modify file(s) +# with this exception, you may extend this exception to your version of the +# file(s), but you are not obligated to do so. If you do not wish to do so, +# delete this exception statement from your version. If you delete this +# exception statement from all source files in the program, then also delete +# it in the license file. +# + +# server setParameters for replication + +global: + cpp_namespace: "mongo::repl" + +imports: + - "mongo/idl/basic_types.idl" + +server_parameters: + # From data_replicator_external_state_impl.cpp + initialSyncOplogBuffer: + description: >- + Set this to specify whether to use a collection to buffer the oplog on the + destination server during initial sync to prevent rolling over the oplog. + set_at: startup + cpp_vartype: std::string + cpp_varname: initialSyncOplogBuffer + default: "collection" + + initialSyncOplogBufferPeekCacheSize: + description: Set this to specify size of read ahead buffer in the OplogBufferCollection. + set_at: startup + cpp_vartype: int + cpp_varname: initialSyncOplogBufferPeekCacheSize + default: 10000 + + # From initial_syncer.cpp + numInitialSyncConnectAttempts: + description: The number of attempts to connect to a sync source + set_at: [ startup, runtime ] + cpp_vartype: AtomicWord<int> + cpp_varname: numInitialSyncConnectAttempts + default: 10 + + numInitialSyncOplogFindAttempts: + description: The number of attempts to call find on the remote oplog + set_at: [ startup, runtime ] + cpp_vartype: AtomicWord<int> + cpp_varname: numInitialSyncOplogFindAttempts + default: 3 + + initialSyncOplogFetcherBatchSize: + description: The batchSize to use for the find/getMore queries called by the OplogFetcher + set_at: startup + cpp_vartype: int + cpp_varname: initialSyncOplogFetcherBatchSize + # 16MB max batch size / 12 byte min doc size * 10 (for good measure) = + # defaultBatchSize to use. + default: + expr: (16 * 1024 * 1024) / 12 * 10 + + # From abstract_oplog_fetcher.cpp + oplogInitialFindMaxSeconds: + description: >- + Number of seconds for the `maxTimeMS` on the initial `find` command. + For the initial 'find' request, we provide a generous timeout, to account for + the potentially slow process of a sync source finding the lastApplied optime + provided in a node's query in its oplog. + set_at: [ startup, runtime ] + cpp_vartype: AtomicWord<int> + cpp_varname: oplogInitialFindMaxSeconds + default: 60 + + oplogRetriedFindMaxSeconds: + description: Number of seconds for the `maxTimeMS` on any retried `find` commands + set_at: [ startup, runtime ] + cpp_vartype: AtomicWord<int> + cpp_varname: oplogRetriedFindMaxSeconds + default: 2 + + # From bgsync.cpp + bgSyncOplogFetcherBatchSize: + description: The batchSize to use for the find/getMore queries called by the OplogFetcher + set_at: startup + cpp_vartype: int + cpp_varname: bgSyncOplogFetcherBatchSize + # 16MB max batch size / 12 byte min doc size * 10 (for good measure) = + # defaultBatchSize to use. + default: + expr: (16 * 1024 * 1024) / 12 * 10 + + rollbackRemoteOplogQueryBatchSize: + description: >- + The batchSize to use for the find/getMore queries called by the rollback + common point resolver. A batchSize of 0 means that the 'find' and 'getMore' + commands will be given no batchSize. We set the default to 2000 to prevent + the sync source from having to read too much data at once, and reduce the + chance of a socket timeout. + set_at: [ startup, runtime ] + cpp_vartype: AtomicWord<int> + cpp_varname: rollbackRemoteOplogQueryBatchSize + # We choose 2000 for (10 minute timeout) * (60 sec / min) * (50 MB / second) / + # (16 MB / document). + default: 2000 + + forceRollbackViaRefetch: + description: >- + If 'forceRollbackViaRefetch' is true, always perform rollbacks via the + refetch algorithm, even if the storage engine supports rollback via recover + to timestamp. + set_at: [ startup, runtime ] + cpp_vartype: AtomicWord<bool> + cpp_varname: forceRollbackViaRefetch + default: false + + # From noop_writer.cpp + writePeriodicNoops: + description: Sets whether to write periodic noops or not + set_at: [ startup, runtime ] + cpp_vartype: AtomicWord<bool> + cpp_varname: writePeriodicNoops + default: true + + # From collection_cloner.cpp + numInitialSyncCollectionCountAttempts: + description: The number of attempts for the count command, which gets the document count. + set_at: [ startup, runtime ] + cpp_vartype: AtomicWord<int> + cpp_varname: numInitialSyncCollectionCountAttempts + default: 3 + + numInitialSyncListIndexesAttempts: + description: The number of attempts for the listIndexes commands. + set_at: [ startup, runtime ] + cpp_vartype: AtomicWord<int> + cpp_varname: numInitialSyncListIndexesAttempts + default: 3 + + numInitialSyncCollectionFindAttempts: + description: The number of attempts for the find command, which gets the data. + set_at: [ startup, runtime ] + cpp_vartype: AtomicWord<int> + cpp_varname: numInitialSyncCollectionFindAttempts + default: 3 + + collectionClonerUsesExhaust: + description: Whether to use the "exhaust cursor" feature when retrieving collection data. + set_at: startup + cpp_vartype: bool + cpp_varname: collectionClonerUsesExhaust + default: true + + # From database_cloner.cpp + collectionClonerBatchSize: + description: >- + The batch size (number of documents) to use for the queries in the + CollectionCloner. Default of '0' means the limit is the number of documents + that will fit in a single BSON object. + set_at: startup + cpp_vartype: int + cpp_varname: collectionClonerBatchSize + default: 0 + validator: + gte: 0 + + numInitialSyncListCollectionsAttempts: + description: The number of attempts for the listCollections commands. + set_at: [ startup, runtime ] + cpp_vartype: AtomicWord<int> + cpp_varname: numInitialSyncListCollectionsAttempts + default: 3 + + # From replication_coordinator_external_state_impl.cpp + oplogFetcherSteadyStateMaxFetcherRestarts: + description: >- + Set this to specify the maximum number of times the oplog fetcher will + consecutively restart the oplog tailing query on non-cancellation errors + during steady state replication. + set_at: [ startup, runtime ] + cpp_vartype: AtomicWord<int> + cpp_varname: oplogFetcherSteadyStateMaxFetcherRestarts + default: 1 + validator: + gte: 0 + + oplogFetcherInitialSyncMaxFetcherRestarts: + description: >- + Set this to specify the maximum number of times the oplog fetcher will + consecutively restart the oplog tailing query on non-cancellation errors + during initial sync. By default we provide a generous amount of restarts + to avoid potentially restarting an entire initial sync from scratch. + set_at: [ startup, runtime ] + cpp_vartype: AtomicWord<int> + cpp_varname: oplogFetcherInitialSyncMaxFetcherRestarts + default: 10 + validator: + gte: 0 + + # From oplog_applier.cpp + replWriterThreadCount: + description: The number of threads in the thread pool used to apply the oplog + set_at: startup + cpp_vartype: int + cpp_varname: replWriterThreadCount + default: 16 + validator: + gte: 1 + lte: 256 + + replBatchLimitOperations: + description: The maximum number of operations to apply in a single batch + set_at: [ startup, runtime ] + cpp_vartype: AtomicWord<int> + cpp_varname: replBatchLimitOperations + default: + expr: 5 * 1000 + validator: + gte: 1 + lte: + expr: 1000 * 1000 + diff --git a/src/mongo/db/repl/repl_set_config.cpp b/src/mongo/db/repl/repl_set_config.cpp index 2f5b4c5d5b8..d0a9f38035b 100644 --- a/src/mongo/db/repl/repl_set_config.cpp +++ b/src/mongo/db/repl/repl_set_config.cpp @@ -38,17 +38,10 @@ #include "mongo/db/jsobj.h" #include "mongo/db/mongod_options.h" #include "mongo/db/server_options.h" -#include "mongo/db/server_parameters.h" #include "mongo/stdx/functional.h" #include "mongo/util/stringutils.h" namespace mongo { -/** - * Dont run any sharding validations. Can not be combined with --configsvr or shardvr. Intended to - * allow restarting config server or shard as an independent replica set. - */ -MONGO_EXPORT_STARTUP_SERVER_PARAMETER(skipShardingConfigurationChecks, bool, false); - namespace repl { const size_t ReplSetConfig::kMaxMembers; diff --git a/src/mongo/db/repl/repl_set_config.idl b/src/mongo/db/repl/repl_set_config.idl new file mode 100644 index 00000000000..92089aef855 --- /dev/null +++ b/src/mongo/db/repl/repl_set_config.idl @@ -0,0 +1,41 @@ +# Copyright (C) 2019-present MongoDB, Inc. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the Server Side Public License, version 1, +# as published by MongoDB, Inc. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# Server Side Public License for more details. +# +# You should have received a copy of the Server Side Public License +# along with this program. If not, see +# <http://www.mongodb.com/licensing/server-side-public-license>. +# +# As a special exception, the copyright holders give permission to link the +# code of portions of this program with the OpenSSL library under certain +# conditions as described in each individual source file and distribute +# linked combinations including the program with the OpenSSL library. You +# must comply with the Server Side Public License in all respects for +# all of the code used other than as permitted herein. If you modify file(s) +# with this exception, you may extend this exception to your version of the +# file(s), but you are not obligated to do so. If you do not wish to do so, +# delete this exception statement from your version. If you delete this +# exception statement from all source files in the program, then also delete +# it in the license file. +# + +global: + cpp_namespace: "mongo" + +server_parameters: + skipShardingConfigurationChecks: + description: Dont run any sharding validations. Can not be combined with --configsvr or + shardvr. Intended to allow restarting config server or shard as an + independent replica set. + set_at: startup + cpp_vartype: bool + cpp_varname: skipShardingConfigurationChecks + default: false + diff --git a/src/mongo/db/repl/repl_settings.cpp b/src/mongo/db/repl/repl_settings.cpp index 1174448c28a..f8abdd02cf8 100644 --- a/src/mongo/db/repl/repl_settings.cpp +++ b/src/mongo/db/repl/repl_settings.cpp @@ -32,20 +32,14 @@ #include "mongo/platform/basic.h" #include "mongo/db/repl/repl_settings.h" -#include "mongo/db/server_parameters.h" +#include "mongo/db/repl/repl_server_parameters_gen.h" +#include "mongo/db/repl/repl_settings_gen.h" #include "mongo/util/log.h" namespace mongo { namespace repl { -namespace { - -// Tells the server to perform replication recovery as a standalone. -constexpr bool recoverFromOplogAsStandaloneDefault = false; -MONGO_EXPORT_STARTUP_SERVER_PARAMETER(recoverFromOplogAsStandalone, - bool, - recoverFromOplogAsStandaloneDefault); -} // namespace + std::string ReplSettings::ourSetName() const { size_t sl = _replSetString.find('/'); diff --git a/src/mongo/db/repl/repl_settings.h b/src/mongo/db/repl/repl_settings.h index 9423ce6185d..e75264cb5be 100644 --- a/src/mongo/db/repl/repl_settings.h +++ b/src/mongo/db/repl/repl_settings.h @@ -32,13 +32,12 @@ #include <string> #include "mongo/db/jsobj.h" +#include "mongo/db/repl/repl_set_config_gen.h" +#include "mongo/db/repl/repl_settings_gen.h" namespace mongo { namespace repl { -extern int maxSyncSourceLagSecs; -extern double replElectionTimeoutOffsetLimitFraction; - class ReplSettings { public: std::string ourSetName() const; diff --git a/src/mongo/db/repl/repl_settings.idl b/src/mongo/db/repl/repl_settings.idl new file mode 100644 index 00000000000..4e01a00d270 --- /dev/null +++ b/src/mongo/db/repl/repl_settings.idl @@ -0,0 +1,61 @@ +# Copyright (C) 2018-present MongoDB, Inc. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the Server Side Public License, version 1, +# as published by MongoDB, Inc. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# Server Side Public License for more details. +# +# You should have received a copy of the Server Side Public License +# along with this program. If not, see +# <http://www.mongodb.com/licensing/server-side-public-license>. +# +# As a special exception, the copyright holders give permission to link the +# code of portions of this program with the OpenSSL library under certain +# conditions as described in each individual source file and distribute +# linked combinations including the program with the OpenSSL library. You +# must comply with the Server Side Public License in all respects for +# all of the code used other than as permitted herein. If you modify file(s) +# with this exception, you may extend this exception to your version of the +# file(s), but you are not obligated to do so. If you do not wish to do so, +# delete this exception statement from your version. If you delete this +# exception statement from all source files in the program, then also delete +# it in the license file. +# + +# server setParameters for replication + +global: + cpp_namespace: "mongo::repl" + +server_parameters: + # From repl_settings.cpp + recoverFromOplogAsStandalone: + description: Tells the server to perform replication recovery as a standalone. + set_at: startup + cpp_vartype: bool + cpp_varname: recoverFromOplogAsStandalone + default: false + + # From repl_settings_init.cpp + maxSyncSourceLagSecs: + description: '' + set_at: startup + cpp_vartype: int + cpp_varname: maxSyncSourceLagSecs + default: 30 + validator: + gt: 0 + + replElectionTimeoutOffsetLimitFraction: + description: '' + set_at: startup + cpp_vartype: double + cpp_varname: replElectionTimeoutOffsetLimitFraction + default: 0.15 + validator: + gt: 0.01 + diff --git a/src/mongo/db/repl/repl_settings_init.cpp b/src/mongo/db/repl/repl_settings_init.cpp deleted file mode 100644 index 66793563f85..00000000000 --- a/src/mongo/db/repl/repl_settings_init.cpp +++ /dev/null @@ -1,52 +0,0 @@ -/** - * Copyright (C) 2018-present MongoDB, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the Server Side Public License, version 1, - * as published by MongoDB, Inc. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * Server Side Public License for more details. - * - * You should have received a copy of the Server Side Public License - * along with this program. If not, see - * <http://www.mongodb.com/licensing/server-side-public-license>. - * - * As a special exception, the copyright holders give permission to link the - * code of portions of this program with the OpenSSL library under certain - * conditions as described in each individual source file and distribute - * linked combinations including the program with the OpenSSL library. You - * must comply with the Server Side Public License in all respects for - * all of the code used other than as permitted herein. If you modify file(s) - * with this exception, you may extend this exception to your version of the - * file(s), but you are not obligated to do so. If you do not wish to do so, - * delete this exception statement from your version. If you delete this - * exception statement from all source files in the program, then also delete - * it in the license file. - */ - -#include "mongo/db/repl/repl_settings.h" - -#include "mongo/base/init.h" -#include "mongo/base/status.h" -#include "mongo/db/server_parameters.h" - -namespace mongo { -namespace repl { - -MONGO_EXPORT_STARTUP_SERVER_PARAMETER(maxSyncSourceLagSecs, int, 30); -MONGO_EXPORT_STARTUP_SERVER_PARAMETER(replElectionTimeoutOffsetLimitFraction, double, 0.15); - -MONGO_INITIALIZER(replSettingsCheck)(InitializerContext*) { - if (maxSyncSourceLagSecs < 1) { - return Status(ErrorCodes::BadValue, "maxSyncSourceLagSecs must be > 0"); - } - if (replElectionTimeoutOffsetLimitFraction <= 0.01) { - return Status(ErrorCodes::BadValue, "electionTimeoutOffsetLimitFraction must be > 0.01"); - } - return Status::OK(); -} -} -} diff --git a/src/mongo/db/repl/replication_coordinator_external_state_impl.cpp b/src/mongo/db/repl/replication_coordinator_external_state_impl.cpp index 2ce1ae9e438..dba73cf46cd 100644 --- a/src/mongo/db/repl/replication_coordinator_external_state_impl.cpp +++ b/src/mongo/db/repl/replication_coordinator_external_state_impl.cpp @@ -66,6 +66,7 @@ #include "mongo/db/repl/oplog.h" #include "mongo/db/repl/oplog_applier_impl.h" #include "mongo/db/repl/oplog_buffer_blocking_queue.h" +#include "mongo/db/repl/repl_server_parameters_gen.h" #include "mongo/db/repl/repl_settings.h" #include "mongo/db/repl/replication_coordinator.h" #include "mongo/db/repl/replication_process.h" @@ -78,7 +79,6 @@ #include "mongo/db/s/sharding_state_recovery.h" #include "mongo/db/s/transaction_coordinator_service.h" #include "mongo/db/server_options.h" -#include "mongo/db/server_parameters.h" #include "mongo/db/service_context.h" #include "mongo/db/session_catalog_mongod.h" #include "mongo/db/storage/storage_engine.h" @@ -125,29 +125,6 @@ const char tsFieldName[] = "ts"; MONGO_FAIL_POINT_DEFINE(dropPendingCollectionReaperHang); -// Set this to specify the maximum number of times the oplog fetcher will consecutively restart the -// oplog tailing query on non-cancellation errors during steady state replication. -MONGO_EXPORT_SERVER_PARAMETER(oplogFetcherSteadyStateMaxFetcherRestarts, int, 1) - ->withValidator([](const int& potentialNewValue) { - if (potentialNewValue < 0) { - return Status(ErrorCodes::BadValue, - "oplogFetcherSteadyStateMaxFetcherRestarts must be nonnegative"); - } - return Status::OK(); - }); - -// Set this to specify the maximum number of times the oplog fetcher will consecutively restart the -// oplog tailing query on non-cancellation errors during initial sync. By default we provide a -// generous amount of restarts to avoid potentially restarting an entire initial sync from scratch. -MONGO_EXPORT_SERVER_PARAMETER(oplogFetcherInitialSyncMaxFetcherRestarts, int, 10) - ->withValidator([](const int& potentialNewValue) { - if (potentialNewValue < 0) { - return Status(ErrorCodes::BadValue, - "oplogFetcherInitialSyncMaxFetcherRestarts must be nonnegative"); - } - return Status::OK(); - }); - // The count of items in the buffer OplogBuffer::Counters bufferGauge; ServerStatusMetricField<Counter64> displayBufferCount("repl.buffer.count", &bufferGauge.count); |