diff options
author | Judah Schvimer <judah@mongodb.com> | 2017-09-13 11:58:43 -0400 |
---|---|---|
committer | Judah Schvimer <judah@mongodb.com> | 2017-09-13 11:58:43 -0400 |
commit | 5dbccb4a861aa2db993dd673097a1300bcdc9cca (patch) | |
tree | e13077ba423a9f5043c6e033a90de731b891e705 /src | |
parent | 76da39708f4d07ed0cf56d986d1c6f3d4353e670 (diff) | |
download | mongo-5dbccb4a861aa2db993dd673097a1300bcdc9cca.tar.gz |
SERVER-19605 make oplog timeout configurable
Diffstat (limited to 'src')
-rw-r--r-- | src/mongo/client/fetcher.cpp | 18 | ||||
-rw-r--r-- | src/mongo/client/fetcher.h | 11 | ||||
-rw-r--r-- | src/mongo/client/fetcher_test.cpp | 4 | ||||
-rw-r--r-- | src/mongo/db/repl/abstract_oplog_fetcher.cpp | 29 | ||||
-rw-r--r-- | src/mongo/db/repl/abstract_oplog_fetcher.h | 18 | ||||
-rw-r--r-- | src/mongo/db/repl/collection_cloner.cpp | 3 | ||||
-rw-r--r-- | src/mongo/db/repl/database_cloner.cpp | 3 | ||||
-rw-r--r-- | src/mongo/db/repl/initial_syncer.cpp | 3 | ||||
-rw-r--r-- | src/mongo/db/repl/oplog_fetcher.cpp | 9 | ||||
-rw-r--r-- | src/mongo/db/repl/oplog_fetcher.h | 2 | ||||
-rw-r--r-- | src/mongo/db/repl/rollback_common_point_resolver.cpp | 5 | ||||
-rw-r--r-- | src/mongo/db/repl/sync_source_resolver.cpp | 6 | ||||
-rw-r--r-- | src/mongo/s/client/shard_remote.cpp | 3 |
13 files changed, 71 insertions, 43 deletions
diff --git a/src/mongo/client/fetcher.cpp b/src/mongo/client/fetcher.cpp index 471de980b22..7128c2800bd 100644 --- a/src/mongo/client/fetcher.cpp +++ b/src/mongo/client/fetcher.cpp @@ -172,7 +172,8 @@ Fetcher::Fetcher(executor::TaskExecutor* executor, const BSONObj& findCmdObj, const CallbackFn& work, const BSONObj& metadata, - Milliseconds timeout, + Milliseconds findNetworkTimeout, + Milliseconds getMoreNetworkTimeout, std::unique_ptr<RemoteCommandRetryScheduler::RetryPolicy> firstCommandRetryPolicy) : _executor(executor), _source(source), @@ -180,10 +181,11 @@ Fetcher::Fetcher(executor::TaskExecutor* executor, _cmdObj(findCmdObj.getOwned()), _metadata(metadata.getOwned()), _work(work), - _timeout(timeout), + _findNetworkTimeout(findNetworkTimeout), + _getMoreNetworkTimeout(getMoreNetworkTimeout), _firstRemoteCommandScheduler( _executor, - RemoteCommandRequest(_source, _dbname, _cmdObj, _metadata, nullptr, _timeout), + RemoteCommandRequest(_source, _dbname, _cmdObj, _metadata, nullptr, _findNetworkTimeout), stdx::bind(&Fetcher::_callback, this, stdx::placeholders::_1, kFirstBatchFieldName), std::move(firstCommandRetryPolicy)) { uassert(ErrorCodes::BadValue, "callback function cannot be null", work); @@ -205,10 +207,6 @@ BSONObj Fetcher::getMetadataObject() const { return _metadata; } -Milliseconds Fetcher::getTimeout() const { - return _timeout; -} - std::string Fetcher::toString() const { return getDiagnosticString(); } @@ -222,7 +220,8 @@ std::string Fetcher::getDiagnosticString() const { output << " query: " << _cmdObj; output << " query metadata: " << _metadata; output << " active: " << _isActive_inlock(); - output << " timeout: " << _timeout; + output << " findNetworkTimeout: " << _findNetworkTimeout; + output << " getMoreNetworkTimeout: " << _getMoreNetworkTimeout; output << " shutting down?: " << _isShuttingDown_inlock(); output << " first: " << _first; output << " firstCommandScheduler: " << _firstRemoteCommandScheduler.toString(); @@ -317,7 +316,8 @@ Status Fetcher::_scheduleGetMore(const BSONObj& cmdObj) { } StatusWith<executor::TaskExecutor::CallbackHandle> scheduleResult = _executor->scheduleRemoteCommand( - RemoteCommandRequest(_source, _dbname, cmdObj, _metadata, nullptr, _timeout), + RemoteCommandRequest( + _source, _dbname, cmdObj, _metadata, nullptr, _getMoreNetworkTimeout), stdx::bind(&Fetcher::_callback, this, stdx::placeholders::_1, kNextBatchFieldName)); if (!scheduleResult.isOK()) { diff --git a/src/mongo/client/fetcher.h b/src/mongo/client/fetcher.h index 660dbe00b5d..94b18802e1f 100644 --- a/src/mongo/client/fetcher.h +++ b/src/mongo/client/fetcher.h @@ -129,7 +129,8 @@ public: const BSONObj& cmdObj, const CallbackFn& work, const BSONObj& metadata = ReadPreferenceSetting::secondaryPreferredMetadata(), - Milliseconds timeout = RemoteCommandRequest::kNoTimeout, + Milliseconds findNetworkTimeout = RemoteCommandRequest::kNoTimeout, + Milliseconds getMoreNetworkTimeout = RemoteCommandRequest::kNoTimeout, std::unique_ptr<RemoteCommandRetryScheduler::RetryPolicy> firstCommandRetryPolicy = RemoteCommandRetryScheduler::makeNoRetryPolicy()); @@ -151,11 +152,6 @@ public: BSONObj getMetadataObject() const; /** - * Returns timeout for remote commands to complete. - */ - Milliseconds getTimeout() const; - - /** * Returns diagnostic information. */ std::string getDiagnosticString() const; @@ -259,7 +255,8 @@ private: executor::TaskExecutor::CallbackHandle _getMoreCallbackHandle; // Socket timeout - Milliseconds _timeout; + Milliseconds _findNetworkTimeout; + Milliseconds _getMoreNetworkTimeout; // First remote command scheduler. RemoteCommandRetryScheduler _firstRemoteCommandScheduler; diff --git a/src/mongo/client/fetcher_test.cpp b/src/mongo/client/fetcher_test.cpp index e68a528b041..1bc8b9008b6 100644 --- a/src/mongo/client/fetcher_test.cpp +++ b/src/mongo/client/fetcher_test.cpp @@ -243,6 +243,7 @@ TEST_F(FetcherTest, InvalidConstruction) { unreachableCallback, rpc::makeEmptyMetadata(), RemoteCommandRequest::kNoTimeout, + RemoteCommandRequest::kNoTimeout, std::unique_ptr<RemoteCommandRetryScheduler::RetryPolicy>()), AssertionException, ErrorCodes::BadValue, @@ -274,7 +275,6 @@ TEST_F(FetcherTest, RemoteCommandRequestShouldContainCommandParametersPassedToCo ASSERT_EQUALS(source, fetcher->getSource()); ASSERT_BSONOBJ_EQ(findCmdObj, fetcher->getCommandObject()); ASSERT_BSONOBJ_EQ(metadataObj, fetcher->getMetadataObject()); - ASSERT_EQUALS(timeout, fetcher->getTimeout()); ASSERT_OK(fetcher->schedule()); @@ -285,6 +285,7 @@ TEST_F(FetcherTest, RemoteCommandRequestShouldContainCommandParametersPassedToCo ASSERT_TRUE(net->hasReadyRequests()); auto noi = net->getNextReadyRequest(); request = noi->getRequest(); + ASSERT_EQUALS(timeout, request.timeout); } ASSERT_EQUALS(source, request.target); @@ -1049,6 +1050,7 @@ TEST_F(FetcherTest, FetcherAppliesRetryPolicyToFirstCommandButNotToGetMoreReques makeCallback(), rpc::makeEmptyMetadata(), executor::RemoteCommandRequest::kNoTimeout, + executor::RemoteCommandRequest::kNoTimeout, std::move(policy)); callbackHook = appendGetMoreRequest; diff --git a/src/mongo/db/repl/abstract_oplog_fetcher.cpp b/src/mongo/db/repl/abstract_oplog_fetcher.cpp index e54cbd6e7e7..a93475f4c19 100644 --- a/src/mongo/db/repl/abstract_oplog_fetcher.cpp +++ b/src/mongo/db/repl/abstract_oplog_fetcher.cpp @@ -36,6 +36,7 @@ #include "mongo/bson/util/bson_extract.h" #include "mongo/db/commands/server_status_metric.h" #include "mongo/db/jsobj.h" +#include "mongo/db/server_parameters.h" #include "mongo/stdx/memory.h" #include "mongo/stdx/mutex.h" #include "mongo/util/assert_util.h" @@ -49,6 +50,18 @@ namespace { Counter64 readersCreatedStats; ServerStatusMetricField<Counter64> displayReadersCreated("repl.network.readersCreated", &readersCreatedStats); + +// Number of seconds for the `maxTimeMS` on the initial `find` command. +MONGO_EXPORT_SERVER_PARAMETER(oplogInitialFindMaxSeconds, int, 60); + +// Number of milliseconds to add to the `find` and `getMore` timeouts to calculate the network +// timeout for the requests. +const Milliseconds kNetworkTimeoutBufferMS{5000}; + +// Default `maxTimeMS` timeout for `getMore`s. +const Milliseconds kDefaultOplogGetMoreMaxMS{5000}; + + } // namespace StatusWith<OpTimeWithHash> AbstractOplogFetcher::parseOpTimeWithHash(const BSONObj& oplogEntryObj) { @@ -66,11 +79,6 @@ StatusWith<OpTimeWithHash> AbstractOplogFetcher::parseOpTimeWithHash(const BSONO return OpTimeWithHash{hash, opTime.getValue()}; } -const Seconds AbstractOplogFetcher::kOplogInitialFindMaxTime{60}; -const Seconds AbstractOplogFetcher::kOplogGetMoreMaxTime{5}; -const Seconds AbstractOplogFetcher::kOplogQueryNetworkTimeout{ - 65}; // 5 seconds past the find command's 1 minute maxTimeMs - AbstractOplogFetcher::AbstractOplogFetcher(executor::TaskExecutor* executor, OpTimeWithHash lastFetched, HostAndPort source, @@ -89,6 +97,14 @@ AbstractOplogFetcher::AbstractOplogFetcher(executor::TaskExecutor* executor, invariant(onShutdownCallbackFn); } +Milliseconds AbstractOplogFetcher::_getFindMaxTime() const { + return Milliseconds(oplogInitialFindMaxSeconds.load() * 1000); +} + +Milliseconds AbstractOplogFetcher::_getGetMoreMaxTime() const { + return kDefaultOplogGetMoreMaxMS; +} + std::string AbstractOplogFetcher::toString() const { stdx::lock_guard<stdx::mutex> lock(_mutex); str::stream msg; @@ -310,7 +326,8 @@ std::unique_ptr<Fetcher> AbstractOplogFetcher::_makeFetcher(const BSONObj& findC stdx::bind( &AbstractOplogFetcher::_callback, this, stdx::placeholders::_1, stdx::placeholders::_3), metadataObj, - kOplogQueryNetworkTimeout); + _getFindMaxTime() + kNetworkTimeoutBufferMS, + _getGetMoreMaxTime() + kNetworkTimeoutBufferMS); } } // namespace repl diff --git a/src/mongo/db/repl/abstract_oplog_fetcher.h b/src/mongo/db/repl/abstract_oplog_fetcher.h index 97511aa272c..087e1c4ac42 100644 --- a/src/mongo/db/repl/abstract_oplog_fetcher.h +++ b/src/mongo/db/repl/abstract_oplog_fetcher.h @@ -71,14 +71,6 @@ public: using OnShutdownCallbackFn = stdx::function<void(const Status& shutdownStatus)>; /** - * Constants used to specify how long the `find` command, `getMore` commands, and network - * operations should wait before timing out. - */ - static const Seconds kOplogInitialFindMaxTime; - static const Seconds kOplogGetMoreMaxTime; - static const Seconds kOplogQueryNetworkTimeout; - - /** * This function takes a BSONObj oplog entry and parses out the OpTime and hash. */ static StatusWith<OpTimeWithHash> parseOpTimeWithHash(const BSONObj& oplogEntryObj); @@ -119,6 +111,16 @@ public: protected: /** + * Returns how long the `find` command should wait before timing out. + */ + virtual Milliseconds _getFindMaxTime() const; + + /** + * Returns how long the `getMore` command should wait before timing out. + */ + virtual Milliseconds _getGetMoreMaxTime() const; + + /** * Returns the sync source from which this oplog fetcher is fetching. */ HostAndPort _getSource() const; diff --git a/src/mongo/db/repl/collection_cloner.cpp b/src/mongo/db/repl/collection_cloner.cpp index 4626948ece9..385dd2d3dfa 100644 --- a/src/mongo/db/repl/collection_cloner.cpp +++ b/src/mongo/db/repl/collection_cloner.cpp @@ -117,7 +117,8 @@ CollectionCloner::CollectionCloner(executor::TaskExecutor* executor, stdx::placeholders::_2, stdx::placeholders::_3), ReadPreferenceSetting::secondaryPreferredMetadata(), - RemoteCommandRequest::kNoTimeout, + RemoteCommandRequest::kNoTimeout /* find network timeout */, + RemoteCommandRequest::kNoTimeout /* getMore network timeout */, RemoteCommandRetryScheduler::makeRetryPolicy( numInitialSyncListIndexesAttempts.load(), executor::RemoteCommandRequest::kNoTimeout, diff --git a/src/mongo/db/repl/database_cloner.cpp b/src/mongo/db/repl/database_cloner.cpp index f6b3cc030ba..fa9c457dcca 100644 --- a/src/mongo/db/repl/database_cloner.cpp +++ b/src/mongo/db/repl/database_cloner.cpp @@ -123,7 +123,8 @@ DatabaseCloner::DatabaseCloner(executor::TaskExecutor* executor, stdx::placeholders::_2, stdx::placeholders::_3), ReadPreferenceSetting::secondaryPreferredMetadata(), - RemoteCommandRequest::kNoTimeout, + RemoteCommandRequest::kNoTimeout /* find network timeout */, + RemoteCommandRequest::kNoTimeout /* getMore network timeout */, RemoteCommandRetryScheduler::makeRetryPolicy( numInitialSyncListCollectionsAttempts.load(), executor::RemoteCommandRequest::kNoTimeout, diff --git a/src/mongo/db/repl/initial_syncer.cpp b/src/mongo/db/repl/initial_syncer.cpp index edd114366a4..b2c21cc62d8 100644 --- a/src/mongo/db/repl/initial_syncer.cpp +++ b/src/mongo/db/repl/initial_syncer.cpp @@ -1155,7 +1155,8 @@ Status InitialSyncer::_scheduleLastOplogEntryFetcher_inlock(Fetcher::CallbackFn query, callback, ReadPreferenceSetting::secondaryPreferredMetadata(), - RemoteCommandRequest::kNoTimeout, + RemoteCommandRequest::kNoTimeout /* find network timeout */, + RemoteCommandRequest::kNoTimeout /* getMore network timeout */, RemoteCommandRetryScheduler::makeRetryPolicy( numInitialSyncOplogFindAttempts.load(), executor::RemoteCommandRequest::kNoTimeout, diff --git a/src/mongo/db/repl/oplog_fetcher.cpp b/src/mongo/db/repl/oplog_fetcher.cpp index 26303f0759d..3228c617164 100644 --- a/src/mongo/db/repl/oplog_fetcher.cpp +++ b/src/mongo/db/repl/oplog_fetcher.cpp @@ -350,8 +350,7 @@ BSONObj OplogFetcher::_makeFindCommandObject(const NamespaceString& nss, cmdBob.append("tailable", true); cmdBob.append("oplogReplay", true); cmdBob.append("awaitData", true); - cmdBob.append("maxTimeMS", - durationCount<Milliseconds>(AbstractOplogFetcher::kOplogInitialFindMaxTime)); + cmdBob.append("maxTimeMS", durationCount<Milliseconds>(_getFindMaxTime())); cmdBob.append("batchSize", _batchSize); if (term != OpTime::kUninitializedTerm) { @@ -379,6 +378,10 @@ BSONObj OplogFetcher::getMetadataObject_forTest() const { } Milliseconds OplogFetcher::getAwaitDataTimeout_forTest() const { + return _getGetMoreMaxTime(); +} + +Milliseconds OplogFetcher::_getGetMoreMaxTime() const { return _awaitDataTimeout; } @@ -501,7 +504,7 @@ StatusWith<BSONObj> OplogFetcher::_onSuccessfulBatch(const Fetcher::QueryRespons return makeGetMoreCommandObject(queryResponse.nss, queryResponse.cursorId, lastCommittedWithCurrentTerm, - _awaitDataTimeout, + _getGetMoreMaxTime(), _batchSize); } } // namespace repl diff --git a/src/mongo/db/repl/oplog_fetcher.h b/src/mongo/db/repl/oplog_fetcher.h index 9d680c59df1..b23d8023cfc 100644 --- a/src/mongo/db/repl/oplog_fetcher.h +++ b/src/mongo/db/repl/oplog_fetcher.h @@ -150,6 +150,8 @@ private: BSONObj _makeMetadataObject() const override; + Milliseconds _getGetMoreMaxTime() const override; + /** * This function is run by the AbstractOplogFetcher on a successful batch of oplog entries. */ diff --git a/src/mongo/db/repl/rollback_common_point_resolver.cpp b/src/mongo/db/repl/rollback_common_point_resolver.cpp index 34a612386cf..6dfe8ad1967 100644 --- a/src/mongo/db/repl/rollback_common_point_resolver.cpp +++ b/src/mongo/db/repl/rollback_common_point_resolver.cpp @@ -123,8 +123,7 @@ BSONObj RollbackCommonPointResolver::_makeFindCommandObject(const NamespaceStrin cmdBob.append("find", nss.coll()); cmdBob.append("filter", BSON("ts" << BSON("$lt" << lastOpTimeFetched.getTimestamp()))); cmdBob.append("sort", BSON("$natural" << -1)); - cmdBob.append("maxTimeMS", - durationCount<Milliseconds>(AbstractOplogFetcher::kOplogInitialFindMaxTime)); + cmdBob.append("maxTimeMS", durationCount<Milliseconds>(_getFindMaxTime())); return cmdBob.obj(); } @@ -250,7 +249,7 @@ StatusWith<BSONObj> RollbackCommonPointResolver::_onSuccessfulBatch( resetLocalOplogIteratorGuard.Dismiss(); return makeGetMoreCommandObject( - queryResponse.nss, queryResponse.cursorId, AbstractOplogFetcher::kOplogGetMoreMaxTime); + queryResponse.nss, queryResponse.cursorId, _getGetMoreMaxTime()); } } // namespace repl diff --git a/src/mongo/db/repl/sync_source_resolver.cpp b/src/mongo/db/repl/sync_source_resolver.cpp index 77579cc24ad..ad7f0ba0bec 100644 --- a/src/mongo/db/repl/sync_source_resolver.cpp +++ b/src/mongo/db/repl/sync_source_resolver.cpp @@ -174,7 +174,8 @@ std::unique_ptr<Fetcher> SyncSourceResolver::_makeFirstOplogEntryFetcher( candidate, earliestOpTimeSeen), ReadPreferenceSetting::secondaryPreferredMetadata(), - kFetcherTimeout); + kFetcherTimeout /* find network timeout */, + kFetcherTimeout /* getMore network timeout */); } std::unique_ptr<Fetcher> SyncSourceResolver::_makeRequiredOpTimeFetcher(HostAndPort candidate, @@ -194,7 +195,8 @@ std::unique_ptr<Fetcher> SyncSourceResolver::_makeRequiredOpTimeFetcher(HostAndP candidate, earliestOpTimeSeen), ReadPreferenceSetting::secondaryPreferredMetadata(), - kFetcherTimeout); + kFetcherTimeout /* find network timeout */, + kFetcherTimeout /* getMore network timeout */); } Status SyncSourceResolver::_scheduleFetcher(std::unique_ptr<Fetcher> fetcher) { diff --git a/src/mongo/s/client/shard_remote.cpp b/src/mongo/s/client/shard_remote.cpp index 1bc9b6e5c97..8657f88651b 100644 --- a/src/mongo/s/client/shard_remote.cpp +++ b/src/mongo/s/client/shard_remote.cpp @@ -323,7 +323,8 @@ StatusWith<Shard::QueryResponse> ShardRemote::_exhaustiveFindOnConfig( findCmdBuilder.done(), fetcherCallback, _appendMetadataForCommand(opCtx, readPrefWithMinOpTime), - maxTimeMS); + maxTimeMS /* find network timeout */, + maxTimeMS /* getMore network timeout */); Status scheduleStatus = fetcher.schedule(); if (!scheduleStatus.isOK()) { return scheduleStatus; |