SERVER-19605 make oplog timeout configurable

13 files changed, 71 insertions, 43 deletions

diff --git a/src/mongo/client/fetcher.cpp b/src/mongo/client/fetcher.cpp
index 471de980b22..7128c2800bd 100644
--- a/src/mongo/client/fetcher.cpp
+++ b/src/mongo/client/fetcher.cpp
@@ -172,7 +172,8 @@ Fetcher::Fetcher(executor::TaskExecutor* executor,
                  const BSONObj& findCmdObj,
                  const CallbackFn& work,
                  const BSONObj& metadata,
-                 Milliseconds timeout,
+                 Milliseconds findNetworkTimeout,
+                 Milliseconds getMoreNetworkTimeout,
                  std::unique_ptr<RemoteCommandRetryScheduler::RetryPolicy> firstCommandRetryPolicy)
     : _executor(executor),
       _source(source),
@@ -180,10 +181,11 @@ Fetcher::Fetcher(executor::TaskExecutor* executor,
       _cmdObj(findCmdObj.getOwned()),
       _metadata(metadata.getOwned()),
       _work(work),
-      _timeout(timeout),
+      _findNetworkTimeout(findNetworkTimeout),
+      _getMoreNetworkTimeout(getMoreNetworkTimeout),
       _firstRemoteCommandScheduler(
           _executor,
-          RemoteCommandRequest(_source, _dbname, _cmdObj, _metadata, nullptr, _timeout),
+          RemoteCommandRequest(_source, _dbname, _cmdObj, _metadata, nullptr, _findNetworkTimeout),
           stdx::bind(&Fetcher::_callback, this, stdx::placeholders::_1, kFirstBatchFieldName),
           std::move(firstCommandRetryPolicy)) {
     uassert(ErrorCodes::BadValue, "callback function cannot be null", work);
@@ -205,10 +207,6 @@ BSONObj Fetcher::getMetadataObject() const {
     return _metadata;
 }
 
-Milliseconds Fetcher::getTimeout() const {
-    return _timeout;
-}
-
 std::string Fetcher::toString() const {
     return getDiagnosticString();
 }
@@ -222,7 +220,8 @@ std::string Fetcher::getDiagnosticString() const {
     output << " query: " << _cmdObj;
     output << " query metadata: " << _metadata;
     output << " active: " << _isActive_inlock();
-    output << " timeout: " << _timeout;
+    output << " findNetworkTimeout: " << _findNetworkTimeout;
+    output << " getMoreNetworkTimeout: " << _getMoreNetworkTimeout;
     output << " shutting down?: " << _isShuttingDown_inlock();
     output << " first: " << _first;
     output << " firstCommandScheduler: " << _firstRemoteCommandScheduler.toString();
@@ -317,7 +316,8 @@ Status Fetcher::_scheduleGetMore(const BSONObj& cmdObj) {
     }
     StatusWith<executor::TaskExecutor::CallbackHandle> scheduleResult =
         _executor->scheduleRemoteCommand(
-            RemoteCommandRequest(_source, _dbname, cmdObj, _metadata, nullptr, _timeout),
+            RemoteCommandRequest(
+                _source, _dbname, cmdObj, _metadata, nullptr, _getMoreNetworkTimeout),
             stdx::bind(&Fetcher::_callback, this, stdx::placeholders::_1, kNextBatchFieldName));
 
     if (!scheduleResult.isOK()) {
diff --git a/src/mongo/client/fetcher.h b/src/mongo/client/fetcher.h
index 660dbe00b5d..94b18802e1f 100644
--- a/src/mongo/client/fetcher.h
+++ b/src/mongo/client/fetcher.h
@@ -129,7 +129,8 @@ public:
             const BSONObj& cmdObj,
             const CallbackFn& work,
             const BSONObj& metadata = ReadPreferenceSetting::secondaryPreferredMetadata(),
-            Milliseconds timeout = RemoteCommandRequest::kNoTimeout,
+            Milliseconds findNetworkTimeout = RemoteCommandRequest::kNoTimeout,
+            Milliseconds getMoreNetworkTimeout = RemoteCommandRequest::kNoTimeout,
             std::unique_ptr<RemoteCommandRetryScheduler::RetryPolicy> firstCommandRetryPolicy =
                 RemoteCommandRetryScheduler::makeNoRetryPolicy());
 
@@ -151,11 +152,6 @@ public:
     BSONObj getMetadataObject() const;
 
     /**
-     * Returns timeout for remote commands to complete.
-     */
-    Milliseconds getTimeout() const;
-
-    /**
      * Returns diagnostic information.
      */
     std::string getDiagnosticString() const;
@@ -259,7 +255,8 @@ private:
     executor::TaskExecutor::CallbackHandle _getMoreCallbackHandle;
 
     // Socket timeout
-    Milliseconds _timeout;
+    Milliseconds _findNetworkTimeout;
+    Milliseconds _getMoreNetworkTimeout;
 
     // First remote command scheduler.
     RemoteCommandRetryScheduler _firstRemoteCommandScheduler;
diff --git a/src/mongo/client/fetcher_test.cpp b/src/mongo/client/fetcher_test.cpp
index e68a528b041..1bc8b9008b6 100644
--- a/src/mongo/client/fetcher_test.cpp
+++ b/src/mongo/client/fetcher_test.cpp
@@ -243,6 +243,7 @@ TEST_F(FetcherTest, InvalidConstruction) {
                 unreachableCallback,
                 rpc::makeEmptyMetadata(),
                 RemoteCommandRequest::kNoTimeout,
+                RemoteCommandRequest::kNoTimeout,
                 std::unique_ptr<RemoteCommandRetryScheduler::RetryPolicy>()),
         AssertionException,
         ErrorCodes::BadValue,
@@ -274,7 +275,6 @@ TEST_F(FetcherTest, RemoteCommandRequestShouldContainCommandParametersPassedToCo
     ASSERT_EQUALS(source, fetcher->getSource());
     ASSERT_BSONOBJ_EQ(findCmdObj, fetcher->getCommandObject());
     ASSERT_BSONOBJ_EQ(metadataObj, fetcher->getMetadataObject());
-    ASSERT_EQUALS(timeout, fetcher->getTimeout());
 
     ASSERT_OK(fetcher->schedule());
 
@@ -285,6 +285,7 @@ TEST_F(FetcherTest, RemoteCommandRequestShouldContainCommandParametersPassedToCo
         ASSERT_TRUE(net->hasReadyRequests());
         auto noi = net->getNextReadyRequest();
         request = noi->getRequest();
+        ASSERT_EQUALS(timeout, request.timeout);
     }
 
     ASSERT_EQUALS(source, request.target);
@@ -1049,6 +1050,7 @@ TEST_F(FetcherTest, FetcherAppliesRetryPolicyToFirstCommandButNotToGetMoreReques
                                          makeCallback(),
                                          rpc::makeEmptyMetadata(),
                                          executor::RemoteCommandRequest::kNoTimeout,
+                                         executor::RemoteCommandRequest::kNoTimeout,
                                          std::move(policy));
 
     callbackHook = appendGetMoreRequest;
diff --git a/src/mongo/db/repl/abstract_oplog_fetcher.cpp b/src/mongo/db/repl/abstract_oplog_fetcher.cpp
index e54cbd6e7e7..a93475f4c19 100644
--- a/src/mongo/db/repl/abstract_oplog_fetcher.cpp
+++ b/src/mongo/db/repl/abstract_oplog_fetcher.cpp
@@ -36,6 +36,7 @@
 #include "mongo/bson/util/bson_extract.h"
 #include "mongo/db/commands/server_status_metric.h"
 #include "mongo/db/jsobj.h"
+#include "mongo/db/server_parameters.h"
 #include "mongo/stdx/memory.h"
 #include "mongo/stdx/mutex.h"
 #include "mongo/util/assert_util.h"
@@ -49,6 +50,18 @@ namespace {
 Counter64 readersCreatedStats;
 ServerStatusMetricField<Counter64> displayReadersCreated("repl.network.readersCreated",
                                                          &readersCreatedStats);
+
+// Number of seconds for the `maxTimeMS` on the initial `find` command.
+MONGO_EXPORT_SERVER_PARAMETER(oplogInitialFindMaxSeconds, int, 60);
+
+// Number of milliseconds to add to the `find` and `getMore` timeouts to calculate the network
+// timeout for the requests.
+const Milliseconds kNetworkTimeoutBufferMS{5000};
+
+// Default `maxTimeMS` timeout for `getMore`s.
+const Milliseconds kDefaultOplogGetMoreMaxMS{5000};
+
+
 }  // namespace
 
 StatusWith<OpTimeWithHash> AbstractOplogFetcher::parseOpTimeWithHash(const BSONObj& oplogEntryObj) {
@@ -66,11 +79,6 @@ StatusWith<OpTimeWithHash> AbstractOplogFetcher::parseOpTimeWithHash(const BSONO
     return OpTimeWithHash{hash, opTime.getValue()};
 }
 
-const Seconds AbstractOplogFetcher::kOplogInitialFindMaxTime{60};
-const Seconds AbstractOplogFetcher::kOplogGetMoreMaxTime{5};
-const Seconds AbstractOplogFetcher::kOplogQueryNetworkTimeout{
-    65};  // 5 seconds past the find command's 1 minute maxTimeMs
-
 AbstractOplogFetcher::AbstractOplogFetcher(executor::TaskExecutor* executor,
                                            OpTimeWithHash lastFetched,
                                            HostAndPort source,
@@ -89,6 +97,14 @@ AbstractOplogFetcher::AbstractOplogFetcher(executor::TaskExecutor* executor,
     invariant(onShutdownCallbackFn);
 }
 
+Milliseconds AbstractOplogFetcher::_getFindMaxTime() const {
+    return Milliseconds(oplogInitialFindMaxSeconds.load() * 1000);
+}
+
+Milliseconds AbstractOplogFetcher::_getGetMoreMaxTime() const {
+    return kDefaultOplogGetMoreMaxMS;
+}
+
 std::string AbstractOplogFetcher::toString() const {
     stdx::lock_guard<stdx::mutex> lock(_mutex);
     str::stream msg;
@@ -310,7 +326,8 @@ std::unique_ptr<Fetcher> AbstractOplogFetcher::_makeFetcher(const BSONObj& findC
         stdx::bind(
             &AbstractOplogFetcher::_callback, this, stdx::placeholders::_1, stdx::placeholders::_3),
         metadataObj,
-        kOplogQueryNetworkTimeout);
+        _getFindMaxTime() + kNetworkTimeoutBufferMS,
+        _getGetMoreMaxTime() + kNetworkTimeoutBufferMS);
 }
 
 }  // namespace repl
diff --git a/src/mongo/db/repl/abstract_oplog_fetcher.h b/src/mongo/db/repl/abstract_oplog_fetcher.h
index 97511aa272c..087e1c4ac42 100644
--- a/src/mongo/db/repl/abstract_oplog_fetcher.h
+++ b/src/mongo/db/repl/abstract_oplog_fetcher.h
@@ -71,14 +71,6 @@ public:
     using OnShutdownCallbackFn = stdx::function<void(const Status& shutdownStatus)>;
 
     /**
-     * Constants used to specify how long the `find` command, `getMore` commands, and network
-     * operations should wait before timing out.
-     */
-    static const Seconds kOplogInitialFindMaxTime;
-    static const Seconds kOplogGetMoreMaxTime;
-    static const Seconds kOplogQueryNetworkTimeout;
-
-    /**
      * This function takes a BSONObj oplog entry and parses out the OpTime and hash.
      */
     static StatusWith<OpTimeWithHash> parseOpTimeWithHash(const BSONObj& oplogEntryObj);
@@ -119,6 +111,16 @@ public:
 
 protected:
     /**
+     * Returns how long the `find` command should wait before timing out.
+     */
+    virtual Milliseconds _getFindMaxTime() const;
+
+    /**
+     * Returns how long the `getMore` command should wait before timing out.
+     */
+    virtual Milliseconds _getGetMoreMaxTime() const;
+
+    /**
      * Returns the sync source from which this oplog fetcher is fetching.
      */
     HostAndPort _getSource() const;
diff --git a/src/mongo/db/repl/collection_cloner.cpp b/src/mongo/db/repl/collection_cloner.cpp
index 4626948ece9..385dd2d3dfa 100644
--- a/src/mongo/db/repl/collection_cloner.cpp
+++ b/src/mongo/db/repl/collection_cloner.cpp
@@ -117,7 +117,8 @@ CollectionCloner::CollectionCloner(executor::TaskExecutor* executor,
                                      stdx::placeholders::_2,
                                      stdx::placeholders::_3),
                           ReadPreferenceSetting::secondaryPreferredMetadata(),
-                          RemoteCommandRequest::kNoTimeout,
+                          RemoteCommandRequest::kNoTimeout /* find network timeout */,
+                          RemoteCommandRequest::kNoTimeout /* getMore network timeout */,
                           RemoteCommandRetryScheduler::makeRetryPolicy(
                               numInitialSyncListIndexesAttempts.load(),
                               executor::RemoteCommandRequest::kNoTimeout,
diff --git a/src/mongo/db/repl/database_cloner.cpp b/src/mongo/db/repl/database_cloner.cpp
index f6b3cc030ba..fa9c457dcca 100644
--- a/src/mongo/db/repl/database_cloner.cpp
+++ b/src/mongo/db/repl/database_cloner.cpp
@@ -123,7 +123,8 @@ DatabaseCloner::DatabaseCloner(executor::TaskExecutor* executor,
                                          stdx::placeholders::_2,
                                          stdx::placeholders::_3),
                               ReadPreferenceSetting::secondaryPreferredMetadata(),
-                              RemoteCommandRequest::kNoTimeout,
+                              RemoteCommandRequest::kNoTimeout /* find network timeout */,
+                              RemoteCommandRequest::kNoTimeout /* getMore network timeout */,
                               RemoteCommandRetryScheduler::makeRetryPolicy(
                                   numInitialSyncListCollectionsAttempts.load(),
                                   executor::RemoteCommandRequest::kNoTimeout,
diff --git a/src/mongo/db/repl/initial_syncer.cpp b/src/mongo/db/repl/initial_syncer.cpp
index edd114366a4..b2c21cc62d8 100644
--- a/src/mongo/db/repl/initial_syncer.cpp
+++ b/src/mongo/db/repl/initial_syncer.cpp
@@ -1155,7 +1155,8 @@ Status InitialSyncer::_scheduleLastOplogEntryFetcher_inlock(Fetcher::CallbackFn
                                    query,
                                    callback,
                                    ReadPreferenceSetting::secondaryPreferredMetadata(),
-                                   RemoteCommandRequest::kNoTimeout,
+                                   RemoteCommandRequest::kNoTimeout /* find network timeout */,
+                                   RemoteCommandRequest::kNoTimeout /* getMore network timeout */,
                                    RemoteCommandRetryScheduler::makeRetryPolicy(
                                        numInitialSyncOplogFindAttempts.load(),
                                        executor::RemoteCommandRequest::kNoTimeout,
diff --git a/src/mongo/db/repl/oplog_fetcher.cpp b/src/mongo/db/repl/oplog_fetcher.cpp
index 26303f0759d..3228c617164 100644
--- a/src/mongo/db/repl/oplog_fetcher.cpp
+++ b/src/mongo/db/repl/oplog_fetcher.cpp
@@ -350,8 +350,7 @@ BSONObj OplogFetcher::_makeFindCommandObject(const NamespaceString& nss,
     cmdBob.append("tailable", true);
     cmdBob.append("oplogReplay", true);
     cmdBob.append("awaitData", true);
-    cmdBob.append("maxTimeMS",
-                  durationCount<Milliseconds>(AbstractOplogFetcher::kOplogInitialFindMaxTime));
+    cmdBob.append("maxTimeMS", durationCount<Milliseconds>(_getFindMaxTime()));
     cmdBob.append("batchSize", _batchSize);
 
     if (term != OpTime::kUninitializedTerm) {
@@ -379,6 +378,10 @@ BSONObj OplogFetcher::getMetadataObject_forTest() const {
 }
 
 Milliseconds OplogFetcher::getAwaitDataTimeout_forTest() const {
+    return _getGetMoreMaxTime();
+}
+
+Milliseconds OplogFetcher::_getGetMoreMaxTime() const {
     return _awaitDataTimeout;
 }
 
@@ -501,7 +504,7 @@ StatusWith<BSONObj> OplogFetcher::_onSuccessfulBatch(const Fetcher::QueryRespons
     return makeGetMoreCommandObject(queryResponse.nss,
                                     queryResponse.cursorId,
                                     lastCommittedWithCurrentTerm,
-                                    _awaitDataTimeout,
+                                    _getGetMoreMaxTime(),
                                     _batchSize);
 }
 }  // namespace repl
diff --git a/src/mongo/db/repl/oplog_fetcher.h b/src/mongo/db/repl/oplog_fetcher.h
index 9d680c59df1..b23d8023cfc 100644
--- a/src/mongo/db/repl/oplog_fetcher.h
+++ b/src/mongo/db/repl/oplog_fetcher.h
@@ -150,6 +150,8 @@ private:
 
     BSONObj _makeMetadataObject() const override;
 
+    Milliseconds _getGetMoreMaxTime() const override;
+
     /**
      * This function is run by the AbstractOplogFetcher on a successful batch of oplog entries.
      */
diff --git a/src/mongo/db/repl/rollback_common_point_resolver.cpp b/src/mongo/db/repl/rollback_common_point_resolver.cpp
index 34a612386cf..6dfe8ad1967 100644
--- a/src/mongo/db/repl/rollback_common_point_resolver.cpp
+++ b/src/mongo/db/repl/rollback_common_point_resolver.cpp
@@ -123,8 +123,7 @@ BSONObj RollbackCommonPointResolver::_makeFindCommandObject(const NamespaceStrin
     cmdBob.append("find", nss.coll());
     cmdBob.append("filter", BSON("ts" << BSON("$lt" << lastOpTimeFetched.getTimestamp())));
     cmdBob.append("sort", BSON("$natural" << -1));
-    cmdBob.append("maxTimeMS",
-                  durationCount<Milliseconds>(AbstractOplogFetcher::kOplogInitialFindMaxTime));
+    cmdBob.append("maxTimeMS", durationCount<Milliseconds>(_getFindMaxTime()));
     return cmdBob.obj();
 }
 
@@ -250,7 +249,7 @@ StatusWith<BSONObj> RollbackCommonPointResolver::_onSuccessfulBatch(
     resetLocalOplogIteratorGuard.Dismiss();
 
     return makeGetMoreCommandObject(
-        queryResponse.nss, queryResponse.cursorId, AbstractOplogFetcher::kOplogGetMoreMaxTime);
+        queryResponse.nss, queryResponse.cursorId, _getGetMoreMaxTime());
 }
 
 }  // namespace repl
diff --git a/src/mongo/db/repl/sync_source_resolver.cpp b/src/mongo/db/repl/sync_source_resolver.cpp
index 77579cc24ad..ad7f0ba0bec 100644
--- a/src/mongo/db/repl/sync_source_resolver.cpp
+++ b/src/mongo/db/repl/sync_source_resolver.cpp
@@ -174,7 +174,8 @@ std::unique_ptr<Fetcher> SyncSourceResolver::_makeFirstOplogEntryFetcher(
                    candidate,
                    earliestOpTimeSeen),
         ReadPreferenceSetting::secondaryPreferredMetadata(),
-        kFetcherTimeout);
+        kFetcherTimeout /* find network timeout */,
+        kFetcherTimeout /* getMore network timeout */);
 }
 
 std::unique_ptr<Fetcher> SyncSourceResolver::_makeRequiredOpTimeFetcher(HostAndPort candidate,
@@ -194,7 +195,8 @@ std::unique_ptr<Fetcher> SyncSourceResolver::_makeRequiredOpTimeFetcher(HostAndP
                    candidate,
                    earliestOpTimeSeen),
         ReadPreferenceSetting::secondaryPreferredMetadata(),
-        kFetcherTimeout);
+        kFetcherTimeout /* find network timeout */,
+        kFetcherTimeout /* getMore network timeout */);
 }
 
 Status SyncSourceResolver::_scheduleFetcher(std::unique_ptr<Fetcher> fetcher) {
diff --git a/src/mongo/s/client/shard_remote.cpp b/src/mongo/s/client/shard_remote.cpp
index 1bc9b6e5c97..8657f88651b 100644
--- a/src/mongo/s/client/shard_remote.cpp
+++ b/src/mongo/s/client/shard_remote.cpp
@@ -323,7 +323,8 @@ StatusWith<Shard::QueryResponse> ShardRemote::_exhaustiveFindOnConfig(
                     findCmdBuilder.done(),
                     fetcherCallback,
                     _appendMetadataForCommand(opCtx, readPrefWithMinOpTime),
-                    maxTimeMS);
+                    maxTimeMS /* find network timeout */,
+                    maxTimeMS /* getMore network timeout */);
     Status scheduleStatus = fetcher.schedule();
     if (!scheduleStatus.isOK()) {
         return scheduleStatus;