SERVER-15312 move replIndexPrefetch parameter out of legacy and into Applier (BGSync)

9 files changed, 233 insertions, 230 deletions

diff --git a/src/mongo/db/prefetch.cpp b/src/mongo/db/prefetch.cpp
index 21b0d15d770..0f4a6c76a77 100644
--- a/src/mongo/db/prefetch.cpp
+++ b/src/mongo/db/prefetch.cpp
@@ -26,123 +26,53 @@
 *    it in the license file.
 */
 
-#define MONGO_LOG_DEFAULT_COMPONENT ::mongo::logger::LogComponent::kQuery
+#define MONGO_LOG_DEFAULT_COMPONENT ::mongo::logger::LogComponent::kReplication
 
 #include "mongo/platform/basic.h"
 
 #include "mongo/db/prefetch.h"
 
+#include "mongo/db/catalog/collection.h"
+#include "mongo/db/commands/server_status_metric.h"
 #include "mongo/db/dbhelpers.h"
-#include "mongo/db/diskloc.h"
 #include "mongo/db/index/index_access_method.h"
-#include "mongo/db/catalog/collection.h"
 #include "mongo/db/jsobj.h"
-#include "mongo/db/repl/rs.h"
+#include "mongo/db/repl/bgsync.h"
+#include "mongo/db/repl/repl_coordinator.h"
+#include "mongo/db/repl/repl_coordinator_global.h"
+#include "mongo/db/server_parameters.h"
 #include "mongo/db/stats/timer_stats.h"
-#include "mongo/db/commands/server_status_metric.h"
 #include "mongo/util/log.h"
+#include "mongo/util/mmap.h"
 
 namespace mongo {
-
+namespace repl {
+namespace {
     // todo / idea: the prefetcher, when it fetches _id, on an upsert, will see if the record exists. if it does not, 
     //              at write time, we can just do an insert, which will be faster.
 
     //The count (of batches) and time spent fetching pages before application
     //    -- meaning depends on the prefetch behavior: all, _id index, none, etc.)
-    static TimerStats prefetchIndexStats;
-    static ServerStatusMetricField<TimerStats> displayPrefetchIndexPages(
-                                                    "repl.preload.indexes",
-                                                    &prefetchIndexStats );
-    static TimerStats prefetchDocStats;
-    static ServerStatusMetricField<TimerStats> displayPrefetchDocPages(
-                                                    "repl.preload.docs",
-                                                    &prefetchDocStats );
-
-    void prefetchIndexPages(OperationContext* txn,
-                            Collection* collection,
-                            const repl::ReplSetImpl::IndexPrefetchConfig& prefetchConfig,
-                            const BSONObj& obj);
-
-    void prefetchRecordPages(OperationContext* txn, const char* ns, const BSONObj& obj);
-
-
-    // prefetch for an oplog operation
-    void prefetchPagesForReplicatedOp(OperationContext* txn,
-                                      Database* db,
-                                      const repl::ReplSetImpl::IndexPrefetchConfig& prefetchConfig,
-                                      const BSONObj& op) {
-        const char *opField;
-        const char *opType = op.getStringField("op");
-        switch (*opType) {
-        case 'i': // insert
-        case 'd': // delete
-            opField = "o";
-            break;
-        case 'u': // update
-            opField = "o2";
-            break;
-        default:
-            // prefetch ignores other ops
-            return;
-        }
-
-        BSONObj obj = op.getObjectField(opField);
-        const char *ns = op.getStringField("ns");
-
-        Collection* collection = db->getCollection( txn, ns );
-        if ( !collection )
-            return;
-
-        LOG(4) << "index prefetch for op " << *opType << endl;
-
-        DEV txn->lockState()->assertAtLeastReadLocked(ns);
-
-        // should we prefetch index pages on updates? if the update is in-place and doesn't change 
-        // indexed values, it is actually slower - a lot slower if there are a dozen indexes or 
-        // lots of multikeys.  possible variations (not all mutually exclusive):
-        //  1) current behavior: full prefetch
-        //  2) don't do it for updates
-        //  3) don't do multikey indexes for updates
-        //  4) don't prefetchIndexPages on some heuristic; e.g., if it's an $inc.
-        //  5) if not prefetching index pages (#2), we should do it if we are upsertings and it 
-        //     will be an insert. to do that we could do the prefetchRecordPage first and if DNE
-        //     then we do #1.
-        // 
-        // note that on deletes 'obj' does not have all the keys we would want to prefetch on.
-        // a way to achieve that would be to prefetch the record first, and then afterwards do 
-        // this part.
-        //
-        prefetchIndexPages(txn, collection, prefetchConfig, obj);
-
-        // do not prefetch the data for inserts; it doesn't exist yet
-        // 
-        // we should consider doing the record prefetch for the delete op case as we hit the record
-        // when we delete.  note if done we only want to touch the first page.
-        // 
-        // update: do record prefetch. 
-        if ((*opType == 'u') &&
-            // do not prefetch the data for capped collections because
-            // they typically do not have an _id index for findById() to use.
-            !collection->isCapped()) {
-            prefetchRecordPages(txn, ns, obj);
-        }
-    }
+    TimerStats prefetchIndexStats;
+    ServerStatusMetricField<TimerStats> displayPrefetchIndexPages("repl.preload.indexes",
+                                                                  &prefetchIndexStats );
+    TimerStats prefetchDocStats;
+    ServerStatusMetricField<TimerStats> displayPrefetchDocPages("repl.preload.docs",
+                                                                &prefetchDocStats );
 
     // page in pages needed for all index lookups on a given object
     void prefetchIndexPages(OperationContext* txn,
                             Collection* collection,
-                            const repl::ReplSetImpl::IndexPrefetchConfig& prefetchConfig,
+                            const BackgroundSync::IndexPrefetchConfig& prefetchConfig,
                             const BSONObj& obj) {
-        DiskLoc unusedDl; // unused
-        BSONObjSet unusedKeys;
 
         // do we want prefetchConfig to be (1) as-is, (2) for update ops only, or (3) configured per op type?  
         // One might want PREFETCH_NONE for updates, but it's more rare that it is a bad idea for inserts.  
         // #3 (per op), a big issue would be "too many knobs".
         switch (prefetchConfig) {
-        case repl::ReplSetImpl::PREFETCH_NONE:
+        case BackgroundSync::PREFETCH_NONE:
             return;
-        case repl::ReplSetImpl::PREFETCH_ID_ONLY:
+        case BackgroundSync::PREFETCH_ID_ONLY:
         {
             TimerHolder timer( &prefetchIndexStats);
             // on the update op case, the call to prefetchRecordPages will touch the _id index.
@@ -160,7 +90,7 @@ namespace mongo {
             }
             break;
         }
-        case repl::ReplSetImpl::PREFETCH_ALL:
+        case BackgroundSync::PREFETCH_ALL:
         {
             // indexCount includes all indexes, including ones
             // in the process of being built
@@ -178,7 +108,6 @@ namespace mongo {
                 catch (const DBException& e) {
                     LOG(2) << "ignoring exception in prefetchIndexPages(): " << e.what() << endl;
                 }
-                unusedKeys.clear();
             }
             break;
         }
@@ -215,4 +144,134 @@ namespace mongo {
             }
         }
     }
-}
+} // namespace
+
+    // prefetch for an oplog operation
+    void prefetchPagesForReplicatedOp(OperationContext* txn,
+                                      Database* db,
+                                      const BSONObj& op) {
+        const BackgroundSync::IndexPrefetchConfig prefetchConfig =
+            BackgroundSync::get()->getIndexPrefetchConfig();
+        const char *opField;
+        const char *opType = op.getStringField("op");
+        switch (*opType) {
+        case 'i': // insert
+        case 'd': // delete
+            opField = "o";
+            break;
+        case 'u': // update
+            opField = "o2";
+            break;
+        default:
+            // prefetch ignores other ops
+            return;
+        }
+
+        BSONObj obj = op.getObjectField(opField);
+        const char *ns = op.getStringField("ns");
+
+        Collection* collection = db->getCollection( txn, ns );
+        if ( !collection )
+            return;
+
+        LOG(4) << "index prefetch for op " << *opType << endl;
+
+        DEV txn->lockState()->assertAtLeastReadLocked(ns);
+
+        // should we prefetch index pages on updates? if the update is in-place and doesn't change 
+        // indexed values, it is actually slower - a lot slower if there are a dozen indexes or 
+        // lots of multikeys.  possible variations (not all mutually exclusive):
+        //  1) current behavior: full prefetch
+        //  2) don't do it for updates
+        //  3) don't do multikey indexes for updates
+        //  4) don't prefetchIndexPages on some heuristic; e.g., if it's an $inc.
+        //  5) if not prefetching index pages (#2), we should do it if we are upsertings and it 
+        //     will be an insert. to do that we could do the prefetchRecordPage first and if DNE
+        //     then we do #1.
+        // 
+        // note that on deletes 'obj' does not have all the keys we would want to prefetch on.
+        // a way to achieve that would be to prefetch the record first, and then afterwards do 
+        // this part.
+        //
+        prefetchIndexPages(txn, collection, prefetchConfig, obj);
+
+        // do not prefetch the data for inserts; it doesn't exist yet
+        // 
+        // we should consider doing the record prefetch for the delete op case as we hit the record
+        // when we delete.  note if done we only want to touch the first page.
+        // 
+        // update: do record prefetch. 
+        if ((*opType == 'u') &&
+            // do not prefetch the data for capped collections because
+            // they typically do not have an _id index for findById() to use.
+            !collection->isCapped()) {
+            prefetchRecordPages(txn, ns, obj);
+        }
+    }
+
+    class ReplIndexPrefetch : public ServerParameter {
+    public:
+        ReplIndexPrefetch()
+            : ServerParameter( ServerParameterSet::getGlobal(), "replIndexPrefetch" ) {
+        }
+
+        virtual ~ReplIndexPrefetch() {
+        }
+
+        const char * _value() {
+            if (getGlobalReplicationCoordinator()->getReplicationMode() != 
+                ReplicationCoordinator::modeReplSet) {
+                return "uninitialized";
+            }
+            BackgroundSync::IndexPrefetchConfig ip = 
+                BackgroundSync::get()->getIndexPrefetchConfig();
+            switch (ip) {
+            case BackgroundSync::PREFETCH_NONE:
+                return "none";
+            case BackgroundSync::PREFETCH_ID_ONLY:
+                return "_id_only";
+            case BackgroundSync::PREFETCH_ALL:
+                return "all";
+            default:
+                return "invalid";
+            }
+        }
+
+        virtual void append(OperationContext* txn, BSONObjBuilder& b, const string& name) {
+            b.append( name, _value() );
+        }
+
+        virtual Status set( const BSONElement& newValueElement ) {
+            if (getGlobalReplicationCoordinator()->getReplicationMode() != 
+                ReplicationCoordinator::modeReplSet) {
+                return Status( ErrorCodes::BadValue, "replication is not enabled" );
+            }
+
+            std::string prefetch = newValueElement.valuestrsafe();
+            return setFromString( prefetch );
+        }
+
+        virtual Status setFromString( const string& prefetch ) {
+            log() << "changing replication index prefetch behavior to " << prefetch << endl;
+
+            BackgroundSync::IndexPrefetchConfig prefetchConfig;
+
+            if (prefetch == "none")
+                prefetchConfig = BackgroundSync::PREFETCH_NONE;
+            else if (prefetch == "_id_only")
+                prefetchConfig = BackgroundSync::PREFETCH_ID_ONLY;
+            else if (prefetch == "all")
+                prefetchConfig = BackgroundSync::PREFETCH_ALL;
+            else {
+                return Status( ErrorCodes::BadValue,
+                               str::stream() << "unrecognized indexPrefetch setting: " << prefetch );
+            }
+
+            BackgroundSync::get()->setIndexPrefetchConfig(prefetchConfig);
+            return Status::OK();
+        }
+
+    } replIndexPrefetch;
+
+} // namespace repl
+} // namespace mongo
diff --git a/src/mongo/db/prefetch.h b/src/mongo/db/prefetch.h
index 89ac800bea4..97a826092e6 100644
--- a/src/mongo/db/prefetch.h
+++ b/src/mongo/db/prefetch.h
@@ -27,18 +27,15 @@
 */
 #pragma once
 
-#include "mongo/db/jsobj.h"
-#include "mongo/db/diskloc.h"
-#include "mongo/db/repl/repl_set_impl.h"
-
 namespace mongo {
-    class Collection;
+    class BSONObj;
     class Database;
     class OperationContext;
+namespace repl {
 
-    // page in both index and data pages for an op from the oplog
+    // page in possible index and/or data pages for an op from the oplog
     void prefetchPagesForReplicatedOp(OperationContext* txn,
                                       Database* db,
-                                      const repl::ReplSetImpl::IndexPrefetchConfig& prefetchConfig,
                                       const BSONObj& op);
-}
+} // namespace repl
+} // namespace mongo
diff --git a/src/mongo/db/repl/bgsync.cpp b/src/mongo/db/repl/bgsync.cpp
index d25f2690ddf..743a7d6300e 100644
--- a/src/mongo/db/repl/bgsync.cpp
+++ b/src/mongo/db/repl/bgsync.cpp
@@ -108,7 +108,8 @@ namespace {
                                        _appliedBuffer(true),
                                        _assumingPrimary(false),
                                        _replCoord(getGlobalReplicationCoordinator()),
-                                       _initialSyncRequestedFlag(false) {
+                                       _initialSyncRequestedFlag(false),
+                                       _indexPrefetchConfig(PREFETCH_ALL) {
     }
 
     BackgroundSync* BackgroundSync::get() {
diff --git a/src/mongo/db/repl/bgsync.h b/src/mongo/db/repl/bgsync.h
index 5548204a3db..67b9a74d0d1 100644
--- a/src/mongo/db/repl/bgsync.h
+++ b/src/mongo/db/repl/bgsync.h
@@ -67,6 +67,60 @@ namespace repl {
      * 3. BackgroundSync::_mutex
      */
     class BackgroundSync : public BackgroundSyncInterface {
+    public:
+        // Allow index prefetching to be turned on/off
+        enum IndexPrefetchConfig {
+            PREFETCH_NONE=0, PREFETCH_ID_ONLY=1, PREFETCH_ALL=2
+        };
+
+        static BackgroundSync* get();
+
+        // stop syncing (when this node becomes a primary, e.g.)
+        void stop();
+        bool isAssumingPrimary_inlock();
+
+
+        void shutdown();
+        void notify();
+
+        virtual ~BackgroundSync() {}
+
+        // starts the producer thread
+        void producerThread();
+        // starts the sync target notifying thread
+        void notifierThread();
+
+        HostAndPort getSyncTarget();
+
+        // Interface implementation
+
+        virtual bool peek(BSONObj* op);
+        virtual void consume();
+        virtual void clearSyncTarget();
+        virtual void waitForMore();
+
+        // For monitoring
+        BSONObj getCounters();
+
+        // Wait for replication to finish and buffer to be applied so that the member can become
+        // primary.
+        void stopReplicationAndFlushBuffer();
+
+        long long getLastAppliedHash() const;
+        void setLastAppliedHash(long long oldH);
+        void loadLastAppliedHash(OperationContext* txn);
+
+        bool getInitialSyncRequestedFlag();
+        void setInitialSyncRequestedFlag(bool value);
+
+        void setIndexPrefetchConfig(const IndexPrefetchConfig cfg) {
+            _indexPrefetchConfig = cfg;
+        }
+
+        IndexPrefetchConfig getIndexPrefetchConfig() {
+            return _indexPrefetchConfig;
+        }
+
     private:
         static BackgroundSync *s_instance;
         // protects creation of s_instance
@@ -121,44 +175,9 @@ namespace repl {
         bool _initialSyncRequestedFlag;
         boost::mutex _initialSyncMutex;
 
-    public:
-        // stop syncing (when this node becomes a primary, e.g.)
-        void stop();
-        bool isAssumingPrimary_inlock();
-
-        static BackgroundSync* get();
-        void shutdown();
-        void notify();
-
-        virtual ~BackgroundSync() {}
-
-        // starts the producer thread
-        void producerThread();
-        // starts the sync target notifying thread
-        void notifierThread();
-
-        HostAndPort getSyncTarget();
-
-        // Interface implementation
-
-        virtual bool peek(BSONObj* op);
-        virtual void consume();
-        virtual void clearSyncTarget();
-        virtual void waitForMore();
-
-        // For monitoring
-        BSONObj getCounters();
-
-        // Wait for replication to finish and buffer to be applied so that the member can become
-        // primary.
-        void stopReplicationAndFlushBuffer();
-
-        long long getLastAppliedHash() const;
-        void setLastAppliedHash(long long oldH);
-        void loadLastAppliedHash(OperationContext* txn);
+        // This setting affects the Applier prefetcher behavior.
+        IndexPrefetchConfig _indexPrefetchConfig;
 
-        bool getInitialSyncRequestedFlag();
-        void setInitialSyncRequestedFlag(bool value);
     };
 
 
diff --git a/src/mongo/db/repl/repl_set_impl.cpp b/src/mongo/db/repl/repl_set_impl.cpp
index 186df3c2ce7..86e880151ef 100644
--- a/src/mongo/db/repl/repl_set_impl.cpp
+++ b/src/mongo/db/repl/repl_set_impl.cpp
@@ -379,21 +379,6 @@ namespace {
                       << " is not present in the current repl set config" << rsLog;
             }
         }
-
-        // Figure out indexPrefetch setting
-        std::string& prefetch = getGlobalReplicationCoordinator()->getSettings().rsIndexPrefetch;
-        if (!prefetch.empty()) {
-            IndexPrefetchConfig prefetchConfig = PREFETCH_ALL;
-            if (prefetch == "none")
-                prefetchConfig = PREFETCH_NONE;
-            else if (prefetch == "_id_only")
-                prefetchConfig = PREFETCH_ID_ONLY;
-            else if (prefetch == "all")
-                prefetchConfig = PREFETCH_ALL;
-            else
-                warning() << "unrecognized indexPrefetch setting: " << prefetch << endl;
-            setIndexPrefetchConfig(prefetchConfig);
-        }
     }
 
     ReplSetImpl::ReplSetImpl() :
@@ -402,8 +387,7 @@ namespace {
         _hbmsgTime(0),
         _self(0),
         _maintenanceMode(0),
-        mgr(0),
-        _indexPrefetchConfig(PREFETCH_ALL) {
+        mgr(0) {
     }
 
     void ReplSetImpl::loadLastOpTimeWritten(OperationContext* txn, bool quiet) {
diff --git a/src/mongo/db/repl/repl_set_impl.h b/src/mongo/db/repl/repl_set_impl.h
index 75fc8ca56e4..79c909c194e 100644
--- a/src/mongo/db/repl/repl_set_impl.h
+++ b/src/mongo/db/repl/repl_set_impl.h
@@ -281,18 +281,6 @@ namespace repl {
         // keep a list of hosts that we've tried recently that didn't work
         map<string,time_t> _veto;
 
-        // Allow index prefetching to be turned on/off
-        enum IndexPrefetchConfig {
-            PREFETCH_NONE=0, PREFETCH_ID_ONLY=1, PREFETCH_ALL=2
-        };
-
-        void setIndexPrefetchConfig(const IndexPrefetchConfig cfg) {
-            _indexPrefetchConfig = cfg;
-        }
-        IndexPrefetchConfig getIndexPrefetchConfig() {
-            return _indexPrefetchConfig;
-        }
-
         const ReplSetConfig::MemberCfg& myConfig() const { return _config; }
         const OpTime lastOtherOpTime() const;
         /**
@@ -301,8 +289,6 @@ namespace repl {
         const OpTime lastOtherElectableOpTime() const;
 
         BSONObj getLastErrorDefault;
-    private:
-        IndexPrefetchConfig _indexPrefetchConfig;
     };
 } // namespace repl
 } // namespace mongo
diff --git a/src/mongo/db/repl/rs.cpp b/src/mongo/db/repl/rs.cpp
index 026814ee39e..7644e5c9ae9 100644
--- a/src/mongo/db/repl/rs.cpp
+++ b/src/mongo/db/repl/rs.cpp
@@ -139,64 +139,5 @@ namespace repl {
         cc().getAuthorizationSession()->grantInternalAuthorization();
     }
 
-    class ReplIndexPrefetch : public ServerParameter {
-    public:
-        ReplIndexPrefetch()
-            : ServerParameter( ServerParameterSet::getGlobal(), "replIndexPrefetch" ) {
-        }
-
-        virtual ~ReplIndexPrefetch() {
-        }
-
-        const char * _value() {
-            if (!theReplSet)
-                return "uninitialized";
-            ReplSetImpl::IndexPrefetchConfig ip = theReplSet->getIndexPrefetchConfig();
-            switch (ip) {
-            case ReplSetImpl::PREFETCH_NONE:
-                return "none";
-            case ReplSetImpl::PREFETCH_ID_ONLY:
-                return "_id_only";
-            case ReplSetImpl::PREFETCH_ALL:
-                return "all";
-            default:
-                return "invalid";
-            }
-        }
-
-        virtual void append(OperationContext* txn, BSONObjBuilder& b, const string& name) {
-            b.append( name, _value() );
-        }
-
-        virtual Status set( const BSONElement& newValueElement ) {
-            if (!theReplSet) {
-                return Status( ErrorCodes::BadValue, "replication is not enabled" );
-            }
-
-            std::string prefetch = newValueElement.valuestrsafe();
-            return setFromString( prefetch );
-        }
-
-        virtual Status setFromString( const string& prefetch ) {
-            log() << "changing replication index prefetch behavior to " << prefetch << endl;
-
-            ReplSetImpl::IndexPrefetchConfig prefetchConfig;
-
-            if (prefetch == "none")
-                prefetchConfig = ReplSetImpl::PREFETCH_NONE;
-            else if (prefetch == "_id_only")
-                prefetchConfig = ReplSetImpl::PREFETCH_ID_ONLY;
-            else if (prefetch == "all")
-                prefetchConfig = ReplSetImpl::PREFETCH_ALL;
-            else {
-                return Status( ErrorCodes::BadValue,
-                               str::stream() << "unrecognized indexPrefetch setting: " << prefetch );
-            }
-
-            theReplSet->setIndexPrefetchConfig(prefetchConfig);
-            return Status::OK();
-        }
-
-    } replIndexPrefetch;
 } // namespace repl
 } // namespace mongo
diff --git a/src/mongo/db/repl/rs_sync.cpp b/src/mongo/db/repl/rs_sync.cpp
index d670de456c9..ad47956aa91 100644
--- a/src/mongo/db/repl/rs_sync.cpp
+++ b/src/mongo/db/repl/rs_sync.cpp
@@ -41,7 +41,6 @@
 #include "mongo/db/curop.h"
 #include "mongo/db/concurrency/d_concurrency.h"
 #include "mongo/db/namespace_string.h"
-#include "mongo/db/prefetch.h"
 #include "mongo/db/repl/bgsync.h"
 #include "mongo/db/repl/member.h"
 #include "mongo/db/repl/minvalid.h"
@@ -159,6 +158,24 @@ namespace repl {
         Client::initThread("rsSync");
         replLocalAuth();
         ReplicationCoordinator* replCoord = getGlobalReplicationCoordinator();
+
+        // Set initial indexPrefetch setting
+        std::string& prefetch = replCoord->getSettings().rsIndexPrefetch;
+        if (!prefetch.empty()) {
+            BackgroundSync::IndexPrefetchConfig prefetchConfig = BackgroundSync::PREFETCH_ALL;
+            if (prefetch == "none")
+                prefetchConfig = BackgroundSync::PREFETCH_NONE;
+            else if (prefetch == "_id_only")
+                prefetchConfig = BackgroundSync::PREFETCH_ID_ONLY;
+            else if (prefetch == "all")
+                prefetchConfig = BackgroundSync::PREFETCH_ALL;
+            else {
+                warning() << "unrecognized indexPrefetch setting " << prefetch << ", defaulting "
+                          << "to \"all\"";
+            }
+            BackgroundSync::get()->setIndexPrefetchConfig(prefetchConfig);
+        }
+
         while (!inShutdown()) {
             // After a reconfig, we may not be in the replica set anymore, so
             // check that we are in the set (and not an arbiter) before
diff --git a/src/mongo/db/repl/sync_tail.cpp b/src/mongo/db/repl/sync_tail.cpp
index 7de91643586..fceafc04b6e 100644
--- a/src/mongo/db/repl/sync_tail.cpp
+++ b/src/mongo/db/repl/sync_tail.cpp
@@ -150,7 +150,6 @@ namespace repl {
                 Client::ReadContext ctx(&txn, ns);
                 prefetchPagesForReplicatedOp(&txn,
                                              ctx.ctx().db(),
-                                             theReplSet->getIndexPrefetchConfig(),
                                              op);
             }
             catch (const DBException& e) {