diff options
author | Eric Milkie <milkie@10gen.com> | 2012-08-08 11:32:39 -0400 |
---|---|---|
committer | Eric Milkie <milkie@10gen.com> | 2012-08-08 11:39:46 -0400 |
commit | c7d800c733ecbfdde9e2594c11d1b25cd1e7ceeb (patch) | |
tree | 7f9ec31024c797519269c45c584b7e69ef6c2671 | |
parent | c879a4c3a04cad2f13c1ced41ac648f93dbbb839 (diff) | |
download | mongo-c7d800c733ecbfdde9e2594c11d1b25cd1e7ceeb.tar.gz |
SERVER-6718 new parameter to control replication index prefetching behavior
New parameter replIndexPrefetch.
Settable via the command line or use setParameter() command on a running server.
Only valid on a server started with --replSet parameter.
This allows a user to disable the prefetching of all index pages prior to the application
of a replicated op on a secondary.
For most use cases, prefetching all indexes for a given op will provide good performance.
For use cases that do many in-place updates to documents in a collection with many indexes
that are unaffected by such updates, using the '_id_only' setting may provide better performance
than the 'all' setting. The '_id_only' setting only prefetches index pages for the _id index,
which will always be used to find the document needed to be updated.
The 'none' setting will be used for debugging only; there should be no production use cases for
this setting.
getParameter() supports getting the current value of this parameter.
-rw-r--r-- | src/mongo/db/cmdline.h | 2 | ||||
-rw-r--r-- | src/mongo/db/db.cpp | 4 | ||||
-rw-r--r-- | src/mongo/db/dbcommands.cpp | 37 | ||||
-rw-r--r-- | src/mongo/db/dbcommands_generic.cpp | 6 | ||||
-rw-r--r-- | src/mongo/db/prefetch.cpp | 52 | ||||
-rw-r--r-- | src/mongo/db/repl/rs.cpp | 15 | ||||
-rw-r--r-- | src/mongo/db/repl/rs.h | 16 | ||||
-rw-r--r-- | src/mongo/s/commands_public.cpp | 4 |
8 files changed, 127 insertions, 9 deletions
diff --git a/src/mongo/db/cmdline.h b/src/mongo/db/cmdline.h index bbd06f62672..11a94b36988 100644 --- a/src/mongo/db/cmdline.h +++ b/src/mongo/db/cmdline.h @@ -68,6 +68,8 @@ namespace mongo { } bool usingReplSets() const { return !_replSet.empty(); } + string rsIndexPrefetch;// --indexPrefetch + // for master/slave replication string source; // --source string only; // --only diff --git a/src/mongo/db/db.cpp b/src/mongo/db/db.cpp index 2a64041e509..6a80fe9e8b4 100644 --- a/src/mongo/db/db.cpp +++ b/src/mongo/db/db.cpp @@ -701,6 +701,7 @@ static int mongoDbMain(int argc, char* argv[]) { rs_options.add_options() ("replSet", po::value<string>(), "arg is <setname>[/<optionalseedhostlist>]") + ("replIndexPrefetch", po::value<string>(), "specify index prefetching behavior (if secondary) [none|_id_only|all]") ; sharding_options.add_options() @@ -959,6 +960,9 @@ static int mongoDbMain(int argc, char* argv[]) { /* seed list of hosts for the repl set */ cmdLine._replSet = params["replSet"].as<string>().c_str(); } + if (params.count("replIndexPrefetch")) { + cmdLine.rsIndexPrefetch = params["replIndexPrefetch"].as<std::string>(); + } if (params.count("only")) { cmdLine.only = params["only"].as<string>().c_str(); } diff --git a/src/mongo/db/dbcommands.cpp b/src/mongo/db/dbcommands.cpp index 48599aa907d..c9dcb7f4144 100644 --- a/src/mongo/db/dbcommands.cpp +++ b/src/mongo/db/dbcommands.cpp @@ -66,9 +66,46 @@ namespace mongo { dur::setAgeOutJournalFiles(r); return true; } + if( cmdObj.hasElement( "replIndexPrefetch" ) ) { + if (!theReplSet) { + errmsg = "replication is not enabled"; + return false; + } + std::string prefetch = cmdObj["replIndexPrefetch"].valuestrsafe(); + log() << "changing replication index prefetch behavior to " << prefetch << endl; + // default: + ReplSetImpl::IndexPrefetchConfig prefetchConfig = ReplSetImpl::PREFETCH_ALL; + if (prefetch == "none") + prefetchConfig = ReplSetImpl::PREFETCH_NONE; + else if (prefetch == "_id_only") + prefetchConfig = ReplSetImpl::PREFETCH_ID_ONLY; + else if (prefetch == "all") + prefetchConfig = ReplSetImpl::PREFETCH_ALL; + else { + warning() << "unrecognized indexPrefetch setting: " << prefetch << endl; + } + theReplSet->setIndexPrefetchConfig(prefetchConfig); + return true; + } + return false; } + const char* fetchReplIndexPrefetchParam() { + if (!theReplSet) return "uninitialized"; + ReplSetImpl::IndexPrefetchConfig ip = theReplSet->getIndexPrefetchConfig(); + switch (ip) { + case ReplSetImpl::PREFETCH_NONE: + return "none"; + case ReplSetImpl::PREFETCH_ID_ONLY: + return "_id_only"; + case ReplSetImpl::PREFETCH_ALL: + return "all"; + default: + return "invalid"; + } + } + /* reset any errors so that getlasterror comes back clean. useful before performing a long series of operations where we want to diff --git a/src/mongo/db/dbcommands_generic.cpp b/src/mongo/db/dbcommands_generic.cpp index 84d57b8769e..ff128dbd1ff 100644 --- a/src/mongo/db/dbcommands_generic.cpp +++ b/src/mongo/db/dbcommands_generic.cpp @@ -124,6 +124,8 @@ namespace mongo { */ unsigned replApplyBatchSize = 1; + const char* fetchReplIndexPrefetchParam(); + class CmdGet : public Command { public: CmdGet() : Command( "getParameter" ) { } @@ -160,7 +162,9 @@ namespace mongo { if( all || cmdObj.hasElement("replApplyBatchSize") ) { result.append("replApplyBatchSize", replApplyBatchSize); } - + if (all || cmdObj.hasElement("replIndexPrefetch")) { + result.append("replIndexPrefetch", fetchReplIndexPrefetchParam()); + } if ( before == result.len() ) { errmsg = "no option found to get"; return false; diff --git a/src/mongo/db/prefetch.cpp b/src/mongo/db/prefetch.cpp index ee6a836e5c8..2af66b36cdf 100644 --- a/src/mongo/db/prefetch.cpp +++ b/src/mongo/db/prefetch.cpp @@ -24,6 +24,7 @@ #include "mongo/db/index_update.h" #include "mongo/db/jsobj.h" #include "mongo/db/namespace_details.h" +#include "mongo/db/repl/rs.h" namespace mongo { @@ -67,23 +68,58 @@ namespace mongo { void prefetchIndexPages(NamespaceDetails *nsd, const BSONObj& obj) { DiskLoc unusedDl; // unused IndexInterface::IndexInserter inserter; - - // includes all indexes, including ones - // in the process of being built - int indexCount = nsd->nIndexesBeingBuilt(); BSONObjSet unusedKeys; - for ( int indexNo = 0; indexNo < indexCount; indexNo++ ) { - // This will page in all index pages for the given object. + ReplSetImpl::IndexPrefetchConfig prefetchConfig = theReplSet->getIndexPrefetchConfig(); + switch (prefetchConfig) { + case ReplSetImpl::PREFETCH_NONE: + return; + case ReplSetImpl::PREFETCH_ID_ONLY: + { + int indexNo = nsd->findIdIndex(); + if (indexNo == -1) return; try { - fetchIndexInserters(/*out*/unusedKeys, inserter, nsd, indexNo, obj, unusedDl, /*allowDups*/true); + fetchIndexInserters(/*out*/unusedKeys, + inserter, + nsd, + indexNo, + obj, + unusedDl, + /*allowDups*/true); } catch (const DBException& e) { LOG(2) << "ignoring exception in prefetchIndexPages(): " << e.what() << endl; } - unusedKeys.clear(); + break; + } + case ReplSetImpl::PREFETCH_ALL: + { + // indexCount includes all indexes, including ones + // in the process of being built + int indexCount = nsd->nIndexesBeingBuilt(); + for ( int indexNo = 0; indexNo < indexCount; indexNo++ ) { + // This will page in all index pages for the given object. + try { + fetchIndexInserters(/*out*/unusedKeys, + inserter, + nsd, + indexNo, + obj, + unusedDl, + /*allowDups*/true); + } + catch (const DBException& e) { + LOG(2) << "ignoring exception in prefetchIndexPages(): " << e.what() << endl; + } + unusedKeys.clear(); + } + break; + } + default: + fassertFailed(16427); } } + void prefetchRecordPages(const char* ns, const BSONObj& obj) { BSONElement _id; if( obj.getObjectID(_id) ) { diff --git a/src/mongo/db/repl/rs.cpp b/src/mongo/db/repl/rs.cpp index 33e3115be42..0e6cffe738d 100644 --- a/src/mongo/db/repl/rs.cpp +++ b/src/mongo/db/repl/rs.cpp @@ -397,6 +397,21 @@ namespace mongo { log() << "replSet warning command line seed " << i->toString() << " is not present in the current repl set config" << rsLog; } } + + // Figure out indexPrefetch setting + std::string& prefetch = cmdLine.rsIndexPrefetch; + if (!prefetch.empty()) { + IndexPrefetchConfig prefetchConfig = PREFETCH_ALL; + if (prefetch == "none") + prefetchConfig = PREFETCH_NONE; + else if (prefetch == "_id_only") + prefetchConfig = PREFETCH_ID_ONLY; + else if (prefetch == "all") + prefetchConfig = PREFETCH_ALL; + else + warning() << "unrecognized indexPrefetch setting: " << prefetch << endl; + setIndexPrefetchConfig(prefetchConfig); + } } ReplSetImpl::ReplSetImpl() : diff --git a/src/mongo/db/repl/rs.h b/src/mongo/db/repl/rs.h index 84de1875032..e0425b42a6c 100644 --- a/src/mongo/db/repl/rs.h +++ b/src/mongo/db/repl/rs.h @@ -515,16 +515,32 @@ namespace mongo { threadpool::ThreadPool _prefetcherPool; public: + // Allow index prefetching to be turned on/off + enum IndexPrefetchConfig { + PREFETCH_NONE=0, PREFETCH_ID_ONLY=1, PREFETCH_ALL=2 + }; + + void setIndexPrefetchConfig(const IndexPrefetchConfig cfg) { + _indexPrefetchConfig = cfg; + } + IndexPrefetchConfig getIndexPrefetchConfig() { + return _indexPrefetchConfig; + } + static const int replWriterThreadCount; static const int replPrefetcherThreadCount; threadpool::ThreadPool& getPrefetchPool() { return _prefetcherPool; } threadpool::ThreadPool& getWriterPool() { return _writerPool; } + const ReplSetConfig::MemberCfg& myConfig() const { return _config; } bool tryToGoLiveAsASecondary(OpTime&); // readlocks void syncRollback(OplogReader& r); void syncThread(); const OpTime lastOtherOpTime() const; + + private: + IndexPrefetchConfig _indexPrefetchConfig; }; class ReplSet : public ReplSetImpl { diff --git a/src/mongo/s/commands_public.cpp b/src/mongo/s/commands_public.cpp index 41e0d1a5133..b23eebd2915 100644 --- a/src/mongo/s/commands_public.cpp +++ b/src/mongo/s/commands_public.cpp @@ -45,6 +45,10 @@ namespace mongo { return true; } + const char* fetchReplIndexPrefetchParam() { + return "unsupported"; + } + namespace dbgrid_pub_cmds { class PublicGridCommand : public Command { |