From d9a10c7f8bfd3f28b74bb1208188d1f1fb280c5f Mon Sep 17 00:00:00 2001 From: Eric Milkie Date: Mon, 15 Dec 2014 17:09:03 -0500 Subject: SERVER-16313 whitelist of storage options for WiredTiger --- jstests/disk/datafile_options.js | 19 +++ src/mongo/db/storage/wiredtiger/SConscript | 5 +- .../wiredtiger/wiredtiger_global_options.cpp | 163 +++++++++++++++++---- .../storage/wiredtiger/wiredtiger_global_options.h | 22 ++- .../db/storage/wiredtiger/wiredtiger_index.cpp | 20 ++- .../db/storage/wiredtiger/wiredtiger_kv_engine.cpp | 34 ++++- .../storage/wiredtiger/wiredtiger_options_init.cpp | 22 +-- .../storage/wiredtiger/wiredtiger_record_store.cpp | 16 +- src/mongo/util/options_parser/option_section.cpp | 2 +- .../util/options_parser/startup_option_init.h | 2 +- 10 files changed, 244 insertions(+), 61 deletions(-) diff --git a/jstests/disk/datafile_options.js b/jstests/disk/datafile_options.js index 62d4b137311..9bd8de40fd7 100644 --- a/jstests/disk/datafile_options.js +++ b/jstests/disk/datafile_options.js @@ -51,4 +51,23 @@ expectedResult = { testGetCmdLineOptsMongod({ config : "jstests/libs/config_files/disable_noprealloc.ini" }, expectedResult); +// Test that --syncdelay is an alias for both mmapv1 and wiredtiger options +jsTest.log("Testing syncdelay command line option"); +var expectedResult = { + "parsed" : { + "storage" : { + "mmapv1" : { + "syncPeriodSecs" : 2 + }, + "wiredTiger" : { + "engineConfig" : { + "checkpointDelaySecs" : 2 + } + } + } + } +}; +testGetCmdLineOptsMongod({ syncdelay : 2 }, expectedResult); + + print(baseName + " succeeded."); diff --git a/src/mongo/db/storage/wiredtiger/SConscript b/src/mongo/db/storage/wiredtiger/SConscript index d7966a083e4..b5394248fc2 100644 --- a/src/mongo/db/storage/wiredtiger/SConscript +++ b/src/mongo/db/storage/wiredtiger/SConscript @@ -8,8 +8,9 @@ if wiredtiger: # This is the smallest possible set of files that wraps WT wtEnv.Library( - target= 'storage_wiredtiger_core', + target='storage_wiredtiger_core', source= [ + 'wiredtiger_global_options.cpp', 'wiredtiger_index.cpp', 'wiredtiger_kv_engine.cpp', 'wiredtiger_record_store.cpp', @@ -37,7 +38,6 @@ if wiredtiger: wtEnv.Library( target='storage_wiredtiger', source=[ - 'wiredtiger_global_options.cpp', 'wiredtiger_init.cpp', 'wiredtiger_options_init.cpp', 'wiredtiger_parameters.cpp', @@ -86,4 +86,3 @@ if wiredtiger: 'storage_wiredtiger_core', ], ) - diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_global_options.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_global_options.cpp index 51b0e7bfc46..a9c42292b8b 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_global_options.cpp +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_global_options.cpp @@ -31,56 +31,159 @@ #define MONGO_LOG_DEFAULT_COMPONENT ::mongo::logger::LogComponent::kStorage #include "mongo/base/status.h" -#include "mongo/util/log.h" #include "mongo/db/storage/wiredtiger/wiredtiger_global_options.h" +#include "mongo/db/storage/wiredtiger/wiredtiger_record_store.h" +#include "mongo/util/log.h" +#include "mongo/util/options_parser/constraints.h" namespace mongo { + WiredTigerGlobalOptions wiredTigerGlobalOptions; + Status WiredTigerGlobalOptions::add(moe::OptionSection* options) { moe::OptionSection wiredTigerOptions("WiredTiger options"); - // Add WiredTiger storage engine specific options. - wiredTigerOptions.addOptionChaining("storage.wiredTiger.engineConfig", - "wiredTigerEngineConfig", moe::String, "WiredTiger storage engine configuration settings"); - wiredTigerOptions.addOptionChaining("storage.wiredTiger.collectionConfig", - "wiredTigerCollectionConfig", moe::String, "WiredTiger collection configuration settings"); - wiredTigerOptions.addOptionChaining("storage.wiredTiger.indexConfig", - "wiredTigerIndexConfig", moe::String, "WiredTiger index configuration settings"); - - wiredTigerOptions.addOptionChaining("storage.wiredTiger.directoryForIndexes", + // WiredTiger storage engine options + wiredTigerOptions.addOptionChaining("storage.wiredTiger.engineConfig.cacheSizeGB", + "wiredTigerCacheSizeGB", + moe::Int, + "maximum amount of memory to allocate for cache;" + "defaults to 1/2 of physical RAM") + .validRange(1,10000); + wiredTigerOptions.addOptionChaining("storage.wiredTiger.engineConfig.checkpointDelaySecs", + "wiredTigerCheckpointDelaySecs", + moe::Int, + "seconds to wait between each checkpoint; " + "setting this value to 0 turns off checkpoints") + .validRange(0, 100000) + .setDefault(moe::Value(60)); + wiredTigerOptions.addOptionChaining( + "storage.wiredTiger.engineConfig.statisticsLogDelaySecs", + "wiredTigerStatisticsLogDelaySecs", + moe::Int, + "seconds to wait between each write to a statistics file in the dbpath; " + "0 means do not log statistics") + .validRange(0, 100000) + .setDefault(moe::Value(0)); + wiredTigerOptions.addOptionChaining("storage.wiredTiger.engineConfig.journalCompressor", + "wiredTigerJournalCompressor", + moe::String, + "use a compressor for log records [none|snappy|zlib]") + .format("(:?none)|(:?snappy)|(:?zlib)", "(none/snappy/zlib)") + .setDefault(moe::Value(std::string("snappy"))); + wiredTigerOptions.addOptionChaining("storage.wiredTiger.engineConfig.directoryForIndexes", "wiredTigerDirectoryForIndexes", moe::Switch, - "Put indexes and data in different directories" ); + "Put indexes and data in different directories"); + wiredTigerOptions.addOptionChaining("storage.wiredTiger.engineConfig.configString", + "wiredTigerEngineConfigString", + moe::String, + "WiredTiger storage engine custom " + "configuration settings") + .hidden(); - return options->addSection(wiredTigerOptions); - } + // WiredTiger collection and index options + wiredTigerOptions.addOptionChaining("storage.wiredTiger.collectionConfig.blockCompressor", + "wiredTigerCollectionBlockCompressor", + moe::String, + "block compression algorithm for collection data " + "[none|snappy|zlib]") + .format("(:?none)|(:?snappy)|(:?zlib)", "(none/snappy/zlib)") + .setDefault(moe::Value(std::string("snappy"))); + wiredTigerOptions.addOptionChaining("storage.wiredTiger.indexConfig.blockCompressor", + "wiredTigerIndexBlockCompressor", + moe::String, + "block compression algorithm for index data " + "[none|snappy|zlib]") + .format("(:?none)|(:?snappy)|(:?zlib)", "(none/snappy/zlib)") + .setDefault(moe::Value(std::string("none"))); + + wiredTigerOptions.addOptionChaining("storage.wiredTiger.collectionConfig.prefixCompression", + "wiredTigerCollectionPrefixCompression", + moe::Bool, + "use prefix compression on row-store leaf pages") + .setDefault(moe::Value(false)); + wiredTigerOptions.addOptionChaining("storage.wiredTiger.indexConfig.prefixCompression", + "wiredTigerIndexPrefixCompression", + moe::Bool, + "use prefix compression on row-store leaf pages") + .setDefault(moe::Value(true)); - bool WiredTigerGlobalOptions::handlePreValidation(const moe::Environment& params) { - return true; + + wiredTigerOptions.addOptionChaining("storage.wiredTiger.collectionConfig.configString", + "wiredTigerCollectionConfigString", + moe::String, + "WiredTiger custom collection configuration settings") + .hidden(); + wiredTigerOptions.addOptionChaining("storage.wiredTiger.indexConfig.configString", + "wiredTigerIndexConfigString", + moe::String, + "WiredTiger custom index configuration settings") + .hidden(); + + return options->addSection(wiredTigerOptions); } Status WiredTigerGlobalOptions::store(const moe::Environment& params, - const std::vector& args) { - if (params.count("storage.wiredTiger.engineConfig")) { + const std::vector& args) { + + // WiredTiger storage engine options + if (params.count("storage.wiredTiger.engineConfig.cacheSizeGB")) { + wiredTigerGlobalOptions.cacheSizeGB = + params["storage.wiredTiger.engineConfig.cacheSizeGB"].as(); + } + if (params.count("storage.wiredTiger.engineConfig.checkpointDelaySecs")) { + wiredTigerGlobalOptions.checkpointDelaySecs = + params["storage.wiredTiger.engineConfig.checkpointDelaySecs"].as(); + } + if (params.count("storage.wiredTiger.engineConfig.statisticsLogDelaySecs")) { + wiredTigerGlobalOptions.statisticsLogDelaySecs = + params["storage.wiredTiger.engineConfig.statisticsLogDelaySecs"].as(); + } + if (params.count("storage.wiredTiger.engineConfig.journalCompressor")) { + wiredTigerGlobalOptions.journalCompressor = + params["storage.wiredTiger.engineConfig.journalCompressor"].as(); + } + if (params.count("storage.wiredTiger.engineConfig.directoryForIndexes")) { + wiredTigerGlobalOptions.directoryForIndexes = + params["storage.wiredTiger.engineConfig.directoryForIndexes"].as(); + } + if (params.count("storage.wiredTiger.engineConfig.configString")) { wiredTigerGlobalOptions.engineConfig = - params["storage.wiredTiger.engineConfig"].as(); - log() << "Engine option: " << wiredTigerGlobalOptions.engineConfig; + params["storage.wiredTiger.engineConfig.configString"].as(); + log() << "Engine custom option: " << wiredTigerGlobalOptions.engineConfig; + } + + // WiredTiger collection and index options + + if (params.count("storage.wiredTiger.collectionConfig.blockCompressor")) { + wiredTigerGlobalOptions.collectionBlockCompressor = + params["storage.wiredTiger.collectionConfig.blockCompressor"].as(); } - if (params.count("storage.wiredTiger.collectionConfig")) { + if (params.count("storage.wiredTiger.indexConfig.blockCompressor")) { + wiredTigerGlobalOptions.indexBlockCompressor = + params["storage.wiredTiger.indexConfig.blockCompressor"].as(); + } + if (params.count("storage.wiredTiger.collectionConfig.prefixCompression")) { + wiredTigerGlobalOptions.useCollectionPrefixCompression = + params["storage.wiredTiger.collectionConfig.prefixCompression"].as(); + } + if (params.count("storage.wiredTiger.indexConfig.prefixCompression")) { + wiredTigerGlobalOptions.useIndexPrefixCompression = + params["storage.wiredTiger.indexConfig.prefixCompression"].as(); + } + if (params.count("storage.wiredTiger.collectionConfig.configString")) { wiredTigerGlobalOptions.collectionConfig = - params["storage.wiredTiger.collectionConfig"].as(); - log() << "Collection option: " << wiredTigerGlobalOptions.collectionConfig; + params["storage.wiredTiger.collectionConfig.configString"].as(); + log() << "Collection custom option: " << wiredTigerGlobalOptions.collectionConfig; } - if (params.count("storage.wiredTiger.indexConfig")) { + if (params.count("storage.wiredTiger.indexConfig.configString")) { wiredTigerGlobalOptions.indexConfig = - params["storage.wiredTiger.indexConfig"].as(); - log() << "Index option: " << wiredTigerGlobalOptions.indexConfig; - } - if (params.count("storage.wiredTiger.directoryForIndexes")) { - wiredTigerGlobalOptions.directoryForIndexes = - params["storage.wiredTiger.directoryForIndexes"].as(); + params["storage.wiredTiger.indexConfig.configString"].as(); + log() << "Index custom option: " << wiredTigerGlobalOptions.indexConfig; } return Status::OK(); } -} + +} // namespace mongo diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_global_options.h b/src/mongo/db/storage/wiredtiger/wiredtiger_global_options.h index 6afa99c109b..622d3d61b03 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_global_options.h +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_global_options.h @@ -39,19 +39,33 @@ namespace mongo { class WiredTigerGlobalOptions { public: - WiredTigerGlobalOptions() : directoryForIndexes(false) {}; + WiredTigerGlobalOptions() : cacheSizeGB(0), + checkpointDelaySecs(0), + statisticsLogDelaySecs(0), + directoryForIndexes(false), + useCollectionPrefixCompression(false), + useIndexPrefixCompression(false) + {}; Status add(moe::OptionSection* options); - bool handlePreValidation(const moe::Environment& params); Status store(const moe::Environment& params, const std::vector& args); + size_t cacheSizeGB; + size_t checkpointDelaySecs; + size_t statisticsLogDelaySecs; + std::string journalCompressor; + bool directoryForIndexes; std::string engineConfig; + + std::string collectionBlockCompressor; + std::string indexBlockCompressor; + bool useCollectionPrefixCompression; + bool useIndexPrefixCompression; std::string collectionConfig; std::string indexConfig; - bool directoryForIndexes; + }; extern WiredTigerGlobalOptions wiredTigerGlobalOptions; } - diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_index.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_index.cpp index 35493f0307c..88442a40711 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_index.cpp +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_index.cpp @@ -39,6 +39,7 @@ #include "mongo/db/json.h" #include "mongo/db/catalog/index_catalog_entry.h" #include "mongo/db/index/index_descriptor.h" +#include "mongo/db/storage/wiredtiger/wiredtiger_global_options.h" #include "mongo/db/storage/wiredtiger/wiredtiger_record_store.h" #include "mongo/db/storage/wiredtiger/wiredtiger_session_cache.h" #include "mongo/db/storage/wiredtiger/wiredtiger_util.h" @@ -150,7 +151,7 @@ namespace { const WT_ITEM *b, int *cmp) { - try { + try { WiredTigerIndexCollator *c = static_cast(coll); *cmp = c->Compare(s, a, b); return 0; @@ -283,8 +284,21 @@ namespace { // Separate out a prefix and suffix in the default string. User configuration will // override values in the prefix, but not values in the suffix. - ss << "type=file,leaf_page_max=16k,prefix_compression,"; + ss << "type=file,leaf_page_max=16k,"; + if (wiredTigerGlobalOptions.useIndexPrefixCompression) { + ss << "prefix_compression,"; + } + // TODO: remove this; SERVER-16568 + std::string localIndexBlockCompressor; + if (wiredTigerGlobalOptions.indexBlockCompressor == "none") { + localIndexBlockCompressor = ""; + } + else { + localIndexBlockCompressor = wiredTigerGlobalOptions.indexBlockCompressor; + } + + ss << "block_compressor=" << localIndexBlockCompressor << ","; ss << extraConfig; // Validate configuration object. @@ -316,7 +330,7 @@ namespace { << "infoObj=" << desc.infoObj().jsonString() << "),"; - + LOG(3) << "index create string: " << ss.ss.str(); return StatusWith(ss); } diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp index ae2a564f174..7431fff7e05 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp @@ -38,6 +38,7 @@ #include "mongo/db/concurrency/write_conflict_exception.h" #include "mongo/db/index/index_descriptor.h" +#include "mongo/db/storage/wiredtiger/wiredtiger_global_options.h" #include "mongo/db/storage/wiredtiger/wiredtiger_index.h" #include "mongo/db/storage/wiredtiger/wiredtiger_record_store.h" #include "mongo/db/storage/wiredtiger/wiredtiger_recovery_unit.h" @@ -105,14 +106,14 @@ namespace mongo { _eventHandler.handle_progress = mdb_handle_progress; _eventHandler.handle_close = mdb_handle_close; - int cacheSizeGB = 1; - - { + size_t cacheSizeGB = wiredTigerGlobalOptions.cacheSizeGB; + if (cacheSizeGB == 0) { + // Since the user didn't provide a cache size, choose a reasonable default value. ProcessInfo pi; unsigned long long memSizeMB = pi.getMemSizeMB(); if ( memSizeMB > 0 ) { double cacheMB = memSizeMB / 2; - cacheSizeGB = static_cast( cacheMB / 1024 ); + cacheSizeGB = static_cast( cacheMB / 1024 ); if ( cacheSizeGB < 1 ) cacheSizeGB = 1; } @@ -139,9 +140,21 @@ namespace mongo { ss << "extensions=[local=(entry=index_collator_extension)],"; ss << "statistics=(all),"; if ( _durable ) { - ss << "log=(enabled=true,archive=true,path=journal,compressor=snappy),"; + ss << "log=(enabled=true,archive=true,path=journal,compressor="; + + // TODO: remove this; SERVER-16568 + std::string localJournalCompressor; + if (wiredTigerGlobalOptions.journalCompressor == "none") { + localJournalCompressor = ""; + } + else { + localJournalCompressor = wiredTigerGlobalOptions.journalCompressor; + } + ss << localJournalCompressor << "),"; } - ss << "checkpoint=(wait=60,log_size=2GB),"; + ss << "checkpoint=(wait=" << wiredTigerGlobalOptions.checkpointDelaySecs; + ss << ",log_size=2GB),"; + ss << "statistics_log=(wait=" << wiredTigerGlobalOptions.statisticsLogDelaySecs << "),"; ss << extraOpenOptions; string config = ss.str(); log() << "wiredtiger_open config: " << config; @@ -296,7 +309,7 @@ namespace mongo { string uri = _uri( ident ); WT_SESSION* s = session.getSession(); - LOG(1) << "WiredTigerKVEngine::createRecordStore uri: " << uri << " config: " << config; + LOG(2) << "WiredTigerKVEngine::createRecordStore uri: " << uri << " config: " << config; return wtRCToStatus( s->create( s, uri.c_str(), config.c_str() ) ); } @@ -331,7 +344,12 @@ namespace mongo { if (!result.isOK()) { return result.getStatus(); } - return wtRCToStatus(WiredTigerIndex::Create(opCtx, _uri(ident), result.getValue())); + + std::string config = result.getValue(); + + LOG(2) << "WiredTigerKVEngine::createSortedDataInterface ident: " << ident + << " config: " << config; + return wtRCToStatus(WiredTigerIndex::Create(opCtx, _uri(ident), config)); } SortedDataInterface* WiredTigerKVEngine::getSortedDataInterface( OperationContext* opCtx, diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_options_init.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_options_init.cpp index 508425b438d..e72c08ce231 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_options_init.cpp +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_options_init.cpp @@ -27,7 +27,7 @@ * exception statement from all source files in the program, then also delete * it in the license file. */ - + #include "mongo/util/options_parser/startup_option_init.h" #include "mongo/util/options_parser/startup_options.h" @@ -35,20 +35,22 @@ namespace mongo { - // Interface to MongoDB option parsing. - WiredTigerGlobalOptions wiredTigerGlobalOptions; - MONGO_MODULE_STARTUP_OPTIONS_REGISTER(WiredTigerOptions)(InitializerContext* context) { return wiredTigerGlobalOptions.add(&moe::startupOptions); } MONGO_STARTUP_OPTIONS_VALIDATE(WiredTigerOptions)(InitializerContext* context) { - if (!wiredTigerGlobalOptions.handlePreValidation(moe::startupOptionsParsed)) { - ::_exit(EXIT_SUCCESS); - } - Status ret = moe::startupOptionsParsed.validate(); - if (!ret.isOK()) { - return ret; + + // Make --syncdelay (syncPeriodSecs in mmapv1) an alias for checkpointDelaySecs in WT + moe::Value syncdelayVal; + if (moe::startupOptionsParsed.get("storage.mmapv1.syncPeriodSecs", &syncdelayVal).isOK()) { + // syncdelay is a double but checkpointDelaySecs is an int. + moe::Value newVal(static_cast(syncdelayVal.as())); + Status ret = moe::startupOptionsParsed.set( + "storage.wiredTiger.engineConfig.checkpointDelaySecs", newVal); + if (!ret.isOK()) { + return ret; + } } return Status::OK(); } diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.cpp index 660e69e104b..37510238447 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.cpp +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.cpp @@ -41,6 +41,7 @@ #include "mongo/db/namespace_string.h" #include "mongo/db/operation_context.h" #include "mongo/db/storage/oplog_hack.h" +#include "mongo/db/storage/wiredtiger/wiredtiger_global_options.h" #include "mongo/db/storage/wiredtiger/wiredtiger_recovery_unit.h" #include "mongo/db/storage/wiredtiger/wiredtiger_session_cache.h" #include "mongo/db/storage/wiredtiger/wiredtiger_size_storer.h" @@ -109,7 +110,20 @@ namespace { ss << "type=file,"; ss << "memory_page_max=100m,"; ss << "leaf_value_max=1MB,"; - ss << "block_compressor=snappy,"; + if (wiredTigerGlobalOptions.useCollectionPrefixCompression) { + ss << "prefix_compression,"; + } + + // TODO: remove this; SERVER-16568 + std::string localCollectionBlockCompressor; + if (wiredTigerGlobalOptions.collectionBlockCompressor == "none") { + localCollectionBlockCompressor = ""; + } + else { + localCollectionBlockCompressor = wiredTigerGlobalOptions.collectionBlockCompressor; + } + + ss << "block_compressor=" << localCollectionBlockCompressor << ","; ss << extraStrings << ","; diff --git a/src/mongo/util/options_parser/option_section.cpp b/src/mongo/util/options_parser/option_section.cpp index 6118334d2f8..1d9611e742d 100644 --- a/src/mongo/util/options_parser/option_section.cpp +++ b/src/mongo/util/options_parser/option_section.cpp @@ -327,7 +327,7 @@ namespace optionenvironment { << oditerator->_dottedName << "\", but trying to use it on the command line " << "or INI config file. Only options that are exclusive to the YAML config " << "file can have an empty single name"; - return Status(ErrorCodes::InternalError, oditerator->_dottedName); + return Status(ErrorCodes::InternalError, sb.str()); } boostOptions->add_options()(oditerator->_singleName.c_str(), diff --git a/src/mongo/util/options_parser/startup_option_init.h b/src/mongo/util/options_parser/startup_option_init.h index fa13a6c3007..ce8671e0e05 100644 --- a/src/mongo/util/options_parser/startup_option_init.h +++ b/src/mongo/util/options_parser/startup_option_init.h @@ -120,7 +120,7 @@ * that depends on them being separate. * * Usage: - * MONGO_STARTUP_OPTIONS_PARSE(MongodOptions)(InitializerContext* context) { + * MONGO_STARTUP_OPTIONS_STORE(MongodOptions)(InitializerContext* context) { * return Status::OK(); * } */ -- cgit v1.2.1