diff options
31 files changed, 312 insertions, 283 deletions
diff --git a/buildscripts/resmokeconfig/suites/disk.yml b/buildscripts/resmokeconfig/suites/disk.yml index 43a758412ab..159881bc87a 100644 --- a/buildscripts/resmokeconfig/suites/disk.yml +++ b/buildscripts/resmokeconfig/suites/disk.yml @@ -7,6 +7,11 @@ selector: executor: js_test: + config: + shell_options: + global_vars: + TestData: + storageEngine: mmapv1 hooks: - class: CleanEveryN n: 20 @@ -16,3 +21,4 @@ executor: nopreallocj: '' set_parameters: enableTestCommands: 1 + storageEngine: mmapv1 diff --git a/buildscripts/resmokeconfig/suites/durability.yml b/buildscripts/resmokeconfig/suites/durability.yml index 7a89ded05dd..6226858071c 100644 --- a/buildscripts/resmokeconfig/suites/durability.yml +++ b/buildscripts/resmokeconfig/suites/durability.yml @@ -11,4 +11,7 @@ executor: js_test: config: shell_options: + global_vars: + TestData: + storageEngine: mmapv1 nodb: '' diff --git a/buildscripts/resmokeconfig/suites/mmap.yml b/buildscripts/resmokeconfig/suites/mmap.yml index 82f3806e76a..57dfe9f0a10 100644 --- a/buildscripts/resmokeconfig/suites/mmap.yml +++ b/buildscripts/resmokeconfig/suites/mmap.yml @@ -5,6 +5,11 @@ selector: executor: js_test: + config: + shell_options: + global_vars: + TestData: + storageEngine: mmapv1 hooks: - class: CleanEveryN n: 20 diff --git a/etc/evergreen.yml b/etc/evergreen.yml index 11b426d5ec2..d857f161947 100644 --- a/etc/evergreen.yml +++ b/etc/evergreen.yml @@ -624,15 +624,7 @@ tasks: - func: "do setup" - func: "run tests" vars: - resmoke_args: -j8 --suites=core_auth --storageEngine=mmapv1 - -- <<: *task_template - name: jsCore_auth_WT - commands: - - func: "do setup" - - func: "run tests" - vars: - resmoke_args: -j8 --suites=core_auth --storageEngine=wiredTiger + resmoke_args: -j8 --suites=core_auth - <<: *task_template name: jsCore_op_command @@ -2118,7 +2110,7 @@ buildvariants: push_arch: i686 compile_flags: --release --distarch=i686 -j$(grep -c ^processor /proc/cpuinfo) CC=/opt/mongodbtoolchain/bin/gcc CXX=/opt/mongodbtoolchain/bin/g++ --variant-dir="linux2/release" --wiredtiger=off CCFLAGS="-m32" LINKFLAGS="-m32" num_jobs_unittests: $(grep -c ^processor /proc/cpuinfo) - test_flags: --continueOnFailure -j1 # Avoid starting too many mongod's on 32-bit systems. + test_flags: --continueOnFailure --storageEngine=mmapv1 -j1 # Avoid starting too many mongod's on 32-bit systems. has_debugsymbols: true tasks: - name: compile @@ -2155,7 +2147,7 @@ buildvariants: push_name: linux-debug push_arch: i686 num_jobs_unittests: $(grep -c ^processor /proc/cpuinfo) - test_flags: --continueOnFailure -j1 # Avoid starting too many mongod's on 32-bit systems. + test_flags: --continueOnFailure --storageEngine=mmapv1 -j1 # Avoid starting too many mongod's on 32-bit systems. has_debugsymbols: true compile_flags: --dbg=on --distarch=i686 --opt=on -j$(grep -c ^processor /proc/cpuinfo) CC=/opt/mongodbtoolchain/bin/gcc CXX=/opt/mongodbtoolchain/bin/g++ --variant-dir="linux2/debug" --wiredtiger=off CCFLAGS="-m32" LINKFLAGS="-m32" tasks: @@ -2534,7 +2526,7 @@ buildvariants: content_type: application/zip compile_flags: --release -j$(grep -c ^processor /proc/cpuinfo) --wiredtiger=off TARGET_ARCH=i386 --variant-dir=win32 num_jobs_unittests: $(grep -c ^processor /proc/cpuinfo) - test_flags: --continueOnFailure -j1 # Avoid starting too many mongod's on 32-bit systems. + test_flags: --continueOnFailure --storageEngine=mmapv1 -j1 # Avoid starting too many mongod's on 32-bit systems. ext: zip # TODO: Remove the "is_windows" expansion after PyYAML is installed on all build variants. is_windows: true diff --git a/jstests/core/apitest_dbcollection.js b/jstests/core/apitest_dbcollection.js index 6d405e8c716..1434ef3170c 100644 --- a/jstests/core/apitest_dbcollection.js +++ b/jstests/core/apitest_dbcollection.js @@ -193,8 +193,10 @@ assert(db.getCollection( "test_db" ).getIndexes().length == 0,24); 'indexDetails missing from ' + 'db.collection.stats(' + tojson(options) + ') result: ' + tojson(collectionStats)); // Currently, indexDetails is only supported with WiredTiger. - if (jsTest.options().storageEngine == undefined) { return; } - if (jsTest.options().storageEngine.toLowerCase() != "wiredtiger") { return; } + var storageEngine = jsTest.options().storageEngine; + if (storageEngine && storageEngine !== 'wiredTiger') { + return; + } assert.eq(1, Object.keys(collectionStats.indexDetails).length, 'indexDetails must have exactly one entry'); assert(collectionStats.indexDetails[indexName], diff --git a/jstests/multiVersion/downgrade_replset.js b/jstests/multiVersion/downgrade_replset.js index c3d8460eb0c..17581827f11 100644 --- a/jstests/multiVersion/downgrade_replset.js +++ b/jstests/multiVersion/downgrade_replset.js @@ -12,7 +12,7 @@ var nodes = {n1: {binVersion: newVersion}, n2: {binVersion: newVersion}, n3: {binVersion: newVersion}}; -var rst = new ReplSetTest({name: name, nodes: 3}); +var rst = new ReplSetTest({name: name, nodes: nodes, nodeOptions: {storageEngine: 'mmapv1'}}); rst.startSet(); rst.initiate(); diff --git a/jstests/multiVersion/libs/verify_collection_data.js b/jstests/multiVersion/libs/verify_collection_data.js index 9e8423c1db2..72c3e01dac7 100644 --- a/jstests/multiVersion/libs/verify_collection_data.js +++ b/jstests/multiVersion/libs/verify_collection_data.js @@ -68,83 +68,47 @@ createCollectionWithData = function (db, collectionName, dataGenerator) { // the saved state function CollectionDataValidator() { - var initialized = false; - var collectionStats = {}; - var indexData = []; - var collectionData = []; + var _initialized = false; + var _collectionInfo = {}; + var _indexData = []; + var _collectionData = []; + + // Returns the options of the specified collection. + this.getCollectionInfo = function(collection) { + var infoObj = collection.getDB().getCollectionInfos({name: collection.getName()}); + assert.eq(1, infoObj.length, "expected collection '" + collection.getName() + "'to exist"); + return infoObj[0]; + }; // Saves the current state of the collection passed in this.recordCollectionData = function (collection) { + // Save the metadata for this collection for later comparison. + _collectionInfo = this.getCollectionInfo(collection); // Save the indexes for this collection for later comparison - indexData = collection.getIndexes().sort(function(a,b) { + _indexData = collection.getIndexes().sort(function(a,b) { if (a.name > b.name) return 1; else return -1; }); // Save the data for this collection for later comparison - collectionData = collection.find().sort({"_id":1}).toArray(); - - // Save the metadata for this collection for later comparison. - // NOTE: We do this last since the data and indexes affect this output - collectionStats = collection.stats(); - - // XXX: in 2.4 avgObjSize was a double, but in 2.6 it is an int - collectionStats['avgObjSize'] = Math.floor(collectionStats['avgObjSize']); + _collectionData = collection.find().sort({"_id":1}).toArray(); - // Delete keys that appear just because we shard - delete collectionStats["primary"]; - delete collectionStats["sharded"]; - - initialized = true; + _initialized = true; return collection; } this.validateCollectionData = function (collection) { - if (!initialized) { + if (!_initialized) { throw Error("validateCollectionWithAllData called, but data is not initialized"); } // Get the metadata for this collection - var newCollectionStats = collection.stats(); - - // XXX: in 2.4 avgObjSize was a double, but in 2.6 it is an int - newCollectionStats['avgObjSize'] = Math.floor(newCollectionStats['avgObjSize']); - - // as of 2.7.1, we no longer use systemFlags - delete collectionStats.systemFlags; - delete newCollectionStats.systemFlags; - - // as of 2.7.7, we no longer use paddingFactor and introduced paddingFactorNote - delete collectionStats.paddingFactor; - delete collectionStats.paddingFactorNote; - delete newCollectionStats.paddingFactor; - delete newCollectionStats.paddingFactorNote; - - // Delete keys that appear just because we shard - delete newCollectionStats["primary"]; - delete newCollectionStats["sharded"]; - - // as of 2.7.8, we added maxSize - // TODO: when 2.6 is no longer tested, remove following two lines - delete newCollectionStats["maxSize"]; - delete collectionStats["maxSize"]; - - // Delete key added in 2.8-rc3 - delete collectionStats["indexDetails"]; - delete newCollectionStats["indexDetails"]; - - // Delete capped:false added in 2.8.0-rc5 - if (newCollectionStats["capped"] == false) { - delete newCollectionStats["capped"]; - } - if (collectionStats["capped"] == false) { - delete collectionStats["capped"]; - } + var newCollectionInfo = this.getCollectionInfo(collection); - assert.docEq(collectionStats, newCollectionStats, "collection metadata not equal"); + assert.docEq(_collectionInfo, newCollectionInfo, "collection metadata not equal"); // Get the indexes for this collection var newIndexData = collection.getIndexes().sort(function(a,b) { @@ -152,13 +116,13 @@ function CollectionDataValidator() { else return -1; }); for (var i = 0; i < newIndexData.length; i++) { - assert.docEq(indexData[i], newIndexData[i], "indexes not equal"); + assert.docEq(_indexData[i], newIndexData[i], "indexes not equal"); } // Save the data for this collection for later comparison var newCollectionData = collection.find().sort({"_id":1}).toArray(); for (var i = 0; i < newCollectionData.length; i++) { - assert.docEq(collectionData[i], newCollectionData[i], "data not equal"); + assert.docEq(_collectionData[i], newCollectionData[i], "data not equal"); } return true; } diff --git a/jstests/multiVersion/mmapv1_overrides_default_storage_engine.js b/jstests/multiVersion/mmapv1_overrides_default_storage_engine.js new file mode 100644 index 00000000000..9cad40c23bd --- /dev/null +++ b/jstests/multiVersion/mmapv1_overrides_default_storage_engine.js @@ -0,0 +1,96 @@ +/** + * Test the upgrade process for 2.6 ~~> 3.2 and 3.0 ~~> 3.2, where mmapv1 should continue to be the + * default storage engine. Repeat the process with --directoryperdb set. + */ +(function() { + 'use strict'; + + var testCases = [ + { + binVersion: '2.6', + }, + { + binVersion: '2.6', + directoryperdb: '', + }, + { + binVersion: '3.0', + }, + { + binVersion: '3.0', + directoryperdb: '', + }, + ]; + + // The mongod should start up with mmapv1 when the --storageEngine flag is omitted, or when + // --storageEngine=mmapv1 is explicitly specified. + testCases.forEach(function(testCase) { + [null, 'mmapv1'].forEach(function(storageEngine) { + jsTest.log('Upgrading from a ' + testCase.binVersion + ' instance with options=' + + tojson(testCase) + ' to the latest version. This should succeed when the' + + ' latest version ' + + (storageEngine ? ('explicitly specifies --storageEngine=' + storageEngine) + : 'omits the --storageEngine flag')); + + var dbpath = MongoRunner.dataPath + 'mmapv1_overrides_default_storage_engine'; + resetDbpath(dbpath); + + var defaultOptions = { + dbpath: dbpath, + noCleanData: true, + }; + + // Start the old version. + var mongodOptions = Object.merge(defaultOptions, testCase); + var conn = MongoRunner.runMongod(mongodOptions); + assert.neq(null, conn, + 'mongod was unable to start up with options ' + tojson(mongodOptions)); + assert.commandWorked(conn.getDB('test').runCommand({ping: 1})); + MongoRunner.stopMongod(conn); + + // Start the newest version. + mongodOptions = Object.extend({}, defaultOptions); + if (storageEngine) { + mongodOptions.storageEngine = storageEngine; + } + if (testCase.hasOwnProperty('directoryperdb')) { + mongodOptions.directoryperdb = testCase.directoryperdb; + } + conn = MongoRunner.runMongod(mongodOptions); + assert.neq(null, conn, + 'mongod was unable to start up with options ' + tojson(mongodOptions)); + assert.commandWorked(conn.getDB('test').runCommand({ping: 1})); + MongoRunner.stopMongod(conn); + }); + }); + + // The mongod should not start up when --storageEngine=wiredTiger is specified. + testCases.forEach(function(testCase) { + jsTest.log('Upgrading from a ' + testCase.binVersion + ' instance with options=' + + tojson(testCase) + ' to the latest version. This should fail when the latest' + + ' version specifies --storageEngine=wiredTiger'); + + var dbpath = MongoRunner.dataPath + 'mmapv1_overrides_default_storage_engine'; + resetDbpath(dbpath); + + var defaultOptions = { + dbpath: dbpath, + noCleanData: true, + }; + + // Start the old version. + var mongodOptions = Object.merge(defaultOptions, testCase); + var conn = MongoRunner.runMongod(mongodOptions); + assert.neq(null, conn, + 'mongod was unable to start up with options ' + tojson(mongodOptions)); + assert.commandWorked(conn.getDB('test').runCommand({ping: 1})); + MongoRunner.stopMongod(conn); + + // Start the newest version. + mongodOptions = Object.extend({storageEngine: 'wiredTiger'}, defaultOptions); + conn = MongoRunner.runMongod(mongodOptions); + assert.eq(null, conn, + 'mongod should not have been able to start up with options ' + + tojson(mongodOptions)); + }); +}()); diff --git a/jstests/noPassthrough/dir_per_db_and_split.js b/jstests/noPassthrough/dir_per_db_and_split.js index e03ad7e6746..8047ec9fda2 100644 --- a/jstests/noPassthrough/dir_per_db_and_split.js +++ b/jstests/noPassthrough/dir_per_db_and_split.js @@ -1,5 +1,5 @@ -if ( jsTest.options().storageEngine == "wiredTiger" ) { +if (!jsTest.options().storageEngine || jsTest.options().storageEngine === "wiredTiger") { var baseDir = "jstests_per_db_and_split_c_and_i"; port = allocatePorts( 1 )[ 0 ]; diff --git a/jstests/noPassthrough/split_collections_and_indexes.js b/jstests/noPassthrough/split_collections_and_indexes.js index 41916d6e9a0..73d2eede111 100644 --- a/jstests/noPassthrough/split_collections_and_indexes.js +++ b/jstests/noPassthrough/split_collections_and_indexes.js @@ -1,5 +1,5 @@ -if ( jsTest.options().storageEngine == "wiredTiger" ) { +if (!jsTest.options().storageEngine || jsTest.options().storageEngine === "wiredTiger") { var baseDir = "jstests_split_c_and_i"; port = allocatePorts( 1 )[ 0 ]; diff --git a/jstests/noPassthrough/wt_nojournal_fsync.js b/jstests/noPassthrough/wt_nojournal_fsync.js index 5673e1208ee..46a881de16d 100644 --- a/jstests/noPassthrough/wt_nojournal_fsync.js +++ b/jstests/noPassthrough/wt_nojournal_fsync.js @@ -30,10 +30,7 @@ function writeDataAndRestart(doFsync) { } // This test can only be run if the storageEngine is wiredTiger -// This check will have to change when we change the default storageEngine -if ( typeof(TestData) != "object" || - !TestData.storageEngine || - TestData.storageEngine != "wiredTiger" ) { +if (jsTest.options().storageEngine && jsTest.options().storageEngine !== "wiredTiger") { jsTestLog("Skipping test because storageEngine is not wiredTiger"); } else { diff --git a/jstests/noPassthrough/wt_nojournal_repl.js b/jstests/noPassthrough/wt_nojournal_repl.js index 79ec8f797a2..01bd23b10da 100644 --- a/jstests/noPassthrough/wt_nojournal_repl.js +++ b/jstests/noPassthrough/wt_nojournal_repl.js @@ -26,9 +26,7 @@ var contains = function(logLines, func) { } // This test can only be run if the storageEngine is wiredTiger -if ( typeof(TestData) != "object" || - !TestData.storageEngine || - TestData.storageEngine != "wiredTiger" ) { +if (jsTest.options().storageEngine && jsTest.options().storageEngine !== "wiredTiger") { jsTestLog("Skipping test because storageEngine is not wiredTiger"); } else { diff --git a/jstests/noPassthroughWithMongod/index_check10.js b/jstests/noPassthroughWithMongod/index_check10.js index f507c687731..5ace6951652 100644 --- a/jstests/noPassthroughWithMongod/index_check10.js +++ b/jstests/noPassthroughWithMongod/index_check10.js @@ -135,7 +135,7 @@ function doIt( indexVersion ) { for( var z = 0; z < 5; ++z ) { var indexVersion = z % 2; var storageEngine = jsTest.options().storageEngine; - if (!storageEngine || storageEngine === 'mmapv1' || indexVersion !== 0) { + if (storageEngine === 'mmapv1' || indexVersion !== 0) { doIt(indexVersion); } } diff --git a/jstests/replsets/initSyncV1Index.js b/jstests/replsets/initSyncV1Index.js index c3daa471b44..10b91949942 100644 --- a/jstests/replsets/initSyncV1Index.js +++ b/jstests/replsets/initSyncV1Index.js @@ -6,7 +6,7 @@ 'use strict'; var storageEngine = jsTest.options().storageEngine; - if (storageEngine && storageEngine !== 'mmapv1') { + if (storageEngine !== 'mmapv1') { return; } diff --git a/jstests/sharding/stats.js b/jstests/sharding/stats.js index 6c21e3861da..52ec40556d7 100644 --- a/jstests/sharding/stats.js +++ b/jstests/sharding/stats.js @@ -132,7 +132,8 @@ collStatComp(coll_not_scaled, coll_scaled_1024, 1024, true); assert.commandWorked(t.ensureIndex({a: 1})); assert.eq(2, t.getIndexes().length); - var isWiredTiger = (jsTest.options().storageEngine == "wiredTiger"); + var isWiredTiger = (!jsTest.options().storageEngine + || jsTest.options().storageEngine === "wiredTiger"); var stats = assert.commandWorked(t.stats({indexDetails: true})); var shardName; diff --git a/jstests/sharding/user_flags_sharded.js b/jstests/sharding/user_flags_sharded.js index 2d7a341e4f4..e5b5f8a41dd 100644 --- a/jstests/sharding/user_flags_sharded.js +++ b/jstests/sharding/user_flags_sharded.js @@ -1,9 +1,7 @@ // Test that when user flags are set on a collection, // then collection is sharded, flags get carried over. -if ( typeof(TestData) != "object" || - !TestData.storageEngine || - TestData.storageEngine == "mmapv1" ) { +if (jsTest.options().storageEngine === "mmapv1") { // the dbname and collection we'll be working with var dbname = "testDB"; diff --git a/src/mongo/db/db.cpp b/src/mongo/db/db.cpp index fa360169fa0..d7e69a0c5ca 100644 --- a/src/mongo/db/db.cpp +++ b/src/mongo/db/db.cpp @@ -34,6 +34,7 @@ #include <boost/thread/thread.hpp> #include <boost/filesystem/operations.hpp> +#include <boost/optional.hpp> #include <boost/shared_ptr.hpp> #include <fstream> #include <iostream> @@ -417,6 +418,8 @@ namespace mongo { dbWebServer->setupSockets(); } + getGlobalServiceContext()->initializeGlobalStorageEngine(); + // Warn if we detect configurations for multiple registered storage engines in // the same configuration file/environment. if (serverGlobalParams.parsedOpts.hasField("storage")) { @@ -441,7 +444,6 @@ namespace mongo { } } - getGlobalServiceContext()->setGlobalStorageEngine(storageGlobalParams.engine); getGlobalServiceContext()->setOpObserver(stdx::make_unique<OpObserver>()); const repl::ReplSettings& replSettings = diff --git a/src/mongo/db/mongod_options.cpp b/src/mongo/db/mongod_options.cpp index bb283121bf3..a2daab9c201 100644 --- a/src/mongo/db/mongod_options.cpp +++ b/src/mongo/db/mongod_options.cpp @@ -165,8 +165,7 @@ namespace mongo { // Storage Options storage_options.addOptionChaining("storage.engine", "storageEngine", moe::String, - "what storage engine to use") - .setDefault(moe::Value(std::string("mmapv1"))); + "what storage engine to use - defaults to wiredTiger if no data files present"); #ifdef _WIN32 @@ -894,7 +893,10 @@ namespace mongo { "files"); } - storageGlobalParams.engine = params["storage.engine"].as<string>(); + if (params.count("storage.engine")) { + storageGlobalParams.engine = params["storage.engine"].as<string>(); + storageGlobalParams.engineSetByUser = true; + } if (params.count("storage.dbPath")) { storageGlobalParams.dbpath = params["storage.dbPath"].as<string>(); diff --git a/src/mongo/db/repl/sync_tail_test.cpp b/src/mongo/db/repl/sync_tail_test.cpp index 1f2dc692252..9e20ce78818 100644 --- a/src/mongo/db/repl/sync_tail_test.cpp +++ b/src/mongo/db/repl/sync_tail_test.cpp @@ -103,7 +103,9 @@ namespace { // go away after the global storage engine is initialized. unittest::TempDir tempDir("sync_tail_test"); mongo::storageGlobalParams.dbpath = tempDir.path(); - serviceContext->setGlobalStorageEngine("devnull"); + mongo::storageGlobalParams.engine = "devnull"; + mongo::storageGlobalParams.engineSetByUser = true; + serviceContext->initializeGlobalStorageEngine(); } _prevCoordinator = getGlobalReplicationCoordinator(); ReplSettings replSettings; diff --git a/src/mongo/db/service_context.h b/src/mongo/db/service_context.h index 40cee393d25..3833ff69d85 100644 --- a/src/mongo/db/service_context.h +++ b/src/mongo/db/service_context.h @@ -197,10 +197,7 @@ namespace mongo { */ virtual StorageFactoriesIterator* makeStorageFactoriesIterator() = 0; - /** - * Set the storage engine. The engine must have been registered via registerStorageEngine. - */ - virtual void setGlobalStorageEngine(const std::string& name) = 0; + virtual void initializeGlobalStorageEngine() = 0; /** * Shuts down storage engine cleanly and releases any locks on mongod.lock. diff --git a/src/mongo/db/service_context_d.cpp b/src/mongo/db/service_context_d.cpp index 83427b0cef2..eaef2f6f639 100644 --- a/src/mongo/db/service_context_d.cpp +++ b/src/mongo/db/service_context_d.cpp @@ -32,6 +32,8 @@ #include "mongo/db/service_context_d.h" +#include <boost/optional.hpp> + #include "mongo/base/init.h" #include "mongo/base/initializer.h" #include "mongo/db/client.h" @@ -45,6 +47,7 @@ #include "mongo/scripting/engine.h" #include "mongo/stdx/memory.h" #include "mongo/util/log.h" +#include "mongo/util/map_util.h" #include "mongo/util/mongoutils/str.h" #include "mongo/util/scopeguard.h" @@ -72,21 +75,52 @@ namespace mongo { extern bool _supportsDocLocking; - void ServiceContextMongoD::setGlobalStorageEngine(const std::string& name) { + void ServiceContextMongoD::initializeGlobalStorageEngine() { // This should be set once. invariant(!_storageEngine); - const StorageEngine::Factory* factory = _storageFactories[name]; + const std::string dbpath = storageGlobalParams.dbpath; + if (auto existingStorageEngine = StorageEngineMetadata::getStorageEngineForPath(dbpath)) { + if (storageGlobalParams.engineSetByUser) { + // Verify that the name of the user-supplied storage engine matches the contents of + // the metadata file. + const StorageEngine::Factory* factory = mapFindWithDefault( + _storageFactories, + storageGlobalParams.engine, + static_cast<const StorageEngine::Factory*>(nullptr)); + + if (factory) { + uassert(28662, str::stream() + << "Cannot start server. Detected data files in " << dbpath << " created by" + << " the '" << *existingStorageEngine << "' storage engine, but the" + << " specified storage engine was '" << factory->getCanonicalName() << "'.", + factory->getCanonicalName() == *existingStorageEngine); + } + } + else { + // Otherwise set the active storage engine as the contents of the metadata file. + log() << "Detected data files in " << dbpath << " created by the '" + << *existingStorageEngine << "' storage engine, so setting the active" + << " storage engine to '" << *existingStorageEngine << "'."; + storageGlobalParams.engine = *existingStorageEngine; + } + } + else if (!storageGlobalParams.engineSetByUser) { + // Ensure the default storage engine is available with this build of mongod. + uassert(28663, str::stream() + << "Cannot start server. The default storage engine '" << storageGlobalParams.engine + << "' is not available with this build of mongod. Please specify a different" + << " storage engine explicitly, e.g. --storageEngine=mmapv1.", + isRegisteredStorageEngine(storageGlobalParams.engine)); + } + + const StorageEngine::Factory* factory = _storageFactories[storageGlobalParams.engine]; uassert(18656, str::stream() - << "Cannot start server with an unknown storage engine: " << name, + << "Cannot start server with an unknown storage engine: " << storageGlobalParams.engine, factory); - std::string canonicalName = factory->getCanonicalName().toString(); - - // Do not proceed if data directory has been used by a different storage engine previously. - std::auto_ptr<StorageEngineMetadata> metadata = - StorageEngineMetadata::validate(storageGlobalParams.dbpath, canonicalName); + std::unique_ptr<StorageEngineMetadata> metadata = StorageEngineMetadata::forPath(dbpath); // Validate options in metadata against current startup options. if (metadata.get()) { @@ -116,7 +150,7 @@ namespace mongo { // Write a new metadata file if it is not present. if (!metadata.get()) { metadata.reset(new StorageEngineMetadata(storageGlobalParams.dbpath)); - metadata->setStorageEngine(canonicalName); + metadata->setStorageEngine(factory->getCanonicalName().toString()); metadata->setStorageEngineOptions(factory->createMetadataOptions(storageGlobalParams)); uassertStatusOK(metadata->write()); } diff --git a/src/mongo/db/service_context_d.h b/src/mongo/db/service_context_d.h index caa6ba17bdf..9437d37aa07 100644 --- a/src/mongo/db/service_context_d.h +++ b/src/mongo/db/service_context_d.h @@ -49,7 +49,7 @@ namespace mongo { StorageEngine* getGlobalStorageEngine(); - void setGlobalStorageEngine(const std::string& name); + void initializeGlobalStorageEngine(); void shutdownGlobalStorageEngineCleanly(); diff --git a/src/mongo/db/service_context_noop.cpp b/src/mongo/db/service_context_noop.cpp index 8b43c4e1bbf..47c3efe8797 100644 --- a/src/mongo/db/service_context_noop.cpp +++ b/src/mongo/db/service_context_noop.cpp @@ -39,7 +39,7 @@ namespace mongo { return NULL; } - void ServiceContextNoop::setGlobalStorageEngine(const std::string& name) { + void ServiceContextNoop::initializeGlobalStorageEngine() { } void ServiceContextNoop::shutdownGlobalStorageEngineCleanly() { diff --git a/src/mongo/db/service_context_noop.h b/src/mongo/db/service_context_noop.h index 679b683bb74..b7f541b784f 100644 --- a/src/mongo/db/service_context_noop.h +++ b/src/mongo/db/service_context_noop.h @@ -34,7 +34,7 @@ namespace mongo { public: StorageEngine* getGlobalStorageEngine(); - void setGlobalStorageEngine(const std::string& name); + void initializeGlobalStorageEngine(); void shutdownGlobalStorageEngineCleanly(); diff --git a/src/mongo/db/storage/storage_engine_metadata.cpp b/src/mongo/db/storage/storage_engine_metadata.cpp index 03dce664bd1..2881c41d689 100644 --- a/src/mongo/db/storage/storage_engine_metadata.cpp +++ b/src/mongo/db/storage/storage_engine_metadata.cpp @@ -34,6 +34,7 @@ #include <cstdio> #include <boost/filesystem.hpp> +#include <boost/optional.hpp> #include <fstream> #include <limits> #include <ostream> @@ -62,42 +63,33 @@ namespace { } // namespace // static - std::auto_ptr<StorageEngineMetadata> StorageEngineMetadata::validate( - const std::string& dbpath, - const std::string& storageEngine) { - - std::auto_ptr<StorageEngineMetadata> metadata; - std::string previousStorageEngine; + std::unique_ptr<StorageEngineMetadata> StorageEngineMetadata::forPath( + const std::string& dbpath) { + std::unique_ptr<StorageEngineMetadata> metadata; if (boost::filesystem::exists(boost::filesystem::path(dbpath) / kMetadataBasename)) { metadata.reset(new StorageEngineMetadata(dbpath)); Status status = metadata->read(); - if (status.isOK()) { - previousStorageEngine = metadata->getStorageEngine(); - } - else { - // The storage metadata file is present but there was an issue - // reading its contents. - warning() << "Unable to read the existing storage engine metadata: " - << status.toString(); - return std::auto_ptr<StorageEngineMetadata>(); + if (!status.isOK()) { + error() << "Unable to read the storage engine metadata file: " << status; + fassertFailed(28661); } } - else if (containsMMapV1LocalNsFile(dbpath)) { - previousStorageEngine = "mmapv1"; - } - else { - // Directory contains neither metadata nor mmapv1 files. - // Allow validation to succeed. - return metadata; - } + return metadata; + } - uassert(28574, str::stream() - << "Cannot start server. Detected data files in " << dbpath - << " created by storage engine '" << previousStorageEngine - << "'. The configured storage engine is '" << storageEngine << "'.", - previousStorageEngine == storageEngine); + // static + boost::optional<std::string> StorageEngineMetadata::getStorageEngineForPath( + const std::string& dbpath) { + if (auto metadata = StorageEngineMetadata::forPath(dbpath)) { + return {metadata->getStorageEngine()}; + } - return metadata; + // Fallback to checking for MMAPv1-specific files to handle upgrades from before the + // storage.bson metadata file was introduced in 3.0. + if (containsMMapV1LocalNsFile(dbpath)) { + return {std::string("mmapv1")}; + } + return {}; } StorageEngineMetadata::StorageEngineMetadata(const std::string& dbpath) diff --git a/src/mongo/db/storage/storage_engine_metadata.h b/src/mongo/db/storage/storage_engine_metadata.h index 4f87f2fc50c..a4dafdf9bfa 100644 --- a/src/mongo/db/storage/storage_engine_metadata.h +++ b/src/mongo/db/storage/storage_engine_metadata.h @@ -28,6 +28,7 @@ #pragma once +#include <boost/optional.hpp> #include <memory> #include <string> @@ -49,23 +50,16 @@ namespace mongo { public: /** - * Validates metadata in data directory against current storage engine. - * 1) If the metadata file exists, ensure that the information in the file - * is consistent with the current storage engine. Otherwise, raise an error. - * Returns the metadata object on successful validation. - * 2) If the metadata file exists but is not readable (eg. corrupted), - * return NULL. This allows the startup process to overwrite the corrupted - * metadata file with a valid copy. - * 3) If the metadata file does not exist, look for local.ns or local/local.ns - * in the data directory. If we detect either file, raise an error - * only if the current storage engine is not 'mmapv1'. - * This makes validation more forgiving of situations where - * application data is placed in the data directory prior - * to server start up. - * Returns NULL on successful validation. + * Returns a metadata object describing the storage engine that backs the data files + * contained in 'dbpath', and nullptr otherwise. */ - static std::auto_ptr<StorageEngineMetadata> validate(const std::string& dbpath, - const std::string& storageEngine); + static std::unique_ptr<StorageEngineMetadata> forPath(const std::string& dbpath); + + /** + * Returns the name of the storage engine that backs the data files contained in 'dbpath', + * and none otherwise. + */ + static boost::optional<std::string> getStorageEngineForPath(const std::string& dbpath); /** * Sets fields to defaults. diff --git a/src/mongo/db/storage/storage_engine_metadata_test.cpp b/src/mongo/db/storage/storage_engine_metadata_test.cpp index afb76972082..27508dfe7a5 100644 --- a/src/mongo/db/storage/storage_engine_metadata_test.cpp +++ b/src/mongo/db/storage/storage_engine_metadata_test.cpp @@ -29,6 +29,8 @@ #include "mongo/platform/basic.h" #include <boost/filesystem.hpp> +#include <boost/optional.hpp> +#include <boost/optional/optional_io.hpp> #include <fstream> #include <ios> #include <ostream> @@ -197,169 +199,105 @@ namespace { } } - TEST(StorageEngineMetadataTest, ValidateEmptyDirectory) { - TempDir tempDir("StorageEngineMetadataTest_ValidateEmptyDirectory"); - std::auto_ptr<StorageEngineMetadata> metadata = - StorageEngineMetadata::validate(tempDir.path(), "storageEngine1"); - ASSERT_FALSE(metadata.get()); - } + TEST(StorageEngineMetadataTest, ValidateStorageEngineOption) { + // It is fine to provide an invalid data directory as long as we do not + // call read() or write(). + StorageEngineMetadata metadata("no_such_directory"); + BSONObj options = fromjson("{x: true, y: false, z: 123}"); + metadata.setStorageEngineOptions(options); - // Data directory is not empty but metadata is missing. - // Data directory contains local.ns. - // Current storage engine is 'mmapv1'. - TEST(StorageEngineMetadataTest, ValidateMissingMetadataStorageEngineMMapV1) { - TempDir tempDir("StorageEngineMetadataTest_ValidateMissingMetadataStorageEngineMMapV1"); - { - std::string filename(tempDir.path() + "/local.ns"); - std::ofstream ofs(filename.c_str()); - ofs << "unused data" << std::endl; - } - std::auto_ptr<StorageEngineMetadata> metadata = - StorageEngineMetadata::validate(tempDir.path(), "mmapv1"); - ASSERT_FALSE(metadata.get()); + // Non-existent field. + ASSERT_OK(metadata.validateStorageEngineOption("w", true)); + ASSERT_OK(metadata.validateStorageEngineOption("w", false)); + + // Non-boolean field. + Status status = metadata.validateStorageEngineOption("z", true); + ASSERT_NOT_OK(status); + ASSERT_EQUALS(ErrorCodes::FailedToParse, status.code()); + status = metadata.validateStorageEngineOption("z", false); + ASSERT_NOT_OK(status); + ASSERT_EQUALS(ErrorCodes::FailedToParse, status.code()); + + // Boolean fields. + ASSERT_OK(metadata.validateStorageEngineOption("x", true)); + status = metadata.validateStorageEngineOption("x", false); + ASSERT_NOT_OK(status); + ASSERT_EQUALS(ErrorCodes::InvalidOptions, status.code()); + + ASSERT_OK(metadata.validateStorageEngineOption("y", false)); + status = metadata.validateStorageEngineOption("y", true); + ASSERT_NOT_OK(status); + ASSERT_EQUALS(ErrorCodes::InvalidOptions, status.code()); } - // Data directory is not empty but metadata is missing. - // Data directory contains local.ns. - // Current storage engine is not 'mmapv1'. - TEST(StorageEngineMetadataTest, ValidateMissingMetadataStorageEngineNotMMapV1) { - TempDir tempDir("StorageEngineMetadataTest_ValidateMissingMetadataStorageEngineNotMMapV1"); - { - std::string filename(tempDir.path() + "/local.ns"); - std::ofstream ofs(filename.c_str()); - ofs << "unused data" << std::endl; - } - ASSERT_THROWS(StorageEngineMetadata::validate(tempDir.path(), "engine1"), UserException); + // Do not override the active storage engine when the data directory is empty. + TEST(StorageEngineMetadataTest, StorageEngineForPath_EmptyDirectory) { + TempDir tempDir("StorageEngineMetadataTest_StorageEngineForPath_EmptyDirectory"); + auto storageEngine = StorageEngineMetadata::getStorageEngineForPath(tempDir.path()); + ASSERT_FALSE(storageEngine); } - // Data directory is not empty but metadata is missing. - // Data directory contains local/local.ns. - // Current storage engine is 'mmapv1'. - TEST(StorageEngineMetadataTest, ValidateMissingMetadataDirectoryPerDb1) { - TempDir tempDir("StorageEngineMetadataTest_ValidateMissingMetadataDirectoryPerDb1"); + // Override the active storage engine with "mmapv1" when the data directory contains local.ns. + TEST(StorageEngineMetadataTest, StorageEngineForPath_DataFilesExist) { + TempDir tempDir("StorageEngineMetadataTest_StorageEngineForPath_DataFilesExist"); { - boost::filesystem::create_directory(tempDir.path() + "/local"); - std::string filename(tempDir.path() + "/local/local.ns"); + std::string filename(tempDir.path() + "/local.ns"); std::ofstream ofs(filename.c_str()); ofs << "unused data" << std::endl; } - std::auto_ptr<StorageEngineMetadata> metadata = - StorageEngineMetadata::validate(tempDir.path(), "mmapv1"); - ASSERT_FALSE(metadata.get()); + ASSERT_EQUALS(std::string("mmapv1"), + StorageEngineMetadata::getStorageEngineForPath(tempDir.path())); } - // Data directory is not empty but metadata is missing. - // Data directory contains local.ns. - // Current storage engine is not 'mmapv1'. - TEST(StorageEngineMetadataTest, ValidateMissingMetadataDirectoryPerDb2) { - TempDir tempDir("StorageEngineMetadataTest_ValidateMissingMetadataDirectoryPerDb2"); + // Override the active storage engine with "mmapv1" when the data directory contains + // local/local.ns. + TEST(StorageEngineMetadataTest, StorageEngineForPath_DataFilesExist_DirPerDB) { + TempDir tempDir("StorageEngineMetadataTest_StorageEngineForPath_DataFilesExist_DirPerDB"); { boost::filesystem::create_directory(tempDir.path() + "/local"); std::string filename(tempDir.path() + "/local/local.ns"); std::ofstream ofs(filename.c_str()); ofs << "unused data" << std::endl; } - ASSERT_THROWS(StorageEngineMetadata::validate(tempDir.path(), "engine1"), UserException); + ASSERT_EQUALS(std::string("mmapv1"), + StorageEngineMetadata::getStorageEngineForPath(tempDir.path())); } - // Data directory is not empty but metadata is missing. - // Data directory does not contain either local.ns or local/local.ns. - // Current storage engine is 'mmapv1'. - TEST(StorageEngineMetadataTest, ValidateMissingMetadataMissingMMapV1Files1) { - TempDir tempDir("StorageEngineMetadataTest_ValidateMissingMetadataMissingMMapV1Files1"); + // Do not override the active storage engine when the data directory is nonempty, but does not + // contain either local.ns or local/local.ns. + TEST(StorageEngineMetadataTest, StorageEngineForPath_NoDataFilesExist) { + TempDir tempDir("StorageEngineMetadataTest_StorageEngineForPath_NoDataFilesExist"); { std::string filename(tempDir.path() + "/user_data.txt"); std::ofstream ofs(filename.c_str()); ofs << "unused data" << std::endl; } - std::auto_ptr<StorageEngineMetadata> metadata = - StorageEngineMetadata::validate(tempDir.path(), "mmapv1"); - ASSERT_FALSE(metadata.get()); + auto storageEngine = StorageEngineMetadata::getStorageEngineForPath(tempDir.path()); + ASSERT_FALSE(storageEngine); } - // Data directory is not empty but metadata is missing. - // Data directory does not contain either local.ns or local/local.ns. - // Current storage engine is not 'mmapv1'. - TEST(StorageEngineMetadataTest, ValidateMissingMetadataMissingMMapV1Files2) { - TempDir tempDir("StorageEngineMetadataTest_ValidateMissingMetadataMissingMMapV1Files2"); - { - std::string filename(tempDir.path() + "/user_data.txt"); - std::ofstream ofs(filename.c_str()); - ofs << "unused data" << std::endl; - } - std::auto_ptr<StorageEngineMetadata> metadata = - StorageEngineMetadata::validate(tempDir.path(), "engine1"); - ASSERT_FALSE(metadata.get()); - } - - TEST(StorageEngineMetadataTest, ValidateMetadataMatchesStorageEngine) { - TempDir tempDir("StorageEngineMetadataTest_ValidateMetadataMatchesStorageEngine"); + // Override the active storage engine with "mmapv1" when the metadata file specifies "mmapv1". + TEST(StorageEngineMetadataTest, StorageEngineForPath_MetadataFile_mmapv1) { + TempDir tempDir("StorageEngineMetadataTest_StorageEngineForPath_MetadataFile_mmapv1"); { StorageEngineMetadata metadata(tempDir.path()); - metadata.setStorageEngine("storageEngine1"); + metadata.setStorageEngine("mmapv1"); ASSERT_OK(metadata.write()); } - std::auto_ptr<StorageEngineMetadata> metadata = - StorageEngineMetadata::validate(tempDir.path(), "storageEngine1"); - ASSERT_TRUE(metadata.get()); - ASSERT_EQUALS("storageEngine1", metadata->getStorageEngine()); + ASSERT_EQUALS(std::string("mmapv1"), + StorageEngineMetadata::getStorageEngineForPath(tempDir.path())); } - TEST(StorageEngineMetadataTest, ValidateMetadataDifferentFromStorageEngine) { - TempDir tempDir("StorageEngineMetadataTest_ValidateMetadataDifferentFromStorageEngine"); + // Override the active storage engine whatever the metadata file specifies. + TEST(StorageEngineMetadataTest, StorageEngineForPath_MetadataFile_someEngine) { + TempDir tempDir("StorageEngineMetadataTest_StorageEngineForPath_MetadataFile_someEngine"); { StorageEngineMetadata metadata(tempDir.path()); - metadata.setStorageEngine("storageEngine1"); + metadata.setStorageEngine("someEngine"); ASSERT_OK(metadata.write()); } - ASSERT_THROWS(StorageEngineMetadata::validate(tempDir.path(), "engine2"), UserException); - } - - TEST(StorageEngineMetadataTest, ValidateMetadataReadFailed) { - TempDir tempDir("StorageEngineMetadataTest_ValidateMetadataReadFailed"); - { - std::string filename(tempDir.path() + "/storage.bson"); - std::ofstream ofs(filename.c_str()); - // BSON document of size -1 and EOO as first element. - BSONObj obj = fromjson("{x: 1}"); - ofs.write("\xff\xff\xff\xff", 4); - ofs.write(obj.objdata()+4, obj.objsize()-4); - ofs.flush(); - } - std::auto_ptr<StorageEngineMetadata> metadata = - StorageEngineMetadata::validate(tempDir.path(), "engine2"); - ASSERT_FALSE(metadata.get()); - } - - TEST(StorageEngineMetadataTest, ValidateStorageEngineOption) { - // It is fine to provide an invalid data directory as long as we do not - // call read() or write(). - StorageEngineMetadata metadata("no_such_directory"); - BSONObj options = fromjson("{x: true, y: false, z: 123}"); - metadata.setStorageEngineOptions(options); - - // Non-existent field. - ASSERT_OK(metadata.validateStorageEngineOption("w", true)); - ASSERT_OK(metadata.validateStorageEngineOption("w", false)); - - // Non-boolean field. - Status status = metadata.validateStorageEngineOption("z", true); - ASSERT_NOT_OK(status); - ASSERT_EQUALS(ErrorCodes::FailedToParse, status.code()); - status = metadata.validateStorageEngineOption("z", false); - ASSERT_NOT_OK(status); - ASSERT_EQUALS(ErrorCodes::FailedToParse, status.code()); - - // Boolean fields. - ASSERT_OK(metadata.validateStorageEngineOption("x", true)); - status = metadata.validateStorageEngineOption("x", false); - ASSERT_NOT_OK(status); - ASSERT_EQUALS(ErrorCodes::InvalidOptions, status.code()); - - ASSERT_OK(metadata.validateStorageEngineOption("y", false)); - status = metadata.validateStorageEngineOption("y", true); - ASSERT_NOT_OK(status); - ASSERT_EQUALS(ErrorCodes::InvalidOptions, status.code()); + ASSERT_EQUALS(std::string("someEngine"), + StorageEngineMetadata::getStorageEngineForPath(tempDir.path())); } } // namespace diff --git a/src/mongo/db/storage_options.h b/src/mongo/db/storage_options.h index 60e007b1ff6..3967af7139a 100644 --- a/src/mongo/db/storage_options.h +++ b/src/mongo/db/storage_options.h @@ -49,7 +49,8 @@ namespace mongo { static const char* kDefaultConfigDbPath; StorageGlobalParams() : - engine("mmapv1"), + engine("wiredTiger"), + engineSetByUser(false), dbpath(kDefaultDbPath), upgrade(false), repair(false), @@ -65,6 +66,9 @@ namespace mongo { // storage engine for this instance of mongod. std::string engine; + // True if --storageEngine was passed on the command line, and false otherwise. + bool engineSetByUser; + // The directory where the mongod instance stores its data. std::string dbpath; diff --git a/src/mongo/dbtests/framework.cpp b/src/mongo/dbtests/framework.cpp index 6a670a3e66b..3b405853665 100644 --- a/src/mongo/dbtests/framework.cpp +++ b/src/mongo/dbtests/framework.cpp @@ -121,7 +121,7 @@ namespace mongo { printGitVersion(); printOpenSSLVersion(); - getGlobalServiceContext()->setGlobalStorageEngine(storageGlobalParams.engine); + getGlobalServiceContext()->initializeGlobalStorageEngine(); // Initialize the sharding state so we can run starding tests in isolation shardingState.initialize("$dummy:10000"); diff --git a/src/mongo/dbtests/framework_options.cpp b/src/mongo/dbtests/framework_options.cpp index cf0c553d8b7..2e8daff0aff 100644 --- a/src/mongo/dbtests/framework_options.cpp +++ b/src/mongo/dbtests/framework_options.cpp @@ -88,7 +88,7 @@ namespace mongo { options->addOptionChaining("storage.engine", "storageEngine", moe::String, "what storage engine to use") - .setDefault(moe::Value(std::string("mmapv1"))); + .setDefault(moe::Value(std::string("wiredTiger"))); options->addOptionChaining("suites", "suites", moe::StringVector, "test suites to run") .hidden() diff --git a/src/mongo/shell/servers.js b/src/mongo/shell/servers.js index c4ec1491a1f..1da9db35ffe 100755 --- a/src/mongo/shell/servers.js +++ b/src/mongo/shell/servers.js @@ -882,6 +882,7 @@ startMongoProgram = function(){ runMongoProgram = function() { var args = argumentsToArray( arguments ); + args = appendSetParameterArgs(args); var progName = args[0]; if ( jsTestOptions().auth ) { @@ -907,6 +908,7 @@ runMongoProgram = function() { // command line arguments to the program. Returns pid of the spawned program. startMongoProgramNoConnect = function() { var args = argumentsToArray( arguments ); + args = appendSetParameterArgs(args); var progName = args[0]; if ( jsTestOptions().auth ) { |