diff options
31 files changed, 283 insertions, 312 deletions
diff --git a/buildscripts/resmokeconfig/suites/disk.yml b/buildscripts/resmokeconfig/suites/disk.yml index 159881bc87a..43a758412ab 100644 --- a/buildscripts/resmokeconfig/suites/disk.yml +++ b/buildscripts/resmokeconfig/suites/disk.yml @@ -7,11 +7,6 @@ selector: executor: js_test: - config: - shell_options: - global_vars: - TestData: - storageEngine: mmapv1 hooks: - class: CleanEveryN n: 20 @@ -21,4 +16,3 @@ executor: nopreallocj: '' set_parameters: enableTestCommands: 1 - storageEngine: mmapv1 diff --git a/buildscripts/resmokeconfig/suites/durability.yml b/buildscripts/resmokeconfig/suites/durability.yml index 6226858071c..7a89ded05dd 100644 --- a/buildscripts/resmokeconfig/suites/durability.yml +++ b/buildscripts/resmokeconfig/suites/durability.yml @@ -11,7 +11,4 @@ executor: js_test: config: shell_options: - global_vars: - TestData: - storageEngine: mmapv1 nodb: '' diff --git a/buildscripts/resmokeconfig/suites/mmap.yml b/buildscripts/resmokeconfig/suites/mmap.yml index 57dfe9f0a10..82f3806e76a 100644 --- a/buildscripts/resmokeconfig/suites/mmap.yml +++ b/buildscripts/resmokeconfig/suites/mmap.yml @@ -5,11 +5,6 @@ selector: executor: js_test: - config: - shell_options: - global_vars: - TestData: - storageEngine: mmapv1 hooks: - class: CleanEveryN n: 20 diff --git a/etc/evergreen.yml b/etc/evergreen.yml index d857f161947..11b426d5ec2 100644 --- a/etc/evergreen.yml +++ b/etc/evergreen.yml @@ -624,7 +624,15 @@ tasks: - func: "do setup" - func: "run tests" vars: - resmoke_args: -j8 --suites=core_auth + resmoke_args: -j8 --suites=core_auth --storageEngine=mmapv1 + +- <<: *task_template + name: jsCore_auth_WT + commands: + - func: "do setup" + - func: "run tests" + vars: + resmoke_args: -j8 --suites=core_auth --storageEngine=wiredTiger - <<: *task_template name: jsCore_op_command @@ -2110,7 +2118,7 @@ buildvariants: push_arch: i686 compile_flags: --release --distarch=i686 -j$(grep -c ^processor /proc/cpuinfo) CC=/opt/mongodbtoolchain/bin/gcc CXX=/opt/mongodbtoolchain/bin/g++ --variant-dir="linux2/release" --wiredtiger=off CCFLAGS="-m32" LINKFLAGS="-m32" num_jobs_unittests: $(grep -c ^processor /proc/cpuinfo) - test_flags: --continueOnFailure --storageEngine=mmapv1 -j1 # Avoid starting too many mongod's on 32-bit systems. + test_flags: --continueOnFailure -j1 # Avoid starting too many mongod's on 32-bit systems. has_debugsymbols: true tasks: - name: compile @@ -2147,7 +2155,7 @@ buildvariants: push_name: linux-debug push_arch: i686 num_jobs_unittests: $(grep -c ^processor /proc/cpuinfo) - test_flags: --continueOnFailure --storageEngine=mmapv1 -j1 # Avoid starting too many mongod's on 32-bit systems. + test_flags: --continueOnFailure -j1 # Avoid starting too many mongod's on 32-bit systems. has_debugsymbols: true compile_flags: --dbg=on --distarch=i686 --opt=on -j$(grep -c ^processor /proc/cpuinfo) CC=/opt/mongodbtoolchain/bin/gcc CXX=/opt/mongodbtoolchain/bin/g++ --variant-dir="linux2/debug" --wiredtiger=off CCFLAGS="-m32" LINKFLAGS="-m32" tasks: @@ -2526,7 +2534,7 @@ buildvariants: content_type: application/zip compile_flags: --release -j$(grep -c ^processor /proc/cpuinfo) --wiredtiger=off TARGET_ARCH=i386 --variant-dir=win32 num_jobs_unittests: $(grep -c ^processor /proc/cpuinfo) - test_flags: --continueOnFailure --storageEngine=mmapv1 -j1 # Avoid starting too many mongod's on 32-bit systems. + test_flags: --continueOnFailure -j1 # Avoid starting too many mongod's on 32-bit systems. ext: zip # TODO: Remove the "is_windows" expansion after PyYAML is installed on all build variants. is_windows: true diff --git a/jstests/core/apitest_dbcollection.js b/jstests/core/apitest_dbcollection.js index 1434ef3170c..6d405e8c716 100644 --- a/jstests/core/apitest_dbcollection.js +++ b/jstests/core/apitest_dbcollection.js @@ -193,10 +193,8 @@ assert(db.getCollection( "test_db" ).getIndexes().length == 0,24); 'indexDetails missing from ' + 'db.collection.stats(' + tojson(options) + ') result: ' + tojson(collectionStats)); // Currently, indexDetails is only supported with WiredTiger. - var storageEngine = jsTest.options().storageEngine; - if (storageEngine && storageEngine !== 'wiredTiger') { - return; - } + if (jsTest.options().storageEngine == undefined) { return; } + if (jsTest.options().storageEngine.toLowerCase() != "wiredtiger") { return; } assert.eq(1, Object.keys(collectionStats.indexDetails).length, 'indexDetails must have exactly one entry'); assert(collectionStats.indexDetails[indexName], diff --git a/jstests/multiVersion/downgrade_replset.js b/jstests/multiVersion/downgrade_replset.js index 17581827f11..c3d8460eb0c 100644 --- a/jstests/multiVersion/downgrade_replset.js +++ b/jstests/multiVersion/downgrade_replset.js @@ -12,7 +12,7 @@ var nodes = {n1: {binVersion: newVersion}, n2: {binVersion: newVersion}, n3: {binVersion: newVersion}}; -var rst = new ReplSetTest({name: name, nodes: nodes, nodeOptions: {storageEngine: 'mmapv1'}}); +var rst = new ReplSetTest({name: name, nodes: 3}); rst.startSet(); rst.initiate(); diff --git a/jstests/multiVersion/libs/verify_collection_data.js b/jstests/multiVersion/libs/verify_collection_data.js index 72c3e01dac7..9e8423c1db2 100644 --- a/jstests/multiVersion/libs/verify_collection_data.js +++ b/jstests/multiVersion/libs/verify_collection_data.js @@ -68,47 +68,83 @@ createCollectionWithData = function (db, collectionName, dataGenerator) { // the saved state function CollectionDataValidator() { - var _initialized = false; - var _collectionInfo = {}; - var _indexData = []; - var _collectionData = []; - - // Returns the options of the specified collection. - this.getCollectionInfo = function(collection) { - var infoObj = collection.getDB().getCollectionInfos({name: collection.getName()}); - assert.eq(1, infoObj.length, "expected collection '" + collection.getName() + "'to exist"); - return infoObj[0]; - }; + var initialized = false; + var collectionStats = {}; + var indexData = []; + var collectionData = []; // Saves the current state of the collection passed in this.recordCollectionData = function (collection) { - // Save the metadata for this collection for later comparison. - _collectionInfo = this.getCollectionInfo(collection); // Save the indexes for this collection for later comparison - _indexData = collection.getIndexes().sort(function(a,b) { + indexData = collection.getIndexes().sort(function(a,b) { if (a.name > b.name) return 1; else return -1; }); // Save the data for this collection for later comparison - _collectionData = collection.find().sort({"_id":1}).toArray(); + collectionData = collection.find().sort({"_id":1}).toArray(); + + // Save the metadata for this collection for later comparison. + // NOTE: We do this last since the data and indexes affect this output + collectionStats = collection.stats(); + + // XXX: in 2.4 avgObjSize was a double, but in 2.6 it is an int + collectionStats['avgObjSize'] = Math.floor(collectionStats['avgObjSize']); - _initialized = true; + // Delete keys that appear just because we shard + delete collectionStats["primary"]; + delete collectionStats["sharded"]; + + initialized = true; return collection; } this.validateCollectionData = function (collection) { - if (!_initialized) { + if (!initialized) { throw Error("validateCollectionWithAllData called, but data is not initialized"); } // Get the metadata for this collection - var newCollectionInfo = this.getCollectionInfo(collection); + var newCollectionStats = collection.stats(); + + // XXX: in 2.4 avgObjSize was a double, but in 2.6 it is an int + newCollectionStats['avgObjSize'] = Math.floor(newCollectionStats['avgObjSize']); + + // as of 2.7.1, we no longer use systemFlags + delete collectionStats.systemFlags; + delete newCollectionStats.systemFlags; + + // as of 2.7.7, we no longer use paddingFactor and introduced paddingFactorNote + delete collectionStats.paddingFactor; + delete collectionStats.paddingFactorNote; + delete newCollectionStats.paddingFactor; + delete newCollectionStats.paddingFactorNote; + + // Delete keys that appear just because we shard + delete newCollectionStats["primary"]; + delete newCollectionStats["sharded"]; + + // as of 2.7.8, we added maxSize + // TODO: when 2.6 is no longer tested, remove following two lines + delete newCollectionStats["maxSize"]; + delete collectionStats["maxSize"]; + + // Delete key added in 2.8-rc3 + delete collectionStats["indexDetails"]; + delete newCollectionStats["indexDetails"]; + + // Delete capped:false added in 2.8.0-rc5 + if (newCollectionStats["capped"] == false) { + delete newCollectionStats["capped"]; + } + if (collectionStats["capped"] == false) { + delete collectionStats["capped"]; + } - assert.docEq(_collectionInfo, newCollectionInfo, "collection metadata not equal"); + assert.docEq(collectionStats, newCollectionStats, "collection metadata not equal"); // Get the indexes for this collection var newIndexData = collection.getIndexes().sort(function(a,b) { @@ -116,13 +152,13 @@ function CollectionDataValidator() { else return -1; }); for (var i = 0; i < newIndexData.length; i++) { - assert.docEq(_indexData[i], newIndexData[i], "indexes not equal"); + assert.docEq(indexData[i], newIndexData[i], "indexes not equal"); } // Save the data for this collection for later comparison var newCollectionData = collection.find().sort({"_id":1}).toArray(); for (var i = 0; i < newCollectionData.length; i++) { - assert.docEq(_collectionData[i], newCollectionData[i], "data not equal"); + assert.docEq(collectionData[i], newCollectionData[i], "data not equal"); } return true; } diff --git a/jstests/multiVersion/mmapv1_overrides_default_storage_engine.js b/jstests/multiVersion/mmapv1_overrides_default_storage_engine.js deleted file mode 100644 index 9cad40c23bd..00000000000 --- a/jstests/multiVersion/mmapv1_overrides_default_storage_engine.js +++ /dev/null @@ -1,96 +0,0 @@ -/** - * Test the upgrade process for 2.6 ~~> 3.2 and 3.0 ~~> 3.2, where mmapv1 should continue to be the - * default storage engine. Repeat the process with --directoryperdb set. - */ -(function() { - 'use strict'; - - var testCases = [ - { - binVersion: '2.6', - }, - { - binVersion: '2.6', - directoryperdb: '', - }, - { - binVersion: '3.0', - }, - { - binVersion: '3.0', - directoryperdb: '', - }, - ]; - - // The mongod should start up with mmapv1 when the --storageEngine flag is omitted, or when - // --storageEngine=mmapv1 is explicitly specified. - testCases.forEach(function(testCase) { - [null, 'mmapv1'].forEach(function(storageEngine) { - jsTest.log('Upgrading from a ' + testCase.binVersion + ' instance with options=' - + tojson(testCase) + ' to the latest version. This should succeed when the' - + ' latest version ' - + (storageEngine ? ('explicitly specifies --storageEngine=' + storageEngine) - : 'omits the --storageEngine flag')); - - var dbpath = MongoRunner.dataPath + 'mmapv1_overrides_default_storage_engine'; - resetDbpath(dbpath); - - var defaultOptions = { - dbpath: dbpath, - noCleanData: true, - }; - - // Start the old version. - var mongodOptions = Object.merge(defaultOptions, testCase); - var conn = MongoRunner.runMongod(mongodOptions); - assert.neq(null, conn, - 'mongod was unable to start up with options ' + tojson(mongodOptions)); - assert.commandWorked(conn.getDB('test').runCommand({ping: 1})); - MongoRunner.stopMongod(conn); - - // Start the newest version. - mongodOptions = Object.extend({}, defaultOptions); - if (storageEngine) { - mongodOptions.storageEngine = storageEngine; - } - if (testCase.hasOwnProperty('directoryperdb')) { - mongodOptions.directoryperdb = testCase.directoryperdb; - } - conn = MongoRunner.runMongod(mongodOptions); - assert.neq(null, conn, - 'mongod was unable to start up with options ' + tojson(mongodOptions)); - assert.commandWorked(conn.getDB('test').runCommand({ping: 1})); - MongoRunner.stopMongod(conn); - }); - }); - - // The mongod should not start up when --storageEngine=wiredTiger is specified. - testCases.forEach(function(testCase) { - jsTest.log('Upgrading from a ' + testCase.binVersion + ' instance with options=' - + tojson(testCase) + ' to the latest version. This should fail when the latest' - + ' version specifies --storageEngine=wiredTiger'); - - var dbpath = MongoRunner.dataPath + 'mmapv1_overrides_default_storage_engine'; - resetDbpath(dbpath); - - var defaultOptions = { - dbpath: dbpath, - noCleanData: true, - }; - - // Start the old version. - var mongodOptions = Object.merge(defaultOptions, testCase); - var conn = MongoRunner.runMongod(mongodOptions); - assert.neq(null, conn, - 'mongod was unable to start up with options ' + tojson(mongodOptions)); - assert.commandWorked(conn.getDB('test').runCommand({ping: 1})); - MongoRunner.stopMongod(conn); - - // Start the newest version. - mongodOptions = Object.extend({storageEngine: 'wiredTiger'}, defaultOptions); - conn = MongoRunner.runMongod(mongodOptions); - assert.eq(null, conn, - 'mongod should not have been able to start up with options ' - + tojson(mongodOptions)); - }); -}()); diff --git a/jstests/noPassthrough/dir_per_db_and_split.js b/jstests/noPassthrough/dir_per_db_and_split.js index 8047ec9fda2..e03ad7e6746 100644 --- a/jstests/noPassthrough/dir_per_db_and_split.js +++ b/jstests/noPassthrough/dir_per_db_and_split.js @@ -1,5 +1,5 @@ -if (!jsTest.options().storageEngine || jsTest.options().storageEngine === "wiredTiger") { +if ( jsTest.options().storageEngine == "wiredTiger" ) { var baseDir = "jstests_per_db_and_split_c_and_i"; port = allocatePorts( 1 )[ 0 ]; diff --git a/jstests/noPassthrough/split_collections_and_indexes.js b/jstests/noPassthrough/split_collections_and_indexes.js index 73d2eede111..41916d6e9a0 100644 --- a/jstests/noPassthrough/split_collections_and_indexes.js +++ b/jstests/noPassthrough/split_collections_and_indexes.js @@ -1,5 +1,5 @@ -if (!jsTest.options().storageEngine || jsTest.options().storageEngine === "wiredTiger") { +if ( jsTest.options().storageEngine == "wiredTiger" ) { var baseDir = "jstests_split_c_and_i"; port = allocatePorts( 1 )[ 0 ]; diff --git a/jstests/noPassthrough/wt_nojournal_fsync.js b/jstests/noPassthrough/wt_nojournal_fsync.js index 46a881de16d..5673e1208ee 100644 --- a/jstests/noPassthrough/wt_nojournal_fsync.js +++ b/jstests/noPassthrough/wt_nojournal_fsync.js @@ -30,7 +30,10 @@ function writeDataAndRestart(doFsync) { } // This test can only be run if the storageEngine is wiredTiger -if (jsTest.options().storageEngine && jsTest.options().storageEngine !== "wiredTiger") { +// This check will have to change when we change the default storageEngine +if ( typeof(TestData) != "object" || + !TestData.storageEngine || + TestData.storageEngine != "wiredTiger" ) { jsTestLog("Skipping test because storageEngine is not wiredTiger"); } else { diff --git a/jstests/noPassthrough/wt_nojournal_repl.js b/jstests/noPassthrough/wt_nojournal_repl.js index 01bd23b10da..79ec8f797a2 100644 --- a/jstests/noPassthrough/wt_nojournal_repl.js +++ b/jstests/noPassthrough/wt_nojournal_repl.js @@ -26,7 +26,9 @@ var contains = function(logLines, func) { } // This test can only be run if the storageEngine is wiredTiger -if (jsTest.options().storageEngine && jsTest.options().storageEngine !== "wiredTiger") { +if ( typeof(TestData) != "object" || + !TestData.storageEngine || + TestData.storageEngine != "wiredTiger" ) { jsTestLog("Skipping test because storageEngine is not wiredTiger"); } else { diff --git a/jstests/noPassthroughWithMongod/index_check10.js b/jstests/noPassthroughWithMongod/index_check10.js index 5ace6951652..f507c687731 100644 --- a/jstests/noPassthroughWithMongod/index_check10.js +++ b/jstests/noPassthroughWithMongod/index_check10.js @@ -135,7 +135,7 @@ function doIt( indexVersion ) { for( var z = 0; z < 5; ++z ) { var indexVersion = z % 2; var storageEngine = jsTest.options().storageEngine; - if (storageEngine === 'mmapv1' || indexVersion !== 0) { + if (!storageEngine || storageEngine === 'mmapv1' || indexVersion !== 0) { doIt(indexVersion); } } diff --git a/jstests/replsets/initSyncV1Index.js b/jstests/replsets/initSyncV1Index.js index 10b91949942..c3daa471b44 100644 --- a/jstests/replsets/initSyncV1Index.js +++ b/jstests/replsets/initSyncV1Index.js @@ -6,7 +6,7 @@ 'use strict'; var storageEngine = jsTest.options().storageEngine; - if (storageEngine !== 'mmapv1') { + if (storageEngine && storageEngine !== 'mmapv1') { return; } diff --git a/jstests/sharding/stats.js b/jstests/sharding/stats.js index 52ec40556d7..6c21e3861da 100644 --- a/jstests/sharding/stats.js +++ b/jstests/sharding/stats.js @@ -132,8 +132,7 @@ collStatComp(coll_not_scaled, coll_scaled_1024, 1024, true); assert.commandWorked(t.ensureIndex({a: 1})); assert.eq(2, t.getIndexes().length); - var isWiredTiger = (!jsTest.options().storageEngine - || jsTest.options().storageEngine === "wiredTiger"); + var isWiredTiger = (jsTest.options().storageEngine == "wiredTiger"); var stats = assert.commandWorked(t.stats({indexDetails: true})); var shardName; diff --git a/jstests/sharding/user_flags_sharded.js b/jstests/sharding/user_flags_sharded.js index e5b5f8a41dd..2d7a341e4f4 100644 --- a/jstests/sharding/user_flags_sharded.js +++ b/jstests/sharding/user_flags_sharded.js @@ -1,7 +1,9 @@ // Test that when user flags are set on a collection, // then collection is sharded, flags get carried over. -if (jsTest.options().storageEngine === "mmapv1") { +if ( typeof(TestData) != "object" || + !TestData.storageEngine || + TestData.storageEngine == "mmapv1" ) { // the dbname and collection we'll be working with var dbname = "testDB"; diff --git a/src/mongo/db/db.cpp b/src/mongo/db/db.cpp index d7e69a0c5ca..fa360169fa0 100644 --- a/src/mongo/db/db.cpp +++ b/src/mongo/db/db.cpp @@ -34,7 +34,6 @@ #include <boost/thread/thread.hpp> #include <boost/filesystem/operations.hpp> -#include <boost/optional.hpp> #include <boost/shared_ptr.hpp> #include <fstream> #include <iostream> @@ -418,8 +417,6 @@ namespace mongo { dbWebServer->setupSockets(); } - getGlobalServiceContext()->initializeGlobalStorageEngine(); - // Warn if we detect configurations for multiple registered storage engines in // the same configuration file/environment. if (serverGlobalParams.parsedOpts.hasField("storage")) { @@ -444,6 +441,7 @@ namespace mongo { } } + getGlobalServiceContext()->setGlobalStorageEngine(storageGlobalParams.engine); getGlobalServiceContext()->setOpObserver(stdx::make_unique<OpObserver>()); const repl::ReplSettings& replSettings = diff --git a/src/mongo/db/mongod_options.cpp b/src/mongo/db/mongod_options.cpp index a2daab9c201..bb283121bf3 100644 --- a/src/mongo/db/mongod_options.cpp +++ b/src/mongo/db/mongod_options.cpp @@ -165,7 +165,8 @@ namespace mongo { // Storage Options storage_options.addOptionChaining("storage.engine", "storageEngine", moe::String, - "what storage engine to use - defaults to wiredTiger if no data files present"); + "what storage engine to use") + .setDefault(moe::Value(std::string("mmapv1"))); #ifdef _WIN32 @@ -893,10 +894,7 @@ namespace mongo { "files"); } - if (params.count("storage.engine")) { - storageGlobalParams.engine = params["storage.engine"].as<string>(); - storageGlobalParams.engineSetByUser = true; - } + storageGlobalParams.engine = params["storage.engine"].as<string>(); if (params.count("storage.dbPath")) { storageGlobalParams.dbpath = params["storage.dbPath"].as<string>(); diff --git a/src/mongo/db/repl/sync_tail_test.cpp b/src/mongo/db/repl/sync_tail_test.cpp index 9e20ce78818..1f2dc692252 100644 --- a/src/mongo/db/repl/sync_tail_test.cpp +++ b/src/mongo/db/repl/sync_tail_test.cpp @@ -103,9 +103,7 @@ namespace { // go away after the global storage engine is initialized. unittest::TempDir tempDir("sync_tail_test"); mongo::storageGlobalParams.dbpath = tempDir.path(); - mongo::storageGlobalParams.engine = "devnull"; - mongo::storageGlobalParams.engineSetByUser = true; - serviceContext->initializeGlobalStorageEngine(); + serviceContext->setGlobalStorageEngine("devnull"); } _prevCoordinator = getGlobalReplicationCoordinator(); ReplSettings replSettings; diff --git a/src/mongo/db/service_context.h b/src/mongo/db/service_context.h index 3833ff69d85..40cee393d25 100644 --- a/src/mongo/db/service_context.h +++ b/src/mongo/db/service_context.h @@ -197,7 +197,10 @@ namespace mongo { */ virtual StorageFactoriesIterator* makeStorageFactoriesIterator() = 0; - virtual void initializeGlobalStorageEngine() = 0; + /** + * Set the storage engine. The engine must have been registered via registerStorageEngine. + */ + virtual void setGlobalStorageEngine(const std::string& name) = 0; /** * Shuts down storage engine cleanly and releases any locks on mongod.lock. diff --git a/src/mongo/db/service_context_d.cpp b/src/mongo/db/service_context_d.cpp index eaef2f6f639..83427b0cef2 100644 --- a/src/mongo/db/service_context_d.cpp +++ b/src/mongo/db/service_context_d.cpp @@ -32,8 +32,6 @@ #include "mongo/db/service_context_d.h" -#include <boost/optional.hpp> - #include "mongo/base/init.h" #include "mongo/base/initializer.h" #include "mongo/db/client.h" @@ -47,7 +45,6 @@ #include "mongo/scripting/engine.h" #include "mongo/stdx/memory.h" #include "mongo/util/log.h" -#include "mongo/util/map_util.h" #include "mongo/util/mongoutils/str.h" #include "mongo/util/scopeguard.h" @@ -75,52 +72,21 @@ namespace mongo { extern bool _supportsDocLocking; - void ServiceContextMongoD::initializeGlobalStorageEngine() { + void ServiceContextMongoD::setGlobalStorageEngine(const std::string& name) { // This should be set once. invariant(!_storageEngine); - const std::string dbpath = storageGlobalParams.dbpath; - if (auto existingStorageEngine = StorageEngineMetadata::getStorageEngineForPath(dbpath)) { - if (storageGlobalParams.engineSetByUser) { - // Verify that the name of the user-supplied storage engine matches the contents of - // the metadata file. - const StorageEngine::Factory* factory = mapFindWithDefault( - _storageFactories, - storageGlobalParams.engine, - static_cast<const StorageEngine::Factory*>(nullptr)); - - if (factory) { - uassert(28662, str::stream() - << "Cannot start server. Detected data files in " << dbpath << " created by" - << " the '" << *existingStorageEngine << "' storage engine, but the" - << " specified storage engine was '" << factory->getCanonicalName() << "'.", - factory->getCanonicalName() == *existingStorageEngine); - } - } - else { - // Otherwise set the active storage engine as the contents of the metadata file. - log() << "Detected data files in " << dbpath << " created by the '" - << *existingStorageEngine << "' storage engine, so setting the active" - << " storage engine to '" << *existingStorageEngine << "'."; - storageGlobalParams.engine = *existingStorageEngine; - } - } - else if (!storageGlobalParams.engineSetByUser) { - // Ensure the default storage engine is available with this build of mongod. - uassert(28663, str::stream() - << "Cannot start server. The default storage engine '" << storageGlobalParams.engine - << "' is not available with this build of mongod. Please specify a different" - << " storage engine explicitly, e.g. --storageEngine=mmapv1.", - isRegisteredStorageEngine(storageGlobalParams.engine)); - } - - const StorageEngine::Factory* factory = _storageFactories[storageGlobalParams.engine]; + const StorageEngine::Factory* factory = _storageFactories[name]; uassert(18656, str::stream() - << "Cannot start server with an unknown storage engine: " << storageGlobalParams.engine, + << "Cannot start server with an unknown storage engine: " << name, factory); - std::unique_ptr<StorageEngineMetadata> metadata = StorageEngineMetadata::forPath(dbpath); + std::string canonicalName = factory->getCanonicalName().toString(); + + // Do not proceed if data directory has been used by a different storage engine previously. + std::auto_ptr<StorageEngineMetadata> metadata = + StorageEngineMetadata::validate(storageGlobalParams.dbpath, canonicalName); // Validate options in metadata against current startup options. if (metadata.get()) { @@ -150,7 +116,7 @@ namespace mongo { // Write a new metadata file if it is not present. if (!metadata.get()) { metadata.reset(new StorageEngineMetadata(storageGlobalParams.dbpath)); - metadata->setStorageEngine(factory->getCanonicalName().toString()); + metadata->setStorageEngine(canonicalName); metadata->setStorageEngineOptions(factory->createMetadataOptions(storageGlobalParams)); uassertStatusOK(metadata->write()); } diff --git a/src/mongo/db/service_context_d.h b/src/mongo/db/service_context_d.h index 9437d37aa07..caa6ba17bdf 100644 --- a/src/mongo/db/service_context_d.h +++ b/src/mongo/db/service_context_d.h @@ -49,7 +49,7 @@ namespace mongo { StorageEngine* getGlobalStorageEngine(); - void initializeGlobalStorageEngine(); + void setGlobalStorageEngine(const std::string& name); void shutdownGlobalStorageEngineCleanly(); diff --git a/src/mongo/db/service_context_noop.cpp b/src/mongo/db/service_context_noop.cpp index 47c3efe8797..8b43c4e1bbf 100644 --- a/src/mongo/db/service_context_noop.cpp +++ b/src/mongo/db/service_context_noop.cpp @@ -39,7 +39,7 @@ namespace mongo { return NULL; } - void ServiceContextNoop::initializeGlobalStorageEngine() { + void ServiceContextNoop::setGlobalStorageEngine(const std::string& name) { } void ServiceContextNoop::shutdownGlobalStorageEngineCleanly() { diff --git a/src/mongo/db/service_context_noop.h b/src/mongo/db/service_context_noop.h index b7f541b784f..679b683bb74 100644 --- a/src/mongo/db/service_context_noop.h +++ b/src/mongo/db/service_context_noop.h @@ -34,7 +34,7 @@ namespace mongo { public: StorageEngine* getGlobalStorageEngine(); - void initializeGlobalStorageEngine(); + void setGlobalStorageEngine(const std::string& name); void shutdownGlobalStorageEngineCleanly(); diff --git a/src/mongo/db/storage/storage_engine_metadata.cpp b/src/mongo/db/storage/storage_engine_metadata.cpp index 2881c41d689..03dce664bd1 100644 --- a/src/mongo/db/storage/storage_engine_metadata.cpp +++ b/src/mongo/db/storage/storage_engine_metadata.cpp @@ -34,7 +34,6 @@ #include <cstdio> #include <boost/filesystem.hpp> -#include <boost/optional.hpp> #include <fstream> #include <limits> #include <ostream> @@ -63,33 +62,42 @@ namespace { } // namespace // static - std::unique_ptr<StorageEngineMetadata> StorageEngineMetadata::forPath( - const std::string& dbpath) { - std::unique_ptr<StorageEngineMetadata> metadata; + std::auto_ptr<StorageEngineMetadata> StorageEngineMetadata::validate( + const std::string& dbpath, + const std::string& storageEngine) { + + std::auto_ptr<StorageEngineMetadata> metadata; + std::string previousStorageEngine; if (boost::filesystem::exists(boost::filesystem::path(dbpath) / kMetadataBasename)) { metadata.reset(new StorageEngineMetadata(dbpath)); Status status = metadata->read(); - if (!status.isOK()) { - error() << "Unable to read the storage engine metadata file: " << status; - fassertFailed(28661); + if (status.isOK()) { + previousStorageEngine = metadata->getStorageEngine(); + } + else { + // The storage metadata file is present but there was an issue + // reading its contents. + warning() << "Unable to read the existing storage engine metadata: " + << status.toString(); + return std::auto_ptr<StorageEngineMetadata>(); } } - return metadata; - } - - // static - boost::optional<std::string> StorageEngineMetadata::getStorageEngineForPath( - const std::string& dbpath) { - if (auto metadata = StorageEngineMetadata::forPath(dbpath)) { - return {metadata->getStorageEngine()}; + else if (containsMMapV1LocalNsFile(dbpath)) { + previousStorageEngine = "mmapv1"; } - - // Fallback to checking for MMAPv1-specific files to handle upgrades from before the - // storage.bson metadata file was introduced in 3.0. - if (containsMMapV1LocalNsFile(dbpath)) { - return {std::string("mmapv1")}; + else { + // Directory contains neither metadata nor mmapv1 files. + // Allow validation to succeed. + return metadata; } - return {}; + + uassert(28574, str::stream() + << "Cannot start server. Detected data files in " << dbpath + << " created by storage engine '" << previousStorageEngine + << "'. The configured storage engine is '" << storageEngine << "'.", + previousStorageEngine == storageEngine); + + return metadata; } StorageEngineMetadata::StorageEngineMetadata(const std::string& dbpath) diff --git a/src/mongo/db/storage/storage_engine_metadata.h b/src/mongo/db/storage/storage_engine_metadata.h index a4dafdf9bfa..4f87f2fc50c 100644 --- a/src/mongo/db/storage/storage_engine_metadata.h +++ b/src/mongo/db/storage/storage_engine_metadata.h @@ -28,7 +28,6 @@ #pragma once -#include <boost/optional.hpp> #include <memory> #include <string> @@ -50,16 +49,23 @@ namespace mongo { public: /** - * Returns a metadata object describing the storage engine that backs the data files - * contained in 'dbpath', and nullptr otherwise. + * Validates metadata in data directory against current storage engine. + * 1) If the metadata file exists, ensure that the information in the file + * is consistent with the current storage engine. Otherwise, raise an error. + * Returns the metadata object on successful validation. + * 2) If the metadata file exists but is not readable (eg. corrupted), + * return NULL. This allows the startup process to overwrite the corrupted + * metadata file with a valid copy. + * 3) If the metadata file does not exist, look for local.ns or local/local.ns + * in the data directory. If we detect either file, raise an error + * only if the current storage engine is not 'mmapv1'. + * This makes validation more forgiving of situations where + * application data is placed in the data directory prior + * to server start up. + * Returns NULL on successful validation. */ - static std::unique_ptr<StorageEngineMetadata> forPath(const std::string& dbpath); - - /** - * Returns the name of the storage engine that backs the data files contained in 'dbpath', - * and none otherwise. - */ - static boost::optional<std::string> getStorageEngineForPath(const std::string& dbpath); + static std::auto_ptr<StorageEngineMetadata> validate(const std::string& dbpath, + const std::string& storageEngine); /** * Sets fields to defaults. diff --git a/src/mongo/db/storage/storage_engine_metadata_test.cpp b/src/mongo/db/storage/storage_engine_metadata_test.cpp index 27508dfe7a5..afb76972082 100644 --- a/src/mongo/db/storage/storage_engine_metadata_test.cpp +++ b/src/mongo/db/storage/storage_engine_metadata_test.cpp @@ -29,8 +29,6 @@ #include "mongo/platform/basic.h" #include <boost/filesystem.hpp> -#include <boost/optional.hpp> -#include <boost/optional/optional_io.hpp> #include <fstream> #include <ios> #include <ostream> @@ -199,105 +197,169 @@ namespace { } } - TEST(StorageEngineMetadataTest, ValidateStorageEngineOption) { - // It is fine to provide an invalid data directory as long as we do not - // call read() or write(). - StorageEngineMetadata metadata("no_such_directory"); - BSONObj options = fromjson("{x: true, y: false, z: 123}"); - metadata.setStorageEngineOptions(options); - - // Non-existent field. - ASSERT_OK(metadata.validateStorageEngineOption("w", true)); - ASSERT_OK(metadata.validateStorageEngineOption("w", false)); - - // Non-boolean field. - Status status = metadata.validateStorageEngineOption("z", true); - ASSERT_NOT_OK(status); - ASSERT_EQUALS(ErrorCodes::FailedToParse, status.code()); - status = metadata.validateStorageEngineOption("z", false); - ASSERT_NOT_OK(status); - ASSERT_EQUALS(ErrorCodes::FailedToParse, status.code()); - - // Boolean fields. - ASSERT_OK(metadata.validateStorageEngineOption("x", true)); - status = metadata.validateStorageEngineOption("x", false); - ASSERT_NOT_OK(status); - ASSERT_EQUALS(ErrorCodes::InvalidOptions, status.code()); - - ASSERT_OK(metadata.validateStorageEngineOption("y", false)); - status = metadata.validateStorageEngineOption("y", true); - ASSERT_NOT_OK(status); - ASSERT_EQUALS(ErrorCodes::InvalidOptions, status.code()); + TEST(StorageEngineMetadataTest, ValidateEmptyDirectory) { + TempDir tempDir("StorageEngineMetadataTest_ValidateEmptyDirectory"); + std::auto_ptr<StorageEngineMetadata> metadata = + StorageEngineMetadata::validate(tempDir.path(), "storageEngine1"); + ASSERT_FALSE(metadata.get()); } - // Do not override the active storage engine when the data directory is empty. - TEST(StorageEngineMetadataTest, StorageEngineForPath_EmptyDirectory) { - TempDir tempDir("StorageEngineMetadataTest_StorageEngineForPath_EmptyDirectory"); - auto storageEngine = StorageEngineMetadata::getStorageEngineForPath(tempDir.path()); - ASSERT_FALSE(storageEngine); + // Data directory is not empty but metadata is missing. + // Data directory contains local.ns. + // Current storage engine is 'mmapv1'. + TEST(StorageEngineMetadataTest, ValidateMissingMetadataStorageEngineMMapV1) { + TempDir tempDir("StorageEngineMetadataTest_ValidateMissingMetadataStorageEngineMMapV1"); + { + std::string filename(tempDir.path() + "/local.ns"); + std::ofstream ofs(filename.c_str()); + ofs << "unused data" << std::endl; + } + std::auto_ptr<StorageEngineMetadata> metadata = + StorageEngineMetadata::validate(tempDir.path(), "mmapv1"); + ASSERT_FALSE(metadata.get()); } - // Override the active storage engine with "mmapv1" when the data directory contains local.ns. - TEST(StorageEngineMetadataTest, StorageEngineForPath_DataFilesExist) { - TempDir tempDir("StorageEngineMetadataTest_StorageEngineForPath_DataFilesExist"); + // Data directory is not empty but metadata is missing. + // Data directory contains local.ns. + // Current storage engine is not 'mmapv1'. + TEST(StorageEngineMetadataTest, ValidateMissingMetadataStorageEngineNotMMapV1) { + TempDir tempDir("StorageEngineMetadataTest_ValidateMissingMetadataStorageEngineNotMMapV1"); { std::string filename(tempDir.path() + "/local.ns"); std::ofstream ofs(filename.c_str()); ofs << "unused data" << std::endl; } - ASSERT_EQUALS(std::string("mmapv1"), - StorageEngineMetadata::getStorageEngineForPath(tempDir.path())); + ASSERT_THROWS(StorageEngineMetadata::validate(tempDir.path(), "engine1"), UserException); } - // Override the active storage engine with "mmapv1" when the data directory contains - // local/local.ns. - TEST(StorageEngineMetadataTest, StorageEngineForPath_DataFilesExist_DirPerDB) { - TempDir tempDir("StorageEngineMetadataTest_StorageEngineForPath_DataFilesExist_DirPerDB"); + // Data directory is not empty but metadata is missing. + // Data directory contains local/local.ns. + // Current storage engine is 'mmapv1'. + TEST(StorageEngineMetadataTest, ValidateMissingMetadataDirectoryPerDb1) { + TempDir tempDir("StorageEngineMetadataTest_ValidateMissingMetadataDirectoryPerDb1"); { boost::filesystem::create_directory(tempDir.path() + "/local"); std::string filename(tempDir.path() + "/local/local.ns"); std::ofstream ofs(filename.c_str()); ofs << "unused data" << std::endl; } - ASSERT_EQUALS(std::string("mmapv1"), - StorageEngineMetadata::getStorageEngineForPath(tempDir.path())); + std::auto_ptr<StorageEngineMetadata> metadata = + StorageEngineMetadata::validate(tempDir.path(), "mmapv1"); + ASSERT_FALSE(metadata.get()); } - // Do not override the active storage engine when the data directory is nonempty, but does not - // contain either local.ns or local/local.ns. - TEST(StorageEngineMetadataTest, StorageEngineForPath_NoDataFilesExist) { - TempDir tempDir("StorageEngineMetadataTest_StorageEngineForPath_NoDataFilesExist"); + // Data directory is not empty but metadata is missing. + // Data directory contains local.ns. + // Current storage engine is not 'mmapv1'. + TEST(StorageEngineMetadataTest, ValidateMissingMetadataDirectoryPerDb2) { + TempDir tempDir("StorageEngineMetadataTest_ValidateMissingMetadataDirectoryPerDb2"); + { + boost::filesystem::create_directory(tempDir.path() + "/local"); + std::string filename(tempDir.path() + "/local/local.ns"); + std::ofstream ofs(filename.c_str()); + ofs << "unused data" << std::endl; + } + ASSERT_THROWS(StorageEngineMetadata::validate(tempDir.path(), "engine1"), UserException); + } + + // Data directory is not empty but metadata is missing. + // Data directory does not contain either local.ns or local/local.ns. + // Current storage engine is 'mmapv1'. + TEST(StorageEngineMetadataTest, ValidateMissingMetadataMissingMMapV1Files1) { + TempDir tempDir("StorageEngineMetadataTest_ValidateMissingMetadataMissingMMapV1Files1"); { std::string filename(tempDir.path() + "/user_data.txt"); std::ofstream ofs(filename.c_str()); ofs << "unused data" << std::endl; } - auto storageEngine = StorageEngineMetadata::getStorageEngineForPath(tempDir.path()); - ASSERT_FALSE(storageEngine); + std::auto_ptr<StorageEngineMetadata> metadata = + StorageEngineMetadata::validate(tempDir.path(), "mmapv1"); + ASSERT_FALSE(metadata.get()); } - // Override the active storage engine with "mmapv1" when the metadata file specifies "mmapv1". - TEST(StorageEngineMetadataTest, StorageEngineForPath_MetadataFile_mmapv1) { - TempDir tempDir("StorageEngineMetadataTest_StorageEngineForPath_MetadataFile_mmapv1"); + // Data directory is not empty but metadata is missing. + // Data directory does not contain either local.ns or local/local.ns. + // Current storage engine is not 'mmapv1'. + TEST(StorageEngineMetadataTest, ValidateMissingMetadataMissingMMapV1Files2) { + TempDir tempDir("StorageEngineMetadataTest_ValidateMissingMetadataMissingMMapV1Files2"); + { + std::string filename(tempDir.path() + "/user_data.txt"); + std::ofstream ofs(filename.c_str()); + ofs << "unused data" << std::endl; + } + std::auto_ptr<StorageEngineMetadata> metadata = + StorageEngineMetadata::validate(tempDir.path(), "engine1"); + ASSERT_FALSE(metadata.get()); + } + + TEST(StorageEngineMetadataTest, ValidateMetadataMatchesStorageEngine) { + TempDir tempDir("StorageEngineMetadataTest_ValidateMetadataMatchesStorageEngine"); { StorageEngineMetadata metadata(tempDir.path()); - metadata.setStorageEngine("mmapv1"); + metadata.setStorageEngine("storageEngine1"); ASSERT_OK(metadata.write()); } - ASSERT_EQUALS(std::string("mmapv1"), - StorageEngineMetadata::getStorageEngineForPath(tempDir.path())); + std::auto_ptr<StorageEngineMetadata> metadata = + StorageEngineMetadata::validate(tempDir.path(), "storageEngine1"); + ASSERT_TRUE(metadata.get()); + ASSERT_EQUALS("storageEngine1", metadata->getStorageEngine()); } - // Override the active storage engine whatever the metadata file specifies. - TEST(StorageEngineMetadataTest, StorageEngineForPath_MetadataFile_someEngine) { - TempDir tempDir("StorageEngineMetadataTest_StorageEngineForPath_MetadataFile_someEngine"); + TEST(StorageEngineMetadataTest, ValidateMetadataDifferentFromStorageEngine) { + TempDir tempDir("StorageEngineMetadataTest_ValidateMetadataDifferentFromStorageEngine"); { StorageEngineMetadata metadata(tempDir.path()); - metadata.setStorageEngine("someEngine"); + metadata.setStorageEngine("storageEngine1"); ASSERT_OK(metadata.write()); } - ASSERT_EQUALS(std::string("someEngine"), - StorageEngineMetadata::getStorageEngineForPath(tempDir.path())); + ASSERT_THROWS(StorageEngineMetadata::validate(tempDir.path(), "engine2"), UserException); + } + + TEST(StorageEngineMetadataTest, ValidateMetadataReadFailed) { + TempDir tempDir("StorageEngineMetadataTest_ValidateMetadataReadFailed"); + { + std::string filename(tempDir.path() + "/storage.bson"); + std::ofstream ofs(filename.c_str()); + // BSON document of size -1 and EOO as first element. + BSONObj obj = fromjson("{x: 1}"); + ofs.write("\xff\xff\xff\xff", 4); + ofs.write(obj.objdata()+4, obj.objsize()-4); + ofs.flush(); + } + std::auto_ptr<StorageEngineMetadata> metadata = + StorageEngineMetadata::validate(tempDir.path(), "engine2"); + ASSERT_FALSE(metadata.get()); + } + + TEST(StorageEngineMetadataTest, ValidateStorageEngineOption) { + // It is fine to provide an invalid data directory as long as we do not + // call read() or write(). + StorageEngineMetadata metadata("no_such_directory"); + BSONObj options = fromjson("{x: true, y: false, z: 123}"); + metadata.setStorageEngineOptions(options); + + // Non-existent field. + ASSERT_OK(metadata.validateStorageEngineOption("w", true)); + ASSERT_OK(metadata.validateStorageEngineOption("w", false)); + + // Non-boolean field. + Status status = metadata.validateStorageEngineOption("z", true); + ASSERT_NOT_OK(status); + ASSERT_EQUALS(ErrorCodes::FailedToParse, status.code()); + status = metadata.validateStorageEngineOption("z", false); + ASSERT_NOT_OK(status); + ASSERT_EQUALS(ErrorCodes::FailedToParse, status.code()); + + // Boolean fields. + ASSERT_OK(metadata.validateStorageEngineOption("x", true)); + status = metadata.validateStorageEngineOption("x", false); + ASSERT_NOT_OK(status); + ASSERT_EQUALS(ErrorCodes::InvalidOptions, status.code()); + + ASSERT_OK(metadata.validateStorageEngineOption("y", false)); + status = metadata.validateStorageEngineOption("y", true); + ASSERT_NOT_OK(status); + ASSERT_EQUALS(ErrorCodes::InvalidOptions, status.code()); } } // namespace diff --git a/src/mongo/db/storage_options.h b/src/mongo/db/storage_options.h index 3967af7139a..60e007b1ff6 100644 --- a/src/mongo/db/storage_options.h +++ b/src/mongo/db/storage_options.h @@ -49,8 +49,7 @@ namespace mongo { static const char* kDefaultConfigDbPath; StorageGlobalParams() : - engine("wiredTiger"), - engineSetByUser(false), + engine("mmapv1"), dbpath(kDefaultDbPath), upgrade(false), repair(false), @@ -66,9 +65,6 @@ namespace mongo { // storage engine for this instance of mongod. std::string engine; - // True if --storageEngine was passed on the command line, and false otherwise. - bool engineSetByUser; - // The directory where the mongod instance stores its data. std::string dbpath; diff --git a/src/mongo/dbtests/framework.cpp b/src/mongo/dbtests/framework.cpp index 3b405853665..6a670a3e66b 100644 --- a/src/mongo/dbtests/framework.cpp +++ b/src/mongo/dbtests/framework.cpp @@ -121,7 +121,7 @@ namespace mongo { printGitVersion(); printOpenSSLVersion(); - getGlobalServiceContext()->initializeGlobalStorageEngine(); + getGlobalServiceContext()->setGlobalStorageEngine(storageGlobalParams.engine); // Initialize the sharding state so we can run starding tests in isolation shardingState.initialize("$dummy:10000"); diff --git a/src/mongo/dbtests/framework_options.cpp b/src/mongo/dbtests/framework_options.cpp index 2e8daff0aff..cf0c553d8b7 100644 --- a/src/mongo/dbtests/framework_options.cpp +++ b/src/mongo/dbtests/framework_options.cpp @@ -88,7 +88,7 @@ namespace mongo { options->addOptionChaining("storage.engine", "storageEngine", moe::String, "what storage engine to use") - .setDefault(moe::Value(std::string("wiredTiger"))); + .setDefault(moe::Value(std::string("mmapv1"))); options->addOptionChaining("suites", "suites", moe::StringVector, "test suites to run") .hidden() diff --git a/src/mongo/shell/servers.js b/src/mongo/shell/servers.js index 1da9db35ffe..c4ec1491a1f 100755 --- a/src/mongo/shell/servers.js +++ b/src/mongo/shell/servers.js @@ -882,7 +882,6 @@ startMongoProgram = function(){ runMongoProgram = function() { var args = argumentsToArray( arguments ); - args = appendSetParameterArgs(args); var progName = args[0]; if ( jsTestOptions().auth ) { @@ -908,7 +907,6 @@ runMongoProgram = function() { // command line arguments to the program. Returns pid of the spawned program. startMongoProgramNoConnect = function() { var args = argumentsToArray( arguments ); - args = appendSetParameterArgs(args); var progName = args[0]; if ( jsTestOptions().auth ) { |