diff options
author | Andy Schwerin <schwerin@mongodb.com> | 2016-04-04 19:01:43 -0400 |
---|---|---|
committer | Andy Schwerin <schwerin@mongodb.com> | 2016-04-07 11:32:38 -0400 |
commit | 5f23fb00bf69a19b67c385e7e953137c70eaa3e0 (patch) | |
tree | dc3635ecfa30af793d571261524b2feb16a62723 | |
parent | 0953a237e58d2af33a55c420d8aefbbb3d4ff88b (diff) | |
download | mongo-5f23fb00bf69a19b67c385e7e953137c70eaa3e0.tar.gz |
SERVER-23299 When starting mongod, clear temp flags from collections if appropriate.
See the description in SERVER-23299 for the definition of appropriate.
(cherry picked from commit 5e11e4484b84c8f913a4b14b24fbb7c9c46ee40f)
8 files changed, 242 insertions, 6 deletions
diff --git a/jstests/multiVersion/server-23299-1.js b/jstests/multiVersion/server-23299-1.js new file mode 100644 index 00000000000..02e34994c47 --- /dev/null +++ b/jstests/multiVersion/server-23299-1.js @@ -0,0 +1,92 @@ +/** + * This test confirms the correct implementation of SERVER-23299. Due + * to a bug fixed as SERVER-23274, replica set secondaries sometimes + * fail to clear the "temp" flag on collections that have been renamed + * as a means of marking the collection as permanent. This + * particularly happens with collections created by the $out + * aggregation stage, in versions 3.2.0 through 3.2.1. + * + * This test creates a collection using $out in 3.2.1, confirms that + * the collection gets incorrectly deleted on secondary failover, then + * confirms that if the user upgrades from 3.2.1 to latest before a + * failover, then a collection produced with $out in 3.2.1 is not + * deleted. + */ + +load('./jstests/multiVersion/libs/multi_rs.js'); + +(function() { + "use strict"; + + var oldVersion = "3.2.1"; + var newVersion = "latest"; + + var ex; + function getTestDbForNode(node) { + return node.getDB("test"); + } + + function expectTargetCollectionSize(node, sz) { + assert.eq(sz, getTestDbForNode(node).target.find().itcount(), "On node " + node.host); + } + + var rst = new ReplSetTest({nodes: 2}); + rst.startSet({binVersion: oldVersion}); + rst.initiate(); + var n0 = rst.getPrimary(); + getTestDbForNode(n0).source.insert({_id: 0}); + assert.gleOK(getTestDbForNode(n0).getLastErrorObj()); + + jsTest.log("Performing aggregation to create target collection on " + n0.host); + getTestDbForNode(n0).source.aggregate({$out: "target"}); + expectTargetCollectionSize(n0, 1); + rst.awaitReplication(); + expectTargetCollectionSize(rst.getSecondary(), 1); + + jsTest.log("Stepping down " + n0.host); + try { + n0.adminCommand({replSetStepDown: 1200}); + } catch (ex) { + assert(tojson(ex).includes( + "network error while attempting to run command 'replSetStepDown' on host"), + tojson(ex)); + } + var n1 = rst.getPrimary(); + + jsTest.log("Confirming that SERVER-23274 is present in " + oldVersion); + + assert.neq( + n1.host, n0.host, "Failed to switch primary to other node in set away from " + n1.host); + expectTargetCollectionSize(n1, 0); + rst.awaitReplication(); + expectTargetCollectionSize(n0, 0); + + jsTest.log("Performing aggregation to create target collection on " + n1.host); + getTestDbForNode(n1).source.aggregate({$out: "target"}); + expectTargetCollectionSize(n1, 1); + rst.awaitReplication(); + expectTargetCollectionSize(n0, 1); + + rst.upgradeSet({binVersion: "latest"}); + n1 = rst.getPrimary(); + n0 = rst.getSecondary(); + + jsTest.log("Confirming that target collection remained after upgrade"); + expectTargetCollectionSize(n1, 1); + expectTargetCollectionSize(n0, 1); + + jsTest.log("Confirming that target collection remained after switching primaries"); + jsTest.log("Stepping down " + n0.host); + try { + n1.adminCommand({replSetStepDown: 1200}); + } catch (ex) { + assert(tojson(ex).includes( + "network error while attempting to run command 'replSetStepDown' on host"), + tojson(ex)); + } + var n0 = rst.getPrimary(); + assert.neq( + n1.host, n0.host, "Failed to switch primary to other node in set away from " + n1.host); + expectTargetCollectionSize(n1, 1); + expectTargetCollectionSize(n0, 1); +}()); diff --git a/jstests/multiVersion/server-23299-2.js b/jstests/multiVersion/server-23299-2.js new file mode 100644 index 00000000000..77288750f3a --- /dev/null +++ b/jstests/multiVersion/server-23299-2.js @@ -0,0 +1,44 @@ +/** + * This test confirms that the behavior prescribed in SERVER-23299 is + * only applied when starting up a node where the immediately prior + * start-up was for versions 3.2.0 through 3.2.4, inclusive. + */ + +load('./jstests/multiVersion/libs/verify_versions.js'); + +(function() { + "use strict"; + + var versionsSubjectToSERVER23299 = ['3.2.1']; + + // A smattering of versions not subject to the bug, but that we could legally encounter + var versionsNotSubjectToSERVER23299 = ['latest', '3.0']; + + function doTest(priorVersion, expectTempToDrop) { + jsTest.log((expectTempToDrop ? "" : " not") + " expecting temp collections created in " + + priorVersion + " to be dropped when starting latest mongod version"); + var mongod = MongoRunner.runMongod({binVersion: priorVersion}); + assert.binVersion(mongod, priorVersion); + assert.commandWorked(mongod.getDB("test").createCollection("tempcoll", {temp: true}), + priorVersion); + mongod.getDB("test").tempcoll.insert({_id: 0}); + assert.gleOK(mongod.getDB("test").getLastErrorObj(), priorVersion); + assert.eq(1, mongod.getDB("test").tempcoll.find().itcount(), priorVersion); + + MongoRunner.stopMongod(mongod); + var newOpts = Object.extend({}, mongod.fullOptions); + mongod = MongoRunner.runMongod(Object.extend(Object.extend({}, mongod.fullOptions), + {restart: true, binVersion: "latest"})); + assert.binVersion(mongod, "latest"); + assert.eq(expectTempToDrop ? 0 : 1, + mongod.getDB("test").tempcoll.find().itcount(), + priorVersion); + } + + versionsNotSubjectToSERVER23299.forEach(function(priorVersion) { + doTest(priorVersion, true); + }); + versionsSubjectToSERVER23299.forEach(function(priorVersion) { + doTest(priorVersion, false); + }); +}()); diff --git a/src/mongo/db/catalog/collection_catalog_entry.h b/src/mongo/db/catalog/collection_catalog_entry.h index e1261e6a409..a31843b3cce 100644 --- a/src/mongo/db/catalog/collection_catalog_entry.h +++ b/src/mongo/db/catalog/collection_catalog_entry.h @@ -98,6 +98,11 @@ public: virtual void updateFlags(OperationContext* txn, int newValue) = 0; /** + * Clears the "temp" flag from the CollectionOptions of this collection. + */ + virtual void clearTempFlag(OperationContext* txn) = 0; + + /** * Updates the validator for this collection. * * An empty validator removes all validation. diff --git a/src/mongo/db/db.cpp b/src/mongo/db/db.cpp index 583907c6619..b49310a0046 100644 --- a/src/mongo/db/db.cpp +++ b/src/mongo/db/db.cpp @@ -48,6 +48,7 @@ #include "mongo/db/auth/authorization_manager.h" #include "mongo/db/auth/authorization_manager_global.h" #include "mongo/db/catalog/collection.h" +#include "mongo/db/catalog/collection_catalog_entry.h" #include "mongo/db/catalog/database.h" #include "mongo/db/catalog/database_catalog_entry.h" #include "mongo/db/catalog/database_holder.h" @@ -56,8 +57,11 @@ #include "mongo/db/client.h" #include "mongo/db/clientcursor.h" #include "mongo/db/concurrency/d_concurrency.h" +#include "mongo/db/concurrency/lock_state.h" +#include "mongo/db/concurrency/write_conflict_exception.h" #include "mongo/db/db_raii.h" #include "mongo/db/dbdirectclient.h" +#include "mongo/db/dbhelpers.h" #include "mongo/db/dbmessage.h" #include "mongo/db/dbwebserver.h" #include "mongo/db/ftdc/ftdc_mongod.h" @@ -145,6 +149,8 @@ void (*snmpInit)() = NULL; extern int diagLogging; +static const NamespaceString startupLogCollectionName("local.startup_log"); + #ifdef _WIN32 ntservice::NtServiceDefaultStrings defaultServiceStrings = { L"MongoDB", L"MongoDB", L"MongoDB Server"}; @@ -232,18 +238,17 @@ static void logStartup(OperationContext* txn) { ScopedTransaction transaction(txn, MODE_X); Lock::GlobalWrite lk(txn->lockState()); - AutoGetOrCreateDb autoDb(txn, "local", mongo::MODE_X); + AutoGetOrCreateDb autoDb(txn, startupLogCollectionName.db(), mongo::MODE_X); Database* db = autoDb.getDb(); - const std::string ns = "local.startup_log"; - Collection* collection = db->getCollection(ns); + Collection* collection = db->getCollection(startupLogCollectionName); WriteUnitOfWork wunit(txn); if (!collection) { BSONObj options = BSON("capped" << true << "size" << 10 * 1024 * 1024); bool shouldReplicateWrites = txn->writesAreReplicated(); txn->setReplicatedWrites(false); ON_BLOCK_EXIT(&OperationContext::setReplicatedWrites, txn, shouldReplicateWrites); - uassertStatusOK(userCreateNS(txn, db, ns, options)); - collection = db->getCollection(ns); + uassertStatusOK(userCreateNS(txn, db, startupLogCollectionName.ns(), options)); + collection = db->getCollection(startupLogCollectionName); } invariant(collection); uassertStatusOK(collection->insertDocument(txn, o, false)); @@ -299,6 +304,76 @@ static unsigned long long checkIfReplMissingFromCommandLine(OperationContext* tx return 0; } +/** + * Due to SERVER-23274, versions 3.2.0 through 3.2.4 of MongoDB incorrectly mark the final output + * collections of aggregations with $out stages as temporary on most replica set secondaries. Rather + * than risk deleting collections that the user did not intend to be temporary when newer nodes + * start up or get promoted to be replica set primaries, newer nodes clear the temp flags left by + * these versions. + */ +static bool isSubjectToSERVER23299(OperationContext* txn) { + dbHolder().openDb(txn, startupLogCollectionName.db()); + AutoGetCollectionForRead autoColl(txn, startupLogCollectionName); + // No startup log or an empty one means either that the user was not running an affected + // version, or that they manually deleted the startup collection since they last started an + // affected version. + LOG(1) << "Checking node for SERVER-23299 eligibility"; + if (!autoColl.getCollection()) { + LOG(1) << "Didn't find " << startupLogCollectionName; + return false; + } + LOG(1) << "Checking node for SERVER-23299 applicability - reading startup log"; + BSONObj lastStartupLogDoc; + if (!Helpers::getLast(txn, startupLogCollectionName.ns().c_str(), lastStartupLogDoc)) { + return false; + } + std::vector<int> versionComponents; + try { + for (auto elem : lastStartupLogDoc["buildinfo"]["versionArray"].Obj()) { + versionComponents.push_back(elem.Int()); + } + uassert(40050, + str::stream() << "Expected three elements in buildinfo.versionArray; found " + << versionComponents.size(), + versionComponents.size() >= 3); + } catch (const DBException& ex) { + log() << "Last entry of " << startupLogCollectionName + << " has no well-formed buildinfo.versionArray field; ignoring " << causedBy(ex); + return false; + } + LOG(1) + << "Checking node for SERVER-23299 applicability - checking version 3.2.x for x in [0, 4]"; + if (versionComponents[0] != 3) + return false; + if (versionComponents[1] != 2) + return false; + if (versionComponents[2] > 4) + return false; + LOG(1) << "Node eligible for SERVER-23299"; + return true; +} + +static void handleSERVER23299ForDb(OperationContext* txn, Database* db) { + log() << "Scanning " << db->name() << " db for SERVER-23299 eligibility"; + const auto dbEntry = db->getDatabaseCatalogEntry(); + list<string> collNames; + dbEntry->getCollectionNamespaces(&collNames); + for (const auto& collName : collNames) { + const auto collEntry = dbEntry->getCollectionCatalogEntry(collName); + const auto collOptions = collEntry->getCollectionOptions(txn); + if (!collOptions.temp) + continue; + log() << "Marking collection " << collName << " as permanent per SERVER-23299"; + MONGO_WRITE_CONFLICT_RETRY_LOOP_BEGIN { + WriteUnitOfWork wuow(txn); + collEntry->clearTempFlag(txn); + wuow.commit(); + } + MONGO_WRITE_CONFLICT_RETRY_LOOP_END(txn, "repair SERVER-23299", collEntry->ns().ns()); + } + log() << "Done scanning " << db->name() << " for SERVER-23299 eligibility"; +} + static void repairDatabasesAndCheckVersion(OperationContext* txn) { LOG(1) << "enter repairDatabases (to check pdfile version #)" << endl; @@ -307,7 +382,7 @@ static void repairDatabasesAndCheckVersion(OperationContext* txn) { vector<string> dbNames; - StorageEngine* storageEngine = getGlobalServiceContext()->getGlobalStorageEngine(); + StorageEngine* storageEngine = txn->getServiceContext()->getGlobalStorageEngine(); storageEngine->listDatabases(&dbNames); // Repair all databases first, so that we do not try to open them if they are in bad shape @@ -330,6 +405,8 @@ static void repairDatabasesAndCheckVersion(OperationContext* txn) { !(checkIfReplMissingFromCommandLine(txn) || replSettings.usingReplSets() || replSettings.isSlave()); + const bool shouldDoCleanupForSERVER23299 = isSubjectToSERVER23299(txn); + for (vector<string>::const_iterator i = dbNames.begin(); i != dbNames.end(); ++i) { const string dbName = *i; LOG(1) << " Recovering database: " << dbName << endl; @@ -398,6 +475,10 @@ static void repairDatabasesAndCheckVersion(OperationContext* txn) { checkForIdIndexes(txn, db); } + if (shouldDoCleanupForSERVER23299) { + handleSERVER23299ForDb(txn, db); + } + if (shouldClearNonLocalTmpCollections || dbName == "local") { db->clearTmpCollections(txn); } diff --git a/src/mongo/db/storage/kv/kv_collection_catalog_entry.cpp b/src/mongo/db/storage/kv/kv_collection_catalog_entry.cpp index 2381a80fff6..38a9d2b3b09 100644 --- a/src/mongo/db/storage/kv/kv_collection_catalog_entry.cpp +++ b/src/mongo/db/storage/kv/kv_collection_catalog_entry.cpp @@ -163,6 +163,12 @@ void KVCollectionCatalogEntry::updateFlags(OperationContext* txn, int newValue) _catalog->putMetaData(txn, ns().toString(), md); } +void KVCollectionCatalogEntry::clearTempFlag(OperationContext* txn) { + MetaData md = _getMetaData(txn); + md.options.temp = false; + _catalog->putMetaData(txn, ns().ns(), md); +} + void KVCollectionCatalogEntry::updateValidator(OperationContext* txn, const BSONObj& validator, StringData validationLevel, diff --git a/src/mongo/db/storage/kv/kv_collection_catalog_entry.h b/src/mongo/db/storage/kv/kv_collection_catalog_entry.h index 11cc4ecafc6..3a3704e82d5 100644 --- a/src/mongo/db/storage/kv/kv_collection_catalog_entry.h +++ b/src/mongo/db/storage/kv/kv_collection_catalog_entry.h @@ -68,6 +68,8 @@ public: void updateFlags(OperationContext* txn, int newValue) final; + void clearTempFlag(OperationContext* txn) final; + void updateValidator(OperationContext* txn, const BSONObj& validator, StringData validationLevel, diff --git a/src/mongo/db/storage/mmap_v1/catalog/namespace_details_collection_entry.cpp b/src/mongo/db/storage/mmap_v1/catalog/namespace_details_collection_entry.cpp index 736c3c11acc..278474cabb0 100644 --- a/src/mongo/db/storage/mmap_v1/catalog/namespace_details_collection_entry.cpp +++ b/src/mongo/db/storage/mmap_v1/catalog/namespace_details_collection_entry.cpp @@ -372,6 +372,10 @@ void NamespaceDetailsCollectionCatalogEntry::updateFlags(OperationContext* txn, _updateSystemNamespaces(txn, BSON("$set" << BSON("options.flags" << newValue))); } +void NamespaceDetailsCollectionCatalogEntry::clearTempFlag(OperationContext* txn) { + _updateSystemNamespaces(txn, BSON("$set" << BSON("options.temp" << false))); +} + void NamespaceDetailsCollectionCatalogEntry::updateValidator(OperationContext* txn, const BSONObj& validator, StringData validationLevel, diff --git a/src/mongo/db/storage/mmap_v1/catalog/namespace_details_collection_entry.h b/src/mongo/db/storage/mmap_v1/catalog/namespace_details_collection_entry.h index 4bdebf31d5e..89760adf99a 100644 --- a/src/mongo/db/storage/mmap_v1/catalog/namespace_details_collection_entry.h +++ b/src/mongo/db/storage/mmap_v1/catalog/namespace_details_collection_entry.h @@ -93,6 +93,8 @@ public: void updateFlags(OperationContext* txn, int newValue) final; + void clearTempFlag(OperationContext* txn) final; + void updateValidator(OperationContext* txn, const BSONObj& validator, StringData validationLevel, |