diff options
author | Xiangyu Yao <xiangyu.yao@mongodb.com> | 2018-08-20 16:43:41 -0400 |
---|---|---|
committer | Xiangyu Yao <xiangyu.yao@mongodb.com> | 2018-08-23 17:14:48 -0400 |
commit | 238af0b7739e6c4702e234cfb28e3a714f119523 (patch) | |
tree | c7509a9ba6f10961498362981d796181869aacd7 | |
parent | e7afdfb2941808f078cae1aafa05e40bdb079766 (diff) | |
download | mongo-238af0b7739e6c4702e234cfb28e3a714f119523.tar.gz |
SERVER-36768 Use namespace orphan for the recovered orphaned collections
-rw-r--r-- | jstests/disk/wt_repair_corrupt_files.js | 5 | ||||
-rw-r--r-- | jstests/disk/wt_repair_orphaned_idents.js | 88 | ||||
-rw-r--r-- | src/mongo/db/namespace_string.h | 2 | ||||
-rw-r--r-- | src/mongo/db/storage/kv/kv_catalog.cpp | 11 | ||||
-rw-r--r-- | src/mongo/db/storage/kv/kv_catalog.h | 2 | ||||
-rw-r--r-- | src/mongo/db/storage/kv/kv_storage_engine.cpp | 15 | ||||
-rw-r--r-- | src/mongo/db/storage/kv/kv_storage_engine_test.cpp | 8 |
7 files changed, 118 insertions, 13 deletions
diff --git a/jstests/disk/wt_repair_corrupt_files.js b/jstests/disk/wt_repair_corrupt_files.js index 331828ed763..c19a6d672d1 100644 --- a/jstests/disk/wt_repair_corrupt_files.js +++ b/jstests/disk/wt_repair_corrupt_files.js @@ -84,7 +84,7 @@ /** * Test 3. Corrupt the _mdb_catalog in an unrecoverable way. Verify that repair suceeds * in creating an empty catalog and recovers the orphaned testColl, which will still be - * accessible in the 'local.system.orphan-' namespace. + * accessible in the 'local.orphan-' namespace. */ let mdbCatalogFile = dbpath + "_mdb_catalog.wt"; @@ -100,7 +100,8 @@ assert.eq(testColl.count(), 0); // Ensure the collection orphan was created with the existing document. - let orphanColl = mongod.getDB('local').getCollection('system.orphan-' + testCollUri); + const orphanCollName = "orphan." + testCollUri.replace(/-/g, "_"); + let orphanColl = mongod.getDB('local').getCollection(orphanCollName); assert(orphanColl.exists()); assert.eq(orphanColl.find(doc).itcount(), 1); assert.eq(orphanColl.count(), 1); diff --git a/jstests/disk/wt_repair_orphaned_idents.js b/jstests/disk/wt_repair_orphaned_idents.js new file mode 100644 index 00000000000..43d57f1ff28 --- /dev/null +++ b/jstests/disk/wt_repair_orphaned_idents.js @@ -0,0 +1,88 @@ +/** + * Tests that --repair on WiredTiger creates new entries for orphaned idents in the catalog. + * + * @tags: [requires_wiredtiger] + */ + +(function() { + + load('jstests/disk/libs/wt_file_helper.js'); + + const baseName = "wt_repair_orphaned_idents"; + const dbpath = MongoRunner.dataPath + baseName + "/"; + + resetDbpath(dbpath); + + // Create a collection and insert a doc. + let mongod = MongoRunner.runMongod({dbpath: dbpath}); + const importantCollName = "importantColl"; + const importantDocId = "importantDoc"; + const importantColl = mongod.getDB("test")[importantCollName]; + assert.commandWorked(importantColl.insert({_id: importantDocId})); + const importantCollIdent = getUriForColl(importantColl); + MongoRunner.stopMongod(mongod); + + // Delete the _mdb_catalog. + let mdbCatalogFile = dbpath + "_mdb_catalog.wt"; + jsTestLog("deleting catalog file: " + mdbCatalogFile); + removeFile(mdbCatalogFile); + + // Repair crates the _mdb_catalog and catalog entries for all the orphaned idents. + jsTestLog("running mongod with --repair"); + assert.eq(0, runMongoProgram("mongod", "--repair", "--port", mongod.port, "--dbpath", dbpath)); + + jsTestLog("restarting mongod"); + mongod = MongoRunner.runMongod({dbpath: dbpath, noCleanData: true}); + + let localDb = mongod.getDB("local"); + let res = localDb.runCommand({listCollections: 1}); + assert.commandWorked(res, tojson(res)); + + // This is the function that 'show collections' uses. + let collNames = localDb.getCollectionNames(); + + const orphanPrefix = "orphan."; + let recoveredCount = 0; + const orphanedImportantCollName = "orphan." + importantCollIdent.replace(/-/g, "_"); + for (let collName of collNames) { + if (collName.startsWith(orphanPrefix)) { + // Manually create the _id index. + assert.commandWorked(localDb[collName].createIndex({_id: 1})); + + if (collName == orphanedImportantCollName) { + assert.commandWorked(localDb.adminCommand( + {renameCollection: "local." + collName, to: "test." + importantCollName})); + } else { + assert.commandWorked(localDb.adminCommand({ + renameCollection: "local." + collName, + to: "test.recovered" + recoveredCount + })); + } + recoveredCount++; + } + } + assert.gt(recoveredCount, 0); + + let testDb = mongod.getDB("test"); + + // Assert the recovered collection still has the original document. + assert.eq(testDb[importantCollName].find({_id: importantDocId}).count(), 1); + + res = testDb.runCommand({listCollections: 1}); + assert.commandWorked(res); + assert.eq(res.cursor.firstBatch.length, recoveredCount); + for (let entry of res.cursor.firstBatch) { + let collName = entry.name; + assert(collName.startsWith("recovered") || collName == importantCollName); + + // Assert _id index has been successfully created. + assert("idIndex" in entry); + + // Make sure we can interact with the recovered collections. + assert.commandWorked(testDb.runCommand({find: collName})); + assert.commandWorked(testDb[collName].insert({x: 1})); + assert(testDb[collName].drop()); + } + + MongoRunner.stopMongod(mongod); +})(); diff --git a/src/mongo/db/namespace_string.h b/src/mongo/db/namespace_string.h index 9334d5a6f84..f6ef0ce8ca6 100644 --- a/src/mongo/db/namespace_string.h +++ b/src/mongo/db/namespace_string.h @@ -61,7 +61,7 @@ public: static constexpr StringData kSystemDotViewsCollectionName = "system.views"_sd; // Prefix for orphan collections - static constexpr StringData kOrphanCollectionPrefix = "system.orphan-"_sd; + static constexpr StringData kOrphanCollectionPrefix = "orphan."_sd; static constexpr StringData kOrphanCollectionDb = "local"_sd; // Namespace for storing configuration data, which needs to be replicated if the server is diff --git a/src/mongo/db/storage/kv/kv_catalog.cpp b/src/mongo/db/storage/kv/kv_catalog.cpp index cb74359dd34..51c6e4943f8 100644 --- a/src/mongo/db/storage/kv/kv_catalog.cpp +++ b/src/mongo/db/storage/kv/kv_catalog.cpp @@ -632,14 +632,19 @@ bool KVCatalog::isCollectionIdent(StringData ident) const { } StatusWith<std::string> KVCatalog::newOrphanedIdent(OperationContext* opCtx, std::string ident) { - // The collection will be named local.system.orphan-xxxxx. + // The collection will be named local.orphan.xxxxx. + std::string identNs = ident; + std::replace(identNs.begin(), identNs.end(), '-', '_'); std::string ns = NamespaceString(NamespaceString::kOrphanCollectionDb, - NamespaceString::kOrphanCollectionPrefix + ident) + NamespaceString::kOrphanCollectionPrefix + identNs) .ns(); stdx::lock_guard<stdx::mutex> lk(_identsLock); Entry& old = _idents[ns]; - invariant(old.ident.empty()); + if (!old.ident.empty()) { + return Status(ErrorCodes::NamespaceExists, + str::stream() << ns << " already exists in the catalog"); + } opCtx->recoveryUnit()->registerChange(new AddIdentChange(this, ns)); // Generate a new UUID for the orphaned collection. diff --git a/src/mongo/db/storage/kv/kv_catalog.h b/src/mongo/db/storage/kv/kv_catalog.h index d8a1a6e91d3..a9827a1fce8 100644 --- a/src/mongo/db/storage/kv/kv_catalog.h +++ b/src/mongo/db/storage/kv/kv_catalog.h @@ -106,6 +106,8 @@ public: /** * Create an entry in the catalog for an orphaned collection found in the * storage engine. Return the generated ns of the collection. + * Note that this function does not recreate the _id index on the collection because it does not + * have access to index catalog. */ StatusWith<std::string> newOrphanedIdent(OperationContext* opCtx, std::string ident); diff --git a/src/mongo/db/storage/kv/kv_storage_engine.cpp b/src/mongo/db/storage/kv/kv_storage_engine.cpp index 978458820eb..8980b43ab43 100644 --- a/src/mongo/db/storage/kv/kv_storage_engine.cpp +++ b/src/mongo/db/storage/kv/kv_storage_engine.cpp @@ -155,7 +155,7 @@ void KVStorageEngine::loadCatalog(OperationContext* opCtx) { if (_options.forRepair) { // It's possible that there are collection files on disk that are unknown to the catalog. In // a repair context, if we can't find an ident in the catalog, we generate a catalog entry - // 'local.system.orphan-xxxxx' for it. However, in a nonrepair context, the orphaned idents + // 'local.orphan.xxxxx' for it. However, in a nonrepair context, the orphaned idents // will be dropped in reconcileCatalogAndIdents(). for (const auto& ident : identsKnownToStorageEngine) { if (_catalog->isCollectionIdent(ident)) { @@ -171,9 +171,13 @@ void KVStorageEngine::loadCatalog(OperationContext* opCtx) { StatusWith<std::string> statusWithNs = _catalog->newOrphanedIdent(opCtx, ident); if (statusWithNs.isOK()) { wuow.commit(); + auto orphanCollNs = statusWithNs.getValue(); log() << "Successfully created an entry in the catalog for the orphaned " "collection: " - << statusWithNs.getValue(); + << orphanCollNs; + warning() << orphanCollNs + << " does not have the _id index. Please manually " + "build the index."; StorageRepairObserver::get(getGlobalServiceContext()) ->onModification(str::stream() << "Orphan collection created: " @@ -182,9 +186,10 @@ void KVStorageEngine::loadCatalog(OperationContext* opCtx) { } else { // Log an error message if we cannot create the entry. // reconcileCatalogAndIdents() will later drop this ident. - error() - << "Cannot create an entry in the catalog for the orphaned collection: " - << ident << " due to " << statusWithNs.getStatus().reason(); + error() << "Cannot create an entry in the catalog for the orphaned " + "collection ident: " + << ident << " due to " << statusWithNs.getStatus().reason(); + error() << "Restarting the server will remove this ident."; } } } diff --git a/src/mongo/db/storage/kv/kv_storage_engine_test.cpp b/src/mongo/db/storage/kv/kv_storage_engine_test.cpp index c89b0d9ac3a..752f16e43f9 100644 --- a/src/mongo/db/storage/kv/kv_storage_engine_test.cpp +++ b/src/mongo/db/storage/kv/kv_storage_engine_test.cpp @@ -336,7 +336,9 @@ TEST_F(KVStorageEngineRepairTest, LoadCatalogRecoversOrphansInCatalog) { // When in a repair context, loadCatalog() recreates catalog entries for orphaned idents. _storageEngine->loadCatalog(opCtx.get()); - NamespaceString orphanNs = NamespaceString("local.system.orphan-" + swIdentName.getValue()); + auto identNs = swIdentName.getValue(); + std::replace(identNs.begin(), identNs.end(), '-', '_'); + NamespaceString orphanNs = NamespaceString("local.orphan." + identNs); ASSERT(identExists(opCtx.get(), swIdentName.getValue())); ASSERT(collectionExists(opCtx.get(), orphanNs)); @@ -367,7 +369,9 @@ TEST_F(KVStorageEngineTest, LoadCatalogDropsOrphans) { ASSERT_OK(reconcile(opCtx.get()).getStatus()); ASSERT(!identExists(opCtx.get(), swIdentName.getValue())); - NamespaceString orphanNs = NamespaceString("local.system.orphan-" + swIdentName.getValue()); + auto identNs = swIdentName.getValue(); + std::replace(identNs.begin(), identNs.end(), '-', '_'); + NamespaceString orphanNs = NamespaceString("local.orphan." + identNs); ASSERT(!collectionExists(opCtx.get(), orphanNs)); } } // namespace |