diff options
author | Louis Williams <louis.williams@mongodb.com> | 2018-06-26 14:29:03 -0400 |
---|---|---|
committer | Louis Williams <louis.williams@mongodb.com> | 2018-07-10 15:31:33 -0400 |
commit | ca2b902002110013479af34f0ce6dee9906d9ad9 (patch) | |
tree | 1d30c86aac8ea372ee9fbf6d8d1fde02b41f2669 | |
parent | acd9fd112d1c2f591f04a31ed6489d9f4b0ec0e8 (diff) | |
download | mongo-ca2b902002110013479af34f0ce6dee9906d9ad9.tar.gz |
SERVER-28734 Provide a way to recover data files when lacking WiredTiger metadata, but have _mdb_catalog data
-rw-r--r-- | src/mongo/db/service_context_d_test_fixture.cpp | 8 | ||||
-rw-r--r-- | src/mongo/db/service_context_d_test_fixture.h | 4 | ||||
-rw-r--r-- | src/mongo/db/storage/devnull/devnull_kv_engine.h | 7 | ||||
-rw-r--r-- | src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_engine.h | 7 | ||||
-rw-r--r-- | src/mongo/db/storage/kv/kv_engine.h | 18 | ||||
-rw-r--r-- | src/mongo/db/storage/kv/kv_storage_engine.cpp | 81 | ||||
-rw-r--r-- | src/mongo/db/storage/kv/kv_storage_engine.h | 15 | ||||
-rw-r--r-- | src/mongo/db/storage/kv/kv_storage_engine_test.cpp | 86 | ||||
-rw-r--r-- | src/mongo/db/storage/mobile/mobile_kv_engine.h | 7 | ||||
-rw-r--r-- | src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp | 76 | ||||
-rw-r--r-- | src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h | 61 | ||||
-rw-r--r-- | src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine_test.cpp | 173 |
12 files changed, 483 insertions, 60 deletions
diff --git a/src/mongo/db/service_context_d_test_fixture.cpp b/src/mongo/db/service_context_d_test_fixture.cpp index 8f444a5a17c..5eec5cb6a24 100644 --- a/src/mongo/db/service_context_d_test_fixture.cpp +++ b/src/mongo/db/service_context_d_test_fixture.cpp @@ -51,11 +51,16 @@ namespace mongo { ServiceContextMongoDTest::ServiceContextMongoDTest() : ServiceContextMongoDTest("ephemeralForTest") {} -ServiceContextMongoDTest::ServiceContextMongoDTest(std::string engine) { +ServiceContextMongoDTest::ServiceContextMongoDTest(std::string engine) + : ServiceContextMongoDTest(engine, RepairAction::kNoRepair) {} + +ServiceContextMongoDTest::ServiceContextMongoDTest(std::string engine, RepairAction repair) { _stashedStorageParams.engine = std::exchange(storageGlobalParams.engine, std::move(engine)); _stashedStorageParams.engineSetByUser = std::exchange(storageGlobalParams.engineSetByUser, true); + _stashedStorageParams.repair = + std::exchange(storageGlobalParams.repair, (repair == RepairAction::kRepair)); auto const serviceContext = getServiceContext(); serviceContext->setServiceEntryPoint(std::make_unique<ServiceEntryPointMongod>(serviceContext)); @@ -86,6 +91,7 @@ ServiceContextMongoDTest::~ServiceContextMongoDTest() { shutdownGlobalStorageEngineCleanly(getGlobalServiceContext()); std::swap(storageGlobalParams.engine, _stashedStorageParams.engine); std::swap(storageGlobalParams.engineSetByUser, _stashedStorageParams.engineSetByUser); + std::swap(storageGlobalParams.repair, _stashedStorageParams.repair); } } // namespace mongo diff --git a/src/mongo/db/service_context_d_test_fixture.h b/src/mongo/db/service_context_d_test_fixture.h index 19619f0d0fe..44e6a8df507 100644 --- a/src/mongo/db/service_context_d_test_fixture.h +++ b/src/mongo/db/service_context_d_test_fixture.h @@ -39,18 +39,22 @@ namespace mongo { */ class ServiceContextMongoDTest : public ServiceContextTest { protected: + enum class RepairAction { kNoRepair, kRepair }; + ServiceContextMongoDTest(); /** * Build a ServiceContextMongoDTest, using the named storage engine. */ explicit ServiceContextMongoDTest(std::string engine); + explicit ServiceContextMongoDTest(std::string engine, RepairAction repair); virtual ~ServiceContextMongoDTest(); private: struct { std::string engine; bool engineSetByUser; + bool repair; } _stashedStorageParams; }; diff --git a/src/mongo/db/storage/devnull/devnull_kv_engine.h b/src/mongo/db/storage/devnull/devnull_kv_engine.h index df356cfaae1..82af7c0d37e 100644 --- a/src/mongo/db/storage/devnull/devnull_kv_engine.h +++ b/src/mongo/db/storage/devnull/devnull_kv_engine.h @@ -107,6 +107,13 @@ public: return Status::OK(); } + virtual Status recoverOrphanedIdent(OperationContext* opCtx, + StringData ns, + StringData ident, + const CollectionOptions& options) { + return Status::OK(); + } + virtual bool hasIdent(OperationContext* opCtx, StringData ident) const { return true; } diff --git a/src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_engine.h b/src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_engine.h index 38c9ea37cef..107c494d02d 100644 --- a/src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_engine.h +++ b/src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_engine.h @@ -96,6 +96,13 @@ public: return Status::OK(); } + virtual Status recoverOrphanedIdent(OperationContext* opCtx, + StringData ns, + StringData ident, + const CollectionOptions& options) override { + return createRecordStore(opCtx, ns, ident, options); + } + virtual void cleanShutdown(){}; virtual bool hasIdent(OperationContext* opCtx, StringData ident) const { diff --git a/src/mongo/db/storage/kv/kv_engine.h b/src/mongo/db/storage/kv/kv_engine.h index d462f8158a5..e4dfab66664 100644 --- a/src/mongo/db/storage/kv/kv_engine.h +++ b/src/mongo/db/storage/kv/kv_engine.h @@ -163,6 +163,24 @@ public: virtual Status dropIdent(OperationContext* opCtx, StringData ident) = 0; + /** + * Attempts to locate and recover a file that is "orphaned" from the storage engine's metadata, + * but may still exist on disk if this is a durable storage engine. Returns Status::OK if a new + * record store was successfully created. + * + * This may return an error if a storage engine chooses not to implement recovery, is unable to + * implement recovery because it is not durable, or if the storage engine attempted to recover + * the file and failed. A storage engine that is not durable may choose to return an OK status + * after creating a new, empty record store. + * + * This recovery process makes no guarantees about the integrity of data recovered or even that + * it still exists when recovered. + */ + virtual Status recoverOrphanedIdent(OperationContext* opCtx, + StringData ns, + StringData ident, + const CollectionOptions& options) = 0; + virtual void alterIdentMetadata(OperationContext* opCtx, StringData ident, const IndexDescriptor* desc){}; diff --git a/src/mongo/db/storage/kv/kv_storage_engine.cpp b/src/mongo/db/storage/kv/kv_storage_engine.cpp index 16eed29b2f9..157bb27e788 100644 --- a/src/mongo/db/storage/kv/kv_storage_engine.cpp +++ b/src/mongo/db/storage/kv/kv_storage_engine.cpp @@ -127,10 +127,13 @@ void KVStorageEngine::loadCatalog(OperationContext* opCtx) { _catalogRecordStore.get(), _options.directoryPerDB, _options.directoryForIndexes)); _catalog->init(opCtx); - // We populate 'identsKnownToStorageEngine' only if we are loading after an unclean shutdown. + // We populate 'identsKnownToStorageEngine' only if we are loading after an unclean shutdown or + // doing repair. + const bool loadingFromUncleanShutdownOrRepair = + startingAfterUncleanShutdown(getGlobalServiceContext()) || _options.forRepair; + std::vector<std::string> identsKnownToStorageEngine; - const bool loadingFromUncleanShutdown = startingAfterUncleanShutdown(getGlobalServiceContext()); - if (loadingFromUncleanShutdown) { + if (loadingFromUncleanShutdownOrRepair) { identsKnownToStorageEngine = _engine->getAllIdents(opCtx); std::sort(identsKnownToStorageEngine.begin(), identsKnownToStorageEngine.end()); } @@ -143,21 +146,18 @@ void KVStorageEngine::loadCatalog(OperationContext* opCtx) { NamespaceString nss(coll); std::string dbName = nss.db().toString(); - if (loadingFromUncleanShutdown) { - // If we are loading the catalog after an unclean shutdown, it's possible that there are - // collections in the catalog that are unknown to the storage engine. If we can't find - // it in the list of storage engine idents, remove the collection and move on to the - // next one. + if (loadingFromUncleanShutdownOrRepair) { + // If we are loading the catalog after an unclean shutdown or during repair, it's + // possible that there are collections in the catalog that are unknown to the storage + // engine. If we can't find a table in the list of storage engine idents, either + // attempt to recover the ident or drop it. const auto collectionIdent = _catalog->getCollectionIdent(coll); - if (!std::binary_search(identsKnownToStorageEngine.begin(), - identsKnownToStorageEngine.end(), - collectionIdent)) { - log() << "Dropping collection " << coll - << " unknown to storage engine after unclean shutdown"; - - WriteUnitOfWork wuow(opCtx); - fassert(50716, _catalog->dropCollection(opCtx, coll)); - wuow.commit(); + bool orphan = !std::binary_search(identsKnownToStorageEngine.begin(), + identsKnownToStorageEngine.end(), + collectionIdent); + // If the storage engine is missing a collection and is unable to create a new record + // store, continue past the following logic. + if (orphan && !_recoverOrDropOrphanedCollection(opCtx, coll, collectionIdent)) { continue; } } @@ -198,6 +198,37 @@ void KVStorageEngine::closeCatalog(OperationContext* opCtx) { _catalogRecordStore.reset(nullptr); } +bool KVStorageEngine::_recoverOrDropOrphanedCollection(OperationContext* opCtx, + StringData collectionName, + StringData collectionIdent) { + + if (_options.forRepair) { + log() << "Storage engine is missing collection '" << collectionName + << "' from its metadata. Attempting to locate and recover the data for " + << collectionIdent; + + WriteUnitOfWork wuow(opCtx); + const auto metadata = _catalog->getMetaData(opCtx, collectionName); + auto status = + _engine->recoverOrphanedIdent(opCtx, collectionName, collectionIdent, metadata.options); + if (status.isOK()) { + wuow.commit(); + return true; + } + + warning() << "Failed to recover orphaned data file for collection '" << collectionName + << "': " << status; + } + + log() << "Dropping collection " << collectionName + << " from the catalog unknown to the storage engine"; + + WriteUnitOfWork wuow(opCtx); + fassert(50716, _catalog->dropCollection(opCtx, collectionName)); + wuow.commit(); + return false; +} + /** * This method reconciles differences between idents the KVEngine is aware of and the * KVCatalog. There are three differences to consider: @@ -263,13 +294,15 @@ KVStorageEngine::reconcileCatalogAndIdents(OperationContext* opCtx) { // other contexts such as `recoverToStableTimestamp`. std::vector<std::string> collections; _catalog->getAllCollections(&collections); - for (const auto& coll : collections) { - const auto& identForColl = _catalog->getCollectionIdent(coll); - if (engineIdents.find(identForColl) == engineIdents.end()) { - return {ErrorCodes::UnrecoverableRollbackError, - str::stream() << "Expected collection does not exist. Collection: " << coll - << " Ident: " - << identForColl}; + if (!_options.forRepair) { + for (const auto& coll : collections) { + const auto& identForColl = _catalog->getCollectionIdent(coll); + if (engineIdents.find(identForColl) == engineIdents.end()) { + return {ErrorCodes::UnrecoverableRollbackError, + str::stream() << "Expected collection does not exist. Collection: " << coll + << " Ident: " + << identForColl}; + } } } diff --git a/src/mongo/db/storage/kv/kv_storage_engine.h b/src/mongo/db/storage/kv/kv_storage_engine.h index d839f149af6..f57b0e0d7d9 100644 --- a/src/mongo/db/storage/kv/kv_storage_engine.h +++ b/src/mongo/db/storage/kv/kv_storage_engine.h @@ -191,6 +191,21 @@ private: CollIter begin, CollIter end); + /** + * When called in a repair context (_options.forRepair=true), attempts to recover a collection + * whose entry is present in the KVCatalog, but missing from the KVEngine. Returns 'false' and + * removes collection metadata from the KVCatalog if called outside of a repair context, the + * collection is unrecoverable, or the implementation of KVEngine::recoverOrphanedIdent returns + * an error. + * + * Returns 'true' if the collection was recovered in the KVEngine and a new record store was + * created. Recovery does not make any guarantees about the integrity of the data in the + * collection. + */ + bool _recoverOrDropOrphanedCollection(OperationContext* opCtx, + StringData collectionName, + StringData collectionIdent); + void _dumpCatalog(OperationContext* opCtx); class RemoveDBChange; diff --git a/src/mongo/db/storage/kv/kv_storage_engine_test.cpp b/src/mongo/db/storage/kv/kv_storage_engine_test.cpp index e9d6c0accab..3a6c10a74aa 100644 --- a/src/mongo/db/storage/kv/kv_storage_engine_test.cpp +++ b/src/mongo/db/storage/kv/kv_storage_engine_test.cpp @@ -45,6 +45,7 @@ #include "mongo/db/storage/kv/kv_database_catalog_entry_mock.h" #include "mongo/db/storage/kv/kv_engine.h" #include "mongo/db/storage/kv/kv_storage_engine.h" +#include "mongo/db/unclean_shutdown.h" #include "mongo/stdx/memory.h" #include "mongo/unittest/unittest.h" @@ -53,10 +54,12 @@ namespace { class KVStorageEngineTest : public ServiceContextMongoDTest { public: - KVStorageEngineTest() - : ServiceContextMongoDTest("ephemeralForTest"), + KVStorageEngineTest(RepairAction repair) + : ServiceContextMongoDTest("ephemeralForTest", repair), _storageEngine(checked_cast<KVStorageEngine*>(getServiceContext()->getStorageEngine())) {} + KVStorageEngineTest() : KVStorageEngineTest(RepairAction::kNoRepair) {} + /** * Create a collection in the catalog and in the KVEngine. Return the storage engine's `ident`. */ @@ -100,6 +103,17 @@ public: return _storageEngine->getEngine()->getAllIdents(opCtx); } + bool collectionExists(OperationContext* opCtx, const NamespaceString& nss) { + std::vector<std::string> allCollections; + _storageEngine->getCatalog()->getAllCollections(&allCollections); + return std::find(allCollections.begin(), allCollections.end(), nss.toString()) != + allCollections.end(); + } + bool identExists(OperationContext* opCtx, std::string ident) { + auto idents = getAllKVEngineIdents(opCtx); + return std::find(idents.begin(), idents.end(), ident) != idents.end(); + } + /** * Create an index with a key of `{<key>: 1}` and a `name` of <key>. */ @@ -130,6 +144,11 @@ public: KVStorageEngine* _storageEngine; }; +class KVStorageEngineRepairTest : public KVStorageEngineTest { +public: + KVStorageEngineRepairTest() : KVStorageEngineTest(RepairAction::kRepair) {} +}; + TEST_F(KVStorageEngineTest, ReconcileIdentsTest) { auto opCtx = cc().makeOperationContext(); @@ -222,5 +241,68 @@ TEST_F(KVStorageEngineTest, RecreateIndexes) { })); } +TEST_F(KVStorageEngineTest, LoadCatalogDropsOrphansAfterUncleanShutdown) { + auto opCtx = cc().makeOperationContext(); + + const NamespaceString collNs("db.coll1"); + auto swIdentName = createCollection(opCtx.get(), collNs); + ASSERT_OK(swIdentName); + + ASSERT_OK(dropIdent(opCtx.get(), swIdentName.getValue())); + ASSERT(collectionExists(opCtx.get(), collNs)); + + // After the catalog is reloaded, we expect that the collection has been dropped because the + // KVEngine was started after an unclean shutdown but not in a repair context. + { + Lock::GlobalWrite writeLock(opCtx.get(), Date_t::max(), Lock::InterruptBehavior::kThrow); + _storageEngine->closeCatalog(opCtx.get()); + startingAfterUncleanShutdown(getGlobalServiceContext()) = true; + _storageEngine->loadCatalog(opCtx.get()); + } + + ASSERT(!identExists(opCtx.get(), swIdentName.getValue())); + ASSERT(!collectionExists(opCtx.get(), collNs)); +} + +TEST_F(KVStorageEngineRepairTest, LoadCatalogRecoversOrphans) { + auto opCtx = cc().makeOperationContext(); + + const NamespaceString collNs("db.coll1"); + auto swIdentName = createCollection(opCtx.get(), collNs); + ASSERT_OK(swIdentName); + + ASSERT_OK(dropIdent(opCtx.get(), swIdentName.getValue())); + ASSERT(collectionExists(opCtx.get(), collNs)); + + // After the catalog is reloaded, we expect that the ident has been recovered because the + // KVEngine was started in a repair context. + { + Lock::GlobalWrite writeLock(opCtx.get(), Date_t::max(), Lock::InterruptBehavior::kThrow); + _storageEngine->closeCatalog(opCtx.get()); + _storageEngine->loadCatalog(opCtx.get()); + } + + ASSERT(identExists(opCtx.get(), swIdentName.getValue())); + ASSERT(collectionExists(opCtx.get(), collNs)); +} + +TEST_F(KVStorageEngineRepairTest, ReconcileSucceeds) { + auto opCtx = cc().makeOperationContext(); + + const NamespaceString collNs("db.coll1"); + auto swIdentName = createCollection(opCtx.get(), collNs); + ASSERT_OK(swIdentName); + + ASSERT_OK(dropIdent(opCtx.get(), swIdentName.getValue())); + ASSERT(collectionExists(opCtx.get(), collNs)); + + // Reconcile would normally return an error if a collection existed with a missing ident in the + // storage engine. When in a repair context, that should not be the case. + ASSERT_OK(reconcile(opCtx.get()).getStatus()); + + ASSERT(!identExists(opCtx.get(), swIdentName.getValue())); + ASSERT(collectionExists(opCtx.get(), collNs)); +} + } // namespace } // namespace mongo diff --git a/src/mongo/db/storage/mobile/mobile_kv_engine.h b/src/mongo/db/storage/mobile/mobile_kv_engine.h index 8618cd4510a..011cdd28d9c 100644 --- a/src/mongo/db/storage/mobile/mobile_kv_engine.h +++ b/src/mongo/db/storage/mobile/mobile_kv_engine.h @@ -109,6 +109,13 @@ public: return Status::OK(); } + virtual Status recoverOrphanedIdent(OperationContext* opCtx, + StringData ns, + StringData ident, + const CollectionOptions& options) override { + return createRecordStore(opCtx, ns, ident, options); + } + void cleanShutdown() override{}; bool hasIdent(OperationContext* opCtx, StringData ident) const override; diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp index 4e06354af8b..9fd42546d26 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp @@ -47,6 +47,8 @@ #include <boost/filesystem.hpp> #include <boost/filesystem/operations.hpp> +#include <boost/filesystem/path.hpp> +#include <boost/system/error_code.hpp> #include <valgrind/valgrind.h> #include "mongo/base/error_codes.h" @@ -827,6 +829,68 @@ Status WiredTigerKVEngine::createGroupedRecordStore(OperationContext* opCtx, return wtRCToStatus(s->create(s, uri.c_str(), config.c_str())); } +Status WiredTigerKVEngine::recoverOrphanedIdent(OperationContext* opCtx, + StringData ns, + StringData ident, + const CollectionOptions& options) { + invariant(_inRepairMode); + + // Moves the data file to a temporary name so that a new RecordStore can be created with the + // same ident name. We will delete the new empty collection and rename the data file back so it + // can be salvaged. + + boost::optional<boost::filesystem::path> identFilePath = getDataFilePathForIdent(ident); + if (!identFilePath) { + return {ErrorCodes::UnknownError, "Data file for ident " + ident + " not found"}; + } + + boost::system::error_code ec; + invariant(boost::filesystem::exists(*identFilePath, ec)); + + boost::filesystem::path tmpFile{*identFilePath}; + tmpFile += ".tmp"; + if (boost::filesystem::exists(tmpFile, ec)) { + return {ErrorCodes::FileRenameFailed, + "Attempted to rename data file to an existing temporary file: " + tmpFile.string()}; + } + + log() << "Renaming data file " + identFilePath->string() + " to temporary file " + + tmpFile.string(); + + boost::filesystem::rename(*identFilePath, tmpFile, ec); + if (ec) { + return {ErrorCodes::FileRenameFailed, + "Error renaming data file to temporary file: " + ec.message()}; + } + + log() << "Creating new RecordStore for collection " + ns + " with UUID: " + + (options.uuid ? options.uuid->toString() : "none"); + + auto status = createGroupedRecordStore(opCtx, ns, ident, options, KVPrefix::kNotPrefixed); + if (!status.isOK()) { + return status; + } + + log() << "Moving orphaned data file back as " + identFilePath->string(); + + boost::filesystem::remove(*identFilePath, ec); + if (ec) { + return {ErrorCodes::UnknownError, "Error deleting empty data file: " + ec.message()}; + } + + boost::filesystem::rename(tmpFile, *identFilePath, ec); + if (ec) { + return {ErrorCodes::FileRenameFailed, + "Error renaming data file back from temporary file: " + ec.message()}; + } + + log() << "Salvaging ident " + ident; + + WiredTigerSession sessionWrapper(_conn); + WT_SESSION* session = sessionWrapper.getSession(); + return wtRCToStatus(session->salvage(session, _uri(ident).c_str(), NULL), "Salvage failed: "); +} + std::unique_ptr<RecordStore> WiredTigerKVEngine::getGroupedRecordStore( OperationContext* opCtx, StringData ns, @@ -1104,6 +1168,18 @@ std::vector<std::string> WiredTigerKVEngine::getAllIdents(OperationContext* opCt return all; } +boost::optional<boost::filesystem::path> WiredTigerKVEngine::getDataFilePathForIdent( + StringData ident) const { + boost::filesystem::path identPath = _path; + identPath /= ident.toString() + ".wt"; + + boost::system::error_code ec; + if (!boost::filesystem::exists(identPath, ec)) { + return boost::none; + } + return identPath; +} + int WiredTigerKVEngine::reconfigure(const char* str) { return _conn->reconfigure(_conn, str); } diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h index a0eb2575f31..08a1e609787 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h @@ -35,6 +35,7 @@ #include <memory> #include <string> +#include <boost/filesystem/path.hpp> #include <wiredtiger.h> #include "mongo/bson/ordering.h" @@ -81,43 +82,43 @@ public: void setRecordStoreExtraOptions(const std::string& options); void setSortedDataInterfaceExtraOptions(const std::string& options); - virtual bool supportsDocLocking() const; + virtual bool supportsDocLocking() const override; - virtual bool supportsDirectoryPerDB() const; + virtual bool supportsDirectoryPerDB() const override; - virtual bool isDurable() const { + virtual bool isDurable() const override { return _durable; } - virtual bool isEphemeral() const { + virtual bool isEphemeral() const override { return _ephemeral; } - virtual RecoveryUnit* newRecoveryUnit(); + virtual RecoveryUnit* newRecoveryUnit() override; virtual Status createRecordStore(OperationContext* opCtx, StringData ns, StringData ident, - const CollectionOptions& options) { + const CollectionOptions& options) override { return createGroupedRecordStore(opCtx, ns, ident, options, KVPrefix::kNotPrefixed); } virtual std::unique_ptr<RecordStore> getRecordStore(OperationContext* opCtx, StringData ns, StringData ident, - const CollectionOptions& options) { + const CollectionOptions& options) override { return getGroupedRecordStore(opCtx, ns, ident, options, KVPrefix::kNotPrefixed); } virtual Status createSortedDataInterface(OperationContext* opCtx, StringData ident, - const IndexDescriptor* desc) { + const IndexDescriptor* desc) override { return createGroupedSortedDataInterface(opCtx, ident, desc, KVPrefix::kNotPrefixed); } virtual SortedDataInterface* getSortedDataInterface(OperationContext* opCtx, StringData ident, - const IndexDescriptor* desc) { + const IndexDescriptor* desc) override { return getGroupedSortedDataInterface(opCtx, ident, desc, KVPrefix::kNotPrefixed); } @@ -125,51 +126,56 @@ public: StringData ns, StringData ident, const CollectionOptions& options, - KVPrefix prefix); + KVPrefix prefix) override; virtual std::unique_ptr<RecordStore> getGroupedRecordStore(OperationContext* opCtx, StringData ns, StringData ident, const CollectionOptions& options, - KVPrefix prefix); + KVPrefix prefix) override; virtual Status createGroupedSortedDataInterface(OperationContext* opCtx, StringData ident, const IndexDescriptor* desc, - KVPrefix prefix); + KVPrefix prefix) override; virtual SortedDataInterface* getGroupedSortedDataInterface(OperationContext* opCtx, StringData ident, const IndexDescriptor* desc, - KVPrefix prefix); + KVPrefix prefix) override; - virtual Status dropIdent(OperationContext* opCtx, StringData ident); + virtual Status dropIdent(OperationContext* opCtx, StringData ident) override; virtual void alterIdentMetadata(OperationContext* opCtx, StringData ident, - const IndexDescriptor* desc); + const IndexDescriptor* desc) override; virtual Status okToRename(OperationContext* opCtx, StringData fromNS, StringData toNS, StringData ident, - const RecordStore* originalRecordStore) const; + const RecordStore* originalRecordStore) const override; - virtual int flushAllFiles(OperationContext* opCtx, bool sync); + virtual int flushAllFiles(OperationContext* opCtx, bool sync) override; - virtual Status beginBackup(OperationContext* opCtx); + virtual Status beginBackup(OperationContext* opCtx) override; - virtual void endBackup(OperationContext* opCtx); + virtual void endBackup(OperationContext* opCtx) override; - virtual int64_t getIdentSize(OperationContext* opCtx, StringData ident); + virtual int64_t getIdentSize(OperationContext* opCtx, StringData ident) override; - virtual Status repairIdent(OperationContext* opCtx, StringData ident); + virtual Status repairIdent(OperationContext* opCtx, StringData ident) override; - virtual bool hasIdent(OperationContext* opCtx, StringData ident) const; + virtual Status recoverOrphanedIdent(OperationContext* opCtx, + StringData ns, + StringData ident, + const CollectionOptions& options) override; - std::vector<std::string> getAllIdents(OperationContext* opCtx) const; + virtual bool hasIdent(OperationContext* opCtx, StringData ident) const override; - virtual void cleanShutdown(); + std::vector<std::string> getAllIdents(OperationContext* opCtx) const override; + + virtual void cleanShutdown() override; SnapshotManager* getSnapshotManager() const final { return &_sessionCache->snapshotManager(); @@ -277,6 +283,13 @@ public: Timestamp getStableTimestamp() const; Timestamp getOldestTimestamp() const; + /** + * Returns the data file path associated with an ident on disk. Returns boost::none if the data + * file can not be found. This will attempt to locate a file even if the storage engine's own + * metadata is not aware of the ident. This is intented for database repair purposes only. + */ + boost::optional<boost::filesystem::path> getDataFilePathForIdent(StringData ident) const; + private: class WiredTigerJournalFlusher; class WiredTigerCheckpointThread; diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine_test.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine_test.cpp index 2832d72bbf2..ab7cad20971 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine_test.cpp +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine_test.cpp @@ -31,7 +31,11 @@ #include "mongo/db/storage/kv/kv_engine_test_harness.h" +#include <boost/filesystem.hpp> +#include <boost/filesystem/path.hpp> + #include "mongo/base/init.h" +#include "mongo/db/operation_context_noop.h" #include "mongo/db/repl/repl_settings.h" #include "mongo/db/repl/replication_coordinator_mock.h" #include "mongo/db/service_context.h" @@ -39,6 +43,7 @@ #include "mongo/db/storage/wiredtiger/wiredtiger_record_store.h" #include "mongo/stdx/memory.h" #include "mongo/unittest/temp_dir.h" +#include "mongo/unittest/unittest.h" #include "mongo/util/clock_source_mock.h" namespace mongo { @@ -46,9 +51,9 @@ namespace { class WiredTigerKVHarnessHelper : public KVHarnessHelper { public: - WiredTigerKVHarnessHelper() : _dbpath("wt-kv-harness") { - _engine.reset(new WiredTigerKVEngine( - kWiredTigerEngineName, _dbpath.path(), _cs.get(), "", 1, false, false, false, false)); + WiredTigerKVHarnessHelper(bool forRepair = false) + : _dbpath("wt-kv-harness"), _forRepair(forRepair) { + _engine.reset(makeEngine()); repl::ReplicationCoordinator::set( getGlobalServiceContext(), std::unique_ptr<repl::ReplicationCoordinator>(new repl::ReplicationCoordinatorMock( @@ -56,26 +61,176 @@ public: } virtual ~WiredTigerKVHarnessHelper() { - _engine.reset(NULL); + _engine.reset(nullptr); + } + + virtual KVEngine* restartEngine() override { + _engine.reset(nullptr); + _engine.reset(makeEngine()); + return _engine.get(); } - virtual KVEngine* restartEngine() { - _engine.reset(NULL); - _engine.reset(new WiredTigerKVEngine( - kWiredTigerEngineName, _dbpath.path(), _cs.get(), "", 1, false, false, false, false)); + virtual KVEngine* getEngine() override { return _engine.get(); } - virtual KVEngine* getEngine() { + virtual WiredTigerKVEngine* getWiredTigerKVEngine() { return _engine.get(); } private: + WiredTigerKVEngine* makeEngine() { + return new WiredTigerKVEngine(kWiredTigerEngineName, + _dbpath.path(), + _cs.get(), + "", + 1, + false, + false, + _forRepair, + false); + } + const std::unique_ptr<ClockSource> _cs = stdx::make_unique<ClockSourceMock>(); unittest::TempDir _dbpath; std::unique_ptr<WiredTigerKVEngine> _engine; + bool _forRepair; +}; + +class WiredTigerKVEngineTest : public unittest::Test { +public: + void setUp() override { + setGlobalServiceContext(ServiceContext::make()); + Client::initThread(getThreadName()); + + _helper = makeHelper(); + _engine = _helper->getWiredTigerKVEngine(); + } + + void tearDown() override { + _helper.reset(nullptr); + Client::destroy(); + setGlobalServiceContext({}); + } + + std::unique_ptr<OperationContext> makeOperationContext() { + return std::make_unique<OperationContextNoop>(_engine->newRecoveryUnit()); + } + +protected: + virtual std::unique_ptr<WiredTigerKVHarnessHelper> makeHelper() { + return std::make_unique<WiredTigerKVHarnessHelper>(); + } + + std::unique_ptr<WiredTigerKVHarnessHelper> _helper; + WiredTigerKVEngine* _engine; }; +class WiredTigerKVEngineRepairTest : public WiredTigerKVEngineTest { + virtual std::unique_ptr<WiredTigerKVHarnessHelper> makeHelper() override { + return std::make_unique<WiredTigerKVHarnessHelper>(true /* repair */); + } +}; + +TEST_F(WiredTigerKVEngineRepairTest, OrphanedDataFilesCanBeRecovered) { + auto opCtxPtr = makeOperationContext(); + + std::string ns = "a.b"; + std::string ident = "collection-1234"; + std::string record = "abcd"; + CollectionOptions options; + + std::unique_ptr<RecordStore> rs; + ASSERT_OK(_engine->createRecordStore(opCtxPtr.get(), ns, ident, options)); + rs = _engine->getRecordStore(opCtxPtr.get(), ns, ident, options); + ASSERT(rs); + + RecordId loc; + { + WriteUnitOfWork uow(opCtxPtr.get()); + StatusWith<RecordId> res = rs->insertRecord( + opCtxPtr.get(), record.c_str(), record.length() + 1, Timestamp(), false); + ASSERT_OK(res.getStatus()); + loc = res.getValue(); + uow.commit(); + } + + const boost::optional<boost::filesystem::path> dataFilePath = + _engine->getDataFilePathForIdent(ident); + ASSERT(dataFilePath); + + ASSERT(boost::filesystem::exists(*dataFilePath)); + + const boost::filesystem::path tmpFile{dataFilePath->string() + ".tmp"}; + ASSERT(!boost::filesystem::exists(tmpFile)); + + // Move the data file out of the way so the ident can be dropped. + boost::system::error_code err; + boost::filesystem::rename(*dataFilePath, tmpFile, err); + ASSERT(!err) << err.message(); + + ASSERT_OK(_engine->dropIdent(opCtxPtr.get(), ident)); + + // The data file is moved back in place so that it becomes an "orphan" of the storage + // engine and the restoration process can be tested. + boost::filesystem::rename(tmpFile, *dataFilePath, err); + ASSERT(!err) << err.message(); + + ASSERT_OK(_engine->recoverOrphanedIdent(opCtxPtr.get(), ns, ident, options)); + + // The existing RecordStore is still usable with a different OperationContext and + // RecoveryUnit because a new session with new cursors will be opened. + ASSERT_EQUALS(record, rs->dataFor(opCtxPtr.get(), loc).data()); +} + +TEST_F(WiredTigerKVEngineRepairTest, UnrecoverableOrphanedDataFilesFailGracefully) { + auto opCtxPtr = makeOperationContext(); + + std::string ns = "a.b"; + std::string ident = "collection-1234"; + std::string record = "abcd"; + CollectionOptions options; + + std::unique_ptr<RecordStore> rs; + ASSERT_OK(_engine->createRecordStore(opCtxPtr.get(), ns, ident, options)); + rs = _engine->getRecordStore(opCtxPtr.get(), ns, ident, options); + ASSERT(rs); + + RecordId loc; + { + WriteUnitOfWork uow(opCtxPtr.get()); + StatusWith<RecordId> res = rs->insertRecord( + opCtxPtr.get(), record.c_str(), record.length() + 1, Timestamp(), false); + ASSERT_OK(res.getStatus()); + loc = res.getValue(); + uow.commit(); + } + + const boost::optional<boost::filesystem::path> dataFilePath = + _engine->getDataFilePathForIdent(ident); + ASSERT(dataFilePath); + + ASSERT(boost::filesystem::exists(*dataFilePath)); + + ASSERT_OK(_engine->dropIdent(opCtxPtr.get(), ident)); + + // The ident may not get immediately dropped, so ensure it is completely gone. + boost::system::error_code err; + boost::filesystem::remove(*dataFilePath, err); + ASSERT(!err) << err.message(); + + // Create an empty data file. The subsequent call to recreate the collection will fail because + // it is unsalvageable. + boost::filesystem::ofstream fileStream(*dataFilePath); + fileStream << ""; + fileStream.close(); + + ASSERT(boost::filesystem::exists(*dataFilePath)); + + // This should fail gracefully and not cause any crashing. + ASSERT_NOT_OK(_engine->recoverOrphanedIdent(opCtxPtr.get(), ns, ident, options)); +} + std::unique_ptr<KVHarnessHelper> makeHelper() { return stdx::make_unique<WiredTigerKVHarnessHelper>(); } |