summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--src/mongo/db/service_context_d_test_fixture.cpp8
-rw-r--r--src/mongo/db/service_context_d_test_fixture.h4
-rw-r--r--src/mongo/db/storage/devnull/devnull_kv_engine.h7
-rw-r--r--src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_engine.h7
-rw-r--r--src/mongo/db/storage/kv/kv_engine.h18
-rw-r--r--src/mongo/db/storage/kv/kv_storage_engine.cpp81
-rw-r--r--src/mongo/db/storage/kv/kv_storage_engine.h15
-rw-r--r--src/mongo/db/storage/kv/kv_storage_engine_test.cpp86
-rw-r--r--src/mongo/db/storage/mobile/mobile_kv_engine.h7
-rw-r--r--src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp76
-rw-r--r--src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h61
-rw-r--r--src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine_test.cpp173
12 files changed, 483 insertions, 60 deletions
diff --git a/src/mongo/db/service_context_d_test_fixture.cpp b/src/mongo/db/service_context_d_test_fixture.cpp
index 8f444a5a17c..5eec5cb6a24 100644
--- a/src/mongo/db/service_context_d_test_fixture.cpp
+++ b/src/mongo/db/service_context_d_test_fixture.cpp
@@ -51,11 +51,16 @@ namespace mongo {
ServiceContextMongoDTest::ServiceContextMongoDTest()
: ServiceContextMongoDTest("ephemeralForTest") {}
-ServiceContextMongoDTest::ServiceContextMongoDTest(std::string engine) {
+ServiceContextMongoDTest::ServiceContextMongoDTest(std::string engine)
+ : ServiceContextMongoDTest(engine, RepairAction::kNoRepair) {}
+
+ServiceContextMongoDTest::ServiceContextMongoDTest(std::string engine, RepairAction repair) {
_stashedStorageParams.engine = std::exchange(storageGlobalParams.engine, std::move(engine));
_stashedStorageParams.engineSetByUser =
std::exchange(storageGlobalParams.engineSetByUser, true);
+ _stashedStorageParams.repair =
+ std::exchange(storageGlobalParams.repair, (repair == RepairAction::kRepair));
auto const serviceContext = getServiceContext();
serviceContext->setServiceEntryPoint(std::make_unique<ServiceEntryPointMongod>(serviceContext));
@@ -86,6 +91,7 @@ ServiceContextMongoDTest::~ServiceContextMongoDTest() {
shutdownGlobalStorageEngineCleanly(getGlobalServiceContext());
std::swap(storageGlobalParams.engine, _stashedStorageParams.engine);
std::swap(storageGlobalParams.engineSetByUser, _stashedStorageParams.engineSetByUser);
+ std::swap(storageGlobalParams.repair, _stashedStorageParams.repair);
}
} // namespace mongo
diff --git a/src/mongo/db/service_context_d_test_fixture.h b/src/mongo/db/service_context_d_test_fixture.h
index 19619f0d0fe..44e6a8df507 100644
--- a/src/mongo/db/service_context_d_test_fixture.h
+++ b/src/mongo/db/service_context_d_test_fixture.h
@@ -39,18 +39,22 @@ namespace mongo {
*/
class ServiceContextMongoDTest : public ServiceContextTest {
protected:
+ enum class RepairAction { kNoRepair, kRepair };
+
ServiceContextMongoDTest();
/**
* Build a ServiceContextMongoDTest, using the named storage engine.
*/
explicit ServiceContextMongoDTest(std::string engine);
+ explicit ServiceContextMongoDTest(std::string engine, RepairAction repair);
virtual ~ServiceContextMongoDTest();
private:
struct {
std::string engine;
bool engineSetByUser;
+ bool repair;
} _stashedStorageParams;
};
diff --git a/src/mongo/db/storage/devnull/devnull_kv_engine.h b/src/mongo/db/storage/devnull/devnull_kv_engine.h
index df356cfaae1..82af7c0d37e 100644
--- a/src/mongo/db/storage/devnull/devnull_kv_engine.h
+++ b/src/mongo/db/storage/devnull/devnull_kv_engine.h
@@ -107,6 +107,13 @@ public:
return Status::OK();
}
+ virtual Status recoverOrphanedIdent(OperationContext* opCtx,
+ StringData ns,
+ StringData ident,
+ const CollectionOptions& options) {
+ return Status::OK();
+ }
+
virtual bool hasIdent(OperationContext* opCtx, StringData ident) const {
return true;
}
diff --git a/src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_engine.h b/src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_engine.h
index 38c9ea37cef..107c494d02d 100644
--- a/src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_engine.h
+++ b/src/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_engine.h
@@ -96,6 +96,13 @@ public:
return Status::OK();
}
+ virtual Status recoverOrphanedIdent(OperationContext* opCtx,
+ StringData ns,
+ StringData ident,
+ const CollectionOptions& options) override {
+ return createRecordStore(opCtx, ns, ident, options);
+ }
+
virtual void cleanShutdown(){};
virtual bool hasIdent(OperationContext* opCtx, StringData ident) const {
diff --git a/src/mongo/db/storage/kv/kv_engine.h b/src/mongo/db/storage/kv/kv_engine.h
index d462f8158a5..e4dfab66664 100644
--- a/src/mongo/db/storage/kv/kv_engine.h
+++ b/src/mongo/db/storage/kv/kv_engine.h
@@ -163,6 +163,24 @@ public:
virtual Status dropIdent(OperationContext* opCtx, StringData ident) = 0;
+ /**
+ * Attempts to locate and recover a file that is "orphaned" from the storage engine's metadata,
+ * but may still exist on disk if this is a durable storage engine. Returns Status::OK if a new
+ * record store was successfully created.
+ *
+ * This may return an error if a storage engine chooses not to implement recovery, is unable to
+ * implement recovery because it is not durable, or if the storage engine attempted to recover
+ * the file and failed. A storage engine that is not durable may choose to return an OK status
+ * after creating a new, empty record store.
+ *
+ * This recovery process makes no guarantees about the integrity of data recovered or even that
+ * it still exists when recovered.
+ */
+ virtual Status recoverOrphanedIdent(OperationContext* opCtx,
+ StringData ns,
+ StringData ident,
+ const CollectionOptions& options) = 0;
+
virtual void alterIdentMetadata(OperationContext* opCtx,
StringData ident,
const IndexDescriptor* desc){};
diff --git a/src/mongo/db/storage/kv/kv_storage_engine.cpp b/src/mongo/db/storage/kv/kv_storage_engine.cpp
index 16eed29b2f9..157bb27e788 100644
--- a/src/mongo/db/storage/kv/kv_storage_engine.cpp
+++ b/src/mongo/db/storage/kv/kv_storage_engine.cpp
@@ -127,10 +127,13 @@ void KVStorageEngine::loadCatalog(OperationContext* opCtx) {
_catalogRecordStore.get(), _options.directoryPerDB, _options.directoryForIndexes));
_catalog->init(opCtx);
- // We populate 'identsKnownToStorageEngine' only if we are loading after an unclean shutdown.
+ // We populate 'identsKnownToStorageEngine' only if we are loading after an unclean shutdown or
+ // doing repair.
+ const bool loadingFromUncleanShutdownOrRepair =
+ startingAfterUncleanShutdown(getGlobalServiceContext()) || _options.forRepair;
+
std::vector<std::string> identsKnownToStorageEngine;
- const bool loadingFromUncleanShutdown = startingAfterUncleanShutdown(getGlobalServiceContext());
- if (loadingFromUncleanShutdown) {
+ if (loadingFromUncleanShutdownOrRepair) {
identsKnownToStorageEngine = _engine->getAllIdents(opCtx);
std::sort(identsKnownToStorageEngine.begin(), identsKnownToStorageEngine.end());
}
@@ -143,21 +146,18 @@ void KVStorageEngine::loadCatalog(OperationContext* opCtx) {
NamespaceString nss(coll);
std::string dbName = nss.db().toString();
- if (loadingFromUncleanShutdown) {
- // If we are loading the catalog after an unclean shutdown, it's possible that there are
- // collections in the catalog that are unknown to the storage engine. If we can't find
- // it in the list of storage engine idents, remove the collection and move on to the
- // next one.
+ if (loadingFromUncleanShutdownOrRepair) {
+ // If we are loading the catalog after an unclean shutdown or during repair, it's
+ // possible that there are collections in the catalog that are unknown to the storage
+ // engine. If we can't find a table in the list of storage engine idents, either
+ // attempt to recover the ident or drop it.
const auto collectionIdent = _catalog->getCollectionIdent(coll);
- if (!std::binary_search(identsKnownToStorageEngine.begin(),
- identsKnownToStorageEngine.end(),
- collectionIdent)) {
- log() << "Dropping collection " << coll
- << " unknown to storage engine after unclean shutdown";
-
- WriteUnitOfWork wuow(opCtx);
- fassert(50716, _catalog->dropCollection(opCtx, coll));
- wuow.commit();
+ bool orphan = !std::binary_search(identsKnownToStorageEngine.begin(),
+ identsKnownToStorageEngine.end(),
+ collectionIdent);
+ // If the storage engine is missing a collection and is unable to create a new record
+ // store, continue past the following logic.
+ if (orphan && !_recoverOrDropOrphanedCollection(opCtx, coll, collectionIdent)) {
continue;
}
}
@@ -198,6 +198,37 @@ void KVStorageEngine::closeCatalog(OperationContext* opCtx) {
_catalogRecordStore.reset(nullptr);
}
+bool KVStorageEngine::_recoverOrDropOrphanedCollection(OperationContext* opCtx,
+ StringData collectionName,
+ StringData collectionIdent) {
+
+ if (_options.forRepair) {
+ log() << "Storage engine is missing collection '" << collectionName
+ << "' from its metadata. Attempting to locate and recover the data for "
+ << collectionIdent;
+
+ WriteUnitOfWork wuow(opCtx);
+ const auto metadata = _catalog->getMetaData(opCtx, collectionName);
+ auto status =
+ _engine->recoverOrphanedIdent(opCtx, collectionName, collectionIdent, metadata.options);
+ if (status.isOK()) {
+ wuow.commit();
+ return true;
+ }
+
+ warning() << "Failed to recover orphaned data file for collection '" << collectionName
+ << "': " << status;
+ }
+
+ log() << "Dropping collection " << collectionName
+ << " from the catalog unknown to the storage engine";
+
+ WriteUnitOfWork wuow(opCtx);
+ fassert(50716, _catalog->dropCollection(opCtx, collectionName));
+ wuow.commit();
+ return false;
+}
+
/**
* This method reconciles differences between idents the KVEngine is aware of and the
* KVCatalog. There are three differences to consider:
@@ -263,13 +294,15 @@ KVStorageEngine::reconcileCatalogAndIdents(OperationContext* opCtx) {
// other contexts such as `recoverToStableTimestamp`.
std::vector<std::string> collections;
_catalog->getAllCollections(&collections);
- for (const auto& coll : collections) {
- const auto& identForColl = _catalog->getCollectionIdent(coll);
- if (engineIdents.find(identForColl) == engineIdents.end()) {
- return {ErrorCodes::UnrecoverableRollbackError,
- str::stream() << "Expected collection does not exist. Collection: " << coll
- << " Ident: "
- << identForColl};
+ if (!_options.forRepair) {
+ for (const auto& coll : collections) {
+ const auto& identForColl = _catalog->getCollectionIdent(coll);
+ if (engineIdents.find(identForColl) == engineIdents.end()) {
+ return {ErrorCodes::UnrecoverableRollbackError,
+ str::stream() << "Expected collection does not exist. Collection: " << coll
+ << " Ident: "
+ << identForColl};
+ }
}
}
diff --git a/src/mongo/db/storage/kv/kv_storage_engine.h b/src/mongo/db/storage/kv/kv_storage_engine.h
index d839f149af6..f57b0e0d7d9 100644
--- a/src/mongo/db/storage/kv/kv_storage_engine.h
+++ b/src/mongo/db/storage/kv/kv_storage_engine.h
@@ -191,6 +191,21 @@ private:
CollIter begin,
CollIter end);
+ /**
+ * When called in a repair context (_options.forRepair=true), attempts to recover a collection
+ * whose entry is present in the KVCatalog, but missing from the KVEngine. Returns 'false' and
+ * removes collection metadata from the KVCatalog if called outside of a repair context, the
+ * collection is unrecoverable, or the implementation of KVEngine::recoverOrphanedIdent returns
+ * an error.
+ *
+ * Returns 'true' if the collection was recovered in the KVEngine and a new record store was
+ * created. Recovery does not make any guarantees about the integrity of the data in the
+ * collection.
+ */
+ bool _recoverOrDropOrphanedCollection(OperationContext* opCtx,
+ StringData collectionName,
+ StringData collectionIdent);
+
void _dumpCatalog(OperationContext* opCtx);
class RemoveDBChange;
diff --git a/src/mongo/db/storage/kv/kv_storage_engine_test.cpp b/src/mongo/db/storage/kv/kv_storage_engine_test.cpp
index e9d6c0accab..3a6c10a74aa 100644
--- a/src/mongo/db/storage/kv/kv_storage_engine_test.cpp
+++ b/src/mongo/db/storage/kv/kv_storage_engine_test.cpp
@@ -45,6 +45,7 @@
#include "mongo/db/storage/kv/kv_database_catalog_entry_mock.h"
#include "mongo/db/storage/kv/kv_engine.h"
#include "mongo/db/storage/kv/kv_storage_engine.h"
+#include "mongo/db/unclean_shutdown.h"
#include "mongo/stdx/memory.h"
#include "mongo/unittest/unittest.h"
@@ -53,10 +54,12 @@ namespace {
class KVStorageEngineTest : public ServiceContextMongoDTest {
public:
- KVStorageEngineTest()
- : ServiceContextMongoDTest("ephemeralForTest"),
+ KVStorageEngineTest(RepairAction repair)
+ : ServiceContextMongoDTest("ephemeralForTest", repair),
_storageEngine(checked_cast<KVStorageEngine*>(getServiceContext()->getStorageEngine())) {}
+ KVStorageEngineTest() : KVStorageEngineTest(RepairAction::kNoRepair) {}
+
/**
* Create a collection in the catalog and in the KVEngine. Return the storage engine's `ident`.
*/
@@ -100,6 +103,17 @@ public:
return _storageEngine->getEngine()->getAllIdents(opCtx);
}
+ bool collectionExists(OperationContext* opCtx, const NamespaceString& nss) {
+ std::vector<std::string> allCollections;
+ _storageEngine->getCatalog()->getAllCollections(&allCollections);
+ return std::find(allCollections.begin(), allCollections.end(), nss.toString()) !=
+ allCollections.end();
+ }
+ bool identExists(OperationContext* opCtx, std::string ident) {
+ auto idents = getAllKVEngineIdents(opCtx);
+ return std::find(idents.begin(), idents.end(), ident) != idents.end();
+ }
+
/**
* Create an index with a key of `{<key>: 1}` and a `name` of <key>.
*/
@@ -130,6 +144,11 @@ public:
KVStorageEngine* _storageEngine;
};
+class KVStorageEngineRepairTest : public KVStorageEngineTest {
+public:
+ KVStorageEngineRepairTest() : KVStorageEngineTest(RepairAction::kRepair) {}
+};
+
TEST_F(KVStorageEngineTest, ReconcileIdentsTest) {
auto opCtx = cc().makeOperationContext();
@@ -222,5 +241,68 @@ TEST_F(KVStorageEngineTest, RecreateIndexes) {
}));
}
+TEST_F(KVStorageEngineTest, LoadCatalogDropsOrphansAfterUncleanShutdown) {
+ auto opCtx = cc().makeOperationContext();
+
+ const NamespaceString collNs("db.coll1");
+ auto swIdentName = createCollection(opCtx.get(), collNs);
+ ASSERT_OK(swIdentName);
+
+ ASSERT_OK(dropIdent(opCtx.get(), swIdentName.getValue()));
+ ASSERT(collectionExists(opCtx.get(), collNs));
+
+ // After the catalog is reloaded, we expect that the collection has been dropped because the
+ // KVEngine was started after an unclean shutdown but not in a repair context.
+ {
+ Lock::GlobalWrite writeLock(opCtx.get(), Date_t::max(), Lock::InterruptBehavior::kThrow);
+ _storageEngine->closeCatalog(opCtx.get());
+ startingAfterUncleanShutdown(getGlobalServiceContext()) = true;
+ _storageEngine->loadCatalog(opCtx.get());
+ }
+
+ ASSERT(!identExists(opCtx.get(), swIdentName.getValue()));
+ ASSERT(!collectionExists(opCtx.get(), collNs));
+}
+
+TEST_F(KVStorageEngineRepairTest, LoadCatalogRecoversOrphans) {
+ auto opCtx = cc().makeOperationContext();
+
+ const NamespaceString collNs("db.coll1");
+ auto swIdentName = createCollection(opCtx.get(), collNs);
+ ASSERT_OK(swIdentName);
+
+ ASSERT_OK(dropIdent(opCtx.get(), swIdentName.getValue()));
+ ASSERT(collectionExists(opCtx.get(), collNs));
+
+ // After the catalog is reloaded, we expect that the ident has been recovered because the
+ // KVEngine was started in a repair context.
+ {
+ Lock::GlobalWrite writeLock(opCtx.get(), Date_t::max(), Lock::InterruptBehavior::kThrow);
+ _storageEngine->closeCatalog(opCtx.get());
+ _storageEngine->loadCatalog(opCtx.get());
+ }
+
+ ASSERT(identExists(opCtx.get(), swIdentName.getValue()));
+ ASSERT(collectionExists(opCtx.get(), collNs));
+}
+
+TEST_F(KVStorageEngineRepairTest, ReconcileSucceeds) {
+ auto opCtx = cc().makeOperationContext();
+
+ const NamespaceString collNs("db.coll1");
+ auto swIdentName = createCollection(opCtx.get(), collNs);
+ ASSERT_OK(swIdentName);
+
+ ASSERT_OK(dropIdent(opCtx.get(), swIdentName.getValue()));
+ ASSERT(collectionExists(opCtx.get(), collNs));
+
+ // Reconcile would normally return an error if a collection existed with a missing ident in the
+ // storage engine. When in a repair context, that should not be the case.
+ ASSERT_OK(reconcile(opCtx.get()).getStatus());
+
+ ASSERT(!identExists(opCtx.get(), swIdentName.getValue()));
+ ASSERT(collectionExists(opCtx.get(), collNs));
+}
+
} // namespace
} // namespace mongo
diff --git a/src/mongo/db/storage/mobile/mobile_kv_engine.h b/src/mongo/db/storage/mobile/mobile_kv_engine.h
index 8618cd4510a..011cdd28d9c 100644
--- a/src/mongo/db/storage/mobile/mobile_kv_engine.h
+++ b/src/mongo/db/storage/mobile/mobile_kv_engine.h
@@ -109,6 +109,13 @@ public:
return Status::OK();
}
+ virtual Status recoverOrphanedIdent(OperationContext* opCtx,
+ StringData ns,
+ StringData ident,
+ const CollectionOptions& options) override {
+ return createRecordStore(opCtx, ns, ident, options);
+ }
+
void cleanShutdown() override{};
bool hasIdent(OperationContext* opCtx, StringData ident) const override;
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp
index 4e06354af8b..9fd42546d26 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp
@@ -47,6 +47,8 @@
#include <boost/filesystem.hpp>
#include <boost/filesystem/operations.hpp>
+#include <boost/filesystem/path.hpp>
+#include <boost/system/error_code.hpp>
#include <valgrind/valgrind.h>
#include "mongo/base/error_codes.h"
@@ -827,6 +829,68 @@ Status WiredTigerKVEngine::createGroupedRecordStore(OperationContext* opCtx,
return wtRCToStatus(s->create(s, uri.c_str(), config.c_str()));
}
+Status WiredTigerKVEngine::recoverOrphanedIdent(OperationContext* opCtx,
+ StringData ns,
+ StringData ident,
+ const CollectionOptions& options) {
+ invariant(_inRepairMode);
+
+ // Moves the data file to a temporary name so that a new RecordStore can be created with the
+ // same ident name. We will delete the new empty collection and rename the data file back so it
+ // can be salvaged.
+
+ boost::optional<boost::filesystem::path> identFilePath = getDataFilePathForIdent(ident);
+ if (!identFilePath) {
+ return {ErrorCodes::UnknownError, "Data file for ident " + ident + " not found"};
+ }
+
+ boost::system::error_code ec;
+ invariant(boost::filesystem::exists(*identFilePath, ec));
+
+ boost::filesystem::path tmpFile{*identFilePath};
+ tmpFile += ".tmp";
+ if (boost::filesystem::exists(tmpFile, ec)) {
+ return {ErrorCodes::FileRenameFailed,
+ "Attempted to rename data file to an existing temporary file: " + tmpFile.string()};
+ }
+
+ log() << "Renaming data file " + identFilePath->string() + " to temporary file " +
+ tmpFile.string();
+
+ boost::filesystem::rename(*identFilePath, tmpFile, ec);
+ if (ec) {
+ return {ErrorCodes::FileRenameFailed,
+ "Error renaming data file to temporary file: " + ec.message()};
+ }
+
+ log() << "Creating new RecordStore for collection " + ns + " with UUID: " +
+ (options.uuid ? options.uuid->toString() : "none");
+
+ auto status = createGroupedRecordStore(opCtx, ns, ident, options, KVPrefix::kNotPrefixed);
+ if (!status.isOK()) {
+ return status;
+ }
+
+ log() << "Moving orphaned data file back as " + identFilePath->string();
+
+ boost::filesystem::remove(*identFilePath, ec);
+ if (ec) {
+ return {ErrorCodes::UnknownError, "Error deleting empty data file: " + ec.message()};
+ }
+
+ boost::filesystem::rename(tmpFile, *identFilePath, ec);
+ if (ec) {
+ return {ErrorCodes::FileRenameFailed,
+ "Error renaming data file back from temporary file: " + ec.message()};
+ }
+
+ log() << "Salvaging ident " + ident;
+
+ WiredTigerSession sessionWrapper(_conn);
+ WT_SESSION* session = sessionWrapper.getSession();
+ return wtRCToStatus(session->salvage(session, _uri(ident).c_str(), NULL), "Salvage failed: ");
+}
+
std::unique_ptr<RecordStore> WiredTigerKVEngine::getGroupedRecordStore(
OperationContext* opCtx,
StringData ns,
@@ -1104,6 +1168,18 @@ std::vector<std::string> WiredTigerKVEngine::getAllIdents(OperationContext* opCt
return all;
}
+boost::optional<boost::filesystem::path> WiredTigerKVEngine::getDataFilePathForIdent(
+ StringData ident) const {
+ boost::filesystem::path identPath = _path;
+ identPath /= ident.toString() + ".wt";
+
+ boost::system::error_code ec;
+ if (!boost::filesystem::exists(identPath, ec)) {
+ return boost::none;
+ }
+ return identPath;
+}
+
int WiredTigerKVEngine::reconfigure(const char* str) {
return _conn->reconfigure(_conn, str);
}
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h
index a0eb2575f31..08a1e609787 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h
@@ -35,6 +35,7 @@
#include <memory>
#include <string>
+#include <boost/filesystem/path.hpp>
#include <wiredtiger.h>
#include "mongo/bson/ordering.h"
@@ -81,43 +82,43 @@ public:
void setRecordStoreExtraOptions(const std::string& options);
void setSortedDataInterfaceExtraOptions(const std::string& options);
- virtual bool supportsDocLocking() const;
+ virtual bool supportsDocLocking() const override;
- virtual bool supportsDirectoryPerDB() const;
+ virtual bool supportsDirectoryPerDB() const override;
- virtual bool isDurable() const {
+ virtual bool isDurable() const override {
return _durable;
}
- virtual bool isEphemeral() const {
+ virtual bool isEphemeral() const override {
return _ephemeral;
}
- virtual RecoveryUnit* newRecoveryUnit();
+ virtual RecoveryUnit* newRecoveryUnit() override;
virtual Status createRecordStore(OperationContext* opCtx,
StringData ns,
StringData ident,
- const CollectionOptions& options) {
+ const CollectionOptions& options) override {
return createGroupedRecordStore(opCtx, ns, ident, options, KVPrefix::kNotPrefixed);
}
virtual std::unique_ptr<RecordStore> getRecordStore(OperationContext* opCtx,
StringData ns,
StringData ident,
- const CollectionOptions& options) {
+ const CollectionOptions& options) override {
return getGroupedRecordStore(opCtx, ns, ident, options, KVPrefix::kNotPrefixed);
}
virtual Status createSortedDataInterface(OperationContext* opCtx,
StringData ident,
- const IndexDescriptor* desc) {
+ const IndexDescriptor* desc) override {
return createGroupedSortedDataInterface(opCtx, ident, desc, KVPrefix::kNotPrefixed);
}
virtual SortedDataInterface* getSortedDataInterface(OperationContext* opCtx,
StringData ident,
- const IndexDescriptor* desc) {
+ const IndexDescriptor* desc) override {
return getGroupedSortedDataInterface(opCtx, ident, desc, KVPrefix::kNotPrefixed);
}
@@ -125,51 +126,56 @@ public:
StringData ns,
StringData ident,
const CollectionOptions& options,
- KVPrefix prefix);
+ KVPrefix prefix) override;
virtual std::unique_ptr<RecordStore> getGroupedRecordStore(OperationContext* opCtx,
StringData ns,
StringData ident,
const CollectionOptions& options,
- KVPrefix prefix);
+ KVPrefix prefix) override;
virtual Status createGroupedSortedDataInterface(OperationContext* opCtx,
StringData ident,
const IndexDescriptor* desc,
- KVPrefix prefix);
+ KVPrefix prefix) override;
virtual SortedDataInterface* getGroupedSortedDataInterface(OperationContext* opCtx,
StringData ident,
const IndexDescriptor* desc,
- KVPrefix prefix);
+ KVPrefix prefix) override;
- virtual Status dropIdent(OperationContext* opCtx, StringData ident);
+ virtual Status dropIdent(OperationContext* opCtx, StringData ident) override;
virtual void alterIdentMetadata(OperationContext* opCtx,
StringData ident,
- const IndexDescriptor* desc);
+ const IndexDescriptor* desc) override;
virtual Status okToRename(OperationContext* opCtx,
StringData fromNS,
StringData toNS,
StringData ident,
- const RecordStore* originalRecordStore) const;
+ const RecordStore* originalRecordStore) const override;
- virtual int flushAllFiles(OperationContext* opCtx, bool sync);
+ virtual int flushAllFiles(OperationContext* opCtx, bool sync) override;
- virtual Status beginBackup(OperationContext* opCtx);
+ virtual Status beginBackup(OperationContext* opCtx) override;
- virtual void endBackup(OperationContext* opCtx);
+ virtual void endBackup(OperationContext* opCtx) override;
- virtual int64_t getIdentSize(OperationContext* opCtx, StringData ident);
+ virtual int64_t getIdentSize(OperationContext* opCtx, StringData ident) override;
- virtual Status repairIdent(OperationContext* opCtx, StringData ident);
+ virtual Status repairIdent(OperationContext* opCtx, StringData ident) override;
- virtual bool hasIdent(OperationContext* opCtx, StringData ident) const;
+ virtual Status recoverOrphanedIdent(OperationContext* opCtx,
+ StringData ns,
+ StringData ident,
+ const CollectionOptions& options) override;
- std::vector<std::string> getAllIdents(OperationContext* opCtx) const;
+ virtual bool hasIdent(OperationContext* opCtx, StringData ident) const override;
- virtual void cleanShutdown();
+ std::vector<std::string> getAllIdents(OperationContext* opCtx) const override;
+
+ virtual void cleanShutdown() override;
SnapshotManager* getSnapshotManager() const final {
return &_sessionCache->snapshotManager();
@@ -277,6 +283,13 @@ public:
Timestamp getStableTimestamp() const;
Timestamp getOldestTimestamp() const;
+ /**
+ * Returns the data file path associated with an ident on disk. Returns boost::none if the data
+ * file can not be found. This will attempt to locate a file even if the storage engine's own
+ * metadata is not aware of the ident. This is intented for database repair purposes only.
+ */
+ boost::optional<boost::filesystem::path> getDataFilePathForIdent(StringData ident) const;
+
private:
class WiredTigerJournalFlusher;
class WiredTigerCheckpointThread;
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine_test.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine_test.cpp
index 2832d72bbf2..ab7cad20971 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine_test.cpp
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine_test.cpp
@@ -31,7 +31,11 @@
#include "mongo/db/storage/kv/kv_engine_test_harness.h"
+#include <boost/filesystem.hpp>
+#include <boost/filesystem/path.hpp>
+
#include "mongo/base/init.h"
+#include "mongo/db/operation_context_noop.h"
#include "mongo/db/repl/repl_settings.h"
#include "mongo/db/repl/replication_coordinator_mock.h"
#include "mongo/db/service_context.h"
@@ -39,6 +43,7 @@
#include "mongo/db/storage/wiredtiger/wiredtiger_record_store.h"
#include "mongo/stdx/memory.h"
#include "mongo/unittest/temp_dir.h"
+#include "mongo/unittest/unittest.h"
#include "mongo/util/clock_source_mock.h"
namespace mongo {
@@ -46,9 +51,9 @@ namespace {
class WiredTigerKVHarnessHelper : public KVHarnessHelper {
public:
- WiredTigerKVHarnessHelper() : _dbpath("wt-kv-harness") {
- _engine.reset(new WiredTigerKVEngine(
- kWiredTigerEngineName, _dbpath.path(), _cs.get(), "", 1, false, false, false, false));
+ WiredTigerKVHarnessHelper(bool forRepair = false)
+ : _dbpath("wt-kv-harness"), _forRepair(forRepair) {
+ _engine.reset(makeEngine());
repl::ReplicationCoordinator::set(
getGlobalServiceContext(),
std::unique_ptr<repl::ReplicationCoordinator>(new repl::ReplicationCoordinatorMock(
@@ -56,26 +61,176 @@ public:
}
virtual ~WiredTigerKVHarnessHelper() {
- _engine.reset(NULL);
+ _engine.reset(nullptr);
+ }
+
+ virtual KVEngine* restartEngine() override {
+ _engine.reset(nullptr);
+ _engine.reset(makeEngine());
+ return _engine.get();
}
- virtual KVEngine* restartEngine() {
- _engine.reset(NULL);
- _engine.reset(new WiredTigerKVEngine(
- kWiredTigerEngineName, _dbpath.path(), _cs.get(), "", 1, false, false, false, false));
+ virtual KVEngine* getEngine() override {
return _engine.get();
}
- virtual KVEngine* getEngine() {
+ virtual WiredTigerKVEngine* getWiredTigerKVEngine() {
return _engine.get();
}
private:
+ WiredTigerKVEngine* makeEngine() {
+ return new WiredTigerKVEngine(kWiredTigerEngineName,
+ _dbpath.path(),
+ _cs.get(),
+ "",
+ 1,
+ false,
+ false,
+ _forRepair,
+ false);
+ }
+
const std::unique_ptr<ClockSource> _cs = stdx::make_unique<ClockSourceMock>();
unittest::TempDir _dbpath;
std::unique_ptr<WiredTigerKVEngine> _engine;
+ bool _forRepair;
+};
+
+class WiredTigerKVEngineTest : public unittest::Test {
+public:
+ void setUp() override {
+ setGlobalServiceContext(ServiceContext::make());
+ Client::initThread(getThreadName());
+
+ _helper = makeHelper();
+ _engine = _helper->getWiredTigerKVEngine();
+ }
+
+ void tearDown() override {
+ _helper.reset(nullptr);
+ Client::destroy();
+ setGlobalServiceContext({});
+ }
+
+ std::unique_ptr<OperationContext> makeOperationContext() {
+ return std::make_unique<OperationContextNoop>(_engine->newRecoveryUnit());
+ }
+
+protected:
+ virtual std::unique_ptr<WiredTigerKVHarnessHelper> makeHelper() {
+ return std::make_unique<WiredTigerKVHarnessHelper>();
+ }
+
+ std::unique_ptr<WiredTigerKVHarnessHelper> _helper;
+ WiredTigerKVEngine* _engine;
};
+class WiredTigerKVEngineRepairTest : public WiredTigerKVEngineTest {
+ virtual std::unique_ptr<WiredTigerKVHarnessHelper> makeHelper() override {
+ return std::make_unique<WiredTigerKVHarnessHelper>(true /* repair */);
+ }
+};
+
+TEST_F(WiredTigerKVEngineRepairTest, OrphanedDataFilesCanBeRecovered) {
+ auto opCtxPtr = makeOperationContext();
+
+ std::string ns = "a.b";
+ std::string ident = "collection-1234";
+ std::string record = "abcd";
+ CollectionOptions options;
+
+ std::unique_ptr<RecordStore> rs;
+ ASSERT_OK(_engine->createRecordStore(opCtxPtr.get(), ns, ident, options));
+ rs = _engine->getRecordStore(opCtxPtr.get(), ns, ident, options);
+ ASSERT(rs);
+
+ RecordId loc;
+ {
+ WriteUnitOfWork uow(opCtxPtr.get());
+ StatusWith<RecordId> res = rs->insertRecord(
+ opCtxPtr.get(), record.c_str(), record.length() + 1, Timestamp(), false);
+ ASSERT_OK(res.getStatus());
+ loc = res.getValue();
+ uow.commit();
+ }
+
+ const boost::optional<boost::filesystem::path> dataFilePath =
+ _engine->getDataFilePathForIdent(ident);
+ ASSERT(dataFilePath);
+
+ ASSERT(boost::filesystem::exists(*dataFilePath));
+
+ const boost::filesystem::path tmpFile{dataFilePath->string() + ".tmp"};
+ ASSERT(!boost::filesystem::exists(tmpFile));
+
+ // Move the data file out of the way so the ident can be dropped.
+ boost::system::error_code err;
+ boost::filesystem::rename(*dataFilePath, tmpFile, err);
+ ASSERT(!err) << err.message();
+
+ ASSERT_OK(_engine->dropIdent(opCtxPtr.get(), ident));
+
+ // The data file is moved back in place so that it becomes an "orphan" of the storage
+ // engine and the restoration process can be tested.
+ boost::filesystem::rename(tmpFile, *dataFilePath, err);
+ ASSERT(!err) << err.message();
+
+ ASSERT_OK(_engine->recoverOrphanedIdent(opCtxPtr.get(), ns, ident, options));
+
+ // The existing RecordStore is still usable with a different OperationContext and
+ // RecoveryUnit because a new session with new cursors will be opened.
+ ASSERT_EQUALS(record, rs->dataFor(opCtxPtr.get(), loc).data());
+}
+
+TEST_F(WiredTigerKVEngineRepairTest, UnrecoverableOrphanedDataFilesFailGracefully) {
+ auto opCtxPtr = makeOperationContext();
+
+ std::string ns = "a.b";
+ std::string ident = "collection-1234";
+ std::string record = "abcd";
+ CollectionOptions options;
+
+ std::unique_ptr<RecordStore> rs;
+ ASSERT_OK(_engine->createRecordStore(opCtxPtr.get(), ns, ident, options));
+ rs = _engine->getRecordStore(opCtxPtr.get(), ns, ident, options);
+ ASSERT(rs);
+
+ RecordId loc;
+ {
+ WriteUnitOfWork uow(opCtxPtr.get());
+ StatusWith<RecordId> res = rs->insertRecord(
+ opCtxPtr.get(), record.c_str(), record.length() + 1, Timestamp(), false);
+ ASSERT_OK(res.getStatus());
+ loc = res.getValue();
+ uow.commit();
+ }
+
+ const boost::optional<boost::filesystem::path> dataFilePath =
+ _engine->getDataFilePathForIdent(ident);
+ ASSERT(dataFilePath);
+
+ ASSERT(boost::filesystem::exists(*dataFilePath));
+
+ ASSERT_OK(_engine->dropIdent(opCtxPtr.get(), ident));
+
+ // The ident may not get immediately dropped, so ensure it is completely gone.
+ boost::system::error_code err;
+ boost::filesystem::remove(*dataFilePath, err);
+ ASSERT(!err) << err.message();
+
+ // Create an empty data file. The subsequent call to recreate the collection will fail because
+ // it is unsalvageable.
+ boost::filesystem::ofstream fileStream(*dataFilePath);
+ fileStream << "";
+ fileStream.close();
+
+ ASSERT(boost::filesystem::exists(*dataFilePath));
+
+ // This should fail gracefully and not cause any crashing.
+ ASSERT_NOT_OK(_engine->recoverOrphanedIdent(opCtxPtr.get(), ns, ident, options));
+}
+
std::unique_ptr<KVHarnessHelper> makeHelper() {
return stdx::make_unique<WiredTigerKVHarnessHelper>();
}
View Lorry Controller Status