diff options
Diffstat (limited to 'src/mongo/db/storage')
5 files changed, 93 insertions, 46 deletions
diff --git a/src/mongo/db/storage/wiredtiger/SConscript b/src/mongo/db/storage/wiredtiger/SConscript index 62f80ce6869..295dfd6ff36 100644 --- a/src/mongo/db/storage/wiredtiger/SConscript +++ b/src/mongo/db/storage/wiredtiger/SConscript @@ -83,6 +83,7 @@ if wiredtiger: LIBDEPS_PRIVATE= [ '$BUILD_DIR/mongo/db/snapshot_window_options', '$BUILD_DIR/mongo/util/options_parser/options_parser', + '$BUILD_DIR/mongo/db/storage/storage_repair_observer', ], ) @@ -147,6 +148,7 @@ if wiredtiger: '$BUILD_DIR/mongo/util/clock_source_mock', ], LIBDEPS_PRIVATE=[ + '$BUILD_DIR/mongo/db/auth/authmocks', '$BUILD_DIR/mongo/db/repl/repl_coordinator_interface', '$BUILD_DIR/mongo/db/repl/replmocks', ], @@ -191,6 +193,7 @@ if wiredtiger: 'additional_wiredtiger_record_store_tests', ], LIBDEPS_PRIVATE=[ + '$BUILD_DIR/mongo/db/auth/authmocks', '$BUILD_DIR/mongo/db/repl/repl_coordinator_interface', '$BUILD_DIR/mongo/db/repl/replmocks', ], @@ -205,6 +208,7 @@ if wiredtiger: 'additional_wiredtiger_record_store_tests', ], LIBDEPS_PRIVATE=[ + '$BUILD_DIR/mongo/db/auth/authmocks', '$BUILD_DIR/mongo/db/repl/repl_coordinator_interface', '$BUILD_DIR/mongo/db/repl/replmocks', ], @@ -218,6 +222,9 @@ if wiredtiger: LIBDEPS=[ 'additional_wiredtiger_index_tests', ], + LIBDEPS_PRIVATE=[ + '$BUILD_DIR/mongo/db/auth/authmocks', + ] ) wtEnv.CppUnitTest( @@ -228,6 +235,9 @@ if wiredtiger: LIBDEPS=[ 'additional_wiredtiger_index_tests', ], + LIBDEPS_PRIVATE=[ + '$BUILD_DIR/mongo/db/auth/authmocks', + ] ) wtEnv.CppUnitTest( diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp index 79326f3f1cc..eb41d13245f 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp @@ -71,6 +71,7 @@ #include "mongo/db/storage/journal_listener.h" #include "mongo/db/storage/storage_file_util.h" #include "mongo/db/storage/storage_options.h" +#include "mongo/db/storage/storage_repair_observer.h" #include "mongo/db/storage/wiredtiger/wiredtiger_customization_hooks.h" #include "mongo/db/storage/wiredtiger/wiredtiger_extensions.h" #include "mongo/db/storage/wiredtiger/wiredtiger_global_options.h" @@ -159,48 +160,6 @@ std::string WiredTigerFileVersion::getDowngradeString() { return "compatibility=(release=3.1)"; } -namespace { -void openWiredTiger(const std::string& path, - WT_EVENT_HANDLER* eventHandler, - const std::string& wtOpenConfig, - WT_CONNECTION** connOut, - WiredTigerFileVersion* fileVersionOut) { - std::string configStr = wtOpenConfig + ",compatibility=(require_min=\"3.1.0\")"; - int ret = wiredtiger_open(path.c_str(), eventHandler, configStr.c_str(), connOut); - if (!ret) { - *fileVersionOut = {WiredTigerFileVersion::StartupVersion::IS_40}; - return; - } - - // Arbiters do not replicate the FCV document. Due to arbiter FCV semantics on 4.0, shutting - // down a 4.0 arbiter may either downgrade the data files to WT compatibility 2.9 or 3.0. Thus, - // 4.2 binaries must allow starting up on 2.9 and 3.0 files. - configStr = wtOpenConfig + ",compatibility=(require_min=\"3.0.0\")"; - ret = wiredtiger_open(path.c_str(), eventHandler, configStr.c_str(), connOut); - if (!ret) { - *fileVersionOut = {WiredTigerFileVersion::StartupVersion::IS_36}; - return; - } - - configStr = wtOpenConfig + ",compatibility=(require_min=\"2.9.0\")"; - ret = wiredtiger_open(path.c_str(), eventHandler, configStr.c_str(), connOut); - if (!ret) { - *fileVersionOut = {WiredTigerFileVersion::StartupVersion::IS_34}; - return; - } - - severe() << "Failed to start up WiredTiger under any compatibility version."; - if (ret == EINVAL) { - fassertFailedNoTrace(28561); - } - - severe() << "Reason: " << wtRCToStatus(ret).reason(); - severe() << "Failed to open a WiredTiger connection. This may be due to metadata corruption. " - << kWTRepairMsg; - fassertFailedNoTrace(28595); -} -} // namespace - using std::set; using std::string; @@ -572,7 +531,7 @@ WiredTigerKVEngine::WiredTigerKVEngine(const std::string& canonicalName, string config = ss.str(); log() << "wiredtiger_open config: " << config; - openWiredTiger(path, _eventHandler.getWtEventHandler(), config, &_conn, &_fileVersion); + _openWiredTiger(path, config); _eventHandler.setStartupSuccessful(); _wtOpenConfig = config; @@ -645,6 +604,64 @@ void WiredTigerKVEngine::appendGlobalStats(BSONObjBuilder& b) { bb.done(); } +void WiredTigerKVEngine::_openWiredTiger(const std::string& path, const std::string& wtOpenConfig) { + std::string configStr = wtOpenConfig + ",compatibility=(require_min=\"3.1.0\")"; + + auto wtEventHandler = _eventHandler.getWtEventHandler(); + + int ret = wiredtiger_open(path.c_str(), wtEventHandler, configStr.c_str(), &_conn); + if (!ret) { + _fileVersion = {WiredTigerFileVersion::StartupVersion::IS_40}; + return; + } + + // Arbiters do not replicate the FCV document. Due to arbiter FCV semantics on 4.0, shutting + // down a 4.0 arbiter may either downgrade the data files to WT compatibility 2.9 or 3.0. Thus, + // 4.2 binaries must allow starting up on 2.9 and 3.0 files. + configStr = wtOpenConfig + ",compatibility=(require_min=\"3.0.0\")"; + ret = wiredtiger_open(path.c_str(), wtEventHandler, configStr.c_str(), &_conn); + if (!ret) { + _fileVersion = {WiredTigerFileVersion::StartupVersion::IS_36}; + return; + } + + configStr = wtOpenConfig + ",compatibility=(require_min=\"2.9.0\")"; + ret = wiredtiger_open(path.c_str(), wtEventHandler, configStr.c_str(), &_conn); + if (!ret) { + _fileVersion = {WiredTigerFileVersion::StartupVersion::IS_34}; + return; + } + + warning() << "Failed to start up WiredTiger under any compatibility version."; + if (ret == EINVAL) { + fassertFailedNoTrace(28561); + } + + if (ret == WT_TRY_SALVAGE) { + warning() << "WiredTiger metadata corruption detected"; + + if (!_inRepairMode) { + severe() << kWTRepairMsg; + fassertFailedNoTrace(50944); + } + + warning() << "Attempting to salvage WiredTiger metadata"; + configStr = wtOpenConfig + ",salvage=true"; + ret = wiredtiger_open(path.c_str(), wtEventHandler, configStr.c_str(), &_conn); + if (!ret) { + StorageRepairObserver::get(getGlobalServiceContext()) + ->onModification("WiredTiger metadata salvaged"); + return; + } + + severe() << "Failed to salvage WiredTiger metadata: " + wtRCToStatus(ret).reason(); + fassertFailedNoTrace(50947); + } + + severe() << "Reason: " << wtRCToStatus(ret).reason(); + fassertFailedNoTrace(28595); +} + void WiredTigerKVEngine::cleanShutdown() { log() << "WiredTigerKVEngine shutting down"; if (!_readOnly) diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h index f466a4eb35e..32d8e93d438 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h @@ -336,6 +336,15 @@ private: class WiredTigerJournalFlusher; class WiredTigerCheckpointThread; + /** + * Opens a connection on the WiredTiger database 'path' with the configuration 'wtOpenConfig'. + * Only returns when successful. Intializes both '_conn' and '_fileVersion'. + * + * If corruption is detected and _inRepairMode is 'true', attempts to salvage the WiredTiger + * metadata. + */ + void _openWiredTiger(const std::string& path, const std::string& wtOpenConfig); + Status _salvageIfNeeded(const char* uri); void _ensureIdentPath(StringData ident); @@ -383,6 +392,7 @@ private: void _setOldestTimestamp(Timestamp newOldestTimestamp, bool force); WT_CONNECTION* _conn; + WiredTigerFileVersion _fileVersion; WiredTigerEventHandler _eventHandler; std::unique_ptr<WiredTigerSessionCache> _sessionCache; ClockSource* const _clockSource; @@ -418,7 +428,6 @@ private: std::unique_ptr<WiredTigerSession> _backupSession; Timestamp _recoveryTimestamp; - WiredTigerFileVersion _fileVersion; // Tracks the stable and oldest timestamps we've set on the storage engine. AtomicWord<std::uint64_t> _oldestTimestamp; diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_session_cache.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_session_cache.cpp index 8021fafd771..1007eb0193c 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_session_cache.cpp +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_session_cache.cpp @@ -131,7 +131,7 @@ WT_CURSOR* WiredTigerSession::getCursor(const std::string& uri, uint64_t id, boo } if (ret != 0) { error() << "Failed to open a WiredTiger cursor: " << uri; - error() << "This may be due to metadata corruption. " << kWTRepairMsg; + error() << "This may be due to data corruption. " << kWTRepairMsg; fassertFailedNoTrace(50882); } diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp index 15e5913a4d3..004062a6d04 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp @@ -61,7 +61,8 @@ Status wtRCToStatus_slow(int retCode, const char* prefix) { throw WriteConflictException(); } - fassert(28559, retCode != WT_PANIC); + // Don't abort on WT_PANIC when repairing, as the error will be handled at a higher layer. + fassert(28559, retCode != WT_PANIC || storageGlobalParams.repair); str::stream s; if (prefix) @@ -403,6 +404,11 @@ int mdb_handle_error_with_startup_suppression(WT_EVENT_HANDLER* handler, error() << "WiredTiger error (" << errorCode << ") " << redact(message) << " Raw: " << message; + + // Don't abort on WT_PANIC when repairing, as the error will be handled at a higher layer. + if (storageGlobalParams.repair) { + return 0; + } fassert(50853, errorCode != WT_PANIC); } catch (...) { std::terminate(); @@ -416,6 +422,11 @@ int mdb_handle_error(WT_EVENT_HANDLER* handler, const char* message) { try { error() << "WiredTiger error (" << errorCode << ") " << redact(message); + + // Don't abort on WT_PANIC when repairing, as the error will be handled at a higher layer. + if (storageGlobalParams.repair) { + return 0; + } fassert(28558, errorCode != WT_PANIC); } catch (...) { std::terminate(); |