diff options
Diffstat (limited to 'src/mongo/db')
12 files changed, 212 insertions, 30 deletions
diff --git a/src/mongo/db/storage/wiredtiger/SConscript b/src/mongo/db/storage/wiredtiger/SConscript index e1f8b71113f..2a6755c7f49 100644 --- a/src/mongo/db/storage/wiredtiger/SConscript +++ b/src/mongo/db/storage/wiredtiger/SConscript @@ -52,7 +52,9 @@ if wiredtiger: '$BUILD_DIR/mongo/db/concurrency/lock_manager', '$BUILD_DIR/mongo/db/concurrency/write_conflict_exception', '$BUILD_DIR/mongo/db/index/index_descriptor', + '$BUILD_DIR/mongo/db/mongod_options', '$BUILD_DIR/mongo/db/namespace_string', + '$BUILD_DIR/mongo/db/repl/repl_settings', '$BUILD_DIR/mongo/db/server_options_core', '$BUILD_DIR/mongo/db/service_context', '$BUILD_DIR/mongo/db/storage/index_entry_comparison', diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_index.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_index.cpp index ae536a2fdf0..db1d8c048f4 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_index.cpp +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_index.cpp @@ -41,6 +41,8 @@ #include "mongo/db/concurrency/write_conflict_exception.h" #include "mongo/db/index/index_descriptor.h" #include "mongo/db/json.h" +#include "mongo/db/mongod_options.h" +#include "mongo/db/repl/repl_settings.h" #include "mongo/db/service_context.h" #include "mongo/db/storage/key_string.h" #include "mongo/db/storage/storage_options.h" @@ -217,6 +219,15 @@ StatusWith<std::string> WiredTigerIndex::generateCreateString(const std::string& << "formatVersion=" << keyStringVersion << ',' << "infoObj=" << desc.infoObj().jsonString() << "),"; + const bool keepOldLoggingSettings = true; + if (keepOldLoggingSettings || + WiredTigerUtil::useTableLogging(NamespaceString(desc.parentNS()), + getGlobalReplSettings().usingReplSets())) { + ss << "log=(enabled=true)"; + } else { + ss << "log=(enabled=false)"; + } + LOG(3) << "index create string: " << ss.ss.str(); return StatusWith<std::string>(ss); } @@ -234,7 +245,8 @@ int WiredTigerIndex::Create(OperationContext* opCtx, WiredTigerIndex::WiredTigerIndex(OperationContext* ctx, const std::string& uri, const IndexDescriptor* desc, - KVPrefix prefix) + KVPrefix prefix, + bool isReadOnly) : _ordering(Ordering::make(desc->keyPattern())), _uri(uri), _tableId(WiredTigerSession::genTableId()), @@ -256,6 +268,14 @@ WiredTigerIndex::WiredTigerIndex(OperationContext* ctx, } _keyStringVersion = version.getValue() == kKeyStringV1Version ? KeyString::Version::V1 : KeyString::Version::V0; + + if (!isReadOnly) { + uassertStatusOK(WiredTigerUtil::setTableLogging( + ctx, + uri, + WiredTigerUtil::useTableLogging(NamespaceString(desc->parentNS()), + getGlobalReplSettings().usingReplSets()))); + } } Status WiredTigerIndex::insert(OperationContext* opCtx, @@ -1081,8 +1101,9 @@ public: WiredTigerIndexUnique::WiredTigerIndexUnique(OperationContext* ctx, const std::string& uri, const IndexDescriptor* desc, - KVPrefix prefix) - : WiredTigerIndex(ctx, uri, desc, prefix), _partial(desc->isPartial()) {} + KVPrefix prefix, + bool isReadOnly) + : WiredTigerIndex(ctx, uri, desc, prefix, isReadOnly), _partial(desc->isPartial()) {} std::unique_ptr<SortedDataInterface::Cursor> WiredTigerIndexUnique::newCursor( OperationContext* opCtx, bool forward) const { @@ -1267,8 +1288,9 @@ void WiredTigerIndexUnique::_unindex(WT_CURSOR* c, WiredTigerIndexStandard::WiredTigerIndexStandard(OperationContext* ctx, const std::string& uri, const IndexDescriptor* desc, - KVPrefix prefix) - : WiredTigerIndex(ctx, uri, desc, prefix) {} + KVPrefix prefix, + bool isReadOnly) + : WiredTigerIndex(ctx, uri, desc, prefix, isReadOnly) {} std::unique_ptr<SortedDataInterface::Cursor> WiredTigerIndexStandard::newCursor( OperationContext* opCtx, bool forward) const { diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_index.h b/src/mongo/db/storage/wiredtiger/wiredtiger_index.h index 4fe86efa009..a90f770a529 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_index.h +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_index.h @@ -79,7 +79,8 @@ public: WiredTigerIndex(OperationContext* ctx, const std::string& uri, const IndexDescriptor* desc, - KVPrefix prefix); + KVPrefix prefix, + bool readOnly); virtual Status insert(OperationContext* opCtx, const BSONObj& key, @@ -169,7 +170,8 @@ public: WiredTigerIndexUnique(OperationContext* ctx, const std::string& uri, const IndexDescriptor* desc, - KVPrefix prefix); + KVPrefix prefix, + bool readOnly = false); std::unique_ptr<SortedDataInterface::Cursor> newCursor(OperationContext* opCtx, bool forward) const override; @@ -193,7 +195,8 @@ public: WiredTigerIndexStandard(OperationContext* ctx, const std::string& uri, const IndexDescriptor* desc, - KVPrefix prefix); + KVPrefix prefix, + bool readOnly = false); std::unique_ptr<SortedDataInterface::Cursor> newCursor(OperationContext* opCtx, bool forward) const override; diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp index a1de28f80b1..0cee2bcd042 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp @@ -54,6 +54,9 @@ #include "mongo/db/concurrency/locker.h" #include "mongo/db/concurrency/write_conflict_exception.h" #include "mongo/db/index/index_descriptor.h" +#include "mongo/db/mongod_options.h" +#include "mongo/db/repl/repl_settings.h" +#include "mongo/db/server_options.h" #include "mongo/db/server_parameters.h" #include "mongo/db/service_context.h" #include "mongo/db/storage/journal_listener.h" @@ -316,6 +319,7 @@ WiredTigerKVEngine::WiredTigerKVEngine(const std::string& canonicalName, } string config = ss.str(); log() << "wiredtiger_open config: " << config; + _wtOpenConfig = config; int ret = wiredtiger_open(path.c_str(), &_eventHandler, config.c_str(), &_conn); // Invalid argument (EINVAL) is usually caused by invalid configuration string. // We still fassert() but without a stack trace. @@ -344,7 +348,10 @@ WiredTigerKVEngine::WiredTigerKVEngine(const std::string& canonicalName, log() << "Repairing size cache"; fassertNoTrace(28577, _salvageIfNeeded(_sizeStorerUri.c_str())); } - _sizeStorer.reset(new WiredTigerSizeStorer(_conn, _sizeStorerUri)); + + const bool sizeStorerLoggingEnabled = !getGlobalReplSettings().usingReplSets(); + _sizeStorer.reset( + new WiredTigerSizeStorer(_conn, _sizeStorerUri, sizeStorerLoggingEnabled, _readOnly)); _sizeStorer->fillCache(); Locker::setGlobalThrottling(&openReadTransaction, &openWriteTransaction); @@ -398,18 +405,66 @@ void WiredTigerKVEngine::cleanShutdown() { #else bool leak_memory = false; #endif - const char* config = nullptr; + const char* closeConfig = nullptr; if (RUNNING_ON_VALGRIND) { leak_memory = false; } if (leak_memory) { - config = "leak_memory=true"; + closeConfig = "leak_memory=true"; } - invariantWTOK(_conn->close(_conn, config)); - _conn = NULL; + const bool keepOldBehavior = true; + const bool needsDowngrade = !keepOldBehavior && !_readOnly && + serverGlobalParams.featureCompatibility.version.load() == + ServerGlobalParams::FeatureCompatibility::Version::k34; + + invariantWTOK(_conn->close(_conn, closeConfig)); + _conn = nullptr; + + // If FCV 3.4, enable WT logging on all tables. + if (needsDowngrade) { + log() << "Downgrading files to FCV 3.4"; + WT_CONNECTION* conn; + std::stringstream openConfig; + openConfig << _wtOpenConfig << ",log=(archive=false)"; + + const bool NEW_WT_DROPPED = false; + if (NEW_WT_DROPPED) { + openConfig << ",compatibility=(release=2.9)"; + } + + invariantWTOK( + wiredtiger_open(_path.c_str(), &_eventHandler, openConfig.str().c_str(), &conn)); + + WT_SESSION* session; + conn->open_session(conn, nullptr, "", &session); + + WT_CURSOR* tableCursor; + invariantWTOK( + session->open_cursor(session, "metadata:", nullptr, nullptr, &tableCursor)); + while (tableCursor->next(tableCursor) == 0) { + const char* raw; + tableCursor->get_key(tableCursor, &raw); + StringData key(raw); + size_t idx = key.find(':'); + if (idx == string::npos) { + continue; + } + + StringData type = key.substr(0, idx); + if (type != "table") { + continue; + } + + uassertStatusOK(WiredTigerUtil::setTableLogging(session, raw, true)); + } + + tableCursor->close(tableCursor); + session->close(session, nullptr); + invariantWTOK(conn->close(conn, closeConfig)); + } } } @@ -538,7 +593,8 @@ Status WiredTigerKVEngine::createGroupedRecordStore(OperationContext* opCtx, string uri = _uri(ident); WT_SESSION* s = session.getSession(); - LOG(2) << "WiredTigerKVEngine::createRecordStore uri: " << uri << " config: " << config; + LOG(2) << "WiredTigerKVEngine::createRecordStore ns: " << ns << " uri: " << uri + << " config: " << config; return wtRCToStatus(s->create(s, uri.c_str(), config.c_str())); } @@ -557,6 +613,7 @@ std::unique_ptr<RecordStore> WiredTigerKVEngine::getGroupedRecordStore( params.isEphemeral = _ephemeral; params.cappedCallback = nullptr; params.sizeStorer = _sizeStorer.get(); + params.isReadOnly = _readOnly; params.cappedMaxSize = -1; if (options.capped) { @@ -616,8 +673,8 @@ Status WiredTigerKVEngine::createGroupedSortedDataInterface(OperationContext* op std::string config = result.getValue(); - LOG(2) << "WiredTigerKVEngine::createSortedDataInterface ident: " << ident - << " config: " << config; + LOG(2) << "WiredTigerKVEngine::createSortedDataInterface ns: " << collection->ns() + << " ident: " << ident << " config: " << config; return wtRCToStatus(WiredTigerIndex::Create(opCtx, _uri(ident), config)); } @@ -626,8 +683,8 @@ SortedDataInterface* WiredTigerKVEngine::getGroupedSortedDataInterface(Operation const IndexDescriptor* desc, KVPrefix prefix) { if (desc->unique()) - return new WiredTigerIndexUnique(opCtx, _uri(ident), desc, prefix); - return new WiredTigerIndexStandard(opCtx, _uri(ident), desc, prefix); + return new WiredTigerIndexUnique(opCtx, _uri(ident), desc, prefix, _readOnly); + return new WiredTigerIndexStandard(opCtx, _uri(ident), desc, prefix, _readOnly); } Status WiredTigerKVEngine::dropIdent(OperationContext* opCtx, StringData ident) { diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h index 493ae38c4e1..c6078a06015 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h @@ -209,6 +209,7 @@ private: std::unique_ptr<WiredTigerSessionCache> _sessionCache; std::string _canonicalName; std::string _path; + std::string _wtOpenConfig; std::unique_ptr<WiredTigerSizeStorer> _sizeStorer; std::string _sizeStorerUri; diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.cpp index c79afd052c4..c47701a087f 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.cpp +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.cpp @@ -40,8 +40,10 @@ #include "mongo/bson/util/builder.h" #include "mongo/db/concurrency/locker.h" #include "mongo/db/concurrency/write_conflict_exception.h" +#include "mongo/db/mongod_options.h" #include "mongo/db/namespace_string.h" #include "mongo/db/operation_context.h" +#include "mongo/db/repl/repl_settings.h" #include "mongo/db/service_context.h" #include "mongo/db/storage/oplog_hack.h" #include "mongo/db/storage/wiredtiger/wiredtiger_customization_hooks.h" @@ -594,6 +596,15 @@ StatusWith<std::string> WiredTigerRecordStore::generateCreateString( } ss << ")"; + const bool keepOldLoggingSettings = true; + if (keepOldLoggingSettings || + WiredTigerUtil::useTableLogging(NamespaceString(ns), + getGlobalReplSettings().usingReplSets())) { + ss << ",log=(enabled=true)"; + } else { + ss << ",log=(enabled=false)"; + } + return StatusWith<std::string>(ss); } @@ -635,6 +646,14 @@ WiredTigerRecordStore::WiredTigerRecordStore(OperationContext* ctx, Params param invariant(_cappedMaxSize == -1); invariant(_cappedMaxDocs == -1); } + + if (!params.isReadOnly) { + uassertStatusOK(WiredTigerUtil::setTableLogging( + ctx, + _uri, + WiredTigerUtil::useTableLogging(NamespaceString(ns()), + getGlobalReplSettings().usingReplSets()))); + } } WiredTigerRecordStore::~WiredTigerRecordStore() { diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.h b/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.h index aca92fbea1d..5ff06004cdd 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.h +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_record_store.h @@ -110,6 +110,7 @@ public: int64_t cappedMaxDocs; CappedCallback* cappedCallback; WiredTigerSizeStorer* sizeStorer; + bool isReadOnly; }; WiredTigerRecordStore(OperationContext* opCtx, Params params); diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_size_storer.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_size_storer.cpp index 7783cea4cbf..23a8c8f90f5 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_size_storer.cpp +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_size_storer.cpp @@ -55,18 +55,27 @@ namespace { int MAGIC = 123123; } -WiredTigerSizeStorer::WiredTigerSizeStorer(WT_CONNECTION* conn, const std::string& storageUri) +WiredTigerSizeStorer::WiredTigerSizeStorer(WT_CONNECTION* conn, + const std::string& storageUri, + bool logSizeStorerTable, + bool readOnly) : _session(conn) { WT_SESSION* session = _session.getSession(); - int ret = session->open_cursor(session, storageUri.c_str(), NULL, "overwrite=true", &_cursor); - if (ret == ENOENT) { - // Need to create table. - std::string config = WiredTigerCustomizationHooks::get(getGlobalServiceContext()) - ->getTableCreateConfig(storageUri); + + std::string config = WiredTigerCustomizationHooks::get(getGlobalServiceContext()) + ->getTableCreateConfig(storageUri); + if (!readOnly) { invariantWTOK(session->create(session, storageUri.c_str(), config.c_str())); - ret = session->open_cursor(session, storageUri.c_str(), NULL, "overwrite=true", &_cursor); + const bool keepOldLoggingSettings = true; + if (keepOldLoggingSettings) { + logSizeStorerTable = true; + } + uassertStatusOK( + WiredTigerUtil::setTableLogging(session, storageUri.c_str(), logSizeStorerTable)); } - invariantWTOK(ret); + + invariantWTOK( + session->open_cursor(session, storageUri.c_str(), NULL, "overwrite=true", &_cursor)); _magic = MAGIC; } diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_size_storer.h b/src/mongo/db/storage/wiredtiger/wiredtiger_size_storer.h index 3dcbc6622a8..67ec4d25c6a 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_size_storer.h +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_size_storer.h @@ -46,7 +46,10 @@ class WiredTigerSession; class WiredTigerSizeStorer { public: - WiredTigerSizeStorer(WT_CONNECTION* conn, const std::string& storageUri); + WiredTigerSizeStorer(WT_CONNECTION* conn, + const std::string& storageUri, + const bool isWiredTigerLoggingEnabled, + const bool readOnly = false); ~WiredTigerSizeStorer(); void onCreate(WiredTigerRecordStore* rs, long long nr, long long ds); diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_standard_record_store_test.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_standard_record_store_test.cpp index 7e9f5d3eb77..8f0f862f881 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_standard_record_store_test.cpp +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_standard_record_store_test.cpp @@ -217,7 +217,8 @@ TEST(WiredTigerRecordStoreTest, SizeStorer1) { string uri = checked_cast<WiredTigerRecordStore*>(rs.get())->getURI(); string indexUri = "table:myindex"; - WiredTigerSizeStorer ss(harnessHelper->conn(), indexUri); + const bool enableWtLogging = false; + WiredTigerSizeStorer ss(harnessHelper->conn(), indexUri, enableWtLogging); checked_cast<WiredTigerRecordStore*>(rs.get())->setSizeStorer(&ss); int N = 12; @@ -287,7 +288,8 @@ TEST(WiredTigerRecordStoreTest, SizeStorer1) { { ServiceContext::UniqueOperationContext opCtx(harnessHelper->newOperationContext()); - WiredTigerSizeStorer ss2(harnessHelper->conn(), indexUri); + const bool enableWtLogging = false; + WiredTigerSizeStorer ss2(harnessHelper->conn(), indexUri, enableWtLogging); ss2.fillCache(); long long numRecords; long long dataSize; @@ -318,7 +320,9 @@ class SizeStorerValidateTest : public mongo::unittest::Test { private: virtual void setUp() { harnessHelper.reset(new WiredTigerHarnessHelper()); - sizeStorer.reset(new WiredTigerSizeStorer(harnessHelper->conn(), "table:sizeStorer")); + const bool enableWtLogging = false; + sizeStorer.reset( + new WiredTigerSizeStorer(harnessHelper->conn(), "table:sizeStorer", enableWtLogging)); rs = harnessHelper->newNonCappedRecordStore(); WiredTigerRecordStore* wtrs = checked_cast<WiredTigerRecordStore*>(rs.get()); wtrs->setSizeStorer(sizeStorer.get()); diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp index 5b8306381b9..a367d2d61a6 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp @@ -439,6 +439,60 @@ int WiredTigerUtil::verifyTable(OperationContext* opCtx, return (session->verify)(session, uri.c_str(), NULL); } +bool WiredTigerUtil::useTableLogging(NamespaceString ns, bool replEnabled) { + if (!replEnabled) { + // All tables on standalones are logged. + return true; + } + + // Of the replica set configurations: + if (ns.db() != "local") { + // All replicated collections are not logged. + return false; + } + + if (ns.coll() == "replset.checkpointTimestamp" || ns.coll() == "replset.minvalid") { + // Of local collections, these two are derived from the state of the data and therefore + // are not logged. + return false; + } + + // The remainder of local gets logged. In particular, the oplog and user created collections. + return true; +} + +Status WiredTigerUtil::setTableLogging(OperationContext* opCtx, const std::string& uri, bool on) { + WiredTigerRecoveryUnit* recoveryUnit = WiredTigerRecoveryUnit::get(opCtx); + return setTableLogging(recoveryUnit->getSession(opCtx)->getSession(), uri, on); +} + +Status WiredTigerUtil::setTableLogging(WT_SESSION* session, const std::string& uri, bool on) { + const bool NEW_WT_DROPPED = false; + if (!NEW_WT_DROPPED) { + return Status::OK(); + } + + LOG(3) << "Changing logging values. Uri: " << uri << " Enabled? " << on; + int ret; + if (on) { + ret = session->alter(session, uri.c_str(), "log=(enabled=true)"); + } else { + ret = session->alter(session, uri.c_str(), "log=(enabled=false)"); + } + + if (ret) { + return Status(ErrorCodes::WriteConflict, + str::stream() << "Failed to update log setting. Uri: " << uri << " Enable? " + << on + << " Ret: " + << ret + << " Msg: " + << session->strerror(session, ret)); + } + + return Status::OK(); +} + Status WiredTigerUtil::exportTableToBSON(WT_SESSION* session, const std::string& uri, const std::string& config, diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_util.h b/src/mongo/db/storage/wiredtiger/wiredtiger_util.h index 248ac7b8450..5817a5855f6 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_util.h +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_util.h @@ -37,6 +37,7 @@ #include "mongo/base/status.h" #include "mongo/base/status_with.h" #include "mongo/bson/bsonobj.h" +#include "mongo/db/namespace_string.h" #include "mongo/util/assert_util.h" namespace mongo { @@ -218,6 +219,12 @@ public: const std::string& uri, std::vector<std::string>* errors = NULL); + static bool useTableLogging(NamespaceString ns, bool replEnabled); + + static Status setTableLogging(OperationContext* opCtx, const std::string& uri, bool on); + + static Status setTableLogging(WT_SESSION* session, const std::string& uri, bool on); + private: /** * Casts unsigned 64-bit statistics value to T. |