diff options
Diffstat (limited to 'src/mongo/db/storage')
18 files changed, 20 insertions, 147 deletions
diff --git a/src/mongo/db/storage/biggie/SConscript b/src/mongo/db/storage/biggie/SConscript index 11706d48971..f96f7e7c249 100644 --- a/src/mongo/db/storage/biggie/SConscript +++ b/src/mongo/db/storage/biggie/SConscript @@ -22,7 +22,6 @@ env.Library( ], LIBDEPS_PRIVATE=[ '$BUILD_DIR/mongo/db/storage/key_string', - '$BUILD_DIR/mongo/db/snapshot_window_options', '$BUILD_DIR/mongo/db/storage/oplog_hack', '$BUILD_DIR/mongo/db/storage/write_unit_of_work', ], diff --git a/src/mongo/db/storage/biggie/biggie_kv_engine.cpp b/src/mongo/db/storage/biggie/biggie_kv_engine.cpp index 3e77b6a8cf3..7c83398126b 100644 --- a/src/mongo/db/storage/biggie/biggie_kv_engine.cpp +++ b/src/mongo/db/storage/biggie/biggie_kv_engine.cpp @@ -36,7 +36,6 @@ #include <memory> #include "mongo/db/index/index_descriptor.h" -#include "mongo/db/snapshot_window_options.h" #include "mongo/db/storage/biggie/biggie_recovery_unit.h" #include "mongo/db/storage/key_string.h" #include "mongo/db/storage/record_store.h" @@ -49,10 +48,6 @@ mongo::RecoveryUnit* KVEngine::newRecoveryUnit() { return new RecoveryUnit(this, nullptr); } -void KVEngine::setCachePressureForTest(int pressure) { - // TODO : implement. -} - Status KVEngine::createRecordStore(OperationContext* opCtx, StringData ns, StringData ident, diff --git a/src/mongo/db/storage/biggie/biggie_kv_engine.h b/src/mongo/db/storage/biggie/biggie_kv_engine.h index 2ce35d07cbf..63e89e8b579 100644 --- a/src/mongo/db/storage/biggie/biggie_kv_engine.h +++ b/src/mongo/db/storage/biggie/biggie_kv_engine.h @@ -105,12 +105,6 @@ public: return true; } - virtual bool isCacheUnderPressure(OperationContext* opCtx) const override { - return false; - } - - virtual void setCachePressureForTest(int pressure) override; - virtual int64_t getIdentSize(OperationContext* opCtx, StringData ident) { return 0; } diff --git a/src/mongo/db/storage/devnull/SConscript b/src/mongo/db/storage/devnull/SConscript index 66dcb6763a7..f2963e59380 100644 --- a/src/mongo/db/storage/devnull/SConscript +++ b/src/mongo/db/storage/devnull/SConscript @@ -13,9 +13,6 @@ env.Library( '$BUILD_DIR/mongo/db/storage/ephemeral_for_test/ephemeral_for_test_record_store', '$BUILD_DIR/mongo/db/storage/kv/kv_prefix', ], - LIBDEPS_PRIVATE=[ - '$BUILD_DIR/mongo/db/snapshot_window_options', - ], ) env.Library( diff --git a/src/mongo/db/storage/devnull/devnull_kv_engine.cpp b/src/mongo/db/storage/devnull/devnull_kv_engine.cpp index 4bbc0b9dc11..f70b7ad94fd 100644 --- a/src/mongo/db/storage/devnull/devnull_kv_engine.cpp +++ b/src/mongo/db/storage/devnull/devnull_kv_engine.cpp @@ -33,7 +33,6 @@ #include <memory> -#include "mongo/db/snapshot_window_options.h" #include "mongo/db/storage/ephemeral_for_test/ephemeral_for_test_record_store.h" #include "mongo/db/storage/record_store.h" #include "mongo/db/storage/sorted_data_interface.h" @@ -244,15 +243,6 @@ std::unique_ptr<SortedDataInterface> DevNullKVEngine::getSortedDataInterface( return std::make_unique<DevNullSortedDataInterface>(); } -bool DevNullKVEngine::isCacheUnderPressure(OperationContext* opCtx) const { - return (_cachePressureForTest >= snapshotWindowParams.cachePressureThreshold.load()); -} - -void DevNullKVEngine::setCachePressureForTest(int pressure) { - invariant(pressure >= 0 && pressure <= 100); - _cachePressureForTest = pressure; -} - namespace { class StreamingCursorImpl : public StorageEngine::StreamingCursor { diff --git a/src/mongo/db/storage/devnull/devnull_kv_engine.h b/src/mongo/db/storage/devnull/devnull_kv_engine.h index f271ae0f0f8..dd1854e0256 100644 --- a/src/mongo/db/storage/devnull/devnull_kv_engine.h +++ b/src/mongo/db/storage/devnull/devnull_kv_engine.h @@ -97,10 +97,6 @@ public: return true; } - virtual bool isCacheUnderPressure(OperationContext* opCtx) const override; - - virtual void setCachePressureForTest(int pressure) override; - virtual int64_t getIdentSize(OperationContext* opCtx, StringData ident) { return 1; } diff --git a/src/mongo/db/storage/kv/kv_engine.h b/src/mongo/db/storage/kv/kv_engine.h index f20897f8be2..f5ceccc0b53 100644 --- a/src/mongo/db/storage/kv/kv_engine.h +++ b/src/mongo/db/storage/kv/kv_engine.h @@ -356,18 +356,6 @@ public: virtual void setOldestTimestamp(Timestamp newOldestTimestamp, bool force) {} /** - * See `StorageEngine::isCacheUnderPressure()` - */ - virtual bool isCacheUnderPressure(OperationContext* opCtx) const { - return false; - } - - /** - * See 'StorageEngine::setCachePressureForTest()' - */ - virtual void setCachePressureForTest(int pressure) {} - - /** * See `StorageEngine::supportsRecoverToStableTimestamp` */ virtual bool supportsRecoverToStableTimestamp() const { diff --git a/src/mongo/db/storage/storage_engine.h b/src/mongo/db/storage/storage_engine.h index 98285ff0f30..8d7df601b5a 100644 --- a/src/mongo/db/storage/storage_engine.h +++ b/src/mongo/db/storage/storage_engine.h @@ -517,21 +517,6 @@ public: virtual void setOldestActiveTransactionTimestampCallback( OldestActiveTransactionTimestampCallback callback) = 0; - /** - * Indicates whether the storage engine cache is under pressure. - * - * Retrieves a cache pressure value in the range [0, 100] from the storage engine, and compares - * it against storageGlobalParams.cachePressureThreshold, a dynamic server parameter, to - * determine whether cache pressure is too high. - */ - virtual bool isCacheUnderPressure(OperationContext* opCtx) const = 0; - - /** - * For unit tests only. Sets the cache pressure value with which isCacheUnderPressure() - * evalutates to 'pressure'. - */ - virtual void setCachePressureForTest(int pressure) = 0; - struct IndexIdentifier { const RecordId catalogId; const NamespaceString nss; diff --git a/src/mongo/db/storage/storage_engine_impl.cpp b/src/mongo/db/storage/storage_engine_impl.cpp index 91ac87542c6..d45ccf93c22 100644 --- a/src/mongo/db/storage/storage_engine_impl.cpp +++ b/src/mongo/db/storage/storage_engine_impl.cpp @@ -783,14 +783,6 @@ void StorageEngineImpl::setOldestActiveTransactionTimestampCallback( _engine->setOldestActiveTransactionTimestampCallback(callback); } -bool StorageEngineImpl::isCacheUnderPressure(OperationContext* opCtx) const { - return _engine->isCacheUnderPressure(opCtx); -} - -void StorageEngineImpl::setCachePressureForTest(int pressure) { - return _engine->setCachePressureForTest(pressure); -} - bool StorageEngineImpl::supportsRecoverToStableTimestamp() const { return _engine->supportsRecoverToStableTimestamp(); } diff --git a/src/mongo/db/storage/storage_engine_impl.h b/src/mongo/db/storage/storage_engine_impl.h index 3fd3bab86d1..70bc31830c3 100644 --- a/src/mongo/db/storage/storage_engine_impl.h +++ b/src/mongo/db/storage/storage_engine_impl.h @@ -133,10 +133,6 @@ public: virtual void setOldestActiveTransactionTimestampCallback( StorageEngine::OldestActiveTransactionTimestampCallback) override; - virtual bool isCacheUnderPressure(OperationContext* opCtx) const override; - - virtual void setCachePressureForTest(int pressure) override; - virtual bool supportsRecoverToStableTimestamp() const override; virtual bool supportsRecoveryTimestamp() const override; diff --git a/src/mongo/db/storage/storage_engine_mock.h b/src/mongo/db/storage/storage_engine_mock.h index b8cb1cbde02..a4dfe3c677f 100644 --- a/src/mongo/db/storage/storage_engine_mock.h +++ b/src/mongo/db/storage/storage_engine_mock.h @@ -138,10 +138,7 @@ public: void setOldestTimestamp(Timestamp timestamp) final {} void setOldestActiveTransactionTimestampCallback( OldestActiveTransactionTimestampCallback callback) final {} - bool isCacheUnderPressure(OperationContext* opCtx) const final { - return false; - } - void setCachePressureForTest(int pressure) final {} + StatusWith<StorageEngine::ReconcileResult> reconcileCatalogAndIdents( OperationContext* opCtx) final { return ReconcileResult{}; diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp index 352699a9de8..b9749e79eab 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp @@ -69,7 +69,7 @@ #include "mongo/db/server_options.h" #include "mongo/db/server_recovery.h" #include "mongo/db/service_context.h" -#include "mongo/db/snapshot_window_options.h" +#include "mongo/db/snapshot_window_options_gen.h" #include "mongo/db/storage/journal_listener.h" #include "mongo/db/storage/storage_file_util.h" #include "mongo/db/storage/storage_options.h" @@ -681,7 +681,7 @@ WiredTigerKVEngine::WiredTigerKVEngine(const std::string& canonicalName, // We do not maintain any snapshot history for the ephemeral storage engine in production // because replication and sharded transactions do not currently run on the inMemory engine. // It is live in testing, however. - snapshotWindowParams.minSnapshotHistoryWindowInSeconds.store(0); + minSnapshotHistoryWindowInSeconds.store(0); } _sizeStorerUri = _uri("sizeStorer"); @@ -2017,20 +2017,20 @@ Timestamp WiredTigerKVEngine::_calculateHistoryLagFromStableTimestamp(Timestamp if (_ephemeral && !getTestCommandsEnabled()) { // No history should be maintained for the inMemory engine because it is not used yet. - invariant(snapshotWindowParams.minSnapshotHistoryWindowInSeconds.load() == 0); + invariant(minSnapshotHistoryWindowInSeconds.load() == 0); } if (stableTimestamp.getSecs() < - static_cast<unsigned>(snapshotWindowParams.minSnapshotHistoryWindowInSeconds.load())) { + static_cast<unsigned>(minSnapshotHistoryWindowInSeconds.load())) { // The history window is larger than the timestamp history thus far. We must wait for // the history to reach the window size before moving oldest_timestamp forward. This should // only happen in unit tests. return Timestamp(); } - Timestamp calculatedOldestTimestamp( - stableTimestamp.getSecs() - snapshotWindowParams.minSnapshotHistoryWindowInSeconds.load(), - stableTimestamp.getInc()); + Timestamp calculatedOldestTimestamp(stableTimestamp.getSecs() - + minSnapshotHistoryWindowInSeconds.load(), + stableTimestamp.getInc()); if (calculatedOldestTimestamp.asULL() <= _oldestTimestamp.load()) { // The stable_timestamp is not far enough ahead of the oldest_timestamp for the @@ -2285,16 +2285,6 @@ void WiredTigerKVEngine::haltOplogManager() { } } -bool WiredTigerKVEngine::isCacheUnderPressure(OperationContext* opCtx) const { - WiredTigerSession* session = WiredTigerRecoveryUnit::get(opCtx)->getSessionNoTxn(); - invariant(session); - - int64_t score = uassertStatusOK(WiredTigerUtil::getStatisticsValue( - session->getSession(), "statistics:", "", WT_STAT_CONN_CACHE_LOOKASIDE_SCORE)); - - return (score >= snapshotWindowParams.cachePressureThreshold.load()); -} - Timestamp WiredTigerKVEngine::getStableTimestamp() const { return Timestamp(_stableTimestamp.load()); } diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h index 4bebe95e99b..61b0589b617 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h @@ -273,8 +273,6 @@ public: bool supportsOplogStones() const final override; - bool isCacheUnderPressure(OperationContext* opCtx) const override; - bool supportsReadConcernMajority() const final; // wiredtiger specific diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_recovery_unit.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_recovery_unit.cpp index f9d9751d464..8d6e087a86f 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_recovery_unit.cpp +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_recovery_unit.cpp @@ -58,6 +58,8 @@ MONGO_FAIL_POINT_DEFINE(doUntimestampedWritesForIdempotencyTests); } // namespace +AtomicWord<std::int64_t> snapshotTooOldErrorCount{0}; + using Section = WiredTigerOperationStats::Section; std::map<int, std::pair<StringData, Section>> WiredTigerOperationStats::_statNameMap = { @@ -531,6 +533,9 @@ void WiredTigerRecoveryUnit::_txnOpen() { auto status = txnOpen.setReadSnapshot(_readAtTimestamp); if (!status.isOK() && status.code() == ErrorCodes::BadValue) { + // SnapshotTooOld errors indicate that PIT ops are failing to find an available + // snapshot at their specified atClusterTime. + snapshotTooOldErrorCount.addAndFetch(1); uasserted(ErrorCodes::SnapshotTooOld, str::stream() << "Read timestamp " << _readAtTimestamp.toString() << " is older than the oldest available timestamp."); diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_recovery_unit.h b/src/mongo/db/storage/wiredtiger/wiredtiger_recovery_unit.h index 373794582a2..d2e599181de 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_recovery_unit.h +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_recovery_unit.h @@ -51,6 +51,8 @@ namespace mongo { using RoundUpPreparedTimestamps = WiredTigerBeginTxnBlock::RoundUpPreparedTimestamps; using RoundUpReadTimestamp = WiredTigerBeginTxnBlock::RoundUpReadTimestamp; +extern AtomicWord<std::int64_t> snapshotTooOldErrorCount; + class BSONObjBuilder; class WiredTigerOperationStats final : public StorageStats { diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_recovery_unit_test.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_recovery_unit_test.cpp index 1f440d319b7..b50d4b79889 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_recovery_unit_test.cpp +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_recovery_unit_test.cpp @@ -263,48 +263,6 @@ TEST_F(WiredTigerRecoveryUnitTestFixture, NoOverlapReadSource) { ASSERT_TRUE(rs->findRecord(opCtx1, rid3, &unused)); } -TEST_F(WiredTigerRecoveryUnitTestFixture, CreateAndCheckForCachePressure) { - int time = 1; - - // Reconfigure the size of the cache to be very small so that building cache pressure is fast. - WiredTigerKVEngine* engine = harnessHelper->getEngine(); - std::string cacheSizeReconfig = "cache_size=1MB"; - ASSERT_EQ(engine->reconfigure(cacheSizeReconfig.c_str()), 0); - - OperationContext* opCtx = clientAndCtx1.second.get(); - std::unique_ptr<RecordStore> rs(harnessHelper->createRecordStore(opCtx, "a.b")); - - // Insert one document so that we can then update it in a loop to create cache pressure. - // Note: inserts will not create cache pressure. - WriteUnitOfWork wu(opCtx); - ASSERT_OK(ru1->setTimestamp(Timestamp(time++))); - std::string str = str::stream() << "foobarbaz"; - StatusWith<RecordId> ress = rs->insertRecord(opCtx, str.c_str(), str.size() + 1, Timestamp()); - ASSERT_OK(ress.getStatus()); - auto recordId = ress.getValue(); - wu.commit(); - - for (int j = 0; j < 1000; ++j) { - // Once we hit the cache pressure threshold, i.e. have successfully created cache pressure - // that is detectable, we are done. - if (engine->isCacheUnderPressure(opCtx)) { - invariant(j != 0); - break; - } - - try { - WriteUnitOfWork wuow(opCtx); - ASSERT_OK(ru1->setTimestamp(Timestamp(time++))); - std::string s = str::stream() - << "abcbcdcdedefefgfghghihijijkjklklmlmnmnomopopqpqrqrsrststutuv" << j; - ASSERT_OK(rs->updateRecord(opCtx, recordId, s.c_str(), s.size() + 1)); - wuow.commit(); - } catch (const DBException& ex) { - invariant(ex.toStatus().code() == ErrorCodes::WriteConflict); - } - } -} - TEST_F(WiredTigerRecoveryUnitTestFixture, LocalReadOnADocumentBeingPreparedWithoutIgnoringPreparedTriggersPrepareConflict) { // Prepare but don't commit a transaction diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp index d2187782a2b..2b8f2d667fe 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp @@ -38,7 +38,7 @@ #include "mongo/base/simple_string_data_comparator.h" #include "mongo/bson/bsonobjbuilder.h" #include "mongo/db/concurrency/write_conflict_exception.h" -#include "mongo/db/snapshot_window_options.h" +#include "mongo/db/snapshot_window_options_gen.h" #include "mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h" #include "mongo/db/storage/wiredtiger/wiredtiger_recovery_unit.h" #include "mongo/db/storage/wiredtiger/wiredtiger_session_cache.h" @@ -738,18 +738,12 @@ void WiredTigerUtil::appendSnapshotWindowSettings(WiredTigerKVEngine* engine, const unsigned currentAvailableSnapshotWindow = stableTimestamp.getSecs() - oldestTimestamp.getSecs(); - int64_t score = uassertStatusOK(WiredTigerUtil::getStatisticsValue( - session->getSession(), "statistics:", "", WT_STAT_CONN_CACHE_LOOKASIDE_SCORE)); - - auto totalNumberOfSnapshotTooOldErrors = snapshotWindowParams.snapshotTooOldErrorCount.load(); + auto totalNumberOfSnapshotTooOldErrors = snapshotTooOldErrorCount.load(); BSONObjBuilder settings(bob->subobjStart("snapshot-window-settings")); - settings.append("cache pressure percentage threshold", - snapshotWindowParams.cachePressureThreshold.load()); - settings.append("current cache pressure percentage", score); settings.append("total number of SnapshotTooOld errors", totalNumberOfSnapshotTooOldErrors); settings.append("minimum target snapshot window size in seconds", - snapshotWindowParams.minSnapshotHistoryWindowInSeconds.load()); + minSnapshotHistoryWindowInSeconds.load()); settings.append("current available snapshot window size in seconds", static_cast<int>(currentAvailableSnapshotWindow)); settings.append("latest majority snapshot timestamp available", diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_util.h b/src/mongo/db/storage/wiredtiger/wiredtiger_util.h index 72ccee41b4f..e1e1a190b28 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_util.h +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_util.h @@ -158,12 +158,9 @@ public: * that affect that window of maintained history. * * "snapshot-window-settings" : { - * "cache pressure percentage threshold" : <num>, - * "current cache pressure percentage" : <num>, * "total number of SnapshotTooOld errors" : <num>, - * "max target available snapshots window size in seconds" : <num>, - * "target available snapshots window size in seconds" : <num>, - * "current available snapshots window size in seconds" : <num>, + * "minimum target snapshot window size in seconds" : <num>, + * "current available snapshot window size in seconds" : <num>, * "latest majority snapshot timestamp available" : <num>, * "oldest majority snapshot timestamp available" : <num> * } |