diff options
author | Dianna Hohensee <dianna.hohensee@10gen.com> | 2019-07-01 10:01:15 -0400 |
---|---|---|
committer | Dianna Hohensee <dianna.hohensee@10gen.com> | 2019-07-09 13:51:20 -0400 |
commit | 1cccd2d05271c018b702bcc3e30a7516457a192a (patch) | |
tree | 9a7a375182a90e36be0ddda96e814e510f469716 | |
parent | 41b3ba1faa0841c7c189ecdae76bb1bf4a21c46c (diff) | |
download | mongo-1cccd2d05271c018b702bcc3e30a7516457a192a.tar.gz |
Revert "SERVER-36956 SnapshotTooOld errors will always increase the snapshot history window size"
This reverts commit 8899b34e1044b08aec7ad9f8546652456472702c.
(cherry picked from commit 8bb53a07a5c593d85b6229a2afe096b3e1efe21d)
23 files changed, 200 insertions, 196 deletions
diff --git a/jstests/noPassthrough/snapshotWindow_serverParameters.js b/jstests/noPassthrough/snapshotWindow_serverParameters.js index 3a05dc3354f..4b41954ed61 100644 --- a/jstests/noPassthrough/snapshotWindow_serverParameters.js +++ b/jstests/noPassthrough/snapshotWindow_serverParameters.js @@ -17,6 +17,17 @@ false /*hasUpperBound*/, "unused" /*upperOutOfBounds*/); + // Valid parameter values are in the range [0, 100]. + testNumericServerParameter("cachePressureThreshold", + true /*isStartupParameter*/, + true /*isRuntimeParameter*/, + 50 /*defaultValue*/, + 70 /*nonDefaultValidValue*/, + true /*hasLowerBound*/, + -1 /*lowerOutOfBounds*/, + true /*hasUpperBound*/, + 101 /*upperOutOfBounds*/); + // Valid parameter values are in the range (0, 1). testNumericServerParameter("snapshotWindowMultiplicativeDecrease", true /*isStartupParameter*/, @@ -32,7 +43,7 @@ testNumericServerParameter("snapshotWindowAdditiveIncreaseSeconds", true /*isStartupParameter*/, true /*isRuntimeParameter*/, - 1 /*defaultValue*/, + 2 /*defaultValue*/, 10 /*nonDefaultValidValue*/, true /*hasLowerBound*/, 0 /*lowerOutOfBounds*/, @@ -40,10 +51,10 @@ "unused" /*upperOutOfBounds*/); // Valid parameter values are in the range [1, infinity). - testNumericServerParameter("decreaseHistoryIfNotNeededPeriodSeconds", + testNumericServerParameter("checkCachePressurePeriodSeconds", true /*isStartupParameter*/, true /*isRuntimeParameter*/, - 15 /*defaultValue*/, + 5 /*defaultValue*/, 8 /*nonDefaultValidValue*/, true /*hasLowerBound*/, 0 /*lowerOutOfBounds*/, @@ -60,4 +71,15 @@ 0 /*lowerOutOfBounds*/, false /*hasUpperBound*/, "unused" /*upperOutOfBounds*/); + + // Valid parameter values are in the range [1, infinity). + testNumericServerParameter("minMillisBetweenSnapshotWindowDec", + true /*isStartupParameter*/, + true /*isRuntimeParameter*/, + 500 /*defaultValue*/, + 2 * 1000 /*nonDefaultValidValue*/, + true /*hasLowerBound*/, + 0 /*lowerOutOfBounds*/, + false /*hasUpperBound*/, + "unused" /*upperOutOfBounds*/); })(); diff --git a/src/mongo/db/db.cpp b/src/mongo/db/db.cpp index e6dd1dc014d..e496d8c60ac 100644 --- a/src/mongo/db/db.cpp +++ b/src/mongo/db/db.cpp @@ -615,7 +615,7 @@ ExitCode _initAndListen(int listenPort) { // release periodically in order to avoid storage cache pressure build up. if (storageEngine->supportsReadConcernSnapshot()) { startPeriodicThreadToAbortExpiredTransactions(serviceContext); - startPeriodicThreadToDecreaseSnapshotHistoryIfNotNeeded(serviceContext); + startPeriodicThreadToDecreaseSnapshotHistoryCachePressure(serviceContext); } // Set up the logical session cache diff --git a/src/mongo/db/periodic_runner_job_decrease_snapshot_cache_pressure.cpp b/src/mongo/db/periodic_runner_job_decrease_snapshot_cache_pressure.cpp index 8e4f6594bab..3a5a41b5ee3 100644 --- a/src/mongo/db/periodic_runner_job_decrease_snapshot_cache_pressure.cpp +++ b/src/mongo/db/periodic_runner_job_decrease_snapshot_cache_pressure.cpp @@ -43,7 +43,7 @@ namespace mongo { -void startPeriodicThreadToDecreaseSnapshotHistoryIfNotNeeded(ServiceContext* serviceContext) { +void startPeriodicThreadToDecreaseSnapshotHistoryCachePressure(ServiceContext* serviceContext) { // Enforce calling this function once, and only once. static bool firstCall = true; invariant(firstCall); @@ -53,7 +53,7 @@ void startPeriodicThreadToDecreaseSnapshotHistoryIfNotNeeded(ServiceContext* ser invariant(periodicRunner); PeriodicRunner::PeriodicJob job( - "startPeriodicThreadToDecreaseSnapshotHistoryIfNotNeeded", + "startPeriodicThreadToDecreaseSnapshotHistoryCachePressure", [](Client* client) { try { // The opCtx destructor handles unsetting itself from the Client. @@ -69,18 +69,18 @@ void startPeriodicThreadToDecreaseSnapshotHistoryIfNotNeeded(ServiceContext* ser } } }, - Seconds(snapshotWindowParams.decreaseHistoryIfNotNeededPeriodSeconds.load())); + Seconds(snapshotWindowParams.checkCachePressurePeriodSeconds.load())); auto handle = periodicRunner->makeJob(std::move(job)); handle->start(); - SnapshotWindowParams::observeDecreaseHistoryIfNotNeededPeriodSeconds + SnapshotWindowParams::observeCheckCachePressurePeriodSeconds .addObserver([handle = std::move(handle)](const auto& secs) { try { handle->setPeriod(Seconds(secs)); } catch (const DBException& ex) { log() << "Failed to update the period of the thread which decreases data history " - "target window size if there have been no new SnapshotTooOld errors." + "target window size if there is cache pressure." << ex.toStatus(); } }); diff --git a/src/mongo/db/periodic_runner_job_decrease_snapshot_cache_pressure.h b/src/mongo/db/periodic_runner_job_decrease_snapshot_cache_pressure.h index 1b6c2e65b18..fca8b832fbf 100644 --- a/src/mongo/db/periodic_runner_job_decrease_snapshot_cache_pressure.h +++ b/src/mongo/db/periodic_runner_job_decrease_snapshot_cache_pressure.h @@ -34,15 +34,13 @@ namespace mongo { class ServiceContext; /** - * Periodically checks whether there has been any storage engine cache pressure and SnapshotTooOld - * errors to determine whether the maintained snapshot history window target setting should be - * decreased. If there has been cache pressure and no new SnapshotTooOld errors in the last period, - * then the target window size will be decrease. Maintaining too much snapshot and write history can - * slow down the system. Runs once every decreaseHistoryIfNotNeededPeriodSeconds. + * Periodically checks for storage engine cache pressure to determine whether the maintained + * snapshot history window target setting should be decreased. Maintaining too much snapshot and + * write history can slow down the system. Runs once every checkCachePressurePeriodSeconds. * * This function should only ever be called once, during mongod server startup (db.cpp). * The PeriodicRunner will handle shutting down the job on shutdown, no extra handling necessary. */ -void startPeriodicThreadToDecreaseSnapshotHistoryIfNotNeeded(ServiceContext* serviceContext); +void startPeriodicThreadToDecreaseSnapshotHistoryCachePressure(ServiceContext* serviceContext); } // namespace mongo diff --git a/src/mongo/db/service_entry_point_common.cpp b/src/mongo/db/service_entry_point_common.cpp index 5c48ade9178..281198f62bb 100644 --- a/src/mongo/db/service_entry_point_common.cpp +++ b/src/mongo/db/service_entry_point_common.cpp @@ -860,7 +860,6 @@ void execCommandDatabase(OperationContext* opCtx, // of successful future PIT atClusterTime requests. auto engine = opCtx->getServiceContext()->getStorageEngine(); if (engine && engine->supportsReadConcernSnapshot()) { - SnapshotWindowUtil::incrementSnapshotTooOldErrorCount(); SnapshotWindowUtil::increaseTargetSnapshotWindowSize(opCtx); } } else { diff --git a/src/mongo/db/snapshot_window_options.h b/src/mongo/db/snapshot_window_options.h index 93b3de975c9..95683969ca0 100644 --- a/src/mongo/db/snapshot_window_options.h +++ b/src/mongo/db/snapshot_window_options.h @@ -64,42 +64,49 @@ struct SnapshotWindowParams { AtomicWord<int> targetSnapshotHistoryWindowInSeconds{ maxTargetSnapshotHistoryWindowInSeconds.load()}; + // cachePressureThreshold (startup & runtime server paramter, range [0, 100]). + // + // Dictates what percentage of cache in use is considered too high. This setting helps preempt + // storage cache pressure immobilizing the system. Attempts to increase + // targetSnapshotHistoryWindowInSeconds will be ignored when the cache pressure reaches this + // threshold. Additionally, a periodic task will decrease targetSnapshotHistoryWindowInSeconds + // when cache pressure exceeds the threshold. + AtomicWord<int> cachePressureThreshold{50}; + // snapshotWindowMultiplicativeDecrease (startup & runtime server paramter, range (0,1)). // - // Controls by what multiplier the target snapshot history window size setting is decreased - // when there is cache pressure. + // Controls by what multiplier the target snapshot history window setting is decreased when + // cache pressure becomes too high, per the cachePressureThreshold setting. AtomicDouble snapshotWindowMultiplicativeDecrease{0.75}; // snapshotWindowAdditiveIncreaseSeconds (startup & runtime server paramter, range 1+). // - // Controls by how much the target snapshot history window size setting is increased when we - // need to service older snapshots for global point-in-time reads. - AtomicWord<int> snapshotWindowAdditiveIncreaseSeconds{1}; + // Controls by how much the target snapshot history window setting is increased when cache + // pressure is OK, per cachePressureThreshold, and we need to service older snapshots for global + // point-in-time reads. + AtomicWord<int> snapshotWindowAdditiveIncreaseSeconds{2}; // minMillisBetweenSnapshotWindowInc (startup & runtime server paramter, range 0+). + // minMillisBetweenSnapshotWindowDec (startup & runtime server paramter, range 0+). // - // Controls how often attempting to increase the target snapshot window will have an effect. - // Multiple callers within minMillisBetweenSnapshotWindowInc will have the same effect as one. - // This protects the system because it takes time for the target snapshot window to affect the - // actual storage engine snapshot window. The stable timestamp must move forward for the window - // between it and oldest timestamp to grow or shrink. + // Controls how often attempting to increase/decrease the target snapshot window will have an + // effect. Multiple callers within minMillisBetweenSnapshotWindowInc will have the same effect + // as one. This protects the system because it takes time for the target snapshot window to + // affect the actual storage engine snapshot window. The stable timestamp must move forward for + // the window between it and oldest timestamp to grow or shrink. AtomicWord<int> minMillisBetweenSnapshotWindowInc{500}; + AtomicWord<int> minMillisBetweenSnapshotWindowDec{500}; - // decreaseHistoryIfNotNeededPeriodSeconds (startup & runtime server paramter, range 1+) + // checkCachePressurePeriodSeconds (startup & runtime server paramter, range 1+) // // Controls the period of the task that checks for cache pressure and decreases - // targetSnapshotHistoryWindowInSeconds if there has been pressure and no new SnapshotTooOld - // errors. - // - // This should not run very frequently. It is preferable to increase the window size, and cache - // pressure, rather than failing PIT reads. - AtomicWord<int> decreaseHistoryIfNotNeededPeriodSeconds{15}; - - static inline MutableObserverRegistry<decltype( - decreaseHistoryIfNotNeededPeriodSeconds)::WordType> - observeDecreaseHistoryIfNotNeededPeriodSeconds; + // targetSnapshotHistoryWindowInSeconds if the pressure is above cachePressureThreshold. The + // target window size setting must not be decreased too fast because time must be allowed for + // the storage engine to attempt to act on the new setting. + AtomicWord<int> checkCachePressurePeriodSeconds{5}; - AtomicWord<long long> snapshotTooOldErrorCount{0}; + static inline MutableObserverRegistry<decltype(checkCachePressurePeriodSeconds)::WordType> + observeCheckCachePressurePeriodSeconds; }; extern SnapshotWindowParams snapshotWindowParams; diff --git a/src/mongo/db/snapshot_window_options.idl b/src/mongo/db/snapshot_window_options.idl index 8f1095bbac6..0c10ef0985d 100644 --- a/src/mongo/db/snapshot_window_options.idl +++ b/src/mongo/db/snapshot_window_options.idl @@ -41,6 +41,14 @@ server_parameters: cpp_varname: "snapshotWindowParams.maxTargetSnapshotHistoryWindowInSeconds" validator: { gte: 0 } + cachePressureThreshold: + description: "Cache pressure threshold" + set_at: [ startup, runtime ] + cpp_varname: "snapshotWindowParams.cachePressureThreshold" + validator: + gte: 0 + lte: 100 + snapshotWindowMultiplicativeDecrease: description: "Snapshot window multiplicative decrease" set_at: [ startup, runtime ] @@ -61,9 +69,15 @@ server_parameters: cpp_varname: "snapshotWindowParams.minMillisBetweenSnapshotWindowInc" validator: { gte: 1 } - decreaseHistoryIfNotNeededPeriodSeconds: + minMillisBetweenSnapshotWindowDec: + description: "Minimum duration between snapshot window decrement, in milliseconds" + set_at: [ startup, runtime ] + cpp_varname: "snapshotWindowParams.minMillisBetweenSnapshotWindowDec" + validator: { gte: 1 } + + checkCachePressurePeriodSeconds: description: "Check cache pressure period, in seconds" set_at: [ startup, runtime ] - cpp_varname: "snapshotWindowParams.decreaseHistoryIfNotNeededPeriodSeconds" + cpp_varname: "snapshotWindowParams.checkCachePressurePeriodSeconds" validator: { gte: 1 } - on_update: std::ref(SnapshotWindowParams::observeDecreaseHistoryIfNotNeededPeriodSeconds) + on_update: std::ref(SnapshotWindowParams::observeCheckCachePressurePeriodSeconds) diff --git a/src/mongo/db/snapshot_window_util.cpp b/src/mongo/db/snapshot_window_util.cpp index eaaf84adc5b..1ed27ca417b 100644 --- a/src/mongo/db/snapshot_window_util.cpp +++ b/src/mongo/db/snapshot_window_util.cpp @@ -52,12 +52,25 @@ namespace SnapshotWindowUtil { // Adds concurrency control to increaseTargetSnapshotWindowSize() and // decreaseTargetSnapshotWindowSize(). They should not run concurrently with themselves or one // another, since they act on and modify the same storage parameters. Further guards the static -// variable "_snapshotWindowLastIncreasedAt" used in increaseTargetSnapshotWindowSize(). +// variables "_snapshotWindowLastDecreasedAt" and "_snapshotWindowLastIncreasedAt" used in +// increaseTargetSnapshotWindowSize() and decreaseSnapshowWindow(). stdx::mutex snapshotWindowMutex; namespace { -void _decreaseTargetSnapshotWindowSize(WithLock, OperationContext* opCtx) { +void _decreaseTargetSnapshotWindowSize(WithLock lock, OperationContext* opCtx) { + // Tracks the last time that the snapshot window was decreased so that it does not go down so + // fast that the system does not have time to react and reduce snapshot availability. + static Date_t _snapshotWindowLastDecreasedAt{Date_t::min()}; + + if (_snapshotWindowLastDecreasedAt > + (Date_t::now() - + Milliseconds(snapshotWindowParams.minMillisBetweenSnapshotWindowDec.load()))) { + // We have already decreased the window size in the last minMillisBetweenSnapshotWindowDec + // milliseconds. + return; + } + snapshotWindowParams.targetSnapshotHistoryWindowInSeconds.store( snapshotWindowParams.targetSnapshotHistoryWindowInSeconds.load() * snapshotWindowParams.snapshotWindowMultiplicativeDecrease.load()); @@ -67,6 +80,8 @@ void _decreaseTargetSnapshotWindowSize(WithLock, OperationContext* opCtx) { StorageEngine* engine = opCtx->getServiceContext()->getStorageEngine(); invariant(engine); engine->setOldestTimestampFromStable(); + + _snapshotWindowLastDecreasedAt = Date_t::now(); } } // namespace @@ -90,6 +105,23 @@ void increaseTargetSnapshotWindowSize(OperationContext* opCtx) { return; } + // If the cache pressure is already too high, we will not put more pressure on it by increasing + // the window size. + StorageEngine* engine = opCtx->getServiceContext()->getStorageEngine(); + if (engine && engine->isCacheUnderPressure(opCtx)) { + warning() << "Attempted to increase the time window of available snapshots for " + "point-in-time operations (readConcern level 'snapshot' or transactions), but " + "the storage engine cache pressure, per the cachePressureThreshold setting of " + "'" + << snapshotWindowParams.cachePressureThreshold.load() + << "', is too high to allow it to increase. If this happens frequently, consider " + "either increasing the cache pressure threshold or increasing the memory " + "available to the storage engine cache, in order to improve the success rate " + "or speed of point-in-time requests."; + _decreaseTargetSnapshotWindowSize(lock, opCtx); + return; + } + if (snapshotWindowParams.targetSnapshotHistoryWindowInSeconds.load() == snapshotWindowParams.maxTargetSnapshotHistoryWindowInSeconds.load()) { warning() << "Attempted to increase the time window of available snapshots for " @@ -119,28 +151,10 @@ void decreaseTargetSnapshotWindowSize(OperationContext* opCtx) { stdx::unique_lock<stdx::mutex> lock(snapshotWindowMutex); StorageEngine* engine = opCtx->getServiceContext()->getStorageEngine(); - if (engine) { - static auto lastInsertsCount = 0; - static auto lastSnapshotErrorCount = 0; - - auto currentInsertsCount = engine->getCacheOverflowTableInsertCount(opCtx); - auto currentSnapshotErrorCount = snapshotWindowParams.snapshotTooOldErrorCount.load(); - - // Only decrease the snapshot window size if there were writes to the cache overflow table - // and there has been no new SnapshotTooOld errors in the same time period. - if (currentInsertsCount > lastInsertsCount && - currentSnapshotErrorCount == lastSnapshotErrorCount) { - _decreaseTargetSnapshotWindowSize(lock, opCtx); - } - - lastInsertsCount = currentInsertsCount; - lastSnapshotErrorCount = currentSnapshotErrorCount; + if (engine && engine->isCacheUnderPressure(opCtx)) { + _decreaseTargetSnapshotWindowSize(lock, opCtx); } } -void incrementSnapshotTooOldErrorCount() { - snapshotWindowParams.snapshotTooOldErrorCount.addAndFetch(1); -} - } // namespace SnapshotWindowUtil } // namespace mongo diff --git a/src/mongo/db/snapshot_window_util.h b/src/mongo/db/snapshot_window_util.h index 0610a6cd92c..cd13ddd2112 100644 --- a/src/mongo/db/snapshot_window_util.h +++ b/src/mongo/db/snapshot_window_util.h @@ -40,9 +40,10 @@ class OperationContext; namespace SnapshotWindowUtil { /** - * Increases the setting that controls the window of time between stable_timestamp and + * Attempts to increase the setting that controls the window of time between stable_timestamp and * oldest_timestamp, in order to provide a greater range of available snapshots for point-in-time - * operations. This function will be called when server requests return SnapshotTooOld (or similar) + * operations. The window will not be increased, however, if the cache pressure is currently too + * high. This function will be called when server requests return SnapshotTooOld (or similar) * errors. Note that this will not immediately affect the oldest_timestamp. Rather, it affects * actions taken next time oldest_timestamp is updated, usually when the stable timestamp is * advanced. @@ -59,15 +60,12 @@ namespace SnapshotWindowUtil { void increaseTargetSnapshotWindowSize(OperationContext* opCtx); /** - * Decreases the target snapshot window size if there has been cache pressure and no new - * SnapshotTooOld errors as tracked via the incrementSnapshotTooOldErrorCount() below since the last - * time this function was called. - * - * We do not decrease the window size when there have been SnapshotTooOld errors recently, even if - * there is some cache pressure, because that will cause sharded transaction commands to fail. - * - * Note that this will not necessarily immediately affect the actual window size; rather, - * it affects actions taken whenever oldest_timestamp is updated, usually when the stable_timestamp + * Attempts to decrease (if not already zero) the setting that affects the size of the window of + * time between stable_timestamp and oldest_timestamp in order to reduce storage engine cache + * pressure. The window target will not be decreased, however, if the cache is not currently under + * pressure. Pressure can occur when too much history is being maintained for point-in-time + * snapshots. Note that this will not necessarily immediately affect the actual window size; rather, + * it affects actions taken whenever oldest_timestamp is updated, usually when the stable timestamp * is advanced. * * This will make one attempt to immediately adjust the window size if possible. @@ -76,15 +74,5 @@ void increaseTargetSnapshotWindowSize(OperationContext* opCtx); */ void decreaseTargetSnapshotWindowSize(OperationContext* opCtx); -/** - * Registers on a counter SnapshotTooOld errors encountered in the command layer. - * decreaseTargetSnapshotWindowSize() above uses the counter value: if there are any errors since - * the last decreaseTargetSnapshotWindowSize() call, then decreaseTargetSnapshotWindowSize() will - * choose not decrease the target snapshot window size setting. - * - * Concurrency safe, the internal counter is atomic. - */ -void incrementSnapshotTooOldErrorCount(); - } // namespace SnapshotWindowUtil } // namespace mongo diff --git a/src/mongo/db/snapshot_window_util_test.cpp b/src/mongo/db/snapshot_window_util_test.cpp index 3956a9f9b71..19e1b60c4a1 100644 --- a/src/mongo/db/snapshot_window_util_test.cpp +++ b/src/mongo/db/snapshot_window_util_test.cpp @@ -69,7 +69,9 @@ TEST_F(SnapshotWindowTest, DecreaseAndIncreaseSnapshotWindow) { snapshotWindowParams.targetSnapshotHistoryWindowInSeconds.store(100); // Lower the time enforced between function calls to speed up testing. + // Dec must match Inc b/c increaseTargetWindowSize can call into decreaseTargetWindowSize. snapshotWindowParams.minMillisBetweenSnapshotWindowInc.store(100); + snapshotWindowParams.minMillisBetweenSnapshotWindowDec.store(100); // Stabilize for the test so we know that we are testing things as expected. snapshotWindowParams.snapshotWindowAdditiveIncreaseSeconds.store(2); @@ -84,117 +86,78 @@ TEST_F(SnapshotWindowTest, DecreaseAndIncreaseSnapshotWindow) { snapshotWindowParams.snapshotWindowMultiplicativeDecrease.load(); auto windowAdditiveIncrease = snapshotWindowParams.snapshotWindowAdditiveIncreaseSeconds.load(); + auto cachePressureThreshold = snapshotWindowParams.cachePressureThreshold.load(); auto minTimeBetweenInc = snapshotWindowParams.minMillisBetweenSnapshotWindowInc.load(); /** - * Test that trying to decrease the window size FAILS when there have been no writes to the - * cache overflow table. + * Test that decreasing the size succeeds when cache pressure is ABOVE the threshold */ - decreaseTargetSnapshotWindowSize(_opCtx.get()); - auto snapshotWindowSecondsOne = - snapshotWindowParams.targetSnapshotHistoryWindowInSeconds.load(); - - ASSERT_EQ(snapshotWindowSeconds, snapshotWindowSecondsOne); - - /** - * Test that trying to decrease the window size SUCCEEDS when there have been writes to the - * cache overflow table. - */ - - engine->setCacheOverflowTableInsertCountForTest(1); + engine->setCachePressureForTest(cachePressureThreshold + 5); decreaseTargetSnapshotWindowSize(_opCtx.get()); - auto snapshotWindowSecondsTwo = + auto snapshotWindowSecondsOne = snapshotWindowParams.targetSnapshotHistoryWindowInSeconds.load(); - ASSERT_GT(snapshotWindowSecondsOne, snapshotWindowSecondsTwo); - ASSERT_EQ(snapshotWindowSecondsTwo, + ASSERT_GT(snapshotWindowSeconds, snapshotWindowSecondsOne); + ASSERT_EQ(snapshotWindowSecondsOne, static_cast<int>(snapshotWindowSeconds * windowMultiplicativeDecrease)); /** - * Test that trying to decrease the window size FAILS when there have been writes to the - * cache overflow table AND SnapshotTooOld errors have occurred. - */ - - engine->setCacheOverflowTableInsertCountForTest(2); - incrementSnapshotTooOldErrorCount(); - - decreaseTargetSnapshotWindowSize(_opCtx.get()); - auto snapshotWindowSecondsThree = - snapshotWindowParams.targetSnapshotHistoryWindowInSeconds.load(); - - ASSERT_EQ(snapshotWindowSecondsTwo, snapshotWindowSecondsThree); - - /** - * Now test again that decreasing the size SUCCEEDS when there have been writes to the cache - * overflow table again (without any further SnapshotTooOld errors). + * Test that increasing the size SUCCEEDS when the cache pressure is BELOW the threshold. */ - engine->setCacheOverflowTableInsertCountForTest(3); - - decreaseTargetSnapshotWindowSize(_opCtx.get()); - auto snapshotWindowSecondsFour = - snapshotWindowParams.targetSnapshotHistoryWindowInSeconds.load(); - - ASSERT_GT(snapshotWindowSecondsThree, snapshotWindowSecondsFour); - ASSERT_EQ(snapshotWindowSecondsFour, - static_cast<int>(snapshotWindowSecondsThree * windowMultiplicativeDecrease)); - - /** - * Test that increasing the size SUCCEEDS. - */ + engine->setCachePressureForTest(cachePressureThreshold - 5); increaseTargetSnapshotWindowSize(_opCtx.get()); - auto snapshotWindowSecondsFive = + auto snapshotWindowSecondsTwo = snapshotWindowParams.targetSnapshotHistoryWindowInSeconds.load(); - ASSERT_EQ(snapshotWindowSecondsFive, snapshotWindowSecondsFour + windowAdditiveIncrease); + ASSERT_EQ(snapshotWindowSecondsTwo, snapshotWindowSecondsOne + windowAdditiveIncrease); /** - * Test that increasing the size SUCCEEDS even when there have been writes to the cache overflow - * table. + * Test that increasing the size FAILS when the cache pressure is ABOVE the threshold, and + * instead this causes the size to be decreased. */ + engine->setCachePressureForTest(cachePressureThreshold + 5); + // Sleep for a time because increaseTargetSnapshotWindowSize() enforces a wait time between // updates. sleepmillis(2 * minTimeBetweenInc); - engine->setCacheOverflowTableInsertCountForTest(4); - increaseTargetSnapshotWindowSize(_opCtx.get()); - auto snapshotWindowSecondsSix = + auto snapshotWindowSecondsThree = snapshotWindowParams.targetSnapshotHistoryWindowInSeconds.load(); - ASSERT_EQ(snapshotWindowSecondsSix, snapshotWindowSecondsFive + windowAdditiveIncrease); + ASSERT_EQ(snapshotWindowSecondsThree, + static_cast<int>(snapshotWindowSecondsTwo * windowMultiplicativeDecrease)); + + engine->setCachePressureForTest(cachePressureThreshold - 5); /** * Test that the size cannot be increased above the maximum size. */ - // Bump up the additive increase to make this run faster. - snapshotWindowParams.snapshotWindowAdditiveIncreaseSeconds.store(9); - windowAdditiveIncrease = snapshotWindowParams.snapshotWindowAdditiveIncreaseSeconds.load(); - // Integers round down, so add 1 to make sure it reaches the max. int numIncreasesToReachMax = - (maxTargetSnapshotWindowSeconds - snapshotWindowSecondsSix) / windowAdditiveIncrease + 1; + (maxTargetSnapshotWindowSeconds - snapshotWindowSecondsThree) / windowAdditiveIncrease + 1; for (int i = 0; i < numIncreasesToReachMax; ++i) { sleepmillis(2 * minTimeBetweenInc); increaseTargetSnapshotWindowSize(_opCtx.get()); } // Should be at max. - auto snapshotWindowSecondsSeven = + auto snapshotWindowSecondsFour = snapshotWindowParams.targetSnapshotHistoryWindowInSeconds.load(); - ASSERT_EQ(snapshotWindowSecondsSeven, maxTargetSnapshotWindowSeconds); + ASSERT_EQ(snapshotWindowSecondsFour, maxTargetSnapshotWindowSeconds); // An attempt to increase beyond max should have no effect. sleepmillis(2 * minTimeBetweenInc); increaseTargetSnapshotWindowSize(_opCtx.get()); - auto snapshotWindowSecondsEight = + auto snapshotWindowSecondsFive = snapshotWindowParams.targetSnapshotHistoryWindowInSeconds.load(); - ASSERT_EQ(snapshotWindowSecondsEight, maxTargetSnapshotWindowSeconds); + ASSERT_EQ(snapshotWindowSecondsFive, maxTargetSnapshotWindowSeconds); } } // namespace diff --git a/src/mongo/db/storage/biggie/biggie_kv_engine.cpp b/src/mongo/db/storage/biggie/biggie_kv_engine.cpp index 6c85b99f1a1..5419a88535b 100644 --- a/src/mongo/db/storage/biggie/biggie_kv_engine.cpp +++ b/src/mongo/db/storage/biggie/biggie_kv_engine.cpp @@ -47,7 +47,7 @@ mongo::RecoveryUnit* KVEngine::newRecoveryUnit() { return new RecoveryUnit(this, nullptr); } -void KVEngine::setCacheOverflowTableInsertCountForTest(int insertCount) { +void KVEngine::setCachePressureForTest(int pressure) { // TODO : implement. } diff --git a/src/mongo/db/storage/biggie/biggie_kv_engine.h b/src/mongo/db/storage/biggie/biggie_kv_engine.h index 393b0cc7c2b..09b2b3a7ae4 100644 --- a/src/mongo/db/storage/biggie/biggie_kv_engine.h +++ b/src/mongo/db/storage/biggie/biggie_kv_engine.h @@ -106,11 +106,11 @@ public: return true; } - virtual int64_t getCacheOverflowTableInsertCount(OperationContext* opCtx) const override { - return 0; + virtual bool isCacheUnderPressure(OperationContext* opCtx) const override { + return false; } - virtual void setCacheOverflowTableInsertCountForTest(int insertCount) override; + virtual void setCachePressureForTest(int pressure) override; virtual int64_t getIdentSize(OperationContext* opCtx, StringData ident) { return 0; diff --git a/src/mongo/db/storage/devnull/devnull_kv_engine.cpp b/src/mongo/db/storage/devnull/devnull_kv_engine.cpp index e9437a18908..6ae745c5587 100644 --- a/src/mongo/db/storage/devnull/devnull_kv_engine.cpp +++ b/src/mongo/db/storage/devnull/devnull_kv_engine.cpp @@ -261,13 +261,13 @@ SortedDataInterface* DevNullKVEngine::getSortedDataInterface(OperationContext* o return new DevNullSortedDataInterface(); } -int64_t DevNullKVEngine::getCacheOverflowTableInsertCount(OperationContext* opCtx) const { - return _overflowTableInsertCountForTest; +bool DevNullKVEngine::isCacheUnderPressure(OperationContext* opCtx) const { + return (_cachePressureForTest >= snapshotWindowParams.cachePressureThreshold.load()); } -void DevNullKVEngine::setCacheOverflowTableInsertCountForTest(int insertsCount) { - invariant(insertsCount >= 0); - _overflowTableInsertCountForTest = insertsCount; +void DevNullKVEngine::setCachePressureForTest(int pressure) { + invariant(pressure >= 0 && pressure <= 100); + _cachePressureForTest = pressure; } StatusWith<std::vector<std::string>> DevNullKVEngine::beginNonBlockingBackup( diff --git a/src/mongo/db/storage/devnull/devnull_kv_engine.h b/src/mongo/db/storage/devnull/devnull_kv_engine.h index 4fb60668149..59a99505e3e 100644 --- a/src/mongo/db/storage/devnull/devnull_kv_engine.h +++ b/src/mongo/db/storage/devnull/devnull_kv_engine.h @@ -98,9 +98,9 @@ public: return true; } - virtual int64_t getCacheOverflowTableInsertCount(OperationContext* opCtx) const override; + virtual bool isCacheUnderPressure(OperationContext* opCtx) const override; - virtual void setCacheOverflowTableInsertCountForTest(int insertCount) override; + virtual void setCachePressureForTest(int pressure) override; virtual int64_t getIdentSize(OperationContext* opCtx, StringData ident) { return 1; @@ -155,6 +155,6 @@ public: private: std::shared_ptr<void> _catalogInfo; - int _overflowTableInsertCountForTest; + int _cachePressureForTest; }; } diff --git a/src/mongo/db/storage/kv/kv_engine.h b/src/mongo/db/storage/kv/kv_engine.h index 2c0e56a0c33..20eb9f93b83 100644 --- a/src/mongo/db/storage/kv/kv_engine.h +++ b/src/mongo/db/storage/kv/kv_engine.h @@ -334,16 +334,16 @@ public: virtual void setOldestTimestamp(Timestamp newOldestTimestamp, bool force) {} /** - * See 'StorageEngine::getCacheOverflowTableInsertCount' + * See `StorageEngine::isCacheUnderPressure()` */ - virtual int64_t getCacheOverflowTableInsertCount(OperationContext* opCtx) const { - return 0; + virtual bool isCacheUnderPressure(OperationContext* opCtx) const { + return false; } /** - * See 'StorageEngine::setCacheOverflowTableInsertCountForTest()' + * See 'StorageEngine::setCachePressureForTest()' */ - virtual void setCacheOverflowTableInsertCountForTest(int insertCount) {} + virtual void setCachePressureForTest(int pressure) {} /** * See `StorageEngine::supportsRecoverToStableTimestamp` diff --git a/src/mongo/db/storage/storage_engine.h b/src/mongo/db/storage/storage_engine.h index 0ef67b009ed..3d1fe86e120 100644 --- a/src/mongo/db/storage/storage_engine.h +++ b/src/mongo/db/storage/storage_engine.h @@ -453,19 +453,21 @@ public: OldestActiveTransactionTimestampCallback callback){}; /** - * Retrieves the number of inserts done to the cache overflow table. Writes to that table only - * occur when the read/write cache size exceeds the allotted in-memory cache capacity and must - * write to disk, which is slow. Tracking this value over time is indicative of cache pressure. + * Indicates whether the storage engine cache is under pressure. + * + * Retrieves a cache pressure value in the range [0, 100] from the storage engine, and compares + * it against storageGlobalParams.cachePressureThreshold, a dynamic server parameter, to + * determine whether cache pressure is too high. */ - virtual int64_t getCacheOverflowTableInsertCount(OperationContext* opCtx) const { - return 0; + virtual bool isCacheUnderPressure(OperationContext* opCtx) const { + return false; } /** - * For unit tests only. Sets the counter for the number of cache overflow table inserts that the - * getCacheOverflowTableInsertCount() function above returns. + * For unit tests only. Sets the cache pressure value with which isCacheUnderPressure() + * evalutates to 'pressure'. */ - virtual void setCacheOverflowTableInsertCountForTest(int insertCount) {} + virtual void setCachePressureForTest(int pressure) {} /** * Notifies the storage engine that a replication batch has completed. diff --git a/src/mongo/db/storage/storage_engine_impl.cpp b/src/mongo/db/storage/storage_engine_impl.cpp index 5977640eb5a..812981c7b56 100644 --- a/src/mongo/db/storage/storage_engine_impl.cpp +++ b/src/mongo/db/storage/storage_engine_impl.cpp @@ -730,12 +730,12 @@ void StorageEngineImpl::setOldestActiveTransactionTimestampCallback( _engine->setOldestActiveTransactionTimestampCallback(callback); } -int64_t StorageEngineImpl::getCacheOverflowTableInsertCount(OperationContext* opCtx) const { - return _engine->getCacheOverflowTableInsertCount(opCtx); +bool StorageEngineImpl::isCacheUnderPressure(OperationContext* opCtx) const { + return _engine->isCacheUnderPressure(opCtx); } -void StorageEngineImpl::setCacheOverflowTableInsertCountForTest(int insertCount) { - return _engine->setCacheOverflowTableInsertCountForTest(insertCount); +void StorageEngineImpl::setCachePressureForTest(int pressure) { + return _engine->setCachePressureForTest(pressure); } bool StorageEngineImpl::supportsRecoverToStableTimestamp() const { diff --git a/src/mongo/db/storage/storage_engine_impl.h b/src/mongo/db/storage/storage_engine_impl.h index accc1bfc5f3..f934035c162 100644 --- a/src/mongo/db/storage/storage_engine_impl.h +++ b/src/mongo/db/storage/storage_engine_impl.h @@ -125,9 +125,9 @@ public: virtual void setOldestActiveTransactionTimestampCallback( StorageEngine::OldestActiveTransactionTimestampCallback) override; - virtual int64_t getCacheOverflowTableInsertCount(OperationContext* opCtx) const override; + virtual bool isCacheUnderPressure(OperationContext* opCtx) const override; - virtual void setCacheOverflowTableInsertCountForTest(int insertCount) override; + virtual void setCachePressureForTest(int pressure) override; virtual bool supportsRecoverToStableTimestamp() const override; diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp index 5b12f859060..e6245a14533 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp @@ -1933,14 +1933,14 @@ void WiredTigerKVEngine::replicationBatchIsComplete() const { _oplogManager->triggerJournalFlush(); } -int64_t WiredTigerKVEngine::getCacheOverflowTableInsertCount(OperationContext* opCtx) const { +bool WiredTigerKVEngine::isCacheUnderPressure(OperationContext* opCtx) const { WiredTigerSession* session = WiredTigerRecoveryUnit::get(opCtx)->getSessionNoTxn(); invariant(session); - int64_t insertCount = uassertStatusOK(WiredTigerUtil::getStatisticsValueAs<int64_t>( - session->getSession(), "statistics:", "", WT_STAT_CONN_CACHE_LOOKASIDE_INSERT)); + int64_t score = uassertStatusOK(WiredTigerUtil::getStatisticsValueAs<int64_t>( + session->getSession(), "statistics:", "", WT_STAT_CONN_CACHE_LOOKASIDE_SCORE)); - return insertCount; + return (score >= snapshotWindowParams.cachePressureThreshold.load()); } Timestamp WiredTigerKVEngine::getStableTimestamp() const { diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h index 10458bd02e4..ae0fffd31a5 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h @@ -246,7 +246,7 @@ public: */ void replicationBatchIsComplete() const override; - int64_t getCacheOverflowTableInsertCount(OperationContext* opCtx) const override; + bool isCacheUnderPressure(OperationContext* opCtx) const override; bool supportsReadConcernMajority() const final; diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_recovery_unit_test.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_recovery_unit_test.cpp index 7e97f110190..e45d693b12c 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_recovery_unit_test.cpp +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_recovery_unit_test.cpp @@ -192,11 +192,10 @@ TEST_F(WiredTigerRecoveryUnitTestFixture, CreateAndCheckForCachePressure) { auto recordId = ress.getValue(); wu.commit(); - auto startOverflowInsertsCount = engine->getCacheOverflowTableInsertCount(opCtx); for (int j = 0; j < 1000; ++j) { - // We know that we have successfully created cache pressure when the in-memory cache - // overflows to disk by writing to the cache overflow table (the WT lookaside table). - if (engine->getCacheOverflowTableInsertCount(opCtx) > startOverflowInsertsCount) { + // Once we hit the cache pressure threshold, i.e. have successfully created cache pressure + // that is detectable, we are done. + if (engine->isCacheUnderPressure(opCtx)) { invariant(j != 0); break; } diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp index 9e38db35291..9d51217f1d1 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp @@ -688,15 +688,13 @@ void WiredTigerUtil::appendSnapshotWindowSettings(WiredTigerKVEngine* engine, const unsigned currentAvailableSnapshotWindow = stableTimestamp.getSecs() - oldestTimestamp.getSecs(); - int64_t overflowTableInsertCount = - uassertStatusOK(WiredTigerUtil::getStatisticsValueAs<int64_t>( - session->getSession(), "statistics:", "", WT_STAT_CONN_CACHE_LOOKASIDE_INSERT)); - long long totalNumberOfSnapshotTooOldErrors = - snapshotWindowParams.snapshotTooOldErrorCount.load(); + int64_t score = uassertStatusOK(WiredTigerUtil::getStatisticsValueAs<int64_t>( + session->getSession(), "statistics:", "", WT_STAT_CONN_CACHE_LOOKASIDE_SCORE)); BSONObjBuilder settings(bob->subobjStart("snapshot-window-settings")); - settings.append("total number of cache overflow disk writes", overflowTableInsertCount); - settings.append("total number of SnapshotTooOld errors", totalNumberOfSnapshotTooOldErrors); + settings.append("cache pressure percentage threshold", + snapshotWindowParams.cachePressureThreshold.load()); + settings.append("current cache pressure percentage", score); settings.append("max target available snapshots window size in seconds", snapshotWindowParams.maxTargetSnapshotHistoryWindowInSeconds.load()); settings.append("target available snapshots window size in seconds", diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_util.h b/src/mongo/db/storage/wiredtiger/wiredtiger_util.h index 217c640cd4e..07764f2dd40 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_util.h +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_util.h @@ -158,8 +158,8 @@ public: * that affect that window of maintained history. * * "snapshot-window-settings" : { - * "total number of cache overflow disk writes", - * "total number of SnapshotTooOld errors", + * "cache pressure percentage threshold" : <num>, + * "current cache pressure percentage" : <num>, * "max target available snapshots window size in seconds" : <num>, * "target available snapshots window size in seconds" : <num>, * "current available snapshots window size in seconds" : <num>, |