diff options
25 files changed, 209 insertions, 207 deletions
diff --git a/jstests/noPassthrough/snapshotWindow_serverParameters.js b/jstests/noPassthrough/snapshotWindow_serverParameters.js index 4b41954ed61..3a05dc3354f 100644 --- a/jstests/noPassthrough/snapshotWindow_serverParameters.js +++ b/jstests/noPassthrough/snapshotWindow_serverParameters.js @@ -17,17 +17,6 @@ false /*hasUpperBound*/, "unused" /*upperOutOfBounds*/); - // Valid parameter values are in the range [0, 100]. - testNumericServerParameter("cachePressureThreshold", - true /*isStartupParameter*/, - true /*isRuntimeParameter*/, - 50 /*defaultValue*/, - 70 /*nonDefaultValidValue*/, - true /*hasLowerBound*/, - -1 /*lowerOutOfBounds*/, - true /*hasUpperBound*/, - 101 /*upperOutOfBounds*/); - // Valid parameter values are in the range (0, 1). testNumericServerParameter("snapshotWindowMultiplicativeDecrease", true /*isStartupParameter*/, @@ -43,7 +32,7 @@ testNumericServerParameter("snapshotWindowAdditiveIncreaseSeconds", true /*isStartupParameter*/, true /*isRuntimeParameter*/, - 2 /*defaultValue*/, + 1 /*defaultValue*/, 10 /*nonDefaultValidValue*/, true /*hasLowerBound*/, 0 /*lowerOutOfBounds*/, @@ -51,10 +40,10 @@ "unused" /*upperOutOfBounds*/); // Valid parameter values are in the range [1, infinity). - testNumericServerParameter("checkCachePressurePeriodSeconds", + testNumericServerParameter("decreaseHistoryIfNotNeededPeriodSeconds", true /*isStartupParameter*/, true /*isRuntimeParameter*/, - 5 /*defaultValue*/, + 15 /*defaultValue*/, 8 /*nonDefaultValidValue*/, true /*hasLowerBound*/, 0 /*lowerOutOfBounds*/, @@ -71,15 +60,4 @@ 0 /*lowerOutOfBounds*/, false /*hasUpperBound*/, "unused" /*upperOutOfBounds*/); - - // Valid parameter values are in the range [1, infinity). - testNumericServerParameter("minMillisBetweenSnapshotWindowDec", - true /*isStartupParameter*/, - true /*isRuntimeParameter*/, - 500 /*defaultValue*/, - 2 * 1000 /*nonDefaultValidValue*/, - true /*hasLowerBound*/, - 0 /*lowerOutOfBounds*/, - false /*hasUpperBound*/, - "unused" /*upperOutOfBounds*/); })(); diff --git a/src/mongo/db/db.cpp b/src/mongo/db/db.cpp index 4a7f4c39fe6..f3709e08328 100644 --- a/src/mongo/db/db.cpp +++ b/src/mongo/db/db.cpp @@ -615,7 +615,7 @@ ExitCode _initAndListen(int listenPort) { // release periodically in order to avoid storage cache pressure build up. if (storageEngine->supportsReadConcernSnapshot()) { startPeriodicThreadToAbortExpiredTransactions(serviceContext); - startPeriodicThreadToDecreaseSnapshotHistoryCachePressure(serviceContext); + startPeriodicThreadToDecreaseSnapshotHistoryIfNotNeeded(serviceContext); } // Set up the logical session cache diff --git a/src/mongo/db/periodic_runner_job_decrease_snapshot_cache_pressure.cpp b/src/mongo/db/periodic_runner_job_decrease_snapshot_cache_pressure.cpp index a79f39ae68d..8e4f6594bab 100644 --- a/src/mongo/db/periodic_runner_job_decrease_snapshot_cache_pressure.cpp +++ b/src/mongo/db/periodic_runner_job_decrease_snapshot_cache_pressure.cpp @@ -43,7 +43,7 @@ namespace mongo { -void startPeriodicThreadToDecreaseSnapshotHistoryCachePressure(ServiceContext* serviceContext) { +void startPeriodicThreadToDecreaseSnapshotHistoryIfNotNeeded(ServiceContext* serviceContext) { // Enforce calling this function once, and only once. static bool firstCall = true; invariant(firstCall); @@ -53,7 +53,7 @@ void startPeriodicThreadToDecreaseSnapshotHistoryCachePressure(ServiceContext* s invariant(periodicRunner); PeriodicRunner::PeriodicJob job( - "startPeriodicThreadToDecreaseSnapshotHistoryCachePressure", + "startPeriodicThreadToDecreaseSnapshotHistoryIfNotNeeded", [](Client* client) { try { // The opCtx destructor handles unsetting itself from the Client. @@ -69,17 +69,18 @@ void startPeriodicThreadToDecreaseSnapshotHistoryCachePressure(ServiceContext* s } } }, - Seconds(snapshotWindowParams.checkCachePressurePeriodSeconds.load())); + Seconds(snapshotWindowParams.decreaseHistoryIfNotNeededPeriodSeconds.load())); auto handle = periodicRunner->makeJob(std::move(job)); handle->start(); - SnapshotWindowParams::observeCheckCachePressurePeriodSeconds + SnapshotWindowParams::observeDecreaseHistoryIfNotNeededPeriodSeconds .addObserver([handle = std::move(handle)](const auto& secs) { try { handle->setPeriod(Seconds(secs)); } catch (const DBException& ex) { - log() << "Failed to update period of thread which checks snapshot cache pressure " + log() << "Failed to update the period of the thread which decreases data history " + "target window size if there have been no new SnapshotTooOld errors." << ex.toStatus(); } }); diff --git a/src/mongo/db/periodic_runner_job_decrease_snapshot_cache_pressure.h b/src/mongo/db/periodic_runner_job_decrease_snapshot_cache_pressure.h index fca8b832fbf..1b6c2e65b18 100644 --- a/src/mongo/db/periodic_runner_job_decrease_snapshot_cache_pressure.h +++ b/src/mongo/db/periodic_runner_job_decrease_snapshot_cache_pressure.h @@ -34,13 +34,15 @@ namespace mongo { class ServiceContext; /** - * Periodically checks for storage engine cache pressure to determine whether the maintained - * snapshot history window target setting should be decreased. Maintaining too much snapshot and - * write history can slow down the system. Runs once every checkCachePressurePeriodSeconds. + * Periodically checks whether there has been any storage engine cache pressure and SnapshotTooOld + * errors to determine whether the maintained snapshot history window target setting should be + * decreased. If there has been cache pressure and no new SnapshotTooOld errors in the last period, + * then the target window size will be decrease. Maintaining too much snapshot and write history can + * slow down the system. Runs once every decreaseHistoryIfNotNeededPeriodSeconds. * * This function should only ever be called once, during mongod server startup (db.cpp). * The PeriodicRunner will handle shutting down the job on shutdown, no extra handling necessary. */ -void startPeriodicThreadToDecreaseSnapshotHistoryCachePressure(ServiceContext* serviceContext); +void startPeriodicThreadToDecreaseSnapshotHistoryIfNotNeeded(ServiceContext* serviceContext); } // namespace mongo diff --git a/src/mongo/db/service_entry_point_common.cpp b/src/mongo/db/service_entry_point_common.cpp index 6fb6df55378..f9c38575808 100644 --- a/src/mongo/db/service_entry_point_common.cpp +++ b/src/mongo/db/service_entry_point_common.cpp @@ -836,6 +836,7 @@ void execCommandDatabase(OperationContext* opCtx, // of successful future PIT atClusterTime requests. auto engine = opCtx->getServiceContext()->getStorageEngine(); if (engine && engine->supportsReadConcernSnapshot()) { + SnapshotWindowUtil::incrementSnapshotTooOldErrorCount(); SnapshotWindowUtil::increaseTargetSnapshotWindowSize(opCtx); } } else { diff --git a/src/mongo/db/snapshot_window_options.h b/src/mongo/db/snapshot_window_options.h index 683c48c9253..93b3de975c9 100644 --- a/src/mongo/db/snapshot_window_options.h +++ b/src/mongo/db/snapshot_window_options.h @@ -36,9 +36,9 @@ namespace mongo { /** - * This is a collection of parameters that affect how much snapshot history the storage engine will - * maintain to support point-in-time transactions (read or write). This is referred to as the - * snapshot window. + * These are parameters that affect how much snapshot history the storage engine will keep to + * support point-in-time transactions (read or write). This is referred to as the snapshot window. + * The window is between the stable timestamp and the oldest timestamp. */ struct SnapshotWindowParams { @@ -59,53 +59,47 @@ struct SnapshotWindowParams { // Note that this is the history window we attempt to maintain, but our current system state may // not always reflect it: the window can only change as more writes come in, so it can take time // for the actual window size to catch up with a change. This value guides actions whenever the - // system goes to update the oldest_timestamp value. + // system goes to update the oldest_timestamp value (usually when the stable_timestamp is + // updated). AtomicWord<int> targetSnapshotHistoryWindowInSeconds{ maxTargetSnapshotHistoryWindowInSeconds.load()}; - // cachePressureThreshold (startup & runtime server paramter, range [0, 100]). - // - // Dictates what percentage of cache in use is considered too high. This setting helps preempt - // storage cache pressure immobilizing the system. Attempts to increase - // targetSnapshotHistoryWindowInSeconds will be ignored when the cache pressure reaches this - // threshold. Additionally, a periodic task will decrease targetSnapshotHistoryWindowInSeconds - // when cache pressure exceeds the threshold. - AtomicWord<int> cachePressureThreshold{50}; - // snapshotWindowMultiplicativeDecrease (startup & runtime server paramter, range (0,1)). // - // Controls by what multiplier the target snapshot history window setting is decreased when - // cache pressure becomes too high, per the cachePressureThreshold setting. + // Controls by what multiplier the target snapshot history window size setting is decreased + // when there is cache pressure. AtomicDouble snapshotWindowMultiplicativeDecrease{0.75}; // snapshotWindowAdditiveIncreaseSeconds (startup & runtime server paramter, range 1+). // - // Controls by how much the target snapshot history window setting is increased when cache - // pressure is OK, per cachePressureThreshold, and we need to service older snapshots for global - // point-in-time reads. - AtomicWord<int> snapshotWindowAdditiveIncreaseSeconds{2}; + // Controls by how much the target snapshot history window size setting is increased when we + // need to service older snapshots for global point-in-time reads. + AtomicWord<int> snapshotWindowAdditiveIncreaseSeconds{1}; // minMillisBetweenSnapshotWindowInc (startup & runtime server paramter, range 0+). - // minMillisBetweenSnapshotWindowDec (startup & runtime server paramter, range 0+). // - // Controls how often attempting to increase/decrease the target snapshot window will have an - // effect. Multiple callers within minMillisBetweenSnapshotWindowInc will have the same effect - // as one. This protects the system because it takes time for the target snapshot window to - // affect the actual storage engine snapshot window. The stable timestamp must move forward for - // the window between it and oldest timestamp to grow or shrink. + // Controls how often attempting to increase the target snapshot window will have an effect. + // Multiple callers within minMillisBetweenSnapshotWindowInc will have the same effect as one. + // This protects the system because it takes time for the target snapshot window to affect the + // actual storage engine snapshot window. The stable timestamp must move forward for the window + // between it and oldest timestamp to grow or shrink. AtomicWord<int> minMillisBetweenSnapshotWindowInc{500}; - AtomicWord<int> minMillisBetweenSnapshotWindowDec{500}; - // checkCachePressurePeriodSeconds (startup & runtime server paramter, range 1+) + // decreaseHistoryIfNotNeededPeriodSeconds (startup & runtime server paramter, range 1+) // // Controls the period of the task that checks for cache pressure and decreases - // targetSnapshotHistoryWindowInSeconds if the pressure is above cachePressureThreshold. The - // target window size setting must not be decreased too fast because time must be allowed for - // the storage engine to attempt to act on the new setting. - AtomicWord<int> checkCachePressurePeriodSeconds{5}; + // targetSnapshotHistoryWindowInSeconds if there has been pressure and no new SnapshotTooOld + // errors. + // + // This should not run very frequently. It is preferable to increase the window size, and cache + // pressure, rather than failing PIT reads. + AtomicWord<int> decreaseHistoryIfNotNeededPeriodSeconds{15}; + + static inline MutableObserverRegistry<decltype( + decreaseHistoryIfNotNeededPeriodSeconds)::WordType> + observeDecreaseHistoryIfNotNeededPeriodSeconds; - static inline MutableObeserverRegistry<decltype(checkCachePressurePeriodSeconds)::WordType> - observeCheckCachePressurePeriodSeconds; + AtomicWord<long long> snapshotTooOldErrorCount{0}; }; extern SnapshotWindowParams snapshotWindowParams; diff --git a/src/mongo/db/snapshot_window_options.idl b/src/mongo/db/snapshot_window_options.idl index 0c10ef0985d..8f1095bbac6 100644 --- a/src/mongo/db/snapshot_window_options.idl +++ b/src/mongo/db/snapshot_window_options.idl @@ -41,14 +41,6 @@ server_parameters: cpp_varname: "snapshotWindowParams.maxTargetSnapshotHistoryWindowInSeconds" validator: { gte: 0 } - cachePressureThreshold: - description: "Cache pressure threshold" - set_at: [ startup, runtime ] - cpp_varname: "snapshotWindowParams.cachePressureThreshold" - validator: - gte: 0 - lte: 100 - snapshotWindowMultiplicativeDecrease: description: "Snapshot window multiplicative decrease" set_at: [ startup, runtime ] @@ -69,15 +61,9 @@ server_parameters: cpp_varname: "snapshotWindowParams.minMillisBetweenSnapshotWindowInc" validator: { gte: 1 } - minMillisBetweenSnapshotWindowDec: - description: "Minimum duration between snapshot window decrement, in milliseconds" - set_at: [ startup, runtime ] - cpp_varname: "snapshotWindowParams.minMillisBetweenSnapshotWindowDec" - validator: { gte: 1 } - - checkCachePressurePeriodSeconds: + decreaseHistoryIfNotNeededPeriodSeconds: description: "Check cache pressure period, in seconds" set_at: [ startup, runtime ] - cpp_varname: "snapshotWindowParams.checkCachePressurePeriodSeconds" + cpp_varname: "snapshotWindowParams.decreaseHistoryIfNotNeededPeriodSeconds" validator: { gte: 1 } - on_update: std::ref(SnapshotWindowParams::observeCheckCachePressurePeriodSeconds) + on_update: std::ref(SnapshotWindowParams::observeDecreaseHistoryIfNotNeededPeriodSeconds) diff --git a/src/mongo/db/snapshot_window_util.cpp b/src/mongo/db/snapshot_window_util.cpp index 1ed27ca417b..eaaf84adc5b 100644 --- a/src/mongo/db/snapshot_window_util.cpp +++ b/src/mongo/db/snapshot_window_util.cpp @@ -52,25 +52,12 @@ namespace SnapshotWindowUtil { // Adds concurrency control to increaseTargetSnapshotWindowSize() and // decreaseTargetSnapshotWindowSize(). They should not run concurrently with themselves or one // another, since they act on and modify the same storage parameters. Further guards the static -// variables "_snapshotWindowLastDecreasedAt" and "_snapshotWindowLastIncreasedAt" used in -// increaseTargetSnapshotWindowSize() and decreaseSnapshowWindow(). +// variable "_snapshotWindowLastIncreasedAt" used in increaseTargetSnapshotWindowSize(). stdx::mutex snapshotWindowMutex; namespace { -void _decreaseTargetSnapshotWindowSize(WithLock lock, OperationContext* opCtx) { - // Tracks the last time that the snapshot window was decreased so that it does not go down so - // fast that the system does not have time to react and reduce snapshot availability. - static Date_t _snapshotWindowLastDecreasedAt{Date_t::min()}; - - if (_snapshotWindowLastDecreasedAt > - (Date_t::now() - - Milliseconds(snapshotWindowParams.minMillisBetweenSnapshotWindowDec.load()))) { - // We have already decreased the window size in the last minMillisBetweenSnapshotWindowDec - // milliseconds. - return; - } - +void _decreaseTargetSnapshotWindowSize(WithLock, OperationContext* opCtx) { snapshotWindowParams.targetSnapshotHistoryWindowInSeconds.store( snapshotWindowParams.targetSnapshotHistoryWindowInSeconds.load() * snapshotWindowParams.snapshotWindowMultiplicativeDecrease.load()); @@ -80,8 +67,6 @@ void _decreaseTargetSnapshotWindowSize(WithLock lock, OperationContext* opCtx) { StorageEngine* engine = opCtx->getServiceContext()->getStorageEngine(); invariant(engine); engine->setOldestTimestampFromStable(); - - _snapshotWindowLastDecreasedAt = Date_t::now(); } } // namespace @@ -105,23 +90,6 @@ void increaseTargetSnapshotWindowSize(OperationContext* opCtx) { return; } - // If the cache pressure is already too high, we will not put more pressure on it by increasing - // the window size. - StorageEngine* engine = opCtx->getServiceContext()->getStorageEngine(); - if (engine && engine->isCacheUnderPressure(opCtx)) { - warning() << "Attempted to increase the time window of available snapshots for " - "point-in-time operations (readConcern level 'snapshot' or transactions), but " - "the storage engine cache pressure, per the cachePressureThreshold setting of " - "'" - << snapshotWindowParams.cachePressureThreshold.load() - << "', is too high to allow it to increase. If this happens frequently, consider " - "either increasing the cache pressure threshold or increasing the memory " - "available to the storage engine cache, in order to improve the success rate " - "or speed of point-in-time requests."; - _decreaseTargetSnapshotWindowSize(lock, opCtx); - return; - } - if (snapshotWindowParams.targetSnapshotHistoryWindowInSeconds.load() == snapshotWindowParams.maxTargetSnapshotHistoryWindowInSeconds.load()) { warning() << "Attempted to increase the time window of available snapshots for " @@ -151,10 +119,28 @@ void decreaseTargetSnapshotWindowSize(OperationContext* opCtx) { stdx::unique_lock<stdx::mutex> lock(snapshotWindowMutex); StorageEngine* engine = opCtx->getServiceContext()->getStorageEngine(); - if (engine && engine->isCacheUnderPressure(opCtx)) { - _decreaseTargetSnapshotWindowSize(lock, opCtx); + if (engine) { + static auto lastInsertsCount = 0; + static auto lastSnapshotErrorCount = 0; + + auto currentInsertsCount = engine->getCacheOverflowTableInsertCount(opCtx); + auto currentSnapshotErrorCount = snapshotWindowParams.snapshotTooOldErrorCount.load(); + + // Only decrease the snapshot window size if there were writes to the cache overflow table + // and there has been no new SnapshotTooOld errors in the same time period. + if (currentInsertsCount > lastInsertsCount && + currentSnapshotErrorCount == lastSnapshotErrorCount) { + _decreaseTargetSnapshotWindowSize(lock, opCtx); + } + + lastInsertsCount = currentInsertsCount; + lastSnapshotErrorCount = currentSnapshotErrorCount; } } +void incrementSnapshotTooOldErrorCount() { + snapshotWindowParams.snapshotTooOldErrorCount.addAndFetch(1); +} + } // namespace SnapshotWindowUtil } // namespace mongo diff --git a/src/mongo/db/snapshot_window_util.h b/src/mongo/db/snapshot_window_util.h index 3645b0cab53..0610a6cd92c 100644 --- a/src/mongo/db/snapshot_window_util.h +++ b/src/mongo/db/snapshot_window_util.h @@ -40,10 +40,9 @@ class OperationContext; namespace SnapshotWindowUtil { /** - * Attempts to increase the setting that controls the window of time between stable_timestamp and + * Increases the setting that controls the window of time between stable_timestamp and * oldest_timestamp, in order to provide a greater range of available snapshots for point-in-time - * operations. The window will not be increased, however, if the cache pressure is currently too - * high. This function will be called when server requests return SnapshotTooOld (or similar) + * operations. This function will be called when server requests return SnapshotTooOld (or similar) * errors. Note that this will not immediately affect the oldest_timestamp. Rather, it affects * actions taken next time oldest_timestamp is updated, usually when the stable timestamp is * advanced. @@ -54,18 +53,21 @@ namespace SnapshotWindowUtil { * tracked and attempts to increase the window are limited to once in * minMillisBetweenSnapshotWindowInc. This is to protect against a sudden wave of function calls due * to simultaneous SnapshotTooOld errors. Some time must be allowed for the increased target - * snapshot window size to have an effect. The target size can also never exceed + * snapshot window size setting to have an effect. The target size can also never exceed * maxTargetSnapshotHistoryWindowInSeconds. */ void increaseTargetSnapshotWindowSize(OperationContext* opCtx); /** - * Attempts to decrease (if not already zero) the setting that affects the size of the window of - * time between stable_timestamp and oldest_timestamp in order to reduce storage engine cache - * pressure. The window target will not be decreased, however, if the cache is not currently under - * pressure. Pressure can occur when too much history is being maintained for point-in-time - * snapshots. Note that this will not necessarily immediately affect the actual window size; rather, - * it affects actions taken whenever oldest_timestamp is updated, usually when the stable timestamp + * Decreases the target snapshot window size if there has been cache pressure and no new + * SnapshotTooOld errors as tracked via the incrementSnapshotTooOldErrorCount() below since the last + * time this function was called. + * + * We do not decrease the window size when there have been SnapshotTooOld errors recently, even if + * there is some cache pressure, because that will cause sharded transaction commands to fail. + * + * Note that this will not necessarily immediately affect the actual window size; rather, + * it affects actions taken whenever oldest_timestamp is updated, usually when the stable_timestamp * is advanced. * * This will make one attempt to immediately adjust the window size if possible. @@ -74,5 +76,15 @@ void increaseTargetSnapshotWindowSize(OperationContext* opCtx); */ void decreaseTargetSnapshotWindowSize(OperationContext* opCtx); +/** + * Registers on a counter SnapshotTooOld errors encountered in the command layer. + * decreaseTargetSnapshotWindowSize() above uses the counter value: if there are any errors since + * the last decreaseTargetSnapshotWindowSize() call, then decreaseTargetSnapshotWindowSize() will + * choose not decrease the target snapshot window size setting. + * + * Concurrency safe, the internal counter is atomic. + */ +void incrementSnapshotTooOldErrorCount(); + } // namespace SnapshotWindowUtil } // namespace mongo diff --git a/src/mongo/db/snapshot_window_util_test.cpp b/src/mongo/db/snapshot_window_util_test.cpp index 5f8d7a8dd34..3956a9f9b71 100644 --- a/src/mongo/db/snapshot_window_util_test.cpp +++ b/src/mongo/db/snapshot_window_util_test.cpp @@ -69,9 +69,11 @@ TEST_F(SnapshotWindowTest, DecreaseAndIncreaseSnapshotWindow) { snapshotWindowParams.targetSnapshotHistoryWindowInSeconds.store(100); // Lower the time enforced between function calls to speed up testing. - // Dec must match Inc b/c increaseTargetWindowSize can call into decreaseTargetWindowSize. snapshotWindowParams.minMillisBetweenSnapshotWindowInc.store(100); - snapshotWindowParams.minMillisBetweenSnapshotWindowDec.store(100); + + // Stabilize for the test so we know that we are testing things as expected. + snapshotWindowParams.snapshotWindowAdditiveIncreaseSeconds.store(2); + snapshotWindowParams.snapshotWindowMultiplicativeDecrease.store(0.75); auto maxTargetSnapshotWindowSeconds = snapshotWindowParams.maxTargetSnapshotHistoryWindowInSeconds.load(); @@ -82,78 +84,117 @@ TEST_F(SnapshotWindowTest, DecreaseAndIncreaseSnapshotWindow) { snapshotWindowParams.snapshotWindowMultiplicativeDecrease.load(); auto windowAdditiveIncrease = snapshotWindowParams.snapshotWindowAdditiveIncreaseSeconds.load(); - auto cachePressureThreshold = snapshotWindowParams.cachePressureThreshold.load(); auto minTimeBetweenInc = snapshotWindowParams.minMillisBetweenSnapshotWindowInc.load(); /** - * Test that decreasing the size succeeds when cache pressure is ABOVE the threshold + * Test that trying to decrease the window size FAILS when there have been no writes to the + * cache overflow table. */ - engine->setCachePressureForTest(cachePressureThreshold + 5); - decreaseTargetSnapshotWindowSize(_opCtx.get()); auto snapshotWindowSecondsOne = snapshotWindowParams.targetSnapshotHistoryWindowInSeconds.load(); - ASSERT_GT(snapshotWindowSeconds, snapshotWindowSecondsOne); - ASSERT_EQ(snapshotWindowSecondsOne, - static_cast<int>(snapshotWindowSeconds * windowMultiplicativeDecrease)); + ASSERT_EQ(snapshotWindowSeconds, snapshotWindowSecondsOne); /** - * Test that increasing the size SUCCEEDS when the cache pressure is BELOW the threshold. + * Test that trying to decrease the window size SUCCEEDS when there have been writes to the + * cache overflow table. */ - engine->setCachePressureForTest(cachePressureThreshold - 5); + engine->setCacheOverflowTableInsertCountForTest(1); - increaseTargetSnapshotWindowSize(_opCtx.get()); + decreaseTargetSnapshotWindowSize(_opCtx.get()); auto snapshotWindowSecondsTwo = snapshotWindowParams.targetSnapshotHistoryWindowInSeconds.load(); - ASSERT_EQ(snapshotWindowSecondsTwo, snapshotWindowSecondsOne + windowAdditiveIncrease); + ASSERT_GT(snapshotWindowSecondsOne, snapshotWindowSecondsTwo); + ASSERT_EQ(snapshotWindowSecondsTwo, + static_cast<int>(snapshotWindowSeconds * windowMultiplicativeDecrease)); /** - * Test that increasing the size FAILS when the cache pressure is ABOVE the threshold, and - * instead this causes the size to be decreased. + * Test that trying to decrease the window size FAILS when there have been writes to the + * cache overflow table AND SnapshotTooOld errors have occurred. */ - engine->setCachePressureForTest(cachePressureThreshold + 5); + engine->setCacheOverflowTableInsertCountForTest(2); + incrementSnapshotTooOldErrorCount(); + + decreaseTargetSnapshotWindowSize(_opCtx.get()); + auto snapshotWindowSecondsThree = + snapshotWindowParams.targetSnapshotHistoryWindowInSeconds.load(); + + ASSERT_EQ(snapshotWindowSecondsTwo, snapshotWindowSecondsThree); + + /** + * Now test again that decreasing the size SUCCEEDS when there have been writes to the cache + * overflow table again (without any further SnapshotTooOld errors). + */ + + engine->setCacheOverflowTableInsertCountForTest(3); + + decreaseTargetSnapshotWindowSize(_opCtx.get()); + auto snapshotWindowSecondsFour = + snapshotWindowParams.targetSnapshotHistoryWindowInSeconds.load(); + + ASSERT_GT(snapshotWindowSecondsThree, snapshotWindowSecondsFour); + ASSERT_EQ(snapshotWindowSecondsFour, + static_cast<int>(snapshotWindowSecondsThree * windowMultiplicativeDecrease)); + + /** + * Test that increasing the size SUCCEEDS. + */ + + increaseTargetSnapshotWindowSize(_opCtx.get()); + auto snapshotWindowSecondsFive = + snapshotWindowParams.targetSnapshotHistoryWindowInSeconds.load(); + + ASSERT_EQ(snapshotWindowSecondsFive, snapshotWindowSecondsFour + windowAdditiveIncrease); + + /** + * Test that increasing the size SUCCEEDS even when there have been writes to the cache overflow + * table. + */ // Sleep for a time because increaseTargetSnapshotWindowSize() enforces a wait time between // updates. sleepmillis(2 * minTimeBetweenInc); + engine->setCacheOverflowTableInsertCountForTest(4); + increaseTargetSnapshotWindowSize(_opCtx.get()); - auto snapshotWindowSecondsThree = + auto snapshotWindowSecondsSix = snapshotWindowParams.targetSnapshotHistoryWindowInSeconds.load(); - ASSERT_EQ(snapshotWindowSecondsThree, - static_cast<int>(snapshotWindowSecondsTwo * windowMultiplicativeDecrease)); - - engine->setCachePressureForTest(cachePressureThreshold - 5); + ASSERT_EQ(snapshotWindowSecondsSix, snapshotWindowSecondsFive + windowAdditiveIncrease); /** * Test that the size cannot be increased above the maximum size. */ + // Bump up the additive increase to make this run faster. + snapshotWindowParams.snapshotWindowAdditiveIncreaseSeconds.store(9); + windowAdditiveIncrease = snapshotWindowParams.snapshotWindowAdditiveIncreaseSeconds.load(); + // Integers round down, so add 1 to make sure it reaches the max. int numIncreasesToReachMax = - (maxTargetSnapshotWindowSeconds - snapshotWindowSecondsThree) / windowAdditiveIncrease + 1; + (maxTargetSnapshotWindowSeconds - snapshotWindowSecondsSix) / windowAdditiveIncrease + 1; for (int i = 0; i < numIncreasesToReachMax; ++i) { sleepmillis(2 * minTimeBetweenInc); increaseTargetSnapshotWindowSize(_opCtx.get()); } // Should be at max. - auto snapshotWindowSecondsFour = + auto snapshotWindowSecondsSeven = snapshotWindowParams.targetSnapshotHistoryWindowInSeconds.load(); - ASSERT_EQ(snapshotWindowSecondsFour, maxTargetSnapshotWindowSeconds); + ASSERT_EQ(snapshotWindowSecondsSeven, maxTargetSnapshotWindowSeconds); // An attempt to increase beyond max should have no effect. sleepmillis(2 * minTimeBetweenInc); increaseTargetSnapshotWindowSize(_opCtx.get()); - auto snapshotWindowSecondsFive = + auto snapshotWindowSecondsEight = snapshotWindowParams.targetSnapshotHistoryWindowInSeconds.load(); - ASSERT_EQ(snapshotWindowSecondsFive, maxTargetSnapshotWindowSeconds); + ASSERT_EQ(snapshotWindowSecondsEight, maxTargetSnapshotWindowSeconds); } } // namespace diff --git a/src/mongo/db/storage/biggie/biggie_kv_engine.cpp b/src/mongo/db/storage/biggie/biggie_kv_engine.cpp index 1ece635c229..43e4b3672d0 100644 --- a/src/mongo/db/storage/biggie/biggie_kv_engine.cpp +++ b/src/mongo/db/storage/biggie/biggie_kv_engine.cpp @@ -47,7 +47,7 @@ mongo::RecoveryUnit* KVEngine::newRecoveryUnit() { return new RecoveryUnit(this, nullptr); } -void KVEngine::setCachePressureForTest(int pressure) { +void KVEngine::setCacheOverflowTableInsertCountForTest(int insertCount) { // TODO : implement. } diff --git a/src/mongo/db/storage/biggie/biggie_kv_engine.h b/src/mongo/db/storage/biggie/biggie_kv_engine.h index a1617bc89bf..59366823c27 100644 --- a/src/mongo/db/storage/biggie/biggie_kv_engine.h +++ b/src/mongo/db/storage/biggie/biggie_kv_engine.h @@ -105,11 +105,11 @@ public: return true; } - virtual bool isCacheUnderPressure(OperationContext* opCtx) const override { - return false; + virtual int64_t getCacheOverflowTableInsertCount(OperationContext* opCtx) const override { + return 0; } - virtual void setCachePressureForTest(int pressure) override; + virtual void setCacheOverflowTableInsertCountForTest(int insertCount) override; virtual int64_t getIdentSize(OperationContext* opCtx, StringData ident) { return 0; diff --git a/src/mongo/db/storage/devnull/devnull_kv_engine.cpp b/src/mongo/db/storage/devnull/devnull_kv_engine.cpp index 6ae745c5587..e9437a18908 100644 --- a/src/mongo/db/storage/devnull/devnull_kv_engine.cpp +++ b/src/mongo/db/storage/devnull/devnull_kv_engine.cpp @@ -261,13 +261,13 @@ SortedDataInterface* DevNullKVEngine::getSortedDataInterface(OperationContext* o return new DevNullSortedDataInterface(); } -bool DevNullKVEngine::isCacheUnderPressure(OperationContext* opCtx) const { - return (_cachePressureForTest >= snapshotWindowParams.cachePressureThreshold.load()); +int64_t DevNullKVEngine::getCacheOverflowTableInsertCount(OperationContext* opCtx) const { + return _overflowTableInsertCountForTest; } -void DevNullKVEngine::setCachePressureForTest(int pressure) { - invariant(pressure >= 0 && pressure <= 100); - _cachePressureForTest = pressure; +void DevNullKVEngine::setCacheOverflowTableInsertCountForTest(int insertsCount) { + invariant(insertsCount >= 0); + _overflowTableInsertCountForTest = insertsCount; } StatusWith<std::vector<std::string>> DevNullKVEngine::beginNonBlockingBackup( diff --git a/src/mongo/db/storage/devnull/devnull_kv_engine.h b/src/mongo/db/storage/devnull/devnull_kv_engine.h index e753e8805c0..04900f06fae 100644 --- a/src/mongo/db/storage/devnull/devnull_kv_engine.h +++ b/src/mongo/db/storage/devnull/devnull_kv_engine.h @@ -97,9 +97,9 @@ public: return true; } - virtual bool isCacheUnderPressure(OperationContext* opCtx) const override; + virtual int64_t getCacheOverflowTableInsertCount(OperationContext* opCtx) const override; - virtual void setCachePressureForTest(int pressure) override; + virtual void setCacheOverflowTableInsertCountForTest(int insertCount) override; virtual int64_t getIdentSize(OperationContext* opCtx, StringData ident) { return 1; @@ -150,6 +150,6 @@ public: private: std::shared_ptr<void> _catalogInfo; - int _cachePressureForTest; + int _overflowTableInsertCountForTest; }; } diff --git a/src/mongo/db/storage/kv/kv_engine.h b/src/mongo/db/storage/kv/kv_engine.h index ebdf490b78f..b2a195b750e 100644 --- a/src/mongo/db/storage/kv/kv_engine.h +++ b/src/mongo/db/storage/kv/kv_engine.h @@ -332,16 +332,16 @@ public: virtual void setOldestTimestamp(Timestamp newOldestTimestamp, bool force) {} /** - * See `StorageEngine::isCacheUnderPressure()` + * See 'StorageEngine::getCacheOverflowTableInsertCount' */ - virtual bool isCacheUnderPressure(OperationContext* opCtx) const { - return false; + virtual int64_t getCacheOverflowTableInsertCount(OperationContext* opCtx) const { + return 0; } /** - * See 'StorageEngine::setCachePressureForTest()' + * See 'StorageEngine::setCacheOverflowTableInsertCountForTest()' */ - virtual void setCachePressureForTest(int pressure) {} + virtual void setCacheOverflowTableInsertCountForTest(int insertCount) {} /** * See `StorageEngine::supportsRecoverToStableTimestamp` diff --git a/src/mongo/db/storage/kv/kv_storage_engine.cpp b/src/mongo/db/storage/kv/kv_storage_engine.cpp index 9931bcf93ee..fe1ea716d72 100644 --- a/src/mongo/db/storage/kv/kv_storage_engine.cpp +++ b/src/mongo/db/storage/kv/kv_storage_engine.cpp @@ -681,12 +681,12 @@ void KVStorageEngine::setOldestActiveTransactionTimestampCallback( _engine->setOldestActiveTransactionTimestampCallback(callback); } -bool KVStorageEngine::isCacheUnderPressure(OperationContext* opCtx) const { - return _engine->isCacheUnderPressure(opCtx); +int64_t KVStorageEngine::getCacheOverflowTableInsertCount(OperationContext* opCtx) const { + return _engine->getCacheOverflowTableInsertCount(opCtx); } -void KVStorageEngine::setCachePressureForTest(int pressure) { - return _engine->setCachePressureForTest(pressure); +void KVStorageEngine::setCacheOverflowTableInsertCountForTest(int insertCount) { + return _engine->setCacheOverflowTableInsertCountForTest(insertCount); } bool KVStorageEngine::supportsRecoverToStableTimestamp() const { diff --git a/src/mongo/db/storage/kv/kv_storage_engine.h b/src/mongo/db/storage/kv/kv_storage_engine.h index 8bba8926a9a..92119d24dfe 100644 --- a/src/mongo/db/storage/kv/kv_storage_engine.h +++ b/src/mongo/db/storage/kv/kv_storage_engine.h @@ -125,9 +125,9 @@ public: virtual void setOldestActiveTransactionTimestampCallback( StorageEngine::OldestActiveTransactionTimestampCallback) override; - virtual bool isCacheUnderPressure(OperationContext* opCtx) const override; + virtual int64_t getCacheOverflowTableInsertCount(OperationContext* opCtx) const override; - virtual void setCachePressureForTest(int pressure) override; + virtual void setCacheOverflowTableInsertCountForTest(int insertCount) override; virtual bool supportsRecoverToStableTimestamp() const override; diff --git a/src/mongo/db/storage/storage_engine.h b/src/mongo/db/storage/storage_engine.h index c74c7de72ab..64c37bc811a 100644 --- a/src/mongo/db/storage/storage_engine.h +++ b/src/mongo/db/storage/storage_engine.h @@ -450,21 +450,19 @@ public: OldestActiveTransactionTimestampCallback callback){}; /** - * Indicates whether the storage engine cache is under pressure. - * - * Retrieves a cache pressure value in the range [0, 100] from the storage engine, and compares - * it against storageGlobalParams.cachePressureThreshold, a dynamic server parameter, to - * determine whether cache pressure is too high. + * Retrieves the number of inserts done to the cache overflow table. Writes to that table only + * occur when the read/write cache size exceeds the allotted in-memory cache capacity and must + * write to disk, which is slow. Tracking this value over time is indicative of cache pressure. */ - virtual bool isCacheUnderPressure(OperationContext* opCtx) const { - return false; + virtual int64_t getCacheOverflowTableInsertCount(OperationContext* opCtx) const { + return 0; } /** - * For unit tests only. Sets the cache pressure value with which isCacheUnderPressure() - * evalutates to 'pressure'. + * For unit tests only. Sets the counter for the number of cache overflow table inserts that the + * getCacheOverflowTableInsertCount() function above returns. */ - virtual void setCachePressureForTest(int pressure) {} + virtual void setCacheOverflowTableInsertCountForTest(int insertCount) {} /** * Notifies the storage engine that a replication batch has completed. diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp index 20c52321578..efbaa237c18 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp @@ -1955,14 +1955,14 @@ void WiredTigerKVEngine::replicationBatchIsComplete() const { _oplogManager->triggerJournalFlush(); } -bool WiredTigerKVEngine::isCacheUnderPressure(OperationContext* opCtx) const { +int64_t WiredTigerKVEngine::getCacheOverflowTableInsertCount(OperationContext* opCtx) const { WiredTigerSession* session = WiredTigerRecoveryUnit::get(opCtx)->getSessionNoTxn(); invariant(session); - int64_t score = uassertStatusOK(WiredTigerUtil::getStatisticsValueAs<int64_t>( - session->getSession(), "statistics:", "", WT_STAT_CONN_CACHE_LOOKASIDE_SCORE)); + int64_t insertCount = uassertStatusOK(WiredTigerUtil::getStatisticsValueAs<int64_t>( + session->getSession(), "statistics:", "", WT_STAT_CONN_CACHE_LOOKASIDE_INSERT)); - return (score >= snapshotWindowParams.cachePressureThreshold.load()); + return insertCount; } Timestamp WiredTigerKVEngine::getStableTimestamp() const { diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h index c7f6efd48cc..6297389285b 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h @@ -243,7 +243,7 @@ public: */ void replicationBatchIsComplete() const override; - bool isCacheUnderPressure(OperationContext* opCtx) const override; + int64_t getCacheOverflowTableInsertCount(OperationContext* opCtx) const override; bool supportsReadConcernMajority() const final; diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_recovery_unit_test.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_recovery_unit_test.cpp index 99fc4f51477..bd0b508f05b 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_recovery_unit_test.cpp +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_recovery_unit_test.cpp @@ -192,10 +192,11 @@ TEST_F(WiredTigerRecoveryUnitTestFixture, CreateAndCheckForCachePressure) { auto recordId = ress.getValue(); wu.commit(); + auto startOverflowInsertsCount = engine->getCacheOverflowTableInsertCount(opCtx); for (int j = 0; j < 1000; ++j) { - // Once we hit the cache pressure threshold, i.e. have successfully created cache pressure - // that is detectable, we are done. - if (engine->isCacheUnderPressure(opCtx)) { + // We know that we have successfully created cache pressure when the in-memory cache + // overflows to disk by writing to the cache overflow table (the WT lookaside table). + if (engine->getCacheOverflowTableInsertCount(opCtx) > startOverflowInsertsCount) { invariant(j != 0); break; } diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp index 01f5a34480c..8bf7008b7aa 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp @@ -675,13 +675,15 @@ void WiredTigerUtil::appendSnapshotWindowSettings(WiredTigerKVEngine* engine, const unsigned currentAvailableSnapshotWindow = stableTimestamp.getSecs() - oldestTimestamp.getSecs(); - int64_t score = uassertStatusOK(WiredTigerUtil::getStatisticsValueAs<int64_t>( - session->getSession(), "statistics:", "", WT_STAT_CONN_CACHE_LOOKASIDE_SCORE)); + int64_t overflowTableInsertCount = + uassertStatusOK(WiredTigerUtil::getStatisticsValueAs<int64_t>( + session->getSession(), "statistics:", "", WT_STAT_CONN_CACHE_LOOKASIDE_INSERT)); + long long totalNumberOfSnapshotTooOldErrors = + snapshotWindowParams.snapshotTooOldErrorCount.load(); BSONObjBuilder settings(bob->subobjStart("snapshot-window-settings")); - settings.append("cache pressure percentage threshold", - snapshotWindowParams.cachePressureThreshold.load()); - settings.append("current cache pressure percentage", score); + settings.append("total number of cache overflow disk writes", overflowTableInsertCount); + settings.append("total number of SnapshotTooOld errors", totalNumberOfSnapshotTooOldErrors); settings.append("max target available snapshots window size in seconds", snapshotWindowParams.maxTargetSnapshotHistoryWindowInSeconds.load()); settings.append("target available snapshots window size in seconds", diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_util.h b/src/mongo/db/storage/wiredtiger/wiredtiger_util.h index 23015814cf7..7f7d390f721 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_util.h +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_util.h @@ -152,8 +152,8 @@ public: * that affect that window of maintained history. * * "snapshot-window-settings" : { - * "cache pressure percentage threshold" : <num>, - * "current cache pressure percentage" : <num>, + * "total number of cache overflow disk writes", + * "total number of SnapshotTooOld errors", * "max target available snapshots window size in seconds" : <num>, * "target available snapshots window size in seconds" : <num>, * "current available snapshots window size in seconds" : <num>, diff --git a/src/mongo/db/transaction_participant.h b/src/mongo/db/transaction_participant.h index 2e343ea0d70..c7abe3b43ec 100644 --- a/src/mongo/db/transaction_participant.h +++ b/src/mongo/db/transaction_participant.h @@ -187,7 +187,7 @@ class TransactionParticipant { }; public: - static inline MutableObeserverRegistry<int32_t> observeTransactionLifetimeLimitSeconds; + static inline MutableObserverRegistry<int32_t> observeTransactionLifetimeLimitSeconds; TransactionParticipant(); diff --git a/src/mongo/idl/mutable_observer_registry.h b/src/mongo/idl/mutable_observer_registry.h index b787372cc6f..89ceaf2b2fb 100644 --- a/src/mongo/idl/mutable_observer_registry.h +++ b/src/mongo/idl/mutable_observer_registry.h @@ -46,7 +46,7 @@ namespace mongo { * Then adding observers via someGlobalMutableObserverRegistry.addObserver(); */ template <typename T> -class MutableObeserverRegistry { +class MutableObserverRegistry { public: using Argument = T; |