summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorDianna Hohensee <dianna.hohensee@10gen.com>2019-07-01 10:01:15 -0400
committerDianna Hohensee <dianna.hohensee@10gen.com>2019-07-09 13:51:20 -0400
commit1cccd2d05271c018b702bcc3e30a7516457a192a (patch)
tree9a7a375182a90e36be0ddda96e814e510f469716
parent41b3ba1faa0841c7c189ecdae76bb1bf4a21c46c (diff)
downloadmongo-1cccd2d05271c018b702bcc3e30a7516457a192a.tar.gz
Revert "SERVER-36956 SnapshotTooOld errors will always increase the snapshot history window size"
This reverts commit 8899b34e1044b08aec7ad9f8546652456472702c. (cherry picked from commit 8bb53a07a5c593d85b6229a2afe096b3e1efe21d)
-rw-r--r--jstests/noPassthrough/snapshotWindow_serverParameters.js28
-rw-r--r--src/mongo/db/db.cpp2
-rw-r--r--src/mongo/db/periodic_runner_job_decrease_snapshot_cache_pressure.cpp10
-rw-r--r--src/mongo/db/periodic_runner_job_decrease_snapshot_cache_pressure.h10
-rw-r--r--src/mongo/db/service_entry_point_common.cpp1
-rw-r--r--src/mongo/db/snapshot_window_options.h51
-rw-r--r--src/mongo/db/snapshot_window_options.idl20
-rw-r--r--src/mongo/db/snapshot_window_util.cpp58
-rw-r--r--src/mongo/db/snapshot_window_util.h30
-rw-r--r--src/mongo/db/snapshot_window_util_test.cpp89
-rw-r--r--src/mongo/db/storage/biggie/biggie_kv_engine.cpp2
-rw-r--r--src/mongo/db/storage/biggie/biggie_kv_engine.h6
-rw-r--r--src/mongo/db/storage/devnull/devnull_kv_engine.cpp10
-rw-r--r--src/mongo/db/storage/devnull/devnull_kv_engine.h6
-rw-r--r--src/mongo/db/storage/kv/kv_engine.h10
-rw-r--r--src/mongo/db/storage/storage_engine.h18
-rw-r--r--src/mongo/db/storage/storage_engine_impl.cpp8
-rw-r--r--src/mongo/db/storage/storage_engine_impl.h4
-rw-r--r--src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp8
-rw-r--r--src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h2
-rw-r--r--src/mongo/db/storage/wiredtiger/wiredtiger_recovery_unit_test.cpp7
-rw-r--r--src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp12
-rw-r--r--src/mongo/db/storage/wiredtiger/wiredtiger_util.h4
23 files changed, 200 insertions, 196 deletions
diff --git a/jstests/noPassthrough/snapshotWindow_serverParameters.js b/jstests/noPassthrough/snapshotWindow_serverParameters.js
index 3a05dc3354f..4b41954ed61 100644
--- a/jstests/noPassthrough/snapshotWindow_serverParameters.js
+++ b/jstests/noPassthrough/snapshotWindow_serverParameters.js
@@ -17,6 +17,17 @@
false /*hasUpperBound*/,
"unused" /*upperOutOfBounds*/);
+ // Valid parameter values are in the range [0, 100].
+ testNumericServerParameter("cachePressureThreshold",
+ true /*isStartupParameter*/,
+ true /*isRuntimeParameter*/,
+ 50 /*defaultValue*/,
+ 70 /*nonDefaultValidValue*/,
+ true /*hasLowerBound*/,
+ -1 /*lowerOutOfBounds*/,
+ true /*hasUpperBound*/,
+ 101 /*upperOutOfBounds*/);
+
// Valid parameter values are in the range (0, 1).
testNumericServerParameter("snapshotWindowMultiplicativeDecrease",
true /*isStartupParameter*/,
@@ -32,7 +43,7 @@
testNumericServerParameter("snapshotWindowAdditiveIncreaseSeconds",
true /*isStartupParameter*/,
true /*isRuntimeParameter*/,
- 1 /*defaultValue*/,
+ 2 /*defaultValue*/,
10 /*nonDefaultValidValue*/,
true /*hasLowerBound*/,
0 /*lowerOutOfBounds*/,
@@ -40,10 +51,10 @@
"unused" /*upperOutOfBounds*/);
// Valid parameter values are in the range [1, infinity).
- testNumericServerParameter("decreaseHistoryIfNotNeededPeriodSeconds",
+ testNumericServerParameter("checkCachePressurePeriodSeconds",
true /*isStartupParameter*/,
true /*isRuntimeParameter*/,
- 15 /*defaultValue*/,
+ 5 /*defaultValue*/,
8 /*nonDefaultValidValue*/,
true /*hasLowerBound*/,
0 /*lowerOutOfBounds*/,
@@ -60,4 +71,15 @@
0 /*lowerOutOfBounds*/,
false /*hasUpperBound*/,
"unused" /*upperOutOfBounds*/);
+
+ // Valid parameter values are in the range [1, infinity).
+ testNumericServerParameter("minMillisBetweenSnapshotWindowDec",
+ true /*isStartupParameter*/,
+ true /*isRuntimeParameter*/,
+ 500 /*defaultValue*/,
+ 2 * 1000 /*nonDefaultValidValue*/,
+ true /*hasLowerBound*/,
+ 0 /*lowerOutOfBounds*/,
+ false /*hasUpperBound*/,
+ "unused" /*upperOutOfBounds*/);
})();
diff --git a/src/mongo/db/db.cpp b/src/mongo/db/db.cpp
index e6dd1dc014d..e496d8c60ac 100644
--- a/src/mongo/db/db.cpp
+++ b/src/mongo/db/db.cpp
@@ -615,7 +615,7 @@ ExitCode _initAndListen(int listenPort) {
// release periodically in order to avoid storage cache pressure build up.
if (storageEngine->supportsReadConcernSnapshot()) {
startPeriodicThreadToAbortExpiredTransactions(serviceContext);
- startPeriodicThreadToDecreaseSnapshotHistoryIfNotNeeded(serviceContext);
+ startPeriodicThreadToDecreaseSnapshotHistoryCachePressure(serviceContext);
}
// Set up the logical session cache
diff --git a/src/mongo/db/periodic_runner_job_decrease_snapshot_cache_pressure.cpp b/src/mongo/db/periodic_runner_job_decrease_snapshot_cache_pressure.cpp
index 8e4f6594bab..3a5a41b5ee3 100644
--- a/src/mongo/db/periodic_runner_job_decrease_snapshot_cache_pressure.cpp
+++ b/src/mongo/db/periodic_runner_job_decrease_snapshot_cache_pressure.cpp
@@ -43,7 +43,7 @@
namespace mongo {
-void startPeriodicThreadToDecreaseSnapshotHistoryIfNotNeeded(ServiceContext* serviceContext) {
+void startPeriodicThreadToDecreaseSnapshotHistoryCachePressure(ServiceContext* serviceContext) {
// Enforce calling this function once, and only once.
static bool firstCall = true;
invariant(firstCall);
@@ -53,7 +53,7 @@ void startPeriodicThreadToDecreaseSnapshotHistoryIfNotNeeded(ServiceContext* ser
invariant(periodicRunner);
PeriodicRunner::PeriodicJob job(
- "startPeriodicThreadToDecreaseSnapshotHistoryIfNotNeeded",
+ "startPeriodicThreadToDecreaseSnapshotHistoryCachePressure",
[](Client* client) {
try {
// The opCtx destructor handles unsetting itself from the Client.
@@ -69,18 +69,18 @@ void startPeriodicThreadToDecreaseSnapshotHistoryIfNotNeeded(ServiceContext* ser
}
}
},
- Seconds(snapshotWindowParams.decreaseHistoryIfNotNeededPeriodSeconds.load()));
+ Seconds(snapshotWindowParams.checkCachePressurePeriodSeconds.load()));
auto handle = periodicRunner->makeJob(std::move(job));
handle->start();
- SnapshotWindowParams::observeDecreaseHistoryIfNotNeededPeriodSeconds
+ SnapshotWindowParams::observeCheckCachePressurePeriodSeconds
.addObserver([handle = std::move(handle)](const auto& secs) {
try {
handle->setPeriod(Seconds(secs));
} catch (const DBException& ex) {
log() << "Failed to update the period of the thread which decreases data history "
- "target window size if there have been no new SnapshotTooOld errors."
+ "target window size if there is cache pressure."
<< ex.toStatus();
}
});
diff --git a/src/mongo/db/periodic_runner_job_decrease_snapshot_cache_pressure.h b/src/mongo/db/periodic_runner_job_decrease_snapshot_cache_pressure.h
index 1b6c2e65b18..fca8b832fbf 100644
--- a/src/mongo/db/periodic_runner_job_decrease_snapshot_cache_pressure.h
+++ b/src/mongo/db/periodic_runner_job_decrease_snapshot_cache_pressure.h
@@ -34,15 +34,13 @@ namespace mongo {
class ServiceContext;
/**
- * Periodically checks whether there has been any storage engine cache pressure and SnapshotTooOld
- * errors to determine whether the maintained snapshot history window target setting should be
- * decreased. If there has been cache pressure and no new SnapshotTooOld errors in the last period,
- * then the target window size will be decrease. Maintaining too much snapshot and write history can
- * slow down the system. Runs once every decreaseHistoryIfNotNeededPeriodSeconds.
+ * Periodically checks for storage engine cache pressure to determine whether the maintained
+ * snapshot history window target setting should be decreased. Maintaining too much snapshot and
+ * write history can slow down the system. Runs once every checkCachePressurePeriodSeconds.
*
* This function should only ever be called once, during mongod server startup (db.cpp).
* The PeriodicRunner will handle shutting down the job on shutdown, no extra handling necessary.
*/
-void startPeriodicThreadToDecreaseSnapshotHistoryIfNotNeeded(ServiceContext* serviceContext);
+void startPeriodicThreadToDecreaseSnapshotHistoryCachePressure(ServiceContext* serviceContext);
} // namespace mongo
diff --git a/src/mongo/db/service_entry_point_common.cpp b/src/mongo/db/service_entry_point_common.cpp
index 5c48ade9178..281198f62bb 100644
--- a/src/mongo/db/service_entry_point_common.cpp
+++ b/src/mongo/db/service_entry_point_common.cpp
@@ -860,7 +860,6 @@ void execCommandDatabase(OperationContext* opCtx,
// of successful future PIT atClusterTime requests.
auto engine = opCtx->getServiceContext()->getStorageEngine();
if (engine && engine->supportsReadConcernSnapshot()) {
- SnapshotWindowUtil::incrementSnapshotTooOldErrorCount();
SnapshotWindowUtil::increaseTargetSnapshotWindowSize(opCtx);
}
} else {
diff --git a/src/mongo/db/snapshot_window_options.h b/src/mongo/db/snapshot_window_options.h
index 93b3de975c9..95683969ca0 100644
--- a/src/mongo/db/snapshot_window_options.h
+++ b/src/mongo/db/snapshot_window_options.h
@@ -64,42 +64,49 @@ struct SnapshotWindowParams {
AtomicWord<int> targetSnapshotHistoryWindowInSeconds{
maxTargetSnapshotHistoryWindowInSeconds.load()};
+ // cachePressureThreshold (startup & runtime server paramter, range [0, 100]).
+ //
+ // Dictates what percentage of cache in use is considered too high. This setting helps preempt
+ // storage cache pressure immobilizing the system. Attempts to increase
+ // targetSnapshotHistoryWindowInSeconds will be ignored when the cache pressure reaches this
+ // threshold. Additionally, a periodic task will decrease targetSnapshotHistoryWindowInSeconds
+ // when cache pressure exceeds the threshold.
+ AtomicWord<int> cachePressureThreshold{50};
+
// snapshotWindowMultiplicativeDecrease (startup & runtime server paramter, range (0,1)).
//
- // Controls by what multiplier the target snapshot history window size setting is decreased
- // when there is cache pressure.
+ // Controls by what multiplier the target snapshot history window setting is decreased when
+ // cache pressure becomes too high, per the cachePressureThreshold setting.
AtomicDouble snapshotWindowMultiplicativeDecrease{0.75};
// snapshotWindowAdditiveIncreaseSeconds (startup & runtime server paramter, range 1+).
//
- // Controls by how much the target snapshot history window size setting is increased when we
- // need to service older snapshots for global point-in-time reads.
- AtomicWord<int> snapshotWindowAdditiveIncreaseSeconds{1};
+ // Controls by how much the target snapshot history window setting is increased when cache
+ // pressure is OK, per cachePressureThreshold, and we need to service older snapshots for global
+ // point-in-time reads.
+ AtomicWord<int> snapshotWindowAdditiveIncreaseSeconds{2};
// minMillisBetweenSnapshotWindowInc (startup & runtime server paramter, range 0+).
+ // minMillisBetweenSnapshotWindowDec (startup & runtime server paramter, range 0+).
//
- // Controls how often attempting to increase the target snapshot window will have an effect.
- // Multiple callers within minMillisBetweenSnapshotWindowInc will have the same effect as one.
- // This protects the system because it takes time for the target snapshot window to affect the
- // actual storage engine snapshot window. The stable timestamp must move forward for the window
- // between it and oldest timestamp to grow or shrink.
+ // Controls how often attempting to increase/decrease the target snapshot window will have an
+ // effect. Multiple callers within minMillisBetweenSnapshotWindowInc will have the same effect
+ // as one. This protects the system because it takes time for the target snapshot window to
+ // affect the actual storage engine snapshot window. The stable timestamp must move forward for
+ // the window between it and oldest timestamp to grow or shrink.
AtomicWord<int> minMillisBetweenSnapshotWindowInc{500};
+ AtomicWord<int> minMillisBetweenSnapshotWindowDec{500};
- // decreaseHistoryIfNotNeededPeriodSeconds (startup & runtime server paramter, range 1+)
+ // checkCachePressurePeriodSeconds (startup & runtime server paramter, range 1+)
//
// Controls the period of the task that checks for cache pressure and decreases
- // targetSnapshotHistoryWindowInSeconds if there has been pressure and no new SnapshotTooOld
- // errors.
- //
- // This should not run very frequently. It is preferable to increase the window size, and cache
- // pressure, rather than failing PIT reads.
- AtomicWord<int> decreaseHistoryIfNotNeededPeriodSeconds{15};
-
- static inline MutableObserverRegistry<decltype(
- decreaseHistoryIfNotNeededPeriodSeconds)::WordType>
- observeDecreaseHistoryIfNotNeededPeriodSeconds;
+ // targetSnapshotHistoryWindowInSeconds if the pressure is above cachePressureThreshold. The
+ // target window size setting must not be decreased too fast because time must be allowed for
+ // the storage engine to attempt to act on the new setting.
+ AtomicWord<int> checkCachePressurePeriodSeconds{5};
- AtomicWord<long long> snapshotTooOldErrorCount{0};
+ static inline MutableObserverRegistry<decltype(checkCachePressurePeriodSeconds)::WordType>
+ observeCheckCachePressurePeriodSeconds;
};
extern SnapshotWindowParams snapshotWindowParams;
diff --git a/src/mongo/db/snapshot_window_options.idl b/src/mongo/db/snapshot_window_options.idl
index 8f1095bbac6..0c10ef0985d 100644
--- a/src/mongo/db/snapshot_window_options.idl
+++ b/src/mongo/db/snapshot_window_options.idl
@@ -41,6 +41,14 @@ server_parameters:
cpp_varname: "snapshotWindowParams.maxTargetSnapshotHistoryWindowInSeconds"
validator: { gte: 0 }
+ cachePressureThreshold:
+ description: "Cache pressure threshold"
+ set_at: [ startup, runtime ]
+ cpp_varname: "snapshotWindowParams.cachePressureThreshold"
+ validator:
+ gte: 0
+ lte: 100
+
snapshotWindowMultiplicativeDecrease:
description: "Snapshot window multiplicative decrease"
set_at: [ startup, runtime ]
@@ -61,9 +69,15 @@ server_parameters:
cpp_varname: "snapshotWindowParams.minMillisBetweenSnapshotWindowInc"
validator: { gte: 1 }
- decreaseHistoryIfNotNeededPeriodSeconds:
+ minMillisBetweenSnapshotWindowDec:
+ description: "Minimum duration between snapshot window decrement, in milliseconds"
+ set_at: [ startup, runtime ]
+ cpp_varname: "snapshotWindowParams.minMillisBetweenSnapshotWindowDec"
+ validator: { gte: 1 }
+
+ checkCachePressurePeriodSeconds:
description: "Check cache pressure period, in seconds"
set_at: [ startup, runtime ]
- cpp_varname: "snapshotWindowParams.decreaseHistoryIfNotNeededPeriodSeconds"
+ cpp_varname: "snapshotWindowParams.checkCachePressurePeriodSeconds"
validator: { gte: 1 }
- on_update: std::ref(SnapshotWindowParams::observeDecreaseHistoryIfNotNeededPeriodSeconds)
+ on_update: std::ref(SnapshotWindowParams::observeCheckCachePressurePeriodSeconds)
diff --git a/src/mongo/db/snapshot_window_util.cpp b/src/mongo/db/snapshot_window_util.cpp
index eaaf84adc5b..1ed27ca417b 100644
--- a/src/mongo/db/snapshot_window_util.cpp
+++ b/src/mongo/db/snapshot_window_util.cpp
@@ -52,12 +52,25 @@ namespace SnapshotWindowUtil {
// Adds concurrency control to increaseTargetSnapshotWindowSize() and
// decreaseTargetSnapshotWindowSize(). They should not run concurrently with themselves or one
// another, since they act on and modify the same storage parameters. Further guards the static
-// variable "_snapshotWindowLastIncreasedAt" used in increaseTargetSnapshotWindowSize().
+// variables "_snapshotWindowLastDecreasedAt" and "_snapshotWindowLastIncreasedAt" used in
+// increaseTargetSnapshotWindowSize() and decreaseSnapshowWindow().
stdx::mutex snapshotWindowMutex;
namespace {
-void _decreaseTargetSnapshotWindowSize(WithLock, OperationContext* opCtx) {
+void _decreaseTargetSnapshotWindowSize(WithLock lock, OperationContext* opCtx) {
+ // Tracks the last time that the snapshot window was decreased so that it does not go down so
+ // fast that the system does not have time to react and reduce snapshot availability.
+ static Date_t _snapshotWindowLastDecreasedAt{Date_t::min()};
+
+ if (_snapshotWindowLastDecreasedAt >
+ (Date_t::now() -
+ Milliseconds(snapshotWindowParams.minMillisBetweenSnapshotWindowDec.load()))) {
+ // We have already decreased the window size in the last minMillisBetweenSnapshotWindowDec
+ // milliseconds.
+ return;
+ }
+
snapshotWindowParams.targetSnapshotHistoryWindowInSeconds.store(
snapshotWindowParams.targetSnapshotHistoryWindowInSeconds.load() *
snapshotWindowParams.snapshotWindowMultiplicativeDecrease.load());
@@ -67,6 +80,8 @@ void _decreaseTargetSnapshotWindowSize(WithLock, OperationContext* opCtx) {
StorageEngine* engine = opCtx->getServiceContext()->getStorageEngine();
invariant(engine);
engine->setOldestTimestampFromStable();
+
+ _snapshotWindowLastDecreasedAt = Date_t::now();
}
} // namespace
@@ -90,6 +105,23 @@ void increaseTargetSnapshotWindowSize(OperationContext* opCtx) {
return;
}
+ // If the cache pressure is already too high, we will not put more pressure on it by increasing
+ // the window size.
+ StorageEngine* engine = opCtx->getServiceContext()->getStorageEngine();
+ if (engine && engine->isCacheUnderPressure(opCtx)) {
+ warning() << "Attempted to increase the time window of available snapshots for "
+ "point-in-time operations (readConcern level 'snapshot' or transactions), but "
+ "the storage engine cache pressure, per the cachePressureThreshold setting of "
+ "'"
+ << snapshotWindowParams.cachePressureThreshold.load()
+ << "', is too high to allow it to increase. If this happens frequently, consider "
+ "either increasing the cache pressure threshold or increasing the memory "
+ "available to the storage engine cache, in order to improve the success rate "
+ "or speed of point-in-time requests.";
+ _decreaseTargetSnapshotWindowSize(lock, opCtx);
+ return;
+ }
+
if (snapshotWindowParams.targetSnapshotHistoryWindowInSeconds.load() ==
snapshotWindowParams.maxTargetSnapshotHistoryWindowInSeconds.load()) {
warning() << "Attempted to increase the time window of available snapshots for "
@@ -119,28 +151,10 @@ void decreaseTargetSnapshotWindowSize(OperationContext* opCtx) {
stdx::unique_lock<stdx::mutex> lock(snapshotWindowMutex);
StorageEngine* engine = opCtx->getServiceContext()->getStorageEngine();
- if (engine) {
- static auto lastInsertsCount = 0;
- static auto lastSnapshotErrorCount = 0;
-
- auto currentInsertsCount = engine->getCacheOverflowTableInsertCount(opCtx);
- auto currentSnapshotErrorCount = snapshotWindowParams.snapshotTooOldErrorCount.load();
-
- // Only decrease the snapshot window size if there were writes to the cache overflow table
- // and there has been no new SnapshotTooOld errors in the same time period.
- if (currentInsertsCount > lastInsertsCount &&
- currentSnapshotErrorCount == lastSnapshotErrorCount) {
- _decreaseTargetSnapshotWindowSize(lock, opCtx);
- }
-
- lastInsertsCount = currentInsertsCount;
- lastSnapshotErrorCount = currentSnapshotErrorCount;
+ if (engine && engine->isCacheUnderPressure(opCtx)) {
+ _decreaseTargetSnapshotWindowSize(lock, opCtx);
}
}
-void incrementSnapshotTooOldErrorCount() {
- snapshotWindowParams.snapshotTooOldErrorCount.addAndFetch(1);
-}
-
} // namespace SnapshotWindowUtil
} // namespace mongo
diff --git a/src/mongo/db/snapshot_window_util.h b/src/mongo/db/snapshot_window_util.h
index 0610a6cd92c..cd13ddd2112 100644
--- a/src/mongo/db/snapshot_window_util.h
+++ b/src/mongo/db/snapshot_window_util.h
@@ -40,9 +40,10 @@ class OperationContext;
namespace SnapshotWindowUtil {
/**
- * Increases the setting that controls the window of time between stable_timestamp and
+ * Attempts to increase the setting that controls the window of time between stable_timestamp and
* oldest_timestamp, in order to provide a greater range of available snapshots for point-in-time
- * operations. This function will be called when server requests return SnapshotTooOld (or similar)
+ * operations. The window will not be increased, however, if the cache pressure is currently too
+ * high. This function will be called when server requests return SnapshotTooOld (or similar)
* errors. Note that this will not immediately affect the oldest_timestamp. Rather, it affects
* actions taken next time oldest_timestamp is updated, usually when the stable timestamp is
* advanced.
@@ -59,15 +60,12 @@ namespace SnapshotWindowUtil {
void increaseTargetSnapshotWindowSize(OperationContext* opCtx);
/**
- * Decreases the target snapshot window size if there has been cache pressure and no new
- * SnapshotTooOld errors as tracked via the incrementSnapshotTooOldErrorCount() below since the last
- * time this function was called.
- *
- * We do not decrease the window size when there have been SnapshotTooOld errors recently, even if
- * there is some cache pressure, because that will cause sharded transaction commands to fail.
- *
- * Note that this will not necessarily immediately affect the actual window size; rather,
- * it affects actions taken whenever oldest_timestamp is updated, usually when the stable_timestamp
+ * Attempts to decrease (if not already zero) the setting that affects the size of the window of
+ * time between stable_timestamp and oldest_timestamp in order to reduce storage engine cache
+ * pressure. The window target will not be decreased, however, if the cache is not currently under
+ * pressure. Pressure can occur when too much history is being maintained for point-in-time
+ * snapshots. Note that this will not necessarily immediately affect the actual window size; rather,
+ * it affects actions taken whenever oldest_timestamp is updated, usually when the stable timestamp
* is advanced.
*
* This will make one attempt to immediately adjust the window size if possible.
@@ -76,15 +74,5 @@ void increaseTargetSnapshotWindowSize(OperationContext* opCtx);
*/
void decreaseTargetSnapshotWindowSize(OperationContext* opCtx);
-/**
- * Registers on a counter SnapshotTooOld errors encountered in the command layer.
- * decreaseTargetSnapshotWindowSize() above uses the counter value: if there are any errors since
- * the last decreaseTargetSnapshotWindowSize() call, then decreaseTargetSnapshotWindowSize() will
- * choose not decrease the target snapshot window size setting.
- *
- * Concurrency safe, the internal counter is atomic.
- */
-void incrementSnapshotTooOldErrorCount();
-
} // namespace SnapshotWindowUtil
} // namespace mongo
diff --git a/src/mongo/db/snapshot_window_util_test.cpp b/src/mongo/db/snapshot_window_util_test.cpp
index 3956a9f9b71..19e1b60c4a1 100644
--- a/src/mongo/db/snapshot_window_util_test.cpp
+++ b/src/mongo/db/snapshot_window_util_test.cpp
@@ -69,7 +69,9 @@ TEST_F(SnapshotWindowTest, DecreaseAndIncreaseSnapshotWindow) {
snapshotWindowParams.targetSnapshotHistoryWindowInSeconds.store(100);
// Lower the time enforced between function calls to speed up testing.
+ // Dec must match Inc b/c increaseTargetWindowSize can call into decreaseTargetWindowSize.
snapshotWindowParams.minMillisBetweenSnapshotWindowInc.store(100);
+ snapshotWindowParams.minMillisBetweenSnapshotWindowDec.store(100);
// Stabilize for the test so we know that we are testing things as expected.
snapshotWindowParams.snapshotWindowAdditiveIncreaseSeconds.store(2);
@@ -84,117 +86,78 @@ TEST_F(SnapshotWindowTest, DecreaseAndIncreaseSnapshotWindow) {
snapshotWindowParams.snapshotWindowMultiplicativeDecrease.load();
auto windowAdditiveIncrease = snapshotWindowParams.snapshotWindowAdditiveIncreaseSeconds.load();
+ auto cachePressureThreshold = snapshotWindowParams.cachePressureThreshold.load();
auto minTimeBetweenInc = snapshotWindowParams.minMillisBetweenSnapshotWindowInc.load();
/**
- * Test that trying to decrease the window size FAILS when there have been no writes to the
- * cache overflow table.
+ * Test that decreasing the size succeeds when cache pressure is ABOVE the threshold
*/
- decreaseTargetSnapshotWindowSize(_opCtx.get());
- auto snapshotWindowSecondsOne =
- snapshotWindowParams.targetSnapshotHistoryWindowInSeconds.load();
-
- ASSERT_EQ(snapshotWindowSeconds, snapshotWindowSecondsOne);
-
- /**
- * Test that trying to decrease the window size SUCCEEDS when there have been writes to the
- * cache overflow table.
- */
-
- engine->setCacheOverflowTableInsertCountForTest(1);
+ engine->setCachePressureForTest(cachePressureThreshold + 5);
decreaseTargetSnapshotWindowSize(_opCtx.get());
- auto snapshotWindowSecondsTwo =
+ auto snapshotWindowSecondsOne =
snapshotWindowParams.targetSnapshotHistoryWindowInSeconds.load();
- ASSERT_GT(snapshotWindowSecondsOne, snapshotWindowSecondsTwo);
- ASSERT_EQ(snapshotWindowSecondsTwo,
+ ASSERT_GT(snapshotWindowSeconds, snapshotWindowSecondsOne);
+ ASSERT_EQ(snapshotWindowSecondsOne,
static_cast<int>(snapshotWindowSeconds * windowMultiplicativeDecrease));
/**
- * Test that trying to decrease the window size FAILS when there have been writes to the
- * cache overflow table AND SnapshotTooOld errors have occurred.
- */
-
- engine->setCacheOverflowTableInsertCountForTest(2);
- incrementSnapshotTooOldErrorCount();
-
- decreaseTargetSnapshotWindowSize(_opCtx.get());
- auto snapshotWindowSecondsThree =
- snapshotWindowParams.targetSnapshotHistoryWindowInSeconds.load();
-
- ASSERT_EQ(snapshotWindowSecondsTwo, snapshotWindowSecondsThree);
-
- /**
- * Now test again that decreasing the size SUCCEEDS when there have been writes to the cache
- * overflow table again (without any further SnapshotTooOld errors).
+ * Test that increasing the size SUCCEEDS when the cache pressure is BELOW the threshold.
*/
- engine->setCacheOverflowTableInsertCountForTest(3);
-
- decreaseTargetSnapshotWindowSize(_opCtx.get());
- auto snapshotWindowSecondsFour =
- snapshotWindowParams.targetSnapshotHistoryWindowInSeconds.load();
-
- ASSERT_GT(snapshotWindowSecondsThree, snapshotWindowSecondsFour);
- ASSERT_EQ(snapshotWindowSecondsFour,
- static_cast<int>(snapshotWindowSecondsThree * windowMultiplicativeDecrease));
-
- /**
- * Test that increasing the size SUCCEEDS.
- */
+ engine->setCachePressureForTest(cachePressureThreshold - 5);
increaseTargetSnapshotWindowSize(_opCtx.get());
- auto snapshotWindowSecondsFive =
+ auto snapshotWindowSecondsTwo =
snapshotWindowParams.targetSnapshotHistoryWindowInSeconds.load();
- ASSERT_EQ(snapshotWindowSecondsFive, snapshotWindowSecondsFour + windowAdditiveIncrease);
+ ASSERT_EQ(snapshotWindowSecondsTwo, snapshotWindowSecondsOne + windowAdditiveIncrease);
/**
- * Test that increasing the size SUCCEEDS even when there have been writes to the cache overflow
- * table.
+ * Test that increasing the size FAILS when the cache pressure is ABOVE the threshold, and
+ * instead this causes the size to be decreased.
*/
+ engine->setCachePressureForTest(cachePressureThreshold + 5);
+
// Sleep for a time because increaseTargetSnapshotWindowSize() enforces a wait time between
// updates.
sleepmillis(2 * minTimeBetweenInc);
- engine->setCacheOverflowTableInsertCountForTest(4);
-
increaseTargetSnapshotWindowSize(_opCtx.get());
- auto snapshotWindowSecondsSix =
+ auto snapshotWindowSecondsThree =
snapshotWindowParams.targetSnapshotHistoryWindowInSeconds.load();
- ASSERT_EQ(snapshotWindowSecondsSix, snapshotWindowSecondsFive + windowAdditiveIncrease);
+ ASSERT_EQ(snapshotWindowSecondsThree,
+ static_cast<int>(snapshotWindowSecondsTwo * windowMultiplicativeDecrease));
+
+ engine->setCachePressureForTest(cachePressureThreshold - 5);
/**
* Test that the size cannot be increased above the maximum size.
*/
- // Bump up the additive increase to make this run faster.
- snapshotWindowParams.snapshotWindowAdditiveIncreaseSeconds.store(9);
- windowAdditiveIncrease = snapshotWindowParams.snapshotWindowAdditiveIncreaseSeconds.load();
-
// Integers round down, so add 1 to make sure it reaches the max.
int numIncreasesToReachMax =
- (maxTargetSnapshotWindowSeconds - snapshotWindowSecondsSix) / windowAdditiveIncrease + 1;
+ (maxTargetSnapshotWindowSeconds - snapshotWindowSecondsThree) / windowAdditiveIncrease + 1;
for (int i = 0; i < numIncreasesToReachMax; ++i) {
sleepmillis(2 * minTimeBetweenInc);
increaseTargetSnapshotWindowSize(_opCtx.get());
}
// Should be at max.
- auto snapshotWindowSecondsSeven =
+ auto snapshotWindowSecondsFour =
snapshotWindowParams.targetSnapshotHistoryWindowInSeconds.load();
- ASSERT_EQ(snapshotWindowSecondsSeven, maxTargetSnapshotWindowSeconds);
+ ASSERT_EQ(snapshotWindowSecondsFour, maxTargetSnapshotWindowSeconds);
// An attempt to increase beyond max should have no effect.
sleepmillis(2 * minTimeBetweenInc);
increaseTargetSnapshotWindowSize(_opCtx.get());
- auto snapshotWindowSecondsEight =
+ auto snapshotWindowSecondsFive =
snapshotWindowParams.targetSnapshotHistoryWindowInSeconds.load();
- ASSERT_EQ(snapshotWindowSecondsEight, maxTargetSnapshotWindowSeconds);
+ ASSERT_EQ(snapshotWindowSecondsFive, maxTargetSnapshotWindowSeconds);
}
} // namespace
diff --git a/src/mongo/db/storage/biggie/biggie_kv_engine.cpp b/src/mongo/db/storage/biggie/biggie_kv_engine.cpp
index 6c85b99f1a1..5419a88535b 100644
--- a/src/mongo/db/storage/biggie/biggie_kv_engine.cpp
+++ b/src/mongo/db/storage/biggie/biggie_kv_engine.cpp
@@ -47,7 +47,7 @@ mongo::RecoveryUnit* KVEngine::newRecoveryUnit() {
return new RecoveryUnit(this, nullptr);
}
-void KVEngine::setCacheOverflowTableInsertCountForTest(int insertCount) {
+void KVEngine::setCachePressureForTest(int pressure) {
// TODO : implement.
}
diff --git a/src/mongo/db/storage/biggie/biggie_kv_engine.h b/src/mongo/db/storage/biggie/biggie_kv_engine.h
index 393b0cc7c2b..09b2b3a7ae4 100644
--- a/src/mongo/db/storage/biggie/biggie_kv_engine.h
+++ b/src/mongo/db/storage/biggie/biggie_kv_engine.h
@@ -106,11 +106,11 @@ public:
return true;
}
- virtual int64_t getCacheOverflowTableInsertCount(OperationContext* opCtx) const override {
- return 0;
+ virtual bool isCacheUnderPressure(OperationContext* opCtx) const override {
+ return false;
}
- virtual void setCacheOverflowTableInsertCountForTest(int insertCount) override;
+ virtual void setCachePressureForTest(int pressure) override;
virtual int64_t getIdentSize(OperationContext* opCtx, StringData ident) {
return 0;
diff --git a/src/mongo/db/storage/devnull/devnull_kv_engine.cpp b/src/mongo/db/storage/devnull/devnull_kv_engine.cpp
index e9437a18908..6ae745c5587 100644
--- a/src/mongo/db/storage/devnull/devnull_kv_engine.cpp
+++ b/src/mongo/db/storage/devnull/devnull_kv_engine.cpp
@@ -261,13 +261,13 @@ SortedDataInterface* DevNullKVEngine::getSortedDataInterface(OperationContext* o
return new DevNullSortedDataInterface();
}
-int64_t DevNullKVEngine::getCacheOverflowTableInsertCount(OperationContext* opCtx) const {
- return _overflowTableInsertCountForTest;
+bool DevNullKVEngine::isCacheUnderPressure(OperationContext* opCtx) const {
+ return (_cachePressureForTest >= snapshotWindowParams.cachePressureThreshold.load());
}
-void DevNullKVEngine::setCacheOverflowTableInsertCountForTest(int insertsCount) {
- invariant(insertsCount >= 0);
- _overflowTableInsertCountForTest = insertsCount;
+void DevNullKVEngine::setCachePressureForTest(int pressure) {
+ invariant(pressure >= 0 && pressure <= 100);
+ _cachePressureForTest = pressure;
}
StatusWith<std::vector<std::string>> DevNullKVEngine::beginNonBlockingBackup(
diff --git a/src/mongo/db/storage/devnull/devnull_kv_engine.h b/src/mongo/db/storage/devnull/devnull_kv_engine.h
index 4fb60668149..59a99505e3e 100644
--- a/src/mongo/db/storage/devnull/devnull_kv_engine.h
+++ b/src/mongo/db/storage/devnull/devnull_kv_engine.h
@@ -98,9 +98,9 @@ public:
return true;
}
- virtual int64_t getCacheOverflowTableInsertCount(OperationContext* opCtx) const override;
+ virtual bool isCacheUnderPressure(OperationContext* opCtx) const override;
- virtual void setCacheOverflowTableInsertCountForTest(int insertCount) override;
+ virtual void setCachePressureForTest(int pressure) override;
virtual int64_t getIdentSize(OperationContext* opCtx, StringData ident) {
return 1;
@@ -155,6 +155,6 @@ public:
private:
std::shared_ptr<void> _catalogInfo;
- int _overflowTableInsertCountForTest;
+ int _cachePressureForTest;
};
}
diff --git a/src/mongo/db/storage/kv/kv_engine.h b/src/mongo/db/storage/kv/kv_engine.h
index 2c0e56a0c33..20eb9f93b83 100644
--- a/src/mongo/db/storage/kv/kv_engine.h
+++ b/src/mongo/db/storage/kv/kv_engine.h
@@ -334,16 +334,16 @@ public:
virtual void setOldestTimestamp(Timestamp newOldestTimestamp, bool force) {}
/**
- * See 'StorageEngine::getCacheOverflowTableInsertCount'
+ * See `StorageEngine::isCacheUnderPressure()`
*/
- virtual int64_t getCacheOverflowTableInsertCount(OperationContext* opCtx) const {
- return 0;
+ virtual bool isCacheUnderPressure(OperationContext* opCtx) const {
+ return false;
}
/**
- * See 'StorageEngine::setCacheOverflowTableInsertCountForTest()'
+ * See 'StorageEngine::setCachePressureForTest()'
*/
- virtual void setCacheOverflowTableInsertCountForTest(int insertCount) {}
+ virtual void setCachePressureForTest(int pressure) {}
/**
* See `StorageEngine::supportsRecoverToStableTimestamp`
diff --git a/src/mongo/db/storage/storage_engine.h b/src/mongo/db/storage/storage_engine.h
index 0ef67b009ed..3d1fe86e120 100644
--- a/src/mongo/db/storage/storage_engine.h
+++ b/src/mongo/db/storage/storage_engine.h
@@ -453,19 +453,21 @@ public:
OldestActiveTransactionTimestampCallback callback){};
/**
- * Retrieves the number of inserts done to the cache overflow table. Writes to that table only
- * occur when the read/write cache size exceeds the allotted in-memory cache capacity and must
- * write to disk, which is slow. Tracking this value over time is indicative of cache pressure.
+ * Indicates whether the storage engine cache is under pressure.
+ *
+ * Retrieves a cache pressure value in the range [0, 100] from the storage engine, and compares
+ * it against storageGlobalParams.cachePressureThreshold, a dynamic server parameter, to
+ * determine whether cache pressure is too high.
*/
- virtual int64_t getCacheOverflowTableInsertCount(OperationContext* opCtx) const {
- return 0;
+ virtual bool isCacheUnderPressure(OperationContext* opCtx) const {
+ return false;
}
/**
- * For unit tests only. Sets the counter for the number of cache overflow table inserts that the
- * getCacheOverflowTableInsertCount() function above returns.
+ * For unit tests only. Sets the cache pressure value with which isCacheUnderPressure()
+ * evalutates to 'pressure'.
*/
- virtual void setCacheOverflowTableInsertCountForTest(int insertCount) {}
+ virtual void setCachePressureForTest(int pressure) {}
/**
* Notifies the storage engine that a replication batch has completed.
diff --git a/src/mongo/db/storage/storage_engine_impl.cpp b/src/mongo/db/storage/storage_engine_impl.cpp
index 5977640eb5a..812981c7b56 100644
--- a/src/mongo/db/storage/storage_engine_impl.cpp
+++ b/src/mongo/db/storage/storage_engine_impl.cpp
@@ -730,12 +730,12 @@ void StorageEngineImpl::setOldestActiveTransactionTimestampCallback(
_engine->setOldestActiveTransactionTimestampCallback(callback);
}
-int64_t StorageEngineImpl::getCacheOverflowTableInsertCount(OperationContext* opCtx) const {
- return _engine->getCacheOverflowTableInsertCount(opCtx);
+bool StorageEngineImpl::isCacheUnderPressure(OperationContext* opCtx) const {
+ return _engine->isCacheUnderPressure(opCtx);
}
-void StorageEngineImpl::setCacheOverflowTableInsertCountForTest(int insertCount) {
- return _engine->setCacheOverflowTableInsertCountForTest(insertCount);
+void StorageEngineImpl::setCachePressureForTest(int pressure) {
+ return _engine->setCachePressureForTest(pressure);
}
bool StorageEngineImpl::supportsRecoverToStableTimestamp() const {
diff --git a/src/mongo/db/storage/storage_engine_impl.h b/src/mongo/db/storage/storage_engine_impl.h
index accc1bfc5f3..f934035c162 100644
--- a/src/mongo/db/storage/storage_engine_impl.h
+++ b/src/mongo/db/storage/storage_engine_impl.h
@@ -125,9 +125,9 @@ public:
virtual void setOldestActiveTransactionTimestampCallback(
StorageEngine::OldestActiveTransactionTimestampCallback) override;
- virtual int64_t getCacheOverflowTableInsertCount(OperationContext* opCtx) const override;
+ virtual bool isCacheUnderPressure(OperationContext* opCtx) const override;
- virtual void setCacheOverflowTableInsertCountForTest(int insertCount) override;
+ virtual void setCachePressureForTest(int pressure) override;
virtual bool supportsRecoverToStableTimestamp() const override;
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp
index 5b12f859060..e6245a14533 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp
@@ -1933,14 +1933,14 @@ void WiredTigerKVEngine::replicationBatchIsComplete() const {
_oplogManager->triggerJournalFlush();
}
-int64_t WiredTigerKVEngine::getCacheOverflowTableInsertCount(OperationContext* opCtx) const {
+bool WiredTigerKVEngine::isCacheUnderPressure(OperationContext* opCtx) const {
WiredTigerSession* session = WiredTigerRecoveryUnit::get(opCtx)->getSessionNoTxn();
invariant(session);
- int64_t insertCount = uassertStatusOK(WiredTigerUtil::getStatisticsValueAs<int64_t>(
- session->getSession(), "statistics:", "", WT_STAT_CONN_CACHE_LOOKASIDE_INSERT));
+ int64_t score = uassertStatusOK(WiredTigerUtil::getStatisticsValueAs<int64_t>(
+ session->getSession(), "statistics:", "", WT_STAT_CONN_CACHE_LOOKASIDE_SCORE));
- return insertCount;
+ return (score >= snapshotWindowParams.cachePressureThreshold.load());
}
Timestamp WiredTigerKVEngine::getStableTimestamp() const {
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h
index 10458bd02e4..ae0fffd31a5 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h
@@ -246,7 +246,7 @@ public:
*/
void replicationBatchIsComplete() const override;
- int64_t getCacheOverflowTableInsertCount(OperationContext* opCtx) const override;
+ bool isCacheUnderPressure(OperationContext* opCtx) const override;
bool supportsReadConcernMajority() const final;
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_recovery_unit_test.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_recovery_unit_test.cpp
index 7e97f110190..e45d693b12c 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_recovery_unit_test.cpp
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_recovery_unit_test.cpp
@@ -192,11 +192,10 @@ TEST_F(WiredTigerRecoveryUnitTestFixture, CreateAndCheckForCachePressure) {
auto recordId = ress.getValue();
wu.commit();
- auto startOverflowInsertsCount = engine->getCacheOverflowTableInsertCount(opCtx);
for (int j = 0; j < 1000; ++j) {
- // We know that we have successfully created cache pressure when the in-memory cache
- // overflows to disk by writing to the cache overflow table (the WT lookaside table).
- if (engine->getCacheOverflowTableInsertCount(opCtx) > startOverflowInsertsCount) {
+ // Once we hit the cache pressure threshold, i.e. have successfully created cache pressure
+ // that is detectable, we are done.
+ if (engine->isCacheUnderPressure(opCtx)) {
invariant(j != 0);
break;
}
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp
index 9e38db35291..9d51217f1d1 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp
@@ -688,15 +688,13 @@ void WiredTigerUtil::appendSnapshotWindowSettings(WiredTigerKVEngine* engine,
const unsigned currentAvailableSnapshotWindow =
stableTimestamp.getSecs() - oldestTimestamp.getSecs();
- int64_t overflowTableInsertCount =
- uassertStatusOK(WiredTigerUtil::getStatisticsValueAs<int64_t>(
- session->getSession(), "statistics:", "", WT_STAT_CONN_CACHE_LOOKASIDE_INSERT));
- long long totalNumberOfSnapshotTooOldErrors =
- snapshotWindowParams.snapshotTooOldErrorCount.load();
+ int64_t score = uassertStatusOK(WiredTigerUtil::getStatisticsValueAs<int64_t>(
+ session->getSession(), "statistics:", "", WT_STAT_CONN_CACHE_LOOKASIDE_SCORE));
BSONObjBuilder settings(bob->subobjStart("snapshot-window-settings"));
- settings.append("total number of cache overflow disk writes", overflowTableInsertCount);
- settings.append("total number of SnapshotTooOld errors", totalNumberOfSnapshotTooOldErrors);
+ settings.append("cache pressure percentage threshold",
+ snapshotWindowParams.cachePressureThreshold.load());
+ settings.append("current cache pressure percentage", score);
settings.append("max target available snapshots window size in seconds",
snapshotWindowParams.maxTargetSnapshotHistoryWindowInSeconds.load());
settings.append("target available snapshots window size in seconds",
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_util.h b/src/mongo/db/storage/wiredtiger/wiredtiger_util.h
index 217c640cd4e..07764f2dd40 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_util.h
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_util.h
@@ -158,8 +158,8 @@ public:
* that affect that window of maintained history.
*
* "snapshot-window-settings" : {
- * "total number of cache overflow disk writes",
- * "total number of SnapshotTooOld errors",
+ * "cache pressure percentage threshold" : <num>,
+ * "current cache pressure percentage" : <num>,
* "max target available snapshots window size in seconds" : <num>,
* "target available snapshots window size in seconds" : <num>,
* "current available snapshots window size in seconds" : <num>,