SERVER-41561 add SnapshotTooOld error counting metric to serverStatus

(cherry picked from commit 823c44416e3aef98796ae0aecb6f574ad71f7c94)

7 files changed, 33 insertions, 4 deletions

diff --git a/src/mongo/db/service_entry_point_common.cpp b/src/mongo/db/service_entry_point_common.cpp
index 281198f62bb..90f5a89225c 100644
--- a/src/mongo/db/service_entry_point_common.cpp
+++ b/src/mongo/db/service_entry_point_common.cpp
@@ -854,14 +854,17 @@ void execCommandDatabase(OperationContext* opCtx,
         }
     } catch (const DBException& e) {
         if (e.code() == ErrorCodes::SnapshotTooOld) {
+            // SnapshotTooOld errors should never be thrown unless we are using a storage engine
+            // that supports snapshot read concern.
+            auto engine = opCtx->getServiceContext()->getStorageEngine();
+            invariant(engine && engine->supportsReadConcernSnapshot());
+
             // SnapshotTooOld errors indicate that PIT ops are failing to find an available snapshot
             // at their specified atClusterTime. Therefore, we'll try to increase the snapshot
             // history window that the storage engine maintains in order to increase the likelihood
             // of successful future PIT atClusterTime requests.
-            auto engine = opCtx->getServiceContext()->getStorageEngine();
-            if (engine && engine->supportsReadConcernSnapshot()) {
-                SnapshotWindowUtil::increaseTargetSnapshotWindowSize(opCtx);
-            }
+            SnapshotWindowUtil::incrementSnapshotTooOldErrorCount();
+            SnapshotWindowUtil::increaseTargetSnapshotWindowSize(opCtx);
         } else {
             behaviors.handleException(e, opCtx);
         }
diff --git a/src/mongo/db/snapshot_window_options.h b/src/mongo/db/snapshot_window_options.h
index 95683969ca0..0af39be1a3b 100644
--- a/src/mongo/db/snapshot_window_options.h
+++ b/src/mongo/db/snapshot_window_options.h
@@ -107,6 +107,8 @@ struct SnapshotWindowParams {
 
     static inline MutableObserverRegistry<decltype(checkCachePressurePeriodSeconds)::WordType>
         observeCheckCachePressurePeriodSeconds;
+
+    AtomicWord<std::int64_t> snapshotTooOldErrorCount{0};
 };
 
 extern SnapshotWindowParams snapshotWindowParams;
diff --git a/src/mongo/db/snapshot_window_util.cpp b/src/mongo/db/snapshot_window_util.cpp
index b11f4fd2dec..75d59c4abbd 100644
--- a/src/mongo/db/snapshot_window_util.cpp
+++ b/src/mongo/db/snapshot_window_util.cpp
@@ -159,5 +159,9 @@ void decreaseTargetSnapshotWindowSize(OperationContext* opCtx) {
     }
 }
 
+void incrementSnapshotTooOldErrorCount() {
+    snapshotWindowParams.snapshotTooOldErrorCount.addAndFetch(1);
+}
+
 }  // namespace SnapshotWindowUtil
 }  // namespace mongo
diff --git a/src/mongo/db/snapshot_window_util.h b/src/mongo/db/snapshot_window_util.h
index cd13ddd2112..dceaa11ceff 100644
--- a/src/mongo/db/snapshot_window_util.h
+++ b/src/mongo/db/snapshot_window_util.h
@@ -74,5 +74,13 @@ void increaseTargetSnapshotWindowSize(OperationContext* opCtx);
  */
 void decreaseTargetSnapshotWindowSize(OperationContext* opCtx);
 
+/**
+ * Increments a counter of SnapshotTooOld errors encountered in the command layer. Used for
+ * statistics purposes. Continuing SnapshotTooOld errors can indicate cache pressure.
+ *
+ * Concurrency safe, the internal counter is atomic.
+ */
+void incrementSnapshotTooOldErrorCount();
+
 }  // namespace SnapshotWindowUtil
 }  // namespace mongo
diff --git a/src/mongo/db/snapshot_window_util_test.cpp b/src/mongo/db/snapshot_window_util_test.cpp
index 3d7a23d8815..d12393007d1 100644
--- a/src/mongo/db/snapshot_window_util_test.cpp
+++ b/src/mongo/db/snapshot_window_util_test.cpp
@@ -162,5 +162,13 @@ TEST_F(SnapshotWindowTest, DecreaseAndIncreaseSnapshotWindow) {
     ASSERT_EQ(snapshotWindowSecondsFive, maxTargetSnapshotWindowSeconds);
 }
 
+TEST_F(SnapshotWindowTest, IncrementSnapshotTooOldErrorCount) {
+    auto beforeCount = snapshotWindowParams.snapshotTooOldErrorCount.load();
+    incrementSnapshotTooOldErrorCount();
+    incrementSnapshotTooOldErrorCount();
+    auto afterCount = snapshotWindowParams.snapshotTooOldErrorCount.load();
+    ASSERT_EQ(beforeCount + 2, afterCount);
+}
+
 }  // namespace
 }  // namespace mongo
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp
index 9d51217f1d1..829dbd1a99f 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp
@@ -691,10 +691,13 @@ void WiredTigerUtil::appendSnapshotWindowSettings(WiredTigerKVEngine* engine,
     int64_t score = uassertStatusOK(WiredTigerUtil::getStatisticsValueAs<int64_t>(
         session->getSession(), "statistics:", "", WT_STAT_CONN_CACHE_LOOKASIDE_SCORE));
 
+    auto totalNumberOfSnapshotTooOldErrors = snapshotWindowParams.snapshotTooOldErrorCount.load();
+
     BSONObjBuilder settings(bob->subobjStart("snapshot-window-settings"));
     settings.append("cache pressure percentage threshold",
                     snapshotWindowParams.cachePressureThreshold.load());
     settings.append("current cache pressure percentage", score);
+    settings.append("total number of SnapshotTooOld errors", totalNumberOfSnapshotTooOldErrors);
     settings.append("max target available snapshots window size in seconds",
                     snapshotWindowParams.maxTargetSnapshotHistoryWindowInSeconds.load());
     settings.append("target available snapshots window size in seconds",
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_util.h b/src/mongo/db/storage/wiredtiger/wiredtiger_util.h
index 07764f2dd40..ced08bc3753 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_util.h
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_util.h
@@ -160,6 +160,7 @@ public:
      * "snapshot-window-settings" : {
      *      "cache pressure percentage threshold" : <num>,
      *      "current cache pressure percentage" : <num>,
+     *      "total number of SnapshotTooOld errors" : <num>,
      *      "max target available snapshots window size in seconds" : <num>,
      *      "target available snapshots window size in seconds" : <num>,
      *      "current available snapshots window size in seconds" : <num>,