SERVER-48519 Include mirrored read metrics in ftdc

(cherry picked from commit 7423d1ad84b43b2b519d7ec4551eb127ef74764d)

4 files changed, 96 insertions, 2 deletions

diff --git a/jstests/noPassthrough/ftdc_mirrored_reads.js b/jstests/noPassthrough/ftdc_mirrored_reads.js
new file mode 100644
index 00000000000..705769cc6a7
--- /dev/null
+++ b/jstests/noPassthrough/ftdc_mirrored_reads.js
@@ -0,0 +1,88 @@
+/**
+ * Verify the FTDC metrics for mirrored reads.
+ *
+ * @tags: [requires_replication]
+ */
+load('jstests/libs/ftdc.js');
+
+(function() {
+'use strict';
+
+const kDbName = "mirrored_reads_ftdc_test";
+const kCollName = "test";
+const kOperations = 100;
+
+function getMirroredReadsStats(rst) {
+    return rst.getPrimary().getDB(kDbName).serverStatus({mirroredReads: 1}).mirroredReads;
+}
+
+function getDiagnosticData(rst) {
+    let db = rst.getPrimary().getDB('admin');
+    const stats = verifyGetDiagnosticData(db).serverStatus;
+    assert(stats.hasOwnProperty('mirroredReads'));
+    return stats.mirroredReads;
+}
+
+function sendAndCheckReads(rst) {
+    let seenBeforeReads = getMirroredReadsStats(rst).seen;
+
+    jsTestLog(`Sending ${kOperations} reads to primary`);
+    for (var i = 0; i < kOperations; ++i) {
+        rst.getPrimary().getDB(kDbName).runCommand({find: kCollName, filter: {}});
+    }
+
+    jsTestLog("Verifying reads were seen by the maestro");
+    let seenAfterReads = getMirroredReadsStats(rst).seen;
+    assert.lte(seenBeforeReads + kOperations, seenAfterReads);
+}
+
+function activateFailPoint(rst) {
+    const db = rst.getPrimary().getDB(kDbName);
+    assert.commandWorked(db.adminCommand({
+        configureFailPoint: "mirrorMaestroExpectsResponse",
+        mode: "alwaysOn",
+    }));
+}
+
+const rst = new ReplSetTest({nodes: 3});
+rst.startSet();
+rst.initiateWithHighElectionTimeout();
+
+// Mirror every mirror-able command.
+assert.commandWorked(
+    rst.getPrimary().adminCommand({setParameter: 1, mirrorReads: {samplingRate: 1.0}}));
+
+jsTestLog("Verifying diagnostic collection for mirrored reads");
+{
+    let statsBeforeReads = getDiagnosticData(rst);
+    // The following metrics are not included by default.
+    assert(!statsBeforeReads.hasOwnProperty('resolved'));
+    assert(!statsBeforeReads.hasOwnProperty('resolvedBreakdown'));
+
+    let seenBeforeReads = statsBeforeReads.seen;
+    sendAndCheckReads(rst);
+    assert.soon(() => {
+        let seenAfterReads = getDiagnosticData(rst).seen;
+        jsTestLog(`Seen ${seenAfterReads} mirrored reads so far`);
+        return seenBeforeReads + kOperations <= seenAfterReads;
+    }, "Failed to update FTDC metrics within time limit", 5000);
+}
+
+jsTestLog("Verifying diagnostic collection when mirrorMaestroExpectsResponse");
+{
+    activateFailPoint(rst);
+    assert.soon(() => {
+        return getDiagnosticData(rst).hasOwnProperty('resolved');
+    }, "Failed to find 'resolved' in mirrored reads FTDC metrics within time limit", 5000);
+    let resolvedBeforeReads = getDiagnosticData(rst).resolved;
+    sendAndCheckReads(rst);
+    assert.soon(() => {
+        let resolvedAfterReads = getDiagnosticData(rst).resolved;
+        jsTestLog(`Mirrored ${resolvedAfterReads} reads so far`);
+        // There are two secondaries, so `kOperations * 2` reads must be resolved.
+        return resolvedBeforeReads + kOperations * 2 <= resolvedAfterReads;
+    }, "Failed to update extended FTDC metrics within time limit", 10000);
+}
+
+rst.stopSet();
+})();
diff --git a/src/mongo/db/ftdc/ftdc_server.cpp b/src/mongo/db/ftdc/ftdc_server.cpp
index fa215af1dce..f4fcf1e0ec2 100644
--- a/src/mongo/db/ftdc/ftdc_server.cpp
+++ b/src/mongo/db/ftdc/ftdc_server.cpp
@@ -44,6 +44,7 @@
 #include "mongo/db/ftdc/ftdc_server_gen.h"
 #include "mongo/db/ftdc/ftdc_system_stats.h"
 #include "mongo/db/jsobj.h"
+#include "mongo/db/mirror_maestro.h"
 #include "mongo/db/service_context.h"
 #include "mongo/util/synchronized_value.h"
 
@@ -207,6 +208,8 @@ public:
         // of active migrations.
         // "timing" is filtered out because it triggers frequent schema changes.
         // "defaultRWConcern" is excluded because it changes rarely and instead included in rotation
+        // "mirroredReads" is included to append the number of mirror-able operations observed and
+        // mirrored by this process in FTDC collections.
 
         BSONObjBuilder commandBuilder;
         commandBuilder.append(kCommand, 1);
@@ -214,6 +217,7 @@ public:
         commandBuilder.append("sharding", false);
         commandBuilder.append("timing", false);
         commandBuilder.append("defaultRWConcern", false);
+        commandBuilder.append(MirrorMaestro::kServerStatusSectionName, true);
 
         if (gDiagnosticDataCollectionEnableLatencyHistograms.load()) {
             BSONObjBuilder subObjBuilder(commandBuilder.subobjStart("opLatencies"));
diff --git a/src/mongo/db/mirror_maestro.cpp b/src/mongo/db/mirror_maestro.cpp
index ebaecf32b28..b54183fdc7b 100644
--- a/src/mongo/db/mirror_maestro.cpp
+++ b/src/mongo/db/mirror_maestro.cpp
@@ -68,7 +68,6 @@ constexpr auto kMirrorMaestroConnPoolMaxSize = 4ull;       // Never use more tha
 
 constexpr auto kMirroredReadsParamName = "mirrorReads"_sd;
 
-constexpr auto kMirroredReadsName = "mirroredReads"_sd;
 constexpr auto kMirroredReadsSeenKey = "seen"_sd;
 constexpr auto kMirroredReadsSentKey = "sent"_sd;
 constexpr auto kMirroredReadsResolvedKey = "resolved"_sd;
@@ -170,7 +169,8 @@ class MirroredReadsSection final : public ServerStatusSection {
 public:
     using CounterT = long long;
 
-    MirroredReadsSection() : ServerStatusSection(kMirroredReadsName.toString()) {}
+    MirroredReadsSection()
+        : ServerStatusSection(MirrorMaestro::kServerStatusSectionName.toString()) {}
 
     bool includeByDefault() const override {
         return false;
diff --git a/src/mongo/db/mirror_maestro.h b/src/mongo/db/mirror_maestro.h
index f595fd0bcc9..8d22586bfdd 100644
--- a/src/mongo/db/mirror_maestro.h
+++ b/src/mongo/db/mirror_maestro.h
@@ -71,6 +71,8 @@ public:
      * This function will noop if the MirrorMaestro is currently being initialized or shutdown.
      */
     static void tryMirrorRequest(OperationContext* opCtx) noexcept;
+
+    static constexpr auto kServerStatusSectionName = "mirroredReads"_sd;
 };
 
 }  // namespace mongo