SERVER-65289 Improvements to Mirrored Reads Metrics and Visibility

15 files changed, 98 insertions, 6 deletions

diff --git a/src/mongo/db/commands.h b/src/mongo/db/commands.h
index d54d86e7c50..41d93c75a4c 100644
--- a/src/mongo/db/commands.h
+++ b/src/mongo/db/commands.h
@@ -757,6 +757,20 @@ public:
     }
 
     /**
+     * Returns if this invocation is a mirrored read.
+     */
+    bool isMirrored() const {
+        return _mirrored;
+    }
+
+    /**
+     * Sets that this operation is a mirrored read.
+     */
+    void markMirrored() {
+        _mirrored = true;
+    }
+
+    /**
      * Returns true if command allows afterClusterTime in its readConcern. The command may not allow
      * it if it is specifically intended not to take any LockManager locks. Waiting for
      * afterClusterTime takes the MODE_IS lock.
@@ -814,6 +828,8 @@ private:
     virtual void doCheckAuthorization(OperationContext* opCtx) const = 0;
 
     const Command* const _definition;
+
+    bool _mirrored = false;
 };
 
 /**
diff --git a/src/mongo/db/commands/count_cmd.cpp b/src/mongo/db/commands/count_cmd.cpp
index f8bb2ddbeb9..4c653cbe98e 100644
--- a/src/mongo/db/commands/count_cmd.cpp
+++ b/src/mongo/db/commands/count_cmd.cpp
@@ -242,6 +242,10 @@ public:
         if (shouldDoFLERewrite(request)) {
             processFLECountD(opCtx, nss, &request);
         }
+        if (request.getMirrored().value_or(false)) {
+            const auto& invocation = CommandInvocation::get(opCtx);
+            invocation->markMirrored();
+        }
 
         if (ctx->getView()) {
             auto viewAggregation = countCommandAsAggregationCommand(request, nss);
diff --git a/src/mongo/db/commands/distinct.cpp b/src/mongo/db/commands/distinct.cpp
index 7b298885cde..f1effaf7b91 100644
--- a/src/mongo/db/commands/distinct.cpp
+++ b/src/mongo/db/commands/distinct.cpp
@@ -231,6 +231,11 @@ public:
         auto parsedDistinct = uassertStatusOK(
             ParsedDistinct::parse(opCtx, nss, cmdObj, extensionsCallback, false, defaultCollation));
 
+        if (parsedDistinct.isMirrored()) {
+            const auto& invocation = CommandInvocation::get(opCtx);
+            invocation->markMirrored();
+        }
+
         if (ctx->getView()) {
             // Relinquish locks. The aggregation command will re-acquire them.
             ctx.reset();
diff --git a/src/mongo/db/commands/find_and_modify.cpp b/src/mongo/db/commands/find_and_modify.cpp
index c5bffda7673..2751fa9b8dd 100644
--- a/src/mongo/db/commands/find_and_modify.cpp
+++ b/src/mongo/db/commands/find_and_modify.cpp
@@ -638,6 +638,11 @@ write_ops::FindAndModifyCommandReply CmdFindAndModify::Invocation::typedRun(
         return processFLEFindAndModify(opCtx, req);
     }
 
+    if (req.getMirrored().value_or(false)) {
+        const auto& invocation = CommandInvocation::get(opCtx);
+        invocation->markMirrored();
+    }
+
     const NamespaceString& nsString = req.getNamespace();
     uassertStatusOK(userAllowedWriteNS(opCtx, nsString));
     auto const curOp = CurOp::get(opCtx);
diff --git a/src/mongo/db/commands/find_cmd.cpp b/src/mongo/db/commands/find_cmd.cpp
index eda3c32b291..f2be7b56ad6 100644
--- a/src/mongo/db/commands/find_cmd.cpp
+++ b/src/mongo/db/commands/find_cmd.cpp
@@ -92,6 +92,11 @@ std::unique_ptr<FindCommandRequest> parseCmdObjectToFindCommandRequest(Operation
         processFLEFindD(opCtx, findCommand->getNamespaceOrUUID().nss().get(), findCommand.get());
     }
 
+    if (findCommand->getMirrored().value_or(false)) {
+        const auto& invocation = CommandInvocation::get(opCtx);
+        invocation->markMirrored();
+    }
+
     return findCommand;
 }
 
diff --git a/src/mongo/db/curop.cpp b/src/mongo/db/curop.cpp
index 9fce4d9d7f9..6fc209bd270 100644
--- a/src/mongo/db/curop.cpp
+++ b/src/mongo/db/curop.cpp
@@ -939,6 +939,12 @@ void OpDebug::report(OperationContext* opCtx,
         pAttrs->add("protocol", getProtoString(networkOp));
     }
 
+    if (const auto& invocation = CommandInvocation::get(opCtx);
+        invocation && invocation->isMirrored()) {
+        const bool mirrored = true;
+        OPDEBUG_TOATTR_HELP_BOOL(mirrored);
+    }
+
     if (remoteOpWaitTime) {
         pAttrs->add("remoteOpWaitMillis", durationCount<Milliseconds>(*remoteOpWaitTime));
     }
diff --git a/src/mongo/db/mirror_maestro.cpp b/src/mongo/db/mirror_maestro.cpp
index c2d0fe9e67b..f1b2358c97d 100644
--- a/src/mongo/db/mirror_maestro.cpp
+++ b/src/mongo/db/mirror_maestro.cpp
@@ -72,6 +72,7 @@ constexpr auto kMirroredReadsParamName = "mirrorReads"_sd;
 
 constexpr auto kMirroredReadsSeenKey = "seen"_sd;
 constexpr auto kMirroredReadsSentKey = "sent"_sd;
+constexpr auto kMirroredReadsReceivedKey = "received"_sd;
 constexpr auto kMirroredReadsResolvedKey = "resolved"_sd;
 constexpr auto kMirroredReadsResolvedBreakdownKey = "resolvedBreakdown"_sd;
 constexpr auto kMirroredReadsPendingKey = "pending"_sd;
@@ -82,17 +83,17 @@ MONGO_FAIL_POINT_DEFINE(mirrorMaestroTracksPending);
 class MirrorMaestroImpl {
 public:
     /**
-     * Make the TaskExecutor and initialize other components
+     * Make the TaskExecutor and initialize other components.
      */
     void init(ServiceContext* serviceContext) noexcept;
 
     /**
-     * Shutdown the TaskExecutor and cancel any outstanding work
+     * Shutdown the TaskExecutor and cancel any outstanding work.
      */
     void shutdown() noexcept;
 
     /**
-     * Mirror only if this maestro has been initialized
+     * Mirror only if this maestro has been initialized.
      */
     void tryMirror(std::shared_ptr<CommandInvocation> invocation) noexcept;
 
@@ -184,6 +185,7 @@ public:
         BSONObjBuilder section;
         section.append(kMirroredReadsSeenKey, seen.loadRelaxed());
         section.append(kMirroredReadsSentKey, sent.loadRelaxed());
+        section.append(kMirroredReadsReceivedKey, received.loadRelaxed());
 
         if (MONGO_unlikely(mirrorMaestroExpectsResponse.shouldFail())) {
             // We only can see if the command resolved if we got a response
@@ -235,6 +237,8 @@ public:
     AtomicWord<CounterT> resolved;
     // Counts the number of operations that are scheduled to be mirrored, but haven't yet been sent.
     AtomicWord<CounterT> pending;
+    // Counts the number of mirrored operations received by this node as a secondary.
+    AtomicWord<CounterT> received;
 } gMirroredReadsSection;
 
 auto parseMirroredReadsParameters(const BSONObj& obj) {
@@ -296,6 +300,13 @@ void MirrorMaestro::tryMirrorRequest(OperationContext* opCtx) noexcept {
     impl.tryMirror(std::move(invocation));
 }
 
+void MirrorMaestro::onReceiveMirroredRead(OperationContext* opCtx) noexcept {
+    const auto& invocation = CommandInvocation::get(opCtx);
+    if (MONGO_unlikely(invocation->isMirrored())) {
+        gMirroredReadsSection.received.fetchAndAddRelaxed(1);
+    }
+}
+
 void MirrorMaestroImpl::tryMirror(std::shared_ptr<CommandInvocation> invocation) noexcept {
     if (!_isInitialized.load()) {
         // If we're not even available, nothing to do
@@ -372,6 +383,9 @@ void MirrorMaestroImpl::_mirror(const std::vector<HostAndPort>& hosts,
         // Limit the maxTimeMS
         bob.append("maxTimeMS", params.getMaxTimeMS());
 
+        // Indicate that this is a mirrored read.
+        bob.append("mirrored", true);
+
         {
             // Set secondaryPreferred read preference
             BSONObjBuilder rpBob = bob.subobjStart("$readPreference");
diff --git a/src/mongo/db/mirror_maestro.h b/src/mongo/db/mirror_maestro.h
index 8d22586bfdd..df6c9c8ce44 100644
--- a/src/mongo/db/mirror_maestro.h
+++ b/src/mongo/db/mirror_maestro.h
@@ -72,6 +72,11 @@ public:
      */
     static void tryMirrorRequest(OperationContext* opCtx) noexcept;
 
+    /**
+     * Runs custom logic as part of receiving a mirrored operation.
+     */
+    static void onReceiveMirroredRead(OperationContext* opCtx) noexcept;
+
     static constexpr auto kServerStatusSectionName = "mirroredReads"_sd;
 };
 
diff --git a/src/mongo/db/mongod_main.cpp b/src/mongo/db/mongod_main.cpp
index 531b875918c..954761086b3 100644
--- a/src/mongo/db/mongod_main.cpp
+++ b/src/mongo/db/mongod_main.cpp
@@ -315,6 +315,7 @@ void initializeCommandHooks(ServiceContext* serviceContext) {
 
         void onAfterRun(OperationContext* opCtx, const OpMsgRequest&, CommandInvocation*) {
             MirrorMaestro::tryMirrorRequest(opCtx);
+            MirrorMaestro::onReceiveMirroredRead(opCtx);
         }
     };
 
diff --git a/src/mongo/db/ops/write_ops.idl b/src/mongo/db/ops/write_ops.idl
index 4a5dda8aaf3..b8d53411527 100644
--- a/src/mongo/db/ops/write_ops.idl
+++ b/src/mongo/db/ops/write_ops.idl
@@ -382,6 +382,10 @@ commands:
                 type: LegacyRuntimeConstants
                 optional: true
                 unstable: false
+            mirrored:
+                description: "Indicates whether the operation is a mirrored read"
+                type: optionalBool
+                unstable: true
 
     delete:
         description: "Parser for the 'delete' command."
@@ -531,3 +535,7 @@ commands:
                 type: EncryptionInformation
                 optional: true
                 unstable: true
+            mirrored:
+                description: "Indicates whether the operation is a mirrored read"
+                type: optionalBool
+                unstable: true
diff --git a/src/mongo/db/query/count_command.idl b/src/mongo/db/query/count_command.idl
index e27659b06eb..93eb9f7a84c 100644
--- a/src/mongo/db/query/count_command.idl
+++ b/src/mongo/db/query/count_command.idl
@@ -126,3 +126,7 @@ commands:
                 type: EncryptionInformation
                 optional: true
                 unstable: true
+            mirrored:
+                description: "Indicates whether the operation is a mirrored read"
+                type: optionalBool
+                unstable: true
diff --git a/src/mongo/db/query/distinct_command.idl b/src/mongo/db/query/distinct_command.idl
index 668ff5b48d3..8bfde758758 100644
--- a/src/mongo/db/query/distinct_command.idl
+++ b/src/mongo/db/query/distinct_command.idl
@@ -53,3 +53,6 @@ commands:
                 description: "Optional collation for the command."
                 type: object
                 optional: true
+            mirrored:
+                description: "Indicates whether the operation is a mirrored read"
+                type: optionalBool
diff --git a/src/mongo/db/query/find_command.idl b/src/mongo/db/query/find_command.idl
index 33925d3f8d1..972a25f220c 100644
--- a/src/mongo/db/query/find_command.idl
+++ b/src/mongo/db/query/find_command.idl
@@ -244,3 +244,8 @@ commands:
         type: EncryptionInformation
         optional: true
         unstable: true
+      mirrored:
+        description: "Indicates whether the operation is a mirrored read"
+        type: optionalBool
+        unstable: true
+
diff --git a/src/mongo/db/query/parsed_distinct.cpp b/src/mongo/db/query/parsed_distinct.cpp
index d98f92e6728..8f4afd0408d 100644
--- a/src/mongo/db/query/parsed_distinct.cpp
+++ b/src/mongo/db/query/parsed_distinct.cpp
@@ -320,7 +320,9 @@ StatusWith<ParsedDistinct> ParsedDistinct::parse(OperationContext* opCtx,
         cq.getValue()->setCollator(defaultCollator->clone());
     }
 
-    return ParsedDistinct(std::move(cq.getValue()), parsedDistinct.getKey().toString());
+    return ParsedDistinct(std::move(cq.getValue()),
+                          parsedDistinct.getKey().toString(),
+                          parsedDistinct.getMirrored().value_or(false));
 }
 
 }  // namespace mongo
diff --git a/src/mongo/db/query/parsed_distinct.h b/src/mongo/db/query/parsed_distinct.h
index 353ce171884..b5cabef4680 100644
--- a/src/mongo/db/query/parsed_distinct.h
+++ b/src/mongo/db/query/parsed_distinct.h
@@ -53,8 +53,10 @@ public:
     static const char kCommentField[];
     static const char kUnwoundArrayFieldForViewUnwind[];
 
-    ParsedDistinct(std::unique_ptr<CanonicalQuery> query, const std::string key)
-        : _query(std::move(query)), _key(std::move(key)) {}
+    ParsedDistinct(std::unique_ptr<CanonicalQuery> query,
+                   const std::string key,
+                   const bool mirrored = false)
+        : _query(std::move(query)), _key(std::move(key)), _mirrored(std::move(mirrored)) {}
 
     const CanonicalQuery* getQuery() const {
         return _query.get();
@@ -72,6 +74,10 @@ public:
         return _key;
     }
 
+    bool isMirrored() const {
+        return _mirrored;
+    }
+
     /**
      * Convert this ParsedDistinct into an aggregation command object.
      */
@@ -93,6 +99,9 @@ private:
 
     // The field for which we are getting distinct values.
     const std::string _key;
+
+    // Indicates that this was a mirrored operation.
+    bool _mirrored = false;
 };
 
 }  // namespace mongo