SERVER-46566 increment hedging metrics counters

5 files changed, 197 insertions, 3 deletions

diff --git a/src/mongo/executor/SConscript b/src/mongo/executor/SConscript
index 27d49b0f0ae..003dd969957 100644
--- a/src/mongo/executor/SConscript
+++ b/src/mongo/executor/SConscript
@@ -113,6 +113,16 @@ env.Library(target='egress_tag_closer_manager',
             ])
 
 env.Library(
+    target='hedging_metrics',
+    source=[
+        'hedging_metrics.cpp',
+    ],
+    LIBDEPS=[
+        '$BUILD_DIR/mongo/base',
+    ]
+)
+
+env.Library(
     target='network_interface_tl',
     source=[
         'connection_pool_tl.cpp',
@@ -121,6 +131,7 @@ env.Library(
     LIBDEPS=[
         '$BUILD_DIR/mongo/client/async_client',
         '$BUILD_DIR/mongo/transport/transport_layer',
+        'hedging_metrics',
     ],
     LIBDEPS_PRIVATE=[
         '$BUILD_DIR/mongo/db/commands/test_commands_enabled',
diff --git a/src/mongo/executor/hedging_metrics.cpp b/src/mongo/executor/hedging_metrics.cpp
new file mode 100644
index 00000000000..9f42ebfe8aa
--- /dev/null
+++ b/src/mongo/executor/hedging_metrics.cpp
@@ -0,0 +1,80 @@
+/**
+ *    Copyright (C) 2020-present MongoDB, Inc.
+ *
+ *    This program is free software: you can redistribute it and/or modify
+ *    it under the terms of the Server Side Public License, version 1,
+ *    as published by MongoDB, Inc.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    Server Side Public License for more details.
+ *
+ *    You should have received a copy of the Server Side Public License
+ *    along with this program. If not, see
+ *    <http://www.mongodb.com/licensing/server-side-public-license>.
+ *
+ *    As a special exception, the copyright holders give permission to link the
+ *    code of portions of this program with the OpenSSL library under certain
+ *    conditions as described in each individual source file and distribute
+ *    linked combinations including the program with the OpenSSL library. You
+ *    must comply with the Server Side Public License in all respects for
+ *    all of the code used other than as permitted herein. If you modify file(s)
+ *    with this exception, you may extend this exception to your version of the
+ *    file(s), but you are not obligated to do so. If you do not wish to do so,
+ *    delete this exception statement from your version. If you delete this
+ *    exception statement from all source files in the program, then also delete
+ *    it in the license file.
+ */
+
+#include "mongo/executor/hedging_metrics.h"
+
+namespace mongo {
+
+namespace {
+const auto HedgingMetricsDecoration = ServiceContext::declareDecoration<HedgingMetrics>();
+}  // namespace
+
+HedgingMetrics* HedgingMetrics::get(ServiceContext* service) {
+    return &HedgingMetricsDecoration(service);
+}
+
+HedgingMetrics* HedgingMetrics::get(OperationContext* opCtx) {
+    return get(opCtx->getServiceContext());
+}
+
+long long HedgingMetrics::getNumTotalOperations() const {
+    return _numTotalOperations.load();
+}
+
+void HedgingMetrics::incrementNumTotalOperations() {
+    _numTotalOperations.fetchAndAdd(1);
+}
+
+long long HedgingMetrics::getNumTotalHedgedOperations() const {
+    return _numTotalHedgedOperations.load();
+}
+
+void HedgingMetrics::incrementNumTotalHedgedOperations() {
+    _numTotalHedgedOperations.fetchAndAdd(1);
+}
+
+long long HedgingMetrics::getNumAdvantageouslyHedgedOperations() const {
+    return _numAdvantageouslyHedgedOperations.load();
+}
+
+void HedgingMetrics::incrementNumAdvantageouslyHedgedOperations() {
+    _numAdvantageouslyHedgedOperations.fetchAndAdd(1);
+}
+
+BSONObj HedgingMetrics::toBSON() const {
+    BSONObjBuilder builder;
+
+    builder.append("numTotalOperations", _numTotalOperations.load());
+    builder.append("numTotalHedgedOperations", _numTotalHedgedOperations.load());
+    builder.append("numAdvantageouslyHedgedOperations", _numAdvantageouslyHedgedOperations.load());
+
+    return builder.obj();
+}
+
+}  // namespace mongo
diff --git a/src/mongo/executor/hedging_metrics.h b/src/mongo/executor/hedging_metrics.h
new file mode 100644
index 00000000000..fae226f342a
--- /dev/null
+++ b/src/mongo/executor/hedging_metrics.h
@@ -0,0 +1,73 @@
+/**
+ *    Copyright (C) 2020-present MongoDB, Inc.
+ *
+ *    This program is free software: you can redistribute it and/or modify
+ *    it under the terms of the Server Side Public License, version 1,
+ *    as published by MongoDB, Inc.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    Server Side Public License for more details.
+ *
+ *    You should have received a copy of the Server Side Public License
+ *    along with this program. If not, see
+ *    <http://www.mongodb.com/licensing/server-side-public-license>.
+ *
+ *    As a special exception, the copyright holders give permission to link the
+ *    code of portions of this program with the OpenSSL library under certain
+ *    conditions as described in each individual source file and distribute
+ *    linked combinations including the program with the OpenSSL library. You
+ *    must comply with the Server Side Public License in all respects for
+ *    all of the code used other than as permitted herein. If you modify file(s)
+ *    with this exception, you may extend this exception to your version of the
+ *    file(s), but you are not obligated to do so. If you do not wish to do so,
+ *    delete this exception statement from your version. If you delete this
+ *    exception statement from all source files in the program, then also delete
+ *    it in the license file.
+ */
+
+#pragma once
+
+#include "mongo/db/operation_context.h"
+#include "mongo/db/service_context.h"
+
+namespace mongo {
+
+/**
+ * Container for server-wide hedging metrics.
+ */
+class HedgingMetrics {
+    HedgingMetrics(const HedgingMetrics&) = delete;
+    HedgingMetrics& operator=(const HedgingMetrics&) = delete;
+
+public:
+    HedgingMetrics() = default;
+
+    static HedgingMetrics* get(ServiceContext* service);
+    static HedgingMetrics* get(OperationContext* opCtx);
+
+    long long getNumTotalOperations() const;
+    void incrementNumTotalOperations();
+
+    long long getNumTotalHedgedOperations() const;
+    void incrementNumTotalHedgedOperations();
+
+    long long getNumAdvantageouslyHedgedOperations() const;
+    void incrementNumAdvantageouslyHedgedOperations();
+
+    BSONObj toBSON() const;
+
+private:
+    // The number of all operations with readPreference options such that they could be hedged.
+    AtomicWord<long long> _numTotalOperations{0};
+
+    // The number of all operations that actually dispatched an additional rpc.
+    AtomicWord<long long> _numTotalHedgedOperations{0};
+
+    // The number of all operations where a rpc other than the first one fulfilled the client
+    // request.
+    AtomicWord<long long> _numAdvantageouslyHedgedOperations{0};
+};
+
+}  // namespace mongo
diff --git a/src/mongo/executor/network_interface_tl.cpp b/src/mongo/executor/network_interface_tl.cpp
index 506aa01f517..2e49b840ccf 100644
--- a/src/mongo/executor/network_interface_tl.cpp
+++ b/src/mongo/executor/network_interface_tl.cpp
@@ -36,6 +36,7 @@
 #include "mongo/db/commands/test_commands_enabled.h"
 #include "mongo/db/server_options.h"
 #include "mongo/executor/connection_pool_tl.h"
+#include "mongo/executor/hedging_metrics.h"
 #include "mongo/logv2/log.h"
 #include "mongo/rpc/get_status_from_command_result.h"
 #include "mongo/transport/transport_layer_manager.h"
@@ -548,7 +549,11 @@ std::shared_ptr<NetworkInterfaceTL::RequestState>
 NetworkInterfaceTL::RequestManager::getNextRequest() {
     stdx::lock_guard<Latch> lk(mutex);
     if (sentIdx.load() < requests.size()) {
-        return requests[sentIdx.fetchAndAdd(1)].lock();
+        auto req = requests[sentIdx.fetchAndAdd(1)].lock();
+        if (sentIdx.load() > 1) {
+            req->isHedge = true;
+        }
+        return req;
     } else {
         return nullptr;
     }
@@ -614,7 +619,10 @@ void NetworkInterfaceTL::RequestManager::trySend(
     auto req = getNextRequest();
     if (req) {
         RemoteCommandRequest remoteReq({cmdStatePtr->requestOnAny, idx});
-        if (sentIdx.load() > 1) {  // this is a hedged read
+        if (remoteReq.hedgeOptions) {
+            req->hasHedgeOptions = true;
+        }
+        if (req->isHedge) {  // this is a hedged read
             invariant(remoteReq.hedgeOptions);
             auto maxTimeMS = remoteReq.hedgeOptions->maxTimeMSForHedgedReads;
             if (remoteReq.timeout == remoteReq.kNoTimeout ||
@@ -637,6 +645,14 @@ void NetworkInterfaceTL::RequestManager::trySend(
                     "idx"_attr = idx);
             }
         }
+        if (cmdStatePtr->interface->_svcCtx && remoteReq.hedgeOptions) {
+            auto hm = HedgingMetrics::get(cmdStatePtr->interface->_svcCtx);
+            invariant(hm);
+            hm->incrementNumTotalOperations();
+            if (req->isHedge) {
+                hm->incrementNumTotalHedgedOperations();
+            }
+        }
         req->send(std::move(swConn), remoteReq);
     }
 }
@@ -674,6 +690,8 @@ void NetworkInterfaceTL::RequestState::resolve(Future<RemoteCommandResponse> fut
     auto& reactor = interface()->_reactor;
     auto& baton = cmdState->baton;
 
+    isSent = true;
+
     // Convert the RemoteCommandResponse to a RemoteCommandOnAnyResponse and wrap any error
     auto anyFuture =
         std::move(future)
@@ -695,6 +713,11 @@ void NetworkInterfaceTL::RequestState::resolve(Future<RemoteCommandResponse> fut
                 auto response = uassertStatusOK(swr);
                 auto status = swr.getValue().status;
                 if (cmdState->finishLine.arriveStrongly()) {
+                    if (hasHedgeOptions && isHedge) {
+                        auto hm = HedgingMetrics::get(cmdState->interface->_svcCtx);
+                        invariant(hm);
+                        hm->incrementNumAdvantageouslyHedgedOperations();
+                    }
                     cmdState->fulfillFinalPromise(std::move(response));
                 }
 
@@ -714,7 +737,11 @@ void NetworkInterfaceTL::RequestState::resolve(Future<RemoteCommandResponse> fut
                 if (!cmdState->finishLine.arriveStrongly()) {
                     return;
                 }
-
+                if (hasHedgeOptions && isHedge) {
+                    auto hm = HedgingMetrics::get(cmdState->interface->_svcCtx);
+                    invariant(hm);
+                    hm->incrementNumAdvantageouslyHedgedOperations();
+                }
                 cmdState->fulfillFinalPromise(std::move(response));
             });
     }
diff --git a/src/mongo/executor/network_interface_tl.h b/src/mongo/executor/network_interface_tl.h
index 9d599452af3..d1ea602271a 100644
--- a/src/mongo/executor/network_interface_tl.h
+++ b/src/mongo/executor/network_interface_tl.h
@@ -307,6 +307,9 @@ private:
         boost::optional<RemoteCommandRequest> request;
         HostAndPort host;
         ConnectionPool::ConnectionHandle conn;
+        bool isHedge{false};
+        bool hasHedgeOptions{false};
+        bool isSent{false};
         size_t reqId;
     };