SERVER-34722 Add new server status metrics about oplog application

2 files changed, 41 insertions, 0 deletions

diff --git a/src/mongo/db/repl/bgsync.cpp b/src/mongo/db/repl/bgsync.cpp
index 8b6a0d4221c..497a0fefca9 100644
--- a/src/mongo/db/repl/bgsync.cpp
+++ b/src/mongo/db/repl/bgsync.cpp
@@ -74,6 +74,31 @@ const int kSleepToAllowBatchingMillis = 2;
 const int kSmallBatchLimitBytes = 40000;
 const Milliseconds kRollbackOplogSocketTimeout(10 * 60 * 1000);
 
+// The number of times a node attempted to choose a node to sync from among the available sync
+// source options. This occurs if we re-evaluate our sync source, receive an error from the source,
+// or step down.
+Counter64 numSyncSourceSelections;
+ServerStatusMetricField<Counter64> displayNumSyncSourceSelections("repl.syncSource.numSelections",
+                                                                  &numSyncSourceSelections);
+
+// The number of times a node kept it's original sync source after re-evaluating if its current sync
+// source was optimal.
+Counter64 numTimesChoseSameSyncSource;
+ServerStatusMetricField<Counter64> displayNumTimesChoseSameSyncSource(
+    "repl.syncSource.numTimesChoseSame", &numTimesChoseSameSyncSource);
+
+// The number of times a node chose a new sync source after re-evaluating if its current sync source
+// was optimal.
+Counter64 numTimesChoseDifferentSyncSource;
+ServerStatusMetricField<Counter64> displayNumTimesChoseDifferentSyncSource(
+    "repl.syncSource.numTimesChoseDifferent", &numTimesChoseDifferentSyncSource);
+
+// The number of times a node could not find a sync source when choosing a node to sync from among
+// the available options.
+Counter64 numTimesCouldNotFindSyncSource;
+ServerStatusMetricField<Counter64> displayNumTimesCouldNotFindSyncSource(
+    "repl.syncSource.numTimesCouldNotFind", &numTimesCouldNotFindSyncSource);
+
 /**
  * Extends DataReplicatorExternalStateImpl to be member state aware.
  */
@@ -298,6 +323,8 @@ void BackgroundSync::_produce() {
         _syncSourceResolver.reset();
     }
 
+    numSyncSourceSelections.increment(1);
+
     if (syncSourceResp.syncSourceStatus == ErrorCodes::OplogStartMissing) {
         // All (accessible) sync sources are too far ahead of us.
         if (_replCoord->getMemberState().primary()) {
@@ -351,11 +378,13 @@ void BackgroundSync::_produce() {
             log() << "Chose same sync source candidate as last time, " << source
                   << ". Sleeping for 1 second to avoid immediately choosing a new sync source for "
                      "the same reason as last time.";
+            numTimesChoseSameSyncSource.increment(1);
             sleepsecs(1);
         } else {
             log() << "Changed sync source from "
                   << (oldSource.empty() ? std::string("empty") : oldSource.toString()) << " to "
                   << source;
+            numTimesChoseDifferentSyncSource.increment(1);
         }
     } else {
         if (!syncSourceResp.isOK()) {
@@ -363,6 +392,9 @@ void BackgroundSync::_produce() {
                   << syncSourceResp.syncSourceStatus.getStatus();
         }
         // No sync source found.
+        LOG(1) << "Could not find a sync source. Sleeping for 1 second before trying again.";
+        numTimesCouldNotFindSyncSource.increment(1);
+
         sleepsecs(1);
         return;
     }
diff --git a/src/mongo/db/repl/reporter.cpp b/src/mongo/db/repl/reporter.cpp
index d659ee83965..f66eb930893 100644
--- a/src/mongo/db/repl/reporter.cpp
+++ b/src/mongo/db/repl/reporter.cpp
@@ -33,7 +33,9 @@
 
 #include "mongo/db/repl/reporter.h"
 
+#include "mongo/base/counter.h"
 #include "mongo/bson/util/bson_extract.h"
+#include "mongo/db/commands/server_status_metric.h"
 #include "mongo/db/repl/update_position_args.h"
 #include "mongo/rpc/get_status_from_command_result.h"
 #include "mongo/util/assert_util.h"
@@ -47,6 +49,11 @@ namespace {
 
 const char kConfigVersionFieldName[] = "configVersion";
 
+// The number of replSetUpdatePosition commands a node sent to its sync source.
+Counter64 numUpdatePosition;
+ServerStatusMetricField<Counter64> displayNumUpdatePosition(
+    "repl.network.replSetUpdatePosition.num", &numUpdatePosition);
+
 /**
  * Returns configuration version in update command object.
  * Returns -1 on failure.
@@ -233,6 +240,8 @@ void Reporter::_sendCommand_inlock(BSONObj commandRequest, Milliseconds netTimeo
         return;
     }
 
+    numUpdatePosition.increment(1);
+
     _remoteCommandCallbackHandle = scheduleResult.getValue();
 }